caio/clang-p2996
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[libc++][format] Switches to Unicode 15.1. (#86543)

Browse Source 
In addition to changes in the tables the extended grapheme clustering
algorithm has been overhauled. Before I considered a separate state
machine to implement the rules. With the new rule GB9c this became more
attractive and the design has changed.

This change initially had quite an impact on the performance. By making
the state machine persistent the performance was improved greatly. Note
it is still slower than before due to the larger Unicode tables.

Before
--------------------------------------------------------------------
Benchmark                          Time             CPU   Iterations
--------------------------------------------------------------------
BM_ascii_text<char>             1891 ns         1889 ns       369504
BM_unicode_text<char>         106642 ns       106397 ns         6576
BM_cyrillic_text<char>         73420 ns        73277 ns         9445
BM_japanese_text<char>         62485 ns        62387 ns        11153
BM_emoji_text<char>             1895 ns         1893 ns       369525
BM_ascii_text<wchar_t>          2015 ns         2013 ns       346887
BM_unicode_text<wchar_t>       92119 ns        92017 ns         7598
BM_cyrillic_text<wchar_t>      62637 ns        62568 ns        11117
BM_japanese_text<wchar_t>      53850 ns        53785 ns        12803
BM_emoji_text<wchar_t>          2016 ns         2014 ns       347325

After
--------------------------------------------------------------------
Benchmark                          Time             CPU   Iterations
--------------------------------------------------------------------
BM_ascii_text<char>             1906 ns         1904 ns       369409
BM_unicode_text<char>         265462 ns       265175 ns         2628
BM_cyrillic_text<char>        181063 ns       180865 ns         3871
BM_japanese_text<char>        130927 ns       130789 ns         5324
BM_emoji_text<char>             1892 ns         1890 ns       370537
BM_ascii_text<wchar_t>          2038 ns         2035 ns       343689
BM_unicode_text<wchar_t>      277603 ns       277282 ns         2526
BM_cyrillic_text<wchar_t>     188558 ns       188339 ns         3727
BM_japanese_text<wchar_t>     133084 ns       132943 ns         5262
BM_emoji_text<wchar_t>          2012 ns         2010 ns       348015

Persistent
--------------------------------------------------------------------
Benchmark                          Time             CPU   Iterations
--------------------------------------------------------------------
BM_ascii_text<char>             1904 ns         1899 ns       367472
BM_unicode_text<char>         133609 ns       133287 ns         5246
BM_cyrillic_text<char>         90185 ns        89941 ns         7796
BM_japanese_text<char>         75137 ns        74946 ns         9316
BM_emoji_text<char>             1906 ns         1901 ns       368081
BM_ascii_text<wchar_t>          2703 ns         2696 ns       259153
BM_unicode_text<wchar_t>      131497 ns       131168 ns         5341
BM_cyrillic_text<wchar_t>      87071 ns        86840 ns         8076
BM_japanese_text<wchar_t>      72279 ns        72099 ns         9682
BM_emoji_text<wchar_t>          2021 ns         2016 ns       346767
This commit is contained in:
Mark de Wever
2024-04-09 19:20:06 +02:00
committed by GitHub
parent cf6feff56b
commit 59e66c515a
19 changed files with 6429 additions and 3025 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
libcxx/docs/ReleaseNotes/19.rst
View File

@@ -56,6 +56,7 @@ Improvements and New Features
resulting in a performance increase of up to 1400x.
- The ``std::mismatch`` algorithm has been optimized for integral types, which can lead up to 40x performance
improvements.
- The ``std::ranges::minmax`` algorithm has been optimized for integral types, resulting in a performance increase of
up to 100x.
@@ -64,6 +65,8 @@ Improvements and New Features
- The ``_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT`` macro has been added to make the declarations in ``<locale>``
available.
- The formatting library is updated to Unicode 15.1.0.
Deprecations and Removals
-------------------------

1
libcxx/include/CMakeLists.txt
View File

@@ -395,6 +395,7 @@ set(files
__format/formatter_pointer.h
__format/formatter_string.h
__format/formatter_tuple.h
__format/indic_conjunct_break_table.h
__format/parser_std_format_spec.h
__format/range_default_formatter.h
__format/range_formatter.h

11
libcxx/include/__format/escaped_output_table.h
View File

@@ -110,7 +110,7 @@ namespace __escaped_output_table {
/// - bits [0, 10] The size of the range, allowing 2048 elements.
/// - bits [11, 31] The lower bound code point of the range. The upper bound of
/// the range is lower bound + size.
_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[893] = {
_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[894] = {
0x00000020,
0x0003f821,
0x00056800,
@@ -464,14 +464,14 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[893] = {
0x0174d000,
0x0177a00b,
0x017eb019,
0x017fe004,
0x01800000,
0x01815005,
0x01820000,
0x0184b803,
0x01880004,
0x01898000,
0x018c7800,
0x018f200b,
0x018f200a,
0x0190f800,
0x05246802,
0x05263808,
@@ -1000,8 +1000,9 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[893] = {
0x15b9d005,
0x15c0f001,
0x1675100d,
0x175f0fff,
0x179f0c1e,
0x175f080e,
0x1772f7ff,
0x17b2f1a1,
0x17d0f5e1,
0x189a5804};

350
libcxx/include/__format/indic_conjunct_break_table.h Normal file
View File

@@ -0,0 +1,350 @@
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
// WARNING, this entire header is generated by
// utils/generate_indic_conjunct_break_table.py
// DO NOT MODIFY!
// UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE
//
// See Terms of Use <https://www.unicode.org/copyright.html>
// for definitions of Unicode Inc.'s Data Files and Software.
//
// NOTICE TO USER: Carefully read the following legal agreement.
// BY DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S
// DATA FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"),
// YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE
// TERMS AND CONDITIONS OF THIS AGREEMENT.
// IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE
// THE DATA FILES OR SOFTWARE.
//
// COPYRIGHT AND PERMISSION NOTICE
//
// Copyright (c) 1991-2022 Unicode, Inc. All rights reserved.
// Distributed under the Terms of Use in https://www.unicode.org/copyright.html.
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of the Unicode data files and any associated documentation
// (the "Data Files") or Unicode software and any associated documentation
// (the "Software") to deal in the Data Files or Software
// without restriction, including without limitation the rights to use,
// copy, modify, merge, publish, distribute, and/or sell copies of
// the Data Files or Software, and to permit persons to whom the Data Files
// or Software are furnished to do so, provided that either
// (a) this copyright and permission notice appear with all copies
// of the Data Files or Software, or
// (b) this copyright and permission notice appear in associated
// Documentation.
//
// THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF
// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT OF THIRD PARTY RIGHTS.
// IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS
// NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
// DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
// DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THE DATA FILES OR SOFTWARE.
//
// Except as contained in this notice, the name of a copyright holder
// shall not be used in advertising or otherwise to promote the sale,
// use or other dealings in these Data Files or Software without prior
// written authorization of the copyright holder.
#ifndef _LIBCPP___FORMAT_INDIC_CONJUNCT_BREAK_TABLE_H
#define _LIBCPP___FORMAT_INDIC_CONJUNCT_BREAK_TABLE_H
#include <__algorithm/ranges_upper_bound.h>
#include <__config>
#include <__iterator/access.h>
#include <cstddef>
#include <cstdint>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
#endif
_LIBCPP_BEGIN_NAMESPACE_STD
#if _LIBCPP_STD_VER >= 20
namespace __indic_conjunct_break {
enum class __property : uint8_t {
// Values generated from the data files.
__Consonant,
__Extend,
__Linker,
// The code unit has none of above properties.
__none
};
/// The entries of the indic conjunct break property table.
///
/// The data is generated from
/// - https://www.unicode.org/Public/UCD/latest/ucd/DerivedCoreProperties.txt
///
/// The data has 3 values
/// - bits [0, 1] The property. One of the values generated from the datafiles
/// of \ref __property
/// - bits [2, 10] The size of the range.
/// - bits [11, 31] The lower bound code point of the range. The upper bound of
/// the range is lower bound + size.
///
/// The 9 bits for the size allow a maximum range of 512 elements. Some ranges
/// in the Unicode tables are larger. They are stored in multiple consecutive
/// ranges in the data table. An alternative would be to store the sizes in a
/// separate 16-bit value. The original MSVC STL code had such an approach, but
/// this approach uses less space for the data and is about 4% faster in the
/// following benchmark.
/// libcxx/benchmarks/std_format_spec_string_unicode.bench.cpp
// clang-format off
_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[201] = {
0x00180139,
0x001a807d,
0x00241811,
0x002c88b1,
0x002df801,
0x002e0805,
0x002e2005,
0x002e3801,
0x00308029,
0x00325851,
0x00338001,
0x0036b019,
0x0036f815,
0x00373805,
0x0037500d,
0x00388801,
0x00398069,
0x003f5821,
0x003fe801,
0x0040b00d,
0x0040d821,
0x00412809,
0x00414811,
0x0042c809,
0x0044c01d,
0x0046505d,
0x00471871,
0x0048a890,
0x0049e001,
0x004a6802,
0x004a880d,
0x004ac01c,
0x004bc01c,
0x004ca84c,
0x004d5018,
0x004d9000,
0x004db00c,
0x004de001,
0x004e6802,
0x004ee004,
0x004ef800,
0x004f8004,
0x004ff001,
0x0051e001,
0x0054a84c,
0x00555018,
0x00559004,
0x0055a810,
0x0055e001,
0x00566802,
0x0057c800,
0x0058a84c,
0x00595018,
0x00599004,
0x0059a810,
0x0059e001,
0x005a6802,
0x005ae004,
0x005af800,
0x005b8800,
0x0060a84c,
0x0061503c,
0x0061e001,
0x00626802,
0x0062a805,
0x0062c008,
0x0065e001,
0x0068a894,
0x0069d805,
0x006a6802,
0x0071c009,
0x0072400d,
0x0075c009,
0x0076400d,
0x0078c005,
0x0079a801,
0x0079b801,
0x0079c801,
0x007b8805,
0x007ba001,
0x007bd00d,
0x007c0001,
0x007c1009,
0x007c3005,
0x007e3001,
0x0081b801,
0x0081c805,
0x00846801,
0x009ae809,
0x00b8a001,
0x00be9001,
0x00bee801,
0x00c54801,
0x00c9c809,
0x00d0b805,
0x00d30001,
0x00d3a81d,
0x00d3f801,
0x00d58035,
0x00d5f83d,
0x00d9a001,
0x00db5821,
0x00dd5801,
0x00df3001,
0x00e1b801,
0x00e68009,
0x00e6a031,
0x00e71019,
0x00e76801,
0x00e7a001,
0x00e7c005,
0x00ee00fd,
0x01006801,
0x01068031,
0x01070801,
0x0107282d,
0x01677809,
0x016bf801,
0x016f007d,
0x01815015,
0x0184c805,
0x05337801,
0x0533a025,
0x0534f005,
0x05378005,
0x05416001,
0x05470045,
0x05495809,
0x054d9801,
0x05558001,
0x05559009,
0x0555b805,
0x0555f005,
0x05560801,
0x0557b001,
0x055f6801,
0x07d8f001,
0x07f1003d,
0x080fe801,
0x08170001,
0x081bb011,
0x08506801,
0x08507801,
0x0851c009,
0x0851f801,
0x08572805,
0x0869200d,
0x08755805,
0x0877e809,
0x087a3029,
0x087c100d,
0x08838001,
0x0883f801,
0x0885d001,
0x08880009,
0x08899805,
0x088b9801,
0x088e5001,
0x0891b001,
0x08974805,
0x0899d805,
0x089b3019,
0x089b8011,
0x08a23001,
0x08a2f001,
0x08a61801,
0x08ae0001,
0x08b5b801,
0x08b95801,
0x08c1d001,
0x08c9f001,
0x08ca1801,
0x08d1a001,
0x08d23801,
0x08d4c801,
0x08ea1001,
0x08ea2005,
0x08ecb801,
0x08fa1001,
0x0b578011,
0x0b598019,
0x0de4f001,
0x0e8b2801,
0x0e8b3809,
0x0e8b7011,
0x0e8bd81d,
0x0e8c2819,
0x0e8d500d,
0x0e921009,
0x0f000019,
0x0f004041,
0x0f00d819,
0x0f011805,
0x0f013011,
0x0f047801,
0x0f098019,
0x0f157001,
0x0f17600d,
0x0f27600d,
0x0f468019,
0x0f4a2019};
// clang-format on
/// Returns the indic conjuct break property of a code point.
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __property __get_property(const char32_t __code_point) noexcept {
// The algorithm searches for the upper bound of the range and, when found,
// steps back one entry. This algorithm is used since the code point can be
// anywhere in the range. After a lower bound is found the next step is to
// compare whether the code unit is indeed in the range.
//
// Since the entry contains a code unit, size, and property the code point
// being sought needs to be adjusted. Just shifting the code point to the
// proper position doesn't work; suppose an entry has property 0, size 1,
// and lower bound 3. This results in the entry 0x1810.
// When searching for code point 3 it will search for 0x1800, find 0x1810
// and moves to the previous entry. Thus the lower bound value will never
// be found.
// The simple solution is to set the bits belonging to the property and
// size. Then the upper bound for code point 3 will return the entry after
// 0x1810. After moving to the previous entry the algorithm arrives at the
// correct entry.
ptrdiff_t __i = std::ranges::upper_bound(__entries, (__code_point << 11) | 0x7ffu) - __entries;
if (__i == 0)
return __property::__none;
--__i;
uint32_t __upper_bound = (__entries[__i] >> 11) + ((__entries[__i] >> 2) & 0b1'1111'1111);
if (__code_point <= __upper_bound)
return static_cast<__property>(__entries[__i] & 0b11);
return __property::__none;
}
} // namespace __indic_conjunct_break
#endif //_LIBCPP_STD_VER >= 20
_LIBCPP_END_NAMESPACE_STD
#endif // _LIBCPP___FORMAT_INDIC_CONJUNCT_BREAK_TABLE_H

347
libcxx/include/__format/unicode.h
View File

@@ -15,8 +15,10 @@
#include <__concepts/same_as.h>
#include <__config>
#include <__format/extended_grapheme_cluster_table.h>
#include <__format/indic_conjunct_break_table.h>
#include <__iterator/concepts.h>
#include <__iterator/readable_traits.h> // iter_value_t
#include <__utility/unreachable.h>
#include <string_view>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -292,84 +294,231 @@ private:
};
# endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
_LIBCPP_HIDE_FROM_ABI constexpr bool __at_extended_grapheme_cluster_break(
bool& __ri_break_allowed,
bool __has_extened_pictographic,
__extended_grapheme_custer_property_boundary::__property __prev,
__extended_grapheme_custer_property_boundary::__property __next) {
using __extended_grapheme_custer_property_boundary::__property;
// State machine to implement the Extended Grapheme Cluster Boundary
//
// The exact rules may change between Unicode versions.
// This implements the extended rules see
// https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries
class __extended_grapheme_cluster_break {
using __EGC_property = __extended_grapheme_custer_property_boundary::__property;
using __inCB_property = __indic_conjunct_break::__property;
__has_extened_pictographic |= __prev == __property::__Extended_Pictographic;
// https://www.unicode.org/reports/tr29/tr29-39.html#Grapheme_Cluster_Boundary_Rules
// *** Break at the start and end of text, unless the text is empty. ***
_LIBCPP_ASSERT_INTERNAL(__prev != __property::__sot, "should be handled in the constructor"); // GB1
_LIBCPP_ASSERT_INTERNAL(__prev != __property::__eot, "should be handled by our caller"); // GB2
// *** Do not break between a CR and LF. Otherwise, break before and after controls. ***
if (__prev == __property::__CR && __next == __property::__LF) // GB3
return false;
if (__prev == __property::__Control || __prev == __property::__CR || __prev == __property::__LF) // GB4
return true;
if (__next == __property::__Control || __next == __property::__CR || __next == __property::__LF) // GB5
return true;
// *** Do not break Hangul syllable sequences. ***
if (__prev == __property::__L && (__next == __property::__L || __next == __property::__V ||
__next == __property::__LV || __next == __property::__LVT)) // GB6
return false;
if ((__prev == __property::__LV || __prev == __property::__V) &&
(__next == __property::__V || __next == __property::__T)) // GB7
return false;
if ((__prev == __property::__LVT || __prev == __property::__T) && __next == __property::__T) // GB8
return false;
// *** Do not break before extending characters or ZWJ. ***
if (__next == __property::__Extend || __next == __property::__ZWJ)
return false; // GB9
// *** Do not break before SpacingMarks, or after Prepend characters. ***
if (__next == __property::__SpacingMark) // GB9a
return false;
if (__prev == __property::__Prepend) // GB9b
return false;
// *** Do not break within emoji modifier sequences or emoji zwj sequences. ***
// GB11 \p{Extended_Pictographic} Extend* ZWJ x \p{Extended_Pictographic}
//
// Note that several parts of this rule are matched by GB9: Any x (Extend | ZWJ)
// - \p{Extended_Pictographic} x Extend
// - Extend x Extend
// - \p{Extended_Pictographic} x ZWJ
// - Extend x ZWJ
//
// So the only case left to test is
// - \p{Extended_Pictographic}' x ZWJ x \p{Extended_Pictographic}
// where \p{Extended_Pictographic}' is stored in __has_extened_pictographic
if (__has_extened_pictographic && __prev == __property::__ZWJ && __next == __property::__Extended_Pictographic)
return false;
// *** Do not break within emoji flag sequences ***
// That is, do not break between regional indicator (RI) symbols if there
// is an odd number of RI characters before the break point.
if (__prev == __property::__Regional_Indicator && __next == __property::__Regional_Indicator) { // GB12 + GB13
__ri_break_allowed = !__ri_break_allowed;
return __ri_break_allowed;
public:
_LIBCPP_HIDE_FROM_ABI constexpr explicit __extended_grapheme_cluster_break(char32_t __first_code_point)
: __prev_code_point_(__first_code_point),
__prev_property_(__extended_grapheme_custer_property_boundary::__get_property(__first_code_point)) {
// Initializes the active rule.
if (__prev_property_ == __EGC_property::__Extended_Pictographic)
__active_rule_ = __rule::__GB11_emoji;
else if (__prev_property_ == __EGC_property::__Regional_Indicator)
__active_rule_ = __rule::__GB12_GB13_regional_indicator;
else if (__indic_conjunct_break::__get_property(__first_code_point) == __inCB_property::__Consonant)
__active_rule_ = __rule::__GB9c_indic_conjunct_break;
}
// *** Otherwise, break everywhere. ***
return true; // GB999
}
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(char32_t __next_code_point) {
__EGC_property __next_property = __extended_grapheme_custer_property_boundary::__get_property(__next_code_point);
bool __result = __evaluate(__next_code_point, __next_property);
__prev_code_point_ = __next_code_point;
__prev_property_ = __next_property;
return __result;
}
// The code point whose break propery are considered during the next
// evaluation cyle.
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr char32_t __current_code_point() const { return __prev_code_point_; }
private:
// The naming of the identifiers matches the Unicode standard.
// NOLINTBEGIN(readability-identifier-naming)
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
__evaluate(char32_t __next_code_point, __EGC_property __next_property) {
switch (__active_rule_) {
case __rule::__none:
return __evaluate_none(__next_code_point, __next_property);
case __rule::__GB9c_indic_conjunct_break:
return __evaluate_GB9c_indic_conjunct_break(__next_code_point, __next_property);
case __rule::__GB11_emoji:
return __evaluate_GB11_emoji(__next_code_point, __next_property);
case __rule::__GB12_GB13_regional_indicator:
return __evaluate_GB12_GB13_regional_indicator(__next_code_point, __next_property);
}
__libcpp_unreachable();
}
_LIBCPP_HIDE_FROM_ABI constexpr bool __evaluate_none(char32_t __next_code_point, __EGC_property __next_property) {
// *** Break at the start and end of text, unless the text is empty. ***
_LIBCPP_ASSERT_INTERNAL(__prev_property_ != __EGC_property::__sot, "should be handled in the constructor"); // GB1
_LIBCPP_ASSERT_INTERNAL(__prev_property_ != __EGC_property::__eot, "should be handled by our caller"); // GB2
// *** Do not break between a CR and LF. Otherwise, break before and after controls. ***
if (__prev_property_ == __EGC_property::__CR && __next_property == __EGC_property::__LF) // GB3
return false;
if (__prev_property_ == __EGC_property::__Control || __prev_property_ == __EGC_property::__CR ||
__prev_property_ == __EGC_property::__LF) // GB4
return true;
if (__next_property == __EGC_property::__Control || __next_property == __EGC_property::__CR ||
__next_property == __EGC_property::__LF) // GB5
return true;
// *** Do not break Hangul syllable sequences. ***
if (__prev_property_ == __EGC_property::__L &&
(__next_property == __EGC_property::__L || __next_property == __EGC_property::__V ||
__next_property == __EGC_property::__LV || __next_property == __EGC_property::__LVT)) // GB6
return false;
if ((__prev_property_ == __EGC_property::__LV || __prev_property_ == __EGC_property::__V) &&
(__next_property == __EGC_property::__V || __next_property == __EGC_property::__T)) // GB7
return false;
if ((__prev_property_ == __EGC_property::__LVT || __prev_property_ == __EGC_property::__T) &&
__next_property == __EGC_property::__T) // GB8
return false;
// *** Do not break before extending characters or ZWJ. ***
if (__next_property == __EGC_property::__Extend || __next_property == __EGC_property::__ZWJ)
return false; // GB9
// *** Do not break before SpacingMarks, or after Prepend characters. ***
if (__next_property == __EGC_property::__SpacingMark) // GB9a
return false;
if (__prev_property_ == __EGC_property::__Prepend) // GB9b
return false;
// *** Do not break within certain combinations with Indic_Conjunct_Break (InCB)=Linker. ***
if (__indic_conjunct_break::__get_property(__next_code_point) == __inCB_property::__Consonant) {
__active_rule_ = __rule::__GB9c_indic_conjunct_break;
__GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant;
return true;
}
// *** Do not break within emoji modifier sequences or emoji zwj sequences. ***
if (__next_property == __EGC_property::__Extended_Pictographic) {
__active_rule_ = __rule::__GB11_emoji;
__GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic;
return true;
}
// *** Do not break within emoji flag sequences ***
// That is, do not break between regional indicator (RI) symbols if there
// is an odd number of RI characters before the break point.
if (__next_property == __EGC_property::__Regional_Indicator) { // GB12 + GB13
__active_rule_ = __rule::__GB12_GB13_regional_indicator;
return true;
}
// *** Otherwise, break everywhere. ***
return true; // GB999
}
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
__evaluate_GB9c_indic_conjunct_break(char32_t __next_code_point, __EGC_property __next_property) {
__inCB_property __break = __indic_conjunct_break::__get_property(__next_code_point);
if (__break == __inCB_property::__none) {
__active_rule_ = __rule::__none;
return __evaluate_none(__next_code_point, __next_property);
}
switch (__GB9c_indic_conjunct_break_state_) {
case __GB9c_indic_conjunct_break_state::__Consonant:
if (__break == __inCB_property::__Extend) {
return false;
}
if (__break == __inCB_property::__Linker) {
__GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Linker;
return false;
}
__active_rule_ = __rule::__none;
return __evaluate_none(__next_code_point, __next_property);
case __GB9c_indic_conjunct_break_state::__Linker:
if (__break == __inCB_property::__Extend) {
return false;
}
if (__break == __inCB_property::__Linker) {
return false;
}
if (__break == __inCB_property::__Consonant) {
__GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant;
return false;
}
__active_rule_ = __rule::__none;
return __evaluate_none(__next_code_point, __next_property);
}
__libcpp_unreachable();
}
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
__evaluate_GB11_emoji(char32_t __next_code_point, __EGC_property __next_property) {
switch (__GB11_emoji_state_) {
case __GB11_emoji_state::__Extended_Pictographic:
if (__next_property == __EGC_property::__Extend) {
__GB11_emoji_state_ = __GB11_emoji_state::__Extend;
return false;
}
[[fallthrough]];
case __GB11_emoji_state::__Extend:
if (__next_property == __EGC_property::__ZWJ) {
__GB11_emoji_state_ = __GB11_emoji_state::__ZWJ;
return false;
}
if (__next_property == __EGC_property::__Extend)
return false;
__active_rule_ = __rule::__none;
return __evaluate_none(__next_code_point, __next_property);
case __GB11_emoji_state::__ZWJ:
if (__next_property == __EGC_property::__Extended_Pictographic) {
__GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic;
return false;
}
__active_rule_ = __rule::__none;
return __evaluate_none(__next_code_point, __next_property);
}
__libcpp_unreachable();
}
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
__evaluate_GB12_GB13_regional_indicator(char32_t __next_code_point, __EGC_property __next_property) {
__active_rule_ = __rule::__none;
if (__next_property == __EGC_property::__Regional_Indicator)
return false;
return __evaluate_none(__next_code_point, __next_property);
}
char32_t __prev_code_point_;
__EGC_property __prev_property_;
enum class __rule {
__none,
__GB9c_indic_conjunct_break,
__GB11_emoji,
__GB12_GB13_regional_indicator,
};
__rule __active_rule_ = __rule::__none;
enum class __GB11_emoji_state {
__Extended_Pictographic,
__Extend,
__ZWJ,
};
__GB11_emoji_state __GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic;
enum class __GB9c_indic_conjunct_break_state {
__Consonant,
__Linker,
};
__GB9c_indic_conjunct_break_state __GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant;
// NOLINTEND(readability-identifier-naming)
};
/// Helper class to extract an extended grapheme cluster from a Unicode character range.
///
@@ -382,9 +531,7 @@ class __extended_grapheme_cluster_view {
public:
_LIBCPP_HIDE_FROM_ABI constexpr explicit __extended_grapheme_cluster_view(_Iterator __first, _Iterator __last)
: __code_point_view_(__first, __last),
__next_code_point_(__code_point_view_.__consume().__code_point),
__next_prop_(__extended_grapheme_custer_property_boundary::__get_property(__next_code_point_)) {}
: __code_point_view_(__first, __last), __at_break_(__code_point_view_.__consume().__code_point) {}
struct __cluster {
/// The first code point of the extended grapheme cluster.
@@ -400,44 +547,20 @@ public:
_Iterator __last_;
};
_LIBCPP_HIDE_FROM_ABI constexpr __cluster __consume() {
_LIBCPP_ASSERT_INTERNAL(__next_prop_ != __extended_grapheme_custer_property_boundary::__property::__eot,
"can't move beyond the end of input");
char32_t __code_point = __next_code_point_;
if (!__code_point_view_.__at_end())
return {__code_point, __get_break()};
__next_prop_ = __extended_grapheme_custer_property_boundary::__property::__eot;
return {__code_point, __code_point_view_.__position()};
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __cluster __consume() {
char32_t __code_point = __at_break_.__current_code_point();
_Iterator __position = __code_point_view_.__position();
while (!__code_point_view_.__at_end()) {
if (__at_break_(__code_point_view_.__consume().__code_point))
break;
__position = __code_point_view_.__position();
}
return {__code_point, __position};
}
private:
__code_point_view<_CharT> __code_point_view_;
char32_t __next_code_point_;
__extended_grapheme_custer_property_boundary::__property __next_prop_;
_LIBCPP_HIDE_FROM_ABI constexpr _Iterator __get_break() {
bool __ri_break_allowed = true;
bool __has_extened_pictographic = false;
while (true) {
_Iterator __result = __code_point_view_.__position();
__extended_grapheme_custer_property_boundary::__property __prev = __next_prop_;
if (__code_point_view_.__at_end()) {
__next_prop_ = __extended_grapheme_custer_property_boundary::__property::__eot;
return __result;
}
__next_code_point_ = __code_point_view_.__consume().__code_point;
__next_prop_ = __extended_grapheme_custer_property_boundary::__get_property(__next_code_point_);
__has_extened_pictographic |=
__prev == __extended_grapheme_custer_property_boundary::__property::__Extended_Pictographic;
if (__at_extended_grapheme_cluster_break(__ri_break_allowed, __has_extened_pictographic, __prev, __next_prop_))
return __result;
}
}
__extended_grapheme_cluster_break __at_break_;
};
template <contiguous_iterator _Iterator>

7
libcxx/include/__format/width_estimation_table.h
View File

@@ -119,7 +119,7 @@ namespace __width_estimation_table {
/// - bits [0, 13] The size of the range, allowing 16384 elements.
/// - bits [14, 31] The lower bound code point of the range. The upper bound of
/// the range is lower bound + size.
_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[108] = {
_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[107] = {
0x0440005f /* 00001100 - 0000115f [ 96] */, //
0x08c68001 /* 0000231a - 0000231b [ 2] */, //
0x08ca4001 /* 00002329 - 0000232a [ 2] */, //
@@ -158,14 +158,13 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[108] = {
0x0ba00019 /* 00002e80 - 00002e99 [ 26] */, //
0x0ba6c058 /* 00002e9b - 00002ef3 [ 89] */, //
0x0bc000d5 /* 00002f00 - 00002fd5 [ 214] */, //
0x0bfc000b /* 00002ff0 - 00002ffb [ 12] */, //
0x0c00003e /* 00003000 - 0000303e [ 63] */, //
0x0bfc004e /* 00002ff0 - 0000303e [ 79] */, //
0x0c104055 /* 00003041 - 00003096 [ 86] */, //
0x0c264066 /* 00003099 - 000030ff [ 103] */, //
0x0c41402a /* 00003105 - 0000312f [ 43] */, //
0x0c4c405d /* 00003131 - 0000318e [ 94] */, //
0x0c640053 /* 00003190 - 000031e3 [ 84] */, //
0x0c7c002e /* 000031f0 - 0000321e [ 47] */, //
0x0c7bc02f /* 000031ef - 0000321e [ 48] */, //
0x0c880027 /* 00003220 - 00003247 [ 40] */, //
0x0c943fff /* 00003250 - 0000724f [16384] */, //
0x1c94323c /* 00007250 - 0000a48c [12861] */, //

1
libcxx/include/libcxx.imp
View File

@@ -389,6 +389,7 @@
{ include: [ "<__format/formatter_pointer.h>", "private", "<format>", "public" ] },
{ include: [ "<__format/formatter_string.h>", "private", "<format>", "public" ] },
{ include: [ "<__format/formatter_tuple.h>", "private", "<format>", "public" ] },
{ include: [ "<__format/indic_conjunct_break_table.h>", "private", "<format>", "public" ] },
{ include: [ "<__format/parser_std_format_spec.h>", "private", "<format>", "public" ] },
{ include: [ "<__format/range_default_formatter.h>", "private", "<format>", "public" ] },
{ include: [ "<__format/range_formatter.h>", "private", "<format>", "public" ] },

2
libcxx/include/module.modulemap
View File

@@ -1339,12 +1339,14 @@ module std_private_format_formatter_output [system] { header "__f
module std_private_format_formatter_pointer [system] { header "__format/formatter_pointer.h" }
module std_private_format_formatter_string [system] { header "__format/formatter_string.h" }
module std_private_format_formatter_tuple [system] { header "__format/formatter_tuple.h" }
module std_private_format_indic_conjunct_break_table [system] { header "__format/indic_conjunct_break_table.h" }
module std_private_format_parser_std_format_spec [system] { header "__format/parser_std_format_spec.h" }
module std_private_format_range_default_formatter [system] { header "__format/range_default_formatter.h" }
module std_private_format_range_formatter [system] { header "__format/range_formatter.h" }
module std_private_format_unicode [system] {
header "__format/unicode.h"
export std_private_format_extended_grapheme_cluster_table
export std_private_format_indic_conjunct_break_table
}
module std_private_format_width_estimation_table [system] { header "__format/width_estimation_table.h" }
module std_private_format_write_escaped [system] { header "__format/write_escaped.h" }

2163
libcxx/test/libcxx/utilities/format/format.string/format.string.std/extended_grapheme_cluster.h
View File

File diff suppressed because it is too large Load Diff

15
libcxx/test/libcxx/utilities/format/format.string/format.string.std/extended_grapheme_cluster.pass.cpp
View File

@@ -53,6 +53,21 @@ static_assert(count_entries(cluster::__property::__LVT) == 10773);
static_assert(count_entries(cluster::__property::__ZWJ) == 1);
static_assert(count_entries(cluster::__property::__Extended_Pictographic) == 3537);
namespace inCB = std::__indic_conjunct_break;
constexpr int count_entries(inCB::__property property) {
return std::transform_reduce(
std::begin(inCB::__entries), std::end(inCB::__entries), 0, std::plus{}, [property](auto entry) {
if (static_cast<inCB::__property>(entry & 0b11) != property)
return 0;
return 1 + static_cast<int>((entry >> 2) & 0b1'1111'1111);
});
}
static_assert(count_entries(inCB::__property::__Linker) == 6);
static_assert(count_entries(inCB::__property::__Consonant) == 240);
static_assert(count_entries(inCB::__property::__Extend) == 884);
} // namespace
template <class Data>

8
libcxx/utils/CMakeLists.txt
View File

@@ -48,6 +48,13 @@ add_custom_target(libcxx-generate-width-estimation-table
"${LIBCXX_SOURCE_DIR}/include/__format/width_estimation_table.h"
COMMENT "Generate the width estimation header")
add_custom_target(libcxx-indic-conjunct-break-table
COMMAND
"${Python3_EXECUTABLE}"
"${LIBCXX_SOURCE_DIR}/utils/generate_indic_conjunct_break_table.py"
"${LIBCXX_SOURCE_DIR}/include/__format/indic_conjunct_break_table.h"
COMMENT "Generate the Indic Conjunct Break header")
add_custom_target(libcxx-generate-iwyu-mapping
COMMAND
"${Python3_EXECUTABLE}"
@@ -63,5 +70,6 @@ add_custom_target(libcxx-generate-files
libcxx-generate-extended-grapheme-cluster-tests
libcxx-generate-escaped-output-table
libcxx-generate-width-estimation-table
libcxx-indic-conjunct-break-table
libcxx-generate-iwyu-mapping
COMMENT "Create all the auto-generated files in libc++ and its tests.")

277
libcxx/utils/data/unicode/DerivedCoreProperties.txt
View File

@@ -1,6 +1,6 @@
# DerivedCoreProperties-15.0.0.txt
# Date: 2022-08-05, 22:17:05 GMT
# © 2022 Unicode®, Inc.
# DerivedCoreProperties-15.1.0.txt
# Date: 2023-08-07, 15:21:24 GMT
# © 2023 Unicode®, Inc.
# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
# For terms of use, see https://www.unicode.org/terms_of_use.html
#
@@ -1397,11 +1397,12 @@ FFDA..FFDC ; Alphabetic # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANG
2B740..2B81D ; Alphabetic # Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D
2B820..2CEA1 ; Alphabetic # Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1
2CEB0..2EBE0 ; Alphabetic # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0
2EBF0..2EE5D ; Alphabetic # Lo [622] CJK UNIFIED IDEOGRAPH-2EBF0..CJK UNIFIED IDEOGRAPH-2EE5D
2F800..2FA1D ; Alphabetic # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D
30000..3134A ; Alphabetic # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A
31350..323AF ; Alphabetic # Lo [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF
# Total code points: 137765
# Total code points: 138387
# ================================================
@@ -6853,11 +6854,12 @@ FFDA..FFDC ; ID_Start # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL
2B740..2B81D ; ID_Start # Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D
2B820..2CEA1 ; ID_Start # Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1
2CEB0..2EBE0 ; ID_Start # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0
2EBF0..2EE5D ; ID_Start # Lo [622] CJK UNIFIED IDEOGRAPH-2EBF0..CJK UNIFIED IDEOGRAPH-2EE5D
2F800..2FA1D ; ID_Start # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D
30000..3134A ; ID_Start # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A
31350..323AF ; ID_Start # Lo [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF
# Total code points: 136345
# Total code points: 136967
# ================================================
@@ -7438,6 +7440,7 @@ FFDA..FFDC ; ID_Start # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL
1FE0..1FEC ; ID_Continue # L& [13] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA
1FF2..1FF4 ; ID_Continue # L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI
1FF6..1FFC ; ID_Continue # L& [7] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
200C..200D ; ID_Continue # Cf [2] ZERO WIDTH NON-JOINER..ZERO WIDTH JOINER
203F..2040 ; ID_Continue # Pc [2] UNDERTIE..CHARACTER TIE
2054 ; ID_Continue # Pc INVERTED UNDERTIE
2071 ; ID_Continue # Lm SUPERSCRIPT LATIN SMALL LETTER I
@@ -7504,6 +7507,7 @@ FFDA..FFDC ; ID_Start # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL
309D..309E ; ID_Continue # Lm [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK
309F ; ID_Continue # Lo HIRAGANA DIGRAPH YORI
30A1..30FA ; ID_Continue # Lo [90] KATAKANA LETTER SMALL A..KATAKANA LETTER VO
30FB ; ID_Continue # Po KATAKANA MIDDLE DOT
30FC..30FE ; ID_Continue # Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK
30FF ; ID_Continue # Lo KATAKANA DIGRAPH KOTO
3105..312F ; ID_Continue # Lo [43] BOPOMOFO LETTER B..BOPOMOFO LETTER NN
@@ -7683,6 +7687,7 @@ FF10..FF19 ; ID_Continue # Nd [10] FULLWIDTH DIGIT ZERO..FULLWIDTH DIGIT NIN
FF21..FF3A ; ID_Continue # L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z
FF3F ; ID_Continue # Pc FULLWIDTH LOW LINE
FF41..FF5A ; ID_Continue # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z
FF65 ; ID_Continue # Po HALFWIDTH KATAKANA MIDDLE DOT
FF66..FF6F ; ID_Continue # Lo [10] HALFWIDTH KATAKANA LETTER WO..HALFWIDTH KATAKANA LETTER SMALL TU
FF70 ; ID_Continue # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK
FF71..FF9D ; ID_Continue # Lo [45] HALFWIDTH KATAKANA LETTER A..HALFWIDTH KATAKANA LETTER N
@@ -8207,12 +8212,13 @@ FFDA..FFDC ; ID_Continue # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HAN
2B740..2B81D ; ID_Continue # Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D
2B820..2CEA1 ; ID_Continue # Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1
2CEB0..2EBE0 ; ID_Continue # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0
2EBF0..2EE5D ; ID_Continue # Lo [622] CJK UNIFIED IDEOGRAPH-2EBF0..CJK UNIFIED IDEOGRAPH-2EE5D
2F800..2FA1D ; ID_Continue # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D
30000..3134A ; ID_Continue # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A
31350..323AF ; ID_Continue # Lo [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF
E0100..E01EF ; ID_Continue # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256
# Total code points: 139482
# Total code points: 140108
# ================================================
@@ -8962,11 +8968,12 @@ FFDA..FFDC ; XID_Start # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGU
2B740..2B81D ; XID_Start # Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D
2B820..2CEA1 ; XID_Start # Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1
2CEB0..2EBE0 ; XID_Start # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0
2EBF0..2EE5D ; XID_Start # Lo [622] CJK UNIFIED IDEOGRAPH-2EBF0..CJK UNIFIED IDEOGRAPH-2EE5D
2F800..2FA1D ; XID_Start # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D
30000..3134A ; XID_Start # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A
31350..323AF ; XID_Start # Lo [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF
# Total code points: 136322
# Total code points: 136944
# ================================================
@@ -9543,6 +9550,7 @@ FFDA..FFDC ; XID_Start # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGU
1FE0..1FEC ; XID_Continue # L& [13] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA
1FF2..1FF4 ; XID_Continue # L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI
1FF6..1FFC ; XID_Continue # L& [7] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
200C..200D ; XID_Continue # Cf [2] ZERO WIDTH NON-JOINER..ZERO WIDTH JOINER
203F..2040 ; XID_Continue # Pc [2] UNDERTIE..CHARACTER TIE
2054 ; XID_Continue # Pc INVERTED UNDERTIE
2071 ; XID_Continue # Lm SUPERSCRIPT LATIN SMALL LETTER I
@@ -9608,6 +9616,7 @@ FFDA..FFDC ; XID_Start # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGU
309D..309E ; XID_Continue # Lm [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK
309F ; XID_Continue # Lo HIRAGANA DIGRAPH YORI
30A1..30FA ; XID_Continue # Lo [90] KATAKANA LETTER SMALL A..KATAKANA LETTER VO
30FB ; XID_Continue # Po KATAKANA MIDDLE DOT
30FC..30FE ; XID_Continue # Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK
30FF ; XID_Continue # Lo KATAKANA DIGRAPH KOTO
3105..312F ; XID_Continue # Lo [43] BOPOMOFO LETTER B..BOPOMOFO LETTER NN
@@ -9793,6 +9802,7 @@ FF10..FF19 ; XID_Continue # Nd [10] FULLWIDTH DIGIT ZERO..FULLWIDTH DIGIT NI
FF21..FF3A ; XID_Continue # L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z
FF3F ; XID_Continue # Pc FULLWIDTH LOW LINE
FF41..FF5A ; XID_Continue # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z
FF65 ; XID_Continue # Po HALFWIDTH KATAKANA MIDDLE DOT
FF66..FF6F ; XID_Continue # Lo [10] HALFWIDTH KATAKANA LETTER WO..HALFWIDTH KATAKANA LETTER SMALL TU
FF70 ; XID_Continue # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK
FF71..FF9D ; XID_Continue # Lo [45] HALFWIDTH KATAKANA LETTER A..HALFWIDTH KATAKANA LETTER N
@@ -10317,12 +10327,13 @@ FFDA..FFDC ; XID_Continue # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HA
2B740..2B81D ; XID_Continue # Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D
2B820..2CEA1 ; XID_Continue # Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1
2CEB0..2EBE0 ; XID_Continue # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0
2EBF0..2EE5D ; XID_Continue # Lo [622] CJK UNIFIED IDEOGRAPH-2EBF0..CJK UNIFIED IDEOGRAPH-2EE5D
2F800..2FA1D ; XID_Continue # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D
30000..3134A ; XID_Continue # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A
31350..323AF ; XID_Continue # Lo [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF
E0100..E01EF ; XID_Continue # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256
# Total code points: 139463
# Total code points: 140089
# ================================================
@@ -10335,6 +10346,15 @@ E0100..E01EF ; XID_Continue # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTO
# - FFF9..FFFB (Interlinear annotation format characters)
# - 13430..13440 (Egyptian hieroglyph format characters)
# - Prepended_Concatenation_Mark (Exceptional format characters that should be visible)
#
# There are currently no stability guarantees for DICP. However, the
# values of DICP interact with the derivation of XID_Continue
# and NFKC_CF, for which there are stability guarantees.
# Maintainers of this property should note that in the
# unlikely case that the DICP value changes for an existing character
# which is also XID_Continue=Yes, then exceptions must be put
# in place to ensure that the NFKC_CF mapping value for that
# existing character does not change.
00AD ; Default_Ignorable_Code_Point # Cf SOFT HYPHEN
034F ; Default_Ignorable_Code_Point # Mn COMBINING GRAPHEME JOINER
@@ -11602,7 +11622,7 @@ E0100..E01EF ; Grapheme_Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELE
2E80..2E99 ; Grapheme_Base # So [26] CJK RADICAL REPEAT..CJK RADICAL RAP
2E9B..2EF3 ; Grapheme_Base # So [89] CJK RADICAL CHOKE..CJK RADICAL C-SIMPLIFIED TURTLE
2F00..2FD5 ; Grapheme_Base # So [214] KANGXI RADICAL ONE..KANGXI RADICAL FLUTE
2FF0..2FFB ; Grapheme_Base # So [12] IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER OVERLAID
2FF0..2FFF ; Grapheme_Base # So [16] IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER ROTATION
3000 ; Grapheme_Base # Zs IDEOGRAPHIC SPACE
3001..3003 ; Grapheme_Base # Po [3] IDEOGRAPHIC COMMA..DITTO MARK
3004 ; Grapheme_Base # So JAPANESE INDUSTRIAL STANDARD SYMBOL
@@ -11657,6 +11677,7 @@ E0100..E01EF ; Grapheme_Extend # Mn [240] VARIATION SELECTOR-17..VARIATION SELE
3196..319F ; Grapheme_Base # So [10] IDEOGRAPHIC ANNOTATION TOP MARK..IDEOGRAPHIC ANNOTATION MAN MARK
31A0..31BF ; Grapheme_Base # Lo [32] BOPOMOFO LETTER BU..BOPOMOFO LETTER AH
31C0..31E3 ; Grapheme_Base # So [36] CJK STROKE T..CJK STROKE Q
31EF ; Grapheme_Base # So IDEOGRAPHIC DESCRIPTION CHARACTER SUBTRACTION
31F0..31FF ; Grapheme_Base # Lo [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO
3200..321E ; Grapheme_Base # So [31] PARENTHESIZED HANGUL KIYEOK..PARENTHESIZED KOREAN CHARACTER O HU
3220..3229 ; Grapheme_Base # No [10] PARENTHESIZED IDEOGRAPH ONE..PARENTHESIZED IDEOGRAPH TEN
@@ -12497,11 +12518,12 @@ FFFC..FFFD ; Grapheme_Base # So [2] OBJECT REPLACEMENT CHARACTER..REPLACEME
2B740..2B81D ; Grapheme_Base # Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D
2B820..2CEA1 ; Grapheme_Base # Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1
2CEB0..2EBE0 ; Grapheme_Base # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0
2EBF0..2EE5D ; Grapheme_Base # Lo [622] CJK UNIFIED IDEOGRAPH-2EBF0..CJK UNIFIED IDEOGRAPH-2EE5D
2F800..2FA1D ; Grapheme_Base # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D
30000..3134A ; Grapheme_Base # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A
31350..323AF ; Grapheme_Base # Lo [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF
# Total code points: 146986
# Total code points: 147613
# ================================================
@@ -12572,4 +12594,239 @@ ABED ; Grapheme_Link # Mn MEETEI MAYEK APUN IYEK
# Total code points: 65
# ================================================
# Derived Property: Indic_Conjunct_Break
# Generated from the Grapheme_Cluster_Break, Indic_Syllabic_Category,
# Canonical_Combining_Class, and Script properties as described in UAX #44:
# https://www.unicode.org/reports/tr44/.
# All code points not explicitly listed for Indic_Conjunct_Break
# have the value None.
# @missing: 0000..10FFFF; InCB; None
# ================================================
# Indic_Conjunct_Break=Linker
094D ; InCB; Linker # Mn DEVANAGARI SIGN VIRAMA
09CD ; InCB; Linker # Mn BENGALI SIGN VIRAMA
0ACD ; InCB; Linker # Mn GUJARATI SIGN VIRAMA
0B4D ; InCB; Linker # Mn ORIYA SIGN VIRAMA
0C4D ; InCB; Linker # Mn TELUGU SIGN VIRAMA
0D4D ; InCB; Linker # Mn MALAYALAM SIGN VIRAMA
# Total code points: 6
# ================================================
# Indic_Conjunct_Break=Consonant
0915..0939 ; InCB; Consonant # Lo [37] DEVANAGARI LETTER KA..DEVANAGARI LETTER HA
0958..095F ; InCB; Consonant # Lo [8] DEVANAGARI LETTER QA..DEVANAGARI LETTER YYA
0978..097F ; InCB; Consonant # Lo [8] DEVANAGARI LETTER MARWARI DDA..DEVANAGARI LETTER BBA
0995..09A8 ; InCB; Consonant # Lo [20] BENGALI LETTER KA..BENGALI LETTER NA
09AA..09B0 ; InCB; Consonant # Lo [7] BENGALI LETTER PA..BENGALI LETTER RA
09B2 ; InCB; Consonant # Lo BENGALI LETTER LA
09B6..09B9 ; InCB; Consonant # Lo [4] BENGALI LETTER SHA..BENGALI LETTER HA
09DC..09DD ; InCB; Consonant # Lo [2] BENGALI LETTER RRA..BENGALI LETTER RHA
09DF ; InCB; Consonant # Lo BENGALI LETTER YYA
09F0..09F1 ; InCB; Consonant # Lo [2] BENGALI LETTER RA WITH MIDDLE DIAGONAL..BENGALI LETTER RA WITH LOWER DIAGONAL
0A95..0AA8 ; InCB; Consonant # Lo [20] GUJARATI LETTER KA..GUJARATI LETTER NA
0AAA..0AB0 ; InCB; Consonant # Lo [7] GUJARATI LETTER PA..GUJARATI LETTER RA
0AB2..0AB3 ; InCB; Consonant # Lo [2] GUJARATI LETTER LA..GUJARATI LETTER LLA
0AB5..0AB9 ; InCB; Consonant # Lo [5] GUJARATI LETTER VA..GUJARATI LETTER HA
0AF9 ; InCB; Consonant # Lo GUJARATI LETTER ZHA
0B15..0B28 ; InCB; Consonant # Lo [20] ORIYA LETTER KA..ORIYA LETTER NA
0B2A..0B30 ; InCB; Consonant # Lo [7] ORIYA LETTER PA..ORIYA LETTER RA
0B32..0B33 ; InCB; Consonant # Lo [2] ORIYA LETTER LA..ORIYA LETTER LLA
0B35..0B39 ; InCB; Consonant # Lo [5] ORIYA LETTER VA..ORIYA LETTER HA
0B5C..0B5D ; InCB; Consonant # Lo [2] ORIYA LETTER RRA..ORIYA LETTER RHA
0B5F ; InCB; Consonant # Lo ORIYA LETTER YYA
0B71 ; InCB; Consonant # Lo ORIYA LETTER WA
0C15..0C28 ; InCB; Consonant # Lo [20] TELUGU LETTER KA..TELUGU LETTER NA
0C2A..0C39 ; InCB; Consonant # Lo [16] TELUGU LETTER PA..TELUGU LETTER HA
0C58..0C5A ; InCB; Consonant # Lo [3] TELUGU LETTER TSA..TELUGU LETTER RRRA
0D15..0D3A ; InCB; Consonant # Lo [38] MALAYALAM LETTER KA..MALAYALAM LETTER TTTA
# Total code points: 240
# ================================================
# Indic_Conjunct_Break=Extend
0300..034E ; InCB; Extend # Mn [79] COMBINING GRAVE ACCENT..COMBINING UPWARDS ARROW BELOW
0350..036F ; InCB; Extend # Mn [32] COMBINING RIGHT ARROWHEAD ABOVE..COMBINING LATIN SMALL LETTER X
0483..0487 ; InCB; Extend # Mn [5] COMBINING CYRILLIC TITLO..COMBINING CYRILLIC POKRYTIE
0591..05BD ; InCB; Extend # Mn [45] HEBREW ACCENT ETNAHTA..HEBREW POINT METEG
05BF ; InCB; Extend # Mn HEBREW POINT RAFE
05C1..05C2 ; InCB; Extend # Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT
05C4..05C5 ; InCB; Extend # Mn [2] HEBREW MARK UPPER DOT..HEBREW MARK LOWER DOT
05C7 ; InCB; Extend # Mn HEBREW POINT QAMATS QATAN
0610..061A ; InCB; Extend # Mn [11] ARABIC SIGN SALLALLAHOU ALAYHE WASSALLAM..ARABIC SMALL KASRA
064B..065F ; InCB; Extend # Mn [21] ARABIC FATHATAN..ARABIC WAVY HAMZA BELOW
0670 ; InCB; Extend # Mn ARABIC LETTER SUPERSCRIPT ALEF
06D6..06DC ; InCB; Extend # Mn [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN
06DF..06E4 ; InCB; Extend # Mn [6] ARABIC SMALL HIGH ROUNDED ZERO..ARABIC SMALL HIGH MADDA
06E7..06E8 ; InCB; Extend # Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON
06EA..06ED ; InCB; Extend # Mn [4] ARABIC EMPTY CENTRE LOW STOP..ARABIC SMALL LOW MEEM
0711 ; InCB; Extend # Mn SYRIAC LETTER SUPERSCRIPT ALAPH
0730..074A ; InCB; Extend # Mn [27] SYRIAC PTHAHA ABOVE..SYRIAC BARREKH
07EB..07F3 ; InCB; Extend # Mn [9] NKO COMBINING SHORT HIGH TONE..NKO COMBINING DOUBLE DOT ABOVE
07FD ; InCB; Extend # Mn NKO DANTAYALAN
0816..0819 ; InCB; Extend # Mn [4] SAMARITAN MARK IN..SAMARITAN MARK DAGESH
081B..0823 ; InCB; Extend # Mn [9] SAMARITAN MARK EPENTHETIC YUT..SAMARITAN VOWEL SIGN A
0825..0827 ; InCB; Extend # Mn [3] SAMARITAN VOWEL SIGN SHORT A..SAMARITAN VOWEL SIGN U
0829..082D ; InCB; Extend # Mn [5] SAMARITAN VOWEL SIGN LONG I..SAMARITAN MARK NEQUDAA
0859..085B ; InCB; Extend # Mn [3] MANDAIC AFFRICATION MARK..MANDAIC GEMINATION MARK
0898..089F ; InCB; Extend # Mn [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA
08CA..08E1 ; InCB; Extend # Mn [24] ARABIC SMALL HIGH FARSI YEH..ARABIC SMALL HIGH SIGN SAFHA
08E3..08FF ; InCB; Extend # Mn [29] ARABIC TURNED DAMMA BELOW..ARABIC MARK SIDEWAYS NOON GHUNNA
093C ; InCB; Extend # Mn DEVANAGARI SIGN NUKTA
0951..0954 ; InCB; Extend # Mn [4] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI ACUTE ACCENT
09BC ; InCB; Extend # Mn BENGALI SIGN NUKTA
09FE ; InCB; Extend # Mn BENGALI SANDHI MARK
0A3C ; InCB; Extend # Mn GURMUKHI SIGN NUKTA
0ABC ; InCB; Extend # Mn GUJARATI SIGN NUKTA
0B3C ; InCB; Extend # Mn ORIYA SIGN NUKTA
0C3C ; InCB; Extend # Mn TELUGU SIGN NUKTA
0C55..0C56 ; InCB; Extend # Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK
0CBC ; InCB; Extend # Mn KANNADA SIGN NUKTA
0D3B..0D3C ; InCB; Extend # Mn [2] MALAYALAM SIGN VERTICAL BAR VIRAMA..MALAYALAM SIGN CIRCULAR VIRAMA
0E38..0E3A ; InCB; Extend # Mn [3] THAI CHARACTER SARA U..THAI CHARACTER PHINTHU
0E48..0E4B ; InCB; Extend # Mn [4] THAI CHARACTER MAI EK..THAI CHARACTER MAI CHATTAWA
0EB8..0EBA ; InCB; Extend # Mn [3] LAO VOWEL SIGN U..LAO SIGN PALI VIRAMA
0EC8..0ECB ; InCB; Extend # Mn [4] LAO TONE MAI EK..LAO TONE MAI CATAWA
0F18..0F19 ; InCB; Extend # Mn [2] TIBETAN ASTROLOGICAL SIGN -KHYUD PA..TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS
0F35 ; InCB; Extend # Mn TIBETAN MARK NGAS BZUNG NYI ZLA
0F37 ; InCB; Extend # Mn TIBETAN MARK NGAS BZUNG SGOR RTAGS
0F39 ; InCB; Extend # Mn TIBETAN MARK TSA -PHRU
0F71..0F72 ; InCB; Extend # Mn [2] TIBETAN VOWEL SIGN AA..TIBETAN VOWEL SIGN I
0F74 ; InCB; Extend # Mn TIBETAN VOWEL SIGN U
0F7A..0F7D ; InCB; Extend # Mn [4] TIBETAN VOWEL SIGN E..TIBETAN VOWEL SIGN OO
0F80 ; InCB; Extend # Mn TIBETAN VOWEL SIGN REVERSED I
0F82..0F84 ; InCB; Extend # Mn [3] TIBETAN SIGN NYI ZLA NAA DA..TIBETAN MARK HALANTA
0F86..0F87 ; InCB; Extend # Mn [2] TIBETAN SIGN LCI RTAGS..TIBETAN SIGN YANG RTAGS
0FC6 ; InCB; Extend # Mn TIBETAN SYMBOL PADMA GDAN
1037 ; InCB; Extend # Mn MYANMAR SIGN DOT BELOW
1039..103A ; InCB; Extend # Mn [2] MYANMAR SIGN VIRAMA..MYANMAR SIGN ASAT
108D ; InCB; Extend # Mn MYANMAR SIGN SHAN COUNCIL EMPHATIC TONE
135D..135F ; InCB; Extend # Mn [3] ETHIOPIC COMBINING GEMINATION AND VOWEL LENGTH MARK..ETHIOPIC COMBINING GEMINATION MARK
1714 ; InCB; Extend # Mn TAGALOG SIGN VIRAMA
17D2 ; InCB; Extend # Mn KHMER SIGN COENG
17DD ; InCB; Extend # Mn KHMER SIGN ATTHACAN
18A9 ; InCB; Extend # Mn MONGOLIAN LETTER ALI GALI DAGALGA
1939..193B ; InCB; Extend # Mn [3] LIMBU SIGN MUKPHRENG..LIMBU SIGN SA-I
1A17..1A18 ; InCB; Extend # Mn [2] BUGINESE VOWEL SIGN I..BUGINESE VOWEL SIGN U
1A60 ; InCB; Extend # Mn TAI THAM SIGN SAKOT
1A75..1A7C ; InCB; Extend # Mn [8] TAI THAM SIGN TONE-1..TAI THAM SIGN KHUEN-LUE KARAN
1A7F ; InCB; Extend # Mn TAI THAM COMBINING CRYPTOGRAMMIC DOT
1AB0..1ABD ; InCB; Extend # Mn [14] COMBINING DOUBLED CIRCUMFLEX ACCENT..COMBINING PARENTHESES BELOW
1ABF..1ACE ; InCB; Extend # Mn [16] COMBINING LATIN SMALL LETTER W BELOW..COMBINING LATIN SMALL LETTER INSULAR T
1B34 ; InCB; Extend # Mn BALINESE SIGN REREKAN
1B6B..1B73 ; InCB; Extend # Mn [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG
1BAB ; InCB; Extend # Mn SUNDANESE SIGN VIRAMA
1BE6 ; InCB; Extend # Mn BATAK SIGN TOMPI
1C37 ; InCB; Extend # Mn LEPCHA SIGN NUKTA
1CD0..1CD2 ; InCB; Extend # Mn [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA
1CD4..1CE0 ; InCB; Extend # Mn [13] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC TONE RIGVEDIC KASHMIRI INDEPENDENT SVARITA
1CE2..1CE8 ; InCB; Extend # Mn [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL
1CED ; InCB; Extend # Mn VEDIC SIGN TIRYAK
1CF4 ; InCB; Extend # Mn VEDIC TONE CANDRA ABOVE
1CF8..1CF9 ; InCB; Extend # Mn [2] VEDIC TONE RING ABOVE..VEDIC TONE DOUBLE RING ABOVE
1DC0..1DFF ; InCB; Extend # Mn [64] COMBINING DOTTED GRAVE ACCENT..COMBINING RIGHT ARROWHEAD AND DOWN ARROWHEAD BELOW
200D ; InCB; Extend # Cf ZERO WIDTH JOINER
20D0..20DC ; InCB; Extend # Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE
20E1 ; InCB; Extend # Mn COMBINING LEFT RIGHT ARROW ABOVE
20E5..20F0 ; InCB; Extend # Mn [12] COMBINING REVERSE SOLIDUS OVERLAY..COMBINING ASTERISK ABOVE
2CEF..2CF1 ; InCB; Extend # Mn [3] COPTIC COMBINING NI ABOVE..COPTIC COMBINING SPIRITUS LENIS
2D7F ; InCB; Extend # Mn TIFINAGH CONSONANT JOINER
2DE0..2DFF ; InCB; Extend # Mn [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS
302A..302D ; InCB; Extend # Mn [4] IDEOGRAPHIC LEVEL TONE MARK..IDEOGRAPHIC ENTERING TONE MARK
302E..302F ; InCB; Extend # Mc [2] HANGUL SINGLE DOT TONE MARK..HANGUL DOUBLE DOT TONE MARK
3099..309A ; InCB; Extend # Mn [2] COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK..COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK
A66F ; InCB; Extend # Mn COMBINING CYRILLIC VZMET
A674..A67D ; InCB; Extend # Mn [10] COMBINING CYRILLIC LETTER UKRAINIAN IE..COMBINING CYRILLIC PAYEROK
A69E..A69F ; InCB; Extend # Mn [2] COMBINING CYRILLIC LETTER EF..COMBINING CYRILLIC LETTER IOTIFIED E
A6F0..A6F1 ; InCB; Extend # Mn [2] BAMUM COMBINING MARK KOQNDON..BAMUM COMBINING MARK TUKWENTIS
A82C ; InCB; Extend # Mn SYLOTI NAGRI SIGN ALTERNATE HASANTA
A8E0..A8F1 ; InCB; Extend # Mn [18] COMBINING DEVANAGARI DIGIT ZERO..COMBINING DEVANAGARI SIGN AVAGRAHA
A92B..A92D ; InCB; Extend # Mn [3] KAYAH LI TONE PLOPHU..KAYAH LI TONE CALYA PLOPHU
A9B3 ; InCB; Extend # Mn JAVANESE SIGN CECAK TELU
AAB0 ; InCB; Extend # Mn TAI VIET MAI KANG
AAB2..AAB4 ; InCB; Extend # Mn [3] TAI VIET VOWEL I..TAI VIET VOWEL U
AAB7..AAB8 ; InCB; Extend # Mn [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA
AABE..AABF ; InCB; Extend # Mn [2] TAI VIET VOWEL AM..TAI VIET TONE MAI EK
AAC1 ; InCB; Extend # Mn TAI VIET TONE MAI THO
AAF6 ; InCB; Extend # Mn MEETEI MAYEK VIRAMA
ABED ; InCB; Extend # Mn MEETEI MAYEK APUN IYEK
FB1E ; InCB; Extend # Mn HEBREW POINT JUDEO-SPANISH VARIKA
FE20..FE2F ; InCB; Extend # Mn [16] COMBINING LIGATURE LEFT HALF..COMBINING CYRILLIC TITLO RIGHT HALF
101FD ; InCB; Extend # Mn PHAISTOS DISC SIGN COMBINING OBLIQUE STROKE
102E0 ; InCB; Extend # Mn COPTIC EPACT THOUSANDS MARK
10376..1037A ; InCB; Extend # Mn [5] COMBINING OLD PERMIC LETTER AN..COMBINING OLD PERMIC LETTER SII
10A0D ; InCB; Extend # Mn KHAROSHTHI SIGN DOUBLE RING BELOW
10A0F ; InCB; Extend # Mn KHAROSHTHI SIGN VISARGA
10A38..10A3A ; InCB; Extend # Mn [3] KHAROSHTHI SIGN BAR ABOVE..KHAROSHTHI SIGN DOT BELOW
10A3F ; InCB; Extend # Mn KHAROSHTHI VIRAMA
10AE5..10AE6 ; InCB; Extend # Mn [2] MANICHAEAN ABBREVIATION MARK ABOVE..MANICHAEAN ABBREVIATION MARK BELOW
10D24..10D27 ; InCB; Extend # Mn [4] HANIFI ROHINGYA SIGN HARBAHAY..HANIFI ROHINGYA SIGN TASSI
10EAB..10EAC ; InCB; Extend # Mn [2] YEZIDI COMBINING HAMZA MARK..YEZIDI COMBINING MADDA MARK
10EFD..10EFF ; InCB; Extend # Mn [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA
10F46..10F50 ; InCB; Extend # Mn [11] SOGDIAN COMBINING DOT BELOW..SOGDIAN COMBINING STROKE BELOW
10F82..10F85 ; InCB; Extend # Mn [4] OLD UYGHUR COMBINING DOT ABOVE..OLD UYGHUR COMBINING TWO DOTS BELOW
11070 ; InCB; Extend # Mn BRAHMI SIGN OLD TAMIL VIRAMA
1107F ; InCB; Extend # Mn BRAHMI NUMBER JOINER
110BA ; InCB; Extend # Mn KAITHI SIGN NUKTA
11100..11102 ; InCB; Extend # Mn [3] CHAKMA SIGN CANDRABINDU..CHAKMA SIGN VISARGA
11133..11134 ; InCB; Extend # Mn [2] CHAKMA VIRAMA..CHAKMA MAAYYAA
11173 ; InCB; Extend # Mn MAHAJANI SIGN NUKTA
111CA ; InCB; Extend # Mn SHARADA SIGN NUKTA
11236 ; InCB; Extend # Mn KHOJKI SIGN NUKTA
112E9..112EA ; InCB; Extend # Mn [2] KHUDAWADI SIGN NUKTA..KHUDAWADI SIGN VIRAMA
1133B..1133C ; InCB; Extend # Mn [2] COMBINING BINDU BELOW..GRANTHA SIGN NUKTA
11366..1136C ; InCB; Extend # Mn [7] COMBINING GRANTHA DIGIT ZERO..COMBINING GRANTHA DIGIT SIX
11370..11374 ; InCB; Extend # Mn [5] COMBINING GRANTHA LETTER A..COMBINING GRANTHA LETTER PA
11446 ; InCB; Extend # Mn NEWA SIGN NUKTA
1145E ; InCB; Extend # Mn NEWA SANDHI MARK
114C3 ; InCB; Extend # Mn TIRHUTA SIGN NUKTA
115C0 ; InCB; Extend # Mn SIDDHAM SIGN NUKTA
116B7 ; InCB; Extend # Mn TAKRI SIGN NUKTA
1172B ; InCB; Extend # Mn AHOM SIGN KILLER
1183A ; InCB; Extend # Mn DOGRA SIGN NUKTA
1193E ; InCB; Extend # Mn DIVES AKURU VIRAMA
11943 ; InCB; Extend # Mn DIVES AKURU SIGN NUKTA
11A34 ; InCB; Extend # Mn ZANABAZAR SQUARE SIGN VIRAMA
11A47 ; InCB; Extend # Mn ZANABAZAR SQUARE SUBJOINER
11A99 ; InCB; Extend # Mn SOYOMBO SUBJOINER
11D42 ; InCB; Extend # Mn MASARAM GONDI SIGN NUKTA
11D44..11D45 ; InCB; Extend # Mn [2] MASARAM GONDI SIGN HALANTA..MASARAM GONDI VIRAMA
11D97 ; InCB; Extend # Mn GUNJALA GONDI VIRAMA
11F42 ; InCB; Extend # Mn KAWI CONJOINER
16AF0..16AF4 ; InCB; Extend # Mn [5] BASSA VAH COMBINING HIGH TONE..BASSA VAH COMBINING HIGH-LOW TONE
16B30..16B36 ; InCB; Extend # Mn [7] PAHAWH HMONG MARK CIM TUB..PAHAWH HMONG MARK CIM TAUM
1BC9E ; InCB; Extend # Mn DUPLOYAN DOUBLE MARK
1D165 ; InCB; Extend # Mc MUSICAL SYMBOL COMBINING STEM
1D167..1D169 ; InCB; Extend # Mn [3] MUSICAL SYMBOL COMBINING TREMOLO-1..MUSICAL SYMBOL COMBINING TREMOLO-3
1D16E..1D172 ; InCB; Extend # Mc [5] MUSICAL SYMBOL COMBINING FLAG-1..MUSICAL SYMBOL COMBINING FLAG-5
1D17B..1D182 ; InCB; Extend # Mn [8] MUSICAL SYMBOL COMBINING ACCENT..MUSICAL SYMBOL COMBINING LOURE
1D185..1D18B ; InCB; Extend # Mn [7] MUSICAL SYMBOL COMBINING DOIT..MUSICAL SYMBOL COMBINING TRIPLE TONGUE
1D1AA..1D1AD ; InCB; Extend # Mn [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO
1D242..1D244 ; InCB; Extend # Mn [3] COMBINING GREEK MUSICAL TRISEME..COMBINING GREEK MUSICAL PENTASEME
1E000..1E006 ; InCB; Extend # Mn [7] COMBINING GLAGOLITIC LETTER AZU..COMBINING GLAGOLITIC LETTER ZHIVETE
1E008..1E018 ; InCB; Extend # Mn [17] COMBINING GLAGOLITIC LETTER ZEMLJA..COMBINING GLAGOLITIC LETTER HERU
1E01B..1E021 ; InCB; Extend # Mn [7] COMBINING GLAGOLITIC LETTER SHTA..COMBINING GLAGOLITIC LETTER YATI
1E023..1E024 ; InCB; Extend # Mn [2] COMBINING GLAGOLITIC LETTER YU..COMBINING GLAGOLITIC LETTER SMALL YUS
1E026..1E02A ; InCB; Extend # Mn [5] COMBINING GLAGOLITIC LETTER YO..COMBINING GLAGOLITIC LETTER FITA
1E08F ; InCB; Extend # Mn COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I
1E130..1E136 ; InCB; Extend # Mn [7] NYIAKENG PUACHUE HMONG TONE-B..NYIAKENG PUACHUE HMONG TONE-D
1E2AE ; InCB; Extend # Mn TOTO SIGN RISING TONE
1E2EC..1E2EF ; InCB; Extend # Mn [4] WANCHO TONE TUP..WANCHO TONE KOINI
1E4EC..1E4EF ; InCB; Extend # Mn [4] NAG MUNDARI SIGN MUHOR..NAG MUNDARI SIGN SUTUH
1E8D0..1E8D6 ; InCB; Extend # Mn [7] MENDE KIKAKUI COMBINING NUMBER TEENS..MENDE KIKAKUI COMBINING NUMBER MILLIONS
1E944..1E94A ; InCB; Extend # Mn [7] ADLAM ALIF LENGTHENER..ADLAM NUKTA
# Total code points: 884
# EOF

22
libcxx/utils/data/unicode/DerivedGeneralCategory.txt
View File

@@ -1,6 +1,6 @@
# DerivedGeneralCategory-15.0.0.txt
# Date: 2022-04-26, 23:14:35 GMT
# © 2022 Unicode®, Inc.
# DerivedGeneralCategory-15.1.0.txt
# Date: 2023-07-28, 23:34:02 GMT
# © 2023 Unicode®, Inc.
# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
# For terms of use, see https://www.unicode.org/terms_of_use.html
#
@@ -284,13 +284,12 @@
2E9A ; Cn # <reserved-2E9A>
2EF4..2EFF ; Cn # [12] <reserved-2EF4>..<reserved-2EFF>
2FD6..2FEF ; Cn # [26] <reserved-2FD6>..<reserved-2FEF>
2FFC..2FFF ; Cn # [4] <reserved-2FFC>..<reserved-2FFF>
3040 ; Cn # <reserved-3040>
3097..3098 ; Cn # [2] <reserved-3097>..<reserved-3098>
3100..3104 ; Cn # [5] <reserved-3100>..<reserved-3104>
3130 ; Cn # <reserved-3130>
318F ; Cn # <reserved-318F>
31E4..31EF ; Cn # [12] <reserved-31E4>..<reserved-31EF>
31E4..31EE ; Cn # [11] <reserved-31E4>..<reserved-31EE>
321F ; Cn # <reserved-321F>
A48D..A48F ; Cn # [3] <reserved-A48D>..<reserved-A48F>
A4C7..A4CF ; Cn # [9] <reserved-A4C7>..<reserved-A4CF>
@@ -713,7 +712,8 @@ FFFE..FFFF ; Cn # [2] <noncharacter-FFFE>..<noncharacter-FFFF>
2B73A..2B73F ; Cn # [6] <reserved-2B73A>..<reserved-2B73F>
2B81E..2B81F ; Cn # [2] <reserved-2B81E>..<reserved-2B81F>
2CEA2..2CEAF ; Cn # [14] <reserved-2CEA2>..<reserved-2CEAF>
2EBE1..2F7FF ; Cn # [3103] <reserved-2EBE1>..<reserved-2F7FF>
2EBE1..2EBEF ; Cn # [15] <reserved-2EBE1>..<reserved-2EBEF>
2EE5E..2F7FF ; Cn # [2466] <reserved-2EE5E>..<reserved-2F7FF>
2FA1E..2FFFF ; Cn # [1506] <reserved-2FA1E>..<noncharacter-2FFFF>
3134B..3134F ; Cn # [5] <reserved-3134B>..<reserved-3134F>
323B0..E0000 ; Cn # [711761] <reserved-323B0>..<reserved-E0000>
@@ -723,7 +723,7 @@ E01F0..EFFFF ; Cn # [65040] <reserved-E01F0>..<noncharacter-EFFFF>
FFFFE..FFFFF ; Cn # [2] <noncharacter-FFFFE>..<noncharacter-FFFFF>
10FFFE..10FFFF; Cn # [2] <noncharacter-10FFFE>..<noncharacter-10FFFF>
# Total code points: 825345
# Total code points: 824718
# ================================================
@@ -2649,11 +2649,12 @@ FFDA..FFDC ; Lo # [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I
2B740..2B81D ; Lo # [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D
2B820..2CEA1 ; Lo # [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1
2CEB0..2EBE0 ; Lo # [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0
2EBF0..2EE5D ; Lo # [622] CJK UNIFIED IDEOGRAPH-2EBF0..CJK UNIFIED IDEOGRAPH-2EE5D
2F800..2FA1D ; Lo # [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D
30000..3134A ; Lo # [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A
31350..323AF ; Lo # [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF
# Total code points: 131612
# Total code points: 132234
# ================================================
@@ -4092,7 +4093,7 @@ FFE3 ; Sk # FULLWIDTH MACRON
2E80..2E99 ; So # [26] CJK RADICAL REPEAT..CJK RADICAL RAP
2E9B..2EF3 ; So # [89] CJK RADICAL CHOKE..CJK RADICAL C-SIMPLIFIED TURTLE
2F00..2FD5 ; So # [214] KANGXI RADICAL ONE..KANGXI RADICAL FLUTE
2FF0..2FFB ; So # [12] IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER OVERLAID
2FF0..2FFF ; So # [16] IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER ROTATION
3004 ; So # JAPANESE INDUSTRIAL STANDARD SYMBOL
3012..3013 ; So # [2] POSTAL MARK..GETA MARK
3020 ; So # POSTAL MARK FACE
@@ -4101,6 +4102,7 @@ FFE3 ; Sk # FULLWIDTH MACRON
3190..3191 ; So # [2] IDEOGRAPHIC ANNOTATION LINKING MARK..IDEOGRAPHIC ANNOTATION REVERSE MARK
3196..319F ; So # [10] IDEOGRAPHIC ANNOTATION TOP MARK..IDEOGRAPHIC ANNOTATION MAN MARK
31C0..31E3 ; So # [36] CJK STROKE T..CJK STROKE Q
31EF ; So # IDEOGRAPHIC DESCRIPTION CHARACTER SUBTRACTION
3200..321E ; So # [31] PARENTHESIZED HANGUL KIYEOK..PARENTHESIZED KOREAN CHARACTER O HU
322A..3247 ; So # [30] PARENTHESIZED IDEOGRAPH MOON..CIRCLED IDEOGRAPH KOTO
3250 ; So # PARTNERSHIP SIGN
@@ -4191,7 +4193,7 @@ FFFC..FFFD ; So # [2] OBJECT REPLACEMENT CHARACTER..REPLACEMENT CHARACTER
1FB00..1FB92 ; So # [147] BLOCK SEXTANT-1..UPPER HALF INVERSE MEDIUM SHADE AND LOWER HALF BLOCK
1FB94..1FBCA ; So # [55] LEFT HALF INVERSE MEDIUM SHADE AND RIGHT HALF BLOCK..WHITE UP-POINTING CHEVRON
# Total code points: 6634
# Total code points: 6639
# ================================================

5170
libcxx/utils/data/unicode/EastAsianWidth.txt
View File

File diff suppressed because it is too large Load Diff

6
libcxx/utils/data/unicode/GraphemeBreakProperty.txt
View File

@@ -1,6 +1,6 @@
# GraphemeBreakProperty-15.0.0.txt
# Date: 2022-04-27, 17:07:38 GMT
# © 2022 Unicode®, Inc.
# GraphemeBreakProperty-15.1.0.txt
# Date: 2023-01-05, 20:34:41 GMT
# © 2023 Unicode®, Inc.
# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
# For terms of use, see https://www.unicode.org/terms_of_use.html
#

727
libcxx/utils/data/unicode/GraphemeBreakTest.txt
View File

File diff suppressed because it is too large Load Diff

10
libcxx/utils/data/unicode/emoji-data.txt
View File

@@ -1,16 +1,16 @@
# emoji-data.txt
# Date: 2022-08-02, 00:26:10 GMT
# © 2022 Unicode®, Inc.
# Date: 2023-02-01, 02:22:54 GMT
# © 2023 Unicode®, Inc.
# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
# For terms of use, see https://www.unicode.org/terms_of_use.html
#
# Emoji Data for UTS #51
# Used with Emoji Version 15.0 and subsequent minor revisions (if any)
# Used with Emoji Version 15.1 and subsequent minor revisions (if any)
#
# For documentation and usage, see https://www.unicode.org/reports/tr51
#
# Format:
# <codepoint(s)> ; <property> # <comments>
# Format:
# <codepoint(s)> ; <property> # <comments>
# Note: there is no guarantee as to the structure of whitespace or comments
#
# Characters and sequences are listed in code point order. Users should be shown a more natural order.

25
libcxx/utils/generate_extended_grapheme_cluster_table.py
View File

@@ -289,25 +289,16 @@ def generate_cpp_data(prop_name: str, ranges: list[PropertyRange]) -> str:
def generate_data_tables() -> str:
"""
Generate Unicode data for inclusion into <format> from
GraphemeBreakProperty.txt and emoji-data.txt.
- https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakProperty.txt
- https://www.unicode.org/Public/UCD/latest/ucd/emoji/emoji-data.txt
- https://www.unicode.org/Public/UCD/latest/ucd/DerivedCoreProperties.txt
GraphemeBreakProperty.txt can be found at
https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakProperty.txt
emoji-data.txt can be found at
https://www.unicode.org/Public/UCD/latest/ucd/emoji/emoji-data.txt
Both files are expected to be in the same directory as this script.
These files are expected to be stored in the same directory as this script.
"""
gbp_data_path = (
Path(__file__).absolute().parent
/ "data"
/ "unicode"
/ "GraphemeBreakProperty.txt"
)
emoji_data_path = (
Path(__file__).absolute().parent / "data" / "unicode" / "emoji-data.txt"
)
root = Path(__file__).absolute().parent / "data" / "unicode"
gbp_data_path = root / "GraphemeBreakProperty.txt"
emoji_data_path = root / "emoji-data.txt"
gbp_ranges = list()
emoji_ranges = list()
with gbp_data_path.open(encoding="utf-8") as f:

309
libcxx/utils/generate_indic_conjunct_break_table.py Executable file
View File

@@ -0,0 +1,309 @@
#!/usr/bin/env python
# ===----------------------------------------------------------------------===##
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
# ===----------------------------------------------------------------------===##
# The code is based on
# https://github.com/microsoft/STL/blob/main/tools/unicode_properties_parse/grapheme_break_property_data_gen.py
#
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
from io import StringIO
from pathlib import Path
from dataclasses import dataclass
from typing import Optional
import re
import sys
@dataclass
class PropertyRange:
lower: int = -1
upper: int = -1
prop: str = None
@dataclass
class Entry:
lower: int = -1
offset: int = -1
prop: int = -1
LINE_REGEX = re.compile(
r"^(?P<lower>[0-9A-F]{4,5})(?:\.\.(?P<upper>[0-9A-F]{4,5}))?\s*;\s*InCB;\s*(?P<prop>\w+)"
)
def parsePropertyLine(inputLine: str) -> Optional[PropertyRange]:
result = PropertyRange()
if m := LINE_REGEX.match(inputLine):
lower_str, upper_str, result.prop = m.group("lower", "upper", "prop")
result.lower = int(lower_str, base=16)
result.upper = result.lower
if upper_str is not None:
result.upper = int(upper_str, base=16)
return result
else:
return None
def compactPropertyRanges(input: list[PropertyRange]) -> list[PropertyRange]:
"""
Merges consecutive ranges with the same property to one range.
Merging the ranges results in fewer ranges in the output table,
reducing binary and improving lookup performance.
"""
result = list()
for x in input:
if (
len(result)
and result[-1].prop == x.prop
and result[-1].upper + 1 == x.lower
):
result[-1].upper = x.upper
continue
result.append(x)
return result
PROP_VALUE_ENUMERATOR_TEMPLATE = " __{}"
PROP_VALUE_ENUM_TEMPLATE = """
enum class __property : uint8_t {{
// Values generated from the data files.
{enumerators},
// The code unit has none of above properties.
__none
}};
"""
DATA_ARRAY_TEMPLATE = """
/// The entries of the indic conjunct break property table.
///
/// The data is generated from
/// - https://www.unicode.org/Public/UCD/latest/ucd/DerivedCoreProperties.txt
///
/// The data has 3 values
/// - bits [0, 1] The property. One of the values generated from the datafiles
/// of \\ref __property
/// - bits [2, 10] The size of the range.
/// - bits [11, 31] The lower bound code point of the range. The upper bound of
/// the range is lower bound + size.
///
/// The 9 bits for the size allow a maximum range of 512 elements. Some ranges
/// in the Unicode tables are larger. They are stored in multiple consecutive
/// ranges in the data table. An alternative would be to store the sizes in a
/// separate 16-bit value. The original MSVC STL code had such an approach, but
/// this approach uses less space for the data and is about 4% faster in the
/// following benchmark.
/// libcxx/benchmarks/std_format_spec_string_unicode.bench.cpp
// clang-format off
_LIBCPP_HIDE_FROM_ABI inline constexpr uint32_t __entries[{size}] = {{
{entries}}};
// clang-format on
/// Returns the indic conjuct break property of a code point.
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __property __get_property(const char32_t __code_point) noexcept {{
// The algorithm searches for the upper bound of the range and, when found,
// steps back one entry. This algorithm is used since the code point can be
// anywhere in the range. After a lower bound is found the next step is to
// compare whether the code unit is indeed in the range.
//
// Since the entry contains a code unit, size, and property the code point
// being sought needs to be adjusted. Just shifting the code point to the
// proper position doesn't work; suppose an entry has property 0, size 1,
// and lower bound 3. This results in the entry 0x1810.
// When searching for code point 3 it will search for 0x1800, find 0x1810
// and moves to the previous entry. Thus the lower bound value will never
// be found.
// The simple solution is to set the bits belonging to the property and
// size. Then the upper bound for code point 3 will return the entry after
// 0x1810. After moving to the previous entry the algorithm arrives at the
// correct entry.
ptrdiff_t __i = std::ranges::upper_bound(__entries, (__code_point << 11) | 0x7ffu) - __entries;
if (__i == 0)
return __property::__none;
--__i;
uint32_t __upper_bound = (__entries[__i] >> 11) + ((__entries[__i] >> 2) & 0b1'1111'1111);
if (__code_point <= __upper_bound)
return static_cast<__property>(__entries[__i] & 0b11);
return __property::__none;
}}
"""
MSVC_FORMAT_UCD_TABLES_HPP_TEMPLATE = """
// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
// WARNING, this entire header is generated by
// utils/generate_indic_conjunct_break_table.py
// DO NOT MODIFY!
// UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE
//
// See Terms of Use <https://www.unicode.org/copyright.html>
// for definitions of Unicode Inc.'s Data Files and Software.
//
// NOTICE TO USER: Carefully read the following legal agreement.
// BY DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S
// DATA FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"),
// YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE
// TERMS AND CONDITIONS OF THIS AGREEMENT.
// IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE
// THE DATA FILES OR SOFTWARE.
//
// COPYRIGHT AND PERMISSION NOTICE
//
// Copyright (c) 1991-2022 Unicode, Inc. All rights reserved.
// Distributed under the Terms of Use in https://www.unicode.org/copyright.html.
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of the Unicode data files and any associated documentation
// (the "Data Files") or Unicode software and any associated documentation
// (the "Software") to deal in the Data Files or Software
// without restriction, including without limitation the rights to use,
// copy, modify, merge, publish, distribute, and/or sell copies of
// the Data Files or Software, and to permit persons to whom the Data Files
// or Software are furnished to do so, provided that either
// (a) this copyright and permission notice appear with all copies
// of the Data Files or Software, or
// (b) this copyright and permission notice appear in associated
// Documentation.
//
// THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF
// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT OF THIRD PARTY RIGHTS.
// IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS
// NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
// DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
// DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
// TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THE DATA FILES OR SOFTWARE.
//
// Except as contained in this notice, the name of a copyright holder
// shall not be used in advertising or otherwise to promote the sale,
// use or other dealings in these Data Files or Software without prior
// written authorization of the copyright holder.
#ifndef _LIBCPP___FORMAT_INDIC_CONJUNCT_BREAK_TABLE_H
#define _LIBCPP___FORMAT_INDIC_CONJUNCT_BREAK_TABLE_H
#include <__algorithm/ranges_upper_bound.h>
#include <__config>
#include <__iterator/access.h>
#include <cstddef>
#include <cstdint>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
#endif
_LIBCPP_BEGIN_NAMESPACE_STD
#if _LIBCPP_STD_VER >= 20
namespace __indic_conjunct_break {{
{content}
}} // namespace __indic_conjunct_break
#endif //_LIBCPP_STD_VER >= 20
_LIBCPP_END_NAMESPACE_STD
#endif // _LIBCPP___FORMAT_INDIC_CONJUNCT_BREAK_TABLE_H"""
def property_ranges_to_table(
ranges: list[PropertyRange], props: list[str]
) -> list[Entry]:
assert len(props) < 4
result = list[Entry]()
high = -1
for range in sorted(ranges, key=lambda x: x.lower):
# Validate overlapping ranges
assert range.lower > high
high = range.upper
while True:
e = Entry(range.lower, range.upper - range.lower, props.index(range.prop))
if e.offset <= 511:
result.append(e)
break
e.offset = 511
result.append(e)
range.lower += 512
return result
cpp_entrytemplate = " 0x{:08x}"
def generate_cpp_data(prop_name: str, ranges: list[PropertyRange]) -> str:
result = StringIO()
prop_values = sorted(set(x.prop for x in ranges))
table = property_ranges_to_table(ranges, prop_values)
enumerator_values = [PROP_VALUE_ENUMERATOR_TEMPLATE.format(x) for x in prop_values]
result.write(
PROP_VALUE_ENUM_TEMPLATE.format(enumerators=",\n".join(enumerator_values))
)
result.write(
DATA_ARRAY_TEMPLATE.format(
prop_name=prop_name,
size=len(table),
entries=",\n".join(
[
cpp_entrytemplate.format(x.lower << 11 | x.offset << 2 | x.prop)
for x in table
]
),
)
)
return result.getvalue()
def generate_data_tables() -> str:
"""
Generate Unicode data for inclusion into <format> from
- https://www.unicode.org/Public/UCD/latest/ucd/DerivedCoreProperties.txt
These files are expected to be stored in the same directory as this script.
"""
root = Path(__file__).absolute().parent / "data" / "unicode"
derived_core_path = root / "DerivedCoreProperties.txt"
indic_conjunct_break = list()
with derived_core_path.open(encoding="utf-8") as f:
indic_conjunct_break_ranges = compactPropertyRanges(
[x for line in f if (x := parsePropertyLine(line))]
)
indic_conjunct_break_data = generate_cpp_data("Grapheme_Break", indic_conjunct_break_ranges)
return "\n".join([indic_conjunct_break_data])
if __name__ == "__main__":
if len(sys.argv) == 2:
sys.stdout = open(sys.argv[1], "w")
print(
MSVC_FORMAT_UCD_TABLES_HPP_TEMPLATE.lstrip().format(
content=generate_data_tables()
)
)