Files
clang-p2996/libcxx/include/__format/unicode.h
Mark de Wever 59e66c515a [libc++][format] Switches to Unicode 15.1. (#86543)
In addition to changes in the tables the extended grapheme clustering
algorithm has been overhauled. Before I considered a separate state
machine to implement the rules. With the new rule GB9c this became more
attractive and the design has changed.

This change initially had quite an impact on the performance. By making
the state machine persistent the performance was improved greatly. Note
it is still slower than before due to the larger Unicode tables.

Before
--------------------------------------------------------------------
Benchmark                          Time             CPU   Iterations
--------------------------------------------------------------------
BM_ascii_text<char>             1891 ns         1889 ns       369504
BM_unicode_text<char>         106642 ns       106397 ns         6576
BM_cyrillic_text<char>         73420 ns        73277 ns         9445
BM_japanese_text<char>         62485 ns        62387 ns        11153
BM_emoji_text<char>             1895 ns         1893 ns       369525
BM_ascii_text<wchar_t>          2015 ns         2013 ns       346887
BM_unicode_text<wchar_t>       92119 ns        92017 ns         7598
BM_cyrillic_text<wchar_t>      62637 ns        62568 ns        11117
BM_japanese_text<wchar_t>      53850 ns        53785 ns        12803
BM_emoji_text<wchar_t>          2016 ns         2014 ns       347325

After
--------------------------------------------------------------------
Benchmark                          Time             CPU   Iterations
--------------------------------------------------------------------
BM_ascii_text<char>             1906 ns         1904 ns       369409
BM_unicode_text<char>         265462 ns       265175 ns         2628
BM_cyrillic_text<char>        181063 ns       180865 ns         3871
BM_japanese_text<char>        130927 ns       130789 ns         5324
BM_emoji_text<char>             1892 ns         1890 ns       370537
BM_ascii_text<wchar_t>          2038 ns         2035 ns       343689
BM_unicode_text<wchar_t>      277603 ns       277282 ns         2526
BM_cyrillic_text<wchar_t>     188558 ns       188339 ns         3727
BM_japanese_text<wchar_t>     133084 ns       132943 ns         5262
BM_emoji_text<wchar_t>          2012 ns         2010 ns       348015

Persistent
--------------------------------------------------------------------
Benchmark                          Time             CPU   Iterations
--------------------------------------------------------------------
BM_ascii_text<char>             1904 ns         1899 ns       367472
BM_unicode_text<char>         133609 ns       133287 ns         5246
BM_cyrillic_text<char>         90185 ns        89941 ns         7796
BM_japanese_text<char>         75137 ns        74946 ns         9316
BM_emoji_text<char>             1906 ns         1901 ns       368081
BM_ascii_text<wchar_t>          2703 ns         2696 ns       259153
BM_unicode_text<wchar_t>      131497 ns       131168 ns         5341
BM_cyrillic_text<wchar_t>      87071 ns        86840 ns         8076
BM_japanese_text<wchar_t>      72279 ns        72099 ns         9682
BM_emoji_text<wchar_t>          2021 ns         2016 ns       346767
2024-04-09 19:20:06 +02:00

603 lines
23 KiB
C++

// -*- C++ -*-
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef _LIBCPP___FORMAT_UNICODE_H
#define _LIBCPP___FORMAT_UNICODE_H
#include <__assert>
#include <__bit/countl.h>
#include <__concepts/same_as.h>
#include <__config>
#include <__format/extended_grapheme_cluster_table.h>
#include <__format/indic_conjunct_break_table.h>
#include <__iterator/concepts.h>
#include <__iterator/readable_traits.h> // iter_value_t
#include <__utility/unreachable.h>
#include <string_view>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
#endif
_LIBCPP_BEGIN_NAMESPACE_STD
#if _LIBCPP_STD_VER >= 20
namespace __unicode {
// Helper struct for the result of a consume operation.
//
// The status value for a correct code point is 0. This allows a valid value to
// be used without masking.
// When the decoding fails it know the number of code units affected. For the
// current use-cases that value is not needed, therefore it is not stored.
// The escape routine needs the number of code units for both a valid and
// invalid character and keeps track of it itself. Doing it in this result
// unconditionally would give some overhead when the value is unneeded.
struct __consume_result {
// When __status == __ok it contains the decoded code point.
// Else it contains the replacement character U+FFFD
char32_t __code_point : 31;
enum : char32_t {
// Consumed a well-formed code point.
__ok = 0,
// Encountered invalid UTF-8
__error = 1
} __status : 1 {__ok};
};
static_assert(sizeof(__consume_result) == sizeof(char32_t));
# ifndef _LIBCPP_HAS_NO_UNICODE
/// Implements the grapheme cluster boundary rules
///
/// These rules are used to implement format's width estimation as stated in
/// [format.string.std]/11
///
/// The Standard refers to UAX \#29 for Unicode 12.0.0
/// https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundary_Rules
///
/// The data tables used are
/// https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakProperty.txt
/// https://www.unicode.org/Public/UCD/latest/ucd/emoji/emoji-data.txt
/// https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakTest.txt (for testing only)
inline constexpr char32_t __replacement_character = U'\ufffd';
// The error of a consume operation.
//
// This sets the code point to the replacement character. This code point does
// not participate in the grapheme clustering, so grapheme clustering code can
// ignore the error status and always use the code point.
inline constexpr __consume_result __consume_result_error{__replacement_character, __consume_result::__error};
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool __is_high_surrogate(char32_t __value) {
return __value >= 0xd800 && __value <= 0xdbff;
}
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool __is_low_surrogate(char32_t __value) {
return __value >= 0xdc00 && __value <= 0xdfff;
}
// https://www.unicode.org/glossary/#surrogate_code_point
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool __is_surrogate(char32_t __value) {
return __value >= 0xd800 && __value <= 0xdfff;
}
// https://www.unicode.org/glossary/#code_point
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool __is_code_point(char32_t __value) {
return __value <= 0x10ffff;
}
// https://www.unicode.org/glossary/#unicode_scalar_value
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool __is_scalar_value(char32_t __value) {
return __unicode::__is_code_point(__value) && !__unicode::__is_surrogate(__value);
}
template <contiguous_iterator _Iterator>
requires same_as<iter_value_t<_Iterator>, char>
_LIBCPP_HIDE_FROM_ABI constexpr bool __is_continuation(_Iterator __char, int __count) {
do {
if ((*__char & 0b1100'0000) != 0b1000'0000)
return false;
--__count;
++__char;
} while (__count);
return true;
}
/// Helper class to extract a code unit from a Unicode character range.
///
/// The stored range is a view. There are multiple specialization for different
/// character types.
template <class _CharT>
class __code_point_view;
/// UTF-8 specialization.
template <>
class __code_point_view<char> {
using _Iterator = basic_string_view<char>::const_iterator;
public:
_LIBCPP_HIDE_FROM_ABI constexpr explicit __code_point_view(_Iterator __first, _Iterator __last)
: __first_(__first), __last_(__last) {}
_LIBCPP_HIDE_FROM_ABI constexpr bool __at_end() const noexcept { return __first_ == __last_; }
_LIBCPP_HIDE_FROM_ABI constexpr _Iterator __position() const noexcept { return __first_; }
// https://www.unicode.org/versions/latest/ch03.pdf#G7404
// Based on Table 3-7, Well-Formed UTF-8 Byte Sequences
//
// Code Points First Byte Second Byte Third Byte Fourth Byte Remarks
// U+0000..U+007F 00..7F U+0000..U+007F 1 code unit range
// C0..C1 80..BF invalid overlong encoding
// U+0080..U+07FF C2..DF 80..BF U+0080..U+07FF 2 code unit range
// E0 80..9F 80..BF invalid overlong encoding
// U+0800..U+0FFF E0 A0..BF 80..BF U+0800..U+FFFF 3 code unit range
// U+1000..U+CFFF E1..EC 80..BF 80..BF
// U+D000..U+D7FF ED 80..9F 80..BF
// U+D800..U+DFFF ED A0..BF 80..BF invalid encoding of surrogate code point
// U+E000..U+FFFF EE..EF 80..BF 80..BF
// F0 80..8F 80..BF 80..BF invalid overlong encoding
// U+10000..U+3FFFF F0 90..BF 80..BF 80..BF U+10000..U+10FFFF 4 code unit range
// U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF
// U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
// F4 90..BF 80..BF 80..BF U+110000.. invalid code point range
//
// Unlike other parsers, these invalid entries are tested after decoding.
// - The parser always needs to consume these code units
// - The code is optimized for well-formed UTF-8
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept {
_LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input");
// Based on the number of leading 1 bits the number of code units in the
// code point can be determined. See
// https://en.wikipedia.org/wiki/UTF-8#Encoding
switch (std::countl_one(static_cast<unsigned char>(*__first_))) {
case 0:
return {static_cast<unsigned char>(*__first_++)};
case 2: {
if (__last_ - __first_ < 2 || !__unicode::__is_continuation(__first_ + 1, 1)) [[unlikely]]
break;
char32_t __value = static_cast<unsigned char>(*__first_++) & 0x1f;
__value <<= 6;
__value |= static_cast<unsigned char>(*__first_++) & 0x3f;
// These values should be encoded in 1 UTF-8 code unit.
if (__value < 0x0080) [[unlikely]]
return __consume_result_error;
return {__value};
}
case 3: {
if (__last_ - __first_ < 3 || !__unicode::__is_continuation(__first_ + 1, 2)) [[unlikely]]
break;
char32_t __value = static_cast<unsigned char>(*__first_++) & 0x0f;
__value <<= 6;
__value |= static_cast<unsigned char>(*__first_++) & 0x3f;
__value <<= 6;
__value |= static_cast<unsigned char>(*__first_++) & 0x3f;
// These values should be encoded in 1 or 2 UTF-8 code units.
if (__value < 0x0800) [[unlikely]]
return __consume_result_error;
// A surrogate value is always encoded in 3 UTF-8 code units.
if (__unicode::__is_surrogate(__value)) [[unlikely]]
return __consume_result_error;
return {__value};
}
case 4: {
if (__last_ - __first_ < 4 || !__unicode::__is_continuation(__first_ + 1, 3)) [[unlikely]]
break;
char32_t __value = static_cast<unsigned char>(*__first_++) & 0x07;
__value <<= 6;
__value |= static_cast<unsigned char>(*__first_++) & 0x3f;
__value <<= 6;
__value |= static_cast<unsigned char>(*__first_++) & 0x3f;
__value <<= 6;
__value |= static_cast<unsigned char>(*__first_++) & 0x3f;
// These values should be encoded in 1, 2, or 3 UTF-8 code units.
if (__value < 0x10000) [[unlikely]]
return __consume_result_error;
// A value too large is always encoded in 4 UTF-8 code units.
if (!__unicode::__is_code_point(__value)) [[unlikely]]
return __consume_result_error;
return {__value};
}
}
// An invalid number of leading ones can be garbage or a code unit in the
// middle of a code point. By consuming one code unit the parser may get
// "in sync" after a few code units.
++__first_;
return __consume_result_error;
}
private:
_Iterator __first_;
_Iterator __last_;
};
# ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
_LIBCPP_HIDE_FROM_ABI constexpr bool __is_surrogate_pair_high(wchar_t __value) {
return __value >= 0xd800 && __value <= 0xdbff;
}
_LIBCPP_HIDE_FROM_ABI constexpr bool __is_surrogate_pair_low(wchar_t __value) {
return __value >= 0xdc00 && __value <= 0xdfff;
}
/// This specialization depends on the size of wchar_t
/// - 2 UTF-16 (for example Windows and AIX)
/// - 4 UTF-32 (for example Linux)
template <>
class __code_point_view<wchar_t> {
using _Iterator = typename basic_string_view<wchar_t>::const_iterator;
public:
static_assert(sizeof(wchar_t) == 2 || sizeof(wchar_t) == 4, "sizeof(wchar_t) has a not implemented value");
_LIBCPP_HIDE_FROM_ABI constexpr explicit __code_point_view(_Iterator __first, _Iterator __last)
: __first_(__first), __last_(__last) {}
_LIBCPP_HIDE_FROM_ABI constexpr _Iterator __position() const noexcept { return __first_; }
_LIBCPP_HIDE_FROM_ABI constexpr bool __at_end() const noexcept { return __first_ == __last_; }
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept {
_LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input");
char32_t __value = static_cast<char32_t>(*__first_++);
if constexpr (sizeof(wchar_t) == 2) {
if (__unicode::__is_low_surrogate(__value)) [[unlikely]]
return __consume_result_error;
if (__unicode::__is_high_surrogate(__value)) {
if (__first_ == __last_ || !__unicode::__is_low_surrogate(static_cast<char32_t>(*__first_))) [[unlikely]]
return __consume_result_error;
__value -= 0xd800;
__value <<= 10;
__value += static_cast<char32_t>(*__first_++) - 0xdc00;
__value += 0x10000;
if (!__unicode::__is_code_point(__value)) [[unlikely]]
return __consume_result_error;
}
} else {
if (!__unicode::__is_scalar_value(__value)) [[unlikely]]
return __consume_result_error;
}
return {__value};
}
private:
_Iterator __first_;
_Iterator __last_;
};
# endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
// State machine to implement the Extended Grapheme Cluster Boundary
//
// The exact rules may change between Unicode versions.
// This implements the extended rules see
// https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries
class __extended_grapheme_cluster_break {
using __EGC_property = __extended_grapheme_custer_property_boundary::__property;
using __inCB_property = __indic_conjunct_break::__property;
public:
_LIBCPP_HIDE_FROM_ABI constexpr explicit __extended_grapheme_cluster_break(char32_t __first_code_point)
: __prev_code_point_(__first_code_point),
__prev_property_(__extended_grapheme_custer_property_boundary::__get_property(__first_code_point)) {
// Initializes the active rule.
if (__prev_property_ == __EGC_property::__Extended_Pictographic)
__active_rule_ = __rule::__GB11_emoji;
else if (__prev_property_ == __EGC_property::__Regional_Indicator)
__active_rule_ = __rule::__GB12_GB13_regional_indicator;
else if (__indic_conjunct_break::__get_property(__first_code_point) == __inCB_property::__Consonant)
__active_rule_ = __rule::__GB9c_indic_conjunct_break;
}
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(char32_t __next_code_point) {
__EGC_property __next_property = __extended_grapheme_custer_property_boundary::__get_property(__next_code_point);
bool __result = __evaluate(__next_code_point, __next_property);
__prev_code_point_ = __next_code_point;
__prev_property_ = __next_property;
return __result;
}
// The code point whose break propery are considered during the next
// evaluation cyle.
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr char32_t __current_code_point() const { return __prev_code_point_; }
private:
// The naming of the identifiers matches the Unicode standard.
// NOLINTBEGIN(readability-identifier-naming)
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
__evaluate(char32_t __next_code_point, __EGC_property __next_property) {
switch (__active_rule_) {
case __rule::__none:
return __evaluate_none(__next_code_point, __next_property);
case __rule::__GB9c_indic_conjunct_break:
return __evaluate_GB9c_indic_conjunct_break(__next_code_point, __next_property);
case __rule::__GB11_emoji:
return __evaluate_GB11_emoji(__next_code_point, __next_property);
case __rule::__GB12_GB13_regional_indicator:
return __evaluate_GB12_GB13_regional_indicator(__next_code_point, __next_property);
}
__libcpp_unreachable();
}
_LIBCPP_HIDE_FROM_ABI constexpr bool __evaluate_none(char32_t __next_code_point, __EGC_property __next_property) {
// *** Break at the start and end of text, unless the text is empty. ***
_LIBCPP_ASSERT_INTERNAL(__prev_property_ != __EGC_property::__sot, "should be handled in the constructor"); // GB1
_LIBCPP_ASSERT_INTERNAL(__prev_property_ != __EGC_property::__eot, "should be handled by our caller"); // GB2
// *** Do not break between a CR and LF. Otherwise, break before and after controls. ***
if (__prev_property_ == __EGC_property::__CR && __next_property == __EGC_property::__LF) // GB3
return false;
if (__prev_property_ == __EGC_property::__Control || __prev_property_ == __EGC_property::__CR ||
__prev_property_ == __EGC_property::__LF) // GB4
return true;
if (__next_property == __EGC_property::__Control || __next_property == __EGC_property::__CR ||
__next_property == __EGC_property::__LF) // GB5
return true;
// *** Do not break Hangul syllable sequences. ***
if (__prev_property_ == __EGC_property::__L &&
(__next_property == __EGC_property::__L || __next_property == __EGC_property::__V ||
__next_property == __EGC_property::__LV || __next_property == __EGC_property::__LVT)) // GB6
return false;
if ((__prev_property_ == __EGC_property::__LV || __prev_property_ == __EGC_property::__V) &&
(__next_property == __EGC_property::__V || __next_property == __EGC_property::__T)) // GB7
return false;
if ((__prev_property_ == __EGC_property::__LVT || __prev_property_ == __EGC_property::__T) &&
__next_property == __EGC_property::__T) // GB8
return false;
// *** Do not break before extending characters or ZWJ. ***
if (__next_property == __EGC_property::__Extend || __next_property == __EGC_property::__ZWJ)
return false; // GB9
// *** Do not break before SpacingMarks, or after Prepend characters. ***
if (__next_property == __EGC_property::__SpacingMark) // GB9a
return false;
if (__prev_property_ == __EGC_property::__Prepend) // GB9b
return false;
// *** Do not break within certain combinations with Indic_Conjunct_Break (InCB)=Linker. ***
if (__indic_conjunct_break::__get_property(__next_code_point) == __inCB_property::__Consonant) {
__active_rule_ = __rule::__GB9c_indic_conjunct_break;
__GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant;
return true;
}
// *** Do not break within emoji modifier sequences or emoji zwj sequences. ***
if (__next_property == __EGC_property::__Extended_Pictographic) {
__active_rule_ = __rule::__GB11_emoji;
__GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic;
return true;
}
// *** Do not break within emoji flag sequences ***
// That is, do not break between regional indicator (RI) symbols if there
// is an odd number of RI characters before the break point.
if (__next_property == __EGC_property::__Regional_Indicator) { // GB12 + GB13
__active_rule_ = __rule::__GB12_GB13_regional_indicator;
return true;
}
// *** Otherwise, break everywhere. ***
return true; // GB999
}
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
__evaluate_GB9c_indic_conjunct_break(char32_t __next_code_point, __EGC_property __next_property) {
__inCB_property __break = __indic_conjunct_break::__get_property(__next_code_point);
if (__break == __inCB_property::__none) {
__active_rule_ = __rule::__none;
return __evaluate_none(__next_code_point, __next_property);
}
switch (__GB9c_indic_conjunct_break_state_) {
case __GB9c_indic_conjunct_break_state::__Consonant:
if (__break == __inCB_property::__Extend) {
return false;
}
if (__break == __inCB_property::__Linker) {
__GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Linker;
return false;
}
__active_rule_ = __rule::__none;
return __evaluate_none(__next_code_point, __next_property);
case __GB9c_indic_conjunct_break_state::__Linker:
if (__break == __inCB_property::__Extend) {
return false;
}
if (__break == __inCB_property::__Linker) {
return false;
}
if (__break == __inCB_property::__Consonant) {
__GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant;
return false;
}
__active_rule_ = __rule::__none;
return __evaluate_none(__next_code_point, __next_property);
}
__libcpp_unreachable();
}
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
__evaluate_GB11_emoji(char32_t __next_code_point, __EGC_property __next_property) {
switch (__GB11_emoji_state_) {
case __GB11_emoji_state::__Extended_Pictographic:
if (__next_property == __EGC_property::__Extend) {
__GB11_emoji_state_ = __GB11_emoji_state::__Extend;
return false;
}
[[fallthrough]];
case __GB11_emoji_state::__Extend:
if (__next_property == __EGC_property::__ZWJ) {
__GB11_emoji_state_ = __GB11_emoji_state::__ZWJ;
return false;
}
if (__next_property == __EGC_property::__Extend)
return false;
__active_rule_ = __rule::__none;
return __evaluate_none(__next_code_point, __next_property);
case __GB11_emoji_state::__ZWJ:
if (__next_property == __EGC_property::__Extended_Pictographic) {
__GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic;
return false;
}
__active_rule_ = __rule::__none;
return __evaluate_none(__next_code_point, __next_property);
}
__libcpp_unreachable();
}
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
__evaluate_GB12_GB13_regional_indicator(char32_t __next_code_point, __EGC_property __next_property) {
__active_rule_ = __rule::__none;
if (__next_property == __EGC_property::__Regional_Indicator)
return false;
return __evaluate_none(__next_code_point, __next_property);
}
char32_t __prev_code_point_;
__EGC_property __prev_property_;
enum class __rule {
__none,
__GB9c_indic_conjunct_break,
__GB11_emoji,
__GB12_GB13_regional_indicator,
};
__rule __active_rule_ = __rule::__none;
enum class __GB11_emoji_state {
__Extended_Pictographic,
__Extend,
__ZWJ,
};
__GB11_emoji_state __GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic;
enum class __GB9c_indic_conjunct_break_state {
__Consonant,
__Linker,
};
__GB9c_indic_conjunct_break_state __GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant;
// NOLINTEND(readability-identifier-naming)
};
/// Helper class to extract an extended grapheme cluster from a Unicode character range.
///
/// This function is used to determine the column width of an extended grapheme
/// cluster. In order to do that only the first code point is evaluated.
/// Therefore only this code point is extracted.
template <class _CharT>
class __extended_grapheme_cluster_view {
using _Iterator = typename basic_string_view<_CharT>::const_iterator;
public:
_LIBCPP_HIDE_FROM_ABI constexpr explicit __extended_grapheme_cluster_view(_Iterator __first, _Iterator __last)
: __code_point_view_(__first, __last), __at_break_(__code_point_view_.__consume().__code_point) {}
struct __cluster {
/// The first code point of the extended grapheme cluster.
///
/// The first code point is used to estimate the width of the extended
/// grapheme cluster.
char32_t __code_point_;
/// Points one beyond the last code unit in the extended grapheme cluster.
///
/// It's expected the caller has the start position and thus can determine
/// the code unit range of the extended grapheme cluster.
_Iterator __last_;
};
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __cluster __consume() {
char32_t __code_point = __at_break_.__current_code_point();
_Iterator __position = __code_point_view_.__position();
while (!__code_point_view_.__at_end()) {
if (__at_break_(__code_point_view_.__consume().__code_point))
break;
__position = __code_point_view_.__position();
}
return {__code_point, __position};
}
private:
__code_point_view<_CharT> __code_point_view_;
__extended_grapheme_cluster_break __at_break_;
};
template <contiguous_iterator _Iterator>
__extended_grapheme_cluster_view(_Iterator, _Iterator) -> __extended_grapheme_cluster_view<iter_value_t<_Iterator>>;
# else // _LIBCPP_HAS_NO_UNICODE
// For ASCII every character is a "code point".
// This makes it easier to write code agnostic of the _LIBCPP_HAS_NO_UNICODE define.
template <class _CharT>
class __code_point_view {
using _Iterator = typename basic_string_view<_CharT>::const_iterator;
public:
_LIBCPP_HIDE_FROM_ABI constexpr explicit __code_point_view(_Iterator __first, _Iterator __last)
: __first_(__first), __last_(__last) {}
_LIBCPP_HIDE_FROM_ABI constexpr bool __at_end() const noexcept { return __first_ == __last_; }
_LIBCPP_HIDE_FROM_ABI constexpr _Iterator __position() const noexcept { return __first_; }
[[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept {
_LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input");
return {static_cast<char32_t>(*__first_++)};
}
private:
_Iterator __first_;
_Iterator __last_;
};
# endif // _LIBCPP_HAS_NO_UNICODE
} // namespace __unicode
#endif //_LIBCPP_STD_VER >= 20
_LIBCPP_END_NAMESPACE_STD
#endif // _LIBCPP___FORMAT_UNICODE_H