diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst index 6ab4cafd2bb7..71de10abb6ea 100644 --- a/libcxx/docs/ReleaseNotes/19.rst +++ b/libcxx/docs/ReleaseNotes/19.rst @@ -58,6 +58,7 @@ Improvements and New Features ----------------------------- - The performance of growing ``std::vector`` has been improved for trivially relocatable types. +- A lot of types are considered trivially relocatable now, including ``vector`` and ``string``. - The performance of ``ranges::fill`` and ``ranges::fill_n`` has been improved for ``vector::iterator``\s, resulting in a performance increase of up to 1400x. - The ``std::mismatch`` algorithm has been optimized for integral types, which can lead up to 40x performance diff --git a/libcxx/include/__exception/exception_ptr.h b/libcxx/include/__exception/exception_ptr.h index 0a8337fa39de..edb81caac099 100644 --- a/libcxx/include/__exception/exception_ptr.h +++ b/libcxx/include/__exception/exception_ptr.h @@ -66,6 +66,9 @@ class _LIBCPP_EXPORTED_FROM_ABI exception_ptr { friend _LIBCPP_HIDE_FROM_ABI exception_ptr make_exception_ptr(_Ep) _NOEXCEPT; public: + // exception_ptr is basically a COW string. + using __trivially_relocatable = exception_ptr; + _LIBCPP_HIDE_FROM_ABI exception_ptr() _NOEXCEPT : __ptr_() {} _LIBCPP_HIDE_FROM_ABI exception_ptr(nullptr_t) _NOEXCEPT : __ptr_() {} diff --git a/libcxx/include/__expected/expected.h b/libcxx/include/__expected/expected.h index d7adaac7567b..0f994e297a87 100644 --- a/libcxx/include/__expected/expected.h +++ b/libcxx/include/__expected/expected.h @@ -31,6 +31,7 @@ #include <__type_traits/is_swappable.h> #include <__type_traits/is_trivially_constructible.h> #include <__type_traits/is_trivially_destructible.h> +#include <__type_traits/is_trivially_relocatable.h> #include <__type_traits/is_void.h> #include <__type_traits/lazy.h> #include <__type_traits/negation.h> @@ -463,6 +464,11 @@ public: using error_type = _Err; using unexpected_type = unexpected<_Err>; + using __trivially_relocatable = + __conditional_t<__libcpp_is_trivially_relocatable<_Tp>::value && __libcpp_is_trivially_relocatable<_Err>::value, + expected, + void>; + template using rebind = expected<_Up, error_type>; diff --git a/libcxx/include/__locale b/libcxx/include/__locale index 1e97c7594c8b..4b382764b446 100644 --- a/libcxx/include/__locale +++ b/libcxx/include/__locale @@ -49,6 +49,9 @@ _LIBCPP_HIDE_FROM_ABI const _Facet& use_facet(const locale&); class _LIBCPP_EXPORTED_FROM_ABI locale { public: + // locale is essentially a shared_ptr that doesn't support weak_ptrs and never got a move constructor. + using __trivially_relocatable = locale; + // types: class _LIBCPP_EXPORTED_FROM_ABI facet; class _LIBCPP_EXPORTED_FROM_ABI id; diff --git a/libcxx/include/__memory/shared_ptr.h b/libcxx/include/__memory/shared_ptr.h index a8ff189df2aa..358a851958db 100644 --- a/libcxx/include/__memory/shared_ptr.h +++ b/libcxx/include/__memory/shared_ptr.h @@ -419,6 +419,10 @@ public: typedef _Tp element_type; #endif + // A shared_ptr contains only two raw pointers which point to the heap and move constructing already doesn't require + // any bookkeeping, so it's always trivially relocatable. + using __trivially_relocatable = shared_ptr; + private: element_type* __ptr_; __shared_weak_count* __cntrl_; @@ -1301,6 +1305,10 @@ public: typedef _Tp element_type; #endif + // A weak_ptr contains only two raw pointers which point to the heap and move constructing already doesn't require + // any bookkeeping, so it's always trivially relocatable. + using __trivially_relocatable = weak_ptr; + private: element_type* __ptr_; __shared_weak_count* __cntrl_; diff --git a/libcxx/include/__split_buffer b/libcxx/include/__split_buffer index c68349e0979c..365a5fc4a2c6 100644 --- a/libcxx/include/__split_buffer +++ b/libcxx/include/__split_buffer @@ -24,12 +24,14 @@ #include <__memory/pointer_traits.h> #include <__memory/swap_allocator.h> #include <__type_traits/add_lvalue_reference.h> +#include <__type_traits/conditional.h> #include <__type_traits/enable_if.h> #include <__type_traits/integral_constant.h> #include <__type_traits/is_nothrow_assignable.h> #include <__type_traits/is_nothrow_constructible.h> #include <__type_traits/is_swappable.h> #include <__type_traits/is_trivially_destructible.h> +#include <__type_traits/is_trivially_relocatable.h> #include <__type_traits/remove_reference.h> #include <__utility/forward.h> #include <__utility/move.h> @@ -64,6 +66,15 @@ public: using iterator = pointer; using const_iterator = const_pointer; + // A __split_buffer contains the following members which may be trivially relocatable: + // - pointer: may be trivially relocatable, so it's checked + // - allocator_type: may be trivially relocatable, so it's checked + // __split_buffer doesn't have any self-references, so it's trivially relocatable if its members are. + using __trivially_relocatable = __conditional_t< + __libcpp_is_trivially_relocatable::value && __libcpp_is_trivially_relocatable::value, + __split_buffer, + void>; + pointer __first_; pointer __begin_; pointer __end_; diff --git a/libcxx/include/__utility/pair.h b/libcxx/include/__utility/pair.h index 3ffab2f7fe4f..7e17c4c415c4 100644 --- a/libcxx/include/__utility/pair.h +++ b/libcxx/include/__utility/pair.h @@ -34,6 +34,7 @@ #include <__type_traits/is_nothrow_constructible.h> #include <__type_traits/is_same.h> #include <__type_traits/is_swappable.h> +#include <__type_traits/is_trivially_relocatable.h> #include <__type_traits/nat.h> #include <__type_traits/remove_cvref.h> #include <__type_traits/unwrap_ref.h> @@ -71,6 +72,11 @@ struct _LIBCPP_TEMPLATE_VIS pair _T1 first; _T2 second; + using __trivially_relocatable = + __conditional_t<__libcpp_is_trivially_relocatable<_T1>::value && __libcpp_is_trivially_relocatable<_T2>::value, + pair, + void>; + _LIBCPP_HIDE_FROM_ABI pair(pair const&) = default; _LIBCPP_HIDE_FROM_ABI pair(pair&&) = default; diff --git a/libcxx/include/array b/libcxx/include/array index 6ea094deec32..e030c6f32e9b 100644 --- a/libcxx/include/array +++ b/libcxx/include/array @@ -130,6 +130,7 @@ template const T&& get(const array&&) noexce #include <__type_traits/is_nothrow_constructible.h> #include <__type_traits/is_same.h> #include <__type_traits/is_swappable.h> +#include <__type_traits/is_trivially_relocatable.h> #include <__type_traits/remove_cv.h> #include <__utility/empty.h> #include <__utility/integer_sequence.h> @@ -166,6 +167,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD template struct _LIBCPP_TEMPLATE_VIS array { + using __trivially_relocatable = __conditional_t<__libcpp_is_trivially_relocatable<_Tp>::value, array, void>; + // types: using __self = array; using value_type = _Tp; diff --git a/libcxx/include/deque b/libcxx/include/deque index 8ffd8193b74e..051bb6551212 100644 --- a/libcxx/include/deque +++ b/libcxx/include/deque @@ -477,6 +477,16 @@ public: using reverse_iterator = std::reverse_iterator; using const_reverse_iterator = std::reverse_iterator; + // A deque contains the following members which may be trivially relocatable: + // - __map: is a `__split_buffer`, see `__split_buffer` for more information on when it is trivially relocatable + // - size_type: is always trivially relocatable, since it is required to be an integral type + // - allocator_type: may not be trivially relocatable, so it's checked + // None of these are referencing the `deque` itself, so if all of them are trivially relocatable, `deque` is too. + using __trivially_relocatable = __conditional_t< + __libcpp_is_trivially_relocatable<__map>::value && __libcpp_is_trivially_relocatable::value, + deque, + void>; + static_assert(is_same >::value, "[allocator.requirements] states that rebinding an allocator to the same type should result in the " "original allocator"); diff --git a/libcxx/include/optional b/libcxx/include/optional index 622e150f7a9f..45e0987623f2 100644 --- a/libcxx/include/optional +++ b/libcxx/include/optional @@ -209,6 +209,7 @@ namespace std { #include <__type_traits/is_trivially_assignable.h> #include <__type_traits/is_trivially_constructible.h> #include <__type_traits/is_trivially_destructible.h> +#include <__type_traits/is_trivially_relocatable.h> #include <__type_traits/negation.h> #include <__type_traits/remove_const.h> #include <__type_traits/remove_cvref.h> @@ -580,6 +581,8 @@ class _LIBCPP_DECLSPEC_EMPTY_BASES optional public: using value_type = _Tp; + using __trivially_relocatable = conditional_t<__libcpp_is_trivially_relocatable<_Tp>::value, optional, void>; + private: // Disable the reference extension using this static assert. static_assert(!is_same_v<__remove_cvref_t, in_place_t>, diff --git a/libcxx/include/tuple b/libcxx/include/tuple index e7b43af7d13c..e614e37cfaec 100644 --- a/libcxx/include/tuple +++ b/libcxx/include/tuple @@ -240,6 +240,7 @@ template #include <__type_traits/is_reference.h> #include <__type_traits/is_same.h> #include <__type_traits/is_swappable.h> +#include <__type_traits/is_trivially_relocatable.h> #include <__type_traits/lazy.h> #include <__type_traits/maybe_const.h> #include <__type_traits/nat.h> @@ -537,6 +538,8 @@ class _LIBCPP_TEMPLATE_VIS tuple { get(const tuple<_Up...>&&) _NOEXCEPT; public: + using __trivially_relocatable = __conditional_t<_And<__libcpp_is_trivially_relocatable<_Tp>...>::value, tuple, void>; + // [tuple.cnstr] // tuple() constructors (including allocator_arg_t variants) diff --git a/libcxx/include/variant b/libcxx/include/variant index 7ebd0534b164..de451729768b 100644 --- a/libcxx/include/variant +++ b/libcxx/include/variant @@ -241,6 +241,7 @@ namespace std { #include <__type_traits/is_trivially_assignable.h> #include <__type_traits/is_trivially_constructible.h> #include <__type_traits/is_trivially_destructible.h> +#include <__type_traits/is_trivially_relocatable.h> #include <__type_traits/is_void.h> #include <__type_traits/remove_const.h> #include <__type_traits/remove_cvref.h> @@ -1180,6 +1181,9 @@ class _LIBCPP_TEMPLATE_VIS _LIBCPP_DECLSPEC_EMPTY_BASES variant using __first_type = variant_alternative_t<0, variant>; public: + using __trivially_relocatable = + conditional_t<_And<__libcpp_is_trivially_relocatable<_Types>...>::value, variant, void>; + template , _Dummy>::value, int> = 0> _LIBCPP_HIDE_FROM_ABI constexpr variant() noexcept(is_nothrow_default_constructible_v<__first_type>) diff --git a/libcxx/include/vector b/libcxx/include/vector index cbfc2cefa1fd..fb03f77dbe76 100644 --- a/libcxx/include/vector +++ b/libcxx/include/vector @@ -406,6 +406,15 @@ public: typedef std::reverse_iterator reverse_iterator; typedef std::reverse_iterator const_reverse_iterator; + // A vector containers the following members which may be trivially relocatable: + // - pointer: may be trivially relocatable, so it's checked + // - allocator_type: may be trivially relocatable, so it's checked + // vector doesn't contain any self-references, so it's trivially relocatable if its members are. + using __trivially_relocatable = __conditional_t< + __libcpp_is_trivially_relocatable::value && __libcpp_is_trivially_relocatable::value, + vector, + void>; + static_assert((is_same::value), "Allocator::value_type must be same type as value_type"); diff --git a/libcxx/test/libcxx/transitive_includes/cxx03.csv b/libcxx/test/libcxx/transitive_includes/cxx03.csv index b0431d9b2b55..c720a0c9a05e 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx03.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx03.csv @@ -903,6 +903,7 @@ thread type_traits thread version tuple compare tuple cstddef +tuple cstdint tuple exception tuple iosfwd tuple new diff --git a/libcxx/test/libcxx/transitive_includes/cxx11.csv b/libcxx/test/libcxx/transitive_includes/cxx11.csv index 6fc8fe5858c0..b00436a1be7f 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx11.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx11.csv @@ -910,6 +910,7 @@ thread type_traits thread version tuple compare tuple cstddef +tuple cstdint tuple exception tuple iosfwd tuple new diff --git a/libcxx/test/libcxx/transitive_includes/cxx14.csv b/libcxx/test/libcxx/transitive_includes/cxx14.csv index 5771e2ba0761..4b8c12929de0 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx14.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx14.csv @@ -913,6 +913,7 @@ thread type_traits thread version tuple compare tuple cstddef +tuple cstdint tuple exception tuple iosfwd tuple new diff --git a/libcxx/test/libcxx/transitive_includes/cxx17.csv b/libcxx/test/libcxx/transitive_includes/cxx17.csv index deba8c03d2f6..a51a7e2de1d3 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx17.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx17.csv @@ -914,6 +914,7 @@ thread type_traits thread version tuple compare tuple cstddef +tuple cstdint tuple exception tuple iosfwd tuple new diff --git a/libcxx/test/libcxx/transitive_includes/cxx20.csv b/libcxx/test/libcxx/transitive_includes/cxx20.csv index b76ec5cf1990..5280d75cbf2e 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx20.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx20.csv @@ -920,6 +920,7 @@ thread type_traits thread version tuple compare tuple cstddef +tuple cstdint tuple exception tuple iosfwd tuple new diff --git a/libcxx/test/libcxx/transitive_includes/cxx23.csv b/libcxx/test/libcxx/transitive_includes/cxx23.csv index bd0ba126d407..bfb8b5def622 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx23.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx23.csv @@ -151,6 +151,7 @@ exception typeinfo exception version execution version expected cstddef +expected cstdint expected initializer_list expected new expected version @@ -484,6 +485,7 @@ regex typeinfo regex vector regex version scoped_allocator cstddef +scoped_allocator cstdint scoped_allocator limits scoped_allocator new scoped_allocator tuple @@ -640,6 +642,7 @@ thread typeinfo thread version tuple compare tuple cstddef +tuple cstdint tuple version type_traits cstddef type_traits cstdint @@ -674,6 +677,7 @@ unordered_set tuple unordered_set version utility compare utility cstddef +utility cstdint utility initializer_list utility limits utility version diff --git a/libcxx/test/libcxx/transitive_includes/cxx26.csv b/libcxx/test/libcxx/transitive_includes/cxx26.csv index bd0ba126d407..bfb8b5def622 100644 --- a/libcxx/test/libcxx/transitive_includes/cxx26.csv +++ b/libcxx/test/libcxx/transitive_includes/cxx26.csv @@ -151,6 +151,7 @@ exception typeinfo exception version execution version expected cstddef +expected cstdint expected initializer_list expected new expected version @@ -484,6 +485,7 @@ regex typeinfo regex vector regex version scoped_allocator cstddef +scoped_allocator cstdint scoped_allocator limits scoped_allocator new scoped_allocator tuple @@ -640,6 +642,7 @@ thread typeinfo thread version tuple compare tuple cstddef +tuple cstdint tuple version type_traits cstddef type_traits cstdint @@ -674,6 +677,7 @@ unordered_set tuple unordered_set version utility compare utility cstddef +utility cstdint utility initializer_list utility limits utility version diff --git a/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp b/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp index 4d1a8ad9e229..64d2c54e7dd8 100644 --- a/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp +++ b/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp @@ -7,11 +7,24 @@ //===----------------------------------------------------------------------===// #include <__type_traits/is_trivially_relocatable.h> +#include +#include +#include +#include #include +#include #include +#include +#include +#include #include "constexpr_char_traits.h" #include "test_allocator.h" +#include "test_macros.h" + +#ifndef TEST_HAS_NO_LOCALIZATION +# include +#endif static_assert(std::__libcpp_is_trivially_relocatable::value, ""); static_assert(std::__libcpp_is_trivially_relocatable::value, ""); @@ -44,9 +57,27 @@ static_assert(std::__libcpp_is_trivially_relocatable: #else static_assert(!std::__libcpp_is_trivially_relocatable::value, ""); #endif + +// library-internal types +// ---------------------- + +// __split_buffer +static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(!std::__libcpp_is_trivially_relocatable > >::value, ""); + // standard library types // ---------------------- +// array +static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable, 0> >::value, ""); + +static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable, 1> >::value, ""); + // basic_string #if defined(_LIBCPP_HAS_NO_ASAN) || !defined(_LIBCPP_INSTRUMENTED_WITH_ASAN) struct MyChar { @@ -80,6 +111,71 @@ static_assert( std::basic_string, test_allocator > >::value, ""); #endif + +// deque +static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(!std::__libcpp_is_trivially_relocatable > >::value, ""); + +// exception_ptr +#ifndef _LIBCPP_ABI_MICROSOFT // FIXME: Is this also the case on windows? +static_assert(std::__libcpp_is_trivially_relocatable::value, ""); +#endif + +// expected +#if TEST_STD_VER >= 23 +static_assert(std::__libcpp_is_trivially_relocatable >::value); +static_assert(std::__libcpp_is_trivially_relocatable, int>>::value); +static_assert(std::__libcpp_is_trivially_relocatable>>::value); +static_assert(std::__libcpp_is_trivially_relocatable, std::unique_ptr>>::value); + +static_assert(!std::__libcpp_is_trivially_relocatable>::value); +static_assert(!std::__libcpp_is_trivially_relocatable>::value); +static_assert( + !std::__libcpp_is_trivially_relocatable>::value); +#endif + +// locale +#ifndef TEST_HAS_NO_LOCALIZATION +static_assert(std::__libcpp_is_trivially_relocatable::value, ""); +#endif + +// optional +#if TEST_STD_VER >= 17 +static_assert(std::__libcpp_is_trivially_relocatable>::value, ""); +static_assert(!std::__libcpp_is_trivially_relocatable>::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable>>::value, ""); +#endif // TEST_STD_VER >= 17 + +// pair +static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(!std::__libcpp_is_trivially_relocatable >::value, + ""); +static_assert(std::__libcpp_is_trivially_relocatable, std::unique_ptr > >::value, + ""); + +// shared_ptr +static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); + +// tuple +#if TEST_STD_VER >= 11 +static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); + +static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable > >::value, ""); + +static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(!std::__libcpp_is_trivially_relocatable >::value, + ""); +static_assert(std::__libcpp_is_trivially_relocatable, std::unique_ptr > >::value, + ""); +#endif // TEST_STD_VER >= 11 + // unique_ptr struct NotTriviallyRelocatableDeleter { NotTriviallyRelocatableDeleter(const NotTriviallyRelocatableDeleter&); @@ -113,4 +209,27 @@ static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); +// variant +#if TEST_STD_VER >= 17 +static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable > >::value, ""); + +static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(!std::__libcpp_is_trivially_relocatable >::value, + ""); +static_assert(std::__libcpp_is_trivially_relocatable, std::unique_ptr > >::value, + ""); +#endif // TEST_STD_VER >= 17 + +// vector +static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(!std::__libcpp_is_trivially_relocatable > >::value, ""); + +// weak_ptr +static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); + // TODO: Mark all the trivially relocatable STL types as such