25#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_H
26#define _GLIBCXX_EXPERIMENTAL_SIMD_H
28#if __cplusplus >= 201703L
30#include "simd_detail.h"
34#ifdef _GLIBCXX_DEBUG_UB
44#if _GLIBCXX_SIMD_X86INTRIN
46#elif _GLIBCXX_SIMD_HAVE_NEON
47#pragma GCC diagnostic push
50#pragma GCC diagnostic ignored "-Wnarrowing"
52#pragma GCC diagnostic pop
54#if _GLIBCXX_SIMD_HAVE_SVE
81_GLIBCXX_SIMD_BEGIN_NAMESPACE
83#if !_GLIBCXX_SIMD_X86INTRIN
84using __m128 [[__gnu__::__vector_size__(16)]] = float;
85using __m128d [[__gnu__::__vector_size__(16)]] = double;
86using __m128i [[__gnu__::__vector_size__(16)]] =
long long;
87using __m256 [[__gnu__::__vector_size__(32)]] = float;
88using __m256d [[__gnu__::__vector_size__(32)]] = double;
89using __m256i [[__gnu__::__vector_size__(32)]] =
long long;
90using __m512 [[__gnu__::__vector_size__(64)]] = float;
91using __m512d [[__gnu__::__vector_size__(64)]] = double;
92using __m512i [[__gnu__::__vector_size__(64)]] =
long long;
95#if _GLIBCXX_SIMD_HAVE_SVE
96constexpr inline int __sve_vectorized_size_bytes = __ARM_FEATURE_SVE_BITS / 8;
98constexpr inline int __sve_vectorized_size_bytes = 0;
120template <
int _UsedBytes>
123template <
int _UsedBytes>
124 struct _VecBltnBtmsk;
126template <
int _UsedBytes,
int _TotalBytes = __sve_vectorized_size_
bytes>
129template <
typename _Tp,
int _Np>
130 using _VecN = _VecBuiltin<
sizeof(_Tp) * _Np>;
132template <
int _UsedBytes = 16>
133 using _Sse = _VecBuiltin<_UsedBytes>;
135template <
int _UsedBytes = 32>
136 using _Avx = _VecBuiltin<_UsedBytes>;
138template <
int _UsedBytes = 64>
139 using _Avx512 = _VecBltnBtmsk<_UsedBytes>;
141template <
int _UsedBytes = 16>
142 using _Neon = _VecBuiltin<_UsedBytes>;
144template <
int _UsedBytes = __sve_vectorized_size_
bytes>
145 using _Sve = _SveAbi<_UsedBytes, __sve_vectorized_size_bytes>;
150using __avx512 = _Avx512<>;
151using __neon = _Neon<>;
152using __neon128 = _Neon<16>;
153using __neon64 = _Neon<8>;
157template <
typename _Tp,
size_t _Np,
typename...>
161 using fixed_size = _Fixed<_Np>;
163using scalar = _Scalar;
168template <
typename _Tp>
171template <
typename _Tp>
174template <
typename _Tp,
typename _Abi>
177template <
typename _Tp,
typename _Abi>
180template <
typename _Tp,
typename _Abi>
185struct element_aligned_tag
187 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
188 static constexpr size_t _S_alignment =
alignof(_Up);
190 template <
typename _Tp,
typename _Up>
191 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
196struct vector_aligned_tag
198 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
199 static constexpr size_t _S_alignment
200 = std::__bit_ceil(
sizeof(_Up) * _Tp::size());
202 template <
typename _Tp,
typename _Up>
203 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
205 {
return static_cast<_Up*
>(__builtin_assume_aligned(__ptr, _S_alignment<_Tp, _Up>)); }
208template <
size_t _Np>
struct overaligned_tag
210 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
211 static constexpr size_t _S_alignment = _Np;
213 template <
typename _Tp,
typename _Up>
214 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
216 {
return static_cast<_Up*
>(__builtin_assume_aligned(__ptr, _Np)); }
219inline constexpr element_aligned_tag element_aligned = {};
221inline constexpr vector_aligned_tag vector_aligned = {};
224 inline constexpr overaligned_tag<_Np> overaligned = {};
228 using _SizeConstant = integral_constant<size_t, _Xp>;
230constexpr inline bool __have_mmx = _GLIBCXX_SIMD_HAVE_MMX;
231constexpr inline bool __have_sse = _GLIBCXX_SIMD_HAVE_SSE;
232constexpr inline bool __have_sse2 = _GLIBCXX_SIMD_HAVE_SSE2;
233constexpr inline bool __have_sse3 = _GLIBCXX_SIMD_HAVE_SSE3;
234constexpr inline bool __have_ssse3 = _GLIBCXX_SIMD_HAVE_SSSE3;
235constexpr inline bool __have_sse4_1 = _GLIBCXX_SIMD_HAVE_SSE4_1;
236constexpr inline bool __have_sse4_2 = _GLIBCXX_SIMD_HAVE_SSE4_2;
237constexpr inline bool __have_xop = _GLIBCXX_SIMD_HAVE_XOP;
238constexpr inline bool __have_avx = _GLIBCXX_SIMD_HAVE_AVX;
239constexpr inline bool __have_avx2 = _GLIBCXX_SIMD_HAVE_AVX2;
240constexpr inline bool __have_bmi = _GLIBCXX_SIMD_HAVE_BMI1;
241constexpr inline bool __have_bmi2 = _GLIBCXX_SIMD_HAVE_BMI2;
242constexpr inline bool __have_lzcnt = _GLIBCXX_SIMD_HAVE_LZCNT;
243constexpr inline bool __have_sse4a = _GLIBCXX_SIMD_HAVE_SSE4A;
244constexpr inline bool __have_fma = _GLIBCXX_SIMD_HAVE_FMA;
245constexpr inline bool __have_fma4 = _GLIBCXX_SIMD_HAVE_FMA4;
246constexpr inline bool __have_f16c = _GLIBCXX_SIMD_HAVE_F16C;
247constexpr inline bool __have_popcnt = _GLIBCXX_SIMD_HAVE_POPCNT;
248constexpr inline bool __have_avx512f = _GLIBCXX_SIMD_HAVE_AVX512F;
249constexpr inline bool __have_avx512dq = _GLIBCXX_SIMD_HAVE_AVX512DQ;
250constexpr inline bool __have_avx512vl = _GLIBCXX_SIMD_HAVE_AVX512VL;
251constexpr inline bool __have_avx512bw = _GLIBCXX_SIMD_HAVE_AVX512BW;
252constexpr inline bool __have_avx512dq_vl = __have_avx512dq && __have_avx512vl;
253constexpr inline bool __have_avx512bw_vl = __have_avx512bw && __have_avx512vl;
254constexpr inline bool __have_avx512bitalg = _GLIBCXX_SIMD_HAVE_AVX512BITALG;
255constexpr inline bool __have_avx512vbmi2 = _GLIBCXX_SIMD_HAVE_AVX512VBMI2;
256constexpr inline bool __have_avx512vbmi = _GLIBCXX_SIMD_HAVE_AVX512VBMI;
257constexpr inline bool __have_avx512ifma = _GLIBCXX_SIMD_HAVE_AVX512IFMA;
258constexpr inline bool __have_avx512cd = _GLIBCXX_SIMD_HAVE_AVX512CD;
259constexpr inline bool __have_avx512vnni = _GLIBCXX_SIMD_HAVE_AVX512VNNI;
260constexpr inline bool __have_avx512vpopcntdq = _GLIBCXX_SIMD_HAVE_AVX512VPOPCNTDQ;
261constexpr inline bool __have_avx512vp2intersect = _GLIBCXX_SIMD_HAVE_AVX512VP2INTERSECT;
263constexpr inline bool __have_neon = _GLIBCXX_SIMD_HAVE_NEON;
264constexpr inline bool __have_neon_a32 = _GLIBCXX_SIMD_HAVE_NEON_A32;
265constexpr inline bool __have_neon_a64 = _GLIBCXX_SIMD_HAVE_NEON_A64;
266constexpr inline bool __support_neon_float =
267#if defined __GCC_IEC_559
269#elif defined __FAST_MATH__
275constexpr inline bool __have_sve = _GLIBCXX_SIMD_HAVE_SVE;
276constexpr inline bool __have_sve2 = _GLIBCXX_SIMD_HAVE_SVE2;
279constexpr inline bool __have_power10vec =
true;
281constexpr inline bool __have_power10vec =
false;
283#ifdef __POWER9_VECTOR__
284constexpr inline bool __have_power9vec =
true;
286constexpr inline bool __have_power9vec =
false;
288#if defined __POWER8_VECTOR__
289constexpr inline bool __have_power8vec =
true;
291constexpr inline bool __have_power8vec = __have_power9vec;
294constexpr inline bool __have_power_vsx =
true;
296constexpr inline bool __have_power_vsx = __have_power8vec;
298#if defined __ALTIVEC__
299constexpr inline bool __have_power_vmx =
true;
301constexpr inline bool __have_power_vmx = __have_power_vsx;
308#ifdef math_errhandling
312 template <
int = math_errhandling>
314 __handle_fpexcept_impl(
int)
315 {
return math_errhandling & MATH_ERREXCEPT; }
321 __handle_fpexcept_impl(
float)
323#if defined __FAST_MATH__
331 static constexpr bool _S_handle_fpexcept = __handle_fpexcept_impl(0);
333 constexpr std::uint_least64_t
334 __floating_point_flags()
336 std::uint_least64_t __flags = 0;
337 if constexpr (_S_handle_fpexcept)
341#elif __FINITE_MATH_ONLY__
343#elif __GCC_IEC_559 < 2
346 __flags |= (__FLT_EVAL_METHOD__ + 1) << 3;
350 constexpr std::uint_least64_t
353 if constexpr (__have_mmx || __have_sse)
358 | (__have_ssse3 << 4)
359 | (__have_sse4_1 << 5)
360 | (__have_sse4_2 << 6)
365 | (__have_bmi2 << 11)
366 | (__have_lzcnt << 12)
367 | (__have_sse4a << 13)
369 | (__have_fma4 << 15)
370 | (__have_f16c << 16)
371 | (__have_popcnt << 17)
372 | (__have_avx512f << 18)
373 | (__have_avx512dq << 19)
374 | (__have_avx512vl << 20)
375 | (__have_avx512bw << 21)
376 | (__have_avx512bitalg << 22)
377 | (__have_avx512vbmi2 << 23)
378 | (__have_avx512vbmi << 24)
379 | (__have_avx512ifma << 25)
380 | (__have_avx512cd << 26)
381 | (__have_avx512vnni << 27)
382 | (__have_avx512vpopcntdq << 28)
383 | (__have_avx512vp2intersect << 29);
384 else if constexpr (__have_neon || __have_sve)
386 | (__have_neon_a32 << 1)
387 | (__have_neon_a64 << 2)
388 | (__have_neon_a64 << 2)
389 | (__support_neon_float << 3)
391 | (__have_sve2 << 5);
392 else if constexpr (__have_power_vmx)
393 return __have_power_vmx
394 | (__have_power_vsx << 1)
395 | (__have_power8vec << 2)
396 | (__have_power9vec << 3)
397 | (__have_power10vec << 4);
404 struct _OdrEnforcer {};
407 template <std::uint_least64_t...>
408 struct _MachineFlagsTemplate {};
421 _MachineFlagsTemplate<__machine_flags(), __floating_point_flags()>>;
425 template <
typename _Tp>
426 _GLIBCXX_SIMD_INTRINSIC
constexpr
428 operator()(_Tp __a, _Tp __b)
const
431 return min(__a, __b);
437 template <
typename _Tp>
438 _GLIBCXX_SIMD_INTRINSIC
constexpr
440 operator()(_Tp __a, _Tp __b)
const
443 return max(__a, __b);
450template <
typename _Fp,
size_t... _I>
451 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC
constexpr
453 __execute_on_index_sequence(_Fp&& __f, index_sequence<_I...>)
454 { ((void)__f(_SizeConstant<_I>()), ...); }
456template <
typename _Fp>
457 _GLIBCXX_SIMD_INTRINSIC
constexpr void
458 __execute_on_index_sequence(_Fp&&, index_sequence<>)
461template <
size_t _Np,
typename _Fp>
462 _GLIBCXX_SIMD_INTRINSIC
constexpr void
463 __execute_n_times(_Fp&& __f)
465 __execute_on_index_sequence(
static_cast<_Fp&&
>(__f),
466 make_index_sequence<_Np>{});
471template <
typename _R,
typename _Fp,
size_t... _I>
472 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC
constexpr
474 __execute_on_index_sequence_with_return(_Fp&& __f, index_sequence<_I...>)
475 {
return _R{__f(_SizeConstant<_I>())...}; }
477template <
size_t _Np,
typename _R,
typename _Fp>
478 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
479 __generate_from_n_evaluations(_Fp&& __f)
481 return __execute_on_index_sequence_with_return<_R>(
482 static_cast<_Fp&&
>(__f), make_index_sequence<_Np>{});
487template <
size_t... _I,
typename _F0,
typename _FArgs>
488 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC
constexpr
490 __call_with_n_evaluations(index_sequence<_I...>, _F0&& __f0, _FArgs&& __fargs)
491 {
return __f0(__fargs(_SizeConstant<_I>())...); }
493template <
size_t _Np,
typename _F0,
typename _FArgs>
494 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
495 __call_with_n_evaluations(_F0&& __f0, _FArgs&& __fargs)
497 return __call_with_n_evaluations(make_index_sequence<_Np>{},
498 static_cast<_F0&&
>(__f0),
499 static_cast<_FArgs&&
>(__fargs));
504template <
size_t _First = 0,
size_t... _It,
typename _Tp,
typename _Fp>
505 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC
constexpr
507 __call_with_subscripts(_Tp&& __x, index_sequence<_It...>, _Fp&& __fun)
508 {
return __fun(__x[_First + _It]...); }
510template <
size_t _Np,
size_t _First = 0,
typename _Tp,
typename _Fp>
511 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
512 __call_with_subscripts(_Tp&& __x, _Fp&& __fun)
514 return __call_with_subscripts<_First>(
static_cast<_Tp&&
>(__x),
515 make_index_sequence<_Np>(),
516 static_cast<_Fp&&
>(__fun));
523using _UChar =
unsigned char;
524using _SChar =
signed char;
525using _UShort =
unsigned short;
526using _UInt =
unsigned int;
527using _ULong =
unsigned long;
528using _ULLong =
unsigned long long;
529using _LLong =
long long;
533template <
typename _T0,
typename...>
534 struct __first_of_pack
535 {
using type = _T0; };
537template <
typename... _Ts>
538 using __first_of_pack_t =
typename __first_of_pack<_Ts...>::type;
542template <
typename _Tp>
543 typename _Tp::value_type
544 __value_type_or_identity_impl(
int);
546template <
typename _Tp>
548 __value_type_or_identity_impl(
float);
550template <
typename _Tp>
551 using __value_type_or_identity_t
552 =
decltype(__value_type_or_identity_impl<_Tp>(
int()));
556template <
typename _Tp>
557 struct __is_vectorizable :
public is_arithmetic<_Tp> {};
560 struct __is_vectorizable<bool> :
public false_type {};
562template <
typename _Tp>
563 inline constexpr bool __is_vectorizable_v = __is_vectorizable<_Tp>::value;
566template <
typename _Tp,
typename = enable_if_t<__is_vectorizable_v<_Tp>>>
567 using _Vectorizable = _Tp;
571template <
typename _Ptr,
typename _ValueType>
572 struct __is_possible_loadstore_conversion
573 : conjunction<__is_vectorizable<_Ptr>, __is_vectorizable<_ValueType>> {};
576 struct __is_possible_loadstore_conversion<bool, bool> :
true_type {};
579template <
typename _Ptr,
typename _ValueType,
581 __is_possible_loadstore_conversion<_Ptr, _ValueType>::value>>
582 using _LoadStorePtr = _Ptr;
586template <
typename _Tp,
typename =
void_t<>>
589template <
typename _Tp>
590 inline constexpr bool __is_bitmask_v = __is_bitmask<_Tp>::value;
593template <
typename _Tp>
594 struct __is_bitmask<_Tp,
595 void_t<decltype(
declval<unsigned&>() = declval<_Tp>() & 1u)>>
600#pragma GCC diagnostic push
601#pragma GCC diagnostic ignored "-Wpedantic"
602template <
size_t _Bytes>
606 static_assert(_Bytes > 0);
607 if constexpr (_Bytes ==
sizeof(int))
610 else if constexpr (_Bytes ==
sizeof(char))
613 else if constexpr (_Bytes ==
sizeof(_SChar))
616 else if constexpr (_Bytes ==
sizeof(short))
619 else if constexpr (_Bytes ==
sizeof(long))
622 else if constexpr (_Bytes ==
sizeof(_LLong))
624 #ifdef __SIZEOF_INT128__
625 else if constexpr (_Bytes ==
sizeof(__int128))
628 else if constexpr (_Bytes %
sizeof(int) == 0)
630 constexpr size_t _Np = _Bytes /
sizeof(int);
635 _GLIBCXX_SIMD_INTRINSIC
constexpr _Ip
638 return __generate_from_n_evaluations<_Np, _Ip>(
639 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
640 return __rhs._M_data[__i] & _M_data[__i];
644 _GLIBCXX_SIMD_INTRINSIC
constexpr _Ip
647 return __generate_from_n_evaluations<_Np, _Ip>(
648 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
649 return __rhs._M_data[__i] | _M_data[__i];
653 _GLIBCXX_SIMD_INTRINSIC
constexpr _Ip
656 return __generate_from_n_evaluations<_Np, _Ip>(
657 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
658 return __rhs._M_data[__i] ^ _M_data[__i];
662 _GLIBCXX_SIMD_INTRINSIC
constexpr _Ip
665 return __generate_from_n_evaluations<_Np, _Ip>(
666 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
return ~_M_data[__i]; });
672 static_assert(_Bytes == 0,
"this should be unreachable");
674#pragma GCC diagnostic pop
676template <
typename _Tp>
677 using __int_for_sizeof_t =
decltype(__int_for_sizeof<sizeof(_Tp)>());
680 using __int_with_sizeof_t =
decltype(__int_for_sizeof<_Np>());
684template <
typename _Tp>
688 struct __is_fixed_size_abi<simd_abi::fixed_size<_Np>> :
true_type {};
690template <
typename _Tp>
691 inline constexpr bool __is_fixed_size_abi_v = __is_fixed_size_abi<_Tp>::value;
695template <
typename _Abi>
698 {
return is_same_v<simd_abi::scalar, _Abi>; }
702template <
template <
int>
class _Abi,
int _Bytes>
704 __abi_bytes_impl(_Abi<_Bytes>*)
707template <
typename _Tp>
709 __abi_bytes_impl(_Tp*)
712template <
typename _Abi>
713 inline constexpr int __abi_bytes_v
714 = __abi_bytes_impl(
static_cast<_Abi*
>(
nullptr));
718template <
typename _Abi>
720 __is_builtin_bitmask_abi()
721 {
return is_same_v<simd_abi::_VecBltnBtmsk<__abi_bytes_v<_Abi>>, _Abi>; }
725template <
typename _Abi>
729 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
730 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
735template <
typename _Abi>
739 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
740 return _Bytes > 16 && _Bytes <= 32
741 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
746template <
typename _Abi>
750 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
751 return _Bytes <= 64 && is_same_v<simd_abi::_Avx512<_Bytes>, _Abi>;
756template <
typename _Abi>
760 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
761 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
766template <
typename _Abi>
770 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
771 return _Bytes <= __sve_vectorized_size_bytes && is_same_v<simd_abi::_Sve<_Bytes>, _Abi>;
776template <
typename,
typename _Up>
777 struct __make_dependent
778 {
using type = _Up; };
780template <
typename _Tp,
typename _Up>
781 using __make_dependent_t =
typename __make_dependent<_Tp, _Up>::type;
787template <
typename... _Args>
788 [[noreturn]] _GLIBCXX_SIMD_ALWAYS_INLINE
void
789 __invoke_ub([[maybe_unused]]
const char* __msg, [[maybe_unused]]
const _Args&... __args)
791#ifdef _GLIBCXX_DEBUG_UB
792 __builtin_fprintf(stderr, __msg, __args...);
795 __builtin_unreachable();
801template <
typename _Tp>
802 struct __assert_unreachable
803 {
static_assert(!is_same_v<_Tp, _Tp>,
"this should be unreachable"); };
807template <typename _Tp, typename _Ap, size_t _Np = simd_size<_Tp, _Ap>::value>
809 __size_or_zero_dispatch(
int)
812template <
typename _Tp,
typename _Ap>
814 __size_or_zero_dispatch(
float)
817template <
typename _Tp,
typename _Ap>
818 inline constexpr size_t __size_or_zero_v
819 = __size_or_zero_dispatch<_Tp, _Ap>(0);
823inline constexpr size_t
824__div_roundup(
size_t __a,
size_t __b)
825{
return (__a + __b - 1) / __b; }
834 _GLIBCXX_SIMD_INTRINSIC
constexpr
835 _ExactBool(
bool __b) : _M_data(__b) {}
837 _ExactBool(
int) =
delete;
839 _GLIBCXX_SIMD_INTRINSIC
constexpr
840 operator bool()
const
851template <
typename _Tp>
852 using __may_alias [[__gnu__::__may_alias__]] = _Tp;
857struct _UnsupportedBase
859 _UnsupportedBase() =
delete;
860 _UnsupportedBase(
const _UnsupportedBase&) =
delete;
861 _UnsupportedBase& operator=(
const _UnsupportedBase&) =
delete;
862 ~_UnsupportedBase() =
delete;
878 using _SimdBase = _UnsupportedBase;
879 using _MaskBase = _UnsupportedBase;
881 static constexpr size_t _S_full_size = 0;
882 static constexpr bool _S_is_partial =
false;
884 static constexpr size_t _S_simd_align = 1;
886 struct _SimdMember {};
887 struct _SimdCastType;
889 static constexpr size_t _S_mask_align = 1;
891 struct _MaskMember {};
892 struct _MaskCastType;
897template <
typename _Tp,
typename _Abi,
typename =
void_t<>>
898 struct _SimdTraits : _InvalidTraits {};
906inline constexpr struct _PrivateInit {} __private_init = {};
908inline constexpr struct _BitsetInit {} __bitset_init = {};
912template <
typename _From,
typename _To,
bool = is_arithmetic_v<_From>,
913 bool = is_arithmetic_v<_To>>
914 struct __is_narrowing_conversion;
918template <
typename _From,
typename _To>
919 struct __is_narrowing_conversion<_From, _To, true, true>
920 :
public __bool_constant<(
921 __digits_v<_From> > __digits_v<_To>
922 || __finite_max_v<_From> > __finite_max_v<_To>
923 || __finite_min_v<_From> < __finite_min_v<_To>
924 || (is_signed_v<_From> && is_unsigned_v<_To>))> {};
926template <
typename _Tp>
927 struct __is_narrowing_conversion<_Tp, bool, true, true>
931 struct __is_narrowing_conversion<bool, bool, true, true>
934template <
typename _Tp>
935 struct __is_narrowing_conversion<_Tp, _Tp, true, true>
938template <
typename _From,
typename _To>
939 struct __is_narrowing_conversion<_From, _To, false, true>
940 :
public negation<is_convertible<_From, _To>> {};
944template <
typename _From,
typename _To,
bool = (sizeof(_From) < sizeof(_To))>
945 struct __converts_to_higher_
integer_rank : public true_type {};
948template <
typename _From,
typename _To>
949 struct __converts_to_higher_integer_rank<_From, _To, false>
950 :
public is_same<decltype(declval<_From>() + declval<_To>()), _To> {};
954template <
typename _Tp,
typename _Ap>
955 _GLIBCXX_SIMD_INTRINSIC
constexpr const auto&
956 __data(
const simd<_Tp, _Ap>& __x);
958template <
typename _Tp,
typename _Ap>
959 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
960 __data(simd<_Tp, _Ap>& __x);
962template <
typename _Tp,
typename _Ap>
963 _GLIBCXX_SIMD_INTRINSIC
constexpr const auto&
964 __data(
const simd_mask<_Tp, _Ap>& __x);
966template <
typename _Tp,
typename _Ap>
967 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
968 __data(simd_mask<_Tp, _Ap>& __x);
972template <
typename _FromT,
typename _FromA,
typename _ToT,
typename _ToA,
974 struct _SimdConverter;
976template <
typename _Tp,
typename _Ap>
977 struct _SimdConverter<_Tp, _Ap, _Tp, _Ap, void>
979 template <
typename _Up>
980 _GLIBCXX_SIMD_INTRINSIC
const _Up&
981 operator()(
const _Up& __x)
987template <
typename _V>
988 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
989 __to_value_type_or_member_type(
const _V& __x) ->
decltype(__data(__x))
990 {
return __data(__x); }
992template <
typename _V>
993 _GLIBCXX_SIMD_INTRINSIC
constexpr const typename _V::value_type&
994 __to_value_type_or_member_type(
const typename _V::value_type& __x)
999template <
size_t _Size>
1000 struct __bool_storage_member_type;
1002template <
size_t _Size>
1003 using __bool_storage_member_type_t =
1004 typename __bool_storage_member_type<_Size>::type;
1014template <
typename _Tp,
typename... _Abis>
1019template <
typename _Tp,
int _Np>
1020 struct __fixed_size_storage;
1022template <
typename _Tp,
int _Np>
1023 using __fixed_size_storage_t =
typename __fixed_size_storage<_Tp, _Np>::type;
1027template <
typename _Tp,
size_t _Size,
typename =
void_t<>>
1028 struct _SimdWrapper;
1030template <
typename _Tp>
1031 using _SimdWrapper8 = _SimdWrapper<_Tp, 8 /
sizeof(_Tp)>;
1032template <
typename _Tp>
1033 using _SimdWrapper16 = _SimdWrapper<_Tp, 16 /
sizeof(_Tp)>;
1034template <
typename _Tp>
1035 using _SimdWrapper32 = _SimdWrapper<_Tp, 32 /
sizeof(_Tp)>;
1036template <
typename _Tp>
1037 using _SimdWrapper64 = _SimdWrapper<_Tp, 64 /
sizeof(_Tp)>;
1039template <
typename _Tp,
size_t _W
idth>
1040 struct _SveSimdWrapper;
1044template <
typename _Tp>
1047template <
typename _Tp,
size_t _Np>
1048 struct __is_simd_wrapper<_SimdWrapper<_Tp, _Np>> :
true_type {};
1050template <
typename _Tp>
1051 inline constexpr bool __is_simd_wrapper_v = __is_simd_wrapper<_Tp>::value;
1058 template <
typename _Tp,
typename _Fp>
1060 _S_bit_iteration(_Tp __mask, _Fp&& __f)
1062 static_assert(
sizeof(_ULLong) >=
sizeof(_Tp));
1063 conditional_t<
sizeof(_Tp) <=
sizeof(_UInt), _UInt, _ULLong> __k;
1064 if constexpr (is_convertible_v<_Tp,
decltype(__k)>)
1067 __k = __mask.to_ullong();
1070 __f(std::__countr_zero(__k));
1080template <
typename _Tp =
void>
1082 {
constexpr _Tp operator()(_Tp __a)
const {
return ++__a; } };
1085 struct __increment<void>
1087 template <
typename _Tp>
1089 operator()(_Tp __a)
const
1093template <
typename _Tp =
void>
1095 {
constexpr _Tp operator()(_Tp __a)
const {
return --__a; } };
1098 struct __decrement<void>
1100 template <
typename _Tp>
1102 operator()(_Tp __a)
const
1108template <
typename _From,
typename _To,
1110 __is_narrowing_conversion<__remove_cvref_t<_From>, _To>>::value>>
1111 using _ValuePreserving = _From;
1113template <
typename _From,
typename _To,
1114 typename _DecayedFrom = __remove_cvref_t<_From>,
1116 is_convertible<_From, _To>,
1118 is_same<_DecayedFrom, _To>, is_same<_DecayedFrom, int>,
1119 conjunction<is_same<_DecayedFrom, _UInt>, is_unsigned<_To>>,
1120 negation<__is_narrowing_conversion<_DecayedFrom, _To>>>>::value>>
1121 using _ValuePreservingOrInt = _From;
1125template <
typename _Tp,
size_t _Bytes,
typename =
void_t<>>
1126 struct __intrinsic_type;
1128template <
typename _Tp,
size_t _Size>
1129 using __intrinsic_type_t =
1130 typename __intrinsic_type<_Tp, _Size *
sizeof(_Tp)>::type;
1132template <
typename _Tp>
1133 using __intrinsic_type2_t =
typename __intrinsic_type<_Tp, 2>::type;
1134template <
typename _Tp>
1135 using __intrinsic_type4_t =
typename __intrinsic_type<_Tp, 4>::type;
1136template <
typename _Tp>
1137 using __intrinsic_type8_t =
typename __intrinsic_type<_Tp, 8>::type;
1138template <
typename _Tp>
1139 using __intrinsic_type16_t =
typename __intrinsic_type<_Tp, 16>::type;
1140template <
typename _Tp>
1141 using __intrinsic_type32_t =
typename __intrinsic_type<_Tp, 32>::type;
1142template <
typename _Tp>
1143 using __intrinsic_type64_t =
typename __intrinsic_type<_Tp, 64>::type;
1147template <
size_t _Np,
bool _Sanitized = false>
1150template <
size_t _Np,
bool _Sanitized>
1151 struct __is_bitmask<_BitMask<_Np, _Sanitized>, void> :
true_type {};
1153template <
size_t _Np>
1154 using _SanitizedBitMask = _BitMask<_Np, true>;
1156template <
size_t _Np,
bool _Sanitized>
1159 static_assert(_Np > 0);
1161 static constexpr size_t _NBytes = __div_roundup(_Np, __CHAR_BIT__);
1165 sizeof(_ULLong), std::__bit_ceil(_NBytes))>>>;
1167 static constexpr int _S_array_size = __div_roundup(_NBytes,
sizeof(_Tp));
1169 _Tp _M_bits[_S_array_size];
1171 static constexpr int _S_unused_bits
1172 = _Np == 1 ? 0 : _S_array_size *
sizeof(_Tp) * __CHAR_BIT__ - _Np;
1174 static constexpr _Tp _S_bitmask = +_Tp(~_Tp()) >> _S_unused_bits;
1176 constexpr _BitMask() noexcept = default;
1178 constexpr _BitMask(
unsigned long long __x) noexcept
1179 : _M_bits{
static_cast<_Tp
>(__x)} {}
1181 _BitMask(bitset<_Np> __x) noexcept : _BitMask(__x.to_ullong()) {}
1183 constexpr _BitMask(
const _BitMask&)
noexcept =
default;
1185 template <
bool _RhsSanitized,
typename =
enable_if_t<_RhsSanitized ==
false
1186 && _Sanitized ==
true>>
1187 constexpr _BitMask(
const _BitMask<_Np, _RhsSanitized>& __rhs) noexcept
1188 : _BitMask(__rhs._M_sanitized()) {}
1190 constexpr operator _SimdWrapper<bool, _Np>() const noexcept
1192 static_assert(_S_array_size == 1);
1198 _M_to_bits() const noexcept
1200 static_assert(_S_array_size == 1);
1205 constexpr unsigned long long
1206 to_ullong() const noexcept
1208 static_assert(_S_array_size == 1);
1213 constexpr unsigned long
1214 to_ulong() const noexcept
1216 static_assert(_S_array_size == 1);
1220 constexpr bitset<_Np>
1221 _M_to_bitset() const noexcept
1223 static_assert(_S_array_size == 1);
1227 constexpr decltype(
auto)
1228 _M_sanitized()
const noexcept
1230 if constexpr (_Sanitized)
1232 else if constexpr (_Np == 1)
1233 return _SanitizedBitMask<_Np>(_M_bits[0]);
1236 _SanitizedBitMask<_Np> __r = {};
1237 for (
int __i = 0; __i < _S_array_size; ++__i)
1238 __r._M_bits[__i] = _M_bits[__i];
1239 if constexpr (_S_unused_bits > 0)
1240 __r._M_bits[_S_array_size - 1] &= _S_bitmask;
1245 template <
size_t _Mp,
bool _LSanitized>
1246 constexpr _BitMask<_Np + _Mp, _Sanitized>
1247 _M_prepend(_BitMask<_Mp, _LSanitized> __lsb)
const noexcept
1249 constexpr size_t _RN = _Np + _Mp;
1250 using _Rp = _BitMask<_RN, _Sanitized>;
1251 if constexpr (_Rp::_S_array_size == 1)
1253 _Rp __r{{_M_bits[0]}};
1254 __r._M_bits[0] <<= _Mp;
1255 __r._M_bits[0] |= __lsb._M_sanitized()._M_bits[0];
1259 __assert_unreachable<_Rp>();
1265 template <
size_t _DropLsb,
size_t _NewSize = _Np - _DropLsb>
1267 _M_extract() const noexcept
1269 static_assert(_Np > _DropLsb);
1270 static_assert(_DropLsb + _NewSize <=
sizeof(_ULLong) * __CHAR_BIT__,
1271 "not implemented for bitmasks larger than one ullong");
1272 if constexpr (_NewSize == 1)
1274 return _SanitizedBitMask<1>(_M_bits[0] & (_Tp(1) << _DropLsb));
1276 return _BitMask<_NewSize,
1277 ((_NewSize + _DropLsb ==
sizeof(_Tp) * __CHAR_BIT__
1278 && _NewSize + _DropLsb <= _Np)
1279 || ((_Sanitized || _Np ==
sizeof(_Tp) * __CHAR_BIT__)
1280 && _NewSize + _DropLsb >= _Np))>(_M_bits[0]
1286 all() const noexcept
1288 if constexpr (_Np == 1)
1290 else if constexpr (!_Sanitized)
1291 return _M_sanitized().all();
1294 constexpr _Tp __allbits = ~_Tp();
1295 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1296 if (_M_bits[__i] != __allbits)
1298 return _M_bits[_S_array_size - 1] == _S_bitmask;
1305 any() const noexcept
1307 if constexpr (_Np == 1)
1309 else if constexpr (!_Sanitized)
1310 return _M_sanitized().any();
1313 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1314 if (_M_bits[__i] != 0)
1316 return _M_bits[_S_array_size - 1] != 0;
1322 none() const noexcept
1324 if constexpr (_Np == 1)
1326 else if constexpr (!_Sanitized)
1327 return _M_sanitized().none();
1330 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1331 if (_M_bits[__i] != 0)
1333 return _M_bits[_S_array_size - 1] == 0;
1340 count() const noexcept
1342 if constexpr (_Np == 1)
1344 else if constexpr (!_Sanitized)
1345 return _M_sanitized().none();
1348 int __result = __builtin_popcountll(_M_bits[0]);
1349 for (
int __i = 1; __i < _S_array_size; ++__i)
1350 __result += __builtin_popcountll(_M_bits[__i]);
1357 operator[](
size_t __i)
const noexcept
1359 if constexpr (_Np == 1)
1361 else if constexpr (_S_array_size == 1)
1362 return (_M_bits[0] >> __i) & 1;
1365 const size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1366 const size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1367 return (_M_bits[__j] >> __shift) & 1;
1371 template <
size_t __i>
1373 operator[](_SizeConstant<__i>)
const noexcept
1375 static_assert(__i < _Np);
1376 constexpr size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1377 constexpr size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1378 return static_cast<bool>(_M_bits[__j] & (_Tp(1) << __shift));
1383 set(
size_t __i,
bool __x)
noexcept
1385 if constexpr (_Np == 1)
1387 else if constexpr (_S_array_size == 1)
1389 _M_bits[0] &= ~_Tp(_Tp(1) << __i);
1390 _M_bits[0] |= _Tp(_Tp(__x) << __i);
1394 const size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1395 const size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1396 _M_bits[__j] &= ~_Tp(_Tp(1) << __shift);
1397 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1401 template <
size_t __i>
1403 set(_SizeConstant<__i>,
bool __x)
noexcept
1405 static_assert(__i < _Np);
1406 if constexpr (_Np == 1)
1410 constexpr size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1411 constexpr size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1412 constexpr _Tp __mask = ~_Tp(_Tp(1) << __shift);
1413 _M_bits[__j] &= __mask;
1414 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1420 operator~() const noexcept
1422 if constexpr (_Np == 1)
1426 _BitMask __result{};
1427 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1428 __result._M_bits[__i] = ~_M_bits[__i];
1429 if constexpr (_Sanitized)
1430 __result._M_bits[_S_array_size - 1]
1431 = _M_bits[_S_array_size - 1] ^ _S_bitmask;
1433 __result._M_bits[_S_array_size - 1] = ~_M_bits[_S_array_size - 1];
1439 operator^=(
const _BitMask& __b) &
noexcept
1441 __execute_n_times<_S_array_size>(
1442 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] ^= __b._M_bits[__i]; });
1447 operator|=(
const _BitMask& __b) &
noexcept
1449 __execute_n_times<_S_array_size>(
1450 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] |= __b._M_bits[__i]; });
1455 operator&=(
const _BitMask& __b) &
noexcept
1457 __execute_n_times<_S_array_size>(
1458 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] &= __b._M_bits[__i]; });
1462 friend constexpr _BitMask
1463 operator^(
const _BitMask& __a,
const _BitMask& __b)
noexcept
1470 friend constexpr _BitMask
1471 operator|(
const _BitMask& __a,
const _BitMask& __b)
noexcept
1478 friend constexpr _BitMask
1479 operator&(
const _BitMask& __a,
const _BitMask& __b)
noexcept
1486 _GLIBCXX_SIMD_INTRINSIC
1488 _M_is_constprop()
const
1490 if constexpr (_S_array_size == 0)
1491 return __builtin_constant_p(_M_bits[0]);
1494 for (
int __i = 0; __i < _S_array_size; ++__i)
1495 if (!__builtin_constant_p(_M_bits[__i]))
1506template <
typename _Tp =
void>
1507 static inline constexpr int __min_vector_size = 2 *
sizeof(_Tp);
1509#if _GLIBCXX_SIMD_HAVE_NEON
1511 inline constexpr int __min_vector_size<void> = 8;
1514 inline constexpr int __min_vector_size<void> = 16;
1519template <
typename _Tp,
size_t _Np,
typename =
void>
1520 struct __vector_type_n {};
1523template <
typename _Tp>
1524 struct __vector_type_n<_Tp, 0, void> {};
1527template <
typename _Tp>
1528 struct __vector_type_n<_Tp, 1,
enable_if_t<__is_vectorizable_v<_Tp>>>
1529 {
using type = _Tp; };
1532template <
typename _Tp,
size_t _Np>
1533 struct __vector_type_n<_Tp, _Np,
enable_if_t<__is_vectorizable_v<_Tp> && _Np >= 2>>
1535 static constexpr size_t _S_Np2 = std::__bit_ceil(_Np *
sizeof(_Tp));
1537 static constexpr size_t _S_Bytes =
1543 _S_Np2 < __min_vector_size<_Tp> ? __min_vector_size<_Tp>
1546 using type [[__gnu__::__vector_size__(_S_Bytes)]] = _Tp;
1549template <
typename _Tp,
size_t _Bytes,
size_t = _Bytes % sizeof(_Tp)>
1550 struct __vector_type;
1552template <
typename _Tp,
size_t _Bytes>
1553 struct __vector_type<_Tp, _Bytes, 0>
1554 : __vector_type_n<_Tp, _Bytes / sizeof(_Tp)> {};
1556template <
typename _Tp,
size_t _Size>
1557 using __vector_type_t =
typename __vector_type_n<_Tp, _Size>::type;
1559template <
typename _Tp>
1560 using __vector_type2_t =
typename __vector_type<_Tp, 2>::type;
1561template <
typename _Tp>
1562 using __vector_type4_t =
typename __vector_type<_Tp, 4>::type;
1563template <
typename _Tp>
1564 using __vector_type8_t =
typename __vector_type<_Tp, 8>::type;
1565template <
typename _Tp>
1566 using __vector_type16_t =
typename __vector_type<_Tp, 16>::type;
1567template <
typename _Tp>
1568 using __vector_type32_t =
typename __vector_type<_Tp, 32>::type;
1569template <
typename _Tp>
1570 using __vector_type64_t =
typename __vector_type<_Tp, 64>::type;
1574template <
typename _Tp,
typename =
void_t<>>
1577template <
typename _Tp>
1578 struct __is_vector_type<
1579 _Tp,
void_t<typename __vector_type<
1581 : is_same<_Tp, typename __vector_type<
1582 remove_reference_t<decltype(declval<_Tp>()[0])>,
1583 sizeof(_Tp)>::type> {};
1585template <
typename _Tp>
1586 inline constexpr bool __is_vector_type_v = __is_vector_type<_Tp>::value;
1590#if _GLIBCXX_SIMD_HAVE_SSE_ABI
1591template <
typename _Tp>
1592 using __is_intrinsic_type = __is_vector_type<_Tp>;
1594template <
typename _Tp,
typename =
void_t<>>
1597template <
typename _Tp>
1598 struct __is_intrinsic_type<
1599 _Tp,
void_t<typename __intrinsic_type<
1601 : is_same<_Tp, typename __intrinsic_type<
1602 remove_reference_t<decltype(declval<_Tp>()[0])>,
1603 sizeof(_Tp)>::type> {};
1606template <
typename _Tp>
1607 inline constexpr bool __is_intrinsic_type_v = __is_intrinsic_type<_Tp>::value;
1611template <
typename _Tp,
typename =
void_t<>>
1612 struct _VectorTraitsImpl;
1614template <
typename _Tp>
1615 struct _VectorTraitsImpl<_Tp,
enable_if_t<__is_vector_type_v<_Tp>
1616 || __is_intrinsic_type_v<_Tp>>>
1619 using value_type = remove_reference_t<decltype(declval<_Tp>()[0])>;
1620 static constexpr int _S_full_size =
sizeof(_Tp) /
sizeof(value_type);
1621 using _Wrapper = _SimdWrapper<value_type, _S_full_size>;
1622 template <
typename _Up,
int _W = _S_full_size>
1623 static constexpr bool _S_is
1624 = is_same_v<value_type, _Up> && _W == _S_full_size;
1627template <
typename _Tp,
size_t _Np>
1628 struct _VectorTraitsImpl<_SimdWrapper<_Tp, _Np>,
1629 void_t<__vector_type_t<_Tp, _Np>>>
1631 using type = __vector_type_t<_Tp, _Np>;
1632 using value_type = _Tp;
1633 static constexpr int _S_full_size =
sizeof(type) /
sizeof(value_type);
1634 using _Wrapper = _SimdWrapper<_Tp, _Np>;
1635 static constexpr bool _S_is_partial = (_Np == _S_full_size);
1636 static constexpr int _S_partial_width = _Np;
1637 template <
typename _Up,
int _W = _S_full_size>
1638 static constexpr bool _S_is
1639 = is_same_v<value_type, _Up>&& _W == _S_full_size;
1642template <typename _Tp, typename = typename _VectorTraitsImpl<_Tp>::type>
1643 using _VectorTraits = _VectorTraitsImpl<_Tp>;
1647template <
typename _V>
1648 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
1651 if constexpr (__is_vector_type_v<_V>)
1653 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1654 return __data(__x)._M_data;
1655 else if constexpr (__is_vectorizable_v<_V>)
1656 return __vector_type_t<_V, 2>{__x};
1663template <
size_t _Np = 0,
typename _V>
1664 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
1665 __as_wrapper(_V __x)
1667 if constexpr (__is_vector_type_v<_V>)
1668 return _SimdWrapper<
typename _VectorTraits<_V>::value_type,
1669 (_Np > 0 ? _Np : _VectorTraits<_V>::_S_full_size)>(__x);
1670 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1672 static_assert(_V::size() == _Np);
1677 static_assert(_V::_S_size == _Np);
1684template <
typename _To,
typename _From>
1685 _GLIBCXX_SIMD_INTRINSIC
constexpr _To
1686 __intrin_bitcast(_From __v)
1688 static_assert((__is_vector_type_v<_From> || __is_intrinsic_type_v<_From>)
1689 && (__is_vector_type_v<_To> || __is_intrinsic_type_v<_To>));
1690 if constexpr (
sizeof(_To) ==
sizeof(_From))
1691 return reinterpret_cast<_To
>(__v);
1692 else if constexpr (
sizeof(_From) >
sizeof(_To))
1693 if constexpr (
sizeof(_To) >= 16)
1694 return reinterpret_cast<const __may_alias<_To>&
>(__v);
1698 __builtin_memcpy(&__r, &__v,
sizeof(_To));
1701#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1702 else if constexpr (__have_avx &&
sizeof(_From) == 16 &&
sizeof(_To) == 32)
1703 return reinterpret_cast<_To
>(__builtin_ia32_ps256_ps(
1704 reinterpret_cast<__vector_type_t<float, 4>
>(__v)));
1705 else if constexpr (__have_avx512f &&
sizeof(_From) == 16
1706 &&
sizeof(_To) == 64)
1707 return reinterpret_cast<_To
>(__builtin_ia32_ps512_ps(
1708 reinterpret_cast<__vector_type_t<float, 4>
>(__v)));
1709 else if constexpr (__have_avx512f &&
sizeof(_From) == 32
1710 &&
sizeof(_To) == 64)
1711 return reinterpret_cast<_To
>(__builtin_ia32_ps512_256ps(
1712 reinterpret_cast<__vector_type_t<float, 8>
>(__v)));
1714 else if constexpr (
sizeof(__v) <= 8)
1715 return reinterpret_cast<_To
>(
1716 __vector_type_t<__int_for_sizeof_t<_From>,
sizeof(_To) /
sizeof(_From)>{
1717 reinterpret_cast<__int_for_sizeof_t<_From>
>(__v)});
1720 static_assert(
sizeof(_To) >
sizeof(_From));
1722 __builtin_memcpy(&__r, &__v,
sizeof(_From));
1729template <
typename _To,
size_t _NN = 0,
typename _From,
1730 typename _FromVT = _VectorTraits<_From>,
1731 size_t _Np = _NN == 0 ?
sizeof(_From) /
sizeof(_To) : _NN>
1732 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_To, _Np>
1733 __vector_bitcast(_From __x)
1735 using _R = __vector_type_t<_To, _Np>;
1736 return __intrin_bitcast<_R>(__x);
1739template <
typename _To,
size_t _NN = 0,
typename _Tp,
size_t _Nx,
1741 = _NN == 0 ?
sizeof(_SimdWrapper<_Tp, _Nx>) /
sizeof(_To) : _NN>
1742 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_To, _Np>
1743 __vector_bitcast(
const _SimdWrapper<_Tp, _Nx>& __x)
1745 static_assert(_Np > 1);
1746 return __intrin_bitcast<__vector_type_t<_To, _Np>>(__x._M_data);
1751#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
1752template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1753 _To __convert_x86(_Tp);
1755template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1756 _To __convert_x86(_Tp, _Tp);
1758template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1759 _To __convert_x86(_Tp, _Tp, _Tp, _Tp);
1761template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1762 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp);
1764template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1765 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp,
1766 _Tp, _Tp, _Tp, _Tp);
1771template <
typename _To,
typename _From>
1772 _GLIBCXX_SIMD_INTRINSIC
constexpr _To
1773 __bit_cast(
const _From __x)
1775#if __has_builtin(__builtin_bit_cast)
1776 return __builtin_bit_cast(_To, __x);
1778 static_assert(
sizeof(_To) ==
sizeof(_From));
1779 constexpr bool __to_is_vectorizable
1780 = is_arithmetic_v<_To> || is_enum_v<_To>;
1781 constexpr bool __from_is_vectorizable
1782 = is_arithmetic_v<_From> || is_enum_v<_From>;
1783 if constexpr (__is_vector_type_v<_To> && __is_vector_type_v<_From>)
1784 return reinterpret_cast<_To
>(__x);
1785 else if constexpr (__is_vector_type_v<_To> && __from_is_vectorizable)
1787 using _FV [[__gnu__::__vector_size__(
sizeof(_From))]] = _From;
1788 return reinterpret_cast<_To
>(_FV{__x});
1790 else if constexpr (__to_is_vectorizable && __from_is_vectorizable)
1792 using _TV [[__gnu__::__vector_size__(
sizeof(_To))]] = _To;
1793 using _FV [[__gnu__::__vector_size__(
sizeof(_From))]] = _From;
1794 return reinterpret_cast<_TV
>(_FV{__x})[0];
1796 else if constexpr (__to_is_vectorizable && __is_vector_type_v<_From>)
1798 using _TV [[__gnu__::__vector_size__(
sizeof(_To))]] = _To;
1799 return reinterpret_cast<_TV
>(__x)[0];
1804 __builtin_memcpy(
reinterpret_cast<char*
>(&__r),
1805 reinterpret_cast<const char*
>(&__x),
sizeof(_To));
1813template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>,
1814 typename _R = __
intrinsic_type_t<
typename _TVT::value_type, _TVT::_S_full_size>>
1815 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
1816 __to_intrin(_Tp __x)
1818 static_assert(
sizeof(__x) <=
sizeof(_R),
1819 "__to_intrin may never drop values off the end");
1820 if constexpr (
sizeof(__x) ==
sizeof(_R))
1821 return reinterpret_cast<_R
>(__as_vector(__x));
1824 using _Up = __int_for_sizeof_t<_Tp>;
1825 return reinterpret_cast<_R
>(
1826 __vector_type_t<_Up,
sizeof(_R) /
sizeof(_Up)>{__bit_cast<_Up>(__x)});
1832template <
typename _Tp,
typename... _Args>
1833 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_Tp,
sizeof...(_Args)>
1834 __make_vector(
const _Args&... __args)
1835 {
return __vector_type_t<_Tp,
sizeof...(_Args)>{
static_cast<_Tp
>(__args)...}; }
1839template <
size_t _Np,
typename _Tp,
size_t... _I>
1840 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_Tp, _Np>
1841 __vector_broadcast_impl(_Tp __x, index_sequence<_I...>)
1842 {
return __vector_type_t<_Tp, _Np>{((void)_I, __x)...}; }
1844template <
size_t _Np,
typename _Tp>
1845 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_Tp, _Np>
1846 __vector_broadcast(_Tp __x)
1847 {
return __vector_broadcast_impl<_Np, _Tp>(__x, make_index_sequence<_Np>()); }
1851 template <
typename _Tp,
size_t _Np,
typename _Gp,
size_t... _I>
1852 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_Tp, _Np>
1853 __generate_vector_impl(_Gp&& __gen, index_sequence<_I...>)
1854 {
return __vector_type_t<_Tp, _Np>{
static_cast<_Tp
>(__gen(_SizeConstant<_I>()))...}; }
1856template <
typename _V,
typename _VVT = _VectorTraits<_V>,
typename _Gp>
1857 _GLIBCXX_SIMD_INTRINSIC
constexpr _V
1858 __generate_vector(_Gp&& __gen)
1860 if constexpr (__is_vector_type_v<_V>)
1861 return __generate_vector_impl<
typename _VVT::value_type,
1862 _VVT::_S_full_size>(
1863 static_cast<_Gp&&
>(__gen), make_index_sequence<_VVT::_S_full_size>());
1865 return __generate_vector_impl<
typename _VVT::value_type,
1866 _VVT::_S_partial_width>(
1867 static_cast<_Gp&&
>(__gen),
1868 make_index_sequence<_VVT::_S_partial_width>());
1871template <
typename _Tp,
size_t _Np,
typename _Gp>
1872 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_Tp, _Np>
1873 __generate_vector(_Gp&& __gen)
1875 return __generate_vector_impl<_Tp, _Np>(
static_cast<_Gp&&
>(__gen),
1876 make_index_sequence<_Np>());
1881template <
typename _TW>
1882 _GLIBCXX_SIMD_INTRINSIC
constexpr _TW
1883 __xor(_TW __a, _TW __b)
noexcept
1885 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1887 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1888 _VectorTraitsImpl<_TW>>::value_type;
1889 if constexpr (is_floating_point_v<_Tp>)
1891 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1892 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1893 ^ __vector_bitcast<_Ip>(__b));
1895 else if constexpr (__is_vector_type_v<_TW>)
1898 return __a._M_data ^ __b._M_data;
1906template <
typename _TW>
1907 _GLIBCXX_SIMD_INTRINSIC
constexpr _TW
1908 __or(_TW __a, _TW __b)
noexcept
1910 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1912 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1913 _VectorTraitsImpl<_TW>>::value_type;
1914 if constexpr (is_floating_point_v<_Tp>)
1916 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1917 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1918 | __vector_bitcast<_Ip>(__b));
1920 else if constexpr (__is_vector_type_v<_TW>)
1923 return __a._M_data | __b._M_data;
1931template <
typename _TW>
1932 _GLIBCXX_SIMD_INTRINSIC
constexpr _TW
1933 __and(_TW __a, _TW __b)
noexcept
1935 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1937 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1938 _VectorTraitsImpl<_TW>>::value_type;
1939 if constexpr (is_floating_point_v<_Tp>)
1941 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1942 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1943 & __vector_bitcast<_Ip>(__b));
1945 else if constexpr (__is_vector_type_v<_TW>)
1948 return __a._M_data & __b._M_data;
1956#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1957static constexpr struct
1959 _GLIBCXX_SIMD_INTRINSIC __v4sf
1960 operator()(__v4sf __a, __v4sf __b)
const noexcept
1961 {
return __builtin_ia32_andnps(__a, __b); }
1963 _GLIBCXX_SIMD_INTRINSIC __v2df
1964 operator()(__v2df __a, __v2df __b)
const noexcept
1965 {
return __builtin_ia32_andnpd(__a, __b); }
1967 _GLIBCXX_SIMD_INTRINSIC __v2di
1968 operator()(__v2di __a, __v2di __b)
const noexcept
1969 {
return __builtin_ia32_pandn128(__a, __b); }
1971 _GLIBCXX_SIMD_INTRINSIC __v8sf
1972 operator()(__v8sf __a, __v8sf __b)
const noexcept
1973 {
return __builtin_ia32_andnps256(__a, __b); }
1975 _GLIBCXX_SIMD_INTRINSIC __v4df
1976 operator()(__v4df __a, __v4df __b)
const noexcept
1977 {
return __builtin_ia32_andnpd256(__a, __b); }
1979 _GLIBCXX_SIMD_INTRINSIC __v4di
1980 operator()(__v4di __a, __v4di __b)
const noexcept
1982 if constexpr (__have_avx2)
1983 return __builtin_ia32_andnotsi256(__a, __b);
1985 return reinterpret_cast<__v4di
>(
1986 __builtin_ia32_andnpd256(
reinterpret_cast<__v4df
>(__a),
1987 reinterpret_cast<__v4df
>(__b)));
1990 _GLIBCXX_SIMD_INTRINSIC __v16sf
1991 operator()(__v16sf __a, __v16sf __b)
const noexcept
1993 if constexpr (__have_avx512dq)
1994 return _mm512_andnot_ps(__a, __b);
1996 return reinterpret_cast<__v16sf
>(
1997 _mm512_andnot_si512(
reinterpret_cast<__v8di
>(__a),
1998 reinterpret_cast<__v8di
>(__b)));
2001 _GLIBCXX_SIMD_INTRINSIC __v8df
2002 operator()(__v8df __a, __v8df __b)
const noexcept
2004 if constexpr (__have_avx512dq)
2005 return _mm512_andnot_pd(__a, __b);
2007 return reinterpret_cast<__v8df
>(
2008 _mm512_andnot_si512(
reinterpret_cast<__v8di
>(__a),
2009 reinterpret_cast<__v8di
>(__b)));
2012 _GLIBCXX_SIMD_INTRINSIC __v8di
2013 operator()(__v8di __a, __v8di __b)
const noexcept
2014 {
return _mm512_andnot_si512(__a, __b); }
2018template <
typename _TW>
2019 _GLIBCXX_SIMD_INTRINSIC
constexpr _TW
2020 __andnot(_TW __a, _TW __b)
noexcept
2022 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
2024 using _TVT = conditional_t<__is_simd_wrapper_v<_TW>, _TW,
2025 _VectorTraitsImpl<_TW>>;
2026 using _Tp =
typename _TVT::value_type;
2027#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
2028 if constexpr (
sizeof(_TW) >= 16)
2030 const auto __ai = __to_intrin(__a);
2031 const auto __bi = __to_intrin(__b);
2032 if (!__builtin_is_constant_evaluated()
2033 && !(__builtin_constant_p(__ai) && __builtin_constant_p(__bi)))
2035 const auto __r = _S_x86_andnot(__ai, __bi);
2036 if constexpr (is_convertible_v<
decltype(__r), _TW>)
2039 return reinterpret_cast<typename _TVT::type
>(__r);
2043 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
2044 return __vector_bitcast<_Tp>(~__vector_bitcast<_Ip>(__a)
2045 & __vector_bitcast<_Ip>(__b));
2053template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
2054 _GLIBCXX_SIMD_INTRINSIC
constexpr _Tp
2055 __not(_Tp __a)
noexcept
2057 if constexpr (is_floating_point_v<typename _TVT::value_type>)
2058 return reinterpret_cast<typename _TVT::type
>(
2059 ~__vector_bitcast<unsigned>(__a));
2066template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>,
2067 typename _R = __vector_type_t<
typename _TVT::value_type, _TVT::_S_full_size * 2>>
2069 __concat(_Tp a_, _Tp b_)
2071#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2073 = conditional_t<is_floating_point_v<typename _TVT::value_type>, double,
2075 long long,
typename _TVT::value_type>>;
2076 constexpr int input_width =
sizeof(_Tp) /
sizeof(_W);
2077 const auto __a = __vector_bitcast<_W>(a_);
2078 const auto __b = __vector_bitcast<_W>(b_);
2079 using _Up = __vector_type_t<_W,
sizeof(_R) /
sizeof(_W)>;
2081 constexpr int input_width = _TVT::_S_full_size;
2082 const _Tp& __a = a_;
2083 const _Tp& __b = b_;
2086 if constexpr (input_width == 2)
2087 return reinterpret_cast<_R
>(_Up{__a[0], __a[1], __b[0], __b[1]});
2088 else if constexpr (input_width == 4)
2089 return reinterpret_cast<_R
>(
2090 _Up{__a[0], __a[1], __a[2], __a[3], __b[0], __b[1], __b[2], __b[3]});
2091 else if constexpr (input_width == 8)
2092 return reinterpret_cast<_R
>(
2093 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7],
2094 __b[0], __b[1], __b[2], __b[3], __b[4], __b[5], __b[6], __b[7]});
2095 else if constexpr (input_width == 16)
2096 return reinterpret_cast<_R
>(
2097 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
2098 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
2099 __a[14], __a[15], __b[0], __b[1], __b[2], __b[3], __b[4],
2100 __b[5], __b[6], __b[7], __b[8], __b[9], __b[10], __b[11],
2101 __b[12], __b[13], __b[14], __b[15]});
2102 else if constexpr (input_width == 32)
2103 return reinterpret_cast<_R
>(
2104 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
2105 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
2106 __a[14], __a[15], __a[16], __a[17], __a[18], __a[19], __a[20],
2107 __a[21], __a[22], __a[23], __a[24], __a[25], __a[26], __a[27],
2108 __a[28], __a[29], __a[30], __a[31], __b[0], __b[1], __b[2],
2109 __b[3], __b[4], __b[5], __b[6], __b[7], __b[8], __b[9],
2110 __b[10], __b[11], __b[12], __b[13], __b[14], __b[15], __b[16],
2111 __b[17], __b[18], __b[19], __b[20], __b[21], __b[22], __b[23],
2112 __b[24], __b[25], __b[26], __b[27], __b[28], __b[29], __b[30],
2118template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
2119 struct _ZeroExtendProxy
2121 using value_type =
typename _TVT::value_type;
2122 static constexpr size_t _Np = _TVT::_S_full_size;
2125 template <
typename _To,
typename _ToVT = _VectorTraits<_To>,
2127 = enable_if_t<is_same_v<
typename _ToVT::value_type, value_type>>>
2128 _GLIBCXX_SIMD_INTRINSIC
operator _To()
const
2130 constexpr size_t _ToN = _ToVT::_S_full_size;
2131 if constexpr (_ToN == _Np)
2133 else if constexpr (_ToN == 2 * _Np)
2135#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2136 if constexpr (__have_avx && _TVT::template _S_is<float, 4>)
2137 return __vector_bitcast<value_type>(
2138 _mm256_insertf128_ps(__m256(), __x, 0));
2139 else if constexpr (__have_avx && _TVT::template _S_is<double, 2>)
2140 return __vector_bitcast<value_type>(
2141 _mm256_insertf128_pd(__m256d(), __x, 0));
2142 else if constexpr (__have_avx2 && _Np *
sizeof(value_type) == 16)
2143 return __vector_bitcast<value_type>(
2144 _mm256_insertf128_si256(__m256i(), __to_intrin(__x), 0));
2145 else if constexpr (__have_avx512f && _TVT::template _S_is<float, 8>)
2147 if constexpr (__have_avx512dq)
2148 return __vector_bitcast<value_type>(
2149 _mm512_insertf32x8(__m512(), __x, 0));
2151 return reinterpret_cast<__m512
>(
2152 _mm512_insertf64x4(__m512d(),
2153 reinterpret_cast<__m256d
>(__x), 0));
2155 else if constexpr (__have_avx512f
2156 && _TVT::template _S_is<double, 4>)
2157 return __vector_bitcast<value_type>(
2158 _mm512_insertf64x4(__m512d(), __x, 0));
2159 else if constexpr (__have_avx512f && _Np *
sizeof(value_type) == 32)
2160 return __vector_bitcast<value_type>(
2161 _mm512_inserti64x4(__m512i(), __to_intrin(__x), 0));
2163 return __concat(__x, _Tp());
2165 else if constexpr (_ToN == 4 * _Np)
2167#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2168 if constexpr (__have_avx512dq && _TVT::template _S_is<double, 2>)
2170 return __vector_bitcast<value_type>(
2171 _mm512_insertf64x2(__m512d(), __x, 0));
2173 else if constexpr (__have_avx512f
2174 && is_floating_point_v<value_type>)
2176 return __vector_bitcast<value_type>(
2177 _mm512_insertf32x4(__m512(),
reinterpret_cast<__m128
>(__x),
2180 else if constexpr (__have_avx512f && _Np *
sizeof(value_type) == 16)
2182 return __vector_bitcast<value_type>(
2183 _mm512_inserti32x4(__m512i(), __to_intrin(__x), 0));
2186 return __concat(__concat(__x, _Tp()),
2187 __vector_type_t<value_type, _Np * 2>());
2189 else if constexpr (_ToN == 8 * _Np)
2190 return __concat(
operator __vector_type_t<value_type, _Np * 4>(),
2191 __vector_type_t<value_type, _Np * 4>());
2192 else if constexpr (_ToN == 16 * _Np)
2193 return __concat(
operator __vector_type_t<value_type, _Np * 8>(),
2194 __vector_type_t<value_type, _Np * 8>());
2196 __assert_unreachable<_Tp>();
2200template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
2201 _GLIBCXX_SIMD_INTRINSIC _ZeroExtendProxy<_Tp, _TVT>
2202 __zero_extend(_Tp __x)
2207template <
int _Offset,
2210 typename _TVT = _VectorTraits<_Tp>,
2211 typename _R = __vector_type_t<
typename _TVT::value_type, _TVT::_S_full_size / _SplitBy>>
2212 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
2215 using value_type =
typename _TVT::value_type;
2216#if _GLIBCXX_SIMD_X86INTRIN
2217 if constexpr (
sizeof(_Tp) == 64 && _SplitBy == 4 && _Offset > 0)
2219 if constexpr (__have_avx512dq && is_same_v<double, value_type>)
2220 return _mm512_extractf64x2_pd(__to_intrin(__in), _Offset);
2221 else if constexpr (is_floating_point_v<value_type>)
2222 return __vector_bitcast<value_type>(
2223 _mm512_extractf32x4_ps(__intrin_bitcast<__m512>(__in), _Offset));
2225 return reinterpret_cast<_R
>(
2226 _mm512_extracti32x4_epi32(__intrin_bitcast<__m512i>(__in),
2232#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2234 is_floating_point_v<value_type>, double,
2236 static_assert(
sizeof(_R) %
sizeof(_W) == 0);
2237 constexpr int __return_width =
sizeof(_R) /
sizeof(_W);
2238 using _Up = __vector_type_t<_W, __return_width>;
2239 const auto __x = __vector_bitcast<_W>(__in);
2241 constexpr int __return_width = _TVT::_S_full_size / _SplitBy;
2243 const __vector_type_t<value_type, _TVT::_S_full_size>& __x
2246 constexpr int _O = _Offset * __return_width;
2247 return __call_with_subscripts<__return_width, _O>(
2248 __x, [](
auto... __entries) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2249 return reinterpret_cast<_R
>(_Up{__entries...});
2256template <
typename _Tp,
2257 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2258 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
2262 __builtin_memcpy(&__r, &__x, 8);
2266template <
typename _Tp,
2267 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2268 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
2271 static_assert(
sizeof(_Tp) == 16,
"use __hi64z if you meant it");
2273 __builtin_memcpy(&__r,
reinterpret_cast<const char*
>(&__x) + 8, 8);
2277template <
typename _Tp,
2278 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2279 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
2280 __hi64z([[maybe_unused]] _Tp __x)
2283 if constexpr (
sizeof(_Tp) == 16)
2284 __builtin_memcpy(&__r,
reinterpret_cast<const char*
>(&__x) + 8, 8);
2290template <
typename _Tp>
2291 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
2293 {
return __extract<0,
sizeof(_Tp) / 16>(__x); }
2295template <
typename _Tp>
2296 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
2299 static_assert(
sizeof(__x) == 32);
2300 return __extract<1, 2>(__x);
2305template <
typename _Tp>
2306 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
2309 static_assert(
sizeof(__x) == 64);
2310 return __extract<0, 2>(__x);
2313template <
typename _Tp>
2314 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
2317 static_assert(
sizeof(__x) == 64);
2318 return __extract<1, 2>(__x);
2323template <
typename _Tp>
2326 static_assert(__is_vector_type_v<_Tp>);
2330 template <
typename _Up,
typename _UVT = _VectorTraits<_Up>>
2331 _GLIBCXX_SIMD_INTRINSIC
constexpr operator _Up()
const
2332 {
return __intrin_bitcast<typename _UVT::type>(__x); }
2335template <
typename _Tp>
2336 _GLIBCXX_SIMD_INTRINSIC
constexpr _AutoCast<_Tp>
2337 __auto_bitcast(
const _Tp& __x)
2340template <
typename _Tp,
size_t _Np>
2341 _GLIBCXX_SIMD_INTRINSIC
constexpr
2342 _AutoCast<typename _SimdWrapper<_Tp, _Np>::_BuiltinType>
2343 __auto_bitcast(
const _SimdWrapper<_Tp, _Np>& __x)
2344 {
return {__x._M_data}; }
2349#if _GLIBCXX_SIMD_HAVE_SSE_ABI
2351#if _GLIBCXX_SIMD_HAVE_AVX512F && _GLIBCXX_SIMD_X86INTRIN
2352template <
size_t _Size>
2353 struct __bool_storage_member_type
2355 static_assert((_Size & (_Size - 1)) != 0,
2356 "This trait may only be used for non-power-of-2 sizes. "
2357 "Power-of-2 sizes must be specialized.");
2359 typename __bool_storage_member_type<std::__bit_ceil(_Size)>::type;
2363 struct __bool_storage_member_type<1> {
using type = bool; };
2366 struct __bool_storage_member_type<2> {
using type = __mmask8; };
2369 struct __bool_storage_member_type<4> {
using type = __mmask8; };
2372 struct __bool_storage_member_type<8> {
using type = __mmask8; };
2375 struct __bool_storage_member_type<16> {
using type = __mmask16; };
2378 struct __bool_storage_member_type<32> {
using type = __mmask32; };
2381 struct __bool_storage_member_type<64> {
using type = __mmask64; };
2387#if _GLIBCXX_SIMD_HAVE_SSE
2388template <
typename _Tp,
size_t _Bytes>
2389 struct __intrinsic_type<_Tp, _Bytes,
enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 64>>
2391 static_assert(!is_same_v<_Tp, long double>,
2392 "no __intrinsic_type support for long double on x86");
2394 static constexpr size_t _S_VBytes = _Bytes <= 16 ? 16 : _Bytes <= 32 ? 32 : 64;
2396 using type [[__gnu__::__vector_size__(_S_VBytes)]]
2397 = conditional_t<is_integral_v<_Tp>,
long long int, _Tp>;
2404#if _GLIBCXX_SIMD_HAVE_NEON
2406 struct __intrinsic_type<float, 8, void>
2407 {
using type = float32x2_t; };
2410 struct __intrinsic_type<float, 16, void>
2411 {
using type = float32x4_t; };
2414 struct __intrinsic_type<double, 8, void>
2416#if _GLIBCXX_SIMD_HAVE_NEON_A64
2417 using type = float64x1_t;
2422 struct __intrinsic_type<double, 16, void>
2424#if _GLIBCXX_SIMD_HAVE_NEON_A64
2425 using type = float64x2_t;
2429#define _GLIBCXX_SIMD_ARM_INTRIN(_Bits, _Np) \
2431 struct __intrinsic_type<__int_with_sizeof_t<_Bits / 8>, \
2432 _Np * _Bits / 8, void> \
2433 { using type = int##_Bits##x##_Np##_t; }; \
2435 struct __intrinsic_type<make_unsigned_t<__int_with_sizeof_t<_Bits / 8>>, \
2436 _Np * _Bits / 8, void> \
2437 { using type = uint##_Bits##x##_Np##_t; }
2438_GLIBCXX_SIMD_ARM_INTRIN(8, 8);
2439_GLIBCXX_SIMD_ARM_INTRIN(8, 16);
2440_GLIBCXX_SIMD_ARM_INTRIN(16, 4);
2441_GLIBCXX_SIMD_ARM_INTRIN(16, 8);
2442_GLIBCXX_SIMD_ARM_INTRIN(32, 2);
2443_GLIBCXX_SIMD_ARM_INTRIN(32, 4);
2444_GLIBCXX_SIMD_ARM_INTRIN(64, 1);
2445_GLIBCXX_SIMD_ARM_INTRIN(64, 2);
2446#undef _GLIBCXX_SIMD_ARM_INTRIN
2448template <
typename _Tp,
size_t _Bytes>
2449 struct __intrinsic_type<_Tp, _Bytes,
enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2451 static constexpr int _SVecBytes = _Bytes <= 8 ? 8 : 16;
2453 using _Ip = __int_for_sizeof_t<_Tp>;
2456 is_floating_point_v<_Tp>, _Tp,
2457 conditional_t<is_unsigned_v<_Tp>, make_unsigned_t<_Ip>, _Ip>>;
2459 static_assert(!is_same_v<_Tp, _Up> || _SVecBytes != _Bytes,
2460 "should use explicit specialization above");
2462 using type =
typename __intrinsic_type<_Up, _SVecBytes>::type;
2469template <
typename _Tp>
2470 struct __intrinsic_type_impl;
2472#define _GLIBCXX_SIMD_PPC_INTRIN(_Tp) \
2474 struct __intrinsic_type_impl<_Tp> { using type = __vector _Tp; }
2475_GLIBCXX_SIMD_PPC_INTRIN(
float);
2477_GLIBCXX_SIMD_PPC_INTRIN(
double);
2479_GLIBCXX_SIMD_PPC_INTRIN(
signed char);
2480_GLIBCXX_SIMD_PPC_INTRIN(
unsigned char);
2481_GLIBCXX_SIMD_PPC_INTRIN(
signed short);
2482_GLIBCXX_SIMD_PPC_INTRIN(
unsigned short);
2483_GLIBCXX_SIMD_PPC_INTRIN(
signed int);
2484_GLIBCXX_SIMD_PPC_INTRIN(
unsigned int);
2485#if defined __VSX__ || __SIZEOF_LONG__ == 4
2486_GLIBCXX_SIMD_PPC_INTRIN(
signed long);
2487_GLIBCXX_SIMD_PPC_INTRIN(
unsigned long);
2490_GLIBCXX_SIMD_PPC_INTRIN(
signed long long);
2491_GLIBCXX_SIMD_PPC_INTRIN(
unsigned long long);
2493#undef _GLIBCXX_SIMD_PPC_INTRIN
2495template <
typename _Tp,
size_t _Bytes>
2496 struct __intrinsic_type<_Tp, _Bytes,
enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2498 static constexpr bool _S_is_ldouble = is_same_v<_Tp, long double>;
2501 static_assert(!(_S_is_ldouble &&
sizeof(
long double) >
sizeof(
double)),
2502 "no __intrinsic_type support for 128-bit floating point on PowerPC");
2505 static_assert(!(is_same_v<_Tp, double>
2506 || (_S_is_ldouble &&
sizeof(
long double) ==
sizeof(
double))),
2507 "no __intrinsic_type support for 64-bit floating point on PowerPC w/o VSX");
2510 static constexpr auto __element_type()
2512 if constexpr (is_floating_point_v<_Tp>)
2514 if constexpr (_S_is_ldouble)
2519 else if constexpr (is_signed_v<_Tp>)
2521 if constexpr (
sizeof(_Tp) ==
sizeof(_SChar))
2523 else if constexpr (
sizeof(_Tp) ==
sizeof(
short))
2525 else if constexpr (
sizeof(_Tp) ==
sizeof(
int))
2527 else if constexpr (
sizeof(_Tp) ==
sizeof(_LLong))
2532 if constexpr (
sizeof(_Tp) ==
sizeof(_UChar))
2534 else if constexpr (
sizeof(_Tp) ==
sizeof(_UShort))
2536 else if constexpr (
sizeof(_Tp) ==
sizeof(_UInt))
2538 else if constexpr (
sizeof(_Tp) ==
sizeof(_ULLong))
2543 using type =
typename __intrinsic_type_impl<
decltype(__element_type())>::type;
2549template <
size_t _W
idth>
2550 struct _SimdWrapper<bool, _Width,
2551 void_t<typename __bool_storage_member_type<_Width>::type>>
2553 using _BuiltinType =
typename __bool_storage_member_type<_Width>::type;
2554 using value_type = bool;
2556 static constexpr size_t _S_full_size =
sizeof(_BuiltinType) * __CHAR_BIT__;
2558 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper<bool, _S_full_size>
2559 __as_full_vector()
const
2562 _GLIBCXX_SIMD_INTRINSIC
constexpr
2563 _SimdWrapper() =
default;
2565 _GLIBCXX_SIMD_INTRINSIC
constexpr
2566 _SimdWrapper(_BuiltinType __k) : _M_data(__k) {};
2568 _GLIBCXX_SIMD_INTRINSIC
2569 operator const _BuiltinType&()
const
2572 _GLIBCXX_SIMD_INTRINSIC
2573 operator _BuiltinType&()
2576 _GLIBCXX_SIMD_INTRINSIC _BuiltinType
2580 _GLIBCXX_SIMD_INTRINSIC
constexpr value_type
2581 operator[](
size_t __i)
const
2582 {
return _M_data & (_BuiltinType(1) << __i); }
2584 template <
size_t __i>
2585 _GLIBCXX_SIMD_INTRINSIC
constexpr value_type
2586 operator[](_SizeConstant<__i>)
const
2587 {
return _M_data & (_BuiltinType(1) << __i); }
2589 _GLIBCXX_SIMD_INTRINSIC
constexpr void
2590 _M_set(
size_t __i, value_type __x)
2593 _M_data |= (_BuiltinType(1) << __i);
2595 _M_data &= ~(_BuiltinType(1) << __i);
2598 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
2599 _M_is_constprop()
const
2600 {
return __builtin_constant_p(_M_data); }
2602 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
2603 _M_is_constprop_none_of()
const
2605 if (__builtin_constant_p(_M_data))
2607 constexpr int __nbits =
sizeof(_BuiltinType) * __CHAR_BIT__;
2608 constexpr _BuiltinType __active_mask
2609 = ~_BuiltinType() >> (__nbits - _Width);
2610 return (_M_data & __active_mask) == 0;
2615 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
2616 _M_is_constprop_all_of()
const
2618 if (__builtin_constant_p(_M_data))
2620 constexpr int __nbits =
sizeof(_BuiltinType) * __CHAR_BIT__;
2621 constexpr _BuiltinType __active_mask
2622 = ~_BuiltinType() >> (__nbits - _Width);
2623 return (_M_data & __active_mask) == __active_mask;
2628 _BuiltinType _M_data;
2632template <
bool _MustZeroInitPadding,
typename _BuiltinType>
2633 struct _SimdWrapperBase;
2635template <
typename _BuiltinType>
2636 struct _SimdWrapperBase<false, _BuiltinType>
2638 _GLIBCXX_SIMD_INTRINSIC
constexpr
2639 _SimdWrapperBase() =
default;
2641 _GLIBCXX_SIMD_INTRINSIC
constexpr
2642 _SimdWrapperBase(_BuiltinType __init) : _M_data(__init) {}
2644 _BuiltinType _M_data;
2647template <
typename _BuiltinType>
2648 struct _SimdWrapperBase<true, _BuiltinType>
2651 _GLIBCXX_SIMD_INTRINSIC
constexpr
2652 _SimdWrapperBase() : _M_data() {}
2654 _GLIBCXX_SIMD_INTRINSIC
constexpr
2655 _SimdWrapperBase(_BuiltinType __init) : _M_data(__init) {}
2657 _BuiltinType _M_data;
2662template <
typename _Tp,
size_t _W
idth>
2663 struct _SimdWrapper<
2665 void_t<__vector_type_t<_Tp, _Width>, __intrinsic_type_t<_Tp, _Width>>>
2666 : _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2667 && sizeof(_Tp) * _Width
2668 == sizeof(__vector_type_t<_Tp, _Width>),
2669 __vector_type_t<_Tp, _Width>>
2672 = _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2673 &&
sizeof(_Tp) * _Width
2674 ==
sizeof(__vector_type_t<_Tp, _Width>),
2675 __vector_type_t<_Tp, _Width>>;
2677 static_assert(__is_vectorizable_v<_Tp>);
2678 static_assert(_Width >= 2);
2680 using _BuiltinType = __vector_type_t<_Tp, _Width>;
2681 using value_type = _Tp;
2683 static inline constexpr size_t _S_full_size
2684 =
sizeof(_BuiltinType) /
sizeof(value_type);
2685 static inline constexpr int _S_size = _Width;
2686 static inline constexpr bool _S_is_partial = _S_full_size != _S_size;
2688 using _Base::_M_data;
2690 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper<_Tp, _S_full_size>
2691 __as_full_vector()
const
2694 _GLIBCXX_SIMD_INTRINSIC
constexpr
2695 _SimdWrapper(initializer_list<_Tp> __init)
2696 : _Base(__generate_from_n_evaluations<_Width, _BuiltinType>(
2697 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2698 return __init.
begin()[__i.value];
2701 _GLIBCXX_SIMD_INTRINSIC
constexpr
2702 _SimdWrapper() =
default;
2704 _GLIBCXX_SIMD_INTRINSIC
constexpr
2705 _SimdWrapper(
const _SimdWrapper&) =
default;
2707 _GLIBCXX_SIMD_INTRINSIC
constexpr
2708 _SimdWrapper(_SimdWrapper&&) =
default;
2710 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper&
2711 operator=(
const _SimdWrapper&) =
default;
2713 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper&
2714 operator=(_SimdWrapper&&) =
default;
2716 template <
typename _V,
typename =
enable_if_t<disjunction_v<
2717 is_same<_V, __vector_type_t<_Tp, _Width>>,
2718 is_same<_V, __intrinsic_type_t<_Tp, _Width>>>>>
2719 _GLIBCXX_SIMD_INTRINSIC
constexpr
2720 _SimdWrapper(_V __x)
2722 : _Base(__vector_bitcast<_Tp, _Width>(__x)) {}
2724 template <
typename... _As,
2725 typename =
enable_if_t<((is_same_v<simd_abi::scalar, _As> && ...)
2726 &&
sizeof...(_As) <= _Width)>>
2727 _GLIBCXX_SIMD_INTRINSIC
constexpr
2728 operator _SimdTuple<_Tp, _As...>()
const
2730 return __generate_from_n_evaluations<
sizeof...(_As), _SimdTuple<_Tp, _As...>>(
2731 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
2732 {
return _M_data[int(__i)]; });
2735 _GLIBCXX_SIMD_INTRINSIC
constexpr
2736 operator const _BuiltinType&()
const
2739 _GLIBCXX_SIMD_INTRINSIC
constexpr
2740 operator _BuiltinType&()
2743 _GLIBCXX_SIMD_INTRINSIC
constexpr _Tp
2744 operator[](
size_t __i)
const
2745 {
return _M_data[__i]; }
2747 template <
size_t __i>
2748 _GLIBCXX_SIMD_INTRINSIC
constexpr _Tp
2749 operator[](_SizeConstant<__i>)
const
2750 {
return _M_data[__i]; }
2752 _GLIBCXX_SIMD_INTRINSIC
constexpr void
2753 _M_set(
size_t __i, _Tp __x)
2755 if (__builtin_is_constant_evaluated())
2756 _M_data = __generate_from_n_evaluations<_Width, _BuiltinType>([&](
auto __j) {
2757 return __j == __i ? __x : _M_data[__j()];
2763 _GLIBCXX_SIMD_INTRINSIC
2765 _M_is_constprop()
const
2766 {
return __builtin_constant_p(_M_data); }
2768 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
2769 _M_is_constprop_none_of()
const
2771 if (__builtin_constant_p(_M_data))
2774 if constexpr (is_floating_point_v<_Tp>)
2776 using _Ip = __int_for_sizeof_t<_Tp>;
2777 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2778 __execute_n_times<_Width>(
2779 [&](
auto __i) { __r &= __intdata[__i.value] == _Ip(); });
2782 __execute_n_times<_Width>(
2783 [&](
auto __i) { __r &= _M_data[__i.value] == _Tp(); });
2784 if (__builtin_constant_p(__r))
2790 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
2791 _M_is_constprop_all_of()
const
2793 if (__builtin_constant_p(_M_data))
2796 if constexpr (is_floating_point_v<_Tp>)
2798 using _Ip = __int_for_sizeof_t<_Tp>;
2799 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2800 __execute_n_times<_Width>(
2801 [&](
auto __i) { __r &= __intdata[__i.value] == ~_Ip(); });
2804 __execute_n_times<_Width>(
2805 [&](
auto __i) { __r &= _M_data[__i.value] == ~_Tp(); });
2806 if (__builtin_constant_p(__r))
2816template <
typename _Tp>
2818 __vectorized_sizeof()
2820 if constexpr (!__is_vectorizable_v<_Tp>)
2823 if constexpr (
sizeof(_Tp) <= 8)
2826 if constexpr (__have_avx512bw)
2828 if constexpr (__have_avx512f &&
sizeof(_Tp) >= 4)
2830 if constexpr (__have_avx2)
2832 if constexpr (__have_avx && is_floating_point_v<_Tp>)
2834 if constexpr (__have_sse2)
2836 if constexpr (__have_sse && is_same_v<_Tp, float>)
2846 if constexpr (__have_power8vec
2847 || (__have_power_vmx && (
sizeof(_Tp) < 8))
2848 || (__have_power_vsx && is_floating_point_v<_Tp>) )
2852 if constexpr (__have_neon_a64)
2854 if constexpr (__have_neon_a32 and (not is_floating_point_v<_Tp>
2855 or is_same_v<_Tp, float>))
2857 if constexpr (__have_neon
2862 && (__support_neon_float || !is_floating_point_v<_Tp>))
2872template <
typename _Tp>
2873 inline constexpr int max_fixed_size
2874 = ((__have_avx512bw &&
sizeof(_Tp) == 1)
2875 || (__have_sve && __sve_vectorized_size_bytes/
sizeof(_Tp) >= 64)) ? 64 : 32;
2878#if defined __x86_64__ || defined __aarch64__
2879template <
typename _Tp>
2880 using compatible =
conditional_t<(
sizeof(_Tp) <= 8), _VecBuiltin<16>, scalar>;
2881#elif defined __ARM_NEON
2884template <
typename _Tp>
2887 && (__support_neon_float || !is_floating_point_v<_Tp>)),
2888 _VecBuiltin<16>, scalar>;
2891 using compatible = scalar;
2896template <
typename _Tp>
2898 __determine_native_abi()
2900 constexpr size_t __bytes = __vectorized_sizeof<_Tp>();
2901 if constexpr (__bytes ==
sizeof(_Tp))
2902 return static_cast<scalar*
>(
nullptr);
2903 else if constexpr (__have_sve)
2904 return static_cast<_SveAbi<__sve_vectorized_size_bytes>*
>(
nullptr);
2905 else if constexpr (__have_avx512vl || (__have_avx512f && __bytes == 64))
2906 return static_cast<_VecBltnBtmsk<__bytes>*
>(
nullptr);
2908 return static_cast<_VecBuiltin<__bytes>*
>(
nullptr);
2911template <
typename _Tp,
typename = enable_if_t<__is_vectorizable_v<_Tp>>>
2912 using native = remove_pointer_t<decltype(__determine_native_abi<_Tp>())>;
2916#if defined _GLIBCXX_SIMD_DEFAULT_ABI
2917template <
typename _Tp>
2918 using __default_abi = _GLIBCXX_SIMD_DEFAULT_ABI<_Tp>;
2920template <
typename _Tp>
2921 using __default_abi = compatible<_Tp>;
2928template <
typename _Tp>
2929 struct is_simd_flag_type
2934 struct is_simd_flag_type<element_aligned_tag>
2939 struct is_simd_flag_type<vector_aligned_tag>
2943template <
size_t _Np>
2944 struct is_simd_flag_type<overaligned_tag<_Np>>
2945 : __bool_constant<(_Np > 0) and __has_single_bit(_Np)>
2948template <
typename _Tp>
2949 inline constexpr bool is_simd_flag_type_v = is_simd_flag_type<_Tp>::value;
2951template <
typename _Tp,
typename = enable_if_t<is_simd_flag_type_v<_Tp>>>
2952 using _IsSimdFlagType = _Tp;
2955template <
typename _Tp,
typename =
void_t<>>
2958template <
typename _Tp>
2959 struct is_abi_tag<_Tp,
void_t<typename _Tp::_IsValidAbiTag>>
2960 :
public _Tp::_IsValidAbiTag {};
2962template <
typename _Tp>
2963 inline constexpr bool is_abi_tag_v = is_abi_tag<_Tp>::value;
2966template <
typename _Tp>
2969template <
typename _Tp>
2970 inline constexpr bool is_simd_v = is_simd<_Tp>::value;
2972template <
typename _Tp>
2975template <
typename _Tp>
2976inline constexpr bool is_simd_mask_v = is_simd_mask<_Tp>::value;
2979template <
typename _Tp,
typename _Abi,
typename =
void>
2980 struct __simd_size_impl {};
2982template <
typename _Tp,
typename _Abi>
2983 struct __simd_size_impl<
2985 enable_if_t<conjunction_v<__is_vectorizable<_Tp>, is_abi_tag<_Abi>>>>
2986 : _SizeConstant<_Abi::template _S_size<_Tp>> {};
2988template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2989 struct simd_size : __simd_size_impl<_Tp, _Abi> {};
2991template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2992 inline constexpr size_t simd_size_v = simd_size<_Tp, _Abi>::value;
2995template <
typename _Tp,
size_t _Np,
typename =
void>
2996 struct __deduce_impl;
2998template <
typename _Tp,
size_t _Np,
typename =
void>
2999 struct __no_sve_deduce_impl;
3010template <
typename _Tp,
size_t _Np,
typename...>
3013template <
typename _Tp,
size_t _Np,
typename... _Abis>
3014 using deduce_t =
typename deduce<_Tp, _Np, _Abis...>::type;
3016template <
typename _Tp,
size_t _Np,
typename...>
3017 struct __no_sve_deduce : __no_sve_deduce_impl<_Tp, _Np> {};
3019template <
typename _Tp,
size_t _Np,
typename... _Abis>
3020 using __no_sve_deduce_t =
typename __no_sve_deduce<_Tp, _Np, _Abis...>::type;
3025template <
typename _Tp,
typename _V,
typename =
void>
3028template <
typename _Tp,
typename _Up,
typename _Abi>
3029 struct rebind_simd<_Tp, simd<_Up, _Abi>,
3031 simd_abi::__no_sve_deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>,
3032 simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>>
3035 !__is_sve_abi<_Abi>(),
3036 simd_abi::__no_sve_deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>,
3037 simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>;
3040template <
typename _Tp,
typename _Up,
typename _Abi>
3041 struct rebind_simd<_Tp, simd_mask<_Up, _Abi>,
3043 simd_abi::__no_sve_deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>,
3044 simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>>
3047 !__is_sve_abi<_Abi>(),
3048 simd_abi::__no_sve_deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>,
3049 simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>;
3052template <
typename _Tp,
typename _V>
3053 using rebind_simd_t =
typename rebind_simd<_Tp, _V>::type;
3056template <
int _Np,
typename _V,
typename =
void>
3059template <
int _Np,
typename _Tp,
typename _Abi>
3060 struct resize_simd<_Np, simd<_Tp, _Abi>,
void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
3061 {
using type = simd<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
3063template <
int _Np,
typename _Tp,
typename _Abi>
3064 struct resize_simd<_Np, simd_mask<_Tp, _Abi>,
void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
3065 {
using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
3067template <
int _Np,
typename _V>
3068 using resize_simd_t =
typename resize_simd<_Np, _V>::type;
3072template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
3073 struct memory_alignment
3074 :
public _SizeConstant<vector_aligned_tag::_S_alignment<_Tp, _Up>> {};
3076template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
3077 inline constexpr size_t memory_alignment_v = memory_alignment<_Tp, _Up>::value;
3080template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
3083template <
typename _Tp,
typename _Abi>
3084 struct is_simd<simd<_Tp, _Abi>> :
public true_type {};
3086template <
typename _Tp>
3087 using native_simd = simd<_Tp, simd_abi::native<_Tp>>;
3089template <
typename _Tp,
int _Np>
3090 using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
3092template <
typename _Tp,
size_t _Np>
3093 using __deduced_simd = simd<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
3096template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
3099template <
typename _Tp,
typename _Abi>
3100 struct is_simd_mask<simd_mask<_Tp, _Abi>> :
public true_type {};
3102template <
typename _Tp>
3103 using native_simd_mask = simd_mask<_Tp, simd_abi::native<_Tp>>;
3105template <
typename _Tp,
int _Np>
3106 using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_Np>>;
3108template <
typename _Tp,
size_t _Np>
3109 using __deduced_simd_mask = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
3113template <
typename _Tp,
typename _Up,
typename _Ap,
bool = is_simd_v<_Tp>,
typename =
void>
3114 struct __static_simd_cast_return_type;
3116template <
typename _Tp,
typename _A0,
typename _Up,
typename _Ap>
3117 struct __static_simd_cast_return_type<simd_mask<_Tp, _A0>, _Up, _Ap, false, void>
3118 : __static_simd_cast_return_type<simd<_Tp, _A0>, _Up, _Ap> {};
3120template <
typename _Tp,
typename _Up,
typename _Ap>
3121 struct __static_simd_cast_return_type<
3122 _Tp, _Up, _Ap, true,
enable_if_t<_Tp::
size() == simd_size_v<_Up, _Ap>>>
3123 {
using type = _Tp; };
3125template <
typename _Tp,
typename _Ap>
3126 struct __static_simd_cast_return_type<_Tp, _Tp, _Ap, false,
3127#ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
3133 {
using type = simd<_Tp, _Ap>; };
3135template <
typename _Tp,
typename =
void>
3136 struct __safe_make_signed {
using type = _Tp;};
3138template <
typename _Tp>
3139 struct __safe_make_signed<_Tp,
enable_if_t<is_integral_v<_Tp>>>
3142 using type = make_signed_t<make_unsigned_t<_Tp>>;
3145template <
typename _Tp>
3146 using safe_make_signed_t =
typename __safe_make_signed<_Tp>::type;
3148template <
typename _Tp,
typename _Up,
typename _Ap>
3149 struct __static_simd_cast_return_type<_Tp, _Up, _Ap, false,
3150#ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
3158 (is_integral_v<_Up> && is_integral_v<_Tp> &&
3159#ifndef _GLIBCXX_SIMD_FIX_P2TS_ISSUE65
3160 is_signed_v<_Up> != is_signed_v<_Tp> &&
3162 is_same_v<safe_make_signed_t<_Up>, safe_make_signed_t<_Tp>>),
3163 simd<_Tp, _Ap>, fixed_size_simd<_Tp, simd_size_v<_Up, _Ap>>>;
3166template <
typename _Tp,
typename _Up,
typename _Ap,
3168 =
typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
3169 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _R
3170 static_simd_cast(
const simd<_Up, _Ap>& __x)
3172 if constexpr (is_same<_R, simd<_Up, _Ap>>::value)
3176 _SimdConverter<_Up, _Ap, typename _R::value_type, typename _R::abi_type>
3178 return _R(__private_init, __c(__data(__x)));
3182namespace __proposed {
3183template <
typename _Tp,
typename _Up,
typename _Ap,
3185 =
typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
3186 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
typename _R::mask_type
3187 static_simd_cast(
const simd_mask<_Up, _Ap>& __x)
3189 using _RM =
typename _R::mask_type;
3190 return {__private_init, _RM::abi_type::_MaskImpl::template _S_convert<
3191 typename _RM::simd_type::value_type>(__x)};
3194template <
typename _To,
typename _Up,
typename _Abi>
3195 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3197 simd_bit_cast(
const simd<_Up, _Abi>& __x)
3199 using _Tp =
typename _To::value_type;
3200 using _ToMember =
typename _SimdTraits<_Tp, typename _To::abi_type>::_SimdMember;
3201 using _From = simd<_Up, _Abi>;
3202 using _FromMember =
typename _SimdTraits<_Up, _Abi>::_SimdMember;
3204 static_assert(
sizeof(_To) ==
sizeof(_From));
3205 static_assert(is_trivially_copyable_v<_Tp> && is_trivially_copyable_v<_Up>);
3206 static_assert(is_trivially_copyable_v<_ToMember> && is_trivially_copyable_v<_FromMember>);
3207#if __has_builtin(__builtin_bit_cast)
3208 return {__private_init, __builtin_bit_cast(_ToMember, __data(__x))};
3210 return {__private_init, __bit_cast<_ToMember>(__data(__x))};
3214template <
typename _To,
typename _Up,
typename _Abi>
3215 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3217 simd_bit_cast(
const simd_mask<_Up, _Abi>& __x)
3219 using _From = simd_mask<_Up, _Abi>;
3220 static_assert(
sizeof(_To) ==
sizeof(_From));
3221 static_assert(is_trivially_copyable_v<_From>);
3224 if constexpr (is_simd_v<_To>)
3226 using _Tp =
typename _To::value_type;
3227 using _ToMember =
typename _SimdTraits<_Tp, typename _To::abi_type>::_SimdMember;
3228 static_assert(is_trivially_copyable_v<_ToMember>);
3229#if __has_builtin(__builtin_bit_cast)
3230 return {__private_init, __builtin_bit_cast(_ToMember, __x)};
3232 return {__private_init, __bit_cast<_ToMember>(__x)};
3237 static_assert(is_trivially_copyable_v<_To>);
3238#if __has_builtin(__builtin_bit_cast)
3239 return __builtin_bit_cast(_To, __x);
3241 return __bit_cast<_To>(__x);
3248template <
typename _Tp,
typename _Up,
typename _Ap,
3249 typename _To = __value_type_or_identity_t<_Tp>>
3250 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
auto
3251 simd_cast(
const simd<_ValuePreserving<_Up, _To>, _Ap>& __x)
3252 ->
decltype(static_simd_cast<_Tp>(__x))
3253 {
return static_simd_cast<_Tp>(__x); }
3255namespace __proposed {
3256template <
typename _Tp,
typename _Up,
typename _Ap,
3257 typename _To = __value_type_or_identity_t<_Tp>>
3258 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
auto
3259 simd_cast(
const simd_mask<_ValuePreserving<_Up, _To>, _Ap>& __x)
3260 ->
decltype(static_simd_cast<_Tp>(__x))
3261 {
return static_simd_cast<_Tp>(__x); }
3266namespace __proposed {
3293template <
typename _Tp,
typename _Up,
typename _Ap>
3294 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
enable_if_t<
3295 conjunction_v<is_simd<_Tp>, is_same<typename _Tp::value_type, _Up>>, _Tp>
3296 resizing_simd_cast(
const simd<_Up, _Ap>& __x)
3298 if constexpr (is_same_v<typename _Tp::abi_type, _Ap>)
3300 else if (__builtin_is_constant_evaluated())
3301 return _Tp([&](
auto __i)
constexpr {
3302 return __i < simd_size_v<_Up, _Ap> ? __x[__i] : _Up();
3304 else if constexpr (simd_size_v<_Up, _Ap> == 1)
3310 else if constexpr (_Tp::size() == 1)
3312 else if constexpr (
sizeof(_Tp) ==
sizeof(__x)
3313 && !__is_fixed_size_abi_v<_Ap> && !__is_sve_abi<_Ap>())
3314 return {__private_init,
3315 __vector_bitcast<typename _Tp::value_type, _Tp::size()>(
3316 _Ap::_S_masked(__data(__x))._M_data)};
3320 __builtin_memcpy(&__data(__r), &__data(__x),
3322 *
std::min(_Tp::size(), simd_size_v<_Up, _Ap>));
3327template <
typename _Tp,
typename _Up,
typename _Ap>
3328 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3329 enable_if_t<is_simd_mask_v<_Tp>, _Tp>
3330 resizing_simd_cast(
const simd_mask<_Up, _Ap>& __x)
3332 return {__private_init, _Tp::abi_type::_MaskImpl::template _S_convert<
3333 typename _Tp::simd_type::value_type>(__x)};
3339template <
typename _Tp,
int _Np>
3340 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, _Np>
3341 to_fixed_size(
const fixed_size_simd<_Tp, _Np>& __x)
3344template <
typename _Tp,
int _Np>
3345 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, _Np>
3346 to_fixed_size(
const fixed_size_simd_mask<_Tp, _Np>& __x)
3349template <
typename _Tp,
typename _Ap>
3350 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, simd_size_v<_Tp, _Ap>>
3351 to_fixed_size(
const simd<_Tp, _Ap>& __x)
3353 using _Rp = fixed_size_simd<_Tp, simd_size_v<_Tp, _Ap>>;
3354 return _Rp([&__x](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
return __x[__i]; });
3357template <
typename _Tp,
typename _Ap>
3358 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, simd_size_v<_Tp, _Ap>>
3359 to_fixed_size(
const simd_mask<_Tp, _Ap>& __x)
3361 return {__private_init,
3362 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
return __x[__i]; }};
3366template <
typename _Tp,
int _Np>
3367 _GLIBCXX_SIMD_INTRINSIC
3368 enable_if_t<(_Np == native_simd<_Tp>::size()), native_simd<_Tp>>
3369 to_native(
const fixed_size_simd<_Tp, _Np>& __x)
3371 alignas(memory_alignment_v<native_simd<_Tp>>) _Tp __mem[_Np];
3372 __x.copy_to(__mem, vector_aligned);
3373 return {__mem, vector_aligned};
3376template <
typename _Tp,
int _Np>
3377 _GLIBCXX_SIMD_INTRINSIC
3378 enable_if_t<(_Np == native_simd_mask<_Tp>::size()), native_simd_mask<_Tp>>
3379 to_native(
const fixed_size_simd_mask<_Tp, _Np>& __x)
3381 return native_simd_mask<_Tp>(
3383 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
return __x[__i]; });
3387template <
typename _Tp,
int _Np>
3388 _GLIBCXX_SIMD_INTRINSIC
enable_if_t<(_Np == simd<_Tp>::size()), simd<_Tp>>
3389 to_compatible(
const simd<_Tp, simd_abi::fixed_size<_Np>>& __x)
3391 alignas(memory_alignment_v<simd<_Tp>>) _Tp __mem[_Np];
3392 __x.copy_to(__mem, vector_aligned);
3393 return {__mem, vector_aligned};
3396template <
typename _Tp,
int _Np>
3397 _GLIBCXX_SIMD_INTRINSIC
3398 enable_if_t<(_Np == simd_mask<_Tp>::size()), simd_mask<_Tp>>
3399 to_compatible(
const simd_mask<_Tp, simd_abi::fixed_size<_Np>>& __x)
3401 return simd_mask<_Tp>(
3403 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
return __x[__i]; });
3410template <
typename _M,
typename _Tp>
3411 class const_where_expression
3414 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3416 struct _Wrapper {
using value_type = _V; };
3419 using _Impl =
typename _V::_Impl;
3422 typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3424 _GLIBCXX_SIMD_INTRINSIC
friend const _M&
3425 __get_mask(
const const_where_expression& __x)
3426 {
return __x._M_k; }
3428 _GLIBCXX_SIMD_INTRINSIC
friend const _Tp&
3429 __get_lvalue(
const const_where_expression& __x)
3430 {
return __x._M_value; }
3436 const_where_expression(
const const_where_expression&) =
delete;
3438 const_where_expression& operator=(
const const_where_expression&) =
delete;
3440 _GLIBCXX_SIMD_INTRINSIC
constexpr
3441 const_where_expression(
const _M& __kk,
const _Tp& dd)
3442 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3444 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3447 return {__private_init,
3448 _Impl::template _S_masked_unary<negate>(__data(_M_k),
3452 template <
typename _Up,
typename _Flags>
3453 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3454 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>)
const&&
3456 return {__private_init,
3457 _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3458 _Flags::template _S_apply<_V>(__mem))};
3461 template <
typename _Up,
typename _Flags>
3462 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3463 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>)
const&&
3465 _Impl::_S_masked_store(__data(_M_value),
3466 _Flags::template _S_apply<_V>(__mem),
3472template <
typename _Tp>
3473 class const_where_expression<bool, _Tp>
3478 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3480 struct _Wrapper {
using value_type = _V; };
3484 =
typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3486 _GLIBCXX_SIMD_INTRINSIC
friend const _M&
3487 __get_mask(
const const_where_expression& __x)
3488 {
return __x._M_k; }
3490 _GLIBCXX_SIMD_INTRINSIC
friend const _Tp&
3491 __get_lvalue(
const const_where_expression& __x)
3492 {
return __x._M_value; }
3498 const_where_expression(
const const_where_expression&) =
delete;
3499 const_where_expression& operator=(
const const_where_expression&) =
delete;
3501 _GLIBCXX_SIMD_INTRINSIC
constexpr
3502 const_where_expression(
const bool __kk,
const _Tp& dd)
3503 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3505 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3507 {
return _M_k ? -_M_value : _M_value; }
3509 template <
typename _Up,
typename _Flags>
3510 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3511 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>)
const&&
3512 {
return _M_k ?
static_cast<_V
>(__mem[0]) : _M_value; }
3514 template <
typename _Up,
typename _Flags>
3515 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3516 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>)
const&&
3519 __mem[0] = _M_value;
3524template <
typename _M,
typename _Tp>
3525 class where_expression :
public const_where_expression<_M, _Tp>
3527 using _Impl =
typename const_where_expression<_M, _Tp>::_Impl;
3529 static_assert(!is_const<_Tp>::value,
3530 "where_expression may only be instantiated with __a non-const "
3533 using typename const_where_expression<_M, _Tp>::value_type;
3534 using const_where_expression<_M, _Tp>::_M_k;
3535 using const_where_expression<_M, _Tp>::_M_value;
3538 is_same<typename _M::abi_type, typename _Tp::abi_type>::value,
"");
3539 static_assert(_M::size() == _Tp::size(),
"");
3541 _GLIBCXX_SIMD_INTRINSIC
friend constexpr _Tp&
3542 __get_lvalue(where_expression& __x)
3543 {
return __x._M_value; }
3546 where_expression(
const where_expression&) =
delete;
3547 where_expression& operator=(
const where_expression&) =
delete;
3549 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3550 where_expression(
const _M& __kk, _Tp& dd)
3551 : const_where_expression<_M, _Tp>(__kk, dd) {}
3553 template <
typename _Up>
3554 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3555 operator=(_Up&& __x) &&
3557 _Impl::_S_masked_assign(__data(_M_k), __data(_M_value),
3558 __to_value_type_or_member_type<_Tp>(
3559 static_cast<_Up&&
>(__x)));
3562#define _GLIBCXX_SIMD_OP_(__op, __name) \
3563 template <typename _Up> \
3564 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void \
3565 operator __op##=(_Up&& __x)&& \
3567 _Impl::template _S_masked_cassign( \
3568 __data(_M_k), __data(_M_value), \
3569 __to_value_type_or_member_type<_Tp>(static_cast<_Up&&>(__x)), \
3570 [](auto __impl, auto __lhs, auto __rhs) \
3571 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \
3572 { return __impl.__name(__lhs, __rhs); }); \
3575 _GLIBCXX_SIMD_OP_(+, _S_plus);
3576 _GLIBCXX_SIMD_OP_(-, _S_minus);
3577 _GLIBCXX_SIMD_OP_(*, _S_multiplies);
3578 _GLIBCXX_SIMD_OP_(/, _S_divides);
3579 _GLIBCXX_SIMD_OP_(%, _S_modulus);
3580 _GLIBCXX_SIMD_OP_(&, _S_bit_and);
3581 _GLIBCXX_SIMD_OP_(|, _S_bit_or);
3582 _GLIBCXX_SIMD_OP_(^, _S_bit_xor);
3583 _GLIBCXX_SIMD_OP_(<<, _S_shift_left);
3584 _GLIBCXX_SIMD_OP_(>>, _S_shift_right);
3585#undef _GLIBCXX_SIMD_OP_
3587 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3591 = _Impl::template _S_masked_unary<__increment>(__data(_M_k), __data(_M_value));
3594 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3598 = _Impl::template _S_masked_unary<__increment>(__data(_M_k), __data(_M_value));
3601 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3605 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k), __data(_M_value));
3608 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3612 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k), __data(_M_value));
3616 template <
typename _Up,
typename _Flags>
3617 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3618 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) &&
3620 __data(_M_value) = _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3621 _Flags::template _S_apply<_Tp>(__mem));
3626template <
typename _Tp>
3627 class where_expression<bool, _Tp>
3628 :
public const_where_expression<bool, _Tp>
3631 using typename const_where_expression<_M, _Tp>::value_type;
3632 using const_where_expression<_M, _Tp>::_M_k;
3633 using const_where_expression<_M, _Tp>::_M_value;
3636 where_expression(
const where_expression&) =
delete;
3637 where_expression& operator=(
const where_expression&) =
delete;
3639 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3640 where_expression(
const _M& __kk, _Tp& dd)
3641 : const_where_expression<_M, _Tp>(__kk, dd) {}
3643#define _GLIBCXX_SIMD_OP_(__op) \
3644 template <typename _Up> \
3645 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void \
3646 operator __op(_Up&& __x)&& \
3647 { if (_M_k) _M_value __op static_cast<_Up&&>(__x); }
3649 _GLIBCXX_SIMD_OP_(=)
3650 _GLIBCXX_SIMD_OP_(+=)
3651 _GLIBCXX_SIMD_OP_(-=)
3652 _GLIBCXX_SIMD_OP_(*=)
3653 _GLIBCXX_SIMD_OP_(/=)
3654 _GLIBCXX_SIMD_OP_(%=)
3655 _GLIBCXX_SIMD_OP_(&=)
3656 _GLIBCXX_SIMD_OP_(|=)
3657 _GLIBCXX_SIMD_OP_(^=)
3658 _GLIBCXX_SIMD_OP_(<<=)
3659 _GLIBCXX_SIMD_OP_(>>=)
3660 #undef _GLIBCXX_SIMD_OP_
3662 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3664 {
if (_M_k) ++_M_value; }
3666 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3668 {
if (_M_k) ++_M_value; }
3670 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3672 {
if (_M_k) --_M_value; }
3674 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3676 {
if (_M_k) --_M_value; }
3679 template <
typename _Up,
typename _Flags>
3680 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
void
3681 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) &&
3682 {
if (_M_k) _M_value = __mem[0]; }
3686template <
typename _Tp,
typename _Ap>
3687 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3688 where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3689 where(
const typename simd<_Tp, _Ap>::mask_type& __k, simd<_Tp, _Ap>& __value)
3690 {
return {__k, __value}; }
3692template <
typename _Tp,
typename _Ap>
3693 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3694 const_where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3695 where(
const typename simd<_Tp, _Ap>::mask_type& __k,
const simd<_Tp, _Ap>& __value)
3696 {
return {__k, __value}; }
3698template <
typename _Tp,
typename _Ap>
3699 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3700 where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3701 where(
const remove_const_t<simd_mask<_Tp, _Ap>>& __k, simd_mask<_Tp, _Ap>& __value)
3702 {
return {__k, __value}; }
3704template <
typename _Tp,
typename _Ap>
3705 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3706 const_where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3707 where(
const remove_const_t<simd_mask<_Tp, _Ap>>& __k,
const simd_mask<_Tp, _Ap>& __value)
3708 {
return {__k, __value}; }
3710template <
typename _Tp>
3711 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR where_expression<bool, _Tp>
3712 where(_ExactBool __k, _Tp& __value)
3713 {
return {__k, __value}; }
3715template <
typename _Tp>
3716 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR const_where_expression<bool, _Tp>
3717 where(_ExactBool __k,
const _Tp& __value)
3718 {
return {__k, __value}; }
3720template <
typename _Tp,
typename _Ap>
3721 _GLIBCXX_SIMD_CONSTEXPR
void
3722 where(
bool __k, simd<_Tp, _Ap>& __value) =
delete;
3724template <
typename _Tp,
typename _Ap>
3725 _GLIBCXX_SIMD_CONSTEXPR
void
3726 where(
bool __k,
const simd<_Tp, _Ap>& __value) =
delete;
3729namespace __proposed {
3730template <
size_t _Np>
3733 const bitset<_Np> __bits;
3736 where_range(bitset<_Np> __b) : __bits(__b) {}
3743 _GLIBCXX_SIMD_INTRINSIC
void
3745 { __bit = __builtin_ctzl(__mask); }
3747 _GLIBCXX_SIMD_INTRINSIC
void
3751 __mask &= (__mask - 1);
3756 iterator(
decltype(__mask) __m) : __mask(__m) { __next_bit(); }
3757 iterator(
const iterator&) =
default;
3758 iterator(iterator&&) =
default;
3760 _GLIBCXX_SIMD_ALWAYS_INLINE
size_t
3764 _GLIBCXX_SIMD_ALWAYS_INLINE
size_t
3768 _GLIBCXX_SIMD_ALWAYS_INLINE iterator&
3776 _GLIBCXX_SIMD_ALWAYS_INLINE iterator
3779 iterator __tmp = *
this;
3785 _GLIBCXX_SIMD_ALWAYS_INLINE
bool
3786 operator==(
const iterator& __rhs)
const
3787 {
return __mask == __rhs.__mask; }
3789 _GLIBCXX_SIMD_ALWAYS_INLINE
bool
3790 operator!=(
const iterator& __rhs)
const
3791 {
return __mask != __rhs.__mask; }
3796 {
return __bits.to_ullong(); }
3803template <
typename _Tp,
typename _Ap>
3804 where_range<simd_size_v<_Tp, _Ap>>
3805 where(
const simd_mask<_Tp, _Ap>& __k)
3806 {
return __k.__to_bitset(); }
3812template <
typename _Tp,
typename _Abi,
typename _BinaryOperation = plus<>>
3813 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3814 reduce(
const simd<_Tp, _Abi>& __v, _BinaryOperation __binary_op = _BinaryOperation())
3815 {
return _Abi::_SimdImpl::_S_reduce(__v, __binary_op); }
3817template <
typename _M,
typename _V,
typename _BinaryOperation = plus<>>
3818 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3819 reduce(
const const_where_expression<_M, _V>& __x,
3820 typename _V::value_type __identity_element, _BinaryOperation __binary_op)
3822 if (__builtin_expect(none_of(__get_mask(__x)),
false))
3823 return __identity_element;
3825 _V __tmp = __identity_element;
3826 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3827 __data(__get_lvalue(__x)));
3828 return reduce(__tmp, __binary_op);
3831template <
typename _M,
typename _V>
3832 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3833 reduce(
const const_where_expression<_M, _V>& __x, plus<> __binary_op = {})
3834 {
return reduce(__x, 0, __binary_op); }
3836template <
typename _M,
typename _V>
3837 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3838 reduce(
const const_where_expression<_M, _V>& __x, multiplies<> __binary_op)
3839 {
return reduce(__x, 1, __binary_op); }
3841template <
typename _M,
typename _V>
3842 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3843 reduce(
const const_where_expression<_M, _V>& __x, bit_and<> __binary_op)
3844 {
return reduce(__x, ~
typename _V::value_type(), __binary_op); }
3846template <
typename _M,
typename _V>
3847 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3848 reduce(
const const_where_expression<_M, _V>& __x, bit_or<> __binary_op)
3849 {
return reduce(__x, 0, __binary_op); }
3851template <
typename _M,
typename _V>
3852 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3853 reduce(
const const_where_expression<_M, _V>& __x, bit_xor<> __binary_op)
3854 {
return reduce(__x, 0, __binary_op); }
3856template <
typename _Tp,
typename _Abi>
3857 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3858 hmin(
const simd<_Tp, _Abi>& __v)
noexcept
3859 {
return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Minimum()); }
3861template <
typename _Tp,
typename _Abi>
3862 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3863 hmax(
const simd<_Tp, _Abi>& __v)
noexcept
3864 {
return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Maximum()); }
3866template <
typename _M,
typename _V>
3867 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3868 typename _V::value_type
3869 hmin(
const const_where_expression<_M, _V>& __x)
noexcept
3871 using _Tp =
typename _V::value_type;
3872 constexpr _Tp __id_elem =
3873#ifdef __FINITE_MATH_ONLY__
3874 __finite_max_v<_Tp>;
3876 __value_or<__infinity, _Tp>(__finite_max_v<_Tp>);
3878 _V __tmp = __id_elem;
3879 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3880 __data(__get_lvalue(__x)));
3881 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Minimum());
3884template <
typename _M,
typename _V>
3885 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3886 typename _V::value_type
3887 hmax(
const const_where_expression<_M, _V>& __x)
noexcept
3889 using _Tp =
typename _V::value_type;
3890 constexpr _Tp __id_elem =
3891#ifdef __FINITE_MATH_ONLY__
3892 __finite_min_v<_Tp>;
3895 if constexpr (__value_exists_v<__infinity, _Tp>)
3896 return -__infinity_v<_Tp>;
3898 return __finite_min_v<_Tp>;
3901 _V __tmp = __id_elem;
3902 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3903 __data(__get_lvalue(__x)));
3904 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Maximum());
3909template <
typename _Tp,
typename _Ap>
3910 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3911 min(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3912 {
return {__private_init, _Ap::_SimdImpl::_S_min(__data(__a), __data(__b))}; }
3914template <
typename _Tp,
typename _Ap>
3915 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3916 max(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3917 {
return {__private_init, _Ap::_SimdImpl::_S_max(__data(__a), __data(__b))}; }
3919template <
typename _Tp,
typename _Ap>
3920 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3921 pair<simd<_Tp, _Ap>, simd<_Tp, _Ap>>
3922 minmax(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3924 const auto pair_of_members
3925 = _Ap::_SimdImpl::_S_minmax(__data(__a), __data(__b));
3926 return {simd<_Tp, _Ap>(__private_init, pair_of_members.first),
3927 simd<_Tp, _Ap>(__private_init, pair_of_members.second)};
3930template <
typename _Tp,
typename _Ap>
3931 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3932 clamp(
const simd<_Tp, _Ap>& __v,
const simd<_Tp, _Ap>& __lo,
const simd<_Tp, _Ap>& __hi)
3934 using _Impl =
typename _Ap::_SimdImpl;
3935 return {__private_init,
3936 _Impl::_S_min(__data(__hi),
3937 _Impl::_S_max(__data(__lo), __data(__v)))};
3942template <
size_t... _Sizes,
typename _Tp,
typename _Ap,
3943 typename =
enable_if_t<((_Sizes + ...) == simd<_Tp, _Ap>::size())>>
3944 inline tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3945 split(
const simd<_Tp, _Ap>&);
3948template <
int _Index,
int _Total,
int _Combine = 1,
typename _Tp,
size_t _Np>
3949 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST
constexpr
3950 _SimdWrapper<_Tp, _Np / _Total * _Combine>
3951 __extract_part(
const _SimdWrapper<_Tp, _Np> __x);
3953template <
int _Index,
int _Parts,
int _Combine = 1,
typename _Tp,
typename _A0,
typename... _As>
3954 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
3955 __extract_part(
const _SimdTuple<_Tp, _A0, _As...>& __x);
3959template <
size_t _V0,
size_t... _Values>
3962 template <
size_t _I>
3963 static constexpr size_t
3964 _S_at(_SizeConstant<_I> = {})
3966 if constexpr (_I == 0)
3969 return _SizeList<_Values...>::template _S_at<_I - 1>();
3972 template <
size_t _I>
3973 static constexpr auto
3974 _S_before(_SizeConstant<_I> = {})
3976 if constexpr (_I == 0)
3977 return _SizeConstant<0>();
3979 return _SizeConstant<
3980 _V0 + _SizeList<_Values...>::template _S_before<_I - 1>()>();
3983 template <
size_t _Np>
3984 static constexpr auto
3985 _S_pop_front(_SizeConstant<_Np> = {})
3987 if constexpr (_Np == 0)
3990 return _SizeList<_Values...>::template _S_pop_front<_Np - 1>();
3996template <
typename _Tp,
size_t _Np>
3997 _GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _Np / 2>
3998 __extract_center(_SimdWrapper<_Tp, _Np> __x)
4000 static_assert(_Np >= 4);
4001 static_assert(_Np % 4 == 0);
4002#if _GLIBCXX_SIMD_X86INTRIN
4003 if constexpr (__have_avx512f &&
sizeof(_Tp) * _Np == 64)
4005 const auto __intrin = __to_intrin(__x);
4006 if constexpr (is_integral_v<_Tp>)
4007 return __vector_bitcast<_Tp>(_mm512_castsi512_si256(
4008 _mm512_shuffle_i32x4(__intrin, __intrin,
4009 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
4010 else if constexpr (
sizeof(_Tp) == 4)
4011 return __vector_bitcast<_Tp>(_mm512_castps512_ps256(
4012 _mm512_shuffle_f32x4(__intrin, __intrin,
4013 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
4014 else if constexpr (
sizeof(_Tp) == 8)
4015 return __vector_bitcast<_Tp>(_mm512_castpd512_pd256(
4016 _mm512_shuffle_f64x2(__intrin, __intrin,
4017 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
4019 __assert_unreachable<_Tp>();
4021 else if constexpr (
sizeof(_Tp) * _Np == 32 && is_floating_point_v<_Tp>)
4022 return __vector_bitcast<_Tp>(
4023 _mm_shuffle_pd(__lo128(__vector_bitcast<double>(__x)),
4024 __hi128(__vector_bitcast<double>(__x)), 1));
4025 else if constexpr (
sizeof(__x) == 32 &&
sizeof(_Tp) * _Np <= 32)
4026 return __vector_bitcast<_Tp>(
4027 _mm_alignr_epi8(__hi128(__vector_bitcast<_LLong>(__x)),
4028 __lo128(__vector_bitcast<_LLong>(__x)),
4029 sizeof(_Tp) * _Np / 4));
4033 __vector_type_t<_Tp, _Np / 2> __r;
4034 __builtin_memcpy(&__r,
4035 reinterpret_cast<const char*
>(&__x)
4036 +
sizeof(_Tp) * _Np / 4,
4037 sizeof(_Tp) * _Np / 2);
4042template <
typename _Tp,
typename _A0,
typename... _As>
4043 _GLIBCXX_SIMD_INTRINSIC
4044 _SimdWrapper<_Tp, _SimdTuple<_Tp, _A0, _As...>::_S_size() / 2>
4045 __extract_center(
const _SimdTuple<_Tp, _A0, _As...>& __x)
4047 if constexpr (
sizeof...(_As) == 0)
4048 return __extract_center(__x.first);
4050 return __extract_part<1, 4, 2>(__x);
4055template <
size_t... _Sizes,
typename _Tp,
typename... _As>
4057 __split_wrapper(_SizeList<_Sizes...>,
const _SimdTuple<_Tp, _As...>& __x)
4059 return split<_Sizes...>(
4060 fixed_size_simd<_Tp, _SimdTuple<_Tp, _As...>::_S_size()>(__private_init,
4067template <
typename _V,
typename _Ap,
4068 size_t _Parts = simd_size_v<typename _V::value_type, _Ap> / _V::size()>
4069 enable_if_t<simd_size_v<typename _V::value_type, _Ap> == _Parts * _V::size()
4070 && is_simd_v<_V>, array<_V, _Parts>>
4071 split(
const simd<typename _V::value_type, _Ap>& __x)
4073 using _Tp =
typename _V::value_type;
4075 auto __gen_fallback = [&]()
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4076 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4077 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4078 return _V([&](
auto __j)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
4079 {
return __x[__i * _V::size() + __j]; });
4083 if constexpr (_Parts == 1)
4085 return {simd_cast<_V>(__x)};
4087 else if (__x._M_is_constprop())
4089 return __gen_fallback();
4091#if _GLIBCXX_SIMD_HAVE_SVE
4092 else if constexpr(__is_sve_abi<_Ap>)
4094 return __gen_fallback();
4098 __is_fixed_size_abi_v<_Ap>
4099 && (is_same_v<typename _V::abi_type, simd_abi::scalar>
4100 || (__is_fixed_size_abi_v<typename _V::abi_type>
4101 &&
sizeof(_V) ==
sizeof(_Tp) * _V::size()
4105#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
4106 const __may_alias<_Tp>*
const __element_ptr
4107 =
reinterpret_cast<const __may_alias<_Tp>*
>(&__data(__x));
4108 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4109 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
4110 {
return _V(__element_ptr + __i * _V::size(), vector_aligned); });
4112 const auto& __xx = __data(__x);
4113 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4114 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4115 [[maybe_unused]]
constexpr size_t __offset
4116 =
decltype(__i)::value * _V::size();
4117 return _V([&](
auto __j)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4118 constexpr _SizeConstant<__j + __offset> __k;
4124 else if constexpr (is_same_v<typename _V::abi_type, simd_abi::scalar>)
4127 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4128 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
return __x[__i]; });
4132 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4133 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4134 if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
4135 return _V([&](
auto __j)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4136 return __x[__i * _V::size() + __j];
4139 return _V(__private_init,
4140 __extract_part<
decltype(__i)::value, _Parts>(__data(__x)));
4147template <
typename _V,
typename _Ap,
4148 size_t _Parts = simd_size_v<typename _V::simd_type::value_type, _Ap> / _V::size()>
4149 enable_if_t<is_simd_mask_v<_V> && simd_size_v<
typename
4150 _V::simd_type::value_type, _Ap> == _Parts * _V::size(), array<_V, _Parts>>
4151 split(
const simd_mask<typename _V::simd_type::value_type, _Ap>& __x)
4153 if constexpr (is_same_v<_Ap, typename _V::abi_type>)
4155 else if constexpr (_Parts == 1)
4156 return {__proposed::static_simd_cast<_V>(__x)};
4157 else if constexpr (_Parts == 2 && __is_sse_abi<typename _V::abi_type>()
4158 && __is_avx_abi<_Ap>())
4159 return {_V(__private_init, __lo128(__data(__x))),
4160 _V(__private_init, __hi128(__data(__x)))};
4161 else if constexpr (_V::size() <= __CHAR_BIT__ *
sizeof(_ULLong))
4163 const bitset __bits = __x.__to_bitset();
4164 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4165 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4166 constexpr size_t __offset = __i * _V::size();
4167 return _V(__bitset_init, (__bits >> __offset).to_ullong());
4172 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4173 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4174 constexpr size_t __offset = __i * _V::size();
4175 return _V(__private_init,
4176 [&](
auto __j)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4177 return __x[__j + __offset];
4185template <
size_t... _Sizes,
typename _Tp,
typename _Ap,
typename>
4186 _GLIBCXX_SIMD_ALWAYS_INLINE
4187 tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
4188 split(
const simd<_Tp, _Ap>& __x)
4190 using _SL = _SizeList<_Sizes...>;
4191 using _Tuple = tuple<__deduced_simd<_Tp, _Sizes>...>;
4192 constexpr size_t _Np = simd_size_v<_Tp, _Ap>;
4193 constexpr size_t _N0 = _SL::template _S_at<0>();
4194 using _V = __deduced_simd<_Tp, _N0>;
4196 auto __gen_fallback = [&]()
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
4198 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>(
4199 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4200 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4201 constexpr size_t __offset = _SL::_S_before(__i);
4202 return _Vi([&](
auto __j)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4203 return __x[__offset + __j];
4208 if (__x._M_is_constprop())
4210#if _GLIBCXX_SIMD_HAVE_SVE
4211 else if constexpr (__have_sve)
4214 else if constexpr (_Np == _N0)
4216 static_assert(
sizeof...(_Sizes) == 1);
4217 return {simd_cast<_V>(__x)};
4220 (__is_fixed_size_abi_v<
4221 _Ap> && __fixed_size_storage_t<_Tp, _Np>::_S_first_size == _N0)
4224 !__is_fixed_size_abi_v<typename _V::abi_type>,
4225 "How can <_Tp, _Np> be __a single _SimdTuple entry but __a "
4231 __split_wrapper(_SL::template _S_pop_front<1>(),
4232 __data(__x).second));
4234 else if constexpr ((!is_same_v<simd_abi::scalar,
4235 simd_abi::deduce_t<_Tp, _Sizes>> && ...)
4236 && (!__is_fixed_size_abi_v<
4237 simd_abi::deduce_t<_Tp, _Sizes>> && ...))
4239 if constexpr (((_Sizes * 2 == _Np) && ...))
4240 return {{__private_init, __extract_part<0, 2>(__data(__x))},
4241 {__private_init, __extract_part<1, 2>(__data(__x))}};
4242 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4243 _SizeList<_Np / 3, _Np / 3, _Np / 3>>)
4244 return {{__private_init, __extract_part<0, 3>(__data(__x))},
4245 {__private_init, __extract_part<1, 3>(__data(__x))},
4246 {__private_init, __extract_part<2, 3>(__data(__x))}};
4247 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4248 _SizeList<2 * _Np / 3, _Np / 3>>)
4249 return {{__private_init, __extract_part<0, 3, 2>(__data(__x))},
4250 {__private_init, __extract_part<2, 3>(__data(__x))}};
4251 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4252 _SizeList<_Np / 3, 2 * _Np / 3>>)
4253 return {{__private_init, __extract_part<0, 3>(__data(__x))},
4254 {__private_init, __extract_part<1, 3, 2>(__data(__x))}};
4255 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4256 _SizeList<_Np / 2, _Np / 4, _Np / 4>>)
4257 return {{__private_init, __extract_part<0, 2>(__data(__x))},
4258 {__private_init, __extract_part<2, 4>(__data(__x))},
4259 {__private_init, __extract_part<3, 4>(__data(__x))}};
4260 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4261 _SizeList<_Np / 4, _Np / 4, _Np / 2>>)
4262 return {{__private_init, __extract_part<0, 4>(__data(__x))},
4263 {__private_init, __extract_part<1, 4>(__data(__x))},
4264 {__private_init, __extract_part<1, 2>(__data(__x))}};
4265 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4266 _SizeList<_Np / 4, _Np / 2, _Np / 4>>)
4267 return {{__private_init, __extract_part<0, 4>(__data(__x))},
4268 {__private_init, __extract_center(__data(__x))},
4269 {__private_init, __extract_part<3, 4>(__data(__x))}};
4270 else if constexpr (((_Sizes * 4 == _Np) && ...))
4271 return {{__private_init, __extract_part<0, 4>(__data(__x))},
4272 {__private_init, __extract_part<1, 4>(__data(__x))},
4273 {__private_init, __extract_part<2, 4>(__data(__x))},
4274 {__private_init, __extract_part<3, 4>(__data(__x))}};
4277#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
4278 const __may_alias<_Tp>*
const __element_ptr
4279 =
reinterpret_cast<const __may_alias<_Tp>*
>(&__x);
4280 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>(
4281 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4282 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4283 constexpr size_t __offset = _SL::_S_before(__i);
4284 constexpr size_t __base_align =
alignof(simd<_Tp, _Ap>);
4285 constexpr size_t __a
4286 = __base_align - ((__offset *
sizeof(_Tp)) % __base_align);
4287 constexpr size_t __b = ((__a - 1) & __a) ^ __a;
4288 constexpr size_t __alignment = __b == 0 ? __a : __b;
4289 return _Vi(__element_ptr + __offset, overaligned<__alignment>);
4292 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>(
4293 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4294 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4295 const auto& __xx = __data(__x);
4296 using _Offset =
decltype(_SL::_S_before(__i));
4297 return _Vi([&](
auto __j)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4298 constexpr _SizeConstant<_Offset::value + __j> __k;
4308template <
size_t _I,
typename _Tp,
typename _Ap,
typename... _As>
4309 _GLIBCXX_SIMD_INTRINSIC
constexpr _Tp
4310 __subscript_in_pack(
const simd<_Tp, _Ap>& __x,
const simd<_Tp, _As>&... __xs)
4312 if constexpr (_I < simd_size_v<_Tp, _Ap>)
4315 return __subscript_in_pack<_I - simd_size_v<_Tp, _Ap>>(__xs...);
4320template <
typename _Tp,
typename _A0,
typename... _As>
4321 _GLIBCXX_SIMD_INTRINSIC
void
4322 __store_pack_of_simd(
char* __mem,
const simd<_Tp, _A0>& __x0,
const simd<_Tp, _As>&... __xs)
4324 constexpr size_t __n_bytes =
sizeof(_Tp) * simd_size_v<_Tp, _A0>;
4325 __builtin_memcpy(__mem, &__data(__x0), __n_bytes);
4326 if constexpr (
sizeof...(__xs) > 0)
4327 __store_pack_of_simd(__mem + __n_bytes, __xs...);
4332template <
typename _Tp,
typename... _As,
typename = __detail::__odr_helper>
4333 inline _GLIBCXX_SIMD_CONSTEXPR
4334 simd<_Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>
4335 concat(
const simd<_Tp, _As>&... __xs)
4337 using _Rp = __deduced_simd<_Tp, (simd_size_v<_Tp, _As> + ...)>;
4338 if constexpr (
sizeof...(__xs) == 1)
4339 return simd_cast<_Rp>(__xs...);
4340 else if ((... && __xs._M_is_constprop()))
4342 simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>(
4343 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
4344 {
return __subscript_in_pack<__i>(__xs...); });
4348 __store_pack_of_simd(
reinterpret_cast<char*
>(&__data(__r)), __xs...);
4355template <
typename _Tp,
typename _Abi,
size_t _Np>
4356 _GLIBCXX_SIMD_ALWAYS_INLINE
4357 _GLIBCXX_SIMD_CONSTEXPR __deduced_simd<_Tp, simd_size_v<_Tp, _Abi> * _Np>
4358 concat(
const array<simd<_Tp, _Abi>, _Np>& __x)
4360 return __call_with_subscripts<_Np>(
4361 __x, [](
const auto&... __xs) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4362 return concat(__xs...);
4370template <
typename _Up,
typename _Accessor = _Up,
4371 typename _ValueType =
typename _Up::value_type>
4372 class _SmartReference
4378 _GLIBCXX_SIMD_INTRINSIC
constexpr _ValueType
4379 _M_read() const noexcept
4381 if constexpr (is_arithmetic_v<_Up>)
4384 return _M_obj[_M_index];
4387 template <
typename _Tp>
4388 _GLIBCXX_SIMD_INTRINSIC
constexpr void
4389 _M_write(_Tp&& __x)
const
4390 { _Accessor::_S_set(_M_obj, _M_index,
static_cast<_Tp&&
>(__x)); }
4393 _GLIBCXX_SIMD_INTRINSIC
constexpr
4394 _SmartReference(_Up& __o,
int __i) noexcept
4395 : _M_index(__i), _M_obj(__o) {}
4397 using value_type = _ValueType;
4399 _GLIBCXX_SIMD_INTRINSIC
4400 _SmartReference(
const _SmartReference&) =
delete;
4402 _GLIBCXX_SIMD_INTRINSIC
constexpr
4403 operator value_type() const noexcept
4404 {
return _M_read(); }
4406 template <
typename _Tp,
typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, value_type>>
4407 _GLIBCXX_SIMD_INTRINSIC
constexpr _SmartReference
4408 operator=(_Tp&& __x) &&
4410 _M_write(
static_cast<_Tp&&
>(__x));
4411 return {_M_obj, _M_index};
4414#define _GLIBCXX_SIMD_OP_(__op) \
4415 template <typename _Tp, \
4416 typename _TT = decltype(declval<value_type>() __op declval<_Tp>()), \
4417 typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, _TT>, \
4418 typename = _ValuePreservingOrInt<_TT, value_type>> \
4419 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference \
4420 operator __op##=(_Tp&& __x) && \
4422 const value_type& __lhs = _M_read(); \
4423 _M_write(__lhs __op __x); \
4424 return {_M_obj, _M_index}; \
4426 _GLIBCXX_SIMD_ALL_ARITHMETICS(_GLIBCXX_SIMD_OP_);
4427 _GLIBCXX_SIMD_ALL_SHIFTS(_GLIBCXX_SIMD_OP_);
4428 _GLIBCXX_SIMD_ALL_BINARY(_GLIBCXX_SIMD_OP_);
4429#undef _GLIBCXX_SIMD_OP_
4431 template <
typename _Tp = void,
4432 typename =
decltype(++declval<conditional_t<true, value_type, _Tp>&>())>
4433 _GLIBCXX_SIMD_INTRINSIC
constexpr _SmartReference
4436 value_type __x = _M_read();
4438 return {_M_obj, _M_index};
4441 template <
typename _Tp = void,
4442 typename =
decltype(declval<conditional_t<true, value_type, _Tp>&>()++)>
4443 _GLIBCXX_SIMD_INTRINSIC
constexpr value_type
4446 const value_type __r = _M_read();
4447 value_type __x = __r;
4452 template <
typename _Tp = void,
4453 typename =
decltype(--declval<conditional_t<true, value_type, _Tp>&>())>
4454 _GLIBCXX_SIMD_INTRINSIC
constexpr _SmartReference
4457 value_type __x = _M_read();
4459 return {_M_obj, _M_index};
4462 template <
typename _Tp = void,
4463 typename =
decltype(declval<conditional_t<true, value_type, _Tp>&>()--)>
4464 _GLIBCXX_SIMD_INTRINSIC
constexpr value_type
4467 const value_type __r = _M_read();
4468 value_type __x = __r;
4473 _GLIBCXX_SIMD_INTRINSIC
friend void
4474 swap(_SmartReference&& __a, _SmartReference&& __b)
noexcept(
4476 is_nothrow_constructible<value_type, _SmartReference&&>,
4477 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4479 value_type __tmp =
static_cast<_SmartReference&&
>(__a);
4480 static_cast<_SmartReference&&
>(__a) =
static_cast<value_type
>(__b);
4481 static_cast<_SmartReference&&
>(__b) =
std::move(__tmp);
4484 _GLIBCXX_SIMD_INTRINSIC
friend void
4485 swap(value_type& __a, _SmartReference&& __b)
noexcept(
4487 is_nothrow_constructible<value_type, value_type&&>,
4488 is_nothrow_assignable<value_type&, value_type&&>,
4489 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4492 __a =
static_cast<value_type
>(__b);
4493 static_cast<_SmartReference&&
>(__b) =
std::move(__tmp);
4496 _GLIBCXX_SIMD_INTRINSIC
friend void
4497 swap(_SmartReference&& __a, value_type& __b)
noexcept(
4499 is_nothrow_constructible<value_type, _SmartReference&&>,
4500 is_nothrow_assignable<value_type&, value_type&&>,
4501 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4503 value_type __tmp(__a);
4504 static_cast<_SmartReference&&
>(__a) =
std::move(__b);
4511template <
int _Bytes>
4512 struct __scalar_abi_wrapper
4514 template <
typename _Tp>
static constexpr size_t _S_full_size = 1;
4515 template <
typename _Tp>
static constexpr size_t _S_size = 1;
4516 template <
typename _Tp>
static constexpr size_t _S_is_partial =
false;
4518 template <
typename _Tp,
typename _Abi = simd_abi::scalar>
4519 static constexpr bool _S_is_valid_v
4520 = _Abi::template _IsValid<_Tp>::value &&
sizeof(_Tp) == _Bytes;
4525template <
typename _Tp>
4526 struct __decay_abi {
using type = _Tp; };
4528template <
int _Bytes>
4529 struct __decay_abi<__scalar_abi_wrapper<_Bytes>>
4530 {
using type = simd_abi::scalar; };
4538template <
template <
int>
class _Abi,
int _Bytes,
typename _Tp>
4539 struct __find_next_valid_abi
4541 static constexpr auto
4544 constexpr int _NextBytes = std::__bit_ceil(_Bytes) / 2;
4545 using _NextAbi = _Abi<_NextBytes>;
4546 if constexpr (_NextBytes <
sizeof(_Tp) * 2)
4547 return _Abi<_Bytes>();
4548 else if constexpr (_NextAbi::template _S_is_partial<_Tp> ==
false
4549 && _NextAbi::template _S_is_valid_v<_Tp>)
4552 return __find_next_valid_abi<_Abi, _NextBytes, _Tp>::_S_choose();
4555 using type =
decltype(_S_choose());
4558template <
int _Bytes,
typename _Tp>
4559 struct __find_next_valid_abi<__scalar_abi_wrapper, _Bytes, _Tp>
4560 {
using type = simd_abi::scalar; };
4563template <
template <
int>
class...>
4566 template <
typename,
int>
static constexpr bool _S_has_valid_abi =
false;
4567 template <
typename,
int>
using _FirstValidAbi = void;
4568 template <
typename,
int>
using _BestAbi = void;
4571template <
template <
int>
class _A0,
template <
int>
class... _Rest>
4572 struct _AbiList<_A0, _Rest...>
4574 template <
typename _Tp,
int _Np>
4575 static constexpr bool _S_has_valid_abi
4576 = _A0<
sizeof(_Tp) * _Np>::template _S_is_valid_v<
4577 _Tp> || _AbiList<_Rest...>::template _S_has_valid_abi<_Tp, _Np>;
4579 template <
typename _Tp,
int _Np>
4581 _A0<
sizeof(_Tp) * _Np>::template _S_is_valid_v<_Tp>,
4582 typename __decay_abi<_A0<
sizeof(_Tp) * _Np>>::type,
4583 typename _AbiList<_Rest...>::template _FirstValidAbi<_Tp, _Np>>;
4585 template <
typename _Tp,
int _Np>
4586 static constexpr auto
4587 _S_determine_best_abi()
4589 static_assert(_Np >= 1);
4590 constexpr int _Bytes =
sizeof(_Tp) * _Np;
4591 if constexpr (_Np == 1)
4592 return __make_dependent_t<_Tp, simd_abi::scalar>{};
4595 constexpr int __fullsize = _A0<_Bytes>::template _S_full_size<_Tp>;
4600 if constexpr (_A0<_Bytes>::template _S_is_valid_v<_Tp>
4601 && ((__is_sve_abi<_A0<_Bytes>>() && __have_sve
4602 && (_Np <= __sve_vectorized_size_bytes/
sizeof(_Tp)))
4603 || (__fullsize / 2 < _Np))
4605 return typename __decay_abi<_A0<_Bytes>>::type{};
4609 typename __find_next_valid_abi<_A0, _Bytes, _Tp>::type;
4610 if constexpr (_Bp::template _S_is_valid_v<
4611 _Tp> && _Bp::template _S_size<_Tp> <= _Np)
4615 typename _AbiList<_Rest...>::template _BestAbi<_Tp, _Np>{};
4620 template <
typename _Tp,
int _Np>
4621 using _BestAbi =
decltype(_S_determine_best_abi<_Tp, _Np>());
4629using _AllNativeAbis = _AbiList<
4630#if _GLIBCXX_SIMD_HAVE_SVE
4633 simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin, __scalar_abi_wrapper>;
4635using _NoSveAllNativeAbis = _AbiList<simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin,
4636 __scalar_abi_wrapper>;
4639template <
typename _Tp,
typename _Abi>
4640 struct _SimdTraits<_Tp, _Abi,
void_t<typename _Abi::template _IsValid<_Tp>>>
4641 : _Abi::template __traits<_Tp> {};
4645template <
typename _Tp,
size_t _Np>
4646 struct __deduce_impl<
4647 _Tp, _Np,
enable_if_t<_AllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>>
4648 {
using type = _AllNativeAbis::_FirstValidAbi<_Tp, _Np>; };
4650template <
typename _Tp,
size_t _Np>
4651 struct __no_sve_deduce_impl<
4652 _Tp, _Np,
enable_if_t<_NoSveAllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>>
4653 {
using type = _NoSveAllNativeAbis::_FirstValidAbi<_Tp, _Np>; };
4656template <
typename _Tp,
size_t _Np,
typename =
void>
4657 struct __deduce_fixed_size_fallback {};
4659template <
typename _Tp,
size_t _Np>
4660 struct __deduce_fixed_size_fallback<_Tp, _Np,
4661 enable_if_t<simd_abi::fixed_size<_Np>::template _S_is_valid_v<_Tp>>>
4662 {
using type = simd_abi::fixed_size<_Np>; };
4664template <
typename _Tp,
size_t _Np,
typename>
4665 struct __deduce_impl :
public __deduce_fixed_size_fallback<_Tp, _Np> {};
4667template <
typename _Tp,
size_t _Np,
typename>
4668 struct __no_sve_deduce_impl
4669 :
public __deduce_fixed_size_fallback<_Tp, _Np>
4677template <
typename _Tp,
typename _Abi>
4678 class simd_mask :
public _SimdTraits<_Tp, _Abi>::_MaskBase
4681 using _Traits = _SimdTraits<_Tp, _Abi>;
4682 using _MemberType =
typename _Traits::_MaskMember;
4688 using _Ip = __int_for_sizeof_t<_Tp>;
4689 static constexpr _Ip* _S_type_tag =
nullptr;
4691 friend typename _Traits::_MaskBase;
4692 friend class simd<_Tp, _Abi>;
4693 friend typename _Traits::_SimdImpl;
4696 using _Impl =
typename _Traits::_MaskImpl;
4701 using value_type = bool;
4702 using reference = _SmartReference<_MemberType, _Impl, value_type>;
4703 using simd_type = simd<_Tp, _Abi>;
4704 using abi_type = _Abi;
4707 static constexpr size_t size()
4708 {
return __size_or_zero_v<_Tp, _Abi>; }
4712 simd_mask() =
default;
4713 simd_mask(
const simd_mask&) =
default;
4714 simd_mask(simd_mask&&) =
default;
4715 simd_mask& operator=(
const simd_mask&) =
default;
4716 simd_mask& operator=(simd_mask&&) =
default;
4720 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
explicit
4721 simd_mask(
typename _Traits::_MaskCastType __init)
4722 : _M_data{__init} {}
4731 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
static simd_mask
4732 __from_bitset(bitset<
size()> bs)
4733 {
return {__bitset_init, bs}; }
4735 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bitset<
size()>
4737 {
return _Impl::_S_to_bits(_M_data)._M_to_bitset(); }
4741 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
4742 simd_mask(value_type __x)
4743 : _M_data(_Impl::template _S_broadcast<_Ip>(__x)) {}
4747 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4749 template <
typename _Up,
typename _A2,
4750 typename = enable_if_t<simd_size_v<_Up, _A2> ==
size()>>
4751 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit(
sizeof(_MemberType)
4752 !=
sizeof(
typename _SimdTraits<_Up, _A2>::_MaskMember))
4753 simd_mask(
const simd_mask<_Up, _A2>& __x)
4754 : simd_mask(__proposed::static_simd_cast<simd_mask>(__x)) {}
4757 template <
typename _Up,
typename =
enable_if_t<conjunction<
4758 is_same<abi_type, simd_abi::fixed_size<
size()>>,
4759 is_same<_Up, _Up>>::value>>
4760 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4761 simd_mask(
const simd_mask<_Up, simd_abi::fixed_size<
size()>>& __x)
4762 : _M_data(_Impl::_S_from_bitmask(__data(__x), _S_type_tag)) {}
4767 template <
typename _Flags>
4768 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4769 simd_mask(
const value_type* __mem, _IsSimdFlagType<_Flags>)
4770 : _M_data(_Impl::template _S_load<_Ip>(_Flags::template _S_apply<simd_mask>(__mem))) {}
4772 template <
typename _Flags>
4773 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4774 simd_mask(
const value_type* __mem, simd_mask __k, _IsSimdFlagType<_Flags>)
4777 _M_data = _Impl::_S_masked_load(_M_data, __k._M_data,
4778 _Flags::template _S_apply<simd_mask>(__mem));
4783 template <
typename _Flags>
4784 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
void
4785 copy_from(
const value_type* __mem, _IsSimdFlagType<_Flags>)
4786 { _M_data = _Impl::template _S_load<_Ip>(_Flags::template _S_apply<simd_mask>(__mem)); }
4790 template <
typename _Flags>
4791 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
void
4792 copy_to(value_type* __mem, _IsSimdFlagType<_Flags>)
const
4793 { _Impl::_S_store(_M_data, _Flags::template _S_apply<simd_mask>(__mem)); }
4797 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
4798 operator[](
size_t __i)
4801 __invoke_ub(
"Subscript %d is out of range [0, %d]", __i,
size() - 1);
4802 return {_M_data, int(__i)};
4805 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
4806 operator[](
size_t __i)
const
4809 __invoke_ub(
"Subscript %d is out of range [0, %d]", __i,
size() - 1);
4810 if constexpr (__is_scalar_abi<_Abi>())
4813 return static_cast<bool>(_M_data[__i]);
4818 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd_mask
4820 {
return {__private_init, _Impl::_S_bit_not(_M_data)}; }
4824 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4826 template <
typename _Up,
typename _A2,
4827 typename = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4828 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4829 operator&&(
const simd_mask& __x,
const simd_mask<_Up, _A2>& __y)
4831 return {__private_init,
4832 _Impl::_S_logical_and(__x._M_data, simd_mask(__y)._M_data)};
4835 template <
typename _Up,
typename _A2,
4836 typename = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4837 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4838 operator||(
const simd_mask& __x,
const simd_mask<_Up, _A2>& __y)
4840 return {__private_init,
4841 _Impl::_S_logical_or(__x._M_data, simd_mask(__y)._M_data)};
4845 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4846 operator&&(
const simd_mask& __x,
const simd_mask& __y)
4847 {
return {__private_init, _Impl::_S_logical_and(__x._M_data, __y._M_data)}; }
4849 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4850 operator||(
const simd_mask& __x,
const simd_mask& __y)
4851 {
return {__private_init, _Impl::_S_logical_or(__x._M_data, __y._M_data)}; }
4853 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4854 operator&(
const simd_mask& __x,
const simd_mask& __y)
4855 {
return {__private_init, _Impl::_S_bit_and(__x._M_data, __y._M_data)}; }
4857 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4858 operator|(
const simd_mask& __x,
const simd_mask& __y)
4859 {
return {__private_init, _Impl::_S_bit_or(__x._M_data, __y._M_data)}; }
4861 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4862 operator^(
const simd_mask& __x,
const simd_mask& __y)
4863 {
return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4865 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask&
4866 operator&=(simd_mask& __x,
const simd_mask& __y)
4868 __x._M_data = _Impl::_S_bit_and(__x._M_data, __y._M_data);
4872 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask&
4873 operator|=(simd_mask& __x,
const simd_mask& __y)
4875 __x._M_data = _Impl::_S_bit_or(__x._M_data, __y._M_data);
4879 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask&
4880 operator^=(simd_mask& __x,
const simd_mask& __y)
4882 __x._M_data = _Impl::_S_bit_xor(__x._M_data, __y._M_data);
4888 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4889 operator==(
const simd_mask& __x,
const simd_mask& __y)
4890 {
return !operator!=(__x, __y); }
4892 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4893 operator!=(
const simd_mask& __x,
const simd_mask& __y)
4894 {
return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4898 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
4899 simd_mask(_PrivateInit,
typename _Traits::_MaskMember __init)
4900 : _M_data(__init) {}
4904 template <typename _Fp, typename = decltype(bool(declval<_Fp>()(size_t())))>
4905 _GLIBCXX_SIMD_INTRINSIC
constexpr
4906 simd_mask(_PrivateInit, _Fp&& __gen)
4909 __execute_n_times<size()>([&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4910 _Impl::_S_set(_M_data, __i, __gen(__i));
4916 _GLIBCXX_SIMD_INTRINSIC
constexpr
4917 simd_mask(_BitsetInit, bitset<
size()> __init)
4918 : _M_data(_Impl::_S_from_bitmask(_SanitizedBitMask<
size()>(__init), _S_type_tag))
4929 template <
typename _Up,
typename _A2,
4930 typename = enable_if_t<simd_size_v<_Up, _A2> == simd_size_v<_Tp, _Abi>>>
4931 _GLIBCXX_SIMD_ALWAYS_INLINE
4932 operator simd_mask<_Up, _A2>() &&
4934 using namespace std::experimental::__proposed;
4935 return static_simd_cast<simd_mask<_Up, _A2>>(_M_data);
4938 const simd_mask<_Tp, _Abi>& _M_data;
4941 _GLIBCXX_SIMD_INTRINSIC _CvtProxy
4947 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
4948 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4949 operator?:(
const simd_mask& __k,
const simd_mask& __where_true,
4950 const simd_mask& __where_false)
4952 auto __ret = __where_false;
4953 _Impl::_S_masked_assign(__k._M_data, __ret._M_data, __where_true._M_data);
4957 template <
typename _U1,
typename _U2,
4958 typename _Rp = simd<common_type_t<_U1, _U2>, _Abi>,
4960 is_convertible<_U1, _Rp>, is_convertible<_U2, _Rp>,
4961 is_convertible<simd_mask, typename _Rp::mask_type>>>>
4962 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend _Rp
4963 operator?:(
const simd_mask& __k,
const _U1& __where_true,
4964 const _U2& __where_false)
4966 _Rp __ret = __where_false;
4967 _Rp::_Impl::_S_masked_assign(
4968 __data(
static_cast<typename _Rp::mask_type
>(__k)), __data(__ret),
4969 __data(
static_cast<_Rp
>(__where_true)));
4973 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4974 template <
typename _Kp,
typename _Ak,
typename _Up,
typename _Au,
4976 conjunction_v<is_convertible<simd_mask<_Kp, _Ak>, simd_mask>,
4977 is_convertible<simd_mask<_Up, _Au>, simd_mask>>>>
4978 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4979 operator?:(
const simd_mask<_Kp, _Ak>& __k,
const simd_mask& __where_true,
4980 const simd_mask<_Up, _Au>& __where_false)
4982 simd_mask __ret = __where_false;
4983 _Impl::_S_masked_assign(simd_mask(__k)._M_data, __ret._M_data,
4984 __where_true._M_data);
4992 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
4993 _M_is_constprop()
const
4995 if constexpr (__is_scalar_abi<_Abi>())
4996 return __builtin_constant_p(_M_data);
4998 return _M_data._M_is_constprop();
5004 friend const auto& __data<_Tp, abi_type>(
const simd_mask&);
5005 friend auto& __data<_Tp, abi_type>(simd_mask&);
5006 alignas(_Traits::_S_mask_align) _MemberType _M_data;
5013template <
typename _Tp,
typename _Ap>
5014 _GLIBCXX_SIMD_INTRINSIC
constexpr const auto&
5015 __data(
const simd_mask<_Tp, _Ap>& __x)
5016 {
return __x._M_data; }
5018template <
typename _Tp,
typename _Ap>
5019 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
5020 __data(simd_mask<_Tp, _Ap>& __x)
5021 {
return __x._M_data; }
5027template <
typename _Tp,
typename _Abi>
5028 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
5029 all_of(
const simd_mask<_Tp, _Abi>& __k)
noexcept
5031 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5033 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
5039 return _Abi::_MaskImpl::_S_all_of(__k);
5042template <
typename _Tp,
typename _Abi>
5043 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
5044 any_of(
const simd_mask<_Tp, _Abi>& __k)
noexcept
5046 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5048 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
5054 return _Abi::_MaskImpl::_S_any_of(__k);
5057template <
typename _Tp,
typename _Abi>
5058 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
5059 none_of(
const simd_mask<_Tp, _Abi>& __k)
noexcept
5061 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5063 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
5069 return _Abi::_MaskImpl::_S_none_of(__k);
5072template <
typename _Tp,
typename _Abi>
5073 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
5074 some_of(
const simd_mask<_Tp, _Abi>& __k)
noexcept
5076 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5078 for (
size_t __i = 1; __i < simd_size_v<_Tp, _Abi>; ++__i)
5079 if (__k[__i] != __k[__i - 1])
5084 return _Abi::_MaskImpl::_S_some_of(__k);
5087template <
typename _Tp,
typename _Abi>
5088 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
5089 popcount(
const simd_mask<_Tp, _Abi>& __k)
noexcept
5091 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5093 const int __r = __call_with_subscripts<simd_size_v<_Tp, _Abi>>(
5094 __k, [](
auto... __elements) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5095 return ((__elements != 0) + ...);
5097 if (__builtin_is_constant_evaluated() || __builtin_constant_p(__r))
5100 return _Abi::_MaskImpl::_S_popcount(__k);
5103template <
typename _Tp,
typename _Abi>
5104 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
5105 find_first_set(
const simd_mask<_Tp, _Abi>& __k)
5107 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5109 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
5110 const size_t _Idx = __call_with_n_evaluations<_Np>(
5111 [](
auto... __indexes) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5113 }, [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5114 return __k[__i] ? +__i : _Np;
5117 __invoke_ub(
"find_first_set(empty mask) is UB");
5118 if (__builtin_constant_p(_Idx))
5121 return _Abi::_MaskImpl::_S_find_first_set(__k);
5124template <
typename _Tp,
typename _Abi>
5125 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
5126 find_last_set(
const simd_mask<_Tp, _Abi>& __k)
5128 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5130 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
5131 const int _Idx = __call_with_n_evaluations<_Np>(
5132 [](
auto... __indexes) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5134 }, [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5135 return __k[__i] ? int(__i) : -1;
5138 __invoke_ub(
"find_first_set(empty mask) is UB");
5139 if (__builtin_constant_p(_Idx))
5142 return _Abi::_MaskImpl::_S_find_last_set(__k);
5145_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
5146all_of(_ExactBool __x)
noexcept
5149_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
5150any_of(_ExactBool __x)
noexcept
5153_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
5154none_of(_ExactBool __x)
noexcept
5157_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
5158some_of(_ExactBool)
noexcept
5161_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
5162popcount(_ExactBool __x)
noexcept
5165_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
5166find_first_set(_ExactBool)
5169_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
5170find_last_set(_ExactBool)
5177template <
typename _V,
typename _Tp,
typename _Abi,
bool>
5178 class _SimdIntOperators {};
5180template <
typename _V,
typename _Tp,
typename _Abi>
5181 class _SimdIntOperators<_V, _Tp, _Abi, true>
5183 using _Impl =
typename _SimdTraits<_Tp, _Abi>::_SimdImpl;
5185 _GLIBCXX_SIMD_INTRINSIC
constexpr const _V&
5187 {
return *
static_cast<const _V*
>(
this); }
5189 template <
typename _Up>
5190 _GLIBCXX_SIMD_INTRINSIC
static _GLIBCXX_SIMD_CONSTEXPR _V
5191 _S_make_derived(_Up&& __d)
5192 {
return {__private_init,
static_cast<_Up&&
>(__d)}; }
5195 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5197 operator%=(_V& __lhs,
const _V& __x)
5198 {
return __lhs = __lhs % __x; }
5200 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5202 operator&=(_V& __lhs,
const _V& __x)
5203 {
return __lhs = __lhs & __x; }
5205 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5207 operator|=(_V& __lhs,
const _V& __x)
5208 {
return __lhs = __lhs | __x; }
5210 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5212 operator^=(_V& __lhs,
const _V& __x)
5213 {
return __lhs = __lhs ^ __x; }
5215 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5217 operator<<=(_V& __lhs,
const _V& __x)
5218 {
return __lhs = __lhs << __x; }
5220 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5222 operator>>=(_V& __lhs,
const _V& __x)
5223 {
return __lhs = __lhs >> __x; }
5225 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5227 operator<<=(_V& __lhs,
int __x)
5228 {
return __lhs = __lhs << __x; }
5230 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5232 operator>>=(_V& __lhs,
int __x)
5233 {
return __lhs = __lhs >> __x; }
5235 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5239 return _SimdIntOperators::_S_make_derived(
5240 _Impl::_S_modulus(__data(__x), __data(__y)));
5243 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5247 return _SimdIntOperators::_S_make_derived(
5248 _Impl::_S_bit_and(__data(__x), __data(__y)));
5251 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5255 return _SimdIntOperators::_S_make_derived(
5256 _Impl::_S_bit_or(__data(__x), __data(__y)));
5259 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5263 return _SimdIntOperators::_S_make_derived(
5264 _Impl::_S_bit_xor(__data(__x), __data(__y)));
5267 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5271 return _SimdIntOperators::_S_make_derived(
5272 _Impl::_S_bit_shift_left(__data(__x), __data(__y)));
5275 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5279 return _SimdIntOperators::_S_make_derived(
5280 _Impl::_S_bit_shift_right(__data(__x), __data(__y)));
5283 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5288 __invoke_ub(
"The behavior is undefined if the right operand of a "
5289 "shift operation is negative. [expr.shift]\nA shift by "
5292 if (
size_t(__y) >=
sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
5294 "The behavior is undefined if the right operand of a "
5295 "shift operation is greater than or equal to the width of the "
5296 "promoted left operand. [expr.shift]\nA shift by %d was requested",
5298 return _SimdIntOperators::_S_make_derived(
5299 _Impl::_S_bit_shift_left(__data(__x), __y));
5302 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5308 "The behavior is undefined if the right operand of a shift "
5309 "operation is negative. [expr.shift]\nA shift by %d was requested",
5311 if (
size_t(__y) >=
sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
5313 "The behavior is undefined if the right operand of a shift "
5314 "operation is greater than or equal to the width of the promoted "
5315 "left operand. [expr.shift]\nA shift by %d was requested",
5317 return _SimdIntOperators::_S_make_derived(
5318 _Impl::_S_bit_shift_right(__data(__x), __y));
5322 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5325 {
return {__private_init, _Impl::_S_complement(__derived()._M_data)}; }
5332template <
typename _Tp,
typename _Abi>
5333 class simd :
public _SimdIntOperators<
5334 simd<_Tp, _Abi>, _Tp, _Abi,
5335 conjunction<is_integral<_Tp>,
5336 typename _SimdTraits<_Tp, _Abi>::_IsValid>::value>,
5337 public _SimdTraits<_Tp, _Abi>::_SimdBase
5339 using _Traits = _SimdTraits<_Tp, _Abi>;
5340 using _MemberType =
typename _Traits::_SimdMember;
5341 using _CastType =
typename _Traits::_SimdCastType;
5342 static constexpr _Tp* _S_type_tag =
nullptr;
5343 friend typename _Traits::_SimdBase;
5346 using _Impl =
typename _Traits::_SimdImpl;
5348 friend _SimdIntOperators<simd, _Tp, _Abi, true>;
5350 using value_type = _Tp;
5351 using reference = _SmartReference<_MemberType, _Impl, value_type>;
5352 using mask_type = simd_mask<_Tp, _Abi>;
5353 using abi_type = _Abi;
5355 static constexpr size_t size()
5356 {
return __size_or_zero_v<_Tp, _Abi>; }
5358 _GLIBCXX_SIMD_CONSTEXPR simd() =
default;
5359 _GLIBCXX_SIMD_CONSTEXPR simd(
const simd&) =
default;
5360 _GLIBCXX_SIMD_CONSTEXPR simd(simd&&) noexcept = default;
5361 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(const simd&) = default;
5362 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(simd&&) noexcept = default;
5365 template <typename _Up,
5366 typename = enable_if_t<!is_same_v<__remove_cvref_t<_Up>,
bool>>>
5367 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5368 simd(_ValuePreservingOrInt<_Up, value_type>&& __x)
5370 _Impl::_S_broadcast(static_cast<value_type>(static_cast<_Up&&>(__x))))
5375 template <
typename _Up>
5376 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5377 simd(
const simd<_Up, simd_abi::fixed_size<
size()>>& __x,
5380 is_same<simd_abi::fixed_size<
size()>, abi_type>,
5381 negation<__is_narrowing_conversion<_Up, value_type>>,
5382 __converts_to_higher_integer_rank<_Up, value_type>>::value,
5384 : simd{static_cast<array<_Up,
size()>>(__x).
data(), vector_aligned} {}
5387#ifdef _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5388 template <
typename _Up,
typename _A2,
5389 typename =
decltype(static_simd_cast<simd>(
5390 declval<
const simd<_Up, _A2>&>()))>
5391 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
5392 simd(
const simd<_Up, _A2>& __x)
5393 : simd(static_simd_cast<simd>(__x)) {}
5397 template <
typename _Fp>
5398 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
5399 simd(_Fp&& __gen, _ValuePreservingOrInt<
decltype(declval<_Fp>()(
5400 declval<_SizeConstant<0>&>())),
5401 value_type>* =
nullptr)
5402 : _M_data(_Impl::_S_generator(static_cast<_Fp&&>(__gen), _S_type_tag)) {}
5405 template <
typename _Up,
typename _Flags>
5406 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5407 simd(
const _Up* __mem, _IsSimdFlagType<_Flags>)
5409 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag))
5413 template <
typename _Up,
typename _Flags>
5414 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
void
5415 copy_from(
const _Vectorizable<_Up>* __mem, _IsSimdFlagType<_Flags>)
5417 _M_data =
static_cast<decltype(_M_data)
>(
5418 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag));
5422 template <
typename _Up,
typename _Flags>
5423 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
void
5424 copy_to(_Vectorizable<_Up>* __mem, _IsSimdFlagType<_Flags>)
const
5426 _Impl::_S_store(_M_data, _Flags::template _S_apply<simd>(__mem),
5431 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
5432 operator[](
size_t __i)
5433 {
return {_M_data, int(__i)}; }
5435 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
5436 operator[]([[maybe_unused]]
size_t __i)
const
5438 if constexpr (__is_scalar_abi<_Abi>())
5440 _GLIBCXX_DEBUG_ASSERT(__i == 0);
5444 return _M_data[__i];
5448 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5451 _Impl::_S_increment(_M_data);
5455 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5459 _Impl::_S_increment(_M_data);
5463 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5466 _Impl::_S_decrement(_M_data);
5470 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5474 _Impl::_S_decrement(_M_data);
5479 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR mask_type
5481 {
return {__private_init, _Impl::_S_negate(_M_data)}; }
5483 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5487 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5489 {
return {__private_init, _Impl::_S_unary_minus(_M_data)}; }
5492 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
5493 simd(_CastType __init) : _M_data(__init) {}
5496 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5497 operator+=(simd& __lhs,
const simd& __x)
5498 {
return __lhs = __lhs + __x; }
5500 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5501 operator-=(simd& __lhs,
const simd& __x)
5502 {
return __lhs = __lhs - __x; }
5504 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5505 operator*=(simd& __lhs,
const simd& __x)
5506 {
return __lhs = __lhs * __x; }
5508 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5509 operator/=(simd& __lhs,
const simd& __x)
5510 {
return __lhs = __lhs / __x; }
5513 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5514 operator+(
const simd& __x,
const simd& __y)
5515 {
return {__private_init, _Impl::_S_plus(__x._M_data, __y._M_data)}; }
5517 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5518 operator-(
const simd& __x,
const simd& __y)
5519 {
return {__private_init, _Impl::_S_minus(__x._M_data, __y._M_data)}; }
5521 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5522 operator*(
const simd& __x,
const simd& __y)
5523 {
return {__private_init, _Impl::_S_multiplies(__x._M_data, __y._M_data)}; }
5525 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5526 operator/(
const simd& __x,
const simd& __y)
5527 {
return {__private_init, _Impl::_S_divides(__x._M_data, __y._M_data)}; }
5530 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5531 operator==(
const simd& __x,
const simd& __y)
5532 {
return simd::_S_make_mask(_Impl::_S_equal_to(__x._M_data, __y._M_data)); }
5534 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5535 operator!=(
const simd& __x,
const simd& __y)
5537 return simd::_S_make_mask(
5538 _Impl::_S_not_equal_to(__x._M_data, __y._M_data));
5541 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5542 operator<(
const simd& __x,
const simd& __y)
5543 {
return simd::_S_make_mask(_Impl::_S_less(__x._M_data, __y._M_data)); }
5545 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5548 return simd::_S_make_mask(_Impl::_S_less_equal(__x._M_data, __y._M_data));
5551 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5552 operator>(
const simd& __x,
const simd& __y)
5553 {
return simd::_S_make_mask(_Impl::_S_less(__y._M_data, __x._M_data)); }
5555 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5558 return simd::_S_make_mask(_Impl::_S_less_equal(__y._M_data, __x._M_data));
5562#ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
5563 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5564 operator?:(
const mask_type& __k,
const simd& __where_true,
5565 const simd& __where_false)
5567 auto __ret = __where_false;
5568 _Impl::_S_masked_assign(__data(__k), __data(__ret), __data(__where_true));
5576 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5577 simd(_PrivateInit,
const _MemberType& __init)
5578 : _M_data(__init) {}
5581 _GLIBCXX_SIMD_INTRINSIC
5582 simd(_BitsetInit, bitset<
size()> __init) : _M_data()
5583 { where(mask_type(__bitset_init, __init), *
this) = ~*
this; }
5585 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
5586 _M_is_constprop()
const
5588 if constexpr (__is_scalar_abi<_Abi>())
5589 return __builtin_constant_p(_M_data);
5591 return _M_data._M_is_constprop();
5595 _GLIBCXX_SIMD_INTRINSIC
static constexpr mask_type
5596 _S_make_mask(
typename mask_type::_MemberType __k)
5597 {
return {__private_init, __k}; }
5599 friend const auto& __data<value_type, abi_type>(
const simd&);
5600 friend auto& __data<value_type, abi_type>(simd&);
5601 alignas(_Traits::_S_simd_align) _MemberType _M_data;
5607template <
typename _Tp,
typename _Ap>
5608 _GLIBCXX_SIMD_INTRINSIC
constexpr const auto&
5609 __data(
const simd<_Tp, _Ap>& __x)
5610 {
return __x._M_data; }
5612template <
typename _Tp,
typename _Ap>
5613 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
5614 __data(simd<_Tp, _Ap>& __x)
5615 {
return __x._M_data; }
5618namespace __float_bitwise_operators {
5619template <
typename _Tp,
typename _Ap>
5620 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5621 operator^(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5622 {
return {__private_init, _Ap::_SimdImpl::_S_bit_xor(__data(__a), __data(__b))}; }
5624template <
typename _Tp,
typename _Ap>
5625 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5626 operator|(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5627 {
return {__private_init, _Ap::_SimdImpl::_S_bit_or(__data(__a), __data(__b))}; }
5629template <
typename _Tp,
typename _Ap>
5630 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5631 operator&(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5632 {
return {__private_init, _Ap::_SimdImpl::_S_bit_and(__data(__a), __data(__b))}; }
5634template <
typename _Tp,
typename _Ap>
5635 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5636 enable_if_t<is_floating_point_v<_Tp>, simd<_Tp, _Ap>>
5637 operator~(
const simd<_Tp, _Ap>& __a)
5638 {
return {__private_init, _Ap::_SimdImpl::_S_complement(__data(__a))}; }
5643_GLIBCXX_SIMD_END_NAMESPACE
constexpr bool operator<=(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
constexpr bool operator>=(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
constexpr duration< __common_rep_t< _Rep1, __disable_if_is_duration< _Rep2 > >, _Period > operator%(const duration< _Rep1, _Period > &__d, const _Rep2 &__s)
constexpr bool operator<(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
constexpr bool operator>(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
constexpr complex< _Tp > operator*(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x times y.
constexpr complex< _Tp > operator-(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x minus y.
constexpr complex< _Tp > operator+(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x plus y.
constexpr complex< _Tp > operator/(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x divided by y.
typename remove_reference< _Tp >::type remove_reference_t
Alias template for remove_reference.
typename make_unsigned< _Tp >::type make_unsigned_t
Alias template for make_unsigned.
__bool_constant< true > true_type
The type used as a compile-time boolean with true value.
typename conditional< _Cond, _Iftrue, _Iffalse >::type conditional_t
Alias template for conditional.
__bool_constant< false > false_type
The type used as a compile-time boolean with false value.
typename enable_if< _Cond, _Tp >::type enable_if_t
Alias template for enable_if.
constexpr auto tuple_cat(_Tpls &&... __tpls) -> typename __tuple_cat_result< _Tpls... >::__type
Create a tuple containing all elements from multiple tuple-like objects.
auto declval() noexcept -> decltype(__declval< _Tp >(0))
constexpr tuple< typename __decay_and_strip< _Elements >::__type... > make_tuple(_Elements &&... __args)
Create a tuple containing copies of the arguments.
constexpr std::remove_reference< _Tp >::type && move(_Tp &&__t) noexcept
Convert a value to an rvalue.
_Tp * end(valarray< _Tp > &__va) noexcept
Return an iterator pointing to one past the last element of the valarray.
_Tp * begin(valarray< _Tp > &__va) noexcept
Return an iterator pointing to the first element of the valarray.
constexpr const _Tp & max(const _Tp &, const _Tp &)
This does what you think it does.
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op)
Calculate reduction of values in a range.
void void_t
A metafunction that always yields void, used for detecting valid types.
ISO C++ entities toplevel namespace is std.
constexpr auto size(const _Container &__cont) noexcept(noexcept(__cont.size())) -> decltype(__cont.size())
Return the size of a container.
constexpr bitset< _Nb > operator^(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
std::basic_istream< _CharT, _Traits > & operator>>(std::basic_istream< _CharT, _Traits > &__is, bitset< _Nb > &__x)
Global I/O operators for bitsets.
std::basic_ostream< _CharT, _Traits > & operator<<(std::basic_ostream< _CharT, _Traits > &__os, const bitset< _Nb > &__x)
Global I/O operators for bitsets.
constexpr auto data(_Container &__cont) noexcept(noexcept(__cont.data())) -> decltype(__cont.data())
Return the data pointer of a container.
constexpr bitset< _Nb > operator|(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
constexpr bitset< _Nb > operator&(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.