libstdc++
simd.h
1// Definition of the public simd interfaces -*- C++ -*-
2
3// Copyright (C) 2020-2021 Free Software Foundation, Inc.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_H
26#define _GLIBCXX_EXPERIMENTAL_SIMD_H
27
28#if __cplusplus >= 201703L
29
30#include "simd_detail.h"
31#include "numeric_traits.h"
32#include <bit>
33#include <bitset>
34#ifdef _GLIBCXX_DEBUG_UB
35#include <cstdio> // for stderr
36#endif
37#include <cstring>
38#include <functional>
39#include <iosfwd>
40#include <utility>
41
42#if _GLIBCXX_SIMD_X86INTRIN
43#include <x86intrin.h>
44#elif _GLIBCXX_SIMD_HAVE_NEON
45#pragma GCC diagnostic push
46// narrowing conversion of '__a' from 'uint64_t' {aka 'long long unsigned int'} to
47// 'int64x1_t' {aka 'long long int'} [-Wnarrowing]
48#pragma GCC diagnostic ignored "-Wnarrowing"
49#include <arm_neon.h>
50#pragma GCC diagnostic pop
51#endif
52
53/** @ingroup ts_simd
54 * @{
55 */
56/* There are several closely related types, with the following naming
57 * convention:
58 * _Tp: vectorizable (arithmetic) type (or any type)
59 * _TV: __vector_type_t<_Tp, _Np>
60 * _TW: _SimdWrapper<_Tp, _Np>
61 * _TI: __intrinsic_type_t<_Tp, _Np>
62 * _TVT: _VectorTraits<_TV> or _VectorTraits<_TW>
63 * If one additional type is needed use _U instead of _T.
64 * Otherwise use _T\d, _TV\d, _TW\d, TI\d, _TVT\d.
65 *
66 * More naming conventions:
67 * _Ap or _Abi: An ABI tag from the simd_abi namespace
68 * _Ip: often used for integer types with sizeof(_Ip) == sizeof(_Tp),
69 * _IV, _IW as for _TV, _TW
70 * _Np: number of elements (not bytes)
71 * _Bytes: number of bytes
72 *
73 * Variable names:
74 * __k: mask object (vector- or bitmask)
75 */
76_GLIBCXX_SIMD_BEGIN_NAMESPACE
77
78#if !_GLIBCXX_SIMD_X86INTRIN
79using __m128 [[__gnu__::__vector_size__(16)]] = float;
80using __m128d [[__gnu__::__vector_size__(16)]] = double;
81using __m128i [[__gnu__::__vector_size__(16)]] = long long;
82using __m256 [[__gnu__::__vector_size__(32)]] = float;
83using __m256d [[__gnu__::__vector_size__(32)]] = double;
84using __m256i [[__gnu__::__vector_size__(32)]] = long long;
85using __m512 [[__gnu__::__vector_size__(64)]] = float;
86using __m512d [[__gnu__::__vector_size__(64)]] = double;
87using __m512i [[__gnu__::__vector_size__(64)]] = long long;
88#endif
89
90namespace simd_abi {
91// simd_abi forward declarations {{{
92// implementation details:
93struct _Scalar;
94
95template <int _Np>
96 struct _Fixed;
97
98// There are two major ABIs that appear on different architectures.
99// Both have non-boolean values packed into an N Byte register
100// -> #elements = N / sizeof(T)
101// Masks differ:
102// 1. Use value vector registers for masks (all 0 or all 1)
103// 2. Use bitmasks (mask registers) with one bit per value in the corresponding
104// value vector
105//
106// Both can be partially used, masking off the rest when doing horizontal
107// operations or operations that can trap (e.g. FP_INVALID or integer division
108// by 0). This is encoded as the number of used bytes.
109template <int _UsedBytes>
110 struct _VecBuiltin;
111
112template <int _UsedBytes>
113 struct _VecBltnBtmsk;
114
115template <typename _Tp, int _Np>
116 using _VecN = _VecBuiltin<sizeof(_Tp) * _Np>;
117
118template <int _UsedBytes = 16>
119 using _Sse = _VecBuiltin<_UsedBytes>;
120
121template <int _UsedBytes = 32>
122 using _Avx = _VecBuiltin<_UsedBytes>;
123
124template <int _UsedBytes = 64>
125 using _Avx512 = _VecBltnBtmsk<_UsedBytes>;
126
127template <int _UsedBytes = 16>
128 using _Neon = _VecBuiltin<_UsedBytes>;
129
130// implementation-defined:
131using __sse = _Sse<>;
132using __avx = _Avx<>;
133using __avx512 = _Avx512<>;
134using __neon = _Neon<>;
135using __neon128 = _Neon<16>;
136using __neon64 = _Neon<8>;
137
138// standard:
139template <typename _Tp, size_t _Np, typename...>
140 struct deduce;
141
142template <int _Np>
143 using fixed_size = _Fixed<_Np>;
144
145using scalar = _Scalar;
146
147// }}}
148} // namespace simd_abi
149// forward declarations is_simd(_mask), simd(_mask), simd_size {{{
150template <typename _Tp>
151 struct is_simd;
152
153template <typename _Tp>
154 struct is_simd_mask;
155
156template <typename _Tp, typename _Abi>
157 class simd;
158
159template <typename _Tp, typename _Abi>
160 class simd_mask;
161
162template <typename _Tp, typename _Abi>
163 struct simd_size;
164
165// }}}
166// load/store flags {{{
167struct element_aligned_tag
168{
169 template <typename _Tp, typename _Up = typename _Tp::value_type>
170 static constexpr size_t _S_alignment = alignof(_Up);
171
172 template <typename _Tp, typename _Up>
173 _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
174 _S_apply(_Up* __ptr)
175 { return __ptr; }
176};
177
178struct vector_aligned_tag
179{
180 template <typename _Tp, typename _Up = typename _Tp::value_type>
181 static constexpr size_t _S_alignment
182 = std::__bit_ceil(sizeof(_Up) * _Tp::size());
183
184 template <typename _Tp, typename _Up>
185 _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
186 _S_apply(_Up* __ptr)
187 { return static_cast<_Up*>(__builtin_assume_aligned(__ptr, _S_alignment<_Tp, _Up>)); }
188};
189
190template <size_t _Np> struct overaligned_tag
191{
192 template <typename _Tp, typename _Up = typename _Tp::value_type>
193 static constexpr size_t _S_alignment = _Np;
194
195 template <typename _Tp, typename _Up>
196 _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
197 _S_apply(_Up* __ptr)
198 { return static_cast<_Up*>(__builtin_assume_aligned(__ptr, _Np)); }
199};
200
201inline constexpr element_aligned_tag element_aligned = {};
202
203inline constexpr vector_aligned_tag vector_aligned = {};
204
205template <size_t _Np>
206 inline constexpr overaligned_tag<_Np> overaligned = {};
207
208// }}}
209template <size_t _Xp>
210 using _SizeConstant = integral_constant<size_t, _Xp>;
211
212namespace __detail
213{
214 struct _Minimum
215 {
216 template <typename _Tp>
217 _GLIBCXX_SIMD_INTRINSIC constexpr
218 _Tp
219 operator()(_Tp __a, _Tp __b) const
220 {
221 using std::min;
222 return min(__a, __b);
223 }
224 };
225
226 struct _Maximum
227 {
228 template <typename _Tp>
229 _GLIBCXX_SIMD_INTRINSIC constexpr
230 _Tp
231 operator()(_Tp __a, _Tp __b) const
232 {
233 using std::max;
234 return max(__a, __b);
235 }
236 };
237} // namespace __detail
238
239// unrolled/pack execution helpers
240// __execute_n_times{{{
241template <typename _Fp, size_t... _I>
242 _GLIBCXX_SIMD_INTRINSIC constexpr void
243 __execute_on_index_sequence(_Fp&& __f, index_sequence<_I...>)
244 { ((void)__f(_SizeConstant<_I>()), ...); }
245
246template <typename _Fp>
247 _GLIBCXX_SIMD_INTRINSIC constexpr void
248 __execute_on_index_sequence(_Fp&&, index_sequence<>)
249 { }
250
251template <size_t _Np, typename _Fp>
252 _GLIBCXX_SIMD_INTRINSIC constexpr void
253 __execute_n_times(_Fp&& __f)
254 {
255 __execute_on_index_sequence(static_cast<_Fp&&>(__f),
256 make_index_sequence<_Np>{});
257 }
258
259// }}}
260// __generate_from_n_evaluations{{{
261template <typename _R, typename _Fp, size_t... _I>
262 _GLIBCXX_SIMD_INTRINSIC constexpr _R
263 __execute_on_index_sequence_with_return(_Fp&& __f, index_sequence<_I...>)
264 { return _R{__f(_SizeConstant<_I>())...}; }
265
266template <size_t _Np, typename _R, typename _Fp>
267 _GLIBCXX_SIMD_INTRINSIC constexpr _R
268 __generate_from_n_evaluations(_Fp&& __f)
269 {
270 return __execute_on_index_sequence_with_return<_R>(
271 static_cast<_Fp&&>(__f), make_index_sequence<_Np>{});
272 }
273
274// }}}
275// __call_with_n_evaluations{{{
276template <size_t... _I, typename _F0, typename _FArgs>
277 _GLIBCXX_SIMD_INTRINSIC constexpr auto
278 __call_with_n_evaluations(index_sequence<_I...>, _F0&& __f0, _FArgs&& __fargs)
279 { return __f0(__fargs(_SizeConstant<_I>())...); }
280
281template <size_t _Np, typename _F0, typename _FArgs>
282 _GLIBCXX_SIMD_INTRINSIC constexpr auto
283 __call_with_n_evaluations(_F0&& __f0, _FArgs&& __fargs)
284 {
285 return __call_with_n_evaluations(make_index_sequence<_Np>{},
286 static_cast<_F0&&>(__f0),
287 static_cast<_FArgs&&>(__fargs));
288 }
289
290// }}}
291// __call_with_subscripts{{{
292template <size_t _First = 0, size_t... _It, typename _Tp, typename _Fp>
293 _GLIBCXX_SIMD_INTRINSIC constexpr auto
294 __call_with_subscripts(_Tp&& __x, index_sequence<_It...>, _Fp&& __fun)
295 { return __fun(__x[_First + _It]...); }
296
297template <size_t _Np, size_t _First = 0, typename _Tp, typename _Fp>
298 _GLIBCXX_SIMD_INTRINSIC constexpr auto
299 __call_with_subscripts(_Tp&& __x, _Fp&& __fun)
300 {
301 return __call_with_subscripts<_First>(static_cast<_Tp&&>(__x),
302 make_index_sequence<_Np>(),
303 static_cast<_Fp&&>(__fun));
304 }
305
306// }}}
307
308// vvv ---- type traits ---- vvv
309// integer type aliases{{{
310using _UChar = unsigned char;
311using _SChar = signed char;
312using _UShort = unsigned short;
313using _UInt = unsigned int;
314using _ULong = unsigned long;
315using _ULLong = unsigned long long;
316using _LLong = long long;
317
318//}}}
319// __first_of_pack{{{
320template <typename _T0, typename...>
321 struct __first_of_pack
322 { using type = _T0; };
323
324template <typename... _Ts>
325 using __first_of_pack_t = typename __first_of_pack<_Ts...>::type;
326
327//}}}
328// __value_type_or_identity_t {{{
329template <typename _Tp>
330 typename _Tp::value_type
331 __value_type_or_identity_impl(int);
332
333template <typename _Tp>
334 _Tp
335 __value_type_or_identity_impl(float);
336
337template <typename _Tp>
338 using __value_type_or_identity_t
339 = decltype(__value_type_or_identity_impl<_Tp>(int()));
340
341// }}}
342// __is_vectorizable {{{
343template <typename _Tp>
344 struct __is_vectorizable : public is_arithmetic<_Tp> {};
345
346template <>
347 struct __is_vectorizable<bool> : public false_type {};
348
349template <typename _Tp>
350 inline constexpr bool __is_vectorizable_v = __is_vectorizable<_Tp>::value;
351
352// Deduces to a vectorizable type
353template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
354 using _Vectorizable = _Tp;
355
356// }}}
357// _LoadStorePtr / __is_possible_loadstore_conversion {{{
358template <typename _Ptr, typename _ValueType>
359 struct __is_possible_loadstore_conversion
360 : conjunction<__is_vectorizable<_Ptr>, __is_vectorizable<_ValueType>> {};
361
362template <>
363 struct __is_possible_loadstore_conversion<bool, bool> : true_type {};
364
365// Deduces to a type allowed for load/store with the given value type.
366template <typename _Ptr, typename _ValueType,
367 typename = enable_if_t<
368 __is_possible_loadstore_conversion<_Ptr, _ValueType>::value>>
369 using _LoadStorePtr = _Ptr;
370
371// }}}
372// __is_bitmask{{{
373template <typename _Tp, typename = void_t<>>
374 struct __is_bitmask : false_type {};
375
376template <typename _Tp>
377 inline constexpr bool __is_bitmask_v = __is_bitmask<_Tp>::value;
378
379// the __mmaskXX case:
380template <typename _Tp>
381 struct __is_bitmask<_Tp,
382 void_t<decltype(declval<unsigned&>() = declval<_Tp>() & 1u)>>
383 : true_type {};
384
385// }}}
386// __int_for_sizeof{{{
387#pragma GCC diagnostic push
388#pragma GCC diagnostic ignored "-Wpedantic"
389template <size_t _Bytes>
390 constexpr auto
391 __int_for_sizeof()
392 {
393 static_assert(_Bytes > 0);
394 if constexpr (_Bytes == sizeof(int))
395 return int();
396 #ifdef __clang__
397 else if constexpr (_Bytes == sizeof(char))
398 return char();
399 #else
400 else if constexpr (_Bytes == sizeof(_SChar))
401 return _SChar();
402 #endif
403 else if constexpr (_Bytes == sizeof(short))
404 return short();
405 #ifndef __clang__
406 else if constexpr (_Bytes == sizeof(long))
407 return long();
408 #endif
409 else if constexpr (_Bytes == sizeof(_LLong))
410 return _LLong();
411 #ifdef __SIZEOF_INT128__
412 else if constexpr (_Bytes == sizeof(__int128))
413 return __int128();
414 #endif // __SIZEOF_INT128__
415 else if constexpr (_Bytes % sizeof(int) == 0)
416 {
417 constexpr size_t _Np = _Bytes / sizeof(int);
418 struct _Ip
419 {
420 int _M_data[_Np];
421
422 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
423 operator&(_Ip __rhs) const
424 {
425 return __generate_from_n_evaluations<_Np, _Ip>(
426 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
427 return __rhs._M_data[__i] & _M_data[__i];
428 });
429 }
430
431 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
432 operator|(_Ip __rhs) const
433 {
434 return __generate_from_n_evaluations<_Np, _Ip>(
435 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
436 return __rhs._M_data[__i] | _M_data[__i];
437 });
438 }
439
440 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
441 operator^(_Ip __rhs) const
442 {
443 return __generate_from_n_evaluations<_Np, _Ip>(
444 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
445 return __rhs._M_data[__i] ^ _M_data[__i];
446 });
447 }
448
449 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
450 operator~() const
451 {
452 return __generate_from_n_evaluations<_Np, _Ip>(
453 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return ~_M_data[__i]; });
454 }
455 };
456 return _Ip{};
457 }
458 else
459 static_assert(_Bytes == 0, "this should be unreachable");
460 }
461#pragma GCC diagnostic pop
462
463template <typename _Tp>
464 using __int_for_sizeof_t = decltype(__int_for_sizeof<sizeof(_Tp)>());
465
466template <size_t _Np>
467 using __int_with_sizeof_t = decltype(__int_for_sizeof<_Np>());
468
469// }}}
470// __is_fixed_size_abi{{{
471template <typename _Tp>
472 struct __is_fixed_size_abi : false_type {};
473
474template <int _Np>
475 struct __is_fixed_size_abi<simd_abi::fixed_size<_Np>> : true_type {};
476
477template <typename _Tp>
478 inline constexpr bool __is_fixed_size_abi_v = __is_fixed_size_abi<_Tp>::value;
479
480// }}}
481// constexpr feature detection{{{
482constexpr inline bool __have_mmx = _GLIBCXX_SIMD_HAVE_MMX;
483constexpr inline bool __have_sse = _GLIBCXX_SIMD_HAVE_SSE;
484constexpr inline bool __have_sse2 = _GLIBCXX_SIMD_HAVE_SSE2;
485constexpr inline bool __have_sse3 = _GLIBCXX_SIMD_HAVE_SSE3;
486constexpr inline bool __have_ssse3 = _GLIBCXX_SIMD_HAVE_SSSE3;
487constexpr inline bool __have_sse4_1 = _GLIBCXX_SIMD_HAVE_SSE4_1;
488constexpr inline bool __have_sse4_2 = _GLIBCXX_SIMD_HAVE_SSE4_2;
489constexpr inline bool __have_xop = _GLIBCXX_SIMD_HAVE_XOP;
490constexpr inline bool __have_avx = _GLIBCXX_SIMD_HAVE_AVX;
491constexpr inline bool __have_avx2 = _GLIBCXX_SIMD_HAVE_AVX2;
492constexpr inline bool __have_bmi = _GLIBCXX_SIMD_HAVE_BMI1;
493constexpr inline bool __have_bmi2 = _GLIBCXX_SIMD_HAVE_BMI2;
494constexpr inline bool __have_lzcnt = _GLIBCXX_SIMD_HAVE_LZCNT;
495constexpr inline bool __have_sse4a = _GLIBCXX_SIMD_HAVE_SSE4A;
496constexpr inline bool __have_fma = _GLIBCXX_SIMD_HAVE_FMA;
497constexpr inline bool __have_fma4 = _GLIBCXX_SIMD_HAVE_FMA4;
498constexpr inline bool __have_f16c = _GLIBCXX_SIMD_HAVE_F16C;
499constexpr inline bool __have_popcnt = _GLIBCXX_SIMD_HAVE_POPCNT;
500constexpr inline bool __have_avx512f = _GLIBCXX_SIMD_HAVE_AVX512F;
501constexpr inline bool __have_avx512dq = _GLIBCXX_SIMD_HAVE_AVX512DQ;
502constexpr inline bool __have_avx512vl = _GLIBCXX_SIMD_HAVE_AVX512VL;
503constexpr inline bool __have_avx512bw = _GLIBCXX_SIMD_HAVE_AVX512BW;
504constexpr inline bool __have_avx512dq_vl = __have_avx512dq && __have_avx512vl;
505constexpr inline bool __have_avx512bw_vl = __have_avx512bw && __have_avx512vl;
506
507constexpr inline bool __have_neon = _GLIBCXX_SIMD_HAVE_NEON;
508constexpr inline bool __have_neon_a32 = _GLIBCXX_SIMD_HAVE_NEON_A32;
509constexpr inline bool __have_neon_a64 = _GLIBCXX_SIMD_HAVE_NEON_A64;
510constexpr inline bool __support_neon_float =
511#if defined __GCC_IEC_559
512 __GCC_IEC_559 == 0;
513#elif defined __FAST_MATH__
514 true;
515#else
516 false;
517#endif
518
519#ifdef _ARCH_PWR10
520constexpr inline bool __have_power10vec = true;
521#else
522constexpr inline bool __have_power10vec = false;
523#endif
524#ifdef __POWER9_VECTOR__
525constexpr inline bool __have_power9vec = true;
526#else
527constexpr inline bool __have_power9vec = false;
528#endif
529#if defined __POWER8_VECTOR__
530constexpr inline bool __have_power8vec = true;
531#else
532constexpr inline bool __have_power8vec = __have_power9vec;
533#endif
534#if defined __VSX__
535constexpr inline bool __have_power_vsx = true;
536#else
537constexpr inline bool __have_power_vsx = __have_power8vec;
538#endif
539#if defined __ALTIVEC__
540constexpr inline bool __have_power_vmx = true;
541#else
542constexpr inline bool __have_power_vmx = __have_power_vsx;
543#endif
544
545// }}}
546// __is_scalar_abi {{{
547template <typename _Abi>
548 constexpr bool
549 __is_scalar_abi()
550 { return is_same_v<simd_abi::scalar, _Abi>; }
551
552// }}}
553// __abi_bytes_v {{{
554template <template <int> class _Abi, int _Bytes>
555 constexpr int
556 __abi_bytes_impl(_Abi<_Bytes>*)
557 { return _Bytes; }
558
559template <typename _Tp>
560 constexpr int
561 __abi_bytes_impl(_Tp*)
562 { return -1; }
563
564template <typename _Abi>
565 inline constexpr int __abi_bytes_v
566 = __abi_bytes_impl(static_cast<_Abi*>(nullptr));
567
568// }}}
569// __is_builtin_bitmask_abi {{{
570template <typename _Abi>
571 constexpr bool
572 __is_builtin_bitmask_abi()
573 { return is_same_v<simd_abi::_VecBltnBtmsk<__abi_bytes_v<_Abi>>, _Abi>; }
574
575// }}}
576// __is_sse_abi {{{
577template <typename _Abi>
578 constexpr bool
579 __is_sse_abi()
580 {
581 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
582 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
583 }
584
585// }}}
586// __is_avx_abi {{{
587template <typename _Abi>
588 constexpr bool
589 __is_avx_abi()
590 {
591 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
592 return _Bytes > 16 && _Bytes <= 32
593 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
594 }
595
596// }}}
597// __is_avx512_abi {{{
598template <typename _Abi>
599 constexpr bool
600 __is_avx512_abi()
601 {
602 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
603 return _Bytes <= 64 && is_same_v<simd_abi::_Avx512<_Bytes>, _Abi>;
604 }
605
606// }}}
607// __is_neon_abi {{{
608template <typename _Abi>
609 constexpr bool
610 __is_neon_abi()
611 {
612 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
613 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
614 }
615
616// }}}
617// __make_dependent_t {{{
618template <typename, typename _Up>
619 struct __make_dependent
620 { using type = _Up; };
621
622template <typename _Tp, typename _Up>
623 using __make_dependent_t = typename __make_dependent<_Tp, _Up>::type;
624
625// }}}
626// ^^^ ---- type traits ---- ^^^
627
628// __invoke_ub{{{
629template <typename... _Args>
630 [[noreturn]] _GLIBCXX_SIMD_ALWAYS_INLINE void
631 __invoke_ub([[maybe_unused]] const char* __msg, [[maybe_unused]] const _Args&... __args)
632 {
633#ifdef _GLIBCXX_DEBUG_UB
634 __builtin_fprintf(stderr, __msg, __args...);
635 __builtin_trap();
636#else
637 __builtin_unreachable();
638#endif
639 }
640
641// }}}
642// __assert_unreachable{{{
643template <typename _Tp>
644 struct __assert_unreachable
645 { static_assert(!is_same_v<_Tp, _Tp>, "this should be unreachable"); };
646
647// }}}
648// __size_or_zero_v {{{
649template <typename _Tp, typename _Ap, size_t _Np = simd_size<_Tp, _Ap>::value>
650 constexpr size_t
651 __size_or_zero_dispatch(int)
652 { return _Np; }
653
654template <typename _Tp, typename _Ap>
655 constexpr size_t
656 __size_or_zero_dispatch(float)
657 { return 0; }
658
659template <typename _Tp, typename _Ap>
660 inline constexpr size_t __size_or_zero_v
661 = __size_or_zero_dispatch<_Tp, _Ap>(0);
662
663// }}}
664// __div_roundup {{{
665inline constexpr size_t
666__div_roundup(size_t __a, size_t __b)
667{ return (__a + __b - 1) / __b; }
668
669// }}}
670// _ExactBool{{{
671class _ExactBool
672{
673 const bool _M_data;
674
675public:
676 _GLIBCXX_SIMD_INTRINSIC constexpr
677 _ExactBool(bool __b) : _M_data(__b) {}
678
679 _ExactBool(int) = delete;
680
681 _GLIBCXX_SIMD_INTRINSIC constexpr
682 operator bool() const
683 { return _M_data; }
684};
685
686// }}}
687// __may_alias{{{
688/**@internal
689 * Helper __may_alias<_Tp> that turns _Tp into the type to be used for an
690 * aliasing pointer. This adds the __may_alias attribute to _Tp (with compilers
691 * that support it).
692 */
693template <typename _Tp>
694 using __may_alias [[__gnu__::__may_alias__]] = _Tp;
695
696// }}}
697// _UnsupportedBase {{{
698// simd and simd_mask base for unsupported <_Tp, _Abi>
699struct _UnsupportedBase
700{
701 _UnsupportedBase() = delete;
702 _UnsupportedBase(const _UnsupportedBase&) = delete;
703 _UnsupportedBase& operator=(const _UnsupportedBase&) = delete;
704 ~_UnsupportedBase() = delete;
705};
706
707// }}}
708// _InvalidTraits {{{
709/**
710 * @internal
711 * Defines the implementation of __a given <_Tp, _Abi>.
712 *
713 * Implementations must ensure that only valid <_Tp, _Abi> instantiations are
714 * possible. Static assertions in the type definition do not suffice. It is
715 * important that SFINAE works.
716 */
717struct _InvalidTraits
718{
719 using _IsValid = false_type;
720 using _SimdBase = _UnsupportedBase;
721 using _MaskBase = _UnsupportedBase;
722
723 static constexpr size_t _S_full_size = 0;
724 static constexpr bool _S_is_partial = false;
725
726 static constexpr size_t _S_simd_align = 1;
727 struct _SimdImpl;
728 struct _SimdMember {};
729 struct _SimdCastType;
730
731 static constexpr size_t _S_mask_align = 1;
732 struct _MaskImpl;
733 struct _MaskMember {};
734 struct _MaskCastType;
735};
736
737// }}}
738// _SimdTraits {{{
739template <typename _Tp, typename _Abi, typename = void_t<>>
740 struct _SimdTraits : _InvalidTraits {};
741
742// }}}
743// __private_init, __bitset_init{{{
744/**
745 * @internal
746 * Tag used for private init constructor of simd and simd_mask
747 */
748inline constexpr struct _PrivateInit {} __private_init = {};
749
750inline constexpr struct _BitsetInit {} __bitset_init = {};
751
752// }}}
753// __is_narrowing_conversion<_From, _To>{{{
754template <typename _From, typename _To, bool = is_arithmetic_v<_From>,
755 bool = is_arithmetic_v<_To>>
756 struct __is_narrowing_conversion;
757
758// ignore "signed/unsigned mismatch" in the following trait.
759// The implicit conversions will do the right thing here.
760template <typename _From, typename _To>
761 struct __is_narrowing_conversion<_From, _To, true, true>
762 : public __bool_constant<(
763 __digits_v<_From> > __digits_v<_To>
764 || __finite_max_v<_From> > __finite_max_v<_To>
765 || __finite_min_v<_From> < __finite_min_v<_To>
766 || (is_signed_v<_From> && is_unsigned_v<_To>))> {};
767
768template <typename _Tp>
769 struct __is_narrowing_conversion<_Tp, bool, true, true>
770 : public true_type {};
771
772template <>
773 struct __is_narrowing_conversion<bool, bool, true, true>
774 : public false_type {};
775
776template <typename _Tp>
777 struct __is_narrowing_conversion<_Tp, _Tp, true, true>
778 : public false_type {};
779
780template <typename _From, typename _To>
781 struct __is_narrowing_conversion<_From, _To, false, true>
782 : public negation<is_convertible<_From, _To>> {};
783
784// }}}
785// __converts_to_higher_integer_rank{{{
786template <typename _From, typename _To, bool = (sizeof(_From) < sizeof(_To))>
787 struct __converts_to_higher_integer_rank : public true_type {};
788
789// this may fail for char -> short if sizeof(char) == sizeof(short)
790template <typename _From, typename _To>
791 struct __converts_to_higher_integer_rank<_From, _To, false>
792 : public is_same<decltype(declval<_From>() + declval<_To>()), _To> {};
793
794// }}}
795// __data(simd/simd_mask) {{{
796template <typename _Tp, typename _Ap>
797 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
798 __data(const simd<_Tp, _Ap>& __x);
799
800template <typename _Tp, typename _Ap>
801 _GLIBCXX_SIMD_INTRINSIC constexpr auto&
802 __data(simd<_Tp, _Ap>& __x);
803
804template <typename _Tp, typename _Ap>
805 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
806 __data(const simd_mask<_Tp, _Ap>& __x);
807
808template <typename _Tp, typename _Ap>
809 _GLIBCXX_SIMD_INTRINSIC constexpr auto&
810 __data(simd_mask<_Tp, _Ap>& __x);
811
812// }}}
813// _SimdConverter {{{
814template <typename _FromT, typename _FromA, typename _ToT, typename _ToA,
815 typename = void>
816 struct _SimdConverter;
817
818template <typename _Tp, typename _Ap>
819 struct _SimdConverter<_Tp, _Ap, _Tp, _Ap, void>
820 {
821 template <typename _Up>
822 _GLIBCXX_SIMD_INTRINSIC const _Up&
823 operator()(const _Up& __x)
824 { return __x; }
825 };
826
827// }}}
828// __to_value_type_or_member_type {{{
829template <typename _V>
830 _GLIBCXX_SIMD_INTRINSIC constexpr auto
831 __to_value_type_or_member_type(const _V& __x) -> decltype(__data(__x))
832 { return __data(__x); }
833
834template <typename _V>
835 _GLIBCXX_SIMD_INTRINSIC constexpr const typename _V::value_type&
836 __to_value_type_or_member_type(const typename _V::value_type& __x)
837 { return __x; }
838
839// }}}
840// __bool_storage_member_type{{{
841template <size_t _Size>
842 struct __bool_storage_member_type;
843
844template <size_t _Size>
845 using __bool_storage_member_type_t =
846 typename __bool_storage_member_type<_Size>::type;
847
848// }}}
849// _SimdTuple {{{
850// why not tuple?
851// 1. tuple gives no guarantee about the storage order, but I require
852// storage
853// equivalent to array<_Tp, _Np>
854// 2. direct access to the element type (first template argument)
855// 3. enforces equal element type, only different _Abi types are allowed
856template <typename _Tp, typename... _Abis>
857 struct _SimdTuple;
858
859//}}}
860// __fixed_size_storage_t {{{
861template <typename _Tp, int _Np>
862 struct __fixed_size_storage;
863
864template <typename _Tp, int _Np>
865 using __fixed_size_storage_t = typename __fixed_size_storage<_Tp, _Np>::type;
866
867// }}}
868// _SimdWrapper fwd decl{{{
869template <typename _Tp, size_t _Size, typename = void_t<>>
870 struct _SimdWrapper;
871
872template <typename _Tp>
873 using _SimdWrapper8 = _SimdWrapper<_Tp, 8 / sizeof(_Tp)>;
874template <typename _Tp>
875 using _SimdWrapper16 = _SimdWrapper<_Tp, 16 / sizeof(_Tp)>;
876template <typename _Tp>
877 using _SimdWrapper32 = _SimdWrapper<_Tp, 32 / sizeof(_Tp)>;
878template <typename _Tp>
879 using _SimdWrapper64 = _SimdWrapper<_Tp, 64 / sizeof(_Tp)>;
880
881// }}}
882// __is_simd_wrapper {{{
883template <typename _Tp>
884 struct __is_simd_wrapper : false_type {};
885
886template <typename _Tp, size_t _Np>
887 struct __is_simd_wrapper<_SimdWrapper<_Tp, _Np>> : true_type {};
888
889template <typename _Tp>
890 inline constexpr bool __is_simd_wrapper_v = __is_simd_wrapper<_Tp>::value;
891
892// }}}
893// _BitOps {{{
894struct _BitOps
895{
896 // _S_bit_iteration {{{
897 template <typename _Tp, typename _Fp>
898 static void
899 _S_bit_iteration(_Tp __mask, _Fp&& __f)
900 {
901 static_assert(sizeof(_ULLong) >= sizeof(_Tp));
902 conditional_t<sizeof(_Tp) <= sizeof(_UInt), _UInt, _ULLong> __k;
903 if constexpr (is_convertible_v<_Tp, decltype(__k)>)
904 __k = __mask;
905 else
906 __k = __mask.to_ullong();
907 while(__k)
908 {
909 __f(std::__countr_zero(__k));
910 __k &= (__k - 1);
911 }
912 }
913
914 //}}}
915};
916
917//}}}
918// __increment, __decrement {{{
919template <typename _Tp = void>
920 struct __increment
921 { constexpr _Tp operator()(_Tp __a) const { return ++__a; } };
922
923template <>
924 struct __increment<void>
925 {
926 template <typename _Tp>
927 constexpr _Tp
928 operator()(_Tp __a) const
929 { return ++__a; }
930 };
931
932template <typename _Tp = void>
933 struct __decrement
934 { constexpr _Tp operator()(_Tp __a) const { return --__a; } };
935
936template <>
937 struct __decrement<void>
938 {
939 template <typename _Tp>
940 constexpr _Tp
941 operator()(_Tp __a) const
942 { return --__a; }
943 };
944
945// }}}
946// _ValuePreserving(OrInt) {{{
947template <typename _From, typename _To,
948 typename = enable_if_t<negation<
949 __is_narrowing_conversion<__remove_cvref_t<_From>, _To>>::value>>
950 using _ValuePreserving = _From;
951
952template <typename _From, typename _To,
953 typename _DecayedFrom = __remove_cvref_t<_From>,
954 typename = enable_if_t<conjunction<
955 is_convertible<_From, _To>,
956 disjunction<
957 is_same<_DecayedFrom, _To>, is_same<_DecayedFrom, int>,
958 conjunction<is_same<_DecayedFrom, _UInt>, is_unsigned<_To>>,
959 negation<__is_narrowing_conversion<_DecayedFrom, _To>>>>::value>>
960 using _ValuePreservingOrInt = _From;
961
962// }}}
963// __intrinsic_type {{{
964template <typename _Tp, size_t _Bytes, typename = void_t<>>
965 struct __intrinsic_type;
966
967template <typename _Tp, size_t _Size>
968 using __intrinsic_type_t =
969 typename __intrinsic_type<_Tp, _Size * sizeof(_Tp)>::type;
970
971template <typename _Tp>
972 using __intrinsic_type2_t = typename __intrinsic_type<_Tp, 2>::type;
973template <typename _Tp>
974 using __intrinsic_type4_t = typename __intrinsic_type<_Tp, 4>::type;
975template <typename _Tp>
976 using __intrinsic_type8_t = typename __intrinsic_type<_Tp, 8>::type;
977template <typename _Tp>
978 using __intrinsic_type16_t = typename __intrinsic_type<_Tp, 16>::type;
979template <typename _Tp>
980 using __intrinsic_type32_t = typename __intrinsic_type<_Tp, 32>::type;
981template <typename _Tp>
982 using __intrinsic_type64_t = typename __intrinsic_type<_Tp, 64>::type;
983
984// }}}
985// _BitMask {{{
986template <size_t _Np, bool _Sanitized = false>
987 struct _BitMask;
988
989template <size_t _Np, bool _Sanitized>
990 struct __is_bitmask<_BitMask<_Np, _Sanitized>, void> : true_type {};
991
992template <size_t _Np>
993 using _SanitizedBitMask = _BitMask<_Np, true>;
994
995template <size_t _Np, bool _Sanitized>
996 struct _BitMask
997 {
998 static_assert(_Np > 0);
999
1000 static constexpr size_t _NBytes = __div_roundup(_Np, __CHAR_BIT__);
1001
1002 using _Tp = conditional_t<_Np == 1, bool,
1003 make_unsigned_t<__int_with_sizeof_t<std::min(
1004 sizeof(_ULLong), std::__bit_ceil(_NBytes))>>>;
1005
1006 static constexpr int _S_array_size = __div_roundup(_NBytes, sizeof(_Tp));
1007
1008 _Tp _M_bits[_S_array_size];
1009
1010 static constexpr int _S_unused_bits
1011 = _Np == 1 ? 0 : _S_array_size * sizeof(_Tp) * __CHAR_BIT__ - _Np;
1012
1013 static constexpr _Tp _S_bitmask = +_Tp(~_Tp()) >> _S_unused_bits;
1014
1015 constexpr _BitMask() noexcept = default;
1016
1017 constexpr _BitMask(unsigned long long __x) noexcept
1018 : _M_bits{static_cast<_Tp>(__x)} {}
1019
1020 _BitMask(bitset<_Np> __x) noexcept : _BitMask(__x.to_ullong()) {}
1021
1022 constexpr _BitMask(const _BitMask&) noexcept = default;
1023
1024 template <bool _RhsSanitized, typename = enable_if_t<_RhsSanitized == false
1025 && _Sanitized == true>>
1026 constexpr _BitMask(const _BitMask<_Np, _RhsSanitized>& __rhs) noexcept
1027 : _BitMask(__rhs._M_sanitized()) {}
1028
1029 constexpr operator _SimdWrapper<bool, _Np>() const noexcept
1030 {
1031 static_assert(_S_array_size == 1);
1032 return _M_bits[0];
1033 }
1034
1035 // precondition: is sanitized
1036 constexpr _Tp
1037 _M_to_bits() const noexcept
1038 {
1039 static_assert(_S_array_size == 1);
1040 return _M_bits[0];
1041 }
1042
1043 // precondition: is sanitized
1044 constexpr unsigned long long
1045 to_ullong() const noexcept
1046 {
1047 static_assert(_S_array_size == 1);
1048 return _M_bits[0];
1049 }
1050
1051 // precondition: is sanitized
1052 constexpr unsigned long
1053 to_ulong() const noexcept
1054 {
1055 static_assert(_S_array_size == 1);
1056 return _M_bits[0];
1057 }
1058
1059 constexpr bitset<_Np>
1060 _M_to_bitset() const noexcept
1061 {
1062 static_assert(_S_array_size == 1);
1063 return _M_bits[0];
1064 }
1065
1066 constexpr decltype(auto)
1067 _M_sanitized() const noexcept
1068 {
1069 if constexpr (_Sanitized)
1070 return *this;
1071 else if constexpr (_Np == 1)
1072 return _SanitizedBitMask<_Np>(_M_bits[0]);
1073 else
1074 {
1075 _SanitizedBitMask<_Np> __r = {};
1076 for (int __i = 0; __i < _S_array_size; ++__i)
1077 __r._M_bits[__i] = _M_bits[__i];
1078 if constexpr (_S_unused_bits > 0)
1079 __r._M_bits[_S_array_size - 1] &= _S_bitmask;
1080 return __r;
1081 }
1082 }
1083
1084 template <size_t _Mp, bool _LSanitized>
1085 constexpr _BitMask<_Np + _Mp, _Sanitized>
1086 _M_prepend(_BitMask<_Mp, _LSanitized> __lsb) const noexcept
1087 {
1088 constexpr size_t _RN = _Np + _Mp;
1089 using _Rp = _BitMask<_RN, _Sanitized>;
1090 if constexpr (_Rp::_S_array_size == 1)
1091 {
1092 _Rp __r{{_M_bits[0]}};
1093 __r._M_bits[0] <<= _Mp;
1094 __r._M_bits[0] |= __lsb._M_sanitized()._M_bits[0];
1095 return __r;
1096 }
1097 else
1098 __assert_unreachable<_Rp>();
1099 }
1100
1101 // Return a new _BitMask with size _NewSize while dropping _DropLsb least
1102 // significant bits. If the operation implicitly produces a sanitized bitmask,
1103 // the result type will have _Sanitized set.
1104 template <size_t _DropLsb, size_t _NewSize = _Np - _DropLsb>
1105 constexpr auto
1106 _M_extract() const noexcept
1107 {
1108 static_assert(_Np > _DropLsb);
1109 static_assert(_DropLsb + _NewSize <= sizeof(_ULLong) * __CHAR_BIT__,
1110 "not implemented for bitmasks larger than one ullong");
1111 if constexpr (_NewSize == 1)
1112 // must sanitize because the return _Tp is bool
1113 return _SanitizedBitMask<1>(_M_bits[0] & (_Tp(1) << _DropLsb));
1114 else
1115 return _BitMask<_NewSize,
1116 ((_NewSize + _DropLsb == sizeof(_Tp) * __CHAR_BIT__
1117 && _NewSize + _DropLsb <= _Np)
1118 || ((_Sanitized || _Np == sizeof(_Tp) * __CHAR_BIT__)
1119 && _NewSize + _DropLsb >= _Np))>(_M_bits[0]
1120 >> _DropLsb);
1121 }
1122
1123 // True if all bits are set. Implicitly sanitizes if _Sanitized == false.
1124 constexpr bool
1125 all() const noexcept
1126 {
1127 if constexpr (_Np == 1)
1128 return _M_bits[0];
1129 else if constexpr (!_Sanitized)
1130 return _M_sanitized().all();
1131 else
1132 {
1133 constexpr _Tp __allbits = ~_Tp();
1134 for (int __i = 0; __i < _S_array_size - 1; ++__i)
1135 if (_M_bits[__i] != __allbits)
1136 return false;
1137 return _M_bits[_S_array_size - 1] == _S_bitmask;
1138 }
1139 }
1140
1141 // True if at least one bit is set. Implicitly sanitizes if _Sanitized ==
1142 // false.
1143 constexpr bool
1144 any() const noexcept
1145 {
1146 if constexpr (_Np == 1)
1147 return _M_bits[0];
1148 else if constexpr (!_Sanitized)
1149 return _M_sanitized().any();
1150 else
1151 {
1152 for (int __i = 0; __i < _S_array_size - 1; ++__i)
1153 if (_M_bits[__i] != 0)
1154 return true;
1155 return _M_bits[_S_array_size - 1] != 0;
1156 }
1157 }
1158
1159 // True if no bit is set. Implicitly sanitizes if _Sanitized == false.
1160 constexpr bool
1161 none() const noexcept
1162 {
1163 if constexpr (_Np == 1)
1164 return !_M_bits[0];
1165 else if constexpr (!_Sanitized)
1166 return _M_sanitized().none();
1167 else
1168 {
1169 for (int __i = 0; __i < _S_array_size - 1; ++__i)
1170 if (_M_bits[__i] != 0)
1171 return false;
1172 return _M_bits[_S_array_size - 1] == 0;
1173 }
1174 }
1175
1176 // Returns the number of set bits. Implicitly sanitizes if _Sanitized ==
1177 // false.
1178 constexpr int
1179 count() const noexcept
1180 {
1181 if constexpr (_Np == 1)
1182 return _M_bits[0];
1183 else if constexpr (!_Sanitized)
1184 return _M_sanitized().none();
1185 else
1186 {
1187 int __result = __builtin_popcountll(_M_bits[0]);
1188 for (int __i = 1; __i < _S_array_size; ++__i)
1189 __result += __builtin_popcountll(_M_bits[__i]);
1190 return __result;
1191 }
1192 }
1193
1194 // Returns the bit at offset __i as bool.
1195 constexpr bool
1196 operator[](size_t __i) const noexcept
1197 {
1198 if constexpr (_Np == 1)
1199 return _M_bits[0];
1200 else if constexpr (_S_array_size == 1)
1201 return (_M_bits[0] >> __i) & 1;
1202 else
1203 {
1204 const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1205 const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1206 return (_M_bits[__j] >> __shift) & 1;
1207 }
1208 }
1209
1210 template <size_t __i>
1211 constexpr bool
1212 operator[](_SizeConstant<__i>) const noexcept
1213 {
1214 static_assert(__i < _Np);
1215 constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1216 constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1217 return static_cast<bool>(_M_bits[__j] & (_Tp(1) << __shift));
1218 }
1219
1220 // Set the bit at offset __i to __x.
1221 constexpr void
1222 set(size_t __i, bool __x) noexcept
1223 {
1224 if constexpr (_Np == 1)
1225 _M_bits[0] = __x;
1226 else if constexpr (_S_array_size == 1)
1227 {
1228 _M_bits[0] &= ~_Tp(_Tp(1) << __i);
1229 _M_bits[0] |= _Tp(_Tp(__x) << __i);
1230 }
1231 else
1232 {
1233 const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1234 const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1235 _M_bits[__j] &= ~_Tp(_Tp(1) << __shift);
1236 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1237 }
1238 }
1239
1240 template <size_t __i>
1241 constexpr void
1242 set(_SizeConstant<__i>, bool __x) noexcept
1243 {
1244 static_assert(__i < _Np);
1245 if constexpr (_Np == 1)
1246 _M_bits[0] = __x;
1247 else
1248 {
1249 constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1250 constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1251 constexpr _Tp __mask = ~_Tp(_Tp(1) << __shift);
1252 _M_bits[__j] &= __mask;
1253 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1254 }
1255 }
1256
1257 // Inverts all bits. Sanitized input leads to sanitized output.
1258 constexpr _BitMask
1259 operator~() const noexcept
1260 {
1261 if constexpr (_Np == 1)
1262 return !_M_bits[0];
1263 else
1264 {
1265 _BitMask __result{};
1266 for (int __i = 0; __i < _S_array_size - 1; ++__i)
1267 __result._M_bits[__i] = ~_M_bits[__i];
1268 if constexpr (_Sanitized)
1269 __result._M_bits[_S_array_size - 1]
1270 = _M_bits[_S_array_size - 1] ^ _S_bitmask;
1271 else
1272 __result._M_bits[_S_array_size - 1] = ~_M_bits[_S_array_size - 1];
1273 return __result;
1274 }
1275 }
1276
1277 constexpr _BitMask&
1278 operator^=(const _BitMask& __b) & noexcept
1279 {
1280 __execute_n_times<_S_array_size>(
1281 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] ^= __b._M_bits[__i]; });
1282 return *this;
1283 }
1284
1285 constexpr _BitMask&
1286 operator|=(const _BitMask& __b) & noexcept
1287 {
1288 __execute_n_times<_S_array_size>(
1289 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] |= __b._M_bits[__i]; });
1290 return *this;
1291 }
1292
1293 constexpr _BitMask&
1294 operator&=(const _BitMask& __b) & noexcept
1295 {
1296 __execute_n_times<_S_array_size>(
1297 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] &= __b._M_bits[__i]; });
1298 return *this;
1299 }
1300
1301 friend constexpr _BitMask
1302 operator^(const _BitMask& __a, const _BitMask& __b) noexcept
1303 {
1304 _BitMask __r = __a;
1305 __r ^= __b;
1306 return __r;
1307 }
1308
1309 friend constexpr _BitMask
1310 operator|(const _BitMask& __a, const _BitMask& __b) noexcept
1311 {
1312 _BitMask __r = __a;
1313 __r |= __b;
1314 return __r;
1315 }
1316
1317 friend constexpr _BitMask
1318 operator&(const _BitMask& __a, const _BitMask& __b) noexcept
1319 {
1320 _BitMask __r = __a;
1321 __r &= __b;
1322 return __r;
1323 }
1324
1325 _GLIBCXX_SIMD_INTRINSIC
1326 constexpr bool
1327 _M_is_constprop() const
1328 {
1329 if constexpr (_S_array_size == 0)
1330 return __builtin_constant_p(_M_bits[0]);
1331 else
1332 {
1333 for (int __i = 0; __i < _S_array_size; ++__i)
1334 if (!__builtin_constant_p(_M_bits[__i]))
1335 return false;
1336 return true;
1337 }
1338 }
1339 };
1340
1341// }}}
1342
1343// vvv ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- vvv
1344// __min_vector_size {{{
1345template <typename _Tp = void>
1346 static inline constexpr int __min_vector_size = 2 * sizeof(_Tp);
1347
1348#if _GLIBCXX_SIMD_HAVE_NEON
1349template <>
1350 inline constexpr int __min_vector_size<void> = 8;
1351#else
1352template <>
1353 inline constexpr int __min_vector_size<void> = 16;
1354#endif
1355
1356// }}}
1357// __vector_type {{{
1358template <typename _Tp, size_t _Np, typename = void>
1359 struct __vector_type_n {};
1360
1361// substition failure for 0-element case
1362template <typename _Tp>
1363 struct __vector_type_n<_Tp, 0, void> {};
1364
1365// special case 1-element to be _Tp itself
1366template <typename _Tp>
1367 struct __vector_type_n<_Tp, 1, enable_if_t<__is_vectorizable_v<_Tp>>>
1368 { using type = _Tp; };
1369
1370// else, use GNU-style builtin vector types
1371template <typename _Tp, size_t _Np>
1372 struct __vector_type_n<_Tp, _Np, enable_if_t<__is_vectorizable_v<_Tp> && _Np >= 2>>
1373 {
1374 static constexpr size_t _S_Np2 = std::__bit_ceil(_Np * sizeof(_Tp));
1375
1376 static constexpr size_t _S_Bytes =
1377#ifdef __i386__
1378 // Using [[gnu::vector_size(8)]] would wreak havoc on the FPU because
1379 // those objects are passed via MMX registers and nothing ever calls EMMS.
1380 _S_Np2 == 8 ? 16 :
1381#endif
1382 _S_Np2 < __min_vector_size<_Tp> ? __min_vector_size<_Tp>
1383 : _S_Np2;
1384
1385 using type [[__gnu__::__vector_size__(_S_Bytes)]] = _Tp;
1386 };
1387
1388template <typename _Tp, size_t _Bytes, size_t = _Bytes % sizeof(_Tp)>
1389 struct __vector_type;
1390
1391template <typename _Tp, size_t _Bytes>
1392 struct __vector_type<_Tp, _Bytes, 0>
1393 : __vector_type_n<_Tp, _Bytes / sizeof(_Tp)> {};
1394
1395template <typename _Tp, size_t _Size>
1396 using __vector_type_t = typename __vector_type_n<_Tp, _Size>::type;
1397
1398template <typename _Tp>
1399 using __vector_type2_t = typename __vector_type<_Tp, 2>::type;
1400template <typename _Tp>
1401 using __vector_type4_t = typename __vector_type<_Tp, 4>::type;
1402template <typename _Tp>
1403 using __vector_type8_t = typename __vector_type<_Tp, 8>::type;
1404template <typename _Tp>
1405 using __vector_type16_t = typename __vector_type<_Tp, 16>::type;
1406template <typename _Tp>
1407 using __vector_type32_t = typename __vector_type<_Tp, 32>::type;
1408template <typename _Tp>
1409 using __vector_type64_t = typename __vector_type<_Tp, 64>::type;
1410
1411// }}}
1412// __is_vector_type {{{
1413template <typename _Tp, typename = void_t<>>
1414 struct __is_vector_type : false_type {};
1415
1416template <typename _Tp>
1417 struct __is_vector_type<
1418 _Tp, void_t<typename __vector_type<
1419 remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>>
1420 : is_same<_Tp, typename __vector_type<
1421 remove_reference_t<decltype(declval<_Tp>()[0])>,
1422 sizeof(_Tp)>::type> {};
1423
1424template <typename _Tp>
1425 inline constexpr bool __is_vector_type_v = __is_vector_type<_Tp>::value;
1426
1427// }}}
1428// __is_intrinsic_type {{{
1429#if _GLIBCXX_SIMD_HAVE_SSE_ABI
1430template <typename _Tp>
1431 using __is_intrinsic_type = __is_vector_type<_Tp>;
1432#else // not SSE (x86)
1433template <typename _Tp, typename = void_t<>>
1434 struct __is_intrinsic_type : false_type {};
1435
1436template <typename _Tp>
1437 struct __is_intrinsic_type<
1438 _Tp, void_t<typename __intrinsic_type<
1439 remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>>
1440 : is_same<_Tp, typename __intrinsic_type<
1441 remove_reference_t<decltype(declval<_Tp>()[0])>,
1442 sizeof(_Tp)>::type> {};
1443#endif
1444
1445template <typename _Tp>
1446 inline constexpr bool __is_intrinsic_type_v = __is_intrinsic_type<_Tp>::value;
1447
1448// }}}
1449// _VectorTraits{{{
1450template <typename _Tp, typename = void_t<>>
1451 struct _VectorTraitsImpl;
1452
1453template <typename _Tp>
1454 struct _VectorTraitsImpl<_Tp, enable_if_t<__is_vector_type_v<_Tp>
1455 || __is_intrinsic_type_v<_Tp>>>
1456 {
1457 using type = _Tp;
1458 using value_type = remove_reference_t<decltype(declval<_Tp>()[0])>;
1459 static constexpr int _S_full_size = sizeof(_Tp) / sizeof(value_type);
1460 using _Wrapper = _SimdWrapper<value_type, _S_full_size>;
1461 template <typename _Up, int _W = _S_full_size>
1462 static constexpr bool _S_is
1463 = is_same_v<value_type, _Up> && _W == _S_full_size;
1464 };
1465
1466template <typename _Tp, size_t _Np>
1467 struct _VectorTraitsImpl<_SimdWrapper<_Tp, _Np>,
1468 void_t<__vector_type_t<_Tp, _Np>>>
1469 {
1470 using type = __vector_type_t<_Tp, _Np>;
1471 using value_type = _Tp;
1472 static constexpr int _S_full_size = sizeof(type) / sizeof(value_type);
1473 using _Wrapper = _SimdWrapper<_Tp, _Np>;
1474 static constexpr bool _S_is_partial = (_Np == _S_full_size);
1475 static constexpr int _S_partial_width = _Np;
1476 template <typename _Up, int _W = _S_full_size>
1477 static constexpr bool _S_is
1478 = is_same_v<value_type, _Up>&& _W == _S_full_size;
1479 };
1480
1481template <typename _Tp, typename = typename _VectorTraitsImpl<_Tp>::type>
1482 using _VectorTraits = _VectorTraitsImpl<_Tp>;
1483
1484// }}}
1485// __as_vector{{{
1486template <typename _V>
1487 _GLIBCXX_SIMD_INTRINSIC constexpr auto
1488 __as_vector(_V __x)
1489 {
1490 if constexpr (__is_vector_type_v<_V>)
1491 return __x;
1492 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1493 return __data(__x)._M_data;
1494 else if constexpr (__is_vectorizable_v<_V>)
1495 return __vector_type_t<_V, 2>{__x};
1496 else
1497 return __x._M_data;
1498 }
1499
1500// }}}
1501// __as_wrapper{{{
1502template <size_t _Np = 0, typename _V>
1503 _GLIBCXX_SIMD_INTRINSIC constexpr auto
1504 __as_wrapper(_V __x)
1505 {
1506 if constexpr (__is_vector_type_v<_V>)
1507 return _SimdWrapper<typename _VectorTraits<_V>::value_type,
1508 (_Np > 0 ? _Np : _VectorTraits<_V>::_S_full_size)>(__x);
1509 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1510 {
1511 static_assert(_V::size() == _Np);
1512 return __data(__x);
1513 }
1514 else
1515 {
1516 static_assert(_V::_S_size == _Np);
1517 return __x;
1518 }
1519 }
1520
1521// }}}
1522// __intrin_bitcast{{{
1523template <typename _To, typename _From>
1524 _GLIBCXX_SIMD_INTRINSIC constexpr _To
1525 __intrin_bitcast(_From __v)
1526 {
1527 static_assert((__is_vector_type_v<_From> || __is_intrinsic_type_v<_From>)
1528 && (__is_vector_type_v<_To> || __is_intrinsic_type_v<_To>));
1529 if constexpr (sizeof(_To) == sizeof(_From))
1530 return reinterpret_cast<_To>(__v);
1531 else if constexpr (sizeof(_From) > sizeof(_To))
1532 if constexpr (sizeof(_To) >= 16)
1533 return reinterpret_cast<const __may_alias<_To>&>(__v);
1534 else
1535 {
1536 _To __r;
1537 __builtin_memcpy(&__r, &__v, sizeof(_To));
1538 return __r;
1539 }
1540#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1541 else if constexpr (__have_avx && sizeof(_From) == 16 && sizeof(_To) == 32)
1542 return reinterpret_cast<_To>(__builtin_ia32_ps256_ps(
1543 reinterpret_cast<__vector_type_t<float, 4>>(__v)));
1544 else if constexpr (__have_avx512f && sizeof(_From) == 16
1545 && sizeof(_To) == 64)
1546 return reinterpret_cast<_To>(__builtin_ia32_ps512_ps(
1547 reinterpret_cast<__vector_type_t<float, 4>>(__v)));
1548 else if constexpr (__have_avx512f && sizeof(_From) == 32
1549 && sizeof(_To) == 64)
1550 return reinterpret_cast<_To>(__builtin_ia32_ps512_256ps(
1551 reinterpret_cast<__vector_type_t<float, 8>>(__v)));
1552#endif // _GLIBCXX_SIMD_X86INTRIN
1553 else if constexpr (sizeof(__v) <= 8)
1554 return reinterpret_cast<_To>(
1555 __vector_type_t<__int_for_sizeof_t<_From>, sizeof(_To) / sizeof(_From)>{
1556 reinterpret_cast<__int_for_sizeof_t<_From>>(__v)});
1557 else
1558 {
1559 static_assert(sizeof(_To) > sizeof(_From));
1560 _To __r = {};
1561 __builtin_memcpy(&__r, &__v, sizeof(_From));
1562 return __r;
1563 }
1564 }
1565
1566// }}}
1567// __vector_bitcast{{{
1568template <typename _To, size_t _NN = 0, typename _From,
1569 typename _FromVT = _VectorTraits<_From>,
1570 size_t _Np = _NN == 0 ? sizeof(_From) / sizeof(_To) : _NN>
1571 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1572 __vector_bitcast(_From __x)
1573 {
1574 using _R = __vector_type_t<_To, _Np>;
1575 return __intrin_bitcast<_R>(__x);
1576 }
1577
1578template <typename _To, size_t _NN = 0, typename _Tp, size_t _Nx,
1579 size_t _Np
1580 = _NN == 0 ? sizeof(_SimdWrapper<_Tp, _Nx>) / sizeof(_To) : _NN>
1581 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1582 __vector_bitcast(const _SimdWrapper<_Tp, _Nx>& __x)
1583 {
1584 static_assert(_Np > 1);
1585 return __intrin_bitcast<__vector_type_t<_To, _Np>>(__x._M_data);
1586 }
1587
1588// }}}
1589// __convert_x86 declarations {{{
1590#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
1591template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1592 _To __convert_x86(_Tp);
1593
1594template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1595 _To __convert_x86(_Tp, _Tp);
1596
1597template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1598 _To __convert_x86(_Tp, _Tp, _Tp, _Tp);
1599
1600template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1601 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp);
1602
1603template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1604 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp,
1605 _Tp, _Tp, _Tp, _Tp);
1606#endif // _GLIBCXX_SIMD_WORKAROUND_PR85048
1607
1608//}}}
1609// __bit_cast {{{
1610template <typename _To, typename _From>
1611 _GLIBCXX_SIMD_INTRINSIC constexpr _To
1612 __bit_cast(const _From __x)
1613 {
1614 // TODO: implement with / replace by __builtin_bit_cast ASAP
1615 static_assert(sizeof(_To) == sizeof(_From));
1616 constexpr bool __to_is_vectorizable
1617 = is_arithmetic_v<_To> || is_enum_v<_To>;
1618 constexpr bool __from_is_vectorizable
1619 = is_arithmetic_v<_From> || is_enum_v<_From>;
1620 if constexpr (__is_vector_type_v<_To> && __is_vector_type_v<_From>)
1621 return reinterpret_cast<_To>(__x);
1622 else if constexpr (__is_vector_type_v<_To> && __from_is_vectorizable)
1623 {
1624 using _FV [[__gnu__::__vector_size__(sizeof(_From))]] = _From;
1625 return reinterpret_cast<_To>(_FV{__x});
1626 }
1627 else if constexpr (__to_is_vectorizable && __from_is_vectorizable)
1628 {
1629 using _TV [[__gnu__::__vector_size__(sizeof(_To))]] = _To;
1630 using _FV [[__gnu__::__vector_size__(sizeof(_From))]] = _From;
1631 return reinterpret_cast<_TV>(_FV{__x})[0];
1632 }
1633 else if constexpr (__to_is_vectorizable && __is_vector_type_v<_From>)
1634 {
1635 using _TV [[__gnu__::__vector_size__(sizeof(_To))]] = _To;
1636 return reinterpret_cast<_TV>(__x)[0];
1637 }
1638 else
1639 {
1640 _To __r;
1641 __builtin_memcpy(reinterpret_cast<char*>(&__r),
1642 reinterpret_cast<const char*>(&__x), sizeof(_To));
1643 return __r;
1644 }
1645 }
1646
1647// }}}
1648// __to_intrin {{{
1649template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
1650 typename _R = __intrinsic_type_t<typename _TVT::value_type, _TVT::_S_full_size>>
1651 _GLIBCXX_SIMD_INTRINSIC constexpr _R
1652 __to_intrin(_Tp __x)
1653 {
1654 static_assert(sizeof(__x) <= sizeof(_R),
1655 "__to_intrin may never drop values off the end");
1656 if constexpr (sizeof(__x) == sizeof(_R))
1657 return reinterpret_cast<_R>(__as_vector(__x));
1658 else
1659 {
1660 using _Up = __int_for_sizeof_t<_Tp>;
1661 return reinterpret_cast<_R>(
1662 __vector_type_t<_Up, sizeof(_R) / sizeof(_Up)>{__bit_cast<_Up>(__x)});
1663 }
1664 }
1665
1666// }}}
1667// __make_vector{{{
1668template <typename _Tp, typename... _Args>
1669 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, sizeof...(_Args)>
1670 __make_vector(const _Args&... __args)
1671 { return __vector_type_t<_Tp, sizeof...(_Args)>{static_cast<_Tp>(__args)...}; }
1672
1673// }}}
1674// __vector_broadcast{{{
1675template <size_t _Np, typename _Tp, size_t... _I>
1676 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1677 __vector_broadcast_impl(_Tp __x, index_sequence<_I...>)
1678 { return __vector_type_t<_Tp, _Np>{((void)_I, __x)...}; }
1679
1680template <size_t _Np, typename _Tp>
1681 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1682 __vector_broadcast(_Tp __x)
1683 { return __vector_broadcast_impl<_Np, _Tp>(__x, make_index_sequence<_Np>()); }
1684
1685// }}}
1686// __generate_vector{{{
1687 template <typename _Tp, size_t _Np, typename _Gp, size_t... _I>
1688 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1689 __generate_vector_impl(_Gp&& __gen, index_sequence<_I...>)
1690 { return __vector_type_t<_Tp, _Np>{ static_cast<_Tp>(__gen(_SizeConstant<_I>()))...}; }
1691
1692template <typename _V, typename _VVT = _VectorTraits<_V>, typename _Gp>
1693 _GLIBCXX_SIMD_INTRINSIC constexpr _V
1694 __generate_vector(_Gp&& __gen)
1695 {
1696 if constexpr (__is_vector_type_v<_V>)
1697 return __generate_vector_impl<typename _VVT::value_type,
1698 _VVT::_S_full_size>(
1699 static_cast<_Gp&&>(__gen), make_index_sequence<_VVT::_S_full_size>());
1700 else
1701 return __generate_vector_impl<typename _VVT::value_type,
1702 _VVT::_S_partial_width>(
1703 static_cast<_Gp&&>(__gen),
1704 make_index_sequence<_VVT::_S_partial_width>());
1705 }
1706
1707template <typename _Tp, size_t _Np, typename _Gp>
1708 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1709 __generate_vector(_Gp&& __gen)
1710 {
1711 return __generate_vector_impl<_Tp, _Np>(static_cast<_Gp&&>(__gen),
1712 make_index_sequence<_Np>());
1713 }
1714
1715// }}}
1716// __xor{{{
1717template <typename _TW>
1718 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1719 __xor(_TW __a, _TW __b) noexcept
1720 {
1721 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1722 {
1723 using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1724 _VectorTraitsImpl<_TW>>::value_type;
1725 if constexpr (is_floating_point_v<_Tp>)
1726 {
1727 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1728 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1729 ^ __vector_bitcast<_Ip>(__b));
1730 }
1731 else if constexpr (__is_vector_type_v<_TW>)
1732 return __a ^ __b;
1733 else
1734 return __a._M_data ^ __b._M_data;
1735 }
1736 else
1737 return __a ^ __b;
1738 }
1739
1740// }}}
1741// __or{{{
1742template <typename _TW>
1743 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1744 __or(_TW __a, _TW __b) noexcept
1745 {
1746 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1747 {
1748 using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1749 _VectorTraitsImpl<_TW>>::value_type;
1750 if constexpr (is_floating_point_v<_Tp>)
1751 {
1752 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1753 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1754 | __vector_bitcast<_Ip>(__b));
1755 }
1756 else if constexpr (__is_vector_type_v<_TW>)
1757 return __a | __b;
1758 else
1759 return __a._M_data | __b._M_data;
1760 }
1761 else
1762 return __a | __b;
1763 }
1764
1765// }}}
1766// __and{{{
1767template <typename _TW>
1768 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1769 __and(_TW __a, _TW __b) noexcept
1770 {
1771 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1772 {
1773 using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1774 _VectorTraitsImpl<_TW>>::value_type;
1775 if constexpr (is_floating_point_v<_Tp>)
1776 {
1777 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1778 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1779 & __vector_bitcast<_Ip>(__b));
1780 }
1781 else if constexpr (__is_vector_type_v<_TW>)
1782 return __a & __b;
1783 else
1784 return __a._M_data & __b._M_data;
1785 }
1786 else
1787 return __a & __b;
1788 }
1789
1790// }}}
1791// __andnot{{{
1792#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1793static constexpr struct
1794{
1795 _GLIBCXX_SIMD_INTRINSIC __v4sf
1796 operator()(__v4sf __a, __v4sf __b) const noexcept
1797 { return __builtin_ia32_andnps(__a, __b); }
1798
1799 _GLIBCXX_SIMD_INTRINSIC __v2df
1800 operator()(__v2df __a, __v2df __b) const noexcept
1801 { return __builtin_ia32_andnpd(__a, __b); }
1802
1803 _GLIBCXX_SIMD_INTRINSIC __v2di
1804 operator()(__v2di __a, __v2di __b) const noexcept
1805 { return __builtin_ia32_pandn128(__a, __b); }
1806
1807 _GLIBCXX_SIMD_INTRINSIC __v8sf
1808 operator()(__v8sf __a, __v8sf __b) const noexcept
1809 { return __builtin_ia32_andnps256(__a, __b); }
1810
1811 _GLIBCXX_SIMD_INTRINSIC __v4df
1812 operator()(__v4df __a, __v4df __b) const noexcept
1813 { return __builtin_ia32_andnpd256(__a, __b); }
1814
1815 _GLIBCXX_SIMD_INTRINSIC __v4di
1816 operator()(__v4di __a, __v4di __b) const noexcept
1817 {
1818 if constexpr (__have_avx2)
1819 return __builtin_ia32_andnotsi256(__a, __b);
1820 else
1821 return reinterpret_cast<__v4di>(
1822 __builtin_ia32_andnpd256(reinterpret_cast<__v4df>(__a),
1823 reinterpret_cast<__v4df>(__b)));
1824 }
1825
1826 _GLIBCXX_SIMD_INTRINSIC __v16sf
1827 operator()(__v16sf __a, __v16sf __b) const noexcept
1828 {
1829 if constexpr (__have_avx512dq)
1830 return _mm512_andnot_ps(__a, __b);
1831 else
1832 return reinterpret_cast<__v16sf>(
1833 _mm512_andnot_si512(reinterpret_cast<__v8di>(__a),
1834 reinterpret_cast<__v8di>(__b)));
1835 }
1836
1837 _GLIBCXX_SIMD_INTRINSIC __v8df
1838 operator()(__v8df __a, __v8df __b) const noexcept
1839 {
1840 if constexpr (__have_avx512dq)
1841 return _mm512_andnot_pd(__a, __b);
1842 else
1843 return reinterpret_cast<__v8df>(
1844 _mm512_andnot_si512(reinterpret_cast<__v8di>(__a),
1845 reinterpret_cast<__v8di>(__b)));
1846 }
1847
1848 _GLIBCXX_SIMD_INTRINSIC __v8di
1849 operator()(__v8di __a, __v8di __b) const noexcept
1850 { return _mm512_andnot_si512(__a, __b); }
1851} _S_x86_andnot;
1852#endif // _GLIBCXX_SIMD_X86INTRIN && !__clang__
1853
1854template <typename _TW>
1855 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1856 __andnot(_TW __a, _TW __b) noexcept
1857 {
1858 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1859 {
1860 using _TVT = conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1861 _VectorTraitsImpl<_TW>>;
1862 using _Tp = typename _TVT::value_type;
1863#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1864 if constexpr (sizeof(_TW) >= 16)
1865 {
1866 const auto __ai = __to_intrin(__a);
1867 const auto __bi = __to_intrin(__b);
1868 if (!__builtin_is_constant_evaluated()
1869 && !(__builtin_constant_p(__ai) && __builtin_constant_p(__bi)))
1870 {
1871 const auto __r = _S_x86_andnot(__ai, __bi);
1872 if constexpr (is_convertible_v<decltype(__r), _TW>)
1873 return __r;
1874 else
1875 return reinterpret_cast<typename _TVT::type>(__r);
1876 }
1877 }
1878#endif // _GLIBCXX_SIMD_X86INTRIN
1879 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1880 return __vector_bitcast<_Tp>(~__vector_bitcast<_Ip>(__a)
1881 & __vector_bitcast<_Ip>(__b));
1882 }
1883 else
1884 return ~__a & __b;
1885 }
1886
1887// }}}
1888// __not{{{
1889template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1890 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
1891 __not(_Tp __a) noexcept
1892 {
1893 if constexpr (is_floating_point_v<typename _TVT::value_type>)
1894 return reinterpret_cast<typename _TVT::type>(
1895 ~__vector_bitcast<unsigned>(__a));
1896 else
1897 return ~__a;
1898 }
1899
1900// }}}
1901// __concat{{{
1902template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
1903 typename _R = __vector_type_t<typename _TVT::value_type, _TVT::_S_full_size * 2>>
1904 constexpr _R
1905 __concat(_Tp a_, _Tp b_)
1906 {
1907#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
1908 using _W
1909 = conditional_t<is_floating_point_v<typename _TVT::value_type>, double,
1910 conditional_t<(sizeof(_Tp) >= 2 * sizeof(long long)),
1911 long long, typename _TVT::value_type>>;
1912 constexpr int input_width = sizeof(_Tp) / sizeof(_W);
1913 const auto __a = __vector_bitcast<_W>(a_);
1914 const auto __b = __vector_bitcast<_W>(b_);
1915 using _Up = __vector_type_t<_W, sizeof(_R) / sizeof(_W)>;
1916#else
1917 constexpr int input_width = _TVT::_S_full_size;
1918 const _Tp& __a = a_;
1919 const _Tp& __b = b_;
1920 using _Up = _R;
1921#endif
1922 if constexpr (input_width == 2)
1923 return reinterpret_cast<_R>(_Up{__a[0], __a[1], __b[0], __b[1]});
1924 else if constexpr (input_width == 4)
1925 return reinterpret_cast<_R>(
1926 _Up{__a[0], __a[1], __a[2], __a[3], __b[0], __b[1], __b[2], __b[3]});
1927 else if constexpr (input_width == 8)
1928 return reinterpret_cast<_R>(
1929 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7],
1930 __b[0], __b[1], __b[2], __b[3], __b[4], __b[5], __b[6], __b[7]});
1931 else if constexpr (input_width == 16)
1932 return reinterpret_cast<_R>(
1933 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
1934 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
1935 __a[14], __a[15], __b[0], __b[1], __b[2], __b[3], __b[4],
1936 __b[5], __b[6], __b[7], __b[8], __b[9], __b[10], __b[11],
1937 __b[12], __b[13], __b[14], __b[15]});
1938 else if constexpr (input_width == 32)
1939 return reinterpret_cast<_R>(
1940 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
1941 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
1942 __a[14], __a[15], __a[16], __a[17], __a[18], __a[19], __a[20],
1943 __a[21], __a[22], __a[23], __a[24], __a[25], __a[26], __a[27],
1944 __a[28], __a[29], __a[30], __a[31], __b[0], __b[1], __b[2],
1945 __b[3], __b[4], __b[5], __b[6], __b[7], __b[8], __b[9],
1946 __b[10], __b[11], __b[12], __b[13], __b[14], __b[15], __b[16],
1947 __b[17], __b[18], __b[19], __b[20], __b[21], __b[22], __b[23],
1948 __b[24], __b[25], __b[26], __b[27], __b[28], __b[29], __b[30],
1949 __b[31]});
1950 }
1951
1952// }}}
1953// __zero_extend {{{
1954template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1955 struct _ZeroExtendProxy
1956 {
1957 using value_type = typename _TVT::value_type;
1958 static constexpr size_t _Np = _TVT::_S_full_size;
1959 const _Tp __x;
1960
1961 template <typename _To, typename _ToVT = _VectorTraits<_To>,
1962 typename
1963 = enable_if_t<is_same_v<typename _ToVT::value_type, value_type>>>
1964 _GLIBCXX_SIMD_INTRINSIC operator _To() const
1965 {
1966 constexpr size_t _ToN = _ToVT::_S_full_size;
1967 if constexpr (_ToN == _Np)
1968 return __x;
1969 else if constexpr (_ToN == 2 * _Np)
1970 {
1971#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
1972 if constexpr (__have_avx && _TVT::template _S_is<float, 4>)
1973 return __vector_bitcast<value_type>(
1974 _mm256_insertf128_ps(__m256(), __x, 0));
1975 else if constexpr (__have_avx && _TVT::template _S_is<double, 2>)
1976 return __vector_bitcast<value_type>(
1977 _mm256_insertf128_pd(__m256d(), __x, 0));
1978 else if constexpr (__have_avx2 && _Np * sizeof(value_type) == 16)
1979 return __vector_bitcast<value_type>(
1980 _mm256_insertf128_si256(__m256i(), __to_intrin(__x), 0));
1981 else if constexpr (__have_avx512f && _TVT::template _S_is<float, 8>)
1982 {
1983 if constexpr (__have_avx512dq)
1984 return __vector_bitcast<value_type>(
1985 _mm512_insertf32x8(__m512(), __x, 0));
1986 else
1987 return reinterpret_cast<__m512>(
1988 _mm512_insertf64x4(__m512d(),
1989 reinterpret_cast<__m256d>(__x), 0));
1990 }
1991 else if constexpr (__have_avx512f
1992 && _TVT::template _S_is<double, 4>)
1993 return __vector_bitcast<value_type>(
1994 _mm512_insertf64x4(__m512d(), __x, 0));
1995 else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 32)
1996 return __vector_bitcast<value_type>(
1997 _mm512_inserti64x4(__m512i(), __to_intrin(__x), 0));
1998#endif
1999 return __concat(__x, _Tp());
2000 }
2001 else if constexpr (_ToN == 4 * _Np)
2002 {
2003#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2004 if constexpr (__have_avx512dq && _TVT::template _S_is<double, 2>)
2005 {
2006 return __vector_bitcast<value_type>(
2007 _mm512_insertf64x2(__m512d(), __x, 0));
2008 }
2009 else if constexpr (__have_avx512f
2010 && is_floating_point_v<value_type>)
2011 {
2012 return __vector_bitcast<value_type>(
2013 _mm512_insertf32x4(__m512(), reinterpret_cast<__m128>(__x),
2014 0));
2015 }
2016 else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 16)
2017 {
2018 return __vector_bitcast<value_type>(
2019 _mm512_inserti32x4(__m512i(), __to_intrin(__x), 0));
2020 }
2021#endif
2022 return __concat(__concat(__x, _Tp()),
2023 __vector_type_t<value_type, _Np * 2>());
2024 }
2025 else if constexpr (_ToN == 8 * _Np)
2026 return __concat(operator __vector_type_t<value_type, _Np * 4>(),
2027 __vector_type_t<value_type, _Np * 4>());
2028 else if constexpr (_ToN == 16 * _Np)
2029 return __concat(operator __vector_type_t<value_type, _Np * 8>(),
2030 __vector_type_t<value_type, _Np * 8>());
2031 else
2032 __assert_unreachable<_Tp>();
2033 }
2034 };
2035
2036template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2037 _GLIBCXX_SIMD_INTRINSIC _ZeroExtendProxy<_Tp, _TVT>
2038 __zero_extend(_Tp __x)
2039 { return {__x}; }
2040
2041// }}}
2042// __extract<_Np, By>{{{
2043template <int _Offset,
2044 int _SplitBy,
2045 typename _Tp,
2046 typename _TVT = _VectorTraits<_Tp>,
2047 typename _R = __vector_type_t<typename _TVT::value_type, _TVT::_S_full_size / _SplitBy>>
2048 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2049 __extract(_Tp __in)
2050 {
2051 using value_type = typename _TVT::value_type;
2052#if _GLIBCXX_SIMD_X86INTRIN // {{{
2053 if constexpr (sizeof(_Tp) == 64 && _SplitBy == 4 && _Offset > 0)
2054 {
2055 if constexpr (__have_avx512dq && is_same_v<double, value_type>)
2056 return _mm512_extractf64x2_pd(__to_intrin(__in), _Offset);
2057 else if constexpr (is_floating_point_v<value_type>)
2058 return __vector_bitcast<value_type>(
2059 _mm512_extractf32x4_ps(__intrin_bitcast<__m512>(__in), _Offset));
2060 else
2061 return reinterpret_cast<_R>(
2062 _mm512_extracti32x4_epi32(__intrin_bitcast<__m512i>(__in),
2063 _Offset));
2064 }
2065 else
2066#endif // _GLIBCXX_SIMD_X86INTRIN }}}
2067 {
2068#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2069 using _W = conditional_t<
2070 is_floating_point_v<value_type>, double,
2071 conditional_t<(sizeof(_R) >= 16), long long, value_type>>;
2072 static_assert(sizeof(_R) % sizeof(_W) == 0);
2073 constexpr int __return_width = sizeof(_R) / sizeof(_W);
2074 using _Up = __vector_type_t<_W, __return_width>;
2075 const auto __x = __vector_bitcast<_W>(__in);
2076#else
2077 constexpr int __return_width = _TVT::_S_full_size / _SplitBy;
2078 using _Up = _R;
2079 const __vector_type_t<value_type, _TVT::_S_full_size>& __x
2080 = __in; // only needed for _Tp = _SimdWrapper<value_type, _Np>
2081#endif
2082 constexpr int _O = _Offset * __return_width;
2083 return __call_with_subscripts<__return_width, _O>(
2084 __x, [](auto... __entries) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2085 return reinterpret_cast<_R>(_Up{__entries...});
2086 });
2087 }
2088 }
2089
2090// }}}
2091// __lo/__hi64[z]{{{
2092template <typename _Tp,
2093 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2094 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2095 __lo64(_Tp __x)
2096 {
2097 _R __r{};
2098 __builtin_memcpy(&__r, &__x, 8);
2099 return __r;
2100 }
2101
2102template <typename _Tp,
2103 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2104 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2105 __hi64(_Tp __x)
2106 {
2107 static_assert(sizeof(_Tp) == 16, "use __hi64z if you meant it");
2108 _R __r{};
2109 __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
2110 return __r;
2111 }
2112
2113template <typename _Tp,
2114 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2115 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2116 __hi64z([[maybe_unused]] _Tp __x)
2117 {
2118 _R __r{};
2119 if constexpr (sizeof(_Tp) == 16)
2120 __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
2121 return __r;
2122 }
2123
2124// }}}
2125// __lo/__hi128{{{
2126template <typename _Tp>
2127 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2128 __lo128(_Tp __x)
2129 { return __extract<0, sizeof(_Tp) / 16>(__x); }
2130
2131template <typename _Tp>
2132 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2133 __hi128(_Tp __x)
2134 {
2135 static_assert(sizeof(__x) == 32);
2136 return __extract<1, 2>(__x);
2137 }
2138
2139// }}}
2140// __lo/__hi256{{{
2141template <typename _Tp>
2142 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2143 __lo256(_Tp __x)
2144 {
2145 static_assert(sizeof(__x) == 64);
2146 return __extract<0, 2>(__x);
2147 }
2148
2149template <typename _Tp>
2150 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2151 __hi256(_Tp __x)
2152 {
2153 static_assert(sizeof(__x) == 64);
2154 return __extract<1, 2>(__x);
2155 }
2156
2157// }}}
2158// __auto_bitcast{{{
2159template <typename _Tp>
2160 struct _AutoCast
2161 {
2162 static_assert(__is_vector_type_v<_Tp>);
2163
2164 const _Tp __x;
2165
2166 template <typename _Up, typename _UVT = _VectorTraits<_Up>>
2167 _GLIBCXX_SIMD_INTRINSIC constexpr operator _Up() const
2168 { return __intrin_bitcast<typename _UVT::type>(__x); }
2169 };
2170
2171template <typename _Tp>
2172 _GLIBCXX_SIMD_INTRINSIC constexpr _AutoCast<_Tp>
2173 __auto_bitcast(const _Tp& __x)
2174 { return {__x}; }
2175
2176template <typename _Tp, size_t _Np>
2177 _GLIBCXX_SIMD_INTRINSIC constexpr
2178 _AutoCast<typename _SimdWrapper<_Tp, _Np>::_BuiltinType>
2179 __auto_bitcast(const _SimdWrapper<_Tp, _Np>& __x)
2180 { return {__x._M_data}; }
2181
2182// }}}
2183// ^^^ ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- ^^^
2184
2185#if _GLIBCXX_SIMD_HAVE_SSE_ABI
2186// __bool_storage_member_type{{{
2187#if _GLIBCXX_SIMD_HAVE_AVX512F && _GLIBCXX_SIMD_X86INTRIN
2188template <size_t _Size>
2189 struct __bool_storage_member_type
2190 {
2191 static_assert((_Size & (_Size - 1)) != 0,
2192 "This trait may only be used for non-power-of-2 sizes. "
2193 "Power-of-2 sizes must be specialized.");
2194 using type =
2195 typename __bool_storage_member_type<std::__bit_ceil(_Size)>::type;
2196 };
2197
2198template <>
2199 struct __bool_storage_member_type<1> { using type = bool; };
2200
2201template <>
2202 struct __bool_storage_member_type<2> { using type = __mmask8; };
2203
2204template <>
2205 struct __bool_storage_member_type<4> { using type = __mmask8; };
2206
2207template <>
2208 struct __bool_storage_member_type<8> { using type = __mmask8; };
2209
2210template <>
2211 struct __bool_storage_member_type<16> { using type = __mmask16; };
2212
2213template <>
2214 struct __bool_storage_member_type<32> { using type = __mmask32; };
2215
2216template <>
2217 struct __bool_storage_member_type<64> { using type = __mmask64; };
2218#endif // _GLIBCXX_SIMD_HAVE_AVX512F
2219
2220// }}}
2221// __intrinsic_type (x86){{{
2222// the following excludes bool via __is_vectorizable
2223#if _GLIBCXX_SIMD_HAVE_SSE
2224template <typename _Tp, size_t _Bytes>
2225 struct __intrinsic_type<_Tp, _Bytes, enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 64>>
2226 {
2227 static_assert(!is_same_v<_Tp, long double>,
2228 "no __intrinsic_type support for long double on x86");
2229
2230 static constexpr size_t _S_VBytes = _Bytes <= 16 ? 16 : _Bytes <= 32 ? 32 : 64;
2231
2232 using type [[__gnu__::__vector_size__(_S_VBytes)]]
2233 = conditional_t<is_integral_v<_Tp>, long long int, _Tp>;
2234 };
2235#endif // _GLIBCXX_SIMD_HAVE_SSE
2236
2237// }}}
2238#endif // _GLIBCXX_SIMD_HAVE_SSE_ABI
2239// __intrinsic_type (ARM){{{
2240#if _GLIBCXX_SIMD_HAVE_NEON
2241template <>
2242 struct __intrinsic_type<float, 8, void>
2243 { using type = float32x2_t; };
2244
2245template <>
2246 struct __intrinsic_type<float, 16, void>
2247 { using type = float32x4_t; };
2248
2249template <>
2250 struct __intrinsic_type<double, 8, void>
2251 {
2252#if _GLIBCXX_SIMD_HAVE_NEON_A64
2253 using type = float64x1_t;
2254#endif
2255 };
2256
2257template <>
2258 struct __intrinsic_type<double, 16, void>
2259 {
2260#if _GLIBCXX_SIMD_HAVE_NEON_A64
2261 using type = float64x2_t;
2262#endif
2263 };
2264
2265#define _GLIBCXX_SIMD_ARM_INTRIN(_Bits, _Np) \
2266template <> \
2267 struct __intrinsic_type<__int_with_sizeof_t<_Bits / 8>, \
2268 _Np * _Bits / 8, void> \
2269 { using type = int##_Bits##x##_Np##_t; }; \
2270template <> \
2271 struct __intrinsic_type<make_unsigned_t<__int_with_sizeof_t<_Bits / 8>>, \
2272 _Np * _Bits / 8, void> \
2273 { using type = uint##_Bits##x##_Np##_t; }
2274_GLIBCXX_SIMD_ARM_INTRIN(8, 8);
2275_GLIBCXX_SIMD_ARM_INTRIN(8, 16);
2276_GLIBCXX_SIMD_ARM_INTRIN(16, 4);
2277_GLIBCXX_SIMD_ARM_INTRIN(16, 8);
2278_GLIBCXX_SIMD_ARM_INTRIN(32, 2);
2279_GLIBCXX_SIMD_ARM_INTRIN(32, 4);
2280_GLIBCXX_SIMD_ARM_INTRIN(64, 1);
2281_GLIBCXX_SIMD_ARM_INTRIN(64, 2);
2282#undef _GLIBCXX_SIMD_ARM_INTRIN
2283
2284template <typename _Tp, size_t _Bytes>
2285 struct __intrinsic_type<_Tp, _Bytes, enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2286 {
2287 static constexpr int _SVecBytes = _Bytes <= 8 ? 8 : 16;
2288
2289 using _Ip = __int_for_sizeof_t<_Tp>;
2290
2291 using _Up = conditional_t<
2292 is_floating_point_v<_Tp>, _Tp,
2293 conditional_t<is_unsigned_v<_Tp>, make_unsigned_t<_Ip>, _Ip>>;
2294
2295 static_assert(!is_same_v<_Tp, _Up> || _SVecBytes != _Bytes,
2296 "should use explicit specialization above");
2297
2298 using type = typename __intrinsic_type<_Up, _SVecBytes>::type;
2299 };
2300#endif // _GLIBCXX_SIMD_HAVE_NEON
2301
2302// }}}
2303// __intrinsic_type (PPC){{{
2304#ifdef __ALTIVEC__
2305template <typename _Tp>
2306 struct __intrinsic_type_impl;
2307
2308#define _GLIBCXX_SIMD_PPC_INTRIN(_Tp) \
2309 template <> \
2310 struct __intrinsic_type_impl<_Tp> { using type = __vector _Tp; }
2311_GLIBCXX_SIMD_PPC_INTRIN(float);
2312#ifdef __VSX__
2313_GLIBCXX_SIMD_PPC_INTRIN(double);
2314#endif
2315_GLIBCXX_SIMD_PPC_INTRIN(signed char);
2316_GLIBCXX_SIMD_PPC_INTRIN(unsigned char);
2317_GLIBCXX_SIMD_PPC_INTRIN(signed short);
2318_GLIBCXX_SIMD_PPC_INTRIN(unsigned short);
2319_GLIBCXX_SIMD_PPC_INTRIN(signed int);
2320_GLIBCXX_SIMD_PPC_INTRIN(unsigned int);
2321#if defined __VSX__ || __SIZEOF_LONG__ == 4
2322_GLIBCXX_SIMD_PPC_INTRIN(signed long);
2323_GLIBCXX_SIMD_PPC_INTRIN(unsigned long);
2324#endif
2325#ifdef __VSX__
2326_GLIBCXX_SIMD_PPC_INTRIN(signed long long);
2327_GLIBCXX_SIMD_PPC_INTRIN(unsigned long long);
2328#endif
2329#undef _GLIBCXX_SIMD_PPC_INTRIN
2330
2331template <typename _Tp, size_t _Bytes>
2332 struct __intrinsic_type<_Tp, _Bytes, enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2333 {
2334 static constexpr bool _S_is_ldouble = is_same_v<_Tp, long double>;
2335
2336 // allow _Tp == long double with -mlong-double-64
2337 static_assert(!(_S_is_ldouble && sizeof(long double) > sizeof(double)),
2338 "no __intrinsic_type support for 128-bit floating point on PowerPC");
2339
2340#ifndef __VSX__
2341 static_assert(!(is_same_v<_Tp, double>
2342 || (_S_is_ldouble && sizeof(long double) == sizeof(double))),
2343 "no __intrinsic_type support for 64-bit floating point on PowerPC w/o VSX");
2344#endif
2345
2346 static constexpr auto __element_type()
2347 {
2348 if constexpr (is_floating_point_v<_Tp>)
2349 {
2350 if constexpr (_S_is_ldouble)
2351 return double {};
2352 else
2353 return _Tp {};
2354 }
2355 else if constexpr (is_signed_v<_Tp>)
2356 {
2357 if constexpr (sizeof(_Tp) == sizeof(_SChar))
2358 return _SChar {};
2359 else if constexpr (sizeof(_Tp) == sizeof(short))
2360 return short {};
2361 else if constexpr (sizeof(_Tp) == sizeof(int))
2362 return int {};
2363 else if constexpr (sizeof(_Tp) == sizeof(_LLong))
2364 return _LLong {};
2365 }
2366 else
2367 {
2368 if constexpr (sizeof(_Tp) == sizeof(_UChar))
2369 return _UChar {};
2370 else if constexpr (sizeof(_Tp) == sizeof(_UShort))
2371 return _UShort {};
2372 else if constexpr (sizeof(_Tp) == sizeof(_UInt))
2373 return _UInt {};
2374 else if constexpr (sizeof(_Tp) == sizeof(_ULLong))
2375 return _ULLong {};
2376 }
2377 }
2378
2379 using type = typename __intrinsic_type_impl<decltype(__element_type())>::type;
2380 };
2381#endif // __ALTIVEC__
2382
2383// }}}
2384// _SimdWrapper<bool>{{{1
2385template <size_t _Width>
2386 struct _SimdWrapper<bool, _Width,
2387 void_t<typename __bool_storage_member_type<_Width>::type>>
2388 {
2389 using _BuiltinType = typename __bool_storage_member_type<_Width>::type;
2390 using value_type = bool;
2391
2392 static constexpr size_t _S_full_size = sizeof(_BuiltinType) * __CHAR_BIT__;
2393
2394 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<bool, _S_full_size>
2395 __as_full_vector() const
2396 { return _M_data; }
2397
2398 _GLIBCXX_SIMD_INTRINSIC constexpr
2399 _SimdWrapper() = default;
2400
2401 _GLIBCXX_SIMD_INTRINSIC constexpr
2402 _SimdWrapper(_BuiltinType __k) : _M_data(__k) {};
2403
2404 _GLIBCXX_SIMD_INTRINSIC
2405 operator const _BuiltinType&() const
2406 { return _M_data; }
2407
2408 _GLIBCXX_SIMD_INTRINSIC
2409 operator _BuiltinType&()
2410 { return _M_data; }
2411
2412 _GLIBCXX_SIMD_INTRINSIC _BuiltinType
2413 __intrin() const
2414 { return _M_data; }
2415
2416 _GLIBCXX_SIMD_INTRINSIC constexpr value_type
2417 operator[](size_t __i) const
2418 { return _M_data & (_BuiltinType(1) << __i); }
2419
2420 template <size_t __i>
2421 _GLIBCXX_SIMD_INTRINSIC constexpr value_type
2422 operator[](_SizeConstant<__i>) const
2423 { return _M_data & (_BuiltinType(1) << __i); }
2424
2425 _GLIBCXX_SIMD_INTRINSIC constexpr void
2426 _M_set(size_t __i, value_type __x)
2427 {
2428 if (__x)
2429 _M_data |= (_BuiltinType(1) << __i);
2430 else
2431 _M_data &= ~(_BuiltinType(1) << __i);
2432 }
2433
2434 _GLIBCXX_SIMD_INTRINSIC constexpr bool
2435 _M_is_constprop() const
2436 { return __builtin_constant_p(_M_data); }
2437
2438 _GLIBCXX_SIMD_INTRINSIC constexpr bool
2439 _M_is_constprop_none_of() const
2440 {
2441 if (__builtin_constant_p(_M_data))
2442 {
2443 constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__;
2444 constexpr _BuiltinType __active_mask
2445 = ~_BuiltinType() >> (__nbits - _Width);
2446 return (_M_data & __active_mask) == 0;
2447 }
2448 return false;
2449 }
2450
2451 _GLIBCXX_SIMD_INTRINSIC constexpr bool
2452 _M_is_constprop_all_of() const
2453 {
2454 if (__builtin_constant_p(_M_data))
2455 {
2456 constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__;
2457 constexpr _BuiltinType __active_mask
2458 = ~_BuiltinType() >> (__nbits - _Width);
2459 return (_M_data & __active_mask) == __active_mask;
2460 }
2461 return false;
2462 }
2463
2464 _BuiltinType _M_data;
2465 };
2466
2467// _SimdWrapperBase{{{1
2468template <bool _MustZeroInitPadding, typename _BuiltinType>
2469 struct _SimdWrapperBase;
2470
2471template <typename _BuiltinType>
2472 struct _SimdWrapperBase<false, _BuiltinType> // no padding or no SNaNs
2473 {
2474 _GLIBCXX_SIMD_INTRINSIC constexpr
2475 _SimdWrapperBase() = default;
2476
2477 _GLIBCXX_SIMD_INTRINSIC constexpr
2478 _SimdWrapperBase(_BuiltinType __init) : _M_data(__init) {}
2479
2480 _BuiltinType _M_data;
2481 };
2482
2483template <typename _BuiltinType>
2484 struct _SimdWrapperBase<true, _BuiltinType> // with padding that needs to
2485 // never become SNaN
2486 {
2487 _GLIBCXX_SIMD_INTRINSIC constexpr
2488 _SimdWrapperBase() : _M_data() {}
2489
2490 _GLIBCXX_SIMD_INTRINSIC constexpr
2491 _SimdWrapperBase(_BuiltinType __init) : _M_data(__init) {}
2492
2493 _BuiltinType _M_data;
2494 };
2495
2496// }}}
2497// _SimdWrapper{{{
2498template <typename _Tp, size_t _Width>
2499 struct _SimdWrapper<
2500 _Tp, _Width,
2501 void_t<__vector_type_t<_Tp, _Width>, __intrinsic_type_t<_Tp, _Width>>>
2502 : _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2503 && sizeof(_Tp) * _Width
2504 == sizeof(__vector_type_t<_Tp, _Width>),
2505 __vector_type_t<_Tp, _Width>>
2506 {
2507 using _Base
2508 = _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2509 && sizeof(_Tp) * _Width
2510 == sizeof(__vector_type_t<_Tp, _Width>),
2511 __vector_type_t<_Tp, _Width>>;
2512
2513 static_assert(__is_vectorizable_v<_Tp>);
2514 static_assert(_Width >= 2); // 1 doesn't make sense, use _Tp directly then
2515
2516 using _BuiltinType = __vector_type_t<_Tp, _Width>;
2517 using value_type = _Tp;
2518
2519 static inline constexpr size_t _S_full_size
2520 = sizeof(_BuiltinType) / sizeof(value_type);
2521 static inline constexpr int _S_size = _Width;
2522 static inline constexpr bool _S_is_partial = _S_full_size != _S_size;
2523
2524 using _Base::_M_data;
2525
2526 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _S_full_size>
2527 __as_full_vector() const
2528 { return _M_data; }
2529
2530 _GLIBCXX_SIMD_INTRINSIC constexpr
2531 _SimdWrapper(initializer_list<_Tp> __init)
2532 : _Base(__generate_from_n_evaluations<_Width, _BuiltinType>(
2533 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2534 return __init.begin()[__i.value];
2535 })) {}
2536
2537 _GLIBCXX_SIMD_INTRINSIC constexpr
2538 _SimdWrapper() = default;
2539
2540 _GLIBCXX_SIMD_INTRINSIC constexpr
2541 _SimdWrapper(const _SimdWrapper&) = default;
2542
2543 _GLIBCXX_SIMD_INTRINSIC constexpr
2544 _SimdWrapper(_SimdWrapper&&) = default;
2545
2546 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2547 operator=(const _SimdWrapper&) = default;
2548
2549 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2550 operator=(_SimdWrapper&&) = default;
2551
2552 template <typename _V, typename = enable_if_t<disjunction_v<
2553 is_same<_V, __vector_type_t<_Tp, _Width>>,
2554 is_same<_V, __intrinsic_type_t<_Tp, _Width>>>>>
2555 _GLIBCXX_SIMD_INTRINSIC constexpr
2556 _SimdWrapper(_V __x)
2557 // __vector_bitcast can convert e.g. __m128 to __vector(2) float
2558 : _Base(__vector_bitcast<_Tp, _Width>(__x)) {}
2559
2560 template <typename... _As,
2561 typename = enable_if_t<((is_same_v<simd_abi::scalar, _As> && ...)
2562 && sizeof...(_As) <= _Width)>>
2563 _GLIBCXX_SIMD_INTRINSIC constexpr
2564 operator _SimdTuple<_Tp, _As...>() const
2565 {
2566 return __generate_from_n_evaluations<sizeof...(_As), _SimdTuple<_Tp, _As...>>(
2567 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
2568 { return _M_data[int(__i)]; });
2569 }
2570
2571 _GLIBCXX_SIMD_INTRINSIC constexpr
2572 operator const _BuiltinType&() const
2573 { return _M_data; }
2574
2575 _GLIBCXX_SIMD_INTRINSIC constexpr
2576 operator _BuiltinType&()
2577 { return _M_data; }
2578
2579 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
2580 operator[](size_t __i) const
2581 { return _M_data[__i]; }
2582
2583 template <size_t __i>
2584 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
2585 operator[](_SizeConstant<__i>) const
2586 { return _M_data[__i]; }
2587
2588 _GLIBCXX_SIMD_INTRINSIC constexpr void
2589 _M_set(size_t __i, _Tp __x)
2590 {
2591 if (__builtin_is_constant_evaluated())
2592 _M_data = __generate_from_n_evaluations<_Width, _BuiltinType>([&](auto __j) {
2593 return __j == __i ? __x : _M_data[__j()];
2594 });
2595 else
2596 _M_data[__i] = __x;
2597 }
2598
2599 _GLIBCXX_SIMD_INTRINSIC
2600 constexpr bool
2601 _M_is_constprop() const
2602 { return __builtin_constant_p(_M_data); }
2603
2604 _GLIBCXX_SIMD_INTRINSIC constexpr bool
2605 _M_is_constprop_none_of() const
2606 {
2607 if (__builtin_constant_p(_M_data))
2608 {
2609 bool __r = true;
2610 if constexpr (is_floating_point_v<_Tp>)
2611 {
2612 using _Ip = __int_for_sizeof_t<_Tp>;
2613 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2614 __execute_n_times<_Width>(
2615 [&](auto __i) { __r &= __intdata[__i.value] == _Ip(); });
2616 }
2617 else
2618 __execute_n_times<_Width>(
2619 [&](auto __i) { __r &= _M_data[__i.value] == _Tp(); });
2620 if (__builtin_constant_p(__r))
2621 return __r;
2622 }
2623 return false;
2624 }
2625
2626 _GLIBCXX_SIMD_INTRINSIC constexpr bool
2627 _M_is_constprop_all_of() const
2628 {
2629 if (__builtin_constant_p(_M_data))
2630 {
2631 bool __r = true;
2632 if constexpr (is_floating_point_v<_Tp>)
2633 {
2634 using _Ip = __int_for_sizeof_t<_Tp>;
2635 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2636 __execute_n_times<_Width>(
2637 [&](auto __i) { __r &= __intdata[__i.value] == ~_Ip(); });
2638 }
2639 else
2640 __execute_n_times<_Width>(
2641 [&](auto __i) { __r &= _M_data[__i.value] == ~_Tp(); });
2642 if (__builtin_constant_p(__r))
2643 return __r;
2644 }
2645 return false;
2646 }
2647 };
2648
2649// }}}
2650
2651// __vectorized_sizeof {{{
2652template <typename _Tp>
2653 constexpr size_t
2654 __vectorized_sizeof()
2655 {
2656 if constexpr (!__is_vectorizable_v<_Tp>)
2657 return 0;
2658
2659 if constexpr (sizeof(_Tp) <= 8)
2660 {
2661 // X86:
2662 if constexpr (__have_avx512bw)
2663 return 64;
2664 if constexpr (__have_avx512f && sizeof(_Tp) >= 4)
2665 return 64;
2666 if constexpr (__have_avx2)
2667 return 32;
2668 if constexpr (__have_avx && is_floating_point_v<_Tp>)
2669 return 32;
2670 if constexpr (__have_sse2)
2671 return 16;
2672 if constexpr (__have_sse && is_same_v<_Tp, float>)
2673 return 16;
2674 /* The following is too much trouble because of mixed MMX and x87 code.
2675 * While nothing here explicitly calls MMX instructions of registers,
2676 * they are still emitted but no EMMS cleanup is done.
2677 if constexpr (__have_mmx && sizeof(_Tp) <= 4 && is_integral_v<_Tp>)
2678 return 8;
2679 */
2680
2681 // PowerPC:
2682 if constexpr (__have_power8vec
2683 || (__have_power_vmx && (sizeof(_Tp) < 8))
2684 || (__have_power_vsx && is_floating_point_v<_Tp>) )
2685 return 16;
2686
2687 // ARM:
2688 if constexpr (__have_neon_a64
2689 || (__have_neon_a32 && !is_same_v<_Tp, double>) )
2690 return 16;
2691 if constexpr (__have_neon
2692 && sizeof(_Tp) < 8
2693 // Only allow fp if the user allows non-ICE559 fp (e.g.
2694 // via -ffast-math). ARMv7 NEON fp is not conforming to
2695 // IEC559.
2696 && (__support_neon_float || !is_floating_point_v<_Tp>))
2697 return 16;
2698 }
2699
2700 return sizeof(_Tp);
2701 }
2702
2703// }}}
2704namespace simd_abi {
2705// most of simd_abi is defined in simd_detail.h
2706template <typename _Tp>
2707 inline constexpr int max_fixed_size
2708 = (__have_avx512bw && sizeof(_Tp) == 1) ? 64 : 32;
2709
2710// compatible {{{
2711#if defined __x86_64__ || defined __aarch64__
2712template <typename _Tp>
2713 using compatible = conditional_t<(sizeof(_Tp) <= 8), _VecBuiltin<16>, scalar>;
2714#elif defined __ARM_NEON
2715// FIXME: not sure, probably needs to be scalar (or dependent on the hard-float
2716// ABI?)
2717template <typename _Tp>
2718 using compatible
2719 = conditional_t<(sizeof(_Tp) < 8
2720 && (__support_neon_float || !is_floating_point_v<_Tp>)),
2721 _VecBuiltin<16>, scalar>;
2722#else
2723template <typename>
2724 using compatible = scalar;
2725#endif
2726
2727// }}}
2728// native {{{
2729template <typename _Tp>
2730 constexpr auto
2731 __determine_native_abi()
2732 {
2733 constexpr size_t __bytes = __vectorized_sizeof<_Tp>();
2734 if constexpr (__bytes == sizeof(_Tp))
2735 return static_cast<scalar*>(nullptr);
2736 else if constexpr (__have_avx512vl || (__have_avx512f && __bytes == 64))
2737 return static_cast<_VecBltnBtmsk<__bytes>*>(nullptr);
2738 else
2739 return static_cast<_VecBuiltin<__bytes>*>(nullptr);
2740 }
2741
2742template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
2743 using native = remove_pointer_t<decltype(__determine_native_abi<_Tp>())>;
2744
2745// }}}
2746// __default_abi {{{
2747#if defined _GLIBCXX_SIMD_DEFAULT_ABI
2748template <typename _Tp>
2749 using __default_abi = _GLIBCXX_SIMD_DEFAULT_ABI<_Tp>;
2750#else
2751template <typename _Tp>
2752 using __default_abi = compatible<_Tp>;
2753#endif
2754
2755// }}}
2756} // namespace simd_abi
2757
2758// traits {{{1
2759template <typename _Tp>
2760 struct is_simd_flag_type
2761 : false_type
2762 {};
2763
2764template <>
2765 struct is_simd_flag_type<element_aligned_tag>
2766 : true_type
2767 {};
2768
2769template <>
2770 struct is_simd_flag_type<vector_aligned_tag>
2771 : true_type
2772 {};
2773
2774template <size_t _Np>
2775 struct is_simd_flag_type<overaligned_tag<_Np>>
2776 : __bool_constant<(_Np > 0) and __has_single_bit(_Np)>
2777 {};
2778
2779template <typename _Tp>
2780 inline constexpr bool is_simd_flag_type_v = is_simd_flag_type<_Tp>::value;
2781
2782template <typename _Tp, typename = enable_if_t<is_simd_flag_type_v<_Tp>>>
2783 using _IsSimdFlagType = _Tp;
2784
2785// is_abi_tag {{{2
2786template <typename _Tp, typename = void_t<>>
2787 struct is_abi_tag : false_type {};
2788
2789template <typename _Tp>
2790 struct is_abi_tag<_Tp, void_t<typename _Tp::_IsValidAbiTag>>
2791 : public _Tp::_IsValidAbiTag {};
2792
2793template <typename _Tp>
2794 inline constexpr bool is_abi_tag_v = is_abi_tag<_Tp>::value;
2795
2796// is_simd(_mask) {{{2
2797template <typename _Tp>
2798 struct is_simd : public false_type {};
2799
2800template <typename _Tp>
2801 inline constexpr bool is_simd_v = is_simd<_Tp>::value;
2802
2803template <typename _Tp>
2804 struct is_simd_mask : public false_type {};
2805
2806template <typename _Tp>
2807inline constexpr bool is_simd_mask_v = is_simd_mask<_Tp>::value;
2808
2809// simd_size {{{2
2810template <typename _Tp, typename _Abi, typename = void>
2811 struct __simd_size_impl {};
2812
2813template <typename _Tp, typename _Abi>
2814 struct __simd_size_impl<
2815 _Tp, _Abi,
2816 enable_if_t<conjunction_v<__is_vectorizable<_Tp>, is_abi_tag<_Abi>>>>
2817 : _SizeConstant<_Abi::template _S_size<_Tp>> {};
2818
2819template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2820 struct simd_size : __simd_size_impl<_Tp, _Abi> {};
2821
2822template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2823 inline constexpr size_t simd_size_v = simd_size<_Tp, _Abi>::value;
2824
2825// simd_abi::deduce {{{2
2826template <typename _Tp, size_t _Np, typename = void>
2827 struct __deduce_impl;
2828
2829namespace simd_abi {
2830/**
2831 * @tparam _Tp The requested `value_type` for the elements.
2832 * @tparam _Np The requested number of elements.
2833 * @tparam _Abis This parameter is ignored, since this implementation cannot
2834 * make any use of it. Either __a good native ABI is matched and used as `type`
2835 * alias, or the `fixed_size<_Np>` ABI is used, which internally is built from
2836 * the best matching native ABIs.
2837 */
2838template <typename _Tp, size_t _Np, typename...>
2839 struct deduce : __deduce_impl<_Tp, _Np> {};
2840
2841template <typename _Tp, size_t _Np, typename... _Abis>
2842 using deduce_t = typename deduce<_Tp, _Np, _Abis...>::type;
2843} // namespace simd_abi
2844
2845// }}}2
2846// rebind_simd {{{2
2847template <typename _Tp, typename _V, typename = void>
2848 struct rebind_simd;
2849
2850template <typename _Tp, typename _Up, typename _Abi>
2851 struct rebind_simd<_Tp, simd<_Up, _Abi>,
2852 void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2853 { using type = simd<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>; };
2854
2855template <typename _Tp, typename _Up, typename _Abi>
2856 struct rebind_simd<_Tp, simd_mask<_Up, _Abi>,
2857 void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2858 { using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>; };
2859
2860template <typename _Tp, typename _V>
2861 using rebind_simd_t = typename rebind_simd<_Tp, _V>::type;
2862
2863// resize_simd {{{2
2864template <int _Np, typename _V, typename = void>
2865 struct resize_simd;
2866
2867template <int _Np, typename _Tp, typename _Abi>
2868 struct resize_simd<_Np, simd<_Tp, _Abi>, void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2869 { using type = simd<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2870
2871template <int _Np, typename _Tp, typename _Abi>
2872 struct resize_simd<_Np, simd_mask<_Tp, _Abi>, void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2873 { using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2874
2875template <int _Np, typename _V>
2876 using resize_simd_t = typename resize_simd<_Np, _V>::type;
2877
2878// }}}2
2879// memory_alignment {{{2
2880template <typename _Tp, typename _Up = typename _Tp::value_type>
2881 struct memory_alignment
2882 : public _SizeConstant<vector_aligned_tag::_S_alignment<_Tp, _Up>> {};
2883
2884template <typename _Tp, typename _Up = typename _Tp::value_type>
2885 inline constexpr size_t memory_alignment_v = memory_alignment<_Tp, _Up>::value;
2886
2887// class template simd [simd] {{{1
2888template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2889 class simd;
2890
2891template <typename _Tp, typename _Abi>
2892 struct is_simd<simd<_Tp, _Abi>> : public true_type {};
2893
2894template <typename _Tp>
2895 using native_simd = simd<_Tp, simd_abi::native<_Tp>>;
2896
2897template <typename _Tp, int _Np>
2898 using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
2899
2900template <typename _Tp, size_t _Np>
2901 using __deduced_simd = simd<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2902
2903// class template simd_mask [simd_mask] {{{1
2904template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2905 class simd_mask;
2906
2907template <typename _Tp, typename _Abi>
2908 struct is_simd_mask<simd_mask<_Tp, _Abi>> : public true_type {};
2909
2910template <typename _Tp>
2911 using native_simd_mask = simd_mask<_Tp, simd_abi::native<_Tp>>;
2912
2913template <typename _Tp, int _Np>
2914 using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_Np>>;
2915
2916template <typename _Tp, size_t _Np>
2917 using __deduced_simd_mask = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2918
2919// casts [simd.casts] {{{1
2920// static_simd_cast {{{2
2921template <typename _Tp, typename _Up, typename _Ap, bool = is_simd_v<_Tp>, typename = void>
2922 struct __static_simd_cast_return_type;
2923
2924template <typename _Tp, typename _A0, typename _Up, typename _Ap>
2925 struct __static_simd_cast_return_type<simd_mask<_Tp, _A0>, _Up, _Ap, false, void>
2926 : __static_simd_cast_return_type<simd<_Tp, _A0>, _Up, _Ap> {};
2927
2928template <typename _Tp, typename _Up, typename _Ap>
2929 struct __static_simd_cast_return_type<
2930 _Tp, _Up, _Ap, true, enable_if_t<_Tp::size() == simd_size_v<_Up, _Ap>>>
2931 { using type = _Tp; };
2932
2933template <typename _Tp, typename _Ap>
2934 struct __static_simd_cast_return_type<_Tp, _Tp, _Ap, false,
2935#ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2936 enable_if_t<__is_vectorizable_v<_Tp>>
2937#else
2938 void
2939#endif
2940 >
2941 { using type = simd<_Tp, _Ap>; };
2942
2943template <typename _Tp, typename = void>
2944 struct __safe_make_signed { using type = _Tp;};
2945
2946template <typename _Tp>
2947 struct __safe_make_signed<_Tp, enable_if_t<is_integral_v<_Tp>>>
2948 {
2949 // the extra make_unsigned_t is because of PR85951
2950 using type = make_signed_t<make_unsigned_t<_Tp>>;
2951 };
2952
2953template <typename _Tp>
2954 using safe_make_signed_t = typename __safe_make_signed<_Tp>::type;
2955
2956template <typename _Tp, typename _Up, typename _Ap>
2957 struct __static_simd_cast_return_type<_Tp, _Up, _Ap, false,
2958#ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2959 enable_if_t<__is_vectorizable_v<_Tp>>
2960#else
2961 void
2962#endif
2963 >
2964 {
2965 using type = conditional_t<
2966 (is_integral_v<_Up> && is_integral_v<_Tp> &&
2967#ifndef _GLIBCXX_SIMD_FIX_P2TS_ISSUE65
2968 is_signed_v<_Up> != is_signed_v<_Tp> &&
2969#endif
2970 is_same_v<safe_make_signed_t<_Up>, safe_make_signed_t<_Tp>>),
2971 simd<_Tp, _Ap>, fixed_size_simd<_Tp, simd_size_v<_Up, _Ap>>>;
2972 };
2973
2974template <typename _Tp, typename _Up, typename _Ap,
2975 typename _R
2976 = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
2977 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _R
2978 static_simd_cast(const simd<_Up, _Ap>& __x)
2979 {
2980 if constexpr (is_same<_R, simd<_Up, _Ap>>::value)
2981 return __x;
2982 else
2983 {
2984 _SimdConverter<_Up, _Ap, typename _R::value_type, typename _R::abi_type>
2985 __c;
2986 return _R(__private_init, __c(__data(__x)));
2987 }
2988 }
2989
2990namespace __proposed {
2991template <typename _Tp, typename _Up, typename _Ap,
2992 typename _R
2993 = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
2994 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR typename _R::mask_type
2995 static_simd_cast(const simd_mask<_Up, _Ap>& __x)
2996 {
2997 using _RM = typename _R::mask_type;
2998 return {__private_init, _RM::abi_type::_MaskImpl::template _S_convert<
2999 typename _RM::simd_type::value_type>(__x)};
3000 }
3001} // namespace __proposed
3002
3003// simd_cast {{{2
3004template <typename _Tp, typename _Up, typename _Ap,
3005 typename _To = __value_type_or_identity_t<_Tp>>
3006 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto
3007 simd_cast(const simd<_ValuePreserving<_Up, _To>, _Ap>& __x)
3008 -> decltype(static_simd_cast<_Tp>(__x))
3009 { return static_simd_cast<_Tp>(__x); }
3010
3011namespace __proposed {
3012template <typename _Tp, typename _Up, typename _Ap,
3013 typename _To = __value_type_or_identity_t<_Tp>>
3014 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto
3015 simd_cast(const simd_mask<_ValuePreserving<_Up, _To>, _Ap>& __x)
3016 -> decltype(static_simd_cast<_Tp>(__x))
3017 { return static_simd_cast<_Tp>(__x); }
3018} // namespace __proposed
3019
3020// }}}2
3021// resizing_simd_cast {{{
3022namespace __proposed {
3023/* Proposed spec:
3024
3025template <class T, class U, class Abi>
3026T resizing_simd_cast(const simd<U, Abi>& x)
3027
3028p1 Constraints:
3029 - is_simd_v<T> is true and
3030 - T::value_type is the same type as U
3031
3032p2 Returns:
3033 A simd object with the i^th element initialized to x[i] for all i in the
3034 range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger
3035 than simd_size_v<U, Abi>, the remaining elements are value-initialized.
3036
3037template <class T, class U, class Abi>
3038T resizing_simd_cast(const simd_mask<U, Abi>& x)
3039
3040p1 Constraints: is_simd_mask_v<T> is true
3041
3042p2 Returns:
3043 A simd_mask object with the i^th element initialized to x[i] for all i in
3044the range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger
3045 than simd_size_v<U, Abi>, the remaining elements are initialized to false.
3046
3047 */
3048
3049template <typename _Tp, typename _Up, typename _Ap>
3050 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR enable_if_t<
3051 conjunction_v<is_simd<_Tp>, is_same<typename _Tp::value_type, _Up>>, _Tp>
3052 resizing_simd_cast(const simd<_Up, _Ap>& __x)
3053 {
3054 if constexpr (is_same_v<typename _Tp::abi_type, _Ap>)
3055 return __x;
3056 else if (__builtin_is_constant_evaluated())
3057 return _Tp([&](auto __i) constexpr {
3058 return __i < simd_size_v<_Up, _Ap> ? __x[__i] : _Up();
3059 });
3060 else if constexpr (simd_size_v<_Up, _Ap> == 1)
3061 {
3062 _Tp __r{};
3063 __r[0] = __x[0];
3064 return __r;
3065 }
3066 else if constexpr (_Tp::size() == 1)
3067 return __x[0];
3068 else if constexpr (sizeof(_Tp) == sizeof(__x)
3069 && !__is_fixed_size_abi_v<_Ap>)
3070 return {__private_init,
3071 __vector_bitcast<typename _Tp::value_type, _Tp::size()>(
3072 _Ap::_S_masked(__data(__x))._M_data)};
3073 else
3074 {
3075 _Tp __r{};
3076 __builtin_memcpy(&__data(__r), &__data(__x),
3077 sizeof(_Up)
3078 * std::min(_Tp::size(), simd_size_v<_Up, _Ap>));
3079 return __r;
3080 }
3081 }
3082
3083template <typename _Tp, typename _Up, typename _Ap>
3084 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3085 enable_if_t<is_simd_mask_v<_Tp>, _Tp>
3086 resizing_simd_cast(const simd_mask<_Up, _Ap>& __x)
3087 {
3088 return {__private_init, _Tp::abi_type::_MaskImpl::template _S_convert<
3089 typename _Tp::simd_type::value_type>(__x)};
3090 }
3091} // namespace __proposed
3092
3093// }}}
3094// to_fixed_size {{{2
3095template <typename _Tp, int _Np>
3096 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, _Np>
3097 to_fixed_size(const fixed_size_simd<_Tp, _Np>& __x)
3098 { return __x; }
3099
3100template <typename _Tp, int _Np>
3101 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, _Np>
3102 to_fixed_size(const fixed_size_simd_mask<_Tp, _Np>& __x)
3103 { return __x; }
3104
3105template <typename _Tp, typename _Ap>
3106 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, simd_size_v<_Tp, _Ap>>
3107 to_fixed_size(const simd<_Tp, _Ap>& __x)
3108 {
3109 using _Rp = fixed_size_simd<_Tp, simd_size_v<_Tp, _Ap>>;
3110 return _Rp([&__x](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3111 }
3112
3113template <typename _Tp, typename _Ap>
3114 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, simd_size_v<_Tp, _Ap>>
3115 to_fixed_size(const simd_mask<_Tp, _Ap>& __x)
3116 {
3117 return {__private_init,
3118 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; }};
3119 }
3120
3121// to_native {{{2
3122template <typename _Tp, int _Np>
3123 _GLIBCXX_SIMD_INTRINSIC
3124 enable_if_t<(_Np == native_simd<_Tp>::size()), native_simd<_Tp>>
3125 to_native(const fixed_size_simd<_Tp, _Np>& __x)
3126 {
3127 alignas(memory_alignment_v<native_simd<_Tp>>) _Tp __mem[_Np];
3128 __x.copy_to(__mem, vector_aligned);
3129 return {__mem, vector_aligned};
3130 }
3131
3132template <typename _Tp, int _Np>
3133 _GLIBCXX_SIMD_INTRINSIC
3134 enable_if_t<(_Np == native_simd_mask<_Tp>::size()), native_simd_mask<_Tp>>
3135 to_native(const fixed_size_simd_mask<_Tp, _Np>& __x)
3136 {
3137 return native_simd_mask<_Tp>(
3138 __private_init,
3139 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3140 }
3141
3142// to_compatible {{{2
3143template <typename _Tp, int _Np>
3144 _GLIBCXX_SIMD_INTRINSIC enable_if_t<(_Np == simd<_Tp>::size()), simd<_Tp>>
3145 to_compatible(const simd<_Tp, simd_abi::fixed_size<_Np>>& __x)
3146 {
3147 alignas(memory_alignment_v<simd<_Tp>>) _Tp __mem[_Np];
3148 __x.copy_to(__mem, vector_aligned);
3149 return {__mem, vector_aligned};
3150 }
3151
3152template <typename _Tp, int _Np>
3153 _GLIBCXX_SIMD_INTRINSIC
3154 enable_if_t<(_Np == simd_mask<_Tp>::size()), simd_mask<_Tp>>
3155 to_compatible(const simd_mask<_Tp, simd_abi::fixed_size<_Np>>& __x)
3156 {
3157 return simd_mask<_Tp>(
3158 __private_init,
3159 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3160 }
3161
3162// masked assignment [simd_mask.where] {{{1
3163
3164// where_expression {{{1
3165// const_where_expression<M, T> {{{2
3166template <typename _M, typename _Tp>
3167 class const_where_expression
3168 {
3169 using _V = _Tp;
3170 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3171
3172 struct _Wrapper { using value_type = _V; };
3173
3174 protected:
3175 using _Impl = typename _V::_Impl;
3176
3177 using value_type =
3178 typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3179
3180 _GLIBCXX_SIMD_INTRINSIC friend const _M&
3181 __get_mask(const const_where_expression& __x)
3182 { return __x._M_k; }
3183
3184 _GLIBCXX_SIMD_INTRINSIC friend const _Tp&
3185 __get_lvalue(const const_where_expression& __x)
3186 { return __x._M_value; }
3187
3188 const _M& _M_k;
3189 _Tp& _M_value;
3190
3191 public:
3192 const_where_expression(const const_where_expression&) = delete;
3193
3194 const_where_expression& operator=(const const_where_expression&) = delete;
3195
3196 _GLIBCXX_SIMD_INTRINSIC constexpr
3197 const_where_expression(const _M& __kk, const _Tp& dd)
3198 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3199
3200 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3201 operator-() const&&
3202 {
3203 return {__private_init,
3204 _Impl::template _S_masked_unary<negate>(__data(_M_k),
3205 __data(_M_value))};
3206 }
3207
3208 template <typename _Up, typename _Flags>
3209 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3210 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&&
3211 {
3212 return {__private_init,
3213 _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3214 _Flags::template _S_apply<_V>(__mem))};
3215 }
3216
3217 template <typename _Up, typename _Flags>
3218 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3219 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&&
3220 {
3221 _Impl::_S_masked_store(__data(_M_value),
3222 _Flags::template _S_apply<_V>(__mem),
3223 __data(_M_k));
3224 }
3225 };
3226
3227// const_where_expression<bool, T> {{{2
3228template <typename _Tp>
3229 class const_where_expression<bool, _Tp>
3230 {
3231 using _M = bool;
3232 using _V = _Tp;
3233
3234 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3235
3236 struct _Wrapper { using value_type = _V; };
3237
3238 protected:
3239 using value_type
3240 = typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3241
3242 _GLIBCXX_SIMD_INTRINSIC friend const _M&
3243 __get_mask(const const_where_expression& __x)
3244 { return __x._M_k; }
3245
3246 _GLIBCXX_SIMD_INTRINSIC friend const _Tp&
3247 __get_lvalue(const const_where_expression& __x)
3248 { return __x._M_value; }
3249
3250 const bool _M_k;
3251 _Tp& _M_value;
3252
3253 public:
3254 const_where_expression(const const_where_expression&) = delete;
3255 const_where_expression& operator=(const const_where_expression&) = delete;
3256
3257 _GLIBCXX_SIMD_INTRINSIC constexpr
3258 const_where_expression(const bool __kk, const _Tp& dd)
3259 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3260
3261 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3262 operator-() const&&
3263 { return _M_k ? -_M_value : _M_value; }
3264
3265 template <typename _Up, typename _Flags>
3266 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3267 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&&
3268 { return _M_k ? static_cast<_V>(__mem[0]) : _M_value; }
3269
3270 template <typename _Up, typename _Flags>
3271 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3272 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&&
3273 {
3274 if (_M_k)
3275 __mem[0] = _M_value;
3276 }
3277 };
3278
3279// where_expression<M, T> {{{2
3280template <typename _M, typename _Tp>
3281 class where_expression : public const_where_expression<_M, _Tp>
3282 {
3283 using _Impl = typename const_where_expression<_M, _Tp>::_Impl;
3284
3285 static_assert(!is_const<_Tp>::value,
3286 "where_expression may only be instantiated with __a non-const "
3287 "_Tp parameter");
3288
3289 using typename const_where_expression<_M, _Tp>::value_type;
3290 using const_where_expression<_M, _Tp>::_M_k;
3291 using const_where_expression<_M, _Tp>::_M_value;
3292
3293 static_assert(
3294 is_same<typename _M::abi_type, typename _Tp::abi_type>::value, "");
3295 static_assert(_M::size() == _Tp::size(), "");
3296
3297 _GLIBCXX_SIMD_INTRINSIC friend constexpr _Tp&
3298 __get_lvalue(where_expression& __x)
3299 { return __x._M_value; }
3300
3301 public:
3302 where_expression(const where_expression&) = delete;
3303 where_expression& operator=(const where_expression&) = delete;
3304
3305 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3306 where_expression(const _M& __kk, _Tp& dd)
3307 : const_where_expression<_M, _Tp>(__kk, dd) {}
3308
3309 template <typename _Up>
3310 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3311 operator=(_Up&& __x) &&
3312 {
3313 _Impl::_S_masked_assign(__data(_M_k), __data(_M_value),
3314 __to_value_type_or_member_type<_Tp>(
3315 static_cast<_Up&&>(__x)));
3316 }
3317
3318#define _GLIBCXX_SIMD_OP_(__op, __name) \
3319 template <typename _Up> \
3320 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void \
3321 operator __op##=(_Up&& __x)&& \
3322 { \
3323 _Impl::template _S_masked_cassign( \
3324 __data(_M_k), __data(_M_value), \
3325 __to_value_type_or_member_type<_Tp>(static_cast<_Up&&>(__x)), \
3326 [](auto __impl, auto __lhs, auto __rhs) \
3327 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \
3328 { return __impl.__name(__lhs, __rhs); }); \
3329 } \
3330 static_assert(true)
3331 _GLIBCXX_SIMD_OP_(+, _S_plus);
3332 _GLIBCXX_SIMD_OP_(-, _S_minus);
3333 _GLIBCXX_SIMD_OP_(*, _S_multiplies);
3334 _GLIBCXX_SIMD_OP_(/, _S_divides);
3335 _GLIBCXX_SIMD_OP_(%, _S_modulus);
3336 _GLIBCXX_SIMD_OP_(&, _S_bit_and);
3337 _GLIBCXX_SIMD_OP_(|, _S_bit_or);
3338 _GLIBCXX_SIMD_OP_(^, _S_bit_xor);
3339 _GLIBCXX_SIMD_OP_(<<, _S_shift_left);
3340 _GLIBCXX_SIMD_OP_(>>, _S_shift_right);
3341#undef _GLIBCXX_SIMD_OP_
3342
3343 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3344 operator++() &&
3345 {
3346 __data(_M_value)
3347 = _Impl::template _S_masked_unary<__increment>(__data(_M_k), __data(_M_value));
3348 }
3349
3350 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3351 operator++(int) &&
3352 {
3353 __data(_M_value)
3354 = _Impl::template _S_masked_unary<__increment>(__data(_M_k), __data(_M_value));
3355 }
3356
3357 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3358 operator--() &&
3359 {
3360 __data(_M_value)
3361 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k), __data(_M_value));
3362 }
3363
3364 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3365 operator--(int) &&
3366 {
3367 __data(_M_value)
3368 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k), __data(_M_value));
3369 }
3370
3371 // intentionally hides const_where_expression::copy_from
3372 template <typename _Up, typename _Flags>
3373 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3374 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) &&
3375 {
3376 __data(_M_value) = _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3377 _Flags::template _S_apply<_Tp>(__mem));
3378 }
3379 };
3380
3381// where_expression<bool, T> {{{2
3382template <typename _Tp>
3383 class where_expression<bool, _Tp>
3384 : public const_where_expression<bool, _Tp>
3385 {
3386 using _M = bool;
3387 using typename const_where_expression<_M, _Tp>::value_type;
3388 using const_where_expression<_M, _Tp>::_M_k;
3389 using const_where_expression<_M, _Tp>::_M_value;
3390
3391 public:
3392 where_expression(const where_expression&) = delete;
3393 where_expression& operator=(const where_expression&) = delete;
3394
3395 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3396 where_expression(const _M& __kk, _Tp& dd)
3397 : const_where_expression<_M, _Tp>(__kk, dd) {}
3398
3399#define _GLIBCXX_SIMD_OP_(__op) \
3400 template <typename _Up> \
3401 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void \
3402 operator __op(_Up&& __x)&& \
3403 { if (_M_k) _M_value __op static_cast<_Up&&>(__x); }
3404
3405 _GLIBCXX_SIMD_OP_(=)
3406 _GLIBCXX_SIMD_OP_(+=)
3407 _GLIBCXX_SIMD_OP_(-=)
3408 _GLIBCXX_SIMD_OP_(*=)
3409 _GLIBCXX_SIMD_OP_(/=)
3410 _GLIBCXX_SIMD_OP_(%=)
3411 _GLIBCXX_SIMD_OP_(&=)
3412 _GLIBCXX_SIMD_OP_(|=)
3413 _GLIBCXX_SIMD_OP_(^=)
3414 _GLIBCXX_SIMD_OP_(<<=)
3415 _GLIBCXX_SIMD_OP_(>>=)
3416 #undef _GLIBCXX_SIMD_OP_
3417
3418 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3419 operator++() &&
3420 { if (_M_k) ++_M_value; }
3421
3422 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3423 operator++(int) &&
3424 { if (_M_k) ++_M_value; }
3425
3426 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3427 operator--() &&
3428 { if (_M_k) --_M_value; }
3429
3430 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3431 operator--(int) &&
3432 { if (_M_k) --_M_value; }
3433
3434 // intentionally hides const_where_expression::copy_from
3435 template <typename _Up, typename _Flags>
3436 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3437 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) &&
3438 { if (_M_k) _M_value = __mem[0]; }
3439 };
3440
3441// where {{{1
3442template <typename _Tp, typename _Ap>
3443 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3444 where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3445 where(const typename simd<_Tp, _Ap>::mask_type& __k, simd<_Tp, _Ap>& __value)
3446 { return {__k, __value}; }
3447
3448template <typename _Tp, typename _Ap>
3449 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3450 const_where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3451 where(const typename simd<_Tp, _Ap>::mask_type& __k, const simd<_Tp, _Ap>& __value)
3452 { return {__k, __value}; }
3453
3454template <typename _Tp, typename _Ap>
3455 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3456 where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3457 where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k, simd_mask<_Tp, _Ap>& __value)
3458 { return {__k, __value}; }
3459
3460template <typename _Tp, typename _Ap>
3461 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3462 const_where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3463 where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k, const simd_mask<_Tp, _Ap>& __value)
3464 { return {__k, __value}; }
3465
3466template <typename _Tp>
3467 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR where_expression<bool, _Tp>
3468 where(_ExactBool __k, _Tp& __value)
3469 { return {__k, __value}; }
3470
3471template <typename _Tp>
3472 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR const_where_expression<bool, _Tp>
3473 where(_ExactBool __k, const _Tp& __value)
3474 { return {__k, __value}; }
3475
3476template <typename _Tp, typename _Ap>
3477 _GLIBCXX_SIMD_CONSTEXPR void
3478 where(bool __k, simd<_Tp, _Ap>& __value) = delete;
3479
3480template <typename _Tp, typename _Ap>
3481 _GLIBCXX_SIMD_CONSTEXPR void
3482 where(bool __k, const simd<_Tp, _Ap>& __value) = delete;
3483
3484// proposed mask iterations {{{1
3485namespace __proposed {
3486template <size_t _Np>
3487 class where_range
3488 {
3489 const bitset<_Np> __bits;
3490
3491 public:
3492 where_range(bitset<_Np> __b) : __bits(__b) {}
3493
3494 class iterator
3495 {
3496 size_t __mask;
3497 size_t __bit;
3498
3499 _GLIBCXX_SIMD_INTRINSIC void
3500 __next_bit()
3501 { __bit = __builtin_ctzl(__mask); }
3502
3503 _GLIBCXX_SIMD_INTRINSIC void
3504 __reset_lsb()
3505 {
3506 // 01100100 - 1 = 01100011
3507 __mask &= (__mask - 1);
3508 // __asm__("btr %1,%0" : "+r"(__mask) : "r"(__bit));
3509 }
3510
3511 public:
3512 iterator(decltype(__mask) __m) : __mask(__m) { __next_bit(); }
3513 iterator(const iterator&) = default;
3514 iterator(iterator&&) = default;
3515
3516 _GLIBCXX_SIMD_ALWAYS_INLINE size_t
3517 operator->() const
3518 { return __bit; }
3519
3520 _GLIBCXX_SIMD_ALWAYS_INLINE size_t
3521 operator*() const
3522 { return __bit; }
3523
3524 _GLIBCXX_SIMD_ALWAYS_INLINE iterator&
3525 operator++()
3526 {
3527 __reset_lsb();
3528 __next_bit();
3529 return *this;
3530 }
3531
3532 _GLIBCXX_SIMD_ALWAYS_INLINE iterator
3533 operator++(int)
3534 {
3535 iterator __tmp = *this;
3536 __reset_lsb();
3537 __next_bit();
3538 return __tmp;
3539 }
3540
3541 _GLIBCXX_SIMD_ALWAYS_INLINE bool
3542 operator==(const iterator& __rhs) const
3543 { return __mask == __rhs.__mask; }
3544
3545 _GLIBCXX_SIMD_ALWAYS_INLINE bool
3546 operator!=(const iterator& __rhs) const
3547 { return __mask != __rhs.__mask; }
3548 };
3549
3550 iterator
3551 begin() const
3552 { return __bits.to_ullong(); }
3553
3554 iterator
3555 end() const
3556 { return 0; }
3557 };
3558
3559template <typename _Tp, typename _Ap>
3560 where_range<simd_size_v<_Tp, _Ap>>
3561 where(const simd_mask<_Tp, _Ap>& __k)
3562 { return __k.__to_bitset(); }
3563
3564} // namespace __proposed
3565
3566// }}}1
3567// reductions [simd.reductions] {{{1
3568template <typename _Tp, typename _Abi, typename _BinaryOperation = plus<>>
3569 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3570 reduce(const simd<_Tp, _Abi>& __v, _BinaryOperation __binary_op = _BinaryOperation())
3571 { return _Abi::_SimdImpl::_S_reduce(__v, __binary_op); }
3572
3573template <typename _M, typename _V, typename _BinaryOperation = plus<>>
3574 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3575 reduce(const const_where_expression<_M, _V>& __x,
3576 typename _V::value_type __identity_element, _BinaryOperation __binary_op)
3577 {
3578 if (__builtin_expect(none_of(__get_mask(__x)), false))
3579 return __identity_element;
3580
3581 _V __tmp = __identity_element;
3582 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3583 __data(__get_lvalue(__x)));
3584 return reduce(__tmp, __binary_op);
3585 }
3586
3587template <typename _M, typename _V>
3588 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3589 reduce(const const_where_expression<_M, _V>& __x, plus<> __binary_op = {})
3590 { return reduce(__x, 0, __binary_op); }
3591
3592template <typename _M, typename _V>
3593 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3594 reduce(const const_where_expression<_M, _V>& __x, multiplies<> __binary_op)
3595 { return reduce(__x, 1, __binary_op); }
3596
3597template <typename _M, typename _V>
3598 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3599 reduce(const const_where_expression<_M, _V>& __x, bit_and<> __binary_op)
3600 { return reduce(__x, ~typename _V::value_type(), __binary_op); }
3601
3602template <typename _M, typename _V>
3603 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3604 reduce(const const_where_expression<_M, _V>& __x, bit_or<> __binary_op)
3605 { return reduce(__x, 0, __binary_op); }
3606
3607template <typename _M, typename _V>
3608 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3609 reduce(const const_where_expression<_M, _V>& __x, bit_xor<> __binary_op)
3610 { return reduce(__x, 0, __binary_op); }
3611
3612template <typename _Tp, typename _Abi>
3613 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3614 hmin(const simd<_Tp, _Abi>& __v) noexcept
3615 { return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Minimum()); }
3616
3617template <typename _Tp, typename _Abi>
3618 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3619 hmax(const simd<_Tp, _Abi>& __v) noexcept
3620 { return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Maximum()); }
3621
3622template <typename _M, typename _V>
3623 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3624 typename _V::value_type
3625 hmin(const const_where_expression<_M, _V>& __x) noexcept
3626 {
3627 using _Tp = typename _V::value_type;
3628 constexpr _Tp __id_elem =
3629#ifdef __FINITE_MATH_ONLY__
3630 __finite_max_v<_Tp>;
3631#else
3632 __value_or<__infinity, _Tp>(__finite_max_v<_Tp>);
3633#endif
3634 _V __tmp = __id_elem;
3635 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3636 __data(__get_lvalue(__x)));
3637 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Minimum());
3638 }
3639
3640template <typename _M, typename _V>
3641 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3642 typename _V::value_type
3643 hmax(const const_where_expression<_M, _V>& __x) noexcept
3644 {
3645 using _Tp = typename _V::value_type;
3646 constexpr _Tp __id_elem =
3647#ifdef __FINITE_MATH_ONLY__
3648 __finite_min_v<_Tp>;
3649#else
3650 [] {
3651 if constexpr (__value_exists_v<__infinity, _Tp>)
3652 return -__infinity_v<_Tp>;
3653 else
3654 return __finite_min_v<_Tp>;
3655 }();
3656#endif
3657 _V __tmp = __id_elem;
3658 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3659 __data(__get_lvalue(__x)));
3660 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Maximum());
3661 }
3662
3663// }}}1
3664// algorithms [simd.alg] {{{
3665template <typename _Tp, typename _Ap>
3666 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3667 min(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3668 { return {__private_init, _Ap::_SimdImpl::_S_min(__data(__a), __data(__b))}; }
3669
3670template <typename _Tp, typename _Ap>
3671 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3672 max(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3673 { return {__private_init, _Ap::_SimdImpl::_S_max(__data(__a), __data(__b))}; }
3674
3675template <typename _Tp, typename _Ap>
3676 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3677 pair<simd<_Tp, _Ap>, simd<_Tp, _Ap>>
3678 minmax(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3679 {
3680 const auto pair_of_members
3681 = _Ap::_SimdImpl::_S_minmax(__data(__a), __data(__b));
3682 return {simd<_Tp, _Ap>(__private_init, pair_of_members.first),
3683 simd<_Tp, _Ap>(__private_init, pair_of_members.second)};
3684 }
3685
3686template <typename _Tp, typename _Ap>
3687 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3688 clamp(const simd<_Tp, _Ap>& __v, const simd<_Tp, _Ap>& __lo, const simd<_Tp, _Ap>& __hi)
3689 {
3690 using _Impl = typename _Ap::_SimdImpl;
3691 return {__private_init,
3692 _Impl::_S_min(__data(__hi),
3693 _Impl::_S_max(__data(__lo), __data(__v)))};
3694 }
3695
3696// }}}
3697
3698template <size_t... _Sizes, typename _Tp, typename _Ap,
3699 typename = enable_if_t<((_Sizes + ...) == simd<_Tp, _Ap>::size())>>
3700 inline tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3701 split(const simd<_Tp, _Ap>&);
3702
3703// __extract_part {{{
3704template <int _Index, int _Total, int _Combine = 1, typename _Tp, size_t _Np>
3705 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST constexpr
3706 _SimdWrapper<_Tp, _Np / _Total * _Combine>
3707 __extract_part(const _SimdWrapper<_Tp, _Np> __x);
3708
3709template <int _Index, int _Parts, int _Combine = 1, typename _Tp, typename _A0, typename... _As>
3710 _GLIBCXX_SIMD_INTRINSIC constexpr auto
3711 __extract_part(const _SimdTuple<_Tp, _A0, _As...>& __x);
3712
3713// }}}
3714// _SizeList {{{
3715template <size_t _V0, size_t... _Values>
3716 struct _SizeList
3717 {
3718 template <size_t _I>
3719 static constexpr size_t
3720 _S_at(_SizeConstant<_I> = {})
3721 {
3722 if constexpr (_I == 0)
3723 return _V0;
3724 else
3725 return _SizeList<_Values...>::template _S_at<_I - 1>();
3726 }
3727
3728 template <size_t _I>
3729 static constexpr auto
3730 _S_before(_SizeConstant<_I> = {})
3731 {
3732 if constexpr (_I == 0)
3733 return _SizeConstant<0>();
3734 else
3735 return _SizeConstant<
3736 _V0 + _SizeList<_Values...>::template _S_before<_I - 1>()>();
3737 }
3738
3739 template <size_t _Np>
3740 static constexpr auto
3741 _S_pop_front(_SizeConstant<_Np> = {})
3742 {
3743 if constexpr (_Np == 0)
3744 return _SizeList();
3745 else
3746 return _SizeList<_Values...>::template _S_pop_front<_Np - 1>();
3747 }
3748 };
3749
3750// }}}
3751// __extract_center {{{
3752template <typename _Tp, size_t _Np>
3753 _GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _Np / 2>
3754 __extract_center(_SimdWrapper<_Tp, _Np> __x)
3755 {
3756 static_assert(_Np >= 4);
3757 static_assert(_Np % 4 == 0); // x0 - x1 - x2 - x3 -> return {x1, x2}
3758#if _GLIBCXX_SIMD_X86INTRIN // {{{
3759 if constexpr (__have_avx512f && sizeof(_Tp) * _Np == 64)
3760 {
3761 const auto __intrin = __to_intrin(__x);
3762 if constexpr (is_integral_v<_Tp>)
3763 return __vector_bitcast<_Tp>(_mm512_castsi512_si256(
3764 _mm512_shuffle_i32x4(__intrin, __intrin,
3765 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3766 else if constexpr (sizeof(_Tp) == 4)
3767 return __vector_bitcast<_Tp>(_mm512_castps512_ps256(
3768 _mm512_shuffle_f32x4(__intrin, __intrin,
3769 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3770 else if constexpr (sizeof(_Tp) == 8)
3771 return __vector_bitcast<_Tp>(_mm512_castpd512_pd256(
3772 _mm512_shuffle_f64x2(__intrin, __intrin,
3773 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3774 else
3775 __assert_unreachable<_Tp>();
3776 }
3777 else if constexpr (sizeof(_Tp) * _Np == 32 && is_floating_point_v<_Tp>)
3778 return __vector_bitcast<_Tp>(
3779 _mm_shuffle_pd(__lo128(__vector_bitcast<double>(__x)),
3780 __hi128(__vector_bitcast<double>(__x)), 1));
3781 else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) * _Np <= 32)
3782 return __vector_bitcast<_Tp>(
3783 _mm_alignr_epi8(__hi128(__vector_bitcast<_LLong>(__x)),
3784 __lo128(__vector_bitcast<_LLong>(__x)),
3785 sizeof(_Tp) * _Np / 4));
3786 else
3787#endif // _GLIBCXX_SIMD_X86INTRIN }}}
3788 {
3789 __vector_type_t<_Tp, _Np / 2> __r;
3790 __builtin_memcpy(&__r,
3791 reinterpret_cast<const char*>(&__x)
3792 + sizeof(_Tp) * _Np / 4,
3793 sizeof(_Tp) * _Np / 2);
3794 return __r;
3795 }
3796 }
3797
3798template <typename _Tp, typename _A0, typename... _As>
3799 _GLIBCXX_SIMD_INTRINSIC
3800 _SimdWrapper<_Tp, _SimdTuple<_Tp, _A0, _As...>::_S_size() / 2>
3801 __extract_center(const _SimdTuple<_Tp, _A0, _As...>& __x)
3802 {
3803 if constexpr (sizeof...(_As) == 0)
3804 return __extract_center(__x.first);
3805 else
3806 return __extract_part<1, 4, 2>(__x);
3807 }
3808
3809// }}}
3810// __split_wrapper {{{
3811template <size_t... _Sizes, typename _Tp, typename... _As>
3812 auto
3813 __split_wrapper(_SizeList<_Sizes...>, const _SimdTuple<_Tp, _As...>& __x)
3814 {
3815 return split<_Sizes...>(
3816 fixed_size_simd<_Tp, _SimdTuple<_Tp, _As...>::_S_size()>(__private_init,
3817 __x));
3818 }
3819
3820// }}}
3821
3822// split<simd>(simd) {{{
3823template <typename _V, typename _Ap,
3824 size_t _Parts = simd_size_v<typename _V::value_type, _Ap> / _V::size()>
3825 enable_if_t<simd_size_v<typename _V::value_type, _Ap> == _Parts * _V::size()
3826 && is_simd_v<_V>, array<_V, _Parts>>
3827 split(const simd<typename _V::value_type, _Ap>& __x)
3828 {
3829 using _Tp = typename _V::value_type;
3830 if constexpr (_Parts == 1)
3831 {
3832 return {simd_cast<_V>(__x)};
3833 }
3834 else if (__x._M_is_constprop())
3835 {
3836 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3837 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3838 return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
3839 { return __x[__i * _V::size() + __j]; });
3840 });
3841 }
3842 else if constexpr (
3843 __is_fixed_size_abi_v<_Ap>
3844 && (is_same_v<typename _V::abi_type, simd_abi::scalar>
3845 || (__is_fixed_size_abi_v<typename _V::abi_type>
3846 && sizeof(_V) == sizeof(_Tp) * _V::size() // _V doesn't have padding
3847 )))
3848 {
3849 // fixed_size -> fixed_size (w/o padding) or scalar
3850#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
3851 const __may_alias<_Tp>* const __element_ptr
3852 = reinterpret_cast<const __may_alias<_Tp>*>(&__data(__x));
3853 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3854 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
3855 { return _V(__element_ptr + __i * _V::size(), vector_aligned); });
3856#else
3857 const auto& __xx = __data(__x);
3858 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3859 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3860 [[maybe_unused]] constexpr size_t __offset
3861 = decltype(__i)::value * _V::size();
3862 return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3863 constexpr _SizeConstant<__j + __offset> __k;
3864 return __xx[__k];
3865 });
3866 });
3867#endif
3868 }
3869 else if constexpr (is_same_v<typename _V::abi_type, simd_abi::scalar>)
3870 {
3871 // normally memcpy should work here as well
3872 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3873 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3874 }
3875 else
3876 {
3877 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3878 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3879 if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
3880 return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3881 return __x[__i * _V::size() + __j];
3882 });
3883 else
3884 return _V(__private_init,
3885 __extract_part<decltype(__i)::value, _Parts>(__data(__x)));
3886 });
3887 }
3888 }
3889
3890// }}}
3891// split<simd_mask>(simd_mask) {{{
3892template <typename _V, typename _Ap,
3893 size_t _Parts = simd_size_v<typename _V::simd_type::value_type, _Ap> / _V::size()>
3894 enable_if_t<is_simd_mask_v<_V> && simd_size_v<typename
3895 _V::simd_type::value_type, _Ap> == _Parts * _V::size(), array<_V, _Parts>>
3896 split(const simd_mask<typename _V::simd_type::value_type, _Ap>& __x)
3897 {
3898 if constexpr (is_same_v<_Ap, typename _V::abi_type>)
3899 return {__x};
3900 else if constexpr (_Parts == 1)
3901 return {__proposed::static_simd_cast<_V>(__x)};
3902 else if constexpr (_Parts == 2 && __is_sse_abi<typename _V::abi_type>()
3903 && __is_avx_abi<_Ap>())
3904 return {_V(__private_init, __lo128(__data(__x))),
3905 _V(__private_init, __hi128(__data(__x)))};
3906 else if constexpr (_V::size() <= __CHAR_BIT__ * sizeof(_ULLong))
3907 {
3908 const bitset __bits = __x.__to_bitset();
3909 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3910 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3911 constexpr size_t __offset = __i * _V::size();
3912 return _V(__bitset_init, (__bits >> __offset).to_ullong());
3913 });
3914 }
3915 else
3916 {
3917 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3918 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3919 constexpr size_t __offset = __i * _V::size();
3920 return _V(__private_init,
3921 [&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3922 return __x[__j + __offset];
3923 });
3924 });
3925 }
3926 }
3927
3928// }}}
3929// split<_Sizes...>(simd) {{{
3930template <size_t... _Sizes, typename _Tp, typename _Ap, typename>
3931 _GLIBCXX_SIMD_ALWAYS_INLINE
3932 tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3933 split(const simd<_Tp, _Ap>& __x)
3934 {
3935 using _SL = _SizeList<_Sizes...>;
3936 using _Tuple = tuple<__deduced_simd<_Tp, _Sizes>...>;
3937 constexpr size_t _Np = simd_size_v<_Tp, _Ap>;
3938 constexpr size_t _N0 = _SL::template _S_at<0>();
3939 using _V = __deduced_simd<_Tp, _N0>;
3940
3941 if (__x._M_is_constprop())
3942 return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>(
3943 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3944 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3945 constexpr size_t __offset = _SL::_S_before(__i);
3946 return _Vi([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3947 return __x[__offset + __j];
3948 });
3949 });
3950 else if constexpr (_Np == _N0)
3951 {
3952 static_assert(sizeof...(_Sizes) == 1);
3953 return {simd_cast<_V>(__x)};
3954 }
3955 else if constexpr // split from fixed_size, such that __x::first.size == _N0
3956 (__is_fixed_size_abi_v<
3957 _Ap> && __fixed_size_storage_t<_Tp, _Np>::_S_first_size == _N0)
3958 {
3959 static_assert(
3960 !__is_fixed_size_abi_v<typename _V::abi_type>,
3961 "How can <_Tp, _Np> be __a single _SimdTuple entry but __a "
3962 "fixed_size_simd "
3963 "when deduced?");
3964 // extract first and recurse (__split_wrapper is needed to deduce a new
3965 // _Sizes pack)
3966 return tuple_cat(make_tuple(_V(__private_init, __data(__x).first)),
3967 __split_wrapper(_SL::template _S_pop_front<1>(),
3968 __data(__x).second));
3969 }
3970 else if constexpr ((!is_same_v<simd_abi::scalar,
3971 simd_abi::deduce_t<_Tp, _Sizes>> && ...)
3972 && (!__is_fixed_size_abi_v<
3973 simd_abi::deduce_t<_Tp, _Sizes>> && ...))
3974 {
3975 if constexpr (((_Sizes * 2 == _Np) && ...))
3976 return {{__private_init, __extract_part<0, 2>(__data(__x))},
3977 {__private_init, __extract_part<1, 2>(__data(__x))}};
3978 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3979 _SizeList<_Np / 3, _Np / 3, _Np / 3>>)
3980 return {{__private_init, __extract_part<0, 3>(__data(__x))},
3981 {__private_init, __extract_part<1, 3>(__data(__x))},
3982 {__private_init, __extract_part<2, 3>(__data(__x))}};
3983 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3984 _SizeList<2 * _Np / 3, _Np / 3>>)
3985 return {{__private_init, __extract_part<0, 3, 2>(__data(__x))},
3986 {__private_init, __extract_part<2, 3>(__data(__x))}};
3987 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3988 _SizeList<_Np / 3, 2 * _Np / 3>>)
3989 return {{__private_init, __extract_part<0, 3>(__data(__x))},
3990 {__private_init, __extract_part<1, 3, 2>(__data(__x))}};
3991 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3992 _SizeList<_Np / 2, _Np / 4, _Np / 4>>)
3993 return {{__private_init, __extract_part<0, 2>(__data(__x))},
3994 {__private_init, __extract_part<2, 4>(__data(__x))},
3995 {__private_init, __extract_part<3, 4>(__data(__x))}};
3996 else if constexpr (is_same_v<_SizeList<_Sizes...>,
3997 _SizeList<_Np / 4, _Np / 4, _Np / 2>>)
3998 return {{__private_init, __extract_part<0, 4>(__data(__x))},
3999 {__private_init, __extract_part<1, 4>(__data(__x))},
4000 {__private_init, __extract_part<1, 2>(__data(__x))}};
4001 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4002 _SizeList<_Np / 4, _Np / 2, _Np / 4>>)
4003 return {{__private_init, __extract_part<0, 4>(__data(__x))},
4004 {__private_init, __extract_center(__data(__x))},
4005 {__private_init, __extract_part<3, 4>(__data(__x))}};
4006 else if constexpr (((_Sizes * 4 == _Np) && ...))
4007 return {{__private_init, __extract_part<0, 4>(__data(__x))},
4008 {__private_init, __extract_part<1, 4>(__data(__x))},
4009 {__private_init, __extract_part<2, 4>(__data(__x))},
4010 {__private_init, __extract_part<3, 4>(__data(__x))}};
4011 // else fall through
4012 }
4013#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
4014 const __may_alias<_Tp>* const __element_ptr
4015 = reinterpret_cast<const __may_alias<_Tp>*>(&__x);
4016 return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>(
4017 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4018 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4019 constexpr size_t __offset = _SL::_S_before(__i);
4020 constexpr size_t __base_align = alignof(simd<_Tp, _Ap>);
4021 constexpr size_t __a
4022 = __base_align - ((__offset * sizeof(_Tp)) % __base_align);
4023 constexpr size_t __b = ((__a - 1) & __a) ^ __a;
4024 constexpr size_t __alignment = __b == 0 ? __a : __b;
4025 return _Vi(__element_ptr + __offset, overaligned<__alignment>);
4026 });
4027#else
4028 return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>(
4029 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4030 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4031 const auto& __xx = __data(__x);
4032 using _Offset = decltype(_SL::_S_before(__i));
4033 return _Vi([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4034 constexpr _SizeConstant<_Offset::value + __j> __k;
4035 return __xx[__k];
4036 });
4037 });
4038#endif
4039 }
4040
4041// }}}
4042
4043// __subscript_in_pack {{{
4044template <size_t _I, typename _Tp, typename _Ap, typename... _As>
4045 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
4046 __subscript_in_pack(const simd<_Tp, _Ap>& __x, const simd<_Tp, _As>&... __xs)
4047 {
4048 if constexpr (_I < simd_size_v<_Tp, _Ap>)
4049 return __x[_I];
4050 else
4051 return __subscript_in_pack<_I - simd_size_v<_Tp, _Ap>>(__xs...);
4052 }
4053
4054// }}}
4055// __store_pack_of_simd {{{
4056template <typename _Tp, typename _A0, typename... _As>
4057 _GLIBCXX_SIMD_INTRINSIC void
4058 __store_pack_of_simd(char* __mem, const simd<_Tp, _A0>& __x0, const simd<_Tp, _As>&... __xs)
4059 {
4060 constexpr size_t __n_bytes = sizeof(_Tp) * simd_size_v<_Tp, _A0>;
4061 __builtin_memcpy(__mem, &__data(__x0), __n_bytes);
4062 if constexpr (sizeof...(__xs) > 0)
4063 __store_pack_of_simd(__mem + __n_bytes, __xs...);
4064 }
4065
4066// }}}
4067// concat(simd...) {{{
4068template <typename _Tp, typename... _As>
4069 inline _GLIBCXX_SIMD_CONSTEXPR
4070 simd<_Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>
4071 concat(const simd<_Tp, _As>&... __xs)
4072 {
4073 using _Rp = __deduced_simd<_Tp, (simd_size_v<_Tp, _As> + ...)>;
4074 if constexpr (sizeof...(__xs) == 1)
4075 return simd_cast<_Rp>(__xs...);
4076 else if ((... && __xs._M_is_constprop()))
4077 return simd<_Tp,
4078 simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>(
4079 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
4080 { return __subscript_in_pack<__i>(__xs...); });
4081 else
4082 {
4083 _Rp __r{};
4084 __store_pack_of_simd(reinterpret_cast<char*>(&__data(__r)), __xs...);
4085 return __r;
4086 }
4087 }
4088
4089// }}}
4090// concat(array<simd>) {{{
4091template <typename _Tp, typename _Abi, size_t _Np>
4092 _GLIBCXX_SIMD_ALWAYS_INLINE
4093 _GLIBCXX_SIMD_CONSTEXPR __deduced_simd<_Tp, simd_size_v<_Tp, _Abi> * _Np>
4094 concat(const array<simd<_Tp, _Abi>, _Np>& __x)
4095 {
4096 return __call_with_subscripts<_Np>(
4097 __x, [](const auto&... __xs) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4098 return concat(__xs...);
4099 });
4100 }
4101
4102// }}}
4103
4104/// @cond undocumented
4105// _SmartReference {{{
4106template <typename _Up, typename _Accessor = _Up,
4107 typename _ValueType = typename _Up::value_type>
4108 class _SmartReference
4109 {
4110 friend _Accessor;
4111 int _M_index;
4112 _Up& _M_obj;
4113
4114 _GLIBCXX_SIMD_INTRINSIC constexpr _ValueType
4115 _M_read() const noexcept
4116 {
4117 if constexpr (is_arithmetic_v<_Up>)
4118 return _M_obj;
4119 else
4120 return _M_obj[_M_index];
4121 }
4122
4123 template <typename _Tp>
4124 _GLIBCXX_SIMD_INTRINSIC constexpr void
4125 _M_write(_Tp&& __x) const
4126 { _Accessor::_S_set(_M_obj, _M_index, static_cast<_Tp&&>(__x)); }
4127
4128 public:
4129 _GLIBCXX_SIMD_INTRINSIC constexpr
4130 _SmartReference(_Up& __o, int __i) noexcept
4131 : _M_index(__i), _M_obj(__o) {}
4132
4133 using value_type = _ValueType;
4134
4135 _GLIBCXX_SIMD_INTRINSIC
4136 _SmartReference(const _SmartReference&) = delete;
4137
4138 _GLIBCXX_SIMD_INTRINSIC constexpr
4139 operator value_type() const noexcept
4140 { return _M_read(); }
4141
4142 template <typename _Tp, typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, value_type>>
4143 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference
4144 operator=(_Tp&& __x) &&
4145 {
4146 _M_write(static_cast<_Tp&&>(__x));
4147 return {_M_obj, _M_index};
4148 }
4149
4150#define _GLIBCXX_SIMD_OP_(__op) \
4151 template <typename _Tp, \
4152 typename _TT = decltype(declval<value_type>() __op declval<_Tp>()), \
4153 typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, _TT>, \
4154 typename = _ValuePreservingOrInt<_TT, value_type>> \
4155 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference \
4156 operator __op##=(_Tp&& __x) && \
4157 { \
4158 const value_type& __lhs = _M_read(); \
4159 _M_write(__lhs __op __x); \
4160 return {_M_obj, _M_index}; \
4161 }
4162 _GLIBCXX_SIMD_ALL_ARITHMETICS(_GLIBCXX_SIMD_OP_);
4163 _GLIBCXX_SIMD_ALL_SHIFTS(_GLIBCXX_SIMD_OP_);
4164 _GLIBCXX_SIMD_ALL_BINARY(_GLIBCXX_SIMD_OP_);
4165#undef _GLIBCXX_SIMD_OP_
4166
4167 template <typename _Tp = void,
4168 typename = decltype(++declval<conditional_t<true, value_type, _Tp>&>())>
4169 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference
4170 operator++() &&
4171 {
4172 value_type __x = _M_read();
4173 _M_write(++__x);
4174 return {_M_obj, _M_index};
4175 }
4176
4177 template <typename _Tp = void,
4178 typename = decltype(declval<conditional_t<true, value_type, _Tp>&>()++)>
4179 _GLIBCXX_SIMD_INTRINSIC constexpr value_type
4180 operator++(int) &&
4181 {
4182 const value_type __r = _M_read();
4183 value_type __x = __r;
4184 _M_write(++__x);
4185 return __r;
4186 }
4187
4188 template <typename _Tp = void,
4189 typename = decltype(--declval<conditional_t<true, value_type, _Tp>&>())>
4190 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference
4191 operator--() &&
4192 {
4193 value_type __x = _M_read();
4194 _M_write(--__x);
4195 return {_M_obj, _M_index};
4196 }
4197
4198 template <typename _Tp = void,
4199 typename = decltype(declval<conditional_t<true, value_type, _Tp>&>()--)>
4200 _GLIBCXX_SIMD_INTRINSIC constexpr value_type
4201 operator--(int) &&
4202 {
4203 const value_type __r = _M_read();
4204 value_type __x = __r;
4205 _M_write(--__x);
4206 return __r;
4207 }
4208
4209 _GLIBCXX_SIMD_INTRINSIC friend void
4210 swap(_SmartReference&& __a, _SmartReference&& __b) noexcept(
4211 conjunction<
4212 is_nothrow_constructible<value_type, _SmartReference&&>,
4213 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4214 {
4215 value_type __tmp = static_cast<_SmartReference&&>(__a);
4216 static_cast<_SmartReference&&>(__a) = static_cast<value_type>(__b);
4217 static_cast<_SmartReference&&>(__b) = std::move(__tmp);
4218 }
4219
4220 _GLIBCXX_SIMD_INTRINSIC friend void
4221 swap(value_type& __a, _SmartReference&& __b) noexcept(
4222 conjunction<
4223 is_nothrow_constructible<value_type, value_type&&>,
4224 is_nothrow_assignable<value_type&, value_type&&>,
4225 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4226 {
4227 value_type __tmp(std::move(__a));
4228 __a = static_cast<value_type>(__b);
4229 static_cast<_SmartReference&&>(__b) = std::move(__tmp);
4230 }
4231
4232 _GLIBCXX_SIMD_INTRINSIC friend void
4233 swap(_SmartReference&& __a, value_type& __b) noexcept(
4234 conjunction<
4235 is_nothrow_constructible<value_type, _SmartReference&&>,
4236 is_nothrow_assignable<value_type&, value_type&&>,
4237 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4238 {
4239 value_type __tmp(__a);
4240 static_cast<_SmartReference&&>(__a) = std::move(__b);
4241 __b = std::move(__tmp);
4242 }
4243 };
4244
4245// }}}
4246// __scalar_abi_wrapper {{{
4247template <int _Bytes>
4248 struct __scalar_abi_wrapper
4249 {
4250 template <typename _Tp> static constexpr size_t _S_full_size = 1;
4251 template <typename _Tp> static constexpr size_t _S_size = 1;
4252 template <typename _Tp> static constexpr size_t _S_is_partial = false;
4253
4254 template <typename _Tp, typename _Abi = simd_abi::scalar>
4255 static constexpr bool _S_is_valid_v
4256 = _Abi::template _IsValid<_Tp>::value && sizeof(_Tp) == _Bytes;
4257 };
4258
4259// }}}
4260// __decay_abi metafunction {{{
4261template <typename _Tp>
4262 struct __decay_abi { using type = _Tp; };
4263
4264template <int _Bytes>
4265 struct __decay_abi<__scalar_abi_wrapper<_Bytes>>
4266 { using type = simd_abi::scalar; };
4267
4268// }}}
4269// __find_next_valid_abi metafunction {{{1
4270// Given an ABI tag A<N>, find an N2 < N such that A<N2>::_S_is_valid_v<_Tp> ==
4271// true, N2 is a power-of-2, and A<N2>::_S_is_partial<_Tp> is false. Break
4272// recursion at 2 elements in the resulting ABI tag. In this case
4273// type::_S_is_valid_v<_Tp> may be false.
4274template <template <int> class _Abi, int _Bytes, typename _Tp>
4275 struct __find_next_valid_abi
4276 {
4277 static constexpr auto
4278 _S_choose()
4279 {
4280 constexpr int _NextBytes = std::__bit_ceil(_Bytes) / 2;
4281 using _NextAbi = _Abi<_NextBytes>;
4282 if constexpr (_NextBytes < sizeof(_Tp) * 2) // break recursion
4283 return _Abi<_Bytes>();
4284 else if constexpr (_NextAbi::template _S_is_partial<_Tp> == false
4285 && _NextAbi::template _S_is_valid_v<_Tp>)
4286 return _NextAbi();
4287 else
4288 return __find_next_valid_abi<_Abi, _NextBytes, _Tp>::_S_choose();
4289 }
4290
4291 using type = decltype(_S_choose());
4292 };
4293
4294template <int _Bytes, typename _Tp>
4295 struct __find_next_valid_abi<__scalar_abi_wrapper, _Bytes, _Tp>
4296 { using type = simd_abi::scalar; };
4297
4298// _AbiList {{{1
4299template <template <int> class...>
4300 struct _AbiList
4301 {
4302 template <typename, int> static constexpr bool _S_has_valid_abi = false;
4303 template <typename, int> using _FirstValidAbi = void;
4304 template <typename, int> using _BestAbi = void;
4305 };
4306
4307template <template <int> class _A0, template <int> class... _Rest>
4308 struct _AbiList<_A0, _Rest...>
4309 {
4310 template <typename _Tp, int _Np>
4311 static constexpr bool _S_has_valid_abi
4312 = _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<
4313 _Tp> || _AbiList<_Rest...>::template _S_has_valid_abi<_Tp, _Np>;
4314
4315 template <typename _Tp, int _Np>
4316 using _FirstValidAbi = conditional_t<
4317 _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<_Tp>,
4318 typename __decay_abi<_A0<sizeof(_Tp) * _Np>>::type,
4319 typename _AbiList<_Rest...>::template _FirstValidAbi<_Tp, _Np>>;
4320
4321 template <typename _Tp, int _Np>
4322 static constexpr auto
4323 _S_determine_best_abi()
4324 {
4325 static_assert(_Np >= 1);
4326 constexpr int _Bytes = sizeof(_Tp) * _Np;
4327 if constexpr (_Np == 1)
4328 return __make_dependent_t<_Tp, simd_abi::scalar>{};
4329 else
4330 {
4331 constexpr int __fullsize = _A0<_Bytes>::template _S_full_size<_Tp>;
4332 // _A0<_Bytes> is good if:
4333 // 1. The ABI tag is valid for _Tp
4334 // 2. The storage overhead is no more than padding to fill the next
4335 // power-of-2 number of bytes
4336 if constexpr (_A0<_Bytes>::template _S_is_valid_v<
4337 _Tp> && __fullsize / 2 < _Np)
4338 return typename __decay_abi<_A0<_Bytes>>::type{};
4339 else
4340 {
4341 using _Bp =
4342 typename __find_next_valid_abi<_A0, _Bytes, _Tp>::type;
4343 if constexpr (_Bp::template _S_is_valid_v<
4344 _Tp> && _Bp::template _S_size<_Tp> <= _Np)
4345 return _Bp{};
4346 else
4347 return
4348 typename _AbiList<_Rest...>::template _BestAbi<_Tp, _Np>{};
4349 }
4350 }
4351 }
4352
4353 template <typename _Tp, int _Np>
4354 using _BestAbi = decltype(_S_determine_best_abi<_Tp, _Np>());
4355 };
4356
4357// }}}1
4358
4359// the following lists all native ABIs, which makes them accessible to
4360// simd_abi::deduce and select_best_vector_type_t (for fixed_size). Order
4361// matters: Whatever comes first has higher priority.
4362using _AllNativeAbis = _AbiList<simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin,
4363 __scalar_abi_wrapper>;
4364
4365// valid _SimdTraits specialization {{{1
4366template <typename _Tp, typename _Abi>
4367 struct _SimdTraits<_Tp, _Abi, void_t<typename _Abi::template _IsValid<_Tp>>>
4368 : _Abi::template __traits<_Tp> {};
4369
4370// __deduce_impl specializations {{{1
4371// try all native ABIs (including scalar) first
4372template <typename _Tp, size_t _Np>
4373 struct __deduce_impl<
4374 _Tp, _Np, enable_if_t<_AllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>>
4375 { using type = _AllNativeAbis::_FirstValidAbi<_Tp, _Np>; };
4376
4377// fall back to fixed_size only if scalar and native ABIs don't match
4378template <typename _Tp, size_t _Np, typename = void>
4379 struct __deduce_fixed_size_fallback {};
4380
4381template <typename _Tp, size_t _Np>
4382 struct __deduce_fixed_size_fallback<_Tp, _Np,
4383 enable_if_t<simd_abi::fixed_size<_Np>::template _S_is_valid_v<_Tp>>>
4384 { using type = simd_abi::fixed_size<_Np>; };
4385
4386template <typename _Tp, size_t _Np, typename>
4387 struct __deduce_impl : public __deduce_fixed_size_fallback<_Tp, _Np> {};
4388
4389//}}}1
4390/// @endcond
4391
4392// simd_mask {{{
4393template <typename _Tp, typename _Abi>
4394 class simd_mask : public _SimdTraits<_Tp, _Abi>::_MaskBase
4395 {
4396 // types, tags, and friends {{{
4397 using _Traits = _SimdTraits<_Tp, _Abi>;
4398 using _MemberType = typename _Traits::_MaskMember;
4399
4400 // We map all masks with equal element sizeof to a single integer type, the
4401 // one given by __int_for_sizeof_t<_Tp>. This is the approach
4402 // [[gnu::vector_size(N)]] types take as well and it reduces the number of
4403 // template specializations in the implementation classes.
4404 using _Ip = __int_for_sizeof_t<_Tp>;
4405 static constexpr _Ip* _S_type_tag = nullptr;
4406
4407 friend typename _Traits::_MaskBase;
4408 friend class simd<_Tp, _Abi>; // to construct masks on return
4409 friend typename _Traits::_SimdImpl; // to construct masks on return and
4410 // inspect data on masked operations
4411 public:
4412 using _Impl = typename _Traits::_MaskImpl;
4413 friend _Impl;
4414
4415 // }}}
4416 // member types {{{
4417 using value_type = bool;
4418 using reference = _SmartReference<_MemberType, _Impl, value_type>;
4419 using simd_type = simd<_Tp, _Abi>;
4420 using abi_type = _Abi;
4421
4422 // }}}
4423 static constexpr size_t size() // {{{
4424 { return __size_or_zero_v<_Tp, _Abi>; }
4425
4426 // }}}
4427 // constructors & assignment {{{
4428 simd_mask() = default;
4429 simd_mask(const simd_mask&) = default;
4430 simd_mask(simd_mask&&) = default;
4431 simd_mask& operator=(const simd_mask&) = default;
4432 simd_mask& operator=(simd_mask&&) = default;
4433
4434 // }}}
4435 // access to internal representation (optional feature) {{{
4436 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR explicit
4437 simd_mask(typename _Traits::_MaskCastType __init)
4438 : _M_data{__init} {}
4439 // conversions to internal type is done in _MaskBase
4440
4441 // }}}
4442 // bitset interface (extension to be proposed) {{{
4443 // TS_FEEDBACK:
4444 // Conversion of simd_mask to and from bitset makes it much easier to
4445 // interface with other facilities. I suggest adding `static
4446 // simd_mask::from_bitset` and `simd_mask::to_bitset`.
4447 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR static simd_mask
4448 __from_bitset(bitset<size()> bs)
4449 { return {__bitset_init, bs}; }
4450
4451 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bitset<size()>
4452 __to_bitset() const
4453 { return _Impl::_S_to_bits(_M_data)._M_to_bitset(); }
4454
4455 // }}}
4456 // explicit broadcast constructor {{{
4457 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
4458 simd_mask(value_type __x)
4459 : _M_data(_Impl::template _S_broadcast<_Ip>(__x)) {}
4460
4461 // }}}
4462 // implicit type conversion constructor {{{
4463 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4464 // proposed improvement
4465 template <typename _Up, typename _A2,
4466 typename = enable_if_t<simd_size_v<_Up, _A2> == size()>>
4467 _GLIBCXX_SIMD_ALWAYS_INLINE explicit(sizeof(_MemberType)
4468 != sizeof(typename _SimdTraits<_Up, _A2>::_MaskMember))
4469 simd_mask(const simd_mask<_Up, _A2>& __x)
4470 : simd_mask(__proposed::static_simd_cast<simd_mask>(__x)) {}
4471 #else
4472 // conforming to ISO/IEC 19570:2018
4473 template <typename _Up, typename = enable_if_t<conjunction<
4474 is_same<abi_type, simd_abi::fixed_size<size()>>,
4475 is_same<_Up, _Up>>::value>>
4476 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4477 simd_mask(const simd_mask<_Up, simd_abi::fixed_size<size()>>& __x)
4478 : _M_data(_Impl::_S_from_bitmask(__data(__x), _S_type_tag)) {}
4479 #endif
4480
4481 // }}}
4482 // load constructor {{{
4483 template <typename _Flags>
4484 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4485 simd_mask(const value_type* __mem, _IsSimdFlagType<_Flags>)
4486 : _M_data(_Impl::template _S_load<_Ip>(_Flags::template _S_apply<simd_mask>(__mem))) {}
4487
4488 template <typename _Flags>
4489 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4490 simd_mask(const value_type* __mem, simd_mask __k, _IsSimdFlagType<_Flags>)
4491 : _M_data{}
4492 {
4493 _M_data = _Impl::_S_masked_load(_M_data, __k._M_data,
4494 _Flags::template _S_apply<simd_mask>(__mem));
4495 }
4496
4497 // }}}
4498 // loads [simd_mask.load] {{{
4499 template <typename _Flags>
4500 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void
4501 copy_from(const value_type* __mem, _IsSimdFlagType<_Flags>)
4502 { _M_data = _Impl::template _S_load<_Ip>(_Flags::template _S_apply<simd_mask>(__mem)); }
4503
4504 // }}}
4505 // stores [simd_mask.store] {{{
4506 template <typename _Flags>
4507 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void
4508 copy_to(value_type* __mem, _IsSimdFlagType<_Flags>) const
4509 { _Impl::_S_store(_M_data, _Flags::template _S_apply<simd_mask>(__mem)); }
4510
4511 // }}}
4512 // scalar access {{{
4513 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
4514 operator[](size_t __i)
4515 {
4516 if (__i >= size())
4517 __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1);
4518 return {_M_data, int(__i)};
4519 }
4520
4521 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
4522 operator[](size_t __i) const
4523 {
4524 if (__i >= size())
4525 __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1);
4526 if constexpr (__is_scalar_abi<_Abi>())
4527 return _M_data;
4528 else
4529 return static_cast<bool>(_M_data[__i]);
4530 }
4531
4532 // }}}
4533 // negation {{{
4534 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd_mask
4535 operator!() const
4536 { return {__private_init, _Impl::_S_bit_not(_M_data)}; }
4537
4538 // }}}
4539 // simd_mask binary operators [simd_mask.binary] {{{
4540 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4541 // simd_mask<int> && simd_mask<uint> needs disambiguation
4542 template <typename _Up, typename _A2,
4543 typename = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4544 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4545 operator&&(const simd_mask& __x, const simd_mask<_Up, _A2>& __y)
4546 {
4547 return {__private_init,
4548 _Impl::_S_logical_and(__x._M_data, simd_mask(__y)._M_data)};
4549 }
4550
4551 template <typename _Up, typename _A2,
4552 typename = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4553 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4554 operator||(const simd_mask& __x, const simd_mask<_Up, _A2>& __y)
4555 {
4556 return {__private_init,
4557 _Impl::_S_logical_or(__x._M_data, simd_mask(__y)._M_data)};
4558 }
4559 #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4560
4561 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4562 operator&&(const simd_mask& __x, const simd_mask& __y)
4563 { return {__private_init, _Impl::_S_logical_and(__x._M_data, __y._M_data)}; }
4564
4565 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4566 operator||(const simd_mask& __x, const simd_mask& __y)
4567 { return {__private_init, _Impl::_S_logical_or(__x._M_data, __y._M_data)}; }
4568
4569 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4570 operator&(const simd_mask& __x, const simd_mask& __y)
4571 { return {__private_init, _Impl::_S_bit_and(__x._M_data, __y._M_data)}; }
4572
4573 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4574 operator|(const simd_mask& __x, const simd_mask& __y)
4575 { return {__private_init, _Impl::_S_bit_or(__x._M_data, __y._M_data)}; }
4576
4577 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4578 operator^(const simd_mask& __x, const simd_mask& __y)
4579 { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4580
4581 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask&
4582 operator&=(simd_mask& __x, const simd_mask& __y)
4583 {
4584 __x._M_data = _Impl::_S_bit_and(__x._M_data, __y._M_data);
4585 return __x;
4586 }
4587
4588 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask&
4589 operator|=(simd_mask& __x, const simd_mask& __y)
4590 {
4591 __x._M_data = _Impl::_S_bit_or(__x._M_data, __y._M_data);
4592 return __x;
4593 }
4594
4595 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask&
4596 operator^=(simd_mask& __x, const simd_mask& __y)
4597 {
4598 __x._M_data = _Impl::_S_bit_xor(__x._M_data, __y._M_data);
4599 return __x;
4600 }
4601
4602 // }}}
4603 // simd_mask compares [simd_mask.comparison] {{{
4604 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4605 operator==(const simd_mask& __x, const simd_mask& __y)
4606 { return !operator!=(__x, __y); }
4607
4608 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4609 operator!=(const simd_mask& __x, const simd_mask& __y)
4610 { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4611
4612 // }}}
4613 // private_init ctor {{{
4614 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
4615 simd_mask(_PrivateInit, typename _Traits::_MaskMember __init)
4616 : _M_data(__init) {}
4617
4618 // }}}
4619 // private_init generator ctor {{{
4620 template <typename _Fp, typename = decltype(bool(declval<_Fp>()(size_t())))>
4621 _GLIBCXX_SIMD_INTRINSIC constexpr
4622 simd_mask(_PrivateInit, _Fp&& __gen)
4623 : _M_data()
4624 {
4625 __execute_n_times<size()>([&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4626 _Impl::_S_set(_M_data, __i, __gen(__i));
4627 });
4628 }
4629
4630 // }}}
4631 // bitset_init ctor {{{
4632 _GLIBCXX_SIMD_INTRINSIC constexpr
4633 simd_mask(_BitsetInit, bitset<size()> __init)
4634 : _M_data(_Impl::_S_from_bitmask(_SanitizedBitMask<size()>(__init), _S_type_tag))
4635 {}
4636
4637 // }}}
4638 // __cvt {{{
4639 // TS_FEEDBACK:
4640 // The conversion operator this implements should be a ctor on simd_mask.
4641 // Once you call .__cvt() on a simd_mask it converts conveniently.
4642 // A useful variation: add `explicit(sizeof(_Tp) != sizeof(_Up))`
4643 struct _CvtProxy
4644 {
4645 template <typename _Up, typename _A2,
4646 typename = enable_if_t<simd_size_v<_Up, _A2> == simd_size_v<_Tp, _Abi>>>
4647 operator simd_mask<_Up, _A2>() &&
4648 {
4649 using namespace std::experimental::__proposed;
4650 return static_simd_cast<simd_mask<_Up, _A2>>(_M_data);
4651 }
4652
4653 const simd_mask<_Tp, _Abi>& _M_data;
4654 };
4655
4656 _GLIBCXX_SIMD_INTRINSIC _CvtProxy
4657 __cvt() const
4658 { return {*this}; }
4659
4660 // }}}
4661 // operator?: overloads (suggested extension) {{{
4662 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
4663 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4664 operator?:(const simd_mask& __k, const simd_mask& __where_true,
4665 const simd_mask& __where_false)
4666 {
4667 auto __ret = __where_false;
4668 _Impl::_S_masked_assign(__k._M_data, __ret._M_data, __where_true._M_data);
4669 return __ret;
4670 }
4671
4672 template <typename _U1, typename _U2,
4673 typename _Rp = simd<common_type_t<_U1, _U2>, _Abi>,
4674 typename = enable_if_t<conjunction_v<
4675 is_convertible<_U1, _Rp>, is_convertible<_U2, _Rp>,
4676 is_convertible<simd_mask, typename _Rp::mask_type>>>>
4677 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend _Rp
4678 operator?:(const simd_mask& __k, const _U1& __where_true,
4679 const _U2& __where_false)
4680 {
4681 _Rp __ret = __where_false;
4682 _Rp::_Impl::_S_masked_assign(
4683 __data(static_cast<typename _Rp::mask_type>(__k)), __data(__ret),
4684 __data(static_cast<_Rp>(__where_true)));
4685 return __ret;
4686 }
4687
4688 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4689 template <typename _Kp, typename _Ak, typename _Up, typename _Au,
4690 typename = enable_if_t<
4691 conjunction_v<is_convertible<simd_mask<_Kp, _Ak>, simd_mask>,
4692 is_convertible<simd_mask<_Up, _Au>, simd_mask>>>>
4693 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4694 operator?:(const simd_mask<_Kp, _Ak>& __k, const simd_mask& __where_true,
4695 const simd_mask<_Up, _Au>& __where_false)
4696 {
4697 simd_mask __ret = __where_false;
4698 _Impl::_S_masked_assign(simd_mask(__k)._M_data, __ret._M_data,
4699 __where_true._M_data);
4700 return __ret;
4701 }
4702 #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4703 #endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__
4704
4705 // }}}
4706 // _M_is_constprop {{{
4707 _GLIBCXX_SIMD_INTRINSIC constexpr bool
4708 _M_is_constprop() const
4709 {
4710 if constexpr (__is_scalar_abi<_Abi>())
4711 return __builtin_constant_p(_M_data);
4712 else
4713 return _M_data._M_is_constprop();
4714 }
4715
4716 // }}}
4717
4718 private:
4719 friend const auto& __data<_Tp, abi_type>(const simd_mask&);
4720 friend auto& __data<_Tp, abi_type>(simd_mask&);
4721 alignas(_Traits::_S_mask_align) _MemberType _M_data;
4722 };
4723
4724// }}}
4725
4726/// @cond undocumented
4727// __data(simd_mask) {{{
4728template <typename _Tp, typename _Ap>
4729 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
4730 __data(const simd_mask<_Tp, _Ap>& __x)
4731 { return __x._M_data; }
4732
4733template <typename _Tp, typename _Ap>
4734 _GLIBCXX_SIMD_INTRINSIC constexpr auto&
4735 __data(simd_mask<_Tp, _Ap>& __x)
4736 { return __x._M_data; }
4737
4738// }}}
4739/// @endcond
4740
4741// simd_mask reductions [simd_mask.reductions] {{{
4742template <typename _Tp, typename _Abi>
4743 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4744 all_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4745 {
4746 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4747 {
4748 for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4749 if (!__k[__i])
4750 return false;
4751 return true;
4752 }
4753 else
4754 return _Abi::_MaskImpl::_S_all_of(__k);
4755 }
4756
4757template <typename _Tp, typename _Abi>
4758 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4759 any_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4760 {
4761 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4762 {
4763 for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4764 if (__k[__i])
4765 return true;
4766 return false;
4767 }
4768 else
4769 return _Abi::_MaskImpl::_S_any_of(__k);
4770 }
4771
4772template <typename _Tp, typename _Abi>
4773 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4774 none_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4775 {
4776 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4777 {
4778 for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4779 if (__k[__i])
4780 return false;
4781 return true;
4782 }
4783 else
4784 return _Abi::_MaskImpl::_S_none_of(__k);
4785 }
4786
4787template <typename _Tp, typename _Abi>
4788 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4789 some_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4790 {
4791 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4792 {
4793 for (size_t __i = 1; __i < simd_size_v<_Tp, _Abi>; ++__i)
4794 if (__k[__i] != __k[__i - 1])
4795 return true;
4796 return false;
4797 }
4798 else
4799 return _Abi::_MaskImpl::_S_some_of(__k);
4800 }
4801
4802template <typename _Tp, typename _Abi>
4803 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4804 popcount(const simd_mask<_Tp, _Abi>& __k) noexcept
4805 {
4806 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4807 {
4808 const int __r = __call_with_subscripts<simd_size_v<_Tp, _Abi>>(
4809 __k, [](auto... __elements) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4810 return ((__elements != 0) + ...);
4811 });
4812 if (__builtin_is_constant_evaluated() || __builtin_constant_p(__r))
4813 return __r;
4814 }
4815 return _Abi::_MaskImpl::_S_popcount(__k);
4816 }
4817
4818template <typename _Tp, typename _Abi>
4819 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4820 find_first_set(const simd_mask<_Tp, _Abi>& __k)
4821 {
4822 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4823 {
4824 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
4825 const size_t _Idx = __call_with_n_evaluations<_Np>(
4826 [](auto... __indexes) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4827 return std::min({__indexes...});
4828 }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4829 return __k[__i] ? +__i : _Np;
4830 });
4831 if (_Idx >= _Np)
4832 __invoke_ub("find_first_set(empty mask) is UB");
4833 if (__builtin_constant_p(_Idx))
4834 return _Idx;
4835 }
4836 return _Abi::_MaskImpl::_S_find_first_set(__k);
4837 }
4838
4839template <typename _Tp, typename _Abi>
4840 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4841 find_last_set(const simd_mask<_Tp, _Abi>& __k)
4842 {
4843 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4844 {
4845 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
4846 const int _Idx = __call_with_n_evaluations<_Np>(
4847 [](auto... __indexes) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4848 return std::max({__indexes...});
4849 }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4850 return __k[__i] ? int(__i) : -1;
4851 });
4852 if (_Idx < 0)
4853 __invoke_ub("find_first_set(empty mask) is UB");
4854 if (__builtin_constant_p(_Idx))
4855 return _Idx;
4856 }
4857 return _Abi::_MaskImpl::_S_find_last_set(__k);
4858 }
4859
4860_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4861all_of(_ExactBool __x) noexcept
4862{ return __x; }
4863
4864_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4865any_of(_ExactBool __x) noexcept
4866{ return __x; }
4867
4868_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4869none_of(_ExactBool __x) noexcept
4870{ return !__x; }
4871
4872_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4873some_of(_ExactBool) noexcept
4874{ return false; }
4875
4876_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4877popcount(_ExactBool __x) noexcept
4878{ return __x; }
4879
4880_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4881find_first_set(_ExactBool)
4882{ return 0; }
4883
4884_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4885find_last_set(_ExactBool)
4886{ return 0; }
4887
4888// }}}
4889
4890/// @cond undocumented
4891// _SimdIntOperators{{{1
4892template <typename _V, typename _Impl, bool>
4893 class _SimdIntOperators {};
4894
4895template <typename _V, typename _Impl>
4896 class _SimdIntOperators<_V, _Impl, true>
4897 {
4898 _GLIBCXX_SIMD_INTRINSIC constexpr const _V&
4899 __derived() const
4900 { return *static_cast<const _V*>(this); }
4901
4902 template <typename _Tp>
4903 _GLIBCXX_SIMD_INTRINSIC static _GLIBCXX_SIMD_CONSTEXPR _V
4904 _S_make_derived(_Tp&& __d)
4905 { return {__private_init, static_cast<_Tp&&>(__d)}; }
4906
4907 public:
4908 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator%=(_V& __lhs, const _V& __x)
4909 { return __lhs = __lhs % __x; }
4910
4911 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator&=(_V& __lhs, const _V& __x)
4912 { return __lhs = __lhs & __x; }
4913
4914 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator|=(_V& __lhs, const _V& __x)
4915 { return __lhs = __lhs | __x; }
4916
4917 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator^=(_V& __lhs, const _V& __x)
4918 { return __lhs = __lhs ^ __x; }
4919
4920 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator<<=(_V& __lhs, const _V& __x)
4921 { return __lhs = __lhs << __x; }
4922
4923 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator>>=(_V& __lhs, const _V& __x)
4924 { return __lhs = __lhs >> __x; }
4925
4926 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator<<=(_V& __lhs, int __x)
4927 { return __lhs = __lhs << __x; }
4928
4929 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator>>=(_V& __lhs, int __x)
4930 { return __lhs = __lhs >> __x; }
4931
4932 _GLIBCXX_SIMD_CONSTEXPR friend _V operator%(const _V& __x, const _V& __y)
4933 {
4934 return _SimdIntOperators::_S_make_derived(
4935 _Impl::_S_modulus(__data(__x), __data(__y)));
4936 }
4937
4938 _GLIBCXX_SIMD_CONSTEXPR friend _V operator&(const _V& __x, const _V& __y)
4939 {
4940 return _SimdIntOperators::_S_make_derived(
4941 _Impl::_S_bit_and(__data(__x), __data(__y)));
4942 }
4943
4944 _GLIBCXX_SIMD_CONSTEXPR friend _V operator|(const _V& __x, const _V& __y)
4945 {
4946 return _SimdIntOperators::_S_make_derived(
4947 _Impl::_S_bit_or(__data(__x), __data(__y)));
4948 }
4949
4950 _GLIBCXX_SIMD_CONSTEXPR friend _V operator^(const _V& __x, const _V& __y)
4951 {
4952 return _SimdIntOperators::_S_make_derived(
4953 _Impl::_S_bit_xor(__data(__x), __data(__y)));
4954 }
4955
4956 _GLIBCXX_SIMD_CONSTEXPR friend _V operator<<(const _V& __x, const _V& __y)
4957 {
4958 return _SimdIntOperators::_S_make_derived(
4959 _Impl::_S_bit_shift_left(__data(__x), __data(__y)));
4960 }
4961
4962 _GLIBCXX_SIMD_CONSTEXPR friend _V operator>>(const _V& __x, const _V& __y)
4963 {
4964 return _SimdIntOperators::_S_make_derived(
4965 _Impl::_S_bit_shift_right(__data(__x), __data(__y)));
4966 }
4967
4968 template <typename _VV = _V>
4969 _GLIBCXX_SIMD_CONSTEXPR friend _V operator<<(const _V& __x, int __y)
4970 {
4971 using _Tp = typename _VV::value_type;
4972 if (__y < 0)
4973 __invoke_ub("The behavior is undefined if the right operand of a "
4974 "shift operation is negative. [expr.shift]\nA shift by "
4975 "%d was requested",
4976 __y);
4977 if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
4978 __invoke_ub(
4979 "The behavior is undefined if the right operand of a "
4980 "shift operation is greater than or equal to the width of the "
4981 "promoted left operand. [expr.shift]\nA shift by %d was requested",
4982 __y);
4983 return _SimdIntOperators::_S_make_derived(
4984 _Impl::_S_bit_shift_left(__data(__x), __y));
4985 }
4986
4987 template <typename _VV = _V>
4988 _GLIBCXX_SIMD_CONSTEXPR friend _V operator>>(const _V& __x, int __y)
4989 {
4990 using _Tp = typename _VV::value_type;
4991 if (__y < 0)
4992 __invoke_ub(
4993 "The behavior is undefined if the right operand of a shift "
4994 "operation is negative. [expr.shift]\nA shift by %d was requested",
4995 __y);
4996 if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
4997 __invoke_ub(
4998 "The behavior is undefined if the right operand of a shift "
4999 "operation is greater than or equal to the width of the promoted "
5000 "left operand. [expr.shift]\nA shift by %d was requested",
5001 __y);
5002 return _SimdIntOperators::_S_make_derived(
5003 _Impl::_S_bit_shift_right(__data(__x), __y));
5004 }
5005
5006 // unary operators (for integral _Tp)
5007 _GLIBCXX_SIMD_CONSTEXPR _V operator~() const
5008 { return {__private_init, _Impl::_S_complement(__derived()._M_data)}; }
5009 };
5010
5011//}}}1
5012/// @endcond
5013
5014// simd {{{
5015template <typename _Tp, typename _Abi>
5016 class simd : public _SimdIntOperators<
5017 simd<_Tp, _Abi>, typename _SimdTraits<_Tp, _Abi>::_SimdImpl,
5018 conjunction<is_integral<_Tp>,
5019 typename _SimdTraits<_Tp, _Abi>::_IsValid>::value>,
5020 public _SimdTraits<_Tp, _Abi>::_SimdBase
5021 {
5022 using _Traits = _SimdTraits<_Tp, _Abi>;
5023 using _MemberType = typename _Traits::_SimdMember;
5024 using _CastType = typename _Traits::_SimdCastType;
5025 static constexpr _Tp* _S_type_tag = nullptr;
5026 friend typename _Traits::_SimdBase;
5027
5028 public:
5029 using _Impl = typename _Traits::_SimdImpl;
5030 friend _Impl;
5031 friend _SimdIntOperators<simd, _Impl, true>;
5032
5033 using value_type = _Tp;
5034 using reference = _SmartReference<_MemberType, _Impl, value_type>;
5035 using mask_type = simd_mask<_Tp, _Abi>;
5036 using abi_type = _Abi;
5037
5038 static constexpr size_t size()
5039 { return __size_or_zero_v<_Tp, _Abi>; }
5040
5041 _GLIBCXX_SIMD_CONSTEXPR simd() = default;
5042 _GLIBCXX_SIMD_CONSTEXPR simd(const simd&) = default;
5043 _GLIBCXX_SIMD_CONSTEXPR simd(simd&&) noexcept = default;
5044 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(const simd&) = default;
5045 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(simd&&) noexcept = default;
5046
5047 // implicit broadcast constructor
5048 template <typename _Up,
5049 typename = enable_if_t<!is_same_v<__remove_cvref_t<_Up>, bool>>>
5050 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5051 simd(_ValuePreservingOrInt<_Up, value_type>&& __x)
5052 : _M_data(
5053 _Impl::_S_broadcast(static_cast<value_type>(static_cast<_Up&&>(__x))))
5054 {}
5055
5056 // implicit type conversion constructor (convert from fixed_size to
5057 // fixed_size)
5058 template <typename _Up>
5059 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5060 simd(const simd<_Up, simd_abi::fixed_size<size()>>& __x,
5061 enable_if_t<
5062 conjunction<
5063 is_same<simd_abi::fixed_size<size()>, abi_type>,
5064 negation<__is_narrowing_conversion<_Up, value_type>>,
5065 __converts_to_higher_integer_rank<_Up, value_type>>::value,
5066 void*> = nullptr)
5067 : simd{static_cast<array<_Up, size()>>(__x).data(), vector_aligned} {}
5068
5069 // explicit type conversion constructor
5070#ifdef _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5071 template <typename _Up, typename _A2,
5072 typename = decltype(static_simd_cast<simd>(
5073 declval<const simd<_Up, _A2>&>()))>
5074 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5075 simd(const simd<_Up, _A2>& __x)
5076 : simd(static_simd_cast<simd>(__x)) {}
5077#endif // _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5078
5079 // generator constructor
5080 template <typename _Fp>
5081 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5082 simd(_Fp&& __gen, _ValuePreservingOrInt<decltype(declval<_Fp>()(
5083 declval<_SizeConstant<0>&>())),
5084 value_type>* = nullptr)
5085 : _M_data(_Impl::_S_generator(static_cast<_Fp&&>(__gen), _S_type_tag)) {}
5086
5087 // load constructor
5088 template <typename _Up, typename _Flags>
5089 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5090 simd(const _Up* __mem, _IsSimdFlagType<_Flags>)
5091 : _M_data(
5092 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag))
5093 {}
5094
5095 // loads [simd.load]
5096 template <typename _Up, typename _Flags>
5097 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void
5098 copy_from(const _Vectorizable<_Up>* __mem, _IsSimdFlagType<_Flags>)
5099 {
5100 _M_data = static_cast<decltype(_M_data)>(
5101 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag));
5102 }
5103
5104 // stores [simd.store]
5105 template <typename _Up, typename _Flags>
5106 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void
5107 copy_to(_Vectorizable<_Up>* __mem, _IsSimdFlagType<_Flags>) const
5108 {
5109 _Impl::_S_store(_M_data, _Flags::template _S_apply<simd>(__mem),
5110 _S_type_tag);
5111 }
5112
5113 // scalar access
5114 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
5115 operator[](size_t __i)
5116 { return {_M_data, int(__i)}; }
5117
5118 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
5119 operator[]([[maybe_unused]] size_t __i) const
5120 {
5121 if constexpr (__is_scalar_abi<_Abi>())
5122 {
5123 _GLIBCXX_DEBUG_ASSERT(__i == 0);
5124 return _M_data;
5125 }
5126 else
5127 return _M_data[__i];
5128 }
5129
5130 // increment and decrement:
5131 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5132 operator++()
5133 {
5134 _Impl::_S_increment(_M_data);
5135 return *this;
5136 }
5137
5138 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5139 operator++(int)
5140 {
5141 simd __r = *this;
5142 _Impl::_S_increment(_M_data);
5143 return __r;
5144 }
5145
5146 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5147 operator--()
5148 {
5149 _Impl::_S_decrement(_M_data);
5150 return *this;
5151 }
5152
5153 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5154 operator--(int)
5155 {
5156 simd __r = *this;
5157 _Impl::_S_decrement(_M_data);
5158 return __r;
5159 }
5160
5161 // unary operators (for any _Tp)
5162 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR mask_type
5163 operator!() const
5164 { return {__private_init, _Impl::_S_negate(_M_data)}; }
5165
5166 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5167 operator+() const
5168 { return *this; }
5169
5170 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5171 operator-() const
5172 { return {__private_init, _Impl::_S_unary_minus(_M_data)}; }
5173
5174 // access to internal representation (suggested extension)
5175 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5176 simd(_CastType __init) : _M_data(__init) {}
5177
5178 // compound assignment [simd.cassign]
5179 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5180 operator+=(simd& __lhs, const simd& __x)
5181 { return __lhs = __lhs + __x; }
5182
5183 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5184 operator-=(simd& __lhs, const simd& __x)
5185 { return __lhs = __lhs - __x; }
5186
5187 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5188 operator*=(simd& __lhs, const simd& __x)
5189 { return __lhs = __lhs * __x; }
5190
5191 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5192 operator/=(simd& __lhs, const simd& __x)
5193 { return __lhs = __lhs / __x; }
5194
5195 // binary operators [simd.binary]
5196 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5197 operator+(const simd& __x, const simd& __y)
5198 { return {__private_init, _Impl::_S_plus(__x._M_data, __y._M_data)}; }
5199
5200 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5201 operator-(const simd& __x, const simd& __y)
5202 { return {__private_init, _Impl::_S_minus(__x._M_data, __y._M_data)}; }
5203
5204 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5205 operator*(const simd& __x, const simd& __y)
5206 { return {__private_init, _Impl::_S_multiplies(__x._M_data, __y._M_data)}; }
5207
5208 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5209 operator/(const simd& __x, const simd& __y)
5210 { return {__private_init, _Impl::_S_divides(__x._M_data, __y._M_data)}; }
5211
5212 // compares [simd.comparison]
5213 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5214 operator==(const simd& __x, const simd& __y)
5215 { return simd::_S_make_mask(_Impl::_S_equal_to(__x._M_data, __y._M_data)); }
5216
5217 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5218 operator!=(const simd& __x, const simd& __y)
5219 {
5220 return simd::_S_make_mask(
5221 _Impl::_S_not_equal_to(__x._M_data, __y._M_data));
5222 }
5223
5224 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5225 operator<(const simd& __x, const simd& __y)
5226 { return simd::_S_make_mask(_Impl::_S_less(__x._M_data, __y._M_data)); }
5227
5228 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5229 operator<=(const simd& __x, const simd& __y)
5230 {
5231 return simd::_S_make_mask(_Impl::_S_less_equal(__x._M_data, __y._M_data));
5232 }
5233
5234 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5235 operator>(const simd& __x, const simd& __y)
5236 { return simd::_S_make_mask(_Impl::_S_less(__y._M_data, __x._M_data)); }
5237
5238 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5239 operator>=(const simd& __x, const simd& __y)
5240 {
5241 return simd::_S_make_mask(_Impl::_S_less_equal(__y._M_data, __x._M_data));
5242 }
5243
5244 // operator?: overloads (suggested extension) {{{
5245#ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
5246 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5247 operator?:(const mask_type& __k, const simd& __where_true,
5248 const simd& __where_false)
5249 {
5250 auto __ret = __where_false;
5251 _Impl::_S_masked_assign(__data(__k), __data(__ret), __data(__where_true));
5252 return __ret;
5253 }
5254
5255#endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__
5256 // }}}
5257
5258 // "private" because of the first arguments's namespace
5259 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5260 simd(_PrivateInit, const _MemberType& __init)
5261 : _M_data(__init) {}
5262
5263 // "private" because of the first arguments's namespace
5264 _GLIBCXX_SIMD_INTRINSIC
5265 simd(_BitsetInit, bitset<size()> __init) : _M_data()
5266 { where(mask_type(__bitset_init, __init), *this) = ~*this; }
5267
5268 _GLIBCXX_SIMD_INTRINSIC constexpr bool
5269 _M_is_constprop() const
5270 {
5271 if constexpr (__is_scalar_abi<_Abi>())
5272 return __builtin_constant_p(_M_data);
5273 else
5274 return _M_data._M_is_constprop();
5275 }
5276
5277 private:
5278 _GLIBCXX_SIMD_INTRINSIC static constexpr mask_type
5279 _S_make_mask(typename mask_type::_MemberType __k)
5280 { return {__private_init, __k}; }
5281
5282 friend const auto& __data<value_type, abi_type>(const simd&);
5283 friend auto& __data<value_type, abi_type>(simd&);
5284 alignas(_Traits::_S_simd_align) _MemberType _M_data;
5285 };
5286
5287// }}}
5288/// @cond undocumented
5289// __data {{{
5290template <typename _Tp, typename _Ap>
5291 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
5292 __data(const simd<_Tp, _Ap>& __x)
5293 { return __x._M_data; }
5294
5295template <typename _Tp, typename _Ap>
5296 _GLIBCXX_SIMD_INTRINSIC constexpr auto&
5297 __data(simd<_Tp, _Ap>& __x)
5298 { return __x._M_data; }
5299
5300// }}}
5301namespace __float_bitwise_operators { //{{{
5302template <typename _Tp, typename _Ap>
5303 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5304 operator^(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5305 { return {__private_init, _Ap::_SimdImpl::_S_bit_xor(__data(__a), __data(__b))}; }
5306
5307template <typename _Tp, typename _Ap>
5308 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5309 operator|(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5310 { return {__private_init, _Ap::_SimdImpl::_S_bit_or(__data(__a), __data(__b))}; }
5311
5312template <typename _Tp, typename _Ap>
5313 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5314 operator&(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5315 { return {__private_init, _Ap::_SimdImpl::_S_bit_and(__data(__a), __data(__b))}; }
5316} // namespace __float_bitwise_operators }}}
5317/// @endcond
5318
5319/// @}
5320_GLIBCXX_SIMD_END_NAMESPACE
5321
5322#endif // __cplusplus >= 201703L
5323#endif // _GLIBCXX_EXPERIMENTAL_SIMD_H
5324
5325// vim: foldmethod=marker foldmarker={{{,}}}
constexpr duration< __common_rep_t< _Rep2, _Rep1 >, _Period > operator*(const _Rep1 &__s, const duration< _Rep2, _Period > &__d)
Definition chrono:700
constexpr duration< __common_rep_t< _Rep1, __disable_if_is_duration< _Rep2 > >, _Period > operator%(const duration< _Rep1, _Period > &__d, const _Rep2 &__s)
Definition chrono:729
constexpr time_point< _Clock, typename common_type< duration< _Rep1, _Period1 >, _Dur2 >::type > operator+(const duration< _Rep1, _Period1 > &__lhs, const time_point< _Clock, _Dur2 > &__rhs)
Adjust a time point forwards by the given duration.
Definition chrono:1016
constexpr common_type< duration< _Rep1, _Period1 >, duration< _Rep2, _Period2 > >::type operator-(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
The difference between two durations.
Definition chrono:660
constexpr duration< __common_rep_t< _Rep1, __disable_if_is_duration< _Rep2 > >, _Period > operator/(const duration< _Rep1, _Period > &__d, const _Rep2 &__s)
Definition chrono:706
typename remove_reference< _Tp >::type remove_reference_t
Alias template for remove_reference.
Definition type_traits:1639
typename make_unsigned< _Tp >::type make_unsigned_t
Alias template for make_unsigned.
Definition type_traits:1974
void void_t
A metafunction that always yields void, used for detecting valid types.
Definition type_traits:2601
integral_constant< bool, true > true_type
The type used as a compile-time boolean with true value.
Definition type_traits:83
typename conditional< _Cond, _Iftrue, _Iffalse >::type conditional_t
Alias template for conditional.
Definition type_traits:2583
integral_constant< bool, false > false_type
The type used as a compile-time boolean with false value.
Definition type_traits:86
typename enable_if< _Cond, _Tp >::type enable_if_t
Alias template for enable_if.
Definition type_traits:2579
constexpr auto tuple_cat(_Tpls &&... __tpls) -> typename __tuple_cat_result< _Tpls... >::__type
tuple_cat
Definition tuple:1746
auto declval() noexcept -> decltype(__declval< _Tp >(0))
Definition type_traits:2358
constexpr std::remove_reference< _Tp >::type && move(_Tp &&__t) noexcept
Convert a value to an rvalue.
Definition move.h:104
void swap(any &__x, any &__y) noexcept
Exchange the states of two any objects.
Definition any:428
_Tp * end(valarray< _Tp > &__va) noexcept
Return an iterator pointing to one past the last element of the valarray.
Definition valarray:1239
_Tp * begin(valarray< _Tp > &__va) noexcept
Return an iterator pointing to the first element of the valarray.
Definition valarray:1217
constexpr const _Tp & clamp(const _Tp &, const _Tp &, const _Tp &)
Returns the value clamped between lo and hi.
Definition stl_algo.h:3656
constexpr const _Tp & max(const _Tp &, const _Tp &)
This does what you think it does.
constexpr pair< const _Tp &, const _Tp & > minmax(const _Tp &, const _Tp &)
Determines min and max at once as an ordered pair.
Definition stl_algo.h:3301
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op)
Calculate reduction of values in a range.
Definition numeric:289
bitset< _Nb > operator&(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition bitset:1435
constexpr auto size(const _Container &__cont) noexcept(noexcept(__cont.size())) -> decltype(__cont.size())
Return the size of a container.
bitset< _Nb > operator|(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition bitset:1444
std::basic_istream< _CharT, _Traits > & operator>>(std::basic_istream< _CharT, _Traits > &__is, bitset< _Nb > &__x)
Global I/O operators for bitsets.
Definition bitset:1472
std::basic_ostream< _CharT, _Traits > & operator<<(std::basic_ostream< _CharT, _Traits > &__os, const bitset< _Nb > &__x)
Global I/O operators for bitsets.
Definition bitset:1540
constexpr auto data(_Container &__cont) noexcept(noexcept(__cont.data())) -> decltype(__cont.data())
Return the data pointer of a container.
bitset< _Nb > operator^(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition bitset:1453