libstdc++
simd.h
1 // Definition of the public simd interfaces -*- C++ -*-
2 
3 // Copyright (C) 2020-2021 Free Software Foundation, Inc.
4 //
5 // This file is part of the GNU ISO C++ Library. This library is free
6 // software; you can redistribute it and/or modify it under the
7 // terms of the GNU General Public License as published by the
8 // Free Software Foundation; either version 3, or (at your option)
9 // any later version.
10 
11 // This library is distributed in the hope that it will be useful,
12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 // GNU General Public License for more details.
15 
16 // Under Section 7 of GPL version 3, you are granted additional
17 // permissions described in the GCC Runtime Library Exception, version
18 // 3.1, as published by the Free Software Foundation.
19 
20 // You should have received a copy of the GNU General Public License and
21 // a copy of the GCC Runtime Library Exception along with this program;
22 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23 // <http://www.gnu.org/licenses/>.
24 
25 #ifndef _GLIBCXX_EXPERIMENTAL_SIMD_H
26 #define _GLIBCXX_EXPERIMENTAL_SIMD_H
27 
28 #if __cplusplus >= 201703L
29 
30 #include "simd_detail.h"
31 #include "numeric_traits.h"
32 #include <bit>
33 #include <bitset>
34 #ifdef _GLIBCXX_DEBUG_UB
35 #include <cstdio> // for stderr
36 #endif
37 #include <cstring>
38 #include <functional>
39 #include <iosfwd>
40 #include <utility>
41 
42 #if _GLIBCXX_SIMD_X86INTRIN
43 #include <x86intrin.h>
44 #elif _GLIBCXX_SIMD_HAVE_NEON
45 #pragma GCC diagnostic push
46 // narrowing conversion of '__a' from 'uint64_t' {aka 'long long unsigned int'} to
47 // 'int64x1_t' {aka 'long long int'} [-Wnarrowing]
48 #pragma GCC diagnostic ignored "-Wnarrowing"
49 #include <arm_neon.h>
50 #pragma GCC diagnostic pop
51 #endif
52 
53 /** @ingroup ts_simd
54  * @{
55  */
56 /* There are several closely related types, with the following naming
57  * convention:
58  * _Tp: vectorizable (arithmetic) type (or any type)
59  * _TV: __vector_type_t<_Tp, _Np>
60  * _TW: _SimdWrapper<_Tp, _Np>
61  * _TI: __intrinsic_type_t<_Tp, _Np>
62  * _TVT: _VectorTraits<_TV> or _VectorTraits<_TW>
63  * If one additional type is needed use _U instead of _T.
64  * Otherwise use _T\d, _TV\d, _TW\d, TI\d, _TVT\d.
65  *
66  * More naming conventions:
67  * _Ap or _Abi: An ABI tag from the simd_abi namespace
68  * _Ip: often used for integer types with sizeof(_Ip) == sizeof(_Tp),
69  * _IV, _IW as for _TV, _TW
70  * _Np: number of elements (not bytes)
71  * _Bytes: number of bytes
72  *
73  * Variable names:
74  * __k: mask object (vector- or bitmask)
75  */
76 _GLIBCXX_SIMD_BEGIN_NAMESPACE
77 
78 #if !_GLIBCXX_SIMD_X86INTRIN
79 using __m128 [[__gnu__::__vector_size__(16)]] = float;
80 using __m128d [[__gnu__::__vector_size__(16)]] = double;
81 using __m128i [[__gnu__::__vector_size__(16)]] = long long;
82 using __m256 [[__gnu__::__vector_size__(32)]] = float;
83 using __m256d [[__gnu__::__vector_size__(32)]] = double;
84 using __m256i [[__gnu__::__vector_size__(32)]] = long long;
85 using __m512 [[__gnu__::__vector_size__(64)]] = float;
86 using __m512d [[__gnu__::__vector_size__(64)]] = double;
87 using __m512i [[__gnu__::__vector_size__(64)]] = long long;
88 #endif
89 
90 namespace simd_abi {
91 // simd_abi forward declarations {{{
92 // implementation details:
93 struct _Scalar;
94 
95 template <int _Np>
96  struct _Fixed;
97 
98 // There are two major ABIs that appear on different architectures.
99 // Both have non-boolean values packed into an N Byte register
100 // -> #elements = N / sizeof(T)
101 // Masks differ:
102 // 1. Use value vector registers for masks (all 0 or all 1)
103 // 2. Use bitmasks (mask registers) with one bit per value in the corresponding
104 // value vector
105 //
106 // Both can be partially used, masking off the rest when doing horizontal
107 // operations or operations that can trap (e.g. FP_INVALID or integer division
108 // by 0). This is encoded as the number of used bytes.
109 template <int _UsedBytes>
110  struct _VecBuiltin;
111 
112 template <int _UsedBytes>
113  struct _VecBltnBtmsk;
114 
115 template <typename _Tp, int _Np>
116  using _VecN = _VecBuiltin<sizeof(_Tp) * _Np>;
117 
118 template <int _UsedBytes = 16>
119  using _Sse = _VecBuiltin<_UsedBytes>;
120 
121 template <int _UsedBytes = 32>
122  using _Avx = _VecBuiltin<_UsedBytes>;
123 
124 template <int _UsedBytes = 64>
125  using _Avx512 = _VecBltnBtmsk<_UsedBytes>;
126 
127 template <int _UsedBytes = 16>
128  using _Neon = _VecBuiltin<_UsedBytes>;
129 
130 // implementation-defined:
131 using __sse = _Sse<>;
132 using __avx = _Avx<>;
133 using __avx512 = _Avx512<>;
134 using __neon = _Neon<>;
135 using __neon128 = _Neon<16>;
136 using __neon64 = _Neon<8>;
137 
138 // standard:
139 template <typename _Tp, size_t _Np, typename...>
140  struct deduce;
141 
142 template <int _Np>
143  using fixed_size = _Fixed<_Np>;
144 
145 using scalar = _Scalar;
146 
147 // }}}
148 } // namespace simd_abi
149 // forward declarations is_simd(_mask), simd(_mask), simd_size {{{
150 template <typename _Tp>
151  struct is_simd;
152 
153 template <typename _Tp>
154  struct is_simd_mask;
155 
156 template <typename _Tp, typename _Abi>
157  class simd;
158 
159 template <typename _Tp, typename _Abi>
160  class simd_mask;
161 
162 template <typename _Tp, typename _Abi>
163  struct simd_size;
164 
165 // }}}
166 // load/store flags {{{
167 struct element_aligned_tag
168 {
169  template <typename _Tp, typename _Up = typename _Tp::value_type>
170  static constexpr size_t _S_alignment = alignof(_Up);
171 
172  template <typename _Tp, typename _Up>
173  _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
174  _S_apply(_Up* __ptr)
175  { return __ptr; }
176 };
177 
178 struct vector_aligned_tag
179 {
180  template <typename _Tp, typename _Up = typename _Tp::value_type>
181  static constexpr size_t _S_alignment
182  = std::__bit_ceil(sizeof(_Up) * _Tp::size());
183 
184  template <typename _Tp, typename _Up>
185  _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
186  _S_apply(_Up* __ptr)
187  { return static_cast<_Up*>(__builtin_assume_aligned(__ptr, _S_alignment<_Tp, _Up>)); }
188 };
189 
190 template <size_t _Np> struct overaligned_tag
191 {
192  template <typename _Tp, typename _Up = typename _Tp::value_type>
193  static constexpr size_t _S_alignment = _Np;
194 
195  template <typename _Tp, typename _Up>
196  _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
197  _S_apply(_Up* __ptr)
198  { return static_cast<_Up*>(__builtin_assume_aligned(__ptr, _Np)); }
199 };
200 
201 inline constexpr element_aligned_tag element_aligned = {};
202 
203 inline constexpr vector_aligned_tag vector_aligned = {};
204 
205 template <size_t _Np>
206  inline constexpr overaligned_tag<_Np> overaligned = {};
207 
208 // }}}
209 template <size_t _Xp>
210  using _SizeConstant = integral_constant<size_t, _Xp>;
211 
212 namespace __detail
213 {
214  struct _Minimum
215  {
216  template <typename _Tp>
217  _GLIBCXX_SIMD_INTRINSIC constexpr
218  _Tp
219  operator()(_Tp __a, _Tp __b) const
220  {
221  using std::min;
222  return min(__a, __b);
223  }
224  };
225 
226  struct _Maximum
227  {
228  template <typename _Tp>
229  _GLIBCXX_SIMD_INTRINSIC constexpr
230  _Tp
231  operator()(_Tp __a, _Tp __b) const
232  {
233  using std::max;
234  return max(__a, __b);
235  }
236  };
237 } // namespace __detail
238 
239 // unrolled/pack execution helpers
240 // __execute_n_times{{{
241 template <typename _Fp, size_t... _I>
242  _GLIBCXX_SIMD_INTRINSIC constexpr void
243  __execute_on_index_sequence(_Fp&& __f, index_sequence<_I...>)
244  { ((void)__f(_SizeConstant<_I>()), ...); }
245 
246 template <typename _Fp>
247  _GLIBCXX_SIMD_INTRINSIC constexpr void
248  __execute_on_index_sequence(_Fp&&, index_sequence<>)
249  { }
250 
251 template <size_t _Np, typename _Fp>
252  _GLIBCXX_SIMD_INTRINSIC constexpr void
253  __execute_n_times(_Fp&& __f)
254  {
255  __execute_on_index_sequence(static_cast<_Fp&&>(__f),
256  make_index_sequence<_Np>{});
257  }
258 
259 // }}}
260 // __generate_from_n_evaluations{{{
261 template <typename _R, typename _Fp, size_t... _I>
262  _GLIBCXX_SIMD_INTRINSIC constexpr _R
263  __execute_on_index_sequence_with_return(_Fp&& __f, index_sequence<_I...>)
264  { return _R{__f(_SizeConstant<_I>())...}; }
265 
266 template <size_t _Np, typename _R, typename _Fp>
267  _GLIBCXX_SIMD_INTRINSIC constexpr _R
268  __generate_from_n_evaluations(_Fp&& __f)
269  {
270  return __execute_on_index_sequence_with_return<_R>(
271  static_cast<_Fp&&>(__f), make_index_sequence<_Np>{});
272  }
273 
274 // }}}
275 // __call_with_n_evaluations{{{
276 template <size_t... _I, typename _F0, typename _FArgs>
277  _GLIBCXX_SIMD_INTRINSIC constexpr auto
278  __call_with_n_evaluations(index_sequence<_I...>, _F0&& __f0, _FArgs&& __fargs)
279  { return __f0(__fargs(_SizeConstant<_I>())...); }
280 
281 template <size_t _Np, typename _F0, typename _FArgs>
282  _GLIBCXX_SIMD_INTRINSIC constexpr auto
283  __call_with_n_evaluations(_F0&& __f0, _FArgs&& __fargs)
284  {
285  return __call_with_n_evaluations(make_index_sequence<_Np>{},
286  static_cast<_F0&&>(__f0),
287  static_cast<_FArgs&&>(__fargs));
288  }
289 
290 // }}}
291 // __call_with_subscripts{{{
292 template <size_t _First = 0, size_t... _It, typename _Tp, typename _Fp>
293  _GLIBCXX_SIMD_INTRINSIC constexpr auto
294  __call_with_subscripts(_Tp&& __x, index_sequence<_It...>, _Fp&& __fun)
295  { return __fun(__x[_First + _It]...); }
296 
297 template <size_t _Np, size_t _First = 0, typename _Tp, typename _Fp>
298  _GLIBCXX_SIMD_INTRINSIC constexpr auto
299  __call_with_subscripts(_Tp&& __x, _Fp&& __fun)
300  {
301  return __call_with_subscripts<_First>(static_cast<_Tp&&>(__x),
302  make_index_sequence<_Np>(),
303  static_cast<_Fp&&>(__fun));
304  }
305 
306 // }}}
307 
308 // vvv ---- type traits ---- vvv
309 // integer type aliases{{{
310 using _UChar = unsigned char;
311 using _SChar = signed char;
312 using _UShort = unsigned short;
313 using _UInt = unsigned int;
314 using _ULong = unsigned long;
315 using _ULLong = unsigned long long;
316 using _LLong = long long;
317 
318 //}}}
319 // __first_of_pack{{{
320 template <typename _T0, typename...>
321  struct __first_of_pack
322  { using type = _T0; };
323 
324 template <typename... _Ts>
325  using __first_of_pack_t = typename __first_of_pack<_Ts...>::type;
326 
327 //}}}
328 // __value_type_or_identity_t {{{
329 template <typename _Tp>
330  typename _Tp::value_type
331  __value_type_or_identity_impl(int);
332 
333 template <typename _Tp>
334  _Tp
335  __value_type_or_identity_impl(float);
336 
337 template <typename _Tp>
338  using __value_type_or_identity_t
339  = decltype(__value_type_or_identity_impl<_Tp>(int()));
340 
341 // }}}
342 // __is_vectorizable {{{
343 template <typename _Tp>
344  struct __is_vectorizable : public is_arithmetic<_Tp> {};
345 
346 template <>
347  struct __is_vectorizable<bool> : public false_type {};
348 
349 template <typename _Tp>
350  inline constexpr bool __is_vectorizable_v = __is_vectorizable<_Tp>::value;
351 
352 // Deduces to a vectorizable type
353 template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
354  using _Vectorizable = _Tp;
355 
356 // }}}
357 // _LoadStorePtr / __is_possible_loadstore_conversion {{{
358 template <typename _Ptr, typename _ValueType>
359  struct __is_possible_loadstore_conversion
360  : conjunction<__is_vectorizable<_Ptr>, __is_vectorizable<_ValueType>> {};
361 
362 template <>
363  struct __is_possible_loadstore_conversion<bool, bool> : true_type {};
364 
365 // Deduces to a type allowed for load/store with the given value type.
366 template <typename _Ptr, typename _ValueType,
367  typename = enable_if_t<
368  __is_possible_loadstore_conversion<_Ptr, _ValueType>::value>>
369  using _LoadStorePtr = _Ptr;
370 
371 // }}}
372 // __is_bitmask{{{
373 template <typename _Tp, typename = void_t<>>
374  struct __is_bitmask : false_type {};
375 
376 template <typename _Tp>
377  inline constexpr bool __is_bitmask_v = __is_bitmask<_Tp>::value;
378 
379 // the __mmaskXX case:
380 template <typename _Tp>
381  struct __is_bitmask<_Tp,
382  void_t<decltype(declval<unsigned&>() = declval<_Tp>() & 1u)>>
383  : true_type {};
384 
385 // }}}
386 // __int_for_sizeof{{{
387 #pragma GCC diagnostic push
388 #pragma GCC diagnostic ignored "-Wpedantic"
389 template <size_t _Bytes>
390  constexpr auto
391  __int_for_sizeof()
392  {
393  static_assert(_Bytes > 0);
394  if constexpr (_Bytes == sizeof(int))
395  return int();
396  #ifdef __clang__
397  else if constexpr (_Bytes == sizeof(char))
398  return char();
399  #else
400  else if constexpr (_Bytes == sizeof(_SChar))
401  return _SChar();
402  #endif
403  else if constexpr (_Bytes == sizeof(short))
404  return short();
405  #ifndef __clang__
406  else if constexpr (_Bytes == sizeof(long))
407  return long();
408  #endif
409  else if constexpr (_Bytes == sizeof(_LLong))
410  return _LLong();
411  #ifdef __SIZEOF_INT128__
412  else if constexpr (_Bytes == sizeof(__int128))
413  return __int128();
414  #endif // __SIZEOF_INT128__
415  else if constexpr (_Bytes % sizeof(int) == 0)
416  {
417  constexpr size_t _Np = _Bytes / sizeof(int);
418  struct _Ip
419  {
420  int _M_data[_Np];
421 
422  _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
423  operator&(_Ip __rhs) const
424  {
425  return __generate_from_n_evaluations<_Np, _Ip>(
426  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
427  return __rhs._M_data[__i] & _M_data[__i];
428  });
429  }
430 
431  _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
432  operator|(_Ip __rhs) const
433  {
434  return __generate_from_n_evaluations<_Np, _Ip>(
435  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
436  return __rhs._M_data[__i] | _M_data[__i];
437  });
438  }
439 
440  _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
441  operator^(_Ip __rhs) const
442  {
443  return __generate_from_n_evaluations<_Np, _Ip>(
444  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
445  return __rhs._M_data[__i] ^ _M_data[__i];
446  });
447  }
448 
449  _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
450  operator~() const
451  {
452  return __generate_from_n_evaluations<_Np, _Ip>(
453  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return ~_M_data[__i]; });
454  }
455  };
456  return _Ip{};
457  }
458  else
459  static_assert(_Bytes == 0, "this should be unreachable");
460  }
461 #pragma GCC diagnostic pop
462 
463 template <typename _Tp>
464  using __int_for_sizeof_t = decltype(__int_for_sizeof<sizeof(_Tp)>());
465 
466 template <size_t _Np>
467  using __int_with_sizeof_t = decltype(__int_for_sizeof<_Np>());
468 
469 // }}}
470 // __is_fixed_size_abi{{{
471 template <typename _Tp>
472  struct __is_fixed_size_abi : false_type {};
473 
474 template <int _Np>
475  struct __is_fixed_size_abi<simd_abi::fixed_size<_Np>> : true_type {};
476 
477 template <typename _Tp>
478  inline constexpr bool __is_fixed_size_abi_v = __is_fixed_size_abi<_Tp>::value;
479 
480 // }}}
481 // constexpr feature detection{{{
482 constexpr inline bool __have_mmx = _GLIBCXX_SIMD_HAVE_MMX;
483 constexpr inline bool __have_sse = _GLIBCXX_SIMD_HAVE_SSE;
484 constexpr inline bool __have_sse2 = _GLIBCXX_SIMD_HAVE_SSE2;
485 constexpr inline bool __have_sse3 = _GLIBCXX_SIMD_HAVE_SSE3;
486 constexpr inline bool __have_ssse3 = _GLIBCXX_SIMD_HAVE_SSSE3;
487 constexpr inline bool __have_sse4_1 = _GLIBCXX_SIMD_HAVE_SSE4_1;
488 constexpr inline bool __have_sse4_2 = _GLIBCXX_SIMD_HAVE_SSE4_2;
489 constexpr inline bool __have_xop = _GLIBCXX_SIMD_HAVE_XOP;
490 constexpr inline bool __have_avx = _GLIBCXX_SIMD_HAVE_AVX;
491 constexpr inline bool __have_avx2 = _GLIBCXX_SIMD_HAVE_AVX2;
492 constexpr inline bool __have_bmi = _GLIBCXX_SIMD_HAVE_BMI1;
493 constexpr inline bool __have_bmi2 = _GLIBCXX_SIMD_HAVE_BMI2;
494 constexpr inline bool __have_lzcnt = _GLIBCXX_SIMD_HAVE_LZCNT;
495 constexpr inline bool __have_sse4a = _GLIBCXX_SIMD_HAVE_SSE4A;
496 constexpr inline bool __have_fma = _GLIBCXX_SIMD_HAVE_FMA;
497 constexpr inline bool __have_fma4 = _GLIBCXX_SIMD_HAVE_FMA4;
498 constexpr inline bool __have_f16c = _GLIBCXX_SIMD_HAVE_F16C;
499 constexpr inline bool __have_popcnt = _GLIBCXX_SIMD_HAVE_POPCNT;
500 constexpr inline bool __have_avx512f = _GLIBCXX_SIMD_HAVE_AVX512F;
501 constexpr inline bool __have_avx512dq = _GLIBCXX_SIMD_HAVE_AVX512DQ;
502 constexpr inline bool __have_avx512vl = _GLIBCXX_SIMD_HAVE_AVX512VL;
503 constexpr inline bool __have_avx512bw = _GLIBCXX_SIMD_HAVE_AVX512BW;
504 constexpr inline bool __have_avx512dq_vl = __have_avx512dq && __have_avx512vl;
505 constexpr inline bool __have_avx512bw_vl = __have_avx512bw && __have_avx512vl;
506 
507 constexpr inline bool __have_neon = _GLIBCXX_SIMD_HAVE_NEON;
508 constexpr inline bool __have_neon_a32 = _GLIBCXX_SIMD_HAVE_NEON_A32;
509 constexpr inline bool __have_neon_a64 = _GLIBCXX_SIMD_HAVE_NEON_A64;
510 constexpr inline bool __support_neon_float =
511 #if defined __GCC_IEC_559
512  __GCC_IEC_559 == 0;
513 #elif defined __FAST_MATH__
514  true;
515 #else
516  false;
517 #endif
518 
519 #ifdef _ARCH_PWR10
520 constexpr inline bool __have_power10vec = true;
521 #else
522 constexpr inline bool __have_power10vec = false;
523 #endif
524 #ifdef __POWER9_VECTOR__
525 constexpr inline bool __have_power9vec = true;
526 #else
527 constexpr inline bool __have_power9vec = false;
528 #endif
529 #if defined __POWER8_VECTOR__
530 constexpr inline bool __have_power8vec = true;
531 #else
532 constexpr inline bool __have_power8vec = __have_power9vec;
533 #endif
534 #if defined __VSX__
535 constexpr inline bool __have_power_vsx = true;
536 #else
537 constexpr inline bool __have_power_vsx = __have_power8vec;
538 #endif
539 #if defined __ALTIVEC__
540 constexpr inline bool __have_power_vmx = true;
541 #else
542 constexpr inline bool __have_power_vmx = __have_power_vsx;
543 #endif
544 
545 // }}}
546 // __is_scalar_abi {{{
547 template <typename _Abi>
548  constexpr bool
549  __is_scalar_abi()
550  { return is_same_v<simd_abi::scalar, _Abi>; }
551 
552 // }}}
553 // __abi_bytes_v {{{
554 template <template <int> class _Abi, int _Bytes>
555  constexpr int
556  __abi_bytes_impl(_Abi<_Bytes>*)
557  { return _Bytes; }
558 
559 template <typename _Tp>
560  constexpr int
561  __abi_bytes_impl(_Tp*)
562  { return -1; }
563 
564 template <typename _Abi>
565  inline constexpr int __abi_bytes_v
566  = __abi_bytes_impl(static_cast<_Abi*>(nullptr));
567 
568 // }}}
569 // __is_builtin_bitmask_abi {{{
570 template <typename _Abi>
571  constexpr bool
572  __is_builtin_bitmask_abi()
573  { return is_same_v<simd_abi::_VecBltnBtmsk<__abi_bytes_v<_Abi>>, _Abi>; }
574 
575 // }}}
576 // __is_sse_abi {{{
577 template <typename _Abi>
578  constexpr bool
579  __is_sse_abi()
580  {
581  constexpr auto _Bytes = __abi_bytes_v<_Abi>;
582  return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
583  }
584 
585 // }}}
586 // __is_avx_abi {{{
587 template <typename _Abi>
588  constexpr bool
589  __is_avx_abi()
590  {
591  constexpr auto _Bytes = __abi_bytes_v<_Abi>;
592  return _Bytes > 16 && _Bytes <= 32
593  && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
594  }
595 
596 // }}}
597 // __is_avx512_abi {{{
598 template <typename _Abi>
599  constexpr bool
600  __is_avx512_abi()
601  {
602  constexpr auto _Bytes = __abi_bytes_v<_Abi>;
603  return _Bytes <= 64 && is_same_v<simd_abi::_Avx512<_Bytes>, _Abi>;
604  }
605 
606 // }}}
607 // __is_neon_abi {{{
608 template <typename _Abi>
609  constexpr bool
610  __is_neon_abi()
611  {
612  constexpr auto _Bytes = __abi_bytes_v<_Abi>;
613  return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
614  }
615 
616 // }}}
617 // __make_dependent_t {{{
618 template <typename, typename _Up>
619  struct __make_dependent
620  { using type = _Up; };
621 
622 template <typename _Tp, typename _Up>
623  using __make_dependent_t = typename __make_dependent<_Tp, _Up>::type;
624 
625 // }}}
626 // ^^^ ---- type traits ---- ^^^
627 
628 // __invoke_ub{{{
629 template <typename... _Args>
630  [[noreturn]] _GLIBCXX_SIMD_ALWAYS_INLINE void
631  __invoke_ub([[maybe_unused]] const char* __msg, [[maybe_unused]] const _Args&... __args)
632  {
633 #ifdef _GLIBCXX_DEBUG_UB
634  __builtin_fprintf(stderr, __msg, __args...);
635  __builtin_trap();
636 #else
637  __builtin_unreachable();
638 #endif
639  }
640 
641 // }}}
642 // __assert_unreachable{{{
643 template <typename _Tp>
644  struct __assert_unreachable
645  { static_assert(!is_same_v<_Tp, _Tp>, "this should be unreachable"); };
646 
647 // }}}
648 // __size_or_zero_v {{{
649 template <typename _Tp, typename _Ap, size_t _Np = simd_size<_Tp, _Ap>::value>
650  constexpr size_t
651  __size_or_zero_dispatch(int)
652  { return _Np; }
653 
654 template <typename _Tp, typename _Ap>
655  constexpr size_t
656  __size_or_zero_dispatch(float)
657  { return 0; }
658 
659 template <typename _Tp, typename _Ap>
660  inline constexpr size_t __size_or_zero_v
661  = __size_or_zero_dispatch<_Tp, _Ap>(0);
662 
663 // }}}
664 // __div_roundup {{{
665 inline constexpr size_t
666 __div_roundup(size_t __a, size_t __b)
667 { return (__a + __b - 1) / __b; }
668 
669 // }}}
670 // _ExactBool{{{
671 class _ExactBool
672 {
673  const bool _M_data;
674 
675 public:
676  _GLIBCXX_SIMD_INTRINSIC constexpr
677  _ExactBool(bool __b) : _M_data(__b) {}
678 
679  _ExactBool(int) = delete;
680 
681  _GLIBCXX_SIMD_INTRINSIC constexpr
682  operator bool() const
683  { return _M_data; }
684 };
685 
686 // }}}
687 // __may_alias{{{
688 /**@internal
689  * Helper __may_alias<_Tp> that turns _Tp into the type to be used for an
690  * aliasing pointer. This adds the __may_alias attribute to _Tp (with compilers
691  * that support it).
692  */
693 template <typename _Tp>
694  using __may_alias [[__gnu__::__may_alias__]] = _Tp;
695 
696 // }}}
697 // _UnsupportedBase {{{
698 // simd and simd_mask base for unsupported <_Tp, _Abi>
699 struct _UnsupportedBase
700 {
701  _UnsupportedBase() = delete;
702  _UnsupportedBase(const _UnsupportedBase&) = delete;
703  _UnsupportedBase& operator=(const _UnsupportedBase&) = delete;
704  ~_UnsupportedBase() = delete;
705 };
706 
707 // }}}
708 // _InvalidTraits {{{
709 /**
710  * @internal
711  * Defines the implementation of __a given <_Tp, _Abi>.
712  *
713  * Implementations must ensure that only valid <_Tp, _Abi> instantiations are
714  * possible. Static assertions in the type definition do not suffice. It is
715  * important that SFINAE works.
716  */
717 struct _InvalidTraits
718 {
719  using _IsValid = false_type;
720  using _SimdBase = _UnsupportedBase;
721  using _MaskBase = _UnsupportedBase;
722 
723  static constexpr size_t _S_full_size = 0;
724  static constexpr bool _S_is_partial = false;
725 
726  static constexpr size_t _S_simd_align = 1;
727  struct _SimdImpl;
728  struct _SimdMember {};
729  struct _SimdCastType;
730 
731  static constexpr size_t _S_mask_align = 1;
732  struct _MaskImpl;
733  struct _MaskMember {};
734  struct _MaskCastType;
735 };
736 
737 // }}}
738 // _SimdTraits {{{
739 template <typename _Tp, typename _Abi, typename = void_t<>>
740  struct _SimdTraits : _InvalidTraits {};
741 
742 // }}}
743 // __private_init, __bitset_init{{{
744 /**
745  * @internal
746  * Tag used for private init constructor of simd and simd_mask
747  */
748 inline constexpr struct _PrivateInit {} __private_init = {};
749 
750 inline constexpr struct _BitsetInit {} __bitset_init = {};
751 
752 // }}}
753 // __is_narrowing_conversion<_From, _To>{{{
754 template <typename _From, typename _To, bool = is_arithmetic_v<_From>,
755  bool = is_arithmetic_v<_To>>
756  struct __is_narrowing_conversion;
757 
758 // ignore "signed/unsigned mismatch" in the following trait.
759 // The implicit conversions will do the right thing here.
760 template <typename _From, typename _To>
761  struct __is_narrowing_conversion<_From, _To, true, true>
762  : public __bool_constant<(
763  __digits_v<_From> > __digits_v<_To>
764  || __finite_max_v<_From> > __finite_max_v<_To>
765  || __finite_min_v<_From> < __finite_min_v<_To>
766  || (is_signed_v<_From> && is_unsigned_v<_To>))> {};
767 
768 template <typename _Tp>
769  struct __is_narrowing_conversion<_Tp, bool, true, true>
770  : public true_type {};
771 
772 template <>
773  struct __is_narrowing_conversion<bool, bool, true, true>
774  : public false_type {};
775 
776 template <typename _Tp>
777  struct __is_narrowing_conversion<_Tp, _Tp, true, true>
778  : public false_type {};
779 
780 template <typename _From, typename _To>
781  struct __is_narrowing_conversion<_From, _To, false, true>
782  : public negation<is_convertible<_From, _To>> {};
783 
784 // }}}
785 // __converts_to_higher_integer_rank{{{
786 template <typename _From, typename _To, bool = (sizeof(_From) < sizeof(_To))>
787  struct __converts_to_higher_integer_rank : public true_type {};
788 
789 // this may fail for char -> short if sizeof(char) == sizeof(short)
790 template <typename _From, typename _To>
791  struct __converts_to_higher_integer_rank<_From, _To, false>
792  : public is_same<decltype(declval<_From>() + declval<_To>()), _To> {};
793 
794 // }}}
795 // __data(simd/simd_mask) {{{
796 template <typename _Tp, typename _Ap>
797  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
798  __data(const simd<_Tp, _Ap>& __x);
799 
800 template <typename _Tp, typename _Ap>
801  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
802  __data(simd<_Tp, _Ap>& __x);
803 
804 template <typename _Tp, typename _Ap>
805  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
806  __data(const simd_mask<_Tp, _Ap>& __x);
807 
808 template <typename _Tp, typename _Ap>
809  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
810  __data(simd_mask<_Tp, _Ap>& __x);
811 
812 // }}}
813 // _SimdConverter {{{
814 template <typename _FromT, typename _FromA, typename _ToT, typename _ToA,
815  typename = void>
816  struct _SimdConverter;
817 
818 template <typename _Tp, typename _Ap>
819  struct _SimdConverter<_Tp, _Ap, _Tp, _Ap, void>
820  {
821  template <typename _Up>
822  _GLIBCXX_SIMD_INTRINSIC const _Up&
823  operator()(const _Up& __x)
824  { return __x; }
825  };
826 
827 // }}}
828 // __to_value_type_or_member_type {{{
829 template <typename _V>
830  _GLIBCXX_SIMD_INTRINSIC constexpr auto
831  __to_value_type_or_member_type(const _V& __x) -> decltype(__data(__x))
832  { return __data(__x); }
833 
834 template <typename _V>
835  _GLIBCXX_SIMD_INTRINSIC constexpr const typename _V::value_type&
836  __to_value_type_or_member_type(const typename _V::value_type& __x)
837  { return __x; }
838 
839 // }}}
840 // __bool_storage_member_type{{{
841 template <size_t _Size>
842  struct __bool_storage_member_type;
843 
844 template <size_t _Size>
845  using __bool_storage_member_type_t =
846  typename __bool_storage_member_type<_Size>::type;
847 
848 // }}}
849 // _SimdTuple {{{
850 // why not tuple?
851 // 1. tuple gives no guarantee about the storage order, but I require
852 // storage
853 // equivalent to array<_Tp, _Np>
854 // 2. direct access to the element type (first template argument)
855 // 3. enforces equal element type, only different _Abi types are allowed
856 template <typename _Tp, typename... _Abis>
857  struct _SimdTuple;
858 
859 //}}}
860 // __fixed_size_storage_t {{{
861 template <typename _Tp, int _Np>
862  struct __fixed_size_storage;
863 
864 template <typename _Tp, int _Np>
865  using __fixed_size_storage_t = typename __fixed_size_storage<_Tp, _Np>::type;
866 
867 // }}}
868 // _SimdWrapper fwd decl{{{
869 template <typename _Tp, size_t _Size, typename = void_t<>>
870  struct _SimdWrapper;
871 
872 template <typename _Tp>
873  using _SimdWrapper8 = _SimdWrapper<_Tp, 8 / sizeof(_Tp)>;
874 template <typename _Tp>
875  using _SimdWrapper16 = _SimdWrapper<_Tp, 16 / sizeof(_Tp)>;
876 template <typename _Tp>
877  using _SimdWrapper32 = _SimdWrapper<_Tp, 32 / sizeof(_Tp)>;
878 template <typename _Tp>
879  using _SimdWrapper64 = _SimdWrapper<_Tp, 64 / sizeof(_Tp)>;
880 
881 // }}}
882 // __is_simd_wrapper {{{
883 template <typename _Tp>
884  struct __is_simd_wrapper : false_type {};
885 
886 template <typename _Tp, size_t _Np>
887  struct __is_simd_wrapper<_SimdWrapper<_Tp, _Np>> : true_type {};
888 
889 template <typename _Tp>
890  inline constexpr bool __is_simd_wrapper_v = __is_simd_wrapper<_Tp>::value;
891 
892 // }}}
893 // _BitOps {{{
894 struct _BitOps
895 {
896  // _S_bit_iteration {{{
897  template <typename _Tp, typename _Fp>
898  static void
899  _S_bit_iteration(_Tp __mask, _Fp&& __f)
900  {
901  static_assert(sizeof(_ULLong) >= sizeof(_Tp));
902  conditional_t<sizeof(_Tp) <= sizeof(_UInt), _UInt, _ULLong> __k;
903  if constexpr (is_convertible_v<_Tp, decltype(__k)>)
904  __k = __mask;
905  else
906  __k = __mask.to_ullong();
907  while(__k)
908  {
909  __f(std::__countr_zero(__k));
910  __k &= (__k - 1);
911  }
912  }
913 
914  //}}}
915 };
916 
917 //}}}
918 // __increment, __decrement {{{
919 template <typename _Tp = void>
920  struct __increment
921  { constexpr _Tp operator()(_Tp __a) const { return ++__a; } };
922 
923 template <>
924  struct __increment<void>
925  {
926  template <typename _Tp>
927  constexpr _Tp
928  operator()(_Tp __a) const
929  { return ++__a; }
930  };
931 
932 template <typename _Tp = void>
933  struct __decrement
934  { constexpr _Tp operator()(_Tp __a) const { return --__a; } };
935 
936 template <>
937  struct __decrement<void>
938  {
939  template <typename _Tp>
940  constexpr _Tp
941  operator()(_Tp __a) const
942  { return --__a; }
943  };
944 
945 // }}}
946 // _ValuePreserving(OrInt) {{{
947 template <typename _From, typename _To,
948  typename = enable_if_t<negation<
949  __is_narrowing_conversion<__remove_cvref_t<_From>, _To>>::value>>
950  using _ValuePreserving = _From;
951 
952 template <typename _From, typename _To,
953  typename _DecayedFrom = __remove_cvref_t<_From>,
954  typename = enable_if_t<conjunction<
955  is_convertible<_From, _To>,
956  disjunction<
957  is_same<_DecayedFrom, _To>, is_same<_DecayedFrom, int>,
958  conjunction<is_same<_DecayedFrom, _UInt>, is_unsigned<_To>>,
959  negation<__is_narrowing_conversion<_DecayedFrom, _To>>>>::value>>
960  using _ValuePreservingOrInt = _From;
961 
962 // }}}
963 // __intrinsic_type {{{
964 template <typename _Tp, size_t _Bytes, typename = void_t<>>
965  struct __intrinsic_type;
966 
967 template <typename _Tp, size_t _Size>
968  using __intrinsic_type_t =
969  typename __intrinsic_type<_Tp, _Size * sizeof(_Tp)>::type;
970 
971 template <typename _Tp>
972  using __intrinsic_type2_t = typename __intrinsic_type<_Tp, 2>::type;
973 template <typename _Tp>
974  using __intrinsic_type4_t = typename __intrinsic_type<_Tp, 4>::type;
975 template <typename _Tp>
976  using __intrinsic_type8_t = typename __intrinsic_type<_Tp, 8>::type;
977 template <typename _Tp>
978  using __intrinsic_type16_t = typename __intrinsic_type<_Tp, 16>::type;
979 template <typename _Tp>
980  using __intrinsic_type32_t = typename __intrinsic_type<_Tp, 32>::type;
981 template <typename _Tp>
982  using __intrinsic_type64_t = typename __intrinsic_type<_Tp, 64>::type;
983 
984 // }}}
985 // _BitMask {{{
986 template <size_t _Np, bool _Sanitized = false>
987  struct _BitMask;
988 
989 template <size_t _Np, bool _Sanitized>
990  struct __is_bitmask<_BitMask<_Np, _Sanitized>, void> : true_type {};
991 
992 template <size_t _Np>
993  using _SanitizedBitMask = _BitMask<_Np, true>;
994 
995 template <size_t _Np, bool _Sanitized>
996  struct _BitMask
997  {
998  static_assert(_Np > 0);
999 
1000  static constexpr size_t _NBytes = __div_roundup(_Np, __CHAR_BIT__);
1001 
1002  using _Tp = conditional_t<_Np == 1, bool,
1003  make_unsigned_t<__int_with_sizeof_t<std::min(
1004  sizeof(_ULLong), std::__bit_ceil(_NBytes))>>>;
1005 
1006  static constexpr int _S_array_size = __div_roundup(_NBytes, sizeof(_Tp));
1007 
1008  _Tp _M_bits[_S_array_size];
1009 
1010  static constexpr int _S_unused_bits
1011  = _Np == 1 ? 0 : _S_array_size * sizeof(_Tp) * __CHAR_BIT__ - _Np;
1012 
1013  static constexpr _Tp _S_bitmask = +_Tp(~_Tp()) >> _S_unused_bits;
1014 
1015  constexpr _BitMask() noexcept = default;
1016 
1017  constexpr _BitMask(unsigned long long __x) noexcept
1018  : _M_bits{static_cast<_Tp>(__x)} {}
1019 
1020  _BitMask(bitset<_Np> __x) noexcept : _BitMask(__x.to_ullong()) {}
1021 
1022  constexpr _BitMask(const _BitMask&) noexcept = default;
1023 
1024  template <bool _RhsSanitized, typename = enable_if_t<_RhsSanitized == false
1025  && _Sanitized == true>>
1026  constexpr _BitMask(const _BitMask<_Np, _RhsSanitized>& __rhs) noexcept
1027  : _BitMask(__rhs._M_sanitized()) {}
1028 
1029  constexpr operator _SimdWrapper<bool, _Np>() const noexcept
1030  {
1031  static_assert(_S_array_size == 1);
1032  return _M_bits[0];
1033  }
1034 
1035  // precondition: is sanitized
1036  constexpr _Tp
1037  _M_to_bits() const noexcept
1038  {
1039  static_assert(_S_array_size == 1);
1040  return _M_bits[0];
1041  }
1042 
1043  // precondition: is sanitized
1044  constexpr unsigned long long
1045  to_ullong() const noexcept
1046  {
1047  static_assert(_S_array_size == 1);
1048  return _M_bits[0];
1049  }
1050 
1051  // precondition: is sanitized
1052  constexpr unsigned long
1053  to_ulong() const noexcept
1054  {
1055  static_assert(_S_array_size == 1);
1056  return _M_bits[0];
1057  }
1058 
1059  constexpr bitset<_Np>
1060  _M_to_bitset() const noexcept
1061  {
1062  static_assert(_S_array_size == 1);
1063  return _M_bits[0];
1064  }
1065 
1066  constexpr decltype(auto)
1067  _M_sanitized() const noexcept
1068  {
1069  if constexpr (_Sanitized)
1070  return *this;
1071  else if constexpr (_Np == 1)
1072  return _SanitizedBitMask<_Np>(_M_bits[0]);
1073  else
1074  {
1075  _SanitizedBitMask<_Np> __r = {};
1076  for (int __i = 0; __i < _S_array_size; ++__i)
1077  __r._M_bits[__i] = _M_bits[__i];
1078  if constexpr (_S_unused_bits > 0)
1079  __r._M_bits[_S_array_size - 1] &= _S_bitmask;
1080  return __r;
1081  }
1082  }
1083 
1084  template <size_t _Mp, bool _LSanitized>
1085  constexpr _BitMask<_Np + _Mp, _Sanitized>
1086  _M_prepend(_BitMask<_Mp, _LSanitized> __lsb) const noexcept
1087  {
1088  constexpr size_t _RN = _Np + _Mp;
1089  using _Rp = _BitMask<_RN, _Sanitized>;
1090  if constexpr (_Rp::_S_array_size == 1)
1091  {
1092  _Rp __r{{_M_bits[0]}};
1093  __r._M_bits[0] <<= _Mp;
1094  __r._M_bits[0] |= __lsb._M_sanitized()._M_bits[0];
1095  return __r;
1096  }
1097  else
1098  __assert_unreachable<_Rp>();
1099  }
1100 
1101  // Return a new _BitMask with size _NewSize while dropping _DropLsb least
1102  // significant bits. If the operation implicitly produces a sanitized bitmask,
1103  // the result type will have _Sanitized set.
1104  template <size_t _DropLsb, size_t _NewSize = _Np - _DropLsb>
1105  constexpr auto
1106  _M_extract() const noexcept
1107  {
1108  static_assert(_Np > _DropLsb);
1109  static_assert(_DropLsb + _NewSize <= sizeof(_ULLong) * __CHAR_BIT__,
1110  "not implemented for bitmasks larger than one ullong");
1111  if constexpr (_NewSize == 1)
1112  // must sanitize because the return _Tp is bool
1113  return _SanitizedBitMask<1>(_M_bits[0] & (_Tp(1) << _DropLsb));
1114  else
1115  return _BitMask<_NewSize,
1116  ((_NewSize + _DropLsb == sizeof(_Tp) * __CHAR_BIT__
1117  && _NewSize + _DropLsb <= _Np)
1118  || ((_Sanitized || _Np == sizeof(_Tp) * __CHAR_BIT__)
1119  && _NewSize + _DropLsb >= _Np))>(_M_bits[0]
1120  >> _DropLsb);
1121  }
1122 
1123  // True if all bits are set. Implicitly sanitizes if _Sanitized == false.
1124  constexpr bool
1125  all() const noexcept
1126  {
1127  if constexpr (_Np == 1)
1128  return _M_bits[0];
1129  else if constexpr (!_Sanitized)
1130  return _M_sanitized().all();
1131  else
1132  {
1133  constexpr _Tp __allbits = ~_Tp();
1134  for (int __i = 0; __i < _S_array_size - 1; ++__i)
1135  if (_M_bits[__i] != __allbits)
1136  return false;
1137  return _M_bits[_S_array_size - 1] == _S_bitmask;
1138  }
1139  }
1140 
1141  // True if at least one bit is set. Implicitly sanitizes if _Sanitized ==
1142  // false.
1143  constexpr bool
1144  any() const noexcept
1145  {
1146  if constexpr (_Np == 1)
1147  return _M_bits[0];
1148  else if constexpr (!_Sanitized)
1149  return _M_sanitized().any();
1150  else
1151  {
1152  for (int __i = 0; __i < _S_array_size - 1; ++__i)
1153  if (_M_bits[__i] != 0)
1154  return true;
1155  return _M_bits[_S_array_size - 1] != 0;
1156  }
1157  }
1158 
1159  // True if no bit is set. Implicitly sanitizes if _Sanitized == false.
1160  constexpr bool
1161  none() const noexcept
1162  {
1163  if constexpr (_Np == 1)
1164  return !_M_bits[0];
1165  else if constexpr (!_Sanitized)
1166  return _M_sanitized().none();
1167  else
1168  {
1169  for (int __i = 0; __i < _S_array_size - 1; ++__i)
1170  if (_M_bits[__i] != 0)
1171  return false;
1172  return _M_bits[_S_array_size - 1] == 0;
1173  }
1174  }
1175 
1176  // Returns the number of set bits. Implicitly sanitizes if _Sanitized ==
1177  // false.
1178  constexpr int
1179  count() const noexcept
1180  {
1181  if constexpr (_Np == 1)
1182  return _M_bits[0];
1183  else if constexpr (!_Sanitized)
1184  return _M_sanitized().none();
1185  else
1186  {
1187  int __result = __builtin_popcountll(_M_bits[0]);
1188  for (int __i = 1; __i < _S_array_size; ++__i)
1189  __result += __builtin_popcountll(_M_bits[__i]);
1190  return __result;
1191  }
1192  }
1193 
1194  // Returns the bit at offset __i as bool.
1195  constexpr bool
1196  operator[](size_t __i) const noexcept
1197  {
1198  if constexpr (_Np == 1)
1199  return _M_bits[0];
1200  else if constexpr (_S_array_size == 1)
1201  return (_M_bits[0] >> __i) & 1;
1202  else
1203  {
1204  const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1205  const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1206  return (_M_bits[__j] >> __shift) & 1;
1207  }
1208  }
1209 
1210  template <size_t __i>
1211  constexpr bool
1212  operator[](_SizeConstant<__i>) const noexcept
1213  {
1214  static_assert(__i < _Np);
1215  constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1216  constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1217  return static_cast<bool>(_M_bits[__j] & (_Tp(1) << __shift));
1218  }
1219 
1220  // Set the bit at offset __i to __x.
1221  constexpr void
1222  set(size_t __i, bool __x) noexcept
1223  {
1224  if constexpr (_Np == 1)
1225  _M_bits[0] = __x;
1226  else if constexpr (_S_array_size == 1)
1227  {
1228  _M_bits[0] &= ~_Tp(_Tp(1) << __i);
1229  _M_bits[0] |= _Tp(_Tp(__x) << __i);
1230  }
1231  else
1232  {
1233  const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1234  const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1235  _M_bits[__j] &= ~_Tp(_Tp(1) << __shift);
1236  _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1237  }
1238  }
1239 
1240  template <size_t __i>
1241  constexpr void
1242  set(_SizeConstant<__i>, bool __x) noexcept
1243  {
1244  static_assert(__i < _Np);
1245  if constexpr (_Np == 1)
1246  _M_bits[0] = __x;
1247  else
1248  {
1249  constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1250  constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1251  constexpr _Tp __mask = ~_Tp(_Tp(1) << __shift);
1252  _M_bits[__j] &= __mask;
1253  _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1254  }
1255  }
1256 
1257  // Inverts all bits. Sanitized input leads to sanitized output.
1258  constexpr _BitMask
1259  operator~() const noexcept
1260  {
1261  if constexpr (_Np == 1)
1262  return !_M_bits[0];
1263  else
1264  {
1265  _BitMask __result{};
1266  for (int __i = 0; __i < _S_array_size - 1; ++__i)
1267  __result._M_bits[__i] = ~_M_bits[__i];
1268  if constexpr (_Sanitized)
1269  __result._M_bits[_S_array_size - 1]
1270  = _M_bits[_S_array_size - 1] ^ _S_bitmask;
1271  else
1272  __result._M_bits[_S_array_size - 1] = ~_M_bits[_S_array_size - 1];
1273  return __result;
1274  }
1275  }
1276 
1277  constexpr _BitMask&
1278  operator^=(const _BitMask& __b) & noexcept
1279  {
1280  __execute_n_times<_S_array_size>(
1281  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] ^= __b._M_bits[__i]; });
1282  return *this;
1283  }
1284 
1285  constexpr _BitMask&
1286  operator|=(const _BitMask& __b) & noexcept
1287  {
1288  __execute_n_times<_S_array_size>(
1289  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] |= __b._M_bits[__i]; });
1290  return *this;
1291  }
1292 
1293  constexpr _BitMask&
1294  operator&=(const _BitMask& __b) & noexcept
1295  {
1296  __execute_n_times<_S_array_size>(
1297  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] &= __b._M_bits[__i]; });
1298  return *this;
1299  }
1300 
1301  friend constexpr _BitMask
1302  operator^(const _BitMask& __a, const _BitMask& __b) noexcept
1303  {
1304  _BitMask __r = __a;
1305  __r ^= __b;
1306  return __r;
1307  }
1308 
1309  friend constexpr _BitMask
1310  operator|(const _BitMask& __a, const _BitMask& __b) noexcept
1311  {
1312  _BitMask __r = __a;
1313  __r |= __b;
1314  return __r;
1315  }
1316 
1317  friend constexpr _BitMask
1318  operator&(const _BitMask& __a, const _BitMask& __b) noexcept
1319  {
1320  _BitMask __r = __a;
1321  __r &= __b;
1322  return __r;
1323  }
1324 
1325  _GLIBCXX_SIMD_INTRINSIC
1326  constexpr bool
1327  _M_is_constprop() const
1328  {
1329  if constexpr (_S_array_size == 0)
1330  return __builtin_constant_p(_M_bits[0]);
1331  else
1332  {
1333  for (int __i = 0; __i < _S_array_size; ++__i)
1334  if (!__builtin_constant_p(_M_bits[__i]))
1335  return false;
1336  return true;
1337  }
1338  }
1339  };
1340 
1341 // }}}
1342 
1343 // vvv ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- vvv
1344 // __min_vector_size {{{
1345 template <typename _Tp = void>
1346  static inline constexpr int __min_vector_size = 2 * sizeof(_Tp);
1347 
1348 #if _GLIBCXX_SIMD_HAVE_NEON
1349 template <>
1350  inline constexpr int __min_vector_size<void> = 8;
1351 #else
1352 template <>
1353  inline constexpr int __min_vector_size<void> = 16;
1354 #endif
1355 
1356 // }}}
1357 // __vector_type {{{
1358 template <typename _Tp, size_t _Np, typename = void>
1359  struct __vector_type_n {};
1360 
1361 // substition failure for 0-element case
1362 template <typename _Tp>
1363  struct __vector_type_n<_Tp, 0, void> {};
1364 
1365 // special case 1-element to be _Tp itself
1366 template <typename _Tp>
1367  struct __vector_type_n<_Tp, 1, enable_if_t<__is_vectorizable_v<_Tp>>>
1368  { using type = _Tp; };
1369 
1370 // else, use GNU-style builtin vector types
1371 template <typename _Tp, size_t _Np>
1372  struct __vector_type_n<_Tp, _Np, enable_if_t<__is_vectorizable_v<_Tp> && _Np >= 2>>
1373  {
1374  static constexpr size_t _S_Np2 = std::__bit_ceil(_Np * sizeof(_Tp));
1375 
1376  static constexpr size_t _S_Bytes =
1377 #ifdef __i386__
1378  // Using [[gnu::vector_size(8)]] would wreak havoc on the FPU because
1379  // those objects are passed via MMX registers and nothing ever calls EMMS.
1380  _S_Np2 == 8 ? 16 :
1381 #endif
1382  _S_Np2 < __min_vector_size<_Tp> ? __min_vector_size<_Tp>
1383  : _S_Np2;
1384 
1385  using type [[__gnu__::__vector_size__(_S_Bytes)]] = _Tp;
1386  };
1387 
1388 template <typename _Tp, size_t _Bytes, size_t = _Bytes % sizeof(_Tp)>
1389  struct __vector_type;
1390 
1391 template <typename _Tp, size_t _Bytes>
1392  struct __vector_type<_Tp, _Bytes, 0>
1393  : __vector_type_n<_Tp, _Bytes / sizeof(_Tp)> {};
1394 
1395 template <typename _Tp, size_t _Size>
1396  using __vector_type_t = typename __vector_type_n<_Tp, _Size>::type;
1397 
1398 template <typename _Tp>
1399  using __vector_type2_t = typename __vector_type<_Tp, 2>::type;
1400 template <typename _Tp>
1401  using __vector_type4_t = typename __vector_type<_Tp, 4>::type;
1402 template <typename _Tp>
1403  using __vector_type8_t = typename __vector_type<_Tp, 8>::type;
1404 template <typename _Tp>
1405  using __vector_type16_t = typename __vector_type<_Tp, 16>::type;
1406 template <typename _Tp>
1407  using __vector_type32_t = typename __vector_type<_Tp, 32>::type;
1408 template <typename _Tp>
1409  using __vector_type64_t = typename __vector_type<_Tp, 64>::type;
1410 
1411 // }}}
1412 // __is_vector_type {{{
1413 template <typename _Tp, typename = void_t<>>
1414  struct __is_vector_type : false_type {};
1415 
1416 template <typename _Tp>
1417  struct __is_vector_type<
1418  _Tp, void_t<typename __vector_type<
1419  remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>>
1420  : is_same<_Tp, typename __vector_type<
1421  remove_reference_t<decltype(declval<_Tp>()[0])>,
1422  sizeof(_Tp)>::type> {};
1423 
1424 template <typename _Tp>
1425  inline constexpr bool __is_vector_type_v = __is_vector_type<_Tp>::value;
1426 
1427 // }}}
1428 // __is_intrinsic_type {{{
1429 #if _GLIBCXX_SIMD_HAVE_SSE_ABI
1430 template <typename _Tp>
1431  using __is_intrinsic_type = __is_vector_type<_Tp>;
1432 #else // not SSE (x86)
1433 template <typename _Tp, typename = void_t<>>
1434  struct __is_intrinsic_type : false_type {};
1435 
1436 template <typename _Tp>
1437  struct __is_intrinsic_type<
1438  _Tp, void_t<typename __intrinsic_type<
1439  remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>>
1440  : is_same<_Tp, typename __intrinsic_type<
1441  remove_reference_t<decltype(declval<_Tp>()[0])>,
1442  sizeof(_Tp)>::type> {};
1443 #endif
1444 
1445 template <typename _Tp>
1446  inline constexpr bool __is_intrinsic_type_v = __is_intrinsic_type<_Tp>::value;
1447 
1448 // }}}
1449 // _VectorTraits{{{
1450 template <typename _Tp, typename = void_t<>>
1451  struct _VectorTraitsImpl;
1452 
1453 template <typename _Tp>
1454  struct _VectorTraitsImpl<_Tp, enable_if_t<__is_vector_type_v<_Tp>
1455  || __is_intrinsic_type_v<_Tp>>>
1456  {
1457  using type = _Tp;
1458  using value_type = remove_reference_t<decltype(declval<_Tp>()[0])>;
1459  static constexpr int _S_full_size = sizeof(_Tp) / sizeof(value_type);
1460  using _Wrapper = _SimdWrapper<value_type, _S_full_size>;
1461  template <typename _Up, int _W = _S_full_size>
1462  static constexpr bool _S_is
1463  = is_same_v<value_type, _Up> && _W == _S_full_size;
1464  };
1465 
1466 template <typename _Tp, size_t _Np>
1467  struct _VectorTraitsImpl<_SimdWrapper<_Tp, _Np>,
1468  void_t<__vector_type_t<_Tp, _Np>>>
1469  {
1470  using type = __vector_type_t<_Tp, _Np>;
1471  using value_type = _Tp;
1472  static constexpr int _S_full_size = sizeof(type) / sizeof(value_type);
1473  using _Wrapper = _SimdWrapper<_Tp, _Np>;
1474  static constexpr bool _S_is_partial = (_Np == _S_full_size);
1475  static constexpr int _S_partial_width = _Np;
1476  template <typename _Up, int _W = _S_full_size>
1477  static constexpr bool _S_is
1478  = is_same_v<value_type, _Up>&& _W == _S_full_size;
1479  };
1480 
1481 template <typename _Tp, typename = typename _VectorTraitsImpl<_Tp>::type>
1482  using _VectorTraits = _VectorTraitsImpl<_Tp>;
1483 
1484 // }}}
1485 // __as_vector{{{
1486 template <typename _V>
1487  _GLIBCXX_SIMD_INTRINSIC constexpr auto
1488  __as_vector(_V __x)
1489  {
1490  if constexpr (__is_vector_type_v<_V>)
1491  return __x;
1492  else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1493  return __data(__x)._M_data;
1494  else if constexpr (__is_vectorizable_v<_V>)
1495  return __vector_type_t<_V, 2>{__x};
1496  else
1497  return __x._M_data;
1498  }
1499 
1500 // }}}
1501 // __as_wrapper{{{
1502 template <size_t _Np = 0, typename _V>
1503  _GLIBCXX_SIMD_INTRINSIC constexpr auto
1504  __as_wrapper(_V __x)
1505  {
1506  if constexpr (__is_vector_type_v<_V>)
1507  return _SimdWrapper<typename _VectorTraits<_V>::value_type,
1508  (_Np > 0 ? _Np : _VectorTraits<_V>::_S_full_size)>(__x);
1509  else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1510  {
1511  static_assert(_V::size() == _Np);
1512  return __data(__x);
1513  }
1514  else
1515  {
1516  static_assert(_V::_S_size == _Np);
1517  return __x;
1518  }
1519  }
1520 
1521 // }}}
1522 // __intrin_bitcast{{{
1523 template <typename _To, typename _From>
1524  _GLIBCXX_SIMD_INTRINSIC constexpr _To
1525  __intrin_bitcast(_From __v)
1526  {
1527  static_assert((__is_vector_type_v<_From> || __is_intrinsic_type_v<_From>)
1528  && (__is_vector_type_v<_To> || __is_intrinsic_type_v<_To>));
1529  if constexpr (sizeof(_To) == sizeof(_From))
1530  return reinterpret_cast<_To>(__v);
1531  else if constexpr (sizeof(_From) > sizeof(_To))
1532  if constexpr (sizeof(_To) >= 16)
1533  return reinterpret_cast<const __may_alias<_To>&>(__v);
1534  else
1535  {
1536  _To __r;
1537  __builtin_memcpy(&__r, &__v, sizeof(_To));
1538  return __r;
1539  }
1540 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1541  else if constexpr (__have_avx && sizeof(_From) == 16 && sizeof(_To) == 32)
1542  return reinterpret_cast<_To>(__builtin_ia32_ps256_ps(
1543  reinterpret_cast<__vector_type_t<float, 4>>(__v)));
1544  else if constexpr (__have_avx512f && sizeof(_From) == 16
1545  && sizeof(_To) == 64)
1546  return reinterpret_cast<_To>(__builtin_ia32_ps512_ps(
1547  reinterpret_cast<__vector_type_t<float, 4>>(__v)));
1548  else if constexpr (__have_avx512f && sizeof(_From) == 32
1549  && sizeof(_To) == 64)
1550  return reinterpret_cast<_To>(__builtin_ia32_ps512_256ps(
1551  reinterpret_cast<__vector_type_t<float, 8>>(__v)));
1552 #endif // _GLIBCXX_SIMD_X86INTRIN
1553  else if constexpr (sizeof(__v) <= 8)
1554  return reinterpret_cast<_To>(
1555  __vector_type_t<__int_for_sizeof_t<_From>, sizeof(_To) / sizeof(_From)>{
1556  reinterpret_cast<__int_for_sizeof_t<_From>>(__v)});
1557  else
1558  {
1559  static_assert(sizeof(_To) > sizeof(_From));
1560  _To __r = {};
1561  __builtin_memcpy(&__r, &__v, sizeof(_From));
1562  return __r;
1563  }
1564  }
1565 
1566 // }}}
1567 // __vector_bitcast{{{
1568 template <typename _To, size_t _NN = 0, typename _From,
1569  typename _FromVT = _VectorTraits<_From>,
1570  size_t _Np = _NN == 0 ? sizeof(_From) / sizeof(_To) : _NN>
1571  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1572  __vector_bitcast(_From __x)
1573  {
1574  using _R = __vector_type_t<_To, _Np>;
1575  return __intrin_bitcast<_R>(__x);
1576  }
1577 
1578 template <typename _To, size_t _NN = 0, typename _Tp, size_t _Nx,
1579  size_t _Np
1580  = _NN == 0 ? sizeof(_SimdWrapper<_Tp, _Nx>) / sizeof(_To) : _NN>
1581  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1582  __vector_bitcast(const _SimdWrapper<_Tp, _Nx>& __x)
1583  {
1584  static_assert(_Np > 1);
1585  return __intrin_bitcast<__vector_type_t<_To, _Np>>(__x._M_data);
1586  }
1587 
1588 // }}}
1589 // __convert_x86 declarations {{{
1590 #ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
1591 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1592  _To __convert_x86(_Tp);
1593 
1594 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1595  _To __convert_x86(_Tp, _Tp);
1596 
1597 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1598  _To __convert_x86(_Tp, _Tp, _Tp, _Tp);
1599 
1600 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1601  _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp);
1602 
1603 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1604  _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp,
1605  _Tp, _Tp, _Tp, _Tp);
1606 #endif // _GLIBCXX_SIMD_WORKAROUND_PR85048
1607 
1608 //}}}
1609 // __bit_cast {{{
1610 template <typename _To, typename _From>
1611  _GLIBCXX_SIMD_INTRINSIC constexpr _To
1612  __bit_cast(const _From __x)
1613  {
1614  // TODO: implement with / replace by __builtin_bit_cast ASAP
1615  static_assert(sizeof(_To) == sizeof(_From));
1616  constexpr bool __to_is_vectorizable
1617  = is_arithmetic_v<_To> || is_enum_v<_To>;
1618  constexpr bool __from_is_vectorizable
1619  = is_arithmetic_v<_From> || is_enum_v<_From>;
1620  if constexpr (__is_vector_type_v<_To> && __is_vector_type_v<_From>)
1621  return reinterpret_cast<_To>(__x);
1622  else if constexpr (__is_vector_type_v<_To> && __from_is_vectorizable)
1623  {
1624  using _FV [[__gnu__::__vector_size__(sizeof(_From))]] = _From;
1625  return reinterpret_cast<_To>(_FV{__x});
1626  }
1627  else if constexpr (__to_is_vectorizable && __from_is_vectorizable)
1628  {
1629  using _TV [[__gnu__::__vector_size__(sizeof(_To))]] = _To;
1630  using _FV [[__gnu__::__vector_size__(sizeof(_From))]] = _From;
1631  return reinterpret_cast<_TV>(_FV{__x})[0];
1632  }
1633  else if constexpr (__to_is_vectorizable && __is_vector_type_v<_From>)
1634  {
1635  using _TV [[__gnu__::__vector_size__(sizeof(_To))]] = _To;
1636  return reinterpret_cast<_TV>(__x)[0];
1637  }
1638  else
1639  {
1640  _To __r;
1641  __builtin_memcpy(reinterpret_cast<char*>(&__r),
1642  reinterpret_cast<const char*>(&__x), sizeof(_To));
1643  return __r;
1644  }
1645  }
1646 
1647 // }}}
1648 // __to_intrin {{{
1649 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
1650  typename _R = __intrinsic_type_t<typename _TVT::value_type, _TVT::_S_full_size>>
1651  _GLIBCXX_SIMD_INTRINSIC constexpr _R
1652  __to_intrin(_Tp __x)
1653  {
1654  static_assert(sizeof(__x) <= sizeof(_R),
1655  "__to_intrin may never drop values off the end");
1656  if constexpr (sizeof(__x) == sizeof(_R))
1657  return reinterpret_cast<_R>(__as_vector(__x));
1658  else
1659  {
1660  using _Up = __int_for_sizeof_t<_Tp>;
1661  return reinterpret_cast<_R>(
1662  __vector_type_t<_Up, sizeof(_R) / sizeof(_Up)>{__bit_cast<_Up>(__x)});
1663  }
1664  }
1665 
1666 // }}}
1667 // __make_vector{{{
1668 template <typename _Tp, typename... _Args>
1669  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, sizeof...(_Args)>
1670  __make_vector(const _Args&... __args)
1671  { return __vector_type_t<_Tp, sizeof...(_Args)>{static_cast<_Tp>(__args)...}; }
1672 
1673 // }}}
1674 // __vector_broadcast{{{
1675 template <size_t _Np, typename _Tp, size_t... _I>
1676  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1677  __vector_broadcast_impl(_Tp __x, index_sequence<_I...>)
1678  { return __vector_type_t<_Tp, _Np>{((void)_I, __x)...}; }
1679 
1680 template <size_t _Np, typename _Tp>
1681  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1682  __vector_broadcast(_Tp __x)
1683  { return __vector_broadcast_impl<_Np, _Tp>(__x, make_index_sequence<_Np>()); }
1684 
1685 // }}}
1686 // __generate_vector{{{
1687  template <typename _Tp, size_t _Np, typename _Gp, size_t... _I>
1688  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1689  __generate_vector_impl(_Gp&& __gen, index_sequence<_I...>)
1690  { return __vector_type_t<_Tp, _Np>{ static_cast<_Tp>(__gen(_SizeConstant<_I>()))...}; }
1691 
1692 template <typename _V, typename _VVT = _VectorTraits<_V>, typename _Gp>
1693  _GLIBCXX_SIMD_INTRINSIC constexpr _V
1694  __generate_vector(_Gp&& __gen)
1695  {
1696  if constexpr (__is_vector_type_v<_V>)
1697  return __generate_vector_impl<typename _VVT::value_type,
1698  _VVT::_S_full_size>(
1699  static_cast<_Gp&&>(__gen), make_index_sequence<_VVT::_S_full_size>());
1700  else
1701  return __generate_vector_impl<typename _VVT::value_type,
1702  _VVT::_S_partial_width>(
1703  static_cast<_Gp&&>(__gen),
1704  make_index_sequence<_VVT::_S_partial_width>());
1705  }
1706 
1707 template <typename _Tp, size_t _Np, typename _Gp>
1708  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1709  __generate_vector(_Gp&& __gen)
1710  {
1711  return __generate_vector_impl<_Tp, _Np>(static_cast<_Gp&&>(__gen),
1712  make_index_sequence<_Np>());
1713  }
1714 
1715 // }}}
1716 // __xor{{{
1717 template <typename _TW>
1718  _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1719  __xor(_TW __a, _TW __b) noexcept
1720  {
1721  if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1722  {
1723  using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1724  _VectorTraitsImpl<_TW>>::value_type;
1725  if constexpr (is_floating_point_v<_Tp>)
1726  {
1727  using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1728  return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1729  ^ __vector_bitcast<_Ip>(__b));
1730  }
1731  else if constexpr (__is_vector_type_v<_TW>)
1732  return __a ^ __b;
1733  else
1734  return __a._M_data ^ __b._M_data;
1735  }
1736  else
1737  return __a ^ __b;
1738  }
1739 
1740 // }}}
1741 // __or{{{
1742 template <typename _TW>
1743  _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1744  __or(_TW __a, _TW __b) noexcept
1745  {
1746  if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1747  {
1748  using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1749  _VectorTraitsImpl<_TW>>::value_type;
1750  if constexpr (is_floating_point_v<_Tp>)
1751  {
1752  using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1753  return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1754  | __vector_bitcast<_Ip>(__b));
1755  }
1756  else if constexpr (__is_vector_type_v<_TW>)
1757  return __a | __b;
1758  else
1759  return __a._M_data | __b._M_data;
1760  }
1761  else
1762  return __a | __b;
1763  }
1764 
1765 // }}}
1766 // __and{{{
1767 template <typename _TW>
1768  _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1769  __and(_TW __a, _TW __b) noexcept
1770  {
1771  if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1772  {
1773  using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1774  _VectorTraitsImpl<_TW>>::value_type;
1775  if constexpr (is_floating_point_v<_Tp>)
1776  {
1777  using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1778  return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1779  & __vector_bitcast<_Ip>(__b));
1780  }
1781  else if constexpr (__is_vector_type_v<_TW>)
1782  return __a & __b;
1783  else
1784  return __a._M_data & __b._M_data;
1785  }
1786  else
1787  return __a & __b;
1788  }
1789 
1790 // }}}
1791 // __andnot{{{
1792 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1793 static constexpr struct
1794 {
1795  _GLIBCXX_SIMD_INTRINSIC __v4sf
1796  operator()(__v4sf __a, __v4sf __b) const noexcept
1797  { return __builtin_ia32_andnps(__a, __b); }
1798 
1799  _GLIBCXX_SIMD_INTRINSIC __v2df
1800  operator()(__v2df __a, __v2df __b) const noexcept
1801  { return __builtin_ia32_andnpd(__a, __b); }
1802 
1803  _GLIBCXX_SIMD_INTRINSIC __v2di
1804  operator()(__v2di __a, __v2di __b) const noexcept
1805  { return __builtin_ia32_pandn128(__a, __b); }
1806 
1807  _GLIBCXX_SIMD_INTRINSIC __v8sf
1808  operator()(__v8sf __a, __v8sf __b) const noexcept
1809  { return __builtin_ia32_andnps256(__a, __b); }
1810 
1811  _GLIBCXX_SIMD_INTRINSIC __v4df
1812  operator()(__v4df __a, __v4df __b) const noexcept
1813  { return __builtin_ia32_andnpd256(__a, __b); }
1814 
1815  _GLIBCXX_SIMD_INTRINSIC __v4di
1816  operator()(__v4di __a, __v4di __b) const noexcept
1817  {
1818  if constexpr (__have_avx2)
1819  return __builtin_ia32_andnotsi256(__a, __b);
1820  else
1821  return reinterpret_cast<__v4di>(
1822  __builtin_ia32_andnpd256(reinterpret_cast<__v4df>(__a),
1823  reinterpret_cast<__v4df>(__b)));
1824  }
1825 
1826  _GLIBCXX_SIMD_INTRINSIC __v16sf
1827  operator()(__v16sf __a, __v16sf __b) const noexcept
1828  {
1829  if constexpr (__have_avx512dq)
1830  return _mm512_andnot_ps(__a, __b);
1831  else
1832  return reinterpret_cast<__v16sf>(
1833  _mm512_andnot_si512(reinterpret_cast<__v8di>(__a),
1834  reinterpret_cast<__v8di>(__b)));
1835  }
1836 
1837  _GLIBCXX_SIMD_INTRINSIC __v8df
1838  operator()(__v8df __a, __v8df __b) const noexcept
1839  {
1840  if constexpr (__have_avx512dq)
1841  return _mm512_andnot_pd(__a, __b);
1842  else
1843  return reinterpret_cast<__v8df>(
1844  _mm512_andnot_si512(reinterpret_cast<__v8di>(__a),
1845  reinterpret_cast<__v8di>(__b)));
1846  }
1847 
1848  _GLIBCXX_SIMD_INTRINSIC __v8di
1849  operator()(__v8di __a, __v8di __b) const noexcept
1850  { return _mm512_andnot_si512(__a, __b); }
1851 } _S_x86_andnot;
1852 #endif // _GLIBCXX_SIMD_X86INTRIN && !__clang__
1853 
1854 template <typename _TW>
1855  _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1856  __andnot(_TW __a, _TW __b) noexcept
1857  {
1858  if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1859  {
1860  using _TVT = conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1861  _VectorTraitsImpl<_TW>>;
1862  using _Tp = typename _TVT::value_type;
1863 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1864  if constexpr (sizeof(_TW) >= 16)
1865  {
1866  const auto __ai = __to_intrin(__a);
1867  const auto __bi = __to_intrin(__b);
1868  if (!__builtin_is_constant_evaluated()
1869  && !(__builtin_constant_p(__ai) && __builtin_constant_p(__bi)))
1870  {
1871  const auto __r = _S_x86_andnot(__ai, __bi);
1872  if constexpr (is_convertible_v<decltype(__r), _TW>)
1873  return __r;
1874  else
1875  return reinterpret_cast<typename _TVT::type>(__r);
1876  }
1877  }
1878 #endif // _GLIBCXX_SIMD_X86INTRIN
1879  using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1880  return __vector_bitcast<_Tp>(~__vector_bitcast<_Ip>(__a)
1881  & __vector_bitcast<_Ip>(__b));
1882  }
1883  else
1884  return ~__a & __b;
1885  }
1886 
1887 // }}}
1888 // __not{{{
1889 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1890  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
1891  __not(_Tp __a) noexcept
1892  {
1893  if constexpr (is_floating_point_v<typename _TVT::value_type>)
1894  return reinterpret_cast<typename _TVT::type>(
1895  ~__vector_bitcast<unsigned>(__a));
1896  else
1897  return ~__a;
1898  }
1899 
1900 // }}}
1901 // __concat{{{
1902 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
1903  typename _R = __vector_type_t<typename _TVT::value_type, _TVT::_S_full_size * 2>>
1904  constexpr _R
1905  __concat(_Tp a_, _Tp b_)
1906  {
1907 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
1908  using _W
1909  = conditional_t<is_floating_point_v<typename _TVT::value_type>, double,
1910  conditional_t<(sizeof(_Tp) >= 2 * sizeof(long long)),
1911  long long, typename _TVT::value_type>>;
1912  constexpr int input_width = sizeof(_Tp) / sizeof(_W);
1913  const auto __a = __vector_bitcast<_W>(a_);
1914  const auto __b = __vector_bitcast<_W>(b_);
1915  using _Up = __vector_type_t<_W, sizeof(_R) / sizeof(_W)>;
1916 #else
1917  constexpr int input_width = _TVT::_S_full_size;
1918  const _Tp& __a = a_;
1919  const _Tp& __b = b_;
1920  using _Up = _R;
1921 #endif
1922  if constexpr (input_width == 2)
1923  return reinterpret_cast<_R>(_Up{__a[0], __a[1], __b[0], __b[1]});
1924  else if constexpr (input_width == 4)
1925  return reinterpret_cast<_R>(
1926  _Up{__a[0], __a[1], __a[2], __a[3], __b[0], __b[1], __b[2], __b[3]});
1927  else if constexpr (input_width == 8)
1928  return reinterpret_cast<_R>(
1929  _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7],
1930  __b[0], __b[1], __b[2], __b[3], __b[4], __b[5], __b[6], __b[7]});
1931  else if constexpr (input_width == 16)
1932  return reinterpret_cast<_R>(
1933  _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
1934  __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
1935  __a[14], __a[15], __b[0], __b[1], __b[2], __b[3], __b[4],
1936  __b[5], __b[6], __b[7], __b[8], __b[9], __b[10], __b[11],
1937  __b[12], __b[13], __b[14], __b[15]});
1938  else if constexpr (input_width == 32)
1939  return reinterpret_cast<_R>(
1940  _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
1941  __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
1942  __a[14], __a[15], __a[16], __a[17], __a[18], __a[19], __a[20],
1943  __a[21], __a[22], __a[23], __a[24], __a[25], __a[26], __a[27],
1944  __a[28], __a[29], __a[30], __a[31], __b[0], __b[1], __b[2],
1945  __b[3], __b[4], __b[5], __b[6], __b[7], __b[8], __b[9],
1946  __b[10], __b[11], __b[12], __b[13], __b[14], __b[15], __b[16],
1947  __b[17], __b[18], __b[19], __b[20], __b[21], __b[22], __b[23],
1948  __b[24], __b[25], __b[26], __b[27], __b[28], __b[29], __b[30],
1949  __b[31]});
1950  }
1951 
1952 // }}}
1953 // __zero_extend {{{
1954 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1955  struct _ZeroExtendProxy
1956  {
1957  using value_type = typename _TVT::value_type;
1958  static constexpr size_t _Np = _TVT::_S_full_size;
1959  const _Tp __x;
1960 
1961  template <typename _To, typename _ToVT = _VectorTraits<_To>,
1962  typename
1963  = enable_if_t<is_same_v<typename _ToVT::value_type, value_type>>>
1964  _GLIBCXX_SIMD_INTRINSIC operator _To() const
1965  {
1966  constexpr size_t _ToN = _ToVT::_S_full_size;
1967  if constexpr (_ToN == _Np)
1968  return __x;
1969  else if constexpr (_ToN == 2 * _Np)
1970  {
1971 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
1972  if constexpr (__have_avx && _TVT::template _S_is<float, 4>)
1973  return __vector_bitcast<value_type>(
1974  _mm256_insertf128_ps(__m256(), __x, 0));
1975  else if constexpr (__have_avx && _TVT::template _S_is<double, 2>)
1976  return __vector_bitcast<value_type>(
1977  _mm256_insertf128_pd(__m256d(), __x, 0));
1978  else if constexpr (__have_avx2 && _Np * sizeof(value_type) == 16)
1979  return __vector_bitcast<value_type>(
1980  _mm256_insertf128_si256(__m256i(), __to_intrin(__x), 0));
1981  else if constexpr (__have_avx512f && _TVT::template _S_is<float, 8>)
1982  {
1983  if constexpr (__have_avx512dq)
1984  return __vector_bitcast<value_type>(
1985  _mm512_insertf32x8(__m512(), __x, 0));
1986  else
1987  return reinterpret_cast<__m512>(
1988  _mm512_insertf64x4(__m512d(),
1989  reinterpret_cast<__m256d>(__x), 0));
1990  }
1991  else if constexpr (__have_avx512f
1992  && _TVT::template _S_is<double, 4>)
1993  return __vector_bitcast<value_type>(
1994  _mm512_insertf64x4(__m512d(), __x, 0));
1995  else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 32)
1996  return __vector_bitcast<value_type>(
1997  _mm512_inserti64x4(__m512i(), __to_intrin(__x), 0));
1998 #endif
1999  return __concat(__x, _Tp());
2000  }
2001  else if constexpr (_ToN == 4 * _Np)
2002  {
2003 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2004  if constexpr (__have_avx512dq && _TVT::template _S_is<double, 2>)
2005  {
2006  return __vector_bitcast<value_type>(
2007  _mm512_insertf64x2(__m512d(), __x, 0));
2008  }
2009  else if constexpr (__have_avx512f
2010  && is_floating_point_v<value_type>)
2011  {
2012  return __vector_bitcast<value_type>(
2013  _mm512_insertf32x4(__m512(), reinterpret_cast<__m128>(__x),
2014  0));
2015  }
2016  else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 16)
2017  {
2018  return __vector_bitcast<value_type>(
2019  _mm512_inserti32x4(__m512i(), __to_intrin(__x), 0));
2020  }
2021 #endif
2022  return __concat(__concat(__x, _Tp()),
2023  __vector_type_t<value_type, _Np * 2>());
2024  }
2025  else if constexpr (_ToN == 8 * _Np)
2026  return __concat(operator __vector_type_t<value_type, _Np * 4>(),
2027  __vector_type_t<value_type, _Np * 4>());
2028  else if constexpr (_ToN == 16 * _Np)
2029  return __concat(operator __vector_type_t<value_type, _Np * 8>(),
2030  __vector_type_t<value_type, _Np * 8>());
2031  else
2032  __assert_unreachable<_Tp>();
2033  }
2034  };
2035 
2036 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2037  _GLIBCXX_SIMD_INTRINSIC _ZeroExtendProxy<_Tp, _TVT>
2038  __zero_extend(_Tp __x)
2039  { return {__x}; }
2040 
2041 // }}}
2042 // __extract<_Np, By>{{{
2043 template <int _Offset,
2044  int _SplitBy,
2045  typename _Tp,
2046  typename _TVT = _VectorTraits<_Tp>,
2047  typename _R = __vector_type_t<typename _TVT::value_type, _TVT::_S_full_size / _SplitBy>>
2048  _GLIBCXX_SIMD_INTRINSIC constexpr _R
2049  __extract(_Tp __in)
2050  {
2051  using value_type = typename _TVT::value_type;
2052 #if _GLIBCXX_SIMD_X86INTRIN // {{{
2053  if constexpr (sizeof(_Tp) == 64 && _SplitBy == 4 && _Offset > 0)
2054  {
2055  if constexpr (__have_avx512dq && is_same_v<double, value_type>)
2056  return _mm512_extractf64x2_pd(__to_intrin(__in), _Offset);
2057  else if constexpr (is_floating_point_v<value_type>)
2058  return __vector_bitcast<value_type>(
2059  _mm512_extractf32x4_ps(__intrin_bitcast<__m512>(__in), _Offset));
2060  else
2061  return reinterpret_cast<_R>(
2062  _mm512_extracti32x4_epi32(__intrin_bitcast<__m512i>(__in),
2063  _Offset));
2064  }
2065  else
2066 #endif // _GLIBCXX_SIMD_X86INTRIN }}}
2067  {
2068 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2069  using _W = conditional_t<
2070  is_floating_point_v<value_type>, double,
2071  conditional_t<(sizeof(_R) >= 16), long long, value_type>>;
2072  static_assert(sizeof(_R) % sizeof(_W) == 0);
2073  constexpr int __return_width = sizeof(_R) / sizeof(_W);
2074  using _Up = __vector_type_t<_W, __return_width>;
2075  const auto __x = __vector_bitcast<_W>(__in);
2076 #else
2077  constexpr int __return_width = _TVT::_S_full_size / _SplitBy;
2078  using _Up = _R;
2079  const __vector_type_t<value_type, _TVT::_S_full_size>& __x
2080  = __in; // only needed for _Tp = _SimdWrapper<value_type, _Np>
2081 #endif
2082  constexpr int _O = _Offset * __return_width;
2083  return __call_with_subscripts<__return_width, _O>(
2084  __x, [](auto... __entries) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2085  return reinterpret_cast<_R>(_Up{__entries...});
2086  });
2087  }
2088  }
2089 
2090 // }}}
2091 // __lo/__hi64[z]{{{
2092 template <typename _Tp,
2093  typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2094  _GLIBCXX_SIMD_INTRINSIC constexpr _R
2095  __lo64(_Tp __x)
2096  {
2097  _R __r{};
2098  __builtin_memcpy(&__r, &__x, 8);
2099  return __r;
2100  }
2101 
2102 template <typename _Tp,
2103  typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2104  _GLIBCXX_SIMD_INTRINSIC constexpr _R
2105  __hi64(_Tp __x)
2106  {
2107  static_assert(sizeof(_Tp) == 16, "use __hi64z if you meant it");
2108  _R __r{};
2109  __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
2110  return __r;
2111  }
2112 
2113 template <typename _Tp,
2114  typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2115  _GLIBCXX_SIMD_INTRINSIC constexpr _R
2116  __hi64z([[maybe_unused]] _Tp __x)
2117  {
2118  _R __r{};
2119  if constexpr (sizeof(_Tp) == 16)
2120  __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
2121  return __r;
2122  }
2123 
2124 // }}}
2125 // __lo/__hi128{{{
2126 template <typename _Tp>
2127  _GLIBCXX_SIMD_INTRINSIC constexpr auto
2128  __lo128(_Tp __x)
2129  { return __extract<0, sizeof(_Tp) / 16>(__x); }
2130 
2131 template <typename _Tp>
2132  _GLIBCXX_SIMD_INTRINSIC constexpr auto
2133  __hi128(_Tp __x)
2134  {
2135  static_assert(sizeof(__x) == 32);
2136  return __extract<1, 2>(__x);
2137  }
2138 
2139 // }}}
2140 // __lo/__hi256{{{
2141 template <typename _Tp>
2142  _GLIBCXX_SIMD_INTRINSIC constexpr auto
2143  __lo256(_Tp __x)
2144  {
2145  static_assert(sizeof(__x) == 64);
2146  return __extract<0, 2>(__x);
2147  }
2148 
2149 template <typename _Tp>
2150  _GLIBCXX_SIMD_INTRINSIC constexpr auto
2151  __hi256(_Tp __x)
2152  {
2153  static_assert(sizeof(__x) == 64);
2154  return __extract<1, 2>(__x);
2155  }
2156 
2157 // }}}
2158 // __auto_bitcast{{{
2159 template <typename _Tp>
2160  struct _AutoCast
2161  {
2162  static_assert(__is_vector_type_v<_Tp>);
2163 
2164  const _Tp __x;
2165 
2166  template <typename _Up, typename _UVT = _VectorTraits<_Up>>
2167  _GLIBCXX_SIMD_INTRINSIC constexpr operator _Up() const
2168  { return __intrin_bitcast<typename _UVT::type>(__x); }
2169  };
2170 
2171 template <typename _Tp>
2172  _GLIBCXX_SIMD_INTRINSIC constexpr _AutoCast<_Tp>
2173  __auto_bitcast(const _Tp& __x)
2174  { return {__x}; }
2175 
2176 template <typename _Tp, size_t _Np>
2177  _GLIBCXX_SIMD_INTRINSIC constexpr
2178  _AutoCast<typename _SimdWrapper<_Tp, _Np>::_BuiltinType>
2179  __auto_bitcast(const _SimdWrapper<_Tp, _Np>& __x)
2180  { return {__x._M_data}; }
2181 
2182 // }}}
2183 // ^^^ ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- ^^^
2184 
2185 #if _GLIBCXX_SIMD_HAVE_SSE_ABI
2186 // __bool_storage_member_type{{{
2187 #if _GLIBCXX_SIMD_HAVE_AVX512F && _GLIBCXX_SIMD_X86INTRIN
2188 template <size_t _Size>
2189  struct __bool_storage_member_type
2190  {
2191  static_assert((_Size & (_Size - 1)) != 0,
2192  "This trait may only be used for non-power-of-2 sizes. "
2193  "Power-of-2 sizes must be specialized.");
2194  using type =
2195  typename __bool_storage_member_type<std::__bit_ceil(_Size)>::type;
2196  };
2197 
2198 template <>
2199  struct __bool_storage_member_type<1> { using type = bool; };
2200 
2201 template <>
2202  struct __bool_storage_member_type<2> { using type = __mmask8; };
2203 
2204 template <>
2205  struct __bool_storage_member_type<4> { using type = __mmask8; };
2206 
2207 template <>
2208  struct __bool_storage_member_type<8> { using type = __mmask8; };
2209 
2210 template <>
2211  struct __bool_storage_member_type<16> { using type = __mmask16; };
2212 
2213 template <>
2214  struct __bool_storage_member_type<32> { using type = __mmask32; };
2215 
2216 template <>
2217  struct __bool_storage_member_type<64> { using type = __mmask64; };
2218 #endif // _GLIBCXX_SIMD_HAVE_AVX512F
2219 
2220 // }}}
2221 // __intrinsic_type (x86){{{
2222 // the following excludes bool via __is_vectorizable
2223 #if _GLIBCXX_SIMD_HAVE_SSE
2224 template <typename _Tp, size_t _Bytes>
2225  struct __intrinsic_type<_Tp, _Bytes, enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 64>>
2226  {
2227  static_assert(!is_same_v<_Tp, long double>,
2228  "no __intrinsic_type support for long double on x86");
2229 
2230  static constexpr size_t _S_VBytes = _Bytes <= 16 ? 16 : _Bytes <= 32 ? 32 : 64;
2231 
2232  using type [[__gnu__::__vector_size__(_S_VBytes)]]
2233  = conditional_t<is_integral_v<_Tp>, long long int, _Tp>;
2234  };
2235 #endif // _GLIBCXX_SIMD_HAVE_SSE
2236 
2237 // }}}
2238 #endif // _GLIBCXX_SIMD_HAVE_SSE_ABI
2239 // __intrinsic_type (ARM){{{
2240 #if _GLIBCXX_SIMD_HAVE_NEON
2241 template <>
2242  struct __intrinsic_type<float, 8, void>
2243  { using type = float32x2_t; };
2244 
2245 template <>
2246  struct __intrinsic_type<float, 16, void>
2247  { using type = float32x4_t; };
2248 
2249 template <>
2250  struct __intrinsic_type<double, 8, void>
2251  {
2252 #if _GLIBCXX_SIMD_HAVE_NEON_A64
2253  using type = float64x1_t;
2254 #endif
2255  };
2256 
2257 template <>
2258  struct __intrinsic_type<double, 16, void>
2259  {
2260 #if _GLIBCXX_SIMD_HAVE_NEON_A64
2261  using type = float64x2_t;
2262 #endif
2263  };
2264 
2265 #define _GLIBCXX_SIMD_ARM_INTRIN(_Bits, _Np) \
2266 template <> \
2267  struct __intrinsic_type<__int_with_sizeof_t<_Bits / 8>, \
2268  _Np * _Bits / 8, void> \
2269  { using type = int##_Bits##x##_Np##_t; }; \
2270 template <> \
2271  struct __intrinsic_type<make_unsigned_t<__int_with_sizeof_t<_Bits / 8>>, \
2272  _Np * _Bits / 8, void> \
2273  { using type = uint##_Bits##x##_Np##_t; }
2274 _GLIBCXX_SIMD_ARM_INTRIN(8, 8);
2275 _GLIBCXX_SIMD_ARM_INTRIN(8, 16);
2276 _GLIBCXX_SIMD_ARM_INTRIN(16, 4);
2277 _GLIBCXX_SIMD_ARM_INTRIN(16, 8);
2278 _GLIBCXX_SIMD_ARM_INTRIN(32, 2);
2279 _GLIBCXX_SIMD_ARM_INTRIN(32, 4);
2280 _GLIBCXX_SIMD_ARM_INTRIN(64, 1);
2281 _GLIBCXX_SIMD_ARM_INTRIN(64, 2);
2282 #undef _GLIBCXX_SIMD_ARM_INTRIN
2283 
2284 template <typename _Tp, size_t _Bytes>
2285  struct __intrinsic_type<_Tp, _Bytes, enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2286  {
2287  static constexpr int _SVecBytes = _Bytes <= 8 ? 8 : 16;
2288 
2289  using _Ip = __int_for_sizeof_t<_Tp>;
2290 
2291  using _Up = conditional_t<
2292  is_floating_point_v<_Tp>, _Tp,
2293  conditional_t<is_unsigned_v<_Tp>, make_unsigned_t<_Ip>, _Ip>>;
2294 
2295  static_assert(!is_same_v<_Tp, _Up> || _SVecBytes != _Bytes,
2296  "should use explicit specialization above");
2297 
2298  using type = typename __intrinsic_type<_Up, _SVecBytes>::type;
2299  };
2300 #endif // _GLIBCXX_SIMD_HAVE_NEON
2301 
2302 // }}}
2303 // __intrinsic_type (PPC){{{
2304 #ifdef __ALTIVEC__
2305 template <typename _Tp>
2306  struct __intrinsic_type_impl;
2307 
2308 #define _GLIBCXX_SIMD_PPC_INTRIN(_Tp) \
2309  template <> \
2310  struct __intrinsic_type_impl<_Tp> { using type = __vector _Tp; }
2311 _GLIBCXX_SIMD_PPC_INTRIN(float);
2312 #ifdef __VSX__
2313 _GLIBCXX_SIMD_PPC_INTRIN(double);
2314 #endif
2315 _GLIBCXX_SIMD_PPC_INTRIN(signed char);
2316 _GLIBCXX_SIMD_PPC_INTRIN(unsigned char);
2317 _GLIBCXX_SIMD_PPC_INTRIN(signed short);
2318 _GLIBCXX_SIMD_PPC_INTRIN(unsigned short);
2319 _GLIBCXX_SIMD_PPC_INTRIN(signed int);
2320 _GLIBCXX_SIMD_PPC_INTRIN(unsigned int);
2321 #if defined __VSX__ || __SIZEOF_LONG__ == 4
2322 _GLIBCXX_SIMD_PPC_INTRIN(signed long);
2323 _GLIBCXX_SIMD_PPC_INTRIN(unsigned long);
2324 #endif
2325 #ifdef __VSX__
2326 _GLIBCXX_SIMD_PPC_INTRIN(signed long long);
2327 _GLIBCXX_SIMD_PPC_INTRIN(unsigned long long);
2328 #endif
2329 #undef _GLIBCXX_SIMD_PPC_INTRIN
2330 
2331 template <typename _Tp, size_t _Bytes>
2332  struct __intrinsic_type<_Tp, _Bytes, enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2333  {
2334  static constexpr bool _S_is_ldouble = is_same_v<_Tp, long double>;
2335 
2336  // allow _Tp == long double with -mlong-double-64
2337  static_assert(!(_S_is_ldouble && sizeof(long double) > sizeof(double)),
2338  "no __intrinsic_type support for 128-bit floating point on PowerPC");
2339 
2340 #ifndef __VSX__
2341  static_assert(!(is_same_v<_Tp, double>
2342  || (_S_is_ldouble && sizeof(long double) == sizeof(double))),
2343  "no __intrinsic_type support for 64-bit floating point on PowerPC w/o VSX");
2344 #endif
2345 
2346  static constexpr auto __element_type()
2347  {
2348  if constexpr (is_floating_point_v<_Tp>)
2349  {
2350  if constexpr (_S_is_ldouble)
2351  return double {};
2352  else
2353  return _Tp {};
2354  }
2355  else if constexpr (is_signed_v<_Tp>)
2356  {
2357  if constexpr (sizeof(_Tp) == sizeof(_SChar))
2358  return _SChar {};
2359  else if constexpr (sizeof(_Tp) == sizeof(short))
2360  return short {};
2361  else if constexpr (sizeof(_Tp) == sizeof(int))
2362  return int {};
2363  else if constexpr (sizeof(_Tp) == sizeof(_LLong))
2364  return _LLong {};
2365  }
2366  else
2367  {
2368  if constexpr (sizeof(_Tp) == sizeof(_UChar))
2369  return _UChar {};
2370  else if constexpr (sizeof(_Tp) == sizeof(_UShort))
2371  return _UShort {};
2372  else if constexpr (sizeof(_Tp) == sizeof(_UInt))
2373  return _UInt {};
2374  else if constexpr (sizeof(_Tp) == sizeof(_ULLong))
2375  return _ULLong {};
2376  }
2377  }
2378 
2379  using type = typename __intrinsic_type_impl<decltype(__element_type())>::type;
2380  };
2381 #endif // __ALTIVEC__
2382 
2383 // }}}
2384 // _SimdWrapper<bool>{{{1
2385 template <size_t _Width>
2386  struct _SimdWrapper<bool, _Width,
2387  void_t<typename __bool_storage_member_type<_Width>::type>>
2388  {
2389  using _BuiltinType = typename __bool_storage_member_type<_Width>::type;
2390  using value_type = bool;
2391 
2392  static constexpr size_t _S_full_size = sizeof(_BuiltinType) * __CHAR_BIT__;
2393 
2394  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<bool, _S_full_size>
2395  __as_full_vector() const
2396  { return _M_data; }
2397 
2398  _GLIBCXX_SIMD_INTRINSIC constexpr
2399  _SimdWrapper() = default;
2400 
2401  _GLIBCXX_SIMD_INTRINSIC constexpr
2402  _SimdWrapper(_BuiltinType __k) : _M_data(__k) {};
2403 
2404  _GLIBCXX_SIMD_INTRINSIC
2405  operator const _BuiltinType&() const
2406  { return _M_data; }
2407 
2408  _GLIBCXX_SIMD_INTRINSIC
2409  operator _BuiltinType&()
2410  { return _M_data; }
2411 
2412  _GLIBCXX_SIMD_INTRINSIC _BuiltinType
2413  __intrin() const
2414  { return _M_data; }
2415 
2416  _GLIBCXX_SIMD_INTRINSIC constexpr value_type
2417  operator[](size_t __i) const
2418  { return _M_data & (_BuiltinType(1) << __i); }
2419 
2420  template <size_t __i>
2421  _GLIBCXX_SIMD_INTRINSIC constexpr value_type
2422  operator[](_SizeConstant<__i>) const
2423  { return _M_data & (_BuiltinType(1) << __i); }
2424 
2425  _GLIBCXX_SIMD_INTRINSIC constexpr void
2426  _M_set(size_t __i, value_type __x)
2427  {
2428  if (__x)
2429  _M_data |= (_BuiltinType(1) << __i);
2430  else
2431  _M_data &= ~(_BuiltinType(1) << __i);
2432  }
2433 
2434  _GLIBCXX_SIMD_INTRINSIC constexpr bool
2435  _M_is_constprop() const
2436  { return __builtin_constant_p(_M_data); }
2437 
2438  _GLIBCXX_SIMD_INTRINSIC constexpr bool
2439  _M_is_constprop_none_of() const
2440  {
2441  if (__builtin_constant_p(_M_data))
2442  {
2443  constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__;
2444  constexpr _BuiltinType __active_mask
2445  = ~_BuiltinType() >> (__nbits - _Width);
2446  return (_M_data & __active_mask) == 0;
2447  }
2448  return false;
2449  }
2450 
2451  _GLIBCXX_SIMD_INTRINSIC constexpr bool
2452  _M_is_constprop_all_of() const
2453  {
2454  if (__builtin_constant_p(_M_data))
2455  {
2456  constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__;
2457  constexpr _BuiltinType __active_mask
2458  = ~_BuiltinType() >> (__nbits - _Width);
2459  return (_M_data & __active_mask) == __active_mask;
2460  }
2461  return false;
2462  }
2463 
2464  _BuiltinType _M_data;
2465  };
2466 
2467 // _SimdWrapperBase{{{1
2468 template <bool _MustZeroInitPadding, typename _BuiltinType>
2469  struct _SimdWrapperBase;
2470 
2471 template <typename _BuiltinType>
2472  struct _SimdWrapperBase<false, _BuiltinType> // no padding or no SNaNs
2473  {
2474  _GLIBCXX_SIMD_INTRINSIC constexpr
2475  _SimdWrapperBase() = default;
2476 
2477  _GLIBCXX_SIMD_INTRINSIC constexpr
2478  _SimdWrapperBase(_BuiltinType __init) : _M_data(__init) {}
2479 
2480  _BuiltinType _M_data;
2481  };
2482 
2483 template <typename _BuiltinType>
2484  struct _SimdWrapperBase<true, _BuiltinType> // with padding that needs to
2485  // never become SNaN
2486  {
2487  _GLIBCXX_SIMD_INTRINSIC constexpr
2488  _SimdWrapperBase() : _M_data() {}
2489 
2490  _GLIBCXX_SIMD_INTRINSIC constexpr
2491  _SimdWrapperBase(_BuiltinType __init) : _M_data(__init) {}
2492 
2493  _BuiltinType _M_data;
2494  };
2495 
2496 // }}}
2497 // _SimdWrapper{{{
2498 template <typename _Tp, size_t _Width>
2499  struct _SimdWrapper<
2500  _Tp, _Width,
2501  void_t<__vector_type_t<_Tp, _Width>, __intrinsic_type_t<_Tp, _Width>>>
2502  : _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2503  && sizeof(_Tp) * _Width
2504  == sizeof(__vector_type_t<_Tp, _Width>),
2505  __vector_type_t<_Tp, _Width>>
2506  {
2507  using _Base
2508  = _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2509  && sizeof(_Tp) * _Width
2510  == sizeof(__vector_type_t<_Tp, _Width>),
2511  __vector_type_t<_Tp, _Width>>;
2512 
2513  static_assert(__is_vectorizable_v<_Tp>);
2514  static_assert(_Width >= 2); // 1 doesn't make sense, use _Tp directly then
2515 
2516  using _BuiltinType = __vector_type_t<_Tp, _Width>;
2517  using value_type = _Tp;
2518 
2519  static inline constexpr size_t _S_full_size
2520  = sizeof(_BuiltinType) / sizeof(value_type);
2521  static inline constexpr int _S_size = _Width;
2522  static inline constexpr bool _S_is_partial = _S_full_size != _S_size;
2523 
2524  using _Base::_M_data;
2525 
2526  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _S_full_size>
2527  __as_full_vector() const
2528  { return _M_data; }
2529 
2530  _GLIBCXX_SIMD_INTRINSIC constexpr
2531  _SimdWrapper(initializer_list<_Tp> __init)
2532  : _Base(__generate_from_n_evaluations<_Width, _BuiltinType>(
2533  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2534  return __init.begin()[__i.value];
2535  })) {}
2536 
2537  _GLIBCXX_SIMD_INTRINSIC constexpr
2538  _SimdWrapper() = default;
2539 
2540  _GLIBCXX_SIMD_INTRINSIC constexpr
2541  _SimdWrapper(const _SimdWrapper&) = default;
2542 
2543  _GLIBCXX_SIMD_INTRINSIC constexpr
2544  _SimdWrapper(_SimdWrapper&&) = default;
2545 
2546  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2547  operator=(const _SimdWrapper&) = default;
2548 
2549  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2550  operator=(_SimdWrapper&&) = default;
2551 
2552  template <typename _V, typename = enable_if_t<disjunction_v<
2553  is_same<_V, __vector_type_t<_Tp, _Width>>,
2554  is_same<_V, __intrinsic_type_t<_Tp, _Width>>>>>
2555  _GLIBCXX_SIMD_INTRINSIC constexpr
2556  _SimdWrapper(_V __x)
2557  // __vector_bitcast can convert e.g. __m128 to __vector(2) float
2558  : _Base(__vector_bitcast<_Tp, _Width>(__x)) {}
2559 
2560  template <typename... _As,
2561  typename = enable_if_t<((is_same_v<simd_abi::scalar, _As> && ...)
2562  && sizeof...(_As) <= _Width)>>
2563  _GLIBCXX_SIMD_INTRINSIC constexpr
2564  operator _SimdTuple<_Tp, _As...>() const
2565  {
2566  return __generate_from_n_evaluations<sizeof...(_As), _SimdTuple<_Tp, _As...>>(
2567  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
2568  { return _M_data[int(__i)]; });
2569  }
2570 
2571  _GLIBCXX_SIMD_INTRINSIC constexpr
2572  operator const _BuiltinType&() const
2573  { return _M_data; }
2574 
2575  _GLIBCXX_SIMD_INTRINSIC constexpr
2576  operator _BuiltinType&()
2577  { return _M_data; }
2578 
2579  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
2580  operator[](size_t __i) const
2581  { return _M_data[__i]; }
2582 
2583  template <size_t __i>
2584  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
2585  operator[](_SizeConstant<__i>) const
2586  { return _M_data[__i]; }
2587 
2588  _GLIBCXX_SIMD_INTRINSIC constexpr void
2589  _M_set(size_t __i, _Tp __x)
2590  {
2591  if (__builtin_is_constant_evaluated())
2592  _M_data = __generate_from_n_evaluations<_Width, _BuiltinType>([&](auto __j) {
2593  return __j == __i ? __x : _M_data[__j()];
2594  });
2595  else
2596  _M_data[__i] = __x;
2597  }
2598 
2599  _GLIBCXX_SIMD_INTRINSIC
2600  constexpr bool
2601  _M_is_constprop() const
2602  { return __builtin_constant_p(_M_data); }
2603 
2604  _GLIBCXX_SIMD_INTRINSIC constexpr bool
2605  _M_is_constprop_none_of() const
2606  {
2607  if (__builtin_constant_p(_M_data))
2608  {
2609  bool __r = true;
2610  if constexpr (is_floating_point_v<_Tp>)
2611  {
2612  using _Ip = __int_for_sizeof_t<_Tp>;
2613  const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2614  __execute_n_times<_Width>(
2615  [&](auto __i) { __r &= __intdata[__i.value] == _Ip(); });
2616  }
2617  else
2618  __execute_n_times<_Width>(
2619  [&](auto __i) { __r &= _M_data[__i.value] == _Tp(); });
2620  if (__builtin_constant_p(__r))
2621  return __r;
2622  }
2623  return false;
2624  }
2625 
2626  _GLIBCXX_SIMD_INTRINSIC constexpr bool
2627  _M_is_constprop_all_of() const
2628  {
2629  if (__builtin_constant_p(_M_data))
2630  {
2631  bool __r = true;
2632  if constexpr (is_floating_point_v<_Tp>)
2633  {
2634  using _Ip = __int_for_sizeof_t<_Tp>;
2635  const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2636  __execute_n_times<_Width>(
2637  [&](auto __i) { __r &= __intdata[__i.value] == ~_Ip(); });
2638  }
2639  else
2640  __execute_n_times<_Width>(
2641  [&](auto __i) { __r &= _M_data[__i.value] == ~_Tp(); });
2642  if (__builtin_constant_p(__r))
2643  return __r;
2644  }
2645  return false;
2646  }
2647  };
2648 
2649 // }}}
2650 
2651 // __vectorized_sizeof {{{
2652 template <typename _Tp>
2653  constexpr size_t
2654  __vectorized_sizeof()
2655  {
2656  if constexpr (!__is_vectorizable_v<_Tp>)
2657  return 0;
2658 
2659  if constexpr (sizeof(_Tp) <= 8)
2660  {
2661  // X86:
2662  if constexpr (__have_avx512bw)
2663  return 64;
2664  if constexpr (__have_avx512f && sizeof(_Tp) >= 4)
2665  return 64;
2666  if constexpr (__have_avx2)
2667  return 32;
2668  if constexpr (__have_avx && is_floating_point_v<_Tp>)
2669  return 32;
2670  if constexpr (__have_sse2)
2671  return 16;
2672  if constexpr (__have_sse && is_same_v<_Tp, float>)
2673  return 16;
2674  /* The following is too much trouble because of mixed MMX and x87 code.
2675  * While nothing here explicitly calls MMX instructions of registers,
2676  * they are still emitted but no EMMS cleanup is done.
2677  if constexpr (__have_mmx && sizeof(_Tp) <= 4 && is_integral_v<_Tp>)
2678  return 8;
2679  */
2680 
2681  // PowerPC:
2682  if constexpr (__have_power8vec
2683  || (__have_power_vmx && (sizeof(_Tp) < 8))
2684  || (__have_power_vsx && is_floating_point_v<_Tp>) )
2685  return 16;
2686 
2687  // ARM:
2688  if constexpr (__have_neon_a64
2689  || (__have_neon_a32 && !is_same_v<_Tp, double>) )
2690  return 16;
2691  if constexpr (__have_neon
2692  && sizeof(_Tp) < 8
2693  // Only allow fp if the user allows non-ICE559 fp (e.g.
2694  // via -ffast-math). ARMv7 NEON fp is not conforming to
2695  // IEC559.
2696  && (__support_neon_float || !is_floating_point_v<_Tp>))
2697  return 16;
2698  }
2699 
2700  return sizeof(_Tp);
2701  }
2702 
2703 // }}}
2704 namespace simd_abi {
2705 // most of simd_abi is defined in simd_detail.h
2706 template <typename _Tp>
2707  inline constexpr int max_fixed_size
2708  = (__have_avx512bw && sizeof(_Tp) == 1) ? 64 : 32;
2709 
2710 // compatible {{{
2711 #if defined __x86_64__ || defined __aarch64__
2712 template <typename _Tp>
2713  using compatible = conditional_t<(sizeof(_Tp) <= 8), _VecBuiltin<16>, scalar>;
2714 #elif defined __ARM_NEON
2715 // FIXME: not sure, probably needs to be scalar (or dependent on the hard-float
2716 // ABI?)
2717 template <typename _Tp>
2718  using compatible
2719  = conditional_t<(sizeof(_Tp) < 8
2720  && (__support_neon_float || !is_floating_point_v<_Tp>)),
2721  _VecBuiltin<16>, scalar>;
2722 #else
2723 template <typename>
2724  using compatible = scalar;
2725 #endif
2726 
2727 // }}}
2728 // native {{{
2729 template <typename _Tp>
2730  constexpr auto
2731  __determine_native_abi()
2732  {
2733  constexpr size_t __bytes = __vectorized_sizeof<_Tp>();
2734  if constexpr (__bytes == sizeof(_Tp))
2735  return static_cast<scalar*>(nullptr);
2736  else if constexpr (__have_avx512vl || (__have_avx512f && __bytes == 64))
2737  return static_cast<_VecBltnBtmsk<__bytes>*>(nullptr);
2738  else
2739  return static_cast<_VecBuiltin<__bytes>*>(nullptr);
2740  }
2741 
2742 template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
2743  using native = remove_pointer_t<decltype(__determine_native_abi<_Tp>())>;
2744 
2745 // }}}
2746 // __default_abi {{{
2747 #if defined _GLIBCXX_SIMD_DEFAULT_ABI
2748 template <typename _Tp>
2749  using __default_abi = _GLIBCXX_SIMD_DEFAULT_ABI<_Tp>;
2750 #else
2751 template <typename _Tp>
2752  using __default_abi = compatible<_Tp>;
2753 #endif
2754 
2755 // }}}
2756 } // namespace simd_abi
2757 
2758 // traits {{{1
2759 template <typename _Tp>
2760  struct is_simd_flag_type
2761  : false_type
2762  {};
2763 
2764 template <>
2765  struct is_simd_flag_type<element_aligned_tag>
2766  : true_type
2767  {};
2768 
2769 template <>
2770  struct is_simd_flag_type<vector_aligned_tag>
2771  : true_type
2772  {};
2773 
2774 template <size_t _Np>
2775  struct is_simd_flag_type<overaligned_tag<_Np>>
2776  : __bool_constant<(_Np > 0) and __has_single_bit(_Np)>
2777  {};
2778 
2779 template <typename _Tp>
2780  inline constexpr bool is_simd_flag_type_v = is_simd_flag_type<_Tp>::value;
2781 
2782 template <typename _Tp, typename = enable_if_t<is_simd_flag_type_v<_Tp>>>
2783  using _IsSimdFlagType = _Tp;
2784 
2785 // is_abi_tag {{{2
2786 template <typename _Tp, typename = void_t<>>
2787  struct is_abi_tag : false_type {};
2788 
2789 template <typename _Tp>
2790  struct is_abi_tag<_Tp, void_t<typename _Tp::_IsValidAbiTag>>
2791  : public _Tp::_IsValidAbiTag {};
2792 
2793 template <typename _Tp>
2794  inline constexpr bool is_abi_tag_v = is_abi_tag<_Tp>::value;
2795 
2796 // is_simd(_mask) {{{2
2797 template <typename _Tp>
2798  struct is_simd : public false_type {};
2799 
2800 template <typename _Tp>
2801  inline constexpr bool is_simd_v = is_simd<_Tp>::value;
2802 
2803 template <typename _Tp>
2804  struct is_simd_mask : public false_type {};
2805 
2806 template <typename _Tp>
2807 inline constexpr bool is_simd_mask_v = is_simd_mask<_Tp>::value;
2808 
2809 // simd_size {{{2
2810 template <typename _Tp, typename _Abi, typename = void>
2811  struct __simd_size_impl {};
2812 
2813 template <typename _Tp, typename _Abi>
2814  struct __simd_size_impl<
2815  _Tp, _Abi,
2816  enable_if_t<conjunction_v<__is_vectorizable<_Tp>, is_abi_tag<_Abi>>>>
2817  : _SizeConstant<_Abi::template _S_size<_Tp>> {};
2818 
2819 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2820  struct simd_size : __simd_size_impl<_Tp, _Abi> {};
2821 
2822 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2823  inline constexpr size_t simd_size_v = simd_size<_Tp, _Abi>::value;
2824 
2825 // simd_abi::deduce {{{2
2826 template <typename _Tp, size_t _Np, typename = void>
2827  struct __deduce_impl;
2828 
2829 namespace simd_abi {
2830 /**
2831  * @tparam _Tp The requested `value_type` for the elements.
2832  * @tparam _Np The requested number of elements.
2833  * @tparam _Abis This parameter is ignored, since this implementation cannot
2834  * make any use of it. Either __a good native ABI is matched and used as `type`
2835  * alias, or the `fixed_size<_Np>` ABI is used, which internally is built from
2836  * the best matching native ABIs.
2837  */
2838 template <typename _Tp, size_t _Np, typename...>
2839  struct deduce : __deduce_impl<_Tp, _Np> {};
2840 
2841 template <typename _Tp, size_t _Np, typename... _Abis>
2842  using deduce_t = typename deduce<_Tp, _Np, _Abis...>::type;
2843 } // namespace simd_abi
2844 
2845 // }}}2
2846 // rebind_simd {{{2
2847 template <typename _Tp, typename _V, typename = void>
2848  struct rebind_simd;
2849 
2850 template <typename _Tp, typename _Up, typename _Abi>
2851  struct rebind_simd<_Tp, simd<_Up, _Abi>,
2852  void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2853  { using type = simd<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>; };
2854 
2855 template <typename _Tp, typename _Up, typename _Abi>
2856  struct rebind_simd<_Tp, simd_mask<_Up, _Abi>,
2857  void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2858  { using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>; };
2859 
2860 template <typename _Tp, typename _V>
2861  using rebind_simd_t = typename rebind_simd<_Tp, _V>::type;
2862 
2863 // resize_simd {{{2
2864 template <int _Np, typename _V, typename = void>
2865  struct resize_simd;
2866 
2867 template <int _Np, typename _Tp, typename _Abi>
2868  struct resize_simd<_Np, simd<_Tp, _Abi>, void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2869  { using type = simd<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2870 
2871 template <int _Np, typename _Tp, typename _Abi>
2872  struct resize_simd<_Np, simd_mask<_Tp, _Abi>, void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2873  { using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2874 
2875 template <int _Np, typename _V>
2876  using resize_simd_t = typename resize_simd<_Np, _V>::type;
2877 
2878 // }}}2
2879 // memory_alignment {{{2
2880 template <typename _Tp, typename _Up = typename _Tp::value_type>
2881  struct memory_alignment
2882  : public _SizeConstant<vector_aligned_tag::_S_alignment<_Tp, _Up>> {};
2883 
2884 template <typename _Tp, typename _Up = typename _Tp::value_type>
2885  inline constexpr size_t memory_alignment_v = memory_alignment<_Tp, _Up>::value;
2886 
2887 // class template simd [simd] {{{1
2888 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2889  class simd;
2890 
2891 template <typename _Tp, typename _Abi>
2892  struct is_simd<simd<_Tp, _Abi>> : public true_type {};
2893 
2894 template <typename _Tp>
2895  using native_simd = simd<_Tp, simd_abi::native<_Tp>>;
2896 
2897 template <typename _Tp, int _Np>
2898  using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
2899 
2900 template <typename _Tp, size_t _Np>
2901  using __deduced_simd = simd<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2902 
2903 // class template simd_mask [simd_mask] {{{1
2904 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2905  class simd_mask;
2906 
2907 template <typename _Tp, typename _Abi>
2908  struct is_simd_mask<simd_mask<_Tp, _Abi>> : public true_type {};
2909 
2910 template <typename _Tp>
2911  using native_simd_mask = simd_mask<_Tp, simd_abi::native<_Tp>>;
2912 
2913 template <typename _Tp, int _Np>
2914  using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_Np>>;
2915 
2916 template <typename _Tp, size_t _Np>
2917  using __deduced_simd_mask = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2918 
2919 // casts [simd.casts] {{{1
2920 // static_simd_cast {{{2
2921 template <typename _Tp, typename _Up, typename _Ap, bool = is_simd_v<_Tp>, typename = void>
2922  struct __static_simd_cast_return_type;
2923 
2924 template <typename _Tp, typename _A0, typename _Up, typename _Ap>
2925  struct __static_simd_cast_return_type<simd_mask<_Tp, _A0>, _Up, _Ap, false, void>
2926  : __static_simd_cast_return_type<simd<_Tp, _A0>, _Up, _Ap> {};
2927 
2928 template <typename _Tp, typename _Up, typename _Ap>
2929  struct __static_simd_cast_return_type<
2930  _Tp, _Up, _Ap, true, enable_if_t<_Tp::size() == simd_size_v<_Up, _Ap>>>
2931  { using type = _Tp; };
2932 
2933 template <typename _Tp, typename _Ap>
2934  struct __static_simd_cast_return_type<_Tp, _Tp, _Ap, false,
2935 #ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2936  enable_if_t<__is_vectorizable_v<_Tp>>
2937 #else
2938  void
2939 #endif
2940  >
2941  { using type = simd<_Tp, _Ap>; };
2942 
2943 template <typename _Tp, typename = void>
2944  struct __safe_make_signed { using type = _Tp;};
2945 
2946 template <typename _Tp>
2947  struct __safe_make_signed<_Tp, enable_if_t<is_integral_v<_Tp>>>
2948  {
2949  // the extra make_unsigned_t is because of PR85951
2950  using type = make_signed_t<make_unsigned_t<_Tp>>;
2951  };
2952 
2953 template <typename _Tp>
2954  using safe_make_signed_t = typename __safe_make_signed<_Tp>::type;
2955 
2956 template <typename _Tp, typename _Up, typename _Ap>
2957  struct __static_simd_cast_return_type<_Tp, _Up, _Ap, false,
2958 #ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2959  enable_if_t<__is_vectorizable_v<_Tp>>
2960 #else
2961  void
2962 #endif
2963  >
2964  {
2965  using type = conditional_t<
2966  (is_integral_v<_Up> && is_integral_v<_Tp> &&
2967 #ifndef _GLIBCXX_SIMD_FIX_P2TS_ISSUE65
2968  is_signed_v<_Up> != is_signed_v<_Tp> &&
2969 #endif
2970  is_same_v<safe_make_signed_t<_Up>, safe_make_signed_t<_Tp>>),
2971  simd<_Tp, _Ap>, fixed_size_simd<_Tp, simd_size_v<_Up, _Ap>>>;
2972  };
2973 
2974 template <typename _Tp, typename _Up, typename _Ap,
2975  typename _R
2976  = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
2977  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _R
2978  static_simd_cast(const simd<_Up, _Ap>& __x)
2979  {
2980  if constexpr (is_same<_R, simd<_Up, _Ap>>::value)
2981  return __x;
2982  else
2983  {
2984  _SimdConverter<_Up, _Ap, typename _R::value_type, typename _R::abi_type>
2985  __c;
2986  return _R(__private_init, __c(__data(__x)));
2987  }
2988  }
2989 
2990 namespace __proposed {
2991 template <typename _Tp, typename _Up, typename _Ap,
2992  typename _R
2993  = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
2994  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR typename _R::mask_type
2995  static_simd_cast(const simd_mask<_Up, _Ap>& __x)
2996  {
2997  using _RM = typename _R::mask_type;
2998  return {__private_init, _RM::abi_type::_MaskImpl::template _S_convert<
2999  typename _RM::simd_type::value_type>(__x)};
3000  }
3001 } // namespace __proposed
3002 
3003 // simd_cast {{{2
3004 template <typename _Tp, typename _Up, typename _Ap,
3005  typename _To = __value_type_or_identity_t<_Tp>>
3006  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto
3007  simd_cast(const simd<_ValuePreserving<_Up, _To>, _Ap>& __x)
3008  -> decltype(static_simd_cast<_Tp>(__x))
3009  { return static_simd_cast<_Tp>(__x); }
3010 
3011 namespace __proposed {
3012 template <typename _Tp, typename _Up, typename _Ap,
3013  typename _To = __value_type_or_identity_t<_Tp>>
3014  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto
3015  simd_cast(const simd_mask<_ValuePreserving<_Up, _To>, _Ap>& __x)
3016  -> decltype(static_simd_cast<_Tp>(__x))
3017  { return static_simd_cast<_Tp>(__x); }
3018 } // namespace __proposed
3019 
3020 // }}}2
3021 // resizing_simd_cast {{{
3022 namespace __proposed {
3023 /* Proposed spec:
3024 
3025 template <class T, class U, class Abi>
3026 T resizing_simd_cast(const simd<U, Abi>& x)
3027 
3028 p1 Constraints:
3029  - is_simd_v<T> is true and
3030  - T::value_type is the same type as U
3031 
3032 p2 Returns:
3033  A simd object with the i^th element initialized to x[i] for all i in the
3034  range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger
3035  than simd_size_v<U, Abi>, the remaining elements are value-initialized.
3036 
3037 template <class T, class U, class Abi>
3038 T resizing_simd_cast(const simd_mask<U, Abi>& x)
3039 
3040 p1 Constraints: is_simd_mask_v<T> is true
3041 
3042 p2 Returns:
3043  A simd_mask object with the i^th element initialized to x[i] for all i in
3044 the range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger
3045  than simd_size_v<U, Abi>, the remaining elements are initialized to false.
3046 
3047  */
3048 
3049 template <typename _Tp, typename _Up, typename _Ap>
3050  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR enable_if_t<
3051  conjunction_v<is_simd<_Tp>, is_same<typename _Tp::value_type, _Up>>, _Tp>
3052  resizing_simd_cast(const simd<_Up, _Ap>& __x)
3053  {
3054  if constexpr (is_same_v<typename _Tp::abi_type, _Ap>)
3055  return __x;
3056  else if (__builtin_is_constant_evaluated())
3057  return _Tp([&](auto __i) constexpr {
3058  return __i < simd_size_v<_Up, _Ap> ? __x[__i] : _Up();
3059  });
3060  else if constexpr (simd_size_v<_Up, _Ap> == 1)
3061  {
3062  _Tp __r{};
3063  __r[0] = __x[0];
3064  return __r;
3065  }
3066  else if constexpr (_Tp::size() == 1)
3067  return __x[0];
3068  else if constexpr (sizeof(_Tp) == sizeof(__x)
3069  && !__is_fixed_size_abi_v<_Ap>)
3070  return {__private_init,
3071  __vector_bitcast<typename _Tp::value_type, _Tp::size()>(
3072  _Ap::_S_masked(__data(__x))._M_data)};
3073  else
3074  {
3075  _Tp __r{};
3076  __builtin_memcpy(&__data(__r), &__data(__x),
3077  sizeof(_Up)
3078  * std::min(_Tp::size(), simd_size_v<_Up, _Ap>));
3079  return __r;
3080  }
3081  }
3082 
3083 template <typename _Tp, typename _Up, typename _Ap>
3084  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3085  enable_if_t<is_simd_mask_v<_Tp>, _Tp>
3086  resizing_simd_cast(const simd_mask<_Up, _Ap>& __x)
3087  {
3088  return {__private_init, _Tp::abi_type::_MaskImpl::template _S_convert<
3089  typename _Tp::simd_type::value_type>(__x)};
3090  }
3091 } // namespace __proposed
3092 
3093 // }}}
3094 // to_fixed_size {{{2
3095 template <typename _Tp, int _Np>
3096  _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, _Np>
3097  to_fixed_size(const fixed_size_simd<_Tp, _Np>& __x)
3098  { return __x; }
3099 
3100 template <typename _Tp, int _Np>
3101  _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, _Np>
3102  to_fixed_size(const fixed_size_simd_mask<_Tp, _Np>& __x)
3103  { return __x; }
3104 
3105 template <typename _Tp, typename _Ap>
3106  _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, simd_size_v<_Tp, _Ap>>
3107  to_fixed_size(const simd<_Tp, _Ap>& __x)
3108  {
3109  using _Rp = fixed_size_simd<_Tp, simd_size_v<_Tp, _Ap>>;
3110  return _Rp([&__x](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3111  }
3112 
3113 template <typename _Tp, typename _Ap>
3114  _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, simd_size_v<_Tp, _Ap>>
3115  to_fixed_size(const simd_mask<_Tp, _Ap>& __x)
3116  {
3117  return {__private_init,
3118  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; }};
3119  }
3120 
3121 // to_native {{{2
3122 template <typename _Tp, int _Np>
3123  _GLIBCXX_SIMD_INTRINSIC
3124  enable_if_t<(_Np == native_simd<_Tp>::size()), native_simd<_Tp>>
3125  to_native(const fixed_size_simd<_Tp, _Np>& __x)
3126  {
3127  alignas(memory_alignment_v<native_simd<_Tp>>) _Tp __mem[_Np];
3128  __x.copy_to(__mem, vector_aligned);
3129  return {__mem, vector_aligned};
3130  }
3131 
3132 template <typename _Tp, int _Np>
3133  _GLIBCXX_SIMD_INTRINSIC
3134  enable_if_t<(_Np == native_simd_mask<_Tp>::size()), native_simd_mask<_Tp>>
3135  to_native(const fixed_size_simd_mask<_Tp, _Np>& __x)
3136  {
3137  return native_simd_mask<_Tp>(
3138  __private_init,
3139  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3140  }
3141 
3142 // to_compatible {{{2
3143 template <typename _Tp, int _Np>
3144  _GLIBCXX_SIMD_INTRINSIC enable_if_t<(_Np == simd<_Tp>::size()), simd<_Tp>>
3145  to_compatible(const simd<_Tp, simd_abi::fixed_size<_Np>>& __x)
3146  {
3147  alignas(memory_alignment_v<simd<_Tp>>) _Tp __mem[_Np];
3148  __x.copy_to(__mem, vector_aligned);
3149  return {__mem, vector_aligned};
3150  }
3151 
3152 template <typename _Tp, int _Np>
3153  _GLIBCXX_SIMD_INTRINSIC
3154  enable_if_t<(_Np == simd_mask<_Tp>::size()), simd_mask<_Tp>>
3155  to_compatible(const simd_mask<_Tp, simd_abi::fixed_size<_Np>>& __x)
3156  {
3157  return simd_mask<_Tp>(
3158  __private_init,
3159  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3160  }
3161 
3162 // masked assignment [simd_mask.where] {{{1
3163 
3164 // where_expression {{{1
3165 // const_where_expression<M, T> {{{2
3166 template <typename _M, typename _Tp>
3167  class const_where_expression
3168  {
3169  using _V = _Tp;
3170  static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3171 
3172  struct _Wrapper { using value_type = _V; };
3173 
3174  protected:
3175  using _Impl = typename _V::_Impl;
3176 
3177  using value_type =
3178  typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3179 
3180  _GLIBCXX_SIMD_INTRINSIC friend const _M&
3181  __get_mask(const const_where_expression& __x)
3182  { return __x._M_k; }
3183 
3184  _GLIBCXX_SIMD_INTRINSIC friend const _Tp&
3185  __get_lvalue(const const_where_expression& __x)
3186  { return __x._M_value; }
3187 
3188  const _M& _M_k;
3189  _Tp& _M_value;
3190 
3191  public:
3192  const_where_expression(const const_where_expression&) = delete;
3193 
3194  const_where_expression& operator=(const const_where_expression&) = delete;
3195 
3196  _GLIBCXX_SIMD_INTRINSIC constexpr
3197  const_where_expression(const _M& __kk, const _Tp& dd)
3198  : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3199 
3200  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3201  operator-() const&&
3202  {
3203  return {__private_init,
3204  _Impl::template _S_masked_unary<negate>(__data(_M_k),
3205  __data(_M_value))};
3206  }
3207 
3208  template <typename _Up, typename _Flags>
3209  [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3210  copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&&
3211  {
3212  return {__private_init,
3213  _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3214  _Flags::template _S_apply<_V>(__mem))};
3215  }
3216 
3217  template <typename _Up, typename _Flags>
3218  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3219  copy_to(_LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&&
3220  {
3221  _Impl::_S_masked_store(__data(_M_value),
3222  _Flags::template _S_apply<_V>(__mem),
3223  __data(_M_k));
3224  }
3225  };
3226 
3227 // const_where_expression<bool, T> {{{2
3228 template <typename _Tp>
3229  class const_where_expression<bool, _Tp>
3230  {
3231  using _M = bool;
3232  using _V = _Tp;
3233 
3234  static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3235 
3236  struct _Wrapper { using value_type = _V; };
3237 
3238  protected:
3239  using value_type
3240  = typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3241 
3242  _GLIBCXX_SIMD_INTRINSIC friend const _M&
3243  __get_mask(const const_where_expression& __x)
3244  { return __x._M_k; }
3245 
3246  _GLIBCXX_SIMD_INTRINSIC friend const _Tp&
3247  __get_lvalue(const const_where_expression& __x)
3248  { return __x._M_value; }
3249 
3250  const bool _M_k;
3251  _Tp& _M_value;
3252 
3253  public:
3254  const_where_expression(const const_where_expression&) = delete;
3255  const_where_expression& operator=(const const_where_expression&) = delete;
3256 
3257  _GLIBCXX_SIMD_INTRINSIC constexpr
3258  const_where_expression(const bool __kk, const _Tp& dd)
3259  : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3260 
3261  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3262  operator-() const&&
3263  { return _M_k ? -_M_value : _M_value; }
3264 
3265  template <typename _Up, typename _Flags>
3266  [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3267  copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&&
3268  { return _M_k ? static_cast<_V>(__mem[0]) : _M_value; }
3269 
3270  template <typename _Up, typename _Flags>
3271  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3272  copy_to(_LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&&
3273  {
3274  if (_M_k)
3275  __mem[0] = _M_value;
3276  }
3277  };
3278 
3279 // where_expression<M, T> {{{2
3280 template <typename _M, typename _Tp>
3281  class where_expression : public const_where_expression<_M, _Tp>
3282  {
3283  using _Impl = typename const_where_expression<_M, _Tp>::_Impl;
3284 
3285  static_assert(!is_const<_Tp>::value,
3286  "where_expression may only be instantiated with __a non-const "
3287  "_Tp parameter");
3288 
3289  using typename const_where_expression<_M, _Tp>::value_type;
3290  using const_where_expression<_M, _Tp>::_M_k;
3291  using const_where_expression<_M, _Tp>::_M_value;
3292 
3293  static_assert(
3294  is_same<typename _M::abi_type, typename _Tp::abi_type>::value, "");
3295  static_assert(_M::size() == _Tp::size(), "");
3296 
3297  _GLIBCXX_SIMD_INTRINSIC friend constexpr _Tp&
3298  __get_lvalue(where_expression& __x)
3299  { return __x._M_value; }
3300 
3301  public:
3302  where_expression(const where_expression&) = delete;
3303  where_expression& operator=(const where_expression&) = delete;
3304 
3305  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3306  where_expression(const _M& __kk, _Tp& dd)
3307  : const_where_expression<_M, _Tp>(__kk, dd) {}
3308 
3309  template <typename _Up>
3310  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3311  operator=(_Up&& __x) &&
3312  {
3313  _Impl::_S_masked_assign(__data(_M_k), __data(_M_value),
3314  __to_value_type_or_member_type<_Tp>(
3315  static_cast<_Up&&>(__x)));
3316  }
3317 
3318 #define _GLIBCXX_SIMD_OP_(__op, __name) \
3319  template <typename _Up> \
3320  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void \
3321  operator __op##=(_Up&& __x)&& \
3322  { \
3323  _Impl::template _S_masked_cassign( \
3324  __data(_M_k), __data(_M_value), \
3325  __to_value_type_or_member_type<_Tp>(static_cast<_Up&&>(__x)), \
3326  [](auto __impl, auto __lhs, auto __rhs) \
3327  constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \
3328  { return __impl.__name(__lhs, __rhs); }); \
3329  } \
3330  static_assert(true)
3331  _GLIBCXX_SIMD_OP_(+, _S_plus);
3332  _GLIBCXX_SIMD_OP_(-, _S_minus);
3333  _GLIBCXX_SIMD_OP_(*, _S_multiplies);
3334  _GLIBCXX_SIMD_OP_(/, _S_divides);
3335  _GLIBCXX_SIMD_OP_(%, _S_modulus);
3336  _GLIBCXX_SIMD_OP_(&, _S_bit_and);
3337  _GLIBCXX_SIMD_OP_(|, _S_bit_or);
3338  _GLIBCXX_SIMD_OP_(^, _S_bit_xor);
3339  _GLIBCXX_SIMD_OP_(<<, _S_shift_left);
3340  _GLIBCXX_SIMD_OP_(>>, _S_shift_right);
3341 #undef _GLIBCXX_SIMD_OP_
3342 
3343  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3344  operator++() &&
3345  {
3346  __data(_M_value)
3347  = _Impl::template _S_masked_unary<__increment>(__data(_M_k), __data(_M_value));
3348  }
3349 
3350  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3351  operator++(int) &&
3352  {
3353  __data(_M_value)
3354  = _Impl::template _S_masked_unary<__increment>(__data(_M_k), __data(_M_value));
3355  }
3356 
3357  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3358  operator--() &&
3359  {
3360  __data(_M_value)
3361  = _Impl::template _S_masked_unary<__decrement>(__data(_M_k), __data(_M_value));
3362  }
3363 
3364  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3365  operator--(int) &&
3366  {
3367  __data(_M_value)
3368  = _Impl::template _S_masked_unary<__decrement>(__data(_M_k), __data(_M_value));
3369  }
3370 
3371  // intentionally hides const_where_expression::copy_from
3372  template <typename _Up, typename _Flags>
3373  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3374  copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) &&
3375  {
3376  __data(_M_value) = _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3377  _Flags::template _S_apply<_Tp>(__mem));
3378  }
3379  };
3380 
3381 // where_expression<bool, T> {{{2
3382 template <typename _Tp>
3383  class where_expression<bool, _Tp>
3384  : public const_where_expression<bool, _Tp>
3385  {
3386  using _M = bool;
3387  using typename const_where_expression<_M, _Tp>::value_type;
3388  using const_where_expression<_M, _Tp>::_M_k;
3389  using const_where_expression<_M, _Tp>::_M_value;
3390 
3391  public:
3392  where_expression(const where_expression&) = delete;
3393  where_expression& operator=(const where_expression&) = delete;
3394 
3395  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3396  where_expression(const _M& __kk, _Tp& dd)
3397  : const_where_expression<_M, _Tp>(__kk, dd) {}
3398 
3399 #define _GLIBCXX_SIMD_OP_(__op) \
3400  template <typename _Up> \
3401  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void \
3402  operator __op(_Up&& __x)&& \
3403  { if (_M_k) _M_value __op static_cast<_Up&&>(__x); }
3404 
3405  _GLIBCXX_SIMD_OP_(=)
3406  _GLIBCXX_SIMD_OP_(+=)
3407  _GLIBCXX_SIMD_OP_(-=)
3408  _GLIBCXX_SIMD_OP_(*=)
3409  _GLIBCXX_SIMD_OP_(/=)
3410  _GLIBCXX_SIMD_OP_(%=)
3411  _GLIBCXX_SIMD_OP_(&=)
3412  _GLIBCXX_SIMD_OP_(|=)
3413  _GLIBCXX_SIMD_OP_(^=)
3414  _GLIBCXX_SIMD_OP_(<<=)
3415  _GLIBCXX_SIMD_OP_(>>=)
3416  #undef _GLIBCXX_SIMD_OP_
3417 
3418  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3419  operator++() &&
3420  { if (_M_k) ++_M_value; }
3421 
3422  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3423  operator++(int) &&
3424  { if (_M_k) ++_M_value; }
3425 
3426  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3427  operator--() &&
3428  { if (_M_k) --_M_value; }
3429 
3430  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3431  operator--(int) &&
3432  { if (_M_k) --_M_value; }
3433 
3434  // intentionally hides const_where_expression::copy_from
3435  template <typename _Up, typename _Flags>
3436  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3437  copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) &&
3438  { if (_M_k) _M_value = __mem[0]; }
3439  };
3440 
3441 // where {{{1
3442 template <typename _Tp, typename _Ap>
3443  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3444  where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3445  where(const typename simd<_Tp, _Ap>::mask_type& __k, simd<_Tp, _Ap>& __value)
3446  { return {__k, __value}; }
3447 
3448 template <typename _Tp, typename _Ap>
3449  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3450  const_where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3451  where(const typename simd<_Tp, _Ap>::mask_type& __k, const simd<_Tp, _Ap>& __value)
3452  { return {__k, __value}; }
3453 
3454 template <typename _Tp, typename _Ap>
3455  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3456  where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3457  where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k, simd_mask<_Tp, _Ap>& __value)
3458  { return {__k, __value}; }
3459 
3460 template <typename _Tp, typename _Ap>
3461  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3462  const_where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3463  where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k, const simd_mask<_Tp, _Ap>& __value)
3464  { return {__k, __value}; }
3465 
3466 template <typename _Tp>
3467  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR where_expression<bool, _Tp>
3468  where(_ExactBool __k, _Tp& __value)
3469  { return {__k, __value}; }
3470 
3471 template <typename _Tp>
3472  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR const_where_expression<bool, _Tp>
3473  where(_ExactBool __k, const _Tp& __value)
3474  { return {__k, __value}; }
3475 
3476 template <typename _Tp, typename _Ap>
3477  _GLIBCXX_SIMD_CONSTEXPR void
3478  where(bool __k, simd<_Tp, _Ap>& __value) = delete;
3479 
3480 template <typename _Tp, typename _Ap>
3481  _GLIBCXX_SIMD_CONSTEXPR void
3482  where(bool __k, const simd<_Tp, _Ap>& __value) = delete;
3483 
3484 // proposed mask iterations {{{1
3485 namespace __proposed {
3486 template <size_t _Np>
3487  class where_range
3488  {
3489  const bitset<_Np> __bits;
3490 
3491  public:
3492  where_range(bitset<_Np> __b) : __bits(__b) {}
3493 
3494  class iterator
3495  {
3496  size_t __mask;
3497  size_t __bit;
3498 
3499  _GLIBCXX_SIMD_INTRINSIC void
3500  __next_bit()
3501  { __bit = __builtin_ctzl(__mask); }
3502 
3503  _GLIBCXX_SIMD_INTRINSIC void
3504  __reset_lsb()
3505  {
3506  // 01100100 - 1 = 01100011
3507  __mask &= (__mask - 1);
3508  // __asm__("btr %1,%0" : "+r"(__mask) : "r"(__bit));
3509  }
3510 
3511  public:
3512  iterator(decltype(__mask) __m) : __mask(__m) { __next_bit(); }
3513  iterator(const iterator&) = default;
3514  iterator(iterator&&) = default;
3515 
3516  _GLIBCXX_SIMD_ALWAYS_INLINE size_t
3517  operator->() const
3518  { return __bit; }
3519 
3520  _GLIBCXX_SIMD_ALWAYS_INLINE size_t
3521  operator*() const
3522  { return __bit; }
3523 
3524  _GLIBCXX_SIMD_ALWAYS_INLINE iterator&
3525  operator++()
3526  {
3527  __reset_lsb();
3528  __next_bit();
3529  return *this;
3530  }
3531 
3532  _GLIBCXX_SIMD_ALWAYS_INLINE iterator
3533  operator++(int)
3534  {
3535  iterator __tmp = *this;
3536  __reset_lsb();
3537  __next_bit();
3538  return __tmp;
3539  }
3540 
3541  _GLIBCXX_SIMD_ALWAYS_INLINE bool
3542  operator==(const iterator& __rhs) const
3543  { return __mask == __rhs.__mask; }
3544 
3545  _GLIBCXX_SIMD_ALWAYS_INLINE bool
3546  operator!=(const iterator& __rhs) const
3547  { return __mask != __rhs.__mask; }
3548  };
3549 
3550  iterator
3551  begin() const
3552  { return __bits.to_ullong(); }
3553 
3554  iterator
3555  end() const
3556  { return 0; }
3557  };
3558 
3559 template <typename _Tp, typename _Ap>
3560  where_range<simd_size_v<_Tp, _Ap>>
3561  where(const simd_mask<_Tp, _Ap>& __k)
3562  { return __k.__to_bitset(); }
3563 
3564 } // namespace __proposed
3565 
3566 // }}}1
3567 // reductions [simd.reductions] {{{1
3568 template <typename _Tp, typename _Abi, typename _BinaryOperation = plus<>>
3569  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3570  reduce(const simd<_Tp, _Abi>& __v, _BinaryOperation __binary_op = _BinaryOperation())
3571  { return _Abi::_SimdImpl::_S_reduce(__v, __binary_op); }
3572 
3573 template <typename _M, typename _V, typename _BinaryOperation = plus<>>
3574  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3575  reduce(const const_where_expression<_M, _V>& __x,
3576  typename _V::value_type __identity_element, _BinaryOperation __binary_op)
3577  {
3578  if (__builtin_expect(none_of(__get_mask(__x)), false))
3579  return __identity_element;
3580 
3581  _V __tmp = __identity_element;
3582  _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3583  __data(__get_lvalue(__x)));
3584  return reduce(__tmp, __binary_op);
3585  }
3586 
3587 template <typename _M, typename _V>
3588  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3589  reduce(const const_where_expression<_M, _V>& __x, plus<> __binary_op = {})
3590  { return reduce(__x, 0, __binary_op); }
3591 
3592 template <typename _M, typename _V>
3593  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3594  reduce(const const_where_expression<_M, _V>& __x, multiplies<> __binary_op)
3595  { return reduce(__x, 1, __binary_op); }
3596 
3597 template <typename _M, typename _V>
3598  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3599  reduce(const const_where_expression<_M, _V>& __x, bit_and<> __binary_op)
3600  { return reduce(__x, ~typename _V::value_type(), __binary_op); }
3601 
3602 template <typename _M, typename _V>
3603  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3604  reduce(const const_where_expression<_M, _V>& __x, bit_or<> __binary_op)
3605  { return reduce(__x, 0, __binary_op); }
3606 
3607 template <typename _M, typename _V>
3608  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3609  reduce(const const_where_expression<_M, _V>& __x, bit_xor<> __binary_op)
3610  { return reduce(__x, 0, __binary_op); }
3611 
3612 template <typename _Tp, typename _Abi>
3613  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3614  hmin(const simd<_Tp, _Abi>& __v) noexcept
3615  { return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Minimum()); }
3616 
3617 template <typename _Tp, typename _Abi>
3618  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3619  hmax(const simd<_Tp, _Abi>& __v) noexcept
3620  { return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Maximum()); }
3621 
3622 template <typename _M, typename _V>
3623  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3624  typename _V::value_type
3625  hmin(const const_where_expression<_M, _V>& __x) noexcept
3626  {
3627  using _Tp = typename _V::value_type;
3628  constexpr _Tp __id_elem =
3629 #ifdef __FINITE_MATH_ONLY__
3630  __finite_max_v<_Tp>;
3631 #else
3632  __value_or<__infinity, _Tp>(__finite_max_v<_Tp>);
3633 #endif
3634  _V __tmp = __id_elem;
3635  _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3636  __data(__get_lvalue(__x)));
3637  return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Minimum());
3638  }
3639 
3640 template <typename _M, typename _V>
3641  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3642  typename _V::value_type
3643  hmax(const const_where_expression<_M, _V>& __x) noexcept
3644  {
3645  using _Tp = typename _V::value_type;
3646  constexpr _Tp __id_elem =
3647 #ifdef __FINITE_MATH_ONLY__
3648  __finite_min_v<_Tp>;
3649 #else
3650  [] {
3651  if constexpr (__value_exists_v<__infinity, _Tp>)
3652  return -__infinity_v<_Tp>;
3653  else
3654  return __finite_min_v<_Tp>;
3655  }();
3656 #endif
3657  _V __tmp = __id_elem;
3658  _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3659  __data(__get_lvalue(__x)));
3660  return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Maximum());
3661  }
3662 
3663 // }}}1
3664 // algorithms [simd.alg] {{{
3665 template <typename _Tp, typename _Ap>
3666  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3667  min(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3668  { return {__private_init, _Ap::_SimdImpl::_S_min(__data(__a), __data(__b))}; }
3669 
3670 template <typename _Tp, typename _Ap>
3671  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3672  max(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3673  { return {__private_init, _Ap::_SimdImpl::_S_max(__data(__a), __data(__b))}; }
3674 
3675 template <typename _Tp, typename _Ap>
3676  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3677  pair<simd<_Tp, _Ap>, simd<_Tp, _Ap>>
3678  minmax(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3679  {
3680  const auto pair_of_members
3681  = _Ap::_SimdImpl::_S_minmax(__data(__a), __data(__b));
3682  return {simd<_Tp, _Ap>(__private_init, pair_of_members.first),
3683  simd<_Tp, _Ap>(__private_init, pair_of_members.second)};
3684  }
3685 
3686 template <typename _Tp, typename _Ap>
3687  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3688  clamp(const simd<_Tp, _Ap>& __v, const simd<_Tp, _Ap>& __lo, const simd<_Tp, _Ap>& __hi)
3689  {
3690  using _Impl = typename _Ap::_SimdImpl;
3691  return {__private_init,
3692  _Impl::_S_min(__data(__hi),
3693  _Impl::_S_max(__data(__lo), __data(__v)))};
3694  }
3695 
3696 // }}}
3697 
3698 template <size_t... _Sizes, typename _Tp, typename _Ap,
3699  typename = enable_if_t<((_Sizes + ...) == simd<_Tp, _Ap>::size())>>
3700  inline tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3701  split(const simd<_Tp, _Ap>&);
3702 
3703 // __extract_part {{{
3704 template <int _Index, int _Total, int _Combine = 1, typename _Tp, size_t _Np>
3705  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST constexpr
3706  _SimdWrapper<_Tp, _Np / _Total * _Combine>
3707  __extract_part(const _SimdWrapper<_Tp, _Np> __x);
3708 
3709 template <int _Index, int _Parts, int _Combine = 1, typename _Tp, typename _A0, typename... _As>
3710  _GLIBCXX_SIMD_INTRINSIC constexpr auto
3711  __extract_part(const _SimdTuple<_Tp, _A0, _As...>& __x);
3712 
3713 // }}}
3714 // _SizeList {{{
3715 template <size_t _V0, size_t... _Values>
3716  struct _SizeList
3717  {
3718  template <size_t _I>
3719  static constexpr size_t
3720  _S_at(_SizeConstant<_I> = {})
3721  {
3722  if constexpr (_I == 0)
3723  return _V0;
3724  else
3725  return _SizeList<_Values...>::template _S_at<_I - 1>();
3726  }
3727 
3728  template <size_t _I>
3729  static constexpr auto
3730  _S_before(_SizeConstant<_I> = {})
3731  {
3732  if constexpr (_I == 0)
3733  return _SizeConstant<0>();
3734  else
3735  return _SizeConstant<
3736  _V0 + _SizeList<_Values...>::template _S_before<_I - 1>()>();
3737  }
3738 
3739  template <size_t _Np>
3740  static constexpr auto
3741  _S_pop_front(_SizeConstant<_Np> = {})
3742  {
3743  if constexpr (_Np == 0)
3744  return _SizeList();
3745  else
3746  return _SizeList<_Values...>::template _S_pop_front<_Np - 1>();
3747  }
3748  };
3749 
3750 // }}}
3751 // __extract_center {{{
3752 template <typename _Tp, size_t _Np>
3753  _GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _Np / 2>
3754  __extract_center(_SimdWrapper<_Tp, _Np> __x)
3755  {
3756  static_assert(_Np >= 4);
3757  static_assert(_Np % 4 == 0); // x0 - x1 - x2 - x3 -> return {x1, x2}
3758 #if _GLIBCXX_SIMD_X86INTRIN // {{{
3759  if constexpr (__have_avx512f && sizeof(_Tp) * _Np == 64)
3760  {
3761  const auto __intrin = __to_intrin(__x);
3762  if constexpr (is_integral_v<_Tp>)
3763  return __vector_bitcast<_Tp>(_mm512_castsi512_si256(
3764  _mm512_shuffle_i32x4(__intrin, __intrin,
3765  1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3766  else if constexpr (sizeof(_Tp) == 4)
3767  return __vector_bitcast<_Tp>(_mm512_castps512_ps256(
3768  _mm512_shuffle_f32x4(__intrin, __intrin,
3769  1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3770  else if constexpr (sizeof(_Tp) == 8)
3771  return __vector_bitcast<_Tp>(_mm512_castpd512_pd256(
3772  _mm512_shuffle_f64x2(__intrin, __intrin,
3773  1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3774  else
3775  __assert_unreachable<_Tp>();
3776  }
3777  else if constexpr (sizeof(_Tp) * _Np == 32 && is_floating_point_v<_Tp>)
3778  return __vector_bitcast<_Tp>(
3779  _mm_shuffle_pd(__lo128(__vector_bitcast<double>(__x)),
3780  __hi128(__vector_bitcast<double>(__x)), 1));
3781  else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) * _Np <= 32)
3782  return __vector_bitcast<_Tp>(
3783  _mm_alignr_epi8(__hi128(__vector_bitcast<_LLong>(__x)),
3784  __lo128(__vector_bitcast<_LLong>(__x)),
3785  sizeof(_Tp) * _Np / 4));
3786  else
3787 #endif // _GLIBCXX_SIMD_X86INTRIN }}}
3788  {
3789  __vector_type_t<_Tp, _Np / 2> __r;
3790  __builtin_memcpy(&__r,
3791  reinterpret_cast<const char*>(&__x)
3792  + sizeof(_Tp) * _Np / 4,
3793  sizeof(_Tp) * _Np / 2);
3794  return __r;
3795  }
3796  }
3797 
3798 template <typename _Tp, typename _A0, typename... _As>
3799  _GLIBCXX_SIMD_INTRINSIC
3800  _SimdWrapper<_Tp, _SimdTuple<_Tp, _A0, _As...>::_S_size() / 2>
3801  __extract_center(const _SimdTuple<_Tp, _A0, _As...>& __x)
3802  {
3803  if constexpr (sizeof...(_As) == 0)
3804  return __extract_center(__x.first);
3805  else
3806  return __extract_part<1, 4, 2>(__x);
3807  }
3808 
3809 // }}}
3810 // __split_wrapper {{{
3811 template <size_t... _Sizes, typename _Tp, typename... _As>
3812  auto
3813  __split_wrapper(_SizeList<_Sizes...>, const _SimdTuple<_Tp, _As...>& __x)
3814  {
3815  return split<_Sizes...>(
3816  fixed_size_simd<_Tp, _SimdTuple<_Tp, _As...>::_S_size()>(__private_init,
3817  __x));
3818  }
3819 
3820 // }}}
3821 
3822 // split<simd>(simd) {{{
3823 template <typename _V, typename _Ap,
3824  size_t _Parts = simd_size_v<typename _V::value_type, _Ap> / _V::size()>
3825  enable_if_t<simd_size_v<typename _V::value_type, _Ap> == _Parts * _V::size()
3826  && is_simd_v<_V>, array<_V, _Parts>>
3827  split(const simd<typename _V::value_type, _Ap>& __x)
3828  {
3829  using _Tp = typename _V::value_type;
3830  if constexpr (_Parts == 1)
3831  {
3832  return {simd_cast<_V>(__x)};
3833  }
3834  else if (__x._M_is_constprop())
3835  {
3836  return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3837  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3838  return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
3839  { return __x[__i * _V::size() + __j]; });
3840  });
3841  }
3842  else if constexpr (
3843  __is_fixed_size_abi_v<_Ap>
3844  && (is_same_v<typename _V::abi_type, simd_abi::scalar>
3845  || (__is_fixed_size_abi_v<typename _V::abi_type>
3846  && sizeof(_V) == sizeof(_Tp) * _V::size() // _V doesn't have padding
3847  )))
3848  {
3849  // fixed_size -> fixed_size (w/o padding) or scalar
3850 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
3851  const __may_alias<_Tp>* const __element_ptr
3852  = reinterpret_cast<const __may_alias<_Tp>*>(&__data(__x));
3853  return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3854  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
3855  { return _V(__element_ptr + __i * _V::size(), vector_aligned); });
3856 #else
3857  const auto& __xx = __data(__x);
3858  return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3859  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3860  [[maybe_unused]] constexpr size_t __offset
3861  = decltype(__i)::value * _V::size();
3862  return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3863  constexpr _SizeConstant<__j + __offset> __k;
3864  return __xx[__k];
3865  });
3866  });
3867 #endif
3868  }
3869  else if constexpr (is_same_v<typename _V::abi_type, simd_abi::scalar>)
3870  {
3871  // normally memcpy should work here as well
3872  return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3873  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3874  }
3875  else
3876  {
3877  return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3878  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3879  if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
3880  return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3881  return __x[__i * _V::size() + __j];
3882  });
3883  else
3884  return _V(__private_init,
3885  __extract_part<decltype(__i)::value, _Parts>(__data(__x)));
3886  });
3887  }
3888  }
3889 
3890 // }}}
3891 // split<simd_mask>(simd_mask) {{{
3892 template <typename _V, typename _Ap,
3893  size_t _Parts = simd_size_v<typename _V::simd_type::value_type, _Ap> / _V::size()>
3894  enable_if_t<is_simd_mask_v<_V> && simd_size_v<typename
3895  _V::simd_type::value_type, _Ap> == _Parts * _V::size(), array<_V, _Parts>>
3896  split(const simd_mask<typename _V::simd_type::value_type, _Ap>& __x)
3897  {
3898  if constexpr (is_same_v<_Ap, typename _V::abi_type>)
3899  return {__x};
3900  else if constexpr (_Parts == 1)
3901  return {__proposed::static_simd_cast<_V>(__x)};
3902  else if constexpr (_Parts == 2 && __is_sse_abi<typename _V::abi_type>()
3903  && __is_avx_abi<_Ap>())
3904  return {_V(__private_init, __lo128(__data(__x))),
3905  _V(__private_init, __hi128(__data(__x)))};
3906  else if constexpr (_V::size() <= __CHAR_BIT__ * sizeof(_ULLong))
3907  {
3908  const bitset __bits = __x.__to_bitset();
3909  return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3910  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3911  constexpr size_t __offset = __i * _V::size();
3912  return _V(__bitset_init, (__bits >> __offset).to_ullong());
3913  });
3914  }
3915  else
3916  {
3917  return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3918  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3919  constexpr size_t __offset = __i * _V::size();
3920  return _V(__private_init,
3921  [&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3922  return __x[__j + __offset];
3923  });
3924  });
3925  }
3926  }
3927 
3928 // }}}
3929 // split<_Sizes...>(simd) {{{
3930 template <size_t... _Sizes, typename _Tp, typename _Ap, typename>
3931  _GLIBCXX_SIMD_ALWAYS_INLINE
3932  tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3933  split(const simd<_Tp, _Ap>& __x)
3934  {
3935  using _SL = _SizeList<_Sizes...>;
3936  using _Tuple = tuple<__deduced_simd<_Tp, _Sizes>...>;
3937  constexpr size_t _Np = simd_size_v<_Tp, _Ap>;
3938  constexpr size_t _N0 = _SL::template _S_at<0>();
3939  using _V = __deduced_simd<_Tp, _N0>;
3940 
3941  if (__x._M_is_constprop())
3942  return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>(
3943  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3944  using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3945  constexpr size_t __offset = _SL::_S_before(__i);
3946  return _Vi([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3947  return __x[__offset + __j];
3948  });
3949  });
3950  else if constexpr (_Np == _N0)
3951  {
3952  static_assert(sizeof...(_Sizes) == 1);
3953  return {simd_cast<_V>(__x)};
3954  }
3955  else if constexpr // split from fixed_size, such that __x::first.size == _N0
3956  (__is_fixed_size_abi_v<
3957  _Ap> && __fixed_size_storage_t<_Tp, _Np>::_S_first_size == _N0)
3958  {
3959  static_assert(
3960  !__is_fixed_size_abi_v<typename _V::abi_type>,
3961  "How can <_Tp, _Np> be __a single _SimdTuple entry but __a "
3962  "fixed_size_simd "
3963  "when deduced?");
3964  // extract first and recurse (__split_wrapper is needed to deduce a new
3965  // _Sizes pack)
3966  return tuple_cat(make_tuple(_V(__private_init, __data(__x).first)),
3967  __split_wrapper(_SL::template _S_pop_front<1>(),
3968  __data(__x).second));
3969  }
3970  else if constexpr ((!is_same_v<simd_abi::scalar,
3971  simd_abi::deduce_t<_Tp, _Sizes>> && ...)
3972  && (!__is_fixed_size_abi_v<
3973  simd_abi::deduce_t<_Tp, _Sizes>> && ...))
3974  {
3975  if constexpr (((_Sizes * 2 == _Np) && ...))
3976  return {{__private_init, __extract_part<0, 2>(__data(__x))},
3977  {__private_init, __extract_part<1, 2>(__data(__x))}};
3978  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3979  _SizeList<_Np / 3, _Np / 3, _Np / 3>>)
3980  return {{__private_init, __extract_part<0, 3>(__data(__x))},
3981  {__private_init, __extract_part<1, 3>(__data(__x))},
3982  {__private_init, __extract_part<2, 3>(__data(__x))}};
3983  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3984  _SizeList<2 * _Np / 3, _Np / 3>>)
3985  return {{__private_init, __extract_part<0, 3, 2>(__data(__x))},
3986  {__private_init, __extract_part<2, 3>(__data(__x))}};
3987  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3988  _SizeList<_Np / 3, 2 * _Np / 3>>)
3989  return {{__private_init, __extract_part<0, 3>(__data(__x))},
3990  {__private_init, __extract_part<1, 3, 2>(__data(__x))}};
3991  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3992  _SizeList<_Np / 2, _Np / 4, _Np / 4>>)
3993  return {{__private_init, __extract_part<0, 2>(__data(__x))},
3994  {__private_init, __extract_part<2, 4>(__data(__x))},
3995  {__private_init, __extract_part<3, 4>(__data(__x))}};
3996  else if constexpr (is_same_v<_SizeList<_Sizes...>,
3997  _SizeList<_Np / 4, _Np / 4, _Np / 2>>)
3998  return {{__private_init, __extract_part<0, 4>(__data(__x))},
3999  {__private_init, __extract_part<1, 4>(__data(__x))},
4000  {__private_init, __extract_part<1, 2>(__data(__x))}};
4001  else if constexpr (is_same_v<_SizeList<_Sizes...>,
4002  _SizeList<_Np / 4, _Np / 2, _Np / 4>>)
4003  return {{__private_init, __extract_part<0, 4>(__data(__x))},
4004  {__private_init, __extract_center(__data(__x))},
4005  {__private_init, __extract_part<3, 4>(__data(__x))}};
4006  else if constexpr (((_Sizes * 4 == _Np) && ...))
4007  return {{__private_init, __extract_part<0, 4>(__data(__x))},
4008  {__private_init, __extract_part<1, 4>(__data(__x))},
4009  {__private_init, __extract_part<2, 4>(__data(__x))},
4010  {__private_init, __extract_part<3, 4>(__data(__x))}};
4011  // else fall through
4012  }
4013 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
4014  const __may_alias<_Tp>* const __element_ptr
4015  = reinterpret_cast<const __may_alias<_Tp>*>(&__x);
4016  return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>(
4017  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4018  using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4019  constexpr size_t __offset = _SL::_S_before(__i);
4020  constexpr size_t __base_align = alignof(simd<_Tp, _Ap>);
4021  constexpr size_t __a
4022  = __base_align - ((__offset * sizeof(_Tp)) % __base_align);
4023  constexpr size_t __b = ((__a - 1) & __a) ^ __a;
4024  constexpr size_t __alignment = __b == 0 ? __a : __b;
4025  return _Vi(__element_ptr + __offset, overaligned<__alignment>);
4026  });
4027 #else
4028  return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>(
4029  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4030  using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4031  const auto& __xx = __data(__x);
4032  using _Offset = decltype(_SL::_S_before(__i));
4033  return _Vi([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4034  constexpr _SizeConstant<_Offset::value + __j> __k;
4035  return __xx[__k];
4036  });
4037  });
4038 #endif
4039  }
4040 
4041 // }}}
4042 
4043 // __subscript_in_pack {{{
4044 template <size_t _I, typename _Tp, typename _Ap, typename... _As>
4045  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
4046  __subscript_in_pack(const simd<_Tp, _Ap>& __x, const simd<_Tp, _As>&... __xs)
4047  {
4048  if constexpr (_I < simd_size_v<_Tp, _Ap>)
4049  return __x[_I];
4050  else
4051  return __subscript_in_pack<_I - simd_size_v<_Tp, _Ap>>(__xs...);
4052  }
4053 
4054 // }}}
4055 // __store_pack_of_simd {{{
4056 template <typename _Tp, typename _A0, typename... _As>
4057  _GLIBCXX_SIMD_INTRINSIC void
4058  __store_pack_of_simd(char* __mem, const simd<_Tp, _A0>& __x0, const simd<_Tp, _As>&... __xs)
4059  {
4060  constexpr size_t __n_bytes = sizeof(_Tp) * simd_size_v<_Tp, _A0>;
4061  __builtin_memcpy(__mem, &__data(__x0), __n_bytes);
4062  if constexpr (sizeof...(__xs) > 0)
4063  __store_pack_of_simd(__mem + __n_bytes, __xs...);
4064  }
4065 
4066 // }}}
4067 // concat(simd...) {{{
4068 template <typename _Tp, typename... _As>
4069  inline _GLIBCXX_SIMD_CONSTEXPR
4070  simd<_Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>
4071  concat(const simd<_Tp, _As>&... __xs)
4072  {
4073  using _Rp = __deduced_simd<_Tp, (simd_size_v<_Tp, _As> + ...)>;
4074  if constexpr (sizeof...(__xs) == 1)
4075  return simd_cast<_Rp>(__xs...);
4076  else if ((... && __xs._M_is_constprop()))
4077  return simd<_Tp,
4078  simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>(
4079  [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
4080  { return __subscript_in_pack<__i>(__xs...); });
4081  else
4082  {
4083  _Rp __r{};
4084  __store_pack_of_simd(reinterpret_cast<char*>(&__data(__r)), __xs...);
4085  return __r;
4086  }
4087  }
4088 
4089 // }}}
4090 // concat(array<simd>) {{{
4091 template <typename _Tp, typename _Abi, size_t _Np>
4092  _GLIBCXX_SIMD_ALWAYS_INLINE
4093  _GLIBCXX_SIMD_CONSTEXPR __deduced_simd<_Tp, simd_size_v<_Tp, _Abi> * _Np>
4094  concat(const array<simd<_Tp, _Abi>, _Np>& __x)
4095  {
4096  return __call_with_subscripts<_Np>(
4097  __x, [](const auto&... __xs) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4098  return concat(__xs...);
4099  });
4100  }
4101 
4102 // }}}
4103 
4104 /// @cond undocumented
4105 // _SmartReference {{{
4106 template <typename _Up, typename _Accessor = _Up,
4107  typename _ValueType = typename _Up::value_type>
4108  class _SmartReference
4109  {
4110  friend _Accessor;
4111  int _M_index;
4112  _Up& _M_obj;
4113 
4114  _GLIBCXX_SIMD_INTRINSIC constexpr _ValueType
4115  _M_read() const noexcept
4116  {
4117  if constexpr (is_arithmetic_v<_Up>)
4118  return _M_obj;
4119  else
4120  return _M_obj[_M_index];
4121  }
4122 
4123  template <typename _Tp>
4124  _GLIBCXX_SIMD_INTRINSIC constexpr void
4125  _M_write(_Tp&& __x) const
4126  { _Accessor::_S_set(_M_obj, _M_index, static_cast<_Tp&&>(__x)); }
4127 
4128  public:
4129  _GLIBCXX_SIMD_INTRINSIC constexpr
4130  _SmartReference(_Up& __o, int __i) noexcept
4131  : _M_index(__i), _M_obj(__o) {}
4132 
4133  using value_type = _ValueType;
4134 
4135  _GLIBCXX_SIMD_INTRINSIC
4136  _SmartReference(const _SmartReference&) = delete;
4137 
4138  _GLIBCXX_SIMD_INTRINSIC constexpr
4139  operator value_type() const noexcept
4140  { return _M_read(); }
4141 
4142  template <typename _Tp, typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, value_type>>
4143  _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference
4144  operator=(_Tp&& __x) &&
4145  {
4146  _M_write(static_cast<_Tp&&>(__x));
4147  return {_M_obj, _M_index};
4148  }
4149 
4150 #define _GLIBCXX_SIMD_OP_(__op) \
4151  template <typename _Tp, \
4152  typename _TT = decltype(declval<value_type>() __op declval<_Tp>()), \
4153  typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, _TT>, \
4154  typename = _ValuePreservingOrInt<_TT, value_type>> \
4155  _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference \
4156  operator __op##=(_Tp&& __x) && \
4157  { \
4158  const value_type& __lhs = _M_read(); \
4159  _M_write(__lhs __op __x); \
4160  return {_M_obj, _M_index}; \
4161  }
4162  _GLIBCXX_SIMD_ALL_ARITHMETICS(_GLIBCXX_SIMD_OP_);
4163  _GLIBCXX_SIMD_ALL_SHIFTS(_GLIBCXX_SIMD_OP_);
4164  _GLIBCXX_SIMD_ALL_BINARY(_GLIBCXX_SIMD_OP_);
4165 #undef _GLIBCXX_SIMD_OP_
4166 
4167  template <typename _Tp = void,
4168  typename = decltype(++declval<conditional_t<true, value_type, _Tp>&>())>
4169  _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference
4170  operator++() &&
4171  {
4172  value_type __x = _M_read();
4173  _M_write(++__x);
4174  return {_M_obj, _M_index};
4175  }
4176 
4177  template <typename _Tp = void,
4178  typename = decltype(declval<conditional_t<true, value_type, _Tp>&>()++)>
4179  _GLIBCXX_SIMD_INTRINSIC constexpr value_type
4180  operator++(int) &&
4181  {
4182  const value_type __r = _M_read();
4183  value_type __x = __r;
4184  _M_write(++__x);
4185  return __r;
4186  }
4187 
4188  template <typename _Tp = void,
4189  typename = decltype(--declval<conditional_t<true, value_type, _Tp>&>())>
4190  _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference
4191  operator--() &&
4192  {
4193  value_type __x = _M_read();
4194  _M_write(--__x);
4195  return {_M_obj, _M_index};
4196  }
4197 
4198  template <typename _Tp = void,
4199  typename = decltype(declval<conditional_t<true, value_type, _Tp>&>()--)>
4200  _GLIBCXX_SIMD_INTRINSIC constexpr value_type
4201  operator--(int) &&
4202  {
4203  const value_type __r = _M_read();
4204  value_type __x = __r;
4205  _M_write(--__x);
4206  return __r;
4207  }
4208 
4209  _GLIBCXX_SIMD_INTRINSIC friend void
4210  swap(_SmartReference&& __a, _SmartReference&& __b) noexcept(
4211  conjunction<
4212  is_nothrow_constructible<value_type, _SmartReference&&>,
4213  is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4214  {
4215  value_type __tmp = static_cast<_SmartReference&&>(__a);
4216  static_cast<_SmartReference&&>(__a) = static_cast<value_type>(__b);
4217  static_cast<_SmartReference&&>(__b) = std::move(__tmp);
4218  }
4219 
4220  _GLIBCXX_SIMD_INTRINSIC friend void
4221  swap(value_type& __a, _SmartReference&& __b) noexcept(
4222  conjunction<
4223  is_nothrow_constructible<value_type, value_type&&>,
4224  is_nothrow_assignable<value_type&, value_type&&>,
4225  is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4226  {
4227  value_type __tmp(std::move(__a));
4228  __a = static_cast<value_type>(__b);
4229  static_cast<_SmartReference&&>(__b) = std::move(__tmp);
4230  }
4231 
4232  _GLIBCXX_SIMD_INTRINSIC friend void
4233  swap(_SmartReference&& __a, value_type& __b) noexcept(
4234  conjunction<
4235  is_nothrow_constructible<value_type, _SmartReference&&>,
4236  is_nothrow_assignable<value_type&, value_type&&>,
4237  is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4238  {
4239  value_type __tmp(__a);
4240  static_cast<_SmartReference&&>(__a) = std::move(__b);
4241  __b = std::move(__tmp);
4242  }
4243  };
4244 
4245 // }}}
4246 // __scalar_abi_wrapper {{{
4247 template <int _Bytes>
4248  struct __scalar_abi_wrapper
4249  {
4250  template <typename _Tp> static constexpr size_t _S_full_size = 1;
4251  template <typename _Tp> static constexpr size_t _S_size = 1;
4252  template <typename _Tp> static constexpr size_t _S_is_partial = false;
4253 
4254  template <typename _Tp, typename _Abi = simd_abi::scalar>
4255  static constexpr bool _S_is_valid_v
4256  = _Abi::template _IsValid<_Tp>::value && sizeof(_Tp) == _Bytes;
4257  };
4258 
4259 // }}}
4260 // __decay_abi metafunction {{{
4261 template <typename _Tp>
4262  struct __decay_abi { using type = _Tp; };
4263 
4264 template <int _Bytes>
4265  struct __decay_abi<__scalar_abi_wrapper<_Bytes>>
4266  { using type = simd_abi::scalar; };
4267 
4268 // }}}
4269 // __find_next_valid_abi metafunction {{{1
4270 // Given an ABI tag A<N>, find an N2 < N such that A<N2>::_S_is_valid_v<_Tp> ==
4271 // true, N2 is a power-of-2, and A<N2>::_S_is_partial<_Tp> is false. Break
4272 // recursion at 2 elements in the resulting ABI tag. In this case
4273 // type::_S_is_valid_v<_Tp> may be false.
4274 template <template <int> class _Abi, int _Bytes, typename _Tp>
4275  struct __find_next_valid_abi
4276  {
4277  static constexpr auto
4278  _S_choose()
4279  {
4280  constexpr int _NextBytes = std::__bit_ceil(_Bytes) / 2;
4281  using _NextAbi = _Abi<_NextBytes>;
4282  if constexpr (_NextBytes < sizeof(_Tp) * 2) // break recursion
4283  return _Abi<_Bytes>();
4284  else if constexpr (_NextAbi::template _S_is_partial<_Tp> == false
4285  && _NextAbi::template _S_is_valid_v<_Tp>)
4286  return _NextAbi();
4287  else
4288  return __find_next_valid_abi<_Abi, _NextBytes, _Tp>::_S_choose();
4289  }
4290 
4291  using type = decltype(_S_choose());
4292  };
4293 
4294 template <int _Bytes, typename _Tp>
4295  struct __find_next_valid_abi<__scalar_abi_wrapper, _Bytes, _Tp>
4296  { using type = simd_abi::scalar; };
4297 
4298 // _AbiList {{{1
4299 template <template <int> class...>
4300  struct _AbiList
4301  {
4302  template <typename, int> static constexpr bool _S_has_valid_abi = false;
4303  template <typename, int> using _FirstValidAbi = void;
4304  template <typename, int> using _BestAbi = void;
4305  };
4306 
4307 template <template <int> class _A0, template <int> class... _Rest>
4308  struct _AbiList<_A0, _Rest...>
4309  {
4310  template <typename _Tp, int _Np>
4311  static constexpr bool _S_has_valid_abi
4312  = _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<
4313  _Tp> || _AbiList<_Rest...>::template _S_has_valid_abi<_Tp, _Np>;
4314 
4315  template <typename _Tp, int _Np>
4316  using _FirstValidAbi = conditional_t<
4317  _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<_Tp>,
4318  typename __decay_abi<_A0<sizeof(_Tp) * _Np>>::type,
4319  typename _AbiList<_Rest...>::template _FirstValidAbi<_Tp, _Np>>;
4320 
4321  template <typename _Tp, int _Np>
4322  static constexpr auto
4323  _S_determine_best_abi()
4324  {
4325  static_assert(_Np >= 1);
4326  constexpr int _Bytes = sizeof(_Tp) * _Np;
4327  if constexpr (_Np == 1)
4328  return __make_dependent_t<_Tp, simd_abi::scalar>{};
4329  else
4330  {
4331  constexpr int __fullsize = _A0<_Bytes>::template _S_full_size<_Tp>;
4332  // _A0<_Bytes> is good if:
4333  // 1. The ABI tag is valid for _Tp
4334  // 2. The storage overhead is no more than padding to fill the next
4335  // power-of-2 number of bytes
4336  if constexpr (_A0<_Bytes>::template _S_is_valid_v<
4337  _Tp> && __fullsize / 2 < _Np)
4338  return typename __decay_abi<_A0<_Bytes>>::type{};
4339  else
4340  {
4341  using _Bp =
4342  typename __find_next_valid_abi<_A0, _Bytes, _Tp>::type;
4343  if constexpr (_Bp::template _S_is_valid_v<
4344  _Tp> && _Bp::template _S_size<_Tp> <= _Np)
4345  return _Bp{};
4346  else
4347  return
4348  typename _AbiList<_Rest...>::template _BestAbi<_Tp, _Np>{};
4349  }
4350  }
4351  }
4352 
4353  template <typename _Tp, int _Np>
4354  using _BestAbi = decltype(_S_determine_best_abi<_Tp, _Np>());
4355  };
4356 
4357 // }}}1
4358 
4359 // the following lists all native ABIs, which makes them accessible to
4360 // simd_abi::deduce and select_best_vector_type_t (for fixed_size). Order
4361 // matters: Whatever comes first has higher priority.
4362 using _AllNativeAbis = _AbiList<simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin,
4363  __scalar_abi_wrapper>;
4364 
4365 // valid _SimdTraits specialization {{{1
4366 template <typename _Tp, typename _Abi>
4367  struct _SimdTraits<_Tp, _Abi, void_t<typename _Abi::template _IsValid<_Tp>>>
4368  : _Abi::template __traits<_Tp> {};
4369 
4370 // __deduce_impl specializations {{{1
4371 // try all native ABIs (including scalar) first
4372 template <typename _Tp, size_t _Np>
4373  struct __deduce_impl<
4374  _Tp, _Np, enable_if_t<_AllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>>
4375  { using type = _AllNativeAbis::_FirstValidAbi<_Tp, _Np>; };
4376 
4377 // fall back to fixed_size only if scalar and native ABIs don't match
4378 template <typename _Tp, size_t _Np, typename = void>
4379  struct __deduce_fixed_size_fallback {};
4380 
4381 template <typename _Tp, size_t _Np>
4382  struct __deduce_fixed_size_fallback<_Tp, _Np,
4383  enable_if_t<simd_abi::fixed_size<_Np>::template _S_is_valid_v<_Tp>>>
4384  { using type = simd_abi::fixed_size<_Np>; };
4385 
4386 template <typename _Tp, size_t _Np, typename>
4387  struct __deduce_impl : public __deduce_fixed_size_fallback<_Tp, _Np> {};
4388 
4389 //}}}1
4390 /// @endcond
4391 
4392 // simd_mask {{{
4393 template <typename _Tp, typename _Abi>
4394  class simd_mask : public _SimdTraits<_Tp, _Abi>::_MaskBase
4395  {
4396  // types, tags, and friends {{{
4397  using _Traits = _SimdTraits<_Tp, _Abi>;
4398  using _MemberType = typename _Traits::_MaskMember;
4399 
4400  // We map all masks with equal element sizeof to a single integer type, the
4401  // one given by __int_for_sizeof_t<_Tp>. This is the approach
4402  // [[gnu::vector_size(N)]] types take as well and it reduces the number of
4403  // template specializations in the implementation classes.
4404  using _Ip = __int_for_sizeof_t<_Tp>;
4405  static constexpr _Ip* _S_type_tag = nullptr;
4406 
4407  friend typename _Traits::_MaskBase;
4408  friend class simd<_Tp, _Abi>; // to construct masks on return
4409  friend typename _Traits::_SimdImpl; // to construct masks on return and
4410  // inspect data on masked operations
4411  public:
4412  using _Impl = typename _Traits::_MaskImpl;
4413  friend _Impl;
4414 
4415  // }}}
4416  // member types {{{
4417  using value_type = bool;
4418  using reference = _SmartReference<_MemberType, _Impl, value_type>;
4419  using simd_type = simd<_Tp, _Abi>;
4420  using abi_type = _Abi;
4421 
4422  // }}}
4423  static constexpr size_t size() // {{{
4424  { return __size_or_zero_v<_Tp, _Abi>; }
4425 
4426  // }}}
4427  // constructors & assignment {{{
4428  simd_mask() = default;
4429  simd_mask(const simd_mask&) = default;
4430  simd_mask(simd_mask&&) = default;
4431  simd_mask& operator=(const simd_mask&) = default;
4432  simd_mask& operator=(simd_mask&&) = default;
4433 
4434  // }}}
4435  // access to internal representation (optional feature) {{{
4436  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR explicit
4437  simd_mask(typename _Traits::_MaskCastType __init)
4438  : _M_data{__init} {}
4439  // conversions to internal type is done in _MaskBase
4440 
4441  // }}}
4442  // bitset interface (extension to be proposed) {{{
4443  // TS_FEEDBACK:
4444  // Conversion of simd_mask to and from bitset makes it much easier to
4445  // interface with other facilities. I suggest adding `static
4446  // simd_mask::from_bitset` and `simd_mask::to_bitset`.
4447  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR static simd_mask
4448  __from_bitset(bitset<size()> bs)
4449  { return {__bitset_init, bs}; }
4450 
4451  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bitset<size()>
4452  __to_bitset() const
4453  { return _Impl::_S_to_bits(_M_data)._M_to_bitset(); }
4454 
4455  // }}}
4456  // explicit broadcast constructor {{{
4457  _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
4458  simd_mask(value_type __x)
4459  : _M_data(_Impl::template _S_broadcast<_Ip>(__x)) {}
4460 
4461  // }}}
4462  // implicit type conversion constructor {{{
4463  #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4464  // proposed improvement
4465  template <typename _Up, typename _A2,
4466  typename = enable_if_t<simd_size_v<_Up, _A2> == size()>>
4467  _GLIBCXX_SIMD_ALWAYS_INLINE explicit(sizeof(_MemberType)
4468  != sizeof(typename _SimdTraits<_Up, _A2>::_MaskMember))
4469  simd_mask(const simd_mask<_Up, _A2>& __x)
4470  : simd_mask(__proposed::static_simd_cast<simd_mask>(__x)) {}
4471  #else
4472  // conforming to ISO/IEC 19570:2018
4473  template <typename _Up, typename = enable_if_t<conjunction<
4474  is_same<abi_type, simd_abi::fixed_size<size()>>,
4475  is_same<_Up, _Up>>::value>>
4476  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4477  simd_mask(const simd_mask<_Up, simd_abi::fixed_size<size()>>& __x)
4478  : _M_data(_Impl::_S_from_bitmask(__data(__x), _S_type_tag)) {}
4479  #endif
4480 
4481  // }}}
4482  // load constructor {{{
4483  template <typename _Flags>
4484  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4485  simd_mask(const value_type* __mem, _IsSimdFlagType<_Flags>)
4486  : _M_data(_Impl::template _S_load<_Ip>(_Flags::template _S_apply<simd_mask>(__mem))) {}
4487 
4488  template <typename _Flags>
4489  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4490  simd_mask(const value_type* __mem, simd_mask __k, _IsSimdFlagType<_Flags>)
4491  : _M_data{}
4492  {
4493  _M_data = _Impl::_S_masked_load(_M_data, __k._M_data,
4494  _Flags::template _S_apply<simd_mask>(__mem));
4495  }
4496 
4497  // }}}
4498  // loads [simd_mask.load] {{{
4499  template <typename _Flags>
4500  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void
4501  copy_from(const value_type* __mem, _IsSimdFlagType<_Flags>)
4502  { _M_data = _Impl::template _S_load<_Ip>(_Flags::template _S_apply<simd_mask>(__mem)); }
4503 
4504  // }}}
4505  // stores [simd_mask.store] {{{
4506  template <typename _Flags>
4507  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void
4508  copy_to(value_type* __mem, _IsSimdFlagType<_Flags>) const
4509  { _Impl::_S_store(_M_data, _Flags::template _S_apply<simd_mask>(__mem)); }
4510 
4511  // }}}
4512  // scalar access {{{
4513  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
4514  operator[](size_t __i)
4515  {
4516  if (__i >= size())
4517  __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1);
4518  return {_M_data, int(__i)};
4519  }
4520 
4521  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
4522  operator[](size_t __i) const
4523  {
4524  if (__i >= size())
4525  __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1);
4526  if constexpr (__is_scalar_abi<_Abi>())
4527  return _M_data;
4528  else
4529  return static_cast<bool>(_M_data[__i]);
4530  }
4531 
4532  // }}}
4533  // negation {{{
4534  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd_mask
4535  operator!() const
4536  { return {__private_init, _Impl::_S_bit_not(_M_data)}; }
4537 
4538  // }}}
4539  // simd_mask binary operators [simd_mask.binary] {{{
4540  #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4541  // simd_mask<int> && simd_mask<uint> needs disambiguation
4542  template <typename _Up, typename _A2,
4543  typename = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4544  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4545  operator&&(const simd_mask& __x, const simd_mask<_Up, _A2>& __y)
4546  {
4547  return {__private_init,
4548  _Impl::_S_logical_and(__x._M_data, simd_mask(__y)._M_data)};
4549  }
4550 
4551  template <typename _Up, typename _A2,
4552  typename = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4553  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4554  operator||(const simd_mask& __x, const simd_mask<_Up, _A2>& __y)
4555  {
4556  return {__private_init,
4557  _Impl::_S_logical_or(__x._M_data, simd_mask(__y)._M_data)};
4558  }
4559  #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4560 
4561  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4562  operator&&(const simd_mask& __x, const simd_mask& __y)
4563  { return {__private_init, _Impl::_S_logical_and(__x._M_data, __y._M_data)}; }
4564 
4565  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4566  operator||(const simd_mask& __x, const simd_mask& __y)
4567  { return {__private_init, _Impl::_S_logical_or(__x._M_data, __y._M_data)}; }
4568 
4569  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4570  operator&(const simd_mask& __x, const simd_mask& __y)
4571  { return {__private_init, _Impl::_S_bit_and(__x._M_data, __y._M_data)}; }
4572 
4573  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4574  operator|(const simd_mask& __x, const simd_mask& __y)
4575  { return {__private_init, _Impl::_S_bit_or(__x._M_data, __y._M_data)}; }
4576 
4577  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4578  operator^(const simd_mask& __x, const simd_mask& __y)
4579  { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4580 
4581  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask&
4582  operator&=(simd_mask& __x, const simd_mask& __y)
4583  {
4584  __x._M_data = _Impl::_S_bit_and(__x._M_data, __y._M_data);
4585  return __x;
4586  }
4587 
4588  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask&
4589  operator|=(simd_mask& __x, const simd_mask& __y)
4590  {
4591  __x._M_data = _Impl::_S_bit_or(__x._M_data, __y._M_data);
4592  return __x;
4593  }
4594 
4595  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask&
4596  operator^=(simd_mask& __x, const simd_mask& __y)
4597  {
4598  __x._M_data = _Impl::_S_bit_xor(__x._M_data, __y._M_data);
4599  return __x;
4600  }
4601 
4602  // }}}
4603  // simd_mask compares [simd_mask.comparison] {{{
4604  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4605  operator==(const simd_mask& __x, const simd_mask& __y)
4606  { return !operator!=(__x, __y); }
4607 
4608  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4609  operator!=(const simd_mask& __x, const simd_mask& __y)
4610  { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4611 
4612  // }}}
4613  // private_init ctor {{{
4614  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
4615  simd_mask(_PrivateInit, typename _Traits::_MaskMember __init)
4616  : _M_data(__init) {}
4617 
4618  // }}}
4619  // private_init generator ctor {{{
4620  template <typename _Fp, typename = decltype(bool(declval<_Fp>()(size_t())))>
4621  _GLIBCXX_SIMD_INTRINSIC constexpr
4622  simd_mask(_PrivateInit, _Fp&& __gen)
4623  : _M_data()
4624  {
4625  __execute_n_times<size()>([&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4626  _Impl::_S_set(_M_data, __i, __gen(__i));
4627  });
4628  }
4629 
4630  // }}}
4631  // bitset_init ctor {{{
4632  _GLIBCXX_SIMD_INTRINSIC constexpr
4633  simd_mask(_BitsetInit, bitset<size()> __init)
4634  : _M_data(_Impl::_S_from_bitmask(_SanitizedBitMask<size()>(__init), _S_type_tag))
4635  {}
4636 
4637  // }}}
4638  // __cvt {{{
4639  // TS_FEEDBACK:
4640  // The conversion operator this implements should be a ctor on simd_mask.
4641  // Once you call .__cvt() on a simd_mask it converts conveniently.
4642  // A useful variation: add `explicit(sizeof(_Tp) != sizeof(_Up))`
4643  struct _CvtProxy
4644  {
4645  template <typename _Up, typename _A2,
4646  typename = enable_if_t<simd_size_v<_Up, _A2> == simd_size_v<_Tp, _Abi>>>
4647  operator simd_mask<_Up, _A2>() &&
4648  {
4649  using namespace std::experimental::__proposed;
4650  return static_simd_cast<simd_mask<_Up, _A2>>(_M_data);
4651  }
4652 
4653  const simd_mask<_Tp, _Abi>& _M_data;
4654  };
4655 
4656  _GLIBCXX_SIMD_INTRINSIC _CvtProxy
4657  __cvt() const
4658  { return {*this}; }
4659 
4660  // }}}
4661  // operator?: overloads (suggested extension) {{{
4662  #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
4663  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4664  operator?:(const simd_mask& __k, const simd_mask& __where_true,
4665  const simd_mask& __where_false)
4666  {
4667  auto __ret = __where_false;
4668  _Impl::_S_masked_assign(__k._M_data, __ret._M_data, __where_true._M_data);
4669  return __ret;
4670  }
4671 
4672  template <typename _U1, typename _U2,
4673  typename _Rp = simd<common_type_t<_U1, _U2>, _Abi>,
4674  typename = enable_if_t<conjunction_v<
4675  is_convertible<_U1, _Rp>, is_convertible<_U2, _Rp>,
4676  is_convertible<simd_mask, typename _Rp::mask_type>>>>
4677  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend _Rp
4678  operator?:(const simd_mask& __k, const _U1& __where_true,
4679  const _U2& __where_false)
4680  {
4681  _Rp __ret = __where_false;
4682  _Rp::_Impl::_S_masked_assign(
4683  __data(static_cast<typename _Rp::mask_type>(__k)), __data(__ret),
4684  __data(static_cast<_Rp>(__where_true)));
4685  return __ret;
4686  }
4687 
4688  #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4689  template <typename _Kp, typename _Ak, typename _Up, typename _Au,
4690  typename = enable_if_t<
4691  conjunction_v<is_convertible<simd_mask<_Kp, _Ak>, simd_mask>,
4692  is_convertible<simd_mask<_Up, _Au>, simd_mask>>>>
4693  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4694  operator?:(const simd_mask<_Kp, _Ak>& __k, const simd_mask& __where_true,
4695  const simd_mask<_Up, _Au>& __where_false)
4696  {
4697  simd_mask __ret = __where_false;
4698  _Impl::_S_masked_assign(simd_mask(__k)._M_data, __ret._M_data,
4699  __where_true._M_data);
4700  return __ret;
4701  }
4702  #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4703  #endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__
4704 
4705  // }}}
4706  // _M_is_constprop {{{
4707  _GLIBCXX_SIMD_INTRINSIC constexpr bool
4708  _M_is_constprop() const
4709  {
4710  if constexpr (__is_scalar_abi<_Abi>())
4711  return __builtin_constant_p(_M_data);
4712  else
4713  return _M_data._M_is_constprop();
4714  }
4715 
4716  // }}}
4717 
4718  private:
4719  friend const auto& __data<_Tp, abi_type>(const simd_mask&);
4720  friend auto& __data<_Tp, abi_type>(simd_mask&);
4721  alignas(_Traits::_S_mask_align) _MemberType _M_data;
4722  };
4723 
4724 // }}}
4725 
4726 /// @cond undocumented
4727 // __data(simd_mask) {{{
4728 template <typename _Tp, typename _Ap>
4729  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
4730  __data(const simd_mask<_Tp, _Ap>& __x)
4731  { return __x._M_data; }
4732 
4733 template <typename _Tp, typename _Ap>
4734  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
4735  __data(simd_mask<_Tp, _Ap>& __x)
4736  { return __x._M_data; }
4737 
4738 // }}}
4739 /// @endcond
4740 
4741 // simd_mask reductions [simd_mask.reductions] {{{
4742 template <typename _Tp, typename _Abi>
4743  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4744  all_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4745  {
4746  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4747  {
4748  for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4749  if (!__k[__i])
4750  return false;
4751  return true;
4752  }
4753  else
4754  return _Abi::_MaskImpl::_S_all_of(__k);
4755  }
4756 
4757 template <typename _Tp, typename _Abi>
4758  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4759  any_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4760  {
4761  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4762  {
4763  for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4764  if (__k[__i])
4765  return true;
4766  return false;
4767  }
4768  else
4769  return _Abi::_MaskImpl::_S_any_of(__k);
4770  }
4771 
4772 template <typename _Tp, typename _Abi>
4773  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4774  none_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4775  {
4776  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4777  {
4778  for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4779  if (__k[__i])
4780  return false;
4781  return true;
4782  }
4783  else
4784  return _Abi::_MaskImpl::_S_none_of(__k);
4785  }
4786 
4787 template <typename _Tp, typename _Abi>
4788  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4789  some_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4790  {
4791  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4792  {
4793  for (size_t __i = 1; __i < simd_size_v<_Tp, _Abi>; ++__i)
4794  if (__k[__i] != __k[__i - 1])
4795  return true;
4796  return false;
4797  }
4798  else
4799  return _Abi::_MaskImpl::_S_some_of(__k);
4800  }
4801 
4802 template <typename _Tp, typename _Abi>
4803  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4804  popcount(const simd_mask<_Tp, _Abi>& __k) noexcept
4805  {
4806  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4807  {
4808  const int __r = __call_with_subscripts<simd_size_v<_Tp, _Abi>>(
4809  __k, [](auto... __elements) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4810  return ((__elements != 0) + ...);
4811  });
4812  if (__builtin_is_constant_evaluated() || __builtin_constant_p(__r))
4813  return __r;
4814  }
4815  return _Abi::_MaskImpl::_S_popcount(__k);
4816  }
4817 
4818 template <typename _Tp, typename _Abi>
4819  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4820  find_first_set(const simd_mask<_Tp, _Abi>& __k)
4821  {
4822  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4823  {
4824  constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
4825  const size_t _Idx = __call_with_n_evaluations<_Np>(
4826  [](auto... __indexes) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4827  return std::min({__indexes...});
4828  }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4829  return __k[__i] ? +__i : _Np;
4830  });
4831  if (_Idx >= _Np)
4832  __invoke_ub("find_first_set(empty mask) is UB");
4833  if (__builtin_constant_p(_Idx))
4834  return _Idx;
4835  }
4836  return _Abi::_MaskImpl::_S_find_first_set(__k);
4837  }
4838 
4839 template <typename _Tp, typename _Abi>
4840  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4841  find_last_set(const simd_mask<_Tp, _Abi>& __k)
4842  {
4843  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4844  {
4845  constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
4846  const int _Idx = __call_with_n_evaluations<_Np>(
4847  [](auto... __indexes) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4848  return std::max({__indexes...});
4849  }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4850  return __k[__i] ? int(__i) : -1;
4851  });
4852  if (_Idx < 0)
4853  __invoke_ub("find_first_set(empty mask) is UB");
4854  if (__builtin_constant_p(_Idx))
4855  return _Idx;
4856  }
4857  return _Abi::_MaskImpl::_S_find_last_set(__k);
4858  }
4859 
4860 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4861 all_of(_ExactBool __x) noexcept
4862 { return __x; }
4863 
4864 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4865 any_of(_ExactBool __x) noexcept
4866 { return __x; }
4867 
4868 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4869 none_of(_ExactBool __x) noexcept
4870 { return !__x; }
4871 
4872 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4873 some_of(_ExactBool) noexcept
4874 { return false; }
4875 
4876 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4877 popcount(_ExactBool __x) noexcept
4878 { return __x; }
4879 
4880 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4881 find_first_set(_ExactBool)
4882 { return 0; }
4883 
4884 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4885 find_last_set(_ExactBool)
4886 { return 0; }
4887 
4888 // }}}
4889 
4890 /// @cond undocumented
4891 // _SimdIntOperators{{{1
4892 template <typename _V, typename _Impl, bool>
4893  class _SimdIntOperators {};
4894 
4895 template <typename _V, typename _Impl>
4896  class _SimdIntOperators<_V, _Impl, true>
4897  {
4898  _GLIBCXX_SIMD_INTRINSIC constexpr const _V&
4899  __derived() const
4900  { return *static_cast<const _V*>(this); }
4901 
4902  template <typename _Tp>
4903  _GLIBCXX_SIMD_INTRINSIC static _GLIBCXX_SIMD_CONSTEXPR _V
4904  _S_make_derived(_Tp&& __d)
4905  { return {__private_init, static_cast<_Tp&&>(__d)}; }
4906 
4907  public:
4908  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator%=(_V& __lhs, const _V& __x)
4909  { return __lhs = __lhs % __x; }
4910 
4911  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator&=(_V& __lhs, const _V& __x)
4912  { return __lhs = __lhs & __x; }
4913 
4914  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator|=(_V& __lhs, const _V& __x)
4915  { return __lhs = __lhs | __x; }
4916 
4917  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator^=(_V& __lhs, const _V& __x)
4918  { return __lhs = __lhs ^ __x; }
4919 
4920  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator<<=(_V& __lhs, const _V& __x)
4921  { return __lhs = __lhs << __x; }
4922 
4923  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator>>=(_V& __lhs, const _V& __x)
4924  { return __lhs = __lhs >> __x; }
4925 
4926  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator<<=(_V& __lhs, int __x)
4927  { return __lhs = __lhs << __x; }
4928 
4929  _GLIBCXX_SIMD_CONSTEXPR friend _V& operator>>=(_V& __lhs, int __x)
4930  { return __lhs = __lhs >> __x; }
4931 
4932  _GLIBCXX_SIMD_CONSTEXPR friend _V operator%(const _V& __x, const _V& __y)
4933  {
4934  return _SimdIntOperators::_S_make_derived(
4935  _Impl::_S_modulus(__data(__x), __data(__y)));
4936  }
4937 
4938  _GLIBCXX_SIMD_CONSTEXPR friend _V operator&(const _V& __x, const _V& __y)
4939  {
4940  return _SimdIntOperators::_S_make_derived(
4941  _Impl::_S_bit_and(__data(__x), __data(__y)));
4942  }
4943 
4944  _GLIBCXX_SIMD_CONSTEXPR friend _V operator|(const _V& __x, const _V& __y)
4945  {
4946  return _SimdIntOperators::_S_make_derived(
4947  _Impl::_S_bit_or(__data(__x), __data(__y)));
4948  }
4949 
4950  _GLIBCXX_SIMD_CONSTEXPR friend _V operator^(const _V& __x, const _V& __y)
4951  {
4952  return _SimdIntOperators::_S_make_derived(
4953  _Impl::_S_bit_xor(__data(__x), __data(__y)));
4954  }
4955 
4956  _GLIBCXX_SIMD_CONSTEXPR friend _V operator<<(const _V& __x, const _V& __y)
4957  {
4958  return _SimdIntOperators::_S_make_derived(
4959  _Impl::_S_bit_shift_left(__data(__x), __data(__y)));
4960  }
4961 
4962  _GLIBCXX_SIMD_CONSTEXPR friend _V operator>>(const _V& __x, const _V& __y)
4963  {
4964  return _SimdIntOperators::_S_make_derived(
4965  _Impl::_S_bit_shift_right(__data(__x), __data(__y)));
4966  }
4967 
4968  template <typename _VV = _V>
4969  _GLIBCXX_SIMD_CONSTEXPR friend _V operator<<(const _V& __x, int __y)
4970  {
4971  using _Tp = typename _VV::value_type;
4972  if (__y < 0)
4973  __invoke_ub("The behavior is undefined if the right operand of a "
4974  "shift operation is negative. [expr.shift]\nA shift by "
4975  "%d was requested",
4976  __y);
4977  if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
4978  __invoke_ub(
4979  "The behavior is undefined if the right operand of a "
4980  "shift operation is greater than or equal to the width of the "
4981  "promoted left operand. [expr.shift]\nA shift by %d was requested",
4982  __y);
4983  return _SimdIntOperators::_S_make_derived(
4984  _Impl::_S_bit_shift_left(__data(__x), __y));
4985  }
4986 
4987  template <typename _VV = _V>
4988  _GLIBCXX_SIMD_CONSTEXPR friend _V operator>>(const _V& __x, int __y)
4989  {
4990  using _Tp = typename _VV::value_type;
4991  if (__y < 0)
4992  __invoke_ub(
4993  "The behavior is undefined if the right operand of a shift "
4994  "operation is negative. [expr.shift]\nA shift by %d was requested",
4995  __y);
4996  if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
4997  __invoke_ub(
4998  "The behavior is undefined if the right operand of a shift "
4999  "operation is greater than or equal to the width of the promoted "
5000  "left operand. [expr.shift]\nA shift by %d was requested",
5001  __y);
5002  return _SimdIntOperators::_S_make_derived(
5003  _Impl::_S_bit_shift_right(__data(__x), __y));
5004  }
5005 
5006  // unary operators (for integral _Tp)
5007  _GLIBCXX_SIMD_CONSTEXPR _V operator~() const
5008  { return {__private_init, _Impl::_S_complement(__derived()._M_data)}; }
5009  };
5010 
5011 //}}}1
5012 /// @endcond
5013 
5014 // simd {{{
5015 template <typename _Tp, typename _Abi>
5016  class simd : public _SimdIntOperators<
5017  simd<_Tp, _Abi>, typename _SimdTraits<_Tp, _Abi>::_SimdImpl,
5018  conjunction<is_integral<_Tp>,
5019  typename _SimdTraits<_Tp, _Abi>::_IsValid>::value>,
5020  public _SimdTraits<_Tp, _Abi>::_SimdBase
5021  {
5022  using _Traits = _SimdTraits<_Tp, _Abi>;
5023  using _MemberType = typename _Traits::_SimdMember;
5024  using _CastType = typename _Traits::_SimdCastType;
5025  static constexpr _Tp* _S_type_tag = nullptr;
5026  friend typename _Traits::_SimdBase;
5027 
5028  public:
5029  using _Impl = typename _Traits::_SimdImpl;
5030  friend _Impl;
5031  friend _SimdIntOperators<simd, _Impl, true>;
5032 
5033  using value_type = _Tp;
5034  using reference = _SmartReference<_MemberType, _Impl, value_type>;
5035  using mask_type = simd_mask<_Tp, _Abi>;
5036  using abi_type = _Abi;
5037 
5038  static constexpr size_t size()
5039  { return __size_or_zero_v<_Tp, _Abi>; }
5040 
5041  _GLIBCXX_SIMD_CONSTEXPR simd() = default;
5042  _GLIBCXX_SIMD_CONSTEXPR simd(const simd&) = default;
5043  _GLIBCXX_SIMD_CONSTEXPR simd(simd&&) noexcept = default;
5044  _GLIBCXX_SIMD_CONSTEXPR simd& operator=(const simd&) = default;
5045  _GLIBCXX_SIMD_CONSTEXPR simd& operator=(simd&&) noexcept = default;
5046 
5047  // implicit broadcast constructor
5048  template <typename _Up,
5049  typename = enable_if_t<!is_same_v<__remove_cvref_t<_Up>, bool>>>
5050  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5051  simd(_ValuePreservingOrInt<_Up, value_type>&& __x)
5052  : _M_data(
5053  _Impl::_S_broadcast(static_cast<value_type>(static_cast<_Up&&>(__x))))
5054  {}
5055 
5056  // implicit type conversion constructor (convert from fixed_size to
5057  // fixed_size)
5058  template <typename _Up>
5059  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5060  simd(const simd<_Up, simd_abi::fixed_size<size()>>& __x,
5061  enable_if_t<
5062  conjunction<
5063  is_same<simd_abi::fixed_size<size()>, abi_type>,
5064  negation<__is_narrowing_conversion<_Up, value_type>>,
5065  __converts_to_higher_integer_rank<_Up, value_type>>::value,
5066  void*> = nullptr)
5067  : simd{static_cast<array<_Up, size()>>(__x).data(), vector_aligned} {}
5068 
5069  // explicit type conversion constructor
5070 #ifdef _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5071  template <typename _Up, typename _A2,
5072  typename = decltype(static_simd_cast<simd>(
5073  declval<const simd<_Up, _A2>&>()))>
5074  _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5075  simd(const simd<_Up, _A2>& __x)
5076  : simd(static_simd_cast<simd>(__x)) {}
5077 #endif // _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5078 
5079  // generator constructor
5080  template <typename _Fp>
5081  _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5082  simd(_Fp&& __gen, _ValuePreservingOrInt<decltype(declval<_Fp>()(
5083  declval<_SizeConstant<0>&>())),
5084  value_type>* = nullptr)
5085  : _M_data(_Impl::_S_generator(static_cast<_Fp&&>(__gen), _S_type_tag)) {}
5086 
5087  // load constructor
5088  template <typename _Up, typename _Flags>
5089  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5090  simd(const _Up* __mem, _IsSimdFlagType<_Flags>)
5091  : _M_data(
5092  _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag))
5093  {}
5094 
5095  // loads [simd.load]
5096  template <typename _Up, typename _Flags>
5097  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void
5098  copy_from(const _Vectorizable<_Up>* __mem, _IsSimdFlagType<_Flags>)
5099  {
5100  _M_data = static_cast<decltype(_M_data)>(
5101  _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag));
5102  }
5103 
5104  // stores [simd.store]
5105  template <typename _Up, typename _Flags>
5106  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void
5107  copy_to(_Vectorizable<_Up>* __mem, _IsSimdFlagType<_Flags>) const
5108  {
5109  _Impl::_S_store(_M_data, _Flags::template _S_apply<simd>(__mem),
5110  _S_type_tag);
5111  }
5112 
5113  // scalar access
5114  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
5115  operator[](size_t __i)
5116  { return {_M_data, int(__i)}; }
5117 
5118  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
5119  operator[]([[maybe_unused]] size_t __i) const
5120  {
5121  if constexpr (__is_scalar_abi<_Abi>())
5122  {
5123  _GLIBCXX_DEBUG_ASSERT(__i == 0);
5124  return _M_data;
5125  }
5126  else
5127  return _M_data[__i];
5128  }
5129 
5130  // increment and decrement:
5131  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5132  operator++()
5133  {
5134  _Impl::_S_increment(_M_data);
5135  return *this;
5136  }
5137 
5138  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5139  operator++(int)
5140  {
5141  simd __r = *this;
5142  _Impl::_S_increment(_M_data);
5143  return __r;
5144  }
5145 
5146  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5147  operator--()
5148  {
5149  _Impl::_S_decrement(_M_data);
5150  return *this;
5151  }
5152 
5153  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5154  operator--(int)
5155  {
5156  simd __r = *this;
5157  _Impl::_S_decrement(_M_data);
5158  return __r;
5159  }
5160 
5161  // unary operators (for any _Tp)
5162  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR mask_type
5163  operator!() const
5164  { return {__private_init, _Impl::_S_negate(_M_data)}; }
5165 
5166  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5167  operator+() const
5168  { return *this; }
5169 
5170  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5171  operator-() const
5172  { return {__private_init, _Impl::_S_unary_minus(_M_data)}; }
5173 
5174  // access to internal representation (suggested extension)
5175  _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5176  simd(_CastType __init) : _M_data(__init) {}
5177 
5178  // compound assignment [simd.cassign]
5179  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5180  operator+=(simd& __lhs, const simd& __x)
5181  { return __lhs = __lhs + __x; }
5182 
5183  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5184  operator-=(simd& __lhs, const simd& __x)
5185  { return __lhs = __lhs - __x; }
5186 
5187  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5188  operator*=(simd& __lhs, const simd& __x)
5189  { return __lhs = __lhs * __x; }
5190 
5191  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5192  operator/=(simd& __lhs, const simd& __x)
5193  { return __lhs = __lhs / __x; }
5194 
5195  // binary operators [simd.binary]
5196  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5197  operator+(const simd& __x, const simd& __y)
5198  { return {__private_init, _Impl::_S_plus(__x._M_data, __y._M_data)}; }
5199 
5200  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5201  operator-(const simd& __x, const simd& __y)
5202  { return {__private_init, _Impl::_S_minus(__x._M_data, __y._M_data)}; }
5203 
5204  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5205  operator*(const simd& __x, const simd& __y)
5206  { return {__private_init, _Impl::_S_multiplies(__x._M_data, __y._M_data)}; }
5207 
5208  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5209  operator/(const simd& __x, const simd& __y)
5210  { return {__private_init, _Impl::_S_divides(__x._M_data, __y._M_data)}; }
5211 
5212  // compares [simd.comparison]
5213  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5214  operator==(const simd& __x, const simd& __y)
5215  { return simd::_S_make_mask(_Impl::_S_equal_to(__x._M_data, __y._M_data)); }
5216 
5217  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5218  operator!=(const simd& __x, const simd& __y)
5219  {
5220  return simd::_S_make_mask(
5221  _Impl::_S_not_equal_to(__x._M_data, __y._M_data));
5222  }
5223 
5224  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5225  operator<(const simd& __x, const simd& __y)
5226  { return simd::_S_make_mask(_Impl::_S_less(__x._M_data, __y._M_data)); }
5227 
5228  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5229  operator<=(const simd& __x, const simd& __y)
5230  {
5231  return simd::_S_make_mask(_Impl::_S_less_equal(__x._M_data, __y._M_data));
5232  }
5233 
5234  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5235  operator>(const simd& __x, const simd& __y)
5236  { return simd::_S_make_mask(_Impl::_S_less(__y._M_data, __x._M_data)); }
5237 
5238  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5239  operator>=(const simd& __x, const simd& __y)
5240  {
5241  return simd::_S_make_mask(_Impl::_S_less_equal(__y._M_data, __x._M_data));
5242  }
5243 
5244  // operator?: overloads (suggested extension) {{{
5245 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
5246  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5247  operator?:(const mask_type& __k, const simd& __where_true,
5248  const simd& __where_false)
5249  {
5250  auto __ret = __where_false;
5251  _Impl::_S_masked_assign(__data(__k), __data(__ret), __data(__where_true));
5252  return __ret;
5253  }
5254 
5255 #endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__
5256  // }}}
5257 
5258  // "private" because of the first arguments's namespace
5259  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5260  simd(_PrivateInit, const _MemberType& __init)
5261  : _M_data(__init) {}
5262 
5263  // "private" because of the first arguments's namespace
5264  _GLIBCXX_SIMD_INTRINSIC
5265  simd(_BitsetInit, bitset<size()> __init) : _M_data()
5266  { where(mask_type(__bitset_init, __init), *this) = ~*this; }
5267 
5268  _GLIBCXX_SIMD_INTRINSIC constexpr bool
5269  _M_is_constprop() const
5270  {
5271  if constexpr (__is_scalar_abi<_Abi>())
5272  return __builtin_constant_p(_M_data);
5273  else
5274  return _M_data._M_is_constprop();
5275  }
5276 
5277  private:
5278  _GLIBCXX_SIMD_INTRINSIC static constexpr mask_type
5279  _S_make_mask(typename mask_type::_MemberType __k)
5280  { return {__private_init, __k}; }
5281 
5282  friend const auto& __data<value_type, abi_type>(const simd&);
5283  friend auto& __data<value_type, abi_type>(simd&);
5284  alignas(_Traits::_S_simd_align) _MemberType _M_data;
5285  };
5286 
5287 // }}}
5288 /// @cond undocumented
5289 // __data {{{
5290 template <typename _Tp, typename _Ap>
5291  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
5292  __data(const simd<_Tp, _Ap>& __x)
5293  { return __x._M_data; }
5294 
5295 template <typename _Tp, typename _Ap>
5296  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
5297  __data(simd<_Tp, _Ap>& __x)
5298  { return __x._M_data; }
5299 
5300 // }}}
5301 namespace __float_bitwise_operators { //{{{
5302 template <typename _Tp, typename _Ap>
5303  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5304  operator^(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5305  { return {__private_init, _Ap::_SimdImpl::_S_bit_xor(__data(__a), __data(__b))}; }
5306 
5307 template <typename _Tp, typename _Ap>
5308  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5309  operator|(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5310  { return {__private_init, _Ap::_SimdImpl::_S_bit_or(__data(__a), __data(__b))}; }
5311 
5312 template <typename _Tp, typename _Ap>
5313  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5314  operator&(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5315  { return {__private_init, _Ap::_SimdImpl::_S_bit_and(__data(__a), __data(__b))}; }
5316 } // namespace __float_bitwise_operators }}}
5317 /// @endcond
5318 
5319 /// @}
5320 _GLIBCXX_SIMD_END_NAMESPACE
5321 
5322 #endif // __cplusplus >= 201703L
5323 #endif // _GLIBCXX_EXPERIMENTAL_SIMD_H
5324 
5325 // vim: foldmethod=marker foldmarker={{{,}}}
constexpr duration< __common_rep_t< _Rep1, __disable_if_is_duration< _Rep2 > >, _Period > operator%(const duration< _Rep1, _Period > &__d, const _Rep2 &__s)
Definition: chrono:729
constexpr time_point< _Clock, typename common_type< duration< _Rep1, _Period1 >, _Dur2 >::type > operator+(const duration< _Rep1, _Period1 > &__lhs, const time_point< _Clock, _Dur2 > &__rhs)
Adjust a time point forwards by the given duration.
Definition: chrono:1016
constexpr duration< __common_rep_t< _Rep2, _Rep1 >, _Period > operator*(const _Rep1 &__s, const duration< _Rep2, _Period > &__d)
Definition: chrono:700
constexpr common_type< duration< _Rep1, _Period1 >, duration< _Rep2, _Period2 > >::type operator-(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
The difference between two durations.
Definition: chrono:660
constexpr duration< __common_rep_t< _Rep1, __disable_if_is_duration< _Rep2 > >, _Period > operator/(const duration< _Rep1, _Period > &__d, const _Rep2 &__s)
Definition: chrono:706
typename remove_reference< _Tp >::type remove_reference_t
Alias template for remove_reference.
Definition: type_traits:1639
typename make_unsigned< _Tp >::type make_unsigned_t
Alias template for make_unsigned.
Definition: type_traits:1974
void void_t
A metafunction that always yields void, used for detecting valid types.
Definition: type_traits:2601
integral_constant< bool, true > true_type
The type used as a compile-time boolean with true value.
Definition: type_traits:83
typename conditional< _Cond, _Iftrue, _Iffalse >::type conditional_t
Alias template for conditional.
Definition: type_traits:2583
typename remove_pointer< _Tp >::type remove_pointer_t
Alias template for remove_pointer.
Definition: type_traits:2049
integral_constant< bool, false > false_type
The type used as a compile-time boolean with false value.
Definition: type_traits:86
typename remove_const< _Tp >::type remove_const_t
Alias template for remove_const.
Definition: type_traits:1570
typename enable_if< _Cond, _Tp >::type enable_if_t
Alias template for enable_if.
Definition: type_traits:2579
constexpr auto tuple_cat(_Tpls &&... __tpls) -> typename __tuple_cat_result< _Tpls... >::__type
tuple_cat
Definition: tuple:1746
auto declval() noexcept -> decltype(__declval< _Tp >(0))
Definition: type_traits:2358
constexpr std::remove_reference< _Tp >::type && move(_Tp &&__t) noexcept
Convert a value to an rvalue.
Definition: move.h:104
void swap(any &__x, any &__y) noexcept
Exchange the states of two any objects.
Definition: any:428
_Tp * end(valarray< _Tp > &__va) noexcept
Return an iterator pointing to one past the last element of the valarray.
Definition: valarray:1239
_Tp * begin(valarray< _Tp > &__va) noexcept
Return an iterator pointing to the first element of the valarray.
Definition: valarray:1217
constexpr const _Tp & clamp(const _Tp &, const _Tp &, const _Tp &)
Returns the value clamped between lo and hi.
Definition: stl_algo.h:3656
constexpr const _Tp & max(const _Tp &, const _Tp &)
This does what you think it does.
Definition: stl_algobase.h:254
constexpr pair< const _Tp &, const _Tp & > minmax(const _Tp &, const _Tp &)
Determines min and max at once as an ordered pair.
Definition: stl_algo.h:3301
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
Definition: stl_algobase.h:230
constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op)
Calculate reduction of values in a range.
Definition: numeric:289
std::basic_istream< _CharT, _Traits > & operator>>(std::basic_istream< _CharT, _Traits > &__is, bitset< _Nb > &__x)
Global I/O operators for bitsets.
Definition: bitset:1472
bitset< _Nb > operator^(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1453
std::basic_ostream< _CharT, _Traits > & operator<<(std::basic_ostream< _CharT, _Traits > &__os, const bitset< _Nb > &__x)
Global I/O operators for bitsets.
Definition: bitset:1540
bitset< _Nb > operator|(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1444
bitset< _Nb > operator&(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1435
constexpr auto size(const _Container &__cont) noexcept(noexcept(__cont.size())) -> decltype(__cont.size())
Return the size of a container.
Definition: range_access.h:245
constexpr auto data(_Container &__cont) noexcept(noexcept(__cont.data())) -> decltype(__cont.data())
Return the data pointer of a container.
Definition: range_access.h:290