libstdc++
simd_x86.h
1// Simd x86 specific implementations -*- C++ -*-
2
3// Copyright (C) 2020-2023 Free Software Foundation, Inc.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_X86_H_
26#define _GLIBCXX_EXPERIMENTAL_SIMD_X86_H_
27
28#if __cplusplus >= 201703L
29
30#if !_GLIBCXX_SIMD_X86INTRIN
31#error \
32 "simd_x86.h may only be included when MMX or SSE on x86(_64) are available"
33#endif
34
35_GLIBCXX_SIMD_BEGIN_NAMESPACE
36
37// __to_masktype {{{
38// Given <T, N> return <__int_for_sizeof_t<T>, N>. For _SimdWrapper and
39// __vector_type_t.
40template <typename _Tp, size_t _Np>
41 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<__int_for_sizeof_t<_Tp>, _Np>
42 __to_masktype(_SimdWrapper<_Tp, _Np> __x)
43 { return reinterpret_cast<__vector_type_t<__int_for_sizeof_t<_Tp>, _Np>>(__x._M_data); }
44
45template <typename _TV,
46 typename _TVT
47 = enable_if_t<__is_vector_type_v<_TV>, _VectorTraits<_TV>>,
48 typename _Up = __int_for_sizeof_t<typename _TVT::value_type>>
49 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Up, _TVT::_S_full_size>
50 __to_masktype(_TV __x)
51 { return reinterpret_cast<__vector_type_t<_Up, _TVT::_S_full_size>>(__x); }
52
53// }}}
54// __interleave128_lo {{{
55template <typename _Ap, typename _Bp, typename _Tp = common_type_t<_Ap, _Bp>,
56 typename _Trait = _VectorTraits<_Tp>>
57 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
58 __interleave128_lo(const _Ap& __av, const _Bp& __bv)
59 {
60 const _Tp __a(__av);
61 const _Tp __b(__bv);
62 if constexpr (sizeof(_Tp) == 16 && _Trait::_S_full_size == 2)
63 return _Tp{__a[0], __b[0]};
64 else if constexpr (sizeof(_Tp) == 16 && _Trait::_S_full_size == 4)
65 return _Tp{__a[0], __b[0], __a[1], __b[1]};
66 else if constexpr (sizeof(_Tp) == 16 && _Trait::_S_full_size == 8)
67 return _Tp{__a[0], __b[0], __a[1], __b[1],
68 __a[2], __b[2], __a[3], __b[3]};
69 else if constexpr (sizeof(_Tp) == 16 && _Trait::_S_full_size == 16)
70 return _Tp{__a[0], __b[0], __a[1], __b[1], __a[2], __b[2],
71 __a[3], __b[3], __a[4], __b[4], __a[5], __b[5],
72 __a[6], __b[6], __a[7], __b[7]};
73 else if constexpr (sizeof(_Tp) == 32 && _Trait::_S_full_size == 4)
74 return _Tp{__a[0], __b[0], __a[2], __b[2]};
75 else if constexpr (sizeof(_Tp) == 32 && _Trait::_S_full_size == 8)
76 return _Tp{__a[0], __b[0], __a[1], __b[1],
77 __a[4], __b[4], __a[5], __b[5]};
78 else if constexpr (sizeof(_Tp) == 32 && _Trait::_S_full_size == 16)
79 return _Tp{__a[0], __b[0], __a[1], __b[1], __a[2], __b[2],
80 __a[3], __b[3], __a[8], __b[8], __a[9], __b[9],
81 __a[10], __b[10], __a[11], __b[11]};
82 else if constexpr (sizeof(_Tp) == 32 && _Trait::_S_full_size == 32)
83 return _Tp{__a[0], __b[0], __a[1], __b[1], __a[2], __b[2], __a[3],
84 __b[3], __a[4], __b[4], __a[5], __b[5], __a[6], __b[6],
85 __a[7], __b[7], __a[16], __b[16], __a[17], __b[17], __a[18],
86 __b[18], __a[19], __b[19], __a[20], __b[20], __a[21], __b[21],
87 __a[22], __b[22], __a[23], __b[23]};
88 else if constexpr (sizeof(_Tp) == 64 && _Trait::_S_full_size == 8)
89 return _Tp{__a[0], __b[0], __a[2], __b[2],
90 __a[4], __b[4], __a[6], __b[6]};
91 else if constexpr (sizeof(_Tp) == 64 && _Trait::_S_full_size == 16)
92 return _Tp{__a[0], __b[0], __a[1], __b[1], __a[4], __b[4],
93 __a[5], __b[5], __a[8], __b[8], __a[9], __b[9],
94 __a[12], __b[12], __a[13], __b[13]};
95 else if constexpr (sizeof(_Tp) == 64 && _Trait::_S_full_size == 32)
96 return _Tp{__a[0], __b[0], __a[1], __b[1], __a[2], __b[2], __a[3],
97 __b[3], __a[8], __b[8], __a[9], __b[9], __a[10], __b[10],
98 __a[11], __b[11], __a[16], __b[16], __a[17], __b[17], __a[18],
99 __b[18], __a[19], __b[19], __a[24], __b[24], __a[25], __b[25],
100 __a[26], __b[26], __a[27], __b[27]};
101 else if constexpr (sizeof(_Tp) == 64 && _Trait::_S_full_size == 64)
102 return _Tp{__a[0], __b[0], __a[1], __b[1], __a[2], __b[2], __a[3],
103 __b[3], __a[4], __b[4], __a[5], __b[5], __a[6], __b[6],
104 __a[7], __b[7], __a[16], __b[16], __a[17], __b[17], __a[18],
105 __b[18], __a[19], __b[19], __a[20], __b[20], __a[21], __b[21],
106 __a[22], __b[22], __a[23], __b[23], __a[32], __b[32], __a[33],
107 __b[33], __a[34], __b[34], __a[35], __b[35], __a[36], __b[36],
108 __a[37], __b[37], __a[38], __b[38], __a[39], __b[39], __a[48],
109 __b[48], __a[49], __b[49], __a[50], __b[50], __a[51], __b[51],
110 __a[52], __b[52], __a[53], __b[53], __a[54], __b[54], __a[55],
111 __b[55]};
112 else
113 __assert_unreachable<_Tp>();
114 }
115
116// }}}
117// __is_zero{{{
118template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
119 _GLIBCXX_SIMD_INTRINSIC constexpr bool
120 __is_zero(_Tp __a)
121 {
122 if (!__builtin_is_constant_evaluated())
123 {
124 if constexpr (__have_avx)
125 {
126 if constexpr (_TVT::template _S_is<float, 8>)
127 return _mm256_testz_ps(__a, __a);
128 else if constexpr (_TVT::template _S_is<double, 4>)
129 return _mm256_testz_pd(__a, __a);
130 else if constexpr (sizeof(_Tp) == 32)
131 return _mm256_testz_si256(__to_intrin(__a), __to_intrin(__a));
132 else if constexpr (_TVT::template _S_is<float>)
133 return _mm_testz_ps(__to_intrin(__a), __to_intrin(__a));
134 else if constexpr (_TVT::template _S_is<double, 2>)
135 return _mm_testz_pd(__a, __a);
136 else
137 return _mm_testz_si128(__to_intrin(__a), __to_intrin(__a));
138 }
139 else if constexpr (__have_sse4_1)
140 return _mm_testz_si128(__intrin_bitcast<__m128i>(__a),
141 __intrin_bitcast<__m128i>(__a));
142 }
143 else if constexpr (sizeof(_Tp) <= 8)
144 return reinterpret_cast<__int_for_sizeof_t<_Tp>>(__a) == 0;
145 else
146 {
147 const auto __b = __vector_bitcast<_LLong>(__a);
148 if constexpr (sizeof(__b) == 16)
149 return (__b[0] | __b[1]) == 0;
150 else if constexpr (sizeof(__b) == 32)
151 return __is_zero(__lo128(__b) | __hi128(__b));
152 else if constexpr (sizeof(__b) == 64)
153 return __is_zero(__lo256(__b) | __hi256(__b));
154 else
155 __assert_unreachable<_Tp>();
156 }
157 }
158
159// }}}
160// __movemask{{{
161template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
162 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST int
163 __movemask(_Tp __a)
164 {
165 if constexpr (sizeof(_Tp) == 32)
166 {
167 if constexpr (_TVT::template _S_is<float>)
168 return _mm256_movemask_ps(__to_intrin(__a));
169 else if constexpr (_TVT::template _S_is<double>)
170 return _mm256_movemask_pd(__to_intrin(__a));
171 else
172 return _mm256_movemask_epi8(__to_intrin(__a));
173 }
174 else if constexpr (_TVT::template _S_is<float>)
175 return _mm_movemask_ps(__to_intrin(__a));
176 else if constexpr (_TVT::template _S_is<double>)
177 return _mm_movemask_pd(__to_intrin(__a));
178 else
179 return _mm_movemask_epi8(__to_intrin(__a));
180 }
181
182// }}}
183// __testz{{{
184template <typename _TI, typename _TVT = _VectorTraits<_TI>>
185 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST constexpr int
186 __testz(_TI __a, _TI __b)
187 {
188 static_assert(is_same_v<_TI, __intrinsic_type_t<typename _TVT::value_type,
189 _TVT::_S_full_size>>);
190 if (!__builtin_is_constant_evaluated())
191 {
192 if constexpr (sizeof(_TI) == 32)
193 {
194 if constexpr (_TVT::template _S_is<float>)
195 return _mm256_testz_ps(__to_intrin(__a), __to_intrin(__b));
196 else if constexpr (_TVT::template _S_is<double>)
197 return _mm256_testz_pd(__to_intrin(__a), __to_intrin(__b));
198 else
199 return _mm256_testz_si256(__to_intrin(__a), __to_intrin(__b));
200 }
201 else if constexpr (_TVT::template _S_is<float> && __have_avx)
202 return _mm_testz_ps(__to_intrin(__a), __to_intrin(__b));
203 else if constexpr (_TVT::template _S_is<double> && __have_avx)
204 return _mm_testz_pd(__to_intrin(__a), __to_intrin(__b));
205 else if constexpr (__have_sse4_1)
206 return _mm_testz_si128(__intrin_bitcast<__m128i>(__to_intrin(__a)),
207 __intrin_bitcast<__m128i>(__to_intrin(__b)));
208 else
209 return __movemask(0 == __and(__a, __b)) != 0;
210 }
211 else
212 return __is_zero(__and(__a, __b));
213 }
214
215// }}}
216// __testc{{{
217// requires SSE4.1 or above
218template <typename _TI, typename _TVT = _VectorTraits<_TI>>
219 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST constexpr int
220 __testc(_TI __a, _TI __b)
221 {
222 static_assert(is_same_v<_TI, __intrinsic_type_t<typename _TVT::value_type,
223 _TVT::_S_full_size>>);
224 if (__builtin_is_constant_evaluated())
225 return __is_zero(__andnot(__a, __b));
226
227 if constexpr (sizeof(_TI) == 32)
228 {
229 if constexpr (_TVT::template _S_is<float>)
230 return _mm256_testc_ps(__a, __b);
231 else if constexpr (_TVT::template _S_is<double>)
232 return _mm256_testc_pd(__a, __b);
233 else
234 return _mm256_testc_si256(__to_intrin(__a), __to_intrin(__b));
235 }
236 else if constexpr (_TVT::template _S_is<float> && __have_avx)
237 return _mm_testc_ps(__to_intrin(__a), __to_intrin(__b));
238 else if constexpr (_TVT::template _S_is<double> && __have_avx)
239 return _mm_testc_pd(__to_intrin(__a), __to_intrin(__b));
240 else
241 {
242 static_assert(is_same_v<_TI, _TI> && __have_sse4_1);
243 return _mm_testc_si128(__intrin_bitcast<__m128i>(__to_intrin(__a)),
244 __intrin_bitcast<__m128i>(__to_intrin(__b)));
245 }
246 }
247
248// }}}
249// __testnzc{{{
250template <typename _TI, typename _TVT = _VectorTraits<_TI>>
251 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST constexpr int
252 __testnzc(_TI __a, _TI __b)
253 {
254 static_assert(is_same_v<_TI, __intrinsic_type_t<typename _TVT::value_type,
255 _TVT::_S_full_size>>);
256 if (!__builtin_is_constant_evaluated())
257 {
258 if constexpr (sizeof(_TI) == 32)
259 {
260 if constexpr (_TVT::template _S_is<float>)
261 return _mm256_testnzc_ps(__a, __b);
262 else if constexpr (_TVT::template _S_is<double>)
263 return _mm256_testnzc_pd(__a, __b);
264 else
265 return _mm256_testnzc_si256(__to_intrin(__a), __to_intrin(__b));
266 }
267 else if constexpr (_TVT::template _S_is<float> && __have_avx)
268 return _mm_testnzc_ps(__to_intrin(__a), __to_intrin(__b));
269 else if constexpr (_TVT::template _S_is<double> && __have_avx)
270 return _mm_testnzc_pd(__to_intrin(__a), __to_intrin(__b));
271 else if constexpr (__have_sse4_1)
272 return _mm_testnzc_si128(__intrin_bitcast<__m128i>(__to_intrin(__a)),
273 __intrin_bitcast<__m128i>(__to_intrin(__b)));
274 else
275 return __movemask(0 == __and(__a, __b)) == 0
276 && __movemask(0 == __andnot(__a, __b)) == 0;
277 }
278 else
279 return !(__is_zero(__and(__a, __b)) || __is_zero(__andnot(__a, __b)));
280 }
281
282// }}}
283// __xzyw{{{
284// shuffles the complete vector, swapping the inner two quarters. Often useful
285// for AVX for fixing up a shuffle result.
286template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
287 _GLIBCXX_SIMD_INTRINSIC _Tp
288 __xzyw(_Tp __a)
289 {
290 if constexpr (sizeof(_Tp) == 16)
291 {
292 const auto __x = __vector_bitcast<conditional_t<
293 is_floating_point_v<typename _TVT::value_type>, float, int>>(__a);
294 return reinterpret_cast<_Tp>(
295 decltype(__x){__x[0], __x[2], __x[1], __x[3]});
296 }
297 else if constexpr (sizeof(_Tp) == 32)
298 {
299 const auto __x = __vector_bitcast<conditional_t<
300 is_floating_point_v<typename _TVT::value_type>, double, _LLong>>(__a);
301 return reinterpret_cast<_Tp>(
302 decltype(__x){__x[0], __x[2], __x[1], __x[3]});
303 }
304 else if constexpr (sizeof(_Tp) == 64)
305 {
306 const auto __x = __vector_bitcast<conditional_t<
307 is_floating_point_v<typename _TVT::value_type>, double, _LLong>>(__a);
308 return reinterpret_cast<_Tp>(decltype(__x){__x[0], __x[1], __x[4],
309 __x[5], __x[2], __x[3],
310 __x[6], __x[7]});
311 }
312 else
313 __assert_unreachable<_Tp>();
314 }
315
316// }}}
317// __maskload_epi32{{{
318template <typename _Tp>
319 _GLIBCXX_SIMD_INTRINSIC auto
320 __maskload_epi32(const int* __ptr, _Tp __k)
321 {
322 if constexpr (sizeof(__k) == 16)
323 return _mm_maskload_epi32(__ptr, __k);
324 else
325 return _mm256_maskload_epi32(__ptr, __k);
326 }
327
328// }}}
329// __maskload_epi64{{{
330template <typename _Tp>
331 _GLIBCXX_SIMD_INTRINSIC auto
332 __maskload_epi64(const _LLong* __ptr, _Tp __k)
333 {
334 if constexpr (sizeof(__k) == 16)
335 return _mm_maskload_epi64(__ptr, __k);
336 else
337 return _mm256_maskload_epi64(__ptr, __k);
338 }
339
340// }}}
341// __maskload_ps{{{
342template <typename _Tp>
343 _GLIBCXX_SIMD_INTRINSIC auto
344 __maskload_ps(const float* __ptr, _Tp __k)
345 {
346 if constexpr (sizeof(__k) == 16)
347 return _mm_maskload_ps(__ptr, __k);
348 else
349 return _mm256_maskload_ps(__ptr, __k);
350 }
351
352// }}}
353// __maskload_pd{{{
354template <typename _Tp>
355 _GLIBCXX_SIMD_INTRINSIC auto
356 __maskload_pd(const double* __ptr, _Tp __k)
357 {
358 if constexpr (sizeof(__k) == 16)
359 return _mm_maskload_pd(__ptr, __k);
360 else
361 return _mm256_maskload_pd(__ptr, __k);
362 }
363
364// }}}
365
366#ifdef __clang__
367template <size_t _Np, typename _Tp, typename _Kp>
368 _GLIBCXX_SIMD_INTRINSIC constexpr auto
369 __movm(_Kp __k) noexcept
370 {
371 static_assert(is_unsigned_v<_Kp>);
372 if constexpr (sizeof(_Tp) == 1 && __have_avx512bw)
373 {
374 if constexpr (_Np <= 16 && __have_avx512vl)
375 return __builtin_ia32_cvtmask2b128(__k);
376 else if constexpr (_Np <= 32 && __have_avx512vl)
377 return __builtin_ia32_cvtmask2b256(__k);
378 else
379 return __builtin_ia32_cvtmask2b512(__k);
380 }
381 else if constexpr (sizeof(_Tp) == 2 && __have_avx512bw)
382 {
383 if constexpr (_Np <= 8 && __have_avx512vl)
384 return __builtin_ia32_cvtmask2w128(__k);
385 else if constexpr (_Np <= 16 && __have_avx512vl)
386 return __builtin_ia32_cvtmask2w256(__k);
387 else
388 return __builtin_ia32_cvtmask2w512(__k);
389 }
390 else if constexpr (sizeof(_Tp) == 4 && __have_avx512dq)
391 {
392 if constexpr (_Np <= 4 && __have_avx512vl)
393 return __builtin_ia32_cvtmask2d128(__k);
394 else if constexpr (_Np <= 8 && __have_avx512vl)
395 return __builtin_ia32_cvtmask2d256(__k);
396 else
397 return __builtin_ia32_cvtmask2d512(__k);
398 }
399 else if constexpr (sizeof(_Tp) == 8 && __have_avx512dq)
400 {
401 if constexpr (_Np <= 2 && __have_avx512vl)
402 return __builtin_ia32_cvtmask2q128(__k);
403 else if constexpr (_Np <= 4 && __have_avx512vl)
404 return __builtin_ia32_cvtmask2q256(__k);
405 else
406 return __builtin_ia32_cvtmask2q512(__k);
407 }
408 else
409 __assert_unreachable<_Tp>();
410 }
411#endif // __clang__
412
413#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
414#include "simd_x86_conversions.h"
415#endif
416
417// ISA & type detection {{{
418template <typename _Tp, size_t _Np>
419 constexpr bool
420 __is_sse_ps()
421 {
422 return __have_sse
423 && is_same_v<_Tp,
424 float> && sizeof(__intrinsic_type_t<_Tp, _Np>) == 16;
425 }
426
427template <typename _Tp, size_t _Np>
428 constexpr bool
429 __is_sse_pd()
430 {
431 return __have_sse2
432 && is_same_v<_Tp,
433 double> && sizeof(__intrinsic_type_t<_Tp, _Np>) == 16;
434 }
435
436template <typename _Tp, size_t _Np>
437 constexpr bool
438 __is_avx_ps()
439 {
440 return __have_avx
441 && is_same_v<_Tp,
442 float> && sizeof(__intrinsic_type_t<_Tp, _Np>) == 32;
443 }
444
445template <typename _Tp, size_t _Np>
446 constexpr bool
447 __is_avx_pd()
448 {
449 return __have_avx
450 && is_same_v<_Tp,
451 double> && sizeof(__intrinsic_type_t<_Tp, _Np>) == 32;
452 }
453
454template <typename _Tp, size_t _Np>
455 constexpr bool
456 __is_avx512_ps()
457 {
458 return __have_avx512f
459 && is_same_v<_Tp,
460 float> && sizeof(__intrinsic_type_t<_Tp, _Np>) == 64;
461 }
462
463template <typename _Tp, size_t _Np>
464 constexpr bool
465 __is_avx512_pd()
466 {
467 return __have_avx512f
468 && is_same_v<_Tp,
469 double> && sizeof(__intrinsic_type_t<_Tp, _Np>) == 64;
470 }
471
472// }}}
473struct _MaskImplX86Mixin;
474
475// _CommonImplX86 {{{
476struct _CommonImplX86 : _CommonImplBuiltin
477{
478#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
479 // _S_converts_via_decomposition {{{
480 template <typename _From, typename _To, size_t _ToSize>
481 static constexpr bool
482 _S_converts_via_decomposition()
483 {
484 if constexpr (is_integral_v<
485 _From> && is_integral_v<_To> && sizeof(_From) == 8
486 && _ToSize == 16)
487 return (sizeof(_To) == 2 && !__have_ssse3)
488 || (sizeof(_To) == 1 && !__have_avx512f);
489 else if constexpr (is_floating_point_v<_From> && is_integral_v<_To>)
490 return ((sizeof(_From) == 4 || sizeof(_From) == 8) && sizeof(_To) == 8
491 && !__have_avx512dq)
492 || (sizeof(_From) == 8 && sizeof(_To) == 4 && !__have_sse4_1
493 && _ToSize == 16);
494 else if constexpr (
495 is_integral_v<_From> && is_floating_point_v<_To> && sizeof(_From) == 8
496 && !__have_avx512dq)
497 return (sizeof(_To) == 4 && _ToSize == 16)
498 || (sizeof(_To) == 8 && _ToSize < 64);
499 else
500 return false;
501 }
502
503 template <typename _From, typename _To, size_t _ToSize>
504 static inline constexpr bool __converts_via_decomposition_v
505 = _S_converts_via_decomposition<_From, _To, _ToSize>();
506
507 // }}}
508#endif
509 // _S_store {{{
510 using _CommonImplBuiltin::_S_store;
511
512 template <typename _Tp, size_t _Np>
513 _GLIBCXX_SIMD_INTRINSIC static void
514 _S_store(_SimdWrapper<_Tp, _Np> __x, void* __addr)
515 {
516 constexpr size_t _Bytes = _Np * sizeof(_Tp);
517
518 if constexpr ((_Bytes & (_Bytes - 1)) != 0 && __have_avx512bw_vl)
519 {
520 const auto __v = __to_intrin(__x);
521
522 if constexpr (_Bytes & 1)
523 {
524 if constexpr (_Bytes < 16)
525 _mm_mask_storeu_epi8(__addr, 0xffffu >> (16 - _Bytes),
526 __intrin_bitcast<__m128i>(__v));
527 else if constexpr (_Bytes < 32)
528 _mm256_mask_storeu_epi8(__addr, 0xffffffffu >> (32 - _Bytes),
529 __intrin_bitcast<__m256i>(__v));
530 else
531 _mm512_mask_storeu_epi8(__addr,
532 0xffffffffffffffffull >> (64 - _Bytes),
533 __intrin_bitcast<__m512i>(__v));
534 }
535 else if constexpr (_Bytes & 2)
536 {
537 if constexpr (_Bytes < 16)
538 _mm_mask_storeu_epi16(__addr, 0xffu >> (8 - _Bytes / 2),
539 __intrin_bitcast<__m128i>(__v));
540 else if constexpr (_Bytes < 32)
541 _mm256_mask_storeu_epi16(__addr, 0xffffu >> (16 - _Bytes / 2),
542 __intrin_bitcast<__m256i>(__v));
543 else
544 _mm512_mask_storeu_epi16(__addr,
545 0xffffffffull >> (32 - _Bytes / 2),
546 __intrin_bitcast<__m512i>(__v));
547 }
548 else if constexpr (_Bytes & 4)
549 {
550 if constexpr (_Bytes < 16)
551 _mm_mask_storeu_epi32(__addr, 0xfu >> (4 - _Bytes / 4),
552 __intrin_bitcast<__m128i>(__v));
553 else if constexpr (_Bytes < 32)
554 _mm256_mask_storeu_epi32(__addr, 0xffu >> (8 - _Bytes / 4),
555 __intrin_bitcast<__m256i>(__v));
556 else
557 _mm512_mask_storeu_epi32(__addr, 0xffffull >> (16 - _Bytes / 4),
558 __intrin_bitcast<__m512i>(__v));
559 }
560 else
561 {
562 static_assert(
563 _Bytes > 16,
564 "_Bytes < 16 && (_Bytes & 7) == 0 && (_Bytes & (_Bytes "
565 "- 1)) != 0 is impossible");
566 if constexpr (_Bytes < 32)
567 _mm256_mask_storeu_epi64(__addr, 0xfu >> (4 - _Bytes / 8),
568 __intrin_bitcast<__m256i>(__v));
569 else
570 _mm512_mask_storeu_epi64(__addr, 0xffull >> (8 - _Bytes / 8),
571 __intrin_bitcast<__m512i>(__v));
572 }
573 }
574 else
575 _CommonImplBuiltin::_S_store(__x, __addr);
576 }
577
578 // }}}
579 // _S_store_bool_array(_BitMask) {{{
580 template <size_t _Np, bool _Sanitized>
581 _GLIBCXX_SIMD_INTRINSIC static constexpr void
582 _S_store_bool_array(const _BitMask<_Np, _Sanitized> __x, bool* __mem)
583 {
584 if constexpr (__have_avx512bw_vl) // don't care for BW w/o VL
585 _S_store<_Np>(1 & __vector_bitcast<_UChar, _Np>(
586 [=]() constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
587 if constexpr (_Np <= 16)
588 return _mm_movm_epi8(__x._M_to_bits());
589 else if constexpr (_Np <= 32)
590 return _mm256_movm_epi8(__x._M_to_bits());
591 else if constexpr (_Np <= 64)
592 return _mm512_movm_epi8(__x._M_to_bits());
593 else
594 __assert_unreachable<_SizeConstant<_Np>>();
595 }()),
596 __mem);
597 else if constexpr (__have_bmi2)
598 {
599 if constexpr (_Np <= 4)
600 _S_store<_Np>(_pdep_u32(__x._M_to_bits(), 0x01010101U), __mem);
601 else
602 __execute_n_times<__div_roundup(_Np, sizeof(size_t))>(
603 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
604 constexpr size_t __offset = __i * sizeof(size_t);
605 constexpr int __todo = std::min(sizeof(size_t), _Np - __offset);
606 if constexpr (__todo == 1)
607 __mem[__offset] = __x[__offset];
608 else
609 {
610 const auto __bools =
611#ifdef __x86_64__
612 _pdep_u64(__x.template _M_extract<__offset>().to_ullong(),
613 0x0101010101010101ULL);
614#else // __x86_64__
615 _pdep_u32(
616 __x.template _M_extract<__offset>()._M_to_bits(),
617 0x01010101U);
618#endif // __x86_64__
619 _S_store<__todo>(__bools, __mem + __offset);
620 }
621 });
622 }
623 else if constexpr (__have_sse2 && _Np > 7)
624 __execute_n_times<__div_roundup(_Np, 16)>([&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
625 constexpr int __offset = __i * 16;
626 constexpr int __todo = std::min(16, int(_Np) - __offset);
627 const int __bits = __x.template _M_extract<__offset>()._M_to_bits();
628 __vector_type16_t<_UChar> __bools;
629 if constexpr (__have_avx512f)
630 {
631 auto __as32bits
632 = _mm512_maskz_mov_epi32(__bits, __to_intrin(
633 __vector_broadcast<16>(1)));
634 auto __as16bits
635 = __xzyw(_mm256_packs_epi32(__lo256(__as32bits),
636 __todo > 8 ? __hi256(__as32bits)
637 : __m256i()));
638 __bools = __vector_bitcast<_UChar>(
639 _mm_packs_epi16(__lo128(__as16bits), __hi128(__as16bits)));
640 }
641 else
642 {
643 using _V = __vector_type_t<_UChar, 16>;
644 auto __tmp = _mm_cvtsi32_si128(__bits);
645 __tmp = _mm_unpacklo_epi8(__tmp, __tmp);
646 __tmp = _mm_unpacklo_epi16(__tmp, __tmp);
647 __tmp = _mm_unpacklo_epi32(__tmp, __tmp);
648 _V __tmp2 = reinterpret_cast<_V>(__tmp);
649 __tmp2 &= _V{1, 2, 4, 8, 16, 32, 64, 128,
650 1, 2, 4, 8, 16, 32, 64, 128}; // mask bit index
651 __bools = (__tmp2 == 0) + 1; // 0xff -> 0x00 | 0x00 -> 0x01
652 }
653 _S_store<__todo>(__bools, __mem + __offset);
654 });
655 else
656 _CommonImplBuiltin::_S_store_bool_array(__x, __mem);
657 }
658
659 // }}}
660 // _S_blend_avx512 {{{
661 // Returns: __k ? __b : __a
662 // TODO: reverse __a and __b to match COND_EXPR
663 // Requires: _TV to be a __vector_type_t matching valuetype for the bitmask
664 // __k
665 template <typename _Kp, typename _TV>
666 _GLIBCXX_SIMD_INTRINSIC static _TV
667 _S_blend_avx512(const _Kp __k, const _TV __a, const _TV __b) noexcept
668 {
669 static_assert(__is_vector_type_v<_TV>);
670 using _Tp = typename _VectorTraits<_TV>::value_type;
671 static_assert(sizeof(_TV) >= 16);
672 static_assert(sizeof(_Tp) <= 8);
673#ifdef __clang__
674 return __movm<_VectorTraits<_TV>::_S_full_size, _Tp>(__k) ? __b : __a;
675#else
676 using _IntT
677 = conditional_t<(sizeof(_Tp) > 2),
678 conditional_t<sizeof(_Tp) == 4, int, long long>,
679 conditional_t<sizeof(_Tp) == 1, char, short>>;
680 [[maybe_unused]] const auto __aa = __vector_bitcast<_IntT>(__a);
681 [[maybe_unused]] const auto __bb = __vector_bitcast<_IntT>(__b);
682 if constexpr (sizeof(_TV) == 64)
683 {
684 if constexpr (sizeof(_Tp) == 1)
685 return reinterpret_cast<_TV>(
686 __builtin_ia32_blendmb_512_mask(__aa, __bb, __k));
687 else if constexpr (sizeof(_Tp) == 2)
688 return reinterpret_cast<_TV>(
689 __builtin_ia32_blendmw_512_mask(__aa, __bb, __k));
690 else if constexpr (sizeof(_Tp) == 4 && is_floating_point_v<_Tp>)
691 return __builtin_ia32_blendmps_512_mask(__a, __b, __k);
692 else if constexpr (sizeof(_Tp) == 4)
693 return reinterpret_cast<_TV>(
694 __builtin_ia32_blendmd_512_mask(__aa, __bb, __k));
695 else if constexpr (sizeof(_Tp) == 8 && is_floating_point_v<_Tp>)
696 return __builtin_ia32_blendmpd_512_mask(__a, __b, __k);
697 else if constexpr (sizeof(_Tp) == 8)
698 return reinterpret_cast<_TV>(
699 __builtin_ia32_blendmq_512_mask(__aa, __bb, __k));
700 }
701 else if constexpr (sizeof(_TV) == 32)
702 {
703 if constexpr (sizeof(_Tp) == 1)
704 return reinterpret_cast<_TV>(
705 __builtin_ia32_blendmb_256_mask(__aa, __bb, __k));
706 else if constexpr (sizeof(_Tp) == 2)
707 return reinterpret_cast<_TV>(
708 __builtin_ia32_blendmw_256_mask(__aa, __bb, __k));
709 else if constexpr (sizeof(_Tp) == 4 && is_floating_point_v<_Tp>)
710 return __builtin_ia32_blendmps_256_mask(__a, __b, __k);
711 else if constexpr (sizeof(_Tp) == 4)
712 return reinterpret_cast<_TV>(
713 __builtin_ia32_blendmd_256_mask(__aa, __bb, __k));
714 else if constexpr (sizeof(_Tp) == 8 && is_floating_point_v<_Tp>)
715 return __builtin_ia32_blendmpd_256_mask(__a, __b, __k);
716 else if constexpr (sizeof(_Tp) == 8)
717 return reinterpret_cast<_TV>(
718 __builtin_ia32_blendmq_256_mask(__aa, __bb, __k));
719 }
720 else if constexpr (sizeof(_TV) == 16)
721 {
722 if constexpr (sizeof(_Tp) == 1)
723 return reinterpret_cast<_TV>(
724 __builtin_ia32_blendmb_128_mask(__aa, __bb, __k));
725 else if constexpr (sizeof(_Tp) == 2)
726 return reinterpret_cast<_TV>(
727 __builtin_ia32_blendmw_128_mask(__aa, __bb, __k));
728 else if constexpr (sizeof(_Tp) == 4 && is_floating_point_v<_Tp>)
729 return __builtin_ia32_blendmps_128_mask(__a, __b, __k);
730 else if constexpr (sizeof(_Tp) == 4)
731 return reinterpret_cast<_TV>(
732 __builtin_ia32_blendmd_128_mask(__aa, __bb, __k));
733 else if constexpr (sizeof(_Tp) == 8 && is_floating_point_v<_Tp>)
734 return __builtin_ia32_blendmpd_128_mask(__a, __b, __k);
735 else if constexpr (sizeof(_Tp) == 8)
736 return reinterpret_cast<_TV>(
737 __builtin_ia32_blendmq_128_mask(__aa, __bb, __k));
738 }
739#endif
740 }
741
742 // }}}
743 // _S_blend_intrin {{{
744 // Returns: __k ? __b : __a
745 // TODO: reverse __a and __b to match COND_EXPR
746 // Requires: _Tp to be an intrinsic type (integers blend per byte) and 16/32
747 // Bytes wide
748 template <typename _Tp>
749 _GLIBCXX_SIMD_INTRINSIC static _Tp
750 _S_blend_intrin(_Tp __k, _Tp __a, _Tp __b) noexcept
751 {
752 static_assert(is_same_v<decltype(__to_intrin(__a)), _Tp>);
753 constexpr struct
754 {
755 _GLIBCXX_SIMD_INTRINSIC __m128 operator()(__m128 __a, __m128 __b,
756 __m128 __k) const noexcept
757 {
758 return __builtin_ia32_blendvps(__a, __b, __k);
759 }
760 _GLIBCXX_SIMD_INTRINSIC __m128d operator()(__m128d __a, __m128d __b,
761 __m128d __k) const noexcept
762 {
763 return __builtin_ia32_blendvpd(__a, __b, __k);
764 }
765 _GLIBCXX_SIMD_INTRINSIC __m128i operator()(__m128i __a, __m128i __b,
766 __m128i __k) const noexcept
767 {
768 return reinterpret_cast<__m128i>(
769 __builtin_ia32_pblendvb128(reinterpret_cast<__v16qi>(__a),
770 reinterpret_cast<__v16qi>(__b),
771 reinterpret_cast<__v16qi>(__k)));
772 }
773 _GLIBCXX_SIMD_INTRINSIC __m256 operator()(__m256 __a, __m256 __b,
774 __m256 __k) const noexcept
775 {
776 return __builtin_ia32_blendvps256(__a, __b, __k);
777 }
778 _GLIBCXX_SIMD_INTRINSIC __m256d operator()(__m256d __a, __m256d __b,
779 __m256d __k) const noexcept
780 {
781 return __builtin_ia32_blendvpd256(__a, __b, __k);
782 }
783 _GLIBCXX_SIMD_INTRINSIC __m256i operator()(__m256i __a, __m256i __b,
784 __m256i __k) const noexcept
785 {
786 if constexpr (__have_avx2)
787 return reinterpret_cast<__m256i>(
788 __builtin_ia32_pblendvb256(reinterpret_cast<__v32qi>(__a),
789 reinterpret_cast<__v32qi>(__b),
790 reinterpret_cast<__v32qi>(__k)));
791 else
792 return reinterpret_cast<__m256i>(
793 __builtin_ia32_blendvps256(reinterpret_cast<__v8sf>(__a),
794 reinterpret_cast<__v8sf>(__b),
795 reinterpret_cast<__v8sf>(__k)));
796 }
797 } __eval;
798 return __eval(__a, __b, __k);
799 }
800
801 // }}}
802 // _S_blend {{{
803 // Returns: __k ? __at1 : __at0
804 // TODO: reverse __at0 and __at1 to match COND_EXPR
805 template <typename _Tp, size_t _Np>
806 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
807 _S_blend(_SimdWrapper<bool, _Np> __k, _SimdWrapper<_Tp, _Np> __at0,
808 _SimdWrapper<_Tp, _Np> __at1)
809 {
810 static_assert(is_same_v<_Tp, _Tp> && __have_avx512f);
811 if (__k._M_is_constprop() && __at0._M_is_constprop()
812 && __at1._M_is_constprop())
813 return __generate_from_n_evaluations<_Np, __vector_type_t<_Tp, _Np>>(
814 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
815 return __k[__i] ? __at1[__i] : __at0[__i];
816 });
817 else if constexpr (sizeof(__at0) == 64
818 || (__have_avx512vl && sizeof(__at0) >= 16))
819 return _S_blend_avx512(__k._M_data, __at0._M_data, __at1._M_data);
820 else
821 {
822 static_assert((__have_avx512vl && sizeof(__at0) < 16)
823 || !__have_avx512vl);
824 constexpr size_t __size = (__have_avx512vl ? 16 : 64) / sizeof(_Tp);
825 return __vector_bitcast<_Tp, _Np>(
826 _S_blend_avx512(__k._M_data, __vector_bitcast<_Tp, __size>(__at0),
827 __vector_bitcast<_Tp, __size>(__at1)));
828 }
829 }
830
831 template <typename _Tp, size_t _Np>
832 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
833 _S_blend(_SimdWrapper<__int_for_sizeof_t<_Tp>, _Np> __k,
834 _SimdWrapper<_Tp, _Np> __at0, _SimdWrapper<_Tp, _Np> __at1)
835 {
836 const auto __kk = __wrapper_bitcast<_Tp>(__k);
837 if (__builtin_is_constant_evaluated()
838 || (__kk._M_is_constprop() && __at0._M_is_constprop()
839 && __at1._M_is_constprop()))
840 {
841 auto __r = __or(__andnot(__kk, __at0), __and(__kk, __at1));
842 if (__r._M_is_constprop())
843 return __r;
844 }
845 if constexpr (((__have_avx512f && sizeof(__at0) == 64) || __have_avx512vl)
846 && (sizeof(_Tp) >= 4 || __have_avx512bw))
847 // convert to bitmask and call overload above
848 return _S_blend(
849 _SimdWrapper<bool, _Np>(
850 __make_dependent_t<_Tp, _MaskImplX86Mixin>::_S_to_bits(__k)
851 ._M_to_bits()),
852 __at0, __at1);
853 else
854 {
855 // Since GCC does not assume __k to be a mask, using the builtin
856 // conditional operator introduces an extra compare against 0 before
857 // blending. So we rather call the intrinsic here.
858 if constexpr (__have_sse4_1)
859 return _S_blend_intrin(__to_intrin(__kk), __to_intrin(__at0),
860 __to_intrin(__at1));
861 else
862 return __or(__andnot(__kk, __at0), __and(__kk, __at1));
863 }
864 }
865
866 // }}}
867};
868
869// }}}
870// _SimdImplX86 {{{
871template <typename _Abi, typename>
872 struct _SimdImplX86 : _SimdImplBuiltin<_Abi>
873 {
874 using _Base = _SimdImplBuiltin<_Abi>;
875
876 template <typename _Tp>
877 using _MaskMember = typename _Base::template _MaskMember<_Tp>;
878
879 template <typename _Tp>
880 static constexpr size_t _S_full_size = _Abi::template _S_full_size<_Tp>;
881
882 template <typename _Tp>
883 static constexpr size_t _S_size = _Abi::template _S_size<_Tp>;
884
885 template <typename _Tp>
886 static constexpr size_t _S_max_store_size
887 = (sizeof(_Tp) >= 4 && __have_avx512f) || __have_avx512bw ? 64
888 : (is_floating_point_v<_Tp>&& __have_avx) || __have_avx2 ? 32
889 : 16;
890
891 using _MaskImpl = typename _Abi::_MaskImpl;
892
893 // _S_masked_load {{{
894 template <typename _Tp, size_t _Np, typename _Up>
895 static inline _SimdWrapper<_Tp, _Np>
896 _S_masked_load(_SimdWrapper<_Tp, _Np> __merge, _MaskMember<_Tp> __k,
897 const _Up* __mem) noexcept
898 {
899 static_assert(_Np == _S_size<_Tp>);
900 if constexpr (is_same_v<_Tp, _Up> || // no conversion
901 (sizeof(_Tp) == sizeof(_Up)
902 && is_integral_v<
903 _Tp> == is_integral_v<_Up>) // conversion via bit
904 // reinterpretation
905 )
906 {
907 [[maybe_unused]] const auto __intrin = __to_intrin(__merge);
908 if constexpr ((__is_avx512_abi<_Abi>() || __have_avx512bw_vl)
909 && sizeof(_Tp) == 1)
910 {
911 const auto __kk = _MaskImpl::_S_to_bits(__k)._M_to_bits();
912 if constexpr (sizeof(__intrin) == 16)
913 __merge = __vector_bitcast<_Tp, _Np>(
914 _mm_mask_loadu_epi8(__intrin, __kk, __mem));
915 else if constexpr (sizeof(__merge) == 32)
916 __merge = __vector_bitcast<_Tp, _Np>(
917 _mm256_mask_loadu_epi8(__intrin, __kk, __mem));
918 else if constexpr (sizeof(__merge) == 64)
919 __merge = __vector_bitcast<_Tp, _Np>(
920 _mm512_mask_loadu_epi8(__intrin, __kk, __mem));
921 else
922 __assert_unreachable<_Tp>();
923 }
924 else if constexpr ((__is_avx512_abi<_Abi>() || __have_avx512bw_vl)
925 && sizeof(_Tp) == 2)
926 {
927 const auto __kk = _MaskImpl::_S_to_bits(__k)._M_to_bits();
928 if constexpr (sizeof(__intrin) == 16)
929 __merge = __vector_bitcast<_Tp, _Np>(
930 _mm_mask_loadu_epi16(__intrin, __kk, __mem));
931 else if constexpr (sizeof(__intrin) == 32)
932 __merge = __vector_bitcast<_Tp, _Np>(
933 _mm256_mask_loadu_epi16(__intrin, __kk, __mem));
934 else if constexpr (sizeof(__intrin) == 64)
935 __merge = __vector_bitcast<_Tp, _Np>(
936 _mm512_mask_loadu_epi16(__intrin, __kk, __mem));
937 else
938 __assert_unreachable<_Tp>();
939 }
940 else if constexpr ((__is_avx512_abi<_Abi>() || __have_avx512vl)
941 && sizeof(_Tp) == 4 && is_integral_v<_Up>)
942 {
943 const auto __kk = _MaskImpl::_S_to_bits(__k)._M_to_bits();
944 if constexpr (sizeof(__intrin) == 16)
945 __merge = __vector_bitcast<_Tp, _Np>(
946 _mm_mask_loadu_epi32(__intrin, __kk, __mem));
947 else if constexpr (sizeof(__intrin) == 32)
948 __merge = __vector_bitcast<_Tp, _Np>(
949 _mm256_mask_loadu_epi32(__intrin, __kk, __mem));
950 else if constexpr (sizeof(__intrin) == 64)
951 __merge = __vector_bitcast<_Tp, _Np>(
952 _mm512_mask_loadu_epi32(__intrin, __kk, __mem));
953 else
954 __assert_unreachable<_Tp>();
955 }
956 else if constexpr ((__is_avx512_abi<_Abi>() || __have_avx512vl)
957 && sizeof(_Tp) == 4 && is_floating_point_v<_Up>)
958 {
959 const auto __kk = _MaskImpl::_S_to_bits(__k)._M_to_bits();
960 if constexpr (sizeof(__intrin) == 16)
961 __merge = __vector_bitcast<_Tp, _Np>(
962 _mm_mask_loadu_ps(__intrin, __kk, __mem));
963 else if constexpr (sizeof(__intrin) == 32)
964 __merge = __vector_bitcast<_Tp, _Np>(
965 _mm256_mask_loadu_ps(__intrin, __kk, __mem));
966 else if constexpr (sizeof(__intrin) == 64)
967 __merge = __vector_bitcast<_Tp, _Np>(
968 _mm512_mask_loadu_ps(__intrin, __kk, __mem));
969 else
970 __assert_unreachable<_Tp>();
971 }
972 else if constexpr (__have_avx2 && sizeof(_Tp) == 4
973 && is_integral_v<_Up>)
974 {
975 static_assert(sizeof(__intrin) == 16 || sizeof(__intrin) == 32);
976 __merge
977 = __or(__andnot(__vector_bitcast<_Tp>(__k), __merge._M_data),
978 __vector_bitcast<_Tp, _Np>(
979 __maskload_epi32(reinterpret_cast<const int*>(__mem),
980 __to_intrin(__k))));
981 }
982 else if constexpr (__have_avx && sizeof(_Tp) == 4)
983 {
984 static_assert(sizeof(__intrin) == 16 || sizeof(__intrin) == 32);
985 __merge
986 = __or(__andnot(__vector_bitcast<_Tp>(__k), __merge._M_data),
987 __vector_bitcast<_Tp, _Np>(
988 __maskload_ps(reinterpret_cast<const float*>(__mem),
989 __to_intrin(__k))));
990 }
991 else if constexpr ((__is_avx512_abi<_Abi>() || __have_avx512vl)
992 && sizeof(_Tp) == 8 && is_integral_v<_Up>)
993 {
994 const auto __kk = _MaskImpl::_S_to_bits(__k)._M_to_bits();
995 if constexpr (sizeof(__intrin) == 16)
996 __merge = __vector_bitcast<_Tp, _Np>(
997 _mm_mask_loadu_epi64(__intrin, __kk, __mem));
998 else if constexpr (sizeof(__intrin) == 32)
999 __merge = __vector_bitcast<_Tp, _Np>(
1000 _mm256_mask_loadu_epi64(__intrin, __kk, __mem));
1001 else if constexpr (sizeof(__intrin) == 64)
1002 __merge = __vector_bitcast<_Tp, _Np>(
1003 _mm512_mask_loadu_epi64(__intrin, __kk, __mem));
1004 else
1005 __assert_unreachable<_Tp>();
1006 }
1007 else if constexpr ((__is_avx512_abi<_Abi>() || __have_avx512vl)
1008 && sizeof(_Tp) == 8 && is_floating_point_v<_Up>)
1009 {
1010 const auto __kk = _MaskImpl::_S_to_bits(__k)._M_to_bits();
1011 if constexpr (sizeof(__intrin) == 16)
1012 __merge = __vector_bitcast<_Tp, _Np>(
1013 _mm_mask_loadu_pd(__intrin, __kk, __mem));
1014 else if constexpr (sizeof(__intrin) == 32)
1015 __merge = __vector_bitcast<_Tp, _Np>(
1016 _mm256_mask_loadu_pd(__intrin, __kk, __mem));
1017 else if constexpr (sizeof(__intrin) == 64)
1018 __merge = __vector_bitcast<_Tp, _Np>(
1019 _mm512_mask_loadu_pd(__intrin, __kk, __mem));
1020 else
1021 __assert_unreachable<_Tp>();
1022 }
1023 else if constexpr (__have_avx2 && sizeof(_Tp) == 8
1024 && is_integral_v<_Up>)
1025 {
1026 static_assert(sizeof(__intrin) == 16 || sizeof(__intrin) == 32);
1027 __merge
1028 = __or(__andnot(__vector_bitcast<_Tp>(__k), __merge._M_data),
1029 __vector_bitcast<_Tp, _Np>(__maskload_epi64(
1030 reinterpret_cast<const _LLong*>(__mem),
1031 __to_intrin(__k))));
1032 }
1033 else if constexpr (__have_avx && sizeof(_Tp) == 8)
1034 {
1035 static_assert(sizeof(__intrin) == 16 || sizeof(__intrin) == 32);
1036 __merge
1037 = __or(__andnot(__vector_bitcast<_Tp>(__k), __merge._M_data),
1038 __vector_bitcast<_Tp, _Np>(
1039 __maskload_pd(reinterpret_cast<const double*>(__mem),
1040 __to_intrin(__k))));
1041 }
1042 else
1043 _BitOps::_S_bit_iteration(_MaskImpl::_S_to_bits(__k),
1044 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1045 __merge._M_set(__i, static_cast<_Tp>(__mem[__i]));
1046 });
1047 }
1048 /* Very uncertain, that the following improves anything. Needs
1049 benchmarking
1050 * before it's activated.
1051 else if constexpr (sizeof(_Up) <= 8 && // no long double
1052 !__converts_via_decomposition_v<
1053 _Up, _Tp,
1054 sizeof(__merge)> // conversion via decomposition
1055 // is better handled via the
1056 // bit_iteration fallback below
1057 )
1058 {
1059 // TODO: copy pattern from _S_masked_store, which doesn't resort to
1060 // fixed_size
1061 using _Ap = simd_abi::deduce_t<_Up, _Np>;
1062 using _ATraits = _SimdTraits<_Up, _Ap>;
1063 using _AImpl = typename _ATraits::_SimdImpl;
1064 typename _ATraits::_SimdMember __uncvted{};
1065 typename _ATraits::_MaskMember __kk = _Ap::_MaskImpl::template
1066 _S_convert<_Up>(__k);
1067 __uncvted = _AImpl::_S_masked_load(__uncvted, __kk, __mem);
1068 _SimdConverter<_Up, _Ap, _Tp, _Abi> __converter;
1069 _Base::_S_masked_assign(__k, __merge, __converter(__uncvted));
1070 }
1071 */
1072 else
1073 __merge = _Base::_S_masked_load(__merge, __k, __mem);
1074 return __merge;
1075 }
1076
1077 // }}}
1078 // _S_masked_store_nocvt {{{
1079 template <typename _Tp, size_t _Np>
1080 _GLIBCXX_SIMD_INTRINSIC static void
1081 _S_masked_store_nocvt(_SimdWrapper<_Tp, _Np> __v, _Tp* __mem, _SimdWrapper<bool, _Np> __k)
1082 {
1083 [[maybe_unused]] const auto __vi = __to_intrin(__v);
1084 if constexpr (sizeof(__vi) == 64)
1085 {
1086 static_assert(sizeof(__v) == 64 && __have_avx512f);
1087 if constexpr (__have_avx512bw && sizeof(_Tp) == 1)
1088 _mm512_mask_storeu_epi8(__mem, __k, __vi);
1089 else if constexpr (__have_avx512bw && sizeof(_Tp) == 2)
1090 _mm512_mask_storeu_epi16(__mem, __k, __vi);
1091 else if constexpr (__have_avx512f && sizeof(_Tp) == 4)
1092 {
1093 if constexpr (is_integral_v<_Tp>)
1094 _mm512_mask_storeu_epi32(__mem, __k, __vi);
1095 else
1096 _mm512_mask_storeu_ps(__mem, __k, __vi);
1097 }
1098 else if constexpr (__have_avx512f && sizeof(_Tp) == 8)
1099 {
1100 if constexpr (is_integral_v<_Tp>)
1101 _mm512_mask_storeu_epi64(__mem, __k, __vi);
1102 else
1103 _mm512_mask_storeu_pd(__mem, __k, __vi);
1104 }
1105#if 0 // with KNL either sizeof(_Tp) >= 4 or sizeof(_vi) <= 32
1106 // with Skylake-AVX512, __have_avx512bw is true
1107 else if constexpr (__have_sse2)
1108 {
1109 using _M = __vector_type_t<_Tp, _Np>;
1110 using _MVT = _VectorTraits<_M>;
1111 _mm_maskmoveu_si128(__auto_bitcast(__extract<0, 4>(__v._M_data)),
1112 __auto_bitcast(_MaskImpl::template _S_convert<_Tp, _Np>(__k._M_data)),
1113 reinterpret_cast<char*>(__mem));
1114 _mm_maskmoveu_si128(__auto_bitcast(__extract<1, 4>(__v._M_data)),
1115 __auto_bitcast(_MaskImpl::template _S_convert<_Tp, _Np>(
1116 __k._M_data >> 1 * _MVT::_S_full_size)),
1117 reinterpret_cast<char*>(__mem) + 1 * 16);
1118 _mm_maskmoveu_si128(__auto_bitcast(__extract<2, 4>(__v._M_data)),
1119 __auto_bitcast(_MaskImpl::template _S_convert<_Tp, _Np>(
1120 __k._M_data >> 2 * _MVT::_S_full_size)),
1121 reinterpret_cast<char*>(__mem) + 2 * 16);
1122 if constexpr (_Np > 48 / sizeof(_Tp))
1123 _mm_maskmoveu_si128(
1124 __auto_bitcast(__extract<3, 4>(__v._M_data)),
1125 __auto_bitcast(_MaskImpl::template _S_convert<_Tp, _Np>(
1126 __k._M_data >> 3 * _MVT::_S_full_size)),
1127 reinterpret_cast<char*>(__mem) + 3 * 16);
1128 }
1129#endif
1130 else
1131 __assert_unreachable<_Tp>();
1132 }
1133 else if constexpr (sizeof(__vi) == 32)
1134 {
1135 if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 1)
1136 _mm256_mask_storeu_epi8(__mem, __k, __vi);
1137 else if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 2)
1138 _mm256_mask_storeu_epi16(__mem, __k, __vi);
1139 else if constexpr (__have_avx512vl && sizeof(_Tp) == 4)
1140 {
1141 if constexpr (is_integral_v<_Tp>)
1142 _mm256_mask_storeu_epi32(__mem, __k, __vi);
1143 else
1144 _mm256_mask_storeu_ps(__mem, __k, __vi);
1145 }
1146 else if constexpr (__have_avx512vl && sizeof(_Tp) == 8)
1147 {
1148 if constexpr (is_integral_v<_Tp>)
1149 _mm256_mask_storeu_epi64(__mem, __k, __vi);
1150 else
1151 _mm256_mask_storeu_pd(__mem, __k, __vi);
1152 }
1153 else if constexpr (__have_avx512f
1154 && (sizeof(_Tp) >= 4 || __have_avx512bw))
1155 {
1156 // use a 512-bit maskstore, using zero-extension of the bitmask
1157 _S_masked_store_nocvt(
1158 _SimdWrapper64<_Tp>(
1159 __intrin_bitcast<__vector_type64_t<_Tp>>(__v._M_data)),
1160 __mem, _SimdWrapper<bool, 64 / sizeof(_Tp)>(__k._M_data));
1161 }
1162 else
1163 _S_masked_store_nocvt(__v, __mem,
1164 _MaskImpl::template _S_to_maskvector<
1165 __int_for_sizeof_t<_Tp>, _Np>(__k));
1166 }
1167 else if constexpr (sizeof(__vi) == 16)
1168 {
1169 if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 1)
1170 _mm_mask_storeu_epi8(__mem, __k, __vi);
1171 else if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 2)
1172 _mm_mask_storeu_epi16(__mem, __k, __vi);
1173 else if constexpr (__have_avx512vl && sizeof(_Tp) == 4)
1174 {
1175 if constexpr (is_integral_v<_Tp>)
1176 _mm_mask_storeu_epi32(__mem, __k, __vi);
1177 else
1178 _mm_mask_storeu_ps(__mem, __k, __vi);
1179 }
1180 else if constexpr (__have_avx512vl && sizeof(_Tp) == 8)
1181 {
1182 if constexpr (is_integral_v<_Tp>)
1183 _mm_mask_storeu_epi64(__mem, __k, __vi);
1184 else
1185 _mm_mask_storeu_pd(__mem, __k, __vi);
1186 }
1187 else if constexpr (__have_avx512f
1188 && (sizeof(_Tp) >= 4 || __have_avx512bw))
1189 {
1190 // use a 512-bit maskstore, using zero-extension of the bitmask
1191 _S_masked_store_nocvt(
1192 _SimdWrapper64<_Tp>(
1193 __intrin_bitcast<__intrinsic_type64_t<_Tp>>(__v._M_data)),
1194 __mem, _SimdWrapper<bool, 64 / sizeof(_Tp)>(__k._M_data));
1195 }
1196 else
1197 _S_masked_store_nocvt(__v, __mem,
1198 _MaskImpl::template _S_to_maskvector<
1199 __int_for_sizeof_t<_Tp>, _Np>(__k));
1200 }
1201 else
1202 __assert_unreachable<_Tp>();
1203 }
1204
1205 template <typename _Tp, size_t _Np>
1206 _GLIBCXX_SIMD_INTRINSIC static void
1207 _S_masked_store_nocvt(_SimdWrapper<_Tp, _Np> __v, _Tp* __mem,
1208 _SimdWrapper<__int_for_sizeof_t<_Tp>, _Np> __k)
1209 {
1210 if constexpr (sizeof(__v) <= 16)
1211 {
1212 [[maybe_unused]] const auto __vi
1213 = __intrin_bitcast<__m128i>(__as_vector(__v));
1214 [[maybe_unused]] const auto __ki
1215 = __intrin_bitcast<__m128i>(__as_vector(__k));
1216 if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 1)
1217 _mm_mask_storeu_epi8(__mem, _mm_movepi8_mask(__ki), __vi);
1218 else if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 2)
1219 _mm_mask_storeu_epi16(__mem, _mm_movepi16_mask(__ki), __vi);
1220 else if constexpr (__have_avx2 && sizeof(_Tp) == 4
1221 && is_integral_v<_Tp>)
1222 _mm_maskstore_epi32(reinterpret_cast<int*>(__mem), __ki, __vi);
1223 else if constexpr (__have_avx && sizeof(_Tp) == 4)
1224 _mm_maskstore_ps(reinterpret_cast<float*>(__mem), __ki,
1225 __vector_bitcast<float>(__vi));
1226 else if constexpr (__have_avx2 && sizeof(_Tp) == 8
1227 && is_integral_v<_Tp>)
1228 _mm_maskstore_epi64(reinterpret_cast<_LLong*>(__mem), __ki, __vi);
1229 else if constexpr (__have_avx && sizeof(_Tp) == 8)
1230 _mm_maskstore_pd(reinterpret_cast<double*>(__mem), __ki,
1231 __vector_bitcast<double>(__vi));
1232 else if constexpr (__have_sse2)
1233 _mm_maskmoveu_si128(__vi, __ki, reinterpret_cast<char*>(__mem));
1234 }
1235 else if constexpr (sizeof(__v) == 32)
1236 {
1237 [[maybe_unused]] const auto __vi
1238 = __intrin_bitcast<__m256i>(__as_vector(__v));
1239 [[maybe_unused]] const auto __ki
1240 = __intrin_bitcast<__m256i>(__as_vector(__k));
1241 if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 1)
1242 _mm256_mask_storeu_epi8(__mem, _mm256_movepi8_mask(__ki), __vi);
1243 else if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 2)
1244 _mm256_mask_storeu_epi16(__mem, _mm256_movepi16_mask(__ki), __vi);
1245 else if constexpr (__have_avx2 && sizeof(_Tp) == 4
1246 && is_integral_v<_Tp>)
1247 _mm256_maskstore_epi32(reinterpret_cast<int*>(__mem), __ki, __vi);
1248 else if constexpr (sizeof(_Tp) == 4)
1249 _mm256_maskstore_ps(reinterpret_cast<float*>(__mem), __ki,
1250 __vector_bitcast<float>(__v));
1251 else if constexpr (__have_avx2 && sizeof(_Tp) == 8
1252 && is_integral_v<_Tp>)
1253 _mm256_maskstore_epi64(reinterpret_cast<_LLong*>(__mem), __ki,
1254 __vi);
1255 else if constexpr (__have_avx && sizeof(_Tp) == 8)
1256 _mm256_maskstore_pd(reinterpret_cast<double*>(__mem), __ki,
1257 __vector_bitcast<double>(__v));
1258 else if constexpr (__have_sse2)
1259 {
1260 _mm_maskmoveu_si128(__lo128(__vi), __lo128(__ki),
1261 reinterpret_cast<char*>(__mem));
1262 _mm_maskmoveu_si128(__hi128(__vi), __hi128(__ki),
1263 reinterpret_cast<char*>(__mem) + 16);
1264 }
1265 }
1266 else
1267 __assert_unreachable<_Tp>();
1268 }
1269
1270 // }}}
1271 // _S_masked_store {{{
1272 template <typename _Tp, size_t _Np, typename _Up>
1273 _GLIBCXX_SIMD_INTRINSIC static void
1274 _S_masked_store(const _SimdWrapper<_Tp, _Np> __v, _Up* __mem,
1275 const _MaskMember<_Tp> __k) noexcept
1276 {
1277 if constexpr (is_integral_v<
1278 _Tp> && is_integral_v<_Up> && sizeof(_Tp) > sizeof(_Up)
1279 && __have_avx512f && (sizeof(_Tp) >= 4 || __have_avx512bw)
1280 && (sizeof(__v) == 64 || __have_avx512vl))
1281 { // truncating store
1282 const auto __vi = __to_intrin(__v);
1283 const auto __kk = _MaskImpl::_S_to_bits(__k)._M_to_bits();
1284 if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 4
1285 && sizeof(__vi) == 64)
1286 _mm512_mask_cvtepi64_storeu_epi32(__mem, __kk, __vi);
1287 else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 4
1288 && sizeof(__vi) == 32)
1289 _mm256_mask_cvtepi64_storeu_epi32(__mem, __kk, __vi);
1290 else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 4
1291 && sizeof(__vi) == 16)
1292 _mm_mask_cvtepi64_storeu_epi32(__mem, __kk, __vi);
1293 else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 2
1294 && sizeof(__vi) == 64)
1295 _mm512_mask_cvtepi64_storeu_epi16(__mem, __kk, __vi);
1296 else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 2
1297 && sizeof(__vi) == 32)
1298 _mm256_mask_cvtepi64_storeu_epi16(__mem, __kk, __vi);
1299 else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 2
1300 && sizeof(__vi) == 16)
1301 _mm_mask_cvtepi64_storeu_epi16(__mem, __kk, __vi);
1302 else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 1
1303 && sizeof(__vi) == 64)
1304 _mm512_mask_cvtepi64_storeu_epi8(__mem, __kk, __vi);
1305 else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 1
1306 && sizeof(__vi) == 32)
1307 _mm256_mask_cvtepi64_storeu_epi8(__mem, __kk, __vi);
1308 else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 1
1309 && sizeof(__vi) == 16)
1310 _mm_mask_cvtepi64_storeu_epi8(__mem, __kk, __vi);
1311 else if constexpr (sizeof(_Tp) == 4 && sizeof(_Up) == 2
1312 && sizeof(__vi) == 64)
1313 _mm512_mask_cvtepi32_storeu_epi16(__mem, __kk, __vi);
1314 else if constexpr (sizeof(_Tp) == 4 && sizeof(_Up) == 2
1315 && sizeof(__vi) == 32)
1316 _mm256_mask_cvtepi32_storeu_epi16(__mem, __kk, __vi);
1317 else if constexpr (sizeof(_Tp) == 4 && sizeof(_Up) == 2
1318 && sizeof(__vi) == 16)
1319 _mm_mask_cvtepi32_storeu_epi16(__mem, __kk, __vi);
1320 else if constexpr (sizeof(_Tp) == 4 && sizeof(_Up) == 1
1321 && sizeof(__vi) == 64)
1322 _mm512_mask_cvtepi32_storeu_epi8(__mem, __kk, __vi);
1323 else if constexpr (sizeof(_Tp) == 4 && sizeof(_Up) == 1
1324 && sizeof(__vi) == 32)
1325 _mm256_mask_cvtepi32_storeu_epi8(__mem, __kk, __vi);
1326 else if constexpr (sizeof(_Tp) == 4 && sizeof(_Up) == 1
1327 && sizeof(__vi) == 16)
1328 _mm_mask_cvtepi32_storeu_epi8(__mem, __kk, __vi);
1329 else if constexpr (sizeof(_Tp) == 2 && sizeof(_Up) == 1
1330 && sizeof(__vi) == 64)
1331 _mm512_mask_cvtepi16_storeu_epi8(__mem, __kk, __vi);
1332 else if constexpr (sizeof(_Tp) == 2 && sizeof(_Up) == 1
1333 && sizeof(__vi) == 32)
1334 _mm256_mask_cvtepi16_storeu_epi8(__mem, __kk, __vi);
1335 else if constexpr (sizeof(_Tp) == 2 && sizeof(_Up) == 1
1336 && sizeof(__vi) == 16)
1337 _mm_mask_cvtepi16_storeu_epi8(__mem, __kk, __vi);
1338 else
1339 __assert_unreachable<_Tp>();
1340 }
1341 else
1342 _Base::_S_masked_store(__v, __mem, __k);
1343 }
1344
1345 // }}}
1346 // _S_multiplies {{{
1347 template <typename _V, typename _VVT = _VectorTraits<_V>>
1348 _GLIBCXX_SIMD_INTRINSIC static constexpr _V
1349 _S_multiplies(_V __x, _V __y)
1350 {
1351 using _Tp = typename _VVT::value_type;
1352 if (__builtin_is_constant_evaluated() || __x._M_is_constprop()
1353 || __y._M_is_constprop())
1354 return __as_vector(__x) * __as_vector(__y);
1355 else if constexpr (sizeof(_Tp) == 1)
1356 {
1357 if constexpr (sizeof(_V) == 2)
1358 {
1359 const auto __xs = reinterpret_cast<short>(__x._M_data);
1360 const auto __ys = reinterpret_cast<short>(__y._M_data);
1361 return reinterpret_cast<__vector_type_t<_Tp, 2>>(short(
1362 ((__xs * __ys) & 0xff) | ((__xs >> 8) * (__ys & 0xff00))));
1363 }
1364 else if constexpr (sizeof(_V) == 4 && _VVT::_S_partial_width == 3)
1365 {
1366 const auto __xi = reinterpret_cast<int>(__x._M_data);
1367 const auto __yi = reinterpret_cast<int>(__y._M_data);
1368 return reinterpret_cast<__vector_type_t<_Tp, 3>>(
1369 ((__xi * __yi) & 0xff)
1370 | (((__xi >> 8) * (__yi & 0xff00)) & 0xff00)
1371 | ((__xi >> 16) * (__yi & 0xff0000)));
1372 }
1373 else if constexpr (sizeof(_V) == 4)
1374 {
1375 const auto __xi = reinterpret_cast<int>(__x._M_data);
1376 const auto __yi = reinterpret_cast<int>(__y._M_data);
1377 return reinterpret_cast<__vector_type_t<_Tp, 4>>(
1378 ((__xi * __yi) & 0xff)
1379 | (((__xi >> 8) * (__yi & 0xff00)) & 0xff00)
1380 | (((__xi >> 16) * (__yi & 0xff0000)) & 0xff0000)
1381 | ((__xi >> 24) * (__yi & 0xff000000u)));
1382 }
1383 else if constexpr (sizeof(_V) == 8 && __have_avx2
1384 && is_signed_v<_Tp>)
1385 return __convert<typename _VVT::type>(
1386 __vector_bitcast<short>(_mm_cvtepi8_epi16(__to_intrin(__x)))
1387 * __vector_bitcast<short>(_mm_cvtepi8_epi16(__to_intrin(__y))));
1388 else if constexpr (sizeof(_V) == 8 && __have_avx2
1389 && is_unsigned_v<_Tp>)
1390 return __convert<typename _VVT::type>(
1391 __vector_bitcast<short>(_mm_cvtepu8_epi16(__to_intrin(__x)))
1392 * __vector_bitcast<short>(_mm_cvtepu8_epi16(__to_intrin(__y))));
1393 else
1394 {
1395 // codegen of `x*y` is suboptimal (as of GCC 9.0.1)
1396 constexpr size_t __full_size = _VVT::_S_full_size;
1397 constexpr int _Np = sizeof(_V) >= 16 ? __full_size / 2 : 8;
1398 using _ShortW = _SimdWrapper<short, _Np>;
1399 const _ShortW __even = __vector_bitcast<short, _Np>(__x)
1400 * __vector_bitcast<short, _Np>(__y);
1401 _ShortW __high_byte = _ShortW()._M_data - 256;
1402 //[&]() { asm("" : "+x"(__high_byte._M_data)); }();
1403 const _ShortW __odd
1404 = (__vector_bitcast<short, _Np>(__x) >> 8)
1405 * (__vector_bitcast<short, _Np>(__y) & __high_byte._M_data);
1406 if constexpr (__have_avx512bw && sizeof(_V) > 2)
1407 return _CommonImplX86::_S_blend_avx512(
1408 0xaaaa'aaaa'aaaa'aaaaLL, __vector_bitcast<_Tp>(__even),
1409 __vector_bitcast<_Tp>(__odd));
1410 else if constexpr (__have_sse4_1 && sizeof(_V) > 2)
1411 return _CommonImplX86::_S_blend_intrin(__to_intrin(
1412 __high_byte),
1413 __to_intrin(__even),
1414 __to_intrin(__odd));
1415 else
1416 return __to_intrin(
1417 __or(__andnot(__high_byte, __even), __odd));
1418 }
1419 }
1420 else
1421 return _Base::_S_multiplies(__x, __y);
1422 }
1423
1424 // }}}
1425 // _S_divides {{{
1426#ifdef _GLIBCXX_SIMD_WORKAROUND_PR90993
1427 template <typename _Tp, size_t _Np>
1428 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
1429 _S_divides(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
1430 {
1431 if (!__builtin_is_constant_evaluated()
1432 && !__builtin_constant_p(__y._M_data))
1433 if constexpr (is_integral_v<_Tp> && sizeof(_Tp) <= 4)
1434 { // use divps - codegen of `x/y` is suboptimal (as of GCC 9.0.1)
1435 // Note that using floating-point division is likely to raise the
1436 // *Inexact* exception flag and thus appears like an invalid
1437 // "as-if" transformation. However, C++ doesn't specify how the
1438 // fpenv can be observed and points to C. C says that function
1439 // calls are assumed to potentially raise fp exceptions, unless
1440 // documented otherwise. Consequently, operator/, which is a
1441 // function call, may raise fp exceptions.
1442 /*const struct _CsrGuard
1443 {
1444 const unsigned _M_data = _mm_getcsr();
1445 _CsrGuard()
1446 {
1447 _mm_setcsr(0x9f80); // turn off FP exceptions and
1448 flush-to-zero
1449 }
1450 ~_CsrGuard() { _mm_setcsr(_M_data); }
1451 } __csr;*/
1452 using _Float = conditional_t<sizeof(_Tp) == 4, double, float>;
1453 constexpr size_t __n_intermediate
1454 = std::min(_Np, (__have_avx512f ? 64
1455 : __have_avx ? 32
1456 : 16)
1457 / sizeof(_Float));
1458 using _FloatV = __vector_type_t<_Float, __n_intermediate>;
1459 constexpr size_t __n_floatv
1460 = __div_roundup(_Np, __n_intermediate);
1461 using _R = __vector_type_t<_Tp, _Np>;
1462 const auto __xf = __convert_all<_FloatV, __n_floatv>(__x);
1463 const auto __yf = __convert_all<_FloatV, __n_floatv>(
1464 _Abi::__make_padding_nonzero(__as_vector(__y)));
1465 return __call_with_n_evaluations<__n_floatv>(
1466 [](auto... __quotients) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1467 return __vector_convert<_R>(__quotients...);
1468 },
1469 [&__xf, &__yf](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
1470 -> _SimdWrapper<_Float, __n_intermediate>
1471 {
1472#if __RECIPROCAL_MATH__
1473 // If -freciprocal-math is active, using the `/` operator is
1474 // incorrect because it may be translated to an imprecise
1475 // multiplication with reciprocal. We need to use inline
1476 // assembly to force a real division.
1477 _FloatV __r;
1478 if constexpr (__have_avx) // -mno-sse2avx is irrelevant
1479 // because once -mavx is given, GCC
1480 // emits VEX encoded vdivp[sd]
1481 {
1482 if constexpr (sizeof(_Tp) == 4)
1483 asm("vdivpd\t{%2, %1, %0|%0, %1, %2}"
1484 : "=x"(__r)
1485 : "x"(__xf[__i]), "x"(__yf[__i]));
1486 else
1487 asm("vdivps\t{%2, %1, %0|%0, %1, %2}"
1488 : "=x"(__r)
1489 : "x"(__xf[__i]), "x"(__yf[__i]));
1490 }
1491 else
1492 {
1493 __r = __xf[__i];
1494 if constexpr (sizeof(_Tp) == 4)
1495 asm("divpd\t{%1, %0|%0, %1}"
1496 : "=x"(__r)
1497 : "x"(__yf[__i]));
1498 else
1499 asm("divps\t{%1, %0|%0, %1}"
1500 : "=x"(__r)
1501 : "x"(__yf[__i]));
1502 }
1503 return __r;
1504#else
1505 return __xf[__i] / __yf[__i];
1506#endif
1507 });
1508 }
1509 /* 64-bit int division is potentially optimizable via double division if
1510 * the value in __x is small enough and the conversion between
1511 * int<->double is efficient enough:
1512 else if constexpr (is_integral_v<_Tp> && is_unsigned_v<_Tp> &&
1513 sizeof(_Tp) == 8)
1514 {
1515 if constexpr (__have_sse4_1 && sizeof(__x) == 16)
1516 {
1517 if (_mm_test_all_zeros(__x, __m128i{0xffe0'0000'0000'0000ull,
1518 0xffe0'0000'0000'0000ull}))
1519 {
1520 __x._M_data | 0x __vector_convert<__m128d>(__x._M_data)
1521 }
1522 }
1523 }
1524 */
1525 return _Base::_S_divides(__x, __y);
1526 }
1527#else
1528 using _Base::_S_divides;
1529#endif // _GLIBCXX_SIMD_WORKAROUND_PR90993
1530
1531 // }}}
1532 // _S_modulus {{{
1533 template <typename _Tp, size_t _Np>
1534 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
1535 _S_modulus(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
1536 {
1537 if (__builtin_is_constant_evaluated()
1538 || __builtin_constant_p(__y._M_data) || sizeof(_Tp) >= 8)
1539 return _Base::_S_modulus(__x, __y);
1540 else
1541 return _Base::_S_minus(__x, _S_multiplies(__y, _S_divides(__x, __y)));
1542 }
1543
1544 // }}}
1545 // _S_bit_shift_left {{{
1546 // Notes on UB. C++2a [expr.shift] says:
1547 // -1- [...] The operands shall be of integral or unscoped enumeration type
1548 // and integral promotions are performed. The type of the result is that
1549 // of the promoted left operand. The behavior is undefined if the right
1550 // operand is negative, or greater than or equal to the width of the
1551 // promoted left operand.
1552 // -2- The value of E1 << E2 is the unique value congruent to E1×2^E2 modulo
1553 // 2^N, where N is the width of the type of the result.
1554 //
1555 // C++17 [expr.shift] says:
1556 // -2- The value of E1 << E2 is E1 left-shifted E2 bit positions; vacated
1557 // bits are zero-filled. If E1 has an unsigned type, the value of the
1558 // result is E1 × 2^E2 , reduced modulo one more than the maximum value
1559 // representable in the result type. Otherwise, if E1 has a signed type
1560 // and non-negative value, and E1 × 2^E2 is representable in the
1561 // corresponding unsigned type of the result type, then that value,
1562 // converted to the result type, is the resulting value; otherwise, the
1563 // behavior is undefined.
1564 //
1565 // Consequences:
1566 // With C++2a signed and unsigned types have the same UB
1567 // characteristics:
1568 // - left shift is not UB for 0 <= RHS < max(32, #bits(T))
1569 //
1570 // With C++17 there's little room for optimizations because the standard
1571 // requires all shifts to happen on promoted integrals (i.e. int). Thus,
1572 // short and char shifts must assume shifts affect bits of neighboring
1573 // values.
1574 #ifndef _GLIBCXX_SIMD_NO_SHIFT_OPT
1575 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1576 constexpr inline _GLIBCXX_CONST static typename _TVT::type
1577 _S_bit_shift_left(_Tp __xx, int __y)
1578 {
1579 using _V = typename _TVT::type;
1580 using _Up = typename _TVT::value_type;
1581 _V __x = __xx;
1582 [[maybe_unused]] const auto __ix = __to_intrin(__x);
1583 if (__builtin_is_constant_evaluated())
1584 return __x << __y;
1585#if __cplusplus > 201703
1586 // after C++17, signed shifts have no UB, and behave just like unsigned
1587 // shifts
1588 else if constexpr (sizeof(_Up) == 1 && is_signed_v<_Up>)
1589 return __vector_bitcast<_Up>(
1590 _S_bit_shift_left(__vector_bitcast<make_unsigned_t<_Up>>(__x),
1591 __y));
1592#endif
1593 else if constexpr (sizeof(_Up) == 1)
1594 {
1595 // (cf. https://gcc.gnu.org/bugzilla/show_bug.cgi?id=83894)
1596 if (__builtin_constant_p(__y))
1597 {
1598 if (__y == 0)
1599 return __x;
1600 else if (__y == 1)
1601 return __x + __x;
1602 else if (__y == 2)
1603 {
1604 __x = __x + __x;
1605 return __x + __x;
1606 }
1607 else if (__y > 2 && __y < 8)
1608 {
1609 if constexpr (sizeof(__x) > sizeof(unsigned))
1610 {
1611 const _UChar __mask = 0xff << __y; // precomputed vector
1612 return __vector_bitcast<_Up>(
1613 __vector_bitcast<_UChar>(
1614 __vector_bitcast<unsigned>(__x) << __y)
1615 & __mask);
1616 }
1617 else
1618 {
1619 const unsigned __mask
1620 = (0xff & (0xff << __y)) * 0x01010101u;
1621 return reinterpret_cast<_V>(
1622 static_cast<__int_for_sizeof_t<_V>>(
1623 unsigned(
1624 reinterpret_cast<__int_for_sizeof_t<_V>>(__x)
1625 << __y)
1626 & __mask));
1627 }
1628 }
1629 else if (__y >= 8 && __y < 32)
1630 return _V();
1631 else
1632 __builtin_unreachable();
1633 }
1634 // general strategy in the following: use an sllv instead of sll
1635 // instruction, because it's 2 to 4 times faster:
1636 else if constexpr (__have_avx512bw_vl && sizeof(__x) == 16)
1637 return __vector_bitcast<_Up>(_mm256_cvtepi16_epi8(
1638 _mm256_sllv_epi16(_mm256_cvtepi8_epi16(__ix),
1639 _mm256_set1_epi16(__y))));
1640 else if constexpr (__have_avx512bw && sizeof(__x) == 32)
1641 return __vector_bitcast<_Up>(_mm512_cvtepi16_epi8(
1642 _mm512_sllv_epi16(_mm512_cvtepi8_epi16(__ix),
1643 _mm512_set1_epi16(__y))));
1644 else if constexpr (__have_avx512bw && sizeof(__x) == 64)
1645 {
1646 const auto __shift = _mm512_set1_epi16(__y);
1647 return __vector_bitcast<_Up>(
1648 __concat(_mm512_cvtepi16_epi8(_mm512_sllv_epi16(
1649 _mm512_cvtepi8_epi16(__lo256(__ix)), __shift)),
1650 _mm512_cvtepi16_epi8(_mm512_sllv_epi16(
1651 _mm512_cvtepi8_epi16(__hi256(__ix)), __shift))));
1652 }
1653 else if constexpr (__have_avx2 && sizeof(__x) == 32)
1654 {
1655#if 1
1656 const auto __shift = _mm_cvtsi32_si128(__y);
1657 auto __k
1658 = _mm256_sll_epi16(_mm256_slli_epi16(~__m256i(), 8), __shift);
1659 __k |= _mm256_srli_epi16(__k, 8);
1660 return __vector_bitcast<_Up>(_mm256_sll_epi32(__ix, __shift)
1661 & __k);
1662#else
1663 const _Up __k = 0xff << __y;
1664 return __vector_bitcast<_Up>(__vector_bitcast<int>(__x) << __y)
1665 & __k;
1666#endif
1667 }
1668 else
1669 {
1670 const auto __shift = _mm_cvtsi32_si128(__y);
1671 auto __k
1672 = _mm_sll_epi16(_mm_slli_epi16(~__m128i(), 8), __shift);
1673 __k |= _mm_srli_epi16(__k, 8);
1674 return __intrin_bitcast<_V>(_mm_sll_epi16(__ix, __shift) & __k);
1675 }
1676 }
1677 return __x << __y;
1678 }
1679
1680 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1681 constexpr inline _GLIBCXX_CONST static typename _TVT::type
1682 _S_bit_shift_left(_Tp __xx, typename _TVT::type __y)
1683 {
1684 using _V = typename _TVT::type;
1685 using _Up = typename _TVT::value_type;
1686 _V __x = __xx;
1687 [[maybe_unused]] const auto __ix = __to_intrin(__x);
1688 [[maybe_unused]] const auto __iy = __to_intrin(__y);
1689 if (__builtin_is_constant_evaluated())
1690 return __x << __y;
1691#if __cplusplus > 201703
1692 // after C++17, signed shifts have no UB, and behave just like unsigned
1693 // shifts
1694 else if constexpr (is_signed_v<_Up>)
1695 return __vector_bitcast<_Up>(
1696 _S_bit_shift_left(__vector_bitcast<make_unsigned_t<_Up>>(__x),
1697 __vector_bitcast<make_unsigned_t<_Up>>(__y)));
1698#endif
1699 else if constexpr (sizeof(_Up) == 1)
1700 {
1701 if constexpr (sizeof __ix == 64 && __have_avx512bw)
1702 return __vector_bitcast<_Up>(__concat(
1703 _mm512_cvtepi16_epi8(
1704 _mm512_sllv_epi16(_mm512_cvtepu8_epi16(__lo256(__ix)),
1705 _mm512_cvtepu8_epi16(__lo256(__iy)))),
1706 _mm512_cvtepi16_epi8(
1707 _mm512_sllv_epi16(_mm512_cvtepu8_epi16(__hi256(__ix)),
1708 _mm512_cvtepu8_epi16(__hi256(__iy))))));
1709 else if constexpr (sizeof __ix == 32 && __have_avx512bw)
1710 return __vector_bitcast<_Up>(_mm512_cvtepi16_epi8(
1711 _mm512_sllv_epi16(_mm512_cvtepu8_epi16(__ix),
1712 _mm512_cvtepu8_epi16(__iy))));
1713 else if constexpr (sizeof __x <= 8 && __have_avx512bw_vl)
1714 return __intrin_bitcast<_V>(
1715 _mm_cvtepi16_epi8(_mm_sllv_epi16(_mm_cvtepu8_epi16(__ix),
1716 _mm_cvtepu8_epi16(__iy))));
1717 else if constexpr (sizeof __ix == 16 && __have_avx512bw_vl)
1718 return __intrin_bitcast<_V>(_mm256_cvtepi16_epi8(
1719 _mm256_sllv_epi16(_mm256_cvtepu8_epi16(__ix),
1720 _mm256_cvtepu8_epi16(__iy))));
1721 else if constexpr (sizeof __ix == 16 && __have_avx512bw)
1722 return __intrin_bitcast<_V>(
1723 __lo128(_mm512_cvtepi16_epi8(_mm512_sllv_epi16(
1724 _mm512_cvtepu8_epi16(_mm256_castsi128_si256(__ix)),
1725 _mm512_cvtepu8_epi16(_mm256_castsi128_si256(__iy))))));
1726 else if constexpr (__have_sse4_1 && sizeof(__x) == 16)
1727 {
1728 auto __mask
1729 = __vector_bitcast<_Up>(__vector_bitcast<short>(__y) << 5);
1730 auto __x4
1731 = __vector_bitcast<_Up>(__vector_bitcast<short>(__x) << 4);
1732 __x4 &= char(0xf0);
1733 __x = reinterpret_cast<_V>(_CommonImplX86::_S_blend_intrin(
1734 __to_intrin(__mask), __to_intrin(__x), __to_intrin(__x4)));
1735 __mask += __mask;
1736 auto __x2
1737 = __vector_bitcast<_Up>(__vector_bitcast<short>(__x) << 2);
1738 __x2 &= char(0xfc);
1739 __x = reinterpret_cast<_V>(_CommonImplX86::_S_blend_intrin(
1740 __to_intrin(__mask), __to_intrin(__x), __to_intrin(__x2)));
1741 __mask += __mask;
1742 auto __x1 = __x + __x;
1743 __x = reinterpret_cast<_V>(_CommonImplX86::_S_blend_intrin(
1744 __to_intrin(__mask), __to_intrin(__x), __to_intrin(__x1)));
1745 return __x
1746 & ((__y & char(0xf8)) == 0); // y > 7 nulls the result
1747 }
1748 else if constexpr (sizeof(__x) == 16)
1749 {
1750 auto __mask
1751 = __vector_bitcast<_UChar>(__vector_bitcast<short>(__y) << 5);
1752 auto __x4
1753 = __vector_bitcast<_Up>(__vector_bitcast<short>(__x) << 4);
1754 __x4 &= char(0xf0);
1755 __x = __vector_bitcast<_SChar>(__mask) < 0 ? __x4 : __x;
1756 __mask += __mask;
1757 auto __x2
1758 = __vector_bitcast<_Up>(__vector_bitcast<short>(__x) << 2);
1759 __x2 &= char(0xfc);
1760 __x = __vector_bitcast<_SChar>(__mask) < 0 ? __x2 : __x;
1761 __mask += __mask;
1762 auto __x1 = __x + __x;
1763 __x = __vector_bitcast<_SChar>(__mask) < 0 ? __x1 : __x;
1764 return __x
1765 & ((__y & char(0xf8)) == 0); // y > 7 nulls the result
1766 }
1767 else
1768 return __x << __y;
1769 }
1770 else if constexpr (sizeof(_Up) == 2)
1771 {
1772 if constexpr (sizeof __ix == 64 && __have_avx512bw)
1773 return __vector_bitcast<_Up>(_mm512_sllv_epi16(__ix, __iy));
1774 else if constexpr (sizeof __ix == 32 && __have_avx512bw_vl)
1775 return __vector_bitcast<_Up>(_mm256_sllv_epi16(__ix, __iy));
1776 else if constexpr (sizeof __ix == 32 && __have_avx512bw)
1777 return __vector_bitcast<_Up>(
1778 __lo256(_mm512_sllv_epi16(_mm512_castsi256_si512(__ix),
1779 _mm512_castsi256_si512(__iy))));
1780 else if constexpr (sizeof __ix == 32 && __have_avx2)
1781 {
1782 const auto __ux = __vector_bitcast<unsigned>(__x);
1783 const auto __uy = __vector_bitcast<unsigned>(__y);
1784 return __vector_bitcast<_Up>(_mm256_blend_epi16(
1785 __auto_bitcast(__ux << (__uy & 0x0000ffffu)),
1786 __auto_bitcast((__ux & 0xffff0000u) << (__uy >> 16)), 0xaa));
1787 }
1788 else if constexpr (sizeof __ix == 16 && __have_avx512bw_vl)
1789 return __intrin_bitcast<_V>(_mm_sllv_epi16(__ix, __iy));
1790 else if constexpr (sizeof __ix == 16 && __have_avx512bw)
1791 return __intrin_bitcast<_V>(
1792 __lo128(_mm512_sllv_epi16(_mm512_castsi128_si512(__ix),
1793 _mm512_castsi128_si512(__iy))));
1794 else if constexpr (sizeof __ix == 16 && __have_avx2)
1795 {
1796 const auto __ux = __vector_bitcast<unsigned>(__ix);
1797 const auto __uy = __vector_bitcast<unsigned>(__iy);
1798 return __intrin_bitcast<_V>(_mm_blend_epi16(
1799 __auto_bitcast(__ux << (__uy & 0x0000ffffu)),
1800 __auto_bitcast((__ux & 0xffff0000u) << (__uy >> 16)), 0xaa));
1801 }
1802 else if constexpr (sizeof __ix == 16)
1803 {
1804 using _Float4 = __vector_type_t<float, 4>;
1805 using _Int4 = __vector_type_t<int, 4>;
1806 using _UInt4 = __vector_type_t<unsigned, 4>;
1807 const _UInt4 __yu
1808 = reinterpret_cast<_UInt4>(__to_intrin(__y + (0x3f8 >> 3)));
1809 return __x
1810 * __intrin_bitcast<_V>(
1811 __vector_convert<_Int4>(_SimdWrapper<float, 4>(
1812 reinterpret_cast<_Float4>(__yu << 23)))
1813 | (__vector_convert<_Int4>(_SimdWrapper<float, 4>(
1814 reinterpret_cast<_Float4>((__yu >> 16) << 23)))
1815 << 16));
1816 }
1817 else
1818 __assert_unreachable<_Tp>();
1819 }
1820 else if constexpr (sizeof(_Up) == 4 && sizeof __ix == 16
1821 && !__have_avx2)
1822 // latency is suboptimal, but throughput is at full speedup
1823 return __intrin_bitcast<_V>(
1824 __vector_bitcast<unsigned>(__ix)
1825 * __vector_convert<__vector_type16_t<int>>(
1826 _SimdWrapper<float, 4>(__vector_bitcast<float>(
1827 (__vector_bitcast<unsigned, 4>(__y) << 23) + 0x3f80'0000))));
1828 else if constexpr (sizeof(_Up) == 8 && sizeof __ix == 16
1829 && !__have_avx2)
1830 {
1831 const auto __lo = _mm_sll_epi64(__ix, __iy);
1832 const auto __hi
1833 = _mm_sll_epi64(__ix, _mm_unpackhi_epi64(__iy, __iy));
1834 if constexpr (__have_sse4_1)
1835 return __vector_bitcast<_Up>(_mm_blend_epi16(__lo, __hi, 0xf0));
1836 else
1837 return __vector_bitcast<_Up>(
1838 _mm_move_sd(__vector_bitcast<double>(__hi),
1839 __vector_bitcast<double>(__lo)));
1840 }
1841 else
1842 return __x << __y;
1843 }
1844#endif // _GLIBCXX_SIMD_NO_SHIFT_OPT
1845
1846 // }}}
1847 // _S_bit_shift_right {{{
1848#ifndef _GLIBCXX_SIMD_NO_SHIFT_OPT
1849 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1850 constexpr inline _GLIBCXX_CONST static typename _TVT::type
1851 _S_bit_shift_right(_Tp __xx, int __y)
1852 {
1853 using _V = typename _TVT::type;
1854 using _Up = typename _TVT::value_type;
1855 _V __x = __xx;
1856 [[maybe_unused]] const auto __ix = __to_intrin(__x);
1857 if (__builtin_is_constant_evaluated())
1858 return __x >> __y;
1859 else if (__builtin_constant_p(__y)
1860 && is_unsigned_v<
1861 _Up> && __y >= int(sizeof(_Up) * __CHAR_BIT__))
1862 return _V();
1863 else if constexpr (sizeof(_Up) == 1 && is_unsigned_v<_Up>) //{{{
1864 return __intrin_bitcast<_V>(__vector_bitcast<_UShort>(__ix) >> __y)
1865 & _Up(0xff >> __y);
1866 //}}}
1867 else if constexpr (sizeof(_Up) == 1 && is_signed_v<_Up>) //{{{
1868 return __intrin_bitcast<_V>(
1869 (__vector_bitcast<_UShort>(__vector_bitcast<short>(__ix)
1870 >> (__y + 8))
1871 << 8)
1872 | (__vector_bitcast<_UShort>(
1873 __vector_bitcast<short>(__vector_bitcast<_UShort>(__ix) << 8)
1874 >> __y)
1875 >> 8));
1876 //}}}
1877 // GCC optimizes sizeof == 2, 4, and unsigned 8 as expected
1878 else if constexpr (sizeof(_Up) == 8 && is_signed_v<_Up>) //{{{
1879 {
1880 if (__y > 32)
1881 return (__intrin_bitcast<_V>(__vector_bitcast<int>(__ix) >> 32)
1882 & _Up(0xffff'ffff'0000'0000ull))
1883 | __vector_bitcast<_Up>(
1884 __vector_bitcast<int>(__vector_bitcast<_ULLong>(__ix)
1885 >> 32)
1886 >> (__y - 32));
1887 else
1888 return __intrin_bitcast<_V>(__vector_bitcast<_ULLong>(__ix)
1889 >> __y)
1890 | __vector_bitcast<_Up>(
1891 __vector_bitcast<int>(__ix & -0x8000'0000'0000'0000ll)
1892 >> __y);
1893 }
1894 //}}}
1895 else
1896 return __x >> __y;
1897 }
1898
1899 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1900 constexpr inline _GLIBCXX_CONST static typename _TVT::type
1901 _S_bit_shift_right(_Tp __xx, typename _TVT::type __y)
1902 {
1903 using _V = typename _TVT::type;
1904 using _Up = typename _TVT::value_type;
1905 _V __x = __xx;
1906 [[maybe_unused]] const auto __ix = __to_intrin(__x);
1907 [[maybe_unused]] const auto __iy = __to_intrin(__y);
1908 if (__builtin_is_constant_evaluated()
1909 || (__builtin_constant_p(__x) && __builtin_constant_p(__y)))
1910 return __x >> __y;
1911 else if constexpr (sizeof(_Up) == 1) //{{{
1912 {
1913 if constexpr (sizeof(__x) <= 8 && __have_avx512bw_vl)
1914 return __intrin_bitcast<_V>(_mm_cvtepi16_epi8(
1915 is_signed_v<_Up> ? _mm_srav_epi16(_mm_cvtepi8_epi16(__ix),
1916 _mm_cvtepi8_epi16(__iy))
1917 : _mm_srlv_epi16(_mm_cvtepu8_epi16(__ix),
1918 _mm_cvtepu8_epi16(__iy))));
1919 if constexpr (sizeof(__x) == 16 && __have_avx512bw_vl)
1920 return __intrin_bitcast<_V>(_mm256_cvtepi16_epi8(
1921 is_signed_v<_Up>
1922 ? _mm256_srav_epi16(_mm256_cvtepi8_epi16(__ix),
1923 _mm256_cvtepi8_epi16(__iy))
1924 : _mm256_srlv_epi16(_mm256_cvtepu8_epi16(__ix),
1925 _mm256_cvtepu8_epi16(__iy))));
1926 else if constexpr (sizeof(__x) == 32 && __have_avx512bw)
1927 return __vector_bitcast<_Up>(_mm512_cvtepi16_epi8(
1928 is_signed_v<_Up>
1929 ? _mm512_srav_epi16(_mm512_cvtepi8_epi16(__ix),
1930 _mm512_cvtepi8_epi16(__iy))
1931 : _mm512_srlv_epi16(_mm512_cvtepu8_epi16(__ix),
1932 _mm512_cvtepu8_epi16(__iy))));
1933 else if constexpr (sizeof(__x) == 64 && is_signed_v<_Up>)
1934 return __vector_bitcast<_Up>(_mm512_mask_mov_epi8(
1935 _mm512_srav_epi16(__ix, _mm512_srli_epi16(__iy, 8)),
1936 0x5555'5555'5555'5555ull,
1937 _mm512_srav_epi16(
1938 _mm512_slli_epi16(__ix, 8),
1939 _mm512_maskz_add_epi8(0x5555'5555'5555'5555ull, __iy,
1940 _mm512_set1_epi16(8)))));
1941 else if constexpr (sizeof(__x) == 64 && is_unsigned_v<_Up>)
1942 return __vector_bitcast<_Up>(_mm512_mask_mov_epi8(
1943 _mm512_srlv_epi16(__ix, _mm512_srli_epi16(__iy, 8)),
1944 0x5555'5555'5555'5555ull,
1945 _mm512_srlv_epi16(
1946 _mm512_maskz_mov_epi8(0x5555'5555'5555'5555ull, __ix),
1947 _mm512_maskz_mov_epi8(0x5555'5555'5555'5555ull, __iy))));
1948 /* This has better throughput but higher latency than the impl below
1949 else if constexpr (__have_avx2 && sizeof(__x) == 16 &&
1950 is_unsigned_v<_Up>)
1951 {
1952 const auto __shorts = __to_intrin(_S_bit_shift_right(
1953 __vector_bitcast<_UShort>(_mm256_cvtepu8_epi16(__ix)),
1954 __vector_bitcast<_UShort>(_mm256_cvtepu8_epi16(__iy))));
1955 return __vector_bitcast<_Up>(
1956 _mm_packus_epi16(__lo128(__shorts), __hi128(__shorts)));
1957 }
1958 */
1959 else if constexpr (__have_avx2 && sizeof(__x) > 8)
1960 // the following uses vpsr[al]vd, which requires AVX2
1961 if constexpr (is_signed_v<_Up>)
1962 {
1963 const auto r3 = __vector_bitcast<_UInt>(
1964 (__vector_bitcast<int>(__x)
1965 >> (__vector_bitcast<_UInt>(__y) >> 24)))
1966 & 0xff000000u;
1967 const auto r2
1968 = __vector_bitcast<_UInt>(
1969 ((__vector_bitcast<int>(__x) << 8)
1970 >> ((__vector_bitcast<_UInt>(__y) << 8) >> 24)))
1971 & 0xff000000u;
1972 const auto r1
1973 = __vector_bitcast<_UInt>(
1974 ((__vector_bitcast<int>(__x) << 16)
1975 >> ((__vector_bitcast<_UInt>(__y) << 16) >> 24)))
1976 & 0xff000000u;
1977 const auto r0 = __vector_bitcast<_UInt>(
1978 (__vector_bitcast<int>(__x) << 24)
1979 >> ((__vector_bitcast<_UInt>(__y) << 24) >> 24));
1980 return __vector_bitcast<_Up>(r3 | (r2 >> 8) | (r1 >> 16)
1981 | (r0 >> 24));
1982 }
1983 else
1984 {
1985 const auto r3 = (__vector_bitcast<_UInt>(__x)
1986 >> (__vector_bitcast<_UInt>(__y) >> 24))
1987 & 0xff000000u;
1988 const auto r2
1989 = ((__vector_bitcast<_UInt>(__x) << 8)
1990 >> ((__vector_bitcast<_UInt>(__y) << 8) >> 24))
1991 & 0xff000000u;
1992 const auto r1
1993 = ((__vector_bitcast<_UInt>(__x) << 16)
1994 >> ((__vector_bitcast<_UInt>(__y) << 16) >> 24))
1995 & 0xff000000u;
1996 const auto r0
1997 = (__vector_bitcast<_UInt>(__x) << 24)
1998 >> ((__vector_bitcast<_UInt>(__y) << 24) >> 24);
1999 return __vector_bitcast<_Up>(r3 | (r2 >> 8) | (r1 >> 16)
2000 | (r0 >> 24));
2001 }
2002 else if constexpr (__have_sse4_1
2003 && is_unsigned_v<_Up> && sizeof(__x) > 2)
2004 {
2005 auto __x128 = __vector_bitcast<_Up>(__ix);
2006 auto __mask
2007 = __vector_bitcast<_Up>(__vector_bitcast<_UShort>(__iy) << 5);
2008 auto __x4 = __vector_bitcast<_Up>(
2009 (__vector_bitcast<_UShort>(__x128) >> 4) & _UShort(0xff0f));
2010 __x128 = __vector_bitcast<_Up>(_CommonImplX86::_S_blend_intrin(
2011 __to_intrin(__mask), __to_intrin(__x128), __to_intrin(__x4)));
2012 __mask += __mask;
2013 auto __x2 = __vector_bitcast<_Up>(
2014 (__vector_bitcast<_UShort>(__x128) >> 2) & _UShort(0xff3f));
2015 __x128 = __vector_bitcast<_Up>(_CommonImplX86::_S_blend_intrin(
2016 __to_intrin(__mask), __to_intrin(__x128), __to_intrin(__x2)));
2017 __mask += __mask;
2018 auto __x1 = __vector_bitcast<_Up>(
2019 (__vector_bitcast<_UShort>(__x128) >> 1) & _UShort(0xff7f));
2020 __x128 = __vector_bitcast<_Up>(_CommonImplX86::_S_blend_intrin(
2021 __to_intrin(__mask), __to_intrin(__x128), __to_intrin(__x1)));
2022 return __intrin_bitcast<_V>(
2023 __x128
2024 & ((__vector_bitcast<_Up>(__iy) & char(0xf8))
2025 == 0)); // y > 7 nulls the result
2026 }
2027 else if constexpr (__have_sse4_1
2028 && is_signed_v<_Up> && sizeof(__x) > 2)
2029 {
2030 auto __mask = __vector_bitcast<_UChar>(
2031 __vector_bitcast<_UShort>(__iy) << 5);
2032 auto __maskl = [&]() _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2033 return __to_intrin(__vector_bitcast<_UShort>(__mask) << 8);
2034 };
2035 auto __xh = __vector_bitcast<short>(__ix);
2036 auto __xl = __vector_bitcast<short>(__ix) << 8;
2037 auto __xh4 = __xh >> 4;
2038 auto __xl4 = __xl >> 4;
2039 __xh = __vector_bitcast<short>(_CommonImplX86::_S_blend_intrin(
2040 __to_intrin(__mask), __to_intrin(__xh), __to_intrin(__xh4)));
2041 __xl = __vector_bitcast<short>(
2042 _CommonImplX86::_S_blend_intrin(__maskl(), __to_intrin(__xl),
2043 __to_intrin(__xl4)));
2044 __mask += __mask;
2045 auto __xh2 = __xh >> 2;
2046 auto __xl2 = __xl >> 2;
2047 __xh = __vector_bitcast<short>(_CommonImplX86::_S_blend_intrin(
2048 __to_intrin(__mask), __to_intrin(__xh), __to_intrin(__xh2)));
2049 __xl = __vector_bitcast<short>(
2050 _CommonImplX86::_S_blend_intrin(__maskl(), __to_intrin(__xl),
2051 __to_intrin(__xl2)));
2052 __mask += __mask;
2053 auto __xh1 = __xh >> 1;
2054 auto __xl1 = __xl >> 1;
2055 __xh = __vector_bitcast<short>(_CommonImplX86::_S_blend_intrin(
2056 __to_intrin(__mask), __to_intrin(__xh), __to_intrin(__xh1)));
2057 __xl = __vector_bitcast<short>(
2058 _CommonImplX86::_S_blend_intrin(__maskl(), __to_intrin(__xl),
2059 __to_intrin(__xl1)));
2060 return __intrin_bitcast<_V>(
2061 (__vector_bitcast<_Up>((__xh & short(0xff00)))
2062 | __vector_bitcast<_Up>(__vector_bitcast<_UShort>(__xl)
2063 >> 8))
2064 & ((__vector_bitcast<_Up>(__iy) & char(0xf8))
2065 == 0)); // y > 7 nulls the result
2066 }
2067 else if constexpr (is_unsigned_v<_Up> && sizeof(__x) > 2) // SSE2
2068 {
2069 auto __mask
2070 = __vector_bitcast<_Up>(__vector_bitcast<_UShort>(__y) << 5);
2071 auto __x4 = __vector_bitcast<_Up>(
2072 (__vector_bitcast<_UShort>(__x) >> 4) & _UShort(0xff0f));
2073 __x = __mask > 0x7f ? __x4 : __x;
2074 __mask += __mask;
2075 auto __x2 = __vector_bitcast<_Up>(
2076 (__vector_bitcast<_UShort>(__x) >> 2) & _UShort(0xff3f));
2077 __x = __mask > 0x7f ? __x2 : __x;
2078 __mask += __mask;
2079 auto __x1 = __vector_bitcast<_Up>(
2080 (__vector_bitcast<_UShort>(__x) >> 1) & _UShort(0xff7f));
2081 __x = __mask > 0x7f ? __x1 : __x;
2082 return __x
2083 & ((__y & char(0xf8)) == 0); // y > 7 nulls the result
2084 }
2085 else if constexpr (sizeof(__x) > 2) // signed SSE2
2086 {
2087 static_assert(is_signed_v<_Up>);
2088 auto __maskh = __vector_bitcast<_UShort>(__y) << 5;
2089 auto __maskl = __vector_bitcast<_UShort>(__y) << (5 + 8);
2090 auto __xh = __vector_bitcast<short>(__x);
2091 auto __xl = __vector_bitcast<short>(__x) << 8;
2092 auto __xh4 = __xh >> 4;
2093 auto __xl4 = __xl >> 4;
2094 __xh = __maskh > 0x7fff ? __xh4 : __xh;
2095 __xl = __maskl > 0x7fff ? __xl4 : __xl;
2096 __maskh += __maskh;
2097 __maskl += __maskl;
2098 auto __xh2 = __xh >> 2;
2099 auto __xl2 = __xl >> 2;
2100 __xh = __maskh > 0x7fff ? __xh2 : __xh;
2101 __xl = __maskl > 0x7fff ? __xl2 : __xl;
2102 __maskh += __maskh;
2103 __maskl += __maskl;
2104 auto __xh1 = __xh >> 1;
2105 auto __xl1 = __xl >> 1;
2106 __xh = __maskh > 0x7fff ? __xh1 : __xh;
2107 __xl = __maskl > 0x7fff ? __xl1 : __xl;
2108 __x = __vector_bitcast<_Up>((__xh & short(0xff00)))
2109 | __vector_bitcast<_Up>(__vector_bitcast<_UShort>(__xl)
2110 >> 8);
2111 return __x
2112 & ((__y & char(0xf8)) == 0); // y > 7 nulls the result
2113 }
2114 else
2115 return __x >> __y;
2116 } //}}}
2117 else if constexpr (sizeof(_Up) == 2 && sizeof(__x) >= 4) //{{{
2118 {
2119 [[maybe_unused]] auto __blend_0xaa
2120 = [](auto __a, auto __b) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2121 if constexpr (sizeof(__a) == 16)
2122 return _mm_blend_epi16(__to_intrin(__a), __to_intrin(__b),
2123 0xaa);
2124 else if constexpr (sizeof(__a) == 32)
2125 return _mm256_blend_epi16(__to_intrin(__a), __to_intrin(__b),
2126 0xaa);
2127 else if constexpr (sizeof(__a) == 64)
2128 return _mm512_mask_blend_epi16(0xaaaa'aaaaU, __to_intrin(__a),
2129 __to_intrin(__b));
2130 else
2131 __assert_unreachable<decltype(__a)>();
2132 };
2133 if constexpr (__have_avx512bw_vl && sizeof(_Tp) <= 16)
2134 return __intrin_bitcast<_V>(is_signed_v<_Up>
2135 ? _mm_srav_epi16(__ix, __iy)
2136 : _mm_srlv_epi16(__ix, __iy));
2137 else if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 32)
2138 return __vector_bitcast<_Up>(is_signed_v<_Up>
2139 ? _mm256_srav_epi16(__ix, __iy)
2140 : _mm256_srlv_epi16(__ix, __iy));
2141 else if constexpr (__have_avx512bw && sizeof(_Tp) == 64)
2142 return __vector_bitcast<_Up>(is_signed_v<_Up>
2143 ? _mm512_srav_epi16(__ix, __iy)
2144 : _mm512_srlv_epi16(__ix, __iy));
2145 else if constexpr (__have_avx2 && is_signed_v<_Up>)
2146 return __intrin_bitcast<_V>(
2147 __blend_0xaa(((__vector_bitcast<int>(__ix) << 16)
2148 >> (__vector_bitcast<int>(__iy) & 0xffffu))
2149 >> 16,
2150 __vector_bitcast<int>(__ix)
2151 >> (__vector_bitcast<int>(__iy) >> 16)));
2152 else if constexpr (__have_avx2 && is_unsigned_v<_Up>)
2153 return __intrin_bitcast<_V>(
2154 __blend_0xaa((__vector_bitcast<_UInt>(__ix) & 0xffffu)
2155 >> (__vector_bitcast<_UInt>(__iy) & 0xffffu),
2156 __vector_bitcast<_UInt>(__ix)
2157 >> (__vector_bitcast<_UInt>(__iy) >> 16)));
2158 else if constexpr (__have_sse4_1)
2159 {
2160 auto __mask = __vector_bitcast<_UShort>(__iy);
2161 auto __x128 = __vector_bitcast<_Up>(__ix);
2162 //__mask *= 0x0808;
2163 __mask = (__mask << 3) | (__mask << 11);
2164 // do __x128 = 0 where __y[4] is set
2165 __x128 = __vector_bitcast<_Up>(
2166 _mm_blendv_epi8(__to_intrin(__x128), __m128i(),
2167 __to_intrin(__mask)));
2168 // do __x128 =>> 8 where __y[3] is set
2169 __x128 = __vector_bitcast<_Up>(
2170 _mm_blendv_epi8(__to_intrin(__x128), __to_intrin(__x128 >> 8),
2171 __to_intrin(__mask += __mask)));
2172 // do __x128 =>> 4 where __y[2] is set
2173 __x128 = __vector_bitcast<_Up>(
2174 _mm_blendv_epi8(__to_intrin(__x128), __to_intrin(__x128 >> 4),
2175 __to_intrin(__mask += __mask)));
2176 // do __x128 =>> 2 where __y[1] is set
2177 __x128 = __vector_bitcast<_Up>(
2178 _mm_blendv_epi8(__to_intrin(__x128), __to_intrin(__x128 >> 2),
2179 __to_intrin(__mask += __mask)));
2180 // do __x128 =>> 1 where __y[0] is set
2181 return __intrin_bitcast<_V>(
2182 _mm_blendv_epi8(__to_intrin(__x128), __to_intrin(__x128 >> 1),
2183 __to_intrin(__mask + __mask)));
2184 }
2185 else
2186 {
2187 auto __k = __vector_bitcast<_UShort>(__iy) << 11;
2188 auto __x128 = __vector_bitcast<_Up>(__ix);
2189 auto __mask
2190 = [](__vector_type16_t<_UShort> __kk) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2191 return __vector_bitcast<short>(__kk) < 0;
2192 };
2193 // do __x128 = 0 where __y[4] is set
2194 __x128 = __mask(__k) ? decltype(__x128)() : __x128;
2195 // do __x128 =>> 8 where __y[3] is set
2196 __x128 = __mask(__k += __k) ? __x128 >> 8 : __x128;
2197 // do __x128 =>> 4 where __y[2] is set
2198 __x128 = __mask(__k += __k) ? __x128 >> 4 : __x128;
2199 // do __x128 =>> 2 where __y[1] is set
2200 __x128 = __mask(__k += __k) ? __x128 >> 2 : __x128;
2201 // do __x128 =>> 1 where __y[0] is set
2202 return __intrin_bitcast<_V>(__mask(__k + __k) ? __x128 >> 1
2203 : __x128);
2204 }
2205 } //}}}
2206 else if constexpr (sizeof(_Up) == 4 && !__have_avx2) //{{{
2207 {
2208 if constexpr (is_unsigned_v<_Up>)
2209 {
2210 // x >> y == x * 2^-y == (x * 2^(31-y)) >> 31
2211 const __m128 __factor_f = reinterpret_cast<__m128>(
2212 0x4f00'0000u - (__vector_bitcast<unsigned, 4>(__y) << 23));
2213 const __m128i __factor
2214 = __builtin_constant_p(__factor_f)
2215 ? __to_intrin(
2216 __make_vector<unsigned>(__factor_f[0], __factor_f[1],
2217 __factor_f[2], __factor_f[3]))
2218 : _mm_cvttps_epi32(__factor_f);
2219 const auto __r02
2220 = _mm_srli_epi64(_mm_mul_epu32(__ix, __factor), 31);
2221 const auto __r13 = _mm_mul_epu32(_mm_srli_si128(__ix, 4),
2222 _mm_srli_si128(__factor, 4));
2223 if constexpr (__have_sse4_1)
2224 return __intrin_bitcast<_V>(
2225 _mm_blend_epi16(_mm_slli_epi64(__r13, 1), __r02, 0x33));
2226 else
2227 return __intrin_bitcast<_V>(
2228 __r02 | _mm_slli_si128(_mm_srli_epi64(__r13, 31), 4));
2229 }
2230 else
2231 {
2232 auto __shift = [](auto __a, auto __b) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2233 if constexpr (is_signed_v<_Up>)
2234 return _mm_sra_epi32(__a, __b);
2235 else
2236 return _mm_srl_epi32(__a, __b);
2237 };
2238 const auto __r0
2239 = __shift(__ix, _mm_unpacklo_epi32(__iy, __m128i()));
2240 const auto __r1 = __shift(__ix, _mm_srli_epi64(__iy, 32));
2241 const auto __r2
2242 = __shift(__ix, _mm_unpackhi_epi32(__iy, __m128i()));
2243 const auto __r3 = __shift(__ix, _mm_srli_si128(__iy, 12));
2244 if constexpr (__have_sse4_1)
2245 return __intrin_bitcast<_V>(
2246 _mm_blend_epi16(_mm_blend_epi16(__r1, __r0, 0x3),
2247 _mm_blend_epi16(__r3, __r2, 0x30), 0xf0));
2248 else
2249 return __intrin_bitcast<_V>(_mm_unpacklo_epi64(
2250 _mm_unpacklo_epi32(__r0, _mm_srli_si128(__r1, 4)),
2251 _mm_unpackhi_epi32(__r2, _mm_srli_si128(__r3, 4))));
2252 }
2253 } //}}}
2254 else
2255 return __x >> __y;
2256 }
2257#endif // _GLIBCXX_SIMD_NO_SHIFT_OPT
2258
2259 // }}}
2260 // compares {{{
2261 // _S_equal_to {{{
2262 template <typename _Tp, size_t _Np>
2263 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
2264 _S_equal_to(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
2265 {
2266 if constexpr (__is_avx512_abi<_Abi>()) // {{{
2267 {
2268 if (__builtin_is_constant_evaluated()
2269 || (__x._M_is_constprop() && __y._M_is_constprop()))
2270 return _MaskImpl::_S_to_bits(
2271 __as_wrapper<_Np>(__x._M_data == __y._M_data));
2272
2273 constexpr auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
2274 [[maybe_unused]] const auto __xi = __to_intrin(__x);
2275 [[maybe_unused]] const auto __yi = __to_intrin(__y);
2276 if constexpr (is_floating_point_v<_Tp>)
2277 {
2278 if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
2279 return _mm512_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_EQ_OQ);
2280 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
2281 return _mm512_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_EQ_OQ);
2282 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
2283 return _mm256_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_EQ_OQ);
2284 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
2285 return _mm256_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_EQ_OQ);
2286 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
2287 return _mm_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_EQ_OQ);
2288 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
2289 return _mm_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_EQ_OQ);
2290 else
2291 __assert_unreachable<_Tp>();
2292 }
2293 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
2294 return _mm512_mask_cmpeq_epi64_mask(__k1, __xi, __yi);
2295 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
2296 return _mm512_mask_cmpeq_epi32_mask(__k1, __xi, __yi);
2297 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 2)
2298 return _mm512_mask_cmpeq_epi16_mask(__k1, __xi, __yi);
2299 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 1)
2300 return _mm512_mask_cmpeq_epi8_mask(__k1, __xi, __yi);
2301 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
2302 return _mm256_mask_cmpeq_epi64_mask(__k1, __xi, __yi);
2303 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
2304 return _mm256_mask_cmpeq_epi32_mask(__k1, __xi, __yi);
2305 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 2)
2306 return _mm256_mask_cmpeq_epi16_mask(__k1, __xi, __yi);
2307 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 1)
2308 return _mm256_mask_cmpeq_epi8_mask(__k1, __xi, __yi);
2309 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
2310 return _mm_mask_cmpeq_epi64_mask(__k1, __xi, __yi);
2311 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
2312 return _mm_mask_cmpeq_epi32_mask(__k1, __xi, __yi);
2313 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 2)
2314 return _mm_mask_cmpeq_epi16_mask(__k1, __xi, __yi);
2315 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 1)
2316 return _mm_mask_cmpeq_epi8_mask(__k1, __xi, __yi);
2317 else
2318 __assert_unreachable<_Tp>();
2319 } // }}}
2320 else if (__builtin_is_constant_evaluated())
2321 return _Base::_S_equal_to(__x, __y);
2322 else if constexpr (sizeof(__x) == 8) // {{{
2323 {
2324 const auto __r128 = __vector_bitcast<_Tp, 16 / sizeof(_Tp)>(__x)
2325 == __vector_bitcast<_Tp, 16 / sizeof(_Tp)>(__y);
2326 _MaskMember<_Tp> __r64;
2327 __builtin_memcpy(&__r64._M_data, &__r128, sizeof(__r64));
2328 return __r64;
2329 } // }}}
2330 else
2331 return _Base::_S_equal_to(__x, __y);
2332 }
2333
2334 // }}}
2335 // _S_not_equal_to {{{
2336 template <typename _Tp, size_t _Np>
2337 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
2338 _S_not_equal_to(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
2339 {
2340 if constexpr (__is_avx512_abi<_Abi>()) // {{{
2341 {
2342 if (__builtin_is_constant_evaluated()
2343 || (__x._M_is_constprop() && __y._M_is_constprop()))
2344 return _MaskImpl::_S_to_bits(
2345 __as_wrapper<_Np>(__x._M_data != __y._M_data));
2346
2347 constexpr auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
2348 [[maybe_unused]] const auto __xi = __to_intrin(__x);
2349 [[maybe_unused]] const auto __yi = __to_intrin(__y);
2350 if constexpr (is_floating_point_v<_Tp>)
2351 {
2352 if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
2353 return _mm512_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_NEQ_UQ);
2354 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
2355 return _mm512_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_NEQ_UQ);
2356 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
2357 return _mm256_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_NEQ_UQ);
2358 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
2359 return _mm256_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_NEQ_UQ);
2360 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
2361 return _mm_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_NEQ_UQ);
2362 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
2363 return _mm_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_NEQ_UQ);
2364 else
2365 __assert_unreachable<_Tp>();
2366 }
2367 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
2368 return ~_mm512_mask_cmpeq_epi64_mask(__k1, __xi, __yi);
2369 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
2370 return ~_mm512_mask_cmpeq_epi32_mask(__k1, __xi, __yi);
2371 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 2)
2372 return ~_mm512_mask_cmpeq_epi16_mask(__k1, __xi, __yi);
2373 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 1)
2374 return ~_mm512_mask_cmpeq_epi8_mask(__k1, __xi, __yi);
2375 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
2376 return ~_mm256_mask_cmpeq_epi64_mask(__k1, __xi, __yi);
2377 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
2378 return ~_mm256_mask_cmpeq_epi32_mask(__k1, __xi, __yi);
2379 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 2)
2380 return ~_mm256_mask_cmpeq_epi16_mask(__k1, __xi, __yi);
2381 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 1)
2382 return ~_mm256_mask_cmpeq_epi8_mask(__k1, __xi, __yi);
2383 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
2384 return ~_mm_mask_cmpeq_epi64_mask(__k1, __xi, __yi);
2385 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
2386 return ~_mm_mask_cmpeq_epi32_mask(__k1, __xi, __yi);
2387 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 2)
2388 return ~_mm_mask_cmpeq_epi16_mask(__k1, __xi, __yi);
2389 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 1)
2390 return ~_mm_mask_cmpeq_epi8_mask(__k1, __xi, __yi);
2391 else
2392 __assert_unreachable<_Tp>();
2393 } // }}}
2394 else if (__builtin_is_constant_evaluated())
2395 return _Base::_S_not_equal_to(__x, __y);
2396 else if constexpr (sizeof(__x) == 8)
2397 {
2398 const auto __r128 = __vector_bitcast<_Tp, 16 / sizeof(_Tp)>(__x)
2399 != __vector_bitcast<_Tp, 16 / sizeof(_Tp)>(__y);
2400 _MaskMember<_Tp> __r64;
2401 __builtin_memcpy(&__r64._M_data, &__r128, sizeof(__r64));
2402 return __r64;
2403 }
2404 else
2405 return _Base::_S_not_equal_to(__x, __y);
2406 }
2407
2408 // }}}
2409 // _S_less {{{
2410 template <typename _Tp, size_t _Np>
2411 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
2412 _S_less(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
2413 {
2414 if constexpr (__is_avx512_abi<_Abi>()) // {{{
2415 {
2416 if (__builtin_is_constant_evaluated()
2417 || (__x._M_is_constprop() && __y._M_is_constprop()))
2418 return _MaskImpl::_S_to_bits(
2419 __as_wrapper<_Np>(__x._M_data < __y._M_data));
2420
2421 constexpr auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
2422 [[maybe_unused]] const auto __xi = __to_intrin(__x);
2423 [[maybe_unused]] const auto __yi = __to_intrin(__y);
2424 if constexpr (sizeof(__xi) == 64)
2425 {
2426 if constexpr (is_same_v<_Tp, float>)
2427 return _mm512_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LT_OS);
2428 else if constexpr (is_same_v<_Tp, double>)
2429 return _mm512_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LT_OS);
2430 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 1)
2431 return _mm512_mask_cmplt_epi8_mask(__k1, __xi, __yi);
2432 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 2)
2433 return _mm512_mask_cmplt_epi16_mask(__k1, __xi, __yi);
2434 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 4)
2435 return _mm512_mask_cmplt_epi32_mask(__k1, __xi, __yi);
2436 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 8)
2437 return _mm512_mask_cmplt_epi64_mask(__k1, __xi, __yi);
2438 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 1)
2439 return _mm512_mask_cmplt_epu8_mask(__k1, __xi, __yi);
2440 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 2)
2441 return _mm512_mask_cmplt_epu16_mask(__k1, __xi, __yi);
2442 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 4)
2443 return _mm512_mask_cmplt_epu32_mask(__k1, __xi, __yi);
2444 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 8)
2445 return _mm512_mask_cmplt_epu64_mask(__k1, __xi, __yi);
2446 else
2447 __assert_unreachable<_Tp>();
2448 }
2449 else if constexpr (sizeof(__xi) == 32)
2450 {
2451 if constexpr (is_same_v<_Tp, float>)
2452 return _mm256_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LT_OS);
2453 else if constexpr (is_same_v<_Tp, double>)
2454 return _mm256_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LT_OS);
2455 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 1)
2456 return _mm256_mask_cmplt_epi8_mask(__k1, __xi, __yi);
2457 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 2)
2458 return _mm256_mask_cmplt_epi16_mask(__k1, __xi, __yi);
2459 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 4)
2460 return _mm256_mask_cmplt_epi32_mask(__k1, __xi, __yi);
2461 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 8)
2462 return _mm256_mask_cmplt_epi64_mask(__k1, __xi, __yi);
2463 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 1)
2464 return _mm256_mask_cmplt_epu8_mask(__k1, __xi, __yi);
2465 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 2)
2466 return _mm256_mask_cmplt_epu16_mask(__k1, __xi, __yi);
2467 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 4)
2468 return _mm256_mask_cmplt_epu32_mask(__k1, __xi, __yi);
2469 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 8)
2470 return _mm256_mask_cmplt_epu64_mask(__k1, __xi, __yi);
2471 else
2472 __assert_unreachable<_Tp>();
2473 }
2474 else if constexpr (sizeof(__xi) == 16)
2475 {
2476 if constexpr (is_same_v<_Tp, float>)
2477 return _mm_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LT_OS);
2478 else if constexpr (is_same_v<_Tp, double>)
2479 return _mm_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LT_OS);
2480 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 1)
2481 return _mm_mask_cmplt_epi8_mask(__k1, __xi, __yi);
2482 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 2)
2483 return _mm_mask_cmplt_epi16_mask(__k1, __xi, __yi);
2484 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 4)
2485 return _mm_mask_cmplt_epi32_mask(__k1, __xi, __yi);
2486 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 8)
2487 return _mm_mask_cmplt_epi64_mask(__k1, __xi, __yi);
2488 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 1)
2489 return _mm_mask_cmplt_epu8_mask(__k1, __xi, __yi);
2490 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 2)
2491 return _mm_mask_cmplt_epu16_mask(__k1, __xi, __yi);
2492 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 4)
2493 return _mm_mask_cmplt_epu32_mask(__k1, __xi, __yi);
2494 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 8)
2495 return _mm_mask_cmplt_epu64_mask(__k1, __xi, __yi);
2496 else
2497 __assert_unreachable<_Tp>();
2498 }
2499 else
2500 __assert_unreachable<_Tp>();
2501 } // }}}
2502 else if (__builtin_is_constant_evaluated())
2503 return _Base::_S_less(__x, __y);
2504 else if constexpr (sizeof(__x) == 8)
2505 {
2506 const auto __r128 = __vector_bitcast<_Tp, 16 / sizeof(_Tp)>(__x)
2507 < __vector_bitcast<_Tp, 16 / sizeof(_Tp)>(__y);
2508 _MaskMember<_Tp> __r64;
2509 __builtin_memcpy(&__r64._M_data, &__r128, sizeof(__r64));
2510 return __r64;
2511 }
2512 else
2513 return _Base::_S_less(__x, __y);
2514 }
2515
2516 // }}}
2517 // _S_less_equal {{{
2518 template <typename _Tp, size_t _Np>
2519 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
2520 _S_less_equal(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
2521 {
2522 if constexpr (__is_avx512_abi<_Abi>()) // {{{
2523 {
2524 if (__builtin_is_constant_evaluated()
2525 || (__x._M_is_constprop() && __y._M_is_constprop()))
2526 return _MaskImpl::_S_to_bits(
2527 __as_wrapper<_Np>(__x._M_data <= __y._M_data));
2528
2529 constexpr auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
2530 [[maybe_unused]] const auto __xi = __to_intrin(__x);
2531 [[maybe_unused]] const auto __yi = __to_intrin(__y);
2532 if constexpr (sizeof(__xi) == 64)
2533 {
2534 if constexpr (is_same_v<_Tp, float>)
2535 return _mm512_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LE_OS);
2536 else if constexpr (is_same_v<_Tp, double>)
2537 return _mm512_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LE_OS);
2538 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 1)
2539 return _mm512_mask_cmple_epi8_mask(__k1, __xi, __yi);
2540 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 2)
2541 return _mm512_mask_cmple_epi16_mask(__k1, __xi, __yi);
2542 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 4)
2543 return _mm512_mask_cmple_epi32_mask(__k1, __xi, __yi);
2544 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 8)
2545 return _mm512_mask_cmple_epi64_mask(__k1, __xi, __yi);
2546 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 1)
2547 return _mm512_mask_cmple_epu8_mask(__k1, __xi, __yi);
2548 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 2)
2549 return _mm512_mask_cmple_epu16_mask(__k1, __xi, __yi);
2550 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 4)
2551 return _mm512_mask_cmple_epu32_mask(__k1, __xi, __yi);
2552 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 8)
2553 return _mm512_mask_cmple_epu64_mask(__k1, __xi, __yi);
2554 else
2555 __assert_unreachable<_Tp>();
2556 }
2557 else if constexpr (sizeof(__xi) == 32)
2558 {
2559 if constexpr (is_same_v<_Tp, float>)
2560 return _mm256_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LE_OS);
2561 else if constexpr (is_same_v<_Tp, double>)
2562 return _mm256_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LE_OS);
2563 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 1)
2564 return _mm256_mask_cmple_epi8_mask(__k1, __xi, __yi);
2565 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 2)
2566 return _mm256_mask_cmple_epi16_mask(__k1, __xi, __yi);
2567 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 4)
2568 return _mm256_mask_cmple_epi32_mask(__k1, __xi, __yi);
2569 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 8)
2570 return _mm256_mask_cmple_epi64_mask(__k1, __xi, __yi);
2571 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 1)
2572 return _mm256_mask_cmple_epu8_mask(__k1, __xi, __yi);
2573 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 2)
2574 return _mm256_mask_cmple_epu16_mask(__k1, __xi, __yi);
2575 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 4)
2576 return _mm256_mask_cmple_epu32_mask(__k1, __xi, __yi);
2577 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 8)
2578 return _mm256_mask_cmple_epu64_mask(__k1, __xi, __yi);
2579 else
2580 __assert_unreachable<_Tp>();
2581 }
2582 else if constexpr (sizeof(__xi) == 16)
2583 {
2584 if constexpr (is_same_v<_Tp, float>)
2585 return _mm_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LE_OS);
2586 else if constexpr (is_same_v<_Tp, double>)
2587 return _mm_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LE_OS);
2588 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 1)
2589 return _mm_mask_cmple_epi8_mask(__k1, __xi, __yi);
2590 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 2)
2591 return _mm_mask_cmple_epi16_mask(__k1, __xi, __yi);
2592 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 4)
2593 return _mm_mask_cmple_epi32_mask(__k1, __xi, __yi);
2594 else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 8)
2595 return _mm_mask_cmple_epi64_mask(__k1, __xi, __yi);
2596 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 1)
2597 return _mm_mask_cmple_epu8_mask(__k1, __xi, __yi);
2598 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 2)
2599 return _mm_mask_cmple_epu16_mask(__k1, __xi, __yi);
2600 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 4)
2601 return _mm_mask_cmple_epu32_mask(__k1, __xi, __yi);
2602 else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 8)
2603 return _mm_mask_cmple_epu64_mask(__k1, __xi, __yi);
2604 else
2605 __assert_unreachable<_Tp>();
2606 }
2607 else
2608 __assert_unreachable<_Tp>();
2609 } // }}}
2610 else if (__builtin_is_constant_evaluated())
2611 return _Base::_S_less_equal(__x, __y);
2612 else if constexpr (sizeof(__x) == 8)
2613 {
2614 const auto __r128 = __vector_bitcast<_Tp, 16 / sizeof(_Tp)>(__x)
2615 <= __vector_bitcast<_Tp, 16 / sizeof(_Tp)>(__y);
2616 _MaskMember<_Tp> __r64;
2617 __builtin_memcpy(&__r64._M_data, &__r128, sizeof(__r64));
2618 return __r64;
2619 }
2620 else
2621 return _Base::_S_less_equal(__x, __y);
2622 }
2623
2624 // }}} }}}
2625 // negation {{{
2626 template <typename _Tp, size_t _Np>
2627 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
2628 _S_negate(_SimdWrapper<_Tp, _Np> __x) noexcept
2629 {
2630 if constexpr (__is_avx512_abi<_Abi>())
2631 return _S_equal_to(__x, _SimdWrapper<_Tp, _Np>());
2632 else
2633 return _Base::_S_negate(__x);
2634 }
2635
2636 // }}}
2637 // math {{{
2638 using _Base::_S_abs;
2639
2640 // _S_sqrt {{{
2641 template <typename _Tp, size_t _Np>
2642 _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
2643 _S_sqrt(_SimdWrapper<_Tp, _Np> __x)
2644 {
2645 if constexpr (__is_sse_ps<_Tp, _Np>())
2646 return __auto_bitcast(_mm_sqrt_ps(__to_intrin(__x)));
2647 else if constexpr (__is_sse_pd<_Tp, _Np>())
2648 return _mm_sqrt_pd(__x);
2649 else if constexpr (__is_avx_ps<_Tp, _Np>())
2650 return _mm256_sqrt_ps(__x);
2651 else if constexpr (__is_avx_pd<_Tp, _Np>())
2652 return _mm256_sqrt_pd(__x);
2653 else if constexpr (__is_avx512_ps<_Tp, _Np>())
2654 return _mm512_sqrt_ps(__x);
2655 else if constexpr (__is_avx512_pd<_Tp, _Np>())
2656 return _mm512_sqrt_pd(__x);
2657 else
2658 __assert_unreachable<_Tp>();
2659 }
2660
2661 // }}}
2662 // _S_ldexp {{{
2663 template <typename _Tp, size_t _Np>
2664 _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
2665 _S_ldexp(_SimdWrapper<_Tp, _Np> __x,
2666 __fixed_size_storage_t<int, _Np> __exp)
2667 {
2668 if constexpr (sizeof(__x) == 64 || __have_avx512vl)
2669 {
2670 const auto __xi = __to_intrin(__x);
2671 constexpr _SimdConverter<int, simd_abi::fixed_size<_Np>, _Tp, _Abi>
2672 __cvt;
2673 const auto __expi = __to_intrin(__cvt(__exp));
2674 using _Up = __bool_storage_member_type_t<_Np>;
2675 constexpr _Up __k1 = _Np < sizeof(_Up) * __CHAR_BIT__ ? _Up((1ULL << _Np) - 1) : ~_Up();
2676 if constexpr (sizeof(__xi) == 16)
2677 {
2678 if constexpr (sizeof(_Tp) == 8)
2679 return _mm_maskz_scalef_pd(__k1, __xi, __expi);
2680 else
2681 return _mm_maskz_scalef_ps(__k1, __xi, __expi);
2682 }
2683 else if constexpr (sizeof(__xi) == 32)
2684 {
2685 if constexpr (sizeof(_Tp) == 8)
2686 return _mm256_maskz_scalef_pd(__k1, __xi, __expi);
2687 else
2688 return _mm256_maskz_scalef_ps(__k1, __xi, __expi);
2689 }
2690 else
2691 {
2692 static_assert(sizeof(__xi) == 64);
2693 if constexpr (sizeof(_Tp) == 8)
2694 return _mm512_maskz_scalef_pd(__k1, __xi, __expi);
2695 else
2696 return _mm512_maskz_scalef_ps(__k1, __xi, __expi);
2697 }
2698 }
2699 else
2700 return _Base::_S_ldexp(__x, __exp);
2701 }
2702
2703 // }}}
2704 // _S_trunc {{{
2705 template <typename _Tp, size_t _Np>
2706 _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
2707 _S_trunc(_SimdWrapper<_Tp, _Np> __x)
2708 {
2709 if constexpr (__is_avx512_ps<_Tp, _Np>())
2710 return _mm512_roundscale_ps(__x, 0x0b);
2711 else if constexpr (__is_avx512_pd<_Tp, _Np>())
2712 return _mm512_roundscale_pd(__x, 0x0b);
2713 else if constexpr (__is_avx_ps<_Tp, _Np>())
2714 return _mm256_round_ps(__x, 0xb);
2715 else if constexpr (__is_avx_pd<_Tp, _Np>())
2716 return _mm256_round_pd(__x, 0xb);
2717 else if constexpr (__have_sse4_1 && __is_sse_ps<_Tp, _Np>())
2718 return __auto_bitcast(_mm_round_ps(__to_intrin(__x), 0xb));
2719 else if constexpr (__have_sse4_1 && __is_sse_pd<_Tp, _Np>())
2720 return _mm_round_pd(__x, 0xb);
2721 else if constexpr (__is_sse_ps<_Tp, _Np>())
2722 {
2723 auto __truncated
2724 = _mm_cvtepi32_ps(_mm_cvttps_epi32(__to_intrin(__x)));
2725 const auto __no_fractional_values
2726 = __vector_bitcast<int>(__vector_bitcast<_UInt>(__to_intrin(__x))
2727 & 0x7f800000u)
2728 < 0x4b000000; // the exponent is so large that no mantissa bits
2729 // signify fractional values (0x3f8 + 23*8 =
2730 // 0x4b0)
2731 return __no_fractional_values ? __truncated : __to_intrin(__x);
2732 }
2733 else
2734 return _Base::_S_trunc(__x);
2735 }
2736
2737 // }}}
2738 // _S_round {{{
2739 template <typename _Tp, size_t _Np>
2740 _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
2741 _S_round(_SimdWrapper<_Tp, _Np> __x)
2742 {
2743 // Note that _MM_FROUND_TO_NEAREST_INT rounds ties to even, not away
2744 // from zero as required by std::round. Therefore this function is more
2745 // complicated.
2746 using _V = __vector_type_t<_Tp, _Np>;
2747 _V __truncated;
2748 if constexpr (__is_avx512_ps<_Tp, _Np>())
2749 __truncated = _mm512_roundscale_ps(__x._M_data, 0x0b);
2750 else if constexpr (__is_avx512_pd<_Tp, _Np>())
2751 __truncated = _mm512_roundscale_pd(__x._M_data, 0x0b);
2752 else if constexpr (__is_avx_ps<_Tp, _Np>())
2753 __truncated = _mm256_round_ps(__x._M_data,
2754 _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
2755 else if constexpr (__is_avx_pd<_Tp, _Np>())
2756 __truncated = _mm256_round_pd(__x._M_data,
2757 _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
2758 else if constexpr (__have_sse4_1 && __is_sse_ps<_Tp, _Np>())
2759 __truncated = __auto_bitcast(
2760 _mm_round_ps(__to_intrin(__x),
2761 _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC));
2762 else if constexpr (__have_sse4_1 && __is_sse_pd<_Tp, _Np>())
2763 __truncated
2764 = _mm_round_pd(__x._M_data, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
2765 else if constexpr (__is_sse_ps<_Tp, _Np>())
2766 __truncated = __auto_bitcast(
2767 _mm_cvtepi32_ps(_mm_cvttps_epi32(__to_intrin(__x))));
2768 else
2769 return _Base::_S_round(__x);
2770
2771 // x < 0 => truncated <= 0 && truncated >= x => x - truncated <= 0
2772 // x > 0 => truncated >= 0 && truncated <= x => x - truncated >= 0
2773
2774 const _V __rounded
2775 = __truncated
2776 + (__and(_S_absmask<_V>, __x._M_data - __truncated) >= _Tp(.5)
2777 ? __or(__and(_S_signmask<_V>, __x._M_data), _V() + 1)
2778 : _V());
2779 if constexpr (__have_sse4_1)
2780 return __rounded;
2781 else // adjust for missing range in cvttps_epi32
2782 return __and(_S_absmask<_V>, __x._M_data) < 0x1p23f ? __rounded
2783 : __x._M_data;
2784 }
2785
2786 // }}}
2787 // _S_nearbyint {{{
2788 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2789 _GLIBCXX_SIMD_INTRINSIC static _Tp
2790 _S_nearbyint(_Tp __x) noexcept
2791 {
2792 if constexpr (_TVT::template _S_is<float, 16>)
2793 return _mm512_roundscale_ps(__x, 0x0c);
2794 else if constexpr (_TVT::template _S_is<double, 8>)
2795 return _mm512_roundscale_pd(__x, 0x0c);
2796 else if constexpr (_TVT::template _S_is<float, 8>)
2797 return _mm256_round_ps(__x,
2798 _MM_FROUND_CUR_DIRECTION | _MM_FROUND_NO_EXC);
2799 else if constexpr (_TVT::template _S_is<double, 4>)
2800 return _mm256_round_pd(__x,
2801 _MM_FROUND_CUR_DIRECTION | _MM_FROUND_NO_EXC);
2802 else if constexpr (__have_sse4_1 && _TVT::template _S_is<float, 4>)
2803 return _mm_round_ps(__x,
2804 _MM_FROUND_CUR_DIRECTION | _MM_FROUND_NO_EXC);
2805 else if constexpr (__have_sse4_1 && _TVT::template _S_is<double, 2>)
2806 return _mm_round_pd(__x,
2807 _MM_FROUND_CUR_DIRECTION | _MM_FROUND_NO_EXC);
2808 else
2809 return _Base::_S_nearbyint(__x);
2810 }
2811
2812 // }}}
2813 // _S_rint {{{
2814 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2815 _GLIBCXX_SIMD_INTRINSIC static _Tp
2816 _S_rint(_Tp __x) noexcept
2817 {
2818 if constexpr (_TVT::template _S_is<float, 16>)
2819 return _mm512_roundscale_ps(__x, 0x04);
2820 else if constexpr (_TVT::template _S_is<double, 8>)
2821 return _mm512_roundscale_pd(__x, 0x04);
2822 else if constexpr (_TVT::template _S_is<float, 8>)
2823 return _mm256_round_ps(__x, _MM_FROUND_CUR_DIRECTION);
2824 else if constexpr (_TVT::template _S_is<double, 4>)
2825 return _mm256_round_pd(__x, _MM_FROUND_CUR_DIRECTION);
2826 else if constexpr (__have_sse4_1 && _TVT::template _S_is<float, 4>)
2827 return _mm_round_ps(__x, _MM_FROUND_CUR_DIRECTION);
2828 else if constexpr (__have_sse4_1 && _TVT::template _S_is<double, 2>)
2829 return _mm_round_pd(__x, _MM_FROUND_CUR_DIRECTION);
2830 else
2831 return _Base::_S_rint(__x);
2832 }
2833
2834 // }}}
2835 // _S_floor {{{
2836 template <typename _Tp, size_t _Np>
2837 _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
2838 _S_floor(_SimdWrapper<_Tp, _Np> __x)
2839 {
2840 if constexpr (__is_avx512_ps<_Tp, _Np>())
2841 return _mm512_roundscale_ps(__x, 0x09);
2842 else if constexpr (__is_avx512_pd<_Tp, _Np>())
2843 return _mm512_roundscale_pd(__x, 0x09);
2844 else if constexpr (__is_avx_ps<_Tp, _Np>())
2845 return _mm256_round_ps(__x, 0x9);
2846 else if constexpr (__is_avx_pd<_Tp, _Np>())
2847 return _mm256_round_pd(__x, 0x9);
2848 else if constexpr (__have_sse4_1 && __is_sse_ps<_Tp, _Np>())
2849 return __auto_bitcast(_mm_round_ps(__to_intrin(__x), 0x9));
2850 else if constexpr (__have_sse4_1 && __is_sse_pd<_Tp, _Np>())
2851 return _mm_round_pd(__x, 0x9);
2852 else
2853 return _Base::_S_floor(__x);
2854 }
2855
2856 // }}}
2857 // _S_ceil {{{
2858 template <typename _Tp, size_t _Np>
2859 _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
2860 _S_ceil(_SimdWrapper<_Tp, _Np> __x)
2861 {
2862 if constexpr (__is_avx512_ps<_Tp, _Np>())
2863 return _mm512_roundscale_ps(__x, 0x0a);
2864 else if constexpr (__is_avx512_pd<_Tp, _Np>())
2865 return _mm512_roundscale_pd(__x, 0x0a);
2866 else if constexpr (__is_avx_ps<_Tp, _Np>())
2867 return _mm256_round_ps(__x, 0xa);
2868 else if constexpr (__is_avx_pd<_Tp, _Np>())
2869 return _mm256_round_pd(__x, 0xa);
2870 else if constexpr (__have_sse4_1 && __is_sse_ps<_Tp, _Np>())
2871 return __auto_bitcast(_mm_round_ps(__to_intrin(__x), 0xa));
2872 else if constexpr (__have_sse4_1 && __is_sse_pd<_Tp, _Np>())
2873 return _mm_round_pd(__x, 0xa);
2874 else
2875 return _Base::_S_ceil(__x);
2876 }
2877
2878 // }}}
2879 // _S_signbit {{{
2880 template <typename _Tp, size_t _Np>
2881 _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
2882 _S_signbit(_SimdWrapper<_Tp, _Np> __x)
2883 {
2884 if constexpr (__is_avx512_abi<_Abi>() && __have_avx512dq)
2885 {
2886 if constexpr (sizeof(__x) == 64 && sizeof(_Tp) == 4)
2887 return _mm512_movepi32_mask(
2888 __intrin_bitcast<__m512i>(__x._M_data));
2889 else if constexpr (sizeof(__x) == 64 && sizeof(_Tp) == 8)
2890 return _mm512_movepi64_mask(
2891 __intrin_bitcast<__m512i>(__x._M_data));
2892 else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) == 4)
2893 return _mm256_movepi32_mask(
2894 __intrin_bitcast<__m256i>(__x._M_data));
2895 else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) == 8)
2896 return _mm256_movepi64_mask(
2897 __intrin_bitcast<__m256i>(__x._M_data));
2898 else if constexpr (sizeof(__x) <= 16 && sizeof(_Tp) == 4)
2899 return _mm_movepi32_mask(__intrin_bitcast<__m128i>(__x._M_data));
2900 else if constexpr (sizeof(__x) <= 16 && sizeof(_Tp) == 8)
2901 return _mm_movepi64_mask(__intrin_bitcast<__m128i>(__x._M_data));
2902 }
2903 else if constexpr (__is_avx512_abi<_Abi>())
2904 {
2905 const auto __xi = __to_intrin(__x);
2906 [[maybe_unused]] constexpr auto __k1
2907 = _Abi::template _S_implicit_mask_intrin<_Tp>();
2908 if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
2909 return _mm_movemask_ps(__xi);
2910 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
2911 return _mm_movemask_pd(__xi);
2912 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
2913 return _mm256_movemask_ps(__xi);
2914 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
2915 return _mm256_movemask_pd(__xi);
2916 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
2917 return _mm512_mask_cmplt_epi32_mask(
2918 __k1, __intrin_bitcast<__m512i>(__xi), __m512i());
2919 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
2920 return _mm512_mask_cmplt_epi64_mask(
2921 __k1, __intrin_bitcast<__m512i>(__xi), __m512i());
2922 else
2923 __assert_unreachable<_Tp>();
2924 }
2925 else
2926 return _Base::_S_signbit(__x);
2927 /*{
2928 using _I = __int_for_sizeof_t<_Tp>;
2929 if constexpr (sizeof(__x) == 64)
2930 return _S_less(__vector_bitcast<_I>(__x), _I());
2931 else
2932 {
2933 const auto __xx = __vector_bitcast<_I>(__x._M_data);
2934 [[maybe_unused]] constexpr _I __signmask = __finite_min_v<_I>;
2935 if constexpr ((sizeof(_Tp) == 4 &&
2936 (__have_avx2 || sizeof(__x) == 16)) ||
2937 __have_avx512vl)
2938 {
2939 return __vector_bitcast<_Tp>(__xx >> __digits_v<_I>);
2940 }
2941 else if constexpr ((__have_avx2 ||
2942 (__have_ssse3 && sizeof(__x) == 16)))
2943 {
2944 return __vector_bitcast<_Tp>((__xx & __signmask) ==
2945 __signmask);
2946 }
2947 else
2948 { // SSE2/3 or AVX (w/o AVX2)
2949 constexpr auto __one = __vector_broadcast<_Np, _Tp>(1);
2950 return __vector_bitcast<_Tp>(
2951 __vector_bitcast<_Tp>(
2952 (__xx & __signmask) |
2953 __vector_bitcast<_I>(__one)) // -1 or 1
2954 != __one);
2955 }
2956 }
2957 }*/
2958 }
2959
2960 // }}}
2961 // _S_isnonzerovalue_mask {{{
2962 // (isnormal | is subnormal == !isinf & !isnan & !is zero)
2963 template <typename _Tp>
2964 _GLIBCXX_SIMD_INTRINSIC static auto
2965 _S_isnonzerovalue_mask(_Tp __x)
2966 {
2967 using _Traits = _VectorTraits<_Tp>;
2968 if constexpr (__have_avx512dq_vl)
2969 {
2970 if constexpr (_Traits::template _S_is<
2971 float, 2> || _Traits::template _S_is<float, 4>)
2972 return _knot_mask8(_mm_fpclass_ps_mask(__to_intrin(__x), 0x9f));
2973 else if constexpr (_Traits::template _S_is<float, 8>)
2974 return _knot_mask8(_mm256_fpclass_ps_mask(__x, 0x9f));
2975 else if constexpr (_Traits::template _S_is<float, 16>)
2976 return _knot_mask16(_mm512_fpclass_ps_mask(__x, 0x9f));
2977 else if constexpr (_Traits::template _S_is<double, 2>)
2978 return _knot_mask8(_mm_fpclass_pd_mask(__x, 0x9f));
2979 else if constexpr (_Traits::template _S_is<double, 4>)
2980 return _knot_mask8(_mm256_fpclass_pd_mask(__x, 0x9f));
2981 else if constexpr (_Traits::template _S_is<double, 8>)
2982 return _knot_mask8(_mm512_fpclass_pd_mask(__x, 0x9f));
2983 else
2984 __assert_unreachable<_Tp>();
2985 }
2986 else
2987 {
2988 using _Up = typename _Traits::value_type;
2989 constexpr size_t _Np = _Traits::_S_full_size;
2990 const auto __a = __x * __infinity_v<_Up>; // NaN if __x == 0
2991 const auto __b = __x * _Up(); // NaN if __x == inf
2992 if constexpr (__have_avx512vl && __is_sse_ps<_Up, _Np>())
2993 return _mm_cmp_ps_mask(__to_intrin(__a), __to_intrin(__b),
2994 _CMP_ORD_Q);
2995 else if constexpr (__have_avx512f && __is_sse_ps<_Up, _Np>())
2996 return __mmask8(0xf
2997 & _mm512_cmp_ps_mask(__auto_bitcast(__a),
2998 __auto_bitcast(__b),
2999 _CMP_ORD_Q));
3000 else if constexpr (__have_avx512vl && __is_sse_pd<_Up, _Np>())
3001 return _mm_cmp_pd_mask(__a, __b, _CMP_ORD_Q);
3002 else if constexpr (__have_avx512f && __is_sse_pd<_Up, _Np>())
3003 return __mmask8(0x3
3004 & _mm512_cmp_pd_mask(__auto_bitcast(__a),
3005 __auto_bitcast(__b),
3006 _CMP_ORD_Q));
3007 else if constexpr (__have_avx512vl && __is_avx_ps<_Up, _Np>())
3008 return _mm256_cmp_ps_mask(__a, __b, _CMP_ORD_Q);
3009 else if constexpr (__have_avx512f && __is_avx_ps<_Up, _Np>())
3010 return __mmask8(_mm512_cmp_ps_mask(__auto_bitcast(__a),
3011 __auto_bitcast(__b),
3012 _CMP_ORD_Q));
3013 else if constexpr (__have_avx512vl && __is_avx_pd<_Up, _Np>())
3014 return _mm256_cmp_pd_mask(__a, __b, _CMP_ORD_Q);
3015 else if constexpr (__have_avx512f && __is_avx_pd<_Up, _Np>())
3016 return __mmask8(0xf
3017 & _mm512_cmp_pd_mask(__auto_bitcast(__a),
3018 __auto_bitcast(__b),
3019 _CMP_ORD_Q));
3020 else if constexpr (__is_avx512_ps<_Up, _Np>())
3021 return _mm512_cmp_ps_mask(__a, __b, _CMP_ORD_Q);
3022 else if constexpr (__is_avx512_pd<_Up, _Np>())
3023 return _mm512_cmp_pd_mask(__a, __b, _CMP_ORD_Q);
3024 else
3025 __assert_unreachable<_Tp>();
3026 }
3027 }
3028
3029 // }}}
3030 // _S_isfinite {{{
3031 template <typename _Tp, size_t _Np>
3032 _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
3033 _S_isfinite(_SimdWrapper<_Tp, _Np> __x)
3034 {
3035 static_assert(is_floating_point_v<_Tp>);
3036#if !__FINITE_MATH_ONLY__
3037 if constexpr (__is_avx512_abi<_Abi>() && __have_avx512dq)
3038 {
3039 const auto __xi = __to_intrin(__x);
3040 constexpr auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
3041 if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
3042 return __k1 ^ _mm512_mask_fpclass_ps_mask(__k1, __xi, 0x99);
3043 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
3044 return __k1 ^ _mm512_mask_fpclass_pd_mask(__k1, __xi, 0x99);
3045 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
3046 return __k1 ^ _mm256_mask_fpclass_ps_mask(__k1, __xi, 0x99);
3047 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
3048 return __k1 ^ _mm256_mask_fpclass_pd_mask(__k1, __xi, 0x99);
3049 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
3050 return __k1 ^ _mm_mask_fpclass_ps_mask(__k1, __xi, 0x99);
3051 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
3052 return __k1 ^ _mm_mask_fpclass_pd_mask(__k1, __xi, 0x99);
3053 }
3054 else if constexpr (__is_avx512_abi<_Abi>())
3055 {
3056 // if all exponent bits are set, __x is either inf or NaN
3057 using _I = __int_for_sizeof_t<_Tp>;
3058 const auto __inf = __vector_bitcast<_I>(
3059 __vector_broadcast<_Np>(__infinity_v<_Tp>));
3060 return _S_less<_I, _Np>(__vector_bitcast<_I>(__x) & __inf, __inf);
3061 }
3062 else
3063#endif
3064 return _Base::_S_isfinite(__x);
3065 }
3066
3067 // }}}
3068 // _S_isinf {{{
3069 template <typename _Tp, size_t _Np>
3070 _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
3071 _S_isinf(_SimdWrapper<_Tp, _Np> __x)
3072 {
3073#if !__FINITE_MATH_ONLY__
3074 if constexpr (__is_avx512_abi<_Abi>() && __have_avx512dq)
3075 {
3076 const auto __xi = __to_intrin(__x);
3077 if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
3078 return _mm512_fpclass_ps_mask(__xi, 0x18);
3079 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
3080 return _mm512_fpclass_pd_mask(__xi, 0x18);
3081 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
3082 return _mm256_fpclass_ps_mask(__xi, 0x18);
3083 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
3084 return _mm256_fpclass_pd_mask(__xi, 0x18);
3085 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
3086 return _mm_fpclass_ps_mask(__xi, 0x18);
3087 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
3088 return _mm_fpclass_pd_mask(__xi, 0x18);
3089 else
3090 __assert_unreachable<_Tp>();
3091 }
3092 else if constexpr (__have_avx512dq_vl)
3093 {
3094 if constexpr (__is_sse_pd<_Tp, _Np>())
3095 return _mm_movm_epi64(_mm_fpclass_pd_mask(__x, 0x18));
3096 else if constexpr (__is_avx_pd<_Tp, _Np>())
3097 return _mm256_movm_epi64(_mm256_fpclass_pd_mask(__x, 0x18));
3098 else if constexpr (__is_sse_ps<_Tp, _Np>())
3099 return _mm_movm_epi32(
3100 _mm_fpclass_ps_mask(__to_intrin(__x), 0x18));
3101 else if constexpr (__is_avx_ps<_Tp, _Np>())
3102 return _mm256_movm_epi32(_mm256_fpclass_ps_mask(__x, 0x18));
3103 else
3104 __assert_unreachable<_Tp>();
3105 }
3106 else
3107#endif
3108 return _Base::_S_isinf(__x);
3109 }
3110
3111 // }}}
3112 // _S_isnormal {{{
3113 template <typename _Tp, size_t _Np>
3114 _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
3115 _S_isnormal(_SimdWrapper<_Tp, _Np> __x)
3116 {
3117#if __FINITE_MATH_ONLY__
3118 [[maybe_unused]] constexpr int __mode = 0x26;
3119#else
3120 [[maybe_unused]] constexpr int __mode = 0xbf;
3121#endif
3122 if constexpr (__is_avx512_abi<_Abi>() && __have_avx512dq)
3123 {
3124 const auto __xi = __to_intrin(__x);
3125 const auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
3126 if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
3127 return __k1 ^ _mm512_mask_fpclass_ps_mask(__k1, __xi, __mode);
3128 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
3129 return __k1 ^ _mm512_mask_fpclass_pd_mask(__k1, __xi, __mode);
3130 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
3131 return __k1 ^ _mm256_mask_fpclass_ps_mask(__k1, __xi, __mode);
3132 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
3133 return __k1 ^ _mm256_mask_fpclass_pd_mask(__k1, __xi, __mode);
3134 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
3135 return __k1 ^ _mm_mask_fpclass_ps_mask(__k1, __xi, __mode);
3136 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
3137 return __k1 ^ _mm_mask_fpclass_pd_mask(__k1, __xi, __mode);
3138 else
3139 __assert_unreachable<_Tp>();
3140 }
3141 else if constexpr (__have_avx512dq)
3142 {
3143 if constexpr (__have_avx512vl && __is_sse_ps<_Tp, _Np>())
3144 return _mm_movm_epi32(
3145 _knot_mask8(_mm_fpclass_ps_mask(__to_intrin(__x), __mode)));
3146 else if constexpr (__have_avx512vl && __is_avx_ps<_Tp, _Np>())
3147 return _mm256_movm_epi32(
3148 _knot_mask8(_mm256_fpclass_ps_mask(__x, __mode)));
3149 else if constexpr (__is_avx512_ps<_Tp, _Np>())
3150 return _knot_mask16(_mm512_fpclass_ps_mask(__x, __mode));
3151 else if constexpr (__have_avx512vl && __is_sse_pd<_Tp, _Np>())
3152 return _mm_movm_epi64(
3153 _knot_mask8(_mm_fpclass_pd_mask(__x, __mode)));
3154 else if constexpr (__have_avx512vl && __is_avx_pd<_Tp, _Np>())
3155 return _mm256_movm_epi64(
3156 _knot_mask8(_mm256_fpclass_pd_mask(__x, __mode)));
3157 else if constexpr (__is_avx512_pd<_Tp, _Np>())
3158 return _knot_mask8(_mm512_fpclass_pd_mask(__x, __mode));
3159 else
3160 __assert_unreachable<_Tp>();
3161 }
3162 else if constexpr (__is_avx512_abi<_Abi>())
3163 {
3164 using _I = __int_for_sizeof_t<_Tp>;
3165 const auto absn = __vector_bitcast<_I>(_S_abs(__x));
3166 const auto minn = __vector_bitcast<_I>(
3167 __vector_broadcast<_Np>(__norm_min_v<_Tp>));
3168#if __FINITE_MATH_ONLY__
3169 return _S_less_equal<_I, _Np>(minn, absn);
3170#else
3171 const auto infn
3172 = __vector_bitcast<_I>(__vector_broadcast<_Np>(__infinity_v<_Tp>));
3173 return __and(_S_less_equal<_I, _Np>(minn, absn),
3174 _S_less<_I, _Np>(absn, infn));
3175#endif
3176 }
3177 else
3178 return _Base::_S_isnormal(__x);
3179 }
3180
3181 // }}}
3182 // _S_isnan {{{
3183 template <typename _Tp, size_t _Np>
3184 _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
3185 _S_isnan(_SimdWrapper<_Tp, _Np> __x)
3186 { return _S_isunordered(__x, __x); }
3187
3188 // }}}
3189 // _S_isunordered {{{
3190 template <typename _Tp, size_t _Np>
3191 _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
3192 _S_isunordered([[maybe_unused]] _SimdWrapper<_Tp, _Np> __x,
3193 [[maybe_unused]] _SimdWrapper<_Tp, _Np> __y)
3194 {
3195#if __FINITE_MATH_ONLY__
3196 return {}; // false
3197#else
3198 const auto __xi = __to_intrin(__x);
3199 const auto __yi = __to_intrin(__y);
3200 if constexpr (__is_avx512_abi<_Abi>())
3201 {
3202 constexpr auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
3203 if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
3204 return _mm512_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_UNORD_Q);
3205 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
3206 return _mm512_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_UNORD_Q);
3207 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
3208 return _mm256_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_UNORD_Q);
3209 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
3210 return _mm256_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_UNORD_Q);
3211 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
3212 return _mm_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_UNORD_Q);
3213 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
3214 return _mm_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_UNORD_Q);
3215 }
3216 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
3217 return __to_masktype(_mm256_cmp_ps(__xi, __yi, _CMP_UNORD_Q));
3218 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
3219 return __to_masktype(_mm256_cmp_pd(__xi, __yi, _CMP_UNORD_Q));
3220 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
3221 return __auto_bitcast(_mm_cmpunord_ps(__xi, __yi));
3222 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
3223 return __to_masktype(_mm_cmpunord_pd(__xi, __yi));
3224 else
3225 __assert_unreachable<_Tp>();
3226#endif
3227 }
3228
3229 // }}}
3230 // _S_isgreater {{{
3231 template <typename _Tp, size_t _Np>
3232 static constexpr _MaskMember<_Tp>
3233 _S_isgreater(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
3234 {
3235 const auto __xi = __to_intrin(__x);
3236 const auto __yi = __to_intrin(__y);
3237 if constexpr (__is_avx512_abi<_Abi>())
3238 {
3239 const auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
3240 if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
3241 return _mm512_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_GT_OQ);
3242 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
3243 return _mm512_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_GT_OQ);
3244 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
3245 return _mm256_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_GT_OQ);
3246 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
3247 return _mm256_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_GT_OQ);
3248 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
3249 return _mm_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_GT_OQ);
3250 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
3251 return _mm_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_GT_OQ);
3252 else
3253 __assert_unreachable<_Tp>();
3254 }
3255 else if constexpr (__have_avx)
3256 {
3257 if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
3258 return __to_masktype(_mm256_cmp_ps(__xi, __yi, _CMP_GT_OQ));
3259 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
3260 return __to_masktype(_mm256_cmp_pd(__xi, __yi, _CMP_GT_OQ));
3261 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
3262 return __auto_bitcast(_mm_cmp_ps(__xi, __yi, _CMP_GT_OQ));
3263 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
3264 return __to_masktype(_mm_cmp_pd(__xi, __yi, _CMP_GT_OQ));
3265 else
3266 __assert_unreachable<_Tp>();
3267 }
3268 else if constexpr (__have_sse2 && sizeof(__xi) == 16
3269 && sizeof(_Tp) == 4)
3270 {
3271 const auto __xn = __vector_bitcast<int>(__xi);
3272 const auto __yn = __vector_bitcast<int>(__yi);
3273 const auto __xp = __xn < 0 ? -(__xn & 0x7fff'ffff) : __xn;
3274 const auto __yp = __yn < 0 ? -(__yn & 0x7fff'ffff) : __yn;
3275 return __auto_bitcast(
3276 __and(__to_masktype(_mm_cmpord_ps(__xi, __yi)), __xp > __yp));
3277 }
3278 else if constexpr (__have_sse2 && sizeof(__xi) == 16
3279 && sizeof(_Tp) == 8)
3280 return __vector_type_t<__int_with_sizeof_t<8>, 2>{
3281 -_mm_ucomigt_sd(__xi, __yi),
3282 -_mm_ucomigt_sd(_mm_unpackhi_pd(__xi, __xi),
3283 _mm_unpackhi_pd(__yi, __yi))};
3284 else
3285 return _Base::_S_isgreater(__x, __y);
3286 }
3287
3288 // }}}
3289 // _S_isgreaterequal {{{
3290 template <typename _Tp, size_t _Np>
3291 static constexpr _MaskMember<_Tp>
3292 _S_isgreaterequal(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
3293 {
3294 const auto __xi = __to_intrin(__x);
3295 const auto __yi = __to_intrin(__y);
3296 if constexpr (__is_avx512_abi<_Abi>())
3297 {
3298 const auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
3299 if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
3300 return _mm512_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_GE_OQ);
3301 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
3302 return _mm512_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_GE_OQ);
3303 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
3304 return _mm256_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_GE_OQ);
3305 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
3306 return _mm256_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_GE_OQ);
3307 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
3308 return _mm_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_GE_OQ);
3309 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
3310 return _mm_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_GE_OQ);
3311 else
3312 __assert_unreachable<_Tp>();
3313 }
3314 else if constexpr (__have_avx)
3315 {
3316 if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
3317 return __to_masktype(_mm256_cmp_ps(__xi, __yi, _CMP_GE_OQ));
3318 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
3319 return __to_masktype(_mm256_cmp_pd(__xi, __yi, _CMP_GE_OQ));
3320 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
3321 return __auto_bitcast(_mm_cmp_ps(__xi, __yi, _CMP_GE_OQ));
3322 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
3323 return __to_masktype(_mm_cmp_pd(__xi, __yi, _CMP_GE_OQ));
3324 else
3325 __assert_unreachable<_Tp>();
3326 }
3327 else if constexpr (__have_sse2 && sizeof(__xi) == 16
3328 && sizeof(_Tp) == 4)
3329 {
3330 const auto __xn = __vector_bitcast<int>(__xi);
3331 const auto __yn = __vector_bitcast<int>(__yi);
3332 const auto __xp = __xn < 0 ? -(__xn & 0x7fff'ffff) : __xn;
3333 const auto __yp = __yn < 0 ? -(__yn & 0x7fff'ffff) : __yn;
3334 return __auto_bitcast(
3335 __and(__to_masktype(_mm_cmpord_ps(__xi, __yi)), __xp >= __yp));
3336 }
3337 else if constexpr (__have_sse2 && sizeof(__xi) == 16
3338 && sizeof(_Tp) == 8)
3339 return __vector_type_t<__int_with_sizeof_t<8>, 2>{
3340 -_mm_ucomige_sd(__xi, __yi),
3341 -_mm_ucomige_sd(_mm_unpackhi_pd(__xi, __xi),
3342 _mm_unpackhi_pd(__yi, __yi))};
3343 else
3344 return _Base::_S_isgreaterequal(__x, __y);
3345 }
3346
3347 // }}}
3348 // _S_isless {{{
3349 template <typename _Tp, size_t _Np>
3350 static constexpr _MaskMember<_Tp>
3351 _S_isless(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
3352 {
3353 const auto __xi = __to_intrin(__x);
3354 const auto __yi = __to_intrin(__y);
3355 if constexpr (__is_avx512_abi<_Abi>())
3356 {
3357 const auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
3358 if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
3359 return _mm512_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LT_OQ);
3360 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
3361 return _mm512_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LT_OQ);
3362 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
3363 return _mm256_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LT_OQ);
3364 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
3365 return _mm256_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LT_OQ);
3366 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
3367 return _mm_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LT_OQ);
3368 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
3369 return _mm_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LT_OQ);
3370 else
3371 __assert_unreachable<_Tp>();
3372 }
3373 else if constexpr (__have_avx)
3374 {
3375 if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
3376 return __to_masktype(_mm256_cmp_ps(__xi, __yi, _CMP_LT_OQ));
3377 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
3378 return __to_masktype(_mm256_cmp_pd(__xi, __yi, _CMP_LT_OQ));
3379 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
3380 return __auto_bitcast(_mm_cmp_ps(__xi, __yi, _CMP_LT_OQ));
3381 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
3382 return __to_masktype(_mm_cmp_pd(__xi, __yi, _CMP_LT_OQ));
3383 else
3384 __assert_unreachable<_Tp>();
3385 }
3386 else if constexpr (__have_sse2 && sizeof(__xi) == 16
3387 && sizeof(_Tp) == 4)
3388 {
3389 const auto __xn = __vector_bitcast<int>(__xi);
3390 const auto __yn = __vector_bitcast<int>(__yi);
3391 const auto __xp = __xn < 0 ? -(__xn & 0x7fff'ffff) : __xn;
3392 const auto __yp = __yn < 0 ? -(__yn & 0x7fff'ffff) : __yn;
3393 return __auto_bitcast(
3394 __and(__to_masktype(_mm_cmpord_ps(__xi, __yi)), __xp < __yp));
3395 }
3396 else if constexpr (__have_sse2 && sizeof(__xi) == 16
3397 && sizeof(_Tp) == 8)
3398 return __vector_type_t<__int_with_sizeof_t<8>, 2>{
3399 -_mm_ucomigt_sd(__yi, __xi),
3400 -_mm_ucomigt_sd(_mm_unpackhi_pd(__yi, __yi),
3401 _mm_unpackhi_pd(__xi, __xi))};
3402 else
3403 return _Base::_S_isless(__x, __y);
3404 }
3405
3406 // }}}
3407 // _S_islessequal {{{
3408 template <typename _Tp, size_t _Np>
3409 static constexpr _MaskMember<_Tp>
3410 _S_islessequal(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
3411 {
3412 const auto __xi = __to_intrin(__x);
3413 const auto __yi = __to_intrin(__y);
3414 if constexpr (__is_avx512_abi<_Abi>())
3415 {
3416 const auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
3417 if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
3418 return _mm512_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LE_OQ);
3419 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
3420 return _mm512_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LE_OQ);
3421 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
3422 return _mm256_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LE_OQ);
3423 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
3424 return _mm256_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LE_OQ);
3425 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
3426 return _mm_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LE_OQ);
3427 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
3428 return _mm_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LE_OQ);
3429 else
3430 __assert_unreachable<_Tp>();
3431 }
3432 else if constexpr (__have_avx)
3433 {
3434 if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
3435 return __to_masktype(_mm256_cmp_ps(__xi, __yi, _CMP_LE_OQ));
3436 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
3437 return __to_masktype(_mm256_cmp_pd(__xi, __yi, _CMP_LE_OQ));
3438 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
3439 return __auto_bitcast(_mm_cmp_ps(__xi, __yi, _CMP_LE_OQ));
3440 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
3441 return __to_masktype(_mm_cmp_pd(__xi, __yi, _CMP_LE_OQ));
3442 else
3443 __assert_unreachable<_Tp>();
3444 }
3445 else if constexpr (__have_sse2 && sizeof(__xi) == 16
3446 && sizeof(_Tp) == 4)
3447 {
3448 const auto __xn = __vector_bitcast<int>(__xi);
3449 const auto __yn = __vector_bitcast<int>(__yi);
3450 const auto __xp = __xn < 0 ? -(__xn & 0x7fff'ffff) : __xn;
3451 const auto __yp = __yn < 0 ? -(__yn & 0x7fff'ffff) : __yn;
3452 return __auto_bitcast(
3453 __and(__to_masktype(_mm_cmpord_ps(__xi, __yi)), __xp <= __yp));
3454 }
3455 else if constexpr (__have_sse2 && sizeof(__xi) == 16
3456 && sizeof(_Tp) == 8)
3457 return __vector_type_t<__int_with_sizeof_t<8>, 2>{
3458 -_mm_ucomige_sd(__yi, __xi),
3459 -_mm_ucomige_sd(_mm_unpackhi_pd(__yi, __yi),
3460 _mm_unpackhi_pd(__xi, __xi))};
3461 else
3462 return _Base::_S_islessequal(__x, __y);
3463 }
3464
3465 // }}}
3466 // _S_islessgreater {{{
3467 template <typename _Tp, size_t _Np>
3468 static constexpr _MaskMember<_Tp>
3469 _S_islessgreater(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
3470 {
3471 const auto __xi = __to_intrin(__x);
3472 const auto __yi = __to_intrin(__y);
3473 if constexpr (__is_avx512_abi<_Abi>())
3474 {
3475 const auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
3476 if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
3477 return _mm512_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_NEQ_OQ);
3478 else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
3479 return _mm512_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_NEQ_OQ);
3480 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
3481 return _mm256_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_NEQ_OQ);
3482 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
3483 return _mm256_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_NEQ_OQ);
3484 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
3485 return _mm_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_NEQ_OQ);
3486 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
3487 return _mm_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_NEQ_OQ);
3488 else
3489 __assert_unreachable<_Tp>();
3490 }
3491 else if constexpr (__have_avx)
3492 {
3493 if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
3494 return __to_masktype(_mm256_cmp_ps(__xi, __yi, _CMP_NEQ_OQ));
3495 else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
3496 return __to_masktype(_mm256_cmp_pd(__xi, __yi, _CMP_NEQ_OQ));
3497 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
3498 return __auto_bitcast(_mm_cmp_ps(__xi, __yi, _CMP_NEQ_OQ));
3499 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
3500 return __to_masktype(_mm_cmp_pd(__xi, __yi, _CMP_NEQ_OQ));
3501 else
3502 __assert_unreachable<_Tp>();
3503 }
3504 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
3505 return __auto_bitcast(
3506 __and(_mm_cmpord_ps(__xi, __yi), _mm_cmpneq_ps(__xi, __yi)));
3507 else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
3508 return __to_masktype(
3509 __and(_mm_cmpord_pd(__xi, __yi), _mm_cmpneq_pd(__xi, __yi)));
3510 else
3511 __assert_unreachable<_Tp>();
3512 }
3513
3514 //}}} }}}
3515 template <template <typename> class _Op, typename _Tp, typename _K, size_t _Np>
3516 _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
3517 _S_masked_unary(const _SimdWrapper<_K, _Np> __k, const _SimdWrapper<_Tp, _Np> __v)
3518 {
3519 if (__k._M_is_constprop_none_of())
3520 return __v;
3521 else if (__k._M_is_constprop_all_of())
3522 {
3523 auto __vv = _Base::_M_make_simd(__v);
3524 _Op<decltype(__vv)> __op;
3525 return __data(__op(__vv));
3526 }
3527 else if constexpr (__is_bitmask_v<decltype(__k)>
3528 && (is_same_v<_Op<void>, __increment<void>>
3529 || is_same_v<_Op<void>, __decrement<void>>))
3530 {
3531 // optimize masked unary increment and decrement as masked sub +/-1
3532 constexpr int __pm_one
3533 = is_same_v<_Op<void>, __increment<void>> ? -1 : 1;
3534#ifdef __clang__
3535 return __movm<_Np, _Tp>(__k._M_data) ? __v._M_data - __pm_one : __v._M_data;
3536#else // __clang__
3537 if constexpr (is_integral_v<_Tp>)
3538 {
3539 constexpr bool __lp64 = sizeof(long) == sizeof(long long);
3540 using _Ip = std::make_signed_t<_Tp>;
3541 using _Up = std::conditional_t<
3542 std::is_same_v<_Ip, long>,
3543 std::conditional_t<__lp64, long long, int>,
3544 std::conditional_t<
3545 std::is_same_v<_Ip, signed char>, char, _Ip>>;
3546 const auto __value = __vector_bitcast<_Up>(__v._M_data);
3547#define _GLIBCXX_SIMD_MASK_SUB(_Sizeof, _Width, _Instr) \
3548 if constexpr (sizeof(_Tp) == _Sizeof && sizeof(__v) == _Width) \
3549 return __vector_bitcast<_Tp>(__builtin_ia32_##_Instr##_mask(__value, \
3550 __vector_broadcast<_Np>(_Up(__pm_one)), __value, __k._M_data))
3551 _GLIBCXX_SIMD_MASK_SUB(1, 64, psubb512);
3552 _GLIBCXX_SIMD_MASK_SUB(1, 32, psubb256);
3553 _GLIBCXX_SIMD_MASK_SUB(1, 16, psubb128);
3554 _GLIBCXX_SIMD_MASK_SUB(2, 64, psubw512);
3555 _GLIBCXX_SIMD_MASK_SUB(2, 32, psubw256);
3556 _GLIBCXX_SIMD_MASK_SUB(2, 16, psubw128);
3557 _GLIBCXX_SIMD_MASK_SUB(4, 64, psubd512);
3558 _GLIBCXX_SIMD_MASK_SUB(4, 32, psubd256);
3559 _GLIBCXX_SIMD_MASK_SUB(4, 16, psubd128);
3560 _GLIBCXX_SIMD_MASK_SUB(8, 64, psubq512);
3561 _GLIBCXX_SIMD_MASK_SUB(8, 32, psubq256);
3562 _GLIBCXX_SIMD_MASK_SUB(8, 16, psubq128);
3563#undef _GLIBCXX_SIMD_MASK_SUB
3564 }
3565 else
3566 {
3567#define _GLIBCXX_SIMD_MASK_SUB(_Sizeof, _Width, _Instr) \
3568 if constexpr (sizeof(_Tp) == _Sizeof && sizeof(__v) == _Width) \
3569 return __builtin_ia32_##_Instr##_mask( \
3570 __v._M_data, __vector_broadcast<_Np>(_Tp(__pm_one)), __v._M_data, \
3571 __k._M_data, _MM_FROUND_CUR_DIRECTION)
3572 _GLIBCXX_SIMD_MASK_SUB(4, 64, subps512);
3573 _GLIBCXX_SIMD_MASK_SUB(4, 32, subps256);
3574 _GLIBCXX_SIMD_MASK_SUB(4, 16, subps128);
3575 _GLIBCXX_SIMD_MASK_SUB(8, 64, subpd512);
3576 _GLIBCXX_SIMD_MASK_SUB(8, 32, subpd256);
3577 _GLIBCXX_SIMD_MASK_SUB(8, 16, subpd128);
3578#undef _GLIBCXX_SIMD_MASK_SUB
3579 }
3580#endif // __clang__
3581 }
3582 else
3583 return _Base::template _S_masked_unary<_Op>(__k, __v);
3584 }
3585 };
3586
3587// }}}
3588// _MaskImplX86Mixin {{{
3589struct _MaskImplX86Mixin
3590{
3591 template <typename _Tp>
3592 using _TypeTag = _Tp*;
3593
3594 using _Base = _MaskImplBuiltinMixin;
3595
3596 // _S_to_maskvector(bool) {{{
3597 template <typename _Up, size_t _ToN = 1, typename _Tp>
3598 _GLIBCXX_SIMD_INTRINSIC static constexpr
3599 enable_if_t<is_same_v<_Tp, bool>, _SimdWrapper<_Up, _ToN>>
3600 _S_to_maskvector(_Tp __x)
3601 {
3602 static_assert(is_same_v<_Up, __int_for_sizeof_t<_Up>>);
3603 return __x ? __vector_type_t<_Up, _ToN>{~_Up()}
3604 : __vector_type_t<_Up, _ToN>();
3605 }
3606
3607 // }}}
3608 // _S_to_maskvector(_SanitizedBitMask) {{{
3609 template <typename _Up, size_t _UpN = 0, size_t _Np, size_t _ToN = _UpN == 0 ? _Np : _UpN>
3610 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Up, _ToN>
3611 _S_to_maskvector(_SanitizedBitMask<_Np> __x)
3612 {
3613 static_assert(is_same_v<_Up, __int_for_sizeof_t<_Up>>);
3614 using _UV = __vector_type_t<_Up, _ToN>;
3615 using _UI = __intrinsic_type_t<_Up, _ToN>;
3616 [[maybe_unused]] const auto __k = __x._M_to_bits();
3617 if constexpr (_Np == 1)
3618 return _S_to_maskvector<_Up, _ToN>(__k);
3619 else if (__x._M_is_constprop() || __builtin_is_constant_evaluated())
3620 return __generate_from_n_evaluations<std::min(_ToN, _Np), _UV>(
3621 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA -> _Up { return -__x[__i.value]; });
3622 else if constexpr (sizeof(_Up) == 1)
3623 {
3624 if constexpr (sizeof(_UI) == 16)
3625 {
3626 if constexpr (__have_avx512bw_vl)
3627 return __intrin_bitcast<_UV>(_mm_movm_epi8(__k));
3628 else if constexpr (__have_avx512bw)
3629 return __intrin_bitcast<_UV>(__lo128(_mm512_movm_epi8(__k)));
3630 else if constexpr (__have_avx512f)
3631 {
3632 auto __as32bits = _mm512_maskz_mov_epi32(__k, ~__m512i());
3633 auto __as16bits
3634 = __xzyw(_mm256_packs_epi32(__lo256(__as32bits),
3635 __hi256(__as32bits)));
3636 return __intrin_bitcast<_UV>(
3637 _mm_packs_epi16(__lo128(__as16bits), __hi128(__as16bits)));
3638 }
3639 else if constexpr (__have_ssse3)
3640 {
3641 const auto __bitmask = __to_intrin(
3642 __make_vector<_UChar>(1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4,
3643 8, 16, 32, 64, 128));
3644 return __intrin_bitcast<_UV>(
3645 __vector_bitcast<_Up>(
3646 _mm_shuffle_epi8(__to_intrin(
3647 __vector_type_t<_ULLong, 2>{__k}),
3648 _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1,
3649 1, 1, 1, 1, 1, 1, 1))
3650 & __bitmask)
3651 != 0);
3652 }
3653 // else fall through
3654 }
3655 else if constexpr (sizeof(_UI) == 32)
3656 {
3657 if constexpr (__have_avx512bw_vl)
3658 return __vector_bitcast<_Up>(_mm256_movm_epi8(__k));
3659 else if constexpr (__have_avx512bw)
3660 return __vector_bitcast<_Up>(__lo256(_mm512_movm_epi8(__k)));
3661 else if constexpr (__have_avx512f)
3662 {
3663 auto __as16bits = // 0 16 1 17 ... 15 31
3664 _mm512_srli_epi32(_mm512_maskz_mov_epi32(__k, ~__m512i()),
3665 16)
3666 | _mm512_slli_epi32(_mm512_maskz_mov_epi32(__k >> 16,
3667 ~__m512i()),
3668 16);
3669 auto __0_16_1_17 = __xzyw(_mm256_packs_epi16(
3670 __lo256(__as16bits),
3671 __hi256(__as16bits)) // 0 16 1 17 2 18 3 19 8 24 9 25 ...
3672 );
3673 // deinterleave:
3674 return __vector_bitcast<_Up>(__xzyw(_mm256_shuffle_epi8(
3675 __0_16_1_17, // 0 16 1 17 2 ...
3676 _mm256_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9,
3677 11, 13, 15, 0, 2, 4, 6, 8, 10, 12, 14, 1,
3678 3, 5, 7, 9, 11, 13,
3679 15)))); // 0-7 16-23 8-15 24-31 -> xzyw
3680 // 0-3 8-11 16-19 24-27
3681 // 4-7 12-15 20-23 28-31
3682 }
3683 else if constexpr (__have_avx2)
3684 {
3685 const auto __bitmask
3686 = _mm256_broadcastsi128_si256(__to_intrin(
3687 __make_vector<_UChar>(1, 2, 4, 8, 16, 32, 64, 128, 1, 2,
3688 4, 8, 16, 32, 64, 128)));
3689 return __vector_bitcast<_Up>(
3690 __vector_bitcast<_Up>(
3691 _mm256_shuffle_epi8(
3692 _mm256_broadcastsi128_si256(
3693 __to_intrin(__vector_type_t<_ULLong, 2>{__k})),
3694 _mm256_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1,
3695 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3,
3696 3, 3, 3, 3, 3, 3))
3697 & __bitmask)
3698 != 0);
3699 }
3700 // else fall through
3701 }
3702 else if constexpr (sizeof(_UI) == 64)
3703 return reinterpret_cast<_UV>(_mm512_movm_epi8(__k));
3704 if constexpr (std::min(_ToN, _Np) <= 4)
3705 {
3706 if constexpr (_Np > 7) // avoid overflow
3707 __x &= _SanitizedBitMask<_Np>(0x0f);
3708 const _UInt __char_mask
3709 = ((_UInt(__x.to_ulong()) * 0x00204081U) & 0x01010101ULL)
3710 * 0xff;
3711 _UV __r = {};
3712 __builtin_memcpy(&__r, &__char_mask,
3713 std::min(sizeof(__r), sizeof(__char_mask)));
3714 return __r;
3715 }
3716 else if constexpr (std::min(_ToN, _Np) <= 7)
3717 {
3718 if constexpr (_Np > 7) // avoid overflow
3719 __x &= _SanitizedBitMask<_Np>(0x7f);
3720 const _ULLong __char_mask
3721 = ((__x.to_ulong() * 0x40810204081ULL) & 0x0101010101010101ULL)
3722 * 0xff;
3723 _UV __r = {};
3724 __builtin_memcpy(&__r, &__char_mask,
3725 std::min(sizeof(__r), sizeof(__char_mask)));
3726 return __r;
3727 }
3728 }
3729 else if constexpr (sizeof(_Up) == 2)
3730 {
3731 if constexpr (sizeof(_UI) == 16)
3732 {
3733 if constexpr (__have_avx512bw_vl)
3734 return __intrin_bitcast<_UV>(_mm_movm_epi16(__k));
3735 else if constexpr (__have_avx512bw)
3736 return __intrin_bitcast<_UV>(__lo128(_mm512_movm_epi16(__k)));
3737 else if constexpr (__have_avx512f)
3738 {
3739 __m256i __as32bits = {};
3740 if constexpr (__have_avx512vl)
3741 __as32bits = _mm256_maskz_mov_epi32(__k, ~__m256i());
3742 else
3743 __as32bits
3744 = __lo256(_mm512_maskz_mov_epi32(__k, ~__m512i()));
3745 return __intrin_bitcast<_UV>(
3746 _mm_packs_epi32(__lo128(__as32bits), __hi128(__as32bits)));
3747 }
3748 // else fall through
3749 }
3750 else if constexpr (sizeof(_UI) == 32)
3751 {
3752 if constexpr (__have_avx512bw_vl)
3753 return __vector_bitcast<_Up>(_mm256_movm_epi16(__k));
3754 else if constexpr (__have_avx512bw)
3755 return __vector_bitcast<_Up>(__lo256(_mm512_movm_epi16(__k)));
3756 else if constexpr (__have_avx512f)
3757 {
3758 auto __as32bits = _mm512_maskz_mov_epi32(__k, ~__m512i());
3759 return __vector_bitcast<_Up>(
3760 __xzyw(_mm256_packs_epi32(__lo256(__as32bits),
3761 __hi256(__as32bits))));
3762 }
3763 // else fall through
3764 }
3765 else if constexpr (sizeof(_UI) == 64)
3766 return __vector_bitcast<_Up>(_mm512_movm_epi16(__k));
3767 }
3768 else if constexpr (sizeof(_Up) == 4)
3769 {
3770 if constexpr (sizeof(_UI) == 16)
3771 {
3772 if constexpr (__have_avx512dq_vl)
3773 return __intrin_bitcast<_UV>(_mm_movm_epi32(__k));
3774 else if constexpr (__have_avx512dq)
3775 return __intrin_bitcast<_UV>(__lo128(_mm512_movm_epi32(__k)));
3776 else if constexpr (__have_avx512vl)
3777 return __intrin_bitcast<_UV>(
3778 _mm_maskz_mov_epi32(__k, ~__m128i()));
3779 else if constexpr (__have_avx512f)
3780 return __intrin_bitcast<_UV>(
3781 __lo128(_mm512_maskz_mov_epi32(__k, ~__m512i())));
3782 // else fall through
3783 }
3784 else if constexpr (sizeof(_UI) == 32)
3785 {
3786 if constexpr (__have_avx512dq_vl)
3787 return __vector_bitcast<_Up>(_mm256_movm_epi32(__k));
3788 else if constexpr (__have_avx512dq)
3789 return __vector_bitcast<_Up>(__lo256(_mm512_movm_epi32(__k)));
3790 else if constexpr (__have_avx512vl)
3791 return __vector_bitcast<_Up>(
3792 _mm256_maskz_mov_epi32(__k, ~__m256i()));
3793 else if constexpr (__have_avx512f)
3794 return __vector_bitcast<_Up>(
3795 __lo256(_mm512_maskz_mov_epi32(__k, ~__m512i())));
3796 // else fall through
3797 }
3798 else if constexpr (sizeof(_UI) == 64)
3799 return __vector_bitcast<_Up>(
3800 __have_avx512dq ? _mm512_movm_epi32(__k)
3801 : _mm512_maskz_mov_epi32(__k, ~__m512i()));
3802 }
3803 else if constexpr (sizeof(_Up) == 8)
3804 {
3805 if constexpr (sizeof(_UI) == 16)
3806 {
3807 if constexpr (__have_avx512dq_vl)
3808 return __vector_bitcast<_Up>(_mm_movm_epi64(__k));
3809 else if constexpr (__have_avx512dq)
3810 return __vector_bitcast<_Up>(__lo128(_mm512_movm_epi64(__k)));
3811 else if constexpr (__have_avx512vl)
3812 return __vector_bitcast<_Up>(
3813 _mm_maskz_mov_epi64(__k, ~__m128i()));
3814 else if constexpr (__have_avx512f)
3815 return __vector_bitcast<_Up>(
3816 __lo128(_mm512_maskz_mov_epi64(__k, ~__m512i())));
3817 // else fall through
3818 }
3819 else if constexpr (sizeof(_UI) == 32)
3820 {
3821 if constexpr (__have_avx512dq_vl)
3822 return __vector_bitcast<_Up>(_mm256_movm_epi64(__k));
3823 else if constexpr (__have_avx512dq)
3824 return __vector_bitcast<_Up>(__lo256(_mm512_movm_epi64(__k)));
3825 else if constexpr (__have_avx512vl)
3826 return __vector_bitcast<_Up>(
3827 _mm256_maskz_mov_epi64(__k, ~__m256i()));
3828 else if constexpr (__have_avx512f)
3829 return __vector_bitcast<_Up>(
3830 __lo256(_mm512_maskz_mov_epi64(__k, ~__m512i())));
3831 // else fall through
3832 }
3833 else if constexpr (sizeof(_UI) == 64)
3834 return __vector_bitcast<_Up>(
3835 __have_avx512dq ? _mm512_movm_epi64(__k)
3836 : _mm512_maskz_mov_epi64(__k, ~__m512i()));
3837 }
3838
3839 using _UpUInt = make_unsigned_t<_Up>;
3840 using _V = __vector_type_t<_UpUInt, _ToN>;
3841 constexpr size_t __bits_per_element = sizeof(_Up) * __CHAR_BIT__;
3842 if constexpr (_ToN == 2)
3843 {
3844 return __vector_bitcast<_Up>(_V{_UpUInt(-__x[0]), _UpUInt(-__x[1])});
3845 }
3846 else if constexpr (!__have_avx2 && __have_avx && sizeof(_V) == 32)
3847 {
3848 if constexpr (sizeof(_Up) == 4)
3849 return __vector_bitcast<_Up>(_mm256_cmp_ps(
3850 _mm256_and_ps(_mm256_castsi256_ps(_mm256_set1_epi32(__k)),
3851 _mm256_castsi256_ps(_mm256_setr_epi32(
3852 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80))),
3853 _mm256_setzero_ps(), _CMP_NEQ_UQ));
3854 else if constexpr (sizeof(_Up) == 8)
3855 return __vector_bitcast<_Up>(_mm256_cmp_pd(
3856 _mm256_and_pd(_mm256_castsi256_pd(_mm256_set1_epi64x(__k)),
3857 _mm256_castsi256_pd(
3858 _mm256_setr_epi64x(0x01, 0x02, 0x04, 0x08))),
3859 _mm256_setzero_pd(), _CMP_NEQ_UQ));
3860 else
3861 __assert_unreachable<_Up>();
3862 }
3863 else if constexpr (__bits_per_element >= _ToN)
3864 {
3865 constexpr auto __bitmask
3866 = __generate_vector<_V>([](auto __i)
3867 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA -> _UpUInt
3868 { return __i < _ToN ? 1ull << __i : 0; });
3869 const auto __bits
3870 = __vector_broadcast<_ToN, _UpUInt>(__k) & __bitmask;
3871 if constexpr (__bits_per_element > _ToN)
3872 return __vector_bitcast<_Up>(__bits) > 0;
3873 else
3874 return __vector_bitcast<_Up>(__bits != 0);
3875 }
3876 else
3877 {
3878 const _V __tmp
3879 = __generate_vector<_V>([&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3880 return static_cast<_UpUInt>(
3881 __k >> (__bits_per_element * (__i / __bits_per_element)));
3882 })
3883 & __generate_vector<_V>([](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3884 return static_cast<_UpUInt>(1ull
3885 << (__i % __bits_per_element));
3886 }); // mask bit index
3887 return __intrin_bitcast<_UV>(__tmp != _V());
3888 }
3889 }
3890
3891 // }}}
3892 // _S_to_maskvector(_SimdWrapper) {{{
3893 template <typename _Up, size_t _UpN = 0, typename _Tp, size_t _Np,
3894 size_t _ToN = _UpN == 0 ? _Np : _UpN>
3895 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Up, _ToN>
3896 _S_to_maskvector(_SimdWrapper<_Tp, _Np> __x)
3897 {
3898 static_assert(is_same_v<_Up, __int_for_sizeof_t<_Up>>);
3899 using _TW = _SimdWrapper<_Tp, _Np>;
3900 using _UW = _SimdWrapper<_Up, _ToN>;
3901 using _UI = __intrinsic_type_t<_Up, _ToN>;
3902 if constexpr (is_same_v<_Tp, bool>) // bits -> vector
3903 return _S_to_maskvector<_Up, _ToN>(
3904 _BitMask<_Np>(__x._M_data)._M_sanitized());
3905 // vector -> vector bitcast
3906 else if constexpr (sizeof(_Up) == sizeof(_Tp)
3907 && sizeof(_TW) == sizeof(_UW))
3908 return __wrapper_bitcast<_Up, _ToN>(
3909 _ToN <= _Np
3910 ? __x
3911 : simd_abi::_VecBuiltin<sizeof(_Tp) * _Np>::_S_masked(__x));
3912 else // vector -> vector {{{
3913 {
3914 if (__x._M_is_constprop() || __builtin_is_constant_evaluated())
3915 {
3916 const auto __y = __vector_bitcast<__int_for_sizeof_t<_Tp>>(__x);
3917 return __generate_from_n_evaluations<std::min(_ToN, _Np),
3918 __vector_type_t<_Up, _ToN>>(
3919 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA -> _Up { return __y[__i.value]; });
3920 }
3921 using _To = __vector_type_t<_Up, _ToN>;
3922 [[maybe_unused]] constexpr size_t _FromN = _Np;
3923 constexpr int _FromBytes = sizeof(_Tp);
3924 constexpr int _ToBytes = sizeof(_Up);
3925 const auto __k = __x._M_data;
3926
3927 if constexpr (_FromBytes == _ToBytes)
3928 return __intrin_bitcast<_To>(__k);
3929 else if constexpr (sizeof(_UI) == 16 && sizeof(__k) == 16)
3930 { // SSE -> SSE {{{
3931 if constexpr (_FromBytes == 4 && _ToBytes == 8)
3932 return __intrin_bitcast<_To>(__interleave128_lo(__k, __k));
3933 else if constexpr (_FromBytes == 2 && _ToBytes == 8)
3934 {
3935 const auto __y
3936 = __vector_bitcast<int>(__interleave128_lo(__k, __k));
3937 return __intrin_bitcast<_To>(__interleave128_lo(__y, __y));
3938 }
3939 else if constexpr (_FromBytes == 1 && _ToBytes == 8)
3940 {
3941 auto __y
3942 = __vector_bitcast<short>(__interleave128_lo(__k, __k));
3943 auto __z
3944 = __vector_bitcast<int>(__interleave128_lo(__y, __y));
3945 return __intrin_bitcast<_To>(__interleave128_lo(__z, __z));
3946 }
3947 else if constexpr (_FromBytes == 8 && _ToBytes == 4
3948 && __have_sse2)
3949 return __intrin_bitcast<_To>(
3950 _mm_packs_epi32(__vector_bitcast<_LLong>(__k), __m128i()));
3951 else if constexpr (_FromBytes == 8 && _ToBytes == 4)
3952 return __vector_shuffle<1, 3, 6, 7>(__vector_bitcast<_Up>(__k),
3953 _UI());
3954 else if constexpr (_FromBytes == 2 && _ToBytes == 4)
3955 return __intrin_bitcast<_To>(__interleave128_lo(__k, __k));
3956 else if constexpr (_FromBytes == 1 && _ToBytes == 4)
3957 {
3958 const auto __y
3959 = __vector_bitcast<short>(__interleave128_lo(__k, __k));
3960 return __intrin_bitcast<_To>(__interleave128_lo(__y, __y));
3961 }
3962 else if constexpr (_FromBytes == 8 && _ToBytes == 2)
3963 {
3964 if constexpr (__have_sse2 && !__have_ssse3)
3965 return __intrin_bitcast<_To>(_mm_packs_epi32(
3966 _mm_packs_epi32(__vector_bitcast<_LLong>(__k), __m128i()),
3967 __m128i()));
3968 else
3969 return __intrin_bitcast<_To>(
3970 __vector_permute<3, 7, -1, -1, -1, -1, -1, -1>(
3971 __vector_bitcast<_Up>(__k)));
3972 }
3973 else if constexpr (_FromBytes == 4 && _ToBytes == 2)
3974 return __intrin_bitcast<_To>(
3975 _mm_packs_epi32(__vector_bitcast<_LLong>(__k), __m128i()));
3976 else if constexpr (_FromBytes == 1 && _ToBytes == 2)
3977 return __intrin_bitcast<_To>(__interleave128_lo(__k, __k));
3978 else if constexpr (_FromBytes == 8 && _ToBytes == 1
3979 && __have_ssse3)
3980 return __intrin_bitcast<_To>(
3981 _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
3982 _mm_setr_epi8(7, 15, -1, -1, -1, -1, -1, -1,
3983 -1, -1, -1, -1, -1, -1, -1,
3984 -1)));
3985 else if constexpr (_FromBytes == 8 && _ToBytes == 1)
3986 {
3987 auto __y
3988 = _mm_packs_epi32(__vector_bitcast<_LLong>(__k), __m128i());
3989 __y = _mm_packs_epi32(__y, __m128i());
3990 return __intrin_bitcast<_To>(_mm_packs_epi16(__y, __m128i()));
3991 }
3992 else if constexpr (_FromBytes == 4 && _ToBytes == 1
3993 && __have_ssse3)
3994 return __intrin_bitcast<_To>(
3995 _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
3996 _mm_setr_epi8(3, 7, 11, 15, -1, -1, -1, -1,
3997 -1, -1, -1, -1, -1, -1, -1,
3998 -1)));
3999 else if constexpr (_FromBytes == 4 && _ToBytes == 1)
4000 {
4001 const auto __y
4002 = _mm_packs_epi32(__vector_bitcast<_LLong>(__k), __m128i());
4003 return __intrin_bitcast<_To>(_mm_packs_epi16(__y, __m128i()));
4004 }
4005 else if constexpr (_FromBytes == 2 && _ToBytes == 1)
4006 return __intrin_bitcast<_To>(
4007 _mm_packs_epi16(__vector_bitcast<_LLong>(__k), __m128i()));
4008 else
4009 __assert_unreachable<_Tp>();
4010 } // }}}
4011 else if constexpr (sizeof(_UI) == 32 && sizeof(__k) == 32)
4012 { // AVX -> AVX {{{
4013 if constexpr (_FromBytes == _ToBytes)
4014 __assert_unreachable<_Tp>();
4015 else if constexpr (_FromBytes == _ToBytes * 2)
4016 {
4017 const auto __y = __vector_bitcast<_LLong>(__k);
4018 return __intrin_bitcast<_To>(_mm256_castsi128_si256(
4019 _mm_packs_epi16(__lo128(__y), __hi128(__y))));
4020 }
4021 else if constexpr (_FromBytes == _ToBytes * 4)
4022 {
4023 const auto __y = __vector_bitcast<_LLong>(__k);
4024 return __intrin_bitcast<_To>(_mm256_castsi128_si256(
4025 _mm_packs_epi16(_mm_packs_epi16(__lo128(__y), __hi128(__y)),
4026 __m128i())));
4027 }
4028 else if constexpr (_FromBytes == _ToBytes * 8)
4029 {
4030 const auto __y = __vector_bitcast<_LLong>(__k);
4031 return __intrin_bitcast<_To>(
4032 _mm256_castsi128_si256(_mm_shuffle_epi8(
4033 _mm_packs_epi16(__lo128(__y), __hi128(__y)),
4034 _mm_setr_epi8(3, 7, 11, 15, -1, -1, -1, -1, -1, -1, -1,
4035 -1, -1, -1, -1, -1))));
4036 }
4037 else if constexpr (_FromBytes * 2 == _ToBytes)
4038 {
4039 auto __y = __xzyw(__to_intrin(__k));
4040 if constexpr (is_floating_point_v<
4041 _Tp> || (!__have_avx2 && _FromBytes == 4))
4042 {
4043 const auto __yy = __vector_bitcast<float>(__y);
4044 return __intrin_bitcast<_To>(
4045 _mm256_unpacklo_ps(__yy, __yy));
4046 }
4047 else
4048 return __intrin_bitcast<_To>(
4049 _mm256_unpacklo_epi8(__y, __y));
4050 }
4051 else if constexpr (_FromBytes * 4 == _ToBytes)
4052 {
4053 auto __y
4054 = _mm_unpacklo_epi8(__lo128(__vector_bitcast<_LLong>(__k)),
4055 __lo128(__vector_bitcast<_LLong>(
4056 __k))); // drops 3/4 of input
4057 return __intrin_bitcast<_To>(
4058 __concat(_mm_unpacklo_epi16(__y, __y),
4059 _mm_unpackhi_epi16(__y, __y)));
4060 }
4061 else if constexpr (_FromBytes == 1 && _ToBytes == 8)
4062 {
4063 auto __y
4064 = _mm_unpacklo_epi8(__lo128(__vector_bitcast<_LLong>(__k)),
4065 __lo128(__vector_bitcast<_LLong>(
4066 __k))); // drops 3/4 of input
4067 __y
4068 = _mm_unpacklo_epi16(__y,
4069 __y); // drops another 1/2 => 7/8 total
4070 return __intrin_bitcast<_To>(
4071 __concat(_mm_unpacklo_epi32(__y, __y),
4072 _mm_unpackhi_epi32(__y, __y)));
4073 }
4074 else
4075 __assert_unreachable<_Tp>();
4076 } // }}}
4077 else if constexpr (sizeof(_UI) == 32 && sizeof(__k) == 16)
4078 { // SSE -> AVX {{{
4079 if constexpr (_FromBytes == _ToBytes)
4080 return __intrin_bitcast<_To>(
4081 __intrinsic_type_t<_Tp, 32 / sizeof(_Tp)>(
4082 __zero_extend(__to_intrin(__k))));
4083 else if constexpr (_FromBytes * 2 == _ToBytes)
4084 { // keep all
4085 return __intrin_bitcast<_To>(
4086 __concat(_mm_unpacklo_epi8(__vector_bitcast<_LLong>(__k),
4087 __vector_bitcast<_LLong>(__k)),
4088 _mm_unpackhi_epi8(__vector_bitcast<_LLong>(__k),
4089 __vector_bitcast<_LLong>(__k))));
4090 }
4091 else if constexpr (_FromBytes * 4 == _ToBytes)
4092 {
4093 if constexpr (__have_avx2)
4094 {
4095 return __intrin_bitcast<_To>(_mm256_shuffle_epi8(
4096 __concat(__vector_bitcast<_LLong>(__k),
4097 __vector_bitcast<_LLong>(__k)),
4098 _mm256_setr_epi8(0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3,
4099 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6,
4100 6, 6, 7, 7, 7, 7)));
4101 }
4102 else
4103 {
4104 return __intrin_bitcast<_To>(__concat(
4105 _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
4106 _mm_setr_epi8(0, 0, 0, 0, 1, 1, 1, 1,
4107 2, 2, 2, 2, 3, 3, 3, 3)),
4108 _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
4109 _mm_setr_epi8(4, 4, 4, 4, 5, 5, 5, 5,
4110 6, 6, 6, 6, 7, 7, 7,
4111 7))));
4112 }
4113 }
4114 else if constexpr (_FromBytes * 8 == _ToBytes)
4115 {
4116 if constexpr (__have_avx2)
4117 {
4118 return __intrin_bitcast<_To>(_mm256_shuffle_epi8(
4119 __concat(__vector_bitcast<_LLong>(__k),
4120 __vector_bitcast<_LLong>(__k)),
4121 _mm256_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1,
4122 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3,
4123 3, 3, 3, 3, 3, 3)));
4124 }
4125 else
4126 {
4127 return __intrin_bitcast<_To>(__concat(
4128 _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
4129 _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0,
4130 1, 1, 1, 1, 1, 1, 1, 1)),
4131 _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
4132 _mm_setr_epi8(2, 2, 2, 2, 2, 2, 2, 2,
4133 3, 3, 3, 3, 3, 3, 3,
4134 3))));
4135 }
4136 }
4137 else if constexpr (_FromBytes == _ToBytes * 2)
4138 return __intrin_bitcast<_To>(__m256i(__zero_extend(
4139 _mm_packs_epi16(__vector_bitcast<_LLong>(__k), __m128i()))));
4140 else if constexpr (_FromBytes == 8 && _ToBytes == 2)
4141 {
4142 return __intrin_bitcast<_To>(__m256i(__zero_extend(
4143 _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
4144 _mm_setr_epi8(6, 7, 14, 15, -1, -1, -1, -1,
4145 -1, -1, -1, -1, -1, -1, -1,
4146 -1)))));
4147 }
4148 else if constexpr (_FromBytes == 4 && _ToBytes == 1)
4149 {
4150 return __intrin_bitcast<_To>(__m256i(__zero_extend(
4151 _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
4152 _mm_setr_epi8(3, 7, 11, 15, -1, -1, -1, -1,
4153 -1, -1, -1, -1, -1, -1, -1,
4154 -1)))));
4155 }
4156 else if constexpr (_FromBytes == 8 && _ToBytes == 1)
4157 {
4158 return __intrin_bitcast<_To>(__m256i(__zero_extend(
4159 _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
4160 _mm_setr_epi8(7, 15, -1, -1, -1, -1, -1,
4161 -1, -1, -1, -1, -1, -1, -1,
4162 -1, -1)))));
4163 }
4164 else
4165 static_assert(!is_same_v<_Tp, _Tp>, "should be unreachable");
4166 } // }}}
4167 else if constexpr (sizeof(_UI) == 16 && sizeof(__k) == 32)
4168 { // AVX -> SSE {{{
4169 if constexpr (_FromBytes == _ToBytes)
4170 { // keep low 1/2
4171 return __intrin_bitcast<_To>(__lo128(__k));
4172 }
4173 else if constexpr (_FromBytes == _ToBytes * 2)
4174 { // keep all
4175 auto __y = __vector_bitcast<_LLong>(__k);
4176 return __intrin_bitcast<_To>(
4177 _mm_packs_epi16(__lo128(__y), __hi128(__y)));
4178 }
4179 else if constexpr (_FromBytes == _ToBytes * 4)
4180 { // add 1/2 undef
4181 auto __y = __vector_bitcast<_LLong>(__k);
4182 return __intrin_bitcast<_To>(
4183 _mm_packs_epi16(_mm_packs_epi16(__lo128(__y), __hi128(__y)),
4184 __m128i()));
4185 }
4186 else if constexpr (_FromBytes == 8 && _ToBytes == 1)
4187 { // add 3/4 undef
4188 auto __y = __vector_bitcast<_LLong>(__k);
4189 return __intrin_bitcast<_To>(_mm_shuffle_epi8(
4190 _mm_packs_epi16(__lo128(__y), __hi128(__y)),
4191 _mm_setr_epi8(3, 7, 11, 15, -1, -1, -1, -1, -1, -1, -1, -1,
4192 -1, -1, -1, -1)));
4193 }
4194 else if constexpr (_FromBytes * 2 == _ToBytes)
4195 { // keep low 1/4
4196 auto __y = __lo128(__vector_bitcast<_LLong>(__k));
4197 return __intrin_bitcast<_To>(_mm_unpacklo_epi8(__y, __y));
4198 }
4199 else if constexpr (_FromBytes * 4 == _ToBytes)
4200 { // keep low 1/8
4201 auto __y = __lo128(__vector_bitcast<_LLong>(__k));
4202 __y = _mm_unpacklo_epi8(__y, __y);
4203 return __intrin_bitcast<_To>(_mm_unpacklo_epi8(__y, __y));
4204 }
4205 else if constexpr (_FromBytes * 8 == _ToBytes)
4206 { // keep low 1/16
4207 auto __y = __lo128(__vector_bitcast<_LLong>(__k));
4208 __y = _mm_unpacklo_epi8(__y, __y);
4209 __y = _mm_unpacklo_epi8(__y, __y);
4210 return __intrin_bitcast<_To>(_mm_unpacklo_epi8(__y, __y));
4211 }
4212 else
4213 static_assert(!is_same_v<_Tp, _Tp>, "should be unreachable");
4214 } // }}}
4215 else
4216 return _Base::template _S_to_maskvector<_Up, _ToN>(__x);
4217 /*
4218 if constexpr (_FromBytes > _ToBytes) {
4219 const _To __y = __vector_bitcast<_Up>(__k);
4220 return [&] <size_t... _Is> (index_sequence<_Is...>) {
4221 constexpr int _Stride = _FromBytes / _ToBytes;
4222 return _To{__y[(_Is + 1) * _Stride - 1]...};
4223 }(make_index_sequence<std::min(_ToN, _FromN)>());
4224 } else {
4225 // {0, 0, 1, 1} (_Dups = 2, _Is<4>)
4226 // {0, 0, 0, 0, 1, 1, 1, 1} (_Dups = 4, _Is<8>)
4227 // {0, 0, 1, 1, 2, 2, 3, 3} (_Dups = 2, _Is<8>)
4228 // ...
4229 return [&] <size_t... _Is> (index_sequence<_Is...>) {
4230 constexpr int __dup = _ToBytes / _FromBytes;
4231 return __intrin_bitcast<_To>(_From{__k[_Is / __dup]...});
4232 }(make_index_sequence<_FromN>());
4233 }
4234 */
4235 } // }}}
4236 }
4237
4238 // }}}
4239 // _S_to_bits {{{
4240 template <typename _Tp, size_t _Np>
4241 _GLIBCXX_SIMD_INTRINSIC static constexpr _SanitizedBitMask<_Np>
4242 _S_to_bits(_SimdWrapper<_Tp, _Np> __x)
4243 {
4244 if constexpr (is_same_v<_Tp, bool>)
4245 return _BitMask<_Np>(__x._M_data)._M_sanitized();
4246 else
4247 {
4248 static_assert(is_same_v<_Tp, __int_for_sizeof_t<_Tp>>);
4249 if (__builtin_is_constant_evaluated()
4250 || __builtin_constant_p(__x._M_data))
4251 {
4252 const auto __bools = -__x._M_data;
4253 const _ULLong __k = __call_with_n_evaluations<_Np>(
4254 [](auto... __bits) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4255 return (__bits | ...);
4256 }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4257 return _ULLong(__bools[+__i]) << __i;
4258 });
4259 if (__builtin_is_constant_evaluated()
4260 || __builtin_constant_p(__k))
4261 return __k;
4262 }
4263 const auto __xi = __to_intrin(__x);
4264 if constexpr (sizeof(_Tp) == 1)
4265 if constexpr (sizeof(__xi) == 16)
4266 if constexpr (__have_avx512bw_vl)
4267 return _BitMask<_Np>(_mm_movepi8_mask(__xi));
4268 else // implies SSE2
4269 return _BitMask<_Np>(_mm_movemask_epi8(__xi));
4270 else if constexpr (sizeof(__xi) == 32)
4271 if constexpr (__have_avx512bw_vl)
4272 return _BitMask<_Np>(_mm256_movepi8_mask(__xi));
4273 else // implies AVX2
4274 return _BitMask<_Np>(_mm256_movemask_epi8(__xi));
4275 else // implies AVX512BW
4276 return _BitMask<_Np>(_mm512_movepi8_mask(__xi));
4277
4278 else if constexpr (sizeof(_Tp) == 2)
4279 if constexpr (sizeof(__xi) == 16)
4280 if constexpr (__have_avx512bw_vl)
4281 return _BitMask<_Np>(_mm_movepi16_mask(__xi));
4282 else if constexpr (__have_avx512bw)
4283 return _BitMask<_Np>(_mm512_movepi16_mask(__zero_extend(__xi)));
4284 else // implies SSE2
4285 return _BitMask<_Np>(
4286 _mm_movemask_epi8(_mm_packs_epi16(__xi, __m128i())));
4287 else if constexpr (sizeof(__xi) == 32)
4288 if constexpr (__have_avx512bw_vl)
4289 return _BitMask<_Np>(_mm256_movepi16_mask(__xi));
4290 else if constexpr (__have_avx512bw)
4291 return _BitMask<_Np>(_mm512_movepi16_mask(__zero_extend(__xi)));
4292 else // implies SSE2
4293 return _BitMask<_Np>(_mm_movemask_epi8(
4294 _mm_packs_epi16(__lo128(__xi), __hi128(__xi))));
4295 else // implies AVX512BW
4296 return _BitMask<_Np>(_mm512_movepi16_mask(__xi));
4297
4298 else if constexpr (sizeof(_Tp) == 4)
4299 if constexpr (sizeof(__xi) == 16)
4300 if constexpr (__have_avx512dq_vl)
4301 return _BitMask<_Np>(_mm_movepi32_mask(__xi));
4302 else if constexpr (__have_avx512vl)
4303 return _BitMask<_Np>(_mm_cmplt_epi32_mask(__xi, __m128i()));
4304 else if constexpr (__have_avx512dq)
4305 return _BitMask<_Np>(_mm512_movepi32_mask(__zero_extend(__xi)));
4306 else if constexpr (__have_avx512f)
4307 return _BitMask<_Np>(
4308 _mm512_cmplt_epi32_mask(__zero_extend(__xi), __m512i()));
4309 else // implies SSE
4310 return _BitMask<_Np>(
4311 _mm_movemask_ps(reinterpret_cast<__m128>(__xi)));
4312 else if constexpr (sizeof(__xi) == 32)
4313 if constexpr (__have_avx512dq_vl)
4314 return _BitMask<_Np>(_mm256_movepi32_mask(__xi));
4315 else if constexpr (__have_avx512dq)
4316 return _BitMask<_Np>(_mm512_movepi32_mask(__zero_extend(__xi)));
4317 else if constexpr (__have_avx512vl)
4318 return _BitMask<_Np>(_mm256_cmplt_epi32_mask(__xi, __m256i()));
4319 else if constexpr (__have_avx512f)
4320 return _BitMask<_Np>(
4321 _mm512_cmplt_epi32_mask(__zero_extend(__xi), __m512i()));
4322 else // implies AVX
4323 return _BitMask<_Np>(
4324 _mm256_movemask_ps(reinterpret_cast<__m256>(__xi)));
4325 else // implies AVX512??
4326 if constexpr (__have_avx512dq)
4327 return _BitMask<_Np>(_mm512_movepi32_mask(__xi));
4328 else // implies AVX512F
4329 return _BitMask<_Np>(_mm512_cmplt_epi32_mask(__xi, __m512i()));
4330
4331 else if constexpr (sizeof(_Tp) == 8)
4332 if constexpr (sizeof(__xi) == 16)
4333 if constexpr (__have_avx512dq_vl)
4334 return _BitMask<_Np>(_mm_movepi64_mask(__xi));
4335 else if constexpr (__have_avx512dq)
4336 return _BitMask<_Np>(_mm512_movepi64_mask(__zero_extend(__xi)));
4337 else if constexpr (__have_avx512vl)
4338 return _BitMask<_Np>(_mm_cmplt_epi64_mask(__xi, __m128i()));
4339 else if constexpr (__have_avx512f)
4340 return _BitMask<_Np>(
4341 _mm512_cmplt_epi64_mask(__zero_extend(__xi), __m512i()));
4342 else // implies SSE2
4343 return _BitMask<_Np>(
4344 _mm_movemask_pd(reinterpret_cast<__m128d>(__xi)));
4345 else if constexpr (sizeof(__xi) == 32)
4346 if constexpr (__have_avx512dq_vl)
4347 return _BitMask<_Np>(_mm256_movepi64_mask(__xi));
4348 else if constexpr (__have_avx512dq)
4349 return _BitMask<_Np>(_mm512_movepi64_mask(__zero_extend(__xi)));
4350 else if constexpr (__have_avx512vl)
4351 return _BitMask<_Np>(_mm256_cmplt_epi64_mask(__xi, __m256i()));
4352 else if constexpr (__have_avx512f)
4353 return _BitMask<_Np>(
4354 _mm512_cmplt_epi64_mask(__zero_extend(__xi), __m512i()));
4355 else // implies AVX
4356 return _BitMask<_Np>(
4357 _mm256_movemask_pd(reinterpret_cast<__m256d>(__xi)));
4358 else // implies AVX512??
4359 if constexpr (__have_avx512dq)
4360 return _BitMask<_Np>(_mm512_movepi64_mask(__xi));
4361 else // implies AVX512F
4362 return _BitMask<_Np>(_mm512_cmplt_epi64_mask(__xi, __m512i()));
4363
4364 else
4365 __assert_unreachable<_Tp>();
4366 }
4367 }
4368 // }}}
4369};
4370
4371// }}}
4372// _MaskImplX86 {{{
4373template <typename _Abi, typename>
4374 struct _MaskImplX86 : _MaskImplX86Mixin, _MaskImplBuiltin<_Abi>
4375 {
4376 using _MaskImplX86Mixin::_S_to_bits;
4377 using _MaskImplX86Mixin::_S_to_maskvector;
4378 using _MaskImplBuiltin<_Abi>::_S_convert;
4379
4380 // member types {{{
4381 template <typename _Tp>
4382 using _SimdMember = typename _Abi::template __traits<_Tp>::_SimdMember;
4383
4384 template <typename _Tp>
4385 using _MaskMember = typename _Abi::template _MaskMember<_Tp>;
4386
4387 template <typename _Tp>
4388 static constexpr size_t _S_size = simd_size_v<_Tp, _Abi>;
4389
4390 using _Base = _MaskImplBuiltin<_Abi>;
4391
4392 // }}}
4393 // _S_broadcast {{{
4394 template <typename _Tp>
4395 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
4396 _S_broadcast(bool __x)
4397 {
4398 if constexpr (__is_avx512_abi<_Abi>())
4399 return __x ? _Abi::_S_masked(_MaskMember<_Tp>(-1))
4400 : _MaskMember<_Tp>();
4401 else
4402 return _Base::template _S_broadcast<_Tp>(__x);
4403 }
4404
4405 // }}}
4406 // _S_load {{{
4407 template <typename _Tp>
4408 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
4409 _S_load(const bool* __mem)
4410 {
4411 static_assert(is_same_v<_Tp, __int_for_sizeof_t<_Tp>>);
4412 if constexpr (__have_avx512bw)
4413 {
4414 const auto __to_vec_or_bits
4415 = [](auto __bits) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA -> decltype(auto) {
4416 if constexpr (__is_avx512_abi<_Abi>())
4417 return __bits;
4418 else
4419 return _S_to_maskvector<_Tp>(
4420 _BitMask<_S_size<_Tp>>(__bits)._M_sanitized());
4421 };
4422
4423 if constexpr (_S_size<_Tp> <= 16 && __have_avx512vl)
4424 {
4425 __m128i __a = {};
4426 __builtin_memcpy(&__a, __mem, _S_size<_Tp>);
4427 return __to_vec_or_bits(_mm_test_epi8_mask(__a, __a));
4428 }
4429 else if constexpr (_S_size<_Tp> <= 32 && __have_avx512vl)
4430 {
4431 __m256i __a = {};
4432 __builtin_memcpy(&__a, __mem, _S_size<_Tp>);
4433 return __to_vec_or_bits(_mm256_test_epi8_mask(__a, __a));
4434 }
4435 else if constexpr (_S_size<_Tp> <= 64)
4436 {
4437 __m512i __a = {};
4438 __builtin_memcpy(&__a, __mem, _S_size<_Tp>);
4439 return __to_vec_or_bits(_mm512_test_epi8_mask(__a, __a));
4440 }
4441 }
4442 else if constexpr (__is_avx512_abi<_Abi>())
4443 {
4444 if constexpr (_S_size<_Tp> <= 8)
4445 {
4446 __m128i __a = {};
4447 __builtin_memcpy(&__a, __mem, _S_size<_Tp>);
4448 const auto __b = _mm512_cvtepi8_epi64(__a);
4449 return _mm512_test_epi64_mask(__b, __b);
4450 }
4451 else if constexpr (_S_size<_Tp> <= 16)
4452 {
4453 __m128i __a = {};
4454 __builtin_memcpy(&__a, __mem, _S_size<_Tp>);
4455 const auto __b = _mm512_cvtepi8_epi32(__a);
4456 return _mm512_test_epi32_mask(__b, __b);
4457 }
4458 else if constexpr (_S_size<_Tp> <= 32)
4459 {
4460 __m128i __a = {};
4461 __builtin_memcpy(&__a, __mem, 16);
4462 const auto __b = _mm512_cvtepi8_epi32(__a);
4463 __builtin_memcpy(&__a, __mem + 16, _S_size<_Tp> - 16);
4464 const auto __c = _mm512_cvtepi8_epi32(__a);
4465 return _mm512_test_epi32_mask(__b, __b)
4466 | (_mm512_test_epi32_mask(__c, __c) << 16);
4467 }
4468 else if constexpr (_S_size<_Tp> <= 64)
4469 {
4470 __m128i __a = {};
4471 __builtin_memcpy(&__a, __mem, 16);
4472 const auto __b = _mm512_cvtepi8_epi32(__a);
4473 __builtin_memcpy(&__a, __mem + 16, 16);
4474 const auto __c = _mm512_cvtepi8_epi32(__a);
4475 if constexpr (_S_size<_Tp> <= 48)
4476 {
4477 __builtin_memcpy(&__a, __mem + 32, _S_size<_Tp> - 32);
4478 const auto __d = _mm512_cvtepi8_epi32(__a);
4479 return _mm512_test_epi32_mask(__b, __b)
4480 | (_mm512_test_epi32_mask(__c, __c) << 16)
4481 | (_ULLong(_mm512_test_epi32_mask(__d, __d)) << 32);
4482 }
4483 else
4484 {
4485 __builtin_memcpy(&__a, __mem + 16, 16);
4486 const auto __d = _mm512_cvtepi8_epi32(__a);
4487 __builtin_memcpy(&__a, __mem + 32, _S_size<_Tp> - 48);
4488 const auto __e = _mm512_cvtepi8_epi32(__a);
4489 return _mm512_test_epi32_mask(__b, __b)
4490 | (_mm512_test_epi32_mask(__c, __c) << 16)
4491 | (_ULLong(_mm512_test_epi32_mask(__d, __d)) << 32)
4492 | (_ULLong(_mm512_test_epi32_mask(__e, __e)) << 48);
4493 }
4494 }
4495 else
4496 __assert_unreachable<_Tp>();
4497 }
4498 else if constexpr (sizeof(_Tp) == 8 && _S_size<_Tp> == 2)
4499 return __vector_bitcast<_Tp>(
4500 __vector_type16_t<int>{-int(__mem[0]), -int(__mem[0]),
4501 -int(__mem[1]), -int(__mem[1])});
4502 else if constexpr (sizeof(_Tp) == 8 && _S_size<_Tp> <= 4 && __have_avx)
4503 {
4504 int __bool4 = 0;
4505 __builtin_memcpy(&__bool4, __mem, _S_size<_Tp>);
4506 const auto __k = __to_intrin(
4507 (__vector_broadcast<4>(__bool4)
4508 & __make_vector<int>(0x1, 0x100, 0x10000,
4509 _S_size<_Tp> == 4 ? 0x1000000 : 0))
4510 != 0);
4511 return __vector_bitcast<_Tp>(
4512 __concat(_mm_unpacklo_epi32(__k, __k),
4513 _mm_unpackhi_epi32(__k, __k)));
4514 }
4515 else if constexpr (sizeof(_Tp) == 4 && _S_size<_Tp> <= 4)
4516 {
4517 int __bools = 0;
4518 __builtin_memcpy(&__bools, __mem, _S_size<_Tp>);
4519 if constexpr (__have_sse2)
4520 {
4521 __m128i __k = _mm_cvtsi32_si128(__bools);
4522 __k = _mm_cmpgt_epi16(_mm_unpacklo_epi8(__k, __k), __m128i());
4523 return __vector_bitcast<_Tp, _S_size<_Tp>>(
4524 _mm_unpacklo_epi16(__k, __k));
4525 }
4526 else
4527 {
4528 __m128 __k = _mm_cvtpi8_ps(_mm_cvtsi32_si64(__bools));
4529 _mm_empty();
4530 return __vector_bitcast<_Tp, _S_size<_Tp>>(
4531 _mm_cmpgt_ps(__k, __m128()));
4532 }
4533 }
4534 else if constexpr (sizeof(_Tp) == 4 && _S_size<_Tp> <= 8)
4535 {
4536 __m128i __k = {};
4537 __builtin_memcpy(&__k, __mem, _S_size<_Tp>);
4538 __k = _mm_cmpgt_epi16(_mm_unpacklo_epi8(__k, __k), __m128i());
4539 return __vector_bitcast<_Tp>(
4540 __concat(_mm_unpacklo_epi16(__k, __k),
4541 _mm_unpackhi_epi16(__k, __k)));
4542 }
4543 else if constexpr (sizeof(_Tp) == 2 && _S_size<_Tp> <= 16)
4544 {
4545 __m128i __k = {};
4546 __builtin_memcpy(&__k, __mem, _S_size<_Tp>);
4547 __k = _mm_cmpgt_epi8(__k, __m128i());
4548 if constexpr (_S_size<_Tp> <= 8)
4549 return __vector_bitcast<_Tp, _S_size<_Tp>>(
4550 _mm_unpacklo_epi8(__k, __k));
4551 else
4552 return __concat(_mm_unpacklo_epi8(__k, __k),
4553 _mm_unpackhi_epi8(__k, __k));
4554 }
4555 else
4556 return _Base::template _S_load<_Tp>(__mem);
4557 }
4558
4559 // }}}
4560 // _S_from_bitmask{{{
4561 template <size_t _Np, typename _Tp>
4562 _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
4563 _S_from_bitmask(_SanitizedBitMask<_Np> __bits, _TypeTag<_Tp>)
4564 {
4565 static_assert(is_same_v<_Tp, __int_for_sizeof_t<_Tp>>);
4566 if constexpr (__is_avx512_abi<_Abi>())
4567 return __bits._M_to_bits();
4568 else
4569 return _S_to_maskvector<_Tp, _S_size<_Tp>>(__bits);
4570 }
4571
4572 // }}}
4573 // _S_masked_load {{{2
4574 template <typename _Tp, size_t _Np>
4575 static inline _SimdWrapper<_Tp, _Np>
4576 _S_masked_load(_SimdWrapper<_Tp, _Np> __merge,
4577 _SimdWrapper<_Tp, _Np> __mask, const bool* __mem) noexcept
4578 {
4579 if constexpr (__is_avx512_abi<_Abi>())
4580 {
4581 if constexpr (__have_avx512bw_vl)
4582 {
4583 if constexpr (_Np <= 16)
4584 {
4585 const auto __a
4586 = _mm_mask_loadu_epi8(__m128i(), __mask, __mem);
4587 return (__merge & ~__mask) | _mm_test_epi8_mask(__a, __a);
4588 }
4589 else if constexpr (_Np <= 32)
4590 {
4591 const auto __a
4592 = _mm256_mask_loadu_epi8(__m256i(), __mask, __mem);
4593 return (__merge & ~__mask)
4594 | _mm256_test_epi8_mask(__a, __a);
4595 }
4596 else if constexpr (_Np <= 64)
4597 {
4598 const auto __a
4599 = _mm512_mask_loadu_epi8(__m512i(), __mask, __mem);
4600 return (__merge & ~__mask)
4601 | _mm512_test_epi8_mask(__a, __a);
4602 }
4603 else
4604 __assert_unreachable<_Tp>();
4605 }
4606 else
4607 {
4608 _BitOps::_S_bit_iteration(__mask, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4609 __merge._M_set(__i, __mem[__i]);
4610 });
4611 return __merge;
4612 }
4613 }
4614 else if constexpr (__have_avx512bw_vl && _Np == 32 && sizeof(_Tp) == 1)
4615 {
4616 const auto __k = _S_to_bits(__mask)._M_to_bits();
4617 __merge = _mm256_mask_sub_epi8(__to_intrin(__merge), __k, __m256i(),
4618 _mm256_mask_loadu_epi8(__m256i(),
4619 __k, __mem));
4620 }
4621 else if constexpr (__have_avx512bw_vl && _Np == 16 && sizeof(_Tp) == 1)
4622 {
4623 const auto __k = _S_to_bits(__mask)._M_to_bits();
4624 __merge
4625 = _mm_mask_sub_epi8(__vector_bitcast<_LLong>(__merge), __k,
4626 __m128i(),
4627 _mm_mask_loadu_epi8(__m128i(), __k, __mem));
4628 }
4629 else if constexpr (__have_avx512bw_vl && _Np == 16 && sizeof(_Tp) == 2)
4630 {
4631 const auto __k = _S_to_bits(__mask)._M_to_bits();
4632 __merge = _mm256_mask_sub_epi16(
4633 __vector_bitcast<_LLong>(__merge), __k, __m256i(),
4634 _mm256_cvtepi8_epi16(_mm_mask_loadu_epi8(__m128i(), __k, __mem)));
4635 }
4636 else if constexpr (__have_avx512bw_vl && _Np == 8 && sizeof(_Tp) == 2)
4637 {
4638 const auto __k = _S_to_bits(__mask)._M_to_bits();
4639 __merge = _mm_mask_sub_epi16(
4640 __vector_bitcast<_LLong>(__merge), __k, __m128i(),
4641 _mm_cvtepi8_epi16(_mm_mask_loadu_epi8(__m128i(), __k, __mem)));
4642 }
4643 else if constexpr (__have_avx512bw_vl && _Np == 8 && sizeof(_Tp) == 4)
4644 {
4645 const auto __k = _S_to_bits(__mask)._M_to_bits();
4646 __merge = __vector_bitcast<_Tp>(_mm256_mask_sub_epi32(
4647 __vector_bitcast<_LLong>(__merge), __k, __m256i(),
4648 _mm256_cvtepi8_epi32(
4649 _mm_mask_loadu_epi8(__m128i(), __k, __mem))));
4650 }
4651 else if constexpr (__have_avx512bw_vl && _Np == 4 && sizeof(_Tp) == 4)
4652 {
4653 const auto __k = _S_to_bits(__mask)._M_to_bits();
4654 __merge = __vector_bitcast<_Tp>(_mm_mask_sub_epi32(
4655 __vector_bitcast<_LLong>(__merge), __k, __m128i(),
4656 _mm_cvtepi8_epi32(_mm_mask_loadu_epi8(__m128i(), __k, __mem))));
4657 }
4658 else if constexpr (__have_avx512bw_vl && _Np == 4 && sizeof(_Tp) == 8)
4659 {
4660 const auto __k = _S_to_bits(__mask)._M_to_bits();
4661 __merge = __vector_bitcast<_Tp>(_mm256_mask_sub_epi64(
4662 __vector_bitcast<_LLong>(__merge), __k, __m256i(),
4663 _mm256_cvtepi8_epi64(
4664 _mm_mask_loadu_epi8(__m128i(), __k, __mem))));
4665 }
4666 else if constexpr (__have_avx512bw_vl && _Np == 2 && sizeof(_Tp) == 8)
4667 {
4668 const auto __k = _S_to_bits(__mask)._M_to_bits();
4669 __merge = __vector_bitcast<_Tp>(_mm_mask_sub_epi64(
4670 __vector_bitcast<_LLong>(__merge), __k, __m128i(),
4671 _mm_cvtepi8_epi64(_mm_mask_loadu_epi8(__m128i(), __k, __mem))));
4672 }
4673 else
4674 return _Base::_S_masked_load(__merge, __mask, __mem);
4675 return __merge;
4676 }
4677
4678 // _S_store {{{2
4679 template <typename _Tp, size_t _Np>
4680 _GLIBCXX_SIMD_INTRINSIC static void
4681 _S_store(_SimdWrapper<_Tp, _Np> __v, bool* __mem) noexcept
4682 {
4683 if constexpr (__is_avx512_abi<_Abi>())
4684 {
4685 if constexpr (__have_avx512bw_vl)
4686 _CommonImplX86::_S_store<_Np>(
4687 __vector_bitcast<char>([](auto __data) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4688 if constexpr (_Np <= 16)
4689 return _mm_maskz_set1_epi8(__data, 1);
4690 else if constexpr (_Np <= 32)
4691 return _mm256_maskz_set1_epi8(__data, 1);
4692 else
4693 return _mm512_maskz_set1_epi8(__data, 1);
4694 }(__v._M_data)),
4695 __mem);
4696 else if constexpr (_Np <= 8)
4697 _CommonImplX86::_S_store<_Np>(
4698 __vector_bitcast<char>(
4699#if defined __x86_64__
4700 __make_wrapper<_ULLong>(
4701 _pdep_u64(__v._M_data, 0x0101010101010101ULL), 0ull)
4702#else
4703 __make_wrapper<_UInt>(_pdep_u32(__v._M_data, 0x01010101U),
4704 _pdep_u32(__v._M_data >> 4,
4705 0x01010101U))
4706#endif
4707 ),
4708 __mem);
4709 else if constexpr (_Np <= 16)
4710 _mm512_mask_cvtepi32_storeu_epi8(
4711 __mem, 0xffffu >> (16 - _Np),
4712 _mm512_maskz_set1_epi32(__v._M_data, 1));
4713 else
4714 __assert_unreachable<_Tp>();
4715 }
4716 else if constexpr (__is_sse_abi<_Abi>()) //{{{
4717 {
4718 if constexpr (_Np == 2 && sizeof(_Tp) == 8)
4719 {
4720 const auto __k = __vector_bitcast<int>(__v);
4721 __mem[0] = -__k[1];
4722 __mem[1] = -__k[3];
4723 }
4724 else if constexpr (_Np <= 4 && sizeof(_Tp) == 4)
4725 {
4726 if constexpr (__have_sse2)
4727 {
4728 const unsigned __bool4
4729 = __vector_bitcast<_UInt>(_mm_packs_epi16(
4730 _mm_packs_epi32(__intrin_bitcast<__m128i>(
4731 __to_intrin(__v)),
4732 __m128i()),
4733 __m128i()))[0]
4734 & 0x01010101u;
4735 __builtin_memcpy(__mem, &__bool4, _Np);
4736 }
4737 else if constexpr (__have_mmx)
4738 {
4739 const __m64 __k = _mm_cvtps_pi8(
4740 __and(__to_intrin(__v), _mm_set1_ps(1.f)));
4741 __builtin_memcpy(__mem, &__k, _Np);
4742 _mm_empty();
4743 }
4744 else
4745 return _Base::_S_store(__v, __mem);
4746 }
4747 else if constexpr (_Np <= 8 && sizeof(_Tp) == 2)
4748 {
4749 _CommonImplX86::_S_store<_Np>(
4750 __vector_bitcast<char>(_mm_packs_epi16(
4751 __to_intrin(__vector_bitcast<_UShort>(__v) >> 15),
4752 __m128i())),
4753 __mem);
4754 }
4755 else if constexpr (_Np <= 16 && sizeof(_Tp) == 1)
4756 _CommonImplX86::_S_store<_Np>(__v._M_data & 1, __mem);
4757 else
4758 __assert_unreachable<_Tp>();
4759 } // }}}
4760 else if constexpr (__is_avx_abi<_Abi>()) // {{{
4761 {
4762 if constexpr (_Np <= 4 && sizeof(_Tp) == 8)
4763 {
4764 auto __k = __intrin_bitcast<__m256i>(__to_intrin(__v));
4765 int __bool4;
4766 if constexpr (__have_avx2)
4767 __bool4 = _mm256_movemask_epi8(__k);
4768 else
4769 __bool4 = (_mm_movemask_epi8(__lo128(__k))
4770 | (_mm_movemask_epi8(__hi128(__k)) << 16));
4771 __bool4 &= 0x01010101;
4772 __builtin_memcpy(__mem, &__bool4, _Np);
4773 }
4774 else if constexpr (_Np <= 8 && sizeof(_Tp) == 4)
4775 {
4776 const auto __k = __intrin_bitcast<__m256i>(__to_intrin(__v));
4777 const auto __k2
4778 = _mm_srli_epi16(_mm_packs_epi16(__lo128(__k), __hi128(__k)),
4779 15);
4780 const auto __k3
4781 = __vector_bitcast<char>(_mm_packs_epi16(__k2, __m128i()));
4782 _CommonImplX86::_S_store<_Np>(__k3, __mem);
4783 }
4784 else if constexpr (_Np <= 16 && sizeof(_Tp) == 2)
4785 {
4786 if constexpr (__have_avx2)
4787 {
4788 const auto __x = _mm256_srli_epi16(__to_intrin(__v), 15);
4789 const auto __bools = __vector_bitcast<char>(
4790 _mm_packs_epi16(__lo128(__x), __hi128(__x)));
4791 _CommonImplX86::_S_store<_Np>(__bools, __mem);
4792 }
4793 else
4794 {
4795 const auto __bools
4796 = 1
4797 & __vector_bitcast<_UChar>(
4798 _mm_packs_epi16(__lo128(__to_intrin(__v)),
4799 __hi128(__to_intrin(__v))));
4800 _CommonImplX86::_S_store<_Np>(__bools, __mem);
4801 }
4802 }
4803 else if constexpr (_Np <= 32 && sizeof(_Tp) == 1)
4804 _CommonImplX86::_S_store<_Np>(1 & __v._M_data, __mem);
4805 else
4806 __assert_unreachable<_Tp>();
4807 } // }}}
4808 else
4809 __assert_unreachable<_Tp>();
4810 }
4811
4812 // _S_masked_store {{{2
4813 template <typename _Tp, size_t _Np>
4814 static inline void
4815 _S_masked_store(const _SimdWrapper<_Tp, _Np> __v, bool* __mem,
4816 const _SimdWrapper<_Tp, _Np> __k) noexcept
4817 {
4818 if constexpr (__is_avx512_abi<_Abi>())
4819 {
4820 static_assert(is_same_v<_Tp, bool>);
4821 if constexpr (_Np <= 16 && __have_avx512bw_vl)
4822 _mm_mask_storeu_epi8(__mem, __k, _mm_maskz_set1_epi8(__v, 1));
4823 else if constexpr (_Np <= 16)
4824 _mm512_mask_cvtepi32_storeu_epi8(__mem, __k,
4825 _mm512_maskz_set1_epi32(__v, 1));
4826 else if constexpr (_Np <= 32 && __have_avx512bw_vl)
4827 _mm256_mask_storeu_epi8(__mem, __k,
4828 _mm256_maskz_set1_epi8(__v, 1));
4829 else if constexpr (_Np <= 32 && __have_avx512bw)
4830 _mm256_mask_storeu_epi8(__mem, __k,
4831 __lo256(_mm512_maskz_set1_epi8(__v, 1)));
4832 else if constexpr (_Np <= 64 && __have_avx512bw)
4833 _mm512_mask_storeu_epi8(__mem, __k,
4834 _mm512_maskz_set1_epi8(__v, 1));
4835 else
4836 __assert_unreachable<_Tp>();
4837 }
4838 else
4839 _Base::_S_masked_store(__v, __mem, __k);
4840 }
4841
4842 // logical and bitwise operators {{{2
4843 template <typename _Tp, size_t _Np>
4844 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
4845 _S_logical_and(const _SimdWrapper<_Tp, _Np>& __x, const _SimdWrapper<_Tp, _Np>& __y)
4846 {
4847 if constexpr (is_same_v<_Tp, bool>)
4848 {
4849 if constexpr (__have_avx512dq && _Np <= 8)
4850 return _kand_mask8(__x._M_data, __y._M_data);
4851 else if constexpr (_Np <= 16)
4852 return _kand_mask16(__x._M_data, __y._M_data);
4853 else if constexpr (__have_avx512bw && _Np <= 32)
4854 return _kand_mask32(__x._M_data, __y._M_data);
4855 else if constexpr (__have_avx512bw && _Np <= 64)
4856 return _kand_mask64(__x._M_data, __y._M_data);
4857 else
4858 __assert_unreachable<_Tp>();
4859 }
4860 else
4861 return _Base::_S_logical_and(__x, __y);
4862 }
4863
4864 template <typename _Tp, size_t _Np>
4865 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
4866 _S_logical_or(const _SimdWrapper<_Tp, _Np>& __x, const _SimdWrapper<_Tp, _Np>& __y)
4867 {
4868 if constexpr (is_same_v<_Tp, bool>)
4869 {
4870 if constexpr (__have_avx512dq && _Np <= 8)
4871 return _kor_mask8(__x._M_data, __y._M_data);
4872 else if constexpr (_Np <= 16)
4873 return _kor_mask16(__x._M_data, __y._M_data);
4874 else if constexpr (__have_avx512bw && _Np <= 32)
4875 return _kor_mask32(__x._M_data, __y._M_data);
4876 else if constexpr (__have_avx512bw && _Np <= 64)
4877 return _kor_mask64(__x._M_data, __y._M_data);
4878 else
4879 __assert_unreachable<_Tp>();
4880 }
4881 else
4882 return _Base::_S_logical_or(__x, __y);
4883 }
4884
4885 template <typename _Tp, size_t _Np>
4886 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
4887 _S_bit_not(const _SimdWrapper<_Tp, _Np>& __x)
4888 {
4889 if constexpr (is_same_v<_Tp, bool>)
4890 {
4891 if constexpr (__have_avx512dq && _Np <= 8)
4892 return _kandn_mask8(__x._M_data,
4893 _Abi::template __implicit_mask_n<_Np>());
4894 else if constexpr (_Np <= 16)
4895 return _kandn_mask16(__x._M_data,
4896 _Abi::template __implicit_mask_n<_Np>());
4897 else if constexpr (__have_avx512bw && _Np <= 32)
4898 return _kandn_mask32(__x._M_data,
4899 _Abi::template __implicit_mask_n<_Np>());
4900 else if constexpr (__have_avx512bw && _Np <= 64)
4901 return _kandn_mask64(__x._M_data,
4902 _Abi::template __implicit_mask_n<_Np>());
4903 else
4904 __assert_unreachable<_Tp>();
4905 }
4906 else
4907 return _Base::_S_bit_not(__x);
4908 }
4909
4910 template <typename _Tp, size_t _Np>
4911 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
4912 _S_bit_and(const _SimdWrapper<_Tp, _Np>& __x, const _SimdWrapper<_Tp, _Np>& __y)
4913 {
4914 if constexpr (is_same_v<_Tp, bool>)
4915 {
4916 if constexpr (__have_avx512dq && _Np <= 8)
4917 return _kand_mask8(__x._M_data, __y._M_data);
4918 else if constexpr (_Np <= 16)
4919 return _kand_mask16(__x._M_data, __y._M_data);
4920 else if constexpr (__have_avx512bw && _Np <= 32)
4921 return _kand_mask32(__x._M_data, __y._M_data);
4922 else if constexpr (__have_avx512bw && _Np <= 64)
4923 return _kand_mask64(__x._M_data, __y._M_data);
4924 else
4925 __assert_unreachable<_Tp>();
4926 }
4927 else
4928 return _Base::_S_bit_and(__x, __y);
4929 }
4930
4931 template <typename _Tp, size_t _Np>
4932 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
4933 _S_bit_or(const _SimdWrapper<_Tp, _Np>& __x, const _SimdWrapper<_Tp, _Np>& __y)
4934 {
4935 if constexpr (is_same_v<_Tp, bool>)
4936 {
4937 if constexpr (__have_avx512dq && _Np <= 8)
4938 return _kor_mask8(__x._M_data, __y._M_data);
4939 else if constexpr (_Np <= 16)
4940 return _kor_mask16(__x._M_data, __y._M_data);
4941 else if constexpr (__have_avx512bw && _Np <= 32)
4942 return _kor_mask32(__x._M_data, __y._M_data);
4943 else if constexpr (__have_avx512bw && _Np <= 64)
4944 return _kor_mask64(__x._M_data, __y._M_data);
4945 else
4946 __assert_unreachable<_Tp>();
4947 }
4948 else
4949 return _Base::_S_bit_or(__x, __y);
4950 }
4951
4952 template <typename _Tp, size_t _Np>
4953 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
4954 _S_bit_xor(const _SimdWrapper<_Tp, _Np>& __x, const _SimdWrapper<_Tp, _Np>& __y)
4955 {
4956 if constexpr (is_same_v<_Tp, bool>)
4957 {
4958 if constexpr (__have_avx512dq && _Np <= 8)
4959 return _kxor_mask8(__x._M_data, __y._M_data);
4960 else if constexpr (_Np <= 16)
4961 return _kxor_mask16(__x._M_data, __y._M_data);
4962 else if constexpr (__have_avx512bw && _Np <= 32)
4963 return _kxor_mask32(__x._M_data, __y._M_data);
4964 else if constexpr (__have_avx512bw && _Np <= 64)
4965 return _kxor_mask64(__x._M_data, __y._M_data);
4966 else
4967 __assert_unreachable<_Tp>();
4968 }
4969 else
4970 return _Base::_S_bit_xor(__x, __y);
4971 }
4972
4973 //}}}2
4974 // _S_masked_assign{{{
4975 template <size_t _Np>
4976 _GLIBCXX_SIMD_INTRINSIC static void
4977 _S_masked_assign(_SimdWrapper<bool, _Np> __k,
4978 _SimdWrapper<bool, _Np>& __lhs, _SimdWrapper<bool, _Np> __rhs)
4979 {
4980 __lhs._M_data
4981 = (~__k._M_data & __lhs._M_data) | (__k._M_data & __rhs._M_data);
4982 }
4983
4984 template <size_t _Np>
4985 _GLIBCXX_SIMD_INTRINSIC static void
4986 _S_masked_assign(_SimdWrapper<bool, _Np> __k,
4987 _SimdWrapper<bool, _Np>& __lhs, bool __rhs)
4988 {
4989 if (__rhs)
4990 __lhs._M_data = __k._M_data | __lhs._M_data;
4991 else
4992 __lhs._M_data = ~__k._M_data & __lhs._M_data;
4993 }
4994
4995 using _MaskImplBuiltin<_Abi>::_S_masked_assign;
4996
4997 //}}}
4998 // _S_all_of {{{
4999 template <typename _Tp>
5000 _GLIBCXX_SIMD_INTRINSIC static bool
5001 _S_all_of(simd_mask<_Tp, _Abi> __k)
5002 {
5003 if constexpr (__is_sse_abi<_Abi>() || __is_avx_abi<_Abi>())
5004 {
5005 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
5006 using _TI = __intrinsic_type_t<_Tp, _Np>;
5007 const _TI __a = reinterpret_cast<_TI>(__to_intrin(__data(__k)));
5008 if constexpr (__have_sse4_1)
5009 {
5010 _GLIBCXX_SIMD_USE_CONSTEXPR _TI __b
5011 = _Abi::template _S_implicit_mask_intrin<_Tp>();
5012 return 0 != __testc(__a, __b);
5013 }
5014 else if constexpr (is_same_v<_Tp, float>)
5015 return (_mm_movemask_ps(__a) & ((1 << _Np) - 1))
5016 == (1 << _Np) - 1;
5017 else if constexpr (is_same_v<_Tp, double>)
5018 return (_mm_movemask_pd(__a) & ((1 << _Np) - 1))
5019 == (1 << _Np) - 1;
5020 else
5021 return (_mm_movemask_epi8(__a) & ((1 << (_Np * sizeof(_Tp))) - 1))
5022 == (1 << (_Np * sizeof(_Tp))) - 1;
5023 }
5024 else if constexpr (__is_avx512_abi<_Abi>())
5025 {
5026 constexpr auto _Mask = _Abi::template _S_implicit_mask<_Tp>();
5027 const auto __kk = __k._M_data._M_data;
5028 if constexpr (sizeof(__kk) == 1)
5029 {
5030 if constexpr (__have_avx512dq)
5031 return _kortestc_mask8_u8(__kk, _Mask == 0xff
5032 ? __kk
5033 : __mmask8(~_Mask));
5034 else
5035 return _kortestc_mask16_u8(__kk, __mmask16(~_Mask));
5036 }
5037 else if constexpr (sizeof(__kk) == 2)
5038 return _kortestc_mask16_u8(__kk, _Mask == 0xffff
5039 ? __kk
5040 : __mmask16(~_Mask));
5041 else if constexpr (sizeof(__kk) == 4 && __have_avx512bw)
5042 return _kortestc_mask32_u8(__kk, _Mask == 0xffffffffU
5043 ? __kk
5044 : __mmask32(~_Mask));
5045 else if constexpr (sizeof(__kk) == 8 && __have_avx512bw)
5046 return _kortestc_mask64_u8(__kk, _Mask == 0xffffffffffffffffULL
5047 ? __kk
5048 : __mmask64(~_Mask));
5049 else
5050 __assert_unreachable<_Tp>();
5051 }
5052 }
5053
5054 // }}}
5055 // _S_any_of {{{
5056 template <typename _Tp>
5057 _GLIBCXX_SIMD_INTRINSIC static bool
5058 _S_any_of(simd_mask<_Tp, _Abi> __k)
5059 {
5060 if constexpr (__is_sse_abi<_Abi>() || __is_avx_abi<_Abi>())
5061 {
5062 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
5063 using _TI = __intrinsic_type_t<_Tp, _Np>;
5064 const _TI __a = reinterpret_cast<_TI>(__to_intrin(__data(__k)));
5065 if constexpr (__have_sse4_1)
5066 {
5067 if constexpr (_Abi::template _S_is_partial<
5068 _Tp> || sizeof(__k) < 16)
5069 {
5070 _GLIBCXX_SIMD_USE_CONSTEXPR _TI __b
5071 = _Abi::template _S_implicit_mask_intrin<_Tp>();
5072 return 0 == __testz(__a, __b);
5073 }
5074 else
5075 return 0 == __testz(__a, __a);
5076 }
5077 else if constexpr (is_same_v<_Tp, float>)
5078 return (_mm_movemask_ps(__a) & ((1 << _Np) - 1)) != 0;
5079 else if constexpr (is_same_v<_Tp, double>)
5080 return (_mm_movemask_pd(__a) & ((1 << _Np) - 1)) != 0;
5081 else
5082 return (_mm_movemask_epi8(__a) & ((1 << (_Np * sizeof(_Tp))) - 1))
5083 != 0;
5084 }
5085 else if constexpr (__is_avx512_abi<_Abi>())
5086 return (__k._M_data._M_data & _Abi::template _S_implicit_mask<_Tp>())
5087 != 0;
5088 }
5089
5090 // }}}
5091 // _S_none_of {{{
5092 template <typename _Tp>
5093 _GLIBCXX_SIMD_INTRINSIC static bool
5094 _S_none_of(simd_mask<_Tp, _Abi> __k)
5095 {
5096 if constexpr (__is_sse_abi<_Abi>() || __is_avx_abi<_Abi>())
5097 {
5098 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
5099 using _TI = __intrinsic_type_t<_Tp, _Np>;
5100 const _TI __a = reinterpret_cast<_TI>(__to_intrin(__data(__k)));
5101 if constexpr (__have_sse4_1)
5102 {
5103 if constexpr (_Abi::template _S_is_partial<
5104 _Tp> || sizeof(__k) < 16)
5105 {
5106 _GLIBCXX_SIMD_USE_CONSTEXPR _TI __b
5107 = _Abi::template _S_implicit_mask_intrin<_Tp>();
5108 return 0 != __testz(__a, __b);
5109 }
5110 else
5111 return 0 != __testz(__a, __a);
5112 }
5113 else if constexpr (is_same_v<_Tp, float>)
5114 return (__movemask(__a) & ((1 << _Np) - 1)) == 0;
5115 else if constexpr (is_same_v<_Tp, double>)
5116 return (__movemask(__a) & ((1 << _Np) - 1)) == 0;
5117 else
5118 return (__movemask(__a) & int((1ull << (_Np * sizeof(_Tp))) - 1))
5119 == 0;
5120 }
5121 else if constexpr (__is_avx512_abi<_Abi>())
5122 return (__k._M_data._M_data & _Abi::template _S_implicit_mask<_Tp>())
5123 == 0;
5124 }
5125
5126 // }}}
5127 // _S_some_of {{{
5128 template <typename _Tp>
5129 _GLIBCXX_SIMD_INTRINSIC static bool
5130 _S_some_of(simd_mask<_Tp, _Abi> __k)
5131 {
5132 if constexpr (__is_sse_abi<_Abi>() || __is_avx_abi<_Abi>())
5133 {
5134 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
5135 using _TI = __intrinsic_type_t<_Tp, _Np>;
5136 const _TI __a = reinterpret_cast<_TI>(__to_intrin(__data(__k)));
5137 if constexpr (__have_sse4_1)
5138 {
5139 _GLIBCXX_SIMD_USE_CONSTEXPR _TI __b
5140 = _Abi::template _S_implicit_mask_intrin<_Tp>();
5141 return 0 != __testnzc(__a, __b);
5142 }
5143 else if constexpr (is_same_v<_Tp, float>)
5144 {
5145 constexpr int __allbits = (1 << _Np) - 1;
5146 const auto __tmp = _mm_movemask_ps(__a) & __allbits;
5147 return __tmp > 0 && __tmp < __allbits;
5148 }
5149 else if constexpr (is_same_v<_Tp, double>)
5150 {
5151 constexpr int __allbits = (1 << _Np) - 1;
5152 const auto __tmp = _mm_movemask_pd(__a) & __allbits;
5153 return __tmp > 0 && __tmp < __allbits;
5154 }
5155 else
5156 {
5157 constexpr int __allbits = (1 << (_Np * sizeof(_Tp))) - 1;
5158 const auto __tmp = _mm_movemask_epi8(__a) & __allbits;
5159 return __tmp > 0 && __tmp < __allbits;
5160 }
5161 }
5162 else if constexpr (__is_avx512_abi<_Abi>())
5163 return _S_any_of(__k) && !_S_all_of(__k);
5164 else
5165 __assert_unreachable<_Tp>();
5166 }
5167
5168 // }}}
5169 // _S_popcount {{{
5170 template <typename _Tp>
5171 _GLIBCXX_SIMD_INTRINSIC static int
5172 _S_popcount(simd_mask<_Tp, _Abi> __k)
5173 {
5174 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
5175 const auto __kk = _Abi::_S_masked(__k._M_data)._M_data;
5176 if constexpr (__is_avx512_abi<_Abi>())
5177 {
5178 if constexpr (_Np > 32)
5179 return __builtin_popcountll(__kk);
5180 else
5181 return __builtin_popcount(__kk);
5182 }
5183 else
5184 {
5185 if constexpr (__have_popcnt)
5186 {
5187 int __bits
5188 = __movemask(__to_intrin(__vector_bitcast<_Tp>(__kk)));
5189 const int __count = __builtin_popcount(__bits);
5190 return is_integral_v<_Tp> ? __count / sizeof(_Tp) : __count;
5191 }
5192 else if constexpr (_Np == 2 && sizeof(_Tp) == 8)
5193 {
5194 const int mask = _mm_movemask_pd(__auto_bitcast(__kk));
5195 return mask - (mask >> 1);
5196 }
5197 else if constexpr (_Np <= 4 && sizeof(_Tp) == 8)
5198 {
5199 auto __x = -(__lo128(__kk) + __hi128(__kk));
5200 return __x[0] + __x[1];
5201 }
5202 else if constexpr (_Np <= 4 && sizeof(_Tp) == 4)
5203 {
5204 if constexpr (__have_sse2)
5205 {
5206 __m128i __x = __intrin_bitcast<__m128i>(__to_intrin(__kk));
5207 __x = _mm_add_epi32(
5208 __x, _mm_shuffle_epi32(__x, _MM_SHUFFLE(0, 1, 2, 3)));
5209 __x = _mm_add_epi32(
5210 __x, _mm_shufflelo_epi16(__x, _MM_SHUFFLE(1, 0, 3, 2)));
5211 return -_mm_cvtsi128_si32(__x);
5212 }
5213 else
5214 return __builtin_popcount(
5215 _mm_movemask_ps(__auto_bitcast(__kk)));
5216 }
5217 else if constexpr (_Np <= 8 && sizeof(_Tp) == 2)
5218 {
5219 auto __x = __to_intrin(__kk);
5220 __x = _mm_add_epi16(__x,
5221 _mm_shuffle_epi32(__x,
5222 _MM_SHUFFLE(0, 1, 2, 3)));
5223 __x = _mm_add_epi16(
5224 __x, _mm_shufflelo_epi16(__x, _MM_SHUFFLE(0, 1, 2, 3)));
5225 __x = _mm_add_epi16(
5226 __x, _mm_shufflelo_epi16(__x, _MM_SHUFFLE(2, 3, 0, 1)));
5227 return -short(_mm_extract_epi16(__x, 0));
5228 }
5229 else if constexpr (_Np <= 16 && sizeof(_Tp) == 1)
5230 {
5231 auto __x = __to_intrin(__kk);
5232 __x = _mm_add_epi8(__x,
5233 _mm_shuffle_epi32(__x,
5234 _MM_SHUFFLE(0, 1, 2, 3)));
5235 __x = _mm_add_epi8(__x,
5236 _mm_shufflelo_epi16(__x, _MM_SHUFFLE(0, 1, 2,
5237 3)));
5238 __x = _mm_add_epi8(__x,
5239 _mm_shufflelo_epi16(__x, _MM_SHUFFLE(2, 3, 0,
5240 1)));
5241 auto __y = -__vector_bitcast<_UChar>(__x);
5242 if constexpr (__have_sse4_1)
5243 return __y[0] + __y[1];
5244 else
5245 {
5246 unsigned __z = _mm_extract_epi16(__to_intrin(__y), 0);
5247 return (__z & 0xff) + (__z >> 8);
5248 }
5249 }
5250 else if constexpr (sizeof(__kk) == 32)
5251 {
5252 // The following works only as long as the implementations above
5253 // use a summation
5254 using _I = __int_for_sizeof_t<_Tp>;
5255 const auto __as_int = __vector_bitcast<_I>(__kk);
5256 _MaskImplX86<simd_abi::__sse>::_S_popcount(
5257 simd_mask<_I, simd_abi::__sse>(__private_init,
5258 __lo128(__as_int)
5259 + __hi128(__as_int)));
5260 }
5261 else
5262 __assert_unreachable<_Tp>();
5263 }
5264 }
5265
5266 // }}}
5267 // _S_find_first_set {{{
5268 template <typename _Tp>
5269 _GLIBCXX_SIMD_INTRINSIC static int
5270 _S_find_first_set(simd_mask<_Tp, _Abi> __k)
5271 {
5272 if constexpr (__is_avx512_abi<_Abi>())
5273 return std::__countr_zero(__k._M_data._M_data);
5274 else
5275 return _Base::_S_find_first_set(__k);
5276 }
5277
5278 // }}}
5279 // _S_find_last_set {{{
5280 template <typename _Tp>
5281 _GLIBCXX_SIMD_INTRINSIC static int
5282 _S_find_last_set(simd_mask<_Tp, _Abi> __k)
5283 {
5284 if constexpr (__is_avx512_abi<_Abi>())
5285 return std::__bit_width(__k._M_data._M_data) - 1;
5286 else
5287 return _Base::_S_find_last_set(__k);
5288 }
5289
5290 // }}}
5291 };
5292
5293// }}}
5294
5295_GLIBCXX_SIMD_END_NAMESPACE
5296#endif // __cplusplus >= 201703L
5297#endif // _GLIBCXX_EXPERIMENTAL_SIMD_X86_H_
5298
5299// vim: foldmethod=marker sw=2 noet ts=8 sts=2 tw=80
std::conditional_t
typename conditional< _Cond, _Iftrue, _Iffalse >::type conditional_t
Alias template for conditional.
Definition: type_traits:2614
std::make_signed_t
typename make_signed< _Tp >::type make_signed_t
Alias template for make_signed.
Definition: type_traits:1979
std::min
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
Definition: stl_algobase.h:233