libstdc++
simd_builtin.h
1// Simd Abi specific implementations -*- C++ -*-
2
3// Copyright (C) 2020-2023 Free Software Foundation, Inc.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_ABIS_H_
26#define _GLIBCXX_EXPERIMENTAL_SIMD_ABIS_H_
27
28#if __cplusplus >= 201703L
29
30#include <array>
31#include <cmath>
32#include <cstdlib>
33
34_GLIBCXX_SIMD_BEGIN_NAMESPACE
35// _S_allbits{{{
36template <typename _V>
37 static inline _GLIBCXX_SIMD_USE_CONSTEXPR _V _S_allbits
38 = reinterpret_cast<_V>(~__vector_type_t<char, sizeof(_V) / sizeof(char)>());
39
40// }}}
41// _S_signmask, _S_absmask{{{
42template <typename _V, typename = _VectorTraits<_V>>
43 static inline _GLIBCXX_SIMD_USE_CONSTEXPR _V _S_signmask
44 = __xor(_V() + 1, _V() - 1);
45
46template <typename _V, typename = _VectorTraits<_V>>
47 static inline _GLIBCXX_SIMD_USE_CONSTEXPR _V _S_absmask
48 = __andnot(_S_signmask<_V>, _S_allbits<_V>);
49
50//}}}
51// __vector_permute<Indices...>{{{
52// Index == -1 requests zeroing of the output element
53template <int... _Indices, typename _Tp, typename _TVT = _VectorTraits<_Tp>,
54 typename = __detail::__odr_helper>
55 _Tp
56 __vector_permute(_Tp __x)
57 {
58 static_assert(sizeof...(_Indices) == _TVT::_S_full_size);
59 return __make_vector<typename _TVT::value_type>(
60 (_Indices == -1 ? 0 : __x[_Indices == -1 ? 0 : _Indices])...);
61 }
62
63// }}}
64// __vector_shuffle<Indices...>{{{
65// Index == -1 requests zeroing of the output element
66template <int... _Indices, typename _Tp, typename _TVT = _VectorTraits<_Tp>,
67 typename = __detail::__odr_helper>
68 _Tp
69 __vector_shuffle(_Tp __x, _Tp __y)
70 {
71 return _Tp{(_Indices == -1 ? 0
72 : _Indices < _TVT::_S_full_size
73 ? __x[_Indices]
74 : __y[_Indices - _TVT::_S_full_size])...};
75 }
76
77// }}}
78// __make_wrapper{{{
79template <typename _Tp, typename... _Args>
80 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, sizeof...(_Args)>
81 __make_wrapper(const _Args&... __args)
82 { return __make_vector<_Tp>(__args...); }
83
84// }}}
85// __wrapper_bitcast{{{
86template <typename _Tp, size_t _ToN = 0, typename _Up, size_t _M,
87 size_t _Np = _ToN != 0 ? _ToN : sizeof(_Up) * _M / sizeof(_Tp)>
88 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _Np>
89 __wrapper_bitcast(_SimdWrapper<_Up, _M> __x)
90 {
91 static_assert(_Np > 1);
92 return __intrin_bitcast<__vector_type_t<_Tp, _Np>>(__x._M_data);
93 }
94
95// }}}
96// __shift_elements_right{{{
97// if (__shift % 2ⁿ == 0) => the low n Bytes are correct
98template <unsigned __shift, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
99 _GLIBCXX_SIMD_INTRINSIC _Tp
100 __shift_elements_right(_Tp __v)
101 {
102 [[maybe_unused]] const auto __iv = __to_intrin(__v);
103 static_assert(__shift <= sizeof(_Tp));
104 if constexpr (__shift == 0)
105 return __v;
106 else if constexpr (__shift == sizeof(_Tp))
107 return _Tp();
108#if _GLIBCXX_SIMD_X86INTRIN // {{{
109 else if constexpr (__have_sse && __shift == 8
110 && _TVT::template _S_is<float, 4>)
111 return _mm_movehl_ps(__iv, __iv);
112 else if constexpr (__have_sse2 && __shift == 8
113 && _TVT::template _S_is<double, 2>)
114 return _mm_unpackhi_pd(__iv, __iv);
115 else if constexpr (__have_sse2 && sizeof(_Tp) == 16)
116 return reinterpret_cast<typename _TVT::type>(
117 _mm_srli_si128(reinterpret_cast<__m128i>(__iv), __shift));
118 else if constexpr (__shift == 16 && sizeof(_Tp) == 32)
119 {
120 /*if constexpr (__have_avx && _TVT::template _S_is<double, 4>)
121 return _mm256_permute2f128_pd(__iv, __iv, 0x81);
122 else if constexpr (__have_avx && _TVT::template _S_is<float, 8>)
123 return _mm256_permute2f128_ps(__iv, __iv, 0x81);
124 else if constexpr (__have_avx)
125 return reinterpret_cast<typename _TVT::type>(
126 _mm256_permute2f128_si256(__iv, __iv, 0x81));
127 else*/
128 return __zero_extend(__hi128(__v));
129 }
130 else if constexpr (__have_avx2 && sizeof(_Tp) == 32 && __shift < 16)
131 {
132 const auto __vll = __vector_bitcast<_LLong>(__v);
133 return reinterpret_cast<typename _TVT::type>(
134 _mm256_alignr_epi8(_mm256_permute2x128_si256(__vll, __vll, 0x81),
135 __vll, __shift));
136 }
137 else if constexpr (__have_avx && sizeof(_Tp) == 32 && __shift < 16)
138 {
139 const auto __vll = __vector_bitcast<_LLong>(__v);
140 return reinterpret_cast<typename _TVT::type>(
141 __concat(_mm_alignr_epi8(__hi128(__vll), __lo128(__vll), __shift),
142 _mm_srli_si128(__hi128(__vll), __shift)));
143 }
144 else if constexpr (sizeof(_Tp) == 32 && __shift > 16)
145 return __zero_extend(__shift_elements_right<__shift - 16>(__hi128(__v)));
146 else if constexpr (sizeof(_Tp) == 64 && __shift == 32)
147 return __zero_extend(__hi256(__v));
148 else if constexpr (__have_avx512f && sizeof(_Tp) == 64)
149 {
150 if constexpr (__shift >= 48)
151 return __zero_extend(
152 __shift_elements_right<__shift - 48>(__extract<3, 4>(__v)));
153 else if constexpr (__shift >= 32)
154 return __zero_extend(
155 __shift_elements_right<__shift - 32>(__hi256(__v)));
156 else if constexpr (__shift % 8 == 0)
157 return reinterpret_cast<typename _TVT::type>(
158 _mm512_alignr_epi64(__m512i(), __intrin_bitcast<__m512i>(__v),
159 __shift / 8));
160 else if constexpr (__shift % 4 == 0)
161 return reinterpret_cast<typename _TVT::type>(
162 _mm512_alignr_epi32(__m512i(), __intrin_bitcast<__m512i>(__v),
163 __shift / 4));
164 else if constexpr (__have_avx512bw && __shift < 16)
165 {
166 const auto __vll = __vector_bitcast<_LLong>(__v);
167 return reinterpret_cast<typename _TVT::type>(
168 _mm512_alignr_epi8(_mm512_shuffle_i32x4(__vll, __vll, 0xf9),
169 __vll, __shift));
170 }
171 else if constexpr (__have_avx512bw && __shift < 32)
172 {
173 const auto __vll = __vector_bitcast<_LLong>(__v);
174 return reinterpret_cast<typename _TVT::type>(
175 _mm512_alignr_epi8(_mm512_shuffle_i32x4(__vll, __m512i(), 0xee),
176 _mm512_shuffle_i32x4(__vll, __vll, 0xf9),
177 __shift - 16));
178 }
179 else
180 __assert_unreachable<_Tp>();
181 }
182 /*
183 } else if constexpr (__shift % 16 == 0 && sizeof(_Tp) == 64)
184 return __auto_bitcast(__extract<__shift / 16, 4>(__v));
185 */
186#endif // _GLIBCXX_SIMD_X86INTRIN }}}
187 else
188 {
189 constexpr int __chunksize = __shift % 8 == 0 ? 8
190 : __shift % 4 == 0 ? 4
191 : __shift % 2 == 0 ? 2
192 : 1;
193 auto __w = __vector_bitcast<__int_with_sizeof_t<__chunksize>>(__v);
194 using _Up = decltype(__w);
195 return __intrin_bitcast<_Tp>(
196 __call_with_n_evaluations<(sizeof(_Tp) - __shift) / __chunksize>(
197 [](auto... __chunks) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
198 return _Up{__chunks...};
199 }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
200 return __w[__shift / __chunksize + __i];
201 }));
202 }
203 }
204
205// }}}
206// __extract_part(_SimdWrapper<_Tp, _Np>) {{{
207template <int _Index, int _Total, int _Combine, typename _Tp, size_t _Np>
208 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST
209 _SimdWrapper<_Tp, _Np / _Total * _Combine>
210 __extract_part(const _SimdWrapper<_Tp, _Np> __x)
211 {
212 if constexpr (_Index % 2 == 0 && _Total % 2 == 0 && _Combine % 2 == 0)
213 return __extract_part<_Index / 2, _Total / 2, _Combine / 2>(__x);
214 else
215 {
216 constexpr size_t __values_per_part = _Np / _Total;
217 constexpr size_t __values_to_skip = _Index * __values_per_part;
218 constexpr size_t __return_size = __values_per_part * _Combine;
219 using _R = __vector_type_t<_Tp, __return_size>;
220 static_assert((_Index + _Combine) * __values_per_part * sizeof(_Tp)
221 <= sizeof(__x),
222 "out of bounds __extract_part");
223 // the following assertion would ensure no "padding" to be read
224 // static_assert(_Total >= _Index + _Combine, "_Total must be greater
225 // than _Index");
226
227 // static_assert(__return_size * _Total == _Np, "_Np must be divisible
228 // by _Total");
229 if (__x._M_is_constprop())
230 return __generate_from_n_evaluations<__return_size, _R>(
231 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
232 return __x[__values_to_skip + __i];
233 });
234 if constexpr (_Index == 0 && _Total == 1)
235 return __x;
236 else if constexpr (_Index == 0)
237 return __intrin_bitcast<_R>(__as_vector(__x));
238#if _GLIBCXX_SIMD_X86INTRIN // {{{
239 else if constexpr (sizeof(__x) == 32
240 && __return_size * sizeof(_Tp) <= 16)
241 {
242 constexpr size_t __bytes_to_skip = __values_to_skip * sizeof(_Tp);
243 if constexpr (__bytes_to_skip == 16)
244 return __vector_bitcast<_Tp, __return_size>(
245 __hi128(__as_vector(__x)));
246 else
247 return __vector_bitcast<_Tp, __return_size>(
248 _mm_alignr_epi8(__hi128(__vector_bitcast<_LLong>(__x)),
249 __lo128(__vector_bitcast<_LLong>(__x)),
250 __bytes_to_skip));
251 }
252#endif // _GLIBCXX_SIMD_X86INTRIN }}}
253 else if constexpr (_Index > 0
254 && (__values_to_skip % __return_size != 0
255 || sizeof(_R) >= 8)
256 && (__values_to_skip + __return_size) * sizeof(_Tp)
257 <= 64
258 && sizeof(__x) >= 16)
259 return __intrin_bitcast<_R>(
260 __shift_elements_right<__values_to_skip * sizeof(_Tp)>(
261 __as_vector(__x)));
262 else
263 {
264 _R __r = {};
265 __builtin_memcpy(&__r,
266 reinterpret_cast<const char*>(&__x)
267 + sizeof(_Tp) * __values_to_skip,
268 __return_size * sizeof(_Tp));
269 return __r;
270 }
271 }
272 }
273
274// }}}
275// __extract_part(_SimdWrapper<bool, _Np>) {{{
276template <int _Index, int _Total, int _Combine = 1, size_t _Np>
277 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<bool, _Np / _Total * _Combine>
278 __extract_part(const _SimdWrapper<bool, _Np> __x)
279 {
280 static_assert(_Combine == 1, "_Combine != 1 not implemented");
281 static_assert(__have_avx512f && _Np == _Np);
282 static_assert(_Total >= 2 && _Index + _Combine <= _Total && _Index >= 0);
283 return __x._M_data >> (_Index * _Np / _Total);
284 }
285
286// }}}
287
288// __vector_convert {{{
289// implementation requires an index sequence
290template <typename _To, typename _From, size_t... _I>
291 _GLIBCXX_SIMD_INTRINSIC constexpr _To
292 __vector_convert(_From __a, index_sequence<_I...>)
293 {
294 using _Tp = typename _VectorTraits<_To>::value_type;
295 return _To{static_cast<_Tp>(__a[_I])...};
296 }
297
298template <typename _To, typename _From, size_t... _I>
299 _GLIBCXX_SIMD_INTRINSIC constexpr _To
300 __vector_convert(_From __a, _From __b, index_sequence<_I...>)
301 {
302 using _Tp = typename _VectorTraits<_To>::value_type;
303 return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])...};
304 }
305
306template <typename _To, typename _From, size_t... _I>
307 _GLIBCXX_SIMD_INTRINSIC constexpr _To
308 __vector_convert(_From __a, _From __b, _From __c, index_sequence<_I...>)
309 {
310 using _Tp = typename _VectorTraits<_To>::value_type;
311 return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])...,
312 static_cast<_Tp>(__c[_I])...};
313 }
314
315template <typename _To, typename _From, size_t... _I>
316 _GLIBCXX_SIMD_INTRINSIC constexpr _To
317 __vector_convert(_From __a, _From __b, _From __c, _From __d,
318 index_sequence<_I...>)
319 {
320 using _Tp = typename _VectorTraits<_To>::value_type;
321 return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])...,
322 static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])...};
323 }
324
325template <typename _To, typename _From, size_t... _I>
326 _GLIBCXX_SIMD_INTRINSIC constexpr _To
327 __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e,
328 index_sequence<_I...>)
329 {
330 using _Tp = typename _VectorTraits<_To>::value_type;
331 return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])...,
332 static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])...,
333 static_cast<_Tp>(__e[_I])...};
334 }
335
336template <typename _To, typename _From, size_t... _I>
337 _GLIBCXX_SIMD_INTRINSIC constexpr _To
338 __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e,
339 _From __f, index_sequence<_I...>)
340 {
341 using _Tp = typename _VectorTraits<_To>::value_type;
342 return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])...,
343 static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])...,
344 static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])...};
345 }
346
347template <typename _To, typename _From, size_t... _I>
348 _GLIBCXX_SIMD_INTRINSIC constexpr _To
349 __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e,
350 _From __f, _From __g, index_sequence<_I...>)
351 {
352 using _Tp = typename _VectorTraits<_To>::value_type;
353 return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])...,
354 static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])...,
355 static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])...,
356 static_cast<_Tp>(__g[_I])...};
357 }
358
359template <typename _To, typename _From, size_t... _I>
360 _GLIBCXX_SIMD_INTRINSIC constexpr _To
361 __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e,
362 _From __f, _From __g, _From __h, index_sequence<_I...>)
363 {
364 using _Tp = typename _VectorTraits<_To>::value_type;
365 return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])...,
366 static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])...,
367 static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])...,
368 static_cast<_Tp>(__g[_I])..., static_cast<_Tp>(__h[_I])...};
369 }
370
371template <typename _To, typename _From, size_t... _I>
372 _GLIBCXX_SIMD_INTRINSIC constexpr _To
373 __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e,
374 _From __f, _From __g, _From __h, _From __i,
375 index_sequence<_I...>)
376 {
377 using _Tp = typename _VectorTraits<_To>::value_type;
378 return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])...,
379 static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])...,
380 static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])...,
381 static_cast<_Tp>(__g[_I])..., static_cast<_Tp>(__h[_I])...,
382 static_cast<_Tp>(__i[_I])...};
383 }
384
385template <typename _To, typename _From, size_t... _I>
386 _GLIBCXX_SIMD_INTRINSIC constexpr _To
387 __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e,
388 _From __f, _From __g, _From __h, _From __i, _From __j,
389 index_sequence<_I...>)
390 {
391 using _Tp = typename _VectorTraits<_To>::value_type;
392 return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])...,
393 static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])...,
394 static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])...,
395 static_cast<_Tp>(__g[_I])..., static_cast<_Tp>(__h[_I])...,
396 static_cast<_Tp>(__i[_I])..., static_cast<_Tp>(__j[_I])...};
397 }
398
399template <typename _To, typename _From, size_t... _I>
400 _GLIBCXX_SIMD_INTRINSIC constexpr _To
401 __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e,
402 _From __f, _From __g, _From __h, _From __i, _From __j,
403 _From __k, index_sequence<_I...>)
404 {
405 using _Tp = typename _VectorTraits<_To>::value_type;
406 return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])...,
407 static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])...,
408 static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])...,
409 static_cast<_Tp>(__g[_I])..., static_cast<_Tp>(__h[_I])...,
410 static_cast<_Tp>(__i[_I])..., static_cast<_Tp>(__j[_I])...,
411 static_cast<_Tp>(__k[_I])...};
412 }
413
414template <typename _To, typename _From, size_t... _I>
415 _GLIBCXX_SIMD_INTRINSIC constexpr _To
416 __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e,
417 _From __f, _From __g, _From __h, _From __i, _From __j,
418 _From __k, _From __l, index_sequence<_I...>)
419 {
420 using _Tp = typename _VectorTraits<_To>::value_type;
421 return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])...,
422 static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])...,
423 static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])...,
424 static_cast<_Tp>(__g[_I])..., static_cast<_Tp>(__h[_I])...,
425 static_cast<_Tp>(__i[_I])..., static_cast<_Tp>(__j[_I])...,
426 static_cast<_Tp>(__k[_I])..., static_cast<_Tp>(__l[_I])...};
427 }
428
429template <typename _To, typename _From, size_t... _I>
430 _GLIBCXX_SIMD_INTRINSIC constexpr _To
431 __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e,
432 _From __f, _From __g, _From __h, _From __i, _From __j,
433 _From __k, _From __l, _From __m, index_sequence<_I...>)
434 {
435 using _Tp = typename _VectorTraits<_To>::value_type;
436 return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])...,
437 static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])...,
438 static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])...,
439 static_cast<_Tp>(__g[_I])..., static_cast<_Tp>(__h[_I])...,
440 static_cast<_Tp>(__i[_I])..., static_cast<_Tp>(__j[_I])...,
441 static_cast<_Tp>(__k[_I])..., static_cast<_Tp>(__l[_I])...,
442 static_cast<_Tp>(__m[_I])...};
443 }
444
445template <typename _To, typename _From, size_t... _I>
446 _GLIBCXX_SIMD_INTRINSIC constexpr _To
447 __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e,
448 _From __f, _From __g, _From __h, _From __i, _From __j,
449 _From __k, _From __l, _From __m, _From __n,
450 index_sequence<_I...>)
451 {
452 using _Tp = typename _VectorTraits<_To>::value_type;
453 return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])...,
454 static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])...,
455 static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])...,
456 static_cast<_Tp>(__g[_I])..., static_cast<_Tp>(__h[_I])...,
457 static_cast<_Tp>(__i[_I])..., static_cast<_Tp>(__j[_I])...,
458 static_cast<_Tp>(__k[_I])..., static_cast<_Tp>(__l[_I])...,
459 static_cast<_Tp>(__m[_I])..., static_cast<_Tp>(__n[_I])...};
460 }
461
462template <typename _To, typename _From, size_t... _I>
463 _GLIBCXX_SIMD_INTRINSIC constexpr _To
464 __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e,
465 _From __f, _From __g, _From __h, _From __i, _From __j,
466 _From __k, _From __l, _From __m, _From __n, _From __o,
467 index_sequence<_I...>)
468 {
469 using _Tp = typename _VectorTraits<_To>::value_type;
470 return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])...,
471 static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])...,
472 static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])...,
473 static_cast<_Tp>(__g[_I])..., static_cast<_Tp>(__h[_I])...,
474 static_cast<_Tp>(__i[_I])..., static_cast<_Tp>(__j[_I])...,
475 static_cast<_Tp>(__k[_I])..., static_cast<_Tp>(__l[_I])...,
476 static_cast<_Tp>(__m[_I])..., static_cast<_Tp>(__n[_I])...,
477 static_cast<_Tp>(__o[_I])...};
478 }
479
480template <typename _To, typename _From, size_t... _I>
481 _GLIBCXX_SIMD_INTRINSIC constexpr _To
482 __vector_convert(_From __a, _From __b, _From __c, _From __d, _From __e,
483 _From __f, _From __g, _From __h, _From __i, _From __j,
484 _From __k, _From __l, _From __m, _From __n, _From __o,
485 _From __p, index_sequence<_I...>)
486 {
487 using _Tp = typename _VectorTraits<_To>::value_type;
488 return _To{static_cast<_Tp>(__a[_I])..., static_cast<_Tp>(__b[_I])...,
489 static_cast<_Tp>(__c[_I])..., static_cast<_Tp>(__d[_I])...,
490 static_cast<_Tp>(__e[_I])..., static_cast<_Tp>(__f[_I])...,
491 static_cast<_Tp>(__g[_I])..., static_cast<_Tp>(__h[_I])...,
492 static_cast<_Tp>(__i[_I])..., static_cast<_Tp>(__j[_I])...,
493 static_cast<_Tp>(__k[_I])..., static_cast<_Tp>(__l[_I])...,
494 static_cast<_Tp>(__m[_I])..., static_cast<_Tp>(__n[_I])...,
495 static_cast<_Tp>(__o[_I])..., static_cast<_Tp>(__p[_I])...};
496 }
497
498// Defer actual conversion to the overload that takes an index sequence. Note
499// that this function adds zeros or drops values off the end if you don't ensure
500// matching width.
501template <typename _To, typename... _From, size_t _FromSize>
502 _GLIBCXX_SIMD_INTRINSIC constexpr _To
503 __vector_convert(_SimdWrapper<_From, _FromSize>... __xs)
504 {
505#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
506 using _From0 = __first_of_pack_t<_From...>;
507 using _FW = _SimdWrapper<_From0, _FromSize>;
508 if (!_FW::_S_is_partial && !(... && __xs._M_is_constprop()))
509 {
510 if constexpr ((sizeof...(_From) & (sizeof...(_From) - 1))
511 == 0) // power-of-two number of arguments
512 return __convert_x86<_To>(__as_vector(__xs)...);
513 else // append zeros and recurse until the above branch is taken
514 return __vector_convert<_To>(__xs..., _FW{});
515 }
516 else
517#endif
518 return __vector_convert<_To>(
519 __as_vector(__xs)...,
520 make_index_sequence<(sizeof...(__xs) == 1 ? std::min(
521 _VectorTraits<_To>::_S_full_size, int(_FromSize))
522 : _FromSize)>());
523 }
524
525// }}}
526// __convert function{{{
527template <typename _To, typename _From, typename... _More>
528 _GLIBCXX_SIMD_INTRINSIC constexpr auto
529 __convert(_From __v0, _More... __vs)
530 {
531 static_assert((true && ... && is_same_v<_From, _More>) );
532 if constexpr (__is_vectorizable_v<_From>)
533 {
534 using _V = typename _VectorTraits<_To>::type;
535 using _Tp = typename _VectorTraits<_To>::value_type;
536 return _V{static_cast<_Tp>(__v0), static_cast<_Tp>(__vs)...};
537 }
538 else if constexpr (__is_vector_type_v<_From>)
539 return __convert<_To>(__as_wrapper(__v0), __as_wrapper(__vs)...);
540 else // _SimdWrapper arguments
541 {
542 constexpr size_t __input_size = _From::_S_size * (1 + sizeof...(_More));
543 if constexpr (__is_vectorizable_v<_To>)
544 return __convert<__vector_type_t<_To, __input_size>>(__v0, __vs...);
545 else if constexpr (!__is_vector_type_v<_To>)
546 return _To(__convert<typename _To::_BuiltinType>(__v0, __vs...));
547 else
548 {
549 static_assert(
550 sizeof...(_More) == 0
551 || _VectorTraits<_To>::_S_full_size >= __input_size,
552 "__convert(...) requires the input to fit into the output");
553 return __vector_convert<_To>(__v0, __vs...);
554 }
555 }
556 }
557
558// }}}
559// __convert_all{{{
560// Converts __v into array<_To, N>, where N is _NParts if non-zero or
561// otherwise deduced from _To such that N * #elements(_To) <= #elements(__v).
562// Note: this function may return less than all converted elements
563template <typename _To,
564 size_t _NParts = 0, // allows to convert fewer or more (only last
565 // _To, to be partially filled) than all
566 size_t _Offset = 0, // where to start, # of elements (not Bytes or
567 // Parts)
568 typename _From, typename _FromVT = _VectorTraits<_From>>
569 _GLIBCXX_SIMD_INTRINSIC auto
570 __convert_all(_From __v)
571 {
572 if constexpr (is_arithmetic_v<_To> && _NParts != 1)
573 {
574 static_assert(_Offset < _FromVT::_S_full_size);
575 constexpr auto _Np
576 = _NParts == 0 ? _FromVT::_S_partial_width - _Offset : _NParts;
577 return __generate_from_n_evaluations<_Np, array<_To, _Np>>(
578 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
579 return static_cast<_To>(__v[__i + _Offset]);
580 });
581 }
582 else
583 {
584 static_assert(__is_vector_type_v<_To>);
585 using _ToVT = _VectorTraits<_To>;
586 if constexpr (__is_vector_type_v<_From>)
587 return __convert_all<_To, _NParts>(__as_wrapper(__v));
588 else if constexpr (_NParts == 1)
589 {
590 static_assert(_Offset % _ToVT::_S_full_size == 0);
591 return array<_To, 1>{__vector_convert<_To>(
592 __extract_part<_Offset / _ToVT::_S_full_size,
593 __div_roundup(_FromVT::_S_partial_width,
594 _ToVT::_S_full_size)>(__v))};
595 }
596#if _GLIBCXX_SIMD_X86INTRIN // {{{
597 else if constexpr (!__have_sse4_1 && _Offset == 0
598 && is_integral_v<typename _FromVT::value_type>
599 && sizeof(typename _FromVT::value_type)
600 < sizeof(typename _ToVT::value_type)
601 && !(sizeof(typename _FromVT::value_type) == 4
602 && is_same_v<typename _ToVT::value_type, double>))
603 {
604 using _ToT = typename _ToVT::value_type;
605 using _FromT = typename _FromVT::value_type;
606 constexpr size_t _Np
607 = _NParts != 0
608 ? _NParts
609 : (_FromVT::_S_partial_width / _ToVT::_S_full_size);
610 using _R = array<_To, _Np>;
611 // __adjust modifies its input to have _Np (use _SizeConstant)
612 // entries so that no unnecessary intermediate conversions are
613 // requested and, more importantly, no intermediate conversions are
614 // missing
615 [[maybe_unused]] auto __adjust
616 = [](auto __n,
617 auto __vv) -> _SimdWrapper<_FromT, decltype(__n)::value> {
618 return __vector_bitcast<_FromT, decltype(__n)::value>(__vv);
619 };
620 [[maybe_unused]] const auto __vi = __to_intrin(__v);
621 auto&& __make_array
622 = [](auto __x0, [[maybe_unused]] auto __x1) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
623 if constexpr (_Np == 1)
624 return _R{__intrin_bitcast<_To>(__x0)};
625 else
626 return _R{__intrin_bitcast<_To>(__x0),
627 __intrin_bitcast<_To>(__x1)};
628 };
629
630 if constexpr (_Np == 0)
631 return _R{};
632 else if constexpr (sizeof(_FromT) == 1 && sizeof(_ToT) == 2)
633 {
634 static_assert(is_integral_v<_FromT>);
635 static_assert(is_integral_v<_ToT>);
636 if constexpr (is_unsigned_v<_FromT>)
637 return __make_array(_mm_unpacklo_epi8(__vi, __m128i()),
638 _mm_unpackhi_epi8(__vi, __m128i()));
639 else
640 return __make_array(
641 _mm_srai_epi16(_mm_unpacklo_epi8(__vi, __vi), 8),
642 _mm_srai_epi16(_mm_unpackhi_epi8(__vi, __vi), 8));
643 }
644 else if constexpr (sizeof(_FromT) == 2 && sizeof(_ToT) == 4)
645 {
646 static_assert(is_integral_v<_FromT>);
647 if constexpr (is_floating_point_v<_ToT>)
648 {
649 const auto __ints
650 = __convert_all<__vector_type16_t<int>, _Np>(
651 __adjust(_SizeConstant<_Np * 4>(), __v));
652 return __generate_from_n_evaluations<_Np, _R>(
653 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
654 return __vector_convert<_To>(__as_wrapper(__ints[__i]));
655 });
656 }
657 else if constexpr (is_unsigned_v<_FromT>)
658 return __make_array(_mm_unpacklo_epi16(__vi, __m128i()),
659 _mm_unpackhi_epi16(__vi, __m128i()));
660 else
661 return __make_array(
662 _mm_srai_epi32(_mm_unpacklo_epi16(__vi, __vi), 16),
663 _mm_srai_epi32(_mm_unpackhi_epi16(__vi, __vi), 16));
664 }
665 else if constexpr (sizeof(_FromT) == 4 && sizeof(_ToT) == 8
666 && is_integral_v<_FromT> && is_integral_v<_ToT>)
667 {
668 if constexpr (is_unsigned_v<_FromT>)
669 return __make_array(_mm_unpacklo_epi32(__vi, __m128i()),
670 _mm_unpackhi_epi32(__vi, __m128i()));
671 else
672 return __make_array(
673 _mm_unpacklo_epi32(__vi, _mm_srai_epi32(__vi, 31)),
674 _mm_unpackhi_epi32(__vi, _mm_srai_epi32(__vi, 31)));
675 }
676 else if constexpr (sizeof(_FromT) == 4 && sizeof(_ToT) == 8
677 && is_integral_v<_FromT> && is_integral_v<_ToT>)
678 {
679 if constexpr (is_unsigned_v<_FromT>)
680 return __make_array(_mm_unpacklo_epi32(__vi, __m128i()),
681 _mm_unpackhi_epi32(__vi, __m128i()));
682 else
683 return __make_array(
684 _mm_unpacklo_epi32(__vi, _mm_srai_epi32(__vi, 31)),
685 _mm_unpackhi_epi32(__vi, _mm_srai_epi32(__vi, 31)));
686 }
687 else if constexpr (sizeof(_FromT) == 1 && sizeof(_ToT) >= 4
688 && is_signed_v<_FromT>)
689 {
690 const __m128i __vv[2] = {_mm_unpacklo_epi8(__vi, __vi),
691 _mm_unpackhi_epi8(__vi, __vi)};
692 const __vector_type_t<int, 4> __vvvv[4] = {
693 __vector_bitcast<int>(_mm_unpacklo_epi16(__vv[0], __vv[0])),
694 __vector_bitcast<int>(_mm_unpackhi_epi16(__vv[0], __vv[0])),
695 __vector_bitcast<int>(_mm_unpacklo_epi16(__vv[1], __vv[1])),
696 __vector_bitcast<int>(_mm_unpackhi_epi16(__vv[1], __vv[1]))};
697 if constexpr (sizeof(_ToT) == 4)
698 return __generate_from_n_evaluations<_Np, _R>(
699 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
700 return __vector_convert<_To>(
701 _SimdWrapper<int, 4>(__vvvv[__i] >> 24));
702 });
703 else if constexpr (is_integral_v<_ToT>)
704 return __generate_from_n_evaluations<_Np, _R>(
705 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
706 const auto __signbits = __to_intrin(__vvvv[__i / 2] >> 31);
707 const auto __sx32 = __to_intrin(__vvvv[__i / 2] >> 24);
708 return __vector_bitcast<_ToT>(
709 __i % 2 == 0 ? _mm_unpacklo_epi32(__sx32, __signbits)
710 : _mm_unpackhi_epi32(__sx32, __signbits));
711 });
712 else
713 return __generate_from_n_evaluations<_Np, _R>(
714 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
715 const _SimdWrapper<int, 4> __int4 = __vvvv[__i / 2] >> 24;
716 return __vector_convert<_To>(
717 __i % 2 == 0 ? __int4
718 : _SimdWrapper<int, 4>(
719 _mm_unpackhi_epi64(__to_intrin(__int4),
720 __to_intrin(__int4))));
721 });
722 }
723 else if constexpr (sizeof(_FromT) == 1 && sizeof(_ToT) == 4)
724 {
725 const auto __shorts = __convert_all<__vector_type16_t<
726 conditional_t<is_signed_v<_FromT>, short, unsigned short>>>(
727 __adjust(_SizeConstant<(_Np + 1) / 2 * 8>(), __v));
728 return __generate_from_n_evaluations<_Np, _R>(
729 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
730 return __convert_all<_To>(__shorts[__i / 2])[__i % 2];
731 });
732 }
733 else if constexpr (sizeof(_FromT) == 2 && sizeof(_ToT) == 8
734 && is_signed_v<_FromT> && is_integral_v<_ToT>)
735 {
736 const __m128i __vv[2] = {_mm_unpacklo_epi16(__vi, __vi),
737 _mm_unpackhi_epi16(__vi, __vi)};
738 const __vector_type16_t<int> __vvvv[4]
739 = {__vector_bitcast<int>(
740 _mm_unpacklo_epi32(_mm_srai_epi32(__vv[0], 16),
741 _mm_srai_epi32(__vv[0], 31))),
742 __vector_bitcast<int>(
743 _mm_unpackhi_epi32(_mm_srai_epi32(__vv[0], 16),
744 _mm_srai_epi32(__vv[0], 31))),
745 __vector_bitcast<int>(
746 _mm_unpacklo_epi32(_mm_srai_epi32(__vv[1], 16),
747 _mm_srai_epi32(__vv[1], 31))),
748 __vector_bitcast<int>(
749 _mm_unpackhi_epi32(_mm_srai_epi32(__vv[1], 16),
750 _mm_srai_epi32(__vv[1], 31)))};
751 return __generate_from_n_evaluations<_Np, _R>(
752 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
753 return __vector_bitcast<_ToT>(__vvvv[__i]);
754 });
755 }
756 else if constexpr (sizeof(_FromT) <= 2 && sizeof(_ToT) == 8)
757 {
758 const auto __ints
759 = __convert_all<__vector_type16_t<conditional_t<
760 is_signed_v<_FromT> || is_floating_point_v<_ToT>, int,
761 unsigned int>>>(
762 __adjust(_SizeConstant<(_Np + 1) / 2 * 4>(), __v));
763 return __generate_from_n_evaluations<_Np, _R>(
764 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
765 return __convert_all<_To>(__ints[__i / 2])[__i % 2];
766 });
767 }
768 else
769 __assert_unreachable<_To>();
770 }
771#endif // _GLIBCXX_SIMD_X86INTRIN }}}
772 else if constexpr ((_FromVT::_S_partial_width - _Offset)
773 > _ToVT::_S_full_size)
774 {
775 /*
776 static_assert(
777 (_FromVT::_S_partial_width & (_FromVT::_S_partial_width - 1)) ==
778 0,
779 "__convert_all only supports power-of-2 number of elements.
780 Otherwise " "the return type cannot be array<_To, N>.");
781 */
782 constexpr size_t _NTotal
783 = (_FromVT::_S_partial_width - _Offset) / _ToVT::_S_full_size;
784 constexpr size_t _Np = _NParts == 0 ? _NTotal : _NParts;
785 static_assert(
786 _Np <= _NTotal
787 || (_Np == _NTotal + 1
788 && (_FromVT::_S_partial_width - _Offset) % _ToVT::_S_full_size
789 > 0));
790 using _R = array<_To, _Np>;
791 if constexpr (_Np == 1)
792 return _R{__vector_convert<_To>(
793 __extract_part<_Offset, _FromVT::_S_partial_width,
794 _ToVT::_S_full_size>(__v))};
795 else
796 return __generate_from_n_evaluations<_Np, _R>(
797 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
798 auto __part
799 = __extract_part<__i * _ToVT::_S_full_size + _Offset,
800 _FromVT::_S_partial_width,
801 _ToVT::_S_full_size>(__v);
802 return __vector_convert<_To>(__part);
803 });
804 }
805 else if constexpr (_Offset == 0)
806 return array<_To, 1>{__vector_convert<_To>(__v)};
807 else
808 return array<_To, 1>{__vector_convert<_To>(
809 __extract_part<_Offset, _FromVT::_S_partial_width,
810 _FromVT::_S_partial_width - _Offset>(__v))};
811 }
812 }
813
814// }}}
815
816// _GnuTraits {{{
817template <typename _Tp, typename _Mp, typename _Abi, size_t _Np>
818 struct _GnuTraits
819 {
820 using _IsValid = true_type;
821 using _SimdImpl = typename _Abi::_SimdImpl;
822 using _MaskImpl = typename _Abi::_MaskImpl;
823
824 // simd and simd_mask member types {{{
825 using _SimdMember = _SimdWrapper<_Tp, _Np>;
826 using _MaskMember = _SimdWrapper<_Mp, _Np>;
827 static constexpr size_t _S_simd_align = alignof(_SimdMember);
828 static constexpr size_t _S_mask_align = alignof(_MaskMember);
829
830 // }}}
831 // size metadata {{{
832 static constexpr size_t _S_full_size = _SimdMember::_S_full_size;
833 static constexpr bool _S_is_partial = _SimdMember::_S_is_partial;
834
835 // }}}
836 // _SimdBase / base class for simd, providing extra conversions {{{
837 struct _SimdBase2
838 {
839 _GLIBCXX_SIMD_ALWAYS_INLINE explicit
840 operator __intrinsic_type_t<_Tp, _Np>() const
841 { return __to_intrin(static_cast<const simd<_Tp, _Abi>*>(this)->_M_data); }
842
843 _GLIBCXX_SIMD_ALWAYS_INLINE explicit
844 operator __vector_type_t<_Tp, _Np>() const
845 { return static_cast<const simd<_Tp, _Abi>*>(this)->_M_data.__builtin(); }
846 };
847
848 struct _SimdBase1
849 {
850 _GLIBCXX_SIMD_ALWAYS_INLINE explicit
851 operator __intrinsic_type_t<_Tp, _Np>() const
852 { return __data(*static_cast<const simd<_Tp, _Abi>*>(this)); }
853 };
854
855 using _SimdBase = conditional_t<
856 is_same<__intrinsic_type_t<_Tp, _Np>, __vector_type_t<_Tp, _Np>>::value,
857 _SimdBase1, _SimdBase2>;
858
859 // }}}
860 // _MaskBase {{{
861 struct _MaskBase2
862 {
863 _GLIBCXX_SIMD_ALWAYS_INLINE explicit
864 operator __intrinsic_type_t<_Tp, _Np>() const
865 { return static_cast<const simd_mask<_Tp, _Abi>*>(this) ->_M_data.__intrin(); }
866
867 _GLIBCXX_SIMD_ALWAYS_INLINE explicit
868 operator __vector_type_t<_Tp, _Np>() const
869 { return static_cast<const simd_mask<_Tp, _Abi>*>(this)->_M_data._M_data; }
870 };
871
872 struct _MaskBase1
873 {
874 _GLIBCXX_SIMD_ALWAYS_INLINE explicit
875 operator __intrinsic_type_t<_Tp, _Np>() const
876 { return __data(*static_cast<const simd_mask<_Tp, _Abi>*>(this)); }
877 };
878
879 using _MaskBase = conditional_t<
880 is_same<__intrinsic_type_t<_Tp, _Np>, __vector_type_t<_Tp, _Np>>::value,
881 _MaskBase1, _MaskBase2>;
882
883 // }}}
884 // _MaskCastType {{{
885 // parameter type of one explicit simd_mask constructor
886 class _MaskCastType
887 {
888 using _Up = __intrinsic_type_t<_Tp, _Np>;
889 _Up _M_data;
890
891 public:
892 _GLIBCXX_SIMD_ALWAYS_INLINE
893 _MaskCastType(_Up __x) : _M_data(__x) {}
894
895 _GLIBCXX_SIMD_ALWAYS_INLINE
896 operator _MaskMember() const { return _M_data; }
897 };
898
899 // }}}
900 // _SimdCastType {{{
901 // parameter type of one explicit simd constructor
902 class _SimdCastType1
903 {
904 using _Ap = __intrinsic_type_t<_Tp, _Np>;
905 _SimdMember _M_data;
906
907 public:
908 _GLIBCXX_SIMD_ALWAYS_INLINE
909 _SimdCastType1(_Ap __a) : _M_data(__vector_bitcast<_Tp>(__a)) {}
910
911 _GLIBCXX_SIMD_ALWAYS_INLINE
912 operator _SimdMember() const { return _M_data; }
913 };
914
915 class _SimdCastType2
916 {
917 using _Ap = __intrinsic_type_t<_Tp, _Np>;
918 using _Bp = __vector_type_t<_Tp, _Np>;
919 _SimdMember _M_data;
920
921 public:
922 _GLIBCXX_SIMD_ALWAYS_INLINE
923 _SimdCastType2(_Ap __a) : _M_data(__vector_bitcast<_Tp>(__a)) {}
924
925 _GLIBCXX_SIMD_ALWAYS_INLINE
926 _SimdCastType2(_Bp __b) : _M_data(__b) {}
927
928 _GLIBCXX_SIMD_ALWAYS_INLINE
929 operator _SimdMember() const { return _M_data; }
930 };
931
932 using _SimdCastType = conditional_t<
933 is_same<__intrinsic_type_t<_Tp, _Np>, __vector_type_t<_Tp, _Np>>::value,
934 _SimdCastType1, _SimdCastType2>;
935 //}}}
936 };
937
938// }}}
939struct _CommonImplX86;
940struct _CommonImplNeon;
941struct _CommonImplBuiltin;
942template <typename _Abi, typename = __detail::__odr_helper> struct _SimdImplBuiltin;
943template <typename _Abi, typename = __detail::__odr_helper> struct _MaskImplBuiltin;
944template <typename _Abi, typename = __detail::__odr_helper> struct _SimdImplX86;
945template <typename _Abi, typename = __detail::__odr_helper> struct _MaskImplX86;
946template <typename _Abi, typename = __detail::__odr_helper> struct _SimdImplNeon;
947template <typename _Abi, typename = __detail::__odr_helper> struct _MaskImplNeon;
948template <typename _Abi, typename = __detail::__odr_helper> struct _SimdImplPpc;
949template <typename _Abi, typename = __detail::__odr_helper> struct _MaskImplPpc;
950
951// simd_abi::_VecBuiltin {{{
952template <int _UsedBytes>
953 struct simd_abi::_VecBuiltin
954 {
955 template <typename _Tp>
956 static constexpr size_t _S_size = _UsedBytes / sizeof(_Tp);
957
958 // validity traits {{{
959 struct _IsValidAbiTag : __bool_constant<(_UsedBytes > 1)> {};
960
961 template <typename _Tp>
962 struct _IsValidSizeFor
963 : __bool_constant<(_UsedBytes / sizeof(_Tp) > 1
964 && _UsedBytes % sizeof(_Tp) == 0
965 && _UsedBytes <= __vectorized_sizeof<_Tp>()
966 && (!__have_avx512f || _UsedBytes <= 32))> {};
967
968 template <typename _Tp>
969 struct _IsValid : conjunction<_IsValidAbiTag, __is_vectorizable<_Tp>,
970 _IsValidSizeFor<_Tp>> {};
971
972 template <typename _Tp>
973 static constexpr bool _S_is_valid_v = _IsValid<_Tp>::value;
974
975 // }}}
976 // _SimdImpl/_MaskImpl {{{
977#if _GLIBCXX_SIMD_X86INTRIN
978 using _CommonImpl = _CommonImplX86;
979 using _SimdImpl = _SimdImplX86<_VecBuiltin<_UsedBytes>>;
980 using _MaskImpl = _MaskImplX86<_VecBuiltin<_UsedBytes>>;
981#elif _GLIBCXX_SIMD_HAVE_NEON
982 using _CommonImpl = _CommonImplNeon;
983 using _SimdImpl = _SimdImplNeon<_VecBuiltin<_UsedBytes>>;
984 using _MaskImpl = _MaskImplNeon<_VecBuiltin<_UsedBytes>>;
985#else
986 using _CommonImpl = _CommonImplBuiltin;
987#ifdef __ALTIVEC__
988 using _SimdImpl = _SimdImplPpc<_VecBuiltin<_UsedBytes>>;
989 using _MaskImpl = _MaskImplPpc<_VecBuiltin<_UsedBytes>>;
990#else
991 using _SimdImpl = _SimdImplBuiltin<_VecBuiltin<_UsedBytes>>;
992 using _MaskImpl = _MaskImplBuiltin<_VecBuiltin<_UsedBytes>>;
993#endif
994#endif
995
996 // }}}
997 // __traits {{{
998 template <typename _Tp>
999 using _MaskValueType = __int_for_sizeof_t<_Tp>;
1000
1001 template <typename _Tp>
1002 using __traits
1003 = conditional_t<_S_is_valid_v<_Tp>,
1004 _GnuTraits<_Tp, _MaskValueType<_Tp>,
1005 _VecBuiltin<_UsedBytes>, _S_size<_Tp>>,
1006 _InvalidTraits>;
1007
1008 //}}}
1009 // size metadata {{{
1010 template <typename _Tp>
1011 static constexpr size_t _S_full_size = __traits<_Tp>::_S_full_size;
1012
1013 template <typename _Tp>
1014 static constexpr bool _S_is_partial = __traits<_Tp>::_S_is_partial;
1015
1016 // }}}
1017 // implicit masks {{{
1018 template <typename _Tp>
1019 using _MaskMember = _SimdWrapper<_MaskValueType<_Tp>, _S_size<_Tp>>;
1020
1021 template <typename _Tp>
1022 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
1023 _S_implicit_mask()
1024 {
1025 using _UV = typename _MaskMember<_Tp>::_BuiltinType;
1026 if constexpr (!_MaskMember<_Tp>::_S_is_partial)
1027 return ~_UV();
1028 else
1029 {
1030 constexpr auto __size = _S_size<_Tp>;
1031 _GLIBCXX_SIMD_USE_CONSTEXPR auto __r
1032 = __generate_vector<_UV>([](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
1033 { return __i < __size ? -1 : 0; });
1034 return __r;
1035 }
1036 }
1037
1038 template <typename _Tp>
1039 _GLIBCXX_SIMD_INTRINSIC static constexpr __intrinsic_type_t<_Tp, _S_size<_Tp>>
1040 _S_implicit_mask_intrin()
1041 { return __to_intrin(__vector_bitcast<_Tp>(_S_implicit_mask<_Tp>()._M_data)); }
1042
1043 template <typename _TW, typename _TVT = _VectorTraits<_TW>>
1044 _GLIBCXX_SIMD_INTRINSIC static constexpr _TW
1045 _S_masked(_TW __x)
1046 {
1047 using _Tp = typename _TVT::value_type;
1048 if constexpr (!_MaskMember<_Tp>::_S_is_partial)
1049 return __x;
1050 else
1051 return __and(__as_vector(__x),
1052 __vector_bitcast<_Tp>(_S_implicit_mask<_Tp>()));
1053 }
1054
1055 template <typename _TW, typename _TVT = _VectorTraits<_TW>>
1056 _GLIBCXX_SIMD_INTRINSIC static constexpr auto
1057 __make_padding_nonzero(_TW __x)
1058 {
1059 using _Tp = typename _TVT::value_type;
1060 if constexpr (!_S_is_partial<_Tp>)
1061 return __x;
1062 else
1063 {
1064 _GLIBCXX_SIMD_USE_CONSTEXPR auto __implicit_mask
1065 = __vector_bitcast<_Tp>(_S_implicit_mask<_Tp>());
1066 if constexpr (is_integral_v<_Tp>)
1067 return __or(__x, ~__implicit_mask);
1068 else
1069 {
1070 _GLIBCXX_SIMD_USE_CONSTEXPR auto __one
1071 = __andnot(__implicit_mask,
1072 __vector_broadcast<_S_full_size<_Tp>>(_Tp(1)));
1073 // it's not enough to return `x | 1_in_padding` because the
1074 // padding in x might be inf or nan (independent of
1075 // __FINITE_MATH_ONLY__, because it's about padding bits)
1076 return __or(__and(__x, __implicit_mask), __one);
1077 }
1078 }
1079 }
1080 // }}}
1081 };
1082
1083// }}}
1084// simd_abi::_VecBltnBtmsk {{{
1085template <int _UsedBytes>
1086 struct simd_abi::_VecBltnBtmsk
1087 {
1088 template <typename _Tp>
1089 static constexpr size_t _S_size = _UsedBytes / sizeof(_Tp);
1090
1091 // validity traits {{{
1092 struct _IsValidAbiTag : __bool_constant<(_UsedBytes > 1)> {};
1093
1094 template <typename _Tp>
1095 struct _IsValidSizeFor
1096 : __bool_constant<(_UsedBytes / sizeof(_Tp) > 1
1097 && _UsedBytes % sizeof(_Tp) == 0 && _UsedBytes <= 64
1098 && (_UsedBytes > 32 || __have_avx512vl))> {};
1099
1100 // Bitmasks require at least AVX512F. If sizeof(_Tp) < 4 the AVX512BW is also
1101 // required.
1102 template <typename _Tp>
1103 struct _IsValid
1104 : conjunction<
1105 _IsValidAbiTag, __bool_constant<__have_avx512f>,
1106 __bool_constant<__have_avx512bw || (sizeof(_Tp) >= 4)>,
1107 __bool_constant<(__vectorized_sizeof<_Tp>() > sizeof(_Tp))>,
1108 _IsValidSizeFor<_Tp>> {};
1109
1110 template <typename _Tp>
1111 static constexpr bool _S_is_valid_v = _IsValid<_Tp>::value;
1112
1113 // }}}
1114 // simd/_MaskImpl {{{
1115 #if _GLIBCXX_SIMD_X86INTRIN
1116 using _CommonImpl = _CommonImplX86;
1117 using _SimdImpl = _SimdImplX86<_VecBltnBtmsk<_UsedBytes>>;
1118 using _MaskImpl = _MaskImplX86<_VecBltnBtmsk<_UsedBytes>>;
1119 #else
1120 template <int>
1121 struct _MissingImpl;
1122
1123 using _CommonImpl = _MissingImpl<_UsedBytes>;
1124 using _SimdImpl = _MissingImpl<_UsedBytes>;
1125 using _MaskImpl = _MissingImpl<_UsedBytes>;
1126 #endif
1127
1128 // }}}
1129 // __traits {{{
1130 template <typename _Tp>
1131 using _MaskMember = _SimdWrapper<bool, _S_size<_Tp>>;
1132
1133 template <typename _Tp>
1134 using __traits = conditional_t<
1135 _S_is_valid_v<_Tp>,
1136 _GnuTraits<_Tp, bool, _VecBltnBtmsk<_UsedBytes>, _S_size<_Tp>>,
1137 _InvalidTraits>;
1138
1139 //}}}
1140 // size metadata {{{
1141 template <typename _Tp>
1142 static constexpr size_t _S_full_size = __traits<_Tp>::_S_full_size;
1143 template <typename _Tp>
1144 static constexpr bool _S_is_partial = __traits<_Tp>::_S_is_partial;
1145
1146 // }}}
1147 // implicit mask {{{
1148 private:
1149 template <typename _Tp>
1150 using _ImplicitMask = _SimdWrapper<bool, _S_size<_Tp>>;
1151
1152 public:
1153 template <size_t _Np>
1154 _GLIBCXX_SIMD_INTRINSIC static constexpr __bool_storage_member_type_t<_Np>
1155 __implicit_mask_n()
1156 {
1157 using _Tp = __bool_storage_member_type_t<_Np>;
1158 return _Np < sizeof(_Tp) * __CHAR_BIT__ ? _Tp((1ULL << _Np) - 1) : ~_Tp();
1159 }
1160
1161 template <typename _Tp>
1162 _GLIBCXX_SIMD_INTRINSIC static constexpr _ImplicitMask<_Tp>
1163 _S_implicit_mask()
1164 { return __implicit_mask_n<_S_size<_Tp>>(); }
1165
1166 template <typename _Tp>
1167 _GLIBCXX_SIMD_INTRINSIC static constexpr __bool_storage_member_type_t<_S_size<_Tp>>
1168 _S_implicit_mask_intrin()
1169 { return __implicit_mask_n<_S_size<_Tp>>(); }
1170
1171 template <typename _Tp, size_t _Np>
1172 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
1173 _S_masked(_SimdWrapper<_Tp, _Np> __x)
1174 {
1175 if constexpr (is_same_v<_Tp, bool>)
1176 if constexpr (_Np < 8 || (_Np & (_Np - 1)) != 0)
1177 return _MaskImpl::_S_bit_and(
1178 __x, _SimdWrapper<_Tp, _Np>(
1179 __bool_storage_member_type_t<_Np>((1ULL << _Np) - 1)));
1180 else
1181 return __x;
1182 else
1183 return _S_masked(__x._M_data);
1184 }
1185
1186 template <typename _TV>
1187 _GLIBCXX_SIMD_INTRINSIC static constexpr _TV
1188 _S_masked(_TV __x)
1189 {
1190 using _Tp = typename _VectorTraits<_TV>::value_type;
1191 static_assert(
1192 !__is_bitmask_v<_TV>,
1193 "_VecBltnBtmsk::_S_masked cannot work on bitmasks, since it doesn't "
1194 "know the number of elements. Use _SimdWrapper<bool, N> instead.");
1195 if constexpr (_S_is_partial<_Tp>)
1196 {
1197 constexpr size_t _Np = _S_size<_Tp>;
1198 return __make_dependent_t<_TV, _CommonImpl>::_S_blend(
1199 _S_implicit_mask<_Tp>(), _SimdWrapper<_Tp, _Np>(),
1200 _SimdWrapper<_Tp, _Np>(__x));
1201 }
1202 else
1203 return __x;
1204 }
1205
1206 template <typename _TV, typename _TVT = _VectorTraits<_TV>>
1207 _GLIBCXX_SIMD_INTRINSIC static constexpr auto
1208 __make_padding_nonzero(_TV __x)
1209 {
1210 using _Tp = typename _TVT::value_type;
1211 if constexpr (!_S_is_partial<_Tp>)
1212 return __x;
1213 else
1214 {
1215 constexpr size_t _Np = _S_size<_Tp>;
1216 if constexpr (is_integral_v<typename _TVT::value_type>)
1217 return __x
1218 | __generate_vector<_Tp, _S_full_size<_Tp>>(
1219 [](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA -> _Tp {
1220 if (__i < _Np)
1221 return 0;
1222 else
1223 return 1;
1224 });
1225 else
1226 return __make_dependent_t<_TV, _CommonImpl>::_S_blend(
1227 _S_implicit_mask<_Tp>(),
1228 _SimdWrapper<_Tp, _Np>(
1229 __vector_broadcast<_S_full_size<_Tp>>(_Tp(1))),
1230 _SimdWrapper<_Tp, _Np>(__x))
1231 ._M_data;
1232 }
1233 }
1234
1235 // }}}
1236 };
1237
1238//}}}
1239// _CommonImplBuiltin {{{
1240struct _CommonImplBuiltin
1241{
1242 // _S_converts_via_decomposition{{{
1243 // This lists all cases where a __vector_convert needs to fall back to
1244 // conversion of individual scalars (i.e. decompose the input vector into
1245 // scalars, convert, compose output vector). In those cases, _S_masked_load &
1246 // _S_masked_store prefer to use the _S_bit_iteration implementation.
1247 template <typename _From, typename _To, size_t _ToSize>
1248 static inline constexpr bool __converts_via_decomposition_v
1249 = sizeof(_From) != sizeof(_To);
1250
1251 // }}}
1252 // _S_load{{{
1253 template <typename _Tp, size_t _Np, size_t _Bytes = _Np * sizeof(_Tp)>
1254 _GLIBCXX_SIMD_INTRINSIC static __vector_type_t<_Tp, _Np>
1255 _S_load(const void* __p)
1256 {
1257 static_assert(_Np > 1);
1258 static_assert(_Bytes % sizeof(_Tp) == 0);
1259 using _Rp = __vector_type_t<_Tp, _Np>;
1260 if constexpr (sizeof(_Rp) == _Bytes)
1261 {
1262 _Rp __r;
1263 __builtin_memcpy(&__r, __p, _Bytes);
1264 return __r;
1265 }
1266 else
1267 {
1268#ifdef _GLIBCXX_SIMD_WORKAROUND_PR90424
1269 using _Up = conditional_t<
1270 is_integral_v<_Tp>,
1271 conditional_t<_Bytes % 4 == 0,
1272 conditional_t<_Bytes % 8 == 0, long long, int>,
1273 conditional_t<_Bytes % 2 == 0, short, signed char>>,
1274 conditional_t<(_Bytes < 8 || _Np % 2 == 1 || _Np == 2), _Tp,
1275 double>>;
1276 using _V = __vector_type_t<_Up, _Np * sizeof(_Tp) / sizeof(_Up)>;
1277 if constexpr (sizeof(_V) != sizeof(_Rp))
1278 { // on i386 with 4 < _Bytes <= 8
1279 _Rp __r{};
1280 __builtin_memcpy(&__r, __p, _Bytes);
1281 return __r;
1282 }
1283 else
1284#else // _GLIBCXX_SIMD_WORKAROUND_PR90424
1285 using _V = _Rp;
1286#endif // _GLIBCXX_SIMD_WORKAROUND_PR90424
1287 {
1288 _V __r{};
1289 static_assert(_Bytes <= sizeof(_V));
1290 __builtin_memcpy(&__r, __p, _Bytes);
1291 return reinterpret_cast<_Rp>(__r);
1292 }
1293 }
1294 }
1295
1296 // }}}
1297 // _S_store {{{
1298 template <size_t _ReqBytes = 0, typename _TV>
1299 _GLIBCXX_SIMD_INTRINSIC static void
1300 _S_store(_TV __x, void* __addr)
1301 {
1302 constexpr size_t _Bytes = _ReqBytes == 0 ? sizeof(__x) : _ReqBytes;
1303 static_assert(sizeof(__x) >= _Bytes);
1304
1305 if constexpr (__is_vector_type_v<_TV>)
1306 {
1307 using _Tp = typename _VectorTraits<_TV>::value_type;
1308 constexpr size_t _Np = _Bytes / sizeof(_Tp);
1309 static_assert(_Np * sizeof(_Tp) == _Bytes);
1310
1311#ifdef _GLIBCXX_SIMD_WORKAROUND_PR90424
1312 using _Up = conditional_t<
1313 (is_integral_v<_Tp> || _Bytes < 4),
1314 conditional_t<(sizeof(__x) > sizeof(long long)), long long, _Tp>,
1315 float>;
1316 const auto __v = __vector_bitcast<_Up>(__x);
1317#else // _GLIBCXX_SIMD_WORKAROUND_PR90424
1318 const __vector_type_t<_Tp, _Np> __v = __x;
1319#endif // _GLIBCXX_SIMD_WORKAROUND_PR90424
1320
1321 if constexpr ((_Bytes & (_Bytes - 1)) != 0)
1322 {
1323 constexpr size_t _MoreBytes = std::__bit_ceil(_Bytes);
1324 alignas(decltype(__v)) char __tmp[_MoreBytes];
1325 __builtin_memcpy(__tmp, &__v, _MoreBytes);
1326 __builtin_memcpy(__addr, __tmp, _Bytes);
1327 }
1328 else
1329 __builtin_memcpy(__addr, &__v, _Bytes);
1330 }
1331 else
1332 __builtin_memcpy(__addr, &__x, _Bytes);
1333 }
1334
1335 template <typename _Tp, size_t _Np>
1336 _GLIBCXX_SIMD_INTRINSIC static void
1337 _S_store(_SimdWrapper<_Tp, _Np> __x, void* __addr)
1338 { _S_store<_Np * sizeof(_Tp)>(__x._M_data, __addr); }
1339
1340 // }}}
1341 // _S_store_bool_array(_BitMask) {{{
1342 template <size_t _Np, bool _Sanitized>
1343 _GLIBCXX_SIMD_INTRINSIC static constexpr void
1344 _S_store_bool_array(_BitMask<_Np, _Sanitized> __x, bool* __mem)
1345 {
1346 if constexpr (_Np == 1)
1347 __mem[0] = __x[0];
1348 else if constexpr (_Np == 2)
1349 {
1350 short __bool2 = (__x._M_to_bits() * 0x81) & 0x0101;
1351 _S_store<_Np>(__bool2, __mem);
1352 }
1353 else if constexpr (_Np == 3)
1354 {
1355 int __bool3 = (__x._M_to_bits() * 0x4081) & 0x010101;
1356 _S_store<_Np>(__bool3, __mem);
1357 }
1358 else
1359 {
1360 __execute_n_times<__div_roundup(_Np, 4)>(
1361 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1362 constexpr int __offset = __i * 4;
1363 constexpr int __remaining = _Np - __offset;
1364 if constexpr (__remaining > 4 && __remaining <= 7)
1365 {
1366 const _ULLong __bool7
1367 = (__x.template _M_extract<__offset>()._M_to_bits()
1368 * 0x40810204081ULL)
1369 & 0x0101010101010101ULL;
1370 _S_store<__remaining>(__bool7, __mem + __offset);
1371 }
1372 else if constexpr (__remaining >= 4)
1373 {
1374 int __bits = __x.template _M_extract<__offset>()._M_to_bits();
1375 if constexpr (__remaining > 7)
1376 __bits &= 0xf;
1377 const int __bool4 = (__bits * 0x204081) & 0x01010101;
1378 _S_store<4>(__bool4, __mem + __offset);
1379 }
1380 });
1381 }
1382 }
1383
1384 // }}}
1385 // _S_blend{{{
1386 template <typename _Tp, size_t _Np>
1387 _GLIBCXX_SIMD_INTRINSIC static constexpr auto
1388 _S_blend(_SimdWrapper<__int_for_sizeof_t<_Tp>, _Np> __k,
1389 _SimdWrapper<_Tp, _Np> __at0, _SimdWrapper<_Tp, _Np> __at1)
1390 { return __k._M_data ? __at1._M_data : __at0._M_data; }
1391
1392 // }}}
1393};
1394
1395// }}}
1396// _SimdImplBuiltin {{{1
1397template <typename _Abi, typename>
1398 struct _SimdImplBuiltin
1399 {
1400 // member types {{{2
1401 template <typename _Tp>
1402 static constexpr size_t _S_max_store_size = 16;
1403
1404 using abi_type = _Abi;
1405
1406 template <typename _Tp>
1407 using _TypeTag = _Tp*;
1408
1409 template <typename _Tp>
1410 using _SimdMember = typename _Abi::template __traits<_Tp>::_SimdMember;
1411
1412 template <typename _Tp>
1413 using _MaskMember = typename _Abi::template _MaskMember<_Tp>;
1414
1415 template <typename _Tp>
1416 static constexpr size_t _S_size = _Abi::template _S_size<_Tp>;
1417
1418 template <typename _Tp>
1419 static constexpr size_t _S_full_size = _Abi::template _S_full_size<_Tp>;
1420
1421 using _CommonImpl = typename _Abi::_CommonImpl;
1422 using _SuperImpl = typename _Abi::_SimdImpl;
1423 using _MaskImpl = typename _Abi::_MaskImpl;
1424
1425 // _M_make_simd(_SimdWrapper/__intrinsic_type_t) {{{2
1426 template <typename _Tp, size_t _Np>
1427 _GLIBCXX_SIMD_INTRINSIC static simd<_Tp, _Abi>
1428 _M_make_simd(_SimdWrapper<_Tp, _Np> __x)
1429 { return {__private_init, __x}; }
1430
1431 template <typename _Tp, size_t _Np>
1432 _GLIBCXX_SIMD_INTRINSIC static simd<_Tp, _Abi>
1433 _M_make_simd(__intrinsic_type_t<_Tp, _Np> __x)
1434 { return {__private_init, __vector_bitcast<_Tp>(__x)}; }
1435
1436 // _S_broadcast {{{2
1437 template <typename _Tp>
1438 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdMember<_Tp>
1439 _S_broadcast(_Tp __x) noexcept
1440 { return __vector_broadcast<_S_full_size<_Tp>>(__x); }
1441
1442 // _S_generator {{{2
1443 template <typename _Fp, typename _Tp>
1444 inline static constexpr _SimdMember<_Tp>
1445 _S_generator(_Fp&& __gen, _TypeTag<_Tp>)
1446 {
1447 return __generate_vector<_Tp, _S_full_size<_Tp>>(
1448 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1449 if constexpr (__i < _S_size<_Tp>)
1450 return __gen(__i);
1451 else
1452 return 0;
1453 });
1454 }
1455
1456 // _S_load {{{2
1457 template <typename _Tp, typename _Up>
1458 _GLIBCXX_SIMD_INTRINSIC static _SimdMember<_Tp>
1459 _S_load(const _Up* __mem, _TypeTag<_Tp>) noexcept
1460 {
1461 constexpr size_t _Np = _S_size<_Tp>;
1462 constexpr size_t __max_load_size
1463 = (sizeof(_Up) >= 4 && __have_avx512f) || __have_avx512bw ? 64
1464 : (is_floating_point_v<_Up> && __have_avx) || __have_avx2 ? 32
1465 : 16;
1466 constexpr size_t __bytes_to_load = sizeof(_Up) * _Np;
1467 if constexpr (sizeof(_Up) > 8)
1468 return __generate_vector<_Tp, _SimdMember<_Tp>::_S_full_size>(
1469 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1470 return static_cast<_Tp>(__i < _Np ? __mem[__i] : 0);
1471 });
1472 else if constexpr (is_same_v<_Up, _Tp>)
1473 return _CommonImpl::template _S_load<_Tp, _S_full_size<_Tp>,
1474 _Np * sizeof(_Tp)>(__mem);
1475 else if constexpr (__bytes_to_load <= __max_load_size)
1476 return __convert<_SimdMember<_Tp>>(
1477 _CommonImpl::template _S_load<_Up, _Np>(__mem));
1478 else if constexpr (__bytes_to_load % __max_load_size == 0)
1479 {
1480 constexpr size_t __n_loads = __bytes_to_load / __max_load_size;
1481 constexpr size_t __elements_per_load = _Np / __n_loads;
1482 return __call_with_n_evaluations<__n_loads>(
1483 [](auto... __uncvted) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1484 return __convert<_SimdMember<_Tp>>(__uncvted...);
1485 }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1486 return _CommonImpl::template _S_load<_Up, __elements_per_load>(
1487 __mem + __i * __elements_per_load);
1488 });
1489 }
1490 else if constexpr (__bytes_to_load % (__max_load_size / 2) == 0
1491 && __max_load_size > 16)
1492 { // e.g. int[] -> <char, 12> with AVX2
1493 constexpr size_t __n_loads
1494 = __bytes_to_load / (__max_load_size / 2);
1495 constexpr size_t __elements_per_load = _Np / __n_loads;
1496 return __call_with_n_evaluations<__n_loads>(
1497 [](auto... __uncvted) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1498 return __convert<_SimdMember<_Tp>>(__uncvted...);
1499 }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1500 return _CommonImpl::template _S_load<_Up, __elements_per_load>(
1501 __mem + __i * __elements_per_load);
1502 });
1503 }
1504 else // e.g. int[] -> <char, 9>
1505 return __call_with_subscripts(
1506 __mem, make_index_sequence<_Np>(),
1507 [](auto... __args) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1508 return __vector_type_t<_Tp, _S_full_size<_Tp>>{static_cast<_Tp>(__args)...};
1509 });
1510 }
1511
1512 // _S_masked_load {{{2
1513 template <typename _Tp, size_t _Np, typename _Up>
1514 static inline _SimdWrapper<_Tp, _Np>
1515 _S_masked_load(_SimdWrapper<_Tp, _Np> __merge, _MaskMember<_Tp> __k,
1516 const _Up* __mem) noexcept
1517 {
1518 _BitOps::_S_bit_iteration(_MaskImpl::_S_to_bits(__k),
1519 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1520 __merge._M_set(__i, static_cast<_Tp>(__mem[__i]));
1521 });
1522 return __merge;
1523 }
1524
1525 // _S_store {{{2
1526 template <typename _Tp, typename _Up>
1527 _GLIBCXX_SIMD_INTRINSIC static void
1528 _S_store(_SimdMember<_Tp> __v, _Up* __mem, _TypeTag<_Tp>) noexcept
1529 {
1530 // TODO: converting int -> "smaller int" can be optimized with AVX512
1531 constexpr size_t _Np = _S_size<_Tp>;
1532 constexpr size_t __max_store_size
1533 = _SuperImpl::template _S_max_store_size<_Up>;
1534 if constexpr (sizeof(_Up) > 8)
1535 __execute_n_times<_Np>([&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1536 __mem[__i] = __v[__i];
1537 });
1538 else if constexpr (is_same_v<_Up, _Tp>)
1539 _CommonImpl::_S_store(__v, __mem);
1540 else if constexpr (sizeof(_Up) * _Np <= __max_store_size)
1541 _CommonImpl::_S_store(_SimdWrapper<_Up, _Np>(__convert<_Up>(__v)),
1542 __mem);
1543 else
1544 {
1545 constexpr size_t __vsize = __max_store_size / sizeof(_Up);
1546 // round up to convert the last partial vector as well:
1547 constexpr size_t __stores = __div_roundup(_Np, __vsize);
1548 constexpr size_t __full_stores = _Np / __vsize;
1549 using _V = __vector_type_t<_Up, __vsize>;
1550 const array<_V, __stores> __converted
1551 = __convert_all<_V, __stores>(__v);
1552 __execute_n_times<__full_stores>(
1553 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1554 _CommonImpl::_S_store(__converted[__i], __mem + __i * __vsize);
1555 });
1556 if constexpr (__full_stores < __stores)
1557 _CommonImpl::template _S_store<(_Np - __full_stores * __vsize)
1558 * sizeof(_Up)>(
1559 __converted[__full_stores], __mem + __full_stores * __vsize);
1560 }
1561 }
1562
1563 // _S_masked_store_nocvt {{{2
1564 template <typename _Tp, size_t _Np>
1565 _GLIBCXX_SIMD_INTRINSIC static void
1566 _S_masked_store_nocvt(_SimdWrapper<_Tp, _Np> __v, _Tp* __mem, _MaskMember<_Tp> __k)
1567 {
1568 _BitOps::_S_bit_iteration(
1569 _MaskImpl::_S_to_bits(__k),
1570 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1571 __mem[__i] = __v[__i];
1572 });
1573 }
1574
1575 // _S_masked_store {{{2
1576 template <typename _TW, typename _TVT = _VectorTraits<_TW>,
1577 typename _Tp = typename _TVT::value_type, typename _Up>
1578 static inline void
1579 _S_masked_store(const _TW __v, _Up* __mem, const _MaskMember<_Tp> __k) noexcept
1580 {
1581 constexpr size_t _TV_size = _S_size<_Tp>;
1582 [[maybe_unused]] const auto __vi = __to_intrin(__v);
1583 constexpr size_t __max_store_size
1584 = _SuperImpl::template _S_max_store_size<_Up>;
1585 if constexpr (
1586 is_same_v<
1587 _Tp,
1588 _Up> || (is_integral_v<_Tp> && is_integral_v<_Up> && sizeof(_Tp) == sizeof(_Up)))
1589 {
1590 // bitwise or no conversion, reinterpret:
1591 const _MaskMember<_Up> __kk = [&]() _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1592 if constexpr (__is_bitmask_v<decltype(__k)>)
1593 return _MaskMember<_Up>(__k._M_data);
1594 else
1595 return __wrapper_bitcast<__int_for_sizeof_t<_Up>>(__k);
1596 }();
1597 _SuperImpl::_S_masked_store_nocvt(__wrapper_bitcast<_Up>(__v),
1598 __mem, __kk);
1599 }
1600 else if constexpr (__vectorized_sizeof<_Up>() > sizeof(_Up)
1601 && !_CommonImpl::
1602 template __converts_via_decomposition_v<
1603 _Tp, _Up, __max_store_size>)
1604 { // conversion via decomposition is better handled via the
1605 // bit_iteration
1606 // fallback below
1607 constexpr size_t _UW_size
1608 = std::min(_TV_size, __max_store_size / sizeof(_Up));
1609 static_assert(_UW_size <= _TV_size);
1610 using _UW = _SimdWrapper<_Up, _UW_size>;
1611 using _UV = __vector_type_t<_Up, _UW_size>;
1612 using _UAbi = simd_abi::deduce_t<_Up, _UW_size>;
1613 if constexpr (_UW_size == _TV_size) // one convert+store
1614 {
1615 const _UW __converted = __convert<_UW>(__v);
1616 _SuperImpl::_S_masked_store_nocvt(
1617 __converted, __mem,
1618 _UAbi::_MaskImpl::template _S_convert<
1619 __int_for_sizeof_t<_Up>>(__k));
1620 }
1621 else
1622 {
1623 static_assert(_UW_size * sizeof(_Up) == __max_store_size);
1624 constexpr size_t _NFullStores = _TV_size / _UW_size;
1625 constexpr size_t _NAllStores
1626 = __div_roundup(_TV_size, _UW_size);
1627 constexpr size_t _NParts = _S_full_size<_Tp> / _UW_size;
1628 const array<_UV, _NAllStores> __converted
1629 = __convert_all<_UV, _NAllStores>(__v);
1630 __execute_n_times<_NFullStores>([&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1631 _SuperImpl::_S_masked_store_nocvt(
1632 _UW(__converted[__i]), __mem + __i * _UW_size,
1633 _UAbi::_MaskImpl::template _S_convert<
1634 __int_for_sizeof_t<_Up>>(
1635 __extract_part<__i, _NParts>(__k.__as_full_vector())));
1636 });
1637 if constexpr (_NAllStores
1638 > _NFullStores) // one partial at the end
1639 _SuperImpl::_S_masked_store_nocvt(
1640 _UW(__converted[_NFullStores]),
1641 __mem + _NFullStores * _UW_size,
1642 _UAbi::_MaskImpl::template _S_convert<
1643 __int_for_sizeof_t<_Up>>(
1644 __extract_part<_NFullStores, _NParts>(
1645 __k.__as_full_vector())));
1646 }
1647 }
1648 else
1649 _BitOps::_S_bit_iteration(_MaskImpl::_S_to_bits(__k),
1650 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1651 __mem[__i] = static_cast<_Up>(__v[__i]);
1652 });
1653 }
1654
1655 // _S_complement {{{2
1656 template <typename _Tp, size_t _Np>
1657 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
1658 _S_complement(_SimdWrapper<_Tp, _Np> __x) noexcept
1659 {
1660 if constexpr (is_floating_point_v<_Tp>)
1661 return __vector_bitcast<_Tp>(~__vector_bitcast<__int_for_sizeof_t<_Tp>>(__x));
1662 else
1663 return ~__x._M_data;
1664 }
1665
1666 // _S_unary_minus {{{2
1667 template <typename _Tp, size_t _Np>
1668 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
1669 _S_unary_minus(_SimdWrapper<_Tp, _Np> __x) noexcept
1670 {
1671 // GCC doesn't use the psign instructions, but pxor & psub seem to be
1672 // just as good a choice as pcmpeqd & psign. So meh.
1673 return -__x._M_data;
1674 }
1675
1676 // arithmetic operators {{{2
1677 template <typename _Tp, size_t _Np>
1678 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
1679 _S_plus(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
1680 { return __x._M_data + __y._M_data; }
1681
1682 template <typename _Tp, size_t _Np>
1683 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
1684 _S_minus(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
1685 { return __x._M_data - __y._M_data; }
1686
1687 template <typename _Tp, size_t _Np>
1688 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
1689 _S_multiplies(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
1690 { return __x._M_data * __y._M_data; }
1691
1692 template <typename _Tp, size_t _Np>
1693 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
1694 _S_divides(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
1695 {
1696 // Note that division by 0 is always UB, so we must ensure we avoid the
1697 // case for partial registers
1698 if constexpr (!_Abi::template _S_is_partial<_Tp>)
1699 return __x._M_data / __y._M_data;
1700 else
1701 return __x._M_data / _Abi::__make_padding_nonzero(__y._M_data);
1702 }
1703
1704 template <typename _Tp, size_t _Np>
1705 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
1706 _S_modulus(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
1707 {
1708 if constexpr (!_Abi::template _S_is_partial<_Tp>)
1709 return __x._M_data % __y._M_data;
1710 else
1711 return __as_vector(__x)
1712 % _Abi::__make_padding_nonzero(__as_vector(__y));
1713 }
1714
1715 template <typename _Tp, size_t _Np>
1716 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
1717 _S_bit_and(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
1718 { return __and(__x, __y); }
1719
1720 template <typename _Tp, size_t _Np>
1721 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
1722 _S_bit_or(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
1723 { return __or(__x, __y); }
1724
1725 template <typename _Tp, size_t _Np>
1726 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
1727 _S_bit_xor(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
1728 { return __xor(__x, __y); }
1729
1730 template <typename _Tp, size_t _Np>
1731 _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
1732 _S_bit_shift_left(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
1733 { return __x._M_data << __y._M_data; }
1734
1735 template <typename _Tp, size_t _Np>
1736 _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
1737 _S_bit_shift_right(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
1738 { return __x._M_data >> __y._M_data; }
1739
1740 template <typename _Tp, size_t _Np>
1741 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
1742 _S_bit_shift_left(_SimdWrapper<_Tp, _Np> __x, int __y)
1743 { return __x._M_data << __y; }
1744
1745 template <typename _Tp, size_t _Np>
1746 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
1747 _S_bit_shift_right(_SimdWrapper<_Tp, _Np> __x, int __y)
1748 { return __x._M_data >> __y; }
1749
1750 // compares {{{2
1751 // _S_equal_to {{{3
1752 template <typename _Tp, size_t _Np>
1753 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
1754 _S_equal_to(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
1755 { return __x._M_data == __y._M_data; }
1756
1757 // _S_not_equal_to {{{3
1758 template <typename _Tp, size_t _Np>
1759 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
1760 _S_not_equal_to(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
1761 { return __x._M_data != __y._M_data; }
1762
1763 // _S_less {{{3
1764 template <typename _Tp, size_t _Np>
1765 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
1766 _S_less(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
1767 { return __x._M_data < __y._M_data; }
1768
1769 // _S_less_equal {{{3
1770 template <typename _Tp, size_t _Np>
1771 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
1772 _S_less_equal(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
1773 { return __x._M_data <= __y._M_data; }
1774
1775 // _S_negate {{{2
1776 template <typename _Tp, size_t _Np>
1777 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
1778 _S_negate(_SimdWrapper<_Tp, _Np> __x) noexcept
1779 { return !__x._M_data; }
1780
1781 // _S_min, _S_max, _S_minmax {{{2
1782 template <typename _Tp, size_t _Np>
1783 _GLIBCXX_SIMD_NORMAL_MATH _GLIBCXX_SIMD_INTRINSIC static constexpr
1784 _SimdWrapper<_Tp, _Np>
1785 _S_min(_SimdWrapper<_Tp, _Np> __a, _SimdWrapper<_Tp, _Np> __b)
1786 { return __a._M_data < __b._M_data ? __a._M_data : __b._M_data; }
1787
1788 template <typename _Tp, size_t _Np>
1789 _GLIBCXX_SIMD_NORMAL_MATH _GLIBCXX_SIMD_INTRINSIC static constexpr
1790 _SimdWrapper<_Tp, _Np>
1791 _S_max(_SimdWrapper<_Tp, _Np> __a, _SimdWrapper<_Tp, _Np> __b)
1792 { return __a._M_data > __b._M_data ? __a._M_data : __b._M_data; }
1793
1794 template <typename _Tp, size_t _Np>
1795 _GLIBCXX_SIMD_NORMAL_MATH _GLIBCXX_SIMD_INTRINSIC static constexpr
1796 pair<_SimdWrapper<_Tp, _Np>, _SimdWrapper<_Tp, _Np>>
1797 _S_minmax(_SimdWrapper<_Tp, _Np> __a, _SimdWrapper<_Tp, _Np> __b)
1798 {
1799 return {__a._M_data < __b._M_data ? __a._M_data : __b._M_data,
1800 __a._M_data < __b._M_data ? __b._M_data : __a._M_data};
1801 }
1802
1803 // reductions {{{2
1804 template <size_t _Np, size_t... _Is, size_t... _Zeros, typename _Tp,
1805 typename _BinaryOperation>
1806 _GLIBCXX_SIMD_INTRINSIC static _Tp
1807 _S_reduce_partial(index_sequence<_Is...>, index_sequence<_Zeros...>,
1808 simd<_Tp, _Abi> __x, _BinaryOperation&& __binary_op)
1809 {
1810 using _V = __vector_type_t<_Tp, _Np / 2>;
1811 static_assert(sizeof(_V) <= sizeof(__x));
1812 // _S_full_size is the size of the smallest native SIMD register that
1813 // can store _Np/2 elements:
1814 using _FullSimd = __deduced_simd<_Tp, _VectorTraits<_V>::_S_full_size>;
1815 using _HalfSimd = __deduced_simd<_Tp, _Np / 2>;
1816 const auto __xx = __as_vector(__x);
1817 return _HalfSimd::abi_type::_SimdImpl::_S_reduce(
1818 static_cast<_HalfSimd>(__as_vector(__binary_op(
1819 static_cast<_FullSimd>(__intrin_bitcast<_V>(__xx)),
1820 static_cast<_FullSimd>(__intrin_bitcast<_V>(
1821 __vector_permute<(_Np / 2 + _Is)..., (int(_Zeros * 0) - 1)...>(
1822 __xx)))))),
1823 __binary_op);
1824 }
1825
1826 template <typename _Tp, typename _BinaryOperation>
1827 _GLIBCXX_SIMD_INTRINSIC static constexpr _Tp
1828 _S_reduce(simd<_Tp, _Abi> __x, _BinaryOperation&& __binary_op)
1829 {
1830 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
1831 if constexpr (_Np == 1)
1832 return __x[0];
1833 else if constexpr (_Np == 2)
1834 return __binary_op(simd<_Tp, simd_abi::scalar>(__x[0]),
1835 simd<_Tp, simd_abi::scalar>(__x[1]))[0];
1836 else if constexpr (_Abi::template _S_is_partial<_Tp>) //{{{
1837 {
1838 [[maybe_unused]] constexpr auto __full_size
1839 = _Abi::template _S_full_size<_Tp>;
1840 if constexpr (_Np == 3)
1841 return __binary_op(
1842 __binary_op(simd<_Tp, simd_abi::scalar>(__x[0]),
1843 simd<_Tp, simd_abi::scalar>(__x[1])),
1844 simd<_Tp, simd_abi::scalar>(__x[2]))[0];
1845 else if constexpr (is_same_v<__remove_cvref_t<_BinaryOperation>,
1846 plus<>>)
1847 {
1848 using _Ap = simd_abi::deduce_t<_Tp, __full_size>;
1849 return _Ap::_SimdImpl::_S_reduce(
1850 simd<_Tp, _Ap>(__private_init,
1851 _Abi::_S_masked(__as_vector(__x))),
1852 __binary_op);
1853 }
1854 else if constexpr (is_same_v<__remove_cvref_t<_BinaryOperation>,
1855 multiplies<>>)
1856 {
1857 using _Ap = simd_abi::deduce_t<_Tp, __full_size>;
1858 using _TW = _SimdWrapper<_Tp, __full_size>;
1859 _GLIBCXX_SIMD_USE_CONSTEXPR auto __implicit_mask_full
1860 = _Abi::template _S_implicit_mask<_Tp>().__as_full_vector();
1861 _GLIBCXX_SIMD_USE_CONSTEXPR _TW __one
1862 = __vector_broadcast<__full_size>(_Tp(1));
1863 const _TW __x_full = __data(__x).__as_full_vector();
1864 const _TW __x_padded_with_ones
1865 = _Ap::_CommonImpl::_S_blend(__implicit_mask_full, __one,
1866 __x_full);
1867 return _Ap::_SimdImpl::_S_reduce(
1868 simd<_Tp, _Ap>(__private_init, __x_padded_with_ones),
1869 __binary_op);
1870 }
1871 else if constexpr (_Np & 1)
1872 {
1873 using _Ap = simd_abi::deduce_t<_Tp, _Np - 1>;
1874 return __binary_op(
1875 simd<_Tp, simd_abi::scalar>(_Ap::_SimdImpl::_S_reduce(
1876 simd<_Tp, _Ap>(
1877 __intrin_bitcast<__vector_type_t<_Tp, _Np - 1>>(
1878 __as_vector(__x))),
1879 __binary_op)),
1880 simd<_Tp, simd_abi::scalar>(__x[_Np - 1]))[0];
1881 }
1882 else
1883 return _S_reduce_partial<_Np>(
1884 make_index_sequence<_Np / 2>(),
1885 make_index_sequence<__full_size - _Np / 2>(), __x, __binary_op);
1886 } //}}}
1887 else if constexpr (sizeof(__x) == 16) //{{{
1888 {
1889 if constexpr (_Np == 16)
1890 {
1891 const auto __y = __data(__x);
1892 __x = __binary_op(
1893 _M_make_simd<_Tp, _Np>(
1894 __vector_permute<0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6,
1895 7, 7>(__y)),
1896 _M_make_simd<_Tp, _Np>(
1897 __vector_permute<8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13,
1898 14, 14, 15, 15>(__y)));
1899 }
1900 if constexpr (_Np >= 8)
1901 {
1902 const auto __y = __vector_bitcast<short>(__data(__x));
1903 __x = __binary_op(
1904 _M_make_simd<_Tp, _Np>(__vector_bitcast<_Tp>(
1905 __vector_permute<0, 0, 1, 1, 2, 2, 3, 3>(__y))),
1906 _M_make_simd<_Tp, _Np>(__vector_bitcast<_Tp>(
1907 __vector_permute<4, 4, 5, 5, 6, 6, 7, 7>(__y))));
1908 }
1909 if constexpr (_Np >= 4)
1910 {
1911 using _Up = conditional_t<is_floating_point_v<_Tp>, float, int>;
1912 const auto __y = __vector_bitcast<_Up>(__data(__x));
1913 __x = __binary_op(__x,
1914 _M_make_simd<_Tp, _Np>(__vector_bitcast<_Tp>(
1915 __vector_permute<3, 2, 1, 0>(__y))));
1916 }
1917 using _Up = conditional_t<is_floating_point_v<_Tp>, double, _LLong>;
1918 const auto __y = __vector_bitcast<_Up>(__data(__x));
1919 __x = __binary_op(__x, _M_make_simd<_Tp, _Np>(__vector_bitcast<_Tp>(
1920 __vector_permute<1, 1>(__y))));
1921 return __x[0];
1922 } //}}}
1923 else
1924 {
1925 static_assert(sizeof(__x) > __min_vector_size<_Tp>);
1926 static_assert((_Np & (_Np - 1)) == 0); // _Np must be a power of 2
1927 using _Ap = simd_abi::deduce_t<_Tp, _Np / 2>;
1928 using _V = simd<_Tp, _Ap>;
1929 return _Ap::_SimdImpl::_S_reduce(
1930 __binary_op(_V(__private_init, __extract<0, 2>(__as_vector(__x))),
1931 _V(__private_init,
1932 __extract<1, 2>(__as_vector(__x)))),
1933 static_cast<_BinaryOperation&&>(__binary_op));
1934 }
1935 }
1936
1937 // math {{{2
1938 // frexp, modf and copysign implemented in simd_math.h
1939#define _GLIBCXX_SIMD_MATH_FALLBACK(__name) \
1940 template <typename _Tp, typename... _More> \
1941 static _Tp \
1942 _S_##__name(const _Tp& __x, const _More&... __more) \
1943 { \
1944 return __generate_vector<_Tp>( \
1945 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { \
1946 return __name(__x[__i], __more[__i]...); \
1947 }); \
1948 }
1949
1950#define _GLIBCXX_SIMD_MATH_FALLBACK_MASKRET(__name) \
1951 template <typename _Tp, typename... _More> \
1952 static typename _Tp::mask_type \
1953 _S_##__name(const _Tp& __x, const _More&... __more) \
1954 { \
1955 return __generate_vector<_Tp>( \
1956 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { \
1957 return __name(__x[__i], __more[__i]...); \
1958 }); \
1959 }
1960
1961#define _GLIBCXX_SIMD_MATH_FALLBACK_FIXEDRET(_RetTp, __name) \
1962 template <typename _Tp, typename... _More> \
1963 static auto \
1964 _S_##__name(const _Tp& __x, const _More&... __more) \
1965 { \
1966 return __fixed_size_storage_t<_RetTp, \
1967 _VectorTraits<_Tp>::_S_partial_width>:: \
1968 _S_generate([&](auto __meta) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { \
1969 return __meta._S_generator( \
1970 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { \
1971 return __name(__x[__meta._S_offset + __i], \
1972 __more[__meta._S_offset + __i]...); \
1973 }, \
1974 static_cast<_RetTp*>(nullptr)); \
1975 }); \
1976 }
1977
1978 _GLIBCXX_SIMD_MATH_FALLBACK(acos)
1979 _GLIBCXX_SIMD_MATH_FALLBACK(asin)
1980 _GLIBCXX_SIMD_MATH_FALLBACK(atan)
1981 _GLIBCXX_SIMD_MATH_FALLBACK(atan2)
1982 _GLIBCXX_SIMD_MATH_FALLBACK(cos)
1983 _GLIBCXX_SIMD_MATH_FALLBACK(sin)
1984 _GLIBCXX_SIMD_MATH_FALLBACK(tan)
1985 _GLIBCXX_SIMD_MATH_FALLBACK(acosh)
1986 _GLIBCXX_SIMD_MATH_FALLBACK(asinh)
1987 _GLIBCXX_SIMD_MATH_FALLBACK(atanh)
1988 _GLIBCXX_SIMD_MATH_FALLBACK(cosh)
1989 _GLIBCXX_SIMD_MATH_FALLBACK(sinh)
1990 _GLIBCXX_SIMD_MATH_FALLBACK(tanh)
1991 _GLIBCXX_SIMD_MATH_FALLBACK(exp)
1992 _GLIBCXX_SIMD_MATH_FALLBACK(exp2)
1993 _GLIBCXX_SIMD_MATH_FALLBACK(expm1)
1994 _GLIBCXX_SIMD_MATH_FALLBACK(ldexp)
1995 _GLIBCXX_SIMD_MATH_FALLBACK_FIXEDRET(int, ilogb)
1996 _GLIBCXX_SIMD_MATH_FALLBACK(log)
1997 _GLIBCXX_SIMD_MATH_FALLBACK(log10)
1998 _GLIBCXX_SIMD_MATH_FALLBACK(log1p)
1999 _GLIBCXX_SIMD_MATH_FALLBACK(log2)
2000 _GLIBCXX_SIMD_MATH_FALLBACK(logb)
2001
2002 // modf implemented in simd_math.h
2003 _GLIBCXX_SIMD_MATH_FALLBACK(scalbn)
2004 _GLIBCXX_SIMD_MATH_FALLBACK(scalbln)
2005 _GLIBCXX_SIMD_MATH_FALLBACK(cbrt)
2006 _GLIBCXX_SIMD_MATH_FALLBACK(fabs)
2007 _GLIBCXX_SIMD_MATH_FALLBACK(pow)
2008 _GLIBCXX_SIMD_MATH_FALLBACK(sqrt)
2009 _GLIBCXX_SIMD_MATH_FALLBACK(erf)
2010 _GLIBCXX_SIMD_MATH_FALLBACK(erfc)
2011 _GLIBCXX_SIMD_MATH_FALLBACK(lgamma)
2012 _GLIBCXX_SIMD_MATH_FALLBACK(tgamma)
2013
2014 _GLIBCXX_SIMD_MATH_FALLBACK_FIXEDRET(long, lrint)
2015 _GLIBCXX_SIMD_MATH_FALLBACK_FIXEDRET(long long, llrint)
2016
2017 _GLIBCXX_SIMD_MATH_FALLBACK_FIXEDRET(long, lround)
2018 _GLIBCXX_SIMD_MATH_FALLBACK_FIXEDRET(long long, llround)
2019
2020 _GLIBCXX_SIMD_MATH_FALLBACK(fmod)
2021 _GLIBCXX_SIMD_MATH_FALLBACK(remainder)
2022
2023 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2024 static _Tp
2025 _S_remquo(const _Tp __x, const _Tp __y,
2026 __fixed_size_storage_t<int, _TVT::_S_partial_width>* __z)
2027 {
2028 return __generate_vector<_Tp>([&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2029 int __tmp;
2030 auto __r = remquo(__x[__i], __y[__i], &__tmp);
2031 __z->_M_set(__i, __tmp);
2032 return __r;
2033 });
2034 }
2035
2036 // copysign in simd_math.h
2037 _GLIBCXX_SIMD_MATH_FALLBACK(nextafter)
2038 _GLIBCXX_SIMD_MATH_FALLBACK(fdim)
2039 _GLIBCXX_SIMD_MATH_FALLBACK(fmax)
2040 _GLIBCXX_SIMD_MATH_FALLBACK(fmin)
2041 _GLIBCXX_SIMD_MATH_FALLBACK(fma)
2042
2043 template <typename _Tp, size_t _Np>
2044 static constexpr _MaskMember<_Tp>
2045 _S_isgreater(_SimdWrapper<_Tp, _Np> __x,
2046 _SimdWrapper<_Tp, _Np> __y) noexcept
2047 {
2048 using _Ip = __int_for_sizeof_t<_Tp>;
2049 const auto __xn = __vector_bitcast<_Ip>(__x);
2050 const auto __yn = __vector_bitcast<_Ip>(__y);
2051 const auto __xp = __xn < 0 ? -(__xn & __finite_max_v<_Ip>) : __xn;
2052 const auto __yp = __yn < 0 ? -(__yn & __finite_max_v<_Ip>) : __yn;
2053 return __andnot(_SuperImpl::_S_isunordered(__x, __y)._M_data,
2054 __xp > __yp);
2055 }
2056
2057 template <typename _Tp, size_t _Np>
2058 static constexpr _MaskMember<_Tp>
2059 _S_isgreaterequal(_SimdWrapper<_Tp, _Np> __x,
2060 _SimdWrapper<_Tp, _Np> __y) noexcept
2061 {
2062 using _Ip = __int_for_sizeof_t<_Tp>;
2063 const auto __xn = __vector_bitcast<_Ip>(__x);
2064 const auto __yn = __vector_bitcast<_Ip>(__y);
2065 const auto __xp = __xn < 0 ? -(__xn & __finite_max_v<_Ip>) : __xn;
2066 const auto __yp = __yn < 0 ? -(__yn & __finite_max_v<_Ip>) : __yn;
2067 return __andnot(_SuperImpl::_S_isunordered(__x, __y)._M_data,
2068 __xp >= __yp);
2069 }
2070
2071 template <typename _Tp, size_t _Np>
2072 static constexpr _MaskMember<_Tp>
2073 _S_isless(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y) noexcept
2074 {
2075 using _Ip = __int_for_sizeof_t<_Tp>;
2076 const auto __xn = __vector_bitcast<_Ip>(__x);
2077 const auto __yn = __vector_bitcast<_Ip>(__y);
2078 const auto __xp = __xn < 0 ? -(__xn & __finite_max_v<_Ip>) : __xn;
2079 const auto __yp = __yn < 0 ? -(__yn & __finite_max_v<_Ip>) : __yn;
2080 return __andnot(_SuperImpl::_S_isunordered(__x, __y)._M_data,
2081 __xp < __yp);
2082 }
2083
2084 template <typename _Tp, size_t _Np>
2085 static constexpr _MaskMember<_Tp>
2086 _S_islessequal(_SimdWrapper<_Tp, _Np> __x,
2087 _SimdWrapper<_Tp, _Np> __y) noexcept
2088 {
2089 using _Ip = __int_for_sizeof_t<_Tp>;
2090 const auto __xn = __vector_bitcast<_Ip>(__x);
2091 const auto __yn = __vector_bitcast<_Ip>(__y);
2092 const auto __xp = __xn < 0 ? -(__xn & __finite_max_v<_Ip>) : __xn;
2093 const auto __yp = __yn < 0 ? -(__yn & __finite_max_v<_Ip>) : __yn;
2094 return __andnot(_SuperImpl::_S_isunordered(__x, __y)._M_data,
2095 __xp <= __yp);
2096 }
2097
2098 template <typename _Tp, size_t _Np>
2099 static constexpr _MaskMember<_Tp>
2100 _S_islessgreater(_SimdWrapper<_Tp, _Np> __x,
2101 _SimdWrapper<_Tp, _Np> __y) noexcept
2102 {
2103 return __andnot(_SuperImpl::_S_isunordered(__x, __y),
2104 _SuperImpl::_S_not_equal_to(__x, __y));
2105 }
2106
2107#undef _GLIBCXX_SIMD_MATH_FALLBACK
2108#undef _GLIBCXX_SIMD_MATH_FALLBACK_MASKRET
2109#undef _GLIBCXX_SIMD_MATH_FALLBACK_FIXEDRET
2110 // _S_abs {{{3
2111 template <typename _Tp, size_t _Np>
2112 _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
2113 _S_abs(_SimdWrapper<_Tp, _Np> __x) noexcept
2114 {
2115 // if (__builtin_is_constant_evaluated())
2116 // {
2117 // return __x._M_data < 0 ? -__x._M_data : __x._M_data;
2118 // }
2119 if constexpr (is_floating_point_v<_Tp>)
2120 // `v < 0 ? -v : v` cannot compile to the efficient implementation of
2121 // masking the signbit off because it must consider v == -0
2122
2123 // ~(-0.) & v would be easy, but breaks with fno-signed-zeros
2124 return __and(_S_absmask<__vector_type_t<_Tp, _Np>>, __x._M_data);
2125 else
2126 return __x._M_data < 0 ? -__x._M_data : __x._M_data;
2127 }
2128
2129 // }}}3
2130 // _S_plus_minus {{{
2131 // Returns __x + __y - __y without -fassociative-math optimizing to __x.
2132 // - _TV must be __vector_type_t<floating-point type, N>.
2133 // - _UV must be _TV or floating-point type.
2134 template <typename _TV, typename _UV>
2135 _GLIBCXX_SIMD_INTRINSIC static constexpr _TV
2136 _S_plus_minus(_TV __x, _UV __y) noexcept
2137 {
2138#if defined __i386__ && !defined __SSE_MATH__
2139 if constexpr (sizeof(__x) == 8)
2140 { // operations on __x would use the FPU
2141 static_assert(is_same_v<_TV, __vector_type_t<float, 2>>);
2142 const auto __x4 = __vector_bitcast<float, 4>(__x);
2143 if constexpr (is_same_v<_TV, _UV>)
2144 return __vector_bitcast<float, 2>(
2145 _S_plus_minus(__x4, __vector_bitcast<float, 4>(__y)));
2146 else
2147 return __vector_bitcast<float, 2>(_S_plus_minus(__x4, __y));
2148 }
2149#endif
2150#if !defined __clang__ && __GCC_IEC_559 == 0
2151 if (__builtin_is_constant_evaluated()
2152 || (__builtin_constant_p(__x) && __builtin_constant_p(__y)))
2153 return (__x + __y) - __y;
2154 else
2155 return [&] {
2156 __x += __y;
2157 if constexpr(__have_sse)
2158 {
2159 if constexpr (sizeof(__x) >= 16)
2160 asm("" : "+x"(__x));
2161 else if constexpr (is_same_v<__vector_type_t<float, 2>, _TV>)
2162 asm("" : "+x"(__x[0]), "+x"(__x[1]));
2163 else
2164 __assert_unreachable<_TV>();
2165 }
2166 else if constexpr(__have_neon)
2167 asm("" : "+w"(__x));
2168 else if constexpr (__have_power_vmx)
2169 {
2170 if constexpr (is_same_v<__vector_type_t<float, 2>, _TV>)
2171 asm("" : "+fgr"(__x[0]), "+fgr"(__x[1]));
2172 else
2173 asm("" : "+v"(__x));
2174 }
2175 else
2176 asm("" : "+g"(__x));
2177 return __x - __y;
2178 }();
2179#else
2180 return (__x + __y) - __y;
2181#endif
2182 }
2183
2184 // }}}
2185 // _S_nearbyint {{{3
2186 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2187 _GLIBCXX_SIMD_INTRINSIC static _Tp
2188 _S_nearbyint(_Tp __x_) noexcept
2189 {
2190 using value_type = typename _TVT::value_type;
2191 using _V = typename _TVT::type;
2192 const _V __x = __x_;
2193 const _V __absx = __and(__x, _S_absmask<_V>);
2194 static_assert(__CHAR_BIT__ * sizeof(1ull) >= __digits_v<value_type>);
2195 _GLIBCXX_SIMD_USE_CONSTEXPR _V __shifter_abs
2196 = _V() + (1ull << (__digits_v<value_type> - 1));
2197 const _V __shifter = __or(__and(_S_signmask<_V>, __x), __shifter_abs);
2198 const _V __shifted = _S_plus_minus(__x, __shifter);
2199 return __absx < __shifter_abs ? __shifted : __x;
2200 }
2201
2202 // _S_rint {{{3
2203 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2204 _GLIBCXX_SIMD_INTRINSIC static _Tp
2205 _S_rint(_Tp __x) noexcept
2206 { return _SuperImpl::_S_nearbyint(__x); }
2207
2208 // _S_trunc {{{3
2209 template <typename _Tp, size_t _Np>
2210 _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
2211 _S_trunc(_SimdWrapper<_Tp, _Np> __x)
2212 {
2213 using _V = __vector_type_t<_Tp, _Np>;
2214 const _V __absx = __and(__x._M_data, _S_absmask<_V>);
2215 static_assert(__CHAR_BIT__ * sizeof(1ull) >= __digits_v<_Tp>);
2216 constexpr _Tp __shifter = 1ull << (__digits_v<_Tp> - 1);
2217 _V __truncated = _S_plus_minus(__absx, __shifter);
2218 __truncated -= __truncated > __absx ? _V() + 1 : _V();
2219 return __absx < __shifter ? __or(__xor(__absx, __x._M_data), __truncated)
2220 : __x._M_data;
2221 }
2222
2223 // _S_round {{{3
2224 template <typename _Tp, size_t _Np>
2225 _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
2226 _S_round(_SimdWrapper<_Tp, _Np> __x)
2227 {
2228 const auto __abs_x = _SuperImpl::_S_abs(__x);
2229 const auto __t_abs = _SuperImpl::_S_trunc(__abs_x)._M_data;
2230 const auto __r_abs // round(abs(x)) =
2231 = __t_abs + (__abs_x._M_data - __t_abs >= _Tp(.5) ? _Tp(1) : 0);
2232 return __or(__xor(__abs_x._M_data, __x._M_data), __r_abs);
2233 }
2234
2235 // _S_floor {{{3
2236 template <typename _Tp, size_t _Np>
2237 _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
2238 _S_floor(_SimdWrapper<_Tp, _Np> __x)
2239 {
2240 const auto __y = _SuperImpl::_S_trunc(__x)._M_data;
2241 const auto __negative_input
2242 = __vector_bitcast<_Tp>(__x._M_data < __vector_broadcast<_Np, _Tp>(0));
2243 const auto __mask
2244 = __andnot(__vector_bitcast<_Tp>(__y == __x._M_data), __negative_input);
2245 return __or(__andnot(__mask, __y),
2246 __and(__mask, __y - __vector_broadcast<_Np, _Tp>(1)));
2247 }
2248
2249 // _S_ceil {{{3
2250 template <typename _Tp, size_t _Np>
2251 _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
2252 _S_ceil(_SimdWrapper<_Tp, _Np> __x)
2253 {
2254 const auto __y = _SuperImpl::_S_trunc(__x)._M_data;
2255 const auto __negative_input
2256 = __vector_bitcast<_Tp>(__x._M_data < __vector_broadcast<_Np, _Tp>(0));
2257 const auto __inv_mask
2258 = __or(__vector_bitcast<_Tp>(__y == __x._M_data), __negative_input);
2259 return __or(__and(__inv_mask, __y),
2260 __andnot(__inv_mask, __y + __vector_broadcast<_Np, _Tp>(1)));
2261 }
2262
2263 // _S_isnan {{{3
2264 template <typename _Tp, size_t _Np>
2265 _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
2266 _S_isnan([[maybe_unused]] _SimdWrapper<_Tp, _Np> __x)
2267 {
2268#if __FINITE_MATH_ONLY__
2269 return {}; // false
2270#elif !defined __SUPPORT_SNAN__
2271 return ~(__x._M_data == __x._M_data);
2272#elif defined __STDC_IEC_559__
2273 using _Ip = __int_for_sizeof_t<_Tp>;
2274 const auto __absn = __vector_bitcast<_Ip>(_SuperImpl::_S_abs(__x));
2275 const auto __infn
2276 = __vector_bitcast<_Ip>(__vector_broadcast<_Np>(__infinity_v<_Tp>));
2277 return __infn < __absn;
2278#else
2279#error "Not implemented: how to support SNaN but non-IEC559 floating-point?"
2280#endif
2281 }
2282
2283 // _S_isfinite {{{3
2284 template <typename _Tp, size_t _Np>
2285 _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
2286 _S_isfinite([[maybe_unused]] _SimdWrapper<_Tp, _Np> __x)
2287 {
2288#if __FINITE_MATH_ONLY__
2289 using _UV = typename _MaskMember<_Tp>::_BuiltinType;
2290 _GLIBCXX_SIMD_USE_CONSTEXPR _UV __alltrue = ~_UV();
2291 return __alltrue;
2292#else
2293 // if all exponent bits are set, __x is either inf or NaN
2294 using _Ip = __int_for_sizeof_t<_Tp>;
2295 const auto __absn = __vector_bitcast<_Ip>(_SuperImpl::_S_abs(__x));
2296 const auto __maxn
2297 = __vector_bitcast<_Ip>(__vector_broadcast<_Np>(__finite_max_v<_Tp>));
2298 return __absn <= __maxn;
2299#endif
2300 }
2301
2302 // _S_isunordered {{{3
2303 template <typename _Tp, size_t _Np>
2304 _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
2305 _S_isunordered(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
2306 { return __or(_S_isnan(__x), _S_isnan(__y)); }
2307
2308 // _S_signbit {{{3
2309 template <typename _Tp, size_t _Np>
2310 _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
2311 _S_signbit(_SimdWrapper<_Tp, _Np> __x)
2312 {
2313 using _Ip = __int_for_sizeof_t<_Tp>;
2314 return __vector_bitcast<_Ip>(__x) < 0;
2315 // Arithmetic right shift (SRA) would also work (instead of compare), but
2316 // 64-bit SRA isn't available on x86 before AVX512. And in general,
2317 // compares are more likely to be efficient than SRA.
2318 }
2319
2320 // _S_isinf {{{3
2321 template <typename _Tp, size_t _Np>
2322 _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
2323 _S_isinf([[maybe_unused]] _SimdWrapper<_Tp, _Np> __x)
2324 {
2325#if __FINITE_MATH_ONLY__
2326 return {}; // false
2327#else
2328 return _SuperImpl::template _S_equal_to<_Tp, _Np>(_SuperImpl::_S_abs(__x),
2329 __vector_broadcast<_Np>(
2330 __infinity_v<_Tp>));
2331 // alternative:
2332 // compare to inf using the corresponding integer type
2333 /*
2334 return
2335 __vector_bitcast<_Tp>(__vector_bitcast<__int_for_sizeof_t<_Tp>>(
2336 _S_abs(__x)._M_data)
2337 ==
2338 __vector_bitcast<__int_for_sizeof_t<_Tp>>(__vector_broadcast<_Np>(
2339 __infinity_v<_Tp>)));
2340 */
2341#endif
2342 }
2343
2344 // _S_isnormal {{{3
2345 template <typename _Tp, size_t _Np>
2346 _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
2347 _S_isnormal(_SimdWrapper<_Tp, _Np> __x)
2348 {
2349 using _Ip = __int_for_sizeof_t<_Tp>;
2350 const auto __absn = __vector_bitcast<_Ip>(_SuperImpl::_S_abs(__x));
2351 const auto __minn
2352 = __vector_bitcast<_Ip>(__vector_broadcast<_Np>(__norm_min_v<_Tp>));
2353#if __FINITE_MATH_ONLY__
2354 return __absn >= __minn;
2355#else
2356 const auto __maxn
2357 = __vector_bitcast<_Ip>(__vector_broadcast<_Np>(__finite_max_v<_Tp>));
2358 return __minn <= __absn && __absn <= __maxn;
2359#endif
2360 }
2361
2362 // _S_fpclassify {{{3
2363 template <typename _Tp, size_t _Np>
2364 _GLIBCXX_SIMD_INTRINSIC static __fixed_size_storage_t<int, _Np>
2365 _S_fpclassify(_SimdWrapper<_Tp, _Np> __x)
2366 {
2367 using _I = __int_for_sizeof_t<_Tp>;
2368 const auto __xn
2369 = __vector_bitcast<_I>(__to_intrin(_SuperImpl::_S_abs(__x)));
2370 constexpr size_t _NI = sizeof(__xn) / sizeof(_I);
2371 _GLIBCXX_SIMD_USE_CONSTEXPR auto __minn
2372 = __vector_bitcast<_I>(__vector_broadcast<_NI>(__norm_min_v<_Tp>));
2373 _GLIBCXX_SIMD_USE_CONSTEXPR auto __infn
2374 = __vector_bitcast<_I>(__vector_broadcast<_NI>(__infinity_v<_Tp>));
2375
2376 _GLIBCXX_SIMD_USE_CONSTEXPR auto __fp_normal
2377 = __vector_broadcast<_NI, _I>(FP_NORMAL);
2378#if !__FINITE_MATH_ONLY__
2379 _GLIBCXX_SIMD_USE_CONSTEXPR auto __fp_nan
2380 = __vector_broadcast<_NI, _I>(FP_NAN);
2381 _GLIBCXX_SIMD_USE_CONSTEXPR auto __fp_infinite
2382 = __vector_broadcast<_NI, _I>(FP_INFINITE);
2383#endif
2384#ifndef __FAST_MATH__
2385 _GLIBCXX_SIMD_USE_CONSTEXPR auto __fp_subnormal
2386 = __vector_broadcast<_NI, _I>(FP_SUBNORMAL);
2387#endif
2388 _GLIBCXX_SIMD_USE_CONSTEXPR auto __fp_zero
2389 = __vector_broadcast<_NI, _I>(FP_ZERO);
2390
2391 __vector_type_t<_I, _NI>
2392 __tmp = __xn < __minn
2393 #ifdef __FAST_MATH__
2394 ? __fp_zero
2395 #else
2396 ? (__xn == 0 ? __fp_zero : __fp_subnormal)
2397 #endif
2398 #if __FINITE_MATH_ONLY__
2399 : __fp_normal;
2400 #else
2401 : (__xn < __infn ? __fp_normal
2402 : (__xn == __infn ? __fp_infinite : __fp_nan));
2403 #endif
2404
2405 if constexpr (sizeof(_I) == sizeof(int))
2406 {
2407 using _FixedInt = __fixed_size_storage_t<int, _Np>;
2408 const auto __as_int = __vector_bitcast<int, _Np>(__tmp);
2409 if constexpr (_FixedInt::_S_tuple_size == 1)
2410 return {__as_int};
2411 else if constexpr (_FixedInt::_S_tuple_size == 2
2412 && is_same_v<
2413 typename _FixedInt::_SecondType::_FirstAbi,
2414 simd_abi::scalar>)
2415 return {__extract<0, 2>(__as_int), __as_int[_Np - 1]};
2416 else if constexpr (_FixedInt::_S_tuple_size == 2)
2417 return {__extract<0, 2>(__as_int),
2418 __auto_bitcast(__extract<1, 2>(__as_int))};
2419 else
2420 __assert_unreachable<_Tp>();
2421 }
2422 else if constexpr (_Np == 2 && sizeof(_I) == 8
2423 && __fixed_size_storage_t<int, _Np>::_S_tuple_size == 2)
2424 {
2425 const auto __aslong = __vector_bitcast<_LLong>(__tmp);
2426 return {int(__aslong[0]), {int(__aslong[1])}};
2427 }
2428#if _GLIBCXX_SIMD_X86INTRIN
2429 else if constexpr (sizeof(_Tp) == 8 && sizeof(__tmp) == 32
2430 && __fixed_size_storage_t<int, _Np>::_S_tuple_size == 1)
2431 return {_mm_packs_epi32(__to_intrin(__lo128(__tmp)),
2432 __to_intrin(__hi128(__tmp)))};
2433 else if constexpr (sizeof(_Tp) == 8 && sizeof(__tmp) == 64
2434 && __fixed_size_storage_t<int, _Np>::_S_tuple_size == 1)
2435 return {_mm512_cvtepi64_epi32(__to_intrin(__tmp))};
2436#endif // _GLIBCXX_SIMD_X86INTRIN
2437 else if constexpr (__fixed_size_storage_t<int, _Np>::_S_tuple_size == 1)
2438 return {__call_with_subscripts<_Np>(__vector_bitcast<_LLong>(__tmp),
2439 [](auto... __l) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2440 return __make_wrapper<int>(__l...);
2441 })};
2442 else
2443 __assert_unreachable<_Tp>();
2444 }
2445
2446 // _S_increment & _S_decrement{{{2
2447 template <typename _Tp, size_t _Np>
2448 _GLIBCXX_SIMD_INTRINSIC static void
2449 _S_increment(_SimdWrapper<_Tp, _Np>& __x)
2450 { __x = __x._M_data + 1; }
2451
2452 template <typename _Tp, size_t _Np>
2453 _GLIBCXX_SIMD_INTRINSIC static void
2454 _S_decrement(_SimdWrapper<_Tp, _Np>& __x)
2455 { __x = __x._M_data - 1; }
2456
2457 // smart_reference access {{{2
2458 template <typename _Tp, size_t _Np, typename _Up>
2459 _GLIBCXX_SIMD_INTRINSIC constexpr static void
2460 _S_set(_SimdWrapper<_Tp, _Np>& __v, int __i, _Up&& __x) noexcept
2461 { __v._M_set(__i, static_cast<_Up&&>(__x)); }
2462
2463 // _S_masked_assign{{{2
2464 template <typename _Tp, typename _K, size_t _Np>
2465 _GLIBCXX_SIMD_INTRINSIC static void
2466 _S_masked_assign(_SimdWrapper<_K, _Np> __k, _SimdWrapper<_Tp, _Np>& __lhs,
2467 __type_identity_t<_SimdWrapper<_Tp, _Np>> __rhs)
2468 {
2469 if (__k._M_is_constprop_none_of())
2470 return;
2471 else if (__k._M_is_constprop_all_of())
2472 __lhs = __rhs;
2473 else
2474 __lhs = _CommonImpl::_S_blend(__k, __lhs, __rhs);
2475 }
2476
2477 template <typename _Tp, typename _K, size_t _Np>
2478 _GLIBCXX_SIMD_INTRINSIC static void
2479 _S_masked_assign(_SimdWrapper<_K, _Np> __k, _SimdWrapper<_Tp, _Np>& __lhs,
2480 __type_identity_t<_Tp> __rhs)
2481 {
2482 if (__k._M_is_constprop_none_of())
2483 return;
2484 else if (__k._M_is_constprop_all_of())
2485 __lhs = __vector_broadcast<_Np>(__rhs);
2486 else if (__builtin_constant_p(__rhs) && __rhs == 0)
2487 {
2488 if constexpr (!is_same_v<bool, _K>)
2489 // the __andnot optimization only makes sense if __k._M_data is a
2490 // vector register
2491 __lhs._M_data
2492 = __andnot(__vector_bitcast<_Tp>(__k), __lhs._M_data);
2493 else
2494 // for AVX512/__mmask, a _mm512_maskz_mov is best
2495 __lhs
2496 = _CommonImpl::_S_blend(__k, __lhs, _SimdWrapper<_Tp, _Np>());
2497 }
2498 else
2499 __lhs = _CommonImpl::_S_blend(__k, __lhs,
2500 _SimdWrapper<_Tp, _Np>(
2501 __vector_broadcast<_Np>(__rhs)));
2502 }
2503
2504 // _S_masked_cassign {{{2
2505 template <typename _Op, typename _Tp, typename _K, size_t _Np>
2506 _GLIBCXX_SIMD_INTRINSIC static void
2507 _S_masked_cassign(const _SimdWrapper<_K, _Np> __k,
2508 _SimdWrapper<_Tp, _Np>& __lhs,
2509 const __type_identity_t<_SimdWrapper<_Tp, _Np>> __rhs,
2510 _Op __op)
2511 {
2512 if (__k._M_is_constprop_none_of())
2513 return;
2514 else if (__k._M_is_constprop_all_of())
2515 __lhs = __op(_SuperImpl{}, __lhs, __rhs);
2516 else
2517 __lhs = _CommonImpl::_S_blend(__k, __lhs,
2518 __op(_SuperImpl{}, __lhs, __rhs));
2519 }
2520
2521 template <typename _Op, typename _Tp, typename _K, size_t _Np>
2522 _GLIBCXX_SIMD_INTRINSIC static void
2523 _S_masked_cassign(const _SimdWrapper<_K, _Np> __k,
2524 _SimdWrapper<_Tp, _Np>& __lhs,
2525 const __type_identity_t<_Tp> __rhs, _Op __op)
2526 { _S_masked_cassign(__k, __lhs, __vector_broadcast<_Np>(__rhs), __op); }
2527
2528 // _S_masked_unary {{{2
2529 template <template <typename> class _Op, typename _Tp, typename _K,
2530 size_t _Np>
2531 _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
2532 _S_masked_unary(const _SimdWrapper<_K, _Np> __k,
2533 const _SimdWrapper<_Tp, _Np> __v)
2534 {
2535 if (__k._M_is_constprop_none_of())
2536 return __v;
2537 auto __vv = _M_make_simd(__v);
2538 _Op<decltype(__vv)> __op;
2539 if (__k._M_is_constprop_all_of())
2540 return __data(__op(__vv));
2541 else if constexpr (is_same_v<_Op<void>, __increment<void>>)
2542 {
2543 static_assert(not std::is_same_v<_K, bool>);
2544 if constexpr (is_integral_v<_Tp>)
2545 // Take a shortcut knowing that __k is an integer vector with values -1 or 0.
2546 return __v._M_data - __vector_bitcast<_Tp>(__k._M_data);
2547 else if constexpr (not __have_avx2)
2548 return __v._M_data
2549 + __vector_bitcast<_Tp>(__k._M_data & __builtin_bit_cast(
2550 _K, _Tp(1)));
2551 // starting with AVX2 it is more efficient to blend after add
2552 }
2553 else if constexpr (is_same_v<_Op<void>, __decrement<void>>)
2554 {
2555 static_assert(not std::is_same_v<_K, bool>);
2556 if constexpr (is_integral_v<_Tp>)
2557 // Take a shortcut knowing that __k is an integer vector with values -1 or 0.
2558 return __v._M_data + __vector_bitcast<_Tp>(__k._M_data);
2559 else if constexpr (not __have_avx2)
2560 return __v._M_data
2561 - __vector_bitcast<_Tp>(__k._M_data & __builtin_bit_cast(
2562 _K, _Tp(1)));
2563 // starting with AVX2 it is more efficient to blend after sub
2564 }
2565 return _CommonImpl::_S_blend(__k, __v, __data(__op(__vv)));
2566 }
2567
2568 //}}}2
2569 };
2570
2571// _MaskImplBuiltinMixin {{{1
2572struct _MaskImplBuiltinMixin
2573{
2574 template <typename _Tp>
2575 using _TypeTag = _Tp*;
2576
2577 // _S_to_maskvector {{{
2578 template <typename _Up, size_t _ToN = 1>
2579 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Up, _ToN>
2580 _S_to_maskvector(bool __x)
2581 {
2582 static_assert(is_same_v<_Up, __int_for_sizeof_t<_Up>>);
2583 return __x ? __vector_type_t<_Up, _ToN>{~_Up()}
2584 : __vector_type_t<_Up, _ToN>{};
2585 }
2586
2587 template <typename _Up, size_t _UpN = 0, size_t _Np, bool _Sanitized,
2588 size_t _ToN = _UpN == 0 ? _Np : _UpN>
2589 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Up, _ToN>
2590 _S_to_maskvector(_BitMask<_Np, _Sanitized> __x)
2591 {
2592 static_assert(is_same_v<_Up, __int_for_sizeof_t<_Up>>);
2593 return __generate_vector<__vector_type_t<_Up, _ToN>>(
2594 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2595 if constexpr (__i < _Np)
2596 return __x[__i] ? ~_Up() : _Up();
2597 else
2598 return _Up();
2599 });
2600 }
2601
2602 template <typename _Up, size_t _UpN = 0, typename _Tp, size_t _Np,
2603 size_t _ToN = _UpN == 0 ? _Np : _UpN>
2604 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Up, _ToN>
2605 _S_to_maskvector(_SimdWrapper<_Tp, _Np> __x)
2606 {
2607 static_assert(is_same_v<_Up, __int_for_sizeof_t<_Up>>);
2608 using _TW = _SimdWrapper<_Tp, _Np>;
2609 using _UW = _SimdWrapper<_Up, _ToN>;
2610 if constexpr (sizeof(_Up) == sizeof(_Tp) && sizeof(_TW) == sizeof(_UW))
2611 return __wrapper_bitcast<_Up, _ToN>(__x);
2612 else if constexpr (is_same_v<_Tp, bool>) // bits -> vector
2613 return _S_to_maskvector<_Up, _ToN>(_BitMask<_Np>(__x._M_data));
2614 else
2615 { // vector -> vector
2616 /*
2617 [[maybe_unused]] const auto __y = __vector_bitcast<_Up>(__x._M_data);
2618 if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 4 && sizeof(__y) ==
2619 16) return __vector_permute<1, 3, -1, -1>(__y); else if constexpr
2620 (sizeof(_Tp) == 4 && sizeof(_Up) == 2
2621 && sizeof(__y) == 16)
2622 return __vector_permute<1, 3, 5, 7, -1, -1, -1, -1>(__y);
2623 else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 2
2624 && sizeof(__y) == 16)
2625 return __vector_permute<3, 7, -1, -1, -1, -1, -1, -1>(__y);
2626 else if constexpr (sizeof(_Tp) == 2 && sizeof(_Up) == 1
2627 && sizeof(__y) == 16)
2628 return __vector_permute<1, 3, 5, 7, 9, 11, 13, 15, -1, -1, -1, -1,
2629 -1, -1, -1, -1>(__y); else if constexpr (sizeof(_Tp) == 4 &&
2630 sizeof(_Up) == 1
2631 && sizeof(__y) == 16)
2632 return __vector_permute<3, 7, 11, 15, -1, -1, -1, -1, -1, -1, -1,
2633 -1, -1, -1, -1, -1>(__y); else if constexpr (sizeof(_Tp) == 8 &&
2634 sizeof(_Up) == 1
2635 && sizeof(__y) == 16)
2636 return __vector_permute<7, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1,
2637 -1, -1, -1, -1, -1>(__y); else
2638 */
2639 {
2640 return __generate_vector<__vector_type_t<_Up, _ToN>>(
2641 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2642 if constexpr (__i < _Np)
2643 return _Up(__x[__i.value]);
2644 else
2645 return _Up();
2646 });
2647 }
2648 }
2649 }
2650
2651 // }}}
2652 // _S_to_bits {{{
2653 template <typename _Tp, size_t _Np>
2654 _GLIBCXX_SIMD_INTRINSIC static constexpr _SanitizedBitMask<_Np>
2655 _S_to_bits(_SimdWrapper<_Tp, _Np> __x)
2656 {
2657 static_assert(!is_same_v<_Tp, bool>);
2658 static_assert(_Np <= __CHAR_BIT__ * sizeof(_ULLong));
2659 using _Up = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
2660 const auto __bools
2661 = __vector_bitcast<_Up>(__x) >> (sizeof(_Up) * __CHAR_BIT__ - 1);
2662 _ULLong __r = 0;
2663 __execute_n_times<_Np>(
2664 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2665 __r |= _ULLong(__bools[__i.value]) << __i;
2666 });
2667 return __r;
2668 }
2669
2670 // }}}
2671};
2672
2673// _MaskImplBuiltin {{{1
2674template <typename _Abi, typename>
2675 struct _MaskImplBuiltin : _MaskImplBuiltinMixin
2676 {
2677 using _MaskImplBuiltinMixin::_S_to_bits;
2678 using _MaskImplBuiltinMixin::_S_to_maskvector;
2679
2680 // member types {{{
2681 template <typename _Tp>
2682 using _SimdMember = typename _Abi::template __traits<_Tp>::_SimdMember;
2683
2684 template <typename _Tp>
2685 using _MaskMember = typename _Abi::template _MaskMember<_Tp>;
2686
2687 using _SuperImpl = typename _Abi::_MaskImpl;
2688 using _CommonImpl = typename _Abi::_CommonImpl;
2689
2690 template <typename _Tp>
2691 static constexpr size_t _S_size = simd_size_v<_Tp, _Abi>;
2692
2693 // }}}
2694 // _S_broadcast {{{
2695 template <typename _Tp>
2696 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
2697 _S_broadcast(bool __x)
2698 { return __x ? _Abi::template _S_implicit_mask<_Tp>() : _MaskMember<_Tp>(); }
2699
2700 // }}}
2701 // _S_load {{{
2702 template <typename _Tp>
2703 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
2704 _S_load(const bool* __mem)
2705 {
2706 using _I = __int_for_sizeof_t<_Tp>;
2707 if constexpr (sizeof(_Tp) == sizeof(bool))
2708 {
2709 const auto __bools
2710 = _CommonImpl::template _S_load<_I, _S_size<_Tp>>(__mem);
2711 // bool is {0, 1}, everything else is UB
2712 return __bools > 0;
2713 }
2714 else
2715 return __generate_vector<_I, _S_size<_Tp>>(
2716 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2717 return __mem[__i] ? ~_I() : _I();
2718 });
2719 }
2720
2721 // }}}
2722 // _S_convert {{{
2723 template <typename _Tp, size_t _Np, bool _Sanitized>
2724 _GLIBCXX_SIMD_INTRINSIC static constexpr auto
2725 _S_convert(_BitMask<_Np, _Sanitized> __x)
2726 {
2727 if constexpr (__is_builtin_bitmask_abi<_Abi>())
2728 return _SimdWrapper<bool, simd_size_v<_Tp, _Abi>>(__x._M_to_bits());
2729 else
2730 return _SuperImpl::template _S_to_maskvector<__int_for_sizeof_t<_Tp>,
2731 _S_size<_Tp>>(
2732 __x._M_sanitized());
2733 }
2734
2735 template <typename _Tp, size_t _Np>
2736 _GLIBCXX_SIMD_INTRINSIC static constexpr auto
2737 _S_convert(_SimdWrapper<bool, _Np> __x)
2738 {
2739 if constexpr (__is_builtin_bitmask_abi<_Abi>())
2740 return _SimdWrapper<bool, simd_size_v<_Tp, _Abi>>(__x._M_data);
2741 else
2742 return _SuperImpl::template _S_to_maskvector<__int_for_sizeof_t<_Tp>,
2743 _S_size<_Tp>>(
2744 _BitMask<_Np>(__x._M_data)._M_sanitized());
2745 }
2746
2747 template <typename _Tp, typename _Up, size_t _Np>
2748 _GLIBCXX_SIMD_INTRINSIC static constexpr auto
2749 _S_convert(_SimdWrapper<_Up, _Np> __x)
2750 {
2751 if constexpr (__is_builtin_bitmask_abi<_Abi>())
2752 return _SimdWrapper<bool, simd_size_v<_Tp, _Abi>>(
2753 _SuperImpl::_S_to_bits(__x));
2754 else
2755 return _SuperImpl::template _S_to_maskvector<__int_for_sizeof_t<_Tp>,
2756 _S_size<_Tp>>(__x);
2757 }
2758
2759 template <typename _Tp, typename _Up, typename _UAbi>
2760 _GLIBCXX_SIMD_INTRINSIC static constexpr auto
2761 _S_convert(simd_mask<_Up, _UAbi> __x)
2762 {
2763 if constexpr (__is_builtin_bitmask_abi<_Abi>())
2764 {
2765 using _R = _SimdWrapper<bool, simd_size_v<_Tp, _Abi>>;
2766 if constexpr (__is_builtin_bitmask_abi<_UAbi>()) // bits -> bits
2767 return _R(__data(__x));
2768 else if constexpr (__is_scalar_abi<_UAbi>()) // bool -> bits
2769 return _R(__data(__x));
2770 else if constexpr (__is_fixed_size_abi_v<_UAbi>) // bitset -> bits
2771 return _R(__data(__x)._M_to_bits());
2772 else // vector -> bits
2773 return _R(_UAbi::_MaskImpl::_S_to_bits(__data(__x))._M_to_bits());
2774 }
2775 else
2776 return _SuperImpl::template _S_to_maskvector<__int_for_sizeof_t<_Tp>,
2777 _S_size<_Tp>>(
2778 __data(__x));
2779 }
2780
2781 // }}}
2782 // _S_masked_load {{{2
2783 template <typename _Tp, size_t _Np>
2784 static inline _SimdWrapper<_Tp, _Np>
2785 _S_masked_load(_SimdWrapper<_Tp, _Np> __merge,
2786 _SimdWrapper<_Tp, _Np> __mask, const bool* __mem) noexcept
2787 {
2788 // AVX(2) has 32/64 bit maskload, but nothing at 8 bit granularity
2789 auto __tmp = __wrapper_bitcast<__int_for_sizeof_t<_Tp>>(__merge);
2790 _BitOps::_S_bit_iteration(_SuperImpl::_S_to_bits(__mask),
2791 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2792 __tmp._M_set(__i, -__mem[__i]);
2793 });
2794 __merge = __wrapper_bitcast<_Tp>(__tmp);
2795 return __merge;
2796 }
2797
2798 // _S_store {{{2
2799 template <typename _Tp, size_t _Np>
2800 _GLIBCXX_SIMD_INTRINSIC static void
2801 _S_store(_SimdWrapper<_Tp, _Np> __v, bool* __mem) noexcept
2802 {
2803 __execute_n_times<_Np>([&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2804 __mem[__i] = __v[__i];
2805 });
2806 }
2807
2808 // _S_masked_store {{{2
2809 template <typename _Tp, size_t _Np>
2810 static inline void
2811 _S_masked_store(const _SimdWrapper<_Tp, _Np> __v, bool* __mem,
2812 const _SimdWrapper<_Tp, _Np> __k) noexcept
2813 {
2814 _BitOps::_S_bit_iteration(_SuperImpl::_S_to_bits(__k),
2815 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2816 __mem[__i] = __v[__i];
2817 });
2818 }
2819
2820 // _S_from_bitmask{{{2
2821 template <size_t _Np, typename _Tp>
2822 _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
2823 _S_from_bitmask(_SanitizedBitMask<_Np> __bits, _TypeTag<_Tp>)
2824 { return _SuperImpl::template _S_to_maskvector<_Tp, _S_size<_Tp>>(__bits); }
2825
2826 // logical and bitwise operators {{{2
2827 template <typename _Tp, size_t _Np>
2828 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
2829 _S_logical_and(const _SimdWrapper<_Tp, _Np>& __x, const _SimdWrapper<_Tp, _Np>& __y)
2830 { return __and(__x._M_data, __y._M_data); }
2831
2832 template <typename _Tp, size_t _Np>
2833 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
2834 _S_logical_or(const _SimdWrapper<_Tp, _Np>& __x, const _SimdWrapper<_Tp, _Np>& __y)
2835 { return __or(__x._M_data, __y._M_data); }
2836
2837 template <typename _Tp, size_t _Np>
2838 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
2839 _S_bit_not(const _SimdWrapper<_Tp, _Np>& __x)
2840 {
2841 if constexpr (_Abi::template _S_is_partial<_Tp>)
2842 return __andnot(__x, __wrapper_bitcast<_Tp>(
2843 _Abi::template _S_implicit_mask<_Tp>()));
2844 else
2845 return __not(__x._M_data);
2846 }
2847
2848 template <typename _Tp, size_t _Np>
2849 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
2850 _S_bit_and(const _SimdWrapper<_Tp, _Np>& __x, const _SimdWrapper<_Tp, _Np>& __y)
2851 { return __and(__x._M_data, __y._M_data); }
2852
2853 template <typename _Tp, size_t _Np>
2854 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
2855 _S_bit_or(const _SimdWrapper<_Tp, _Np>& __x, const _SimdWrapper<_Tp, _Np>& __y)
2856 { return __or(__x._M_data, __y._M_data); }
2857
2858 template <typename _Tp, size_t _Np>
2859 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
2860 _S_bit_xor(const _SimdWrapper<_Tp, _Np>& __x, const _SimdWrapper<_Tp, _Np>& __y)
2861 { return __xor(__x._M_data, __y._M_data); }
2862
2863 // smart_reference access {{{2
2864 template <typename _Tp, size_t _Np>
2865 static constexpr void
2866 _S_set(_SimdWrapper<_Tp, _Np>& __k, int __i, bool __x) noexcept
2867 {
2868 if constexpr (is_same_v<_Tp, bool>)
2869 __k._M_set(__i, __x);
2870 else
2871 {
2872 static_assert(is_same_v<_Tp, __int_for_sizeof_t<_Tp>>);
2873 if (__builtin_is_constant_evaluated())
2874 {
2875 __k = __generate_from_n_evaluations<_Np,
2876 __vector_type_t<_Tp, _Np>>(
2877 [&](auto __j) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2878 if (__i == static_cast<int>(__j))
2879 return _Tp(-__x);
2880 else
2881 return __k[+__j];
2882 });
2883 }
2884 else
2885 __k._M_data[__i] = -__x;
2886 }
2887 }
2888
2889 // _S_masked_assign{{{2
2890 template <typename _Tp, size_t _Np>
2891 _GLIBCXX_SIMD_INTRINSIC static void
2892 _S_masked_assign(_SimdWrapper<_Tp, _Np> __k, _SimdWrapper<_Tp, _Np>& __lhs,
2893 __type_identity_t<_SimdWrapper<_Tp, _Np>> __rhs)
2894 { __lhs = _CommonImpl::_S_blend(__k, __lhs, __rhs); }
2895
2896 template <typename _Tp, size_t _Np>
2897 _GLIBCXX_SIMD_INTRINSIC static void
2898 _S_masked_assign(_SimdWrapper<_Tp, _Np> __k, _SimdWrapper<_Tp, _Np>& __lhs, bool __rhs)
2899 {
2900 if (__builtin_constant_p(__rhs))
2901 {
2902 if (__rhs == false)
2903 __lhs = __andnot(__k, __lhs);
2904 else
2905 __lhs = __or(__k, __lhs);
2906 return;
2907 }
2908 __lhs = _CommonImpl::_S_blend(__k, __lhs,
2909 __data(simd_mask<_Tp, _Abi>(__rhs)));
2910 }
2911
2912 //}}}2
2913 // _S_all_of {{{
2914 template <typename _Tp>
2915 _GLIBCXX_SIMD_INTRINSIC static bool
2916 _S_all_of(simd_mask<_Tp, _Abi> __k)
2917 {
2918 return __call_with_subscripts(
2919 __data(__k), make_index_sequence<_S_size<_Tp>>(),
2920 [](const auto... __ent) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
2921 { return (... && !(__ent == 0)); });
2922 }
2923
2924 // }}}
2925 // _S_any_of {{{
2926 template <typename _Tp>
2927 _GLIBCXX_SIMD_INTRINSIC static bool
2928 _S_any_of(simd_mask<_Tp, _Abi> __k)
2929 {
2930 return __call_with_subscripts(
2931 __data(__k), make_index_sequence<_S_size<_Tp>>(),
2932 [](const auto... __ent) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
2933 { return (... || !(__ent == 0)); });
2934 }
2935
2936 // }}}
2937 // _S_none_of {{{
2938 template <typename _Tp>
2939 _GLIBCXX_SIMD_INTRINSIC static bool
2940 _S_none_of(simd_mask<_Tp, _Abi> __k)
2941 {
2942 return __call_with_subscripts(
2943 __data(__k), make_index_sequence<_S_size<_Tp>>(),
2944 [](const auto... __ent) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
2945 { return (... && (__ent == 0)); });
2946 }
2947
2948 // }}}
2949 // _S_some_of {{{
2950 template <typename _Tp>
2951 _GLIBCXX_SIMD_INTRINSIC static bool
2952 _S_some_of(simd_mask<_Tp, _Abi> __k)
2953 {
2954 const int __n_true = _SuperImpl::_S_popcount(__k);
2955 return __n_true > 0 && __n_true < int(_S_size<_Tp>);
2956 }
2957
2958 // }}}
2959 // _S_popcount {{{
2960 template <typename _Tp>
2961 _GLIBCXX_SIMD_INTRINSIC static int
2962 _S_popcount(simd_mask<_Tp, _Abi> __k)
2963 {
2964 using _I = __int_for_sizeof_t<_Tp>;
2965 if constexpr (is_default_constructible_v<simd<_I, _Abi>>)
2966 return -reduce(
2967 simd<_I, _Abi>(__private_init, __wrapper_bitcast<_I>(__data(__k))));
2968 else
2969 return -reduce(__bit_cast<rebind_simd_t<_I, simd<_Tp, _Abi>>>(
2970 simd<_Tp, _Abi>(__private_init, __data(__k))));
2971 }
2972
2973 // }}}
2974 // _S_find_first_set {{{
2975 template <typename _Tp>
2976 _GLIBCXX_SIMD_INTRINSIC static int
2977 _S_find_first_set(simd_mask<_Tp, _Abi> __k)
2978 { return std::__countr_zero(_SuperImpl::_S_to_bits(__data(__k))._M_to_bits()); }
2979
2980 // }}}
2981 // _S_find_last_set {{{
2982 template <typename _Tp>
2983 _GLIBCXX_SIMD_INTRINSIC static int
2984 _S_find_last_set(simd_mask<_Tp, _Abi> __k)
2985 { return std::__bit_width(_SuperImpl::_S_to_bits(__data(__k))._M_to_bits()) - 1; }
2986
2987 // }}}
2988 };
2989
2990//}}}1
2991_GLIBCXX_SIMD_END_NAMESPACE
2992#endif // __cplusplus >= 201703L
2993#endif // _GLIBCXX_EXPERIMENTAL_SIMD_ABIS_H_
2994
2995// vim: foldmethod=marker foldmarker={{{,}}} sw=2 noet ts=8 sts=2 tw=100
std::true_type
integral_constant< bool, true > true_type
The type used as a compile-time boolean with true value.
Definition: type_traits:82
std::conditional_t
typename conditional< _Cond, _Iftrue, _Iffalse >::type conditional_t
Alias template for conditional.
Definition: type_traits:2614
std::min
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
Definition: stl_algobase.h:233
std::reduce
constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op)
Calculate reduction of values in a range.
Definition: numeric:287