Grok 10.0.5
traits-inl.h
Go to the documentation of this file.
1// Copyright 2021 Google LLC
2// SPDX-License-Identifier: Apache-2.0
3//
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at
7//
8// http://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15
16// Per-target
17#if defined(HIGHWAY_HWY_CONTRIB_SORT_TRAITS_TOGGLE) == \
18 defined(HWY_TARGET_TOGGLE)
19#ifdef HIGHWAY_HWY_CONTRIB_SORT_TRAITS_TOGGLE
20#undef HIGHWAY_HWY_CONTRIB_SORT_TRAITS_TOGGLE
21#else
22#define HIGHWAY_HWY_CONTRIB_SORT_TRAITS_TOGGLE
23#endif
24
25#include <string>
26
27#include "hwy/contrib/sort/shared-inl.h" // SortConstants
28#include "hwy/contrib/sort/vqsort.h" // SortDescending
29#include "hwy/highway.h"
30#include "hwy/print.h"
31
33namespace hwy {
34namespace HWY_NAMESPACE {
35namespace detail {
36
37#if VQSORT_ENABLED || HWY_IDE
38
39// Highway does not provide a lane type for 128-bit keys, so we use uint64_t
40// along with an abstraction layer for single-lane vs. lane-pair, which is
41// independent of the order.
42template <typename T>
43struct KeyLane {
44 static constexpr bool Is128() { return false; }
45 // False indicates the entire key (i.e. lane) should be compared. KV stands
46 // for key-value.
47 static constexpr bool IsKV() { return false; }
48 constexpr size_t LanesPerKey() const { return 1; }
49
50 // What type bench_sort should allocate for generating inputs.
51 using LaneType = T;
52 // What type to pass to Sorter::operator().
53 using KeyType = T;
54
55 std::string KeyString() const {
56 char string100[100];
57 hwy::detail::TypeName(hwy::detail::MakeTypeInfo<KeyType>(), 1, string100);
58 return string100;
59 }
60
61 // For HeapSort
62 HWY_INLINE void Swap(T* a, T* b) const {
63 const T temp = *a;
64 *a = *b;
65 *b = temp;
66 }
67
68 template <class V, class M>
69 HWY_INLINE V CompressKeys(V keys, M mask) const {
70 return CompressNot(keys, mask);
71 }
72
73 // Broadcasts one key into a vector
74 template <class D>
75 HWY_INLINE Vec<D> SetKey(D d, const T* key) const {
76 return Set(d, *key);
77 }
78
79 template <class D>
80 HWY_INLINE Mask<D> EqualKeys(D /*tag*/, Vec<D> a, Vec<D> b) const {
81 return Eq(a, b);
82 }
83
84 template <class D>
85 HWY_INLINE Mask<D> NotEqualKeys(D /*tag*/, Vec<D> a, Vec<D> b) const {
86 return Ne(a, b);
87 }
88
89 // For keys=lanes, any difference counts.
90 template <class D>
91 HWY_INLINE bool NoKeyDifference(D /*tag*/, Vec<D> diff) const {
92 // Must avoid floating-point comparisons (for -0)
93 const RebindToUnsigned<D> du;
94 return AllTrue(du, Eq(BitCast(du, diff), Zero(du)));
95 }
96
97 HWY_INLINE bool Equal1(const T* a, const T* b) const { return *a == *b; }
98
99 template <class D>
100 HWY_INLINE Vec<D> ReverseKeys(D d, Vec<D> v) const {
101 return Reverse(d, v);
102 }
103
104 template <class D>
105 HWY_INLINE Vec<D> ReverseKeys2(D d, Vec<D> v) const {
106 return Reverse2(d, v);
107 }
108
109 template <class D>
110 HWY_INLINE Vec<D> ReverseKeys4(D d, Vec<D> v) const {
111 return Reverse4(d, v);
112 }
113
114 template <class D>
115 HWY_INLINE Vec<D> ReverseKeys8(D d, Vec<D> v) const {
116 return Reverse8(d, v);
117 }
118
119 template <class D>
120 HWY_INLINE Vec<D> ReverseKeys16(D d, Vec<D> v) const {
121 static_assert(SortConstants::kMaxCols <= 16, "Assumes u32x16 = 512 bit");
122 return ReverseKeys(d, v);
123 }
124
125 template <class V>
126 HWY_INLINE V OddEvenKeys(const V odd, const V even) const {
127 return OddEven(odd, even);
128 }
129
130 template <class D, HWY_IF_LANE_SIZE_D(D, 2)>
131 HWY_INLINE Vec<D> SwapAdjacentPairs(D d, const Vec<D> v) const {
132 const Repartition<uint32_t, D> du32;
133 return BitCast(d, Shuffle2301(BitCast(du32, v)));
134 }
135 template <class D, HWY_IF_LANE_SIZE_D(D, 4)>
136 HWY_INLINE Vec<D> SwapAdjacentPairs(D /* tag */, const Vec<D> v) const {
137 return Shuffle1032(v);
138 }
139 template <class D, HWY_IF_LANE_SIZE_D(D, 8)>
140 HWY_INLINE Vec<D> SwapAdjacentPairs(D /* tag */, const Vec<D> v) const {
141 return SwapAdjacentBlocks(v);
142 }
143
144 template <class D, HWY_IF_NOT_LANE_SIZE_D(D, 8)>
145 HWY_INLINE Vec<D> SwapAdjacentQuads(D d, const Vec<D> v) const {
146#if HWY_HAVE_FLOAT64 // in case D is float32
147 const RepartitionToWide<D> dw;
148#else
149 const RepartitionToWide<RebindToUnsigned<D> > dw;
150#endif
151 return BitCast(d, SwapAdjacentPairs(dw, BitCast(dw, v)));
152 }
153 template <class D, HWY_IF_LANE_SIZE_D(D, 8)>
154 HWY_INLINE Vec<D> SwapAdjacentQuads(D d, const Vec<D> v) const {
155 // Assumes max vector size = 512
156 return ConcatLowerUpper(d, v, v);
157 }
158
159 template <class D, HWY_IF_NOT_LANE_SIZE_D(D, 8)>
160 HWY_INLINE Vec<D> OddEvenPairs(D d, const Vec<D> odd,
161 const Vec<D> even) const {
162#if HWY_HAVE_FLOAT64 // in case D is float32
163 const RepartitionToWide<D> dw;
164#else
165 const RepartitionToWide<RebindToUnsigned<D> > dw;
166#endif
167 return BitCast(d, OddEven(BitCast(dw, odd), BitCast(dw, even)));
168 }
169 template <class D, HWY_IF_LANE_SIZE_D(D, 8)>
170 HWY_INLINE Vec<D> OddEvenPairs(D /* tag */, Vec<D> odd, Vec<D> even) const {
171 return OddEvenBlocks(odd, even);
172 }
173
174 template <class D, HWY_IF_NOT_LANE_SIZE_D(D, 8)>
175 HWY_INLINE Vec<D> OddEvenQuads(D d, Vec<D> odd, Vec<D> even) const {
176#if HWY_HAVE_FLOAT64 // in case D is float32
177 const RepartitionToWide<D> dw;
178#else
179 const RepartitionToWide<RebindToUnsigned<D> > dw;
180#endif
181 return BitCast(d, OddEvenPairs(dw, BitCast(dw, odd), BitCast(dw, even)));
182 }
183 template <class D, HWY_IF_LANE_SIZE_D(D, 8)>
184 HWY_INLINE Vec<D> OddEvenQuads(D d, Vec<D> odd, Vec<D> even) const {
185 return ConcatUpperLower(d, odd, even);
186 }
187};
188
189// Anything order-related depends on the key traits *and* the order (see
190// FirstOfLanes). We cannot implement just one Compare function because Lt128
191// only compiles if the lane type is u64. Thus we need either overloaded
192// functions with a tag type, class specializations, or separate classes.
193// We avoid overloaded functions because we want all functions to be callable
194// from a SortTraits without per-function wrappers. Specializing would work, but
195// we are anyway going to specialize at a higher level.
196template <typename T>
197struct OrderAscending : public KeyLane<T> {
198 using Order = SortAscending;
199
200 HWY_INLINE bool Compare1(const T* a, const T* b) { return *a < *b; }
201
202 template <class D>
203 HWY_INLINE Mask<D> Compare(D /* tag */, Vec<D> a, Vec<D> b) const {
204 return Lt(a, b);
205 }
206
207 // Two halves of Sort2, used in ScanMinMax.
208 template <class D>
209 HWY_INLINE Vec<D> First(D /* tag */, const Vec<D> a, const Vec<D> b) const {
210 return Min(a, b);
211 }
212
213 template <class D>
214 HWY_INLINE Vec<D> Last(D /* tag */, const Vec<D> a, const Vec<D> b) const {
215 return Max(a, b);
216 }
217
218 template <class D>
219 HWY_INLINE Vec<D> FirstOfLanes(D d, Vec<D> v,
220 T* HWY_RESTRICT /* buf */) const {
221 return MinOfLanes(d, v);
222 }
223
224 template <class D>
225 HWY_INLINE Vec<D> LastOfLanes(D d, Vec<D> v,
226 T* HWY_RESTRICT /* buf */) const {
227 return MaxOfLanes(d, v);
228 }
229
230 template <class D>
231 HWY_INLINE Vec<D> FirstValue(D d) const {
232 return Set(d, hwy::LowestValue<T>());
233 }
234
235 template <class D>
236 HWY_INLINE Vec<D> LastValue(D d) const {
237 return Set(d, hwy::HighestValue<T>());
238 }
239
240 template <class D>
241 HWY_INLINE Vec<D> PrevValue(D d, Vec<D> v) const {
242 return Sub(v, Set(d, hwy::Epsilon<T>()));
243 }
244};
245
246template <typename T>
247struct OrderDescending : public KeyLane<T> {
248 using Order = SortDescending;
249
250 HWY_INLINE bool Compare1(const T* a, const T* b) { return *b < *a; }
251
252 template <class D>
253 HWY_INLINE Mask<D> Compare(D /* tag */, Vec<D> a, Vec<D> b) const {
254 return Lt(b, a);
255 }
256
257 template <class D>
258 HWY_INLINE Vec<D> First(D /* tag */, const Vec<D> a, const Vec<D> b) const {
259 return Max(a, b);
260 }
261
262 template <class D>
263 HWY_INLINE Vec<D> Last(D /* tag */, const Vec<D> a, const Vec<D> b) const {
264 return Min(a, b);
265 }
266
267 template <class D>
268 HWY_INLINE Vec<D> FirstOfLanes(D d, Vec<D> v,
269 T* HWY_RESTRICT /* buf */) const {
270 return MaxOfLanes(d, v);
271 }
272
273 template <class D>
274 HWY_INLINE Vec<D> LastOfLanes(D d, Vec<D> v,
275 T* HWY_RESTRICT /* buf */) const {
276 return MinOfLanes(d, v);
277 }
278
279 template <class D>
280 HWY_INLINE Vec<D> FirstValue(D d) const {
281 return Set(d, hwy::HighestValue<T>());
282 }
283
284 template <class D>
285 HWY_INLINE Vec<D> LastValue(D d) const {
286 return Set(d, hwy::LowestValue<T>());
287 }
288
289 template <class D>
290 HWY_INLINE Vec<D> PrevValue(D d, Vec<D> v) const {
291 return Add(v, Set(d, hwy::Epsilon<T>()));
292 }
293};
294
295struct KeyValue64 : public KeyLane<uint64_t> {
296 // True indicates only part of the key (i.e. lane) should be compared. KV
297 // stands for key-value.
298 static constexpr bool IsKV() { return true; }
299
300 template <class D>
301 HWY_INLINE Mask<D> EqualKeys(D /*tag*/, Vec<D> a, Vec<D> b) const {
302 return Eq(ShiftRight<32>(a), ShiftRight<32>(b));
303 }
304
305 template <class D>
306 HWY_INLINE Mask<D> NotEqualKeys(D /*tag*/, Vec<D> a, Vec<D> b) const {
307 return Ne(ShiftRight<32>(a), ShiftRight<32>(b));
308 }
309
310 HWY_INLINE bool Equal1(const uint64_t* a, const uint64_t* b) const {
311 return (*a >> 32) == (*b >> 32);
312 }
313
314 // Only count differences in the actual key, not the value.
315 template <class D>
316 HWY_INLINE bool NoKeyDifference(D /*tag*/, Vec<D> diff) const {
317 // Must avoid floating-point comparisons (for -0)
318 const RebindToUnsigned<D> du;
319 const Vec<decltype(du)> zero = Zero(du);
320 const Vec<decltype(du)> keys = ShiftRight<32>(diff); // clear values
321 return AllTrue(du, Eq(BitCast(du, keys), zero));
322 }
323};
324
325struct OrderAscendingKV64 : public KeyValue64 {
326 using Order = SortAscending;
327
328 HWY_INLINE bool Compare1(const LaneType* a, const LaneType* b) {
329 return (*a >> 32) < (*b >> 32);
330 }
331
332 template <class D>
333 HWY_INLINE Mask<D> Compare(D /* tag */, Vec<D> a, Vec<D> b) const {
334 return Lt(ShiftRight<32>(a), ShiftRight<32>(b));
335 }
336
337 // Not required to be stable (preserving the order of equivalent keys), so
338 // we can include the value in the comparison.
339 template <class D>
340 HWY_INLINE Vec<D> First(D /* tag */, const Vec<D> a, const Vec<D> b) const {
341 return Min(a, b);
342 }
343
344 template <class D>
345 HWY_INLINE Vec<D> Last(D /* tag */, const Vec<D> a, const Vec<D> b) const {
346 return Max(a, b);
347 }
348
349 template <class D>
350 HWY_INLINE Vec<D> FirstOfLanes(D d, Vec<D> v,
351 uint64_t* HWY_RESTRICT /* buf */) const {
352 return MinOfLanes(d, v);
353 }
354
355 template <class D>
356 HWY_INLINE Vec<D> LastOfLanes(D d, Vec<D> v,
357 uint64_t* HWY_RESTRICT /* buf */) const {
358 return MaxOfLanes(d, v);
359 }
360
361 // Same as for regular lanes.
362 template <class D>
363 HWY_INLINE Vec<D> FirstValue(D d) const {
364 return Set(d, hwy::LowestValue<TFromD<D> >());
365 }
366
367 template <class D>
368 HWY_INLINE Vec<D> LastValue(D d) const {
369 return Set(d, hwy::HighestValue<TFromD<D> >());
370 }
371
372 template <class D>
373 HWY_INLINE Vec<D> PrevValue(D d, Vec<D> v) const {
374 return Sub(v, Set(d, uint64_t{1}));
375 }
376};
377
378struct OrderDescendingKV64 : public KeyValue64 {
379 using Order = SortDescending;
380
381 HWY_INLINE bool Compare1(const LaneType* a, const LaneType* b) {
382 return (*b >> 32) < (*a >> 32);
383 }
384
385 template <class D>
386 HWY_INLINE Mask<D> Compare(D /* tag */, Vec<D> a, Vec<D> b) const {
387 return Lt(ShiftRight<32>(b), ShiftRight<32>(a));
388 }
389
390 // Not required to be stable (preserving the order of equivalent keys), so
391 // we can include the value in the comparison.
392 template <class D>
393 HWY_INLINE Vec<D> First(D /* tag */, const Vec<D> a, const Vec<D> b) const {
394 return Max(a, b);
395 }
396
397 template <class D>
398 HWY_INLINE Vec<D> Last(D /* tag */, const Vec<D> a, const Vec<D> b) const {
399 return Min(a, b);
400 }
401
402 template <class D>
403 HWY_INLINE Vec<D> FirstOfLanes(D d, Vec<D> v,
404 uint64_t* HWY_RESTRICT /* buf */) const {
405 return MaxOfLanes(d, v);
406 }
407
408 template <class D>
409 HWY_INLINE Vec<D> LastOfLanes(D d, Vec<D> v,
410 uint64_t* HWY_RESTRICT /* buf */) const {
411 return MinOfLanes(d, v);
412 }
413
414 template <class D>
415 HWY_INLINE Vec<D> FirstValue(D d) const {
416 return Set(d, hwy::HighestValue<TFromD<D> >());
417 }
418
419 template <class D>
420 HWY_INLINE Vec<D> LastValue(D d) const {
421 return Set(d, hwy::LowestValue<TFromD<D> >());
422 }
423
424 template <class D>
425 HWY_INLINE Vec<D> PrevValue(D d, Vec<D> v) const {
426 return Add(v, Set(d, uint64_t{1}));
427 }
428};
429
430// Shared code that depends on Order.
431template <class Base>
432struct TraitsLane : public Base {
433 // For each lane i: replaces a[i] with the first and b[i] with the second
434 // according to Base.
435 // Corresponds to a conditional swap, which is one "node" of a sorting
436 // network. Min/Max are cheaper than compare + blend at least for integers.
437 template <class D>
438 HWY_INLINE void Sort2(D d, Vec<D>& a, Vec<D>& b) const {
439 const Base* base = static_cast<const Base*>(this);
440
441 const Vec<D> a_copy = a;
442 // Prior to AVX3, there is no native 64-bit Min/Max, so they compile to 4
443 // instructions. We can reduce it to a compare + 2 IfThenElse.
444#if HWY_AVX3 < HWY_TARGET && HWY_TARGET <= HWY_SSSE3
445 if (sizeof(TFromD<D>) == 8) {
446 const Mask<D> cmp = base->Compare(d, a, b);
447 a = IfThenElse(cmp, a, b);
448 b = IfThenElse(cmp, b, a_copy);
449 return;
450 }
451#endif
452 a = base->First(d, a, b);
453 b = base->Last(d, a_copy, b);
454 }
455
456 // Conditionally swaps even-numbered lanes with their odd-numbered neighbor.
457 template <class D, HWY_IF_LANE_SIZE_D(D, 8)>
458 HWY_INLINE Vec<D> SortPairsDistance1(D d, Vec<D> v) const {
459 const Base* base = static_cast<const Base*>(this);
460 Vec<D> swapped = base->ReverseKeys2(d, v);
461 // Further to the above optimization, Sort2+OddEvenKeys compile to four
462 // instructions; we can save one by combining two blends.
463#if HWY_AVX3 < HWY_TARGET && HWY_TARGET <= HWY_SSSE3
464 const Vec<D> cmp = VecFromMask(d, base->Compare(d, v, swapped));
465 return IfVecThenElse(DupOdd(cmp), swapped, v);
466#else
467 Sort2(d, v, swapped);
468 return base->OddEvenKeys(swapped, v);
469#endif
470 }
471
472 // (See above - we use Sort2 for non-64-bit types.)
473 template <class D, HWY_IF_NOT_LANE_SIZE_D(D, 8)>
474 HWY_INLINE Vec<D> SortPairsDistance1(D d, Vec<D> v) const {
475 const Base* base = static_cast<const Base*>(this);
476 Vec<D> swapped = base->ReverseKeys2(d, v);
477 Sort2(d, v, swapped);
478 return base->OddEvenKeys(swapped, v);
479 }
480
481 // Swaps with the vector formed by reversing contiguous groups of 4 keys.
482 template <class D>
483 HWY_INLINE Vec<D> SortPairsReverse4(D d, Vec<D> v) const {
484 const Base* base = static_cast<const Base*>(this);
485 Vec<D> swapped = base->ReverseKeys4(d, v);
486 Sort2(d, v, swapped);
487 return base->OddEvenPairs(d, swapped, v);
488 }
489
490 // Conditionally swaps lane 0 with 4, 1 with 5 etc.
491 template <class D>
492 HWY_INLINE Vec<D> SortPairsDistance4(D d, Vec<D> v) const {
493 const Base* base = static_cast<const Base*>(this);
494 Vec<D> swapped = base->SwapAdjacentQuads(d, v);
495 // Only used in Merge16, so this will not be used on AVX2 (which only has 4
496 // u64 lanes), so skip the above optimization for 64-bit AVX2.
497 Sort2(d, v, swapped);
498 return base->OddEvenQuads(d, swapped, v);
499 }
500};
501
502#else
503
504// Base class shared between OrderAscending, OrderDescending.
505template <typename T>
506struct KeyLane {
507 constexpr bool Is128() const { return false; }
508 constexpr size_t LanesPerKey() const { return 1; }
509
510 using LaneType = T;
511 using KeyType = T;
512
513 std::string KeyString() const {
514 char string100[100];
515 hwy::detail::TypeName(hwy::detail::MakeTypeInfo<KeyType>(), 1, string100);
516 return string100;
517 }
518};
519
520template <typename T>
521struct OrderAscending : public KeyLane<T> {
523
524 HWY_INLINE bool Compare1(const T* a, const T* b) { return *a < *b; }
525
526 template <class D>
528 return Lt(a, b);
529 }
530};
531
532template <typename T>
533struct OrderDescending : public KeyLane<T> {
535
536 HWY_INLINE bool Compare1(const T* a, const T* b) { return *b < *a; }
537
538 template <class D>
540 return Lt(b, a);
541 }
542};
543
544template <class Order>
545struct TraitsLane : public Order {
546 // For HeapSort
547 template <typename T> // MSVC doesn't find typename Order::LaneType.
548 HWY_INLINE void Swap(T* a, T* b) const {
549 const T temp = *a;
550 *a = *b;
551 *b = temp;
552 }
553
554 template <class D>
555 HWY_INLINE Vec<D> SetKey(D d, const TFromD<D>* key) const {
556 return Set(d, *key);
557 }
558};
559
560#endif // VQSORT_ENABLED
561
562} // namespace detail
563// NOLINTNEXTLINE(google-readability-namespace-comments)
564} // namespace HWY_NAMESPACE
565} // namespace hwy
567
568#endif // HIGHWAY_HWY_CONTRIB_SORT_TRAITS_TOGGLE
#define HWY_RESTRICT
Definition base.h:64
#define HWY_INLINE
Definition base.h:70
HWY_API Vec128< T, N > Shuffle2301(const Vec128< T, N > a, const Vec128< T, N > b)
Definition wasm_128-inl.h:2413
HWY_INLINE bool AllTrue(hwy::SizeTag< 1 >, const Mask128< T > m)
Definition wasm_128-inl.h:3661
HWY_INLINE Vec128< T, N > Add(hwy::NonFloatTag, Vec128< T, N > a, Vec128< T, N > b)
Definition emu128-inl.h:535
HWY_INLINE Vec128< T, N > OddEven(hwy::SizeTag< 1 >, const Vec128< T, N > a, const Vec128< T, N > b)
Definition wasm_128-inl.h:3023
HWY_INLINE Vec128< T, N > Min(hwy::NonFloatTag, Vec128< T, N > a, const Vec128< T, N > b)
Definition emu128-inl.h:663
HWY_INLINE Vec128< T, 1 > MinOfLanes(hwy::SizeTag< sizeof(T)>, const Vec128< T, 1 > v)
Definition arm_neon-inl.h:5063
HWY_INLINE Vec128< T, N > CompressNot(Vec128< T, N > v, const uint64_t mask_bits)
Definition arm_neon-inl.h:6162
HWY_INLINE Vec128< T, N > Sub(hwy::NonFloatTag, Vec128< T, N > a, Vec128< T, N > b)
Definition emu128-inl.h:545
HWY_INLINE Vec128< T, N > Max(hwy::NonFloatTag, Vec128< T, N > a, const Vec128< T, N > b)
Definition emu128-inl.h:671
HWY_INLINE Vec128< T, N > IfThenElse(hwy::SizeTag< 1 >, Mask128< T, N > mask, Vec128< T, N > yes, Vec128< T, N > no)
Definition x86_128-inl.h:670
HWY_INLINE Vec128< T, 1 > MaxOfLanes(hwy::SizeTag< sizeof(T)>, const Vec128< T, 1 > v)
Definition arm_neon-inl.h:5068
d
Definition rvv-inl.h:1998
HWY_API Vec128< T, N > OddEvenBlocks(Vec128< T, N >, Vec128< T, N > even)
Definition arm_neon-inl.h:4697
HWY_API Vec128< T, N > DupOdd(Vec128< T, N > v)
Definition arm_neon-inl.h:4662
HWY_API bool AllTrue(const Full128< T > d, const Mask128< T > m)
Definition arm_neon-inl.h:5716
HWY_API Vec128< T > Shuffle1032(const Vec128< T > v)
Definition arm_neon-inl.h:4131
HWY_API Vec128< uint64_t, N > Min(const Vec128< uint64_t, N > a, const Vec128< uint64_t, N > b)
Definition arm_neon-inl.h:2517
HWY_API Vec128< uint64_t, N > Max(const Vec128< uint64_t, N > a, const Vec128< uint64_t, N > b)
Definition arm_neon-inl.h:2555
HWY_API Vec128< T, N > ConcatLowerUpper(const Simd< T, N, 0 > d, Vec128< T, N > hi, Vec128< T, N > lo)
Definition arm_neon-inl.h:4544
HWY_API Vec128< T, N > IfVecThenElse(Vec128< T, N > mask, Vec128< T, N > yes, Vec128< T, N > no)
Definition arm_neon-inl.h:2047
HWY_API Vec128< T, N > VecFromMask(Simd< T, N, 0 > d, const Mask128< T, N > v)
Definition arm_neon-inl.h:2223
HWY_API Vec128< T, N > SwapAdjacentBlocks(Vec128< T, N > v)
Definition arm_neon-inl.h:4704
HWY_API Vec128< T, N > Reverse2(Simd< T, N, 0 > d, const Vec128< T, N > v)
Definition arm_neon-inl.h:4061
svuint16_t Set(Simd< bfloat16_t, N, kPow2 > d, bfloat16_t arg)
Definition arm_sve-inl.h:322
HWY_API Vec128< T, N > Reverse8(Simd< T, N, 0 > d, const Vec128< T, N > v)
Definition arm_neon-inl.h:4113
HWY_API Vec128< T, N > ConcatUpperLower(Simd< T, N, 0 > d, Vec128< T, N > hi, Vec128< T, N > lo)
Definition arm_neon-inl.h:4570
HWY_API Vec128< T, N > BitCast(Simd< T, N, 0 > d, Vec128< FromT, N *sizeof(T)/sizeof(FromT)> v)
Definition arm_neon-inl.h:997
HWY_API Vec128< T, N > Zero(Simd< T, N, 0 > d)
Definition arm_neon-inl.h:1020
decltype(MaskFromVec(Zero(D()))) Mask
Definition generic_ops-inl.h:46
HWY_API Vec128< T, N > Reverse4(Simd< T, N, 0 > d, const Vec128< T, N > v)
Definition arm_neon-inl.h:4090
HWY_API Vec128< T, 1 > Reverse(Simd< T, 1, 0 >, const Vec128< T, 1 > v)
Definition arm_neon-inl.h:4030
const vfloat64m1_t v
Definition rvv-inl.h:1998
typename D::T TFromD
Definition ops/shared-inl.h:203
decltype(Zero(D())) Vec
Definition generic_ops-inl.h:40
HWY_DLLEXPORT void TypeName(const TypeInfo &info, size_t N, char *string100)
Definition aligned_allocator.h:27
HWY_API constexpr T HighestValue()
Definition base.h:684
HWY_API constexpr T LowestValue()
Definition base.h:671
#define HWY_NAMESPACE
Definition set_macros-inl.h:82
Definition traits-inl.h:506
T LaneType
Definition traits-inl.h:510
constexpr size_t LanesPerKey() const
Definition traits-inl.h:508
T KeyType
Definition traits-inl.h:511
constexpr bool Is128() const
Definition traits-inl.h:507
std::string KeyString() const
Definition traits-inl.h:513
HWY_INLINE bool Compare1(const T *a, const T *b)
Definition traits-inl.h:524
HWY_INLINE Mask< D > Compare(D, Vec< D > a, Vec< D > b)
Definition traits-inl.h:527
SortAscending Order
Definition traits-inl.h:522
HWY_INLINE bool Compare1(const T *a, const T *b)
Definition traits-inl.h:536
HWY_INLINE Mask< D > Compare(D, Vec< D > a, Vec< D > b)
Definition traits-inl.h:539
SortDescending Order
Definition traits-inl.h:534
Definition traits-inl.h:545
HWY_INLINE void Swap(T *a, T *b) const
Definition traits-inl.h:548
HWY_INLINE Vec< D > SetKey(D d, const TFromD< D > *key) const
Definition traits-inl.h:555
Definition vqsort.h:32
static constexpr size_t kMaxCols
Definition contrib/sort/shared-inl.h:34
Definition vqsort.h:35
HWY_AFTER_NAMESPACE()
HWY_BEFORE_NAMESPACE()