AVX512/PacketMath.h
Go to the documentation of this file.
1 // This file is part of Eigen, a lightweight C++ template library
2 // for linear algebra.
3 //
4 // Copyright (C) 2016 Benoit Steiner (benoit.steiner.goog@gmail.com)
5 //
6 // This Source Code Form is subject to the terms of the Mozilla
7 // Public License v. 2.0. If a copy of the MPL was not distributed
8 // with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9 
10 #ifndef EIGEN_PACKET_MATH_AVX512_H
11 #define EIGEN_PACKET_MATH_AVX512_H
12 
13 namespace Eigen {
14 
15 namespace internal {
16 
17 #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
18 #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
19 #endif
20 
21 #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
22 #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS (2*sizeof(void*))
23 #endif
24 
25 #ifdef __FMA__
26 #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
27 #define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
28 #endif
29 #endif
30 
31 typedef __m512 Packet16f;
32 typedef __m512i Packet16i;
33 typedef __m512d Packet8d;
34 
35 template <>
36 struct is_arithmetic<__m512> {
37  enum { value = true };
38 };
39 template <>
40 struct is_arithmetic<__m512i> {
41  enum { value = true };
42 };
43 template <>
44 struct is_arithmetic<__m512d> {
45  enum { value = true };
46 };
47 
48 template<> struct packet_traits<float> : default_packet_traits
49 {
50  typedef Packet16f type;
51  typedef Packet8f half;
52  enum {
53  Vectorizable = 1,
54  AlignedOnScalar = 1,
55  size = 16,
56  HasHalfPacket = 1,
57 #if EIGEN_GNUC_AT_LEAST(5, 3)
58 #ifdef EIGEN_VECTORIZE_AVX512DQ
59  HasLog = 1,
60 #endif
61  HasExp = 1,
62  HasSqrt = 1,
63  HasRsqrt = 1,
64 #endif
65  HasDiv = 1
66  };
67  };
68 template<> struct packet_traits<double> : default_packet_traits
69 {
70  typedef Packet8d type;
71  typedef Packet4d half;
72  enum {
73  Vectorizable = 1,
74  AlignedOnScalar = 1,
75  size = 8,
76  HasHalfPacket = 1,
77 #if EIGEN_GNUC_AT_LEAST(5, 3)
78  HasSqrt = 1,
79  HasRsqrt = EIGEN_FAST_MATH,
80 #endif
81  HasDiv = 1
82  };
83 };
84 
85 /* TODO Implement AVX512 for integers
86 template<> struct packet_traits<int> : default_packet_traits
87 {
88  typedef Packet16i type;
89  enum {
90  Vectorizable = 1,
91  AlignedOnScalar = 1,
92  size=8
93  };
94 };
95 */
96 
97 template <>
98 struct unpacket_traits<Packet16f> {
99  typedef float type;
100  typedef Packet8f half;
101  enum { size = 16, alignment=Aligned64 };
102 };
103 template <>
104 struct unpacket_traits<Packet8d> {
105  typedef double type;
106  typedef Packet4d half;
107  enum { size = 8, alignment=Aligned64 };
108 };
109 template <>
110 struct unpacket_traits<Packet16i> {
111  typedef int type;
112  typedef Packet8i half;
113  enum { size = 16, alignment=Aligned64 };
114 };
115 
116 template <>
117 EIGEN_STRONG_INLINE Packet16f pset1<Packet16f>(const float& from) {
118  return _mm512_set1_ps(from);
119 }
120 template <>
121 EIGEN_STRONG_INLINE Packet8d pset1<Packet8d>(const double& from) {
122  return _mm512_set1_pd(from);
123 }
124 template <>
125 EIGEN_STRONG_INLINE Packet16i pset1<Packet16i>(const int& from) {
126  return _mm512_set1_epi32(from);
127 }
128 
129 template <>
130 EIGEN_STRONG_INLINE Packet16f pload1<Packet16f>(const float* from) {
131  return _mm512_broadcastss_ps(_mm_load_ps1(from));
132 }
133 template <>
134 EIGEN_STRONG_INLINE Packet8d pload1<Packet8d>(const double* from) {
135  return _mm512_broadcastsd_pd(_mm_load_pd1(from));
136 }
137 
138 template <>
139 EIGEN_STRONG_INLINE Packet16f plset<Packet16f>(const float& a) {
140  return _mm512_add_ps(
141  _mm512_set1_ps(a),
142  _mm512_set_ps(15.0f, 14.0f, 13.0f, 12.0f, 11.0f, 10.0f, 9.0f, 8.0f, 7.0f, 6.0f, 5.0f,
143  4.0f, 3.0f, 2.0f, 1.0f, 0.0f));
144 }
145 template <>
146 EIGEN_STRONG_INLINE Packet8d plset<Packet8d>(const double& a) {
147  return _mm512_add_pd(_mm512_set1_pd(a),
148  _mm512_set_pd(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0));
149 }
150 
151 template <>
152 EIGEN_STRONG_INLINE Packet16f padd<Packet16f>(const Packet16f& a,
153  const Packet16f& b) {
154  return _mm512_add_ps(a, b);
155 }
156 template <>
157 EIGEN_STRONG_INLINE Packet8d padd<Packet8d>(const Packet8d& a,
158  const Packet8d& b) {
159  return _mm512_add_pd(a, b);
160 }
161 
162 template <>
163 EIGEN_STRONG_INLINE Packet16f psub<Packet16f>(const Packet16f& a,
164  const Packet16f& b) {
165  return _mm512_sub_ps(a, b);
166 }
167 template <>
168 EIGEN_STRONG_INLINE Packet8d psub<Packet8d>(const Packet8d& a,
169  const Packet8d& b) {
170  return _mm512_sub_pd(a, b);
171 }
172 
173 template <>
174 EIGEN_STRONG_INLINE Packet16f pnegate(const Packet16f& a) {
175  return _mm512_sub_ps(_mm512_set1_ps(0.0), a);
176 }
177 template <>
178 EIGEN_STRONG_INLINE Packet8d pnegate(const Packet8d& a) {
179  return _mm512_sub_pd(_mm512_set1_pd(0.0), a);
180 }
181 
182 template <>
183 EIGEN_STRONG_INLINE Packet16f pconj(const Packet16f& a) {
184  return a;
185 }
186 template <>
187 EIGEN_STRONG_INLINE Packet8d pconj(const Packet8d& a) {
188  return a;
189 }
190 template <>
191 EIGEN_STRONG_INLINE Packet16i pconj(const Packet16i& a) {
192  return a;
193 }
194 
195 template <>
196 EIGEN_STRONG_INLINE Packet16f pmul<Packet16f>(const Packet16f& a,
197  const Packet16f& b) {
198  return _mm512_mul_ps(a, b);
199 }
200 template <>
201 EIGEN_STRONG_INLINE Packet8d pmul<Packet8d>(const Packet8d& a,
202  const Packet8d& b) {
203  return _mm512_mul_pd(a, b);
204 }
205 
206 template <>
207 EIGEN_STRONG_INLINE Packet16f pdiv<Packet16f>(const Packet16f& a,
208  const Packet16f& b) {
209  return _mm512_div_ps(a, b);
210 }
211 template <>
212 EIGEN_STRONG_INLINE Packet8d pdiv<Packet8d>(const Packet8d& a,
213  const Packet8d& b) {
214  return _mm512_div_pd(a, b);
215 }
216 
217 #ifdef __FMA__
218 template <>
219 EIGEN_STRONG_INLINE Packet16f pmadd(const Packet16f& a, const Packet16f& b,
220  const Packet16f& c) {
221  return _mm512_fmadd_ps(a, b, c);
222 }
223 template <>
224 EIGEN_STRONG_INLINE Packet8d pmadd(const Packet8d& a, const Packet8d& b,
225  const Packet8d& c) {
226  return _mm512_fmadd_pd(a, b, c);
227 }
228 #endif
229 
230 template <>
231 EIGEN_STRONG_INLINE Packet16f pmin<Packet16f>(const Packet16f& a,
232  const Packet16f& b) {
233  return _mm512_min_ps(a, b);
234 }
235 template <>
236 EIGEN_STRONG_INLINE Packet8d pmin<Packet8d>(const Packet8d& a,
237  const Packet8d& b) {
238  return _mm512_min_pd(a, b);
239 }
240 
241 template <>
242 EIGEN_STRONG_INLINE Packet16f pmax<Packet16f>(const Packet16f& a,
243  const Packet16f& b) {
244  return _mm512_max_ps(a, b);
245 }
246 template <>
247 EIGEN_STRONG_INLINE Packet8d pmax<Packet8d>(const Packet8d& a,
248  const Packet8d& b) {
249  return _mm512_max_pd(a, b);
250 }
251 
252 template <>
253 EIGEN_STRONG_INLINE Packet16f pand<Packet16f>(const Packet16f& a,
254  const Packet16f& b) {
255 #ifdef EIGEN_VECTORIZE_AVX512DQ
256  return _mm512_and_ps(a, b);
257 #else
258  Packet16f res = _mm512_undefined_ps();
259  Packet4f lane0_a = _mm512_extractf32x4_ps(a, 0);
260  Packet4f lane0_b = _mm512_extractf32x4_ps(b, 0);
261  res = _mm512_insertf32x4(res, _mm_and_ps(lane0_a, lane0_b), 0);
262 
263  Packet4f lane1_a = _mm512_extractf32x4_ps(a, 1);
264  Packet4f lane1_b = _mm512_extractf32x4_ps(b, 1);
265  res = _mm512_insertf32x4(res, _mm_and_ps(lane1_a, lane1_b), 1);
266 
267  Packet4f lane2_a = _mm512_extractf32x4_ps(a, 2);
268  Packet4f lane2_b = _mm512_extractf32x4_ps(b, 2);
269  res = _mm512_insertf32x4(res, _mm_and_ps(lane2_a, lane2_b), 2);
270 
271  Packet4f lane3_a = _mm512_extractf32x4_ps(a, 3);
272  Packet4f lane3_b = _mm512_extractf32x4_ps(b, 3);
273  res = _mm512_insertf32x4(res, _mm_and_ps(lane3_a, lane3_b), 3);
274 
275  return res;
276 #endif
277 }
278 template <>
279 EIGEN_STRONG_INLINE Packet8d pand<Packet8d>(const Packet8d& a,
280  const Packet8d& b) {
281 #ifdef EIGEN_VECTORIZE_AVX512DQ
282  return _mm512_and_pd(a, b);
283 #else
284  Packet8d res = _mm512_undefined_pd();
285  Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0);
286  Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0);
287  res = _mm512_insertf64x4(res, _mm256_and_pd(lane0_a, lane0_b), 0);
288 
289  Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1);
290  Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1);
291  res = _mm512_insertf64x4(res, _mm256_and_pd(lane1_a, lane1_b), 1);
292 
293  return res;
294 #endif
295 }
296 template <>
297 EIGEN_STRONG_INLINE Packet16f por<Packet16f>(const Packet16f& a,
298  const Packet16f& b) {
299 #ifdef EIGEN_VECTORIZE_AVX512DQ
300  return _mm512_or_ps(a, b);
301 #else
302  Packet16f res = _mm512_undefined_ps();
303  Packet4f lane0_a = _mm512_extractf32x4_ps(a, 0);
304  Packet4f lane0_b = _mm512_extractf32x4_ps(b, 0);
305  res = _mm512_insertf32x4(res, _mm_or_ps(lane0_a, lane0_b), 0);
306 
307  Packet4f lane1_a = _mm512_extractf32x4_ps(a, 1);
308  Packet4f lane1_b = _mm512_extractf32x4_ps(b, 1);
309  res = _mm512_insertf32x4(res, _mm_or_ps(lane1_a, lane1_b), 1);
310 
311  Packet4f lane2_a = _mm512_extractf32x4_ps(a, 2);
312  Packet4f lane2_b = _mm512_extractf32x4_ps(b, 2);
313  res = _mm512_insertf32x4(res, _mm_or_ps(lane2_a, lane2_b), 2);
314 
315  Packet4f lane3_a = _mm512_extractf32x4_ps(a, 3);
316  Packet4f lane3_b = _mm512_extractf32x4_ps(b, 3);
317  res = _mm512_insertf32x4(res, _mm_or_ps(lane3_a, lane3_b), 3);
318 
319  return res;
320 #endif
321 }
322 
323 template <>
324 EIGEN_STRONG_INLINE Packet8d por<Packet8d>(const Packet8d& a,
325  const Packet8d& b) {
326 #ifdef EIGEN_VECTORIZE_AVX512DQ
327  return _mm512_or_pd(a, b);
328 #else
329  Packet8d res = _mm512_undefined_pd();
330  Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0);
331  Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0);
332  res = _mm512_insertf64x4(res, _mm256_or_pd(lane0_a, lane0_b), 0);
333 
334  Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1);
335  Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1);
336  res = _mm512_insertf64x4(res, _mm256_or_pd(lane1_a, lane1_b), 1);
337 
338  return res;
339 #endif
340 }
341 
342 template <>
343 EIGEN_STRONG_INLINE Packet16f pxor<Packet16f>(const Packet16f& a,
344  const Packet16f& b) {
345 #ifdef EIGEN_VECTORIZE_AVX512DQ
346  return _mm512_xor_ps(a, b);
347 #else
348  Packet16f res = _mm512_undefined_ps();
349  Packet4f lane0_a = _mm512_extractf32x4_ps(a, 0);
350  Packet4f lane0_b = _mm512_extractf32x4_ps(b, 0);
351  res = _mm512_insertf32x4(res, _mm_xor_ps(lane0_a, lane0_b), 0);
352 
353  Packet4f lane1_a = _mm512_extractf32x4_ps(a, 1);
354  Packet4f lane1_b = _mm512_extractf32x4_ps(b, 1);
355  res = _mm512_insertf32x4(res, _mm_xor_ps(lane1_a, lane1_b), 1);
356 
357  Packet4f lane2_a = _mm512_extractf32x4_ps(a, 2);
358  Packet4f lane2_b = _mm512_extractf32x4_ps(b, 2);
359  res = _mm512_insertf32x4(res, _mm_xor_ps(lane2_a, lane2_b), 2);
360 
361  Packet4f lane3_a = _mm512_extractf32x4_ps(a, 3);
362  Packet4f lane3_b = _mm512_extractf32x4_ps(b, 3);
363  res = _mm512_insertf32x4(res, _mm_xor_ps(lane3_a, lane3_b), 3);
364 
365  return res;
366 #endif
367 }
368 template <>
369 EIGEN_STRONG_INLINE Packet8d pxor<Packet8d>(const Packet8d& a,
370  const Packet8d& b) {
371 #ifdef EIGEN_VECTORIZE_AVX512DQ
372  return _mm512_xor_pd(a, b);
373 #else
374  Packet8d res = _mm512_undefined_pd();
375  Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0);
376  Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0);
377  res = _mm512_insertf64x4(res, _mm256_xor_pd(lane0_a, lane0_b), 0);
378 
379  Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1);
380  Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1);
381  res = _mm512_insertf64x4(res, _mm256_xor_pd(lane1_a, lane1_b), 1);
382 
383  return res;
384 #endif
385 }
386 
387 template <>
388 EIGEN_STRONG_INLINE Packet16f pandnot<Packet16f>(const Packet16f& a,
389  const Packet16f& b) {
390 #ifdef EIGEN_VECTORIZE_AVX512DQ
391  return _mm512_andnot_ps(a, b);
392 #else
393  Packet16f res = _mm512_undefined_ps();
394  Packet4f lane0_a = _mm512_extractf32x4_ps(a, 0);
395  Packet4f lane0_b = _mm512_extractf32x4_ps(b, 0);
396  res = _mm512_insertf32x4(res, _mm_andnot_ps(lane0_a, lane0_b), 0);
397 
398  Packet4f lane1_a = _mm512_extractf32x4_ps(a, 1);
399  Packet4f lane1_b = _mm512_extractf32x4_ps(b, 1);
400  res = _mm512_insertf32x4(res, _mm_andnot_ps(lane1_a, lane1_b), 1);
401 
402  Packet4f lane2_a = _mm512_extractf32x4_ps(a, 2);
403  Packet4f lane2_b = _mm512_extractf32x4_ps(b, 2);
404  res = _mm512_insertf32x4(res, _mm_andnot_ps(lane2_a, lane2_b), 2);
405 
406  Packet4f lane3_a = _mm512_extractf32x4_ps(a, 3);
407  Packet4f lane3_b = _mm512_extractf32x4_ps(b, 3);
408  res = _mm512_insertf32x4(res, _mm_andnot_ps(lane3_a, lane3_b), 3);
409 
410  return res;
411 #endif
412 }
413 template <>
414 EIGEN_STRONG_INLINE Packet8d pandnot<Packet8d>(const Packet8d& a,
415  const Packet8d& b) {
416 #ifdef EIGEN_VECTORIZE_AVX512DQ
417  return _mm512_andnot_pd(a, b);
418 #else
419  Packet8d res = _mm512_undefined_pd();
420  Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0);
421  Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0);
422  res = _mm512_insertf64x4(res, _mm256_andnot_pd(lane0_a, lane0_b), 0);
423 
424  Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1);
425  Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1);
426  res = _mm512_insertf64x4(res, _mm256_andnot_pd(lane1_a, lane1_b), 1);
427 
428  return res;
429 #endif
430 }
431 
432 template <>
433 EIGEN_STRONG_INLINE Packet16f pload<Packet16f>(const float* from) {
434  EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_ps(from);
435 }
436 template <>
437 EIGEN_STRONG_INLINE Packet8d pload<Packet8d>(const double* from) {
438  EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_pd(from);
439 }
440 template <>
441 EIGEN_STRONG_INLINE Packet16i pload<Packet16i>(const int* from) {
442  EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_si512(
443  reinterpret_cast<const __m512i*>(from));
444 }
445 
446 template <>
447 EIGEN_STRONG_INLINE Packet16f ploadu<Packet16f>(const float* from) {
448  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_ps(from);
449 }
450 template <>
451 EIGEN_STRONG_INLINE Packet8d ploadu<Packet8d>(const double* from) {
452  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_pd(from);
453 }
454 template <>
455 EIGEN_STRONG_INLINE Packet16i ploadu<Packet16i>(const int* from) {
456  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_si512(
457  reinterpret_cast<const __m512i*>(from));
458 }
459 
460 // Loads 8 floats from memory a returns the packet
461 // {a0, a0 a1, a1, a2, a2, a3, a3, a4, a4, a5, a5, a6, a6, a7, a7}
462 template <>
463 EIGEN_STRONG_INLINE Packet16f ploaddup<Packet16f>(const float* from) {
464  Packet8f lane0 = _mm256_broadcast_ps((const __m128*)(const void*)from);
465  // mimic an "inplace" permutation of the lower 128bits using a blend
466  lane0 = _mm256_blend_ps(
467  lane0, _mm256_castps128_ps256(_mm_permute_ps(
468  _mm256_castps256_ps128(lane0), _MM_SHUFFLE(1, 0, 1, 0))),
469  15);
470  // then we can perform a consistent permutation on the global register to get
471  // everything in shape:
472  lane0 = _mm256_permute_ps(lane0, _MM_SHUFFLE(3, 3, 2, 2));
473 
474  Packet8f lane1 = _mm256_broadcast_ps((const __m128*)(const void*)(from + 4));
475  // mimic an "inplace" permutation of the lower 128bits using a blend
476  lane1 = _mm256_blend_ps(
477  lane1, _mm256_castps128_ps256(_mm_permute_ps(
478  _mm256_castps256_ps128(lane1), _MM_SHUFFLE(1, 0, 1, 0))),
479  15);
480  // then we can perform a consistent permutation on the global register to get
481  // everything in shape:
482  lane1 = _mm256_permute_ps(lane1, _MM_SHUFFLE(3, 3, 2, 2));
483 
484 #ifdef EIGEN_VECTORIZE_AVX512DQ
485  Packet16f res = _mm512_undefined_ps();
486  return _mm512_insertf32x8(res, lane0, 0);
487  return _mm512_insertf32x8(res, lane1, 1);
488  return res;
489 #else
490  Packet16f res = _mm512_undefined_ps();
491  res = _mm512_insertf32x4(res, _mm256_extractf128_ps(lane0, 0), 0);
492  res = _mm512_insertf32x4(res, _mm256_extractf128_ps(lane0, 1), 1);
493  res = _mm512_insertf32x4(res, _mm256_extractf128_ps(lane1, 0), 2);
494  res = _mm512_insertf32x4(res, _mm256_extractf128_ps(lane1, 1), 3);
495  return res;
496 #endif
497 }
498 // Loads 4 doubles from memory a returns the packet {a0, a0 a1, a1, a2, a2, a3,
499 // a3}
500 template <>
501 EIGEN_STRONG_INLINE Packet8d ploaddup<Packet8d>(const double* from) {
502  Packet4d lane0 = _mm256_broadcast_pd((const __m128d*)(const void*)from);
503  lane0 = _mm256_permute_pd(lane0, 3 << 2);
504 
505  Packet4d lane1 = _mm256_broadcast_pd((const __m128d*)(const void*)(from + 2));
506  lane1 = _mm256_permute_pd(lane1, 3 << 2);
507 
508  Packet8d res = _mm512_undefined_pd();
509  res = _mm512_insertf64x4(res, lane0, 0);
510  return _mm512_insertf64x4(res, lane1, 1);
511 }
512 
513 // Loads 4 floats from memory a returns the packet
514 // {a0, a0 a0, a0, a1, a1, a1, a1, a2, a2, a2, a2, a3, a3, a3, a3}
515 template <>
516 EIGEN_STRONG_INLINE Packet16f ploadquad<Packet16f>(const float* from) {
517  Packet16f tmp = _mm512_undefined_ps();
518  tmp = _mm512_insertf32x4(tmp, _mm_load_ps1(from), 0);
519  tmp = _mm512_insertf32x4(tmp, _mm_load_ps1(from + 1), 1);
520  tmp = _mm512_insertf32x4(tmp, _mm_load_ps1(from + 2), 2);
521  tmp = _mm512_insertf32x4(tmp, _mm_load_ps1(from + 3), 3);
522  return tmp;
523 }
524 // Loads 2 doubles from memory a returns the packet
525 // {a0, a0 a0, a0, a1, a1, a1, a1}
526 template <>
527 EIGEN_STRONG_INLINE Packet8d ploadquad<Packet8d>(const double* from) {
528  Packet8d tmp = _mm512_undefined_pd();
529  Packet2d tmp0 = _mm_load_pd1(from);
530  Packet2d tmp1 = _mm_load_pd1(from + 1);
531  Packet4d lane0 = _mm256_broadcastsd_pd(tmp0);
532  Packet4d lane1 = _mm256_broadcastsd_pd(tmp1);
533  tmp = _mm512_insertf64x4(tmp, lane0, 0);
534  return _mm512_insertf64x4(tmp, lane1, 1);
535 }
536 
537 template <>
538 EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet16f& from) {
539  EIGEN_DEBUG_ALIGNED_STORE _mm512_store_ps(to, from);
540 }
541 template <>
542 EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet8d& from) {
543  EIGEN_DEBUG_ALIGNED_STORE _mm512_store_pd(to, from);
544 }
545 template <>
546 EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet16i& from) {
547  EIGEN_DEBUG_ALIGNED_STORE _mm512_storeu_si512(reinterpret_cast<__m512i*>(to),
548  from);
549 }
550 
551 template <>
552 EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet16f& from) {
553  EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_ps(to, from);
554 }
555 template <>
556 EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet8d& from) {
557  EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_pd(to, from);
558 }
559 template <>
560 EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet16i& from) {
561  EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_si512(
562  reinterpret_cast<__m512i*>(to), from);
563 }
564 
565 template <>
566 EIGEN_DEVICE_FUNC inline Packet16f pgather<float, Packet16f>(const float* from,
567  Index stride) {
568  Packet16i stride_vector = _mm512_set1_epi32(stride);
569  Packet16i stride_multiplier =
570  _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
571  Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
572 
573  return _mm512_i32gather_ps(indices, from, 4);
574 }
575 template <>
576 EIGEN_DEVICE_FUNC inline Packet8d pgather<double, Packet8d>(const double* from,
577  Index stride) {
578  Packet8i stride_vector = _mm256_set1_epi32(stride);
579  Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
580  Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
581 
582  return _mm512_i32gather_pd(indices, from, 8);
583 }
584 
585 template <>
586 EIGEN_DEVICE_FUNC inline void pscatter<float, Packet16f>(float* to,
587  const Packet16f& from,
588  Index stride) {
589  Packet16i stride_vector = _mm512_set1_epi32(stride);
590  Packet16i stride_multiplier =
591  _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
592  Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
593  _mm512_i32scatter_ps(to, indices, from, 4);
594 }
595 template <>
596 EIGEN_DEVICE_FUNC inline void pscatter<double, Packet8d>(double* to,
597  const Packet8d& from,
598  Index stride) {
599  Packet8i stride_vector = _mm256_set1_epi32(stride);
600  Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
601  Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
602  _mm512_i32scatter_pd(to, indices, from, 8);
603 }
604 
605 template <>
606 EIGEN_STRONG_INLINE void pstore1<Packet16f>(float* to, const float& a) {
607  Packet16f pa = pset1<Packet16f>(a);
608  pstore(to, pa);
609 }
610 template <>
611 EIGEN_STRONG_INLINE void pstore1<Packet8d>(double* to, const double& a) {
612  Packet8d pa = pset1<Packet8d>(a);
613  pstore(to, pa);
614 }
615 template <>
616 EIGEN_STRONG_INLINE void pstore1<Packet16i>(int* to, const int& a) {
617  Packet16i pa = pset1<Packet16i>(a);
618  pstore(to, pa);
619 }
620 
621 template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
622 template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
623 template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
624 
625 template <>
626 EIGEN_STRONG_INLINE float pfirst<Packet16f>(const Packet16f& a) {
627  return _mm_cvtss_f32(_mm512_extractf32x4_ps(a, 0));
628 }
629 template <>
630 EIGEN_STRONG_INLINE double pfirst<Packet8d>(const Packet8d& a) {
631  return _mm_cvtsd_f64(_mm256_extractf128_pd(_mm512_extractf64x4_pd(a, 0), 0));
632 }
633 template <>
634 EIGEN_STRONG_INLINE int pfirst<Packet16i>(const Packet16i& a) {
635  return _mm_extract_epi32(_mm512_extracti32x4_epi32(a, 0), 0);
636 }
637 
638 template<> EIGEN_STRONG_INLINE Packet16f preverse(const Packet16f& a)
639 {
640  return _mm512_permutexvar_ps(_mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), a);
641 }
642 
643 template<> EIGEN_STRONG_INLINE Packet8d preverse(const Packet8d& a)
644 {
645  return _mm512_permutexvar_pd(_mm512_set_epi32(0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7), a);
646 }
647 
648 template<> EIGEN_STRONG_INLINE Packet16f pabs(const Packet16f& a)
649 {
650  // _mm512_abs_ps intrinsic not found, so hack around it
651  return (__m512)_mm512_and_si512((__m512i)a, _mm512_set1_epi32(0x7fffffff));
652 }
653 template <>
654 EIGEN_STRONG_INLINE Packet8d pabs(const Packet8d& a) {
655  // _mm512_abs_ps intrinsic not found, so hack around it
656  return (__m512d)_mm512_and_si512((__m512i)a,
657  _mm512_set1_epi64(0x7fffffffffffffff));
658 }
659 
660 #ifdef EIGEN_VECTORIZE_AVX512DQ
661 // AVX512F does not define _mm512_extractf32x8_ps to extract _m256 from _m512
662 #define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \
663  __m256 OUTPUT##_0 = _mm512_extractf32x8_ps(INPUT, 0) __m256 OUTPUT##_1 = \
664  _mm512_extractf32x8_ps(INPUT, 1)
665 #else
666 #define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \
667  __m256 OUTPUT##_0 = _mm256_insertf128_ps( \
668  _mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 0)), \
669  _mm512_extractf32x4_ps(INPUT, 1), 1); \
670  __m256 OUTPUT##_1 = _mm256_insertf128_ps( \
671  _mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 2)), \
672  _mm512_extractf32x4_ps(INPUT, 3), 1);
673 #endif
674 
675 #ifdef EIGEN_VECTORIZE_AVX512DQ
676 #define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \
677  OUTPUT = _mm512_insertf32x8(OUTPUT, INPUTA, 0); \
678  OUTPUT = _mm512_insertf32x8(OUTPUT, INPUTB, 1);
679 #else
680 #define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \
681  OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 0), 0); \
682  OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 1), 1); \
683  OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 0), 2); \
684  OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 1), 3);
685 #endif
686 template<> EIGEN_STRONG_INLINE Packet16f preduxp<Packet16f>(const Packet16f*
687 vecs)
688 {
689  EIGEN_EXTRACT_8f_FROM_16f(vecs[0], vecs0);
690  EIGEN_EXTRACT_8f_FROM_16f(vecs[1], vecs1);
691  EIGEN_EXTRACT_8f_FROM_16f(vecs[2], vecs2);
692  EIGEN_EXTRACT_8f_FROM_16f(vecs[3], vecs3);
693  EIGEN_EXTRACT_8f_FROM_16f(vecs[4], vecs4);
694  EIGEN_EXTRACT_8f_FROM_16f(vecs[5], vecs5);
695  EIGEN_EXTRACT_8f_FROM_16f(vecs[6], vecs6);
696  EIGEN_EXTRACT_8f_FROM_16f(vecs[7], vecs7);
697  EIGEN_EXTRACT_8f_FROM_16f(vecs[8], vecs8);
698  EIGEN_EXTRACT_8f_FROM_16f(vecs[9], vecs9);
699  EIGEN_EXTRACT_8f_FROM_16f(vecs[10], vecs10);
700  EIGEN_EXTRACT_8f_FROM_16f(vecs[11], vecs11);
701  EIGEN_EXTRACT_8f_FROM_16f(vecs[12], vecs12);
702  EIGEN_EXTRACT_8f_FROM_16f(vecs[13], vecs13);
703  EIGEN_EXTRACT_8f_FROM_16f(vecs[14], vecs14);
704  EIGEN_EXTRACT_8f_FROM_16f(vecs[15], vecs15);
705 
706  __m256 hsum1 = _mm256_hadd_ps(vecs0_0, vecs1_0);
707  __m256 hsum2 = _mm256_hadd_ps(vecs2_0, vecs3_0);
708  __m256 hsum3 = _mm256_hadd_ps(vecs4_0, vecs5_0);
709  __m256 hsum4 = _mm256_hadd_ps(vecs6_0, vecs7_0);
710 
711  __m256 hsum5 = _mm256_hadd_ps(hsum1, hsum1);
712  __m256 hsum6 = _mm256_hadd_ps(hsum2, hsum2);
713  __m256 hsum7 = _mm256_hadd_ps(hsum3, hsum3);
714  __m256 hsum8 = _mm256_hadd_ps(hsum4, hsum4);
715 
716  __m256 perm1 = _mm256_permute2f128_ps(hsum5, hsum5, 0x23);
717  __m256 perm2 = _mm256_permute2f128_ps(hsum6, hsum6, 0x23);
718  __m256 perm3 = _mm256_permute2f128_ps(hsum7, hsum7, 0x23);
719  __m256 perm4 = _mm256_permute2f128_ps(hsum8, hsum8, 0x23);
720 
721  __m256 sum1 = _mm256_add_ps(perm1, hsum5);
722  __m256 sum2 = _mm256_add_ps(perm2, hsum6);
723  __m256 sum3 = _mm256_add_ps(perm3, hsum7);
724  __m256 sum4 = _mm256_add_ps(perm4, hsum8);
725 
726  __m256 blend1 = _mm256_blend_ps(sum1, sum2, 0xcc);
727  __m256 blend2 = _mm256_blend_ps(sum3, sum4, 0xcc);
728 
729  __m256 final = _mm256_blend_ps(blend1, blend2, 0xf0);
730 
731  hsum1 = _mm256_hadd_ps(vecs0_1, vecs1_1);
732  hsum2 = _mm256_hadd_ps(vecs2_1, vecs3_1);
733  hsum3 = _mm256_hadd_ps(vecs4_1, vecs5_1);
734  hsum4 = _mm256_hadd_ps(vecs6_1, vecs7_1);
735 
736  hsum5 = _mm256_hadd_ps(hsum1, hsum1);
737  hsum6 = _mm256_hadd_ps(hsum2, hsum2);
738  hsum7 = _mm256_hadd_ps(hsum3, hsum3);
739  hsum8 = _mm256_hadd_ps(hsum4, hsum4);
740 
741  perm1 = _mm256_permute2f128_ps(hsum5, hsum5, 0x23);
742  perm2 = _mm256_permute2f128_ps(hsum6, hsum6, 0x23);
743  perm3 = _mm256_permute2f128_ps(hsum7, hsum7, 0x23);
744  perm4 = _mm256_permute2f128_ps(hsum8, hsum8, 0x23);
745 
746  sum1 = _mm256_add_ps(perm1, hsum5);
747  sum2 = _mm256_add_ps(perm2, hsum6);
748  sum3 = _mm256_add_ps(perm3, hsum7);
749  sum4 = _mm256_add_ps(perm4, hsum8);
750 
751  blend1 = _mm256_blend_ps(sum1, sum2, 0xcc);
752  blend2 = _mm256_blend_ps(sum3, sum4, 0xcc);
753 
754  final = padd(final, _mm256_blend_ps(blend1, blend2, 0xf0));
755 
756  hsum1 = _mm256_hadd_ps(vecs8_0, vecs9_0);
757  hsum2 = _mm256_hadd_ps(vecs10_0, vecs11_0);
758  hsum3 = _mm256_hadd_ps(vecs12_0, vecs13_0);
759  hsum4 = _mm256_hadd_ps(vecs14_0, vecs15_0);
760 
761  hsum5 = _mm256_hadd_ps(hsum1, hsum1);
762  hsum6 = _mm256_hadd_ps(hsum2, hsum2);
763  hsum7 = _mm256_hadd_ps(hsum3, hsum3);
764  hsum8 = _mm256_hadd_ps(hsum4, hsum4);
765 
766  perm1 = _mm256_permute2f128_ps(hsum5, hsum5, 0x23);
767  perm2 = _mm256_permute2f128_ps(hsum6, hsum6, 0x23);
768  perm3 = _mm256_permute2f128_ps(hsum7, hsum7, 0x23);
769  perm4 = _mm256_permute2f128_ps(hsum8, hsum8, 0x23);
770 
771  sum1 = _mm256_add_ps(perm1, hsum5);
772  sum2 = _mm256_add_ps(perm2, hsum6);
773  sum3 = _mm256_add_ps(perm3, hsum7);
774  sum4 = _mm256_add_ps(perm4, hsum8);
775 
776  blend1 = _mm256_blend_ps(sum1, sum2, 0xcc);
777  blend2 = _mm256_blend_ps(sum3, sum4, 0xcc);
778 
779  __m256 final_1 = _mm256_blend_ps(blend1, blend2, 0xf0);
780 
781  hsum1 = _mm256_hadd_ps(vecs8_1, vecs9_1);
782  hsum2 = _mm256_hadd_ps(vecs10_1, vecs11_1);
783  hsum3 = _mm256_hadd_ps(vecs12_1, vecs13_1);
784  hsum4 = _mm256_hadd_ps(vecs14_1, vecs15_1);
785 
786  hsum5 = _mm256_hadd_ps(hsum1, hsum1);
787  hsum6 = _mm256_hadd_ps(hsum2, hsum2);
788  hsum7 = _mm256_hadd_ps(hsum3, hsum3);
789  hsum8 = _mm256_hadd_ps(hsum4, hsum4);
790 
791  perm1 = _mm256_permute2f128_ps(hsum5, hsum5, 0x23);
792  perm2 = _mm256_permute2f128_ps(hsum6, hsum6, 0x23);
793  perm3 = _mm256_permute2f128_ps(hsum7, hsum7, 0x23);
794  perm4 = _mm256_permute2f128_ps(hsum8, hsum8, 0x23);
795 
796  sum1 = _mm256_add_ps(perm1, hsum5);
797  sum2 = _mm256_add_ps(perm2, hsum6);
798  sum3 = _mm256_add_ps(perm3, hsum7);
799  sum4 = _mm256_add_ps(perm4, hsum8);
800 
801  blend1 = _mm256_blend_ps(sum1, sum2, 0xcc);
802  blend2 = _mm256_blend_ps(sum3, sum4, 0xcc);
803 
804  final_1 = padd(final_1, _mm256_blend_ps(blend1, blend2, 0xf0));
805 
806  __m512 final_output;
807 
808  EIGEN_INSERT_8f_INTO_16f(final_output, final, final_1);
809  return final_output;
810 }
811 
812 template<> EIGEN_STRONG_INLINE Packet8d preduxp<Packet8d>(const Packet8d* vecs)
813 {
814  Packet4d vecs0_0 = _mm512_extractf64x4_pd(vecs[0], 0);
815  Packet4d vecs0_1 = _mm512_extractf64x4_pd(vecs[0], 1);
816 
817  Packet4d vecs1_0 = _mm512_extractf64x4_pd(vecs[1], 0);
818  Packet4d vecs1_1 = _mm512_extractf64x4_pd(vecs[1], 1);
819 
820  Packet4d vecs2_0 = _mm512_extractf64x4_pd(vecs[2], 0);
821  Packet4d vecs2_1 = _mm512_extractf64x4_pd(vecs[2], 1);
822 
823  Packet4d vecs3_0 = _mm512_extractf64x4_pd(vecs[3], 0);
824  Packet4d vecs3_1 = _mm512_extractf64x4_pd(vecs[3], 1);
825 
826  Packet4d vecs4_0 = _mm512_extractf64x4_pd(vecs[4], 0);
827  Packet4d vecs4_1 = _mm512_extractf64x4_pd(vecs[4], 1);
828 
829  Packet4d vecs5_0 = _mm512_extractf64x4_pd(vecs[5], 0);
830  Packet4d vecs5_1 = _mm512_extractf64x4_pd(vecs[5], 1);
831 
832  Packet4d vecs6_0 = _mm512_extractf64x4_pd(vecs[6], 0);
833  Packet4d vecs6_1 = _mm512_extractf64x4_pd(vecs[6], 1);
834 
835  Packet4d vecs7_0 = _mm512_extractf64x4_pd(vecs[7], 0);
836  Packet4d vecs7_1 = _mm512_extractf64x4_pd(vecs[7], 1);
837 
838  Packet4d tmp0, tmp1;
839 
840  tmp0 = _mm256_hadd_pd(vecs0_0, vecs1_0);
841  tmp0 = _mm256_add_pd(tmp0, _mm256_permute2f128_pd(tmp0, tmp0, 1));
842 
843  tmp1 = _mm256_hadd_pd(vecs2_0, vecs3_0);
844  tmp1 = _mm256_add_pd(tmp1, _mm256_permute2f128_pd(tmp1, tmp1, 1));
845 
846  __m256d final_0 = _mm256_blend_pd(tmp0, tmp1, 0xC);
847 
848  tmp0 = _mm256_hadd_pd(vecs0_1, vecs1_1);
849  tmp0 = _mm256_add_pd(tmp0, _mm256_permute2f128_pd(tmp0, tmp0, 1));
850 
851  tmp1 = _mm256_hadd_pd(vecs2_1, vecs3_1);
852  tmp1 = _mm256_add_pd(tmp1, _mm256_permute2f128_pd(tmp1, tmp1, 1));
853 
854  final_0 = padd(final_0, _mm256_blend_pd(tmp0, tmp1, 0xC));
855 
856  tmp0 = _mm256_hadd_pd(vecs4_0, vecs5_0);
857  tmp0 = _mm256_add_pd(tmp0, _mm256_permute2f128_pd(tmp0, tmp0, 1));
858 
859  tmp1 = _mm256_hadd_pd(vecs6_0, vecs7_0);
860  tmp1 = _mm256_add_pd(tmp1, _mm256_permute2f128_pd(tmp1, tmp1, 1));
861 
862  __m256d final_1 = _mm256_blend_pd(tmp0, tmp1, 0xC);
863 
864  tmp0 = _mm256_hadd_pd(vecs4_1, vecs5_1);
865  tmp0 = _mm256_add_pd(tmp0, _mm256_permute2f128_pd(tmp0, tmp0, 1));
866 
867  tmp1 = _mm256_hadd_pd(vecs6_1, vecs7_1);
868  tmp1 = _mm256_add_pd(tmp1, _mm256_permute2f128_pd(tmp1, tmp1, 1));
869 
870  final_1 = padd(final_1, _mm256_blend_pd(tmp0, tmp1, 0xC));
871 
872  __m512d final_output = _mm512_insertf64x4(final_output, final_0, 0);
873 
874  return _mm512_insertf64x4(final_output, final_1, 1);
875 }
876 
877 template <>
878 EIGEN_STRONG_INLINE float predux<Packet16f>(const Packet16f& a) {
879  //#ifdef EIGEN_VECTORIZE_AVX512DQ
880 #if 0
881  Packet8f lane0 = _mm512_extractf32x8_ps(a, 0);
882  Packet8f lane1 = _mm512_extractf32x8_ps(a, 1);
883  Packet8f sum = padd(lane0, lane1);
884  Packet8f tmp0 = _mm256_hadd_ps(sum, _mm256_permute2f128_ps(a, a, 1));
885  tmp0 = _mm256_hadd_ps(tmp0, tmp0);
886  return pfirst(_mm256_hadd_ps(tmp0, tmp0));
887 #else
888  Packet4f lane0 = _mm512_extractf32x4_ps(a, 0);
889  Packet4f lane1 = _mm512_extractf32x4_ps(a, 1);
890  Packet4f lane2 = _mm512_extractf32x4_ps(a, 2);
891  Packet4f lane3 = _mm512_extractf32x4_ps(a, 3);
892  Packet4f sum = padd(padd(lane0, lane1), padd(lane2, lane3));
893  sum = _mm_hadd_ps(sum, sum);
894  sum = _mm_hadd_ps(sum, _mm_permute_ps(sum, 1));
895  return pfirst(sum);
896 #endif
897 }
898 template <>
899 EIGEN_STRONG_INLINE double predux<Packet8d>(const Packet8d& a) {
900  Packet4d lane0 = _mm512_extractf64x4_pd(a, 0);
901  Packet4d lane1 = _mm512_extractf64x4_pd(a, 1);
902  Packet4d sum = padd(lane0, lane1);
903  Packet4d tmp0 = _mm256_hadd_pd(sum, _mm256_permute2f128_pd(sum, sum, 1));
904  return pfirst(_mm256_hadd_pd(tmp0, tmp0));
905 }
906 
907 template <>
908 EIGEN_STRONG_INLINE Packet8f predux_downto4<Packet16f>(const Packet16f& a) {
909 #ifdef EIGEN_VECTORIZE_AVX512DQ
910  Packet8f lane0 = _mm512_extractf32x8_ps(a, 0);
911  Packet8f lane1 = _mm512_extractf32x8_ps(a, 1);
912  return padd(lane0, lane1);
913 #else
914  Packet4f lane0 = _mm512_extractf32x4_ps(a, 0);
915  Packet4f lane1 = _mm512_extractf32x4_ps(a, 1);
916  Packet4f lane2 = _mm512_extractf32x4_ps(a, 2);
917  Packet4f lane3 = _mm512_extractf32x4_ps(a, 3);
918  Packet4f sum0 = padd(lane0, lane2);
919  Packet4f sum1 = padd(lane1, lane3);
920  return _mm256_insertf128_ps(_mm256_castps128_ps256(sum0), sum1, 1);
921 #endif
922 }
923 template <>
924 EIGEN_STRONG_INLINE Packet4d predux_downto4<Packet8d>(const Packet8d& a) {
925  Packet4d lane0 = _mm512_extractf64x4_pd(a, 0);
926  Packet4d lane1 = _mm512_extractf64x4_pd(a, 1);
927  Packet4d res = padd(lane0, lane1);
928  return res;
929 }
930 
931 template <>
932 EIGEN_STRONG_INLINE float predux_mul<Packet16f>(const Packet16f& a) {
933 //#ifdef EIGEN_VECTORIZE_AVX512DQ
934 #if 0
935  Packet8f lane0 = _mm512_extractf32x8_ps(a, 0);
936  Packet8f lane1 = _mm512_extractf32x8_ps(a, 1);
937  Packet8f res = pmul(lane0, lane1);
938  res = pmul(res, _mm256_permute2f128_ps(res, res, 1));
939  res = pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2)));
940  return pfirst(pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1))));
941 #else
942  Packet4f lane0 = _mm512_extractf32x4_ps(a, 0);
943  Packet4f lane1 = _mm512_extractf32x4_ps(a, 1);
944  Packet4f lane2 = _mm512_extractf32x4_ps(a, 2);
945  Packet4f lane3 = _mm512_extractf32x4_ps(a, 3);
946  Packet4f res = pmul(pmul(lane0, lane1), pmul(lane2, lane3));
947  res = pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2)));
948  return pfirst(pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1))));
949 #endif
950 }
951 template <>
952 EIGEN_STRONG_INLINE double predux_mul<Packet8d>(const Packet8d& a) {
953  Packet4d lane0 = _mm512_extractf64x4_pd(a, 0);
954  Packet4d lane1 = _mm512_extractf64x4_pd(a, 1);
955  Packet4d res = pmul(lane0, lane1);
956  res = pmul(res, _mm256_permute2f128_pd(res, res, 1));
957  return pfirst(pmul(res, _mm256_shuffle_pd(res, res, 1)));
958 }
959 
960 template <>
961 EIGEN_STRONG_INLINE float predux_min<Packet16f>(const Packet16f& a) {
962  Packet4f lane0 = _mm512_extractf32x4_ps(a, 0);
963  Packet4f lane1 = _mm512_extractf32x4_ps(a, 1);
964  Packet4f lane2 = _mm512_extractf32x4_ps(a, 2);
965  Packet4f lane3 = _mm512_extractf32x4_ps(a, 3);
966  Packet4f res = _mm_min_ps(_mm_min_ps(lane0, lane1), _mm_min_ps(lane2, lane3));
967  res = _mm_min_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2)));
968  return pfirst(_mm_min_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1))));
969 }
970 template <>
971 EIGEN_STRONG_INLINE double predux_min<Packet8d>(const Packet8d& a) {
972  Packet4d lane0 = _mm512_extractf64x4_pd(a, 0);
973  Packet4d lane1 = _mm512_extractf64x4_pd(a, 1);
974  Packet4d res = _mm256_min_pd(lane0, lane1);
975  res = _mm256_min_pd(res, _mm256_permute2f128_pd(res, res, 1));
976  return pfirst(_mm256_min_pd(res, _mm256_shuffle_pd(res, res, 1)));
977 }
978 
979 template <>
980 EIGEN_STRONG_INLINE float predux_max<Packet16f>(const Packet16f& a) {
981  Packet4f lane0 = _mm512_extractf32x4_ps(a, 0);
982  Packet4f lane1 = _mm512_extractf32x4_ps(a, 1);
983  Packet4f lane2 = _mm512_extractf32x4_ps(a, 2);
984  Packet4f lane3 = _mm512_extractf32x4_ps(a, 3);
985  Packet4f res = _mm_max_ps(_mm_max_ps(lane0, lane1), _mm_max_ps(lane2, lane3));
986  res = _mm_max_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2)));
987  return pfirst(_mm_max_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1))));
988 }
989 template <>
990 EIGEN_STRONG_INLINE double predux_max<Packet8d>(const Packet8d& a) {
991  Packet4d lane0 = _mm512_extractf64x4_pd(a, 0);
992  Packet4d lane1 = _mm512_extractf64x4_pd(a, 1);
993  Packet4d res = _mm256_max_pd(lane0, lane1);
994  res = _mm256_max_pd(res, _mm256_permute2f128_pd(res, res, 1));
995  return pfirst(_mm256_max_pd(res, _mm256_shuffle_pd(res, res, 1)));
996 }
997 
998 template <int Offset>
999 struct palign_impl<Offset, Packet16f> {
1000  static EIGEN_STRONG_INLINE void run(Packet16f& first,
1001  const Packet16f& second) {
1002  if (Offset != 0) {
1003  __m512i first_idx = _mm512_set_epi32(
1004  Offset + 15, Offset + 14, Offset + 13, Offset + 12, Offset + 11,
1005  Offset + 10, Offset + 9, Offset + 8, Offset + 7, Offset + 6,
1006  Offset + 5, Offset + 4, Offset + 3, Offset + 2, Offset + 1, Offset);
1007 
1008  __m512i second_idx =
1009  _mm512_set_epi32(Offset - 1, Offset - 2, Offset - 3, Offset - 4,
1010  Offset - 5, Offset - 6, Offset - 7, Offset - 8,
1011  Offset - 9, Offset - 10, Offset - 11, Offset - 12,
1012  Offset - 13, Offset - 14, Offset - 15, Offset - 16);
1013 
1014  unsigned short mask = 0xFFFF;
1015  mask <<= (16 - Offset);
1016 
1017  first = _mm512_permutexvar_ps(first_idx, first);
1018  Packet16f tmp = _mm512_permutexvar_ps(second_idx, second);
1019  first = _mm512_mask_blend_ps(mask, first, tmp);
1020  }
1021  }
1022 };
1023 template <int Offset>
1024 struct palign_impl<Offset, Packet8d> {
1025  static EIGEN_STRONG_INLINE void run(Packet8d& first, const Packet8d& second) {
1026  if (Offset != 0) {
1027  __m512i first_idx = _mm512_set_epi32(
1028  0, Offset + 7, 0, Offset + 6, 0, Offset + 5, 0, Offset + 4, 0,
1029  Offset + 3, 0, Offset + 2, 0, Offset + 1, 0, Offset);
1030 
1031  __m512i second_idx = _mm512_set_epi32(
1032  0, Offset - 1, 0, Offset - 2, 0, Offset - 3, 0, Offset - 4, 0,
1033  Offset - 5, 0, Offset - 6, 0, Offset - 7, 0, Offset - 8);
1034 
1035  unsigned char mask = 0xFF;
1036  mask <<= (8 - Offset);
1037 
1038  first = _mm512_permutexvar_pd(first_idx, first);
1039  Packet8d tmp = _mm512_permutexvar_pd(second_idx, second);
1040  first = _mm512_mask_blend_pd(mask, first, tmp);
1041  }
1042  }
1043 };
1044 
1045 
1046 #define PACK_OUTPUT(OUTPUT, INPUT, INDEX, STRIDE) \
1047  EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[INDEX], INPUT[INDEX + STRIDE]);
1048 
1049 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 16>& kernel) {
1050  __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1051  __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1052  __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1053  __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1054  __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4], kernel.packet[5]);
1055  __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4], kernel.packet[5]);
1056  __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6], kernel.packet[7]);
1057  __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6], kernel.packet[7]);
1058  __m512 T8 = _mm512_unpacklo_ps(kernel.packet[8], kernel.packet[9]);
1059  __m512 T9 = _mm512_unpackhi_ps(kernel.packet[8], kernel.packet[9]);
1060  __m512 T10 = _mm512_unpacklo_ps(kernel.packet[10], kernel.packet[11]);
1061  __m512 T11 = _mm512_unpackhi_ps(kernel.packet[10], kernel.packet[11]);
1062  __m512 T12 = _mm512_unpacklo_ps(kernel.packet[12], kernel.packet[13]);
1063  __m512 T13 = _mm512_unpackhi_ps(kernel.packet[12], kernel.packet[13]);
1064  __m512 T14 = _mm512_unpacklo_ps(kernel.packet[14], kernel.packet[15]);
1065  __m512 T15 = _mm512_unpackhi_ps(kernel.packet[14], kernel.packet[15]);
1066  __m512 S0 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1067  __m512 S1 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1068  __m512 S2 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1069  __m512 S3 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1070  __m512 S4 = _mm512_shuffle_ps(T4, T6, _MM_SHUFFLE(1, 0, 1, 0));
1071  __m512 S5 = _mm512_shuffle_ps(T4, T6, _MM_SHUFFLE(3, 2, 3, 2));
1072  __m512 S6 = _mm512_shuffle_ps(T5, T7, _MM_SHUFFLE(1, 0, 1, 0));
1073  __m512 S7 = _mm512_shuffle_ps(T5, T7, _MM_SHUFFLE(3, 2, 3, 2));
1074  __m512 S8 = _mm512_shuffle_ps(T8, T10, _MM_SHUFFLE(1, 0, 1, 0));
1075  __m512 S9 = _mm512_shuffle_ps(T8, T10, _MM_SHUFFLE(3, 2, 3, 2));
1076  __m512 S10 = _mm512_shuffle_ps(T9, T11, _MM_SHUFFLE(1, 0, 1, 0));
1077  __m512 S11 = _mm512_shuffle_ps(T9, T11, _MM_SHUFFLE(3, 2, 3, 2));
1078  __m512 S12 = _mm512_shuffle_ps(T12, T14, _MM_SHUFFLE(1, 0, 1, 0));
1079  __m512 S13 = _mm512_shuffle_ps(T12, T14, _MM_SHUFFLE(3, 2, 3, 2));
1080  __m512 S14 = _mm512_shuffle_ps(T13, T15, _MM_SHUFFLE(1, 0, 1, 0));
1081  __m512 S15 = _mm512_shuffle_ps(T13, T15, _MM_SHUFFLE(3, 2, 3, 2));
1082 
1083  EIGEN_EXTRACT_8f_FROM_16f(S0, S0);
1084  EIGEN_EXTRACT_8f_FROM_16f(S1, S1);
1085  EIGEN_EXTRACT_8f_FROM_16f(S2, S2);
1086  EIGEN_EXTRACT_8f_FROM_16f(S3, S3);
1087  EIGEN_EXTRACT_8f_FROM_16f(S4, S4);
1088  EIGEN_EXTRACT_8f_FROM_16f(S5, S5);
1089  EIGEN_EXTRACT_8f_FROM_16f(S6, S6);
1090  EIGEN_EXTRACT_8f_FROM_16f(S7, S7);
1091  EIGEN_EXTRACT_8f_FROM_16f(S8, S8);
1092  EIGEN_EXTRACT_8f_FROM_16f(S9, S9);
1093  EIGEN_EXTRACT_8f_FROM_16f(S10, S10);
1094  EIGEN_EXTRACT_8f_FROM_16f(S11, S11);
1095  EIGEN_EXTRACT_8f_FROM_16f(S12, S12);
1096  EIGEN_EXTRACT_8f_FROM_16f(S13, S13);
1097  EIGEN_EXTRACT_8f_FROM_16f(S14, S14);
1098  EIGEN_EXTRACT_8f_FROM_16f(S15, S15);
1099 
1100  PacketBlock<Packet8f, 32> tmp;
1101 
1102  tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S4_0, 0x20);
1103  tmp.packet[1] = _mm256_permute2f128_ps(S1_0, S5_0, 0x20);
1104  tmp.packet[2] = _mm256_permute2f128_ps(S2_0, S6_0, 0x20);
1105  tmp.packet[3] = _mm256_permute2f128_ps(S3_0, S7_0, 0x20);
1106  tmp.packet[4] = _mm256_permute2f128_ps(S0_0, S4_0, 0x31);
1107  tmp.packet[5] = _mm256_permute2f128_ps(S1_0, S5_0, 0x31);
1108  tmp.packet[6] = _mm256_permute2f128_ps(S2_0, S6_0, 0x31);
1109  tmp.packet[7] = _mm256_permute2f128_ps(S3_0, S7_0, 0x31);
1110 
1111  tmp.packet[8] = _mm256_permute2f128_ps(S0_1, S4_1, 0x20);
1112  tmp.packet[9] = _mm256_permute2f128_ps(S1_1, S5_1, 0x20);
1113  tmp.packet[10] = _mm256_permute2f128_ps(S2_1, S6_1, 0x20);
1114  tmp.packet[11] = _mm256_permute2f128_ps(S3_1, S7_1, 0x20);
1115  tmp.packet[12] = _mm256_permute2f128_ps(S0_1, S4_1, 0x31);
1116  tmp.packet[13] = _mm256_permute2f128_ps(S1_1, S5_1, 0x31);
1117  tmp.packet[14] = _mm256_permute2f128_ps(S2_1, S6_1, 0x31);
1118  tmp.packet[15] = _mm256_permute2f128_ps(S3_1, S7_1, 0x31);
1119 
1120  // Second set of _m256 outputs
1121  tmp.packet[16] = _mm256_permute2f128_ps(S8_0, S12_0, 0x20);
1122  tmp.packet[17] = _mm256_permute2f128_ps(S9_0, S13_0, 0x20);
1123  tmp.packet[18] = _mm256_permute2f128_ps(S10_0, S14_0, 0x20);
1124  tmp.packet[19] = _mm256_permute2f128_ps(S11_0, S15_0, 0x20);
1125  tmp.packet[20] = _mm256_permute2f128_ps(S8_0, S12_0, 0x31);
1126  tmp.packet[21] = _mm256_permute2f128_ps(S9_0, S13_0, 0x31);
1127  tmp.packet[22] = _mm256_permute2f128_ps(S10_0, S14_0, 0x31);
1128  tmp.packet[23] = _mm256_permute2f128_ps(S11_0, S15_0, 0x31);
1129 
1130  tmp.packet[24] = _mm256_permute2f128_ps(S8_1, S12_1, 0x20);
1131  tmp.packet[25] = _mm256_permute2f128_ps(S9_1, S13_1, 0x20);
1132  tmp.packet[26] = _mm256_permute2f128_ps(S10_1, S14_1, 0x20);
1133  tmp.packet[27] = _mm256_permute2f128_ps(S11_1, S15_1, 0x20);
1134  tmp.packet[28] = _mm256_permute2f128_ps(S8_1, S12_1, 0x31);
1135  tmp.packet[29] = _mm256_permute2f128_ps(S9_1, S13_1, 0x31);
1136  tmp.packet[30] = _mm256_permute2f128_ps(S10_1, S14_1, 0x31);
1137  tmp.packet[31] = _mm256_permute2f128_ps(S11_1, S15_1, 0x31);
1138 
1139  // Pack them into the output
1140  PACK_OUTPUT(kernel.packet, tmp.packet, 0, 16);
1141  PACK_OUTPUT(kernel.packet, tmp.packet, 1, 16);
1142  PACK_OUTPUT(kernel.packet, tmp.packet, 2, 16);
1143  PACK_OUTPUT(kernel.packet, tmp.packet, 3, 16);
1144 
1145  PACK_OUTPUT(kernel.packet, tmp.packet, 4, 16);
1146  PACK_OUTPUT(kernel.packet, tmp.packet, 5, 16);
1147  PACK_OUTPUT(kernel.packet, tmp.packet, 6, 16);
1148  PACK_OUTPUT(kernel.packet, tmp.packet, 7, 16);
1149 
1150  PACK_OUTPUT(kernel.packet, tmp.packet, 8, 16);
1151  PACK_OUTPUT(kernel.packet, tmp.packet, 9, 16);
1152  PACK_OUTPUT(kernel.packet, tmp.packet, 10, 16);
1153  PACK_OUTPUT(kernel.packet, tmp.packet, 11, 16);
1154 
1155  PACK_OUTPUT(kernel.packet, tmp.packet, 12, 16);
1156  PACK_OUTPUT(kernel.packet, tmp.packet, 13, 16);
1157  PACK_OUTPUT(kernel.packet, tmp.packet, 14, 16);
1158  PACK_OUTPUT(kernel.packet, tmp.packet, 15, 16);
1159 }
1160 #define PACK_OUTPUT_2(OUTPUT, INPUT, INDEX, STRIDE) \
1161  EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[2 * INDEX], \
1162  INPUT[2 * INDEX + STRIDE]);
1163 
1164 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 4>& kernel) {
1165  __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1166  __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1167  __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1168  __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1169 
1170  __m512 S0 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1171  __m512 S1 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1172  __m512 S2 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1173  __m512 S3 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1174 
1175  EIGEN_EXTRACT_8f_FROM_16f(S0, S0);
1176  EIGEN_EXTRACT_8f_FROM_16f(S1, S1);
1177  EIGEN_EXTRACT_8f_FROM_16f(S2, S2);
1178  EIGEN_EXTRACT_8f_FROM_16f(S3, S3);
1179 
1180  PacketBlock<Packet8f, 8> tmp;
1181 
1182  tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S1_0, 0x20);
1183  tmp.packet[1] = _mm256_permute2f128_ps(S2_0, S3_0, 0x20);
1184  tmp.packet[2] = _mm256_permute2f128_ps(S0_0, S1_0, 0x31);
1185  tmp.packet[3] = _mm256_permute2f128_ps(S2_0, S3_0, 0x31);
1186 
1187  tmp.packet[4] = _mm256_permute2f128_ps(S0_1, S1_1, 0x20);
1188  tmp.packet[5] = _mm256_permute2f128_ps(S2_1, S3_1, 0x20);
1189  tmp.packet[6] = _mm256_permute2f128_ps(S0_1, S1_1, 0x31);
1190  tmp.packet[7] = _mm256_permute2f128_ps(S2_1, S3_1, 0x31);
1191 
1192  PACK_OUTPUT_2(kernel.packet, tmp.packet, 0, 1);
1193  PACK_OUTPUT_2(kernel.packet, tmp.packet, 1, 1);
1194  PACK_OUTPUT_2(kernel.packet, tmp.packet, 2, 1);
1195  PACK_OUTPUT_2(kernel.packet, tmp.packet, 3, 1);
1196 }
1197 
1198 #define PACK_OUTPUT_SQ_D(OUTPUT, INPUT, INDEX, STRIDE) \
1199  OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX], 0); \
1200  OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX + STRIDE], 1);
1201 
1202 #define PACK_OUTPUT_D(OUTPUT, INPUT, INDEX, STRIDE) \
1203  OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX)], 0); \
1204  OUTPUT[INDEX] = \
1205  _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX) + STRIDE], 1);
1206 
1207 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8d, 4>& kernel) {
1208  __m512d T0 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0);
1209  __m512d T1 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0xff);
1210  __m512d T2 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0);
1211  __m512d T3 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0xff);
1212 
1213  PacketBlock<Packet4d, 8> tmp;
1214 
1215  tmp.packet[0] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0),
1216  _mm512_extractf64x4_pd(T2, 0), 0x20);
1217  tmp.packet[1] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0),
1218  _mm512_extractf64x4_pd(T3, 0), 0x20);
1219  tmp.packet[2] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0),
1220  _mm512_extractf64x4_pd(T2, 0), 0x31);
1221  tmp.packet[3] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0),
1222  _mm512_extractf64x4_pd(T3, 0), 0x31);
1223 
1224  tmp.packet[4] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1),
1225  _mm512_extractf64x4_pd(T2, 1), 0x20);
1226  tmp.packet[5] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1),
1227  _mm512_extractf64x4_pd(T3, 1), 0x20);
1228  tmp.packet[6] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1),
1229  _mm512_extractf64x4_pd(T2, 1), 0x31);
1230  tmp.packet[7] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1),
1231  _mm512_extractf64x4_pd(T3, 1), 0x31);
1232 
1233  PACK_OUTPUT_D(kernel.packet, tmp.packet, 0, 1);
1234  PACK_OUTPUT_D(kernel.packet, tmp.packet, 1, 1);
1235  PACK_OUTPUT_D(kernel.packet, tmp.packet, 2, 1);
1236  PACK_OUTPUT_D(kernel.packet, tmp.packet, 3, 1);
1237 }
1238 
1239 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8d, 8>& kernel) {
1240  __m512d T0 = _mm512_unpacklo_pd(kernel.packet[0], kernel.packet[1]);
1241  __m512d T1 = _mm512_unpackhi_pd(kernel.packet[0], kernel.packet[1]);
1242  __m512d T2 = _mm512_unpacklo_pd(kernel.packet[2], kernel.packet[3]);
1243  __m512d T3 = _mm512_unpackhi_pd(kernel.packet[2], kernel.packet[3]);
1244  __m512d T4 = _mm512_unpacklo_pd(kernel.packet[4], kernel.packet[5]);
1245  __m512d T5 = _mm512_unpackhi_pd(kernel.packet[4], kernel.packet[5]);
1246  __m512d T6 = _mm512_unpacklo_pd(kernel.packet[6], kernel.packet[7]);
1247  __m512d T7 = _mm512_unpackhi_pd(kernel.packet[6], kernel.packet[7]);
1248 
1249  PacketBlock<Packet4d, 16> tmp;
1250 
1251  tmp.packet[0] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0),
1252  _mm512_extractf64x4_pd(T2, 0), 0x20);
1253  tmp.packet[1] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0),
1254  _mm512_extractf64x4_pd(T3, 0), 0x20);
1255  tmp.packet[2] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0),
1256  _mm512_extractf64x4_pd(T2, 0), 0x31);
1257  tmp.packet[3] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0),
1258  _mm512_extractf64x4_pd(T3, 0), 0x31);
1259 
1260  tmp.packet[4] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1),
1261  _mm512_extractf64x4_pd(T2, 1), 0x20);
1262  tmp.packet[5] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1),
1263  _mm512_extractf64x4_pd(T3, 1), 0x20);
1264  tmp.packet[6] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1),
1265  _mm512_extractf64x4_pd(T2, 1), 0x31);
1266  tmp.packet[7] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1),
1267  _mm512_extractf64x4_pd(T3, 1), 0x31);
1268 
1269  tmp.packet[8] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 0),
1270  _mm512_extractf64x4_pd(T6, 0), 0x20);
1271  tmp.packet[9] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 0),
1272  _mm512_extractf64x4_pd(T7, 0), 0x20);
1273  tmp.packet[10] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 0),
1274  _mm512_extractf64x4_pd(T6, 0), 0x31);
1275  tmp.packet[11] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 0),
1276  _mm512_extractf64x4_pd(T7, 0), 0x31);
1277 
1278  tmp.packet[12] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 1),
1279  _mm512_extractf64x4_pd(T6, 1), 0x20);
1280  tmp.packet[13] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 1),
1281  _mm512_extractf64x4_pd(T7, 1), 0x20);
1282  tmp.packet[14] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 1),
1283  _mm512_extractf64x4_pd(T6, 1), 0x31);
1284  tmp.packet[15] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 1),
1285  _mm512_extractf64x4_pd(T7, 1), 0x31);
1286 
1287  PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 0, 8);
1288  PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 1, 8);
1289  PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 2, 8);
1290  PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 3, 8);
1291 
1292  PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 4, 8);
1293  PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 5, 8);
1294  PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 6, 8);
1295  PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 7, 8);
1296 }
1297 template <>
1298 EIGEN_STRONG_INLINE Packet16f pblend(const Selector<16>& /*ifPacket*/,
1299  const Packet16f& /*thenPacket*/,
1300  const Packet16f& /*elsePacket*/) {
1301  assert(false && "To be implemented");
1302  return Packet16f();
1303 }
1304 template <>
1305 EIGEN_STRONG_INLINE Packet8d pblend(const Selector<8>& /*ifPacket*/,
1306  const Packet8d& /*thenPacket*/,
1307  const Packet8d& /*elsePacket*/) {
1308  assert(false && "To be implemented");
1309  return Packet8d();
1310 }
1311 
1312 } // end namespace internal
1313 
1314 } // end namespace Eigen
1315 
1316 #endif // EIGEN_PACKET_MATH_AVX512_H
Eigen::internal::pmax< Packet16f >
EIGEN_STRONG_INLINE Packet16f pmax< Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:242
Eigen::internal::pgather< double, Packet8d >
EIGEN_DEVICE_FUNC Packet8d pgather< double, Packet8d >(const double *from, Index stride)
Definition: AVX512/PacketMath.h:576
Eigen::internal::pfirst< Packet8d >
EIGEN_STRONG_INLINE double pfirst< Packet8d >(const Packet8d &a)
Definition: AVX512/PacketMath.h:630
EIGEN_STRONG_INLINE
#define EIGEN_STRONG_INLINE
Definition: Macros.h:493
Eigen::internal::unpacket_traits< Packet8d >::half
Packet4d half
Definition: AVX512/PacketMath.h:106
PACK_OUTPUT_2
#define PACK_OUTPUT_2(OUTPUT, INPUT, INDEX, STRIDE)
Definition: AVX512/PacketMath.h:1160
Eigen::internal::unpacket_traits
Definition: XprHelper.h:158
Eigen::internal::predux_min< Packet8d >
EIGEN_STRONG_INLINE double predux_min< Packet8d >(const Packet8d &a)
Definition: AVX512/PacketMath.h:971
Eigen::internal::pstore1< Packet8d >
EIGEN_STRONG_INLINE void pstore1< Packet8d >(double *to, const double &a)
Definition: AVX512/PacketMath.h:611
Eigen::internal::pload1< Packet16f >
EIGEN_STRONG_INLINE Packet16f pload1< Packet16f >(const float *from)
Definition: AVX512/PacketMath.h:130
f
f
Eigen::internal::ploadu< Packet16i >
EIGEN_STRONG_INLINE Packet16i ploadu< Packet16i >(const int *from)
Definition: AVX512/PacketMath.h:455
EIGEN_DEBUG_UNALIGNED_LOAD
#define EIGEN_DEBUG_UNALIGNED_LOAD
Definition: GenericPacketMath.h:31
Eigen::internal::palign_impl< Offset, Packet8d >::run
static EIGEN_STRONG_INLINE void run(Packet8d &first, const Packet8d &second)
Definition: AVX512/PacketMath.h:1025
Eigen::internal::is_arithmetic::value
Definition: Meta.h:85
Eigen::internal::packet_traits< double >::type
Packet8d type
Definition: AVX512/PacketMath.h:70
Eigen::internal::pand< Packet16f >
EIGEN_STRONG_INLINE Packet16f pand< Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:253
Eigen::internal::padd< Packet16f >
EIGEN_STRONG_INLINE Packet16f padd< Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:152
EIGEN_EXTRACT_8f_FROM_16f
#define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT)
Definition: AVX512/PacketMath.h:666
Eigen::internal::pload< Packet16f >
EIGEN_STRONG_INLINE Packet16f pload< Packet16f >(const float *from)
Definition: AVX512/PacketMath.h:433
Eigen::internal::pset1< Packet16f >
EIGEN_STRONG_INLINE Packet16f pset1< Packet16f >(const float &from)
Definition: AVX512/PacketMath.h:117
Eigen::internal::pxor< Packet16f >
EIGEN_STRONG_INLINE Packet16f pxor< Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:343
EIGEN_DEBUG_ALIGNED_STORE
#define EIGEN_DEBUG_ALIGNED_STORE
Definition: GenericPacketMath.h:35
Eigen
Definition: LDLT.h:16
Eigen::internal::pmul< Packet16f >
EIGEN_STRONG_INLINE Packet16f pmul< Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:196
Eigen::internal::pmin< Packet16f >
EIGEN_STRONG_INLINE Packet16f pmin< Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:231
Eigen::internal::psub< Packet8d >
EIGEN_STRONG_INLINE Packet8d psub< Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:168
EIGEN_DEBUG_UNALIGNED_STORE
#define EIGEN_DEBUG_UNALIGNED_STORE
Definition: GenericPacketMath.h:39
Eigen::internal::pandnot< Packet16f >
EIGEN_STRONG_INLINE Packet16f pandnot< Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:388
Eigen::internal::prefetch< float >
EIGEN_STRONG_INLINE void prefetch< float >(const float *addr)
Definition: AltiVec/PacketMath.h:516
Eigen::internal::pxor< Packet8d >
EIGEN_STRONG_INLINE Packet8d pxor< Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:369
Eigen::internal::pstoreu< double >
EIGEN_STRONG_INLINE void pstoreu< double >(double *to, const Packet4d &from)
Definition: AVX/PacketMath.h:255
Eigen::internal::pdiv< Packet16f >
EIGEN_STRONG_INLINE Packet16f pdiv< Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:207
Eigen::internal::pscatter< double, Packet8d >
EIGEN_DEVICE_FUNC void pscatter< double, Packet8d >(double *to, const Packet8d &from, Index stride)
Definition: AVX512/PacketMath.h:596
Eigen::internal::padd< Packet8d >
EIGEN_STRONG_INLINE Packet8d padd< Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:157
Eigen::internal::unpacket_traits< Packet8d >::type
double type
Definition: AVX512/PacketMath.h:105
EIGEN_DEBUG_ALIGNED_LOAD
#define EIGEN_DEBUG_ALIGNED_LOAD
Definition: GenericPacketMath.h:27
Eigen::internal::pload1< Packet8d >
EIGEN_STRONG_INLINE Packet8d pload1< Packet8d >(const double *from)
Definition: AVX512/PacketMath.h:134
Eigen::internal::unpacket_traits< Packet16i >::type
int type
Definition: AVX512/PacketMath.h:111
Eigen::internal::predux< Packet8d >
EIGEN_STRONG_INLINE double predux< Packet8d >(const Packet8d &a)
Definition: AVX512/PacketMath.h:899
Eigen::internal::pset1< Packet8d >
EIGEN_STRONG_INLINE Packet8d pset1< Packet8d >(const double &from)
Definition: AVX512/PacketMath.h:121
PACK_OUTPUT
#define PACK_OUTPUT(OUTPUT, INPUT, INDEX, STRIDE)
Definition: AVX512/PacketMath.h:1046
Eigen::internal::padd
EIGEN_DEVICE_FUNC Packet padd(const Packet &a, const Packet &b)
Definition: GenericPacketMath.h:151
Eigen::internal::pload< Packet16i >
EIGEN_STRONG_INLINE Packet16i pload< Packet16i >(const int *from)
Definition: AVX512/PacketMath.h:441
Eigen::internal::packet_traits
Definition: GenericPacketMath.h:96
Eigen::internal::pstore< double >
EIGEN_STRONG_INLINE void pstore< double >(double *to, const Packet4d &from)
Definition: AVX/PacketMath.h:251
Eigen::internal::pandnot< Packet8d >
EIGEN_STRONG_INLINE Packet8d pandnot< Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:414
Eigen::internal::unpacket_traits< Packet16f >::half
Packet8f half
Definition: AVX512/PacketMath.h:100
Eigen::internal::predux_mul< Packet8d >
EIGEN_STRONG_INLINE double predux_mul< Packet8d >(const Packet8d &a)
Definition: AVX512/PacketMath.h:952
EIGEN_FAST_MATH
#define EIGEN_FAST_MATH
Definition: Macros.h:472
Eigen::internal::prefetch< int >
EIGEN_STRONG_INLINE void prefetch< int >(const int *addr)
Definition: AltiVec/PacketMath.h:517
Eigen::internal::default_packet_traits
Definition: GenericPacketMath.h:42
Eigen::internal::packet_traits< float >::half
Packet8f half
Definition: AVX512/PacketMath.h:51
PACK_OUTPUT_SQ_D
#define PACK_OUTPUT_SQ_D(OUTPUT, INPUT, INDEX, STRIDE)
Definition: AVX512/PacketMath.h:1198
Eigen::internal::pscatter< float, Packet16f >
EIGEN_DEVICE_FUNC void pscatter< float, Packet16f >(float *to, const Packet16f &from, Index stride)
Definition: AVX512/PacketMath.h:586
Eigen::internal::unpacket_traits< Packet16i >::half
Packet8i half
Definition: AVX512/PacketMath.h:112
Eigen::internal::psub< Packet16f >
EIGEN_STRONG_INLINE Packet16f psub< Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:163
Eigen::internal::ptranspose
EIGEN_STRONG_INLINE void ptranspose(PacketBlock< Packet2cf, 2 > &kernel)
Definition: AltiVec/Complex.h:258
Eigen::Aligned64
Definition: Constants.h:232
Eigen::internal::por< Packet16f >
EIGEN_STRONG_INLINE Packet16f por< Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:297
Eigen::internal::pstoreu< int >
EIGEN_STRONG_INLINE void pstoreu< int >(int *to, const Packet4i &from)
Definition: AltiVec/PacketMath.h:504
Eigen::internal::pstore< float >
EIGEN_STRONG_INLINE void pstore< float >(float *to, const Packet4f &from)
Definition: AltiVec/PacketMath.h:259
Eigen::internal::is_arithmetic
Definition: Meta.h:85
Eigen::Index
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:33
Eigen::internal::pand< Packet8d >
EIGEN_STRONG_INLINE Packet8d pand< Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:279
Eigen::internal::ploaddup< Packet16f >
EIGEN_STRONG_INLINE Packet16f ploaddup< Packet16f >(const float *from)
Definition: AVX512/PacketMath.h:463
Eigen::internal::pstore1< Packet16i >
EIGEN_STRONG_INLINE void pstore1< Packet16i >(int *to, const int &a)
Definition: AVX512/PacketMath.h:616
Eigen::internal::Packet8i
__vector short int Packet8i
Definition: AltiVec/PacketMath.h:38
Eigen::internal::pfirst
EIGEN_DEVICE_FUNC unpacket_traits< Packet >::type pfirst(const Packet &a)
Definition: GenericPacketMath.h:315
Eigen::internal::palign_impl< Offset, Packet16f >::run
static EIGEN_STRONG_INLINE void run(Packet16f &first, const Packet16f &second)
Definition: AVX512/PacketMath.h:1000
Eigen::internal::predux_mul< Packet16f >
EIGEN_STRONG_INLINE float predux_mul< Packet16f >(const Packet16f &a)
Definition: AVX512/PacketMath.h:932
EIGEN_INSERT_8f_INTO_16f
#define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB)
Definition: AVX512/PacketMath.h:680
Eigen::internal::pconj
EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf &a)
Definition: AltiVec/Complex.h:101
Eigen::internal::pmul< Packet8d >
EIGEN_STRONG_INLINE Packet8d pmul< Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:201
Eigen::internal::plset< Packet16f >
EIGEN_STRONG_INLINE Packet16f plset< Packet16f >(const float &a)
Definition: AVX512/PacketMath.h:139
Eigen::internal::pfirst< Packet16i >
EIGEN_STRONG_INLINE int pfirst< Packet16i >(const Packet16i &a)
Definition: AVX512/PacketMath.h:634
Eigen::internal::pstoreu< float >
EIGEN_STRONG_INLINE void pstoreu< float >(float *to, const Packet4f &from)
Definition: AltiVec/PacketMath.h:509
Eigen::internal::pstore
EIGEN_DEVICE_FUNC void pstore(Scalar *to, const Packet &from)
Definition: GenericPacketMath.h:285
Eigen::internal::ploadu< Packet8d >
EIGEN_STRONG_INLINE Packet8d ploadu< Packet8d >(const double *from)
Definition: AVX512/PacketMath.h:451
Eigen::internal::packet_traits< float >::type
Packet16f type
Definition: AVX512/PacketMath.h:50
Eigen::internal::preduxp< Packet8d >
EIGEN_STRONG_INLINE Packet8d preduxp< Packet8d >(const Packet8d *vecs)
Definition: AVX512/PacketMath.h:812
Eigen::internal::plset< Packet8d >
EIGEN_STRONG_INLINE Packet8d plset< Packet8d >(const double &a)
Definition: AVX512/PacketMath.h:146
Eigen::internal::pload< Packet8d >
EIGEN_STRONG_INLINE Packet8d pload< Packet8d >(const double *from)
Definition: AVX512/PacketMath.h:437
Eigen::internal::predux_max< Packet16f >
EIGEN_STRONG_INLINE float predux_max< Packet16f >(const Packet16f &a)
Definition: AVX512/PacketMath.h:980
Eigen::internal::pfirst< Packet16f >
EIGEN_STRONG_INLINE float pfirst< Packet16f >(const Packet16f &a)
Definition: AVX512/PacketMath.h:626
Eigen::internal::pnegate
EIGEN_STRONG_INLINE Packet2cf pnegate(const Packet2cf &a)
Definition: AltiVec/Complex.h:100
Eigen::internal::packet_traits< double >::half
Packet4d half
Definition: AVX512/PacketMath.h:71
Eigen::internal::predux< Packet16f >
EIGEN_STRONG_INLINE float predux< Packet16f >(const Packet16f &a)
Definition: AVX512/PacketMath.h:878
Eigen::internal::unpacket_traits< Packet16f >::type
float type
Definition: AVX512/PacketMath.h:99
Eigen::internal::ploadquad< Packet16f >
EIGEN_STRONG_INLINE Packet16f ploadquad< Packet16f >(const float *from)
Definition: AVX512/PacketMath.h:516
Eigen::internal::pstore< int >
EIGEN_STRONG_INLINE void pstore< int >(int *to, const Packet4i &from)
Definition: AltiVec/PacketMath.h:269
Eigen::internal::pset1< Packet16i >
EIGEN_STRONG_INLINE Packet16i pset1< Packet16i >(const int &from)
Definition: AVX512/PacketMath.h:125
Eigen::internal::pmadd
EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f &a, const Packet4f &b, const Packet4f &c)
Definition: AltiVec/PacketMath.h:388
Eigen::internal::pmin< Packet8d >
EIGEN_STRONG_INLINE Packet8d pmin< Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:236
Eigen::internal::ploadquad< Packet8d >
EIGEN_STRONG_INLINE Packet8d ploadquad< Packet8d >(const double *from)
Definition: AVX512/PacketMath.h:527
Eigen::internal::por< Packet8d >
EIGEN_STRONG_INLINE Packet8d por< Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:324
PACK_OUTPUT_D
#define PACK_OUTPUT_D(OUTPUT, INPUT, INDEX, STRIDE)
Definition: AVX512/PacketMath.h:1202
Eigen::internal::preduxp< Packet16f >
EIGEN_STRONG_INLINE Packet16f preduxp< Packet16f >(const Packet16f *vecs)
Definition: AVX512/PacketMath.h:686
Eigen::internal::palign_impl
Definition: GenericPacketMath.h:492
Eigen::internal::pstore1< Packet16f >
EIGEN_STRONG_INLINE void pstore1< Packet16f >(float *to, const float &a)
Definition: AVX512/PacketMath.h:606
Eigen::internal::pmax< Packet8d >
EIGEN_STRONG_INLINE Packet8d pmax< Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:247
Eigen::internal::ploadu< Packet16f >
EIGEN_STRONG_INLINE Packet16f ploadu< Packet16f >(const float *from)
Definition: AVX512/PacketMath.h:447
Eigen::internal::predux_downto4< Packet16f >
EIGEN_STRONG_INLINE Packet8f predux_downto4< Packet16f >(const Packet16f &a)
Definition: AVX512/PacketMath.h:908
Eigen::internal::pdiv< Packet8d >
EIGEN_STRONG_INLINE Packet8d pdiv< Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:212
Eigen::internal::pmul
EIGEN_DEVICE_FUNC Packet pmul(const Packet &a, const Packet &b)
Definition: GenericPacketMath.h:170
Eigen::internal::prefetch< double >
EIGEN_STRONG_INLINE void prefetch< double >(const double *addr)
Definition: AVX/PacketMath.h:312
Eigen::internal::predux_max< Packet8d >
EIGEN_STRONG_INLINE double predux_max< Packet8d >(const Packet8d &a)
Definition: AVX512/PacketMath.h:990
Eigen::internal::pblend
EIGEN_STRONG_INLINE Packet4i pblend(const Selector< 4 > &ifPacket, const Packet4i &thenPacket, const Packet4i &elsePacket)
Definition: AltiVec/PacketMath.h:741
Eigen::internal::predux_min< Packet16f >
EIGEN_STRONG_INLINE float predux_min< Packet16f >(const Packet16f &a)
Definition: AVX512/PacketMath.h:961
Eigen::internal::preverse
EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf &a)
Definition: AltiVec/Complex.h:137
Eigen::internal::pabs
EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f &a)
Definition: AltiVec/PacketMath.h:530
Eigen::internal::pgather< float, Packet16f >
EIGEN_DEVICE_FUNC Packet16f pgather< float, Packet16f >(const float *from, Index stride)
Definition: AVX512/PacketMath.h:566
Eigen::internal::ploaddup< Packet8d >
EIGEN_STRONG_INLINE Packet8d ploaddup< Packet8d >(const double *from)
Definition: AVX512/PacketMath.h:501
Eigen::internal::predux_downto4< Packet8d >
EIGEN_STRONG_INLINE Packet4d predux_downto4< Packet8d >(const Packet8d &a)
Definition: AVX512/PacketMath.h:924

co_scan
Author(s):
autogenerated on Mon Feb 28 2022 23:00:45