PacketMath.h
Go to the documentation of this file.
00001 // This file is part of Eigen, a lightweight C++ template library
00002 // for linear algebra.
00003 //
00004 // Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr>
00005 //
00006 // Eigen is free software; you can redistribute it and/or
00007 // modify it under the terms of the GNU Lesser General Public
00008 // License as published by the Free Software Foundation; either
00009 // version 3 of the License, or (at your option) any later version.
00010 //
00011 // Alternatively, you can redistribute it and/or
00012 // modify it under the terms of the GNU General Public License as
00013 // published by the Free Software Foundation; either version 2 of
00014 // the License, or (at your option) any later version.
00015 //
00016 // Eigen is distributed in the hope that it will be useful, but WITHOUT ANY
00017 // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
00018 // FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License or the
00019 // GNU General Public License for more details.
00020 //
00021 // You should have received a copy of the GNU Lesser General Public
00022 // License and a copy of the GNU General Public License along with
00023 // Eigen. If not, see <http://www.gnu.org/licenses/>.
00024 
00025 #ifndef EIGEN_PACKET_MATH_SSE_H
00026 #define EIGEN_PACKET_MATH_SSE_H
00027 
00028 namespace internal {
00029 
00030 #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
00031 #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
00032 #endif
00033 
00034 #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
00035 #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS (2*sizeof(void*))
00036 #endif
00037 
00038 typedef __m128  Packet4f;
00039 typedef __m128i Packet4i;
00040 typedef __m128d Packet2d;
00041 
00042 template<> struct is_arithmetic<__m128>  { enum { value = true }; };
00043 template<> struct is_arithmetic<__m128i> { enum { value = true }; };
00044 template<> struct is_arithmetic<__m128d> { enum { value = true }; };
00045 
00046 #define vec4f_swizzle1(v,p,q,r,s) \
00047   (_mm_castsi128_ps(_mm_shuffle_epi32( _mm_castps_si128(v), ((s)<<6|(r)<<4|(q)<<2|(p)))))
00048 
00049 #define vec4i_swizzle1(v,p,q,r,s) \
00050   (_mm_shuffle_epi32( v, ((s)<<6|(r)<<4|(q)<<2|(p))))
00051 
00052 #define vec2d_swizzle1(v,p,q) \
00053   (_mm_castsi128_pd(_mm_shuffle_epi32( _mm_castpd_si128(v), ((q*2+1)<<6|(q*2)<<4|(p*2+1)<<2|(p*2)))))
00054   
00055 #define vec4f_swizzle2(a,b,p,q,r,s) \
00056   (_mm_shuffle_ps( (a), (b), ((s)<<6|(r)<<4|(q)<<2|(p))))
00057 
00058 #define vec4i_swizzle2(a,b,p,q,r,s) \
00059   (_mm_castps_si128( (_mm_shuffle_ps( _mm_castsi128_ps(a), _mm_castsi128_ps(b), ((s)<<6|(r)<<4|(q)<<2|(p))))))
00060 
00061 #define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \
00062   const Packet4f p4f_##NAME = pset1<Packet4f>(X)
00063 
00064 #define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \
00065   const Packet4f p4f_##NAME = _mm_castsi128_ps(pset1<Packet4i>(X))
00066 
00067 #define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \
00068   const Packet4i p4i_##NAME = pset1<Packet4i>(X)
00069 
00070 
00071 template<> struct packet_traits<float>  : default_packet_traits
00072 {
00073   typedef Packet4f type;
00074   enum {
00075     Vectorizable = 1,
00076     AlignedOnScalar = 1,
00077     size=4,
00078 
00079     HasDiv    = 1,
00080     HasSin  = EIGEN_FAST_MATH,
00081     HasCos  = EIGEN_FAST_MATH,
00082     HasLog  = 1,
00083     HasExp  = 1,
00084     HasSqrt = 1
00085   };
00086 };
00087 template<> struct packet_traits<double> : default_packet_traits
00088 {
00089   typedef Packet2d type;
00090   enum {
00091     Vectorizable = 1,
00092     AlignedOnScalar = 1,
00093     size=2,
00094 
00095     HasDiv    = 1
00096   };
00097 };
00098 template<> struct packet_traits<int>    : default_packet_traits
00099 {
00100   typedef Packet4i type;
00101   enum {
00102     // FIXME check the Has*
00103     Vectorizable = 1,
00104     AlignedOnScalar = 1,
00105     size=4
00106   };
00107 };
00108 
00109 template<> struct unpacket_traits<Packet4f> { typedef float  type; enum {size=4}; };
00110 template<> struct unpacket_traits<Packet2d> { typedef double type; enum {size=2}; };
00111 template<> struct unpacket_traits<Packet4i> { typedef int    type; enum {size=4}; };
00112 
00113 template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float&  from) { return _mm_set1_ps(from); }
00114 template<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) { return _mm_set1_pd(from); }
00115 template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int&    from) { return _mm_set1_epi32(from); }
00116 
00117 template<> EIGEN_STRONG_INLINE Packet4f plset<float>(const float& a) { return _mm_add_ps(pset1<Packet4f>(a), _mm_set_ps(3,2,1,0)); }
00118 template<> EIGEN_STRONG_INLINE Packet2d plset<double>(const double& a) { return _mm_add_pd(pset1<Packet2d>(a),_mm_set_pd(1,0)); }
00119 template<> EIGEN_STRONG_INLINE Packet4i plset<int>(const int& a) { return _mm_add_epi32(pset1<Packet4i>(a),_mm_set_epi32(3,2,1,0)); }
00120 
00121 template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_add_ps(a,b); }
00122 template<> EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_add_pd(a,b); }
00123 template<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_add_epi32(a,b); }
00124 
00125 template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_sub_ps(a,b); }
00126 template<> EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_sub_pd(a,b); }
00127 template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_sub_epi32(a,b); }
00128 
00129 template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a)
00130 {
00131   const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(0x80000000,0x80000000,0x80000000,0x80000000));
00132   return _mm_xor_ps(a,mask);
00133 }
00134 template<> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a)
00135 {
00136   const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0x0,0x80000000,0x0,0x80000000));
00137   return _mm_xor_pd(a,mask);
00138 }
00139 template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a)
00140 {
00141   return psub(_mm_setr_epi32(0,0,0,0), a);
00142 }
00143 
00144 template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_mul_ps(a,b); }
00145 template<> EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_mul_pd(a,b); }
00146 template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b)
00147 {
00148 #ifdef EIGEN_VECTORIZE_SSE4_1
00149   return _mm_mullo_epi32(a,b);
00150 #else
00151   // this version is slightly faster than 4 scalar products
00152   return vec4i_swizzle1(
00153             vec4i_swizzle2(
00154               _mm_mul_epu32(a,b),
00155               _mm_mul_epu32(vec4i_swizzle1(a,1,0,3,2),
00156                             vec4i_swizzle1(b,1,0,3,2)),
00157               0,2,0,2),
00158             0,2,1,3);
00159 #endif
00160 }
00161 
00162 template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_div_ps(a,b); }
00163 template<> EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_div_pd(a,b); }
00164 template<> EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& /*a*/, const Packet4i& /*b*/)
00165 { eigen_assert(false && "packet integer division are not supported by SSE");
00166   return pset1<Packet4i>(0);
00167 }
00168 
00169 // for some weird raisons, it has to be overloaded for packet of integers
00170 template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return padd(pmul(a,b), c); }
00171 
00172 template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_min_ps(a,b); }
00173 template<> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_min_pd(a,b); }
00174 template<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b)
00175 {
00176   // after some bench, this version *is* faster than a scalar implementation
00177   Packet4i mask = _mm_cmplt_epi32(a,b);
00178   return _mm_or_si128(_mm_and_si128(mask,a),_mm_andnot_si128(mask,b));
00179 }
00180 
00181 template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_max_ps(a,b); }
00182 template<> EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_max_pd(a,b); }
00183 template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b)
00184 {
00185   // after some bench, this version *is* faster than a scalar implementation
00186   Packet4i mask = _mm_cmpgt_epi32(a,b);
00187   return _mm_or_si128(_mm_and_si128(mask,a),_mm_andnot_si128(mask,b));
00188 }
00189 
00190 template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_and_ps(a,b); }
00191 template<> EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_and_pd(a,b); }
00192 template<> EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_and_si128(a,b); }
00193 
00194 template<> EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_or_ps(a,b); }
00195 template<> EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_or_pd(a,b); }
00196 template<> EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_or_si128(a,b); }
00197 
00198 template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_xor_ps(a,b); }
00199 template<> EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_xor_pd(a,b); }
00200 template<> EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_xor_si128(a,b); }
00201 
00202 template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) { return _mm_andnot_ps(a,b); }
00203 template<> EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) { return _mm_andnot_pd(a,b); }
00204 template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) { return _mm_andnot_si128(a,b); }
00205 
00206 template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float*   from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_ps(from); }
00207 template<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double*  from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_pd(from); }
00208 template<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int*     from) { EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_si128(reinterpret_cast<const Packet4i*>(from)); }
00209 
00210 #if defined(_MSC_VER)
00211   template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float*  from) {
00212     EIGEN_DEBUG_UNALIGNED_LOAD
00213     #if (_MSC_VER==1600)
00214     // NOTE Some version of MSVC10 generates bad code when using _mm_loadu_ps
00215     // (i.e., it does not generate an unaligned load!!
00216     // TODO On most architectures this version should also be faster than a single _mm_loadu_ps
00217     // so we could also enable it for MSVC08 but first we have to make this later does not generate crap when doing so...
00218     __m128 res = _mm_loadl_pi(_mm_set1_ps(0.0f), (const __m64*)(from));
00219     res = _mm_loadh_pi(res, (const __m64*)(from+2));
00220     return res;
00221     #else
00222     return _mm_loadu_ps(from);
00223     #endif
00224   }
00225   template<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_pd(from); }
00226   template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int*    from) { EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_si128(reinterpret_cast<const Packet4i*>(from)); }
00227 #else
00228 // Fast unaligned loads. Note that here we cannot directly use intrinsics: this would
00229 // require pointer casting to incompatible pointer types and leads to invalid code
00230 // because of the strict aliasing rule. The "dummy" stuff are required to enforce
00231 // a correct instruction dependency.
00232 // TODO: do the same for MSVC (ICC is compatible)
00233 // NOTE: with the code below, MSVC's compiler crashes!
00234 
00235 #if defined(__GNUC__) && defined(__i386__)
00236   // bug 195: gcc/i386 emits weird x87 fldl/fstpl instructions for _mm_load_sd
00237   #define EIGEN_AVOID_CUSTOM_UNALIGNED_LOADS 1
00238 #elif defined(__clang__)
00239   // bug 201: Segfaults in __mm_loadh_pd with clang 2.8
00240   #define EIGEN_AVOID_CUSTOM_UNALIGNED_LOADS 1
00241 #else
00242   #define EIGEN_AVOID_CUSTOM_UNALIGNED_LOADS 0
00243 #endif
00244 
00245 template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from)
00246 {
00247   EIGEN_DEBUG_UNALIGNED_LOAD
00248 #if EIGEN_AVOID_CUSTOM_UNALIGNED_LOADS
00249   return _mm_loadu_ps(from);
00250 #else
00251   __m128d res;
00252   res =  _mm_load_sd((const double*)(from)) ;
00253   res =  _mm_loadh_pd(res, (const double*)(from+2)) ;
00254   return _mm_castpd_ps(res);
00255 #endif
00256 }
00257 template<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from)
00258 {
00259   EIGEN_DEBUG_UNALIGNED_LOAD
00260 #if EIGEN_AVOID_CUSTOM_UNALIGNED_LOADS
00261   return _mm_loadu_pd(from);
00262 #else
00263   __m128d res;
00264   res = _mm_load_sd(from) ;
00265   res = _mm_loadh_pd(res,from+1);
00266   return res;
00267 #endif
00268 }
00269 template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from)
00270 {
00271   EIGEN_DEBUG_UNALIGNED_LOAD
00272 #if EIGEN_AVOID_CUSTOM_UNALIGNED_LOADS
00273   return _mm_loadu_si128(reinterpret_cast<const Packet4i*>(from));
00274 #else
00275   __m128d res;
00276   res =  _mm_load_sd((const double*)(from)) ;
00277   res =  _mm_loadh_pd(res, (const double*)(from+2)) ;
00278   return _mm_castpd_si128(res);
00279 #endif
00280 }
00281 #endif
00282 
00283 template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float*   from)
00284 {
00285   return vec4f_swizzle1(_mm_castpd_ps(_mm_load_sd((const double*)from)), 0, 0, 1, 1);
00286 }
00287 template<> EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double*  from)
00288 { return pset1<Packet2d>(from[0]); }
00289 template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int*     from)
00290 {
00291   Packet4i tmp;
00292   tmp = _mm_loadl_epi64(reinterpret_cast<const Packet4i*>(from));
00293   return vec4i_swizzle1(tmp, 0, 0, 1, 1);
00294 }
00295 
00296 template<> EIGEN_STRONG_INLINE void pstore<float>(float*   to, const Packet4f& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_ps(to, from); }
00297 template<> EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_pd(to, from); }
00298 template<> EIGEN_STRONG_INLINE void pstore<int>(int*       to, const Packet4i& from) { EIGEN_DEBUG_ALIGNED_STORE _mm_store_si128(reinterpret_cast<Packet4i*>(to), from); }
00299 
00300 template<> EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from) {
00301   EIGEN_DEBUG_UNALIGNED_STORE
00302   _mm_storel_pd((to), from);
00303   _mm_storeh_pd((to+1), from);
00304 }
00305 template<> EIGEN_STRONG_INLINE void pstoreu<float>(float*  to, const Packet4f& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, _mm_castps_pd(from)); }
00306 template<> EIGEN_STRONG_INLINE void pstoreu<int>(int*      to, const Packet4i& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, _mm_castsi128_pd(from)); }
00307 
00308 // some compilers might be tempted to perform multiple moves instead of using a vector path.
00309 template<> EIGEN_STRONG_INLINE void pstore1<Packet4f>(float* to, const float& a)
00310 {
00311   Packet4f pa = _mm_set_ss(a);
00312   pstore(to, vec4f_swizzle1(pa,0,0,0,0));
00313 }
00314 // some compilers might be tempted to perform multiple moves instead of using a vector path.
00315 template<> EIGEN_STRONG_INLINE void pstore1<Packet2d>(double* to, const double& a)
00316 {
00317   Packet2d pa = _mm_set_sd(a);
00318   pstore(to, vec2d_swizzle1(pa,0,0));
00319 }
00320 
00321 template<> EIGEN_STRONG_INLINE void prefetch<float>(const float*   addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
00322 template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
00323 template<> EIGEN_STRONG_INLINE void prefetch<int>(const int*       addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
00324 
00325 #if defined(_MSC_VER) && defined(_WIN64) && !defined(__INTEL_COMPILER)
00326 // The temporary variable fixes an internal compilation error in vs <= 2008 and a wrong-result bug in vs 2010
00327 // Direct of the struct members fixed bug #62.
00328 template<> EIGEN_STRONG_INLINE float  pfirst<Packet4f>(const Packet4f& a) { return a.m128_f32[0]; }
00329 template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { return a.m128d_f64[0]; }
00330 template<> EIGEN_STRONG_INLINE int    pfirst<Packet4i>(const Packet4i& a) { int x = _mm_cvtsi128_si32(a); return x; }
00331 #elif defined(_MSC_VER) && !defined(__INTEL_COMPILER)
00332 // The temporary variable fixes an internal compilation error in vs <= 2008 and a wrong-result bug in vs 2010
00333 template<> EIGEN_STRONG_INLINE float  pfirst<Packet4f>(const Packet4f& a) { float x = _mm_cvtss_f32(a); return x; }
00334 template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { double x = _mm_cvtsd_f64(a); return x; }
00335 template<> EIGEN_STRONG_INLINE int    pfirst<Packet4i>(const Packet4i& a) { int x = _mm_cvtsi128_si32(a); return x; }
00336 #else
00337 template<> EIGEN_STRONG_INLINE float  pfirst<Packet4f>(const Packet4f& a) { return _mm_cvtss_f32(a); }
00338 template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { return _mm_cvtsd_f64(a); }
00339 template<> EIGEN_STRONG_INLINE int    pfirst<Packet4i>(const Packet4i& a) { return _mm_cvtsi128_si32(a); }
00340 #endif
00341 
00342 template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a)
00343 { return _mm_shuffle_ps(a,a,0x1B); }
00344 template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a)
00345 { return _mm_shuffle_pd(a,a,0x1); }
00346 template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a)
00347 { return _mm_shuffle_epi32(a,0x1B); }
00348 
00349 
00350 template<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a)
00351 {
00352   const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF,0x7FFFFFFF));
00353   return _mm_and_ps(a,mask);
00354 }
00355 template<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a)
00356 {
00357   const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0xFFFFFFFF,0x7FFFFFFF,0xFFFFFFFF,0x7FFFFFFF));
00358   return _mm_and_pd(a,mask);
00359 }
00360 template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a)
00361 {
00362   #ifdef EIGEN_VECTORIZE_SSSE3
00363   return _mm_abs_epi32(a);
00364   #else
00365   Packet4i aux = _mm_srai_epi32(a,31);
00366   return _mm_sub_epi32(_mm_xor_si128(a,aux),aux);
00367   #endif
00368 }
00369 
00370 EIGEN_STRONG_INLINE void punpackp(Packet4f* vecs)
00371 {
00372   vecs[1] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0x55));
00373   vecs[2] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0xAA));
00374   vecs[3] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0xFF));
00375   vecs[0] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0x00));
00376 }
00377 
00378 #ifdef EIGEN_VECTORIZE_SSE3
00379 // TODO implement SSE2 versions as well as integer versions
00380 template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs)
00381 {
00382   return _mm_hadd_ps(_mm_hadd_ps(vecs[0], vecs[1]),_mm_hadd_ps(vecs[2], vecs[3]));
00383 }
00384 template<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(const Packet2d* vecs)
00385 {
00386   return _mm_hadd_pd(vecs[0], vecs[1]);
00387 }
00388 // SSSE3 version:
00389 // EIGEN_STRONG_INLINE Packet4i preduxp(const Packet4i* vecs)
00390 // {
00391 //   return _mm_hadd_epi32(_mm_hadd_epi32(vecs[0], vecs[1]),_mm_hadd_epi32(vecs[2], vecs[3]));
00392 // }
00393 
00394 template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a)
00395 {
00396   Packet4f tmp0 = _mm_hadd_ps(a,a);
00397   return pfirst(_mm_hadd_ps(tmp0, tmp0));
00398 }
00399 
00400 template<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) { return pfirst(_mm_hadd_pd(a, a)); }
00401 
00402 // SSSE3 version:
00403 // EIGEN_STRONG_INLINE float predux(const Packet4i& a)
00404 // {
00405 //   Packet4i tmp0 = _mm_hadd_epi32(a,a);
00406 //   return pfirst(_mm_hadd_epi32(tmp0, tmp0));
00407 // }
00408 #else
00409 // SSE2 versions
00410 template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a)
00411 {
00412   Packet4f tmp = _mm_add_ps(a, _mm_movehl_ps(a,a));
00413   return pfirst(_mm_add_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
00414 }
00415 template<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a)
00416 {
00417   return pfirst(_mm_add_sd(a, _mm_unpackhi_pd(a,a)));
00418 }
00419 
00420 template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs)
00421 {
00422   Packet4f tmp0, tmp1, tmp2;
00423   tmp0 = _mm_unpacklo_ps(vecs[0], vecs[1]);
00424   tmp1 = _mm_unpackhi_ps(vecs[0], vecs[1]);
00425   tmp2 = _mm_unpackhi_ps(vecs[2], vecs[3]);
00426   tmp0 = _mm_add_ps(tmp0, tmp1);
00427   tmp1 = _mm_unpacklo_ps(vecs[2], vecs[3]);
00428   tmp1 = _mm_add_ps(tmp1, tmp2);
00429   tmp2 = _mm_movehl_ps(tmp1, tmp0);
00430   tmp0 = _mm_movelh_ps(tmp0, tmp1);
00431   return _mm_add_ps(tmp0, tmp2);
00432 }
00433 
00434 template<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(const Packet2d* vecs)
00435 {
00436   return _mm_add_pd(_mm_unpacklo_pd(vecs[0], vecs[1]), _mm_unpackhi_pd(vecs[0], vecs[1]));
00437 }
00438 #endif  // SSE3
00439 
00440 template<> EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a)
00441 {
00442   Packet4i tmp = _mm_add_epi32(a, _mm_unpackhi_epi64(a,a));
00443   return pfirst(tmp) + pfirst(_mm_shuffle_epi32(tmp, 1));
00444 }
00445 
00446 template<> EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(const Packet4i* vecs)
00447 {
00448   Packet4i tmp0, tmp1, tmp2;
00449   tmp0 = _mm_unpacklo_epi32(vecs[0], vecs[1]);
00450   tmp1 = _mm_unpackhi_epi32(vecs[0], vecs[1]);
00451   tmp2 = _mm_unpackhi_epi32(vecs[2], vecs[3]);
00452   tmp0 = _mm_add_epi32(tmp0, tmp1);
00453   tmp1 = _mm_unpacklo_epi32(vecs[2], vecs[3]);
00454   tmp1 = _mm_add_epi32(tmp1, tmp2);
00455   tmp2 = _mm_unpacklo_epi64(tmp0, tmp1);
00456   tmp0 = _mm_unpackhi_epi64(tmp0, tmp1);
00457   return _mm_add_epi32(tmp0, tmp2);
00458 }
00459 
00460 // Other reduction functions:
00461 
00462 // mul
00463 template<> EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a)
00464 {
00465   Packet4f tmp = _mm_mul_ps(a, _mm_movehl_ps(a,a));
00466   return pfirst(_mm_mul_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
00467 }
00468 template<> EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a)
00469 {
00470   return pfirst(_mm_mul_sd(a, _mm_unpackhi_pd(a,a)));
00471 }
00472 template<> EIGEN_STRONG_INLINE int predux_mul<Packet4i>(const Packet4i& a)
00473 {
00474   // after some experiments, it is seems this is the fastest way to implement it
00475   // for GCC (eg., reusing pmul is very slow !)
00476   // TODO try to call _mm_mul_epu32 directly
00477   EIGEN_ALIGN16 int aux[4];
00478   pstore(aux, a);
00479   return  (aux[0] * aux[1]) * (aux[2] * aux[3]);;
00480 }
00481 
00482 // min
00483 template<> EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a)
00484 {
00485   Packet4f tmp = _mm_min_ps(a, _mm_movehl_ps(a,a));
00486   return pfirst(_mm_min_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
00487 }
00488 template<> EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a)
00489 {
00490   return pfirst(_mm_min_sd(a, _mm_unpackhi_pd(a,a)));
00491 }
00492 template<> EIGEN_STRONG_INLINE int predux_min<Packet4i>(const Packet4i& a)
00493 {
00494   // after some experiments, it is seems this is the fastest way to implement it
00495   // for GCC (eg., it does not like using std::min after the pstore !!)
00496   EIGEN_ALIGN16 int aux[4];
00497   pstore(aux, a);
00498   register int aux0 = aux[0]<aux[1] ? aux[0] : aux[1];
00499   register int aux2 = aux[2]<aux[3] ? aux[2] : aux[3];
00500   return aux0<aux2 ? aux0 : aux2;
00501 }
00502 
00503 // max
00504 template<> EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a)
00505 {
00506   Packet4f tmp = _mm_max_ps(a, _mm_movehl_ps(a,a));
00507   return pfirst(_mm_max_ss(tmp, _mm_shuffle_ps(tmp,tmp, 1)));
00508 }
00509 template<> EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a)
00510 {
00511   return pfirst(_mm_max_sd(a, _mm_unpackhi_pd(a,a)));
00512 }
00513 template<> EIGEN_STRONG_INLINE int predux_max<Packet4i>(const Packet4i& a)
00514 {
00515   // after some experiments, it is seems this is the fastest way to implement it
00516   // for GCC (eg., it does not like using std::min after the pstore !!)
00517   EIGEN_ALIGN16 int aux[4];
00518   pstore(aux, a);
00519   register int aux0 = aux[0]>aux[1] ? aux[0] : aux[1];
00520   register int aux2 = aux[2]>aux[3] ? aux[2] : aux[3];
00521   return aux0>aux2 ? aux0 : aux2;
00522 }
00523 
00524 #if (defined __GNUC__)
00525 // template <> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f&  a, const Packet4f&  b, const Packet4f&  c)
00526 // {
00527 //   Packet4f res = b;
00528 //   asm("mulps %[a], %[b] \n\taddps %[c], %[b]" : [b] "+x" (res) : [a] "x" (a), [c] "x" (c));
00529 //   return res;
00530 // }
00531 // EIGEN_STRONG_INLINE Packet4i _mm_alignr_epi8(const Packet4i&  a, const Packet4i&  b, const int i)
00532 // {
00533 //   Packet4i res = a;
00534 //   asm("palignr %[i], %[a], %[b] " : [b] "+x" (res) : [a] "x" (a), [i] "i" (i));
00535 //   return res;
00536 // }
00537 #endif
00538 
00539 #ifdef EIGEN_VECTORIZE_SSSE3
00540 // SSSE3 versions
00541 template<int Offset>
00542 struct palign_impl<Offset,Packet4f>
00543 {
00544   EIGEN_STRONG_INLINE static void run(Packet4f& first, const Packet4f& second)
00545   {
00546     if (Offset!=0)
00547       first = _mm_castsi128_ps(_mm_alignr_epi8(_mm_castps_si128(second), _mm_castps_si128(first), Offset*4));
00548   }
00549 };
00550 
00551 template<int Offset>
00552 struct palign_impl<Offset,Packet4i>
00553 {
00554   EIGEN_STRONG_INLINE static void run(Packet4i& first, const Packet4i& second)
00555   {
00556     if (Offset!=0)
00557       first = _mm_alignr_epi8(second,first, Offset*4);
00558   }
00559 };
00560 
00561 template<int Offset>
00562 struct palign_impl<Offset,Packet2d>
00563 {
00564   EIGEN_STRONG_INLINE static void run(Packet2d& first, const Packet2d& second)
00565   {
00566     if (Offset==1)
00567       first = _mm_castsi128_pd(_mm_alignr_epi8(_mm_castpd_si128(second), _mm_castpd_si128(first), 8));
00568   }
00569 };
00570 #else
00571 // SSE2 versions
00572 template<int Offset>
00573 struct palign_impl<Offset,Packet4f>
00574 {
00575   EIGEN_STRONG_INLINE static void run(Packet4f& first, const Packet4f& second)
00576   {
00577     if (Offset==1)
00578     {
00579       first = _mm_move_ss(first,second);
00580       first = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(first),0x39));
00581     }
00582     else if (Offset==2)
00583     {
00584       first = _mm_movehl_ps(first,first);
00585       first = _mm_movelh_ps(first,second);
00586     }
00587     else if (Offset==3)
00588     {
00589       first = _mm_move_ss(first,second);
00590       first = _mm_shuffle_ps(first,second,0x93);
00591     }
00592   }
00593 };
00594 
00595 template<int Offset>
00596 struct palign_impl<Offset,Packet4i>
00597 {
00598   EIGEN_STRONG_INLINE static void run(Packet4i& first, const Packet4i& second)
00599   {
00600     if (Offset==1)
00601     {
00602       first = _mm_castps_si128(_mm_move_ss(_mm_castsi128_ps(first),_mm_castsi128_ps(second)));
00603       first = _mm_shuffle_epi32(first,0x39);
00604     }
00605     else if (Offset==2)
00606     {
00607       first = _mm_castps_si128(_mm_movehl_ps(_mm_castsi128_ps(first),_mm_castsi128_ps(first)));
00608       first = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(first),_mm_castsi128_ps(second)));
00609     }
00610     else if (Offset==3)
00611     {
00612       first = _mm_castps_si128(_mm_move_ss(_mm_castsi128_ps(first),_mm_castsi128_ps(second)));
00613       first = _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(first),_mm_castsi128_ps(second),0x93));
00614     }
00615   }
00616 };
00617 
00618 template<int Offset>
00619 struct palign_impl<Offset,Packet2d>
00620 {
00621   EIGEN_STRONG_INLINE static void run(Packet2d& first, const Packet2d& second)
00622   {
00623     if (Offset==1)
00624     {
00625       first = _mm_castps_pd(_mm_movehl_ps(_mm_castpd_ps(first),_mm_castpd_ps(first)));
00626       first = _mm_castps_pd(_mm_movelh_ps(_mm_castpd_ps(first),_mm_castpd_ps(second)));
00627     }
00628   }
00629 };
00630 #endif
00631 
00632 } // end namespace internal
00633 
00634 #endif // EIGEN_PACKET_MATH_SSE_H


libicr
Author(s): Robert Krug
autogenerated on Mon Jan 6 2014 11:33:08