00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019 #ifndef EIGEN_MEMORY_H
00020 #define EIGEN_MEMORY_H
00021
00022 #ifndef EIGEN_MALLOC_ALREADY_ALIGNED
00023
00024
00025
00026
00027
00028
00029
00030
00031
00032
00033 #if defined(__GLIBC__) && ((__GLIBC__>=2 && __GLIBC_MINOR__ >= 8) || __GLIBC__>2) \
00034 && defined(__LP64__) && ! defined( __SANITIZE_ADDRESS__ )
00035 #define EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED 1
00036 #else
00037 #define EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED 0
00038 #endif
00039
00040
00041
00042
00043
00044 #if defined(__FreeBSD__) && !defined(__arm__) && !defined(__mips__)
00045 #define EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED 1
00046 #else
00047 #define EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED 0
00048 #endif
00049
00050 #if defined(__APPLE__) \
00051 || defined(_WIN64) \
00052 || EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED \
00053 || EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED
00054 #define EIGEN_MALLOC_ALREADY_ALIGNED 1
00055 #else
00056 #define EIGEN_MALLOC_ALREADY_ALIGNED 0
00057 #endif
00058
00059 #endif
00060
00061
00062
00063
00064 #if defined(__unix__) || defined(__unix)
00065 #include <unistd.h>
00066 #if ((defined __QNXNTO__) || (defined _GNU_SOURCE) || (defined __PGI) || ((defined _XOPEN_SOURCE) && (_XOPEN_SOURCE >= 600))) && (defined _POSIX_ADVISORY_INFO) && (_POSIX_ADVISORY_INFO > 0)
00067 #define EIGEN_HAS_POSIX_MEMALIGN 1
00068 #endif
00069 #endif
00070
00071 #ifndef EIGEN_HAS_POSIX_MEMALIGN
00072 #define EIGEN_HAS_POSIX_MEMALIGN 0
00073 #endif
00074
00075 #ifdef EIGEN_VECTORIZE_SSE
00076 #define EIGEN_HAS_MM_MALLOC 1
00077 #else
00078 #define EIGEN_HAS_MM_MALLOC 0
00079 #endif
00080
00081 namespace Eigen {
00082
00083 namespace internal {
00084
00085 inline void throw_std_bad_alloc()
00086 {
00087 #ifdef EIGEN_EXCEPTIONS
00088 throw std::bad_alloc();
00089 #else
00090 std::size_t huge = -1;
00091 new int[huge];
00092 #endif
00093 }
00094
00095
00096
00097
00098
00099
00100
00104 inline void* handmade_aligned_malloc(std::size_t size)
00105 {
00106 void *original = std::malloc(size+16);
00107 if (original == 0) return 0;
00108 void *aligned = reinterpret_cast<void*>((reinterpret_cast<std::size_t>(original) & ~(std::size_t(15))) + 16);
00109 *(reinterpret_cast<void**>(aligned) - 1) = original;
00110 return aligned;
00111 }
00112
00114 inline void handmade_aligned_free(void *ptr)
00115 {
00116 if (ptr) std::free(*(reinterpret_cast<void**>(ptr) - 1));
00117 }
00118
00124 inline void* handmade_aligned_realloc(void* ptr, std::size_t size, std::size_t = 0)
00125 {
00126 if (ptr == 0) return handmade_aligned_malloc(size);
00127 void *original = *(reinterpret_cast<void**>(ptr) - 1);
00128 std::ptrdiff_t previous_offset = static_cast<char *>(ptr)-static_cast<char *>(original);
00129 original = std::realloc(original,size+16);
00130 if (original == 0) return 0;
00131 void *aligned = reinterpret_cast<void*>((reinterpret_cast<std::size_t>(original) & ~(std::size_t(15))) + 16);
00132 void *previous_aligned = static_cast<char *>(original)+previous_offset;
00133 if(aligned!=previous_aligned)
00134 std::memmove(aligned, previous_aligned, size);
00135
00136 *(reinterpret_cast<void**>(aligned) - 1) = original;
00137 return aligned;
00138 }
00139
00140
00141
00142
00143
00144 void* aligned_malloc(std::size_t size);
00145 void aligned_free(void *ptr);
00146
00152 inline void* generic_aligned_realloc(void* ptr, size_t size, size_t old_size)
00153 {
00154 if (ptr==0)
00155 return aligned_malloc(size);
00156
00157 if (size==0)
00158 {
00159 aligned_free(ptr);
00160 return 0;
00161 }
00162
00163 void* newptr = aligned_malloc(size);
00164 if (newptr == 0)
00165 {
00166 #ifdef EIGEN_HAS_ERRNO
00167 errno = ENOMEM;
00168 #endif
00169 return 0;
00170 }
00171
00172 if (ptr != 0)
00173 {
00174 std::memcpy(newptr, ptr, (std::min)(size,old_size));
00175 aligned_free(ptr);
00176 }
00177
00178 return newptr;
00179 }
00180
00181
00182
00183
00184
00185 #ifdef EIGEN_NO_MALLOC
00186 inline void check_that_malloc_is_allowed()
00187 {
00188 eigen_assert(false && "heap allocation is forbidden (EIGEN_NO_MALLOC is defined)");
00189 }
00190 #elif defined EIGEN_RUNTIME_NO_MALLOC
00191 inline bool is_malloc_allowed_impl(bool update, bool new_value = false)
00192 {
00193 static bool value = true;
00194 if (update == 1)
00195 value = new_value;
00196 return value;
00197 }
00198 inline bool is_malloc_allowed() { return is_malloc_allowed_impl(false); }
00199 inline bool set_is_malloc_allowed(bool new_value) { return is_malloc_allowed_impl(true, new_value); }
00200 inline void check_that_malloc_is_allowed()
00201 {
00202 eigen_assert(is_malloc_allowed() && "heap allocation is forbidden (EIGEN_RUNTIME_NO_MALLOC is defined and g_is_malloc_allowed is false)");
00203 }
00204 #else
00205 inline void check_that_malloc_is_allowed()
00206 {}
00207 #endif
00208
00212 inline void* aligned_malloc(size_t size)
00213 {
00214 check_that_malloc_is_allowed();
00215
00216 void *result;
00217 #if !EIGEN_ALIGN
00218 result = std::malloc(size);
00219 #elif EIGEN_MALLOC_ALREADY_ALIGNED
00220 result = std::malloc(size);
00221 #elif EIGEN_HAS_POSIX_MEMALIGN
00222 if(posix_memalign(&result, 16, size)) result = 0;
00223 #elif EIGEN_HAS_MM_MALLOC
00224 result = _mm_malloc(size, 16);
00225 #elif defined(_MSC_VER) && (!defined(_WIN32_WCE))
00226 result = _aligned_malloc(size, 16);
00227 #else
00228 result = handmade_aligned_malloc(size);
00229 #endif
00230
00231 if(!result && size)
00232 throw_std_bad_alloc();
00233
00234 return result;
00235 }
00236
00238 inline void aligned_free(void *ptr)
00239 {
00240 #if !EIGEN_ALIGN
00241 std::free(ptr);
00242 #elif EIGEN_MALLOC_ALREADY_ALIGNED
00243 std::free(ptr);
00244 #elif EIGEN_HAS_POSIX_MEMALIGN
00245 std::free(ptr);
00246 #elif EIGEN_HAS_MM_MALLOC
00247 _mm_free(ptr);
00248 #elif defined(_MSC_VER) && (!defined(_WIN32_WCE))
00249 _aligned_free(ptr);
00250 #else
00251 handmade_aligned_free(ptr);
00252 #endif
00253 }
00254
00260 inline void* aligned_realloc(void *ptr, size_t new_size, size_t old_size)
00261 {
00262 EIGEN_UNUSED_VARIABLE(old_size);
00263
00264 void *result;
00265 #if !EIGEN_ALIGN
00266 result = std::realloc(ptr,new_size);
00267 #elif EIGEN_MALLOC_ALREADY_ALIGNED
00268 result = std::realloc(ptr,new_size);
00269 #elif EIGEN_HAS_POSIX_MEMALIGN
00270 result = generic_aligned_realloc(ptr,new_size,old_size);
00271 #elif EIGEN_HAS_MM_MALLOC
00272
00273
00274
00275 #if defined(_MSC_VER) && (!defined(_WIN32_WCE)) && defined(_mm_free)
00276 result = _aligned_realloc(ptr,new_size,16);
00277 #else
00278 result = generic_aligned_realloc(ptr,new_size,old_size);
00279 #endif
00280 #elif defined(_MSC_VER) && (!defined(_WIN32_WCE))
00281 result = _aligned_realloc(ptr,new_size,16);
00282 #else
00283 result = handmade_aligned_realloc(ptr,new_size,old_size);
00284 #endif
00285
00286 if (!result && new_size)
00287 throw_std_bad_alloc();
00288
00289 return result;
00290 }
00291
00292
00293
00294
00295
00299 template<bool Align> inline void* conditional_aligned_malloc(size_t size)
00300 {
00301 return aligned_malloc(size);
00302 }
00303
00304 template<> inline void* conditional_aligned_malloc<false>(size_t size)
00305 {
00306 check_that_malloc_is_allowed();
00307
00308 void *result = std::malloc(size);
00309 if(!result && size)
00310 throw_std_bad_alloc();
00311 return result;
00312 }
00313
00315 template<bool Align> inline void conditional_aligned_free(void *ptr)
00316 {
00317 aligned_free(ptr);
00318 }
00319
00320 template<> inline void conditional_aligned_free<false>(void *ptr)
00321 {
00322 std::free(ptr);
00323 }
00324
00325 template<bool Align> inline void* conditional_aligned_realloc(void* ptr, size_t new_size, size_t old_size)
00326 {
00327 return aligned_realloc(ptr, new_size, old_size);
00328 }
00329
00330 template<> inline void* conditional_aligned_realloc<false>(void* ptr, size_t new_size, size_t)
00331 {
00332 return std::realloc(ptr, new_size);
00333 }
00334
00335
00336
00337
00338
00342 template<typename T> inline T* construct_elements_of_array(T *ptr, size_t size)
00343 {
00344 for (size_t i=0; i < size; ++i) ::new (ptr + i) T;
00345 return ptr;
00346 }
00347
00351 template<typename T> inline void destruct_elements_of_array(T *ptr, size_t size)
00352 {
00353
00354 if(ptr)
00355 while(size) ptr[--size].~T();
00356 }
00357
00358
00359
00360
00361
00362 template<typename T>
00363 EIGEN_ALWAYS_INLINE void check_size_for_overflow(size_t size)
00364 {
00365 if(size > size_t(-1) / sizeof(T))
00366 throw_std_bad_alloc();
00367 }
00368
00373 template<typename T> inline T* aligned_new(size_t size)
00374 {
00375 check_size_for_overflow<T>(size);
00376 T *result = reinterpret_cast<T*>(aligned_malloc(sizeof(T)*size));
00377 return construct_elements_of_array(result, size);
00378 }
00379
00380 template<typename T, bool Align> inline T* conditional_aligned_new(size_t size)
00381 {
00382 check_size_for_overflow<T>(size);
00383 T *result = reinterpret_cast<T*>(conditional_aligned_malloc<Align>(sizeof(T)*size));
00384 return construct_elements_of_array(result, size);
00385 }
00386
00390 template<typename T> inline void aligned_delete(T *ptr, size_t size)
00391 {
00392 destruct_elements_of_array<T>(ptr, size);
00393 aligned_free(ptr);
00394 }
00395
00399 template<typename T, bool Align> inline void conditional_aligned_delete(T *ptr, size_t size)
00400 {
00401 destruct_elements_of_array<T>(ptr, size);
00402 conditional_aligned_free<Align>(ptr);
00403 }
00404
00405 template<typename T, bool Align> inline T* conditional_aligned_realloc_new(T* pts, size_t new_size, size_t old_size)
00406 {
00407 check_size_for_overflow<T>(new_size);
00408 check_size_for_overflow<T>(old_size);
00409 if(new_size < old_size)
00410 destruct_elements_of_array(pts+new_size, old_size-new_size);
00411 T *result = reinterpret_cast<T*>(conditional_aligned_realloc<Align>(reinterpret_cast<void*>(pts), sizeof(T)*new_size, sizeof(T)*old_size));
00412 if(new_size > old_size)
00413 construct_elements_of_array(result+old_size, new_size-old_size);
00414 return result;
00415 }
00416
00417
00418 template<typename T, bool Align> inline T* conditional_aligned_new_auto(size_t size)
00419 {
00420 if(size==0)
00421 return 0;
00422 check_size_for_overflow<T>(size);
00423 T *result = reinterpret_cast<T*>(conditional_aligned_malloc<Align>(sizeof(T)*size));
00424 if(NumTraits<T>::RequireInitialization)
00425 construct_elements_of_array(result, size);
00426 return result;
00427 }
00428
00429 template<typename T, bool Align> inline T* conditional_aligned_realloc_new_auto(T* pts, size_t new_size, size_t old_size)
00430 {
00431 check_size_for_overflow<T>(new_size);
00432 check_size_for_overflow<T>(old_size);
00433 if(NumTraits<T>::RequireInitialization && (new_size < old_size))
00434 destruct_elements_of_array(pts+new_size, old_size-new_size);
00435 T *result = reinterpret_cast<T*>(conditional_aligned_realloc<Align>(reinterpret_cast<void*>(pts), sizeof(T)*new_size, sizeof(T)*old_size));
00436 if(NumTraits<T>::RequireInitialization && (new_size > old_size))
00437 construct_elements_of_array(result+old_size, new_size-old_size);
00438 return result;
00439 }
00440
00441 template<typename T, bool Align> inline void conditional_aligned_delete_auto(T *ptr, size_t size)
00442 {
00443 if(NumTraits<T>::RequireInitialization)
00444 destruct_elements_of_array<T>(ptr, size);
00445 conditional_aligned_free<Align>(ptr);
00446 }
00447
00448
00449
00466 template<typename Scalar, typename Index>
00467 static inline Index first_aligned(const Scalar* array, Index size)
00468 {
00469 static const Index PacketSize = packet_traits<Scalar>::size;
00470 static const Index PacketAlignedMask = PacketSize-1;
00471
00472 if(PacketSize==1)
00473 {
00474
00475
00476 return 0;
00477 }
00478 else if(size_t(array) & (sizeof(Scalar)-1))
00479 {
00480
00481
00482 return size;
00483 }
00484 else
00485 {
00486 return std::min<Index>( (PacketSize - (Index((size_t(array)/sizeof(Scalar))) & PacketAlignedMask))
00487 & PacketAlignedMask, size);
00488 }
00489 }
00490
00493 template<typename Index>
00494 inline static Index first_multiple(Index size, Index base)
00495 {
00496 return ((size+base-1)/base)*base;
00497 }
00498
00499
00500
00501 template<typename T, bool UseMemcpy> struct smart_copy_helper;
00502
00503 template<typename T> void smart_copy(const T* start, const T* end, T* target)
00504 {
00505 smart_copy_helper<T,!NumTraits<T>::RequireInitialization>::run(start, end, target);
00506 }
00507
00508 template<typename T> struct smart_copy_helper<T,true> {
00509 static inline void run(const T* start, const T* end, T* target)
00510 { memcpy(target, start, std::ptrdiff_t(end)-std::ptrdiff_t(start)); }
00511 };
00512
00513 template<typename T> struct smart_copy_helper<T,false> {
00514 static inline void run(const T* start, const T* end, T* target)
00515 { std::copy(start, end, target); }
00516 };
00517
00518
00519
00520
00521
00522
00523
00524
00525 #ifndef EIGEN_ALLOCA
00526 #if (defined __linux__) || (defined __APPLE__) || (defined alloca)
00527 #define EIGEN_ALLOCA alloca
00528 #elif defined(_MSC_VER)
00529 #define EIGEN_ALLOCA _alloca
00530 #endif
00531 #endif
00532
00533
00534
00535 template<typename T> class aligned_stack_memory_handler
00536 {
00537 public:
00538
00539
00540
00541
00542
00543
00544 aligned_stack_memory_handler(T* ptr, size_t size, bool dealloc)
00545 : m_ptr(ptr), m_size(size), m_deallocate(dealloc)
00546 {
00547 if(NumTraits<T>::RequireInitialization && m_ptr)
00548 Eigen::internal::construct_elements_of_array(m_ptr, size);
00549 }
00550 ~aligned_stack_memory_handler()
00551 {
00552 if(NumTraits<T>::RequireInitialization && m_ptr)
00553 Eigen::internal::destruct_elements_of_array<T>(m_ptr, m_size);
00554 if(m_deallocate)
00555 Eigen::internal::aligned_free(m_ptr);
00556 }
00557 protected:
00558 T* m_ptr;
00559 size_t m_size;
00560 bool m_deallocate;
00561 };
00562
00563 }
00564
00580 #ifdef EIGEN_ALLOCA
00581
00582 #if defined(__arm__) || defined(_WIN32)
00583 #define EIGEN_ALIGNED_ALLOCA(SIZE) reinterpret_cast<void*>((reinterpret_cast<size_t>(EIGEN_ALLOCA(SIZE+16)) & ~(size_t(15))) + 16)
00584 #else
00585 #define EIGEN_ALIGNED_ALLOCA EIGEN_ALLOCA
00586 #endif
00587
00588 #define ei_declare_aligned_stack_constructed_variable(TYPE,NAME,SIZE,BUFFER) \
00589 Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \
00590 TYPE* NAME = (BUFFER)!=0 ? (BUFFER) \
00591 : reinterpret_cast<TYPE*>( \
00592 (sizeof(TYPE)*SIZE<=EIGEN_STACK_ALLOCATION_LIMIT) ? EIGEN_ALIGNED_ALLOCA(sizeof(TYPE)*SIZE) \
00593 : Eigen::internal::aligned_malloc(sizeof(TYPE)*SIZE) ); \
00594 Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME,_stack_memory_destructor)((BUFFER)==0 ? NAME : 0,SIZE,sizeof(TYPE)*SIZE>EIGEN_STACK_ALLOCATION_LIMIT)
00595
00596 #else
00597
00598 #define ei_declare_aligned_stack_constructed_variable(TYPE,NAME,SIZE,BUFFER) \
00599 Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \
00600 TYPE* NAME = (BUFFER)!=0 ? BUFFER : reinterpret_cast<TYPE*>(Eigen::internal::aligned_malloc(sizeof(TYPE)*SIZE)); \
00601 Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME,_stack_memory_destructor)((BUFFER)==0 ? NAME : 0,SIZE,true)
00602
00603 #endif
00604
00605
00606
00607
00608
00609
00610 #if EIGEN_ALIGN
00611 #ifdef EIGEN_EXCEPTIONS
00612 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \
00613 void* operator new(size_t size, const std::nothrow_t&) throw() { \
00614 try { return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); } \
00615 catch (...) { return 0; } \
00616 }
00617 #else
00618 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \
00619 void* operator new(size_t size, const std::nothrow_t&) throw() { \
00620 return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \
00621 }
00622 #endif
00623
00624 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign) \
00625 void *operator new(size_t size) { \
00626 return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \
00627 } \
00628 void *operator new[](size_t size) { \
00629 return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \
00630 } \
00631 void operator delete(void * ptr) throw() { Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); } \
00632 void operator delete[](void * ptr) throw() { Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); } \
00633 \
00634 \
00635 \
00636 static void *operator new(size_t size, void *ptr) { return ::operator new(size,ptr); } \
00637 static void *operator new[](size_t size, void* ptr) { return ::operator new[](size,ptr); } \
00638 void operator delete(void * memory, void *ptr) throw() { return ::operator delete(memory,ptr); } \
00639 void operator delete[](void * memory, void *ptr) throw() { return ::operator delete[](memory,ptr); } \
00640 \
00641 EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \
00642 void operator delete(void *ptr, const std::nothrow_t&) throw() { \
00643 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
00644 } \
00645 typedef void eigen_aligned_operator_new_marker_type;
00646 #else
00647 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign)
00648 #endif
00649
00650 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(true)
00651 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar,Size) \
00652 EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(bool(((Size)!=Eigen::Dynamic) && ((sizeof(Scalar)*(Size))%16==0)))
00653
00654
00655
00672 template<class T>
00673 class aligned_allocator
00674 {
00675 public:
00676 typedef size_t size_type;
00677 typedef std::ptrdiff_t difference_type;
00678 typedef T* pointer;
00679 typedef const T* const_pointer;
00680 typedef T& reference;
00681 typedef const T& const_reference;
00682 typedef T value_type;
00683
00684 template<class U>
00685 struct rebind
00686 {
00687 typedef aligned_allocator<U> other;
00688 };
00689
00690 pointer address( reference value ) const
00691 {
00692 return &value;
00693 }
00694
00695 const_pointer address( const_reference value ) const
00696 {
00697 return &value;
00698 }
00699
00700 aligned_allocator()
00701 {
00702 }
00703
00704 aligned_allocator( const aligned_allocator& )
00705 {
00706 }
00707
00708 template<class U>
00709 aligned_allocator( const aligned_allocator<U>& )
00710 {
00711 }
00712
00713 ~aligned_allocator()
00714 {
00715 }
00716
00717 size_type max_size() const
00718 {
00719 return (std::numeric_limits<size_type>::max)();
00720 }
00721
00722 pointer allocate( size_type num, const void* hint = 0 )
00723 {
00724 EIGEN_UNUSED_VARIABLE(hint);
00725 internal::check_size_for_overflow<T>(num);
00726 return static_cast<pointer>( internal::aligned_malloc( num * sizeof(T) ) );
00727 }
00728
00729 void construct( pointer p, const T& value )
00730 {
00731 ::new( p ) T( value );
00732 }
00733
00734 void destroy( pointer p )
00735 {
00736 p->~T();
00737 }
00738
00739 void deallocate( pointer p, size_type )
00740 {
00741 internal::aligned_free( p );
00742 }
00743
00744 bool operator!=(const aligned_allocator<T>& ) const
00745 { return false; }
00746
00747 bool operator==(const aligned_allocator<T>& ) const
00748 { return true; }
00749 };
00750
00751
00752
00753 #if !defined(EIGEN_NO_CPUID)
00754 # if defined(__GNUC__) && ( defined(__i386__) || defined(__x86_64__) )
00755 # if defined(__PIC__) && defined(__i386__)
00756
00757 # define EIGEN_CPUID(abcd,func,id) \
00758 __asm__ __volatile__ ("xchgl %%ebx, %k1;cpuid; xchgl %%ebx,%k1": "=a" (abcd[0]), "=&r" (abcd[1]), "=c" (abcd[2]), "=d" (abcd[3]) : "a" (func), "c" (id));
00759 # elif defined(__PIC__) && defined(__x86_64__)
00760
00761
00762 # define EIGEN_CPUID(abcd,func,id) \
00763 __asm__ __volatile__ ("xchg{q}\t{%%}rbx, %q1; cpuid; xchg{q}\t{%%}rbx, %q1": "=a" (abcd[0]), "=&r" (abcd[1]), "=c" (abcd[2]), "=d" (abcd[3]) : "0" (func), "2" (id));
00764 # else
00765
00766 # define EIGEN_CPUID(abcd,func,id) \
00767 __asm__ __volatile__ ("cpuid": "=a" (abcd[0]), "=b" (abcd[1]), "=c" (abcd[2]), "=d" (abcd[3]) : "0" (func), "2" (id) );
00768 # endif
00769 # elif defined(_MSC_VER)
00770 # if (_MSC_VER > 1500) && ( defined(_M_IX86) || defined(_M_X64) )
00771 # define EIGEN_CPUID(abcd,func,id) __cpuidex((int*)abcd,func,id)
00772 # endif
00773 # endif
00774 #endif
00775
00776 namespace internal {
00777
00778 #ifdef EIGEN_CPUID
00779
00780 inline bool cpuid_is_vendor(int abcd[4], const int vendor[3])
00781 {
00782 return abcd[1]==vendor[0] && abcd[3]==vendor[1] && abcd[2]==vendor[2];
00783 }
00784
00785 inline void queryCacheSizes_intel_direct(int& l1, int& l2, int& l3)
00786 {
00787 int abcd[4];
00788 l1 = l2 = l3 = 0;
00789 int cache_id = 0;
00790 int cache_type = 0;
00791 do {
00792 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
00793 EIGEN_CPUID(abcd,0x4,cache_id);
00794 cache_type = (abcd[0] & 0x0F) >> 0;
00795 if(cache_type==1||cache_type==3)
00796 {
00797 int cache_level = (abcd[0] & 0xE0) >> 5;
00798 int ways = (abcd[1] & 0xFFC00000) >> 22;
00799 int partitions = (abcd[1] & 0x003FF000) >> 12;
00800 int line_size = (abcd[1] & 0x00000FFF) >> 0;
00801 int sets = (abcd[2]);
00802
00803 int cache_size = (ways+1) * (partitions+1) * (line_size+1) * (sets+1);
00804
00805 switch(cache_level)
00806 {
00807 case 1: l1 = cache_size; break;
00808 case 2: l2 = cache_size; break;
00809 case 3: l3 = cache_size; break;
00810 default: break;
00811 }
00812 }
00813 cache_id++;
00814 } while(cache_type>0 && cache_id<16);
00815 }
00816
00817 inline void queryCacheSizes_intel_codes(int& l1, int& l2, int& l3)
00818 {
00819 int abcd[4];
00820 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
00821 l1 = l2 = l3 = 0;
00822 EIGEN_CPUID(abcd,0x00000002,0);
00823 unsigned char * bytes = reinterpret_cast<unsigned char *>(abcd)+2;
00824 bool check_for_p2_core2 = false;
00825 for(int i=0; i<14; ++i)
00826 {
00827 switch(bytes[i])
00828 {
00829 case 0x0A: l1 = 8; break;
00830 case 0x0C: l1 = 16; break;
00831 case 0x0E: l1 = 24; break;
00832 case 0x10: l1 = 16; break;
00833 case 0x15: l1 = 16; break;
00834 case 0x2C: l1 = 32; break;
00835 case 0x30: l1 = 32; break;
00836 case 0x60: l1 = 16; break;
00837 case 0x66: l1 = 8; break;
00838 case 0x67: l1 = 16; break;
00839 case 0x68: l1 = 32; break;
00840 case 0x1A: l2 = 96; break;
00841 case 0x22: l3 = 512; break;
00842 case 0x23: l3 = 1024; break;
00843 case 0x25: l3 = 2048; break;
00844 case 0x29: l3 = 4096; break;
00845 case 0x39: l2 = 128; break;
00846 case 0x3A: l2 = 192; break;
00847 case 0x3B: l2 = 128; break;
00848 case 0x3C: l2 = 256; break;
00849 case 0x3D: l2 = 384; break;
00850 case 0x3E: l2 = 512; break;
00851 case 0x40: l2 = 0; break;
00852 case 0x41: l2 = 128; break;
00853 case 0x42: l2 = 256; break;
00854 case 0x43: l2 = 512; break;
00855 case 0x44: l2 = 1024; break;
00856 case 0x45: l2 = 2048; break;
00857 case 0x46: l3 = 4096; break;
00858 case 0x47: l3 = 8192; break;
00859 case 0x48: l2 = 3072; break;
00860 case 0x49: if(l2!=0) l3 = 4096; else {check_for_p2_core2=true; l3 = l2 = 4096;} break;
00861 case 0x4A: l3 = 6144; break;
00862 case 0x4B: l3 = 8192; break;
00863 case 0x4C: l3 = 12288; break;
00864 case 0x4D: l3 = 16384; break;
00865 case 0x4E: l2 = 6144; break;
00866 case 0x78: l2 = 1024; break;
00867 case 0x79: l2 = 128; break;
00868 case 0x7A: l2 = 256; break;
00869 case 0x7B: l2 = 512; break;
00870 case 0x7C: l2 = 1024; break;
00871 case 0x7D: l2 = 2048; break;
00872 case 0x7E: l2 = 256; break;
00873 case 0x7F: l2 = 512; break;
00874 case 0x80: l2 = 512; break;
00875 case 0x81: l2 = 128; break;
00876 case 0x82: l2 = 256; break;
00877 case 0x83: l2 = 512; break;
00878 case 0x84: l2 = 1024; break;
00879 case 0x85: l2 = 2048; break;
00880 case 0x86: l2 = 512; break;
00881 case 0x87: l2 = 1024; break;
00882 case 0x88: l3 = 2048; break;
00883 case 0x89: l3 = 4096; break;
00884 case 0x8A: l3 = 8192; break;
00885 case 0x8D: l3 = 3072; break;
00886
00887 default: break;
00888 }
00889 }
00890 if(check_for_p2_core2 && l2 == l3)
00891 l3 = 0;
00892 l1 *= 1024;
00893 l2 *= 1024;
00894 l3 *= 1024;
00895 }
00896
00897 inline void queryCacheSizes_intel(int& l1, int& l2, int& l3, int max_std_funcs)
00898 {
00899 if(max_std_funcs>=4)
00900 queryCacheSizes_intel_direct(l1,l2,l3);
00901 else
00902 queryCacheSizes_intel_codes(l1,l2,l3);
00903 }
00904
00905 inline void queryCacheSizes_amd(int& l1, int& l2, int& l3)
00906 {
00907 int abcd[4];
00908 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
00909 EIGEN_CPUID(abcd,0x80000005,0);
00910 l1 = (abcd[2] >> 24) * 1024;
00911 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
00912 EIGEN_CPUID(abcd,0x80000006,0);
00913 l2 = (abcd[2] >> 16) * 1024;
00914 l3 = ((abcd[3] & 0xFFFC000) >> 18) * 512 * 1024;
00915 }
00916 #endif
00917
00920 inline void queryCacheSizes(int& l1, int& l2, int& l3)
00921 {
00922 #ifdef EIGEN_CPUID
00923 int abcd[4];
00924 const int GenuineIntel[] = {0x756e6547, 0x49656e69, 0x6c65746e};
00925 const int AuthenticAMD[] = {0x68747541, 0x69746e65, 0x444d4163};
00926 const int AMDisbetter_[] = {0x69444d41, 0x74656273, 0x21726574};
00927
00928
00929 EIGEN_CPUID(abcd,0x0,0);
00930 int max_std_funcs = abcd[1];
00931 if(cpuid_is_vendor(abcd,GenuineIntel))
00932 queryCacheSizes_intel(l1,l2,l3,max_std_funcs);
00933 else if(cpuid_is_vendor(abcd,AuthenticAMD) || cpuid_is_vendor(abcd,AMDisbetter_))
00934 queryCacheSizes_amd(l1,l2,l3);
00935 else
00936
00937 queryCacheSizes_intel(l1,l2,l3,max_std_funcs);
00938
00939
00940
00941
00942
00943
00944
00945
00946
00947
00948
00949
00950 #else
00951 l1 = l2 = l3 = -1;
00952 #endif
00953 }
00954
00957 inline int queryL1CacheSize()
00958 {
00959 int l1(-1), l2, l3;
00960 queryCacheSizes(l1,l2,l3);
00961 return l1;
00962 }
00963
00966 inline int queryTopLevelCacheSize()
00967 {
00968 int l1, l2(-1), l3(-1);
00969 queryCacheSizes(l1,l2,l3);
00970 return (std::max)(l2,l3);
00971 }
00972
00973 }
00974
00975 }
00976
00977 #endif // EIGEN_MEMORY_H