20 #ifndef EIGEN_MEMORY_H
21 #define EIGEN_MEMORY_H
23 #ifndef EIGEN_MALLOC_ALREADY_ALIGNED
34 #if defined(__GLIBC__) && ((__GLIBC__>=2 && __GLIBC_MINOR__ >= 8) || __GLIBC__>2) \
35 && defined(__LP64__) && ! defined( __SANITIZE_ADDRESS__ ) && (EIGEN_DEFAULT_ALIGN_BYTES == 16)
36 #define EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED 1
38 #define EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED 0
45 #if defined(__FreeBSD__) && !(EIGEN_ARCH_ARM || EIGEN_ARCH_MIPS) && (EIGEN_DEFAULT_ALIGN_BYTES == 16)
46 #define EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED 1
48 #define EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED 0
51 #if (EIGEN_OS_MAC && (EIGEN_DEFAULT_ALIGN_BYTES == 16)) \
52 || (EIGEN_OS_WIN64 && (EIGEN_DEFAULT_ALIGN_BYTES == 16)) \
53 || EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED \
54 || EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED
55 #define EIGEN_MALLOC_ALREADY_ALIGNED 1
57 #define EIGEN_MALLOC_ALREADY_ALIGNED 0
69 #ifdef EIGEN_EXCEPTIONS
70 throw std::bad_alloc();
73 #if defined(EIGEN_HIPCC)
85 void* unused = ::operator
new(huge);
102 eigen_assert(alignment >=
sizeof(
void*) && (alignment & (alignment-1)) == 0 &&
"Alignment must be at least sizeof(void*) and a power of 2");
105 void *original = malloc(
size+alignment);
107 if (original == 0)
return 0;
108 void *aligned =
reinterpret_cast<void*
>((
reinterpret_cast<std::size_t>(original) & ~(
std::size_t(alignment-1))) + alignment);
109 *(
reinterpret_cast<void**
>(aligned) - 1) = original;
118 free(*(
reinterpret_cast<void**
>(ptr) - 1));
130 void *original = *(
reinterpret_cast<void**
>(ptr) - 1);
131 std::ptrdiff_t previous_offset =
static_cast<char *
>(ptr)-
static_cast<char *
>(original);
133 if (original == 0)
return 0;
135 void *previous_aligned =
static_cast<char *
>(original)+previous_offset;
136 if(aligned!=previous_aligned)
137 std::memmove(aligned, previous_aligned,
size);
139 *(
reinterpret_cast<void**
>(aligned) - 1) = original;
147 #ifdef EIGEN_NO_MALLOC
150 eigen_assert(
false &&
"heap allocation is forbidden (EIGEN_NO_MALLOC is defined)");
152 #elif defined EIGEN_RUNTIME_NO_MALLOC
155 static bool value =
true;
160 EIGEN_DEVICE_FUNC inline bool is_malloc_allowed() {
return is_malloc_allowed_impl(
false); }
161 EIGEN_DEVICE_FUNC inline bool set_is_malloc_allowed(
bool new_value) {
return is_malloc_allowed_impl(
true, new_value); }
164 eigen_assert(is_malloc_allowed() &&
"heap allocation is forbidden (EIGEN_RUNTIME_NO_MALLOC is defined and g_is_malloc_allowed is false)");
179 #if (EIGEN_DEFAULT_ALIGN_BYTES==0) || EIGEN_MALLOC_ALREADY_ALIGNED
184 #if EIGEN_DEFAULT_ALIGN_BYTES==16
185 eigen_assert((
size<16 || (
std::size_t(
result)%16)==0) &&
"System's malloc returned an unaligned pointer. Compile with EIGEN_MALLOC_ALREADY_ALIGNED=0 to fallback to handmade aligned memory allocator.");
200 #if (EIGEN_DEFAULT_ALIGN_BYTES==0) || EIGEN_MALLOC_ALREADY_ALIGNED
220 #if (EIGEN_DEFAULT_ALIGN_BYTES==0) || EIGEN_MALLOC_ALREADY_ALIGNED
221 result = std::realloc(ptr,new_size);
275 return std::realloc(ptr, new_size);
300 for (
i = 0;
i <
size; ++
i) ::
new (ptr +
i)
T;
328 check_size_for_overflow<T>(
size);
344 check_size_for_overflow<T>(
size);
345 T *
result =
reinterpret_cast<T*
>(conditional_aligned_malloc<Align>(
sizeof(
T)*
size));
352 conditional_aligned_free<Align>(
result);
363 destruct_elements_of_array<T>(ptr,
size);
372 destruct_elements_of_array<T>(ptr,
size);
373 conditional_aligned_free<Align>(ptr);
378 check_size_for_overflow<T>(new_size);
379 check_size_for_overflow<T>(old_size);
380 if(new_size < old_size)
382 T *
result =
reinterpret_cast<T*
>(conditional_aligned_realloc<Align>(
reinterpret_cast<void*
>(pts),
sizeof(
T)*new_size,
sizeof(
T)*old_size));
383 if(new_size > old_size)
391 conditional_aligned_free<Align>(
result);
403 check_size_for_overflow<T>(
size);
404 T *
result =
reinterpret_cast<T*
>(conditional_aligned_malloc<Align>(
sizeof(
T)*
size));
413 conditional_aligned_free<Align>(
result);
422 check_size_for_overflow<T>(new_size);
423 check_size_for_overflow<T>(old_size);
426 T *
result =
reinterpret_cast<T*
>(conditional_aligned_realloc<Align>(
reinterpret_cast<void*
>(pts),
sizeof(
T)*new_size,
sizeof(
T)*old_size));
435 conditional_aligned_free<Align>(
result);
445 destruct_elements_of_array<T>(ptr,
size);
446 conditional_aligned_free<Align>(ptr);
468 template<
int Alignment,
typename Scalar,
typename Index>
472 const Index AlignmentSize = Alignment / ScalarSize;
473 const Index AlignmentMask = AlignmentSize-1;
496 template<
typename Scalar,
typename Index>
500 return first_aligned<unpacket_traits<DefaultPacketType>::alignment>(
array,
size);
505 template<
typename Index>
527 memcpy(target, start,
size);
533 { std::copy(start,
end, target); }
545 static inline void run(
const T* start,
const T*
end,
T* target)
550 std::memmove(target, start,
size);
555 static inline void run(
const T* start,
const T*
end,
T* target)
559 std::copy(start,
end, target);
563 std::ptrdiff_t count = (std::ptrdiff_t(
end)-std::ptrdiff_t(start)) /
sizeof(
T);
564 std::copy_backward(start,
end, target + count);
569 #if EIGEN_HAS_RVALUE_REFERENCES
572 return std::move(start,
end, target);
577 return std::copy(start,
end, target);
587 #if ! defined EIGEN_ALLOCA && ! defined EIGEN_GPU_COMPILE_PHASE
588 #if EIGEN_OS_LINUX || EIGEN_OS_MAC || (defined alloca)
589 #define EIGEN_ALLOCA alloca
590 #elif EIGEN_COMP_MSVC
591 #define EIGEN_ALLOCA _alloca
600 #if defined(__clang__) && defined(__thumb__)
626 Eigen::internal::destruct_elements_of_array<T>(
m_ptr,
m_size);
638 template<
typename Xpr,
int NbEvaluations,
641 struct local_nested_eval_wrapper
643 static const bool NeedExternalBuffer =
false;
649 local_nested_eval_wrapper(
const Xpr& xpr,
Scalar* ptr) :
object(xpr)
656 template<
typename Xpr,
int NbEvaluations>
657 struct local_nested_eval_wrapper<Xpr,NbEvaluations,true>
659 static const bool NeedExternalBuffer =
true;
666 local_nested_eval_wrapper(
const Xpr& xpr,
Scalar* ptr)
676 ~local_nested_eval_wrapper()
688 #endif // EIGEN_ALLOCA
706 operator const T*()
const {
return m_ptr; }
743 #if EIGEN_DEFAULT_ALIGN_BYTES>0
746 #define EIGEN_ALIGNED_ALLOCA(SIZE) reinterpret_cast<void*>((internal::UIntPtr(EIGEN_ALLOCA(SIZE+EIGEN_DEFAULT_ALIGN_BYTES-1)) + EIGEN_DEFAULT_ALIGN_BYTES-1) & ~(std::size_t(EIGEN_DEFAULT_ALIGN_BYTES-1)))
748 #define EIGEN_ALIGNED_ALLOCA(SIZE) EIGEN_ALLOCA(SIZE)
751 #define ei_declare_aligned_stack_constructed_variable(TYPE,NAME,SIZE,BUFFER) \
752 Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \
753 TYPE* NAME = (BUFFER)!=0 ? (BUFFER) \
754 : reinterpret_cast<TYPE*>( \
755 (sizeof(TYPE)*SIZE<=EIGEN_STACK_ALLOCATION_LIMIT) ? EIGEN_ALIGNED_ALLOCA(sizeof(TYPE)*SIZE) \
756 : Eigen::internal::aligned_malloc(sizeof(TYPE)*SIZE) ); \
757 Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME,_stack_memory_destructor)((BUFFER)==0 ? NAME : 0,SIZE,sizeof(TYPE)*SIZE>EIGEN_STACK_ALLOCATION_LIMIT)
760 #define ei_declare_local_nested_eval(XPR_T,XPR,N,NAME) \
761 Eigen::internal::local_nested_eval_wrapper<XPR_T,N> EIGEN_CAT(NAME,_wrapper)(XPR, reinterpret_cast<typename XPR_T::Scalar*>( \
762 ( (Eigen::internal::local_nested_eval_wrapper<XPR_T,N>::NeedExternalBuffer) && ((sizeof(typename XPR_T::Scalar)*XPR.size())<=EIGEN_STACK_ALLOCATION_LIMIT) ) \
763 ? EIGEN_ALIGNED_ALLOCA( sizeof(typename XPR_T::Scalar)*XPR.size() ) : 0 ) ) ; \
764 typename Eigen::internal::local_nested_eval_wrapper<XPR_T,N>::ObjectType NAME(EIGEN_CAT(NAME,_wrapper).object)
768 #define ei_declare_aligned_stack_constructed_variable(TYPE,NAME,SIZE,BUFFER) \
769 Eigen::internal::check_size_for_overflow<TYPE>(SIZE); \
770 TYPE* NAME = (BUFFER)!=0 ? BUFFER : reinterpret_cast<TYPE*>(Eigen::internal::aligned_malloc(sizeof(TYPE)*SIZE)); \
771 Eigen::internal::aligned_stack_memory_handler<TYPE> EIGEN_CAT(NAME,_stack_memory_destructor)((BUFFER)==0 ? NAME : 0,SIZE,true)
774 #define ei_declare_local_nested_eval(XPR_T,XPR,N,NAME) typename Eigen::internal::nested_eval<XPR_T,N>::type NAME(XPR)
783 #if EIGEN_HAS_CXX17_OVERALIGN
787 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign)
788 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign)
789 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW
790 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar,Size)
795 #if EIGEN_MAX_ALIGN_BYTES!=0 && !defined(EIGEN_HIP_DEVICE_COMPILE)
796 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \
798 void* operator new(std::size_t size, const std::nothrow_t&) EIGEN_NO_THROW { \
799 EIGEN_TRY { return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); } \
800 EIGEN_CATCH (...) { return 0; } \
802 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign) \
804 void *operator new(std::size_t size) { \
805 return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \
808 void *operator new[](std::size_t size) { \
809 return Eigen::internal::conditional_aligned_malloc<NeedsToAlign>(size); \
812 void operator delete(void * ptr) EIGEN_NO_THROW { Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); } \
814 void operator delete[](void * ptr) EIGEN_NO_THROW { Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); } \
816 void operator delete(void * ptr, std::size_t ) EIGEN_NO_THROW { Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); } \
818 void operator delete[](void * ptr, std::size_t ) EIGEN_NO_THROW { Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); } \
823 static void *operator new(std::size_t size, void *ptr) { return ::operator new(size,ptr); } \
825 static void *operator new[](std::size_t size, void* ptr) { return ::operator new[](size,ptr); } \
827 void operator delete(void * memory, void *ptr) EIGEN_NO_THROW { return ::operator delete(memory,ptr); } \
829 void operator delete[](void * memory, void *ptr) EIGEN_NO_THROW { return ::operator delete[](memory,ptr); } \
831 EIGEN_MAKE_ALIGNED_OPERATOR_NEW_NOTHROW(NeedsToAlign) \
833 void operator delete(void *ptr, const std::nothrow_t&) EIGEN_NO_THROW { \
834 Eigen::internal::conditional_aligned_free<NeedsToAlign>(ptr); \
836 typedef void eigen_aligned_operator_new_marker_type;
838 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign)
841 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(true)
842 #define EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(Scalar,Size) \
843 EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(bool( \
844 ((Size)!=Eigen::Dynamic) && \
845 (((EIGEN_MAX_ALIGN_BYTES>=16) && ((sizeof(Scalar)*(Size))%(EIGEN_MAX_ALIGN_BYTES )==0)) || \
846 ((EIGEN_MAX_ALIGN_BYTES>=32) && ((sizeof(Scalar)*(Size))%(EIGEN_MAX_ALIGN_BYTES/2)==0)) || \
847 ((EIGEN_MAX_ALIGN_BYTES>=64) && ((sizeof(Scalar)*(Size))%(EIGEN_MAX_ALIGN_BYTES/4)==0)) )))
904 #if EIGEN_COMP_GNUC_STRICT && EIGEN_GNUC_AT_LEAST(7,0)
915 internal::check_size_for_overflow<T>(num);
927 #if !defined(EIGEN_NO_CPUID)
928 # if EIGEN_COMP_GNUC && EIGEN_ARCH_i386_OR_x86_64
929 # if defined(__PIC__) && EIGEN_ARCH_i386
931 # define EIGEN_CPUID(abcd,func,id) \
932 __asm__ __volatile__ ("xchgl %%ebx, %k1;cpuid; xchgl %%ebx,%k1": "=a" (abcd[0]), "=&r" (abcd[1]), "=c" (abcd[2]), "=d" (abcd[3]) : "a" (func), "c" (id));
933 # elif defined(__PIC__) && EIGEN_ARCH_x86_64
936 # define EIGEN_CPUID(abcd,func,id) \
937 __asm__ __volatile__ ("xchg{q}\t{%%}rbx, %q1; cpuid; xchg{q}\t{%%}rbx, %q1": "=a" (abcd[0]), "=&r" (abcd[1]), "=c" (abcd[2]), "=d" (abcd[3]) : "0" (func), "2" (id));
940 # define EIGEN_CPUID(abcd,func,id) \
941 __asm__ __volatile__ ("cpuid": "=a" (abcd[0]), "=b" (abcd[1]), "=c" (abcd[2]), "=d" (abcd[3]) : "0" (func), "2" (id) );
943 # elif EIGEN_COMP_MSVC
944 # if (EIGEN_COMP_MSVC > 1500) && EIGEN_ARCH_i386_OR_x86_64
945 # define EIGEN_CPUID(abcd,func,id) __cpuidex((int*)abcd,func,id)
954 inline bool cpuid_is_vendor(
int abcd[4],
const int vendor[3])
956 return abcd[1]==vendor[0] && abcd[3]==vendor[1] && abcd[2]==vendor[2];
959 inline void queryCacheSizes_intel_direct(
int&
l1,
int&
l2,
int&
l3)
966 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
967 EIGEN_CPUID(abcd,0
x4,cache_id);
968 cache_type = (abcd[0] & 0x0F) >> 0;
969 if(cache_type==1||cache_type==3)
971 int cache_level = (abcd[0] & 0xE0) >> 5;
972 int ways = (abcd[1] & 0xFFC00000) >> 22;
973 int partitions = (abcd[1] & 0x003FF000) >> 12;
974 int line_size = (abcd[1] & 0x00000FFF) >> 0;
975 int sets = (abcd[2]);
977 int cache_size = (ways+1) * (partitions+1) * (line_size+1) * (sets+1);
981 case 1:
l1 = cache_size;
break;
982 case 2:
l2 = cache_size;
break;
983 case 3:
l3 = cache_size;
break;
988 }
while(cache_type>0 && cache_id<16);
991 inline void queryCacheSizes_intel_codes(
int&
l1,
int&
l2,
int&
l3)
994 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
996 EIGEN_CPUID(abcd,0x00000002,0);
997 unsigned char *
bytes =
reinterpret_cast<unsigned char *
>(abcd)+2;
998 bool check_for_p2_core2 =
false;
999 for(
int i=0;
i<14; ++
i)
1003 case 0x0A:
l1 = 8;
break;
1004 case 0x0C:
l1 = 16;
break;
1005 case 0x0E:
l1 = 24;
break;
1006 case 0x10:
l1 = 16;
break;
1007 case 0x15:
l1 = 16;
break;
1008 case 0x2C:
l1 = 32;
break;
1009 case 0x30:
l1 = 32;
break;
1010 case 0x60:
l1 = 16;
break;
1011 case 0x66:
l1 = 8;
break;
1012 case 0x67:
l1 = 16;
break;
1013 case 0x68:
l1 = 32;
break;
1014 case 0x1A:
l2 = 96;
break;
1015 case 0x22:
l3 = 512;
break;
1016 case 0x23:
l3 = 1024;
break;
1017 case 0x25:
l3 = 2048;
break;
1018 case 0x29:
l3 = 4096;
break;
1019 case 0x39:
l2 = 128;
break;
1020 case 0x3A:
l2 = 192;
break;
1021 case 0x3B:
l2 = 128;
break;
1022 case 0x3C:
l2 = 256;
break;
1023 case 0x3D:
l2 = 384;
break;
1024 case 0x3E:
l2 = 512;
break;
1025 case 0x40:
l2 = 0;
break;
1026 case 0x41:
l2 = 128;
break;
1027 case 0x42:
l2 = 256;
break;
1028 case 0x43:
l2 = 512;
break;
1029 case 0x44:
l2 = 1024;
break;
1030 case 0x45:
l2 = 2048;
break;
1031 case 0x46:
l3 = 4096;
break;
1032 case 0x47:
l3 = 8192;
break;
1033 case 0x48:
l2 = 3072;
break;
1034 case 0x49:
if(
l2!=0)
l3 = 4096;
else {check_for_p2_core2=
true;
l3 =
l2 = 4096;}
break;
1035 case 0x4A:
l3 = 6144;
break;
1036 case 0x4B:
l3 = 8192;
break;
1037 case 0x4C:
l3 = 12288;
break;
1038 case 0x4D:
l3 = 16384;
break;
1039 case 0x4E:
l2 = 6144;
break;
1040 case 0x78:
l2 = 1024;
break;
1041 case 0x79:
l2 = 128;
break;
1042 case 0x7A:
l2 = 256;
break;
1043 case 0x7B:
l2 = 512;
break;
1044 case 0x7C:
l2 = 1024;
break;
1045 case 0x7D:
l2 = 2048;
break;
1046 case 0x7E:
l2 = 256;
break;
1047 case 0x7F:
l2 = 512;
break;
1048 case 0x80:
l2 = 512;
break;
1049 case 0x81:
l2 = 128;
break;
1050 case 0x82:
l2 = 256;
break;
1051 case 0x83:
l2 = 512;
break;
1052 case 0x84:
l2 = 1024;
break;
1053 case 0x85:
l2 = 2048;
break;
1054 case 0x86:
l2 = 512;
break;
1055 case 0x87:
l2 = 1024;
break;
1056 case 0x88:
l3 = 2048;
break;
1057 case 0x89:
l3 = 4096;
break;
1058 case 0x8A:
l3 = 8192;
break;
1059 case 0x8D:
l3 = 3072;
break;
1064 if(check_for_p2_core2 &&
l2 ==
l3)
1071 inline void queryCacheSizes_intel(
int&
l1,
int&
l2,
int&
l3,
int max_std_funcs)
1073 if(max_std_funcs>=4)
1074 queryCacheSizes_intel_direct(
l1,
l2,
l3);
1075 else if(max_std_funcs>=2)
1076 queryCacheSizes_intel_codes(
l1,
l2,
l3);
1081 inline void queryCacheSizes_amd(
int&
l1,
int&
l2,
int&
l3)
1084 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
1087 EIGEN_CPUID(abcd,0x80000000,0);
1090 EIGEN_CPUID(abcd,0x80000005,0);
1091 l1 = (abcd[2] >> 24) * 1024;
1092 abcd[0] = abcd[1] = abcd[2] = abcd[3] = 0;
1093 EIGEN_CPUID(abcd,0x80000006,0);
1094 l2 = (abcd[2] >> 16) * 1024;
1095 l3 = ((abcd[3] & 0xFFFC000) >> 18) * 512 * 1024;
1110 const int GenuineIntel[] = {0x756e6547, 0x49656e69, 0x6c65746e};
1111 const int AuthenticAMD[] = {0x68747541, 0x69746e65, 0x444d4163};
1112 const int AMDisbetter_[] = {0x69444d41, 0x74656273, 0x21726574};
1115 EIGEN_CPUID(abcd,0
x0,0);
1116 int max_std_funcs = abcd[0];
1117 if(cpuid_is_vendor(abcd,GenuineIntel))
1118 queryCacheSizes_intel(
l1,
l2,
l3,max_std_funcs);
1119 else if(cpuid_is_vendor(abcd,AuthenticAMD) || cpuid_is_vendor(abcd,AMDisbetter_))
1120 queryCacheSizes_amd(
l1,
l2,
l3);
1123 queryCacheSizes_intel(
l1,
l2,
l3,max_std_funcs);
1163 #endif // EIGEN_MEMORY_H