12 #ifndef EIGEN_ASSIGN_EVALUATOR_H 13 #define EIGEN_ASSIGN_EVALUATOR_H 27 template <
typename DstEvaluator,
typename SrcEvaluator,
typename AssignFunc>
30 typedef typename DstEvaluator::XprType
Dst;
50 :
int(Dst::RowsAtCompileTime),
53 :
int(Dst::MaxRowsAtCompileTime),
119 MayUnrollCompletely =
int(Dst::SizeAtCompileTime) !=
Dynamic 120 &&
int(Dst::SizeAtCompileTime) * (
int(DstEvaluator::CoeffReadCost)+
int(SrcEvaluator::CoeffReadCost)) <=
int(
UnrollingLimit),
148 #ifdef EIGEN_DEBUG_ASSIGN 151 std::cerr <<
"DstXpr: " <<
typeid(
typename DstEvaluator::XprType).
name() << std::endl;
152 std::cerr <<
"SrcXpr: " <<
typeid(
typename SrcEvaluator::XprType).
name() << std::endl;
153 std::cerr.setf(std::ios::hex, std::ios::basefield);
154 std::cerr <<
"DstFlags" <<
" = " <<
DstFlags <<
" (" << demangle_flags(
DstFlags) <<
" )" << std::endl;
155 std::cerr <<
"SrcFlags" <<
" = " <<
SrcFlags <<
" (" << demangle_flags(
SrcFlags) <<
" )" << std::endl;
156 std::cerr.unsetf(std::ios::hex);
173 std::cerr <<
"Traversal" <<
" = " <<
Traversal <<
" (" << demangle_traversal(
Traversal) <<
")" << std::endl;
178 std::cerr <<
"Unrolling" <<
" = " <<
Unrolling <<
" (" << demangle_unrolling(
Unrolling) <<
")" << std::endl;
179 std::cerr << std::endl;
192 template<
typename Kernel,
int Index,
int Stop>
200 outer =
Index / DstXprType::InnerSizeAtCompileTime,
201 inner =
Index % DstXprType::InnerSizeAtCompileTime
206 kernel.assignCoeffByOuterInner(outer, inner);
211 template<
typename Kernel,
int Stop>
217 template<
typename Kernel,
int Index_,
int Stop>
222 kernel.assignCoeffByOuterInner(outer, Index_);
227 template<
typename Kernel,
int Stop>
237 template<
typename Kernel,
int Index,
int Stop>
242 kernel.assignCoeff(
Index);
247 template<
typename Kernel,
int Stop>
257 template<
typename Kernel,
int Index,
int Stop>
266 outer =
Index / DstXprType::InnerSizeAtCompileTime,
267 inner =
Index % DstXprType::InnerSizeAtCompileTime,
274 kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, inner);
280 template<
typename Kernel,
int Stop>
286 template<
typename Kernel,
int Index_,
int Stop,
int SrcAlignment,
int DstAlignment>
292 kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, Index_);
298 template<
typename Kernel,
int Stop,
int SrcAlignment,
int DstAlignment>
310 template<
typename Kernel,
311 int Traversal = Kernel::AssignmentTraits::Traversal,
312 int Unrolling = Kernel::AssignmentTraits::Unrolling>
319 template<
typename Kernel>
324 for(
Index outer = 0; outer < kernel.outerSize(); ++outer) {
325 for(
Index inner = 0; inner < kernel.innerSize(); ++inner) {
326 kernel.assignCoeffByOuterInner(outer, inner);
332 template<
typename Kernel>
337 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
342 template<
typename Kernel>
347 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
349 const Index outerSize = kernel.outerSize();
350 for(
Index outer = 0; outer < outerSize; ++outer)
363 template <
bool IsAligned = false>
367 template <
typename Kernel>
378 template <
typename Kernel>
383 template <
typename Kernel>
389 for (
Index index = start; index <
end; ++index)
390 kernel.assignCoeff(index);
394 template<
typename Kernel>
401 typedef typename Kernel::PacketType
PacketType;
403 requestedAlignment = Kernel::AssignmentTraits::LinearRequiredAlignment,
405 dstIsAligned =
int(Kernel::AssignmentTraits::DstAlignment)>=
int(requestedAlignment),
407 :
int(Kernel::AssignmentTraits::DstAlignment),
408 srcAlignment = Kernel::AssignmentTraits::JointAlignment
410 const Index alignedStart = dstIsAligned ? 0 : internal::first_aligned<requestedAlignment>(kernel.dstDataPtr(),
size);
411 const Index alignedEnd = alignedStart + ((size-alignedStart)/packetSize)*packetSize;
415 for(
Index index = alignedStart; index < alignedEnd; index += packetSize)
416 kernel.template assignPacket<dstAlignment, srcAlignment, PacketType>(index);
422 template<
typename Kernel>
427 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
428 typedef typename Kernel::PacketType
PacketType;
430 enum {
size = DstXprType::SizeAtCompileTime,
432 alignedSize = (
size/packetSize)*packetSize };
443 template<
typename Kernel>
453 const Index innerSize = kernel.innerSize();
454 const Index outerSize = kernel.outerSize();
456 for(
Index outer = 0; outer < outerSize; ++outer)
457 for(
Index inner = 0; inner < innerSize; inner+=packetSize)
458 kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, inner);
462 template<
typename Kernel>
467 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
472 template<
typename Kernel>
477 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
478 typedef typename Kernel::AssignmentTraits Traits;
479 const Index outerSize = kernel.outerSize();
480 for(
Index outer = 0; outer < outerSize; ++outer)
482 Traits::SrcAlignment, Traits::DstAlignment>::
run(kernel, outer);
490 template<
typename Kernel>
497 kernel.assignCoeff(
i);
501 template<
typename Kernel>
506 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
515 template<
typename Kernel>
521 typedef typename Kernel::PacketType
PacketType;
524 requestedAlignment =
int(Kernel::AssignmentTraits::InnerRequiredAlignment),
526 dstIsAligned =
int(Kernel::AssignmentTraits::DstAlignment)>=
int(requestedAlignment),
527 dstAlignment = alignable ?
int(requestedAlignment)
528 :
int(Kernel::AssignmentTraits::DstAlignment)
530 const Scalar *dst_ptr = kernel.dstDataPtr();
531 if((!
bool(dstIsAligned)) && (
UIntPtr(dst_ptr) %
sizeof(Scalar))>0)
536 const Index packetAlignedMask = packetSize - 1;
537 const Index innerSize = kernel.innerSize();
538 const Index outerSize = kernel.outerSize();
539 const Index alignedStep = alignable ? (packetSize - kernel.outerStride() % packetSize) & packetAlignedMask : 0;
540 Index alignedStart = ((!alignable) ||
bool(dstIsAligned)) ? 0 : internal::first_aligned<requestedAlignment>(dst_ptr, innerSize);
542 for(
Index outer = 0; outer < outerSize; ++outer)
544 const Index alignedEnd = alignedStart + ((innerSize-alignedStart) & ~packetAlignedMask);
546 for(
Index inner = 0; inner<alignedStart ; ++inner)
547 kernel.assignCoeffByOuterInner(outer, inner);
550 for(
Index inner = alignedStart; inner<alignedEnd; inner+=packetSize)
551 kernel.template assignPacketByOuterInner<dstAlignment, Unaligned, PacketType>(outer, inner);
554 for(
Index inner = alignedEnd; inner<innerSize ; ++inner)
555 kernel.assignCoeffByOuterInner(outer, inner);
557 alignedStart =
numext::mini((alignedStart+alignedStep)%packetSize, innerSize);
562 #if EIGEN_UNALIGNED_VECTORIZE 563 template<
typename Kernel>
568 typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
569 typedef typename Kernel::PacketType
PacketType;
571 enum {
size = DstXprType::InnerSizeAtCompileTime,
573 vectorizableSize = (
size/packetSize)*packetSize };
575 for(
Index outer = 0; outer < kernel.outerSize(); ++outer)
595 template<
typename DstEvaluatorTypeT,
typename SrcEvaluatorTypeT,
typename Functor,
int Version = Specialized>
611 : m_dst(dst), m_src(src), m_functor(func), m_dstExpr(dstExpr)
613 #ifdef EIGEN_DEBUG_ASSIGN 618 EIGEN_DEVICE_FUNC
Index size()
const {
return m_dstExpr.size(); }
621 EIGEN_DEVICE_FUNC
Index rows()
const {
return m_dstExpr.rows(); }
622 EIGEN_DEVICE_FUNC
Index cols()
const {
return m_dstExpr.cols(); }
626 EIGEN_DEVICE_FUNC
const SrcEvaluatorType&
srcEvaluator()
const {
return m_src; }
631 m_functor.assignCoeff(m_dst.coeffRef(row,col), m_src.coeff(row,col));
637 m_functor.assignCoeff(m_dst.coeffRef(index), m_src.coeff(index));
643 Index row = rowIndexByOuterInner(outer, inner);
644 Index col = colIndexByOuterInner(outer, inner);
645 assignCoeff(row, col);
649 template<
int StoreMode,
int LoadMode,
typename PacketType>
652 m_functor.template assignPacket<StoreMode>(&m_dst.coeffRef(row,col), m_src.template packet<LoadMode,PacketType>(
row,
col));
655 template<
int StoreMode,
int LoadMode,
typename PacketType>
658 m_functor.template assignPacket<StoreMode>(&m_dst.coeffRef(index), m_src.template packet<LoadMode,PacketType>(index));
661 template<
int StoreMode,
int LoadMode,
typename PacketType>
664 Index row = rowIndexByOuterInner(outer, inner);
665 Index col = colIndexByOuterInner(outer, inner);
666 assignPacket<StoreMode,LoadMode,PacketType>(
row,
col);
671 typedef typename DstEvaluatorType::ExpressionTraits Traits;
672 return int(Traits::RowsAtCompileTime) == 1 ? 0
673 :
int(Traits::ColsAtCompileTime) == 1 ? inner
674 :
int(DstEvaluatorType::Flags)&RowMajorBit ? outer
680 typedef typename DstEvaluatorType::ExpressionTraits Traits;
681 return int(Traits::ColsAtCompileTime) == 1 ? 0
682 :
int(Traits::RowsAtCompileTime) == 1 ? inner
683 :
int(DstEvaluatorType::Flags)&RowMajorBit ? inner
689 return m_dstExpr.data();
704 template<
typename DstXprType,
typename SrcXprType,
typename Functor>
710 eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
713 template<
typename DstXprType,
typename SrcXprType,
typename T1,
typename T2>
717 Index dstRows = src.rows();
718 Index dstCols = src.cols();
719 if(((dst.rows()!=dstRows) || (dst.cols()!=dstCols)))
720 dst.resize(dstRows, dstCols);
721 eigen_assert(dst.rows() == dstRows && dst.cols() == dstCols);
724 template<
typename DstXprType,
typename SrcXprType,
typename Functor>
730 SrcEvaluatorType srcEvaluator(src);
736 DstEvaluatorType dstEvaluator(dst);
739 Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived());
744 template<
typename DstXprType,
typename SrcXprType>
767 template<
typename DstXprType,
typename SrcXprType,
typename Functor,
778 template<
typename Dst,
typename Src>
784 template<
typename Dst,
typename Src>
792 template<
typename Dst,
typename Src,
typename Func>
800 template<
typename Dst,
typename Src,
typename Func>
809 template<
typename Dst,
template <
typename>
class StorageBase,
typename Src,
typename Func>
817 template<
typename Dst,
typename Src,
typename Func>
822 NeedToTranspose = ( (
int(Dst::RowsAtCompileTime) == 1 &&
int(Src::ColsAtCompileTime) == 1)
823 || (
int(Dst::ColsAtCompileTime) == 1 &&
int(Src::RowsAtCompileTime) == 1)
824 ) &&
int(Dst::SizeAtCompileTime) != 1
828 typedef typename internal::conditional<NeedToTranspose, Transpose<Dst>, Dst&>
::type ActualDstType;
829 ActualDstType actualDst(dst);
838 template<
typename Dst,
typename Src>
845 template<
typename Dst,
typename Src,
typename Func>
856 template<
typename Dst,
typename Src>
864 template<
typename Dst,
typename Src>
void check_for_aliasing(
const Dst &dst,
const Src &src);
869 template<
typename DstXprType,
typename SrcXprType,
typename Functor,
typename Weak>
875 #ifndef EIGEN_NO_DEBUG 887 template<
typename DstXprType,
typename SrcXprType,
typename Functor,
typename Weak>
893 Index dstRows = src.rows();
894 Index dstCols = src.cols();
895 if((dst.rows()!=dstRows) || (dst.cols()!=dstCols))
896 dst.resize(dstRows, dstCols);
898 eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
904 template<
typename SrcScalarType>
908 Index dstRows = src.rows();
909 Index dstCols = src.cols();
910 if((dst.rows()!=dstRows) || (dst.cols()!=dstCols))
911 dst.resize(dstRows, dstCols);
913 eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
917 template<
typename SrcScalarType>
921 Index dstRows = src.rows();
922 Index dstCols = src.cols();
923 if((dst.rows()!=dstRows) || (dst.cols()!=dstCols))
924 dst.resize(dstRows, dstCols);
926 eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
935 #endif // EIGEN_ASSIGN_EVALUATOR_H Kernel::DstEvaluatorType DstEvaluatorType
find_best_packet_helper< Size, typename packet_traits< T >::type >::type type
static EIGEN_DEVICE_FUNC void EIGEN_STRONG_INLINE run(Kernel &kernel)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_assignment_no_alias(Dst &dst, const Src &src, const Func &func)
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &kernel, Index outer)
const unsigned int ActualPacketAccessBit
#define EIGEN_STRONG_INLINE
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &kernel)
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &, Index)
EIGEN_DEVICE_FUNC Index outerStride() const
EIGEN_DEVICE_FUNC const SrcEvaluatorType & srcEvaluator() const
SrcEvaluatorTypeT::XprType SrcXprType
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void resize_if_allowed(DstXprType &dst, const SrcXprType &src, const Functor &)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_dense_assignment_loop(DstXprType &dst, const SrcXprType &src, const Functor &func)
const SrcEvaluatorType & m_src
#define EIGEN_DEBUG_VAR(x)
EIGEN_DEVICE_FUNC Index outerSize() const
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &kernel)
Pseudo expression providing an operator = assuming no aliasing.
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &kernel)
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &kernel)
const unsigned int DirectAccessBit
void check_for_aliasing(const Dst &dst, const Src &src)
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &kernel)
DstEvaluatorType::Scalar Scalar
Namespace containing all symbols from the Eigen library.
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacketByOuterInner(Index outer, Index inner)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacket(Index row, Index col)
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &, Index, Index)
EIGEN_DEVICE_FUNC Index size() const
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &kernel)
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(DstXprType &dst, const SrcXprType &src, const Functor &func)
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &kernel)
const unsigned int RowMajorBit
Kernel::PacketType PacketType
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &kernel, Index outer)
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &)
#define EIGEN_STATIC_ASSERT_LVALUE(Derived)
#define EIGEN_DONT_INLINE
EIGEN_DEVICE_FUNC Index rows() const
SrcEvaluatorTypeT SrcEvaluatorType
const Functor & m_functor
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &kernel)
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &)
EIGEN_DEVICE_FUNC DstEvaluatorType & dstEvaluator()
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeffByOuterInner(Index outer, Index inner)
DstEvaluatorType::XprType DstXprType
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
DstEvaluatorTypeT DstEvaluatorType
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &, Index)
find_best_packet< DstScalar, InnerSize >::type InnerPacketType
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op< typename DstXprType::Scalar, typename SrcXprType::Scalar > &)
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &kernel)
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(DstXprType &dst, const SrcXprType &src, const internal::sub_assign_op< typename DstXprType::Scalar, SrcScalarType > &)
DstEvaluator::XprType Dst
copy_using_evaluator_traits< DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor > AssignmentTraits
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index rowIndexByOuterInner(Index outer, Index inner)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(Index row, Index col)
Assign src(row,col) to dst(row,col) through the assignment functor.
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T mini(const T &x, const T &y)
EIGEN_DEVICE_FUNC Index cols() const
DstEvaluatorTypeT::XprType DstXprType
EIGEN_DEVICE_FUNC Index innerSize() const
EIGEN_DEVICE_FUNC generic_dense_assignment_kernel(DstEvaluatorType &dst, const SrcEvaluatorType &src, const Functor &func, DstXprType &dstExpr)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_assignment_no_alias_no_transpose(Dst &dst, const Src &src, const Func &func)
#define EIGEN_PLAIN_ENUM_MIN(a, b)
storage_kind_to_shape< typename traits< T >::StorageKind >::Shape Shape
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &kernel, Index start, Index end)
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &)
Kernel::PacketType PacketType
#define EIGEN_CHECK_BINARY_COMPATIBILIY(BINOP, LHS, RHS)
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index colIndexByOuterInner(Index outer, Index inner)
EIGEN_DEVICE_FUNC ExpressionType & expression() const
Kernel::DstEvaluatorType DstEvaluatorType
AssignmentTraits::PacketType PacketType
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(Index index)
conditional< int(Traversal)==LinearVectorizedTraversal, LinearPacketType, InnerPacketType >::type PacketType
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &kernel)
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(DstXprType &dst, const SrcXprType &src, const internal::add_assign_op< typename DstXprType::Scalar, SrcScalarType > &)
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &kernel)
Convenience specialization of Stride to specify only an outer stride See class Map for some examples...
void run(Expr &expr, Dev &dev)
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &kernel)
EIGEN_DEVICE_FUNC const Scalar * dstDataPtr() const
const unsigned int LinearAccessBit
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacket(Index index)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_assignment(Dst &dst, const Src &src)
#define EIGEN_UNROLLING_LIMIT
DstEvaluatorType::XprType DstXprType
#define EIGEN_UNALIGNED_VECTORIZE
find_best_packet< DstScalar, Dst::SizeAtCompileTime >::type LinearPacketType
Kernel::PacketType PacketType
#define EIGEN_ONLY_USED_FOR_DEBUG(x)
static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Kernel &kernel)
#define EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(TYPE0, TYPE1)