26 template<
typename Func,
typename Derived>
31 PacketSize = packet_traits<typename Derived::Scalar>::size,
32 InnerMaxSize = int(Derived::IsRowMajor)
33 ? Derived::MaxColsAtCompileTime
34 : Derived::MaxRowsAtCompileTime
39 && (functor_traits<Func>::PacketAccess),
40 MayLinearVectorize = MightVectorize && (int(Derived::Flags)&
LinearAccessBit),
41 MaySliceVectorize = MightVectorize &&
int(InnerMaxSize)>=3*PacketSize
46 Traversal = int(MayLinearVectorize) ? int(LinearVectorizedTraversal)
47 : int(MaySliceVectorize) ? int(SliceVectorizedTraversal)
48 : int(DefaultTraversal)
53 Cost = Derived::SizeAtCompileTime == Dynamic ? HugeCost
54 : Derived::SizeAtCompileTime * Derived::CoeffReadCost + (Derived::SizeAtCompileTime-1) * functor_traits<Func>::Cost,
55 UnrollingLimit = EIGEN_UNROLLING_LIMIT * (
int(Traversal) == int(DefaultTraversal) ? 1 : int(PacketSize))
60 Unrolling = Cost <= UnrollingLimit ? CompleteUnrolling : NoUnrolling
63 #ifdef EIGEN_DEBUG_ASSIGN
66 std::cerr <<
"Xpr: " <<
typeid(
typename Derived::XprType).name() << std::endl;
67 std::cerr.setf(std::ios::hex, std::ios::basefield);
68 EIGEN_DEBUG_VAR(Derived::Flags)
69 std::cerr.unsetf(
std::ios::hex);
70 EIGEN_DEBUG_VAR(InnerMaxSize)
71 EIGEN_DEBUG_VAR(PacketSize)
72 EIGEN_DEBUG_VAR(MightVectorize)
73 EIGEN_DEBUG_VAR(MayLinearVectorize)
74 EIGEN_DEBUG_VAR(MaySliceVectorize)
75 EIGEN_DEBUG_VAR(Traversal)
76 EIGEN_DEBUG_VAR(UnrollingLimit)
77 EIGEN_DEBUG_VAR(Unrolling)
89 template<
typename Func,
typename Derived,
int Start,
int Length>
90 struct redux_novec_unroller
96 typedef typename Derived::Scalar Scalar;
99 static EIGEN_STRONG_INLINE Scalar run(
const Derived &mat,
const Func& func)
101 return func(redux_novec_unroller<Func, Derived, Start, HalfLength>::run(mat,func),
102 redux_novec_unroller<Func, Derived, Start+HalfLength, Length-HalfLength>::run(mat,func));
106 template<
typename Func,
typename Derived,
int Start>
107 struct redux_novec_unroller<Func, Derived, Start, 1>
110 outer = Start / Derived::InnerSizeAtCompileTime,
111 inner = Start % Derived::InnerSizeAtCompileTime
114 typedef typename Derived::Scalar Scalar;
117 static EIGEN_STRONG_INLINE Scalar run(
const Derived &mat,
const Func&)
119 return mat.coeffByOuterInner(outer, inner);
126 template<
typename Func,
typename Derived,
int Start>
127 struct redux_novec_unroller<Func, Derived, Start, 0>
129 typedef typename Derived::Scalar Scalar;
131 static EIGEN_STRONG_INLINE Scalar run(
const Derived&,
const Func&) {
return Scalar(); }
136 template<
typename Func,
typename Derived,
int Start,
int Length>
137 struct redux_vec_unroller
140 PacketSize = packet_traits<typename Derived::Scalar>::size,
141 HalfLength = Length/2
144 typedef typename Derived::Scalar Scalar;
145 typedef typename packet_traits<Scalar>::type PacketScalar;
147 static EIGEN_STRONG_INLINE PacketScalar run(
const Derived &mat,
const Func& func)
149 return func.packetOp(
150 redux_vec_unroller<Func, Derived, Start, HalfLength>::run(mat,func),
151 redux_vec_unroller<Func, Derived, Start+HalfLength, Length-HalfLength>::run(mat,func) );
155 template<
typename Func,
typename Derived,
int Start>
156 struct redux_vec_unroller<Func, Derived, Start, 1>
159 index = Start * packet_traits<typename Derived::Scalar>::size,
160 outer = index / int(Derived::InnerSizeAtCompileTime),
161 inner = index % int(Derived::InnerSizeAtCompileTime),
162 alignment = Derived::Alignment
165 typedef typename Derived::Scalar Scalar;
166 typedef typename packet_traits<Scalar>::type PacketScalar;
168 static EIGEN_STRONG_INLINE PacketScalar run(
const Derived &mat,
const Func&)
170 return mat.template packetByOuterInner<alignment,PacketScalar>(outer, inner);
178 template<
typename Func,
typename Derived,
179 int Traversal = redux_traits<Func, Derived>::Traversal,
180 int Unrolling = redux_traits<Func, Derived>::Unrolling
184 template<
typename Func,
typename Derived>
185 struct redux_impl<Func, Derived, DefaultTraversal, NoUnrolling>
187 typedef typename Derived::Scalar Scalar;
189 static EIGEN_STRONG_INLINE Scalar run(
const Derived &mat,
const Func& func)
191 eigen_assert(mat.rows()>0 && mat.cols()>0 &&
"you are using an empty matrix");
193 res = mat.coeffByOuterInner(0, 0);
194 for(Index i = 1; i < mat.innerSize(); ++i)
195 res = func(res, mat.coeffByOuterInner(0, i));
196 for(Index i = 1; i < mat.outerSize(); ++i)
197 for(Index j = 0; j < mat.innerSize(); ++j)
198 res = func(res, mat.coeffByOuterInner(i, j));
203 template<
typename Func,
typename Derived>
204 struct redux_impl<Func,Derived, DefaultTraversal, CompleteUnrolling>
205 :
public redux_novec_unroller<Func,Derived, 0, Derived::SizeAtCompileTime>
208 template<
typename Func,
typename Derived>
209 struct redux_impl<Func, Derived, LinearVectorizedTraversal, NoUnrolling>
211 typedef typename Derived::Scalar Scalar;
212 typedef typename packet_traits<Scalar>::type PacketScalar;
214 static Scalar run(
const Derived &mat,
const Func& func)
216 const Index size = mat.size();
218 const Index packetSize = packet_traits<Scalar>::size;
219 const int packetAlignment = unpacket_traits<PacketScalar>::alignment;
221 alignment0 = (bool(Derived::Flags &
DirectAccessBit) && bool(packet_traits<Scalar>::AlignedOnScalar)) ?
int(packetAlignment) : int(
Unaligned),
222 alignment = EIGEN_PLAIN_ENUM_MAX(alignment0, Derived::Alignment)
224 const Index alignedStart = internal::first_default_aligned(mat.nestedExpression());
225 const Index alignedSize2 = ((size-alignedStart)/(2*packetSize))*(2*packetSize);
226 const Index alignedSize = ((size-alignedStart)/(packetSize))*(packetSize);
227 const Index alignedEnd2 = alignedStart + alignedSize2;
228 const Index alignedEnd = alignedStart + alignedSize;
232 PacketScalar packet_res0 = mat.template packet<alignment,PacketScalar>(alignedStart);
233 if(alignedSize>packetSize)
235 PacketScalar packet_res1 = mat.template packet<alignment,PacketScalar>(alignedStart+packetSize);
236 for(Index index = alignedStart + 2*packetSize; index < alignedEnd2; index += 2*packetSize)
238 packet_res0 = func.packetOp(packet_res0, mat.template packet<alignment,PacketScalar>(index));
239 packet_res1 = func.packetOp(packet_res1, mat.template packet<alignment,PacketScalar>(index+packetSize));
242 packet_res0 = func.packetOp(packet_res0,packet_res1);
243 if(alignedEnd>alignedEnd2)
244 packet_res0 = func.packetOp(packet_res0, mat.template packet<alignment,PacketScalar>(alignedEnd2));
246 res = func.predux(packet_res0);
248 for(Index index = 0; index < alignedStart; ++index)
249 res = func(res,mat.coeff(index));
251 for(Index index = alignedEnd; index < size; ++index)
252 res = func(res,mat.coeff(index));
258 for(Index index = 1; index < size; ++index)
259 res = func(res,mat.coeff(index));
267 template<
typename Func,
typename Derived,
int Unrolling>
268 struct redux_impl<Func, Derived, SliceVectorizedTraversal, Unrolling>
270 typedef typename Derived::Scalar Scalar;
271 typedef typename packet_traits<Scalar>::type PacketType;
273 EIGEN_DEVICE_FUNC
static Scalar run(
const Derived &mat,
const Func& func)
275 eigen_assert(mat.rows()>0 && mat.cols()>0 &&
"you are using an empty matrix");
276 const Index innerSize = mat.innerSize();
277 const Index outerSize = mat.outerSize();
279 packetSize = packet_traits<Scalar>::size
281 const Index packetedInnerSize = ((innerSize)/packetSize)*packetSize;
283 if(packetedInnerSize)
285 PacketType packet_res = mat.template packet<Unaligned,PacketType>(0,0);
286 for(Index j=0; j<outerSize; ++j)
287 for(Index i=(j==0?packetSize:0); i<packetedInnerSize; i+=Index(packetSize))
288 packet_res = func.packetOp(packet_res, mat.template packetByOuterInner<Unaligned,PacketType>(j,i));
290 res = func.predux(packet_res);
291 for(Index j=0; j<outerSize; ++j)
292 for(Index i=packetedInnerSize; i<innerSize; ++i)
293 res = func(res, mat.coeffByOuterInner(j,i));
298 res = redux_impl<Func, Derived, DefaultTraversal, NoUnrolling>::run(mat, func);
305 template<
typename Func,
typename Derived>
306 struct redux_impl<Func, Derived, LinearVectorizedTraversal, CompleteUnrolling>
308 typedef typename Derived::Scalar Scalar;
309 typedef typename packet_traits<Scalar>::type PacketScalar;
311 PacketSize = packet_traits<Scalar>::size,
312 Size = Derived::SizeAtCompileTime,
313 VectorizedSize = (Size / PacketSize) * PacketSize
315 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Scalar run(
const Derived &mat,
const Func& func)
317 eigen_assert(mat.rows()>0 && mat.cols()>0 &&
"you are using an empty matrix");
318 if (VectorizedSize > 0) {
319 Scalar res = func.predux(redux_vec_unroller<Func, Derived, 0, Size / PacketSize>::run(mat,func));
320 if (VectorizedSize != Size)
321 res = func(res,redux_novec_unroller<Func, Derived, VectorizedSize, Size-VectorizedSize>::run(mat,func));
325 return redux_novec_unroller<Func, Derived, 0, Size>::run(mat,func);
331 template<
typename _XprType>
332 class redux_evaluator
335 typedef _XprType XprType;
336 EIGEN_DEVICE_FUNC
explicit redux_evaluator(
const XprType &xpr) : m_evaluator(xpr), m_xpr(xpr) {}
338 typedef typename XprType::Scalar Scalar;
339 typedef typename XprType::CoeffReturnType CoeffReturnType;
340 typedef typename XprType::PacketScalar PacketScalar;
341 typedef typename XprType::PacketReturnType PacketReturnType;
344 MaxRowsAtCompileTime = XprType::MaxRowsAtCompileTime,
345 MaxColsAtCompileTime = XprType::MaxColsAtCompileTime,
348 IsRowMajor = XprType::IsRowMajor,
349 SizeAtCompileTime = XprType::SizeAtCompileTime,
350 InnerSizeAtCompileTime = XprType::InnerSizeAtCompileTime,
351 CoeffReadCost = evaluator<XprType>::CoeffReadCost,
352 Alignment = evaluator<XprType>::Alignment
355 EIGEN_DEVICE_FUNC Index rows()
const {
return m_xpr.rows(); }
356 EIGEN_DEVICE_FUNC Index cols()
const {
return m_xpr.cols(); }
357 EIGEN_DEVICE_FUNC Index size()
const {
return m_xpr.size(); }
358 EIGEN_DEVICE_FUNC Index innerSize()
const {
return m_xpr.innerSize(); }
359 EIGEN_DEVICE_FUNC Index outerSize()
const {
return m_xpr.outerSize(); }
362 CoeffReturnType coeff(Index row, Index col)
const
363 {
return m_evaluator.coeff(row, col); }
366 CoeffReturnType coeff(Index index)
const
367 {
return m_evaluator.coeff(index); }
369 template<
int LoadMode,
typename PacketType>
370 PacketReturnType packet(Index row, Index col)
const
371 {
return m_evaluator.template packet<LoadMode,PacketType>(row, col); }
373 template<
int LoadMode,
typename PacketType>
374 PacketReturnType packet(Index index)
const
375 {
return m_evaluator.template packet<LoadMode,PacketType>(index); }
378 CoeffReturnType coeffByOuterInner(Index outer, Index inner)
const
379 {
return m_evaluator.coeff(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer); }
381 template<
int LoadMode,
typename PacketType>
382 PacketReturnType packetByOuterInner(Index outer, Index inner)
const
383 {
return m_evaluator.template packet<LoadMode,PacketType>(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer); }
385 const XprType & nestedExpression()
const {
return m_xpr; }
388 internal::evaluator<XprType> m_evaluator;
389 const XprType &m_xpr;
406 template<
typename Derived>
407 template<
typename Func>
408 typename internal::traits<Derived>::Scalar
411 eigen_assert(this->rows()>0 && this->cols()>0 &&
"you are using an empty matrix");
413 typedef typename internal::redux_evaluator<Derived> ThisEvaluator;
414 ThisEvaluator thisEval(derived());
416 return internal::redux_impl<Func, ThisEvaluator>::run(thisEval, func);
422 template<
typename Derived>
423 EIGEN_STRONG_INLINE
typename internal::traits<Derived>::Scalar
426 return derived().redux(Eigen::internal::scalar_min_op<Scalar>());
432 template<
typename Derived>
433 EIGEN_STRONG_INLINE
typename internal::traits<Derived>::Scalar
436 return derived().redux(Eigen::internal::scalar_max_op<Scalar>());
443 template<
typename Derived>
444 EIGEN_STRONG_INLINE
typename internal::traits<Derived>::Scalar
447 if(SizeAtCompileTime==0 || (SizeAtCompileTime==Dynamic && size()==0))
449 return derived().redux(Eigen::internal::scalar_sum_op<Scalar>());
456 template<
typename Derived>
457 EIGEN_STRONG_INLINE
typename internal::traits<Derived>::Scalar
460 return Scalar(derived().redux(Eigen::internal::scalar_sum_op<Scalar>())) / Scalar(this->size());
470 template<
typename Derived>
471 EIGEN_STRONG_INLINE
typename internal::traits<Derived>::Scalar
474 if(SizeAtCompileTime==0 || (SizeAtCompileTime==Dynamic && size()==0))
476 return derived().redux(Eigen::internal::scalar_product_op<Scalar>());
485 template<
typename Derived>
486 EIGEN_STRONG_INLINE
typename internal::traits<Derived>::Scalar
489 return derived().diagonal().sum();
494 #endif // EIGEN_REDUX_H
Scalar prod() const
Definition: Redux.h:472
const unsigned int DirectAccessBit
Definition: Constants.h:149
internal::traits< Derived >::Scalar maxCoeff() const
Definition: Redux.h:434
Definition: StdDeque.h:58
Base class for all dense matrices, vectors, and arrays.
Definition: DenseBase.h:41
Scalar mean() const
Definition: Redux.h:458
Definition: Constants.h:228
Scalar sum() const
Definition: Redux.h:445
internal::traits< Derived >::Scalar minCoeff() const
Definition: Redux.h:424
Definition: Eigen_Colamd.h:54
Scalar trace() const
Definition: Redux.h:487
const unsigned int ActualPacketAccessBit
Definition: Constants.h:99
const unsigned int LinearAccessBit
Definition: Constants.h:124