Mantid
Loading...
Searching...
No Matches
BoxController.h
Go to the documentation of this file.
1// Mantid Repository : https://github.com/mantidproject/mantid
2//
3// Copyright © 2018 ISIS Rutherford Appleton Laboratory UKRI,
4// NScD Oak Ridge National Laboratory, European Spallation Source,
5// Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
6// SPDX - License - Identifier: GPL - 3.0 +
7#pragma once
8
9#include "MantidAPI/DllConfig.h"
13#include "MantidKernel/System.h"
15#include <nexus/NeXusFile.hpp>
16
17#include <boost/optional.hpp>
18#include <numeric>
19#include <vector>
20
21namespace Mantid {
22namespace API {
23
33class MANTID_API_DLL BoxController {
34public:
35 //-----------------------------------------------------------------------------------
41 BoxController(size_t nd)
42 : nd(nd), m_maxId(0), m_SplitThreshold(1024), m_splitTopInto(boost::none), m_numSplit(1), m_numTopSplit(1),
43 m_fileIO(std::shared_ptr<API::IBoxControllerIO>()) {
44 // TODO: Smarter ways to determine all of these values
45 m_maxDepth = 5;
46 m_numEventsAtMax = 0;
47 m_addingEvents_eventsPerTask = 1000;
48 m_significantEventsNumber = 10000000;
49 m_addingEvents_numTasksPerBlock = Kernel::ThreadPool::getNumPhysicalCores() * 5;
50 m_splitInto.resize(this->nd, 1);
51 resetNumBoxes();
52 }
53
54 virtual ~BoxController();
55 // create new box controller from the existing one
56 virtual BoxController *clone() const;
58 std::string toXMLString() const;
59
61 void fromXMLString(const std::string &xml);
62
64 bool operator==(const BoxController &other) const;
65
66 //-----------------------------------------------------------------------------------
70 size_t getNDims() const { return nd; }
71
72 //-----------------------------------------------------------------------------------
75 size_t getNextId() { return m_maxId++; }
76
77 //-----------------------------------------------------------------------------------
80 size_t getMaxId() const { return m_maxId; }
81
82 //-----------------------------------------------------------------------------------
87 void setMaxId(size_t newMaxId) { m_maxId = newMaxId; }
88
89 //-----------------------------------------------------------------------------------
91 inline std::mutex &getIdMutex() { return m_idMutex; }
92
93 //-----------------------------------------------------------------------------------
100 bool willSplit(size_t numPoints, size_t depth) const {
101 return (numPoints > m_SplitThreshold) && (depth < m_maxDepth);
102 }
103
104 //-----------------------------------------------------------------------------------
106 size_t getSplitThreshold() const { return m_SplitThreshold; }
107
111 void setSplitThreshold(size_t threshold) { m_SplitThreshold = threshold; }
112
113 //-----------------------------------------------------------------------------------
119 size_t getSplitInto(size_t dim) const {
120 // if (dim >= nd)
121 // throw std::invalid_argument("BoxController::setSplitInto() called
122 // with too high of a dimension index.");
123 return m_splitInto[dim];
124 }
125
126 //-----------------------------------------------------------------------------------
132 const std::vector<size_t> &getSplitIntoAll() const { return m_splitInto; }
133 //-----------------------------------------------------------------------------------
138 boost::optional<std::vector<size_t>> getSplitTopInto() const {
139 // if (dim >= nd)
140 // throw std::invalid_argument("BoxController::setSplitInto() called
141 // with too high of a dimension index.");
142 return m_splitTopInto;
143 }
144
146 size_t getNumSplit() const { return m_numSplit; }
147
148 //-----------------------------------------------------------------------------------
152 void setSplitInto(size_t num) {
153 m_splitInto.clear();
154 m_splitInto.resize(nd, num);
155 calcNumSplit();
156 }
157
158 //-----------------------------------------------------------------------------------
164 void setSplitInto(size_t dim, size_t num) {
165 if (dim >= nd)
166 throw std::invalid_argument("BoxController::setSplitInto() called with "
167 "too high of a dimension index.");
168 m_splitInto[dim] = num;
169 calcNumSplit();
170 }
171
172 //-----------------------------------------------------------------------------------
178 void setSplitTopInto(size_t dim, size_t num) {
179 if (dim >= nd)
180 throw std::invalid_argument("BoxController::setSplitTopInto() called with "
181 "too high of a dimension index.");
182 // If the vector is not created, then create it
183 if (!m_splitTopInto) {
184 m_splitTopInto = std::vector<size_t>(nd, 1);
185 }
186 m_splitTopInto.get()[dim] = num;
187 calcNumTopSplit();
188 }
189
190 //-----------------------------------------------------------------------------------
195 void setAddingEvents_eventsPerTask(size_t m_addingEvents_eventsPerTask) {
196 this->m_addingEvents_eventsPerTask = m_addingEvents_eventsPerTask;
197 }
199 size_t getAddingEvents_eventsPerTask() const { return m_addingEvents_eventsPerTask; }
200
205 void setAddingEvents_numTasksPerBlock(size_t m_addingEvents_numTasksPerBlock) {
206 this->m_addingEvents_numTasksPerBlock = m_addingEvents_numTasksPerBlock;
207 }
208
210 size_t getAddingEvents_numTasksPerBlock() const { return m_addingEvents_numTasksPerBlock; }
211
212 //-----------------------------------------------------------------------------------
226 void getAddingEventsParameters(size_t &eventsPerTask, size_t &numTasksPerBlock) const {
227 // TODO: Smarter values here depending on nd, etc.
228 eventsPerTask = m_addingEvents_eventsPerTask;
229 numTasksPerBlock = m_addingEvents_numTasksPerBlock;
230 }
231
232 //-----------------------------------------------------------------------------------
234 size_t getMaxDepth() const { return m_maxDepth; }
235
240 void setMaxDepth(size_t value) {
241 m_maxDepth = value;
242 resetNumBoxes();
243 }
244
246 size_t getSignificantEventsNumber() const { return m_significantEventsNumber; }
247
248 //-----------------------------------------------------------------------------------
261 bool shouldSplitBoxes(size_t nEventsInOutput, size_t eventsAdded, size_t numMDBoxes) const {
262 // Avoid divide by zero
263 if (numMDBoxes == 0)
264 return false;
265 // Performance depends pretty strongly on WHEN you split the boxes.
266 // This is an empirically-determined way to optimize the splitting calls.
267 // Split when adding 1/16^th as many events as are already in the output,
268 // (because when the workspace gets very large you should split less often)
269 // But no more often than every 10 million events.
270 const size_t comparisonPoint = std::max(nEventsInOutput / 16, m_significantEventsNumber);
271 if (eventsAdded > comparisonPoint)
272 return true;
273
274 // Return true if the average # of events per box is big enough to split.
275 return ((eventsAdded / numMDBoxes) > m_SplitThreshold);
276 }
277 //-----------------------------------------------------------------------------------
278
279 void clearBoxesCounter(size_t depth) {
280 std::lock_guard<std::mutex> lock(m_mutexNumMDBoxes);
281 m_numMDBoxes[depth] = 0;
282 }
283
284 void clearGridBoxesCounter(size_t depth) {
285 std::lock_guard<std::mutex> lock(m_mutexNumMDBoxes);
286 m_numMDGridBoxes[depth] = 0;
287 }
288 void incGridBoxesCounter(size_t depth, size_t inc = 1) {
289 std::lock_guard<std::mutex> lock(m_mutexNumMDBoxes);
290 m_numMDGridBoxes[depth] += inc;
291 }
292
293 void incBoxesCounter(size_t depth, size_t inc = 1) {
294 std::lock_guard<std::mutex> lock(m_mutexNumMDBoxes);
295 m_numMDBoxes[depth] += inc;
296 }
297
308 void trackNumBoxes(size_t depth) {
309 std::lock_guard<std::mutex> lock(m_mutexNumMDBoxes);
310 if (m_numMDBoxes[depth] > 0) {
311 m_numMDBoxes[depth]--;
312 }
313 m_numMDGridBoxes[depth]++;
314
315 // We need to account for optional top level splitting
316 if (depth == 0 && m_splitTopInto) {
317
318 const auto &splitTopInto = m_splitTopInto.get();
319 size_t numSplitTop =
320 std::accumulate(splitTopInto.cbegin(), splitTopInto.cend(), size_t{1}, std::multiplies<size_t>());
321 m_numMDBoxes[depth + 1] += numSplitTop;
322 } else {
323 m_numMDBoxes[depth + 1] += m_numSplit;
324 }
325 }
326
328 const std::vector<size_t> &getNumMDBoxes() const { return m_numMDBoxes; }
329
332 const std::vector<size_t> &getNumMDGridBoxes() const { return m_numMDGridBoxes; }
333
336 const std::vector<double> &getMaxNumMDBoxes() const { return m_maxNumMDBoxes; }
337
339 size_t getTotalNumMDBoxes() const {
340 return std::accumulate(m_numMDBoxes.cbegin(), m_numMDBoxes.cend(), size_t{0}, std::plus<size_t>());
341 }
342
344 size_t getTotalNumMDGridBoxes() const {
345 return std::accumulate(m_numMDGridBoxes.cbegin(), m_numMDGridBoxes.cend(), size_t{0}, std::plus<size_t>());
346 }
347
350 double getAverageDepth() const {
351 double total = 0;
352 double maxNumberOfFinestBoxes = m_maxNumMDBoxes.back();
353 for (size_t depth = 0; depth < m_numMDBoxes.size(); depth++) {
354 // Add up the number of MDBoxes at that depth, weighed by their volume in
355 // units of the volume of the finest possible box.
356 // I.e. a box at level 1 is 100 x bigger than a box at level 2, so it
357 // counts 100x more.
358 total += double(depth * m_numMDBoxes[depth]) * (maxNumberOfFinestBoxes / m_maxNumMDBoxes[depth]);
359 }
360 return total / maxNumberOfFinestBoxes;
361 }
362
365 std::lock_guard<std::mutex> lock(m_mutexNumMDBoxes);
366 m_numMDBoxes.clear();
367 m_numMDBoxes.resize(m_maxDepth + 1, 0); // Reset to 0
368 m_numMDGridBoxes.resize(m_maxDepth + 1, 0); // Reset to 0
369 m_numMDBoxes[0] = 1; // Start at 1 at depth 0.
370 resetMaxNumBoxes(); // Also the maximums
371 }
372
373 // { return m_useWriteBuffer; }
376 bool isFileBacked() const { return bool(m_fileIO); }
378 IBoxControllerIO *getFileIO() { return m_fileIO.get(); }
381 void setFileBacked(const std::shared_ptr<IBoxControllerIO> &newFileIO, const std::string &fileName = "");
382 void clearFileBacked();
383 //-----------------------------------------------------------------------------------
384 // BoxCtrlChangesInterface *getChangesList(){return m_ChangesList;}
385 // void setChangesList(BoxCtrlChangesInterface *pl){m_ChangesList=pl;}
386 //-----------------------------------------------------------------------------------
387 // increase the counter, calculatinb events at max;
388 void rizeEventAtMax() { ++m_numEventsAtMax; }
391 size_t getNumEventAtMax() const { return m_numEventsAtMax; }
393 size_t claimIDRange(size_t range);
394
397 std::string getFilename() const;
400 bool useWriteBuffer() const;
401
402private:
405 m_numSplit = 1;
406 for (size_t d = 0; d < nd; d++) {
407 m_numSplit *= m_splitInto[d];
408 }
410 resetMaxNumBoxes();
411 }
412
415 m_numTopSplit = 1;
416 for (size_t d = 0; d < nd; d++) {
417 m_numTopSplit *= m_splitTopInto.get()[d];
418 }
420 resetMaxNumBoxes();
421 }
422
425 // Now calculate the max # of boxes
426 m_maxNumMDBoxes.resize(m_maxDepth + 1, 0); // Reset to 0
427 m_maxNumMDBoxes[0] = 1;
428 for (size_t depth = 1; depth < m_maxNumMDBoxes.size(); depth++) {
429 if (depth == 1 && m_splitTopInto) {
430 m_maxNumMDBoxes[depth] = m_maxNumMDBoxes[depth - 1] * double(m_numTopSplit);
431 } else {
432 m_maxNumMDBoxes[depth] = m_maxNumMDBoxes[depth - 1] * double(m_numSplit);
433 }
434 }
435 }
436
437protected:
440 BoxController(const BoxController &other);
442
443private:
445 size_t nd;
446
449 size_t m_maxId;
450
453
457
471 volatile size_t m_numEventsAtMax;
472
474 std::vector<size_t> m_splitInto;
475
477 boost::optional<std::vector<size_t>> m_splitTopInto;
478
481
484
487
490
493 std::vector<size_t> m_numMDBoxes;
494
497 std::vector<size_t> m_numMDGridBoxes;
498
501
504 std::vector<double> m_maxNumMDBoxes;
505
507 std::mutex m_idMutex;
508
509 // the class which does actual IO operations, including MRU support list
510 std::shared_ptr<IBoxControllerIO> m_fileIO;
511
513 // size_t m_bytesPerEvent;
514public:
515};
516
518using BoxController_sptr = std::shared_ptr<BoxController>;
519
521using BoxController_const_sptr = std::shared_ptr<const BoxController>;
522
523} // namespace API
524
525} // namespace Mantid
double value
The value of the point.
Definition: FitMW.cpp:51
This class is used by MDBox and MDGridBox in order to intelligently determine optimal behavior.
Definition: BoxController.h:33
bool shouldSplitBoxes(size_t nEventsInOutput, size_t eventsAdded, size_t numMDBoxes) const
Determine when would be a good time to split MDBoxes into MDGridBoxes.
size_t getSplitInto(size_t dim) const
Return into how many to split along a dimension.
void getAddingEventsParameters(size_t &eventsPerTask, size_t &numTasksPerBlock) const
Get parameters for adding events to a MDGridBox, trying to optimize parallel CPU use.
void incGridBoxesCounter(size_t depth, size_t inc=1)
IBoxControllerIO * getFileIO()
returns the pointer to the class, responsible for fileIO operations;
boost::optional< std::vector< size_t > > m_splitTopInto
Splittin # for all dimensions in the top level.
const std::vector< size_t > & getNumMDGridBoxes() const
Return the vector giving the number of MD Grid Boxes as a function of depth.
double getAverageDepth() const
Return the average recursion depth of gridding.
const std::vector< size_t > & getNumMDBoxes() const
Return the vector giving the number of MD Boxes as a function of depth.
std::shared_ptr< IBoxControllerIO > m_fileIO
std::vector< size_t > m_splitInto
Splitting # for all dimensions.
volatile size_t m_numEventsAtMax
number of events sitting in the boxes which should be split but are already split up to the max depth
size_t m_addingEvents_numTasksPerBlock
For adding events tasks.
bool isFileBacked() const
Returns if current box controller is file backed.
void setSplitInto(size_t dim, size_t num)
Set the way splitting will be done.
size_t m_numSplit
When you split a MDBox, it becomes this many sub-boxes.
size_t m_numTopSplit
When you split a top level MDBox by force, it becomes this many sub boxes.
void clearGridBoxesCounter(size_t depth)
void resetNumBoxes()
Reset the number of boxes tracked in m_numMDBoxes.
size_t m_maxDepth
Maximum splitting depth: don't go further than this many levels of recursion.
std::mutex m_mutexNumMDBoxes
Mutex for changing the number of MD Boxes.
size_t getNDims() const
Get # of dimensions.
Definition: BoxController.h:70
size_t m_significantEventsNumber
This empirically-determined number of events takes a noticeable time to process and triggers box spli...
size_t getNumEventAtMax() const
return the numner of events, which are sitting at max depth and would be split if not due to the max ...
size_t getNumSplit() const
Return how many boxes (total) a MDGridBox will contain.
size_t getTotalNumMDBoxes() const
Return the total number of MD Boxes, irrespective of depth.
std::vector< size_t > m_numMDGridBoxes
For tracking how many MDGridBoxes (not MDBoxes) are at each recursion level.
void setSplitInto(size_t num)
Set the way splitting will be done.
void setAddingEvents_eventsPerTask(size_t m_addingEvents_eventsPerTask)
When adding events, how many events per task should be done?
size_t nd
Number of dimensions.
std::vector< size_t > m_numMDBoxes
For tracking how many MDBoxes (not MDGridBoxes) are at each recursion level.
void setSplitTopInto(size_t dim, size_t num)
Set the way splitting will be done for the top level.
size_t getAddingEvents_eventsPerTask() const
void setMaxDepth(size_t value)
Sets the max recursion depth allowed for grid box splitting.
size_t getSignificantEventsNumber() const
The number of events that triggers box splitting.
void calcNumTopSplit()
When you split an MDBox by force, it becomes this many sub boxes.
void setSplitThreshold(size_t threshold)
Set the splitting threshold.
void calcNumSplit()
When you split a MDBox, it becomes this many sub-boxes.
void clearBoxesCounter(size_t depth)
size_t m_addingEvents_eventsPerTask
For adding events tasks.
BoxController(size_t nd)
Constructor.
Definition: BoxController.h:41
void setMaxId(size_t newMaxId)
Set the new maximum ID number anywhere in the workspace.
Definition: BoxController.h:87
size_t m_SplitThreshold
Splitting threshold.
size_t getTotalNumMDGridBoxes() const
Return the total number of MDGridBox'es, irrespective of depth.
BoxController & operator=(const BoxController &)=delete
size_t getSplitThreshold() const
Return the splitting threshold, in # of events.
std::mutex m_idMutex
Mutex for getting IDs.
boost::optional< std::vector< size_t > > getSplitTopInto() const
Return into how many to split along a dimension for the top level.
void incBoxesCounter(size_t depth, size_t inc=1)
const std::vector< size_t > & getSplitIntoAll() const
Return into how many to split along a every dimension.
void trackNumBoxes(size_t depth)
Call to track the number of MDBoxes are contained in the MDEventWorkspace This should be called when ...
void resetMaxNumBoxes()
Calculate the vector of the max # of MDBoxes per level.
size_t getAddingEvents_numTasksPerBlock() const
std::vector< double > m_maxNumMDBoxes
This is the maximum number of MD boxes there could be at each recursion level (e.g.
const std::vector< double > & getMaxNumMDBoxes() const
Return the vector giving the MAXIMUM number of MD Boxes as a function of depth.
void setAddingEvents_numTasksPerBlock(size_t m_addingEvents_numTasksPerBlock)
When adding events, how many events tasks per block should be done?
size_t m_maxId
The maximum ID number of any boxes in the workspace (not inclusive, i.e.
bool willSplit(size_t numPoints, size_t depth) const
Return true if the MDBox should split, given :
The header describes interface to IO Operations perfomed by the box controller May be replaced by a b...
std::shared_ptr< const BoxController > BoxController_const_sptr
Shared ptr to a const BoxController.
std::shared_ptr< BoxController > BoxController_sptr
Shared ptr to BoxController.
Helper class which provides the Collimation Length for SANS instruments.
Definition: NDArray.h:49
STL namespace.
constexpr bool operator==(const wide_integer< Bits, Signed > &lhs, const wide_integer< Bits2, Signed2 > &rhs)