Mantid
Loading...
Searching...
No Matches
NexusTester.cpp
Go to the documentation of this file.
1// Mantid Repository : https://github.com/mantidproject/mantid
2//
3// Copyright © 2018 ISIS Rutherford Appleton Laboratory UKRI,
4// NScD Oak Ridge National Laboratory, European Spallation Source,
5// Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
6// SPDX - License - Identifier: GPL - 3.0 +
11
12#include <Poco/File.h>
13#include <Poco/Thread.h>
14
15#include <cstdlib>
16#include <nexus/NeXusFile.hpp>
17
18using namespace Mantid::Kernel;
19using namespace Mantid::API;
20
21namespace Mantid::DataHandling {
22
23// Register the algorithm into the AlgorithmFactory
25
26//----------------------------------------------------------------------------------------------
28const std::string NexusTester::name() const { return "NexusTester"; }
29
31int NexusTester::version() const { return 1; }
32
34const std::string NexusTester::category() const { return "Utility\\Development"; }
35
36//----------------------------------------------------------------------------------------------
37
38//----------------------------------------------------------------------------------------------
42 std::initializer_list<std::string> exts = {".nxs"};
43
44 declareProperty(std::make_unique<FileProperty>("SaveFilename", "", FileProperty::OptionalSave, exts),
45 "The name of the Nexus file to write.");
46
47 declareProperty(std::make_unique<FileProperty>("LoadFilename", "", FileProperty::OptionalLoad, exts),
48 "The name of the Nexus file to load (optional).\n"
49 "Must have been written by NexusTester algorithm.");
50
51 declareProperty("ChunkSize", 10, "Chunk size for writing/loading, in kb of data");
52 declareProperty("NumChunks", 10, "Number of chunks to load or write");
53 declareProperty("Compress", true, "For writing: compress the data.");
54 declareProperty("HDFCacheSize", 2000000, "HDF cache size, in bytes");
55 declareProperty("ClearDiskCache", false,
56 "Clear the linux disk cache before loading.\n"
57 "Only works on linux AND you need to run MantidPlot in sudo mode (!).");
58
59 std::vector<std::string> types{"Zeros", "Incrementing Numbers", "Random Numbers"};
60 declareProperty("FakeData", "Incrementing Numbers", std::make_shared<StringListValidator>(types),
61 "For writing: type of fake data to generate.");
62
63 declareProperty("CompressionFactor", 0.0, "The size of the file divided by the the size of the data on disk.",
65 declareProperty("SaveSpeed", 0.0, "The measured rate of saving the file, in MB (of data)/sec.", Direction::Output);
66 declareProperty("LoadSpeed", 0.0, "The measured rate of loading the file, in MB (of data)/sec.", Direction::Output);
67}
68
69//----------------------------------------------------------------------------------------------
73 std::string SaveFilename = getPropertyValue("SaveFilename");
74 std::string LoadFilename = getPropertyValue("LoadFilename");
75 std::string FakeDataType = getPropertyValue("FakeData");
76 int ChunkSizeKb = getProperty("ChunkSize");
77 int NumChunks = getProperty("NumChunks");
78 bool Compress = getProperty("Compress");
79
80 if (ChunkSizeKb <= 0)
81 throw std::invalid_argument("ChunkSize must be > 0");
82 if (NumChunks <= 0)
83 throw std::invalid_argument("NumChunks must be > 0");
84
85 // Size of the chunk in number of integers
86 size_t chunkSize = ChunkSizeKb * 1024 / sizeof(uint32_t);
87 // ----------- Generate the fake data -----------------------------
88 auto fakeData = std::vector<uint32_t>(chunkSize);
89 if (FakeDataType == "Zeros") {
90 for (size_t i = 0; i < chunkSize; i++)
91 fakeData[i] = 0;
92 } else if (FakeDataType == "Incrementing Numbers") {
93 for (size_t i = 0; i < chunkSize; i++)
94 fakeData[i] = uint32_t(i);
95 } else if (FakeDataType == "Random Numbers") {
96 for (size_t i = 0; i < chunkSize; i++)
97 fakeData[i] = rand();
98 }
99
100 std::vector<int64_t> dims;
101 dims.emplace_back(int64_t(chunkSize) * NumChunks);
102 std::vector<int64_t> chunkDims;
103 chunkDims.emplace_back(int64_t(chunkSize));
104
105 // Total size in BYTES
106 double dataSizeMB = double(chunkSize * NumChunks * sizeof(uint32_t)) / (1024. * 1024.);
107 g_log.notice() << "Data size is " << dataSizeMB << " MB\n";
108
109 // ------------------------ Save a File ----------------------------
110 if (!SaveFilename.empty()) {
111 ::NeXus::File file(SaveFilename, NXACC_CREATE5);
112 file.makeGroup("FakeDataGroup", "NXdata", true);
113 file.makeCompData("FakeData", ::NeXus::UINT32, dims, Compress ? ::NeXus::LZW : ::NeXus::NONE, chunkDims, true);
114 Progress prog(this, 0.0, 1.0, NumChunks);
115 CPUTimer tim;
116 for (int i = 0; i < NumChunks; i++) {
117 std::vector<int64_t> startDims;
118 startDims.emplace_back(i * int64_t(chunkSize));
119 file.putSlab(fakeData.data(), startDims, chunkDims);
120 prog.report();
121 }
122 file.close();
123 double seconds = tim.elapsedWallClock(false);
124 double MBperSec = dataSizeMB / seconds;
125 g_log.notice() << tim << " to save the file = " << MBperSec << " MB/sec\n";
126 this->setProperty("SaveSpeed", MBperSec);
127 }
128
129 // Check the size of the file created/loaded
130 Poco::File info(SaveFilename.empty() ? LoadFilename : SaveFilename);
131 double fileSizeMB = double(info.getSize()) / (1024. * 1024.);
132 g_log.notice() << "File size is " << fileSizeMB << " MB\n";
133
134 double CompressionFactor = fileSizeMB / dataSizeMB;
135 this->setProperty("CompressionFactor", CompressionFactor);
136
137 bool ClearDiskCache = this->getProperty("ClearDiskCache");
138 if (ClearDiskCache) {
139 g_log.information() << "Clearing disk cache.\n";
140 if (system("sync ; echo 3 > /proc/sys/vm/drop_caches") != 0)
141 g_log.error("Error clearing disk cache");
142 Poco::Thread::sleep(100);
143 }
144
145 // ------------------------ Load a File ----------------------------
146 if (!LoadFilename.empty()) {
147 ::NeXus::File file(LoadFilename, NXACC_READ);
148 int HDFCacheSize = getProperty("HDFCacheSize");
149 NXsetcache(HDFCacheSize);
150 file.openGroup("FakeDataGroup", "NXdata");
151 Progress prog(this, 0.0, 1.0, NumChunks);
152 CPUTimer tim;
153
154 for (int i = 0; i < NumChunks; i++) {
155 file.openData("FakeData");
156 std::vector<int64_t> startDims;
157 startDims.emplace_back(i * int64_t(chunkSize));
158 file.getSlab(fakeData.data(), startDims, chunkDims);
159 prog.report();
160 file.closeData();
161 }
162
163 file.close();
164
165 double seconds = tim.elapsedWallClock(false);
166 double MBperSec = dataSizeMB / seconds;
167 this->setProperty("LoadSpeed", MBperSec);
168 g_log.notice() << tim << " to load the file = " << MBperSec << " MB/sec (data), " << MBperSec * CompressionFactor
169 << " MB/sec (file)\n";
170 }
171}
172
173} // namespace Mantid::DataHandling
#define DECLARE_ALGORITHM(classname)
Definition: Algorithm.h:576
void declareProperty(std::unique_ptr< Kernel::Property > p, const std::string &doc="") override
Add a property to the list of managed properties.
Definition: Algorithm.cpp:1913
std::string getPropertyValue(const std::string &name) const override
Get the value of a property as a string.
Definition: Algorithm.cpp:2026
TypedValue getProperty(const std::string &name) const override
Get the value of a property.
Definition: Algorithm.cpp:2076
Kernel::Logger & g_log
Definition: Algorithm.h:451
@ OptionalSave
to specify a file to write to but an empty string is
Definition: FileProperty.h:50
@ OptionalLoad
to specify a file to read but the file doesn't have to exist
Definition: FileProperty.h:53
Helper class for reporting progress from algorithms.
Definition: Progress.h:25
NexusTester : debugging/performance testing algorithm for nexus file loading and saving.
Definition: NexusTester.h:20
const std::string category() const override
Algorithm's category for identification.
Definition: NexusTester.cpp:34
void exec() override
Execute the algorithm.
Definition: NexusTester.cpp:72
int version() const override
Algorithm's version for identification.
Definition: NexusTester.cpp:31
void init() override
Initialize the algorithm's properties.
Definition: NexusTester.cpp:41
CPUTimer : Timer that uses the CPU time, rather than wall-clock time to measure execution time.
Definition: CPUTimer.h:24
float elapsedWallClock(bool doReset=true)
Calculate the elapsed wall-clock time, reseting the timer if specified.
Definition: CPUTimer.cpp:54
IPropertyManager * setProperty(const std::string &name, const T &value)
Templated method to set the value of a PropertyWithValue.
void notice(const std::string &msg)
Logs at notice level.
Definition: Logger.cpp:95
void error(const std::string &msg)
Logs at error level.
Definition: Logger.cpp:77
void information(const std::string &msg)
Logs at information level.
Definition: Logger.cpp:105
void report()
Increments the loop counter by 1, then sends the progress notification on behalf of its algorithm.
Definition: ProgressBase.h:51
STL namespace.
@ Output
An output workspace.
Definition: Property.h:54