Mantid
Loading...
Searching...
No Matches
LoadNexusLogs.cpp
Go to the documentation of this file.
1// Mantid Repository : https://github.com/mantidproject/mantid
2//
3// Copyright © 2018 ISIS Rutherford Appleton Laboratory UKRI,
4// NScD Oak Ridge National Laboratory, European Spallation Source,
5// Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
6// SPDX - License - Identifier: GPL - 3.0 +
10#include "MantidAPI/Run.h"
13#include <locale>
14#include <nexus/NeXusException.hpp>
15
16#include <Poco/DateTimeFormat.h>
17#include <Poco/DateTimeFormatter.h>
18#include <Poco/DateTimeParser.h>
19#include <Poco/Glob.h>
20#include <Poco/Path.h>
21
23#include <boost/scoped_array.hpp>
24
25#include <algorithm>
26
27namespace Mantid::DataHandling {
28// Register the algorithm into the algorithm factory
29DECLARE_ALGORITHM(LoadNexusLogs)
30
31using namespace Kernel;
32using API::FileProperty;
33using API::MatrixWorkspace;
35using API::WorkspaceProperty;
36using std::size_t;
37using Types::Core::DateAndTime;
38
39// Anonymous namespace
40namespace {
47bool loadAndApplyMeasurementInfo(::NeXus::File *const file, API::MatrixWorkspace &workspace) {
48
49 bool successfullyApplied = false;
50 try {
51 file->openGroup("measurement", "NXcollection");
52
53 // If we can open the measurement group. We assume that the following will
54 // be available.
55 file->openData("id");
56 workspace.mutableRun().addLogData(
57 new Mantid::Kernel::PropertyWithValue<std::string>("measurement_id", file->getStrData()));
58 file->closeData();
59 file->openData("label");
60 workspace.mutableRun().addLogData(
61 new Mantid::Kernel::PropertyWithValue<std::string>("measurement_label", file->getStrData()));
62 file->closeData();
63 file->openData("subid");
64 workspace.mutableRun().addLogData(
65 new Mantid::Kernel::PropertyWithValue<std::string>("measurement_subid", file->getStrData()));
66 file->closeData();
67 file->openData("type");
68 workspace.mutableRun().addLogData(
69 new Mantid::Kernel::PropertyWithValue<std::string>("measurement_type", file->getStrData()));
70 file->closeData();
71 file->closeGroup();
72 successfullyApplied = true;
73 } catch (::NeXus::Exception &) {
74 successfullyApplied = false;
75 }
76 return successfullyApplied;
77}
78
85bool loadAndApplyRunTitle(::NeXus::File *const file, API::MatrixWorkspace &workspace) {
86
87 bool successfullyApplied = false;
88 try {
89 file->openData("title");
90 workspace.mutableRun().addLogData(
91 new Mantid::Kernel::PropertyWithValue<std::string>("run_title", file->getStrData()));
92 file->closeData();
93 successfullyApplied = true;
94 } catch (::NeXus::Exception &) {
95 successfullyApplied = false;
96 }
97 return successfullyApplied;
98}
99
113bool isControlValue(const char &c, const std::string &propName, Kernel::Logger &log) {
114 // Have to check it falls within range accepted by c style check
115 if (c <= -1) {
116 log.warning("Found an invalid character in property " + propName);
117 // Pretend this is a control value so it is sanitized
118 return true;
119 } else {
120 // Use default global c++ locale within this program
121 std::locale locale{};
122 // Use c++ style call so we don't need to cast from int to bool
123 return std::iscntrl(c, locale);
124 }
125}
126
139std::unique_ptr<Kernel::Property> createTimeSeries(::NeXus::File &file, const std::string &propName,
140 const std::string &freqStart, Kernel::Logger &log) {
141 file.openData("time");
142 //----- Start time is an ISO8601 string date and time. ------
143 std::string start;
144 try {
145 file.getAttr("start", start);
146 } catch (::NeXus::Exception &) {
147 // Some logs have "offset" instead of start
148 try {
149 file.getAttr("offset", start);
150 } catch (::NeXus::Exception &) {
151 log.warning() << "Log entry has no start time indicated.\n";
152 file.closeData();
153 throw;
154 }
155 }
156 if (start == "No Time") {
157 start = freqStart;
158 }
159
160 // Convert to date and time
161 Types::Core::DateAndTime start_time = Types::Core::DateAndTime(start);
162 std::string time_units;
163 file.getAttr("units", time_units);
164 if (time_units.compare("second") < 0 && time_units != "s" &&
165 time_units != "minutes") // Can be s/second/seconds/minutes
166 {
167 file.closeData();
168 throw ::NeXus::Exception("Unsupported time unit '" + time_units + "'");
169 }
170 //--- Load the seconds into a double array ---
171 std::vector<double> time_double;
172 try {
173 file.getDataCoerce(time_double);
174 } catch (::NeXus::Exception &e) {
175 log.warning() << "Log entry's time field could not be loaded: '" << e.what() << "'.\n";
176 file.closeData();
177 throw;
178 }
179 file.closeData(); // Close time data
180 log.debug() << " done reading \"time\" array\n";
181
182 // Convert to seconds if needed
183 if (time_units == "minutes") {
184 using std::placeholders::_1;
185 std::transform(time_double.begin(), time_double.end(), time_double.begin(),
186 std::bind(std::multiplies<double>(), _1, 60.0));
187 }
188
189 // Now the values: Could be a string, int or double
190 file.openData("value");
191 // Get the units of the property
192 std::string value_units;
193 try {
194 file.getAttr("units", value_units);
195 } catch (::NeXus::Exception &) {
196 // Ignore missing units field.
197 value_units = "";
198 }
199
200 // Now the actual data
201 ::NeXus::Info info = file.getInfo();
202 // Check the size
203 if (size_t(info.dims[0]) != time_double.size()) {
204 file.closeData();
205 throw ::NeXus::Exception("Invalid value entry for time series");
206 }
207 if (file.isDataInt()) // Int type
208 {
209 std::vector<int> values;
210 try {
211 file.getDataCoerce(values);
212 file.closeData();
213 } catch (::NeXus::Exception &) {
214 file.closeData();
215 throw;
216 }
217 // Make an int TSP
218 auto tsp = std::make_unique<TimeSeriesProperty<int>>(propName);
219 tsp->create(start_time, time_double, values);
220 tsp->setUnits(value_units);
221 log.debug() << " done reading \"value\" array\n";
222 return tsp;
223 } else if (info.type == ::NeXus::CHAR) {
224 std::string values;
225 const int64_t item_length = info.dims[1];
226 try {
227 const int64_t nitems = info.dims[0];
228 const int64_t total_length = nitems * item_length;
229 boost::scoped_array<char> val_array(new char[total_length]);
230 file.getData(val_array.get());
231 file.closeData();
232 values = std::string(val_array.get(), total_length);
233 } catch (::NeXus::Exception &) {
234 file.closeData();
235 throw;
236 }
237 // The string may contain non-printable (i.e. control) characters, replace
238 // these
239 std::replace_if(
240 values.begin(), values.end(), [&](const char &c) { return isControlValue(c, propName, log); }, ' ');
241 auto tsp = std::make_unique<TimeSeriesProperty<std::string>>(propName);
242 std::vector<DateAndTime> times;
243 DateAndTime::createVector(start_time, time_double, times);
244 const size_t ntimes = times.size();
245 for (size_t i = 0; i < ntimes; ++i) {
246 std::string value_i = std::string(values.data() + i * item_length, item_length);
247 tsp->addValue(times[i], value_i);
248 }
249 tsp->setUnits(value_units);
250 log.debug() << " done reading \"value\" array\n";
251 return tsp;
252 } else if (info.type == ::NeXus::FLOAT32 || info.type == ::NeXus::FLOAT64) {
253 std::vector<double> values;
254 try {
255 file.getDataCoerce(values);
256 file.closeData();
257 } catch (::NeXus::Exception &) {
258 file.closeData();
259 throw;
260 }
261 auto tsp = std::make_unique<TimeSeriesProperty<double>>(propName);
262 tsp->create(start_time, time_double, values);
263 tsp->setUnits(value_units);
264 log.debug() << " done reading \"value\" array\n";
265 return tsp;
266 } else {
267 throw ::NeXus::Exception("Invalid value type for time series. Only int, double or strings are "
268 "supported");
269 }
270}
271
282std::unique_ptr<Kernel::Property> createTimeSeriesValidityFilter(::NeXus::File &file, const Kernel::Property &prop,
283 Kernel::Logger &log) {
284 const auto tsProp = dynamic_cast<const Kernel::ITimeSeriesProperty *>(&prop);
285 const auto times = tsProp->timesAsVector();
286 std::vector<int> values;
287 std::vector<bool> boolValues;
288 // Now the the validity of the values
289 // this should be a match int array to the data values (or times)
290 // If not present assume all data is valid
291 try {
292 file.openData("value_valid");
293
294 // Now the validity data
295 ::NeXus::Info info = file.getInfo();
296 // Check the size
297 if (size_t(info.dims[0]) != times.size()) {
298 throw ::NeXus::Exception("Invalid value entry for validity data");
299 }
300 if (file.isDataInt()) // Int type
301 {
302 try {
303 file.getDataCoerce(values);
304 file.closeData();
305 } catch (::NeXus::Exception &) {
306 throw;
307 }
308 } else {
309 throw ::NeXus::Exception("Invalid value type for validity data. Only int is supported");
310 }
311 } catch (::NeXus::Exception &ex) {
312 std::string error_msg = ex.what();
313 if (error_msg != "NXopendata(value_valid) failed") {
314 log.warning() << error_msg << "\n";
315 file.closeData();
316 // no data found
317 return std::unique_ptr<Kernel::Property>(nullptr);
318 }
319 }
320
321 bool invalidDataFound = false;
322 boolValues.reserve(values.size());
323 // convert the integer values to boolean with 0=invalid data
324 for (size_t i = 0; i < values.size(); i++) {
325 bool isInvalidData = (values[i] == 0);
326 boolValues.emplace_back(!isInvalidData);
327 if (isInvalidData && !invalidDataFound) {
328 invalidDataFound = true;
329 }
330 }
331 if (invalidDataFound) {
332 const auto tspName = API::LogManager::getInvalidValuesFilterLogName(prop.name());
333 auto tsp = std::make_unique<TimeSeriesProperty<bool>>(tspName);
334 tsp->create(times, boolValues);
335 log.debug() << " done reading \"value_valid\" array\n";
336
337 return tsp;
338 } else {
339 // no data found
340 return std::unique_ptr<Kernel::Property>(nullptr);
341 }
342}
343
358void appendEndTimeLog(Kernel::Property *prop, const API::Run &run) {
359 try {
360 auto tsLog = dynamic_cast<TimeSeriesProperty<double> *>(prop);
361 const auto endTime = run.endTime();
362
363 // First check if it is valid to add a additional log entry
364 if (!tsLog || tsLog->size() == 0 || endTime <= tsLog->lastTime() || prop->name() == "proton_charge")
365 return;
366
367 tsLog->addValue(endTime, tsLog->lastValue());
368 } catch (const Exception::NotFoundError &) {
369 // pass
370 } catch (const std::runtime_error &) {
371 // pass
372 }
373}
374
381void readStartAndEndTime(::NeXus::File &file, API::Run &run) {
382 try {
383 // Read the start and end time strings
384 file.openData("start_time");
385 Types::Core::DateAndTime start(file.getStrData());
386 file.closeData();
387 file.openData("end_time");
388 Types::Core::DateAndTime end(file.getStrData());
389 file.closeData();
390 run.setStartAndEndTime(start, end);
391 } catch (::NeXus::Exception &) {
392 }
393}
394
395} // End of anonymous namespace
396
399
402 declareProperty(std::make_unique<WorkspaceProperty<MatrixWorkspace>>("Workspace", "Anonymous", Direction::InOut),
403 "The name of the workspace that will be filled with the logs.");
404 const std::vector<std::string> exts{".nxs", ".n*"};
405 declareProperty(std::make_unique<FileProperty>("Filename", "", FileProperty::Load, exts),
406 "Path to the .nxs file to load. Can be an EventNeXus or a "
407 "histogrammed NeXus.");
408 declareProperty(std::make_unique<PropertyWithValue<bool>>("OverwriteLogs", true, Direction::Input),
409 "If true then some existing logs will be overwritten, if false they will "
410 "not.");
411 declareProperty(std::make_unique<PropertyWithValue<std::string>>("NXentryName", "", Direction::Input),
412 "Entry in the nexus file from which to read the logs");
413 declareProperty(std::make_unique<PropertyWithValue<std::vector<std::string>>>("AllowList", std::vector<std::string>(),
415 "If specified, only these logs will be loaded from the file (each "
416 "separated by a comma).");
417 declareProperty(std::make_unique<PropertyWithValue<std::vector<std::string>>>("BlockList", std::vector<std::string>(),
419 "If specified, logs matching one of the patterns will NOT be loaded from the file (each "
420 "separated by a comma).");
421}
422
431 std::string filename = getPropertyValue("Filename");
433
434 std::string entry_name = getPropertyValue("NXentryName");
435
436 std::vector<std::string> allow_list = getProperty("AllowList");
437 std::vector<std::string> block_list = getProperty("BlockList");
438
439 // Find the entry name to use (normally "entry" for SNS, "raw_data_1" for
440 // ISIS) if entry name is empty
441 if (entry_name.empty()) {
442 entry_name = LoadTOFRawNexus::getEntryName(filename);
443 }
444 ::NeXus::File file(filename);
445 // Find the root entry
446 try {
447 file.openGroup(entry_name, "NXentry");
448 } catch (::NeXus::Exception &) {
449 throw std::invalid_argument("Unknown NeXus file format found in file '" + filename + "', or '" + entry_name +
450 "' is not a valid NXentry");
451 }
452
455 try {
456 file.openPath("DASlogs");
457 try {
458 file.openGroup("frequency", "NXlog");
459 try {
460 file.openData("time");
461
462 //----- Start time is an ISO8601 string date and time. ------
463 try {
464 file.getAttr("start", freqStart);
465
466 } catch (::NeXus::Exception &) {
467 // Some logs have "offset" instead of start
468 try {
469 file.getAttr("offset", freqStart);
470 } catch (::NeXus::Exception &) {
471 g_log.warning() << "Log entry has no start time indicated.\n";
472 file.closeData();
473 throw;
474 }
475 }
476 file.closeData();
477 } catch (::NeXus::Exception &) {
478 // No time. This is not an SNS SNAP file
479 }
480 file.closeGroup();
481 } catch (::NeXus::Exception &) {
482 // No time. This is not an SNS frequency group
483 }
484 file.closeGroup();
485 } catch (::NeXus::Exception &) {
486 // No time. This is not an SNS group
487 }
488
489 readStartAndEndTime(file, workspace->mutableRun());
490
491 if (!allow_list.empty() && !block_list.empty()) {
492 throw std::runtime_error("BlockList and AllowList are mutually exclusive! "
493 "Please only enter values for one of these fields.");
494 }
495
496 const std::map<std::string, std::set<std::string>> &allEntries = getFileInfo()->getAllEntries();
497
498 auto lf_LoadLogsByClass = [&](const std::string &group_class, const bool isLog) {
499 auto itGroupClass = allEntries.find(group_class);
500 if (itGroupClass == allEntries.end()) {
501 return;
502 }
503 const std::set<std::string> &entries = itGroupClass->second;
504 // still a linear search
505 for (const std::string &entry : entries) {
506 // match for 2nd level entry /a/b
507 if (std::count(entry.begin(), entry.end(), '/') == 2) {
508 if (isLog) {
509 loadLogs(file, entry, group_class, workspace, allow_list, block_list);
510 } else {
511 loadNPeriods(file, workspace);
512 }
513 }
514 }
515 };
516 lf_LoadLogsByClass("IXselog", true);
517 lf_LoadLogsByClass("IXrunlog", true);
518 lf_LoadLogsByClass("IXperiods", false);
519
520 auto lf_LoadLogsByName = [&](const std::string &group_name) {
521 for (auto itGroupClass = allEntries.begin(); itGroupClass != allEntries.end(); ++itGroupClass) {
522
523 const std::string &group_class = itGroupClass->first;
524 const std::set<std::string> &entries = itGroupClass->second;
525
526 const std::string absoluteGroupName = "/" + entry_name + "/" + group_name;
527 auto itGroupName = entries.find(absoluteGroupName);
528 if (itGroupName == entries.end()) {
529 continue;
530 }
531 // here we must search only in NxLogs and NXpositioner sets
532 loadLogs(file, absoluteGroupName, group_class, workspace, allow_list, block_list);
533 }
534 };
535
536 lf_LoadLogsByName("DASlogs");
537 lf_LoadLogsByName("framelog");
538
539 // If there's measurement information, load that info as logs.
540 loadAndApplyMeasurementInfo(&file, *workspace);
541 // If there's title information, load that info as logs.
542 loadAndApplyRunTitle(&file, *workspace);
543
544 // Freddie Akeroyd 12/10/2011
545 // current ISIS implementation contains an additional indirection between
546 // collected frames via an
547 // "event_frame_number" array in NXevent_data (which eliminates frames with no
548 // events).
549 // the proton_log is for all frames and so is longer than the event_index
550 // array, so we need to
551 // filter the proton_charge log based on event_frame_number
552 // This difference will be removed in future for compatibility with SNS, but
553 // the code below will allow current SANS2D files to load
554 if (workspace->mutableRun().hasProperty("proton_log")) {
555 std::vector<int> event_frame_number;
556 this->getLogger().notice() << "Using old ISIS proton_log and event_frame_number indirection...\n";
557 try {
558 // Find the bank/name corresponding to the first event data entry, i.e.
559 // one with type NXevent_data.
560 file.openPath("/" + entry_name);
561 auto itEventData = allEntries.find("NXevent_data");
562 if (itEventData != allEntries.end()) {
563 const std::set<std::string> &events = itEventData->second;
564 for (const std::string &event : events) {
565 const std::string eventEntry = event.substr(event.find_last_of("/") + 1);
566
567 this->getLogger().debug() << "Opening"
568 << " /" + entry_name + "/" + eventEntry + "/event_frame_number"
569 << " to find the event_frame_number\n";
570 file.openPath("/" + entry_name + "/" + eventEntry + "/event_frame_number");
571 file.getData(event_frame_number);
572 }
573 }
574 } catch (const ::NeXus::Exception &) {
575 this->getLogger().warning() << "Unable to load event_frame_number - "
576 "filtering events by time will not work \n";
577 }
578 file.openPath("/" + entry_name);
579 if (!event_frame_number.empty()) // ISIS indirection - see above comments
580 {
582 dynamic_cast<Kernel::TimeSeriesProperty<double> *>(workspace->mutableRun().getProperty("proton_log"));
583 if (!plog)
584 throw std::runtime_error("Could not cast (interpret) proton_log as a time "
585 "series property. Cannot continue.");
587 std::vector<double> pval;
588 std::vector<Mantid::Types::Core::DateAndTime> ptime;
589 pval.reserve(event_frame_number.size());
590 ptime.reserve(event_frame_number.size());
591 std::vector<Mantid::Types::Core::DateAndTime> plogt = plog->timesAsVector();
592 std::vector<double> plogv = plog->valuesAsVector();
593 for (auto number : event_frame_number) {
594 ptime.emplace_back(plogt[number]);
595 pval.emplace_back(plogv[number]);
596 }
597 pcharge->create(ptime, pval);
598 pcharge->setUnits("uAh");
599 workspace->mutableRun().addProperty(pcharge, true);
600 }
601 }
602
603 if (!workspace->run().hasProperty("gd_prtn_chrg")) {
604 // Try pulling it from the main proton_charge entry first
605 try {
606 file.openData("proton_charge");
607 std::vector<double> values;
608 file.getDataCoerce(values);
609 std::string units;
610 file.getAttr("units", units);
611 double charge = values.front();
612 if (units.find("picoCoulomb") != std::string::npos) {
613 charge *= 1.e-06 / 3600.;
614 }
615 workspace->mutableRun().setProtonCharge(charge);
616 } catch (::NeXus::Exception &) {
617 // Try and integrate the proton logs
618 try {
619 // Use the DAS logs to integrate the proton charge (if any).
620 workspace->mutableRun().getProtonCharge();
621 } catch (Exception::NotFoundError &) {
622 // Ignore not found property error.
623 }
624 }
625 }
626
627 if (!allow_list.empty()) {
628 for (const auto &allow : allow_list) {
629 if (!workspace->run().hasProperty(allow)) {
630 g_log.notice() << "could not load entry '" << allow << "' that was specified in the allow list"
631 << "\n";
632 }
633 }
634 }
635
636 // Close the file
637 file.close();
638
639 if (m_logsWithInvalidValues.size() > 0) {
640 if (m_logsWithInvalidValues.size() == 1) {
641 g_log.warning() << "Sample Log \"" << m_logsWithInvalidValues[0]
642 << "\" contains invalid values, click \"Show Sample Logs\" "
643 "for details.\n";
644 }
645 auto other_string = (m_logsWithInvalidValues.size() < 2) ? " other " : " others";
646 g_log.warning() << "Sample Log \"" << m_logsWithInvalidValues[0] << "\" and " << m_logsWithInvalidValues.size() - 1
647 << other_string
648 << " contain invalid values, click \"Show Sample Logs\" for "
649 "details.\n";
650 }
651}
652
659void LoadNexusLogs::loadVetoPulses(::NeXus::File &file, const std::shared_ptr<API::MatrixWorkspace> &workspace) const {
660 try {
661 file.openGroup("Veto_pulse", "NXgroup");
662 } catch (::NeXus::Exception &) {
663 // No group. This is common in older files
664 return;
665 }
666 file.openData("veto_pulse_time");
667
668 // Load the start date/time as ISO8601 string.
669 std::string start_time;
670 file.getAttr("start_time", start_time);
671 DateAndTime start(start_time);
672
673 // Read the offsets
674 std::vector<double> time_double;
675 file.getData(time_double);
676
677 // Fake values with zeroes.
678 std::vector<double> values(time_double.size(), 0.0);
679 TimeSeriesProperty<double> *tsp = new TimeSeriesProperty<double>("veto_pulse_time");
680 tsp->create(start, time_double, values);
681 tsp->setUnits("");
682
683 // Add the log
684 workspace->mutableRun().addProperty(tsp);
685
686 file.closeData();
687 file.closeGroup();
688}
689
690void LoadNexusLogs::loadNPeriods(::NeXus::File &file, const std::shared_ptr<API::MatrixWorkspace> &workspace) const {
691 int value = 1; // Default to 1-period unless
692 try {
693 file.openGroup("periods", "IXperiods");
694 file.openData("number");
695 file.getData(&value);
696 file.closeData();
697 file.closeGroup();
698 } catch (::NeXus::Exception &) {
699 // Likely missing IXperiods.
700 return;
701 }
702
703 API::Run &run = workspace->mutableRun();
704 const std::string nPeriodsLabel = "nperiods";
705 if (!run.hasProperty(nPeriodsLabel)) {
706 run.addProperty(new PropertyWithValue<int>(nPeriodsLabel, value));
707 }
708
709 // For ISIS Nexus only, fabricate an additional log containing an array of
710 // proton charge information from the periods group.
711 try {
712 file.openGroup("periods", "IXperiods");
713
714 // Get the number of periods again
715 file.openData("number");
716 int numberOfPeriods = 0;
717 file.getData(&numberOfPeriods);
718 file.closeData();
719
720 // Get the proton charge vector
721 std::vector<double> protonChargeByPeriod(numberOfPeriods);
722 file.openData("proton_charge");
723 file.getDataCoerce(protonChargeByPeriod);
724 file.closeData();
725
726 // Add the proton charge vector
727 const std::string protonChargeByPeriodLabel = "proton_charge_by_period";
728 if (!run.hasProperty(protonChargeByPeriodLabel)) {
729 run.addProperty(new ArrayProperty<double>(protonChargeByPeriodLabel, std::move(protonChargeByPeriod)));
730 }
731 file.closeGroup();
732 } catch (::NeXus::Exception &) {
733 this->g_log.debug("Cannot read periods information from the nexus file. "
734 "This group may be absent.");
735 file.closeGroup();
736 } catch (std::runtime_error &) {
737 this->g_log.debug("Cannot read periods information from the nexus file. "
738 "This group may be absent.");
739 file.closeGroup();
740 }
741}
742
753void LoadNexusLogs::loadLogs(::NeXus::File &file, const std::string &absolute_entry_name,
754 const std::string &entry_class, const std::shared_ptr<API::MatrixWorkspace> &workspace,
755 const std::vector<std::string> &allow_list,
756 const std::vector<std::string> &block_list) const {
757
758 const std::map<std::string, std::set<std::string>> &allEntries = getFileInfo()->getAllEntries();
759
760 auto lf_LoadByLogClass = [&](const std::string &logClass, const bool isNxLog) {
761 auto itLogClass = allEntries.find(logClass);
762 if (itLogClass == allEntries.end()) {
763 return;
764 }
765 const std::set<std::string> &logsSet = itLogClass->second;
766 auto itPrefixBegin = logsSet.lower_bound(absolute_entry_name);
767
768 if (allow_list.empty()) {
769 for (auto it = itPrefixBegin;
770 it != logsSet.end() && it->compare(0, absolute_entry_name.size(), absolute_entry_name) == 0; ++it) {
771 // must be third level entry
772 if (std::count(it->begin(), it->end(), '/') == 3) {
773 if (!block_list.empty()) {
774 bool skip = false;
775 for (const auto &block : block_list) {
776 Poco::Glob glob(block);
777 if (glob.match((*it).substr((*it).find_last_of("/") + 1))) {
778 skip = true;
779 break;
780 }
781 }
782 if (skip) {
783 continue;
784 }
785 }
786
787 if (isNxLog) {
788 loadNXLog(file, *it, logClass, workspace);
789 } else {
790 loadSELog(file, *it, workspace);
791 }
792 }
793 }
794 } else {
795 for (const auto &allow : allow_list) {
796 itPrefixBegin = logsSet.find(absolute_entry_name + "/" + allow);
797 if (itPrefixBegin == logsSet.end()) {
798 // don't print warning yet since it might be found in another log
799 // class
800 continue;
801 }
802 auto it = itPrefixBegin;
803 // must be third level entry
804 if (std::count(it->begin(), it->end(), '/') == 3) {
805 if (isNxLog) {
806 loadNXLog(file, *it, logClass, workspace);
807 } else {
808 loadSELog(file, *it, workspace);
809 }
810 }
811 }
812 }
813 };
814
815 const std::string entry_name = absolute_entry_name.substr(absolute_entry_name.find_last_of("/") + 1);
816 file.openGroup(entry_name, entry_class);
817 lf_LoadByLogClass("NXlog", true);
818 lf_LoadByLogClass("NXpositioner", true);
819 lf_LoadByLogClass("IXseblock", false);
821
822 file.closeGroup();
823}
824
833void LoadNexusLogs::loadNXLog(::NeXus::File &file, const std::string &absolute_entry_name,
834 const std::string &entry_class,
835 const std::shared_ptr<API::MatrixWorkspace> &workspace) const {
836
837 const std::string entry_name = absolute_entry_name.substr(absolute_entry_name.find_last_of("/") + 1);
838 g_log.debug() << "processing " << entry_name << ":" << entry_class << "\n";
839 file.openGroup(entry_name, entry_class);
840 // Validate the NX log class.
841 // Just verify that time and value entries exist
842 const std::string timeEntry = absolute_entry_name + "/time";
843 const std::string valueEntry = absolute_entry_name + "/value";
844 bool foundValue = false;
845 bool foundTime = false;
846
847 const std::map<std::string, std::set<std::string>> &allEntries = getFileInfo()->getAllEntries();
848 // reverse search to take advantage of the fact that these are located in SDS
849 for (auto it = allEntries.rbegin(); it != allEntries.rend(); ++it) {
850 const std::set<std::string> &entriesSet = it->second;
851 if (entriesSet.count(timeEntry) == 1) {
852 foundTime = true;
853 }
854 if (entriesSet.count(valueEntry) == 1) {
855 foundValue = true;
856 }
857 if (foundTime && foundValue) {
858 break;
859 }
860 }
861
862 if (!foundTime || !foundValue) {
863 g_log.warning() << "Invalid NXlog entry " << entry_name << " found. Did not contain 'value' and 'time'.\n";
864 file.closeGroup();
865 return;
866 }
867
868 // whether or not to overwrite logs on workspace
869 bool overwritelogs = this->getProperty("OverwriteLogs");
870 try {
871 if (overwritelogs || !(workspace->run().hasProperty(entry_name))) {
872 auto logValue = createTimeSeries(file, entry_name, freqStart, g_log);
873 auto validityLogValue = createTimeSeriesValidityFilter(file, *logValue, g_log);
874 if (validityLogValue) {
875 appendEndTimeLog(validityLogValue.get(), workspace->run());
876 workspace->mutableRun().addProperty(std::move(validityLogValue), overwritelogs);
877 m_logsWithInvalidValues.emplace_back(entry_name);
878 }
879 appendEndTimeLog(logValue.get(), workspace->run());
880 workspace->mutableRun().addProperty(std::move(logValue), overwritelogs);
881 }
882 } catch (::NeXus::Exception &e) {
883 g_log.warning() << "NXlog entry " << entry_name << " gave an error when loading:'" << e.what() << "'.\n";
884 } catch (std::invalid_argument &e) {
885 g_log.warning() << "NXlog entry " << entry_name << " gave an error when loading:'" << e.what() << "'.\n";
886 }
887
888 file.closeGroup();
889}
890
891void LoadNexusLogs::loadSELog(::NeXus::File &file, const std::string &absolute_entry_name,
892 const std::shared_ptr<API::MatrixWorkspace> &workspace) const {
893 // Open the entry
894 const std::string entry_name = absolute_entry_name.substr(absolute_entry_name.find_last_of("/") + 1);
895
896 file.openGroup(entry_name, "IXseblock");
897 std::string propName = entry_name;
898 if (workspace->run().hasProperty(propName)) {
899 propName = "selog_" + propName;
900 }
901 // There are two possible entries:
902 // value_log - A time series entry. This can contain a corrupt value entry
903 // so if it does use the value one
904 // value - A single value float entry
905 const std::string valueEntry = absolute_entry_name + "/value";
906 const std::string valueLogEntry = absolute_entry_name + "/value_log";
907 bool foundValue = false;
908 bool foundValueLog = false;
909
910 const std::map<std::string, std::set<std::string>> &allEntries = getFileInfo()->getAllEntries();
911
912 for (auto it = allEntries.rbegin(); it != allEntries.rend(); ++it) {
913 const std::set<std::string> &entriesSet = it->second;
914 if (entriesSet.count(valueEntry) == 1) {
915 foundValue = true;
916 }
917 if (entriesSet.count(valueLogEntry) == 1) {
918 foundValueLog = true;
919 // takes precedence
920 break;
921 }
922 }
923
924 std::unique_ptr<Kernel::Property> logValue;
925 if (foundValueLog) {
926 try {
927 try {
928 file.openGroup("value_log", "NXlog");
929 } catch (::NeXus::Exception &) {
930 file.closeGroup();
931 throw;
932 }
933
934 logValue = createTimeSeries(file, propName, freqStart, g_log);
935 auto validityLogValue = createTimeSeriesValidityFilter(file, *logValue, g_log);
936 if (validityLogValue) {
937 appendEndTimeLog(validityLogValue.get(), workspace->run());
938 workspace->mutableRun().addProperty(std::move(validityLogValue));
939 m_logsWithInvalidValues.emplace_back(propName);
940 }
941 appendEndTimeLog(logValue.get(), workspace->run());
942
943 file.closeGroup();
944 } catch (std::exception &e) {
945 g_log.warning() << "IXseblock entry '" << entry_name << "' gave an error when loading "
946 << "a time series:'" << e.what() << "'. Skipping entry\n";
947 file.closeGroup(); // value_log
948 file.closeGroup(); // entry_name
949 return;
950 }
951 } else if (foundValue) {
952 try {
953 // This may have a larger dimension than 1 bit it has no time field so
954 // take the first entry
955 file.openData("value");
956 ::NeXus::Info info = file.getInfo();
957 if (info.type == ::NeXus::FLOAT32) {
958 boost::scoped_array<float> value(new float[info.dims[0]]);
959 file.getData(value.get());
960 file.closeData();
961 logValue = std::make_unique<Kernel::PropertyWithValue<double>>(propName, static_cast<double>(value[0]), true);
962 } else {
963 file.closeGroup();
964 return;
965 }
966 } catch (::NeXus::Exception &e) {
967 g_log.warning() << "IXseblock entry " << entry_name << " gave an error when loading "
968 << "a single value:'" << e.what() << "'.\n";
969 file.closeData();
970 file.closeGroup();
971 return;
972 }
973 } else {
974 g_log.warning() << "IXseblock entry " << entry_name << " cannot be read, skipping entry.\n";
975 file.closeGroup();
976 return;
977 }
978 workspace->mutableRun().addProperty(std::move(logValue));
979 file.closeGroup();
980}
981
982} // namespace Mantid::DataHandling
#define DECLARE_ALGORITHM(classname)
Definition: Algorithm.h:576
double value
The value of the point.
Definition: FitMW.cpp:51
IPeaksWorkspace_sptr workspace
Definition: IndexPeaks.cpp:114
void declareProperty(std::unique_ptr< Kernel::Property > p, const std::string &doc="") override
Add a property to the list of managed properties.
Definition: Algorithm.cpp:1913
std::string getPropertyValue(const std::string &name) const override
Get the value of a property as a string.
Definition: Algorithm.cpp:2026
TypedValue getProperty(const std::string &name) const override
Get the value of a property.
Definition: Algorithm.cpp:2076
Kernel::Logger & getLogger() const
Returns a reference to the logger.
Definition: Algorithm.cpp:1660
@ Load
allowed here which will be passed to the algorithm
Definition: FileProperty.h:52
bool hasProperty(const std::string &name) const
Does the property exist on the object.
Definition: LogManager.cpp:265
void addProperty(Kernel::Property *prop, bool overwrite=false)
Add data to the object in the form of a property.
Definition: LogManager.h:79
static std::string getInvalidValuesFilterLogName(const std::string &logName)
Gets the correct log name for the matching invalid values log for a given log name.
Definition: LogManager.cpp:565
virtual const std::shared_ptr< Mantid::Kernel::NexusHDF5Descriptor > getFileInfo() const noexcept
Required to pass m_fileInfo to static functions Keeping it shared_ptr to match setFileInfo signature ...
This class stores information regarding an experimental run as a series of log entries.
Definition: Run.h:38
A property class for workspaces.
std::vector< std::string > m_logsWithInvalidValues
void init() override
Overwrites Algorithm method.
void loadSELog(::NeXus::File &file, const std::string &absolute_entry_name, const std::shared_ptr< API::MatrixWorkspace > &workspace) const
Load an IXseblock entry.
LoadNexusLogs()
Default constructor.
void loadLogs(::NeXus::File &file, const std::string &absolute_entry_name, const std::string &entry_class, const std::shared_ptr< API::MatrixWorkspace > &workspace, const std::vector< std::string > &allow_list, const std::vector< std::string > &block_list) const
Load log data from a group.
void execLoader() override
Overwrites Algorithm method.
std::string freqStart
Use frequency start for Monitor19 and Special1_19 logs with "No Time" for SNAP.
void loadVetoPulses(::NeXus::File &file, const std::shared_ptr< API::MatrixWorkspace > &workspace) const
Try to load the "Veto_pulse" field in DASLogs and convert it to a sample log.
void loadNXLog(::NeXus::File &file, const std::string &absolute_entry_name, const std::string &entry_class, const std::shared_ptr< API::MatrixWorkspace > &workspace) const
Load an NXlog entry.
void loadNPeriods(::NeXus::File &file, const std::shared_ptr< API::MatrixWorkspace > &workspace) const
static std::string getEntryName(const std::string &filename)
Support for a property that holds an array of values.
Definition: ArrayProperty.h:28
Exception for when an item is not found in a collection.
Definition: Exception.h:145
void debug(const std::string &msg)
Logs at debug level.
Definition: Logger.cpp:114
void notice(const std::string &msg)
Logs at notice level.
Definition: Logger.cpp:95
void warning(const std::string &msg)
Logs at warning level.
Definition: Logger.cpp:86
The concrete, templated class for properties.
virtual void setUnits(const std::string &unit)
Sets the units of the property, as a string.
Definition: Property.cpp:186
A specialised Property class for holding a series of time-value pairs.
std::vector< TYPE > valuesAsVector() const
Return the time series's values (unfiltered) as a vector<TYPE>
void create(const Types::Core::DateAndTime &start_time, const std::vector< double > &time_sec, const std::vector< TYPE > &new_values)
Clears and creates a TimeSeriesProperty from these parameters.
std::vector< Types::Core::DateAndTime > timesAsVector() const override
Return the time series's times as a vector<DateAndTime>
Kernel::Logger g_log("ExperimentInfo")
static logger object
std::shared_ptr< MatrixWorkspace > MatrixWorkspace_sptr
shared pointer to the matrix workspace base class
@ InOut
Both an input & output workspace.
Definition: Property.h:55
@ Input
An input workspace.
Definition: Property.h:53