14#include <nexus/NeXusException.hpp>
16#include <Poco/DateTimeFormat.h>
17#include <Poco/DateTimeFormatter.h>
18#include <Poco/DateTimeParser.h>
23#include <boost/scoped_array.hpp>
31using namespace Kernel;
32using API::FileProperty;
33using API::MatrixWorkspace;
35using API::WorkspaceProperty;
37using Types::Core::DateAndTime;
47bool loadAndApplyMeasurementInfo(::NeXus::File *
const file, API::MatrixWorkspace &
workspace) {
49 bool successfullyApplied =
false;
51 file->openGroup(
"measurement",
"NXcollection");
59 file->openData(
"label");
63 file->openData(
"subid");
67 file->openData(
"type");
72 successfullyApplied =
true;
73 }
catch (::NeXus::Exception &) {
74 successfullyApplied =
false;
76 return successfullyApplied;
85bool loadAndApplyRunTitle(::NeXus::File *
const file, API::MatrixWorkspace &
workspace) {
87 bool successfullyApplied =
false;
89 file->openData(
"title");
93 successfullyApplied =
true;
94 }
catch (::NeXus::Exception &) {
95 successfullyApplied =
false;
97 return successfullyApplied;
113bool isControlValue(
const char &c,
const std::string &propName, Kernel::Logger &log) {
116 log.warning(
"Found an invalid character in property " + propName);
121 std::locale locale{};
123 return std::iscntrl(c, locale);
139std::unique_ptr<Kernel::Property> createTimeSeries(::NeXus::File &file,
const std::string &propName,
140 const std::string &freqStart, Kernel::Logger &log) {
141 file.openData(
"time");
145 file.getAttr(
"start", start);
146 }
catch (::NeXus::Exception &) {
149 file.getAttr(
"offset", start);
150 }
catch (::NeXus::Exception &) {
151 log.warning() <<
"Log entry has no start time indicated.\n";
156 if (start ==
"No Time") {
161 Types::Core::DateAndTime start_time = Types::Core::DateAndTime(start);
162 std::string time_units;
163 file.getAttr(
"units", time_units);
164 if (time_units.compare(
"second") < 0 && time_units !=
"s" &&
165 time_units !=
"minutes")
168 throw ::NeXus::Exception(
"Unsupported time unit '" + time_units +
"'");
171 std::vector<double> time_double;
173 file.getDataCoerce(time_double);
174 }
catch (::NeXus::Exception &e) {
175 log.warning() <<
"Log entry's time field could not be loaded: '" << e.what() <<
"'.\n";
180 log.debug() <<
" done reading \"time\" array\n";
183 if (time_units ==
"minutes") {
184 using std::placeholders::_1;
185 std::transform(time_double.begin(), time_double.end(), time_double.begin(),
186 std::bind(std::multiplies<double>(), _1, 60.0));
190 file.openData(
"value");
192 std::string value_units;
194 file.getAttr(
"units", value_units);
195 }
catch (::NeXus::Exception &) {
201 ::NeXus::Info info = file.getInfo();
203 if (
size_t(info.dims[0]) != time_double.size()) {
205 throw ::NeXus::Exception(
"Invalid value entry for time series");
207 if (file.isDataInt())
209 std::vector<int> values;
211 file.getDataCoerce(values);
213 }
catch (::NeXus::Exception &) {
218 auto tsp = std::make_unique<TimeSeriesProperty<int>>(propName);
219 tsp->create(start_time, time_double, values);
220 tsp->setUnits(value_units);
221 log.debug() <<
" done reading \"value\" array\n";
223 }
else if (info.type == ::NeXus::CHAR) {
225 const int64_t item_length = info.dims[1];
227 const int64_t nitems = info.dims[0];
228 const int64_t total_length = nitems * item_length;
229 boost::scoped_array<char> val_array(
new char[total_length]);
230 file.getData(val_array.get());
232 values = std::string(val_array.get(), total_length);
233 }
catch (::NeXus::Exception &) {
240 values.begin(), values.end(), [&](
const char &c) { return isControlValue(c, propName, log); },
' ');
241 auto tsp = std::make_unique<TimeSeriesProperty<std::string>>(propName);
242 std::vector<DateAndTime> times;
243 DateAndTime::createVector(start_time, time_double, times);
244 const size_t ntimes = times.size();
245 for (
size_t i = 0; i < ntimes; ++i) {
246 std::string value_i = std::string(values.data() + i * item_length, item_length);
247 tsp->addValue(times[i], value_i);
249 tsp->setUnits(value_units);
250 log.debug() <<
" done reading \"value\" array\n";
252 }
else if (info.type == ::NeXus::FLOAT32 || info.type == ::NeXus::FLOAT64) {
253 std::vector<double> values;
255 file.getDataCoerce(values);
257 }
catch (::NeXus::Exception &) {
261 auto tsp = std::make_unique<TimeSeriesProperty<double>>(propName);
262 tsp->create(start_time, time_double, values);
263 tsp->setUnits(value_units);
264 log.debug() <<
" done reading \"value\" array\n";
267 throw ::NeXus::Exception(
"Invalid value type for time series. Only int, double or strings are "
282std::unique_ptr<Kernel::Property> createTimeSeriesValidityFilter(::NeXus::File &file,
const Kernel::Property &prop,
283 Kernel::Logger &log) {
284 const auto tsProp =
dynamic_cast<const Kernel::ITimeSeriesProperty *
>(&prop);
285 const auto times = tsProp->timesAsVector();
286 std::vector<int> values;
287 std::vector<bool> boolValues;
292 file.openData(
"value_valid");
295 ::NeXus::Info info = file.getInfo();
297 if (
size_t(info.dims[0]) != times.size()) {
298 throw ::NeXus::Exception(
"Invalid value entry for validity data");
300 if (file.isDataInt())
303 file.getDataCoerce(values);
305 }
catch (::NeXus::Exception &) {
309 throw ::NeXus::Exception(
"Invalid value type for validity data. Only int is supported");
311 }
catch (::NeXus::Exception &ex) {
312 std::string error_msg = ex.what();
313 if (error_msg !=
"NXopendata(value_valid) failed") {
314 log.warning() << error_msg <<
"\n";
317 return std::unique_ptr<Kernel::Property>(
nullptr);
321 bool invalidDataFound =
false;
322 boolValues.reserve(values.size());
324 for (
size_t i = 0; i < values.size(); i++) {
325 bool isInvalidData = (values[i] == 0);
326 boolValues.emplace_back(!isInvalidData);
327 if (isInvalidData && !invalidDataFound) {
328 invalidDataFound =
true;
331 if (invalidDataFound) {
333 auto tsp = std::make_unique<TimeSeriesProperty<bool>>(tspName);
334 tsp->create(times, boolValues);
335 log.debug() <<
" done reading \"value_valid\" array\n";
340 return std::unique_ptr<Kernel::Property>(
nullptr);
358void appendEndTimeLog(Kernel::Property *prop,
const API::Run &run) {
360 auto tsLog =
dynamic_cast<TimeSeriesProperty<double> *
>(prop);
361 const auto endTime = run.endTime();
364 if (!tsLog || tsLog->size() == 0 || endTime <= tsLog->lastTime() || prop->name() ==
"proton_charge")
367 tsLog->addValue(endTime, tsLog->lastValue());
368 }
catch (
const Exception::NotFoundError &) {
370 }
catch (
const std::runtime_error &) {
381void readStartAndEndTime(::NeXus::File &file, API::Run &run) {
384 file.openData(
"start_time");
385 Types::Core::DateAndTime start(file.getStrData());
387 file.openData(
"end_time");
388 Types::Core::DateAndTime end(file.getStrData());
390 run.setStartAndEndTime(start, end);
391 }
catch (::NeXus::Exception &) {
403 "The name of the workspace that will be filled with the logs.");
404 const std::vector<std::string> exts{
".nxs",
".n*"};
406 "Path to the .nxs file to load. Can be an EventNeXus or a "
407 "histogrammed NeXus.");
409 "If true then some existing logs will be overwritten, if false they will "
412 "Entry in the nexus file from which to read the logs");
415 "If specified, only these logs will be loaded from the file (each "
416 "separated by a comma).");
419 "If specified, logs matching one of the patterns will NOT be loaded from the file (each "
420 "separated by a comma).");
436 std::vector<std::string> allow_list =
getProperty(
"AllowList");
437 std::vector<std::string> block_list =
getProperty(
"BlockList");
441 if (entry_name.empty()) {
444 ::NeXus::File file(filename);
447 file.openGroup(entry_name,
"NXentry");
448 }
catch (::NeXus::Exception &) {
449 throw std::invalid_argument(
"Unknown NeXus file format found in file '" + filename +
"', or '" + entry_name +
450 "' is not a valid NXentry");
456 file.openPath(
"DASlogs");
458 file.openGroup(
"frequency",
"NXlog");
460 file.openData(
"time");
466 }
catch (::NeXus::Exception &) {
470 }
catch (::NeXus::Exception &) {
471 g_log.
warning() <<
"Log entry has no start time indicated.\n";
477 }
catch (::NeXus::Exception &) {
481 }
catch (::NeXus::Exception &) {
485 }
catch (::NeXus::Exception &) {
489 readStartAndEndTime(file,
workspace->mutableRun());
491 if (!allow_list.empty() && !block_list.empty()) {
492 throw std::runtime_error(
"BlockList and AllowList are mutually exclusive! "
493 "Please only enter values for one of these fields.");
496 const std::map<std::string, std::set<std::string>> &allEntries =
getFileInfo()->getAllEntries();
498 auto lf_LoadLogsByClass = [&](
const std::string &group_class,
const bool isLog) {
499 auto itGroupClass = allEntries.find(group_class);
500 if (itGroupClass == allEntries.end()) {
503 const std::set<std::string> &entries = itGroupClass->second;
505 for (
const std::string &entry : entries) {
507 if (std::count(entry.begin(), entry.end(),
'/') == 2) {
516 lf_LoadLogsByClass(
"IXselog",
true);
517 lf_LoadLogsByClass(
"IXrunlog",
true);
518 lf_LoadLogsByClass(
"IXperiods",
false);
520 auto lf_LoadLogsByName = [&](
const std::string &group_name) {
521 for (
auto itGroupClass = allEntries.begin(); itGroupClass != allEntries.end(); ++itGroupClass) {
523 const std::string &group_class = itGroupClass->first;
524 const std::set<std::string> &entries = itGroupClass->second;
526 const std::string absoluteGroupName =
"/" + entry_name +
"/" + group_name;
527 auto itGroupName = entries.find(absoluteGroupName);
528 if (itGroupName == entries.end()) {
532 loadLogs(file, absoluteGroupName, group_class,
workspace, allow_list, block_list);
536 lf_LoadLogsByName(
"DASlogs");
537 lf_LoadLogsByName(
"framelog");
540 loadAndApplyMeasurementInfo(&file, *
workspace);
554 if (
workspace->mutableRun().hasProperty(
"proton_log")) {
555 std::vector<int> event_frame_number;
556 this->
getLogger().
notice() <<
"Using old ISIS proton_log and event_frame_number indirection...\n";
560 file.openPath(
"/" + entry_name);
561 auto itEventData = allEntries.find(
"NXevent_data");
562 if (itEventData != allEntries.end()) {
563 const std::set<std::string> &events = itEventData->second;
564 for (
const std::string &event : events) {
565 const std::string eventEntry =
event.substr(event.find_last_of(
"/") + 1);
568 <<
" /" + entry_name +
"/" + eventEntry +
"/event_frame_number"
569 <<
" to find the event_frame_number\n";
570 file.openPath(
"/" + entry_name +
"/" + eventEntry +
"/event_frame_number");
571 file.getData(event_frame_number);
574 }
catch (const ::NeXus::Exception &) {
576 "filtering events by time will not work \n";
578 file.openPath(
"/" + entry_name);
579 if (!event_frame_number.empty())
584 throw std::runtime_error(
"Could not cast (interpret) proton_log as a time "
585 "series property. Cannot continue.");
587 std::vector<double> pval;
588 std::vector<Mantid::Types::Core::DateAndTime> ptime;
589 pval.reserve(event_frame_number.size());
590 ptime.reserve(event_frame_number.size());
591 std::vector<Mantid::Types::Core::DateAndTime> plogt = plog->
timesAsVector();
593 for (
auto number : event_frame_number) {
594 ptime.emplace_back(plogt[number]);
595 pval.emplace_back(plogv[number]);
597 pcharge->
create(ptime, pval);
599 workspace->mutableRun().addProperty(pcharge,
true);
603 if (!
workspace->run().hasProperty(
"gd_prtn_chrg")) {
606 file.openData(
"proton_charge");
607 std::vector<double> values;
608 file.getDataCoerce(values);
610 file.getAttr(
"units", units);
611 double charge = values.front();
612 if (units.find(
"picoCoulomb") != std::string::npos) {
613 charge *= 1.e-06 / 3600.;
615 workspace->mutableRun().setProtonCharge(charge);
616 }
catch (::NeXus::Exception &) {
620 workspace->mutableRun().getProtonCharge();
627 if (!allow_list.empty()) {
628 for (
const auto &allow : allow_list) {
629 if (!
workspace->run().hasProperty(allow)) {
630 g_log.
notice() <<
"could not load entry '" << allow <<
"' that was specified in the allow list"
642 <<
"\" contains invalid values, click \"Show Sample Logs\" "
648 <<
" contain invalid values, click \"Show Sample Logs\" for "
661 file.openGroup(
"Veto_pulse",
"NXgroup");
662 }
catch (::NeXus::Exception &) {
666 file.openData(
"veto_pulse_time");
669 std::string start_time;
670 file.getAttr(
"start_time", start_time);
671 DateAndTime start(start_time);
674 std::vector<double> time_double;
675 file.getData(time_double);
678 std::vector<double> values(time_double.size(), 0.0);
680 tsp->
create(start, time_double, values);
684 workspace->mutableRun().addProperty(tsp);
693 file.openGroup(
"periods",
"IXperiods");
694 file.openData(
"number");
695 file.getData(&
value);
698 }
catch (::NeXus::Exception &) {
704 const std::string nPeriodsLabel =
"nperiods";
712 file.openGroup(
"periods",
"IXperiods");
715 file.openData(
"number");
716 int numberOfPeriods = 0;
717 file.getData(&numberOfPeriods);
721 std::vector<double> protonChargeByPeriod(numberOfPeriods);
722 file.openData(
"proton_charge");
723 file.getDataCoerce(protonChargeByPeriod);
727 const std::string protonChargeByPeriodLabel =
"proton_charge_by_period";
732 }
catch (::NeXus::Exception &) {
733 this->
g_log.
debug(
"Cannot read periods information from the nexus file. "
734 "This group may be absent.");
736 }
catch (std::runtime_error &) {
737 this->
g_log.
debug(
"Cannot read periods information from the nexus file. "
738 "This group may be absent.");
754 const std::string &entry_class,
const std::shared_ptr<API::MatrixWorkspace> &
workspace,
755 const std::vector<std::string> &allow_list,
756 const std::vector<std::string> &block_list)
const {
758 const std::map<std::string, std::set<std::string>> &allEntries =
getFileInfo()->getAllEntries();
760 auto lf_LoadByLogClass = [&](
const std::string &logClass,
const bool isNxLog) {
761 auto itLogClass = allEntries.find(logClass);
762 if (itLogClass == allEntries.end()) {
765 const std::set<std::string> &logsSet = itLogClass->second;
766 auto itPrefixBegin = logsSet.lower_bound(absolute_entry_name);
768 if (allow_list.empty()) {
769 for (
auto it = itPrefixBegin;
770 it != logsSet.end() && it->compare(0, absolute_entry_name.size(), absolute_entry_name) == 0; ++it) {
772 if (std::count(it->begin(), it->end(),
'/') == 3) {
773 if (!block_list.empty()) {
775 for (
const auto &block : block_list) {
776 Poco::Glob glob(block);
777 if (glob.match((*it).substr((*it).find_last_of(
"/") + 1))) {
795 for (
const auto &allow : allow_list) {
796 itPrefixBegin = logsSet.find(absolute_entry_name +
"/" + allow);
797 if (itPrefixBegin == logsSet.end()) {
802 auto it = itPrefixBegin;
804 if (std::count(it->begin(), it->end(),
'/') == 3) {
815 const std::string entry_name = absolute_entry_name.substr(absolute_entry_name.find_last_of(
"/") + 1);
816 file.openGroup(entry_name, entry_class);
817 lf_LoadByLogClass(
"NXlog",
true);
818 lf_LoadByLogClass(
"NXpositioner",
true);
819 lf_LoadByLogClass(
"IXseblock",
false);
834 const std::string &entry_class,
835 const std::shared_ptr<API::MatrixWorkspace> &
workspace)
const {
837 const std::string entry_name = absolute_entry_name.substr(absolute_entry_name.find_last_of(
"/") + 1);
838 g_log.
debug() <<
"processing " << entry_name <<
":" << entry_class <<
"\n";
839 file.openGroup(entry_name, entry_class);
842 const std::string timeEntry = absolute_entry_name +
"/time";
843 const std::string valueEntry = absolute_entry_name +
"/value";
844 bool foundValue =
false;
845 bool foundTime =
false;
847 const std::map<std::string, std::set<std::string>> &allEntries =
getFileInfo()->getAllEntries();
849 for (
auto it = allEntries.rbegin(); it != allEntries.rend(); ++it) {
850 const std::set<std::string> &entriesSet = it->second;
851 if (entriesSet.count(timeEntry) == 1) {
854 if (entriesSet.count(valueEntry) == 1) {
857 if (foundTime && foundValue) {
862 if (!foundTime || !foundValue) {
863 g_log.
warning() <<
"Invalid NXlog entry " << entry_name <<
" found. Did not contain 'value' and 'time'.\n";
869 bool overwritelogs = this->
getProperty(
"OverwriteLogs");
871 if (overwritelogs || !(
workspace->run().hasProperty(entry_name))) {
872 auto logValue = createTimeSeries(file, entry_name,
freqStart,
g_log);
873 auto validityLogValue = createTimeSeriesValidityFilter(file, *logValue,
g_log);
874 if (validityLogValue) {
875 appendEndTimeLog(validityLogValue.get(),
workspace->run());
876 workspace->mutableRun().addProperty(std::move(validityLogValue), overwritelogs);
879 appendEndTimeLog(logValue.get(),
workspace->run());
880 workspace->mutableRun().addProperty(std::move(logValue), overwritelogs);
882 }
catch (::NeXus::Exception &e) {
883 g_log.
warning() <<
"NXlog entry " << entry_name <<
" gave an error when loading:'" << e.what() <<
"'.\n";
884 }
catch (std::invalid_argument &e) {
885 g_log.
warning() <<
"NXlog entry " << entry_name <<
" gave an error when loading:'" << e.what() <<
"'.\n";
892 const std::shared_ptr<API::MatrixWorkspace> &
workspace)
const {
894 const std::string entry_name = absolute_entry_name.substr(absolute_entry_name.find_last_of(
"/") + 1);
896 file.openGroup(entry_name,
"IXseblock");
897 std::string propName = entry_name;
898 if (
workspace->run().hasProperty(propName)) {
899 propName =
"selog_" + propName;
905 const std::string valueEntry = absolute_entry_name +
"/value";
906 const std::string valueLogEntry = absolute_entry_name +
"/value_log";
907 bool foundValue =
false;
908 bool foundValueLog =
false;
910 const std::map<std::string, std::set<std::string>> &allEntries =
getFileInfo()->getAllEntries();
912 for (
auto it = allEntries.rbegin(); it != allEntries.rend(); ++it) {
913 const std::set<std::string> &entriesSet = it->second;
914 if (entriesSet.count(valueEntry) == 1) {
917 if (entriesSet.count(valueLogEntry) == 1) {
918 foundValueLog =
true;
924 std::unique_ptr<Kernel::Property> logValue;
928 file.openGroup(
"value_log",
"NXlog");
929 }
catch (::NeXus::Exception &) {
935 auto validityLogValue = createTimeSeriesValidityFilter(file, *logValue,
g_log);
936 if (validityLogValue) {
937 appendEndTimeLog(validityLogValue.get(),
workspace->run());
938 workspace->mutableRun().addProperty(std::move(validityLogValue));
941 appendEndTimeLog(logValue.get(),
workspace->run());
944 }
catch (std::exception &e) {
945 g_log.
warning() <<
"IXseblock entry '" << entry_name <<
"' gave an error when loading "
946 <<
"a time series:'" << e.what() <<
"'. Skipping entry\n";
951 }
else if (foundValue) {
955 file.openData(
"value");
956 ::NeXus::Info info = file.getInfo();
957 if (info.type == ::NeXus::FLOAT32) {
958 boost::scoped_array<float>
value(
new float[info.dims[0]]);
959 file.getData(
value.get());
961 logValue = std::make_unique<Kernel::PropertyWithValue<double>>(propName,
static_cast<double>(
value[0]),
true);
966 }
catch (::NeXus::Exception &e) {
967 g_log.
warning() <<
"IXseblock entry " << entry_name <<
" gave an error when loading "
968 <<
"a single value:'" << e.what() <<
"'.\n";
974 g_log.
warning() <<
"IXseblock entry " << entry_name <<
" cannot be read, skipping entry.\n";
978 workspace->mutableRun().addProperty(std::move(logValue));
#define DECLARE_ALGORITHM(classname)
double value
The value of the point.
IPeaksWorkspace_sptr workspace
void declareProperty(std::unique_ptr< Kernel::Property > p, const std::string &doc="") override
Add a property to the list of managed properties.
std::string getPropertyValue(const std::string &name) const override
Get the value of a property as a string.
TypedValue getProperty(const std::string &name) const override
Get the value of a property.
Kernel::Logger & getLogger() const
Returns a reference to the logger.
@ Load
allowed here which will be passed to the algorithm
bool hasProperty(const std::string &name) const
Does the property exist on the object.
void addProperty(Kernel::Property *prop, bool overwrite=false)
Add data to the object in the form of a property.
static std::string getInvalidValuesFilterLogName(const std::string &logName)
Gets the correct log name for the matching invalid values log for a given log name.
virtual const std::shared_ptr< Mantid::Kernel::NexusHDF5Descriptor > getFileInfo() const noexcept
Required to pass m_fileInfo to static functions Keeping it shared_ptr to match setFileInfo signature ...
This class stores information regarding an experimental run as a series of log entries.
A property class for workspaces.
std::vector< std::string > m_logsWithInvalidValues
void init() override
Overwrites Algorithm method.
void loadSELog(::NeXus::File &file, const std::string &absolute_entry_name, const std::shared_ptr< API::MatrixWorkspace > &workspace) const
Load an IXseblock entry.
LoadNexusLogs()
Default constructor.
void loadLogs(::NeXus::File &file, const std::string &absolute_entry_name, const std::string &entry_class, const std::shared_ptr< API::MatrixWorkspace > &workspace, const std::vector< std::string > &allow_list, const std::vector< std::string > &block_list) const
Load log data from a group.
void execLoader() override
Overwrites Algorithm method.
std::string freqStart
Use frequency start for Monitor19 and Special1_19 logs with "No Time" for SNAP.
void loadVetoPulses(::NeXus::File &file, const std::shared_ptr< API::MatrixWorkspace > &workspace) const
Try to load the "Veto_pulse" field in DASLogs and convert it to a sample log.
void loadNXLog(::NeXus::File &file, const std::string &absolute_entry_name, const std::string &entry_class, const std::shared_ptr< API::MatrixWorkspace > &workspace) const
Load an NXlog entry.
void loadNPeriods(::NeXus::File &file, const std::shared_ptr< API::MatrixWorkspace > &workspace) const
static std::string getEntryName(const std::string &filename)
Support for a property that holds an array of values.
Exception for when an item is not found in a collection.
void debug(const std::string &msg)
Logs at debug level.
void notice(const std::string &msg)
Logs at notice level.
void warning(const std::string &msg)
Logs at warning level.
The concrete, templated class for properties.
virtual void setUnits(const std::string &unit)
Sets the units of the property, as a string.
A specialised Property class for holding a series of time-value pairs.
std::vector< TYPE > valuesAsVector() const
Return the time series's values (unfiltered) as a vector<TYPE>
void create(const Types::Core::DateAndTime &start_time, const std::vector< double > &time_sec, const std::vector< TYPE > &new_values)
Clears and creates a TimeSeriesProperty from these parameters.
std::vector< Types::Core::DateAndTime > timesAsVector() const override
Return the time series's times as a vector<DateAndTime>
Kernel::Logger g_log("ExperimentInfo")
static logger object
std::shared_ptr< MatrixWorkspace > MatrixWorkspace_sptr
shared pointer to the matrix workspace base class
@ InOut
Both an input & output workspace.
@ Input
An input workspace.