LoadNexusLogs.cpp

Go to the documentation of this file.

// Mantid Repository : https://github.com/mantidproject/mantid
//
// Copyright © 2018 ISIS Rutherford Appleton Laboratory UKRI,
//   NScD Oak Ridge National Laboratory, European Spallation Source,
//   Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
// SPDX - License - Identifier: GPL - 3.0 +
#include "MantidDataHandling/LoadNexusLogs.h"
#include "MantidAPI/FileProperty.h"
#include "MantidAPI/LogManager.h"
#include "MantidAPI/Run.h"
#include "MantidDataHandling/LoadTOFRawNexus.h"
#include "MantidKernel/ArrayProperty.h"
#include "MantidKernel/TimeSeriesProperty.h"
#include "MantidNexus/NexusException.h"
 
#include <Poco/DateTimeFormat.h>
#include <Poco/DateTimeFormatter.h>
#include <Poco/DateTimeParser.h>
#include <Poco/Glob.h>
#include <boost/scoped_array.hpp>
 
#include <algorithm>
#include <locale>
 
namespace Mantid::DataHandling {
// Register the algorithm into the algorithm factory
DECLARE_ALGORITHM(LoadNexusLogs)
 
using namespace Kernel;
using API::FileProperty;
using API::MatrixWorkspace;
using API::MatrixWorkspace_sptr;
using API::WorkspaceProperty;
using std::size_t;
using Types::Core::DateAndTime;
 
// Anonymous namespace
namespace {
bool loadAndApplyMeasurementInfo(Nexus::File *const file, API::MatrixWorkspace &workspace) {
 
  bool successfullyApplied = false;
  try {
    file->openGroup("measurement", "NXcollection");
 
    // If we can open the measurement group. We assume that the following will
    // be available.
    file->openData("id");
    workspace.mutableRun().addLogData(
        new Mantid::Kernel::PropertyWithValue<std::string>("measurement_id", file->getStrData()));
    file->closeData();
    file->openData("label");
    workspace.mutableRun().addLogData(
        new Mantid::Kernel::PropertyWithValue<std::string>("measurement_label", file->getStrData()));
    file->closeData();
    file->openData("subid");
    workspace.mutableRun().addLogData(
        new Mantid::Kernel::PropertyWithValue<std::string>("measurement_subid", file->getStrData()));
    file->closeData();
    file->openData("type");
    workspace.mutableRun().addLogData(
        new Mantid::Kernel::PropertyWithValue<std::string>("measurement_type", file->getStrData()));
    file->closeData();
    file->closeGroup();
    successfullyApplied = true;
  } catch (Nexus::Exception const &) {
    successfullyApplied = false;
  }
  return successfullyApplied;
}
 
bool loadAndApplyRunTitle(Nexus::File *const file, API::MatrixWorkspace &workspace) {
 
  bool successfullyApplied = false;
  try {
    file->openData("title");
    workspace.mutableRun().addLogData(
        new Mantid::Kernel::PropertyWithValue<std::string>("run_title", file->getStrData()));
    file->closeData();
    successfullyApplied = true;
  } catch (Nexus::Exception const &) {
    successfullyApplied = false;
  }
  return successfullyApplied;
}
 
bool isControlValue(const char &c, const std::string &propName, Kernel::Logger &log) {
  // Have to check it falls within range accepted by c style check
  if (c <= -1) {
    log.warning("Found an invalid character in property " + propName);
    // Pretend this is a control value so it is sanitized
    return true;
  } else {
    // Use default global c++ locale within this program
    std::locale locale{};
    // Use c++ style call so we don't need to cast from int to bool
    return std::iscntrl(c, locale);
  }
}
 
std::unique_ptr<Kernel::Property> createTimeSeries(Nexus::File &file, const std::string &propName,
                                                   const std::string &freqStart, Kernel::Logger &log) {
  file.openData("time");
  //----- Start time is an ISO8601 string date and time. ------
  std::string start;
  try {
    file.getAttr("start", start);
  } catch (Nexus::Exception const &) {
    // Some logs have "offset" instead of start
    try {
      file.getAttr("offset", start);
    } catch (Nexus::Exception const &) {
      log.warning() << "Log entry has no start time indicated.\n";
      file.closeData();
      throw;
    }
  }
  if (start == "No Time") {
    start = freqStart;
  }
 
  // Convert to date and time
  Types::Core::DateAndTime start_time = Types::Core::DateAndTime(start);
  std::string time_units;
  file.getAttr("units", time_units);
  if (time_units.compare("second") < 0 && time_units != "s" &&
      time_units != "minutes") // Can be s/second/seconds/minutes
  {
    file.closeData();
    throw Nexus::Exception("Unsupported time unit '" + time_units + "'");
  }
  //--- Load the seconds into a double array ---
  std::vector<double> time_double;
  try {
    file.getDataCoerce(time_double);
  } catch (Nexus::Exception const &e) {
    log.warning() << "Log entry's time field could not be loaded: '" << e.what() << "'.\n";
    file.closeData();
    throw;
  }
  file.closeData(); // Close time data
  log.debug() << "   done reading \"time\" array\n";
 
  // Convert to seconds if needed
  if (time_units == "minutes") {
    using std::placeholders::_1;
    std::transform(time_double.begin(), time_double.end(), time_double.begin(),
                   std::bind(std::multiplies<double>(), _1, 60.0));
  }
 
  // Now the values: Could be a string, int or double
  file.openData("value");
  // Get the units of the property
  std::string value_units;
  try {
    file.getAttr("units", value_units);
  } catch (Nexus::Exception const &) {
    // Ignore missing units field.
    value_units = "";
  }
 
  // Now the actual data
  Nexus::Info info = file.getInfo();
  if (file.isDataInt()) { // Int type
    // Check the size
    if (size_t(info.dims[0]) != time_double.size()) {
      file.closeData();
      throw Nexus::Exception("Invalid value entry for time series");
    }
    std::vector<int> values;
    try {
      file.getDataCoerce(values);
      file.closeData();
    } catch (Nexus::Exception const &) {
      file.closeData();
      throw;
    }
    // Make an int TSP
    auto tsp = std::make_unique<TimeSeriesProperty<int>>(propName);
    tsp->create(start_time, time_double, values);
    tsp->setUnits(value_units);
    log.debug() << "   done reading \"value\" array\n";
    return tsp;
  } else if (info.type == NXnumtype::CHAR) {
    std::string values;
    // the length of each string is normally stored in the second dimension (1),
    // but some older files (wrongly) put this in the first dimension (0) if the rank is 1
    const int64_t item_length = (info.dims.size() > 1 ? info.dims[1] : info.dims[0]);
    try {
      const std::size_t total_length =
          std::accumulate(info.dims.cbegin(), info.dims.cend(), std::size_t(1),
                          [](std::size_t acc, int64_t x) { return acc * static_cast<std::size_t>(x); });
      boost::scoped_array<char> val_array(new char[total_length + 1]);
      file.getData(val_array.get());
      file.closeData();
      values = std::string(val_array.get(), total_length);
    } catch (Nexus::Exception const &) {
      file.closeData();
      throw;
    }
    // The string may contain non-printable (i.e. control) characters, replace these
    std::replace_if(values.begin(), values.end(), [&](const char &c) { return isControlValue(c, propName, log); }, ' ');
    auto tsp = std::make_unique<TimeSeriesProperty<std::string>>(propName);
    std::vector<DateAndTime> times;
    DateAndTime::createVector(start_time, time_double, times);
    const size_t ntimes = times.size();
    for (size_t i = 0; i < ntimes; ++i) {
      std::string value_i = std::string(values.data() + i * item_length, item_length);
      tsp->addValue(times[i], value_i);
    }
    tsp->setUnits(value_units);
    log.debug() << "   done reading \"value\" array\n";
    return tsp;
  } else if (info.type == NXnumtype::FLOAT32 || info.type == NXnumtype::FLOAT64) {
    std::vector<double> values;
    try {
      file.getDataCoerce(values);
      file.closeData();
    } catch (Nexus::Exception const &) {
      file.closeData();
      throw;
    }
    auto tsp = std::make_unique<TimeSeriesProperty<double>>(propName);
    tsp->create(start_time, time_double, values);
    tsp->setUnits(value_units);
    log.debug() << "   done reading \"value\" array\n";
    return tsp;
  } else {
    throw Nexus::Exception("Invalid value type for time series. Only int, double or strings are "
                           "supported");
  }
}
 
std::unique_ptr<Kernel::Property> createTimeSeriesValidityFilter(Nexus::File &file, const Kernel::Property &prop,
                                                                 Kernel::Logger &log) {
  const auto tsProp = dynamic_cast<const Kernel::ITimeSeriesProperty *>(&prop);
  const auto times = tsProp->timesAsVector();
  std::vector<int> values;
  std::vector<bool> boolValues;
  // Now the validity of the values
  // this should be a match int array to the data values (or times)
  // If not present assume all data is valid
  if (file.hasData("value_valid")) {
    file.openData("value_valid");
    try {
      // Now the validity data
      Nexus::Info info = file.getInfo();
      // Check the size
      if (size_t(info.dims[0]) != times.size()) {
        throw Nexus::Exception("Invalid value entry for validity data");
      }
      if (file.isDataInt()) // Int type
      {
        try {
          file.getDataCoerce(values);
          file.closeData();
        } catch (Nexus::Exception const &) {
          throw;
        }
      } else {
        throw Nexus::Exception("Invalid value type for validity data. Only int is supported");
      }
    } catch (std::exception const &ex) {
      std::string error_msg = ex.what();
      log.warning() << error_msg << "\n";
      file.closeData();
      // no data found
      return std::unique_ptr<Kernel::Property>(nullptr);
    }
  }
 
  bool invalidDataFound = false;
  boolValues.reserve(values.size());
  // convert the integer values to boolean with 0=invalid data
  for (size_t i = 0; i < values.size(); i++) {
    bool isInvalidData = (values[i] == 0);
    boolValues.emplace_back(!isInvalidData);
    if (isInvalidData && !invalidDataFound) {
      invalidDataFound = true;
    }
  }
  if (invalidDataFound) {
    // Prepare the TimeSeriesProperty<bool>
    // It's name will be the name of the property plus suffix "_invalid_values"
    const auto tspName = API::LogManager::getInvalidValuesFilterLogName(prop.name());
    auto tsp = std::make_unique<TimeSeriesProperty<bool>>(tspName);
    tsp->create(times, boolValues);
    log.debug() << "   done reading \"value_valid\" array\n";
 
    return tsp;
  } else {
    // no data found
    return std::unique_ptr<Kernel::Property>(nullptr);
  }
}
 
void appendEndTimeLog(Kernel::Property *prop, const API::Run &run) {
  // do not modify proton charge
  if (prop->name() == "proton_charge")
    return;
 
  try {
    auto tsLog = dynamic_cast<TimeSeriesProperty<double> *>(prop);
    const auto endTime = run.endTime();
 
    // First check if it is valid to append a log entry
    if (!tsLog || tsLog->size() == 0 || endTime <= tsLog->lastTime())
      return;
 
    tsLog->addValue(endTime, tsLog->lastValue());
  } catch (const Exception::NotFoundError &) {
    // pass
  } catch (const std::runtime_error &) {
    // pass
  }
}
 
void readStartAndEndTime(Nexus::File &file, API::Run &run) {
  try {
    // Read the start and end time strings
    file.openData("start_time");
    Types::Core::DateAndTime start(file.getStrData());
    file.closeData();
    file.openData("end_time");
    Types::Core::DateAndTime end(file.getStrData());
    file.closeData();
    run.setStartAndEndTime(start, end);
  } catch (Nexus::Exception const &) {
  }
}
 
} // End of anonymous namespace
 
LoadNexusLogs::LoadNexusLogs() = default;
 
void LoadNexusLogs::init() {
  declareProperty(std::make_unique<WorkspaceProperty<MatrixWorkspace>>("Workspace", "Anonymous", Direction::InOut),
                  "The name of the workspace that will be filled with the logs.");
  const std::vector<std::string> exts{".nxs", ".n*"};
  declareProperty(std::make_unique<FileProperty>("Filename", "", FileProperty::Load, exts),
                  "Path to the .nxs file to load. Can be an EventNeXus or a "
                  "histogrammed NeXus.");
  declareProperty(std::make_unique<PropertyWithValue<bool>>("OverwriteLogs", true, Direction::Input),
                  "If true then some existing logs will be overwritten, if false they will "
                  "not.");
  declareProperty(std::make_unique<PropertyWithValue<std::string>>("NXentryName", "", Direction::Input),
                  "Entry in the nexus file from which to read the logs");
  declareProperty(std::make_unique<PropertyWithValue<std::vector<std::string>>>("AllowList", std::vector<std::string>(),
                                                                                Direction::Input),
                  "If specified, only these logs will be loaded from the file (each "
                  "separated by a comma).");
  declareProperty(std::make_unique<PropertyWithValue<std::vector<std::string>>>("BlockList", std::vector<std::string>(),
                                                                                Direction::Input),
                  "If specified, logs matching one of the patterns will NOT be loaded from the file (each "
                  "separated by a comma).");
}
 
void LoadNexusLogs::exec() {
  std::string filename = getPropertyValue("Filename");
  MatrixWorkspace_sptr workspace = getProperty("Workspace");
 
  std::string entry_name = getPropertyValue("NXentryName");
 
  std::vector<std::string> allow_list = getProperty("AllowList");
  std::vector<std::string> block_list = getProperty("BlockList");
 
  // Find the entry name to use (normally "entry" for SNS, "raw_data_1" for
  // ISIS) if entry name is empty
  if (entry_name.empty()) {
    entry_name = LoadTOFRawNexus::getEntryName(filename);
  }
  Nexus::File file(filename);
  // Find the root entry
  try {
    file.openGroup(entry_name, "NXentry");
  } catch (Nexus::Exception const &) {
    throw std::invalid_argument("Unknown NeXus file format found in file '" + filename + "', or '" + entry_name +
                                "' is not a valid NXentry");
  }
 
  try {
    file.openAddress("DASlogs");
    try {
      file.openGroup("frequency", "NXlog");
      try {
        file.openData("time");
 
        //----- Start time is an ISO8601 string date and time. ------
        try {
          file.getAttr("start", freqStart);
 
        } catch (Nexus::Exception const &) {
          // Some logs have "offset" instead of start
          try {
            file.getAttr("offset", freqStart);
          } catch (Nexus::Exception const &) {
            g_log.warning() << "Log entry has no start time indicated.\n";
            file.closeData();
            throw;
          }
        }
        file.closeData();
      } catch (Nexus::Exception const &) {
        // No time. This is not an SNS SNAP file
      }
      file.closeGroup();
    } catch (Nexus::Exception const &) {
      // No time. This is not an SNS frequency group
    }
    file.closeGroup();
  } catch (Nexus::Exception const &) {
    // No time. This is not an SNS group
  }
 
  readStartAndEndTime(file, workspace->mutableRun());
 
  if (!allow_list.empty() && !block_list.empty()) {
    throw std::runtime_error("BlockList and AllowList are mutually exclusive! "
                             "Please only enter values for one of these fields.");
  }
 
  auto lf_LoadLogsByClass = [&](const std::string &group_class, const bool isLog) {
    const std::set<std::string> &entries = file.getEntriesByClass(group_class);
    // still a linear search
    for (const std::string &entry : entries) {
      // match for 2nd level entry /a/b
      if (std::count(entry.begin(), entry.end(), '/') == 2) {
        if (isLog) {
          loadLogs(file, entry, group_class, workspace, allow_list, block_list);
        } else {
          loadNPeriods(file, workspace);
        }
      }
    }
  };
  lf_LoadLogsByClass("IXselog", true);
  lf_LoadLogsByClass("IXrunlog", true);
  lf_LoadLogsByClass("IXperiods", false);
 
  auto lf_LoadLogsByName = [&](const std::string &group_name) {
    std::string const absoluteGroupName = "/" + entry_name + "/" + group_name;
    if (file.hasAddress(absoluteGroupName)) {
      std::string const &group_class = file.classForEntry(absoluteGroupName);
      loadLogs(file, absoluteGroupName, group_class, workspace, allow_list, block_list);
    }
  };
 
  lf_LoadLogsByName("DASlogs");
  lf_LoadLogsByName("framelog");
 
  // If there's measurement information, load that info as logs.
  loadAndApplyMeasurementInfo(&file, *workspace);
  // If there's title information, load that info as logs.
  loadAndApplyRunTitle(&file, *workspace);
 
  // Freddie Akeroyd 12/10/2011
  // current ISIS implementation contains an additional indirection between
  // collected frames via an
  // "event_frame_number" array in NXevent_data (which eliminates frames with no
  // events).
  // the proton_log is for all frames and so is longer than the event_index
  // array, so we need to
  // filter the proton_charge log based on event_frame_number
  // This difference will be removed in future for compatibility with SNS, but
  // the code below will allow current SANS2D files to load
  if (workspace->mutableRun().hasProperty("proton_log")) {
    std::vector<int> event_frame_number;
    this->getLogger().notice() << "Using old ISIS proton_log and event_frame_number indirection...\n";
    try {
      // Find the bank/name corresponding to the first event data entry, i.e.
      // one with type NXevent_data.
      file.openAddress("/" + entry_name);
      std::set<std::string> const &events = file.getEntriesByClass("NXevent_data");
      for (std::string const &event : events) {
        std::string const eventEntry = event.substr(event.find_last_of("/") + 1);
 
        this->getLogger().debug() << "Opening"
                                  << " /" + entry_name + "/" + eventEntry + "/event_frame_number"
                                  << " to find the event_frame_number\n";
        file.openAddress("/" + entry_name + "/" + eventEntry + "/event_frame_number");
        file.getData(event_frame_number);
      }
    } catch (Nexus::Exception const &) {
      this->getLogger().warning() << "Unable to load event_frame_number - "
                                     "filtering events by time will not work \n";
    }
    file.openAddress("/" + entry_name);
    if (!event_frame_number.empty()) // ISIS indirection - see above comments
    {
      Kernel::TimeSeriesProperty<double> const *plog =
          dynamic_cast<Kernel::TimeSeriesProperty<double> *>(workspace->mutableRun().getProperty("proton_log"));
      if (!plog)
        throw std::runtime_error("Could not cast (interpret) proton_log as a time "
                                 "series property. Cannot continue.");
      Kernel::TimeSeriesProperty<double> *pcharge = new Kernel::TimeSeriesProperty<double>("proton_charge");
      std::vector<double> pval;
      std::vector<Mantid::Types::Core::DateAndTime> ptime;
      pval.reserve(event_frame_number.size());
      ptime.reserve(event_frame_number.size());
      std::vector<Mantid::Types::Core::DateAndTime> plogt = plog->timesAsVector();
      std::vector<double> plogv = plog->valuesAsVector();
      for (auto number : event_frame_number) {
        ptime.emplace_back(plogt[number]);
        pval.emplace_back(plogv[number]);
      }
      pcharge->create(ptime, pval);
      pcharge->setUnits("uAh");
      workspace->mutableRun().addProperty(pcharge, true);
    }
  }
 
  if (!workspace->run().hasProperty(workspace->run().getProtonChargeLogName())) {
    try {
      // Try pulling proton charge from the main proton_charge entry
      file.openData("proton_charge");
      std::vector<double> values;
      file.getDataCoerce(values);
      std::string units;
      file.getAttr("units", units);
      double charge = values.front();
      if (units.find("picoCoulomb") != std::string::npos) {
        charge *= 1.e-06 / 3600.;
      }
      workspace->mutableRun().setProtonCharge(charge);
    } catch (Nexus::Exception const &) {
      // Try and integrate the proton logs
      try {
        // Use the DAS logs to integrate the proton charge (if any).
        workspace->mutableRun().getProtonCharge();
      } catch (Exception::NotFoundError &) {
        // Ignore not found property error.
      }
    }
  }
  // For period data mark proton charge log value as unfiltered to enable subsequent filtering by period.
  if (workspace->run().hasProperty("proton_charge_by_period")) {
    Kernel::PropertyWithValue<int> *pChargeUnfiltered =
        new Kernel::PropertyWithValue<int>(workspace->run().getProtonChargeUnfilteredName(), 1);
    workspace->mutableRun().addProperty(pChargeUnfiltered, true);
  }
 
  if (!allow_list.empty()) {
    for (const auto &allow : allow_list) {
      if (!workspace->run().hasProperty(allow)) {
        g_log.notice() << "could not load entry '" << allow << "' that was specified in the allow list"
                       << "\n";
      }
    }
  }
 
  // Close the file
  file.close();
 
  if (m_logsWithInvalidValues.size() > 0) {
    if (m_logsWithInvalidValues.size() == 1) {
      g_log.warning() << "Sample Log \"" << m_logsWithInvalidValues[0]
                      << "\" contains invalid values, click \"Show Sample Logs\" "
                         "for details.\n";
    }
    auto other_string = (m_logsWithInvalidValues.size() < 2) ? " other " : " others";
    g_log.warning() << "Sample Log \"" << m_logsWithInvalidValues[0] << "\" and " << m_logsWithInvalidValues.size() - 1
                    << other_string
                    << " contain invalid values, click \"Show Sample Logs\" for "
                       "details.\n";
  }
}
 
void LoadNexusLogs::loadVetoPulses(Nexus::File &file, const std::shared_ptr<API::MatrixWorkspace> &workspace) const {
  try {
    file.openGroup("Veto_pulse", "NXgroup");
  } catch (Nexus::Exception const &) {
    // No group. This is common in older files
    return;
  }
  file.openData("veto_pulse_time");
 
  // Load the start date/time as ISO8601 string.
  std::string start_time;
  file.getAttr("start_time", start_time);
  DateAndTime start(start_time);
 
  // Read the offsets
  std::vector<double> time_double;
  file.getData(time_double);
 
  // Fake values with zeroes.
  std::vector<double> values(time_double.size(), 0.0);
  TimeSeriesProperty<double> *tsp = new TimeSeriesProperty<double>("veto_pulse_time");
  tsp->create(start, time_double, values);
  tsp->setUnits("");
 
  // Add the log
  workspace->mutableRun().addProperty(tsp);
 
  file.closeData();
  file.closeGroup();
}
 
void LoadNexusLogs::loadNPeriods(Nexus::File &file, const std::shared_ptr<API::MatrixWorkspace> &workspace) const {
  int value = 1; // Default to 1-period unless
  try {
    file.openGroup("periods", "IXperiods");
    file.openData("number");
    file.getData(&value);
    file.closeData();
    file.closeGroup();
  } catch (Nexus::Exception const &) {
    // Likely missing IXperiods.
    return;
  }
 
  API::Run &run = workspace->mutableRun();
  const std::string nPeriodsLabel = "nperiods";
  if (!run.hasProperty(nPeriodsLabel)) {
    run.addProperty(new PropertyWithValue<int>(nPeriodsLabel, value));
  }
 
  // For ISIS Nexus only, fabricate an additional log containing an array of
  // proton charge information from the periods group.
  try {
    file.openGroup("periods", "IXperiods");
 
    // Get the number of periods again
    file.openData("number");
    int numberOfPeriods = 0;
    file.getData(&numberOfPeriods);
    file.closeData();
 
    // Get the proton charge vector
    std::vector<double> protonChargeByPeriod(numberOfPeriods);
    file.openData("proton_charge");
    file.getDataCoerce(protonChargeByPeriod);
    file.closeData();
 
    // Add the proton charge vector
    const std::string protonChargeByPeriodLabel = "proton_charge_by_period";
    if (!run.hasProperty(protonChargeByPeriodLabel)) {
      run.addProperty(new ArrayProperty<double>(protonChargeByPeriodLabel, std::move(protonChargeByPeriod)));
    }
    file.closeGroup();
  } catch (Nexus::Exception const &) {
    this->g_log.debug("Cannot read periods information from the nexus file. "
                      "This group may be absent.");
    file.closeGroup();
  } catch (std::runtime_error &) {
    this->g_log.debug("Cannot read periods information from the nexus file. "
                      "This group may be absent.");
    file.closeGroup();
  }
}
 
void LoadNexusLogs::loadLogs(Nexus::File &file, const std::string &absolute_entry_name, const std::string &entry_class,
                             const std::shared_ptr<API::MatrixWorkspace> &workspace,
                             const std::vector<std::string> &allow_list,
                             const std::vector<std::string> &block_list) const {
 
  auto lf_LoadByLogClass = [&](std::string const &logClass, const bool isNxLog) {
    std::set<std::string> const &logsSet = file.getEntriesByClass(logClass);
    if (logsSet.empty()) {
      return;
    }
    auto itPrefixBegin = logsSet.lower_bound(absolute_entry_name);
 
    if (allow_list.empty()) {
      // convert the blocklist into a bunch of objects to handle globbing
      const bool has_block_list = (!block_list.empty());
      std::vector<std::unique_ptr<Poco::Glob>> globblock_list;
      if (has_block_list) {
        std::transform(block_list.cbegin(), block_list.cend(), std::back_inserter(globblock_list),
                       [](const auto &block) { return std::make_unique<Poco::Glob>(block); });
      }
 
      for (auto it = itPrefixBegin;
           it != logsSet.end() && it->compare(0, absolute_entry_name.size(), absolute_entry_name) == 0; ++it) {
        // must be third level entry
        if (std::count(it->begin(), it->end(), '/') == 3) {
          if (has_block_list) {
            bool skip = false;
            for (auto &block : globblock_list) {
              if (block->match((*it).substr((*it).find_last_of("/") + 1))) {
                skip = true;
                break; // from the loop of block items
              }
            }
            if (skip) {
              continue; // go to next log
            }
          } // end of looping over block_list
 
          if (isNxLog) {
            loadNXLog(file, *it, logClass, workspace);
          } else {
            loadSELog(file, *it, workspace);
          }
        }
      }
    } else {
      for (const auto &allow : allow_list) {
        itPrefixBegin = logsSet.find(absolute_entry_name + "/" + allow);
        if (itPrefixBegin == logsSet.end()) {
          // don't print warning yet since it might be found in another log
          // class
          continue;
        }
        auto it = itPrefixBegin;
        // must be third level entry
        if (std::count(it->begin(), it->end(), '/') == 3) {
          if (isNxLog) {
            loadNXLog(file, *it, logClass, workspace);
          } else {
            loadSELog(file, *it, workspace);
          }
        }
      }
    }
  };
 
  const std::string entry_name = absolute_entry_name.substr(absolute_entry_name.find_last_of("/") + 1);
  file.openGroup(entry_name, entry_class);
  lf_LoadByLogClass("NXlog", true);
  lf_LoadByLogClass("NXpositioner", true);
  lf_LoadByLogClass("IXseblock", false);
  loadVetoPulses(file, workspace);
 
  file.closeGroup();
}
 
void LoadNexusLogs::loadNXLog(Nexus::File &file, const std::string &absolute_entry_name, const std::string &entry_class,
                              const std::shared_ptr<API::MatrixWorkspace> &workspace) const {
 
  const std::string entry_name = absolute_entry_name.substr(absolute_entry_name.find_last_of("/") + 1);
  g_log.debug() << "processing " << entry_name << ":" << entry_class << "\n";
  file.openGroup(entry_name, entry_class);
  // Validate the NX log class.
  // Just verify that time and value entries exist
  std::string const timeEntry = absolute_entry_name + "/time";
  std::string const valueEntry = absolute_entry_name + "/value";
  std::string const validatorEntry = absolute_entry_name + "/value_valid";
  bool const foundValue = file.hasAddress(valueEntry);
  bool const foundTime = file.hasAddress(timeEntry);
  bool const foundValidator = file.hasAddress(validatorEntry);
 
  if (!foundTime || !foundValue) {
    g_log.warning() << "Invalid NXlog entry " << entry_name << " found. Did not contain 'value' and 'time'.\n";
    file.closeGroup();
    return;
  }
 
  // whether to overwrite logs on workspace
  bool overwritelogs = this->getProperty("OverwriteLogs");
  try {
    if (overwritelogs || !(workspace->run().hasProperty(entry_name))) {
      auto logValue = createTimeSeries(file, entry_name, freqStart, g_log);
      // Create (possibly) a boolean time series, companion to time series `entry_name`
      if (foundValidator) {
        auto validityLogValue = createTimeSeriesValidityFilter(file, *logValue, g_log);
        if (validityLogValue) {
          appendEndTimeLog(validityLogValue.get(), workspace->run());
          workspace->mutableRun().addProperty(std::move(validityLogValue), overwritelogs);
          m_logsWithInvalidValues.emplace_back(entry_name);
        }
      }
      appendEndTimeLog(logValue.get(), workspace->run());
      workspace->mutableRun().addProperty(std::move(logValue), overwritelogs);
    }
  } catch (Nexus::Exception const &e) {
    g_log.warning() << "NXlog entry " << entry_name << " gave an error when loading:'" << e.what() << "'.\n";
  } catch (std::invalid_argument &e) {
    g_log.warning() << "NXlog entry " << entry_name << " gave an error when loading:'" << e.what() << "'.\n";
  }
 
  file.closeGroup();
}
 
void LoadNexusLogs::loadSELog(Nexus::File &file, const std::string &absolute_entry_name,
                              const std::shared_ptr<API::MatrixWorkspace> &workspace) const {
  // Open the entry
  const std::string entry_name = absolute_entry_name.substr(absolute_entry_name.find_last_of("/") + 1);
 
  file.openGroup(entry_name, "IXseblock");
  std::string propName = entry_name;
  if (workspace->run().hasProperty(propName)) {
    propName = "selog_" + propName;
  }
  // There are two possible entries:
  //   value_log - A time series entry. This can contain a corrupt value entry
  //   so if it does use the value one
  //   value - A single value float entry
  std::string const valueEntry = absolute_entry_name + "/value";
  std::string const valueLogEntry = absolute_entry_name + "/value_log";
  bool const foundValue = file.hasAddress(valueEntry);
  bool const foundValueLog = file.hasAddress(valueLogEntry);
 
  std::unique_ptr<Kernel::Property> logValue;
  if (foundValueLog) {
    try {
      try {
        file.openGroup("value_log", "NXlog");
      } catch (Nexus::Exception const &) {
        file.closeGroup();
        throw;
      }
 
      logValue = createTimeSeries(file, propName, freqStart, g_log);
      // Create (possibly) a boolean time series, companion to time series `logValue`.
      auto validityLogValue = createTimeSeriesValidityFilter(file, *logValue, g_log);
      if (validityLogValue) {
        appendEndTimeLog(validityLogValue.get(), workspace->run());
        workspace->mutableRun().addProperty(std::move(validityLogValue));
        m_logsWithInvalidValues.emplace_back(propName);
      }
      appendEndTimeLog(logValue.get(), workspace->run());
 
      file.closeGroup();
    } catch (std::exception &e) {
      g_log.warning() << "IXseblock entry '" << entry_name << "' gave an error when loading "
                      << "a time series:'" << e.what() << "'. Skipping entry\n";
      file.closeGroup(); // value_log
      file.closeGroup(); // entry_name
      return;
    }
  } else if (foundValue) {
    try {
      // This may have a larger dimension than 1 bit it has no time field so
      // take the first entry
      file.openData("value");
      Nexus::Info info = file.getInfo();
      if (info.type == NXnumtype::FLOAT32) {
        boost::scoped_array<float> value(new float[info.dims[0]]);
        file.getData(value.get());
        file.closeData();
        logValue = std::make_unique<Kernel::PropertyWithValue<double>>(propName, static_cast<double>(value[0]), true);
      } else {
        file.closeGroup();
        return;
      }
    } catch (Nexus::Exception const &e) {
      g_log.warning() << "IXseblock entry " << entry_name << " gave an error when loading "
                      << "a single value:'" << e.what() << "'.\n";
      file.closeData();
      file.closeGroup();
      return;
    }
  } else {
    g_log.warning() << "IXseblock entry " << entry_name << " cannot be read, skipping entry.\n";
    file.closeGroup();
    return;
  }
  workspace->mutableRun().addProperty(std::move(logValue));
  file.closeGroup();
}
 
} // namespace Mantid::DataHandling