LogManager.cpp

Go to the documentation of this file.

// Mantid Repository : https://github.com/mantidproject/mantid
//
// Copyright © 2018 ISIS Rutherford Appleton Laboratory UKRI,
//   NScD Oak Ridge National Laboratory, European Spallation Source,
//   Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
// SPDX - License - Identifier: GPL - 3.0 +
#include "MantidAPI/LogManager.h"
#include "MantidKernel/Cache.h"
#include "MantidKernel/Exception.h"
#include "MantidKernel/PropertyManager.h"
#include "MantidKernel/PropertyNexus.h"
#include "MantidKernel/TimeROI.h"
#include "MantidKernel/TimeSeriesProperty.h"
#include "MantidNexus/NexusFile.h"
 
namespace Mantid::API {
 
using namespace Kernel;
using namespace Types::Core;
 
namespace {
Logger g_log("LogManager");
 
const std::string START_TIME_NAME("start_time");
const std::string END_TIME_NAME("end_time");
 
template <typename T> bool convertSingleValue(const Property *property, double &value) {
  if (auto log = dynamic_cast<const PropertyWithValue<T> *>(property)) {
    value = static_cast<double>(*log);
    return true;
  } else {
    return false;
  }
}
 
bool convertSingleValueToDouble(const Property *property, double &value) {
  // Order these with double and int first, and less likely options later.
  // The first one to succeed short-circuits and the value is returned.
  // If all fail, returns false.
  return convertSingleValue<double>(property, value) || convertSingleValue<int32_t>(property, value) ||
         convertSingleValue<int64_t>(property, value) || convertSingleValue<uint32_t>(property, value) ||
         convertSingleValue<uint64_t>(property, value) || convertSingleValue<float>(property, value);
}
 
template <typename T>
bool convertTimeSeriesToDouble(const Property *property, double &value, const Math::StatisticType &function,
                               const Kernel::TimeROI *timeRoi = nullptr) {
  if (const auto *log = dynamic_cast<const ITimeSeriesProperty *>(property)) {
    value = log->extractStatistic(function, timeRoi);
    return true;
  } else {
    return false;
  }
}
 
template <typename T>
bool convertPropertyToDouble(const Property *property, double &value, const Math::StatisticType &function,
                             const Kernel::TimeROI *timeRoi = nullptr) {
  return convertSingleValue<T>(property, value) || convertTimeSeriesToDouble<T>(property, value, function, timeRoi);
}
 
bool convertPropertyToDouble(const Property *property, double &value, const Math::StatisticType &function,
                             const Kernel::TimeROI *timeRoi = nullptr) {
  // Order these with double and int first, and less likely options later.
  // The first one to succeed short-circuits and the value is returned.
  // If all fail, returns false.
  return convertPropertyToDouble<double>(property, value, function, timeRoi) ||
         convertPropertyToDouble<int32_t>(property, value, function, timeRoi) ||
         convertPropertyToDouble<int64_t>(property, value, function, timeRoi) ||
         convertPropertyToDouble<uint32_t>(property, value, function, timeRoi) ||
         convertPropertyToDouble<uint64_t>(property, value, function, timeRoi) ||
         convertPropertyToDouble<float>(property, value, function, timeRoi);
}
} // namespace
 
const std::string LogManager::PROTON_CHARGE_LOG_NAME = "gd_prtn_chrg";
const std::string LogManager::PROTON_CHARGE_UNFILTERED_LOG_NAME = LogManager::PROTON_CHARGE_LOG_NAME + "_unfiltered";
 
//----------------------------------------------------------------------
// Public member functions
//----------------------------------------------------------------------
 
LogManager::LogManager()
    : m_manager(std::make_unique<Kernel::PropertyManager>()), m_timeroi(std::make_unique<Kernel::TimeROI>()),
      m_singleValueCache(
          std::make_unique<Kernel::Cache<std::pair<std::string, Kernel::Math::StatisticType>, double>>()) {}
 
LogManager::LogManager(const LogManager &other)
    : m_manager(std::make_unique<Kernel::PropertyManager>(*other.m_manager)),
      m_timeroi(std::make_unique<Kernel::TimeROI>(*other.m_timeroi)),
      m_singleValueCache(std::make_unique<Kernel::Cache<std::pair<std::string, Kernel::Math::StatisticType>, double>>(
          *other.m_singleValueCache)) {}
 
// Defined as default in source for forward declaration with std::unique_ptr.
LogManager::~LogManager() = default;
 
LogManager &LogManager::operator=(const LogManager &other) {
  *m_manager = *other.m_manager;
  *m_timeroi = *other.m_timeroi;
  m_singleValueCache = std::make_unique<Kernel::Cache<std::pair<std::string, Kernel::Math::StatisticType>, double>>(
      *other.m_singleValueCache);
  return *this;
}
 
void LogManager::setStartAndEndTime(const Types::Core::DateAndTime &start, const Types::Core::DateAndTime &end) {
  this->addProperty<std::string>(START_TIME_NAME, start.toISO8601String(), true);
  this->addProperty<std::string>(END_TIME_NAME, end.toISO8601String(), true);
}
 
const Types::Core::DateAndTime LogManager::startTime() const {
  if (hasProperty(START_TIME_NAME)) {
    try {
      DateAndTime start_time(getProperty(START_TIME_NAME)->value());
      if (start_time != DateAndTime::GPS_EPOCH) {
        return start_time;
      }
    } catch (std::invalid_argument &) { /*Swallow and move on*/
    }
  }
 
  const std::string run_start_prop("run_start");
  if (hasProperty(run_start_prop)) {
    try {
      DateAndTime start_time(getProperty(run_start_prop)->value());
      if (start_time != DateAndTime::GPS_EPOCH) {
        return start_time;
      }
    } catch (std::invalid_argument &) { /*Swallow and move on*/
    }
  }
 
  std::string errorMsg{"No valid start time has been set for this run."};
  if (hasValidProtonChargeLog(errorMsg))
    return getFirstPulseTime();
 
  throw std::runtime_error(errorMsg);
}
 
const Types::Core::DateAndTime LogManager::endTime() const {
  if (hasProperty(END_TIME_NAME)) {
    try {
      return DateAndTime(getProperty(END_TIME_NAME)->value());
    } catch (std::invalid_argument &) { /*Swallow and move on*/
    }
  }
 
  const std::string run_end_prop("run_end");
  if (hasProperty(run_end_prop)) {
    try {
      return DateAndTime(getProperty(run_end_prop)->value());
    } catch (std::invalid_argument &) { /*Swallow and move on*/
    }
  }
 
  std::string errorMsg{"No valid end time has been set for this run."};
  if (hasValidProtonChargeLog(errorMsg))
    return getLastPulseTime();
 
  throw std::runtime_error(errorMsg);
}
 
bool LogManager::hasStartTime() const {
  std::string temp;
  return hasProperty(START_TIME_NAME) || hasProperty("run_start") || hasValidProtonChargeLog(temp);
}
 
bool LogManager::hasEndTime() const {
  std::string temp;
  return hasProperty(END_TIME_NAME) || hasProperty("run_end") || hasValidProtonChargeLog(temp);
}
 
const DateAndTime LogManager::getFirstPulseTime() const {
  const TimeSeriesProperty<double> *log =
      getTimeSeriesProperty<double>("proton_charge"); // guaranteed to be a valid pointer, if the call succeeds
  if (log->realSize() == 0)
    throw std::runtime_error("First pulse time is not available. Log \"proton_charge\" is empty.");
 
  // NOTE, this method has been migrated from MatrixWorkspace class, where it had a comment:
  // "Pulse times before 1991 (up to 100) are skipped. This is to avoid
  // a DAS bug at SNS around Mar 2011 where the first pulse time is Jan 1, 1990."
  // There was no explanation why 100 was picked as the maximum number of times to skip.
  // In the refactored algorithm below we keep 100 as is.
  const DateAndTime reference("1991-01-01T00:00:00");
  const std::vector<DateAndTime> &times = log->timesAsVector();
  const size_t maxSkip{100};
  const size_t maxIndex = std::min(static_cast<size_t>(log->realSize()), maxSkip);
  const auto it = std::find_if(times.cbegin(), times.cbegin() + maxIndex,
                               [&reference](const auto &time) { return time >= reference; });
  if (it != times.cbegin() + maxIndex) {
    return *it;
  }
  return times[maxIndex - 1];
}
 
const DateAndTime LogManager::getLastPulseTime() const {
  const TimeSeriesProperty<double> *log =
      getTimeSeriesProperty<double>("proton_charge"); // guaranteed to be a valid pointer, if the call succeeds
  if (log->realSize() == 0)
    throw std::runtime_error("Last pulse time is not available. Log \"proton_charge\" is empty.");
  return log->lastTime();
}
 
bool LogManager::hasValidProtonChargeLog(std::string &error) const {
  const std::string log_name{"proton_charge"};
  if (!hasProperty(log_name)) {
    error += " Log " + log_name + " is not found.";
    return false;
  }
 
  Kernel::Property *prop = getProperty(log_name);
  const TimeSeriesProperty<double> *log;
  if (!(log = dynamic_cast<Kernel::TimeSeriesProperty<double> *>(prop))) {
    error += " Log " + log_name + " is not a time series of floating-point values.";
    return false;
  }
 
  if (log->realSize() == 0) {
    error += " Log " + log_name + " is empty.";
    return false;
  }
 
  return true;
}
 
//-----------------------------------------------------------------------------------------------
void LogManager::filterByTime(const Types::Core::DateAndTime start, const Types::Core::DateAndTime stop) {
  this->setTimeROI(TimeROI(start, stop));
  this->removeDataOutsideTimeROI();
}
 
LogManager *LogManager::cloneInTimeROI(const Kernel::TimeROI &timeROI) {
  LogManager *newMgr = new LogManager();
  newMgr->m_manager = std::unique_ptr<PropertyManager>(m_manager->cloneInTimeROI(timeROI));
 
  // This LogManager object may have filtered out some data previously, in which case it would be holding the TimeROI
  // used. Therefore, the cloned object's TimeROI should be an intersection of the input TimeROI with the one being
  // held.
  TimeROI outputTimeROI(timeROI);
  outputTimeROI.update_or_replace_intersection(*m_timeroi);
  newMgr->m_timeroi = std::make_unique<Kernel::TimeROI>(outputTimeROI);
 
  return newMgr;
}
 
void LogManager::copyAndFilterProperties(const LogManager &other, const Kernel::TimeROI &timeROI) {
  this->m_manager = std::unique_ptr<PropertyManager>(other.m_manager->cloneInTimeROI(timeROI));
  this->setTimeROI(timeROI);
  this->clearSingleValueCache();
}
 
void LogManager::removeDataOutsideTimeROI() {
  m_manager->removeDataOutsideTimeROI(*m_timeroi);
  this->clearSingleValueCache();
}
 
//----------------------------------------------------------------------------------------------
void LogManager::filterByLog(Mantid::Kernel::LogFilter *filter, const std::vector<std::string> &excludedFromFiltering) {
  // This will invalidate the cache
  this->clearSingleValueCache();
  m_manager->filterByProperty(filter, excludedFromFiltering);
}
 
//-----------------------------------------------------------------------------------------------
void LogManager::addProperty(std::unique_ptr<Kernel::Property> prop, bool overwrite) {
  // Make an exception for the proton charge
  // and overwrite its value as we don't want to store the proton charge in two
  // separate locations
  // Similar we don't want more than one run_title
  std::string name = prop->name();
  if (hasProperty(name) && (overwrite || prop->name() == PROTON_CHARGE_LOG_NAME || prop->name() == "run_title")) {
    removeProperty(name);
  }
  m_manager->declareProperty(std::move(prop), "");
}
 
//-----------------------------------------------------------------------------------------------
bool LogManager::hasProperty(const std::string &name) const { return m_manager->existsProperty(name); }
 
//-----------------------------------------------------------------------------------------------
void LogManager::removeProperty(const std::string &name, bool delProperty) {
  // Remove any cached entries for this log. Need to make this more general
  for (unsigned int stat = 0; stat < 7; ++stat) {
    m_singleValueCache->removeCache(std::make_pair(name, static_cast<Math::StatisticType>(stat)));
  }
  m_manager->removeProperty(name, delProperty);
}
 
const std::vector<Kernel::Property *> &LogManager::getProperties() const { return m_manager->getProperties(); }
 
//-----------------------------------------------------------------------------------------------
size_t LogManager::getMemorySize() const {
  size_t total{m_timeroi->getMemorySize()};
 
  for (const auto &p : m_manager->getProperties()) {
    if (p) {
      // cppcheck-suppress useStlAlgorithm
      total += p->getMemorySize() + sizeof(Property *); // cppcheck-suppress useStlAlgorithm
    }
  }
 
  return total;
}
 
template <typename T> Kernel::TimeSeriesProperty<T> *LogManager::getTimeSeriesProperty(const std::string &name) const {
  Kernel::Property *prop = getProperty(name);
  if (auto *tsp = dynamic_cast<Kernel::TimeSeriesProperty<T> *>(prop)) {
    return tsp;
  } else {
    throw std::invalid_argument("LogManager::getTimeSeriesProperty - '" + name + "' is not a TimeSeriesProperty");
  }
}
 
double LogManager::getTimeAveragedStd(const std::string &name) const {
  return getStatistics(name).time_standard_deviation;
}
 
double LogManager::getTimeAveragedValue(const std::string &name) const { return getStatistics(name).time_mean; }
 
Kernel::TimeSeriesPropertyStatistics LogManager::getStatistics(const std::string &name) const {
  const Kernel::Property *prop = getProperty(name);
 
  // statistics from a TimeSeriesProperty object
  if (auto *timeSeriesProp = dynamic_cast<const Kernel::ITimeSeriesProperty *>(prop))
    return timeSeriesProp->getStatistics(m_timeroi.get());
 
  // statistics from a PropertyWithValue object
  double value;
  if (Mantid::API::convertSingleValueToDouble(prop, value))
    return TimeSeriesPropertyStatistics(value);
 
  // return values set to NAN, signaling no statistics can be obtained
  TimeSeriesPropertyStatistics invalid;
  invalid.setAllToNan();
  return invalid;
}
 
template <typename HeldType> HeldType LogManager::getPropertyValueAsType(const std::string &name) const {
  Kernel::Property *prop = getProperty(name);
  if (auto *valueProp = dynamic_cast<Kernel::PropertyWithValue<HeldType> *>(prop)) {
    return (*valueProp)();
  } else {
    throw std::invalid_argument("Run::getPropertyValueAsType - '" + name + "' is not of the requested type");
  }
}
 
double LogManager::getPropertyAsSingleValue(const std::string &name, Kernel::Math::StatisticType statistic) const {
  double singleValue(0.0);
  const auto key = std::make_pair(name, statistic);
  if (!m_singleValueCache->getCache(key, singleValue)) {
    const Property *log = getProperty(name);
    const TimeROI &filter = this->getTimeROI();
    if (!convertPropertyToDouble(log, singleValue, statistic, &filter)) {
      if (const auto stringLog = dynamic_cast<const PropertyWithValue<std::string> *>(log)) {
        // Try to lexically cast string to a double
        try {
          singleValue = std::stod(stringLog->value());
        } catch (const std::invalid_argument &) {
          throw std::invalid_argument("Run::getPropertyAsSingleValue - Property \"" + name +
                                      "\" cannot be converted to a numeric value.");
        }
      } else {
        throw std::invalid_argument("Run::getPropertyAsSingleValue - Property \"" + name +
                                    "\" is not a single numeric value or numeric time series.");
      }
    }
    // Put it in the cache
    m_singleValueCache->setCache(key, singleValue);
  }
  return singleValue;
}
 
int LogManager::getPropertyAsIntegerValue(const std::string &name) const {
  int singleValue(0);
  double discard(0);
 
  Property *prop = getProperty(name);
 
  if (convertSingleValue<int32_t>(prop, discard) || convertSingleValue<int64_t>(prop, discard) ||
      convertSingleValue<uint32_t>(prop, discard) || convertSingleValue<uint64_t>(prop, discard)) {
    singleValue = std::stoi(prop->value());
  } else {
    throw std::invalid_argument("Run::getPropertyAsIntegerValue - Property \"" + name +
                                "\" cannot be converted to an integer value.");
  }
 
  return singleValue;
}
 
Kernel::Property *LogManager::getProperty(const std::string &name) const { return m_manager->getProperty(name); }
 
void LogManager::clearTimeSeriesLogs() {
  auto &props = getProperties();
 
  // Loop over the set of properties, identifying those that are time-series
  // properties
  // and then clearing them out.
  for (auto prop : props) {
    if (auto tsp = dynamic_cast<ITimeSeriesProperty *>(prop)) {
      tsp->clear();
    }
  }
}
 
void LogManager::clearOutdatedTimeSeriesLogValues() {
  auto &props = getProperties();
  for (auto prop : props) {
    if (auto tsp = dynamic_cast<ITimeSeriesProperty *>(prop)) {
      tsp->clearOutdated();
    }
  }
}
 
const Kernel::TimeROI &LogManager::getTimeROI() const { return *(m_timeroi.get()); }
 
void LogManager::setTimeROI(const Kernel::TimeROI &timeroi) {
  m_timeroi->replaceROI(timeroi);
  this->clearSingleValueCache();
}
 
//--------------------------------------------------------------------------------------------
void LogManager::saveNexus(Nexus::File *file, const std::string &group, bool keepOpen) const {
  file->makeGroup(group, "NXgroup", true);
  file->putAttr("version", 1);
 
  // Save all the properties as NXlog
  std::vector<Property *> props = m_manager->getProperties();
  for (auto &prop : props) {
    try {
      prop->saveProperty(file);
    } catch (std::invalid_argument &exc) {
      g_log.warning(exc.what());
    }
  }
  // save the timeROI to the nexus file
  if (!(m_timeroi->useAll()))
    m_timeroi->saveNexus(file);
 
  if (!keepOpen)
    file->closeGroup();
}
 
//--------------------------------------------------------------------------------------------
void LogManager::loadNexus(Nexus::File * /*file*/, const std::string & /*group*/,
                           const Nexus::NexusDescriptor & /*fileInfo*/, const std::string & /*prefix*/,
                           bool /*keepOpen*/) {
  throw std::runtime_error("LogManager::loadNexus should not be used");
}
 
//--------------------------------------------------------------------------------------------
void LogManager::loadNexus(Nexus::File *file, const std::string &group, bool keepOpen) {
  if (!group.empty()) {
    file->openGroup(group, "NXgroup");
  }
  std::map<std::string, std::string> entries;
  file->getEntries(entries);
  LogManager::loadNexus(file, entries);
 
  if (!(group.empty() || keepOpen)) {
    file->closeGroup();
  }
}
 
void LogManager::loadNexus(Nexus::File *file, const Nexus::NexusDescriptor &fileInfo, const std::string &prefix) {
 
  // Only load from NXlog entries
  const auto &allEntries = fileInfo.getAllEntries();
  auto itNxLogEntries = allEntries.find("NXlog");
  const auto nxLogEntries = (itNxLogEntries != allEntries.end()) ? itNxLogEntries->second : std::set<std::string>{};
 
  const auto levels = std::count(prefix.begin(), prefix.end(), '/');
 
  auto itLower = nxLogEntries.lower_bound(prefix);
 
  if (itLower == nxLogEntries.end()) {
    return;
  }
  if (itLower->compare(0, prefix.size(), prefix) != 0) {
    return;
  }
 
  for (auto it = itLower; it != nxLogEntries.end() && it->compare(0, prefix.size(), prefix) == 0; ++it) {
    // only next level entries
    const std::string &absoluteEntryName = *it;
    if (std::count(absoluteEntryName.begin(), absoluteEntryName.end(), '/') != levels + 1) {
      continue;
    }
    const std::string nameClass = absoluteEntryName.substr(absoluteEntryName.find_last_of('/') + 1);
 
    auto prop = PropertyNexus::loadProperty(file, nameClass, fileInfo, prefix);
    if (prop) {
      // get TimeROI
      if (prop->name() == Kernel::TimeROI::NAME) {
        auto boolProp = dynamic_cast<TimeSeriesProperty<bool> *>(prop.get());
        if (boolProp) {
          m_timeroi->replaceROI(boolProp);
        } else {
          throw std::runtime_error("Kernel_TimeROI is not a TimeSeriesPropertyBool");
        }
      } else {
        // everything else gets added to the list of properties
        m_manager->declareOrReplaceProperty(std::move(prop));
      }
    }
  }
}
 
//--------------------------------------------------------------------------------------------
void LogManager::loadNexus(Nexus::File *file, const std::map<std::string, std::string> &entries) {
 
  for (const auto &name_class : entries) {
    // NXLog types are the main one.
    if (name_class.second == "NXlog") {
      auto prop = PropertyNexus::loadProperty(file, name_class.first);
      if (prop) {
        // get TimeROI
        if (prop->name() == Kernel::TimeROI::NAME) {
          auto boolProp = dynamic_cast<TimeSeriesProperty<bool> *>(prop.get());
          if (boolProp) {
            m_timeroi->replaceROI(boolProp);
          } else {
            throw std::runtime_error("Kernel_TimeROI is not a TimeSeriesPropertyBool");
          }
        } else {
          // everything else gets added to the list of properties
          m_manager->declareOrReplaceProperty(std::move(prop));
        }
      }
    }
  }
}
 
void LogManager::clearLogs() { m_manager->clear(); }
 
void LogManager::clearSingleValueCache() { m_singleValueCache->clear(); }
 
std::string LogManager::getInvalidValuesFilterLogName(const std::string &logName) {
  return PropertyManager::getInvalidValuesFilterLogName(logName);
}
 
bool LogManager::hasInvalidValuesFilter(const std::string &logName) const {
  return hasProperty(getInvalidValuesFilterLogName(logName));
}
 
Kernel::TimeSeriesProperty<bool> *LogManager::getInvalidValuesFilter(const std::string &logName) const {
  try {
    auto log = getLogData(getInvalidValuesFilterLogName(logName));
    if (auto tsp = dynamic_cast<TimeSeriesProperty<bool> *>(log)) {
      return tsp;
    }
  } catch (Exception::NotFoundError &) {
    // do nothing, just drop through tto the return line below
  }
  return nullptr;
}
 
bool LogManager::operator==(const LogManager &other) const {
  return (*m_manager == *(other.m_manager)) && (*m_timeroi == *(other.m_timeroi));
}
 
bool LogManager::operator!=(const LogManager &other) const {
  return (*m_timeroi != *(other.m_timeroi)) || (*m_manager != *(other.m_manager));
}
 
//-----------------------------------------------------------------------------------------------------------------------
// Private methods
//-----------------------------------------------------------------------------------------------------------------------
 
#define INSTANTIATE(TYPE)                                                                                              \
  template MANTID_API_DLL Kernel::TimeSeriesProperty<TYPE> *LogManager::getTimeSeriesProperty(const std::string &)     \
      const;                                                                                                           \
  template MANTID_API_DLL TYPE LogManager::getPropertyValueAsType(const std::string &) const;
 
INSTANTIATE(double)
INSTANTIATE(int32_t)
INSTANTIATE(int64_t)
INSTANTIATE(uint32_t)
INSTANTIATE(uint64_t)
INSTANTIATE(std::string)
INSTANTIATE(bool)
 
template MANTID_API_DLL uint16_t LogManager::getPropertyValueAsType(const std::string &) const;
template MANTID_API_DLL std::vector<double> LogManager::getPropertyValueAsType(const std::string &) const;
template MANTID_API_DLL std::vector<size_t> LogManager::getPropertyValueAsType(const std::string &) const;
template MANTID_API_DLL std::vector<int> LogManager::getPropertyValueAsType(const std::string &) const;
template MANTID_API_DLL std::vector<long> LogManager::getPropertyValueAsType(const std::string &) const;
} // namespace Mantid::API