SampleLogsBehaviour.cpp

Go to the documentation of this file.

// Mantid Repository : https://github.com/mantidproject/mantid
//
// Copyright © 2018 ISIS Rutherford Appleton Laboratory UKRI,
//   NScD Oak Ridge National Laboratory, European Spallation Source,
//   Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
// SPDX - License - Identifier: GPL - 3.0 +
#include <utility>
 
#include "MantidAPI/MatrixWorkspace.h"
#include "MantidAPI/Run.h"
#include "MantidAlgorithms/RunCombinationHelpers/SampleLogsBehaviour.h"
#include "MantidGeometry/Instrument.h"
#include "MantidKernel/FloatingPointComparison.h"
#include "MantidKernel/Logger.h"
#include "MantidKernel/Property.h"
#include "MantidKernel/StringTokenizer.h"
#include "MantidKernel/Strings.h"
#include "MantidKernel/TimeSeriesProperty.h"
 
namespace Mantid::Algorithms {
 
using namespace Kernel;
using namespace API;
 
namespace {
std::string generateDifferenceMessage(const std::string &item, const std::string &wsName, const std::string &wsValue,
                                      const std::string &firstValue) {
  std::stringstream stringstream;
  stringstream << "Item \"" << item << "\" has different values in workspaces! Found: " << wsValue << " in workspace "
               << wsName << " but the value in the first workspace was: " << firstValue << "." << std::endl;
  return stringstream.str();
}
} // namespace
 
// Names and docs from the properties allowing to override the default (IPF
// controlled) merging behaviour.
// These are common between e.g. MergeRuns and ConjoinXRuns.
const std::string SampleLogsBehaviour::TIME_SERIES_PROP = "SampleLogsTimeSeries";
const std::string SampleLogsBehaviour::TIME_SERIES_DOC = "A comma separated list of the sample logs to merge into a "
                                                         "time series. The initial times are taken as the start times "
                                                         "for the run. Sample logs must be numeric.";
const std::string SampleLogsBehaviour::LIST_PROP = "SampleLogsList";
const std::string SampleLogsBehaviour::LIST_DOC = "A comma separated list of the sample logs to merge into a "
                                                  "list.";
const std::string SampleLogsBehaviour::WARN_PROP = "SampleLogsWarn";
const std::string SampleLogsBehaviour::WARN_DOC = "A comma separated list of the sample "
                                                  "logs to generate a warning if "
                                                  "different when merging.";
const std::string SampleLogsBehaviour::WARN_TOL_PROP = "SampleLogsWarnTolerances";
const std::string SampleLogsBehaviour::WARN_TOL_DOC = "The tolerances for warning if sample logs are different. "
                                                      "Can either be empty for a comparison of the strings, a "
                                                      "single value for all warn sample logs, or a comma "
                                                      "separated list of values (must be the same length as "
                                                      "SampleLogsWarn).";
const std::string SampleLogsBehaviour::FAIL_PROP = "SampleLogsFail";
const std::string SampleLogsBehaviour::FAIL_DOC = "The sample logs to fail if different "
                                                  "when merging. If there is a "
                                                  "difference the run is skipped.";
const std::string SampleLogsBehaviour::FAIL_TOL_PROP = "SampleLogsFailTolerances";
const std::string SampleLogsBehaviour::FAIL_TOL_DOC = "The tolerances for failing if sample logs are different. "
                                                      "Can either be empty for a comparison of the strings, a "
                                                      "single value for all fail sample logs, or a comma "
                                                      "separated list of values (must be the same length as "
                                                      "SampleLogsFail).";
const std::string SampleLogsBehaviour::SUM_PROP = "SampleLogsSum";
const std::string SampleLogsBehaviour::SUM_DOC = "A comma separated list of the sample "
                                                 "logs to sum into a single entry.  "
                                                 "Sample logs must be numeric.";
 
SampleLogsBehaviour::SampleLogsBehaviour(const MatrixWorkspace_sptr &ws, Logger &logger,
                                         const SampleLogNames &logEntries, ParameterName parName)
    : parameterNames(std::move(parName)), m_logger(logger) {
  setSampleMap(m_logMap, MergeLogType::Sum, logEntries.sampleLogsSum, *ws);
  setSampleMap(m_logMap, MergeLogType::TimeSeries, logEntries.sampleLogsTimeSeries, *ws);
  setSampleMap(m_logMap, MergeLogType::List, logEntries.sampleLogsList, *ws);
  setSampleMap(m_logMap, MergeLogType::Warn, logEntries.sampleLogsWarn, *ws, logEntries.sampleLogsWarnTolerances);
  setSampleMap(m_logMap, MergeLogType::Fail, logEntries.sampleLogsFail, *ws, logEntries.sampleLogsFailTolerances);
 
  SampleLogsMap instrumentMap;
  this->createSampleLogsMapsFromInstrumentParams(instrumentMap, *ws);
 
  // This adds the parameters from the instrument to the main map, with any
  // duplicates left as the versions in the MergeRuns arguments.
  m_logMap.insert(instrumentMap.begin(), instrumentMap.end());
}
 
void SampleLogsBehaviour::createSampleLogsMapsFromInstrumentParams(SampleLogsMap &map, MatrixWorkspace &ws) {
  std::string params = ws.getInstrument()->getParameterAsString(parameterNames.SUM_MERGE, false);
  setSampleMap(map, MergeLogType::Sum, params, ws, "", true);
 
  params = ws.getInstrument()->getParameterAsString(parameterNames.TIME_SERIES_MERGE, false);
  setSampleMap(map, MergeLogType::TimeSeries, params, ws, "", true);
 
  params = ws.getInstrument()->getParameterAsString(parameterNames.LIST_MERGE, false);
  setSampleMap(map, MergeLogType::List, params, ws, "", true);
 
  params = ws.getInstrument()->getParameterAsString(parameterNames.WARN_MERGE, false);
  std::string paramsTolerances;
  paramsTolerances = ws.getInstrument()->getParameterAsString(parameterNames.WARN_MERGE_TOLERANCES, false);
  setSampleMap(map, MergeLogType::Warn, params, ws, paramsTolerances, true);
 
  params = ws.getInstrument()->getParameterAsString(parameterNames.FAIL_MERGE, false);
  paramsTolerances = ws.getInstrument()->getParameterAsString(parameterNames.FAIL_MERGE_TOLERANCES, false);
  setSampleMap(map, MergeLogType::Fail, params, ws, paramsTolerances, true);
}
 
void SampleLogsBehaviour::setSampleMap(SampleLogsMap &map, const MergeLogType &mergeType, const std::string &params,
                                       MatrixWorkspace &ws, const std::string &paramsTolerances,
                                       bool skipIfInPrimaryMap) {
 
  StringTokenizer tokenizer(params, ",", StringTokenizer::TOK_TRIM | StringTokenizer::TOK_IGNORE_EMPTY);
  StringTokenizer tokenizerTolerances(paramsTolerances, ",",
                                      StringTokenizer::TOK_TRIM | StringTokenizer::TOK_IGNORE_EMPTY);
 
  auto tolerancesStringVector = tokenizerTolerances.asVector();
 
  std::vector<double> tolerancesVector = createTolerancesVector(tokenizer.asVector().size(), tolerancesStringVector);
 
  auto i = tokenizer.begin();
  auto j = tolerancesVector.begin();
 
  for (; i != tokenizer.end() && j != tolerancesVector.end(); ++i, ++j) {
    auto item = *i;
    auto tolerance = *j;
 
    // Check 1: Does the key exist in the primary map? If so ignore it and
    // continue.
    if (skipIfInPrimaryMap && (m_logMap.count(SampleLogsKey(item, mergeType)) != 0)) {
      continue;
    }
 
    // Check 2: If the key (sample log name) already exists in this map throw an
    // error.
    if (map.count(SampleLogsKey(item, mergeType)) != 0) {
      throw std::invalid_argument("Error when making list of merge items, sample log \"" + item +
                                  "\" defined more than once!");
    }
 
    // Check 3: If the sample log is one that should not be combined with
    // others, check other incompatible sample logs do not exist too.
    std::set<MergeLogType> uncombinableLogs = {MergeLogType::Sum, MergeLogType::TimeSeries, MergeLogType::List};
    if (uncombinableLogs.count(mergeType) != 0) {
      bool skipLog = false;
 
      uncombinableLogs.erase(mergeType);
      for (auto &logType : uncombinableLogs) {
        if (map.count(SampleLogsKey(item, logType)) > 0)
          throw std::invalid_argument("Error when making list of merge items, sample log " + item +
                                      " being used for two incompatible merge types!");
        if (skipIfInPrimaryMap && m_logMap.count(SampleLogsKey(item, logType)) > 0)
          skipLog = true;
      }
 
      if (skipLog)
        continue;
    }
 
    // Check 4: Does the sample log exist? If not log an error but continue.
    std::shared_ptr<Property> prop;
    try {
      prop = std::shared_ptr<Property>(ws.getLog(item)->clone());
    } catch (std::invalid_argument &) {
      m_logger.error() << "Could not merge sample log \"" << item
                       << "\", does not exist in workspace! This sample log will be ignored." << std::endl;
      continue;
    }
 
    // Check 5: Can the property be converted to a double? If not, and sum or
    // time series case, log an error but continue.
    bool isNumeric;
    double value = 0.0;
    isNumeric = setNumericValue(item, ws, value);
    if (!isNumeric && (mergeType == MergeLogType::Sum || mergeType == MergeLogType::TimeSeries)) {
      m_logger.error() << item
                       << " could not be converted to a numeric type. "
                          "This sample log will be ignored.\n"
                       << std::endl;
      continue;
    }
 
    // For a TimeSeries or a List we need to add a new property to the workspace
    if (mergeType == MergeLogType::TimeSeries) {
      prop = addPropertyForTimeSeries(item, value, ws);
    } else if (mergeType == MergeLogType::List) {
      prop = addPropertyForList(item, prop->value(), ws);
    }
 
    // Finally add the key-value pair to the map
    map[SampleLogsKey(item, mergeType)] = {prop, tolerance, isNumeric};
  }
}
 
std::vector<double> SampleLogsBehaviour::createTolerancesVector(size_t numberNames,
                                                                const std::vector<std::string> &tolerances) {
  size_t numberTolerances = tolerances.size();
 
  std::vector<double> tolerancesVector(numberNames);
 
  if (numberNames == numberTolerances && numberTolerances > 1) {
    try {
      std::transform(tolerances.begin(), tolerances.end(), tolerancesVector.begin(),
                     [](const std::string &val) { return std::stod(val); });
    } catch (std::invalid_argument &) {
      throw std::invalid_argument("Error when creating tolerances vector. "
                                  "Please ensure each comma separated value is "
                                  "numeric.");
    } catch (std::out_of_range &) {
      throw std::out_of_range("Error when creating tolerances vector. Please "
                              "ensure each comma separated value is within "
                              "double precision range.");
    }
    if (std::any_of(tolerancesVector.cbegin(), tolerancesVector.cend(), [](const auto value) { return value < 0.; })) {
      throw std::out_of_range("Error when creating tolerances vector. Please "
                              "ensure all tolerance values are positive.");
    }
  } else if (tolerances.empty()) {
    std::fill(tolerancesVector.begin(), tolerancesVector.end(), -1.0);
  } else if (numberTolerances == 1) {
    double value;
    try {
      value = std::stod(tolerances.front());
      std::fill(tolerancesVector.begin(), tolerancesVector.end(), value);
    } catch (std::invalid_argument &) {
      throw std::invalid_argument("The single tolerance value requested can "
                                  "not be converted to a number. Please ensure "
                                  "it is a single number, or a comma separated "
                                  "list of numbers.");
    } catch (std::out_of_range &) {
      throw std::out_of_range("The single tolerance value requested can not be "
                              "converted to a double. Please ensure tolerance "
                              "is within double precision range.");
    }
    if (value < 0)
      throw std::out_of_range("The single tolerance value requested is "
                              "negative. Please ensure it is positive.");
  } else {
    throw std::invalid_argument("Invalid length of tolerances, found " + std::to_string(numberTolerances) +
                                " tolerance values but " + std::to_string(numberNames) + " names.");
  }
 
  return tolerancesVector;
}
 
std::shared_ptr<Property> SampleLogsBehaviour::addPropertyForTimeSeries(const std::string &item, const double value,
                                                                        MatrixWorkspace &ws) {
  std::shared_ptr<Property> returnProp;
  const std::string originalUnit = ws.getLog(item)->units();
  try {
    // See if property exists as a TimeSeriesLog already - merging an output of
    // MergeRuns
    ws.run().getTimeSeriesProperty<double>(item);
    returnProp.reset(ws.getLog(item)->clone());
  } catch (std::invalid_argument &) {
    // Property does not already exist, so add it setting the first entry
    std::unique_ptr<Kernel::TimeSeriesProperty<double>> timeSeriesProp(new TimeSeriesProperty<double>(item));
    std::string startTime = ws.run().startTime().toISO8601String();
 
    timeSeriesProp->addValue(startTime, value);
    ws.mutableRun().addProperty(std::move(timeSeriesProp), true);
 
    returnProp.reset(ws.getLog(item)->clone());
  }
  ws.getLog(item)->setUnits(originalUnit); // we lost the unit of the workspace
  return returnProp;
}
 
std::shared_ptr<Property> SampleLogsBehaviour::addPropertyForList(const std::string &item, const std::string &value,
                                                                  MatrixWorkspace &ws) {
  std::shared_ptr<Property> returnProp;
 
  const std::string originalUnit = ws.getLog(item)->units();
  // See if property exists already - merging an output of the calling algorithm
  returnProp.reset(ws.getLog(item)->clone());
  if (returnProp->type() != "string") {
    ws.mutableRun().addProperty(item, value, true);
    returnProp.reset(ws.getLog(item)->clone());
  }
  ws.getLog(item)->setUnits(originalUnit); // we lost the unit of the workspace
  return returnProp;
}
 
bool SampleLogsBehaviour::setNumericValue(const std::string &item, const MatrixWorkspace &ws, double &value) {
  bool isNumeric;
 
  try {
    value = ws.getLogAsSingleValue(item);
    isNumeric = true;
  } catch (std::invalid_argument &) {
    isNumeric = false;
  }
 
  return isNumeric;
}
 
void SampleLogsBehaviour::mergeSampleLogs(const MatrixWorkspace_sptr &addeeWS, const MatrixWorkspace_sptr &outWS) {
  for (const auto &item : m_logMap) {
    const std::string &logName = item.first.first;
 
    const Property *addeeWSProperty = addeeWS->getLog(logName);
    const std::string originalUnit = addeeWS->getLog(logName)->units();
 
    double addeeWSNumericValue = 0.;
    double outWSNumericValue = 0.;
 
    try {
      addeeWSNumericValue = addeeWS->getLogAsSingleValue(logName);
      outWSNumericValue = outWS->getLogAsSingleValue(logName);
    } catch (std::invalid_argument &) {
      if (item.second.isNumeric) {
        throw std::invalid_argument(logName + " could not be converted to a numeric type");
      }
    }
 
    switch (item.first.second) {
    case MergeLogType::Sum: {
      this->updateSumProperty(addeeWSNumericValue, outWSNumericValue, *outWS, logName);
      break;
    }
    case MergeLogType::TimeSeries: {
      this->updateTimeSeriesProperty(*addeeWS, *outWS, logName);
      break;
    }
    case MergeLogType::List: {
      this->updateListProperty(*addeeWS, *outWS, logName);
      break;
    }
    case MergeLogType::Warn:
      this->checkWarnProperty(*addeeWS, addeeWSProperty, item.second, addeeWSNumericValue, outWSNumericValue, logName);
      break;
    case MergeLogType::Fail:
      this->checkErrorProperty(*addeeWS, addeeWSProperty, item.second, addeeWSNumericValue, outWSNumericValue, logName);
      break;
    }
    outWS->getLog(logName)->setUnits(originalUnit);
  }
}
 
void SampleLogsBehaviour::updateSumProperty(double addeeWSNumericValue, double outWSNumericValue,
                                            MatrixWorkspace &outWS, const std::string &name) {
  outWS.mutableRun().addProperty(name, addeeWSNumericValue + outWSNumericValue, true);
}
 
void SampleLogsBehaviour::updateTimeSeriesProperty(const MatrixWorkspace &addeeWS, const MatrixWorkspace &outWS,
                                                   const std::string &name) {
  auto timeSeriesProp = outWS.run().getTimeSeriesProperty<double>(name);
  try {
    const auto addeeTimeSeries = addeeWS.run().getTimeSeriesProperty<double>(name);
    timeSeriesProp->merge(addeeTimeSeries);
  } catch (std::invalid_argument &) {
    Types::Core::DateAndTime startTime = addeeWS.run().startTime();
    double value = addeeWS.run().getLogAsSingleValue(name);
    timeSeriesProp->addValue(startTime, value);
  }
  // Remove this to supress a warning, we will put it back after adding the
  // workspaces in MergeRuns
  const Property *addeeWSProperty = addeeWS.run().getProperty(name);
  m_addeeLogMap.emplace_back(std::shared_ptr<Property>(addeeWSProperty->clone()));
}
 
void SampleLogsBehaviour::updateListProperty(const MatrixWorkspace &addeeWS, MatrixWorkspace &outWS,
                                             const std::string &name) {
  const std::string addeeWSVal = addeeWS.getLog(name)->value();
  const std::string outWSVal = outWS.run().getProperty(name)->value();
  outWS.mutableRun().addProperty(name, outWSVal + ", " + addeeWSVal, true);
}
 
void SampleLogsBehaviour::checkWarnProperty(const MatrixWorkspace &addeeWS, const Property *addeeWSProperty,
                                            const SampleLogBehaviour &behaviour, const double addeeWSNumericValue,
                                            const double outWSNumericValue, const std::string &name) {
 
  if (!isWithinTolerance(behaviour, addeeWSNumericValue, outWSNumericValue) &&
      !stringPropertiesMatch(behaviour, addeeWSProperty)) {
    m_logger.warning() << generateDifferenceMessage(name, addeeWS.getName(), addeeWSProperty->value(),
                                                    behaviour.property->value());
  }
}
 
void SampleLogsBehaviour::checkErrorProperty(const MatrixWorkspace &addeeWS, const Property *addeeWSProperty,
                                             const SampleLogBehaviour &behaviour, const double addeeWSNumericValue,
                                             const double outWSNumericValue, const std::string &name) {
 
  if (!isWithinTolerance(behaviour, addeeWSNumericValue, outWSNumericValue) &&
      !stringPropertiesMatch(behaviour, addeeWSProperty)) {
    throw std::invalid_argument(
        generateDifferenceMessage(name, addeeWS.getName(), addeeWSProperty->value(), behaviour.property->value()));
  }
}
 
bool SampleLogsBehaviour::isWithinTolerance(const SampleLogBehaviour &behaviour, const double addeeWSNumericValue,
                                            const double outWSNumericValue) {
  if (behaviour.isNumeric && behaviour.tolerance > 0.0) {
    return Kernel::withinAbsoluteDifference(addeeWSNumericValue, outWSNumericValue, behaviour.tolerance);
  }
 
  return false;
}
 
bool SampleLogsBehaviour::stringPropertiesMatch(const SampleLogBehaviour &behaviour, const Property *addeeWSProperty) {
  return behaviour.property->value() == addeeWSProperty->value();
}
 
void SampleLogsBehaviour::setUpdatedSampleLogs(const MatrixWorkspace_sptr &outWS) {
  for (auto &item : m_logMap) {
    std::string propertyToReset = item.first.first;
 
    if (item.first.second == MergeLogType::Warn || item.first.second == MergeLogType::Fail) {
      continue;
    }
 
    const Property *outWSProperty = outWS->run().getProperty(propertyToReset);
    item.second.property = std::shared_ptr<Property>(outWSProperty->clone());
  }
}
 
void SampleLogsBehaviour::removeSampleLogsFromWorkspace(const MatrixWorkspace_sptr &addeeWS) {
  for (const auto &prop : m_addeeLogMap) {
    const auto &propName = prop->name();
    addeeWS->mutableRun().removeProperty(propName);
  }
}
 
void SampleLogsBehaviour::readdSampleLogToWorkspace(const MatrixWorkspace_sptr &addeeWS) {
  for (const auto &item : m_addeeLogMap) {
    auto property = std::unique_ptr<Kernel::Property>(item->clone());
    addeeWS->mutableRun().addProperty(std::move(property));
  }
  m_addeeLogMap.clear();
}
 
void SampleLogsBehaviour::resetSampleLogs(const MatrixWorkspace_sptr &ws) {
  for (auto const &item : m_logMap) {
    std::string const &propertyToReset = item.first.first;
 
    if (item.first.second == MergeLogType::TimeSeries) {
      auto property = std::unique_ptr<Kernel::Property>(item.second.property->clone());
      ws->mutableRun().addProperty(std::move(property), true);
    } else if (item.first.second == MergeLogType::Sum || item.first.second == MergeLogType::List) {
      ws->mutableRun().getProperty(propertyToReset)->setValue(item.second.property->value());
    }
  }
}
 
} // namespace Mantid::Algorithms