DataProcessorAlgorithm.cpp

Go to the documentation of this file.

// Mantid Repository : https://github.com/mantidproject/mantid
//
// Copyright © 2018 ISIS Rutherford Appleton Laboratory UKRI,
//   NScD Oak Ridge National Laboratory, European Spallation Source,
//   Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
// SPDX - License - Identifier: GPL - 3.0 +
#include "MantidAPI/DataProcessorAlgorithm.h"
#include "MantidAPI/AlgorithmManager.h"
#include "MantidAPI/AlgorithmProperty.h"
#include "MantidAPI/AnalysisDataService.h"
#include "MantidAPI/FileFinder.h"
#include "MantidAPI/IEventWorkspace.h"
#include "MantidAPI/ITableWorkspace.h"
#include "MantidAPI/WorkspaceFactory.h"
#include "MantidKernel/Exception.h"
#include "MantidKernel/FacilityInfo.h"
#include "MantidKernel/PropertyManager.h"
#include "MantidKernel/PropertyManagerDataService.h"
#include "MantidKernel/System.h"
#include "Poco/Path.h"
#include <stdexcept>
#include <utility>
 
#ifdef MPI_BUILD
#include <boost/mpi.hpp>
#endif
 
using namespace Mantid::Kernel;
using namespace Mantid::API;
 
namespace Mantid::API {
 
//----------------------------------------------------------------------------------------------
template <class Base>
GenericDataProcessorAlgorithm<Base>::GenericDataProcessorAlgorithm()
    : m_useMPI(false), m_loadAlg("Load"), m_accumulateAlg("Plus"), m_loadAlgFileProp("Filename"),
      m_propertyManagerPropertyName("ReductionProperties") {
  Base::enableHistoryRecordingForChild(true);
}
 
//---------------------------------------------------------------------------------------------
template <class Base>
std::shared_ptr<Algorithm>
GenericDataProcessorAlgorithm<Base>::createChildAlgorithm(const std::string &name, const double startProgress,
                                                          const double endProgress, const bool enableLogging,
                                                          const int &version) {
  // call parent method to create the child algorithm
  auto alg = Algorithm::createChildAlgorithm(name, startProgress, endProgress, enableLogging, version);
  alg->enableHistoryRecordingForChild(this->isRecordingHistoryForChild());
  if (this->isRecordingHistoryForChild()) {
    // pass pointer to the history object created in Algorithm to the child
    alg->trackAlgorithmHistory(Base::m_history);
  }
  return alg;
}
 
template <class Base> void GenericDataProcessorAlgorithm<Base>::setLoadAlg(const std::string &alg) {
  if (alg.empty())
    throw std::invalid_argument("Cannot set load algorithm to empty string");
  m_loadAlg = alg;
}
 
template <class Base> void GenericDataProcessorAlgorithm<Base>::setLoadAlgFileProp(const std::string &filePropName) {
  if (filePropName.empty()) {
    throw std::invalid_argument("Cannot set the load algorithm file property name");
  }
  m_loadAlgFileProp = filePropName;
}
 
template <class Base> void GenericDataProcessorAlgorithm<Base>::setAccumAlg(const std::string &alg) {
  if (alg.empty())
    throw std::invalid_argument("Cannot set accumulate algorithm to empty string");
  m_accumulateAlg = alg;
}
 
template <class Base> void GenericDataProcessorAlgorithm<Base>::setPropManagerPropName(const std::string &propName) {
  m_propertyManagerPropertyName = propName;
}
 
template <class Base>
void GenericDataProcessorAlgorithm<Base>::mapPropertyName(const std::string &nameInProp,
                                                          const std::string &nameInPropManager) {
  m_nameToPMName[nameInProp] = nameInPropManager;
}
 
template <class Base>
void GenericDataProcessorAlgorithm<Base>::copyProperty(const API::Algorithm_sptr &alg, const std::string &name) {
  if (!alg->existsProperty(name)) {
    std::stringstream msg;
    msg << "Algorithm \"" << alg->name() << "\" does not have property \"" << name << "\"";
    throw std::runtime_error(msg.str());
  }
 
  auto prop = alg->getPointerToProperty(name);
  Base::declareProperty(std::unique_ptr<Property>(prop->clone()), prop->documentation());
}
 
template <class Base> std::string GenericDataProcessorAlgorithm<Base>::getPropertyValue(const std::string &name) const {
  // explicitly specifying a property wins
  if (!Base::isDefault(name)) {
    return Algorithm::getPropertyValue(name);
  }
 
  // return it if it is in the held property manager
  auto mapping = m_nameToPMName.find(name);
  if (mapping != m_nameToPMName.end()) {
    auto pm = this->getProcessProperties();
    if (pm->existsProperty(mapping->second)) {
      return pm->getPropertyValue(mapping->second);
    }
  }
 
  // let the parent class version win
  return Algorithm::getPropertyValue(name);
}
 
template <class Base>
PropertyManagerOwner::TypedValue GenericDataProcessorAlgorithm<Base>::getProperty(const std::string &name) const {
  // explicitely specifying a property wins
  if (!Base::isDefault(name)) {
    return Base::getProperty(name);
  }
 
  // return it if it is in the held property manager
  auto mapping = m_nameToPMName.find(name);
  if (mapping != m_nameToPMName.end()) {
    auto pm = this->getProcessProperties();
    if (pm->existsProperty(mapping->second)) {
      return pm->getProperty(mapping->second);
    }
  }
 
  // let the parent class version win
  return Algorithm::getProperty(name);
}
 
template <class Base>
ITableWorkspace_sptr GenericDataProcessorAlgorithm<Base>::determineChunk(const std::string &filename) {
  UNUSED_ARG(filename);
 
  throw std::runtime_error("DataProcessorAlgorithm::determineChunk is not implemented");
}
 
template <class Base> MatrixWorkspace_sptr GenericDataProcessorAlgorithm<Base>::loadChunk(const size_t rowIndex) {
  UNUSED_ARG(rowIndex);
 
  throw std::runtime_error("DataProcessorAlgorithm::loadChunk is not implemented");
}
 
template <class Base> Workspace_sptr GenericDataProcessorAlgorithm<Base>::assemble(Workspace_sptr partialWS) {
  Workspace_sptr outputWS = std::move(partialWS);
#ifdef MPI_BUILD
  auto gatherAlg = createChildAlgorithm("GatherWorkspaces");
  gatherAlg->setLogging(true);
  gatherAlg->setAlwaysStoreInADS(true);
  gatherAlg->setProperty("InputWorkspace", partialWS);
  gatherAlg->setProperty("PreserveEvents", true);
  gatherAlg->setPropertyValue("OutputWorkspace", "_total");
  gatherAlg->execute();
 
  if (isMainThread()) {
    outputWS = AnalysisDataService::Instance().retrieve("_total");
  }
#endif
 
  return outputWS;
}
 
template <class Base>
Workspace_sptr GenericDataProcessorAlgorithm<Base>::assemble(const std::string &partialWSName,
                                                             [[maybe_unused]] const std::string &outputWSName) {
#ifdef MPI_BUILD
  std::string threadOutput = partialWSName;
  Workspace_sptr partialWS = AnalysisDataService::Instance().retrieve(partialWSName);
  auto gatherAlg = createChildAlgorithm("GatherWorkspaces");
  gatherAlg->setLogging(true);
  gatherAlg->setAlwaysStoreInADS(true);
  gatherAlg->setProperty("InputWorkspace", partialWS);
  gatherAlg->setProperty("PreserveEvents", true);
  gatherAlg->setPropertyValue("OutputWorkspace", outputWSName);
  gatherAlg->execute();
 
  if (isMainThread())
    threadOutput = outputWSName;
#else
  const std::string &threadOutput = partialWSName;
 
#endif
  Workspace_sptr outputWS = AnalysisDataService::Instance().retrieve(threadOutput);
  return outputWS;
}
 
template <class Base>
void GenericDataProcessorAlgorithm<Base>::saveNexus(const std::string &outputWSName, const std::string &outputFile) {
#ifdef MPI_BUILD
  if (boost::mpi::communicator().rank() <= 0 && !outputFile.empty()) {
#else
  if (!outputFile.empty()) {
#endif
    auto saveAlg = createChildAlgorithm("SaveNexus");
    saveAlg->setPropertyValue("Filename", outputFile);
    saveAlg->setPropertyValue("InputWorkspace", outputWSName);
    saveAlg->execute();
  }
}
 
template <class Base> bool GenericDataProcessorAlgorithm<Base>::isMainThread() {
  bool mainThread;
#ifdef MPI_BUILD
  mainThread = (boost::mpi::communicator().rank() == 0);
#else
  mainThread = true;
#endif
  return mainThread;
}
 
template <class Base> int GenericDataProcessorAlgorithm<Base>::getNThreads() {
#ifdef MPI_BUILD
  return boost::mpi::communicator().size();
#else
  return 1;
#endif
}
 
template <class Base>
Workspace_sptr GenericDataProcessorAlgorithm<Base>::load(const std::string &inputData, const bool loadQuiet) {
  Workspace_sptr inputWS;
 
  // First, check whether we have the name of an existing workspace
  if (AnalysisDataService::Instance().doesExist(inputData)) {
    inputWS = AnalysisDataService::Instance().retrieve(inputData);
  } else {
    std::string foundFile = FileFinder::Instance().getFullPath(inputData);
    if (foundFile.empty()) {
      // Get facility extensions
      FacilityInfo facilityInfo = ConfigService::Instance().getFacility();
      const std::vector<std::string> facilityExts = facilityInfo.extensions();
      foundFile = FileFinder::Instance().findRun(inputData, facilityExts);
    }
 
    if (!foundFile.empty()) {
      Poco::Path p(foundFile);
      const std::string outputWSName = p.getBaseName();
 
      auto loadAlg = createChildAlgorithm(m_loadAlg);
      loadAlg->setProperty(m_loadAlgFileProp, foundFile);
      if (!loadQuiet) {
        loadAlg->setAlwaysStoreInADS(true);
      }
 
// Set up MPI if available
#ifdef MPI_BUILD
      // First, check whether the loader allows use to chunk the data
      if (loadAlg->existsProperty("ChunkNumber") && loadAlg->existsProperty("TotalChunks")) {
        m_useMPI = true;
        // The communicator containing all processes
        boost::mpi::communicator world;
        g_log.notice() << "Chunk/Total: " << world.rank() + 1 << "/" << world.size() << '\n';
        loadAlg->setPropertyValue("OutputWorkspace", outputWSName);
        loadAlg->setProperty("ChunkNumber", world.rank() + 1);
        loadAlg->setProperty("TotalChunks", world.size());
      }
#endif
      loadAlg->execute();
 
      if (loadQuiet) {
        inputWS = loadAlg->getProperty("OutputWorkspace");
      } else {
        inputWS = AnalysisDataService::Instance().retrieve(outputWSName);
      }
    } else
      throw std::runtime_error("DataProcessorAlgorithm::load could process any data");
  }
  return inputWS;
}
 
template <class Base>
std::shared_ptr<PropertyManager>
GenericDataProcessorAlgorithm<Base>::getProcessProperties(const std::string &propertyManager) const {
  std::string propertyManagerName(propertyManager);
  if (propertyManager.empty() && (!m_propertyManagerPropertyName.empty())) {
    if (!Base::existsProperty(m_propertyManagerPropertyName)) {
      std::stringstream msg;
      msg << "Failed to find property \"" << m_propertyManagerPropertyName << "\"";
      throw Exception::NotFoundError(msg.str(), this->name());
    }
    propertyManagerName = this->getPropertyValue(m_propertyManagerPropertyName);
  }
 
  std::shared_ptr<PropertyManager> processProperties;
  if (PropertyManagerDataService::Instance().doesExist(propertyManagerName)) {
    processProperties = PropertyManagerDataService::Instance().retrieve(propertyManagerName);
  } else {
    Base::getLogger().notice() << "Could not find property manager\n";
    processProperties = std::make_shared<PropertyManager>();
    PropertyManagerDataService::Instance().addOrReplace(propertyManagerName, processProperties);
  }
  return processProperties;
}
 
template <class Base>
std::vector<std::string> GenericDataProcessorAlgorithm<Base>::splitInput(const std::string &input) {
  UNUSED_ARG(input);
  throw std::runtime_error("DataProcessorAlgorithm::splitInput is not implemented");
}
 
template <class Base> void GenericDataProcessorAlgorithm<Base>::forwardProperties() {
  throw std::runtime_error("DataProcessorAlgorithm::forwardProperties is not implemented");
}
 
//------------------------------------------------------------------------------------------
// Binary opration implementations for DPA so it can record history
//------------------------------------------------------------------------------------------
 
template <class Base>
MatrixWorkspace_sptr GenericDataProcessorAlgorithm<Base>::divide(const MatrixWorkspace_sptr lhs,
                                                                 const MatrixWorkspace_sptr rhs) {
  return this->executeBinaryAlgorithm<MatrixWorkspace_sptr, MatrixWorkspace_sptr, MatrixWorkspace_sptr>("Divide", lhs,
                                                                                                        rhs);
}
 
template <class Base>
MatrixWorkspace_sptr GenericDataProcessorAlgorithm<Base>::divide(const MatrixWorkspace_sptr lhs,
                                                                 const double &rhsValue) {
  return this->executeBinaryAlgorithm<MatrixWorkspace_sptr, MatrixWorkspace_sptr, MatrixWorkspace_sptr>(
      "Divide", lhs, createWorkspaceSingleValue(rhsValue));
}
 
template <class Base>
MatrixWorkspace_sptr GenericDataProcessorAlgorithm<Base>::multiply(const MatrixWorkspace_sptr lhs,
                                                                   const MatrixWorkspace_sptr rhs) {
  return this->executeBinaryAlgorithm<MatrixWorkspace_sptr, MatrixWorkspace_sptr, MatrixWorkspace_sptr>("Divide", lhs,
                                                                                                        rhs);
}
 
template <class Base>
MatrixWorkspace_sptr GenericDataProcessorAlgorithm<Base>::multiply(const MatrixWorkspace_sptr lhs,
                                                                   const double &rhsValue) {
  return this->executeBinaryAlgorithm<MatrixWorkspace_sptr, MatrixWorkspace_sptr, MatrixWorkspace_sptr>(
      "Multiply", lhs, createWorkspaceSingleValue(rhsValue));
}
 
template <class Base>
MatrixWorkspace_sptr GenericDataProcessorAlgorithm<Base>::plus(const MatrixWorkspace_sptr lhs,
                                                               const MatrixWorkspace_sptr rhs) {
  return this->executeBinaryAlgorithm<MatrixWorkspace_sptr, MatrixWorkspace_sptr, MatrixWorkspace_sptr>("Plus", lhs,
                                                                                                        rhs);
}
 
template <class Base>
MatrixWorkspace_sptr GenericDataProcessorAlgorithm<Base>::plus(const MatrixWorkspace_sptr lhs, const double &rhsValue) {
  return this->executeBinaryAlgorithm<MatrixWorkspace_sptr, MatrixWorkspace_sptr, MatrixWorkspace_sptr>(
      "Plus", lhs, createWorkspaceSingleValue(rhsValue));
}
 
template <class Base>
MatrixWorkspace_sptr GenericDataProcessorAlgorithm<Base>::minus(const MatrixWorkspace_sptr lhs,
                                                                const MatrixWorkspace_sptr rhs) {
  return this->executeBinaryAlgorithm<MatrixWorkspace_sptr, MatrixWorkspace_sptr, MatrixWorkspace_sptr>("Minus", lhs,
                                                                                                        rhs);
}
 
template <class Base>
MatrixWorkspace_sptr GenericDataProcessorAlgorithm<Base>::minus(const MatrixWorkspace_sptr lhs,
                                                                const double &rhsValue) {
  return this->executeBinaryAlgorithm<MatrixWorkspace_sptr, MatrixWorkspace_sptr, MatrixWorkspace_sptr>(
      "Minus", lhs, createWorkspaceSingleValue(rhsValue));
}
 
template <class Base>
MatrixWorkspace_sptr GenericDataProcessorAlgorithm<Base>::createWorkspaceSingleValue(const double &rhsValue) {
  MatrixWorkspace_sptr retVal = WorkspaceFactory::Instance().create("WorkspaceSingleValue", 1, 1, 1);
  retVal->dataY(0)[0] = rhsValue;
 
  return retVal;
}
 
template <typename T> void GenericDataProcessorAlgorithm<T>::visualStudioC4661Workaround() {}
 
template class GenericDataProcessorAlgorithm<Algorithm>;
template class MANTID_API_DLL GenericDataProcessorAlgorithm<SerialAlgorithm>;
template class MANTID_API_DLL GenericDataProcessorAlgorithm<ParallelAlgorithm>;
template class MANTID_API_DLL GenericDataProcessorAlgorithm<DistributedAlgorithm>;
 
template <> MANTID_API_DLL void GenericDataProcessorAlgorithm<Algorithm>::visualStudioC4661Workaround() {}
 
} // namespace Mantid::API