MultiFileNameParser.cpp

Go to the documentation of this file.

// Mantid Repository : https://github.com/mantidproject/mantid
//
// Copyright © 2018 ISIS Rutherford Appleton Laboratory UKRI,
//   NScD Oak Ridge National Laboratory, European Spallation Source,
//   Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
// SPDX - License - Identifier: GPL - 3.0 +
//----------------------------------------------------------------------
// Includes
//----------------------------------------------------------------------
#include "MantidKernel/MultiFileNameParser.h"
 
#include "MantidKernel/ConfigService.h"
#include "MantidKernel/FacilityInfo.h"
#include "MantidKernel/Strings.h"
 
#include <boost/algorithm/string.hpp>
 
#include <numeric>
#include <regex>
#include <sstream>
#include <utility>
 
namespace Mantid::Kernel::MultiFileNameParsing {
// Static constants.
 
namespace Regexs {
const std::string INST = "([A-Za-z]+|PG3|pg3)";
 
const std::string UNDERSCORE = "(_{0,1})";
const std::string SPACE = "(\\s*)";
 
const std::string COMMA = "(" + SPACE + "," + SPACE + ")";
const std::string PLUS = "(" + SPACE + "\\+" + SPACE + ")";
const std::string MINUS = "(" + SPACE + "\\-" + SPACE + ")";
const std::string COLON = "(" + SPACE + ":" + SPACE + ")";
 
const std::string SINGLE = "(" + INST + "*[0-9]+)";
const std::string RANGE = "(" + SINGLE + COLON + SINGLE + ")";
const std::string STEP_RANGE = "(" + SINGLE + COLON + SINGLE + COLON + SINGLE + ")";
const std::string ADD_RANGE = "(" + SINGLE + MINUS + SINGLE + ")";
const std::string ADD_STEP_RANGE = "(" + SINGLE + MINUS + SINGLE + COLON + SINGLE + ")";
const std::string SINGLE_OR_STEP_OR_ADD_RANGE = "(" + ADD_STEP_RANGE + "|" + ADD_RANGE + "|" + SINGLE + ")";
const std::string ADD_LIST = "(" + SINGLE_OR_STEP_OR_ADD_RANGE + "(" + PLUS + SINGLE_OR_STEP_OR_ADD_RANGE + ")+" + ")";
 
const std::string ANY =
    "(" + ADD_STEP_RANGE + "|" + ADD_LIST + "|" + ADD_RANGE + "|" + STEP_RANGE + "|" + RANGE + "|" + SINGLE + ")";
const std::string LIST = "(" + ANY + "(" + COMMA + ANY + ")*" + ")";
} // namespace Regexs
 
// Forward declarations.
 
namespace {
// Anonymous helper functions.
void parseToken(std::vector<std::vector<unsigned int>> &parsedRuns, const std::string &token);
std::vector<std::vector<unsigned int>> generateRange(const unsigned int from, const unsigned int to,
                                                     const unsigned int stepSize, const bool addRuns);
void validateToken(const std::string &token);
bool matchesFully(const std::string &stringToMatch, const std::string &regexString, const bool caseless = false);
std::string getMatchingString(const std::string &regexString, const std::string &toParse, const bool caseless = false);
std::string pad(const unsigned int run, const std::string &instString);
 
std::set<std::pair<unsigned int, unsigned int>>
mergeAdjacentRanges(std::set<std::pair<unsigned int, unsigned int>> ranges,
                    const std::pair<unsigned int, unsigned int> &range);
 
// Helper functor.
struct RangeContainsRun {
  bool operator()(const std::pair<unsigned int, unsigned int> &range, const unsigned int run);
  bool operator()(const unsigned int run, const std::pair<unsigned int, unsigned int> &range);
};
 
std::string toString(const RunRangeList &runRangeList);
std::string accumulateString(std::string output, std::pair<unsigned int, unsigned int> runRange);
} // namespace
 
// Scoped, global functions.
 
std::string suggestWorkspaceName(const std::vector<std::string> &fileNames) {
  Parser parser;
  RunRangeList runs;
 
  // For each file name, parse the run number out of it, and add it to a
  // RunRangeList.
  for (const auto &fileName : fileNames) {
    parser.parse(fileName);
    runs.addRun(parser.runs()[0][0]);
  }
 
  // Return the suggested ws name.
  return parser.instString() + parser.underscoreString() + toString(runs);
}
 
// Comparator class.
 
bool ReverseCaselessCompare::operator()(const std::string &a, const std::string &b) const {
  std::string lowerA;
  lowerA.resize(a.size());
  std::string lowerB;
  lowerB.resize(b.size());
 
  std::transform(a.cbegin(), a.cend(), lowerA.begin(), tolower);
  std::transform(b.cbegin(), b.cend(), lowerB.begin(), tolower);
 
  return lowerA > lowerB;
}
 
// Public member functions of Parser class.
 
Parser::Parser()
    : m_runs(), m_fileNames(), m_multiFileName(), m_dirString(), m_instString(), m_underscoreString(), m_runString(),
      m_extString(), m_validInstNames(), m_compoundLeadExtensions(), m_trimWhiteSpaces(true) {
  const ConfigServiceImpl &config = ConfigService::Instance();
 
  const auto facilities = config.getFacilities();
  for (const auto facility : facilities) {
    for (const auto &instrument : facility->instruments()) {
      m_validInstNames.insert(instrument.name());
      m_validInstNames.insert(instrument.shortName());
    }
 
    // Derive compound-extension leads from FileExtensions. For a two-component
    // entry like ".nxs.h5" the lead is ".nxs"; the parser uses this set in
    // split() to keep the inner extension out of the file stem. Entries with
    // a single dot (e.g. ".nxs", "_event.nxs") or more than two dots are
    // skipped — the parser only models two-component compounds.
    for (const auto &ext : facility->extensions()) {
      if (std::count(ext.cbegin(), ext.cend(), '.') != 2)
        continue;
      auto lead = ext.substr(0, ext.find_last_of('.'));
      if (!lead.empty() && lead.front() == '.')
        m_compoundLeadExtensions.insert(std::move(lead));
    }
  }
}
 
void Parser::parse(const std::string &multiFileName) {
  // Clear any contents of the member variables.
  clear();
 
  // Set the string to parse.
  m_multiFileName = multiFileName;
 
  // Split the string to be parsed into sections, and do some validation.
  split();
 
  // Parse the run section into unsigned ints we can use.
  m_runs = parseMultiRunString(m_runString);
 
  // Set up helper functor.
  GenerateFileName generateFileName(m_dirString, m_extString, m_instString);
 
  // Generate complete file names for each run using helper functor.
  std::transform(m_runs.begin(), m_runs.end(), std::back_inserter(m_fileNames), generateFileName);
}
 
std::vector<std::vector<unsigned int>> Parser::parseMultiRunString(std::string runString) {
  // If the run string is empty, return no runs.
  if (runString.empty())
    return std::vector<std::vector<unsigned int>>();
 
  // Remove whitespaces if requested.
  if (trimWhiteSpaces()) {
    runString.erase(std::remove_if( // ("Erase-remove" idiom.)
                        runString.begin(), runString.end(), isspace),
                    runString.end());
  }
  // Only numeric characters, or occurances of plus, minus, comma and colon are
  // allowed.
  if (!matchesFully(runString, "([0-9]|\\+|\\-|,|:)+")) {
    throw std::runtime_error("Non-numeric or otherwise unaccetable character(s) detected.");
  }
 
  // Tokenize on commas.
  std::vector<std::string> tokens;
  tokens = boost::split(tokens, runString, boost::is_any_of(","));
 
  // Validate each token.
  std::for_each(tokens.begin(), tokens.end(), validateToken);
 
  // Parse each token, accumulate the results, and return them.
  std::vector<std::vector<unsigned int>> runGroups;
  for (auto const &token : tokens) {
    parseToken(runGroups, token);
  }
  return runGroups;
}
 
bool Parser::trimWhiteSpaces() const { return m_trimWhiteSpaces; }
 
void Parser::setTrimWhiteSpaces(const bool &setting) { m_trimWhiteSpaces = setting; }
 
// Private member functions of Parser class.
 
void Parser::clear() {
  m_runs.clear();
  m_fileNames.clear();
  m_multiFileName.clear();
  m_dirString.clear();
  m_instString.clear();
  m_underscoreString.clear();
  m_runString.clear();
  m_extString.clear();
}
 
void Parser::split() {
  if (m_multiFileName.empty())
    throw std::runtime_error("No file name to parse.");
 
  // Clear whitespace before getting extentions and directories, if requested.
  if (trimWhiteSpaces()) {
    m_multiFileName.erase(std::remove_if( // ("Erase-remove" idiom.)
                              m_multiFileName.begin(), m_multiFileName.end(), isspace),
                          m_multiFileName.end());
  }
  // Get the directory, if there is one.
  const size_t lastSeparator = m_multiFileName.find_last_of("/\\");
  if (lastSeparator != std::string::npos)
    m_dirString = m_multiFileName.substr(0, lastSeparator + 1);
 
  // Get the extension. Compound extensions like ".nxs.h5" are kept whole when
  // the leading half (".nxs") matches a known compound lead derived from the
  // facility FileExtensions in Facilities.xml. Otherwise just the final dot's
  // suffix is taken. This avoids misidentifying stems that contain dots
  // (e.g. "run.v2.nxs") as compound-extension files.
  const size_t lastDot = m_multiFileName.find_last_of('.');
  if (lastDot != std::string::npos && lastDot > m_dirString.size()) {
    const size_t prevDot = m_multiFileName.find_last_of('.', lastDot - 1);
    if (prevDot != std::string::npos && prevDot >= m_dirString.size()) {
      const auto midExt = m_multiFileName.substr(prevDot, lastDot - prevDot);
      if (m_compoundLeadExtensions.contains(midExt))
        m_extString = m_multiFileName.substr(prevDot);
    }
    if (m_extString.empty())
      m_extString = m_multiFileName.substr(lastDot);
  }
 
  // If the directory contains an instance of a comma, then the string is
  // a comma separated list of single *full* file names to load.
  if (std::string::npos != m_dirString.find(','))
    throw std::runtime_error("Unable to parse.");
 
  // Slice off the directory and extension.
  std::string base =
      m_multiFileName.substr(m_dirString.size(), m_multiFileName.size() - (m_dirString.size() + m_extString.size()));
 
  if (base.empty())
    throw std::runtime_error("There does not appear to be any runs present.");
 
  auto instrumentNameIt = std::find_if(m_validInstNames.cbegin(), m_validInstNames.cend(), [&base](const auto &name) {
    return matchesFully(base, name + ".*", true);
  }); // USE CASELESS MATCHES HERE.
 
  // See if the user has typed in one of the available instrument names.
  if (instrumentNameIt != m_validInstNames.cend()) {
    m_instString = getMatchingString("^" + *instrumentNameIt, base, true);
  }
 
  // If not, use the default, or throw if we encounter an unrecognisable
  // non-numeric string.
  if (m_instString.empty()) {
    if (base.empty())
      throw std::runtime_error("There does not appear to be any runs present.");
 
    if (isdigit(base[0]))
      m_instString = ConfigService::Instance().getString("default.instrument");
    else
      throw std::runtime_error("There does not appear to be a valid instrument name present.");
  } else {
    // Chop off instrument name.
    base = base.substr(m_instString.size(), base.size());
  }
 
  if (base.empty())
    throw std::runtime_error("There does not appear to be any runs present.");
 
  const auto &instInfo = ConfigService::Instance().getInstrument(m_instString);
  // why?
  // m_instString = instInfo.shortName(); // Make sure we're using the shortened
  // form of the isntrument name.
 
  if (base.starts_with(instInfo.delimiter())) {
    // Store the instrument delimiter, and strip it off the start of the string.
    m_underscoreString = instInfo.delimiter();
    base = base.substr(m_underscoreString.size(), base.size());
  }
 
  // LIST requires at least one run number ([0-9]+) so a base with no digits
  // can never match. Skip the expensive regex and fail fast.
  if (base.find_first_of("0123456789") == std::string::npos)
    throw std::runtime_error("There do not appear to be any runs present.");
 
  m_runString = getMatchingString("^" + Regexs::LIST, base);
 
  if (m_runString.size() != base.size()) {
    throw std::runtime_error("There is an unparsable token present.");
  }
}
 
// Helper functor.
 
GenerateFileName::GenerateFileName(std::string prefix, std::string suffix, std::string instString)
    : m_prefix(std::move(prefix)), m_suffix(std::move(suffix)), m_instString(std::move(instString)) {}
 
std::vector<std::string> GenerateFileName::operator()(const std::vector<unsigned int> &runs) {
  std::vector<std::string> fileNames;
 
  std::transform(runs.begin(), runs.end(), std::back_inserter(fileNames),
                 (*this) // Call other overloaded function operator.
  );
 
  return fileNames;
}
 
std::string GenerateFileName::operator()(const unsigned int run) {
  std::stringstream fileName;
 
  fileName << m_prefix << pad(run, m_instString) << m_suffix;
 
  return fileName.str();
}
 
// Public member functions of RunRangeList class.
 
RunRangeList::RunRangeList() : m_rangeList() {}
 
void RunRangeList::addRun(const unsigned int run) {
  // If the run is inside one of the ranges, do nothing.
  if (std::binary_search(m_rangeList.begin(), m_rangeList.end(), run, RangeContainsRun()))
    return;
 
  // Else create a new range, containing a single run, and add it to the list.
  m_rangeList.emplace(run, run);
 
  // Now merge any ranges that are adjacent.
  m_rangeList = std::accumulate(m_rangeList.begin(), m_rangeList.end(),
                                std::set<std::pair<unsigned int, unsigned int>>(), mergeAdjacentRanges);
}
 
void RunRangeList::addRunRange(unsigned int from, unsigned int to) {
  for (; from <= to; ++from)
    addRun(from);
}
 
void RunRangeList::addRunRange(const std::pair<unsigned int, unsigned int> &range) {
  addRunRange(range.first, range.second);
}
 
// Anonymous helper functions.
 
namespace // anonymous
{
void parseToken(std::vector<std::vector<unsigned int>> &parsedRuns, const std::string &token) {
  std::vector<std::vector<unsigned int>> runs;
  // Tokenise further on plus.
  std::vector<std::string> subTokens;
  boost::split(subTokens, token, boost::is_any_of("+"));
  std::vector<unsigned int> runsToAdd;
  for (auto const &subToken : subTokens) {
    // E.g. "2012".
    if (matchesFully(subToken, Regexs::SINGLE)) {
      runsToAdd.emplace_back(std::stoi(subToken));
    }
    // E.g. "2012:2020".
    else if (matchesFully(subToken, Regexs::RANGE)) {
      // Fill in runs directly.
      constexpr bool addRuns{false};
      std::vector<std::string> rangeDetails;
      rangeDetails.reserve(2);
      boost::split(rangeDetails, subToken, boost::is_any_of(":"));
      runs = generateRange(std::stoi(rangeDetails.front()), std::stoi(rangeDetails.back()), 1, addRuns);
    }
    // E.g. "2012:2020:4".
    else if (matchesFully(subToken, Regexs::STEP_RANGE)) {
      // Fill in runs directly.
      constexpr bool addRuns{false};
      std::vector<std::string> rangeDetails;
      rangeDetails.reserve(3);
      boost::split(rangeDetails, subToken, boost::is_any_of(":"));
      runs = generateRange(std::stoi(rangeDetails[0]), std::stoi(rangeDetails[1]), std::stoi(rangeDetails[2]), addRuns);
    }
    // E.g. "2012-2020".
    else if (matchesFully(subToken, Regexs::ADD_RANGE)) {
      constexpr bool addRuns{true};
      std::vector<std::string> rangeDetails;
      rangeDetails.reserve(2);
      boost::split(rangeDetails, subToken, boost::is_any_of("-"));
      const auto generated = generateRange(std::stoi(rangeDetails.front()), std::stoi(rangeDetails.back()), 1, addRuns);
      std::copy(generated.front().cbegin(), generated.front().cend(), back_inserter(runsToAdd));
    }
    // E.g. "2012-2020:4".
    else if (matchesFully(subToken, Regexs::ADD_STEP_RANGE)) {
      constexpr bool addRuns{true};
      std::vector<std::string> rangeDetails;
      rangeDetails.reserve(3);
      boost::split(rangeDetails, subToken, boost::is_any_of("-:"));
      const auto generated =
          generateRange(std::stoi(rangeDetails[0]), std::stoi(rangeDetails[1]), std::stoi(rangeDetails[2]), addRuns);
      std::copy(generated.front().cbegin(), generated.front().cend(), back_inserter(runsToAdd));
    } else {
      // We should never reach here - the validation done on the token
      // previously should prevent any other possible scenario.
      assert(false);
    }
  }
  if (!runsToAdd.empty()) {
    if (!runs.empty()) {
      // We have either add ranges or step ranges. Never both.
      throw std::runtime_error("Unable to handle a mixture of add ranges and step ranges");
    }
    runs.emplace_back(runsToAdd);
  }
  // Add the runs on to the end of parsedRuns, and return it.
  std::copy(runs.begin(), runs.end(), std::back_inserter(parsedRuns));
}
 
std::vector<std::vector<unsigned int>> generateRange(unsigned int const from, unsigned int const to,
                                                     unsigned int const stepSize, bool const addRuns) {
  if (stepSize == 0)
    throw std::runtime_error("Unable to generate a range with a step size of zero.");
 
  size_t limit;
  auto limitStr = ConfigService::Instance().getValue<std::string>("loading.multifilelimit");
  if (!limitStr.has_value() || !Strings::convert(limitStr.value(), limit)) {
    limit = ConfigService::Instance().getFacility().multiFileLimit();
  }
 
  unsigned int const orderedTo = from > to ? from : to;
  unsigned int const orderedFrom = from > to ? to : from;
  unsigned int const numberOfFiles = (orderedTo - orderedFrom) / stepSize;
  if (numberOfFiles > limit) {
    std::stringstream sstream;
    sstream << "The range from " << orderedFrom << " to " << orderedTo << " with step " << stepSize
            << " would generate " << numberOfFiles << " files.  "
            << "This is greater than the current limit of " << limit << ".  "
            << "This limit can be configured in the Mantid.user.properties "
               "file using the key loading.multifilelimit=200.";
    throw std::range_error(sstream.str());
  }
 
  unsigned int currentRun = from;
  std::vector<std::vector<unsigned int>> runs;
 
  // If ascending range
  if (from <= to) {
    while (currentRun <= to) {
      if (addRuns) {
        if (runs.empty())
          runs.emplace_back(1, currentRun);
        else
          runs.front().emplace_back(currentRun);
      } else {
        runs.emplace_back(1, currentRun);
      }
 
      currentRun += stepSize;
    }
  }
  // Else descending range
  else {
    while (currentRun >= to) {
      if (addRuns) {
        if (runs.empty())
          runs.emplace_back(1, currentRun);
        else
          runs.front().emplace_back(currentRun);
      } else {
        runs.emplace_back(1, currentRun);
      }
 
      // Guard against case where stepSize would take us into negative
      // numbers (not supported by unsigned ints ...).
      if (static_cast<int>(currentRun) - static_cast<int>(stepSize) < 0)
        break;
 
      currentRun -= stepSize;
    }
  }
 
  return runs;
}
 
void validateToken(const std::string &token) {
  // Each token must be non-empty.
  if (token.empty())
    throw std::runtime_error("A comma-separated token is empty.");
 
  // Each token must begin and end with a numeric character.
  if (!matchesFully(token, "[0-9].+[0-9]|[0-9]"))
    throw std::runtime_error("The token \"" + token +
                             "\" is of an incorrect form.  Does it begin or "
                             "end with a plus, minus or colon?");
 
  // Each token must be one of the acceptable forms, i.e. a single run, an added
  // range of runs, etc.
  if (!matchesFully(token, Regexs::ANY))
    throw std::runtime_error("The token \"" + token + "\" is of an incorrect form.");
}
 
bool matchesFully(const std::string &stringToMatch, const std::string &regexString, const bool caseless) {
  const auto flags = caseless ? std::regex::ECMAScript | std::regex::icase : std::regex::ECMAScript;
  const std::regex regex("^(" + regexString + "$)", flags);
  return std::regex_match(stringToMatch, regex);
}
 
std::string getMatchingString(const std::string &regexString, const std::string &toParse, const bool caseless) {
  const auto flags = caseless ? std::regex::ECMAScript | std::regex::icase : std::regex::ECMAScript;
  const std::regex regex(regexString, flags);
 
  std::sregex_iterator it(toParse.begin(), toParse.end(), regex);
  if (it == std::sregex_iterator())
    return "";
 
  return it->str();
}
 
std::string pad(const unsigned int run, const std::string &instString) {
  InstrumentInfo const instInfo = ConfigService::Instance().getInstrument(instString);
  std::string prefix;
  if (!instInfo.facility().noFilePrefix())
    prefix = instInfo.filePrefix(run) + instInfo.delimiter();
  unsigned int padLength = instInfo.zeroPadding(run);
  std::string runStr = std::to_string(run);
  if (runStr.size() < padLength)
    runStr.insert(0, padLength - runStr.size(), '0');
  else if (padLength > 0 && runStr.size() > padLength)
    throw std::runtime_error("Could not parse run number \"" + runStr +
                             "\" since the instrument run number length required is " + std::to_string(padLength));
  runStr.insert(0, prefix);
  return runStr;
}
 
bool RangeContainsRun::operator()(const std::pair<unsigned int, unsigned int> &range, const unsigned int run) {
  return range.second < run;
}
bool RangeContainsRun::operator()(const unsigned int run, const std::pair<unsigned int, unsigned int> &range) {
  return run < range.first;
}
 
std::set<std::pair<unsigned int, unsigned int>>
mergeAdjacentRanges(std::set<std::pair<unsigned int, unsigned int>> ranges,
                    const std::pair<unsigned int, unsigned int> &range) {
  // If ranges is empty, just insert the new range.
  if (ranges.empty()) {
    ranges.insert(range);
  }
  // Else there are already some ranges present ...
  else {
    // ... if the last one is adjacent to the new range, merge the two.
    if (ranges.rbegin()->second + 1 == range.first) {
      unsigned int from = ranges.rbegin()->first;
      unsigned int to = range.second;
      std::pair<unsigned int, unsigned int> temp(from, to);
 
      ranges.erase(--ranges.end(), ranges.end());
      ranges.insert(temp);
    }
    // ... else just insert it.
    else {
      ranges.insert(range);
    }
  }
  return ranges;
}
 
std::string accumulateString(std::string output, std::pair<unsigned int, unsigned int> runRange) {
  if (!output.empty())
    output += "_and_";
 
  if (runRange.first == runRange.second)
    output += std::to_string(runRange.first);
  else
    output += std::to_string(runRange.first) + "_to_" + std::to_string(runRange.second);
 
  return output;
}
 
std::string toString(const RunRangeList &runRangeList) {
  std::set<std::pair<unsigned int, unsigned int>> runRanges = runRangeList.rangeList();
 
  // For each run range (pair of unsigned ints), call accumulateString and
  // return the accumulated result.
  return std::accumulate(runRanges.begin(), runRanges.end(), std::string(), accumulateString);
}
 
} // anonymous namespace
 
} // namespace Mantid::Kernel::MultiFileNameParsing