CatalogDownloadDataFiles.cpp

Go to the documentation of this file.

// Mantid Repository : https://github.com/mantidproject/mantid
//
// Copyright © 2018 ISIS Rutherford Appleton Laboratory UKRI,
//   NScD Oak Ridge National Laboratory, European Spallation Source,
//   Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
// SPDX - License - Identifier: GPL - 3.0 +
#include "MantidICat/CatalogDownloadDataFiles.h"
#include "MantidAPI/CatalogManager.h"
#include "MantidAPI/ICatalogInfoService.h"
#include "MantidAPI/WorkspaceProperty.h"
#include "MantidICat/CatalogAlgorithmHelper.h"
#include "MantidKernel/ArrayProperty.h"
#include "MantidKernel/BoundedValidator.h"
#include "MantidKernel/CatalogInfo.h"
#include "MantidKernel/ConfigService.h"
#include "MantidKernel/ICatalogInfo.h"
#include "MantidKernel/PropertyWithValue.h"
#include "MantidKernel/UserCatalogInfo.h"
 
#include <Poco/Net/AcceptCertificateHandler.h>
#include <Poco/Net/HTTPRequest.h>
#include <Poco/Net/HTTPResponse.h>
#include <Poco/Net/HTTPSClientSession.h>
#include <Poco/Net/PrivateKeyPassphraseHandler.h>
#include <Poco/Net/SSLException.h>
#include <Poco/Net/SSLManager.h>
#include <Poco/Net/SecureStreamSocket.h>
#include <Poco/Path.h>
#include <Poco/StreamCopier.h>
#include <Poco/URI.h>
 
#include <fstream>
#include <iomanip>
#include <memory>
 
namespace Mantid::ICat {
using namespace Kernel;
using namespace API;
 
DECLARE_ALGORITHM(CatalogDownloadDataFiles)
 
 
void CatalogDownloadDataFiles::init() {
  declareProperty(std::make_unique<ArrayProperty<int64_t>>("FileIds"),
                  "List of fileids to download from the data server");
  declareProperty(std::make_unique<ArrayProperty<std::string>>("FileNames"),
                  "List of filenames to download from the data server");
  declareProperty("DownloadPath", "", "The path to save the downloaded files.");
  declareProperty("Session", "", "The session information of the catalog to use.");
  declareProperty(std::make_unique<ArrayProperty<std::string>>("FileLocations", std::vector<std::string>(),
                                                               std::make_shared<NullValidator>(), Direction::Output),
                  "A list of file locations to the catalog datafiles.");
}
 
void CatalogDownloadDataFiles::exec() {
  // Cast a catalog to a catalogInfoService to access downloading functionality.
  auto catalogInfoService = std::dynamic_pointer_cast<API::ICatalogInfoService>(
      API::CatalogManager::Instance().getCatalog(getPropertyValue("Session")));
  // Check if the catalog created supports publishing functionality.
  if (!catalogInfoService)
    throw std::runtime_error("The catalog that you are using does not support "
                             "external downloading.");
 
  std::unique_ptr<CatalogConfigService> catConfigService(makeCatalogConfigServiceAdapter(ConfigService::Instance()));
  UserCatalogInfo catalogInfo(ConfigService::Instance().getFacility().catalogInfo(), *catConfigService);
 
  std::vector<int64_t> fileIDs = getProperty("FileIds");
  std::vector<std::string> fileNames = getProperty("FileNames");
 
  // Stores the paths to the related files located in the archives (if user has
  // access).
  // Otherwise, stores the path to the downloaded file.
  std::vector<std::string> fileLocations;
 
  m_prog = 0.0;
 
  std::vector<int64_t>::const_iterator fileID = fileIDs.begin();
  std::vector<std::string>::const_iterator fileName = fileNames.begin();
 
  // For every file with the given ID.
  for (; fileID != fileIDs.end(); ++fileID, ++fileName) {
    m_prog += 0.1;
    double prog = m_prog / (double(fileIDs.size()) / 10);
 
    progress(prog, "getting location string...");
 
    // The location of the file (on the server) stored in the archives.
    std::string fileLocation = catalogInfoService->getFileLocation(*fileID);
 
    g_log.debug() << "CatalogDownloadDataFiles -> File location before transform is: " << fileLocation
                  << " mac path is : " << catalogInfo.macPrefix() << '\n';
    // Transform the archive path to the path of the user's operating system.
    fileLocation = catalogInfo.transformArchivePath(fileLocation);
    g_log.debug() << "CatalogDownloadDataFiles -> File location after transform is:  " << fileLocation << '\n';
 
    // Can we open the file (Hence, have access to the archives?)
    std::ifstream hasAccessToArchives(fileLocation.c_str());
    if (hasAccessToArchives) {
      g_log.information() << "File (" << *fileName << ") located in archives (" << fileLocation << ").\n";
      fileLocations.emplace_back(fileLocation);
    } else {
      g_log.information() << "Unable to open file (" << *fileName
                          << ") from archive. Beginning to download over Internet.\n";
      progress(prog / 2, "getting the url ....");
      // Obtain URL for related file to download from net.
      const std::string url = catalogInfoService->getDownloadURL(*fileID);
      progress(prog, "downloading over internet...");
      const std::string fullPathDownloadedFile = doDownloadandSavetoLocalDrive(url, *fileName);
      fileLocations.emplace_back(fullPathDownloadedFile);
    }
  }
 
  // Set the fileLocations property
  setProperty("FileLocations", fileLocations);
}
 
bool CatalogDownloadDataFiles::isDataFile(const std::string &fileName) {
  std::string extension = Poco::Path(fileName).getExtension();
  std::transform(extension.begin(), extension.end(), extension.begin(), tolower);
  return (extension == "raw" || extension == "nxs");
}
 
std::string CatalogDownloadDataFiles::doDownloadandSavetoLocalDrive(const std::string &URL,
                                                                    const std::string &fileName) {
  std::string pathToDownloadedDatafile;
 
  try {
    Poco::URI uri(URL);
 
    std::string path(uri.getPathAndQuery());
    clock_t start = clock();
 
    Poco::SharedPtr<Poco::Net::InvalidCertificateHandler> certificateHandler =
        new Poco::Net::AcceptCertificateHandler(true);
    // Currently do not use any means of authentication. This should be updated
    // IDS has signed certificate.
    const Poco::Net::Context::Ptr context =
        new Poco::Net::Context(Poco::Net::Context::CLIENT_USE, "", "", "", Poco::Net::Context::VERIFY_NONE);
    // Create a singleton for holding the default context. E.g. any future
    // requests to publish are made to this certificate and context.
    Poco::Net::SSLManager::instance().initializeClient(nullptr, certificateHandler, context);
 
    // Session takes ownership of socket
    Poco::Net::SecureStreamSocket socket{context};
    Poco::Net::HTTPSClientSession session{socket};
    session.setHost(uri.getHost());
    session.setPort(uri.getPort());
 
    Poco::Net::HTTPRequest request(Poco::Net::HTTPRequest::HTTP_GET, path, Poco::Net::HTTPMessage::HTTP_1_1);
    session.sendRequest(request);
 
    // Close the request by requesting a response.
    Poco::Net::HTTPResponse response;
    // Store the response for use IF an error occurs (e.g. 404).
    std::istream &responseStream = session.receiveResponse(response);
 
    // Obtain the status returned by the server to verify if it was a success.
    Poco::Net::HTTPResponse::HTTPStatus HTTPStatus = response.getStatus();
    // The error message returned by the IDS (if one exists).
    std::string IDSError = CatalogAlgorithmHelper().getIDSError(HTTPStatus, responseStream);
    // Cancel the algorithm and display the message if it exists.
    if (!IDSError.empty()) {
      // As an error occurred we must cancel the algorithm to prevent success
      // message.
      this->cancel();
      // Output an appropriate error message from the JSON object returned by
      // the IDS.
      g_log.error(IDSError);
      return "";
    }
 
    // Save the file to local disk if no errors occurred on the IDS.
    pathToDownloadedDatafile = saveFiletoDisk(responseStream, fileName);
 
    clock_t end = clock();
    float diff = float(end - start) / CLOCKS_PER_SEC;
    g_log.information() << "Time taken to download file " << fileName << " is " << std::fixed << std::setprecision(2)
                        << diff << " seconds\n";
 
  } catch (Poco::Net::SSLException &error) {
    throw std::runtime_error(error.displayText());
  }
  // A strange error occurs (what returns: {I/O error}, while message returns: {
  // 9: The BIO reported an error }.
  // This bug has been fixed in POCO 1.4 and is noted -
  // http://sourceforge.net/p/poco/bugs/403/
  // I have opted to catch the exception and do nothing as this allows the
  // load/download functionality to work.
  // However, the port the user used to download the file will be left open.
  catch (Poco::Exception &error) {
    throw std::runtime_error(error.displayText());
  }
 
  return pathToDownloadedDatafile;
}
 
std::string CatalogDownloadDataFiles::saveFiletoDisk(std::istream &rs, const std::string &fileName) {
  std::string filepath = Poco::Path(getPropertyValue("DownloadPath"), fileName).toString();
  std::ios_base::openmode mode = isDataFile(fileName) ? std::ios_base::binary : std::ios_base::out;
 
  std::ofstream ofs(filepath.c_str(), mode);
  if (ofs.rdstate() & std::ios::failbit)
    throw Exception::FileError("Error on creating File", fileName);
  // copy the input stream to a file.
  Poco::StreamCopier::copyStream(rs, ofs);
 
  return filepath;
}
 
std::string CatalogDownloadDataFiles::testDownload(const std::string &URL, const std::string &fileName) {
  return doDownloadandSavetoLocalDrive(URL, fileName);
}
} // namespace Mantid::ICat