InternetHelper.cpp

Go to the documentation of this file.

// Mantid Repository : https://github.com/mantidproject/mantid
//
// Copyright © 2018 ISIS Rutherford Appleton Laboratory UKRI,
//   NScD Oak Ridge National Laboratory, European Spallation Source,
//   Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
// SPDX - License - Identifier: GPL - 3.0 +
#include "MantidKernel/InternetHelper.h"
#include "MantidKernel/ConfigService.h"
#include "MantidKernel/DateAndTime.h"
#include "MantidKernel/Exception.h"
#include "MantidKernel/Logger.h"
#include "MantidKernel/MantidVersion.h"
 
// Poco
#include <Poco/Net/AcceptCertificateHandler.h>
#include <Poco/Net/HTMLForm.h>
#include <Poco/Net/HTTPRequest.h>
#include <Poco/Net/HTTPResponse.h>
#include <Poco/Net/HTTPSClientSession.h>
#include <Poco/Net/NetException.h>
#include <Poco/Net/PrivateKeyPassphraseHandler.h>
#include <Poco/Net/SSLManager.h>
#include <Poco/StreamCopier.h>
#include <Poco/TemporaryFile.h>
#include <Poco/URI.h>
 
#include <Poco/Exception.h>
#include <Poco/Net/Context.h>
#include <Poco/Net/HTTPClientSession.h>
#include <Poco/Net/HTTPMessage.h>
#include <Poco/Net/InvalidCertificateHandler.h>
#include <Poco/SharedPtr.h>
#include <Poco/Timespan.h>
#include <Poco/Types.h>
 
#if defined(_WIN32) || defined(_WIN64)
#include <Winhttp.h>
#endif
 
#include <boost/lexical_cast.hpp>
 
// std
#include <filesystem>
#include <fstream>
#include <mutex>
#include <utility>
 
namespace Mantid {
using namespace Types::Core;
namespace Kernel {
 
using namespace Poco::Net;
using std::map;
using std::string;
 
namespace {
// anonymous namespace for some utility functions
Logger g_log("InternetHelper");
 
std::once_flag SSL_INIT_FLAG;
 
void doSSLInit() {
  // initialize ssl
  Poco::SharedPtr<InvalidCertificateHandler> certificateHandler = new AcceptCertificateHandler(true);
  // Currently do not use any means of authentication. This should be updated
  // IDS has signed certificate.
  const Context::Ptr context = new Context(Context::CLIENT_USE, "", "", "", Context::VERIFY_NONE);
  // Create a singleton for holding the default context.
  // e.g. any future requests to publish are made to this certificate and
  // context.
  SSLManager::instance().initializeClient(nullptr, certificateHandler, context);
}
 
void initializeSSL() { std::call_once(SSL_INIT_FLAG, doSSLInit); }
} // namespace
 
//----------------------------------------------------------------------------------------------
InternetHelper::InternetHelper()
    : m_proxyInfo(), m_isProxySet(false), m_timeout(30), m_isTimeoutSet(false), m_contentLength(0),
      m_method(HTTPRequest::HTTP_GET), m_contentType("application/json"), m_body(), m_headers(), m_request(nullptr),
      m_response(nullptr) {}
 
//----------------------------------------------------------------------------------------------
InternetHelper::InternetHelper(const Kernel::ProxyInfo &proxy)
    : m_proxyInfo(proxy), m_isProxySet(true), m_timeout(30), m_isTimeoutSet(false), m_contentLength(0),
      m_method(HTTPRequest::HTTP_GET), m_contentType("application/json"), m_body(), m_headers(), m_request(nullptr),
      m_response(nullptr) {}
 
//----------------------------------------------------------------------------------------------
InternetHelper::~InternetHelper() = default;
 
void InternetHelper::setupProxyOnSession(HTTPClientSession &session, const std::string &proxyUrl) {
  auto proxy = this->getProxy(proxyUrl);
  if (!proxy.emptyProxy()) {
    session.setProxyHost(proxy.host());
    session.setProxyPort(static_cast<Poco::UInt16>(proxy.port()));
  }
}
 
void InternetHelper::createRequest(Poco::URI &uri) {
  m_request = std::make_unique<HTTPRequest>(m_method, uri.getPathAndQuery(), HTTPMessage::HTTP_1_1);
  m_response = std::make_unique<HTTPResponse>();
  if (!m_contentType.empty()) {
    m_request->setContentType(m_contentType);
  }
 
  m_request->set("User-Agent",
                 // Use standard User-Agent format as per MDN documentation.
                 std::string("Mantid/") + MantidVersion::version());
  if (m_method == "POST") {
    // HTTP states that the 'Content-Length' header should not be included
    // if the 'Transfer-Encoding' header is set. UNKNOWN_CONTENT_LENGTH
    // indicates to Poco to remove the header field
    m_request->setContentLength(HTTPMessage::UNKNOWN_CONTENT_LENGTH);
    m_request->setChunkedTransferEncoding(true);
  } else if (m_contentLength > 0) {
    m_request->setContentLength(m_contentLength);
  }
 
  for (auto &header : m_headers) {
    m_request->set(header.first, header.second);
  }
}
 
InternetHelper::HTTPStatus InternetHelper::sendRequestAndProcess(HTTPClientSession &session, Poco::URI &uri,
                                                                 std::ostream &responseStream) {
  // create a request
  this->createRequest(uri);
  session.sendRequest(*m_request) << m_body;
 
  // do not dereference null pointer below
  if (!m_response) {
    g_log.warning("Response is null pointer");
    return HTTPStatus::BAD_REQUEST; // generic error
  }
 
  // process the response
  std::istream &rs = session.receiveResponse(*m_response);
  const auto retStatus = static_cast<HTTPStatus>(m_response->getStatus());
  g_log.debug() << "Answer from web: " << static_cast<int>(retStatus) << " " << m_response->getReason() << '\n';
  if (retStatus == HTTPStatus::OK || (retStatus == HTTPStatus::CREATED && m_method == HTTPRequest::HTTP_POST)) {
    Poco::StreamCopier::copyStream(rs, responseStream);
    processResponseHeaders(*m_response);
    return retStatus;
  } else if (isRelocated(retStatus)) {
    return this->processRelocation(*m_response, responseStream);
  } else {
    Poco::StreamCopier::copyStream(rs, responseStream);
    return processErrorStates(*m_response, rs, uri.toString());
  }
}
 
InternetHelper::HTTPStatus InternetHelper::processRelocation(const HTTPResponse &response,
                                                             std::ostream &responseStream) {
  std::string newLocation = response.get("location", "");
  if (!newLocation.empty()) {
    g_log.information() << "url relocated to " << newLocation << "\n";
    return this->sendRequest(newLocation, responseStream);
  } else {
    g_log.warning("Apparent relocation did not give new location\n");
    return static_cast<HTTPStatus>(response.getStatus());
  }
}
 
InternetHelper::HTTPStatus InternetHelper::sendRequest(const std::string &url, std::ostream &responseStream) {
 
  // send the request
  Poco::URI uri(url);
  if (uri.getPath().empty())
    uri = url + "/";
  if ((uri.getScheme() == "https") || (uri.getPort() == 443)) {
    return static_cast<HTTPStatus>(sendHTTPSRequest(uri.toString(), responseStream));
  } else {
    return static_cast<HTTPStatus>(sendHTTPRequest(uri.toString(), responseStream));
  }
}
 
void InternetHelper::logDebugRequestSending(const std::string &schemeName, const std::string &url) const {
  const std::string insecString = "password=";
  if (std::string::npos == url.find(insecString)) {
    g_log.debug() << "Sending " << schemeName << " " << m_method << " request to: " << url << "\n";
  } else {
    g_log.debug() << "Sending " << schemeName << " " << m_method
                  << " request to an url where the query string seems to contain a "
                     "password! (not shown for security reasons)."
                  << "\n";
  }
}
 
InternetHelper::HTTPStatus InternetHelper::sendHTTPRequest(const std::string &url, std::ostream &responseStream) {
  InternetHelper::HTTPStatus retStatus{InternetHelper::HTTPStatus::BAD_REQUEST};
  logDebugRequestSending("http", url);
 
  Poco::URI uri(url);
  // Configure Poco HTTP Client Session
  try {
    Poco::Net::HTTPClientSession session(uri.getHost(), uri.getPort());
    session.setTimeout(Poco::Timespan(getTimeout(), 0));
 
    // configure proxy
    setupProxyOnSession(session, url);
 
    // low level sending the request
    retStatus = this->sendRequestAndProcess(session, uri, responseStream);
  } catch (HostNotFoundException &ex) {
    throwNotConnected(url, ex);
  } catch (Poco::Exception &ex) {
    throw Exception::InternetError("Connection and request failed " + ex.displayText());
  }
  return retStatus;
}
 
InternetHelper::HTTPStatus InternetHelper::sendHTTPSRequest(const std::string &url, std::ostream &responseStream) {
  InternetHelper::HTTPStatus retStatus{InternetHelper::HTTPStatus::BAD_REQUEST};
 
  logDebugRequestSending("https", url);
 
  Poco::URI uri(url);
  try {
    initializeSSL();
    // Create the session
    HTTPSClientSession session(uri.getHost(), static_cast<Poco::UInt16>(uri.getPort()));
    session.setTimeout(Poco::Timespan(getTimeout(), 0));
 
    // HACK:: Currently the automatic proxy detection only supports http proxy
    // detection
    // most locations use the same proxy for http and https, so force it to use
    // the http proxy
    std::string urlforProxy = ConfigService::Instance().getString("proxy.httpsTargetUrl");
    if (urlforProxy.empty()) {
      urlforProxy = "http://" + uri.getHost();
    }
    setupProxyOnSession(session, urlforProxy);
 
    // low level sending the request
    retStatus = this->sendRequestAndProcess(session, uri, responseStream);
  } catch (HostNotFoundException &ex) {
    throwNotConnected(url, ex);
  } catch (Poco::Exception &ex) {
    throw Exception::InternetError("Connection and request failed " + ex.displayText());
  }
  return retStatus;
}
 
Kernel::ProxyInfo &InternetHelper::getProxy(const std::string &url) {
  // set the proxy
  if (!m_isProxySet) {
    setProxy(ConfigService::Instance().getProxy(url));
  }
  return m_proxyInfo;
}
 
void InternetHelper::clearProxy() { m_isProxySet = false; }
 
void InternetHelper::setProxy(const Kernel::ProxyInfo &proxy) {
  m_proxyInfo = proxy;
  m_isProxySet = true;
}
 
void InternetHelper::processResponseHeaders(const HTTPResponse & /*unused*/) {}
 
InternetHelper::HTTPStatus InternetHelper::processErrorStates(const HTTPResponse &res, std::istream &rs,
                                                              const std::string &url) {
  const auto retStatus = static_cast<HTTPStatus>(res.getStatus());
  g_log.debug() << "Answer from web: " << static_cast<int>(res.getStatus()) << " " << res.getReason() << '\n';
 
  // get github api rate limit information if available;
  int rateLimitRemaining;
  DateAndTime rateLimitReset;
  try {
    rateLimitRemaining = boost::lexical_cast<int>(res.get("X-RateLimit-Remaining", "-1"));
    rateLimitReset.set_from_time_t(boost::lexical_cast<int>(res.get("X-RateLimit-Reset", "0")));
  } catch (boost::bad_lexical_cast const &) {
    rateLimitRemaining = -1;
  }
 
  if (retStatus == HTTPStatus::OK) {
    throw Exception::InternetError("Response was ok, processing should never "
                                   "have entered processErrorStates",
                                   static_cast<int>(retStatus));
  } else if (retStatus == HTTPStatus::FOUND) {
    throw Exception::InternetError("Response was HTTP_FOUND, processing should "
                                   "never have entered processErrorStates",
                                   static_cast<int>(retStatus));
  } else if (retStatus == HTTPStatus::MOVED_PERMANENTLY) {
    throw Exception::InternetError("Response was HTTP_MOVED_PERMANENTLY, "
                                   "processing should never have entered "
                                   "processErrorStates",
                                   static_cast<int>(retStatus));
  } else if (retStatus == HTTPStatus::NOT_MODIFIED) {
    throw Exception::InternetError("Not modified since provided date" + rateLimitReset.toSimpleString(),
                                   static_cast<int>(retStatus));
  } else if ((retStatus == HTTPStatus::FORBIDDEN) && (rateLimitRemaining == 0)) {
    throw Exception::InternetError("The Github API rate limit has been reached, try again after " +
                                       rateLimitReset.toSimpleString() + " GMT",
                                   static_cast<int>(retStatus));
  } else {
    std::stringstream info;
    std::stringstream ss;
    Poco::StreamCopier::copyStream(rs, ss);
    if (retStatus == HTTPStatus::NOT_FOUND)
      info << "Failed to download " << url << " with the link "
           << "<a href=\"" << url << "\">.\n"
           << "Hint. Check that link is correct</a>";
    else {
      // show the error
      info << res.getReason();
      info << ss.str();
      g_log.debug() << ss.str();
    }
    throw Exception::InternetError(info.str() + ss.str(), static_cast<int>(retStatus));
  }
  return retStatus; // must return to follow contract
}
 
InternetHelper::HTTPStatus InternetHelper::downloadFile(const std::string &urlFile, const std::string &localFilePath) {
  g_log.debug() << "DownloadFile from \"" << urlFile << "\" to file: \"" << localFilePath << "\"\n";
 
  Poco::TemporaryFile tempFile;
  std::ofstream tempFileStream(tempFile.path());
  const auto retStatus = sendRequest(urlFile, tempFileStream);
  tempFileStream.close();
 
  // if there have been no errors move it to the final location, and turn off
  // automatic deletion.
  // clear the way if the target file path is already in use
  if (std::filesystem::exists(localFilePath)) {
    std::filesystem::remove(localFilePath);
  }
 
  tempFile.moveTo(localFilePath);
  tempFile.keep();
 
  return retStatus;
}
 
void InternetHelper::setTimeout(int seconds) {
  m_timeout = seconds;
  m_isTimeoutSet = true;
}
 
bool InternetHelper::isRelocated(const HTTPStatus &response) {
  return ((response == HTTPStatus::FOUND) || (response == HTTPStatus::MOVED_PERMANENTLY) ||
          (response == HTTPStatus::TEMPORARY_REDIRECT) || (response == HTTPStatus::SEE_OTHER));
}
 
void InternetHelper::throwNotConnected(const std::string &url, const HostNotFoundException &ex) {
  std::stringstream info;
  info << "Failed to access " << url << " because there is no connection to the host " << ex.message()
       << ".\nHint: Check your connection following this link: <a href=\"" << url << "\">" << url << "</a> ";
  throw Exception::InternetError(info.str() + ex.displayText());
}
 
int InternetHelper::getTimeout() {
  if (!m_isTimeoutSet) {
    const auto timeout = ConfigService::Instance().getValue<int>("network.default.timeout");
    m_timeout = timeout.value_or(30);
  }
  return m_timeout;
}
 
void InternetHelper::setMethod(const std::string &method) {
  if (method == "POST") {
    m_method = method;
  } else {
    m_method = "GET";
  }
}
 
const std::string &InternetHelper::getMethod() { return m_method; }
 
void InternetHelper::setContentType(const std::string &contentType) { m_contentType = contentType; }
 
const std::string &InternetHelper::getContentType() { return m_contentType; }
 
void InternetHelper::setContentLength(std::streamsize length) { m_contentLength = length; }
 
std::streamsize InternetHelper::getContentLength() { return m_contentLength; }
 
void InternetHelper::setBody(const std::string &body) {
  m_body = body;
  if (m_body.empty()) {
    m_method = "GET";
  } else {
    m_method = "POST";
  }
  setContentLength(m_body.size());
}
 
void InternetHelper::setBody(const std::ostringstream &body) { setBody(body.str()); }
 
void InternetHelper::setBody(Poco::Net::HTMLForm &form) {
 
  setMethod("POST");
  if (m_request == nullptr) {
    Poco::URI uri("http://www.mantidproject.org");
    createRequest(uri);
  }
  form.prepareSubmit(*m_request);
  setContentType(m_request->getContentType());
 
  std::ostringstream ss;
  form.write(ss);
  m_body = ss.str();
  setContentLength(m_body.size());
}
 
const std::string &InternetHelper::getBody() { return m_body; }
 
InternetHelper::HTTPStatus InternetHelper::getResponseStatus() {
  return static_cast<HTTPStatus>(m_response->getStatus());
}
 
const std::string &InternetHelper::getResponseReason() { return m_response->getReason(); }
 
void InternetHelper::addHeader(const std::string &key, const std::string &value) { m_headers.emplace(key, value); }
 
void InternetHelper::removeHeader(const std::string &key) { m_headers.erase(key); }
 
const std::string &InternetHelper::getHeader(const std::string &key) { return m_headers[key]; }
 
void InternetHelper::clearHeaders() { m_headers.clear(); }
 
std::map<std::string, std::string> &InternetHelper::headers() { return m_headers; }
 
void InternetHelper::reset() {
  m_headers.clear();
  m_timeout = 30;
  m_isTimeoutSet = false;
  m_body = "";
  m_method = HTTPRequest::HTTP_GET;
  m_contentType = "application/json";
  m_request = nullptr;
}
 
} // namespace Kernel
} // namespace Mantid