Mantid
Loading...
Searching...
No Matches
InternetHelper.cpp
Go to the documentation of this file.
1// Mantid Repository : https://github.com/mantidproject/mantid
2//
3// Copyright © 2018 ISIS Rutherford Appleton Laboratory UKRI,
4// NScD Oak Ridge National Laboratory, European Spallation Source,
5// Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
6// SPDX - License - Identifier: GPL - 3.0 +
11#include "MantidKernel/Logger.h"
13
14// Poco
15#include <Poco/Net/AcceptCertificateHandler.h>
16#include <Poco/Net/HTMLForm.h>
17#include <Poco/Net/HTTPRequest.h>
18#include <Poco/Net/HTTPResponse.h>
19#include <Poco/Net/HTTPSClientSession.h>
20#include <Poco/Net/NetException.h>
21#include <Poco/Net/PrivateKeyPassphraseHandler.h>
22#include <Poco/Net/SSLManager.h>
23#include <Poco/StreamCopier.h>
24#include <Poco/TemporaryFile.h>
25#include <Poco/URI.h>
26
27#include <Poco/Exception.h>
28#include <Poco/File.h>
29#include <Poco/FileStream.h>
30#include <Poco/Net/Context.h>
31#include <Poco/Net/HTTPClientSession.h>
32#include <Poco/Net/HTTPMessage.h>
33#include <Poco/Net/InvalidCertificateHandler.h>
34#include <Poco/SharedPtr.h>
35#include <Poco/Timespan.h>
36#include <Poco/Types.h>
37
38#if defined(_WIN32) || defined(_WIN64)
39#include <Winhttp.h>
40#endif
41
42#include <boost/lexical_cast.hpp>
43
44// std
45#include <fstream>
46#include <mutex>
47#include <utility>
48
49namespace Mantid {
50using namespace Types::Core;
51namespace Kernel {
52
53using namespace Poco::Net;
54using std::map;
55using std::string;
56
57namespace {
58// anonymous namespace for some utility functions
60Logger g_log("InternetHelper");
61
63std::once_flag SSL_INIT_FLAG;
64
69void doSSLInit() {
70 // initialize ssl
71 Poco::SharedPtr<InvalidCertificateHandler> certificateHandler = new AcceptCertificateHandler(true);
72 // Currently do not use any means of authentication. This should be updated
73 // IDS has signed certificate.
74 const Context::Ptr context = new Context(Context::CLIENT_USE, "", "", "", Context::VERIFY_NONE);
75 // Create a singleton for holding the default context.
76 // e.g. any future requests to publish are made to this certificate and
77 // context.
78 SSLManager::instance().initializeClient(nullptr, certificateHandler, context);
79}
80
85void initializeSSL() { std::call_once(SSL_INIT_FLAG, doSSLInit); }
86} // namespace
87
88//----------------------------------------------------------------------------------------------
92 : m_proxyInfo(), m_isProxySet(false), m_timeout(30), m_isTimeoutSet(false), m_contentLength(0),
93 m_method(HTTPRequest::HTTP_GET), m_contentType("application/json"), m_body(), m_headers(), m_request(nullptr),
94 m_response(nullptr) {}
95
96//----------------------------------------------------------------------------------------------
100 : m_proxyInfo(proxy), m_isProxySet(true), m_timeout(30), m_isTimeoutSet(false), m_contentLength(0),
101 m_method(HTTPRequest::HTTP_GET), m_contentType("application/json"), m_body(), m_headers(), m_request(nullptr),
102 m_response(nullptr) {}
103
104//----------------------------------------------------------------------------------------------
108
109void InternetHelper::setupProxyOnSession(HTTPClientSession &session, const std::string &proxyUrl) {
110 auto proxy = this->getProxy(proxyUrl);
111 if (!proxy.emptyProxy()) {
112 session.setProxyHost(proxy.host());
113 session.setProxyPort(static_cast<Poco::UInt16>(proxy.port()));
114 }
115}
116
117void InternetHelper::createRequest(Poco::URI &uri) {
118 m_request = std::make_unique<HTTPRequest>(m_method, uri.getPathAndQuery(), HTTPMessage::HTTP_1_1);
119 m_response = std::make_unique<HTTPResponse>();
120 if (!m_contentType.empty()) {
121 m_request->setContentType(m_contentType);
122 }
123
124 m_request->set("User-Agent",
125 // Use standard User-Agent format as per MDN documentation.
126 std::string("Mantid/") + MantidVersion::version());
127 if (m_method == "POST") {
128 // HTTP states that the 'Content-Length' header should not be included
129 // if the 'Transfer-Encoding' header is set. UNKNOWN_CONTENT_LENGTH
130 // indicates to Poco to remove the header field
131 m_request->setContentLength(HTTPMessage::UNKNOWN_CONTENT_LENGTH);
132 m_request->setChunkedTransferEncoding(true);
133 } else if (m_contentLength > 0) {
134 m_request->setContentLength(m_contentLength);
135 }
136
137 for (auto &header : m_headers) {
138 m_request->set(header.first, header.second);
139 }
140}
141
142InternetHelper::HTTPStatus InternetHelper::sendRequestAndProcess(HTTPClientSession &session, Poco::URI &uri,
143 std::ostream &responseStream) {
144 // create a request
145 this->createRequest(uri);
146 session.sendRequest(*m_request) << m_body;
147
148 std::istream &rs = session.receiveResponse(*m_response);
149 const auto retStatus = static_cast<HTTPStatus>(m_response->getStatus());
150 g_log.debug() << "Answer from web: " << static_cast<int>(retStatus) << " " << m_response->getReason() << '\n';
151
152 if (retStatus == HTTPStatus::OK || (retStatus == HTTPStatus::CREATED && m_method == HTTPRequest::HTTP_POST)) {
153 Poco::StreamCopier::copyStream(rs, responseStream);
154 if (m_response)
156 else
157 g_log.warning("Response is null pointer");
158 return retStatus;
159 } else if (isRelocated(retStatus)) {
160 return this->processRelocation(*m_response, responseStream);
161 } else {
162 Poco::StreamCopier::copyStream(rs, responseStream);
163 return processErrorStates(*m_response, rs, uri.toString());
164 }
165}
166
168 std::ostream &responseStream) {
169 std::string newLocation = response.get("location", "");
170 if (!newLocation.empty()) {
171 g_log.information() << "url relocated to " << newLocation << "\n";
172 return this->sendRequest(newLocation, responseStream);
173 } else {
174 g_log.warning("Apparent relocation did not give new location\n");
175 return static_cast<HTTPStatus>(response.getStatus());
176 }
177}
178
183InternetHelper::HTTPStatus InternetHelper::sendRequest(const std::string &url, std::ostream &responseStream) {
184
185 // send the request
186 Poco::URI uri(url);
187 if (uri.getPath().empty())
188 uri = url + "/";
189 if ((uri.getScheme() == "https") || (uri.getPort() == 443)) {
190 return static_cast<HTTPStatus>(sendHTTPSRequest(uri.toString(), responseStream));
191 } else {
192 return static_cast<HTTPStatus>(sendHTTPRequest(uri.toString(), responseStream));
193 }
194}
195
203void InternetHelper::logDebugRequestSending(const std::string &schemeName, const std::string &url) const {
204 const std::string insecString = "password=";
205 if (std::string::npos == url.find(insecString)) {
206 g_log.debug() << "Sending " << schemeName << " " << m_method << " request to: " << url << "\n";
207 } else {
208 g_log.debug() << "Sending " << schemeName << " " << m_method
209 << " request to an url where the query string seems to contain a "
210 "password! (not shown for security reasons)."
211 << "\n";
212 }
213}
214
219InternetHelper::HTTPStatus InternetHelper::sendHTTPRequest(const std::string &url, std::ostream &responseStream) {
221 logDebugRequestSending("http", url);
222
223 Poco::URI uri(url);
224 // Configure Poco HTTP Client Session
225 try {
226 Poco::Net::HTTPClientSession session(uri.getHost(), uri.getPort());
227 session.setTimeout(Poco::Timespan(getTimeout(), 0));
228
229 // configure proxy
230 setupProxyOnSession(session, url);
231
232 // low level sending the request
233 retStatus = this->sendRequestAndProcess(session, uri, responseStream);
234 } catch (HostNotFoundException &ex) {
235 throwNotConnected(url, ex);
236 } catch (Poco::Exception &ex) {
237 throw Exception::InternetError("Connection and request failed " + ex.displayText());
238 }
239 return retStatus;
240}
241
246InternetHelper::HTTPStatus InternetHelper::sendHTTPSRequest(const std::string &url, std::ostream &responseStream) {
248
249 logDebugRequestSending("https", url);
250
251 Poco::URI uri(url);
252 try {
253 initializeSSL();
254 // Create the session
255 HTTPSClientSession session(uri.getHost(), static_cast<Poco::UInt16>(uri.getPort()));
256 session.setTimeout(Poco::Timespan(getTimeout(), 0));
257
258 // HACK:: Currently the automatic proxy detection only supports http proxy
259 // detection
260 // most locations use the same proxy for http and https, so force it to use
261 // the http proxy
262 std::string urlforProxy = ConfigService::Instance().getString("proxy.httpsTargetUrl");
263 if (urlforProxy.empty()) {
264 urlforProxy = "http://" + uri.getHost();
265 }
266 setupProxyOnSession(session, urlforProxy);
267
268 // low level sending the request
269 retStatus = this->sendRequestAndProcess(session, uri, responseStream);
270 } catch (HostNotFoundException &ex) {
271 throwNotConnected(url, ex);
272 } catch (Poco::Exception &ex) {
273 throw Exception::InternetError("Connection and request failed " + ex.displayText());
274 }
275 return retStatus;
276}
277
282 // set the proxy
283 if (!m_isProxySet) {
285 }
286 return m_proxyInfo;
287}
288
292
297 m_proxyInfo = proxy;
298 m_isProxySet = true;
299}
300
304void InternetHelper::processResponseHeaders(const HTTPResponse & /*unused*/) {}
305
315InternetHelper::HTTPStatus InternetHelper::processErrorStates(const HTTPResponse &res, std::istream &rs,
316 const std::string &url) {
317 const auto retStatus = static_cast<HTTPStatus>(res.getStatus());
318 g_log.debug() << "Answer from web: " << static_cast<int>(res.getStatus()) << " " << res.getReason() << '\n';
319
320 // get github api rate limit information if available;
321 int rateLimitRemaining;
322 DateAndTime rateLimitReset;
323 try {
324 rateLimitRemaining = boost::lexical_cast<int>(res.get("X-RateLimit-Remaining", "-1"));
325 rateLimitReset.set_from_time_t(boost::lexical_cast<int>(res.get("X-RateLimit-Reset", "0")));
326 } catch (boost::bad_lexical_cast const &) {
327 rateLimitRemaining = -1;
328 }
329
330 if (retStatus == HTTPStatus::OK) {
331 throw Exception::InternetError("Response was ok, processing should never "
332 "have entered processErrorStates",
333 static_cast<int>(retStatus));
334 } else if (retStatus == HTTPStatus::FOUND) {
335 throw Exception::InternetError("Response was HTTP_FOUND, processing should "
336 "never have entered processErrorStates",
337 static_cast<int>(retStatus));
338 } else if (retStatus == HTTPStatus::MOVED_PERMANENTLY) {
339 throw Exception::InternetError("Response was HTTP_MOVED_PERMANENTLY, "
340 "processing should never have entered "
341 "processErrorStates",
342 static_cast<int>(retStatus));
343 } else if (retStatus == HTTPStatus::NOT_MODIFIED) {
344 throw Exception::InternetError("Not modified since provided date" + rateLimitReset.toSimpleString(),
345 static_cast<int>(retStatus));
346 } else if ((retStatus == HTTPStatus::FORBIDDEN) && (rateLimitRemaining == 0)) {
347 throw Exception::InternetError("The Github API rate limit has been reached, try again after " +
348 rateLimitReset.toSimpleString() + " GMT",
349 static_cast<int>(retStatus));
350 } else {
351 std::stringstream info;
352 std::stringstream ss;
353 Poco::StreamCopier::copyStream(rs, ss);
354 if (retStatus == HTTPStatus::NOT_FOUND)
355 info << "Failed to download " << url << " with the link "
356 << "<a href=\"" << url << "\">.\n"
357 << "Hint. Check that link is correct</a>";
358 else {
359 // show the error
360 info << res.getReason();
361 info << ss.str();
362 g_log.debug() << ss.str();
363 }
364 throw Exception::InternetError(info.str() + ss.str(), static_cast<int>(retStatus));
365 }
366 return retStatus; // must return to follow contract
367}
368
389InternetHelper::HTTPStatus InternetHelper::downloadFile(const std::string &urlFile, const std::string &localFilePath) {
390 g_log.debug() << "DownloadFile from \"" << urlFile << "\" to file: \"" << localFilePath << "\"\n";
391
392 Poco::TemporaryFile tempFile;
393 Poco::FileStream tempFileStream(tempFile.path());
394 const auto retStatus = sendRequest(urlFile, tempFileStream);
395 tempFileStream.close();
396
397 // if there have been no errors move it to the final location, and turn off
398 // automatic deletion.
399 // clear the way if the target file path is already in use
400 Poco::File file(localFilePath);
401 if (file.exists()) {
402 file.remove();
403 }
404
405 tempFile.moveTo(localFilePath);
406 tempFile.keep();
407
408 return retStatus;
409}
410
414void InternetHelper::setTimeout(int seconds) {
415 m_timeout = seconds;
416 m_isTimeoutSet = true;
417}
418
423 return ((response == HTTPStatus::FOUND) || (response == HTTPStatus::MOVED_PERMANENTLY) ||
424 (response == HTTPStatus::TEMPORARY_REDIRECT) || (response == HTTPStatus::SEE_OTHER));
425}
426
431void InternetHelper::throwNotConnected(const std::string &url, const HostNotFoundException &ex) {
432 std::stringstream info;
433 info << "Failed to access " << url << " because there is no connection to the host " << ex.message()
434 << ".\nHint: Check your connection following this link: <a href=\"" << url << "\">" << url << "</a> ";
435 throw Exception::InternetError(info.str() + ex.displayText());
436}
437
442 if (!m_isTimeoutSet) {
443 const auto timeout = ConfigService::Instance().getValue<int>("network.default.timeout");
444 m_timeout = timeout.is_initialized() ? timeout.get() : 30;
445 }
446 return m_timeout;
447}
448
453void InternetHelper::setMethod(const std::string &method) {
454 if (method == "POST") {
455 m_method = method;
456 } else {
457 m_method = "GET";
458 }
459}
460
464const std::string &InternetHelper::getMethod() { return m_method; }
465
469void InternetHelper::setContentType(const std::string &contentType) { m_contentType = contentType; }
470
474const std::string &InternetHelper::getContentType() { return m_contentType; }
475
479void InternetHelper::setContentLength(std::streamsize length) { m_contentLength = length; }
480
485
491void InternetHelper::setBody(const std::string &body) {
492 m_body = body;
493 if (m_body.empty()) {
494 m_method = "GET";
495 } else {
496 m_method = "POST";
497 }
498 setContentLength(m_body.size());
499}
500
506void InternetHelper::setBody(const std::ostringstream &body) { setBody(body.str()); }
507
513void InternetHelper::setBody(Poco::Net::HTMLForm &form) {
514
515 setMethod("POST");
516 if (m_request == nullptr) {
517 Poco::URI uri("http://www.mantidproject.org");
518 createRequest(uri);
519 }
520 form.prepareSubmit(*m_request);
521 setContentType(m_request->getContentType());
522
523 std::ostringstream ss;
524 form.write(ss);
525 m_body = ss.str();
526 setContentLength(m_body.size());
527}
528
532const std::string &InternetHelper::getBody() { return m_body; }
533
538 return static_cast<HTTPStatus>(m_response->getStatus());
539}
540
544const std::string &InternetHelper::getResponseReason() { return m_response->getReason(); }
545
550void InternetHelper::addHeader(const std::string &key, const std::string &value) { m_headers.emplace(key, value); }
551
555void InternetHelper::removeHeader(const std::string &key) { m_headers.erase(key); }
556
561const std::string &InternetHelper::getHeader(const std::string &key) { return m_headers[key]; }
562
566
569std::map<std::string, std::string> &InternetHelper::headers() { return m_headers; }
570
574 m_headers.clear();
575 m_timeout = 30;
576 m_isTimeoutSet = false;
577 m_body = "";
578 m_method = HTTPRequest::HTTP_GET;
579 m_contentType = "application/json";
580 m_request = nullptr;
581}
582
583} // namespace Kernel
584} // namespace Mantid
double value
The value of the point.
Definition: FitMW.cpp:51
BuilderMethod< ArgType > m_method
Exception thrown when error occurs accessing an internet resource.
Definition: Exception.h:321
const std::string & getResponseReason()
Gets the body set for future requests.
const std::string & getBody()
Gets the body set for future requests.
void createRequest(Poco::URI &uri)
void setContentLength(std::streamsize length)
Sets the content length.
InternetHelper::HTTPStatus processRelocation(const Poco::Net::HTTPResponse &response, std::ostream &responseStream)
void logDebugRequestSending(const std::string &schemeName, const std::string &url) const
Helper to log (debug level) the request being sent (careful not to print blatant passwords,...
void setBody(const std::string &body)
Sets the body & content length for future requests, this will also set the method to POST is the body...
bool isRelocated(const HTTPStatus &response)
Checks the HTTP status to decide if this is a relocation.
std::unique_ptr< Poco::Net::HTTPRequest > m_request
void addHeader(const std::string &key, const std::string &value)
Adds a header.
void setupProxyOnSession(Poco::Net::HTTPClientSession &session, const std::string &proxyUrl)
void setProxy(const Kernel::ProxyInfo &proxy)
sets the proxy details.
void throwNotConnected(const std::string &url, const Poco::Net::HostNotFoundException &ex)
Throw an exception occurs when the computer is not connected to the internet.
std::streamsize getContentLength()
Gets the content length.
void clearProxy()
Clears cached proxy details.
HTTPStatus getResponseStatus()
Gets the body set for future requests.
virtual ~InternetHelper()
Destructor.
const std::string & getHeader(const std::string &key)
Gets the value of a header.
virtual HTTPStatus sendHTTPSRequest(const std::string &url, std::ostream &responseStream)
Performs a request using https.
Kernel::ProxyInfo & getProxy(const std::string &url)
Gets proxy details for a system and a url.
void setTimeout(int seconds)
Sets the timeout in seconds.
virtual HTTPStatus processErrorStates(const Poco::Net::HTTPResponse &res, std::istream &rs, const std::string &url)
Process any HTTP errors states.
virtual HTTPStatus sendHTTPRequest(const std::string &url, std::ostream &responseStream)
Performs a request using http.
virtual HTTPStatus sendRequest(const std::string &url, std::ostream &responseStream)
Performs a request using http or https depending on the url.
const std::string & getMethod()
Gets the method.
StringToStringMap & headers()
Returns a reference to the headers map.
virtual HTTPStatus downloadFile(const std::string &urlFile, const std::string &localFilePath="")
Download a url and fetch it inside the local path given.
int getTimeout()
Gets the timeout in seconds.
virtual HTTPStatus sendRequestAndProcess(Poco::Net::HTTPClientSession &session, Poco::URI &uri, std::ostream &responseStream)
void removeHeader(const std::string &key)
Removes a header.
const std::string & getContentType()
Gets the Content Type.
virtual void reset()
Resets properties to defaults (except the proxy)
std::unique_ptr< Poco::Net::HTTPResponse > m_response
void setContentType(const std::string &contentType)
Sets the Content Type.
void clearHeaders()
Clears all headers.
void setMethod(const std::string &method)
Sets the Method.
virtual void processResponseHeaders(const Poco::Net::HTTPResponse &res)
Process any headers from the response stream Basic implementation does nothing.
void debug(const std::string &msg)
Logs at debug level.
Definition: Logger.cpp:114
void warning(const std::string &msg)
Logs at warning level.
Definition: Logger.cpp:86
void information(const std::string &msg)
Logs at information level.
Definition: Logger.cpp:105
static const char * version()
The full version number.
ProxyInfo : Container for carrying around network proxy information.
Definition: ProxyInfo.h:17
static T & Instance()
Return a reference to the Singleton instance, creating it if it does not already exist Creation is do...
Helper class which provides the Collimation Length for SANS instruments.