HttpClientManager.cs 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853
  1. using System;
  2. using System.Collections.Concurrent;
  3. using System.Collections.Generic;
  4. using System.Globalization;
  5. using System.IO;
  6. using System.Linq;
  7. using System.Net;
  8. using System.Net.Cache;
  9. using System.Text;
  10. using System.Threading;
  11. using System.Threading.Tasks;
  12. using MediaBrowser.Common.Configuration;
  13. using MediaBrowser.Common.Extensions;
  14. using MediaBrowser.Common.Net;
  15. using MediaBrowser.Model.IO;
  16. using Microsoft.Extensions.Logging;
  17. using MediaBrowser.Model.Net;
  18. namespace Emby.Server.Implementations.HttpClientManager
  19. {
  20. /// <summary>
  21. /// Class HttpClientManager
  22. /// </summary>
  23. public class HttpClientManager : IHttpClient
  24. {
  25. /// <summary>
  26. /// When one request to a host times out, we'll ban all other requests for this period of time, to prevent scans from stalling
  27. /// </summary>
  28. private const int TimeoutSeconds = 30;
  29. /// <summary>
  30. /// The _logger
  31. /// </summary>
  32. private readonly ILogger _logger;
  33. /// <summary>
  34. /// The _app paths
  35. /// </summary>
  36. private readonly IApplicationPaths _appPaths;
  37. private readonly IFileSystem _fileSystem;
  38. private readonly Func<string> _defaultUserAgentFn;
  39. /// <summary>
  40. /// Initializes a new instance of the <see cref="HttpClientManager" /> class.
  41. /// </summary>
  42. public HttpClientManager(IApplicationPaths appPaths, ILogger logger, IFileSystem fileSystem, Func<string> defaultUserAgentFn)
  43. {
  44. if (appPaths == null)
  45. {
  46. throw new ArgumentNullException(nameof(appPaths));
  47. }
  48. if (logger == null)
  49. {
  50. throw new ArgumentNullException(nameof(logger));
  51. }
  52. _logger = logger;
  53. _fileSystem = fileSystem;
  54. _appPaths = appPaths;
  55. _defaultUserAgentFn = defaultUserAgentFn;
  56. // http://stackoverflow.com/questions/566437/http-post-returns-the-error-417-expectation-failed-c
  57. ServicePointManager.Expect100Continue = false;
  58. #if NET46
  59. // Trakt requests sometimes fail without this
  60. ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls;
  61. #endif
  62. }
  63. /// <summary>
  64. /// Holds a dictionary of http clients by host. Use GetHttpClient(host) to retrieve or create a client for web requests.
  65. /// DON'T dispose it after use.
  66. /// </summary>
  67. /// <value>The HTTP clients.</value>
  68. private readonly ConcurrentDictionary<string, HttpClientInfo> _httpClients = new ConcurrentDictionary<string, HttpClientInfo>();
  69. /// <summary>
  70. /// Gets
  71. /// </summary>
  72. /// <param name="host">The host.</param>
  73. /// <param name="enableHttpCompression">if set to <c>true</c> [enable HTTP compression].</param>
  74. /// <returns>HttpClient.</returns>
  75. /// <exception cref="System.ArgumentNullException">host</exception>
  76. private HttpClientInfo GetHttpClient(string host, bool enableHttpCompression)
  77. {
  78. if (string.IsNullOrEmpty(host))
  79. {
  80. throw new ArgumentNullException(nameof(host));
  81. }
  82. HttpClientInfo client;
  83. var key = host + enableHttpCompression;
  84. if (!_httpClients.TryGetValue(key, out client))
  85. {
  86. client = new HttpClientInfo();
  87. _httpClients.TryAdd(key, client);
  88. }
  89. return client;
  90. }
  91. private static WebRequest CreateWebRequest(string url)
  92. {
  93. try
  94. {
  95. return WebRequest.Create(url);
  96. }
  97. catch (NotSupportedException)
  98. {
  99. //Webrequest creation does fail on MONO randomly when using WebRequest.Create
  100. //the issue occurs in the GetCreator method here: http://www.oschina.net/code/explore/mono-2.8.1/mcs/class/System/System.Net/WebRequest.cs
  101. var type = Type.GetType("System.Net.HttpRequestCreator, System, Version=4.0.0.0,Culture=neutral, PublicKeyToken=b77a5c561934e089");
  102. var creator = Activator.CreateInstance(type, nonPublic: true) as IWebRequestCreate;
  103. return creator.Create(new Uri(url)) as HttpWebRequest;
  104. }
  105. }
  106. private WebRequest GetRequest(HttpRequestOptions options, string method)
  107. {
  108. string url = options.Url;
  109. Uri uriAddress = new Uri(url);
  110. string userInfo = uriAddress.UserInfo;
  111. if (!string.IsNullOrWhiteSpace(userInfo))
  112. {
  113. _logger.LogInformation("Found userInfo in url: {0} ... url: {1}", userInfo, url);
  114. url = url.Replace(userInfo + "@", string.Empty);
  115. }
  116. WebRequest request = CreateWebRequest(url);
  117. if (request is HttpWebRequest httpWebRequest)
  118. {
  119. AddRequestHeaders(httpWebRequest, options);
  120. if (options.EnableHttpCompression)
  121. {
  122. httpWebRequest.AutomaticDecompression = DecompressionMethods.Deflate;
  123. if (options.DecompressionMethod.HasValue
  124. && options.DecompressionMethod.Value == CompressionMethod.Gzip)
  125. {
  126. httpWebRequest.AutomaticDecompression = DecompressionMethods.GZip;
  127. }
  128. }
  129. else
  130. {
  131. httpWebRequest.AutomaticDecompression = DecompressionMethods.None;
  132. }
  133. httpWebRequest.KeepAlive = options.EnableKeepAlive;
  134. if (!string.IsNullOrEmpty(options.Host))
  135. {
  136. httpWebRequest.Host = options.Host;
  137. }
  138. if (!string.IsNullOrEmpty(options.Referer))
  139. {
  140. httpWebRequest.Referer = options.Referer;
  141. }
  142. }
  143. request.CachePolicy = new RequestCachePolicy(RequestCacheLevel.BypassCache);
  144. request.Method = method;
  145. request.Timeout = options.TimeoutMs;
  146. if (!string.IsNullOrWhiteSpace(userInfo))
  147. {
  148. var parts = userInfo.Split(':');
  149. if (parts.Length == 2)
  150. {
  151. request.Credentials = GetCredential(url, parts[0], parts[1]);
  152. // TODO: .net core ??
  153. request.PreAuthenticate = true;
  154. }
  155. }
  156. return request;
  157. }
  158. private static CredentialCache GetCredential(string url, string username, string password)
  159. {
  160. //ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3;
  161. CredentialCache credentialCache = new CredentialCache();
  162. credentialCache.Add(new Uri(url), "Basic", new NetworkCredential(username, password));
  163. return credentialCache;
  164. }
  165. private void AddRequestHeaders(HttpWebRequest request, HttpRequestOptions options)
  166. {
  167. var hasUserAgent = false;
  168. foreach (var header in options.RequestHeaders)
  169. {
  170. if (string.Equals(header.Key, "Accept", StringComparison.OrdinalIgnoreCase))
  171. {
  172. request.Accept = header.Value;
  173. }
  174. else if (string.Equals(header.Key, "User-Agent", StringComparison.OrdinalIgnoreCase))
  175. {
  176. SetUserAgent(request, header.Value);
  177. hasUserAgent = true;
  178. }
  179. else
  180. {
  181. request.Headers.Set(header.Key, header.Value);
  182. }
  183. }
  184. if (!hasUserAgent && options.EnableDefaultUserAgent)
  185. {
  186. SetUserAgent(request, _defaultUserAgentFn());
  187. }
  188. }
  189. private static void SetUserAgent(HttpWebRequest request, string userAgent)
  190. {
  191. request.UserAgent = userAgent;
  192. }
  193. /// <summary>
  194. /// Gets the response internal.
  195. /// </summary>
  196. /// <param name="options">The options.</param>
  197. /// <returns>Task{HttpResponseInfo}.</returns>
  198. public Task<HttpResponseInfo> GetResponse(HttpRequestOptions options)
  199. {
  200. return SendAsync(options, "GET");
  201. }
  202. /// <summary>
  203. /// Performs a GET request and returns the resulting stream
  204. /// </summary>
  205. /// <param name="options">The options.</param>
  206. /// <returns>Task{Stream}.</returns>
  207. public async Task<Stream> Get(HttpRequestOptions options)
  208. {
  209. var response = await GetResponse(options).ConfigureAwait(false);
  210. return response.Content;
  211. }
  212. /// <summary>
  213. /// send as an asynchronous operation.
  214. /// </summary>
  215. /// <param name="options">The options.</param>
  216. /// <param name="httpMethod">The HTTP method.</param>
  217. /// <returns>Task{HttpResponseInfo}.</returns>
  218. /// <exception cref="HttpException">
  219. /// </exception>
  220. public async Task<HttpResponseInfo> SendAsync(HttpRequestOptions options, string httpMethod)
  221. {
  222. if (options.CacheMode == CacheMode.None)
  223. {
  224. return await SendAsyncInternal(options, httpMethod).ConfigureAwait(false);
  225. }
  226. var url = options.Url;
  227. var urlHash = url.ToLower().GetMD5().ToString("N");
  228. var responseCachePath = Path.Combine(_appPaths.CachePath, "httpclient", urlHash);
  229. var response = await GetCachedResponse(responseCachePath, options.CacheLength, url).ConfigureAwait(false);
  230. if (response != null)
  231. {
  232. return response;
  233. }
  234. response = await SendAsyncInternal(options, httpMethod).ConfigureAwait(false);
  235. if (response.StatusCode == HttpStatusCode.OK)
  236. {
  237. await CacheResponse(response, responseCachePath).ConfigureAwait(false);
  238. }
  239. return response;
  240. }
  241. private async Task<HttpResponseInfo> GetCachedResponse(string responseCachePath, TimeSpan cacheLength, string url)
  242. {
  243. try
  244. {
  245. if (_fileSystem.GetLastWriteTimeUtc(responseCachePath).Add(cacheLength) > DateTime.UtcNow)
  246. {
  247. using (var stream = _fileSystem.GetFileStream(responseCachePath, FileOpenMode.Open, FileAccessMode.Read, FileShareMode.Read, true))
  248. {
  249. var memoryStream = new MemoryStream();
  250. await stream.CopyToAsync(memoryStream).ConfigureAwait(false);
  251. memoryStream.Position = 0;
  252. return new HttpResponseInfo
  253. {
  254. ResponseUrl = url,
  255. Content = memoryStream,
  256. StatusCode = HttpStatusCode.OK,
  257. ContentLength = memoryStream.Length
  258. };
  259. }
  260. }
  261. }
  262. catch (FileNotFoundException)
  263. {
  264. }
  265. catch (DirectoryNotFoundException)
  266. {
  267. }
  268. return null;
  269. }
  270. private async Task CacheResponse(HttpResponseInfo response, string responseCachePath)
  271. {
  272. _fileSystem.CreateDirectory(_fileSystem.GetDirectoryName(responseCachePath));
  273. using (var responseStream = response.Content)
  274. {
  275. var memoryStream = new MemoryStream();
  276. await responseStream.CopyToAsync(memoryStream).ConfigureAwait(false);
  277. memoryStream.Position = 0;
  278. using (var fileStream = _fileSystem.GetFileStream(responseCachePath, FileOpenMode.Create, FileAccessMode.Write, FileShareMode.None, true))
  279. {
  280. await memoryStream.CopyToAsync(fileStream).ConfigureAwait(false);
  281. memoryStream.Position = 0;
  282. response.Content = memoryStream;
  283. }
  284. }
  285. }
  286. private async Task<HttpResponseInfo> SendAsyncInternal(HttpRequestOptions options, string httpMethod)
  287. {
  288. ValidateParams(options);
  289. options.CancellationToken.ThrowIfCancellationRequested();
  290. var client = GetHttpClient(GetHostFromUrl(options.Url), options.EnableHttpCompression);
  291. if ((DateTime.UtcNow - client.LastTimeout).TotalSeconds < TimeoutSeconds)
  292. {
  293. throw new HttpException(string.Format("Cancelling connection to {0} due to a previous timeout.", options.Url))
  294. {
  295. IsTimedOut = true
  296. };
  297. }
  298. var httpWebRequest = GetRequest(options, httpMethod);
  299. if (options.RequestContentBytes != null ||
  300. !string.IsNullOrEmpty(options.RequestContent) ||
  301. string.Equals(httpMethod, "post", StringComparison.OrdinalIgnoreCase))
  302. {
  303. try
  304. {
  305. var bytes = options.RequestContentBytes ?? Encoding.UTF8.GetBytes(options.RequestContent ?? string.Empty);
  306. var contentType = options.RequestContentType ?? "application/x-www-form-urlencoded";
  307. if (options.AppendCharsetToMimeType)
  308. {
  309. contentType = contentType.TrimEnd(';') + "; charset=\"utf-8\"";
  310. }
  311. httpWebRequest.ContentType = contentType;
  312. httpWebRequest.ContentLength = bytes.Length;
  313. (await httpWebRequest.GetRequestStreamAsync().ConfigureAwait(false)).Write(bytes, 0, bytes.Length);
  314. }
  315. catch (Exception ex)
  316. {
  317. throw new HttpException(ex.Message) { IsTimedOut = true };
  318. }
  319. }
  320. if (options.ResourcePool != null)
  321. {
  322. await options.ResourcePool.WaitAsync(options.CancellationToken).ConfigureAwait(false);
  323. }
  324. if ((DateTime.UtcNow - client.LastTimeout).TotalSeconds < TimeoutSeconds)
  325. {
  326. options.ResourcePool?.Release();
  327. throw new HttpException(string.Format("Connection to {0} timed out", options.Url)) { IsTimedOut = true };
  328. }
  329. if (options.LogRequest)
  330. {
  331. if (options.LogRequestAsDebug)
  332. {
  333. _logger.LogDebug("HttpClientManager {0}: {1}", httpMethod.ToUpper(), options.Url);
  334. }
  335. else
  336. {
  337. _logger.LogInformation("HttpClientManager {0}: {1}", httpMethod.ToUpper(), options.Url);
  338. }
  339. }
  340. try
  341. {
  342. options.CancellationToken.ThrowIfCancellationRequested();
  343. if (!options.BufferContent)
  344. {
  345. var response = await GetResponseAsync(httpWebRequest, TimeSpan.FromMilliseconds(options.TimeoutMs)).ConfigureAwait(false);
  346. var httpResponse = (HttpWebResponse)response;
  347. EnsureSuccessStatusCode(client, httpResponse, options);
  348. options.CancellationToken.ThrowIfCancellationRequested();
  349. return GetResponseInfo(httpResponse, httpResponse.GetResponseStream(), GetContentLength(httpResponse), httpResponse);
  350. }
  351. using (var response = await GetResponseAsync(httpWebRequest, TimeSpan.FromMilliseconds(options.TimeoutMs)).ConfigureAwait(false))
  352. {
  353. var httpResponse = (HttpWebResponse)response;
  354. EnsureSuccessStatusCode(client, httpResponse, options);
  355. options.CancellationToken.ThrowIfCancellationRequested();
  356. using (var stream = httpResponse.GetResponseStream())
  357. {
  358. var memoryStream = new MemoryStream();
  359. await stream.CopyToAsync(memoryStream).ConfigureAwait(false);
  360. memoryStream.Position = 0;
  361. return GetResponseInfo(httpResponse, memoryStream, memoryStream.Length, null);
  362. }
  363. }
  364. }
  365. catch (OperationCanceledException ex)
  366. {
  367. throw GetCancellationException(options, client, options.CancellationToken, ex);
  368. }
  369. catch (Exception ex)
  370. {
  371. throw GetException(ex, options, client);
  372. }
  373. finally
  374. {
  375. options.ResourcePool?.Release();
  376. }
  377. }
  378. private HttpResponseInfo GetResponseInfo(HttpWebResponse httpResponse, Stream content, long? contentLength, IDisposable disposable)
  379. {
  380. var responseInfo = new HttpResponseInfo(disposable)
  381. {
  382. Content = content,
  383. StatusCode = httpResponse.StatusCode,
  384. ContentType = httpResponse.ContentType,
  385. ContentLength = contentLength,
  386. ResponseUrl = httpResponse.ResponseUri.ToString()
  387. };
  388. if (httpResponse.Headers != null)
  389. {
  390. SetHeaders(httpResponse.Headers, responseInfo);
  391. }
  392. return responseInfo;
  393. }
  394. private HttpResponseInfo GetResponseInfo(HttpWebResponse httpResponse, string tempFile, long? contentLength)
  395. {
  396. var responseInfo = new HttpResponseInfo
  397. {
  398. TempFilePath = tempFile,
  399. StatusCode = httpResponse.StatusCode,
  400. ContentType = httpResponse.ContentType,
  401. ContentLength = contentLength
  402. };
  403. if (httpResponse.Headers != null)
  404. {
  405. SetHeaders(httpResponse.Headers, responseInfo);
  406. }
  407. return responseInfo;
  408. }
  409. private static void SetHeaders(WebHeaderCollection headers, HttpResponseInfo responseInfo)
  410. {
  411. foreach (var key in headers.AllKeys)
  412. {
  413. responseInfo.Headers[key] = headers[key];
  414. }
  415. }
  416. public Task<HttpResponseInfo> Post(HttpRequestOptions options)
  417. {
  418. return SendAsync(options, "POST");
  419. }
  420. /// <summary>
  421. /// Performs a POST request
  422. /// </summary>
  423. /// <param name="options">The options.</param>
  424. /// <param name="postData">Params to add to the POST data.</param>
  425. /// <returns>stream on success, null on failure</returns>
  426. public async Task<Stream> Post(HttpRequestOptions options, Dictionary<string, string> postData)
  427. {
  428. options.SetPostData(postData);
  429. var response = await Post(options).ConfigureAwait(false);
  430. return response.Content;
  431. }
  432. /// <summary>
  433. /// Downloads the contents of a given url into a temporary location
  434. /// </summary>
  435. /// <param name="options">The options.</param>
  436. /// <returns>Task{System.String}.</returns>
  437. public async Task<string> GetTempFile(HttpRequestOptions options)
  438. {
  439. var response = await GetTempFileResponse(options).ConfigureAwait(false);
  440. return response.TempFilePath;
  441. }
  442. public async Task<HttpResponseInfo> GetTempFileResponse(HttpRequestOptions options)
  443. {
  444. ValidateParams(options);
  445. _fileSystem.CreateDirectory(_appPaths.TempDirectory);
  446. var tempFile = Path.Combine(_appPaths.TempDirectory, Guid.NewGuid() + ".tmp");
  447. if (options.Progress == null)
  448. {
  449. throw new ArgumentException("Options did not have a Progress value.",nameof(options));
  450. }
  451. options.CancellationToken.ThrowIfCancellationRequested();
  452. var httpWebRequest = GetRequest(options, "GET");
  453. if (options.ResourcePool != null)
  454. {
  455. await options.ResourcePool.WaitAsync(options.CancellationToken).ConfigureAwait(false);
  456. }
  457. options.Progress.Report(0);
  458. if (options.LogRequest)
  459. {
  460. if (options.LogRequestAsDebug)
  461. {
  462. _logger.LogDebug("HttpClientManager.GetTempFileResponse url: {0}", options.Url);
  463. }
  464. else
  465. {
  466. _logger.LogInformation("HttpClientManager.GetTempFileResponse url: {0}", options.Url);
  467. }
  468. }
  469. var client = GetHttpClient(GetHostFromUrl(options.Url), options.EnableHttpCompression);
  470. try
  471. {
  472. options.CancellationToken.ThrowIfCancellationRequested();
  473. using (var response = await httpWebRequest.GetResponseAsync().ConfigureAwait(false))
  474. {
  475. var httpResponse = (HttpWebResponse)response;
  476. EnsureSuccessStatusCode(client, httpResponse, options);
  477. options.CancellationToken.ThrowIfCancellationRequested();
  478. var contentLength = GetContentLength(httpResponse);
  479. if (contentLength.HasValue)
  480. {
  481. using (var fs = _fileSystem.GetFileStream(tempFile, FileOpenMode.Create, FileAccessMode.Write, FileShareMode.Read, true))
  482. {
  483. await httpResponse.GetResponseStream().CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, options.CancellationToken).ConfigureAwait(false);
  484. }
  485. }
  486. else
  487. {
  488. // We're not able to track progress
  489. using (var stream = httpResponse.GetResponseStream())
  490. using (var fs = _fileSystem.GetFileStream(tempFile, FileOpenMode.Create, FileAccessMode.Write, FileShareMode.Read, true))
  491. {
  492. await stream.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, options.CancellationToken).ConfigureAwait(false);
  493. }
  494. }
  495. options.Progress.Report(100);
  496. return GetResponseInfo(httpResponse, tempFile, contentLength);
  497. }
  498. }
  499. catch (Exception ex)
  500. {
  501. DeleteTempFile(tempFile);
  502. throw GetException(ex, options, client);
  503. }
  504. finally
  505. {
  506. options.ResourcePool?.Release();
  507. }
  508. }
  509. private static long? GetContentLength(HttpWebResponse response)
  510. {
  511. var length = response.ContentLength;
  512. if (length == 0)
  513. {
  514. return null;
  515. }
  516. return length;
  517. }
  518. protected static readonly CultureInfo UsCulture = new CultureInfo("en-US");
  519. private Exception GetException(Exception ex, HttpRequestOptions options, HttpClientInfo client)
  520. {
  521. if (ex is HttpException)
  522. {
  523. return ex;
  524. }
  525. var webException = ex as WebException
  526. ?? ex.InnerException as WebException;
  527. if (webException != null)
  528. {
  529. if (options.LogErrors)
  530. {
  531. _logger.LogError(webException, "Error {status} getting response from {url}", webException.Status, options.Url);
  532. }
  533. var exception = new HttpException(webException.Message, webException);
  534. var response = webException.Response as HttpWebResponse;
  535. if (response != null)
  536. {
  537. exception.StatusCode = response.StatusCode;
  538. if ((int)response.StatusCode == 429)
  539. {
  540. client.LastTimeout = DateTime.UtcNow;
  541. }
  542. }
  543. if (!exception.StatusCode.HasValue)
  544. {
  545. if (webException.Status == WebExceptionStatus.NameResolutionFailure ||
  546. webException.Status == WebExceptionStatus.ConnectFailure)
  547. {
  548. exception.IsTimedOut = true;
  549. }
  550. }
  551. return exception;
  552. }
  553. var operationCanceledException = ex as OperationCanceledException
  554. ?? ex.InnerException as OperationCanceledException;
  555. if (operationCanceledException != null)
  556. {
  557. return GetCancellationException(options, client, options.CancellationToken, operationCanceledException);
  558. }
  559. if (options.LogErrors)
  560. {
  561. _logger.LogError(ex, "Error getting response from {url}", options.Url);
  562. }
  563. return ex;
  564. }
  565. private void DeleteTempFile(string file)
  566. {
  567. try
  568. {
  569. _fileSystem.DeleteFile(file);
  570. }
  571. catch (IOException)
  572. {
  573. // Might not have been created at all. No need to worry.
  574. }
  575. }
  576. private void ValidateParams(HttpRequestOptions options)
  577. {
  578. if (string.IsNullOrEmpty(options.Url))
  579. {
  580. throw new ArgumentNullException(nameof(options));
  581. }
  582. }
  583. /// <summary>
  584. /// Gets the host from URL.
  585. /// </summary>
  586. /// <param name="url">The URL.</param>
  587. /// <returns>System.String.</returns>
  588. private static string GetHostFromUrl(string url)
  589. {
  590. var index = url.IndexOf("://", StringComparison.OrdinalIgnoreCase);
  591. if (index != -1)
  592. {
  593. url = url.Substring(index + 3);
  594. var host = url.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault();
  595. if (!string.IsNullOrWhiteSpace(host))
  596. {
  597. return host;
  598. }
  599. }
  600. return url;
  601. }
  602. /// <summary>
  603. /// Throws the cancellation exception.
  604. /// </summary>
  605. /// <param name="options">The options.</param>
  606. /// <param name="client">The client.</param>
  607. /// <param name="cancellationToken">The cancellation token.</param>
  608. /// <param name="exception">The exception.</param>
  609. /// <returns>Exception.</returns>
  610. private Exception GetCancellationException(HttpRequestOptions options, HttpClientInfo client, CancellationToken cancellationToken, OperationCanceledException exception)
  611. {
  612. // If the HttpClient's timeout is reached, it will cancel the Task internally
  613. if (!cancellationToken.IsCancellationRequested)
  614. {
  615. var msg = string.Format("Connection to {0} timed out", options.Url);
  616. if (options.LogErrors)
  617. {
  618. _logger.LogError(msg);
  619. }
  620. client.LastTimeout = DateTime.UtcNow;
  621. // Throw an HttpException so that the caller doesn't think it was cancelled by user code
  622. return new HttpException(msg, exception)
  623. {
  624. IsTimedOut = true
  625. };
  626. }
  627. return exception;
  628. }
  629. private void EnsureSuccessStatusCode(HttpClientInfo client, HttpWebResponse response, HttpRequestOptions options)
  630. {
  631. var statusCode = response.StatusCode;
  632. var isSuccessful = statusCode >= HttpStatusCode.OK && statusCode <= (HttpStatusCode)299;
  633. if (isSuccessful)
  634. {
  635. return;
  636. }
  637. if (options.LogErrorResponseBody)
  638. {
  639. try
  640. {
  641. using (var stream = response.GetResponseStream())
  642. {
  643. if (stream != null)
  644. {
  645. using (var reader = new StreamReader(stream))
  646. {
  647. var msg = reader.ReadToEnd();
  648. _logger.LogError(msg);
  649. }
  650. }
  651. }
  652. }
  653. catch
  654. {
  655. }
  656. }
  657. throw new HttpException(response.StatusDescription)
  658. {
  659. StatusCode = response.StatusCode
  660. };
  661. }
  662. private static Task<WebResponse> GetResponseAsync(WebRequest request, TimeSpan timeout)
  663. {
  664. var taskCompletion = new TaskCompletionSource<WebResponse>();
  665. Task<WebResponse> asyncTask = Task.Factory.FromAsync<WebResponse>(request.BeginGetResponse, request.EndGetResponse, null);
  666. ThreadPool.RegisterWaitForSingleObject((asyncTask as IAsyncResult).AsyncWaitHandle, TimeoutCallback, request, timeout, true);
  667. var callback = new TaskCallback { taskCompletion = taskCompletion };
  668. asyncTask.ContinueWith(callback.OnSuccess, TaskContinuationOptions.NotOnFaulted);
  669. // Handle errors
  670. asyncTask.ContinueWith(callback.OnError, TaskContinuationOptions.OnlyOnFaulted);
  671. return taskCompletion.Task;
  672. }
  673. private static void TimeoutCallback(object state, bool timedOut)
  674. {
  675. if (timedOut && state != null)
  676. {
  677. WebRequest request = (WebRequest)state;
  678. request.Abort();
  679. }
  680. }
  681. private class TaskCallback
  682. {
  683. public TaskCompletionSource<WebResponse> taskCompletion;
  684. public void OnSuccess(Task<WebResponse> task)
  685. {
  686. taskCompletion.TrySetResult(task.Result);
  687. }
  688. public void OnError(Task<WebResponse> task)
  689. {
  690. if (task.Exception == null)
  691. {
  692. taskCompletion.TrySetException(Enumerable.Empty<Exception>());
  693. }
  694. else
  695. {
  696. taskCompletion.TrySetException(task.Exception);
  697. }
  698. }
  699. }
  700. }
  701. }