HttpClientManager.cs 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855
  1. using System;
  2. using System.Collections.Concurrent;
  3. using System.Collections.Generic;
  4. using System.Globalization;
  5. using System.IO;
  6. using System.Linq;
  7. using System.Net;
  8. using System.Net.Cache;
  9. using System.Text;
  10. using System.Threading;
  11. using System.Threading.Tasks;
  12. using MediaBrowser.Common.Configuration;
  13. using MediaBrowser.Common.Extensions;
  14. using MediaBrowser.Common.Net;
  15. using MediaBrowser.Model.IO;
  16. using MediaBrowser.Model.Net;
  17. using Microsoft.Extensions.Logging;
  18. namespace Emby.Server.Implementations.HttpClientManager
  19. {
  20. /// <summary>
  21. /// Class HttpClientManager
  22. /// </summary>
  23. public class HttpClientManager : IHttpClient
  24. {
  25. /// <summary>
  26. /// When one request to a host times out, we'll ban all other requests for this period of time, to prevent scans from stalling
  27. /// </summary>
  28. private const int TimeoutSeconds = 30;
  29. /// <summary>
  30. /// The _logger
  31. /// </summary>
  32. private readonly ILogger _logger;
  33. /// <summary>
  34. /// The _app paths
  35. /// </summary>
  36. private readonly IApplicationPaths _appPaths;
  37. private readonly IFileSystem _fileSystem;
  38. private readonly Func<string> _defaultUserAgentFn;
  39. /// <summary>
  40. /// Initializes a new instance of the <see cref="HttpClientManager" /> class.
  41. /// </summary>
  42. public HttpClientManager(
  43. IApplicationPaths appPaths,
  44. ILoggerFactory loggerFactory,
  45. IFileSystem fileSystem,
  46. Func<string> defaultUserAgentFn)
  47. {
  48. if (appPaths == null)
  49. {
  50. throw new ArgumentNullException(nameof(appPaths));
  51. }
  52. if (loggerFactory == null)
  53. {
  54. throw new ArgumentNullException(nameof(loggerFactory));
  55. }
  56. _logger = loggerFactory.CreateLogger("HttpClient");
  57. _fileSystem = fileSystem;
  58. _appPaths = appPaths;
  59. _defaultUserAgentFn = defaultUserAgentFn;
  60. // http://stackoverflow.com/questions/566437/http-post-returns-the-error-417-expectation-failed-c
  61. ServicePointManager.Expect100Continue = false;
  62. #if NET46
  63. // Trakt requests sometimes fail without this
  64. ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls;
  65. #endif
  66. }
  67. /// <summary>
  68. /// Holds a dictionary of http clients by host. Use GetHttpClient(host) to retrieve or create a client for web requests.
  69. /// DON'T dispose it after use.
  70. /// </summary>
  71. /// <value>The HTTP clients.</value>
  72. private readonly ConcurrentDictionary<string, HttpClientInfo> _httpClients = new ConcurrentDictionary<string, HttpClientInfo>();
  73. /// <summary>
  74. /// Gets
  75. /// </summary>
  76. /// <param name="host">The host.</param>
  77. /// <param name="enableHttpCompression">if set to <c>true</c> [enable HTTP compression].</param>
  78. /// <returns>HttpClient.</returns>
  79. /// <exception cref="ArgumentNullException">host</exception>
  80. private HttpClientInfo GetHttpClient(string host, bool enableHttpCompression)
  81. {
  82. if (string.IsNullOrEmpty(host))
  83. {
  84. throw new ArgumentNullException(nameof(host));
  85. }
  86. var key = host + enableHttpCompression;
  87. if (!_httpClients.TryGetValue(key, out var client))
  88. {
  89. client = new HttpClientInfo();
  90. _httpClients.TryAdd(key, client);
  91. }
  92. return client;
  93. }
  94. private static WebRequest CreateWebRequest(string url)
  95. {
  96. try
  97. {
  98. return WebRequest.Create(url);
  99. }
  100. catch (NotSupportedException)
  101. {
  102. //Webrequest creation does fail on MONO randomly when using WebRequest.Create
  103. //the issue occurs in the GetCreator method here: http://www.oschina.net/code/explore/mono-2.8.1/mcs/class/System/System.Net/WebRequest.cs
  104. var type = Type.GetType("System.Net.HttpRequestCreator, System, Version=4.0.0.0,Culture=neutral, PublicKeyToken=b77a5c561934e089");
  105. var creator = Activator.CreateInstance(type, nonPublic: true) as IWebRequestCreate;
  106. return creator.Create(new Uri(url)) as HttpWebRequest;
  107. }
  108. }
  109. private WebRequest GetRequest(HttpRequestOptions options, string method)
  110. {
  111. string url = options.Url;
  112. var uriAddress = new Uri(url);
  113. string userInfo = uriAddress.UserInfo;
  114. if (!string.IsNullOrWhiteSpace(userInfo))
  115. {
  116. _logger.LogInformation("Found userInfo in url: {0} ... url: {1}", userInfo, url);
  117. url = url.Replace(userInfo + "@", string.Empty);
  118. }
  119. var request = CreateWebRequest(url);
  120. if (request is HttpWebRequest httpWebRequest)
  121. {
  122. AddRequestHeaders(httpWebRequest, options);
  123. if (options.EnableHttpCompression)
  124. {
  125. httpWebRequest.AutomaticDecompression = DecompressionMethods.Deflate;
  126. if (options.DecompressionMethod.HasValue
  127. && options.DecompressionMethod.Value == CompressionMethod.Gzip)
  128. {
  129. httpWebRequest.AutomaticDecompression = DecompressionMethods.GZip;
  130. }
  131. }
  132. else
  133. {
  134. httpWebRequest.AutomaticDecompression = DecompressionMethods.None;
  135. }
  136. httpWebRequest.KeepAlive = options.EnableKeepAlive;
  137. if (!string.IsNullOrEmpty(options.Host))
  138. {
  139. httpWebRequest.Host = options.Host;
  140. }
  141. if (!string.IsNullOrEmpty(options.Referer))
  142. {
  143. httpWebRequest.Referer = options.Referer;
  144. }
  145. }
  146. request.CachePolicy = new RequestCachePolicy(RequestCacheLevel.BypassCache);
  147. request.Method = method;
  148. request.Timeout = options.TimeoutMs;
  149. if (!string.IsNullOrWhiteSpace(userInfo))
  150. {
  151. var parts = userInfo.Split(':');
  152. if (parts.Length == 2)
  153. {
  154. request.Credentials = GetCredential(url, parts[0], parts[1]);
  155. // TODO: .net core ??
  156. request.PreAuthenticate = true;
  157. }
  158. }
  159. return request;
  160. }
  161. private static CredentialCache GetCredential(string url, string username, string password)
  162. {
  163. //ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3;
  164. var credentialCache = new CredentialCache();
  165. credentialCache.Add(new Uri(url), "Basic", new NetworkCredential(username, password));
  166. return credentialCache;
  167. }
  168. private void AddRequestHeaders(HttpWebRequest request, HttpRequestOptions options)
  169. {
  170. var hasUserAgent = false;
  171. foreach (var header in options.RequestHeaders)
  172. {
  173. if (string.Equals(header.Key, "Accept", StringComparison.OrdinalIgnoreCase))
  174. {
  175. request.Accept = header.Value;
  176. }
  177. else if (string.Equals(header.Key, "User-Agent", StringComparison.OrdinalIgnoreCase))
  178. {
  179. SetUserAgent(request, header.Value);
  180. hasUserAgent = true;
  181. }
  182. else
  183. {
  184. request.Headers.Set(header.Key, header.Value);
  185. }
  186. }
  187. if (!hasUserAgent && options.EnableDefaultUserAgent)
  188. {
  189. SetUserAgent(request, _defaultUserAgentFn());
  190. }
  191. }
  192. private static void SetUserAgent(HttpWebRequest request, string userAgent)
  193. {
  194. request.UserAgent = userAgent;
  195. }
  196. /// <summary>
  197. /// Gets the response internal.
  198. /// </summary>
  199. /// <param name="options">The options.</param>
  200. /// <returns>Task{HttpResponseInfo}.</returns>
  201. public Task<HttpResponseInfo> GetResponse(HttpRequestOptions options)
  202. {
  203. return SendAsync(options, "GET");
  204. }
  205. /// <summary>
  206. /// Performs a GET request and returns the resulting stream
  207. /// </summary>
  208. /// <param name="options">The options.</param>
  209. /// <returns>Task{Stream}.</returns>
  210. public async Task<Stream> Get(HttpRequestOptions options)
  211. {
  212. var response = await GetResponse(options).ConfigureAwait(false);
  213. return response.Content;
  214. }
  215. /// <summary>
  216. /// send as an asynchronous operation.
  217. /// </summary>
  218. /// <param name="options">The options.</param>
  219. /// <param name="httpMethod">The HTTP method.</param>
  220. /// <returns>Task{HttpResponseInfo}.</returns>
  221. /// <exception cref="HttpException">
  222. /// </exception>
  223. public async Task<HttpResponseInfo> SendAsync(HttpRequestOptions options, string httpMethod)
  224. {
  225. if (options.CacheMode == CacheMode.None)
  226. {
  227. return await SendAsyncInternal(options, httpMethod).ConfigureAwait(false);
  228. }
  229. var url = options.Url;
  230. var urlHash = url.ToLower().GetMD5().ToString("N");
  231. var responseCachePath = Path.Combine(_appPaths.CachePath, "httpclient", urlHash);
  232. var response = await GetCachedResponse(responseCachePath, options.CacheLength, url).ConfigureAwait(false);
  233. if (response != null)
  234. {
  235. return response;
  236. }
  237. response = await SendAsyncInternal(options, httpMethod).ConfigureAwait(false);
  238. if (response.StatusCode == HttpStatusCode.OK)
  239. {
  240. await CacheResponse(response, responseCachePath).ConfigureAwait(false);
  241. }
  242. return response;
  243. }
  244. private async Task<HttpResponseInfo> GetCachedResponse(string responseCachePath, TimeSpan cacheLength, string url)
  245. {
  246. try
  247. {
  248. if (_fileSystem.GetLastWriteTimeUtc(responseCachePath).Add(cacheLength) > DateTime.UtcNow)
  249. {
  250. using (var stream = _fileSystem.GetFileStream(responseCachePath, FileOpenMode.Open, FileAccessMode.Read, FileShareMode.Read, true))
  251. {
  252. var memoryStream = new MemoryStream();
  253. await stream.CopyToAsync(memoryStream).ConfigureAwait(false);
  254. memoryStream.Position = 0;
  255. return new HttpResponseInfo
  256. {
  257. ResponseUrl = url,
  258. Content = memoryStream,
  259. StatusCode = HttpStatusCode.OK,
  260. ContentLength = memoryStream.Length
  261. };
  262. }
  263. }
  264. }
  265. catch (FileNotFoundException)
  266. {
  267. }
  268. catch (DirectoryNotFoundException)
  269. {
  270. }
  271. return null;
  272. }
  273. private async Task CacheResponse(HttpResponseInfo response, string responseCachePath)
  274. {
  275. _fileSystem.CreateDirectory(_fileSystem.GetDirectoryName(responseCachePath));
  276. using (var responseStream = response.Content)
  277. {
  278. var memoryStream = new MemoryStream();
  279. await responseStream.CopyToAsync(memoryStream).ConfigureAwait(false);
  280. memoryStream.Position = 0;
  281. using (var fileStream = _fileSystem.GetFileStream(responseCachePath, FileOpenMode.Create, FileAccessMode.Write, FileShareMode.None, true))
  282. {
  283. await memoryStream.CopyToAsync(fileStream).ConfigureAwait(false);
  284. memoryStream.Position = 0;
  285. response.Content = memoryStream;
  286. }
  287. }
  288. }
  289. private async Task<HttpResponseInfo> SendAsyncInternal(HttpRequestOptions options, string httpMethod)
  290. {
  291. ValidateParams(options);
  292. options.CancellationToken.ThrowIfCancellationRequested();
  293. var client = GetHttpClient(GetHostFromUrl(options.Url), options.EnableHttpCompression);
  294. if ((DateTime.UtcNow - client.LastTimeout).TotalSeconds < TimeoutSeconds)
  295. {
  296. throw new HttpException(string.Format("Cancelling connection to {0} due to a previous timeout.", options.Url))
  297. {
  298. IsTimedOut = true
  299. };
  300. }
  301. var httpWebRequest = GetRequest(options, httpMethod);
  302. if (options.RequestContentBytes != null ||
  303. !string.IsNullOrEmpty(options.RequestContent) ||
  304. string.Equals(httpMethod, "post", StringComparison.OrdinalIgnoreCase))
  305. {
  306. try
  307. {
  308. var bytes = options.RequestContentBytes ?? Encoding.UTF8.GetBytes(options.RequestContent ?? string.Empty);
  309. var contentType = options.RequestContentType ?? "application/x-www-form-urlencoded";
  310. if (options.AppendCharsetToMimeType)
  311. {
  312. contentType = contentType.TrimEnd(';') + "; charset=\"utf-8\"";
  313. }
  314. httpWebRequest.ContentType = contentType;
  315. httpWebRequest.ContentLength = bytes.Length;
  316. (await httpWebRequest.GetRequestStreamAsync().ConfigureAwait(false)).Write(bytes, 0, bytes.Length);
  317. }
  318. catch (Exception ex)
  319. {
  320. throw new HttpException(ex.Message) { IsTimedOut = true };
  321. }
  322. }
  323. if (options.ResourcePool != null)
  324. {
  325. await options.ResourcePool.WaitAsync(options.CancellationToken).ConfigureAwait(false);
  326. }
  327. if ((DateTime.UtcNow - client.LastTimeout).TotalSeconds < TimeoutSeconds)
  328. {
  329. options.ResourcePool?.Release();
  330. throw new HttpException($"Connection to {options.Url} timed out") { IsTimedOut = true };
  331. }
  332. if (options.LogRequest)
  333. {
  334. if (options.LogRequestAsDebug)
  335. {
  336. _logger.LogDebug("HttpClientManager {0}: {1}", httpMethod.ToUpper(), options.Url);
  337. }
  338. else
  339. {
  340. _logger.LogInformation("HttpClientManager {0}: {1}", httpMethod.ToUpper(), options.Url);
  341. }
  342. }
  343. try
  344. {
  345. options.CancellationToken.ThrowIfCancellationRequested();
  346. if (!options.BufferContent)
  347. {
  348. var response = await GetResponseAsync(httpWebRequest, TimeSpan.FromMilliseconds(options.TimeoutMs)).ConfigureAwait(false);
  349. var httpResponse = (HttpWebResponse)response;
  350. EnsureSuccessStatusCode(client, httpResponse, options);
  351. options.CancellationToken.ThrowIfCancellationRequested();
  352. return GetResponseInfo(httpResponse, httpResponse.GetResponseStream(), GetContentLength(httpResponse), httpResponse);
  353. }
  354. using (var response = await GetResponseAsync(httpWebRequest, TimeSpan.FromMilliseconds(options.TimeoutMs)).ConfigureAwait(false))
  355. {
  356. var httpResponse = (HttpWebResponse)response;
  357. EnsureSuccessStatusCode(client, httpResponse, options);
  358. options.CancellationToken.ThrowIfCancellationRequested();
  359. using (var stream = httpResponse.GetResponseStream())
  360. {
  361. var memoryStream = new MemoryStream();
  362. await stream.CopyToAsync(memoryStream).ConfigureAwait(false);
  363. memoryStream.Position = 0;
  364. return GetResponseInfo(httpResponse, memoryStream, memoryStream.Length, null);
  365. }
  366. }
  367. }
  368. catch (OperationCanceledException ex)
  369. {
  370. throw GetCancellationException(options, client, options.CancellationToken, ex);
  371. }
  372. catch (Exception ex)
  373. {
  374. throw GetException(ex, options, client);
  375. }
  376. finally
  377. {
  378. options.ResourcePool?.Release();
  379. }
  380. }
  381. private HttpResponseInfo GetResponseInfo(HttpWebResponse httpResponse, Stream content, long? contentLength, IDisposable disposable)
  382. {
  383. var responseInfo = new HttpResponseInfo(disposable)
  384. {
  385. Content = content,
  386. StatusCode = httpResponse.StatusCode,
  387. ContentType = httpResponse.ContentType,
  388. ContentLength = contentLength,
  389. ResponseUrl = httpResponse.ResponseUri.ToString()
  390. };
  391. if (httpResponse.Headers != null)
  392. {
  393. SetHeaders(httpResponse.Headers, responseInfo);
  394. }
  395. return responseInfo;
  396. }
  397. private HttpResponseInfo GetResponseInfo(HttpWebResponse httpResponse, string tempFile, long? contentLength)
  398. {
  399. var responseInfo = new HttpResponseInfo
  400. {
  401. TempFilePath = tempFile,
  402. StatusCode = httpResponse.StatusCode,
  403. ContentType = httpResponse.ContentType,
  404. ContentLength = contentLength
  405. };
  406. if (httpResponse.Headers != null)
  407. {
  408. SetHeaders(httpResponse.Headers, responseInfo);
  409. }
  410. return responseInfo;
  411. }
  412. private static void SetHeaders(WebHeaderCollection headers, HttpResponseInfo responseInfo)
  413. {
  414. foreach (var key in headers.AllKeys)
  415. {
  416. responseInfo.Headers[key] = headers[key];
  417. }
  418. }
  419. public Task<HttpResponseInfo> Post(HttpRequestOptions options)
  420. {
  421. return SendAsync(options, "POST");
  422. }
  423. /// <summary>
  424. /// Performs a POST request
  425. /// </summary>
  426. /// <param name="options">The options.</param>
  427. /// <param name="postData">Params to add to the POST data.</param>
  428. /// <returns>stream on success, null on failure</returns>
  429. public async Task<Stream> Post(HttpRequestOptions options, Dictionary<string, string> postData)
  430. {
  431. options.SetPostData(postData);
  432. var response = await Post(options).ConfigureAwait(false);
  433. return response.Content;
  434. }
  435. /// <summary>
  436. /// Downloads the contents of a given url into a temporary location
  437. /// </summary>
  438. /// <param name="options">The options.</param>
  439. /// <returns>Task{System.String}.</returns>
  440. public async Task<string> GetTempFile(HttpRequestOptions options)
  441. {
  442. var response = await GetTempFileResponse(options).ConfigureAwait(false);
  443. return response.TempFilePath;
  444. }
  445. public async Task<HttpResponseInfo> GetTempFileResponse(HttpRequestOptions options)
  446. {
  447. ValidateParams(options);
  448. _fileSystem.CreateDirectory(_appPaths.TempDirectory);
  449. var tempFile = Path.Combine(_appPaths.TempDirectory, Guid.NewGuid() + ".tmp");
  450. if (options.Progress == null)
  451. {
  452. throw new ArgumentException("Options did not have a Progress value.", nameof(options));
  453. }
  454. options.CancellationToken.ThrowIfCancellationRequested();
  455. var httpWebRequest = GetRequest(options, "GET");
  456. if (options.ResourcePool != null)
  457. {
  458. await options.ResourcePool.WaitAsync(options.CancellationToken).ConfigureAwait(false);
  459. }
  460. options.Progress.Report(0);
  461. if (options.LogRequest)
  462. {
  463. if (options.LogRequestAsDebug)
  464. {
  465. _logger.LogDebug("HttpClientManager.GetTempFileResponse url: {0}", options.Url);
  466. }
  467. else
  468. {
  469. _logger.LogInformation("HttpClientManager.GetTempFileResponse url: {0}", options.Url);
  470. }
  471. }
  472. var client = GetHttpClient(GetHostFromUrl(options.Url), options.EnableHttpCompression);
  473. try
  474. {
  475. options.CancellationToken.ThrowIfCancellationRequested();
  476. using (var response = await httpWebRequest.GetResponseAsync().ConfigureAwait(false))
  477. {
  478. var httpResponse = (HttpWebResponse)response;
  479. EnsureSuccessStatusCode(client, httpResponse, options);
  480. options.CancellationToken.ThrowIfCancellationRequested();
  481. var contentLength = GetContentLength(httpResponse);
  482. if (contentLength.HasValue)
  483. {
  484. using (var fs = _fileSystem.GetFileStream(tempFile, FileOpenMode.Create, FileAccessMode.Write, FileShareMode.Read, true))
  485. {
  486. await httpResponse.GetResponseStream().CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, options.CancellationToken).ConfigureAwait(false);
  487. }
  488. }
  489. else
  490. {
  491. // We're not able to track progress
  492. using (var stream = httpResponse.GetResponseStream())
  493. using (var fs = _fileSystem.GetFileStream(tempFile, FileOpenMode.Create, FileAccessMode.Write, FileShareMode.Read, true))
  494. {
  495. await stream.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, options.CancellationToken).ConfigureAwait(false);
  496. }
  497. }
  498. options.Progress.Report(100);
  499. return GetResponseInfo(httpResponse, tempFile, contentLength);
  500. }
  501. }
  502. catch (Exception ex)
  503. {
  504. DeleteTempFile(tempFile);
  505. throw GetException(ex, options, client);
  506. }
  507. finally
  508. {
  509. options.ResourcePool?.Release();
  510. }
  511. }
  512. private static long? GetContentLength(HttpWebResponse response)
  513. {
  514. var length = response.ContentLength;
  515. if (length == 0)
  516. {
  517. return null;
  518. }
  519. return length;
  520. }
  521. protected static readonly CultureInfo UsCulture = new CultureInfo("en-US");
  522. private Exception GetException(Exception ex, HttpRequestOptions options, HttpClientInfo client)
  523. {
  524. if (ex is HttpException)
  525. {
  526. return ex;
  527. }
  528. var webException = ex as WebException
  529. ?? ex.InnerException as WebException;
  530. if (webException != null)
  531. {
  532. if (options.LogErrors)
  533. {
  534. _logger.LogError(webException, "Error {status} getting response from {url}", webException.Status, options.Url);
  535. }
  536. var exception = new HttpException(webException.Message, webException);
  537. var response = webException.Response as HttpWebResponse;
  538. if (response != null)
  539. {
  540. exception.StatusCode = response.StatusCode;
  541. if ((int)response.StatusCode == 429)
  542. {
  543. client.LastTimeout = DateTime.UtcNow;
  544. }
  545. }
  546. if (!exception.StatusCode.HasValue)
  547. {
  548. if (webException.Status == WebExceptionStatus.NameResolutionFailure ||
  549. webException.Status == WebExceptionStatus.ConnectFailure)
  550. {
  551. exception.IsTimedOut = true;
  552. }
  553. }
  554. return exception;
  555. }
  556. var operationCanceledException = ex as OperationCanceledException
  557. ?? ex.InnerException as OperationCanceledException;
  558. if (operationCanceledException != null)
  559. {
  560. return GetCancellationException(options, client, options.CancellationToken, operationCanceledException);
  561. }
  562. if (options.LogErrors)
  563. {
  564. _logger.LogError(ex, "Error getting response from {url}", options.Url);
  565. }
  566. return ex;
  567. }
  568. private void DeleteTempFile(string file)
  569. {
  570. try
  571. {
  572. _fileSystem.DeleteFile(file);
  573. }
  574. catch (IOException)
  575. {
  576. // Might not have been created at all. No need to worry.
  577. }
  578. }
  579. private void ValidateParams(HttpRequestOptions options)
  580. {
  581. if (string.IsNullOrEmpty(options.Url))
  582. {
  583. throw new ArgumentNullException(nameof(options));
  584. }
  585. }
  586. /// <summary>
  587. /// Gets the host from URL.
  588. /// </summary>
  589. /// <param name="url">The URL.</param>
  590. /// <returns>System.String.</returns>
  591. private static string GetHostFromUrl(string url)
  592. {
  593. var index = url.IndexOf("://", StringComparison.OrdinalIgnoreCase);
  594. if (index != -1)
  595. {
  596. url = url.Substring(index + 3);
  597. var host = url.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault();
  598. if (!string.IsNullOrWhiteSpace(host))
  599. {
  600. return host;
  601. }
  602. }
  603. return url;
  604. }
  605. /// <summary>
  606. /// Throws the cancellation exception.
  607. /// </summary>
  608. /// <param name="options">The options.</param>
  609. /// <param name="client">The client.</param>
  610. /// <param name="cancellationToken">The cancellation token.</param>
  611. /// <param name="exception">The exception.</param>
  612. /// <returns>Exception.</returns>
  613. private Exception GetCancellationException(HttpRequestOptions options, HttpClientInfo client, CancellationToken cancellationToken, OperationCanceledException exception)
  614. {
  615. // If the HttpClient's timeout is reached, it will cancel the Task internally
  616. if (!cancellationToken.IsCancellationRequested)
  617. {
  618. var msg = string.Format("Connection to {0} timed out", options.Url);
  619. if (options.LogErrors)
  620. {
  621. _logger.LogError(msg);
  622. }
  623. client.LastTimeout = DateTime.UtcNow;
  624. // Throw an HttpException so that the caller doesn't think it was cancelled by user code
  625. return new HttpException(msg, exception)
  626. {
  627. IsTimedOut = true
  628. };
  629. }
  630. return exception;
  631. }
  632. private void EnsureSuccessStatusCode(HttpClientInfo client, HttpWebResponse response, HttpRequestOptions options)
  633. {
  634. var statusCode = response.StatusCode;
  635. var isSuccessful = statusCode >= HttpStatusCode.OK && statusCode <= (HttpStatusCode)299;
  636. if (isSuccessful)
  637. {
  638. return;
  639. }
  640. if (options.LogErrorResponseBody)
  641. {
  642. try
  643. {
  644. using (var stream = response.GetResponseStream())
  645. {
  646. if (stream != null)
  647. {
  648. using (var reader = new StreamReader(stream))
  649. {
  650. var msg = reader.ReadToEnd();
  651. _logger.LogError(msg);
  652. }
  653. }
  654. }
  655. }
  656. catch
  657. {
  658. }
  659. }
  660. throw new HttpException(response.StatusDescription)
  661. {
  662. StatusCode = response.StatusCode
  663. };
  664. }
  665. private static Task<WebResponse> GetResponseAsync(WebRequest request, TimeSpan timeout)
  666. {
  667. var taskCompletion = new TaskCompletionSource<WebResponse>();
  668. var asyncTask = Task.Factory.FromAsync(request.BeginGetResponse, request.EndGetResponse, null);
  669. ThreadPool.RegisterWaitForSingleObject((asyncTask as IAsyncResult).AsyncWaitHandle, TimeoutCallback, request, timeout, true);
  670. var callback = new TaskCallback { taskCompletion = taskCompletion };
  671. asyncTask.ContinueWith(callback.OnSuccess, TaskContinuationOptions.NotOnFaulted);
  672. // Handle errors
  673. asyncTask.ContinueWith(callback.OnError, TaskContinuationOptions.OnlyOnFaulted);
  674. return taskCompletion.Task;
  675. }
  676. private static void TimeoutCallback(object state, bool timedOut)
  677. {
  678. if (timedOut && state != null)
  679. {
  680. var request = (WebRequest)state;
  681. request.Abort();
  682. }
  683. }
  684. private class TaskCallback
  685. {
  686. public TaskCompletionSource<WebResponse> taskCompletion;
  687. public void OnSuccess(Task<WebResponse> task)
  688. {
  689. taskCompletion.TrySetResult(task.Result);
  690. }
  691. public void OnError(Task<WebResponse> task)
  692. {
  693. if (task.Exception == null)
  694. {
  695. taskCompletion.TrySetException(Enumerable.Empty<Exception>());
  696. }
  697. else
  698. {
  699. taskCompletion.TrySetException(task.Exception);
  700. }
  701. }
  702. }
  703. }
  704. }