replaced http client cache with longer lived cache

MediaBrowser.Common.Implementations/HttpClientManager/HttpClientManager.cs

@@ -36,7 +36,7 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager
         private readonly IApplicationPaths _appPaths;
 
         private readonly IJsonSerializer _jsonSerializer;
-        //private readonly FileSystemRepository _cacheRepository;
+        private readonly FileSystemRepository _cacheRepository;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="HttpClientManager" /> class.
@@ -64,7 +64,7 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager
             _jsonSerializer = jsonSerializer;
             _appPaths = appPaths;
 
-            //_cacheRepository = new FileSystemRepository(Path.Combine(_appPaths.CachePath, "http"));
+            _cacheRepository = new FileSystemRepository(Path.Combine(_appPaths.CachePath, "downloads"));
         }
 
         /// <summary>
@@ -115,41 +115,57 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager
         {
             ValidateParams(options.Url, options.CancellationToken);
 
-            //var urlHash = url.GetMD5().ToString();
-            //var infoPath = _cacheRepository.GetResourcePath(urlHash + ".js");
-            //var responsePath = _cacheRepository.GetResourcePath(urlHash + ".dat");
+            HttpResponseInfo cachedInfo = null;
 
-            //HttpResponseInfo cachedInfo = null;
+            var urlHash = options.Url.GetMD5().ToString();
+            var cachedInfoPath = _cacheRepository.GetResourcePath(urlHash + ".js");
+            var cachedReponsePath = _cacheRepository.GetResourcePath(urlHash + ".dat");
 
-            //try
-            //{
-            //    cachedInfo = _jsonSerializer.DeserializeFromFile<HttpResponseInfo>(infoPath);
-            //}
-            //catch (FileNotFoundException)
-            //{
+            if (options.EnableResponseCache)
+            {
+                try
+                {
+                    cachedInfo = _jsonSerializer.DeserializeFromFile<HttpResponseInfo>(cachedInfoPath);
+                }
+                catch (FileNotFoundException)
+                {
 
-            //}
+                }
 
-            //if (cachedInfo != null && !cachedInfo.MustRevalidate && cachedInfo.Expires.HasValue && cachedInfo.Expires.Value > DateTime.UtcNow)
-            //{
-            //    return GetCachedResponse(responsePath);
-            //}
+                if (cachedInfo != null)
+                {
+                    var isCacheValid = (!cachedInfo.MustRevalidate && !string.IsNullOrEmpty(cachedInfo.Etag)) 
+                        || (cachedInfo.Expires.HasValue && cachedInfo.Expires.Value > DateTime.UtcNow);
+
+                    if (isCacheValid)
+                    {
+                        try
+                        {
+                            return GetCachedResponse(cachedReponsePath);
+                        }
+                        catch (FileNotFoundException)
+                        {
+
+                        }
+                    }
+                }
+            }
 
             options.CancellationToken.ThrowIfCancellationRequested();
 
             var message = GetHttpRequestMessage(options);
 
-            //if (cachedInfo != null)
-            //{
-            //    if (!string.IsNullOrEmpty(cachedInfo.Etag))
-            //    {
-            //        message.Headers.Add("If-None-Match", cachedInfo.Etag);
-            //    }
-            //    else if (cachedInfo.LastModified.HasValue)
-            //    {
-            //        message.Headers.IfModifiedSince = new DateTimeOffset(cachedInfo.LastModified.Value);
-            //    }
-            //}
+            if (options.EnableResponseCache && cachedInfo != null)
+            {
+                if (!string.IsNullOrEmpty(cachedInfo.Etag))
+                {
+                    message.Headers.Add("If-None-Match", cachedInfo.Etag);
+                }
+                else if (cachedInfo.LastModified.HasValue)
+                {
+                    message.Headers.IfModifiedSince = new DateTimeOffset(cachedInfo.LastModified.Value);
+                }
+            }
 
             if (options.ResourcePool != null)
             {
@@ -168,19 +184,22 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager
 
                 options.CancellationToken.ThrowIfCancellationRequested();
 
-                //cachedInfo = UpdateInfoCache(cachedInfo, url, infoPath, response);
+                if (options.EnableResponseCache)
+                {
+                    cachedInfo = UpdateInfoCache(cachedInfo, options.Url, cachedInfoPath, response);
 
-                //if (response.StatusCode == HttpStatusCode.NotModified)
-                //{
-                //    return GetCachedResponse(responsePath);
-                //}
+                    if (response.StatusCode == HttpStatusCode.NotModified)
+                    {
+                        return GetCachedResponse(cachedReponsePath);
+                    }
 
-                //if (!string.IsNullOrEmpty(cachedInfo.Etag) || cachedInfo.LastModified.HasValue || (cachedInfo.Expires.HasValue && cachedInfo.Expires.Value > DateTime.UtcNow))
-                //{
-                //    await UpdateResponseCache(response, responsePath).ConfigureAwait(false);
+                    if (!string.IsNullOrEmpty(cachedInfo.Etag) || cachedInfo.LastModified.HasValue || (cachedInfo.Expires.HasValue && cachedInfo.Expires.Value > DateTime.UtcNow))
+                    {
+                        await UpdateResponseCache(response, cachedReponsePath).ConfigureAwait(false);
 
-                //    return GetCachedResponse(responsePath);
-                //}
+                        return GetCachedResponse(cachedReponsePath);
+                    }
+                }
 
                 return await response.Content.ReadAsStreamAsync().ConfigureAwait(false);
             }
@@ -237,7 +256,6 @@ namespace MediaBrowser.Common.Implementations.HttpClientManager
             return Get(url, null, cancellationToken);
         }
 
-
         /// <summary>
         /// Gets the cached response.
         /// </summary>

MediaBrowser.Common/Net/HttpRequestOptions.cs

@@ -43,5 +43,11 @@ namespace MediaBrowser.Common.Net
         /// </summary>
         /// <value>The progress.</value>
         public IProgress<double> Progress { get; set; }
+
+        /// <summary>
+        /// Gets or sets a value indicating whether [enable response caching].
+        /// </summary>
+        /// <value><c>true</c> if [enable response caching]; otherwise, <c>false</c>.</value>
+        public bool EnableResponseCache { get; set; }
     }
 }

MediaBrowser.Controller/Providers/Movies/FanArtMovieProvider.cs

@@ -121,7 +121,14 @@ namespace MediaBrowser.Controller.Providers.Movies
 
             try
             {
-                using (var xml = await HttpClient.Get(url, FanArtResourcePool, cancellationToken).ConfigureAwait(false))
+                using (var xml = await HttpClient.Get(new HttpRequestOptions
+                {
+                    Url = url,
+                    ResourcePool = FanArtResourcePool,
+                    CancellationToken = cancellationToken,
+                    EnableResponseCache = true
+
+                }).ConfigureAwait(false))
                 {
                     doc.Load(xml);
                 }

MediaBrowser.Controller/Providers/Movies/MovieDbProvider.cs

@@ -174,7 +174,8 @@ namespace MediaBrowser.Controller.Providers.Movies
                     Url = string.Format(TmdbConfigUrl, ApiKey),
                     CancellationToken = CancellationToken.None,
                     ResourcePool = Current.MovieDbResourcePool,
-                    AcceptHeader = AcceptHeader
+                    AcceptHeader = AcceptHeader,
+                    EnableResponseCache = true
 
                 }).ConfigureAwait(false))
                 {
@@ -546,7 +547,8 @@ namespace MediaBrowser.Controller.Providers.Movies
                     Url = url3,
                     CancellationToken = cancellationToken,
                     ResourcePool = Current.MovieDbResourcePool,
-                    AcceptHeader = AcceptHeader
+                    AcceptHeader = AcceptHeader,
+                    EnableResponseCache = true
 
                 }).ConfigureAwait(false))
                 {
@@ -585,7 +587,8 @@ namespace MediaBrowser.Controller.Providers.Movies
                         Url = url3,
                         CancellationToken = cancellationToken,
                         ResourcePool = Current.MovieDbResourcePool,
-                        AcceptHeader = AcceptHeader
+                        AcceptHeader = AcceptHeader,
+                        EnableResponseCache = true
 
                     }).ConfigureAwait(false))
                     {
@@ -631,7 +634,8 @@ namespace MediaBrowser.Controller.Providers.Movies
                                 Url = url3,
                                 CancellationToken = cancellationToken,
                                 ResourcePool = Current.MovieDbResourcePool,
-                                AcceptHeader = AcceptHeader
+                                AcceptHeader = AcceptHeader,
+                                EnableResponseCache = true
 
                             }).ConfigureAwait(false))
                             {
@@ -720,7 +724,8 @@ namespace MediaBrowser.Controller.Providers.Movies
                         Url = url,
                         CancellationToken = cancellationToken,
                         ResourcePool = Current.MovieDbResourcePool,
-                        AcceptHeader = AcceptHeader
+                        AcceptHeader = AcceptHeader,
+                        EnableResponseCache = true
 
                     }).ConfigureAwait(false))
                     {
@@ -828,7 +833,8 @@ namespace MediaBrowser.Controller.Providers.Movies
                     Url = url,
                     CancellationToken = cancellationToken,
                     ResourcePool = Current.MovieDbResourcePool,
-                    AcceptHeader = AcceptHeader
+                    AcceptHeader = AcceptHeader,
+                    EnableResponseCache = true
 
                 }).ConfigureAwait(false))
                 {
@@ -867,7 +873,8 @@ namespace MediaBrowser.Controller.Providers.Movies
                             Url = url,
                             CancellationToken = cancellationToken,
                             ResourcePool = Current.MovieDbResourcePool,
-                            AcceptHeader = AcceptHeader
+                            AcceptHeader = AcceptHeader,
+                            EnableResponseCache = true
 
                         }).ConfigureAwait(false))
                         {
@@ -910,7 +917,8 @@ namespace MediaBrowser.Controller.Providers.Movies
                     Url = url,
                     CancellationToken = cancellationToken,
                     ResourcePool = Current.MovieDbResourcePool,
-                    AcceptHeader = AcceptHeader
+                    AcceptHeader = AcceptHeader,
+                    EnableResponseCache = true
 
                 }).ConfigureAwait(false))
                 {
@@ -944,7 +952,8 @@ namespace MediaBrowser.Controller.Providers.Movies
                     Url = url,
                     CancellationToken = cancellationToken,
                     ResourcePool = Current.MovieDbResourcePool,
-                    AcceptHeader = AcceptHeader
+                    AcceptHeader = AcceptHeader,
+                    EnableResponseCache = true
 
                 }).ConfigureAwait(false))
                 {
@@ -980,7 +989,8 @@ namespace MediaBrowser.Controller.Providers.Movies
                     Url = url,
                     CancellationToken = cancellationToken,
                     ResourcePool = Current.MovieDbResourcePool,
-                    AcceptHeader = AcceptHeader
+                    AcceptHeader = AcceptHeader,
+                    EnableResponseCache = true
 
                 }).ConfigureAwait(false))
                 {

MediaBrowser.Controller/Providers/Movies/RottenTomatoesMovieProvider.cs

@@ -186,7 +186,14 @@ namespace MediaBrowser.Controller.Providers.Movies
             RTMovieSearchResult hit = null;
 
             // Have IMDB Id
-            using (var stream = await HttpClient.Get(GetMovieImdbUrl(imdbId), _rottenTomatoesResourcePool, cancellationToken).ConfigureAwait(false))
+            using (var stream = await HttpClient.Get(new HttpRequestOptions
+            {
+                Url = GetMovieImdbUrl(imdbId),
+                ResourcePool = _rottenTomatoesResourcePool,
+                CancellationToken = cancellationToken,
+                EnableResponseCache = true
+
+            }).ConfigureAwait(false))
             {
                 var result = JsonSerializer.DeserializeFromStream<RTMovieSearchResult>(stream);
 
@@ -203,7 +210,14 @@ namespace MediaBrowser.Controller.Providers.Movies
                 item.CriticRatingSummary = hit.critics_consensus;
                 item.CriticRating = float.Parse(hit.ratings.critics_score);
 
-                using (var stream = await HttpClient.Get(GetMovieReviewsUrl(hit.id), _rottenTomatoesResourcePool, cancellationToken).ConfigureAwait(false))
+                using (var stream = await HttpClient.Get(new HttpRequestOptions
+                {
+                    Url = GetMovieReviewsUrl(hit.id),
+                    ResourcePool = _rottenTomatoesResourcePool,
+                    CancellationToken = cancellationToken,
+                    EnableResponseCache = true
+
+                }).ConfigureAwait(false))
                 {
 
                     var result = JsonSerializer.DeserializeFromStream<RTReviewList>(stream);

MediaBrowser.Controller/Providers/Movies/TmdbPersonProvider.cs

@@ -168,7 +168,8 @@ namespace MediaBrowser.Controller.Providers.Movies
                     Url = url,
                     CancellationToken = cancellationToken,
                     ResourcePool = MovieDbProvider.Current.MovieDbResourcePool,
-                    AcceptHeader = MovieDbProvider.AcceptHeader
+                    AcceptHeader = MovieDbProvider.AcceptHeader,
+                    EnableResponseCache = true
 
                 }).ConfigureAwait(false))
                 {
@@ -201,7 +202,8 @@ namespace MediaBrowser.Controller.Providers.Movies
                     Url = url,
                     CancellationToken = cancellationToken,
                     ResourcePool = MovieDbProvider.Current.MovieDbResourcePool,
-                    AcceptHeader = MovieDbProvider.AcceptHeader
+                    AcceptHeader = MovieDbProvider.AcceptHeader,
+                    EnableResponseCache = true
 
                 }).ConfigureAwait(false))
                 {
@@ -286,7 +288,8 @@ namespace MediaBrowser.Controller.Providers.Movies
                     Url = url,
                     CancellationToken = cancellationToken,
                     ResourcePool = MovieDbProvider.Current.MovieDbResourcePool,
-                    AcceptHeader = MovieDbProvider.AcceptHeader
+                    AcceptHeader = MovieDbProvider.AcceptHeader,
+                    EnableResponseCache = true
 
                 }).ConfigureAwait(false))
                 {

MediaBrowser.Controller/Providers/Music/FanArtAlbumProvider.cs

@@ -141,7 +141,14 @@ namespace MediaBrowser.Controller.Providers.Music
 
             try
             {
-                using (var xml = await HttpClient.Get(url, FanArtResourcePool, cancellationToken).ConfigureAwait(false))
+                using (var xml = await HttpClient.Get(new HttpRequestOptions
+                {
+                    Url = url,
+                    ResourcePool = FanArtResourcePool,
+                    CancellationToken = cancellationToken,
+                    EnableResponseCache = true
+
+                }).ConfigureAwait(false))
                 {
                     doc.Load(xml);
                 }
@@ -243,7 +250,8 @@ namespace MediaBrowser.Controller.Providers.Music
                     Url = url,
                     CancellationToken = cancellationToken,
                     ResourcePool = _musicBrainzSemaphore,
-                    UserAgent = "MediaBrowserServer/www.mediabrowser3.com"
+                    UserAgent = "MediaBrowserServer/www.mediabrowser3.com",
+                    EnableResponseCache = true
 
                 }).ConfigureAwait(false))
                 {

MediaBrowser.Controller/Providers/Music/FanArtArtistProvider.cs

@@ -117,7 +117,14 @@ namespace MediaBrowser.Controller.Providers.Music
 
             try
             {
-                using (var xml = await HttpClient.Get(url, FanArtResourcePool, cancellationToken).ConfigureAwait(false))
+                using (var xml = await HttpClient.Get(new HttpRequestOptions
+                {
+                    Url = url,
+                    ResourcePool = FanArtResourcePool,
+                    CancellationToken = cancellationToken,
+                    EnableResponseCache = true
+
+                }).ConfigureAwait(false))
                 {
                     doc.Load(xml);
                 }

MediaBrowser.Controller/Providers/Music/LastfmAlbumProvider.cs

@@ -108,7 +108,14 @@ namespace MediaBrowser.Controller.Providers.Music
             // Get albu info using artist and album name
             var url = RootUrl + string.Format("method=album.getInfo&artist={0}&album={1}&api_key={2}&format=json", UrlEncode(artist), UrlEncode(album), ApiKey);
 
-            using (var json = await HttpClient.Get(url, LastfmResourcePool, cancellationToken).ConfigureAwait(false))
+            using (var json = await HttpClient.Get(new HttpRequestOptions
+            {
+                Url = url,
+                ResourcePool = LastfmResourcePool,
+                CancellationToken = cancellationToken,
+                EnableResponseCache = true
+
+            }).ConfigureAwait(false))
             {
                 return JsonSerializer.DeserializeFromStream<LastfmGetAlbumResult>(json);
             }

MediaBrowser.Controller/Providers/Music/LastfmArtistByNameProvider.cs

@@ -2,6 +2,7 @@
 using MediaBrowser.Controller.Configuration;
 using MediaBrowser.Controller.Entities;
 using MediaBrowser.Controller.Entities.Audio;
+using MediaBrowser.Controller.Library;
 using MediaBrowser.Model.Logging;
 using MediaBrowser.Model.Serialization;
 
@@ -20,8 +21,8 @@ namespace MediaBrowser.Controller.Providers.Music
         /// <param name="logManager">The log manager.</param>
         /// <param name="configurationManager">The configuration manager.</param>
         /// <param name="providerManager">The provider manager.</param>
-        public LastfmArtistByNameProvider(IJsonSerializer jsonSerializer, IHttpClient httpClient, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager)
-            : base(jsonSerializer, httpClient, logManager, configurationManager, providerManager)
+        public LastfmArtistByNameProvider(IJsonSerializer jsonSerializer, IHttpClient httpClient, ILogManager logManager, IServerConfigurationManager configurationManager, IProviderManager providerManager, ILibraryManager libraryManager)
+            : base(jsonSerializer, httpClient, logManager, configurationManager, providerManager, libraryManager)
         {
         }
 

MediaBrowser.Controller/Providers/Music/LastfmArtistProvider.cs

@@ -100,7 +100,14 @@ namespace MediaBrowser.Controller.Providers.Music
 
             try
             {
-                using (var json = await HttpClient.Get(url, LastfmResourcePool, cancellationToken).ConfigureAwait(false))
+                using (var json = await HttpClient.Get(new HttpRequestOptions
+                {
+                    Url = url,
+                    ResourcePool = LastfmResourcePool,
+                    CancellationToken = cancellationToken,
+                    EnableResponseCache = true
+
+                }).ConfigureAwait(false))
                 {
                     searchResult = JsonSerializer.DeserializeFromStream<LastfmArtistSearchResults>(json);
                 }
@@ -180,7 +187,14 @@ namespace MediaBrowser.Controller.Providers.Music
 
             LastfmGetArtistResult result;
 
-            using (var json = await HttpClient.Get(url, LastfmResourcePool, cancellationToken).ConfigureAwait(false))
+            using (var json = await HttpClient.Get(new HttpRequestOptions
+            {
+                Url = url,
+                ResourcePool = LastfmResourcePool,
+                CancellationToken = cancellationToken,
+                EnableResponseCache = true
+
+            }).ConfigureAwait(false))
             {
                 result = JsonSerializer.DeserializeFromStream<LastfmGetArtistResult>(json);
             }

MediaBrowser.Controller/Providers/TV/FanArtTVProvider.cs

@@ -76,7 +76,14 @@ namespace MediaBrowser.Controller.Providers.TV
 
             try
             {
-                using (var xml = await HttpClient.Get(url, FanArtResourcePool, cancellationToken).ConfigureAwait(false))
+                using (var xml = await HttpClient.Get(new HttpRequestOptions
+                {
+                    Url = url,
+                    ResourcePool = FanArtResourcePool,
+                    CancellationToken = cancellationToken,
+                    EnableResponseCache = true
+
+                }).ConfigureAwait(false))
                 {
                     doc.Load(xml);
                 }

MediaBrowser.Controller/Providers/TV/RemoteEpisodeProvider.cs

@@ -185,7 +185,14 @@ namespace MediaBrowser.Controller.Providers.TV
 
                 try
                 {
-                    using (var result = await HttpClient.Get(url, RemoteSeriesProvider.Current.TvDbResourcePool, cancellationToken).ConfigureAwait(false))
+                    using (var result = await HttpClient.Get(new HttpRequestOptions
+                    {
+                        Url = url,
+                        ResourcePool = RemoteSeriesProvider.Current.TvDbResourcePool,
+                        CancellationToken = cancellationToken,
+                        EnableResponseCache = true
+
+                    }).ConfigureAwait(false))
                     {
                         doc.Load(result);
                     }
@@ -203,7 +210,14 @@ namespace MediaBrowser.Controller.Providers.TV
 
                     try
                     {
-                        using (var result = await HttpClient.Get(url, RemoteSeriesProvider.Current.TvDbResourcePool, cancellationToken).ConfigureAwait(false))
+                        using (var result = await HttpClient.Get(new HttpRequestOptions
+                        {
+                            Url = url,
+                            ResourcePool = RemoteSeriesProvider.Current.TvDbResourcePool,
+                            CancellationToken = cancellationToken,
+                            EnableResponseCache = true
+
+                        }).ConfigureAwait(false))
                         {
                             if (result != null) doc.Load(result);
                             usingAbsoluteData = true;

MediaBrowser.Controller/Providers/TV/RemoteSeasonProvider.cs

@@ -149,7 +149,14 @@ namespace MediaBrowser.Controller.Providers.TV
 
                     try
                     {
-                        using (var imgs = await HttpClient.Get(url, RemoteSeriesProvider.Current.TvDbResourcePool, cancellationToken).ConfigureAwait(false))
+                        using (var imgs = await HttpClient.Get(new HttpRequestOptions
+                        {
+                            Url = url,
+                            ResourcePool = RemoteSeriesProvider.Current.TvDbResourcePool,
+                            CancellationToken = cancellationToken,
+                            EnableResponseCache = true
+
+                        }).ConfigureAwait(false))
                         {
                             images.Load(imgs);
                         }

MediaBrowser.Controller/Providers/TV/RemoteSeriesProvider.cs

@@ -202,7 +202,14 @@ namespace MediaBrowser.Controller.Providers.TV
 
                 try
                 {
-                    using (var xml = await HttpClient.Get(url, TvDbResourcePool, cancellationToken).ConfigureAwait(false))
+                    using (var xml = await HttpClient.Get(new HttpRequestOptions
+                    {
+                        Url = url,
+                        ResourcePool = TvDbResourcePool,
+                        CancellationToken = cancellationToken,
+                        EnableResponseCache = true
+
+                    }).ConfigureAwait(false))
                     {
                         doc.Load(xml);
                     }
@@ -294,7 +301,14 @@ namespace MediaBrowser.Controller.Providers.TV
 
             try
             {
-                using (var actors = await HttpClient.Get(urlActors, TvDbResourcePool, cancellationToken).ConfigureAwait(false))
+                using (var actors = await HttpClient.Get(new HttpRequestOptions
+                {
+                    Url = urlActors,
+                    ResourcePool = TvDbResourcePool,
+                    CancellationToken = cancellationToken,
+                    EnableResponseCache = true
+
+                }).ConfigureAwait(false))
                 {
                     docActors.Load(actors);
                 }
@@ -366,7 +380,14 @@ namespace MediaBrowser.Controller.Providers.TV
 
                 try
                 {
-                    using (var imgs = await HttpClient.Get(url, TvDbResourcePool, cancellationToken).ConfigureAwait(false))
+                    using (var imgs = await HttpClient.Get(new HttpRequestOptions
+                    {
+                        Url = url,
+                        ResourcePool = TvDbResourcePool,
+                        CancellationToken = cancellationToken,
+                        EnableResponseCache = true
+
+                    }).ConfigureAwait(false))
                     {
                         images.Load(imgs);
                     }
@@ -513,7 +534,14 @@ namespace MediaBrowser.Controller.Providers.TV
 
             try
             {
-                using (var results = await HttpClient.Get(url, TvDbResourcePool, cancellationToken).ConfigureAwait(false))
+                using (var results = await HttpClient.Get(new HttpRequestOptions
+                {
+                    Url = url,
+                    ResourcePool = TvDbResourcePool,
+                    CancellationToken = cancellationToken,
+                    EnableResponseCache = true
+
+                }).ConfigureAwait(false))
                 {
                     doc.Load(results);
                 }