[viewster->contv] remove viewster extractor and add support for contv.com

youtube_dl/extractor/contv.py

@@ -0,0 +1,118 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    int_or_none,
+)
+
+
+class CONtvIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?contv\.com/details-movie/(?P<id>[^/]+)'
+    _TESTS = [{
+        'url': 'https://www.contv.com/details-movie/CEG10022949/days-of-thrills-&-laughter',
+        'info_dict': {
+            'id': 'CEG10022949',
+            'ext': 'mp4',
+            'title': 'Days Of Thrills & Laughter',
+            'description': 'md5:5d6b3d0b1829bb93eb72898c734802eb',
+            'upload_date': '20180703',
+            'timestamp': 1530634789.61,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.contv.com/details-movie/CLIP-show_fotld_bts/fight-of-the-living-dead:-behind-the-scenes-bites',
+        'info_dict': {
+            'id': 'CLIP-show_fotld_bts',
+            'title': 'Fight of the Living Dead: Behind the Scenes Bites',
+        },
+        'playlist_mincount': 7,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        details = self._download_json(
+            'http://metax.contv.live.junctiontv.net/metax/2.5/details/' + video_id,
+            video_id, query={'device': 'web'})
+
+        if details.get('type') == 'episodic':
+            seasons = self._download_json(
+                'http://metax.contv.live.junctiontv.net/metax/2.5/seriesfeed/json/' + video_id,
+                video_id)
+            entries = []
+            for season in seasons:
+                for episode in season.get('episodes', []):
+                    episode_id = episode.get('id')
+                    if not episode_id:
+                        continue
+                    entries.append(self.url_result(
+                        'https://www.contv.com/details-movie/' + episode_id,
+                        CONtvIE.ie_key(), episode_id))
+            return self.playlist_result(entries, video_id, details.get('title'))
+
+        m_details = details['details']
+        title = details['title']
+
+        formats = []
+
+        media_hls_url = m_details.get('media_hls_url')
+        if media_hls_url:
+            formats.extend(self._extract_m3u8_formats(
+                media_hls_url, video_id, 'mp4',
+                m3u8_id='hls', fatal=False))
+
+        media_mp4_url = m_details.get('media_mp4_url')
+        if media_mp4_url:
+            formats.append({
+                'format_id': 'http',
+                'url': media_mp4_url,
+            })
+
+        self._sort_formats(formats)
+
+        subtitles = {}
+        captions = m_details.get('captions') or {}
+        for caption_url in captions.values():
+            subtitles.setdefault('en', []).append({
+                'url': caption_url
+            })
+
+        thumbnails = []
+        for image in m_details.get('images', []):
+            image_url = image.get('url')
+            if not image_url:
+                continue
+            thumbnails.append({
+                'url': image_url,
+                'width': int_or_none(image.get('width')),
+                'height': int_or_none(image.get('height')),
+            })
+
+        description = None
+        for p in ('large_', 'medium_', 'small_', ''):
+            d = m_details.get(p + 'description')
+            if d:
+                description = d
+                break
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'description': description,
+            'timestamp': float_or_none(details.get('metax_added_on'), 1000),
+            'subtitles': subtitles,
+            'duration': float_or_none(m_details.get('duration'), 1000),
+            'view_count': int_or_none(details.get('num_watched')),
+            'like_count': int_or_none(details.get('num_fav')),
+            'categories': details.get('category'),
+            'tags': details.get('tags'),
+            'season_number': int_or_none(details.get('season')),
+            'episode_number': int_or_none(details.get('episode')),
+            'release_year': int_or_none(details.get('pub_year')),
+        }

youtube_dl/extractor/extractors.py

@@ -231,6 +231,7 @@ from .commonprotocols import (
     RtmpIE,
 )
 from .condenast import CondeNastIE
+from .contv import CONtvIE
 from .corus import CorusIE
 from .cracked import CrackedIE
 from .crackle import CrackleIE
@@ -1322,7 +1323,6 @@ from .viewlift import (
     ViewLiftIE,
     ViewLiftEmbedIE,
 )
-from .viewster import ViewsterIE
 from .viidea import ViideaIE
 from .vimeo import (
     VimeoIE,

youtube_dl/extractor/viewster.py

@@ -1,217 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..compat import (
-    compat_HTTPError,
-    compat_urllib_parse_unquote,
-)
-from ..utils import (
-    determine_ext,
-    ExtractorError,
-    int_or_none,
-    parse_iso8601,
-    sanitized_Request,
-    HEADRequest,
-    url_basename,
-)
-
-
-class ViewsterIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?viewster\.com/(?:serie|movie)/(?P<id>\d+-\d+-\d+)'
-    _TESTS = [{
-        # movie, Type=Movie
-        'url': 'http://www.viewster.com/movie/1140-11855-000/the-listening-project/',
-        'md5': 'e642d1b27fcf3a4ffa79f194f5adde36',
-        'info_dict': {
-            'id': '1140-11855-000',
-            'ext': 'mp4',
-            'title': 'The listening Project',
-            'description': 'md5:bac720244afd1a8ea279864e67baa071',
-            'timestamp': 1214870400,
-            'upload_date': '20080701',
-            'duration': 4680,
-        },
-    }, {
-        # series episode, Type=Episode
-        'url': 'http://www.viewster.com/serie/1284-19427-001/the-world-and-a-wall/',
-        'md5': '9243079a8531809efe1b089db102c069',
-        'info_dict': {
-            'id': '1284-19427-001',
-            'ext': 'mp4',
-            'title': 'The World and a Wall',
-            'description': 'md5:24814cf74d3453fdf5bfef9716d073e3',
-            'timestamp': 1428192000,
-            'upload_date': '20150405',
-            'duration': 1500,
-        },
-    }, {
-        # serie, Type=Serie
-        'url': 'http://www.viewster.com/serie/1303-19426-000/',
-        'info_dict': {
-            'id': '1303-19426-000',
-            'title': 'Is It Wrong to Try to Pick up Girls in a Dungeon?',
-            'description': 'md5:eeda9bef25b0d524b3a29a97804c2f11',
-        },
-        'playlist_count': 13,
-    }, {
-        # unfinished serie, no Type
-        'url': 'http://www.viewster.com/serie/1284-19427-000/baby-steps-season-2/',
-        'info_dict': {
-            'id': '1284-19427-000',
-            'title': 'Baby Steps—Season 2',
-            'description': 'md5:e7097a8fc97151e25f085c9eb7a1cdb1',
-        },
-        'playlist_mincount': 16,
-    }, {
-        # geo restricted series
-        'url': 'https://www.viewster.com/serie/1280-18794-002/',
-        'only_matching': True,
-    }, {
-        # geo restricted video
-        'url': 'https://www.viewster.com/serie/1280-18794-002/what-is-extraterritoriality-lawo/',
-        'only_matching': True,
-    }]
-
-    _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'
-
-    def _download_json(self, url, video_id, note='Downloading JSON metadata', fatal=True, query={}):
-        request = sanitized_Request(url)
-        request.add_header('Accept', self._ACCEPT_HEADER)
-        request.add_header('Auth-token', self._AUTH_TOKEN)
-        return super(ViewsterIE, self)._download_json(request, video_id, note, fatal=fatal, query=query)
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        # Get 'api_token' cookie
-        self._request_webpage(
-            HEADRequest('http://www.viewster.com/'),
-            video_id, headers=self.geo_verification_headers())
-        cookies = self._get_cookies('http://www.viewster.com/')
-        self._AUTH_TOKEN = compat_urllib_parse_unquote(cookies['api_token'].value)
-
-        info = self._download_json(
-            'https://public-api.viewster.com/search/%s' % video_id,
-            video_id, 'Downloading entry JSON')
-
-        entry_id = info.get('Id') or info['id']
-
-        # unfinished serie has no Type
-        if info.get('Type') in ('Serie', None):
-            try:
-                episodes = self._download_json(
-                    'https://public-api.viewster.com/series/%s/episodes' % entry_id,
-                    video_id, 'Downloading series JSON')
-            except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
-                    self.raise_geo_restricted()
-                else:
-                    raise
-            entries = [
-                self.url_result(
-                    'http://www.viewster.com/movie/%s' % episode['OriginId'], 'Viewster')
-                for episode in episodes]
-            title = (info.get('Title') or info['Synopsis']['Title']).strip()
-            description = info.get('Synopsis', {}).get('Detailed')
-            return self.playlist_result(entries, video_id, title, description)
-
-        formats = []
-        for language_set in info.get('LanguageSets', []):
-            manifest_url = None
-            m3u8_formats = []
-            audio = language_set.get('Audio') or ''
-            subtitle = language_set.get('Subtitle') or ''
-            base_format_id = audio
-            if subtitle:
-                base_format_id += '-%s' % subtitle
-
-            def concat(suffix, sep='-'):
-                return (base_format_id + '%s%s' % (sep, suffix)) if base_format_id else suffix
-
-            medias = self._download_json(
-                'https://public-api.viewster.com/movies/%s/videos' % entry_id,
-                video_id, fatal=False, query={
-                    'mediaTypes': ['application/f4m+xml', 'application/x-mpegURL', 'video/mp4'],
-                    'language': audio,
-                    'subtitle': subtitle,
-                })
-            if not medias:
-                continue
-            for media in medias:
-                video_url = media.get('Uri')
-                if not video_url:
-                    continue
-                ext = determine_ext(video_url)
-                if ext == 'f4m':
-                    manifest_url = video_url
-                    video_url += '&' if '?' in video_url else '?'
-                    video_url += 'hdcore=3.2.0&plugin=flowplayer-3.2.0.1'
-                    formats.extend(self._extract_f4m_formats(
-                        video_url, video_id, f4m_id=concat('hds')))
-                elif ext == 'm3u8':
-                    manifest_url = video_url
-                    m3u8_formats = self._extract_m3u8_formats(
-                        video_url, video_id, 'mp4', m3u8_id=concat('hls'),
-                        fatal=False)  # m3u8 sometimes fail
-                    if m3u8_formats:
-                        formats.extend(m3u8_formats)
-                else:
-                    qualities_basename = self._search_regex(
-                        r'/([^/]+)\.csmil/',
-                        manifest_url, 'qualities basename', default=None)
-                    if not qualities_basename:
-                        continue
-                    QUALITIES_RE = r'((,\d+k)+,?)'
-                    qualities = self._search_regex(
-                        QUALITIES_RE, qualities_basename,
-                        'qualities', default=None)
-                    if not qualities:
-                        continue
-                    qualities = list(map(lambda q: int(q[:-1]), qualities.strip(',').split(',')))
-                    qualities.sort()
-                    http_template = re.sub(QUALITIES_RE, r'%dk', qualities_basename)
-                    http_url_basename = url_basename(video_url)
-                    if m3u8_formats:
-                        self._sort_formats(m3u8_formats)
-                        m3u8_formats = list(filter(
-                            lambda f: f.get('vcodec') != 'none', m3u8_formats))
-                    if len(qualities) == len(m3u8_formats):
-                        for q, m3u8_format in zip(qualities, m3u8_formats):
-                            f = m3u8_format.copy()
-                            f.update({
-                                'url': video_url.replace(http_url_basename, http_template % q),
-                                'format_id': f['format_id'].replace('hls', 'http'),
-                                'protocol': 'http',
-                            })
-                            formats.append(f)
-                    else:
-                        for q in qualities:
-                            formats.append({
-                                'url': video_url.replace(http_url_basename, http_template % q),
-                                'ext': 'mp4',
-                                'format_id': 'http-%d' % q,
-                                'tbr': q,
-                            })
-
-        if not formats and not info.get('VODSettings'):
-            self.raise_geo_restricted()
-
-        self._sort_formats(formats)
-
-        synopsis = info.get('Synopsis') or {}
-        # Prefer title outside synopsis since it's less messy
-        title = (info.get('Title') or synopsis['Title']).strip()
-        description = synopsis.get('Detailed') or (info.get('Synopsis') or {}).get('Short')
-        duration = int_or_none(info.get('Duration'))
-        timestamp = parse_iso8601(info.get('ReleaseDate'))
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'timestamp': timestamp,
-            'duration': duration,
-            'formats': formats,
-        }