bbc.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722
  1. # coding: utf-8
  2. from __future__ import unicode_literals
  3. import re
  4. import xml.etree.ElementTree
  5. from .common import InfoExtractor
  6. from ..utils import (
  7. ExtractorError,
  8. float_or_none,
  9. int_or_none,
  10. parse_duration,
  11. parse_iso8601,
  12. )
  13. from ..compat import compat_HTTPError
  14. class BBCCoUkIE(InfoExtractor):
  15. IE_NAME = 'bbc.co.uk'
  16. IE_DESC = 'BBC iPlayer'
  17. _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
  18. _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s'
  19. _TESTS = [
  20. {
  21. 'url': 'http://www.bbc.co.uk/programmes/b039g8p7',
  22. 'info_dict': {
  23. 'id': 'b039d07m',
  24. 'ext': 'flv',
  25. 'title': 'Kaleidoscope, Leonard Cohen',
  26. 'description': 'The Canadian poet and songwriter reflects on his musical career.',
  27. 'duration': 1740,
  28. },
  29. 'params': {
  30. # rtmp download
  31. 'skip_download': True,
  32. }
  33. },
  34. {
  35. 'url': 'http://www.bbc.co.uk/iplayer/episode/b00yng5w/The_Man_in_Black_Series_3_The_Printed_Name/',
  36. 'info_dict': {
  37. 'id': 'b00yng1d',
  38. 'ext': 'flv',
  39. 'title': 'The Man in Black: Series 3: The Printed Name',
  40. 'description': "Mark Gatiss introduces Nicholas Pierpan's chilling tale of a writer's devilish pact with a mysterious man. Stars Ewan Bailey.",
  41. 'duration': 1800,
  42. },
  43. 'params': {
  44. # rtmp download
  45. 'skip_download': True,
  46. },
  47. 'skip': 'Episode is no longer available on BBC iPlayer Radio',
  48. },
  49. {
  50. 'url': 'http://www.bbc.co.uk/iplayer/episode/b03vhd1f/The_Voice_UK_Series_3_Blind_Auditions_5/',
  51. 'info_dict': {
  52. 'id': 'b00yng1d',
  53. 'ext': 'flv',
  54. 'title': 'The Voice UK: Series 3: Blind Auditions 5',
  55. 'description': "Emma Willis and Marvin Humes present the fifth set of blind auditions in the singing competition, as the coaches continue to build their teams based on voice alone.",
  56. 'duration': 5100,
  57. },
  58. 'params': {
  59. # rtmp download
  60. 'skip_download': True,
  61. },
  62. 'skip': 'Currently BBC iPlayer TV programmes are available to play in the UK only',
  63. },
  64. {
  65. 'url': 'http://www.bbc.co.uk/iplayer/episode/p026c7jt/tomorrows-worlds-the-unearthly-history-of-science-fiction-2-invasion',
  66. 'info_dict': {
  67. 'id': 'b03k3pb7',
  68. 'ext': 'flv',
  69. 'title': "Tomorrow's Worlds: The Unearthly History of Science Fiction",
  70. 'description': '2. Invasion',
  71. 'duration': 3600,
  72. },
  73. 'params': {
  74. # rtmp download
  75. 'skip_download': True,
  76. },
  77. 'skip': 'Currently BBC iPlayer TV programmes are available to play in the UK only',
  78. }, {
  79. 'url': 'http://www.bbc.co.uk/programmes/b04v20dw',
  80. 'info_dict': {
  81. 'id': 'b04v209v',
  82. 'ext': 'flv',
  83. 'title': 'Pete Tong, The Essential New Tune Special',
  84. 'description': "Pete has a very special mix - all of 2014's Essential New Tunes!",
  85. 'duration': 10800,
  86. },
  87. 'params': {
  88. # rtmp download
  89. 'skip_download': True,
  90. }
  91. }, {
  92. 'url': 'http://www.bbc.co.uk/music/clips/p02frcc3',
  93. 'note': 'Audio',
  94. 'info_dict': {
  95. 'id': 'p02frcch',
  96. 'ext': 'flv',
  97. 'title': 'Pete Tong, Past, Present and Future Special, Madeon - After Hours mix',
  98. 'description': 'French house superstar Madeon takes us out of the club and onto the after party.',
  99. 'duration': 3507,
  100. },
  101. 'params': {
  102. # rtmp download
  103. 'skip_download': True,
  104. }
  105. }, {
  106. 'url': 'http://www.bbc.co.uk/music/clips/p025c0zz',
  107. 'note': 'Video',
  108. 'info_dict': {
  109. 'id': 'p025c103',
  110. 'ext': 'flv',
  111. 'title': 'Reading and Leeds Festival, 2014, Rae Morris - Closer (Live on BBC Three)',
  112. 'description': 'Rae Morris performs Closer for BBC Three at Reading 2014',
  113. 'duration': 226,
  114. },
  115. 'params': {
  116. # rtmp download
  117. 'skip_download': True,
  118. }
  119. }, {
  120. 'url': 'http://www.bbc.co.uk/iplayer/episode/b054fn09/ad/natural-world-20152016-2-super-powered-owls',
  121. 'info_dict': {
  122. 'id': 'p02n76xf',
  123. 'ext': 'flv',
  124. 'title': 'Natural World, 2015-2016: 2. Super Powered Owls',
  125. 'description': 'md5:e4db5c937d0e95a7c6b5e654d429183d',
  126. 'duration': 3540,
  127. },
  128. 'params': {
  129. # rtmp download
  130. 'skip_download': True,
  131. },
  132. 'skip': 'geolocation',
  133. }, {
  134. 'url': 'http://www.bbc.co.uk/iplayer/episode/b05zmgwn/royal-academy-summer-exhibition',
  135. 'info_dict': {
  136. 'id': 'b05zmgw1',
  137. 'ext': 'flv',
  138. 'description': 'Kirsty Wark and Morgan Quaintance visit the Royal Academy as it prepares for its annual artistic extravaganza, meeting people who have come together to make the show unique.',
  139. 'title': 'Royal Academy Summer Exhibition',
  140. 'duration': 3540,
  141. },
  142. 'params': {
  143. # rtmp download
  144. 'skip_download': True,
  145. },
  146. 'skip': 'geolocation',
  147. }, {
  148. 'url': 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4',
  149. 'only_matching': True,
  150. }, {
  151. 'url': 'http://www.bbc.co.uk/music/clips#p02frcc3',
  152. 'only_matching': True,
  153. }, {
  154. 'url': 'http://www.bbc.co.uk/iplayer/cbeebies/episode/b0480276/bing-14-atchoo',
  155. 'only_matching': True,
  156. }
  157. ]
  158. def _extract_asx_playlist(self, connection, programme_id):
  159. asx = self._download_xml(connection.get('href'), programme_id, 'Downloading ASX playlist')
  160. return [ref.get('href') for ref in asx.findall('./Entry/ref')]
  161. def _extract_connection(self, connection, programme_id):
  162. formats = []
  163. protocol = connection.get('protocol')
  164. supplier = connection.get('supplier')
  165. if protocol == 'http':
  166. href = connection.get('href')
  167. transfer_format = connection.get('transferFormat')
  168. # ASX playlist
  169. if supplier == 'asx':
  170. for i, ref in enumerate(self._extract_asx_playlist(connection, programme_id)):
  171. formats.append({
  172. 'url': ref,
  173. 'format_id': 'ref%s_%s' % (i, supplier),
  174. })
  175. # Skip DASH until supported
  176. elif transfer_format == 'dash':
  177. pass
  178. # Direct link
  179. else:
  180. formats.append({
  181. 'url': href,
  182. 'format_id': supplier,
  183. })
  184. elif protocol == 'rtmp':
  185. application = connection.get('application', 'ondemand')
  186. auth_string = connection.get('authString')
  187. identifier = connection.get('identifier')
  188. server = connection.get('server')
  189. formats.append({
  190. 'url': '%s://%s/%s?%s' % (protocol, server, application, auth_string),
  191. 'play_path': identifier,
  192. 'app': '%s?%s' % (application, auth_string),
  193. 'page_url': 'http://www.bbc.co.uk',
  194. 'player_url': 'http://www.bbc.co.uk/emp/releases/iplayer/revisions/617463_618125_4/617463_618125_4_emp.swf',
  195. 'rtmp_live': False,
  196. 'ext': 'flv',
  197. 'format_id': supplier,
  198. })
  199. return formats
  200. def _extract_items(self, playlist):
  201. return playlist.findall('./{http://bbc.co.uk/2008/emp/playlist}item')
  202. def _extract_medias(self, media_selection):
  203. error = media_selection.find('./{http://bbc.co.uk/2008/mp/mediaselection}error')
  204. if error is not None:
  205. raise ExtractorError(
  206. '%s returned error: %s' % (self.IE_NAME, error.get('id')), expected=True)
  207. return media_selection.findall('./{http://bbc.co.uk/2008/mp/mediaselection}media')
  208. def _extract_connections(self, media):
  209. return media.findall('./{http://bbc.co.uk/2008/mp/mediaselection}connection')
  210. def _extract_video(self, media, programme_id):
  211. formats = []
  212. vbr = int_or_none(media.get('bitrate'))
  213. vcodec = media.get('encoding')
  214. service = media.get('service')
  215. width = int_or_none(media.get('width'))
  216. height = int_or_none(media.get('height'))
  217. file_size = int_or_none(media.get('media_file_size'))
  218. for connection in self._extract_connections(media):
  219. conn_formats = self._extract_connection(connection, programme_id)
  220. for format in conn_formats:
  221. format.update({
  222. 'format_id': '%s_%s' % (service, format['format_id']),
  223. 'width': width,
  224. 'height': height,
  225. 'vbr': vbr,
  226. 'vcodec': vcodec,
  227. 'filesize': file_size,
  228. })
  229. formats.extend(conn_formats)
  230. return formats
  231. def _extract_audio(self, media, programme_id):
  232. formats = []
  233. abr = int_or_none(media.get('bitrate'))
  234. acodec = media.get('encoding')
  235. service = media.get('service')
  236. for connection in self._extract_connections(media):
  237. conn_formats = self._extract_connection(connection, programme_id)
  238. for format in conn_formats:
  239. format.update({
  240. 'format_id': '%s_%s' % (service, format['format_id']),
  241. 'abr': abr,
  242. 'acodec': acodec,
  243. })
  244. formats.extend(conn_formats)
  245. return formats
  246. def _get_subtitles(self, media, programme_id):
  247. subtitles = {}
  248. for connection in self._extract_connections(media):
  249. captions = self._download_xml(connection.get('href'), programme_id, 'Downloading captions')
  250. lang = captions.get('{http://www.w3.org/XML/1998/namespace}lang', 'en')
  251. subtitles[lang] = [
  252. {
  253. 'url': connection.get('href'),
  254. 'ext': 'ttml',
  255. },
  256. ]
  257. return subtitles
  258. def _download_media_selector(self, programme_id):
  259. return self._download_media_selector_url(
  260. self._MEDIASELECTOR_URL % programme_id, programme_id)
  261. def _download_media_selector_url(self, url, programme_id=None):
  262. try:
  263. media_selection = self._download_xml(
  264. url, programme_id, 'Downloading media selection XML')
  265. except ExtractorError as ee:
  266. if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
  267. media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().decode('utf-8'))
  268. else:
  269. raise
  270. return self._process_media_selector(media_selection, programme_id)
  271. def _process_media_selector(self, media_selection, programme_id):
  272. formats = []
  273. subtitles = None
  274. for media in self._extract_medias(media_selection):
  275. kind = media.get('kind')
  276. if kind == 'audio':
  277. formats.extend(self._extract_audio(media, programme_id))
  278. elif kind == 'video':
  279. formats.extend(self._extract_video(media, programme_id))
  280. elif kind == 'captions':
  281. subtitles = self.extract_subtitles(media, programme_id)
  282. return formats, subtitles
  283. def _download_playlist(self, playlist_id):
  284. try:
  285. playlist = self._download_json(
  286. 'http://www.bbc.co.uk/programmes/%s/playlist.json' % playlist_id,
  287. playlist_id, 'Downloading playlist JSON')
  288. version = playlist.get('defaultAvailableVersion')
  289. if version:
  290. smp_config = version['smpConfig']
  291. title = smp_config['title']
  292. description = smp_config['summary']
  293. for item in smp_config['items']:
  294. kind = item['kind']
  295. if kind != 'programme' and kind != 'radioProgramme':
  296. continue
  297. programme_id = item.get('vpid')
  298. duration = int_or_none(item.get('duration'))
  299. formats, subtitles = self._download_media_selector(programme_id)
  300. return programme_id, title, description, duration, formats, subtitles
  301. except ExtractorError as ee:
  302. if not (isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404):
  303. raise
  304. # fallback to legacy playlist
  305. return self._process_legacy_playlist(playlist_id)
  306. def _process_legacy_playlist_url(self, url, display_id):
  307. playlist = self._download_legacy_playlist_url(url, display_id)
  308. return self._extract_from_legacy_playlist(playlist, display_id)
  309. def _process_legacy_playlist(self, playlist_id):
  310. return self._process_legacy_playlist_url(
  311. 'http://www.bbc.co.uk/iplayer/playlist/%s' % playlist_id, playlist_id)
  312. def _download_legacy_playlist_url(self, url, playlist_id=None):
  313. return self._download_xml(
  314. url, playlist_id, 'Downloading legacy playlist XML')
  315. def _extract_from_legacy_playlist(self, playlist, playlist_id):
  316. no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
  317. if no_items is not None:
  318. reason = no_items.get('reason')
  319. if reason == 'preAvailability':
  320. msg = 'Episode %s is not yet available' % playlist_id
  321. elif reason == 'postAvailability':
  322. msg = 'Episode %s is no longer available' % playlist_id
  323. elif reason == 'noMedia':
  324. msg = 'Episode %s is not currently available' % playlist_id
  325. else:
  326. msg = 'Episode %s is not available: %s' % (playlist_id, reason)
  327. raise ExtractorError(msg, expected=True)
  328. for item in self._extract_items(playlist):
  329. kind = item.get('kind')
  330. if kind != 'programme' and kind != 'radioProgramme':
  331. continue
  332. title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
  333. description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
  334. def get_programme_id(item):
  335. def get_from_attributes(item):
  336. for p in('identifier', 'group'):
  337. value = item.get(p)
  338. if value and re.match(r'^[pb][\da-z]{7}$', value):
  339. return value
  340. get_from_attributes(item)
  341. mediator = item.find('./{http://bbc.co.uk/2008/emp/playlist}mediator')
  342. if mediator is not None:
  343. return get_from_attributes(mediator)
  344. programme_id = get_programme_id(item)
  345. duration = int_or_none(item.get('duration'))
  346. # TODO: programme_id can be None and media items can be incorporated right inside
  347. # playlist's item (e.g. http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
  348. # as f4m and m3u8
  349. formats, subtitles = self._download_media_selector(programme_id)
  350. return programme_id, title, description, duration, formats, subtitles
  351. def _real_extract(self, url):
  352. group_id = self._match_id(url)
  353. webpage = self._download_webpage(url, group_id, 'Downloading video page')
  354. programme_id = None
  355. tviplayer = self._search_regex(
  356. r'mediator\.bind\(({.+?})\s*,\s*document\.getElementById',
  357. webpage, 'player', default=None)
  358. if tviplayer:
  359. player = self._parse_json(tviplayer, group_id).get('player', {})
  360. duration = int_or_none(player.get('duration'))
  361. programme_id = player.get('vpid')
  362. if not programme_id:
  363. programme_id = self._search_regex(
  364. r'"vpid"\s*:\s*"([\da-z]{8})"', webpage, 'vpid', fatal=False, default=None)
  365. if programme_id:
  366. formats, subtitles = self._download_media_selector(programme_id)
  367. title = self._og_search_title(webpage)
  368. description = self._search_regex(
  369. r'<p class="[^"]*medium-description[^"]*">([^<]+)</p>',
  370. webpage, 'description', fatal=False)
  371. else:
  372. programme_id, title, description, duration, formats, subtitles = self._download_playlist(group_id)
  373. self._sort_formats(formats)
  374. return {
  375. 'id': programme_id,
  376. 'title': title,
  377. 'description': description,
  378. 'thumbnail': self._og_search_thumbnail(webpage, default=None),
  379. 'duration': duration,
  380. 'formats': formats,
  381. 'subtitles': subtitles,
  382. }
  383. class BBCIE(BBCCoUkIE):
  384. IE_NAME = 'bbc'
  385. IE_DESC = 'BBC'
  386. _VALID_URL = r'https?://(?:www\.)?bbc\.(?:com|co\.uk)/(?:[^/]+/)+(?P<id>[^/#?]+)'
  387. # fails with notukerror for some videos
  388. # _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/4/mtis/stream/%s'
  389. _MEDIASELECTOR_URL = 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/journalism-pc/vpid/%s'
  390. _TESTS = [{
  391. # article with multiple videos embedded with data-media-meta containing
  392. # playlist.sxml, externalId and no direct video links
  393. 'url': 'http://www.bbc.com/news/world-europe-32668511',
  394. 'info_dict': {
  395. 'id': 'world-europe-32668511',
  396. 'title': 'Russia stages massive WW2 parade despite Western boycott',
  397. 'description': 'md5:00ff61976f6081841f759a08bf78cc9c',
  398. },
  399. 'playlist_count': 2,
  400. }, {
  401. # article with multiple videos embedded with data-media-meta (more videos)
  402. 'url': 'http://www.bbc.com/news/business-28299555',
  403. 'info_dict': {
  404. 'id': 'business-28299555',
  405. 'title': 'Farnborough Airshow: Video highlights',
  406. 'description': 'BBC reports and video highlights at the Farnborough Airshow.',
  407. },
  408. 'playlist_count': 9,
  409. 'skip': 'Save time',
  410. }, {
  411. # article with multiple videos embedded with `new SMP()`
  412. 'url': 'http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460',
  413. 'info_dict': {
  414. 'id': '3662a707-0af9-3149-963f-47bea720b460',
  415. 'title': 'BBC Blogs - Adam Curtis - BUGGER',
  416. },
  417. 'playlist_count': 18,
  418. }, {
  419. # single video embedded with mediaAssetPage.init()
  420. 'url': 'http://www.bbc.com/news/world-europe-32041533',
  421. 'info_dict': {
  422. 'id': 'p02mprgb',
  423. 'ext': 'flv',
  424. 'title': 'Aerial footage showed the site of the crash in the Alps - courtesy BFM TV',
  425. 'duration': 47,
  426. 'timestamp': 1427219242,
  427. 'upload_date': '20150324',
  428. },
  429. 'params': {
  430. # rtmp download
  431. 'skip_download': True,
  432. }
  433. }, {
  434. # article with single video embedded with data-media-meta containing
  435. # direct video links (for now these are extracted) and playlist.xml (with
  436. # media items as f4m and m3u8 - currently unsupported)
  437. 'url': 'http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu',
  438. 'info_dict': {
  439. 'id': '150615_telabyad_kentin_cogu',
  440. 'ext': 'mp4',
  441. 'title': "YPG: Tel Abyad'ın tamamı kontrolümüzde",
  442. 'duration': 47,
  443. 'timestamp': 1434397334,
  444. 'upload_date': '20150615',
  445. },
  446. 'params': {
  447. 'skip_download': True,
  448. }
  449. }, {
  450. # single video embedded with mediaAssetPage.init() (regional section)
  451. 'url': 'http://www.bbc.com/mundo/video_fotos/2015/06/150619_video_honduras_militares_hospitales_corrupcion_aw',
  452. 'info_dict': {
  453. 'id': '150619_video_honduras_militares_hospitales_corrupcion_aw',
  454. 'ext': 'mp4',
  455. 'title': 'Honduras militariza sus hospitales por nuevo escándalo de corrupción',
  456. 'duration': 87,
  457. 'timestamp': 1434713142,
  458. 'upload_date': '20150619',
  459. },
  460. 'params': {
  461. 'skip_download': True,
  462. }
  463. }, {
  464. # single video story with digitalData
  465. 'url': 'http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret',
  466. 'info_dict': {
  467. 'id': 'p02q6gc4',
  468. 'ext': 'flv',
  469. 'title': 'Sri Lanka’s spicy secret',
  470. 'description': 'As a new train line to Jaffna opens up the country’s north, travellers can experience a truly distinct slice of Tamil culture.',
  471. 'timestamp': 1437674293,
  472. 'upload_date': '20150723',
  473. },
  474. 'params': {
  475. # rtmp download
  476. 'skip_download': True,
  477. }
  478. }, {
  479. # single video story without digitalData
  480. 'url': 'http://www.bbc.com/autos/story/20130513-hyundais-rock-star',
  481. 'info_dict': {
  482. 'id': 'p018zqqg',
  483. 'ext': 'flv',
  484. 'title': 'Hyundai Santa Fe Sport: Rock star',
  485. 'description': 'md5:b042a26142c4154a6e472933cf20793d',
  486. 'timestamp': 1368473503,
  487. 'upload_date': '20130513',
  488. },
  489. 'params': {
  490. # rtmp download
  491. 'skip_download': True,
  492. }
  493. }, {
  494. # single video with playlist.sxml URL
  495. 'url': 'http://www.bbc.com/sport/0/football/33653409',
  496. 'info_dict': {
  497. 'id': 'p02xycnp',
  498. 'ext': 'flv',
  499. 'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
  500. 'description': 'md5:398fca0e2e701c609d726e034fa1fc89',
  501. 'duration': 140,
  502. },
  503. 'params': {
  504. # rtmp download
  505. 'skip_download': True,
  506. }
  507. }, {
  508. # single video with playlist URL from weather section
  509. 'url': 'http://www.bbc.com/weather/features/33601775',
  510. 'only_matching': True,
  511. }, {
  512. # custom redirection to www.bbc.com
  513. 'url': 'http://www.bbc.co.uk/news/science-environment-33661876',
  514. 'only_matching': True,
  515. }]
  516. @classmethod
  517. def suitable(cls, url):
  518. return False if BBCCoUkIE.suitable(url) else super(BBCIE, cls).suitable(url)
  519. def _extract_from_media_meta(self, media_meta, video_id):
  520. # Direct links to media in media metadata (e.g.
  521. # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
  522. # TODO: there are also f4m and m3u8 streams incorporated in playlist.sxml
  523. source_files = media_meta.get('sourceFiles')
  524. if source_files:
  525. return [{
  526. 'url': f['url'],
  527. 'format_id': format_id,
  528. 'ext': f.get('encoding'),
  529. 'tbr': float_or_none(f.get('bitrate'), 1000),
  530. 'filesize': int_or_none(f.get('filesize')),
  531. } for format_id, f in source_files.items() if f.get('url')], []
  532. programme_id = media_meta.get('externalId')
  533. if programme_id:
  534. return self._download_media_selector(programme_id)
  535. # Process playlist.sxml as legacy playlist
  536. href = media_meta.get('href')
  537. if href:
  538. playlist = self._download_legacy_playlist_url(href)
  539. _, _, _, _, formats, subtitles = self._extract_from_legacy_playlist(playlist, video_id)
  540. return formats, subtitles
  541. return [], []
  542. def _real_extract(self, url):
  543. playlist_id = self._match_id(url)
  544. webpage = self._download_webpage(url, playlist_id)
  545. timestamp = parse_iso8601(self._search_regex(
  546. [r'"datePublished":\s*"([^"]+)',
  547. r'<meta[^>]+property="article:published_time"[^>]+content="([^"]+)"',
  548. r'itemprop="datePublished"[^>]+datetime="([^"]+)"'],
  549. webpage, 'date', default=None))
  550. # single video with playlist.sxml URL (e.g. http://www.bbc.com/sport/0/football/3365340ng)
  551. playlist = self._search_regex(
  552. r'<param[^>]+name="playlist"[^>]+value="([^"]+)"',
  553. webpage, 'playlist', default=None)
  554. if playlist:
  555. programme_id, title, description, duration, formats, subtitles = \
  556. self._process_legacy_playlist_url(playlist, playlist_id)
  557. self._sort_formats(formats)
  558. return {
  559. 'id': programme_id,
  560. 'title': title,
  561. 'description': description,
  562. 'duration': duration,
  563. 'timestamp': timestamp,
  564. 'formats': formats,
  565. 'subtitles': subtitles,
  566. }
  567. # single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
  568. programme_id = self._search_regex(
  569. [r'data-video-player-vpid="([\da-z]{8})"',
  570. r'<param[^>]+name="externalIdentifier"[^>]+value="([\da-z]{8})"'],
  571. webpage, 'vpid', default=None)
  572. if programme_id:
  573. formats, subtitles = self._download_media_selector(programme_id)
  574. self._sort_formats(formats)
  575. # digitalData may be missing (e.g. http://www.bbc.com/autos/story/20130513-hyundais-rock-star)
  576. digital_data = self._parse_json(
  577. self._search_regex(
  578. r'var\s+digitalData\s*=\s*({.+?});?\n', webpage, 'digital data', default='{}'),
  579. programme_id, fatal=False)
  580. page_info = digital_data.get('page', {}).get('pageInfo', {})
  581. title = page_info.get('pageName') or self._og_search_title(webpage)
  582. description = page_info.get('description') or self._og_search_description(webpage)
  583. timestamp = parse_iso8601(page_info.get('publicationDate')) or timestamp
  584. return {
  585. 'id': programme_id,
  586. 'title': title,
  587. 'description': description,
  588. 'timestamp': timestamp,
  589. 'formats': formats,
  590. 'subtitles': subtitles,
  591. }
  592. playlist_title = self._html_search_regex(
  593. r'<title>(.*?)(?:\s*-\s*BBC [^ ]+)?</title>', webpage, 'playlist title')
  594. playlist_description = self._og_search_description(webpage, default=None)
  595. def extract_all(pattern):
  596. return list(filter(None, map(
  597. lambda s: self._parse_json(s, playlist_id, fatal=False),
  598. re.findall(pattern, webpage))))
  599. # Multiple video article (e.g.
  600. # http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460)
  601. EMBED_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:[^/]+/)+[\da-z]{8}(?:\b[^"]+)?'
  602. entries = []
  603. for match in extract_all(r'new\s+SMP\(({.+?})\)'):
  604. embed_url = match.get('playerSettings', {}).get('externalEmbedUrl')
  605. if embed_url and re.match(EMBED_URL, embed_url):
  606. entries.append(embed_url)
  607. entries.extend(re.findall(
  608. r'setPlaylist\("(%s)"\)' % EMBED_URL, webpage))
  609. if entries:
  610. return self.playlist_result(
  611. [self.url_result(entry, 'BBCCoUk') for entry in entries],
  612. playlist_id, playlist_title, playlist_description)
  613. # Multiple video article (e.g. http://www.bbc.com/news/world-europe-32668511)
  614. medias = extract_all(r"data-media-meta='({[^']+})'")
  615. if not medias:
  616. # Single video article (e.g. http://www.bbc.com/news/video_and_audio/international)
  617. media_asset_page = self._parse_json(
  618. self._search_regex(
  619. r'mediaAssetPage\.init\(\s*({.+?}), "/', webpage, 'media asset'),
  620. playlist_id)
  621. medias = []
  622. for video in media_asset_page.get('videos', {}).values():
  623. medias.extend(video.values())
  624. entries = []
  625. for num, media_meta in enumerate(medias, start=1):
  626. formats, subtitles = self._extract_from_media_meta(media_meta, playlist_id)
  627. if not formats:
  628. continue
  629. self._sort_formats(formats)
  630. video_id = media_meta.get('externalId')
  631. if not video_id:
  632. video_id = playlist_id if len(medias) == 1 else '%s-%s' % (playlist_id, num)
  633. title = media_meta.get('caption')
  634. if not title:
  635. title = playlist_title if len(medias) == 1 else '%s - Video %s' % (playlist_title, num)
  636. duration = int_or_none(media_meta.get('durationInSeconds')) or parse_duration(media_meta.get('duration'))
  637. images = []
  638. for image in media_meta.get('images', {}).values():
  639. images.extend(image.values())
  640. if 'image' in media_meta:
  641. images.append(media_meta['image'])
  642. thumbnails = [{
  643. 'url': image.get('href'),
  644. 'width': int_or_none(image.get('width')),
  645. 'height': int_or_none(image.get('height')),
  646. } for image in images]
  647. entries.append({
  648. 'id': video_id,
  649. 'title': title,
  650. 'thumbnails': thumbnails,
  651. 'duration': duration,
  652. 'timestamp': timestamp,
  653. 'formats': formats,
  654. 'subtitles': subtitles,
  655. })
  656. return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)