facebook.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649
  1. # coding: utf-8
  2. from __future__ import unicode_literals
  3. import re
  4. import socket
  5. from .common import InfoExtractor
  6. from ..compat import (
  7. compat_etree_fromstring,
  8. compat_http_client,
  9. compat_urllib_error,
  10. compat_urllib_parse_unquote,
  11. compat_urllib_parse_unquote_plus,
  12. )
  13. from ..utils import (
  14. clean_html,
  15. error_to_compat_str,
  16. ExtractorError,
  17. float_or_none,
  18. get_element_by_id,
  19. int_or_none,
  20. js_to_json,
  21. limit_length,
  22. parse_count,
  23. qualities,
  24. sanitized_Request,
  25. try_get,
  26. urlencode_postdata,
  27. urljoin,
  28. )
  29. class FacebookIE(InfoExtractor):
  30. _VALID_URL = r'''(?x)
  31. (?:
  32. https?://
  33. (?:[\w-]+\.)?(?:facebook\.com|facebookcorewwwi\.onion)/
  34. (?:[^#]*?\#!/)?
  35. (?:
  36. (?:
  37. video/video\.php|
  38. photo\.php|
  39. video\.php|
  40. video/embed|
  41. story\.php|
  42. watch(?:/live)?/?
  43. )\?(?:.*?)(?:v|video_id|story_fbid)=|
  44. [^/]+/videos/(?:[^/]+/)?|
  45. [^/]+/posts/|
  46. groups/[^/]+/permalink/
  47. )|
  48. facebook:
  49. )
  50. (?P<id>[0-9]+)
  51. '''
  52. _LOGIN_URL = 'https://www.facebook.com/login.php?next=http%3A%2F%2Ffacebook.com%2Fhome.php&login_attempt=1'
  53. _CHECKPOINT_URL = 'https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1'
  54. _NETRC_MACHINE = 'facebook'
  55. IE_NAME = 'facebook'
  56. _VIDEO_PAGE_TEMPLATE = 'https://www.facebook.com/video/video.php?v=%s'
  57. _VIDEO_PAGE_TAHOE_TEMPLATE = 'https://www.facebook.com/video/tahoe/async/%s/?chain=true&isvideo=true&payloadtype=primary'
  58. _TESTS = [{
  59. 'url': 'https://www.facebook.com/video.php?v=637842556329505&fref=nf',
  60. 'md5': '6a40d33c0eccbb1af76cf0485a052659',
  61. 'info_dict': {
  62. 'id': '637842556329505',
  63. 'ext': 'mp4',
  64. 'title': 're:Did you know Kei Nishikori is the first Asian man to ever reach a Grand Slam',
  65. 'uploader': 'Tennis on Facebook',
  66. 'upload_date': '20140908',
  67. 'timestamp': 1410199200,
  68. },
  69. 'skip': 'Requires logging in',
  70. }, {
  71. # data.video
  72. 'url': 'https://www.facebook.com/video.php?v=274175099429670',
  73. 'info_dict': {
  74. 'id': '274175099429670',
  75. 'ext': 'mp4',
  76. 'title': 're:^Asif Nawab Butt posted a video',
  77. 'uploader': 'Asif Nawab Butt',
  78. 'upload_date': '20140506',
  79. 'timestamp': 1399398998,
  80. 'thumbnail': r're:^https?://.*',
  81. },
  82. 'expected_warnings': [
  83. 'title'
  84. ]
  85. }, {
  86. 'note': 'Video with DASH manifest',
  87. 'url': 'https://www.facebook.com/video.php?v=957955867617029',
  88. 'md5': 'b2c28d528273b323abe5c6ab59f0f030',
  89. 'info_dict': {
  90. 'id': '957955867617029',
  91. 'ext': 'mp4',
  92. 'title': 'When you post epic content on instagram.com/433 8 million followers, this is ...',
  93. 'uploader': 'Demy de Zeeuw',
  94. 'upload_date': '20160110',
  95. 'timestamp': 1452431627,
  96. },
  97. 'skip': 'Requires logging in',
  98. }, {
  99. 'url': 'https://www.facebook.com/maxlayn/posts/10153807558977570',
  100. 'md5': '037b1fa7f3c2d02b7a0d7bc16031ecc6',
  101. 'info_dict': {
  102. 'id': '544765982287235',
  103. 'ext': 'mp4',
  104. 'title': '"What are you doing running in the snow?"',
  105. 'uploader': 'FailArmy',
  106. },
  107. 'skip': 'Video gone',
  108. }, {
  109. 'url': 'https://m.facebook.com/story.php?story_fbid=1035862816472149&id=116132035111903',
  110. 'md5': '1deb90b6ac27f7efcf6d747c8a27f5e3',
  111. 'info_dict': {
  112. 'id': '1035862816472149',
  113. 'ext': 'mp4',
  114. 'title': 'What the Flock Is Going On In New Zealand Credit: ViralHog',
  115. 'uploader': 'S. Saint',
  116. },
  117. 'skip': 'Video gone',
  118. }, {
  119. 'note': 'swf params escaped',
  120. 'url': 'https://www.facebook.com/barackobama/posts/10153664894881749',
  121. 'md5': '97ba073838964d12c70566e0085c2b91',
  122. 'info_dict': {
  123. 'id': '10153664894881749',
  124. 'ext': 'mp4',
  125. 'title': 'Average time to confirm recent Supreme Court nominees: 67 days Longest it\'s t...',
  126. 'thumbnail': r're:^https?://.*',
  127. 'timestamp': 1456259628,
  128. 'upload_date': '20160223',
  129. 'uploader': 'Barack Obama',
  130. },
  131. }, {
  132. # have 1080P, but only up to 720p in swf params
  133. # data.video.story.attachments[].media
  134. 'url': 'https://www.facebook.com/cnn/videos/10155529876156509/',
  135. 'md5': '9571fae53d4165bbbadb17a94651dcdc',
  136. 'info_dict': {
  137. 'id': '10155529876156509',
  138. 'ext': 'mp4',
  139. 'title': 'She survived the holocaust — and years later, she’s getting her citizenship s...',
  140. 'timestamp': 1477818095,
  141. 'upload_date': '20161030',
  142. 'uploader': 'CNN',
  143. 'thumbnail': r're:^https?://.*',
  144. 'view_count': int,
  145. },
  146. }, {
  147. # bigPipe.onPageletArrive ... onPageletArrive pagelet_group_mall
  148. # data.node.comet_sections.content.story.attachments[].style_type_renderer.attachment.media
  149. 'url': 'https://www.facebook.com/yaroslav.korpan/videos/1417995061575415/',
  150. 'info_dict': {
  151. 'id': '1417995061575415',
  152. 'ext': 'mp4',
  153. 'title': 'md5:1db063d6a8c13faa8da727817339c857',
  154. 'timestamp': 1486648217,
  155. 'upload_date': '20170209',
  156. 'uploader': 'Yaroslav Korpan',
  157. },
  158. 'params': {
  159. 'skip_download': True,
  160. },
  161. }, {
  162. 'url': 'https://www.facebook.com/LaGuiaDelVaron/posts/1072691702860471',
  163. 'info_dict': {
  164. 'id': '1072691702860471',
  165. 'ext': 'mp4',
  166. 'title': 'md5:ae2d22a93fbb12dad20dc393a869739d',
  167. 'timestamp': 1477305000,
  168. 'upload_date': '20161024',
  169. 'uploader': 'La Guía Del Varón',
  170. 'thumbnail': r're:^https?://.*',
  171. },
  172. 'params': {
  173. 'skip_download': True,
  174. },
  175. }, {
  176. # data.node.comet_sections.content.story.attachments[].style_type_renderer.attachment.media
  177. 'url': 'https://www.facebook.com/groups/1024490957622648/permalink/1396382447100162/',
  178. 'info_dict': {
  179. 'id': '1396382447100162',
  180. 'ext': 'mp4',
  181. 'title': 'md5:19a428bbde91364e3de815383b54a235',
  182. 'timestamp': 1486035494,
  183. 'upload_date': '20170202',
  184. 'uploader': 'Elisabeth Ahtn',
  185. },
  186. 'params': {
  187. 'skip_download': True,
  188. },
  189. }, {
  190. 'url': 'https://www.facebook.com/video.php?v=10204634152394104',
  191. 'only_matching': True,
  192. }, {
  193. 'url': 'https://www.facebook.com/amogood/videos/1618742068337349/?fref=nf',
  194. 'only_matching': True,
  195. }, {
  196. # data.mediaset.currMedia.edges
  197. 'url': 'https://www.facebook.com/ChristyClarkForBC/videos/vb.22819070941/10153870694020942/?type=2&theater',
  198. 'only_matching': True,
  199. }, {
  200. # data.video.story.attachments[].media
  201. 'url': 'facebook:544765982287235',
  202. 'only_matching': True,
  203. }, {
  204. # data.node.comet_sections.content.story.attachments[].style_type_renderer.attachment.media
  205. 'url': 'https://www.facebook.com/groups/164828000315060/permalink/764967300301124/',
  206. 'only_matching': True,
  207. }, {
  208. # data.video.creation_story.attachments[].media
  209. 'url': 'https://zh-hk.facebook.com/peoplespower/videos/1135894589806027/',
  210. 'only_matching': True,
  211. }, {
  212. # data.video
  213. 'url': 'https://www.facebookcorewwwi.onion/video.php?v=274175099429670',
  214. 'only_matching': True,
  215. }, {
  216. # no title
  217. 'url': 'https://www.facebook.com/onlycleverentertainment/videos/1947995502095005/',
  218. 'only_matching': True,
  219. }, {
  220. # data.video
  221. 'url': 'https://www.facebook.com/WatchESLOne/videos/359649331226507/',
  222. 'info_dict': {
  223. 'id': '359649331226507',
  224. 'ext': 'mp4',
  225. 'title': '#ESLOne VoD - Birmingham Finals Day#1 Fnatic vs. @Evil Geniuses',
  226. 'uploader': 'ESL One Dota 2',
  227. },
  228. 'params': {
  229. 'skip_download': True,
  230. },
  231. }, {
  232. # data.node.comet_sections.content.story.attachments[].style_type_renderer.attachment.all_subattachments.nodes[].media
  233. 'url': 'https://www.facebook.com/100033620354545/videos/106560053808006/',
  234. 'info_dict': {
  235. 'id': '106560053808006',
  236. },
  237. 'playlist_count': 2,
  238. }, {
  239. # data.video.story.attachments[].media
  240. 'url': 'https://www.facebook.com/watch/?v=647537299265662',
  241. 'only_matching': True,
  242. }, {
  243. # data.node.comet_sections.content.story.attachments[].style_type_renderer.attachment.all_subattachments.nodes[].media
  244. 'url': 'https://www.facebook.com/PankajShahLondon/posts/10157667649866271',
  245. 'info_dict': {
  246. 'id': '10157667649866271',
  247. },
  248. 'playlist_count': 3,
  249. }, {
  250. # data.nodes[].comet_sections.content.story.attachments[].style_type_renderer.attachment.media
  251. 'url': 'https://m.facebook.com/Alliance.Police.Department/posts/4048563708499330',
  252. 'info_dict': {
  253. 'id': '117576630041613',
  254. 'ext': 'mp4',
  255. # TODO: title can be extracted from video page
  256. 'title': 'Facebook video #117576630041613',
  257. 'uploader_id': '189393014416438',
  258. 'upload_date': '20201123',
  259. 'timestamp': 1606162592,
  260. },
  261. 'skip': 'Requires logging in',
  262. }, {
  263. # node.comet_sections.content.story.attached_story.attachments.style_type_renderer.attachment.media
  264. 'url': 'https://www.facebook.com/groups/ateistiskselskab/permalink/10154930137678856/',
  265. 'info_dict': {
  266. 'id': '211567722618337',
  267. 'ext': 'mp4',
  268. 'title': 'Facebook video #211567722618337',
  269. 'uploader_id': '127875227654254',
  270. 'upload_date': '20161122',
  271. 'timestamp': 1479793574,
  272. },
  273. }, {
  274. # data.video.creation_story.attachments[].media
  275. 'url': 'https://www.facebook.com/watch/live/?v=1823658634322275',
  276. 'only_matching': True,
  277. }]
  278. _SUPPORTED_PAGLETS_REGEX = r'(?:pagelet_group_mall|permalink_video_pagelet|hyperfeed_story_id_[0-9a-f]+)'
  279. @staticmethod
  280. def _extract_urls(webpage):
  281. urls = []
  282. for mobj in re.finditer(
  283. r'<iframe[^>]+?src=(["\'])(?P<url>https?://www\.facebook\.com/(?:video/embed|plugins/video\.php).+?)\1',
  284. webpage):
  285. urls.append(mobj.group('url'))
  286. # Facebook API embed
  287. # see https://developers.facebook.com/docs/plugins/embedded-video-player
  288. for mobj in re.finditer(r'''(?x)<div[^>]+
  289. class=(?P<q1>[\'"])[^\'"]*\bfb-(?:video|post)\b[^\'"]*(?P=q1)[^>]+
  290. data-href=(?P<q2>[\'"])(?P<url>(?:https?:)?//(?:www\.)?facebook.com/.+?)(?P=q2)''', webpage):
  291. urls.append(mobj.group('url'))
  292. return urls
  293. def _login(self):
  294. useremail, password = self._get_login_info()
  295. if useremail is None:
  296. return
  297. login_page_req = sanitized_Request(self._LOGIN_URL)
  298. self._set_cookie('facebook.com', 'locale', 'en_US')
  299. login_page = self._download_webpage(login_page_req, None,
  300. note='Downloading login page',
  301. errnote='Unable to download login page')
  302. lsd = self._search_regex(
  303. r'<input type="hidden" name="lsd" value="([^"]*)"',
  304. login_page, 'lsd')
  305. lgnrnd = self._search_regex(r'name="lgnrnd" value="([^"]*?)"', login_page, 'lgnrnd')
  306. login_form = {
  307. 'email': useremail,
  308. 'pass': password,
  309. 'lsd': lsd,
  310. 'lgnrnd': lgnrnd,
  311. 'next': 'http://facebook.com/home.php',
  312. 'default_persistent': '0',
  313. 'legacy_return': '1',
  314. 'timezone': '-60',
  315. 'trynum': '1',
  316. }
  317. request = sanitized_Request(self._LOGIN_URL, urlencode_postdata(login_form))
  318. request.add_header('Content-Type', 'application/x-www-form-urlencoded')
  319. try:
  320. login_results = self._download_webpage(request, None,
  321. note='Logging in', errnote='unable to fetch login page')
  322. if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
  323. error = self._html_search_regex(
  324. r'(?s)<div[^>]+class=(["\']).*?login_error_box.*?\1[^>]*><div[^>]*>.*?</div><div[^>]*>(?P<error>.+?)</div>',
  325. login_results, 'login error', default=None, group='error')
  326. if error:
  327. raise ExtractorError('Unable to login: %s' % error, expected=True)
  328. self._downloader.report_warning('unable to log in: bad username/password, or exceeded login rate limit (~3/min). Check credentials or wait.')
  329. return
  330. fb_dtsg = self._search_regex(
  331. r'name="fb_dtsg" value="(.+?)"', login_results, 'fb_dtsg', default=None)
  332. h = self._search_regex(
  333. r'name="h"\s+(?:\w+="[^"]+"\s+)*?value="([^"]+)"', login_results, 'h', default=None)
  334. if not fb_dtsg or not h:
  335. return
  336. check_form = {
  337. 'fb_dtsg': fb_dtsg,
  338. 'h': h,
  339. 'name_action_selected': 'dont_save',
  340. }
  341. check_req = sanitized_Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
  342. check_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
  343. check_response = self._download_webpage(check_req, None,
  344. note='Confirming login')
  345. if re.search(r'id="checkpointSubmitButton"', check_response) is not None:
  346. self._downloader.report_warning('Unable to confirm login, you have to login in your browser and authorize the login.')
  347. except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  348. self._downloader.report_warning('unable to log in: %s' % error_to_compat_str(err))
  349. return
  350. def _real_initialize(self):
  351. self._login()
  352. def _extract_from_url(self, url, video_id):
  353. webpage = self._download_webpage(
  354. url.replace('://m.facebook.com/', '://www.facebook.com/'), video_id)
  355. video_data = None
  356. def extract_video_data(instances):
  357. video_data = []
  358. for item in instances:
  359. if try_get(item, lambda x: x[1][0]) == 'VideoConfig':
  360. video_item = item[2][0]
  361. if video_item.get('video_id'):
  362. video_data.append(video_item['videoData'])
  363. return video_data
  364. server_js_data = self._parse_json(self._search_regex(
  365. [r'handleServerJS\(({.+})(?:\);|,")', r'\bs\.handle\(({.+?})\);'],
  366. webpage, 'server js data', default='{}'), video_id, fatal=False)
  367. if server_js_data:
  368. video_data = extract_video_data(server_js_data.get('instances', []))
  369. def extract_from_jsmods_instances(js_data):
  370. if js_data:
  371. return extract_video_data(try_get(
  372. js_data, lambda x: x['jsmods']['instances'], list) or [])
  373. def extract_dash_manifest(video, formats):
  374. dash_manifest = video.get('dash_manifest')
  375. if dash_manifest:
  376. formats.extend(self._parse_mpd_formats(
  377. compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest))))
  378. def process_formats(formats):
  379. # Downloads with browser's User-Agent are rate limited. Working around
  380. # with non-browser User-Agent.
  381. for f in formats:
  382. f.setdefault('http_headers', {})['User-Agent'] = 'facebookexternalhit/1.1'
  383. self._sort_formats(formats)
  384. if not video_data:
  385. server_js_data = self._parse_json(self._search_regex([
  386. r'bigPipe\.onPageletArrive\(({.+?})\)\s*;\s*}\s*\)\s*,\s*["\']onPageletArrive\s+' + self._SUPPORTED_PAGLETS_REGEX,
  387. r'bigPipe\.onPageletArrive\(({.*?id\s*:\s*"%s".*?})\);' % self._SUPPORTED_PAGLETS_REGEX
  388. ], webpage, 'js data', default='{}'), video_id, js_to_json, False)
  389. video_data = extract_from_jsmods_instances(server_js_data)
  390. if not video_data:
  391. graphql_data = self._parse_json(self._search_regex(
  392. r'handleWithCustomApplyEach\([^,]+,\s*({.*?"(?:dash_manifest|playable_url(?:_quality_hd)?)"\s*:\s*"[^"]+".*?})\);',
  393. webpage, 'graphql data', default='{}'), video_id, fatal=False) or {}
  394. for require in (graphql_data.get('require') or []):
  395. if require[0] == 'RelayPrefetchedStreamCache':
  396. entries = []
  397. def parse_graphql_video(video):
  398. formats = []
  399. q = qualities(['sd', 'hd'])
  400. for (suffix, format_id) in [('', 'sd'), ('_quality_hd', 'hd')]:
  401. playable_url = video.get('playable_url' + suffix)
  402. if not playable_url:
  403. continue
  404. formats.append({
  405. 'format_id': format_id,
  406. 'quality': q(format_id),
  407. 'url': playable_url,
  408. })
  409. extract_dash_manifest(video, formats)
  410. process_formats(formats)
  411. v_id = video.get('videoId') or video.get('id') or video_id
  412. info = {
  413. 'id': v_id,
  414. 'formats': formats,
  415. 'thumbnail': try_get(video, lambda x: x['thumbnailImage']['uri']),
  416. 'uploader_id': try_get(video, lambda x: x['owner']['id']),
  417. 'timestamp': int_or_none(video.get('publish_time')),
  418. 'duration': float_or_none(video.get('playable_duration_in_ms'), 1000),
  419. }
  420. description = try_get(video, lambda x: x['savable_description']['text'])
  421. title = video.get('name')
  422. if title:
  423. info.update({
  424. 'title': title,
  425. 'description': description,
  426. })
  427. else:
  428. info['title'] = description or 'Facebook video #%s' % v_id
  429. entries.append(info)
  430. def parse_attachment(attachment, key='media'):
  431. media = attachment.get(key) or {}
  432. if media.get('__typename') == 'Video':
  433. return parse_graphql_video(media)
  434. data = try_get(require, lambda x: x[3][1]['__bbox']['result']['data'], dict) or {}
  435. nodes = data.get('nodes') or []
  436. node = data.get('node') or {}
  437. if not nodes and node:
  438. nodes.append(node)
  439. for node in nodes:
  440. story = try_get(node, lambda x: x['comet_sections']['content']['story'], dict) or {}
  441. attachments = try_get(story, [
  442. lambda x: x['attached_story']['attachments'],
  443. lambda x: x['attachments']
  444. ], list) or []
  445. for attachment in attachments:
  446. attachment = try_get(attachment, lambda x: x['style_type_renderer']['attachment'], dict)
  447. ns = try_get(attachment, lambda x: x['all_subattachments']['nodes'], list) or []
  448. for n in ns:
  449. parse_attachment(n)
  450. parse_attachment(attachment)
  451. edges = try_get(data, lambda x: x['mediaset']['currMedia']['edges'], list) or []
  452. for edge in edges:
  453. parse_attachment(edge, key='node')
  454. video = data.get('video') or {}
  455. if video:
  456. attachments = try_get(video, [
  457. lambda x: x['story']['attachments'],
  458. lambda x: x['creation_story']['attachments']
  459. ], list) or []
  460. for attachment in attachments:
  461. parse_attachment(attachment)
  462. if not entries:
  463. parse_graphql_video(video)
  464. return self.playlist_result(entries, video_id)
  465. if not video_data:
  466. m_msg = re.search(r'class="[^"]*uiInterstitialContent[^"]*"><div>(.*?)</div>', webpage)
  467. if m_msg is not None:
  468. raise ExtractorError(
  469. 'The video is not available, Facebook said: "%s"' % m_msg.group(1),
  470. expected=True)
  471. elif '>You must log in to continue' in webpage:
  472. self.raise_login_required()
  473. # Video info not in first request, do a secondary request using
  474. # tahoe player specific URL
  475. tahoe_data = self._download_webpage(
  476. self._VIDEO_PAGE_TAHOE_TEMPLATE % video_id, video_id,
  477. data=urlencode_postdata({
  478. '__a': 1,
  479. '__pc': self._search_regex(
  480. r'pkg_cohort["\']\s*:\s*["\'](.+?)["\']', webpage,
  481. 'pkg cohort', default='PHASED:DEFAULT'),
  482. '__rev': self._search_regex(
  483. r'client_revision["\']\s*:\s*(\d+),', webpage,
  484. 'client revision', default='3944515'),
  485. 'fb_dtsg': self._search_regex(
  486. r'"DTSGInitialData"\s*,\s*\[\]\s*,\s*{\s*"token"\s*:\s*"([^"]+)"',
  487. webpage, 'dtsg token', default=''),
  488. }),
  489. headers={
  490. 'Content-Type': 'application/x-www-form-urlencoded',
  491. })
  492. tahoe_js_data = self._parse_json(
  493. self._search_regex(
  494. r'for\s+\(\s*;\s*;\s*\)\s*;(.+)', tahoe_data,
  495. 'tahoe js data', default='{}'),
  496. video_id, fatal=False)
  497. video_data = extract_from_jsmods_instances(tahoe_js_data)
  498. if not video_data:
  499. raise ExtractorError('Cannot parse data')
  500. if len(video_data) > 1:
  501. entries = []
  502. for v in video_data:
  503. video_url = v[0].get('video_url')
  504. if not video_url:
  505. continue
  506. entries.append(self.url_result(urljoin(
  507. url, video_url), self.ie_key(), v[0].get('video_id')))
  508. return self.playlist_result(entries, video_id)
  509. video_data = video_data[0]
  510. formats = []
  511. subtitles = {}
  512. for f in video_data:
  513. format_id = f['stream_type']
  514. if f and isinstance(f, dict):
  515. f = [f]
  516. if not f or not isinstance(f, list):
  517. continue
  518. for quality in ('sd', 'hd'):
  519. for src_type in ('src', 'src_no_ratelimit'):
  520. src = f[0].get('%s_%s' % (quality, src_type))
  521. if src:
  522. preference = -10 if format_id == 'progressive' else 0
  523. if quality == 'hd':
  524. preference += 5
  525. formats.append({
  526. 'format_id': '%s_%s_%s' % (format_id, quality, src_type),
  527. 'url': src,
  528. 'preference': preference,
  529. })
  530. extract_dash_manifest(f[0], formats)
  531. subtitles_src = f[0].get('subtitles_src')
  532. if subtitles_src:
  533. subtitles.setdefault('en', []).append({'url': subtitles_src})
  534. if not formats:
  535. raise ExtractorError('Cannot find video formats')
  536. process_formats(formats)
  537. video_title = self._html_search_regex(
  538. r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>([^<]*)</h2>', webpage,
  539. 'title', default=None)
  540. if not video_title:
  541. video_title = self._html_search_regex(
  542. r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(.*?)</span>',
  543. webpage, 'alternative title', default=None)
  544. if not video_title:
  545. video_title = self._html_search_meta(
  546. 'description', webpage, 'title', default=None)
  547. if video_title:
  548. video_title = limit_length(video_title, 80)
  549. else:
  550. video_title = 'Facebook video #%s' % video_id
  551. uploader = clean_html(get_element_by_id(
  552. 'fbPhotoPageAuthorName', webpage)) or self._search_regex(
  553. r'ownerName\s*:\s*"([^"]+)"', webpage, 'uploader',
  554. default=None) or self._og_search_title(webpage, fatal=False)
  555. timestamp = int_or_none(self._search_regex(
  556. r'<abbr[^>]+data-utime=["\'](\d+)', webpage,
  557. 'timestamp', default=None))
  558. thumbnail = self._html_search_meta(['og:image', 'twitter:image'], webpage)
  559. view_count = parse_count(self._search_regex(
  560. r'\bviewCount\s*:\s*["\']([\d,.]+)', webpage, 'view count',
  561. default=None))
  562. info_dict = {
  563. 'id': video_id,
  564. 'title': video_title,
  565. 'formats': formats,
  566. 'uploader': uploader,
  567. 'timestamp': timestamp,
  568. 'thumbnail': thumbnail,
  569. 'view_count': view_count,
  570. 'subtitles': subtitles,
  571. }
  572. return info_dict
  573. def _real_extract(self, url):
  574. video_id = self._match_id(url)
  575. real_url = self._VIDEO_PAGE_TEMPLATE % video_id if url.startswith('facebook:') else url
  576. return self._extract_from_url(real_url, video_id)
  577. class FacebookPluginsVideoIE(InfoExtractor):
  578. _VALID_URL = r'https?://(?:[\w-]+\.)?facebook\.com/plugins/video\.php\?.*?\bhref=(?P<id>https.+)'
  579. _TESTS = [{
  580. 'url': 'https://www.facebook.com/plugins/video.php?href=https%3A%2F%2Fwww.facebook.com%2Fgov.sg%2Fvideos%2F10154383743583686%2F&show_text=0&width=560',
  581. 'md5': '5954e92cdfe51fe5782ae9bda7058a07',
  582. 'info_dict': {
  583. 'id': '10154383743583686',
  584. 'ext': 'mp4',
  585. 'title': 'What to do during the haze?',
  586. 'uploader': 'Gov.sg',
  587. 'upload_date': '20160826',
  588. 'timestamp': 1472184808,
  589. },
  590. 'add_ie': [FacebookIE.ie_key()],
  591. }, {
  592. 'url': 'https://www.facebook.com/plugins/video.php?href=https%3A%2F%2Fwww.facebook.com%2Fvideo.php%3Fv%3D10204634152394104',
  593. 'only_matching': True,
  594. }, {
  595. 'url': 'https://www.facebook.com/plugins/video.php?href=https://www.facebook.com/gov.sg/videos/10154383743583686/&show_text=0&width=560',
  596. 'only_matching': True,
  597. }]
  598. def _real_extract(self, url):
  599. return self.url_result(
  600. compat_urllib_parse_unquote(self._match_id(url)),
  601. FacebookIE.ie_key())