test_YoutubeDL.py 48 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190
  1. #!/usr/bin/env python
  2. # coding: utf-8
  3. from __future__ import unicode_literals
  4. # Allow direct execution
  5. import os
  6. import sys
  7. import unittest
  8. sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
  9. import copy
  10. import json
  11. from test.helper import (
  12. FakeYDL,
  13. assertRegexpMatches,
  14. try_rm,
  15. )
  16. from youtube_dl import YoutubeDL
  17. from youtube_dl.compat import (
  18. compat_http_cookiejar_Cookie,
  19. compat_http_cookies_SimpleCookie,
  20. compat_kwargs,
  21. compat_str,
  22. compat_urllib_error,
  23. )
  24. from youtube_dl.extractor import YoutubeIE
  25. from youtube_dl.extractor.common import InfoExtractor
  26. from youtube_dl.postprocessor.common import PostProcessor
  27. from youtube_dl.utils import (
  28. ExtractorError,
  29. match_filter_func,
  30. traverse_obj,
  31. )
  32. TEST_URL = 'http://localhost/sample.mp4'
  33. class YDL(FakeYDL):
  34. def __init__(self, *args, **kwargs):
  35. super(YDL, self).__init__(*args, **kwargs)
  36. self.downloaded_info_dicts = []
  37. self.msgs = []
  38. def process_info(self, info_dict):
  39. self.downloaded_info_dicts.append(info_dict.copy())
  40. def to_screen(self, msg):
  41. self.msgs.append(msg)
  42. def dl(self, *args, **kwargs):
  43. assert False, 'Downloader must not be invoked for test_YoutubeDL'
  44. def _make_result(formats, **kwargs):
  45. res = {
  46. 'formats': formats,
  47. 'id': 'testid',
  48. 'title': 'testttitle',
  49. 'extractor': 'testex',
  50. 'extractor_key': 'TestEx',
  51. 'webpage_url': 'http://example.com/watch?v=shenanigans',
  52. }
  53. res.update(**compat_kwargs(kwargs))
  54. return res
  55. class TestFormatSelection(unittest.TestCase):
  56. def test_prefer_free_formats(self):
  57. # Same resolution => download webm
  58. ydl = YDL()
  59. ydl.params['prefer_free_formats'] = True
  60. formats = [
  61. {'ext': 'webm', 'height': 460, 'url': TEST_URL},
  62. {'ext': 'mp4', 'height': 460, 'url': TEST_URL},
  63. ]
  64. info_dict = _make_result(formats)
  65. yie = YoutubeIE(ydl)
  66. yie._sort_formats(info_dict['formats'])
  67. ydl.process_ie_result(info_dict)
  68. downloaded = ydl.downloaded_info_dicts[0]
  69. self.assertEqual(downloaded['ext'], 'webm')
  70. # Different resolution => download best quality (mp4)
  71. ydl = YDL()
  72. ydl.params['prefer_free_formats'] = True
  73. formats = [
  74. {'ext': 'webm', 'height': 720, 'url': TEST_URL},
  75. {'ext': 'mp4', 'height': 1080, 'url': TEST_URL},
  76. ]
  77. info_dict['formats'] = formats
  78. yie = YoutubeIE(ydl)
  79. yie._sort_formats(info_dict['formats'])
  80. ydl.process_ie_result(info_dict)
  81. downloaded = ydl.downloaded_info_dicts[0]
  82. self.assertEqual(downloaded['ext'], 'mp4')
  83. # No prefer_free_formats => prefer mp4 and flv for greater compatibility
  84. ydl = YDL()
  85. ydl.params['prefer_free_formats'] = False
  86. formats = [
  87. {'ext': 'webm', 'height': 720, 'url': TEST_URL},
  88. {'ext': 'mp4', 'height': 720, 'url': TEST_URL},
  89. {'ext': 'flv', 'height': 720, 'url': TEST_URL},
  90. ]
  91. info_dict['formats'] = formats
  92. yie = YoutubeIE(ydl)
  93. yie._sort_formats(info_dict['formats'])
  94. ydl.process_ie_result(info_dict)
  95. downloaded = ydl.downloaded_info_dicts[0]
  96. self.assertEqual(downloaded['ext'], 'mp4')
  97. ydl = YDL()
  98. ydl.params['prefer_free_formats'] = False
  99. formats = [
  100. {'ext': 'flv', 'height': 720, 'url': TEST_URL},
  101. {'ext': 'webm', 'height': 720, 'url': TEST_URL},
  102. ]
  103. info_dict['formats'] = formats
  104. yie = YoutubeIE(ydl)
  105. yie._sort_formats(info_dict['formats'])
  106. ydl.process_ie_result(info_dict)
  107. downloaded = ydl.downloaded_info_dicts[0]
  108. self.assertEqual(downloaded['ext'], 'flv')
  109. def test_format_selection(self):
  110. formats = [
  111. {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
  112. {'format_id': 'example-with-dashes', 'ext': 'webm', 'preference': 1, 'url': TEST_URL},
  113. {'format_id': '45', 'ext': 'webm', 'preference': 2, 'url': TEST_URL},
  114. {'format_id': '47', 'ext': 'webm', 'preference': 3, 'url': TEST_URL},
  115. {'format_id': '2', 'ext': 'flv', 'preference': 4, 'url': TEST_URL},
  116. ]
  117. info_dict = _make_result(formats)
  118. ydl = YDL({'format': '20/47'})
  119. ydl.process_ie_result(info_dict.copy())
  120. downloaded = ydl.downloaded_info_dicts[0]
  121. self.assertEqual(downloaded['format_id'], '47')
  122. ydl = YDL({'format': '20/71/worst'})
  123. ydl.process_ie_result(info_dict.copy())
  124. downloaded = ydl.downloaded_info_dicts[0]
  125. self.assertEqual(downloaded['format_id'], '35')
  126. ydl = YDL()
  127. ydl.process_ie_result(info_dict.copy())
  128. downloaded = ydl.downloaded_info_dicts[0]
  129. self.assertEqual(downloaded['format_id'], '2')
  130. ydl = YDL({'format': 'webm/mp4'})
  131. ydl.process_ie_result(info_dict.copy())
  132. downloaded = ydl.downloaded_info_dicts[0]
  133. self.assertEqual(downloaded['format_id'], '47')
  134. ydl = YDL({'format': '3gp/40/mp4'})
  135. ydl.process_ie_result(info_dict.copy())
  136. downloaded = ydl.downloaded_info_dicts[0]
  137. self.assertEqual(downloaded['format_id'], '35')
  138. ydl = YDL({'format': 'example-with-dashes'})
  139. ydl.process_ie_result(info_dict.copy())
  140. downloaded = ydl.downloaded_info_dicts[0]
  141. self.assertEqual(downloaded['format_id'], 'example-with-dashes')
  142. def test_format_selection_audio(self):
  143. formats = [
  144. {'format_id': 'audio-low', 'ext': 'webm', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
  145. {'format_id': 'audio-mid', 'ext': 'webm', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
  146. {'format_id': 'audio-high', 'ext': 'flv', 'preference': 3, 'vcodec': 'none', 'url': TEST_URL},
  147. {'format_id': 'vid', 'ext': 'mp4', 'preference': 4, 'url': TEST_URL},
  148. ]
  149. info_dict = _make_result(formats)
  150. ydl = YDL({'format': 'bestaudio'})
  151. ydl.process_ie_result(info_dict.copy())
  152. downloaded = ydl.downloaded_info_dicts[0]
  153. self.assertEqual(downloaded['format_id'], 'audio-high')
  154. ydl = YDL({'format': 'worstaudio'})
  155. ydl.process_ie_result(info_dict.copy())
  156. downloaded = ydl.downloaded_info_dicts[0]
  157. self.assertEqual(downloaded['format_id'], 'audio-low')
  158. formats = [
  159. {'format_id': 'vid-low', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
  160. {'format_id': 'vid-high', 'ext': 'mp4', 'preference': 2, 'url': TEST_URL},
  161. ]
  162. info_dict = _make_result(formats)
  163. ydl = YDL({'format': 'bestaudio/worstaudio/best'})
  164. ydl.process_ie_result(info_dict.copy())
  165. downloaded = ydl.downloaded_info_dicts[0]
  166. self.assertEqual(downloaded['format_id'], 'vid-high')
  167. def test_format_selection_audio_exts(self):
  168. formats = [
  169. {'format_id': 'mp3-64', 'ext': 'mp3', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
  170. {'format_id': 'ogg-64', 'ext': 'ogg', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
  171. {'format_id': 'aac-64', 'ext': 'aac', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
  172. {'format_id': 'mp3-32', 'ext': 'mp3', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
  173. {'format_id': 'aac-32', 'ext': 'aac', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
  174. ]
  175. info_dict = _make_result(formats)
  176. ydl = YDL({'format': 'best'})
  177. ie = YoutubeIE(ydl)
  178. ie._sort_formats(info_dict['formats'])
  179. ydl.process_ie_result(copy.deepcopy(info_dict))
  180. downloaded = ydl.downloaded_info_dicts[0]
  181. self.assertEqual(downloaded['format_id'], 'aac-64')
  182. ydl = YDL({'format': 'mp3'})
  183. ie = YoutubeIE(ydl)
  184. ie._sort_formats(info_dict['formats'])
  185. ydl.process_ie_result(copy.deepcopy(info_dict))
  186. downloaded = ydl.downloaded_info_dicts[0]
  187. self.assertEqual(downloaded['format_id'], 'mp3-64')
  188. ydl = YDL({'prefer_free_formats': True})
  189. ie = YoutubeIE(ydl)
  190. ie._sort_formats(info_dict['formats'])
  191. ydl.process_ie_result(copy.deepcopy(info_dict))
  192. downloaded = ydl.downloaded_info_dicts[0]
  193. self.assertEqual(downloaded['format_id'], 'ogg-64')
  194. def test_format_selection_video(self):
  195. formats = [
  196. {'format_id': 'dash-video-low', 'ext': 'mp4', 'preference': 1, 'acodec': 'none', 'url': TEST_URL},
  197. {'format_id': 'dash-video-high', 'ext': 'mp4', 'preference': 2, 'acodec': 'none', 'url': TEST_URL},
  198. {'format_id': 'vid', 'ext': 'mp4', 'preference': 3, 'url': TEST_URL},
  199. ]
  200. info_dict = _make_result(formats)
  201. ydl = YDL({'format': 'bestvideo'})
  202. ydl.process_ie_result(info_dict.copy())
  203. downloaded = ydl.downloaded_info_dicts[0]
  204. self.assertEqual(downloaded['format_id'], 'dash-video-high')
  205. ydl = YDL({'format': 'worstvideo'})
  206. ydl.process_ie_result(info_dict.copy())
  207. downloaded = ydl.downloaded_info_dicts[0]
  208. self.assertEqual(downloaded['format_id'], 'dash-video-low')
  209. ydl = YDL({'format': 'bestvideo[format_id^=dash][format_id$=low]'})
  210. ydl.process_ie_result(info_dict.copy())
  211. downloaded = ydl.downloaded_info_dicts[0]
  212. self.assertEqual(downloaded['format_id'], 'dash-video-low')
  213. formats = [
  214. {'format_id': 'vid-vcodec-dot', 'ext': 'mp4', 'preference': 1, 'vcodec': 'avc1.123456', 'acodec': 'none', 'url': TEST_URL},
  215. ]
  216. info_dict = _make_result(formats)
  217. ydl = YDL({'format': 'bestvideo[vcodec=avc1.123456]'})
  218. ydl.process_ie_result(info_dict.copy())
  219. downloaded = ydl.downloaded_info_dicts[0]
  220. self.assertEqual(downloaded['format_id'], 'vid-vcodec-dot')
  221. def test_format_selection_string_ops(self):
  222. formats = [
  223. {'format_id': 'abc-cba', 'ext': 'mp4', 'url': TEST_URL},
  224. {'format_id': 'zxc-cxz', 'ext': 'webm', 'url': TEST_URL},
  225. ]
  226. info_dict = _make_result(formats)
  227. # equals (=)
  228. ydl = YDL({'format': '[format_id=abc-cba]'})
  229. ydl.process_ie_result(info_dict.copy())
  230. downloaded = ydl.downloaded_info_dicts[0]
  231. self.assertEqual(downloaded['format_id'], 'abc-cba')
  232. # does not equal (!=)
  233. ydl = YDL({'format': '[format_id!=abc-cba]'})
  234. ydl.process_ie_result(info_dict.copy())
  235. downloaded = ydl.downloaded_info_dicts[0]
  236. self.assertEqual(downloaded['format_id'], 'zxc-cxz')
  237. ydl = YDL({'format': '[format_id!=abc-cba][format_id!=zxc-cxz]'})
  238. self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
  239. # starts with (^=)
  240. ydl = YDL({'format': '[format_id^=abc]'})
  241. ydl.process_ie_result(info_dict.copy())
  242. downloaded = ydl.downloaded_info_dicts[0]
  243. self.assertEqual(downloaded['format_id'], 'abc-cba')
  244. # does not start with (!^=)
  245. ydl = YDL({'format': '[format_id!^=abc]'})
  246. ydl.process_ie_result(info_dict.copy())
  247. downloaded = ydl.downloaded_info_dicts[0]
  248. self.assertEqual(downloaded['format_id'], 'zxc-cxz')
  249. ydl = YDL({'format': '[format_id!^=abc][format_id!^=zxc]'})
  250. self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
  251. # ends with ($=)
  252. ydl = YDL({'format': '[format_id$=cba]'})
  253. ydl.process_ie_result(info_dict.copy())
  254. downloaded = ydl.downloaded_info_dicts[0]
  255. self.assertEqual(downloaded['format_id'], 'abc-cba')
  256. # does not end with (!$=)
  257. ydl = YDL({'format': '[format_id!$=cba]'})
  258. ydl.process_ie_result(info_dict.copy())
  259. downloaded = ydl.downloaded_info_dicts[0]
  260. self.assertEqual(downloaded['format_id'], 'zxc-cxz')
  261. ydl = YDL({'format': '[format_id!$=cba][format_id!$=cxz]'})
  262. self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
  263. # contains (*=)
  264. ydl = YDL({'format': '[format_id*=bc-cb]'})
  265. ydl.process_ie_result(info_dict.copy())
  266. downloaded = ydl.downloaded_info_dicts[0]
  267. self.assertEqual(downloaded['format_id'], 'abc-cba')
  268. # does not contain (!*=)
  269. ydl = YDL({'format': '[format_id!*=bc-cb]'})
  270. ydl.process_ie_result(info_dict.copy())
  271. downloaded = ydl.downloaded_info_dicts[0]
  272. self.assertEqual(downloaded['format_id'], 'zxc-cxz')
  273. ydl = YDL({'format': '[format_id!*=abc][format_id!*=zxc]'})
  274. self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
  275. ydl = YDL({'format': '[format_id!*=-]'})
  276. self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
  277. def test_youtube_format_selection(self):
  278. order = [
  279. '38', '37', '46', '22', '45', '35', '44', '18', '34', '43', '6', '5', '17', '36', '13',
  280. # Apple HTTP Live Streaming
  281. '96', '95', '94', '93', '92', '132', '151',
  282. # 3D
  283. '85', '84', '102', '83', '101', '82', '100',
  284. # Dash video
  285. '137', '248', '136', '247', '135', '246',
  286. '245', '244', '134', '243', '133', '242', '160',
  287. # Dash audio
  288. '141', '172', '140', '171', '139',
  289. ]
  290. def format_info(f_id):
  291. info = YoutubeIE._formats[f_id].copy()
  292. # XXX: In real cases InfoExtractor._parse_mpd_formats() fills up 'acodec'
  293. # and 'vcodec', while in tests such information is incomplete since
  294. # commit a6c2c24479e5f4827ceb06f64d855329c0a6f593
  295. # test_YoutubeDL.test_youtube_format_selection is broken without
  296. # this fix
  297. if 'acodec' in info and 'vcodec' not in info:
  298. info['vcodec'] = 'none'
  299. elif 'vcodec' in info and 'acodec' not in info:
  300. info['acodec'] = 'none'
  301. info['format_id'] = f_id
  302. info['url'] = 'url:' + f_id
  303. return info
  304. formats_order = [format_info(f_id) for f_id in order]
  305. info_dict = _make_result(list(formats_order), extractor='youtube')
  306. ydl = YDL({'format': 'bestvideo+bestaudio'})
  307. yie = YoutubeIE(ydl)
  308. yie._sort_formats(info_dict['formats'])
  309. ydl.process_ie_result(info_dict)
  310. downloaded = ydl.downloaded_info_dicts[0]
  311. self.assertEqual(downloaded['format_id'], '137+141')
  312. self.assertEqual(downloaded['ext'], 'mp4')
  313. info_dict = _make_result(list(formats_order), extractor='youtube')
  314. ydl = YDL({'format': 'bestvideo[height>=999999]+bestaudio/best'})
  315. yie = YoutubeIE(ydl)
  316. yie._sort_formats(info_dict['formats'])
  317. ydl.process_ie_result(info_dict)
  318. downloaded = ydl.downloaded_info_dicts[0]
  319. self.assertEqual(downloaded['format_id'], '38')
  320. info_dict = _make_result(list(formats_order), extractor='youtube')
  321. ydl = YDL({'format': 'bestvideo/best,bestaudio'})
  322. yie = YoutubeIE(ydl)
  323. yie._sort_formats(info_dict['formats'])
  324. ydl.process_ie_result(info_dict)
  325. downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
  326. self.assertEqual(downloaded_ids, ['137', '141'])
  327. info_dict = _make_result(list(formats_order), extractor='youtube')
  328. ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])+bestaudio'})
  329. yie = YoutubeIE(ydl)
  330. yie._sort_formats(info_dict['formats'])
  331. ydl.process_ie_result(info_dict)
  332. downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
  333. self.assertEqual(downloaded_ids, ['137+141', '248+141'])
  334. info_dict = _make_result(list(formats_order), extractor='youtube')
  335. ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])[height<=720]+bestaudio'})
  336. yie = YoutubeIE(ydl)
  337. yie._sort_formats(info_dict['formats'])
  338. ydl.process_ie_result(info_dict)
  339. downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
  340. self.assertEqual(downloaded_ids, ['136+141', '247+141'])
  341. info_dict = _make_result(list(formats_order), extractor='youtube')
  342. ydl = YDL({'format': '(bestvideo[ext=none]/bestvideo[ext=webm])+bestaudio'})
  343. yie = YoutubeIE(ydl)
  344. yie._sort_formats(info_dict['formats'])
  345. ydl.process_ie_result(info_dict)
  346. downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
  347. self.assertEqual(downloaded_ids, ['248+141'])
  348. for f1, f2 in zip(formats_order, formats_order[1:]):
  349. info_dict = _make_result([f1, f2], extractor='youtube')
  350. ydl = YDL({'format': 'best/bestvideo'})
  351. yie = YoutubeIE(ydl)
  352. yie._sort_formats(info_dict['formats'])
  353. ydl.process_ie_result(info_dict)
  354. downloaded = ydl.downloaded_info_dicts[0]
  355. self.assertEqual(downloaded['format_id'], f1['format_id'])
  356. info_dict = _make_result([f2, f1], extractor='youtube')
  357. ydl = YDL({'format': 'best/bestvideo'})
  358. yie = YoutubeIE(ydl)
  359. yie._sort_formats(info_dict['formats'])
  360. ydl.process_ie_result(info_dict)
  361. downloaded = ydl.downloaded_info_dicts[0]
  362. self.assertEqual(downloaded['format_id'], f1['format_id'])
  363. def test_audio_only_extractor_format_selection(self):
  364. # For extractors with incomplete formats (all formats are audio-only or
  365. # video-only) best and worst should fallback to corresponding best/worst
  366. # video-only or audio-only formats (as per
  367. # https://github.com/ytdl-org/youtube-dl/pull/5556)
  368. formats = [
  369. {'format_id': 'low', 'ext': 'mp3', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
  370. {'format_id': 'high', 'ext': 'mp3', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
  371. ]
  372. info_dict = _make_result(formats)
  373. ydl = YDL({'format': 'best'})
  374. ydl.process_ie_result(info_dict.copy())
  375. downloaded = ydl.downloaded_info_dicts[0]
  376. self.assertEqual(downloaded['format_id'], 'high')
  377. ydl = YDL({'format': 'worst'})
  378. ydl.process_ie_result(info_dict.copy())
  379. downloaded = ydl.downloaded_info_dicts[0]
  380. self.assertEqual(downloaded['format_id'], 'low')
  381. def test_format_not_available(self):
  382. formats = [
  383. {'format_id': 'regular', 'ext': 'mp4', 'height': 360, 'url': TEST_URL},
  384. {'format_id': 'video', 'ext': 'mp4', 'height': 720, 'acodec': 'none', 'url': TEST_URL},
  385. ]
  386. info_dict = _make_result(formats)
  387. # This must fail since complete video-audio format does not match filter
  388. # and extractor does not provide incomplete only formats (i.e. only
  389. # video-only or audio-only).
  390. ydl = YDL({'format': 'best[height>360]'})
  391. self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
  392. def test_format_selection_issue_10083(self):
  393. # See https://github.com/ytdl-org/youtube-dl/issues/10083
  394. formats = [
  395. {'format_id': 'regular', 'height': 360, 'url': TEST_URL},
  396. {'format_id': 'video', 'height': 720, 'acodec': 'none', 'url': TEST_URL},
  397. {'format_id': 'audio', 'vcodec': 'none', 'url': TEST_URL},
  398. ]
  399. info_dict = _make_result(formats)
  400. ydl = YDL({'format': 'best[height>360]/bestvideo[height>360]+bestaudio'})
  401. ydl.process_ie_result(info_dict.copy())
  402. self.assertEqual(ydl.downloaded_info_dicts[0]['format_id'], 'video+audio')
  403. def test_invalid_format_specs(self):
  404. def assert_syntax_error(format_spec):
  405. ydl = YDL({'format': format_spec})
  406. info_dict = _make_result([{'format_id': 'foo', 'url': TEST_URL}])
  407. self.assertRaises(SyntaxError, ydl.process_ie_result, info_dict)
  408. assert_syntax_error('bestvideo,,best')
  409. assert_syntax_error('+bestaudio')
  410. assert_syntax_error('bestvideo+')
  411. assert_syntax_error('/')
  412. assert_syntax_error('bestvideo+bestvideo+bestaudio')
  413. def test_format_filtering(self):
  414. formats = [
  415. {'format_id': 'A', 'filesize': 500, 'width': 1000},
  416. {'format_id': 'B', 'filesize': 1000, 'width': 500},
  417. {'format_id': 'C', 'filesize': 1000, 'width': 400},
  418. {'format_id': 'D', 'filesize': 2000, 'width': 600},
  419. {'format_id': 'E', 'filesize': 3000},
  420. {'format_id': 'F'},
  421. {'format_id': 'G', 'filesize': 1000000},
  422. ]
  423. for f in formats:
  424. f['url'] = 'http://_/'
  425. f['ext'] = 'unknown'
  426. info_dict = _make_result(formats)
  427. ydl = YDL({'format': 'best[filesize<3000]'})
  428. ydl.process_ie_result(info_dict)
  429. downloaded = ydl.downloaded_info_dicts[0]
  430. self.assertEqual(downloaded['format_id'], 'D')
  431. ydl = YDL({'format': 'best[filesize<=3000]'})
  432. ydl.process_ie_result(info_dict)
  433. downloaded = ydl.downloaded_info_dicts[0]
  434. self.assertEqual(downloaded['format_id'], 'E')
  435. ydl = YDL({'format': 'best[filesize <= ? 3000]'})
  436. ydl.process_ie_result(info_dict)
  437. downloaded = ydl.downloaded_info_dicts[0]
  438. self.assertEqual(downloaded['format_id'], 'F')
  439. ydl = YDL({'format': 'best [filesize = 1000] [width>450]'})
  440. ydl.process_ie_result(info_dict)
  441. downloaded = ydl.downloaded_info_dicts[0]
  442. self.assertEqual(downloaded['format_id'], 'B')
  443. ydl = YDL({'format': 'best [filesize = 1000] [width!=450]'})
  444. ydl.process_ie_result(info_dict)
  445. downloaded = ydl.downloaded_info_dicts[0]
  446. self.assertEqual(downloaded['format_id'], 'C')
  447. ydl = YDL({'format': '[filesize>?1]'})
  448. ydl.process_ie_result(info_dict)
  449. downloaded = ydl.downloaded_info_dicts[0]
  450. self.assertEqual(downloaded['format_id'], 'G')
  451. ydl = YDL({'format': '[filesize<1M]'})
  452. ydl.process_ie_result(info_dict)
  453. downloaded = ydl.downloaded_info_dicts[0]
  454. self.assertEqual(downloaded['format_id'], 'E')
  455. ydl = YDL({'format': '[filesize<1MiB]'})
  456. ydl.process_ie_result(info_dict)
  457. downloaded = ydl.downloaded_info_dicts[0]
  458. self.assertEqual(downloaded['format_id'], 'G')
  459. ydl = YDL({'format': 'all[width>=400][width<=600]'})
  460. ydl.process_ie_result(info_dict)
  461. downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
  462. self.assertEqual(downloaded_ids, ['B', 'C', 'D'])
  463. ydl = YDL({'format': 'best[height<40]'})
  464. try:
  465. ydl.process_ie_result(info_dict)
  466. except ExtractorError:
  467. pass
  468. self.assertEqual(ydl.downloaded_info_dicts, [])
  469. def test_default_format_spec(self):
  470. ydl = YDL({'simulate': True})
  471. self.assertEqual(ydl._default_format_spec({}), 'bestvideo+bestaudio/best')
  472. ydl = YDL({})
  473. self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best/bestvideo+bestaudio')
  474. ydl = YDL({'simulate': True})
  475. self.assertEqual(ydl._default_format_spec({'is_live': True}), 'bestvideo+bestaudio/best')
  476. ydl = YDL({'outtmpl': '-'})
  477. self.assertEqual(ydl._default_format_spec({}), 'best/bestvideo+bestaudio')
  478. ydl = YDL({})
  479. self.assertEqual(ydl._default_format_spec({}, download=False), 'bestvideo+bestaudio/best')
  480. self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best/bestvideo+bestaudio')
  481. class TestYoutubeDL(unittest.TestCase):
  482. def test_subtitles(self):
  483. def s_formats(lang, autocaption=False):
  484. return [{
  485. 'ext': ext,
  486. 'url': 'http://localhost/video.%s.%s' % (lang, ext),
  487. '_auto': autocaption,
  488. } for ext in ['vtt', 'srt', 'ass']]
  489. subtitles = dict((l, s_formats(l)) for l in ['en', 'fr', 'es'])
  490. auto_captions = dict((l, s_formats(l, True)) for l in ['it', 'pt', 'es'])
  491. info_dict = {
  492. 'id': 'test',
  493. 'title': 'Test',
  494. 'url': 'http://localhost/video.mp4',
  495. 'subtitles': subtitles,
  496. 'automatic_captions': auto_captions,
  497. 'extractor': 'TEST',
  498. }
  499. def get_info(params={}):
  500. params.setdefault('simulate', True)
  501. ydl = YDL(params)
  502. ydl.report_warning = lambda *args, **kargs: None
  503. return ydl.process_video_result(info_dict, download=False)
  504. result = get_info()
  505. self.assertFalse(result.get('requested_subtitles'))
  506. self.assertEqual(result['subtitles'], subtitles)
  507. self.assertEqual(result['automatic_captions'], auto_captions)
  508. result = get_info({'writesubtitles': True})
  509. subs = result['requested_subtitles']
  510. self.assertTrue(subs)
  511. self.assertEqual(set(subs.keys()), set(['en']))
  512. self.assertTrue(subs['en'].get('data') is None)
  513. self.assertEqual(subs['en']['ext'], 'ass')
  514. result = get_info({'writesubtitles': True, 'subtitlesformat': 'foo/srt'})
  515. subs = result['requested_subtitles']
  516. self.assertEqual(subs['en']['ext'], 'srt')
  517. result = get_info({'writesubtitles': True, 'subtitleslangs': ['es', 'fr', 'it']})
  518. subs = result['requested_subtitles']
  519. self.assertTrue(subs)
  520. self.assertEqual(set(subs.keys()), set(['es', 'fr']))
  521. result = get_info({'writesubtitles': True, 'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
  522. subs = result['requested_subtitles']
  523. self.assertTrue(subs)
  524. self.assertEqual(set(subs.keys()), set(['es', 'pt']))
  525. self.assertFalse(subs['es']['_auto'])
  526. self.assertTrue(subs['pt']['_auto'])
  527. result = get_info({'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
  528. subs = result['requested_subtitles']
  529. self.assertTrue(subs)
  530. self.assertEqual(set(subs.keys()), set(['es', 'pt']))
  531. self.assertTrue(subs['es']['_auto'])
  532. self.assertTrue(subs['pt']['_auto'])
  533. def test_add_extra_info(self):
  534. test_dict = {
  535. 'extractor': 'Foo',
  536. }
  537. extra_info = {
  538. 'extractor': 'Bar',
  539. 'playlist': 'funny videos',
  540. }
  541. YDL.add_extra_info(test_dict, extra_info)
  542. self.assertEqual(test_dict['extractor'], 'Foo')
  543. self.assertEqual(test_dict['playlist'], 'funny videos')
  544. def test_prepare_filename(self):
  545. info = {
  546. 'id': '1234',
  547. 'ext': 'mp4',
  548. 'width': None,
  549. 'height': 1080,
  550. 'title1': '$PATH',
  551. 'title2': '%PATH%',
  552. }
  553. def fname(templ, na_placeholder='NA'):
  554. params = {'outtmpl': templ}
  555. if na_placeholder != 'NA':
  556. params['outtmpl_na_placeholder'] = na_placeholder
  557. ydl = YoutubeDL(params)
  558. return ydl.prepare_filename(info)
  559. self.assertEqual(fname('%(id)s.%(ext)s'), '1234.mp4')
  560. self.assertEqual(fname('%(id)s-%(width)s.%(ext)s'), '1234-NA.mp4')
  561. NA_TEST_OUTTMPL = '%(uploader_date)s-%(width)d-%(id)s.%(ext)s'
  562. # Replace missing fields with 'NA' by default
  563. self.assertEqual(fname(NA_TEST_OUTTMPL), 'NA-NA-1234.mp4')
  564. # Or by provided placeholder
  565. self.assertEqual(fname(NA_TEST_OUTTMPL, na_placeholder='none'), 'none-none-1234.mp4')
  566. self.assertEqual(fname(NA_TEST_OUTTMPL, na_placeholder=''), '--1234.mp4')
  567. self.assertEqual(fname('%(height)d.%(ext)s'), '1080.mp4')
  568. self.assertEqual(fname('%(height)6d.%(ext)s'), ' 1080.mp4')
  569. self.assertEqual(fname('%(height)-6d.%(ext)s'), '1080 .mp4')
  570. self.assertEqual(fname('%(height)06d.%(ext)s'), '001080.mp4')
  571. self.assertEqual(fname('%(height) 06d.%(ext)s'), ' 01080.mp4')
  572. self.assertEqual(fname('%(height) 06d.%(ext)s'), ' 01080.mp4')
  573. self.assertEqual(fname('%(height)0 6d.%(ext)s'), ' 01080.mp4')
  574. self.assertEqual(fname('%(height)0 6d.%(ext)s'), ' 01080.mp4')
  575. self.assertEqual(fname('%(height) 0 6d.%(ext)s'), ' 01080.mp4')
  576. self.assertEqual(fname('%%'), '%')
  577. self.assertEqual(fname('%%%%'), '%%')
  578. self.assertEqual(fname('%%(height)06d.%(ext)s'), '%(height)06d.mp4')
  579. self.assertEqual(fname('%(width)06d.%(ext)s'), 'NA.mp4')
  580. self.assertEqual(fname('%(width)06d.%%(ext)s'), 'NA.%(ext)s')
  581. self.assertEqual(fname('%%(width)06d.%(ext)s'), '%(width)06d.mp4')
  582. self.assertEqual(fname('Hello %(title1)s'), 'Hello $PATH')
  583. self.assertEqual(fname('Hello %(title2)s'), 'Hello %PATH%')
  584. def test_format_note(self):
  585. ydl = YoutubeDL()
  586. self.assertEqual(ydl._format_note({}), '')
  587. assertRegexpMatches(self, ydl._format_note({
  588. 'vbr': 10,
  589. }), r'^\s*10k$')
  590. assertRegexpMatches(self, ydl._format_note({
  591. 'fps': 30,
  592. }), r'^30fps$')
  593. def test_postprocessors(self):
  594. filename = 'post-processor-testfile.mp4'
  595. audiofile = filename + '.mp3'
  596. class SimplePP(PostProcessor):
  597. def run(self, info):
  598. with open(audiofile, 'wt') as f:
  599. f.write('EXAMPLE')
  600. return [info['filepath']], info
  601. def run_pp(params, PP):
  602. with open(filename, 'wt') as f:
  603. f.write('EXAMPLE')
  604. ydl = YoutubeDL(params)
  605. ydl.add_post_processor(PP())
  606. ydl.post_process(filename, {'filepath': filename})
  607. run_pp({'keepvideo': True}, SimplePP)
  608. self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
  609. self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
  610. os.unlink(filename)
  611. os.unlink(audiofile)
  612. run_pp({'keepvideo': False}, SimplePP)
  613. self.assertFalse(os.path.exists(filename), '%s exists' % filename)
  614. self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
  615. os.unlink(audiofile)
  616. class ModifierPP(PostProcessor):
  617. def run(self, info):
  618. with open(info['filepath'], 'wt') as f:
  619. f.write('MODIFIED')
  620. return [], info
  621. run_pp({'keepvideo': False}, ModifierPP)
  622. self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
  623. os.unlink(filename)
  624. def test_match_filter(self):
  625. class FilterYDL(YDL):
  626. def __init__(self, *args, **kwargs):
  627. super(FilterYDL, self).__init__(*args, **kwargs)
  628. self.params['simulate'] = True
  629. def process_info(self, info_dict):
  630. super(YDL, self).process_info(info_dict)
  631. def _match_entry(self, info_dict, incomplete):
  632. res = super(FilterYDL, self)._match_entry(info_dict, incomplete)
  633. if res is None:
  634. self.downloaded_info_dicts.append(info_dict)
  635. return res
  636. first = {
  637. 'id': '1',
  638. 'url': TEST_URL,
  639. 'title': 'one',
  640. 'extractor': 'TEST',
  641. 'duration': 30,
  642. 'filesize': 10 * 1024,
  643. 'playlist_id': '42',
  644. 'uploader': "變態妍字幕版 太妍 тест",
  645. 'creator': "тест ' 123 ' тест--",
  646. }
  647. second = {
  648. 'id': '2',
  649. 'url': TEST_URL,
  650. 'title': 'two',
  651. 'extractor': 'TEST',
  652. 'duration': 10,
  653. 'description': 'foo',
  654. 'filesize': 5 * 1024,
  655. 'playlist_id': '43',
  656. 'uploader': "тест 123",
  657. }
  658. videos = [first, second]
  659. def get_videos(filter_=None):
  660. ydl = FilterYDL({'match_filter': filter_})
  661. for v in videos:
  662. ydl.process_ie_result(v, download=True)
  663. return [v['id'] for v in ydl.downloaded_info_dicts]
  664. res = get_videos()
  665. self.assertEqual(res, ['1', '2'])
  666. def f(v):
  667. if v['id'] == '1':
  668. return None
  669. else:
  670. return 'Video id is not 1'
  671. res = get_videos(f)
  672. self.assertEqual(res, ['1'])
  673. f = match_filter_func('duration < 30')
  674. res = get_videos(f)
  675. self.assertEqual(res, ['2'])
  676. f = match_filter_func('description = foo')
  677. res = get_videos(f)
  678. self.assertEqual(res, ['2'])
  679. f = match_filter_func('description =? foo')
  680. res = get_videos(f)
  681. self.assertEqual(res, ['1', '2'])
  682. f = match_filter_func('filesize > 5KiB')
  683. res = get_videos(f)
  684. self.assertEqual(res, ['1'])
  685. f = match_filter_func('playlist_id = 42')
  686. res = get_videos(f)
  687. self.assertEqual(res, ['1'])
  688. f = match_filter_func('uploader = "變態妍字幕版 太妍 тест"')
  689. res = get_videos(f)
  690. self.assertEqual(res, ['1'])
  691. f = match_filter_func('uploader != "變態妍字幕版 太妍 тест"')
  692. res = get_videos(f)
  693. self.assertEqual(res, ['2'])
  694. f = match_filter_func('creator = "тест \' 123 \' тест--"')
  695. res = get_videos(f)
  696. self.assertEqual(res, ['1'])
  697. f = match_filter_func("creator = 'тест \\' 123 \\' тест--'")
  698. res = get_videos(f)
  699. self.assertEqual(res, ['1'])
  700. f = match_filter_func(r"creator = 'тест \' 123 \' тест--' & duration > 30")
  701. res = get_videos(f)
  702. self.assertEqual(res, [])
  703. def test_playlist_items_selection(self):
  704. entries = [{
  705. 'id': compat_str(i),
  706. 'title': compat_str(i),
  707. 'url': TEST_URL,
  708. } for i in range(1, 5)]
  709. playlist = {
  710. '_type': 'playlist',
  711. 'id': 'test',
  712. 'entries': entries,
  713. 'extractor': 'test:playlist',
  714. 'extractor_key': 'test:playlist',
  715. 'webpage_url': 'http://example.com',
  716. }
  717. def get_downloaded_info_dicts(params):
  718. ydl = YDL(params)
  719. # make a deep copy because the dictionary and nested entries
  720. # can be modified
  721. ydl.process_ie_result(copy.deepcopy(playlist))
  722. return ydl.downloaded_info_dicts
  723. def get_ids(params):
  724. return [int(v['id']) for v in get_downloaded_info_dicts(params)]
  725. result = get_ids({})
  726. self.assertEqual(result, [1, 2, 3, 4])
  727. result = get_ids({'playlistend': 10})
  728. self.assertEqual(result, [1, 2, 3, 4])
  729. result = get_ids({'playlistend': 2})
  730. self.assertEqual(result, [1, 2])
  731. result = get_ids({'playliststart': 10})
  732. self.assertEqual(result, [])
  733. result = get_ids({'playliststart': 2})
  734. self.assertEqual(result, [2, 3, 4])
  735. result = get_ids({'playlist_items': '2-4'})
  736. self.assertEqual(result, [2, 3, 4])
  737. result = get_ids({'playlist_items': '2,4'})
  738. self.assertEqual(result, [2, 4])
  739. result = get_ids({'playlist_items': '10'})
  740. self.assertEqual(result, [])
  741. result = get_ids({'playlist_items': '3-10'})
  742. self.assertEqual(result, [3, 4])
  743. result = get_ids({'playlist_items': '2-4,3-4,3'})
  744. self.assertEqual(result, [2, 3, 4])
  745. # Tests for https://github.com/ytdl-org/youtube-dl/issues/10591
  746. # @{
  747. result = get_downloaded_info_dicts({'playlist_items': '2-4,3-4,3'})
  748. self.assertEqual(result[0]['playlist_index'], 2)
  749. self.assertEqual(result[1]['playlist_index'], 3)
  750. result = get_downloaded_info_dicts({'playlist_items': '2-4,3-4,3'})
  751. self.assertEqual(result[0]['playlist_index'], 2)
  752. self.assertEqual(result[1]['playlist_index'], 3)
  753. self.assertEqual(result[2]['playlist_index'], 4)
  754. result = get_downloaded_info_dicts({'playlist_items': '4,2'})
  755. self.assertEqual(result[0]['playlist_index'], 4)
  756. self.assertEqual(result[1]['playlist_index'], 2)
  757. # @}
  758. def test_urlopen_no_file_protocol(self):
  759. # see https://github.com/ytdl-org/youtube-dl/issues/8227
  760. ydl = YDL()
  761. self.assertRaises(compat_urllib_error.URLError, ydl.urlopen, 'file:///etc/passwd')
  762. def test_do_not_override_ie_key_in_url_transparent(self):
  763. ydl = YDL()
  764. class Foo1IE(InfoExtractor):
  765. _VALID_URL = r'foo1:'
  766. def _real_extract(self, url):
  767. return {
  768. '_type': 'url_transparent',
  769. 'url': 'foo2:',
  770. 'ie_key': 'Foo2',
  771. 'title': 'foo1 title',
  772. 'id': 'foo1_id',
  773. }
  774. class Foo2IE(InfoExtractor):
  775. _VALID_URL = r'foo2:'
  776. def _real_extract(self, url):
  777. return {
  778. '_type': 'url',
  779. 'url': 'foo3:',
  780. 'ie_key': 'Foo3',
  781. }
  782. class Foo3IE(InfoExtractor):
  783. _VALID_URL = r'foo3:'
  784. def _real_extract(self, url):
  785. return _make_result([{'url': TEST_URL}], title='foo3 title')
  786. ydl.add_info_extractor(Foo1IE(ydl))
  787. ydl.add_info_extractor(Foo2IE(ydl))
  788. ydl.add_info_extractor(Foo3IE(ydl))
  789. ydl.extract_info('foo1:')
  790. downloaded = ydl.downloaded_info_dicts[0]
  791. self.assertEqual(downloaded['url'], TEST_URL)
  792. self.assertEqual(downloaded['title'], 'foo1 title')
  793. self.assertEqual(downloaded['id'], 'testid')
  794. self.assertEqual(downloaded['extractor'], 'testex')
  795. self.assertEqual(downloaded['extractor_key'], 'TestEx')
  796. # Test case for https://github.com/ytdl-org/youtube-dl/issues/27064
  797. def test_ignoreerrors_for_playlist_with_url_transparent_iterable_entries(self):
  798. ydl = YDL({
  799. 'format': 'extra',
  800. 'ignoreerrors': True,
  801. })
  802. ydl.trouble = lambda *_, **__: None
  803. class VideoIE(InfoExtractor):
  804. _VALID_URL = r'video:(?P<id>\d+)'
  805. def _real_extract(self, url):
  806. video_id = self._match_id(url)
  807. formats = [{
  808. 'format_id': 'default',
  809. 'url': 'url:',
  810. }]
  811. if video_id == '0':
  812. raise ExtractorError('foo')
  813. if video_id == '2':
  814. formats.append({
  815. 'format_id': 'extra',
  816. 'url': TEST_URL,
  817. })
  818. return {
  819. 'id': video_id,
  820. 'title': 'Video %s' % video_id,
  821. 'formats': formats,
  822. }
  823. class PlaylistIE(InfoExtractor):
  824. _VALID_URL = r'playlist:'
  825. def _entries(self):
  826. for n in range(3):
  827. video_id = compat_str(n)
  828. yield {
  829. '_type': 'url_transparent',
  830. 'ie_key': VideoIE.ie_key(),
  831. 'id': video_id,
  832. 'url': 'video:%s' % video_id,
  833. 'title': 'Video Transparent %s' % video_id,
  834. }
  835. def _real_extract(self, url):
  836. return self.playlist_result(self._entries())
  837. ydl.add_info_extractor(VideoIE(ydl))
  838. ydl.add_info_extractor(PlaylistIE(ydl))
  839. info = ydl.extract_info('playlist:')
  840. entries = info['entries']
  841. self.assertEqual(len(entries), 3)
  842. self.assertTrue(entries[0] is None)
  843. self.assertTrue(entries[1] is None)
  844. self.assertEqual(len(ydl.downloaded_info_dicts), 1)
  845. downloaded = ydl.downloaded_info_dicts[0]
  846. self.assertEqual(entries[2], downloaded)
  847. self.assertEqual(downloaded['url'], TEST_URL)
  848. self.assertEqual(downloaded['title'], 'Video Transparent 2')
  849. self.assertEqual(downloaded['id'], '2')
  850. self.assertEqual(downloaded['extractor'], 'Video')
  851. self.assertEqual(downloaded['extractor_key'], 'Video')
  852. def test_default_times(self):
  853. """Test addition of missing upload/release/_date from /release_/timestamp"""
  854. info = {
  855. 'id': '1234',
  856. 'url': TEST_URL,
  857. 'title': 'Title',
  858. 'ext': 'mp4',
  859. 'timestamp': 1631352900,
  860. 'release_timestamp': 1632995931,
  861. }
  862. params = {'simulate': True, }
  863. ydl = FakeYDL(params)
  864. out_info = ydl.process_ie_result(info)
  865. self.assertTrue(isinstance(out_info['upload_date'], compat_str))
  866. self.assertEqual(out_info['upload_date'], '20210911')
  867. self.assertTrue(isinstance(out_info['release_date'], compat_str))
  868. self.assertEqual(out_info['release_date'], '20210930')
  869. class TestYoutubeDLCookies(unittest.TestCase):
  870. @staticmethod
  871. def encode_cookie(cookie):
  872. if not isinstance(cookie, dict):
  873. cookie = vars(cookie)
  874. for name, value in cookie.items():
  875. yield name, compat_str(value)
  876. @classmethod
  877. def comparable_cookies(cls, cookies):
  878. # Work around cookiejar cookies not being unicode strings
  879. return sorted(map(tuple, map(sorted, map(cls.encode_cookie, cookies))))
  880. def assertSameCookies(self, c1, c2, msg=None):
  881. return self.assertEqual(
  882. *map(self.comparable_cookies, (c1, c2)),
  883. msg=msg)
  884. def assertSameCookieStrings(self, c1, c2, msg=None):
  885. return self.assertSameCookies(
  886. *map(lambda c: compat_http_cookies_SimpleCookie(c).values(), (c1, c2)),
  887. msg=msg)
  888. def test_header_cookies(self):
  889. ydl = FakeYDL()
  890. ydl.report_warning = lambda *_, **__: None
  891. def cookie(name, value, version=None, domain='', path='', secure=False, expires=None):
  892. return compat_http_cookiejar_Cookie(
  893. version or 0, name, value, None, False,
  894. domain, bool(domain), bool(domain), path, bool(path),
  895. secure, expires, False, None, None, rest={})
  896. test_url, test_domain = (t % ('yt.dl',) for t in ('https://%s/test', '.%s'))
  897. def test(encoded_cookies, cookies, headers=False, round_trip=None, error_re=None):
  898. def _test():
  899. ydl.cookiejar.clear()
  900. ydl._load_cookies(encoded_cookies, autoscope=headers)
  901. if headers:
  902. ydl._apply_header_cookies(test_url)
  903. data = {'url': test_url}
  904. ydl._calc_headers(data)
  905. self.assertSameCookies(
  906. cookies, ydl.cookiejar,
  907. 'Extracted cookiejar.Cookie is not the same')
  908. if not headers:
  909. self.assertSameCookieStrings(
  910. data.get('cookies'), round_trip or encoded_cookies,
  911. msg='Cookie is not the same as round trip')
  912. ydl.__dict__['_YoutubeDL__header_cookies'] = []
  913. try:
  914. _test()
  915. except AssertionError:
  916. raise
  917. except Exception as e:
  918. if not error_re:
  919. raise
  920. assertRegexpMatches(self, e.args[0], error_re.join(('.*',) * 2))
  921. test('test=value; Domain=' + test_domain, [cookie('test', 'value', domain=test_domain)])
  922. test('test=value', [cookie('test', 'value')], error_re='Unscoped cookies are not allowed')
  923. test('cookie1=value1; Domain={0}; Path=/test; cookie2=value2; Domain={0}; Path=/'.format(test_domain), [
  924. cookie('cookie1', 'value1', domain=test_domain, path='/test'),
  925. cookie('cookie2', 'value2', domain=test_domain, path='/')])
  926. cookie_kw = compat_kwargs(
  927. {'domain': test_domain, 'path': '/test', 'secure': True, 'expires': '9999999999', })
  928. test('test=value; Domain={domain}; Path={path}; Secure; Expires={expires}'.format(**cookie_kw), [
  929. cookie('test', 'value', **cookie_kw)])
  930. test('test="value; "; path=/test; domain=' + test_domain, [
  931. cookie('test', 'value; ', domain=test_domain, path='/test')],
  932. round_trip='test="value\\073 "; Domain={0}; Path=/test'.format(test_domain))
  933. test('name=; Domain=' + test_domain, [cookie('name', '', domain=test_domain)],
  934. round_trip='name=""; Domain=' + test_domain)
  935. test('test=value', [cookie('test', 'value', domain=test_domain)], headers=True)
  936. test('cookie1=value; Domain={0}; cookie2=value'.format(test_domain), [],
  937. headers=True, error_re='Invalid syntax')
  938. ydl.report_warning = ydl.report_error
  939. test('test=value', [], headers=True, error_re='Passing cookies as a header is a potential security risk')
  940. def test_infojson_cookies(self):
  941. TEST_FILE = 'test_infojson_cookies.info.json'
  942. TEST_URL = 'https://example.com/example.mp4'
  943. COOKIES = 'a=b; Domain=.example.com; c=d; Domain=.example.com'
  944. COOKIE_HEADER = {'Cookie': 'a=b; c=d'}
  945. ydl = FakeYDL()
  946. ydl.process_info = lambda x: ydl._write_info_json('test', x, TEST_FILE)
  947. def make_info(info_header_cookies=False, fmts_header_cookies=False, cookies_field=False):
  948. fmt = {'url': TEST_URL}
  949. if fmts_header_cookies:
  950. fmt['http_headers'] = COOKIE_HEADER
  951. if cookies_field:
  952. fmt['cookies'] = COOKIES
  953. return _make_result([fmt], http_headers=COOKIE_HEADER if info_header_cookies else None)
  954. def test(initial_info, note):
  955. def failure_msg(why):
  956. return ' when '.join((why, note))
  957. result = {}
  958. result['processed'] = ydl.process_ie_result(initial_info)
  959. self.assertTrue(ydl.cookiejar.get_cookies_for_url(TEST_URL),
  960. msg=failure_msg('No cookies set in cookiejar after initial process'))
  961. ydl.cookiejar.clear()
  962. with open(TEST_FILE) as infojson:
  963. result['loaded'] = ydl.sanitize_info(json.load(infojson), True)
  964. result['final'] = ydl.process_ie_result(result['loaded'].copy(), download=False)
  965. self.assertTrue(ydl.cookiejar.get_cookies_for_url(TEST_URL),
  966. msg=failure_msg('No cookies set in cookiejar after final process'))
  967. ydl.cookiejar.clear()
  968. for key in ('processed', 'loaded', 'final'):
  969. info = result[key]
  970. self.assertIsNone(
  971. traverse_obj(info, ((None, ('formats', 0)), 'http_headers', 'Cookie'), casesense=False, get_all=False),
  972. msg=failure_msg('Cookie header not removed in {0} result'.format(key)))
  973. self.assertSameCookieStrings(
  974. traverse_obj(info, ((None, ('formats', 0)), 'cookies'), get_all=False), COOKIES,
  975. msg=failure_msg('No cookies field found in {0} result'.format(key)))
  976. test({'url': TEST_URL, 'http_headers': COOKIE_HEADER, 'id': '1', 'title': 'x'}, 'no formats field')
  977. test(make_info(info_header_cookies=True), 'info_dict header cokies')
  978. test(make_info(fmts_header_cookies=True), 'format header cookies')
  979. test(make_info(info_header_cookies=True, fmts_header_cookies=True), 'info_dict and format header cookies')
  980. test(make_info(info_header_cookies=True, fmts_header_cookies=True, cookies_field=True), 'all cookies fields')
  981. test(make_info(cookies_field=True), 'cookies format field')
  982. test({'url': TEST_URL, 'cookies': COOKIES, 'id': '1', 'title': 'x'}, 'info_dict cookies field only')
  983. try_rm(TEST_FILE)
  984. def test_add_headers_cookie(self):
  985. def check_for_cookie_header(result):
  986. return traverse_obj(result, ((None, ('formats', 0)), 'http_headers', 'Cookie'), casesense=False, get_all=False)
  987. ydl = FakeYDL({'http_headers': {'Cookie': 'a=b'}})
  988. ydl._apply_header_cookies(_make_result([])['webpage_url']) # Scope to input webpage URL: .example.com
  989. fmt = {'url': 'https://example.com/video.mp4'}
  990. result = ydl.process_ie_result(_make_result([fmt]), download=False)
  991. self.assertIsNone(check_for_cookie_header(result), msg='http_headers cookies in result info_dict')
  992. self.assertEqual(result.get('cookies'), 'a=b; Domain=.example.com', msg='No cookies were set in cookies field')
  993. self.assertIn('a=b', ydl.cookiejar.get_cookie_header(fmt['url']), msg='No cookies were set in cookiejar')
  994. fmt = {'url': 'https://wrong.com/video.mp4'}
  995. result = ydl.process_ie_result(_make_result([fmt]), download=False)
  996. self.assertIsNone(check_for_cookie_header(result), msg='http_headers cookies for wrong domain')
  997. self.assertFalse(result.get('cookies'), msg='Cookies set in cookies field for wrong domain')
  998. self.assertFalse(ydl.cookiejar.get_cookie_header(fmt['url']), msg='Cookies set in cookiejar for wrong domain')
  999. if __name__ == '__main__':
  1000. unittest.main()