release 2013.06.21

Also extracts the description.

Makefile

@@ -9,9 +9,19 @@ cleanall: clean
 PREFIX=/usr/local
 BINDIR=$(PREFIX)/bin
 MANDIR=$(PREFIX)/man
-SYSCONFDIR=/etc
 PYTHON=/usr/bin/env python
 
+# set SYSCONFDIR to /etc if PREFIX=/usr or PREFIX=/usr/local
+ifeq ($(PREFIX),/usr)
+    SYSCONFDIR=/etc
+else
+    ifeq ($(PREFIX),/usr/local)
+        SYSCONFDIR=/etc
+    else
+        SYSCONFDIR=$(PREFIX)/etc
+    endif
+endif
+
 install: youtube-dl youtube-dl.1 youtube-dl.bash-completion
 	install -d $(DESTDIR)$(BINDIR)
 	install -m 755 youtube-dl $(DESTDIR)$(BINDIR)

README.md

@@ -94,6 +94,7 @@ which means you can modify it, redistribute it or use it however you like.
     --skip-download            do not download the video
     -g, --get-url              simulate, quiet but print URL
     -e, --get-title            simulate, quiet but print title
+    --get-id                   simulate, quiet but print id
     --get-thumbnail            simulate, quiet but print thumbnail URL
     --get-description          simulate, quiet but print video description
     --get-filename             simulate, quiet but print output filename
@@ -115,7 +116,7 @@ which means you can modify it, redistribute it or use it however you like.
     -F, --list-formats         list all available formats (currently youtube
                                only)
     --write-sub                write subtitle file (currently youtube only)
-    --only-sub                 downloads only the subtitles (no video)
+    --only-sub                 [deprecated] alias of --skip-download
     --all-subs                 downloads all the available subtitles of the
                                video (currently youtube only)
     --list-subs                lists all available subtitles for the video

devscripts/release.sh

@@ -22,7 +22,7 @@ if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit
 
 /bin/echo -e "\n### First of all, testing..."
 make cleanall
-nosetests --with-coverage --cover-package=youtube_dl --cover-html test --stop || exit 1
+nosetests --verbose --with-coverage --cover-package=youtube_dl --cover-html test --stop || exit 1
 
 /bin/echo -e "\n### Changing version in version.py..."
 sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py

test/test_all_urls.py

@@ -7,7 +7,7 @@ import unittest
 import os
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from youtube_dl.InfoExtractors import YoutubeIE, YoutubePlaylistIE, YoutubeChannelIE
+from youtube_dl.InfoExtractors import YoutubeIE, YoutubePlaylistIE, YoutubeChannelIE, JustinTVIE
 
 class TestAllURLsMatching(unittest.TestCase):
     def test_youtube_playlist_matching(self):
@@ -29,6 +29,22 @@ class TestAllURLsMatching(unittest.TestCase):
         self.assertTrue(YoutubeChannelIE.suitable('https://www.youtube.com/channel/HCtnHdj3df7iM?feature=gb_ch_rec'))
         self.assertTrue(YoutubeChannelIE.suitable('https://www.youtube.com/channel/HCtnHdj3df7iM/videos'))
 
+    def test_justin_tv_channelid_matching(self):
+        self.assertTrue(JustinTVIE.suitable(u"justin.tv/vanillatv"))
+        self.assertTrue(JustinTVIE.suitable(u"twitch.tv/vanillatv"))
+        self.assertTrue(JustinTVIE.suitable(u"www.justin.tv/vanillatv"))
+        self.assertTrue(JustinTVIE.suitable(u"www.twitch.tv/vanillatv"))
+        self.assertTrue(JustinTVIE.suitable(u"http://www.justin.tv/vanillatv"))
+        self.assertTrue(JustinTVIE.suitable(u"http://www.twitch.tv/vanillatv"))
+        self.assertTrue(JustinTVIE.suitable(u"http://www.justin.tv/vanillatv/"))
+        self.assertTrue(JustinTVIE.suitable(u"http://www.twitch.tv/vanillatv/"))
+
+    def test_justintv_videoid_matching(self):
+        self.assertTrue(JustinTVIE.suitable(u"http://www.twitch.tv/vanillatv/b/328087483"))
+
+    def test_justin_tv_chapterid_matching(self):
+        self.assertTrue(JustinTVIE.suitable(u"http://www.twitch.tv/tsm_theoddone/c/2349361"))
+
     def test_youtube_extract(self):
         self.assertEqual(YoutubeIE()._extract_id('http://www.youtube.com/watch?&v=BaW_jenozKc'), 'BaW_jenozKc')
         self.assertEqual(YoutubeIE()._extract_id('https://www.youtube.com/watch?&v=BaW_jenozKc'), 'BaW_jenozKc')

test/test_download.py

@@ -7,8 +7,8 @@ import os
 import json
 import unittest
 import sys
-import hashlib
 import socket
+import binascii
 
 # Allow direct execution
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -38,11 +38,16 @@ def _try_rm(filename):
         if ose.errno != errno.ENOENT:
             raise
 
+md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
+
 class FileDownloader(youtube_dl.FileDownloader):
     def __init__(self, *args, **kwargs):
         self.to_stderr = self.to_screen
         self.processed_info_dicts = []
         return youtube_dl.FileDownloader.__init__(self, *args, **kwargs)
+    def report_warning(self, message):
+        # Don't accept warnings during tests
+        raise ExtractorError(message)
     def process_info(self, info_dict):
         self.processed_info_dicts.append(info_dict)
         return youtube_dl.FileDownloader.process_info(self, info_dict)
@@ -121,7 +126,21 @@ def generator(test_case):
                 with io.open(tc['file'] + '.info.json', encoding='utf-8') as infof:
                     info_dict = json.load(infof)
                 for (info_field, value) in tc.get('info_dict', {}).items():
-                    self.assertEqual(value, info_dict.get(info_field))
+                    if isinstance(value, compat_str) and value.startswith('md5:'):
+                        self.assertEqual(value, 'md5:' + md5(info_dict.get(info_field)))
+                    else:
+                        self.assertEqual(value, info_dict.get(info_field))
+
+                # If checkable fields are missing from the test case, print the info_dict
+                test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value))
+                    for key, value in info_dict.items()
+                    if value and key in ('title', 'description', 'uploader', 'upload_date', 'uploader_id', 'location'))
+                if not all(key in tc.get('info_dict', {}).keys() for key in test_info_dict.keys()):
+                    sys.stderr.write(u'\n"info_dict": ' + json.dumps(test_info_dict, ensure_ascii=False, indent=2) + u'\n')
+
+                # Check for the presence of mandatory fields
+                for key in ('id', 'url', 'title', 'ext'):
+                    self.assertTrue(key in info_dict.keys() and info_dict[key])
         finally:
             for tc in test_cases:
                 _try_rm(tc['file'])

test/test_youtube_lists.py

@@ -53,8 +53,7 @@ class TestYoutubeLists(unittest.TestCase):
         dl = FakeDownloader()
         ie = YoutubePlaylistIE(dl)
         result = ie.extract('PLBB231211A4F62143')[0]
-        self.assertEqual(result['title'], 'Team Fortress 2')
-        self.assertTrue(len(result['entries']) > 40)
+        self.assertTrue(len(result['entries']) > 25)
 
     def test_youtube_playlist_long(self):
         dl = FakeDownloader()
@@ -105,5 +104,11 @@ class TestYoutubeLists(unittest.TestCase):
         result = ie.extract('https://www.youtube.com/user/TheLinuxFoundation')[0]
         self.assertTrue(len(result['entries']) >= 320)
 
+    def test_youtube_safe_search(self):
+        dl = FakeDownloader()
+        ie = YoutubePlaylistIE(dl)
+        result = ie.extract('PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl')[0]
+        self.assertEqual(len(result['entries']), 2)
+
 if __name__ == '__main__':
     unittest.main()

test/test_youtube_subtitles.py

@@ -12,6 +12,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from youtube_dl.InfoExtractors import YoutubeIE
 from youtube_dl.utils import *
+from youtube_dl import FileDownloader
 
 PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
 with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
@@ -24,10 +25,12 @@ proxy_handler = compat_urllib_request.ProxyHandler()
 opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
 compat_urllib_request.install_opener(opener)
 
-class FakeDownloader(object):
+class FakeDownloader(FileDownloader):
     def __init__(self):
         self.result = []
-        self.params = parameters
+        # Different instances of the downloader can't share the same dictionary
+        # some test set the "sublang" parameter, which would break the md5 checks.
+        self.params = dict(parameters)
     def to_screen(self, s):
         print(s)
     def trouble(self, s, tb=None):
@@ -95,6 +98,14 @@ class TestYoutubeSubtitles(unittest.TestCase):
         IE = YoutubeIE(DL)
         info_dict = IE.extract('QRS8MkLhQmM')
         self.assertEqual(info_dict, None)
+    def test_youtube_automatic_captions(self):
+        DL = FakeDownloader()
+        DL.params['writesubtitles'] = True
+        DL.params['subtitleslang'] = 'it'
+        IE = YoutubeIE(DL)
+        info_dict = IE.extract('8YoUxe5ncPo')
+        sub = info_dict[0]['subtitles'][0]
+        self.assertTrue(sub[2] is not None)
 
 if __name__ == '__main__':
     unittest.main()

test/tests.json

@@ -15,43 +15,76 @@
     "name": "Dailymotion",
     "md5":  "392c4b85a60a90dc4792da41ce3144eb",
     "url":  "http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech",
-    "file":  "x33vw9.mp4"
+    "file":  "x33vw9.mp4",
+    "info_dict": {
+      "uploader": "Alex and Van .",
+      "title": "Tutoriel de Youtubeur\"DL DES VIDEO DE YOUTUBE\""
+    }
   },
   {
     "name": "Metacafe",
     "add_ie": ["Youtube"],
     "url":  "http://metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/",
-    "file":  "_aUehQsCQtM.flv"
+    "file":  "_aUehQsCQtM.flv",
+    "info_dict": {
+      "upload_date": "20090102",
+      "title": "The Electric Company | \"Short I\" | PBS KIDS GO!",
+      "description": "md5:2439a8ef6d5a70e380c22f5ad323e5a8",
+      "uploader": "PBS",
+      "uploader_id": "PBS"
+    }
   },
   {
     "name": "BlipTV",
     "md5":  "b2d849efcf7ee18917e4b4d9ff37cafe",
     "url":  "http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352",
-    "file":  "5779306.m4v"
+    "file":  "5779306.m4v",
+    "info_dict": {
+      "upload_date": "20111205",
+      "description": "md5:9bc31f227219cde65e47eeec8d2dc596",
+      "uploader": "Comic Book Resources - CBR TV",
+      "title": "CBR EXCLUSIVE: \"Gotham City Imposters\" Bats VS Jokerz Short 3"
+    }
   },
   {
     "name": "XVideos",
     "md5":  "1d0c835822f0a71a7bf011855db929d0",
     "url":  "http://www.xvideos.com/video939581/funny_porns_by_s_-1",
-    "file":  "939581.flv"
+    "file":  "939581.flv",
+    "info_dict": {
+      "title": "Funny Porns By >>>>S<<<<<< -1"
+    }
   },
   {
     "name": "YouPorn",
     "md5": "c37ddbaaa39058c76a7e86c6813423c1",
     "url": "http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/",
-    "file": "505835.mp4"
+    "file": "505835.mp4",
+    "info_dict": {
+      "upload_date": "20101221",
+      "description": "Love & Sex Answers: http://bit.ly/DanAndJenn -- Is It Unhealthy To Masturbate Daily?",
+      "uploader": "Ask Dan And Jennifer",
+      "title": "Sex Ed: Is It Safe To Masturbate Daily?"
+    }
   },
   {
     "name": "Pornotube",
     "md5": "374dd6dcedd24234453b295209aa69b6",
     "url": "http://pornotube.com/c/173/m/1689755/Marilyn-Monroe-Bathing",
-    "file": "1689755.flv"
+    "file": "1689755.flv",
+    "info_dict": {
+      "upload_date": "20090708",
+      "title": "Marilyn-Monroe-Bathing"
+    }
   },
   {
     "name": "YouJizz",
     "md5": "07e15fa469ba384c7693fd246905547c",
     "url": "http://www.youjizz.com/videos/zeichentrick-1-2189178.html",
-    "file": "2189178.flv"
+    "file": "2189178.flv",
+    "info_dict": {
+      "title": "Zeichentrick 1"
+    }
   },
   {
     "name": "Vimeo",
@@ -70,61 +103,103 @@
     "name": "Soundcloud",
     "md5":  "ebef0a451b909710ed1d7787dddbf0d7",
     "url":  "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy",
-    "file":  "62986583.mp3"
+    "file":  "62986583.mp3",
+    "info_dict": {
+      "upload_date": "20121011",
+      "description": "No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o'd",
+      "uploader": "E.T. ExTerrestrial Music",
+      "title": "Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1"
+    }
   },
   {
     "name": "StanfordOpenClassroom",
     "md5":  "544a9468546059d4e80d76265b0443b8",
     "url":  "http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100",
-    "file":  "PracticalUnix_intro-environment.mp4"
+    "file":  "PracticalUnix_intro-environment.mp4",
+    "info_dict": {
+      "title": "Intro Environment"
+    }
   },
   {
     "name": "XNXX",
     "md5":  "0831677e2b4761795f68d417e0b7b445",
     "url":  "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_",
-    "file":  "1135332.flv"
+    "file":  "1135332.flv",
+    "info_dict": {
+      "title": "lida » Naked Funny Actress  (5)"
+    }
   },
   {
     "name": "Youku",
     "url": "http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html",
     "file": "XNDgyMDQ2NTQw_part00.flv",
     "md5": "ffe3f2e435663dc2d1eea34faeff5b5b",
-    "params": { "test": false }
+    "params": { "test": false },
+    "info_dict": {
+      "title": "youtube-dl test video \"'/\\ä↭𝕐"
+    }
   },
   {
     "name": "NBA",
     "url": "http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html",
     "file": "0021200253-okc-bkn-recap.nba.mp4",
-    "md5": "c0edcfc37607344e2ff8f13c378c88a4"
+    "md5": "c0edcfc37607344e2ff8f13c378c88a4",
+    "info_dict": {
+      "description": "Kevin Durant scores 32 points and dishes out six assists as the Thunder beat the Nets in Brooklyn.",
+      "title": "Thunder vs. Nets"
+    }
   },
   {
     "name": "JustinTV",
     "url": "http://www.twitch.tv/thegamedevhub/b/296128360",
     "file": "296128360.flv",
-    "md5": "ecaa8a790c22a40770901460af191c9a"
+    "md5": "ecaa8a790c22a40770901460af191c9a",
+    "info_dict": {
+      "upload_date": "20110927",
+      "uploader_id": 25114803,
+      "uploader": "thegamedevhub",
+      "title": "Beginner Series - Scripting With Python Pt.1"
+    }
   },
   {
     "name": "MyVideo",
     "url": "http://www.myvideo.de/watch/8229274/bowling_fail_or_win",
     "file": "8229274.flv",
-    "md5": "2d2753e8130479ba2cb7e0a37002053e"
+    "md5": "2d2753e8130479ba2cb7e0a37002053e",
+    "info_dict": {
+      "title": "bowling-fail-or-win"
+    }
   },
   {
     "name": "Escapist",
     "url": "http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate",
     "file": "6618-Breaking-Down-Baldurs-Gate.mp4",
-    "md5": "c6793dbda81388f4264c1ba18684a74d"
+    "md5": "c6793dbda81388f4264c1ba18684a74d",
+    "info_dict": {
+      "description": "Baldur's Gate: Original, Modded or Enhanced Edition? I'll break down what you can expect from the new Baldur's Gate: Enhanced Edition.",
+      "uploader": "the-escapist-presents",
+      "title": "Breaking Down Baldur's Gate"
+    }
   },
   {
     "name": "GooglePlus",
     "url": "https://plus.google.com/u/0/108897254135232129896/posts/ZButuJc6CtH",
-    "file": "ZButuJc6CtH.flv"
+    "file": "ZButuJc6CtH.flv",
+    "info_dict": {
+      "upload_date": "20120613",
+      "uploader": "井上ヨシマサ",
+      "title": "嘆きの天使 降臨"
+    }
   },
   {
     "name": "FunnyOrDie",
     "url": "http://www.funnyordie.com/videos/0732f586d7/heart-shaped-box-literal-video-version",
     "file": "0732f586d7.mp4",
-    "md5": "f647e9e90064b53b6e046e75d0241fbd"
+    "md5": "f647e9e90064b53b6e046e75d0241fbd",
+    "info_dict": {
+      "description": "Lyrics changed to match the video. Spoken cameo by Obscurus Lupa (from ThatGuyWithTheGlasses.com). Based on a concept by Dustin McLean (DustFilms.com). Performed, edited, and written by David A. Scott.",
+      "title": "Heart-Shaped Box: Literal Video Version"
+    }
   },
   {
     "name": "Steam",
@@ -152,7 +227,8 @@
     "file": "20274954.flv",
     "md5": "088f151799e8f572f84eb62f17d73e5c",
     "info_dict": {
-        "title": "Young Americans for Liberty February 7, 2012 2:28 AM"
+        "title": "Young Americans for Liberty February 7, 2012 2:28 AM",
+        "uploader": "Young Americans for Liberty"
     }
   },
   {
@@ -160,6 +236,7 @@
     "url": "http://www.infoq.com/presentations/A-Few-of-My-Favorite-Python-Things",
     "file": "12-jan-pythonthings.mp4",
     "info_dict": {
+      "description": "Mike Pirnat presents some tips and tricks, standard libraries and third party packages that make programming in Python a richer experience.",
       "title": "A Few of My Favorite [Python] Things"
     },
     "params": {
@@ -172,7 +249,10 @@
     "file": "422212.mp4",
     "md5": "4e2f5cb088a83cd8cdb7756132f9739d",
     "info_dict": {
-        "title": "thedailyshow-kristen-stewart part 1"
+      "upload_date": "20121214",
+      "description": "Kristen Stewart",
+      "uploader": "thedailyshow",
+      "title": "thedailyshow-kristen-stewart part 1"
     }
   },
   {
@@ -223,42 +303,48 @@
         "file": "11885679.m4a",
         "md5": "d30b5b5f74217410f4689605c35d1fd7",
         "info_dict": {
-          "title": "youtube-dl project as well - youtube-dl test track 3 \"'/\\\u00e4\u21ad"
+          "title": "youtube-dl project as well - youtube-dl test track 3 \"'/\\\u00e4\u21ad",
+          "uploader_id": "ytdl"
         }
       },
       {
         "file": "11885680.m4a",
         "md5": "4eb0a669317cd725f6bbd336a29f923a",
         "info_dict": {
-          "title": "youtube-dl project as well - youtube-dl test track 4 \"'/\\\u00e4\u21ad"
+          "title": "youtube-dl project as well - youtube-dl test track 4 \"'/\\\u00e4\u21ad",
+          "uploader_id": "ytdl"
         }
       },
       {
         "file": "11885682.m4a",
         "md5": "1893e872e263a2705558d1d319ad19e8",
         "info_dict": {
-          "title": "PH - youtube-dl test track 5 \"'/\\\u00e4\u21ad"
+          "title": "PH - youtube-dl test track 5 \"'/\\\u00e4\u21ad",
+          "uploader_id": "ytdl"
         }
       },
       {
         "file": "11885683.m4a",
         "md5": "b673c46f47a216ab1741ae8836af5899",
         "info_dict": {
-          "title": "PH - youtube-dl test track 6 \"'/\\\u00e4\u21ad"
+          "title": "PH - youtube-dl test track 6 \"'/\\\u00e4\u21ad",
+          "uploader_id": "ytdl"
         }
       },
       {
         "file": "11885684.m4a",
         "md5": "1d74534e95df54986da7f5abf7d842b7",
         "info_dict": {
-          "title": "phihag - youtube-dl test track 7 \"'/\\\u00e4\u21ad"
+          "title": "phihag - youtube-dl test track 7 \"'/\\\u00e4\u21ad",
+          "uploader_id": "ytdl"
         }
       },
       {
         "file": "11885685.m4a",
         "md5": "f081f47af8f6ae782ed131d38b9cd1c0",
         "info_dict": {
-          "title": "phihag - youtube-dl test track 8 \"'/\\\u00e4\u21ad"
+          "title": "phihag - youtube-dl test track 8 \"'/\\\u00e4\u21ad",
+          "uploader_id": "ytdl"
         }
       }
     ]
@@ -269,18 +355,18 @@
     "file": "NODfbab.mp4",
     "md5": "9b0636f8c0f7614afa4ea5e4c6e57e83",
     "info_dict": {
+      "uploader": "ytdl",
       "title": "test chars: \"'/\\ä<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de ."
     }
-
   },
   {
     "name": "TED",
     "url": "http://www.ted.com/talks/dan_dennett_on_our_consciousness.html",
     "file": "102.mp4",
-    "md5": "7bc087e71d16f18f9b8ab9fa62a8a031",
+    "md5": "8cd9dfa41ee000ce658fd48fb5d89a61",
     "info_dict": {
         "title": "Dan Dennett: The illusion of consciousness",
-        "thumbnail": "http://images.ted.com/images/ted/488_389x292.jpg"
+        "description": "md5:c6fa72e6eedbd938c9caf6b2702f5922"
     }
   },
   {
@@ -289,14 +375,19 @@
     "file": "11741.mp4",
     "md5": "0b49f4844a068f8b33f4b7c88405862b",
     "info_dict": {
-        "title": "Absolute Mehrheit vom 17.02.2013 - Die Highlights, Teil 2"
+      "description": "Wer kann in die Fußstapfen von Wolfgang Kubicki treten und die Mehrheit der Zuschauer hinter sich versammeln? Wird vielleicht sogar die Absolute Mehrheit geknackt und der Jackpot von 200.000 Euro mit nach Hause genommen?",
+      "title": "Absolute Mehrheit vom 17.02.2013 - Die Highlights, Teil 2"
     }
   },
   {
     "name": "Generic",
     "url": "http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html",
     "file": "13601338388002.mp4",
-    "md5": "85b90ccc9d73b4acd9138d3af4c27f89"
+    "md5": "85b90ccc9d73b4acd9138d3af4c27f89",
+    "info_dict": {
+      "uploader": "www.hodiho.fr",
+      "title": "Régis plante sa Jeep"
+    }
   },
   {
     "name": "Spiegel",
@@ -324,7 +415,7 @@
     "file": "wshh6a7q1ny0G34ZwuIO.mp4",
     "md5": "9d04de741161603bf7071bbf4e883186",
     "info_dict": {
-        "title": "Video: KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick! "
+        "title": "Video: KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!"
     }
   },
   {
@@ -339,11 +430,11 @@
   },
   {
     "name": "Tumblr",
-    "url": "http://birthdayproject2012.tumblr.com/post/17258355236/a-sample-video-from-leeann-if-you-need-an-idea",
-    "file": "17258355236.mp4",
-    "md5": "7c6a514d691b034ccf8567999e9e88a3",
+    "url": "http://resigno.tumblr.com/post/53364321212/e-de-extrema-importancia-que-esse-video-seja",
+    "file": "53364321212.mp4",
+    "md5": "0716d3dd51baf68a28b40fdf1251494e",
     "info_dict": {
-        "title": "Calling all Pris! - A sample video from LeeAnn. (If you need an idea..."
+        "title": "Rafael Lemos | Tumblr"
     }
   },
   {
@@ -354,42 +445,59 @@
         "file":"30510138.mp3",
         "md5":"f9136bf103901728f29e419d2c70f55d",
         "info_dict": {
-          "title":"D-D-Dance"
+          "upload_date": "20111213",
+          "description": "The Royal Concept from Stockholm\r\nFilip / Povel / David / Magnus\r\nwww.royalconceptband.com",
+          "uploader": "The Royal Concept",
+          "title": "D-D-Dance"
         }
       },
       {
         "file":"47127625.mp3",
         "md5":"09b6758a018470570f8fd423c9453dd8",
         "info_dict": {
-          "title":"The Royal Concept - Gimme Twice"
+          "upload_date": "20120521",
+          "description": "The Royal Concept from Stockholm\r\nFilip / Povel / David / Magnus\r\nwww.royalconceptband.com",
+          "uploader": "The Royal Concept",
+          "title": "The Royal Concept - Gimme Twice"
         }
       },
       {
         "file":"47127627.mp3",
         "md5":"154abd4e418cea19c3b901f1e1306d9c",
         "info_dict": {
-          "title":"Goldrushed"
+          "upload_date": "20120521",
+          "uploader": "The Royal Concept",
+          "title": "Goldrushed"
         }
       },
       {
         "file":"47127629.mp3",
         "md5":"2f5471edc79ad3f33a683153e96a79c1",
         "info_dict": {
-          "title":"In the End"
+          "upload_date": "20120521",
+          "description": "The Royal Concept from Stockholm\r\nFilip / Povel / David / Magnus\r\nwww.royalconceptband.com",
+          "uploader": "The Royal Concept",
+          "title": "In the End"
         }
       },
       {
         "file":"47127631.mp3",
         "md5":"f9ba87aa940af7213f98949254f1c6e2",
         "info_dict": {
-          "title":"Knocked Up"
+          "upload_date": "20120521",
+          "description": "The Royal Concept from Stockholm\r\nFilip / David / Povel / Magnus\r\nwww.theroyalconceptband.com",
+          "uploader": "The Royal Concept",
+          "title": "Knocked Up"
         }
       },
       {
         "file":"75206121.mp3",
         "md5":"f9d1fe9406717e302980c30de4af9353",
         "info_dict": {
-          "title":"World On Fire"
+          "upload_date": "20130116",
+          "description": "The unreleased track World on Fire premiered on the CW's hit show Arrow (8pm/7pm central).  \r\nAs a gift to our fans we would like to offer you a free download of the track!  ",
+          "uploader": "The Royal Concept",
+          "title": "World On Fire"
         }
       }
     ]
@@ -401,7 +509,8 @@
     "md5":"cdeb30cdae1921719a3cbcab696ef53c",
     "info_dict": {
       "title":"youtube-dl test song \"'/\\ä↭"
-    }
+    },
+    "skip": "There is a limit of 200 free downloads / month for the test song"
   },
   {
     "name": "RedTube",
@@ -417,8 +526,10 @@
     "url": "http://media.photobucket.com/user/rachaneronas/media/TiredofLinkBuildingTryBacklinkMyDomaincom_zpsc0c3b9fa.mp4.html?filters[term]=search&filters[primary]=videos&filters[secondary]=images&sort=1&o=0",
     "file": "zpsc0c3b9fa.mp4",
     "md5": "7dabfb92b0a31f6c16cebc0f8e60ff99",
-    "info_dict":{
-      "title":"Tired of Link Building? Try BacklinkMyDomain.com!"
+    "info_dict": {
+      "upload_date": "20130504",
+      "uploader": "rachaneronas",
+      "title": "Tired of Link Building? Try BacklinkMyDomain.com!"
     }
   },
   {
@@ -429,5 +540,95 @@
     "info_dict":{
       "title":"François Hollande \"Je crois que c'est clair\""
     }
+  },
+  {
+    "name": "Yahoo",
+    "url": "http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html",
+    "file": "214727115.flv",
+    "md5": "2e717f169c1be93d84d3794a00d4a325",
+    "info_dict": {
+        "title": "Julian Smith & Travis Legg Watch Julian Smith"
+    },
+    "skip": "Requires rtmpdump"
+  },
+  {
+    "name": "Howcast",
+    "url": "http://www.howcast.com/videos/390161-How-to-Tie-a-Square-Knot-Properly",
+    "file": "390161.mp4",
+    "md5": "1d7ba54e2c9d7dc6935ef39e00529138",
+    "info_dict":{
+      "title":"How to Tie a Square Knot Properly",
+      "description":"The square knot, also known as the reef knot, is one of the oldest, most basic knots to tie, and can be used in many different ways. Here's the proper way to tie a square knot."
+    }
+  },
+  {
+    "name": "Vine",
+    "url": "https://vine.co/v/b9KOOWX7HUx",
+    "file": "b9KOOWX7HUx.mp4",
+    "md5": "2f36fed6235b16da96ce9b4dc890940d",
+    "info_dict":{
+      "title": "Chicken.",
+      "uploader": "Jack Dorsey"
+    }
+  },
+  {
+    "name": "Flickr",
+    "url": "http://www.flickr.com/photos/forestwander-nature-pictures/5645318632/in/photostream/",
+    "file": "5645318632.mp4",
+    "md5": "6fdc01adbc89d72fc9c4f15b4a4ba87b",
+    "info_dict":{
+      "title": "Dark Hollow Waterfalls",
+      "uploader_id": "forestwander-nature-pictures",
+      "description": "Waterfalls in the Springtime at Dark Hollow Waterfalls. These are located just off of Skyline Drive in Virginia. They are only about 6/10 of a mile hike but it is a pretty steep hill and a good climb back up."
+    }
+  },
+  {
+    "name": "Teamcoco",
+    "url": "http://teamcoco.com/video/louis-ck-interview-george-w-bush",
+    "file": "19705.mp4",
+    "md5": "27b6f7527da5acf534b15f21b032656e",
+    "info_dict":{
+      "title": "Louis C.K. Interview Pt. 1 11/3/11",
+      "description": "Louis C.K. got starstruck by George W. Bush, so what? Part one."
+    }
+  },
+  {
+    "name": "XHamster",
+    "url": "http://xhamster.com/movies/1509445/femaleagent_shy_beauty_takes_the_bait.html",
+    "file": "1509445.flv",
+    "md5": "9f48e0e8d58e3076bb236ff412ab62fa",
+    "info_dict": {
+      "upload_date": "20121014",
+      "uploader_id": "Ruseful2011",
+      "title": "FemaleAgent Shy beauty takes the bait"
+    }
+  },
+  {
+    "name": "Hypem",
+    "url": "http://hypem.com/track/1v6ga/BODYWORK+-+TAME",
+    "file": "1v6ga.mp3",
+    "md5": "b9cc91b5af8995e9f0c1cee04c575828",
+    "info_dict":{
+      "title":"Tame"
+    }
+  },
+  {
+    "name": "Vbox7",
+    "url": "http://vbox7.com/play:249bb972c2",
+    "file": "249bb972c2.flv",
+    "md5": "9c70d6d956f888bdc08c124acc120cfe",
+    "info_dict":{
+      "title":"Смях! Чудо - чист за секунди - Скрита камера"
+    }
+  },
+  {
+    "name": "Gametrailers",
+    "url": "http://www.gametrailers.com/videos/zbvr8i/mirror-s-edge-2-e3-2013--debut-trailer",
+    "file": "zbvr8i.flv",
+    "md5": "c3edbc995ab4081976e16779bd96a878",
+    "info_dict": {
+        "title": "E3 2013: Debut Trailer"
+    },
+    "skip": "Requires rtmpdump"
   }
 ]

youtube_dl/FileDownloader.py

@@ -54,6 +54,7 @@ class FileDownloader(object):
     quiet:             Do not print messages to stdout.
     forceurl:          Force printing final URL.
     forcetitle:        Force printing title.
+    forceid:           Force printing ID.
     forcethumbnail:    Force printing thumbnail URL.
     forcedescription:  Force printing description.
     forcefilename:     Force printing final filename.
@@ -82,7 +83,6 @@ class FileDownloader(object):
     writeinfojson:     Write the video description to a .info.json file
     writethumbnail:    Write the thumbnail image to a file
     writesubtitles:    Write the video subtitles to a file
-    onlysubtitles:     Downloads only the subtitles of the video
     allsubtitles:      Downloads all the subtitles of the video
     listsubtitles:     Lists all available subtitles for the video
     subtitlesformat:   Subtitle format [sbv/srt] (default=srt)
@@ -92,6 +92,7 @@ class FileDownloader(object):
     min_filesize:      Skip files smaller than this size
     max_filesize:      Skip files larger than this size
     daterange:         A DateRange object, download only if the upload_date is in the range.
+    skip_download:     Skip the actual download of the video file
     """
 
     params = None
@@ -321,6 +322,9 @@ class FileDownloader(object):
         filetime = timeconvert(timestr)
         if filetime is None:
             return filetime
+        # Ignore obviously invalid dates
+        if filetime == 0:
+            return
         try:
             os.utime(filename, (time.time(), filetime))
         except:
@@ -435,10 +439,11 @@ class FileDownloader(object):
                 return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
         return None
         
-    def extract_info(self, url, download=True, ie_key=None):
+    def extract_info(self, url, download=True, ie_key=None, extra_info={}):
         '''
         Returns a list with a dictionary for each video we find.
         If 'download', also downloads the videos.
+        extra_info is a dict containing the extra values to add to each result
          '''
         
         if ie_key:
@@ -462,10 +467,14 @@ class FileDownloader(object):
                     break
                 if isinstance(ie_result, list):
                     # Backwards compatibility: old IE result format
+                    for result in ie_result:
+                        result.update(extra_info)
                     ie_result = {
                         '_type': 'compat_list',
                         'entries': ie_result,
                     }
+                else:
+                    ie_result.update(extra_info)
                 if 'extractor' not in ie_result:
                     ie_result['extractor'] = ie.IE_NAME
                 return self.process_ie_result(ie_result, download=download)
@@ -481,7 +490,7 @@ class FileDownloader(object):
         else:
             self.report_error(u'no suitable InfoExtractor: %s' % url)
         
-    def process_ie_result(self, ie_result, download=True):
+    def process_ie_result(self, ie_result, download=True, extra_info={}):
         """
         Take the result of the ie(may be modified) and resolve all unresolved
         references (URLs, playlist items).
@@ -500,7 +509,12 @@ class FileDownloader(object):
                 self.process_info(ie_result)
             return ie_result
         elif result_type == 'url':
-            return self.extract_info(ie_result['url'], download, ie_key=ie_result.get('ie_key'))
+            # We have to add extra_info to the results because it may be
+            # contained in a playlist
+            return self.extract_info(ie_result['url'],
+                                     download,
+                                     ie_key=ie_result.get('ie_key'),
+                                     extra_info=extra_info)
         elif result_type == 'playlist':
             # We process each entry in the playlist
             playlist = ie_result.get('title', None) or ie_result.get('id', None)
@@ -524,9 +538,18 @@ class FileDownloader(object):
 
             for i,entry in enumerate(entries,1):
                 self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries))
-                entry['playlist'] = playlist
-                entry['playlist_index'] = i + playliststart
-                entry_result = self.process_ie_result(entry, download=download)
+                extra = {
+                         'playlist': playlist, 
+                         'playlist_index': i + playliststart,
+                         }
+                if not 'extractor' in entry:
+                    # We set the extractor, if it's an url it will be set then to
+                    # the new extractor, but if it's already a video we must make
+                    # sure it's present: see issue #877
+                    entry['extractor'] = ie_result['extractor']
+                entry_result = self.process_ie_result(entry,
+                                                      download=download,
+                                                      extra_info=extra)
                 playlist_results.append(entry_result)
             ie_result['entries'] = playlist_results
             return ie_result
@@ -574,6 +597,8 @@ class FileDownloader(object):
         # Forced printings
         if self.params.get('forcetitle', False):
             compat_print(info_dict['title'])
+        if self.params.get('forceid', False):
+            compat_print(info_dict['id'])
         if self.params.get('forceurl', False):
             compat_print(info_dict['url'])
         if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
@@ -594,7 +619,7 @@ class FileDownloader(object):
 
         try:
             dn = os.path.dirname(encodeFilename(filename))
-            if dn != '' and not os.path.exists(dn): # dn is already encoded
+            if dn != '' and not os.path.exists(dn):
                 os.makedirs(dn)
         except (OSError, IOError) as err:
             self.report_error(u'unable to create directory ' + compat_str(err))
@@ -627,8 +652,6 @@ class FileDownloader(object):
                 except (OSError, IOError):
                     self.report_error(u'Cannot write subtitles file ' + descfn)
                     return
-            if self.params.get('onlysubtitles', False):
-                return 
 
         if self.params.get('allsubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
             subtitles = info_dict['subtitles']
@@ -646,8 +669,6 @@ class FileDownloader(object):
                     except (OSError, IOError):
                         self.report_error(u'Cannot write subtitles file ' + descfn)
                         return
-            if self.params.get('onlysubtitles', False):
-                return 
 
         if self.params.get('writeinfojson', False):
             infofn = filename + u'.info.json'
@@ -735,7 +756,7 @@ class FileDownloader(object):
             except (IOError, OSError):
                 self.report_warning(u'Unable to remove downloaded video file')
 
-    def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path):
+    def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
@@ -745,18 +766,21 @@ class FileDownloader(object):
         except (OSError, IOError):
             self.report_error(u'RTMP download detected but "rtmpdump" could not be run')
             return False
+        verbosity_option = '--verbose' if self.params.get('verbose', False) else '--quiet'
 
         # Download using rtmpdump. rtmpdump returns exit code 2 when
         # the connection was interrumpted and resuming appears to be
         # possible. This is part of rtmpdump's normal usage, AFAIK.
-        basic_args = ['rtmpdump', '-q', '-r', url, '-o', tmpfilename]
+        basic_args = ['rtmpdump', verbosity_option, '-r', url, '-o', tmpfilename]
         if player_url is not None:
-            basic_args += ['-W', player_url]
+            basic_args += ['--swfVfy', player_url]
         if page_url is not None:
             basic_args += ['--pageUrl', page_url]
         if play_path is not None:
-            basic_args += ['-y', play_path]
-        args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
+            basic_args += ['--playpath', play_path]
+        if tc_url is not None:
+            basic_args += ['--tcUrl', url]
+        args = basic_args + [[], ['--resume', '--skip', '1']][self.params.get('continuedl', False)]
         if self.params.get('verbose', False):
             try:
                 import pipes
@@ -794,6 +818,37 @@ class FileDownloader(object):
             self.report_error(u'rtmpdump exited with code %d' % retval)
             return False
 
+    def _download_with_mplayer(self, filename, url):
+        self.report_destination(filename)
+        tmpfilename = self.temp_name(filename)
+
+        args = ['mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy', '-dumpstream', '-dumpfile', tmpfilename, url]
+        # Check for mplayer first
+        try:
+            subprocess.call(['mplayer', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
+        except (OSError, IOError):
+            self.report_error(u'MMS or RTSP download detected but "%s" could not be run' % args[0] )
+            return False
+
+        # Download using mplayer. 
+        retval = subprocess.call(args)
+        if retval == 0:
+            fsize = os.path.getsize(encodeFilename(tmpfilename))
+            self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize))
+            self.try_rename(tmpfilename, filename)
+            self._hook_progress({
+                'downloaded_bytes': fsize,
+                'total_bytes': fsize,
+                'filename': filename,
+                'status': 'finished',
+            })
+            return True
+        else:
+            self.to_stderr(u"\n")
+            self.report_error(u'mplayer exited with code %d' % retval)
+            return False
+
+
     def _do_download(self, filename, info_dict):
         url = info_dict['url']
 
@@ -811,7 +866,12 @@ class FileDownloader(object):
             return self._download_with_rtmpdump(filename, url,
                                                 info_dict.get('player_url', None),
                                                 info_dict.get('page_url', None),
-                                                info_dict.get('play_path', None))
+                                                info_dict.get('play_path', None),
+                                                info_dict.get('tc_url', None))
+
+        # Attempt to download using mplayer
+        if url.startswith('mms') or url.startswith('rtsp'):
+            return self._download_with_mplayer(filename, url)
 
         tmpfilename = self.temp_name(filename)
         stream = None

youtube_dl/PostProcessor.py

@@ -85,8 +85,9 @@ class FFmpegPostProcessor(PostProcessor):
         p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         stdout,stderr = p.communicate()
         if p.returncode != 0:
+            stderr = stderr.decode('utf-8', 'replace')
             msg = stderr.strip().split('\n')[-1]
-            raise FFmpegPostProcessorError(msg.decode('utf-8', 'replace'))
+            raise FFmpegPostProcessorError(msg)
 
     def _ffmpeg_filename_argument(self, fn):
         # ffmpeg broke --, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details

youtube_dl/__init__.py

@@ -191,8 +191,8 @@ def parseOpts(overrideArguments=None):
             action='store_true', dest='writesubtitles',
             help='write subtitle file (currently youtube only)', default=False)
     video_format.add_option('--only-sub',
-            action='store_true', dest='onlysubtitles',
-            help='downloads only the subtitles (no video)', default=False)
+            action='store_true', dest='skip_download',
+            help='[deprecated] alias of --skip-download', default=False)
     video_format.add_option('--all-subs',
             action='store_true', dest='allsubtitles',
             help='downloads all the available subtitles of the video (currently youtube only)', default=False)
@@ -216,6 +216,8 @@ def parseOpts(overrideArguments=None):
             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
     verbosity.add_option('-e', '--get-title',
             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
+    verbosity.add_option('--get-id',
+            action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
     verbosity.add_option('--get-thumbnail',
             action='store_true', dest='getthumbnail',
             help='simulate, quiet but print thumbnail URL', default=False)
@@ -390,8 +392,11 @@ def _real_main(argv=None):
 
     # General configuration
     cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
-    if opts.proxy:
-        proxies = {'http': opts.proxy, 'https': opts.proxy}
+    if opts.proxy is not None:
+        if opts.proxy == '':
+            proxies = {}
+        else:
+            proxies = {'http': opts.proxy, 'https': opts.proxy}
     else:
         proxies = compat_urllib_request.getproxies()
         # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
@@ -495,15 +500,16 @@ def _real_main(argv=None):
         'usenetrc': opts.usenetrc,
         'username': opts.username,
         'password': opts.password,
-        'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
+        'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
         'forceurl': opts.geturl,
         'forcetitle': opts.gettitle,
+        'forceid': opts.getid,
         'forcethumbnail': opts.getthumbnail,
         'forcedescription': opts.getdescription,
         'forcefilename': opts.getfilename,
         'forceformat': opts.getformat,
         'simulate': opts.simulate,
-        'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
+        'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
         'format': opts.format,
         'format_limit': opts.format_limit,
         'listformats': opts.listformats,
@@ -529,7 +535,6 @@ def _real_main(argv=None):
         'writeinfojson': opts.writeinfojson,
         'writethumbnail': opts.writethumbnail,
         'writesubtitles': opts.writesubtitles,
-        'onlysubtitles': opts.onlysubtitles,
         'allsubtitles': opts.allsubtitles,
         'listsubtitles': opts.listsubtitles,
         'subtitlesformat': opts.subtitlesformat,

youtube_dl/utils.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+import errno
 import gzip
 import io
 import json
@@ -11,7 +12,7 @@ import sys
 import traceback
 import zlib
 import email.utils
-import json
+import socket
 import datetime
 
 try:
@@ -149,6 +150,13 @@ try:
 except NameError:
     compat_chr = chr
 
+def compat_ord(c):
+    if type(c) is int: return c
+    else: return ord(c)
+
+# This is not clearly defined otherwise
+compiled_regex_type = type(re.compile(''))
+
 std_headers = {
     'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
     'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
@@ -334,12 +342,20 @@ def sanitize_open(filename, open_mode):
         stream = open(encodeFilename(filename), open_mode)
         return (stream, filename)
     except (IOError, OSError) as err:
-        # In case of error, try to remove win32 forbidden chars
-        filename = re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', filename)
+        if err.errno in (errno.EACCES,):
+            raise
 
-        # An exception here should be caught in the caller
-        stream = open(encodeFilename(filename), open_mode)
-        return (stream, filename)
+        # In case of error, try to remove win32 forbidden chars
+        alt_filename = os.path.join(
+                        re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', path_part)
+                        for path_part in os.path.split(filename)
+                       )
+        if alt_filename == filename:
+            raise
+        else:
+            # An exception here should be caught in the caller
+            stream = open(encodeFilename(filename), open_mode)
+            return (stream, alt_filename)
 
 
 def timeconvert(timestr):
@@ -456,7 +472,11 @@ class ExtractorError(Exception):
     """Error during info extraction."""
     def __init__(self, msg, tb=None):
         """ tb, if given, is the original traceback (so that it can be printed out). """
+
+        if not sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
+            msg = msg + u'; please report this issue on GitHub.'
         super(ExtractorError, self).__init__(msg)
+
         self.traceback = tb
         self.exc_info = sys.exc_info()  # preserve original exception
 

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2013.05.10'
+__version__ = '2013.06.21'