Release 2012.12.11

The new shiny build system

.gitignore

@@ -3,3 +3,13 @@
 *~
 wine-py2exe/
 py2exe.log
+*.kate-swp
+build/
+dist/
+MANIFEST
+README.txt
+youtube-dl.1
+youtube-dl.bash-completion
+youtube-dl
+youtube-dl.exe
+youtube-dl.tar.gz

.travis.yml

@@ -0,0 +1,13 @@
+language: python
+python:
+  - "2.6"
+  - "2.7"
+#  - "3.3"
+script: nosetests test --verbose
+notifications:
+  email:
+    - filippo.valsorda@gmail.com
+  irc:
+    channels:
+      - "irc.freenode.org#youtube-dl"
+    skip_join: true

LATEST_VERSION

@@ -1 +1 @@
-2012.10.09
+9999.99.99

MANIFEST.in

@@ -0,0 +1,3 @@
+include README.md
+include test/*.py
+include test/*.json

Makefile

@@ -1,47 +1,48 @@
-all: youtube-dl README.md youtube-dl.1 youtube-dl.bash-completion LATEST_VERSION
-# TODO: re-add youtube-dl.exe, and make sure it's 1. safe and 2. doesn't need sudo
+all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion
 
 clean:
-	rm -f youtube-dl youtube-dl.exe youtube-dl.1 LATEST_VERSION
+	rm -rf youtube-dl youtube-dl.exe youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/
 
 PREFIX=/usr/local
-install: youtube-dl youtube-dl.1 youtube-dl.bash-completion
-	install -m 755 --owner root --group root youtube-dl $(PREFIX)/bin/
-	install -m 644 --owner root --group root youtube-dl.1 $(PREFIX)/man/man1
-	install -m 644 --owner root --group root youtube-dl.bash-completion /etc/bash_completion.d/youtube-dl
+BINDIR=$(PREFIX)/bin
+MANDIR=$(PREFIX)/man
+SYSCONFDIR=/etc
 
-.PHONY: all clean install README.md youtube-dl.bash-completion
-# TODO un-phony README.md and youtube-dl.bash_completion by reading from .in files and generating from them
+install: youtube-dl youtube-dl.1 youtube-dl.bash-completion
+	install -d $(DESTDIR)$(BINDIR)
+	install -m 755 youtube-dl $(DESTDIR)$(BINDIR)
+	install -d $(DESTDIR)$(MANDIR)/man1
+	install -m 644 youtube-dl.1 $(DESTDIR)$(MANDIR)/man1
+	install -d $(DESTDIR)$(SYSCONFDIR)/bash_completion.d
+	install -m 644 youtube-dl.bash-completion $(DESTDIR)$(SYSCONFDIR)/bash_completion.d/youtube-dl
+
+test:
+	nosetests2 --nocapture test
+
+.PHONY: all clean install test
 
 youtube-dl: youtube_dl/*.py
-	zip --quiet --junk-paths youtube-dl youtube_dl/*.py
+	zip --quiet youtube-dl youtube_dl/*.py
+	zip --quiet --junk-paths youtube-dl youtube_dl/__main__.py
 	echo '#!/usr/bin/env python' > youtube-dl
 	cat youtube-dl.zip >> youtube-dl
 	rm youtube-dl.zip
 	chmod a+x youtube-dl
 
-youtube-dl.exe: youtube_dl/*.py
-	bash devscripts/wine-py2exe.sh build_exe.py
-
 README.md: youtube_dl/*.py
-	@options=$$(COLUMNS=80 python -m youtube_dl --help | sed -e '1,/.*General Options.*/ d' -e 's/^\W\{2\}\(\w\)/## \1/') && \
-		header=$$(sed -e '/.*# OPTIONS/,$$ d' README.md) && \
-		footer=$$(sed -e '1,/.*# FAQ/ d' README.md) && \
-		echo "$${header}" > README.md && \
-		echo >> README.md && \
-		echo '# OPTIONS' >> README.md && \
-		echo "$${options}" >> README.md&& \
-		echo >> README.md && \
-		echo '# FAQ' >> README.md && \
-		echo "$${footer}" >> README.md
+	COLUMNS=80 python -m youtube_dl --help | python devscripts/make_readme.py
+
+README.txt: README.md
+	pandoc -f markdown -t plain README.md -o README.txt
 
 youtube-dl.1: README.md
-	pandoc -s -w man README.md -o youtube-dl.1
+	pandoc -s -f markdown -t man README.md -o youtube-dl.1
 
-youtube-dl.bash-completion: README.md
-	@options=`egrep -o '(--[a-z-]+) ' README.md | sort -u | xargs echo` && \
-		content=`sed "s/opts=\"[^\"]*\"/opts=\"$${options}\"/g" youtube-dl.bash-completion` && \
-		echo "$${content}" > youtube-dl.bash-completion
+youtube-dl.bash-completion: youtube_dl/*.py devscripts/bash-completion.template
+	python devscripts/bash-completion.py
 
-LATEST_VERSION: youtube_dl/__init__.py
-	python -m youtube_dl --version > LATEST_VERSION
+youtube-dl.tar.gz: all
+	tar -czf youtube-dl.tar.gz -s "|^./|./youtube-dl/|" \
+		--exclude="*.pyc" --exclude="*.pyo" --exclude="*~" --exclude="youtube-dl.exe" \
+		--exclude="wine-py2exe/" --exclude="py2exe.log" --exclude="*.kate-swp" \
+		--exclude="build/" --exclude="dist/" --exclude="MANIFEST" --exclude=".git/" .

README.md

@@ -1,4 +1,4 @@
-% youtube-dl(1)
+% YOUTUBE-DL(1)
 
 # NAME
 youtube-dl
@@ -20,6 +20,11 @@ which means you can modify it, redistribute it or use it however you like.
     -i, --ignore-errors      continue on download errors
     -r, --rate-limit LIMIT   download rate limit (e.g. 50k or 44.6m)
     -R, --retries RETRIES    number of retries (default is 10)
+    --buffer-size SIZE       size of download buffer (e.g. 1024 or 16k) (default
+                             is 1024)
+    --no-resize-buffer       do not automatically adjust the buffer size. By
+                             default, the buffer size is automatically resized
+                             from an initial value of SIZE.
     --dump-user-agent        display the current browser identification
     --user-agent UA          specify a custom user agent
     --list-extractors        List all supported extractors and the URLs they
@@ -36,16 +41,22 @@ which means you can modify it, redistribute it or use it however you like.
 
 ## Filesystem Options:
     -t, --title              use title in file name
-    -l, --literal            use literal title in file name
+    --id                     use video ID in file name
+    -l, --literal            [deprecated] alias of --title
     -A, --auto-number        number downloaded files starting from 00000
-    -o, --output TEMPLATE    output filename template. Use %(stitle)s to get the
+    -o, --output TEMPLATE    output filename template. Use %(title)s to get the
                              title, %(uploader)s for the uploader name,
                              %(autonumber)s to get an automatically incremented
                              number, %(ext)s for the filename extension,
                              %(upload_date)s for the upload date (YYYYMMDD),
                              %(extractor)s for the provider (youtube, metacafe,
                              etc), %(id)s for the video id and %% for a literal
-                             percent. Use - to output to stdout.
+                             percent. Use - to output to stdout. Can also be
+                             used to download to a different directory, for
+                             example with -o '/my/downloads/%(uploader)s/%(title
+                             )s-%(id)s.%(ext)s' .
+    --restrict-filenames     Restrict filenames to only ASCII characters, and
+                             avoid "&" and spaces in filenames
     -a, --batch-file FILE    file containing URLs to download ('-' for stdin)
     -w, --no-overwrites      do not overwrite files
     -c, --continue           resume partially downloaded files
@@ -91,7 +102,7 @@ which means you can modify it, redistribute it or use it however you like.
     -n, --netrc              use .netrc authentication data
 
 ## Post-processing Options:
-    --extract-audio          convert video files to audio-only files (requires
+    -x, --extract-audio      convert video files to audio-only files (requires
                              ffmpeg or avconv and ffprobe or avprobe)
     --audio-format FORMAT    "best", "aac", "vorbis", "mp3", "m4a", or "wav";
                              best by default
@@ -101,6 +112,32 @@ which means you can modify it, redistribute it or use it however you like.
     -k, --keep-video         keeps the video file on disk after the post-
                              processing; the video is erased by default
 
+# CONFIGURATION
+
+You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.local/config/youtube-dl.conf`.
+
+# OUTPUT TEMPLATE
+
+The `-o` option allows users to indicate a template for the output file names. The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "http://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences have the format `%(NAME)s`. To clarify, that is a percent symbol followed by a name in parenthesis, followed by a lowercase S. Allowed names are:
+
+ - `id`: The sequence will be replaced by the video identifier.
+ - `url`: The sequence will be replaced by the video URL.
+ - `uploader`: The sequence will be replaced by the nickname of the person who uploaded the video.
+ - `upload_date`: The sequence will be replaced by the upload date in YYYYMMDD format.
+ - `title`: The sequence will be replaced by the video title.
+ - `ext`: The sequence will be replaced by the appropriate extension (like flv or mp4).
+ - `epoch`: The sequence will be replaced by the Unix epoch when creating the file.
+ - `autonumber`: The sequence will be replaced by a five-digit number that will be increased with each download, starting at zero.
+
+The current default template is `%(id)s.%(ext)s`, but that will be switchted to `%(title)s-%(id)s.%(ext)s` (which can be requested with `-t` at the moment).
+
+In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:
+
+    $ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc
+    youtube-dl test video ''_ä↭𝕐.mp4    # All kinds of weird characters
+    $ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filenames
+    youtube-dl_test_video_.mp4          # A simple file name
+
 # FAQ
 
 ### Can you please put the -b option back?
@@ -136,17 +173,9 @@ The error
 
 means you're using an outdated version of Python. Please update to Python 2.6 or 2.7.
 
-To run youtube-dl under Python 2.5, you'll have to manually check it out like this:
-
-	git clone git://github.com/rg3/youtube-dl.git
-	cd youtube-dl
-	python -m youtube_dl --help
-
-Please note that Python 2.5 is not supported anymore.
-
 ### What is this binary file? Where has the code gone?
 
-Since June 2012 (#342) youtube-dl is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repo to see the code. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make compile`.
+Since June 2012 (#342) youtube-dl is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.
 
 ### The exe throws a *Runtime error from Visual C++*
 
@@ -168,3 +197,5 @@ Please include:
 * The output of `youtube-dl --version`
 * The output of `python --version`
 * The name and version of your Operating System ("Ubuntu 11.04 x64" or "Windows 7 x64" is usually enough).
+
+For discussions, join us in the irc channel #youtube-dl on freenode.

bin/youtube-dl

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+import youtube_dl
+
+if __name__ == '__main__':
+    youtube_dl.main()

build_exe.py

@@ -1,48 +0,0 @@
-from distutils.core import setup
-import py2exe
-import sys, os
-
-"""This will create an exe that needs Microsoft Visual C++ 2008 Redistributable Package"""
-
-# If run without args, build executables
-if len(sys.argv) == 1:
-    sys.argv.append("py2exe")
-
-# os.chdir(os.path.dirname(os.path.abspath(sys.argv[0]))) # conflict with wine-py2exe.sh
-sys.path.append('./youtube_dl')
-
-options = {
-    "bundle_files": 1,
-    "compressed": 1,
-    "optimize": 2,
-    "dist_dir": '.',
-    "dll_excludes": ['w9xpopen.exe']
-}
-
-console = [{
-    "script":"./youtube_dl/__main__.py",
-    "dest_base": "youtube-dl",
-}]
-
-init_file = open('./youtube_dl/__init__.py')
-for line in init_file.readlines():
-    if line.startswith('__version__'):
-        version = line[11:].strip(" ='\n")
-        break
-else:
-    version = ''
-
-setup(name='youtube-dl',
-      version=version,
-      description='Small command-line program to download videos from YouTube.com and other video sites',
-      url='https://github.com/rg3/youtube-dl',
-      packages=['youtube_dl'],
-      
-      console = console,
-      options = {"py2exe": options},
-      zipfile = None,
-)
-
-import shutil
-shutil.rmtree("build")
-

devscripts/bash-completion.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+import os
+from os.path import dirname as dirn
+import sys
+
+sys.path.append(dirn(dirn((os.path.abspath(__file__)))))
+import youtube_dl
+
+BASH_COMPLETION_FILE = "youtube-dl.bash-completion"
+BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.template"
+
+def build_completion(opt_parser):
+    opts_flag = []
+    for group in opt_parser.option_groups:
+        for option in group.option_list:
+            #for every long flag
+            opts_flag.append(option.get_opt_string())
+    with open(BASH_COMPLETION_TEMPLATE) as f:
+        template = f.read()
+    with open(BASH_COMPLETION_FILE, "w") as f:
+        #just using the special char
+        filled_template = template.replace("{{flags}}", " ".join(opts_flag))
+        f.write(filled_template)
+
+parser = youtube_dl.parseOpts()[0]
+build_completion(parser)

devscripts/bash-completion.template

@@ -0,0 +1,14 @@
+__youtube-dl()
+{
+    local cur prev opts
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    opts="{{flags}}"
+
+    if [[ ${cur} == * ]] ; then
+        COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+        return 0
+    fi
+}
+
+complete -F __youtube-dl youtube-dl

devscripts/make_readme.py

@@ -0,0 +1,20 @@
+import sys
+import re
+
+README_FILE = 'README.md'
+helptext = sys.stdin.read()
+
+with open(README_FILE) as f:
+    oldreadme = f.read()
+
+header = oldreadme[:oldreadme.index('# OPTIONS')]
+footer = oldreadme[oldreadme.index('# CONFIGURATION'):]
+
+options = helptext[helptext.index('  General Options:')+19:]
+options = re.sub(r'^  (\w.+)$', r'## \1', options, flags=re.M)
+options = '# OPTIONS\n' + options + '\n'
+
+with open(README_FILE, 'w') as f:
+    f.write(header)
+    f.write(options)
+    f.write(footer)

devscripts/transition_helper.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+
+import sys, os
+
+try:
+    import urllib.request as compat_urllib_request
+except ImportError: # Python 2
+    import urllib2 as compat_urllib_request
+
+sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n')
+sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n')
+sys.stderr.write(u'The new location of the binaries is https://github.com/rg3/youtube-dl/downloads, not the git repository.\n\n')
+
+try:
+	raw_input()
+except NameError: # Python 3
+	input()
+
+filename = sys.argv[0]
+
+API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads"
+BIN_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl"
+
+if not os.access(filename, os.W_OK):
+    sys.exit('ERROR: no write permissions on %s' % filename)
+
+try:
+    urlh = compat_urllib_request.urlopen(BIN_URL)
+    newcontent = urlh.read()
+    urlh.close()
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to download latest version')
+
+try:
+    with open(filename, 'wb') as outf:
+        outf.write(newcontent)
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to overwrite current version')
+
+sys.stderr.write(u'Done! Now you can run youtube-dl.\n')

devscripts/transition_helper_exe/setup.py

@@ -0,0 +1,12 @@
+from distutils.core import setup
+import py2exe
+
+py2exe_options = {
+    "bundle_files": 1,
+    "compressed": 1,
+    "optimize": 2,
+    "dist_dir": '.',
+    "dll_excludes": ['w9xpopen.exe']
+}
+
+setup(console=['youtube-dl.py'], options={ "py2exe": py2exe_options }, zipfile=None)

devscripts/transition_helper_exe/youtube-dl.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+
+import sys, os
+import urllib2
+
+sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n')
+sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n')
+sys.stderr.write(u'The new location of the binaries is https://github.com/rg3/youtube-dl/downloads, not the git repository.\n\n')
+
+raw_input()
+
+filename = sys.argv[0]
+
+API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads"
+EXE_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl.exe"
+
+if not os.access(filename, os.W_OK):
+    sys.exit('ERROR: no write permissions on %s' % filename)
+
+exe = os.path.abspath(filename)
+directory = os.path.dirname(exe)
+if not os.access(directory, os.W_OK):
+    sys.exit('ERROR: no write permissions on %s' % directory)
+
+try:
+    urlh = urllib2.urlopen(EXE_URL)
+    newcontent = urlh.read()
+    urlh.close()
+    with open(exe + '.new', 'wb') as outf:
+        outf.write(newcontent)
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to download latest version')
+
+try:
+    bat = os.path.join(directory, 'youtube-dl-updater.bat')
+    b = open(bat, 'w')
+    b.write("""
+echo Updating youtube-dl...
+ping 127.0.0.1 -n 5 -w 1000 > NUL
+move /Y "%s.new" "%s"
+del "%s"
+    \n""" %(exe, exe, bat))
+    b.close()
+
+    os.startfile(bat)
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to overwrite current version')
+
+sys.stderr.write(u'Done! Now you can run youtube-dl.\n')

setup.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
+from distutils.core import setup
+import pkg_resources
+import sys
+
+try:
+    import py2exe
+    """This will create an exe that needs Microsoft Visual C++ 2008 Redistributable Package"""
+except ImportError:
+    if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe':
+        print("Cannot import py2exe", file=sys.stderr)
+        exit(1)
+
+py2exe_options = {
+    "bundle_files": 1,
+    "compressed": 1,
+    "optimize": 2,
+    "dist_dir": '.',
+    "dll_excludes": ['w9xpopen.exe']
+}
+py2exe_console = [{
+    "script": "./youtube_dl/__main__.py",
+    "dest_base": "youtube-dl",
+}]
+py2exe_params = {
+    'console': py2exe_console,
+    'options': { "py2exe": py2exe_options },
+    'zipfile': None
+}
+
+if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe':
+    params = py2exe_params
+else:
+    params = {
+        'scripts': ['bin/youtube-dl'],
+        'data_files': [('etc/bash_completion.d', ['youtube-dl.bash-completion']), # Installing system-wide would require sudo...
+                       ('share/doc/youtube_dl', ['README.txt']),
+                       ('share/man/man1/', ['youtube-dl.1'])]
+    }
+
+# Get the version from youtube_dl/version.py without importing the package
+exec(compile(open('youtube_dl/version.py').read(), 'youtube_dl/version.py', 'exec'))
+
+setup(
+    name = 'youtube_dl',
+    version = __version__,
+    description = 'YouTube video downloader',
+    long_description = 'Small command-line program to download videos from YouTube.com and other video sites.',
+    url = 'https://github.com/rg3/youtube-dl',
+    author = 'Ricardo Garcia',
+    maintainer = 'Philipp Hagemeister',
+    maintainer_email = 'phihag@phihag.de',
+    packages = ['youtube_dl'],
+
+    # Provokes warning on most systems (why?!)
+    #test_suite = 'nose.collector',
+    #test_requires = ['nosetest'],
+
+    classifiers = [
+        "Topic :: Multimedia :: Video",
+        "Development Status :: 5 - Production/Stable",
+        "Environment :: Console",
+        "License :: Public Domain",
+        "Programming Language :: Python :: 2.6",
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.3"
+    ],
+
+    **params
+)

test/gentests.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+
+import io  # for python 2
+import json
+import os
+import sys
+import unittest
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import youtube_dl.InfoExtractors
+
+HEADER = u'''#!/usr/bin/env python
+
+# DO NOT EDIT THIS FILE BY HAND!
+# It is auto-generated from tests.json and gentests.py.
+
+import hashlib
+import io
+import os
+import json
+import unittest
+import sys
+import socket
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import youtube_dl.FileDownloader
+import youtube_dl.InfoExtractors
+from youtube_dl.utils import *
+
+# General configuration (from __init__, not very elegant...)
+jar = compat_cookiejar.CookieJar()
+cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
+proxy_handler = compat_urllib_request.ProxyHandler()
+opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+compat_urllib_request.install_opener(opener)
+socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
+
+class FileDownloader(youtube_dl.FileDownloader):
+    def __init__(self, *args, **kwargs):
+        youtube_dl.FileDownloader.__init__(self, *args, **kwargs)
+        self.to_stderr = self.to_screen
+
+def _file_md5(fn):
+    with open(fn, 'rb') as f:
+        return hashlib.md5(f.read()).hexdigest()
+try:
+    _skip_unless = unittest.skipUnless
+except AttributeError: # Python 2.6
+    def _skip_unless(cond, reason='No reason given'):
+        def resfunc(f):
+            # Start the function name with test to appease nosetests-2.6
+            def test_wfunc(*args, **kwargs):
+                if cond:
+                    return f(*args, **kwargs)
+                else:
+                    print('Skipped test')
+                    return
+            return test_wfunc
+        return resfunc
+_skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs)
+
+class DownloadTest(unittest.TestCase):
+    PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+
+    def setUp(self):
+        # Clear old files
+        self.tearDown()
+
+        with io.open(self.PARAMETERS_FILE, encoding='utf-8') as pf:
+            self.parameters = json.load(pf)
+'''
+
+FOOTER = u'''
+
+if __name__ == '__main__':
+    unittest.main()
+'''
+
+DEF_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tests.json')
+TEST_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'test_download.py')
+
+def gentests():
+    with io.open(DEF_FILE, encoding='utf-8') as deff:
+        defs = json.load(deff)
+    with io.open(TEST_FILE, 'w', encoding='utf-8') as testf:
+        testf.write(HEADER)
+        spaces = ' ' * 4
+        write = lambda l: testf.write(spaces + l + u'\n')
+
+        for d in defs:
+            name = d['name']
+            ie = getattr(youtube_dl.InfoExtractors, name + 'IE')
+            testf.write(u'\n')
+            write('@_skip_unless(youtube_dl.InfoExtractors.' + name + 'IE._WORKING, "IE marked as not _WORKING")')
+            if not d['file']:
+                write('@_skip("No output file specified")')
+            elif 'skip' in d:
+                write('@_skip(' + repr(d['skip']) + ')')
+            write('def test_' + name + '(self):')
+            write('    filename = ' + repr(d['file']))
+            write('    params = self.parameters')
+            for p in d.get('params', {}):
+                write('    params["' + p + '"] = ' + repr(d['params'][p]))
+            write('    fd = FileDownloader(params)')
+            write('    fd.add_info_extractor(youtube_dl.InfoExtractors.' + name + 'IE())')
+            for ien in d.get('addIEs', []):
+                write('    fd.add_info_extractor(youtube_dl.InfoExtractors.' + ien + 'IE())')
+            write('    fd.download([' + repr(d['url']) + '])')
+            write('    self.assertTrue(os.path.exists(filename))')
+            if 'md5' in d:
+                write('    md5_for_file = _file_md5(filename)')
+                write('    self.assertEqual(md5_for_file, ' + repr(d['md5']) + ')')
+
+        testf.write(u'\n\n')
+        write('def tearDown(self):')
+        for d in defs:
+            if d['file']:
+                write('    if os.path.exists(' + repr(d['file']) + '):')
+                write('        os.remove(' + repr(d['file']) + ')')
+            else:
+                write('    # No file specified for ' + d['name'])
+        testf.write(u'\n')
+        testf.write(FOOTER)
+
+if __name__ == '__main__':
+    gentests()

test/parameters.json

@@ -0,0 +1,40 @@
+{
+    "consoletitle": false, 
+    "continuedl": true, 
+    "forcedescription": false, 
+    "forcefilename": false, 
+    "forceformat": false, 
+    "forcethumbnail": false, 
+    "forcetitle": false, 
+    "forceurl": false, 
+    "format": null, 
+    "format_limit": null, 
+    "ignoreerrors": false, 
+    "listformats": null, 
+    "logtostderr": false, 
+    "matchtitle": null, 
+    "max_downloads": null, 
+    "nooverwrites": false, 
+    "nopart": false, 
+    "noprogress": false, 
+    "outtmpl": "%(id)s.%(ext)s", 
+    "password": null, 
+    "playlistend": -1, 
+    "playliststart": 1, 
+    "prefer_free_formats": false, 
+    "quiet": false, 
+    "ratelimit": null, 
+    "rejecttitle": null, 
+    "retries": 10, 
+    "simulate": false, 
+    "skip_download": false, 
+    "subtitleslang": null, 
+    "test": true, 
+    "updatetime": true, 
+    "usenetrc": false, 
+    "username": null, 
+    "verbose": true, 
+    "writedescription": false, 
+    "writeinfojson": false, 
+    "writesubtitles": false
+}

test/test_div.py

@@ -1,29 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Various small unit tests
-
-import os,sys
-sys.path.append(os.path.dirname(os.path.dirname(__file__)))
-
-import youtube_dl
-
-def test_simplify_title():
-	assert youtube_dl._simplify_title(u'abc') == u'abc'
-	assert youtube_dl._simplify_title(u'abc_d-e') == u'abc_d-e'
-
-	assert youtube_dl._simplify_title(u'123') == u'123'
-
-	assert u'/' not in youtube_dl._simplify_title(u'abc/de')
-	assert u'abc' in youtube_dl._simplify_title(u'abc/de')
-	assert u'de' in youtube_dl._simplify_title(u'abc/de')
-	assert u'/' not in youtube_dl._simplify_title(u'abc/de///')
-
-	assert u'\\' not in youtube_dl._simplify_title(u'abc\\de')
-	assert u'abc' in youtube_dl._simplify_title(u'abc\\de')
-	assert u'de' in youtube_dl._simplify_title(u'abc\\de')
-
-	assert youtube_dl._simplify_title(u'ä') == u'ä'
-	assert youtube_dl._simplify_title(u'кириллица') == u'кириллица'
-
-	# Strip underlines
-	assert youtube_dl._simplify_title(u'\'a_') == u'a'

test/test_download.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python
+
+# DO NOT EDIT THIS FILE BY HAND!
+# It is auto-generated from tests.json and gentests.py.
+
+import hashlib
+import io
+import os
+import json
+import unittest
+import sys
+import socket
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import youtube_dl.FileDownloader
+import youtube_dl.InfoExtractors
+from youtube_dl.utils import *
+
+# General configuration (from __init__, not very elegant...)
+jar = compat_cookiejar.CookieJar()
+cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
+proxy_handler = compat_urllib_request.ProxyHandler()
+opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+compat_urllib_request.install_opener(opener)
+socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
+
+class FileDownloader(youtube_dl.FileDownloader):
+    def __init__(self, *args, **kwargs):
+        youtube_dl.FileDownloader.__init__(self, *args, **kwargs)
+        self.to_stderr = self.to_screen
+
+def _file_md5(fn):
+    with open(fn, 'rb') as f:
+        return hashlib.md5(f.read()).hexdigest()
+try:
+    _skip_unless = unittest.skipUnless
+except AttributeError: # Python 2.6
+    def _skip_unless(cond, reason='No reason given'):
+        def resfunc(f):
+            # Start the function name with test to appease nosetests-2.6
+            def test_wfunc(*args, **kwargs):
+                if cond:
+                    return f(*args, **kwargs)
+                else:
+                    print('Skipped test')
+                    return
+            return test_wfunc
+        return resfunc
+_skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs)
+
+class DownloadTest(unittest.TestCase):
+    PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+
+    def setUp(self):
+        # Clear old files
+        self.tearDown()
+
+        with io.open(self.PARAMETERS_FILE, encoding='utf-8') as pf:
+            self.parameters = json.load(pf)
+
+    @_skip_unless(youtube_dl.InfoExtractors.YoutubeIE._WORKING, "IE marked as not _WORKING")
+    def test_Youtube(self):
+        filename = u'BaW_jenozKc.mp4'
+        params = self.parameters
+        fd = FileDownloader(params)
+        fd.add_info_extractor(youtube_dl.InfoExtractors.YoutubeIE())
+        fd.download([u'http://www.youtube.com/watch?v=BaW_jenozKc'])
+        self.assertTrue(os.path.exists(filename))
+
+    @_skip_unless(youtube_dl.InfoExtractors.DailymotionIE._WORKING, "IE marked as not _WORKING")
+    def test_Dailymotion(self):
+        filename = u'x33vw9.mp4'
+        params = self.parameters
+        fd = FileDownloader(params)
+        fd.add_info_extractor(youtube_dl.InfoExtractors.DailymotionIE())
+        fd.download([u'http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech'])
+        self.assertTrue(os.path.exists(filename))
+        md5_for_file = _file_md5(filename)
+        self.assertEqual(md5_for_file, u'392c4b85a60a90dc4792da41ce3144eb')
+
+    @_skip_unless(youtube_dl.InfoExtractors.MetacafeIE._WORKING, "IE marked as not _WORKING")
+    def test_Metacafe(self):
+        filename = u'_aUehQsCQtM.flv'
+        params = self.parameters
+        fd = FileDownloader(params)
+        fd.add_info_extractor(youtube_dl.InfoExtractors.MetacafeIE())
+        fd.add_info_extractor(youtube_dl.InfoExtractors.YoutubeIE())
+        fd.download([u'http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/'])
+        self.assertTrue(os.path.exists(filename))
+
+    @_skip_unless(youtube_dl.InfoExtractors.BlipTVIE._WORKING, "IE marked as not _WORKING")
+    def test_BlipTV(self):
+        filename = u'5779306.m4v'
+        params = self.parameters
+        fd = FileDownloader(params)
+        fd.add_info_extractor(youtube_dl.InfoExtractors.BlipTVIE())
+        fd.download([u'http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352'])
+        self.assertTrue(os.path.exists(filename))
+        md5_for_file = _file_md5(filename)
+        self.assertEqual(md5_for_file, u'b2d849efcf7ee18917e4b4d9ff37cafe')
+
+    @_skip_unless(youtube_dl.InfoExtractors.XVideosIE._WORKING, "IE marked as not _WORKING")
+    def test_XVideos(self):
+        filename = u'939581.flv'
+        params = self.parameters
+        fd = FileDownloader(params)
+        fd.add_info_extractor(youtube_dl.InfoExtractors.XVideosIE())
+        fd.download([u'http://www.xvideos.com/video939581/funny_porns_by_s_-1'])
+        self.assertTrue(os.path.exists(filename))
+        md5_for_file = _file_md5(filename)
+        self.assertEqual(md5_for_file, u'1d0c835822f0a71a7bf011855db929d0')
+
+    @_skip_unless(youtube_dl.InfoExtractors.VimeoIE._WORKING, "IE marked as not _WORKING")
+    def test_Vimeo(self):
+        filename = u'14160053.mp4'
+        params = self.parameters
+        fd = FileDownloader(params)
+        fd.add_info_extractor(youtube_dl.InfoExtractors.VimeoIE())
+        fd.download([u'http://vimeo.com/14160053'])
+        self.assertTrue(os.path.exists(filename))
+        md5_for_file = _file_md5(filename)
+        self.assertEqual(md5_for_file, u'60540a4ec7cc378ec84b919c0aed5023')
+
+    @_skip_unless(youtube_dl.InfoExtractors.SoundcloudIE._WORKING, "IE marked as not _WORKING")
+    def test_Soundcloud(self):
+        filename = u'62986583.mp3'
+        params = self.parameters
+        fd = FileDownloader(params)
+        fd.add_info_extractor(youtube_dl.InfoExtractors.SoundcloudIE())
+        fd.download([u'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy'])
+        self.assertTrue(os.path.exists(filename))
+        md5_for_file = _file_md5(filename)
+        self.assertEqual(md5_for_file, u'ebef0a451b909710ed1d7787dddbf0d7')
+
+    @_skip_unless(youtube_dl.InfoExtractors.StanfordOpenClassroomIE._WORKING, "IE marked as not _WORKING")
+    def test_StanfordOpenClassroom(self):
+        filename = u'PracticalUnix_intro-environment.mp4'
+        params = self.parameters
+        fd = FileDownloader(params)
+        fd.add_info_extractor(youtube_dl.InfoExtractors.StanfordOpenClassroomIE())
+        fd.download([u'http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100'])
+        self.assertTrue(os.path.exists(filename))
+        md5_for_file = _file_md5(filename)
+        self.assertEqual(md5_for_file, u'544a9468546059d4e80d76265b0443b8')
+
+    @_skip_unless(youtube_dl.InfoExtractors.XNXXIE._WORKING, "IE marked as not _WORKING")
+    def test_XNXX(self):
+        filename = u'1135332.flv'
+        params = self.parameters
+        fd = FileDownloader(params)
+        fd.add_info_extractor(youtube_dl.InfoExtractors.XNXXIE())
+        fd.download([u'http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_'])
+        self.assertTrue(os.path.exists(filename))
+        md5_for_file = _file_md5(filename)
+        self.assertEqual(md5_for_file, u'0831677e2b4761795f68d417e0b7b445')
+
+    @_skip_unless(youtube_dl.InfoExtractors.YoukuIE._WORKING, "IE marked as not _WORKING")
+    def test_Youku(self):
+        filename = u'XNDgyMDQ2NTQw_part00.flv'
+        params = self.parameters
+        params["test"] = False
+        fd = FileDownloader(params)
+        fd.add_info_extractor(youtube_dl.InfoExtractors.YoukuIE())
+        fd.download([u'http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html'])
+        self.assertTrue(os.path.exists(filename))
+        md5_for_file = _file_md5(filename)
+        self.assertEqual(md5_for_file, u'ffe3f2e435663dc2d1eea34faeff5b5b')
+
+
+    def tearDown(self):
+        if os.path.exists(u'BaW_jenozKc.mp4'):
+            os.remove(u'BaW_jenozKc.mp4')
+        if os.path.exists(u'x33vw9.mp4'):
+            os.remove(u'x33vw9.mp4')
+        if os.path.exists(u'_aUehQsCQtM.flv'):
+            os.remove(u'_aUehQsCQtM.flv')
+        if os.path.exists(u'5779306.m4v'):
+            os.remove(u'5779306.m4v')
+        if os.path.exists(u'939581.flv'):
+            os.remove(u'939581.flv')
+        if os.path.exists(u'14160053.mp4'):
+            os.remove(u'14160053.mp4')
+        if os.path.exists(u'62986583.mp3'):
+            os.remove(u'62986583.mp3')
+        if os.path.exists(u'PracticalUnix_intro-environment.mp4'):
+            os.remove(u'PracticalUnix_intro-environment.mp4')
+        if os.path.exists(u'1135332.flv'):
+            os.remove(u'1135332.flv')
+        if os.path.exists(u'XNDgyMDQ2NTQw_part00.flv'):
+            os.remove(u'XNDgyMDQ2NTQw_part00.flv')
+
+
+
+if __name__ == '__main__':
+    unittest.main()

test/test_execution.py

@@ -0,0 +1,26 @@
+import unittest
+
+import sys
+import os
+import subprocess
+
+rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+try:
+    _DEV_NULL = subprocess.DEVNULL
+except AttributeError:
+    _DEV_NULL = open(os.devnull, 'wb')
+
+class TestExecution(unittest.TestCase):
+    def test_import(self):
+        subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
+
+    def test_module_exec(self):
+        if sys.version_info >= (2,7): # Python 2.6 doesn't support package execution
+            subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+
+    def test_main_exec(self):
+        subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+
+if __name__ == '__main__':
+    unittest.main()

test/test_utils.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+
+# Various small unit tests
+
+import sys
+import unittest
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+#from youtube_dl.utils import htmlentity_transform
+from youtube_dl.utils import timeconvert
+from youtube_dl.utils import sanitize_filename
+from youtube_dl.utils import unescapeHTML
+from youtube_dl.utils import orderedSet
+
+if sys.version_info < (3, 0):
+    _compat_str = lambda b: b.decode('unicode-escape')
+else:
+    _compat_str = lambda s: s
+
+
+class TestUtil(unittest.TestCase):
+    def test_timeconvert(self):
+        self.assertTrue(timeconvert('') is None)
+        self.assertTrue(timeconvert('bougrg') is None)
+
+    def test_sanitize_filename(self):
+        self.assertEqual(sanitize_filename('abc'), 'abc')
+        self.assertEqual(sanitize_filename('abc_d-e'), 'abc_d-e')
+
+        self.assertEqual(sanitize_filename('123'), '123')
+
+        self.assertEqual('abc_de', sanitize_filename('abc/de'))
+        self.assertFalse('/' in sanitize_filename('abc/de///'))
+
+        self.assertEqual('abc_de', sanitize_filename('abc/<>\\*|de'))
+        self.assertEqual('xxx', sanitize_filename('xxx/<>\\*|'))
+        self.assertEqual('yes no', sanitize_filename('yes? no'))
+        self.assertEqual('this - that', sanitize_filename('this: that'))
+
+        self.assertEqual(sanitize_filename('AT&T'), 'AT&T')
+        aumlaut = _compat_str('\xe4')
+        self.assertEqual(sanitize_filename(aumlaut), aumlaut)
+        tests = _compat_str('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0446\u0430')
+        self.assertEqual(sanitize_filename(tests), tests)
+
+        forbidden = '"\0\\/'
+        for fc in forbidden:
+            for fbc in forbidden:
+                self.assertTrue(fbc not in sanitize_filename(fc))
+
+    def test_sanitize_filename_restricted(self):
+        self.assertEqual(sanitize_filename('abc', restricted=True), 'abc')
+        self.assertEqual(sanitize_filename('abc_d-e', restricted=True), 'abc_d-e')
+
+        self.assertEqual(sanitize_filename('123', restricted=True), '123')
+
+        self.assertEqual('abc_de', sanitize_filename('abc/de', restricted=True))
+        self.assertFalse('/' in sanitize_filename('abc/de///', restricted=True))
+
+        self.assertEqual('abc_de', sanitize_filename('abc/<>\\*|de', restricted=True))
+        self.assertEqual('xxx', sanitize_filename('xxx/<>\\*|', restricted=True))
+        self.assertEqual('yes_no', sanitize_filename('yes? no', restricted=True))
+        self.assertEqual('this_-_that', sanitize_filename('this: that', restricted=True))
+
+        tests = _compat_str('a\xe4b\u4e2d\u56fd\u7684c')
+        self.assertEqual(sanitize_filename(tests, restricted=True), 'a_b_c')
+        self.assertTrue(sanitize_filename(_compat_str('\xf6'), restricted=True) != '')  # No empty filename
+
+        forbidden = '"\0\\/&!: \'\t\n()[]{}$;`^,#'
+        for fc in forbidden:
+            for fbc in forbidden:
+                self.assertTrue(fbc not in sanitize_filename(fc, restricted=True))
+
+        # Handle a common case more neatly
+        self.assertEqual(sanitize_filename(_compat_str('\u5927\u58f0\u5e26 - Song'), restricted=True), 'Song')
+        self.assertEqual(sanitize_filename(_compat_str('\u603b\u7edf: Speech'), restricted=True), 'Speech')
+        # .. but make sure the file name is never empty
+        self.assertTrue(sanitize_filename('-', restricted=True) != '')
+        self.assertTrue(sanitize_filename(':', restricted=True) != '')
+
+    def test_sanitize_ids(self):
+        self.assertEquals(sanitize_filename('_n_cd26wFpw', is_id=True), '_n_cd26wFpw')
+        self.assertEquals(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw')
+        self.assertEquals(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI')
+
+    def test_ordered_set(self):
+        self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
+        self.assertEqual(orderedSet([]), [])
+        self.assertEqual(orderedSet([1]), [1])
+        #keep the list ordered
+        self.assertEqual(orderedSet([135, 1, 1, 1]), [135, 1])
+
+    def test_unescape_html(self):
+        self.assertEqual(unescapeHTML(_compat_str('%20;')), _compat_str('%20;'))
+
+if __name__ == '__main__':
+    unittest.main()

test/test_youtube_lists.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+
+import sys
+import unittest
+import socket
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.InfoExtractors import YoutubePlaylistIE
+from youtube_dl.utils import *
+
+# General configuration (from __init__, not very elegant...)
+jar = compat_cookiejar.CookieJar()
+cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
+proxy_handler = compat_urllib_request.ProxyHandler()
+opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+compat_urllib_request.install_opener(opener)
+socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
+
+class FakeDownloader(object):
+    def __init__(self):
+        self.result = []
+        self.params = {}
+    def to_screen(self, s):
+        print(s)
+    def trouble(self, s):
+        raise Exception(s)
+    def download(self, x):
+        self.result.append(x)
+
+class TestYoutubeLists(unittest.TestCase):
+    def test_youtube_playlist(self):
+        DL = FakeDownloader()
+        IE = YoutubePlaylistIE(DL)
+        IE.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
+        self.assertEqual(DL.result, [
+            ['http://www.youtube.com/watch?v=bV9L5Ht9LgY'],
+            ['http://www.youtube.com/watch?v=FXxLjLQi3Fg'],
+            ['http://www.youtube.com/watch?v=tU3Bgo5qJZE']
+        ])
+
+    def test_youtube_playlist_long(self):
+        DL = FakeDownloader()
+        IE = YoutubePlaylistIE(DL)
+        IE.extract('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q')
+        self.assertTrue(len(DL.result) >= 799)
+
+    def test_youtube_course(self):
+        DL = FakeDownloader()
+        IE = YoutubePlaylistIE(DL)
+        # TODO find a > 100 (paginating?) videos course
+        IE.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
+        self.assertEqual(DL.result[0], ['http://www.youtube.com/watch?v=j9WZyLZCBzs'])
+        self.assertEqual(len(DL.result), 25)
+        self.assertEqual(DL.result[-1], ['http://www.youtube.com/watch?v=rYefUsYuEp0'])
+
+    def test_youtube_channel(self):
+        """I give up, please find a channel that does paginate and test this like test_youtube_playlist_long"""
+        pass # TODO
+
+    def test_youtube_user(self):
+        DL = FakeDownloader()
+        IE = YoutubePlaylistIE(DL)
+        IE.extract('https://www.youtube.com/user/TheLinuxFoundation')
+        self.assertTrue(len(DL.result) >= 320)
+
+if __name__ == '__main__':
+    unittest.main()

test/test_youtube_playlist_ids.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+
+import sys
+import unittest
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.InfoExtractors import YoutubeIE, YoutubePlaylistIE
+
+class TestYoutubePlaylistMatching(unittest.TestCase):
+    def test_playlist_matching(self):
+        self.assertTrue(YoutubePlaylistIE().suitable(u'ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8'))
+        self.assertTrue(YoutubePlaylistIE().suitable(u'PL63F0C78739B09958'))
+        self.assertFalse(YoutubePlaylistIE().suitable(u'PLtS2H6bU1M'))
+
+    def test_youtube_matching(self):
+        self.assertTrue(YoutubeIE().suitable(u'PLtS2H6bU1M'))
+
+if __name__ == '__main__':
+    unittest.main()

test/tests.json

@@ -0,0 +1,62 @@
+[
+  {
+    "name": "Youtube",
+    "url":  "http://www.youtube.com/watch?v=BaW_jenozKc",
+    "file":  "BaW_jenozKc.mp4"
+  },
+  {
+    "name": "Dailymotion",
+    "md5":  "392c4b85a60a90dc4792da41ce3144eb",
+    "url":  "http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech",
+    "file":  "x33vw9.mp4"
+  },
+  {
+    "name": "Metacafe",
+    "addIEs": ["Youtube"],
+    "url":  "http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/",
+    "file":  "_aUehQsCQtM.flv"
+  },
+  {
+    "name": "BlipTV",
+    "md5":  "b2d849efcf7ee18917e4b4d9ff37cafe",
+    "url":  "http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352",
+    "file":  "5779306.m4v"
+  },
+  {
+    "name": "XVideos",
+    "md5":  "1d0c835822f0a71a7bf011855db929d0",
+    "url":  "http://www.xvideos.com/video939581/funny_porns_by_s_-1",
+    "file":  "939581.flv"
+  },
+  {
+    "name": "Vimeo",
+    "md5":  "60540a4ec7cc378ec84b919c0aed5023",
+    "url":  "http://vimeo.com/14160053",
+    "file": "14160053.mp4"
+  },
+  {
+    "name": "Soundcloud",
+    "md5":  "ebef0a451b909710ed1d7787dddbf0d7",
+    "url":  "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy",
+    "file":  "62986583.mp3"
+  },
+  {
+    "name": "StanfordOpenClassroom",
+    "md5":  "544a9468546059d4e80d76265b0443b8",
+    "url":  "http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100",
+    "file":  "PracticalUnix_intro-environment.mp4"
+  },
+  {
+    "name": "XNXX",
+    "md5":  "0831677e2b4761795f68d417e0b7b445",
+    "url":  "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_",
+    "file":  "1135332.flv"
+  },
+  {
+    "name": "Youku",
+    "url": "http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html",
+    "file": "XNDgyMDQ2NTQw_part00.flv",
+    "md5": "ffe3f2e435663dc2d1eea34faeff5b5b",
+    "params": { "test": false }
+  }
+]

youtube-dl.1

@@ -1,239 +0,0 @@
-.TH youtube-dl 1 "" 
-.SH NAME
-.PP
-youtube-dl
-.SH SYNOPSIS
-.PP
-\f[B]youtube-dl\f[] [OPTIONS] URL [URL...]
-.SH DESCRIPTION
-.PP
-\f[B]youtube-dl\f[] is a small command-line program to download videos
-from YouTube.com and a few more sites.
-It requires the Python interpreter, version 2.x (x being at least 6),
-and it is not platform specific.
-It should work in your Unix box, in Windows or in Mac OS X.
-It is released to the public domain, which means you can modify it,
-redistribute it or use it however you like.
-.SH OPTIONS
-.IP
-.nf
-\f[C]
--h,\ --help\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ print\ this\ help\ text\ and\ exit
---version\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ print\ program\ version\ and\ exit
--U,\ --update\ \ \ \ \ \ \ \ \ \ \ \ \ update\ this\ program\ to\ latest\ version
--i,\ --ignore-errors\ \ \ \ \ \ continue\ on\ download\ errors
--r,\ --rate-limit\ LIMIT\ \ \ download\ rate\ limit\ (e.g.\ 50k\ or\ 44.6m)
--R,\ --retries\ RETRIES\ \ \ \ number\ of\ retries\ (default\ is\ 10)
---dump-user-agent\ \ \ \ \ \ \ \ display\ the\ current\ browser\ identification
---user-agent\ UA\ \ \ \ \ \ \ \ \ \ specify\ a\ custom\ user\ agent
---list-extractors\ \ \ \ \ \ \ \ List\ all\ supported\ extractors\ and\ the\ URLs\ they
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ would\ handle
-\f[]
-.fi
-.SS Video Selection:
-.IP
-.nf
-\f[C]
---playlist-start\ NUMBER\ \ playlist\ video\ to\ start\ at\ (default\ is\ 1)
---playlist-end\ NUMBER\ \ \ \ playlist\ video\ to\ end\ at\ (default\ is\ last)
---match-title\ REGEX\ \ \ \ \ \ download\ only\ matching\ titles\ (regex\ or\ caseless
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ sub-string)
---reject-title\ REGEX\ \ \ \ \ skip\ download\ for\ matching\ titles\ (regex\ or
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ caseless\ sub-string)
---max-downloads\ NUMBER\ \ \ Abort\ after\ downloading\ NUMBER\ files
-\f[]
-.fi
-.SS Filesystem Options:
-.IP
-.nf
-\f[C]
--t,\ --title\ \ \ \ \ \ \ \ \ \ \ \ \ \ use\ title\ in\ file\ name
--l,\ --literal\ \ \ \ \ \ \ \ \ \ \ \ use\ literal\ title\ in\ file\ name
--A,\ --auto-number\ \ \ \ \ \ \ \ number\ downloaded\ files\ starting\ from\ 00000
--o,\ --output\ TEMPLATE\ \ \ \ output\ filename\ template.\ Use\ %(stitle)s\ to\ get\ the
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ title,\ %(uploader)s\ for\ the\ uploader\ name,
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(autonumber)s\ to\ get\ an\ automatically\ incremented
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ number,\ %(ext)s\ for\ the\ filename\ extension,
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(upload_date)s\ for\ the\ upload\ date\ (YYYYMMDD),
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(extractor)s\ for\ the\ provider\ (youtube,\ metacafe,
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ etc),\ %(id)s\ for\ the\ video\ id\ and\ %%\ for\ a\ literal
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ percent.\ Use\ -\ to\ output\ to\ stdout.
--a,\ --batch-file\ FILE\ \ \ \ file\ containing\ URLs\ to\ download\ (\[aq]-\[aq]\ for\ stdin)
--w,\ --no-overwrites\ \ \ \ \ \ do\ not\ overwrite\ files
--c,\ --continue\ \ \ \ \ \ \ \ \ \ \ resume\ partially\ downloaded\ files
---no-continue\ \ \ \ \ \ \ \ \ \ \ \ do\ not\ resume\ partially\ downloaded\ files\ (restart
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ from\ beginning)
---cookies\ FILE\ \ \ \ \ \ \ \ \ \ \ file\ to\ read\ cookies\ from\ and\ dump\ cookie\ jar\ in
---no-part\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ do\ not\ use\ .part\ files
---no-mtime\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ do\ not\ use\ the\ Last-modified\ header\ to\ set\ the\ file
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ modification\ time
---write-description\ \ \ \ \ \ write\ video\ description\ to\ a\ .description\ file
---write-info-json\ \ \ \ \ \ \ \ write\ video\ metadata\ to\ a\ .info.json\ file
-\f[]
-.fi
-.SS Verbosity / Simulation Options:
-.IP
-.nf
-\f[C]
--q,\ --quiet\ \ \ \ \ \ \ \ \ \ \ \ \ \ activates\ quiet\ mode
--s,\ --simulate\ \ \ \ \ \ \ \ \ \ \ do\ not\ download\ the\ video\ and\ do\ not\ write\ anything
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ to\ disk
---skip-download\ \ \ \ \ \ \ \ \ \ do\ not\ download\ the\ video
--g,\ --get-url\ \ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ URL
--e,\ --get-title\ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ title
---get-thumbnail\ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ thumbnail\ URL
---get-description\ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ video\ description
---get-filename\ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ output\ filename
---get-format\ \ \ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ output\ format
---no-progress\ \ \ \ \ \ \ \ \ \ \ \ do\ not\ print\ progress\ bar
---console-title\ \ \ \ \ \ \ \ \ \ display\ progress\ in\ console\ titlebar
--v,\ --verbose\ \ \ \ \ \ \ \ \ \ \ \ print\ various\ debugging\ information
-\f[]
-.fi
-.SS Video Format Options:
-.IP
-.nf
-\f[C]
--f,\ --format\ FORMAT\ \ \ \ \ \ video\ format\ code
---all-formats\ \ \ \ \ \ \ \ \ \ \ \ download\ all\ available\ video\ formats
---prefer-free-formats\ \ \ \ prefer\ free\ video\ formats\ unless\ a\ specific\ one\ is
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ requested
---max-quality\ FORMAT\ \ \ \ \ highest\ quality\ format\ to\ download
--F,\ --list-formats\ \ \ \ \ \ \ list\ all\ available\ formats\ (currently\ youtube\ only)
---write-srt\ \ \ \ \ \ \ \ \ \ \ \ \ \ write\ video\ closed\ captions\ to\ a\ .srt\ file
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (currently\ youtube\ only)
---srt-lang\ LANG\ \ \ \ \ \ \ \ \ \ language\ of\ the\ closed\ captions\ to\ download
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (optional)\ use\ IETF\ language\ tags\ like\ \[aq]en\[aq]
-\f[]
-.fi
-.SS Authentication Options:
-.IP
-.nf
-\f[C]
--u,\ --username\ USERNAME\ \ account\ username
--p,\ --password\ PASSWORD\ \ account\ password
--n,\ --netrc\ \ \ \ \ \ \ \ \ \ \ \ \ \ use\ .netrc\ authentication\ data
-\f[]
-.fi
-.SS Post-processing Options:
-.IP
-.nf
-\f[C]
---extract-audio\ \ \ \ \ \ \ \ \ \ convert\ video\ files\ to\ audio-only\ files\ (requires
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ffmpeg\ or\ avconv\ and\ ffprobe\ or\ avprobe)
---audio-format\ FORMAT\ \ \ \ "best",\ "aac",\ "vorbis",\ "mp3",\ "m4a",\ or\ "wav";
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ best\ by\ default
---audio-quality\ QUALITY\ \ ffmpeg/avconv\ audio\ quality\ specification,\ insert\ a
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ value\ between\ 0\ (better)\ and\ 9\ (worse)\ for\ VBR\ or\ a
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ specific\ bitrate\ like\ 128K\ (default\ 5)
--k,\ --keep-video\ \ \ \ \ \ \ \ \ keeps\ the\ video\ file\ on\ disk\ after\ the\ post-
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ processing;\ the\ video\ is\ erased\ by\ default
-\f[]
-.fi
-.SH FAQ
-.SS Can you please put the -b option back?
-.PP
-Most people asking this question are not aware that youtube-dl now
-defaults to downloading the highest available quality as reported by
-YouTube, which will be 1080p or 720p in some cases, so you no longer
-need the -b option.
-For some specific videos, maybe YouTube does not report them to be
-available in a specific high quality format you\[aq]\[aq]re interested
-in.
-In that case, simply request it with the -f option and youtube-dl will
-try to download it.
-.SS I get HTTP error 402 when trying to download a video. What\[aq]s
-this?
-.PP
-Apparently YouTube requires you to pass a CAPTCHA test if you download
-too much.
-We\[aq]\[aq]re considering to provide a way to let you solve the
-CAPTCHA (https://github.com/rg3/youtube-dl/issues/154), but at the
-moment, your best course of action is pointing a webbrowser to the
-youtube URL, solving the CAPTCHA, and restart youtube-dl.
-.SS I have downloaded a video but how can I play it?
-.PP
-Once the video is fully downloaded, use any video player, such as
-vlc (http://www.videolan.org) or mplayer (http://www.mplayerhq.hu/).
-.SS The links provided by youtube-dl -g are not working anymore
-.PP
-The URLs youtube-dl outputs require the downloader to have the correct
-cookies.
-Use the \f[C]--cookies\f[] option to write the required cookies into a
-file, and advise your downloader to read cookies from that file.
-Some sites also require a common user agent to be used, use
-\f[C]--dump-user-agent\f[] to see the one in use by youtube-dl.
-.SS ERROR: no fmt_url_map or conn information found in video info
-.PP
-youtube has switched to a new video info format in July 2011 which is
-not supported by old versions of youtube-dl.
-You can update youtube-dl with \f[C]sudo\ youtube-dl\ --update\f[].
-.SS ERROR: unable to download video
-.PP
-youtube requires an additional signature since September 2012 which is
-not supported by old versions of youtube-dl.
-You can update youtube-dl with \f[C]sudo\ youtube-dl\ --update\f[].
-.SS SyntaxError: Non-ASCII character
-.PP
-The error
-.IP
-.nf
-\f[C]
-File\ "youtube-dl",\ line\ 2
-SyntaxError:\ Non-ASCII\ character\ \[aq]\\x93\[aq]\ ...
-\f[]
-.fi
-.PP
-means you\[aq]re using an outdated version of Python.
-Please update to Python 2.6 or 2.7.
-.PP
-To run youtube-dl under Python 2.5, you\[aq]ll have to manually check it
-out like this:
-.IP
-.nf
-\f[C]
-git\ clone\ git://github.com/rg3/youtube-dl.git
-cd\ youtube-dl
-python\ -m\ youtube_dl\ --help
-\f[]
-.fi
-.PP
-Please note that Python 2.5 is not supported anymore.
-.SS What is this binary file? Where has the code gone?
-.PP
-Since June 2012 (#342) youtube-dl is packed as an executable zipfile,
-simply unzip it (might need renaming to \f[C]youtube-dl.zip\f[] first on
-some systems) or clone the git repo to see the code.
-If you modify the code, you can run it by executing the
-\f[C]__main__.py\f[] file.
-To recompile the executable, run \f[C]make\ compile\f[].
-.SS The exe throws a \f[I]Runtime error from Visual C++\f[]
-.PP
-To run the exe you need to install first the Microsoft Visual C++ 2008
-Redistributable
-Package (http://www.microsoft.com/en-us/download/details.aspx?id=29).
-.SH COPYRIGHT
-.PP
-youtube-dl is released into the public domain by the copyright holders.
-.PP
-This README file was originally written by Daniel Bolton
-(<https://github.com/dbbolton>) and is likewise released into the public
-domain.
-.SH BUGS
-.PP
-Bugs and suggestions should be reported at:
-<https://github.com/rg3/youtube-dl/issues>
-.PP
-Please include:
-.IP \[bu] 2
-Your exact command line, like
-\f[C]youtube-dl\ -t\ "http://www.youtube.com/watch?v=uHlDtZ6Oc3s&feature=channel_video_title"\f[].
-A common mistake is not to escape the \f[C]&\f[].
-Putting URLs in quotes should solve this problem.
-.IP \[bu] 2
-The output of \f[C]youtube-dl\ --version\f[]
-.IP \[bu] 2
-The output of \f[C]python\ --version\f[]
-.IP \[bu] 2
-The name and version of your Operating System ("Ubuntu 11.04 x64" or
-"Windows 7 x64" is usually enough).

youtube-dl.bash-completion

@@ -1,14 +0,0 @@
-__youtube-dl()
-{
-    local cur prev opts
-    COMPREPLY=()
-    cur="${COMP_WORDS[COMP_CWORD]}"
-    opts="--all-formats --audio-format --audio-quality --auto-number --batch-file --console-title --continue --cookies --dump-user-agent --extract-audio --format --get-description --get-filename --get-format --get-thumbnail --get-title --get-url --help --ignore-errors --keep-video --list-extractors --list-formats --literal --match-title --max-downloads --max-quality --netrc --no-continue --no-mtime --no-overwrites --no-part --no-progress --output --password --playlist-end --playlist-start --prefer-free-formats --quiet --rate-limit --reject-title --retries --simulate --skip-download --srt-lang --title --update --user-agent --username --verbose --version --write-description --write-info-json --write-srt"
-
-    if [[ ${cur} == * ]] ; then
-        COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
-        return 0
-    fi
-}
-
-complete -F __youtube-dl youtube-dl

youtube_dl/PostProcessor.py

@@ -1,198 +1,200 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+from __future__ import absolute_import
+
 import os
 import subprocess
 import sys
 import time
 
-from utils import *
+from .utils import *
 
 
 class PostProcessor(object):
-	"""Post Processor class.
+    """Post Processor class.
 
-	PostProcessor objects can be added to downloaders with their
-	add_post_processor() method. When the downloader has finished a
-	successful download, it will take its internal chain of PostProcessors
-	and start calling the run() method on each one of them, first with
-	an initial argument and then with the returned value of the previous
-	PostProcessor.
+    PostProcessor objects can be added to downloaders with their
+    add_post_processor() method. When the downloader has finished a
+    successful download, it will take its internal chain of PostProcessors
+    and start calling the run() method on each one of them, first with
+    an initial argument and then with the returned value of the previous
+    PostProcessor.
 
-	The chain will be stopped if one of them ever returns None or the end
-	of the chain is reached.
+    The chain will be stopped if one of them ever returns None or the end
+    of the chain is reached.
 
-	PostProcessor objects follow a "mutual registration" process similar
-	to InfoExtractor objects.
-	"""
+    PostProcessor objects follow a "mutual registration" process similar
+    to InfoExtractor objects.
+    """
 
-	_downloader = None
+    _downloader = None
 
-	def __init__(self, downloader=None):
-		self._downloader = downloader
+    def __init__(self, downloader=None):
+        self._downloader = downloader
 
-	def set_downloader(self, downloader):
-		"""Sets the downloader for this PP."""
-		self._downloader = downloader
+    def set_downloader(self, downloader):
+        """Sets the downloader for this PP."""
+        self._downloader = downloader
 
-	def run(self, information):
-		"""Run the PostProcessor.
+    def run(self, information):
+        """Run the PostProcessor.
 
-		The "information" argument is a dictionary like the ones
-		composed by InfoExtractors. The only difference is that this
-		one has an extra field called "filepath" that points to the
-		downloaded file.
+        The "information" argument is a dictionary like the ones
+        composed by InfoExtractors. The only difference is that this
+        one has an extra field called "filepath" that points to the
+        downloaded file.
 
-		When this method returns None, the postprocessing chain is
-		stopped. However, this method may return an information
-		dictionary that will be passed to the next postprocessing
-		object in the chain. It can be the one it received after
-		changing some fields.
+        When this method returns None, the postprocessing chain is
+        stopped. However, this method may return an information
+        dictionary that will be passed to the next postprocessing
+        object in the chain. It can be the one it received after
+        changing some fields.
 
-		In addition, this method may raise a PostProcessingError
-		exception that will be taken into account by the downloader
-		it was called from.
-		"""
-		return information # by default, do nothing
+        In addition, this method may raise a PostProcessingError
+        exception that will be taken into account by the downloader
+        it was called from.
+        """
+        return information # by default, do nothing
 
 class AudioConversionError(BaseException):
-	def __init__(self, message):
-		self.message = message
+    def __init__(self, message):
+        self.message = message
 
 class FFmpegExtractAudioPP(PostProcessor):
-	def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, keepvideo=False):
-		PostProcessor.__init__(self, downloader)
-		if preferredcodec is None:
-			preferredcodec = 'best'
-		self._preferredcodec = preferredcodec
-		self._preferredquality = preferredquality
-		self._keepvideo = keepvideo
-		self._exes = self.detect_executables()
+    def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, keepvideo=False):
+        PostProcessor.__init__(self, downloader)
+        if preferredcodec is None:
+            preferredcodec = 'best'
+        self._preferredcodec = preferredcodec
+        self._preferredquality = preferredquality
+        self._keepvideo = keepvideo
+        self._exes = self.detect_executables()
 
-	@staticmethod
-	def detect_executables():
-		def executable(exe):
-			try:
-				subprocess.check_output([exe, '-version'])
-			except OSError:
-				return False
-			return exe
-		programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
-		return dict((program, executable(program)) for program in programs)
+    @staticmethod
+    def detect_executables():
+        def executable(exe):
+            try:
+                subprocess.Popen([exe, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+            except OSError:
+                return False
+            return exe
+        programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
+        return dict((program, executable(program)) for program in programs)
 
-	def get_audio_codec(self, path):
-		if not self._exes['ffprobe'] and not self._exes['avprobe']: return None
-		try:
-			cmd = [self._exes['avprobe'] or self._exes['ffprobe'], '-show_streams', '--', encodeFilename(path)]
-			handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE)
-			output = handle.communicate()[0]
-			if handle.wait() != 0:
-				return None
-		except (IOError, OSError):
-			return None
-		audio_codec = None
-		for line in output.split('\n'):
-			if line.startswith('codec_name='):
-				audio_codec = line.split('=')[1].strip()
-			elif line.strip() == 'codec_type=audio' and audio_codec is not None:
-				return audio_codec
-		return None
+    def get_audio_codec(self, path):
+        if not self._exes['ffprobe'] and not self._exes['avprobe']: return None
+        try:
+            cmd = [self._exes['avprobe'] or self._exes['ffprobe'], '-show_streams', '--', encodeFilename(path)]
+            handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE)
+            output = handle.communicate()[0]
+            if handle.wait() != 0:
+                return None
+        except (IOError, OSError):
+            return None
+        audio_codec = None
+        for line in output.split('\n'):
+            if line.startswith('codec_name='):
+                audio_codec = line.split('=')[1].strip()
+            elif line.strip() == 'codec_type=audio' and audio_codec is not None:
+                return audio_codec
+        return None
 
-	def run_ffmpeg(self, path, out_path, codec, more_opts):
-		if not self._exes['ffmpeg'] and not self._exes['avconv']:
-			raise AudioConversionError('ffmpeg or avconv not found. Please install one.')	
-		if codec is None:
-			acodec_opts = []
-		else:
-			acodec_opts = ['-acodec', codec]
-		cmd = ([self._exes['avconv'] or self._exes['ffmpeg'], '-y', '-i', encodeFilename(path), '-vn']
-			   + acodec_opts + more_opts +
-			   ['--', encodeFilename(out_path)])
-		p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-		stdout,stderr = p.communicate()
-		if p.returncode != 0:
-			msg = stderr.strip().split('\n')[-1]
-			raise AudioConversionError(msg)
+    def run_ffmpeg(self, path, out_path, codec, more_opts):
+        if not self._exes['ffmpeg'] and not self._exes['avconv']:
+            raise AudioConversionError('ffmpeg or avconv not found. Please install one.')   
+        if codec is None:
+            acodec_opts = []
+        else:
+            acodec_opts = ['-acodec', codec]
+        cmd = ([self._exes['avconv'] or self._exes['ffmpeg'], '-y', '-i', encodeFilename(path), '-vn']
+               + acodec_opts + more_opts +
+               ['--', encodeFilename(out_path)])
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout,stderr = p.communicate()
+        if p.returncode != 0:
+            msg = stderr.strip().split('\n')[-1]
+            raise AudioConversionError(msg)
 
-	def run(self, information):
-		path = information['filepath']
+    def run(self, information):
+        path = information['filepath']
 
-		filecodec = self.get_audio_codec(path)
-		if filecodec is None:
-			self._downloader.to_stderr(u'WARNING: unable to obtain file audio codec with ffprobe')
-			return None
+        filecodec = self.get_audio_codec(path)
+        if filecodec is None:
+            self._downloader.to_stderr(u'WARNING: unable to obtain file audio codec with ffprobe')
+            return None
 
-		more_opts = []
-		if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'):
-			if self._preferredcodec == 'm4a' and filecodec == 'aac':
-				# Lossless, but in another container
-				acodec = 'copy'
-				extension = self._preferredcodec
-				more_opts = [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
-			elif filecodec in ['aac', 'mp3', 'vorbis']:
-				# Lossless if possible
-				acodec = 'copy'
-				extension = filecodec
-				if filecodec == 'aac':
-					more_opts = ['-f', 'adts']
-				if filecodec == 'vorbis':
-					extension = 'ogg'
-			else:
-				# MP3 otherwise.
-				acodec = 'libmp3lame'
-				extension = 'mp3'
-				more_opts = []
-				if self._preferredquality is not None:
-					if int(self._preferredquality) < 10:
-						more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
-					else:
-						more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality]
-		else:
-			# We convert the audio (lossy)
-			acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
-			extension = self._preferredcodec
-			more_opts = []
-			if self._preferredquality is not None:
-				if int(self._preferredquality) < 10:
-					more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
-				else:
-					more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality]
-			if self._preferredcodec == 'aac':
-				more_opts += ['-f', 'adts']
-			if self._preferredcodec == 'm4a':
-				more_opts += [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
-			if self._preferredcodec == 'vorbis':
-				extension = 'ogg'
-			if self._preferredcodec == 'wav':
-				extension = 'wav'
-				more_opts += ['-f', 'wav']
+        more_opts = []
+        if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'):
+            if self._preferredcodec == 'm4a' and filecodec == 'aac':
+                # Lossless, but in another container
+                acodec = 'copy'
+                extension = self._preferredcodec
+                more_opts = [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
+            elif filecodec in ['aac', 'mp3', 'vorbis']:
+                # Lossless if possible
+                acodec = 'copy'
+                extension = filecodec
+                if filecodec == 'aac':
+                    more_opts = ['-f', 'adts']
+                if filecodec == 'vorbis':
+                    extension = 'ogg'
+            else:
+                # MP3 otherwise.
+                acodec = 'libmp3lame'
+                extension = 'mp3'
+                more_opts = []
+                if self._preferredquality is not None:
+                    if int(self._preferredquality) < 10:
+                        more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
+                    else:
+                        more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
+        else:
+            # We convert the audio (lossy)
+            acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
+            extension = self._preferredcodec
+            more_opts = []
+            if self._preferredquality is not None:
+                if int(self._preferredquality) < 10:
+                    more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
+                else:
+                    more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
+            if self._preferredcodec == 'aac':
+                more_opts += ['-f', 'adts']
+            if self._preferredcodec == 'm4a':
+                more_opts += [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
+            if self._preferredcodec == 'vorbis':
+                extension = 'ogg'
+            if self._preferredcodec == 'wav':
+                extension = 'wav'
+                more_opts += ['-f', 'wav']
 
-		prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups
-		new_path = prefix + sep + extension
-		self._downloader.to_screen(u'[' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path)
-		try:
-			self.run_ffmpeg(path, new_path, acodec, more_opts)
-		except:
-			etype,e,tb = sys.exc_info()
-			if isinstance(e, AudioConversionError):
-				self._downloader.to_stderr(u'ERROR: audio conversion failed: ' + e.message)
-			else:
-				self._downloader.to_stderr(u'ERROR: error running ' + (self._exes['avconv'] and 'avconv' or 'ffmpeg'))
-			return None
+        prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups
+        new_path = prefix + sep + extension
+        self._downloader.to_screen(u'[' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path)
+        try:
+            self.run_ffmpeg(path, new_path, acodec, more_opts)
+        except:
+            etype,e,tb = sys.exc_info()
+            if isinstance(e, AudioConversionError):
+                self._downloader.to_stderr(u'ERROR: audio conversion failed: ' + e.message)
+            else:
+                self._downloader.to_stderr(u'ERROR: error running ' + (self._exes['avconv'] and 'avconv' or 'ffmpeg'))
+            return None
 
- 		# Try to update the date time for extracted audio file.
-		if information.get('filetime') is not None:
-			try:
-				os.utime(encodeFilename(new_path), (time.time(), information['filetime']))
-			except:
-				self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file')
+        # Try to update the date time for extracted audio file.
+        if information.get('filetime') is not None:
+            try:
+                os.utime(encodeFilename(new_path), (time.time(), information['filetime']))
+            except:
+                self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file')
 
-		if not self._keepvideo:
-			try:
-				os.remove(encodeFilename(path))
-			except (IOError, OSError):
-				self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file')
-				return None
+        if not self._keepvideo:
+            try:
+                os.remove(encodeFilename(path))
+            except (IOError, OSError):
+                self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file')
+                return None
 
-		information['filepath'] = new_path
-		return information
+        information['filepath'] = new_path
+        return information

youtube_dl/__main__.py

@@ -1,7 +1,17 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 
-import __init__
+# Execute with
+# $ python youtube_dl/__main__.py (2.6+)
+# $ python -m youtube_dl          (2.7+)
+
+import sys
+
+if __package__ is None and not hasattr(sys, "frozen"):
+    # direct call of __main__.py
+    import os.path
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import youtube_dl
 
 if __name__ == '__main__':
-	__init__.main()
+    youtube_dl.main()

youtube_dl/utils.py

@@ -2,353 +2,510 @@
 # -*- coding: utf-8 -*-
 
 import gzip
-import htmlentitydefs
-import HTMLParser
+import io
 import locale
 import os
 import re
 import sys
 import zlib
-import urllib2
 import email.utils
 import json
 
 try:
-	import cStringIO as StringIO
-except ImportError:
-	import StringIO
+    import urllib.request as compat_urllib_request
+except ImportError: # Python 2
+    import urllib2 as compat_urllib_request
+
+try:
+    import urllib.error as compat_urllib_error
+except ImportError: # Python 2
+    import urllib2 as compat_urllib_error
+
+try:
+    import urllib.parse as compat_urllib_parse
+except ImportError: # Python 2
+    import urllib as compat_urllib_parse
+
+try:
+    from urllib.parse import urlparse as compat_urllib_parse_urlparse
+except ImportError: # Python 2
+    from urlparse import urlparse as compat_urllib_parse_urlparse
+
+try:
+    import http.cookiejar as compat_cookiejar
+except ImportError: # Python 2
+    import cookielib as compat_cookiejar
+
+try:
+    import html.entities as compat_html_entities
+except ImportError: # Python 2
+    import htmlentitydefs as compat_html_entities
+
+try:
+    import html.parser as compat_html_parser
+except ImportError: # Python 2
+    import HTMLParser as compat_html_parser
+
+try:
+    import http.client as compat_http_client
+except ImportError: # Python 2
+    import httplib as compat_http_client
+
+try:
+    from urllib.parse import parse_qs as compat_parse_qs
+except ImportError: # Python 2
+    # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
+    # Python 2's version is apparently totally broken
+    def _unquote(string, encoding='utf-8', errors='replace'):
+        if string == '':
+            return string
+        res = string.split('%')
+        if len(res) == 1:
+            return string
+        if encoding is None:
+            encoding = 'utf-8'
+        if errors is None:
+            errors = 'replace'
+        # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
+        pct_sequence = b''
+        string = res[0]
+        for item in res[1:]:
+            try:
+                if not item:
+                    raise ValueError
+                pct_sequence += item[:2].decode('hex')
+                rest = item[2:]
+                if not rest:
+                    # This segment was just a single percent-encoded character.
+                    # May be part of a sequence of code units, so delay decoding.
+                    # (Stored in pct_sequence).
+                    continue
+            except ValueError:
+                rest = '%' + item
+            # Encountered non-percent-encoded characters. Flush the current
+            # pct_sequence.
+            string += pct_sequence.decode(encoding, errors) + rest
+            pct_sequence = b''
+        if pct_sequence:
+            # Flush the final pct_sequence
+            string += pct_sequence.decode(encoding, errors)
+        return string
+
+    def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
+                encoding='utf-8', errors='replace'):
+        qs, _coerce_result = qs, unicode
+        pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
+        r = []
+        for name_value in pairs:
+            if not name_value and not strict_parsing:
+                continue
+            nv = name_value.split('=', 1)
+            if len(nv) != 2:
+                if strict_parsing:
+                    raise ValueError("bad query field: %r" % (name_value,))
+                # Handle case of a control-name with no equal sign
+                if keep_blank_values:
+                    nv.append('')
+                else:
+                    continue
+            if len(nv[1]) or keep_blank_values:
+                name = nv[0].replace('+', ' ')
+                name = _unquote(name, encoding=encoding, errors=errors)
+                name = _coerce_result(name)
+                value = nv[1].replace('+', ' ')
+                value = _unquote(value, encoding=encoding, errors=errors)
+                value = _coerce_result(value)
+                r.append((name, value))
+        return r
+
+    def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
+                encoding='utf-8', errors='replace'):
+        parsed_result = {}
+        pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
+                        encoding=encoding, errors=errors)
+        for name, value in pairs:
+            if name in parsed_result:
+                parsed_result[name].append(value)
+            else:
+                parsed_result[name] = [value]
+        return parsed_result
+
+try:
+    compat_str = unicode # Python 2
+except NameError:
+    compat_str = str
+
+try:
+    compat_chr = unichr # Python 2
+except NameError:
+    compat_chr = chr
 
 std_headers = {
-	'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
-	'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
-	'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-	'Accept-Encoding': 'gzip, deflate',
-	'Accept-Language': 'en-us,en;q=0.5',
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
+    'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Encoding': 'gzip, deflate',
+    'Accept-Language': 'en-us,en;q=0.5',
 }
-
 def preferredencoding():
-	"""Get preferred encoding.
+    """Get preferred encoding.
 
-	Returns the best encoding scheme for the system, based on
-	locale.getpreferredencoding() and some further tweaks.
-	"""
-	def yield_preferredencoding():
-		try:
-			pref = locale.getpreferredencoding()
-			u'TEST'.encode(pref)
-		except:
-			pref = 'UTF-8'
-		while True:
-			yield pref
-	return yield_preferredencoding().next()
+    Returns the best encoding scheme for the system, based on
+    locale.getpreferredencoding() and some further tweaks.
+    """
+    try:
+        pref = locale.getpreferredencoding()
+        u'TEST'.encode(pref)
+    except:
+        pref = 'UTF-8'
 
+    return pref
+
+if sys.version_info < (3,0):
+    def compat_print(s):
+        print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
+else:
+    def compat_print(s):
+        assert type(s) == type(u'')
+        print(s)
 
 def htmlentity_transform(matchobj):
-	"""Transforms an HTML entity to a Unicode character.
+    """Transforms an HTML entity to a character.
 
-	This function receives a match object and is intended to be used with
-	the re.sub() function.
-	"""
-	entity = matchobj.group(1)
+    This function receives a match object and is intended to be used with
+    the re.sub() function.
+    """
+    entity = matchobj.group(1)
 
-	# Known non-numeric HTML entity
-	if entity in htmlentitydefs.name2codepoint:
-		return unichr(htmlentitydefs.name2codepoint[entity])
+    # Known non-numeric HTML entity
+    if entity in compat_html_entities.name2codepoint:
+        return compat_chr(compat_html_entities.name2codepoint[entity])
 
-	# Unicode character
-	mobj = re.match(ur'(?u)#(x?\d+)', entity)
-	if mobj is not None:
-		numstr = mobj.group(1)
-		if numstr.startswith(u'x'):
-			base = 16
-			numstr = u'0%s' % numstr
-		else:
-			base = 10
-		return unichr(long(numstr, base))
+    mobj = re.match(u'(?u)#(x?\\d+)', entity)
+    if mobj is not None:
+        numstr = mobj.group(1)
+        if numstr.startswith(u'x'):
+            base = 16
+            numstr = u'0%s' % numstr
+        else:
+            base = 10
+        return compat_chr(int(numstr, base))
 
-	# Unknown entity in name, return its literal representation
-	return (u'&%s;' % entity)
+    # Unknown entity in name, return its literal representation
+    return (u'&%s;' % entity)
 
-HTMLParser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
-class IDParser(HTMLParser.HTMLParser):
-	"""Modified HTMLParser that isolates a tag with the specified id"""
-	def __init__(self, id):
-		self.id = id
-		self.result = None
-		self.started = False
-		self.depth = {}
-		self.html = None
-		self.watch_startpos = False
-		self.error_count = 0
-		HTMLParser.HTMLParser.__init__(self)
+compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
+class IDParser(compat_html_parser.HTMLParser):
+    """Modified HTMLParser that isolates a tag with the specified id"""
+    def __init__(self, id):
+        self.id = id
+        self.result = None
+        self.started = False
+        self.depth = {}
+        self.html = None
+        self.watch_startpos = False
+        self.error_count = 0
+        compat_html_parser.HTMLParser.__init__(self)
 
-	def error(self, message):
-		print >> sys.stderr, self.getpos()
-		if self.error_count > 10 or self.started:
-			raise HTMLParser.HTMLParseError(message, self.getpos())
-		self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
-		self.error_count += 1
-		self.goahead(1)
+    def error(self, message):
+        if self.error_count > 10 or self.started:
+            raise compat_html_parser.HTMLParseError(message, self.getpos())
+        self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
+        self.error_count += 1
+        self.goahead(1)
 
-	def loads(self, html):
-		self.html = html
-		self.feed(html)
-		self.close()
+    def loads(self, html):
+        self.html = html
+        self.feed(html)
+        self.close()
 
-	def handle_starttag(self, tag, attrs):
-		attrs = dict(attrs)
-		if self.started:
-			self.find_startpos(None)
-		if 'id' in attrs and attrs['id'] == self.id:
-			self.result = [tag]
-			self.started = True
-			self.watch_startpos = True
-		if self.started:
-			if not tag in self.depth: self.depth[tag] = 0
-			self.depth[tag] += 1
+    def handle_starttag(self, tag, attrs):
+        attrs = dict(attrs)
+        if self.started:
+            self.find_startpos(None)
+        if 'id' in attrs and attrs['id'] == self.id:
+            self.result = [tag]
+            self.started = True
+            self.watch_startpos = True
+        if self.started:
+            if not tag in self.depth: self.depth[tag] = 0
+            self.depth[tag] += 1
 
-	def handle_endtag(self, tag):
-		if self.started:
-			if tag in self.depth: self.depth[tag] -= 1
-			if self.depth[self.result[0]] == 0:
-				self.started = False
-				self.result.append(self.getpos())
+    def handle_endtag(self, tag):
+        if self.started:
+            if tag in self.depth: self.depth[tag] -= 1
+            if self.depth[self.result[0]] == 0:
+                self.started = False
+                self.result.append(self.getpos())
 
-	def find_startpos(self, x):
-		"""Needed to put the start position of the result (self.result[1])
-		after the opening tag with the requested id"""
-		if self.watch_startpos:
-			self.watch_startpos = False
-			self.result.append(self.getpos())
-	handle_entityref = handle_charref = handle_data = handle_comment = \
-	handle_decl = handle_pi = unknown_decl = find_startpos
+    def find_startpos(self, x):
+        """Needed to put the start position of the result (self.result[1])
+        after the opening tag with the requested id"""
+        if self.watch_startpos:
+            self.watch_startpos = False
+            self.result.append(self.getpos())
+    handle_entityref = handle_charref = handle_data = handle_comment = \
+    handle_decl = handle_pi = unknown_decl = find_startpos
 
-	def get_result(self):
-		if self.result == None: return None
-		if len(self.result) != 3: return None
-		lines = self.html.split('\n')
-		lines = lines[self.result[1][0]-1:self.result[2][0]]
-		lines[0] = lines[0][self.result[1][1]:]
-		if len(lines) == 1:
-			lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
-		lines[-1] = lines[-1][:self.result[2][1]]
-		return '\n'.join(lines).strip()
+    def get_result(self):
+        if self.result is None:
+            return None
+        if len(self.result) != 3:
+            return None
+        lines = self.html.split('\n')
+        lines = lines[self.result[1][0]-1:self.result[2][0]]
+        lines[0] = lines[0][self.result[1][1]:]
+        if len(lines) == 1:
+            lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
+        lines[-1] = lines[-1][:self.result[2][1]]
+        return '\n'.join(lines).strip()
 
 def get_element_by_id(id, html):
-	"""Return the content of the tag with the specified id in the passed HTML document"""
-	parser = IDParser(id)
-	try:
-		parser.loads(html)
-	except HTMLParser.HTMLParseError:
-		pass
-	return parser.get_result()
+    """Return the content of the tag with the specified id in the passed HTML document"""
+    parser = IDParser(id)
+    try:
+        parser.loads(html)
+    except compat_html_parser.HTMLParseError:
+        pass
+    return parser.get_result()
 
 
 def clean_html(html):
-	"""Clean an HTML snippet into a readable string"""
-	# Newline vs <br />
-	html = html.replace('\n', ' ')
-	html = re.sub('\s*<\s*br\s*/?\s*>\s*', '\n', html)
-	# Strip html tags
-	html = re.sub('<.*?>', '', html)
-	# Replace html entities
-	html = unescapeHTML(html)
-	return html
+    """Clean an HTML snippet into a readable string"""
+    # Newline vs <br />
+    html = html.replace('\n', ' ')
+    html = re.sub('\s*<\s*br\s*/?\s*>\s*', '\n', html)
+    # Strip html tags
+    html = re.sub('<.*?>', '', html)
+    # Replace html entities
+    html = unescapeHTML(html)
+    return html
 
 
 def sanitize_open(filename, open_mode):
-	"""Try to open the given filename, and slightly tweak it if this fails.
+    """Try to open the given filename, and slightly tweak it if this fails.
 
-	Attempts to open the given filename. If this fails, it tries to change
-	the filename slightly, step by step, until it's either able to open it
-	or it fails and raises a final exception, like the standard open()
-	function.
+    Attempts to open the given filename. If this fails, it tries to change
+    the filename slightly, step by step, until it's either able to open it
+    or it fails and raises a final exception, like the standard open()
+    function.
 
-	It returns the tuple (stream, definitive_file_name).
-	"""
-	try:
-		if filename == u'-':
-			if sys.platform == 'win32':
-				import msvcrt
-				msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
-			return (sys.stdout, filename)
-		stream = open(encodeFilename(filename), open_mode)
-		return (stream, filename)
-	except (IOError, OSError), err:
-		# In case of error, try to remove win32 forbidden chars
-		filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
+    It returns the tuple (stream, definitive_file_name).
+    """
+    try:
+        if filename == u'-':
+            if sys.platform == 'win32':
+                import msvcrt
+                msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+            return (sys.stdout, filename)
+        stream = open(encodeFilename(filename), open_mode)
+        return (stream, filename)
+    except (IOError, OSError) as err:
+        # In case of error, try to remove win32 forbidden chars
+        filename = re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', filename)
 
-		# An exception here should be caught in the caller
-		stream = open(encodeFilename(filename), open_mode)
-		return (stream, filename)
+        # An exception here should be caught in the caller
+        stream = open(encodeFilename(filename), open_mode)
+        return (stream, filename)
 
 
 def timeconvert(timestr):
-	"""Convert RFC 2822 defined time string into system timestamp"""
-	timestamp = None
-	timetuple = email.utils.parsedate_tz(timestr)
-	if timetuple is not None:
-		timestamp = email.utils.mktime_tz(timetuple)
-	return timestamp
-	
-def sanitize_filename(s):
-	"""Sanitizes a string so it could be used as part of a filename."""
-	def replace_insane(char):
-		if char in u' .\\/|?*<>:"' or ord(char) < 32:
-			return '_'
-		return char
-	return u''.join(map(replace_insane, s)).strip('_')
+    """Convert RFC 2822 defined time string into system timestamp"""
+    timestamp = None
+    timetuple = email.utils.parsedate_tz(timestr)
+    if timetuple is not None:
+        timestamp = email.utils.mktime_tz(timetuple)
+    return timestamp
+
+def sanitize_filename(s, restricted=False, is_id=False):
+    """Sanitizes a string so it could be used as part of a filename.
+    If restricted is set, use a stricter subset of allowed characters.
+    Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
+    """
+    def replace_insane(char):
+        if char == '?' or ord(char) < 32 or ord(char) == 127:
+            return ''
+        elif char == '"':
+            return '' if restricted else '\''
+        elif char == ':':
+            return '_-' if restricted else ' -'
+        elif char in '\\/|*<>':
+            return '_'
+        if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
+            return '_'
+        if restricted and ord(char) > 127:
+            return '_'
+        return char
+
+    result = u''.join(map(replace_insane, s))
+    if not is_id:
+        while '__' in result:
+            result = result.replace('__', '_')
+        result = result.strip('_')
+        # Common case of "Foreign band name - English song title"
+        if restricted and result.startswith('-_'):
+            result = result[2:]
+        if not result:
+            result = '_'
+    return result
 
 def orderedSet(iterable):
-	""" Remove all duplicates from the input iterable """
-	res = []
-	for el in iterable:
-		if el not in res:
-			res.append(el)
-	return res
+    """ Remove all duplicates from the input iterable """
+    res = []
+    for el in iterable:
+        if el not in res:
+            res.append(el)
+    return res
 
 def unescapeHTML(s):
-	"""
-	@param s a string (of type unicode)
-	"""
-	assert type(s) == type(u'')
+    """
+    @param s a string
+    """
+    assert type(s) == type(u'')
 
-	result = re.sub(ur'(?u)&(.+?);', htmlentity_transform, s)
-	return result
+    result = re.sub(u'(?u)&(.+?);', htmlentity_transform, s)
+    return result
 
 def encodeFilename(s):
-	"""
-	@param s The name of the file (of type unicode)
-	"""
+    """
+    @param s The name of the file
+    """
 
-	assert type(s) == type(u'')
+    assert type(s) == type(u'')
 
-	if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
-		# Pass u'' directly to use Unicode APIs on Windows 2000 and up
-		# (Detecting Windows NT 4 is tricky because 'major >= 4' would
-		# match Windows 9x series as well. Besides, NT 4 is obsolete.)
-		return s
-	else:
-		return s.encode(sys.getfilesystemencoding(), 'ignore')
+    # Python 3 has a Unicode API
+    if sys.version_info >= (3, 0):
+        return s
+
+    if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
+        # Pass u'' directly to use Unicode APIs on Windows 2000 and up
+        # (Detecting Windows NT 4 is tricky because 'major >= 4' would
+        # match Windows 9x series as well. Besides, NT 4 is obsolete.)
+        return s
+    else:
+        return s.encode(sys.getfilesystemencoding(), 'ignore')
 
 class DownloadError(Exception):
-	"""Download Error exception.
+    """Download Error exception.
 
-	This exception may be thrown by FileDownloader objects if they are not
-	configured to continue on errors. They will contain the appropriate
-	error message.
-	"""
-	pass
+    This exception may be thrown by FileDownloader objects if they are not
+    configured to continue on errors. They will contain the appropriate
+    error message.
+    """
+    pass
 
 
 class SameFileError(Exception):
-	"""Same File exception.
+    """Same File exception.
 
-	This exception will be thrown by FileDownloader objects if they detect
-	multiple files would have to be downloaded to the same file on disk.
-	"""
-	pass
+    This exception will be thrown by FileDownloader objects if they detect
+    multiple files would have to be downloaded to the same file on disk.
+    """
+    pass
 
 
 class PostProcessingError(Exception):
-	"""Post Processing exception.
+    """Post Processing exception.
 
-	This exception may be raised by PostProcessor's .run() method to
-	indicate an error in the postprocessing task.
-	"""
-	pass
+    This exception may be raised by PostProcessor's .run() method to
+    indicate an error in the postprocessing task.
+    """
+    pass
 
 class MaxDownloadsReached(Exception):
-	""" --max-downloads limit has been reached. """
-	pass
+    """ --max-downloads limit has been reached. """
+    pass
 
 
 class UnavailableVideoError(Exception):
-	"""Unavailable Format exception.
+    """Unavailable Format exception.
 
-	This exception will be thrown when a video is requested
-	in a format that is not available for that video.
-	"""
-	pass
+    This exception will be thrown when a video is requested
+    in a format that is not available for that video.
+    """
+    pass
 
 
 class ContentTooShortError(Exception):
-	"""Content Too Short exception.
+    """Content Too Short exception.
 
-	This exception may be raised by FileDownloader objects when a file they
-	download is too small for what the server announced first, indicating
-	the connection was probably interrupted.
-	"""
-	# Both in bytes
-	downloaded = None
-	expected = None
+    This exception may be raised by FileDownloader objects when a file they
+    download is too small for what the server announced first, indicating
+    the connection was probably interrupted.
+    """
+    # Both in bytes
+    downloaded = None
+    expected = None
 
-	def __init__(self, downloaded, expected):
-		self.downloaded = downloaded
-		self.expected = expected
+    def __init__(self, downloaded, expected):
+        self.downloaded = downloaded
+        self.expected = expected
 
 
 class Trouble(Exception):
-	"""Trouble helper exception
-	
-	This is an exception to be handled with
-	FileDownloader.trouble
-	"""
+    """Trouble helper exception
 
-class YoutubeDLHandler(urllib2.HTTPHandler):
-	"""Handler for HTTP requests and responses.
+    This is an exception to be handled with
+    FileDownloader.trouble
+    """
 
-	This class, when installed with an OpenerDirector, automatically adds
-	the standard headers to every HTTP request and handles gzipped and
-	deflated responses from web servers. If compression is to be avoided in
-	a particular request, the original request in the program code only has
-	to include the HTTP header "Youtubedl-No-Compression", which will be
-	removed before making the real request.
+class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
+    """Handler for HTTP requests and responses.
 
-	Part of this code was copied from:
+    This class, when installed with an OpenerDirector, automatically adds
+    the standard headers to every HTTP request and handles gzipped and
+    deflated responses from web servers. If compression is to be avoided in
+    a particular request, the original request in the program code only has
+    to include the HTTP header "Youtubedl-No-Compression", which will be
+    removed before making the real request.
 
-	http://techknack.net/python-urllib2-handlers/
+    Part of this code was copied from:
 
-	Andrew Rowls, the author of that code, agreed to release it to the
-	public domain.
-	"""
+    http://techknack.net/python-urllib2-handlers/
 
-	@staticmethod
-	def deflate(data):
-		try:
-			return zlib.decompress(data, -zlib.MAX_WBITS)
-		except zlib.error:
-			return zlib.decompress(data)
+    Andrew Rowls, the author of that code, agreed to release it to the
+    public domain.
+    """
 
-	@staticmethod
-	def addinfourl_wrapper(stream, headers, url, code):
-		if hasattr(urllib2.addinfourl, 'getcode'):
-			return urllib2.addinfourl(stream, headers, url, code)
-		ret = urllib2.addinfourl(stream, headers, url)
-		ret.code = code
-		return ret
+    @staticmethod
+    def deflate(data):
+        try:
+            return zlib.decompress(data, -zlib.MAX_WBITS)
+        except zlib.error:
+            return zlib.decompress(data)
 
-	def http_request(self, req):
-		for h in std_headers:
-			if h in req.headers:
-				del req.headers[h]
-			req.add_header(h, std_headers[h])
-		if 'Youtubedl-no-compression' in req.headers:
-			if 'Accept-encoding' in req.headers:
-				del req.headers['Accept-encoding']
-			del req.headers['Youtubedl-no-compression']
-		return req
+    @staticmethod
+    def addinfourl_wrapper(stream, headers, url, code):
+        if hasattr(compat_urllib_request.addinfourl, 'getcode'):
+            return compat_urllib_request.addinfourl(stream, headers, url, code)
+        ret = compat_urllib_request.addinfourl(stream, headers, url)
+        ret.code = code
+        return ret
 
-	def http_response(self, req, resp):
-		old_resp = resp
-		# gzip
-		if resp.headers.get('Content-encoding', '') == 'gzip':
-			gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r')
-			resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
-			resp.msg = old_resp.msg
-		# deflate
-		if resp.headers.get('Content-encoding', '') == 'deflate':
-			gz = StringIO.StringIO(self.deflate(resp.read()))
-			resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
-			resp.msg = old_resp.msg
-		return resp
+    def http_request(self, req):
+        for h in std_headers:
+            if h in req.headers:
+                del req.headers[h]
+            req.add_header(h, std_headers[h])
+        if 'Youtubedl-no-compression' in req.headers:
+            if 'Accept-encoding' in req.headers:
+                del req.headers['Accept-encoding']
+            del req.headers['Youtubedl-no-compression']
+        return req
+
+    def http_response(self, req, resp):
+        old_resp = resp
+        # gzip
+        if resp.headers.get('Content-encoding', '') == 'gzip':
+            gz = gzip.GzipFile(fileobj=io.BytesIO(resp.read()), mode='r')
+            resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
+            resp.msg = old_resp.msg
+        # deflate
+        if resp.headers.get('Content-encoding', '') == 'deflate':
+            gz = io.BytesIO(self.deflate(resp.read()))
+            resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
+            resp.msg = old_resp.msg
+        return resp
+
+    https_request = http_request
+    https_response = http_response

youtube_dl/version.py

@@ -0,0 +1,2 @@
+
+__version__ = '2012.12.11'