release 2012.11.27

Also enable album URLs on Vimeo.

.gitignore

@@ -0,0 +1,5 @@
+*.pyc
+*.pyo
+*~
+wine-py2exe/
+py2exe.log

.travis.yml

@@ -0,0 +1,9 @@
+language: python
+#specify the python version
+python:
+  - "2.6"
+  - "2.7"
+#command to install the setup
+install:
+# command to run tests
+script: nosetests test --nocapture

LATEST_VERSION

@@ -1 +1 @@
-2010.02.13
+2012.11.27

Makefile

@@ -0,0 +1,57 @@
+all: youtube-dl README.md youtube-dl.1 youtube-dl.bash-completion LATEST_VERSION
+# TODO: re-add youtube-dl.exe, and make sure it's 1. safe and 2. doesn't need sudo
+
+clean:
+	rm -f youtube-dl youtube-dl.exe youtube-dl.1 LATEST_VERSION
+
+PREFIX=/usr/local
+BINDIR=$(PREFIX)/bin
+MANDIR=$(PREFIX)/man
+SYSCONFDIR=/etc
+
+install: youtube-dl youtube-dl.1 youtube-dl.bash-completion
+	install -d $(DESTDIR)$(BINDIR)
+	install -m 755 youtube-dl $(DESTDIR)$(BINDIR)
+	install -d $(DESTDIR)$(MANDIR)/man1
+	install -m 644 youtube-dl.1 $(DESTDIR)$(MANDIR)/man1
+	install -d $(DESTDIR)$(SYSCONFDIR)/bash_completion.d
+	install -m 644 youtube-dl.bash-completion $(DESTDIR)$(SYSCONFDIR)/bash_completion.d/youtube-dl
+
+test:
+	nosetests2 --nocapture test
+
+.PHONY: all clean install test README.md youtube-dl.bash-completion
+# TODO un-phony README.md and youtube-dl.bash_completion by reading from .in files and generating from them
+
+youtube-dl: youtube_dl/*.py
+	zip --quiet --junk-paths youtube-dl youtube_dl/*.py
+	echo '#!/usr/bin/env python' > youtube-dl
+	cat youtube-dl.zip >> youtube-dl
+	rm youtube-dl.zip
+	chmod a+x youtube-dl
+
+youtube-dl.exe: youtube_dl/*.py
+	bash devscripts/wine-py2exe.sh build_exe.py
+
+README.md: youtube_dl/*.py
+	@options=$$(COLUMNS=80 python -m youtube_dl --help | sed -e '1,/.*General Options.*/ d' -e 's/^\W\{2\}\(\w\)/## \1/') && \
+		header=$$(sed -e '/.*# OPTIONS/,$$ d' README.md) && \
+		footer=$$(sed -e '1,/.*# CONFIGURATION/ d' README.md) && \
+		echo "$${header}" > README.md && \
+		echo >> README.md && \
+		echo '# OPTIONS' >> README.md && \
+		echo "$${options}" >> README.md&& \
+		echo >> README.md && \
+		echo '# CONFIGURATION' >> README.md && \
+		echo "$${footer}" >> README.md
+
+youtube-dl.1: README.md
+	pandoc -s -w man README.md -o youtube-dl.1
+
+youtube-dl.bash-completion: README.md
+	@options=`egrep -o '(--[a-z-]+) ' README.md | sort -u | xargs echo` && \
+		content=`sed "s/opts=\"[^\"]*\"/opts=\"$${options}\"/g" youtube-dl.bash-completion` && \
+		echo "$${content}" > youtube-dl.bash-completion
+
+LATEST_VERSION: youtube_dl/__init__.py
+	python -m youtube_dl --version > LATEST_VERSION

README.md

@@ -0,0 +1,177 @@
+% youtube-dl(1)
+
+# NAME
+youtube-dl
+
+# SYNOPSIS
+**youtube-dl** [OPTIONS] URL [URL...]
+
+# DESCRIPTION
+**youtube-dl** is a small command-line program to download videos from
+YouTube.com and a few more sites. It requires the Python interpreter, version
+2.x (x being at least 6), and it is not platform specific. It should work in
+your Unix box, in Windows or in Mac OS X. It is released to the public domain,
+which means you can modify it, redistribute it or use it however you like.
+
+# OPTIONS
+    -h, --help               print this help text and exit
+    --version                print program version and exit
+    -U, --update             update this program to latest version
+    -i, --ignore-errors      continue on download errors
+    -r, --rate-limit LIMIT   download rate limit (e.g. 50k or 44.6m)
+    -R, --retries RETRIES    number of retries (default is 10)
+    --dump-user-agent        display the current browser identification
+    --user-agent UA          specify a custom user agent
+    --list-extractors        List all supported extractors and the URLs they
+                             would handle
+
+## Video Selection:
+    --playlist-start NUMBER  playlist video to start at (default is 1)
+    --playlist-end NUMBER    playlist video to end at (default is last)
+    --match-title REGEX      download only matching titles (regex or caseless
+                             sub-string)
+    --reject-title REGEX     skip download for matching titles (regex or
+                             caseless sub-string)
+    --max-downloads NUMBER   Abort after downloading NUMBER files
+
+## Filesystem Options:
+    -t, --title              use title in file name
+    --id                     use video ID in file name
+    -l, --literal            [deprecated] alias of --title
+    -A, --auto-number        number downloaded files starting from 00000
+    -o, --output TEMPLATE    output filename template. Use %(title)s to get the
+                             title, %(uploader)s for the uploader name,
+                             %(autonumber)s to get an automatically incremented
+                             number, %(ext)s for the filename extension,
+                             %(upload_date)s for the upload date (YYYYMMDD),
+                             %(extractor)s for the provider (youtube, metacafe,
+                             etc), %(id)s for the video id and %% for a literal
+                             percent. Use - to output to stdout.
+    --restrict-filenames     Avoid some characters such as "&" and spaces in
+                             filenames
+    -a, --batch-file FILE    file containing URLs to download ('-' for stdin)
+    -w, --no-overwrites      do not overwrite files
+    -c, --continue           resume partially downloaded files
+    --no-continue            do not resume partially downloaded files (restart
+                             from beginning)
+    --cookies FILE           file to read cookies from and dump cookie jar in
+    --no-part                do not use .part files
+    --no-mtime               do not use the Last-modified header to set the file
+                             modification time
+    --write-description      write video description to a .description file
+    --write-info-json        write video metadata to a .info.json file
+
+## Verbosity / Simulation Options:
+    -q, --quiet              activates quiet mode
+    -s, --simulate           do not download the video and do not write anything
+                             to disk
+    --skip-download          do not download the video
+    -g, --get-url            simulate, quiet but print URL
+    -e, --get-title          simulate, quiet but print title
+    --get-thumbnail          simulate, quiet but print thumbnail URL
+    --get-description        simulate, quiet but print video description
+    --get-filename           simulate, quiet but print output filename
+    --get-format             simulate, quiet but print output format
+    --no-progress            do not print progress bar
+    --console-title          display progress in console titlebar
+    -v, --verbose            print various debugging information
+
+## Video Format Options:
+    -f, --format FORMAT      video format code
+    --all-formats            download all available video formats
+    --prefer-free-formats    prefer free video formats unless a specific one is
+                             requested
+    --max-quality FORMAT     highest quality format to download
+    -F, --list-formats       list all available formats (currently youtube only)
+    --write-srt              write video closed captions to a .srt file
+                             (currently youtube only)
+    --srt-lang LANG          language of the closed captions to download
+                             (optional) use IETF language tags like 'en'
+
+## Authentication Options:
+    -u, --username USERNAME  account username
+    -p, --password PASSWORD  account password
+    -n, --netrc              use .netrc authentication data
+
+## Post-processing Options:
+    -x, --extract-audio      convert video files to audio-only files (requires
+                             ffmpeg or avconv and ffprobe or avprobe)
+    --audio-format FORMAT    "best", "aac", "vorbis", "mp3", "m4a", or "wav";
+                             best by default
+    --audio-quality QUALITY  ffmpeg/avconv audio quality specification, insert a
+                             value between 0 (better) and 9 (worse) for VBR or a
+                             specific bitrate like 128K (default 5)
+    -k, --keep-video         keeps the video file on disk after the post-
+                             processing; the video is erased by default
+
+# CONFIGURATION
+
+You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.local/config/youtube-dl.conf`.
+
+# FAQ
+
+### Can you please put the -b option back?
+
+Most people asking this question are not aware that youtube-dl now defaults to downloading the highest available quality as reported by YouTube, which will be 1080p or 720p in some cases, so you no longer need the -b option. For some specific videos, maybe YouTube does not report them to be available in a specific high quality format you''re interested in. In that case, simply request it with the -f option and youtube-dl will try to download it.
+
+### I get HTTP error 402 when trying to download a video. What's this?
+
+Apparently YouTube requires you to pass a CAPTCHA test if you download too much. We''re [considering to provide a way to let you solve the CAPTCHA](https://github.com/rg3/youtube-dl/issues/154), but at the moment, your best course of action is pointing a webbrowser to the youtube URL, solving the CAPTCHA, and restart youtube-dl.
+
+### I have downloaded a video but how can I play it?
+
+Once the video is fully downloaded, use any video player, such as [vlc](http://www.videolan.org) or [mplayer](http://www.mplayerhq.hu/).
+
+### The links provided by youtube-dl -g are not working anymore
+
+The URLs youtube-dl outputs require the downloader to have the correct cookies. Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. Some sites also require a common user agent to be used, use `--dump-user-agent` to see the one in use by youtube-dl.
+
+### ERROR: no fmt_url_map or conn information found in video info
+
+youtube has switched to a new video info format in July 2011 which is not supported by old versions of youtube-dl. You can update youtube-dl with `sudo youtube-dl --update`.
+
+### ERROR: unable to download video ###
+
+youtube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. You can update youtube-dl with `sudo youtube-dl --update`.
+
+### SyntaxError: Non-ASCII character ###
+
+The error
+
+    File "youtube-dl", line 2
+    SyntaxError: Non-ASCII character '\x93' ...
+
+means you're using an outdated version of Python. Please update to Python 2.6 or 2.7.
+
+To run youtube-dl under Python 2.5, you'll have to manually check it out like this:
+
+	git clone git://github.com/rg3/youtube-dl.git
+	cd youtube-dl
+	python -m youtube_dl --help
+
+Please note that Python 2.5 is not supported anymore.
+
+### What is this binary file? Where has the code gone?
+
+Since June 2012 (#342) youtube-dl is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.
+
+### The exe throws a *Runtime error from Visual C++*
+
+To run the exe you need to install first the [Microsoft Visual C++ 2008 Redistributable Package](http://www.microsoft.com/en-us/download/details.aspx?id=29).
+
+# COPYRIGHT
+
+youtube-dl is released into the public domain by the copyright holders.
+
+This README file was originally written by Daniel Bolton (<https://github.com/dbbolton>) and is likewise released into the public domain.
+
+# BUGS
+
+Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues>
+
+Please include:
+
+* Your exact command line, like `youtube-dl -t "http://www.youtube.com/watch?v=uHlDtZ6Oc3s&feature=channel_video_title"`. A common mistake is not to escape the `&`. Putting URLs in quotes should solve this problem.
+* The output of `youtube-dl --version`
+* The output of `python --version`
+* The name and version of your Operating System ("Ubuntu 11.04 x64" or "Windows 7 x64" is usually enough).

build_exe.py

@@ -0,0 +1,48 @@
+from distutils.core import setup
+import py2exe
+import sys, os
+
+"""This will create an exe that needs Microsoft Visual C++ 2008 Redistributable Package"""
+
+# If run without args, build executables
+if len(sys.argv) == 1:
+    sys.argv.append("py2exe")
+
+# os.chdir(os.path.dirname(os.path.abspath(sys.argv[0]))) # conflict with wine-py2exe.sh
+sys.path.append('./youtube_dl')
+
+options = {
+    "bundle_files": 1,
+    "compressed": 1,
+    "optimize": 2,
+    "dist_dir": '.',
+    "dll_excludes": ['w9xpopen.exe']
+}
+
+console = [{
+    "script":"./youtube_dl/__main__.py",
+    "dest_base": "youtube-dl",
+}]
+
+init_file = open('./youtube_dl/__init__.py')
+for line in init_file.readlines():
+    if line.startswith('__version__'):
+        version = line[11:].strip(" ='\n")
+        break
+else:
+    version = ''
+
+setup(name='youtube-dl',
+      version=version,
+      description='Small command-line program to download videos from YouTube.com and other video sites',
+      url='https://github.com/rg3/youtube-dl',
+      packages=['youtube_dl'],
+      
+      console = console,
+      options = {"py2exe": options},
+      zipfile = None,
+)
+
+import shutil
+shutil.rmtree("build")
+

devscripts/posix-locale.sh

@@ -0,0 +1,6 @@
+
+# source this file in your shell to get a POSIX locale (which will break many programs, but that's kind of the point)
+
+export LC_ALL=POSIX
+export LANG=POSIX
+export LANGUAGE=POSIX

devscripts/release.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+if [ -z "$1" ]; then echo "ERROR: specify version number like this: $0 1994.09.06"; exit 1; fi
+version="$1"
+if [ ! -z "`git tag | grep "$version"`" ]; then echo 'ERROR: version already present'; exit 1; fi
+if [ ! -z "`git status --porcelain`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi
+sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/__init__.py
+make all
+git add -A
+git commit -m "release $version"
+git tag -m "Release $version" "$version"

devscripts/wine-py2exe.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Run with as parameter a setup.py that works in the current directory
+# e.g. no os.chdir()
+# It will run twice, the first time will crash
+
+set -e
+
+SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
+
+if [ ! -d wine-py2exe ]; then
+
+    sudo apt-get install wine1.3 axel bsdiff
+
+    mkdir wine-py2exe
+    cd wine-py2exe
+    export WINEPREFIX=`pwd`
+
+    axel -a "http://www.python.org/ftp/python/2.7/python-2.7.msi"
+    axel -a "http://downloads.sourceforge.net/project/py2exe/py2exe/0.6.9/py2exe-0.6.9.win32-py2.7.exe"
+    #axel -a "http://winetricks.org/winetricks"
+
+    # http://appdb.winehq.org/objectManager.php?sClass=version&iId=21957
+    echo "Follow python setup on screen"
+    wine msiexec /i python-2.7.msi
+    
+    echo "Follow py2exe setup on screen"
+    wine py2exe-0.6.9.win32-py2.7.exe
+    
+    #echo "Follow Microsoft Visual C++ 2008 Redistributable Package setup on screen"
+    #bash winetricks vcrun2008
+
+    rm py2exe-0.6.9.win32-py2.7.exe
+    rm python-2.7.msi
+    #rm winetricks
+    
+    # http://bugs.winehq.org/show_bug.cgi?id=3591
+    
+    mv drive_c/Python27/Lib/site-packages/py2exe/run.exe drive_c/Python27/Lib/site-packages/py2exe/run.exe.backup
+    bspatch drive_c/Python27/Lib/site-packages/py2exe/run.exe.backup drive_c/Python27/Lib/site-packages/py2exe/run.exe "$SCRIPT_DIR/SizeOfImage.patch"
+    mv drive_c/Python27/Lib/site-packages/py2exe/run_w.exe drive_c/Python27/Lib/site-packages/py2exe/run_w.exe.backup
+    bspatch drive_c/Python27/Lib/site-packages/py2exe/run_w.exe.backup drive_c/Python27/Lib/site-packages/py2exe/run_w.exe "$SCRIPT_DIR/SizeOfImage_w.patch"
+
+    cd -
+    
+else
+
+    export WINEPREFIX="$( cd wine-py2exe && pwd )"
+
+fi
+
+wine "C:\\Python27\\python.exe" "$1" py2exe > "py2exe.log" 2>&1 || true
+echo '# Copying python27.dll' >> "py2exe.log"
+cp "$WINEPREFIX/drive_c/windows/system32/python27.dll" build/bdist.win32/winexe/bundle-2.7/
+wine "C:\\Python27\\python.exe" "$1" py2exe >> "py2exe.log" 2>&1
+

test/parameters.json

@@ -0,0 +1 @@
+{"username": null, "listformats": null, "skip_download": false, "usenetrc": false, "max_downloads": null, "noprogress": false, "forcethumbnail": false, "forceformat": false, "format_limit": null, "ratelimit": null, "nooverwrites": false, "forceurl": false, "writeinfojson": false, "simulate": false, "playliststart": 1, "continuedl": true, "password": null, "prefer_free_formats": false, "nopart": false, "retries": 10, "updatetime": true, "consoletitle": false, "verbose": true, "forcefilename": false, "ignoreerrors": false, "logtostderr": false, "format": null, "subtitleslang": null, "quiet": false, "outtmpl": "%(id)s.%(ext)s", "rejecttitle": null, "playlistend": -1, "writedescription": false, "forcetitle": false, "forcedescription": false, "writesubtitles": false, "matchtitle": null}

test/test_download.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python2
+import unittest
+import hashlib
+import os
+import json
+
+from youtube_dl.FileDownloader import FileDownloader
+from youtube_dl.InfoExtractors  import YoutubeIE, DailymotionIE
+from youtube_dl.InfoExtractors import  MetacafeIE, BlipTVIE
+
+
+class DownloadTest(unittest.TestCase):
+	PARAMETERS_FILE = "test/parameters.json"
+	#calculated with md5sum:
+	#md5sum (GNU coreutils) 8.19
+
+	YOUTUBE_SIZE = 1993883
+	YOUTUBE_URL = "http://www.youtube.com/watch?v=BaW_jenozKc"
+	YOUTUBE_FILE = "BaW_jenozKc.mp4"
+
+	DAILYMOTION_MD5 = "d363a50e9eb4f22ce90d08d15695bb47"
+	DAILYMOTION_URL = "http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech"
+	DAILYMOTION_FILE = "x33vw9.mp4"
+
+	METACAFE_SIZE = 5754305
+	METACAFE_URL = "http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/"
+	METACAFE_FILE = "_aUehQsCQtM.flv"
+
+	BLIP_MD5 = "93c24d2f4e0782af13b8a7606ea97ba7"
+	BLIP_URL = "http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352"
+	BLIP_FILE = "5779306.m4v"
+
+	XVIDEO_MD5 = ""
+	XVIDEO_URL = ""
+	XVIDEO_FILE = ""
+
+
+	def test_youtube(self):
+		#let's download a file from youtube
+		with open(DownloadTest.PARAMETERS_FILE) as f:
+			fd = FileDownloader(json.load(f))
+		fd.add_info_extractor(YoutubeIE())
+		fd.download([DownloadTest.YOUTUBE_URL])
+		self.assertTrue(os.path.exists(DownloadTest.YOUTUBE_FILE))
+		self.assertEqual(os.path.getsize(DownloadTest.YOUTUBE_FILE), DownloadTest.YOUTUBE_SIZE)
+
+	def test_dailymotion(self):
+		with open(DownloadTest.PARAMETERS_FILE) as f:
+			fd = FileDownloader(json.load(f))
+		fd.add_info_extractor(DailymotionIE())
+		fd.download([DownloadTest.DAILYMOTION_URL])
+		self.assertTrue(os.path.exists(DownloadTest.DAILYMOTION_FILE))
+		md5_down_file = md5_for_file(DownloadTest.DAILYMOTION_FILE)
+		self.assertEqual(md5_down_file, DownloadTest.DAILYMOTION_MD5)
+
+	def test_metacafe(self):
+		#this emulate a skip,to be 2.6 compatible
+		with open(DownloadTest.PARAMETERS_FILE) as f:
+			fd = FileDownloader(json.load(f))
+		fd.add_info_extractor(MetacafeIE())
+		fd.add_info_extractor(YoutubeIE())
+		fd.download([DownloadTest.METACAFE_URL])
+		self.assertTrue(os.path.exists(DownloadTest.METACAFE_FILE))
+		self.assertEqual(os.path.getsize(DownloadTest.METACAFE_FILE), DownloadTest.METACAFE_SIZE)
+
+	def test_blip(self):
+		with open(DownloadTest.PARAMETERS_FILE) as f:
+			fd = FileDownloader(json.load(f))
+		fd.add_info_extractor(BlipTVIE())
+		fd.download([DownloadTest.BLIP_URL])
+		self.assertTrue(os.path.exists(DownloadTest.BLIP_FILE))
+		md5_down_file = md5_for_file(DownloadTest.BLIP_FILE)
+		self.assertEqual(md5_down_file, DownloadTest.BLIP_MD5)
+
+	def tearDown(self):
+		if os.path.exists(DownloadTest.YOUTUBE_FILE):
+			os.remove(DownloadTest.YOUTUBE_FILE)
+		if os.path.exists(DownloadTest.DAILYMOTION_FILE):
+			os.remove(DownloadTest.DAILYMOTION_FILE)
+		if os.path.exists(DownloadTest.METACAFE_FILE):
+			os.remove(DownloadTest.METACAFE_FILE)
+		if os.path.exists(DownloadTest.BLIP_FILE):
+			os.remove(DownloadTest.BLIP_FILE)
+
+def md5_for_file(filename, block_size=2**20):
+    with open(filename) as f:
+        md5 = hashlib.md5()
+        while True:
+            data = f.read(block_size)
+            if not data:
+                break
+            md5.update(data)
+            return md5.hexdigest()

test/test_utils.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+
+# Various small unit tests
+
+import unittest
+
+#from youtube_dl.utils import htmlentity_transform
+from youtube_dl.utils import timeconvert
+from youtube_dl.utils import sanitize_filename
+from youtube_dl.utils import unescapeHTML
+from youtube_dl.utils import orderedSet
+
+
+class TestUtil(unittest.TestCase):
+	def test_timeconvert(self):
+		self.assertTrue(timeconvert('') is None)
+		self.assertTrue(timeconvert('bougrg') is None)
+
+	def test_sanitize_filename(self):
+		self.assertEqual(sanitize_filename(u'abc'), u'abc')
+		self.assertEqual(sanitize_filename(u'abc_d-e'), u'abc_d-e')
+
+		self.assertEqual(sanitize_filename(u'123'), u'123')
+
+		self.assertEqual(u'abc-de', sanitize_filename(u'abc/de'))
+		self.assertFalse(u'/' in sanitize_filename(u'abc/de///'))
+
+		self.assertEqual(u'abc-de', sanitize_filename(u'abc/<>\\*|de'))
+		self.assertEqual(u'xxx', sanitize_filename(u'xxx/<>\\*|'))
+		self.assertEqual(u'yes no', sanitize_filename(u'yes? no'))
+		self.assertEqual(u'this - that', sanitize_filename(u'this: that'))
+
+		self.assertEqual(sanitize_filename(u'AT&T'), u'AT&T')
+		self.assertEqual(sanitize_filename(u'ä'), u'ä')
+		self.assertEqual(sanitize_filename(u'кириллица'), u'кириллица')
+
+		forbidden = u'"\0\\/'
+		for fc in forbidden:
+			for fbc in forbidden:
+				self.assertTrue(fbc not in sanitize_filename(fc))
+
+	def test_sanitize_filename_restricted(self):
+		self.assertEqual(sanitize_filename(u'abc', restricted=True), u'abc')
+		self.assertEqual(sanitize_filename(u'abc_d-e', restricted=True), u'abc_d-e')
+
+		self.assertEqual(sanitize_filename(u'123', restricted=True), u'123')
+
+		self.assertEqual(u'abc-de', sanitize_filename(u'abc/de', restricted=True))
+		self.assertFalse(u'/' in sanitize_filename(u'abc/de///', restricted=True))
+
+		self.assertEqual(u'abc-de', sanitize_filename(u'abc/<>\\*|de', restricted=True))
+		self.assertEqual(u'xxx', sanitize_filename(u'xxx/<>\\*|', restricted=True))
+		self.assertEqual(u'yes_no', sanitize_filename(u'yes? no', restricted=True))
+		self.assertEqual(u'this_-_that', sanitize_filename(u'this: that', restricted=True))
+
+		forbidden = u'"\0\\/&: \'\t\n'
+		for fc in forbidden:
+			print('input: ' + fc + ', result: ' + repr(sanitize_filename(fc, restricted=True)))
+			for fbc in forbidden:
+				self.assertTrue(fbc not in sanitize_filename(fc, restricted=True))
+
+	def test_ordered_set(self):
+		self.assertEqual(orderedSet([1,1,2,3,4,4,5,6,7,3,5]), [1,2,3,4,5,6,7])
+		self.assertEqual(orderedSet([]), [])
+		self.assertEqual(orderedSet([1]), [1])
+		#keep the list ordered
+		self.assertEqual(orderedSet([135,1,1,1]), [135,1])
+
+	def test_unescape_html(self):
+		self.assertEqual(unescapeHTML(u"%20;"), u"%20;")

youtube-dl.1

@@ -0,0 +1,248 @@
+.TH youtube-dl 1 "" 
+.SH NAME
+.PP
+youtube-dl
+.SH SYNOPSIS
+.PP
+\f[B]youtube-dl\f[] [OPTIONS] URL [URL...]
+.SH DESCRIPTION
+.PP
+\f[B]youtube-dl\f[] is a small command-line program to download videos
+from YouTube.com and a few more sites.
+It requires the Python interpreter, version 2.x (x being at least 6),
+and it is not platform specific.
+It should work in your Unix box, in Windows or in Mac OS X.
+It is released to the public domain, which means you can modify it,
+redistribute it or use it however you like.
+.SH OPTIONS
+.IP
+.nf
+\f[C]
+-h,\ --help\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ print\ this\ help\ text\ and\ exit
+--version\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ print\ program\ version\ and\ exit
+-U,\ --update\ \ \ \ \ \ \ \ \ \ \ \ \ update\ this\ program\ to\ latest\ version
+-i,\ --ignore-errors\ \ \ \ \ \ continue\ on\ download\ errors
+-r,\ --rate-limit\ LIMIT\ \ \ download\ rate\ limit\ (e.g.\ 50k\ or\ 44.6m)
+-R,\ --retries\ RETRIES\ \ \ \ number\ of\ retries\ (default\ is\ 10)
+--dump-user-agent\ \ \ \ \ \ \ \ display\ the\ current\ browser\ identification
+--user-agent\ UA\ \ \ \ \ \ \ \ \ \ specify\ a\ custom\ user\ agent
+--list-extractors\ \ \ \ \ \ \ \ List\ all\ supported\ extractors\ and\ the\ URLs\ they
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ would\ handle
+\f[]
+.fi
+.SS Video Selection:
+.IP
+.nf
+\f[C]
+--playlist-start\ NUMBER\ \ playlist\ video\ to\ start\ at\ (default\ is\ 1)
+--playlist-end\ NUMBER\ \ \ \ playlist\ video\ to\ end\ at\ (default\ is\ last)
+--match-title\ REGEX\ \ \ \ \ \ download\ only\ matching\ titles\ (regex\ or\ caseless
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ sub-string)
+--reject-title\ REGEX\ \ \ \ \ skip\ download\ for\ matching\ titles\ (regex\ or
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ caseless\ sub-string)
+--max-downloads\ NUMBER\ \ \ Abort\ after\ downloading\ NUMBER\ files
+\f[]
+.fi
+.SS Filesystem Options:
+.IP
+.nf
+\f[C]
+-t,\ --title\ \ \ \ \ \ \ \ \ \ \ \ \ \ use\ title\ in\ file\ name
+--id\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ use\ video\ ID\ in\ file\ name
+-l,\ --literal\ \ \ \ \ \ \ \ \ \ \ \ [deprecated]\ alias\ of\ --title
+-A,\ --auto-number\ \ \ \ \ \ \ \ number\ downloaded\ files\ starting\ from\ 00000
+-o,\ --output\ TEMPLATE\ \ \ \ output\ filename\ template.\ Use\ %(title)s\ to\ get\ the
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ title,\ %(uploader)s\ for\ the\ uploader\ name,
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(autonumber)s\ to\ get\ an\ automatically\ incremented
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ number,\ %(ext)s\ for\ the\ filename\ extension,
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(upload_date)s\ for\ the\ upload\ date\ (YYYYMMDD),
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(extractor)s\ for\ the\ provider\ (youtube,\ metacafe,
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ etc),\ %(id)s\ for\ the\ video\ id\ and\ %%\ for\ a\ literal
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ percent.\ Use\ -\ to\ output\ to\ stdout.
+--restrict-filenames\ \ \ \ \ Avoid\ some\ characters\ such\ as\ "&"\ and\ spaces\ in
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ filenames
+-a,\ --batch-file\ FILE\ \ \ \ file\ containing\ URLs\ to\ download\ (\[aq]-\[aq]\ for\ stdin)
+-w,\ --no-overwrites\ \ \ \ \ \ do\ not\ overwrite\ files
+-c,\ --continue\ \ \ \ \ \ \ \ \ \ \ resume\ partially\ downloaded\ files
+--no-continue\ \ \ \ \ \ \ \ \ \ \ \ do\ not\ resume\ partially\ downloaded\ files\ (restart
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ from\ beginning)
+--cookies\ FILE\ \ \ \ \ \ \ \ \ \ \ file\ to\ read\ cookies\ from\ and\ dump\ cookie\ jar\ in
+--no-part\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ do\ not\ use\ .part\ files
+--no-mtime\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ do\ not\ use\ the\ Last-modified\ header\ to\ set\ the\ file
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ modification\ time
+--write-description\ \ \ \ \ \ write\ video\ description\ to\ a\ .description\ file
+--write-info-json\ \ \ \ \ \ \ \ write\ video\ metadata\ to\ a\ .info.json\ file
+\f[]
+.fi
+.SS Verbosity / Simulation Options:
+.IP
+.nf
+\f[C]
+-q,\ --quiet\ \ \ \ \ \ \ \ \ \ \ \ \ \ activates\ quiet\ mode
+-s,\ --simulate\ \ \ \ \ \ \ \ \ \ \ do\ not\ download\ the\ video\ and\ do\ not\ write\ anything
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ to\ disk
+--skip-download\ \ \ \ \ \ \ \ \ \ do\ not\ download\ the\ video
+-g,\ --get-url\ \ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ URL
+-e,\ --get-title\ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ title
+--get-thumbnail\ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ thumbnail\ URL
+--get-description\ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ video\ description
+--get-filename\ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ output\ filename
+--get-format\ \ \ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ output\ format
+--no-progress\ \ \ \ \ \ \ \ \ \ \ \ do\ not\ print\ progress\ bar
+--console-title\ \ \ \ \ \ \ \ \ \ display\ progress\ in\ console\ titlebar
+-v,\ --verbose\ \ \ \ \ \ \ \ \ \ \ \ print\ various\ debugging\ information
+\f[]
+.fi
+.SS Video Format Options:
+.IP
+.nf
+\f[C]
+-f,\ --format\ FORMAT\ \ \ \ \ \ video\ format\ code
+--all-formats\ \ \ \ \ \ \ \ \ \ \ \ download\ all\ available\ video\ formats
+--prefer-free-formats\ \ \ \ prefer\ free\ video\ formats\ unless\ a\ specific\ one\ is
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ requested
+--max-quality\ FORMAT\ \ \ \ \ highest\ quality\ format\ to\ download
+-F,\ --list-formats\ \ \ \ \ \ \ list\ all\ available\ formats\ (currently\ youtube\ only)
+--write-srt\ \ \ \ \ \ \ \ \ \ \ \ \ \ write\ video\ closed\ captions\ to\ a\ .srt\ file
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (currently\ youtube\ only)
+--srt-lang\ LANG\ \ \ \ \ \ \ \ \ \ language\ of\ the\ closed\ captions\ to\ download
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (optional)\ use\ IETF\ language\ tags\ like\ \[aq]en\[aq]
+\f[]
+.fi
+.SS Authentication Options:
+.IP
+.nf
+\f[C]
+-u,\ --username\ USERNAME\ \ account\ username
+-p,\ --password\ PASSWORD\ \ account\ password
+-n,\ --netrc\ \ \ \ \ \ \ \ \ \ \ \ \ \ use\ .netrc\ authentication\ data
+\f[]
+.fi
+.SS Post-processing Options:
+.IP
+.nf
+\f[C]
+-x,\ --extract-audio\ \ \ \ \ \ convert\ video\ files\ to\ audio-only\ files\ (requires
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ffmpeg\ or\ avconv\ and\ ffprobe\ or\ avprobe)
+--audio-format\ FORMAT\ \ \ \ "best",\ "aac",\ "vorbis",\ "mp3",\ "m4a",\ or\ "wav";
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ best\ by\ default
+--audio-quality\ QUALITY\ \ ffmpeg/avconv\ audio\ quality\ specification,\ insert\ a
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ value\ between\ 0\ (better)\ and\ 9\ (worse)\ for\ VBR\ or\ a
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ specific\ bitrate\ like\ 128K\ (default\ 5)
+-k,\ --keep-video\ \ \ \ \ \ \ \ \ keeps\ the\ video\ file\ on\ disk\ after\ the\ post-
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ processing;\ the\ video\ is\ erased\ by\ default
+\f[]
+.fi
+.SH CONFIGURATION
+.PP
+You can configure youtube-dl by placing default arguments (such as
+\f[C]--extract-audio\ --no-mtime\f[] to always extract the audio and not
+copy the mtime) into \f[C]/etc/youtube-dl.conf\f[] and/or
+\f[C]~/.local/config/youtube-dl.conf\f[].
+.SH FAQ
+.SS Can you please put the -b option back?
+.PP
+Most people asking this question are not aware that youtube-dl now
+defaults to downloading the highest available quality as reported by
+YouTube, which will be 1080p or 720p in some cases, so you no longer
+need the -b option.
+For some specific videos, maybe YouTube does not report them to be
+available in a specific high quality format you\[aq]\[aq]re interested
+in.
+In that case, simply request it with the -f option and youtube-dl will
+try to download it.
+.SS I get HTTP error 402 when trying to download a video. What\[aq]s
+this?
+.PP
+Apparently YouTube requires you to pass a CAPTCHA test if you download
+too much.
+We\[aq]\[aq]re considering to provide a way to let you solve the
+CAPTCHA (https://github.com/rg3/youtube-dl/issues/154), but at the
+moment, your best course of action is pointing a webbrowser to the
+youtube URL, solving the CAPTCHA, and restart youtube-dl.
+.SS I have downloaded a video but how can I play it?
+.PP
+Once the video is fully downloaded, use any video player, such as
+vlc (http://www.videolan.org) or mplayer (http://www.mplayerhq.hu/).
+.SS The links provided by youtube-dl -g are not working anymore
+.PP
+The URLs youtube-dl outputs require the downloader to have the correct
+cookies.
+Use the \f[C]--cookies\f[] option to write the required cookies into a
+file, and advise your downloader to read cookies from that file.
+Some sites also require a common user agent to be used, use
+\f[C]--dump-user-agent\f[] to see the one in use by youtube-dl.
+.SS ERROR: no fmt_url_map or conn information found in video info
+.PP
+youtube has switched to a new video info format in July 2011 which is
+not supported by old versions of youtube-dl.
+You can update youtube-dl with \f[C]sudo\ youtube-dl\ --update\f[].
+.SS ERROR: unable to download video
+.PP
+youtube requires an additional signature since September 2012 which is
+not supported by old versions of youtube-dl.
+You can update youtube-dl with \f[C]sudo\ youtube-dl\ --update\f[].
+.SS SyntaxError: Non-ASCII character
+.PP
+The error
+.IP
+.nf
+\f[C]
+File\ "youtube-dl",\ line\ 2
+SyntaxError:\ Non-ASCII\ character\ \[aq]\\x93\[aq]\ ...
+\f[]
+.fi
+.PP
+means you\[aq]re using an outdated version of Python.
+Please update to Python 2.6 or 2.7.
+.PP
+To run youtube-dl under Python 2.5, you\[aq]ll have to manually check it
+out like this:
+.IP
+.nf
+\f[C]
+git\ clone\ git://github.com/rg3/youtube-dl.git
+cd\ youtube-dl
+python\ -m\ youtube_dl\ --help
+\f[]
+.fi
+.PP
+Please note that Python 2.5 is not supported anymore.
+.SS What is this binary file? Where has the code gone?
+.PP
+Since June 2012 (#342) youtube-dl is packed as an executable zipfile,
+simply unzip it (might need renaming to \f[C]youtube-dl.zip\f[] first on
+some systems) or clone the git repository, as laid out above.
+If you modify the code, you can run it by executing the
+\f[C]__main__.py\f[] file.
+To recompile the executable, run \f[C]make\ youtube-dl\f[].
+.SS The exe throws a \f[I]Runtime error from Visual C++\f[]
+.PP
+To run the exe you need to install first the Microsoft Visual C++ 2008
+Redistributable
+Package (http://www.microsoft.com/en-us/download/details.aspx?id=29).
+.SH COPYRIGHT
+.PP
+youtube-dl is released into the public domain by the copyright holders.
+.PP
+This README file was originally written by Daniel Bolton
+(<https://github.com/dbbolton>) and is likewise released into the public
+domain.
+.SH BUGS
+.PP
+Bugs and suggestions should be reported at:
+<https://github.com/rg3/youtube-dl/issues>
+.PP
+Please include:
+.IP \[bu] 2
+Your exact command line, like
+\f[C]youtube-dl\ -t\ "http://www.youtube.com/watch?v=uHlDtZ6Oc3s&feature=channel_video_title"\f[].
+A common mistake is not to escape the \f[C]&\f[].
+Putting URLs in quotes should solve this problem.
+.IP \[bu] 2
+The output of \f[C]youtube-dl\ --version\f[]
+.IP \[bu] 2
+The output of \f[C]python\ --version\f[]
+.IP \[bu] 2
+The name and version of your Operating System ("Ubuntu 11.04 x64" or
+"Windows 7 x64" is usually enough).

youtube-dl.bash-completion

@@ -0,0 +1,14 @@
+__youtube-dl()
+{
+    local cur prev opts
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    opts="--all-formats --audio-format --audio-quality --auto-number --batch-file --console-title --continue --cookies --dump-user-agent --extract-audio --format --get-description --get-filename --get-format --get-thumbnail --get-title --get-url --help --id --ignore-errors --keep-video --list-extractors --list-formats --literal --match-title --max-downloads --max-quality --netrc --no-continue --no-mtime --no-overwrites --no-part --no-progress --output --password --playlist-end --playlist-start --prefer-free-formats --quiet --rate-limit --reject-title --restrict-filenames --retries --simulate --skip-download --srt-lang --title --update --user-agent --username --verbose --version --write-description --write-info-json --write-srt"
+
+    if [[ ${cur} == * ]] ; then
+        COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+        return 0
+    fi
+}
+
+complete -F __youtube-dl youtube-dl

youtube_dl/FileDownloader.py

@@ -0,0 +1,696 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import httplib
+import math
+import os
+import re
+import socket
+import subprocess
+import sys
+import time
+import urllib2
+
+if os.name == 'nt':
+	import ctypes
+
+from utils import *
+
+
+class FileDownloader(object):
+	"""File Downloader class.
+
+	File downloader objects are the ones responsible of downloading the
+	actual video file and writing it to disk if the user has requested
+	it, among some other tasks. In most cases there should be one per
+	program. As, given a video URL, the downloader doesn't know how to
+	extract all the needed information, task that InfoExtractors do, it
+	has to pass the URL to one of them.
+
+	For this, file downloader objects have a method that allows
+	InfoExtractors to be registered in a given order. When it is passed
+	a URL, the file downloader handles it to the first InfoExtractor it
+	finds that reports being able to handle it. The InfoExtractor extracts
+	all the information about the video or videos the URL refers to, and
+	asks the FileDownloader to process the video information, possibly
+	downloading the video.
+
+	File downloaders accept a lot of parameters. In order not to saturate
+	the object constructor with arguments, it receives a dictionary of
+	options instead. These options are available through the params
+	attribute for the InfoExtractors to use. The FileDownloader also
+	registers itself as the downloader in charge for the InfoExtractors
+	that are added to it, so this is a "mutual registration".
+
+	Available options:
+
+	username:          Username for authentication purposes.
+	password:          Password for authentication purposes.
+	usenetrc:          Use netrc for authentication instead.
+	quiet:             Do not print messages to stdout.
+	forceurl:          Force printing final URL.
+	forcetitle:        Force printing title.
+	forcethumbnail:    Force printing thumbnail URL.
+	forcedescription:  Force printing description.
+	forcefilename:     Force printing final filename.
+	simulate:          Do not download the video files.
+	format:            Video format code.
+	format_limit:      Highest quality format to try.
+	outtmpl:           Template for output names.
+	restrictfilenames: Do not allow "&" and spaces in file names
+	ignoreerrors:      Do not stop on download errors.
+	ratelimit:         Download speed limit, in bytes/sec.
+	nooverwrites:      Prevent overwriting files.
+	retries:           Number of times to retry for HTTP error 5xx
+	continuedl:        Try to continue downloads if possible.
+	noprogress:        Do not print the progress bar.
+	playliststart:     Playlist item to start at.
+	playlistend:       Playlist item to end at.
+	matchtitle:        Download only matching titles.
+	rejecttitle:       Reject downloads for matching titles.
+	logtostderr:       Log messages to stderr instead of stdout.
+	consoletitle:      Display progress in console window's titlebar.
+	nopart:            Do not use temporary .part files.
+	updatetime:        Use the Last-modified header to set output file timestamps.
+	writedescription:  Write the video description to a .description file
+	writeinfojson:     Write the video description to a .info.json file
+	writesubtitles:    Write the video subtitles to a .srt file
+	subtitleslang:     Language of the subtitles to download
+	"""
+
+	params = None
+	_ies = []
+	_pps = []
+	_download_retcode = None
+	_num_downloads = None
+	_screen_file = None
+
+	def __init__(self, params):
+		"""Create a FileDownloader object with the given options."""
+		self._ies = []
+		self._pps = []
+		self._download_retcode = 0
+		self._num_downloads = 0
+		self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
+		self.params = params
+
+	@staticmethod
+	def format_bytes(bytes):
+		if bytes is None:
+			return 'N/A'
+		if type(bytes) is str:
+			bytes = float(bytes)
+		if bytes == 0.0:
+			exponent = 0
+		else:
+			exponent = long(math.log(bytes, 1024.0))
+		suffix = 'bkMGTPEZY'[exponent]
+		converted = float(bytes) / float(1024 ** exponent)
+		return '%.2f%s' % (converted, suffix)
+
+	@staticmethod
+	def calc_percent(byte_counter, data_len):
+		if data_len is None:
+			return '---.-%'
+		return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
+
+	@staticmethod
+	def calc_eta(start, now, total, current):
+		if total is None:
+			return '--:--'
+		dif = now - start
+		if current == 0 or dif < 0.001: # One millisecond
+			return '--:--'
+		rate = float(current) / dif
+		eta = long((float(total) - float(current)) / rate)
+		(eta_mins, eta_secs) = divmod(eta, 60)
+		if eta_mins > 99:
+			return '--:--'
+		return '%02d:%02d' % (eta_mins, eta_secs)
+
+	@staticmethod
+	def calc_speed(start, now, bytes):
+		dif = now - start
+		if bytes == 0 or dif < 0.001: # One millisecond
+			return '%10s' % '---b/s'
+		return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
+
+	@staticmethod
+	def best_block_size(elapsed_time, bytes):
+		new_min = max(bytes / 2.0, 1.0)
+		new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
+		if elapsed_time < 0.001:
+			return int(new_max)
+		rate = bytes / elapsed_time
+		if rate > new_max:
+			return int(new_max)
+		if rate < new_min:
+			return int(new_min)
+		return int(rate)
+
+	@staticmethod
+	def parse_bytes(bytestr):
+		"""Parse a string indicating a byte quantity into an integer."""
+		matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
+		if matchobj is None:
+			return None
+		number = float(matchobj.group(1))
+		multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
+		return int(round(number * multiplier))
+
+	def add_info_extractor(self, ie):
+		"""Add an InfoExtractor object to the end of the list."""
+		self._ies.append(ie)
+		ie.set_downloader(self)
+
+	def add_post_processor(self, pp):
+		"""Add a PostProcessor object to the end of the chain."""
+		self._pps.append(pp)
+		pp.set_downloader(self)
+
+	def to_screen(self, message, skip_eol=False):
+		"""Print message to stdout if not in quiet mode."""
+		assert type(message) == type(u'')
+		if not self.params.get('quiet', False):
+			terminator = [u'\n', u''][skip_eol]
+			output = message + terminator
+			if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
+				output = output.encode(preferredencoding(), 'ignore')
+			self._screen_file.write(output)
+			self._screen_file.flush()
+
+	def to_stderr(self, message):
+		"""Print message to stderr."""
+		assert type(message) == type(u'')
+		sys.stderr.write((message + u'\n').encode(preferredencoding()))
+
+	def to_cons_title(self, message):
+		"""Set console/terminal window title to message."""
+		if not self.params.get('consoletitle', False):
+			return
+		if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
+			# c_wchar_p() might not be necessary if `message` is
+			# already of type unicode()
+			ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
+		elif 'TERM' in os.environ:
+			sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding()))
+
+	def fixed_template(self):
+		"""Checks if the output template is fixed."""
+		return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
+
+	def trouble(self, message=None):
+		"""Determine action to take when a download problem appears.
+
+		Depending on if the downloader has been configured to ignore
+		download errors or not, this method may throw an exception or
+		not when errors are found, after printing the message.
+		"""
+		if message is not None:
+			self.to_stderr(message)
+		if not self.params.get('ignoreerrors', False):
+			raise DownloadError(message)
+		self._download_retcode = 1
+
+	def slow_down(self, start_time, byte_counter):
+		"""Sleep if the download speed is over the rate limit."""
+		rate_limit = self.params.get('ratelimit', None)
+		if rate_limit is None or byte_counter == 0:
+			return
+		now = time.time()
+		elapsed = now - start_time
+		if elapsed <= 0.0:
+			return
+		speed = float(byte_counter) / elapsed
+		if speed > rate_limit:
+			time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
+
+	def temp_name(self, filename):
+		"""Returns a temporary filename for the given filename."""
+		if self.params.get('nopart', False) or filename == u'-' or \
+				(os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
+			return filename
+		return filename + u'.part'
+
+	def undo_temp_name(self, filename):
+		if filename.endswith(u'.part'):
+			return filename[:-len(u'.part')]
+		return filename
+
+	def try_rename(self, old_filename, new_filename):
+		try:
+			if old_filename == new_filename:
+				return
+			os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
+		except (IOError, OSError), err:
+			self.trouble(u'ERROR: unable to rename file')
+
+	def try_utime(self, filename, last_modified_hdr):
+		"""Try to set the last-modified time of the given file."""
+		if last_modified_hdr is None:
+			return
+		if not os.path.isfile(encodeFilename(filename)):
+			return
+		timestr = last_modified_hdr
+		if timestr is None:
+			return
+		filetime = timeconvert(timestr)
+		if filetime is None:
+			return filetime
+		try:
+			os.utime(filename, (time.time(), filetime))
+		except:
+			pass
+		return filetime
+
+	def report_writedescription(self, descfn):
+		""" Report that the description file is being written """
+		self.to_screen(u'[info] Writing video description to: ' + descfn)
+
+	def report_writesubtitles(self, srtfn):
+		""" Report that the subtitles file is being written """
+		self.to_screen(u'[info] Writing video subtitles to: ' + srtfn)
+
+	def report_writeinfojson(self, infofn):
+		""" Report that the metadata file has been written """
+		self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
+
+	def report_destination(self, filename):
+		"""Report destination filename."""
+		self.to_screen(u'[download] Destination: ' + filename)
+
+	def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
+		"""Report download progress."""
+		if self.params.get('noprogress', False):
+			return
+		self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
+				(percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
+		self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
+				(percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
+
+	def report_resuming_byte(self, resume_len):
+		"""Report attempt to resume at given byte."""
+		self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
+
+	def report_retry(self, count, retries):
+		"""Report retry in case of HTTP error 5xx"""
+		self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
+
+	def report_file_already_downloaded(self, file_name):
+		"""Report file has already been fully downloaded."""
+		try:
+			self.to_screen(u'[download] %s has already been downloaded' % file_name)
+		except (UnicodeEncodeError), err:
+			self.to_screen(u'[download] The file has already been downloaded')
+
+	def report_unable_to_resume(self):
+		"""Report it was impossible to resume download."""
+		self.to_screen(u'[download] Unable to resume')
+
+	def report_finish(self):
+		"""Report download finished."""
+		if self.params.get('noprogress', False):
+			self.to_screen(u'[download] Download completed')
+		else:
+			self.to_screen(u'')
+
+	def increment_downloads(self):
+		"""Increment the ordinal that assigns a number to each file."""
+		self._num_downloads += 1
+
+	def prepare_filename(self, info_dict):
+		"""Generate the output filename."""
+		try:
+			template_dict = dict(info_dict)
+			template_dict['epoch'] = unicode(long(time.time()))
+			template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
+			template_dict['title'] = template_dict['stitle'] # Keep both for backwards compatibility
+			filename = self.params['outtmpl'] % template_dict
+			return filename
+		except (ValueError, KeyError), err:
+			self.trouble(u'ERROR: invalid system charset or erroneous output template')
+			return None
+
+	def _match_entry(self, info_dict):
+		""" Returns None iff the file should be downloaded """
+
+		title = info_dict['title']
+		matchtitle = self.params.get('matchtitle', False)
+		if matchtitle:
+			matchtitle = matchtitle.decode('utf8')
+			if not re.search(matchtitle, title, re.IGNORECASE):
+				return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
+		rejecttitle = self.params.get('rejecttitle', False)
+		if rejecttitle:
+			rejecttitle = rejecttitle.decode('utf8')
+			if re.search(rejecttitle, title, re.IGNORECASE):
+				return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
+		return None
+
+	def process_info(self, info_dict):
+		"""Process a single dictionary returned by an InfoExtractor."""
+
+		info_dict['stitle'] = sanitize_filename(info_dict['title'], self.params.get('restrictfilenames'))
+
+		reason = self._match_entry(info_dict)
+		if reason is not None:
+			self.to_screen(u'[download] ' + reason)
+			return
+
+		max_downloads = self.params.get('max_downloads')
+		if max_downloads is not None:
+			if self._num_downloads > int(max_downloads):
+				raise MaxDownloadsReached()
+
+		filename = self.prepare_filename(info_dict)
+
+		# Forced printings
+		if self.params.get('forcetitle', False):
+			print(info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace'))
+		if self.params.get('forceurl', False):
+			print(info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace'))
+		if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
+			print(info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace'))
+		if self.params.get('forcedescription', False) and 'description' in info_dict:
+			print(info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace'))
+		if self.params.get('forcefilename', False) and filename is not None:
+			print(filename.encode(preferredencoding(), 'xmlcharrefreplace'))
+		if self.params.get('forceformat', False):
+			print(info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace'))
+
+		# Do nothing else if in simulate mode
+		if self.params.get('simulate', False):
+			return
+
+		if filename is None:
+			return
+
+		try:
+			dn = os.path.dirname(encodeFilename(filename))
+			if dn != '' and not os.path.exists(dn): # dn is already encoded
+				os.makedirs(dn)
+		except (OSError, IOError), err:
+			self.trouble(u'ERROR: unable to create directory ' + unicode(err))
+			return
+
+		if self.params.get('writedescription', False):
+			try:
+				descfn = filename + u'.description'
+				self.report_writedescription(descfn)
+				descfile = open(encodeFilename(descfn), 'wb')
+				try:
+					descfile.write(info_dict['description'].encode('utf-8'))
+				finally:
+					descfile.close()
+			except (OSError, IOError):
+				self.trouble(u'ERROR: Cannot write description file ' + descfn)
+				return
+
+		if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
+			# subtitles download errors are already managed as troubles in relevant IE
+			# that way it will silently go on when used with unsupporting IE
+			try:
+				srtfn = filename.rsplit('.', 1)[0] + u'.srt'
+				self.report_writesubtitles(srtfn)
+				srtfile = open(encodeFilename(srtfn), 'wb')
+				try:
+					srtfile.write(info_dict['subtitles'].encode('utf-8'))
+				finally:
+					srtfile.close()
+			except (OSError, IOError):
+				self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
+				return
+
+		if self.params.get('writeinfojson', False):
+			infofn = filename + u'.info.json'
+			self.report_writeinfojson(infofn)
+			try:
+				json.dump
+			except (NameError,AttributeError):
+				self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
+				return
+			try:
+				infof = open(encodeFilename(infofn), 'wb')
+				try:
+					json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',))
+					json.dump(json_info_dict, infof)
+				finally:
+					infof.close()
+			except (OSError, IOError):
+				self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
+				return
+
+		if not self.params.get('skip_download', False):
+			if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
+				success = True
+			else:
+				try:
+					success = self._do_download(filename, info_dict)
+				except (OSError, IOError), err:
+					raise UnavailableVideoError
+				except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+					self.trouble(u'ERROR: unable to download video data: %s' % str(err))
+					return
+				except (ContentTooShortError, ), err:
+					self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
+					return
+
+			if success:
+				try:
+					self.post_process(filename, info_dict)
+				except (PostProcessingError), err:
+					self.trouble(u'ERROR: postprocessing: %s' % str(err))
+					return
+
+	def download(self, url_list):
+		"""Download a given list of URLs."""
+		if len(url_list) > 1 and self.fixed_template():
+			raise SameFileError(self.params['outtmpl'])
+
+		for url in url_list:
+			suitable_found = False
+			for ie in self._ies:
+				# Go to next InfoExtractor if not suitable
+				if not ie.suitable(url):
+					continue
+
+				# Suitable InfoExtractor found
+				suitable_found = True
+
+				# Extract information from URL and process it
+				videos = ie.extract(url)
+				for video in videos or []:
+					video['extractor'] = ie.IE_NAME
+					try:
+						self.increment_downloads()
+						self.process_info(video)
+					except UnavailableVideoError:
+						self.trouble(u'\nERROR: unable to download video')
+
+				# Suitable InfoExtractor had been found; go to next URL
+				break
+
+			if not suitable_found:
+				self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
+
+		return self._download_retcode
+
+	def post_process(self, filename, ie_info):
+		"""Run the postprocessing chain on the given file."""
+		info = dict(ie_info)
+		info['filepath'] = filename
+		for pp in self._pps:
+			info = pp.run(info)
+			if info is None:
+				break
+
+	def _download_with_rtmpdump(self, filename, url, player_url):
+		self.report_destination(filename)
+		tmpfilename = self.temp_name(filename)
+
+		# Check for rtmpdump first
+		try:
+			subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
+		except (OSError, IOError):
+			self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
+			return False
+
+		# Download using rtmpdump. rtmpdump returns exit code 2 when
+		# the connection was interrumpted and resuming appears to be
+		# possible. This is part of rtmpdump's normal usage, AFAIK.
+		basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
+		args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
+		if self.params.get('verbose', False):
+			try:
+				import pipes
+				shell_quote = lambda args: ' '.join(map(pipes.quote, args))
+			except ImportError:
+				shell_quote = repr
+			self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
+		retval = subprocess.call(args)
+		while retval == 2 or retval == 1:
+			prevsize = os.path.getsize(encodeFilename(tmpfilename))
+			self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
+			time.sleep(5.0) # This seems to be needed
+			retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
+			cursize = os.path.getsize(encodeFilename(tmpfilename))
+			if prevsize == cursize and retval == 1:
+				break
+			 # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
+			if prevsize == cursize and retval == 2 and cursize > 1024:
+				self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
+				retval = 0
+				break
+		if retval == 0:
+			self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(encodeFilename(tmpfilename)))
+			self.try_rename(tmpfilename, filename)
+			return True
+		else:
+			self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
+			return False
+
+	def _do_download(self, filename, info_dict):
+		url = info_dict['url']
+		player_url = info_dict.get('player_url', None)
+
+		# Check file already present
+		if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
+			self.report_file_already_downloaded(filename)
+			return True
+
+		# Attempt to download using rtmpdump
+		if url.startswith('rtmp'):
+			return self._download_with_rtmpdump(filename, url, player_url)
+
+		tmpfilename = self.temp_name(filename)
+		stream = None
+
+		# Do not include the Accept-Encoding header
+		headers = {'Youtubedl-no-compression': 'True'}
+		basic_request = urllib2.Request(url, None, headers)
+		request = urllib2.Request(url, None, headers)
+
+		# Establish possible resume length
+		if os.path.isfile(encodeFilename(tmpfilename)):
+			resume_len = os.path.getsize(encodeFilename(tmpfilename))
+		else:
+			resume_len = 0
+
+		open_mode = 'wb'
+		if resume_len != 0:
+			if self.params.get('continuedl', False):
+				self.report_resuming_byte(resume_len)
+				request.add_header('Range','bytes=%d-' % resume_len)
+				open_mode = 'ab'
+			else:
+				resume_len = 0
+
+		count = 0
+		retries = self.params.get('retries', 0)
+		while count <= retries:
+			# Establish connection
+			try:
+				if count == 0 and 'urlhandle' in info_dict:
+					data = info_dict['urlhandle']
+				data = urllib2.urlopen(request)
+				break
+			except (urllib2.HTTPError, ), err:
+				if (err.code < 500 or err.code >= 600) and err.code != 416:
+					# Unexpected HTTP error
+					raise
+				elif err.code == 416:
+					# Unable to resume (requested range not satisfiable)
+					try:
+						# Open the connection again without the range header
+						data = urllib2.urlopen(basic_request)
+						content_length = data.info()['Content-Length']
+					except (urllib2.HTTPError, ), err:
+						if err.code < 500 or err.code >= 600:
+							raise
+					else:
+						# Examine the reported length
+						if (content_length is not None and
+								(resume_len - 100 < long(content_length) < resume_len + 100)):
+							# The file had already been fully downloaded.
+							# Explanation to the above condition: in issue #175 it was revealed that
+							# YouTube sometimes adds or removes a few bytes from the end of the file,
+							# changing the file size slightly and causing problems for some users. So
+							# I decided to implement a suggested change and consider the file
+							# completely downloaded if the file size differs less than 100 bytes from
+							# the one in the hard drive.
+							self.report_file_already_downloaded(filename)
+							self.try_rename(tmpfilename, filename)
+							return True
+						else:
+							# The length does not match, we start the download over
+							self.report_unable_to_resume()
+							open_mode = 'wb'
+							break
+			# Retry
+			count += 1
+			if count <= retries:
+				self.report_retry(count, retries)
+
+		if count > retries:
+			self.trouble(u'ERROR: giving up after %s retries' % retries)
+			return False
+
+		data_len = data.info().get('Content-length', None)
+		if data_len is not None:
+			data_len = long(data_len) + resume_len
+		data_len_str = self.format_bytes(data_len)
+		byte_counter = 0 + resume_len
+		block_size = 1024
+		start = time.time()
+		while True:
+			# Download and write
+			before = time.time()
+			data_block = data.read(block_size)
+			after = time.time()
+			if len(data_block) == 0:
+				break
+			byte_counter += len(data_block)
+
+			# Open file just in time
+			if stream is None:
+				try:
+					(stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
+					assert stream is not None
+					filename = self.undo_temp_name(tmpfilename)
+					self.report_destination(filename)
+				except (OSError, IOError), err:
+					self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
+					return False
+			try:
+				stream.write(data_block)
+			except (IOError, OSError), err:
+				self.trouble(u'\nERROR: unable to write data: %s' % str(err))
+				return False
+			block_size = self.best_block_size(after - before, len(data_block))
+
+			# Progress message
+			speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
+			if data_len is None:
+				self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
+			else:
+				percent_str = self.calc_percent(byte_counter, data_len)
+				eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
+				self.report_progress(percent_str, data_len_str, speed_str, eta_str)
+
+			# Apply rate limit
+			self.slow_down(start, byte_counter - resume_len)
+
+		if stream is None:
+			self.trouble(u'\nERROR: Did not get any data blocks')
+			return False
+		stream.close()
+		self.report_finish()
+		if data_len is not None and byte_counter != data_len:
+			raise ContentTooShortError(byte_counter, long(data_len))
+		self.try_rename(tmpfilename, filename)
+
+		# Update file modification time
+		if self.params.get('updatetime', True):
+			info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
+
+		return True

youtube_dl/PostProcessor.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import subprocess
+import sys
+import time
+
+from utils import *
+
+
+class PostProcessor(object):
+	"""Post Processor class.
+
+	PostProcessor objects can be added to downloaders with their
+	add_post_processor() method. When the downloader has finished a
+	successful download, it will take its internal chain of PostProcessors
+	and start calling the run() method on each one of them, first with
+	an initial argument and then with the returned value of the previous
+	PostProcessor.
+
+	The chain will be stopped if one of them ever returns None or the end
+	of the chain is reached.
+
+	PostProcessor objects follow a "mutual registration" process similar
+	to InfoExtractor objects.
+	"""
+
+	_downloader = None
+
+	def __init__(self, downloader=None):
+		self._downloader = downloader
+
+	def set_downloader(self, downloader):
+		"""Sets the downloader for this PP."""
+		self._downloader = downloader
+
+	def run(self, information):
+		"""Run the PostProcessor.
+
+		The "information" argument is a dictionary like the ones
+		composed by InfoExtractors. The only difference is that this
+		one has an extra field called "filepath" that points to the
+		downloaded file.
+
+		When this method returns None, the postprocessing chain is
+		stopped. However, this method may return an information
+		dictionary that will be passed to the next postprocessing
+		object in the chain. It can be the one it received after
+		changing some fields.
+
+		In addition, this method may raise a PostProcessingError
+		exception that will be taken into account by the downloader
+		it was called from.
+		"""
+		return information # by default, do nothing
+
+class AudioConversionError(BaseException):
+	def __init__(self, message):
+		self.message = message
+
+class FFmpegExtractAudioPP(PostProcessor):
+	def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, keepvideo=False):
+		PostProcessor.__init__(self, downloader)
+		if preferredcodec is None:
+			preferredcodec = 'best'
+		self._preferredcodec = preferredcodec
+		self._preferredquality = preferredquality
+		self._keepvideo = keepvideo
+		self._exes = self.detect_executables()
+
+	@staticmethod
+	def detect_executables():
+		def executable(exe):
+			try:
+				subprocess.Popen([exe, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+			except OSError:
+				return False
+			return exe
+		programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
+		return dict((program, executable(program)) for program in programs)
+
+	def get_audio_codec(self, path):
+		if not self._exes['ffprobe'] and not self._exes['avprobe']: return None
+		try:
+			cmd = [self._exes['avprobe'] or self._exes['ffprobe'], '-show_streams', '--', encodeFilename(path)]
+			handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE)
+			output = handle.communicate()[0]
+			if handle.wait() != 0:
+				return None
+		except (IOError, OSError):
+			return None
+		audio_codec = None
+		for line in output.split('\n'):
+			if line.startswith('codec_name='):
+				audio_codec = line.split('=')[1].strip()
+			elif line.strip() == 'codec_type=audio' and audio_codec is not None:
+				return audio_codec
+		return None
+
+	def run_ffmpeg(self, path, out_path, codec, more_opts):
+		if not self._exes['ffmpeg'] and not self._exes['avconv']:
+			raise AudioConversionError('ffmpeg or avconv not found. Please install one.')	
+		if codec is None:
+			acodec_opts = []
+		else:
+			acodec_opts = ['-acodec', codec]
+		cmd = ([self._exes['avconv'] or self._exes['ffmpeg'], '-y', '-i', encodeFilename(path), '-vn']
+			   + acodec_opts + more_opts +
+			   ['--', encodeFilename(out_path)])
+		p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+		stdout,stderr = p.communicate()
+		if p.returncode != 0:
+			msg = stderr.strip().split('\n')[-1]
+			raise AudioConversionError(msg)
+
+	def run(self, information):
+		path = information['filepath']
+
+		filecodec = self.get_audio_codec(path)
+		if filecodec is None:
+			self._downloader.to_stderr(u'WARNING: unable to obtain file audio codec with ffprobe')
+			return None
+
+		more_opts = []
+		if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'):
+			if self._preferredcodec == 'm4a' and filecodec == 'aac':
+				# Lossless, but in another container
+				acodec = 'copy'
+				extension = self._preferredcodec
+				more_opts = [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
+			elif filecodec in ['aac', 'mp3', 'vorbis']:
+				# Lossless if possible
+				acodec = 'copy'
+				extension = filecodec
+				if filecodec == 'aac':
+					more_opts = ['-f', 'adts']
+				if filecodec == 'vorbis':
+					extension = 'ogg'
+			else:
+				# MP3 otherwise.
+				acodec = 'libmp3lame'
+				extension = 'mp3'
+				more_opts = []
+				if self._preferredquality is not None:
+					if int(self._preferredquality) < 10:
+						more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
+					else:
+						more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
+		else:
+			# We convert the audio (lossy)
+			acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
+			extension = self._preferredcodec
+			more_opts = []
+			if self._preferredquality is not None:
+				if int(self._preferredquality) < 10:
+					more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
+				else:
+					more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
+			if self._preferredcodec == 'aac':
+				more_opts += ['-f', 'adts']
+			if self._preferredcodec == 'm4a':
+				more_opts += [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
+			if self._preferredcodec == 'vorbis':
+				extension = 'ogg'
+			if self._preferredcodec == 'wav':
+				extension = 'wav'
+				more_opts += ['-f', 'wav']
+
+		prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups
+		new_path = prefix + sep + extension
+		self._downloader.to_screen(u'[' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path)
+		try:
+			self.run_ffmpeg(path, new_path, acodec, more_opts)
+		except:
+			etype,e,tb = sys.exc_info()
+			if isinstance(e, AudioConversionError):
+				self._downloader.to_stderr(u'ERROR: audio conversion failed: ' + e.message)
+			else:
+				self._downloader.to_stderr(u'ERROR: error running ' + (self._exes['avconv'] and 'avconv' or 'ffmpeg'))
+			return None
+
+ 		# Try to update the date time for extracted audio file.
+		if information.get('filetime') is not None:
+			try:
+				os.utime(encodeFilename(new_path), (time.time(), information['filetime']))
+			except:
+				self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file')
+
+		if not self._keepvideo:
+			try:
+				os.remove(encodeFilename(path))
+			except (IOError, OSError):
+				self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file')
+				return None
+
+		information['filepath'] = new_path
+		return information

youtube_dl/__init__.py

@@ -0,0 +1,559 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import with_statement
+
+__authors__  = (
+	'Ricardo Garcia Gonzalez',
+	'Danny Colligan',
+	'Benjamin Johnson',
+	'Vasyl\' Vavrychuk',
+	'Witold Baryluk',
+	'Paweł Paprota',
+	'Gergely Imreh',
+	'Rogério Brito',
+	'Philipp Hagemeister',
+	'Sören Schulze',
+	'Kevin Ngo',
+	'Ori Avtalion',
+	'shizeeg',
+	'Filippo Valsorda',
+	)
+
+__license__ = 'Public Domain'
+__version__ = '2012.11.27'
+
+UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
+UPDATE_URL_VERSION = 'https://raw.github.com/rg3/youtube-dl/master/LATEST_VERSION'
+UPDATE_URL_EXE = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl.exe'
+
+
+import cookielib
+import getpass
+import optparse
+import os
+import re
+import shlex
+import socket
+import subprocess
+import sys
+import urllib2
+import warnings
+
+from utils import *
+from FileDownloader import *
+from InfoExtractors import *
+from PostProcessor import *
+
+def updateSelf(downloader, filename):
+	''' Update the program file with the latest version from the repository '''
+	# Note: downloader only used for options
+
+	if not os.access(filename, os.W_OK):
+		sys.exit('ERROR: no write permissions on %s' % filename)
+
+	downloader.to_screen(u'Updating to latest version...')
+
+	urlv = urllib2.urlopen(UPDATE_URL_VERSION)
+	newversion = urlv.read().strip()
+	if newversion == __version__:
+		downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')')
+		return
+	urlv.close()
+
+	if hasattr(sys, "frozen"): #py2exe
+		exe = os.path.abspath(filename)
+		directory = os.path.dirname(exe)
+		if not os.access(directory, os.W_OK):
+			sys.exit('ERROR: no write permissions on %s' % directory)
+
+		try:
+			urlh = urllib2.urlopen(UPDATE_URL_EXE)
+			newcontent = urlh.read()
+			urlh.close()
+			with open(exe + '.new', 'wb') as outf:
+				outf.write(newcontent)
+		except (IOError, OSError), err:
+			sys.exit('ERROR: unable to download latest version')
+
+		try:
+			bat = os.path.join(directory, 'youtube-dl-updater.bat')
+			b = open(bat, 'w')
+			b.write("""
+echo Updating youtube-dl...
+ping 127.0.0.1 -n 5 -w 1000 > NUL
+move /Y "%s.new" "%s"
+del "%s"
+			\n""" %(exe, exe, bat))
+			b.close()
+
+			os.startfile(bat)
+		except (IOError, OSError), err:
+			sys.exit('ERROR: unable to overwrite current version')
+
+	else:
+		try:
+			urlh = urllib2.urlopen(UPDATE_URL)
+			newcontent = urlh.read()
+			urlh.close()
+		except (IOError, OSError), err:
+			sys.exit('ERROR: unable to download latest version')
+
+		try:
+			with open(filename, 'wb') as outf:
+				outf.write(newcontent)
+		except (IOError, OSError), err:
+			sys.exit('ERROR: unable to overwrite current version')
+
+	downloader.to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')
+
+def parseOpts():
+	def _readOptions(filename_bytes):
+		try:
+			optionf = open(filename_bytes)
+		except IOError:
+			return [] # silently skip if file is not present
+		try:
+			res = []
+			for l in optionf:
+				res += shlex.split(l, comments=True)
+		finally:
+			optionf.close()
+		return res
+
+	def _format_option_string(option):
+		''' ('-o', '--option') -> -o, --format METAVAR'''
+
+		opts = []
+
+		if option._short_opts: opts.append(option._short_opts[0])
+		if option._long_opts: opts.append(option._long_opts[0])
+		if len(opts) > 1: opts.insert(1, ', ')
+
+		if option.takes_value(): opts.append(' %s' % option.metavar)
+
+		return "".join(opts)
+
+	def _find_term_columns():
+		columns = os.environ.get('COLUMNS', None)
+		if columns:
+			return int(columns)
+
+		try:
+			sp = subprocess.Popen(['stty', 'size'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+			out,err = sp.communicate()
+			return int(out.split()[1])
+		except:
+			pass
+		return None
+
+	max_width = 80
+	max_help_position = 80
+
+	# No need to wrap help messages if we're on a wide console
+	columns = _find_term_columns()
+	if columns: max_width = columns
+
+	fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
+	fmt.format_option_strings = _format_option_string
+
+	kw = {
+		'version'   : __version__,
+		'formatter' : fmt,
+		'usage' : '%prog [options] url [url...]',
+		'conflict_handler' : 'resolve',
+	}
+
+	parser = optparse.OptionParser(**kw)
+
+	# option groups
+	general        = optparse.OptionGroup(parser, 'General Options')
+	selection      = optparse.OptionGroup(parser, 'Video Selection')
+	authentication = optparse.OptionGroup(parser, 'Authentication Options')
+	video_format   = optparse.OptionGroup(parser, 'Video Format Options')
+	postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
+	filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
+	verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
+
+	general.add_option('-h', '--help',
+			action='help', help='print this help text and exit')
+	general.add_option('-v', '--version',
+			action='version', help='print program version and exit')
+	general.add_option('-U', '--update',
+			action='store_true', dest='update_self', help='update this program to latest version')
+	general.add_option('-i', '--ignore-errors',
+			action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
+	general.add_option('-r', '--rate-limit',
+			dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
+	general.add_option('-R', '--retries',
+			dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
+	general.add_option('--dump-user-agent',
+			action='store_true', dest='dump_user_agent',
+			help='display the current browser identification', default=False)
+	general.add_option('--user-agent',
+			dest='user_agent', help='specify a custom user agent', metavar='UA')
+	general.add_option('--list-extractors',
+			action='store_true', dest='list_extractors',
+			help='List all supported extractors and the URLs they would handle', default=False)
+
+	selection.add_option('--playlist-start',
+			dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is %default)', default=1)
+	selection.add_option('--playlist-end',
+			dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
+	selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
+	selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
+	selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None)
+
+	authentication.add_option('-u', '--username',
+			dest='username', metavar='USERNAME', help='account username')
+	authentication.add_option('-p', '--password',
+			dest='password', metavar='PASSWORD', help='account password')
+	authentication.add_option('-n', '--netrc',
+			action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
+
+
+	video_format.add_option('-f', '--format',
+			action='store', dest='format', metavar='FORMAT', help='video format code')
+	video_format.add_option('--all-formats',
+			action='store_const', dest='format', help='download all available video formats', const='all')
+	video_format.add_option('--prefer-free-formats',
+			action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
+	video_format.add_option('--max-quality',
+			action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
+	video_format.add_option('-F', '--list-formats',
+			action='store_true', dest='listformats', help='list all available formats (currently youtube only)')
+	video_format.add_option('--write-srt',
+			action='store_true', dest='writesubtitles',
+			help='write video closed captions to a .srt file (currently youtube only)', default=False)
+	video_format.add_option('--srt-lang',
+			action='store', dest='subtitleslang', metavar='LANG',
+			help='language of the closed captions to download (optional) use IETF language tags like \'en\'')
+
+
+	verbosity.add_option('-q', '--quiet',
+			action='store_true', dest='quiet', help='activates quiet mode', default=False)
+	verbosity.add_option('-s', '--simulate',
+			action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
+	verbosity.add_option('--skip-download',
+			action='store_true', dest='skip_download', help='do not download the video', default=False)
+	verbosity.add_option('-g', '--get-url',
+			action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
+	verbosity.add_option('-e', '--get-title',
+			action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
+	verbosity.add_option('--get-thumbnail',
+			action='store_true', dest='getthumbnail',
+			help='simulate, quiet but print thumbnail URL', default=False)
+	verbosity.add_option('--get-description',
+			action='store_true', dest='getdescription',
+			help='simulate, quiet but print video description', default=False)
+	verbosity.add_option('--get-filename',
+			action='store_true', dest='getfilename',
+			help='simulate, quiet but print output filename', default=False)
+	verbosity.add_option('--get-format',
+			action='store_true', dest='getformat',
+			help='simulate, quiet but print output format', default=False)
+	verbosity.add_option('--no-progress',
+			action='store_true', dest='noprogress', help='do not print progress bar', default=False)
+	verbosity.add_option('--console-title',
+			action='store_true', dest='consoletitle',
+			help='display progress in console titlebar', default=False)
+	verbosity.add_option('-v', '--verbose',
+			action='store_true', dest='verbose', help='print various debugging information', default=False)
+
+
+	filesystem.add_option('-t', '--title',
+			action='store_true', dest='usetitle', help='use title in file name', default=False)
+	filesystem.add_option('--id',
+			action='store_true', dest='useid', help='use video ID in file name', default=False)
+	filesystem.add_option('-l', '--literal',
+			action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
+	filesystem.add_option('-A', '--auto-number',
+			action='store_true', dest='autonumber',
+			help='number downloaded files starting from 00000', default=False)
+	filesystem.add_option('-o', '--output',
+			dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout.')
+	filesystem.add_option('--restrict-filenames',
+			action='store_true', dest='restrictfilenames',
+			help='Avoid some characters such as "&" and spaces in filenames', default=False)
+	filesystem.add_option('-a', '--batch-file',
+			dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
+	filesystem.add_option('-w', '--no-overwrites',
+			action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
+	filesystem.add_option('-c', '--continue',
+			action='store_true', dest='continue_dl', help='resume partially downloaded files', default=True)
+	filesystem.add_option('--no-continue',
+			action='store_false', dest='continue_dl',
+			help='do not resume partially downloaded files (restart from beginning)')
+	filesystem.add_option('--cookies',
+			dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
+	filesystem.add_option('--no-part',
+			action='store_true', dest='nopart', help='do not use .part files', default=False)
+	filesystem.add_option('--no-mtime',
+			action='store_false', dest='updatetime',
+			help='do not use the Last-modified header to set the file modification time', default=True)
+	filesystem.add_option('--write-description',
+			action='store_true', dest='writedescription',
+			help='write video description to a .description file', default=False)
+	filesystem.add_option('--write-info-json',
+			action='store_true', dest='writeinfojson',
+			help='write video metadata to a .info.json file', default=False)
+
+
+	postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
+			help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
+	postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
+			help='"best", "aac", "vorbis", "mp3", "m4a", or "wav"; best by default')
+	postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
+			help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
+	postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
+			help='keeps the video file on disk after the post-processing; the video is erased by default')
+
+
+	parser.add_option_group(general)
+	parser.add_option_group(selection)
+	parser.add_option_group(filesystem)
+	parser.add_option_group(verbosity)
+	parser.add_option_group(video_format)
+	parser.add_option_group(authentication)
+	parser.add_option_group(postproc)
+
+	xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
+	if xdg_config_home:
+		userConf = os.path.join(xdg_config_home, 'youtube-dl.conf')
+	else:
+		userConf = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
+	argv = _readOptions('/etc/youtube-dl.conf') + _readOptions(userConf) + sys.argv[1:]
+	opts, args = parser.parse_args(argv)
+
+	return parser, opts, args
+
+def gen_extractors():
+	""" Return a list of an instance of every supported extractor.
+	The order does matter; the first extractor matched is the one handling the URL.
+	"""
+	return [
+		YoutubePlaylistIE(),
+		YoutubeChannelIE(),
+		YoutubeUserIE(),
+		YoutubeSearchIE(),
+		YoutubeIE(),
+		MetacafeIE(),
+		DailymotionIE(),
+		GoogleIE(),
+		GoogleSearchIE(),
+		PhotobucketIE(),
+		YahooIE(),
+		YahooSearchIE(),
+		DepositFilesIE(),
+		FacebookIE(),
+		BlipTVUserIE(),
+		BlipTVIE(),
+		VimeoIE(),
+		MyVideoIE(),
+		ComedyCentralIE(),
+		EscapistIE(),
+		CollegeHumorIE(),
+		XVideosIE(),
+		SoundcloudIE(),
+		InfoQIE(),
+		MixcloudIE(),
+		StanfordOpenClassroomIE(),
+		MTVIE(),
+		YoukuIE(),
+		XNXXIE(),
+		GooglePlusIE(),
+
+		GenericIE()
+	]
+
+def _real_main():
+	parser, opts, args = parseOpts()
+
+	# Open appropriate CookieJar
+	if opts.cookiefile is None:
+		jar = cookielib.CookieJar()
+	else:
+		try:
+			jar = cookielib.MozillaCookieJar(opts.cookiefile)
+			if os.path.isfile(opts.cookiefile) and os.access(opts.cookiefile, os.R_OK):
+				jar.load()
+		except (IOError, OSError), err:
+			sys.exit(u'ERROR: unable to open cookie file')
+	# Set user agent
+	if opts.user_agent is not None:
+		std_headers['User-Agent'] = opts.user_agent
+
+	# Dump user agent
+	if opts.dump_user_agent:
+		print std_headers['User-Agent']
+		sys.exit(0)
+
+	# Batch file verification
+	batchurls = []
+	if opts.batchfile is not None:
+		try:
+			if opts.batchfile == '-':
+				batchfd = sys.stdin
+			else:
+				batchfd = open(opts.batchfile, 'r')
+			batchurls = batchfd.readlines()
+			batchurls = [x.strip() for x in batchurls]
+			batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
+		except IOError:
+			sys.exit(u'ERROR: batch file could not be read')
+	all_urls = batchurls + args
+	all_urls = map(lambda url: url.strip(), all_urls)
+
+	# General configuration
+	cookie_processor = urllib2.HTTPCookieProcessor(jar)
+	proxy_handler = urllib2.ProxyHandler()
+	opener = urllib2.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+	urllib2.install_opener(opener)
+	socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
+
+	extractors = gen_extractors()
+
+	if opts.list_extractors:
+		for ie in extractors:
+			print(ie.IE_NAME)
+			matchedUrls = filter(lambda url: ie.suitable(url), all_urls)
+			all_urls = filter(lambda url: url not in matchedUrls, all_urls)
+			for mu in matchedUrls:
+				print(u'  ' + mu)
+		sys.exit(0)
+
+	# Conflicting, missing and erroneous options
+	if opts.usenetrc and (opts.username is not None or opts.password is not None):
+		parser.error(u'using .netrc conflicts with giving username/password')
+	if opts.password is not None and opts.username is None:
+		parser.error(u'account username missing')
+	if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
+		parser.error(u'using output template conflicts with using title, video ID or auto number')
+	if opts.usetitle and opts.useid:
+		parser.error(u'using title conflicts with using video ID')
+	if opts.username is not None and opts.password is None:
+		opts.password = getpass.getpass(u'Type account password and press return:')
+	if opts.ratelimit is not None:
+		numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
+		if numeric_limit is None:
+			parser.error(u'invalid rate limit specified')
+		opts.ratelimit = numeric_limit
+	if opts.retries is not None:
+		try:
+			opts.retries = long(opts.retries)
+		except (TypeError, ValueError), err:
+			parser.error(u'invalid retry count specified')
+	try:
+		opts.playliststart = int(opts.playliststart)
+		if opts.playliststart <= 0:
+			raise ValueError(u'Playlist start must be positive')
+	except (TypeError, ValueError), err:
+		parser.error(u'invalid playlist start number specified')
+	try:
+		opts.playlistend = int(opts.playlistend)
+		if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart):
+			raise ValueError(u'Playlist end must be greater than playlist start')
+	except (TypeError, ValueError), err:
+		parser.error(u'invalid playlist end number specified')
+	if opts.extractaudio:
+		if opts.audioformat not in ['best', 'aac', 'mp3', 'vorbis', 'm4a', 'wav']:
+			parser.error(u'invalid audio format specified')
+	if opts.audioquality:
+		opts.audioquality = opts.audioquality.strip('k').strip('K')
+		if not opts.audioquality.isdigit():
+			parser.error(u'invalid audio quality specified')
+
+	# File downloader
+	fd = FileDownloader({
+		'usenetrc': opts.usenetrc,
+		'username': opts.username,
+		'password': opts.password,
+		'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
+		'forceurl': opts.geturl,
+		'forcetitle': opts.gettitle,
+		'forcethumbnail': opts.getthumbnail,
+		'forcedescription': opts.getdescription,
+		'forcefilename': opts.getfilename,
+		'forceformat': opts.getformat,
+		'simulate': opts.simulate,
+		'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
+		'format': opts.format,
+		'format_limit': opts.format_limit,
+		'listformats': opts.listformats,
+		'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
+			or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
+			or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
+			or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
+			or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
+			or (opts.useid and u'%(id)s.%(ext)s')
+			or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
+			or u'%(id)s.%(ext)s'),
+		'restrictfilenames': opts.restrictfilenames,
+		'ignoreerrors': opts.ignoreerrors,
+		'ratelimit': opts.ratelimit,
+		'nooverwrites': opts.nooverwrites,
+		'retries': opts.retries,
+		'continuedl': opts.continue_dl,
+		'noprogress': opts.noprogress,
+		'playliststart': opts.playliststart,
+		'playlistend': opts.playlistend,
+		'logtostderr': opts.outtmpl == '-',
+		'consoletitle': opts.consoletitle,
+		'nopart': opts.nopart,
+		'updatetime': opts.updatetime,
+		'writedescription': opts.writedescription,
+		'writeinfojson': opts.writeinfojson,
+		'writesubtitles': opts.writesubtitles,
+		'subtitleslang': opts.subtitleslang,
+		'matchtitle': opts.matchtitle,
+		'rejecttitle': opts.rejecttitle,
+		'max_downloads': opts.max_downloads,
+		'prefer_free_formats': opts.prefer_free_formats,
+		'verbose': opts.verbose,
+		})
+
+	if opts.verbose:
+		fd.to_screen(u'[debug] Proxy map: ' + str(proxy_handler.proxies))
+
+	for extractor in extractors:
+		fd.add_info_extractor(extractor)
+
+	# PostProcessors
+	if opts.extractaudio:
+		fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, keepvideo=opts.keepvideo))
+
+	# Update version
+	if opts.update_self:
+		updateSelf(fd, sys.argv[0])
+
+	# Maybe do nothing
+	if len(all_urls) < 1:
+		if not opts.update_self:
+			parser.error(u'you must provide at least one URL')
+		else:
+			sys.exit()
+	
+	try:
+		retcode = fd.download(all_urls)
+	except MaxDownloadsReached:
+		fd.to_screen(u'--max-download limit reached, aborting.')
+		retcode = 101
+
+	# Dump cookie jar if requested
+	if opts.cookiefile is not None:
+		try:
+			jar.save()
+		except (IOError, OSError), err:
+			sys.exit(u'ERROR: unable to save cookie jar')
+
+	sys.exit(retcode)
+
+def main():
+	try:
+		_real_main()
+	except DownloadError:
+		sys.exit(1)
+	except SameFileError:
+		sys.exit(u'ERROR: fixed output name but more than one file to download')
+	except KeyboardInterrupt:
+		sys.exit(u'\nERROR: Interrupted by user')

youtube_dl/__main__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import __init__
+
+if __name__ == '__main__':
+	__init__.main()

youtube_dl/utils.py

@@ -0,0 +1,372 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import gzip
+import htmlentitydefs
+import HTMLParser
+import locale
+import os
+import re
+import sys
+import zlib
+import urllib2
+import email.utils
+import json
+
+try:
+	import cStringIO as StringIO
+except ImportError:
+	import StringIO
+
+std_headers = {
+	'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
+	'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
+	'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+	'Accept-Encoding': 'gzip, deflate',
+	'Accept-Language': 'en-us,en;q=0.5',
+}
+
+try:
+    compat_str = unicode # Python 2
+except NameError:
+    compat_str = str
+
+def preferredencoding():
+	"""Get preferred encoding.
+
+	Returns the best encoding scheme for the system, based on
+	locale.getpreferredencoding() and some further tweaks.
+	"""
+	def yield_preferredencoding():
+		try:
+			pref = locale.getpreferredencoding()
+			u'TEST'.encode(pref)
+		except:
+			pref = 'UTF-8'
+		while True:
+			yield pref
+	return yield_preferredencoding().next()
+
+
+def htmlentity_transform(matchobj):
+	"""Transforms an HTML entity to a Unicode character.
+
+	This function receives a match object and is intended to be used with
+	the re.sub() function.
+	"""
+	entity = matchobj.group(1)
+
+	# Known non-numeric HTML entity
+	if entity in htmlentitydefs.name2codepoint:
+		return unichr(htmlentitydefs.name2codepoint[entity])
+
+	# Unicode character
+	mobj = re.match(ur'(?u)#(x?\d+)', entity)
+	if mobj is not None:
+		numstr = mobj.group(1)
+		if numstr.startswith(u'x'):
+			base = 16
+			numstr = u'0%s' % numstr
+		else:
+			base = 10
+		return unichr(long(numstr, base))
+
+	# Unknown entity in name, return its literal representation
+	return (u'&%s;' % entity)
+
+HTMLParser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
+class IDParser(HTMLParser.HTMLParser):
+	"""Modified HTMLParser that isolates a tag with the specified id"""
+	def __init__(self, id):
+		self.id = id
+		self.result = None
+		self.started = False
+		self.depth = {}
+		self.html = None
+		self.watch_startpos = False
+		self.error_count = 0
+		HTMLParser.HTMLParser.__init__(self)
+
+	def error(self, message):
+		if self.error_count > 10 or self.started:
+			raise HTMLParser.HTMLParseError(message, self.getpos())
+		self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
+		self.error_count += 1
+		self.goahead(1)
+
+	def loads(self, html):
+		self.html = html
+		self.feed(html)
+		self.close()
+
+	def handle_starttag(self, tag, attrs):
+		attrs = dict(attrs)
+		if self.started:
+			self.find_startpos(None)
+		if 'id' in attrs and attrs['id'] == self.id:
+			self.result = [tag]
+			self.started = True
+			self.watch_startpos = True
+		if self.started:
+			if not tag in self.depth: self.depth[tag] = 0
+			self.depth[tag] += 1
+
+	def handle_endtag(self, tag):
+		if self.started:
+			if tag in self.depth: self.depth[tag] -= 1
+			if self.depth[self.result[0]] == 0:
+				self.started = False
+				self.result.append(self.getpos())
+
+	def find_startpos(self, x):
+		"""Needed to put the start position of the result (self.result[1])
+		after the opening tag with the requested id"""
+		if self.watch_startpos:
+			self.watch_startpos = False
+			self.result.append(self.getpos())
+	handle_entityref = handle_charref = handle_data = handle_comment = \
+	handle_decl = handle_pi = unknown_decl = find_startpos
+
+	def get_result(self):
+		if self.result == None: return None
+		if len(self.result) != 3: return None
+		lines = self.html.split('\n')
+		lines = lines[self.result[1][0]-1:self.result[2][0]]
+		lines[0] = lines[0][self.result[1][1]:]
+		if len(lines) == 1:
+			lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
+		lines[-1] = lines[-1][:self.result[2][1]]
+		return '\n'.join(lines).strip()
+
+def get_element_by_id(id, html):
+	"""Return the content of the tag with the specified id in the passed HTML document"""
+	parser = IDParser(id)
+	try:
+		parser.loads(html)
+	except HTMLParser.HTMLParseError:
+		pass
+	return parser.get_result()
+
+
+def clean_html(html):
+	"""Clean an HTML snippet into a readable string"""
+	# Newline vs <br />
+	html = html.replace('\n', ' ')
+	html = re.sub('\s*<\s*br\s*/?\s*>\s*', '\n', html)
+	# Strip html tags
+	html = re.sub('<.*?>', '', html)
+	# Replace html entities
+	html = unescapeHTML(html)
+	return html
+
+
+def sanitize_open(filename, open_mode):
+	"""Try to open the given filename, and slightly tweak it if this fails.
+
+	Attempts to open the given filename. If this fails, it tries to change
+	the filename slightly, step by step, until it's either able to open it
+	or it fails and raises a final exception, like the standard open()
+	function.
+
+	It returns the tuple (stream, definitive_file_name).
+	"""
+	try:
+		if filename == u'-':
+			if sys.platform == 'win32':
+				import msvcrt
+				msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+			return (sys.stdout, filename)
+		stream = open(encodeFilename(filename), open_mode)
+		return (stream, filename)
+	except (IOError, OSError), err:
+		# In case of error, try to remove win32 forbidden chars
+		filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
+
+		# An exception here should be caught in the caller
+		stream = open(encodeFilename(filename), open_mode)
+		return (stream, filename)
+
+
+def timeconvert(timestr):
+	"""Convert RFC 2822 defined time string into system timestamp"""
+	timestamp = None
+	timetuple = email.utils.parsedate_tz(timestr)
+	if timetuple is not None:
+		timestamp = email.utils.mktime_tz(timetuple)
+	return timestamp
+
+def sanitize_filename(s, restricted=False):
+	"""Sanitizes a string so it could be used as part of a filename.
+	If restricted is set, use a stricter subset of allowed characters.
+	"""
+	def replace_insane(char):
+		if char == '?' or ord(char) < 32 or ord(char) == 127:
+			return ''
+		elif char == '"':
+			return '' if restricted else '\''
+		elif char == ':':
+			return '_-' if restricted else ' -'
+		elif char in '\\/|*<>':
+			return '-'
+		if restricted and (char in '&\'' or char.isspace()):
+			return '_'
+		return char
+
+	result = u''.join(map(replace_insane, s))
+	while '--' in result:
+		result = result.replace('--', '-')
+	return result.strip('-')
+
+def orderedSet(iterable):
+	""" Remove all duplicates from the input iterable """
+	res = []
+	for el in iterable:
+		if el not in res:
+			res.append(el)
+	return res
+
+def unescapeHTML(s):
+	"""
+	@param s a string (of type unicode)
+	"""
+	assert type(s) == type(u'')
+
+	result = re.sub(ur'(?u)&(.+?);', htmlentity_transform, s)
+	return result
+
+def encodeFilename(s):
+	"""
+	@param s The name of the file (of type unicode)
+	"""
+
+	assert type(s) == type(u'')
+
+	if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
+		# Pass u'' directly to use Unicode APIs on Windows 2000 and up
+		# (Detecting Windows NT 4 is tricky because 'major >= 4' would
+		# match Windows 9x series as well. Besides, NT 4 is obsolete.)
+		return s
+	else:
+		return s.encode(sys.getfilesystemencoding(), 'ignore')
+
+class DownloadError(Exception):
+	"""Download Error exception.
+
+	This exception may be thrown by FileDownloader objects if they are not
+	configured to continue on errors. They will contain the appropriate
+	error message.
+	"""
+	pass
+
+
+class SameFileError(Exception):
+	"""Same File exception.
+
+	This exception will be thrown by FileDownloader objects if they detect
+	multiple files would have to be downloaded to the same file on disk.
+	"""
+	pass
+
+
+class PostProcessingError(Exception):
+	"""Post Processing exception.
+
+	This exception may be raised by PostProcessor's .run() method to
+	indicate an error in the postprocessing task.
+	"""
+	pass
+
+class MaxDownloadsReached(Exception):
+	""" --max-downloads limit has been reached. """
+	pass
+
+
+class UnavailableVideoError(Exception):
+	"""Unavailable Format exception.
+
+	This exception will be thrown when a video is requested
+	in a format that is not available for that video.
+	"""
+	pass
+
+
+class ContentTooShortError(Exception):
+	"""Content Too Short exception.
+
+	This exception may be raised by FileDownloader objects when a file they
+	download is too small for what the server announced first, indicating
+	the connection was probably interrupted.
+	"""
+	# Both in bytes
+	downloaded = None
+	expected = None
+
+	def __init__(self, downloaded, expected):
+		self.downloaded = downloaded
+		self.expected = expected
+
+
+class Trouble(Exception):
+	"""Trouble helper exception
+	
+	This is an exception to be handled with
+	FileDownloader.trouble
+	"""
+
+class YoutubeDLHandler(urllib2.HTTPHandler):
+	"""Handler for HTTP requests and responses.
+
+	This class, when installed with an OpenerDirector, automatically adds
+	the standard headers to every HTTP request and handles gzipped and
+	deflated responses from web servers. If compression is to be avoided in
+	a particular request, the original request in the program code only has
+	to include the HTTP header "Youtubedl-No-Compression", which will be
+	removed before making the real request.
+
+	Part of this code was copied from:
+
+	http://techknack.net/python-urllib2-handlers/
+
+	Andrew Rowls, the author of that code, agreed to release it to the
+	public domain.
+	"""
+
+	@staticmethod
+	def deflate(data):
+		try:
+			return zlib.decompress(data, -zlib.MAX_WBITS)
+		except zlib.error:
+			return zlib.decompress(data)
+
+	@staticmethod
+	def addinfourl_wrapper(stream, headers, url, code):
+		if hasattr(urllib2.addinfourl, 'getcode'):
+			return urllib2.addinfourl(stream, headers, url, code)
+		ret = urllib2.addinfourl(stream, headers, url)
+		ret.code = code
+		return ret
+
+	def http_request(self, req):
+		for h in std_headers:
+			if h in req.headers:
+				del req.headers[h]
+			req.add_header(h, std_headers[h])
+		if 'Youtubedl-no-compression' in req.headers:
+			if 'Accept-encoding' in req.headers:
+				del req.headers['Accept-encoding']
+			del req.headers['Youtubedl-no-compression']
+		return req
+
+	def http_response(self, req, resp):
+		old_resp = resp
+		# gzip
+		if resp.headers.get('Content-encoding', '') == 'gzip':
+			gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r')
+			resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
+			resp.msg = old_resp.msg
+		# deflate
+		if resp.headers.get('Content-encoding', '') == 'deflate':
+			gz = StringIO.StringIO(self.deflate(resp.read()))
+			resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
+			resp.msg = old_resp.msg
+		return resp