release 2014.01.07.4

[vimeo] Add support for passwords for player. URLs
Fixes #2053
2014-01-07 09:52:29 +01:00 · 2014-01-07 09:52:00 +01:00 · 2014-01-07 09:45:58 +01:00 · 2014-01-07 09:45:40 +01:00 · 2014-01-07 09:42:38 +01:00 · 2014-01-07 09:41:13 +01:00
29 changed files with 837 additions and 343 deletions
--- a/README.md
+++ b/README.md
@@ -193,7 +193,9 @@ which means you can modify it, redistribute it or use it however you like.
                               processed files are overwritten by default
    --embed-subs               embed subtitles in the video (only for mp4
                               videos)
-    --add-metadata             add metadata to the files
+    --add-metadata             write metadata to the video file
+    --xattrs                   write metadata to the video file's xattrs (using
+                               dublin core and xdg standards)

 # CONFIGURATION

--- a/devscripts/bash-completion.in
+++ b/devscripts/bash-completion.in
@@ -6,7 +6,7 @@ __youtube_dl()
    prev="${COMP_WORDS[COMP_CWORD-1]}"
    opts="{{flags}}"
    keywords=":ytfavorites :ytrecommended :ytsubscriptions :ytwatchlater :ythistory"
-    fileopts="-a|--batch-file|--download-archive|--cookies"
+    fileopts="-a|--batch-file|--download-archive|--cookies|--load-info"
    diropts="--cache-dir"

    if [[ ${prev} =~ ${fileopts} ]]; then
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@@ -24,6 +24,8 @@ if [ -z "$1" ]; then echo "ERROR: specify version number like this: $0 1994.09.0
 version="$1"
 if [ ! -z "`git tag | grep "$version"`" ]; then echo 'ERROR: version already present'; exit 1; fi
 if [ ! -z "`git status --porcelain | grep -v CHANGELOG`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi
+useless_files=$(find youtube_dl -type f -not -name '*.py')
+if [ ! -z "$useless_files" ]; then echo "ERROR: Non-.py files in youtube_dl: $useless_files"; exit 1; fi
 if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi

 /bin/echo -e "\n### First of all, testing..."
--- a/setup.py
+++ b/setup.py
@@ -71,7 +71,10 @@ setup(
    author_email='ytdl@yt-dl.org',
    maintainer='Philipp Hagemeister',
    maintainer_email='phihag@phihag.de',
-    packages=['youtube_dl', 'youtube_dl.extractor', 'youtube_dl.downloader'],
+    packages=[
+        'youtube_dl',
+        'youtube_dl.extractor', 'youtube_dl.downloader',
+        'youtube_dl.postprocessor'],

    # Provokes warning on most systems (why?!)
    # test_suite = 'nose.collector',
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -113,6 +113,8 @@ class TestAllURLsMatching(unittest.TestCase):
    def test_vimeo_matching(self):
        self.assertMatch('http://vimeo.com/channels/tributes', ['vimeo:channel'])
        self.assertMatch('http://vimeo.com/user7108434', ['vimeo:user'])
+        self.assertMatch('http://vimeo.com/user7108434/videos', ['vimeo:user'])
+        self.assertMatch('https://vimeo.com/user21297594/review/75524534/3c257a1b5d', ['vimeo:review'])

    # https://github.com/rg3/youtube-dl/issues/1930
    def test_soundcloud_not_matching_sets(self):
--- a/test/test_playlists.py
+++ b/test/test_playlists.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # encoding: utf-8

+from __future__ import unicode_literals

 # Allow direct execution
 import os
@@ -30,6 +31,7 @@ from youtube_dl.extractor import (
    SmotriUserIE,
    IviCompilationIE,
    ImdbListIE,
+    KhanAcademyIE,
 )


@@ -43,7 +45,7 @@ class TestPlaylists(unittest.TestCase):
        ie = DailymotionPlaylistIE(dl)
        result = ie.extract('http://www.dailymotion.com/playlist/xv4bw_nqtv_sport/1#video=xl8v3q')
        self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], u'SPORT')
+        self.assertEqual(result['title'], 'SPORT')
        self.assertTrue(len(result['entries']) > 20)

    def test_dailymotion_user(self):
@@ -51,7 +53,7 @@ class TestPlaylists(unittest.TestCase):
        ie = DailymotionUserIE(dl)
        result = ie.extract('http://www.dailymotion.com/user/generation-quoi/')
        self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], u'Génération Quoi')
+        self.assertEqual(result['title'], 'Génération Quoi')
        self.assertTrue(len(result['entries']) >= 26)

    def test_vimeo_channel(self):
@@ -59,7 +61,7 @@ class TestPlaylists(unittest.TestCase):
        ie = VimeoChannelIE(dl)
        result = ie.extract('http://vimeo.com/channels/tributes')
        self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], u'Vimeo Tributes')
+        self.assertEqual(result['title'], 'Vimeo Tributes')
        self.assertTrue(len(result['entries']) > 24)

    def test_vimeo_user(self):
@@ -67,7 +69,7 @@ class TestPlaylists(unittest.TestCase):
        ie = VimeoUserIE(dl)
        result = ie.extract('http://vimeo.com/nkistudio/videos')
        self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], u'Nki')
+        self.assertEqual(result['title'], 'Nki')
        self.assertTrue(len(result['entries']) > 65)

    def test_vimeo_album(self):
@@ -75,7 +77,7 @@ class TestPlaylists(unittest.TestCase):
        ie = VimeoAlbumIE(dl)
        result = ie.extract('http://vimeo.com/album/2632481')
        self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], u'Staff Favorites: November 2013')
+        self.assertEqual(result['title'], 'Staff Favorites: November 2013')
        self.assertTrue(len(result['entries']) > 12)

    def test_vimeo_groups(self):
@@ -83,7 +85,7 @@ class TestPlaylists(unittest.TestCase):
        ie = VimeoGroupsIE(dl)
        result = ie.extract('http://vimeo.com/groups/rolexawards')
        self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], u'Rolex Awards for Enterprise')
+        self.assertEqual(result['title'], 'Rolex Awards for Enterprise')
        self.assertTrue(len(result['entries']) > 72)

    def test_ustream_channel(self):
@@ -91,7 +93,7 @@ class TestPlaylists(unittest.TestCase):
        ie = UstreamChannelIE(dl)
        result = ie.extract('http://www.ustream.tv/channel/young-americans-for-liberty')
        self.assertIsPlaylist(result)
-        self.assertEqual(result['id'], u'5124905')
+        self.assertEqual(result['id'], '5124905')
        self.assertTrue(len(result['entries']) >= 11)

    def test_soundcloud_set(self):
@@ -99,7 +101,7 @@ class TestPlaylists(unittest.TestCase):
        ie = SoundcloudSetIE(dl)
        result = ie.extract('https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep')
        self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], u'The Royal Concept EP')
+        self.assertEqual(result['title'], 'The Royal Concept EP')
        self.assertTrue(len(result['entries']) >= 6)

    def test_soundcloud_user(self):
@@ -107,7 +109,7 @@ class TestPlaylists(unittest.TestCase):
        ie = SoundcloudUserIE(dl)
        result = ie.extract('https://soundcloud.com/the-concept-band')
        self.assertIsPlaylist(result)
-        self.assertEqual(result['id'], u'9615865')
+        self.assertEqual(result['id'], '9615865')
        self.assertTrue(len(result['entries']) >= 12)

    def test_livestream_event(self):
@@ -115,7 +117,7 @@ class TestPlaylists(unittest.TestCase):
        ie = LivestreamIE(dl)
        result = ie.extract('http://new.livestream.com/tedx/cityenglish')
        self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], u'TEDCity2.0 (English)')
+        self.assertEqual(result['title'], 'TEDCity2.0 (English)')
        self.assertTrue(len(result['entries']) >= 4)

    def test_nhl_videocenter(self):
@@ -123,8 +125,8 @@ class TestPlaylists(unittest.TestCase):
        ie = NHLVideocenterIE(dl)
        result = ie.extract('http://video.canucks.nhl.com/videocenter/console?catid=999')
        self.assertIsPlaylist(result)
-        self.assertEqual(result['id'], u'999')
-        self.assertEqual(result['title'], u'Highlights')
+        self.assertEqual(result['id'], '999')
+        self.assertEqual(result['title'], 'Highlights')
        self.assertEqual(len(result['entries']), 12)

    def test_bambuser_channel(self):
@@ -132,7 +134,7 @@ class TestPlaylists(unittest.TestCase):
        ie = BambuserChannelIE(dl)
        result = ie.extract('http://bambuser.com/channel/pixelversity')
        self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], u'pixelversity')
+        self.assertEqual(result['title'], 'pixelversity')
        self.assertTrue(len(result['entries']) >= 60)

    def test_bandcamp_album(self):
@@ -140,7 +142,7 @@ class TestPlaylists(unittest.TestCase):
        ie = BandcampAlbumIE(dl)
        result = ie.extract('http://mpallante.bandcamp.com/album/nightmare-night-ep')
        self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], u'Nightmare Night EP')
+        self.assertEqual(result['title'], 'Nightmare Night EP')
        self.assertTrue(len(result['entries']) >= 4)
        
    def test_smotri_community(self):
@@ -148,8 +150,8 @@ class TestPlaylists(unittest.TestCase):
        ie = SmotriCommunityIE(dl)
        result = ie.extract('http://smotri.com/community/video/kommuna')
        self.assertIsPlaylist(result)
-        self.assertEqual(result['id'], u'kommuna')
-        self.assertEqual(result['title'], u'КПРФ')
+        self.assertEqual(result['id'], 'kommuna')
+        self.assertEqual(result['title'], 'КПРФ')
        self.assertTrue(len(result['entries']) >= 4)
        
    def test_smotri_user(self):
@@ -157,17 +159,17 @@ class TestPlaylists(unittest.TestCase):
        ie = SmotriUserIE(dl)
        result = ie.extract('http://smotri.com/user/inspector')
        self.assertIsPlaylist(result)
-        self.assertEqual(result['id'], u'inspector')
-        self.assertEqual(result['title'], u'Inspector')
+        self.assertEqual(result['id'], 'inspector')
+        self.assertEqual(result['title'], 'Inspector')
        self.assertTrue(len(result['entries']) >= 9)

    def test_AcademicEarthCourse(self):
        dl = FakeYDL()
        ie = AcademicEarthCourseIE(dl)
-        result = ie.extract(u'http://academicearth.org/courses/building-dynamic-websites/')
+        result = ie.extract('http://academicearth.org/courses/building-dynamic-websites/')
        self.assertIsPlaylist(result)
-        self.assertEqual(result['id'], u'building-dynamic-websites')
-        self.assertEqual(result['title'], u'Building Dynamic Websites')
+        self.assertEqual(result['id'], 'building-dynamic-websites')
+        self.assertEqual(result['title'], 'Building Dynamic Websites')
        self.assertEqual(result['description'], u"Today's websites are increasingly dynamic. Pages are no longer static HTML files but instead generated by scripts and database calls. User interfaces are more seamless, with technologies like Ajax replacing traditional page reloads. This course teaches students how to build dynamic websites with Ajax and with Linux, Apache, MySQL, and PHP (LAMP), one of today's most popular frameworks. Students learn how to set up domain names with DNS, how to structure pages with XHTML and CSS, how to program in JavaScript and PHP, how to configure Apache and MySQL, how to design and query databases with SQL, how to use Ajax with both XML and JSON, and how to build mashups. The course explores issues of security, scalability, and cross-browser support and also discusses enterprise-level deployments of websites, including third-party hosting, virtualization, colocation in data centers, firewalling, and load-balancing.")
        self.assertEqual(len(result['entries']), 10)
        
@@ -176,8 +178,8 @@ class TestPlaylists(unittest.TestCase):
        ie = IviCompilationIE(dl)
        result = ie.extract('http://www.ivi.ru/watch/dezhurnyi_angel')
        self.assertIsPlaylist(result)
-        self.assertEqual(result['id'], u'dezhurnyi_angel')
-        self.assertEqual(result['title'], u'Дежурный ангел (2010 - 2012)')
+        self.assertEqual(result['id'], 'dezhurnyi_angel')
+        self.assertEqual(result['title'], 'Дежурный ангел (2010 - 2012)')
        self.assertTrue(len(result['entries']) >= 36)
        
    def test_ivi_compilation_season(self):
@@ -185,8 +187,8 @@ class TestPlaylists(unittest.TestCase):
        ie = IviCompilationIE(dl)
        result = ie.extract('http://www.ivi.ru/watch/dezhurnyi_angel/season2')
        self.assertIsPlaylist(result)
-        self.assertEqual(result['id'], u'dezhurnyi_angel/season2')
-        self.assertEqual(result['title'], u'Дежурный ангел (2010 - 2012) 2 сезон')
+        self.assertEqual(result['id'], 'dezhurnyi_angel/season2')
+        self.assertEqual(result['title'], 'Дежурный ангел (2010 - 2012) 2 сезон')
        self.assertTrue(len(result['entries']) >= 20)
        
    def test_imdb_list(self):
@@ -194,10 +196,20 @@ class TestPlaylists(unittest.TestCase):
        ie = ImdbListIE(dl)
        result = ie.extract('http://www.imdb.com/list/sMjedvGDd8U')
        self.assertIsPlaylist(result)
-        self.assertEqual(result['id'], u'sMjedvGDd8U')
-        self.assertEqual(result['title'], u'Animated and Family Films')
+        self.assertEqual(result['id'], 'sMjedvGDd8U')
+        self.assertEqual(result['title'], 'Animated and Family Films')
        self.assertTrue(len(result['entries']) >= 48)

+    def test_khanacademy_topic(self):
+        dl = FakeYDL()
+        ie = KhanAcademyIE(dl)
+        result = ie.extract('https://www.khanacademy.org/math/applied-math/cryptography')
+        self.assertIsPlaylist(result)
+        self.assertEqual(result['id'], 'cryptography')
+        self.assertEqual(result['title'], 'Journey into cryptography')
+        self.assertEqual(result['description'], 'How have humans protected their secret messages through history? What has changed today?')
+        self.assertTrue(len(result['entries']) >= 3)
+

 if __name__ == '__main__':
    unittest.main()
--- a/BIN
+++ b/BIN
--- a/youtube-dl.exe
+++ b/youtube-dl.exe
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -55,7 +55,7 @@ from .utils import (
 )
 from .extractor import get_info_extractor, gen_extractors
 from .downloader import get_suitable_downloader
-from .PostProcessor import FFmpegMergerPP
+from .postprocessor import FFmpegMergerPP
 from .version import __version__


@@ -1062,7 +1062,9 @@ class YoutubeDL(object):
                res += '%4dk ' % fdict['tbr']
            if (fdict.get('vcodec') is not None and
                    fdict.get('vcodec') != 'none'):
-                res += '%-5s@' % fdict['vcodec']
+                res += '%-5s' % fdict['vcodec']
+                if fdict.get('vbr') is not None:
+                    res += '@'
            elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
                res += 'video@'
            if fdict.get('vbr') is not None:
--- a/youtube_dl/init.py
+++ b/youtube_dl/init.py
@@ -38,6 +38,7 @@ __authors__  = (
    'Takuya Tsuchida',
    'Sergey M.',
    'Michael Orlitzky',
+    'Chris Gahan',
 )

 __license__ = 'Public Domain'
@@ -74,11 +75,12 @@ from .FileDownloader import (
 from .extractor import gen_extractors
 from .version import __version__
 from .YoutubeDL import YoutubeDL
-from .PostProcessor import (
+from .postprocessor import (
    FFmpegMetadataPP,
    FFmpegVideoConvertor,
    FFmpegExtractAudioPP,
    FFmpegEmbedSubtitlePP,
+    XAttrMetadataPP,
 )


@@ -415,7 +417,9 @@ def parseOpts(overrideArguments=None):
    postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
            help='embed subtitles in the video (only for mp4 videos)')
    postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
-            help='add metadata to the files')
+            help='write metadata to the video file')
+    postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
+            help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')


    parser.add_option_group(general)
@@ -523,7 +527,7 @@ def _real_main(argv=None):
    all_urls = batchurls + args
    all_urls = [url.strip() for url in all_urls]
    _enc = preferredencoding()
-    all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url]
+    all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]

    extractors = gen_extractors()

@@ -717,6 +721,8 @@ def _real_main(argv=None):
            ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
        if opts.embedsubtitles:
            ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
+        if opts.xattrs:
+            ydl.add_post_processor(XAttrMetadataPP())

        # Update version
        if opts.update_self:
--- a/youtube_dl/extractor/init.py
+++ b/youtube_dl/extractor/init.py
@@ -98,6 +98,7 @@ from .justintv import JustinTVIE
 from .jpopsukitv import JpopsukiIE
 from .kankan import KankanIE
 from .keezmovies import KeezMoviesIE
+from .khanacademy import KhanAcademyIE
 from .kickstarter import KickStarterIE
 from .keek import KeekIE
 from .liveleak import LiveLeakIE
@@ -112,6 +113,7 @@ from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mit import TechTVMITIE, MITIE
 from .mixcloud import MixcloudIE
+from .mpora import MporaIE
 from .mofosex import MofosexIE
 from .mtv import MTVIE
 from .muzu import MuzuTVIE
@@ -199,6 +201,7 @@ from .vimeo import (
    VimeoUserIE,
    VimeoAlbumIE,
    VimeoGroupsIE,
+    VimeoReviewIE,
 )
 from .vine import VineIE
 from .viki import VikiIE
--- a/youtube_dl/extractor/blinkx.py
+++ b/youtube_dl/extractor/blinkx.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import datetime
 import json
 import re
@@ -10,19 +12,19 @@ from ..utils import (

 class BlinkxIE(InfoExtractor):
    _VALID_URL = r'^(?:https?://(?:www\.)blinkx\.com/#?ce/|blinkx:)(?P<id>[^?]+)'
-    _IE_NAME = u'blinkx'
+    IE_NAME = 'blinkx'

    _TEST = {
-        u'url': u'http://www.blinkx.com/ce/8aQUy7GVFYgFzpKhT0oqsilwOGFRVXk3R1ZGWWdGenBLaFQwb3FzaWx3OGFRVXk3R1ZGWWdGenB',
-        u'file': u'8aQUy7GV.mp4',
-        u'md5': u'2e9a07364af40163a908edbf10bb2492',
-        u'info_dict': {
-            u"title": u"Police Car Rolls Away",
-            u"uploader": u"stupidvideos.com",
-            u"upload_date": u"20131215",
-            u"description": u"A police car gently rolls away from a fight. Maybe it felt weird being around a confrontation and just had to get out of there!",
-            u"duration": 14.886,
-            u"thumbnails": [{
+        'url': 'http://www.blinkx.com/ce/8aQUy7GVFYgFzpKhT0oqsilwOGFRVXk3R1ZGWWdGenBLaFQwb3FzaWx3OGFRVXk3R1ZGWWdGenB',
+        'file': '8aQUy7GV.mp4',
+        'md5': '2e9a07364af40163a908edbf10bb2492',
+        'info_dict': {
+            "title": "Police Car Rolls Away",
+            "uploader": "stupidvideos.com",
+            "upload_date": "20131215",
+            "description": "A police car gently rolls away from a fight. Maybe it felt weird being around a confrontation and just had to get out of there!",
+            "duration": 14.886,
+            "thumbnails": [{
                "width": 100,
                "height": 76,
                "url": "http://cdn.blinkx.com/stream/b/41/StupidVideos/20131215/1873969261/1873969261_tn_0.jpg",
@@ -30,17 +32,17 @@ class BlinkxIE(InfoExtractor):
        },
    }

-    def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
+    def _real_extract(self, rl):
+        m = re.match(self._VALID_URL, rl)
        video_id = m.group('id')
        display_id = video_id[:8]

        api_url = (u'https://apib4.blinkx.com/api.php?action=play_video&' +
-                   u'video=%s' % video_id)
+                   'video=%s' % video_id)
        data_json = self._download_webpage(api_url, display_id)
        data = json.loads(data_json)['api']['results'][0]
        dt = datetime.datetime.fromtimestamp(data['pubdate_epoch'])
-        upload_date = dt.strftime('%Y%m%d')
+        pload_date = dt.strftime('%Y%m%d')

        duration = None
        thumbnails = []
@@ -86,7 +88,7 @@ class BlinkxIE(InfoExtractor):
            'title': data['title'],
            'formats': formats,
            'uploader': data['channel_name'],
-            'upload_date': upload_date,
+            'upload_date': pload_date,
            'description': data.get('description'),
            'thumbnails': thumbnails,
            'duration': duration,
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -1,4 +1,5 @@
 # encoding: utf-8
+from __future__ import unicode_literals

 import re
 import json
@@ -13,6 +14,7 @@ from ..utils import (
    compat_urllib_request,

    ExtractorError,
+    unsmuggle_url,
 )


@@ -24,47 +26,47 @@ class BrightcoveIE(InfoExtractor):
    _TESTS = [
        {
            # From http://www.8tv.cat/8aldia/videos/xavier-sala-i-martin-aquesta-tarda-a-8-al-dia/
-            u'url': u'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1654948606001&flashID=myExperience&%40videoPlayer=2371591881001',
-            u'file': u'2371591881001.mp4',
-            u'md5': u'5423e113865d26e40624dce2e4b45d95',
-            u'note': u'Test Brightcove downloads and detection in GenericIE',
-            u'info_dict': {
-                u'title': u'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”',
-                u'uploader': u'8TV',
-                u'description': u'md5:a950cc4285c43e44d763d036710cd9cd',
+            'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1654948606001&flashID=myExperience&%40videoPlayer=2371591881001',
+            'file': '2371591881001.mp4',
+            'md5': '5423e113865d26e40624dce2e4b45d95',
+            'note': 'Test Brightcove downloads and detection in GenericIE',
+            'info_dict': {
+                'title': 'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”',
+                'uploader': '8TV',
+                'description': 'md5:a950cc4285c43e44d763d036710cd9cd',
            }
        },
        {
            # From http://medianetwork.oracle.com/video/player/1785452137001
-            u'url': u'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1217746023001&flashID=myPlayer&%40videoPlayer=1785452137001',
-            u'file': u'1785452137001.flv',
-            u'info_dict': {
-                u'title': u'JVMLS 2012: Arrays 2.0 - Opportunities and Challenges',
-                u'description': u'John Rose speaks at the JVM Language Summit, August 1, 2012.',
-                u'uploader': u'Oracle',
+            'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1217746023001&flashID=myPlayer&%40videoPlayer=1785452137001',
+            'file': '1785452137001.flv',
+            'info_dict': {
+                'title': 'JVMLS 2012: Arrays 2.0 - Opportunities and Challenges',
+                'description': 'John Rose speaks at the JVM Language Summit, August 1, 2012.',
+                'uploader': 'Oracle',
            },
        },
        {
            # From http://mashable.com/2013/10/26/thermoelectric-bracelet-lets-you-control-your-body-temperature/
-            u'url': u'http://c.brightcove.com/services/viewer/federated_f9?&playerID=1265504713001&publisherID=AQ%7E%7E%2CAAABBzUwv1E%7E%2CxP-xFHVUstiMFlNYfvF4G9yFnNaqCw_9&videoID=2750934548001',
-            u'info_dict': {
-                u'id': u'2750934548001',
-                u'ext': u'mp4',
-                u'title': u'This Bracelet Acts as a Personal Thermostat',
-                u'description': u'md5:547b78c64f4112766ccf4e151c20b6a0',
-                u'uploader': u'Mashable',
+            'url': 'http://c.brightcove.com/services/viewer/federated_f9?&playerID=1265504713001&publisherID=AQ%7E%7E%2CAAABBzUwv1E%7E%2CxP-xFHVUstiMFlNYfvF4G9yFnNaqCw_9&videoID=2750934548001',
+            'info_dict': {
+                'id': '2750934548001',
+                'ext': 'mp4',
+                'title': 'This Bracelet Acts as a Personal Thermostat',
+                'description': 'md5:547b78c64f4112766ccf4e151c20b6a0',
+                'uploader': 'Mashable',
            },
        },
        {
            # test that the default referer works
            # from http://national.ballet.ca/interact/video/Lost_in_Motion_II/
-            u'url': u'http://link.brightcove.com/services/player/bcpid756015033001?bckey=AQ~~,AAAApYJi_Ck~,GxhXCegT1Dp39ilhXuxMJxasUhVNZiil&bctid=2878862109001',
-            u'info_dict': {
-                u'id': u'2878862109001',
-                u'ext': u'mp4',
-                u'title': u'Lost in Motion II',
-                u'description': u'md5:363109c02998fee92ec02211bd8000df',
-                u'uploader': u'National Ballet of Canada',
+            'url': 'http://link.brightcove.com/services/player/bcpid756015033001?bckey=AQ~~,AAAApYJi_Ck~,GxhXCegT1Dp39ilhXuxMJxasUhVNZiil&bctid=2878862109001',
+            'info_dict': {
+                'id': '2878862109001',
+                'ext': 'mp4',
+                'title': 'Lost in Motion II',
+                'description': 'md5:363109c02998fee92ec02211bd8000df',
+                'uploader': 'National Ballet of Canada',
            },
        },
    ]
@@ -80,10 +82,10 @@ class BrightcoveIE(InfoExtractor):
        object_str = re.sub(r'(<param name="[^"]+" value="[^"]+")>',
                            lambda m: m.group(1) + '/>', object_str)
        # Fix up some stupid XML, see https://github.com/rg3/youtube-dl/issues/1608
-        object_str = object_str.replace(u'<--', u'<!--')
+        object_str = object_str.replace('<--', '<!--')

        object_doc = xml.etree.ElementTree.fromstring(object_str)
-        assert u'BrightcoveExperience' in object_doc.attrib['class']
+        assert 'BrightcoveExperience' in object_doc.attrib['class']
        params = {'flashID': object_doc.attrib['id'],
                  'playerID': find_xpath_attr(object_doc, './param', 'name', 'playerID').attrib['value'],
                  }
@@ -120,6 +122,8 @@ class BrightcoveIE(InfoExtractor):
            return None

    def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+
        # Change the 'videoId' and others field to '@videoPlayer'
        url = re.sub(r'(?<=[?&])(videoI(d|D)|bctid)', '%40videoPlayer', url)
        # Change bckey (used by bcove.me urls) to playerKey
@@ -130,9 +134,10 @@ class BrightcoveIE(InfoExtractor):

        videoPlayer = query.get('@videoPlayer')
        if videoPlayer:
-            return self._get_video_info(videoPlayer[0], query_str, query,
-                # We set the original url as the default 'Referer' header
-                referer=url)
+            # We set the original url as the default 'Referer' header
+            referer = smuggled_data.get('Referer', url)
+            return self._get_video_info(
+                videoPlayer[0], query_str, query, referer=referer)
        else:
            player_key = query['playerKey']
            return self._get_playlist_info(player_key[0])
@@ -156,11 +161,11 @@ class BrightcoveIE(InfoExtractor):

    def _get_playlist_info(self, player_key):
        playlist_info = self._download_webpage(self._PLAYLIST_URL_TEMPLATE % player_key,
-                                               player_key, u'Downloading playlist information')
+                                               player_key, 'Downloading playlist information')

        json_data = json.loads(playlist_info)
        if 'videoList' not in json_data:
-            raise ExtractorError(u'Empty playlist')
+            raise ExtractorError('Empty playlist')
        playlist_info = json_data['videoList']
        videos = [self._extract_video_info(video_info) for video_info in playlist_info['mediaCollectionDTO']['videoDTOs']]

@@ -189,5 +194,5 @@ class BrightcoveIE(InfoExtractor):
                'url': video_info['FLVFullLengthURL'],
            })
        else:
-            raise ExtractorError(u'Unable to extract video url for %s' % info['id'])
+            raise ExtractorError('Unable to extract video url for %s' % info['id'])
        return info
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1,4 +1,5 @@
 import base64
+import json
 import os
 import re
 import socket
@@ -73,6 +74,10 @@ class InfoExtractor(object):
                                 by this field.
                                 -1 for default (order by other properties),
                                 -2 or smaller for less than default.
+                    * quality    Order number of the video quality of this
+                                 format, irrespective of the file format.
+                                 -1 for default (order by other properties),
+                                 -2 or smaller for less than default.
    url:            Final video URL.
    ext:            Video filename extension.
    format:         The video format, defaults to ext (used for --get-format)
@@ -256,6 +261,15 @@ class InfoExtractor(object):
            xml_string = transform_source(xml_string)
        return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8'))

+    def _download_json(self, url_or_request, video_id,
+                       note=u'Downloading JSON metadata',
+                       errnote=u'Unable to download JSON metadata'):
+        json_string = self._download_webpage(url_or_request, video_id, note, errnote)
+        try:
+            return json.loads(json_string)
+        except ValueError as ve:
+            raise ExtractorError('Failed to download JSON', cause=ve)
+
    def report_warning(self, msg, video_id=None):
        idstr = u'' if video_id is None else u'%s: ' % video_id
        self._downloader.report_warning(
@@ -483,6 +497,7 @@ class InfoExtractor(object):

            return (
                preference,
+                f.get('quality') if f.get('quality') is not None else -1,
                f.get('height') if f.get('height') is not None else -1,
                f.get('width') if f.get('width') is not None else -1,
                ext_preference,
--- a/youtube_dl/extractor/defense.py
+++ b/youtube_dl/extractor/defense.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import re
 import json

@@ -5,15 +7,14 @@ from .common import InfoExtractor


 class DefenseGouvFrIE(InfoExtractor):
-    _IE_NAME = 'defense.gouv.fr'
+    IE_NAME = 'defense.gouv.fr'
    _VALID_URL = (r'http://.*?\.defense\.gouv\.fr/layout/set/'
        r'ligthboxvideo/base-de-medias/webtv/(.*)')

    _TEST = {
-        u'url': (u'http://www.defense.gouv.fr/layout/set/ligthboxvideo/'
-        u'base-de-medias/webtv/attaque-chimique-syrienne-du-21-aout-2013-1'),
-        u'file': u'11213.mp4',
-        u'md5': u'75bba6124da7e63d2d60b5244ec9430c',
+        'url': 'http://www.defense.gouv.fr/layout/set/ligthboxvideo/base-de-medias/webtv/attaque-chimique-syrienne-du-21-aout-2013-1',
+        'file': '11213.mp4',
+        'md5': '75bba6124da7e63d2d60b5244ec9430c',
        "info_dict": {
            "title": "attaque-chimique-syrienne-du-21-aout-2013-1"
        }
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1,9 +1,12 @@
 # encoding: utf-8

+from __future__ import unicode_literals
+
 import os
 import re

 from .common import InfoExtractor
+from .youtube import YoutubeIE
 from ..utils import (
    compat_urllib_error,
    compat_urllib_parse,
@@ -22,78 +25,78 @@ from .ooyala import OoyalaIE


 class GenericIE(InfoExtractor):
-    IE_DESC = u'Generic downloader that works on some sites'
+    IE_DESC = 'Generic downloader that works on some sites'
    _VALID_URL = r'.*'
-    IE_NAME = u'generic'
+    IE_NAME = 'generic'
    _TESTS = [
        {
-            u'url': u'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
-            u'file': u'13601338388002.mp4',
-            u'md5': u'6e15c93721d7ec9e9ca3fdbf07982cfd',
-            u'info_dict': {
-                u"uploader": u"www.hodiho.fr",
-                u"title": u"R\u00e9gis plante sa Jeep"
+            'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
+            'file': '13601338388002.mp4',
+            'md5': '6e15c93721d7ec9e9ca3fdbf07982cfd',
+            'info_dict': {
+                'uploader': 'www.hodiho.fr',
+                'title': 'R\u00e9gis plante sa Jeep',
            }
        },
        # embedded vimeo video
        {
-            u'add_ie': ['Vimeo'],
-            u'url': u'http://skillsmatter.com/podcast/home/move-semanticsperfect-forwarding-and-rvalue-references',
-            u'file': u'22444065.mp4',
-            u'md5': u'2903896e23df39722c33f015af0666e2',
-            u'info_dict': {
-                u'title': u'ACCU 2011: Move Semantics,Perfect Forwarding, and Rvalue references- Scott Meyers- 13/04/2011',
-                u"uploader_id": u"skillsmatter",
-                u"uploader": u"Skills Matter",
+            'add_ie': ['Vimeo'],
+            'url': 'http://skillsmatter.com/podcast/home/move-semanticsperfect-forwarding-and-rvalue-references',
+            'file': '22444065.mp4',
+            'md5': '2903896e23df39722c33f015af0666e2',
+            'info_dict': {
+                'title': 'ACCU 2011: Move Semantics,Perfect Forwarding, and Rvalue references- Scott Meyers- 13/04/2011',
+                'uploader_id': 'skillsmatter',
+                'uploader': 'Skills Matter',
            }
        },
        # bandcamp page with custom domain
        {
-            u'add_ie': ['Bandcamp'],
-            u'url': u'http://bronyrock.com/track/the-pony-mash',
-            u'file': u'3235767654.mp3',
-            u'info_dict': {
-                u'title': u'The Pony Mash',
-                u'uploader': u'M_Pallante',
+            'add_ie': ['Bandcamp'],
+            'url': 'http://bronyrock.com/track/the-pony-mash',
+            'file': '3235767654.mp3',
+            'info_dict': {
+                'title': 'The Pony Mash',
+                'uploader': 'M_Pallante',
            },
-            u'skip': u'There is a limit of 200 free downloads / month for the test song',
+            'skip': 'There is a limit of 200 free downloads / month for the test song',
        },
        # embedded brightcove video
        # it also tests brightcove videos that need to set the 'Referer' in the
        # http requests
        {
-            u'add_ie': ['Brightcove'],
-            u'url': u'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
-            u'info_dict': {
-                u'id': u'2765128793001',
-                u'ext': u'mp4',
-                u'title': u'Le cours de bourse : l’analyse technique',
-                u'description': u'md5:7e9ad046e968cb2d1114004aba466fd9',
-                u'uploader': u'BFM BUSINESS',
+            'add_ie': ['Brightcove'],
+            'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
+            'info_dict': {
+                'id': '2765128793001',
+                'ext': 'mp4',
+                'title': 'Le cours de bourse : l’analyse technique',
+                'description': 'md5:7e9ad046e968cb2d1114004aba466fd9',
+                'uploader': 'BFM BUSINESS',
            },
-            u'params': {
-                u'skip_download': True,
+            'params': {
+                'skip_download': True,
            },
        },
        # Direct link to a video
        {
-            u'url': u'http://media.w3.org/2010/05/sintel/trailer.mp4',
-            u'file': u'trailer.mp4',
-            u'md5': u'67d406c2bcb6af27fa886f31aa934bbe',
-            u'info_dict': {
-                u'id': u'trailer',
-                u'title': u'trailer',
-                u'upload_date': u'20100513',
+            'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
+            'file': 'trailer.mp4',
+            'md5': '67d406c2bcb6af27fa886f31aa934bbe',
+            'info_dict': {
+                'id': 'trailer',
+                'title': 'trailer',
+                'upload_date': '20100513',
            }
        },
        # ooyala video
        {
-            u'url': u'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
-            u'md5': u'5644c6ca5d5782c1d0d350dad9bd840c',
-            u'info_dict': {
-                u'id': u'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
-                u'ext': u'mp4',
-                u'title': u'2cc213299525360.mov', #that's what we get
+            'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
+            'md5': '5644c6ca5d5782c1d0d350dad9bd840c',
+            'info_dict': {
+                'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
+                'ext': 'mp4',
+                'title': '2cc213299525360.mov', #that's what we get
            },
        },
    ]
@@ -101,12 +104,12 @@ class GenericIE(InfoExtractor):
    def report_download_webpage(self, video_id):
        """Report webpage download."""
        if not self._downloader.params.get('test', False):
-            self._downloader.report_warning(u'Falling back on generic information extractor.')
+            self._downloader.report_warning('Falling back on generic information extractor.')
        super(GenericIE, self).report_download_webpage(video_id)

    def report_following_redirect(self, new_url):
        """Report information extraction."""
-        self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url)
+        self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)

    def _send_head(self, url):
        """Check if it is a redirect, like url shorteners, in case return the new url."""
@@ -152,7 +155,7 @@ class GenericIE(InfoExtractor):

        response = opener.open(HEADRequest(url))
        if response is None:
-            raise ExtractorError(u'Invalid URL protocol')
+            raise ExtractorError('Invalid URL protocol')
        return response

    def _real_extract(self, url):
@@ -162,7 +165,7 @@ class GenericIE(InfoExtractor):
            return self.url_result('http://' + url)
        video_id = os.path.splitext(url.split('/')[-1])[0]

-        self.to_screen(u'%s: Requesting header' % video_id)
+        self.to_screen('%s: Requesting header' % video_id)

        try:
            response = self._send_head(url)
@@ -186,7 +189,7 @@ class GenericIE(InfoExtractor):
                    'formats': [{
                        'format_id': m.group('format_id'),
                        'url': url,
-                        'vcodec': u'none' if m.group('type') == 'audio' else None
+                        'vcodec': 'none' if m.group('type') == 'audio' else None
                    }],
                    'upload_date': upload_date,
                }
@@ -200,7 +203,7 @@ class GenericIE(InfoExtractor):
        except ValueError:
            # since this is the last-resort InfoExtractor, if
            # this error is thrown, it'll be thrown here
-            raise ExtractorError(u'Failed to download URL: %s' % url)
+            raise ExtractorError('Failed to download URL: %s' % url)

        self.report_extraction(video_id)

@@ -211,18 +214,19 @@ class GenericIE(InfoExtractor):
        #   Video Title - Tagline | Site Name
        # and so on and so forth; it's just not practical
        video_title = self._html_search_regex(
-            r'(?s)<title>(.*?)</title>', webpage, u'video title',
-            default=u'video')
+            r'(?s)<title>(.*?)</title>', webpage, 'video title',
+            default='video')

        # video uploader is domain name
        video_uploader = self._search_regex(
-            r'^(?:https?://)?([^/]*)/.*', url, u'video uploader')
+            r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')

        # Look for BrightCove:
        bc_url = BrightcoveIE._extract_brightcove_url(webpage)
        if bc_url is not None:
-            self.to_screen(u'Brightcove video detected.')
-            return self.url_result(bc_url, 'Brightcove')
+            self.to_screen('Brightcove video detected.')
+            surl = smuggle_url(bc_url, {'Referer': url})
+            return self.url_result(surl, 'Brightcove')

        # Look for embedded (iframe) Vimeo player
        mobj = re.search(
@@ -303,6 +307,11 @@ class GenericIE(InfoExtractor):
        if mobj is not None:
            return self.url_result(mobj.group(1), 'Aparat')

+        # Look for MPORA videos
+        mobj = re.search(r'<iframe .*?src="(http://mpora.com/videos/[^"]+)"', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group(1), 'Mpora')
+
        # Start with something easy: JW Player in SWFObject
        mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
        if mobj is None:
@@ -328,23 +337,27 @@ class GenericIE(InfoExtractor):
            # HTML5 video
            mobj = re.search(r'<video[^<]*(?:>.*?<source.*?)? src="([^"]+)"', webpage, flags=re.DOTALL)
        if mobj is None:
-            raise ExtractorError(u'Unsupported URL: %s' % url)
+            raise ExtractorError('Unsupported URL: %s' % url)

        # It's possible that one of the regexes
        # matched, but returned an empty group:
        if mobj.group(1) is None:
-            raise ExtractorError(u'Did not find a valid video URL at %s' % url)
+            raise ExtractorError('Did not find a valid video URL at %s' % url)

        video_url = mobj.group(1)
        video_url = compat_urlparse.urljoin(url, video_url)
        video_id = compat_urllib_parse.unquote(os.path.basename(video_url))

+        # Sometimes, jwplayer extraction will result in a YouTube URL
+        if YoutubeIE.suitable(video_url):
+            return self.url_result(video_url, 'Youtube')
+
        # here's a fun little line of code for you:
        video_id = os.path.splitext(video_id)[0]

        return {
-            'id':       video_id,
-            'url':      video_url,
+            'id': video_id,
+            'url': video_url,
            'uploader': video_uploader,
-            'title':    video_title,
+            'title': video_title,
        }
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import re
 import json

@@ -9,18 +11,18 @@ from ..utils import (


 class ImdbIE(InfoExtractor):
-    IE_NAME = u'imdb'
-    IE_DESC = u'Internet Movie Database trailers'
+    IE_NAME = 'imdb'
+    IE_DESC = 'Internet Movie Database trailers'
    _VALID_URL = r'http://(?:www|m)\.imdb\.com/video/imdb/vi(?P<id>\d+)'

    _TEST = {
-        u'url': u'http://www.imdb.com/video/imdb/vi2524815897',
-        u'md5': u'9f34fa777ade3a6e57a054fdbcb3a068',
-        u'info_dict': {
-            u'id': u'2524815897',
-            u'ext': u'mp4',
-            u'title': u'Ice Age: Continental Drift Trailer (No. 2) - IMDb',
-            u'description': u'md5:9061c2219254e5d14e03c25c98e96a81',
+        'url': 'http://www.imdb.com/video/imdb/vi2524815897',
+        'md5': '9f34fa777ade3a6e57a054fdbcb3a068',
+        'info_dict': {
+            'id': '2524815897',
+            'ext': 'mp4',
+            'title': 'Ice Age: Continental Drift Trailer (No. 2) - IMDb',
+            'description': 'md5:9061c2219254e5d14e03c25c98e96a81',
        }
    }

@@ -37,10 +39,10 @@ class ImdbIE(InfoExtractor):
            f_path = f_path.strip()
            format_page = self._download_webpage(
                compat_urlparse.urljoin(url, f_path),
-                u'Downloading info for %s format' % f_id)
+                'Downloading info for %s format' % f_id)
            json_data = self._search_regex(
                r'<script[^>]+class="imdb-player-data"[^>]*?>(.*?)</script>',
-                format_page, u'json data', flags=re.DOTALL)
+                format_page, 'json data', flags=re.DOTALL)
            info = json.loads(json_data)
            format_info = info['videoPlayerObject']['video']
            formats.append({
@@ -56,9 +58,10 @@ class ImdbIE(InfoExtractor):
            'thumbnail': format_info['slate'],
        }

+
 class ImdbListIE(InfoExtractor):
-    IE_NAME = u'imdb:list'
-    IE_DESC = u'Internet Movie Database lists'
+    IE_NAME = 'imdb:list'
+    IE_DESC = 'Internet Movie Database lists'
    _VALID_URL = r'http://www\.imdb\.com/list/(?P<id>[\da-zA-Z_-]{11})'
    
    def _real_extract(self, url):
@@ -66,13 +69,13 @@ class ImdbListIE(InfoExtractor):
        list_id = mobj.group('id')
        
        # RSS XML is sometimes malformed
-        rss = self._download_webpage('http://rss.imdb.com/list/%s' % list_id, list_id, u'Downloading list RSS')
-        list_title = self._html_search_regex(r'<title>(.*?)</title>', rss, u'list title')
+        rss = self._download_webpage('http://rss.imdb.com/list/%s' % list_id, list_id, 'Downloading list RSS')
+        list_title = self._html_search_regex(r'<title>(.*?)</title>', rss, 'list title')
        
        # Export is independent of actual author_id, but returns 404 if no author_id is provided.
        # However, passing dummy author_id seems to be enough.
        csv = self._download_webpage('http://www.imdb.com/list/export?list_id=%s&author_id=ur00000000' % list_id,
-                                     list_id, u'Downloading list CSV')
+                                     list_id, 'Downloading list CSV')
        
        entries = []
        for item in csv.split('\n')[1:]:
@@ -83,4 +86,4 @@ class ImdbListIE(InfoExtractor):
            if item_id.startswith('vi'):
                entries.append(self.url_result('http://www.imdb.com/video/imdb/%s' % item_id, 'Imdb'))
        
-        return self.playlist_result(entries, list_id, list_title)
+        return self.playlist_result(entries, list_id, list_title)
--- a/youtube_dl/extractor/khanacademy.py
+++ b/youtube_dl/extractor/khanacademy.py
@@ -0,0 +1,71 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    unified_strdate,
+)
+
+
+class KhanAcademyIE(InfoExtractor):
+    _VALID_URL = r'^https?://(?:www\.)?khanacademy\.org/(?P<key>[^/]+)/(?:[^/]+/){,2}(?P<id>[^?#/]+)(?:$|[?#])'
+    IE_NAME = 'KhanAcademy'
+
+    _TEST = {
+        'url': 'http://www.khanacademy.org/video/one-time-pad',
+        'file': 'one-time-pad.mp4',
+        'md5': '7021db7f2d47d4fff89b13177cb1e8f4',
+        'info_dict': {
+            'title': 'The one-time pad',
+            'description': 'The perfect cipher',
+            'duration': 176,
+            'uploader': 'Brit Cruise',
+            'upload_date': '20120411',
+        }
+    }
+
+    def _real_extract(self, url):
+        m = re.match(self._VALID_URL, url)
+        video_id = m.group('id')
+
+        if m.group('key') == 'video':
+            data = self._download_json(
+                'http://api.khanacademy.org/api/v1/videos/' + video_id,
+                video_id, 'Downloading video info')
+
+            upload_date = unified_strdate(data['date_added'])
+            uploader = ', '.join(data['author_names'])
+            return {
+                '_type': 'url_transparent',
+                'url': data['url'],
+                'id': video_id,
+                'title': data['title'],
+                'thumbnail': data['image_url'],
+                'duration': data['duration'],
+                'description': data['description'],
+                'uploader': uploader,
+                'upload_date': upload_date,
+            }
+        else:
+            # topic
+            data = self._download_json(
+                'http://api.khanacademy.org/api/v1/topic/' + video_id,
+                video_id, 'Downloading topic info')
+
+            entries = [
+                {
+                    '_type': 'url',
+                    'url': c['url'],
+                    'id': c['id'],
+                    'title': c['title'],
+                }
+                for c in data['children'] if c['kind'] in ('Video', 'Topic')]
+
+            return {
+                '_type': 'playlist',
+                'id': video_id,
+                'title': data['title'],
+                'description': data['description'],
+                'entries': entries,
+            }
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -3,11 +3,12 @@ from __future__ import unicode_literals
 import re
 import json

+from .subtitles import SubtitlesInfoExtractor
 from .common import InfoExtractor
 from ..utils import ExtractorError


-class LyndaIE(InfoExtractor):
+class LyndaIE(SubtitlesInfoExtractor):
    IE_NAME = 'lynda'
    IE_DESC = 'lynda.com videos'
    _VALID_URL = r'https?://www\.lynda\.com/[^/]+/[^/]+/\d+/(\d+)-\d\.html'
@@ -16,12 +17,12 @@ class LyndaIE(InfoExtractor):
        'url': 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html',
        'file': '114408.mp4',
        'md5': 'ecfc6862da89489161fb9cd5f5a6fac1',
-        u"info_dict": {
+        'info_dict': {
            'title': 'Using the exercise files',
            'duration': 68
        }
    }
-
+    
    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group(1)
@@ -45,17 +46,59 @@ class LyndaIE(InfoExtractor):
                    'width': fmt['Width'],
                    'height': fmt['Height'],
                    'filesize': fmt['FileSize'],
-                    'format_id': fmt['Resolution']
+                    'format_id': str(fmt['Resolution'])
                    } for fmt in video_json['Formats']]

        self._sort_formats(formats)
-
+        
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(video_id, page)
+            return
+        
+        subtitles = self._fix_subtitles(self.extract_subtitles(video_id, page))
+        
        return {
            'id': video_id,
            'title': title,
            'duration': duration,
+            'subtitles': subtitles,
            'formats': formats
        }
+        
+    _TIMECODE_REGEX = r'\[(?P<timecode>\d+:\d+:\d+[\.,]\d+)\]'
+
+    def _fix_subtitles(self, subtitles):
+        if subtitles is None:
+            return subtitles  # subtitles not requested
+
+        fixed_subtitles = {}
+        for k, v in subtitles.items():
+            subs = json.loads(v)
+            if len(subs) == 0:
+                continue
+            srt = ''
+            for pos in range(0, len(subs) - 1):
+                seq_current = subs[pos]
+                m_current = re.match(self._TIMECODE_REGEX, seq_current['Timecode'])
+                if m_current is None:
+                    continue
+                seq_next = subs[pos + 1]
+                m_next = re.match(self._TIMECODE_REGEX, seq_next['Timecode'])
+                if m_next is None:
+                    continue
+                appear_time = m_current.group('timecode')
+                disappear_time = m_next.group('timecode')
+                text = seq_current['Caption']
+                srt += '%s\r\n%s --> %s\r\n%s' % (str(pos), appear_time, disappear_time, text)
+            if srt:
+                fixed_subtitles[k] = srt
+        return fixed_subtitles
+        
+    def _get_available_subtitles(self, video_id, webpage):
+        url = 'http://www.lynda.com/ajax/player?videoId=%s&type=transcript' % video_id
+        sub = self._download_webpage(url, None, note=False)
+        sub_json = json.loads(sub)
+        return {'en': url} if len(sub_json) > 0 else {}


 class LyndaCourseIE(InfoExtractor):
--- a/youtube_dl/extractor/mpora.py
+++ b/youtube_dl/extractor/mpora.py
@@ -0,0 +1,66 @@
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+)
+
+
+class MporaIE(InfoExtractor):
+    _VALID_URL = r'^https?://(www\.)?mpora\.(?:com|de)/videos/(?P<id>[^?#/]+)'
+    IE_NAME = 'MPORA'
+
+    _TEST = {
+        'url': 'http://mpora.de/videos/AAdo8okx4wiz/embed?locale=de',
+        'file': 'AAdo8okx4wiz.mp4',
+        'md5': 'a7a228473eedd3be741397cf452932eb',
+        'info_dict': {
+            'title': 'Katy Curd -  Winter in the Forest',
+            'duration': 416,
+            'uploader': 'petenewman',
+        },
+    }
+
+    def _real_extract(self, url):
+        m = re.match(self._VALID_URL, url)
+        video_id = m.group('id')
+
+        webpage = self._download_webpage(url, video_id)
+        data_json = self._search_regex(
+            r"new FM\.Player\('[^']+',\s*(\{.*?)\);\n", webpage, 'json')
+
+        data = json.loads(data_json)
+
+        uploader = data['info_overlay']['name']
+        duration = data['video']['duration'] // 1000
+        thumbnail = data['video']['encodings']['sd']['poster']
+        title = data['info_overlay']['title']
+
+        formats = []
+        for encoding_id, edata in data['video']['encodings'].items():
+            for src in edata['sources']:
+                width_str = self._search_regex(
+                    r'_([0-9]+)\.[a-zA-Z0-9]+$', src['src'],
+                    False, default=None)
+                vcodec = src['type'].partition('/')[2]
+                
+                formats.append({
+                    'format_id': encoding_id + '-' + vcodec,
+                    'url': src['src'],
+                    'vcodec': vcodec,
+                    'width': int_or_none(width_str),
+                })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'uploader': uploader,
+            'duration': duration,
+            'thumbnail': thumbnail,
+        }
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -1,54 +1,98 @@
 # coding: utf-8
+from __future__ import unicode_literals

-import re
-import xml.etree.ElementTree
 import json
+import re

 from .common import InfoExtractor
 from ..utils import (
-    compat_urlparse,
-    ExtractorError,
-    find_xpath_attr,
+    HEADRequest,
+    unified_strdate,
 )

+
 class ORFIE(InfoExtractor):
-    _VALID_URL = r'https?://tvthek\.orf\.at/(programs/.+?/episodes|topics/.+?)/(?P<id>\d+)'
+    _VALID_URL = r'https?://tvthek\.orf\.at/(?:programs/.+?/episodes|topics/.+?|program/[^/]+)/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://tvthek.orf.at/program/matinee-Was-Sie-schon-immer-ueber-Klassik-wissen-wollten/7317210/Was-Sie-schon-immer-ueber-Klassik-wissen-wollten/7319746/Was-Sie-schon-immer-ueber-Klassik-wissen-wollten/7319747',
+        'file': '7319747.mp4',
+        'md5': 'bd803c5d8c32d3c64a0ea4b4eeddf375',
+        'info_dict': {
+            'title': 'Was Sie schon immer über Klassik wissen wollten',
+            'description': 'md5:0ddf0d5f0060bd53f744edaa5c2e04a4',
+            'duration': 3508,
+            'upload_date': '20140105',
+        },
+        'skip': 'Blocked outside of Austria',
+    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        playlist_id = mobj.group('id')
        webpage = self._download_webpage(url, playlist_id)

-        flash_xml = self._search_regex('ORF.flashXML = \'(.+?)\'', webpage, u'flash xml')
-        flash_xml = compat_urlparse.parse_qs('xml='+flash_xml)['xml'][0]
-        flash_config = xml.etree.ElementTree.fromstring(flash_xml.encode('utf-8'))
-        playlist_json = self._search_regex(r'playlist\': \'(\[.*?\])\'', webpage, u'playlist').replace(r'\"','"')
-        playlist = json.loads(playlist_json)
+        data_json = self._search_regex(
+            r'initializeAdworx\((.+?)\);\n', webpage, 'video info')
+        all_data = json.loads(data_json)
+        sdata = all_data[0]['values']['segments']

-        videos = []
-        ns = '{http://tempuri.org/XMLSchema.xsd}'
-        xpath = '%(ns)sPlaylist/%(ns)sItems/%(ns)sItem' % {'ns': ns}
-        webpage_description = self._og_search_description(webpage)
-        for (i, (item, info)) in enumerate(zip(flash_config.findall(xpath), playlist), 1):
-            # Get best quality url
-            rtmp_url = None
-            for q in ['Q6A', 'Q4A', 'Q1A']:
-                video_url = find_xpath_attr(item, '%sVideoUrl' % ns, 'quality', q)
-                if video_url is not None:
-                    rtmp_url = video_url.text
-                    break
-            if rtmp_url is None:
-                raise ExtractorError(u'Couldn\'t get video url: %s' % info['id'])
-            description = self._html_search_regex(
-                r'id="playlist_entry_%s".*?<p>(.*?)</p>' % i, webpage,
-                u'description', default=webpage_description, flags=re.DOTALL)
-            videos.append({
+        def quality_to_int(s):
+            m = re.search('([0-9]+)', s)
+            if m is None:
+                return -1
+            return int(m.group(1))
+
+        entries = []
+        for sd in sdata:
+            video_id = sd['id']
+            formats = [{
+                'preference': -10 if fd['delivery'] == 'hls' else None,
+                'format_id': '%s-%s-%s' % (
+                    fd['delivery'], fd['quality'], fd['quality_string']),
+                'url': fd['src'],
+                'protocol': fd['protocol'],
+                'quality': quality_to_int(fd['quality']),
+            } for fd in sd['playlist_item_array']['sources']]
+
+            # Check for geoblocking.
+            # There is a property is_geoprotection, but that's always false
+            geo_str = sd.get('geoprotection_string')
+            if geo_str:
+                try:
+                    http_url = next(
+                        f['url']
+                        for f in formats
+                        if re.match(r'^https?://.*\.mp4$', f['url']))
+                except StopIteration:
+                    pass
+                else:
+                    req = HEADRequest(http_url)
+                    self._request_webpage(
+                        req, video_id,
+                        note='Testing for geoblocking',
+                        errnote=((
+                            'This video seems to be blocked outside of %s. '
+                            'You may want to try the streaming-* formats.')
+                            % geo_str),
+                        fatal=False)
+
+            self._sort_formats(formats)
+
+            upload_date = unified_strdate(sd['created_date'])
+            entries.append({
                '_type': 'video',
-                'id': info['id'],
-                'title': info['title'],
-                'url': rtmp_url,
-                'ext': 'flv',
-                'description': description,
-                })
+                'id': video_id,
+                'title': sd['header'],
+                'formats': formats,
+                'description': sd.get('description'),
+                'duration': int(sd['duration_in_seconds']),
+                'upload_date': upload_date,
+                'thumbnail': sd.get('image_full_url'),
+            })

-        return videos
+        return {
+            '_type': 'playlist',
+            'entries': entries,
+            'id': playlist_id,
+        }
--- a/youtube_dl/extractor/veehd.py
+++ b/youtube_dl/extractor/veehd.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import re
 import json

@@ -8,16 +10,17 @@ from ..utils import (
    clean_html,
 )

+
 class VeeHDIE(InfoExtractor):
    _VALID_URL = r'https?://veehd\.com/video/(?P<id>\d+)'

    _TEST = {
-        u'url': u'http://veehd.com/video/4686958',
-        u'file': u'4686958.mp4',
-        u'info_dict': {
-            u'title': u'Time Lapse View from Space ( ISS)',
-            u'uploader_id': u'spotted',
-            u'description': u'md5:f0094c4cf3a72e22bc4e4239ef767ad7',
+        'url': 'http://veehd.com/video/4686958',
+        'file': '4686958.mp4',
+        'info_dict': {
+            'title': 'Time Lapse View from Space ( ISS)',
+            'uploader_id': 'spotted',
+            'description': 'md5:f0094c4cf3a72e22bc4e4239ef767ad7',
        },
    }

@@ -25,24 +28,30 @@ class VeeHDIE(InfoExtractor):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

+        # VeeHD seems to send garbage on the first request.
+        # See https://github.com/rg3/youtube-dl/issues/2102
+        self._download_webpage(url, video_id, 'Requesting webpage')
        webpage = self._download_webpage(url, video_id)
-        player_path = self._search_regex(r'\$\("#playeriframe"\).attr\({src : "(.+?)"',
-            webpage, u'player path')
+        player_path = self._search_regex(
+            r'\$\("#playeriframe"\).attr\({src : "(.+?)"',
+            webpage, 'player path')
        player_url = compat_urlparse.urljoin(url, player_path)
-        player_page = self._download_webpage(player_url, video_id,
-            u'Downloading player page')
-        config_json = self._search_regex(r'value=\'config=({.+?})\'',
-            player_page, u'config json')
+
+        self._download_webpage(player_url, video_id, 'Requesting player page')
+        player_page = self._download_webpage(
+            player_url, video_id, 'Downloading player page')
+        config_json = self._search_regex(
+            r'value=\'config=({.+?})\'', player_page, 'config json')
        config = json.loads(config_json)

        video_url = compat_urlparse.unquote(config['clip']['url'])
        title = clean_html(get_element_by_id('videoName', webpage).rpartition('|')[0])
        uploader_id = self._html_search_regex(r'<a href="/profile/\d+">(.+?)</a>',
-            webpage, u'uploader')
+            webpage, 'uploader')
        thumbnail = self._search_regex(r'<img id="veehdpreview" src="(.+?)"',
-            webpage, u'thumbnail')
+            webpage, 'thumbnail')
        description = self._html_search_regex(r'<td class="infodropdown".*?<div>(.*?)<ul',
-            webpage, u'description', flags=re.DOTALL)
+            webpage, 'description', flags=re.DOTALL)

        return {
            '_type': 'video',
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -1,4 +1,6 @@
 # encoding: utf-8
+from __future__ import unicode_literals
+
 import json
 import re
 import itertools
@@ -31,54 +33,55 @@ class VimeoIE(InfoExtractor):
        (?P<id>[0-9]+)
        /?(?:[?&].*)?(?:[#].*)?$'''
    _NETRC_MACHINE = 'vimeo'
-    IE_NAME = u'vimeo'
+    IE_NAME = 'vimeo'
    _TESTS = [
        {
-            u'url': u'http://vimeo.com/56015672#at=0',
-            u'file': u'56015672.mp4',
-            u'md5': u'8879b6cc097e987f02484baf890129e5',
-            u'info_dict': {
-                u"upload_date": u"20121220", 
-                u"description": u"This is a test case for youtube-dl.\nFor more information, see github.com/rg3/youtube-dl\nTest chars: \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550", 
-                u"uploader_id": u"user7108434", 
-                u"uploader": u"Filippo Valsorda", 
-                u"title": u"youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
+            'url': 'http://vimeo.com/56015672#at=0',
+            'file': '56015672.mp4',
+            'md5': '8879b6cc097e987f02484baf890129e5',
+            'info_dict': {
+                "upload_date": "20121220", 
+                "description": "This is a test case for youtube-dl.\nFor more information, see github.com/rg3/youtube-dl\nTest chars: \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550", 
+                "uploader_id": "user7108434", 
+                "uploader": "Filippo Valsorda", 
+                "title": "youtube-dl test video - \u2605 \" ' \u5e78 / \\ \u00e4 \u21ad \U0001d550",
            },
        },
        {
-            u'url': u'http://vimeopro.com/openstreetmapus/state-of-the-map-us-2013/video/68093876',
-            u'file': u'68093876.mp4',
-            u'md5': u'3b5ca6aa22b60dfeeadf50b72e44ed82',
-            u'note': u'Vimeo Pro video (#1197)',
-            u'info_dict': {
-                u'uploader_id': u'openstreetmapus', 
-                u'uploader': u'OpenStreetMap US', 
-                u'title': u'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
+            'url': 'http://vimeopro.com/openstreetmapus/state-of-the-map-us-2013/video/68093876',
+            'file': '68093876.mp4',
+            'md5': '3b5ca6aa22b60dfeeadf50b72e44ed82',
+            'note': 'Vimeo Pro video (#1197)',
+            'info_dict': {
+                'uploader_id': 'openstreetmapus',
+                'uploader': 'OpenStreetMap US',
+                'title': 'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
            },
        },
        {
-            u'url': u'http://player.vimeo.com/video/54469442',
-            u'file': u'54469442.mp4',
-            u'md5': u'619b811a4417aa4abe78dc653becf511',
-            u'note': u'Videos that embed the url in the player page',
-            u'info_dict': {
-                u'title': u'Kathy Sierra: Building the minimum Badass User, Business of Software',
-                u'uploader': u'The BLN & Business of Software',
+            'url': 'http://player.vimeo.com/video/54469442',
+            'file': '54469442.mp4',
+            'md5': '619b811a4417aa4abe78dc653becf511',
+            'note': 'Videos that embed the url in the player page',
+            'info_dict': {
+                'title': 'Kathy Sierra: Building the minimum Badass User, Business of Software',
+                'uploader': 'The BLN & Business of Software',
+                'uploader_id': 'theblnbusinessofsoftware',
            },
        },
        {
-            u'url': u'http://vimeo.com/68375962',
-            u'file': u'68375962.mp4',
-            u'md5': u'aaf896bdb7ddd6476df50007a0ac0ae7',
-            u'note': u'Video protected with password',
-            u'info_dict': {
-                u'title': u'youtube-dl password protected test video',
-                u'upload_date': u'20130614',
-                u'uploader_id': u'user18948128',
-                u'uploader': u'Jaime Marquínez Ferrándiz',
+            'url': 'http://vimeo.com/68375962',
+            'file': '68375962.mp4',
+            'md5': 'aaf896bdb7ddd6476df50007a0ac0ae7',
+            'note': 'Video protected with password',
+            'info_dict': {
+                'title': 'youtube-dl password protected test video',
+                'upload_date': '20130614',
+                'uploader_id': 'user18948128',
+                'uploader': 'Jaime Marquínez Ferrándiz',
            },
-            u'params': {
-                u'videopassword': u'youtube-dl',
+            'params': {
+                'videopassword': 'youtube-dl',
            },
        },
    ]
@@ -90,7 +93,7 @@ class VimeoIE(InfoExtractor):
        self.report_login()
        login_url = 'https://vimeo.com/log_in'
        webpage = self._download_webpage(login_url, None, False)
-        token = re.search(r'xsrft: \'(.*?)\'', webpage).group(1)
+        token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
        data = compat_urllib_parse.urlencode({'email': username,
                                              'password': password,
                                              'action': 'login',
@@ -100,13 +103,13 @@ class VimeoIE(InfoExtractor):
        login_request = compat_urllib_request.Request(login_url, data)
        login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
        login_request.add_header('Cookie', 'xsrft=%s' % token)
-        self._download_webpage(login_request, None, False, u'Wrong login info')
+        self._download_webpage(login_request, None, False, 'Wrong login info')

    def _verify_video_password(self, url, video_id, webpage):
        password = self._downloader.params.get('videopassword', None)
        if password is None:
-            raise ExtractorError(u'This video is protected by a password, use the --video-password option')
-        token = re.search(r'xsrft: \'(.*?)\'', webpage).group(1)
+            raise ExtractorError('This video is protected by a password, use the --video-password option')
+        token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
        data = compat_urllib_parse.urlencode({'password': password,
                                              'token': token})
        # I didn't manage to use the password with https
@@ -118,8 +121,21 @@ class VimeoIE(InfoExtractor):
        password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
        password_request.add_header('Cookie', 'xsrft=%s' % token)
        self._download_webpage(password_request, video_id,
-                               u'Verifying the password',
-                               u'Wrong password')
+                               'Verifying the password',
+                               'Wrong password')
+
+    def _verify_player_video_password(self, url, video_id):
+        password = self._downloader.params.get('videopassword', None)
+        if password is None:
+            raise ExtractorError('This video is protected by a password, use the --video-password option')
+        data = compat_urllib_parse.urlencode({'password': password})
+        pass_url = url + '/check-password'
+        password_request = compat_urllib_request.Request(pass_url, data)
+        password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        return self._download_json(
+            password_request, video_id,
+            'Verifying the password',
+            'Wrong password')

    def _real_initialize(self):
        self._login()
@@ -134,7 +150,7 @@ class VimeoIE(InfoExtractor):
        # Extract ID from URL
        mobj = re.match(self._VALID_URL, url)
        if mobj is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)
+            raise ExtractorError('Invalid URL: %s' % url)

        video_id = mobj.group('id')
        if mobj.group('pro') or mobj.group('player'):
@@ -155,7 +171,7 @@ class VimeoIE(InfoExtractor):
        try:
            try:
                config_url = self._html_search_regex(
-                    r' data-config-url="(.+?)"', webpage, u'config URL')
+                    r' data-config-url="(.+?)"', webpage, 'config URL')
                config_json = self._download_webpage(config_url, video_id)
                config = json.loads(config_json)
            except RegexNotFoundError:
@@ -166,19 +182,22 @@ class VimeoIE(InfoExtractor):
                    config_re = r'%s=({.+?});' % re.escape(m_variable_name.group(1))
                else:
                    config_re = [r' = {config:({.+?}),assets:', r'(?:[abc])=({.+?});']
-                config = self._search_regex(config_re, webpage, u'info section',
+                config = self._search_regex(config_re, webpage, 'info section',
                    flags=re.DOTALL)
                config = json.loads(config)
        except Exception as e:
            if re.search('The creator of this video has not given you permission to embed it on this domain.', webpage):
-                raise ExtractorError(u'The author has restricted the access to this video, try with the "--referer" option')
+                raise ExtractorError('The author has restricted the access to this video, try with the "--referer" option')

            if re.search('<form[^>]+?id="pw_form"', webpage) is not None:
                self._verify_video_password(url, video_id, webpage)
                return self._real_extract(url)
            else:
-                raise ExtractorError(u'Unable to extract info section',
+                raise ExtractorError('Unable to extract info section',
                                     cause=e)
+        else:
+            if config.get('view') == 4:
+                config = self._verify_player_video_password(url, video_id)

        # Extract title
        video_title = config["video"]["title"]
@@ -212,9 +231,9 @@ class VimeoIE(InfoExtractor):
            video_upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3)

        try:
-            view_count = int(self._search_regex(r'UserPlays:(\d+)', webpage, u'view count'))
-            like_count = int(self._search_regex(r'UserLikes:(\d+)', webpage, u'like count'))
-            comment_count = int(self._search_regex(r'UserComments:(\d+)', webpage, u'comment count'))
+            view_count = int(self._search_regex(r'UserPlays:(\d+)', webpage, 'view count'))
+            like_count = int(self._search_regex(r'UserLikes:(\d+)', webpage, 'like count'))
+            comment_count = int(self._search_regex(r'UserComments:(\d+)', webpage, 'comment count'))
        except RegexNotFoundError:
            # This info is only available in vimeo.com/{id} urls
            view_count = None
@@ -255,7 +274,7 @@ class VimeoIE(InfoExtractor):
        for key in ('other', 'sd', 'hd'):
            formats += files[key]
        if len(formats) == 0:
-            raise ExtractorError(u'No known codec found')
+            raise ExtractorError('No known codec found')

        return {
            'id':       video_id,
@@ -274,7 +293,7 @@ class VimeoIE(InfoExtractor):


 class VimeoChannelIE(InfoExtractor):
-    IE_NAME = u'vimeo:channel'
+    IE_NAME = 'vimeo:channel'
    _VALID_URL = r'(?:https?://)?vimeo.\com/channels/(?P<id>[^/]+)'
    _MORE_PAGES_INDICATOR = r'<a.+?rel="next"'
    _TITLE_RE = r'<link rel="alternate"[^>]+?title="(.*?)"'
@@ -283,14 +302,14 @@ class VimeoChannelIE(InfoExtractor):
        return '%s/videos/page:%d/' % (base_url, pagenum)

    def _extract_list_title(self, webpage):
-        return self._html_search_regex(self._TITLE_RE, webpage, u'list title')
+        return self._html_search_regex(self._TITLE_RE, webpage, 'list title')

    def _extract_videos(self, list_id, base_url):
        video_ids = []
        for pagenum in itertools.count(1):
            webpage = self._download_webpage(
                self._page_url(base_url, pagenum) ,list_id,
-                u'Downloading page %s' % pagenum)
+                'Downloading page %s' % pagenum)
            video_ids.extend(re.findall(r'id="clip_(\d+?)"', webpage))
            if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
                break
@@ -310,8 +329,8 @@ class VimeoChannelIE(InfoExtractor):


 class VimeoUserIE(VimeoChannelIE):
-    IE_NAME = u'vimeo:user'
-    _VALID_URL = r'(?:https?://)?vimeo.\com/(?P<name>[^/]+)'
+    IE_NAME = 'vimeo:user'
+    _VALID_URL = r'(?:https?://)?vimeo.\com/(?P<name>[^/]+)(?:/videos|[#?]|$)'
    _TITLE_RE = r'<a[^>]+?class="user">([^<>]+?)</a>'

    @classmethod
@@ -327,7 +346,7 @@ class VimeoUserIE(VimeoChannelIE):


 class VimeoAlbumIE(VimeoChannelIE):
-    IE_NAME = u'vimeo:album'
+    IE_NAME = 'vimeo:album'
    _VALID_URL = r'(?:https?://)?vimeo.\com/album/(?P<id>\d+)'
    _TITLE_RE = r'<header id="page_header">\n\s*<h1>(.*?)</h1>'

@@ -336,12 +355,12 @@ class VimeoAlbumIE(VimeoChannelIE):

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
-        album_id =  mobj.group('id')
+        album_id = mobj.group('id')
        return self._extract_videos(album_id, 'http://vimeo.com/album/%s' % album_id)


 class VimeoGroupsIE(VimeoAlbumIE):
-    IE_NAME = u'vimeo:group'
+    IE_NAME = 'vimeo:group'
    _VALID_URL = r'(?:https?://)?vimeo.\com/groups/(?P<name>[^/]+)'

    def _extract_list_title(self, webpage):
@@ -351,3 +370,24 @@ class VimeoGroupsIE(VimeoAlbumIE):
        mobj = re.match(self._VALID_URL, url)
        name = mobj.group('name')
        return self._extract_videos(name, 'http://vimeo.com/groups/%s' % name)
+
+
+class VimeoReviewIE(InfoExtractor):
+    IE_NAME = 'vimeo:review'
+    IE_DESC = 'Review pages on vimeo'
+    _VALID_URL = r'(?:https?://)?vimeo.\com/[^/]+/review/(?P<id>[^/]+)'
+    _TEST = {
+        'url': 'https://vimeo.com/user21297594/review/75524534/3c257a1b5d',
+        'file': '75524534.mp4',
+        'md5': 'c507a72f780cacc12b2248bb4006d253',
+        'info_dict': {
+            'title': "DICK HARDWICK 'Comedian'",
+            'uploader': 'Richard Hardwick',
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        player_url = 'https://player.vimeo.com/player/' + video_id
+        return self.url_result(player_url, 'Vimeo', video_id)
--- a/youtube_dl/postprocessor/init.py
+++ b/youtube_dl/postprocessor/init.py
@@ -0,0 +1,18 @@
+
+from .ffmpeg import (
+    FFmpegMergerPP,
+    FFmpegMetadataPP,
+    FFmpegVideoConvertor,
+    FFmpegExtractAudioPP,
+    FFmpegEmbedSubtitlePP,
+)
+from .xattrpp import XAttrMetadataPP
+
+__all__ = [
+    'FFmpegMergerPP',
+    'FFmpegMetadataPP',
+    'FFmpegVideoConvertor',
+    'FFmpegExtractAudioPP',
+    'FFmpegEmbedSubtitlePP',
+    'XAttrMetadataPP',
+]
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -0,0 +1,49 @@
+from ..utils import PostProcessingError
+
+
+class PostProcessor(object):
+    """Post Processor class.
+
+    PostProcessor objects can be added to downloaders with their
+    add_post_processor() method. When the downloader has finished a
+    successful download, it will take its internal chain of PostProcessors
+    and start calling the run() method on each one of them, first with
+    an initial argument and then with the returned value of the previous
+    PostProcessor.
+
+    The chain will be stopped if one of them ever returns None or the end
+    of the chain is reached.
+
+    PostProcessor objects follow a "mutual registration" process similar
+    to InfoExtractor objects.
+    """
+
+    _downloader = None
+
+    def __init__(self, downloader=None):
+        self._downloader = downloader
+
+    def set_downloader(self, downloader):
+        """Sets the downloader for this PP."""
+        self._downloader = downloader
+
+    def run(self, information):
+        """Run the PostProcessor.
+
+        The "information" argument is a dictionary like the ones
+        composed by InfoExtractors. The only difference is that this
+        one has an extra field called "filepath" that points to the
+        downloaded file.
+
+        This method returns a tuple, the first element of which describes
+        whether the original file should be kept (i.e. not deleted - None for
+        no preference), and the second of which is the updated information.
+
+        In addition, this method may raise a PostProcessingError
+        exception if post processing fails.
+        """
+        return None, information  # by default, keep file and do nothing
+
+
+class AudioConversionError(PostProcessingError):
+    pass
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -4,65 +4,23 @@ import sys
 import time


-from .utils import (
+from .common import AudioConversionError, PostProcessor
+
+from ..utils import (
+    check_executable,
    compat_subprocess_get_DEVNULL,
    encodeFilename,
    PostProcessingError,
+    prepend_extension,
    shell_quote,
    subtitles_filename,
-    prepend_extension,
 )


-class PostProcessor(object):
-    """Post Processor class.
-
-    PostProcessor objects can be added to downloaders with their
-    add_post_processor() method. When the downloader has finished a
-    successful download, it will take its internal chain of PostProcessors
-    and start calling the run() method on each one of them, first with
-    an initial argument and then with the returned value of the previous
-    PostProcessor.
-
-    The chain will be stopped if one of them ever returns None or the end
-    of the chain is reached.
-
-    PostProcessor objects follow a "mutual registration" process similar
-    to InfoExtractor objects.
-    """
-
-    _downloader = None
-
-    def __init__(self, downloader=None):
-        self._downloader = downloader
-
-    def set_downloader(self, downloader):
-        """Sets the downloader for this PP."""
-        self._downloader = downloader
-
-    def run(self, information):
-        """Run the PostProcessor.
-
-        The "information" argument is a dictionary like the ones
-        composed by InfoExtractors. The only difference is that this
-        one has an extra field called "filepath" that points to the
-        downloaded file.
-
-        This method returns a tuple, the first element of which describes
-        whether the original file should be kept (i.e. not deleted - None for
-        no preference), and the second of which is the updated information.
-
-        In addition, this method may raise a PostProcessingError
-        exception if post processing fails.
-        """
-        return None, information # by default, keep file and do nothing

 class FFmpegPostProcessorError(PostProcessingError):
    pass

-class AudioConversionError(PostProcessingError):
-    pass
-
 class FFmpegPostProcessor(PostProcessor):
    def __init__(self,downloader=None):
        PostProcessor.__init__(self, downloader)
@@ -70,14 +28,8 @@ class FFmpegPostProcessor(PostProcessor):

    @staticmethod
    def detect_executables():
-        def executable(exe):
-            try:
-                subprocess.Popen([exe, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
-            except OSError:
-                return False
-            return exe
        programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
-        return dict((program, executable(program)) for program in programs)
+        return dict((program, check_executable(program, ['-version'])) for program in programs)

    def run_ffmpeg_multiple_files(self, input_paths, out_path, opts):
        if not self._exes['ffmpeg'] and not self._exes['avconv']:
@@ -108,6 +60,7 @@ class FFmpegPostProcessor(PostProcessor):
            return u'./' + fn
        return fn

+
 class FFmpegExtractAudioPP(FFmpegPostProcessor):
    def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, nopostoverwrites=False):
        FFmpegPostProcessor.__init__(self, downloader)
@@ -236,6 +189,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
        information['filepath'] = new_path
        return self._nopostoverwrites,information

+
 class FFmpegVideoConvertor(FFmpegPostProcessor):
    def __init__(self, downloader=None,preferedformat=None):
        super(FFmpegVideoConvertor, self).__init__(downloader)
@@ -519,3 +473,4 @@ class FFmpegMergerPP(FFmpegPostProcessor):
        args = ['-c', 'copy']
        self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args)
        return True, info
+
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -0,0 +1,108 @@
+import os
+import subprocess
+import sys
+
+from .common import PostProcessor
+from ..utils import (
+    check_executable,
+    hyphenate_date,
+)
+
+
+class XAttrMetadataPP(PostProcessor):
+
+    #
+    # More info about extended attributes for media:
+    #   http://freedesktop.org/wiki/CommonExtendedAttributes/
+    #   http://www.freedesktop.org/wiki/PhreedomDraft/
+    #   http://dublincore.org/documents/usageguide/elements.shtml
+    #
+    # TODO:
+    #  * capture youtube keywords and put them in 'user.dublincore.subject' (comma-separated)
+    #  * figure out which xattrs can be used for 'duration', 'thumbnail', 'resolution'
+    #
+
+    def run(self, info):
+        """ Set extended attributes on downloaded file (if xattr support is found). """
+
+        # This mess below finds the best xattr tool for the job and creates a
+        # "write_xattr" function.
+        try:
+            # try the pyxattr module...
+            import xattr
+
+            def write_xattr(path, key, value):
+                return xattr.setxattr(path, key, value)
+
+        except ImportError:
+            if os.name == 'nt':
+                # Write xattrs to NTFS Alternate Data Streams:
+                # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
+                def write_xattr(path, key, value):
+                    assert ':' not in key
+                    assert os.path.exists(path)
+
+                    ads_fn = path + ":" + key
+                    with open(ads_fn, "wb") as f:
+                        f.write(value)
+            else:
+                user_has_setfattr = check_executable("setfattr", ['--version'])
+                user_has_xattr = check_executable("xattr", ['-h'])
+
+                if user_has_setfattr or user_has_xattr:
+
+                    def write_xattr(path, key, value):
+                        if user_has_setfattr:
+                            cmd = ['setfattr', '-n', key, '-v', value, path]
+                        elif user_has_xattr:
+                            cmd = ['xattr', '-w', key, value, path]
+
+                        subprocess.check_output(cmd)
+
+                else:
+                    # On Unix, and can't find pyxattr, setfattr, or xattr.
+                    if sys.platform.startswith('linux'):
+                        self._downloader.report_error(
+                            "Couldn't find a tool to set the xattrs. "
+                            "Install either the python 'pyxattr' or 'xattr' "
+                            "modules, or the GNU 'attr' package "
+                            "(which contains the 'setfattr' tool).")
+                    else:
+                        self._downloader.report_error(
+                            "Couldn't find a tool to set the xattrs. "
+                            "Install either the python 'xattr' module, "
+                            "or the 'xattr' binary.")
+
+        # Write the metadata to the file's xattrs
+        self._downloader.to_screen('[metadata] Writing metadata to file\'s xattrs')
+
+        filename = info['filepath']
+
+        try:
+            xattr_mapping = {
+                'user.xdg.referrer.url': 'webpage_url',
+                # 'user.xdg.comment':            'description',
+                'user.dublincore.title': 'title',
+                'user.dublincore.date': 'upload_date',
+                'user.dublincore.description': 'description',
+                'user.dublincore.contributor': 'uploader',
+                'user.dublincore.format': 'format',
+            }
+
+            for xattrname, infoname in xattr_mapping.items():
+
+                value = info.get(infoname)
+
+                if value:
+                    if infoname == "upload_date":
+                        value = hyphenate_date(value)
+
+                    byte_value = value.encode('utf-8')
+                    write_xattr(filename, xattrname, byte_value)
+
+            return True, info
+
+        except (subprocess.CalledProcessError, OSError):
+            self._downloader.report_error("This filesystem doesn't support extended attributes. (You may have to enable them in your /etc/fstab)")
+            return False, info
+
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -764,6 +764,7 @@ def unified_strdate(date_str):
        '%Y-%m-%d',
        '%d/%m/%Y',
        '%Y/%m/%d %H:%M:%S',
+        '%Y-%m-%d %H:%M:%S',
        '%d.%m.%Y %H:%M',
        '%Y-%m-%dT%H:%M:%SZ',
        '%Y-%m-%dT%H:%M:%S.%fZ',
@@ -817,6 +818,15 @@ def date_from_str(date_str):
        return today + delta
    return datetime.datetime.strptime(date_str, "%Y%m%d").date()
    
+def hyphenate_date(date_str):
+    """
+    Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
+    match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
+    if match is not None:
+        return '-'.join(match.groups())
+    else:
+        return date_str
+
 class DateRange(object):
    """Represents a time interval between two dates"""
    def __init__(self, start=None, end=None):
@@ -1026,9 +1036,9 @@ def smuggle_url(url, data):
    return url + u'#' + sdata


-def unsmuggle_url(smug_url):
+def unsmuggle_url(smug_url, default=None):
    if not '#__youtubedl_smuggle' in smug_url:
-        return smug_url, None
+        return smug_url, default
    url, _, sdata = smug_url.rpartition(u'#')
    jsond = compat_parse_qs(sdata)[u'__youtubedl_smuggle'][0]
    data = json.loads(jsond)
@@ -1141,3 +1151,13 @@ def parse_duration(s):
 def prepend_extension(filename, ext):
    name, real_ext = os.path.splitext(filename) 
    return u'{0}.{1}{2}'.format(name, ext, real_ext)
+
+
+def check_executable(exe, args=[]):
+    """ Checks if the given binary is installed somewhere in PATH, and returns its name.
+    args can be a list of arguments for a short output (like -version) """
+    try:
+        subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+    except OSError:
+        return False
+    return exe
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@

-__version__ = '2014.01.05.5'
+__version__ = '2014.01.07.4'
Author	SHA1	Message	Date
Philipp Hagemeister	2537186d43	release 2014.01.07.4	2014-01-07 09:52:29 +01:00
Philipp Hagemeister	0eecc6a417	[vimeo] Add support for passwords for player. URLs Fixes #2053	2014-01-07 09:52:00 +01:00
Philipp Hagemeister	0dc13f4c4a	Correctly set IE_NAME field	2014-01-07 09:45:58 +01:00
Philipp Hagemeister	f577e0ce15	switch more to unicode_literals	2014-01-07 09:45:40 +01:00
Philipp Hagemeister	bd1b906527	Remove unusued import	2014-01-07 09:42:38 +01:00
Philipp Hagemeister	ecfef3e5bf	+unicode_literals	2014-01-07 09:41:13 +01:00
Philipp Hagemeister	3d3538e422	[khanacademy] Add support (Fixes #2066 )	2014-01-07 09:35:34 +01:00
Philipp Hagemeister	0cdad20c75	release 2014.01.07.3	2014-01-07 08:28:13 +01:00
Philipp Hagemeister	50144133c5	[release] Check for useless files before release	2014-01-07 08:28:05 +01:00
Philipp Hagemeister	089cb705e8	release 2014.01.07.2	2014-01-07 08:21:05 +01:00
Philipp Hagemeister	525e1076ad	release 2014.01.07.1	2014-01-07 08:09:08 +01:00
Philipp Hagemeister	282962bd36	--list-formats: Only add "@" if vbr is given	2014-01-07 08:08:48 +01:00
Philipp Hagemeister	c93c2ab1c3	[mpora] Add support (Fixes #2096 )	2014-01-07 08:07:46 +01:00
Philipp Hagemeister	7b09a4d847	[lynda] Fix download if subtitles were not requested	2014-01-07 07:17:49 +01:00
Philipp Hagemeister	73a25b30ea	[lynda] Remove superfluous space	2014-01-07 07:14:46 +01:00
Philipp Hagemeister	ac260dd81e	[lynda] Remove useless u""	2014-01-07 07:14:12 +01:00
Philipp Hagemeister	48a2034671	[vimeo] Fix playlist URL matching	2014-01-07 07:13:47 +01:00
Philipp Hagemeister	a9ce0c631e	[xattr] Correct on Windows	2014-01-07 06:50:24 +01:00
Philipp Hagemeister	afc7bc33cb	[xattr] Always use UTF-8 On Windows and other systems, other encodings would break when trying to encode non-ASCII characters. Simply use UTF-8, like every sane system.	2014-01-07 06:49:15 +01:00
Philipp Hagemeister	168da92b9a	[xattr] Rework In particular, explicitly require NT before trying ADS, and do not try to parse process output that may be localized.	2014-01-07 06:36:34 +01:00
Philipp Hagemeister	d70ad093af	Move check_executable into a helper ufnction	2014-01-07 06:23:41 +01:00
Philipp Hagemeister	2a2e2770cc	[xattr] Always output a warning message on errors	2014-01-07 06:12:28 +01:00
Philipp Hagemeister	42cc71e80b	[xattr] Write bytestrings, not characters	2014-01-07 06:11:21 +01:00
Philipp Hagemeister	496c19234c	Split postprocessor package into multiple modules	2014-01-07 05:59:22 +01:00
Philipp Hagemeister	4f81667d76	[orf] Remove unused variable name	2014-01-07 05:51:46 +01:00
Philipp Hagemeister	56327689a2	Move postprocessor into its own package	2014-01-07 05:49:17 +01:00
Philipp Hagemeister	ad84831537	[xattr] Coding style	2014-01-07 05:45:15 +01:00
Philipp Hagemeister	5f263296ea	Merge remote-tracking branch 'epitron/metadata-pp' Conflicts: youtube_dl/PostProcessor.py	2014-01-07 05:44:44 +01:00
Philipp Hagemeister	89650ea3a6	release 2014.01.07	2014-01-07 05:34:32 +01:00
Philipp Hagemeister	79f8295303	Use original Referer URL in Brightcove requests (Fixes #2110 )	2014-01-07 05:34:14 +01:00
Philipp Hagemeister	400e58103d	[brightcove] Use unicode_literals	2014-01-07 05:23:20 +01:00
Philipp Hagemeister	fcee8ee784	[vimeo] Use _search_regex	2014-01-07 05:19:28 +01:00
Philipp Hagemeister	9148eb002b	[vimeo] Use unicode_literals	2014-01-06 23:38:16 +01:00
Philipp Hagemeister	559e370f44	[vimeo] Proper warning when password is required (Fixes #2053 ) In player. URLs, the password warning is different.	2014-01-06 23:35:27 +01:00
Philipp Hagemeister	cdeb10b5cd	release 2014.01.06.1	2014-01-06 19:25:43 +01:00
Philipp Hagemeister	e6162a90e6	release 2014.01.06	2014-01-06 17:37:24 +01:00
Philipp Hagemeister	9a6422a81e	Merge remote-tracking branch 'origin/master'	2014-01-06 17:37:20 +01:00
Philipp Hagemeister	fcea44c6d5	[vimeo] Add support for review pages Since the regexp is already overboarding and review pages have a distinct URL format (with non-trivial stuff after the ID), use a dedicated IE. Fixes #2106	2014-01-06 17:34:23 +01:00
Philipp Hagemeister	5d73273f6f	[orf] Use new extraction method (Fixes #2057 )	2014-01-06 17:15:27 +01:00
Philipp Hagemeister	c11a0611d9	[veehd] Send requests twice (Fixes #2102 )	2014-01-06 12:54:01 +01:00
Philipp Hagemeister	796495886e	[generic] Use unicode_literals instead of duplicating the u'	2014-01-06 01:47:52 +01:00
Philipp Hagemeister	fa27f667c8	Merge pull request #2104 from dstftw/lynda [lynda] Add subtitles extraction	2014-01-05 16:44:21 -08:00
Philipp Hagemeister	fc9713a1d2	[youtube] Support jwplayer with YouTube URLs (Closes #2075 )	2014-01-06 01:42:58 +01:00
dst	62bcfa8c57	[lynda] Add subtitles extraction	2014-01-05 23:59:33 +07:00
Philipp Hagemeister	7f9886379c	release 2014.01.05.6	2014-01-05 11:44:20 +01:00
Philipp Hagemeister	c6e4b225b1	Restore binary files for backwards compatibility Fixes `9656ee5d1d` New year's resolution: Check which systems of Ubuntu / RHEL still serve the ancient versions. If it's only RHEL, consider removing these binary files in 2015 or so.	2014-01-05 11:41:44 +01:00
Jaime Marquínez Ferrándiz	1c0f31f9f7	[bash-completion] Complete filename if `—load-info` is given	2014-01-05 11:28:01 +01:00
Jaime Marquínez Ferrándiz	41292a3827	Fix list comprehension for decoding the URLs (fixes #2100 ) It wasn’t a comprehension, it was just using the last url from the previous comprehension. That didn’t raise an error in python 2, but in python 3 the variable was not defined.	2014-01-05 10:58:36 +01:00
epitron	e63fc1bed4	Added '--xattrs' option which writes metadata to the file's extended attributes using a youtube-dl postprocessor. Works on Linux, OSX, and Windows.	2014-01-02 07:47:28 -05:00