From 061a75edd6bab2f30978b458fe7402ff9e9c02a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 05:01:01 +0600
Subject: [PATCH] [youtube] Extract base for entry list extractors and support
multi page lists of playlists
---
youtube_dl/extractor/youtube.py | 28 +++++++++++++++++-----------
1 file changed, 17 insertions(+), 11 deletions(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c56f8a0a2..8352ad1da 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -178,15 +178,13 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
return
-class YoutubePlaylistBaseInfoExtractor(InfoExtractor):
- # Extract the video ids from the playlist pages
+class YoutubeEntryListBaseInfoExtractor(InfoExtractor):
+ # Extract entries from page with "Load more" button
def _entries(self, page, playlist_id):
more_widget_html = content_html = page
for page_num in itertools.count(1):
- for video_id, video_title in self.extract_videos_from_page(content_html):
- yield self.url_result(
- video_id, 'Youtube', video_id=video_id,
- video_title=video_title)
+ for entry in self._process_page(content_html):
+ yield entry
mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
if not mobj:
@@ -203,6 +201,12 @@ class YoutubePlaylistBaseInfoExtractor(InfoExtractor):
break
more_widget_html = more['load_more_widget_html']
+
+class YoutubePlaylistBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
+ def _process_page(self, content):
+ for video_id, video_title in self.extract_videos_from_page(content):
+ yield self.url_result(video_id, 'Youtube', video_id, video_title)
+
def extract_videos_from_page(self, page):
ids_in_page = []
titles_in_page = []
@@ -224,15 +228,17 @@ class YoutubePlaylistBaseInfoExtractor(InfoExtractor):
return zip(ids_in_page, titles_in_page)
-class YoutubePlaylistsBaseInfoExtractor(InfoExtractor):
+class YoutubePlaylistsBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
+ def _process_page(self, content):
+ for playlist_id in re.findall(r'href="/?playlist\?list=(.+?)"', content):
+ yield self.url_result(
+ 'https://www.youtube.com/playlist?list=%s' % playlist_id, 'YoutubePlaylist')
+
def _real_extract(self, url):
playlist_id = self._match_id(url)
webpage = self._download_webpage(url, playlist_id)
- entries = [
- self.url_result(compat_urlparse.urljoin(url, playlist), 'YoutubePlaylist')
- for playlist in re.findall(r'href="(/playlist\?list=.+?)"', webpage)]
title = self._og_search_title(webpage, fatal=False)
- return self.playlist_result(entries, playlist_id, title)
+ return self.playlist_result(self._entries(webpage, playlist_id), playlist_id, title)
class YoutubeIE(YoutubeBaseInfoExtractor):