[extractor/cspan] Support of C-Span congress videos (#2295)

Authored by: Grabien

yt_dlp/extractor/bandcamp.py

@@ -212,7 +212,7 @@ class BandcampIE(InfoExtractor):
 
 class BandcampAlbumIE(BandcampIE):
     IE_NAME = 'Bandcamp:album'
-    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?!/music)(?:/album/(?P<id>[^/?#&]+))?'
+    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com/album/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
@@ -257,14 +257,6 @@ class BandcampAlbumIE(BandcampIE):
             'id': 'hierophany-of-the-open-grave',
         },
         'playlist_mincount': 9,
-    }, {
-        'url': 'http://dotscale.bandcamp.com',
-        'info_dict': {
-            'title': 'Loom',
-            'id': 'dotscale',
-            'uploader_id': 'dotscale',
-        },
-        'playlist_mincount': 7,
     }, {
         # with escaped quote in title
         'url': 'https://jstrecords.bandcamp.com/album/entropy-ep',
@@ -391,41 +383,63 @@ class BandcampWeeklyIE(BandcampIE):
         }
 
 
-class BandcampMusicIE(InfoExtractor):
-    _VALID_URL = r'https?://(?P<id>[^/]+)\.bandcamp\.com/music'
+class BandcampUserIE(InfoExtractor):
+    IE_NAME = 'Bandcamp:user'
+    _VALID_URL = r'https?://(?!www\.)(?P<id>[^.]+)\.bandcamp\.com(?:/music)?/?(?:[#?]|$)'
+
     _TESTS = [{
+        # Type 1 Bandcamp user page.
+        'url': 'https://adrianvonziegler.bandcamp.com',
+        'info_dict': {
+            'id': 'adrianvonziegler',
+            'title': 'Discography of adrianvonziegler',
+        },
+        'playlist_mincount': 23,
+    }, {
+        # Bandcamp user page with only one album
+        'url': 'http://dotscale.bandcamp.com',
+        'info_dict': {
+            'id': 'dotscale',
+            'title': 'Discography of dotscale'
+        },
+        'playlist_count': 1,
+    }, {
+        # Type 2 Bandcamp user page.
+        'url': 'https://nightcallofficial.bandcamp.com',
+        'info_dict': {
+            'id': 'nightcallofficial',
+            'title': 'Discography of nightcallofficial',
+        },
+        'playlist_count': 4,
+    }, {
         'url': 'https://steviasphere.bandcamp.com/music',
         'playlist_mincount': 47,
         'info_dict': {
             'id': 'steviasphere',
+            'title': 'Discography of steviasphere',
         },
     }, {
         'url': 'https://coldworldofficial.bandcamp.com/music',
         'playlist_mincount': 10,
         'info_dict': {
             'id': 'coldworldofficial',
+            'title': 'Discography of coldworldofficial',
         },
     }, {
         'url': 'https://nuclearwarnowproductions.bandcamp.com/music',
         'playlist_mincount': 399,
         'info_dict': {
             'id': 'nuclearwarnowproductions',
+            'title': 'Discography of nuclearwarnowproductions',
         },
-    }
-    ]
-
-    _TYPE_IE_DICT = {
-        'album': BandcampAlbumIE.ie_key(),
-        'track': BandcampIE.ie_key()
-    }
+    }]
 
     def _real_extract(self, url):
-        id = self._match_id(url)
-        webpage = self._download_webpage(url, id)
-        items = re.findall(r'href\=\"\/(?P<path>(?P<type>album|track)+/[^\"]+)', webpage)
-        entries = [
-            self.url_result(
-                f'https://{id}.bandcamp.com/{item[0]}',
-                ie=self._TYPE_IE_DICT[item[1]])
-            for item in items]
-        return self.playlist_result(entries, id)
+        uploader = self._match_id(url)
+        webpage = self._download_webpage(url, uploader)
+
+        discography_data = (re.findall(r'<li data-item-id=["\'][^>]+>\s*<a href=["\']([^"\']+)', webpage)
+                            or re.findall(r'<div[^>]+trackTitle["\'][^"\']+["\']([^"\']+)', webpage))
+
+        return self.playlist_from_matches(
+            discography_data, uploader, f'Discography of {uploader}', getter=lambda x: urljoin(url, x))

yt_dlp/extractor/cspan.py

@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_HTMLParseError
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -11,9 +12,11 @@ from ..utils import (
     get_element_by_attribute,
     get_element_by_class,
     int_or_none,
+    join_nonempty,
     js_to_json,
     merge_dicts,
     parse_iso8601,
+    parse_qs,
     smuggle_url,
     str_to_int,
     unescapeHTML,
@@ -126,8 +129,12 @@ class CSpanIE(InfoExtractor):
                         ext = 'vtt'
                     subtitle['ext'] = ext
             ld_info = self._search_json_ld(webpage, video_id, default={})
-            title = get_element_by_class('video-page-title', webpage) or \
-                self._og_search_title(webpage)
+            try:
+                title = get_element_by_class('video-page-title', webpage)
+            except compat_HTMLParseError:
+                title = None
+            if title is None:
+                title = self._og_search_title(webpage)
             description = get_element_by_attribute('itemprop', 'description', webpage) or \
                 self._html_search_meta(['og:description', 'description'], webpage)
             return merge_dicts(info, ld_info, {
@@ -242,3 +249,42 @@ class CSpanIE(InfoExtractor):
                 'title': title,
                 'id': 'c' + video_id if video_type == 'clip' else video_id,
             }
+
+
+class CSpanCongressIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?c-span\.org/congress/'
+    _TESTS = [{
+        'url': 'https://www.c-span.org/congress/?chamber=house&date=2017-12-13&t=1513208380',
+        'info_dict': {
+            'id': 'house_2017-12-13',
+            'title': 'Congressional Chronicle - Members of Congress, Hearings and More',
+            'description': 'md5:54c264b7a8f219937987610243305a84',
+            'thumbnail': r're:https://ximage.c-spanvideo.org/.+',
+            'ext': 'mp4'
+        }
+    }]
+
+    def _real_extract(self, url):
+        query = parse_qs(url)
+        video_date = query.get('date', [None])[0]
+        video_id = join_nonempty(query.get('chamber', ['senate'])[0], video_date, delim='_')
+        webpage = self._download_webpage(url, video_id)
+        if not video_date:
+            jwp_date = re.search(r'jwsetup.clipprogdate = \'(?P<date>\d{4}-\d{2}-\d{2})\';', webpage)
+            if jwp_date:
+                video_id = f'{video_id}_{jwp_date.group("date")}'
+        jwplayer_data = self._parse_json(
+            self._search_regex(r'jwsetup\s*=\s*({(?:.|\n)[^;]+});', webpage, 'player config'),
+            video_id, transform_source=js_to_json)
+
+        title = (self._og_search_title(webpage, default=None)
+                 or self._html_search_regex(r'(?s)<title>(.*?)</title>', webpage, 'video title'))
+        description = (self._og_search_description(webpage, default=None)
+                       or self._html_search_meta('description', webpage, 'description', default=None))
+
+        return {
+            **self._parse_jwplayer_data(jwplayer_data, video_id, False),
+            'title': re.sub(r'\s+', ' ', title.split('|')[0]).strip(),
+            'description': description,
+            'http_headers': {'Referer': 'https://www.c-span.org/'},
+        }

yt_dlp/extractor/extractors.py

@@ -118,7 +118,7 @@ from .bandcamp import (
     BandcampIE,
     BandcampAlbumIE,
     BandcampWeeklyIE,
-    BandcampMusicIE,
+    BandcampUserIE,
 )
 from .bannedvideo import BannedVideoIE
 from .bbc import (
@@ -316,7 +316,7 @@ from .crunchyroll import (
     CrunchyrollBetaIE,
     CrunchyrollBetaShowIE,
 )
-from .cspan import CSpanIE
+from .cspan import CSpanIE, CSpanCongressIE
 from .ctsnews import CtsNewsIE
 from .ctv import CTVIE
 from .ctvnews import CTVNewsIE