[extractor/naver] Add navernow extractor (#3866)

Authored by: ping

yt_dlp/extractor/extractors.py

@@ -548,6 +548,10 @@ from .frontendmasters import (
     FrontendMastersLessonIE,
     FrontendMastersCourseIE
 )
+from .freetv import (
+    FreeTvIE,
+    FreeTvMoviesIE,
+)
 from .fujitv import FujiTVFODPlus7IE
 from .funimation import (
     FunimationIE,
@@ -993,6 +997,7 @@ from .nationalgeographic import (
 from .naver import (
     NaverIE,
     NaverLiveIE,
+    NaverNowIE,
 )
 from .nba import (
     NBAWatchEmbedIE,

yt_dlp/extractor/freetv.py

@@ -0,0 +1,145 @@
+import itertools
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    traverse_obj,
+    urlencode_postdata,
+)
+
+
+class FreeTvBaseIE(InfoExtractor):
+    def _get_api_response(self, content_id, resource_type, postdata):
+        return self._download_json(
+            'https://www.freetv.com/wordpress/wp-admin/admin-ajax.php',
+            content_id, data=urlencode_postdata(postdata),
+            note=f'Downloading {content_id} {resource_type} JSON')['data']
+
+
+class FreeTvMoviesIE(FreeTvBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?freetv\.com/peliculas/(?P<id>[^/]+)'
+    _TESTS = [{
+        'url': 'https://www.freetv.com/peliculas/atrapame-si-puedes/',
+        'md5': 'dc62d5abf0514726640077cd1591aa92',
+        'info_dict': {
+            'id': '428021',
+            'title': 'Atrápame Si Puedes',
+            'description': 'md5:ca63bc00898aeb2f64ec87c6d3a5b982',
+            'ext': 'mp4',
+        }
+    }, {
+        'url': 'https://www.freetv.com/peliculas/monstruoso/',
+        'md5': '509c15c68de41cb708d1f92d071f20aa',
+        'info_dict': {
+            'id': '377652',
+            'title': 'Monstruoso',
+            'description': 'md5:333fc19ee327b457b980e54a911ea4a3',
+            'ext': 'mp4',
+        }
+    }]
+
+    def _extract_video(self, content_id, action='olyott_video_play'):
+        api_response = self._get_api_response(content_id, 'video', {
+            'action': action,
+            'contentID': content_id,
+        })
+
+        video_id, video_url = api_response['displayMeta']['contentID'], api_response['displayMeta']['streamURLVideo']
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(video_url, video_id, 'mp4')
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': traverse_obj(api_response, ('displayMeta', 'title')),
+            'description': traverse_obj(api_response, ('displayMeta', 'desc')),
+            'formats': formats,
+            'subtitles': subtitles,
+        }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        return self._extract_video(
+            self._search_regex((
+                r'class=["\'][^>]+postid-(?P<video_id>\d+)',
+                r'<link[^>]+freetv.com/\?p=(?P<video_id>\d+)',
+                r'<div[^>]+data-params=["\'][^>]+post_id=(?P<video_id>\d+)',
+            ), webpage, 'video id', group='video_id'))
+
+
+class FreeTvIE(FreeTvBaseIE):
+    IE_NAME = 'freetv:series'
+    _VALID_URL = r'https?://(?:www\.)?freetv\.com/series/(?P<id>[^/]+)'
+    _TESTS = [{
+        'url': 'https://www.freetv.com/series/el-detective-l/',
+        'info_dict': {
+            'id': 'el-detective-l',
+            'title': 'El Detective L',
+            'description': 'md5:f9f1143bc33e9856ecbfcbfb97a759be'
+        },
+        'playlist_count': 24,
+    }, {
+        'url': 'https://www.freetv.com/series/esmeraldas/',
+        'info_dict': {
+            'id': 'esmeraldas',
+            'title': 'Esmeraldas',
+            'description': 'md5:43d7ec45bd931d8268a4f5afaf4c77bf'
+        },
+        'playlist_count': 62,
+    }, {
+        'url': 'https://www.freetv.com/series/las-aventuras-de-leonardo/',
+        'info_dict': {
+            'id': 'las-aventuras-de-leonardo',
+            'title': 'Las Aventuras de Leonardo',
+            'description': 'md5:0c47130846c141120a382aca059288f6'
+        },
+        'playlist_count': 13,
+    },
+    ]
+
+    def _extract_series_season(self, season_id, series_title):
+        episodes = self._get_api_response(season_id, 'series', {
+            'contentID': season_id,
+            'action': 'olyott_get_dynamic_series_content',
+            'type': 'list',
+            'perPage': '1000',
+        })['1']
+
+        for episode in episodes:
+            video_id = str(episode['contentID'])
+            formats, subtitles = self._extract_m3u8_formats_and_subtitles(episode['streamURL'], video_id, 'mp4')
+            self._sort_formats(formats)
+
+            yield {
+                'id': video_id,
+                'title': episode.get('fullTitle'),
+                'description': episode.get('description'),
+                'formats': formats,
+                'subtitles': subtitles,
+                'thumbnail': episode.get('thumbnail'),
+                'series': series_title,
+                'series_id': traverse_obj(episode, ('contentMeta', 'displayMeta', 'seriesID')),
+                'season_id': traverse_obj(episode, ('contentMeta', 'displayMeta', 'seasonID')),
+                'season_number': traverse_obj(
+                    episode, ('contentMeta', 'displayMeta', 'seasonNum'), expected_type=int_or_none),
+                'episode_number': traverse_obj(
+                    episode, ('contentMeta', 'displayMeta', 'episodeNum'), expected_type=int_or_none),
+            }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        title = self._html_search_regex(
+            r'<h1[^>]+class=["\']synopis[^>]>(?P<title>[^<]+)', webpage, 'title', group='title', fatal=False)
+        description = self._html_search_regex(
+            r'<div[^>]+class=["\']+synopis content[^>]><p>(?P<description>[^<]+)',
+            webpage, 'description', group='description', fatal=False)
+
+        return self.playlist_result(
+            itertools.chain.from_iterable(
+                self._extract_series_season(season_id, title)
+                for season_id in re.findall(r'<option[^>]+value=["\'](\d+)["\']', webpage)),
+            display_id, title, description)

yt_dlp/extractor/naver.py

@@ -1,13 +1,19 @@
+import itertools
 import re
+from urllib.parse import urlparse, parse_qs
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     clean_html,
     dict_get,
-    ExtractorError,
     int_or_none,
+    merge_dicts,
     parse_duration,
+    traverse_obj,
+    try_call,
     try_get,
+    unified_timestamp,
     update_url_query,
 )
 
@@ -247,3 +253,134 @@ class NaverLiveIE(InfoExtractor):
             'categories': [meta.get('categoryId')],
             'is_live': True
         }
+
+
+class NaverNowIE(NaverBaseIE):
+    IE_NAME = 'navernow'
+    _VALID_URL = r'https?://now\.naver\.com/show/(?P<id>[0-9]+)'
+    _PAGE_SIZE = 30
+    _API_URL = 'https://apis.naver.com/now_web/nowcms-api-xhmac/cms/v1'
+    _TESTS = [{
+        'url': 'https://now.naver.com/show/4759?shareReplayId=5901#replay=',
+        'md5': 'e05854162c21c221481de16b2944a0bc',
+        'info_dict': {
+            'id': '4759-5901',
+            'title': '아이키X노제\r\n💖꽁냥꽁냥💖(1)',
+            'ext': 'mp4',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'timestamp': 1650369600,
+            'upload_date': '20220419',
+            'uploader_id': 'now',
+            'view_count': int,
+        },
+        'params': {
+            'noplaylist': True,
+        }
+    }, {
+        'url': 'https://now.naver.com/show/4759?shareHightlight=1078#highlight=',
+        'md5': '9f6118e398aa0f22b2152f554ea7851b',
+        'info_dict': {
+            'id': '4759-1078',
+            'title': '아이키: 나 리정한테 흔들렸어,,, 질투 폭발하는 노제 여보😾 [아이키의 떰즈업]ㅣ네이버 NOW.',
+            'ext': 'mp4',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'upload_date': '20220504',
+            'timestamp': 1651648042,
+            'uploader_id': 'now',
+            'view_count': int,
+        },
+        'params': {
+            'noplaylist': True,
+        },
+    }, {
+        'url': 'https://now.naver.com/show/4759',
+        'info_dict': {
+            'id': '4759',
+            'title': '아이키의 떰즈업',
+        },
+        'playlist_mincount': 48
+    }, {
+        'url': 'https://now.naver.com/show/4759?shareReplayId=5901#replay',
+        'info_dict': {
+            'id': '4759',
+            'title': '아이키의 떰즈업',
+        },
+        'playlist_mincount': 48,
+    }, {
+        'url': 'https://now.naver.com/show/4759?shareHightlight=1078#highlight=',
+        'info_dict': {
+            'id': '4759',
+            'title': '아이키의 떰즈업',
+        },
+        'playlist_mincount': 48,
+    }]
+
+    def _extract_replay(self, show_id, replay_id):
+        vod_info = self._download_json(f'{self._API_URL}/shows/{show_id}/vod/{replay_id}', replay_id)
+        in_key = self._download_json(f'{self._API_URL}/shows/{show_id}/vod/{replay_id}/inkey', replay_id)['inKey']
+        return merge_dicts({
+            'id': f'{show_id}-{replay_id}',
+            'title': traverse_obj(vod_info, ('episode', 'title')),
+            'timestamp': unified_timestamp(traverse_obj(vod_info, ('episode', 'start_time'))),
+            'thumbnail': vod_info.get('thumbnail_image_url'),
+        }, self._extract_video_info(replay_id, vod_info['video_id'], in_key))
+
+    def _extract_show_replays(self, show_id):
+        page = 0
+        while True:
+            show_vod_info = self._download_json(
+                f'{self._API_URL}/vod-shows/{show_id}', show_id,
+                query={'offset': page * self._PAGE_SIZE, 'limit': self._PAGE_SIZE},
+                note=f'Downloading JSON vod list for show {show_id} - page {page}'
+            )['response']['result']
+            for v in show_vod_info.get('vod_list') or []:
+                yield self._extract_replay(show_id, v['id'])
+
+            if try_call(lambda: show_vod_info['count'] <= self._PAGE_SIZE * (page + 1)):
+                break
+            page += 1
+
+    def _extract_show_highlights(self, show_id, highlight_id=None):
+        page = 0
+        while True:
+            highlights_videos = self._download_json(
+                f'{self._API_URL}/shows/{show_id}/highlights/videos/', show_id,
+                query={'offset': page * self._PAGE_SIZE, 'limit': self._PAGE_SIZE},
+                note=f'Downloading JSON highlights for show {show_id} - page {page}')
+
+            for highlight in highlights_videos.get('results') or []:
+                if highlight_id and highlight.get('id') != int(highlight_id):
+                    continue
+                yield merge_dicts({
+                    'id': f'{show_id}-{highlight["id"]}',
+                    'title': highlight.get('title'),
+                    'timestamp': unified_timestamp(highlight.get('regdate')),
+                    'thumbnail': highlight.get('thumbnail_url'),
+                }, self._extract_video_info(highlight['id'], highlight['video_id'], highlight['video_inkey']))
+
+            if try_call(lambda: highlights_videos['count'] <= self._PAGE_SIZE * (page + 1)):
+                break
+            page += 1
+
+    def _extract_highlight(self, show_id, highlight_id):
+        try:
+            return next(self._extract_show_highlights(show_id, highlight_id))
+        except StopIteration:
+            raise ExtractorError(f'Unable to find highlight {highlight_id} for show {show_id}')
+
+    def _real_extract(self, url):
+        show_id = self._match_id(url)
+        qs = parse_qs(urlparse(url).query)
+
+        if not self._yes_playlist(show_id, qs.get('shareHightlight')):
+            return self._extract_highlight(show_id, qs['shareHightlight'][0])
+        elif not self._yes_playlist(show_id, qs.get('shareReplayId')):
+            return self._extract_replay(show_id, qs['shareReplayId'][0])
+
+        show_info = self._download_json(
+            f'{self._API_URL}/shows/{show_id}', show_id,
+            note=f'Downloading JSON vod list for show {show_id}')
+
+        return self.playlist_result(
+            itertools.chain(self._extract_show_replays(show_id), self._extract_show_highlights(show_id)),
+            show_id, show_info.get('title'))

yt_dlp/extractor/zdf.py

@@ -69,6 +69,7 @@ class ZDFBaseIE(InfoExtractor):
             f.update({
                 'url': format_url,
                 'format_id': join_nonempty('http', meta.get('type'), meta.get('quality')),
+                'tbr': int_or_none(self._search_regex(r'_(\d+)k_', format_url, default=None))
             })
             new_formats = [f]
         formats.extend(merge_dicts(f, {
@@ -108,7 +109,7 @@ class ZDFBaseIE(InfoExtractor):
                                 'class': track.get('class'),
                                 'language': track.get('language'),
                             })
-        self._sort_formats(formats, ('hasaud', 'res', 'quality', 'language_preference'))
+        self._sort_formats(formats, ('tbr', 'res', 'quality', 'language_preference'))
 
         duration = float_or_none(try_get(
             ptmd, lambda x: x['attributes']['duration']['value']), scale=1000)
@@ -187,7 +188,7 @@ class ZDFIE(ZDFBaseIE):
         },
     }, {
         'url': 'https://www.zdf.de/funk/druck-11790/funk-alles-ist-verzaubert-102.html',
-        'md5': '3d6f1049e9682178a11c54b91f3dd065',
+        'md5': '57af4423db0455a3975d2dc4578536bc',
         'info_dict': {
             'ext': 'mp4',
             'id': 'video_funk_1770473',
@@ -230,6 +231,19 @@ class ZDFIE(ZDFBaseIE):
             'timestamp': 1641355200,
             'upload_date': '20220105',
         },
+        'skip': 'No longer available "Diese Seite wurde leider nicht gefunden"'
+    }, {
+        'url': 'https://www.zdf.de/serien/soko-stuttgart/das-geld-anderer-leute-100.html',
+        'info_dict': {
+            'id': '191205_1800_sendung_sok8',
+            'ext': 'mp4',
+            'title': 'Das Geld anderer Leute',
+            'description': 'md5:cb6f660850dc5eb7d1ab776ea094959d',
+            'duration': 2581.0,
+            'timestamp': 1654790700,
+            'upload_date': '20220609',
+            'thumbnail': 'https://epg-image.zdf.de/fotobase-webdelivery/images/e2d7e55a-09f0-424e-ac73-6cac4dd65f35?layout=2400x1350',
+        },
     }]
 
     def _extract_entry(self, url, player, content, video_id):