From cc0780751b3f2f7cd1a21ba7cd6e1f93f02c1a7a Mon Sep 17 00:00:00 2001 From: kclauhk <78251477+kclauhk@users.noreply.github.com> Date: Tue, 20 Aug 2024 20:47:31 +0800 Subject: [PATCH 1/7] [ie/kidoodletv] Add extractor --- yt_dlp/extractor/_extractors.py | 4 + yt_dlp/extractor/kidoodletv.py | 187 ++++++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 yt_dlp/extractor/kidoodletv.py diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py index 9b73fcd75e..85ea19021c 100644 --- a/yt_dlp/extractor/_extractors.py +++ b/yt_dlp/extractor/_extractors.py @@ -945,6 +945,10 @@ from .kick import ( ) from .kicker import KickerIE from .kickstarter import KickStarterIE +from .kidoodletv import ( + KidoodleTVIE, + KidoodleTVSeriesIE, +) from .kinja import KinjaEmbedIE from .kinopoisk import KinoPoiskIE from .kommunetv import KommunetvIE diff --git a/yt_dlp/extractor/kidoodletv.py b/yt_dlp/extractor/kidoodletv.py new file mode 100644 index 0000000000..80d7e48a64 --- /dev/null +++ b/yt_dlp/extractor/kidoodletv.py @@ -0,0 +1,187 @@ +import re +import urllib.parse + +from .common import InfoExtractor +from ..utils import ( + determine_ext, + float_or_none, + int_or_none, + join_nonempty, + js_to_json, + merge_dicts, + traverse_obj, + url_or_none, + urlencode_postdata, +) + + +class KidoodleTVBaseIE(InfoExtractor): + def _extract_data(self, webpage, video_id): + keys = self._html_search_regex(r'__NUXT__=\(function\(([^\)]+)\)\{', webpage, 'key', default=None) + key_list = self._parse_json(js_to_json(f'[{keys}]'), video_id, fatal=False) + data = self._html_search_regex(r'\}\}\}\((".+)\)\);', webpage, 'data', default=None) + data_list = self._parse_json(js_to_json(f'[{data}]'), video_id, fatal=False) + data_set = {} + if key_list and data_list and (len(data_list) == len(key_list)): + for idx, key in enumerate(key_list): + data_set[key] = data_list[idx] + return data_set + + def _extract_by_idx(self, idx, webpage, data, display_id=None): + def slugify(string): + s = string.lower().strip() + s = re.sub(r'[0-9]', '', s) + s = re.sub(r'[^\w\s-]', '', s) + s = re.sub(r'[\s_-]+', '-', s) + return re.sub(r'^-+|-+$', '', s) + + def get_field(field_name, idx, webpage, data): + value = self._html_search_regex(rf'{idx}\.{field_name}=(?P"?(?P.+?)"?);', webpage, + field_name, default=None, group=('a', 'b')) + return value[1] if value[1] != value[0] else (data.get(value[0]) or value[0]) + + video_id = get_field('id', idx, webpage, data) + title = get_field('title', idx, webpage, data) + brief = get_field('shortSummary', idx, webpage, data) or '' + summary = get_field('summary', idx, webpage, data) or '' + description = (summary if brief[:-3] in summary else join_nonempty(brief, summary, delim='\n') + ).replace('\\\"', '\"') + series = get_field('seriesName', idx, webpage, data) + season_episode = get_field('seasonAndEpisode', idx, webpage, data) + season, episode = self._search_regex(r'^S(?P\d+)E(?P\d+)', season_episode, + 'season_episode', group=('season', 'episode')) + if release_date := get_field('premiere_date', idx, webpage, data): + release_date = release_date.replace('-', '') + duration = get_field('duration', idx, webpage, data) + thumbnails, formats, subtitles = [], [], {} + if images := self._html_search_regex(rf'{idx}\.images=(\[[^\]]+]);', webpage, + 'images', default=None): + for image_url in traverse_obj(self._parse_json(js_to_json(images), video_id, fatal=False), + (..., 'url', {lambda v: url_or_none(v.replace('\\u002F', '/'))})): + if determine_ext(image_url) != 'mp4': + thumbnails.append({ + 'url': image_url, + 'preference': -1 if '_large' in image_url else -2, + }) + if video_url := get_field('videoUrl', idx, webpage, data): + video_url = video_url.replace('\\u002F', '/') + if determine_ext(video_url) == 'm3u8': + formats, subtitles = self._extract_m3u8_formats_and_subtitles(video_url, video_id) + return { + 'id': video_id, + 'display_id': display_id or f'{season_episode}-{slugify(title)}', + 'title': title, + 'description': description, + 'thumbnails': thumbnails, + 'release_date': release_date, + 'series': series, + 'season_number': int_or_none(season), + 'episode_number': int_or_none(episode), + 'duration': float_or_none(duration), + 'formats': formats, + 'subtitles': subtitles, + } + + +class KidoodleTVIE(KidoodleTVBaseIE): + _VALID_URL = r'https?://kidoodle\.tv/(?P\d+)/[^/]+/(?P(?PS\d+E\d+)[^/\?]*)' + _TESTS = [{ + 'url': 'https://kidoodle.tv/2376/regal-academy/S1E01-a-school-for-fairy-tales', + 'info_dict': { + 'id': '84499', + 'ext': 'mp4', + 'display_id': 'S1E01-a-school-for-fairy-tales', + 'title': 'A School for Fairy Tales', + 'description': 'md5:4083278308ce6dda1660445b5073b851', + 'thumbnail': 'https://imgcdn.kidoodle.tv/RegalAcademy/S01/keyart_e01_large.jpg', + 'release_date': '20160521', + 'series': 'Regal Academy', + 'series_id': '2376', + 'season': 'Season 1', + 'season_number': 1, + 'episode': 'Episode 1', + 'episode_number': 1, + 'duration': 1423.4, + }, + }, { + 'url': 'https://kidoodle.tv/3083/unicorn-academy/S1E04-fun-with-foals', + 'info_dict': { + 'id': '105372', + 'ext': 'mp4', + 'display_id': 'S1E04-fun-with-foals', + 'title': 'Fun with Foals', + 'description': 'The Sapphire team looks after a newborn baby unicorn!', + 'thumbnail': 'https://imgcdn.kidoodle.tv/UnicornAcademy/S01/keyart_e04_large.jpg', + 'release_date': '20231027', + 'series': 'Unicorn Academy', + 'series_id': '3083', + 'season': 'Season 1', + 'season_number': 1, + 'episode': 'Episode 4', + 'episode_number': 4, + 'duration': 746.816, + }, + }] + + def _real_extract(self, url): + video_id, series_id, season_episode = self._match_valid_url(url).group('id', 'series_id', 'season_episode') + qs = urlencode_postdata({'origin': urllib.parse.urlparse(url).path}) + self._download_webpage(f'https://kidoodle.tv/welcome?{qs}', video_id, note='Downloading welcome page') + self._download_webpage(f'https://kidoodle.tv/welcome/verify?{qs}', video_id, note='Performing age verification') + # the above lines download the webpages to change verification status, not really get verified + webpage = self._download_webpage(url, video_id) + + description = self._html_search_meta('description', webpage, 'description', default=None) + data_set = self._extract_data(webpage, video_id) + info = {} + if idx := self._html_search_regex(rf'([\w-]{{2}})\.seasonAndEpisode="{season_episode}";', + webpage, 'data_idx', default=None): + info = self._extract_by_idx(idx, webpage, data_set, video_id) + + return merge_dicts(info, { + 'id': video_id, + 'description': description, + 'series_id': series_id, + }) + + +class KidoodleTVSeriesIE(KidoodleTVBaseIE): + _VALID_URL = r'https?://kidoodle\.tv/(?P\d+)/(?P[\w-]+)[^/]*/?$' + IE_NAME = 'KidoodleTV:series' + _TESTS = [{ + 'url': 'https://kidoodle.tv/3014/bluey-the-videogame-by-abdallah-smash', + 'info_dict': { + 'id': '3014', + 'title': 'Bluey: The Videogame by Abdallah Smash', + 'description': 'Bluey: The Videogame on Nintendo Switch with no-commentary.', + }, + 'playlist_count': 8, + }, { + 'url': 'https://kidoodle.tv/3083/unicorn-academy?category=What%27s%20NEW', + 'info_dict': { + 'id': '3083', + 'title': 'Unicorn Academy', + 'description': 'md5:d3f92c6bd76cc9941e60d827213b79f3', + }, + 'playlist_count': 4, + }] + + def _real_extract(self, url): + series_id, slug = self._match_valid_url(url).group('id', 'slug') + webpage = self._download_webpage(url, series_id) + + title = self._html_search_regex(r']+>(.*?)', webpage, 'title', default=None) + description = self._html_search_regex(r'

Date: Wed, 21 Aug 2024 00:36:29 +0800 Subject: [PATCH 2/7] to bypass age verification --- yt_dlp/extractor/kidoodletv.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/yt_dlp/extractor/kidoodletv.py b/yt_dlp/extractor/kidoodletv.py index 80d7e48a64..37cd6c3501 100644 --- a/yt_dlp/extractor/kidoodletv.py +++ b/yt_dlp/extractor/kidoodletv.py @@ -37,7 +37,7 @@ class KidoodleTVBaseIE(InfoExtractor): def get_field(field_name, idx, webpage, data): value = self._html_search_regex(rf'{idx}\.{field_name}=(?P"?(?P.+?)"?);', webpage, - field_name, default=None, group=('a', 'b')) + field_name, default=None, group=('a', 'b')) return value[1] if value[1] != value[0] else (data.get(value[0]) or value[0]) video_id = get_field('id', idx, webpage, data) @@ -45,7 +45,7 @@ class KidoodleTVBaseIE(InfoExtractor): brief = get_field('shortSummary', idx, webpage, data) or '' summary = get_field('summary', idx, webpage, data) or '' description = (summary if brief[:-3] in summary else join_nonempty(brief, summary, delim='\n') - ).replace('\\\"', '\"') + ).replace(r'\"', '"') series = get_field('seriesName', idx, webpage, data) season_episode = get_field('seasonAndEpisode', idx, webpage, data) season, episode = self._search_regex(r'^S(?P\d+)E(?P\d+)', season_episode, @@ -84,7 +84,7 @@ class KidoodleTVBaseIE(InfoExtractor): class KidoodleTVIE(KidoodleTVBaseIE): - _VALID_URL = r'https?://kidoodle\.tv/(?P\d+)/[^/]+/(?P(?PS\d+E\d+)[^/\?]*)' + _VALID_URL = r'https?://kidoodle\.tv/(?P\d+)/(?P[^/]+)/(?P(?PS\d+E\d+)[^/\?]*)' _TESTS = [{ 'url': 'https://kidoodle.tv/2376/regal-academy/S1E01-a-school-for-fairy-tales', 'info_dict': { @@ -124,12 +124,16 @@ class KidoodleTVIE(KidoodleTVBaseIE): }] def _real_extract(self, url): - video_id, series_id, season_episode = self._match_valid_url(url).group('id', 'series_id', 'season_episode') - qs = urlencode_postdata({'origin': urllib.parse.urlparse(url).path}) - self._download_webpage(f'https://kidoodle.tv/welcome?{qs}', video_id, note='Downloading welcome page') - self._download_webpage(f'https://kidoodle.tv/welcome/verify?{qs}', video_id, note='Performing age verification') - # the above lines download the webpages to change verification status, not really get verified - webpage = self._download_webpage(url, video_id) + video_id, series_id, series, season_episode = self._match_valid_url(url).group( + 'id', 'series_id', 'series', 'season_episode') + webpage = self._download_webpage(f'https://kidoodle.tv/{series_id}/{series}', video_id, + fatal=False, expected_status=(404, 500)) + if 'Server error' in webpage or 'Something went wrong' in webpage: + qs = urlencode_postdata({'origin': urllib.parse.urlparse(url).path}) + self._download_webpage(f'https://kidoodle.tv/welcome?{qs}', video_id, note='Downloading welcome page') + self._download_webpage(f'https://kidoodle.tv/welcome/verify?{qs}', video_id, note='Performing age verification') + # the above lines download the webpages to change verification status, not really get verified + webpage = self._download_webpage(url, video_id) description = self._html_search_meta('description', webpage, 'description', default=None) data_set = self._extract_data(webpage, video_id) From b7f03bc2b245ee0b447b4981b08e376a63f8d68b Mon Sep 17 00:00:00 2001 From: kclauhk <78251477+kclauhk@users.noreply.github.com> Date: Wed, 21 Aug 2024 03:19:15 +0800 Subject: [PATCH 3/7] Pattern correction --- yt_dlp/extractor/kidoodletv.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/yt_dlp/extractor/kidoodletv.py b/yt_dlp/extractor/kidoodletv.py index 37cd6c3501..3bbdb9022c 100644 --- a/yt_dlp/extractor/kidoodletv.py +++ b/yt_dlp/extractor/kidoodletv.py @@ -40,6 +40,7 @@ class KidoodleTVBaseIE(InfoExtractor): field_name, default=None, group=('a', 'b')) return value[1] if value[1] != value[0] else (data.get(value[0]) or value[0]) + idx = idx.replace('$', r'\$') video_id = get_field('id', idx, webpage, data) title = get_field('title', idx, webpage, data) brief = get_field('shortSummary', idx, webpage, data) or '' @@ -138,7 +139,7 @@ class KidoodleTVIE(KidoodleTVBaseIE): description = self._html_search_meta('description', webpage, 'description', default=None) data_set = self._extract_data(webpage, video_id) info = {} - if idx := self._html_search_regex(rf'([\w-]{{2}})\.seasonAndEpisode="{season_episode}";', + if idx := self._html_search_regex(rf'([\w\$]{{2,4}})\.seasonAndEpisode="{season_episode}";', webpage, 'data_idx', default=None): info = self._extract_by_idx(idx, webpage, data_set, video_id) @@ -179,7 +180,7 @@ class KidoodleTVSeriesIE(KidoodleTVBaseIE): 'description', default=None) data_set = self._extract_data(webpage, series_id) entries = [] - for idx_se in sorted(re.findall(r'([\w-]{2})\.seasonAndEpisode="([^"]+)";', webpage), + for idx_se in sorted(re.findall(r'([\w\$]{2,4})\.seasonAndEpisode="([^"]+)";', webpage), key=lambda x: x[1]): if entry := self._extract_by_idx(idx_se[0], webpage, data_set): entry['series_id'] = series_id From a9c165305cbf15fdfc6a099c46ff955aab873f2a Mon Sep 17 00:00:00 2001 From: kclauhk <78251477+kclauhk@users.noreply.github.com> Date: Wed, 21 Aug 2024 16:11:01 +0800 Subject: [PATCH 4/7] to honor None values --- yt_dlp/extractor/kidoodletv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt_dlp/extractor/kidoodletv.py b/yt_dlp/extractor/kidoodletv.py index 3bbdb9022c..07103722ab 100644 --- a/yt_dlp/extractor/kidoodletv.py +++ b/yt_dlp/extractor/kidoodletv.py @@ -38,7 +38,7 @@ class KidoodleTVBaseIE(InfoExtractor): def get_field(field_name, idx, webpage, data): value = self._html_search_regex(rf'{idx}\.{field_name}=(?P"?(?P.+?)"?);', webpage, field_name, default=None, group=('a', 'b')) - return value[1] if value[1] != value[0] else (data.get(value[0]) or value[0]) + return value[1] if value[1] != value[0] else (data[value[0]] if value[0] in data else value[0]) idx = idx.replace('$', r'\$') video_id = get_field('id', idx, webpage, data) From b28dc2a42444bc9f58e75d9a85c3e3cbd8701246 Mon Sep 17 00:00:00 2001 From: kclauhk <78251477+kclauhk@users.noreply.github.com> Date: Wed, 21 Aug 2024 16:24:44 +0800 Subject: [PATCH 5/7] Revert "to honor None values" This reverts commit a9c165305cbf15fdfc6a099c46ff955aab873f2a. --- yt_dlp/extractor/kidoodletv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt_dlp/extractor/kidoodletv.py b/yt_dlp/extractor/kidoodletv.py index 07103722ab..3bbdb9022c 100644 --- a/yt_dlp/extractor/kidoodletv.py +++ b/yt_dlp/extractor/kidoodletv.py @@ -38,7 +38,7 @@ class KidoodleTVBaseIE(InfoExtractor): def get_field(field_name, idx, webpage, data): value = self._html_search_regex(rf'{idx}\.{field_name}=(?P"?(?P.+?)"?);', webpage, field_name, default=None, group=('a', 'b')) - return value[1] if value[1] != value[0] else (data[value[0]] if value[0] in data else value[0]) + return value[1] if value[1] != value[0] else (data.get(value[0]) or value[0]) idx = idx.replace('$', r'\$') video_id = get_field('id', idx, webpage, data) From 6f6cab852d7268903527daaade9e6da4617cf716 Mon Sep 17 00:00:00 2001 From: kclauhk <78251477+kclauhk@users.noreply.github.com> Date: Wed, 21 Aug 2024 20:54:55 +0800 Subject: [PATCH 6/7] return None when None/key not exist --- yt_dlp/extractor/kidoodletv.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/yt_dlp/extractor/kidoodletv.py b/yt_dlp/extractor/kidoodletv.py index 3bbdb9022c..19a033d10f 100644 --- a/yt_dlp/extractor/kidoodletv.py +++ b/yt_dlp/extractor/kidoodletv.py @@ -38,7 +38,8 @@ class KidoodleTVBaseIE(InfoExtractor): def get_field(field_name, idx, webpage, data): value = self._html_search_regex(rf'{idx}\.{field_name}=(?P"?(?P.+?)"?);', webpage, field_name, default=None, group=('a', 'b')) - return value[1] if value[1] != value[0] else (data.get(value[0]) or value[0]) + return (value[1] if value[1] != value[0] else ( + data.get(value[0]) if re.search(r'^[a-zA-Z_\$]{1,4}$', value[0]) else value[0])) idx = idx.replace('$', r'\$') video_id = get_field('id', idx, webpage, data) @@ -139,7 +140,7 @@ class KidoodleTVIE(KidoodleTVBaseIE): description = self._html_search_meta('description', webpage, 'description', default=None) data_set = self._extract_data(webpage, video_id) info = {} - if idx := self._html_search_regex(rf'([\w\$]{{2,4}})\.seasonAndEpisode="{season_episode}";', + if idx := self._html_search_regex(rf'([\w\$]{{1,4}})\.seasonAndEpisode="{season_episode}";', webpage, 'data_idx', default=None): info = self._extract_by_idx(idx, webpage, data_set, video_id) @@ -180,7 +181,7 @@ class KidoodleTVSeriesIE(KidoodleTVBaseIE): 'description', default=None) data_set = self._extract_data(webpage, series_id) entries = [] - for idx_se in sorted(re.findall(r'([\w\$]{2,4})\.seasonAndEpisode="([^"]+)";', webpage), + for idx_se in sorted(re.findall(r'([\w\$]{1,4})\.seasonAndEpisode="([^"]+)";', webpage), key=lambda x: x[1]): if entry := self._extract_by_idx(idx_se[0], webpage, data_set): entry['series_id'] = series_id From 4a6a19b127055a1947fe8004f3db86e4e1aa03b3 Mon Sep 17 00:00:00 2001 From: kclauhk <78251477+kclauhk@users.noreply.github.com> Date: Sat, 14 Sep 2024 11:06:43 +0800 Subject: [PATCH 7/7] Update test playlist count --- yt_dlp/extractor/kidoodletv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yt_dlp/extractor/kidoodletv.py b/yt_dlp/extractor/kidoodletv.py index 19a033d10f..6ebc38f490 100644 --- a/yt_dlp/extractor/kidoodletv.py +++ b/yt_dlp/extractor/kidoodletv.py @@ -161,7 +161,7 @@ class KidoodleTVSeriesIE(KidoodleTVBaseIE): 'title': 'Bluey: The Videogame by Abdallah Smash', 'description': 'Bluey: The Videogame on Nintendo Switch with no-commentary.', }, - 'playlist_count': 8, + 'playlist_mincount': 8, }, { 'url': 'https://kidoodle.tv/3083/unicorn-academy?category=What%27s%20NEW', 'info_dict': { @@ -169,7 +169,7 @@ class KidoodleTVSeriesIE(KidoodleTVBaseIE): 'title': 'Unicorn Academy', 'description': 'md5:d3f92c6bd76cc9941e60d827213b79f3', }, - 'playlist_count': 4, + 'playlist_mincount': 4, }] def _real_extract(self, url):