From 3b603dbdf139efe187f961dbe8b1b24ba16ae194 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 13 Jan 2022 16:31:08 +0530 Subject: [PATCH] Add option `--concat-playlist` Closes #1855, related: #382 --- README.md | 11 +++++++- yt_dlp/YoutubeDL.py | 25 ++++++++++-------- yt_dlp/__init__.py | 6 +++++ yt_dlp/options.py | 10 +++++++ yt_dlp/postprocessor/__init__.py | 1 + yt_dlp/postprocessor/ffmpeg.py | 45 ++++++++++++++++++++++++++++++++ yt_dlp/utils.py | 1 + 7 files changed, 87 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 6ba9163bba..54b565e593 100644 --- a/README.md +++ b/README.md @@ -893,6 +893,15 @@ You can also fork the project on github and run your fork's [build workflow](.gi multiple times --xattrs Write metadata to the video file's xattrs (using dublin core and xdg standards) + --concat-playlist POLICY Concatenate videos in a playlist. One of + "never" (default), "always", or + "multi_video" (only when the videos form a + single show). All the video files must have + same codecs and number of streams to be + concatable. The "pl_video:" prefix can be + used with "--paths" and "--output" to set + the output filename for the split files. + See "OUTPUT TEMPLATE" for details --fixup POLICY Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn (the @@ -1106,7 +1115,7 @@ To summarize, the general syntax for a field is: %(name[.keys][addition][>strf][,alternate][&replacement][|default])[flags][width][.precision][length]type ``` -Additionally, you can set different output templates for the various metadata files separately from the general output template by specifying the type of file followed by the template separated by a colon `:`. The different file types supported are `subtitle`, `thumbnail`, `description`, `annotation` (deprecated), `infojson`, `link`, `pl_thumbnail`, `pl_description`, `pl_infojson`, `chapter`. For example, `-o "%(title)s.%(ext)s" -o "thumbnail:%(title)s\%(title)s.%(ext)s"` will put the thumbnails in a folder with the same name as the video. If any of the templates (except default) is empty, that type of file will not be written. Eg: `--write-thumbnail -o "thumbnail:"` will write thumbnails only for playlists and not for video. +Additionally, you can set different output templates for the various metadata files separately from the general output template by specifying the type of file followed by the template separated by a colon `:`. The different file types supported are `subtitle`, `thumbnail`, `description`, `annotation` (deprecated), `infojson`, `link`, `pl_thumbnail`, `pl_description`, `pl_infojson`, `chapter`, `pl_video`. For example, `-o "%(title)s.%(ext)s" -o "thumbnail:%(title)s\%(title)s.%(ext)s"` will put the thumbnails in a folder with the same name as the video. If any of the templates (except default) is empty, that type of file will not be written. Eg: `--write-thumbnail -o "thumbnail:"` will write thumbnails only for playlists and not for video. The available fields are: diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 71369bc44c..dfca76bb00 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -1596,6 +1596,19 @@ class YoutubeDL(object): def _ensure_dir_exists(self, path): return make_dir(path, self.report_error) + @staticmethod + def _playlist_infodict(ie_result, **kwargs): + return { + **ie_result, + 'playlist': ie_result.get('title') or ie_result.get('id'), + 'playlist_id': ie_result.get('id'), + 'playlist_title': ie_result.get('title'), + 'playlist_uploader': ie_result.get('uploader'), + 'playlist_uploader_id': ie_result.get('uploader_id'), + 'playlist_index': 0, + **kwargs, + } + def __process_playlist(self, ie_result, download): # We process each entry in the playlist playlist = ie_result.get('title') or ie_result.get('id') @@ -1695,17 +1708,7 @@ class YoutubeDL(object): _infojson_written = False if not self.params.get('simulate') and self.params.get('allow_playlist_files', True): - ie_copy = { - 'playlist': playlist, - 'playlist_id': ie_result.get('id'), - 'playlist_title': ie_result.get('title'), - 'playlist_uploader': ie_result.get('uploader'), - 'playlist_uploader_id': ie_result.get('uploader_id'), - 'playlist_index': 0, - 'n_entries': n_entries, - } - ie_copy.update(dict(ie_result)) - + ie_copy = self._playlist_infodict(ie_result, n_entries=n_entries) _infojson_written = self._write_info_json( 'playlist', ie_result, self.prepare_filename(ie_copy, 'pl_infojson')) if _infojson_written is None: diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index 85f000df4f..f3faf0ce49 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -591,6 +591,12 @@ def _real_main(argv=None): # XAttrMetadataPP should be run after post-processors that may change file contents if opts.xattrs: postprocessors.append({'key': 'XAttrMetadata'}) + if opts.concat_playlist != 'never': + postprocessors.append({ + 'key': 'FFmpegConcat', + 'only_multi_video': opts.concat_playlist != 'always', + 'when': 'playlist', + }) # Exec must be the last PP of each category if opts.exec_before_dl_cmd: opts.exec_cmd.setdefault('before_dl', opts.exec_before_dl_cmd) diff --git a/yt_dlp/options.py b/yt_dlp/options.py index cc0a933be5..cb6f01d4d3 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -1397,6 +1397,16 @@ def create_parser(): '--xattrs', action='store_true', dest='xattrs', default=False, help='Write metadata to the video file\'s xattrs (using dublin core and xdg standards)') + postproc.add_option( + '--concat-playlist', + metavar='POLICY', dest='concat_playlist', default='multi_video', + choices=('never', 'always', 'multi_video'), + help=( + 'Concatenate videos in a playlist. One of "never" (default), "always", or ' + '"multi_video" (only when the videos form a single show). ' + 'All the video files must have same codecs and number of streams to be concatable. ' + 'The "pl_video:" prefix can be used with "--paths" and "--output" to ' + 'set the output filename for the split files. See "OUTPUT TEMPLATE" for details')) postproc.add_option( '--fixup', metavar='POLICY', dest='fixup', default=None, diff --git a/yt_dlp/postprocessor/__init__.py b/yt_dlp/postprocessor/__init__.py index 7f8adb3686..e411cc145c 100644 --- a/yt_dlp/postprocessor/__init__.py +++ b/yt_dlp/postprocessor/__init__.py @@ -7,6 +7,7 @@ from .embedthumbnail import EmbedThumbnailPP from .exec import ExecPP, ExecAfterDownloadPP from .ffmpeg import ( FFmpegPostProcessor, + FFmpegConcatPP, FFmpegEmbedSubtitlePP, FFmpegExtractAudioPP, FFmpegFixupDuplicateMoovPP, diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 43c1b276d0..213de0ecf3 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -1123,3 +1123,48 @@ class FFmpegThumbnailsConvertorPP(FFmpegPostProcessor): if not has_thumbnail: self.to_screen('There aren\'t any thumbnails to convert') return files_to_delete, info + + +class FFmpegConcatPP(FFmpegPostProcessor): + def __init__(self, downloader, only_multi_video=False): + self._only_multi_video = only_multi_video + super().__init__(downloader) + + def concat_files(self, in_files, out_file): + if len(in_files) == 1: + os.replace(in_files[0], out_file) + return + + codecs = [traverse_obj(self.get_metadata_object(file), ('streams', ..., 'codec_name')) for file in in_files] + if len(set(map(tuple, codecs))) > 1: + raise PostProcessingError( + 'The files have different streams/codecs and cannot be concatenated. ' + 'Either select different formats or --recode-video them to a common format') + super().concat_files(in_files, out_file) + + @PostProcessor._restrict_to(images=False) + def run(self, info): + if not info.get('entries') or self._only_multi_video and info['_type'] != 'multi_video': + return [], info + elif None in info['entries']: + raise PostProcessingError('Aborting concatenation because some downloads failed') + elif any(len(entry) > 1 for entry in traverse_obj(info, ('entries', ..., 'requested_downloads')) or []): + raise PostProcessingError('Concatenation is not supported when downloading multiple separate formats') + + in_files = traverse_obj(info, ('entries', ..., 'requested_downloads', 0, 'filepath')) + if not in_files: + self.to_screen('There are no files to concatenate') + return [], info + + ie_copy = self._downloader._playlist_infodict(info) + exts = [traverse_obj(entry, ('requested_downloads', 0, 'ext'), 'ext') for entry in info['entries']] + ie_copy['ext'] = exts[0] if len(set(exts)) == 1 else 'mkv' + out_file = self._downloader.prepare_filename(ie_copy, 'pl_video') + + self.concat_files(in_files, out_file) + + info['requested_downloads'] = [{ + 'filepath': out_file, + 'ext': ie_copy['ext'], + }] + return in_files, info diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index 9b7f658546..b7e718028b 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -4695,6 +4695,7 @@ OUTTMPL_TYPES = { 'annotation': 'annotations.xml', 'infojson': 'info.json', 'link': None, + 'pl_video': None, 'pl_thumbnail': None, 'pl_description': 'description', 'pl_infojson': 'info.json',