administration/yt-dlp
1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2024-11-09 10:50:38 +00:00
Code Issues Projects Releases Wiki Activity

Improved progress reporting (See desc) (#1125)

Browse Source 
* Separate `--console-title` and `--no-progress`
* Add option `--progress` to show progress-bar even in quiet mode
* Fix and refactor `minicurses`
* Use `minicurses` for all progress reporting
* Standardize use of terminal sequences and enable color support for windows 10
* Add option `--progress-template` to customize progress-bar and console-title
* Add postprocessor hooks and progress reporting

Closes: #906, #901, #1085, #1170
This commit is contained in:
pukkandan 2021-10-09 00:41:59 +05:30 committed by GitHub
parent fee3f44f5f
commit 819e05319b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 301 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
README.md
View File

@ -604,7 +604,18 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
(Alias: --force-download-archive)
--newline Output progress bar as new lines
--no-progress Do not print progress bar
--progress Show progress bar, even if in quiet mode
--console-title Display progress in console titlebar
--progress-template [TYPES:]TEMPLATE
Template for progress outputs, optionally
prefixed with one of "download:" (default),
"download-title:" (the console title),
"postprocess:", or "postprocess-title:".
The video's fields are accessible under the
"info" key and the progress attributes are
accessible under "progress" key. Eg:
--console-title --progress-template
"download-title:%(info.id)s-%(progress.eta)s"
-v, --verbose Print various debugging information
--dump-pages Print downloaded pages encoded using base64
to debug problems (very verbose)

3
test/test_YoutubeDL.py
View File

@ -666,8 +666,7 @@ class TestYoutubeDL(unittest.TestCase):
ydl._num_downloads = 1
self.assertEqual(ydl.validate_outtmpl(tmpl), None)
outtmpl, tmpl_dict = ydl.prepare_outtmpl(tmpl, info or self.outtmpl_info)
out = ydl.escape_outtmpl(outtmpl) % tmpl_dict
out = ydl.evaluate_outtmpl(tmpl, info or self.outtmpl_info)
fname = ydl.prepare_filename(info or self.outtmpl_info)
if not isinstance(expected, (list, tuple)):

77
yt_dlp/YoutubeDL.py
View File

@ -42,6 +42,7 @@ from .compat import (
compat_urllib_error,
compat_urllib_request,
compat_urllib_request_DataHandler,
windows_enable_vt_mode,
)
from .cookies import load_cookies
from .utils import (
@ -67,8 +68,6 @@ from .utils import (
float_or_none,
format_bytes,
format_field,
STR_FORMAT_RE_TMPL,
STR_FORMAT_TYPES,
formatSeconds,
GeoRestrictedError,
HEADRequest,
@ -101,9 +100,13 @@ from .utils import (
sanitize_url,
sanitized_Request,
std_headers,
STR_FORMAT_RE_TMPL,
STR_FORMAT_TYPES,
str_or_none,
strftime_or_none,
subtitles_filename,
supports_terminal_sequences,
TERMINAL_SEQUENCES,
ThrottledDownload,
to_high_limit_path,
traverse_obj,
@ -248,6 +251,7 @@ class YoutubeDL(object):
rejecttitle: Reject downloads for matching titles.
logger: Log messages to a logging.Logger instance.
logtostderr: Log messages to stderr instead of stdout.
consoletitle: Display progress in console window's titlebar.
writedescription: Write the video description to a .description file
writeinfojson: Write the video description to a .info.json file
clean_infojson: Remove private fields from the infojson
@ -353,6 +357,15 @@ class YoutubeDL(object):
Progress hooks are guaranteed to be called at least once
(with status "finished") if the download is successful.
postprocessor_hooks: A list of functions that get called on postprocessing
progress, with a dictionary with the entries
* status: One of "started", "processing", or "finished".
Check this first and ignore unknown values.
* postprocessor: Name of the postprocessor
* info_dict: The extracted info_dict
Progress hooks are guaranteed to be called at least twice
(with status "started" and "finished") if the processing is successful.
merge_output_format: Extension to use when merging formats.
final_ext: Expected final extension; used to detect when the file was
already downloaded and converted. "merge_output_format" is
@ -412,11 +425,15 @@ class YoutubeDL(object):
filename, abort-on-error, multistreams, no-live-chat,
no-clean-infojson, no-playlist-metafiles, no-keep-subs.
Refer __init__.py for their implementation
progress_template: Dictionary of templates for progress outputs.
Allowed keys are 'download', 'postprocess',
'download-title' (console title) and 'postprocess-title'.
The template is mapped on a dictionary with keys 'progress' and 'info'
The following parameters are not used by YoutubeDL itself, they are used by
the downloader (see yt_dlp/downloader/common.py):
nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize,
max_filesize, test, noresizebuffer, retries, continuedl, noprogress, consoletitle,
max_filesize, test, noresizebuffer, retries, continuedl, noprogress,
xattr_set_filesize, external_downloader_args, hls_use_mpegts, http_chunk_size.
The following options are used by the post processors:
@ -484,26 +501,27 @@ class YoutubeDL(object):
self._first_webpage_request = True
self._post_hooks = []
self._progress_hooks = []
self._postprocessor_hooks = []
self._download_retcode = 0
self._num_downloads = 0
self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
self._err_file = sys.stderr
self.params = {
# Default parameters
'nocheckcertificate': False,
}
self.params.update(params)
self.params = params
self.cache = Cache(self)
windows_enable_vt_mode()
self.params['no_color'] = self.params.get('no_color') or not supports_terminal_sequences(self._err_file)
if sys.version_info < (3, 6):
self.report_warning(
'Python version %d.%d is not supported! Please update to Python 3.6 or above' % sys.version_info[:2])
if self.params.get('allow_unplayable_formats'):
self.report_warning(
'You have asked for unplayable formats to be listed/downloaded. '
'This is a developer option intended for debugging. '
'If you experience any issues while using this option, DO NOT open a bug report')
f'You have asked for {self._color_text("unplayable formats", "blue")} to be listed/downloaded. '
'This is a developer option intended for debugging. \n'
' If you experience any issues while using this option, '
f'{self._color_text("DO NOT", "red")} open a bug report')
def check_deprecated(param, option, suggestion):
if self.params.get(param) is not None:
@ -675,9 +693,13 @@ class YoutubeDL(object):
self._post_hooks.append(ph)
def add_progress_hook(self, ph):
"""Add the progress hook (currently only for the file downloader)"""
"""Add the download progress hook"""
self._progress_hooks.append(ph)
def add_postprocessor_hook(self, ph):
"""Add the postprocessing progress hook"""
self._postprocessor_hooks.append(ph)
def _bidi_workaround(self, message):
if not hasattr(self, '_output_channel'):
return message
@ -790,6 +812,11 @@ class YoutubeDL(object):
self.to_stdout(
message, skip_eol, quiet=self.params.get('quiet', False))
def _color_text(self, text, color):
if self.params.get('no_color'):
return text
return f'{TERMINAL_SEQUENCES[color.upper()]}{text}{TERMINAL_SEQUENCES["RESET_STYLE"]}'
def report_warning(self, message, only_once=False):
'''
Print the message to stderr, it will be prefixed with 'WARNING:'
@ -800,24 +827,14 @@ class YoutubeDL(object):
else:
if self.params.get('no_warnings'):
return
if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
_msg_header = '\033[0;33mWARNING:\033[0m'
else:
_msg_header = 'WARNING:'
warning_message = '%s %s' % (_msg_header, message)
self.to_stderr(warning_message, only_once)
self.to_stderr(f'{self._color_text("WARNING:", "yellow")} {message}', only_once)
def report_error(self, message, tb=None):
'''
Do the same as trouble, but prefixes the message with 'ERROR:', colored
in red if stderr is a tty file.
'''
if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
_msg_header = '\033[0;31mERROR:\033[0m'
else:
_msg_header = 'ERROR:'
error_message = '%s %s' % (_msg_header, message)
self.trouble(error_message, tb)
self.trouble(f'{self._color_text("ERROR:", "red")} {message}', tb)
def write_debug(self, message, only_once=False):
'''Log debug message or Print message to stderr'''
@ -919,7 +936,7 @@ class YoutubeDL(object):
return err
def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None):
""" Make the template and info_dict suitable for substitution : ydl.outtmpl_escape(outtmpl) % info_dict """
""" Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict """
info_dict.setdefault('epoch', int(time.time())) # keep epoch consistent once set
info_dict = dict(info_dict) # Do not sanitize so as not to consume LazyList
@ -1073,6 +1090,10 @@ class YoutubeDL(object):
return EXTERNAL_FORMAT_RE.sub(create_key, outtmpl), TMPL_DICT
def evaluate_outtmpl(self, outtmpl, info_dict, *args, **kwargs):
outtmpl, info_dict = self.prepare_outtmpl(outtmpl, info_dict, *args, **kwargs)
return self.escape_outtmpl(outtmpl) % info_dict
def _prepare_filename(self, info_dict, tmpl_type='default'):
try:
sanitize = lambda k, v: sanitize_filename(
@ -2431,10 +2452,8 @@ class YoutubeDL(object):
if self.params.get('forceprint') or self.params.get('forcejson'):
self.post_extract(info_dict)
for tmpl in self.params.get('forceprint', []):
if re.match(r'\w+$', tmpl):
tmpl = '%({})s'.format(tmpl)
tmpl, info_copy = self.prepare_outtmpl(tmpl, info_dict)
self.to_stdout(self.escape_outtmpl(tmpl) % info_copy)
self.to_stdout(self.evaluate_outtmpl(
f'%({tmpl})s' if re.match(r'\w+$', tmpl) else tmpl, info_dict))
print_mandatory('title')
print_mandatory('id')

8
yt_dlp/__init__.py
View File

@ -302,11 +302,14 @@ def _real_main(argv=None):
parser.error('invalid %s %r: %s' % (msg, tmpl, error_to_compat_str(err)))
for k, tmpl in opts.outtmpl.items():
validate_outtmpl(tmpl, '%s output template' % k)
validate_outtmpl(tmpl, f'{k} output template')
opts.forceprint = opts.forceprint or []
for tmpl in opts.forceprint or []:
validate_outtmpl(tmpl, 'print template')
validate_outtmpl(opts.sponsorblock_chapter_title, 'SponsorBlock chapter title')
for k, tmpl in opts.progress_template.items():
k = f'{k[:-6]} console title' if '-title' in k else f'{k} progress'
validate_outtmpl(tmpl, f'{k} template')
if opts.extractaudio and not opts.keepvideo and opts.format is None:
opts.format = 'bestaudio/best'
@ -633,8 +636,9 @@ def _real_main(argv=None):
'noresizebuffer': opts.noresizebuffer,
'http_chunk_size': opts.http_chunk_size,
'continuedl': opts.continue_dl,
'noprogress': opts.noprogress,
'noprogress': opts.quiet if opts.noprogress is None else opts.noprogress,
'progress_with_newline': opts.progress_with_newline,
'progress_template': opts.progress_template,
'playliststart': opts.playliststart,
'playlistend': opts.playlistend,
'playlistreverse': opts.playlist_reverse,

7
yt_dlp/compat.py
View File

@ -159,6 +159,12 @@ except ImportError:
compat_pycrypto_AES = None
def windows_enable_vt_mode(): # TODO: Do this the proper way https://bugs.python.org/issue30075
if compat_os_name != 'nt':
return
os.system('')
# Deprecated
compat_basestring = str
@ -281,5 +287,6 @@ __all__ = [
'compat_xml_parse_error',
'compat_xpath',
'compat_zip',
'windows_enable_vt_mode',
'workaround_optparse_bug9161',
]

84
yt_dlp/downloader/common.py
View File

@ -7,7 +7,6 @@ import sys
import time
import random
from ..compat import compat_os_name
from ..utils import (
decodeArgument,
encodeFilename,
@ -17,6 +16,7 @@ from ..utils import (
timeconvert,
)
from ..minicurses import (
MultilineLogger,
MultilinePrinter,
QuietMultilinePrinter,
BreaklineStatusPrinter
@ -44,8 +44,6 @@ class FileDownloader(object):
noresizebuffer: Do not automatically resize the download buffer.
continuedl: Try to continue downloads if possible.
noprogress: Do not print the progress bar.
logtostderr: Log messages to stderr instead of stdout.
consoletitle: Display progress in console window's titlebar.
nopart: Do not use temporary .part files.
updatetime: Use the Last-modified header to set output file timestamps.
test: Download only first bytes to test the downloader.
@ -61,6 +59,7 @@ class FileDownloader(object):
http_chunk_size: Size of a chunk for chunk-based HTTP downloading. May be
useful for bypassing bandwidth throttling imposed by
a webserver (experimental)
progress_template: See YoutubeDL.py
Subclasses of this one must re-define the real_download method.
"""
@ -73,7 +72,7 @@ class FileDownloader(object):
self.ydl = ydl
self._progress_hooks = []
self.params = params
self._multiline = None
self._prepare_multiline_status()
self.add_progress_hook(self.report_progress)
@staticmethod
@ -242,55 +241,46 @@ class FileDownloader(object):
"""Report destination filename."""
self.to_screen('[download] Destination: ' + filename)
def _prepare_multiline_status(self, lines):
if self.params.get('quiet'):
def _prepare_multiline_status(self, lines=1):
if self.params.get('noprogress'):
self._multiline = QuietMultilinePrinter()
elif self.params.get('progress_with_newline', False):
elif self.ydl.params.get('logger'):
self._multiline = MultilineLogger(self.ydl.params['logger'], lines)
elif self.params.get('progress_with_newline'):
self._multiline = BreaklineStatusPrinter(sys.stderr, lines)
elif self.params.get('noprogress', False):
self._multiline = None
else:
self._multiline = MultilinePrinter(sys.stderr, lines)
self._multiline = MultilinePrinter(sys.stderr, lines, not self.params.get('quiet'))
def _finish_multiline_status(self):
if self._multiline is not None:
self._multiline.end()
self._multiline.end()
def _report_progress_status(self, msg, is_last_line=False, progress_line=None):
fullmsg = '[download] ' + msg
if self.params.get('progress_with_newline', False):
self.to_screen(fullmsg)
elif progress_line is not None and self._multiline is not None:
self._multiline.print_at_line(fullmsg, progress_line)
else:
if compat_os_name == 'nt' or not sys.stderr.isatty():
prev_len = getattr(self, '_report_progress_prev_line_length', 0)
if prev_len > len(fullmsg):
fullmsg += ' ' * (prev_len - len(fullmsg))
self._report_progress_prev_line_length = len(fullmsg)
clear_line = '\r'
else:
clear_line = '\r\x1b[K'
self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
self.to_console_title('yt-dlp ' + msg)
def _report_progress_status(self, s):
progress_dict = s.copy()
progress_dict.pop('info_dict')
progress_dict = {'info': s['info_dict'], 'progress': progress_dict}
progress_template = self.params.get('progress_template', {})
self._multiline.print_at_line(self.ydl.evaluate_outtmpl(
progress_template.get('download') or '[download] %(progress._default_template)s',
progress_dict), s.get('progress_idx') or 0)
self.to_console_title(self.ydl.evaluate_outtmpl(
progress_template.get('download-title') or 'yt-dlp %(progress._default_template)s',
progress_dict))
def report_progress(self, s):
if s['status'] == 'finished':
if self.params.get('noprogress', False):
if self.params.get('noprogress'):
self.to_screen('[download] Download completed')
else:
msg_template = '100%%'
if s.get('total_bytes') is not None:
s['_total_bytes_str'] = format_bytes(s['total_bytes'])
msg_template += ' of %(_total_bytes_str)s'
if s.get('elapsed') is not None:
s['_elapsed_str'] = self.format_seconds(s['elapsed'])
msg_template += ' in %(_elapsed_str)s'
self._report_progress_status(
msg_template % s, is_last_line=True, progress_line=s.get('progress_idx'))
return
if self.params.get('noprogress'):
msg_template = '100%%'
if s.get('total_bytes') is not None:
s['_total_bytes_str'] = format_bytes(s['total_bytes'])
msg_template += ' of %(_total_bytes_str)s'
if s.get('elapsed') is not None:
s['_elapsed_str'] = self.format_seconds(s['elapsed'])
msg_template += ' in %(_elapsed_str)s'
s['_percent_str'] = self.format_percent(100)
s['_default_template'] = msg_template % s
self._report_progress_status(s)
return
if s['status'] != 'downloading':
@ -332,8 +322,8 @@ class FileDownloader(object):
msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
else:
msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'
self._report_progress_status(msg_template % s, progress_line=s.get('progress_idx'))
s['_default_template'] = msg_template % s
self._report_progress_status(s)
def report_resuming_byte(self, resume_len):
"""Report attempt to resume at given byte."""
@ -405,7 +395,9 @@ class FileDownloader(object):
'[download] Sleeping %s seconds ...' % (
sleep_interval_sub))
time.sleep(sleep_interval_sub)
return self.real_download(filename, info_dict), True
ret = self.real_download(filename, info_dict)
self._finish_multiline_status()
return ret, True
def real_download(self, filename, info_dict):
"""Real download process. Redefine in subclasses."""

4
yt_dlp/downloader/fragment.py
View File

@ -393,9 +393,7 @@ class FragmentFD(FileDownloader):
result = result and job.result()
finally:
tpe.shutdown(wait=True)
self._finish_multiline_status()
return True
return result
def download_and_append_fragments(self, ctx, fragments, info_dict, *, pack_func=None, finish_func=None, tpe=None):
fragment_retries = self.params.get('fragment_retries', 0)

5
yt_dlp/extractor/common.py
View File

@ -1134,10 +1134,7 @@ class InfoExtractor(object):
if mobj:
break
if not self.get_param('no_color') and compat_os_name != 'nt' and sys.stderr.isatty():
_name = '\033[0;34m%s\033[0m' % name
else:
_name = name
_name = self._downloader._color_text(name, 'blue')
if mobj:
if group is None:

196
yt_dlp/minicurses.py
View File

@ -1,10 +1,12 @@
import os
from threading import Lock
from .utils import compat_os_name, get_windows_version
from .utils import supports_terminal_sequences, TERMINAL_SEQUENCES
class MultilinePrinterBase():
class MultilinePrinterBase:
def __init__(self, stream=None, lines=1):
self.stream = stream
self.maximum = lines - 1
def __enter__(self):
return self
@ -17,119 +19,87 @@ class MultilinePrinterBase():
def end(self):
pass
class MultilinePrinter(MultilinePrinterBase):
def __init__(self, stream, lines):
"""
@param stream stream to write to
@lines number of lines to be written
"""
self.stream = stream
is_win10 = compat_os_name == 'nt' and get_windows_version() >= (10, )
self.CARRIAGE_RETURN = '\r'
if os.getenv('TERM') and self._isatty() or is_win10:
# reason not to use curses https://github.com/yt-dlp/yt-dlp/pull/1036#discussion_r713851492
# escape sequences for Win10 https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences
self.UP = '\x1b[A'
self.DOWN = '\n'
self.ERASE_LINE = '\x1b[K'
self._HAVE_FULLCAP = self._isatty() or is_win10
else:
self.UP = self.DOWN = self.ERASE_LINE = None
self._HAVE_FULLCAP = False
# lines are numbered from top to bottom, counting from 0 to self.maximum
self.maximum = lines - 1
self.lastline = 0
self.lastlength = 0
self.movelock = Lock()
@property
def have_fullcap(self):
"""
True if the TTY is allowing to control cursor,
so that multiline progress works
"""
return self._HAVE_FULLCAP
def _isatty(self):
try:
return self.stream.isatty()
except BaseException:
return False
def _move_cursor(self, dest):
current = min(self.lastline, self.maximum)
self.stream.write(self.CARRIAGE_RETURN)
if current == dest:
# current and dest are at same position, no need to move cursor
return
elif current > dest:
# when maximum == 2,
# 0. dest
# 1.
# 2. current
self.stream.write(self.UP * (current - dest))
elif current < dest:
# when maximum == 2,
# 0. current
# 1.
# 2. dest
self.stream.write(self.DOWN * (dest - current))
self.lastline = dest
def print_at_line(self, text, pos):
with self.movelock:
if self.have_fullcap:
self._move_cursor(pos)
self.stream.write(self.ERASE_LINE)
self.stream.write(text)
else:
if self.maximum != 0:
# let user know about which line is updating the status
text = f'{pos + 1}: {text}'
textlen = len(text)
if self.lastline == pos:
# move cursor at the start of progress when writing to same line
self.stream.write(self.CARRIAGE_RETURN)
if self.lastlength > textlen:
text += ' ' * (self.lastlength - textlen)
self.lastlength = textlen
else:
# otherwise, break the line
self.stream.write('\n')
self.lastlength = 0
self.stream.write(text)
self.lastline = pos
def end(self):
with self.movelock:
# move cursor to the end of the last line, and write line break
# so that other to_screen calls can precede
self._move_cursor(self.maximum)
self.stream.write('\n')
def _add_line_number(self, text, line):
if self.maximum:
return f'{line + 1}: {text}'
return text
class QuietMultilinePrinter(MultilinePrinterBase):
def __init__(self):
self.have_fullcap = True
pass
class MultilineLogger(MultilinePrinterBase):
def print_at_line(self, text, pos):
# stream is the logger object, not an actual stream
self.stream.debug(self._add_line_number(text, pos))
class BreaklineStatusPrinter(MultilinePrinterBase):
def __init__(self, stream, lines):
"""
@param stream stream to write to
"""
self.stream = stream
self.maximum = lines
self.have_fullcap = True
def print_at_line(self, text, pos):
if self.maximum != 0:
# let user know about which line is updating the status
text = f'{pos + 1}: {text}'
self.stream.write(text + '\n')
self.stream.write(self._add_line_number(text, pos) + '\n')
class MultilinePrinter(MultilinePrinterBase):
def __init__(self, stream=None, lines=1, preserve_output=True):
super().__init__(stream, lines)
self.preserve_output = preserve_output
self._lastline = self._lastlength = 0
self._movelock = Lock()
self._HAVE_FULLCAP = supports_terminal_sequences(self.stream)
def lock(func):
def wrapper(self, *args, **kwargs):
with self._movelock:
return func(self, *args, **kwargs)
return wrapper
def _move_cursor(self, dest):
current = min(self._lastline, self.maximum)
self.stream.write('\r')
distance = dest - current
if distance < 0:
self.stream.write(TERMINAL_SEQUENCES['UP'] * -distance)
elif distance > 0:
self.stream.write(TERMINAL_SEQUENCES['DOWN'] * distance)
self._lastline = dest
@lock
def print_at_line(self, text, pos):
if self._HAVE_FULLCAP:
self._move_cursor(pos)
self.stream.write(TERMINAL_SEQUENCES['ERASE_LINE'])
self.stream.write(text)
return
text = self._add_line_number(text, pos)
textlen = len(text)
if self._lastline == pos:
# move cursor at the start of progress when writing to same line
self.stream.write('\r')
if self._lastlength > textlen:
text += ' ' * (self._lastlength - textlen)
self._lastlength = textlen
else:
# otherwise, break the line
self.stream.write('\n')
self._lastlength = textlen
self.stream.write(text)
self._lastline = pos
@lock
def end(self):
# move cursor to the end of the last line, and write line break
# so that other to_screen calls can precede
if self._HAVE_FULLCAP:
self._move_cursor(self.maximum)
if self.preserve_output:
self.stream.write('\n')
return
if self._HAVE_FULLCAP:
self.stream.write(
TERMINAL_SEQUENCES['ERASE_LINE']
+ f'{TERMINAL_SEQUENCES["UP"]}{TERMINAL_SEQUENCES["ERASE_LINE"]}' * self.maximum)
else:
self.stream.write(' ' * self._lastlength)

20
yt_dlp/options.py
View File

@ -910,12 +910,30 @@ def parseOpts(overrideArguments=None):
help='Output progress bar as new lines')
verbosity.add_option(
'--no-progress',
action='store_true', dest='noprogress', default=False,
action='store_true', dest='noprogress', default=None,
help='Do not print progress bar')
verbosity.add_option(
'--progress',
action='store_false', dest='noprogress',
help='Show progress bar, even if in quiet mode')
verbosity.add_option(
'--console-title',
action='store_true', dest='consoletitle', default=False,
help='Display progress in console titlebar')
verbosity.add_option(
'--progress-template',
metavar='[TYPES:]TEMPLATE', dest='progress_template', default={}, type='str',
action='callback', callback=_dict_from_options_callback,
callback_kwargs={
'allowed_keys': '(download|postprocess)(-title)?',
'default_key': 'download'
}, help=(
'Template for progress outputs, optionally prefixed with one of "download:" (default), '
'"download-title:" (the console title), "postprocess:", or "postprocess-title:". '
'The video\'s fields are accessible under the "info" key and '
'the progress attributes are accessible under "progress" key. Eg: '
# TODO: Document the fields inside "progress"
'--console-title --progress-template "download-title:%(info.id)s-%(progress.eta)s"'))
verbosity.add_option(
'-v', '--verbose',
action='store_true', dest='verbose', default=False,

63
yt_dlp/postprocessor/common.py
View File

@ -1,5 +1,6 @@
from __future__ import unicode_literals
import copy
import functools
import os
@ -11,7 +12,26 @@ from ..utils import (
)
class PostProcessor(object):
class PostProcessorMetaClass(type):
@staticmethod
def run_wrapper(func):
@functools.wraps(func)
def run(self, info, *args, **kwargs):
self._hook_progress({'status': 'started'}, info)
ret = func(self, info, *args, **kwargs)
if ret is not None:
_, info = ret
self._hook_progress({'status': 'finished'}, info)
return ret
return run
def __new__(cls, name, bases, attrs):
if 'run' in attrs:
attrs['run'] = cls.run_wrapper(attrs['run'])
return type.__new__(cls, name, bases, attrs)
class PostProcessor(metaclass=PostProcessorMetaClass):
"""Post Processor class.
PostProcessor objects can be added to downloaders with their
@ -34,7 +54,9 @@ class PostProcessor(object):
_downloader = None
def __init__(self, downloader=None):
self._downloader = downloader
self._progress_hooks = []
self.add_progress_hook(self.report_progress)
self.set_downloader(downloader)
self.PP_NAME = self.pp_key()
@classmethod
@ -68,6 +90,10 @@ class PostProcessor(object):
def set_downloader(self, downloader):
"""Sets the downloader for this PP."""
self._downloader = downloader
if not downloader:
return
for ph in downloader._postprocessor_hooks:
self.add_progress_hook(ph)
@staticmethod
def _restrict_to(*, video=True, audio=True, images=True):
@ -115,6 +141,39 @@ class PostProcessor(object):
return _configuration_args(
self.pp_key(), self.get_param('postprocessor_args'), exe, *args, **kwargs)
def _hook_progress(self, status, info_dict):
if not self._progress_hooks:
return
info_dict = dict(info_dict)
for key in ('__original_infodict', '__postprocessors'):
info_dict.pop(key, None)
status.update({
'info_dict': copy.deepcopy(info_dict),
'postprocessor': self.pp_key(),
})
for ph in self._progress_hooks:
ph(status)
def add_progress_hook(self, ph):
# See YoutubeDl.py (search for postprocessor_hooks) for a description of this interface
self._progress_hooks.append(ph)
def report_progress(self, s):
s['_default_template'] = '%(postprocessor)s %(status)s' % s
progress_dict = s.copy()
progress_dict.pop('info_dict')
progress_dict = {'info': s['info_dict'], 'progress': progress_dict}
progress_template = self.get_param('progress_template', {})
tmpl = progress_template.get('postprocess')
if tmpl:
self._downloader.to_stdout(self._downloader.evaluate_outtmpl(tmpl, progress_dict))
self._downloader.to_console_title(self._downloader.evaluate_outtmpl(
progress_template.get('postprocess-title') or 'yt-dlp %(progress._default_template)s',
progress_dict))
class AudioConversionError(PostProcessingError):
pass

3
yt_dlp/postprocessor/metadataparser.py
View File

@ -62,8 +62,7 @@ class MetadataParserPP(PostProcessor):
def interpretter(self, inp, out):
def f(info):
outtmpl, tmpl_dict = self._downloader.prepare_outtmpl(template, info)
data_to_parse = self._downloader.escape_outtmpl(outtmpl) % tmpl_dict
data_to_parse = self._downloader.evaluate_outtmpl(template, info)
self.write_debug(f'Searching for {out_re.pattern!r} in {template!r}')
match = out_re.search(data_to_parse)
if match is None:

3
yt_dlp/postprocessor/modify_chapters.py
View File

@ -292,8 +292,7 @@ class ModifyChaptersPP(FFmpegPostProcessor):
'name': SponsorBlockPP.CATEGORIES[category],
'category_names': [SponsorBlockPP.CATEGORIES[c] for c in cats]
})
outtmpl, tmpl_dict = self._downloader.prepare_outtmpl(self._sponsorblock_chapter_title, c)
c['title'] = self._downloader.escape_outtmpl(outtmpl) % tmpl_dict
c['title'] = self._downloader.evaluate_outtmpl(self._sponsorblock_chapter_title, c)
# Merge identically named sponsors.
if (new_chapters and 'categories' in new_chapters[-1]
and new_chapters[-1]['title'] == c['title']):

23
yt_dlp/utils.py
View File

@ -6440,3 +6440,26 @@ def jwt_encode_hs256(payload_data, key, headers={}):
signature_b64 = base64.b64encode(h.digest())
token = header_b64 + b'.' + payload_b64 + b'.' + signature_b64
return token
def supports_terminal_sequences(stream):
if compat_os_name == 'nt':
if get_windows_version() < (10, ):
return False
elif not os.getenv('TERM'):
return False
try:
return stream.isatty()
except BaseException:
return False
TERMINAL_SEQUENCES = {
'DOWN': '\n',
'UP': '\x1b[A',
'ERASE_LINE': '\x1b[K',
'RED': '\033[0;31m',
'YELLOW': '\033[0;33m',
'BLUE': '\033[0;34m',
'RESET_STYLE': '\033[0m',
}