From 77f9033095cd8e1092a80db67f2b577cf13f95a8 Mon Sep 17 00:00:00 2001 From: felix Date: Fri, 4 Feb 2022 14:37:02 +0100 Subject: [PATCH] [compat] Split into sub-modules (#2173) Authored by: fstirlitz, pukkandan --- test/test_compat.py | 8 - yt_dlp/compat.py | 302 ------------------------------ yt_dlp/compat/__init__.py | 129 +++++++++++++ yt_dlp/compat/_deprecated.py | 47 +++++ yt_dlp/compat/_legacy.py | 54 ++++++ yt_dlp/compat/asyncio/__init__.py | 16 ++ yt_dlp/compat/asyncio/tasks.py | 8 + yt_dlp/compat/re.py | 14 ++ yt_dlp/downloader/websocket.py | 2 +- yt_dlp/extractor/common.py | 5 +- yt_dlp/webvtt.py | 7 +- 11 files changed, 274 insertions(+), 318 deletions(-) delete mode 100644 yt_dlp/compat.py create mode 100644 yt_dlp/compat/__init__.py create mode 100644 yt_dlp/compat/_deprecated.py create mode 100644 yt_dlp/compat/_legacy.py create mode 100644 yt_dlp/compat/asyncio/__init__.py create mode 100644 yt_dlp/compat/asyncio/tasks.py create mode 100644 yt_dlp/compat/re.py diff --git a/test/test_compat.py b/test/test_compat.py index 29e7384f04..8e40a41802 100644 --- a/test/test_compat.py +++ b/test/test_compat.py @@ -42,14 +42,6 @@ class TestCompat(unittest.TestCase): finally: compat_setenv('HOME', old_home or '') - def test_all_present(self): - import yt_dlp.compat - all_names = yt_dlp.compat.__all__ - present_names = set(filter( - lambda c: '_' in c and not c.startswith('_'), - dir(yt_dlp.compat))) - {'unicode_literals'} - self.assertEqual(all_names, sorted(present_names)) - def test_compat_urllib_parse_unquote(self): self.assertEqual(compat_urllib_parse_unquote('abc%20def'), 'abc def') self.assertEqual(compat_urllib_parse_unquote('%7e/abc+def'), '~/abc+def') diff --git a/yt_dlp/compat.py b/yt_dlp/compat.py deleted file mode 100644 index f18c6cce28..0000000000 --- a/yt_dlp/compat.py +++ /dev/null @@ -1,302 +0,0 @@ -import asyncio -import base64 -import collections -import contextlib -import ctypes -import getpass -import html -import html.parser -import http -import http.client -import http.cookiejar -import http.cookies -import http.server -import itertools -import os -import re -import shlex -import shutil -import socket -import struct -import subprocess -import sys -import tokenize -import urllib -import xml.etree.ElementTree as etree -from subprocess import DEVNULL - - -# HTMLParseError has been deprecated in Python 3.3 and removed in -# Python 3.5. Introducing dummy exception for Python >3.5 for compatible -# and uniform cross-version exception handling -class compat_HTMLParseError(Exception): - pass - - -# compat_ctypes_WINFUNCTYPE = ctypes.WINFUNCTYPE -# will not work since ctypes.WINFUNCTYPE does not exist in UNIX machines -def compat_ctypes_WINFUNCTYPE(*args, **kwargs): - return ctypes.WINFUNCTYPE(*args, **kwargs) - - -class _TreeBuilder(etree.TreeBuilder): - def doctype(self, name, pubid, system): - pass - - -def compat_etree_fromstring(text): - return etree.XML(text, parser=etree.XMLParser(target=_TreeBuilder())) - - -compat_os_name = os._name if os.name == 'java' else os.name - - -if compat_os_name == 'nt': - def compat_shlex_quote(s): - return s if re.match(r'^[-_\w./]+$', s) else '"%s"' % s.replace('"', '\\"') -else: - from shlex import quote as compat_shlex_quote # noqa: F401 - - -def compat_ord(c): - return c if isinstance(c, int) else ord(c) - - -def compat_setenv(key, value, env=os.environ): - env[key] = value - - -if compat_os_name == 'nt' and sys.version_info < (3, 8): - # os.path.realpath on Windows does not follow symbolic links - # prior to Python 3.8 (see https://bugs.python.org/issue9949) - def compat_realpath(path): - while os.path.islink(path): - path = os.path.abspath(os.readlink(path)) - return path -else: - compat_realpath = os.path.realpath - - -try: - compat_Pattern = re.Pattern -except AttributeError: - compat_Pattern = type(re.compile('')) - - -try: - compat_Match = re.Match -except AttributeError: - compat_Match = type(re.compile('').match('')) - - -try: - compat_asyncio_run = asyncio.run # >= 3.7 -except AttributeError: - def compat_asyncio_run(coro): - try: - loop = asyncio.get_event_loop() - except RuntimeError: - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - loop.run_until_complete(coro) - - asyncio.run = compat_asyncio_run - - -try: # >= 3.7 - asyncio.tasks.all_tasks -except AttributeError: - asyncio.tasks.all_tasks = asyncio.tasks.Task.all_tasks - -try: - import websockets as compat_websockets -except ImportError: - compat_websockets = None - -# Python 3.8+ does not honor %HOME% on windows, but this breaks compatibility with youtube-dl -# See https://github.com/yt-dlp/yt-dlp/issues/792 -# https://docs.python.org/3/library/os.path.html#os.path.expanduser -if compat_os_name in ('nt', 'ce'): - def compat_expanduser(path): - HOME = os.environ.get('HOME') - if not HOME: - return os.path.expanduser(path) - elif not path.startswith('~'): - return path - i = path.replace('\\', '/', 1).find('/') # ~user - if i < 0: - i = len(path) - userhome = os.path.join(os.path.dirname(HOME), path[1:i]) if i > 1 else HOME - return userhome + path[i:] -else: - compat_expanduser = os.path.expanduser - - -try: - from Cryptodome.Cipher import AES as compat_pycrypto_AES -except ImportError: - try: - from Crypto.Cipher import AES as compat_pycrypto_AES - except ImportError: - compat_pycrypto_AES = None - -try: - import brotlicffi as compat_brotli -except ImportError: - try: - import brotli as compat_brotli - except ImportError: - compat_brotli = None - -WINDOWS_VT_MODE = False if compat_os_name == 'nt' else None - - -def windows_enable_vt_mode(): # TODO: Do this the proper way https://bugs.python.org/issue30075 - if compat_os_name != 'nt': - return - global WINDOWS_VT_MODE - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW - with contextlib.suppress(Exception): - subprocess.Popen('', shell=True, startupinfo=startupinfo).wait() - WINDOWS_VT_MODE = True - - -# Deprecated - -compat_b64decode = base64.b64decode -compat_chr = chr -compat_cookiejar = http.cookiejar -compat_cookiejar_Cookie = http.cookiejar.Cookie -compat_cookies_SimpleCookie = http.cookies.SimpleCookie -compat_get_terminal_size = shutil.get_terminal_size -compat_getenv = os.getenv -compat_getpass = getpass.getpass -compat_html_entities = html.entities -compat_html_entities_html5 = html.entities.html5 -compat_HTMLParser = html.parser.HTMLParser -compat_http_client = http.client -compat_http_server = http.server -compat_HTTPError = urllib.error.HTTPError -compat_itertools_count = itertools.count -compat_parse_qs = urllib.parse.parse_qs -compat_str = str -compat_struct_pack = struct.pack -compat_struct_unpack = struct.unpack -compat_tokenize_tokenize = tokenize.tokenize -compat_urllib_error = urllib.error -compat_urllib_parse_unquote = urllib.parse.unquote -compat_urllib_parse_unquote_plus = urllib.parse.unquote_plus -compat_urllib_parse_urlencode = urllib.parse.urlencode -compat_urllib_parse_urlparse = urllib.parse.urlparse -compat_urllib_request = urllib.request -compat_urlparse = compat_urllib_parse = urllib.parse - - -# To be removed - Do not use - -compat_basestring = str -compat_collections_abc = collections.abc -compat_cookies = http.cookies -compat_etree_Element = etree.Element -compat_etree_register_namespace = etree.register_namespace -compat_filter = filter -compat_input = input -compat_integer_types = (int, ) -compat_kwargs = lambda kwargs: kwargs -compat_map = map -compat_numeric_types = (int, float, complex) -compat_print = print -compat_shlex_split = shlex.split -compat_socket_create_connection = socket.create_connection -compat_Struct = struct.Struct -compat_subprocess_get_DEVNULL = lambda: DEVNULL -compat_urllib_parse_quote = urllib.parse.quote -compat_urllib_parse_quote_plus = urllib.parse.quote_plus -compat_urllib_parse_unquote_to_bytes = urllib.parse.unquote_to_bytes -compat_urllib_parse_urlunparse = urllib.parse.urlunparse -compat_urllib_request_DataHandler = urllib.request.DataHandler -compat_urllib_response = urllib.response -compat_urlretrieve = urllib.request.urlretrieve -compat_xml_parse_error = etree.ParseError -compat_xpath = lambda xpath: xpath -compat_zip = zip -workaround_optparse_bug9161 = lambda: None - - -# Set public objects - -__all__ = [ - 'WINDOWS_VT_MODE', - 'compat_HTMLParseError', - 'compat_HTMLParser', - 'compat_HTTPError', - 'compat_Match', - 'compat_Pattern', - 'compat_Struct', - 'compat_asyncio_run', - 'compat_b64decode', - 'compat_basestring', - 'compat_brotli', - 'compat_chr', - 'compat_collections_abc', - 'compat_cookiejar', - 'compat_cookiejar_Cookie', - 'compat_cookies', - 'compat_cookies_SimpleCookie', - 'compat_ctypes_WINFUNCTYPE', - 'compat_etree_Element', - 'compat_etree_fromstring', - 'compat_etree_register_namespace', - 'compat_expanduser', - 'compat_filter', - 'compat_get_terminal_size', - 'compat_getenv', - 'compat_getpass', - 'compat_html_entities', - 'compat_html_entities_html5', - 'compat_http_client', - 'compat_http_server', - 'compat_input', - 'compat_integer_types', - 'compat_itertools_count', - 'compat_kwargs', - 'compat_map', - 'compat_numeric_types', - 'compat_ord', - 'compat_os_name', - 'compat_parse_qs', - 'compat_print', - 'compat_pycrypto_AES', - 'compat_realpath', - 'compat_setenv', - 'compat_shlex_quote', - 'compat_shlex_split', - 'compat_socket_create_connection', - 'compat_str', - 'compat_struct_pack', - 'compat_struct_unpack', - 'compat_subprocess_get_DEVNULL', - 'compat_tokenize_tokenize', - 'compat_urllib_error', - 'compat_urllib_parse', - 'compat_urllib_parse_quote', - 'compat_urllib_parse_quote_plus', - 'compat_urllib_parse_unquote', - 'compat_urllib_parse_unquote_plus', - 'compat_urllib_parse_unquote_to_bytes', - 'compat_urllib_parse_urlencode', - 'compat_urllib_parse_urlparse', - 'compat_urllib_parse_urlunparse', - 'compat_urllib_request', - 'compat_urllib_request_DataHandler', - 'compat_urllib_response', - 'compat_urlparse', - 'compat_urlretrieve', - 'compat_websockets', - 'compat_xml_parse_error', - 'compat_xpath', - 'compat_zip', - 'windows_enable_vt_mode', - 'workaround_optparse_bug9161', -] diff --git a/yt_dlp/compat/__init__.py b/yt_dlp/compat/__init__.py new file mode 100644 index 0000000000..7a0e829926 --- /dev/null +++ b/yt_dlp/compat/__init__.py @@ -0,0 +1,129 @@ +import contextlib +import os +import subprocess +import sys +import types +import xml.etree.ElementTree as etree + +from . import re +from ._deprecated import * # noqa: F401, F403 + + +# HTMLParseError has been deprecated in Python 3.3 and removed in +# Python 3.5. Introducing dummy exception for Python >3.5 for compatible +# and uniform cross-version exception handling +class compat_HTMLParseError(Exception): + pass + + +class _TreeBuilder(etree.TreeBuilder): + def doctype(self, name, pubid, system): + pass + + +def compat_etree_fromstring(text): + return etree.XML(text, parser=etree.XMLParser(target=_TreeBuilder())) + + +compat_os_name = os._name if os.name == 'java' else os.name + + +if compat_os_name == 'nt': + def compat_shlex_quote(s): + return s if re.match(r'^[-_\w./]+$', s) else '"%s"' % s.replace('"', '\\"') +else: + from shlex import quote as compat_shlex_quote # noqa: F401 + + +def compat_ord(c): + return c if isinstance(c, int) else ord(c) + + +def compat_setenv(key, value, env=os.environ): + env[key] = value + + +if compat_os_name == 'nt' and sys.version_info < (3, 8): + # os.path.realpath on Windows does not follow symbolic links + # prior to Python 3.8 (see https://bugs.python.org/issue9949) + def compat_realpath(path): + while os.path.islink(path): + path = os.path.abspath(os.readlink(path)) + return path +else: + compat_realpath = os.path.realpath + + +try: + import websockets as compat_websockets +except ImportError: + compat_websockets = None + +# Python 3.8+ does not honor %HOME% on windows, but this breaks compatibility with youtube-dl +# See https://github.com/yt-dlp/yt-dlp/issues/792 +# https://docs.python.org/3/library/os.path.html#os.path.expanduser +if compat_os_name in ('nt', 'ce'): + def compat_expanduser(path): + HOME = os.environ.get('HOME') + if not HOME: + return os.path.expanduser(path) + elif not path.startswith('~'): + return path + i = path.replace('\\', '/', 1).find('/') # ~user + if i < 0: + i = len(path) + userhome = os.path.join(os.path.dirname(HOME), path[1:i]) if i > 1 else HOME + return userhome + path[i:] +else: + compat_expanduser = os.path.expanduser + + +try: + from Cryptodome.Cipher import AES as compat_pycrypto_AES +except ImportError: + try: + from Crypto.Cipher import AES as compat_pycrypto_AES + except ImportError: + compat_pycrypto_AES = None + +try: + import brotlicffi as compat_brotli +except ImportError: + try: + import brotli as compat_brotli + except ImportError: + compat_brotli = None + +WINDOWS_VT_MODE = False if compat_os_name == 'nt' else None + + +def windows_enable_vt_mode(): # TODO: Do this the proper way https://bugs.python.org/issue30075 + if compat_os_name != 'nt': + return + global WINDOWS_VT_MODE + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + with contextlib.suppress(Exception): + subprocess.Popen('', shell=True, startupinfo=startupinfo).wait() + WINDOWS_VT_MODE = True + + +class _PassthroughLegacy(types.ModuleType): + def __getattr__(self, attr): + import importlib + with contextlib.suppress(ImportError): + return importlib.import_module(f'.{attr}', __name__) + + legacy = importlib.import_module('._legacy', __name__) + if not hasattr(legacy, attr): + raise AttributeError(f'module {__name__} has no attribute {attr}') + + # XXX: Implement this the same way as other DeprecationWarnings without circular import + import warnings + warnings.warn(DeprecationWarning(f'{__name__}.{attr} is deprecated'), stacklevel=2) + return getattr(legacy, attr) + + +# Python 3.6 does not have module level __getattr__ +# https://peps.python.org/pep-0562/ +sys.modules[__name__].__class__ = _PassthroughLegacy diff --git a/yt_dlp/compat/_deprecated.py b/yt_dlp/compat/_deprecated.py new file mode 100644 index 0000000000..f844398254 --- /dev/null +++ b/yt_dlp/compat/_deprecated.py @@ -0,0 +1,47 @@ +"""Deprecated - New code should avoid these""" + +import base64 +import getpass +import html +import html.parser +import http +import http.client +import http.cookiejar +import http.cookies +import http.server +import itertools +import os +import shutil +import struct +import tokenize +import urllib + +compat_b64decode = base64.b64decode +compat_chr = chr +compat_cookiejar = http.cookiejar +compat_cookiejar_Cookie = http.cookiejar.Cookie +compat_cookies_SimpleCookie = http.cookies.SimpleCookie +compat_get_terminal_size = shutil.get_terminal_size +compat_getenv = os.getenv +compat_getpass = getpass.getpass +compat_html_entities = html.entities +compat_html_entities_html5 = html.entities.html5 +compat_HTMLParser = html.parser.HTMLParser +compat_http_client = http.client +compat_http_server = http.server +compat_HTTPError = urllib.error.HTTPError +compat_itertools_count = itertools.count +compat_parse_qs = urllib.parse.parse_qs +compat_str = str +compat_struct_pack = struct.pack +compat_struct_unpack = struct.unpack +compat_tokenize_tokenize = tokenize.tokenize +compat_urllib_error = urllib.error +compat_urllib_parse_unquote = urllib.parse.unquote +compat_urllib_parse_unquote_plus = urllib.parse.unquote_plus +compat_urllib_parse_urlencode = urllib.parse.urlencode +compat_urllib_parse_urlparse = urllib.parse.urlparse +compat_urllib_request = urllib.request +compat_urlparse = compat_urllib_parse = urllib.parse + +__all__ = [x for x in globals() if x.startswith('compat_')] diff --git a/yt_dlp/compat/_legacy.py b/yt_dlp/compat/_legacy.py new file mode 100644 index 0000000000..f185b7e2fb --- /dev/null +++ b/yt_dlp/compat/_legacy.py @@ -0,0 +1,54 @@ +""" Do not use! """ + +import collections +import ctypes +import http +import http.client +import http.cookiejar +import http.cookies +import http.server +import shlex +import socket +import struct +import urllib +import xml.etree.ElementTree as etree +from subprocess import DEVNULL + +from .asyncio import run as compat_asyncio_run # noqa: F401 +from .re import Pattern as compat_Pattern # noqa: F401 +from .re import match as compat_Match # noqa: F401 + + +# compat_ctypes_WINFUNCTYPE = ctypes.WINFUNCTYPE +# will not work since ctypes.WINFUNCTYPE does not exist in UNIX machines +def compat_ctypes_WINFUNCTYPE(*args, **kwargs): + return ctypes.WINFUNCTYPE(*args, **kwargs) + + +compat_basestring = str +compat_collections_abc = collections.abc +compat_cookies = http.cookies +compat_etree_Element = etree.Element +compat_etree_register_namespace = etree.register_namespace +compat_filter = filter +compat_input = input +compat_integer_types = (int, ) +compat_kwargs = lambda kwargs: kwargs +compat_map = map +compat_numeric_types = (int, float, complex) +compat_print = print +compat_shlex_split = shlex.split +compat_socket_create_connection = socket.create_connection +compat_Struct = struct.Struct +compat_subprocess_get_DEVNULL = lambda: DEVNULL +compat_urllib_parse_quote = urllib.parse.quote +compat_urllib_parse_quote_plus = urllib.parse.quote_plus +compat_urllib_parse_unquote_to_bytes = urllib.parse.unquote_to_bytes +compat_urllib_parse_urlunparse = urllib.parse.urlunparse +compat_urllib_request_DataHandler = urllib.request.DataHandler +compat_urllib_response = urllib.response +compat_urlretrieve = urllib.request.urlretrieve +compat_xml_parse_error = etree.ParseError +compat_xpath = lambda xpath: xpath +compat_zip = zip +workaround_optparse_bug9161 = lambda: None diff --git a/yt_dlp/compat/asyncio/__init__.py b/yt_dlp/compat/asyncio/__init__.py new file mode 100644 index 0000000000..0e8c6cad3f --- /dev/null +++ b/yt_dlp/compat/asyncio/__init__.py @@ -0,0 +1,16 @@ +# flake8: noqa: F405 + +from asyncio import * # noqa: F403 + +from . import tasks # noqa: F401 + +try: + run # >= 3.7 +except NameError: + def run(coro): + try: + loop = get_event_loop() + except RuntimeError: + loop = new_event_loop() + set_event_loop(loop) + loop.run_until_complete(coro) diff --git a/yt_dlp/compat/asyncio/tasks.py b/yt_dlp/compat/asyncio/tasks.py new file mode 100644 index 0000000000..cb31e52fab --- /dev/null +++ b/yt_dlp/compat/asyncio/tasks.py @@ -0,0 +1,8 @@ +# flake8: noqa: F405 + +from asyncio.tasks import * # noqa: F403 + +try: # >= 3.7 + all_tasks +except NameError: + all_tasks = Task.all_tasks diff --git a/yt_dlp/compat/re.py b/yt_dlp/compat/re.py new file mode 100644 index 0000000000..e8a6fabbda --- /dev/null +++ b/yt_dlp/compat/re.py @@ -0,0 +1,14 @@ +# flake8: noqa: F405 + +from re import * # F403 + +try: + Pattern # >= 3.7 +except NameError: + Pattern = type(compile('')) + + +try: + Match # >= 3.7 +except NameError: + Match = type(compile('').match('')) diff --git a/yt_dlp/downloader/websocket.py b/yt_dlp/downloader/websocket.py index 6b190cd90a..8465f97135 100644 --- a/yt_dlp/downloader/websocket.py +++ b/yt_dlp/downloader/websocket.py @@ -1,4 +1,3 @@ -import asyncio import contextlib import os import signal @@ -15,6 +14,7 @@ else: from .common import FileDownloader from .external import FFmpegFD +from ..compat import asyncio class FFmpegSinkFD(FileDownloader): diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index 10b297708b..3ee5e257c6 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -7,7 +7,6 @@ import math import netrc import os import random -import re import sys import time import xml.etree.ElementTree @@ -20,13 +19,13 @@ from ..compat import ( compat_getpass, compat_http_client, compat_os_name, - compat_Pattern, compat_str, compat_urllib_error, compat_urllib_parse_unquote, compat_urllib_parse_urlencode, compat_urllib_request, compat_urlparse, + re, ) from ..downloader import FileDownloader from ..downloader.f4m import get_base_url, remove_encrypted_media @@ -1198,7 +1197,7 @@ class InfoExtractor: """ if string is None: mobj = None - elif isinstance(pattern, (str, compat_Pattern)): + elif isinstance(pattern, (str, re.Pattern)): mobj = re.search(pattern, string, flags) else: for p in pattern: diff --git a/yt_dlp/webvtt.py b/yt_dlp/webvtt.py index 741622b25b..4c222ba8e1 100644 --- a/yt_dlp/webvtt.py +++ b/yt_dlp/webvtt.py @@ -9,9 +9,8 @@ in RFC 8216 ยง3.5 . """ import io -import re -from .compat import compat_Match, compat_Pattern +from .compat import re from .utils import int_or_none, timetuple_from_msec @@ -26,7 +25,7 @@ class _MatchParser: self._pos = 0 def match(self, r): - if isinstance(r, compat_Pattern): + if isinstance(r, re.Pattern): return r.match(self._data, self._pos) if isinstance(r, str): if self._data.startswith(r, self._pos): @@ -37,7 +36,7 @@ class _MatchParser: def advance(self, by): if by is None: amt = 0 - elif isinstance(by, compat_Match): + elif isinstance(by, re.Match): amt = len(by.group(0)) elif isinstance(by, str): amt = len(by)