Allow users to specify encoding in each config files (#4357)

Authored by: Lesmiscore

README.md

@@ -1161,6 +1161,15 @@ Note that options in configuration file are just the same options aka switches u
 
 You can use `--ignore-config` if you want to disable all configuration files for a particular yt-dlp run. If `--ignore-config` is found inside any configuration file, no further configuration will be loaded. For example, having the option in the portable configuration file prevents loading of home, user, and system configurations. Additionally, (for backward compatibility) if `--ignore-config` is found inside the system configuration file, the user configuration is not loaded.
 
+### Specifying encoding of config files
+
+By default, config files are read in the encoding from system locale.
+If you saved your config file in a different encoding than that, you may write `# coding: ENCODING` to the beginning of the file. (e.g. `# coding: shift-jis`)
+
+There must not be any characters before that, including spaces.
+
+If you have BOM enabled, it will be used instead.
+
 ### Authentication with `.netrc` file
 
 You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every yt-dlp execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](https://stackoverflow.com/tags/.netrc/info) on a per extractor basis. For that you will need to create a `.netrc` file in `--netrc-location` and restrict permissions to read/write by only you:

test/test_utils.py

@@ -39,6 +39,7 @@ from yt_dlp.utils import (
     datetime_from_str,
     detect_exe_version,
     determine_ext,
+    determine_file_encoding,
     dfxp2srt,
     dict_get,
     encode_base_n,
@@ -1822,6 +1823,33 @@ Line 1
             with contextlib.suppress(OSError):
                 os.remove(FILE)
 
+    def test_determine_file_encoding(self):
+        self.assertEqual(determine_file_encoding(b''), (None, 0))
+        self.assertEqual(determine_file_encoding(b'--verbose -x --audio-format mkv\n'), (None, 0))
+
+        self.assertEqual(determine_file_encoding(b'\xef\xbb\xbf'), ('utf-8', 3))
+        self.assertEqual(determine_file_encoding(b'\x00\x00\xfe\xff'), ('utf-32-be', 4))
+        self.assertEqual(determine_file_encoding(b'\xff\xfe'), ('utf-16-le', 2))
+
+        self.assertEqual(determine_file_encoding(b'# -*- coding: cp932 -*-'), ('cp932', 0))
+        self.assertEqual(determine_file_encoding(b'# -*- coding: cp932 -*-\n'), ('cp932', 0))
+        self.assertEqual(determine_file_encoding(b'# -*- coding: cp932 -*-\r\n'), ('cp932', 0))
+
+        self.assertEqual(determine_file_encoding(b'# coding: utf-8\n--verbose'), ('utf-8', 0))
+        self.assertEqual(determine_file_encoding(b'# coding: someencodinghere-12345\n--verbose'), ('someencodinghere-12345', 0))
+
+        self.assertEqual(determine_file_encoding(b'# vi: set fileencoding=cp932'), ('cp932', 0))
+        self.assertEqual(determine_file_encoding(b'# vi: set fileencoding=cp932\n'), ('cp932', 0))
+        self.assertEqual(determine_file_encoding(b'# vi: set fileencoding=cp932\r\n'), ('cp932', 0))
+        self.assertEqual(determine_file_encoding(b'# vi: set fileencoding=cp932,euc-jp\r\n'), ('cp932', 0))
+
+        self.assertEqual(determine_file_encoding(
+            b'\0\0\0#\0\0\0 \0\0\0c\0\0\0o\0\0\0d\0\0\0i\0\0\0n\0\0\0g\0\0\0:\0\0\0 \0\0\0u\0\0\0t\0\0\0f\0\0\0-\0\0\x003\0\0\x002\0\0\0-\0\0\0b\0\0\0e'),
+            ('utf-32-be', 0))
+        self.assertEqual(determine_file_encoding(
+            b'#\0 \0c\0o\0d\0i\0n\0g\0:\0 \0u\0t\0f\0-\x001\x006\0-\0l\0e\0'),
+            ('utf-16-le', 0))
+
 
 if __name__ == '__main__':
     unittest.main()

yt_dlp/extractor/_extractors.py

@@ -1794,6 +1794,7 @@ from .trueid import TrueIDIE
 from .trunews import TruNewsIE
 from .trutv import TruTVIE
 from .tube8 import Tube8IE
+from .tubetugraz import TubeTuGrazIE, TubeTuGrazSeriesIE
 from .tubitv import (
     TubiTvIE,
     TubiTvShowIE,

yt_dlp/extractor/common.py

@@ -929,39 +929,37 @@ class InfoExtractor:
 
         return content
 
-    def _parse_xml(self, xml_string, video_id, transform_source=None, fatal=True):
+    def __print_error(self, errnote, fatal, video_id, err):
+        if fatal:
+            raise ExtractorError(f'{video_id}: {errnote} ', cause=err)
+        elif errnote:
+            self.report_warning(f'{video_id}: {errnote} {err}')
+
+    def _parse_xml(self, xml_string, video_id, transform_source=None, fatal=True, errnote=None):
         if transform_source:
             xml_string = transform_source(xml_string)
         try:
             return compat_etree_fromstring(xml_string.encode('utf-8'))
         except xml.etree.ElementTree.ParseError as ve:
-            errmsg = '%s: Failed to parse XML ' % video_id
+            self.__print_error('Failed to parse XML' if errnote is None else errnote, fatal, video_id, ve)
-            if fatal:
-                raise ExtractorError(errmsg, cause=ve)
-            else:
-                self.report_warning(errmsg + str(ve))
 
-    def _parse_json(self, json_string, video_id, transform_source=None, fatal=True, **parser_kwargs):
+    def _parse_json(self, json_string, video_id, transform_source=None, fatal=True, errnote=None, **parser_kwargs):
         try:
             return json.loads(
                 json_string, cls=LenientJSONDecoder, strict=False, transform_source=transform_source, **parser_kwargs)
         except ValueError as ve:
-            errmsg = f'{video_id}: Failed to parse JSON'
+            self.__print_error('Failed to parse JSON' if errnote is None else errnote, fatal, video_id, ve)
-            if fatal:
-                raise ExtractorError(errmsg, cause=ve)
-            else:
-                self.report_warning(f'{errmsg}: {ve}')
 
-    def _parse_socket_response_as_json(self, data, video_id, transform_source=None, fatal=True):
+    def _parse_socket_response_as_json(self, data, *args, **kwargs):
-        return self._parse_json(
+        return self._parse_json(data[data.find('{'):data.rfind('}') + 1], *args, **kwargs)
-            data[data.find('{'):data.rfind('}') + 1],
-            video_id, transform_source, fatal)
 
     def __create_download_methods(name, parser, note, errnote, return_value):
 
-        def parse(ie, content, *args, **kwargs):
+        def parse(ie, content, *args, errnote=errnote, **kwargs):
             if parser is None:
                 return content
+            if errnote is False:
+                kwargs['errnote'] = errnote
             # parser is fetched by name so subclasses can override it
             return getattr(ie, parser)(content, *args, **kwargs)
 
@@ -973,7 +971,7 @@ class InfoExtractor:
             if res is False:
                 return res
             content, urlh = res
-            return parse(self, content, video_id, transform_source=transform_source, fatal=fatal), urlh
+            return parse(self, content, video_id, transform_source=transform_source, fatal=fatal, errnote=errnote), urlh
 
         def download_content(self, url_or_request, video_id, note=note, errnote=errnote, transform_source=None,
                              fatal=True, encoding=None, data=None, headers={}, query={}, expected_status=None):
@@ -988,7 +986,7 @@ class InfoExtractor:
                     self.report_warning(f'Unable to load request from disk: {e}')
                 else:
                     content = self.__decode_webpage(webpage_bytes, encoding, url_or_request.headers)
-                    return parse(self, content, video_id, transform_source, fatal)
+                    return parse(self, content, video_id, transform_source=transform_source, fatal=fatal, errnote=errnote)
             kwargs = {
                 'note': note,
                 'errnote': errnote,

yt_dlp/extractor/tubetugraz.py

@@ -0,0 +1,234 @@
+from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    parse_resolution,
+    traverse_obj,
+    urlencode_postdata,
+    variadic,
+)
+
+
+class TubeTuGrazBaseIE(InfoExtractor):
+    _NETRC_MACHINE = 'tubetugraz'
+
+    _API_EPISODE = 'https://tube.tugraz.at/search/episode.json'
+    _FORMAT_TYPES = ('presentation', 'presenter')
+
+    def _perform_login(self, username, password):
+        urlh = self._request_webpage(
+            'https://tube.tugraz.at/Shibboleth.sso/Login?target=/paella/ui/index.html',
+            None, fatal=False, note='downloading login page', errnote='unable to fetch login page')
+        if not urlh:
+            return
+
+        urlh = self._request_webpage(
+            urlh.geturl(), None, fatal=False, headers={'referer': urlh.geturl()},
+            note='logging in', errnote='unable to log in', data=urlencode_postdata({
+                'lang': 'de',
+                '_eventId_proceed': '',
+                'j_username': username,
+                'j_password': password
+            }))
+
+        if urlh and urlh.geturl() != 'https://tube.tugraz.at/paella/ui/index.html':
+            self.report_warning('unable to login: incorrect password')
+
+    def _extract_episode(self, episode_info):
+        id = episode_info.get('id')
+        formats = list(self._extract_formats(
+            traverse_obj(episode_info, ('mediapackage', 'media', 'track')), id))
+        self._sort_formats(formats)
+
+        title = traverse_obj(episode_info, ('mediapackage', 'title'), 'dcTitle')
+        series_title = traverse_obj(episode_info, ('mediapackage', 'seriestitle'))
+        creator = ', '.join(variadic(traverse_obj(
+            episode_info, ('mediapackage', 'creators', 'creator'), 'dcCreator', default='')))
+        return {
+            'id': id,
+            'title': title,
+            'creator': creator or None,
+            'duration': traverse_obj(episode_info, ('mediapackage', 'duration'), 'dcExtent'),
+            'series': series_title,
+            'series_id': traverse_obj(episode_info, ('mediapackage', 'series'), 'dcIsPartOf'),
+            'episode': series_title and title,
+            'formats': formats
+        }
+
+    def _set_format_type(self, formats, type):
+        for f in formats:
+            f['format_note'] = type
+            if not type.startswith(self._FORMAT_TYPES[0]):
+                f['preference'] = -2
+        return formats
+
+    def _extract_formats(self, format_list, id):
+        has_hls, has_dash = False, False
+
+        for format_info in format_list or []:
+            url = traverse_obj(format_info, ('tags', 'url'), 'url')
+            if url is None:
+                continue
+
+            type = format_info.get('type') or 'unknown'
+            transport = (format_info.get('transport') or 'https').lower()
+
+            if transport == 'https':
+                formats = [{
+                    'url': url,
+                    'abr': float_or_none(traverse_obj(format_info, ('audio', 'bitrate')), 1000),
+                    'vbr': float_or_none(traverse_obj(format_info, ('video', 'bitrate')), 1000),
+                    'fps': traverse_obj(format_info, ('video', 'framerate')),
+                    **parse_resolution(traverse_obj(format_info, ('video', 'resolution'))),
+                }]
+            elif transport == 'hls':
+                has_hls, formats = True, self._extract_m3u8_formats(
+                    url, id, 'mp4', fatal=False, note=f'downloading {type} HLS manifest')
+            elif transport == 'dash':
+                has_dash, formats = True, self._extract_mpd_formats(
+                    url, id, fatal=False, note=f'downloading {type} DASH manifest')
+            else:
+                # RTMP, HDS, SMOOTH, and unknown formats
+                # - RTMP url fails on every tested entry until now
+                # - HDS url 404's on every tested entry until now
+                # - SMOOTH url 404's on every tested entry until now
+                continue
+
+            yield from self._set_format_type(formats, type)
+
+        # TODO: Add test for these
+        for type in self._FORMAT_TYPES:
+            if not has_hls:
+                hls_formats = self._extract_m3u8_formats(
+                    f'https://wowza.tugraz.at/matterhorn_engage/smil:engage-player_{id}_{type}.smil/playlist.m3u8',
+                    id, 'mp4', fatal=False, note=f'Downloading {type} HLS manifest', errnote=False) or []
+                yield from self._set_format_type(hls_formats, type)
+
+            if not has_dash:
+                dash_formats = self._extract_mpd_formats(
+                    f'https://wowza.tugraz.at/matterhorn_engage/smil:engage-player_{id}_{type}.smil/manifest_mpm4sav_mvlist.mpd',
+                    id, fatal=False, note=f'Downloading {type} DASH manifest', errnote=False)
+                yield from self._set_format_type(dash_formats, type)
+
+
+class TubeTuGrazIE(TubeTuGrazBaseIE):
+    IE_DESC = 'tube.tugraz.at'
+
+    _VALID_URL = r'''(?x)
+        https?://tube\.tugraz\.at/paella/ui/watch.html\?id=
+        (?P<id>[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12})
+    '''
+    _TESTS = [
+        {
+            'url': 'https://tube.tugraz.at/paella/ui/watch.html?id=f2634392-e40e-4ac7-9ddc-47764aa23d40',
+            'md5': 'a23a3d5c9aaca2b84932fdba66e17145',
+            'info_dict': {
+                'id': 'f2634392-e40e-4ac7-9ddc-47764aa23d40',
+                'ext': 'mp4',
+                'title': '#6 (23.11.2017)',
+                'episode': '#6 (23.11.2017)',
+                'series': '[INB03001UF] Einführung in die strukturierte Programmierung',
+                'creator': 'Safran C',
+                'duration': 3295818,
+                'series_id': 'b1192fff-2aa7-4bf0-a5cf-7b15c3bd3b34',
+            }
+        }, {
+            'url': 'https://tube.tugraz.at/paella/ui/watch.html?id=2df6d787-e56a-428d-8ef4-d57f07eef238',
+            'md5': 'de0d854a56bf7318d2b693fe1adb89a5',
+            'info_dict': {
+                'id': '2df6d787-e56a-428d-8ef4-d57f07eef238',
+                'title': 'TubeTuGraz video #2df6d787-e56a-428d-8ef4-d57f07eef238',
+                'ext': 'mp4',
+            },
+            'expected_warnings': ['Extractor failed to obtain "title"'],
+        }
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        episode_data = self._download_json(
+            self._API_EPISODE, video_id, query={'id': video_id, 'limit': 1}, note='Downloading episode metadata')
+
+        episode_info = traverse_obj(episode_data, ('search-results', 'result'), default={'id': video_id})
+        return self._extract_episode(episode_info)
+
+
+class TubeTuGrazSeriesIE(TubeTuGrazBaseIE):
+    _VALID_URL = r'''(?x)
+        https?://tube\.tugraz\.at/paella/ui/browse\.html\?series=
+        (?P<id>[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12})
+    '''
+    _TESTS = [{
+        'url': 'https://tube.tugraz.at/paella/ui/browse.html?series=0e6351b7-c372-491e-8a49-2c9b7e21c5a6',
+        'id': '0e6351b7-c372-491e-8a49-2c9b7e21c5a6',
+        'info_dict': {
+            'id': '0e6351b7-c372-491e-8a49-2c9b7e21c5a6',
+            'title': '[209351] Strassenwesen',
+        },
+        'playlist': [
+            {
+                'info_dict': {
+                    'id': 'ee17ce5d-34e2-48b7-a76a-fed148614e11',
+                    'series_id': '0e6351b7-c372-491e-8a49-2c9b7e21c5a6',
+                    'ext': 'mp4',
+                    'title': '#4 Detailprojekt',
+                    'episode': '#4 Detailprojekt',
+                    'series': '[209351] Strassenwesen',
+                    'creator': 'Neuhold R',
+                    'duration': 6127024,
+                }
+            },
+            {
+                'info_dict': {
+                    'id': '87350498-799a-44d3-863f-d1518a98b114',
+                    'series_id': '0e6351b7-c372-491e-8a49-2c9b7e21c5a6',
+                    'ext': 'mp4',
+                    'title': '#3 Generelles Projekt',
+                    'episode': '#3 Generelles Projekt',
+                    'series': '[209351] Strassenwesen',
+                    'creator': 'Neuhold R',
+                    'duration': 5374422,
+                }
+            },
+            {
+                'info_dict': {
+                    'id': '778599ea-489e-4189-9e05-3b4888e19bcd',
+                    'series_id': '0e6351b7-c372-491e-8a49-2c9b7e21c5a6',
+                    'ext': 'mp4',
+                    'title': '#2 Vorprojekt',
+                    'episode': '#2 Vorprojekt',
+                    'series': '[209351] Strassenwesen',
+                    'creator': 'Neuhold R',
+                    'duration': 5566404,
+                }
+            },
+            {
+                'info_dict': {
+                    'id': '75e4c71c-d99d-4e56-b0e6-4f2bcdf11f29',
+                    'series_id': '0e6351b7-c372-491e-8a49-2c9b7e21c5a6',
+                    'ext': 'mp4',
+                    'title': '#1 Variantenstudium',
+                    'episode': '#1 Variantenstudium',
+                    'series': '[209351] Strassenwesen',
+                    'creator': 'Neuhold R',
+                    'duration': 5420200,
+                }
+            }
+        ],
+        'min_playlist_count': 4
+    }]
+
+    def _real_extract(self, url):
+        id = self._match_id(url)
+        episodes_data = self._download_json(self._API_EPISODE, id, query={'sid': id}, note='Downloading episode list')
+        series_data = self._download_json(
+            'https://tube.tugraz.at/series/series.json', id, fatal=False,
+            note='downloading series metadata', errnote='failed to download series metadata',
+            query={
+                'seriesId': id,
+                'count': 1,
+                'sort': 'TITLE'
+            })
+
+        return self.playlist_result(
+            map(self._extract_episode, episodes_data['search-results']['result']), id,
+            traverse_obj(series_data, ('catalogs', 0, 'http://purl.org/dc/terms/', 'title', 0, 'value')))

yt_dlp/utils.py

@@ -3485,17 +3485,19 @@ def age_restricted(content_limit, age_limit):
     return age_limit < content_limit
 
 
+BOMS = [
+    (b'\xef\xbb\xbf', 'utf-8'),
+    (b'\x00\x00\xfe\xff', 'utf-32-be'),
+    (b'\xff\xfe\x00\x00', 'utf-32-le'),
+    (b'\xff\xfe', 'utf-16-le'),
+    (b'\xfe\xff', 'utf-16-be'),
+]
+""" List of known byte-order-marks (BOM) """
+
+
 def is_html(first_bytes):
     """ Detect whether a file contains HTML by examining its first bytes. """
 
-    BOMS = [
-        (b'\xef\xbb\xbf', 'utf-8'),
-        (b'\x00\x00\xfe\xff', 'utf-32-be'),
-        (b'\xff\xfe\x00\x00', 'utf-32-le'),
-        (b'\xff\xfe', 'utf-16-le'),
-        (b'\xfe\xff', 'utf-16-be'),
-    ]
-
     encoding = 'utf-8'
     for bom, enc in BOMS:
         while first_bytes.startswith(bom):
@@ -5394,6 +5396,41 @@ def read_stdin(what):
     return sys.stdin
 
 
+def determine_file_encoding(data):
+    """
+    From the first 512 bytes of a given file,
+    it tries to detect the encoding to be used to read as text.
+
+    @returns (encoding, bytes to skip)
+    """
+
+    for bom, enc in BOMS:
+        # matching BOM beats any declaration
+        # BOMs are skipped to prevent any errors
+        if data.startswith(bom):
+            return enc, len(bom)
+
+    # strip off all null bytes to match even when UTF-16 or UTF-32 is used
+    # endians don't matter
+    data = data.replace(b'\0', b'')
+
+    PREAMBLES = [
+        # "# -*- coding: utf-8 -*-"
+        # "# coding: utf-8"
+        rb'(?m)^#(?:\s+-\*-)?\s*coding\s*:\s*(?P<encoding>\S+)(?:\s+-\*-)?\s*$',
+        # "# vi: set fileencoding=utf-8"
+        rb'^#\s+vi\s*:\s+set\s+fileencoding=(?P<encoding>[^\s,]+)'
+    ]
+    for pb in PREAMBLES:
+        mobj = re.match(pb, data)
+        if not mobj:
+            continue
+        # preambles aren't skipped since they're just ignored when reading as config
+        return mobj.group('encoding').decode(), 0
+
+    return None, 0
+
+
 class Config:
     own_args = None
     parsed_args = None
@@ -5445,12 +5482,17 @@ class Config:
     @staticmethod
     def read_file(filename, default=[]):
         try:
-            optionf = open(filename)
+            optionf = open(filename, 'rb')
         except OSError:
             return default  # silently skip if file is not present
+        try:
+            enc, skip = determine_file_encoding(optionf.read(512))
+            optionf.seek(skip, io.SEEK_SET)
+        except OSError:
+            enc = None  # silently skip read errors
         try:
             # FIXME: https://github.com/ytdl-org/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56
-            contents = optionf.read()
+            contents = optionf.read().decode(enc or preferredencoding())
             res = shlex.split(contents, comments=True)
         except Exception as err:
             raise ValueError(f'Unable to parse "{filename}": {err}')