[version] update

Created by: pukkandan

:ci skip all :ci run dl

.github/ISSUE_TEMPLATE/1_broken_site.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm reporting a broken site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.06.22.1** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.06.29** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -51,12 +51,12 @@ body:
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2022.06.22.1 (exe)
+        [debug] yt-dlp version 2022.06.29 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2022.06.22.1)
+        yt-dlp is up to date (2022.06.29)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/2_site_support_request.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.06.22.1** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.06.29** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -62,12 +62,12 @@ body:
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2022.06.22.1 (exe)
+        [debug] yt-dlp version 2022.06.29 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2022.06.22.1)
+        yt-dlp is up to date (2022.06.29)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/3_site_feature_request.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm requesting a site-specific feature
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.06.22.1** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.06.29** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -60,12 +60,12 @@ body:
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2022.06.22.1 (exe)
+        [debug] yt-dlp version 2022.06.29 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2022.06.22.1)
+        yt-dlp is up to date (2022.06.29)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/4_bug_report.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.06.22.1** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.06.29** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -45,12 +45,12 @@ body:
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2022.06.22.1 (exe)
+        [debug] yt-dlp version 2022.06.29 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2022.06.22.1)
+        yt-dlp is up to date (2022.06.29)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/5_feature_request.yml

@@ -13,7 +13,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.06.22.1** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.06.29** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues including closed ones. DO NOT post duplicates
           required: true

.github/ISSUE_TEMPLATE/6_question.yml

@@ -13,7 +13,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.06.22.1** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.06.29** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions including closed ones. DO NOT post duplicates
           required: true

.github/workflows/build.yml

@@ -8,6 +8,7 @@ jobs:
       version_suffix: ${{ steps.version_suffix.outputs.version_suffix }}
       ytdlp_version: ${{ steps.bump_version.outputs.ytdlp_version }}
       upload_url: ${{ steps.create_release.outputs.upload_url }}
+      release_id: ${{ steps.create_release.outputs.id }}
     steps:
     - uses: actions/checkout@v2
       with:
@@ -58,15 +59,19 @@ jobs:
         tag_name: ${{ steps.bump_version.outputs.ytdlp_version }}
         release_name: yt-dlp ${{ steps.bump_version.outputs.ytdlp_version }}
         commitish: ${{ steps.push_release.outputs.head_sha }}
+        draft: true
+        prerelease: false
         body: |
           #### [A description of the various files]((https://github.com/yt-dlp/yt-dlp#release-files)) are in the README
 
           ---
+          <details open><summary><h3>Changelog</summary>
+          <p>
 
-          ### Changelog:
           ${{ env.changelog }}
-        draft: false
+
-        prerelease: false
+          </p>
+          </details>
 
 
   build_unix:
@@ -443,3 +448,24 @@ jobs:
         asset_path: ./SHA2-512SUMS
         asset_name: SHA2-512SUMS
         asset_content_type: text/plain
+
+    - name: Make Update spec
+      run: |
+        echo "# This file is used for regulating self-update" >> _update_spec
+    - name: Upload update spec
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.create_release.outputs.upload_url }}
+        asset_path: ./_update_spec
+        asset_name: _update_spec
+        asset_content_type: text/plain
+
+    - name: Finalize release
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        gh api -X PATCH -H "Accept: application/vnd.github.v3+json" \
+          /repos/${{ github.repository }}/releases/${{ needs.create_release.outputs.release_id }} \
+          -F draft=false

CONTRIBUTORS

@@ -267,3 +267,8 @@ sqrtNOT
 bubbleguuum
 darkxex
 miseran
+StefanLobbenmeier
+crazymoose77756
+nomevi
+Brett824
+pingiun

Changelog.md

@@ -11,6 +11,45 @@
 -->
 
 
+### 2022.06.29
+
+* Fix `--downloader native`
+* Fix `section_end` of clips
+* Fix playlist error handling
+* Sanitize `chapters`
+* [extractor] Fix `_create_request` when headers is None
+* [extractor] Fix empty `BaseURL` in MPD
+* [ffmpeg] Write full output to debug on error
+* [hls] Warn user when trying to download live HLS
+* [options] Fix `parse_known_args` for `--`
+* [utils] Fix inconsistent default handling between HTTP and HTTPS requests by [coletdjnz](https://github.com/coletdjnz)
+* [build] Draft release until complete
+* [build] Fix release tag commit
+* [build] Standalone x64 builds for MacOS 10.9 by [StefanLobbenmeier](https://github.com/StefanLobbenmeier)
+* [update] Ability to set a maximum version for specific variants
+* [compat] Fix `compat.WINDOWS_VT_MODE`
+* [compat] Remove deprecated functions from core code
+* [compat] Remove more functions
+* [cleanup, extractor] Reduce direct use of `_downloader`
+* [cleanup] Consistent style for file heads
+* [cleanup] Fix some typos by [crazymoose77756](https://github.com/crazymoose77756)
+* [cleanup] Misc fixes and cleanup
+* [extractor/Scrolller] Add extractor by [LunarFang416](https://github.com/LunarFang416)
+* [extractor/ViMP] Add playlist extractor by [FestplattenSchnitzel](https://github.com/FestplattenSchnitzel)
+* [extractor/fuyin] Add extractor by [HobbyistDev](https://github.com/HobbyistDev)
+* [extractor/livestreamfails] Add extractor by [nomevi](https://github.com/nomevi)
+* [extractor/premiershiprugby] Add extractor by [HobbyistDev](https://github.com/HobbyistDev)
+* [extractor/steam] Add broadcast extractor by [HobbyistDev](https://github.com/HobbyistDev)
+* [extractor/youtube] Mark videos as fully watched by [Brett824](https://github.com/Brett824)
+* [extractor/CWTV] Extract thumbnail by [ischmidt20](https://github.com/ischmidt20)
+* [extractor/ViMP] Add thumbnail and support more sites by [FestplattenSchnitzel](https://github.com/FestplattenSchnitzel)
+* [extractor/dropout] Support cookies and login only as needed by [pingiun](https://github.com/pingiun), [pukkandan](https://github.com/pukkandan)
+* [extractor/ertflix] Improve `_VALID_URL`
+* [extractor/lbry] Use HEAD request for redirect URL by [flashdagger](https://github.com/flashdagger)
+* [extractor/mediaset] Improve `_VALID_URL`
+* [extractor/npr] Implement [e50c350](https://github.com/yt-dlp/yt-dlp/commit/e50c3500b43d80e4492569c4b4523c4379c6fbb2) differently
+* [extractor/tennistv] Rewrite extractor by [pukkandan](https://github.com/pukkandan), [zenerdi0de](https://github.com/zenerdi0de)
+
 ### 2022.06.22.1
 
 * [build] Fix updating homebrew formula

README.md

@@ -71,7 +71,7 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
 
 # NEW FEATURES
 
-* Based on **youtube-dl 2021.12.17 [commit/8a158a9](https://github.com/ytdl-org/youtube-dl/commit/8a158a936c8b002ef536e9e2b778ded02c09c0fa)**<!--([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))--> and **youtube-dlc 2020.11.11-3 [commit/f9401f2](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee)**: You get all the features and patches of [youtube-dlc](https://github.com/blackjack4494/yt-dlc) in addition to the latest [youtube-dl](https://github.com/ytdl-org/youtube-dl)
+* Merged with **youtube-dl v2021.12.17+ [commit/a03b977](https://github.com/ytdl-org/youtube-dl/commit/a03b9775d544b06a5b4f2aa630214c7c22fc2229)**<!--([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))--> and **youtube-dlc v2020.11.11-3+ [commit/f9401f2](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee)**: You get all the features and patches of [youtube-dlc](https://github.com/blackjack4494/yt-dlc) in addition to the latest [youtube-dl](https://github.com/ytdl-org/youtube-dl)
 
 * **[SponsorBlock Integration](#sponsorblock-options)**: You can mark/remove sponsor sections in youtube videos by utilizing the [SponsorBlock](https://sponsor.ajay.app) API
 
@@ -79,18 +79,13 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
 
 * **Merged with animelover1984/youtube-dl**: You get most of the features and improvements from [animelover1984/youtube-dl](https://github.com/animelover1984/youtube-dl) including `--write-comments`, `BiliBiliSearch`, `BilibiliChannel`, Embedding thumbnail in mp4/ogg/opus, playlist infojson etc. Note that the NicoNico livestreams are not available. See [#31](https://github.com/yt-dlp/yt-dlp/pull/31) for details.
 
-* **Youtube improvements**:
+* **YouTube improvements**:
-    * All Feeds (`:ytfav`, `:ytwatchlater`, `:ytsubs`, `:ythistory`, `:ytrec`, `:ytnotif`) and private playlists supports downloading multiple pages of content
+    * Supports Clips, Stories (`ytstories:<channel UCID>`), Search (including filters)**\***, YouTube Music Search, Channel-specific search, Search prefixes (`ytsearch:`, `ytsearchdate:`)**\***, Mixes, YouTube Music Albums/Channels ([except self-uploaded music](https://github.com/yt-dlp/yt-dlp/issues/723)), and Feeds (`:ytfav`, `:ytwatchlater`, `:ytsubs`, `:ythistory`, `:ytrec`, `:ytnotif`)
-    * Search (`ytsearch:`, `ytsearchdate:`), search URLs and in-channel search works
+    * Fix for [n-sig based throttling](https://github.com/ytdl-org/youtube-dl/issues/29326) **\***
-    * Mixes supports downloading multiple pages of content
+    * Supports some (but not all) age-gated content without cookies
-    * Some (but not all) age-gated content can be downloaded without cookies
+    * Download livestreams from the start using `--live-from-start` (*experimental*)
-    * Fix for [n-sig based throttling](https://github.com/ytdl-org/youtube-dl/issues/29326)
+    * `255kbps` audio is extracted (if available) from YouTube Music when premium cookies are given
     * Redirect channel's home URL automatically to `/video` to preserve the old behaviour
-    * `255kbps` audio is extracted (if available) from youtube music when premium cookies are given
-    * Youtube music Albums, channels etc can be downloaded ([except self-uploaded music](https://github.com/yt-dlp/yt-dlp/issues/723))
-    * Download livestreams from the start using `--live-from-start` (experimental)
-    * Support for downloading stories (`ytstories:<channel UCID>`) 
-    * Support for downloading clips
 
 * **Cookies from browser**: Cookies can be automatically extracted from all major web browsers using `--cookies-from-browser BROWSER[+KEYRING][:PROFILE]`
 
@@ -124,6 +119,8 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
 
 See [changelog](Changelog.md) or [commits](https://github.com/yt-dlp/yt-dlp/commits) for the full list of changes
 
+Features marked with a **\*** have been back-ported to youtube-dl
+
 ### Differences in default behavior
 
 Some of yt-dlp's default options are different from that of youtube-dl and youtube-dlc:

requirements.txt

@@ -3,4 +3,4 @@ pycryptodomex
 websockets
 brotli; platform_python_implementation=='CPython'
 brotlicffi; platform_python_implementation!='CPython'
-certifi
+certifi

supportedsites.md

@@ -418,6 +418,7 @@
  - **Funk**
  - **Fusion**
  - **Fux**
+ - **FuyinTV**
  - **Gab**
  - **GabTV**
  - **Gaia**: [<abbr title="netrc machine"><em>gaia</em></abbr>]
@@ -618,6 +619,7 @@
  - **LiveJournal**
  - **livestream**
  - **livestream:original**
+ - **Livestreamfails**
  - **Lnk**
  - **LnkGo**
  - **loc**: Library of Congress
@@ -982,6 +984,7 @@
  - **PornoVoisines**
  - **PornoXO**
  - **PornTube**
+ - **PremiershipRugby**
  - **PressTV**
  - **ProjectVeritas**
  - **prosiebensat1**: ProSiebenSat.1 Digital
@@ -1113,6 +1116,7 @@
  - **ScreencastOMatic**
  - **ScrippsNetworks**
  - **scrippsnetworks:watch**
+ - **Scrolller**
  - **SCTE**: [<abbr title="netrc machine"><em>scte</em></abbr>]
  - **SCTECourse**: [<abbr title="netrc machine"><em>scte</em></abbr>]
  - **Seeker**
@@ -1189,6 +1193,7 @@
  - **stanfordoc**: Stanford Open ClassRoom
  - **startv**
  - **Steam**
+ - **SteamCommunityBroadcast**
  - **Stitcher**
  - **StitcherShow**
  - **StoryFire**
@@ -1427,7 +1432,8 @@
  - **vimeo:watchlater**: [<abbr title="netrc machine"><em>vimeo</em></abbr>] Vimeo watch later list, ":vimeowatchlater" keyword (requires authentication)
  - **Vimm:recording**
  - **Vimm:stream**
- - **Vimp**
+ - **ViMP**
+ - **ViMP:Playlist**
  - **Vimple**: Vimple - one-click video hosting
  - **Vine**
  - **vine:user**

test/test_download.py

@@ -273,7 +273,11 @@ def batch_generator(name, num_tests):
 
     def test_template(self):
         for i in range(num_tests):
-            getattr(self, f'test_{name}_{i}' if i else f'test_{name}')()
+            test_name = f'test_{name}_{i}' if i else f'test_{name}'
+            try:
+                getattr(self, test_name)()
+            except unittest.SkipTest:
+                print(f'Skipped {test_name}')
 
     return test_template
 

yt_dlp/YoutubeDL.py

@@ -10,7 +10,6 @@ import json
 import locale
 import operator
 import os
-import platform
 import random
 import re
 import shutil
@@ -110,7 +109,6 @@ from .utils import (
     number_of_digits,
     orderedSet,
     parse_filesize,
-    platform_name,
     preferredencoding,
     prepend_extension,
     register_socks_protocols,
@@ -126,6 +124,7 @@ from .utils import (
     strftime_or_none,
     subtitles_filename,
     supports_terminal_sequences,
+    system_identifier,
     timetuple_from_msec,
     to_high_limit_path,
     traverse_obj,
@@ -577,7 +576,9 @@ class YoutubeDL:
         MIN_SUPPORTED, MIN_RECOMMENDED = (3, 6), (3, 7)
         current_version = sys.version_info[:2]
         if current_version < MIN_RECOMMENDED:
-            msg = 'Support for Python version %d.%d has been deprecated and will break in future versions of yt-dlp'
+            msg = ('Support for Python version %d.%d has been deprecated. '
+                   'See  https://github.com/yt-dlp/yt-dlp/issues/3764  for more details. '
+                   'You will recieve only one more update on this version')
             if current_version < MIN_SUPPORTED:
                 msg = 'Python version %d.%d is no longer supported'
             self.deprecation_warning(
@@ -3532,7 +3533,7 @@ class YoutubeDL:
                     'none', '' if f.get('vcodec') == 'none'
                             else self._format_out('video only', self.Styles.SUPPRESS)),
                 format_field(f, 'abr', '\t%dk'),
-                format_field(f, 'asr', '\t%dHz'),
+                format_field(f, 'asr', '\t%s', func=format_decimal_suffix),
                 join_nonempty(
                     self._format_out('UNSUPPORTED', 'light red') if f.get('ext') in ('f4f', 'f4m') else None,
                     format_field(f, 'language', '[%s]'),
@@ -3656,17 +3657,7 @@ class YoutubeDL:
                 with contextlib.suppress(Exception):
                     sys.exc_clear()
 
-        def python_implementation():
+        write_debug(system_identifier())
-            impl_name = platform.python_implementation()
-            if impl_name == 'PyPy' and hasattr(sys, 'pypy_version_info'):
-                return impl_name + ' version %d.%d.%d' % sys.pypy_version_info[:3]
-            return impl_name
-
-        write_debug('Python version %s (%s %s) - %s' % (
-            platform.python_version(),
-            python_implementation(),
-            platform.architecture()[0],
-            platform_name()))
 
         exe_versions, ffmpeg_features = FFmpegPostProcessor.get_versions_and_features(self)
         ffmpeg_features = {key for key, val in ffmpeg_features.items() if val}

yt_dlp/compat/_legacy.py

@@ -44,14 +44,26 @@ def compat_setenv(key, value, env=os.environ):
 
 
 compat_basestring = str
+compat_chr = chr
 compat_collections_abc = collections.abc
+compat_cookiejar = http.cookiejar
+compat_cookiejar_Cookie = http.cookiejar.Cookie
 compat_cookies = http.cookies
+compat_cookies_SimpleCookie = http.cookies.SimpleCookie
 compat_etree_Element = etree.Element
 compat_etree_register_namespace = etree.register_namespace
 compat_filter = filter
+compat_get_terminal_size = shutil.get_terminal_size
 compat_getenv = os.getenv
+compat_getpass = getpass.getpass
+compat_html_entities = html.entities
+compat_html_entities_html5 = html.entities.html5
+compat_HTMLParser = html.parser.HTMLParser
+compat_http_client = http.client
+compat_http_server = http.server
 compat_input = input
 compat_integer_types = (int, )
+compat_itertools_count = itertools.count
 compat_kwargs = lambda kwargs: kwargs
 compat_map = map
 compat_numeric_types = (int, float, complex)
@@ -59,34 +71,22 @@ compat_print = print
 compat_shlex_split = shlex.split
 compat_socket_create_connection = socket.create_connection
 compat_Struct = struct.Struct
+compat_struct_pack = struct.pack
+compat_struct_unpack = struct.unpack
 compat_subprocess_get_DEVNULL = lambda: DEVNULL
+compat_tokenize_tokenize = tokenize.tokenize
+compat_urllib_error = urllib.error
+compat_urllib_parse = urllib.parse
 compat_urllib_parse_quote = urllib.parse.quote
 compat_urllib_parse_quote_plus = urllib.parse.quote_plus
+compat_urllib_parse_unquote_plus = urllib.parse.unquote_plus
 compat_urllib_parse_unquote_to_bytes = urllib.parse.unquote_to_bytes
 compat_urllib_parse_urlunparse = urllib.parse.urlunparse
-compat_urllib_request_DataHandler = urllib.request.DataHandler
 compat_urllib_request = urllib.request
+compat_urllib_request_DataHandler = urllib.request.DataHandler
 compat_urllib_response = urllib.response
 compat_urlretrieve = urllib.request.urlretrieve
 compat_xml_parse_error = etree.ParseError
 compat_xpath = lambda xpath: xpath
 compat_zip = zip
 workaround_optparse_bug9161 = lambda: None
-compat_getpass = getpass.getpass
-compat_chr = chr
-compat_urllib_parse = urllib.parse
-compat_itertools_count = itertools.count
-compat_cookiejar = http.cookiejar
-compat_cookiejar_Cookie = http.cookiejar.Cookie
-compat_cookies_SimpleCookie = http.cookies.SimpleCookie
-compat_get_terminal_size = shutil.get_terminal_size
-compat_html_entities = html.entities
-compat_html_entities_html5 = html.entities.html5
-compat_tokenize_tokenize = tokenize.tokenize
-compat_HTMLParser = html.parser.HTMLParser
-compat_http_client = http.client
-compat_http_server = http.server
-compat_struct_pack = struct.pack
-compat_struct_unpack = struct.unpack
-compat_urllib_error = urllib.error
-compat_urllib_parse_unquote_plus = urllib.parse.unquote_plus

yt_dlp/downloader/__init__.py

@@ -59,10 +59,11 @@ PROTOCOL_MAP = {
 
 def shorten_protocol_name(proto, simplify=False):
     short_protocol_names = {
-        'm3u8_native': 'm3u8_n',
+        'm3u8_native': 'm3u8',
-        'rtmp_ffmpeg': 'rtmp_f',
+        'm3u8': 'm3u8F',
+        'rtmp_ffmpeg': 'rtmpF',
         'http_dash_segments': 'dash',
-        'http_dash_segments_generator': 'dash_g',
+        'http_dash_segments_generator': 'dashG',
         'niconico_dmc': 'dmc',
         'websocket_frag': 'WSfrag',
     }
@@ -70,6 +71,7 @@ def shorten_protocol_name(proto, simplify=False):
         short_protocol_names.update({
             'https': 'http',
             'ftps': 'ftp',
+            'm3u8': 'm3u8',  # Reverse above m3u8 mapping
             'm3u8_native': 'm3u8',
             'http_dash_segments_generator': 'dash',
             'rtmp_ffmpeg': 'rtmp',

yt_dlp/downloader/hls.py

@@ -69,7 +69,7 @@ class HlsFD(FragmentFD):
             elif no_crypto:
                 message = ('The stream has AES-128 encryption and neither ffmpeg nor pycryptodomex are available; '
                            'Decryption will be performed natively, but will be extremely slow')
-            elif re.search(r'#EXT-X-MEDIA-SEQUENCE:(?!0$)', s):
+            elif info_dict.get('extractor_key') == 'Generic' and re.search(r'(?m)#EXT-X-MEDIA-SEQUENCE:(?!0$)', s):
                 install_ffmpeg = '' if has_ffmpeg else 'install ffmpeg and '
                 message = ('Live HLS streams are not supported by the native downloader. If this is a livestream, '
                            f'please {install_ffmpeg}add "--downloader ffmpeg --hls-use-mpegts" to your command')

yt_dlp/extractor/_extractors.py

@@ -837,6 +837,7 @@ from .livestream import (
     LivestreamOriginalIE,
     LivestreamShortenerIE,
 )
+from .livestreamfails import LivestreamfailsIE
 from .lnkgo import (
     LnkGoIE,
     LnkIE,
@@ -1934,7 +1935,10 @@ from .vice import (
 from .vidbit import VidbitIE
 from .viddler import ViddlerIE
 from .videa import VideaIE
-from .videocampus_sachsen import VideocampusSachsenIE
+from .videocampus_sachsen import (
+    VideocampusSachsenIE,
+    ViMPPlaylistIE,
+)
 from .videodetective import VideoDetectiveIE
 from .videofyme import VideofyMeIE
 from .videomore import (

yt_dlp/extractor/common.py

@@ -63,6 +63,7 @@ from ..utils import (
     str_to_int,
     strip_or_none,
     traverse_obj,
+    try_call,
     try_get,
     unescapeHTML,
     unified_strdate,
@@ -2820,7 +2821,7 @@ class InfoExtractor:
                     base_url = ''
                     for element in (representation, adaptation_set, period, mpd_doc):
                         base_url_e = element.find(_add_ns('BaseURL'))
-                        if base_url_e is not None:
+                        if try_call(lambda: base_url_e.text) is not None:
                             base_url = base_url_e.text + base_url
                             if re.match(r'^https?://', base_url):
                                 break

yt_dlp/extractor/generic.py

@@ -2825,12 +2825,22 @@ class GenericIE(InfoExtractor):
                         new_url, {'force_videoid': force_videoid})
                 return self.url_result(new_url)
 
+        def request_webpage():
+            request = sanitized_Request(url)
+            # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
+            # making it impossible to download only chunk of the file (yet we need only 512kB to
+            # test whether it's HTML or not). According to yt-dlp default Accept-Encoding
+            # that will always result in downloading the whole file that is not desirable.
+            # Therefore for extraction pass we have to override Accept-Encoding to any in order
+            # to accept raw bytes and being able to download only a chunk.
+            # It may probably better to solve this by checking Content-Type for application/octet-stream
+            # after HEAD request finishes, but not sure if we can rely on this.
+            request.add_header('Accept-Encoding', '*')
+            return self._request_webpage(request, video_id)
+
         full_response = None
         if head_response is False:
-            request = sanitized_Request(url)
+            head_response = full_response = request_webpage()
-            request.add_header('Accept-Encoding', '*')
-            full_response = self._request_webpage(request, video_id)
-            head_response = full_response
 
         info_dict = {
             'id': video_id,
@@ -2868,19 +2878,7 @@ class GenericIE(InfoExtractor):
             self.report_warning(
                 '%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))
 
-        if not full_response:
+        full_response = full_response or request_webpage()
-            request = sanitized_Request(url)
-            # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
-            # making it impossible to download only chunk of the file (yet we need only 512kB to
-            # test whether it's HTML or not). According to yt-dlp default Accept-Encoding
-            # that will always result in downloading the whole file that is not desirable.
-            # Therefore for extraction pass we have to override Accept-Encoding to any in order
-            # to accept raw bytes and being able to download only a chunk.
-            # It may probably better to solve this by checking Content-Type for application/octet-stream
-            # after HEAD request finishes, but not sure if we can rely on this.
-            request.add_header('Accept-Encoding', '*')
-            full_response = self._request_webpage(request, video_id)
-
         first_bytes = full_response.read(512)
 
         # Is it an M3U playlist?

yt_dlp/extractor/livestreamfails.py

@@ -0,0 +1,34 @@
+from .common import InfoExtractor
+from ..utils import format_field, traverse_obj, unified_timestamp
+
+
+class LivestreamfailsIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?livestreamfails\.com/clip/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'https://livestreamfails.com/clip/139200',
+        'md5': '8a03aea1a46e94a05af6410337463102',
+        'info_dict': {
+            'id': '139200',
+            'ext': 'mp4',
+            'display_id': 'ConcernedLitigiousSalmonPeteZaroll-O8yo9W2L8OZEKhV2',
+            'title': 'Streamer jumps off a trampoline at full speed',
+            'creator': 'paradeev1ch',
+            'thumbnail': r're:^https?://.+',
+            'timestamp': 1656271785,
+            'upload_date': '20220626',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        api_response = self._download_json(f'https://api.livestreamfails.com/clip/{video_id}', video_id)
+
+        return {
+            'id': video_id,
+            'display_id': api_response.get('sourceId'),
+            'timestamp': unified_timestamp(api_response.get('createdAt')),
+            'url': f'https://livestreamfails-video-prod.b-cdn.net/video/{api_response["videoId"]}',
+            'title': api_response.get('label'),
+            'creator': traverse_obj(api_response, ('streamer', 'label')),
+            'thumbnail': format_field(api_response, 'imageId', 'https://livestreamfails-image-prod.b-cdn.net/image/%s')
+        }

yt_dlp/extractor/videocampus_sachsen.py

@@ -1,8 +1,9 @@
+import functools
 import re
 
 from .common import InfoExtractor
 from ..compat import compat_HTTPError
-from ..utils import ExtractorError
+from ..utils import ExtractorError, OnDemandPagedList, urlencode_postdata
 
 
 class VideocampusSachsenIE(InfoExtractor):
@@ -183,3 +184,71 @@ class VideocampusSachsenIE(InfoExtractor):
             'formats': formats,
             'subtitles': subtitles,
         }
+
+
+class ViMPPlaylistIE(InfoExtractor):
+    IE_NAME = 'ViMP:Playlist'
+    _VALID_URL = r'''(?x)(?P<host>https?://(?:%s))/(?:
+        album/view/aid/(?P<album_id>[0-9]+)|
+        (?P<mode>category|channel)/(?P<name>[\w-]+)/(?P<id>[0-9]+)
+    )''' % '|'.join(map(re.escape, VideocampusSachsenIE._INSTANCES))
+
+    _TESTS = [{
+        'url': 'https://vimp.oth-regensburg.de/channel/Designtheorie-1-SoSe-2020/3',
+        'info_dict': {
+            'id': 'channel-3',
+            'title': 'Designtheorie 1 SoSe 2020 :: Channels :: ViMP OTH Regensburg',
+        },
+        'playlist_mincount': 9,
+    }, {
+        'url': 'https://www.fh-bielefeld.de/medienportal/album/view/aid/208',
+        'info_dict': {
+            'id': 'album-208',
+            'title': 'KG Praktikum ABT/MEC :: Playlists :: FH-Medienportal',
+        },
+        'playlist_mincount': 4,
+    }, {
+        'url': 'https://videocampus.sachsen.de/category/online-tutorials-onyx/91',
+        'info_dict': {
+            'id': 'category-91',
+            'title': 'Online-Seminare ONYX - BPS - Bildungseinrichtungen - VCS',
+        },
+        'playlist_mincount': 7,
+    }]
+    _PAGE_SIZE = 10
+
+    def _fetch_page(self, host, url_part, id, data, page):
+        webpage = self._download_webpage(
+            f'{host}/media/ajax/component/boxList/{url_part}', id,
+            query={'page': page, 'page_only': 1}, data=urlencode_postdata(data))
+        urls = re.findall(r'"([^"]+/video/[^"]+)"', webpage)
+
+        for url in urls:
+            yield self.url_result(host + url, VideocampusSachsenIE)
+
+    def _real_extract(self, url):
+        host, album_id, mode, name, id = self._match_valid_url(url).group(
+            'host', 'album_id', 'mode', 'name', 'id')
+
+        webpage = self._download_webpage(url, album_id or id, fatal=False) or ''
+        title = (self._html_search_meta('title', webpage, fatal=False)
+                 or self._html_extract_title(webpage))
+
+        url_part = (f'aid/{album_id}' if album_id
+                    else f'category/{name}/category_id/{id}' if mode == 'category'
+                    else f'title/{name}/channel/{id}')
+
+        mode = mode or 'album'
+        data = {
+            'vars[mode]': mode,
+            f'vars[{mode}]': album_id or id,
+            'vars[context]': '4' if album_id else '1' if mode == 'category' else '3',
+            'vars[context_id]': album_id or id,
+            'vars[layout]': 'thumb',
+            'vars[per_page][thumb]': str(self._PAGE_SIZE),
+        }
+
+        return self.playlist_result(
+            OnDemandPagedList(functools.partial(
+                self._fetch_page, host, url_part, album_id or id, data), self._PAGE_SIZE),
+            playlist_title=title, id=f'{mode}-{album_id or id}')

yt_dlp/extractor/youtube.py

@@ -2467,6 +2467,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         func_id = f'js_{player_id}_{self._signature_cache_id(example_sig)}'
         assert os.path.basename(func_id) == func_id
 
+        self.write_debug(f'Extracting signature function {func_id}')
         cache_spec = self.cache.load('youtube-sigfuncs', func_id)
         if cache_spec is not None:
             return lambda s: ''.join(s[i] for i in cache_spec)
@@ -2714,10 +2715,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     @classmethod
     def extract_id(cls, url):
-        mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
+        video_id = cls.get_temp_id(url)
-        if mobj is None:
+        if not video_id:
-            raise ExtractorError('Invalid URL: %s' % url)
+            raise ExtractorError(f'Invalid URL: {url}')
-        return mobj.group('id')
+        return video_id
 
     def _extract_chapters_from_json(self, data, duration):
         chapter_list = traverse_obj(

yt_dlp/update.py

@@ -3,17 +3,25 @@ import hashlib
 import json
 import os
 import platform
+import re
 import subprocess
 import sys
 from zipimport import zipimporter
 
 from .compat import functools  # isort: split
 from .compat import compat_realpath
-from .utils import Popen, shell_quote, traverse_obj, version_tuple
+from .utils import (
+    Popen,
+    cached_method,
+    shell_quote,
+    system_identifier,
+    traverse_obj,
+    version_tuple,
+)
 from .version import __version__
 
 REPOSITORY = 'yt-dlp/yt-dlp'
-API_URL = f'https://api.github.com/repos/{REPOSITORY}/releases/latest'
+API_URL = f'https://api.github.com/repos/{REPOSITORY}/releases'
 
 
 @functools.cache
@@ -25,6 +33,8 @@ def _get_variant_and_executable_path():
             return 'py2exe', path
         if sys._MEIPASS == os.path.dirname(path):
             return f'{sys.platform}_dir', path
+        if sys.platform == 'darwin' and version_tuple(platform.mac_ver()[0]) < (10, 15):
+            return 'darwin_legacy_exe', path
         return f'{sys.platform}_exe', path
 
     path = os.path.dirname(__file__)
@@ -45,6 +55,7 @@ _FILE_SUFFIXES = {
     'py2exe': '_min.exe',
     'win32_exe': '.exe',
     'darwin_exe': '_macos',
+    'darwin_legacy_exe': '_macos_legacy',
     'linux_exe': '_linux',
 }
 
@@ -76,9 +87,20 @@ class Updater:
         self.ydl = ydl
 
     @functools.cached_property
-    def _new_version_info(self):
+    def _tag(self):
-        self.ydl.write_debug(f'Fetching release info: {API_URL}')
+        identifier = f'{detect_variant()} {system_identifier()}'
-        return json.loads(self.ydl.urlopen(API_URL).read().decode())
+        for line in self._download('_update_spec', 'latest').decode().splitlines():
+            if not line.startswith('lock '):
+                continue
+            _, tag, pattern = line.split(' ', 2)
+            if re.match(pattern, identifier):
+                return f'tags/{tag}'
+        return 'latest'
+
+    @cached_method
+    def _get_version_info(self, tag):
+        self.ydl.write_debug(f'Fetching release info: {API_URL}/{tag}')
+        return json.loads(self.ydl.urlopen(f'{API_URL}/{tag}').read().decode())
 
     @property
     def current_version(self):
@@ -88,7 +110,7 @@ class Updater:
     @property
     def new_version(self):
         """Version of the latest release"""
-        return self._new_version_info['tag_name']
+        return self._get_version_info(self._tag)['tag_name']
 
     @property
     def has_update(self):
@@ -100,9 +122,8 @@ class Updater:
         """Filename of the executable"""
         return compat_realpath(_get_variant_and_executable_path()[1])
 
-    def _download(self, name=None):
+    def _download(self, name, tag):
-        name = name or self.release_name
+        url = traverse_obj(self._get_version_info(tag), (
-        url = traverse_obj(self._new_version_info, (
             'assets', lambda _, v: v['name'] == name, 'browser_download_url'), get_all=False)
         if not url:
             raise Exception('Unable to find download URL')
@@ -120,7 +141,7 @@ class Updater:
     @functools.cached_property
     def release_hash(self):
         """Hash of the latest release"""
-        hash_data = dict(ln.split()[::-1] for ln in self._download('SHA2-256SUMS').decode().splitlines())
+        hash_data = dict(ln.split()[::-1] for ln in self._download('SHA2-256SUMS', self._tag).decode().splitlines())
         return hash_data[self.release_name]
 
     def _report_error(self, msg, expected=False):
@@ -173,7 +194,7 @@ class Updater:
             return self._report_error('Unable to remove the old version')
 
         try:
-            newcontent = self._download()
+            newcontent = self._download(self.release_name, self._tag)
         except OSError:
             return self._report_network_error('download latest version')
         except Exception:

yt_dlp/utils.py

@@ -18,6 +18,7 @@ import html.parser
 import http.client
 import http.cookiejar
 import importlib.util
+import inspect
 import io
 import itertools
 import json
@@ -233,7 +234,7 @@ DATE_FORMATS_MONTH_FIRST.extend([
 ])
 
 PACKED_CODES_RE = r"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)"
-JSON_LD_RE = r'(?is)<script[^>]+type=(["\']?)application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>'
+JSON_LD_RE = r'(?is)<script[^>]+type=(["\']?)application/ld\+json\1[^>]*>\s*(?P<json_ld>{.+?})\s*</script>'
 
 NUMBER_RE = r'\d+(?:\.\d+)?'
 
@@ -672,8 +673,8 @@ def sanitize_filename(s, restricted=False, is_id=NO_DEFAULT):
     s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)  # Handle timestamps
     result = ''.join(map(replace_insane, s))
     if is_id is NO_DEFAULT:
-        result = re.sub('(\0.)(?:(?=\\1)..)+', r'\1', result)  # Remove repeated substitute chars
+        result = re.sub(r'(\0.)(?:(?=\1)..)+', r'\1', result)  # Remove repeated substitute chars
-        STRIP_RE = '(?:\0.|[ _-])*'
+        STRIP_RE = r'(?:\0.|[ _-])*'
         result = re.sub(f'^\0.{STRIP_RE}|{STRIP_RE}\0.$', '', result)  # Remove substitute chars from start/end
     result = result.replace('\0', '') or '_'
 
@@ -1909,12 +1910,23 @@ class DateRange:
 
 def platform_name():
     """ Returns the platform name as a str """
-    res = platform.platform()
+    write_string('DeprecationWarning: yt_dlp.utils.platform_name is deprecated, use platform.platform instead')
-    if isinstance(res, bytes):
+    return platform.platform()
-        res = res.decode(preferredencoding())
 
-    assert isinstance(res, str)
+
-    return res
+@functools.cache
+def system_identifier():
+    python_implementation = platform.python_implementation()
+    if python_implementation == 'PyPy' and hasattr(sys, 'pypy_version_info'):
+        python_implementation += ' version %d.%d.%d' % sys.pypy_version_info[:3]
+
+    return 'Python %s (%s %s) - %s %s' % (
+        platform.python_version(),
+        python_implementation,
+        platform.architecture()[0],
+        platform.platform(),
+        format_field(join_nonempty(*platform.libc_ver(), delim=' '), None, '(%s)'),
+    )
 
 
 @functools.cache
@@ -2388,8 +2400,7 @@ def remove_quotes(s):
 
 
 def get_domain(url):
-    domain = re.match(r'(?:https?:\/\/)?(?:www\.)?(?P<domain>[^\n\/]+\.[^\n\/]+)(?:\/(.*))?', url)
+    return '.'.join(urllib.parse.urlparse(url).netloc.rsplit('.', 2)[-2:])
-    return domain.group('domain') if domain else None
 
 
 def url_basename(url):
@@ -5544,8 +5555,27 @@ def merge_headers(*dicts):
     return {k.title(): v for k, v in itertools.chain.from_iterable(map(dict.items, dicts))}
 
 
+def cached_method(f):
+    """Cache a method"""
+    signature = inspect.signature(f)
+
+    @functools.wraps(f)
+    def wrapper(self, *args, **kwargs):
+        bound_args = signature.bind(self, *args, **kwargs)
+        bound_args.apply_defaults()
+        key = tuple(bound_args.arguments.values())
+
+        if not hasattr(self, '__cached_method__cache'):
+            self.__cached_method__cache = {}
+        cache = self.__cached_method__cache.setdefault(f.__name__, {})
+        if key not in cache:
+            cache[key] = f(self, *args, **kwargs)
+        return cache[key]
+    return wrapper
+
+
 class classproperty:
-    """classmethod(property(func)) that works in py < 3.9"""
+    """property access for class methods"""
 
     def __init__(self, func):
         functools.update_wrapper(self, func)

yt_dlp/version.py

@@ -1,5 +1,5 @@
 # Autogenerated by devscripts/update-version.py
 
-__version__ = '2022.06.22.1'
+__version__ = '2022.06.29'
 
-RELEASE_GIT_HEAD = 'a86e01e74'
+RELEASE_GIT_HEAD = '9d339c41e'