.github/ISSUE_TEMPLATE/1_broken_site.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm reporting a broken site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.06.29** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.06.22.1** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -51,12 +51,12 @@ body:
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2022.06.29 (exe)
+        [debug] yt-dlp version 2022.06.22.1 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2022.06.29)
+        yt-dlp is up to date (2022.06.22.1)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/2_site_support_request.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.06.29** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.06.22.1** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -62,12 +62,12 @@ body:
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2022.06.29 (exe)
+        [debug] yt-dlp version 2022.06.22.1 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2022.06.29)
+        yt-dlp is up to date (2022.06.22.1)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/3_site_feature_request.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm requesting a site-specific feature
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.06.29** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.06.22.1** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -60,12 +60,12 @@ body:
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2022.06.29 (exe)
+        [debug] yt-dlp version 2022.06.22.1 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2022.06.29)
+        yt-dlp is up to date (2022.06.22.1)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/4_bug_report.yml

@@ -11,7 +11,7 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.06.29** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.06.22.1** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
@@ -45,12 +45,12 @@ body:
         [debug] Portable config file: yt-dlp.conf
         [debug] Portable config: ['-i']
         [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2022.06.29 (exe)
+        [debug] yt-dlp version 2022.06.22.1 (exe)
         [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
         [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
         [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
         [debug] Proxy map: {}
-        yt-dlp is up to date (2022.06.29)
+        yt-dlp is up to date (2022.06.22.1)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/5_feature_request.yml

@@ -13,7 +13,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.06.29** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.06.22.1** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues including closed ones. DO NOT post duplicates
           required: true

.github/ISSUE_TEMPLATE/6_question.yml

@@ -13,7 +13,7 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.06.29** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.06.22.1** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions including closed ones. DO NOT post duplicates
           required: true

.github/workflows/build.yml

@@ -8,7 +8,6 @@ jobs:
       version_suffix: ${{ steps.version_suffix.outputs.version_suffix }}
       ytdlp_version: ${{ steps.bump_version.outputs.ytdlp_version }}
       upload_url: ${{ steps.create_release.outputs.upload_url }}
-      release_id: ${{ steps.create_release.outputs.id }}
     steps:
     - uses: actions/checkout@v2
       with:
@@ -30,7 +29,6 @@ jobs:
         make issuetemplates
 
     - name: Push to release
-      id: push_release
       run: |
         git config --global user.name github-actions
         git config --global user.email github-actions@example.com
@@ -59,19 +57,15 @@ jobs:
         tag_name: ${{ steps.bump_version.outputs.ytdlp_version }}
         release_name: yt-dlp ${{ steps.bump_version.outputs.ytdlp_version }}
         commitish: ${{ steps.push_release.outputs.head_sha }}
-        draft: true
-        prerelease: false
         body: |
           #### [A description of the various files]((https://github.com/yt-dlp/yt-dlp#release-files)) are in the README
 
           ---
-          <details open><summary><h3>Changelog</summary>
-          <p>
 
+          ### Changelog:
           ${{ env.changelog }}
-
+        draft: false
-          </p>
+        prerelease: false
-          </details>
 
 
   build_unix:
@@ -241,52 +235,6 @@ jobs:
         asset_content_type: application/zip
 
 
-  build_macos_legacy:
-    runs-on: macos-latest
-    needs: create_release
-    steps:
-    - uses: actions/checkout@v2
-    - name: Install Python
-      # We need the official Python, because the GA ones only support newer macOS versions
-      env:
-        PYTHON_VERSION: 3.10.5
-        MACOSX_DEPLOYMENT_TARGET: 10.9  # Used up by the Python build tools
-      run: |
-          # Hack to get the latest patch version. Uncomment if needed
-          #brew install python@3.10
-          #export PYTHON_VERSION=$( $(brew --prefix)/opt/python@3.10/bin/python3 --version | cut -d ' ' -f 2 )
-          curl https://www.python.org/ftp/python/${PYTHON_VERSION}/python-${PYTHON_VERSION}-macos11.pkg -o "python.pkg"
-          sudo installer -pkg python.pkg -target /
-          python3 --version
-    - name: Install Requirements
-      run: |
-          brew install coreutils
-          python3 -m pip install -U --user pip Pyinstaller -r requirements.txt
-
-    - name: Prepare
-      run: |
-          python3 devscripts/update-version.py ${{ needs.create_release.outputs.version_suffix }}
-          python3 devscripts/make_lazy_extractors.py
-    - name: Build
-      run: |
-          python3 pyinst.py
-    - name: Get SHA2-SUMS
-      id: get_sha
-      run: |
-          echo "::set-output name=sha256_macos_legacy::$(sha256sum dist/yt-dlp_macos | awk '{print $1}')"
-          echo "::set-output name=sha512_macos_legacy::$(sha512sum dist/yt-dlp_macos | awk '{print $1}')"
-
-    - name: Upload standalone binary
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ needs.create_release.outputs.upload_url }}
-        asset_path: ./dist/yt-dlp_macos
-        asset_name: yt-dlp_macos_legacy
-        asset_content_type: application/octet-stream
-
-
   build_windows:
     runs-on: windows-latest
     needs: create_release
@@ -402,7 +350,7 @@ jobs:
 
   finish:
     runs-on: ubuntu-latest
-    needs: [create_release, build_unix, build_windows, build_windows32, build_macos, build_macos_legacy]
+    needs: [create_release, build_unix, build_windows, build_windows32, build_macos]
 
     steps:
     - name: Make SHA2-SUMS files
@@ -417,7 +365,6 @@ jobs:
         echo "${{ needs.build_windows.outputs.sha256_win_zip }}  yt-dlp_win.zip" >> SHA2-256SUMS
         echo "${{ needs.build_macos.outputs.sha256_macos }}  yt-dlp_macos" >> SHA2-256SUMS
         echo "${{ needs.build_macos.outputs.sha256_macos_zip }}  yt-dlp_macos.zip" >> SHA2-256SUMS
-        echo "${{ needs.build_macos_legacy.outputs.sha256_macos_legacy }}  yt-dlp_macos_legacy" >> SHA2-256SUMS
         echo "${{ needs.build_unix.outputs.sha512_bin }}  yt-dlp" >> SHA2-512SUMS
         echo "${{ needs.build_unix.outputs.sha512_tar }}  yt-dlp.tar.gz" >> SHA2-512SUMS
         echo "${{ needs.build_unix.outputs.sha512_linux }}  yt-dlp_linux" >> SHA2-512SUMS
@@ -428,7 +375,6 @@ jobs:
         echo "${{ needs.build_windows.outputs.sha512_win_zip }}  yt-dlp_win.zip" >> SHA2-512SUMS
         echo "${{ needs.build_macos.outputs.sha512_macos }}  yt-dlp_macos" >> SHA2-512SUMS
         echo "${{ needs.build_macos.outputs.sha512_macos_zip }}  yt-dlp_macos.zip" >> SHA2-512SUMS
-        echo "${{ needs.build_macos_legacy.outputs.sha512_macos_legacy }}  yt-dlp_macos_legacy" >> SHA2-512SUMS
 
     - name: Upload SHA2-256SUMS file
       uses: actions/upload-release-asset@v1
@@ -448,24 +394,3 @@ jobs:
         asset_path: ./SHA2-512SUMS
         asset_name: SHA2-512SUMS
         asset_content_type: text/plain
-
-    - name: Make Update spec
-      run: |
-        echo "# This file is used for regulating self-update" >> _update_spec
-    - name: Upload update spec
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        upload_url: ${{ needs.create_release.outputs.upload_url }}
-        asset_path: ./_update_spec
-        asset_name: _update_spec
-        asset_content_type: text/plain
-
-    - name: Finalize release
-      env:
-        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-        gh api -X PATCH -H "Accept: application/vnd.github.v3+json" \
-          /repos/${{ github.repository }}/releases/${{ needs.create_release.outputs.release_id }} \
-          -F draft=false

CONTRIBUTING.md

@@ -457,7 +457,7 @@ title = self._search_regex(  # incorrect
     webpage, 'title', group='title')
 ```
 
-Here the presence or absence of other attributes including `style` is irrelevant for the data we need, and so the regex must not depend on it
+Here the presence or absence of other attributes including `style` is irrelevent for the data we need, and so the regex must not depend on it
 
 
 #### Keep the regular expressions as simple as possible, but no simpler
@@ -501,7 +501,7 @@ There is a soft limit to keep lines of code under 100 characters long. This mean
 
 For example, you should **never** split long string literals like URLs or some other often copied entities over multiple lines to fit this limit:
 
-Conversely, don't unnecessarily split small lines further. As a rule of thumb, if removing the line split keeps the code under 80 characters, it should be a single line.
+Conversely, don't unecessarily split small lines further. As a rule of thumb, if removing the line split keeps the code under 80 characters, it should be a single line.
 
 ##### Examples
 

CONTRIBUTORS

@@ -267,8 +267,3 @@ sqrtNOT
 bubbleguuum
 darkxex
 miseran
-StefanLobbenmeier
-crazymoose77756
-nomevi
-Brett824
-pingiun

Changelog.md

@@ -11,45 +11,6 @@
 -->
 
 
-### 2022.06.29
-
-* Fix `--downloader native`
-* Fix `section_end` of clips
-* Fix playlist error handling
-* Sanitize `chapters`
-* [extractor] Fix `_create_request` when headers is None
-* [extractor] Fix empty `BaseURL` in MPD
-* [ffmpeg] Write full output to debug on error
-* [hls] Warn user when trying to download live HLS
-* [options] Fix `parse_known_args` for `--`
-* [utils] Fix inconsistent default handling between HTTP and HTTPS requests by [coletdjnz](https://github.com/coletdjnz)
-* [build] Draft release until complete
-* [build] Fix release tag commit
-* [build] Standalone x64 builds for MacOS 10.9 by [StefanLobbenmeier](https://github.com/StefanLobbenmeier)
-* [update] Ability to set a maximum version for specific variants
-* [compat] Fix `compat.WINDOWS_VT_MODE`
-* [compat] Remove deprecated functions from core code
-* [compat] Remove more functions
-* [cleanup, extractor] Reduce direct use of `_downloader`
-* [cleanup] Consistent style for file heads
-* [cleanup] Fix some typos by [crazymoose77756](https://github.com/crazymoose77756)
-* [cleanup] Misc fixes and cleanup
-* [extractor/Scrolller] Add extractor by [LunarFang416](https://github.com/LunarFang416)
-* [extractor/ViMP] Add playlist extractor by [FestplattenSchnitzel](https://github.com/FestplattenSchnitzel)
-* [extractor/fuyin] Add extractor by [HobbyistDev](https://github.com/HobbyistDev)
-* [extractor/livestreamfails] Add extractor by [nomevi](https://github.com/nomevi)
-* [extractor/premiershiprugby] Add extractor by [HobbyistDev](https://github.com/HobbyistDev)
-* [extractor/steam] Add broadcast extractor by [HobbyistDev](https://github.com/HobbyistDev)
-* [extractor/youtube] Mark videos as fully watched by [Brett824](https://github.com/Brett824)
-* [extractor/CWTV] Extract thumbnail by [ischmidt20](https://github.com/ischmidt20)
-* [extractor/ViMP] Add thumbnail and support more sites by [FestplattenSchnitzel](https://github.com/FestplattenSchnitzel)
-* [extractor/dropout] Support cookies and login only as needed by [pingiun](https://github.com/pingiun), [pukkandan](https://github.com/pukkandan)
-* [extractor/ertflix] Improve `_VALID_URL`
-* [extractor/lbry] Use HEAD request for redirect URL by [flashdagger](https://github.com/flashdagger)
-* [extractor/mediaset] Improve `_VALID_URL`
-* [extractor/npr] Implement [e50c350](https://github.com/yt-dlp/yt-dlp/commit/e50c3500b43d80e4492569c4b4523c4379c6fbb2) differently
-* [extractor/tennistv] Rewrite extractor by [pukkandan](https://github.com/pukkandan), [zenerdi0de](https://github.com/zenerdi0de)
-
 ### 2022.06.22.1
 
 * [build] Fix updating homebrew formula
@@ -583,7 +544,7 @@
 * [downloader/ffmpeg] Handle unknown formats better
 * [outtmpl] Handle `-o ""` better
 * [outtmpl] Handle hard-coded file extension better
-* [extractor] Add convenience function `_yes_playlist`
+* [extractor] Add convinience function `_yes_playlist`
 * [extractor] Allow non-fatal `title` extraction
 * [extractor] Extract video inside `Article` json_ld
 * [generic] Allow further processing of json_ld URL
@@ -1717,7 +1678,7 @@
 * [utils] Generalize `traverse_dict` to `traverse_obj`
 * [downloader/ffmpeg] Hide FFmpeg banner unless in verbose mode by [fstirlitz](https://github.com/fstirlitz)
 * [build] Release `yt-dlp.tar.gz`
-* [build,update] Add GNU-style SHA512 and prepare updater for similar SHA256 by [nihil-admirari](https://github.com/nihil-admirari)
+* [build,update] Add GNU-style SHA512 and prepare updater for simlar SHA256 by [nihil-admirari](https://github.com/nihil-admirari)
 * [pyinst] Show Python version in exe metadata by [nihil-admirari](https://github.com/nihil-admirari)
 * [docs] Improve documentation of dependencies
 * [cleanup] Mark unused files

README.md

@@ -71,7 +71,7 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
 
 # NEW FEATURES
 
-* Merged with **youtube-dl v2021.12.17+ [commit/a03b977](https://github.com/ytdl-org/youtube-dl/commit/a03b9775d544b06a5b4f2aa630214c7c22fc2229)**<!--([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))--> and **youtube-dlc v2020.11.11-3+ [commit/f9401f2](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee)**: You get all the features and patches of [youtube-dlc](https://github.com/blackjack4494/yt-dlc) in addition to the latest [youtube-dl](https://github.com/ytdl-org/youtube-dl)
+* Based on **youtube-dl 2021.12.17 [commit/8a158a9](https://github.com/ytdl-org/youtube-dl/commit/8a158a936c8b002ef536e9e2b778ded02c09c0fa)**<!--([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))--> and **youtube-dlc 2020.11.11-3 [commit/f9401f2](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee)**: You get all the features and patches of [youtube-dlc](https://github.com/blackjack4494/yt-dlc) in addition to the latest [youtube-dl](https://github.com/ytdl-org/youtube-dl)
 
 * **[SponsorBlock Integration](#sponsorblock-options)**: You can mark/remove sponsor sections in youtube videos by utilizing the [SponsorBlock](https://sponsor.ajay.app) API
 
@@ -79,13 +79,18 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
 
 * **Merged with animelover1984/youtube-dl**: You get most of the features and improvements from [animelover1984/youtube-dl](https://github.com/animelover1984/youtube-dl) including `--write-comments`, `BiliBiliSearch`, `BilibiliChannel`, Embedding thumbnail in mp4/ogg/opus, playlist infojson etc. Note that the NicoNico livestreams are not available. See [#31](https://github.com/yt-dlp/yt-dlp/pull/31) for details.
 
-* **YouTube improvements**:
+* **Youtube improvements**:
-    * Supports Clips, Stories (`ytstories:<channel UCID>`), Search (including filters)**\***, YouTube Music Search, Channel-specific search, Search prefixes (`ytsearch:`, `ytsearchdate:`)**\***, Mixes, YouTube Music Albums/Channels ([except self-uploaded music](https://github.com/yt-dlp/yt-dlp/issues/723)), and Feeds (`:ytfav`, `:ytwatchlater`, `:ytsubs`, `:ythistory`, `:ytrec`, `:ytnotif`)
+    * All Feeds (`:ytfav`, `:ytwatchlater`, `:ytsubs`, `:ythistory`, `:ytrec`, `:ytnotif`) and private playlists supports downloading multiple pages of content
-    * Fix for [n-sig based throttling](https://github.com/ytdl-org/youtube-dl/issues/29326) **\***
+    * Search (`ytsearch:`, `ytsearchdate:`), search URLs and in-channel search works
-    * Supports some (but not all) age-gated content without cookies
+    * Mixes supports downloading multiple pages of content
-    * Download livestreams from the start using `--live-from-start` (*experimental*)
+    * Some (but not all) age-gated content can be downloaded without cookies
-    * `255kbps` audio is extracted (if available) from YouTube Music when premium cookies are given
+    * Fix for [n-sig based throttling](https://github.com/ytdl-org/youtube-dl/issues/29326)
     * Redirect channel's home URL automatically to `/video` to preserve the old behaviour
+    * `255kbps` audio is extracted (if available) from youtube music when premium cookies are given
+    * Youtube music Albums, channels etc can be downloaded ([except self-uploaded music](https://github.com/yt-dlp/yt-dlp/issues/723))
+    * Download livestreams from the start using `--live-from-start` (experimental)
+    * Support for downloading stories (`ytstories:<channel UCID>`) 
+    * Support for downloading clips
 
 * **Cookies from browser**: Cookies can be automatically extracted from all major web browsers using `--cookies-from-browser BROWSER[+KEYRING][:PROFILE]`
 
@@ -119,8 +124,6 @@ yt-dlp is a [youtube-dl](https://github.com/ytdl-org/youtube-dl) fork based on t
 
 See [changelog](Changelog.md) or [commits](https://github.com/yt-dlp/yt-dlp/commits) for the full list of changes
 
-Features marked with a **\*** have been back-ported to youtube-dl
-
 ### Differences in default behavior
 
 Some of yt-dlp's default options are different from that of youtube-dl and youtube-dlc:
@@ -147,7 +150,7 @@ Some of yt-dlp's default options are different from that of youtube-dl and youtu
 * Some private fields such as filenames are removed by default from the infojson. Use `--no-clean-infojson` or `--compat-options no-clean-infojson` to revert this
 * When `--embed-subs` and `--write-subs` are used together, the subtitles are written to disk and also embedded in the media file. You can use just `--embed-subs` to embed the subs and automatically delete the separate file. See [#630 (comment)](https://github.com/yt-dlp/yt-dlp/issues/630#issuecomment-893659460) for more info. `--compat-options no-keep-subs` can be used to revert this
 * `certifi` will be used for SSL root certificates, if installed. If you want to use only system certificates, use `--compat-options no-certifi`
-* youtube-dl tries to remove some superfluous punctuations from filenames. While this can sometimes be helpful, it is often undesirable. So yt-dlp tries to keep the fields in the filenames as close to their original values as possible. You can use `--compat-options filename-sanitization` to revert to youtube-dl's behavior
+* youtube-dl tries to remove some superfluous punctuations from filenames. While this can sometimes be helpfull, it is often undesirable. So yt-dlp tries to keep the fields in the filenames as close to their original values as possible. You can use `--compat-options filename-sanitization` to revert to youtube-dl's behavior
 
 For ease of use, a few more compat options are available:
 
@@ -236,7 +239,7 @@ If you [installed using Homebrew](#with-homebrew), run `brew upgrade yt-dlp/taps
 
 File|Description
 :---|:---
-[yt-dlp](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp)|Platform-independent [zipimport](https://docs.python.org/3/library/zipimport.html) binary. Needs Python (recommended for **Linux/BSD**)
+[yt-dlp](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp)|Platform-independant [zipimport](https://docs.python.org/3/library/zipimport.html) binary. Needs Python (recommended for **Linux/BSD**)
 [yt-dlp.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.exe)|Windows (Win7 SP1+) standalone x64 binary (recommended for **Windows**)
 [yt-dlp_macos](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos)|MacOS (10.15+) standalone executable (recommended for **MacOS**)
 
@@ -250,7 +253,6 @@ File|Description
 [yt-dlp_linux.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux.zip)|Unpackaged Unix executable (no auto-update)
 [yt-dlp_win.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_win.zip)|Unpackaged Windows executable (no auto-update)
 [yt-dlp_macos.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos.zip)|Unpackaged MacOS (10.15+) executable (no auto-update)
-[yt-dlp_macos_legacy](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos_legacy)|MacOS (10.9+) standalone x64 executable
 
 #### Misc
 
@@ -431,7 +433,7 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                     "-S=aext:ARG0,abr -x --audio-format ARG0".
                                     All defined aliases are listed in the --help
                                     output. Alias options can trigger more
-                                    aliases; so be careful to avoid defining
+                                    aliases; so be carefull to avoid defining
                                     recursive options. As a safety measure, each
                                     alias may be triggered a maximum of 100
                                     times. This option can be used multiple times
@@ -464,7 +466,7 @@ You can also fork the project on github and run your fork's [build workflow](.gi
                                     explicitly provided IP block in CIDR notation
 
 ## Video Selection:
-    -I, --playlist-items ITEM_SPEC  Comma separated playlist_index of the videos
+    -I, --playlist-items ITEM_SPEC  Comma seperated playlist_index of the videos
                                     to download. You can specify a range using
                                     "[START]:[STOP][:STEP]". For backward
                                     compatibility, START-STOP is also supported.

devscripts/bash-completion.py

@@ -1,12 +1,9 @@
 #!/usr/bin/env python3
-
-# Allow direct execution
 import os
 import sys
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
 import yt_dlp
 
 BASH_COMPLETION_FILE = "completions/bash/yt-dlp"

devscripts/check-porn.py

@@ -13,12 +13,10 @@ import sys
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
-import urllib.parse
-import urllib.request
-
 from test.helper import gettestcases
 
+from yt_dlp.utils import compat_urllib_parse_urlparse, compat_urllib_request
+
 if len(sys.argv) > 1:
     METHOD = 'LIST'
     LIST = open(sys.argv[1]).read().decode('utf8').strip()
@@ -28,7 +26,7 @@ else:
 for test in gettestcases():
     if METHOD == 'EURISTIC':
         try:
-            webpage = urllib.request.urlopen(test['url'], timeout=10).read()
+            webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read()
         except Exception:
             print('\nFail: {}'.format(test['name']))
             continue
@@ -38,7 +36,7 @@ for test in gettestcases():
         RESULT = 'porn' in webpage.lower()
 
     elif METHOD == 'LIST':
-        domain = urllib.parse.urlparse(test['url']).netloc
+        domain = compat_urllib_parse_urlparse(test['url']).netloc
         if not domain:
             print('\nFail: {}'.format(test['name']))
             continue

devscripts/fish-completion.py

@@ -1,14 +1,10 @@
 #!/usr/bin/env python3
-
+import optparse
-# Allow direct execution
 import os
 import sys
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
-import optparse
-
 import yt_dlp
 from yt_dlp.utils import shell_quote
 

devscripts/generate_aes_testdata.py

@@ -1,15 +1,11 @@
 #!/usr/bin/env python3
-
+import codecs
-# Allow direct execution
 import os
+import subprocess
 import sys
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
-import codecs
-import subprocess
-
 from yt_dlp.aes import aes_encrypt, key_expansion
 from yt_dlp.utils import intlist_to_bytes
 

devscripts/make_contributing.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 import optparse
 import re
 

devscripts/make_issue_template.py

@@ -1,12 +1,4 @@
 #!/usr/bin/env python3
-
-# Allow direct execution
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-
 import optparse
 
 
@@ -15,7 +7,7 @@ def read(fname):
         return f.read()
 
 
-# Get the version without importing the package
+# Get the version from yt_dlp/version.py without importing the package
 def read_version(fname):
     exec(compile(read(fname), fname, 'exec'))
     return locals()['__version__']

devscripts/make_lazy_extractors.py

@@ -1,15 +1,12 @@
 #!/usr/bin/env python3
-
+import optparse
-# Allow direct execution
 import os
 import sys
+from inspect import getsource
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
-import optparse
-from inspect import getsource
-
 NO_ATTR = object()
 STATIC_CLASS_PROPERTIES = ['IE_NAME', 'IE_DESC', 'SEARCH_KEY', '_WORKING', '_NETRC_MACHINE', 'age_limit']
 CLASS_METHODS = [

devscripts/make_readme.py

@@ -1,11 +1,7 @@
 #!/usr/bin/env python3
 
-"""
+# yt-dlp --help | make_readme.py
-yt-dlp --help | make_readme.py
+# This must be run in a console of correct width
-This must be run in a console of correct width
-"""
-
-
 import functools
 import re
 import sys

devscripts/make_supportedsites.py

@@ -1,14 +1,10 @@
 #!/usr/bin/env python3
-
+import optparse
-# Allow direct execution
 import os
 import sys
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
-import optparse
-
 from yt_dlp.extractor import list_extractor_classes
 
 

devscripts/prepare_manpage.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 import optparse
 import os.path
 import re
@@ -24,7 +23,7 @@ yt\-dlp \- A youtube-dl fork with additional features and patches
 
 def main():
     parser = optparse.OptionParser(usage='%prog OUTFILE.md')
-    _, args = parser.parse_args()
+    options, args = parser.parse_args()
     if len(args) != 1:
         parser.error('Expected an output filename')
 

devscripts/update-formulae.py

@@ -1,15 +1,12 @@
 #!/usr/bin/env python3
-
+import json
-# Allow direct execution
 import os
+import re
 import sys
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
+from yt_dlp.compat import compat_urllib_request
-import json
-import re
-import urllib.request
 
 # usage: python3 ./devscripts/update-formulae.py <path-to-formulae-rb> <version>
 # version can be either 0-aligned (yt-dlp version) or normalized (PyPl version)
@@ -18,7 +15,7 @@ filename, version = sys.argv[1:]
 
 normalized_version = '.'.join(str(int(x)) for x in version.split('.'))
 
-pypi_release = json.loads(urllib.request.urlopen(
+pypi_release = json.loads(compat_urllib_request.urlopen(
     'https://pypi.org/pypi/yt-dlp/%s/json' % normalized_version
 ).read().decode())
 

devscripts/update-version.py

@@ -1,12 +1,4 @@
 #!/usr/bin/env python3
-
-# Allow direct execution
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-
 import subprocess
 import sys
 from datetime import datetime

devscripts/zsh-completion.py

@@ -1,12 +1,9 @@
 #!/usr/bin/env python3
-
-# Allow direct execution
 import os
 import sys
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
 import yt_dlp
 
 ZSH_COMPLETION_FILE = "completions/zsh/_yt-dlp"

pyinst.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 import os
 import platform
 import sys
@@ -44,7 +43,7 @@ def main():
 
 
 def parse_options():
-    # Compatibility with older arguments
+    # Compatability with older arguments
     opts = sys.argv[1:]
     if opts[0:1] in (['32'], ['64']):
         if ARCH != opts[0]:

requirements.txt

@@ -3,4 +3,4 @@ pycryptodomex
 websockets
 brotli; platform_python_implementation=='CPython'
 brotlicffi; platform_python_implementation!='CPython'
-certifi
+certifi

setup.cfg

@@ -37,5 +37,3 @@ line_length = 80
 reverse_relative = true
 ensure_newline_before_comments = true
 include_trailing_comma = true
-known_first_party =
-    test

setup.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 import os.path
 import sys
 import warnings

supportedsites.md

@@ -418,7 +418,6 @@
  - **Funk**
  - **Fusion**
  - **Fux**
- - **FuyinTV**
  - **Gab**
  - **GabTV**
  - **Gaia**: [<abbr title="netrc machine"><em>gaia</em></abbr>]
@@ -619,7 +618,6 @@
  - **LiveJournal**
  - **livestream**
  - **livestream:original**
- - **Livestreamfails**
  - **Lnk**
  - **LnkGo**
  - **loc**: Library of Congress
@@ -984,7 +982,6 @@
  - **PornoVoisines**
  - **PornoXO**
  - **PornTube**
- - **PremiershipRugby**
  - **PressTV**
  - **ProjectVeritas**
  - **prosiebensat1**: ProSiebenSat.1 Digital
@@ -1116,7 +1113,6 @@
  - **ScreencastOMatic**
  - **ScrippsNetworks**
  - **scrippsnetworks:watch**
- - **Scrolller**
  - **SCTE**: [<abbr title="netrc machine"><em>scte</em></abbr>]
  - **SCTECourse**: [<abbr title="netrc machine"><em>scte</em></abbr>]
  - **Seeker**
@@ -1193,7 +1189,6 @@
  - **stanfordoc**: Stanford Open ClassRoom
  - **startv**
  - **Steam**
- - **SteamCommunityBroadcast**
  - **Stitcher**
  - **StitcherShow**
  - **StoryFire**
@@ -1432,8 +1427,7 @@
  - **vimeo:watchlater**: [<abbr title="netrc machine"><em>vimeo</em></abbr>] Vimeo watch later list, ":vimeowatchlater" keyword (requires authentication)
  - **Vimm:recording**
  - **Vimm:stream**
- - **ViMP**
+ - **Vimp**
- - **ViMP:Playlist**
  - **Vimple**: Vimple - one-click video hosting
  - **Vine**
  - **vine:user**

test/helper.py

@@ -9,7 +9,7 @@ import types
 
 import yt_dlp.extractor
 from yt_dlp import YoutubeDL
-from yt_dlp.compat import compat_os_name
+from yt_dlp.compat import compat_os_name, compat_str
 from yt_dlp.utils import preferredencoding, write_string
 
 if 'pytest' in sys.modules:
@@ -96,29 +96,29 @@ md5 = lambda s: hashlib.md5(s.encode()).hexdigest()
 
 
 def expect_value(self, got, expected, field):
-    if isinstance(expected, str) and expected.startswith('re:'):
+    if isinstance(expected, compat_str) and expected.startswith('re:'):
         match_str = expected[len('re:'):]
         match_rex = re.compile(match_str)
 
         self.assertTrue(
-            isinstance(got, str),
+            isinstance(got, compat_str),
-            f'Expected a {str.__name__} object, but got {type(got).__name__} for field {field}')
+            f'Expected a {compat_str.__name__} object, but got {type(got).__name__} for field {field}')
         self.assertTrue(
             match_rex.match(got),
             f'field {field} (value: {got!r}) should match {match_str!r}')
-    elif isinstance(expected, str) and expected.startswith('startswith:'):
+    elif isinstance(expected, compat_str) and expected.startswith('startswith:'):
         start_str = expected[len('startswith:'):]
         self.assertTrue(
-            isinstance(got, str),
+            isinstance(got, compat_str),
-            f'Expected a {str.__name__} object, but got {type(got).__name__} for field {field}')
+            f'Expected a {compat_str.__name__} object, but got {type(got).__name__} for field {field}')
         self.assertTrue(
             got.startswith(start_str),
             f'field {field} (value: {got!r}) should start with {start_str!r}')
-    elif isinstance(expected, str) and expected.startswith('contains:'):
+    elif isinstance(expected, compat_str) and expected.startswith('contains:'):
         contains_str = expected[len('contains:'):]
         self.assertTrue(
-            isinstance(got, str),
+            isinstance(got, compat_str),
-            f'Expected a {str.__name__} object, but got {type(got).__name__} for field {field}')
+            f'Expected a {compat_str.__name__} object, but got {type(got).__name__} for field {field}')
         self.assertTrue(
             contains_str in got,
             f'field {field} (value: {got!r}) should contain {contains_str!r}')
@@ -142,12 +142,12 @@ def expect_value(self, got, expected, field):
                     index, field, type_expected, type_got))
             expect_value(self, item_got, item_expected, field)
     else:
-        if isinstance(expected, str) and expected.startswith('md5:'):
+        if isinstance(expected, compat_str) and expected.startswith('md5:'):
             self.assertTrue(
-                isinstance(got, str),
+                isinstance(got, compat_str),
                 f'Expected field {field} to be a unicode object, but got value {got!r} of type {type(got)!r}')
             got = 'md5:' + md5(got)
-        elif isinstance(expected, str) and re.match(r'^(?:min|max)?count:\d+', expected):
+        elif isinstance(expected, compat_str) and re.match(r'^(?:min|max)?count:\d+', expected):
             self.assertTrue(
                 isinstance(got, (list, dict)),
                 f'Expected field {field} to be a list or a dict, but it is of type {type(got).__name__}')
@@ -236,7 +236,7 @@ def expect_info_dict(self, got_dict, expected_dict):
     missing_keys = set(test_info_dict.keys()) - set(expected_dict.keys())
     if missing_keys:
         def _repr(v):
-            if isinstance(v, str):
+            if isinstance(v, compat_str):
                 return "'%s'" % v.replace('\\', '\\\\').replace("'", "\\'").replace('\n', '\\n')
             elif isinstance(v, type):
                 return v.__name__

test/test_InfoExtractor.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
 import os
 import sys
@@ -7,12 +6,10 @@ import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
-import http.server
 import threading
-
 from test.helper import FakeYDL, expect_dict, expect_value, http_server_port
-from yt_dlp.compat import compat_etree_fromstring
+
+from yt_dlp.compat import compat_etree_fromstring, compat_http_server
 from yt_dlp.extractor import YoutubeIE, get_info_extractor
 from yt_dlp.extractor.common import InfoExtractor
 from yt_dlp.utils import (
@@ -26,7 +23,7 @@ TEAPOT_RESPONSE_STATUS = 418
 TEAPOT_RESPONSE_BODY = "<h1>418 I'm a teapot</h1>"
 
 
-class InfoExtractorTestRequestHandler(http.server.BaseHTTPRequestHandler):
+class InfoExtractorTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
     def log_message(self, format, *args):
         pass
 
@@ -1658,7 +1655,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
         # or the underlying `_download_webpage_handle` returning no content
         # when a response matches `expected_status`.
 
-        httpd = http.server.HTTPServer(
+        httpd = compat_http_server.HTTPServer(
             ('127.0.0.1', 0), InfoExtractorTestRequestHandler)
         port = http_server_port(httpd)
         server_thread = threading.Thread(target=httpd.serve_forever)

test/test_YoutubeDL.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
 import os
 import sys
@@ -7,14 +6,17 @@ import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
 import copy
 import json
-import urllib.error
-
 from test.helper import FakeYDL, assertRegexpMatches
+
 from yt_dlp import YoutubeDL
-from yt_dlp.compat import compat_os_name
+from yt_dlp.compat import (
+    compat_os_name,
+    compat_setenv,
+    compat_str,
+    compat_urllib_error,
+)
 from yt_dlp.extractor import YoutubeIE
 from yt_dlp.extractor.common import InfoExtractor
 from yt_dlp.postprocessor.common import PostProcessor
@@ -839,14 +841,14 @@ class TestYoutubeDL(unittest.TestCase):
         # test('%(foo|)s', ('', '_'))  # fixme
 
         # Environment variable expansion for prepare_filename
-        os.environ['__yt_dlp_var'] = 'expanded'
+        compat_setenv('__yt_dlp_var', 'expanded')
         envvar = '%__yt_dlp_var%' if compat_os_name == 'nt' else '$__yt_dlp_var'
         test(envvar, (envvar, 'expanded'))
         if compat_os_name == 'nt':
             test('%s%', ('%s%', '%s%'))
-            os.environ['s'] = 'expanded'
+            compat_setenv('s', 'expanded')
             test('%s%', ('%s%', 'expanded'))  # %s% should be expanded before escaping %s
-            os.environ['(test)s'] = 'expanded'
+            compat_setenv('(test)s', 'expanded')
             test('%(test)s%', ('NA%', 'expanded'))  # Environment should take priority over template
 
         # Path expansion and escaping
@@ -1099,7 +1101,7 @@ class TestYoutubeDL(unittest.TestCase):
     def test_urlopen_no_file_protocol(self):
         # see https://github.com/ytdl-org/youtube-dl/issues/8227
         ydl = YDL()
-        self.assertRaises(urllib.error.URLError, ydl.urlopen, 'file:///etc/passwd')
+        self.assertRaises(compat_urllib_error.URLError, ydl.urlopen, 'file:///etc/passwd')
 
     def test_do_not_override_ie_key_in_url_transparent(self):
         ydl = YDL()
@@ -1185,7 +1187,7 @@ class TestYoutubeDL(unittest.TestCase):
 
             def _entries(self):
                 for n in range(3):
-                    video_id = str(n)
+                    video_id = compat_str(n)
                     yield {
                         '_type': 'url_transparent',
                         'ie_key': VideoIE.ie_key(),

test/test_YoutubeDLCookieJar.py

@@ -1,16 +1,12 @@
 #!/usr/bin/env python3
-
-# Allow direct execution
 import os
+import re
 import sys
+import tempfile
 import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
-import re
-import tempfile
-
 from yt_dlp.utils import YoutubeDLCookieJar
 
 

test/test_aes.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
 import os
 import sys
@@ -7,7 +6,6 @@ import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
 import base64
 
 from yt_dlp.aes import (

test/test_age_restriction.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
 import os
 import sys
@@ -7,8 +6,8 @@ import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
 from test.helper import is_download_test, try_rm
+
 from yt_dlp import YoutubeDL
 
 

test/test_all_urls.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
+import collections
 import os
 import sys
 import unittest
@@ -8,9 +8,8 @@ import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
-import collections
-
 from test.helper import gettestcases
+
 from yt_dlp.extractor import FacebookIE, YoutubeIE, gen_extractors
 
 

test/test_cache.py

@@ -1,16 +1,15 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
 import os
+import shutil
 import sys
 import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
-import shutil
-
 from test.helper import FakeYDL
+
 from yt_dlp.cache import Cache
 
 

test/test_compat.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
 import os
 import sys
@@ -8,14 +7,16 @@ import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
-import struct
-import urllib.parse
-
 from yt_dlp import compat
 from yt_dlp.compat import (
     compat_etree_fromstring,
     compat_expanduser,
+    compat_getenv,
+    compat_setenv,
+    compat_str,
+    compat_struct_unpack,
     compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
     compat_urllib_parse_urlencode,
 )
 
@@ -25,19 +26,28 @@ class TestCompat(unittest.TestCase):
         with self.assertWarns(DeprecationWarning):
             compat.compat_basestring
 
-        with self.assertWarns(DeprecationWarning):
-            compat.WINDOWS_VT_MODE
-
         compat.asyncio.events  # Must not raise error
 
+    def test_compat_getenv(self):
+        test_str = 'тест'
+        compat_setenv('yt_dlp_COMPAT_GETENV', test_str)
+        self.assertEqual(compat_getenv('yt_dlp_COMPAT_GETENV'), test_str)
+
+    def test_compat_setenv(self):
+        test_var = 'yt_dlp_COMPAT_SETENV'
+        test_str = 'тест'
+        compat_setenv(test_var, test_str)
+        compat_getenv(test_var)
+        self.assertEqual(compat_getenv(test_var), test_str)
+
     def test_compat_expanduser(self):
         old_home = os.environ.get('HOME')
         test_str = R'C:\Documents and Settings\тест\Application Data'
         try:
-            os.environ['HOME'] = test_str
+            compat_setenv('HOME', test_str)
             self.assertEqual(compat_expanduser('~'), test_str)
         finally:
-            os.environ['HOME'] = old_home or ''
+            compat_setenv('HOME', old_home or '')
 
     def test_compat_urllib_parse_unquote(self):
         self.assertEqual(compat_urllib_parse_unquote('abc%20def'), 'abc def')
@@ -59,8 +69,8 @@ class TestCompat(unittest.TestCase):
             '''(^◣_◢^)っ︻デ═一    ⇀    ⇀    ⇀    ⇀    ⇀    ↶%I%Break%Things%''')
 
     def test_compat_urllib_parse_unquote_plus(self):
-        self.assertEqual(urllib.parse.unquote_plus('abc%20def'), 'abc def')
+        self.assertEqual(compat_urllib_parse_unquote_plus('abc%20def'), 'abc def')
-        self.assertEqual(urllib.parse.unquote_plus('%7e/abc+def'), '~/abc def')
+        self.assertEqual(compat_urllib_parse_unquote_plus('%7e/abc+def'), '~/abc def')
 
     def test_compat_urllib_parse_urlencode(self):
         self.assertEqual(compat_urllib_parse_urlencode({'abc': 'def'}), 'abc=def')
@@ -81,11 +91,11 @@ class TestCompat(unittest.TestCase):
             </root>
         '''
         doc = compat_etree_fromstring(xml.encode())
-        self.assertTrue(isinstance(doc.attrib['foo'], str))
+        self.assertTrue(isinstance(doc.attrib['foo'], compat_str))
-        self.assertTrue(isinstance(doc.attrib['spam'], str))
+        self.assertTrue(isinstance(doc.attrib['spam'], compat_str))
-        self.assertTrue(isinstance(doc.find('normal').text, str))
+        self.assertTrue(isinstance(doc.find('normal').text, compat_str))
-        self.assertTrue(isinstance(doc.find('chinese').text, str))
+        self.assertTrue(isinstance(doc.find('chinese').text, compat_str))
-        self.assertTrue(isinstance(doc.find('foo/bar').text, str))
+        self.assertTrue(isinstance(doc.find('foo/bar').text, compat_str))
 
     def test_compat_etree_fromstring_doctype(self):
         xml = '''<?xml version="1.0"?>
@@ -94,7 +104,7 @@ class TestCompat(unittest.TestCase):
         compat_etree_fromstring(xml)
 
     def test_struct_unpack(self):
-        self.assertEqual(struct.unpack('!B', b'\x00'), (0,))
+        self.assertEqual(compat_struct_unpack('!B', b'\x00'), (0,))
 
 
 if __name__ == '__main__':

test/test_download.py

@@ -1,19 +1,14 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
+import hashlib
+import json
 import os
+import socket
 import sys
 import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
-import hashlib
-import http.client
-import json
-import socket
-import urllib.error
-
 from test.helper import (
     assertGreaterEqual,
     expect_info_dict,
@@ -25,7 +20,12 @@ from test.helper import (
     try_rm,
 )
 
-import yt_dlp.YoutubeDL  # isort: split
+import yt_dlp.YoutubeDL
+from yt_dlp.compat import (
+    compat_http_client,
+    compat_HTTPError,
+    compat_urllib_error,
+)
 from yt_dlp.extractor import get_info_extractor
 from yt_dlp.utils import (
     DownloadError,
@@ -167,7 +167,7 @@ def generator(test_case, tname):
                         force_generic_extractor=params.get('force_generic_extractor', False))
                 except (DownloadError, ExtractorError) as err:
                     # Check if the exception is not a network related one
-                    if not err.exc_info[0] in (urllib.error.URLError, socket.timeout, UnavailableVideoError, http.client.BadStatusLine) or (err.exc_info[0] == urllib.error.HTTPError and err.exc_info[1].code == 503):
+                    if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
                         raise
 
                     if try_num == RETRIES:
@@ -273,11 +273,7 @@ def batch_generator(name, num_tests):
 
     def test_template(self):
         for i in range(num_tests):
-            test_name = f'test_{name}_{i}' if i else f'test_{name}'
+            getattr(self, f'test_{name}_{i}' if i else f'test_{name}')()
-            try:
-                getattr(self, test_name)()
-            except unittest.SkipTest:
-                print(f'Skipped {test_name}')
 
     return test_template
 

test/test_downloader_http.py

@@ -1,19 +1,17 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
 import os
+import re
 import sys
 import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
-import http.server
-import re
 import threading
-
 from test.helper import http_server_port, try_rm
+
 from yt_dlp import YoutubeDL
+from yt_dlp.compat import compat_http_server
 from yt_dlp.downloader.http import HttpFD
 from yt_dlp.utils import encodeFilename
 
@@ -23,7 +21,7 @@ TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 TEST_SIZE = 10 * 1024
 
 
-class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler):
+class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
     def log_message(self, format, *args):
         pass
 
@@ -80,7 +78,7 @@ class FakeLogger:
 
 class TestHttpFD(unittest.TestCase):
     def setUp(self):
-        self.httpd = http.server.HTTPServer(
+        self.httpd = compat_http_server.HTTPServer(
             ('127.0.0.1', 0), HTTPTestRequestHandler)
         self.port = http_server_port(self.httpd)
         self.server_thread = threading.Thread(target=self.httpd.serve_forever)

test/test_execution.py

@@ -1,16 +1,12 @@
 #!/usr/bin/env python3
-
+import contextlib
-# Allow direct execution
 import os
+import subprocess
 import sys
 import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
-import contextlib
-import subprocess
-
 from yt_dlp.utils import encodeArgument
 
 rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

test/test_http.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
 import os
 import sys
@@ -7,19 +6,17 @@ import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
-import http.server
 import ssl
 import threading
-import urllib.request
-
 from test.helper import http_server_port
+
 from yt_dlp import YoutubeDL
+from yt_dlp.compat import compat_http_server, compat_urllib_request
 
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
-class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler):
+class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
     def log_message(self, format, *args):
         pass
 
@@ -56,7 +53,7 @@ class FakeLogger:
 
 class TestHTTP(unittest.TestCase):
     def setUp(self):
-        self.httpd = http.server.HTTPServer(
+        self.httpd = compat_http_server.HTTPServer(
             ('127.0.0.1', 0), HTTPTestRequestHandler)
         self.port = http_server_port(self.httpd)
         self.server_thread = threading.Thread(target=self.httpd.serve_forever)
@@ -67,7 +64,7 @@ class TestHTTP(unittest.TestCase):
 class TestHTTPS(unittest.TestCase):
     def setUp(self):
         certfn = os.path.join(TEST_DIR, 'testcert.pem')
-        self.httpd = http.server.HTTPServer(
+        self.httpd = compat_http_server.HTTPServer(
             ('127.0.0.1', 0), HTTPTestRequestHandler)
         sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
         sslctx.load_cert_chain(certfn, None)
@@ -93,7 +90,7 @@ class TestClientCert(unittest.TestCase):
         certfn = os.path.join(TEST_DIR, 'testcert.pem')
         self.certdir = os.path.join(TEST_DIR, 'testdata', 'certificate')
         cacertfn = os.path.join(self.certdir, 'ca.crt')
-        self.httpd = http.server.HTTPServer(('127.0.0.1', 0), HTTPTestRequestHandler)
+        self.httpd = compat_http_server.HTTPServer(('127.0.0.1', 0), HTTPTestRequestHandler)
         sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
         sslctx.verify_mode = ssl.CERT_REQUIRED
         sslctx.load_verify_locations(cafile=cacertfn)
@@ -133,7 +130,7 @@ class TestClientCert(unittest.TestCase):
 
 
 def _build_proxy_handler(name):
-    class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler):
+    class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
         proxy_name = name
 
         def log_message(self, format, *args):
@@ -149,14 +146,14 @@ def _build_proxy_handler(name):
 
 class TestProxy(unittest.TestCase):
     def setUp(self):
-        self.proxy = http.server.HTTPServer(
+        self.proxy = compat_http_server.HTTPServer(
             ('127.0.0.1', 0), _build_proxy_handler('normal'))
         self.port = http_server_port(self.proxy)
         self.proxy_thread = threading.Thread(target=self.proxy.serve_forever)
         self.proxy_thread.daemon = True
         self.proxy_thread.start()
 
-        self.geo_proxy = http.server.HTTPServer(
+        self.geo_proxy = compat_http_server.HTTPServer(
             ('127.0.0.1', 0), _build_proxy_handler('geo'))
         self.geo_port = http_server_port(self.geo_proxy)
         self.geo_proxy_thread = threading.Thread(target=self.geo_proxy.serve_forever)
@@ -173,7 +170,7 @@ class TestProxy(unittest.TestCase):
         response = ydl.urlopen(url).read().decode()
         self.assertEqual(response, f'normal: {url}')
 
-        req = urllib.request.Request(url)
+        req = compat_urllib_request.Request(url)
         req.add_header('Ytdl-request-proxy', geo_proxy)
         response = ydl.urlopen(req).read().decode()
         self.assertEqual(response, f'geo: {url}')

test/test_iqiyi_sdk_interpreter.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
 import os
 import sys
@@ -7,8 +6,8 @@ import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
 from test.helper import FakeYDL, is_download_test
+
 from yt_dlp.extractor import IqiyiIE
 
 

test/test_jsinterp.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
 import os
 import sys
@@ -7,7 +6,6 @@ import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
 from yt_dlp.jsinterp import JSInterpreter
 
 

test/test_netrc.py

@@ -1,6 +1,3 @@
-#!/usr/bin/env python3
-
-# Allow direct execution
 import os
 import sys
 import unittest

test/test_overwrites.py

@@ -1,15 +1,11 @@
 #!/usr/bin/env python3
-
-# Allow direct execution
 import os
+import subprocess
 import sys
 import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
-import subprocess
-
 from test.helper import is_download_test, try_rm
 
 root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

test/test_post_hooks.py

@@ -1,15 +1,13 @@
 #!/usr/bin/env python3
-
-# Allow direct execution
 import os
 import sys
 import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
 from test.helper import get_params, is_download_test, try_rm
-import yt_dlp.YoutubeDL  # isort: split
+
+import yt_dlp.YoutubeDL
 from yt_dlp.utils import DownloadError
 
 

test/test_postprocessors.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
 import os
 import sys
@@ -7,7 +6,6 @@ import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
 from yt_dlp import YoutubeDL
 from yt_dlp.compat import compat_shlex_quote
 from yt_dlp.postprocessor import (

test/test_socks.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
 import os
 import sys
@@ -7,13 +6,12 @@ import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
 import random
 import subprocess
-import urllib.request
-
 from test.helper import FakeYDL, get_params, is_download_test
 
+from yt_dlp.compat import compat_str, compat_urllib_request
+
 
 @is_download_test
 class TestMultipleSocks(unittest.TestCase):
@@ -53,7 +51,7 @@ class TestMultipleSocks(unittest.TestCase):
         if params is None:
             return
         ydl = FakeYDL()
-        req = urllib.request.Request('http://yt-dl.org/ip')
+        req = compat_urllib_request.Request('http://yt-dl.org/ip')
         req.add_header('Ytdl-request-proxy', params['secondary_proxy'])
         self.assertEqual(
             ydl.urlopen(req).read().decode(),
@@ -64,7 +62,7 @@ class TestMultipleSocks(unittest.TestCase):
         if params is None:
             return
         ydl = FakeYDL()
-        req = urllib.request.Request('https://yt-dl.org/ip')
+        req = compat_urllib_request.Request('https://yt-dl.org/ip')
         req.add_header('Ytdl-request-proxy', params['secondary_proxy'])
         self.assertEqual(
             ydl.urlopen(req).read().decode(),
@@ -101,13 +99,13 @@ class TestSocks(unittest.TestCase):
         return ydl.urlopen('http://yt-dl.org/ip').read().decode()
 
     def test_socks4(self):
-        self.assertTrue(isinstance(self._get_ip('socks4'), str))
+        self.assertTrue(isinstance(self._get_ip('socks4'), compat_str))
 
     def test_socks4a(self):
-        self.assertTrue(isinstance(self._get_ip('socks4a'), str))
+        self.assertTrue(isinstance(self._get_ip('socks4a'), compat_str))
 
     def test_socks5(self):
-        self.assertTrue(isinstance(self._get_ip('socks5'), str))
+        self.assertTrue(isinstance(self._get_ip('socks5'), compat_str))
 
 
 if __name__ == '__main__':

test/test_subtitles.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
 import os
 import sys
@@ -7,8 +6,8 @@ import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
 from test.helper import FakeYDL, is_download_test, md5
+
 from yt_dlp.extractor import (
     NPOIE,
     NRKTVIE,

test/test_update.py.disabled

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
 import os
 import sys

test/test_utils.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
+import contextlib
 import os
 import sys
 import unittest
@@ -8,16 +8,19 @@ import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
-import contextlib
+# Various small unit tests
 import io
 import itertools
 import json
 import xml.etree.ElementTree
 
 from yt_dlp.compat import (
+    compat_chr,
     compat_etree_fromstring,
+    compat_getenv,
     compat_HTMLParseError,
     compat_os_name,
+    compat_setenv,
 )
 from yt_dlp.utils import (
     Config,
@@ -263,20 +266,20 @@ class TestUtil(unittest.TestCase):
         def env(var):
             return f'%{var}%' if sys.platform == 'win32' else f'${var}'
 
-        os.environ['yt_dlp_EXPATH_PATH'] = 'expanded'
+        compat_setenv('yt_dlp_EXPATH_PATH', 'expanded')
         self.assertEqual(expand_path(env('yt_dlp_EXPATH_PATH')), 'expanded')
 
         old_home = os.environ.get('HOME')
         test_str = R'C:\Documents and Settings\тест\Application Data'
         try:
-            os.environ['HOME'] = test_str
+            compat_setenv('HOME', test_str)
-            self.assertEqual(expand_path(env('HOME')), os.getenv('HOME'))
+            self.assertEqual(expand_path(env('HOME')), compat_getenv('HOME'))
-            self.assertEqual(expand_path('~'), os.getenv('HOME'))
+            self.assertEqual(expand_path('~'), compat_getenv('HOME'))
             self.assertEqual(
                 expand_path('~/%s' % env('yt_dlp_EXPATH_PATH')),
-                '%s/expanded' % os.getenv('HOME'))
+                '%s/expanded' % compat_getenv('HOME'))
         finally:
-            os.environ['HOME'] = old_home or ''
+            compat_setenv('HOME', old_home or '')
 
     def test_prepend_extension(self):
         self.assertEqual(prepend_extension('abc.ext', 'temp'), 'abc.temp.ext')
@@ -1125,7 +1128,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(extract_attributes('<e x="décompose&#769;">'), {'x': 'décompose\u0301'})
         # "Narrow" Python builds don't support unicode code points outside BMP.
         try:
-            chr(0x10000)
+            compat_chr(0x10000)
             supports_outside_bmp = True
         except ValueError:
             supports_outside_bmp = False

test/test_verbose_output.py

@@ -1,15 +1,11 @@
 #!/usr/bin/env python3
-
-# Allow direct execution
 import os
+import subprocess
 import sys
 import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
-import subprocess
-
 rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 

test/test_write_annotations.py.disabled

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
 import os
 import sys
@@ -7,12 +6,11 @@ import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
 import xml.etree.ElementTree
+from test.helper import get_params, is_download_test, try_rm
 
 import yt_dlp.extractor
 import yt_dlp.YoutubeDL
-from test.helper import get_params, is_download_test, try_rm
 
 
 class YoutubeDL(yt_dlp.YoutubeDL):

test/test_youtube_lists.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
 import os
 import sys
@@ -7,8 +6,8 @@ import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
 from test.helper import FakeYDL, is_download_test
+
 from yt_dlp.extractor import YoutubeIE, YoutubeTabIE
 
 

test/test_youtube_misc.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
 import os
 import sys

test/test_youtube_signature.py

@@ -1,19 +1,18 @@
 #!/usr/bin/env python3
-
 # Allow direct execution
+import contextlib
 import os
 import sys
 import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
-import contextlib
 import re
 import string
 import urllib.request
-
 from test.helper import FakeYDL, is_download_test
+
+from yt_dlp.compat import compat_str
 from yt_dlp.extractor import YoutubeIE
 from yt_dlp.jsinterp import JSInterpreter
 
@@ -158,7 +157,7 @@ def t_factory(name, sig_func, url_pattern):
 def signature(jscode, sig_input):
     func = YoutubeIE(FakeYDL())._parse_sig_js(jscode)
     src_sig = (
-        str(string.printable[:sig_input])
+        compat_str(string.printable[:sig_input])
         if isinstance(sig_input, int) else sig_input)
     return func(src_sig)
 

yt_dlp/YoutubeDL.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 import collections
 import contextlib
 import datetime
@@ -10,6 +11,7 @@ import json
 import locale
 import operator
 import os
+import platform
 import random
 import re
 import shutil
@@ -24,8 +26,15 @@ import urllib.request
 from string import ascii_letters
 
 from .cache import Cache
-from .compat import HAS_LEGACY as compat_has_legacy
+from .compat import (
-from .compat import compat_os_name, compat_shlex_quote
+    HAS_LEGACY as compat_has_legacy,
+    compat_get_terminal_size,
+    compat_os_name,
+    compat_shlex_quote,
+    compat_str,
+    compat_urllib_error,
+    compat_urllib_request,
+)
 from .cookies import load_cookies
 from .downloader import FFmpegFD, get_suitable_downloader, shorten_protocol_name
 from .downloader.rtmp import rtmpdump_version
@@ -109,6 +118,7 @@ from .utils import (
     number_of_digits,
     orderedSet,
     parse_filesize,
+    platform_name,
     preferredencoding,
     prepend_extension,
     register_socks_protocols,
@@ -124,7 +134,6 @@ from .utils import (
     strftime_or_none,
     subtitles_filename,
     supports_terminal_sequences,
-    system_identifier,
     timetuple_from_msec,
     to_high_limit_path,
     traverse_obj,
@@ -576,9 +585,7 @@ class YoutubeDL:
         MIN_SUPPORTED, MIN_RECOMMENDED = (3, 6), (3, 7)
         current_version = sys.version_info[:2]
         if current_version < MIN_RECOMMENDED:
-            msg = ('Support for Python version %d.%d has been deprecated. '
+            msg = 'Support for Python version %d.%d has been deprecated and will break in future versions of yt-dlp'
-                   'See  https://github.com/yt-dlp/yt-dlp/issues/3764  for more details. '
-                   'You will recieve only one more update on this version')
             if current_version < MIN_SUPPORTED:
                 msg = 'Python version %d.%d is no longer supported'
             self.deprecation_warning(
@@ -637,7 +644,7 @@ class YoutubeDL:
             try:
                 import pty
                 master, slave = pty.openpty()
-                width = shutil.get_terminal_size().columns
+                width = compat_get_terminal_size().columns
                 width_args = [] if width is None else ['-w', str(width)]
                 sp_kwargs = {'stdin': subprocess.PIPE, 'stdout': slave, 'stderr': self._out_files.error}
                 try:
@@ -792,7 +799,7 @@ class YoutubeDL:
             return message
 
         assert hasattr(self, '_output_process')
-        assert isinstance(message, str)
+        assert isinstance(message, compat_str)
         line_count = message.count('\n') + 1
         self._output_process.stdin.write((message + '\n').encode())
         self._output_process.stdin.flush()
@@ -828,7 +835,7 @@ class YoutubeDL:
 
     def to_stderr(self, message, only_once=False):
         """Print message to stderr"""
-        assert isinstance(message, str)
+        assert isinstance(message, compat_str)
         if self.params.get('logger'):
             self.params['logger'].error(message)
         else:
@@ -1563,7 +1570,7 @@ class YoutubeDL:
             additional_urls = (ie_result or {}).get('additional_urls')
             if additional_urls:
                 # TODO: Improve MetadataParserPP to allow setting a list
-                if isinstance(additional_urls, str):
+                if isinstance(additional_urls, compat_str):
                     additional_urls = [additional_urls]
                 self.to_screen(
                     '[info] %s: %d additional URL(s) requested' % (ie_result['id'], len(additional_urls)))
@@ -1725,7 +1732,7 @@ class YoutubeDL:
                 resolved_entries.append((playlist_index, entry))
 
             # TODO: Add auto-generated fields
-            if not entry or self._match_entry(entry, incomplete=True) is not None:
+            if self._match_entry(entry, incomplete=True) is not None:
                 continue
 
             self.to_screen('[download] Downloading video %s of %s' % (
@@ -2356,10 +2363,10 @@ class YoutubeDL:
 
         def sanitize_string_field(info, string_field):
             field = info.get(string_field)
-            if field is None or isinstance(field, str):
+            if field is None or isinstance(field, compat_str):
                 return
             report_force_conversion(string_field, 'a string', 'string')
-            info[string_field] = str(field)
+            info[string_field] = compat_str(field)
 
         def sanitize_numeric_fields(info):
             for numeric_field in self._NUMERIC_FIELDS:
@@ -2376,15 +2383,6 @@ class YoutubeDL:
         if (info_dict.get('duration') or 0) <= 0 and info_dict.pop('duration', None):
             self.report_warning('"duration" field is negative, there is an error in extractor')
 
-        chapters = info_dict.get('chapters') or []
-        dummy_chapter = {'end_time': 0, 'start_time': info_dict.get('duration')}
-        for prev, current, next_ in zip(
-                (dummy_chapter, *chapters), chapters, (*chapters[1:], dummy_chapter)):
-            if current.get('start_time') is None:
-                current['start_time'] = prev.get('end_time')
-            if not current.get('end_time'):
-                current['end_time'] = next_.get('start_time')
-
         if 'playlist' not in info_dict:
             # It isn't part of a playlist
             info_dict['playlist'] = None
@@ -2471,7 +2469,7 @@ class YoutubeDL:
             sanitize_numeric_fields(format)
             format['url'] = sanitize_url(format['url'])
             if not format.get('format_id'):
-                format['format_id'] = str(i)
+                format['format_id'] = compat_str(i)
             else:
                 # Sanitize format_id from characters used in format selector expression
                 format['format_id'] = re.sub(r'[\s,/+\[\]()]', '_', format['format_id'])
@@ -2621,7 +2619,7 @@ class YoutubeDL:
                 if chapter or offset:
                     new_info.update({
                         'section_start': offset + chapter.get('start_time', 0),
-                        'section_end': offset + min(chapter.get('end_time', duration), duration),
+                        'section_end': offset + min(chapter.get('end_time', 0), duration),
                         'section_title': chapter.get('title'),
                         'section_number': chapter.get('index'),
                     })
@@ -3533,7 +3531,7 @@ class YoutubeDL:
                     'none', '' if f.get('vcodec') == 'none'
                             else self._format_out('video only', self.Styles.SUPPRESS)),
                 format_field(f, 'abr', '\t%dk'),
-                format_field(f, 'asr', '\t%s', func=format_decimal_suffix),
+                format_field(f, 'asr', '\t%dHz'),
                 join_nonempty(
                     self._format_out('UNSUPPORTED', 'light red') if f.get('ext') in ('f4f', 'f4m') else None,
                     format_field(f, 'language', '[%s]'),
@@ -3657,7 +3655,17 @@ class YoutubeDL:
                 with contextlib.suppress(Exception):
                     sys.exc_clear()
 
-        write_debug(system_identifier())
+        def python_implementation():
+            impl_name = platform.python_implementation()
+            if impl_name == 'PyPy' and hasattr(sys, 'pypy_version_info'):
+                return impl_name + ' version %d.%d.%d' % sys.pypy_version_info[:3]
+            return impl_name
+
+        write_debug('Python version %s (%s %s) - %s' % (
+            platform.python_version(),
+            python_implementation(),
+            platform.architecture()[0],
+            platform_name()))
 
         exe_versions, ffmpeg_features = FFmpegPostProcessor.get_versions_and_features(self)
         ffmpeg_features = {key for key, val in ffmpeg_features.items() if val}
@@ -3716,7 +3724,7 @@ class YoutubeDL:
             else:
                 proxies = {'http': opts_proxy, 'https': opts_proxy}
         else:
-            proxies = urllib.request.getproxies()
+            proxies = compat_urllib_request.getproxies()
             # Set HTTPS proxy to HTTP one if given (https://github.com/ytdl-org/youtube-dl/issues/805)
             if 'http' in proxies and 'https' not in proxies:
                 proxies['https'] = proxies['http']
@@ -3732,13 +3740,13 @@ class YoutubeDL:
         # default FileHandler and allows us to disable the file protocol, which
         # can be used for malicious purposes (see
         # https://github.com/ytdl-org/youtube-dl/issues/8227)
-        file_handler = urllib.request.FileHandler()
+        file_handler = compat_urllib_request.FileHandler()
 
         def file_open(*args, **kwargs):
-            raise urllib.error.URLError('file:// scheme is explicitly disabled in yt-dlp for security reasons')
+            raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in yt-dlp for security reasons')
         file_handler.file_open = file_open
 
-        opener = urllib.request.build_opener(
+        opener = compat_urllib_request.build_opener(
             proxy_handler, https_handler, cookie_processor, ydlh, redirect_handler, data_handler, file_handler)
 
         # Delete the default user-agent header, which would otherwise apply in

yt_dlp/__init__.py

@@ -1,15 +1,15 @@
+#!/usr/bin/env python3
 f'You are using an unsupported version of Python. Only Python versions 3.6 and above are supported by yt-dlp'  # noqa: F541
 
 __license__ = 'Public Domain'
 
-import getpass
 import itertools
 import optparse
 import os
 import re
 import sys
 
-from .compat import compat_shlex_quote
+from .compat import compat_getpass, compat_shlex_quote
 from .cookies import SUPPORTED_BROWSERS, SUPPORTED_KEYRINGS
 from .downloader import FileDownloader
 from .downloader.external import get_external_downloader
@@ -403,8 +403,6 @@ def validate_options(opts):
 
     default_downloader = None
     for proto, path in opts.external_downloader.items():
-        if path == 'native':
-            continue
         ed = get_external_downloader(path)
         if ed is None:
             raise ValueError(
@@ -531,9 +529,9 @@ def validate_options(opts):
 
     # Ask for passwords
     if opts.username is not None and opts.password is None:
-        opts.password = getpass.getpass('Type account password and press [Return]: ')
+        opts.password = compat_getpass('Type account password and press [Return]: ')
     if opts.ap_username is not None and opts.ap_password is None:
-        opts.ap_password = getpass.getpass('Type TV provider account password and press [Return]: ')
+        opts.ap_password = compat_getpass('Type TV provider account password and press [Return]: ')
 
     return warnings, deprecation_warnings
 

yt_dlp/__main__.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 # Execute with
 # $ python -m yt_dlp
 

yt_dlp/aes.py

@@ -1,7 +1,6 @@
-import base64
 from math import ceil
 
-from .compat import compat_ord
+from .compat import compat_b64decode, compat_ord
 from .dependencies import Cryptodome_AES
 from .utils import bytes_to_intlist, intlist_to_bytes
 
@@ -265,7 +264,7 @@ def aes_decrypt_text(data, password, key_size_bytes):
     """
     NONCE_LENGTH_BYTES = 8
 
-    data = bytes_to_intlist(base64.b64decode(data))
+    data = bytes_to_intlist(compat_b64decode(data))
     password = bytes_to_intlist(password.encode())
 
     key = password[:key_size_bytes] + [0] * (key_size_bytes - len(password))

yt_dlp/cache.py

@@ -6,6 +6,7 @@ import re
 import shutil
 import traceback
 
+from .compat import compat_getenv
 from .utils import expand_path, write_json_file
 
 
@@ -16,7 +17,7 @@ class Cache:
     def _get_root_dir(self):
         res = self._ydl.params.get('cachedir')
         if res is None:
-            cache_root = os.getenv('XDG_CACHE_HOME', '~/.cache')
+            cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache')
             res = os.path.join(cache_root, 'yt-dlp')
         return expand_path(res)
 

yt_dlp/compat/__init__.py

@@ -7,6 +7,7 @@ from . import re
 from ._deprecated import *  # noqa: F401, F403
 from .compat_utils import passthrough_module
 
+
 # XXX: Implement this the same way as other DeprecationWarnings without circular import
 try:
     passthrough_module(__name__, '._legacy', callback=lambda attr: warnings.warn(

yt_dlp/compat/_deprecated.py

@@ -1,16 +1,52 @@
 """Deprecated - New code should avoid these"""
 
 import base64
-import urllib.error
+import getpass
-import urllib.parse
+import html
-
+import html.parser
-compat_str = str
+import http
+import http.client
+import http.cookiejar
+import http.cookies
+import http.server
+import itertools
+import os
+import shutil
+import struct
+import tokenize
+import urllib
 
 compat_b64decode = base64.b64decode
-
+compat_chr = chr
+compat_cookiejar = http.cookiejar
+compat_cookiejar_Cookie = http.cookiejar.Cookie
+compat_cookies_SimpleCookie = http.cookies.SimpleCookie
+compat_get_terminal_size = shutil.get_terminal_size
+compat_getenv = os.getenv
+compat_getpass = getpass.getpass
+compat_html_entities = html.entities
+compat_html_entities_html5 = html.entities.html5
+compat_HTMLParser = html.parser.HTMLParser
+compat_http_client = http.client
+compat_http_server = http.server
 compat_HTTPError = urllib.error.HTTPError
-compat_urlparse = urllib.parse
+compat_itertools_count = itertools.count
 compat_parse_qs = urllib.parse.parse_qs
+compat_str = str
+compat_struct_pack = struct.pack
+compat_struct_unpack = struct.unpack
+compat_tokenize_tokenize = tokenize.tokenize
+compat_urllib_error = urllib.error
 compat_urllib_parse_unquote = urllib.parse.unquote
+compat_urllib_parse_unquote_plus = urllib.parse.unquote_plus
 compat_urllib_parse_urlencode = urllib.parse.urlencode
 compat_urllib_parse_urlparse = urllib.parse.urlparse
+compat_urllib_request = urllib.request
+compat_urlparse = compat_urllib_parse = urllib.parse
+
+
+def compat_setenv(key, value, env=os.environ):
+    env[key] = value
+
+
+__all__ = [x for x in globals() if x.startswith('compat_')]

yt_dlp/compat/_legacy.py

@@ -2,27 +2,18 @@
 
 import collections
 import ctypes
-import getpass
+import http
-import html.entities
-import html.parser
 import http.client
 import http.cookiejar
 import http.cookies
 import http.server
-import itertools
-import os
 import shlex
-import shutil
 import socket
 import struct
-import tokenize
+import urllib
-import urllib.error
-import urllib.parse
-import urllib.request
 import xml.etree.ElementTree as etree
 from subprocess import DEVNULL
 
-from .compat_utils import passthrough_module  # isort: split
 from .asyncio import run as compat_asyncio_run  # noqa: F401
 from .re import Pattern as compat_Pattern  # noqa: F401
 from .re import match as compat_Match  # noqa: F401
@@ -30,8 +21,6 @@ from ..dependencies import Cryptodome_AES as compat_pycrypto_AES  # noqa: F401
 from ..dependencies import brotli as compat_brotli  # noqa: F401
 from ..dependencies import websockets as compat_websockets  # noqa: F401
 
-passthrough_module(__name__, '...utils', ('WINDOWS_VT_MODE', 'windows_enable_vt_mode'))
-
 
 # compat_ctypes_WINFUNCTYPE = ctypes.WINFUNCTYPE
 # will not work since ctypes.WINFUNCTYPE does not exist in UNIX machines
@@ -39,31 +28,14 @@ def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
     return ctypes.WINFUNCTYPE(*args, **kwargs)
 
 
-def compat_setenv(key, value, env=os.environ):
-    env[key] = value
-
-
 compat_basestring = str
-compat_chr = chr
 compat_collections_abc = collections.abc
-compat_cookiejar = http.cookiejar
-compat_cookiejar_Cookie = http.cookiejar.Cookie
 compat_cookies = http.cookies
-compat_cookies_SimpleCookie = http.cookies.SimpleCookie
 compat_etree_Element = etree.Element
 compat_etree_register_namespace = etree.register_namespace
 compat_filter = filter
-compat_get_terminal_size = shutil.get_terminal_size
-compat_getenv = os.getenv
-compat_getpass = getpass.getpass
-compat_html_entities = html.entities
-compat_html_entities_html5 = html.entities.html5
-compat_HTMLParser = html.parser.HTMLParser
-compat_http_client = http.client
-compat_http_server = http.server
 compat_input = input
 compat_integer_types = (int, )
-compat_itertools_count = itertools.count
 compat_kwargs = lambda kwargs: kwargs
 compat_map = map
 compat_numeric_types = (int, float, complex)
@@ -71,18 +43,11 @@ compat_print = print
 compat_shlex_split = shlex.split
 compat_socket_create_connection = socket.create_connection
 compat_Struct = struct.Struct
-compat_struct_pack = struct.pack
-compat_struct_unpack = struct.unpack
 compat_subprocess_get_DEVNULL = lambda: DEVNULL
-compat_tokenize_tokenize = tokenize.tokenize
-compat_urllib_error = urllib.error
-compat_urllib_parse = urllib.parse
 compat_urllib_parse_quote = urllib.parse.quote
 compat_urllib_parse_quote_plus = urllib.parse.quote_plus
-compat_urllib_parse_unquote_plus = urllib.parse.unquote_plus
 compat_urllib_parse_unquote_to_bytes = urllib.parse.unquote_to_bytes
 compat_urllib_parse_urlunparse = urllib.parse.urlunparse
-compat_urllib_request = urllib.request
 compat_urllib_request_DataHandler = urllib.request.DataHandler
 compat_urllib_response = urllib.response
 compat_urlretrieve = urllib.request.urlretrieve
@@ -90,3 +55,10 @@ compat_xml_parse_error = etree.ParseError
 compat_xpath = lambda xpath: xpath
 compat_zip = zip
 workaround_optparse_bug9161 = lambda: None
+
+
+def __getattr__(name):
+    if name in ('WINDOWS_VT_MODE', 'windows_enable_vt_mode'):
+        from .. import utils
+        return getattr(utils, name)
+    raise AttributeError(name)

yt_dlp/compat/compat_utils.py

@@ -4,6 +4,7 @@ import importlib
 import sys
 import types
 
+
 _NO_ATTRIBUTE = object()
 
 _Package = collections.namedtuple('Package', ('name', 'version'))
@@ -30,7 +31,7 @@ def _is_package(module):
     return True
 
 
-def passthrough_module(parent, child, allowed_attributes=None, *, callback=lambda _: None):
+def passthrough_module(parent, child, *, callback=lambda _: None):
     parent_module = importlib.import_module(parent)
     child_module = None  # Import child module only as needed
 
@@ -40,30 +41,22 @@ def passthrough_module(parent, child, allowed_attributes=None, *, callback=lambd
                 with contextlib.suppress(ImportError):
                     return importlib.import_module(f'.{attr}', parent)
 
-            ret = self.__from_child(attr)
-            if ret is _NO_ATTRIBUTE:
-                raise AttributeError(f'module {parent} has no attribute {attr}')
-            callback(attr)
-            return ret
-
-        def __from_child(self, attr):
-            if allowed_attributes is None:
-                if attr.startswith('__') and attr.endswith('__'):
-                    return _NO_ATTRIBUTE
-            elif attr not in allowed_attributes:
-                return _NO_ATTRIBUTE
-
             nonlocal child_module
             child_module = child_module or importlib.import_module(child, parent)
 
+            ret = _NO_ATTRIBUTE
             with contextlib.suppress(AttributeError):
-                return getattr(child_module, attr)
+                ret = getattr(child_module, attr)
 
             if _is_package(child_module):
                 with contextlib.suppress(ImportError):
-                    return importlib.import_module(f'.{attr}', child)
+                    ret = importlib.import_module(f'.{attr}', child)
 
-            return _NO_ATTRIBUTE
+            if ret is _NO_ATTRIBUTE:
+                raise AttributeError(f'module {parent} has no attribute {attr}')
+
+            callback(attr)
+            return ret
 
     # Python 3.6 does not have module level __getattr__
     # https://peps.python.org/pep-0562/

yt_dlp/cookies.py

@@ -1,7 +1,5 @@
-import base64
 import contextlib
 import ctypes
-import http.cookiejar
 import json
 import os
 import shutil
@@ -19,6 +17,7 @@ from .aes import (
     aes_gcm_decrypt_and_verify_bytes,
     unpad_pkcs7,
 )
+from .compat import compat_b64decode, compat_cookiejar_Cookie
 from .dependencies import (
     _SECRETSTORAGE_UNAVAILABLE_REASON,
     secretstorage,
@@ -143,7 +142,7 @@ def _extract_firefox_cookies(profile, logger):
                 total_cookie_count = len(table)
                 for i, (host, name, value, path, expiry, is_secure) in enumerate(table):
                     progress_bar.print(f'Loading cookie {i: 6d}/{total_cookie_count: 6d}')
-                    cookie = http.cookiejar.Cookie(
+                    cookie = compat_cookiejar_Cookie(
                         version=0, name=name, value=value, port=None, port_specified=False,
                         domain=host, domain_specified=bool(host), domain_initial_dot=host.startswith('.'),
                         path=path, path_specified=bool(path), secure=is_secure, expires=expiry, discard=False,
@@ -298,7 +297,7 @@ def _process_chrome_cookie(decryptor, host_key, name, value, encrypted_value, pa
         if value is None:
             return is_encrypted, None
 
-    return is_encrypted, http.cookiejar.Cookie(
+    return is_encrypted, compat_cookiejar_Cookie(
         version=0, name=name, value=value, port=None, port_specified=False,
         domain=host_key, domain_specified=bool(host_key), domain_initial_dot=host_key.startswith('.'),
         path=path, path_specified=bool(path), secure=is_secure, expires=expires_utc, discard=False,
@@ -590,7 +589,7 @@ def _parse_safari_cookies_record(data, jar, logger):
 
     p.skip_to(record_size, 'space at the end of the record')
 
-    cookie = http.cookiejar.Cookie(
+    cookie = compat_cookiejar_Cookie(
         version=0, name=name, value=value, port=None, port_specified=False,
         domain=domain, domain_specified=bool(domain), domain_initial_dot=domain.startswith('.'),
         path=path, path_specified=bool(path), secure=is_secure, expires=expiration_date, discard=False,
@@ -836,7 +835,7 @@ def _get_windows_v10_key(browser_root, logger):
     except KeyError:
         logger.error('no encrypted key in Local State')
         return None
-    encrypted_key = base64.b64decode(base64_key)
+    encrypted_key = compat_b64decode(base64_key)
     prefix = b'DPAPI'
     if not encrypted_key.startswith(prefix):
         logger.error('invalid key')

yt_dlp/dependencies.py

@@ -1,6 +1,6 @@
 # flake8: noqa: F401
 """Imports all optional dependencies for the project.
-An attribute "_yt_dlp__identifier" may be inserted into the module if it uses an ambiguous namespace"""
+An attribute "_yt_dlp__identifier" may be inserted into the module if it uses an ambigious namespace"""
 
 try:
     import brotlicffi as brotli

yt_dlp/downloader/__init__.py

@@ -59,11 +59,10 @@ PROTOCOL_MAP = {
 
 def shorten_protocol_name(proto, simplify=False):
     short_protocol_names = {
-        'm3u8_native': 'm3u8',
+        'm3u8_native': 'm3u8_n',
-        'm3u8': 'm3u8F',
+        'rtmp_ffmpeg': 'rtmp_f',
-        'rtmp_ffmpeg': 'rtmpF',
         'http_dash_segments': 'dash',
-        'http_dash_segments_generator': 'dashG',
+        'http_dash_segments_generator': 'dash_g',
         'niconico_dmc': 'dmc',
         'websocket_frag': 'WSfrag',
     }
@@ -71,7 +70,6 @@ def shorten_protocol_name(proto, simplify=False):
         short_protocol_names.update({
             'https': 'http',
             'ftps': 'ftp',
-            'm3u8': 'm3u8',  # Reverse above m3u8 mapping
             'm3u8_native': 'm3u8',
             'http_dash_segments_generator': 'dash',
             'rtmp_ffmpeg': 'rtmp',

yt_dlp/downloader/external.py

@@ -6,7 +6,8 @@ import sys
 import time
 
 from .fragment import FragmentFD
-from ..compat import functools
+from ..compat import functools  # isort: split
+from ..compat import compat_setenv
 from ..postprocessor.ffmpeg import EXT_TO_OUT_FORMATS, FFmpegPostProcessor
 from ..utils import (
     Popen,
@@ -402,8 +403,8 @@ class FFmpegFD(ExternalFD):
             # We could switch to the following code if we are able to detect version properly
             # args += ['-http_proxy', proxy]
             env = os.environ.copy()
-            env['HTTP_PROXY'] = proxy
+            compat_setenv('HTTP_PROXY', proxy, env=env)
-            env['http_proxy'] = proxy
+            compat_setenv('http_proxy', proxy, env=env)
 
         protocol = info_dict.get('protocol')
 

yt_dlp/downloader/f4m.py

@@ -1,13 +1,17 @@
-import base64
 import io
 import itertools
-import struct
 import time
-import urllib.error
-import urllib.parse
 
 from .fragment import FragmentFD
-from ..compat import compat_etree_fromstring
+from ..compat import (
+    compat_b64decode,
+    compat_etree_fromstring,
+    compat_struct_pack,
+    compat_struct_unpack,
+    compat_urllib_error,
+    compat_urllib_parse_urlparse,
+    compat_urlparse,
+)
 from ..utils import fix_xml_ampersands, xpath_text
 
 
@@ -31,13 +35,13 @@ class FlvReader(io.BytesIO):
 
     # Utility functions for reading numbers and strings
     def read_unsigned_long_long(self):
-        return struct.unpack('!Q', self.read_bytes(8))[0]
+        return compat_struct_unpack('!Q', self.read_bytes(8))[0]
 
     def read_unsigned_int(self):
-        return struct.unpack('!I', self.read_bytes(4))[0]
+        return compat_struct_unpack('!I', self.read_bytes(4))[0]
 
     def read_unsigned_char(self):
-        return struct.unpack('!B', self.read_bytes(1))[0]
+        return compat_struct_unpack('!B', self.read_bytes(1))[0]
 
     def read_string(self):
         res = b''
@@ -199,11 +203,11 @@ def build_fragments_list(boot_info):
 
 
 def write_unsigned_int(stream, val):
-    stream.write(struct.pack('!I', val))
+    stream.write(compat_struct_pack('!I', val))
 
 
 def write_unsigned_int_24(stream, val):
-    stream.write(struct.pack('!I', val)[1:])
+    stream.write(compat_struct_pack('!I', val)[1:])
 
 
 def write_flv_header(stream):
@@ -297,12 +301,12 @@ class F4mFD(FragmentFD):
         # 1. http://live-1-1.rutube.ru/stream/1024/HDS/SD/C2NKsS85HQNckgn5HdEmOQ/1454167650/S-s604419906/move/four/dirs/upper/1024-576p.f4m
         bootstrap_url = node.get('url')
         if bootstrap_url:
-            bootstrap_url = urllib.parse.urljoin(
+            bootstrap_url = compat_urlparse.urljoin(
                 base_url, bootstrap_url)
             boot_info = self._get_bootstrap_from_url(bootstrap_url)
         else:
             bootstrap_url = None
-            bootstrap = base64.b64decode(node.text)
+            bootstrap = compat_b64decode(node.text)
             boot_info = read_bootstrap_info(bootstrap)
         return boot_info, bootstrap_url
 
@@ -332,14 +336,14 @@ class F4mFD(FragmentFD):
         # Prefer baseURL for relative URLs as per 11.2 of F4M 3.0 spec.
         man_base_url = get_base_url(doc) or man_url
 
-        base_url = urllib.parse.urljoin(man_base_url, media.attrib['url'])
+        base_url = compat_urlparse.urljoin(man_base_url, media.attrib['url'])
         bootstrap_node = doc.find(_add_ns('bootstrapInfo'))
         boot_info, bootstrap_url = self._parse_bootstrap_node(
             bootstrap_node, man_base_url)
         live = boot_info['live']
         metadata_node = media.find(_add_ns('metadata'))
         if metadata_node is not None:
-            metadata = base64.b64decode(metadata_node.text)
+            metadata = compat_b64decode(metadata_node.text)
         else:
             metadata = None
 
@@ -367,7 +371,7 @@ class F4mFD(FragmentFD):
             if not live:
                 write_metadata_tag(dest_stream, metadata)
 
-        base_url_parsed = urllib.parse.urlparse(base_url)
+        base_url_parsed = compat_urllib_parse_urlparse(base_url)
 
         self._start_frag_download(ctx, info_dict)
 
@@ -407,7 +411,7 @@ class F4mFD(FragmentFD):
                     if box_type == b'mdat':
                         self._append_fragment(ctx, box_data)
                         break
-            except urllib.error.HTTPError as err:
+            except compat_urllib_error.HTTPError as err:
                 if live and (err.code == 404 or err.code == 410):
                     # We didn't keep up with the live window. Continue
                     # with the next available fragment.

yt_dlp/downloader/fragment.py

@@ -4,14 +4,12 @@ import http.client
 import json
 import math
 import os
-import struct
 import time
-import urllib.error
 
 from .common import FileDownloader
 from .http import HttpFD
 from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
-from ..compat import compat_os_name
+from ..compat import compat_os_name, compat_struct_pack, compat_urllib_error
 from ..utils import (
     DownloadError,
     encodeFilename,
@@ -350,7 +348,7 @@ class FragmentFD(FileDownloader):
             decrypt_info = fragment.get('decrypt_info')
             if not decrypt_info or decrypt_info['METHOD'] != 'AES-128':
                 return frag_content
-            iv = decrypt_info.get('IV') or struct.pack('>8xq', fragment['media_sequence'])
+            iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', fragment['media_sequence'])
             decrypt_info['KEY'] = decrypt_info.get('KEY') or _get_key(info_dict.get('_decryption_key_url') or decrypt_info['URI'])
             # Don't decrypt the content in tests since the data is explicitly truncated and it's not to a valid block
             # size (see https://github.com/ytdl-org/youtube-dl/pull/27660). Tests only care that the correct data downloaded,
@@ -459,7 +457,7 @@ class FragmentFD(FileDownloader):
                     if self._download_fragment(ctx, fragment['url'], info_dict, headers):
                         break
                     return
-                except (urllib.error.HTTPError, http.client.IncompleteRead) as err:
+                except (compat_urllib_error.HTTPError, http.client.IncompleteRead) as err:
                     # Unavailable (possibly temporary) fragments may be served.
                     # First we try to retry then either skip or abort.
                     # See https://github.com/ytdl-org/youtube-dl/issues/10165,

yt_dlp/downloader/hls.py

@@ -1,12 +1,12 @@
 import binascii
 import io
 import re
-import urllib.parse
 
 from . import get_suitable_downloader
 from .external import FFmpegFD
 from .fragment import FragmentFD
 from .. import webvtt
+from ..compat import compat_urlparse
 from ..dependencies import Cryptodome_AES
 from ..utils import bug_reports_message, parse_m3u8_attributes, update_url_query
 
@@ -61,18 +61,12 @@ class HlsFD(FragmentFD):
         s = urlh.read().decode('utf-8', 'ignore')
 
         can_download, message = self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')), None
-        if can_download:
+        if can_download and not Cryptodome_AES and '#EXT-X-KEY:METHOD=AES-128' in s:
-            has_ffmpeg = FFmpegFD.available()
+            if FFmpegFD.available():
-            no_crypto = not Cryptodome_AES and '#EXT-X-KEY:METHOD=AES-128' in s
-            if no_crypto and has_ffmpeg:
                 can_download, message = False, 'The stream has AES-128 encryption and pycryptodomex is not available'
-            elif no_crypto:
+            else:
                 message = ('The stream has AES-128 encryption and neither ffmpeg nor pycryptodomex are available; '
                            'Decryption will be performed natively, but will be extremely slow')
-            elif info_dict.get('extractor_key') == 'Generic' and re.search(r'(?m)#EXT-X-MEDIA-SEQUENCE:(?!0$)', s):
-                install_ffmpeg = '' if has_ffmpeg else 'install ffmpeg and '
-                message = ('Live HLS streams are not supported by the native downloader. If this is a livestream, '
-                           f'please {install_ffmpeg}add "--downloader ffmpeg --hls-use-mpegts" to your command')
         if not can_download:
             has_drm = re.search('|'.join([
                 r'#EXT-X-FAXS-CM:',  # Adobe Flash Access
@@ -146,7 +140,7 @@ class HlsFD(FragmentFD):
         extra_query = None
         extra_param_to_segment_url = info_dict.get('extra_param_to_segment_url')
         if extra_param_to_segment_url:
-            extra_query = urllib.parse.parse_qs(extra_param_to_segment_url)
+            extra_query = compat_urlparse.parse_qs(extra_param_to_segment_url)
         i = 0
         media_sequence = 0
         decrypt_info = {'METHOD': 'NONE'}
@@ -168,7 +162,7 @@ class HlsFD(FragmentFD):
                     frag_url = (
                         line
                         if re.match(r'^https?://', line)
-                        else urllib.parse.urljoin(man_url, line))
+                        else compat_urlparse.urljoin(man_url, line))
                     if extra_query:
                         frag_url = update_url_query(frag_url, extra_query)
 
@@ -193,7 +187,7 @@ class HlsFD(FragmentFD):
                     frag_url = (
                         map_info.get('URI')
                         if re.match(r'^https?://', map_info.get('URI'))
-                        else urllib.parse.urljoin(man_url, map_info.get('URI')))
+                        else compat_urlparse.urljoin(man_url, map_info.get('URI')))
                     if extra_query:
                         frag_url = update_url_query(frag_url, extra_query)
 
@@ -221,7 +215,7 @@ class HlsFD(FragmentFD):
                         if 'IV' in decrypt_info:
                             decrypt_info['IV'] = binascii.unhexlify(decrypt_info['IV'][2:].zfill(32))
                         if not re.match(r'^https?://', decrypt_info['URI']):
-                            decrypt_info['URI'] = urllib.parse.urljoin(
+                            decrypt_info['URI'] = compat_urlparse.urljoin(
                                 man_url, decrypt_info['URI'])
                         if extra_query:
                             decrypt_info['URI'] = update_url_query(decrypt_info['URI'], extra_query)

yt_dlp/downloader/http.py

@@ -1,12 +1,11 @@
-import http.client
 import os
 import random
 import socket
 import ssl
 import time
-import urllib.error
 
 from .common import FileDownloader
+from ..compat import compat_http_client, compat_urllib_error
 from ..utils import (
     ContentTooShortError,
     ThrottledDownload,
@@ -25,7 +24,7 @@ RESPONSE_READ_EXCEPTIONS = (
     socket.timeout,  # compat: py < 3.10
     ConnectionError,
     ssl.SSLError,
-    http.client.HTTPException
+    compat_http_client.HTTPException
 )
 
 
@@ -156,7 +155,7 @@ class HttpFD(FileDownloader):
                     ctx.resume_len = 0
                     ctx.open_mode = 'wb'
                 ctx.data_len = ctx.content_len = int_or_none(ctx.data.info().get('Content-length', None))
-            except urllib.error.HTTPError as err:
+            except compat_urllib_error.HTTPError as err:
                 if err.code == 416:
                     # Unable to resume (requested range not satisfiable)
                     try:
@@ -164,7 +163,7 @@ class HttpFD(FileDownloader):
                         ctx.data = self.ydl.urlopen(
                             sanitized_Request(url, request_data, headers))
                         content_length = ctx.data.info()['Content-Length']
-                    except urllib.error.HTTPError as err:
+                    except compat_urllib_error.HTTPError as err:
                         if err.code < 500 or err.code >= 600:
                             raise
                     else:
@@ -197,7 +196,7 @@ class HttpFD(FileDownloader):
                     # Unexpected HTTP error
                     raise
                 raise RetryDownload(err)
-            except urllib.error.URLError as err:
+            except compat_urllib_error.URLError as err:
                 if isinstance(err.reason, ssl.CertificateError):
                     raise
                 raise RetryDownload(err)

yt_dlp/downloader/ism.py

@@ -2,9 +2,9 @@ import binascii
 import io
 import struct
 import time
-import urllib.error
 
 from .fragment import FragmentFD
+from ..compat import compat_urllib_error
 
 u8 = struct.Struct('>B')
 u88 = struct.Struct('>Bx')
@@ -268,7 +268,7 @@ class IsmFD(FragmentFD):
                         extra_state['ism_track_written'] = True
                     self._append_fragment(ctx, frag_content)
                     break
-                except urllib.error.HTTPError as err:
+                except compat_urllib_error.HTTPError as err:
                     count += 1
                     if count <= fragment_retries:
                         self.report_retry_fragment(err, frag_index, count, fragment_retries)

yt_dlp/downloader/rtmp.py

@@ -4,6 +4,7 @@ import subprocess
 import time
 
 from .common import FileDownloader
+from ..compat import compat_str
 from ..utils import (
     Popen,
     check_executable,
@@ -142,7 +143,7 @@ class RtmpFD(FileDownloader):
         if isinstance(conn, list):
             for entry in conn:
                 basic_args += ['--conn', entry]
-        elif isinstance(conn, str):
+        elif isinstance(conn, compat_str):
             basic_args += ['--conn', conn]
         if protocol is not None:
             basic_args += ['--protocol', protocol]

yt_dlp/downloader/youtube_live_chat.py

@@ -1,8 +1,8 @@
 import json
 import time
-import urllib.error
 
 from .fragment import FragmentFD
+from ..compat import compat_urllib_error
 from ..utils import RegexNotFoundError, dict_get, int_or_none, try_get
 
 
@@ -128,7 +128,7 @@ class YoutubeLiveChatFD(FragmentFD):
                     elif info_dict['protocol'] == 'youtube_live_chat':
                         continuation_id, offset, click_tracking_params = parse_actions_live(live_chat_continuation)
                     return True, continuation_id, offset, click_tracking_params
-                except urllib.error.HTTPError as err:
+                except compat_urllib_error.HTTPError as err:
                     count += 1
                     if count <= fragment_retries:
                         self.report_retry_fragment(err, frag_index, count, fragment_retries)

yt_dlp/extractor/_extractors.py

@@ -563,7 +563,6 @@ from .funimation import (
 )
 from .funk import FunkIE
 from .fusion import FusionIE
-from .fuyintv import FuyinTVIE
 from .gab import (
     GabTVIE,
     GabIE,
@@ -837,7 +836,6 @@ from .livestream import (
     LivestreamOriginalIE,
     LivestreamShortenerIE,
 )
-from .livestreamfails import LivestreamfailsIE
 from .lnkgo import (
     LnkGoIE,
     LnkIE,
@@ -1331,7 +1329,6 @@ from .puhutv import (
     PuhuTVIE,
     PuhuTVSerieIE,
 )
-from .premiershiprugby import PremiershipRugbyIE
 from .presstv import PressTVIE
 from .projectveritas import ProjectVeritasIE
 from .prosiebensat1 import ProSiebenSat1IE
@@ -1512,7 +1509,6 @@ from .scte import (
     SCTEIE,
     SCTECourseIE,
 )
-from .scrolller import ScrolllerIE
 from .seeker import SeekerIE
 from .senategov import SenateISVPIE, SenateGovIE
 from .sendtonews import SendtoNewsIE
@@ -1634,10 +1630,7 @@ from .srgssr import (
 from .srmediathek import SRMediathekIE
 from .stanfordoc import StanfordOpenClassroomIE
 from .startv import StarTVIE
-from .steam import (
+from .steam import SteamIE
-    SteamIE,
-    SteamCommunityBroadcastIE,
-)
 from .storyfire import (
     StoryFireIE,
     StoryFireUserIE,
@@ -1935,10 +1928,7 @@ from .vice import (
 from .vidbit import VidbitIE
 from .viddler import ViddlerIE
 from .videa import VideaIE
-from .videocampus_sachsen import (
+from .videocampus_sachsen import VideocampusSachsenIE
-    VideocampusSachsenIE,
-    ViMPPlaylistIE,
-)
 from .videodetective import VideoDetectiveIE
 from .videofyme import VideofyMeIE
 from .videomore import (

yt_dlp/extractor/abematv.py

@@ -7,13 +7,12 @@ import json
 import re
 import struct
 import time
-import urllib.parse
-import urllib.request
 import urllib.response
 import uuid
 
 from .common import InfoExtractor
 from ..aes import aes_ecb_decrypt
+from ..compat import compat_urllib_parse_urlparse, compat_urllib_request
 from ..utils import (
     ExtractorError,
     bytes_to_intlist,
@@ -34,7 +33,7 @@ def add_opener(ydl, handler):
     ''' Add a handler for opening URLs, like _download_webpage '''
     # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L426
     # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L605
-    assert isinstance(ydl._opener, urllib.request.OpenerDirector)
+    assert isinstance(ydl._opener, compat_urllib_request.OpenerDirector)
     ydl._opener.add_handler(handler)
 
 
@@ -47,7 +46,7 @@ def remove_opener(ydl, handler):
     # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L426
     # https://github.com/python/cpython/blob/main/Lib/urllib/request.py#L605
     opener = ydl._opener
-    assert isinstance(ydl._opener, urllib.request.OpenerDirector)
+    assert isinstance(ydl._opener, compat_urllib_request.OpenerDirector)
     if isinstance(handler, (type, tuple)):
         find_cp = lambda x: isinstance(x, handler)
     else:
@@ -97,20 +96,20 @@ def remove_opener(ydl, handler):
         opener.handlers[:] = [x for x in opener.handlers if not find_cp(x)]
 
 
-class AbemaLicenseHandler(urllib.request.BaseHandler):
+class AbemaLicenseHandler(compat_urllib_request.BaseHandler):
     handler_order = 499
     STRTABLE = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
     HKEY = b'3AF0298C219469522A313570E8583005A642E73EDD58E3EA2FB7339D3DF1597E'
 
     def __init__(self, ie: 'AbemaTVIE'):
-        # the protocol that this should really handle is 'abematv-license://'
+        # the protcol that this should really handle is 'abematv-license://'
         # abematv_license_open is just a placeholder for development purposes
         # ref. https://github.com/python/cpython/blob/f4c03484da59049eb62a9bf7777b963e2267d187/Lib/urllib/request.py#L510
         setattr(self, 'abematv-license_open', getattr(self, 'abematv_license_open'))
         self.ie = ie
 
     def _get_videokey_from_ticket(self, ticket):
-        to_show = self.ie.get_param('verbose', False)
+        to_show = self.ie._downloader.params.get('verbose', False)
         media_token = self.ie._get_media_token(to_show=to_show)
 
         license_response = self.ie._download_json(
@@ -137,7 +136,7 @@ class AbemaLicenseHandler(urllib.request.BaseHandler):
 
     def abematv_license_open(self, url):
         url = request_to_url(url)
-        ticket = urllib.parse.urlparse(url).netloc
+        ticket = compat_urllib_parse_urlparse(url).netloc
         response_data = self._get_videokey_from_ticket(ticket)
         return urllib.response.addinfourl(io.BytesIO(response_data), headers={
             'Content-Length': len(response_data),
@@ -312,7 +311,7 @@ class AbemaTVIE(AbemaTVBaseIE):
 
     def _real_extract(self, url):
         # starting download using infojson from this extractor is undefined behavior,
-        # and never be fixed in the future; you must trigger downloads by directly specifying URL.
+        # and never be fixed in the future; you must trigger downloads by directly specifing URL.
         # (unless there's a way to hook before downloading by extractor)
         video_id, video_type = self._match_valid_url(url).group('id', 'type')
         headers = {

yt_dlp/extractor/adobepass.py

@@ -1,4 +1,3 @@
-import getpass
 import json
 import re
 import time
@@ -6,14 +5,18 @@ import urllib.error
 import xml.etree.ElementTree as etree
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
-from ..utils import (
+    compat_urlparse,
-    NO_DEFAULT,
+    compat_getpass
-    ExtractorError,
-    unescapeHTML,
-    unified_timestamp,
-    urlencode_postdata,
 )
+from ..utils import (
+    unescapeHTML,
+    urlencode_postdata,
+    unified_timestamp,
+    ExtractorError,
+    NO_DEFAULT,
+)
+
 
 MSO_INFO = {
     'DTV': {
@@ -1428,7 +1431,7 @@ class AdobePassIE(InfoExtractor):
         guid = xml_text(resource, 'guid') if '<' in resource else resource
         count = 0
         while count < 2:
-            requestor_info = self.cache.load(self._MVPD_CACHE, requestor_id) or {}
+            requestor_info = self._downloader.cache.load(self._MVPD_CACHE, requestor_id) or {}
             authn_token = requestor_info.get('authn_token')
             if authn_token and is_expired(authn_token, 'simpleTokenExpires'):
                 authn_token = None
@@ -1503,7 +1506,7 @@ class AdobePassIE(InfoExtractor):
                             'send_confirm_link': False,
                             'send_token': True
                         }))
-                    philo_code = getpass.getpass('Type auth code you have received [Return]: ')
+                    philo_code = compat_getpass('Type auth code you have received [Return]: ')
                     self._download_webpage(
                         'https://idp.philo.com/auth/update/login_code', video_id, 'Submitting token', data=urlencode_postdata({
                             'token': philo_code
@@ -1723,12 +1726,12 @@ class AdobePassIE(InfoExtractor):
                         raise_mvpd_required()
                     raise
                 if '<pendingLogout' in session:
-                    self.cache.store(self._MVPD_CACHE, requestor_id, {})
+                    self._downloader.cache.store(self._MVPD_CACHE, requestor_id, {})
                     count += 1
                     continue
                 authn_token = unescapeHTML(xml_text(session, 'authnToken'))
                 requestor_info['authn_token'] = authn_token
-                self.cache.store(self._MVPD_CACHE, requestor_id, requestor_info)
+                self._downloader.cache.store(self._MVPD_CACHE, requestor_id, requestor_info)
 
             authz_token = requestor_info.get(guid)
             if authz_token and is_expired(authz_token, 'simpleTokenTTL'):
@@ -1744,14 +1747,14 @@ class AdobePassIE(InfoExtractor):
                         'userMeta': '1',
                     }), headers=mvpd_headers)
                 if '<pendingLogout' in authorize:
-                    self.cache.store(self._MVPD_CACHE, requestor_id, {})
+                    self._downloader.cache.store(self._MVPD_CACHE, requestor_id, {})
                     count += 1
                     continue
                 if '<error' in authorize:
                     raise ExtractorError(xml_text(authorize, 'details'), expected=True)
                 authz_token = unescapeHTML(xml_text(authorize, 'authzToken'))
                 requestor_info[guid] = authz_token
-                self.cache.store(self._MVPD_CACHE, requestor_id, requestor_info)
+                self._downloader.cache.store(self._MVPD_CACHE, requestor_id, requestor_info)
 
             mvpd_headers.update({
                 'ap_19': xml_text(authn_token, 'simpleSamlNameID'),
@@ -1767,7 +1770,7 @@ class AdobePassIE(InfoExtractor):
                     'hashed_guid': 'false',
                 }), headers=mvpd_headers)
             if '<pendingLogout' in short_authorize:
-                self.cache.store(self._MVPD_CACHE, requestor_id, {})
+                self._downloader.cache.store(self._MVPD_CACHE, requestor_id, {})
                 count += 1
                 continue
             return short_authorize

yt_dlp/extractor/archiveorg.py

@@ -1,34 +1,36 @@
-import json
 import re
-import urllib.parse
+import json
-
 from .common import InfoExtractor
-from .youtube import YoutubeBaseInfoExtractor, YoutubeIE
+from .youtube import YoutubeIE, YoutubeBaseInfoExtractor
-from ..compat import compat_HTTPError, compat_urllib_parse_unquote
+from ..compat import (
+    compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
+    compat_HTTPError
+)
 from ..utils import (
-    KNOWN_EXTENSIONS,
-    ExtractorError,
-    HEADRequest,
     bug_reports_message,
     clean_html,
     dict_get,
     extract_attributes,
+    ExtractorError,
     get_element_by_id,
+    HEADRequest,
     int_or_none,
     join_nonempty,
+    KNOWN_EXTENSIONS,
     merge_dicts,
     mimetype2ext,
     orderedSet,
     parse_duration,
     parse_qs,
-    str_or_none,
     str_to_int,
+    str_or_none,
     traverse_obj,
     try_get,
     unified_strdate,
     unified_timestamp,
-    url_or_none,
     urlhandle_detect_ext,
+    url_or_none
 )
 
 
@@ -141,7 +143,7 @@ class ArchiveOrgIE(InfoExtractor):
         return json.loads(extract_attributes(element)['value'])
 
     def _real_extract(self, url):
-        video_id = urllib.parse.unquote_plus(self._match_id(url))
+        video_id = compat_urllib_parse_unquote_plus(self._match_id(url))
         identifier, entry_id = (video_id.split('/', 1) + [None])[:2]
 
         # Archive.org metadata API doesn't clearly demarcate playlist entries

yt_dlp/extractor/audius.py

@@ -1,8 +1,8 @@
 import random
 
 from .common import InfoExtractor
-from ..compat import compat_str, compat_urllib_parse_unquote
+from ..utils import ExtractorError, try_get, compat_str, str_or_none
-from ..utils import ExtractorError, str_or_none, try_get
+from ..compat import compat_urllib_parse_unquote
 
 
 class AudiusBaseIE(InfoExtractor):

yt_dlp/extractor/bbc.py

@@ -1,12 +1,16 @@
+import xml.etree.ElementTree
 import functools
 import itertools
 import json
 import re
-import urllib.error
-import xml.etree.ElementTree
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError, compat_str, compat_urlparse
+from ..compat import (
+    compat_HTTPError,
+    compat_str,
+    compat_urllib_error,
+    compat_urlparse,
+)
 from ..utils import (
     ExtractorError,
     OnDemandPagedList,
@@ -387,7 +391,7 @@ class BBCCoUkIE(InfoExtractor):
                                 href, programme_id, ext='mp4', entry_protocol='m3u8_native',
                                 m3u8_id=format_id, fatal=False)
                         except ExtractorError as e:
-                            if not (isinstance(e.exc_info[1], urllib.error.HTTPError)
+                            if not (isinstance(e.exc_info[1], compat_urllib_error.HTTPError)
                                     and e.exc_info[1].code in (403, 404)):
                                 raise
                             fmts = []

yt_dlp/extractor/brightcove.py

@@ -600,9 +600,9 @@ class BrightcoveNewIE(AdobePassIE):
         account_id, player_id, embed, content_type, video_id = self._match_valid_url(url).groups()
 
         policy_key_id = '%s_%s' % (account_id, player_id)
-        policy_key = self.cache.load('brightcove', policy_key_id)
+        policy_key = self._downloader.cache.load('brightcove', policy_key_id)
         policy_key_extracted = False
-        store_pk = lambda x: self.cache.store('brightcove', policy_key_id, x)
+        store_pk = lambda x: self._downloader.cache.store('brightcove', policy_key_id, x)
 
         def extract_policy_key():
             base_url = 'http://players.brightcove.net/%s/%s_%s/' % (account_id, player_id, embed)

yt_dlp/extractor/cbc.py

@@ -304,13 +304,13 @@ class CBCGemIE(InfoExtractor):
     def _get_claims_token(self, email, password):
         if not self.claims_token_valid():
             self._claims_token = self._new_claims_token(email, password)
-            self.cache.store(self._NETRC_MACHINE, 'claims_token', self._claims_token)
+            self._downloader.cache.store(self._NETRC_MACHINE, 'claims_token', self._claims_token)
         return self._claims_token
 
     def _real_initialize(self):
         if self.claims_token_valid():
             return
-        self._claims_token = self.cache.load(self._NETRC_MACHINE, 'claims_token')
+        self._claims_token = self._downloader.cache.load(self._NETRC_MACHINE, 'claims_token')
 
     def _find_secret_formats(self, formats, video_id):
         """ Find a valid video url and convert it to the secret variant """

yt_dlp/extractor/cda.py

@@ -1,9 +1,13 @@
 import codecs
-import json
 import re
+import json
 
 from .common import InfoExtractor
-from ..compat import compat_ord, compat_urllib_parse_unquote
+from ..compat import (
+    compat_chr,
+    compat_ord,
+    compat_urllib_parse_unquote,
+)
 from ..utils import (
     ExtractorError,
     float_or_none,
@@ -12,8 +16,8 @@ from ..utils import (
     multipart_encode,
     parse_duration,
     random_birthday,
-    try_get,
     urljoin,
+    try_get,
 )
 
 
@@ -140,7 +144,7 @@ class CDAIE(InfoExtractor):
             b = []
             for c in a:
                 f = compat_ord(c)
-                b.append(chr(33 + (f + 14) % 94) if 33 <= f <= 126 else chr(f))
+                b.append(compat_chr(33 + (f + 14) % 94) if 33 <= f <= 126 else compat_chr(f))
             a = ''.join(b)
             a = a.replace('.cda.mp4', '')
             for p in ('.2cda.pl', '.3cda.pl'):

yt_dlp/extractor/chingari.py

@@ -1,11 +1,11 @@
 import itertools
 import json
-import urllib.parse
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote_plus
 from ..utils import (
-    ExtractorError,
     clean_html,
+    ExtractorError,
     int_or_none,
     str_to_int,
     url_or_none,
@@ -47,8 +47,8 @@ class ChingariBaseIE(InfoExtractor):
             'id': id,
             'extractor_key': ChingariIE.ie_key(),
             'extractor': 'Chingari',
-            'title': urllib.parse.unquote_plus(clean_html(post_data.get('caption'))),
+            'title': compat_urllib_parse_unquote_plus(clean_html(post_data.get('caption'))),
-            'description': urllib.parse.unquote_plus(clean_html(post_data.get('caption'))),
+            'description': compat_urllib_parse_unquote_plus(clean_html(post_data.get('caption'))),
             'duration': media_data.get('duration'),
             'thumbnail': url_or_none(thumbnail),
             'like_count': post_data.get('likeCount'),

yt_dlp/extractor/common.py

@@ -1,10 +1,6 @@
 import base64
 import collections
-import getpass
 import hashlib
-import http.client
-import http.cookiejar
-import http.cookies
 import itertools
 import json
 import math
@@ -13,12 +9,24 @@ import os
 import random
 import sys
 import time
-import urllib.parse
-import urllib.request
 import xml.etree.ElementTree
 
 from ..compat import functools, re  # isort: split
-from ..compat import compat_etree_fromstring, compat_expanduser, compat_os_name
+from ..compat import (
+    compat_cookiejar_Cookie,
+    compat_cookies_SimpleCookie,
+    compat_etree_fromstring,
+    compat_expanduser,
+    compat_getpass,
+    compat_http_client,
+    compat_os_name,
+    compat_str,
+    compat_urllib_error,
+    compat_urllib_parse_unquote,
+    compat_urllib_parse_urlencode,
+    compat_urllib_request,
+    compat_urlparse,
+)
 from ..downloader import FileDownloader
 from ..downloader.f4m import get_base_url, remove_encrypted_media
 from ..utils import (
@@ -63,7 +71,6 @@ from ..utils import (
     str_to_int,
     strip_or_none,
     traverse_obj,
-    try_call,
     try_get,
     unescapeHTML,
     unified_strdate,
@@ -392,7 +399,7 @@ class InfoExtractor:
     There must be a key "entries", which is a list, an iterable, or a PagedList
     object, each element of which is a valid dictionary by this specification.
 
-    Additionally, playlists can have "id", "title", and any other relevant
+    Additionally, playlists can have "id", "title", and any other relevent
     attributes with the same semantics as videos (see above).
 
     It can also have the following optional fields:
@@ -664,7 +671,7 @@ class InfoExtractor:
             if hasattr(e, 'countries'):
                 kwargs['countries'] = e.countries
             raise type(e)(e.orig_msg, **kwargs)
-        except http.client.IncompleteRead as e:
+        except compat_http_client.IncompleteRead as e:
             raise ExtractorError('A network error has occurred.', cause=e, expected=True, video_id=self.get_temp_id(url))
         except (KeyError, StopIteration) as e:
             raise ExtractorError('An extractor error has occurred.', cause=e, video_id=self.get_temp_id(url))
@@ -688,16 +695,8 @@ class InfoExtractor:
         """Sets a YoutubeDL instance as the downloader for this IE."""
         self._downloader = downloader
 
-    @property
-    def cache(self):
-        return self._downloader.cache
-
-    @property
-    def cookiejar(self):
-        return self._downloader.cookiejar
-
     def _initialize_pre_login(self):
-        """ Initialization before login. Redefine in subclasses."""
+        """ Intialization before login. Redefine in subclasses."""
         pass
 
     def _perform_login(self, username, password):
@@ -723,7 +722,7 @@ class InfoExtractor:
 
     @staticmethod
     def __can_accept_status_code(err, expected_status):
-        assert isinstance(err, urllib.error.HTTPError)
+        assert isinstance(err, compat_urllib_error.HTTPError)
         if expected_status is None:
             return False
         elif callable(expected_status):
@@ -731,14 +730,14 @@ class InfoExtractor:
         else:
             return err.code in variadic(expected_status)
 
-    def _create_request(self, url_or_request, data=None, headers=None, query=None):
+    def _create_request(self, url_or_request, data=None, headers={}, query={}):
-        if isinstance(url_or_request, urllib.request.Request):
+        if isinstance(url_or_request, compat_urllib_request.Request):
             return update_Request(url_or_request, data=data, headers=headers, query=query)
         if query:
             url_or_request = update_url_query(url_or_request, query)
-        return sanitized_Request(url_or_request, data, headers or {})
+        return sanitized_Request(url_or_request, data, headers)
 
-    def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, data=None, headers=None, query=None, expected_status=None):
+    def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, data=None, headers={}, query={}, expected_status=None):
         """
         Return the response handle.
 
@@ -766,13 +765,13 @@ class InfoExtractor:
         # geo unrestricted country. We will do so once we encounter any
         # geo restriction error.
         if self._x_forwarded_for_ip:
-            headers = (headers or {}).copy()
+            if 'X-Forwarded-For' not in headers:
-            headers.setdefault('X-Forwarded-For', self._x_forwarded_for_ip)
+                headers['X-Forwarded-For'] = self._x_forwarded_for_ip
 
         try:
             return self._downloader.urlopen(self._create_request(url_or_request, data, headers, query))
         except network_exceptions as err:
-            if isinstance(err, urllib.error.HTTPError):
+            if isinstance(err, compat_urllib_error.HTTPError):
                 if self.__can_accept_status_code(err, expected_status):
                     # Retain reference to error to prevent file object from
                     # being closed before it can be read. Works around the
@@ -800,7 +799,7 @@ class InfoExtractor:
 
         Arguments:
         url_or_request -- plain text URL as a string or
-            a urllib.request.Request object
+            a compat_urllib_request.Requestobject
         video_id -- Video/playlist/item identifier (string)
 
         Keyword arguments:
@@ -828,7 +827,7 @@ class InfoExtractor:
         """
 
         # Strip hashes from the URL (#1038)
-        if isinstance(url_or_request, str):
+        if isinstance(url_or_request, (compat_str, str)):
             url_or_request = url_or_request.partition('#')[0]
 
         urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal, data=data, headers=headers, query=query, expected_status=expected_status)
@@ -1049,7 +1048,7 @@ class InfoExtractor:
         while True:
             try:
                 return self.__download_webpage(url_or_request, video_id, note, errnote, None, fatal, *args, **kwargs)
-            except http.client.IncompleteRead as e:
+            except compat_http_client.IncompleteRead as e:
                 try_count += 1
                 if try_count >= tries:
                     raise e
@@ -1285,7 +1284,7 @@ class InfoExtractor:
         if tfa is not None:
             return tfa
 
-        return getpass.getpass('Type %s and press [Return]: ' % note)
+        return compat_getpass('Type %s and press [Return]: ' % note)
 
     # Helper functions for extracting OpenGraph info
     @staticmethod
@@ -1393,25 +1392,27 @@ class InfoExtractor:
         return self._html_search_meta('twitter:player', html,
                                       'twitter card player')
 
-    def _yield_json_ld(self, html, video_id, *, fatal=True, default=NO_DEFAULT):
+    def _search_json_ld(self, html, video_id, expected_type=None, **kwargs):
-        """Yield all json ld objects in the html"""
+        json_ld_list = list(re.finditer(JSON_LD_RE, html))
-        if default is not NO_DEFAULT:
+        default = kwargs.get('default', NO_DEFAULT)
-            fatal = False
+        # JSON-LD may be malformed and thus `fatal` should be respected.
-        for mobj in re.finditer(JSON_LD_RE, html):
+        # At the same time `default` may be passed that assumes `fatal=False`
-            json_ld_item = self._parse_json(mobj.group('json_ld'), video_id, fatal=fatal)
+        # for _search_regex. Let's simulate the same behavior here as well.
-            for json_ld in variadic(json_ld_item):
+        fatal = kwargs.get('fatal', True) if default is NO_DEFAULT else False
-                if isinstance(json_ld, dict):
+        json_ld = []
-                    yield json_ld
+        for mobj in json_ld_list:
-
+            json_ld_item = self._parse_json(
-    def _search_json_ld(self, html, video_id, expected_type=None, *, fatal=True, default=NO_DEFAULT):
+                mobj.group('json_ld'), video_id, fatal=fatal)
-        """Search for a video in any json ld in the html"""
+            if not json_ld_item:
-        if default is not NO_DEFAULT:
+                continue
-            fatal = False
+            if isinstance(json_ld_item, dict):
-        info = self._json_ld(
+                json_ld.append(json_ld_item)
-            list(self._yield_json_ld(html, video_id, fatal=fatal, default=default)),
+            elif isinstance(json_ld_item, (list, tuple)):
-            video_id, fatal=fatal, expected_type=expected_type)
+                json_ld.extend(json_ld_item)
-        if info:
+        if json_ld:
-            return info
+            json_ld = self._json_ld(json_ld, video_id, fatal=fatal, expected_type=expected_type)
+        if json_ld:
+            return json_ld
         if default is not NO_DEFAULT:
             return default
         elif fatal:
@@ -1421,7 +1422,7 @@ class InfoExtractor:
             return {}
 
     def _json_ld(self, json_ld, video_id, fatal=True, expected_type=None):
-        if isinstance(json_ld, str):
+        if isinstance(json_ld, compat_str):
             json_ld = self._parse_json(json_ld, video_id, fatal=fatal)
         if not json_ld:
             return {}
@@ -1499,7 +1500,7 @@ class InfoExtractor:
             assert is_type(e, 'VideoObject')
             author = e.get('author')
             info.update({
-                'url': url_or_none(e.get('contentUrl')),
+                'url': traverse_obj(e, 'contentUrl', 'embedUrl', expected_type=url_or_none),
                 'title': unescapeHTML(e.get('name')),
                 'description': unescapeHTML(e.get('description')),
                 'thumbnails': [{'url': url}
@@ -1511,7 +1512,7 @@ class InfoExtractor:
                 # both types can have 'name' property(inherited from 'Thing' type). [1]
                 # however some websites are using 'Text' type instead.
                 # 1. https://schema.org/VideoObject
-                'uploader': author.get('name') if isinstance(author, dict) else author if isinstance(author, str) else None,
+                'uploader': author.get('name') if isinstance(author, dict) else author if isinstance(author, compat_str) else None,
                 'filesize': int_or_none(float_or_none(e.get('contentSize'))),
                 'tbr': int_or_none(e.get('bitrate')),
                 'width': int_or_none(e.get('width')),
@@ -2160,7 +2161,7 @@ class InfoExtractor:
         ]), m3u8_doc)
 
         def format_url(url):
-            return url if re.match(r'^https?://', url) else urllib.parse.urljoin(m3u8_url, url)
+            return url if re.match(r'^https?://', url) else compat_urlparse.urljoin(m3u8_url, url)
 
         if self.get_param('hls_split_discontinuity', False):
             def _extract_m3u8_playlist_indices(manifest_url=None, m3u8_doc=None):
@@ -2533,7 +2534,7 @@ class InfoExtractor:
                     })
                 continue
 
-            src_url = src if src.startswith('http') else urllib.parse.urljoin(base, src)
+            src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
             src_url = src_url.strip()
 
             if proto == 'm3u8' or src_ext == 'm3u8':
@@ -2556,7 +2557,7 @@ class InfoExtractor:
                         'plugin': 'flowplayer-3.2.0.1',
                     }
                 f4m_url += '&' if '?' in f4m_url else '?'
-                f4m_url += urllib.parse.urlencode(f4m_params)
+                f4m_url += compat_urllib_parse_urlencode(f4m_params)
                 formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))
             elif src_ext == 'mpd':
                 formats.extend(self._extract_mpd_formats(
@@ -2821,12 +2822,12 @@ class InfoExtractor:
                     base_url = ''
                     for element in (representation, adaptation_set, period, mpd_doc):
                         base_url_e = element.find(_add_ns('BaseURL'))
-                        if try_call(lambda: base_url_e.text) is not None:
+                        if base_url_e is not None:
                             base_url = base_url_e.text + base_url
                             if re.match(r'^https?://', base_url):
                                 break
                     if mpd_base_url and base_url.startswith('/'):
-                        base_url = urllib.parse.urljoin(mpd_base_url, base_url)
+                        base_url = compat_urlparse.urljoin(mpd_base_url, base_url)
                     elif mpd_base_url and not re.match(r'^https?://', base_url):
                         if not mpd_base_url.endswith('/'):
                             mpd_base_url += '/'
@@ -3096,7 +3097,7 @@ class InfoExtractor:
                 sampling_rate = int_or_none(track.get('SamplingRate'))
 
                 track_url_pattern = re.sub(r'{[Bb]itrate}', track.attrib['Bitrate'], url_pattern)
-                track_url_pattern = urllib.parse.urljoin(ism_url, track_url_pattern)
+                track_url_pattern = compat_urlparse.urljoin(ism_url, track_url_pattern)
 
                 fragments = []
                 fragment_ctx = {
@@ -3115,7 +3116,7 @@ class InfoExtractor:
                         fragment_ctx['duration'] = (next_fragment_time - fragment_ctx['time']) / fragment_repeat
                     for _ in range(fragment_repeat):
                         fragments.append({
-                            'url': re.sub(r'{start[ _]time}', str(fragment_ctx['time']), track_url_pattern),
+                            'url': re.sub(r'{start[ _]time}', compat_str(fragment_ctx['time']), track_url_pattern),
                             'duration': fragment_ctx['duration'] / stream_timescale,
                         })
                         fragment_ctx['time'] += fragment_ctx['duration']
@@ -3208,7 +3209,7 @@ class InfoExtractor:
 
         entries = []
         # amp-video and amp-audio are very similar to their HTML5 counterparts
-        # so we will include them right here (see
+        # so we wll include them right here (see
         # https://www.ampproject.org/docs/reference/components/amp-video)
         # For dl8-* tags see https://delight-vr.com/documentation/dl8-video/
         _MEDIA_TAG_NAME_RE = r'(?:(?:amp|dl8(?:-live)?)-)?(video|audio)'
@@ -3359,7 +3360,7 @@ class InfoExtractor:
         return formats, subtitles
 
     def _extract_wowza_formats(self, url, video_id, m3u8_entry_protocol='m3u8_native', skip_protocols=[]):
-        query = urllib.parse.urlparse(url).query
+        query = compat_urlparse.urlparse(url).query
         url = re.sub(r'/(?:manifest|playlist|jwplayer)\.(?:m3u8|f4m|mpd|smil)', '', url)
         mobj = re.search(
             r'(?:(?:http|rtmp|rtsp)(?P<s>s)?:)?(?P<url>//[^?]+)', url)
@@ -3465,7 +3466,7 @@ class InfoExtractor:
                     if not isinstance(track, dict):
                         continue
                     track_kind = track.get('kind')
-                    if not track_kind or not isinstance(track_kind, str):
+                    if not track_kind or not isinstance(track_kind, compat_str):
                         continue
                     if track_kind.lower() not in ('captions', 'subtitles'):
                         continue
@@ -3538,7 +3539,7 @@ class InfoExtractor:
                     # Often no height is provided but there is a label in
                     # format like "1080p", "720p SD", or 1080.
                     height = int_or_none(self._search_regex(
-                        r'^(\d{3,4})[pP]?(?:\b|$)', str(source.get('label') or ''),
+                        r'^(\d{3,4})[pP]?(?:\b|$)', compat_str(source.get('label') or ''),
                         'height', default=None))
                 a_format = {
                     'url': source_url,
@@ -3590,15 +3591,15 @@ class InfoExtractor:
 
     def _set_cookie(self, domain, name, value, expire_time=None, port=None,
                     path='/', secure=False, discard=False, rest={}, **kwargs):
-        cookie = http.cookiejar.Cookie(
+        cookie = compat_cookiejar_Cookie(
             0, name, value, port, port is not None, domain, True,
             domain.startswith('.'), path, True, secure, expire_time,
             discard, None, None, rest)
-        self.cookiejar.set_cookie(cookie)
+        self._downloader.cookiejar.set_cookie(cookie)
 
     def _get_cookies(self, url):
-        """ Return a http.cookies.SimpleCookie with the cookies for the url """
+        """ Return a compat_cookies_SimpleCookie with the cookies for the url """
-        return http.cookies.SimpleCookie(self._downloader._calc_cookies(url))
+        return compat_cookies_SimpleCookie(self._downloader._calc_cookies(url))
 
     def _apply_first_set_cookie_header(self, url_handle, cookie):
         """
@@ -3764,10 +3765,10 @@ class InfoExtractor:
         return headers
 
     def _generic_id(self, url):
-        return urllib.parse.unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
+        return compat_urllib_parse_unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
 
     def _generic_title(self, url):
-        return urllib.parse.unquote(os.path.splitext(url_basename(url))[0])
+        return compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0])
 
     @staticmethod
     def _availability(is_private=None, needs_premium=None, needs_subscription=None, needs_auth=None, is_unlisted=None):

yt_dlp/extractor/commonprotocols.py

@@ -1,6 +1,5 @@
-import urllib.parse
-
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 
 
 class RtmpIE(InfoExtractor):
@@ -24,7 +23,7 @@ class RtmpIE(InfoExtractor):
             'formats': [{
                 'url': url,
                 'ext': 'flv',
-                'format_id': urllib.parse.urlparse(url).scheme,
+                'format_id': compat_urlparse.urlparse(url).scheme,
             }],
         }
 

yt_dlp/extractor/crunchyroll.py

@@ -1,20 +1,19 @@
 import base64
-import json
 import re
-import urllib.request
+import json
-import xml.etree.ElementTree
 import zlib
-from hashlib import sha1
-from math import floor, pow, sqrt
 
+import xml.etree.ElementTree
+from hashlib import sha1
+from math import pow, sqrt, floor
 from .common import InfoExtractor
 from .vrv import VRVBaseIE
-from ..aes import aes_cbc_decrypt
 from ..compat import (
     compat_b64decode,
     compat_etree_fromstring,
     compat_str,
     compat_urllib_parse_urlencode,
+    compat_urllib_request,
     compat_urlparse,
 )
 from ..utils import (
@@ -23,8 +22,8 @@ from ..utils import (
     extract_attributes,
     float_or_none,
     format_field,
-    int_or_none,
     intlist_to_bytes,
+    int_or_none,
     join_nonempty,
     lowercase_escape,
     merge_dicts,
@@ -35,6 +34,9 @@ from ..utils import (
     try_get,
     xpath_text,
 )
+from ..aes import (
+    aes_cbc_decrypt,
+)
 
 
 class CrunchyrollBaseIE(InfoExtractor):
@@ -257,7 +259,7 @@ class CrunchyrollIE(CrunchyrollBaseIE, VRVBaseIE):
     }
 
     def _download_webpage(self, url_or_request, *args, **kwargs):
-        request = (url_or_request if isinstance(url_or_request, urllib.request.Request)
+        request = (url_or_request if isinstance(url_or_request, compat_urllib_request.Request)
                    else sanitized_Request(url_or_request))
         # Accept-Language must be set explicitly to accept any language to avoid issues
         # similar to https://github.com/ytdl-org/youtube-dl/issues/6797.

yt_dlp/extractor/curiositystream.py

@@ -1,8 +1,12 @@
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..utils import (
-from ..utils import ExtractorError, int_or_none, urlencode_postdata
+    int_or_none,
+    urlencode_postdata,
+    compat_str,
+    ExtractorError,
+)
 
 
 class CuriosityStreamBaseIE(InfoExtractor):
@@ -46,7 +50,7 @@ class CuriosityStreamIE(CuriosityStreamBaseIE):
     IE_NAME = 'curiositystream'
     _VALID_URL = r'https?://(?:app\.)?curiositystream\.com/video/(?P<id>\d+)'
     _TESTS = [{
-        'url': 'http://app.curiositystream.com/video/2',
+        'url': 'https://app.curiositystream.com/video/2',
         'info_dict': {
             'id': '2',
             'ext': 'mp4',

yt_dlp/extractor/cwtv.py

@@ -91,5 +91,4 @@ class CWTVIE(InfoExtractor):
             'timestamp': parse_iso8601(video_data.get('start_time')),
             'age_limit': parse_age_limit(video_data.get('rating')),
             'ie_key': 'ThePlatform',
-            'thumbnail': video_data.get('large_thumbnail')
         }

yt_dlp/extractor/dropout.py

@@ -119,16 +119,16 @@ class DropoutIE(InfoExtractor):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-
+        login_err, webpage = False, ''
-        webpage = None
+        try:
-        if self._get_cookies('https://www.dropout.tv').get('_session'):
-            webpage = self._download_webpage(url, display_id)
-        if not webpage or '<div id="watch-unauthorized"' in webpage:
             login_err = self._login(display_id)
             webpage = self._download_webpage(url, display_id)
-            if login_err and '<div id="watch-unauthorized"' in webpage:
+        finally:
+            if not login_err:
+                self._download_webpage('https://www.dropout.tv/logout', display_id, note='Logging out', fatal=False)
+            elif '<div id="watch-unauthorized"' in webpage:
                 if login_err is True:
-                    self.raise_login_required(method='any')
+                    self.raise_login_required(method='password')
                 raise ExtractorError(login_err, expected=True)
 
         embed_url = self._search_regex(r'embed_url:\s*["\'](.+?)["\']', webpage, 'embed url')

yt_dlp/extractor/ertgr.py

@@ -119,7 +119,7 @@ class ERTFlixCodenameIE(ERTFlixBaseIE):
 class ERTFlixIE(ERTFlixBaseIE):
     IE_NAME = 'ertflix'
     IE_DESC = 'ERTFLIX videos'
-    _VALID_URL = r'https?://www\.ertflix\.gr/(?:[^/]+/)?(?:series|vod)/(?P<id>[a-z]{3}\.\d+)'
+    _VALID_URL = r'https?://www\.ertflix\.gr/(?:series|vod)/(?P<id>[a-z]{3}\.\d+)'
     _TESTS = [{
         'url': 'https://www.ertflix.gr/vod/vod.173258-aoratoi-ergates',
         'md5': '6479d5e60fd7e520b07ba5411dcdd6e7',
@@ -171,9 +171,6 @@ class ERTFlixIE(ERTFlixBaseIE):
             'title': 'Το δίκτυο',
         },
         'playlist_mincount': 9,
-    }, {
-        'url': 'https://www.ertflix.gr/en/vod/vod.127652-ta-kalytera-mas-chronia-ep1-mia-volta-sto-feggari',
-        'only_matching': True,
     }]
 
     def _extract_episode(self, episode):

yt_dlp/extractor/espn.py

@@ -1,10 +1,10 @@
 import base64
 import json
 import re
-import urllib.parse
+import urllib
 
-from .adobepass import AdobePassIE
 from .common import InfoExtractor
+from .adobepass import AdobePassIE
 from .once import OnceIE
 from ..utils import (
     determine_ext,
@@ -197,7 +197,7 @@ class ESPNArticleIE(InfoExtractor):
 
     @classmethod
     def suitable(cls, url):
-        return False if (ESPNIE.suitable(url) or WatchESPNIE.suitable(url)) else super().suitable(url)
+        return False if (ESPNIE.suitable(url) or WatchESPNIE.suitable(url)) else super(ESPNArticleIE, cls).suitable(url)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

yt_dlp/extractor/facebook.py

@@ -1,18 +1,18 @@
 import json
 import re
-import urllib.parse
 
 from .common import InfoExtractor
 from ..compat import (
     compat_etree_fromstring,
     compat_str,
     compat_urllib_parse_unquote,
+    compat_urllib_parse_unquote_plus,
 )
 from ..utils import (
-    ExtractorError,
     clean_html,
     determine_ext,
     error_to_compat_str,
+    ExtractorError,
     float_or_none,
     get_element_by_id,
     get_first,
@@ -467,7 +467,7 @@ class FacebookIE(InfoExtractor):
             dash_manifest = video.get('dash_manifest')
             if dash_manifest:
                 formats.extend(self._parse_mpd_formats(
-                    compat_etree_fromstring(urllib.parse.unquote_plus(dash_manifest))))
+                    compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest))))
 
         def process_formats(formats):
             # Downloads with browser's User-Agent are rate limited. Working around

yt_dlp/extractor/fc2.py

@@ -78,7 +78,7 @@ class FC2IE(InfoExtractor):
         webpage = None
         if not url.startswith('fc2:'):
             webpage = self._download_webpage(url, video_id)
-            self.cookiejar.clear_session_cookies()  # must clear
+            self._downloader.cookiejar.clear_session_cookies()  # must clear
             self._login()
 
         title, thumbnail, description = None, None, None

yt_dlp/extractor/foxgay.py

@@ -31,7 +31,7 @@ class FoxgayIE(InfoExtractor):
         description = get_element_by_id('inf_tit', webpage)
 
         # The default user-agent with foxgay cookies leads to pages without videos
-        self.cookiejar.clear('.foxgay.com')
+        self._downloader.cookiejar.clear('.foxgay.com')
         # Find the URL for the iFrame which contains the actual video.
         iframe_url = self._html_search_regex(
             r'<iframe[^>]+src=([\'"])(?P<url>[^\'"]+)\1', webpage,

yt_dlp/extractor/fuyintv.py

@@ -1,30 +0,0 @@
-from .common import InfoExtractor
-from ..utils import traverse_obj
-
-
-class FuyinTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?fuyin\.tv/html/(?:\d+)/(?P<id>\d+)\.html'
-    _TESTS = [{
-        'url': 'https://www.fuyin.tv/html/2733/44129.html',
-        'info_dict': {
-            'id': '44129',
-            'ext': 'mp4',
-            'title': '第1集',
-            'description': 'md5:21a3d238dc8d49608e1308e85044b9c3',
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        json_data = self._download_json(
-            'https://www.fuyin.tv/api/api/tv.movie/url',
-            video_id, query={'urlid': f'{video_id}'})
-        webpage = self._download_webpage(url, video_id, fatal=False)
-
-        return {
-            'id': video_id,
-            'title': traverse_obj(json_data, ('data', 'title')),
-            'url': json_data['data']['url'],
-            'ext': 'mp4',
-            'description': self._html_search_meta('description', webpage),
-        }

yt_dlp/extractor/generic.py

@@ -1,6 +1,5 @@
 import os
 import re
-import urllib.parse
 import xml.etree.ElementTree
 
 from .ant1newsgr import Ant1NewsGrEmbedIE
@@ -107,7 +106,12 @@ from .yapfiles import YapFilesIE
 from .youporn import YouPornIE
 from .youtube import YoutubeIE
 from .zype import ZypeIE
-from ..compat import compat_etree_fromstring
+from ..compat import (
+    compat_etree_fromstring,
+    compat_str,
+    compat_urllib_parse_unquote,
+    compat_urlparse,
+)
 from ..utils import (
     KNOWN_EXTENSIONS,
     ExtractorError,
@@ -142,7 +146,7 @@ class GenericIE(InfoExtractor):
     IE_DESC = 'Generic downloader that works on some sites'
     _VALID_URL = r'.*'
     IE_NAME = 'generic'
-    _NETRC_MACHINE = False  # Suppress username warning
+    _NETRC_MACHINE = False  # Supress username warning
     _TESTS = [
         # Direct link to a video
         {
@@ -2699,7 +2703,7 @@ class GenericIE(InfoExtractor):
 
         title = self._html_search_meta('DC.title', webpage, fatal=True)
 
-        camtasia_url = urllib.parse.urljoin(url, camtasia_cfg)
+        camtasia_url = compat_urlparse.urljoin(url, camtasia_cfg)
         camtasia_cfg = self._download_xml(
             camtasia_url, video_id,
             note='Downloading camtasia configuration',
@@ -2715,7 +2719,7 @@ class GenericIE(InfoExtractor):
             entries.append({
                 'id': os.path.splitext(url_n.text.rpartition('/')[2])[0],
                 'title': f'{title} - {n.tag}',
-                'url': urllib.parse.urljoin(url, url_n.text),
+                'url': compat_urlparse.urljoin(url, url_n.text),
                 'duration': float_or_none(n.find('./duration').text),
             })
 
@@ -2767,7 +2771,7 @@ class GenericIE(InfoExtractor):
         if url.startswith('//'):
             return self.url_result(self.http_scheme() + url)
 
-        parsed_url = urllib.parse.urlparse(url)
+        parsed_url = compat_urlparse.urlparse(url)
         if not parsed_url.scheme:
             default_search = self.get_param('default_search')
             if default_search is None:
@@ -2825,22 +2829,12 @@ class GenericIE(InfoExtractor):
                         new_url, {'force_videoid': force_videoid})
                 return self.url_result(new_url)
 
-        def request_webpage():
-            request = sanitized_Request(url)
-            # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
-            # making it impossible to download only chunk of the file (yet we need only 512kB to
-            # test whether it's HTML or not). According to yt-dlp default Accept-Encoding
-            # that will always result in downloading the whole file that is not desirable.
-            # Therefore for extraction pass we have to override Accept-Encoding to any in order
-            # to accept raw bytes and being able to download only a chunk.
-            # It may probably better to solve this by checking Content-Type for application/octet-stream
-            # after HEAD request finishes, but not sure if we can rely on this.
-            request.add_header('Accept-Encoding', '*')
-            return self._request_webpage(request, video_id)
-
         full_response = None
         if head_response is False:
-            head_response = full_response = request_webpage()
+            request = sanitized_Request(url)
+            request.add_header('Accept-Encoding', '*')
+            full_response = self._request_webpage(request, video_id)
+            head_response = full_response
 
         info_dict = {
             'id': video_id,
@@ -2853,7 +2847,7 @@ class GenericIE(InfoExtractor):
         m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
         if m:
             self.report_detected('direct video link')
-            format_id = str(m.group('format_id'))
+            format_id = compat_str(m.group('format_id'))
             subtitles = {}
             if format_id.endswith('mpegurl'):
                 formats, subtitles = self._extract_m3u8_formats_and_subtitles(url, video_id, 'mp4')
@@ -2878,7 +2872,19 @@ class GenericIE(InfoExtractor):
             self.report_warning(
                 '%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))
 
-        full_response = full_response or request_webpage()
+        if not full_response:
+            request = sanitized_Request(url)
+            # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
+            # making it impossible to download only chunk of the file (yet we need only 512kB to
+            # test whether it's HTML or not). According to yt-dlp default Accept-Encoding
+            # that will always result in downloading the whole file that is not desirable.
+            # Therefore for extraction pass we have to override Accept-Encoding to any in order
+            # to accept raw bytes and being able to download only a chunk.
+            # It may probably better to solve this by checking Content-Type for application/octet-stream
+            # after HEAD request finishes, but not sure if we can rely on this.
+            request.add_header('Accept-Encoding', '*')
+            full_response = self._request_webpage(request, video_id)
+
         first_bytes = full_response.read(512)
 
         # Is it an M3U playlist?
@@ -2960,7 +2966,7 @@ class GenericIE(InfoExtractor):
         # Unescaping the whole page allows to handle those cases in a generic way
         # FIXME: unescaping the whole page may break URLs, commenting out for now.
         # There probably should be a second run of generic extractor on unescaped webpage.
-        # webpage = urllib.parse.unquote(webpage)
+        # webpage = compat_urllib_parse_unquote(webpage)
 
         # Unescape squarespace embeds to be detected by generic extractor,
         # see https://github.com/ytdl-org/youtube-dl/issues/21294
@@ -3233,7 +3239,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'))
         mobj = re.search(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P<url>[^&]+)', webpage)
         if mobj is not None:
-            return self.url_result(urllib.parse.unquote(mobj.group('url')))
+            return self.url_result(compat_urllib_parse_unquote(mobj.group('url')))
 
         # Look for funnyordie embed
         matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage)
@@ -3486,7 +3492,7 @@ class GenericIE(InfoExtractor):
             r'<iframe[^>]+src="(?:https?:)?(?P<url>%s)"' % UDNEmbedIE._PROTOCOL_RELATIVE_VALID_URL, webpage)
         if mobj is not None:
             return self.url_result(
-                urllib.parse.urljoin(url, mobj.group('url')), 'UDNEmbed')
+                compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')
 
         # Look for Senate ISVP iframe
         senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
@@ -3719,7 +3725,7 @@ class GenericIE(InfoExtractor):
         if mediasite_urls:
             entries = [
                 self.url_result(smuggle_url(
-                    urllib.parse.urljoin(url, mediasite_url),
+                    compat_urlparse.urljoin(url, mediasite_url),
                     {'UrlReferrer': url}), ie=MediasiteIE.ie_key())
                 for mediasite_url in mediasite_urls]
             return self.playlist_result(entries, video_id, video_title)
@@ -3914,11 +3920,11 @@ class GenericIE(InfoExtractor):
             subtitles = {}
             for source in sources:
                 src = source.get('src')
-                if not src or not isinstance(src, str):
+                if not src or not isinstance(src, compat_str):
                     continue
-                src = urllib.parse.urljoin(url, src)
+                src = compat_urlparse.urljoin(url, src)
                 src_type = source.get('type')
-                if isinstance(src_type, str):
+                if isinstance(src_type, compat_str):
                     src_type = src_type.lower()
                 ext = determine_ext(src).lower()
                 if src_type == 'video/youtube':
@@ -3952,7 +3958,7 @@ class GenericIE(InfoExtractor):
                 if not src:
                     continue
                 subtitles.setdefault(dict_get(sub, ('language', 'srclang')) or 'und', []).append({
-                    'url': urllib.parse.urljoin(url, src),
+                    'url': compat_urlparse.urljoin(url, src),
                     'name': sub.get('label'),
                     'http_headers': {
                         'Referer': full_response.geturl(),
@@ -3979,7 +3985,7 @@ class GenericIE(InfoExtractor):
                 return True
             if RtmpIE.suitable(vurl):
                 return True
-            vpath = urllib.parse.urlparse(vurl).path
+            vpath = compat_urlparse.urlparse(vurl).path
             vext = determine_ext(vpath, None)
             return vext not in (None, 'swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml', 'js', 'xml')
 
@@ -4107,7 +4113,7 @@ class GenericIE(InfoExtractor):
                 if refresh_header:
                     found = re.search(REDIRECT_REGEX, refresh_header)
             if found:
-                new_url = urllib.parse.urljoin(url, unescapeHTML(found.group(1)))
+                new_url = compat_urlparse.urljoin(url, unescapeHTML(found.group(1)))
                 if new_url != url:
                     self.report_following_redirect(new_url)
                     return {
@@ -4133,8 +4139,8 @@ class GenericIE(InfoExtractor):
         for video_url in orderedSet(found):
             video_url = unescapeHTML(video_url)
             video_url = video_url.replace('\\/', '/')
-            video_url = urllib.parse.urljoin(url, video_url)
+            video_url = compat_urlparse.urljoin(url, video_url)
-            video_id = urllib.parse.unquote(os.path.basename(video_url))
+            video_id = compat_urllib_parse_unquote(os.path.basename(video_url))
 
             # Sometimes, jwplayer extraction will result in a YouTube URL
             if YoutubeIE.suitable(video_url):