[version] update

Created by: pukkandan

:ci skip all :ci run dl

.github/ISSUE_TEMPLATE/1_broken_site.yml

@@ -11,13 +11,13 @@ body:
       options:
         - label: I'm reporting a broken site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.06.29** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.07.18** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
         - label: I've checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/ytdl-org/youtube-dl#video-url-contains-an-ampersand-and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command)
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues including closed ones. DO NOT post duplicates
+        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
@@ -26,37 +26,45 @@ body:
     id: region
     attributes:
       label: Region
-      description: "Enter the region the site is accessible from"
-      placeholder: "India"
+      description: Enter the country/region that the site is accessible from
+      placeholder: India
   - type: textarea
     id: description
     attributes:
-      label: Description
-      description: |
-        Provide an explanation of your issue in an arbitrary form.
-        Provide any additional information, any suggested solutions, and as much context and examples as possible
-      placeholder: WRITE DESCRIPTION HERE
+      label: Provide a description that is worded well enough to be understood
+      description: See [is-the-description-of-the-issue-itself-sufficient](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-description-of-the-issue-itself-sufficient)
+      placeholder: Provide any additional information, any suggested solutions, and as much context and examples as possible
     validations:
       required: true
+  - type: checkboxes
+    id: verbose
+    attributes:
+      label: Provide verbose output that clearly demonstrates the problem
+      options:
+        - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU <your command line>`)
+          required: true
+        - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below
+          required: true
   - type: textarea
     id: log
     attributes:
-      label: Verbose log
+      label: Complete Verbose Output
       description: |
-        Provide the complete verbose output of yt-dlp **that clearly demonstrates the problem**.
-        Add the `-vU` flag to your command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
-        It should look similar to this:
+        It should start like this:
       placeholder: |
-        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
-        [debug] Portable config file: yt-dlp.conf
-        [debug] Portable config: ['-i']
-        [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2022.06.29 (exe)
-        [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
-        [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
-        [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
+        [debug] Command-line config: ['-vU', 'test:youtube']
+        [debug] Portable config "yt-dlp.conf": ['-i']
+        [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
+        [debug] yt-dlp version 2022.07.18 [9d339c4] (win32_exe)
+        [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
+        [debug] Checking exe version: ffmpeg -bsfs
+        [debug] Checking exe version: ffprobe -bsfs
+        [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
+        [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
-        yt-dlp is up to date (2022.06.29)
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
+        Latest version: 2022.07.18, Current version: 2022.07.18
+        yt-dlp is up to date (2022.07.18)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/2_site_support_request.yml

@@ -11,13 +11,13 @@ body:
       options:
         - label: I'm reporting a new site support request
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.06.29** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.07.18** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
         - label: I've checked that none of provided URLs [violate any copyrights](https://github.com/ytdl-org/youtube-dl#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free) or contain any [DRM](https://en.wikipedia.org/wiki/Digital_rights_management) to the best of my knowledge
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues including closed ones. DO NOT post duplicates
+        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
@@ -26,8 +26,8 @@ body:
     id: region
     attributes:
       label: Region
-      description: "Enter the region the site is accessible from"
-      placeholder: "India"
+      description: Enter the country/region that the site is accessible from
+      placeholder: India
   - type: textarea
     id: example-urls
     attributes:
@@ -43,31 +43,40 @@ body:
   - type: textarea
     id: description
     attributes:
-      label: Description
-      description: |
-        Provide any additional information
-      placeholder: WRITE DESCRIPTION HERE
+      label: Provide a description that is worded well enough to be understood
+      description: See [is-the-description-of-the-issue-itself-sufficient](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-description-of-the-issue-itself-sufficient)
+      placeholder: Provide any additional information, any suggested solutions, and as much context and examples as possible
     validations:
       required: true
+  - type: checkboxes
+    id: verbose
+    attributes:
+      label: Provide verbose output that clearly demonstrates the problem
+      options:
+        - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU <your command line>`)
+          required: true
+        - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below
+          required: true
   - type: textarea
     id: log
     attributes:
-      label: Verbose log
+      label: Complete Verbose Output
       description: |
-        Provide the complete verbose output **using one of the example URLs provided above**.
-        Add the `-vU` flag to your command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
-        It should look similar to this:
+        It should start like this:
       placeholder: |
-        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
-        [debug] Portable config file: yt-dlp.conf
-        [debug] Portable config: ['-i']
-        [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2022.06.29 (exe)
-        [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
-        [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
-        [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
+        [debug] Command-line config: ['-vU', 'test:youtube']
+        [debug] Portable config "yt-dlp.conf": ['-i']
+        [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
+        [debug] yt-dlp version 2022.07.18 [9d339c4] (win32_exe)
+        [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
+        [debug] Checking exe version: ffmpeg -bsfs
+        [debug] Checking exe version: ffprobe -bsfs
+        [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
+        [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
-        yt-dlp is up to date (2022.06.29)
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
+        Latest version: 2022.07.18, Current version: 2022.07.18
+        yt-dlp is up to date (2022.07.18)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/3_site_feature_request.yml

@@ -11,11 +11,11 @@ body:
       options:
         - label: I'm requesting a site-specific feature
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.06.29** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.07.18** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues including closed ones. DO NOT post duplicates
+        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
@@ -24,8 +24,8 @@ body:
     id: region
     attributes:
       label: Region
-      description: "Enter the region the site is accessible from"
-      placeholder: "India"
+      description: Enter the country/region that the site is accessible from
+      placeholder: India
   - type: textarea
     id: example-urls
     attributes:
@@ -39,33 +39,40 @@ body:
   - type: textarea
     id: description
     attributes:
-      label: Description
-      description: |
-        Provide an explanation of your site feature request in an arbitrary form.
-        Please make sure the description is worded well enough to be understood, see [is-the-description-of-the-issue-itself-sufficient](https://github.com/ytdl-org/youtube-dl#is-the-description-of-the-issue-itself-sufficient).
-        Provide any additional information, any suggested solutions, and as much context and examples as possible
-      placeholder: WRITE DESCRIPTION HERE
+      label: Provide a description that is worded well enough to be understood
+      description: See [is-the-description-of-the-issue-itself-sufficient](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-description-of-the-issue-itself-sufficient)
+      placeholder: Provide any additional information, any suggested solutions, and as much context and examples as possible
     validations:
       required: true
+  - type: checkboxes
+    id: verbose
+    attributes:
+      label: Provide verbose output that clearly demonstrates the problem
+      options:
+        - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU <your command line>`)
+          required: true
+        - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below
+          required: true
   - type: textarea
     id: log
     attributes:
-      label: Verbose log
+      label: Complete Verbose Output
       description: |
-        Provide the complete verbose output of yt-dlp that demonstrates the need for the enhancement.
-        Add the `-vU` flag to your command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
-        It should look similar to this:
+        It should start like this:
       placeholder: |
-        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
-        [debug] Portable config file: yt-dlp.conf
-        [debug] Portable config: ['-i']
-        [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2022.06.29 (exe)
-        [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
-        [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
-        [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
+        [debug] Command-line config: ['-vU', 'test:youtube']
+        [debug] Portable config "yt-dlp.conf": ['-i']
+        [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
+        [debug] yt-dlp version 2022.07.18 [9d339c4] (win32_exe)
+        [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
+        [debug] Checking exe version: ffmpeg -bsfs
+        [debug] Checking exe version: ffprobe -bsfs
+        [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
+        [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
-        yt-dlp is up to date (2022.06.29)
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
+        Latest version: 2022.07.18, Current version: 2022.07.18
+        yt-dlp is up to date (2022.07.18)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/4_bug_report.yml

@@ -11,46 +11,53 @@ body:
       options:
         - label: I'm reporting a bug unrelated to a specific site
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.06.29** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.07.18** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
         - label: I've checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/ytdl-org/youtube-dl#video-url-contains-an-ampersand-and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command)
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues including closed ones. DO NOT post duplicates
+        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
   - type: textarea
     id: description
     attributes:
-      label: Description
-      description: |
-        Provide an explanation of your issue in an arbitrary form.
-        Please make sure the description is worded well enough to be understood, see [is-the-description-of-the-issue-itself-sufficient](https://github.com/ytdl-org/youtube-dl#is-the-description-of-the-issue-itself-sufficient).
-        Provide any additional information, any suggested solutions, and as much context and examples as possible
-      placeholder: WRITE DESCRIPTION HERE
+      label: Provide a description that is worded well enough to be understood
+      description: See [is-the-description-of-the-issue-itself-sufficient](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-description-of-the-issue-itself-sufficient)
+      placeholder: Provide any additional information, any suggested solutions, and as much context and examples as possible
     validations:
       required: true
+    - type: checkboxes
+    id: verbose
+    attributes:
+      label: Provide verbose output that clearly demonstrates the problem
+      options:
+        - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU <your command line>`)
+          required: true
+        - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below
+          required: true
   - type: textarea
     id: log
     attributes:
-      label: Verbose log
+      label: Complete Verbose Output
       description: |
-        Provide the complete verbose output of yt-dlp **that clearly demonstrates the problem**.
-        Add the `-vU` flag to **your** command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
-        It should look similar to this:
+        It should start like this:
       placeholder: |
-        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
-        [debug] Portable config file: yt-dlp.conf
-        [debug] Portable config: ['-i']
-        [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2022.06.29 (exe)
-        [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
-        [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
-        [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
+        [debug] Command-line config: ['-vU', 'test:youtube']
+        [debug] Portable config "yt-dlp.conf": ['-i']
+        [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
+        [debug] yt-dlp version 2022.07.18 [9d339c4] (win32_exe)
+        [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
+        [debug] Checking exe version: ffmpeg -bsfs
+        [debug] Checking exe version: ffprobe -bsfs
+        [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
+        [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
-        yt-dlp is up to date (2022.06.29)
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
+        Latest version: 2022.07.18, Current version: 2022.07.18
+        yt-dlp is up to date (2022.07.18)
         <more lines>
       render: shell
     validations:

.github/ISSUE_TEMPLATE/5_feature_request.yml

@@ -13,41 +13,46 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.06.29** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.07.18** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues including closed ones. DO NOT post duplicates
+        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
   - type: textarea
     id: description
     attributes:
-      label: Description
-      description: |
-        Provide an explanation of your site feature request in an arbitrary form.
-        Please make sure the description is worded well enough to be understood, see [is-the-description-of-the-issue-itself-sufficient](https://github.com/ytdl-org/youtube-dl#is-the-description-of-the-issue-itself-sufficient).
-        Provide any additional information, any suggested solutions, and as much context and examples as possible
-      placeholder: WRITE DESCRIPTION HERE
+      label: Provide a description that is worded well enough to be understood
+      description: See [is-the-description-of-the-issue-itself-sufficient](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-description-of-the-issue-itself-sufficient)
+      placeholder: Provide any additional information, any suggested solutions, and as much context and examples as possible
     validations:
       required: true
+  - type: checkboxes
+    id: verbose
+    attributes:
+      label: Provide verbose output that clearly demonstrates the problem
+      options:
+        - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU <your command line>`)
+        - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below
   - type: textarea
     id: log
     attributes:
-      label: Verbose log
+      label: Complete Verbose Output
       description: |
-        If your feature request involves an existing yt-dlp command, provide the complete verbose output of that command.
-        Add the `-vU` flag to **your** command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
-        It should look similar to this:
+        It should start like this:
       placeholder: |
-        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
-        [debug] Portable config file: yt-dlp.conf
-        [debug] Portable config: ['-i']
-        [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2021.12.01 (exe)
-        [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
-        [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
-        [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
+        [debug] Command-line config: ['-vU', 'test:youtube']
+        [debug] Portable config "yt-dlp.conf": ['-i']
+        [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
+        [debug] yt-dlp version 2022.07.18 [9d339c4] (win32_exe)
+        [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
+        [debug] Checking exe version: ffmpeg -bsfs
+        [debug] Checking exe version: ffprobe -bsfs
+        [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
+        [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
-        yt-dlp is up to date (2021.12.01)
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
+        Latest version: 2022.07.18, Current version: 2022.07.18
+        yt-dlp is up to date (2022.07.18)
         <more lines>
       render: shell

.github/ISSUE_TEMPLATE/6_question.yml

@@ -2,6 +2,12 @@ name: Ask question
 description: Ask yt-dlp related question
 labels: [question]
 body:
+  - type: markdown
+    attributes:
+      value: |
+        ### Make sure you are **only** asking a question and not reporting a bug or requesting a feature.
+        If your question contains "isn't working" or "can you add", this is most likely the wrong template.
+        If you are in doubt whether this is the right template, **use another template**!
   - type: checkboxes
     id: checklist
     attributes:
@@ -13,43 +19,46 @@ body:
           required: true
         - label: I've looked through the [README](https://github.com/yt-dlp/yt-dlp#readme)
           required: true
-        - label: I've verified that I'm running yt-dlp version **2022.06.29** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
+        - label: I've verified that I'm running yt-dlp version **2022.07.18** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions including closed ones. DO NOT post duplicates
+        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
   - type: textarea
     id: question
     attributes:
-      label: Question
-      description: |
-        Ask your question in an arbitrary form.
-        Please make sure it's worded well enough to be understood, see [is-the-description-of-the-issue-itself-sufficient](https://github.com/ytdl-org/youtube-dl#is-the-description-of-the-issue-itself-sufficient).
-        Provide any additional information and as much context and examples as possible.
-        If your question contains "isn't working" or "can you add", this is most likely the wrong template.
-        If you are in doubt if this is the right template, use another template!
-      placeholder: WRITE QUESTION HERE
+      label: Please make sure the question is worded well enough to be understood
+      description: See [is-the-description-of-the-issue-itself-sufficient](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-description-of-the-issue-itself-sufficient)
+      placeholder: Provide any additional information and as much context and examples as possible
     validations:
       required: true
+  - type: checkboxes
+    id: verbose
+    attributes:
+      label: Provide verbose output that clearly demonstrates the problem
+      options:
+        - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU <your command line>`)
+        - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below
   - type: textarea
     id: log
     attributes:
-      label: Verbose log
+      label: Complete Verbose Output
       description: |
-        If your question involves a yt-dlp command, provide the complete verbose output of that command.
-        Add the `-vU` flag to **your** command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
-        It should look similar to this:
+        It should start like this:
       placeholder: |
-        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
-        [debug] Portable config file: yt-dlp.conf
-        [debug] Portable config: ['-i']
-        [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2021.12.01 (exe)
-        [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
-        [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
-        [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
+        [debug] Command-line config: ['-vU', 'test:youtube']
+        [debug] Portable config "yt-dlp.conf": ['-i']
+        [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
+        [debug] yt-dlp version 2022.07.18 [9d339c4] (win32_exe)
+        [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
+        [debug] Checking exe version: ffmpeg -bsfs
+        [debug] Checking exe version: ffprobe -bsfs
+        [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
+        [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
         [debug] Proxy map: {}
-        yt-dlp is up to date (2021.12.01)
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
+        Latest version: 2022.07.18, Current version: 2022.07.18
+        yt-dlp is up to date (2022.07.18)
         <more lines>
       render: shell

.github/ISSUE_TEMPLATE_tmpl/1_broken_site.yml

@@ -17,7 +17,7 @@ body:
           required: true
         - label: I've checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/ytdl-org/youtube-dl#video-url-contains-an-ampersand-and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command)
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues including closed ones. DO NOT post duplicates
+        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
@@ -26,38 +26,14 @@ body:
     id: region
     attributes:
       label: Region
-      description: "Enter the region the site is accessible from"
-      placeholder: "India"
+      description: Enter the country/region that the site is accessible from
+      placeholder: India
   - type: textarea
     id: description
     attributes:
-      label: Description
-      description: |
-        Provide an explanation of your issue in an arbitrary form.
-        Provide any additional information, any suggested solutions, and as much context and examples as possible
-      placeholder: WRITE DESCRIPTION HERE
-    validations:
-      required: true
-  - type: textarea
-    id: log
-    attributes:
-      label: Verbose log
-      description: |
-        Provide the complete verbose output of yt-dlp **that clearly demonstrates the problem**.
-        Add the `-vU` flag to your command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
-        It should look similar to this:
-      placeholder: |
-        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
-        [debug] Portable config file: yt-dlp.conf
-        [debug] Portable config: ['-i']
-        [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version %(version)s (exe)
-        [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
-        [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
-        [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
-        [debug] Proxy map: {}
-        yt-dlp is up to date (%(version)s)
-        <more lines>
-      render: shell
+      label: Provide a description that is worded well enough to be understood
+      description: See [is-the-description-of-the-issue-itself-sufficient](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-description-of-the-issue-itself-sufficient)
+      placeholder: Provide any additional information, any suggested solutions, and as much context and examples as possible
     validations:
       required: true
+  %(verbose)s

.github/ISSUE_TEMPLATE_tmpl/2_site_support_request.yml

@@ -17,7 +17,7 @@ body:
           required: true
         - label: I've checked that none of provided URLs [violate any copyrights](https://github.com/ytdl-org/youtube-dl#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free) or contain any [DRM](https://en.wikipedia.org/wiki/Digital_rights_management) to the best of my knowledge
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues including closed ones. DO NOT post duplicates
+        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
@@ -26,8 +26,8 @@ body:
     id: region
     attributes:
       label: Region
-      description: "Enter the region the site is accessible from"
-      placeholder: "India"
+      description: Enter the country/region that the site is accessible from
+      placeholder: India
   - type: textarea
     id: example-urls
     attributes:
@@ -43,32 +43,9 @@ body:
   - type: textarea
     id: description
     attributes:
-      label: Description
-      description: |
-        Provide any additional information
-      placeholder: WRITE DESCRIPTION HERE
-    validations:
-      required: true
-  - type: textarea
-    id: log
-    attributes:
-      label: Verbose log
-      description: |
-        Provide the complete verbose output **using one of the example URLs provided above**.
-        Add the `-vU` flag to your command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
-        It should look similar to this:
-      placeholder: |
-        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
-        [debug] Portable config file: yt-dlp.conf
-        [debug] Portable config: ['-i']
-        [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version %(version)s (exe)
-        [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
-        [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
-        [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
-        [debug] Proxy map: {}
-        yt-dlp is up to date (%(version)s)
-        <more lines>
-      render: shell
+      label: Provide a description that is worded well enough to be understood
+      description: See [is-the-description-of-the-issue-itself-sufficient](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-description-of-the-issue-itself-sufficient)
+      placeholder: Provide any additional information, any suggested solutions, and as much context and examples as possible
     validations:
       required: true
+  %(verbose)s

.github/ISSUE_TEMPLATE_tmpl/3_site_feature_request.yml

@@ -15,7 +15,7 @@ body:
           required: true
         - label: I've checked that all provided URLs are playable in a browser with the same IP and same login details
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues including closed ones. DO NOT post duplicates
+        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
@@ -24,8 +24,8 @@ body:
     id: region
     attributes:
       label: Region
-      description: "Enter the region the site is accessible from"
-      placeholder: "India"
+      description: Enter the country/region that the site is accessible from
+      placeholder: India
   - type: textarea
     id: example-urls
     attributes:
@@ -39,34 +39,9 @@ body:
   - type: textarea
     id: description
     attributes:
-      label: Description
-      description: |
-        Provide an explanation of your site feature request in an arbitrary form.
-        Please make sure the description is worded well enough to be understood, see [is-the-description-of-the-issue-itself-sufficient](https://github.com/ytdl-org/youtube-dl#is-the-description-of-the-issue-itself-sufficient).
-        Provide any additional information, any suggested solutions, and as much context and examples as possible
-      placeholder: WRITE DESCRIPTION HERE
-    validations:
-      required: true
-  - type: textarea
-    id: log
-    attributes:
-      label: Verbose log
-      description: |
-        Provide the complete verbose output of yt-dlp that demonstrates the need for the enhancement.
-        Add the `-vU` flag to your command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
-        It should look similar to this:
-      placeholder: |
-        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
-        [debug] Portable config file: yt-dlp.conf
-        [debug] Portable config: ['-i']
-        [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version %(version)s (exe)
-        [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
-        [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
-        [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
-        [debug] Proxy map: {}
-        yt-dlp is up to date (%(version)s)
-        <more lines>
-      render: shell
+      label: Provide a description that is worded well enough to be understood
+      description: See [is-the-description-of-the-issue-itself-sufficient](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-description-of-the-issue-itself-sufficient)
+      placeholder: Provide any additional information, any suggested solutions, and as much context and examples as possible
     validations:
       required: true
+  %(verbose)s

.github/ISSUE_TEMPLATE_tmpl/4_bug_report.yml

@@ -17,41 +17,16 @@ body:
           required: true
         - label: I've checked that all URLs and arguments with special characters are [properly quoted or escaped](https://github.com/ytdl-org/youtube-dl#video-url-contains-an-ampersand-and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command)
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues including closed ones. DO NOT post duplicates
+        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
   - type: textarea
     id: description
     attributes:
-      label: Description
-      description: |
-        Provide an explanation of your issue in an arbitrary form.
-        Please make sure the description is worded well enough to be understood, see [is-the-description-of-the-issue-itself-sufficient](https://github.com/ytdl-org/youtube-dl#is-the-description-of-the-issue-itself-sufficient).
-        Provide any additional information, any suggested solutions, and as much context and examples as possible
-      placeholder: WRITE DESCRIPTION HERE
-    validations:
-      required: true
-  - type: textarea
-    id: log
-    attributes:
-      label: Verbose log
-      description: |
-        Provide the complete verbose output of yt-dlp **that clearly demonstrates the problem**.
-        Add the `-vU` flag to **your** command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
-        It should look similar to this:
-      placeholder: |
-        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
-        [debug] Portable config file: yt-dlp.conf
-        [debug] Portable config: ['-i']
-        [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version %(version)s (exe)
-        [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
-        [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
-        [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
-        [debug] Proxy map: {}
-        yt-dlp is up to date (%(version)s)
-        <more lines>
-      render: shell
+      label: Provide a description that is worded well enough to be understood
+      description: See [is-the-description-of-the-issue-itself-sufficient](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-description-of-the-issue-itself-sufficient)
+      placeholder: Provide any additional information, any suggested solutions, and as much context and examples as possible
     validations:
       required: true
+    %(verbose)s

.github/ISSUE_TEMPLATE_tmpl/5_feature_request.yml

@@ -15,39 +15,16 @@ body:
           required: true
         - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues including closed ones. DO NOT post duplicates
+        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar issues **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
   - type: textarea
     id: description
     attributes:
-      label: Description
-      description: |
-        Provide an explanation of your site feature request in an arbitrary form.
-        Please make sure the description is worded well enough to be understood, see [is-the-description-of-the-issue-itself-sufficient](https://github.com/ytdl-org/youtube-dl#is-the-description-of-the-issue-itself-sufficient).
-        Provide any additional information, any suggested solutions, and as much context and examples as possible
-      placeholder: WRITE DESCRIPTION HERE
+      label: Provide a description that is worded well enough to be understood
+      description: See [is-the-description-of-the-issue-itself-sufficient](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-description-of-the-issue-itself-sufficient)
+      placeholder: Provide any additional information, any suggested solutions, and as much context and examples as possible
     validations:
       required: true
-  - type: textarea
-    id: log
-    attributes:
-      label: Verbose log
-      description: |
-        If your feature request involves an existing yt-dlp command, provide the complete verbose output of that command.
-        Add the `-vU` flag to **your** command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
-        It should look similar to this:
-      placeholder: |
-        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
-        [debug] Portable config file: yt-dlp.conf
-        [debug] Portable config: ['-i']
-        [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2021.12.01 (exe)
-        [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
-        [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
-        [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
-        [debug] Proxy map: {}
-        yt-dlp is up to date (2021.12.01)
-        <more lines>
-      render: shell
+  %(verbose_optional)s

.github/ISSUE_TEMPLATE_tmpl/6_question.yml

@@ -2,6 +2,12 @@ name: Ask question
 description: Ask yt-dlp related question
 labels: [question]
 body:
+  - type: markdown
+    attributes:
+      value: |
+        ### Make sure you are **only** asking a question and not reporting a bug or requesting a feature.
+        If your question contains "isn't working" or "can you add", this is most likely the wrong template.
+        If you are in doubt whether this is the right template, **use another template**!
   - type: checkboxes
     id: checklist
     attributes:
@@ -15,41 +21,16 @@ body:
           required: true
         - label: I've verified that I'm running yt-dlp version **%(version)s** ([update instructions](https://github.com/yt-dlp/yt-dlp#update)) or later (specify commit)
           required: true
-        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions including closed ones. DO NOT post duplicates
+        - label: I've searched the [bugtracker](https://github.com/yt-dlp/yt-dlp/issues?q=) for similar questions **including closed ones**. DO NOT post duplicates
           required: true
         - label: I've read the [guidelines for opening an issue](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#opening-an-issue)
           required: true
   - type: textarea
     id: question
     attributes:
-      label: Question
-      description: |
-        Ask your question in an arbitrary form.
-        Please make sure it's worded well enough to be understood, see [is-the-description-of-the-issue-itself-sufficient](https://github.com/ytdl-org/youtube-dl#is-the-description-of-the-issue-itself-sufficient).
-        Provide any additional information and as much context and examples as possible.
-        If your question contains "isn't working" or "can you add", this is most likely the wrong template.
-        If you are in doubt if this is the right template, use another template!
-      placeholder: WRITE QUESTION HERE
+      label: Please make sure the question is worded well enough to be understood
+      description: See [is-the-description-of-the-issue-itself-sufficient](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-description-of-the-issue-itself-sufficient)
+      placeholder: Provide any additional information and as much context and examples as possible
     validations:
       required: true
-  - type: textarea
-    id: log
-    attributes:
-      label: Verbose log
-      description: |
-        If your question involves a yt-dlp command, provide the complete verbose output of that command.
-        Add the `-vU` flag to **your** command line you run yt-dlp with (`yt-dlp -vU <your command line>`), copy the WHOLE output and insert it below.
-        It should look similar to this:
-      placeholder: |
-        [debug] Command-line config: ['-vU', 'http://www.youtube.com/watch?v=BaW_jenozKc']
-        [debug] Portable config file: yt-dlp.conf
-        [debug] Portable config: ['-i']
-        [debug] Encodings: locale cp1252, fs utf-8, stdout utf-8, stderr utf-8, pref cp1252
-        [debug] yt-dlp version 2021.12.01 (exe)
-        [debug] Python version 3.8.8 (CPython 64bit) - Windows-10-10.0.19041-SP0
-        [debug] exe versions: ffmpeg 3.0.1, ffprobe 3.0.1
-        [debug] Optional libraries: Cryptodome, keyring, mutagen, sqlite, websockets
-        [debug] Proxy map: {}
-        yt-dlp is up to date (2021.12.01)
-        <more lines>
-      render: shell
+  %(verbose_optional)s

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,5 +1,8 @@
+<details open><summary>Template</summary> <!-- OPEN is intentional -->
+
 <!--
-# Please follow the guide below
+
+# PLEASE FOLLOW THE GUIDE BELOW
 
 - You will be asked some questions, please read them **carefully** and answer honestly
 - Put an `x` into all the boxes `[ ]` relevant to your *pull request* (like [x])
@@ -22,8 +25,16 @@
 - [ ] Core bug fix/improvement
 - [ ] New feature (It is strongly [recommended to open an issue first](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#adding-new-feature-or-making-overarching-changes))
 
----
-
 ### Description of your *pull request* and other information
 
-Explanation of your *pull request* in arbitrary form goes here. Please **make sure the description explains the purpose and effect** of your *pull request* and is worded well enough to be understood. Provide as much **context and examples** as possible.
+</details>
+
+<!--
+
+Explanation of your *pull request* in arbitrary form goes here. Please **make sure the description explains the purpose and effect** of your *pull request* and is worded well enough to be understood. Provide as much **context and examples** as possible
+
+-->
+
+DESCRIPTION
+
+Fixes #

.github/workflows/build.yml

@@ -244,6 +244,10 @@ jobs:
   build_macos_legacy:
     runs-on: macos-latest
     needs: create_release
+    outputs:
+      sha256_macos_legacy: ${{ steps.get_sha.outputs.sha256_macos_legacy }}
+      sha512_macos_legacy: ${{ steps.get_sha.outputs.sha512_macos_legacy }}
+
     steps:
     - uses: actions/checkout@v2
     - name: Install Python
@@ -452,6 +456,7 @@ jobs:
     - name: Make Update spec
       run: |
         echo "# This file is used for regulating self-update" >> _update_spec
+        echo "lock 2022.07.18 .+ Python 3.6" >> _update_spec
     - name: Upload update spec
       uses: actions/upload-release-asset@v1
       env:

.github/workflows/download.yml

@@ -1,15 +1,31 @@
 name: Download Tests
 on: [push, pull_request]
 jobs:
-  tests:
-    name: Download Tests
+  quick:
+    name: Quick Download Tests
     if: "contains(github.event.head_commit.message, 'ci run dl')"
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+    - name: Install test requirements
+      run: pip install pytest
+    - name: Run tests
+      continue-on-error: true
+      run: ./devscripts/run_tests.sh download
+
+  full:
+    name: Full Download Tests
+    if: "contains(github.event.head_commit.message, 'ci run dl all')"
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: true
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.6', '3.7', '3.9', '3.10', 3.11-dev, pypy-3.6, pypy-3.7, pypy-3.8]
+        python-version: ['3.6', '3.7', '3.10', 3.11-dev, pypy-3.6, pypy-3.7, pypy-3.8]
         run-tests-ext: [sh]
         include:
         # atleast one of each CPython/PyPy tests must be in windows

.gitignore

@@ -27,11 +27,13 @@ cookies
 *.ass
 *.avi
 *.desktop
+*.f4v
 *.flac
 *.flv
 *.jpeg
 *.jpg
 *.m4a
+*.mpga
 *.m4v
 *.mhtml
 *.mkv

CONTRIBUTORS

@@ -272,3 +272,16 @@ crazymoose77756
 nomevi
 Brett824
 pingiun
+dosy4ev
+EhtishamSabir
+Ferdi265
+FirefoxMetzger
+ftk
+lamby
+llamasblade
+lockmatrix
+misaelaguayo
+odo2063
+pritam20ps05
+scy
+sheerluck

Changelog.md

@@ -11,6 +11,73 @@
 -->
 
 
+### 2022.07.18
+
+* Allow users to specify encoding in each config files by [Lesmiscore](https://github.com/Lesmiscore)
+* Discard infodict from memory if no longer needed
+* Do not allow extractors to return `None`
+* Do not load system certificates when `certifi` is used
+* Fix rounding of integers in format table
+* Improve chapter sanitization
+* Skip some fixup if remux/recode is needed by [Lesmiscore](https://github.com/Lesmiscore)
+* Support `--no-progress` for `--wait-for-video`
+* Fix bug in [612f2be](https://github.com/yt-dlp/yt-dlp/commit/612f2be5d3924540158dfbe5f25d841f04cff8c6)
+* [outtmpl] Add alternate form `h` for HTML escaping
+* [aes] Add multiple padding modes in CBC by [elyse0](https://github.com/elyse0)
+* [extractor/common] Passthrough `errnote=False` to parsers
+* [extractor/generic] Remove HEAD request
+* [http] Ensure the file handle is always closed
+* [ModifyChapters] Modify duration in infodict
+* [options] Fix aliases to `--config-location`
+* [utils] Fix `get_domain`
+* [build] Consistent order for lazy extractors by [lamby](https://github.com/lamby)
+* [build] Fix architecture suffix of executables by [odo2063](https://github.com/odo2063)
+* [build] Improve `setup.py`
+* [update] Do not check `_update_spec` when up to date
+* [update] Prepare to remove Python 3.6 support
+* [compat] Let PyInstaller detect _legacy module
+* [devscripts/update-formulae] Do not change dependency section
+* [test] Split download tests so they can be more easily run in CI
+* [docs] Improve docstring of `download_ranges` by [FirefoxMetzger](https://github.com/FirefoxMetzger)
+* [docs] Improve issue templates
+* [build] Fix bug in [6d916fe](https://github.com/yt-dlp/yt-dlp/commit/6d916fe709a38e8c4c69b73843acf170b5165931)
+* [cleanup, utils] Refactor parse_codecs
+* [cleanup] Misc fixes and cleanup
+* [extractor/acfun] Add extractors by [lockmatrix](https://github.com/lockmatrix)
+* [extractor/Audiodraft] Add extractors by [Ashish0804](https://github.com/Ashish0804), [fstirlitz](https://github.com/fstirlitz)
+* [extractor/cellebrite] Add extractor by [HobbyistDev](https://github.com/HobbyistDev)
+* [extractor/detik] Add extractor by [HobbyistDev](https://github.com/HobbyistDev)
+* [extractor/hytale] Add extractor by [llamasblade](https://github.com/llamasblade), [pukkandan](https://github.com/pukkandan)
+* [extractor/liputan6] Add extractor by [HobbyistDev](https://github.com/HobbyistDev)
+* [extractor/mocha] Add extractor by [HobbyistDev](https://github.com/HobbyistDev)
+* [extractor/rtl.lu] Add extractor by [HobbyistDev](https://github.com/HobbyistDev)
+* [extractor/rtvsl] Add extractor by [iw0nderhow](https://github.com/iw0nderhow), [pukkandan](https://github.com/pukkandan)
+* [extractor/StarTrek] Add extractor by [scy](https://github.com/scy)
+* [extractor/syvdk] Add extractor by [misaelaguayo](https://github.com/misaelaguayo)
+* [extractor/theholetv] Add extractor by [dosy4ev](https://github.com/dosy4ev)
+* [extractor/TubeTuGraz] Add extractor by [Ferdi265](https://github.com/Ferdi265), [pukkandan](https://github.com/pukkandan)
+* [extractor/tviplayer] Add extractor by [HobbyistDev](https://github.com/HobbyistDev)
+* [extractor/wetv] Add extractors by [elyse0](https://github.com/elyse0)
+* [extractor/wikimedia] Add extractor by [EhtishamSabir](https://github.com/EhtishamSabir), [pukkandan](https://github.com/pukkandan)
+* [extractor/youtube] Fix duration check for post-live manifestless mode
+* [extractor/youtube] More metadata for storyboards by [ftk](https://github.com/ftk)
+* [extractor/bigo] Fix extractor by [Lesmiscore](https://github.com/Lesmiscore)
+* [extractor/BiliIntl] Fix subtitle extraction by [MinePlayersPE](https://github.com/MinePlayersPE)
+* [extractor/crunchyroll] Improve `_VALID_URL`
+* [extractor/fifa] Fix extractor by [ischmidt20](https://github.com/ischmidt20)
+* [extractor/instagram] Fix post/story extractors by [pritam20ps05](https://github.com/pritam20ps05), [pukkandan](https://github.com/pukkandan)
+* [extractor/iq] Set language correctly for Korean subtitles
+* [extractor/MangoTV] Fix subtitle languages
+* [extractor/Netverse] Improve playlist extractor by [HobbyistDev](https://github.com/HobbyistDev)
+* [extractor/philharmoniedeparis] Fix extractor by [sqrtNOT](https://github.com/sqrtNOT)
+* [extractor/Trovo] Fix extractor by [u-spec-png](https://github.com/u-spec-png)
+* [extractor/twitch] Support storyboards for VODs by [ftk](https://github.com/ftk)
+* [extractor/WatchESPN] Improve `_VALID_URL` by [IONECarter](https://github.com/IONECarter), [dirkf](https://github.com/dirkf)
+* [extractor/WSJArticle] Fix video id extraction by [sqrtNOT](https://github.com/sqrtNOT)
+* [extractor/Ximalaya] Fix extractors by [lockmatrix](https://github.com/lockmatrix)
+* [cleanup, extractor/youtube] Fix tests by [sheerluck](https://github.com/sheerluck)
+
+
 ### 2022.06.29
 
 * Fix `--downloader native`

Makefile

@@ -17,8 +17,8 @@ pypi-files: AUTHORS Changelog.md LICENSE README.md README.txt supportedsites \
 clean-test:
 	rm -rf test/testdata/sigs/player-*.js tmp/ *.annotations.xml *.aria2 *.description *.dump *.frag \
 	*.frag.aria2 *.frag.urls *.info.json *.live_chat.json *.meta *.part* *.tmp *.temp *.unknown_video *.ytdl \
-	*.3gp *.ape *.ass *.avi *.desktop *.flac *.flv *.jpeg *.jpg *.m4a *.m4v *.mhtml *.mkv *.mov *.mp3 \
-	*.mp4 *.ogg *.opus *.png *.sbv *.srt *.swf *.swp *.ttml *.url *.vtt *.wav *.webloc *.webm *.webp
+	*.3gp *.ape *.ass *.avi *.desktop *.f4v *.flac *.flv *.jpeg *.jpg *.m4a *.mpga *.m4v *.mhtml *.mkv *.mov \
+	*.mp3 *.mp4 *.ogg *.opus *.png *.sbv *.srt *.swf *.swp *.ttml *.url *.vtt *.wav *.webloc *.webm *.webp
 clean-dist:
 	rm -rf yt-dlp.1.temp.md yt-dlp.1 README.txt MANIFEST build/ dist/ .coverage cover/ yt-dlp.tar.gz completions/ \
 	yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS .mailmap

README.md

@@ -146,7 +146,7 @@ Some of yt-dlp's default options are different from that of youtube-dl and youtu
 * Thumbnail embedding in `mp4` is done with mutagen if possible. Use `--compat-options embed-thumbnail-atomicparsley` to force the use of AtomicParsley instead
 * Some private fields such as filenames are removed by default from the infojson. Use `--no-clean-infojson` or `--compat-options no-clean-infojson` to revert this
 * When `--embed-subs` and `--write-subs` are used together, the subtitles are written to disk and also embedded in the media file. You can use just `--embed-subs` to embed the subs and automatically delete the separate file. See [#630 (comment)](https://github.com/yt-dlp/yt-dlp/issues/630#issuecomment-893659460) for more info. `--compat-options no-keep-subs` can be used to revert this
-* `certifi` will be used for SSL root certificates, if installed. If you want to use only system certificates, use `--compat-options no-certifi`
+* `certifi` will be used for SSL root certificates, if installed. If you want to use system certificates (e.g. self-signed), use `--compat-options no-certifi`
 * youtube-dl tries to remove some superfluous punctuations from filenames. While this can sometimes be helpful, it is often undesirable. So yt-dlp tries to keep the fields in the filenames as close to their original values as possible. You can use `--compat-options filename-sanitization` to revert to youtube-dl's behavior
 
 For ease of use, a few more compat options are available:
@@ -238,7 +238,7 @@ File|Description
 :---|:---
 [yt-dlp](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp)|Platform-independent [zipimport](https://docs.python.org/3/library/zipimport.html) binary. Needs Python (recommended for **Linux/BSD**)
 [yt-dlp.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp.exe)|Windows (Win7 SP1+) standalone x64 binary (recommended for **Windows**)
-[yt-dlp_macos](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos)|MacOS (10.15+) standalone executable (recommended for **MacOS**)
+[yt-dlp_macos](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos)|Universal MacOS (10.15+) standalone executable (recommended for **MacOS**)
 
 #### Alternatives
 
@@ -246,8 +246,8 @@ File|Description
 :---|:---
 [yt-dlp_x86.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_x86.exe)|Windows (Vista SP2+) standalone x86 (32-bit) binary
 [yt-dlp_min.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_min.exe)|Windows (Win7 SP1+) standalone x64 binary built with `py2exe`<br/> ([Not recommended](#standalone-py2exe-builds-windows))
-[yt-dlp_linux](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux)|UNIX standalone x64 binary
-[yt-dlp_linux.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux.zip)|Unpackaged Unix executable (no auto-update)
+[yt-dlp_linux](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux)|Linux standalone x64 binary
+[yt-dlp_linux.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux.zip)|Unpackaged Linux executable (no auto-update)
 [yt-dlp_win.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_win.zip)|Unpackaged Windows executable (no auto-update)
 [yt-dlp_macos.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos.zip)|Unpackaged MacOS (10.15+) executable (no auto-update)
 [yt-dlp_macos_legacy](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos_legacy)|MacOS (10.9+) standalone x64 executable
@@ -305,7 +305,7 @@ While all the other dependencies are optional, `ffmpeg` and `ffprobe` are highly
 
 To use or redistribute the dependencies, you must agree to their respective licensing terms.
 
-The Windows and MacOS standalone release binaries are built with the Python interpreter and the packages marked with **\*** included.
+The standalone release binaries are built with the Python interpreter and the packages marked with **\*** included.
 
 If you do not have the necessary dependencies for a task you are attempting, yt-dlp will warn you. All the currently available dependencies are visible at the top of the `--verbose` output
 
@@ -414,7 +414,8 @@ You can also fork the project on github and run your fork's [build workflow](.gi
     --no-wait-for-video             Do not wait for scheduled streams (default)
     --mark-watched                  Mark videos watched (even with --simulate)
     --no-mark-watched               Do not mark videos watched (default)
-    --no-colors                     Do not emit color codes in output
+    --no-colors                     Do not emit color codes in output (Alias:
+                                    --no-colours)
     --compat-options OPTS           Options that can help keep compatibility
                                     with youtube-dl or youtube-dlc
                                     configurations by reverting some of the
@@ -1160,6 +1161,12 @@ Note that options in configuration file are just the same options aka switches u
 
 You can use `--ignore-config` if you want to disable all configuration files for a particular yt-dlp run. If `--ignore-config` is found inside any configuration file, no further configuration will be loaded. For example, having the option in the portable configuration file prevents loading of home, user, and system configurations. Additionally, (for backward compatibility) if `--ignore-config` is found inside the system configuration file, the user configuration is not loaded.
 
+### Config file encoding
+
+The config files are decoded according to the UTF BOM if present, and in the encoding from system locale otherwise.
+
+If you want your file to be decoded differently, add `# coding: ENCODING` to the beginning of the file (e.g. `# coding: shift-jis`). There must be no characters before that, even spaces or BOM.
+
 ### Authentication with `.netrc` file
 
 You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every yt-dlp execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](https://stackoverflow.com/tags/.netrc/info) on a per extractor basis. For that you will need to create a `.netrc` file in `--netrc-location` and restrict permissions to read/write by only you:
@@ -1206,7 +1213,7 @@ The field names themselves (the part inside the parenthesis) can also have some
 
 1. **Default**: A literal default value can be specified for when the field is empty using a `|` separator. This overrides `--output-na-template`. Eg: `%(uploader|Unknown)s`
 
-1. **More Conversions**: In addition to the normal format types `diouxXeEfFgGcrs`, `B`, `j`, `l`, `q`, `D`, `S` can be used for converting to **B**ytes, **j**son (flag `#` for pretty-printing), a comma separated **l**ist (flag `#` for `\n` newline-separated), a string **q**uoted for the terminal (flag `#` to split a list into different arguments), to add **D**ecimal suffixes (Eg: 10M) (flag `#` to use 1024 as factor), and to **S**anitize as filename (flag `#` for restricted), respectively
+1. **More Conversions**: In addition to the normal format types `diouxXeEfFgGcrs`, yt-dlp additionally supports converting to `B` = **B**ytes, `j` = **j**son (flag `#` for pretty-printing), `h` = HTML escaping, `l` = a comma separated **l**ist (flag `#` for `\n` newline-separated), `q` = a string **q**uoted for the terminal (flag `#` to split a list into different arguments), `D` = add **D**ecimal suffixes (Eg: 10M) (flag `#` to use 1024 as factor), and `S` = **S**anitize as filename (flag `#` for restricted)
 
 1. **Unicode normalization**: The format type `U` can be used for NFC [unicode normalization](https://docs.python.org/3/library/unicodedata.html#unicodedata.normalize). The alternate form flag (`#`) changes the normalization to NFD and the conversion flag `+` can be used for NFKC/NFKD compatibility equivalence normalization. Eg: `%(title)+.100U` is NFKC
 

devscripts/make_issue_template.py

@@ -8,6 +8,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
 import optparse
+import re
 
 
 def read(fname):
@@ -21,16 +22,56 @@ def read_version(fname):
     return locals()['__version__']
 
 
+VERBOSE_TMPL = '''
+  - type: checkboxes
+    id: verbose
+    attributes:
+      label: Provide verbose output that clearly demonstrates the problem
+      options:
+        - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU <your command line>`)
+          required: true
+        - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below
+          required: true
+  - type: textarea
+    id: log
+    attributes:
+      label: Complete Verbose Output
+      description: |
+        It should start like this:
+      placeholder: |
+        [debug] Command-line config: ['-vU', 'test:youtube']
+        [debug] Portable config "yt-dlp.conf": ['-i']
+        [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8
+        [debug] yt-dlp version %(version)s [9d339c4] (win32_exe)
+        [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0
+        [debug] Checking exe version: ffmpeg -bsfs
+        [debug] Checking exe version: ffprobe -bsfs
+        [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1
+        [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3
+        [debug] Proxy map: {}
+        [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest
+        Latest version: %(version)s, Current version: %(version)s
+        yt-dlp is up to date (%(version)s)
+        <more lines>
+      render: shell
+    validations:
+      required: true
+'''.strip()
+
+
 def main():
     parser = optparse.OptionParser(usage='%prog INFILE OUTFILE')
-    options, args = parser.parse_args()
+    _, args = parser.parse_args()
     if len(args) != 2:
         parser.error('Expected an input and an output filename')
 
+    fields = {'version': read_version('yt_dlp/version.py')}
+    fields['verbose'] = VERBOSE_TMPL % fields
+    fields['verbose_optional'] = re.sub(r'(\n\s+validations:)?\n\s+required: true', '', fields['verbose'])
+
     infile, outfile = args
     with open(outfile, 'w', encoding='utf-8') as outf:
-        outf.write(
-            read(infile) % {'version': read_version('yt_dlp/version.py')})
+        outf.write(read(infile) % fields)
 
 
 if __name__ == '__main__':

devscripts/make_lazy_extractors.py

@@ -94,7 +94,7 @@ def sort_ies(ies, ignored_bases):
         for c in classes[:]:
             bases = set(c.__bases__) - {object, *ignored_bases}
             restart = False
-            for b in bases:
+            for b in sorted(bases, key=lambda x: x.__name__):
                 if b not in classes and b not in returned_classes:
                     assert b.__name__ != 'GenericIE', 'Cannot inherit from GenericIE'
                     classes.insert(0, b)

devscripts/update-formulae.py

@@ -30,8 +30,8 @@ url = tarball_file['url']
 with open(filename) as r:
     formulae_text = r.read()
 
-formulae_text = re.sub(r'sha256 "[0-9a-f]*?"', 'sha256 "%s"' % sha256sum, formulae_text)
-formulae_text = re.sub(r'url "[^"]*?"', 'url "%s"' % url, formulae_text)
+formulae_text = re.sub(r'sha256 "[0-9a-f]*?"', 'sha256 "%s"' % sha256sum, formulae_text, count=1)
+formulae_text = re.sub(r'url "[^"]*?"', 'url "%s"' % url, formulae_text, count=1)
 
 with open(filename, 'w') as w:
     w.write(formulae_text)

pyinst.py

@@ -6,7 +6,10 @@ import sys
 
 from PyInstaller.__main__ import run as run_pyinstaller
 
-OS_NAME, ARCH = sys.platform, platform.architecture()[0][:2]
+OS_NAME, MACHINE, ARCH = sys.platform, platform.machine(), platform.architecture()[0][:2]
+if MACHINE in ('x86_64', 'AMD64') or ('i' in MACHINE and '86' in MACHINE):
+    # NB: Windows x86 has MACHINE = AMD64 irrespective of bitness
+    MACHINE = 'x86' if ARCH == '32' else ''
 
 
 def main():
@@ -18,7 +21,7 @@ def main():
         opts.append('--onefile')
 
     name, final_file = exe(onedir)
-    print(f'Building yt-dlp v{version} {ARCH}bit for {OS_NAME} with options {opts}')
+    print(f'Building yt-dlp v{version} for {OS_NAME} {platform.machine()} with options {opts}')
     print('Remember to update the version using  "devscripts/update-version.py"')
     if not os.path.isfile('yt_dlp/extractor/lazy_extractors.py'):
         print('WARNING: Building without lazy_extractors. Run  '
@@ -30,9 +33,6 @@ def main():
         '--icon=devscripts/logo.ico',
         '--upx-exclude=vcruntime140.dll',
         '--noconfirm',
-        # NB: Modules that are only imported dynamically must be added here.
-        # --collect-submodules may not work correctly if user has a yt-dlp installed via PIP
-        '--hidden-import=yt_dlp.compat._legacy',
         *dependency_options(),
         *opts,
         'yt_dlp/__main__.py',
@@ -65,7 +65,7 @@ def exe(onedir):
     name = '_'.join(filter(None, (
         'yt-dlp',
         {'win32': '', 'darwin': 'macos'}.get(OS_NAME, OS_NAME),
-        ARCH == '32' and 'x86'
+        MACHINE
     )))
     return name, ''.join(filter(None, (
         'dist/',
@@ -122,7 +122,7 @@ def windows_set_version(exe, version):
     )
 
     version_list = version_to_list(version)
-    suffix = '_x86' if ARCH == '32' else ''
+    suffix = MACHINE and f'_{MACHINE}'
     SetVersion(exe, VSVersionInfo(
         ffi=FixedFileInfo(
             filevers=version_list,
@@ -136,9 +136,9 @@ def windows_set_version(exe, version):
         ),
         kids=[
             StringFileInfo([StringTable('040904B0', [
-                StringStruct('Comments', 'yt-dlp%s Command Line Interface.' % suffix),
+                StringStruct('Comments', 'yt-dlp%s Command Line Interface' % suffix),
                 StringStruct('CompanyName', 'https://github.com/yt-dlp'),
-                StringStruct('FileDescription', 'yt-dlp%s' % (' (32 Bit)' if ARCH == '32' else '')),
+                StringStruct('FileDescription', 'yt-dlp%s' % (MACHINE and f' ({MACHINE})')),
                 StringStruct('FileVersion', version),
                 StringStruct('InternalName', f'yt-dlp{suffix}'),
                 StringStruct('LegalCopyright', 'pukkandan.ytdlp@gmail.com | UNLICENSE'),

setup.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import os.path
+import subprocess
 import sys
 import warnings
 
@@ -10,7 +11,6 @@ try:
 except ImportError:
     from distutils.core import Command, setup
     setuptools_available = False
-from distutils.spawn import spawn
 
 
 def read(fname):
@@ -36,12 +36,24 @@ LONG_DESCRIPTION = '\n\n'.join((
 REQUIREMENTS = read('requirements.txt').splitlines()
 
 
-if sys.argv[1:2] == ['py2exe']:
+def packages():
+    if setuptools_available:
+        return find_packages(exclude=('youtube_dl', 'youtube_dlc', 'test', 'ytdlp_plugins'))
+
+    return [
+        'yt_dlp', 'yt_dlp.extractor', 'yt_dlp.downloader', 'yt_dlp.postprocessor', 'yt_dlp.compat',
+        'yt_dlp.extractor.anvato_token_generator',
+    ]
+
+
+def py2exe_params():
     import py2exe  # noqa: F401
+
     warnings.warn(
         'py2exe builds do not support pycryptodomex and needs VC++14 to run. '
         'The recommended way is to use "pyinst.py" to build using pyinstaller')
-    params = {
+
+    return {
         'console': [{
             'script': './yt_dlp/__main__.py',
             'dest_base': 'yt-dlp',
@@ -50,6 +62,7 @@ if sys.argv[1:2] == ['py2exe']:
             'comments': LONG_DESCRIPTION.split('\n')[0],
             'product_name': 'yt-dlp',
             'product_version': VERSION,
+            'icon_resources': [(1, 'devscripts/logo.ico')],
         }],
         'options': {
             'py2exe': {
@@ -66,7 +79,8 @@ if sys.argv[1:2] == ['py2exe']:
         'zipfile': None
     }
 
-else:
+
+def build_params():
     files_spec = [
         ('share/bash-completion/completions', ['completions/bash/yt-dlp']),
         ('share/zsh/site-functions', ['completions/zsh/_yt-dlp']),
@@ -74,25 +88,23 @@ else:
         ('share/doc/yt_dlp', ['README.txt']),
         ('share/man/man1', ['yt-dlp.1'])
     ]
-    root = os.path.dirname(os.path.abspath(__file__))
     data_files = []
     for dirname, files in files_spec:
         resfiles = []
         for fn in files:
             if not os.path.exists(fn):
-                warnings.warn('Skipping file %s since it is not present. Try running `make pypi-files` first' % fn)
+                warnings.warn(f'Skipping file {fn} since it is not present. Try running " make pypi-files " first')
             else:
                 resfiles.append(fn)
         data_files.append((dirname, resfiles))
 
-    params = {
-        'data_files': data_files,
-    }
+    params = {'data_files': data_files}
 
     if setuptools_available:
         params['entry_points'] = {'console_scripts': ['yt-dlp = yt_dlp:main']}
     else:
         params['scripts'] = ['yt-dlp']
+    return params
 
 
 class build_lazy_extractors(Command):
@@ -106,16 +118,13 @@ class build_lazy_extractors(Command):
         pass
 
     def run(self):
-        spawn([sys.executable, 'devscripts/make_lazy_extractors.py', 'yt_dlp/extractor/lazy_extractors.py'],
-              dry_run=self.dry_run)
-
-
-if setuptools_available:
-    packages = find_packages(exclude=('youtube_dl', 'youtube_dlc', 'test', 'ytdlp_plugins'))
-else:
-    packages = ['yt_dlp', 'yt_dlp.downloader', 'yt_dlp.extractor', 'yt_dlp.postprocessor']
+        if self.dry_run:
+            print('Skipping build of lazy extractors in dry run mode')
+            return
+        subprocess.run([sys.executable, 'devscripts/make_lazy_extractors.py', 'yt_dlp/extractor/lazy_extractors.py'])
 
 
+params = py2exe_params() if sys.argv[1:2] == ['py2exe'] else build_params()
 setup(
     name='yt-dlp',
     version=VERSION,
@@ -125,8 +134,9 @@ setup(
     long_description=LONG_DESCRIPTION,
     long_description_content_type='text/markdown',
     url='https://github.com/yt-dlp/yt-dlp',
-    packages=packages,
+    packages=packages(),
     install_requires=REQUIREMENTS,
+    python_requires='>=3.6',
     project_urls={
         'Documentation': 'https://github.com/yt-dlp/yt-dlp#readme',
         'Source': 'https://github.com/yt-dlp/yt-dlp',
@@ -150,8 +160,6 @@ setup(
         'License :: Public Domain',
         'Operating System :: OS Independent',
     ],
-    python_requires='>=3.6',
-
     cmdclass={'build_lazy_extractors': build_lazy_extractors},
     **params
 )

supportedsites.md

@@ -4,6 +4,7 @@
  - **17live**
  - **17live:clip**
  - **1tv**: Первый канал
+ - **20.detik.com**
  - **20min**
  - **23video**
  - **247sports**
@@ -31,6 +32,8 @@
  - **AcademicEarth:Course**
  - **acast**
  - **acast:channel**
+ - **AcFunBangumi**
+ - **AcFunVideo**
  - **ADN**: [<abbr title="netrc machine"><em>animedigitalnetwork</em></abbr>] Anime Digital Network
  - **AdobeConnect**
  - **adobetv**
@@ -94,6 +97,8 @@
  - **ATVAt**
  - **AudiMedia**
  - **AudioBoom**
+ - **Audiodraft:custom**
+ - **Audiodraft:generic**
  - **audiomack**
  - **audiomack:album**
  - **Audius**: Audius.co
@@ -205,6 +210,7 @@
  - **CCMA**
  - **CCTV**: 央视网
  - **CDA**
+ - **Cellebrite**
  - **CeskaTelevize**
  - **CGTN**
  - **channel9**: Channel 9
@@ -503,6 +509,7 @@
  - **HungamaSong**
  - **huya:live**: huya.com
  - **Hypem**
+ - **Hytale**
  - **Icareus**
  - **ign.com**
  - **IGNArticle**
@@ -615,6 +622,7 @@
  - **linkedin:learning**: [<abbr title="netrc machine"><em>linkedin</em></abbr>]
  - **linkedin:learning:course**: [<abbr title="netrc machine"><em>linkedin</em></abbr>]
  - **LinuxAcademy**: [<abbr title="netrc machine"><em>linuxacademy</em></abbr>]
+ - **Liputan6**
  - **LiTV**
  - **LiveJournal**
  - **livestream**
@@ -698,6 +706,7 @@
  - **MLSSoccer**
  - **Mnet**
  - **MNetTV**: [<abbr title="netrc machine"><em>mnettv</em></abbr>]
+ - **MochaVideo**
  - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
  - **Mofosex**
  - **MofosexEmbed**
@@ -1068,10 +1077,14 @@
  - **RTDocumentryPlaylist**
  - **rte**: Raidió Teilifís Éireann TV
  - **rte:radio**: Raidió Teilifís Éireann radio
+ - **rtl.lu:article**
+ - **rtl.lu:tele-vod**
  - **rtl.nl**: rtl.nl and rtlxl.nl
  - **rtl2**
  - **rtl2:you**
  - **rtl2:you:series**
+ - **RTLLuLive**
+ - **RTLLuRadio**
  - **RTNews**
  - **RTP**
  - **RTRFM**
@@ -1083,6 +1096,7 @@
  - **rtve.es:television**
  - **RTVNH**
  - **RTVS**
+ - **rtvslo.si**
  - **RUHD**
  - **Rule34Video**
  - **RumbleChannel**
@@ -1191,6 +1205,7 @@
  - **SRGSSR**
  - **SRGSSRPlay**: srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites
  - **stanfordoc**: Stanford Open ClassRoom
+ - **StarTrek**
  - **startv**
  - **Steam**
  - **SteamCommunityBroadcast**
@@ -1218,6 +1233,7 @@
  - **SVTSeries**
  - **SWRMediathek**
  - **Syfy**
+ - **SYVDK**
  - **SztvHu**
  - **t-online.de**
  - **Tagesschau**
@@ -1256,6 +1272,7 @@
  - **TenPlay**: [<abbr title="netrc machine"><em>10play</em></abbr>]
  - **TF1**
  - **TFO**
+ - **TheHoleTv**
  - **TheIntercept**
  - **ThePlatform**
  - **ThePlatformFeed**
@@ -1298,6 +1315,8 @@
  - **TruNews**
  - **TruTV**
  - **Tube8**
+ - **TubeTuGraz**: [<abbr title="netrc machine"><em>tubetugraz</em></abbr>] tube.tugraz.at
+ - **TubeTuGrazSeries**: [<abbr title="netrc machine"><em>tubetugraz</em></abbr>]
  - **TubiTv**: [<abbr title="netrc machine"><em>tubitv</em></abbr>]
  - **TubiTvShow**
  - **Tumblr**: [<abbr title="netrc machine"><em>tumblr</em></abbr>]
@@ -1326,6 +1345,7 @@
  - **TVCArticle**
  - **TVer**
  - **tvigle**: Интернет-телевидение Tvigle.ru
+ - **TVIPlayer**
  - **tvland.com**
  - **TVN24**
  - **TVNet**
@@ -1498,7 +1518,10 @@
  - **Weibo**
  - **WeiboMobile**
  - **WeiqiTV**: WQTV
+ - **wetv:episode**
+ - **WeTvSeries**
  - **whowatch**
+ - **wikimedia.org**
  - **Willow**
  - **WimTV**
  - **Wistia**

test/test_YoutubeDL.py

@@ -1053,6 +1053,7 @@ class TestYoutubeDL(unittest.TestCase):
                            for v in get_downloaded_info_dicts(params, entries)]
                 self.assertEqual(results, list(enumerate(zip(expected_ids, expected_ids))), f'Entries of {name} for {params}')
                 self.assertEqual(sorted(evaluated), expected_eval, f'Evaluation of {name} for {params}')
+
         test_selection({}, INDICES)
         test_selection({'playlistend': 20}, INDICES, True)
         test_selection({'playlistend': 2}, INDICES[:2])

test/test_aes.py

@@ -24,6 +24,8 @@ from yt_dlp.aes import (
     aes_encrypt,
     aes_gcm_decrypt_and_verify,
     aes_gcm_decrypt_and_verify_bytes,
+    key_expansion,
+    pad_block,
 )
 from yt_dlp.dependencies import Cryptodome_AES
 from yt_dlp.utils import bytes_to_intlist, intlist_to_bytes
@@ -112,6 +114,41 @@ class TestAES(unittest.TestCase):
         decrypted = intlist_to_bytes(aes_ecb_decrypt(data, self.key, self.iv))
         self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
 
+    def test_key_expansion(self):
+        key = '4f6bdaa39e2f8cb07f5e722d9edef314'
+
+        self.assertEqual(key_expansion(bytes_to_intlist(bytearray.fromhex(key))), [
+            0x4F, 0x6B, 0xDA, 0xA3, 0x9E, 0x2F, 0x8C, 0xB0, 0x7F, 0x5E, 0x72, 0x2D, 0x9E, 0xDE, 0xF3, 0x14,
+            0x53, 0x66, 0x20, 0xA8, 0xCD, 0x49, 0xAC, 0x18, 0xB2, 0x17, 0xDE, 0x35, 0x2C, 0xC9, 0x2D, 0x21,
+            0x8C, 0xBE, 0xDD, 0xD9, 0x41, 0xF7, 0x71, 0xC1, 0xF3, 0xE0, 0xAF, 0xF4, 0xDF, 0x29, 0x82, 0xD5,
+            0x2D, 0xAD, 0xDE, 0x47, 0x6C, 0x5A, 0xAF, 0x86, 0x9F, 0xBA, 0x00, 0x72, 0x40, 0x93, 0x82, 0xA7,
+            0xF9, 0xBE, 0x82, 0x4E, 0x95, 0xE4, 0x2D, 0xC8, 0x0A, 0x5E, 0x2D, 0xBA, 0x4A, 0xCD, 0xAF, 0x1D,
+            0x54, 0xC7, 0x26, 0x98, 0xC1, 0x23, 0x0B, 0x50, 0xCB, 0x7D, 0x26, 0xEA, 0x81, 0xB0, 0x89, 0xF7,
+            0x93, 0x60, 0x4E, 0x94, 0x52, 0x43, 0x45, 0xC4, 0x99, 0x3E, 0x63, 0x2E, 0x18, 0x8E, 0xEA, 0xD9,
+            0xCA, 0xE7, 0x7B, 0x39, 0x98, 0xA4, 0x3E, 0xFD, 0x01, 0x9A, 0x5D, 0xD3, 0x19, 0x14, 0xB7, 0x0A,
+            0xB0, 0x4E, 0x1C, 0xED, 0x28, 0xEA, 0x22, 0x10, 0x29, 0x70, 0x7F, 0xC3, 0x30, 0x64, 0xC8, 0xC9,
+            0xE8, 0xA6, 0xC1, 0xE9, 0xC0, 0x4C, 0xE3, 0xF9, 0xE9, 0x3C, 0x9C, 0x3A, 0xD9, 0x58, 0x54, 0xF3,
+            0xB4, 0x86, 0xCC, 0xDC, 0x74, 0xCA, 0x2F, 0x25, 0x9D, 0xF6, 0xB3, 0x1F, 0x44, 0xAE, 0xE7, 0xEC])
+
+    def test_pad_block(self):
+        block = [0x21, 0xA0, 0x43, 0xFF]
+
+        self.assertEqual(pad_block(block, 'pkcs7'),
+                         block + [0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C])
+
+        self.assertEqual(pad_block(block, 'iso7816'),
+                         block + [0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])
+
+        self.assertEqual(pad_block(block, 'whitespace'),
+                         block + [0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20])
+
+        self.assertEqual(pad_block(block, 'zero'),
+                         block + [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])
+
+        block = list(range(16))
+        for mode in ('pkcs7', 'iso7816', 'whitespace', 'zero'):
+            self.assertEqual(pad_block(block, mode), block, mode)
+
 
 if __name__ == '__main__':
     unittest.main()

test/test_utils.py

@@ -39,6 +39,7 @@ from yt_dlp.utils import (
     datetime_from_str,
     detect_exe_version,
     determine_ext,
+    determine_file_encoding,
     dfxp2srt,
     dict_get,
     encode_base_n,
@@ -895,7 +896,7 @@ class TestUtil(unittest.TestCase):
             'dynamic_range': 'HDR10',
         })
         self.assertEqual(parse_codecs('av01.0.12M.10.0.110.09.16.09.0'), {
-            'vcodec': 'av01.0.12M.10',
+            'vcodec': 'av01.0.12M.10.0.110.09.16.09.0',
             'acodec': 'none',
             'dynamic_range': 'HDR10',
         })
@@ -1822,6 +1823,25 @@ Line 1
             with contextlib.suppress(OSError):
                 os.remove(FILE)
 
+    def test_determine_file_encoding(self):
+        self.assertEqual(determine_file_encoding(b''), (None, 0))
+        self.assertEqual(determine_file_encoding(b'--verbose -x --audio-format mkv\n'), (None, 0))
+
+        self.assertEqual(determine_file_encoding(b'\xef\xbb\xbf'), ('utf-8', 3))
+        self.assertEqual(determine_file_encoding(b'\x00\x00\xfe\xff'), ('utf-32-be', 4))
+        self.assertEqual(determine_file_encoding(b'\xff\xfe'), ('utf-16-le', 2))
+
+        self.assertEqual(determine_file_encoding(b'\xff\xfe# coding: utf-8\n--verbose'), ('utf-16-le', 2))
+
+        self.assertEqual(determine_file_encoding(b'# coding: utf-8\n--verbose'), ('utf-8', 0))
+        self.assertEqual(determine_file_encoding(b'# coding: someencodinghere-12345\n--verbose'), ('someencodinghere-12345', 0))
+
+        self.assertEqual(determine_file_encoding(b'#coding:utf-8\n--verbose'), ('utf-8', 0))
+        self.assertEqual(determine_file_encoding(b'#  coding:   utf-8   \r\n--verbose'), ('utf-8', 0))
+
+        self.assertEqual(determine_file_encoding('# coding: utf-32-be'.encode('utf-32-be')), ('utf-32-be', 0))
+        self.assertEqual(determine_file_encoding('# coding: utf-16-le'.encode('utf-16-le')), ('utf-16-le', 0))
+
 
 if __name__ == '__main__':
     unittest.main()

yt_dlp/YoutubeDL.py

@@ -24,7 +24,6 @@ import urllib.request
 from string import ascii_letters
 
 from .cache import Cache
-from .compat import HAS_LEGACY as compat_has_legacy
 from .compat import compat_os_name, compat_shlex_quote
 from .cookies import load_cookies
 from .downloader import FFmpegFD, get_suitable_downloader, shorten_protocol_name
@@ -43,9 +42,11 @@ from .postprocessor import (
     FFmpegFixupTimestampPP,
     FFmpegMergerPP,
     FFmpegPostProcessor,
+    FFmpegVideoConvertorPP,
     MoveFilesAfterDownloadPP,
     get_postprocessor,
 )
+from .postprocessor.ffmpeg import resolve_mapping as resolve_recode_mapping
 from .update import detect_variant
 from .utils import (
     DEFAULT_OUTTMPL,
@@ -84,12 +85,14 @@ from .utils import (
     YoutubeDLRedirectHandler,
     age_restricted,
     args_to_str,
+    bug_reports_message,
     date_from_str,
     determine_ext,
     determine_protocol,
     encode_compat_str,
     encodeFilename,
     error_to_compat_str,
+    escapeHTML,
     expand_path,
     filter_dict,
     float_or_none,
@@ -303,7 +306,7 @@ class YoutubeDL:
     client_certificate_password:  Password for client certificate private key, if encrypted.
                         If not provided and the key is encrypted, yt-dlp will ask interactively
     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
-                       At the moment, this is only supported by YouTube.
+                       (Only supported by some extractors)
     http_headers:      A dictionary of custom headers to be used for all requests
     proxy:             URL of the proxy server to use
     geo_verification_proxy:  URL of the proxy to use for IP address verification
@@ -315,9 +318,14 @@ class YoutubeDL:
     default_search:    Prepend this string if an input url is not valid.
                        'auto' for elaborate guessing
     encoding:          Use this encoding instead of the system-specified.
-    extract_flat:      Do not resolve URLs, return the immediate result.
-                       Pass in 'in_playlist' to only show this behavior for
-                       playlist items.
+    extract_flat:      Whether to resolve and process url_results further
+                       * False:     Always process (default)
+                       * True:      Never process
+                       * 'in_playlist': Do not process inside playlist/multi_video
+                       * 'discard': Always process, but don't return the result
+                                    from inside playlist/multi_video
+                       * 'discard_in_playlist': Same as "discard", but only for
+                                    playlists (not multi_video)
     wait_for_video:    If given, wait for scheduled streams to become available.
                        The value should be a tuple containing the range
                        (min_secs, max_secs) to wait between retries
@@ -421,19 +429,22 @@ class YoutubeDL:
     retry_sleep_functions: Dictionary of functions that takes the number of attempts
                        as argument and returns the time to sleep in seconds.
                        Allowed keys are 'http', 'fragment', 'file_access'
-    download_ranges:   A function that gets called for every video with the signature
-                       (info_dict, *, ydl) -> Iterable[Section].
-                       Only the returned sections will be downloaded. Each Section contains:
+    download_ranges:   A callback function that gets called for every video with
+                       the signature (info_dict, ydl) -> Iterable[Section].
+                       Only the returned sections will be downloaded.
+                       Each Section is a dict with the following keys:
                        * start_time: Start time of the section in seconds
                        * end_time: End time of the section in seconds
                        * title: Section title (Optional)
                        * index: Section number (Optional)
+    force_keyframes_at_cuts: Re-encode the video when downloading ranges to get precise cuts
+    noprogress:        Do not print the progress bar
 
     The following parameters are not used by YoutubeDL itself, they are used by
     the downloader (see yt_dlp/downloader/common.py):
     nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize,
     max_filesize, test, noresizebuffer, retries, file_access_retries, fragment_retries,
-    continuedl, noprogress, xattr_set_filesize, hls_use_mpegts, http_chunk_size,
+    continuedl, xattr_set_filesize, hls_use_mpegts, http_chunk_size,
     external_downloader_args, concurrent_fragment_downloads.
 
     The following options are used by the post processors:
@@ -577,8 +588,8 @@ class YoutubeDL:
         current_version = sys.version_info[:2]
         if current_version < MIN_RECOMMENDED:
             msg = ('Support for Python version %d.%d has been deprecated. '
-                   'See  https://github.com/yt-dlp/yt-dlp/issues/3764  for more details. '
-                   'You will recieve only one more update on this version')
+                   'See  https://github.com/yt-dlp/yt-dlp/issues/3764  for more details.'
+                   '\n                    You will no longer receive updates on this version')
             if current_version < MIN_SUPPORTED:
                 msg = 'Python version %d.%d is no longer supported'
             self.deprecation_warning(
@@ -611,8 +622,6 @@ class YoutubeDL:
             self.deprecation_warning(msg)
 
         self.params['compat_opts'] = set(self.params.get('compat_opts', ()))
-        if not compat_has_legacy:
-            self.params['compat_opts'].add('no-compat-legacy')
         if 'list-formats' in self.params['compat_opts']:
             self.params['listformats_table'] = False
 
@@ -1046,7 +1055,7 @@ class YoutubeDL:
     def validate_outtmpl(cls, outtmpl):
         ''' @return None or Exception object '''
         outtmpl = re.sub(
-            STR_FORMAT_RE_TMPL.format('[^)]*', '[ljqBUDS]'),
+            STR_FORMAT_RE_TMPL.format('[^)]*', '[ljhqBUDS]'),
             lambda mobj: f'{mobj.group(0)[:-1]}s',
             cls._outtmpl_expandpath(outtmpl))
         try:
@@ -1089,7 +1098,7 @@ class YoutubeDL:
         }
 
         TMPL_DICT = {}
-        EXTERNAL_FORMAT_RE = re.compile(STR_FORMAT_RE_TMPL.format('[^)]*', f'[{STR_FORMAT_TYPES}ljqBUDS]'))
+        EXTERNAL_FORMAT_RE = re.compile(STR_FORMAT_RE_TMPL.format('[^)]*', f'[{STR_FORMAT_TYPES}ljhqBUDS]'))
         MATH_FUNCTIONS = {
             '+': float.__add__,
             '-': float.__sub__,
@@ -1198,6 +1207,8 @@ class YoutubeDL:
                 value, fmt = delim.join(map(str, variadic(value, allowed_types=(str, bytes)))), str_fmt
             elif fmt[-1] == 'j':  # json
                 value, fmt = json.dumps(value, default=_dumpjson_default, indent=4 if '#' in flags else None), str_fmt
+            elif fmt[-1] == 'h':  # html
+                value, fmt = escapeHTML(value), str_fmt
             elif fmt[-1] == 'q':  # quoted
                 value = map(str, variadic(value) if '#' in flags else [value])
                 value, fmt = ' '.join(map(compat_shlex_quote, value)), str_fmt
@@ -1455,7 +1466,12 @@ class YoutubeDL:
 
         def progress(msg):
             nonlocal last_msg
-            self.to_screen(msg + ' ' * (len(last_msg) - len(msg)) + '\r', skip_eol=True)
+            full_msg = f'{msg}\n'
+            if not self.params.get('noprogress'):
+                full_msg = msg + ' ' * (len(last_msg) - len(msg)) + '\r'
+            elif last_msg:
+                return
+            self.to_screen(full_msg, skip_eol=True)
             last_msg = msg
 
         min_wait, max_wait = self.params.get('wait_for_video')
@@ -1489,6 +1505,7 @@ class YoutubeDL:
     def __extract_info(self, url, ie, download, extra_info, process):
         ie_result = ie.extract(url)
         if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)
+            self.report_warning(f'Extractor {ie.IE_NAME} returned nothing{bug_reports_message()}')
             return
         if isinstance(ie_result, list):
             # Backwards compatibility: old IE result format
@@ -1673,8 +1690,10 @@ class YoutubeDL:
 
     def __process_playlist(self, ie_result, download):
         """Process each entry in the playlist"""
+        assert ie_result['_type'] in ('playlist', 'multi_video')
+
         title = ie_result.get('title') or ie_result.get('id') or '<Untitled>'
-        self.to_screen(f'[download] Downloading playlist: {title}')
+        self.to_screen(f'[download] Downloading {ie_result["_type"]}: {title}')
 
         all_entries = PlaylistEntries(self, ie_result)
         entries = orderedSet(all_entries.get_requested_items(), lazy=True)
@@ -1718,6 +1737,12 @@ class YoutubeDL:
         self.to_screen(f'[{ie_result["extractor"]}] Playlist {title}: Downloading {n_entries} videos'
                        f'{format_field(ie_result, "playlist_count", " of %s")}')
 
+        keep_resolved_entries = self.params.get('extract_flat') != 'discard'
+        if self.params.get('extract_flat') == 'discard_in_playlist':
+            keep_resolved_entries = ie_result['_type'] != 'playlist'
+        if keep_resolved_entries:
+            self.write_debug('The information of all playlist entries will be held in memory')
+
         failures = 0
         max_failures = self.params.get('skip_playlist_after_errors') or float('inf')
         for i, (playlist_index, entry) in enumerate(entries):
@@ -1758,7 +1783,8 @@ class YoutubeDL:
                 self.report_error(
                     f'Skipping the remaining entries in playlist "{title}" since {failures} items failed extraction')
                 break
-            resolved_entries[i] = (playlist_index, entry_result)
+            if keep_resolved_entries:
+                resolved_entries[i] = (playlist_index, entry_result)
 
         # Update with processed data
         ie_result['requested_entries'], ie_result['entries'] = tuple(zip(*resolved_entries)) or ([], [])
@@ -2377,13 +2403,18 @@ class YoutubeDL:
             self.report_warning('"duration" field is negative, there is an error in extractor')
 
         chapters = info_dict.get('chapters') or []
+        if chapters and chapters[0].get('start_time'):
+            chapters.insert(0, {'start_time': 0})
+
         dummy_chapter = {'end_time': 0, 'start_time': info_dict.get('duration')}
-        for prev, current, next_ in zip(
-                (dummy_chapter, *chapters), chapters, (*chapters[1:], dummy_chapter)):
+        for idx, (prev, current, next_) in enumerate(zip(
+                (dummy_chapter, *chapters), chapters, (*chapters[1:], dummy_chapter)), 1):
             if current.get('start_time') is None:
                 current['start_time'] = prev.get('end_time')
             if not current.get('end_time'):
                 current['end_time'] = next_.get('start_time')
+            if not current.get('title'):
+                current['title'] = f'<Untitled Chapter {idx}>'
 
         if 'playlist' not in info_dict:
             # It isn't part of a playlist
@@ -3173,22 +3204,23 @@ class YoutubeDL:
                             self.report_warning(f'{vid}: {msg}. Install ffmpeg to fix this automatically')
 
                     stretched_ratio = info_dict.get('stretched_ratio')
-                    ffmpeg_fixup(
-                        stretched_ratio not in (1, None),
-                        f'Non-uniform pixel ratio {stretched_ratio}',
-                        FFmpegFixupStretchedPP)
-
-                    ffmpeg_fixup(
-                        (info_dict.get('requested_formats') is None
-                         and info_dict.get('container') == 'm4a_dash'
-                         and info_dict.get('ext') == 'm4a'),
-                        'writing DASH m4a. Only some players support this container',
-                        FFmpegFixupM4aPP)
+                    ffmpeg_fixup(stretched_ratio not in (1, None),
+                                 f'Non-uniform pixel ratio {stretched_ratio}',
+                                 FFmpegFixupStretchedPP)
 
                     downloader = get_suitable_downloader(info_dict, self.params) if 'protocol' in info_dict else None
                     downloader = downloader.FD_NAME if downloader else None
 
-                    if info_dict.get('requested_formats') is None:  # Not necessary if doing merger
+                    ext = info_dict.get('ext')
+                    postprocessed_by_ffmpeg = info_dict.get('requested_formats') or any((
+                        isinstance(pp, FFmpegVideoConvertorPP)
+                        and resolve_recode_mapping(ext, pp.mapping)[0] not in (ext, None)
+                    ) for pp in self._pps['post_process'])
+
+                    if not postprocessed_by_ffmpeg:
+                        ffmpeg_fixup(ext == 'm4a' and info_dict.get('container') == 'm4a_dash',
+                                     'writing DASH m4a. Only some players support this container',
+                                     FFmpegFixupM4aPP)
                         ffmpeg_fixup(downloader == 'hlsnative' and not self.params.get('hls_use_mpegts')
                                      or info_dict.get('is_live') and self.params.get('hls_use_mpegts') is None,
                                      'Possible MPEG-TS in MP4 container or malformed AAC timestamps',
@@ -3512,27 +3544,38 @@ class YoutubeDL:
                 ] for f in formats if f.get('preference') is None or f['preference'] >= -1000]
             return render_table(['format code', 'extension', 'resolution', 'note'], table, extra_gap=1)
 
+        def simplified_codec(f, field):
+            assert field in ('acodec', 'vcodec')
+            codec = f.get(field, 'unknown')
+            if not codec:
+                return 'unknown'
+            elif codec != 'none':
+                return '.'.join(codec.split('.')[:4])
+
+            if field == 'vcodec' and f.get('acodec') == 'none':
+                return 'images'
+            elif field == 'acodec' and f.get('vcodec') == 'none':
+                return ''
+            return self._format_out('audio only' if field == 'vcodec' else 'video only',
+                                    self.Styles.SUPPRESS)
+
         delim = self._format_out('\u2502', self.Styles.DELIM, '|', test_encoding=True)
         table = [
             [
                 self._format_out(format_field(f, 'format_id'), self.Styles.ID),
                 format_field(f, 'ext'),
                 format_field(f, func=self.format_resolution, ignore=('audio only', 'images')),
-                format_field(f, 'fps', '\t%d'),
+                format_field(f, 'fps', '\t%d', func=round),
                 format_field(f, 'dynamic_range', '%s', ignore=(None, 'SDR')).replace('HDR', ''),
                 delim,
                 format_field(f, 'filesize', ' \t%s', func=format_bytes) + format_field(f, 'filesize_approx', '~\t%s', func=format_bytes),
-                format_field(f, 'tbr', '\t%dk'),
+                format_field(f, 'tbr', '\t%dk', func=round),
                 shorten_protocol_name(f.get('protocol', '')),
                 delim,
-                format_field(f, 'vcodec', default='unknown').replace(
-                    'none', 'images' if f.get('acodec') == 'none'
-                            else self._format_out('audio only', self.Styles.SUPPRESS)),
-                format_field(f, 'vbr', '\t%dk'),
-                format_field(f, 'acodec', default='unknown').replace(
-                    'none', '' if f.get('vcodec') == 'none'
-                            else self._format_out('video only', self.Styles.SUPPRESS)),
-                format_field(f, 'abr', '\t%dk'),
+                simplified_codec(f, 'vcodec'),
+                format_field(f, 'vbr', '\t%dk', func=round),
+                simplified_codec(f, 'acodec'),
+                format_field(f, 'abr', '\t%dk', func=round),
                 format_field(f, 'asr', '\t%s', func=format_decimal_suffix),
                 join_nonempty(
                     self._format_out('UNSUPPORTED', 'light red') if f.get('ext') in ('f4f', 'f4m') else None,

yt_dlp/__init__.py

@@ -2,6 +2,7 @@ f'You are using an unsupported version of Python. Only Python versions 3.6 and a
 
 __license__ = 'Public Domain'
 
+import collections
 import getpass
 import itertools
 import optparse
@@ -516,7 +517,7 @@ def validate_options(opts):
         # Do not unnecessarily download audio
         opts.format = 'bestaudio/best'
 
-    if opts.getcomments and opts.writeinfojson is None:
+    if opts.getcomments and opts.writeinfojson is None and not opts.embed_infojson:
         # If JSON is not printed anywhere, but comments are requested, save it to file
         if not opts.dumpjson or opts.print_json or opts.dump_single_json:
             opts.writeinfojson = True
@@ -665,8 +666,11 @@ def get_postprocessors(opts):
         }
 
 
+ParsedOptions = collections.namedtuple('ParsedOptions', ('parser', 'options', 'urls', 'ydl_opts'))
+
+
 def parse_options(argv=None):
-    """ @returns (parser, opts, urls, ydl_opts) """
+    """@returns ParsedOptions(parser, opts, urls, ydl_opts)"""
     parser, opts, urls = parseOpts(argv)
     urls = get_urls(urls, opts.batchfile, opts.verbose)
 
@@ -684,13 +688,28 @@ def parse_options(argv=None):
         'getformat', 'getid', 'getthumbnail', 'gettitle', 'geturl'
     ))
 
+    playlist_pps = [pp for pp in postprocessors if pp.get('when') == 'playlist']
+    write_playlist_infojson = (opts.writeinfojson and not opts.clean_infojson
+                               and opts.allow_playlist_files and opts.outtmpl.get('pl_infojson') != '')
+    if not any((
+        opts.extract_flat,
+        opts.dump_single_json,
+        opts.forceprint.get('playlist'),
+        opts.print_to_file.get('playlist'),
+        write_playlist_infojson,
+    )):
+        if not playlist_pps:
+            opts.extract_flat = 'discard'
+        elif playlist_pps == [{'key': 'FFmpegConcat', 'only_multi_video': True, 'when': 'playlist'}]:
+            opts.extract_flat = 'discard_in_playlist'
+
     final_ext = (
         opts.recodevideo if opts.recodevideo in FFmpegVideoConvertorPP.SUPPORTED_EXTS
         else opts.remuxvideo if opts.remuxvideo in FFmpegVideoRemuxerPP.SUPPORTED_EXTS
         else opts.audioformat if (opts.extractaudio and opts.audioformat in FFmpegExtractAudioPP.SUPPORTED_EXTS)
         else None)
 
-    return parser, opts, urls, {
+    return ParsedOptions(parser, opts, urls, {
         'usenetrc': opts.usenetrc,
         'netrc_location': opts.netrc_location,
         'username': opts.username,
@@ -863,7 +882,7 @@ def parse_options(argv=None):
         '_warnings': warnings,
         '_deprecation_warnings': deprecation_warnings,
         'compat_opts': opts.compat_opts,
-    }
+    })
 
 
 def _real_main(argv=None):

yt_dlp/aes.py

@@ -24,6 +24,10 @@ else:
         return intlist_to_bytes(aes_gcm_decrypt_and_verify(*map(bytes_to_intlist, (data, key, tag, nonce))))
 
 
+def aes_cbc_encrypt_bytes(data, key, iv, **kwargs):
+    return intlist_to_bytes(aes_cbc_encrypt(*map(bytes_to_intlist, (data, key, iv)), **kwargs))
+
+
 def unpad_pkcs7(data):
     return data[:-compat_ord(data[-1])]
 
@@ -31,6 +35,33 @@ def unpad_pkcs7(data):
 BLOCK_SIZE_BYTES = 16
 
 
+def pad_block(block, padding_mode):
+    """
+    Pad a block with the given padding mode
+    @param {int[]} block        block to pad
+    @param padding_mode         padding mode
+    """
+    padding_size = BLOCK_SIZE_BYTES - len(block)
+
+    PADDING_BYTE = {
+        'pkcs7': padding_size,
+        'iso7816': 0x0,
+        'whitespace': 0x20,
+        'zero': 0x0,
+    }
+
+    if padding_size < 0:
+        raise ValueError('Block size exceeded')
+    elif padding_mode not in PADDING_BYTE:
+        raise NotImplementedError(f'Padding mode {padding_mode} is not implemented')
+
+    if padding_mode == 'iso7816' and padding_size:
+        block = block + [0x80]  # NB: += mutates list
+        padding_size -= 1
+
+    return block + [PADDING_BYTE[padding_mode]] * padding_size
+
+
 def aes_ecb_encrypt(data, key, iv=None):
     """
     Encrypt with aes in ECB mode
@@ -137,13 +168,14 @@ def aes_cbc_decrypt(data, key, iv):
     return decrypted_data
 
 
-def aes_cbc_encrypt(data, key, iv):
+def aes_cbc_encrypt(data, key, iv, *, padding_mode='pkcs7'):
     """
-    Encrypt with aes in CBC mode. Using PKCS#7 padding
+    Encrypt with aes in CBC mode
 
     @param {int[]} data        cleartext
     @param {int[]} key         16/24/32-Byte cipher key
     @param {int[]} iv          16-Byte IV
+    @param padding_mode        Padding mode to use
     @returns {int[]}           encrypted data
     """
     expanded_key = key_expansion(key)
@@ -153,8 +185,8 @@ def aes_cbc_encrypt(data, key, iv):
     previous_cipher_block = iv
     for i in range(block_count):
         block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
-        remaining_length = BLOCK_SIZE_BYTES - len(block)
-        block += [remaining_length] * remaining_length
+        block = pad_block(block, padding_mode)
+
         mixed_block = xor(block, previous_cipher_block)
 
         encrypted_block = aes_encrypt(mixed_block, expanded_key)
@@ -502,13 +534,22 @@ def ghash(subkey, data):
 
 
 __all__ = [
-    'aes_ctr_decrypt',
     'aes_cbc_decrypt',
     'aes_cbc_decrypt_bytes',
+    'aes_ctr_decrypt',
     'aes_decrypt_text',
-    'aes_encrypt',
+    'aes_decrypt',
+    'aes_ecb_decrypt',
     'aes_gcm_decrypt_and_verify',
     'aes_gcm_decrypt_and_verify_bytes',
+
+    'aes_cbc_encrypt',
+    'aes_cbc_encrypt_bytes',
+    'aes_ctr_encrypt',
+    'aes_ecb_encrypt',
+    'aes_encrypt',
+
     'key_expansion',
+    'pad_block',
     'unpad_pkcs7',
 ]

yt_dlp/compat/__init__.py

@@ -8,14 +8,8 @@ from ._deprecated import *  # noqa: F401, F403
 from .compat_utils import passthrough_module
 
 # XXX: Implement this the same way as other DeprecationWarnings without circular import
-try:
-    passthrough_module(__name__, '._legacy', callback=lambda attr: warnings.warn(
-        DeprecationWarning(f'{__name__}.{attr} is deprecated'), stacklevel=2))
-    HAS_LEGACY = True
-except ModuleNotFoundError:
-    # Keep working even without _legacy module
-    HAS_LEGACY = False
-del passthrough_module
+passthrough_module(__name__, '._legacy', callback=lambda attr: warnings.warn(
+    DeprecationWarning(f'{__name__}.{attr} is deprecated'), stacklevel=2))
 
 
 # HTMLParseError has been deprecated in Python 3.3 and removed in
@@ -76,3 +70,9 @@ if compat_os_name in ('nt', 'ce'):
         return userhome + path[i:]
 else:
     compat_expanduser = os.path.expanduser
+
+
+# NB: Add modules that are imported dynamically here so that PyInstaller can find them
+# See https://github.com/pyinstaller/pyinstaller-hooks-contrib/issues/438
+if False:
+    from . import _legacy  # noqa: F401

yt_dlp/downloader/common.py

@@ -450,8 +450,7 @@ class FileDownloader:
         raise NotImplementedError('This method must be implemented by subclasses')
 
     def _hook_progress(self, status, info_dict):
-        if not self._progress_hooks:
-            return
+        # Ideally we want to make a copy of the dict, but that is too slow
         status['info_dict'] = info_dict
         # youtube-dl passes the same status object to all the hooks.
         # Some third party scripts seems to be relying on this.

yt_dlp/downloader/http.py

@@ -206,6 +206,12 @@ class HttpFD(FileDownloader):
             except RESPONSE_READ_EXCEPTIONS as err:
                 raise RetryDownload(err)
 
+        def close_stream():
+            if ctx.stream is not None:
+                if not ctx.tmpfilename == '-':
+                    ctx.stream.close()
+                ctx.stream = None
+
         def download():
             data_len = ctx.data.info().get('Content-length', None)
 
@@ -239,12 +245,9 @@ class HttpFD(FileDownloader):
             before = start  # start measuring
 
             def retry(e):
-                to_stdout = ctx.tmpfilename == '-'
-                if ctx.stream is not None:
-                    if not to_stdout:
-                        ctx.stream.close()
-                    ctx.stream = None
-                ctx.resume_len = byte_counter if to_stdout else os.path.getsize(encodeFilename(ctx.tmpfilename))
+                close_stream()
+                ctx.resume_len = (byte_counter if ctx.tmpfilename == '-'
+                                  else os.path.getsize(encodeFilename(ctx.tmpfilename)))
                 raise RetryDownload(e)
 
             while True:
@@ -382,6 +385,9 @@ class HttpFD(FileDownloader):
                 continue
             except SucceedDownload:
                 return True
+            except:  # noqa: E722
+                close_stream()
+                raise
 
         self.report_error('giving up after %s retries' % retries)
         return False

yt_dlp/extractor/_extractors.py

@@ -22,6 +22,7 @@ from .acast import (
     ACastIE,
     ACastChannelIE,
 )
+from .acfun import AcFunVideoIE, AcFunBangumiIE
 from .adn import ADNIE
 from .adobeconnect import AdobeConnectIE
 from .adobetv import (
@@ -104,6 +105,10 @@ from .atttechchannel import ATTTechChannelIE
 from .atvat import ATVAtIE
 from .audimedia import AudiMediaIE
 from .audioboom import AudioBoomIE
+from .audiodraft import (
+    AudiodraftCustomIE,
+    AudiodraftGenericIE,
+)
 from .audiomack import AudiomackIE, AudiomackAlbumIE
 from .audius import (
     AudiusIE,
@@ -258,6 +263,7 @@ from .ccc import (
 from .ccma import CCMAIE
 from .cctv import CCTVIE
 from .cda import CDAIE
+from .cellebrite import CellebriteIE
 from .ceskatelevize import CeskaTelevizeIE
 from .cgtn import CGTNIE
 from .channel9 import Channel9IE
@@ -376,6 +382,7 @@ from .deezer import (
     DeezerAlbumIE,
 )
 from .democracynow import DemocracynowIE
+from .detik import Detik20IE
 from .dfb import DFBIE
 from .dhm import DHMIE
 from .digg import DiggIE
@@ -660,6 +667,7 @@ from .hungama import (
     HungamaAlbumPlaylistIE,
 )
 from .hypem import HypemIE
+from .hytale import HytaleIE
 from .icareus import IcareusIE
 from .ichinanalive import (
     IchinanaLiveIE,
@@ -830,6 +838,7 @@ from .linkedin import (
     LinkedInLearningCourseIE,
 )
 from .linuxacademy import LinuxAcademyIE
+from .liputan6 import Liputan6IE
 from .litv import LiTVIE
 from .livejournal import LiveJournalIE
 from .livestream import (
@@ -943,6 +952,7 @@ from .mlb import (
 )
 from .mlssoccer import MLSSoccerIE
 from .mnet import MnetIE
+from .mocha import MochaVideoIE
 from .moevideo import MoeVideoIE
 from .mofosex import (
     MofosexIE,
@@ -1434,7 +1444,13 @@ from .rottentomatoes import RottenTomatoesIE
 from .rozhlas import RozhlasIE
 from .rtbf import RTBFIE
 from .rte import RteIE, RteRadioIE
-from .rtlnl import RtlNlIE
+from .rtlnl import (
+    RtlNlIE,
+    RTLLuTeleVODIE,
+    RTLLuArticleIE,
+    RTLLuLiveIE,
+    RTLLuRadioIE,
+)
 from .rtl2 import (
     RTL2IE,
     RTL2YouIE,
@@ -1458,6 +1474,7 @@ from .rtve import (
 )
 from .rtvnh import RTVNHIE
 from .rtvs import RTVSIE
+from .rtvslo import RTVSLOIE
 from .ruhd import RUHDIE
 from .rule34video import Rule34VideoIE
 from .rumble import (
@@ -1608,6 +1625,7 @@ from .spike import (
     BellatorIE,
     ParamountNetworkIE,
 )
+from .startrek import StarTrekIE
 from .stitcher import (
     StitcherIE,
     StitcherShowIE,
@@ -1665,6 +1683,7 @@ from .svt import (
     SVTSeriesIE,
 )
 from .swrmediathek import SWRMediathekIE
+from .syvdk import SYVDKIE
 from .syfy import SyfyIE
 from .sztvhu import SztvHuIE
 from .tagesschau import TagesschauIE
@@ -1711,6 +1730,7 @@ from .tenplay import TenPlayIE
 from .testurl import TestURLIE
 from .tf1 import TF1IE
 from .tfo import TFOIE
+from .theholetv import TheHoleTvIE
 from .theintercept import TheInterceptIE
 from .theplatform import (
     ThePlatformIE,
@@ -1775,6 +1795,7 @@ from .trueid import TrueIDIE
 from .trunews import TruNewsIE
 from .trutv import TruTVIE
 from .tube8 import Tube8IE
+from .tubetugraz import TubeTuGrazIE, TubeTuGrazSeriesIE
 from .tubitv import (
     TubiTvIE,
     TubiTvShowIE,
@@ -1823,6 +1844,7 @@ from .tvc import (
 )
 from .tver import TVerIE
 from .tvigle import TvigleIE
+from .tviplayer import TVIPlayerIE
 from .tvland import TVLandIE
 from .tvn24 import TVN24IE
 from .tvnet import TVNetIE
@@ -2066,6 +2088,8 @@ from .weibo import (
     WeiboMobileIE
 )
 from .weiqitv import WeiqiTVIE
+from .wetv import WeTvEpisodeIE, WeTvSeriesIE
+from .wikimedia import WikimediaIE
 from .willow import WillowIE
 from .wimtv import WimTVIE
 from .whowatch import WhoWatchIE

yt_dlp/extractor/acfun.py

@@ -0,0 +1,200 @@
+from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    format_field,
+    int_or_none,
+    traverse_obj,
+    parse_codecs,
+    parse_qs,
+)
+
+
+class AcFunVideoBaseIE(InfoExtractor):
+    def _extract_metadata(self, video_id, video_info):
+        playjson = self._parse_json(video_info['ksPlayJson'], video_id)
+
+        formats, subtitles = [], {}
+        for video in traverse_obj(playjson, ('adaptationSet', 0, 'representation')):
+            fmts, subs = self._extract_m3u8_formats_and_subtitles(video['url'], video_id, 'mp4', fatal=False)
+            formats.extend(fmts)
+            self._merge_subtitles(subs, target=subtitles)
+            for f in fmts:
+                f.update({
+                    'fps': float_or_none(video.get('frameRate')),
+                    'width': int_or_none(video.get('width')),
+                    'height': int_or_none(video.get('height')),
+                    'tbr': float_or_none(video.get('avgBitrate')),
+                    **parse_codecs(video.get('codecs', ''))
+                })
+
+        self._sort_formats(formats)
+        return {
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            'duration': float_or_none(video_info.get('durationMillis'), 1000),
+            'timestamp': int_or_none(video_info.get('uploadTime'), 1000),
+            'http_headers': {'Referer': 'https://www.acfun.cn/'},
+        }
+
+
+class AcFunVideoIE(AcFunVideoBaseIE):
+    _VALID_URL = r'https?://www\.acfun\.cn/v/ac(?P<id>[_\d]+)'
+
+    _TESTS = [{
+        'url': 'https://www.acfun.cn/v/ac35457073',
+        'info_dict': {
+            'id': '35457073',
+            'ext': 'mp4',
+            'duration': 174.208,
+            'timestamp': 1656403967,
+            'title': '1 8 岁 现 状',
+            'description': '“赶紧回去！班主任查班了！”',
+            'uploader': '锤子game',
+            'uploader_id': '51246077',
+            'thumbnail': r're:^https?://.*\.(jpg|jpeg)',
+            'upload_date': '20220628',
+            'like_count': int,
+            'view_count': int,
+            'comment_count': int,
+            'tags': list,
+        },
+    }, {
+        # example for len(video_list) > 1
+        'url': 'https://www.acfun.cn/v/ac35468952_2',
+        'info_dict': {
+            'id': '35468952_2',
+            'ext': 'mp4',
+            'title': '【动画剧集】Rocket & Groot Season 1（2022）/火箭浣熊与格鲁特第1季 P02 S01E02 十拿九穩',
+            'duration': 90.459,
+            'uploader': '比令',
+            'uploader_id': '37259967',
+            'upload_date': '20220629',
+            'timestamp': 1656479962,
+            'tags': list,
+            'like_count': int,
+            'view_count': int,
+            'comment_count': int,
+            'thumbnail': r're:^https?://.*\.(jpg|jpeg)',
+            'description': 'md5:67583aaf3a0f933bd606bc8a2d3ebb17',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        json_all = self._search_json(r'window.videoInfo\s*=\s*', webpage, 'videoInfo', video_id)
+
+        title = json_all.get('title')
+        video_list = json_all.get('videoList') or []
+        video_internal_id = traverse_obj(json_all, ('currentVideoInfo', 'id'))
+        if video_internal_id and len(video_list) > 1:
+            part_idx, part_video_info = next(
+                (idx + 1, v) for (idx, v) in enumerate(video_list)
+                if v['id'] == video_internal_id)
+            title = f'{title} P{part_idx:02d} {part_video_info["title"]}'
+
+        return {
+            **self._extract_metadata(video_id, json_all['currentVideoInfo']),
+            'title': title,
+            'thumbnail': json_all.get('coverUrl'),
+            'description': json_all.get('description'),
+            'uploader': traverse_obj(json_all, ('user', 'name')),
+            'uploader_id': traverse_obj(json_all, ('user', 'href')),
+            'tags': traverse_obj(json_all, ('tagList', ..., 'name')),
+            'view_count': int_or_none(json_all.get('viewCount')),
+            'like_count': int_or_none(json_all.get('likeCountShow')),
+            'comment_count': int_or_none(json_all.get('commentCountShow')),
+        }
+
+
+class AcFunBangumiIE(AcFunVideoBaseIE):
+    _VALID_URL = r'https?://www\.acfun\.cn/bangumi/(?P<id>aa[_\d]+)'
+
+    _TESTS = [{
+        'url': 'https://www.acfun.cn/bangumi/aa6002917_36188_1745457?ac=2',
+        'info_dict': {
+            'id': 'aa6002917_36188_1745457__2',
+            'ext': 'mp4',
+            'title': '【7月】租借女友 水原千鹤角色曲『DATE』特别PV',
+            'upload_date': '20200916',
+            'timestamp': 1600243813,
+            'duration': 92.091,
+        },
+    }, {
+        'url': 'https://www.acfun.cn/bangumi/aa5023171_36188_1750645',
+        'info_dict': {
+            'id': 'aa5023171_36188_1750645',
+            'ext': 'mp4',
+            'title': '红孩儿之趴趴蛙寻石记 第5话 ',
+            'duration': 760.0,
+            'season': '红孩儿之趴趴蛙寻石记',
+            'season_id': 5023171,
+            'season_number': 1,  # series has only 1 season
+            'episode': 'Episode 5',
+            'episode_number': 5,
+            'upload_date': '20181223',
+            'timestamp': 1545552185,
+            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)',
+            'comment_count': int,
+        },
+    }, {
+        'url': 'https://www.acfun.cn/bangumi/aa6065485_36188_1885061',
+        'info_dict': {
+            'id': 'aa6065485_36188_1885061',
+            'ext': 'mp4',
+            'title': '叽歪老表（第二季） 第5话 坚不可摧',
+            'season': '叽歪老表（第二季）',
+            'season_number': 2,
+            'season_id': 6065485,
+            'episode': '坚不可摧',
+            'episode_number': 5,
+            'upload_date': '20220324',
+            'timestamp': 1648082786,
+            'duration': 105.002,
+            'thumbnail': r're:^https?://.*\.(jpg|jpeg|png)',
+            'comment_count': int,
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        ac_idx = parse_qs(url).get('ac', [None])[-1]
+        video_id = f'{video_id}{format_field(ac_idx, template="__%s")}'
+
+        webpage = self._download_webpage(url, video_id)
+        json_bangumi_data = self._search_json(r'window.bangumiData\s*=\s*', webpage, 'bangumiData', video_id)
+
+        if ac_idx:
+            video_info = json_bangumi_data['hlVideoInfo']
+            return {
+                **self._extract_metadata(video_id, video_info),
+                'title': video_info.get('title'),
+            }
+
+        video_info = json_bangumi_data['currentVideoInfo']
+
+        season_id = json_bangumi_data.get('bangumiId')
+        season_number = season_id and next((
+            idx for idx, v in enumerate(json_bangumi_data.get('relatedBangumis') or [], 1)
+            if v.get('id') == season_id), 1)
+
+        json_bangumi_list = self._search_json(
+            r'window\.bangumiList\s*=\s*', webpage, 'bangumiList', video_id, fatal=False)
+        video_internal_id = int_or_none(traverse_obj(json_bangumi_data, ('currentVideoInfo', 'id')))
+        episode_number = video_internal_id and next((
+            idx for idx, v in enumerate(json_bangumi_list.get('items') or [], 1)
+            if v.get('videoId') == video_internal_id), None)
+
+        return {
+            **self._extract_metadata(video_id, video_info),
+            'title': json_bangumi_data.get('showTitle'),
+            'thumbnail': json_bangumi_data.get('image'),
+            'season': json_bangumi_data.get('bangumiTitle'),
+            'season_id': season_id,
+            'season_number': season_number,
+            'episode': json_bangumi_data.get('title'),
+            'episode_number': episode_number,
+            'comment_count': int_or_none(json_bangumi_data.get('commentCount')),
+        }

yt_dlp/extractor/audiodraft.py

@@ -0,0 +1,93 @@
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class AudiodraftBaseIE(InfoExtractor):
+    def _audiodraft_extract_from_id(self, player_entry_id):
+        data_json = self._download_json(
+            'https://www.audiodraft.com/scripts/general/player/getPlayerInfoNew.php', player_entry_id,
+            headers={
+                'Content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
+                'X-Requested-With': 'XMLHttpRequest',
+            }, data=f'id={player_entry_id}'.encode('utf-8'))
+
+        return {
+            'id': str(data_json['entry_id']),
+            'title': data_json.get('entry_title'),
+            'url': data_json['path'],
+            'vcodec': 'none',
+            'ext': 'mp3',
+            'uploader': data_json.get('designer_name'),
+            'uploader_id': data_json.get('designer_id'),
+            'webpage_url': data_json.get('entry_url'),
+            'like_count': int_or_none(data_json.get('entry_likes')),
+            'average_rating': int_or_none(data_json.get('entry_rating')),
+        }
+
+
+class AudiodraftCustomIE(AudiodraftBaseIE):
+    IE_NAME = 'Audiodraft:custom'
+    _VALID_URL = r'https?://(?:[-\w]+)\.audiodraft\.com/entry/(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'http://nokiatune.audiodraft.com/entry/5874',
+        'info_dict': {
+            'id': '9485',
+            'ext': 'mp3',
+            'title': 'Hula Hula Calls',
+            'uploader': 'unclemaki',
+            'uploader_id': '13512',
+            'average_rating': 5,
+            'like_count': int,
+        },
+    }, {
+        'url': 'http://vikinggrace.audiodraft.com/entry/501',
+        'info_dict': {
+            'id': '22241',
+            'ext': 'mp3',
+            'title': 'MVG Happy',
+            'uploader': 'frog',
+            'uploader_id': '19142',
+            'average_rating': 5,
+            'like_count': int,
+        },
+    }, {
+        'url': 'http://timferriss.audiodraft.com/entry/765',
+        'info_dict': {
+            'id': '19710',
+            'ext': 'mp3',
+            'title': 'ferris03',
+            'uploader': 'malex',
+            'uploader_id': '17335',
+            'average_rating': 5,
+            'like_count': int,
+        },
+    }]
+
+    def _real_extract(self, url):
+        id = self._match_id(url)
+        webpage = self._download_webpage(url, id)
+        player_entry_id = self._search_regex(r'playAudio\(\'(player_entry_\d+)\'\);', webpage, id, 'play entry id')
+        return self._audiodraft_extract_from_id(player_entry_id)
+
+
+class AudiodraftGenericIE(AudiodraftBaseIE):
+    IE_NAME = 'Audiodraft:generic'
+    _VALID_URL = r'https?://www\.audiodraft\.com/contests/[^/#]+#entries&eid=(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'https://www.audiodraft.com/contests/570-Score-A-Video-Surprise-Us#entries&eid=30138',
+        'info_dict': {
+            'id': '30138',
+            'ext': 'mp3',
+            'title': 'DROP in sound_V2',
+            'uploader': 'TiagoSilva',
+            'uploader_id': '19452',
+            'average_rating': 4,
+            'like_count': int,
+        },
+    }]
+
+    def _real_extract(self, url):
+        id = self._match_id(url)
+        return self._audiodraft_extract_from_id(f'player_entry_{id}')

yt_dlp/extractor/bigo.py

@@ -28,7 +28,7 @@ class BigoIE(InfoExtractor):
         user_id = self._match_id(url)
 
         info_raw = self._download_json(
-            'https://bigo.tv/studio/getInternalStudioInfo',
+            'https://ta.bigo.tv/official_website/studio/getInternalStudioInfo',
             user_id, data=urlencode_postdata({'siteId': user_id}))
 
         if not isinstance(info_raw, dict):
@@ -41,14 +41,14 @@ class BigoIE(InfoExtractor):
         if not info.get('alive'):
             raise ExtractorError('This user is offline.', expected=True)
 
+        formats, subs = self._extract_m3u8_formats_and_subtitles(
+            info.get('hls_src'), user_id, 'mp4', 'm3u8')
+
         return {
             'id': info.get('roomId') or user_id,
             'title': info.get('roomTopic') or info.get('nick_name') or user_id,
-            'formats': [{
-                'url': info.get('hls_src'),
-                'ext': 'mp4',
-                'protocol': 'm3u8',
-            }],
+            'formats': formats,
+            'subtitles': subs,
             'thumbnail': info.get('snapshot'),
             'uploader': info.get('nick_name'),
             'uploader_id': user_id,

yt_dlp/extractor/bilibili.py

@@ -795,12 +795,14 @@ class BiliIntlBaseIE(InfoExtractor):
 
     def _get_subtitles(self, *, ep_id=None, aid=None):
         sub_json = self._call_api(
-            '/web/v2/subtitle', ep_id or aid, note='Downloading subtitles list',
-            errnote='Unable to download subtitles list', query=filter_dict({
+            '/web/v2/subtitle', ep_id or aid, fatal=False,
+            note='Downloading subtitles list', errnote='Unable to download subtitles list',
+            query=filter_dict({
                 'platform': 'web',
+                's_locale': 'en_US',
                 'episode_id': ep_id,
                 'aid': aid,
-            }))
+            })) or {}
         subtitles = {}
         for sub in sub_json.get('subtitles') or []:
             sub_url = sub.get('url')

yt_dlp/extractor/cellebrite.py

@@ -0,0 +1,64 @@
+from .common import InfoExtractor
+from ..utils import traverse_obj
+
+
+class CellebriteIE(InfoExtractor):
+    _VALID_URL = r'https?://cellebrite\.com/(?:\w+)?/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://cellebrite.com/en/collect-data-from-android-devices-with-cellebrite-ufed/',
+        'info_dict': {
+            'id': '16025876',
+            'ext': 'mp4',
+            'description': 'md5:174571cb97083fd1d457d75c684f4e2b',
+            'thumbnail': 'https://cellebrite.com/wp-content/uploads/2021/05/Chat-Capture-1024x559.png',
+            'title': 'Ask the Expert: Chat Capture - Collect Data from Android Devices in Cellebrite UFED',
+            'duration': 455,
+            'tags': [],
+        }
+    }, {
+        'url': 'https://cellebrite.com/en/how-to-lawfully-collect-the-maximum-amount-of-data-from-android-devices/',
+        'info_dict': {
+            'id': '29018255',
+            'ext': 'mp4',
+            'duration': 134,
+            'tags': [],
+            'description': 'md5:e9a3d124c7287b0b07bad2547061cacf',
+            'thumbnail': 'https://cellebrite.com/wp-content/uploads/2022/07/How-to-Lawfully-Collect-the-Maximum-Amount-of-Data-From-Android-Devices.png',
+            'title': 'Android Extractions Explained',
+        }
+    }]
+
+    def _get_formats_and_subtitles(self, json_data, display_id):
+        formats = [{'url': url} for url in traverse_obj(json_data, ('mp4', ..., 'url')) or []]
+        subtitles = {}
+
+        for url in traverse_obj(json_data, ('hls', ..., 'url')) or []:
+            fmt, sub = self._extract_m3u8_formats_and_subtitles(
+                url, display_id, ext='mp4', headers={'Referer': 'https://play.vidyard.com/'})
+            formats.extend(fmt)
+            self._merge_subtitles(sub, target=subtitles)
+
+        return formats, subtitles
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        player_uuid = self._search_regex(
+            r'<img\s[^>]*\bdata-uuid\s*=\s*"([^"\?]+)', webpage, 'player UUID')
+        json_data = self._download_json(
+            f'https://play.vidyard.com/player/{player_uuid}.json', display_id)['payload']['chapters'][0]
+
+        formats, subtitles = self._get_formats_and_subtitles(json_data['sources'], display_id)
+        self._sort_formats(formats)
+        return {
+            'id': str(json_data['videoId']),
+            'title': json_data.get('name') or self._og_search_title(webpage),
+            'formats': formats,
+            'subtitles': subtitles,
+            'description': json_data.get('description') or self._og_search_description(webpage),
+            'duration': json_data.get('seconds'),
+            'tags': json_data.get('tags'),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'http_headers': {'Referer': 'https://play.vidyard.com/'},
+        }

yt_dlp/extractor/common.py

@@ -383,6 +383,10 @@ class InfoExtractor:
     section_start:  Start time of the section in seconds
     section_end:    End time of the section in seconds
 
+    The following fields should only be set for storyboards:
+    rows:           Number of rows in each storyboard fragment, as an integer
+    columns:        Number of columns in each storyboard fragment, as an integer
+
     Unless mentioned otherwise, the fields should be Unicode strings.
 
     Unless mentioned otherwise, None is equivalent to absence of information.
@@ -925,39 +929,37 @@ class InfoExtractor:
 
         return content
 
-    def _parse_xml(self, xml_string, video_id, transform_source=None, fatal=True):
+    def __print_error(self, errnote, fatal, video_id, err):
+        if fatal:
+            raise ExtractorError(f'{video_id}: {errnote}', cause=err)
+        elif errnote:
+            self.report_warning(f'{video_id}: {errnote}: {err}')
+
+    def _parse_xml(self, xml_string, video_id, transform_source=None, fatal=True, errnote=None):
         if transform_source:
             xml_string = transform_source(xml_string)
         try:
             return compat_etree_fromstring(xml_string.encode('utf-8'))
         except xml.etree.ElementTree.ParseError as ve:
-            errmsg = '%s: Failed to parse XML ' % video_id
-            if fatal:
-                raise ExtractorError(errmsg, cause=ve)
-            else:
-                self.report_warning(errmsg + str(ve))
+            self.__print_error('Failed to parse XML' if errnote is None else errnote, fatal, video_id, ve)
 
-    def _parse_json(self, json_string, video_id, transform_source=None, fatal=True, **parser_kwargs):
+    def _parse_json(self, json_string, video_id, transform_source=None, fatal=True, errnote=None, **parser_kwargs):
         try:
             return json.loads(
                 json_string, cls=LenientJSONDecoder, strict=False, transform_source=transform_source, **parser_kwargs)
         except ValueError as ve:
-            errmsg = f'{video_id}: Failed to parse JSON'
-            if fatal:
-                raise ExtractorError(errmsg, cause=ve)
-            else:
-                self.report_warning(f'{errmsg}: {ve}')
+            self.__print_error('Failed to parse JSON' if errnote is None else errnote, fatal, video_id, ve)
 
-    def _parse_socket_response_as_json(self, data, video_id, transform_source=None, fatal=True):
-        return self._parse_json(
-            data[data.find('{'):data.rfind('}') + 1],
-            video_id, transform_source, fatal)
+    def _parse_socket_response_as_json(self, data, *args, **kwargs):
+        return self._parse_json(data[data.find('{'):data.rfind('}') + 1], *args, **kwargs)
 
     def __create_download_methods(name, parser, note, errnote, return_value):
 
-        def parse(ie, content, *args, **kwargs):
+        def parse(ie, content, *args, errnote=errnote, **kwargs):
             if parser is None:
                 return content
+            if errnote is False:
+                kwargs['errnote'] = errnote
             # parser is fetched by name so subclasses can override it
             return getattr(ie, parser)(content, *args, **kwargs)
 
@@ -969,7 +971,7 @@ class InfoExtractor:
             if res is False:
                 return res
             content, urlh = res
-            return parse(self, content, video_id, transform_source=transform_source, fatal=fatal), urlh
+            return parse(self, content, video_id, transform_source=transform_source, fatal=fatal, errnote=errnote), urlh
 
         def download_content(self, url_or_request, video_id, note=note, errnote=errnote, transform_source=None,
                              fatal=True, encoding=None, data=None, headers={}, query={}, expected_status=None):
@@ -984,7 +986,7 @@ class InfoExtractor:
                     self.report_warning(f'Unable to load request from disk: {e}')
                 else:
                     content = self.__decode_webpage(webpage_bytes, encoding, url_or_request.headers)
-                    return parse(self, content, video_id, transform_source, fatal)
+                    return parse(self, content, video_id, transform_source=transform_source, fatal=fatal, errnote=errnote)
             kwargs = {
                 'note': note,
                 'errnote': errnote,
@@ -1502,7 +1504,7 @@ class InfoExtractor:
                 'url': url_or_none(e.get('contentUrl')),
                 'title': unescapeHTML(e.get('name')),
                 'description': unescapeHTML(e.get('description')),
-                'thumbnails': [{'url': url}
+                'thumbnails': [{'url': unescapeHTML(url)}
                                for url in variadic(traverse_obj(e, 'thumbnailUrl', 'thumbnailURL'))
                                if url_or_none(url)],
                 'duration': parse_duration(e.get('duration')),

yt_dlp/extractor/crunchyroll.py

@@ -113,7 +113,7 @@ class CrunchyrollBaseIE(InfoExtractor):
 
 class CrunchyrollIE(CrunchyrollBaseIE, VRVBaseIE):
     IE_NAME = 'crunchyroll'
-    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|(?:[^/]*/){1,2}[^/?&]*?)(?P<id>[0-9]+))(?:[/?&]|$)'
+    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|(?!series/|watch/)(?:[^/]+/){1,2}[^/?&]*?)(?P<id>[0-9]+))(?:[/?&]|$)'
     _TESTS = [{
         'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
         'info_dict': {

yt_dlp/extractor/detik.py

@@ -0,0 +1,122 @@
+from .common import InfoExtractor
+from ..utils import merge_dicts, str_or_none
+
+
+class Detik20IE(InfoExtractor):
+    IE_NAME = '20.detik.com'
+    _VALID_URL = r'https?://20\.detik\.com/((?!program)[\w-]+)/[\d-]+/(?P<id>[\w-]+)'
+    _TESTS = [{
+        # detikflash
+        'url': 'https://20.detik.com/detikflash/20220705-220705098/zulhas-klaim-sukses-turunkan-harga-migor-jawa-bali',
+        'info_dict': {
+            'id': '220705098',
+            'ext': 'mp4',
+            'duration': 157,
+            'thumbnail': 'https://cdnv.detik.com/videoservice/AdminTV/2022/07/05/bfe0384db04f4bbb9dd5efc869c5d4b1-20220705164334-0s.jpg?w=650&q=80',
+            'description': 'md5:ac18dcee5b107abbec1ed46e0bf400e3',
+            'title': 'Zulhas Klaim Sukses Turunkan Harga Migor Jawa-Bali',
+            'tags': ['zulkifli hasan', 'menteri perdagangan', 'minyak goreng'],
+            'timestamp': 1657039548,
+            'upload_date': '20220705'
+        }
+    }, {
+        # e-flash
+        'url': 'https://20.detik.com/e-flash/20220705-220705109/ahli-level-ppkm-jadi-payung-strategi-protokol-kesehatan',
+        'info_dict': {
+            'id': '220705109',
+            'ext': 'mp4',
+            'tags': ['ppkm jabodetabek', 'dicky budiman', 'ppkm'],
+            'upload_date': '20220705',
+            'duration': 110,
+            'title': 'Ahli: Level PPKM Jadi Payung Strategi Protokol Kesehatan',
+            'thumbnail': 'https://cdnv.detik.com/videoservice/AdminTV/2022/07/05/Ahli-_Level_PPKM_Jadi_Payung_Strat_jOgUMCN-20220705182313-custom.jpg?w=650&q=80',
+            'description': 'md5:4eb825a9842e6bdfefd66f47b364314a',
+            'timestamp': 1657045255,
+        }
+    }, {
+        # otobuzz
+        'url': 'https://20.detik.com/otobuzz/20220704-220704093/mulai-rp-10-jutaan-ini-skema-kredit-mitsubishi-pajero-sport',
+        'info_dict': {
+            'id': '220704093',
+            'ext': 'mp4',
+            'tags': ['cicilan mobil', 'mitsubishi pajero sport', 'mitsubishi', 'pajero sport'],
+            'timestamp': 1656951521,
+            'duration': 83,
+            'upload_date': '20220704',
+            'thumbnail': 'https://cdnv.detik.com/videoservice/AdminTV/2022/07/04/5d6187e402ec4a91877755a5886ff5b6-20220704161859-0s.jpg?w=650&q=80',
+            'description': 'md5:9b2257341b6f375cdcf90106146d5ffb',
+            'title': 'Mulai Rp 10 Jutaan! Ini Skema Kredit Mitsubishi Pajero Sport',
+        }
+    }, {
+        # sport-buzz
+        'url': 'https://20.detik.com/sport-buzz/20220704-220704054/crash-crash-horor-di-paruh-pertama-motogp-2022',
+        'info_dict': {
+            'id': '220704054',
+            'ext': 'mp4',
+            'thumbnail': 'https://cdnv.detik.com/videoservice/AdminTV/2022/07/04/6b172c6fb564411996ea145128315630-20220704090746-0s.jpg?w=650&q=80',
+            'title': 'Crash-crash Horor di Paruh Pertama MotoGP 2022',
+            'description': 'md5:fbcc6687572ad7d16eb521b76daa50e4',
+            'timestamp': 1656925591,
+            'duration': 107,
+            'tags': ['marc marquez', 'fabio quartararo', 'francesco bagnaia', 'motogp crash', 'motogp 2022'],
+            'upload_date': '20220704',
+        }
+    }, {
+        # adu-perspektif
+        'url': 'https://20.detik.com/adu-perspektif/20220518-220518144/24-tahun-reformasi-dan-alarm-demokrasi-dari-filipina',
+        'info_dict': {
+            'id': '220518144',
+            'ext': 'mp4',
+            'title': '24 Tahun Reformasi dan Alarm Demokrasi dari Filipina',
+            'upload_date': '20220518',
+            'timestamp': 1652913823,
+            'duration': 185.0,
+            'tags': ['politik', 'adu perspektif', 'indonesia', 'filipina', 'demokrasi'],
+            'description': 'md5:8eaaf440b839c3d02dca8c9bbbb099a9',
+            'thumbnail': 'https://cdnv.detik.com/videoservice/AdminTV/2022/05/18/adpers_18_mei_compressed-20220518230458-custom.jpg?w=650&q=80',
+        }
+    }, {
+        # sosok
+        'url': 'https://20.detik.com/sosok/20220702-220703032/resa-boenard-si-princess-bantar-gebang',
+        'info_dict': {
+            'id': '220703032',
+            'ext': 'mp4',
+            'timestamp': 1656824438,
+            'thumbnail': 'https://cdnv.detik.com/videoservice/AdminTV/2022/07/02/SOSOK_BGBJ-20220702191138-custom.jpg?w=650&q=80',
+            'title': 'Resa Boenard Si \'Princess Bantar Gebang\'',
+            'description': 'md5:84ea66306a0285330de6a13fc6218b78',
+            'tags': ['sosok', 'sosok20d', 'bantar gebang', 'bgbj', 'resa boenard', 'bantar gebang bgbj', 'bgbj bantar gebang', 'sosok bantar gebang', 'sosok bgbj', 'bgbj resa boenard'],
+            'upload_date': '20220703',
+            'duration': 650,
+        }
+    }, {
+        # viral
+        'url': 'https://20.detik.com/viral/20220603-220603135/merasakan-bus-imut-tanpa-pengemudi-muter-muter-di-kawasan-bsd-city',
+        'info_dict': {
+            'id': '220603135',
+            'ext': 'mp4',
+            'description': 'md5:4771fe101aa303edb829c59c26f9e7c6',
+            'timestamp': 1654304305,
+            'title': 'Merasakan Bus Imut Tanpa Pengemudi, Muter-muter di Kawasan BSD City',
+            'tags': ['viral', 'autonomous vehicle', 'electric', 'shuttle bus'],
+            'thumbnail': 'https://cdnv.detik.com/videoservice/AdminTV/2022/06/03/VIRAL_BUS_NO_SUPIR-20220604004707-custom.jpg?w=650&q=80',
+            'duration': 593,
+            'upload_date': '20220604',
+        }
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        json_ld_data = self._search_json_ld(webpage, display_id)
+
+        video_url = self._html_search_regex(
+            r'videoUrl\s*:\s*"(?P<video_url>[^"]+)', webpage, 'videoUrl')
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(video_url, display_id, ext='mp4')
+
+        return merge_dicts(json_ld_data, {
+            'id': self._html_search_meta('video_id', webpage),
+            'formats': formats,
+            'subtitles': subtitles,
+            'tags': str_or_none(self._html_search_meta(['keywords', 'keyword', 'dtk:keywords'], webpage), '').split(','),
+        })

yt_dlp/extractor/espn.py

@@ -281,7 +281,7 @@ class ESPNCricInfoIE(InfoExtractor):
 
 
 class WatchESPNIE(AdobePassIE):
-    _VALID_URL = r'https://www.espn.com/watch/player/_/id/(?P<id>[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})'
+    _VALID_URL = r'https?://(?:www\.)?espn\.com/(?:watch|espnplus)/player/_/id/(?P<id>[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})'
     _TESTS = [{
         'url': 'https://www.espn.com/watch/player/_/id/ba7d17da-453b-4697-bf92-76a99f61642b',
         'info_dict': {
@@ -304,6 +304,17 @@ class WatchESPNIE(AdobePassIE):
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'https://www.espn.com/espnplus/player/_/id/317f5fd1-c78a-4ebe-824a-129e0d348421',
+        'info_dict': {
+            'id': '317f5fd1-c78a-4ebe-824a-129e0d348421',
+            'ext': 'mp4',
+            'title': 'The Wheel - Episode 10',
+            'thumbnail': 'https://s.secure.espncdn.com/stitcher/artwork/collections/media/317f5fd1-c78a-4ebe-824a-129e0d348421/16x9.jpg?timestamp=202205031523&showBadge=true&cb=12&package=ESPN_PLUS',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     _API_KEY = 'ZXNwbiZicm93c2VyJjEuMC4w.ptUt7QxsteaRruuPmGZFaJByOoqKvDP2a5YkInHrc7c'

yt_dlp/extractor/fifa.py

@@ -16,21 +16,21 @@ class FifaIE(InfoExtractor):
             'title': 'Italy v France | Final | 2006 FIFA World Cup Germany™ | Full Match Replay',
             'description': 'md5:f4520d0ee80529c8ba4134a7d692ff8b',
             'ext': 'mp4',
-            'categories': ['FIFA Tournaments', 'Replay'],
+            'categories': ['FIFA Tournaments'],
             'thumbnail': 'https://digitalhub.fifa.com/transform/fa6f0b3e-a2e9-4cf7-9f32-53c57bcb7360/2006_Final_ITA_FRA',
-            'duration': 8164,
+            'duration': 8165,
         },
         'params': {'skip_download': 'm3u8'},
     }, {
         'url': 'https://www.fifa.com/fifaplus/pt/watch/1cg5r5Qt6Qt12ilkDgb1sV',
         'info_dict': {
             'id': '1cg5r5Qt6Qt12ilkDgb1sV',
-            'title': 'Brasil x Alemanha | Semifinais | Copa do Mundo FIFA Brasil 2014 | Compacto',
-            'description': 'md5:ba4ffcc084802b062beffc3b4c4b19d6',
+            'title': 'Brazil v Germany | Semi-finals | 2014 FIFA World Cup Brazil™ | Extended Highlights',
+            'description': 'md5:d908c74ee66322b804ae2e521b02a855',
             'ext': 'mp4',
             'categories': ['FIFA Tournaments', 'Highlights'],
             'thumbnail': 'https://digitalhub.fifa.com/transform/d8fe6f61-276d-4a73-a7fe-6878a35fd082/FIFAPLS_100EXTHL_2014BRAvGER_TMB',
-            'duration': 901,
+            'duration': 902,
             'release_timestamp': 1404777600,
             'release_date': '20140708',
         },
@@ -39,8 +39,8 @@ class FifaIE(InfoExtractor):
         'url': 'https://www.fifa.com/fifaplus/fr/watch/3C6gQH9C2DLwzNx7BMRQdp',
         'info_dict': {
             'id': '3C6gQH9C2DLwzNx7BMRQdp',
-            'title': 'Le but de Josimar contre le Irlande du Nord | Buts classiques',
-            'description': 'md5:16f9f789f09960bfe7220fe67af31f34',
+            'title': 'Josimar goal against Northern Ireland | Classic Goals',
+            'description': 'md5:cbe7e7bb52f603c9f1fe9a4780fe983b',
             'ext': 'mp4',
             'categories': ['FIFA Tournaments', 'Goal'],
             'duration': 28,
@@ -56,27 +56,13 @@ class FifaIE(InfoExtractor):
         preconnect_link = self._search_regex(
             r'<link[^>]+rel\s*=\s*"preconnect"[^>]+href\s*=\s*"([^"]+)"', webpage, 'Preconnect Link')
 
-        json_data = self._download_json(
-            f'{preconnect_link}/video/GetVideoPlayerData/{video_id}', video_id,
-            'Downloading Video Player Data', query={'includeIdents': True, 'locale': locale})
-
         video_details = self._download_json(
             f'{preconnect_link}/sections/videoDetails/{video_id}', video_id, 'Downloading Video Details', fatal=False)
 
         preplay_parameters = self._download_json(
-            f'{preconnect_link}/video/GetVerizonPreplayParameters', video_id, 'Downloading Preplay Parameters', query={
-                'entryId': video_id,
-                'assetId': json_data['verizonAssetId'],
-                'useExternalId': False,
-                'requiresToken': json_data['requiresToken'],
-                'adConfig': 'fifaplusvideo',
-                'prerollAds': True,
-                'adVideoId': json_data['externalVerizonAssetId'],
-                'preIdentId': json_data['preIdentId'],
-                'postIdentId': json_data['postIdentId'],
-            })
+            f'{preconnect_link}/video/GetVerizonPreplayParameters/{video_id}', video_id, 'Downloading Preplay Parameters')['preplayParameters']
 
-        cid = f'{json_data["preIdentId"]},{json_data["verizonAssetId"]},{json_data["postIdentId"]}'
+        cid = preplay_parameters['contentId']
         content_data = self._download_json(
             f'https://content.uplynk.com/preplay/{cid}/multiple.json', video_id, 'Downloading Content Data', query={
                 'v': preplay_parameters['preplayAPIVersion'],
@@ -98,9 +84,9 @@ class FifaIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': json_data.get('title'),
-            'description': json_data.get('description'),
-            'duration': int_or_none(json_data.get('duration')),
+            'title': video_details.get('title'),
+            'description': video_details.get('description'),
+            'duration': int_or_none(video_details.get('duration')),
             'release_timestamp': unified_timestamp(video_details.get('dateOfRelease')),
             'categories': traverse_obj(video_details, (('videoCategory', 'videoSubcategory'),)),
             'thumbnail': traverse_obj(video_details, ('backgroundImage', 'src')),

yt_dlp/extractor/generic.py

@@ -111,7 +111,6 @@ from ..compat import compat_etree_fromstring
 from ..utils import (
     KNOWN_EXTENSIONS,
     ExtractorError,
-    HEADRequest,
     UnsupportedError,
     determine_ext,
     dict_get,
@@ -124,7 +123,6 @@ from ..utils import (
     orderedSet,
     parse_duration,
     parse_resolution,
-    sanitized_Request,
     smuggle_url,
     str_or_none,
     try_call,
@@ -2807,49 +2805,30 @@ class GenericIE(InfoExtractor):
         else:
             video_id = self._generic_id(url)
 
-        self.to_screen('%s: Requesting header' % video_id)
-
-        head_req = HEADRequest(url)
-        head_response = self._request_webpage(
-            head_req, video_id,
-            note=False, errnote='Could not send HEAD request to %s' % url,
-            fatal=False)
-
-        if head_response is not False:
-            # Check for redirect
-            new_url = head_response.geturl()
-            if url != new_url:
-                self.report_following_redirect(new_url)
-                if force_videoid:
-                    new_url = smuggle_url(
-                        new_url, {'force_videoid': force_videoid})
-                return self.url_result(new_url)
-
-        def request_webpage():
-            request = sanitized_Request(url)
-            # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
-            # making it impossible to download only chunk of the file (yet we need only 512kB to
-            # test whether it's HTML or not). According to yt-dlp default Accept-Encoding
-            # that will always result in downloading the whole file that is not desirable.
-            # Therefore for extraction pass we have to override Accept-Encoding to any in order
-            # to accept raw bytes and being able to download only a chunk.
-            # It may probably better to solve this by checking Content-Type for application/octet-stream
-            # after HEAD request finishes, but not sure if we can rely on this.
-            request.add_header('Accept-Encoding', '*')
-            return self._request_webpage(request, video_id)
-
-        full_response = None
-        if head_response is False:
-            head_response = full_response = request_webpage()
+        # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
+        # making it impossible to download only chunk of the file (yet we need only 512kB to
+        # test whether it's HTML or not). According to yt-dlp default Accept-Encoding
+        # that will always result in downloading the whole file that is not desirable.
+        # Therefore for extraction pass we have to override Accept-Encoding to any in order
+        # to accept raw bytes and being able to download only a chunk.
+        # It may probably better to solve this by checking Content-Type for application/octet-stream
+        # after a HEAD request, but not sure if we can rely on this.
+        full_response = self._request_webpage(url, video_id, headers={'Accept-Encoding': '*'})
+        new_url = full_response.geturl()
+        if url != new_url:
+            self.report_following_redirect(new_url)
+            if force_videoid:
+                new_url = smuggle_url(new_url, {'force_videoid': force_videoid})
+            return self.url_result(new_url)
 
         info_dict = {
             'id': video_id,
             'title': self._generic_title(url),
-            'timestamp': unified_timestamp(head_response.headers.get('Last-Modified'))
+            'timestamp': unified_timestamp(full_response.headers.get('Last-Modified'))
         }
 
         # Check for direct link to a video
-        content_type = head_response.headers.get('Content-Type', '').lower()
+        content_type = full_response.headers.get('Content-Type', '').lower()
         m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
         if m:
             self.report_detected('direct video link')
@@ -2878,7 +2857,6 @@ class GenericIE(InfoExtractor):
             self.report_warning(
                 '%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))
 
-        full_response = full_response or request_webpage()
         first_bytes = full_response.read(512)
 
         # Is it an M3U playlist?
@@ -4103,7 +4081,7 @@ class GenericIE(InfoExtractor):
                 webpage)
             if not found:
                 # Look also in Refresh HTTP header
-                refresh_header = head_response.headers.get('Refresh')
+                refresh_header = full_response.headers.get('Refresh')
                 if refresh_header:
                     found = re.search(REDIRECT_REGEX, refresh_header)
             if found:

yt_dlp/extractor/hytale.py

@@ -0,0 +1,58 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import traverse_obj
+
+
+class HytaleIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?hytale\.com/news/\d+/\d+/(?P<id>[a-z0-9-]+)'
+    _TESTS = [{
+        'url': 'https://hytale.com/news/2021/07/summer-2021-development-update',
+        'info_dict': {
+            'id': 'summer-2021-development-update',
+            'title': 'Summer 2021 Development Update',
+        },
+        'playlist_count': 4,
+        'playlist': [{
+            'md5': '0854ebe347d233ee19b86ab7b2ead610',
+            'info_dict': {
+                'id': 'ed51a2609d21bad6e14145c37c334999',
+                'ext': 'mp4',
+                'title': 'Avatar Personalization',
+                'thumbnail': r're:https://videodelivery\.net/\w+/thumbnails/thumbnail\.jpg',
+            }
+        }]
+    }, {
+        'url': 'https://www.hytale.com/news/2019/11/hytale-graphics-update',
+        'info_dict': {
+            'id': 'hytale-graphics-update',
+            'title': 'Hytale graphics update',
+        },
+        'playlist_count': 2,
+    }]
+
+    def _real_initialize(self):
+        media_webpage = self._download_webpage(
+            'https://hytale.com/media', None, note='Downloading list of media', fatal=False) or ''
+
+        clips_json = traverse_obj(
+            self._search_json(
+                r'window\.__INITIAL_COMPONENTS_STATE__\s*=\s*\[',
+                media_webpage, 'clips json', None),
+            ('media', 'clips')) or []
+
+        self._titles = {clip.get('src'): clip.get('caption') for clip in clips_json}
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        webpage = self._download_webpage(url, playlist_id)
+        entries = [
+            self.url_result(
+                f'https://cloudflarestream.com/{video_hash}/manifest/video.mpd?parentOrigin=https%3A%2F%2Fhytale.com',
+                title=self._titles.get(video_hash), url_transparent=True)
+            for video_hash in re.findall(
+                r'<stream\s+class\s*=\s*"ql-video\s+cf-stream"\s+src\s*=\s*"([a-f0-9]{32})"',
+                webpage)
+        ]
+
+        return self.playlist_result(entries, playlist_id, self._og_search_title(webpage))

yt_dlp/extractor/instagram.py

@@ -1,17 +1,17 @@
-import itertools
 import hashlib
+import itertools
 import json
 import re
 import time
+import urllib.error
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_HTTPError,
-)
 from ..utils import (
     ExtractorError,
-    format_field,
+    decode_base_n,
+    encode_base_n,
     float_or_none,
+    format_field,
     get_element_by_attribute,
     int_or_none,
     lowercase_escape,
@@ -22,6 +22,18 @@ from ..utils import (
     urlencode_postdata,
 )
 
+_ENCODING_CHARS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'
+
+
+def _pk_to_id(id):
+    """Source: https://stackoverflow.com/questions/24437823/getting-instagram-post-url-from-media-id"""
+    return encode_base_n(int(id.split('_')[0]), table=_ENCODING_CHARS)
+
+
+def _id_to_pk(shortcode):
+    """Covert a shortcode to a numeric value"""
+    return decode_base_n(shortcode[:11], table=_ENCODING_CHARS)
+
 
 class InstagramBaseIE(InfoExtractor):
     _NETRC_MACHINE = 'instagram'
@@ -156,6 +168,15 @@ class InstagramBaseIE(InfoExtractor):
         if isinstance(product_info, list):
             product_info = product_info[0]
 
+        comment_data = traverse_obj(product_info, ('edge_media_to_parent_comment', 'edges'))
+        comments = [{
+            'author': traverse_obj(comment_dict, ('node', 'owner', 'username')),
+            'author_id': traverse_obj(comment_dict, ('node', 'owner', 'id')),
+            'id': traverse_obj(comment_dict, ('node', 'id')),
+            'text': traverse_obj(comment_dict, ('node', 'text')),
+            'timestamp': traverse_obj(comment_dict, ('node', 'created_at'), expected_type=int_or_none),
+        } for comment_dict in comment_data] if comment_data else None
+
         user_info = product_info.get('user') or {}
         info_dict = {
             'id': product_info.get('code') or product_info.get('id'),
@@ -168,6 +189,7 @@ class InstagramBaseIE(InfoExtractor):
             'view_count': int_or_none(product_info.get('view_count')),
             'like_count': int_or_none(product_info.get('like_count')),
             'comment_count': int_or_none(product_info.get('comment_count')),
+            'comments': comments,
             'http_headers': {
                 'Referer': 'https://www.instagram.com/',
             }
@@ -214,23 +236,9 @@ class InstagramIOSIE(InfoExtractor):
         'add_ie': ['Instagram']
     }]
 
-    def _get_id(self, id):
-        """Source: https://stackoverflow.com/questions/24437823/getting-instagram-post-url-from-media-id"""
-        chrs = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'
-        media_id = int(id.split('_')[0])
-        shortened_id = ''
-        while media_id > 0:
-            r = media_id % 64
-            media_id = (media_id - r) // 64
-            shortened_id = chrs[r] + shortened_id
-        return shortened_id
-
     def _real_extract(self, url):
-        return {
-            '_type': 'url_transparent',
-            'url': f'http://instagram.com/tv/{self._get_id(self._match_id(url))}/',
-            'ie_key': 'Instagram',
-        }
+        video_id = _pk_to_id(self._match_id(url))
+        return self.url_result(f'http://instagram.com/tv/{video_id}', InstagramIE, video_id)
 
 
 class InstagramIE(InstagramBaseIE):
@@ -358,39 +366,49 @@ class InstagramIE(InstagramBaseIE):
 
     def _real_extract(self, url):
         video_id, url = self._match_valid_url(url).group('id', 'url')
-        webpage, urlh = self._download_webpage_handle(url, video_id)
-        if 'www.instagram.com/accounts/login' in urlh.geturl():
-            self.report_warning('Main webpage is locked behind the login page. '
-                                'Retrying with embed webpage (Note that some metadata might be missing)')
-            webpage = self._download_webpage(
-                'https://www.instagram.com/p/%s/embed/' % video_id, video_id, note='Downloading embed webpage')
-
-        shared_data = self._parse_json(
-            self._search_regex(
-                r'window\._sharedData\s*=\s*({.+?});',
-                webpage, 'shared data', default='{}'),
-            video_id, fatal=False)
-        media = traverse_obj(
-            shared_data,
-            ('entry_data', 'PostPage', 0, 'graphql', 'shortcode_media'),
-            ('entry_data', 'PostPage', 0, 'media'),
-            expected_type=dict)
-
-        # _sharedData.entry_data.PostPage is empty when authenticated (see
-        # https://github.com/ytdl-org/youtube-dl/pull/22880)
+        general_info = self._download_json(
+            f'https://www.instagram.com/graphql/query/?query_hash=9f8827793ef34641b2fb195d4d41151c'
+            f'&variables=%7B"shortcode":"{video_id}",'
+            '"parent_comment_count":10,"has_threaded_comments":true}', video_id, fatal=False, errnote=False,
+            headers={
+                'Accept': '*',
+                'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36',
+                'Authority': 'www.instagram.com',
+                'Referer': 'https://www.instagram.com',
+                'x-ig-app-id': '936619743392459',
+            })
+        media = traverse_obj(general_info, ('data', 'shortcode_media')) or {}
         if not media:
-            additional_data = self._parse_json(
-                self._search_regex(
-                    r'window\.__additionalDataLoaded\s*\(\s*[^,]+,\s*({.+?})\s*\);',
-                    webpage, 'additional data', default='{}'),
-                video_id, fatal=False)
-            product_item = traverse_obj(additional_data, ('items', 0), expected_type=dict)
-            if product_item:
-                return self._extract_product(product_item)
-            media = traverse_obj(additional_data, ('graphql', 'shortcode_media'), 'shortcode_media', expected_type=dict) or {}
+            self.report_warning('General metadata extraction failed', video_id)
 
-        if not media and 'www.instagram.com/accounts/login' in urlh.geturl():
-            self.raise_login_required('You need to log in to access this content')
+        info = self._download_json(
+            f'https://i.instagram.com/api/v1/media/{_id_to_pk(video_id)}/info/', video_id,
+            fatal=False, note='Downloading video info', errnote=False, headers={
+                'Accept': '*',
+                'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36',
+                'Authority': 'www.instagram.com',
+                'Referer': 'https://www.instagram.com',
+                'x-ig-app-id': '936619743392459',
+            })
+        if info:
+            media.update(info['items'][0])
+            return self._extract_product(media)
+
+        webpage = self._download_webpage(
+            f'https://www.instagram.com/p/{video_id}/embed/', video_id,
+            note='Downloading embed webpage', fatal=False)
+        if not webpage:
+            self.raise_login_required('Requested content was not found, the content might be private')
+
+        additional_data = self._search_json(
+            r'window\.__additionalDataLoaded\s*\(\s*[^,]+,\s*', webpage, 'additional data', video_id, fatal=False)
+        product_item = traverse_obj(additional_data, ('items', 0), expected_type=dict)
+        if product_item:
+            media.update(product_item)
+            return self._extract_product(media)
+
+        media.update(traverse_obj(
+            additional_data, ('graphql', 'shortcode_media'), 'shortcode_media', expected_type=dict) or {})
 
         username = traverse_obj(media, ('owner', 'username')) or self._search_regex(
             r'"owner"\s*:\s*{\s*"username"\s*:\s*"(.+?)"', webpage, 'username', fatal=False)
@@ -519,7 +537,7 @@ class InstagramPlaylistBaseIE(InstagramBaseIE):
                 except ExtractorError as e:
                     # if it's an error caused by a bad query, and there are
                     # more GIS templates to try, ignore it and keep trying
-                    if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                    if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
                         if gis_tmpl != gis_tmpls[-1]:
                             continue
                     raise
@@ -629,41 +647,36 @@ class InstagramStoryIE(InstagramBaseIE):
 
     def _real_extract(self, url):
         username, story_id = self._match_valid_url(url).groups()
-
-        story_info_url = f'{username}/{story_id}/?__a=1' if username == 'highlights' else f'{username}/?__a=1'
-        story_info = self._download_json(f'https://www.instagram.com/stories/{story_info_url}', story_id, headers={
-            'X-IG-App-ID': 936619743392459,
-            'X-ASBD-ID': 198387,
-            'X-IG-WWW-Claim': 0,
-            'X-Requested-With': 'XMLHttpRequest',
-            'Referer': url,
-        })
-        user_id = story_info['user']['id']
-        highlight_title = traverse_obj(story_info, ('highlight', 'title'))
+        story_info = self._download_webpage(url, story_id)
+        user_info = self._search_json(r'"user":', story_info, 'user info', story_id, fatal=False)
+        if not user_info:
+            self.raise_login_required('This content is unreachable')
+        user_id = user_info.get('id')
 
         story_info_url = user_id if username != 'highlights' else f'highlight:{story_id}'
-        videos = self._download_json(f'https://i.instagram.com/api/v1/feed/reels_media/?reel_ids={story_info_url}', story_id, headers={
-            'X-IG-App-ID': 936619743392459,
-            'X-ASBD-ID': 198387,
-            'X-IG-WWW-Claim': 0,
-        })['reels']
+        videos = traverse_obj(self._download_json(
+            f'https://i.instagram.com/api/v1/feed/reels_media/?reel_ids={story_info_url}',
+            story_id, errnote=False, fatal=False, headers={
+                'X-IG-App-ID': 936619743392459,
+                'X-ASBD-ID': 198387,
+                'X-IG-WWW-Claim': 0,
+            }), 'reels')
+        if not videos:
+            self.raise_login_required('You need to log in to access this content')
 
-        full_name = traverse_obj(videos, ('user', 'full_name'))
-
-        user_info = {}
-        if not (username and username != 'highlights' and full_name):
-            user_info = self._download_json(
-                f'https://i.instagram.com/api/v1/users/{user_id}/info/', story_id, headers={
-                    'User-Agent': 'Mozilla/5.0 (Linux; Android 11; SM-A505F Build/RP1A.200720.012; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/96.0.4664.45 Mobile Safari/537.36 Instagram 214.1.0.29.120 Android (30/11; 450dpi; 1080x2122; samsung; SM-A505F; a50; exynos9610; en_US; 333717274)',
-                }, note='Downloading user info')
-
-        username = traverse_obj(user_info, ('user', 'username')) or username
-        full_name = traverse_obj(user_info, ('user', 'full_name')) or full_name
+        full_name = traverse_obj(videos, (f'highlight:{story_id}', 'user', 'full_name'), (str(user_id), 'user', 'full_name'))
+        story_title = traverse_obj(videos, (f'highlight:{story_id}', 'title'))
+        if not story_title:
+            story_title = f'Story by {username}'
 
         highlights = traverse_obj(videos, (f'highlight:{story_id}', 'items'), (str(user_id), 'items'))
-        return self.playlist_result([{
-            **self._extract_product(highlight),
-            'title': f'Story by {username}',
-            'uploader': full_name,
-            'uploader_id': user_id,
-        } for highlight in highlights], playlist_id=story_id, playlist_title=highlight_title)
+        info_data = []
+        for highlight in highlights:
+            highlight_data = self._extract_product(highlight)
+            if highlight_data.get('formats'):
+                info_data.append({
+                    **highlight_data,
+                    'uploader': full_name,
+                    'uploader_id': user_id,
+                })
+        return self.playlist_result(info_data, playlist_id=story_id, playlist_title=story_title)

yt_dlp/extractor/iqiyi.py

@@ -441,6 +441,7 @@ class IqIE(InfoExtractor):
         '1': 'zh_CN',
         '2': 'zh_TW',
         '3': 'en',
+        '4': 'kor',
         '18': 'th',
         '21': 'my',
         '23': 'vi',

yt_dlp/extractor/liputan6.py

@@ -0,0 +1,64 @@
+from .common import InfoExtractor
+from .vidio import VidioIE
+
+
+class Liputan6IE(InfoExtractor):
+    _VALID_URL = r'https?://www\.liputan6\.com/\w+/read/\d+/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://www.liputan6.com/news/read/5007510/video-duh-perawat-rs-di-medan-diduga-salah-berikan-obat-ke-pasien',
+        'info_dict': {
+            'id': '7082548',
+            'ext': 'mp4',
+            'title': 'Duh, Perawat RS di Medan Diduga Salah Berikan Obat Ke Pasien',
+            'thumbnail': 'https://thumbor.prod.vidiocdn.com/lOz5pStm9X-jjlTa_VQQUelOPtw=/640x360/filters:quality(70)/vidio-web-prod-video/uploads/video/image/7082548/duh-perawat-rs-di-medan-diduga-salah-berikan-obat-ke-pasien-ca1125.jpg',
+            'channel_id': '185693',
+            'uploader': 'Liputan6.com',
+            'duration': 104,
+            'uploader_url': 'https://www.vidio.com/@liputan6',
+            'description': 'md5:3b58ecff10ec3a41d4304cf98228435a',
+            'timestamp': 1657159427,
+            'uploader_id': 'liputan6',
+            'display_id': 'video-duh-perawat-rs-di-medan-diduga-salah-berikan-obat-ke-pasien',
+            'like_count': int,
+            'view_count': int,
+            'comment_count': int,
+            'tags': ['perawat indonesia', 'rumah sakit', 'Medan', 'viral hari ini', 'viral', 'enamplus'],
+            'channel': 'Default Channel',
+            'dislike_count': int,
+            'upload_date': '20220707'
+        }
+    }, {
+        'url': 'https://www.liputan6.com/tv/read/5007719/video-program-minyakita-minyak-goreng-kemasan-sederhana-seharga-rp-14-ribu',
+        'info_dict': {
+            'id': '7082543',
+            'ext': 'mp4',
+            'title': 'md5:ecb7b3c598b97798bfd0eb50c6233b8c',
+            'channel_id': '604054',
+            'dislike_count': int,
+            'comment_count': int,
+            'timestamp': 1657159211,
+            'upload_date': '20220707',
+            'tags': ['minyakita', 'minyak goreng', 'liputan 6', 'sctv'],
+            'uploader_url': 'https://www.vidio.com/@sctv',
+            'display_id': 'video-program-minyakita-minyak-goreng-kemasan-sederhana-seharga-rp-14-ribu',
+            'like_count': int,
+            'uploader': 'SCTV',
+            'description': 'md5:6c374d82589b71fb98b3d550edb6873f',
+            'duration': 99,
+            'uploader_id': 'sctv',
+            'thumbnail': 'https://thumbor.prod.vidiocdn.com/AAIOjz-64hKojjdw5hr0oNNEeJg=/640x360/filters:quality(70)/vidio-web-prod-video/uploads/video/image/7082543/program-minyakita-minyak-goreng-kemasan-sederhana-seharga-rp14-ribu-_-liputan-6-7d9fbb.jpg',
+            'channel': 'Liputan 6 Pagi',
+            'view_count': int,
+        }
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        json_data = self._search_json(
+            r'window.kmklabs.gtm\s*=\s*', webpage, 'json_data', display_id)
+        video_id = json_data['videos']['video_1']['video_id']
+
+        return self.url_result(
+            f'https://www.vidio.com/watch/{video_id}-{display_id}', ie=VidioIE, video_id=display_id)

yt_dlp/extractor/mgtv.py

@@ -67,7 +67,7 @@ class MGTVIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         tk2 = base64.urlsafe_b64encode(
-            f'did={compat_str(uuid.uuid4()).encode()}|pno=1030|ver=0.3.0301|clit={int(time.time())}'.encode())[::-1]
+            f'did={str(uuid.uuid4())}|pno=1030|ver=0.3.0301|clit={int(time.time())}'.encode())[::-1]
         try:
             api_data = self._download_json(
                 'https://pcweb.api.mgtv.com/player/video', video_id, query={
@@ -137,14 +137,15 @@ class MGTVIE(InfoExtractor):
             url_sub = sub.get('url')
             if not url_sub:
                 continue
-            locale = sub.get('captionCountrySimpleName')
+            locale = sub.get('captionSimpleName') or 'en'
             sub = self._download_json(f'{domain}{url_sub}', video_id, fatal=False,
                                       note=f'Download subtitle for locale {sub.get("name")} ({locale})') or {}
             sub_url = url_or_none(sub.get('info'))
             if not sub_url:
                 continue
-            subtitles.setdefault(locale or 'en', []).append({
+            subtitles.setdefault(locale.lower(), []).append({
                 'url': sub_url,
+                'name': sub.get('name'),
                 'ext': 'srt'
             })
         return subtitles

yt_dlp/extractor/mocha.py

@@ -0,0 +1,66 @@
+from .common import InfoExtractor
+from ..utils import int_or_none, traverse_obj
+
+
+class MochaVideoIE(InfoExtractor):
+    _VALID_URL = r'https?://video.mocha.com.vn/(?P<video_slug>[\w-]+)'
+    _TESTS = [{
+        'url': 'http://video.mocha.com.vn/chuyen-meo-gia-su-tu-thong-diep-cuoc-song-v18694039',
+        'info_dict': {
+            'id': '18694039',
+            'title': 'Chuyện mèo giả sư tử | Thông điệp cuộc sống',
+            'ext': 'mp4',
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'display_id': 'chuyen-meo-gia-su-tu-thong-diep-cuoc-song',
+            'thumbnail': 'http://mcvideomd1fr.keeng.net/playnow/images/20220505/ad0a055d-2f69-42ca-b888-4790041fe6bc_640x480.jpg',
+            'description': '',
+            'duration': 70,
+            'timestamp': 1652254203,
+            'upload_date': '20220511',
+            'comment_count': int,
+            'categories': ['Kids']
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_slug = self._match_valid_url(url).group('video_slug')
+        json_data = self._download_json(
+            'http://apivideo.mocha.com.vn:8081/onMediaBackendBiz/mochavideo/getVideoDetail',
+            video_slug, query={'url': url, 'token': ''})['data']['videoDetail']
+        video_id = str(json_data['id'])
+        video_urls = (json_data.get('list_resolution') or []) + [json_data.get('original_path')]
+
+        formats, subtitles = [], {}
+        for video in video_urls:
+            if isinstance(video, str):
+                formats.extend([{'url': video, 'ext': 'mp4'}])
+            else:
+                fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                    video.get('video_path'), video_id, ext='mp4')
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'display_id': json_data.get('slug') or video_slug,
+            'title': json_data.get('name'),
+            'formats': formats,
+            'subtitles': subtitles,
+            'description': json_data.get('description'),
+            'duration': json_data.get('durationS'),
+            'view_count': json_data.get('total_view'),
+            'like_count': json_data.get('total_like'),
+            'dislike_count': json_data.get('total_unlike'),
+            'thumbnail': json_data.get('image_path_thumb'),
+            'timestamp': int_or_none(json_data.get('publish_time'), scale=1000),
+            'is_live': json_data.get('isLive'),
+            'channel': traverse_obj(json_data, ('channels', '0', 'name')),
+            'channel_id': traverse_obj(json_data, ('channels', '0', 'id')),
+            'channel_follower_count': traverse_obj(json_data, ('channels', '0', 'numfollow')),
+            'categories': traverse_obj(json_data, ('categories', ..., 'categoryname')),
+            'comment_count': json_data.get('total_comment'),
+        }

yt_dlp/extractor/netverse.py

@@ -1,12 +1,6 @@
-import functools
-
 from .common import InfoExtractor
 from .dailymotion import DailymotionIE
-from ..utils import (
-    InAdvancePagedList,
-    smuggle_url,
-    traverse_obj,
-)
+from ..utils import smuggle_url, traverse_obj
 
 
 class NetverseBaseIE(InfoExtractor):
@@ -14,16 +8,13 @@ class NetverseBaseIE(InfoExtractor):
         'watch': 'watchvideo',
         'video': 'watchvideo',
         'webseries': 'webseries',
+        'season': 'webseason_videos',
     }
 
-    def _call_api(self, url, query={}):
-        display_id, sites_type = self._match_valid_url(url).group('display_id', 'type')
-
-        json_data = self._download_json(
-            f'https://api.netverse.id/medias/api/v2/{self._ENDPOINTS[sites_type]}/{display_id}',
-            display_id, query=query)
-
-        return display_id, json_data
+    def _call_api(self, slug, endpoint, query={}, season_id='', display_id=None):
+        return self._download_json(
+            f'https://api.netverse.id/medias/api/v2/{self._ENDPOINTS[endpoint]}/{slug}/{season_id}',
+            display_id or slug, query=query)
 
 
 class NetverseIE(NetverseBaseIE):
@@ -36,10 +27,9 @@ class NetverseIE(NetverseBaseIE):
             'title': 'Waktu Indonesia Bercanda - Edisi Spesial Lebaran 2016',
             'ext': 'mp4',
             'season': 'Season 2016',
-            'description': 'md5:fc27747c0aa85067b6967c816f01617c',
-            'thumbnail': 'https://vplayed-uat.s3-ap-southeast-1.amazonaws.com/images/webseries/thumbnails/2021/11/619cfce45c827.jpeg',
+            'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
+            'thumbnail': r're:https?://s\d+\.dmcdn\.net/v/T7aV31Y0eGRWBbwkK/x1080',
             'episode_number': 22,
-            'series': 'Waktu Indonesia Bercanda',
             'episode': 'Episode 22',
             'uploader_id': 'x2ir3vq',
             'age_limit': 0,
@@ -60,10 +50,9 @@ class NetverseIE(NetverseBaseIE):
             'title': 'Jadoo Seorang Model',
             'ext': 'mp4',
             'season': 'Season 2',
-            'description': 'md5:c616e8e59d3edf2d3d506e3736120d99',
-            'thumbnail': 'https://storage.googleapis.com/netprime-live/images/webseries/thumbnails/2021/11/619cf63f105d3.jpeg',
+            'description': 'md5:8a74f70812cca267e19ee0635f0af835',
+            'thumbnail': r're:https?://s\d+\.dmcdn\.net/v/Thwuy1YURicFmGu0v/x1080',
             'episode_number': 2,
-            'series': 'Hello Jadoo',
             'episode': 'Episode 2',
             'view_count': int,
             'like_count': int,
@@ -85,10 +74,9 @@ class NetverseIE(NetverseBaseIE):
             'ext': 'mp4',
             'title': 'Tetangga Baru',
             'season': 'Season 1',
-            'description': 'md5:ed6dd355bed84d139b1154c3d8d65957',
-            'thumbnail': 'https://vplayed-uat.s3-ap-southeast-1.amazonaws.com/images/webseries/thumbnails/2021/11/619cfd9d32c5f.jpeg',
+            'description': 'md5:23fcf70e97d461d3029d25d59b2ccfb9',
+            'thumbnail': r're:https?://s\d+\.dmcdn\.net/v/T3Ogm1YEnnyjVKAFF/x1080',
             'episode_number': 1,
-            'series': 'Tetangga Masa Gitu',
             'episode': 'Episode 1',
             'timestamp': 1624538169,
             'view_count': int,
@@ -108,12 +96,11 @@ class NetverseIE(NetverseBaseIE):
         'info_dict': {
             'id': 'x887jzz',
             'ext': 'mp4',
-            'thumbnail': 'https://storage.googleapis.com/netprime-live/images/webseries/thumbnails/2021/11/619cf63f105d3.jpeg',
+            'thumbnail': r're:https?://s\d+\.dmcdn\.net/v/TfuZ_1Y6PboJ5An_s/x1080',
             'season': 'Season 1',
             'episode_number': 1,
-            'description': 'md5:c616e8e59d3edf2d3d506e3736120d99',
+            'description': 'md5:d4f627b3e7a3f9acdc55f6cdd5ea41d5',
             'title': 'Namaku Choi Jadoo',
-            'series': 'Hello Jadoo',
             'episode': 'Episode 1',
             'age_limit': 0,
             'like_count': int,
@@ -130,7 +117,8 @@ class NetverseIE(NetverseBaseIE):
     }]
 
     def _real_extract(self, url):
-        display_id, program_json = self._call_api(url)
+        display_id, sites_type = self._match_valid_url(url).group('display_id', 'type')
+        program_json = self._call_api(display_id, sites_type)
         videos = program_json['response']['videos']
 
         return {
@@ -143,34 +131,46 @@ class NetverseIE(NetverseBaseIE):
             'thumbnail': traverse_obj(videos, ('program_detail', 'thumbnail_image')),
             'description': traverse_obj(videos, ('program_detail', 'description')),
             'episode_number': videos.get('episode_order'),
-            'series': traverse_obj(videos, ('program_detail', 'title')),
         }
 
 
 class NetversePlaylistIE(NetverseBaseIE):
     _VALID_URL = r'https?://(?:\w+\.)?netverse\.id/(?P<type>webseries)/(?P<display_id>[^/?#&]+)'
-    _TEST = {
+    _TESTS = [{
+        # multiple season
         'url': 'https://netverse.id/webseries/tetangga-masa-gitu',
         'info_dict': {
             'id': 'tetangga-masa-gitu',
             'title': 'Tetangga Masa Gitu',
         },
-        'playlist_count': 46,
-    }
+        'playlist_count': 519,
+    }, {
+        # single season
+        'url': 'https://netverse.id/webseries/kelas-internasional',
+        'info_dict': {
+            'id': 'kelas-internasional',
+            'title': 'Kelas Internasional',
+        },
+        'playlist_count': 203,
+    }]
 
-    def parse_playlist(self, url, page_num):
-        _, playlist_json = self._call_api(url, query={'page': page_num + 1})
-        for slug in traverse_obj(playlist_json, ('response', 'related', 'data', ..., 'slug')):
-            yield self.url_result(f'https://www.netverse.id/video/{slug}', NetverseIE)
+    def parse_playlist(self, json_data, playlist_id):
+        slug_sample = traverse_obj(json_data, ('related', 'data', ..., 'slug'))[0]
+        for season in traverse_obj(json_data, ('seasons', ..., 'id')):
+            playlist_json = self._call_api(
+                slug_sample, 'season', display_id=playlist_id, season_id=season)
+
+            for current_page in range(playlist_json['response']['season_list']['last_page']):
+                playlist_json = self._call_api(slug_sample, 'season', query={'page': current_page + 1},
+                                               season_id=season, display_id=playlist_id)
+                for slug in traverse_obj(playlist_json, ('response', ..., 'data', ..., 'slug')):
+                    yield self.url_result(f'https://www.netverse.id/video/{slug}', NetverseIE)
 
     def _real_extract(self, url):
-        _, playlist_data = self._call_api(url)
-        webseries_related_info = playlist_data['response']['related']
-        # TODO: get video from other season
-        # The season has id and the next season video is located at api_url/<season_id>?page=<page>
+        playlist_id, sites_type = self._match_valid_url(url).group('display_id', 'type')
+        playlist_data = self._call_api(playlist_id, sites_type)
+
         return self.playlist_result(
-            InAdvancePagedList(functools.partial(self.parse_playlist, url),
-                               webseries_related_info['last_page'],
-                               webseries_related_info['to'] - webseries_related_info['from'] + 1),
+            self.parse_playlist(playlist_data['response'], playlist_id),
             traverse_obj(playlist_data, ('response', 'webseries_info', 'slug')),
             traverse_obj(playlist_data, ('response', 'webseries_info', 'title')))

yt_dlp/extractor/openload.py

@@ -104,9 +104,8 @@ class PhantomJSwrapper:
 
         self.exe = check_executable('phantomjs', ['-v'])
         if not self.exe:
-            raise ExtractorError('PhantomJS executable not found in PATH, '
-                                 'download it from http://phantomjs.org',
-                                 expected=True)
+            raise ExtractorError(
+                'PhantomJS not found, Please download it from https://phantomjs.org/download.html', expected=True)
 
         self.extractor = extractor
 

yt_dlp/extractor/philharmoniedeparis.py

@@ -1,9 +1,6 @@
 from .common import InfoExtractor
 from ..compat import compat_str
-from ..utils import (
-    try_get,
-    urljoin,
-)
+from ..utils import try_get
 
 
 class PhilharmonieDeParisIE(InfoExtractor):
@@ -12,27 +9,29 @@ class PhilharmonieDeParisIE(InfoExtractor):
                     https?://
                         (?:
                             live\.philharmoniedeparis\.fr/(?:[Cc]oncert/|embed(?:app)?/|misc/Playlist\.ashx\?id=)|
-                            pad\.philharmoniedeparis\.fr/doc/CIMU/
+                            pad\.philharmoniedeparis\.fr/(?:doc/CIMU/|player\.aspx\?id=)|
+                            philharmoniedeparis\.fr/fr/live/concert/|
+                            otoplayer\.philharmoniedeparis\.fr/fr/embed/
                         )
                         (?P<id>\d+)
                     '''
     _TESTS = [{
-        'url': 'http://pad.philharmoniedeparis.fr/doc/CIMU/1086697/jazz-a-la-villette-knower',
-        'md5': 'a0a4b195f544645073631cbec166a2c2',
+        'url': 'https://philharmoniedeparis.fr/fr/live/concert/1129666-danses-symphoniques',
+        'md5': '24bdb7e86c200c107680e1f7770330ae',
         'info_dict': {
-            'id': '1086697',
+            'id': '1129666',
             'ext': 'mp4',
-            'title': 'Jazz à la Villette : Knower',
+            'title': 'Danses symphoniques. Orchestre symphonique Divertimento - Zahia Ziouani. Bizet, de Falla, Stravinski, Moussorgski, Saint-Saëns',
         },
     }, {
-        'url': 'http://live.philharmoniedeparis.fr/concert/1032066.html',
+        'url': 'https://philharmoniedeparis.fr/fr/live/concert/1032066-akademie-fur-alte-musik-berlin-rias-kammerchor-rene-jacobs-passion-selon-saint-jean-de-johann',
         'info_dict': {
             'id': '1032066',
-            'title': 'md5:0a031b81807b3593cffa3c9a87a167a0',
+            'title': 'Akademie für alte Musik Berlin, Rias Kammerchor, René Jacobs : Passion selon saint Jean de Johann Sebastian Bach',
         },
         'playlist_mincount': 2,
     }, {
-        'url': 'http://live.philharmoniedeparis.fr/Concert/1030324.html',
+        'url': 'https://philharmoniedeparis.fr/fr/live/concert/1030324-orchestre-philharmonique-de-radio-france-myung-whun-chung-renaud-capucon-pascal-dusapin-johannes',
         'only_matching': True,
     }, {
         'url': 'http://live.philharmoniedeparis.fr/misc/Playlist.ashx?id=1030324&track=&lang=fr',
@@ -41,16 +40,15 @@ class PhilharmonieDeParisIE(InfoExtractor):
         'url': 'https://live.philharmoniedeparis.fr/embedapp/1098406/berlioz-fantastique-lelio-les-siecles-national-youth-choir-of.html?lang=fr-FR',
         'only_matching': True,
     }, {
-        'url': 'https://live.philharmoniedeparis.fr/embed/1098406/berlioz-fantastique-lelio-les-siecles-national-youth-choir-of.html?lang=fr-FR',
+        'url': 'https://otoplayer.philharmoniedeparis.fr/fr/embed/1098406?lang=fr-FR',
         'only_matching': True,
     }]
-    _LIVE_URL = 'https://live.philharmoniedeparis.fr'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         config = self._download_json(
-            '%s/otoPlayer/config.ashx' % self._LIVE_URL, video_id, query={
+            'https://otoplayer.philharmoniedeparis.fr/fr/config/%s.json' % video_id, video_id, query={
                 'id': video_id,
                 'lang': 'fr-FR',
             })
@@ -72,9 +70,8 @@ class PhilharmonieDeParisIE(InfoExtractor):
                 if not format_url or format_url in format_urls:
                     continue
                 format_urls.add(format_url)
-                m3u8_url = urljoin(self._LIVE_URL, format_url)
                 formats.extend(self._extract_m3u8_formats(
-                    m3u8_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    format_url, video_id, 'mp4', entry_protocol='m3u8_native',
                     m3u8_id='hls', fatal=False))
             if not formats and not self.get_param('ignore_no_formats'):
                 return
@@ -82,21 +79,19 @@ class PhilharmonieDeParisIE(InfoExtractor):
             return {
                 'title': title,
                 'formats': formats,
+                'thumbnail': files.get('thumbnail'),
             }
-
-        thumbnail = urljoin(self._LIVE_URL, config.get('image'))
-
         info = extract_entry(config)
         if info:
             info.update({
                 'id': video_id,
-                'thumbnail': thumbnail,
             })
             return info
-
         entries = []
         for num, chapter in enumerate(config['chapters'], start=1):
             entry = extract_entry(chapter)
+            if entry is None:
+                continue
             entry['id'] = '%s-%d' % (video_id, num)
             entries.append(entry)
 

yt_dlp/extractor/rtlnl.py

@@ -141,3 +141,155 @@ class RtlNlIE(InfoExtractor):
             'duration': parse_duration(material.get('duration')),
             'thumbnails': thumbnails,
         }
+
+
+class RTLLuBaseIE(InfoExtractor):
+    _MEDIA_REGEX = {
+        'video': r'<rtl-player\s[^>]*\bhls\s*=\s*"([^"]+)',
+        'audio': r'<rtl-audioplayer\s[^>]*\bsrc\s*=\s*"([^"]+)',
+        'thumbnail': r'<rtl-player\s[^>]*\bposter\s*=\s*"([^"]+)',
+    }
+
+    def get_media_url(self, webpage, video_id, media_type):
+        return self._search_regex(self._MEDIA_REGEX[media_type], webpage, f'{media_type} url', default=None)
+
+    def get_formats_and_subtitles(self, webpage, video_id):
+        video_url, audio_url = self.get_media_url(webpage, video_id, 'video'), self.get_media_url(webpage, video_id, 'audio')
+
+        formats, subtitles = [], {}
+        if video_url is not None:
+            formats, subtitles = self._extract_m3u8_formats_and_subtitles(video_url, video_id)
+        if audio_url is not None:
+            formats.append({'url': audio_url, 'ext': 'mp3', 'vcodec': 'none'})
+
+        return formats, subtitles
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        is_live = video_id in ('live', 'live-2', 'lauschteren')
+
+        # TODO: extract comment from https://www.rtl.lu/comments?status=1&order=desc&context=news|article|<video_id>
+        # we can context from <rtl-comments context=<context> in webpage
+        webpage = self._download_webpage(url, video_id)
+
+        formats, subtitles = self.get_formats_and_subtitles(webpage, video_id)
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage, default=None),
+            'formats': formats,
+            'subtitles': subtitles,
+            'thumbnail': self.get_media_url(webpage, video_id, 'thumbnail') or self._og_search_thumbnail(webpage, default=None),
+            'is_live': is_live,
+        }
+
+
+class RTLLuTeleVODIE(RTLLuBaseIE):
+    IE_NAME = 'rtl.lu:tele-vod'
+    _VALID_URL = r'https?://(?:www\.)?rtl\.lu/(tele/(?P<slug>[\w-]+)/v/|video/)(?P<id>\d+)(\.html)?'
+    _TESTS = [{
+        'url': 'https://www.rtl.lu/tele/de-journal-vun-der-tele/v/3266757.html',
+        'info_dict': {
+            'id': '3266757',
+            'title': 'Informatiounsversammlung Héichwaasser',
+            'ext': 'mp4',
+            'thumbnail': 'https://replay-assets.rtl.lu/2021/11/16/d3647fc4-470d-11ec-adc2-3a00abd6e90f_00008.jpg',
+            'description': 'md5:b1db974408cc858c9fd241812e4a2a14',
+        }
+    }, {
+        'url': 'https://www.rtl.lu/video/3295215',
+        'info_dict': {
+            'id': '3295215',
+            'title': 'Kulturassisen iwwer d\'Bestandsopnam vum Lëtzebuerger Konscht',
+            'ext': 'mp4',
+            'thumbnail': 'https://replay-assets.rtl.lu/2022/06/28/0000_3295215_0000.jpg',
+            'description': 'md5:85bcd4e0490aa6ec969d9bf16927437b',
+        }
+    }]
+
+
+class RTLLuArticleIE(RTLLuBaseIE):
+    IE_NAME = 'rtl.lu:article'
+    _VALID_URL = r'https?://(?:(www|5minutes|today)\.)rtl\.lu/(?:[\w-]+)/(?:[\w-]+)/a/(?P<id>\d+)\.html'
+    _TESTS = [{
+        # Audio-only
+        'url': 'https://www.rtl.lu/sport/news/a/1934360.html',
+        'info_dict': {
+            'id': '1934360',
+            'ext': 'mp3',
+            'thumbnail': 'https://static.rtl.lu/rtl2008.lu/nt/p/2022/06/28/19/e4b37d66ddf00bab4c45617b91a5bb9b.jpeg',
+            'description': 'md5:5eab4a2a911c1fff7efc1682a38f9ef7',
+            'title': 'md5:40aa85f135578fbd549d3c9370321f99',
+        }
+    }, {
+        # 5minutes
+        'url': 'https://5minutes.rtl.lu/espace-frontaliers/frontaliers-en-questions/a/1853173.html',
+        'info_dict': {
+            'id': '1853173',
+            'ext': 'mp4',
+            'description': 'md5:ac031da0740e997a5cf4633173634fee',
+            'title': 'md5:87e17722ed21af0f24be3243f4ec0c46',
+            'thumbnail': 'https://replay-assets.rtl.lu/2022/01/26/screenshot_20220126104933_3274749_12b249833469b0d6e4440a1dec83cdfa.jpg',
+        }
+    }, {
+        # today.lu
+        'url': 'https://today.rtl.lu/entertainment/news/a/1936203.html',
+        'info_dict': {
+            'id': '1936203',
+            'ext': 'mp4',
+            'title': 'Once Upon A Time...zu Lëtzebuerg: The Three Witches\' Tower',
+            'description': 'The witchy theme continues in the latest episode of Once Upon A Time...',
+            'thumbnail': 'https://replay-assets.rtl.lu/2022/07/02/screenshot_20220702122859_3290019_412dc5185951b7f6545a4039c8be9235.jpg',
+        }
+    }]
+
+
+class RTLLuLiveIE(RTLLuBaseIE):
+    _VALID_URL = r'https?://www\.rtl\.lu/(?:tele|radio)/(?P<id>live(?:-\d+)?|lauschteren)'
+    _TESTS = [{
+        # Tele:live
+        'url': 'https://www.rtl.lu/tele/live',
+        'info_dict': {
+            'id': 'live',
+            'ext': 'mp4',
+            'live_status': 'is_live',
+            'title': r're:RTL - Télé LIVE \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
+            'thumbnail': 'https://static.rtl.lu/livestream/channel1.jpg',
+        }
+    }, {
+        # Tele:live-2
+        'url': 'https://www.rtl.lu/tele/live-2',
+        'info_dict': {
+            'id': 'live-2',
+            'ext': 'mp4',
+            'live_status': 'is_live',
+            'title': r're:RTL - Télé LIVE \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
+            'thumbnail': 'https://static.rtl.lu/livestream/channel2.jpg',
+        }
+    }, {
+        # Radio:lauschteren
+        'url': 'https://www.rtl.lu/radio/lauschteren',
+        'info_dict': {
+            'id': 'lauschteren',
+            'ext': 'mp4',
+            'live_status': 'is_live',
+            'title': r're:RTL - Radio LIVE \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
+            'thumbnail': 'https://static.rtl.lu/livestream/rtlradiowebtv.jpg',
+        }
+    }]
+
+
+class RTLLuRadioIE(RTLLuBaseIE):
+    _VALID_URL = r'https?://www\.rtl\.lu/radio/(?:[\w-]+)/s/(?P<id>\d+)(\.html)?'
+    _TESTS = [{
+        'url': 'https://www.rtl.lu/radio/5-vir-12/s/4033058.html',
+        'info_dict': {
+            'id': '4033058',
+            'ext': 'mp3',
+            'description': 'md5:f855a4f3e3235393ae47ed1db5d934b9',
+            'title': '5 vir 12 - Stau um Stau',
+            'thumbnail': 'https://static.rtl.lu/rtlg//2022/06/24/c9c19e5694a14be46a3647a3760e1f62.jpg',
+        }
+    }]

yt_dlp/extractor/rtvslo.py

@@ -0,0 +1,151 @@
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    parse_duration,
+    traverse_obj,
+    unified_timestamp,
+    url_or_none,
+)
+
+
+class RTVSLOIE(InfoExtractor):
+    IE_NAME = 'rtvslo.si'
+    _VALID_URL = r'''(?x)
+        https?://(?:
+            (?:365|4d)\.rtvslo.si/arhiv/[^/?#&;]+|
+            (?:www\.)?rtvslo\.si/rtv365/arhiv
+        )/(?P<id>\d+)'''
+    _GEO_COUNTRIES = ['SI']
+
+    _API_BASE = 'https://api.rtvslo.si/ava/{}/{}?client_id=82013fb3a531d5414f478747c1aca622'
+    SUB_LANGS_MAP = {'Slovenski': 'sl'}
+
+    _TESTS = [
+        {
+            'url': 'https://www.rtvslo.si/rtv365/arhiv/174842550?s=tv',
+            'info_dict': {
+                'id': '174842550',
+                'ext': 'flv',
+                'release_timestamp': 1643140032,
+                'upload_date': '20220125',
+                'series': 'Dnevnik',
+                'thumbnail': 'https://img.rtvcdn.si/_up/ava/ava_misc/show_logos/92/dnevnik_3_wide2.jpg',
+                'description': 'md5:76a18692757aeb8f0f51221106277dd2',
+                'timestamp': 1643137046,
+                'title': 'Dnevnik',
+                'series_id': '92',
+                'release_date': '20220125',
+                'duration': 1789,
+            },
+        }, {
+            'url': 'https://365.rtvslo.si/arhiv/utrip/174843754',
+            'info_dict': {
+                'id': '174843754',
+                'ext': 'mp4',
+                'series_id': '94',
+                'release_date': '20220129',
+                'timestamp': 1643484455,
+                'title': 'Utrip',
+                'duration': 813,
+                'thumbnail': 'https://img.rtvcdn.si/_up/ava/ava_misc/show_logos/94/utrip_1_wide2.jpg',
+                'description': 'md5:77f2892630c7b17bb7a5bb84319020c9',
+                'release_timestamp': 1643485825,
+                'upload_date': '20220129',
+                'series': 'Utrip',
+            },
+        }, {
+            'url': 'https://365.rtvslo.si/arhiv/il-giornale-della-sera/174844609',
+            'info_dict': {
+                'id': '174844609',
+                'ext': 'mp3',
+                'series_id': '106615841',
+                'title': 'Il giornale della sera',
+                'duration': 1328,
+                'series': 'Il giornale della sera',
+                'timestamp': 1643743800,
+                'release_timestamp': 1643745424,
+                'thumbnail': 'https://img.rtvcdn.si/_up/ava/ava_misc/show_logos/il-giornale-della-sera_wide2.jpg',
+                'upload_date': '20220201',
+                'tbr': 128000,
+                'release_date': '20220201',
+            },
+
+        }, {
+            'url': 'https://4d.rtvslo.si/arhiv/dnevnik/174842550',
+            'only_matching': True
+        }
+    ]
+
+    def _real_extract(self, url):
+        v_id = self._match_id(url)
+        meta = self._download_json(self._API_BASE.format('getRecordingDrm', v_id), v_id)['response']
+
+        thumbs = [{'id': k, 'url': v, 'http_headers': {'Accept': 'image/jpeg'}}
+                  for k, v in (meta.get('images') or {}).items()]
+
+        subs = {}
+        for s in traverse_obj(meta, 'subs', 'subtitles', default=[]):
+            lang = self.SUB_LANGS_MAP.get(s.get('language'), s.get('language') or 'und')
+            subs.setdefault(lang, []).append({
+                'url': s.get('file'),
+                'ext': traverse_obj(s, 'format', expected_type=str.lower),
+            })
+
+        jwt = meta.get('jwt')
+        if not jwt:
+            raise ExtractorError('Site did not provide an authentication token, cannot proceed.')
+
+        media = self._download_json(self._API_BASE.format('getMedia', v_id), v_id, query={'jwt': jwt})['response']
+
+        formats = []
+        adaptive_url = traverse_obj(media, ('addaptiveMedia', 'hls_sec'), expected_type=url_or_none)
+        if adaptive_url:
+            formats = self._extract_wowza_formats(adaptive_url, v_id, skip_protocols=['smil'])
+
+        adaptive_url = traverse_obj(media, ('addaptiveMedia_sl', 'hls_sec'), expected_type=url_or_none)
+        if adaptive_url:
+            for f in self._extract_wowza_formats(adaptive_url, v_id, skip_protocols=['smil']):
+                formats.append({
+                    **f,
+                    'format_id': 'sign-' + f['format_id'],
+                    'format_note': 'Sign language interpretation', 'preference': -10,
+                    'language': (
+                        'slv' if f.get('language') == 'eng' and f.get('acodec') != 'none'
+                        else f.get('language'))
+                })
+
+        formats.extend(
+            {
+                'url': f['streams'][strm],
+                'ext': traverse_obj(f, 'mediaType', expected_type=str.lower),
+                'width': f.get('width'),
+                'height': f.get('height'),
+                'tbr': f.get('bitrate'),
+                'filesize': f.get('filesize'),
+            }
+            for strm in ('http', 'https')
+            for f in media.get('mediaFiles') or []
+            if traverse_obj(f, ('streams', strm))
+        )
+
+        if any('intermission.mp4' in x['url'] for x in formats):
+            self.raise_geo_restricted(countries=self._GEO_COUNTRIES, metadata_available=True)
+        if any('dummy_720p.mp4' in x.get('manifest_url', '') for x in formats) and meta.get('stub') == 'error':
+            raise ExtractorError(f'{self.IE_NAME} said: Clip not available', expected=True)
+
+        self._sort_formats(formats)
+        return {
+            'id': v_id,
+            'webpage_url': ''.join(traverse_obj(meta, ('canonical', ('domain', 'path')))),
+            'title': meta.get('title'),
+            'formats': formats,
+            'subtitles': subs,
+            'thumbnails': thumbs,
+            'description': meta.get('description'),
+            'timestamp': unified_timestamp(traverse_obj(meta, 'broadcastDate', ('broadcastDates', 0))),
+            'release_timestamp': unified_timestamp(meta.get('recordingDate')),
+            'duration': meta.get('duration') or parse_duration(meta.get('length')),
+            'tags': meta.get('genre'),
+            'series': meta.get('showName'),
+            'series_id': meta.get('showId'),
+        }

yt_dlp/extractor/startrek.py

@@ -0,0 +1,76 @@
+from .common import InfoExtractor
+from ..utils import int_or_none, urljoin
+
+
+class StarTrekIE(InfoExtractor):
+    _VALID_URL = r'(?P<base>https?://(?:intl|www)\.startrek\.com)/videos/(?P<id>[^/]+)'
+    _TESTS = [{
+        'url': 'https://intl.startrek.com/videos/watch-welcoming-jess-bush-to-the-ready-room',
+        'md5': '491df5035c9d4dc7f63c79caaf9c839e',
+        'info_dict': {
+            'id': 'watch-welcoming-jess-bush-to-the-ready-room',
+            'ext': 'mp4',
+            'title': 'WATCH: Welcoming Jess Bush to The Ready Room',
+            'duration': 1888,
+            'timestamp': 1655388000,
+            'upload_date': '20220616',
+            'description': 'md5:1ffee884e3920afbdd6dd04e926a1221',
+            'thumbnail': r're:https://(?:intl|www)\.startrek\.com/sites/default/files/styles/video_1920x1080/public/images/2022-06/pp_14794_rr_thumb_107_yt_16x9\.jpg(?:\?.+)?',
+            'subtitles': {'en-US': [{
+                'url': r're:https://(?:intl|www)\.startrek\.com/sites/default/files/video/captions/2022-06/TRR_SNW_107_v4\.vtt',
+            }, {
+                'url': 'https://media.startrek.com/2022/06/16/2043801155561/1069981_hls/trr_snw_107_v4-c4bfc25d/stream_vtt.m3u8',
+            }]},
+        }
+    }, {
+        'url': 'https://www.startrek.com/videos/watch-ethan-peck-and-gia-sandhu-beam-down-to-the-ready-room',
+        'md5': 'f5ad74fbb86e91e0882fc0a333178d1d',
+        'info_dict': {
+            'id': 'watch-ethan-peck-and-gia-sandhu-beam-down-to-the-ready-room',
+            'ext': 'mp4',
+            'title': 'WATCH: Ethan Peck and Gia Sandhu Beam Down to The Ready Room',
+            'duration': 1986,
+            'timestamp': 1654221600,
+            'upload_date': '20220603',
+            'description': 'md5:b3aa0edacfe119386567362dec8ed51b',
+            'thumbnail': r're:https://www\.startrek\.com/sites/default/files/styles/video_1920x1080/public/images/2022-06/pp_14792_rr_thumb_105_yt_16x9_1.jpg(?:\?.+)?',
+            'subtitles': {'en-US': [{
+                'url': r're:https://(?:intl|www)\.startrek\.com/sites/default/files/video/captions/2022-06/TRR_SNW_105_v5\.vtt',
+            }]},
+        }
+    }]
+
+    def _real_extract(self, url):
+        urlbase, video_id = self._match_valid_url(url).group('base', 'id')
+        webpage = self._download_webpage(url, video_id)
+
+        player = self._search_regex(
+            r'(<\s*div\s+id\s*=\s*"cvp-player-[^<]+<\s*/div\s*>)', webpage, 'player')
+
+        hls = self._html_search_regex(r'\bdata-hls\s*=\s*"([^"]+)"', player, 'HLS URL')
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(hls, video_id, 'mp4')
+        self._sort_formats(formats)
+
+        captions = self._html_search_regex(
+            r'\bdata-captions-url\s*=\s*"([^"]+)"', player, 'captions URL', fatal=False)
+        if captions:
+            subtitles.setdefault('en-US', [])[:0] = [{'url': urljoin(urlbase, captions)}]
+
+        # NB: Most of the data in the json_ld is undesirable
+        json_ld = self._search_json_ld(webpage, video_id, fatal=False)
+
+        return {
+            'id': video_id,
+            'title': self._html_search_regex(
+                r'\bdata-title\s*=\s*"([^"]+)"', player, 'title', json_ld.get('title')),
+            'description': self._html_search_regex(
+                r'(?s)<\s*div\s+class\s*=\s*"header-body"\s*>(.+?)<\s*/div\s*>',
+                webpage, 'description', fatal=False),
+            'duration': int_or_none(self._html_search_regex(
+                r'\bdata-duration\s*=\s*"(\d+)"', player, 'duration', fatal=False)),
+            'formats': formats,
+            'subtitles': subtitles,
+            'thumbnail': urljoin(urlbase, self._html_search_regex(
+                r'\bdata-poster-url\s*=\s*"([^"]+)"', player, 'thumbnail', fatal=False)),
+            'timestamp': json_ld.get('timestamp'),
+        }

yt_dlp/extractor/syvdk.py

@@ -0,0 +1,33 @@
+from .common import InfoExtractor
+from ..utils import traverse_obj
+
+
+class SYVDKIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?24syv\.dk/episode/(?P<id>[\w-]+)'
+
+    _TESTS = [{
+        'url': 'https://24syv.dk/episode/isabella-arendt-stiller-op-for-de-konservative-2',
+        'md5': '429ce5a423dd4b1e1d0bf3a569558089',
+        'info_dict': {
+            'id': '12215',
+            'display_id': 'isabella-arendt-stiller-op-for-de-konservative-2',
+            'ext': 'mp3',
+            'title': 'Isabella Arendt stiller op for De Konservative',
+            'description': 'md5:f5fa6a431813bf37284f3412ad7c6c06'
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        info_data = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['episodeDetails'][0]
+
+        return {
+            'id': str(info_data['id']),
+            'vcodec': 'none',
+            'ext': 'mp3',
+            'url': info_data['details']['enclosure'],
+            'display_id': video_id,
+            'title': traverse_obj(info_data, ('title', 'rendered')),
+            'description': traverse_obj(info_data, ('details', 'post_title')),
+        }

yt_dlp/extractor/theholetv.py

@@ -0,0 +1,36 @@
+from .common import InfoExtractor
+from ..utils import extract_attributes, remove_end
+
+
+class TheHoleTvIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?the-hole\.tv/episodes/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://the-hole.tv/episodes/gromkii-vopros-sergey-orlov',
+        'md5': 'fea6682f47786f3ae5a6cbd635ec4bf9',
+        'info_dict': {
+            'id': 'gromkii-vopros-sergey-orlov',
+            'ext': 'mp4',
+            'title': 'Сергей Орлов — Громкий вопрос',
+            'thumbnail': 'https://assets-cdn.the-hole.tv/images/t8gan4n6zn627e7wni11b2uemqts',
+            'description': 'md5:45741a9202331f995d9fb76996759379'
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        player_attrs = extract_attributes(self._search_regex(
+            r'(<div[^>]*\bdata-controller="player"[^>]*>)', webpage, 'video player'))
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+            player_attrs['data-player-source-value'], video_id, 'mp4')
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': remove_end(self._html_extract_title(webpage), ' — The Hole'),
+            'description': self._og_search_description(webpage),
+            'thumbnail': player_attrs.get('data-player-poster-value'),
+            'formats': formats,
+            'subtitles': subtitles
+        }

yt_dlp/extractor/trovo.py

@@ -43,7 +43,27 @@ class TrovoBaseIE(InfoExtractor):
 
 
 class TrovoIE(TrovoBaseIE):
-    _VALID_URL = TrovoBaseIE._VALID_URL_BASE + r'(?!(?:clip|video)/)(?P<id>[^/?&#]+)'
+    _VALID_URL = TrovoBaseIE._VALID_URL_BASE + r'(?:s/)?(?!(?:clip|video)/)(?P<id>(?!s/)[^/?&#]+(?![^#]+[?&]vid=))'
+    _TESTS = [{
+        'url': 'https://trovo.live/Exsl',
+        'only_matching': True,
+    }, {
+        'url': 'https://trovo.live/s/SkenonSLive/549759191497',
+        'only_matching': True,
+    }, {
+        'url': 'https://trovo.live/s/zijo987/208251706',
+        'info_dict': {
+            'id': '104125853_104125853_1656439572',
+            'ext': 'flv',
+            'uploader_url': 'https://trovo.live/zijo987',
+            'uploader_id': '104125853',
+            'thumbnail': 'https://livecover.trovo.live/screenshot/73846_104125853_104125853-2022-06-29-04-00-22-852x480.jpg',
+            'uploader': 'zijo987',
+            'title': '💥IGRAMO IGRICE UPADAJTE💥2500/5000 2022-06-28 22:01',
+            'live_status': 'is_live',
+        },
+        'skip': 'May not be live'
+    }]
 
     def _real_extract(self, url):
         username = self._match_id(url)
@@ -71,6 +91,7 @@ class TrovoIE(TrovoBaseIE):
                 'format_id': format_id,
                 'height': int_or_none(format_id[:-1]) if format_id else None,
                 'url': play_url,
+                'tbr': stream_info.get('bitrate'),
                 'http_headers': self._HEADERS,
             })
         self._sort_formats(formats)
@@ -87,7 +108,7 @@ class TrovoIE(TrovoBaseIE):
 
 
 class TrovoVodIE(TrovoBaseIE):
-    _VALID_URL = TrovoBaseIE._VALID_URL_BASE + r'(?:clip|video)/(?P<id>[^/?&#]+)'
+    _VALID_URL = TrovoBaseIE._VALID_URL_BASE + r'(?:clip|video|s)/(?:[^/]+/\d+[^#]*[?&]vid=)?(?P<id>(?<!/s/)[^/?&#]+)'
     _TESTS = [{
         'url': 'https://trovo.live/clip/lc-5285890818705062210?ltab=videos',
         'params': {'getcomments': True},
@@ -108,9 +129,30 @@ class TrovoVodIE(TrovoBaseIE):
             'uploader_url': 'https://trovo.live/OneTappedYou',
             'thumbnail': r're:^https?://.*\.jpg',
         },
+    }, {
+        'url': 'https://trovo.live/s/SkenonSLive/549759191497?vid=ltv-100829718_100829718_387702301737980280',
+        'info_dict': {
+            'id': 'ltv-100829718_100829718_387702301737980280',
+            'ext': 'mp4',
+            'timestamp': 1654909624,
+            'thumbnail': 'http://vod.trovo.live/1f09baf0vodtransger1301120758/ef9ea3f0387702301737980280/coverBySnapshot/coverBySnapshot_10_0.jpg',
+            'uploader_id': '100829718',
+            'uploader': 'SkenonSLive',
+            'title': 'Trovo u secanju, uz par modova i muzike :)',
+            'uploader_url': 'https://trovo.live/SkenonSLive',
+            'duration': 10830,
+            'view_count': int,
+            'like_count': int,
+            'upload_date': '20220611',
+            'comment_count': int,
+            'categories': ['Minecraft'],
+        }
     }, {
         'url': 'https://trovo.live/video/ltv-100095501_100095501_1609596043',
         'only_matching': True,
+    }, {
+        'url': 'https://trovo.live/s/SkenonSLive/549759191497?foo=bar&vid=ltv-100829718_100829718_387702301737980280',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

yt_dlp/extractor/tubetugraz.py

@@ -0,0 +1,234 @@
+from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    parse_resolution,
+    traverse_obj,
+    urlencode_postdata,
+    variadic,
+)
+
+
+class TubeTuGrazBaseIE(InfoExtractor):
+    _NETRC_MACHINE = 'tubetugraz'
+
+    _API_EPISODE = 'https://tube.tugraz.at/search/episode.json'
+    _FORMAT_TYPES = ('presentation', 'presenter')
+
+    def _perform_login(self, username, password):
+        urlh = self._request_webpage(
+            'https://tube.tugraz.at/Shibboleth.sso/Login?target=/paella/ui/index.html',
+            None, fatal=False, note='downloading login page', errnote='unable to fetch login page')
+        if not urlh:
+            return
+
+        urlh = self._request_webpage(
+            urlh.geturl(), None, fatal=False, headers={'referer': urlh.geturl()},
+            note='logging in', errnote='unable to log in', data=urlencode_postdata({
+                'lang': 'de',
+                '_eventId_proceed': '',
+                'j_username': username,
+                'j_password': password
+            }))
+
+        if urlh and urlh.geturl() != 'https://tube.tugraz.at/paella/ui/index.html':
+            self.report_warning('unable to login: incorrect password')
+
+    def _extract_episode(self, episode_info):
+        id = episode_info.get('id')
+        formats = list(self._extract_formats(
+            traverse_obj(episode_info, ('mediapackage', 'media', 'track')), id))
+        self._sort_formats(formats)
+
+        title = traverse_obj(episode_info, ('mediapackage', 'title'), 'dcTitle')
+        series_title = traverse_obj(episode_info, ('mediapackage', 'seriestitle'))
+        creator = ', '.join(variadic(traverse_obj(
+            episode_info, ('mediapackage', 'creators', 'creator'), 'dcCreator', default='')))
+        return {
+            'id': id,
+            'title': title,
+            'creator': creator or None,
+            'duration': traverse_obj(episode_info, ('mediapackage', 'duration'), 'dcExtent'),
+            'series': series_title,
+            'series_id': traverse_obj(episode_info, ('mediapackage', 'series'), 'dcIsPartOf'),
+            'episode': series_title and title,
+            'formats': formats
+        }
+
+    def _set_format_type(self, formats, type):
+        for f in formats:
+            f['format_note'] = type
+            if not type.startswith(self._FORMAT_TYPES[0]):
+                f['preference'] = -2
+        return formats
+
+    def _extract_formats(self, format_list, id):
+        has_hls, has_dash = False, False
+
+        for format_info in format_list or []:
+            url = traverse_obj(format_info, ('tags', 'url'), 'url')
+            if url is None:
+                continue
+
+            type = format_info.get('type') or 'unknown'
+            transport = (format_info.get('transport') or 'https').lower()
+
+            if transport == 'https':
+                formats = [{
+                    'url': url,
+                    'abr': float_or_none(traverse_obj(format_info, ('audio', 'bitrate')), 1000),
+                    'vbr': float_or_none(traverse_obj(format_info, ('video', 'bitrate')), 1000),
+                    'fps': traverse_obj(format_info, ('video', 'framerate')),
+                    **parse_resolution(traverse_obj(format_info, ('video', 'resolution'))),
+                }]
+            elif transport == 'hls':
+                has_hls, formats = True, self._extract_m3u8_formats(
+                    url, id, 'mp4', fatal=False, note=f'downloading {type} HLS manifest')
+            elif transport == 'dash':
+                has_dash, formats = True, self._extract_mpd_formats(
+                    url, id, fatal=False, note=f'downloading {type} DASH manifest')
+            else:
+                # RTMP, HDS, SMOOTH, and unknown formats
+                # - RTMP url fails on every tested entry until now
+                # - HDS url 404's on every tested entry until now
+                # - SMOOTH url 404's on every tested entry until now
+                continue
+
+            yield from self._set_format_type(formats, type)
+
+        # TODO: Add test for these
+        for type in self._FORMAT_TYPES:
+            if not has_hls:
+                hls_formats = self._extract_m3u8_formats(
+                    f'https://wowza.tugraz.at/matterhorn_engage/smil:engage-player_{id}_{type}.smil/playlist.m3u8',
+                    id, 'mp4', fatal=False, note=f'Downloading {type} HLS manifest', errnote=False) or []
+                yield from self._set_format_type(hls_formats, type)
+
+            if not has_dash:
+                dash_formats = self._extract_mpd_formats(
+                    f'https://wowza.tugraz.at/matterhorn_engage/smil:engage-player_{id}_{type}.smil/manifest_mpm4sav_mvlist.mpd',
+                    id, fatal=False, note=f'Downloading {type} DASH manifest', errnote=False)
+                yield from self._set_format_type(dash_formats, type)
+
+
+class TubeTuGrazIE(TubeTuGrazBaseIE):
+    IE_DESC = 'tube.tugraz.at'
+
+    _VALID_URL = r'''(?x)
+        https?://tube\.tugraz\.at/paella/ui/watch.html\?id=
+        (?P<id>[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12})
+    '''
+    _TESTS = [
+        {
+            'url': 'https://tube.tugraz.at/paella/ui/watch.html?id=f2634392-e40e-4ac7-9ddc-47764aa23d40',
+            'md5': 'a23a3d5c9aaca2b84932fdba66e17145',
+            'info_dict': {
+                'id': 'f2634392-e40e-4ac7-9ddc-47764aa23d40',
+                'ext': 'mp4',
+                'title': '#6 (23.11.2017)',
+                'episode': '#6 (23.11.2017)',
+                'series': '[INB03001UF] Einführung in die strukturierte Programmierung',
+                'creator': 'Safran C',
+                'duration': 3295818,
+                'series_id': 'b1192fff-2aa7-4bf0-a5cf-7b15c3bd3b34',
+            }
+        }, {
+            'url': 'https://tube.tugraz.at/paella/ui/watch.html?id=2df6d787-e56a-428d-8ef4-d57f07eef238',
+            'md5': 'de0d854a56bf7318d2b693fe1adb89a5',
+            'info_dict': {
+                'id': '2df6d787-e56a-428d-8ef4-d57f07eef238',
+                'title': 'TubeTuGraz video #2df6d787-e56a-428d-8ef4-d57f07eef238',
+                'ext': 'mp4',
+            },
+            'expected_warnings': ['Extractor failed to obtain "title"'],
+        }
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        episode_data = self._download_json(
+            self._API_EPISODE, video_id, query={'id': video_id, 'limit': 1}, note='Downloading episode metadata')
+
+        episode_info = traverse_obj(episode_data, ('search-results', 'result'), default={'id': video_id})
+        return self._extract_episode(episode_info)
+
+
+class TubeTuGrazSeriesIE(TubeTuGrazBaseIE):
+    _VALID_URL = r'''(?x)
+        https?://tube\.tugraz\.at/paella/ui/browse\.html\?series=
+        (?P<id>[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12})
+    '''
+    _TESTS = [{
+        'url': 'https://tube.tugraz.at/paella/ui/browse.html?series=0e6351b7-c372-491e-8a49-2c9b7e21c5a6',
+        'id': '0e6351b7-c372-491e-8a49-2c9b7e21c5a6',
+        'info_dict': {
+            'id': '0e6351b7-c372-491e-8a49-2c9b7e21c5a6',
+            'title': '[209351] Strassenwesen',
+        },
+        'playlist': [
+            {
+                'info_dict': {
+                    'id': 'ee17ce5d-34e2-48b7-a76a-fed148614e11',
+                    'series_id': '0e6351b7-c372-491e-8a49-2c9b7e21c5a6',
+                    'ext': 'mp4',
+                    'title': '#4 Detailprojekt',
+                    'episode': '#4 Detailprojekt',
+                    'series': '[209351] Strassenwesen',
+                    'creator': 'Neuhold R',
+                    'duration': 6127024,
+                }
+            },
+            {
+                'info_dict': {
+                    'id': '87350498-799a-44d3-863f-d1518a98b114',
+                    'series_id': '0e6351b7-c372-491e-8a49-2c9b7e21c5a6',
+                    'ext': 'mp4',
+                    'title': '#3 Generelles Projekt',
+                    'episode': '#3 Generelles Projekt',
+                    'series': '[209351] Strassenwesen',
+                    'creator': 'Neuhold R',
+                    'duration': 5374422,
+                }
+            },
+            {
+                'info_dict': {
+                    'id': '778599ea-489e-4189-9e05-3b4888e19bcd',
+                    'series_id': '0e6351b7-c372-491e-8a49-2c9b7e21c5a6',
+                    'ext': 'mp4',
+                    'title': '#2 Vorprojekt',
+                    'episode': '#2 Vorprojekt',
+                    'series': '[209351] Strassenwesen',
+                    'creator': 'Neuhold R',
+                    'duration': 5566404,
+                }
+            },
+            {
+                'info_dict': {
+                    'id': '75e4c71c-d99d-4e56-b0e6-4f2bcdf11f29',
+                    'series_id': '0e6351b7-c372-491e-8a49-2c9b7e21c5a6',
+                    'ext': 'mp4',
+                    'title': '#1 Variantenstudium',
+                    'episode': '#1 Variantenstudium',
+                    'series': '[209351] Strassenwesen',
+                    'creator': 'Neuhold R',
+                    'duration': 5420200,
+                }
+            }
+        ],
+        'min_playlist_count': 4
+    }]
+
+    def _real_extract(self, url):
+        id = self._match_id(url)
+        episodes_data = self._download_json(self._API_EPISODE, id, query={'sid': id}, note='Downloading episode list')
+        series_data = self._download_json(
+            'https://tube.tugraz.at/series/series.json', id, fatal=False,
+            note='downloading series metadata', errnote='failed to download series metadata',
+            query={
+                'seriesId': id,
+                'count': 1,
+                'sort': 'TITLE'
+            })
+
+        return self.playlist_result(
+            map(self._extract_episode, episodes_data['search-results']['result']), id,
+            traverse_obj(series_data, ('catalogs', 0, 'http://purl.org/dc/terms/', 'title', 0, 'value')))

yt_dlp/extractor/tviplayer.py

@@ -0,0 +1,65 @@
+from .common import InfoExtractor
+from ..utils import traverse_obj
+
+
+class TVIPlayerIE(InfoExtractor):
+    _VALID_URL = r'https?://tviplayer\.iol\.pt(/programa/[\w-]+/[a-f0-9]+)?/video/(?P<id>[a-f0-9]+)'
+    _TESTS = [{
+        'url': 'https://tviplayer.iol.pt/programa/jornal-das-8/53c6b3903004dc006243d0cf/video/61c8e8b90cf2c7ea0f0f71a9',
+        'info_dict': {
+            'id': '61c8e8b90cf2c7ea0f0f71a9',
+            'ext': 'mp4',
+            'duration': 4167,
+            'title': 'Jornal das 8 - 26 de dezembro de 2021',
+            'thumbnail': 'https://www.iol.pt/multimedia/oratvi/multimedia/imagem/id/61c8ee630cf2cc58e7d98d9f/',
+            'season_number': 8,
+            'season': 'Season 8',
+        }
+    }, {
+        'url': 'https://tviplayer.iol.pt/programa/isabel/62b471090cf26256cd2a8594/video/62be445f0cf2ea4f0a5218e5',
+        'info_dict': {
+            'id': '62be445f0cf2ea4f0a5218e5',
+            'ext': 'mp4',
+            'duration': 3255,
+            'season': 'Season 1',
+            'title': 'Isabel - Episódio 1',
+            'thumbnail': 'https://www.iol.pt/multimedia/oratvi/multimedia/imagem/id/62beac200cf2f9a86eab856b/',
+            'season_number': 1,
+        }
+    }, {
+        'url': 'https://tviplayer.iol.pt/video/62c4131c0cf2f9a86eac06bb',
+        'info_dict': {
+            'id': '62c4131c0cf2f9a86eac06bb',
+            'ext': 'mp4',
+            'title': 'David e Mickael Carreira respondem: «Qual é o próximo a ser pai?»',
+            'thumbnail': 'https://www.iol.pt/multimedia/oratvi/multimedia/imagem/id/62c416490cf2ea367d4433fd/',
+            'season': 'Season 2',
+            'duration': 148,
+            'season_number': 2,
+        }
+    }]
+
+    def _real_initialize(self):
+        self.wms_auth_sign_token = self._download_webpage(
+            'https://services.iol.pt/matrix?userId=', 'wmsAuthSign',
+            note='Trying to get wmsAuthSign token')
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        json_data = self._search_json(
+            r'<script>\s*jsonData\s*=\s*', webpage, 'json_data', video_id)
+
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
+            f'{json_data["videoUrl"]}?wmsAuthSign={self.wms_auth_sign_token}',
+            video_id, ext='mp4')
+        return {
+            'id': video_id,
+            'title': json_data.get('title') or self._og_search_title(webpage),
+            'thumbnail': json_data.get('cover') or self._og_search_thumbnail(webpage),
+            'duration': json_data.get('duration'),
+            'formats': formats,
+            'subtitles': subtitles,
+            'season_number': traverse_obj(json_data, ('program', 'seasonNum')),
+        }

yt_dlp/extractor/twitch.py

@@ -12,6 +12,7 @@ from ..compat import (
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
+    base_url,
     clean_html,
     dict_get,
     ExtractorError,
@@ -52,6 +53,7 @@ class TwitchBaseIE(InfoExtractor):
         'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c',
         'VideoMetadata': '226edb3e692509f727fd56821f5653c05740242c82b0388883e0c0e75dcbf687',
         'VideoPlayer_ChapterSelectButtonVideo': '8d2793384aac3773beab5e59bd5d6f585aedb923d292800119e03d40cd0f9b41',
+        'VideoPlayer_VODSeekbarPreviewVideo': '07e99e4d56c5a7c67117a154777b0baf85a5ffefa393b213f4bc712ccaf85dd6',
     }
 
     def _perform_login(self, username, password):
@@ -202,6 +204,8 @@ class TwitchVodIE(TwitchBaseIE):
             'uploader_id': 'riotgames',
             'view_count': int,
             'start_time': 310,
+            'chapters': [],
+            'live_status': 'was_live',
         },
         'params': {
             # m3u8 download
@@ -270,9 +274,52 @@ class TwitchVodIE(TwitchBaseIE):
                     'title': 'Art'
                 }
             ],
+            'live_status': 'was_live',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'view_count': int,
         },
         'params': {
             'skip_download': True
+        },
+    }, {
+        'note': 'Storyboards',
+        'url': 'https://www.twitch.tv/videos/635475444',
+        'info_dict': {
+            'id': 'v635475444',
+            'format_id': 'sb0',
+            'ext': 'mhtml',
+            'title': 'Riot Games',
+            'duration': 11643,
+            'uploader': 'Riot Games',
+            'uploader_id': 'riotgames',
+            'timestamp': 1590770569,
+            'upload_date': '20200529',
+            'chapters': [
+                {
+                    'start_time': 0,
+                    'end_time': 573,
+                    'title': 'League of Legends'
+                },
+                {
+                    'start_time': 573,
+                    'end_time': 3922,
+                    'title': 'Legends of Runeterra'
+                },
+                {
+                    'start_time': 3922,
+                    'end_time': 11643,
+                    'title': 'Art'
+                }
+            ],
+            'live_status': 'was_live',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'view_count': int,
+            'columns': int,
+            'rows': int,
+        },
+        'params': {
+            'format': 'mhtml',
+            'skip_download': True
         }
     }]
 
@@ -290,16 +337,23 @@ class TwitchVodIE(TwitchBaseIE):
                     'includePrivate': False,
                     'videoID': item_id,
                 },
+            }, {
+                'operationName': 'VideoPlayer_VODSeekbarPreviewVideo',
+                'variables': {
+                    'includePrivate': False,
+                    'videoID': item_id,
+                },
             }],
             'Downloading stream metadata GraphQL')
 
         video = traverse_obj(data, (0, 'data', 'video'))
         video['moments'] = traverse_obj(data, (1, 'data', 'video', 'moments', 'edges', ..., 'node'))
+        video['storyboard'] = traverse_obj(data, (2, 'data', 'video', 'seekPreviewsURL'), expected_type=url_or_none)
 
         if video is None:
             raise ExtractorError(
                 'Video %s does not exist' % item_id, expected=True)
-        return self._extract_info_gql(video, item_id)
+        return video
 
     def _extract_info(self, info):
         status = info.get('status')
@@ -383,10 +437,44 @@ class TwitchVodIE(TwitchBaseIE):
             'was_live': True,
         }
 
+    def _extract_storyboard(self, item_id, storyboard_json_url, duration):
+        if not duration or not storyboard_json_url:
+            return
+        spec = self._download_json(storyboard_json_url, item_id, 'Downloading storyboard metadata JSON', fatal=False) or []
+        # sort from highest quality to lowest
+        # This makes sb0 the highest-quality format, sb1 - lower, etc which is consistent with youtube sb ordering
+        spec.sort(key=lambda x: int_or_none(x.get('width')) or 0, reverse=True)
+        base = base_url(storyboard_json_url)
+        for i, s in enumerate(spec):
+            count = int_or_none(s.get('count'))
+            images = s.get('images')
+            if not (images and count):
+                continue
+            fragment_duration = duration / len(images)
+            yield {
+                'format_id': f'sb{i}',
+                'format_note': 'storyboard',
+                'ext': 'mhtml',
+                'protocol': 'mhtml',
+                'acodec': 'none',
+                'vcodec': 'none',
+                'url': urljoin(base, images[0]),
+                'width': int_or_none(s.get('width')),
+                'height': int_or_none(s.get('height')),
+                'fps': count / duration,
+                'rows': int_or_none(s.get('rows')),
+                'columns': int_or_none(s.get('cols')),
+                'fragments': [{
+                    'url': urljoin(base, path),
+                    'duration': fragment_duration,
+                } for path in images],
+            }
+
     def _real_extract(self, url):
         vod_id = self._match_id(url)
 
-        info = self._download_info(vod_id)
+        video = self._download_info(vod_id)
+        info = self._extract_info_gql(video, vod_id)
         access_token = self._download_access_token(vod_id, 'video', 'id')
 
         formats = self._extract_m3u8_formats(
@@ -403,6 +491,8 @@ class TwitchVodIE(TwitchBaseIE):
                 })),
             vod_id, 'mp4', entry_protocol='m3u8_native')
 
+        formats.extend(self._extract_storyboard(vod_id, video.get('storyboard'), info.get('duration')))
+
         self._prefer_source(formats)
         info['formats'] = formats
 

yt_dlp/extractor/wetv.py

@@ -0,0 +1,208 @@
+import functools
+import re
+import time
+
+from .common import InfoExtractor
+from ..aes import aes_cbc_encrypt_bytes
+from ..utils import determine_ext, int_or_none, traverse_obj, urljoin
+
+
+class WeTvBaseIE(InfoExtractor):
+    _VALID_URL_BASE = r'https?://(?:www\.)?wetv\.vip/(?:[^?#]+/)?play'
+
+    def _get_ckey(self, video_id, url, app_version, platform):
+        ua = self.get_param('http_headers')['User-Agent']
+
+        payload = (f'{video_id}|{int(time.time())}|mg3c3b04ba|{app_version}|0000000000000000|'
+                   f'{platform}|{url[:48]}|{ua.lower()[:48]}||Mozilla|Netscape|Win32|00|')
+
+        return aes_cbc_encrypt_bytes(
+            bytes(f'|{sum(map(ord, payload))}|{payload}', 'utf-8'),
+            b'Ok\xda\xa3\x9e/\x8c\xb0\x7f^r-\x9e\xde\xf3\x14',
+            b'\x01PJ\xf3V\xe6\x19\xcf.B\xbb\xa6\x8c?p\xf9',
+            padding_mode='whitespace').hex()
+
+    def _get_video_api_response(self, video_url, video_id, series_id, subtitle_format, video_format, video_quality):
+        app_version = '3.5.57'
+        platform = '4830201'
+
+        ckey = self._get_ckey(video_id, video_url, app_version, platform)
+        query = {
+            'vid': video_id,
+            'cid': series_id,
+            'cKey': ckey,
+            'encryptVer': '8.1',
+            'spcaptiontype': '1' if subtitle_format == 'vtt' else '0',  # 0 - SRT, 1 - VTT
+            'sphls': '1' if video_format == 'hls' else '0',  # 0 - MP4, 1 - HLS
+            'defn': video_quality,  # '': 480p, 'shd': 720p, 'fhd': 1080p
+            'spsrt': '1',  # Enable subtitles
+            'sphttps': '1',  # Enable HTTPS
+            'otype': 'json',  # Response format: xml, json,
+            'dtype': '1',
+            'spwm': '1',
+            'host': 'wetv.vip',  # These three values are needed for SHD
+            'referer': 'wetv.vip',
+            'ehost': video_url,
+            'appVer': app_version,
+            'platform': platform,
+        }
+
+        return self._search_json(r'QZOutputJson=', self._download_webpage(
+            'https://play.wetv.vip/getvinfo', video_id, query=query), 'api_response', video_id)
+
+    def _get_webpage_metadata(self, webpage, video_id):
+        return self._parse_json(
+            traverse_obj(self._search_nextjs_data(webpage, video_id), ('props', 'pageProps', 'data')),
+            video_id, fatal=False)
+
+
+class WeTvEpisodeIE(WeTvBaseIE):
+    IE_NAME = 'wetv:episode'
+    _VALID_URL = WeTvBaseIE._VALID_URL_BASE + r'/(?P<series_id>\w+)(?:-[^?#]+)?/(?P<id>\w+)(?:-[^?#]+)?'
+
+    _TESTS = [{
+        'url': 'https://wetv.vip/en/play/air11ooo2rdsdi3-Cute-Programmer/v0040pr89t9-EP1-Cute-Programmer',
+        'md5': 'a046f565c9dce9b263a0465a422cd7bf',
+        'info_dict': {
+            'id': 'v0040pr89t9',
+            'ext': 'mp4',
+            'title': 'EP1: Cute Programmer',
+            'description': 'md5:e87beab3bf9f392d6b9e541a63286343',
+            'thumbnail': r're:^https?://[^?#]+air11ooo2rdsdi3',
+            'series': 'Cute Programmer',
+            'episode': 'Episode 1',
+            'episode_number': 1,
+            'duration': 2835,
+        },
+    }, {
+        'url': 'https://wetv.vip/en/play/u37kgfnfzs73kiu/p0039b9nvik',
+        'md5': '4d9d69bcfd11da61f4aae64fc6b316b3',
+        'info_dict': {
+            'id': 'p0039b9nvik',
+            'ext': 'mp4',
+            'title': 'EP1: You Are My Glory',
+            'description': 'md5:831363a4c3b4d7615e1f3854be3a123b',
+            'thumbnail': r're:^https?://[^?#]+u37kgfnfzs73kiu',
+            'series': 'You Are My Glory',
+            'episode': 'Episode 1',
+            'episode_number': 1,
+            'duration': 2454,
+        },
+    }, {
+        'url': 'https://wetv.vip/en/play/lcxgwod5hapghvw-WeTV-PICK-A-BOO/i0042y00lxp-Zhao-Lusi-Describes-The-First-Experiences-She-Had-In-Who-Rules-The-World-%7C-WeTV-PICK-A-BOO',
+        'md5': '71133f5c2d5d6cad3427e1b010488280',
+        'info_dict': {
+            'id': 'i0042y00lxp',
+            'ext': 'mp4',
+            'title': 'md5:f7a0857dbe5fbbe2e7ad630b92b54e6a',
+            'description': 'md5:76260cb9cdc0ef76826d7ca9d92fadfa',
+            'thumbnail': r're:^https?://[^?#]+lcxgwod5hapghvw',
+            'series': 'WeTV PICK-A-BOO',
+            'episode': 'Episode 0',
+            'episode_number': 0,
+            'duration': 442,
+        },
+    }]
+
+    def _extract_video_formats_and_subtitles(self, api_response, video_id, video_quality):
+        video_response = api_response['vl']['vi'][0]
+        video_width = video_response.get('vw')
+        video_height = video_response.get('vh')
+
+        formats, subtitles = [], {}
+        for video_format in video_response['ul']['ui']:
+            if video_format.get('hls'):
+                fmts, subs = self._extract_m3u8_formats_and_subtitles(
+                    video_format['url'] + video_format['hls']['pname'], video_id, 'mp4', fatal=False)
+                for f in fmts:
+                    f['width'] = video_width
+                    f['height'] = video_height
+
+                formats.extend(fmts)
+                self._merge_subtitles(subs, target=subtitles)
+            else:
+                formats.append({
+                    'url': f'{video_format["url"]}{video_response["fn"]}?vkey={video_response["fvkey"]}',
+                    'width': video_width,
+                    'height': video_height,
+                    'ext': 'mp4',
+                })
+
+        return formats, subtitles
+
+    def _extract_video_subtitles(self, api_response, subtitles_format):
+        subtitles = {}
+        for subtitle in traverse_obj(api_response, ('sfl', 'fi')):
+            subtitles.setdefault(subtitle['lang'].lower(), []).append({
+                'url': subtitle['url'],
+                'ext': subtitles_format,
+                'protocol': 'm3u8_native' if determine_ext(subtitle['url']) == 'm3u8' else 'http',
+            })
+
+        return subtitles
+
+    def _real_extract(self, url):
+        video_id, series_id = self._match_valid_url(url).group('id', 'series_id')
+        webpage = self._download_webpage(url, video_id)
+
+        formats, subtitles = [], {}
+        for video_format, subtitle_format, video_quality in (('mp4', 'srt', ''), ('hls', 'vtt', 'shd'), ('hls', 'vtt', 'fhd')):
+            api_response = self._get_video_api_response(url, video_id, series_id, subtitle_format, video_format, video_quality)
+
+            fmts, subs = self._extract_video_formats_and_subtitles(api_response, video_id, video_quality)
+            native_subtitles = self._extract_video_subtitles(api_response, subtitle_format)
+
+            formats.extend(fmts)
+            self._merge_subtitles(subs, native_subtitles, target=subtitles)
+
+        self._sort_formats(formats)
+        webpage_metadata = self._get_webpage_metadata(webpage, video_id)
+
+        return {
+            'id': video_id,
+            'title': (self._og_search_title(webpage)
+                      or traverse_obj(webpage_metadata, ('coverInfo', 'description'))),
+            'description': (self._og_search_description(webpage)
+                            or traverse_obj(webpage_metadata, ('coverInfo', 'description'))),
+            'formats': formats,
+            'subtitles': subtitles,
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'duration': int_or_none(traverse_obj(webpage_metadata, ('videoInfo', 'duration'))),
+            'series': traverse_obj(webpage_metadata, ('coverInfo', 'title')),
+            'episode_number': int_or_none(traverse_obj(webpage_metadata, ('videoInfo', 'episode'))),
+        }
+
+
+class WeTvSeriesIE(WeTvBaseIE):
+    _VALID_URL = WeTvBaseIE._VALID_URL_BASE + r'/(?P<id>\w+)(?:-[^/?#]+)?/?(?:[?#]|$)'
+
+    _TESTS = [{
+        'url': 'https://wetv.vip/play/air11ooo2rdsdi3-Cute-Programmer',
+        'info_dict': {
+            'id': 'air11ooo2rdsdi3',
+            'title': 'Cute Programmer',
+            'description': 'md5:e87beab3bf9f392d6b9e541a63286343',
+        },
+        'playlist_count': 30,
+    }, {
+        'url': 'https://wetv.vip/en/play/u37kgfnfzs73kiu-You-Are-My-Glory',
+        'info_dict': {
+            'id': 'u37kgfnfzs73kiu',
+            'title': 'You Are My Glory',
+            'description': 'md5:831363a4c3b4d7615e1f3854be3a123b',
+        },
+        'playlist_count': 32,
+    }]
+
+    def _real_extract(self, url):
+        series_id = self._match_id(url)
+        webpage = self._download_webpage(url, series_id)
+        webpage_metadata = self._get_webpage_metadata(webpage, series_id)
+
+        episode_paths = (re.findall(r'<a[^>]+class="play-video__link"[^>]+href="(?P<path>[^"]+)', webpage)
+                         or [f'/{series_id}/{episode["vid"]}' for episode in webpage_metadata.get('videoList')])
+
+        return self.playlist_from_matches(
+            episode_paths, series_id, ie=WeTvEpisodeIE, getter=functools.partial(urljoin, url),
+            title=traverse_obj(webpage_metadata, ('coverInfo', 'title')) or self._og_search_title(webpage),
+            description=traverse_obj(webpage_metadata, ('coverInfo', 'description')) or self._og_search_description(webpage))

yt_dlp/extractor/wikimedia.py

@@ -0,0 +1,55 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    get_element_by_class,
+    parse_qs,
+    remove_start,
+    unescapeHTML,
+    urljoin,
+)
+
+
+class WikimediaIE(InfoExtractor):
+    IE_NAME = 'wikimedia.org'
+    _VALID_URL = r'https?://commons\.wikimedia\.org/wiki/File:(?P<id>[^/#?]+)\.\w+'
+    _TESTS = [{
+        'url': 'https://commons.wikimedia.org/wiki/File:Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS.webm',
+        'info_dict': {
+            'url': 're:https?://upload.wikimedia.org/wikipedia',
+            'ext': 'webm',
+            'id': 'Die_Temperaturkurve_der_Erde_(ZDF,_Terra_X)_720p_HD_50FPS',
+            'title': 'Die Temperaturkurve der Erde (ZDF, Terra X) 720p HD 50FPS.webm - Wikimedia Commons',
+            'description': 'md5:7cd84f76e7081f1be033d0b155b4a460',
+            'license': 'Creative Commons Attribution 4.0 International',
+            'uploader': 'ZDF/Terra X/Gruppe 5/Luise Wagner, Jonas Sichert, Andreas Hougardy',
+            'subtitles': 'count:4'
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        subtitles = {}
+        for sub in set(re.findall(r'\bsrc\s*=\s*["\'](/w/api[^"]+)["\']', webpage)):
+            sub = urljoin('https://commons.wikimedia.org', unescapeHTML(sub))
+            qs = parse_qs(sub)
+            lang = qs.get('lang', [None])[-1]
+            sub_ext = qs.get('trackformat', [None])[-1]
+            if lang and sub_ext:
+                subtitles.setdefault(lang, []).append({'ext': sub_ext, 'url': sub})
+
+        return {
+            'id': video_id,
+            'url': self._html_search_regex(r'<source\s[^>]*\bsrc="([^"]+)"', webpage, 'video URL'),
+            'description': clean_html(get_element_by_class('description', webpage)),
+            'title': remove_start(self._og_search_title(webpage), 'File:'),
+            'license': self._html_search_regex(
+                r'licensed under(?: the)? (.+?) license',
+                get_element_by_class('licensetpl', webpage), 'license', default=None),
+            'uploader': self._html_search_regex(
+                r'>\s*Author\s*</td>\s*<td\b[^>]*>\s*([^<]+)\s*</td>', webpage, 'video author', default=None),
+            'subtitles': subtitles,
+        }

yt_dlp/extractor/wsj.py

@@ -116,5 +116,6 @@ class WSJArticleIE(InfoExtractor):
         article_id = self._match_id(url)
         webpage = self._download_webpage(url, article_id)
         video_id = self._search_regex(
-            r'data-src=["\']([a-fA-F0-9-]{36})', webpage, 'video id')
+            r'(?:id=["\']video|video-|iframe\.html\?guid=|data-src=["\'])([a-fA-F0-9-]{36})',
+            webpage, 'video id')
         return self.url_result('wsj:%s' % video_id, WSJIE.ie_key(), video_id)

yt_dlp/extractor/ximalaya.py

@@ -1,7 +1,7 @@
-import itertools
-import re
+import math
 
 from .common import InfoExtractor
+from ..utils import traverse_obj, try_call, InAdvancePagedList
 
 
 class XimalayaBaseIE(InfoExtractor):
@@ -11,11 +11,10 @@ class XimalayaBaseIE(InfoExtractor):
 class XimalayaIE(XimalayaBaseIE):
     IE_NAME = 'ximalaya'
     IE_DESC = '喜马拉雅FM'
-    _VALID_URL = r'https?://(?:www\.|m\.)?ximalaya\.com/(?P<uid>[0-9]+)/sound/(?P<id>[0-9]+)'
-    _USER_URL_FORMAT = '%s://www.ximalaya.com/zhubo/%i/'
+    _VALID_URL = r'https?://(?:www\.|m\.)?ximalaya\.com/(:?(?P<uid>\d+)/)?sound/(?P<id>[0-9]+)'
     _TESTS = [
         {
-            'url': 'http://www.ximalaya.com/61425525/sound/47740352/',
+            'url': 'http://www.ximalaya.com/sound/47740352/',
             'info_dict': {
                 'id': '47740352',
                 'ext': 'm4a',
@@ -24,19 +23,20 @@ class XimalayaIE(XimalayaBaseIE):
                 'uploader_url': 'http://www.ximalaya.com/zhubo/61425525/',
                 'title': '261.唐诗三百首.卷八.送孟浩然之广陵.李白',
                 'description': "contains:《送孟浩然之广陵》\n作者：李白\n故人西辞黄鹤楼，烟花三月下扬州。\n孤帆远影碧空尽，惟见长江天际流。",
+                'thumbnail': r're:^https?://.*\.jpg',
                 'thumbnails': [
                     {
                         'name': 'cover_url',
-                        'url': r're:^https?://.*\.jpg$',
+                        'url': r're:^https?://.*\.jpg',
                     },
                     {
                         'name': 'cover_url_142',
-                        'url': r're:^https?://.*\.jpg$',
+                        'url': r're:^https?://.*\.jpg',
                         'width': 180,
                         'height': 180
                     }
                 ],
-                'categories': ['renwen', '人文'],
+                'categories': ['人文'],
                 'duration': 93,
                 'view_count': int,
                 'like_count': int,
@@ -52,77 +52,42 @@ class XimalayaIE(XimalayaBaseIE):
                 'uploader_url': 'http://www.ximalaya.com/zhubo/61425525/',
                 'title': '261.唐诗三百首.卷八.送孟浩然之广陵.李白',
                 'description': "contains:《送孟浩然之广陵》\n作者：李白\n故人西辞黄鹤楼，烟花三月下扬州。\n孤帆远影碧空尽，惟见长江天际流。",
+                'thumbnail': r're:^https?://.*\.jpg',
                 'thumbnails': [
                     {
                         'name': 'cover_url',
-                        'url': r're:^https?://.*\.jpg$',
+                        'url': r're:^https?://.*\.jpg',
                     },
                     {
                         'name': 'cover_url_142',
-                        'url': r're:^https?://.*\.jpg$',
+                        'url': r're:^https?://.*\.jpg',
                         'width': 180,
                         'height': 180
                     }
                 ],
-                'categories': ['renwen', '人文'],
+                'categories': ['人文'],
                 'duration': 93,
                 'view_count': int,
                 'like_count': int,
             }
-        },
-        {
-            'url': 'https://www.ximalaya.com/11045267/sound/15705996/',
-            'info_dict': {
-                'id': '15705996',
-                'ext': 'm4a',
-                'uploader': '李延隆老师',
-                'uploader_id': 11045267,
-                'uploader_url': 'https://www.ximalaya.com/zhubo/11045267/',
-                'title': 'Lesson 1 Excuse me!',
-                'description': "contains:Listen to the tape then answer\xa0this question. Whose handbag is it?\n"
-                               "听录音，然后回答问题，这是谁的手袋？",
-                'thumbnails': [
-                    {
-                        'name': 'cover_url',
-                        'url': r're:^https?://.*\.jpg$',
-                    },
-                    {
-                        'name': 'cover_url_142',
-                        'url': r're:^https?://.*\.jpg$',
-                        'width': 180,
-                        'height': 180
-                    }
-                ],
-                'categories': ['train', '外语'],
-                'duration': 40,
-                'view_count': int,
-                'like_count': int,
-            }
-        },
+        }
     ]
 
     def _real_extract(self, url):
-
-        is_m = 'm.ximalaya' in url
         scheme = 'https' if url.startswith('https') else 'http'
 
         audio_id = self._match_id(url)
-        webpage = self._download_webpage(url, audio_id,
-                                         note='Download sound page for %s' % audio_id,
-                                         errnote='Unable to get sound page')
-
         audio_info_file = '%s://m.ximalaya.com/tracks/%s.json' % (scheme, audio_id)
         audio_info = self._download_json(audio_info_file, audio_id,
                                          'Downloading info json %s' % audio_info_file,
                                          'Unable to download info file')
 
-        formats = []
-        for bps, k in (('24k', 'play_path_32'), ('64k', 'play_path_64')):
-            if audio_info.get(k):
-                formats.append({
-                    'format_id': bps,
-                    'url': audio_info[k],
-                })
+        formats = [{
+            'format_id': f'{bps}k',
+            'url': audio_info[k],
+            'abr': bps,
+            'vcodec': 'none'
+        } for bps, k in ((24, 'play_path_32'), (64, 'play_path_64')) if audio_info.get(k)]
 
         thumbnails = []
         for k in audio_info.keys():
@@ -136,30 +101,18 @@ class XimalayaIE(XimalayaBaseIE):
 
         audio_uploader_id = audio_info.get('uid')
 
-        if is_m:
-            audio_description = self._html_search_regex(r'(?s)<section\s+class=["\']content[^>]+>(.+?)</section>',
-                                                        webpage, 'audio_description', fatal=False)
-        else:
-            audio_description = self._html_search_regex(r'(?s)<div\s+class=["\']rich_intro[^>]*>(.+?</article>)',
-                                                        webpage, 'audio_description', fatal=False)
-
-        if not audio_description:
-            audio_description_file = '%s://www.ximalaya.com/sounds/%s/rich_intro' % (scheme, audio_id)
-            audio_description = self._download_webpage(audio_description_file, audio_id,
-                                                       note='Downloading description file %s' % audio_description_file,
-                                                       errnote='Unable to download descrip file',
-                                                       fatal=False)
-            audio_description = audio_description.strip() if audio_description else None
+        audio_description = try_call(
+            lambda: audio_info['intro'].replace('\r\n\r\n\r\n ', '\n').replace('\r\n', '\n'))
 
         return {
             'id': audio_id,
             'uploader': audio_info.get('nickname'),
             'uploader_id': audio_uploader_id,
-            'uploader_url': self._USER_URL_FORMAT % (scheme, audio_uploader_id) if audio_uploader_id else None,
+            'uploader_url': f'{scheme}://www.ximalaya.com/zhubo/{audio_uploader_id}/' if audio_uploader_id else None,
             'title': audio_info['title'],
             'thumbnails': thumbnails,
             'description': audio_description,
-            'categories': list(filter(None, (audio_info.get('category_name'), audio_info.get('category_title')))),
+            'categories': list(filter(None, [audio_info.get('category_name')])),
             'duration': audio_info.get('duration'),
             'view_count': audio_info.get('play_count'),
             'like_count': audio_info.get('favorites_count'),
@@ -170,60 +123,38 @@ class XimalayaIE(XimalayaBaseIE):
 class XimalayaAlbumIE(XimalayaBaseIE):
     IE_NAME = 'ximalaya:album'
     IE_DESC = '喜马拉雅FM 专辑'
-    _VALID_URL = r'https?://(?:www\.|m\.)?ximalaya\.com/(?P<uid>[0-9]+)/album/(?P<id>[0-9]+)'
-    _TEMPLATE_URL = '%s://www.ximalaya.com/%s/album/%s/'
-    _BASE_URL_TEMPL = '%s://www.ximalaya.com%s'
-    _LIST_VIDEO_RE = r'<a[^>]+?href="(?P<url>/%s/sound/(?P<id>\d+)/?)"[^>]+?title="(?P<title>[^>]+)">'
+    _VALID_URL = r'https?://(?:www\.|m\.)?ximalaya\.com/\d+/album/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://www.ximalaya.com/61425525/album/5534601/',
         'info_dict': {
             'title': '唐诗三百首（含赏析）',
             'id': '5534601',
         },
-        'playlist_count': 312,
-    }, {
-        'url': 'http://m.ximalaya.com/61425525/album/5534601',
-        'info_dict': {
-            'title': '唐诗三百首（含赏析）',
-            'id': '5534601',
-        },
-        'playlist_count': 312,
-    },
-    ]
+        'playlist_mincount': 323,
+    }]
 
     def _real_extract(self, url):
-        self.scheme = scheme = 'https' if url.startswith('https') else 'http'
+        playlist_id = self._match_id(url)
 
-        mobj = self._match_valid_url(url)
-        uid, playlist_id = mobj.group('uid'), mobj.group('id')
+        first_page = self._fetch_page(playlist_id, 1)
+        page_count = math.ceil(first_page['trackTotalCount'] / first_page['pageSize'])
 
-        webpage = self._download_webpage(self._TEMPLATE_URL % (scheme, uid, playlist_id), playlist_id,
-                                         note='Download album page for %s' % playlist_id,
-                                         errnote='Unable to get album info')
+        entries = InAdvancePagedList(
+            lambda idx: self._get_entries(self._fetch_page(playlist_id, idx + 1) if idx else first_page),
+            page_count, first_page['pageSize'])
 
-        title = self._html_search_regex(r'detailContent_title[^>]*><h1(?:[^>]+)?>([^<]+)</h1>',
-                                        webpage, 'title', fatal=False)
+        title = traverse_obj(first_page, ('tracks', 0, 'albumTitle'), expected_type=str)
 
-        return self.playlist_result(self._entries(webpage, playlist_id, uid), playlist_id, title)
+        return self.playlist_result(entries, playlist_id, title)
 
-    def _entries(self, page, playlist_id, uid):
-        html = page
-        for page_num in itertools.count(1):
-            for entry in self._process_page(html, uid):
-                yield entry
+    def _fetch_page(self, playlist_id, page_idx):
+        return self._download_json(
+            'https://www.ximalaya.com/revision/album/v1/getTracksList',
+            playlist_id, note=f'Downloading tracks list page {page_idx}',
+            query={'albumId': playlist_id, 'pageNum': page_idx, 'sort': 1})['data']
 
-            next_url = self._search_regex(r'<a\s+href=(["\'])(?P<more>[\S]+)\1[^>]+rel=(["\'])next\3',
-                                          html, 'list_next_url', default=None, group='more')
-            if not next_url:
-                break
-
-            next_full_url = self._BASE_URL_TEMPL % (self.scheme, next_url)
-            html = self._download_webpage(next_full_url, playlist_id)
-
-    def _process_page(self, html, uid):
-        find_from = html.index('album_soundlist')
-        for mobj in re.finditer(self._LIST_VIDEO_RE % uid, html[find_from:]):
-            yield self.url_result(self._BASE_URL_TEMPL % (self.scheme, mobj.group('url')),
-                                  XimalayaIE.ie_key(),
-                                  mobj.group('id'),
-                                  mobj.group('title'))
+    def _get_entries(self, page_data):
+        for e in page_data['tracks']:
+            yield self.url_result(
+                self._proto_relative_url(f'//www.ximalaya.com{e["url"]}'),
+                XimalayaIE, e.get('trackId'), e.get('title'))

yt_dlp/extractor/youtube.py

@@ -1074,6 +1074,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'age_limit': 0,
                 'start_time': 1,
                 'end_time': 9,
+                'comment_count': int,
                 'channel_follower_count': int
             }
         },
@@ -1118,6 +1119,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
                 'live_status': 'not_live',
                 'age_limit': 0,
+                'comment_count': int,
                 'channel_follower_count': int
             },
             'params': {
@@ -1260,6 +1262,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'categories': ['Entertainment'],
                 'duration': 106,
                 'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
+                'comment_count': int,
                 'channel_follower_count': int
             },
         },
@@ -1347,7 +1350,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'upload_date': '20150827',
                 'uploader_id': 'olympic',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
-                'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
+                'description': 'md5:04bbbf3ccceb6795947572ca36f45904',
                 'uploader': 'Olympics',
                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
                 'like_count': int,
@@ -1396,6 +1399,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'like_count': int,
                 'live_status': 'not_live',
                 'availability': 'unlisted',
+                'comment_count': int,
                 'channel_follower_count': int
             },
         },
@@ -1624,6 +1628,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'thumbnail': 'https://i.ytimg.com/vi_webp/M4gD1WSo5mA/maxresdefault.webp',
                 'live_status': 'not_live',
                 'playable_in_embed': True,
+                'comment_count': int,
                 'channel_follower_count': int
             },
             'params': {
@@ -1656,6 +1661,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'view_count': int,
                 'live_status': 'not_live',
                 'channel_url': 'https://www.youtube.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
+                'comment_count': int,
                 'channel_follower_count': int
             },
             'params': {
@@ -1920,6 +1926,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'view_count': int,
                 'duration': 522,
                 'channel': 'kudvenkat',
+                'comment_count': int,
                 'channel_follower_count': int
             },
             'params': {
@@ -2141,6 +2148,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'availability': 'public',
                 'channel': 'Leon Nguyen',
                 'thumbnail': 'https://i.ytimg.com/vi_webp/2NUZ8W2llS4/maxresdefault.webp',
+                'comment_count': int,
                 'channel_follower_count': int
             }
         }, {
@@ -2204,7 +2212,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'params': {'skip_download': True}
         }, {
             # Story. Requires specific player params to work.
-            # Note: stories get removed after some period of time
             'url': 'https://www.youtube.com/watch?v=vv8qTUWmulI',
             'info_dict': {
                 'id': 'vv8qTUWmulI',
@@ -2227,7 +2234,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'thumbnail': 'https://i.ytimg.com/vi_webp/vv8qTUWmulI/maxresdefault.webp',
                 'uploader_url': 'http://www.youtube.com/user/BlastfromthePast',
                 'channel_url': 'https://www.youtube.com/channel/UCzIZ8HrzDgc-pNQDUG6avBA',
-            }
+            },
+            'skip': 'stories get removed after some period of time',
         }, {
             'url': 'https://www.youtube.com/watch?v=tjjjtzRLHvA',
             'info_dict': {
@@ -2764,17 +2772,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         if not strict:
             chapter_list.sort(key=lambda c: c['start_time'] or 0)
 
-        chapters = [{'start_time': 0, 'title': '<Untitled>'}]
+        chapters = [{'start_time': 0}]
         for idx, chapter in enumerate(chapter_list):
-            if chapter['start_time'] is None or not chapter['title']:
+            if chapter['start_time'] is None:
                 self.report_warning(f'Incomplete chapter {idx}')
             elif chapters[-1]['start_time'] <= chapter['start_time'] <= duration:
-                chapters[-1]['end_time'] = chapter['start_time']
                 chapters.append(chapter)
             else:
                 self.report_warning(f'Invalid start time for chapter "{chapter["title"]}"')
-        chapters[-1]['end_time'] = duration
-        return chapters if len(chapters) > 1 and chapters[1]['start_time'] else chapters[1:]
+        return chapters[1:]
 
     def _extract_comment(self, comment_renderer, parent=None):
         comment_id = comment_renderer.get('commentId')
@@ -3334,6 +3340,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'url': url,
                 'width': width,
                 'height': height,
+                'fps': frame_count / duration,
+                'rows': rows,
+                'columns': cols,
                 'fragments': [{
                     'url': url.replace('$M', str(j)),
                     'duration': min(fragment_duration, duration - (j * fragment_duration)),
@@ -3449,7 +3458,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         if get_first(video_details, 'isPostLiveDvr'):
             self.write_debug('Video is in Post-Live Manifestless mode')
-            if duration or 0 > 4 * 3600:
+            if (duration or 0) > 4 * 3600:
                 self.report_warning(
                     'The livestream has not finished processing. Only 4 hours of the video can be currently downloaded. '
                     'This is a known issue and patches are welcome')
@@ -5004,7 +5013,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
     }, {
         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
         'info_dict': {
-            'id': 'GgL890LIznQ',  # This will keep changing
+            'id': 'Wq15eF5vCbI',  # This will keep changing
             'ext': 'mp4',
             'title': str,
             'uploader': 'Sky News',
@@ -5124,7 +5133,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'uploader': 'NoCopyrightSounds',
             'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
             'uploader_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
-            'title': 'NCS Releases',
+            'title': 'NCS : All Releases 💿',
             'uploader_url': 'https://www.youtube.com/c/NoCopyrightSounds',
             'channel_url': 'https://www.youtube.com/c/NoCopyrightSounds',
             'modified_date': r're:\d{8}',
@@ -5193,7 +5202,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'title': 'yt-dlp unlisted playlist test',
             'availability': 'unlisted',
             'tags': [],
-            'modified_date': '20211208',
+            'modified_date': '20220418',
             'channel': 'colethedj',
             'view_count': int,
             'description': '',
@@ -5281,6 +5290,7 @@ class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
             'channel': 'pukkandan',
             'description': 'Test for collaborative playlist',
             'title': 'yt-dlp test - collaborative playlist',
+            'view_count': int,
             'uploader_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
         },
         'playlist_mincount': 2
@@ -5488,7 +5498,7 @@ class YoutubePlaylistIE(InfoExtractor):
         'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
     }, {
         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
-        'playlist_mincount': 654,
+        'playlist_mincount': 455,
         'info_dict': {
             'title': '2018 Chinese New Singles (11/6 updated)',
             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
@@ -5561,6 +5571,8 @@ class YoutubeYtBeIE(InfoExtractor):
             'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
             'availability': 'public',
             'duration': 59,
+            'comment_count': int,
+            'channel_follower_count': int
         },
         'params': {
             'noplaylist': True,
@@ -5778,10 +5790,11 @@ class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
         'info_dict': {
             'id': '#cats',
             'title': '#cats',
-            'entries': [{
-                'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
-                'title': '#cats',
-            }],
+            # The test suite does not have support for nested playlists
+            # 'entries': [{
+            #     'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
+            #     'title': '#cats',
+            # }],
         },
     }, {
         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
@@ -5998,6 +6011,25 @@ class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
             'section_start': 29.0,
             'section_end': 39.7,
             'duration': 10.7,
+            'age_limit': 0,
+            'availability': 'public',
+            'categories': ['Gaming'],
+            'channel': 'Scott The Woz',
+            'channel_id': 'UC4rqhyiTs7XyuODcECvuiiQ',
+            'channel_url': 'https://www.youtube.com/channel/UC4rqhyiTs7XyuODcECvuiiQ',
+            'description': 'md5:7a4517a17ea9b4bd98996399d8bb36e7',
+            'like_count': int,
+            'playable_in_embed': True,
+            'tags': 'count:17',
+            'thumbnail': 'https://i.ytimg.com/vi_webp/ScPX26pdQik/maxresdefault.webp',
+            'title': 'Mobile Games on Console - Scott The Woz',
+            'upload_date': '20210920',
+            'uploader': 'Scott The Woz',
+            'uploader_id': 'scottthewoz',
+            'uploader_url': 'http://www.youtube.com/user/scottthewoz',
+            'view_count': int,
+            'live_status': 'not_live',
+            'channel_follower_count': int
         }
     }]
 

yt_dlp/options.py

@@ -114,6 +114,11 @@ def parseOpts(overrideArguments=None, ignore_config_files='if_override'):
                 if user_conf is not None:
                     root.configs.pop(user_conf)
 
+        try:
+            root.configs[0].load_configs()  # Resolve any aliases using --config-location
+        except ValueError as err:
+            raise root.parser.error(err)
+
         opts, args = root.parse_args()
     except optparse.OptParseError:
         with contextlib.suppress(optparse.OptParseError):
@@ -423,9 +428,9 @@ def create_parser():
         action='store_false', dest='mark_watched',
         help='Do not mark videos watched (default)')
     general.add_option(
-        '--no-colors',
+        '--no-colors', '--no-colours',
         action='store_true', dest='no_color', default=False,
-        help='Do not emit color codes in output')
+        help='Do not emit color codes in output (Alias: --no-colours)')
     general.add_option(
         '--compat-options',
         metavar='OPTS', dest='compat_opts', default=set(), type='str',

yt_dlp/postprocessor/ffmpeg.py

@@ -725,11 +725,10 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
                 value = value.replace('\0', '')  # nul character cannot be passed in command line
                 metadata['common'].update({meta_f: value for meta_f in variadic(meta_list)})
 
-        # See [1-4] for some info on media metadata/metadata supported
-        # by ffmpeg.
-        # 1. https://kdenlive.org/en/project/adding-meta-data-to-mp4-video/
-        # 2. https://wiki.multimedia.cx/index.php/FFmpeg_Metadata
-        # 3. https://kodi.wiki/view/Video_file_tagging
+        # Info on media metadata/metadata supported by ffmpeg:
+        # https://wiki.multimedia.cx/index.php/FFmpeg_Metadata
+        # https://kdenlive.org/en/project/adding-meta-data-to-mp4-video/
+        # https://kodi.wiki/view/Video_file_tagging
 
         add('title', ('track', 'title'))
         add('date', 'upload_date')

yt_dlp/postprocessor/modify_chapters.py

@@ -38,8 +38,9 @@ class ModifyChaptersPP(FFmpegPostProcessor):
         if not cuts:
             return [], info
 
-        if self._duration_mismatch(real_duration, info.get('duration'), 1):
-            if not self._duration_mismatch(real_duration, info['chapters'][-1]['end_time']):
+        original_duration, info['duration'] = info.get('duration'), info['chapters'][-1]['end_time']
+        if self._duration_mismatch(real_duration, original_duration, 1):
+            if not self._duration_mismatch(real_duration, info['duration']):
                 self.to_screen(f'Skipping {self.pp_key()} since the video appears to be already cut')
                 return [], info
             if not info.get('__real_download'):

yt_dlp/update.py

@@ -88,6 +88,9 @@ class Updater:
 
     @functools.cached_property
     def _tag(self):
+        if version_tuple(__version__) >= version_tuple(self.latest_version):
+            return 'latest'
+
         identifier = f'{detect_variant()} {system_identifier()}'
         for line in self._download('_update_spec', 'latest').decode().splitlines():
             if not line.startswith('lock '):
@@ -109,9 +112,16 @@ class Updater:
 
     @property
     def new_version(self):
-        """Version of the latest release"""
+        """Version of the latest release we can update to"""
+        if self._tag.startswith('tags/'):
+            return self._tag[5:]
         return self._get_version_info(self._tag)['tag_name']
 
+    @property
+    def latest_version(self):
+        """Version of the latest release"""
+        return self._get_version_info('latest')['tag_name']
+
     @property
     def has_update(self):
         """Whether there is an update available"""
@@ -157,13 +167,15 @@ class Updater:
         """Report whether there is an update available"""
         try:
             self.ydl.to_screen(
-                f'Latest version: {self.new_version}, Current version: {self.current_version}')
+                f'Latest version: {self.latest_version}, Current version: {self.current_version}')
+            if not self.has_update:
+                if self._tag == 'latest':
+                    return self.ydl.to_screen(f'yt-dlp is up to date ({__version__})')
+                return self.ydl.report_warning(
+                    'yt-dlp cannot be updated any further since you are on an older Python version')
         except Exception:
             return self._report_network_error('obtain version info', delim='; Please try again later or')
 
-        if not self.has_update:
-            return self.ydl.to_screen(f'yt-dlp is up to date ({__version__})')
-
         if not is_non_updateable():
             self.ydl.to_screen(f'Current Build Hash {_sha256_file(self.filename)}')
         return True

yt_dlp/utils.py

@@ -950,17 +950,18 @@ def make_HTTPS_handler(params, **kwargs):
     if opts_check_certificate:
         if has_certifi and 'no-certifi' not in params.get('compat_opts', []):
             context.load_verify_locations(cafile=certifi.where())
-        try:
-            context.load_default_certs()
-        # Work around the issue in load_default_certs when there are bad certificates. See:
-        # https://github.com/yt-dlp/yt-dlp/issues/1060,
-        # https://bugs.python.org/issue35665, https://bugs.python.org/issue45312
-        except ssl.SSLError:
-            # enum_certificates is not present in mingw python. See https://github.com/yt-dlp/yt-dlp/issues/1151
-            if sys.platform == 'win32' and hasattr(ssl, 'enum_certificates'):
-                for storename in ('CA', 'ROOT'):
-                    _ssl_load_windows_store_certs(context, storename)
-            context.set_default_verify_paths()
+        else:
+            try:
+                context.load_default_certs()
+                # Work around the issue in load_default_certs when there are bad certificates. See:
+                # https://github.com/yt-dlp/yt-dlp/issues/1060,
+                # https://bugs.python.org/issue35665, https://bugs.python.org/issue45312
+            except ssl.SSLError:
+                # enum_certificates is not present in mingw python. See https://github.com/yt-dlp/yt-dlp/issues/1151
+                if sys.platform == 'win32' and hasattr(ssl, 'enum_certificates'):
+                    for storename in ('CA', 'ROOT'):
+                        _ssl_load_windows_store_certs(context, storename)
+                context.set_default_verify_paths()
 
     client_certfile = params.get('client_certificate')
     if client_certfile:
@@ -1907,6 +1908,10 @@ class DateRange:
     def __str__(self):
         return f'{self.start.isoformat()} - {self.end.isoformat()}'
 
+    def __eq__(self, other):
+        return (isinstance(other, DateRange)
+                and self.start == other.start and self.end == other.end)
+
 
 def platform_name():
     """ Returns the platform name as a str """
@@ -2400,7 +2405,11 @@ def remove_quotes(s):
 
 
 def get_domain(url):
-    return '.'.join(urllib.parse.urlparse(url).netloc.rsplit('.', 2)[-2:])
+    """
+    This implementation is inconsistent, but is kept for compatibility.
+    Use this only for "webpage_url_domain"
+    """
+    return remove_start(urllib.parse.urlparse(url).netloc, 'www.') or None
 
 
 def url_basename(url):
@@ -2659,7 +2668,7 @@ class LazyList(collections.abc.Sequence):
 
     @staticmethod
     def _reverse_index(x):
-        return None if x is None else -(x + 1)
+        return None if x is None else ~x
 
     def __getitem__(self, idx):
         if isinstance(idx, slice):
@@ -3414,24 +3423,23 @@ def parse_codecs(codecs_str):
         str.strip, codecs_str.strip().strip(',').split(','))))
     vcodec, acodec, scodec, hdr = None, None, None, None
     for full_codec in split_codecs:
-        parts = full_codec.split('.')
-        codec = parts[0].replace('0', '')
-        if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2',
-                     'h263', 'h264', 'mp4v', 'hvc1', 'av1', 'theora', 'dvh1', 'dvhe'):
-            if not vcodec:
-                vcodec = '.'.join(parts[:4]) if codec in ('vp9', 'av1', 'hvc1') else full_codec
-                if codec in ('dvh1', 'dvhe'):
-                    hdr = 'DV'
-                elif codec == 'av1' and len(parts) > 3 and parts[3] == '10':
-                    hdr = 'HDR10'
-                elif full_codec.replace('0', '').startswith('vp9.2'):
-                    hdr = 'HDR10'
-        elif codec in ('flac', 'mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'):
-            if not acodec:
-                acodec = full_codec
-        elif codec in ('stpp', 'wvtt',):
-            if not scodec:
-                scodec = full_codec
+        parts = re.sub(r'0+(?=\d)', '', full_codec).split('.')
+        if parts[0] in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2',
+                        'h263', 'h264', 'mp4v', 'hvc1', 'av1', 'theora', 'dvh1', 'dvhe'):
+            if vcodec:
+                continue
+            vcodec = full_codec
+            if parts[0] in ('dvh1', 'dvhe'):
+                hdr = 'DV'
+            elif parts[0] == 'av1' and traverse_obj(parts, 3) == '10':
+                hdr = 'HDR10'
+            elif parts[:2] == ['vp9', '2']:
+                hdr = 'HDR10'
+        elif parts[0] in ('flac', 'mp4a', 'opus', 'vorbis', 'mp3', 'aac',
+                          'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'):
+            acodec = acodec or full_codec
+        elif parts[0] in ('stpp', 'wvtt'):
+            scodec = scodec or full_codec
         else:
             write_string(f'WARNING: Unknown codec {full_codec}\n')
     if vcodec or acodec or scodec:
@@ -3477,17 +3485,19 @@ def age_restricted(content_limit, age_limit):
     return age_limit < content_limit
 
 
+# List of known byte-order-marks (BOM)
+BOMS = [
+    (b'\xef\xbb\xbf', 'utf-8'),
+    (b'\x00\x00\xfe\xff', 'utf-32-be'),
+    (b'\xff\xfe\x00\x00', 'utf-32-le'),
+    (b'\xff\xfe', 'utf-16-le'),
+    (b'\xfe\xff', 'utf-16-be'),
+]
+
+
 def is_html(first_bytes):
     """ Detect whether a file contains HTML by examining its first bytes. """
 
-    BOMS = [
-        (b'\xef\xbb\xbf', 'utf-8'),
-        (b'\x00\x00\xfe\xff', 'utf-32-be'),
-        (b'\xff\xfe\x00\x00', 'utf-32-le'),
-        (b'\xff\xfe', 'utf-16-le'),
-        (b'\xfe\xff', 'utf-16-be'),
-    ]
-
     encoding = 'utf-8'
     for bom, enc in BOMS:
         while first_bytes.startswith(bom):
@@ -3661,21 +3671,26 @@ def match_filter_func(filters):
     return _match_func
 
 
-def download_range_func(chapters, ranges):
-    def inner(info_dict, ydl):
+class download_range_func:
+    def __init__(self, chapters, ranges):
+        self.chapters, self.ranges = chapters, ranges
+
+    def __call__(self, info_dict, ydl):
         warning = ('There are no chapters matching the regex' if info_dict.get('chapters')
                    else 'Cannot match chapters since chapter information is unavailable')
-        for regex in chapters or []:
+        for regex in self.chapters or []:
             for i, chapter in enumerate(info_dict.get('chapters') or []):
                 if re.search(regex, chapter['title']):
                     warning = None
                     yield {**chapter, 'index': i}
-        if chapters and warning:
+        if self.chapters and warning:
             ydl.to_screen(f'[info] {info_dict["id"]}: {warning}')
 
-        yield from ({'start_time': start, 'end_time': end} for start, end in ranges or [])
+        yield from ({'start_time': start, 'end_time': end} for start, end in self.ranges or [])
 
-    return inner
+    def __eq__(self, other):
+        return (isinstance(other, download_range_func)
+                and self.chapters == other.chapters and self.ranges == other.ranges)
 
 
 def parse_dfxp_time_expr(time_expr):
@@ -4755,7 +4770,7 @@ def _base_n_table(n, table):
         raise ValueError('Either table or n must be specified')
     table = (table or '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ')[:n]
 
-    if n != len(table):
+    if n and n != len(table):
         raise ValueError(f'base {n} exceeds table length {len(table)}')
     return table
 
@@ -5381,6 +5396,24 @@ def read_stdin(what):
     return sys.stdin
 
 
+def determine_file_encoding(data):
+    """
+    Detect the text encoding used
+    @returns (encoding, bytes to skip)
+    """
+
+    # BOM marks are given priority over declarations
+    for bom, enc in BOMS:
+        if data.startswith(bom):
+            return enc, len(bom)
+
+    # Strip off all null bytes to match even when UTF-16 or UTF-32 is used.
+    # We ignore the endianness to get a good enough match
+    data = data.replace(b'\0', b'')
+    mobj = re.match(rb'(?m)^#\s*coding\s*:\s*(\S+)\s*$', data)
+    return mobj.group(1).decode() if mobj else None, 0
+
+
 class Config:
     own_args = None
     parsed_args = None
@@ -5393,18 +5426,21 @@ class Config:
 
     def init(self, args=None, filename=None):
         assert not self.__initialized
+        self.own_args, self.filename = args, filename
+        return self.load_configs()
+
+    def load_configs(self):
         directory = ''
-        if filename:
-            location = os.path.realpath(filename)
+        if self.filename:
+            location = os.path.realpath(self.filename)
             directory = os.path.dirname(location)
             if location in self._loaded_paths:
                 return False
             self._loaded_paths.add(location)
 
-        self.own_args, self.__initialized = args, True
-        opts, _ = self.parser.parse_known_args(args)
-        self.parsed_args, self.filename = args, filename
-
+        self.__initialized = True
+        opts, _ = self.parser.parse_known_args(self.own_args)
+        self.parsed_args = self.own_args
         for location in opts.config_locations or []:
             if location == '-':
                 self.append_config(shlex.split(read_stdin('options'), comments=True), label='stdin')
@@ -5429,12 +5465,17 @@ class Config:
     @staticmethod
     def read_file(filename, default=[]):
         try:
-            optionf = open(filename)
+            optionf = open(filename, 'rb')
         except OSError:
             return default  # silently skip if file is not present
+        try:
+            enc, skip = determine_file_encoding(optionf.read(512))
+            optionf.seek(skip, io.SEEK_SET)
+        except OSError:
+            enc = None  # silently skip read errors
         try:
             # FIXME: https://github.com/ytdl-org/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56
-            contents = optionf.read()
+            contents = optionf.read().decode(enc or preferredencoding())
             res = shlex.split(contents, comments=True)
         except Exception as err:
             raise ValueError(f'Unable to parse "{filename}": {err}')

yt_dlp/version.py

@@ -1,5 +1,5 @@
 # Autogenerated by devscripts/update-version.py
 
-__version__ = '2022.06.29'
+__version__ = '2022.07.18'
 
-RELEASE_GIT_HEAD = '9d339c41e'
+RELEASE_GIT_HEAD = '135f05ef6'