[extractor/steam] Add broadcast extractor (#4137)

Closes #4083

Authored by: HobbyistDev

.github/workflows/build.yml

@@ -236,6 +236,52 @@ jobs:
         asset_content_type: application/zip
 
 
+  build_macos_legacy:
+    runs-on: macos-latest
+    needs: create_release
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install Python
+      # We need the official Python, because the GA ones only support newer macOS versions
+      env:
+        PYTHON_VERSION: 3.10.5
+        MACOSX_DEPLOYMENT_TARGET: 10.9  # Used up by the Python build tools
+      run: |
+          # Hack to get the latest patch version. Uncomment if needed
+          #brew install python@3.10
+          #export PYTHON_VERSION=$( $(brew --prefix)/opt/python@3.10/bin/python3 --version | cut -d ' ' -f 2 )
+          curl https://www.python.org/ftp/python/${PYTHON_VERSION}/python-${PYTHON_VERSION}-macos11.pkg -o "python.pkg"
+          sudo installer -pkg python.pkg -target /
+          python3 --version
+    - name: Install Requirements
+      run: |
+          brew install coreutils
+          python3 -m pip install -U --user pip Pyinstaller -r requirements.txt
+
+    - name: Prepare
+      run: |
+          python3 devscripts/update-version.py ${{ needs.create_release.outputs.version_suffix }}
+          python3 devscripts/make_lazy_extractors.py
+    - name: Build
+      run: |
+          python3 pyinst.py
+    - name: Get SHA2-SUMS
+      id: get_sha
+      run: |
+          echo "::set-output name=sha256_macos_legacy::$(sha256sum dist/yt-dlp_macos | awk '{print $1}')"
+          echo "::set-output name=sha512_macos_legacy::$(sha512sum dist/yt-dlp_macos | awk '{print $1}')"
+
+    - name: Upload standalone binary
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.create_release.outputs.upload_url }}
+        asset_path: ./dist/yt-dlp_macos
+        asset_name: yt-dlp_macos_legacy
+        asset_content_type: application/octet-stream
+
+
   build_windows:
     runs-on: windows-latest
     needs: create_release
@@ -351,7 +397,7 @@ jobs:
 
   finish:
     runs-on: ubuntu-latest
-    needs: [create_release, build_unix, build_windows, build_windows32, build_macos]
+    needs: [create_release, build_unix, build_windows, build_windows32, build_macos, build_macos_legacy]
 
     steps:
     - name: Make SHA2-SUMS files
@@ -366,6 +412,7 @@ jobs:
         echo "${{ needs.build_windows.outputs.sha256_win_zip }}  yt-dlp_win.zip" >> SHA2-256SUMS
         echo "${{ needs.build_macos.outputs.sha256_macos }}  yt-dlp_macos" >> SHA2-256SUMS
         echo "${{ needs.build_macos.outputs.sha256_macos_zip }}  yt-dlp_macos.zip" >> SHA2-256SUMS
+        echo "${{ needs.build_macos_legacy.outputs.sha256_macos_legacy }}  yt-dlp_macos_legacy" >> SHA2-256SUMS
         echo "${{ needs.build_unix.outputs.sha512_bin }}  yt-dlp" >> SHA2-512SUMS
         echo "${{ needs.build_unix.outputs.sha512_tar }}  yt-dlp.tar.gz" >> SHA2-512SUMS
         echo "${{ needs.build_unix.outputs.sha512_linux }}  yt-dlp_linux" >> SHA2-512SUMS
@@ -376,6 +423,7 @@ jobs:
         echo "${{ needs.build_windows.outputs.sha512_win_zip }}  yt-dlp_win.zip" >> SHA2-512SUMS
         echo "${{ needs.build_macos.outputs.sha512_macos }}  yt-dlp_macos" >> SHA2-512SUMS
         echo "${{ needs.build_macos.outputs.sha512_macos_zip }}  yt-dlp_macos.zip" >> SHA2-512SUMS
+        echo "${{ needs.build_macos_legacy.outputs.sha512_macos_legacy }}  yt-dlp_macos_legacy" >> SHA2-512SUMS
 
     - name: Upload SHA2-256SUMS file
       uses: actions/upload-release-asset@v1

README.md

@@ -253,6 +253,7 @@ File|Description
 [yt-dlp_linux.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux.zip)|Unpackaged Unix executable (no auto-update)
 [yt-dlp_win.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_win.zip)|Unpackaged Windows executable (no auto-update)
 [yt-dlp_macos.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos.zip)|Unpackaged MacOS (10.15+) executable (no auto-update)
+[yt-dlp_macos_legacy](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_macos_legacy)|MacOS (10.9+) standalone x64 executable
 
 #### Misc
 

yt_dlp/extractor/_extractors.py

@@ -1511,6 +1511,7 @@ from .scte import (
     SCTEIE,
     SCTECourseIE,
 )
+from .scrolller import ScrolllerIE
 from .seeker import SeekerIE
 from .senategov import SenateISVPIE, SenateGovIE
 from .sendtonews import SendtoNewsIE
@@ -1632,7 +1633,10 @@ from .srgssr import (
 from .srmediathek import SRMediathekIE
 from .stanfordoc import StanfordOpenClassroomIE
 from .startv import StarTVIE
-from .steam import SteamIE
+from .steam import (
+    SteamIE,
+    SteamCommunityBroadcastIE,
+)
 from .storyfire import (
     StoryFireIE,
     StoryFireUserIE,

yt_dlp/extractor/scrolller.py

@@ -0,0 +1,104 @@
+import json
+
+from .common import InfoExtractor
+from ..utils import determine_ext, int_or_none
+
+
+class ScrolllerIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?scrolller\.com/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://scrolller.com/a-helping-hand-1k9pxikxkw',
+        'info_dict': {
+            'id': 'a-helping-hand-1k9pxikxkw',
+            'ext': 'mp4',
+            'thumbnail': 'https://zepto.scrolller.com/a-helping-hand-3ty9q8x094-540x960.jpg',
+            'title': 'A helping hand',
+            'age_limit': 0,
+        }
+    }, {
+        'url': 'https://scrolller.com/tigers-chasing-a-drone-c5d1f2so6j',
+        'info_dict': {
+            'id': 'tigers-chasing-a-drone-c5d1f2so6j',
+            'ext': 'mp4',
+            'thumbnail': 'https://zepto.scrolller.com/tigers-chasing-a-drone-az9pkpguwe-540x303.jpg',
+            'title': 'Tigers chasing a drone',
+            'age_limit': 0,
+        }
+    }, {
+        'url': 'https://scrolller.com/baby-rhino-smells-something-9chhugsv9p',
+        'info_dict': {
+            'id': 'baby-rhino-smells-something-9chhugsv9p',
+            'ext': 'mp4',
+            'thumbnail': 'https://atto.scrolller.com/hmm-whats-that-smell-bh54mf2c52-300x224.jpg',
+            'title': 'Baby rhino smells something',
+            'age_limit': 0,
+        }
+    }, {
+        'url': 'https://scrolller.com/its-all-fun-and-games-cco8jjmoh7',
+        'info_dict': {
+            'id': 'its-all-fun-and-games-cco8jjmoh7',
+            'ext': 'mp4',
+            'thumbnail': 'https://atto.scrolller.com/its-all-fun-and-games-3amk9vg7m3-540x649.jpg',
+            'title': 'It\'s all fun and games...',
+            'age_limit': 0,
+        }
+    }, {
+        'url': 'https://scrolller.com/may-the-force-be-with-you-octokuro-yeytg1fs7a',
+        'info_dict': {
+            'id': 'may-the-force-be-with-you-octokuro-yeytg1fs7a',
+            'ext': 'mp4',
+            'thumbnail': 'https://thumbs2.redgifs.com/DarkStarchyNautilus-poster.jpg',
+            'title': 'May the force be with you (Octokuro)',
+            'age_limit': 18,
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        query = {
+            'query': '''{
+                getSubredditPost(url:"/%s"){
+                    id
+                    title
+                    isNsfw
+                    mediaSources{
+                        url
+                        width
+                        height
+                    }
+                }
+            }''' % video_id
+        }
+
+        video_data = self._download_json(
+            'https://api.scrolller.com/api/v2/graphql', video_id, data=json.dumps(query).encode(),
+            headers={'Content-Type': 'application/json'})['data']['getSubredditPost']
+
+        formats, thumbnails = [], []
+        for source in video_data['mediaSources']:
+            if determine_ext(source.get('url')) in ('jpg', 'png'):
+                thumbnails.append({
+                    'url': source['url'],
+                    'width': int_or_none(source.get('width')),
+                    'height': int_or_none(source.get('height')),
+                })
+            elif source.get('url'):
+                formats.append({
+                    'url': source['url'],
+                    'width': int_or_none(source.get('width')),
+                    'height': int_or_none(source.get('height')),
+                })
+
+        if not formats:
+            self.raise_no_formats('There is no video.', expected=True, video_id=video_id)
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video_data.get('title'),
+            'thumbnails': thumbnails,
+            'formats': formats,
+            'age_limit': 18 if video_data.get('isNsfw') else 0
+        }

yt_dlp/extractor/steam.py

@@ -127,3 +127,50 @@ class SteamIE(InfoExtractor):
             raise ExtractorError('Could not find any videos')
 
         return self.playlist_result(entries, playlist_id, playlist_title)
+
+
+class SteamCommunityBroadcastIE(InfoExtractor):
+    _VALID_URL = r'https?://steamcommunity\.(?:com)/broadcast/watch/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://steamcommunity.com/broadcast/watch/76561199073851486',
+        'info_dict': {
+            'id': '76561199073851486',
+            'title': r're:Steam Community :: pepperm!nt :: Broadcast 2022-06-26 \d{2}:\d{2}',
+            'ext': 'mp4',
+            'uploader_id': 1113585758,
+            'uploader': 'pepperm!nt',
+            'live_status': 'is_live',
+        },
+        'skip': 'Stream has ended',
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        json_data = self._download_json(
+            'https://steamcommunity.com/broadcast/getbroadcastmpd/',
+            video_id, query={'steamid': f'{video_id}'})
+
+        formats, subs = self._extract_m3u8_formats_and_subtitles(json_data['hls_url'], video_id)
+
+        ''' # We cannot download live dash atm
+        mpd_formats, mpd_subs = self._extract_mpd_formats_and_subtitles(json_data['url'], video_id)
+        formats.extend(mpd_formats)
+        self._merge_subtitles(mpd_subs, target=subs)
+        '''
+
+        uploader_json = self._download_json(
+            'https://steamcommunity.com/actions/ajaxresolveusers',
+            video_id, query={'steamids': video_id})[0]
+
+        self._sort_formats(formats)
+        return {
+            'id': video_id,
+            'title': self._html_extract_title(webpage) or self._og_search_title(webpage),
+            'formats': formats,
+            'live_status': 'is_live',
+            'view_count': json_data.get('num_view'),
+            'uploader': uploader_json.get('persona_name'),
+            'uploader_id': uploader_json.get('accountid'),
+            'subtitles': subs,
+        }