#70 Allow downloading of unplayable video formats

Video postprocessors are also turned off when this option is used

Co-authored-by: shirtjs <2660574+shirtjs@users.noreply.github.com>
Co-authored-by: pukkandan <pukkandan@gmail.com>

README.md

@@ -537,6 +537,11 @@ Then simply type this
                                      bestvideo+bestaudio), output to given
                                      container format. One of mkv, mp4, ogg,
                                      webm, flv. Ignored if no merge is required
+    --allow-unplayable-formats       Allow unplayable formats to be listed and
+                                     downloaded. All video postprocessing will
+                                     also be turned off
+    --no-allow-unplayable-formats    Do not allow unplayable formats to be
+                                     listed or downloaded (default)
 
 ## Subtitle Options:
     --write-subs                     Write subtitle file

youtube_dlc/YoutubeDL.py

@@ -179,6 +179,7 @@ class YoutubeDL(object):
                        of 'skip_download' or 'simulate'.
     simulate:          Do not download the video files.
     format:            Video format code. see "FORMAT SELECTION" for more details.
+    allow_unplayable_formats:   Allow unplayable formats to be extracted and downloaded.
     format_sort:       How to sort the video formats. see "Sorting Formats"
                        for more details.
     format_sort_force: Force the given format_sort. see "Sorting Formats"
@@ -2291,10 +2292,15 @@ class YoutubeDL(object):
                 if info_dict.get('requested_formats') is not None:
                     downloaded = []
                     merger = FFmpegMergerPP(self)
-                    if not merger.available:
+                    if self.params.get('allow_unplayable_formats'):
-                        self.report_warning('You have requested multiple '
+                        self.report_warning(
-                                            'formats but ffmpeg is not installed.'
+                            'You have requested merging of multiple formats '
-                                            ' The formats won\'t be merged.')
+                            'while also allowing unplayable formats to be downloaded. '
+                            'The formats won\'t be merged to prevent data corruption.')
+                    elif not merger.available:
+                        self.report_warning(
+                            'You have requested merging of multiple formats but ffmpeg is not installed. '
+                            'The formats won\'t be merged.')
 
                     def compatible_formats(formats):
                         # TODO: some formats actually allow this (mkv, webm, ogg, mp4), but not all of them.
@@ -2346,7 +2352,7 @@ class YoutubeDL(object):
                             downloaded.append(fname)
                             partial_success, real_download = dl(fname, new_info)
                             success = success and partial_success
-                        if merger.available:
+                        if merger.available and not self.params.get('allow_unplayable_formats'):
                             info_dict['__postprocessors'].append(merger)
                         info_dict['__files_to_merge'] = downloaded
                         # Even if there were no downloads, it is being merged only now

youtube_dlc/__init__.py

@@ -212,9 +212,6 @@ def _real_main(argv=None):
     if opts.recodevideo is not None:
         if opts.recodevideo not in REMUX_EXTENSIONS:
             parser.error('invalid video recode format specified')
-    if opts.remuxvideo and opts.recodevideo:
-        opts.remuxvideo = None
-        write_string('WARNING: --remux-video is ignored since --recode-video was given\n', out=sys.stderr)
     if opts.remuxvideo is not None:
         opts.remuxvideo = opts.remuxvideo.replace(' ', '')
         remux_regex = r'{0}(?:/{0})*$'.format(r'(?:\w+>)?(?:%s)' % '|'.join(REMUX_EXTENSIONS))
@@ -265,6 +262,40 @@ def _real_main(argv=None):
     any_printing = opts.print_json
     download_archive_fn = expand_path(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 
+    def report_conflict(arg1, arg2):
+        write_string('WARNING: %s is ignored since %s was given\n' % (arg2, arg1), out=sys.stderr)
+    if opts.remuxvideo and opts.recodevideo:
+        report_conflict('--recode-video', '--remux-video')
+        opts.remuxvideo = False
+    if opts.allow_unplayable_formats:
+        if opts.extractaudio:
+            report_conflict('--allow-unplayable-formats', '--extract-audio')
+            opts.extractaudio = False
+        if opts.remuxvideo:
+            report_conflict('--allow-unplayable-formats', '--remux-video')
+            opts.remuxvideo = False
+        if opts.recodevideo:
+            report_conflict('--allow-unplayable-formats', '--recode-video')
+            opts.recodevideo = False
+        if opts.addmetadata:
+            report_conflict('--allow-unplayable-formats', '--add-metadata')
+            opts.addmetadata = False
+        if opts.embedsubtitles:
+            report_conflict('--allow-unplayable-formats', '--embed-subs')
+            opts.embedsubtitles = False
+        if opts.embedthumbnail:
+            report_conflict('--allow-unplayable-formats', '--embed-thumbnail')
+            opts.embedthumbnail = False
+        if opts.xattrs:
+            report_conflict('--allow-unplayable-formats', '--xattrs')
+            opts.xattrs = False
+        if opts.fixup and opts.fixup.lower() not in ('never', 'ignore'):
+            report_conflict('--allow-unplayable-formats', '--fixup')
+        opts.fixup = 'never'
+        if opts.sponskrub:
+            report_conflict('--allow-unplayable-formats', '--sponskrub')
+        opts.sponskrub = False
+
     # PostProcessors
     postprocessors = []
     if opts.metafromfield:
@@ -393,6 +424,7 @@ def _real_main(argv=None):
         'simulate': opts.simulate or any_getting,
         'skip_download': opts.skip_download,
         'format': opts.format,
+        'allow_unplayable_formats': opts.allow_unplayable_formats,
         'format_sort': opts.format_sort,
         'format_sort_force': opts.format_sort_force,
         'allow_multiple_video_streams': opts.allow_multiple_video_streams,

youtube_dlc/downloader/f4m.py

@@ -267,6 +267,7 @@ class F4mFD(FragmentFD):
         media = doc.findall(_add_ns('media'))
         if not media:
             self.report_error('No media found')
+        if not self.params.get('allow_unplayable_formats'):
             for e in (doc.findall(_add_ns('drmAdditionalHeader'))
                       + doc.findall(_add_ns('drmAdditionalHeaderSet'))):
                 # If id attribute is missing it's valid for all media nodes

youtube_dlc/downloader/hls.py

@@ -29,9 +29,8 @@ class HlsFD(FragmentFD):
     FD_NAME = 'hlsnative'
 
     @staticmethod
-    def can_download(manifest, info_dict):
+    def can_download(manifest, info_dict, allow_unplayable_formats=False):
-        UNSUPPORTED_FEATURES = (
+        UNSUPPORTED_FEATURES = [
-            r'#EXT-X-KEY:METHOD=(?!NONE|AES-128)',  # encrypted streams [1]
             # r'#EXT-X-BYTERANGE',  # playlists composed of byte ranges of media files [2]
 
             # Live streams heuristic does not always work (e.g. geo restricted to Germany
@@ -50,7 +49,11 @@ class HlsFD(FragmentFD):
             # 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.2
             # 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5
             # 5. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.2.5
-        )
+        ]
+        if not allow_unplayable_formats:
+            UNSUPPORTED_FEATURES += [
+                r'#EXT-X-KEY:METHOD=(?!NONE|AES-128)',  # encrypted streams [1]
+            ]
         check_results = [not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES]
         is_aes128_enc = '#EXT-X-KEY:METHOD=AES-128' in manifest
         check_results.append(can_decrypt_frag or not is_aes128_enc)
@@ -66,7 +69,7 @@ class HlsFD(FragmentFD):
         man_url = urlh.geturl()
         s = urlh.read().decode('utf-8', 'ignore')
 
-        if not self.can_download(s, info_dict):
+        if not self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')):
             if info_dict.get('extra_param_to_segment_url') or info_dict.get('_decryption_key_url'):
                 self.report_error('pycrypto not found. Please install it.')
                 return False

youtube_dlc/extractor/brightcove.py

@@ -479,10 +479,10 @@ class BrightcoveNewIE(AdobePassIE):
             ext = mimetype2ext(source.get('type'))
             src = source.get('src')
             # https://support.brightcove.com/playback-api-video-fields-reference#key_systems_object
-            if container == 'WVM' or source.get('key_systems'):
+            if not self._downloader.params.get('allow_unplayable_formats') and (container == 'WVM' or source.get('key_systems')):
                 num_drm_sources += 1
                 continue
-            elif ext == 'ism':
+            elif ext == 'ism' and self._downloader.params.get('allow_unplayable_formats'):
                 continue
             elif ext == 'm3u8' or container == 'M2TS':
                 if not src:
@@ -546,7 +546,7 @@ class BrightcoveNewIE(AdobePassIE):
                 error = errors[0]
                 raise ExtractorError(
                     error.get('message') or error.get('error_subcode') or error['error_code'], expected=True)
-            if sources and num_drm_sources == len(sources):
+            if not self._downloader.params.get('allow_unplayable_formats') and sources and num_drm_sources == len(sources):
                 raise ExtractorError('This video is DRM protected.', expected=True)
 
         self._sort_formats(formats)

youtube_dlc/extractor/ceskatelevize.py

@@ -147,7 +147,7 @@ class CeskaTelevizeIE(InfoExtractor):
                 is_live = item.get('type') == 'LIVE'
                 formats = []
                 for format_id, stream_url in item.get('streamUrls', {}).items():
-                    if 'drmOnly=true' in stream_url:
+                    if not self._downloader.params.get('allow_unplayable_formats') and 'drmOnly=true' in stream_url:
                         continue
                     if 'playerType=flash' in stream_url:
                         stream_formats = self._extract_m3u8_formats(

youtube_dlc/extractor/common.py

@@ -2358,6 +2358,8 @@ class InfoExtractor(object):
                         extract_Initialization(segment_template)
             return ms_info
 
+        allow_unplayable_formats = self._downloader.params.get('allow_unplayable_formats')
+
         mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration'))
         formats = []
         for period in mpd_doc.findall(_add_ns('Period')):
@@ -2367,11 +2369,11 @@ class InfoExtractor(object):
                 'timescale': 1,
             })
             for adaptation_set in period.findall(_add_ns('AdaptationSet')):
-                if is_drm_protected(adaptation_set):
+                if is_drm_protected(adaptation_set) and allow_unplayable_formats is False:
                     continue
                 adaption_set_ms_info = extract_multisegment_info(adaptation_set, period_ms_info)
                 for representation in adaptation_set.findall(_add_ns('Representation')):
-                    if is_drm_protected(representation):
+                    if is_drm_protected(representation) and allow_unplayable_formats is False:
                         continue
                     representation_attrib = adaptation_set.attrib.copy()
                     representation_attrib.update(representation.attrib)
@@ -2585,7 +2587,7 @@ class InfoExtractor(object):
          1. [MS-SSTR]: Smooth Streaming Protocol,
             https://msdn.microsoft.com/en-us/library/ff469518.aspx
         """
-        if ism_doc.get('IsLive') == 'TRUE' or ism_doc.find('Protection') is not None:
+        if ism_doc.get('IsLive') == 'TRUE' or (ism_doc.find('Protection') is not None and not self._downloader.params.get('allow_unplayable_formats')):
             return []
 
         duration = int(ism_doc.attrib['Duration'])

youtube_dlc/extractor/crackle.py

@@ -103,7 +103,7 @@ class CrackleIE(InfoExtractor):
 
             formats = []
             for e in media['MediaURLs']:
-                if e.get('UseDRM') is True:
+                if not self._downloader.params.get('allow_unplayable_formats') and e.get('UseDRM') is True:
                     continue
                 format_url = url_or_none(e.get('Path'))
                 if not format_url:

youtube_dlc/extractor/globo.py

@@ -96,7 +96,7 @@ class GloboIE(InfoExtractor):
         video = self._download_json(
             'http://api.globovideos.com/videos/%s/playlist' % video_id,
             video_id)['videos'][0]
-        if video.get('encrypted') is True:
+        if not self._downloader.params.get('allow_unplayable_formats') and video.get('encrypted') is True:
             raise ExtractorError('This video is DRM protected.', expected=True)
 
         title = video['title']

youtube_dlc/extractor/hotstar.py

@@ -141,7 +141,7 @@ class HotStarIE(HotStarBaseIE):
 
         title = video_data['title']
 
-        if video_data.get('drmProtected'):
+        if not self._downloader.params.get('allow_unplayable_formats') and video_data.get('drmProtected'):
             raise ExtractorError('This video is DRM protected.', expected=True)
 
         headers = {'Referer': url}

youtube_dlc/extractor/ivi.py

@@ -163,7 +163,7 @@ class IviIE(InfoExtractor):
         for f in result.get('files', []):
             f_url = f.get('url')
             content_format = f.get('content_format')
-            if not f_url or '-MDRM-' in content_format or '-FPS-' in content_format:
+            if not f_url or (not self._downloader.params.get('allow_unplayable_formats') and ('-MDRM-' in content_format or '-FPS-' in content_format)):
                 continue
             formats.append({
                 'url': f_url,

youtube_dlc/extractor/kaltura.py

@@ -309,7 +309,7 @@ class KalturaIE(InfoExtractor):
             if f.get('fileExt') == 'chun':
                 continue
             # DRM-protected video, cannot be decrypted
-            if f.get('fileExt') == 'wvm':
+            if not self._downloader.params.get('allow_unplayable_formats') and f.get('fileExt') == 'wvm':
                 continue
             if not f.get('fileExt'):
                 # QT indicates QuickTime; some videos have broken fileExt

youtube_dlc/extractor/limelight.py

@@ -96,7 +96,7 @@ class LimelightBaseIE(InfoExtractor):
         urls = []
         for stream in pc_item.get('streams', []):
             stream_url = stream.get('url')
-            if not stream_url or stream.get('drmProtected') or stream_url in urls:
+            if not stream_url or (not self._downloader.params.get('allow_unplayable_formats') and stream.get('drmProtected')) or stream_url in urls:
                 continue
             urls.append(stream_url)
             ext = determine_ext(stream_url)

youtube_dlc/extractor/ninecninemedia.py

@@ -36,7 +36,7 @@ class NineCNineMediaIE(InfoExtractor):
                 '$include': '[HasClosedCaptions]',
             })
 
-        if try_get(content_package, lambda x: x['Constraints']['Security']['Type']):
+        if not self._downloader.params.get('allow_unplayable_formats') and try_get(content_package, lambda x: x['Constraints']['Security']['Type']):
             raise ExtractorError('This video is DRM protected.', expected=True)
 
         manifest_base_url = content_package_url + 'manifest.'

youtube_dlc/extractor/ninenow.py

@@ -66,7 +66,7 @@ class NineNowIE(InfoExtractor):
 
         video_data = common_data['video']
 
-        if video_data.get('drm'):
+        if not self._downloader.params.get('allow_unplayable_formats') and video_data.get('drm'):
             raise ExtractorError('This video is DRM protected.', expected=True)
 
         brightcove_id = video_data.get('brightcoveId') or 'ref:' + video_data['referenceId']

youtube_dlc/extractor/npo.py

@@ -246,7 +246,7 @@ class NPOIE(NPOBaseIE):
                 })
 
         if not formats:
-            if drm:
+            if not self._downloader.params.get('allow_unplayable_formats') and drm:
                 raise ExtractorError('This video is DRM protected.', expected=True)
             return
 

youtube_dlc/extractor/prosiebensat1.py

@@ -34,7 +34,7 @@ class ProSiebenSat1BaseIE(InfoExtractor):
                 'ids': clip_id,
             })[0]
 
-        if video.get('is_protected') is True:
+        if not self._downloader.params.get('allow_unplayable_formats') and video.get('is_protected') is True:
             raise ExtractorError('This video is DRM protected.', expected=True)
 
         formats = []

youtube_dlc/extractor/rtbf.py

@@ -125,7 +125,7 @@ class RTBFIE(InfoExtractor):
                 })
 
         mpd_url = data.get('urlDash')
-        if not data.get('drm') and mpd_url:
+        if (not self._downloader.params.get('allow_unplayable_formats') and not data.get('drm')) and mpd_url:
             formats.extend(self._extract_mpd_formats(
                 mpd_url, media_id, mpd_id='dash', fatal=False))
 

youtube_dlc/extractor/ruutu.py

@@ -201,7 +201,7 @@ class RuutuIE(InfoExtractor):
 
         if not formats:
             drm = xpath_text(video_xml, './Clip/DRM', default=None)
-            if drm:
+            if not self._downloader.params.get('allow_unplayable_formats') and drm:
                 raise ExtractorError('This video is DRM protected.', expected=True)
             ns_st_cds = pv('ns_st_cds')
             if ns_st_cds != 'free':

youtube_dlc/extractor/shahid.py

@@ -111,7 +111,7 @@ class ShahidIE(ShahidBaseIE):
         playout = self._call_api(
             'playout/url/' + video_id, video_id)['playout']
 
-        if playout.get('drm'):
+        if not self._downloader.params.get('allow_unplayable_formats') and playout.get('drm'):
             raise ExtractorError('This video is DRM protected.', expected=True)
 
         formats = self._extract_m3u8_formats(playout['url'], video_id, 'mp4')

youtube_dlc/extractor/sonyliv.py

@@ -75,7 +75,7 @@ class SonyLIVIE(InfoExtractor):
         video_id = self._match_id(url)
         content = self._call_api(
             '1.5', 'IN/CONTENT/VIDEOURL/VOD/' + video_id, video_id)
-        if content.get('isEncrypted'):
+        if not self._downloader.params.get('allow_unplayable_formats') and content.get('isEncrypted'):
             raise ExtractorError('This video is DRM protected.', expected=True)
         dash_url = content['videoURL']
         headers = {

youtube_dlc/extractor/toggle.py

@@ -154,7 +154,7 @@ class ToggleIE(InfoExtractor):
                 })
         if not formats:
             for meta in (info.get('Metas') or []):
-                if meta.get('Key') == 'Encryption' and meta.get('Value') == '1':
+                if not self._downloader.params.get('allow_unplayable_formats') and meta.get('Key') == 'Encryption' and meta.get('Value') == '1':
                     raise ExtractorError(
                         'This video is DRM protected.', expected=True)
             # Most likely because geo-blocked

youtube_dlc/extractor/toutv.py

@@ -74,7 +74,7 @@ class TouTvIE(RadioCanadaIE):
             })
         # IsDrm does not necessarily mean the video is DRM protected (see
         # https://github.com/ytdl-org/youtube-dl/issues/13994).
-        if metadata.get('IsDrm'):
+        if not self._downloader.params.get('allow_unplayable_formats') and metadata.get('IsDrm'):
             self.report_warning('This video is probably DRM protected.', path)
         video_id = metadata['IdMedia']
         details = metadata['Details']

youtube_dlc/extractor/tvnow.py

@@ -69,7 +69,7 @@ class TVNowBaseIE(InfoExtractor):
             if formats:
                 break
         else:
-            if info.get('isDrm'):
+            if not self._downloader.params.get('allow_unplayable_formats') and info.get('isDrm'):
                 raise ExtractorError(
                     'Video %s is DRM protected' % video_id, expected=True)
             if info.get('geoblocked'):

youtube_dlc/extractor/viki.py

@@ -315,7 +315,7 @@ class VikiIE(VikiBaseIE):
                 # Despite CODECS metadata in m3u8 all video-only formats
                 # are actually video+audio
                 for f in m3u8_formats:
-                    if '_drm/index_' in f['url']:
+                    if not self._downloader.params.get('allow_unplayable_formats') and '_drm/index_' in f['url']:
                         continue
                     if f.get('acodec') == 'none' and f.get('vcodec') != 'none':
                         f['acodec'] = None

youtube_dlc/extractor/wakanim.py

@@ -45,7 +45,7 @@ class WakanimIE(InfoExtractor):
         encryption = self._search_regex(
             r'encryption%3D(c(?:enc|bc(?:s-aapl)?))',
             m3u8_url, 'encryption', default=None)
-        if encryption and encryption in ('cenc', 'cbcs-aapl'):
+        if not self._downloader.params.get('allow_unplayable_formats') and encryption and encryption in ('cenc', 'cbcs-aapl'):
             raise ExtractorError('This video is DRM protected.', expected=True)
 
         formats = self._extract_m3u8_formats(

youtube_dlc/extractor/youtube.py

@@ -1618,7 +1618,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     formats.append(f)
 
         if not formats:
-            if streaming_data.get('licenseInfos'):
+            if not self._downloader.params.get('allow_unplayable_formats') and streaming_data.get('licenseInfos'):
                 raise ExtractorError(
                     'This video is DRM protected.', expected=True)
             pemr = try_get(

youtube_dlc/options.py

@@ -519,6 +519,16 @@ def parseOpts(overrideArguments=None):
             'If a merge is required (e.g. bestvideo+bestaudio), '
             'output to given container format. One of mkv, mp4, ogg, webm, flv. '
             'Ignored if no merge is required'))
+    video_format.add_option(
+        '--allow-unplayable-formats',
+        action='store_true', dest='allow_unplayable_formats', default=False,
+        help=(
+            'Allow unplayable formats to be listed and downloaded. '
+            'All video postprocessing will also be turned off'))
+    video_format.add_option(
+        '--no-allow-unplayable-formats',
+        action='store_false', dest='allow_unplayable_formats',
+        help='Do not allow unplayable formats to be listed or downloaded (default)')
 
     subtitles = optparse.OptionGroup(parser, 'Subtitle Options')
     subtitles.add_option(