Option to choose different downloader for different protocols

* Renamed `--external-downloader-args` to `--downloader-args`
* Added `native` as an option for the downloader
* Use similar syntax to `--downloader-args` etc. Eg: `--downloader dash:native --downloader aria2c`
* Deprecated `--hls-prefer-native` and `--hls-prefer-ffmpeg` since the same can now be done with `--downloader "m3u8:native"` and `m3u8:ffmpeg` respectively
* Split `frag_urls` protocol into `m3u8_frag_urls` and `dash_frag_urls`
* Standardize shortening of protocol names with `downloader.shorten_protocol_name`

README.md

@@ -337,10 +337,6 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
     --playlist-random                Download playlist videos in random order
     --xattr-set-filesize             Set file xattribute ytdl.filesize with
                                      expected file size
-    --hls-prefer-native              Use the native HLS downloader instead of
-                                     ffmpeg
-    --hls-prefer-ffmpeg              Use ffmpeg instead of the native HLS
-                                     downloader
     --hls-use-mpegts                 Use the mpegts container for HLS videos;
                                      allowing some players to play the video
                                      while downloading, and reducing the chance
@@ -350,10 +346,19 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
     --no-hls-use-mpegts              Do not use the mpegts container for HLS
                                      videos. This is default when not
                                      downloading live streams
-    --external-downloader NAME       Name or path of the external downloader to
+    --downloader [PROTO:]NAME        Name or path of the external downloader to
-                                     use. Currently supports aria2c, avconv,
+                                     use (optionally) prefixed by the protocols
-                                     axel, curl, ffmpeg, httpie, wget
+                                     (http, ftp, m3u8, dash, rstp, rtmp, mms) to
-                                     (Recommended: aria2c)
+                                     use it for. Currently supports native,
+                                     aria2c, avconv, axel, curl, ffmpeg, httpie,
+                                     wget (Recommended: aria2c). You can use
+                                     this option multiple times to set different
+                                     downloaders for different protocols. For
+                                     example, --downloader aria2c --downloader
+                                     "dash,m3u8:native" will use aria2c for
+                                     http/ftp downloads, and the native
+                                     downloader for dash/m3u8 downloads
+                                     (Alias: --external-downloader)
     --downloader-args NAME:ARGS      Give these arguments to the external
                                      downloader. Specify the downloader name and
                                      the arguments separated by a colon ":". You
@@ -1244,6 +1249,8 @@ These are all the deprecated options and the current alternative to achieve the
     --metadata-from-title FORMAT     --parse-metadata "%(title)s:FORMAT"
     --prefer-avconv                  avconv is no longer officially supported (Alias: --no-prefer-ffmpeg)
     --prefer-ffmpeg                  Default (Alias: --no-prefer-avconv)
+    --hls-prefer-native              --downloader "m3u8:native"
+    --hls-prefer-ffmpeg              --downloader "m3u8:ffmpeg"
     --avconv-location                avconv is no longer officially supported
     -C, --call-home                  Not implemented
     --no-call-home                   Default

yt_dlp/YoutubeDL.py

@@ -111,9 +111,17 @@ from .utils import (
     process_communicate_or_kill,
 )
 from .cache import Cache
-from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER, _PLUGIN_CLASSES
+from .extractor import (
+    gen_extractor_classes,
+    get_info_extractor,
+    _LAZY_LOADER,
+    _PLUGIN_CLASSES
+)
 from .extractor.openload import PhantomJSwrapper
-from .downloader import get_suitable_downloader
+from .downloader import (
+    get_suitable_downloader,
+    shorten_protocol_name
+)
 from .downloader.rtmp import rtmpdump_version
 from .postprocessor import (
     FFmpegFixupM3u8PP,
@@ -359,9 +367,13 @@ class YoutubeDL(object):
                        geo_bypass_country
 
     The following options determine which downloader is picked:
-    external_downloader: Executable of the external downloader to call.
+    external_downloader: A dictionary of protocol keys and the executable of the
-                       None or unset for standard (built-in) downloader.
+                       external downloader to use for it. The allowed protocols
-    hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
+                       are default|http|ftp|m3u8|dash|rtsp|rtmp|mms.
+                       Set the value to 'native' to use the native downloader
+    hls_prefer_native: Deprecated - Use external_downloader = {'m3u8': 'native'}
+                       or {'m3u8': 'ffmpeg'} instead.
+                       Use the native HLS downloader instead of ffmpeg/avconv
                        if True, otherwise use ffmpeg/avconv if False, otherwise
                        use downloader suggested by extractor if None.
 
@@ -2776,7 +2788,7 @@ class YoutubeDL(object):
                     '|',
                     format_field(f, 'filesize', ' %s', func=format_bytes) + format_field(f, 'filesize_approx', '~%s', func=format_bytes),
                     format_field(f, 'tbr', '%4dk'),
-                    f.get('protocol').replace('http_dash_segments', 'dash').replace("native", "n").replace('niconico_', ''),
+                    shorten_protocol_name(f.get('protocol', '').replace("native", "n")),
                     '|',
                     format_field(f, 'vcodec', default='unknown').replace('none', ''),
                     format_field(f, 'vbr', '%4dk'),

yt_dlp/downloader/__init__.py

@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+from ..compat import compat_str
 from ..utils import (
     determine_protocol,
 )
@@ -42,6 +43,23 @@ PROTOCOL_MAP = {
 }
 
 
+def shorten_protocol_name(proto, simplify=False):
+    short_protocol_names = {
+        'm3u8_native': 'm3u8_n',
+        'http_dash_segments': 'dash',
+        'niconico_dmc': 'dmc',
+    }
+    if simplify:
+        short_protocol_names.update({
+            'https': 'http',
+            'ftps': 'ftp',
+            'm3u8_native': 'm3u8',
+            'm3u8_frag_urls': 'm3u8',
+            'dash_frag_urls': 'dash',
+        })
+    return short_protocol_names.get(proto, proto)
+
+
 def get_suitable_downloader(info_dict, params={}, default=HttpFD):
     """Get the downloader class that can handle the info dict."""
     protocol = determine_protocol(info_dict)
@@ -50,8 +68,14 @@ def get_suitable_downloader(info_dict, params={}, default=HttpFD):
     # if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict):
     #     return FFmpegFD
 
-    external_downloader = params.get('external_downloader')
+    downloaders = params.get('external_downloader')
-    if external_downloader is not None:
+    external_downloader = (
+        downloaders if isinstance(downloaders, compat_str)
+        else downloaders.get(shorten_protocol_name(protocol, True), downloaders.get('default')))
+    if external_downloader and external_downloader.lower() == 'native':
+        external_downloader = 'native'
+
+    if external_downloader not in (None, 'native'):
         ed = get_external_downloader(external_downloader)
         if ed.can_download(info_dict, external_downloader):
             return ed
@@ -59,6 +83,8 @@ def get_suitable_downloader(info_dict, params={}, default=HttpFD):
     if protocol.startswith('m3u8'):
         if info_dict.get('is_live'):
             return FFmpegFD
+        elif external_downloader == 'native':
+            return HlsFD
         elif _get_real_downloader(info_dict, 'frag_urls', params, None):
             return HlsFD
         elif params.get('hls_prefer_native') is True:
@@ -70,6 +96,7 @@ def get_suitable_downloader(info_dict, params={}, default=HttpFD):
 
 
 __all__ = [
-    'get_suitable_downloader',
     'FileDownloader',
+    'get_suitable_downloader',
+    'shorten_protocol_name',
 ]

yt_dlp/downloader/dash.py

@@ -20,7 +20,7 @@ from ..utils import (
 class DashSegmentsFD(FragmentFD):
     """
     Download segments in a DASH manifest. External downloaders can take over
-    the fragment downloads by supporting the 'frag_urls' protocol
+    the fragment downloads by supporting the 'dash_frag_urls' protocol
     """
 
     FD_NAME = 'dashsegments'
@@ -30,7 +30,7 @@ class DashSegmentsFD(FragmentFD):
         fragments = info_dict['fragments'][:1] if self.params.get(
             'test', False) else info_dict['fragments']
 
-        real_downloader = _get_real_downloader(info_dict, 'frag_urls', self.params, None)
+        real_downloader = _get_real_downloader(info_dict, 'dash_frag_urls', self.params, None)
 
         ctx = {
             'filename': filename,

yt_dlp/downloader/external.py

@@ -81,11 +81,15 @@ class ExternalFD(FileDownloader):
 
     @property
     def exe(self):
-        return self.params.get('external_downloader')
+        return self.get_basename()
 
     @classmethod
     def available(cls, path=None):
-        return check_executable(path or cls.get_basename(), [cls.AVAILABLE_OPT])
+        path = check_executable(path or cls.get_basename(), [cls.AVAILABLE_OPT])
+        if path:
+            cls.exe = path
+            return path
+        return False
 
     @classmethod
     def supports(cls, info_dict):
@@ -259,7 +263,7 @@ class WgetFD(ExternalFD):
 
 class Aria2cFD(ExternalFD):
     AVAILABLE_OPT = '-v'
-    SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'frag_urls')
+    SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'dash_frag_urls', 'm3u8_frag_urls')
 
     @staticmethod
     def supports_manifest(manifest):
@@ -310,9 +314,11 @@ class Aria2cFD(ExternalFD):
 
 
 class HttpieFD(ExternalFD):
+    AVAILABLE_OPT = '--version'
+
     @classmethod
     def available(cls, path=None):
-        return check_executable(path or 'http', ['--version'])
+        return ExternalFD.available(cls, path or 'http')
 
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
@@ -327,7 +333,8 @@ class FFmpegFD(ExternalFD):
     SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms')
 
     @classmethod
-    def available(cls, path=None):  # path is ignored for ffmpeg
+    def available(cls, path=None):
+        # TODO: Fix path for ffmpeg
         return FFmpegPostProcessor().available
 
     def _call_downloader(self, tmpfilename, info_dict):
@@ -484,4 +491,4 @@ def get_external_downloader(external_downloader):
         downloader . """
     # Drop .exe extension on Windows
     bn = os.path.splitext(os.path.basename(external_downloader))[0]
-    return _BY_NAME[bn]
+    return _BY_NAME.get(bn)

yt_dlp/downloader/hls.py

@@ -32,7 +32,7 @@ from ..utils import (
 class HlsFD(FragmentFD):
     """
     Download segments in a m3u8 manifest. External downloaders can take over
-    the fragment downloads by supporting the 'frag_urls' protocol and
+    the fragment downloads by supporting the 'm3u8_frag_urls' protocol and
     re-defining 'supports_manifest' function
     """
 
@@ -95,7 +95,7 @@ class HlsFD(FragmentFD):
             #     fd.add_progress_hook(ph)
             return fd.real_download(filename, info_dict)
 
-        real_downloader = _get_real_downloader(info_dict, 'frag_urls', self.params, None)
+        real_downloader = _get_real_downloader(info_dict, 'm3u8_frag_urls', self.params, None)
         if real_downloader and not real_downloader.supports_manifest(s):
             real_downloader = None
         if real_downloader:

yt_dlp/options.py

@@ -639,11 +639,11 @@ def parseOpts(overrideArguments=None):
     downloader.add_option(
         '--hls-prefer-native',
         dest='hls_prefer_native', action='store_true', default=None,
-        help='Use the native HLS downloader instead of ffmpeg')
+        help=optparse.SUPPRESS_HELP)
     downloader.add_option(
         '--hls-prefer-ffmpeg',
         dest='hls_prefer_native', action='store_false', default=None,
-        help='Use ffmpeg instead of the native HLS downloader')
+        help=optparse.SUPPRESS_HELP)
     downloader.add_option(
         '--hls-use-mpegts',
         dest='hls_use_mpegts', action='store_true', default=None,
@@ -659,11 +659,20 @@ def parseOpts(overrideArguments=None):
             'Do not use the mpegts container for HLS videos. '
             'This is default when not downloading live streams'))
     downloader.add_option(
-        '--external-downloader',
+        '--downloader', '--external-downloader',
-        dest='external_downloader', metavar='NAME',
+        dest='external_downloader', metavar='[PROTO:]NAME', default={}, type='str',
+        action='callback', callback=_dict_from_multiple_values_options_callback,
+        callback_kwargs={
+            'allowed_keys': 'http|ftp|m3u8|dash|rtsp|rtmp|mms',
+            'default_key': 'default', 'process': lambda x: x.strip()},
         help=(
-            'Name or path of the external downloader to use. '
+            'Name or path of the external downloader to use (optionally) prefixed by '
-            'Currently supports %s (Recommended: aria2c)' % ', '.join(list_external_downloaders())))
+            'the protocols (http, ftp, m3u8, dash, rstp, rtmp, mms) to use it for. '
+            'Currently supports native, %s (Recommended: aria2c). '
+            'You can use this option multiple times to set different downloaders for different protocols. '
+            'For example, --downloader aria2c --downloader "dash,m3u8:native" will use '
+            'aria2c for http/ftp downloads, and the native downloader for dash/m3u8 downloads '
+            '(Alias: --external-downloader)' % ', '.join(list_external_downloaders())))
     downloader.add_option(
         '--downloader-args', '--external-downloader-args',
         metavar='NAME:ARGS', dest='external_downloader_args', default={}, type='str',