[devscripts] make_changelog: Various improvements

- Make single items collapse into one line
- Don't hide "Important changes" in `<details>`
- Move upstream merge into priority
- Properly support comma separated prefixes

Authored by: Grub4K

.github/workflows/publish.yml

@@ -54,9 +54,7 @@ jobs:
           cat >> ./RELEASE_NOTES << EOF
           #### A description of the various files are in the [README](https://github.com/yt-dlp/yt-dlp#release-files)
           ---
-          <details><summary><h3>Changelog</h3></summary>
+          $(python ./devscripts/make_changelog.py -vv --collapsible)
-          $(python ./devscripts/make_changelog.py -vv)
-          </details>
           EOF
           printf '%s\n\n' '**This is an automated nightly pre-release build**' >> ./NIGHTLY_NOTES
           cat ./RELEASE_NOTES >> ./NIGHTLY_NOTES

devscripts/make_changelog.py

@@ -26,7 +26,6 @@ logger = logging.getLogger(__name__)
 
 
 class CommitGroup(enum.Enum):
-    UPSTREAM = None
     PRIORITY = 'Important'
     CORE = 'Core'
     EXTRACTOR = 'Extractor'
@@ -34,6 +33,11 @@ class CommitGroup(enum.Enum):
     POSTPROCESSOR = 'Postprocessor'
     MISC = 'Misc.'
 
+    @classmethod
+    @property
+    def ignorable_prefixes(cls):
+        return ('core', 'downloader', 'extractor', 'misc', 'postprocessor', 'upstream')
+
     @classmethod
     @lru_cache
     def commit_lookup(cls):
@@ -41,7 +45,6 @@ class CommitGroup(enum.Enum):
             name: group
             for group, names in {
                 cls.PRIORITY: {''},
-                cls.UPSTREAM: {'upstream'},
                 cls.CORE: {
                     'aes',
                     'cache',
@@ -54,6 +57,7 @@ class CommitGroup(enum.Enum):
                     'outtmpl',
                     'plugins',
                     'update',
+                    'upstream',
                     'utils',
                 },
                 cls.MISC: {
@@ -111,22 +115,36 @@ class CommitInfo:
         return ((self.details or '').lower(), self.sub_details, self.message)
 
 
+def unique(items):
+    return sorted({item.strip().lower(): item for item in items if item}.values())
+
+
 class Changelog:
     MISC_RE = re.compile(r'(?:^|\b)(?:lint(?:ing)?|misc|format(?:ting)?|fixes)(?:\b|$)', re.IGNORECASE)
+    ALWAYS_SHOWN = (CommitGroup.PRIORITY,)
 
-    def __init__(self, groups, repo):
+    def __init__(self, groups, repo, collapsible=False):
         self._groups = groups
         self._repo = repo
+        self._collapsible = collapsible
 
     def __str__(self):
         return '\n'.join(self._format_groups(self._groups)).replace('\t', '    ')
 
     def _format_groups(self, groups):
+        first = True
         for item in CommitGroup:
+            if self._collapsible and item not in self.ALWAYS_SHOWN and first:
+                first = False
+                yield '\n<details><summary><h3>Changelog</h3></summary>\n'
+
             group = groups[item]
             if group:
                 yield self.format_module(item.value, group)
 
+        if self._collapsible:
+            yield '\n</details>'
+
     def format_module(self, name, group):
         result = f'\n#### {name} changes\n' if name else '\n'
         return result + '\n'.join(self._format_group(group))
@@ -137,62 +155,52 @@ class Changelog:
         for _, items in detail_groups:
             items = list(items)
             details = items[0].details
-            if not details:
-                indent = ''
-            else:
-                yield f'- {details}'
-                indent = '\t'
 
             if details == 'cleanup':
-                items, cleanup_misc_items = self._filter_cleanup_misc_items(items)
+                items = self._prepare_cleanup_misc_items(items)
+
+            prefix = '-'
+            if details:
+                if len(items) == 1:
+                    prefix = f'- **{details}**:'
+                else:
+                    yield f'- **{details}**'
+                    prefix = '\t-'
 
             sub_detail_groups = itertools.groupby(items, lambda item: tuple(map(str.lower, item.sub_details)))
             for sub_details, entries in sub_detail_groups:
                 if not sub_details:
                     for entry in entries:
-                        yield f'{indent}- {self.format_single_change(entry)}'
+                        yield f'{prefix} {self.format_single_change(entry)}'
                     continue
 
                 entries = list(entries)
-                prefix = f'{indent}- {", ".join(entries[0].sub_details)}'
+                sub_prefix = f'{prefix} {", ".join(entries[0].sub_details)}'
                 if len(entries) == 1:
-                    yield f'{prefix}: {self.format_single_change(entries[0])}'
+                    yield f'{sub_prefix}: {self.format_single_change(entries[0])}'
                     continue
 
-                yield prefix
+                yield sub_prefix
                 for entry in entries:
-                    yield f'{indent}\t- {self.format_single_change(entry)}'
+                    yield f'\t{prefix} {self.format_single_change(entry)}'
 
-            if details == 'cleanup' and cleanup_misc_items:
+    def _prepare_cleanup_misc_items(self, items):
-                yield from self._format_cleanup_misc_sub_group(cleanup_misc_items)
-
-    def _filter_cleanup_misc_items(self, items):
         cleanup_misc_items = defaultdict(list)
-        non_misc_items = []
+        sorted_items = []
         for item in items:
             if self.MISC_RE.search(item.message):
                 cleanup_misc_items[tuple(item.commit.authors)].append(item)
             else:
-                non_misc_items.append(item)
+                sorted_items.append(item)
 
-        return non_misc_items, cleanup_misc_items
+        for commit_infos in cleanup_misc_items.values():
-
+            sorted_items.append(CommitInfo(
-    def _format_cleanup_misc_sub_group(self, group):
+                'cleanup', ('Miscellaneous',), ', '.join(
-        prefix = '\t- Miscellaneous'
-        if len(group) == 1:
-            yield f'{prefix}: {next(self._format_cleanup_misc_items(group))}'
-            return
-
-        yield prefix
-        for message in self._format_cleanup_misc_items(group):
-            yield f'\t\t- {message}'
-
-    def _format_cleanup_misc_items(self, group):
-        for authors, infos in group.items():
-            message = ', '.join(
                     self._format_message_link(None, info.commit.hash)
-                for info in sorted(infos, key=lambda item: item.commit.hash or ''))
+                    for info in sorted(commit_infos, key=lambda item: item.commit.hash or '')),
-            yield f'{message} by {self._format_authors(authors)}'
+                [], Commit(None, '', commit_infos[0].commit.authors), []))
+
+        return sorted_items
 
     def format_single_change(self, info):
         message = self._format_message_link(info.message, info.commit.hash)
@@ -236,12 +244,8 @@ class CommitRange:
 
     AUTHOR_INDICATOR_RE = re.compile(r'Authored by:? ', re.IGNORECASE)
     MESSAGE_RE = re.compile(r'''
-        (?:\[
+        (?:\[(?P<prefix>[^\]]+)\]\ )?
-            (?P<prefix>[^\]\/:,]+)
+        (?:(?P<sub_details>`?[^:`]+`?): )?
-            (?:/(?P<details>[^\]:,]+))?
-            (?:[:,](?P<sub_details>[^\]]+))?
-        \]\ )?
-        (?:(?P<sub_details_alt>`?[^:`]+`?): )?
         (?P<message>.+?)
         (?:\ \((?P<issues>\#\d+(?:,\ \#\d+)*)\))?
         ''', re.VERBOSE | re.DOTALL)
@@ -340,27 +344,64 @@ class CommitRange:
         self._commits = {key: value for key, value in reversed(self._commits.items())}
 
     def groups(self):
-        groups = defaultdict(list)
+        group_dict = defaultdict(list)
         for commit in self:
-            upstream_re = self.UPSTREAM_MERGE_RE.match(commit.short)
+            upstream_re = self.UPSTREAM_MERGE_RE.search(commit.short)
             if upstream_re:
-                commit.short = f'[upstream] Merge up to youtube-dl {upstream_re.group(1)}'
+                commit.short = f'[upstream] Merged with youtube-dl {upstream_re.group(1)}'
 
             match = self.MESSAGE_RE.fullmatch(commit.short)
             if not match:
                 logger.error(f'Error parsing short commit message: {commit.short!r}')
                 continue
 
-            prefix, details, sub_details, sub_details_alt, message, issues = match.groups()
+            prefix, sub_details_alt, message, issues = match.groups()
-            group = None
+            issues = [issue.strip()[1:] for issue in issues.split(',')] if issues else []
-            if prefix:
-                if prefix == 'priority':
-                    prefix, _, details = (details or '').partition('/')
-                    logger.debug(f'Priority: {message!r}')
-                    group = CommitGroup.PRIORITY
 
-                if not details and prefix:
+            if prefix:
-                    if prefix not in ('core', 'downloader', 'extractor', 'misc', 'postprocessor', 'upstream'):
+                groups, details, sub_details = zip(*map(self.details_from_prefix, prefix.split(',')))
+                group = next(iter(filter(None, groups)), None)
+                details = ', '.join(unique(details))
+                sub_details = list(itertools.chain.from_iterable(sub_details))
+            else:
+                group = CommitGroup.CORE
+                details = None
+                sub_details = []
+
+            if sub_details_alt:
+                sub_details.append(sub_details_alt)
+            sub_details = tuple(unique(sub_details))
+
+            if not group:
+                if self.EXTRACTOR_INDICATOR_RE.search(commit.short):
+                    group = CommitGroup.EXTRACTOR
+                else:
+                    group = CommitGroup.POSTPROCESSOR
+                logger.warning(f'Failed to map {commit.short!r}, selected {group.name.lower()}')
+
+            commit_info = CommitInfo(
+                details, sub_details, message.strip(),
+                issues, commit, self._fixes[commit.hash])
+
+            logger.debug(f'Resolved {commit.short!r} to {commit_info!r}')
+            group_dict[group].append(commit_info)
+
+        return group_dict
+
+    @staticmethod
+    def details_from_prefix(prefix):
+        if not prefix:
+            return CommitGroup.CORE, None, ()
+
+        prefix, _, details = prefix.partition('/')
+        prefix = prefix.strip().lower()
+        details = details.strip()
+
+        group = CommitGroup.get(prefix)
+        if group is CommitGroup.PRIORITY:
+            prefix, _, details = details.partition('/')
+
+        if not details and prefix and prefix not in CommitGroup.ignorable_prefixes:
             logger.debug(f'Replaced details with {prefix!r}')
             details = prefix or None
 
@@ -368,32 +409,11 @@ class CommitRange:
             details = None
 
         if details:
-                    details = details.strip()
+            details, *sub_details = details.split(':')
-
         else:
-                group = CommitGroup.CORE
+            sub_details = []
 
-            sub_details = f'{sub_details or ""},{sub_details_alt or ""}'.replace(':', ',')
+        return group, details, sub_details
-            sub_details = tuple(filter(None, map(str.strip, sub_details.split(','))))
-
-            issues = [issue.strip()[1:] for issue in issues.split(',')] if issues else []
-
-            if not group:
-                group = CommitGroup.get(prefix.lower())
-                if not group:
-                    if self.EXTRACTOR_INDICATOR_RE.search(commit.short):
-                        group = CommitGroup.EXTRACTOR
-                    else:
-                        group = CommitGroup.POSTPROCESSOR
-                    logger.warning(f'Failed to map {commit.short!r}, selected {group.name}')
-
-            commit_info = CommitInfo(
-                details, sub_details, message.strip(),
-                issues, commit, self._fixes[commit.hash])
-            logger.debug(f'Resolved {commit.short!r} to {commit_info!r}')
-            groups[group].append(commit_info)
-
-        return groups
 
 
 def get_new_contributors(contributors_path, commits):
@@ -444,6 +464,9 @@ if __name__ == '__main__':
     parser.add_argument(
         '--repo', default='yt-dlp/yt-dlp',
         help='the github repository to use for the operations (default: %(default)s)')
+    parser.add_argument(
+        '--collapsible', action='store_true',
+        help='make changelog collapsible (default: %(default)s)')
     args = parser.parse_args()
 
     logging.basicConfig(
@@ -467,4 +490,4 @@ if __name__ == '__main__':
             write_file(args.contributors_path, '\n'.join(new_contributors) + '\n', mode='a')
         logger.info(f'New contributors: {", ".join(new_contributors)}')
 
-    print(Changelog(commits.groups(), args.repo))
+    print(Changelog(commits.groups(), args.repo, args.collapsible))