Uploaded image for project: 'Compass '
  1. Compass
  2. COMPASS-1655

Automate changelog generation for releases

    • Type: Icon: Task Task
    • Resolution: Gone away
    • Priority: Icon: Major - P3 Major - P3
    • No version
    • Affects Version/s: None
    • Component/s: CI, Tech debt
    • Labels:
      None
    • Not Needed

      My current script for doing this is below. Imagine it could live in hadron-build. An example of what this generates: https://github.com/10gen/compass/releases/tag/v1.7.0

      High-level:

      1. Use GitHub API to compare 2 branches (in this case, we were promoting 1.7.0 to stable, for patches we can just compare 2 version tags)
      2. Parse commits from the result of compare to extract Pull Request id and JIRA ticket id
      3. Use GitHub API to get details of Pull Request if id available (lazy cache the response json on disk)
      4. A commit is determined to be "featured" if it contains an opening block for an image (iknowrite?!) likely a screenshot or screencap gif. This could be vastly improved (e.g. sort byo # of reactions on Pull Request
      5. More searching and cleaning pulling request to extract JIRA id's from pull request metadata, replace them with markdown links to the ticket
      6. While doing all of this parsing, keep track of some basic stats (# pull requests, # JIRA tickets seen). Would be nice if this also hit JIRA api to facet JIRA stats for release by bug/task/epic and also make sure all tickets are closed+have correct fixVersion.

      var _ = require('lodash');
      var GitHub = require('github');
      var github = new GitHub({ version: '3.0.0', 'User-Agent': 'hadron-build' });
      
      github.authenticate({
        token: process.env.GITHUB_TOKEN,
        type: 'oauth'
      });
      
      var owner = '10gen';
      var repo = 'compass';
      var base = '1.6-releases';
      var head = '1.7-releases';
      
      var changelog = {
        compare_url: `https://github.com/${owner}/${repo}/compare/${base}...${head}`,
        stats: {
          total_commits: 0,
          ignored_commits: 0,
          include_jira: 0,
          include_jira_unique: 0,
          include_pr: 0,
          include_pr_unique: 0
        },
        ignored: [],
        jira: [],
        pr: [],
        featured: [],
        messages: [],
        other: []
      };
      
      function extractLinksFromCommit(commit) {
        // console.log('parse', commit);
        var message = _.trimStart(_.first(commit.commit.message.split('\n')));
        var res = {
          sha: commit.sha,
          original: message,
          message: message,
          jira: undefined,
          pr: undefined,
          author: commit.author.name
        };
      
        var m = /compass[-\s]?(\d+)/gi.exec(res.message);
        if (m) {
          res.jira = `COMPASS-${m[1]}`.toUpperCase();
          res.message = res.message.replace(m[0], `[${res.jira}][${res.jira}]`);
        }
      
        var pr = /\(#(\d+)\)/.exec(res.message);
        if (pr) {
          res.pr = pr[1];
          res.message = res.message.replace(
            `#${pr[1]}`,
            `[#${res.pr}][pr-${res.pr}]`
          );
        }
      
        return res;
      }
      
      var async = require('async');
      var fs = require('fs');
      var path = require('path');
      
      function getPullRequestDetail(pr, cb) {
        var dest = path.join(__dirname, '..', 'pr-cache', `${pr}.json`);
      
        fs.readFile(dest, function(err, d) {
          if (!err) {
            return cb(null, JSON.parse(d));
          }
      
          github.pullRequests.get(
            { owner: owner, repo: repo, number: Number(pr) },
            function(err, res) {
              if (err) return cb(err);
      
              fs.writeFile(dest, JSON.stringify(res, null, 2), function(err) {
                if (err) return cb(err);
                cb(null, res);
              });
            }
          );
        });
      }
      
      function getAllPullRequests(done) {
        var tasks = changelog.pr.map(function(pr) {
          return function(cb) {
            getPullRequestDetail(pr, cb);
          };
        });
        return async.series(tasks, done);
      }
      
      function parseCommits(commits, done) {
        changelog.stats.total_commits = commits.length;
      
        var tasks = commits.map(function(commit) {
          return function(cb) {
            var res = extractLinksFromCommit(commit);
            if (!res.jira && !res.pr) {
              changelog.ignored.push(res);
              changelog.stats.ignored_commits += 1;
              return cb();
            }
      
            if (_.startsWith(res.message, 'Revert')) {
              changelog.ignored.push(res);
              changelog.stats.ignored_commits += 1;
              return cb();
            }
      
            if (res.jira) {
              if (changelog.jira.indexOf(res.jira) === -1) {
                changelog.stats.include_jira_unique += 1;
              }
              changelog.stats.include_jira += 1;
              changelog.jira.push(res.jira);
            }
      
            if (res.pr) {
              if (changelog.pr.indexOf(res.pr) === -1) {
                changelog.stats.include_pr_unique += 1;
              }
              changelog.stats.include_pr += 1;
              changelog.pr.push(res.pr);
            }
            if (
              res.pr &&
              res.jira &&
              changelog.messages.indexOf(res.message) === -1
            ) {
              changelog.messages.push(res.message);
              getPullRequestDetail(res.pr, function(_err, detail) {
                if (detail.body.indexOf('![') > -1) {
                  changelog.featured.push({ title: res.message, body: detail.body });
                }
                cb();
              });
            } else {
              if (changelog.other.indexOf(res.message) === -1) {
                changelog.other.push(res.message);
              }
              return cb();
            }
          };
        });
      
        async.series(tasks, done);
      }
      
      github.repos.compareCommits(
        {
          owner: owner,
          repo: repo,
          base: base,
          head: head
        },
        function(err, res) {
          if (err) return console.log('err', err);
          changelog.messages = changelog.messages.sort();
          changelog.pr = changelog.pr.sort();
          changelog.jira = changelog.jira.sort();
      
          parseCommits(res.commits, function(_err) {
            if (_err) return console.log('err', _err);
      
            var contents = '';
            contents += '\n\n### Featured\n\n';
            changelog.featured.forEach(function(f) {
              contents += `#### ${f.title}\n\n${f.body}\n\n`;
            });
      
            contents += `\n\n### Whats New ${changelog.messages.length}\n\n`;
            changelog.messages.forEach(function(message) {
              contents += `- ${message}\n`;
            });
      
            contents += `\n\n### Other Changes ${changelog.other.length}\n\n`;
      
            changelog.other.forEach(function(message) {
              contents += `- ${message}\n`;
            });
      
            contents += `\n\n
          <!--
          Pull Requests:
            Total: ${changelog.stats.include_pr}
            Unique: ${changelog.stats.include_pr_unique}
          -->
        `;
      
            changelog.pr.forEach(function(pr) {
              contents += `[pr-${pr}]: https://github.com/10gen/compass/pull/${pr}\n`;
            });
      
            contents += `\n\n
          <!--
          JIRA Tickets:
            Total: ${changelog.stats.include_jira}
            Unique: ${changelog.stats.include_jira_unique}
          -->
        `;
            changelog.jira.forEach(function(jiraId) {
              contents += `[${jiraId}]: https://jira.mongodb.org/browse/${jiraId}\n`;
            });
      
            console.log(contents);
          });
        }
      );
      

            Assignee:
            Unassigned Unassigned
            Reporter:
            lucas.hrabovsky Lucas Hrabovsky (Inactive)
            Votes:
            0 Vote for this issue
            Watchers:
            3 Start watching this issue

              Created:
              Updated:
              Resolved: