Generating release notes and hill-climbing to improve them

const START_DATE = "2023-11-26";
const END_DATE = "2024-01-27";
const REPO_OWNER = "braintrustdata";
const REPO_NAME = "braintrust-sdk";

import { Octokit } from "@octokit/rest";
import { GetResponseTypeFromEndpointMethod } from "@octokit/types";
 
type CommitsResponse = GetResponseTypeFromEndpointMethod<
  typeof octokit.rest.repos.listCommits
>;
type Commit = CommitsResponse["data"][number];
 
// Octokit.js
// https://github.com/octokit/core.js#readme
const octokit: Octokit = new Octokit({
  auth: process.env.GITHUB_ACCESS_TOKEN || "Your Github Access Token",
});
 
const commits: CommitsResponse = await octokit.rest.repos.listCommits({
  owner: REPO_OWNER,
  repo: REPO_NAME,
  since: START_DATE,
  until: END_DATE,
  per_page: 1000,
});
 
console.log("Retrieved", commits.data.length, "commits");

import moment from "moment";
 
interface CommitInfo {
  url: string;
  html_url: string;
  sha: string;
  commit: {
    author: {
      name?: string;
      email?: string;
      date?: string;
    };
    message: string;
  };
}
 
const weeks: Record<string, CommitInfo[]> = {};
for (const commit of commits.data) {
  const week = moment(commit.commit.author.date, "YYYY-MM-DD")
    .startOf("week")
    .format("YYYY-MM-DD");
  weeks[week] = (weeks[week] || []).concat([
    // Simplify the commit data structure
    {
      sha: commit.sha,
      url: commit.url,
      html_url: commit.html_url,
      commit: {
        author: commit.commit.author,
        message: commit.commit.message,
      },
    },
  ]);
}
 
const sortedWeeks = Object.keys(weeks).sort((a, b) =>
  moment(a).diff(moment(b))
);
for (const week of sortedWeeks) {
  console.log(week, weeks[week].length);
  weeks[week].sort((a, b) =>
    moment(a.commit.author.date).diff(moment(b.commit.author.date))
  );
}

const firstWeek = weeks[sortedWeeks[0]];
for (const commit of firstWeek) {
  console.log("-----", commit.sha, "-----");
  console.log(commit.html_url);
  console.log(commit.commit.author.date);
  console.log(commit.commit.message);
  console.log("\n");
}

import { wrapOpenAI } from "braintrust";
import { OpenAI } from "openai";
 
const client = wrapOpenAI(
  new OpenAI({
    apiKey: process.env.OPENAI_API_KEY || "Your OpenAI API Key",
  })
);
 
const MODEL: string = "gpt-3.5-turbo";
const SEED = 123;

import { ChatCompletionMessageParam } from "openai/resources";
import { traced } from "braintrust";
 
function serializeCommit(info: CommitInfo): string {
  return `SHA: ${info.sha}
AUTHOR: ${info.commit.author.name} <${info.commit.author.email}>
DATE: ${info.commit.author.date}
MESSAGE: ${info.commit.message}`;
}
 
function generatePrompt(commits: CommitInfo[]): ChatCompletionMessageParam[] {
  return [
    {
      role: "system",
      content: `You are an expert technical writer who generates release notes for the Braintrust SDK.
You will be provided a list of commits, including their message, author, and date, and you will generate
a full list of release notes, in markdown list format, across the commits. You should include the important
details, but if a commit is not relevant to the release notes, you can skip it.`,
    },
    {
      role: "user",
      content:
        "Commits: \n" + commits.map((c) => serializeCommit(c)).join("\n\n"),
    },
  ];
}
 
async function generateReleaseNotes(input: CommitInfo[]) {
  return traced(
    async (span) => {
      const response = await client.chat.completions.create({
        model: MODEL,
        messages: generatePrompt(input),
        seed: SEED,
      });
      return response.choices[0].message.content;
    },
    {
      name: "generateReleaseNotes",
    }
  );
}
 
const releaseNotes = await generateReleaseNotes(firstWeek);
console.log(releaseNotes);

import { LLMClassifierFromTemplate, Scorer, Score } from "autoevals";
 
const GRADER: string = "gpt-4";
 
const promptTemplate = `You are a technical writer who helps assess how effectively a product team generates
release notes based on git commits. You will look at the commit messages and determine if the release
notes sufficiently cover the changes.
 
Messages:
 
{{input}}
 
 
Release Notes:
 
{{output}}
 
Assess the quality of the release notes by selecting one of the following options. As you think through
the changes, list out which messages are not included in the release notes or info that is made up.
 
a) The release notes are excellent and cover all the changes.
b) The release notes capture some, but not all, of the changes.
c) The release notes include changes that are not in the commit messages.
d) The release notes are not useful and do not cover any changes.`;
 
const evaluator: Scorer<any, { input: string; output: string }> =
  LLMClassifierFromTemplate<{ input: string }>({
    name: "Comprehensiveness",
    promptTemplate,
    choiceScores: { a: 1, b: 0.5, c: 0.25, d: 0 },
    useCoT: true,
    model: GRADER,
  });
 
async function comprehensiveness({
  input,
  output,
}: {
  input: CommitInfo[];
  output: string;
}): Promise<Score> {
  return evaluator({
    input: input.map((c) => "-----\n" + c.commit.message).join("\n\n"),
    output,
  });
}
 
await comprehensiveness({ input: firstWeek, output: releaseNotes });

const promptTemplate = `You are a technical writer who helps assess the writing quality of release notes.
 
Release Notes:
 
{{output}}
 
Assess the quality of the release notes by selecting one of the following options. As you think through
the changes, list out which messages are not included in the release notes or info that is made up.
 
a) The release notes are clear and concise.
b) The release notes are not formatted as markdown/html, but otherwise are well written.
c) The release notes contain superfluous wording, for example statements like "let me know if you have any questions".
d) The release notes contain repeated information.
e) The release notes are off-topic to Braintrust's software and do not contain relevant information.`;
 
const evaluator: Scorer<any, { output: string }> = LLMClassifierFromTemplate({
  name: "WritingQuality",
  promptTemplate,
  choiceScores: { a: 1, b: 0.75, c: 0.5, d: 0.25, e: 0 },
  useCoT: true,
  model: GRADER,
});
 
async function writingQuality({ output }: { output: string }): Promise<Score> {
  return evaluator({
    output,
  });
}
 
await writingQuality({ output: releaseNotes });

import { Eval } from "braintrust";
 
let lastExperiment = await Eval<CommitInfo[], string, unknown>(
  "Release Notes Cookbook",
  {
    data: Object.entries(weeks).map(([week, commits]) => ({
      input: commits,
      metadata: { week },
    })),
    task: generateReleaseNotes,
    scores: [comprehensiveness, writingQuality],
  }
);

function generatePrompt(commits: CommitInfo[]): ChatCompletionMessageParam[] {
  return [
    {
      role: "system",
      content: `You are an expert technical writer who generates release notes for the Braintrust SDK.
You will be provided a list of commits, including their message, author, and date, and you will generate
a full list of release notes, in markdown list format, across the commits. You should make sure to include
some information about each commit, without the commit sha, url, or author info.`,
    },
    {
      role: "user",
      content:
        "Commits: \n" + commits.map((c) => serializeCommit(c)).join("\n\n"),
    },
  ];
}
 
async function generateReleaseNotes(input: CommitInfo[]) {
  return traced(
    async (span) => {
      const response = await client.chat.completions.create({
        model: MODEL,
        messages: generatePrompt(input),
        seed: SEED,
      });
      return response.choices[0].message.content;
    },
    {
      name: "generateReleaseNotes",
    }
  );
}
 
await generateReleaseNotes(firstWeek);

import { BaseExperiment } from "braintrust";
import { Summary } from "autoevals";
 
async function releaseSummary({
  input,
  output,
  expected,
}: {
  input: CommitInfo[];
  output: string;
  expected: string;
}): Promise<Score> {
  return Summary({
    input: input.map((c) => "-----\n" + c.commit.message).join("\n\n"),
    output,
    expected,
    model: GRADER,
    useCoT: true,
  });
}
 
lastExperiment = await Eval<CommitInfo[], string, unknown>(
  "Release Notes Cookbook",
  {
    data: BaseExperiment({ name: lastExperiment.experimentName }),
    task: generateReleaseNotes,
    scores: [comprehensiveness, writingQuality, releaseSummary],
  }
);

import { ChatCompletionMessageParam } from "openai/resources";
import { traced } from "braintrust";
 
function generatePrompt(commits: CommitInfo[]): ChatCompletionMessageParam[] {
  return [
    {
      role: "system",
      content: `You are an expert technical writer who generates release notes for the Braintrust SDK.
You will be provided a list of commits, including their message, author, and date, and you will generate
a full list of release notes, in markdown list format, across the commits. You should make sure to include
some information about each commit, without the commit sha, url, or author info. However, do not mention
version bumps multiple times. If there are multiple version bumps, only mention the latest one.`,
    },
    {
      role: "user",
      content:
        "Commits: \n" + commits.map((c) => serializeCommit(c)).join("\n\n"),
    },
  ];
}
 
async function generateReleaseNotes(input: CommitInfo[]) {
  return traced(
    async (span) => {
      const response = await client.chat.completions.create({
        model: MODEL,
        messages: generatePrompt(input),
        seed: SEED,
      });
      return response.choices[0].message.content;
    },
    {
      name: "generateReleaseNotes",
    }
  );
}
 
const releaseNotes = await generateReleaseNotes(firstWeek);
console.log(releaseNotes);

lastExperiment = await Eval<CommitInfo[], string, unknown>(
  "Release Notes Cookbook",
  {
    data: BaseExperiment({ name: lastExperiment.experimentName }),
    task: generateReleaseNotes,
    scores: [comprehensiveness, writingQuality, releaseSummary],
  }
);

async function generateReleaseNotes(input: CommitInfo[]) {
  return traced(
    async (span) => {
      const response = await client.chat.completions.create({
        model: "gpt-4-1106-preview",
        messages: generatePrompt(input),
        seed: SEED,
      });
      return response.choices[0].message.content;
    },
    {
      name: "generateReleaseNotes",
    }
  );
}

lastExperiment = await Eval<CommitInfo[], string, unknown>(
  "Release Notes Cookbook",
  {
    data: BaseExperiment({ name: lastExperiment.experimentName }),
    task: generateReleaseNotes,
    scores: [comprehensiveness, writingQuality, releaseSummary],
  }
);