1
/**
2
 * PR split cost: what splitting a change into k pull requests costs in waiting for review.
3
 *
4
 * Paste (size, wait) pairs for your own merged PRs: lines changed, and hours from ready-for-review to the first
5
 * review by someone else. The tool fits a straight line to log(wait) against log(size). Its slope e is the
6
 * elasticity: double a PR and its wait is multiplied by 2^e. Busy open-source repositories come out between about
7
 * 0.2 and 0.45: the wait grows much more slowly than size.
8
 *
9
 * From e, three ways a change split into k PRs can be reviewed, each against one PR of the whole change:
10
 *
11
 *  - one after another: each piece waits its own pickup. The expected total is k x (1/k)^e = k^(1-e) times one
12
 *    PR's, exactly, under the fit, whatever the scatter around it, because expected waits add.
13
 *  - side by side, independently: the change waits for the slowest of k pickups. That depends on the scatter, not
14
 *    just on e, so it is computed exactly from the fit's own residuals: the expected largest of k draws.
15
 *  - in one sitting: one reviewer takes all k together, so the change waits for one pickup of a PR k times smaller,
16
 *    k^(-e). That assumes the reviewer starts the stack as soon as they would start one small piece.
17
 *
18
 * All three read e as if size drove the wait. The tool also prints the other end, as if size had no effect: k, the
19
 * slowest of k ordinary pickups, and 1. The truth for a team is somewhere between.
20
 *
21
 * ⚠️ What it is not: a causal model. The fit is a correlation, usually a weak one, and small PRs are often small
22
 * fixes by maintainers, picked up fast for that reason. The tool reports r and how much of the variation in wait size
23
 * explains, so the weakness is on the screen beside the answer.
24
 *
25
 * The result leads with the three ways side by side at the reader's k, slowest first, at both ends. Right under it is
26
 * how weak the fit is, because a 4.76x that rests on a fit explaining 3.5% of the variation must not read as a
27
 * measurement. Then the fit's numbers, the charts, what each way assumes, and a markdown copy for a PR description.
28
 * "In one sitting" is drawn muted, never green: its small multiple rests on an assumption about the reviewer, not on
29
 * the data. Nothing else is toned: no limit the reader typed decides that a multiple is bad.
30
 *
31
 * Seeded: the bootstrap and the draws use mulberry32 from a constant, so the same input always gives the same output.
32
 */
33
import type { Field, Output, Series, Tool } from "@toolbench/sdk";
34

35
type Input = { pairs: string; source: string; elasticity: number; lines: number; split: number; review: string };
36

37
class InputError extends Error {
38
  readonly input: string;
39
  readonly at: number | undefined;
40
  readonly len: number | undefined;
41

42
  constructor(message: string, input: string, at?: number, len?: number) {
43
    super(message);
44
    this.name = "InputError";
45
    this.input = input;
46
    this.at = at;
47
    this.len = len;
48
  }
49
}
50

51
const FLOOR_H = 0.05;
52
const BOOTSTRAPS = 1000;
53
const MIN_PAIRS = 10;
54

55
function rng(seed: number): () => number {
56
  let state = seed >>> 0;
57
  return () => {
58
    state = (state + 0x6d2b79f5) >>> 0;
59
    let t = state;
60
    t = Math.imul(t ^ (t >>> 15), t | 1);
61
    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
62
    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
63
  };
64
}
65

66
/** One pair per line: size and hours, separated by a comma, tab, semicolon or spaces. A header line is skipped. */
67
function parse(text: string): { size: number; wait: number }[] {
68
  const out: { size: number; wait: number }[] = [];
69
  let offset = 0;
70
  const lines = text.split("\n");
71
  for (let i = 0; i < lines.length; i++) {
72
    const line = lines[i]!;
73
    const at = offset;
74
    offset += line.length + 1;
75
    const trimmed = line.trim();
76
    if (trimmed === "") continue;
77
    const cells = trimmed.split(/\s*[,;\t]\s*|\s+/);
78
    if (out.length === 0 && i === lines.findIndex((l) => l.trim() !== "") && cells.some((c) => Number.isNaN(Number(c)))) {
79
      continue; // a header such as size,wait_hours
80
    }
81
    const start = at + line.indexOf(trimmed[0]!);
82
    if (cells.length < 2) throw new InputError(`Line ${i + 1} has one number; each line needs a size and a wait.`, "pairs", start, trimmed.length);
83
    const size = Number(cells[0]);
84
    const wait = Number(cells[1]);
85
    if (!Number.isFinite(size) || !Number.isFinite(wait) || cells[0] === "" || cells[1] === "") {
86
      throw new InputError(`Line ${i + 1} isn't two numbers: "${trimmed.slice(0, 30)}".`, "pairs", start, trimmed.length);
87
    }
88
    if (size < 0 || wait < 0) throw new InputError(`Line ${i + 1} has a negative number.`, "pairs", start, trimmed.length);
89
    out.push({ size: Math.max(1, size), wait: Math.max(FLOOR_H, wait) });
90
  }
91
  return out;
92
}
93

94
type Fit = { e: number; a: number; r: number };
95

96
function fitLogs(xs: Float64Array, ys: Float64Array, pick?: Uint32Array): Fit {
97
  const n = pick ? pick.length : xs.length;
98
  let sx = 0;
99
  let sy = 0;
100
  for (let i = 0; i < n; i++) {
101
    const j = pick ? pick[i]! : i;
102
    sx += xs[j]!;
103
    sy += ys[j]!;
104
  }
105
  const mx = sx / n;
106
  const my = sy / n;
107
  let sxx = 0;
108
  let sxy = 0;
109
  let syy = 0;
110
  for (let i = 0; i < n; i++) {
111
    const j = pick ? pick[i]! : i;
112
    const dx = xs[j]! - mx;
113
    const dy = ys[j]! - my;
114
    sxx += dx * dx;
115
    sxy += dx * dy;
116
    syy += dy * dy;
117
  }
118
  const e = sxy / sxx;
119
  return { e, a: my - e * mx, r: syy > 0 ? sxy / Math.sqrt(sxx * syy) : 0 };
120
}
121

122
/**
123
 * Expected largest of k independent draws from `sorted` (ascending), over their mean. Exact, not sampled: the largest
124
 * of k is the i-th smallest with probability (i/n)^k - ((i-1)/n)^k. No randomness, so the answer can't depend on what
125
 * else was computed first, which a shared sampler did.
126
 */
127
function slowestOf(k: number, sorted: Float64Array, mean: number): number {
128
  const n = sorted.length;
129
  let total = 0;
130
  let below = 0;
131
  for (let i = 1; i <= n; i++) {
132
    const upto = (i / n) ** k;
133
    total += sorted[i - 1]! * (upto - below);
134
    below = upto;
135
  }
136
  return total / mean;
137
}
138

139
const median = (sorted: Float64Array) => {
140
  const n = sorted.length;
141
  return n % 2 ? sorted[(n - 1) / 2]! : (sorted[n / 2 - 1]! + sorted[n / 2]!) / 2;
142
};
143

144
/** A waiting multiple: "4.76x", "8x", "1x". */
145
const times = (v: number) => {
146
  const r = Math.round(v * 100) / 100;
147
  return Number.isInteger(r) ? `${r}x` : `${r.toFixed(2)}x`;
148
};
149
const hours = (h: number) => (h >= 48 ? `${(h / 24).toFixed(1)} days` : h >= 1 ? `${h.toFixed(1)} h` : `${Math.round(h * 60)} min`);
150

151
type Way = "serial" | "parallel" | "sitting";
152
const WAY_NAME: Record<Way, string> = { serial: "One after another", parallel: "Side by side", sitting: "In one sitting" };
153
const WAY_RESTS_ON: Record<Way, string> = {
154
  serial: "Each piece waits for its own first review, and the next opens when the last one merges.",
155
  parallel: "All the pieces open at once and are picked up independently, so the change waits for the slowest.",
156
  sitting: "One reviewer takes them all together, as soon as they would start one small piece. That is an assumption about how they work, not a measurement.",
157
};
158

159
export default {
160
  run({ pairs, source, elasticity, lines, split, review }: Input, ctx) {
161
    try {
162
      if (!Number.isInteger(split) || split < 1 || split > 50) throw new InputError("Split into a whole number of PRs, from 1 to 50.", "split");
163
      if (!(lines >= 1)) throw new InputError("The change needs at least one line.", "lines");
164
      if (!["serial", "parallel", "sitting"].includes(review)) throw new InputError("Pick how the pieces are reviewed.", "review");
165
      const way = review as Way;
166

167
      const metrics: Field[] = [];
168
      let evidence = "";
169
      const charts: Output[] = [];
170
      let e: number;
171
      let lo: number | undefined;
172
      let hi: number | undefined;
173
      let residuals: Float64Array | undefined;
174
      let residualMean = 1;
175
      let residualMedian = 1;
176
      let waits: Float64Array | undefined;
177
      let waitMean = 1;
178
      let a = 0;
179
      let fitNote = "";
180

181
      if (source === "fit") {
182
        const data = parse(pairs);
183
        if (data.length < MIN_PAIRS) {
184
          throw new InputError(`Paste at least ${MIN_PAIRS} PRs to fit; there ${data.length === 1 ? "is" : "are"} ${data.length}.`, "pairs");
185
        }
186
        const xs = Float64Array.from(data, (d) => Math.log(d.size));
187
        const ys = Float64Array.from(data, (d) => Math.log(d.wait));
188
        if (Math.max(...xs) - Math.min(...xs) < 1e-9) throw new InputError("Every PR is the same size, so there is no slope to fit.", "pairs");
189
        const f = fitLogs(xs, ys);
190
        e = f.e;
191
        a = f.a;
192
        const next = rng(1);
193
        const pick = new Uint32Array(data.length);
194
        const es: number[] = [];
195
        for (let b = 0; b < BOOTSTRAPS; b++) {
196
          ctx?.signal?.throwIfAborted();
197
          for (let i = 0; i < pick.length; i++) pick[i] = Math.floor(next() * data.length);
198
          es.push(fitLogs(xs, ys, pick).e);
199
        }
200
        es.sort((p, q) => p - q);
201
        lo = es[Math.floor(0.025 * BOOTSTRAPS)]!;
202
        hi = es[Math.ceil(0.975 * BOOTSTRAPS) - 1]!;
203
        residuals = Float64Array.from(data, (_, i) => Math.exp(ys[i]! - (a + e * xs[i]!))).sort();
204
        residualMean = residuals.reduce((s, v) => s + v, 0) / residuals.length;
205
        residualMedian = median(residuals);
206
        waits = Float64Array.from(data, (d) => d.wait).sort();
207
        waitMean = waits.reduce((s, v) => s + v, 0) / waits.length;
208
        fitNote = `e = ${e.toFixed(2)} from ${data.length} PRs, r = ${f.r.toFixed(2)}`;
209
        const explained = (f.r * f.r * 100).toFixed(1);
210
        // Weak below r = 0.3, where size accounts for under a tenth of the variation: then the multiples only point.
211
        evidence =
212
          Math.abs(f.r) < 0.3
213
            ? `Directional only: size explains ${explained}% of the variation in wait across ${data.length} PRs (r = ${f.r.toFixed(2)}). A correlation, not a cause.`
214
            : `Size explains ${explained}% of the variation in wait across ${data.length} PRs (r = ${f.r.toFixed(2)}). A correlation, not a cause.`;
215
        const typical = Math.exp(a + e * Math.log(lines));
216
        metrics.push(
217
          { label: "Elasticity e", value: e.toFixed(2), note: `95% bootstrap range ${lo.toFixed(2)} to ${hi.toFixed(2)}` },
218
          { label: "Doubling a PR", value: `${(2 ** e).toFixed(2)}x the wait` },
219
          {
220
            label: `Median wait, one PR of ${lines} lines`,
221
            value: hours(typical * residualMedian),
222
            note: `mean ${hours(typical * residualMean)}${residualMean > 1.1 * residualMedian ? ", pulled up by the slowest few" : ""}`,
223
          },
224
        );
225

226
        // The scatter and the line, both on log axes, because that is where the fit is a line.
227
        const sizes = data.map((d) => d.size);
228
        const minS = Math.min(...sizes);
229
        const maxS = Math.max(...sizes);
230
        const lineX = [minS, Math.sqrt(minS * maxS), maxS];
231
        charts.push({
232
          kind: "series",
233
          chart: {
234
            xLabel: "PR size, lines changed",
235
            yLabel: "Hours to first review",
236
            xScale: "log",
237
            yScale: "log",
238
            x: lineX,
239
            series: [
240
              { label: "Pasted PRs", shape: "points", x: sizes, points: data.map((d) => d.wait) },
241
              { label: `Fit, wait ∝ size^${e.toFixed(2)}`, shape: "line", x: lineX, points: lineX.map((s) => Number(Math.exp(a + e * Math.log(s)).toFixed(3))) },
242
            ],
243
            readout: { mode: "point" },
244
          },
245
        });
246
      } else {
247
        if (!(elasticity >= 0 && elasticity <= 1)) throw new InputError("An elasticity between 0 and 1, please.", "elasticity");
248
        e = elasticity;
249
        fitNote = `e = ${e.toFixed(2)}, as given`;
250
        evidence = `e = ${e.toFixed(2)}, typed in rather than fitted, so nothing here says how well size predicts the wait.`;
251
        metrics.push({ label: "Doubling a PR", value: `${(2 ** e).toFixed(2)}x the wait` });
252
      }
253

254
      const serial = (k: number) => k ** (1 - e);
255
      const sitting = (k: number) => k ** -e;
256
      const parallel = (k: number) => (residuals ? sitting(k) * slowestOf(k, residuals, residualMean) : undefined);
257
      const onFit = (w: Way, k: number) => (w === "serial" ? serial(k) : w === "sitting" ? sitting(k) : parallel(k));
258
      // The other end: size has no effect on the pickup, so every piece waits like an ordinary PR.
259
      const noEffect = (w: Way, k: number) => (w === "serial" ? k : w === "sitting" ? 1 : waits ? slowestOf(k, waits, waitMean) : undefined);
260
      // "In one sitting" rests on an assumption about the reviewer, so it is never shown as good news.
261
      const tone = (w: Way) => (w === "sitting" ? "muted" : "normal");
262

263
      const k = split;
264
      const pieces = `${k} PR${k === 1 ? "" : "s"}`;
265

266
      // The three ways at the reader's k, slowest first, with the unpriceable one last. The pick is named in words.
267
      const ways = (["serial", "parallel", "sitting"] as Way[]).sort((p, q) => (onFit(q, k) ?? -1) - (onFit(p, k) ?? -1));
268
      const comparison: Output = {
269
        kind: "table",
270
        role: "comparison",
271
        caption: `${pieces} instead of one of ${lines} lines: the wait for a first review, as a multiple of the one PR's`,
272
        columns: [
273
          { label: "Reviewed", wrap: "prose", rowHeader: true },
274
          { label: source === "fit" ? "Fit says" : `At e = ${e.toFixed(2)}`, align: "end", mono: true, wrap: "none" },
275
          { label: "If size does not matter", align: "end", mono: true, wrap: "none" },
276
        ],
277
        rows: ways.map((w) => {
278
          const v = onFit(w, k);
279
          const b = noEffect(w, k);
280
          const t = tone(w);
281
          return [
282
            { text: `${WAY_NAME[w]}${w === way ? " (your pick)" : ""}`, tone: t },
283
            { text: v === undefined ? "Paste PRs" : times(v), mono: true, tone: t },
284
            { text: b === undefined ? "Paste PRs" : times(b), mono: true, tone: t },
285
          ];
286
        }),
287
      };
288

289
      // Waiting multiple against k, for the three ways, with 1 (one PR) as the line to beat.
290
      const ks = [1, 2, 3, 4, 6, 8, 12, 16, 24, 32].filter((v) => v <= Math.max(16, k * 2));
291
      if (!ks.includes(k)) ks.push(k);
292
      ks.sort((p, q) => p - q);
293
      const lines3: Series[] = [
294
        { label: "One after another", shape: "line", markers: true, points: ks.map((v) => Number(serial(v).toFixed(3))) },
295
        ...(residuals ? [{ label: "Side by side", shape: "line" as const, markers: true, points: ks.map((v) => Number(parallel(v)!.toFixed(3))) }] : []),
296
        { label: "In one sitting", shape: "line", markers: true, points: ks.map((v) => Number(sitting(v).toFixed(3))) },
297
      ];
298
      charts.unshift({
299
        kind: "series",
300
        chart: {
301
          xLabel: "Split into k PRs",
302
          yLabel: "Waiting for a first review, against one PR",
303
          x: ks,
304
          series: lines3,
305
          thresholds: [{ y: 1, label: "one PR" }],
306
          annotations: [{ x: k, label: `${k} PRs` }],
307
        },
308
      });
309

310
      // What each way assumes, as prose under the numbers rather than a column in the comparison.
311
      const method: Output = {
312
        kind: "fields",
313
        role: "method",
314
        fields: [
315
          ...(["serial", "parallel", "sitting"] as Way[]).map((w): Field => ({ label: WAY_NAME[w], value: WAY_RESTS_ON[w], group: "What each way assumes", mono: false })),
316
          { label: "Measured", value: "Hours from ready for review to the first review by someone else. Not the time to merge.", group: "What it counts", mono: false },
317
        ],
318
      };
319

320
      // For a PR description or a design doc, where a tab-separated table doesn't render.
321
      const md = [
322
        `**Splitting a change of ${lines} lines into ${pieces}** (${fitNote})`,
323
        "",
324
        "| Reviewed | If size drives the pickup | If size has no effect |",
325
        "|---|---|---|",
326
        ...(["serial", "parallel", "sitting"] as Way[]).map((w) => {
327
          const v = onFit(w, k);
328
          const b = noEffect(w, k);
329
          return `| ${WAY_NAME[w]} | ${v === undefined ? "needs PRs" : times(v)} | ${b === undefined ? "needs PRs" : times(b)} |`;
330
        }),
331
        "",
332
        "Expected waiting for a first review, against one PR of the whole change. A correlation, not a cause. In one sitting assumes one reviewer takes every piece together.",
333
      ].join("\n");
334

335
      const parts: Output[] = [
336
        comparison,
337
        // Plain text, not an answer: the caveat belongs beside the numbers, not louder than them.
338
        { kind: "text", text: evidence },
339
        { kind: "fields", role: "metrics", fields: metrics },
340
        ...charts,
341
        method,
342
        { kind: "code", lang: "markdown", source: md },
343
      ];
344
      return { kind: "group", parts };
345
    } catch (error) {
346
      if (error instanceof InputError) {
347
        return {
348
          kind: "error",
349
          message: error.message,
350
          input: error.input,
351
          ...(error.at !== undefined ? { at: error.at } : {}),
352
          ...(error.len !== undefined ? { len: error.len } : {}),
353
        };
354
      }
355
      throw error;
356
    }
357
  },
358
} satisfies Tool<Input>;