benchmarks.yml

name: Benchmark

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

concurrency:
  group: benchmarks-${{ github.head_ref || github.ref }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

jobs:
  benchmarks:
    name: run
    runs-on: ubuntu-latest

    # The last step posts the benchmark comparison as a pull request comment,
    # and the default token is read-only, so it failed with "Resource not
    # accessible by integration" — after the build and every benchmark had
    # succeeded. Only pull requests comment, so main pushes stayed green and
    # the job looked healthy. binary-size.yml had the identical bug and was
    # fixed in #22; this workflow was missed.
    permissions:
      contents: read
      pull-requests: write

    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Setup Zig
        uses: pantry-pm/pantry/packages/action@235036fa0f48bae99b2293df5a3dc35c809b1777 # pinned: last SHA whose bundled typescript resolves on linux-x64
      - name: Install Linux dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y libgtk-3-dev libwebkit2gtk-4.1-dev

      - name: Cache Zig artifacts
        uses: actions/cache@v5
        with:
          path: |
            ~/.cache/zig
            packages/zig/zig-cache
          key: bench-${{ runner.os }}-zig-master-${{ hashFiles('packages/zig/build.zig') }}
          restore-keys: |
            bench-${{ runner.os }}-zig-master-

      - name: First-party Zig dependencies
        uses: ./.github/actions/first-party-zig-deps

      - name: Setup Bun
        uses: oven-sh/setup-bun@v2
        with:
          bun-version: latest

      # ---------------------------------------------------------------------
      # Binary load time, A/B against main on this runner
      #
      # What is measured is `craft --help`: process spawn, dynamic linking and
      # argument parsing. It never opens a window, so it cannot see a change in
      # window or webview startup. This step used to call that "Startup Time"
      # and fail pull requests on it, which is how a number that cannot move
      # ended up gating the code that would move it. Real startup is
      # `benchmarks/startup.bench.ts`, and it needs a display.
      #
      # The comparison is A/B on one runner rather than against a stored
      # number, because the stored number compared machines instead of
      # binaries. Measured on one unchanging binary: p50 22.9ms, p95 38.8ms,
      # max 65.6ms, and the old mean-of-5-against-a-cached-number method swung
      # 22ms to 32ms — a 45% spread through a gate that fired at 20%.
      # Interleaving both binaries in one sitting brought the same experiment
      # to 3.5% worst case.
      #
      # So the cache holds main's *binary*, not main's timing.
      # ---------------------------------------------------------------------
      - name: Build binary
        working-directory: packages/zig
        run: |
          zig build -Doptimize=ReleaseFast
          mkdir -p "$RUNNER_TEMP/bench"
          cp zig-out/bin/craft "$RUNNER_TEMP/bench/craft-head"

      - name: Restore main's binary
        if: github.event_name == 'pull_request'
        id: baseline
        uses: actions/cache/restore@v4
        with:
          path: baseline-binary
          key: benchmarks-baseline-binary-${{ github.base_ref }}
          restore-keys: |
            benchmarks-baseline-binary-main

      - name: Compare against main
        if: github.event_name == 'pull_request'
        id: compare
        working-directory: benchmarks
        run: |
          if [ ! -f "../baseline-binary/craft" ]; then
            echo "No baseline binary cached yet — reporting without a comparison."
            echo "compared=false" >> $GITHUB_OUTPUT
            exit 0
          fi
          echo "compared=true" >> $GITHUB_OUTPUT
          chmod +x ../baseline-binary/craft

          # Redirect, then cat — deliberately not `| tee`. `$?` after a
          # pipeline is the last command's status, so `... | tee file` would
          # report tee's success and the gate below could never fire. Whether
          # `pipefail` happens to be set is not something a gate should depend
          # on.
          set +e
          bun run binary-load-ab.ts \
            --base ../baseline-binary/craft \
            --head "$RUNNER_TEMP/bench/craft-head" \
            --rounds 25 > ab-output.txt 2>&1
          RC=$?
          set -e
          cat ab-output.txt

          echo "regressed=$RC" >> $GITHUB_OUTPUT
          DELTA=$(python3 -c "import json;print(json.load(open('binary-load-ab.json'))['delta_percent'])")
          echo "delta=$DELTA" >> $GITHUB_OUTPUT

      - name: Report
        run: |
          echo "## Binary load time" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          if [ "${{ steps.compare.outputs.compared }}" = "true" ]; then
            echo '```' >> $GITHUB_STEP_SUMMARY
            cat benchmarks/ab-output.txt >> $GITHUB_STEP_SUMMARY
            echo '```' >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
          fi
          echo "Measures \`craft --help\` — process spawn, dynamic linking and argument" >> $GITHUB_STEP_SUMMARY
          echo "parsing. It does **not** measure window or webview startup." >> $GITHUB_STEP_SUMMARY

      - name: Save main's binary as the baseline
        if: github.ref == 'refs/heads/main'
        run: |
          mkdir -p baseline-binary
          cp "$RUNNER_TEMP/bench/craft-head" baseline-binary/craft

      - name: Cache main's binary
        if: github.ref == 'refs/heads/main'
        uses: actions/cache/save@v4
        with:
          path: baseline-binary
          key: benchmarks-baseline-binary-main-${{ github.sha }}

      - name: Comment on PR
        if: github.event_name == 'pull_request' && steps.compare.outputs.compared == 'true'
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const out = fs.readFileSync('benchmarks/ab-output.txt', 'utf8');
            const regressed = '${{ steps.compare.outputs.regressed }}' === '1';
            const body = `## ${regressed ? '\u{1F534}' : '\u2705'} Binary load time

            \`\`\`
            ${out.trim()}
            \`\`\`

            <details>
            <summary>What this measures</summary>

            \`craft --help\`: process spawn, dynamic linking and argument parsing.
            It never opens a window, so it **cannot** see a change in window or
            webview startup — real startup is \`benchmarks/startup.bench.ts\`, which
            needs a display.

            Both binaries are measured interleaved on this runner and compared by
            p50, rather than against a number recorded on another machine. On
            byte-identical binaries that method reads within ~3.5%; the old one
            swung 45%.
            </details>`;

            const { data: comments } = await github.rest.issues.listComments({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
            });
            const existing = comments.find(c =>
              c.user.login === 'github-actions[bot]' && c.body.includes('Binary load time')
            );
            if (existing) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner, repo: context.repo.repo,
                comment_id: existing.id, body,
              });
            } else {
              await github.rest.issues.createComment({
                owner: context.repo.owner, repo: context.repo.repo,
                issue_number: context.issue.number, body,
              });
            }

      - name: Upload benchmark results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-results
          path: |
            benchmarks/ab-output.txt
            benchmarks/binary-load-ab.json
          if-no-files-found: ignore
          retention-days: 90

      - name: Fail on regression
        if: steps.compare.outputs.regressed == '1'
        run: |
          echo "::error::Binary load time regressed by ${{ steps.compare.outputs.delta }}% against main, measured A/B on this runner."
          exit 1