test(mobile): run the generated apps on a simulator and an emulator #785
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| concurrency: | |
| group: benchmarks-${{ github.head_ref || github.ref }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| benchmarks: | |
| name: run | |
| runs-on: ubuntu-latest | |
| # The last step posts the benchmark comparison as a pull request comment, | |
| # and the default token is read-only, so it failed with "Resource not | |
| # accessible by integration" — after the build and every benchmark had | |
| # succeeded. Only pull requests comment, so main pushes stayed green and | |
| # the job looked healthy. binary-size.yml had the identical bug and was | |
| # fixed in #22; this workflow was missed. | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Zig | |
| uses: pantry-pm/pantry/packages/action@235036fa0f48bae99b2293df5a3dc35c809b1777 # pinned: last SHA whose bundled typescript resolves on linux-x64 | |
| - name: Install Linux dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libgtk-3-dev libwebkit2gtk-4.1-dev | |
| - name: Cache Zig artifacts | |
| uses: actions/cache@v5 | |
| with: | |
| path: | | |
| ~/.cache/zig | |
| packages/zig/zig-cache | |
| key: bench-${{ runner.os }}-zig-master-${{ hashFiles('packages/zig/build.zig') }} | |
| restore-keys: | | |
| bench-${{ runner.os }}-zig-master- | |
| - name: First-party Zig dependencies | |
| uses: ./.github/actions/first-party-zig-deps | |
| - name: Setup Bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version: latest | |
| # --------------------------------------------------------------------- | |
| # Binary load time, A/B against main on this runner | |
| # | |
| # What is measured is `craft --help`: process spawn, dynamic linking and | |
| # argument parsing. It never opens a window, so it cannot see a change in | |
| # window or webview startup. This step used to call that "Startup Time" | |
| # and fail pull requests on it, which is how a number that cannot move | |
| # ended up gating the code that would move it. Real startup is | |
| # `benchmarks/startup.bench.ts`, and it needs a display. | |
| # | |
| # The comparison is A/B on one runner rather than against a stored | |
| # number, because the stored number compared machines instead of | |
| # binaries. Measured on one unchanging binary: p50 22.9ms, p95 38.8ms, | |
| # max 65.6ms, and the old mean-of-5-against-a-cached-number method swung | |
| # 22ms to 32ms — a 45% spread through a gate that fired at 20%. | |
| # Interleaving both binaries in one sitting brought the same experiment | |
| # to 3.5% worst case. | |
| # | |
| # So the cache holds main's *binary*, not main's timing. | |
| # --------------------------------------------------------------------- | |
| - name: Build binary | |
| working-directory: packages/zig | |
| run: | | |
| zig build -Doptimize=ReleaseFast | |
| mkdir -p "$RUNNER_TEMP/bench" | |
| cp zig-out/bin/craft "$RUNNER_TEMP/bench/craft-head" | |
| - name: Restore main's binary | |
| if: github.event_name == 'pull_request' | |
| id: baseline | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: baseline-binary | |
| key: benchmarks-baseline-binary-${{ github.base_ref }} | |
| restore-keys: | | |
| benchmarks-baseline-binary-main | |
| - name: Compare against main | |
| if: github.event_name == 'pull_request' | |
| id: compare | |
| working-directory: benchmarks | |
| run: | | |
| if [ ! -f "../baseline-binary/craft" ]; then | |
| echo "No baseline binary cached yet — reporting without a comparison." | |
| echo "compared=false" >> $GITHUB_OUTPUT | |
| exit 0 | |
| fi | |
| echo "compared=true" >> $GITHUB_OUTPUT | |
| chmod +x ../baseline-binary/craft | |
| # Redirect, then cat — deliberately not `| tee`. `$?` after a | |
| # pipeline is the last command's status, so `... | tee file` would | |
| # report tee's success and the gate below could never fire. Whether | |
| # `pipefail` happens to be set is not something a gate should depend | |
| # on. | |
| set +e | |
| bun run binary-load-ab.ts \ | |
| --base ../baseline-binary/craft \ | |
| --head "$RUNNER_TEMP/bench/craft-head" \ | |
| --rounds 25 > ab-output.txt 2>&1 | |
| RC=$? | |
| set -e | |
| cat ab-output.txt | |
| echo "regressed=$RC" >> $GITHUB_OUTPUT | |
| DELTA=$(python3 -c "import json;print(json.load(open('binary-load-ab.json'))['delta_percent'])") | |
| echo "delta=$DELTA" >> $GITHUB_OUTPUT | |
| - name: Report | |
| run: | | |
| echo "## Binary load time" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ "${{ steps.compare.outputs.compared }}" = "true" ]; then | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| cat benchmarks/ab-output.txt >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "Measures \`craft --help\` — process spawn, dynamic linking and argument" >> $GITHUB_STEP_SUMMARY | |
| echo "parsing. It does **not** measure window or webview startup." >> $GITHUB_STEP_SUMMARY | |
| - name: Save main's binary as the baseline | |
| if: github.ref == 'refs/heads/main' | |
| run: | | |
| mkdir -p baseline-binary | |
| cp "$RUNNER_TEMP/bench/craft-head" baseline-binary/craft | |
| - name: Cache main's binary | |
| if: github.ref == 'refs/heads/main' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: baseline-binary | |
| key: benchmarks-baseline-binary-main-${{ github.sha }} | |
| - name: Comment on PR | |
| if: github.event_name == 'pull_request' && steps.compare.outputs.compared == 'true' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const out = fs.readFileSync('benchmarks/ab-output.txt', 'utf8'); | |
| const regressed = '${{ steps.compare.outputs.regressed }}' === '1'; | |
| const body = `## ${regressed ? '\u{1F534}' : '\u2705'} Binary load time | |
| \`\`\` | |
| ${out.trim()} | |
| \`\`\` | |
| <details> | |
| <summary>What this measures</summary> | |
| \`craft --help\`: process spawn, dynamic linking and argument parsing. | |
| It never opens a window, so it **cannot** see a change in window or | |
| webview startup — real startup is \`benchmarks/startup.bench.ts\`, which | |
| needs a display. | |
| Both binaries are measured interleaved on this runner and compared by | |
| p50, rather than against a number recorded on another machine. On | |
| byte-identical binaries that method reads within ~3.5%; the old one | |
| swung 45%. | |
| </details>`; | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| }); | |
| const existing = comments.find(c => | |
| c.user.login === 'github-actions[bot]' && c.body.includes('Binary load time') | |
| ); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, repo: context.repo.repo, | |
| comment_id: existing.id, body, | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, repo: context.repo.repo, | |
| issue_number: context.issue.number, body, | |
| }); | |
| } | |
| - name: Upload benchmark results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results | |
| path: | | |
| benchmarks/ab-output.txt | |
| benchmarks/binary-load-ab.json | |
| if-no-files-found: ignore | |
| retention-days: 90 | |
| - name: Fail on regression | |
| if: steps.compare.outputs.regressed == '1' | |
| run: | | |
| echo "::error::Binary load time regressed by ${{ steps.compare.outputs.delta }}% against main, measured A/B on this runner." | |
| exit 1 |