name: On-Demand PR Benchmark

on:
  workflow_dispatch:
    inputs:
      pr_number:
        description: "PR number to benchmark"
        required: true
        type: number
      commands:
        description: "Comma separated predefined commands(equivalent to '-t')  to benchmark (e.g. SET,GET,HSET,LPUSH)"
        required: false
        type: string
        default: "SET,GET"
      arbitrary_command:
        description: "Command to benchmark, e.g. 'GETRANGE key:__rand_int__ 0 100'. Takes precedence over commands. Supports __rand_int__ and __data__."
        required: false
        type: string
        default: ""
      populate_command:
        description: "Seeds the keyspace before an arbitrary read, e.g. 'SET key:__rand_int__ __data__'. Requires arbitrary_command and the SAME key pattern, or the reads will miss."
        required: false
        type: string
        default: ""
      write_command:
        description: "Write side of a concurrent mixed load, e.g. 'SET key:__rand_int__ __data__' or 'SET'. Requires read_command. Not for use with arbitrary_command or populate_command."
        required: false
        type: string
        default: ""
      read_command:
        description: "Read side of a concurrent mixed load, e.g. 'GET key:__rand_int__' or 'GET'. Requires write_command and the SAME key pattern, or the reads will miss."
        required: false
        type: string
        default: ""
      write_ratio:
        description: "Percentage of clients driving write_command; the rest drive read_command. Range: 1-99, default 50."
        required: false
        type: string
        default: ""
      data_size:
        description: "Comma separated data sizes in bytes (e.g. 16,96). Range: 1-1048576"
        required: false
        type: string
        default: ""
      io_threads:
        description: "Comma separated io-threads values (e.g. 1,9). Range: 1-20"
        required: false
        type: string
        default: ""
      benchmark_threads:
        description: "Client thread count for valkey-benchmark ('--threads'). Range: 1-96."
        required: false
        type: string
        default: ""
      pipelines:
        description: "Comma separated pipeline values (e.g. 1,10). Range: 1-1000"
        required: false
        type: string
        default: ""
      cluster_mode:
        description: "Enable cluster mode"
        required: false
        type: boolean
        default: false

defaults:
  run:
    shell: "bash -Eeuo pipefail -x {0}"

# Workflow-level permissions are minimal. Jobs grant additional scopes only
# where they are actually needed (defense-in-depth).
permissions:
  contents: read

jobs:
  resolve-pr:
    if: github.repository == 'valkey-io/valkey'
    runs-on: ubuntu-latest
    # Read-only access to the PR is enough to call pulls.get.
    permissions:
      contents: read
      pull-requests: read
    outputs:
      merge_commit_sha: ${{ steps.pr.outputs.merge_commit_sha }}
      head_sha: ${{ steps.pr.outputs.head_sha }}
      base_ref: ${{ steps.pr.outputs.base_ref }}
    steps:
      - name: Get PR details
        id: pr
        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const pr = await github.rest.pulls.get({
              owner: context.repo.owner,
              repo: context.repo.repo,
              pull_number: ${{ github.event.inputs.pr_number }}
            });

            if (pr.data.state !== 'open') {
              core.setFailed(`PR #${{ github.event.inputs.pr_number }} is not open`);
              return;
            }

            // GitHub returns merge_commit_sha = null when the PR cannot be
            // cleanly merged into its base branch, OR when GitHub has not
            // yet computed the merge (race right after a push).
            if (!pr.data.merge_commit_sha) {
              core.setFailed("The PR has no mergeable commit (likely conflicts with the base branch, or merge not yet computed - try again in a moment).");
              return;
            }

            core.setOutput('merge_commit_sha', pr.data.merge_commit_sha);
            core.setOutput('head_sha', pr.data.head.sha);
            core.setOutput('base_ref', pr.data.base.ref);

  benchmark:
    needs: resolve-pr
    runs-on: ["self-hosted", "ec2-al-2023-pr-benchmarking-arm64"]
    # Benchmark workload is ~5.5h; 7h gives buffer for slow runs/queueing.
    timeout-minutes: 420
    # Write access scoped to the job that posts the result comment.
    permissions:
      contents: read
      pull-requests: write
      issues: write
    steps:
      - name: Checkout valkey
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          path: valkey
          fetch-depth: 0
          ref: ${{ needs.resolve-pr.outputs.merge_commit_sha }}
          persist-credentials: false

      - name: Checkout valkey-perf-benchmark
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          repository: ${{ github.repository_owner }}/valkey-perf-benchmark
          path: valkey-perf-benchmark
          fetch-depth: 1
          persist-credentials: false

      - name: Checkout valkey for latest benchmark
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          repository: valkey-io/valkey
          ref: "unstable"
          path: valkey_latest
          fetch-depth: 0
          persist-credentials: false

      - name: Set up Python
        uses: kishaningithub/setup-python-amazon-linux@a326cdc792983fe0fbd04c81d3d62b59b6123a6c # v1.1.0
        with:
          python-version: "3.10"
          cache: "pip"

      - name: Install dependencies
        working-directory: valkey-perf-benchmark
        run: |
          sudo dnf groupinstall "Development Tools" -y
          sudo dnf install -y gcc gcc-c++ make \
            python3-devel \
            openssl-devel \
            bzip2-devel \
            libffi-devel \
            jq
          pip install --require-hashes -r requirements.txt

      - name: Build latest valkey_latest
        working-directory: valkey_latest
        run: |
          echo "Building latest valkey-benchmark for latest benchmark executable..."
          make distclean || true
          make -j
          if [[ -f "src/valkey-benchmark" ]]; then
            echo "Successfully built latest valkey-benchmark"
            ls -la src/valkey-benchmark
            ./src/valkey-benchmark --version || echo "Version check completed"
          else
            echo "Failed to build valkey-benchmark"
            exit 1
          fi
          VALKEY_BENCHMARK_PATH="$(pwd)/src/valkey-benchmark"
          echo "VALKEY_BENCHMARK_PATH=$VALKEY_BENCHMARK_PATH" >> $GITHUB_ENV
          echo "Latest valkey-benchmark path: $VALKEY_BENCHMARK_PATH"

      - name: Generate benchmark config
        working-directory: valkey-perf-benchmark
        env:
          INPUT_COMMANDS: ${{ github.event.inputs.commands }}
          INPUT_ARBITRARY_COMMAND: ${{ github.event.inputs.arbitrary_command }}
          INPUT_POPULATE_COMMAND: ${{ github.event.inputs.populate_command }}
          INPUT_WRITE_COMMAND: ${{ github.event.inputs.write_command }}
          INPUT_READ_COMMAND: ${{ github.event.inputs.read_command }}
          INPUT_WRITE_RATIO: ${{ github.event.inputs.write_ratio }}
          INPUT_DATA_SIZE: ${{ github.event.inputs.data_size }}
          INPUT_IO_THREADS: ${{ github.event.inputs.io_threads }}
          INPUT_BENCHMARK_THREADS: ${{ github.event.inputs.benchmark_threads }}
          INPUT_PIPELINES: ${{ github.event.inputs.pipelines }}
          INPUT_CLUSTER_MODE: ${{ github.event.inputs.cluster_mode }}
        run: |
          # utils/modify_config.py owns the config schema and validates inputs.
          ARGS=(--config configs/benchmark-config-arm.json)

          # commands has a default, and the script rejects it alongside a
          # generated mode, so only pass it when no generated mode is selected.
          if [[ -n "$INPUT_COMMANDS" && -z "$INPUT_ARBITRARY_COMMAND" \
                && -z "$INPUT_WRITE_COMMAND" && -z "$INPUT_READ_COMMAND" ]]; then
            ARGS+=(--commands "$INPUT_COMMANDS")
          fi

          [[ -n "$INPUT_ARBITRARY_COMMAND" ]] && ARGS+=(--arbitrary-command "$INPUT_ARBITRARY_COMMAND")
          [[ -n "$INPUT_POPULATE_COMMAND" ]]  && ARGS+=(--populate-command "$INPUT_POPULATE_COMMAND")
          [[ -n "$INPUT_WRITE_COMMAND" ]]     && ARGS+=(--write-command "$INPUT_WRITE_COMMAND")
          [[ -n "$INPUT_READ_COMMAND" ]]      && ARGS+=(--read-command "$INPUT_READ_COMMAND")
          [[ -n "$INPUT_WRITE_RATIO" ]]       && ARGS+=(--write-ratio "$INPUT_WRITE_RATIO")
          [[ -n "$INPUT_DATA_SIZE" ]]         && ARGS+=(--data-size "$INPUT_DATA_SIZE")
          [[ -n "$INPUT_IO_THREADS" ]]        && ARGS+=(--io-threads "$INPUT_IO_THREADS")
          [[ -n "$INPUT_BENCHMARK_THREADS" ]] && ARGS+=(--benchmark-threads "$INPUT_BENCHMARK_THREADS")
          [[ -n "$INPUT_PIPELINES" ]]         && ARGS+=(--pipelines "$INPUT_PIPELINES")

          [[ "$INPUT_CLUSTER_MODE" == "true" ]] && ARGS+=(--cluster-mode)

          # Bounds that describe this runner rather than the framework.
          ARGS+=(--server-cpu-ceiling 19 --max-io-threads 20 --max-benchmark-threads 96)

          python utils/modify_config.py "${ARGS[@]}" --print

      - name: Run benchmarks
        working-directory: valkey-perf-benchmark
        run: |
          CONFIG_FILE="configs/benchmark-config-arm.json"

          BENCHMARK_ARGS=(
            --config "$CONFIG_FILE"
            --commits "${{ needs.resolve-pr.outputs.merge_commit_sha }}"
            --baseline "${{ needs.resolve-pr.outputs.base_ref }}"
            --valkey-benchmark-path "$VALKEY_BENCHMARK_PATH"
            --target-ip ${{ secrets.EC2_ARM64_IP }}
            --valkey-path "../valkey"
            --results-dir "results"
            --runs 3
          )

          python ./benchmark.py "${BENCHMARK_ARGS[@]}"

      - name: Compare results
        working-directory: valkey-perf-benchmark
        run: |
          python ./utils/compare_benchmark_results.py \
            --baseline ./results/${{ needs.resolve-pr.outputs.base_ref }}/metrics.json \
            --new ./results/${{ needs.resolve-pr.outputs.merge_commit_sha }}/metrics.json \
            --output ../comparison.md \
            --metrics rps

      - name: Upload artifacts
        if: always()
        continue-on-error: true
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: on-demand-benchmark-pr-${{ github.event.inputs.pr_number }}
          path: |
            ./valkey-perf-benchmark/results/
            comparison.md

      - name: Comment PR with results
        env:
          INPUT_PR_NUMBER: ${{ github.event.inputs.pr_number }}
          INPUT_COMMANDS: ${{ github.event.inputs.commands }}
          INPUT_ARBITRARY_COMMAND: ${{ github.event.inputs.arbitrary_command }}
          INPUT_POPULATE_COMMAND: ${{ github.event.inputs.populate_command }}
          INPUT_WRITE_COMMAND: ${{ github.event.inputs.write_command }}
          INPUT_READ_COMMAND: ${{ github.event.inputs.read_command }}
          INPUT_WRITE_RATIO: ${{ github.event.inputs.write_ratio }}
          INPUT_DATA_SIZE: ${{ github.event.inputs.data_size }}
          INPUT_IO_THREADS: ${{ github.event.inputs.io_threads }}
          INPUT_BENCHMARK_THREADS: ${{ github.event.inputs.benchmark_threads }}
          INPUT_PIPELINES: ${{ github.event.inputs.pipelines }}
          INPUT_CLUSTER_MODE: ${{ github.event.inputs.cluster_mode }}
        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const fs = require('fs');
            const body = fs.readFileSync('comparison.md', 'utf8');
            const {owner, repo} = context.repo;
            const sha = '${{ needs.resolve-pr.outputs.head_sha }}';
            const short = sha.slice(0,7);
            const link = `[\`${short}\`](https://github.com/${owner}/${repo}/commit/${sha})`;
            // Read user inputs from process.env (safer than direct
            // interpolation of ${{ github.event.inputs.* }} into JS strings).
            const prNumber = Number(process.env.INPUT_PR_NUMBER);
            const commands = process.env.INPUT_COMMANDS;
            const arbitraryCommand = process.env.INPUT_ARBITRARY_COMMAND;
            const cluster = process.env.INPUT_CLUSTER_MODE === 'true' ? ' (Cluster Mode)' : '';
            const runUrl = `https://github.com/${owner}/${repo}/actions/runs/${{ github.run_id }}`;

            let header = `**On-Demand Benchmark${cluster} ran on commit:** ${link}\n`;
            const writeCommand = process.env.INPUT_WRITE_COMMAND;
            const readCommand = process.env.INPUT_READ_COMMAND;
            if (writeCommand && readCommand) {
              const writeRatio = Number(process.env.INPUT_WRITE_RATIO) || 50;
              header += `**Mixed load:** \`${writeCommand}\` (write, ${writeRatio}% of clients) + \`${readCommand}\` (read, ${100 - writeRatio}%)\n`;
            } else if (arbitraryCommand) {
              header += `**Arbitrary command:** \`${arbitraryCommand}\`\n`;
              const populateCommand = process.env.INPUT_POPULATE_COMMAND;
              if (populateCommand) {
                header += `**Populate command:** \`${populateCommand}\`\n`;
              }
            } else {
              header += `**Commands:** \`${commands}\`\n`;
            }

            // Show overridden configs
            const overrides = [];
            const dataSize = process.env.INPUT_DATA_SIZE;
            const ioThreads = process.env.INPUT_IO_THREADS;
            const pipelines = process.env.INPUT_PIPELINES;
            if (dataSize) overrides.push(`**Data Size:** \`${dataSize}\``);
            if (ioThreads) overrides.push(`**IO Threads:** \`${ioThreads}\``);
            const benchmarkThreads = process.env.INPUT_BENCHMARK_THREADS;
            if (benchmarkThreads) overrides.push(`**Benchmark Threads:** \`${benchmarkThreads}\``);
            if (pipelines) overrides.push(`**Pipelines:** \`${pipelines}\``);
            if (overrides.length > 0) {
              header += overrides.join(' | ') + '\n';
            }

            header += `**Runs:** 3 | [Workflow Run](${runUrl})\n\n`;

            await github.rest.issues.createComment({
              issue_number: prNumber,
              owner,
              repo,
              body: header + body
            });

      - name: Cleanup any running valkey processes
        if: always()
        continue-on-error: true
        run: |
          rm -rf comparison.md valkey*
          # pkill returns 1 when no processes match, which is the normal case.
          # We need to capture the exit code without tripping `set -e`.
          pkill -f valkey && exit_code=0 || exit_code=$?
          if [ $exit_code -eq 0 ]; then
            echo "Killed running valkey processes"
          elif [ $exit_code -eq 1 ]; then
            echo "No valkey processes found to kill"
          else
            echo "Warning: pkill failed with exit code $exit_code"
          fi
