name: NVIDIA GPU

on:
  workflow_dispatch:  # Enables manual trigger
    inputs:
      suite:
        description: "Which test set to run on manual dispatch"
        type: choice
        default: daily
        options:
          - daily
          - weekly
  schedule:
    # Daily: correctness + cuTeDSL (SM90) on H100, cuTeDSL (SM100) on B200,
    # and cuTile on BOTH archs.
    # Randomized off-peak minute; round hours (esp. 00:00) get throttled/queued by GitHub.
    - cron: '24 9 * * *'
    # Weekly (Sundays): convergence (latest tf) + oldest-tf correctness & convergence.
    # Staggered a few hours after the daily run as an extra guard against overlap.
    - cron: '24 13 * * 0'

# Serialize whole runs so the daily and weekly runs never execute at the same time.
# Together with `max-parallel: 3` on each job this caps concurrent Modal GPU calls at 3
# (a 4th concurrent GPU call is rejected by the runner/Modal limit).
concurrency:
  group: ${{ github.workflow }}
  cancel-in-progress: false

jobs:
  # ---------------------------------------------------------------------------
  # Daily: general correctness + cuTile + cuTeDSL, split across H100 and B200.
  # cuTeDSL runs on BOTH archs on purpose: SM90 tests only run on H100 and SM100
  # tests only run on B200 (each arch self-skips the other half). cuTile likewise
  # runs on BOTH archs to catch arch-specific (e.g. SM100/tcgen05) regressions.
  # ---------------------------------------------------------------------------
  daily:
    name: daily / ${{ matrix.name }}
    if: >-
      github.event.schedule == '24 9 * * *'
      || (github.event_name == 'workflow_dispatch' && github.event.inputs.suite == 'daily')
    strategy:
      max-parallel: 3      # never run more than 3 GPU jobs at once
      fail-fast: false
      matrix:
        include:
          - name: correctness (H100)
            modal_fn: liger_correctness_tests
          - name: cuTile (H100)
            modal_fn: liger_cutile_tests
          - name: cuTile (B200)
            modal_fn: liger_cutile_tests_b200
          - name: cuTeDSL SM90 (H100)
            modal_fn: liger_cutedsl_tests_h100
          - name: cuTeDSL SM100 (B200)
            modal_fn: liger_cutedsl_tests_b200
    runs-on: ubuntu-latest
    env:
      MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
      MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}

    steps:
    - name: Checkout code
      uses: actions/checkout@v6

    - name: Set up Python
      uses: actions/setup-python@v6
      with:
        python-version: '3.10'

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install modal

    - name: Run ${{ matrix.name }}
      run: |
        modal run -m dev.modal.tests::${{ matrix.modal_fn }}

  # ---------------------------------------------------------------------------
  # Weekly (Sundays): convergence (latest tf) + oldest-supported-tf correctness
  # and convergence. All on H100.
  # ---------------------------------------------------------------------------
  weekly:
    name: weekly / ${{ matrix.name }}
    if: >-
      github.event.schedule == '24 13 * * 0'
      || (github.event_name == 'workflow_dispatch' && github.event.inputs.suite == 'weekly')
    strategy:
      max-parallel: 3      # never run more than 3 GPU jobs at once
      fail-fast: false
      matrix:
        include:
          - name: convergence (H100)
            modal_fn: liger_convergence_tests
          - name: oldest-tf correctness (H100)
            modal_fn: liger_oldest_v4_correctness_tests
          - name: oldest-tf convergence (H100)
            modal_fn: liger_oldest_v4_convergence_tests
    runs-on: ubuntu-latest
    env:
      MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
      MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}

    steps:
    - name: Checkout code
      uses: actions/checkout@v6

    - name: Set up Python
      uses: actions/setup-python@v6
      with:
        python-version: '3.10'

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install modal

    - name: Run ${{ matrix.name }}
      run: |
        modal run -m dev.modal.tests::${{ matrix.modal_fn }}
