trigger:
  tags:
    include: ["*"]
  paths:
    include:
      - ".azure/gpu-coverage.yml"
      - "requirements/coverage.txt"
      - "thunder/tests/coverage/**"
  branches:
    include:
      - "main"
      - "release/*"
      - "refs/tags/*"

pr:
  branches:
    include: ["*"]

jobs:
  - job: coverage
    strategy:
      matrix:
        "w/ torch 2.7.1":
          docker-image: "ubuntu24.04-cuda12.8.1-cudnn-fe1.15.0-py3.12-pt_2.8.0-dev"
    # how much time to give 'run always even if cancelled tasks' before stopping them
    cancelTimeoutInMinutes: "2"
    pool: "lit-rtx-3090"
    variables:
      DEVICES: $( python -c 'name = "$(Agent.Name)" ; gpus = name.split("_")[-1] if "_" in name else "0"; print(gpus)' )
      TORCH_HOME: "/var/tmp/torch"
      PIP_CACHE_DIR: "/var/tmp/pip"
      PYTHONHASHSEED: "0"
      NCCL_DEBUG: "INFO"
      ALLOW_COVERAGE_TRACE: "1"
    container:
      image: "pytorchlightning/lightning-thunder:$(docker-image)"
      options: "--gpus=all --shm-size=16g -v /var/tmp:/var/tmp"
    workspace:
      clean: all
    steps:
      - bash: |
          echo $(DEVICES)
          lspci | egrep 'VGA|3D'
          dpkg-query -W -f='${Package} ${Version}\n' libnccl2 libnccl-dev
          whereis nvidia
          nvidia-smi
          which python && which pip
          python --version
          pip --version
          pip list
          echo "##vso[task.setvariable variable=CUDA_VISIBLE_DEVICES]$(DEVICES)"
        displayName: "Image info & NVIDIA"

      - bash: |
          set -ex
          # drop pt from requirements so not to interfere with the existing one
          bash scripts/remove-torch-lines.sh requirements/base.txt
          cat requirements/base.txt

          # double check on test requirements
          pip install -U -r requirements/base.txt -r requirements/coverage.txt

          # https://docs.codecov.com/docs/codecov-uploader
          curl -Os https://uploader.codecov.io/latest/linux/codecov
          chmod +x codecov

          # install this package
          pip install -e .
        displayName: "Install package & ..."

      - bash: bash scripts/sanity-check.sh
        displayName: "Sanity check / details"

      - bash: |
          PYTHONPATH=$(pwd)/thunder/tests pytest thunder/tests/coverage_tests
        timeoutInMinutes: "45"
        displayName: "Testing: coverage_tests"
