group: Disaggregated
depends_on: 
  - image-build
steps:
- label: ":nvidia: (L4) Distributed NixlConnector PD accuracy"
  key: distributed-nixlconnector-pd-accuracy-4-gpus
  timeout_in_minutes: 55
  working_dir: "/vllm-workspace/tests"
  device: l4
  num_devices: 4
  source_file_dependencies:
    - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
    - tests/v1/kv_connector/nixl_integration/
  commands:
    - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
    - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
  mirror:
    amd:
      label: ":amd: (MI355) Distributed NixlConnector PD accuracy"
      dind: false
      device: mi355_4
      timeout_in_minutes: 60
      depends_on:
        - image-build-amd
      source_file_dependencies:
        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
        - tests/v1/kv_connector/nixl_integration/
        - vllm/platforms/rocm.py
      commands:
        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
        - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh

- label: ":nvidia: (L4) Distributed FlashInfer NixlConnector PD accuracy"
  key: distributed-flashinfer-nixlconnector-pd-accuracy-4-gpus
  timeout_in_minutes: 55
  working_dir: "/vllm-workspace/tests"
  device: l4
  num_devices: 4
  source_file_dependencies:
    - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
    - tests/v1/kv_connector/nixl_integration/
  commands:
    - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
    - FLASHINFER=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
  mirror:
    amd:
      label: ":amd: (MI355) Distributed AITER NixlConnector PD accuracy"
      dind: false
      device: mi355_4
      num_devices: 4
      timeout_in_minutes: 60
      working_dir: "/vllm-workspace/tests"
      depends_on:
        - image-build-amd
      source_file_dependencies:
        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
        - tests/v1/kv_connector/nixl_integration/
        - tests/v1/kv_connector/rocm_pd_accuracy_utils.py
        - tests/utils.py
        - vllm/v1/attention/backends/rocm_aiter_unified_attn.py
        - vllm/platforms/rocm.py
        - requirements/common.txt
        - requirements/kv_connectors_rocm.txt
      commands:
        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
        # Preserve all six model/TP pairs with native AITER attention.
        - pytest -v -s v1/kv_connector/nixl_integration/test_accuracy_rocm.py

- label: ":nvidia: (L4) Push NixlConnector PP prefill PD accuracy"
  key: push-nixlconnector-pp-prefill-pd-accuracy-4-gpus
  timeout_in_minutes: 30
  working_dir: "/vllm-workspace/tests"
  device: l4
  num_devices: 4
  source_file_dependencies:
    - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
    - tests/v1/kv_connector/nixl_integration/
    - tests/v1/kv_connector/nixl_push_integration/
  commands:
    - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
    - bash v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh
  mirror:
    amd:
      label: ":amd: (MI300) Push NixlConnector PP prefill PD accuracy"
      dind: false
      device: mi300_4
      timeout_in_minutes: 55
      depends_on:
        - image-build-amd
      source_file_dependencies:
        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
        - tests/v1/kv_connector/nixl_integration/
        - tests/v1/kv_connector/nixl_push_integration/
        - vllm/platforms/rocm.py
      commands:
        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
        - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh

- label: ":nvidia: (L4) DP EP Distributed NixlConnector PD accuracy"
  key: dp-ep-distributed-nixlconnector-pd-accuracy-tests-4-gpus
  timeout_in_minutes: 30
  working_dir: "/vllm-workspace/tests"
  device: l4
  num_devices: 4
  source_file_dependencies:
    - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
    - tests/v1/kv_connector/nixl_integration/
  commands:
    - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
    - DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
  mirror:
    amd:
      label: ":amd: (MI355) DP EP Distributed NixlConnector PD accuracy"
      dind: false
      device: mi355_4
      timeout_in_minutes: 40
      depends_on:
        - image-build-amd
      source_file_dependencies:
        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
        - tests/v1/kv_connector/nixl_integration/
        - vllm/platforms/rocm.py
      commands:
        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
        - DP_EP=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh

- label: ":nvidia: (L4) CrossLayer KV layout Distributed NixlConnector PD accuracy"
  key: crosslayer-kv-layout-distributed-nixlconnector-pd-accuracy-tests-4-gpus
  timeout_in_minutes: 55
  working_dir: "/vllm-workspace/tests"
  device: l4
  num_devices: 4
  source_file_dependencies:
    - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
    - tests/v1/kv_connector/nixl_integration/
  commands:
    - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
    - CROSS_LAYERS_BLOCKS=True bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
  mirror:
    amd:
      label: ":amd: (MI300) CrossLayer KV layout Distributed NixlConnector PD accuracy"
      dind: false
      device: mi300_4
      timeout_in_minutes: 60
      depends_on:
        - image-build-amd
      source_file_dependencies:
        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
        - tests/v1/kv_connector/nixl_integration/
        - vllm/platforms/rocm.py
      commands:
        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
        - CROSS_LAYERS_BLOCKS=True ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh

- label: ":nvidia: (L4) Hybrid SSM NixlConnector PD accuracy"
  key: hybrid-ssm-nixlconnector-pd-accuracy-tests-4-gpus
  timeout_in_minutes: 60
  working_dir: "/vllm-workspace/tests"
  device: l4
  num_devices: 4
  source_file_dependencies:
    - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
    - tests/v1/kv_connector/nixl_integration/
  commands:
    - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
    - HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
  mirror:
    amd:
      label: ":amd: (MI300) Hybrid SSM NixlConnector PD accuracy"
      dind: false
      device: mi300_4
      timeout_in_minutes: 80
      depends_on:
        - image-build-amd
      source_file_dependencies:
        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
        - tests/v1/kv_connector/nixl_integration/
        - vllm/platforms/rocm.py
      commands:
        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
        - HYBRID_SSM=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh

- label: ":nvidia: (L4) NixlConnector PD edge case"
  key: nixlconnector-pd-edge-cases-2-gpus
  timeout_in_minutes: 40
  working_dir: "/vllm-workspace/tests"
  device: l4
  num_devices: 2
  source_file_dependencies:
    - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
    - vllm/v1/core/sched/
    - tests/v1/kv_connector/nixl_integration/
  env:
    PREFILL_GPU_ID: "0"
    DECODE_GPU_ID: "1"
  commands:
    - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
    - bash v1/kv_connector/nixl_integration/run_edge_case_test.sh
  mirror:
    amd:
      label: ":amd: (MI300) NixlConnector PD edge case"
      dind: false
      device: mi300_2
      timeout_in_minutes: 65
      depends_on:
        - image-build-amd
      source_file_dependencies:
        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
        - vllm/v1/core/sched/
        - tests/v1/kv_connector/nixl_integration/
        - vllm/platforms/rocm.py
      commands:
        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
        - PREFILL_GPU_ID=0 DECODE_GPU_ID=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_edge_case_test.sh

- label: ":nvidia: (L4) Hybrid SSM NixlConnector PD prefix cache"
  key: hybrid-ssm-nixlconnector-pd-prefix-cache-2-gpus
  timeout_in_minutes: 25
  working_dir: "/vllm-workspace/tests"
  device: l4
  num_devices: 2
  source_file_dependencies:
    - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
    - vllm/v1/core/sched/
    - vllm/v1/core/kv_cache_coordinator.py
    - tests/v1/kv_connector/nixl_integration/
  commands:
    - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
    - bash v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh
  mirror:
    amd:
      label: ":amd: (MI355) Hybrid SSM NixlConnector PD prefix cache"
      dind: false
      device: mi355_2
      timeout_in_minutes: 25
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
      - vllm/v1/core/sched/
      - vllm/v1/core/kv_cache_coordinator.py
      - tests/v1/kv_connector/nixl_integration/
      - vllm/platforms/rocm.py
      commands:
      - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
      - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh

- label: ":nvidia: (L4) MultiConnector (Nixl+Offloading) PD accuracy"
  key: multiconnector-nixl-offloading-pd-accuracy-2-gpus
  timeout_in_minutes: 40
  working_dir: "/vllm-workspace/tests"
  device: l4
  num_devices: 2
  source_file_dependencies:
    - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
    - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
    - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
    - vllm/distributed/kv_transfer/kv_connector/v1/offloading/
    - tests/v1/kv_connector/nixl_integration/
  commands:
    - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
    - bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh
  mirror:
    amd:
      label: ":amd: (MI355) MultiConnector (Nixl+Offloading) PD accuracy"
      dind: false
      device: mi355_2
      timeout_in_minutes: 40
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
      - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
      - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
      - vllm/distributed/kv_transfer/kv_connector/v1/offloading/
      - tests/v1/kv_connector/nixl_integration/
      - requirements/kv_connectors_rocm.txt
      - vllm/platforms/rocm.py
      commands:
      - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
      - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh

- label: ":nvidia: (A100) NixlConnector PD + Spec Decode acceptance"
  key: nixlconnector-pd-spec-decode-acceptance-2-gpus
  timeout_in_minutes: 45
  device: a100
  working_dir: "/vllm-workspace/tests"
  num_devices: 2
  source_file_dependencies:
    - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
    - vllm/v1/worker/kv_connector_model_runner_mixin.py
    - tests/v1/kv_connector/nixl_integration/
  commands:
    - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
    - bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
  mirror:
    amd:
      label: ":amd: (MI355) NixlConnector PD + Spec Decode acceptance"
      dind: false
      device: mi355_2
      timeout_in_minutes: 45
      depends_on:
        - image-build-amd
      source_file_dependencies:
        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
        - vllm/v1/worker/kv_connector_model_runner_mixin.py
        - tests/v1/kv_connector/nixl_integration/
        - vllm/platforms/rocm.py
      commands:
        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
        - KV_CACHE_MEMORY_BYTES=8G ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh

- label: ":nvidia: (L4) MultiConnector (Nixl+Offloading) PD edge cases"
  key: multiconnector-nixl-offloading-pd-edge-cases-2-gpus
  timeout_in_minutes: 25
  working_dir: "/vllm-workspace/tests"
  device: l4
  num_devices: 2
  source_file_dependencies:
    - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
    - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
    - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
    - vllm/distributed/kv_transfer/kv_connector/v1/offloading/
    - tests/v1/kv_connector/nixl_integration/
  commands:
    - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
    - bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh
  mirror:
    amd:
      label: ":amd: (MI355) MultiConnector (Nixl+Offloading) PD edge cases"
      dind: false
      device: mi355_2
      timeout_in_minutes: 25
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
      - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
      - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
      - vllm/distributed/kv_transfer/kv_connector/v1/offloading/
      - tests/v1/kv_connector/nixl_integration/
      - vllm/platforms/rocm.py
      commands:
      - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
      - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh

# P TP 4 - D DPEP 4 test case for DSv4-Flash
- label: ":nvidia: (H200) DSv4-Flash Disaggregated DP EP"
  key: dsv4-flash-disaggregated
  timeout_in_minutes: 60
  device: h200
  optional: true
  working_dir: "/vllm-workspace/tests"
  num_devices: 8
  env:
    ENABLE_HMA_FLAG: "1"
    DP_EP: "1"
    GPU_MEMORY_UTILIZATION: "0.85"
    PREFILLER_TP_SIZE: "4"
    DECODER_TP_SIZE: "4"
    PREFILL_BLOCK_SIZE: "256"
    DECODE_BLOCK_SIZE: "256"
    MODEL_NAMES: "deepseek-ai/DeepSeek-V4-Flash"
    VLLM_ENGINE_READY_TIMEOUT_S: "1800"
    VLLM_SERVE_EXTRA_ARGS: "--trust-remote-code,--kv-cache-dtype,fp8"
    # The V4 indexer only supports block-outermost layouts; let it pick.
    PREFILLER_KV_LAYOUT: ""
    DECODER_KV_LAYOUT: ""
  commands:
    - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
    - bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
  mirror:
    amd:
      label: ":amd: (MI355) DSv4-Flash Disaggregated DP EP"
      dind: false
      device: mi355_8
      timeout_in_minutes: 85
      depends_on:
        - image-build-amd
      source_file_dependencies:
        - requirements/kv_connectors_rocm.txt
        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
        - vllm/model_executor/layers/fused_moe/
        - vllm/model_executor/layers/quantization/
        - vllm/models/deepseek_v4/
        - vllm/v1/attention/
        - tests/v1/kv_connector/nixl_integration/
        - vllm/_aiter_ops.py
        - vllm/platforms/rocm.py
      commands:
        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
        # A mirror does not inherit the parent env, so it is repeated here in
        # the parent's order. The layouts must stay empty: unset, the script
        # defaults to HND, which the V4 indexer rejects.
        - >-
          VLLM_ROCM_USE_AITER=1
          ENABLE_HMA_FLAG=1
          DP_EP=1
          GPU_MEMORY_UTILIZATION=0.85
          PREFILLER_TP_SIZE=4
          DECODER_TP_SIZE=4
          PREFILL_BLOCK_SIZE=256
          DECODE_BLOCK_SIZE=256
          MODEL_NAMES=deepseek-ai/DeepSeek-V4-Flash
          VLLM_ENGINE_READY_TIMEOUT_S=1800
          VLLM_SERVE_EXTRA_ARGS=--trust-remote-code,--kv-cache-dtype,fp8
          PREFILLER_KV_LAYOUT=
          DECODER_KV_LAYOUT=
          bash v1/kv_connector/nixl_integration/run_accuracy_test.sh

- label: ":nvidia: (H200) Kimi-Linear-48B-A3B Disaggregated DP EP"
  key: kimi-linear-disaggregated
  timeout_in_minutes: 60
  device: h200
  optional: true
  working_dir: "/vllm-workspace/tests"
  num_devices: 4
  env:
    ENABLE_HMA_FLAG: "1"
    DP_EP: "1"
    GPU_MEMORY_UTILIZATION: "0.9"
    VLLM_SSM_CONV_STATE_LAYOUT: "DS"
    PREFILLER_TP_SIZE: "2"
    DECODER_TP_SIZE: "2"
    MODEL_NAMES: "moonshotai/Kimi-Linear-48B-A3B-Instruct"
    # Keep TP=2 prefill and TP=1 decode above both hybrid page-size minima.
    PREFILL_BLOCK_SIZE: "2048"
    DECODE_BLOCK_SIZE: "2048"
    VLLM_SERVE_EXTRA_ARGS: "--trust-remote-code,--mamba-cache-mode align,--max-model-len 32768"
  commands:
    - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
    - bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
  mirror:
    amd:
      label: ":amd: (MI355) Kimi-Linear-48B-A3B Disaggregated DP EP"
      dind: false
      device: mi355_4
      num_devices: 4
      timeout_in_minutes: 105
      working_dir: /vllm-workspace/tests
      depends_on:
      - image-build-amd
      source_file_dependencies:
      - vllm/platforms/rocm.py
      - vllm/models/kimi_k3/
      - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
      - tests/v1/kv_connector/nixl_integration/
      - requirements/kv_connectors_rocm.txt
      env:
        VLLM_WORKER_MULTIPROC_METHOD: "spawn"
      commands:
      - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
      - bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
