Repository navigation
ete_test_gpu-glm5-2-sft-30B-mtp-fp8 #318
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ete_test_gpu | |
| run-name: ete_test_gpu-${{ github.event.inputs.run_case || 'all' }} | |
| permissions: | |
| contents: read | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| repo_org: | |
| required: false | |
| description: 'Tested repository organization name. Default is InternLM' | |
| type: string | |
| default: 'InternLM/xtuner' | |
| repo_ref: | |
| required: false | |
| description: 'Set branch or tag or commit id. Default is "main"' | |
| type: string | |
| default: 'main' | |
| run_case: | |
| required: false | |
| description: >- | |
| Case name(s) from autotest/config.yaml. Use "all" for every case, | |
| one name (e.g. "qwen3-rl-lmdeploy"), or comma-separated names | |
| (e.g. "qwen3-5-rl-vl-lmdeploy-grpo,s2-preview-rl-vl-lmdeploy-grpo"). | |
| type: string | |
| default: 'all' | |
| parallel: | |
| required: false | |
| description: >- | |
| pytest-xdist worker count (-n). Cases from config.yaml run in parallel. | |
| Use a number (e.g. 4), or "all" for one worker per selected case. | |
| Resume phases within one case stay sequential. | |
| type: string | |
| default: '4' | |
| CI_E2E_DEBUG: | |
| required: false | |
| description: >- | |
| CI_E2E_DEBUG:if true, start "sleep inf &" after install and before training so the | |
| cluster job stays alive for interactive debug (including after OOM). | |
| Default false. | |
| type: boolean | |
| default: false | |
| train_image: | |
| required: false | |
| description: >- | |
| Override train image tag only (e.g. ailab-llmrazor/xtuner:pt29_latest). | |
| Empty uses config.yaml default. | |
| type: string | |
| default: '' | |
| clusterx_partition: | |
| required: false | |
| description: >- | |
| clusterx partition (yidian group), e.g. llmrazor_gpu. | |
| Empty uses llmrazor_gpu. | |
| type: string | |
| default: 'llmrazor_gpu' | |
| clusterx_project_name: | |
| required: false | |
| description: >- | |
| clusterx project namespace, e.g. ailab-llmrazor. | |
| Empty uses ailab-llmrazor. | |
| type: string | |
| default: 'ailab-llmrazor' | |
| skip_xtuner_install: | |
| required: false | |
| description: >- | |
| If true, skip ``pip install -e .`` in train jobs (use when the image | |
| already contains a preinstalled xtuner). | |
| type: boolean | |
| default: false | |
| runner_label: | |
| required: false | |
| description: >- | |
| GitHub Actions runner label for the ETE job. | |
| Default is h_cluster_ete. | |
| type: string | |
| default: 'h_cluster_ete' | |
| env: | |
| CI_GPU_IMAGE_REGISTRY: ${{ secrets.CI_GPU_IMAGE_REGISTRY }} | |
| CI_ETE_TRAIN_IMAGE: ${{ github.event.inputs.train_image }} | |
| CI_ETE_CLUSTERX_PARTITION: ${{ github.event.inputs.clusterx_partition || 'llmrazor_gpu' }} | |
| CI_ETE_CLUSTERX_PROJECT_NAME: ${{ github.event.inputs.clusterx_project_name || 'ailab-llmrazor' }} | |
| CI_ETE_SKIP_XTUNER_INSTALL: ${{ github.event.inputs.skip_xtuner_install }} | |
| # ETE PNG reports: llmcitester/xtuner @ branch reports, under ete/{run_id}/ | |
| CI_REPORTS_REPOSITORY: ${{ vars.CI_REPORTS_REPOSITORY || 'llmcitester/xtuner' }} | |
| CI_REPORTS_BRANCH: ${{ vars.CI_REPORTS_BRANCH || 'reports' }} | |
| CI_REPORTS_RAW_URL_BASE: ${{ vars.CI_REPORTS_RAW_URL_BASE || 'https://raw.githubusercontent.com/llmcitester/xtuner/reports/ete' }} | |
| CI_REPORTS_BROWSE_URL_BASE: ${{ vars.CI_REPORTS_BROWSE_URL_BASE || 'https://github.com/llmcitester/xtuner/tree/reports/ete' }} | |
| CI_REPORTS_KEEP_RUNS: ${{ vars.CI_REPORTS_KEEP_RUNS || '30' }} | |
| jobs: | |
| ete_test: | |
| if: ${{!cancelled() }} | |
| runs-on: ${{ github.event.inputs.runner_label || 'h_cluster_ete' }} | |
| timeout-minutes: 1440 | |
| steps: | |
| - name: Clean workdir | |
| run: sudo git clean -ffdx | |
| - name: Clone repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.inputs.repo_org || 'InternLM/xtuner' }} | |
| ref: ${{github.event.inputs.repo_ref || 'main'}} | |
| - name: Init job summary | |
| run: | | |
| { | |
| echo "## ETE test report" | |
| echo "" | |
| echo "- Workflow run: [#${{ github.run_number }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})" | |
| echo "- Reports: [ete/${{ github.run_id }}](${{ env.CI_REPORTS_BROWSE_URL_BASE }}/${{ github.run_id }})" | |
| echo "- Train image: \`${{ env.CI_ETE_TRAIN_IMAGE || 'ailab-llmrazor/xtuner:pt29_latest (config default)' }}\`" | |
| echo "- clusterx partition: \`${{ env.CI_ETE_CLUSTERX_PARTITION }}\`" | |
| echo "- clusterx project: \`${{ env.CI_ETE_CLUSTERX_PROJECT_NAME }}\`" | |
| echo "- runner label: \`${{ github.event.inputs.runner_label || 'h_cluster_ete' }}\`" | |
| echo "- skip xtuner install: \`${{ env.CI_ETE_SKIP_XTUNER_INSTALL }}\`" | |
| echo "" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: run-test | |
| env: | |
| GITHUB_RUN_ID: ${{ github.run_id }} | |
| CI_REPORTS_RAW_URL_BASE: ${{ env.CI_REPORTS_RAW_URL_BASE }} | |
| CI_E2E_DEBUG: ${{ github.event.inputs.CI_E2E_DEBUG || 'false' }} | |
| CI_ETE_TRAIN_IMAGE: ${{ env.CI_ETE_TRAIN_IMAGE }} | |
| CI_ETE_CLUSTERX_PARTITION: ${{ env.CI_ETE_CLUSTERX_PARTITION }} | |
| CI_ETE_CLUSTERX_PROJECT_NAME: ${{ env.CI_ETE_CLUSTERX_PROJECT_NAME }} | |
| CI_ETE_SKIP_XTUNER_INSTALL: ${{ env.CI_ETE_SKIP_XTUNER_INSTALL }} | |
| run: | | |
| source /mnt/shared-storage-user/llmrazor-share/qa-llm-cicd/miniconda3/bin/activate | |
| conda activate clusterx | |
| conda env list | |
| unset HTTP_PROXY;unset HTTPS_PROXY;unset http_proxy;unset https_proxy; | |
| echo "Train image: ${CI_ETE_TRAIN_IMAGE:-ailab-llmrazor/xtuner:pt29_latest (config default)}" | |
| echo "clusterx partition: ${CI_ETE_CLUSTERX_PARTITION}" | |
| echo "clusterx project: ${CI_ETE_CLUSTERX_PROJECT_NAME}" | |
| echo "runner label: ${{ github.event.inputs.runner_label || 'h_cluster_ete' }}" | |
| echo "skip xtuner install: ${CI_ETE_SKIP_XTUNER_INSTALL:-false}" | |
| CASE_INPUT="${{ github.event.inputs.run_case || 'all' }}" | |
| PARALLEL_INPUT="${{ github.event.inputs.parallel || '4' }}" | |
| PYTEST_NODES=() | |
| if [ "$CASE_INPUT" = "all" ]; then | |
| mapfile -t CASE_NAMES < <(python -c 'import yaml; cfg=yaml.safe_load(open("autotest/config.yaml",encoding="utf-8")); print("\n".join(cfg["case"]))') | |
| for case_name in "${CASE_NAMES[@]}"; do | |
| [ -z "$case_name" ] && continue | |
| PYTEST_NODES+=("autotest/test_all.py::test_all[${case_name}]") | |
| done | |
| else | |
| # Support one or more cases: "case_a" or "case_a,case_b, case_c" | |
| IFS=',' read -ra RAW_CASES <<< "$CASE_INPUT" | |
| for raw in "${RAW_CASES[@]}"; do | |
| case_name="$(echo "$raw" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" | |
| [ -z "$case_name" ] && continue | |
| PYTEST_NODES+=("autotest/test_all.py::test_all[${case_name}]") | |
| done | |
| fi | |
| if [ "${#PYTEST_NODES[@]}" -eq 0 ]; then | |
| echo "No valid case names parsed from run_case='${CASE_INPUT}'" | |
| exit 1 | |
| fi | |
| CASE_COUNT="${#PYTEST_NODES[@]}" | |
| if [ "$PARALLEL_INPUT" = "all" ]; then | |
| PARALLEL="$CASE_COUNT" | |
| elif [[ "$PARALLEL_INPUT" =~ ^[1-9][0-9]*$ ]]; then | |
| PARALLEL="$PARALLEL_INPUT" | |
| else | |
| echo "Invalid parallel='${PARALLEL_INPUT}', fallback to 4" | |
| PARALLEL=4 | |
| fi | |
| if [ "$PARALLEL" -gt "$CASE_COUNT" ]; then | |
| PARALLEL="$CASE_COUNT" | |
| fi | |
| echo "Running ${CASE_COUNT} case(s) with pytest-xdist -n ${PARALLEL}" | |
| echo "Cases: ${PYTEST_NODES[*]}" | |
| pytest "${PYTEST_NODES[@]}" -m all -n "${PARALLEL}" -s -vv --run_id ${{ github.run_id }} | |
| - name: Check report files | |
| if: always() | |
| id: reports | |
| run: | | |
| REPORT_DIR="${{ github.workspace }}/${{ github.run_id }}" | |
| if compgen -G "${REPORT_DIR}/*.png" > /dev/null; then | |
| echo "has_reports=true" >> "$GITHUB_OUTPUT" | |
| echo "Found $(ls -1 "${REPORT_DIR}"/*.png | wc -l) comparison plot(s)." | |
| else | |
| echo "has_reports=false" >> "$GITHUB_OUTPUT" | |
| echo "No comparison plots found under ${REPORT_DIR}, skip publish steps." | |
| fi | |
| - name: Upload comparison plots | |
| if: always() && steps.reports.outputs.has_reports == 'true' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: ${{ github.workspace }}/${{ github.run_id }}/*.png | |
| if-no-files-found: error | |
| retention-days: 7 | |
| name: xtuner-ete-${{ github.run_id }} | |
| - name: Publish and prune ete reports | |
| if: always() && steps.reports.outputs.has_reports == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.CI_REPORTS_DEPLOY_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| REPO="${{ env.CI_REPORTS_REPOSITORY }}" | |
| BRANCH="${{ env.CI_REPORTS_BRANCH }}" | |
| KEEP="${{ env.CI_REPORTS_KEEP_RUNS }}" | |
| SRC="${{ github.workspace }}/${{ github.run_id }}" | |
| DEST="ete/${{ github.run_id }}" | |
| git clone --depth=1 -b "${BRANCH}" "https://x-access-token:${GH_TOKEN}@github.com/${REPO}.git" pages | |
| cd pages | |
| rm -rf "${DEST}" | |
| mkdir -p "$(dirname "${DEST}")" | |
| cp -r "${SRC}" "${DEST}" | |
| if [ -d ete ]; then | |
| cd ete | |
| mapfile -t RUN_DIRS < <(find . -maxdepth 1 -type d -regextype posix-egrep -regex './[0-9]+' -printf '%f\n' | sort -rn) | |
| if [ "${#RUN_DIRS[@]}" -gt "${KEEP}" ]; then | |
| for old_run in "${RUN_DIRS[@]:KEEP}"; do | |
| rm -rf "${old_run}" | |
| echo "Removed old report: ${old_run}" | |
| done | |
| fi | |
| cd .. | |
| fi | |
| git add -A | |
| if git diff --staged --quiet; then | |
| echo "No report changes to commit." | |
| else | |
| git -c user.name="github-actions[bot]" -c user.email="github-actions[bot]@users.noreply.github.com" \ | |
| commit -m "chore: publish ete report ${{ github.run_id }}, keep latest ${KEEP} runs" | |
| git push | |
| fi |