diff --git a/.github/workflows/_ci-run-decision.yml b/.github/workflows/_ci-run-decision.yml index 99413f17d05..8e301ad70d8 100644 --- a/.github/workflows/_ci-run-decision.yml +++ b/.github/workflows/_ci-run-decision.yml @@ -14,9 +14,23 @@ name: CI Run Decision # - pull_request / pull_request_target (use path filter instead) # - push events not matching any of the above (path-filtered runs) # -# See ``viable-strict-gate.yml``: viable/strict only advances on -# commits where this is true, so the path-filtered fast path doesn't -# silently advance partial signal. +# viable/strict eligibility is not decided here. It follows from the +# ``ciflow/trunk`` tag that ``promote-to-viable-strict.yml`` pushes; see +# ``viable-strict-gate.yml``. +# +# Do not assume promotion makes the depth sample redundant. Seven +# workflows list ``ciflow/trunk/*`` under ``on.push.tags`` and so re-run +# from a promotion tag: pull, trunk, apple, windows-msvc, riscv64, +# qnn-windows-msvc and viable-strict-gate. Only four of those call this +# workflow. The other seven callers (cuda, cuda-windows, cuda-perf, +# metal, mlx, rocm, vulkan) do have ciflow tags, but none of them listens +# for ``ciflow/trunk``, so a promotion tag never starts them, and they +# hold 32 of the 48 jobs whose ``if:`` gates on ``is-full-run``. Most of +# those 32 also have a changed-files branch and still run on main when +# their own paths are touched; the 12 in mlx.yml do not, so for those the +# depth sample below is the only thing that runs them automatically on a +# push to main. (``workflow_dispatch`` and ``ciflow/mlx`` can still force +# them by hand, which is not a substitute for running on main.) on: workflow_call: diff --git a/.github/workflows/promote-to-viable-strict.yml b/.github/workflows/promote-to-viable-strict.yml index a750bef4d0d..d2d737f1669 100644 --- a/.github/workflows/promote-to-viable-strict.yml +++ b/.github/workflows/promote-to-viable-strict.yml @@ -1,39 +1,42 @@ name: Promote commit to viable/strict -# Manual escape hatch for the sampled-CI gating in -# `_ci-run-decision.yml` + `viable-strict-gate.yml`. +# Grants a commit eligibility for viable/strict by pushing a +# `ciflow/trunk/` tag at it, which: +# 1. Runs `pull.yml` / `trunk.yml` against that commit with +# ``is-full-run = true``, so every gated job runs regardless of the +# path filter. +# 2. Runs `viable-strict-gate.yml`, whose presence is what +# `update-viablestrict` requires before it will advance. # -# Pushes a `ciflow/trunk/` tag at a chosen commit, which: -# 1. Re-triggers `pull.yml` / `trunk.yml` against that commit with -# ``is-full-run = true`` (every gated job runs regardless of -# path filter or SHA sample). -# 2. Triggers `viable-strict-gate.yml` for that commit; the gate -# succeeds because tag pushes always count as a full-run. +# This runs on a schedule, because landing a commit does not make it +# eligible on its own. Without a periodic promotion the tip of main +# would sit outside viable/strict until somebody promoted it by hand. # -# Once those tag-triggered runs all pass, the next -# `update-viablestrict` cron run will be able to advance viable/strict -# to the chosen commit. +# The schedule includes a tick several hours before the nightly branch +# is cut, so the last commits of a quiet night still have time to run +# full CI and reach that day's nightly build. # -# Use cases: -# - Bisecting a regression on a non-sampled commit. -# - Pre-release validation: pin viable/strict to a specific commit -# (e.g. release branch tip) regardless of its SHA's sample bit. -# - Recovering when recent sampled commits all happen to be red. +# Can also be dispatched by hand for a specific commit, for example to +# force full CI on one commit while bisecting a regression. Note that +# promoting a commit that is only on a release branch runs CI for it but +# will never advance viable/strict: the updater only considers commits +# reachable from the branch it checks out, which is main. (The +# `main-branch: main` line in update-viablestrict.yml looks like it +# controls this, but the action does not declare that input and passes a +# hardcoded value, so it has no effect either way.) on: + schedule: + # Every 4 hours. The 08:00 UTC tick is the one that lands ahead of + # the nightly branch cut. + - cron: "0 0,4,8,12,16,20 * * *" workflow_dispatch: inputs: sha: - description: "Full 40-char SHA on main / release/* to promote" - required: true + description: "Full 40-char SHA on main / release/* to promote. Defaults to the tip of main." + required: false type: string -permissions: - contents: write - # Needed to delete the failed `viable-strict-gate` run that the - # original push triggered — see the "Delete failed gate runs" step. - actions: write - concurrency: # One in-flight promotion at a time; safer than racing tag pushes. group: promote-to-viable-strict @@ -43,17 +46,57 @@ jobs: push-ciflow-tag: if: ${{ github.repository_owner == 'pytorch' }} runs-on: ubuntu-22.04 + # GH_PYTORCHBOT_TOKEN is an environment secret, not a repository + # secret, so a job that does not name the environment cannot read it + # and would see an empty string. Same pattern as + # weekly-pytorch-pin-bump.yml. + # + # This environment's deployment branch policy allows `main` only, so + # a hand dispatch from any other ref is refused before the first + # step. That also means this job cannot be exercised from a pull + # request branch; it is first live after merge. + environment: update-commit-hash + permissions: + # Both the checkout and the tag push use the bot token below, so no + # step uses GITHUB_TOKEN at all. Declared read-only rather than + # omitted so that a future step cannot silently inherit write. + contents: read steps: + # Checked before the tag is pushed rather than after, so a missing + # secret is reported as itself instead of as an opaque checkout + # error or, worse, a tag that silently starts nothing. + - name: Check the bot token is configured + env: + BOT_TOKEN: ${{ secrets.GH_PYTORCHBOT_TOKEN }} + run: | + set -eu + if [ -z "${BOT_TOKEN:-}" ]; then + echo "::error::GH_PYTORCHBOT_TOKEN resolved to an empty string. It is an environment secret on the 'update-commit-hash' environment, so check that this job still declares that environment and that the secret is still present on it. Promotion needs a bot token with contents write access, because a ref pushed with the default GITHUB_TOKEN does not start any workflow run and the tag would do nothing." + exit 1 + fi + - uses: actions/checkout@v4 with: fetch-depth: 0 + # A ref pushed with the default GITHUB_TOKEN does not start any + # workflow run, which would leave the tag below doing nothing. + # Push as the bot so the tag triggers full CI and the gate, + # which is the entire point of promoting a commit. + token: ${{ secrets.GH_PYTORCHBOT_TOKEN }} - name: Validate SHA and push ciflow tag env: - SHA: ${{ inputs.sha }} + INPUT_SHA: ${{ inputs.sha }} run: | set -euo pipefail + # A scheduled run has no inputs, so promote the tip of main. + SHA="${INPUT_SHA:-}" + if [ -z "$SHA" ]; then + SHA=$(git rev-parse origin/main) + echo "No SHA given; promoting the tip of main: $SHA" + fi + # Reject anything that isn't a full 40-char lowercase hex SHA. if [[ ! "$SHA" =~ ^[0-9a-f]{40}$ ]]; then echo "::error::Input must be a full 40-char lowercase hex SHA; got: '$SHA'" @@ -90,10 +133,25 @@ jobs: exit 1 fi + # Nothing to promote if viable/strict already contains the + # commit. This is the common case for a scheduled tick that + # finds main unchanged since the last one. + # + # Only applied when we picked the SHA ourselves. Almost all of + # main is already an ancestor of viable/strict, so applying it + # to a hand-dispatched SHA would turn nearly every deliberate + # request into a silent no-op that still reports success. + if [ -z "${INPUT_SHA:-}" ] \ + && git merge-base --is-ancestor "$SHA" origin/viable/strict 2>/dev/null; then + echo "$SHA is already contained in viable/strict; nothing to do." + exit 0 + fi + TAG="ciflow/trunk/$SHA" - # If the tag already exists (e.g. someone already promoted - # this commit), exit cleanly — no-op is a valid outcome. + # If the tag already exists (e.g. an earlier tick promoted + # this same commit), exit cleanly. This is what keeps repeated + # ticks over a quiet period from re-running CI. if git ls-remote --tags --exit-code origin "refs/tags/$TAG" >/dev/null 2>&1; then echo "Tag $TAG already exists on origin; nothing to do." exit 0 @@ -104,42 +162,11 @@ jobs: git tag "$TAG" "$SHA" git push origin "$TAG" - echo "::notice::Pushed $TAG. Watch the tag-triggered workflow runs (pull / trunk / viable-strict-gate); once they pass, the next update-viablestrict cron (every 30 min) will advance viable/strict." - - # Defense-in-depth: the push that originally landed this commit - # triggered a `viable-strict-gate` run that failed (because the - # commit wasn't sampled). The tag push above triggers a NEW run - # of the gate workflow that will succeed. Standard PyTorch viable/ - # strict resolves multiple runs by taking the latest conclusion, - # so this is usually fine — but to remove ambiguity (and keep the - # commit's HUD row clean), explicitly delete any prior failed/ - # cancelled gate runs on this SHA. - - name: Delete failed viable-strict-gate runs on this SHA - env: - GH_TOKEN: ${{ github.token }} - SHA: ${{ inputs.sha }} - REPO: ${{ github.repository }} - run: | - set -euo pipefail - - # List all viable-strict-gate runs for the SHA, filter to - # those that completed unsuccessfully, and delete each one. - # Failures here are non-fatal: the tag push above is the - # primary mechanism; this cleanup is best-effort. - RUNS=$(gh api "repos/$REPO/actions/runs?head_sha=$SHA&per_page=100" \ - --jq '.workflow_runs[] - | select(.name == "viable-strict-gate") - | select(.conclusion == "failure" or .conclusion == "cancelled" or .conclusion == "timed_out") - | .id' 2>/dev/null || true) - - if [ -z "$RUNS" ]; then - echo "No prior failed viable-strict-gate runs to clean up." - exit 0 - fi - - while IFS= read -r RUN_ID; do - [ -z "$RUN_ID" ] && continue - echo "Deleting failed viable-strict-gate run $RUN_ID" - gh api -X DELETE "repos/$REPO/actions/runs/$RUN_ID" || \ - echo "::warning::Failed to delete run $RUN_ID; continuing anyway." - done <<< "$RUNS" + # Only the workflows that list ciflow/trunk/* under `on.push.tags` + # start from this tag. Four of them are in update-viablestrict's + # required set; the other three run but do not gate. `lint` and + # `Build documentation` are required too, and neither has a + # ciflow/trunk trigger, so their rows come from the earlier push + # to main; if either was cancelled on that push, this tag cannot + # revive it. + echo "::notice::Pushed $TAG. Workflow runs started by this tag: pull, trunk, Apple, viable-strict-gate, Windows MSVC Build, Test RISC-V Backend, Test QNN Windows MSVC build. Of those, update-viablestrict only requires pull, trunk, Apple and viable-strict-gate; the other three do not gate advancement. The two remaining required checks, lint and Build documentation, do not start from a ciflow/trunk tag and must already be green from the push to main. Once all six required checks are green, the next update-viablestrict cron (every 30 min) will advance viable/strict." diff --git a/.github/workflows/viable-strict-gate.yml b/.github/workflows/viable-strict-gate.yml index d25b57803b9..63cb7a67388 100644 --- a/.github/workflows/viable-strict-gate.yml +++ b/.github/workflows/viable-strict-gate.yml @@ -1,27 +1,30 @@ name: viable-strict-gate -# Sampled-full-run gating for viable/strict advancement. +# Marks a commit as eligible for viable/strict advancement. # # Path filtering on push to main saves runner cost but risks advancing -# viable/strict on commits where many jobs were skipped — a partial -# green from "no job ran" is indistinguishable from "everything passed" -# at the workflow-conclusion level. +# viable/strict on commits where many jobs were skipped: a partial green +# from "no job ran" is indistinguishable from "everything passed" at the +# workflow-conclusion level. # -# This workflow runs on every push to main / release branches and -# *fails* when ``_ci-run-decision.yml`` says this isn't a full-coverage -# commit (i.e. the SHA isn't sampled and there's no ciflow/* tag). -# Failure => the "viable-strict-gate" workflow conclusion is failure -# => update-viablestrict refuses to advance viable/strict. +# So this workflow runs only on ``ciflow/trunk/`` tags, for which +# ``_ci-run-decision.yml`` always returns ``is-full-run = true``. A plain +# push to main produces no run of this workflow at all, so +# update-viablestrict reports the required check as missing and declines +# to advance. Eligibility is granted by promoting a commit, never by +# landing it. ``promote-to-viable-strict.yml`` pushes those tags, on a +# schedule and on demand. # -# To force a full run on a specific commit (e.g. before tagging a -# release), push a ``ciflow/trunk/`` tag — on tag pushes -# ``_ci-run-decision.yml`` always returns ``is-full-run = true``. +# This workflow used to run on every push and fail on commits outside +# the sample. That wrote a permanent failure: the viable/strict updater +# reads one row per workflow *run* rather than per workflow, so a later +# successful run sat beside the failed one instead of replacing it, and +# a single failure anywhere is fatal. A commit that landed outside the +# sample could therefore never be promoted afterwards. Producing no run +# at all avoids that, because there is nothing to sit beside. on: push: - branches: - - main - - release/* tags: - ciflow/trunk/* @@ -37,6 +40,20 @@ jobs: name: Full CI required for viable/strict runs-on: ubuntu-22.04 steps: + # Only tag pushes reach this workflow, so is-full-run is always + # true here. Kept as an assertion against a future trigger that + # does not force a full run. + # + # Failing is the correct behaviour for a tag push, which is a + # deliberate request, but it is NOT a safe way to reject ordinary + # commits: a failure row here is permanent, for the reason in the + # header. So do not re-add ``push: branches`` above. Adding + # ``workflow_dispatch`` is a different trap: it would make this + # assertion pass and publish a viable-strict-gate success for a + # commit whose CI was path-filtered. ``schedule`` is unsafe for the + # opposite reason: ``_ci-run-decision.yml`` returns false for a + # schedule event, so every tick would fail here and write a + # permanent failure row on the tip of main. - name: Check whether this commit is a full-coverage run env: IS_FULL_RUN: ${{ needs.run-decision.outputs.is-full-run }} @@ -47,6 +64,5 @@ jobs: exit 0 fi echo "::error::Non-full-run commit (path-filtered CI). viable/strict cannot advance from this commit." - echo "Full CI runs on every 4th commit on main / release/* (depth %% 4 == 0)." - echo "To force a full run on this commit, push a 'ciflow/trunk/${{ github.sha }}' tag." + echo "This workflow should only ever run on a 'ciflow/trunk/' tag, which always forces a full run." exit 1