From b29e4d86791040a1619be652ecdc36cb2fc0365a Mon Sep 17 00:00:00 2001 From: Michael Weibel Date: Fri, 3 Jul 2026 11:10:25 +0200 Subject: [PATCH 1/3] release: sign assets --- .github/dependabot.yml | 3 +- .github/workflows/govulncheck.yml | 30 +++++++++++ .github/workflows/release.yml | 86 +++++++++++++++++++++++++++++-- Dockerfile | 4 +- Makefile | 11 ++++ docs/releasing.md | 73 ++++++++++++++++++++++++-- go.mod | 2 +- 7 files changed, 197 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/govulncheck.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4fdc0d8..99e5508 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -32,6 +32,7 @@ updates: - package-ecosystem: "docker" directory: "/" schedule: - interval: "monthly" + interval: "weekly" + day: "monday" cooldown: default-days: 7 diff --git a/.github/workflows/govulncheck.yml b/.github/workflows/govulncheck.yml new file mode 100644 index 0000000..ef310e4 --- /dev/null +++ b/.github/workflows/govulncheck.yml @@ -0,0 +1,30 @@ +name: govulncheck + +permissions: {} + +on: + push: + branches: [main] + pull_request: + schedule: + - cron: '0 6 * * 1' + +jobs: + govulncheck: + permissions: + contents: read + name: Run on Ubuntu + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version-file: go.mod + + - name: Run govulncheck + run: make govulncheck diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e84cb7b..f3a073e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,12 +14,16 @@ concurrency: jobs: release: permissions: - contents: write + contents: write # gh release create + id-token: write # keyless cosign + actions/attest OIDC (Fulcio/Rekor) + attestations: write # actions/attest — persist attestations + artifact-metadata: write # actions/attest — create the artifact storage record name: Build and publish release runs-on: ubuntu-latest environment: release env: TAG: ${{ github.ref_name }} + IMG_REPO: quay.io/cloudscalech/cluster-api-cloudscale-controller IMG: quay.io/cloudscalech/cluster-api-cloudscale-controller:${{ github.ref_name }} steps: - name: Checkout @@ -34,6 +38,11 @@ jobs: go-version-file: go.mod cache: false + # Gate the release on reachable vulnerabilities (incl. stdlib/toolchain) + # before we build or push anything. + - name: Run govulncheck + run: make govulncheck + - name: Login to quay.io uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: @@ -41,15 +50,83 @@ jobs: username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_PASSWORD }} + - name: Set up QEMU + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - name: Build and push multi-arch image - run: make docker-buildx IMG=$IMG PLATFORMS=linux/amd64,linux/arm64 + id: build + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + platforms: linux/amd64 + push: true + tags: ${{ env.IMG }} + build-args: | + VERSION=${{ env.TAG }} + # We attach our own SLSA provenance + SBOM attestations below with + # actions/attest; disable BuildKit's inline provenance so the image + # index stays clean and single-sourced. + provenance: false + + - name: Install cosign + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + + # Keyless signature (Sigstore/OIDC) — verifiable with `cosign verify`. + - name: Sign the container image + env: + DIGEST: ${{ steps.build.outputs.digest }} + run: cosign sign --yes "${IMG_REPO}@${DIGEST}" + + # SLSA build provenance attestation, pushed to quay and the GitHub + # attestation store (verifiable with `gh attestation verify`). + - name: Attest build provenance + uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1 + with: + subject-name: ${{ env.IMG_REPO }} + subject-digest: ${{ steps.build.outputs.digest }} + push-to-registry: true + + - name: Generate SBOM + uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0 + with: + image: ${{ env.IMG_REPO }}@${{ steps.build.outputs.digest }} + format: spdx-json + output-file: sbom.spdx.json + upload-artifact: false + upload-release-assets: false + + # SBOM attestation (the same action switches to SBOM mode when sbom-path + # is set), pushed to quay and the GitHub attestation store. + - name: Attest SBOM + uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1 + with: + subject-name: ${{ env.IMG_REPO }} + subject-digest: ${{ steps.build.outputs.digest }} + sbom-path: sbom.spdx.json + push-to-registry: true - name: Build release manifests run: make release-manifests IMG=$IMG + # Checksums for the clusterctl artifacts + a keyless signature over them. + - name: Generate and sign checksums + run: | + cd dist + sha256sum *.yaml > checksums.txt + cosign sign-blob --yes \ + --bundle checksums.txt.bundle \ + checksums.txt + + # SLSA provenance for the release manifest files (GitHub attestation store; + # verifiable with `gh attestation verify --owner cloudscale-ch`). + - name: Attest release manifests provenance + uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1 + with: + subject-path: 'dist/*.yaml' + - name: Create GitHub Release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -62,4 +139,7 @@ jobs: gh release create "$TAG" \ --generate-notes \ $PRERELEASE_FLAG \ - dist/*.yaml + dist/*.yaml \ + dist/checksums.txt \ + dist/checksums.txt.bundle \ + sbom.spdx.json diff --git a/Dockerfile b/Dockerfile index 5c197df..7bbe481 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -# Build the manager binary -FROM golang:1.26.3 AS builder +# BUILDPLATFORM is a predefined BuildKit arg. For a single-platform `docker build` it resolves to the host and is a no-op. +FROM --platform=${BUILDPLATFORM} golang:1.26.5 AS builder ARG TARGETOS ARG TARGETARCH ARG VERSION=dev diff --git a/Makefile b/Makefile index 32d71e9..35347aa 100644 --- a/Makefile +++ b/Makefile @@ -80,6 +80,10 @@ lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes lint-config: golangci-lint ## Verify golangci-lint linter configuration "$(GOLANGCI_LINT)" config verify +.PHONY: govulncheck +govulncheck: govulncheck-tool ## Run govulncheck to scan for known, reachable vulnerabilities (incl. stdlib/toolchain). + "$(GOVULNCHECK)" ./... + ##@ Dependencies ## Location to install dependencies to @@ -95,6 +99,7 @@ CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest GOLANGCI_LINT = $(LOCALBIN)/golangci-lint GINKGO ?= $(LOCALBIN)/ginkgo +GOVULNCHECK ?= $(LOCALBIN)/govulncheck ##@ E2E Testing @@ -378,6 +383,7 @@ ENVTEST_K8S_VERSION ?= $(shell v='$(call gomodver,k8s.io/api)'; \ printf '%s\n' "$$v" | sed -E 's/^v?[0-9]+\.([0-9]+).*/1.\1/') GOLANGCI_LINT_VERSION ?= v2.12.2 +GOVULNCHECK_VERSION ?= v1.5.0 .PHONY: kustomize kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. $(KUSTOMIZE): $(LOCALBIN) @@ -411,6 +417,11 @@ $(GOLANGCI_LINT): $(LOCALBIN) mv -f $(LOCALBIN)/golangci-lint-custom $(GOLANGCI_LINT); \ } || true +.PHONY: govulncheck-tool +govulncheck-tool: $(GOVULNCHECK) ## Download govulncheck locally if necessary. +$(GOVULNCHECK): $(LOCALBIN) + $(call go-install-tool,$(GOVULNCHECK),golang.org/x/vuln/cmd/govulncheck,$(GOVULNCHECK_VERSION)) + # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist # $1 - target path with name of binary # $2 - package url which can be installed diff --git a/docs/releasing.md b/docs/releasing.md index ed62651..f964edd 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -32,13 +32,19 @@ When introducing a new API version or breaking changes, add a new release series 4. **The release workflow runs automatically** Pushing a tag matching `v*.*.*` triggers `.github/workflows/release.yml`, which: - - Builds and pushes a multi-arch Docker image (linux/amd64, linux/arm64) to + - Gates the release on `make govulncheck` (fails on known, reachable vulnerabilities) + - Builds and pushes a Docker image (linux/amd64) to `quay.io/cloudscalech/cluster-api-cloudscale-controller:` - - Generates release manifests via `make release-manifests` + - Signs the image with keyless [cosign](https://docs.sigstore.dev/) and attaches + SLSA build-provenance and an SPDX SBOM as signed attestations + - Generates release manifests via `make release-manifests`, a `checksums.txt` + signed with `cosign sign-blob`, and SLSA provenance for the manifest files - Creates a GitHub release with auto-generated release notes and the following artifacts: - `infrastructure-components.yaml` — all CRDs, RBAC, and controller deployment - `metadata.yaml` — clusterctl provider metadata - - `cluster-template.yaml` — default workload cluster template + - `cluster-template*.yaml` / `cluster-class*.yaml` — workload cluster templates + - `checksums.txt`, `checksums.txt.bundle` — manifest checksums + Sigstore bundle (signature + certificate) + - `sbom.spdx.json` — SBOM of the controller image ## Post-Release Verification @@ -47,7 +53,64 @@ After the workflow completes: 1. **Check the GitHub Actions run** succeeded without errors 2. **Verify the container image** exists on [quay.io](https://quay.io/repository/cloudscalech/cluster-api-cloudscale-controller) -3. **Verify the GitHub release** has all 3 artifacts attached -4. **Test installation** on a fresh management cluster. +3. **Verify the GitHub release** has all artifacts attached (manifests, checksums, + signature, SBOM) +4. **Verify the signatures/attestations** — see [Verifying a Release](#verifying-a-release) +5. **Test installation** on a fresh management cluster. See [Testing Releases](testing-releases.md) for detailed testing instructions. + +## Verifying a Release + +All release artifacts are signed with keyless [Sigstore](https://www.sigstore.dev/) +signatures using GitHub Actions OIDC. The signing +identity is this repository's release workflow at the release tag, i.e. +`https://github.com/cloudscale-ch/cluster-api-provider-cloudscale/.github/workflows/release.yml@refs/tags/` +issued by `https://token.actions.githubusercontent.com`. + +Requires [`cosign`](https://docs.sigstore.dev/cosign/system_config/installation/) +and the [GitHub CLI](https://cli.github.com/) (`gh`). + +### Container image signature + +```bash +IMG=quay.io/cloudscalech/cluster-api-cloudscale-controller:v1.0.0 +cosign verify "$IMG" \ + --certificate-identity-regexp '^https://github.com/cloudscale-ch/cluster-api-provider-cloudscale/\.github/workflows/release\.yml@refs/tags/' \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com +``` + +### Build provenance & SBOM attestations + +```bash +# SLSA provenance + SBOM attestation attached to the image: +gh attestation verify oci://$IMG --owner cloudscale-ch + +# List everything attached to the image (signature + attestations): +cosign tree "$IMG" +``` + +The SBOM is also attached to the GitHub release as `sbom.spdx.json`. + +### Release manifests + +Download `checksums.txt`, `checksums.txt.bundle` and the +`*.yaml` manifests into the same directory, then (run from that directory): + +```bash +# Verify the manifest digests. `shasum` ships by default on both macOS and Linux +# and reads the checksum file; GNU coreutils users can use `sha256sum -c` instead. +# (Do not use BSD `sha256sum -c` — there `-c` compares against a single digest string.) +shasum -a 256 -c checksums.txt + +# The bundle is a self-describing Sigstore bundle carrying the signature, +# signing certificate, and transparency-log proof. +cosign verify-blob \ + --bundle checksums.txt.bundle \ + --certificate-identity-regexp '^https://github.com/cloudscale-ch/cluster-api-provider-cloudscale/\.github/workflows/release\.yml@refs/tags/' \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + checksums.txt + +# Each manifest also carries a SLSA provenance attestation: +gh attestation verify infrastructure-components.yaml --owner cloudscale-ch +``` diff --git a/go.mod b/go.mod index ec891c8..bce90ae 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/cloudscale-ch/cluster-api-provider-cloudscale -go 1.26.0 +go 1.26.5 require ( github.com/cloudscale-ch/cloudscale-go-sdk/v9 v9.1.0 From 606d9f077b86eacaa00415b9f32b90d84e9fc4d2 Mon Sep 17 00:00:00 2001 From: Michael Weibel Date: Fri, 3 Jul 2026 13:36:21 +0200 Subject: [PATCH 2/3] chore: place tools with os/arch file avoids issues when using e.g. macOS to develop and using linux sandboxes for agentic development --- Makefile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 35347aa..060f138 100644 --- a/Makefile +++ b/Makefile @@ -91,6 +91,9 @@ LOCALBIN := $(shell pwd)/bin $(LOCALBIN): mkdir -p "$(LOCALBIN)" +# Host OS/ARCH used to namespace tool binaries in $(LOCALBIN) +HOST_PLATFORM := $(shell go env GOOS)-$(shell go env GOARCH) + ## Tool Binaries KUBECTL ?= kubectl KIND ?= kind @@ -427,15 +430,15 @@ $(GOVULNCHECK): $(LOCALBIN) # $2 - package url which can be installed # $3 - specific version of package define go-install-tool -@[ -f "$(1)-$(3)" ] && [ "$$(readlink -- "$(1)" 2>/dev/null)" = "$(1)-$(3)" ] || { \ +@[ -f "$(1)-$(3)-$(HOST_PLATFORM)" ] && [ "$$(readlink -- "$(1)" 2>/dev/null)" = "$(1)-$(3)-$(HOST_PLATFORM)" ] || { \ set -e; \ package=$(2)@$(3) ;\ -echo "Downloading $${package}" ;\ +echo "Downloading $${package} ($(HOST_PLATFORM))" ;\ rm -f "$(1)" ;\ GOBIN="$(LOCALBIN)" go install $${package} ;\ -mv "$(LOCALBIN)/$$(basename "$(1)")" "$(1)-$(3)" ;\ +mv "$(LOCALBIN)/$$(basename "$(1)")" "$(1)-$(3)-$(HOST_PLATFORM)" ;\ } ;\ -ln -sf "$$(realpath "$(1)-$(3)")" "$(1)" +ln -sf "$$(realpath "$(1)-$(3)-$(HOST_PLATFORM)")" "$(1)" endef define gomodver From 9655240d87265f87c0ce0245b87c99302e013be9 Mon Sep 17 00:00:00 2001 From: Michael Weibel Date: Mon, 20 Jul 2026 16:43:18 +0200 Subject: [PATCH 3/3] release: workflow and documentation review fixes - removes unused QEMU build step - re-adds linux/arm64 because we might need it when using an arm-based seed cluster (e.g. kind on macOS) - during build verifies the checksum bundle before doing the release - improves release verification documentation Co-authored-by: alakae --- .github/workflows/release.yml | 14 ++++-- docs/releasing.md | 94 ++++++++++++++++++++++++++--------- 2 files changed, 81 insertions(+), 27 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f3a073e..c769502 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -50,9 +50,6 @@ jobs: username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_PASSWORD }} - - name: Set up QEMU - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 @@ -61,7 +58,7 @@ jobs: uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: context: . - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 push: true tags: ${{ env.IMG }} build-args: | @@ -120,6 +117,15 @@ jobs: --bundle checksums.txt.bundle \ checksums.txt + # Verify the signature we just created + - name: Verify blob + run: | + cd dist + cosign verify-blob --bundle checksums.txt.bundle \ + --certificate-identity-regexp '^https://github.com/cloudscale-ch/cluster-api-provider-cloudscale/\.github/workflows/release\.yml@refs/tags/' \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + checksums.txt + # SLSA provenance for the release manifest files (GitHub attestation store; # verifiable with `gh attestation verify --owner cloudscale-ch`). - name: Attest release manifests provenance diff --git a/docs/releasing.md b/docs/releasing.md index f964edd..8fee895 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -53,13 +53,15 @@ After the workflow completes: 1. **Check the GitHub Actions run** succeeded without errors 2. **Verify the container image** exists on [quay.io](https://quay.io/repository/cloudscalech/cluster-api-cloudscale-controller) -3. **Verify the GitHub release** has all artifacts attached (manifests, checksums, - signature, SBOM) -4. **Verify the signatures/attestations** — see [Verifying a Release](#verifying-a-release) -5. **Test installation** on a fresh management cluster. +3. **Verify the GitHub release** has all artifacts attached (manifests, checksums with bundle, + SBOM, release attestation) +4. **Test installation** on a fresh management cluster. See [Testing Releases](testing-releases.md) for detailed testing instructions. +Optionally, the release artifacts can be verified using the section below. This is optional because during release +the verification is already done. + ## Verifying a Release All release artifacts are signed with keyless [Sigstore](https://www.sigstore.dev/) @@ -71,46 +73,92 @@ issued by `https://token.actions.githubusercontent.com`. Requires [`cosign`](https://docs.sigstore.dev/cosign/system_config/installation/) and the [GitHub CLI](https://cli.github.com/) (`gh`). +```bash +export TAG=v1.0.0 +export IMG=quay.io/cloudscalech/cluster-api-cloudscale-controller:$TAG +export ID_REGEXP='^https://github.com/cloudscale-ch/cluster-api-provider-cloudscale/\.github/workflows/release\.yml@refs/tags/' +export ISSUER=https://token.actions.githubusercontent.com +``` + ### Container image signature +`cosign sign` pushes the signature into the registry alongside the image itself, not to a separate file. We can verify +the signature using `cosign verify`. + ```bash -IMG=quay.io/cloudscalech/cluster-api-cloudscale-controller:v1.0.0 cosign verify "$IMG" \ - --certificate-identity-regexp '^https://github.com/cloudscale-ch/cluster-api-provider-cloudscale/\.github/workflows/release\.yml@refs/tags/' \ - --certificate-oidc-issuer https://token.actions.githubusercontent.com + --certificate-identity-regexp "$ID_REGEXP" \ + --certificate-oidc-issuer "$ISSUER" ``` ### Build provenance & SBOM attestations ```bash -# SLSA provenance + SBOM attestation attached to the image: -gh attestation verify oci://$IMG --owner cloudscale-ch - -# List everything attached to the image (signature + attestations): +# Build provenance (predicate type slsa.dev/provenance/v1, this happens to be +# gh's default if --predicate-type is omitted, but named explicitly here anyway) +gh attestation verify oci://$IMG --owner cloudscale-ch \ + --predicate-type https://slsa.dev/provenance/v1 + +# The provenance claim's actual content: which commit/workflow run produced +# this image (buildDefinition, runDetails.builder.id): +gh attestation verify oci://$IMG --owner cloudscale-ch \ + --predicate-type https://slsa.dev/provenance/v1 \ + --format json \ + --jq '.[].verificationResult.statement.predicate' + +# SBOM (predicate type spdx.dev/Document/v2.3, must be given explicitly, +# otherwise gh defaults to provenance and the SBOM is never checked) +gh attestation verify oci://$IMG --owner cloudscale-ch \ + --predicate-type https://spdx.dev/Document/v2.3 + +# A verified copy of the SBOM's actual content, extracted from the checked +# attestation rather than trusted from the release asset on its own: +gh attestation verify oci://$IMG --owner cloudscale-ch \ + --predicate-type https://spdx.dev/Document/v2.3 \ + --format json \ + --jq '.[].verificationResult.statement.predicate' + +# Everything attached at once: cosign tree "$IMG" ``` -The SBOM is also attached to the GitHub release as `sbom.spdx.json`. +`sbom.spdx.json` is also attached to the GitHub release directly, but that copy isn't +independently verifiable as a standalone file: its attestation's subject is the image, +not the file. The SBOM extraction command above prints the verified content instead. ### Release manifests -Download `checksums.txt`, `checksums.txt.bundle` and the -`*.yaml` manifests into the same directory, then (run from that directory): +Unlike the image, these are plain files with no registry to push into, so +`cosign sign-blob` produces `checksums.txt.bundle` as a standalone file instead. Both +checks below matter: `shasum` proves your files match `checksums.txt`; the signature +proves `checksums.txt` itself is genuine. Either alone isn't enough. ```bash -# Verify the manifest digests. `shasum` ships by default on both macOS and Linux -# and reads the checksum file; GNU coreutils users can use `sha256sum -c` instead. -# (Do not use BSD `sha256sum -c` — there `-c` compares against a single digest string.) +cd "$(mktemp -d)" +gh release download "$TAG" \ + -R cloudscale-ch/cluster-api-provider-cloudscale \ + -p 'checksums.txt' \ + -p 'checksums.txt.bundle' \ + -p '*.yaml' + +# Files match their checksums. `shasum` ships by default on macOS and Linux; +# GNU coreutils users can use `sha256sum -c` instead (BSD sha256sum's -c compares +# against a single digest string, not a checksum file). shasum -a 256 -c checksums.txt -# The bundle is a self-describing Sigstore bundle carrying the signature, -# signing certificate, and transparency-log proof. +# checksums.txt itself is signed (predicate type sigstore.dev/cosign/sign/v1). The +# bundle is a self-describing Sigstore bundle: signature + certificate + +# transparency-log proof. cosign verify-blob \ --bundle checksums.txt.bundle \ - --certificate-identity-regexp '^https://github.com/cloudscale-ch/cluster-api-provider-cloudscale/\.github/workflows/release\.yml@refs/tags/' \ - --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity-regexp "$ID_REGEXP" \ + --certificate-oidc-issuer "$ISSUER" \ checksums.txt -# Each manifest also carries a SLSA provenance attestation: -gh attestation verify infrastructure-components.yaml --owner cloudscale-ch +# Each manifest also carries its own build provenance (predicate type +# slsa.dev/provenance/v1, gh's default, named explicitly here anyway): +for f in *.yaml; do + gh attestation verify "$f" --owner cloudscale-ch \ + --predicate-type https://slsa.dev/provenance/v1 +done ```