diff --git a/.forgejo/workflows/backup-verify.yml b/.forgejo/workflows/backup-verify.yml new file mode 100644 index 0000000..78f909e --- /dev/null +++ b/.forgejo/workflows/backup-verify.yml @@ -0,0 +1,62 @@ +# backup-verify — weekly "a backup is not trusted until restored" (CONTRACT_004 +# §4.6, T14 state-dependent half). Produces a fresh bundle and asserts it restores +# from the OFFSITE copy into scratch resources (NON-DESTRUCTIVE — restore.sh never +# touches the live platform). Runs on the foundation's own runner in foundation-ci. +# +# It reuses the operator scripts UNCHANGED (backup/backup.sh + restore.sh), which +# read everything from `pulumi config get` (vm coords, vault root token, offsite + +# age creds — all in the committed, passphrase-encrypted Pulumi.foundation.yaml) and +# orchestrate the heavy lifting on the VM over SSH (ADR-007). Two CI-specific needs: +# 1. the operator SSH key (SSH_PRIVATE_KEY secret → /tmp/op_key) to reach the VM; +# 2. real Pulumi STATE imported into a file backend, because backup.sh embeds a +# `pulumi stack export` (pulumi-state.json, CONTRACT_004 §4.2) in the bundle — +# a bare `stack init` would ship an EMPTY deployment. State is pulled from the +# object run.sh publishes (state-publish.sh), same as pulumi-preview. +# +# Schedule only (+ manual dispatch): never on push — it creates real backup bundles. +name: backup-verify +on: + schedule: + - cron: "17 3 * * 0" # weekly, Sunday 03:17 UTC + workflow_dispatch: + +jobs: + backup-verify: + runs-on: docker + container: + image: foundation-ci:latest + env: + PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }} + SSH_PRIVATE_KEY_PATH: /tmp/op_key + PULUMI_SKIP_UPDATE_CHECK: "true" + steps: + - uses: actions/checkout@v4 + + - name: Materialize operator SSH key (reaches the VM over SSH) + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + run: | + install -m 600 /dev/null /tmp/op_key + printf '%s' "$SSH_PRIVATE_KEY" > /tmp/op_key + [ -z "$(tail -c1 /tmp/op_key)" ] || printf '\n' >> /tmp/op_key + + - name: Import Pulumi state (so the bundle's pulumi-state.json is real) + env: + RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }} + RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }} + working-directory: bootstrap + run: | + mc alias set rfs http://foundation-rustfs:9000 "$RUSTFS_ACCESS_KEY" "$RUSTFS_SECRET_KEY" >/dev/null + mc cp rfs/foundation-ci-state/foundation-stack.json /tmp/foundation-stack.json + mkdir -p state + export PULUMI_BACKEND_URL="file://$(pwd)/state" + pulumi stack select foundation 2>/dev/null || pulumi stack init foundation + pulumi stack import --file /tmp/foundation-stack.json + + - name: Backup, then restore-verify from offsite (CONTRACT_004 §4.6) + run: | + TS=$(date -u +%Y%m%dT%H%M%SZ) + echo "backup-verify: bundle timestamp $TS" + ./backup/backup.sh "$TS" + ./backup/restore.sh "$TS" off + echo "backup-verify: OK ($TS restored from offsite)" diff --git a/.forgejo/workflows/pulumi-preview.yml b/.forgejo/workflows/pulumi-preview.yml new file mode 100644 index 0000000..5ad9562 --- /dev/null +++ b/.forgejo/workflows/pulumi-preview.yml @@ -0,0 +1,64 @@ +# pulumi-preview — read-only infra drift / PR check (T14, state-dependent half). +# +# Runs on the foundation's OWN runner in the baked foundation-ci image. Unlike +# ci.yml (self-contained), this needs Pulumi STATE, which bootstrap/state/ is +# gitignored from. It pulls the latest `pulumi stack export` that run.sh publishes +# to rfs/foundation-ci-state/foundation-stack.json on every `up` (state-publish.sh), +# imports it into a throwaway file backend, and previews. CONFIG + encrypted +# secrets come from the committed Pulumi.foundation.yaml (the passphrase decrypts). +# +# READ-ONLY: `pulumi preview` only — NEVER `up`. It dials the VM docker over SSH +# the same way the operator does (ADR-007), so the operator key is materialized +# from the SSH_PRIVATE_KEY secret. A diff is informational (expected on infra PRs), +# so the job does NOT fail on changes — it fails only if the program/preview errors. +# +# Secrets (repo-scoped, set via the admin API): PULUMI_CONFIG_PASSPHRASE, +# SSH_PRIVATE_KEY (operator ed25519), RUSTFS_ACCESS_KEY/RUSTFS_SECRET_KEY (scoped +# RustFS service account, to fetch the state object over foundation-net). +name: pulumi-preview +on: + push: + pull_request: + workflow_dispatch: + +jobs: + preview: + runs-on: docker + container: + image: foundation-ci:latest + env: + PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }} + SSH_PRIVATE_KEY_PATH: /tmp/op_key + PULUMI_SKIP_UPDATE_CHECK: "true" + steps: + - uses: actions/checkout@v4 + + - name: Install workspace deps (bun workspace, root lockfile) + run: bun install --frozen-lockfile || bun install + + - name: Materialize operator SSH key (provider + index.ts read it) + env: + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + run: | + install -m 600 /dev/null /tmp/op_key + printf '%s' "$SSH_PRIVATE_KEY" > /tmp/op_key + # OpenSSH keys must end in a newline; add one only if missing. + [ -z "$(tail -c1 /tmp/op_key)" ] || printf '\n' >> /tmp/op_key + ssh-keygen -y -f /tmp/op_key > /tmp/op_key.pub + + - name: Fetch Pulumi state from RustFS + env: + RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }} + RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }} + run: | + mc alias set rfs http://foundation-rustfs:9000 "$RUSTFS_ACCESS_KEY" "$RUSTFS_SECRET_KEY" >/dev/null + mc cp rfs/foundation-ci-state/foundation-stack.json /tmp/foundation-stack.json + + - name: Pulumi preview (read-only — never up) + working-directory: bootstrap + run: | + mkdir -p state # bootstrap/state/ is gitignored — create the file backend dir + export PULUMI_BACKEND_URL="file://$(pwd)/state" + pulumi stack select foundation 2>/dev/null || pulumi stack init foundation + pulumi stack import --file /tmp/foundation-stack.json + pulumi preview --non-interactive --diff diff --git a/VERSIONS b/VERSIONS index 8fa8ab5..527f8e3 100644 --- a/VERSIONS +++ b/VERSIONS @@ -94,7 +94,10 @@ IMAGE_CI=foundation-ci:latest # ----------------------------------------------------------------------------- # --- orchestration toolchain --- -TOOL_PULUMI_MIN=3.140.0 +# 3.149 is the floor for the `packagemanager: bun` project option (bootstrap/ +# Pulumi.yaml); older CLIs reject "bun". The foundation-ci image pins a concrete +# version >= this (containers/ci-image/Dockerfile PULUMI_VERSION). +TOOL_PULUMI_MIN=3.149.0 TOOL_BUN_MIN=1.1.0 TOOL_NODE_MIN=20.0.0 diff --git a/bootstrap/Pulumi.foundation.yaml b/bootstrap/Pulumi.foundation.yaml index e1508e1..b4b34ca 100644 --- a/bootstrap/Pulumi.foundation.yaml +++ b/bootstrap/Pulumi.foundation.yaml @@ -36,6 +36,7 @@ config: - forgejo-artifacts - forgejo-lfs - foundation-backups + - foundation-ci-state foundation:forgejo.adminUser: platform-admin foundation:forgejo.orgName: olsitec foundation:runner.labels: diff --git a/bootstrap/components/rustfs.ts b/bootstrap/components/rustfs.ts index 91ee6bb..2fba234 100644 --- a/bootstrap/components/rustfs.ts +++ b/bootstrap/components/rustfs.ts @@ -50,7 +50,7 @@ docker run --rm --network foundation-net --entrypoint sh \ -e MC_HOST_rfs="$MCHOST" -e ROOT_AK="$ROOT_AK" -e SVC_AK="$SVC_AK" -e SVC_SK="$SVC_SK" \ "$MC_IMAGE" -c ' set -e - for b in forgejo-packages forgejo-artifacts forgejo-lfs foundation-backups; do + for b in forgejo-packages forgejo-artifacts forgejo-lfs foundation-backups foundation-ci-state; do mc mb --ignore-existing "rfs/$b" done EXISTING=$(mc admin user svcacct ls rfs "$ROOT_AK" 2>/dev/null || true) diff --git a/bootstrap/run.sh b/bootstrap/run.sh index e00231c..00e6a19 100755 --- a/bootstrap/run.sh +++ b/bootstrap/run.sh @@ -28,4 +28,8 @@ if [ "${1:-}" = "up" ]; then echo "run.sh: captured Vault unseal keys + root token into encrypted config" fi fi + # Publish the fresh stack export to RustFS so CI (pulumi-preview, backup-verify) + # has Pulumi state — bootstrap/state/ is gitignored (T14). Best-effort: the `up` + # already succeeded, so a publish hiccup must not fail the deploy. + "$DIR/state-publish.sh" || echo "run.sh: WARN state-publish failed (CI state not refreshed)" fi diff --git a/bootstrap/state-publish.sh b/bootstrap/state-publish.sh new file mode 100755 index 0000000..043e7a9 --- /dev/null +++ b/bootstrap/state-publish.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# state-publish.sh — publish the latest Pulumi stack export to RustFS so CI has +# stack state (T14). `bootstrap/state/` is gitignored, so a CI checkout has NO +# Pulumi deployment to `preview` against; this pushes a fresh `pulumi stack export` +# to a dedicated RustFS object after every `up` (invoked by run.sh; also runnable +# standalone to re-publish without a deploy). +# +# WHAT TRAVELS: only the resource DEPLOYMENT (stack export). Config + secrets stay +# in the committed Pulumi.foundation.yaml (passphrase-encrypted) that CI gets from +# the git checkout; secrets inside the export itself are likewise passphrase- +# encrypted (`secure:` ciphertext), so the object carries NO plaintext secret. +# +# WHERE: rfs/foundation-ci-state/foundation-stack.json (internal RustFS; the bucket +# is declared in components/rustfs.ts BUCKET_SETUP and created here belt-and-suspenders). +# The push runs ON the VM via a throwaway `mc` container on foundation-net (ADR-007), +# exactly like backup.sh — RustFS 9000 is NOT published off-host. RustFS root creds +# are read on the VM from the running container and never transit the wire. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +DIR="$ROOT/bootstrap" +export PULUMI_BACKEND_URL="file://${DIR}/state" +export PULUMI_CONFIG_PASSPHRASE="${PULUMI_CONFIG_PASSPHRASE:-$(pass olsitec-foundation/PULUMI_CONFIG_PASSPHRASE)}" +KEY="${SSH_PRIVATE_KEY_PATH:-${HOME}/.ssh/foundation-test_ed25519}" +MC_IMAGE="$(grep '^IMAGE_MC=' "$ROOT/VERSIONS" | cut -d= -f2-)" +BUCKET=foundation-ci-state +OBJECT=foundation-stack.json +cd "$DIR" +pulumi stack select foundation >/dev/null + +HOST=$(pulumi config get foundation:vm.host) +PORT=$(pulumi config get foundation:vm.sshPort) +SUSER=$(pulumi config get foundation:vm.user) +SSHX="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=15 -i $KEY -p $PORT $SUSER@$HOST" + +echo "state-publish: exporting stack -> rfs/$BUCKET/$OBJECT" +pulumi stack export | $SSHX "cat > /tmp/ci-stack.json" +# Push from the VM through a throwaway mc container (RAK/RSK read on the VM, not sent). +$SSHX "MC_IMAGE='$MC_IMAGE' BUCKET='$BUCKET' OBJECT='$OBJECT' sh -s" <<'REMOTE' +set -eu +RAK=$(docker inspect foundation-rustfs --format '{{range .Config.Env}}{{println .}}{{end}}' | sed -n 's/^RUSTFS_ACCESS_KEY=//p') +RSK=$(docker inspect foundation-rustfs --format '{{range .Config.Env}}{{println .}}{{end}}' | sed -n 's/^RUSTFS_SECRET_KEY=//p') +docker run --rm --network foundation-net --entrypoint sh -v /tmp:/w \ + -e RAK="$RAK" -e RSK="$RSK" -e BUCKET="$BUCKET" -e OBJECT="$OBJECT" "$MC_IMAGE" -c ' + set -e + mc alias set rfs http://foundation-rustfs:9000 "$RAK" "$RSK" >/dev/null + mc mb --ignore-existing "rfs/$BUCKET" >/dev/null + mc cp /w/ci-stack.json "rfs/$BUCKET/$OBJECT" >/dev/null + ' +rm -f /tmp/ci-stack.json +REMOTE +echo "state-publish: published rfs/$BUCKET/$OBJECT" diff --git a/containers/ci-image/Dockerfile b/containers/ci-image/Dockerfile index 4269dd9..00dc17a 100644 --- a/containers/ci-image/Dockerfile +++ b/containers/ci-image/Dockerfile @@ -10,7 +10,10 @@ # operation (PULUMI_CONFIG_PASSPHRASE + SSH key arrive as CI secrets/env). FROM node:20-bookworm -ARG PULUMI_VERSION=3.145.0 +# Pulumi >= 3.149 is required: the project pins `packagemanager: bun` +# (bootstrap/Pulumi.yaml) and older CLIs reject "bun" as an unknown package +# manager. Matches the operator's CLI line for preview parity (TOOL_PULUMI_MIN). +ARG PULUMI_VERSION=3.243.0 ARG VAULT_VERSION=1.18.5 ARG MC_RELEASE=RELEASE.2025-04-03T17-07-56Z ARG TARGETARCH=amd64