diff --git a/VERSIONS b/VERSIONS index cd2c4f0..6befa05 100644 --- a/VERSIONS +++ b/VERSIONS @@ -64,7 +64,7 @@ IMAGE_FORGEJO=codeberg.org/forgejo/forgejo:11@sha256:d98d860ea64fd36cb0aabf0b46b IMAGE_POSTGRES=postgres:17@sha256:5c855ad7b85e68e48a62f34662853f38b57c1c1d80f3a927ab58034fd6d31c5e IMAGE_VAULT=hashicorp/vault:1.18@sha256:750bb37c1638fa194ab37053a81618c61bb0491ddec6fccac87c07a8e6cd8166 IMAGE_RUSTFS=rustfs/rustfs:latest@sha256:fa19210ac4697c79d7ccca1ec9b0eb91aebacc6691991ffb14014bb3c67e6cc3 -IMAGE_ACT_RUNNER=code.forgejo.org/forgejo/runner:6@sha256:e8dd2880f2fc81984d2308b93f1bc064dfb41187942300676536c09a3b30043d +IMAGE_ACT_RUNNER=code.forgejo.org/forgejo/runner:6@sha256:PIN_DIGEST IMAGE_REGISTRY=registry:2@sha256:PIN_DIGEST # Tool image: MinIO client `mc` — used transiently (never a long-running service) diff --git a/backup/backup-remote.sh b/backup/backup-remote.sh deleted file mode 100755 index a32ce3b..0000000 --- a/backup/backup-remote.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/sh -# backup-remote.sh — the VM-side bundle assembler (CONTRACT_004 producer half). -# Shipped + run by backup/backup.sh; NOT run directly. Secrets arrive on stdin -# (never argv); non-secrets ($TS, $MC_IMAGE, $AGE_RECIPIENT) are args. pulumi-state.json -# is already in $W (the operator placed it there before invoking this). -# -# Produces foundation-backups// in RustFS and replicates it to the offsite -# bucket. Artifacts per CONTRACT_004 §4.2: postgres.sql.gz, forgejo-repos.tar.zst, -# vault-raft.snap, pulumi-state.json, rustfs-blobs.tar.zst, MANIFEST.json. -# -# At-rest encryption (CONTRACT_004 §4.3): every DATA artifact is age-encrypted to -# $AGE_RECIPIENT before upload (`` -> `.age`); only MANIFEST.json travels -# in cleartext (it carries no secrets — it is the inventory + integrity gate, and -# lists each artifact's PLAINTEXT sha256 so restore verifies after decryption). The -# matching identity is in Vault + passphrase-encrypted config (CONTRACT_004 §4.3), -# so {repo + passphrase} can always decrypt even after total Vault loss. -set -eu -IFS= read -r VAULT_TOKEN -IFS= read -r OFF_EP -IFS= read -r OFF_AK -IFS= read -r OFF_SK -IFS= read -r BUCKET -TS="$1" -MC_IMAGE="$2" -AGE_RECIPIENT="$3" -OFFSITE_BUCKET=olsitec-foundation -W="/tmp/foundation-backup-$TS" -mkdir -p "$W" - -echo "[backup] postgres pg_dumpall" >&2 -docker exec foundation-postgres pg_dumpall -U postgres | gzip > "$W/postgres.sql.gz" - -echo "[backup] forgejo git repos (tar.zst)" >&2 -# Forgejo keeps repos under /data/git; use the container's own tar (no extra image). -docker exec foundation-forgejo sh -c 'tar -C /data -cf - git' | zstd -q -T0 > "$W/forgejo-repos.tar.zst" - -echo "[backup] vault raft snapshot" >&2 -docker exec -e VAULT_ADDR=http://127.0.0.1:8200 -e VAULT_TOKEN="$VAULT_TOKEN" foundation-vault \ - sh -c 'vault operator raft snapshot save /tmp/v.snap >/dev/null 2>&1 && cat /tmp/v.snap && rm -f /tmp/v.snap' > "$W/vault-raft.snap" - -# RustFS root creds from the running container (VM-trusted). -RAK=$(docker inspect foundation-rustfs --format '{{range .Config.Env}}{{println .}}{{end}}' | sed -n 's/^RUSTFS_ACCESS_KEY=//p') -RSK=$(docker inspect foundation-rustfs --format '{{range .Config.Env}}{{println .}}{{end}}' | sed -n 's/^RUSTFS_SECRET_KEY=//p') - -echo "[backup] rustfs blobs -> rustfs-blobs.tar.zst" >&2 -# Pull the blob buckets onto the VM fs so the bundle is a single encrypted unit -# (CONTRACT_004 §4.3 "whole bundle"). Tiny at Layer 0; may be made incremental later. -mkdir -p "$W/blobs/forgejo-packages" "$W/blobs/forgejo-artifacts" "$W/blobs/forgejo-lfs" -docker run --rm --network foundation-net --entrypoint sh -v "$W":/w \ - -e RAK="$RAK" -e RSK="$RSK" "$MC_IMAGE" -c ' - set -e - mc alias set rfs http://foundation-rustfs:9000 "$RAK" "$RSK" >/dev/null - for b in forgejo-packages forgejo-artifacts forgejo-lfs; do - mc mirror --overwrite --quiet "rfs/$b" "/w/blobs/$b" >/dev/null 2>&1 || true - done' -tar -C "$W/blobs" -cf - . | zstd -q -T0 > "$W/rustfs-blobs.tar.zst" -rm -rf "$W/blobs" - -echo "[backup] MANIFEST.json" >&2 -( cd "$W" - jq -n --arg ts "$TS" --arg rcpt "$AGE_RECIPIENT" \ - --argjson files "$(for f in postgres.sql.gz forgejo-repos.tar.zst vault-raft.snap pulumi-state.json rustfs-blobs.tar.zst; do - [ -f "$f" ] || continue - jq -n --arg n "$f" --arg sha "$(sha256sum "$f" | cut -d' ' -f1)" --argjson sz "$(stat -c %s "$f")" \ - '{name:$n, sha256:$sha, size:$sz}' - done | jq -s '.')" \ - '{timestamp:$ts, encryption:"age", ageRecipient:$rcpt, restoreOrder:["vault","postgres","rustfs","forgejo"], artifacts:$files}' > MANIFEST.json -) - -echo "[backup] age-encrypt artifacts (-> *.age)" >&2 -for f in postgres.sql.gz forgejo-repos.tar.zst vault-raft.snap pulumi-state.json rustfs-blobs.tar.zst; do - [ -f "$W/$f" ] || continue - age -r "$AGE_RECIPIENT" -o "$W/$f.age" "$W/$f" - rm -f "$W/$f" -done - -echo "[backup] upload to RustFS $BUCKET/$TS + replicate offsite" >&2 -docker run --rm --network foundation-net --entrypoint sh -v "$W":/w \ - -e RAK="$RAK" -e RSK="$RSK" -e OFF_EP="$OFF_EP" -e OFF_AK="$OFF_AK" -e OFF_SK="$OFF_SK" \ - -e BUCKET="$BUCKET" -e TS="$TS" -e OFFB="$OFFSITE_BUCKET" \ - "$MC_IMAGE" -c ' - set -e - mc alias set rfs http://foundation-rustfs:9000 "$RAK" "$RSK" >/dev/null - mc alias set off "$OFF_EP" "$OFF_AK" "$OFF_SK" >/dev/null - mc cp -r /w/ "rfs/$BUCKET/$TS/" >/dev/null - mc mirror --overwrite --quiet "rfs/$BUCKET/$TS" "off/$OFFB/$TS" >/dev/null - ' -rm -rf "$W" -echo "[backup] complete: rfs/$BUCKET/$TS (+ offsite $OFFSITE_BUCKET/$TS), age-encrypted" >&2 diff --git a/backup/backup.sh b/backup/backup.sh deleted file mode 100755 index 6a99317..0000000 --- a/backup/backup.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# backup.sh — CONTRACT_004 backup producer (operator orchestrator). -# -# ./backup/backup.sh [UTC-timestamp] -# -# The timestamp is supplied by the caller (CI/cron) per CONTRACT_004 §4.1; it -# defaults to now for manual runs. The operator contributes the Pulumi state -# (local file backend) and the secrets (from passphrase-encrypted config); the -# heavy lifting runs on the VM via backup-remote.sh. Result: a bundle in RustFS -# foundation-backups// replicated to the offsite bucket. -set -euo pipefail -ROOT="$(cd "$(dirname "$0")/.." && pwd)" -DIR="$ROOT/bootstrap" -TS="${1:-$(date -u +%Y%m%dT%H%M%SZ)}" -export PULUMI_BACKEND_URL="file://${DIR}/state" -export PULUMI_CONFIG_PASSPHRASE="$(pass olsitec-foundation/PULUMI_CONFIG_PASSPHRASE)" -KEY="${SSH_PRIVATE_KEY_PATH:-${HOME}/.ssh/foundation-test_ed25519}" -MC_IMAGE="$(grep '^IMAGE_MC=' "$ROOT/VERSIONS" | cut -d= -f2-)" -cd "$DIR" -pulumi stack select foundation >/dev/null - -RT=$(pulumi config get vaultCredentials:rootToken) -OFF_EP=$(pulumi config get foundation:backup.offsiteEndpoint) -OFF_AK=$(pulumi config get foundation:backup.offsiteAccessKey) -OFF_SK=$(pulumi config get foundation:backup.offsiteSecretKey) -BUCKET=$(pulumi config get foundation:backup.bucket) -AGE_RECIPIENT=$(pulumi config get foundation:backup.ageRecipient) # public; CONTRACT_004 §4.3 -HOST=$(pulumi config get foundation:vm.host) -PORT=$(pulumi config get foundation:vm.sshPort) -SUSER=$(pulumi config get foundation:vm.user) -SSHX="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=15 -i $KEY -p $PORT $SUSER@$HOST" -W="/tmp/foundation-backup-$TS" - -echo "backup: $TS -> rfs/$BUCKET/$TS (+ offsite)" -# Pulumi state + the assembler script onto the VM. -pulumi stack export | $SSHX "mkdir -p $W && cat > $W/pulumi-state.json" -$SSHX "cat > /tmp/backup-remote-$TS.sh" < "$ROOT/backup/backup-remote.sh" -# Run the assembler: secrets on stdin (never argv); TS, MC_IMAGE, age recipient as args. -printf '%s\n%s\n%s\n%s\n%s\n' "$RT" "$OFF_EP" "$OFF_AK" "$OFF_SK" "$BUCKET" \ - | $SSHX "sh /tmp/backup-remote-$TS.sh '$TS' '$MC_IMAGE' '$AGE_RECIPIENT'; rm -f /tmp/backup-remote-$TS.sh" -echo "backup: done ($TS)" diff --git a/backup/restore-remote.sh b/backup/restore-remote.sh deleted file mode 100755 index 68b507c..0000000 --- a/backup/restore-remote.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/sh -# restore-remote.sh — VM-side SCRATCH restore verifier (CONTRACT_004 consumer half, -# §4.6 "a backup is not trusted until restored"). Shipped + run by backup/restore.sh. -# NON-DESTRUCTIVE: it reconstructs into throwaway scratch resources and asserts the -# bundle is restorable — it never touches the live containers/volumes. A real -# disaster restore (overwriting live, restore order Vault->Postgres->RustFS->Forgejo) -# is dr/restore-to-fresh-vm.sh (T13), out of scope here. -# -# Secrets on stdin (OFF_* offsite creds + the age IDENTITY); non-secrets ($TS, -# $MC_IMAGE, $PG_IMAGE, $SRC) as args. The bundle is age-encrypted (CONTRACT_004 -# §4.3): every artifact is pulled as .age and decrypted with the identity -# BEFORE its MANIFEST sha256 (a PLAINTEXT sha) is verified. -set -eu -IFS= read -r OFF_EP -IFS= read -r OFF_AK -IFS= read -r OFF_SK -IFS= read -r BUCKET -IFS= read -r AGE_IDENTITY -TS="$1"; MC_IMAGE="$2"; PG_IMAGE="$3"; SRC="${4:-rfs}" -OFFSITE_BUCKET=olsitec-foundation -W="/tmp/foundation-restore-$TS" -rm -rf "$W"; (umask 077; mkdir -p "$W") -fail() { echo "RESTORE VERIFY FAIL: $1" >&2; docker rm -f foundation-restore-pg >/dev/null 2>&1 || true; rm -f "$W/age.key" 2>/dev/null || true; exit 1; } - -# Materialise the age identity to a 0600 file for `age -d -i` (removed on exit). -( umask 077; printf '%s\n' "$AGE_IDENTITY" > "$W/age.key" ) - -RAK=$(docker inspect foundation-rustfs --format '{{range .Config.Env}}{{println .}}{{end}}' | sed -n 's/^RUSTFS_ACCESS_KEY=//p') -RSK=$(docker inspect foundation-rustfs --format '{{range .Config.Env}}{{println .}}{{end}}' | sed -n 's/^RUSTFS_SECRET_KEY=//p') - -echo "[restore] pull bundle $TS from $SRC" >&2 -docker run --rm --network foundation-net --entrypoint sh -v "$W":/w \ - -e RAK="$RAK" -e RSK="$RSK" -e OFF_EP="$OFF_EP" -e OFF_AK="$OFF_AK" -e OFF_SK="$OFF_SK" \ - -e BUCKET="$BUCKET" -e TS="$TS" -e SRC="$SRC" -e OFFB="$OFFSITE_BUCKET" "$MC_IMAGE" -c ' - set -e - mc alias set rfs http://foundation-rustfs:9000 "$RAK" "$RSK" >/dev/null - if [ "$SRC" = off ]; then - mc alias set off "$OFF_EP" "$OFF_AK" "$OFF_SK" >/dev/null - mc cp -r "off/$OFFB/$TS/" /w/ >/dev/null - else - mc cp -r "rfs/$BUCKET/$TS/" /w/ >/dev/null - fi' -# mc cp -r nests under $TS/ — flatten if needed -[ -f "$W/MANIFEST.json" ] || { [ -d "$W/$TS" ] && mv "$W/$TS"/* "$W"/; } -[ -f "$W/MANIFEST.json" ] || fail "MANIFEST.json missing from pulled bundle" - -cd "$W" -echo "[restore] age-decrypt artifacts" >&2 -for name in $(jq -r '.artifacts[].name' MANIFEST.json); do - [ -f "$name.age" ] || fail "$name.age missing from bundle" - age -d -i age.key -o "$name" "$name.age" 2>/dev/null || fail "age decrypt failed: $name" -done - -echo "[restore] verify MANIFEST sha256 (plaintext)" >&2 -jq -r '.artifacts[] | "\(.sha256) \(.name)"' MANIFEST.json | while read -r sha name; do - [ -f "$name" ] || { echo "missing $name" >&2; exit 1; } - got=$(sha256sum "$name" | cut -d' ' -f1) - [ "$got" = "$sha" ] || { echo "sha mismatch $name" >&2; exit 1; } -done || fail "MANIFEST sha verification failed" - -echo "[restore] scratch Postgres restore + assert" >&2 -docker rm -f foundation-restore-pg >/dev/null 2>&1 || true -docker run -d --name foundation-restore-pg -e POSTGRES_PASSWORD=scratch "$PG_IMAGE" >/dev/null -i=0; until docker exec foundation-restore-pg pg_isready -U postgres >/dev/null 2>&1; do - i=$((i+1)); [ "$i" -gt 30 ] && fail "scratch postgres not ready"; sleep 2; done -gunzip < postgres.sql.gz | docker exec -i foundation-restore-pg psql -U postgres -q >/dev/null 2>&1 || fail "psql restore errored" -ROWS=$(docker exec foundation-restore-pg psql -U postgres -d forgejo -tAc 'SELECT count(*) FROM "user"' 2>/dev/null || echo 0) -[ "${ROWS:-0}" -ge 1 ] || fail "restored forgejo DB has no users (got '$ROWS')" -echo "[restore] postgres OK: forgejo.\"user\" rows=$ROWS" >&2 -docker rm -f foundation-restore-pg >/dev/null 2>&1 || true - -echo "[restore] extract forgejo repos + assert olsitec/foundation present" >&2 -mkdir -p repos -zstd -dc forgejo-repos.tar.zst | tar -C repos -xf - 2>/dev/null || fail "forgejo tar extract failed" -[ -d repos/git/repositories/olsitec/foundation.git ] || fail "olsitec/foundation.git not in repo bundle" -echo "[restore] forgejo repos OK: olsitec/foundation.git present" >&2 - -echo "[restore] extract rustfs blobs + assert packages present" >&2 -mkdir -p blobs -zstd -dc rustfs-blobs.tar.zst | tar -C blobs -xf - 2>/dev/null || fail "rustfs-blobs tar extract failed" -[ -d blobs/forgejo-packages ] || fail "forgejo-packages not in blob bundle" -echo "[restore] rustfs blobs OK: $(find blobs -type f | wc -l | tr -d ' ') object(s)" >&2 - -echo "[restore] vault snapshot sanity" >&2 -[ -s vault-raft.snap ] || fail "vault-raft.snap empty" -echo "[restore] vault snapshot OK: $(stat -c %s vault-raft.snap) bytes" >&2 - -rm -f "$W/age.key" -rm -rf "$W" -echo "RESTORE VERIFY PASS ($TS from $SRC, age-decrypted)" diff --git a/backup/restore.sh b/backup/restore.sh deleted file mode 100755 index 6206690..0000000 --- a/backup/restore.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# restore.sh — CONTRACT_004 §4.6 restore verifier (operator orchestrator). -# -# ./backup/restore.sh [rfs|off] -# -# Pulls the bundle (default from RustFS; `off` checks the offsite copy) and asserts -# it reconstructs into scratch resources — NON-DESTRUCTIVE, it never touches the -# live platform. The real disaster restore is dr/restore-to-fresh-vm.sh (T13). -set -euo pipefail -ROOT="$(cd "$(dirname "$0")/.." && pwd)" -DIR="$ROOT/bootstrap" -TS="${1:?usage: restore.sh [rfs|off]}" -SRC="${2:-rfs}" -export PULUMI_BACKEND_URL="file://${DIR}/state" -export PULUMI_CONFIG_PASSPHRASE="$(pass olsitec-foundation/PULUMI_CONFIG_PASSPHRASE)" -KEY="${SSH_PRIVATE_KEY_PATH:-${HOME}/.ssh/foundation-test_ed25519}" -MC_IMAGE="$(grep '^IMAGE_MC=' "$ROOT/VERSIONS" | cut -d= -f2-)" -PG_IMAGE="$(grep '^IMAGE_POSTGRES=' "$ROOT/VERSIONS" | cut -d= -f2-)" -cd "$DIR" -pulumi stack select foundation >/dev/null - -OFF_EP=$(pulumi config get foundation:backup.offsiteEndpoint) -OFF_AK=$(pulumi config get foundation:backup.offsiteAccessKey) -OFF_SK=$(pulumi config get foundation:backup.offsiteSecretKey) -BUCKET=$(pulumi config get foundation:backup.bucket) -AGE_IDENTITY=$(pulumi config get foundation:backup.ageIdentity) # secret; CONTRACT_004 §4.3 -HOST=$(pulumi config get foundation:vm.host) -PORT=$(pulumi config get foundation:vm.sshPort) -SUSER=$(pulumi config get foundation:vm.user) -SSHX="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=15 -i $KEY -p $PORT $SUSER@$HOST" - -$SSHX "cat > /tmp/restore-remote-$TS.sh" < "$ROOT/backup/restore-remote.sh" -printf '%s\n%s\n%s\n%s\n%s\n' "$OFF_EP" "$OFF_AK" "$OFF_SK" "$BUCKET" "$AGE_IDENTITY" \ - | $SSHX "sh /tmp/restore-remote-$TS.sh '$TS' '$MC_IMAGE' '$PG_IMAGE' '$SRC'; rm -f /tmp/restore-remote-$TS.sh" diff --git a/bootstrap/Pulumi.foundation.yaml b/bootstrap/Pulumi.foundation.yaml index e1508e1..5dba134 100644 --- a/bootstrap/Pulumi.foundation.yaml +++ b/bootstrap/Pulumi.foundation.yaml @@ -63,7 +63,4 @@ config: secure: v1:9YpTkFoQanMwxAQV:dJ4YmXS0aOTHPbuK1H6AJ0SAJ0CjYX0iIyLOQAUNfsOWLsSy5TXxPpGecieBWkzc4AALDkJNlQN9Xo6Q0ZcaSg== vaultCredentials:rootToken: secure: v1:OUpYMjnaftxMUKjv:2m+dydQopXGRleeX6ddhYSHgHP7HHZXYLAvQHXUvaA91qajoxU+VugDB/Rs= - foundation:backup.ageRecipient: age1x6dmgtt2eahpvyzkmy6j80rts28chw2lcam0rcxq3nhc8ld649sslzpsy4 - foundation:backup.ageIdentity: - secure: v1:VCFVXswrmMrXyFbr:p4pfG/Kp2lreetYX4O86rZqpU1xQugRycF+PBBiNGZnaD0c15R+mJuLNrl0rBXY5vJwyZTbNSpFY1zPQ7TwuQcVp9h8oiGcgVEobsbb4BBp3lFhsObllgYM9 encryptionsalt: v1:5YhUt8BVfH0=:v1:DPCHl+7zwn4RaMPj:A19tZzBlZ1NmDtTWrHreEKk5e8idyw== diff --git a/bootstrap/components/credentials.ts b/bootstrap/components/credentials.ts index dce3796..e74b3f6 100644 --- a/bootstrap/components/credentials.ts +++ b/bootstrap/components/credentials.ts @@ -15,7 +15,6 @@ import * as pulumi from "@pulumi/pulumi"; import * as command from "@pulumi/command"; import { RandomPassword } from "@pulumi/random"; -import { BackupSecrets } from "../config"; import { DeployCtx } from "../lib/context"; import { vmConnection } from "../lib/remote"; import { VaultOutputs } from "./vault"; @@ -37,18 +36,14 @@ export interface RustfsCredentials { } /** - * `foundation/forgejo/service-credentials` — admin slice + SECRET_KEY (CONTRACT_002 - * §2.3). The OTHER crypto secrets (forgejoInternalToken/Jwt*) are format-constrained - * (JWTs, not free random) so Forgejo auto-generates them into app.ini and the GATE-B - * mirror captures them. SECRET_KEY, by contrast, is a plain high-entropy string, so - * we generate it here (@pulumi/random, per CONTRACT_002) and inject it via env — - * Forgejo leaves it EMPTY when the web installer is skipped (INSTALL_LOCK), which - * would weaken at-rest encryption of 2FA/mirror/oauth secrets. + * `foundation/forgejo/service-credentials` — the admin slice (CONTRACT_002 §2.3). + * The crypto secrets (forgejoSecretKey/InternalToken/Jwt*) are auto-generated by + * Forgejo into its app.ini (format-constrained — JWTs, not free random), so they + * are not generated here; capturing them into Vault is a later refinement. */ export interface ForgejoCredentials { adminUser: string; // cfg.forgejo.adminUser (deterministic) adminPassword: pulumi.Output; - secretKey: pulumi.Output; // [security] SECRET_KEY (injected via env) } /** Everything generateCredentials() produces; grows as Wave-2 tasks land. */ @@ -84,7 +79,6 @@ export function generateCredentials(ctx: DeployCtx): FoundationCredentials { forgejo: { adminUser: ctx.cfg.forgejo.adminUser, // "platform-admin" adminPassword: secret("forgejo-admin-password"), - secretKey: secret("forgejo-secret-key", 40), // [security] SECRET_KEY }, }; } @@ -104,6 +98,7 @@ IFS= read -r PG_FORGEJO_PW IFS= read -r RUSTFS_ADMIN_PW IFS= read -r RUSTFS_SVC_ID IFS= read -r RUSTFS_SVC_SECRET +IFS= read -r FORGEJO_ADMIN_PW C=foundation-vault VE="-e VAULT_ADDR=http://127.0.0.1:8200 -e VAULT_TOKEN=$ROOT_TOKEN" @@ -121,12 +116,11 @@ jq -n --arg u "$RUSTFS_ADMIN_USER" --arg p "$RUSTFS_ADMIN_PW" --arg ki "$RUSTFS_ '{rustfsAdminUser:$u,rustfsAdminPassword:$p,rustfsServiceKeyId:$ki,rustfsServiceKeySecret:$ks}' \ | put rustfs/service-credentials -echo "vault: wrote postgres + rustfs service-credentials"`; -// NOTE: foundation/forgejo/service-credentials is NOT written here. Its crypto -// secrets (SECRET_KEY/INTERNAL_TOKEN/JWT) only exist after Forgejo first-starts -// and writes app.ini, so the whole forgejo path (admin + crypto) is single-owned -// by writeForgejoCredentialsToVault at GATE B — keeping one writer per Vault path -// avoids a put/patch race on re-runs (CONTRACT_002 "single source of truth"). +jq -n --arg u "$FORGEJO_ADMIN_USER" --arg p "$FORGEJO_ADMIN_PW" \ + '{forgejoAdminUser:$u,forgejoAdminPassword:$p}' \ + | put forgejo/service-credentials + +echo "vault: wrote postgres + rustfs + forgejo service-credentials"`; /** * T06 — distribute the generated data-plane credentials into Vault (CONTRACT_002). @@ -137,11 +131,12 @@ export function writeCredentialsToVault( creds: FoundationCredentials, vault: VaultOutputs, ): command.remote.Command { - // Non-secret usernames are prepended as shell vars; the 5 secret values (root + // Non-secret usernames are prepended as shell vars; the 6 secret values (root // token first, then the order the script `read`s them) arrive on stdin. const create = pulumi.interpolate`PG_SUPER_USER='${creds.postgres.superUser}' PG_FORGEJO_USER='${creds.postgres.forgejoDbUser}' RUSTFS_ADMIN_USER='${creds.rustfs.adminUser}' +FORGEJO_ADMIN_USER='${creds.forgejo.adminUser}' ${WRITE_CREDS}`; return new command.remote.Command( @@ -156,6 +151,7 @@ ${creds.postgres.forgejoDbPassword} ${creds.rustfs.adminPassword} ${creds.rustfs.serviceKeyId} ${creds.rustfs.serviceKeySecret} +${creds.forgejo.adminPassword} `, addPreviousOutputInEnv: false, triggers: [ @@ -165,129 +161,9 @@ ${creds.rustfs.serviceKeySecret} creds.rustfs.adminPassword, creds.rustfs.serviceKeyId, creds.rustfs.serviceKeySecret, + creds.forgejo.adminPassword, ], }, { dependsOn: [vault.init] }, ); } - -// Mirrors the config-seeded backup credentials (offsite S3 creds + the age key) -// into Vault at foundation/backup/backup-credentials (CONTRACT_002 §2.3). Unlike -// the generated data-plane creds these are seeded once into passphrase-encrypted -// config (the age IDENTITY MUST also live there so {repo + passphrase} can decrypt -// a bundle after total Vault loss — CONTRACT_004 §4.3); this writer makes them -// available to in-Vault consumers (Layer-1 ESO, the backup-verify job). Secret -// values on stdin (ADR-007 D2); non-secrets (endpoint, recipient) as shell vars. -const WRITE_BACKUP_CREDS = `set -eu -IFS= read -r ROOT_TOKEN -IFS= read -r OFF_AK -IFS= read -r OFF_SK -IFS= read -r AGE_IDENTITY -C=foundation-vault -VE="-e VAULT_ADDR=http://127.0.0.1:8200 -e VAULT_TOKEN=$ROOT_TOKEN" - -if ! docker exec $VE "$C" vault secrets list -format=json 2>/dev/null | jq -e 'has("foundation/")' >/dev/null; then - docker exec $VE "$C" vault secrets enable -path=foundation kv-v2 >/dev/null -fi - -jq -n --arg ep "$OFF_EP" --arg ak "$OFF_AK" --arg sk "$OFF_SK" --arg ar "$AGE_RECIPIENT" --arg ai "$AGE_IDENTITY" \ - '{offsiteEndpoint:$ep,offsiteAccessKey:$ak,offsiteSecretKey:$sk,backupAgeRecipient:$ar,backupAgeIdentity:$ai}' \ - | docker exec -i $VE "$C" vault kv put foundation/backup/backup-credentials - >/dev/null - -echo "vault: wrote backup/backup-credentials (offsite + age key)"`; - -/** - * Mirror the backup credentials (incl. the age key) into Vault (CONTRACT_002 §2.3). - * Depends on Vault being unsealed (GATE A) via vault.init — same pattern as the - * data-plane creds writer above. - */ -export function writeBackupCredentialsToVault( - ctx: DeployCtx, - vault: VaultOutputs, - backup: BackupSecrets, -): command.remote.Command { - const create = pulumi.interpolate`OFF_EP='${backup.offsiteEndpoint}' -AGE_RECIPIENT='${backup.ageRecipient}' -${WRITE_BACKUP_CREDS}`; - - return new command.remote.Command( - "foundation-backup-credentials", - { - connection: vmConnection(ctx), - create, - update: create, - stdin: pulumi.interpolate`${vault.rootToken} -${backup.offsiteAccessKey} -${backup.offsiteSecretKey} -${backup.ageIdentity} -`, - addPreviousOutputInEnv: false, - triggers: [ - vault.init.id, - backup.offsiteAccessKey, - backup.offsiteSecretKey, - backup.ageIdentity, - ], - }, - { dependsOn: [vault.init] }, - ); -} - -// Single owner of foundation/forgejo/service-credentials (CONTRACT_002 §2.3). The -// admin user/pw are generated (@pulumi/random); the crypto secrets (SECRET_KEY, -// INTERNAL_TOKEN, the LFS + OAuth2 JWT secrets) are auto-generated by Forgejo into -// app.ini on first start — they only exist post-boot, so the whole path is written -// here at GATE B (dependsOn forgejo.ready), read straight off the live app.ini via -// docker-exec (ADR-007). Admin pw on stdin (D2); crypto values are read on the VM -// and never transit the Pulumi command string. Idempotent put. -// -// NOTE: SECRET_KEY can be EMPTY when the bootstrap skips the web installer -// (INSTALL_LOCK) — it is mirrored as-is (faithful), and flagged for hardening. -const WRITE_FORGEJO_CREDS = `set -eu -IFS= read -r ROOT_TOKEN -IFS= read -r ADMIN_PW -C=foundation-vault -F=foundation-forgejo -VE="-e VAULT_ADDR=http://127.0.0.1:8200 -e VAULT_TOKEN=$ROOT_TOKEN" -gv() { docker exec "$F" sh -c "sed -n 's/^$1 *= *//p' /data/gitea/conf/app.ini" | head -1; } -SECRET_KEY=$(gv SECRET_KEY) -INTERNAL_TOKEN=$(gv INTERNAL_TOKEN) -LFS_JWT=$(gv LFS_JWT_SECRET) -OAUTH2_JWT=$(gv JWT_SECRET) -jq -n --arg au "$ADMIN_USER" --arg ap "$ADMIN_PW" \ - --arg sk "$SECRET_KEY" --arg it "$INTERNAL_TOKEN" --arg jt "$LFS_JWT" --arg oj "$OAUTH2_JWT" \ - '{forgejoAdminUser:$au,forgejoAdminPassword:$ap,forgejoSecretKey:$sk,forgejoInternalToken:$it,forgejoJwtSecret:$jt,forgejoOauth2JwtSecret:$oj}' \ - | docker exec -i $VE "$C" vault kv put foundation/forgejo/service-credentials - >/dev/null -echo "vault: wrote forgejo/service-credentials (admin + crypto secrets)"`; - -/** - * Mirror the Forgejo admin + crypto secrets into Vault (CONTRACT_002 §2.3). - * Runs at GATE B: needs Vault unsealed (vault.init) AND Forgejo healthy - * (forgejoReady) so app.ini exists to read the crypto secrets from. - */ -export function writeForgejoCredentialsToVault( - ctx: DeployCtx, - vault: VaultOutputs, - forgejoCreds: ForgejoCredentials, - forgejoReady: command.remote.Command, -): command.remote.Command { - const create = pulumi.interpolate`ADMIN_USER='${forgejoCreds.adminUser}' -${WRITE_FORGEJO_CREDS}`; - - return new command.remote.Command( - "foundation-forgejo-credentials", - { - connection: vmConnection(ctx), - create, - update: create, - stdin: pulumi.interpolate`${vault.rootToken} -${forgejoCreds.adminPassword} -`, - addPreviousOutputInEnv: false, - // forgejoReady.id changes when the container is (re)created → app.ini (hence - // the crypto secrets) regenerated → re-mirror. - triggers: [vault.init.id, forgejoReady.id, forgejoCreds.adminPassword], - }, - { dependsOn: [vault.init, forgejoReady] }, - ); -} diff --git a/bootstrap/components/forgejo.ts b/bootstrap/components/forgejo.ts index 1b9d761..9711fcb 100644 --- a/bootstrap/components/forgejo.ts +++ b/bootstrap/components/forgejo.ts @@ -26,7 +26,6 @@ export interface ForgejoDeps { rustfs: RustfsOutputs; pgCreds: PostgresCredentials; rustfsCreds: RustfsCredentials; - forgejoCreds: ForgejoCredentials; } export interface ForgejoOutputs { @@ -89,11 +88,6 @@ export function deployForgejo( // Go SSH server colliding on :22. SSH_PORT is the clone-URL port; the sshd is // published on host :22 (scp-form goal) + :2222 (CONTRACT_003). "FORGEJO__server__START_SSH_SERVER=false", - // [security] SECRET_KEY — Forgejo leaves this EMPTY when the installer is - // skipped (INSTALL_LOCK); set it explicitly so at-rest encryption of 2FA / - // mirror / oauth secrets is keyed. env-to-ini overwrites it in the volume's - // app.ini (INTERNAL_TOKEN + JWT secrets are left untouched — Forgejo's own). - pulumi.interpolate`FORGEJO__security__SECRET_KEY=${deps.forgejoCreds.secretKey}`, "FORGEJO__server__SSH_LISTEN_PORT=22", `FORGEJO__server__SSH_PORT=${cfg.forgeSshPort}`, `FORGEJO__server__SSH_DOMAIN=${cfg.hosts.git}`, diff --git a/bootstrap/components/network.ts b/bootstrap/components/network.ts index e99b188..e9a7990 100644 --- a/bootstrap/components/network.ts +++ b/bootstrap/components/network.ts @@ -16,17 +16,6 @@ export function deployNetwork(ctx: BaseCtx): docker.Network { attachable: true, ipamConfigs: [{ subnet: ctx.cfg.network.subnet }], // "172.30.0.0/24" }, - { - provider: ctx.provider, - deleteBeforeReplace: true, - // Docker auto-assigns the subnet's first host (.1) as the bridge gateway — - // a field we never declared, so a `pulumi up --refresh` surfaced it as a - // spurious ipamConfigs drift. `gateway` is ForceNew, so reconciling it - // (either by declaring it OR by applying the refreshed diff) would REPLACE - // the network and disconnect every container. The IPAM is immutable by - // design (subnet fixed by CONTRACT_003), so we ignore drift on it: plain - // `up` stays clean AND `up --refresh` no longer wants to recreate the net. - ignoreChanges: ["ipamConfigs"], - }, + { provider: ctx.provider, deleteBeforeReplace: true }, ); } diff --git a/bootstrap/components/runner.ts b/bootstrap/components/runner.ts deleted file mode 100644 index f4a076d..0000000 --- a/bootstrap/components/runner.ts +++ /dev/null @@ -1,103 +0,0 @@ -// components/runner.ts (T10) -// -// foundation-runner — a Forgejo Actions runner (CONTRACT_003 §3.2). Registration -// is idempotent (ADR-007): if the persistent /data/.runner config already exists it -// is reused; otherwise a fresh registration token is minted with -// `forgejo actions generate-runner-token` and consumed by `forgejo-runner register` -// (the token never leaves the VM). The daemon then polls Forgejo for jobs. -// -// SECURITY (PLAN-002 R5 / PLAN-001 §4a): this runner shares the VM and is given the -// host Docker socket so `docker`-label jobs can spawn containers — i.e. it is -// root-equivalent on the host and is NOT fenced from the forge's trust boundary. -// That is the documented day-zero compromise; the steady-state is a throwaway, -// separate privileged runner VM. Do not run untrusted workflows here until fenced. -import * as pulumi from "@pulumi/pulumi"; -import * as docker from "@pulumi/docker"; -import * as command from "@pulumi/command"; -import { DeployCtx } from "../lib/context"; -import { vmConnection } from "../lib/remote"; -import { ForgejoOutputs } from "./forgejo"; - -export interface RunnerOutputs { - container: docker.Container; - registered: command.remote.Command; -} - -export function deployRunner( - ctx: DeployCtx, - forgejo: ForgejoOutputs, -): RunnerOutputs { - const { cfg, provider, network } = ctx; - const img = ctx.image("ACT_RUNNER"); - const labels = cfg.runner.labels.join(","); - - const image = new docker.RemoteImage( - "foundation-runner-image", - { name: img, keepLocally: true }, - { provider }, - ); - const volume = new docker.Volume( - "foundation-runner-data", - { name: "foundation-runner-data" }, - { provider, retainOnDelete: true }, // holds .runner registration secret - ); - - const register = new command.remote.Command( - "foundation-runner-register", - { - connection: vmConnection(ctx), - create: pulumi.interpolate`set -eu -VOL=foundation-runner-data -IMG='${img}' -LABELS='${labels}' -docker volume inspect "$VOL" >/dev/null 2>&1 || docker volume create "$VOL" >/dev/null -if docker run --rm --entrypoint sh -v "$VOL":/data "$IMG" -c '[ -s /data/.runner ]'; then - echo "runner already registered" -else - TOKEN=$(docker exec -u git foundation-forgejo forgejo actions generate-runner-token) - docker run --rm --network foundation-net --entrypoint /bin/forgejo-runner -v "$VOL":/data "$IMG" \\ - register --no-interactive --instance http://foundation-forgejo:3000 --token "$TOKEN" --name foundation-runner --labels "$LABELS" >/dev/null - echo "runner registered" -fi`, - addPreviousOutputInEnv: false, - triggers: [forgejo.ready.id, labels], - }, - { dependsOn: [forgejo.ready] }, - ); - - const container = new docker.Container( - "foundation-runner", - { - name: "foundation-runner", - image: image.imageId, - hostname: "foundation-runner", - restart: "unless-stopped", - entrypoints: ["/bin/forgejo-runner"], - command: ["daemon"], - // The image runs as uid 1000; add the host docker group (gid of - // /var/run/docker.sock) so the daemon can reach the socket without running - // as root. NOTE: 996 is THIS host's docker gid — re-check on DR to a new VM - // (`stat -c %g /var/run/docker.sock`). Socket access is root-equivalent - // regardless (see the security note above). - groupAdds: ["996"], - envs: ["DOCKER_HOST=unix:///var/run/docker.sock"], - volumes: [ - { volumeName: volume.name, containerPath: "/data" }, - { - hostPath: "/var/run/docker.sock", - containerPath: "/var/run/docker.sock", - }, - ], - networksAdvanced: [{ name: network.name, aliases: ["foundation-runner"] }], - logDriver: "json-file", - logOpts: { "max-size": "10m", "max-file": "3" }, - }, - { - provider, - dependsOn: [network, register], - deleteBeforeReplace: true, - }, - ); - - return { container, registered: register }; -} diff --git a/bootstrap/config.ts b/bootstrap/config.ts index d737a00..762c19d 100644 --- a/bootstrap/config.ts +++ b/bootstrap/config.ts @@ -88,33 +88,6 @@ export interface FoundationConfig { }; } -/** - * The backup secret slice (CONTRACT_001 §1.3 + CONTRACT_002 `foundation/backup/ - * backup-credentials`). Kept OUT of the non-secret FoundationConfig surface: these - * are `secure:`-encrypted (offsite creds, age identity) or a public key that only - * the Vault mirror + backup scripts consume. `loadBackupSecrets()` is the single - * reader (composition point), so components still never touch raw pulumi.Config. - */ -export interface BackupSecrets { - offsiteEndpoint: string; // non-secret (also in FoundationConfig.backup) - offsiteAccessKey: pulumi.Output; - offsiteSecretKey: pulumi.Output; - ageRecipient: string; // age1… public key (non-secret) - ageIdentity: pulumi.Output; // AGE-SECRET-KEY-… (secret; survives Vault loss via config) -} - -/** Reads the backup secret slice for the Vault mirror (CONTRACT_002 §2.3). */ -export function loadBackupSecrets(): BackupSecrets { - const c = new pulumi.Config("foundation"); - return { - offsiteEndpoint: c.require("backup.offsiteEndpoint"), - offsiteAccessKey: c.requireSecret("backup.offsiteAccessKey"), - offsiteSecretKey: c.requireSecret("backup.offsiteSecretKey"), - ageRecipient: c.require("backup.ageRecipient"), - ageIdentity: c.requireSecret("backup.ageIdentity"), - }; -} - /** * The SSH private key path is supplied by ENV (CONTRACT_001 §1: `SSH_PRIVATE_KEY_PATH`, * default ~/.ssh/id_rsa) — NEVER by Pulumi config. Exposed separately from diff --git a/bootstrap/index.ts b/bootstrap/index.ts index 9978788..5cf7982 100644 --- a/bootstrap/index.ts +++ b/bootstrap/index.ts @@ -6,22 +6,19 @@ // fill the marked slots — this file is the single composition point; components stay // pure factories in components/*. import * as pulumi from "@pulumi/pulumi"; -import { loadConfig, loadBackupSecrets } from "./config"; +import { loadConfig } from "./config"; import { buildBaseContext, DeployCtx } from "./lib/context"; import { deployNetwork } from "./components/network"; import { deployDns } from "./components/dns"; import { generateCredentials, writeCredentialsToVault, - writeBackupCredentialsToVault, - writeForgejoCredentialsToVault, } from "./components/credentials"; import { deployPostgres } from "./components/postgres"; import { deployRustfs } from "./components/rustfs"; import { deployVault } from "./components/vault"; import { deployProxy } from "./components/proxy"; import { deployForgejo, bootstrapForgejo } from "./components/forgejo"; -import { deployRunner } from "./components/runner"; import * as fs from "fs"; const cfg = loadConfig(); @@ -49,8 +46,6 @@ const vault = deployVault(ctx); // --- GATE A: Vault init + unseal (T05). T06 writes the generated data-plane creds // into Vault (CONTRACT_002), dependsOn vault.init so it runs only once unsealed. const vaultCreds = writeCredentialsToVault(ctx, credentials, vault); -// Mirror the config-seeded backup creds + age key into Vault (CONTRACT_002 §2.3). -const backupCreds = writeBackupCredentialsToVault(ctx, vault, loadBackupSecrets()); // ============================================================================= // PHASE 6 — FORGE (depends on: credentials, GATE A) // T07 caddy ✓ · T08 forgejo · T10 runner @@ -61,7 +56,6 @@ const forgejo = deployForgejo(ctx, { rustfs, pgCreds: credentials.postgres, rustfsCreds: credentials.rustfs, - forgejoCreds: credentials.forgejo, }); // --- GATE B: Forgejo healthy → headless admin + org + repo + operator SSH key @@ -76,29 +70,18 @@ const forgejoBootstrap = bootstrapForgejo(ctx, { repoName: "foundation", sshPublicKey, }); -const runner = cfg.features.runner ? deployRunner(ctx, forgejo) : undefined; -// Mirror Forgejo's admin + app.ini crypto secrets into Vault (CONTRACT_002 §2.3); -// GATE B — needs app.ini, which exists only once Forgejo has started. -const forgejoCreds = writeForgejoCredentialsToVault( - ctx, - vault, - credentials.forgejo, - forgejo.ready, -); +// const runner = deployRunner(ctx, { forgejo, credentials }); // ============================================================================= // Stack outputs (extended as phases land). // vaultCreds (T06) is a gate for Forgejo (T08) — it has no output to export yet. void vaultCreds; -void backupCreds; // CONTRACT_002 backup/backup-credentials mirror; no secret output -void forgejoCreds; // CONTRACT_002 forgejo/service-credentials mirror; no secret output -export const phase = "T10-runner"; // forge + CI runner live +export const phase = "T09-forge-bootstrap"; // admin + org + repo + operator key export const caddyImageId = proxy.imageId; export const forgejoEndpoint = forgejo.endpoint; export const cloneUrl = pulumi.interpolate`git@${cfg.hosts.git}:${cfg.forgejo.orgName}/foundation.git`; void forgejoBootstrap; // GATE B consumer; no secret output to export -void runner; // CI runner (feature-flagged) export const networkName = network.name; export const vmTarget = `${cfg.vm.user}@${cfg.vm.host}`; export const postgresEndpoint = postgres.endpoint; diff --git a/documentation/sessions/HANDOVER.md b/documentation/sessions/HANDOVER.md deleted file mode 100644 index 0638866..0000000 --- a/documentation/sessions/HANDOVER.md +++ /dev/null @@ -1,62 +0,0 @@ -# HANDOVER — next-session prompt (paste into a fresh context) - -> Living doc: overwritten each handover. The durable record is the dated -> `SESSION_*` files. Latest state = `SESSION_2026-06-30_002.md`. - ---- - -Continue the **olsitec-foundation** build. You are the **Lead Agent, HIGH-RISK / INFRA mode**. - -## Required reads (in `~/work/olsitec-foundation/foundation/`) -1. `documentation/sessions/SESSION_2026-06-30_002.md` ← current state + known gaps + next steps -2. `documentation/000_baseline.md` + `000_TOPOLOGY.md` -3. `documentation/contracts/CONTRACT_001–004` + `decisions/ADR_004,005,006,007` - (**ADR-007** is the control-plane mechanism the whole egg runs on — read it first) -4. `documentation/planning/PLAN-002-foundation-implementation.md` §10 - -## Where things stand -**The egg is LIVE and the goal is met.** Wave 2 (T03–T10, T12) is deployed to the Helsinki VM and -committed. `git clone git@git.olsitec.net:olsitec/foundation.git` works (:22 and :2222). Six containers -on `foundation-net`: postgres, rustfs, vault, caddy, forgejo, runner — all healthy. `https://forge.olsitec.net` -= 200 (LE DNS-01). CI green. Backups → RustFS + offsite, restore-verified from both. `cd bootstrap && -./run.sh up` is idempotent (**41 unchanged**). Working tree clean on `master`. - -## Operating essentials -- **VM**: `204.168.234.72`, admin SSH **:222**, key `~/.ssh/foundation-test_ed25519` (also the registered - Forgejo operator key). Git endpoint is :22 (scp-form) + :2222. -- **Deploy**: `cd bootstrap && ./run.sh up` (sets passphrase + key + per-process backend; captures Vault - keys to config after `up`). Master passphrase: `pass olsitec-foundation/PULUMI_CONFIG_PASSPHRASE`. -- **Vault reboot**: `bootstrap/vault-unseal.sh`. **Backup**: `backup/backup.sh [ts]`; - **restore-verify**: `backup/restore.sh [rfs|off]`. -- **Mechanism (ADR-007)**: in-VM control-plane ops = `@pulumi/command` `remote.Command` (docker-exec over - SSH); idempotent, readiness-gated, **secrets on stdin** (never inline — the provider echoes the command - on error). Images are digest-pinned in `VERSIONS`. - -## Watchouts (HIGH-RISK) -- Do **NOT** `pulumi up --refresh` blindly — it surfaces a spurious `foundation-net` ipamConfigs diff; - applying it recreates the network and disconnects every container. Plain `up` ignores it. (Investigate + - fix the drift before enabling refresh in CI.) -- Never print/commit the passphrase, Vault root token, or unseal keys (D2) — only the already-encrypted - `secure: v1:…` values in `Pulumi.foundation.yaml`. -- Don't `pulumi up` against the production `olsicloud4-*` stacks. The `provision`/`offsite-backup` stacks - use the throwaway passphrase `dev-validation-throwaway` + `HCLOUD_TOKEN`/`MINIO_BACKUP_*` from `pass`. -- Commit **atomically per task** (conventional commits; group by concern; don't `git add .`). - -## Next work — remaining PLAN-002 tasks + the known gaps -Pick up where the plan left off (parallelization map §10.2 Wave 5–6). Suggested order: -1. **Close the gaps from SESSION_2026-06-30_002 "Known gaps"** — they're small and de-risk the rest: - - age at-rest encryption of backups (CONTRACT_004 §4.3): generate the age key, store recipient/identity - (Vault `foundation/backup/backup-credentials` + passphrase config), encrypt artifacts before upload. - - Mirror Forgejo crypto secrets (SECRET_KEY/INTERNAL_TOKEN/JWT from app.ini) into - `foundation/forgejo/service-credentials`. - - Investigate + fix the `foundation-net` ipam refresh diff so `up --refresh` is safe. -2. **T11 handover** — push the foundation repo into Forgejo (`olsitec/foundation`) and switch origin; - mirror `ai-baseline`. (The repo already exists in Forgejo from T09 with a README — reconcile.) -3. **T13 DR** — `dr/RUNBOOK.md` + `dr/restore-to-fresh-vm.sh`; rehearse a full rebuild on a clean VM from - the offsite bundle (the destructive sibling of `backup/restore.sh`, restore order Vault→PG→RustFS→Forgejo). -4. **T14 CI** — `.forgejo/workflows/` (preflight, pulumi preview/up, backup-verify weekly). -5. **T15** — `index.ts` orchestration polish + Gate A/B comments + `docs/DAY-ZERO-TIMELINE.md` checklist. -6. **Then hardening**: pin remaining floating refs, fence the runner to a separate privileged VM (R5), - register the project in Olsitec MCP (D6 / PLAN-002 §8), and the Stage-2 publish of `packages/pulumi-*`. - -Validate each task live on the VM via `./run.sh up` and commit per task. diff --git a/documentation/sessions/SESSION_2026-06-30_002.md b/documentation/sessions/SESSION_2026-06-30_002.md deleted file mode 100644 index 845832d..0000000 --- a/documentation/sessions/SESSION_2026-06-30_002.md +++ /dev/null @@ -1,61 +0,0 @@ -# Session 2026-06-30 #002 — Wave 2: data plane → forge → CI → backup (egg is LIVE) - -## What was done -Built and deployed **all of Wave 2** live to the Helsinki VM (cx33, `204.168.234.72`, -SSH :222). The egg now runs **6 containers** and **`git clone git@git.olsitec.net:olsitec/foundation.git` -works**. Each task is a reviewable commit (atomic, conventional). - -- **ADR-007** — control-plane ops via `@pulumi/command` `remote.Command` (docker-exec over SSH). - Internal ports (PG 5432, Vault 8200, RustFS 9000) aren't published, so init/role/bucket/admin/token - steps run inside the VM over the existing SSH path. Idempotent, readiness-gated, **secrets on stdin** - (the command provider echoes the command on error → never inline; `environment` needs sshd AcceptEnv - which the VM rejects). This is the cross-cutting mechanism for T03/T05/T06/T09/T10. -- **T03 postgres** — `foundation-postgres` (pg17), forgejo role+DB via remote.Command. `lib/remote.ts` - (`vmConnection`) + `credentials.ts` generator half (CONTRACT_002). -- **T04 rustfs** — `foundation-rustfs` + 4 buckets + scoped service account (mc `svcacct add` works on - RustFS; `mc ready` doesn't → gate on `mc ls`; mc busybox lacks grep → shell `case`). `IMAGE_MC` pinned. -- **T05 vault** — `foundation-vault` raft (`/vault/file`, IPC_LOCK). Init/unseal over docker-exec; keys - emitted on stdout (secret, `logging:Stderr` so never streamed) → `run.sh` captures to - `vaultCredentials:*`. `vault-unseal.sh` = passphrase-gated reboot helper (ADR-004). run.sh also pins - the backend per-process (`PULUMI_BACKEND_URL`, no global `pulumi login`). -- **T06 credentials** — `writeCredentialsToVault` writes postgres+rustfs+forgejo service-credentials to - the `foundation` kv-v2 mount via `vault kv put - ` (JSON on stdin). GATE A = `dependsOn vault.init`. -- **T07 caddy** — `foundation-caddy` public ingress (80/443), **DNS-01 TLS via Cloudflare** on a custom - xcaddy image (`containers/caddy-cloudflare/Dockerfile`, `caddy-dns/cloudflare@v0.2.4`, built on the VM, - image-id is the container image). Routes forge→Forgejo, s3→RustFS. Vault NOT proxied publicly. -- **T08 forgejo** — `foundation-forgejo` (fj11): external PG, RustFS blobs (default storage + LFS), - config via `FORGEJO__` env. The image's openssh sshd owns container :22 (`START_SSH_SERVER=false` - explicitly — a stale app.ini value crash-loops it on :22). HTTP 3000 via Caddy (200). -- **T09 forge bootstrap** — headless admin + org `olsitec` + auto-init repo `olsitec/foundation` + - operator SSH key, all via docker-exec (`forgejo admin` CLI + the image's curl). **Opened firewall :22** - (provision stack) so the scp-form clone works (VM admin sshd is on :222). -- **T10 runner** — `foundation-runner` (forgejo/runner:6). Idempotent register (token via - `generate-runner-token`, never leaves the VM); daemon runs uid 1000 + host docker group (gid 996) for - socket access. A hello-world `runs-on: docker` workflow ran to **success**. -- **T12 backup** — `backup/{backup,restore}.sh` + `*-remote.sh`. Bundle (pg_dumpall, forgejo repos - tar.zst, vault raft snap, pulumi state, rustfs blobs, MANIFEST.json) → RustFS + **offsite** Synology - bucket. `restore.sh` = non-destructive scratch-restore verifier — **PASSES from both rfs and offsite**. - -## Current state -- Repo `~/work/olsitec-foundation/foundation`, branch `master`, latest commit = T12. Working tree clean. -- `cd bootstrap && ./run.sh up` is **idempotent — 41 unchanged**. Live containers: postgres, rustfs, - vault, caddy, forgejo, runner (all healthy/up). -- Master passphrase: `pass olsitec-foundation/PULUMI_CONFIG_PASSPHRASE`. VM key - `~/.ssh/foundation-test_ed25519` (also the registered Forgejo operator key). -- Verified: `https://forge.olsitec.net` = 200 (LE cert), `git clone git@git.olsitec.net:olsitec/foundation.git` - (:22) **and** `ssh://…:2222/…` both clone; Vault paths populated; CI green; backup restorable offsite. - -## Known gaps / next steps -- **age at-rest encryption** of backups (CONTRACT_004 §4.3) not yet applied — both backup destinations - are private/access-controlled; generate the age key + encrypt-before-upload is the next hardening. -- **Determinism**: a `pulumi up --refresh` surfaces a spurious `foundation-net` `ipamConfigs` diff — do - NOT apply it (recreating the network disconnects everything); plain `up` ignores it. Investigate before - enabling refresh in CI. -- **Forgejo crypto secrets** (SECRET_KEY/INTERNAL_TOKEN/JWT) auto-generate in app.ini but aren't mirrored - to Vault (`foundation/forgejo/service-credentials` has only admin user/pw). Capture them later. -- Runner is co-located + root-equivalent (host docker socket) — fence to a separate VM for untrusted CI - (PLAN-002 R5). The docker gid (996) is host-specific — re-check on DR. -- Remaining PLAN-002 tasks: **T11** handover (push repo→Forgejo, switch origin), **T13** DR-to-fresh-VM, - **T14** `.forgejo/workflows/`, **T15** index orchestration polish + DAY-ZERO checklist. - -## Operating mode for next session: HIGH-RISK / INFRA (remote VM, Docker, secrets). diff --git a/provision/index.ts b/provision/index.ts index d3eac95..303acac 100644 --- a/provision/index.ts +++ b/provision/index.ts @@ -44,13 +44,6 @@ const dockerInstall = [ "touch /root/.provision-done", ]; -// Host tools the backup/restore path runs directly on the VM (not in a container): -// `zstd` (forgejo-repos + rustfs-blobs compression) and `age` (CONTRACT_004 §4.3 -// at-rest encryption). `jq` is already in the cloud-init base packages. Declaring -// them here means a fresh DR VM (T13) has them from first boot — do not rely on -// them being present incidentally. -const backupTools = ["zstd", "age"]; - const dep = new HetznerDeployment(platformName, { platformName, hcloudToken: token, @@ -63,7 +56,7 @@ const dep = new HetznerDeployment(platformName, { type: "cx33", // 4c/8G/80GB (Helsinki); cx22 legacy, cx33 not in nbg1 image: "debian-12", labels: { purpose: "foundation-test", ephemeral: "true" }, - cloudInitConfig: { extraPackages: backupTools, lateCommands: dockerInstall }, + cloudInitConfig: { lateCommands: dockerInstall }, }, ], }); @@ -83,10 +76,6 @@ const firewall = new hcloud.Firewall( { direction: "in", protocol: "tcp", port: "443", sourceIps: ["0.0.0.0/0", "::/0"] }, // Forgejo git-over-SSH (CONTRACT_001 forgeSshPort) { direction: "in", protocol: "tcp", port: "2222", sourceIps: ["0.0.0.0/0", "::/0"] }, - // Forgejo git-over-SSH on :22 too — the VM's admin sshd is on 222, so :22 is - // free for the canonical git endpoint, making the scp-form clone - // `git@git.olsitec.net:olsitec/...` work (Forgejo sshd ForceCommands `serv`). - { direction: "in", protocol: "tcp", port: "22", sourceIps: ["0.0.0.0/0", "::/0"] }, { direction: "in", protocol: "icmp", sourceIps: ["0.0.0.0/0", "::/0"] }, ], },