#!/bin/sh # restore-remote.sh — VM-side SCRATCH restore verifier (CONTRACT_004 consumer half, # §4.6 "a backup is not trusted until restored"). Shipped + run by backup/restore.sh. # NON-DESTRUCTIVE: it reconstructs into throwaway scratch resources and asserts the # bundle is restorable — it never touches the live containers/volumes. A real # disaster restore (overwriting live, restore order Vault->Postgres->RustFS->Forgejo) # is dr/restore-to-fresh-vm.sh (T13), out of scope here. # # Secrets on stdin (OFF_* offsite creds + the age IDENTITY); non-secrets ($TS, # $MC_IMAGE, $PG_IMAGE, $SRC) as args. The bundle is age-encrypted (CONTRACT_004 # §4.3): every artifact is pulled as .age and decrypted with the identity # BEFORE its MANIFEST sha256 (a PLAINTEXT sha) is verified. set -eu IFS= read -r OFF_EP IFS= read -r OFF_AK IFS= read -r OFF_SK IFS= read -r BUCKET IFS= read -r AGE_IDENTITY TS="$1"; MC_IMAGE="$2"; PG_IMAGE="$3"; SRC="${4:-rfs}" OFFSITE_BUCKET=olsitec-foundation W="/tmp/foundation-restore-$TS" rm -rf "$W"; (umask 077; mkdir -p "$W") fail() { echo "RESTORE VERIFY FAIL: $1" >&2; docker rm -f foundation-restore-pg >/dev/null 2>&1 || true; rm -f "$W/age.key" 2>/dev/null || true; exit 1; } # Materialise the age identity to a 0600 file for `age -d -i` (removed on exit). ( umask 077; printf '%s\n' "$AGE_IDENTITY" > "$W/age.key" ) RAK=$(docker inspect foundation-rustfs --format '{{range .Config.Env}}{{println .}}{{end}}' | sed -n 's/^RUSTFS_ACCESS_KEY=//p') RSK=$(docker inspect foundation-rustfs --format '{{range .Config.Env}}{{println .}}{{end}}' | sed -n 's/^RUSTFS_SECRET_KEY=//p') echo "[restore] pull bundle $TS from $SRC" >&2 docker run --rm --network foundation-net --entrypoint sh -v "$W":/w \ -e RAK="$RAK" -e RSK="$RSK" -e OFF_EP="$OFF_EP" -e OFF_AK="$OFF_AK" -e OFF_SK="$OFF_SK" \ -e BUCKET="$BUCKET" -e TS="$TS" -e SRC="$SRC" -e OFFB="$OFFSITE_BUCKET" "$MC_IMAGE" -c ' set -e mc alias set rfs http://foundation-rustfs:9000 "$RAK" "$RSK" >/dev/null if [ "$SRC" = off ]; then mc alias set off "$OFF_EP" "$OFF_AK" "$OFF_SK" >/dev/null mc cp -r "off/$OFFB/$TS/" /w/ >/dev/null else mc cp -r "rfs/$BUCKET/$TS/" /w/ >/dev/null fi' # mc cp -r nests under $TS/ — flatten if needed [ -f "$W/MANIFEST.json" ] || { [ -d "$W/$TS" ] && mv "$W/$TS"/* "$W"/; } [ -f "$W/MANIFEST.json" ] || fail "MANIFEST.json missing from pulled bundle" cd "$W" echo "[restore] age-decrypt artifacts" >&2 for name in $(jq -r '.artifacts[].name' MANIFEST.json); do [ -f "$name.age" ] || fail "$name.age missing from bundle" age -d -i age.key -o "$name" "$name.age" 2>/dev/null || fail "age decrypt failed: $name" done echo "[restore] verify MANIFEST sha256 (plaintext)" >&2 jq -r '.artifacts[] | "\(.sha256) \(.name)"' MANIFEST.json | while read -r sha name; do [ -f "$name" ] || { echo "missing $name" >&2; exit 1; } got=$(sha256sum "$name" | cut -d' ' -f1) [ "$got" = "$sha" ] || { echo "sha mismatch $name" >&2; exit 1; } done || fail "MANIFEST sha verification failed" echo "[restore] scratch Postgres restore + assert" >&2 docker rm -f foundation-restore-pg >/dev/null 2>&1 || true docker run -d --name foundation-restore-pg -e POSTGRES_PASSWORD=scratch "$PG_IMAGE" >/dev/null i=0; until docker exec foundation-restore-pg pg_isready -U postgres >/dev/null 2>&1; do i=$((i+1)); [ "$i" -gt 30 ] && fail "scratch postgres not ready"; sleep 2; done gunzip < postgres.sql.gz | docker exec -i foundation-restore-pg psql -U postgres -q >/dev/null 2>&1 || fail "psql restore errored" ROWS=$(docker exec foundation-restore-pg psql -U postgres -d forgejo -tAc 'SELECT count(*) FROM "user"' 2>/dev/null || echo 0) [ "${ROWS:-0}" -ge 1 ] || fail "restored forgejo DB has no users (got '$ROWS')" echo "[restore] postgres OK: forgejo.\"user\" rows=$ROWS" >&2 docker rm -f foundation-restore-pg >/dev/null 2>&1 || true echo "[restore] extract forgejo repos + assert olsitec/foundation present" >&2 mkdir -p repos zstd -dc forgejo-repos.tar.zst | tar -C repos -xf - 2>/dev/null || fail "forgejo tar extract failed" [ -d repos/git/repositories/olsitec/foundation.git ] || fail "olsitec/foundation.git not in repo bundle" echo "[restore] forgejo repos OK: olsitec/foundation.git present" >&2 echo "[restore] extract rustfs blobs + assert packages present" >&2 mkdir -p blobs zstd -dc rustfs-blobs.tar.zst | tar -C blobs -xf - 2>/dev/null || fail "rustfs-blobs tar extract failed" [ -d blobs/forgejo-packages ] || fail "forgejo-packages not in blob bundle" echo "[restore] rustfs blobs OK: $(find blobs -type f | wc -l | tr -d ' ') object(s)" >&2 echo "[restore] vault snapshot sanity" >&2 [ -s vault-raft.snap ] || fail "vault-raft.snap empty" echo "[restore] vault snapshot OK: $(stat -c %s vault-raft.snap) bytes" >&2 rm -f "$W/age.key" rm -rf "$W" echo "RESTORE VERIFY PASS ($TS from $SRC, age-decrypted)"