foundation/backup/backup-remote.sh

#!/bin/sh
# backup-remote.sh — the VM-side bundle assembler (CONTRACT_004 producer half).
# Shipped + run by backup/backup.sh; NOT run directly. Secrets arrive on stdin
# (never argv); non-secrets ($TS, $MC_IMAGE, $AGE_RECIPIENT) are args. pulumi-state.json
# is already in $W (the operator placed it there before invoking this).
#
# Produces foundation-backups/<TS>/ in RustFS and replicates it to the offsite
# bucket. Artifacts per CONTRACT_004 §4.2: postgres.sql.gz, forgejo-repos.tar.zst,
# vault-raft.snap, pulumi-state.json, rustfs-blobs.tar.zst, MANIFEST.json.
#
# At-rest encryption (CONTRACT_004 §4.3): every DATA artifact is age-encrypted to
# $AGE_RECIPIENT before upload (`<name>` -> `<name>.age`); only MANIFEST.json travels
# in cleartext (it carries no secrets — it is the inventory + integrity gate, and
# lists each artifact's PLAINTEXT sha256 so restore verifies after decryption). The
# matching identity is in Vault + passphrase-encrypted config (CONTRACT_004 §4.3),
# so {repo + passphrase} can always decrypt even after total Vault loss.
set -eu
IFS= read -r VAULT_TOKEN
IFS= read -r OFF_EP
IFS= read -r OFF_AK
IFS= read -r OFF_SK
IFS= read -r BUCKET
TS="$1"
MC_IMAGE="$2"
AGE_RECIPIENT="$3"
OFFSITE_BUCKET=olsitec-foundation
W="/tmp/foundation-backup-$TS"
mkdir -p "$W"

echo "[backup] postgres pg_dumpall" >&2
docker exec foundation-postgres pg_dumpall -U postgres | gzip > "$W/postgres.sql.gz"

echo "[backup] forgejo data volume (tar.zst)" >&2
# CONTRACT_004 §4.2 needs the git repos AND app.ini AND the host SSH keys — all live
# in the /data volume (git/, gitea/conf/app.ini, ssh/ssh_host_*). Tar the WHOLE volume
# (it is ~1 MB; the DB + LFS/packages are externalised to PG + RustFS, so /data has no
# large recreatable state at Layer 0). Use the container's own tar (no extra image).
docker exec foundation-forgejo sh -c 'tar -C /data -cf - .' | zstd -q -T0 > "$W/forgejo-repos.tar.zst"

echo "[backup] vault raft snapshot" >&2
docker exec -e VAULT_ADDR=http://127.0.0.1:8200 -e VAULT_TOKEN="$VAULT_TOKEN" foundation-vault \
  sh -c 'vault operator raft snapshot save /tmp/v.snap >/dev/null 2>&1 && cat /tmp/v.snap && rm -f /tmp/v.snap' > "$W/vault-raft.snap"

# RustFS root creds from the running container (VM-trusted).
RAK=$(docker inspect foundation-rustfs --format '{{range .Config.Env}}{{println .}}{{end}}' | sed -n 's/^RUSTFS_ACCESS_KEY=//p')
RSK=$(docker inspect foundation-rustfs --format '{{range .Config.Env}}{{println .}}{{end}}' | sed -n 's/^RUSTFS_SECRET_KEY=//p')

echo "[backup] rustfs blobs -> rustfs-blobs.tar.zst" >&2
# Pull the blob buckets onto the VM fs so the bundle is a single encrypted unit
# (CONTRACT_004 §4.3 "whole bundle"). Tiny at Layer 0; may be made incremental later.
mkdir -p "$W/blobs/forgejo-packages" "$W/blobs/forgejo-artifacts" "$W/blobs/forgejo-lfs"
docker run --rm --network foundation-net --entrypoint sh -v "$W":/w \
  -e RAK="$RAK" -e RSK="$RSK" "$MC_IMAGE" -c '
    set -e
    mc alias set rfs http://foundation-rustfs:9000 "$RAK" "$RSK" >/dev/null
    for b in forgejo-packages forgejo-artifacts forgejo-lfs; do
      mc mirror --overwrite --quiet "rfs/$b" "/w/blobs/$b" >/dev/null 2>&1 || true
    done'
tar -C "$W/blobs" -cf - . | zstd -q -T0 > "$W/rustfs-blobs.tar.zst"
rm -rf "$W/blobs"

echo "[backup] MANIFEST.json" >&2
( cd "$W"
  jq -n --arg ts "$TS" --arg rcpt "$AGE_RECIPIENT" \
    --argjson files "$(for f in postgres.sql.gz forgejo-repos.tar.zst vault-raft.snap pulumi-state.json rustfs-blobs.tar.zst; do
        [ -f "$f" ] || continue
        jq -n --arg n "$f" --arg sha "$(sha256sum "$f" | cut -d' ' -f1)" --argjson sz "$(stat -c %s "$f")" \
          '{name:$n, sha256:$sha, size:$sz}'
      done | jq -s '.')" \
    '{timestamp:$ts, encryption:"age", ageRecipient:$rcpt, restoreOrder:["vault","postgres","rustfs","forgejo"], artifacts:$files}' > MANIFEST.json
)

echo "[backup] age-encrypt artifacts (-> *.age)" >&2
for f in postgres.sql.gz forgejo-repos.tar.zst vault-raft.snap pulumi-state.json rustfs-blobs.tar.zst; do
  [ -f "$W/$f" ] || continue
  age -r "$AGE_RECIPIENT" -o "$W/$f.age" "$W/$f"
  rm -f "$W/$f"
done

echo "[backup] upload to RustFS $BUCKET/$TS + replicate offsite" >&2
docker run --rm --network foundation-net --entrypoint sh -v "$W":/w \
  -e RAK="$RAK" -e RSK="$RSK" -e OFF_EP="$OFF_EP" -e OFF_AK="$OFF_AK" -e OFF_SK="$OFF_SK" \
  -e BUCKET="$BUCKET" -e TS="$TS" -e OFFB="$OFFSITE_BUCKET" \
  "$MC_IMAGE" -c '
    set -e
    mc alias set rfs http://foundation-rustfs:9000 "$RAK" "$RSK" >/dev/null
    mc alias set off "$OFF_EP" "$OFF_AK" "$OFF_SK" >/dev/null
    mc cp -r /w/ "rfs/$BUCKET/$TS/" >/dev/null
    mc mirror --overwrite --quiet "rfs/$BUCKET/$TS" "off/$OFFB/$TS" >/dev/null
  '
rm -rf "$W"
echo "[backup] complete: rfs/$BUCKET/$TS (+ offsite $OFFSITE_BUCKET/$TS), age-encrypted" >&2