feat(bootstrap): vault init/unseal + capture to encrypted config (T05)

foundation-vault (hashicorp/vault:1.18, digest-pinned) with integrated raft
storage in foundation-vault-data (-> /vault/file, which the entrypoint chowns to
the vault user), IPC_LOCK for mlock, internal only (8200 unpublished). Init +
unseal reuse the olsitec-core pattern but over docker-exec/SSH (ADR-007): the
foundation-vault-init command inits 1-of-1 Shamir, unseals, and emits keys + root
token on stdout — marked secret and NOT streamed (logging:Stderr) so they never
reach the terminal/logs (D2). run.sh captures them into vaultCredentials:* (the
one bootstrap secret that cannot live in Vault, CONTRACT_002 §2.4) with an
idempotent guard that avoids churning the config. vault-unseal.sh is the
passphrase-gated reboot helper (ADR-004): reads keys from config, unseals over an
SSH stdin pipe. run.sh also now pins the Pulumi backend per-process
(PULUMI_BACKEND_URL) instead of a global `pulumi login`.

Live on cx33 Helsinki: initialized + unsealed (raft 1.18.5), keys captured to
encrypted config, idempotent re-up reuses stored keys, container-restart reseal
recovered by vault-unseal.sh. Acceptance T05 met.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

VERSIONS

@@ -59,7 +59,7 @@
 IMAGE_CADDY=caddy:2.10@sha256:PIN_DIGEST
 IMAGE_FORGEJO=codeberg.org/forgejo/forgejo:11@sha256:PIN_DIGEST
 IMAGE_POSTGRES=postgres:17@sha256:5c855ad7b85e68e48a62f34662853f38b57c1c1d80f3a927ab58034fd6d31c5e
-IMAGE_VAULT=hashicorp/vault:1.18@sha256:PIN_DIGEST
+IMAGE_VAULT=hashicorp/vault:1.18@sha256:750bb37c1638fa194ab37053a81618c61bb0491ddec6fccac87c07a8e6cd8166
 IMAGE_RUSTFS=rustfs/rustfs:latest@sha256:fa19210ac4697c79d7ccca1ec9b0eb91aebacc6691991ffb14014bb3c67e6cc3
 IMAGE_ACT_RUNNER=code.forgejo.org/forgejo/runner:6@sha256:PIN_DIGEST
 IMAGE_REGISTRY=registry:2@sha256:PIN_DIGEST

bootstrap/Pulumi.foundation.yaml

@@ -59,4 +59,8 @@ config:
     secure: v1:svEvJ5K9u+FMnpV/:RztjS8VMSxrdgpBtbNpBPA6gfPLVgnABp77diBC6nWGHZRnG
   foundation:backup.offsiteSecretKey:
     secure: v1:lkkGBjgmJqVziusc:gpmw5lkfFAjXzeFikhtQnvWObYpKD3Bq5XSmrBA/vlLaoqqxFGAAO4Cq7V8nOLZ926x3fXukPQI=
+  vaultCredentials:unsealKeys:
+    secure: v1:9YpTkFoQanMwxAQV:dJ4YmXS0aOTHPbuK1H6AJ0SAJ0CjYX0iIyLOQAUNfsOWLsSy5TXxPpGecieBWkzc4AALDkJNlQN9Xo6Q0ZcaSg==
+  vaultCredentials:rootToken:
+    secure: v1:OUpYMjnaftxMUKjv:2m+dydQopXGRleeX6ddhYSHgHP7HHZXYLAvQHXUvaA91qajoxU+VugDB/Rs=
 encryptionsalt: v1:5YhUt8BVfH0=:v1:DPCHl+7zwn4RaMPj:A19tZzBlZ1NmDtTWrHreEKk5e8idyw==

bootstrap/components/vault.ts

@@ -0,0 +1,161 @@
+// components/vault.ts  (T05) — the hard one.
+//
+// foundation-vault — the runtime secret store (CONTRACT_003 §3.2; ADR-004).
+// Integrated raft storage in the named volume foundation-vault-data (→ /vault/file,
+// which the image's entrypoint chowns to the vault user); IPC_LOCK so mlock keeps
+// secrets out of swap. Internal-only (8200 NOT published; Caddy fronts it later).
+//
+// Init + unseal follow the proven olsitec-core pattern (init → capture keys → write
+// back to passphrase-encrypted config → unseal), but the MECHANISM is docker-exec
+// over SSH (ADR-007) because 8200 isn't reachable from the operator. The init
+// command emits the unseal keys + root token on STDOUT (marked secret, and NOT
+// streamed — logging:Stderr); run.sh then captures them into vaultCredentials:*
+// (CONTRACT_001 §1.3 / CONTRACT_002 §2.4 — the one bootstrap exception that cannot
+// live in Vault). On re-runs it reads those stored keys back (via stdin) and only
+// re-unseals. Reboots re-seal Vault → the operator runs vault-unseal.sh (ADR-004).
+import * as pulumi from "@pulumi/pulumi";
+import * as docker from "@pulumi/docker";
+import * as command from "@pulumi/command";
+import { DeployCtx } from "../lib/context";
+import { vmConnection } from "../lib/remote";
+
+export interface VaultOutputs {
+  container: docker.Container;
+  /** Vault is initialized + unsealed once this resolves (GATE A for T06). */
+  init: command.remote.Command;
+  /** JSON-array string of unseal keys (secret) — captured to config by run.sh. */
+  unsealKeys: pulumi.Output<string>;
+  /** Root token (secret) — captured to config by run.sh. */
+  rootToken: pulumi.Output<string>;
+  /** Internal API endpoint (CONTRACT_003 §3.3). */
+  endpoint: string;
+}
+
+// Raft single-node config (api/cluster on the foundation-net name). 1-of-1 Shamir:
+// all shares live together in the same passphrase-encrypted config, so splitting
+// adds no real security here — the passphrase IS the root of trust (PLAN-002 §4.1).
+// Rekey to N/M later if multi-custodian distribution is ever wanted.
+const VAULT_LOCAL_CONFIG = JSON.stringify({
+  storage: { raft: { path: "/vault/file", node_id: "foundation-vault" } },
+  listener: { tcp: { address: "0.0.0.0:8200", tls_disable: true } },
+  api_addr: "http://foundation-vault:8200",
+  cluster_addr: "http://foundation-vault:8201",
+  ui: true,
+  disable_mlock: false,
+});
+
+// Idempotent init/unseal (ADR-007). Stored keys arrive on stdin (line 1 = JSON
+// array of unseal keys or empty on first run; line 2 = root token or empty).
+// Progress → stderr; the captured creds are the ONLY thing on stdout (two lines),
+// and the command sets logging:Stderr so stdout is never streamed to the terminal.
+const INIT_UNSEAL = `set -eu
+IFS= read -r STORED_KEYS_JSON || true
+IFS= read -r STORED_ROOT_TOKEN || true
+C=foundation-vault
+VE='-e VAULT_ADDR=http://127.0.0.1:8200'
+
+vstat() { docker exec $VE "$C" vault status -format=json 2>/dev/null; }
+
+ready=
+for _ in $(seq 1 40); do
+  vstat >/tmp/vstatus && rc=0 || rc=$?
+  # 0 = unsealed, 2 = sealed/uninitialized but reachable; both mean "up"
+  if [ "$rc" = 0 ] || { [ "$rc" = 2 ] && [ -s /tmp/vstatus ]; }; then ready=1; break; fi
+  sleep 2
+done
+[ "$ready" = 1 ] || { echo "foundation-vault not reachable after 80s" >&2; exit 1; }
+
+INITIALIZED=$(vstat | jq -r '.initialized')
+if [ "$INITIALIZED" = "false" ]; then
+  echo "initializing vault (1/1 shamir)" >&2
+  INIT_JSON=$(docker exec $VE "$C" vault operator init -key-shares=1 -key-threshold=1 -format=json)
+  KEYS_JSON=$(printf '%s' "$INIT_JSON" | jq -c '.unseal_keys_b64')
+  ROOT_TOKEN=$(printf '%s' "$INIT_JSON" | jq -r '.root_token')
+else
+  echo "vault already initialized; reusing stored keys" >&2
+  [ -n "$STORED_KEYS_JSON" ] || { echo "vault initialized but no stored unseal keys in config" >&2; exit 1; }
+  KEYS_JSON="$STORED_KEYS_JSON"
+  ROOT_TOKEN="$STORED_ROOT_TOKEN"
+fi
+
+if [ "$(vstat | jq -r '.sealed')" = "true" ]; then
+  echo "unsealing" >&2
+  printf '%s' "$KEYS_JSON" | jq -r '.[]' | while IFS= read -r k; do
+    docker exec $VE "$C" vault operator unseal "$k" >/dev/null
+  done
+fi
+[ "$(vstat | jq -r '.sealed')" = "false" ] || { echo "vault still sealed after unseal" >&2; exit 1; }
+echo "vault unsealed and ready" >&2
+
+# stdout: the two captured secrets (run.sh -> vaultCredentials:*). Never streamed.
+printf '%s\n%s\n' "$KEYS_JSON" "$ROOT_TOKEN"`;
+
+export function deployVault(ctx: DeployCtx): VaultOutputs {
+  const { provider, network } = ctx;
+
+  const image = new docker.RemoteImage(
+    "foundation-vault-image",
+    { name: ctx.image("VAULT"), keepLocally: true },
+    { provider },
+  );
+
+  const volume = new docker.Volume(
+    "foundation-vault-data",
+    { name: "foundation-vault-data" },
+    { provider, retainOnDelete: true }, // raft store — losing it loses ALL secrets
+  );
+
+  const container = new docker.Container(
+    "foundation-vault",
+    {
+      name: "foundation-vault",
+      image: image.imageId,
+      hostname: "foundation-vault",
+      restart: "unless-stopped",
+      command: ["server"], // override the image's default `server -dev`
+      envs: [
+        `VAULT_LOCAL_CONFIG=${VAULT_LOCAL_CONFIG}`,
+        "VAULT_API_ADDR=http://foundation-vault:8200",
+      ],
+      capabilities: { adds: ["IPC_LOCK"] },
+      volumes: [{ volumeName: volume.name, containerPath: "/vault/file" }],
+      networksAdvanced: [{ name: network.name, aliases: ["foundation-vault"] }],
+      logDriver: "json-file",
+      logOpts: { "max-size": "10m", "max-file": "3" },
+    },
+    { provider, dependsOn: [network], deleteBeforeReplace: true },
+  );
+
+  // Stored creds from passphrase-encrypted config (undefined → "" on first init).
+  const vc = new pulumi.Config("vaultCredentials");
+  const stdin = pulumi
+    .all([vc.getSecret("unsealKeys"), vc.getSecret("rootToken")])
+    .apply(([k, t]) => `${k ?? ""}\n${t ?? ""}\n`);
+
+  const init = new command.remote.Command(
+    "foundation-vault-init",
+    {
+      connection: vmConnection(ctx),
+      create: INIT_UNSEAL,
+      update: INIT_UNSEAL, // re-run on stdin change (stored keys appear after capture)
+      stdin,
+      addPreviousOutputInEnv: false,
+      // Keep the captured keys (stdout) OUT of the streamed log — only stderr shows.
+      logging: command.remote.Logging.Stderr,
+      triggers: [container.id, stdin],
+    },
+    { dependsOn: [container], additionalSecretOutputs: ["stdout"] },
+  );
+
+  // stdout = "<unsealKeysJsonArray>\n<rootToken>\n" (secret). Split into the two.
+  const unsealKeys = init.stdout.apply((s) => s.split("\n")[0] ?? "");
+  const rootToken = init.stdout.apply((s) => s.split("\n")[1] ?? "");
+
+  return {
+    container,
+    init,
+    unsealKeys: pulumi.secret(unsealKeys),
+    rootToken: pulumi.secret(rootToken),
+    endpoint: "http://foundation-vault:8200",
+  };
+}

bootstrap/index.ts

@@ -12,6 +12,7 @@ import { deployDns } from "./components/dns";
 import { generateCredentials } from "./components/credentials";
 import { deployPostgres } from "./components/postgres";
 import { deployRustfs } from "./components/rustfs";
+import { deployVault } from "./components/vault";
 
 const cfg = loadConfig();
 
@@ -29,14 +30,14 @@ const credentials = generateCredentials(ctx);
 
 // =============================================================================
 // PHASE 3 — DATA PLANE  (depends on: network)
-//   T03 postgres ✓ · T04 rustfs ✓ · T05 vault (sealed)
+//   T03 postgres ✓ · T04 rustfs ✓ · T05 vault ✓
 // -----------------------------------------------------------------------------
 const postgres = deployPostgres(ctx, credentials.postgres);
 const rustfs = deployRustfs(ctx, credentials.rustfs);
-// const vault    = deployVault(ctx);
+const vault = deployVault(ctx);
 //
-// --- GATE A: Vault init + unseal (T05) → writes unseal keys to encrypted config;
+// --- GATE A: Vault init + unseal (T05) → run.sh captures unseal keys to encrypted
-//     credentials.ts (T06) dependsOn the init resource.
+//     config; credentials.ts (T06) dependsOn vault.init.
 // writeCredentialsToVault(ctx, credentials, { vault });
 //
 // =============================================================================
@@ -50,11 +51,16 @@ const rustfs = deployRustfs(ctx, credentials.rustfs);
 // =============================================================================
 
 // Stack outputs (extended as phases land).
-export const phase = "T04-rustfs"; // network + DNS + data-plane: postgres, rustfs
+export const phase = "T05-vault"; // data-plane complete: postgres, rustfs, vault
 export const networkName = network.name;
 export const vmTarget = `${cfg.vm.user}@${cfg.vm.host}`;
 export const postgresEndpoint = postgres.endpoint;
 export const rustfsEndpoint = rustfs.endpoint;
+export const vaultEndpoint = vault.endpoint;
+// Captured by run.sh into vaultCredentials:* (passphrase-encrypted config) after
+// `up` — the one bootstrap secret that cannot live in Vault (CONTRACT_002 §2.4).
+export const vaultUnsealKeys = vault.unsealKeys;
+export const vaultRootToken = vault.rootToken;
 export const enabledFeatures = Object.entries(cfg.features)
   .filter(([, on]) => on)
   .map(([name]) => name);

bootstrap/run.sh

@@ -2,8 +2,30 @@
 # Reproducible foundation deploy. Master passphrase = the single external secret.
 set -euo pipefail
 DIR="$(cd "$(dirname "$0")" && pwd)"
+# Pin the backend PER-PROCESS via env — NEVER `pulumi login` (that mutates the
+# GLOBAL backend pointer in ~/.pulumi and would misdirect other projects' run.sh).
+export PULUMI_BACKEND_URL="file://${DIR}/state"
 export PULUMI_CONFIG_PASSPHRASE="$(pass olsitec-foundation/PULUMI_CONFIG_PASSPHRASE)"
-# Test/initial deploy uses the dedicated VM key on port 222 (config carries host+port).
 export SSH_PRIVATE_KEY_PATH="${SSH_PRIVATE_KEY_PATH:-${HOME}/.ssh/foundation-test_ed25519}"
-pulumi login "file://${DIR}/state" >/dev/null
+cd "$DIR"
-( cd "$DIR" && (pulumi stack select foundation 2>/dev/null || pulumi stack init foundation) && pulumi "$@" )
+pulumi stack select foundation 2>/dev/null || pulumi stack init foundation
+pulumi "$@"
+
+# After a successful `up`, capture Vault's unseal keys + root token (emitted by the
+# foundation-vault-init command as secret stack outputs) back into the
+# passphrase-encrypted config (vaultCredentials:*). This is the proven
+# olsitec-core/run.sh pattern and the ONE bootstrap secret that cannot live in
+# Vault (CONTRACT_002 §2.4). Idempotent: only writes when the value actually
+# changes, so Pulumi.foundation.yaml is not churned on every deploy.
+if [ "${1:-}" = "up" ]; then
+  uk=$(pulumi stack output vaultUnsealKeys --show-secrets 2>/dev/null || true)
+  rt=$(pulumi stack output vaultRootToken --show-secrets 2>/dev/null || true)
+  if [ -n "$uk" ] && [ -n "$rt" ]; then
+    cur=$(pulumi config get vaultCredentials:unsealKeys 2>/dev/null || true)
+    if [ "$cur" != "$uk" ]; then
+      pulumi config set vaultCredentials:unsealKeys --secret "$uk"
+      pulumi config set vaultCredentials:rootToken  --secret "$rt"
+      echo "run.sh: captured Vault unseal keys + root token into encrypted config"
+    fi
+  fi
+fi

bootstrap/vault-unseal.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Passphrase-gated Vault unseal helper (ADR-004). After a VM reboot foundation-vault
+# re-seals; this reads the unseal keys from the passphrase-encrypted Pulumi config
+# and unseals over SSH via docker exec (ADR-007). No external KMS, no SaaS — reboots
+# require the master passphrase to be available (the ratified Layer-0 trade-off).
+#
+# `pulumi up` unseals as part of a deploy (the foundation-vault-init command); this
+# helper is the break-glass for a reboot WITHOUT a deploy.
+set -euo pipefail
+DIR="$(cd "$(dirname "$0")" && pwd)"
+export PULUMI_BACKEND_URL="file://${DIR}/state"
+export PULUMI_CONFIG_PASSPHRASE="$(pass olsitec-foundation/PULUMI_CONFIG_PASSPHRASE)"
+cd "$DIR"
+pulumi stack select foundation >/dev/null
+
+KEYS_JSON=$(pulumi config get vaultCredentials:unsealKeys 2>/dev/null || true)
+[ -n "$KEYS_JSON" ] || { echo "no vaultCredentials:unsealKeys in config — is Vault initialized?" >&2; exit 1; }
+HOST=$(pulumi config get foundation:vm.host)
+PORT=$(pulumi config get foundation:vm.sshPort 2>/dev/null || echo 22)
+SSHUSER=$(pulumi config get foundation:vm.user)
+KEY="${SSH_PRIVATE_KEY_PATH:-${HOME}/.ssh/foundation-test_ed25519}"
+
+# Keys travel over the SSH stdin pipe, NEVER on the command line. The remote shell
+# unseals foundation-vault with each one via docker exec.
+printf '%s' "$KEYS_JSON" | jq -r '.[]' | \
+  ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$KEY" -p "$PORT" "$SSHUSER@$HOST" '
+    n=0
+    while IFS= read -r k; do
+      docker exec -e VAULT_ADDR=http://127.0.0.1:8200 foundation-vault vault operator unseal "$k" >/dev/null && n=$((n+1))
+    done
+    echo "applied $n unseal key(s)"
+    docker exec -e VAULT_ADDR=http://127.0.0.1:8200 foundation-vault vault status -format=json | jq -r "\"sealed: \" + (.sealed|tostring)"
+  '
+echo "vault-unseal.sh: done"