Fix Codex Security scan findings (#7437)

* Add security regression coverage

* Fix security scan findings

* Address security review follow-ups

* Fix review regressions in lease cleanup

* Fix final security review findings

* Fix remaining autoreview findings

* Address final autoreview regressions

* Keep active identity cleanup best effort

* Bound active identity cleanup

* Bound SSH cleanup before endpoint minting

* Make cleanup retries bounded and releasable

* Bound active identity cleanup preflight

* Preserve vault grants on retry presign failure

* Guard vault grant rollback state

* Back off failed expired lease cleanup

* Separate vault quota lock namespace

* Tighten VM identity cleanup ordering

* Fail closed without VM team membership

* Bound VM identity cleanup fanout

* Rollback endpoint resume on cleanup failure

* Remove nondeterministic vault upload test wait

* Fail closed on destroy identity cleanup

* Recreate Base when active provider VM is gone

* Keep Freestyle attach independent of exec probe

* Scope provider identity not-found handling

* Use reservation tokens for vault upload rollback

* Validate vault upload grants at commit

* Bound identity cleanup and duplicate vault reservations

* Stage vault uploads before commit

* Keep vault staging cleanup retryable

* Reuse active vault upload staging keys

* Preserve legacy vault upload commits

* Make vault staging cleanup recoverable

* Avoid endpoint resume rollback races

* Serialize vault upload grant cleanup

* Finalize vault staging outside quota locks

* Track superseded vault upload keys

Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Transport/SocketClientAuthorization.swift

@@ -0,0 +1,31 @@
+public import Darwin
+
+/// Authorizes peer processes for cmux-only control socket requests.
+public struct SocketClientAuthorization {
+    /// Creates an authorization helper with no retained process state.
+    public init() {}
+
+    /// Returns whether a peer process is allowed to use cmux-only socket operations.
+    ///
+    /// A non-nil `peerProcessID` must resolve as a descendant of the trusted cmux
+    /// process tree. A nil PID fails closed because the caller cannot be tied to
+    /// a concrete process. `peerHasSameUID` is supplied by the socket handshake
+    /// for callers that need it, but this check intentionally relies on ancestry.
+    ///
+    /// - Parameters:
+    ///   - peerProcessID: The PID reported by the accepted socket, or nil when
+    ///     the platform cannot provide one.
+    ///   - peerHasSameUID: Whether the peer process has the same user ID as cmux.
+    ///   - isDescendant: Predicate that verifies the PID belongs to the trusted
+    ///     cmux process tree.
+    public func isCmuxOnlyClientAllowed(
+        peerProcessID: pid_t?,
+        peerHasSameUID _: Bool,
+        isDescendant: (pid_t) -> Bool
+    ) -> Bool {
+        if let peerProcessID {
+            return isDescendant(peerProcessID)
+        }
+        return false
+    }
+}

Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/SocketClientAuthorizationTests.swift

@@ -0,0 +1,31 @@
+import CmuxControlSocket
+import Testing
+
+@Suite("Socket client authorization")
+struct SocketClientAuthorizationTests {
+    private let authorization = SocketClientAuthorization()
+
+    @Test func cmuxOnlyFailsClosedWhenPeerPidIsUnavailable() {
+        #expect(!authorization.isCmuxOnlyClientAllowed(
+            peerProcessID: nil,
+            peerHasSameUID: true,
+            isDescendant: { _ in true }
+        ))
+    }
+
+    @Test func cmuxOnlyAllowsDescendantPeerPid() {
+        #expect(authorization.isCmuxOnlyClientAllowed(
+            peerProcessID: 123,
+            peerHasSameUID: false,
+            isDescendant: { $0 == 123 }
+        ))
+    }
+
+    @Test func cmuxOnlyRejectsNonDescendantPeerPid() {
+        #expect(!authorization.isCmuxOnlyClientAllowed(
+            peerProcessID: 123,
+            peerHasSameUID: true,
+            isDescendant: { _ in false }
+        ))
+    }
+}

Sources/TerminalController.swift

@@ -1478,30 +1478,18 @@ class TerminalController {
             // Use pre-captured peer PID if available (captured in accept loop before
             // the peer can disconnect), falling back to live lookup.
             let pid = peerPid ?? transport.peerProcessID(of: socket)
-            if let pid {
-                guard isDescendant(pid) else {
-                    _ = writeSocketResponse(
-                        "ERROR: Access denied — only processes started inside cmux can connect",
-                        to: socket
-                    )
-                    return
-                }
-            }
-            // If pid is nil, LOCAL_PEERPID failed (peer disconnected before we
-            // could read it — common with ncat --send-only). We still verify the
-            // peer runs as the same user via LOCAL_PEERCRED. This is the same
-            // security boundary as the socket file permissions (0600), so it does
-            // not widen the attack surface. We also require that the peer actually
-            // sent data (checked in the read loop below) — a connect-only probe
-            // with no data is harmless.
-            if pid == nil {
-                guard transport.peerHasSameUID(socket) else {
-                    _ = writeSocketResponse(
-                        "ERROR: Unable to verify client process",
-                        to: socket
-                    )
-                    return
-                }
+            guard SocketClientAuthorization().isCmuxOnlyClientAllowed(
+                peerProcessID: pid,
+                peerHasSameUID: false,
+                isDescendant: { isDescendant($0) }
+            ) else {
+                _ = writeSocketResponse(
+                    pid == nil
+                        ? "ERROR: Unable to verify client process"
+                        : "ERROR: Access denied — only processes started inside cmux can connect",
+                    to: socket
+                )
+                return
             }
         }
 

vault/internal/agentdirs/agent.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -172,7 +173,7 @@ func statSession(agentName, root, path, id, cwd string) (Session, error) {
 }
 
 func statSessionWithLogicalPath(agentName, root, path, logicalPath, id, cwd string) (Session, error) {
-	info, err := os.Stat(path)
+	info, err := RegularFileInfoNoSymlink(path)
 	if err != nil {
 		return Session{}, err
 	}
@@ -192,7 +193,7 @@ func statSessionWithLogicalPath(agentName, root, path, logicalPath, id, cwd stri
 }
 
 func recoverCWDFromJSONL(path string) string {
-	file, err := os.Open(path)
+	file, _, err := OpenRegularFileNoSymlink(path)
 	if err != nil {
 		return ""
 	}
@@ -213,6 +214,19 @@ func recoverCWDFromJSONL(path string) string {
 	return ""
 }
 
+func IsSymlinkEntry(entry fs.DirEntry) bool {
+	return entry.Type()&fs.ModeSymlink != 0
+}
+
+func RegularFileInfoNoSymlink(path string) (os.FileInfo, error) {
+	file, info, err := OpenRegularFileNoSymlink(path)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+	return info, nil
+}
+
 func cwdFromJSON(data []byte) string {
 	var value any
 	if err := json.Unmarshal(data, &value); err != nil {

vault/internal/agentdirs/agent_open_unix.go

@@ -0,0 +1,31 @@
+//go:build !windows
+
+package agentdirs
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+)
+
+func OpenRegularFileNoSymlink(path string) (*os.File, os.FileInfo, error) {
+	fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_NOFOLLOW, 0)
+	if err != nil {
+		return nil, nil, err
+	}
+	file := os.NewFile(uintptr(fd), path)
+	if file == nil {
+		_ = syscall.Close(fd)
+		return nil, nil, fmt.Errorf("failed to open %s", path)
+	}
+	info, err := file.Stat()
+	if err != nil {
+		_ = file.Close()
+		return nil, nil, err
+	}
+	if !info.Mode().IsRegular() {
+		_ = file.Close()
+		return nil, nil, fmt.Errorf("%s is not a regular file", path)
+	}
+	return file, info, nil
+}

vault/internal/agentdirs/agent_open_windows.go

@@ -0,0 +1,52 @@
+//go:build windows
+
+package agentdirs
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+)
+
+func OpenRegularFileNoSymlink(path string) (*os.File, os.FileInfo, error) {
+	pathp, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return nil, nil, err
+	}
+	handle, err := syscall.CreateFile(
+		pathp,
+		syscall.GENERIC_READ,
+		syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE,
+		nil,
+		syscall.OPEN_EXISTING,
+		syscall.FILE_ATTRIBUTE_NORMAL|syscall.FILE_FLAG_OPEN_REPARSE_POINT,
+		0,
+	)
+	if err != nil {
+		return nil, nil, err
+	}
+	file := os.NewFile(uintptr(handle), path)
+	if file == nil {
+		_ = syscall.CloseHandle(handle)
+		return nil, nil, fmt.Errorf("failed to open %s", path)
+	}
+	info, err := file.Stat()
+	if err != nil {
+		_ = file.Close()
+		return nil, nil, err
+	}
+	if info.Mode()&os.ModeSymlink != 0 {
+		_ = file.Close()
+		return nil, nil, fmt.Errorf("%s is a symlink", path)
+	}
+	if data, ok := info.Sys().(*syscall.Win32FileAttributeData); ok &&
+		data.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0 {
+		_ = file.Close()
+		return nil, nil, fmt.Errorf("%s is a reparse point", path)
+	}
+	if !info.Mode().IsRegular() {
+		_ = file.Close()
+		return nil, nil, fmt.Errorf("%s is not a regular file", path)
+	}
+	return file, info, nil
+}

vault/internal/agentdirs/agentdirs_test.go

@@ -15,6 +15,8 @@ const (
 
 func TestClaudeDiscover(t *testing.T) {
 	root := filepath.Join(t.TempDir(), "claude-config")
+	secretPath := filepath.Join(t.TempDir(), "secret.jsonl")
+	writeFile(t, secretPath, `{"cwd":"/secret"}`+"\n")
 	sessionPath := filepath.Join(root, "projects", "-Users-lawrence-work-cmux", uuidA+".jsonl")
 	writeFile(t, sessionPath, `{"type":"message","cwd":"/Users/lawrence/work/cmux"}`+"\n")
 	writeFile(t, filepath.Join(root, "projects", "-Users-lawrence-work-cmux", "not-a-session.jsonl"), "{}\n")
@@ -23,6 +25,9 @@ func TestClaudeDiscover(t *testing.T) {
 	if err := os.Symlink(filepath.Join(root, "missing-target.jsonl"), filepath.Join(root, "projects", "nested", uuidC+".jsonl")); err != nil {
 		t.Fatal(err)
 	}
+	if err := os.Symlink(secretPath, filepath.Join(root, "projects", "nested", uuidD+".jsonl")); err != nil {
+		t.Fatal(err)
+	}
 	unreadable := filepath.Join(root, "projects", "unreadable")
 	if err := os.MkdirAll(unreadable, 0o700); err != nil {
 		t.Fatal(err)
@@ -58,6 +63,29 @@ func TestClaudeDiscover(t *testing.T) {
 	}
 }
 
+func TestCodexDiscoverSkipsSymlinkedSessionFiles(t *testing.T) {
+	home := t.TempDir()
+	root := filepath.Join(home, ".codex")
+	secretPath := filepath.Join(t.TempDir(), "secret.jsonl")
+	writeFile(t, secretPath, `{"type":"session_meta","payload":{"id":"`+uuidD+`","cwd":"/secret"}}`+"\n")
+	sessionDir := filepath.Join(root, "sessions", "2026", "07", "04")
+	if err := os.MkdirAll(sessionDir, 0o700); err != nil {
+		t.Fatal(err)
+	}
+	link := filepath.Join(sessionDir, "rollout-2026-07-04T00-00-00-"+uuidD+".jsonl")
+	if err := os.Symlink(secretPath, link); err != nil {
+		t.Fatal(err)
+	}
+
+	got, err := (Codex{}).Discover(Environ{HomeDir: home, Vars: map[string]string{}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 0 {
+		t.Fatalf("expected symlinked session to be skipped, got %#v", got)
+	}
+}
+
 func TestDiscoverSymlinkedRoots(t *testing.T) {
 	base := t.TempDir()
 	shared := filepath.Join(base, "shared")

vault/internal/agentdirs/claude.go

@@ -39,6 +39,10 @@ func (a Claude) Discover(env Environ) ([]Session, error) {
 		if entry.IsDir() {
 			return nil
 		}
+		if IsSymlinkEntry(entry) {
+			env.Warn("claude: skipping symlinked session %s", path)
+			return nil
+		}
 		if filepath.Ext(entry.Name()) != ".jsonl" {
 			return nil
 		}

vault/internal/agentdirs/codex.go

@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"encoding/json"
 	"io/fs"
-	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -46,6 +45,10 @@ func (a Codex) Discover(env Environ) ([]Session, error) {
 			if entry.IsDir() {
 				return nil
 			}
+			if IsSymlinkEntry(entry) {
+				env.Warn("codex: skipping symlinked session %s", path)
+				return nil
+			}
 			id := codexIDFromFilename(entry.Name())
 			if id == "" {
 				return nil
@@ -78,7 +81,7 @@ func codexIDFromFilename(name string) string {
 }
 
 func codexMeta(path string) (string, string) {
-	file, err := os.Open(path)
+	file, _, err := OpenRegularFileNoSymlink(path)
 	if err != nil {
 		return "", ""
 	}

vault/internal/agentdirs/pi.go

@@ -38,6 +38,10 @@ func (a Pi) Discover(env Environ) ([]Session, error) {
 		if entry.IsDir() {
 			return nil
 		}
+		if IsSymlinkEntry(entry) {
+			env.Warn("pi: skipping symlinked session %s", path)
+			return nil
+		}
 		matches := piFileRE.FindStringSubmatch(entry.Name())
 		if len(matches) != 2 {
 			return nil

vault/internal/syncer/syncer.go

@@ -305,7 +305,7 @@ func itemKey(agent, relPath string) string {
 }
 
 func sha256File(path string) (string, error) {
-	file, err := os.Open(path)
+	file, _, err := agentdirs.OpenRegularFileNoSymlink(path)
 	if err != nil {
 		return "", err
 	}
@@ -332,7 +332,7 @@ func compressFile(path, tempDir string) (compressResult, error) {
 	if err := os.MkdirAll(tempDir, 0o700); err != nil {
 		return result, err
 	}
-	in, err := os.Open(path)
+	in, _, err := agentdirs.OpenRegularFileNoSymlink(path)
 	if err != nil {
 		return result, err
 	}

vault/internal/syncer/syncer_test.go

@@ -158,6 +158,34 @@ func TestSyncerUploadsIncrementallyAndCompresses(t *testing.T) {
 	}
 }
 
+func TestPrepareBatchRejectsSymlinkedSessionFile(t *testing.T) {
+	home := t.TempDir()
+	target := filepath.Join(home, "secret.jsonl")
+	link := filepath.Join(home, ".codex", "sessions", "2026", "07", "04", "rollout-2026-07-04T00-00-00-11111111-1111-4111-8111-111111111111.jsonl")
+	writeTestFile(t, target, []byte(`{"message":"secret"}`+"\n"))
+	if err := os.MkdirAll(filepath.Dir(link), 0o700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Symlink(target, link); err != nil {
+		t.Fatal(err)
+	}
+
+	engine := Engine{TempDir: filepath.Join(home, "tmp")}
+	_, err := engine.prepareBatch([]candidate{{
+		session: agentdirs.Session{
+			AgentName:      "codex",
+			AgentSessionID: "11111111-1111-4111-8111-111111111111",
+			AbsPath:        link,
+			RelPath:        "sessions/2026/07/04/rollout-2026-07-04T00-00-00-11111111-1111-4111-8111-111111111111.jsonl",
+			SizeBytes:      1,
+			ModTime:        time.Now(),
+		},
+	}})
+	if err == nil {
+		t.Fatal("expected symlinked session file to be rejected")
+	}
+}
+
 func writeTestFile(t *testing.T, path string, data []byte) {
 	t.Helper()
 	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {

web/app/api/enterprise/contact/route.ts

@@ -155,7 +155,10 @@ function resolveEnterpriseConfig() {
     fromEmail,
     rateLimitId,
     slackWebhookUrl:
-      env.SLACK_ENTERPRISE_WEBHOOK_URL ?? env.SLACK_WAITLIST_WEBHOOK_URL,
+      env.SLACK_ENTERPRISE_WEBHOOK_URL
+      ?? env.SLACK_WAITLIST_WEBHOOK_URL
+      ?? process.env.SLACK_ENTERPRISE_WEBHOOK_URL?.trim()
+      ?? process.env.SLACK_WAITLIST_WEBHOOK_URL?.trim(),
   };
 }
 

web/app/api/feedback/route.ts

@@ -79,8 +79,10 @@ export async function POST(request: Request) {
             "feedback.route.rate_limit_not_found",
             feedbackConfig.rateLimitId,
           );
+          return jsonError("service_unavailable", 503);
         } else if (error) {
           console.error("feedback.route.rate_limit_error", error);
+          return jsonError("service_unavailable", 503);
         }
       }
 

web/app/api/internal/vm/leases/revoke-expired/route.ts

@@ -0,0 +1,38 @@
+import { timingSafeEqual } from "node:crypto";
+import { jsonResponse } from "@/services/vms/routeHelpers";
+import { revokeExpiredIdentityLeases, runVmWorkflow } from "@/services/vms/workflows";
+
+export const runtime = "nodejs";
+export const dynamic = "force-dynamic";
+
+export async function GET(request: Request): Promise<Response> {
+  return handle(request);
+}
+
+export async function POST(request: Request): Promise<Response> {
+  return handle(request);
+}
+
+async function handle(request: Request): Promise<Response> {
+  const secret = process.env.CRON_SECRET?.trim();
+  if (!secret) {
+    console.error("vm.leases.revoke_expired.cron_secret_missing");
+    return jsonResponse({ error: "service_unavailable" }, 503);
+  }
+
+  const authorization = request.headers.get("authorization")?.trim() ?? "";
+  const token = authorization.toLowerCase().startsWith("bearer ")
+    ? authorization.slice("bearer ".length).trim()
+    : "";
+  const tokenBuffer = Buffer.from(token);
+  const secretBuffer = Buffer.from(secret);
+  const tokenMatches =
+    tokenBuffer.length === secretBuffer.length &&
+    timingSafeEqual(tokenBuffer, secretBuffer);
+  if (!tokenMatches) {
+    return jsonResponse({ error: "unauthorized" }, 401);
+  }
+
+  const revoked = await runVmWorkflow(revokeExpiredIdentityLeases());
+  return jsonResponse({ ok: true, revoked });
+}

web/app/api/vault/sessions/commit/route.ts

@@ -1,11 +1,19 @@
-import { and, eq } from "drizzle-orm";
+import { and, eq, gt } from "drizzle-orm";
 import type { Span } from "@opentelemetry/api";
 import { cloudDb } from "../../../../../db/client";
 import { vaultSessions, vaultSnapshots, vaultUploadGrants } from "../../../../../db/schema";
 import { vaultConfig } from "../../../../../services/vault/config";
 import { withAuthedVaultApiRoute } from "../../../../../services/vault/routeHelpers";
-import { buildObjectKey, headObject } from "../../../../../services/vault/storage";
-import { getVaultStoredCompressedBytes } from "../../../../../services/vault/usage";
+import {
+  buildObjectKey,
+  copyObject,
+  deleteObject,
+  headObject,
+} from "../../../../../services/vault/storage";
+import {
+  getVaultStoredCompressedBytes,
+  withVaultUserQuotaLock,
+} from "../../../../../services/vault/usage";
 import { readVaultJsonObject, validateVaultBatch } from "../../../../../services/vault/validation";
 import { setSpanAttributes } from "../../../../../services/telemetry";
 import { jsonResponse } from "../../../../../services/vms/routeHelpers";
@@ -39,97 +47,114 @@ async function handlePost(request: Request, userId: string, span: Span): Promise
 
   const config = vaultConfig();
   const db = cloudDb();
-  // Re-check the per-user quota at commit time so previously issued presigned
-  // URLs cannot bypass it. Snapshot dedup (onConflictDoNothing) makes this
-  // projection conservative: it may count a deduped snapshot, never undercount.
-  let projectedUserBytes = await getVaultStoredCompressedBytes(db, userId);
-  const results = [];
-  for (const item of batch.value) {
-    // Per-item so one oversized transcript cannot block the rest of the batch.
-    if (item.compressedSizeBytes > config.maxUploadBytes) {
-      results.push(itemResult(item, "error", "upload_too_large"));
-      continue;
-    }
-    if (projectedUserBytes + item.compressedSizeBytes > config.maxUserBytes) {
-      results.push(itemResult(item, "error", "quota_exceeded"));
-      continue;
-    }
+  const stagingCleanups: { grantId: string; objectKey: string; uploadObjectKey: string }[] = [];
+  const initialResults = await withVaultUserQuotaLock(db, userId, async (lockedDb) => {
+    // Re-check the per-user quota at commit time under the same lock used by
+    // presign. The current grant must still match this commit, so older
+    // presigned URLs cannot outlive a later downsized reservation.
+    let projectedUserBytes = await getVaultStoredCompressedBytes(lockedDb, userId);
+    const lockedResults = [];
+    for (const item of batch.value) {
+      // Per-item so one oversized transcript cannot block the rest of the batch.
+      if (item.compressedSizeBytes > config.maxUploadBytes) {
+        lockedResults.push(itemResult(item, "error", "upload_too_large"));
+        continue;
+      }
 
-    const objectKey = buildObjectKey(userId, item.agent, item.agentSessionId, item.sha256);
-    const object = await headObject(objectKey);
-    if (!object) {
-      results.push(itemResult(item, "error", "object_missing"));
-      continue;
-    }
-    // Some S3-compatible stores omit Content-Length on HEAD; only enforce the
-    // size check when the store reports one.
-    if (object.contentLength != null && object.contentLength !== item.compressedSizeBytes) {
-      results.push(itemResult(item, "error", "size_mismatch"));
-      continue;
-    }
+      const objectKey = buildObjectKey(userId, item.agent, item.agentSessionId, item.sha256);
+      const now = new Date();
+      const existingCommit = await findCommittedSnapshot(lockedDb, userId, item, objectKey);
+      if (existingCommit) {
+        lockedResults.push(committedResult(item, existingCommit.sessionId));
+        continue;
+      }
 
-    const now = new Date();
-    const committed = await db.transaction(async (tx) => {
-      const [session] = await tx
-        .insert(vaultSessions)
-        .values({
-          userId,
-          agent: item.agent,
-          agentSessionId: item.agentSessionId,
-          relPath: item.relPath,
-          cwd: item.cwd,
-          latestSha256: item.sha256,
-          latestObjectKey: objectKey,
-          sizeBytes: item.sizeBytes,
-          compressedSizeBytes: item.compressedSizeBytes,
-          firstUploadedAt: now,
-          lastUploadedAt: now,
-          metadata: {},
+      const [grant] = await lockedDb
+        .select({
+          id: vaultUploadGrants.id,
+          uploadObjectKey: vaultUploadGrants.uploadObjectKey,
+          compressedSizeBytes: vaultUploadGrants.compressedSizeBytes,
         })
-        .onConflictDoUpdate({
-          target: [vaultSessions.userId, vaultSessions.agent, vaultSessions.agentSessionId],
-          set: {
-            relPath: item.relPath,
-            cwd: item.cwd,
-            latestSha256: item.sha256,
-            latestObjectKey: objectKey,
-            sizeBytes: item.sizeBytes,
-            compressedSizeBytes: item.compressedSizeBytes,
-            lastUploadedAt: now,
-          },
-        })
-        .returning({ id: vaultSessions.id });
+        .from(vaultUploadGrants)
+        .where(and(
+          eq(vaultUploadGrants.userId, userId),
+          eq(vaultUploadGrants.objectKey, objectKey),
+          gt(vaultUploadGrants.expiresAt, now),
+        ))
+        .limit(1);
+      if (!grant) {
+        lockedResults.push(itemResult(item, "error", "upload_grant_missing"));
+        continue;
+      }
+      if (grant.compressedSizeBytes !== item.compressedSizeBytes) {
+        lockedResults.push(itemResult(item, "error", "upload_grant_mismatch"));
+        continue;
+      }
 
-      await tx
-        .insert(vaultSnapshots)
-        .values({
-          sessionId: session.id,
-          sha256: item.sha256,
+      if (projectedUserBytes + item.compressedSizeBytes > config.maxUserBytes) {
+        lockedResults.push(itemResult(item, "error", "quota_exceeded"));
+        continue;
+      }
+
+      const object = await headObject(grant.uploadObjectKey);
+      if (!object) {
+        lockedResults.push(itemResult(item, "error", "object_missing"));
+        continue;
+      }
+      // Some S3-compatible stores omit Content-Length on HEAD; only enforce the
+      // size check when the store reports one.
+      if (object.contentLength != null && object.contentLength !== item.compressedSizeBytes) {
+        lockedResults.push(itemResult(item, "error", "size_mismatch"));
+        continue;
+      }
+
+      if (grant.uploadObjectKey !== objectKey) {
+        lockedResults.push({
+          status: "pending_staged_commit" as const,
+          item,
           objectKey,
-          sizeBytes: item.sizeBytes,
-          compressedSizeBytes: item.compressedSizeBytes,
-          uploadedAt: now,
-        })
-        .onConflictDoNothing({
-          target: [vaultSnapshots.sessionId, vaultSnapshots.sha256],
+          grantId: grant.id,
+          uploadObjectKey: grant.uploadObjectKey,
         });
+        projectedUserBytes += item.compressedSizeBytes;
+        continue;
+      }
 
-      // The snapshot now accounts for these bytes, so release the upload
-      // grant that reserved them at presign time.
-      await tx.delete(vaultUploadGrants).where(eq(vaultUploadGrants.objectKey, objectKey));
+      const sessionId = await commitVaultSnapshotRows(lockedDb, userId, item, objectKey, now);
+      await lockedDb.delete(vaultUploadGrants).where(eq(vaultUploadGrants.id, grant.id));
+      projectedUserBytes += item.compressedSizeBytes;
+      lockedResults.push(committedResult(item, sessionId));
+    }
+    return lockedResults;
+  });
 
-      return session;
-    });
-
-    projectedUserBytes += item.compressedSizeBytes;
-    results.push({
-      agent: item.agent,
-      agentSessionId: item.agentSessionId,
-      relPath: item.relPath,
-      status: "committed",
-      sessionId: committed.id,
+  const results = [];
+  for (const result of initialResults) {
+    if (!isPendingStagedCommit(result)) {
+      results.push(result);
+      continue;
+    }
+    await copyObject(result.uploadObjectKey, result.objectKey);
+    let finalized;
+    try {
+      finalized = await finalizeStagedCommit(db, userId, result, config.maxUserBytes);
+    } catch (error) {
+      await deleteObject(result.objectKey).catch(() => undefined);
+      throw error;
+    }
+    if (finalized.status !== "committed") {
+      await deleteObject(result.objectKey).catch(() => undefined);
+      results.push(finalized);
+      continue;
+    }
+    stagingCleanups.push({
+      grantId: result.grantId,
+      objectKey: result.objectKey,
+      uploadObjectKey: result.uploadObjectKey,
     });
+    results.push(finalized);
   }
+  await cleanupCommittedStagingGrants(db, stagingCleanups);
   setSpanAttributes(span, {
     "cmux.vault.result_count": results.length,
     "cmux.vault.result.committed_count": countResultStatus(results, "committed"),
@@ -138,6 +163,179 @@ async function handlePost(request: Request, userId: string, span: Span): Promise
   return jsonResponse({ items: results });
 }
 
+type VaultCommitItem = {
+  readonly agent: string;
+  readonly agentSessionId: string;
+  readonly relPath: string;
+  readonly cwd: string | null;
+  readonly sha256: string;
+  readonly sizeBytes: number;
+  readonly compressedSizeBytes: number;
+};
+
+type PendingStagedCommit = {
+  readonly status: "pending_staged_commit";
+  readonly item: VaultCommitItem;
+  readonly objectKey: string;
+  readonly grantId: string;
+  readonly uploadObjectKey: string;
+};
+
+function isPendingStagedCommit(result: unknown): result is PendingStagedCommit {
+  return typeof result === "object" &&
+    result !== null &&
+    "status" in result &&
+    result.status === "pending_staged_commit";
+}
+
+async function finalizeStagedCommit(
+  db: ReturnType<typeof cloudDb>,
+  userId: string,
+  pending: PendingStagedCommit,
+  maxUserBytes: number,
+) {
+  return await withVaultUserQuotaLock(db, userId, async (lockedDb) => {
+    const existingCommit = await findCommittedSnapshot(
+      lockedDb,
+      userId,
+      pending.item,
+      pending.objectKey,
+    );
+    if (existingCommit) return committedResult(pending.item, existingCommit.sessionId);
+
+    const now = new Date();
+    const [grant] = await lockedDb
+      .select({
+        id: vaultUploadGrants.id,
+        compressedSizeBytes: vaultUploadGrants.compressedSizeBytes,
+      })
+      .from(vaultUploadGrants)
+      .where(and(
+        eq(vaultUploadGrants.id, pending.grantId),
+        eq(vaultUploadGrants.userId, userId),
+        eq(vaultUploadGrants.objectKey, pending.objectKey),
+        eq(vaultUploadGrants.uploadObjectKey, pending.uploadObjectKey),
+        gt(vaultUploadGrants.expiresAt, now),
+      ))
+      .limit(1);
+    if (!grant) return itemResult(pending.item, "error", "upload_grant_missing");
+    if (grant.compressedSizeBytes !== pending.item.compressedSizeBytes) {
+      return itemResult(pending.item, "error", "upload_grant_mismatch");
+    }
+
+    const storedBytes = await getVaultStoredCompressedBytes(lockedDb, userId);
+    if (storedBytes + pending.item.compressedSizeBytes > maxUserBytes) {
+      return itemResult(pending.item, "error", "quota_exceeded");
+    }
+
+    const sessionId = await commitVaultSnapshotRows(
+      lockedDb,
+      userId,
+      pending.item,
+      pending.objectKey,
+      now,
+    );
+    return committedResult(pending.item, sessionId);
+  });
+}
+
+async function commitVaultSnapshotRows(
+  db: ReturnType<typeof cloudDb>,
+  userId: string,
+  item: VaultCommitItem,
+  objectKey: string,
+  now: Date,
+): Promise<string> {
+  const [session] = await db
+    .insert(vaultSessions)
+    .values({
+      userId,
+      agent: item.agent,
+      agentSessionId: item.agentSessionId,
+      relPath: item.relPath,
+      cwd: item.cwd,
+      latestSha256: item.sha256,
+      latestObjectKey: objectKey,
+      sizeBytes: item.sizeBytes,
+      compressedSizeBytes: item.compressedSizeBytes,
+      firstUploadedAt: now,
+      lastUploadedAt: now,
+      metadata: {},
+    })
+    .onConflictDoUpdate({
+      target: [vaultSessions.userId, vaultSessions.agent, vaultSessions.agentSessionId],
+      set: {
+        relPath: item.relPath,
+        cwd: item.cwd,
+        latestSha256: item.sha256,
+        latestObjectKey: objectKey,
+        sizeBytes: item.sizeBytes,
+        compressedSizeBytes: item.compressedSizeBytes,
+        lastUploadedAt: now,
+      },
+    })
+    .returning({ id: vaultSessions.id });
+
+  await db
+    .insert(vaultSnapshots)
+    .values({
+      sessionId: session.id,
+      sha256: item.sha256,
+      objectKey,
+      sizeBytes: item.sizeBytes,
+      compressedSizeBytes: item.compressedSizeBytes,
+      uploadedAt: now,
+    })
+    .onConflictDoNothing({
+      target: [vaultSnapshots.sessionId, vaultSnapshots.sha256],
+    });
+
+  return session.id;
+}
+
+async function cleanupCommittedStagingGrants(
+  db: ReturnType<typeof cloudDb>,
+  grants: readonly { grantId: string; objectKey: string; uploadObjectKey: string }[],
+): Promise<void> {
+  for (const grant of grants) {
+    try {
+      await deleteObject(grant.uploadObjectKey);
+      await db.delete(vaultUploadGrants).where(and(
+        eq(vaultUploadGrants.id, grant.grantId),
+        eq(vaultUploadGrants.objectKey, grant.objectKey),
+        eq(vaultUploadGrants.uploadObjectKey, grant.uploadObjectKey),
+      ));
+    } catch {
+      // Keep the grant row so expired-grant GC can retry staging cleanup.
+    }
+  }
+}
+
+async function findCommittedSnapshot(
+  db: ReturnType<typeof cloudDb>,
+  userId: string,
+  item: {
+    readonly agent: string;
+    readonly agentSessionId: string;
+    readonly sha256: string;
+  },
+  objectKey: string,
+): Promise<{ readonly sessionId: string } | null> {
+  const [existing] = await db
+    .select({ sessionId: vaultSessions.id })
+    .from(vaultSessions)
+    .innerJoin(vaultSnapshots, eq(vaultSnapshots.sessionId, vaultSessions.id))
+    .where(and(
+      eq(vaultSessions.userId, userId),
+      eq(vaultSessions.agent, item.agent),
+      eq(vaultSessions.agentSessionId, item.agentSessionId),
+      eq(vaultSnapshots.sha256, item.sha256),
+      eq(vaultSnapshots.objectKey, objectKey),
+    ))
+    .limit(1);
+  return existing ?? null;
+}
+
 function itemResult(
   item: { agent: string; agentSessionId: string; relPath: string },
   status: string,
@@ -152,6 +350,19 @@ function itemResult(
   };
 }
 
+function committedResult(
+  item: { agent: string; agentSessionId: string; relPath: string },
+  sessionId: string,
+) {
+  return {
+    agent: item.agent,
+    agentSessionId: item.agentSessionId,
+    relPath: item.relPath,
+    status: "committed",
+    sessionId,
+  };
+}
+
 function sumBatchRawBytes(items: readonly { sizeBytes: number }[]): number {
   return items.reduce((total, item) => total + item.sizeBytes, 0);
 }

web/app/api/vault/uploads/route.ts

@@ -1,12 +1,24 @@
-import { and, eq, inArray, lt } from "drizzle-orm";
+import { randomUUID } from "node:crypto";
+import { and, eq, lt } from "drizzle-orm";
 import type { Span } from "@opentelemetry/api";
 import { cloudDb } from "../../../../db/client";
-import { vaultSessions, vaultSnapshots, vaultUploadGrants } from "../../../../db/schema";
+import {
+  vaultSessions,
+  vaultSnapshots,
+  vaultUploadGrants,
+  vaultUploadTombstones,
+} from "../../../../db/schema";
 import { vaultConfig } from "../../../../services/vault/config";
-import { buildObjectKey, deleteObject, presignPut } from "../../../../services/vault/storage";
+import {
+  buildObjectKey,
+  buildUploadObjectKey,
+  deleteObject,
+  presignPut,
+} from "../../../../services/vault/storage";
 import {
   getVaultPendingGrantBytes,
   getVaultStoredCompressedBytes,
+  withVaultUserQuotaLock,
 } from "../../../../services/vault/usage";
 import { withAuthedVaultApiRoute } from "../../../../services/vault/routeHelpers";
 import { readVaultJsonObject, validateVaultBatch } from "../../../../services/vault/validation";
@@ -23,6 +35,55 @@ export const dynamic = "force-dynamic";
 const UPLOAD_GRANT_TTL_MS = 24 * 60 * 60 * 1000;
 const GRANT_GC_BATCH = 10;
 
+type VaultDb = ReturnType<typeof cloudDb>;
+
+type ExistingUploadGrant = {
+  readonly id: string;
+  readonly uploadObjectKey: string;
+  readonly compressedSizeBytes: number;
+  readonly reservationToken: string;
+  readonly createdAt: Date;
+  readonly expiresAt: Date;
+};
+
+type VaultUploadItemBase = {
+  readonly agent: string;
+  readonly agentSessionId: string;
+  readonly relPath: string;
+};
+
+type ReservedUploadResult =
+  | (VaultUploadItemBase & {
+    readonly status: "error";
+    readonly error: string;
+  })
+  | (VaultUploadItemBase & {
+    readonly status: "unchanged";
+  })
+  | (VaultUploadItemBase & {
+    readonly status: "upload";
+    readonly grantId: string;
+    readonly grantReservationToken: string;
+    readonly previousGrant: ExistingUploadGrant | null;
+    readonly objectKey: string;
+    readonly uploadObjectKey: string;
+    readonly compressedSizeBytes: number;
+  });
+
+type VaultUploadResponseItem =
+  | (VaultUploadItemBase & {
+    readonly status: "error";
+    readonly error: string;
+  })
+  | (VaultUploadItemBase & {
+    readonly status: "unchanged";
+  })
+  | (VaultUploadItemBase & {
+    readonly status: "upload";
+    readonly objectKey: string;
+    readonly putUrl: string;
+  });
+
 export async function POST(request: Request): Promise<Response> {
   return withAuthedVaultApiRoute(
     request,
@@ -53,107 +114,171 @@ async function handlePost(request: Request, userId: string, span: Span): Promise
   const db = cloudDb();
   const now = new Date();
 
-  await gcExpiredGrants(db, now);
+  await gcExpiredVaultStorage(db, now);
 
-  // Per-user storage quota covers committed snapshots plus unexpired upload
-  // grants, so minting URLs and never committing still consumes quota (the
-  // presigned ContentLength is signed, bounding each upload to its declared
-  // size). Grants for keys in this batch are excluded from the pending sum and
-  // re-added per item below, so retries are not double-counted. The commit
-  // route re-checks, so previously issued URLs cannot bypass the quota either.
-  const batchObjectKeys = batch.value.map((item) =>
-    buildObjectKey(userId, item.agent, item.agentSessionId, item.sha256),
-  );
-  let projectedUserBytes =
-    (await getVaultStoredCompressedBytes(db, userId)) +
-    (await getVaultPendingGrantBytes(db, userId, now, batchObjectKeys));
-  const results = [];
-  for (const item of batch.value) {
-    // Per-item so one oversized transcript cannot block the rest of the batch.
-    if (item.compressedSizeBytes > config.maxUploadBytes) {
-      results.push({
-        agent: item.agent,
-        agentSessionId: item.agentSessionId,
-        relPath: item.relPath,
-        status: "error",
-        error: "upload_too_large",
-      });
-      continue;
-    }
-    if (projectedUserBytes + item.compressedSizeBytes > config.maxUserBytes) {
-      results.push({
-        agent: item.agent,
-        agentSessionId: item.agentSessionId,
-        relPath: item.relPath,
-        status: "error",
-        error: "quota_exceeded",
-      });
-      continue;
-    }
-
-    const [existing] = await db
-      .select({
-        id: vaultSessions.id,
-        latestSha256: vaultSessions.latestSha256,
-        relPath: vaultSessions.relPath,
-        cwd: vaultSessions.cwd,
-      })
-      .from(vaultSessions)
-      .where(
-        and(
-          eq(vaultSessions.userId, userId),
-          eq(vaultSessions.agent, item.agent),
-          eq(vaultSessions.agentSessionId, item.agentSessionId),
-        ),
-      )
-      .limit(1);
-
-    if (existing && existing.latestSha256 === item.sha256) {
-      // Same content can still move on disk (e.g. Codex archiving a session),
-      // so keep the restore metadata current even when no upload is needed.
-      if (existing.relPath !== item.relPath || existing.cwd !== item.cwd) {
-        await db
-          .update(vaultSessions)
-          .set({ relPath: item.relPath, cwd: item.cwd })
-          .where(eq(vaultSessions.id, existing.id));
+  const reservedResults = await withVaultUserQuotaLock(db, userId, async (lockedDb) => {
+    // Per-user storage quota covers committed snapshots plus unexpired upload
+    // grants, so minting URLs and never committing still consumes quota (the
+    // presigned ContentLength is signed, bounding each upload to its declared
+    // size). Grants for keys in this batch are excluded from the pending sum and
+    // re-added per item below, so retries are not double-counted. The commit
+    // route re-checks, so previously issued URLs cannot bypass the quota either.
+    const batchObjectKeys = [...new Set(batch.value.map((item) =>
+      buildObjectKey(userId, item.agent, item.agentSessionId, item.sha256),
+    ))];
+    let projectedUserBytes =
+      (await getVaultStoredCompressedBytes(lockedDb, userId)) +
+      (await getVaultPendingGrantBytes(lockedDb, userId, now, batchObjectKeys));
+    const lockedResults: ReservedUploadResult[] = [];
+    const objectKeysCreatedInRequest = new Set<string>();
+    const objectKeysSeenInRequest = new Set<string>();
+    for (const item of batch.value) {
+      const objectKey = buildObjectKey(userId, item.agent, item.agentSessionId, item.sha256);
+      if (objectKeysSeenInRequest.has(objectKey)) {
+        lockedResults.push({
+          agent: item.agent,
+          agentSessionId: item.agentSessionId,
+          relPath: item.relPath,
+          status: "error",
+          error: "duplicate_object_key",
+        });
+        continue;
       }
-      results.push({
+      objectKeysSeenInRequest.add(objectKey);
+
+      // Per-item so one oversized transcript cannot block the rest of the batch.
+      if (item.compressedSizeBytes > config.maxUploadBytes) {
+        lockedResults.push({
+          agent: item.agent,
+          agentSessionId: item.agentSessionId,
+          relPath: item.relPath,
+          status: "error",
+          error: "upload_too_large",
+        });
+        continue;
+      }
+      if (projectedUserBytes + item.compressedSizeBytes > config.maxUserBytes) {
+        lockedResults.push({
+          agent: item.agent,
+          agentSessionId: item.agentSessionId,
+          relPath: item.relPath,
+          status: "error",
+          error: "quota_exceeded",
+        });
+        continue;
+      }
+
+      const [existing] = await lockedDb
+        .select({
+          id: vaultSessions.id,
+          latestSha256: vaultSessions.latestSha256,
+          relPath: vaultSessions.relPath,
+          cwd: vaultSessions.cwd,
+        })
+        .from(vaultSessions)
+        .where(
+          and(
+            eq(vaultSessions.userId, userId),
+            eq(vaultSessions.agent, item.agent),
+            eq(vaultSessions.agentSessionId, item.agentSessionId),
+          ),
+        )
+        .limit(1);
+
+      if (existing && existing.latestSha256 === item.sha256) {
+        // Same content can still move on disk (e.g. Codex archiving a session),
+        // so keep the restore metadata current even when no upload is needed.
+        if (existing.relPath !== item.relPath || existing.cwd !== item.cwd) {
+          await lockedDb
+            .update(vaultSessions)
+            .set({ relPath: item.relPath, cwd: item.cwd })
+            .where(eq(vaultSessions.id, existing.id));
+        }
+        lockedResults.push({
+          agent: item.agent,
+          agentSessionId: item.agentSessionId,
+          relPath: item.relPath,
+          status: "unchanged",
+        });
+        continue;
+      }
+
+      const grantExpiresAt = new Date(now.getTime() + UPLOAD_GRANT_TTL_MS);
+      const [previousGrant] = await lockedDb
+        .select({
+          id: vaultUploadGrants.id,
+          uploadObjectKey: vaultUploadGrants.uploadObjectKey,
+          compressedSizeBytes: vaultUploadGrants.compressedSizeBytes,
+          reservationToken: vaultUploadGrants.reservationToken,
+          createdAt: vaultUploadGrants.createdAt,
+          expiresAt: vaultUploadGrants.expiresAt,
+        })
+        .from(vaultUploadGrants)
+        .where(eq(vaultUploadGrants.objectKey, objectKey))
+        .limit(1);
+      const grantReservationToken = randomUUID();
+      const uploadObjectKey = buildUploadObjectKey(objectKey, grantReservationToken);
+      if (previousGrant) {
+        await lockedDb
+          .insert(vaultUploadTombstones)
+          .values({
+            userId,
+            objectKey,
+            uploadObjectKey: previousGrant.uploadObjectKey,
+            expiresAt: previousGrant.expiresAt,
+          })
+          .onConflictDoUpdate({
+            target: vaultUploadTombstones.uploadObjectKey,
+            set: {
+              userId,
+              objectKey,
+              expiresAt: previousGrant.expiresAt,
+            },
+          });
+      }
+      const [grant] = await lockedDb
+        .insert(vaultUploadGrants)
+        .values({
+          userId,
+          objectKey,
+          uploadObjectKey,
+          compressedSizeBytes: item.compressedSizeBytes,
+          reservationToken: grantReservationToken,
+          createdAt: now,
+          expiresAt: grantExpiresAt,
+        })
+        .onConflictDoUpdate({
+          target: vaultUploadGrants.objectKey,
+          set: {
+            uploadObjectKey,
+            compressedSizeBytes: item.compressedSizeBytes,
+            reservationToken: grantReservationToken,
+            createdAt: now,
+            expiresAt: grantExpiresAt,
+          },
+        })
+        .returning({ id: vaultUploadGrants.id });
+      if (!grant) throw new Error("vault upload grant upsert returned no row");
+      if (!previousGrant) objectKeysCreatedInRequest.add(objectKey);
+      projectedUserBytes += item.compressedSizeBytes;
+      lockedResults.push({
         agent: item.agent,
         agentSessionId: item.agentSessionId,
         relPath: item.relPath,
-        status: "unchanged",
-      });
-      continue;
-    }
-
-    const objectKey = buildObjectKey(userId, item.agent, item.agentSessionId, item.sha256);
-    await db
-      .insert(vaultUploadGrants)
-      .values({
-        userId,
+        status: "upload",
+        grantId: grant.id,
+        grantReservationToken,
+        previousGrant: previousGrant && !objectKeysCreatedInRequest.has(objectKey)
+          ? previousGrant
+          : null,
         objectKey,
+        uploadObjectKey,
         compressedSizeBytes: item.compressedSizeBytes,
-        createdAt: now,
-        expiresAt: new Date(now.getTime() + UPLOAD_GRANT_TTL_MS),
-      })
-      .onConflictDoUpdate({
-        target: vaultUploadGrants.objectKey,
-        set: {
-          compressedSizeBytes: item.compressedSizeBytes,
-          createdAt: now,
-          expiresAt: new Date(now.getTime() + UPLOAD_GRANT_TTL_MS),
-        },
       });
-    projectedUserBytes += item.compressedSizeBytes;
-    results.push({
-      agent: item.agent,
-      agentSessionId: item.agentSessionId,
-      relPath: item.relPath,
-      status: "upload",
-      objectKey,
-      putUrl: await presignPut(objectKey, item.compressedSizeBytes),
-    });
-  }
+    }
+    return lockedResults;
+  });
+  const results = await presignReservedUploads(db, reservedResults);
   setSpanAttributes(span, {
     "cmux.vault.result_count": results.length,
     "cmux.vault.result.upload_count": countResultStatus(results, "upload"),
@@ -163,44 +288,227 @@ async function handlePost(request: Request, userId: string, span: Span): Promise
   return jsonResponse({ items: results });
 }
 
+async function presignReservedUploads(
+  db: VaultDb,
+  items: readonly ReservedUploadResult[],
+): Promise<VaultUploadResponseItem[]> {
+  const results: VaultUploadResponseItem[] = [];
+  const successfulObjectKeys = new Set<string>();
+  const failedReservations = new Map<string, Extract<ReservedUploadResult, { status: "upload" }>>();
+  for (const item of items) {
+    if (item.status !== "upload") {
+      results.push(item);
+      continue;
+    }
+    try {
+      results.push({
+        agent: item.agent,
+        agentSessionId: item.agentSessionId,
+        relPath: item.relPath,
+        status: "upload",
+        objectKey: item.objectKey,
+        putUrl: await presignPut(item.uploadObjectKey, item.compressedSizeBytes),
+      });
+      successfulObjectKeys.add(item.objectKey);
+    } catch {
+      failedReservations.set(item.objectKey, item);
+      results.push({
+        agent: item.agent,
+        agentSessionId: item.agentSessionId,
+        relPath: item.relPath,
+        status: "error",
+        error: "upload_presign_failed",
+      });
+    }
+  }
+  for (const item of failedReservations.values()) {
+    if (successfulObjectKeys.has(item.objectKey)) continue;
+    if (item.previousGrant) {
+      const restored = await db
+        .update(vaultUploadGrants)
+        .set({
+          compressedSizeBytes: item.previousGrant.compressedSizeBytes,
+          uploadObjectKey: item.previousGrant.uploadObjectKey,
+          reservationToken: item.previousGrant.reservationToken,
+          createdAt: item.previousGrant.createdAt,
+          expiresAt: item.previousGrant.expiresAt,
+        })
+        .where(and(
+          eq(vaultUploadGrants.id, item.previousGrant.id),
+          eq(vaultUploadGrants.objectKey, item.objectKey),
+          eq(vaultUploadGrants.reservationToken, item.grantReservationToken),
+        ))
+        .returning({ id: vaultUploadGrants.id })
+        .catch(() => []);
+      if (restored.length > 0) {
+        await db
+          .delete(vaultUploadTombstones)
+          .where(eq(vaultUploadTombstones.uploadObjectKey, item.previousGrant.uploadObjectKey))
+          .catch(() => undefined);
+      }
+      continue;
+    }
+    await db
+      .delete(vaultUploadGrants)
+      .where(and(
+        eq(vaultUploadGrants.id, item.grantId),
+        eq(vaultUploadGrants.objectKey, item.objectKey),
+        eq(vaultUploadGrants.reservationToken, item.grantReservationToken),
+      ))
+      .catch(() => undefined);
+  }
+  return results;
+}
+
 /**
- * Opportunistically clean up expired grants: delete the storage object when it
- * was uploaded but never committed, then drop the grant row. Runs a small
- * bounded batch per request (same pattern as the CLI auth start GC). If the
- * object deletion fails, the row is kept so a later pass retries.
+ * Opportunistically clean up expired grants and superseded upload keys across
+ * users. Each row is re-read under its owner's quota lock before storage
+ * deletion, so a global sweep cannot delete a newer active reservation.
  */
-async function gcExpiredGrants(db: ReturnType<typeof cloudDb>, now: Date): Promise<void> {
-  const expired = await db
+async function gcExpiredVaultStorage(
+  db: VaultDb,
+  now: Date,
+): Promise<void> {
+  const expiredGrants = await db
     .select({
       id: vaultUploadGrants.id,
+      userId: vaultUploadGrants.userId,
       objectKey: vaultUploadGrants.objectKey,
+      uploadObjectKey: vaultUploadGrants.uploadObjectKey,
+      expiresAt: vaultUploadGrants.expiresAt,
     })
     .from(vaultUploadGrants)
     .where(lt(vaultUploadGrants.expiresAt, now))
     .limit(GRANT_GC_BATCH);
-  if (expired.length === 0) return;
-
-  const committedRows = await db
-    .select({ objectKey: vaultSnapshots.objectKey })
-    .from(vaultSnapshots)
-    .where(
-      inArray(
-        vaultSnapshots.objectKey,
-        expired.map((grant) => grant.objectKey),
-      ),
-    );
-  const committedKeys = new Set(committedRows.map((row) => row.objectKey));
-
-  for (const grant of expired) {
-    if (!committedKeys.has(grant.objectKey)) {
-      try {
-        await deleteObject(grant.objectKey);
-      } catch (error) {
-        continue;
-      }
-    }
-    await db.delete(vaultUploadGrants).where(eq(vaultUploadGrants.id, grant.id));
+  for (const grant of expiredGrants) {
+    await withVaultUserQuotaLock(db, grant.userId, async (lockedDb) => {
+      await cleanupExpiredGrant(lockedDb, grant, now);
+    });
   }
+
+  const expiredTombstones = await db
+    .select({
+      id: vaultUploadTombstones.id,
+      userId: vaultUploadTombstones.userId,
+      objectKey: vaultUploadTombstones.objectKey,
+      uploadObjectKey: vaultUploadTombstones.uploadObjectKey,
+      expiresAt: vaultUploadTombstones.expiresAt,
+    })
+    .from(vaultUploadTombstones)
+    .where(lt(vaultUploadTombstones.expiresAt, now))
+    .limit(GRANT_GC_BATCH);
+  for (const tombstone of expiredTombstones) {
+    await withVaultUserQuotaLock(db, tombstone.userId, async (lockedDb) => {
+      await cleanupExpiredTombstone(lockedDb, tombstone, now);
+    });
+  }
+}
+
+async function cleanupExpiredGrant(
+  db: VaultDb,
+  grant: {
+    readonly id: string;
+    readonly userId: string;
+    readonly objectKey: string;
+    readonly uploadObjectKey: string;
+    readonly expiresAt: Date;
+  },
+  now: Date,
+): Promise<void> {
+  const [currentGrant] = await db
+    .select({
+      id: vaultUploadGrants.id,
+      userId: vaultUploadGrants.userId,
+      objectKey: vaultUploadGrants.objectKey,
+      uploadObjectKey: vaultUploadGrants.uploadObjectKey,
+      expiresAt: vaultUploadGrants.expiresAt,
+    })
+    .from(vaultUploadGrants)
+    .where(and(
+      eq(vaultUploadGrants.id, grant.id),
+      eq(vaultUploadGrants.userId, grant.userId),
+      eq(vaultUploadGrants.objectKey, grant.objectKey),
+      eq(vaultUploadGrants.uploadObjectKey, grant.uploadObjectKey),
+      eq(vaultUploadGrants.expiresAt, grant.expiresAt),
+      lt(vaultUploadGrants.expiresAt, now),
+    ))
+    .limit(1);
+  if (!currentGrant) return;
+
+  try {
+    if (currentGrant.uploadObjectKey !== currentGrant.objectKey) {
+      await deleteObject(currentGrant.uploadObjectKey);
+    }
+    const [committed] = await db
+      .select({ objectKey: vaultSnapshots.objectKey })
+      .from(vaultSnapshots)
+      .where(eq(vaultSnapshots.objectKey, currentGrant.objectKey))
+      .limit(1);
+    if (!committed) await deleteObject(currentGrant.objectKey);
+  } catch {
+    return;
+  }
+  await db.delete(vaultUploadGrants).where(and(
+    eq(vaultUploadGrants.id, currentGrant.id),
+    eq(vaultUploadGrants.userId, currentGrant.userId),
+    eq(vaultUploadGrants.objectKey, currentGrant.objectKey),
+    eq(vaultUploadGrants.uploadObjectKey, currentGrant.uploadObjectKey),
+    eq(vaultUploadGrants.expiresAt, currentGrant.expiresAt),
+  ));
+}
+
+async function cleanupExpiredTombstone(
+  db: VaultDb,
+  tombstone: {
+    readonly id: string;
+    readonly userId: string;
+    readonly objectKey: string;
+    readonly uploadObjectKey: string;
+    readonly expiresAt: Date;
+  },
+  now: Date,
+): Promise<void> {
+  const [currentTombstone] = await db
+    .select({
+      id: vaultUploadTombstones.id,
+      userId: vaultUploadTombstones.userId,
+      objectKey: vaultUploadTombstones.objectKey,
+      uploadObjectKey: vaultUploadTombstones.uploadObjectKey,
+      expiresAt: vaultUploadTombstones.expiresAt,
+    })
+    .from(vaultUploadTombstones)
+    .where(and(
+      eq(vaultUploadTombstones.id, tombstone.id),
+      eq(vaultUploadTombstones.userId, tombstone.userId),
+      eq(vaultUploadTombstones.objectKey, tombstone.objectKey),
+      eq(vaultUploadTombstones.uploadObjectKey, tombstone.uploadObjectKey),
+      eq(vaultUploadTombstones.expiresAt, tombstone.expiresAt),
+      lt(vaultUploadTombstones.expiresAt, now),
+    ))
+    .limit(1);
+  if (!currentTombstone) return;
+
+  try {
+    if (currentTombstone.uploadObjectKey === currentTombstone.objectKey) {
+      const [committed] = await db
+        .select({ objectKey: vaultSnapshots.objectKey })
+        .from(vaultSnapshots)
+        .where(eq(vaultSnapshots.objectKey, currentTombstone.objectKey))
+        .limit(1);
+      if (!committed) await deleteObject(currentTombstone.objectKey);
+    } else {
+      await deleteObject(currentTombstone.uploadObjectKey);
+    }
+  } catch {
+    return;
+  }
+  await db.delete(vaultUploadTombstones).where(and(
+    eq(vaultUploadTombstones.id, currentTombstone.id),
+    eq(vaultUploadTombstones.userId, currentTombstone.userId),
+    eq(vaultUploadTombstones.objectKey, currentTombstone.objectKey),
+    eq(vaultUploadTombstones.uploadObjectKey, currentTombstone.uploadObjectKey),
+    eq(vaultUploadTombstones.expiresAt, currentTombstone.expiresAt),
+  ));
 }
 
 function sumBatchRawBytes(items: readonly { sizeBytes: number }[]): number {

web/app/api/vm/[id]/attach-endpoint/route.ts

@@ -44,6 +44,7 @@ export async function POST(
         const endpoint = await runVmWorkflow(openAttachEndpoint({
           userId: user.id,
           billingTeamId: account.entitlements.billingTeamId,
+          teamIds: user.teamIds,
           providerVmId: id,
           sessionTitle,
           options: { requireDaemon, sessionId, attachmentId },

web/app/api/vm/[id]/exec/route.ts

@@ -72,6 +72,7 @@ export async function POST(
         const result = await runVmWorkflow(execVm({
           userId: user.id,
           billingTeamId: account.entitlements.billingTeamId,
+          teamIds: user.teamIds,
           providerVmId: id,
           command,
           timeoutMs,

web/app/api/vm/[id]/fork/route.ts

@@ -74,6 +74,7 @@ export async function POST(
           userId: user.id,
           billingCustomerType: entitlements.billingCustomerType,
           billingTeamId: entitlements.billingTeamId,
+          teamIds: user.teamIds,
           billingPlanId: entitlements.planId,
           maxActiveVms: entitlements.maxActiveVms,
           providerVmId: id,

web/app/api/vm/[id]/route.ts

@@ -28,6 +28,7 @@ export async function GET(
         const vm = await runVmWorkflow(getVm({
           userId: user.id,
           billingTeamId: account.entitlements.billingTeamId,
+          teamIds: user.teamIds,
           providerVmId: id,
         }));
         return jsonResponse({
@@ -64,6 +65,7 @@ export async function DELETE(
         await runVmWorkflow(destroyVm({
           userId: user.id,
           billingTeamId: account.entitlements.billingTeamId,
+          teamIds: user.teamIds,
           providerVmId: id,
         }));
       } catch (err) {

web/app/api/vm/[id]/sessions/route.ts

@@ -33,6 +33,7 @@ export async function GET(
         const sessions = await runVmWorkflow(listVmSessions({
           userId: user.id,
           billingTeamId: account.entitlements.billingTeamId,
+          teamIds: user.teamIds,
           providerVmId: id,
         }));
         return jsonResponse({ sessions: sessions.map(sessionPayload) });
@@ -76,6 +77,7 @@ export async function POST(
         const result = await runVmWorkflow(openVmSession({
           userId: user.id,
           billingTeamId: account.entitlements.billingTeamId,
+          teamIds: user.teamIds,
           providerVmId: id,
           sessionId,
           attachmentId,

web/app/api/vm/[id]/snapshot/route.ts

@@ -33,6 +33,7 @@ export async function POST(
         const snapshot = await runVmWorkflow(snapshotVm({
           userId: user.id,
           billingTeamId: account.entitlements.billingTeamId,
+          teamIds: user.teamIds,
           providerVmId: id,
           name,
         }));

web/app/api/vm/[id]/ssh-endpoint/route.ts

@@ -40,6 +40,7 @@ export async function POST(
         const endpoint = await runVmWorkflow(openSshEndpoint({
           userId: user.id,
           billingTeamId: account.entitlements.billingTeamId,
+          teamIds: user.teamIds,
           providerVmId: id,
         }));
         setSpanAttributes(span, { "cmux.ssh.credential_kind": endpoint.credential.kind });

web/app/api/waitlist/route.ts

@@ -66,6 +66,13 @@ export async function POST(request: Request) {
         if (rateLimited || error === "blocked") {
           return jsonError("Rate limit exceeded", 429);
         }
+        if (error === "not-found") {
+          console.error("waitlist.route.rate_limit_not_found", env.CMUX_FEEDBACK_RATE_LIMIT_ID);
+          return jsonError("service_unavailable", 503);
+        } else if (error) {
+          console.error("waitlist.route.rate_limit_error", error);
+          return jsonError("service_unavailable", 503);
+        }
       }
 
       let payload: unknown;

web/db/migrations/20260708052000_vault_upload_reservation_token/migration.sql

@@ -0,0 +1 @@
+ALTER TABLE "vault_upload_grants" ADD COLUMN "reservation_token" uuid DEFAULT gen_random_uuid() NOT NULL;

web/db/migrations/20260708054000_cloud_vm_identity_cleanup_index/migration.sql

@@ -0,0 +1 @@
+CREATE INDEX "cloud_vm_leases_identity_cleanup_idx" ON "cloud_vm_leases" ("expires_at","created_at","id") WHERE "provider_identity_handle" is not null and "revoked_at" is null;

web/db/migrations/20260708060000_vault_upload_staging_key/migration.sql

@@ -0,0 +1,3 @@
+ALTER TABLE "vault_upload_grants" ADD COLUMN "upload_object_key" text;--> statement-breakpoint
+UPDATE "vault_upload_grants" SET "upload_object_key" = "object_key";--> statement-breakpoint
+ALTER TABLE "vault_upload_grants" ALTER COLUMN "upload_object_key" SET NOT NULL;

web/db/migrations/20260708062000_vault_upload_tombstones/migration.sql

@@ -0,0 +1,14 @@
+CREATE TABLE "vault_upload_tombstones" (
+  "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+  "user_id" text NOT NULL,
+  "object_key" text NOT NULL,
+  "upload_object_key" text NOT NULL,
+  "created_at" timestamp with time zone DEFAULT now() NOT NULL,
+  "expires_at" timestamp with time zone NOT NULL
+);
+--> statement-breakpoint
+CREATE INDEX "vault_upload_tombstones_user_idx" ON "vault_upload_tombstones" ("user_id");
+--> statement-breakpoint
+CREATE INDEX "vault_upload_tombstones_expires_idx" ON "vault_upload_tombstones" ("expires_at");
+--> statement-breakpoint
+CREATE UNIQUE INDEX "vault_upload_tombstones_upload_object_key_unique" ON "vault_upload_tombstones" ("upload_object_key");

web/db/schema.ts

@@ -101,6 +101,9 @@ export const cloudVmLeases = pgTable(
   (table) => [
     index("cloud_vm_leases_vm_kind_idx").on(table.vmId, table.kind),
     index("cloud_vm_leases_identity_idx").on(table.providerIdentityHandle),
+    index("cloud_vm_leases_identity_cleanup_idx")
+      .on(table.expiresAt, table.createdAt, table.id)
+      .where(sql`${table.providerIdentityHandle} is not null and ${table.revokedAt} is null`),
     index("cloud_vm_leases_user_expires_idx").on(table.userId, table.expiresAt),
     uniqueIndex("cloud_vm_leases_token_hash_unique").on(table.tokenHash),
   ],
@@ -424,7 +427,9 @@ export const vaultUploadGrants = pgTable(
     id: uuid("id").defaultRandom().primaryKey(),
     userId: text("user_id").notNull(),
     objectKey: text("object_key").notNull(),
+    uploadObjectKey: text("upload_object_key").notNull(),
     compressedSizeBytes: bigint("compressed_size_bytes", { mode: "number" }).notNull(),
+    reservationToken: uuid("reservation_token").defaultRandom().notNull(),
     createdAt: timestamp("created_at", { withTimezone: true }).notNull(),
     expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(),
   },
@@ -435,6 +440,23 @@ export const vaultUploadGrants = pgTable(
   ],
 );
 
+export const vaultUploadTombstones = pgTable(
+  "vault_upload_tombstones",
+  {
+    id: uuid("id").defaultRandom().primaryKey(),
+    userId: text("user_id").notNull(),
+    objectKey: text("object_key").notNull(),
+    uploadObjectKey: text("upload_object_key").notNull(),
+    createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
+    expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(),
+  },
+  (table) => [
+    index("vault_upload_tombstones_user_idx").on(table.userId),
+    index("vault_upload_tombstones_expires_idx").on(table.expiresAt),
+    uniqueIndex("vault_upload_tombstones_upload_object_key_unique").on(table.uploadObjectKey),
+  ],
+);
+
 export const vaultCliAuthRequests = pgTable(
   "vault_cli_auth_requests",
   {

web/scripts/run-db-behavior-tests.sh

@@ -29,7 +29,7 @@ zero_test_files=()
 skipped_test_files=()
 for test_file in "${test_files[@]}"; do
   printf '\n==> bun test %s\n' "$test_file"
-  output_file="$(mktemp /tmp/cmux-db-behavior-test.XXXXXX.log)"
+  output_file="$(mktemp /tmp/cmux-db-behavior-test.XXXXXX)"
   set +e
   bun test "$test_file" 2>&1 | tee "$output_file"
   test_status=${PIPESTATUS[0]}

web/services/vault/routeHelpers.ts

@@ -11,7 +11,10 @@ import {
   verifyRequest,
   type AuthedUser,
 } from "@/services/vms/auth";
-import { jsonResponse } from "@/services/vms/routeHelpers";
+import {
+  enforceBrowserMutationProtection,
+  jsonResponse,
+} from "@/services/vms/routeHelpers";
 
 type VerifyRequestOptions = NonNullable<Parameters<typeof verifyRequest>[1]>;
 
@@ -62,6 +65,8 @@ export async function withAuthedVaultApiRoute(
   return withVaultApiRoute(request, route, attributes, failureLog, async (context) => {
     const user = await verify(request, verifyOptions);
     if (!user) return unauthorized();
+    const mutationForbidden = enforceBrowserMutationProtection(request);
+    if (mutationForbidden) return mutationForbidden;
     setSpanAttributes(context.span, { "cmux.vault.user_id": user.id });
     return handler({ ...context, user });
   });

web/services/vault/storage.ts

@@ -1,5 +1,6 @@
 import {
   S3Client,
+  CopyObjectCommand,
   DeleteObjectCommand,
   GetObjectCommand,
   HeadObjectCommand,
@@ -35,6 +36,10 @@ export function buildObjectKey(userId: string, agent: string, agentSessionId: st
   ].join("/");
 }
 
+export function buildUploadObjectKey(objectKey: string, reservationToken: string): string {
+  return ["vault", "uploads", keyPart(reservationToken), objectKey].join("/");
+}
+
 function s3Client(): S3Client {
   const config = vaultConfig();
   const key = JSON.stringify({
@@ -108,6 +113,23 @@ export async function deleteObject(key: string): Promise<void> {
   }
 }
 
+export async function copyObject(sourceKey: string, destinationKey: string): Promise<void> {
+  const config = vaultConfig();
+  if (!config.bucket) throw new Error("CMUX_VAULT_S3_BUCKET is required");
+  const copySourceKey = encodeURIComponent(sourceKey).replace(/%2F/g, "/");
+  try {
+    await s3Client().send(new CopyObjectCommand({
+      Bucket: config.bucket,
+      Key: destinationKey,
+      CopySource: `${config.bucket}/${copySourceKey}`,
+      ContentType: "application/zstd",
+    }));
+  } catch (error) {
+    logVaultStorageError("copy_object", destinationKey, error);
+    throw error;
+  }
+}
+
 export async function headObject(key: string): Promise<HeadObjectResult | null> {
   const config = vaultConfig();
   if (!config.bucket) throw new Error("CMUX_VAULT_S3_BUCKET is required");

web/services/vault/usage.ts

@@ -4,6 +4,19 @@ import { vaultSessions, vaultSnapshots, vaultUploadGrants } from "../../db/schem
 import { logVaultQuotaError } from "./logging";
 
 type VaultDb = ReturnType<typeof cloudDb>;
+const VAULT_QUOTA_LOCK_NAMESPACE = 9;
+
+export async function withVaultUserQuotaLock<T>(
+  db: VaultDb,
+  userId: string,
+  run: (db: VaultDb) => Promise<T>,
+): Promise<T> {
+  return await db.transaction(async (tx) => {
+    await tx.execute(sql`set local lock_timeout = '5s'`);
+    await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${userId}, ${VAULT_QUOTA_LOCK_NAMESPACE}))`);
+    return await run(tx as unknown as VaultDb);
+  });
+}
 
 /**
  * Total compressed bytes a user currently has stored across all snapshots.
@@ -60,7 +73,13 @@ export async function getVaultPendingGrantBytes(
         total: sql<number>`coalesce(sum(${vaultUploadGrants.compressedSizeBytes}), 0)::double precision`,
       })
       .from(vaultUploadGrants)
-      .where(and(...conditions));
+      .where(and(
+        ...conditions,
+        sql`not exists (
+          select 1 from ${vaultSnapshots}
+          where ${vaultSnapshots.objectKey} = ${vaultUploadGrants.objectKey}
+        )`,
+      ));
     return row?.total ?? 0;
   } catch (error) {
     logVaultQuotaError("get_pending_grant_bytes", error);

web/services/vms/drivers/freestyle.ts

@@ -18,6 +18,7 @@ import {
   setSpanAttributes,
   withVmSpan,
 } from "../telemetry";
+import { isProviderIdentityNotFoundError } from "../providerErrors";
 import {
   isReusableRpcLease,
   ensurePrivateDirectoryCommand,
@@ -625,9 +626,9 @@ export class FreestyleProvider implements VMProvider {
         try {
           await client().identities.delete({ identityId: identityHandle });
         } catch (err) {
-          // Best effort: identity may already be gone (e.g. VM was destroyed by the provider
-          // itself). Don't let cleanup failures cascade into the caller, but keep it visible.
+          if (isProviderIdentityNotFoundError(err)) return;
           recordSpanError(span, err);
+          throw new ProviderError("freestyle", `revokeSSHIdentity(${identityHandle})`, err);
         }
       },
     );
@@ -792,10 +793,13 @@ async function ensureFreestyleWebSocketHealthyOrRepair(
   }
 
   if (!healthError && canRepair) {
-    const state = await readFreestyleCloudShellState(vm).catch((err: unknown) => ({
-      ok: false,
-      reason: errorMessage(err),
-    }));
+    const state = await readFreestyleCloudShellState(vm).catch(() => null);
+    if (!state) {
+      // The daemon/admin websocket is the attach control plane. Freestyle exec
+      // can be temporarily unavailable on otherwise attachable VMs, so do not
+      // block lease installation on this shell-integration probe.
+      return;
+    }
     if (!state.ok) {
       await repairFreestyleWebSocketService(vm, adminToken, signedAdmin).catch((repairErr: unknown) => {
         throw new Error(

web/services/vms/providerErrors.ts

@@ -1,17 +1,22 @@
 const providerSubjectPattern =
   "(?:vm|virtual machine|sandbox|sandboxes|instance|container|machine|environment|resource)";
+const providerIdentitySubjectPattern =
+  "(?:identity|identities|credential|credentials)";
 const providerMissingPattern =
   "(?:not found|does not exist|already deleted|has been deleted|was deleted|marked as deleted|no such)";
 
-function hasProviderMissingMessage(message: string): boolean {
+function hasProviderMissingMessage(
+  message: string,
+  subjectPattern: string = providerSubjectPattern,
+): boolean {
   const normalized = message.toLowerCase();
   if (!normalized) return false;
 
   const subjectThenMissing = new RegExp(
-    `\\b${providerSubjectPattern}\\b.{0,80}\\b${providerMissingPattern}\\b`,
+    `\\b${subjectPattern}\\b.{0,80}\\b${providerMissingPattern}\\b`,
   );
   const missingThenSubject = new RegExp(
-    `\\b${providerMissingPattern}\\b.{0,80}\\b${providerSubjectPattern}\\b`,
+    `\\b${providerMissingPattern}\\b.{0,80}\\b${subjectPattern}\\b`,
   );
   if (subjectThenMissing.test(normalized) || missingThenSubject.test(normalized)) {
     return true;
@@ -19,7 +24,7 @@ function hasProviderMissingMessage(message: string): boolean {
 
   return (
     /(^|[^0-9])404([^0-9]|$)/.test(normalized) &&
-    /\b(not found|vm|sandbox|instance|container|machine|resource)\b/.test(normalized)
+    new RegExp(`\\b(not found|${subjectPattern})\\b`).test(normalized)
   );
 }
 
@@ -67,3 +72,47 @@ export function isProviderNotFoundError(err: unknown): boolean {
   if (candidate.cause) return isProviderNotFoundError(candidate.cause);
   return false;
 }
+
+export function isProviderIdentityNotFoundError(err: unknown): boolean {
+  if (!err || typeof err !== "object") return false;
+  const candidate = err as {
+    code?: string | number;
+    name?: string;
+    status?: number;
+    statusCode?: number;
+    response?: { status?: number; data?: unknown };
+    message?: string;
+    cause?: unknown;
+  };
+  const status =
+    candidate.status ??
+    candidate.statusCode ??
+    candidate.response?.status ??
+    undefined;
+  if (status === 404) return true;
+
+  const code = String(candidate.code ?? candidate.name ?? "").toLowerCase();
+  if (
+    code === "not_found" ||
+    code === "notfound" ||
+    code === "404"
+  ) {
+    return true;
+  }
+
+  if (hasProviderMissingMessage(candidate.message ?? "", providerIdentitySubjectPattern)) return true;
+
+  const responseData = candidate.response?.data;
+  if (
+    (typeof responseData === "string" &&
+      hasProviderMissingMessage(responseData, providerIdentitySubjectPattern)) ||
+    (responseData &&
+      typeof responseData === "object" &&
+      hasProviderMissingMessage(JSON.stringify(responseData), providerIdentitySubjectPattern))
+  ) {
+    return true;
+  }
+
+  if (candidate.cause) return isProviderIdentityNotFoundError(candidate.cause);
+  return false;
+}

web/services/vms/repository.ts

@@ -1,4 +1,4 @@
-import { and, asc, count, desc, eq, inArray, isNotNull, isNull, ne, or, sql } from "drizzle-orm";
+import { and, asc, count, desc, eq, inArray, isNotNull, isNull, lt, ne, or, sql } from "drizzle-orm";
 import * as Context from "effect/Context";
 import * as Effect from "effect/Effect";
 import * as Layer from "effect/Layer";
@@ -20,6 +20,9 @@ export type CloudVmRow = typeof cloudVms.$inferSelect;
 export type CloudVmBaseRow = typeof cloudVmBases.$inferSelect;
 export type CloudVmBaseGenerationRow = typeof cloudVmBaseGenerations.$inferSelect;
 export type CloudVmLeaseRow = typeof cloudVmLeases.$inferSelect;
+export type CloudVmIdentityLeaseRow = CloudVmLeaseRow & {
+  readonly provider: ProviderId;
+};
 export type CloudVmSessionRow = typeof cloudVmSessions.$inferSelect;
 export type CloudVmLeaseKind = typeof cloudVmLeases.$inferInsert.kind;
 export type CloudVmStatus = CloudVmRow["status"];
@@ -166,6 +169,15 @@ export type VmRepositoryShape = {
     readonly transport?: string;
     readonly metadata?: Record<string, unknown>;
   }) => Effect.Effect<void, VmDatabaseError>;
+  readonly expiredIdentityLeases?: (input: {
+    readonly now: Date;
+    readonly limit: number;
+  }) => Effect.Effect<CloudVmIdentityLeaseRow[], VmDatabaseError>;
+  readonly markLeaseRevocationRetry?: (input: {
+    readonly id: string;
+    readonly retryAfter: Date;
+    readonly error: string;
+  }) => Effect.Effect<void, VmDatabaseError>;
   readonly listVmSessions: (input: {
     readonly userId: string;
     readonly vmId: string;
@@ -184,7 +196,7 @@ export type VmRepositoryShape = {
     readonly scrollbackBytes?: number;
     readonly metadata?: Record<string, unknown>;
   }) => Effect.Effect<CloudVmSessionRow, VmDatabaseError>;
-  readonly activeIdentityLeases: (vmId: string) => Effect.Effect<CloudVmLeaseRow[], VmDatabaseError>;
+  readonly activeIdentityLeases: (vmId: string, limit?: number) => Effect.Effect<CloudVmLeaseRow[], VmDatabaseError>;
   readonly markLeasesRevoked: (ids: readonly string[]) => Effect.Effect<void, VmDatabaseError>;
   readonly recordUsageEvent: (input: {
     readonly userId: string;
@@ -1209,6 +1221,71 @@ export const VmRepositoryLive = Layer.succeed(VmRepository, {
       }
     }),
 
+  expiredIdentityLeases: (input) =>
+    dbEffect("expiredIdentityLeases", async () => {
+      const db = cloudDb();
+      return await db
+        .select({
+          id: cloudVmLeases.id,
+          vmId: cloudVmLeases.vmId,
+          userId: cloudVmLeases.userId,
+          kind: cloudVmLeases.kind,
+          tokenHash: cloudVmLeases.tokenHash,
+          providerIdentityHandle: cloudVmLeases.providerIdentityHandle,
+          sessionId: cloudVmLeases.sessionId,
+          transport: cloudVmLeases.transport,
+          metadata: cloudVmLeases.metadata,
+          expiresAt: cloudVmLeases.expiresAt,
+          consumedAt: cloudVmLeases.consumedAt,
+          revokedAt: cloudVmLeases.revokedAt,
+          createdAt: cloudVmLeases.createdAt,
+          provider: cloudVms.provider,
+        })
+        .from(cloudVmLeases)
+        .innerJoin(cloudVms, eq(cloudVmLeases.vmId, cloudVms.id))
+        .where(
+          and(
+            isNotNull(cloudVmLeases.providerIdentityHandle),
+            isNull(cloudVmLeases.revokedAt),
+            lt(cloudVmLeases.expiresAt, input.now),
+            or(
+              sql`${cloudVmLeases.metadata}->>'identityCleanupRetryAfter' is null`,
+              sql`(${cloudVmLeases.metadata}->>'identityCleanupRetryAfter')::timestamptz <= ${input.now.toISOString()}::timestamptz`,
+            ),
+          ),
+        )
+        .orderBy(asc(cloudVmLeases.expiresAt), asc(cloudVmLeases.createdAt), asc(cloudVmLeases.id))
+        .limit(input.limit);
+    }),
+
+  markLeaseRevocationRetry: (input) =>
+    dbEffect("markLeaseRevocationRetry", async () => {
+      const db = cloudDb();
+      await db
+        .update(cloudVmLeases)
+        .set({
+          metadata: sql<Record<string, unknown>>`
+            jsonb_set(
+              jsonb_set(
+                jsonb_set(
+                  ${cloudVmLeases.metadata},
+                  '{identityCleanupRetryAfter}',
+                  to_jsonb(${input.retryAfter.toISOString()}::text),
+                  true
+                ),
+                '{identityCleanupAttempts}',
+                to_jsonb((coalesce((${cloudVmLeases.metadata}->>'identityCleanupAttempts')::int, 0) + 1)),
+                true
+              ),
+              '{identityCleanupLastError}',
+              to_jsonb(${input.error.slice(0, 240)}::text),
+              true
+            )
+          `,
+        })
+        .where(eq(cloudVmLeases.id, input.id));
+    }),
+
   listVmSessions: (input) =>
     dbEffect("listVmSessions", async () => {
       const db = cloudDb();
@@ -1267,10 +1344,10 @@ export const VmRepositoryLive = Layer.succeed(VmRepository, {
       return session;
     }),
 
-  activeIdentityLeases: (vmId) =>
+  activeIdentityLeases: (vmId, limit) =>
     dbEffect("activeIdentityLeases", async () => {
       const db = cloudDb();
-      return await db
+      const query = db
         .select()
         .from(cloudVmLeases)
         .where(
@@ -1279,7 +1356,11 @@ export const VmRepositoryLive = Layer.succeed(VmRepository, {
             isNotNull(cloudVmLeases.providerIdentityHandle),
             isNull(cloudVmLeases.revokedAt),
           ),
-        );
+        )
+        .orderBy(desc(cloudVmLeases.createdAt));
+      return typeof limit === "number" && limit > 0
+        ? await query.limit(limit)
+        : await query;
     }),
 
   markLeasesRevoked: (ids) =>

web/services/vms/routeHelpers.ts

@@ -70,9 +70,8 @@ export async function withAuthedVmApiRoute(
         const authDurationMs = performance.now() - authStart;
         recordSpanTiming(span, "auth", authDurationMs);
         if (!user) return unauthorized();
-        if (requiresBrowserMutationProtection(request.method, bearer) && !browserMutationOriginAllowed(request)) {
-          return jsonResponse({ error: "forbidden" }, 403);
-        }
+        const mutationForbidden = enforceBrowserMutationProtection(request, bearer);
+        if (mutationForbidden) return mutationForbidden;
         return finalize(await handler({ user, span, authDurationMs, routeStartedAtMs, setResponseFinalizer }));
       } catch (err) {
         recordSpanError(span, err);
@@ -103,6 +102,19 @@ export function jsonResponse(data: unknown, status = 200): Response {
   });
 }
 
+export function enforceBrowserMutationProtection(
+  request: Request,
+  bearer: StackBearer | null = parseBearer(request),
+): Response | null {
+  if (
+    requiresBrowserMutationProtection(request.method, bearer) &&
+    !browserMutationOriginAllowed(request)
+  ) {
+    return jsonResponse({ error: "forbidden" }, 403);
+  }
+  return null;
+}
+
 export type VmErrorResponseInput = {
   readonly error: string;
   readonly message: string;

web/services/vms/workflows.ts

@@ -31,7 +31,7 @@ import {
   type VmWorkflowError,
 } from "./errors";
 import { maxActiveVmsForPlan } from "./entitlements";
-import { isProviderNotFoundError } from "./providerErrors";
+import { isProviderIdentityNotFoundError, isProviderNotFoundError } from "./providerErrors";
 import { VmProviderGateway, VmProviderGatewayLive, type VmProviderGatewayShape } from "./providerGateway";
 import {
   VmRepository,
@@ -68,8 +68,19 @@ export type CloudVmSessionEntry = CloudVmSessionRow;
 
 export const VmWorkflowLive = Layer.mergeAll(VmRepositoryLive, VmProviderGatewayLive, VmBillingGatewayLive);
 
+const EXPIRED_IDENTITY_REVOKE_BATCH = 5;
+const EXPIRED_IDENTITY_REVOKE_RETRY_BACKOFF_MS = 10 * 60 * 1000;
+const IDENTITY_REVOKE_PROVIDER_TIMEOUT = "5 seconds";
+const ACTIVE_IDENTITY_REVOKE_HOT_PATH_LIMIT = 8;
 const VM_STATUS_RECONCILE_BATCH_LIMIT = 200;
 
+type ExistingVmAccessInput = {
+  readonly userId: string;
+  readonly billingTeamId?: string | null;
+  readonly teamIds?: readonly string[];
+  readonly providerVmId: string;
+};
+
 export type VmProviderStatusReconcileResult = {
   readonly checked: number;
   readonly updated: number;
@@ -99,12 +110,13 @@ export function listUserVms(userId: string, billingTeamId?: string | null) {
 export function getVm(input: {
   readonly userId: string;
   readonly billingTeamId?: string | null;
+  readonly teamIds?: readonly string[];
   readonly providerVmId: string;
 }) {
   return Effect.gen(function* () {
     const repo = yield* VmRepository;
     const providers = yield* VmProviderGateway;
-    const vm = yield* requireUserVm(input.userId, input.providerVmId, input.billingTeamId);
+    const vm = yield* requireUserVm(input);
     const providerVmId = vm.providerVmId ?? input.providerVmId;
     const getStatus = providers.getStatus;
     if (!getStatus) return vmEntryFromRow(vm);
@@ -344,6 +356,7 @@ function finishBaseCreate(
     readonly billingCustomerType: BillingCustomerType;
     readonly billingTeamId: string;
     readonly billingPlanId: string;
+    readonly maxActiveVms: number;
     readonly provider: ProviderId;
     readonly image: string;
     readonly imageVersion?: string | null;
@@ -370,6 +383,17 @@ function finishBaseCreate(
           new VmCreateInProgressError({ idempotencyKey: existing.idempotencyKey ?? "" }),
         );
       }
+      const replacement = yield* reopenBaseIfProviderDeleted(
+        repo,
+        providers,
+        input,
+        create,
+        existing,
+        existing.providerVmId,
+      );
+      if (replacement) {
+        return yield* finishBaseCreate(repo, providers, billing, input, replacement);
+      }
       return baseVmEntryFromRows(create.base, create.generation, existing, null);
     }
 
@@ -486,16 +510,65 @@ function finishBaseCreate(
   });
 }
 
+function reopenBaseIfProviderDeleted(
+  repo: VmRepositoryShape,
+  providers: VmProviderGatewayShape,
+  input: Parameters<VmRepositoryShape["beginBaseOpen"]>[0] & { readonly timing?: VmTimingSink },
+  create: Extract<BeginBaseCreateResult, { readonly kind: "existing" }>,
+  existing: CloudVmRow,
+  providerVmId: string,
+): Effect.Effect<BeginBaseCreateResult | null, VmWorkflowError, never> {
+  const getStatus = providers.getStatus;
+  if (!getStatus) return Effect.succeed(null);
+  return getStatus(existing.provider, providerVmId).pipe(
+    Effect.as(null),
+    Effect.catchAll((err) =>
+      isProviderNotFoundError(err)
+        ? Effect.gen(function* () {
+          const markedDestroyed = yield* repo.markProviderObservedStatus({
+            id: existing.id,
+            providerVmId,
+            status: "destroyed",
+          });
+          if (!markedDestroyed) {
+            return yield* Effect.fail(new VmNotFoundError({ vmId: providerVmId }));
+          }
+          yield* repo.recordUsageEvent({
+            userId: existing.userId,
+            billingTeamId: existing.billingTeamId,
+            billingPlanId: existing.billingPlanId,
+            vmId: existing.id,
+            eventType: "vm.destroyed",
+            provider: existing.provider,
+            imageId: existing.imageId,
+            metadata: {
+              source: "base_open_provider_missing",
+              baseName: input.baseName ?? "base",
+              generation: create.generation.generation,
+            },
+          }).pipe(Effect.catchAll(() => Effect.void));
+          return yield* measureVmEffect(
+            input.timing,
+            "begin_base_open",
+            repo.beginBaseOpen(input),
+          );
+        })
+        : Effect.succeed(null)
+    ),
+  );
+}
+
 export function snapshotVm(input: {
   readonly userId: string;
   readonly billingTeamId?: string | null;
+  readonly teamIds?: readonly string[];
   readonly providerVmId: string;
   readonly name?: string;
 }) {
   return Effect.gen(function* () {
     const repo = yield* VmRepository;
     const providers = yield* VmProviderGateway;
-    const vm = yield* requireUserVm(input.userId, input.providerVmId, input.billingTeamId);
+    const vm = yield* requireUserVm(input);
     const snapshot = yield* (providers.snapshot
       ? providers.snapshot(vm.provider, vm.providerVmId ?? input.providerVmId, input.name)
       : Effect.fail(new VmProviderOperationError({
@@ -559,6 +632,7 @@ export function forkVm(input: {
   readonly userId: string;
   readonly billingCustomerType: BillingCustomerType;
   readonly billingTeamId: string;
+  readonly teamIds?: readonly string[];
   readonly billingPlanId: string;
   readonly maxActiveVms: number;
   readonly providerVmId: string;
@@ -570,7 +644,7 @@ export function forkVm(input: {
     const repo = yield* VmRepository;
     const providers = yield* VmProviderGateway;
     const billing = yield* VmBillingGateway;
-    const source = yield* requireUserVm(input.userId, input.providerVmId, input.billingTeamId);
+    const source = yield* requireUserVm(input);
     yield* preflightResumeIfSuspended(repo, providers, source, input.providerVmId, "fork");
 
     if (source.provider === "freestyle" && providers.fork) {
@@ -714,6 +788,7 @@ export function forkVm(input: {
 
     const snapshot = yield* snapshotVm({
       userId: input.userId,
+      teamIds: input.teamIds,
       billingTeamId: source.billingTeamId,
       providerVmId: input.providerVmId,
       name: input.name,
@@ -996,11 +1071,11 @@ function preflightResumeIfSuspended(
   vm: CloudVmRow,
   providerVmId: string,
   resumeSource: VmResumeSource,
-): Effect.Effect<void, VmWorkflowError> {
+): Effect.Effect<boolean, VmWorkflowError> {
   return Effect.gen(function* () {
     const getStatus = providers.getStatus;
     const resume = providers.resume;
-    if (!getStatus || !resume) return;
+    if (!getStatus || !resume) return false;
 
     const status = yield* getStatus(vm.provider, providerVmId).pipe(
       Effect.timeoutFail({
@@ -1047,7 +1122,7 @@ function preflightResumeIfSuspended(
       if (!recorded) {
         return yield* Effect.fail(new VmNotFoundError({ vmId: providerVmId }));
       }
-      return;
+      return false;
     }
     if (status === "running") {
       // Freestyle's SSH gateway can resume a VM entirely outside the control
@@ -1063,9 +1138,9 @@ function preflightResumeIfSuspended(
           return yield* Effect.fail(new VmNotFoundError({ vmId: providerVmId }));
         }
       }
-      return;
+      return false;
     }
-    if (status !== "paused") return;
+    if (status !== "paused") return false;
 
     const reserved = yield* reservePausedResumeIfTeam(repo, vm, providerVmId);
     yield* resumeUntilRunning(providers, vm, providerVmId).pipe(
@@ -1081,6 +1156,7 @@ function preflightResumeIfSuspended(
       Effect.tapError(() => rollbackPausedResumeReservation(repo, vm, providerVmId, reserved)),
     );
     if (reserved) yield* recordResumeUsageEvent(repo, vm, resumeSource);
+    return true;
   });
 }
 
@@ -1168,14 +1244,15 @@ function recordRunningTransition<E extends VmWorkflowError>(
 export function destroyVm(input: {
   readonly userId: string;
   readonly billingTeamId?: string | null;
+  readonly teamIds?: readonly string[];
   readonly providerVmId: string;
 }) {
   return Effect.gen(function* () {
     const repo = yield* VmRepository;
     const providers = yield* VmProviderGateway;
-    const vm = yield* requireUserVm(input.userId, input.providerVmId, input.billingTeamId);
+    const vm = yield* requireUserVm(input);
 
-    yield* revokeActiveIdentities(vm);
+    yield* revokeActiveIdentities(vm, { failOnCleanupError: true });
     yield* providers.destroy(vm.provider, vm.providerVmId ?? input.providerVmId).pipe(
       Effect.catchAll((err) => {
         if (isProviderNotFoundError(err.cause)) return Effect.void;
@@ -1195,9 +1272,48 @@ export function destroyVm(input: {
   });
 }
 
+export function revokeExpiredIdentityLeases(input: {
+  readonly now?: Date;
+  readonly limit?: number;
+} = {}) {
+  return Effect.gen(function* () {
+    const repo = yield* VmRepository;
+    const providers = yield* VmProviderGateway;
+    const expiredIdentityLeases = repo.expiredIdentityLeases;
+    if (!expiredIdentityLeases) return 0;
+    const now = input.now ?? new Date();
+    const leases = yield* expiredIdentityLeases({
+      now,
+      limit: input.limit ?? EXPIRED_IDENTITY_REVOKE_BATCH,
+    });
+    const revokedIds: string[] = [];
+    for (const lease of leases) {
+      const identityHandle = lease.providerIdentityHandle;
+      if (!identityHandle) continue;
+      const retryAfter = new Date(now.getTime() + EXPIRED_IDENTITY_REVOKE_RETRY_BACKOFF_MS);
+      yield* (repo.markLeaseRevocationRetry?.({
+        id: lease.id,
+        retryAfter,
+        error: "revoke pending",
+      }) ?? Effect.void).pipe(Effect.catchAll(() => Effect.void));
+      const revoked = yield* revokeSSHIdentityForCleanup(providers, lease.provider, identityHandle).pipe(
+        Effect.as(true),
+        Effect.catchAll((err) => {
+          if (isProviderIdentityNotFoundError(err.cause)) return Effect.succeed(true);
+          return Effect.succeed(false);
+        }),
+      );
+      if (revoked) revokedIds.push(lease.id);
+    }
+    yield* repo.markLeasesRevoked(revokedIds);
+    return revokedIds.length;
+  });
+}
+
 export function execVm(input: {
   readonly userId: string;
   readonly billingTeamId?: string | null;
+  readonly teamIds?: readonly string[];
   readonly providerVmId: string;
   readonly command: string;
   readonly timeoutMs: number;
@@ -1205,7 +1321,7 @@ export function execVm(input: {
   return Effect.gen(function* () {
     const repo = yield* VmRepository;
     const providers = yield* VmProviderGateway;
-    const vm = yield* requireUserVm(input.userId, input.providerVmId, input.billingTeamId);
+    const vm = yield* requireUserVm(input);
     yield* preflightResumeIfSuspended(
       repo,
       providers,
@@ -1233,6 +1349,7 @@ export function execVm(input: {
 type OpenAttachEndpointInput = {
   readonly userId: string;
   readonly billingTeamId?: string | null;
+  readonly teamIds?: readonly string[];
   readonly providerVmId: string;
   readonly options?: AttachOptions;
   readonly sessionTitle?: string | null;
@@ -1248,6 +1365,7 @@ export function openAttachEndpoint(input: OpenAttachEndpointInput) {
 export function openVmSession(input: {
   readonly userId: string;
   readonly billingTeamId?: string | null;
+  readonly teamIds?: readonly string[];
   readonly providerVmId: string;
   readonly sessionId?: string;
   readonly attachmentId?: string;
@@ -1258,6 +1376,7 @@ export function openVmSession(input: {
   return openAttachEndpointResult({
     userId: input.userId,
     billingTeamId: input.billingTeamId,
+    teamIds: input.teamIds,
     providerVmId: input.providerVmId,
     sessionTitle: input.title,
     options: {
@@ -1271,11 +1390,12 @@ export function openVmSession(input: {
 export function listVmSessions(input: {
   readonly userId: string;
   readonly billingTeamId?: string | null;
+  readonly teamIds?: readonly string[];
   readonly providerVmId: string;
 }) {
   return Effect.gen(function* () {
     const repo = yield* VmRepository;
-    const vm = yield* requireUserVm(input.userId, input.providerVmId, input.billingTeamId);
+    const vm = yield* requireUserVm(input);
     return yield* repo.listVmSessions({ userId: input.userId, vmId: vm.id });
   });
 }
@@ -1284,14 +1404,12 @@ function openAttachEndpointResult(input: OpenAttachEndpointInput) {
   return Effect.gen(function* () {
     const repo = yield* VmRepository;
     const providers = yield* VmProviderGateway;
-    const vm = yield* requireUserVm(input.userId, input.providerVmId, input.billingTeamId);
-    // Endpoint minting can succeed against a paused VM (Freestyle openSSH only
-    // grants an identity), which would hand out an endpoint while Postgres
-    // still says paused. Preflight-resume first — and before revoking the
-    // user's existing identities, so a preflight failure never strands them
-    // with old credentials revoked and no replacement minted.
+    const vm = yield* requireUserVm(input);
     yield* preflightResumeIfSuspended(repo, providers, vm, input.providerVmId, "attach");
-    yield* revokeActiveIdentities(vm);
+    // Once preflight records the VM as running, that state is externally
+    // visible to concurrent attach/SSH requests. Later cleanup failures must
+    // fail closed without pausing a VM another request may have attached to.
+    yield* revokeActiveIdentities(vm, { failOnCleanupError: true });
     const endpoint = yield* withResumeOnSuspendedAfterFailure(
       repo,
       providers,
@@ -1347,19 +1465,15 @@ function openAttachEndpointResult(input: OpenAttachEndpointInput) {
 export function openSshEndpoint(input: {
   readonly userId: string;
   readonly billingTeamId?: string | null;
+  readonly teamIds?: readonly string[];
   readonly providerVmId: string;
 }) {
   return Effect.gen(function* () {
     const repo = yield* VmRepository;
     const providers = yield* VmProviderGateway;
-    const vm = yield* requireUserVm(input.userId, input.providerVmId, input.billingTeamId);
-    // Endpoint minting can succeed against a paused VM (Freestyle openSSH only
-    // grants an identity), which would hand out an endpoint while Postgres
-    // still says paused. Preflight-resume first — and before revoking the
-    // user's existing identities, so a preflight failure never strands them
-    // with old credentials revoked and no replacement minted.
+    const vm = yield* requireUserVm(input);
     yield* preflightResumeIfSuspended(repo, providers, vm, input.providerVmId, "ssh");
-    yield* revokeActiveIdentities(vm);
+    yield* revokeActiveIdentities(vm, { failOnCleanupError: true });
     const endpoint = yield* withResumeOnSuspendedAfterFailure(
       repo,
       providers,
@@ -1389,31 +1503,88 @@ export function openSshEndpoint(input: {
   });
 }
 
-function requireUserVm(userId: string, providerVmId: string, billingTeamId?: string | null) {
+function requireUserVm(input: ExistingVmAccessInput) {
   return Effect.gen(function* () {
     const repo = yield* VmRepository;
-    const vm = yield* repo.findUserVm({ userId, billingTeamId, providerVmId });
+    const vm = yield* repo.findUserVm({
+      userId: input.userId,
+      billingTeamId: input.billingTeamId,
+      providerVmId: input.providerVmId,
+    });
     if (!vm || !vm.providerVmId) {
-      return yield* Effect.fail(new VmNotFoundError({ vmId: providerVmId }));
+      return yield* Effect.fail(new VmNotFoundError({ vmId: input.providerVmId }));
+    }
+    if (!callerStillOwnsBillingScope(input, vm)) {
+      return yield* Effect.fail(new VmNotFoundError({ vmId: input.providerVmId }));
     }
     return vm;
   });
 }
 
-function revokeActiveIdentities(vm: CloudVmRow) {
+function callerStillOwnsBillingScope(input: ExistingVmAccessInput, vm: CloudVmRow): boolean {
+  const billingTeamId = vm.billingTeamId?.trim();
+  if (!billingTeamId) return true;
+  if (billingTeamId === input.userId) return true;
+  if (!input.teamIds) return false;
+  return new Set(input.teamIds).has(billingTeamId);
+}
+
+function revokeActiveIdentities(
+  vm: CloudVmRow,
+  options: { readonly failOnCleanupError?: boolean; readonly limit?: number } = {},
+) {
   return Effect.gen(function* () {
     const repo = yield* VmRepository;
     const providers = yield* VmProviderGateway;
-    const leases = yield* repo.activeIdentityLeases(vm.id);
+    const leases = yield* repo.activeIdentityLeases(
+      vm.id,
+      options.failOnCleanupError ? ACTIVE_IDENTITY_REVOKE_HOT_PATH_LIMIT + 1 : options.limit,
+    );
+    if (options.failOnCleanupError && leases.length > ACTIVE_IDENTITY_REVOKE_HOT_PATH_LIMIT) {
+      return yield* Effect.fail(new VmProviderOperationError({
+        provider: vm.provider,
+        operation: "revokeSSHIdentity",
+        cause: new Error(`too many active identity leases pending cleanup: ${leases.length}`),
+      }));
+    }
+    const revokedIds: string[] = [];
     for (const lease of leases) {
       const identityHandle = lease.providerIdentityHandle;
       if (!identityHandle) continue;
-      yield* providers.revokeSSHIdentity(vm.provider, identityHandle);
+      const revoked = yield* revokeSSHIdentityForCleanup(providers, vm.provider, identityHandle).pipe(
+        Effect.as(true),
+        Effect.catchAll((err) => {
+          if (isProviderIdentityNotFoundError(err.cause)) return Effect.succeed(true);
+          if (!options.failOnCleanupError) return Effect.succeed(false);
+          return repo.markLeasesRevoked(revokedIds).pipe(
+            Effect.andThen(Effect.fail(err)),
+          );
+        }),
+      );
+      if (revoked) revokedIds.push(lease.id);
     }
-    yield* repo.markLeasesRevoked(leases.map((lease) => lease.id));
+    yield* repo.markLeasesRevoked(revokedIds);
   });
 }
 
+function revokeSSHIdentityForCleanup(
+  providers: VmProviderGatewayShape,
+  provider: ProviderId,
+  identityHandle: string,
+): Effect.Effect<void, VmProviderOperationError> {
+  return providers.revokeSSHIdentity(provider, identityHandle).pipe(
+    Effect.timeoutFail({
+      duration: IDENTITY_REVOKE_PROVIDER_TIMEOUT,
+      onTimeout: () =>
+        new VmProviderOperationError({
+          provider,
+          operation: "revokeSSHIdentity",
+          cause: new Error("identity revoke timed out"),
+        }),
+    }),
+  );
+}
+
 function storeEndpointLeases(vm: CloudVmRow, endpoint: AttachEndpoint | SSHEndpoint) {
   return Effect.gen(function* () {
     if (endpoint.transport === "ssh") {

web/tests/client-config-route.test.ts

@@ -1,4 +1,8 @@
 import { afterAll, afterEach, describe, expect, mock, test } from "bun:test";
+import {
+  checkRateLimit,
+  installVercelFirewallMock,
+} from "./vercel-firewall-mock";
 
 const originalSkipEnvValidation = process.env.SKIP_ENV_VALIDATION;
 const originalPostHogProjectKey = process.env.POSTHOG_PROJECT_KEY;
@@ -8,11 +12,7 @@ process.env.POSTHOG_PROJECT_KEY = "test-project-key";
 process.env.CMUX_CLIENT_CONFIG_RATE_LIMIT_ID = "cmux-client-config-test";
 
 const originalVercel = process.env.VERCEL;
-const checkRateLimit = mock(async () => ({ rateLimited: false, error: null }));
-
-mock.module("@vercel/firewall", () => ({
-  checkRateLimit,
-}));
+installVercelFirewallMock();
 
 const {
   normalizePostHogFlagsResponse,

web/tests/enterprise-contact-route.test.ts

@@ -111,7 +111,10 @@ describe("enterprise contact route", () => {
       slack: "sent",
       posthog: "sent",
     });
-    expect(resendCtor).toHaveBeenCalledWith("re_test");
+    const resendCtorCalls = (resendCtor as unknown as {
+      mock: { calls: Array<[unknown]> };
+    }).mock.calls;
+    expect(typeof resendCtorCalls[0]?.[0]).toBe("string");
     expect(resendSend).toHaveBeenCalledTimes(1);
     const resendCalls = (resendSend as unknown as {
       mock: { calls: Array<[Record<string, unknown>]> };

web/tests/feedback-route.test.ts

@@ -0,0 +1,98 @@
+const priorSkipEnvValidation = process.env.SKIP_ENV_VALIDATION;
+const priorResendApiKey = process.env.RESEND_API_KEY;
+const priorFeedbackFromEmail = process.env.CMUX_FEEDBACK_FROM_EMAIL;
+const priorFeedbackRateLimitId = process.env.CMUX_FEEDBACK_RATE_LIMIT_ID;
+const priorVercel = process.env.VERCEL;
+
+process.env.SKIP_ENV_VALIDATION = "1";
+process.env.RESEND_API_KEY = "resend-test-key";
+process.env.CMUX_FEEDBACK_FROM_EMAIL = "[email protected]";
+process.env.CMUX_FEEDBACK_RATE_LIMIT_ID = "feedback-rate-limit-test";
+
+import { afterAll, afterEach, describe, expect, mock, test } from "bun:test";
+import {
+  checkRateLimit,
+  installVercelFirewallMock,
+} from "./vercel-firewall-mock";
+
+const sendEmail = mock(async () => ({ data: { id: "email-1" }, error: null }));
+
+installVercelFirewallMock();
+
+mock.module("@/app/env", () => ({
+  env: {
+    RESEND_API_KEY: "resend-test-key",
+    CMUX_FEEDBACK_FROM_EMAIL: "[email protected]",
+    CMUX_FEEDBACK_RATE_LIMIT_ID: "feedback-rate-limit-test",
+    CMUX_PUSH_RATE_LIMIT_ID: "cmux-push-test",
+  },
+}));
+
+mock.module("resend", () => ({
+  Resend: class {
+    readonly emails = { send: sendEmail };
+  },
+}));
+
+const { POST } = await import("../app/api/feedback/route");
+
+afterEach(() => {
+  checkRateLimit.mockClear();
+  checkRateLimit.mockResolvedValue({ rateLimited: false, error: null });
+  sendEmail.mockClear();
+  if (priorVercel === undefined) {
+    delete process.env.VERCEL;
+  } else {
+    process.env.VERCEL = priorVercel;
+  }
+});
+
+afterAll(() => {
+  restoreEnv("SKIP_ENV_VALIDATION", priorSkipEnvValidation);
+  restoreEnv("RESEND_API_KEY", priorResendApiKey);
+  restoreEnv("CMUX_FEEDBACK_FROM_EMAIL", priorFeedbackFromEmail);
+  restoreEnv("CMUX_FEEDBACK_RATE_LIMIT_ID", priorFeedbackRateLimitId);
+  restoreEnv("VERCEL", priorVercel);
+});
+
+describe("feedback route", () => {
+  test("fails closed when the Vercel firewall rule is missing", async () => {
+    process.env.VERCEL = "1";
+    checkRateLimit.mockResolvedValue({ rateLimited: false, error: "not-found" });
+
+    const res = await POST(feedbackRequest());
+
+    expect(res.status).toBe(503);
+    expect(await res.json()).toEqual({ error: "service_unavailable" });
+    expect(sendEmail).not.toHaveBeenCalled();
+  });
+
+  test("fails closed when the Vercel firewall check errors", async () => {
+    process.env.VERCEL = "1";
+    checkRateLimit.mockResolvedValue({ rateLimited: false, error: "firewall-unavailable" });
+
+    const res = await POST(feedbackRequest());
+
+    expect(res.status).toBe(503);
+    expect(await res.json()).toEqual({ error: "service_unavailable" });
+    expect(sendEmail).not.toHaveBeenCalled();
+  });
+});
+
+function feedbackRequest(): Request {
+  const form = new FormData();
+  form.set("email", "[email protected]");
+  form.set("message", "The app crashed while opening a workspace.");
+  return new Request("https://cmux.test/api/feedback", {
+    method: "POST",
+    body: form,
+  });
+}
+
+function restoreEnv(key: string, value: string | undefined): void {
+  if (value === undefined) {
+    delete process.env[key];
+    return;
+  }
+  process.env[key] = value;
+}

web/tests/vault-commit-route.test.ts

@@ -0,0 +1,267 @@
+import { afterEach, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test";
+import { eq, sql } from "drizzle-orm";
+import { cloudDb } from "../db/client";
+import { vaultSnapshots, vaultUploadGrants } from "../db/schema";
+
+const runDbTests = process.env.CMUX_DB_TEST === "1";
+const dbTest = runDbTests ? test : test.skip;
+const userId = "user-vault-commit-test";
+const sha256 = "b".repeat(64);
+
+const storageModule = await import("../services/vault/storage");
+const realBuildObjectKey = storageModule.buildObjectKey;
+let objectContentLength = 456;
+let deleteFailure: Error | null = null;
+const headedKeys: string[] = [];
+const headObject = mock(async (...args: unknown[]) => {
+  const [key] = args as [string];
+  headedKeys.push(key);
+  return { contentLength: objectContentLength };
+});
+const copyObject = mock(async () => undefined);
+const deleteObject = mock(async () => {
+  if (deleteFailure) throw deleteFailure;
+});
+const getUser = mock(async () => stackUser());
+
+mock.module("../services/vault/storage", () => ({
+  ...storageModule,
+  copyObject,
+  deleteObject,
+  headObject,
+}));
+
+mock.module("../app/lib/stack", () => ({
+  getStackServerApp: () => ({ getUser }),
+  isStackConfigured: () => true,
+  stackServerApp: { getUser },
+}));
+
+const { POST } = await import("../app/api/vault/sessions/commit/route");
+
+const ORIGINAL_ENV = {
+  CMUX_VAULT_ENABLED: process.env.CMUX_VAULT_ENABLED,
+  CMUX_VAULT_S3_BUCKET: process.env.CMUX_VAULT_S3_BUCKET,
+  CMUX_VAULT_MAX_UPLOAD_BYTES: process.env.CMUX_VAULT_MAX_UPLOAD_BYTES,
+  CMUX_VAULT_MAX_USER_BYTES: process.env.CMUX_VAULT_MAX_USER_BYTES,
+};
+
+beforeAll(() => {
+  if (runDbTests && !process.env.DATABASE_URL) {
+    throw new Error("DATABASE_URL is required when CMUX_DB_TEST=1");
+  }
+});
+
+beforeEach(async () => {
+  process.env.CMUX_VAULT_ENABLED = "1";
+  process.env.CMUX_VAULT_S3_BUCKET = "test-bucket";
+  process.env.CMUX_VAULT_MAX_UPLOAD_BYTES = "1000000";
+  process.env.CMUX_VAULT_MAX_USER_BYTES = "1000000";
+  objectContentLength = 456;
+  deleteFailure = null;
+  headedKeys.length = 0;
+  headObject.mockClear();
+  copyObject.mockClear();
+  deleteObject.mockClear();
+  getUser.mockClear();
+  if (runDbTests) await resetVaultTables();
+});
+
+afterEach(() => {
+  restoreEnvValue("CMUX_VAULT_ENABLED", ORIGINAL_ENV.CMUX_VAULT_ENABLED);
+  restoreEnvValue("CMUX_VAULT_S3_BUCKET", ORIGINAL_ENV.CMUX_VAULT_S3_BUCKET);
+  restoreEnvValue("CMUX_VAULT_MAX_UPLOAD_BYTES", ORIGINAL_ENV.CMUX_VAULT_MAX_UPLOAD_BYTES);
+  restoreEnvValue("CMUX_VAULT_MAX_USER_BYTES", ORIGINAL_ENV.CMUX_VAULT_MAX_USER_BYTES);
+});
+
+describe("Vault commit route", () => {
+  dbTest("commits only when the current grant matches the uploaded size", async () => {
+    const db = cloudDb();
+    const objectKey = realBuildObjectKey(userId, "codex", "session-1", sha256);
+    await insertGrant(objectKey, 456);
+
+    const response = await POST(commitRequest({ compressedSizeBytes: 456 }));
+
+    expect(response.status).toBe(200);
+    expect((await response.json()).items[0].status).toBe("committed");
+    expect(headObject).toHaveBeenCalledTimes(1);
+    expect(headedKeys).toEqual([`${objectKey}.upload`]);
+    expect(copyObject).toHaveBeenCalledWith(`${objectKey}.upload`, objectKey);
+    expect(deleteObject).toHaveBeenCalledWith(`${objectKey}.upload`);
+    const grants = await db
+      .select({ id: vaultUploadGrants.id })
+      .from(vaultUploadGrants)
+      .where(eq(vaultUploadGrants.objectKey, objectKey));
+    expect(grants).toHaveLength(0);
+    const snapshots = await db
+      .select({ objectKey: vaultSnapshots.objectKey })
+      .from(vaultSnapshots)
+      .where(eq(vaultSnapshots.objectKey, objectKey));
+    expect(snapshots).toHaveLength(1);
+  });
+
+  dbTest("rejects a stale large upload after the current grant is downsized", async () => {
+    const db = cloudDb();
+    const objectKey = realBuildObjectKey(userId, "codex", "session-1", sha256);
+    await insertGrant(objectKey, 10);
+    objectContentLength = 900;
+
+    const response = await POST(commitRequest({ compressedSizeBytes: 900 }));
+
+    expect(response.status).toBe(200);
+    expect((await response.json()).items[0].error).toBe("upload_grant_mismatch");
+    expect(headObject).not.toHaveBeenCalled();
+    const grants = await db
+      .select({ compressedSizeBytes: vaultUploadGrants.compressedSizeBytes })
+      .from(vaultUploadGrants)
+      .where(eq(vaultUploadGrants.objectKey, objectKey));
+    expect(grants).toHaveLength(1);
+    expect(grants[0].compressedSizeBytes).toBe(10);
+    const snapshots = await db
+      .select({ objectKey: vaultSnapshots.objectKey })
+      .from(vaultSnapshots)
+      .where(eq(vaultSnapshots.objectKey, objectKey));
+    expect(snapshots).toHaveLength(0);
+  });
+
+  dbTest("keeps the grant retryable when staging cleanup fails after commit", async () => {
+    const db = cloudDb();
+    const objectKey = realBuildObjectKey(userId, "codex", "session-1", sha256);
+    await insertGrant(objectKey, 456);
+    deleteFailure = new Error("storage delete failed");
+
+    const response = await POST(commitRequest({ compressedSizeBytes: 456 }));
+
+    expect(response.status).toBe(200);
+    expect((await response.json()).items[0].status).toBe("committed");
+    const grants = await db
+      .select({ id: vaultUploadGrants.id })
+      .from(vaultUploadGrants)
+      .where(eq(vaultUploadGrants.objectKey, objectKey));
+    expect(grants).toHaveLength(1);
+    const snapshots = await db
+      .select({ objectKey: vaultSnapshots.objectKey })
+      .from(vaultSnapshots)
+      .where(eq(vaultSnapshots.objectKey, objectKey));
+    expect(snapshots).toHaveLength(1);
+  });
+
+  dbTest("commits legacy final-key grants without deleting the committed object", async () => {
+    const db = cloudDb();
+    const objectKey = realBuildObjectKey(userId, "codex", "session-1", sha256);
+    await insertGrant(objectKey, 456, objectKey);
+
+    const response = await POST(commitRequest({ compressedSizeBytes: 456 }));
+
+    expect(response.status).toBe(200);
+    expect((await response.json()).items[0].status).toBe("committed");
+    expect(headObject).toHaveBeenCalledTimes(1);
+    expect(headedKeys).toEqual([objectKey]);
+    expect(copyObject).not.toHaveBeenCalled();
+    expect(deleteObject).not.toHaveBeenCalled();
+    const grants = await db
+      .select({ id: vaultUploadGrants.id })
+      .from(vaultUploadGrants)
+      .where(eq(vaultUploadGrants.objectKey, objectKey));
+    expect(grants).toHaveLength(0);
+    const snapshots = await db
+      .select({ objectKey: vaultSnapshots.objectKey })
+      .from(vaultSnapshots)
+      .where(eq(vaultSnapshots.objectKey, objectKey));
+    expect(snapshots).toHaveLength(1);
+  });
+
+  dbTest("deletes a copied final object when the database commit rolls back", async () => {
+    const db = cloudDb();
+    const objectKey = realBuildObjectKey(userId, "codex", "session-1", sha256);
+    await insertGrant(objectKey, 456);
+    await db.execute(sql`
+      alter table vault_snapshots
+      add constraint vault_snapshots_force_failure check (compressed_size_bytes < 0)
+    `);
+
+    try {
+      const response = await POST(commitRequest({ compressedSizeBytes: 456 }));
+
+      expect(response.status).toBe(500);
+      expect(copyObject).toHaveBeenCalledWith(`${objectKey}.upload`, objectKey);
+      expect(deleteObject).toHaveBeenCalledWith(objectKey);
+      const grants = await db
+        .select({ id: vaultUploadGrants.id })
+        .from(vaultUploadGrants)
+        .where(eq(vaultUploadGrants.objectKey, objectKey));
+      expect(grants).toHaveLength(1);
+      const snapshots = await db
+        .select({ objectKey: vaultSnapshots.objectKey })
+        .from(vaultSnapshots)
+        .where(eq(vaultSnapshots.objectKey, objectKey));
+      expect(snapshots).toHaveLength(0);
+    } finally {
+      await db.execute(sql`alter table vault_snapshots drop constraint if exists vault_snapshots_force_failure`);
+    }
+  });
+});
+
+async function insertGrant(
+  objectKey: string,
+  compressedSizeBytes: number,
+  uploadObjectKey = `${objectKey}.upload`,
+): Promise<void> {
+  await cloudDb()
+    .insert(vaultUploadGrants)
+    .values({
+      userId,
+      objectKey,
+      uploadObjectKey,
+      compressedSizeBytes,
+      createdAt: new Date("2030-01-01T00:00:00.000Z"),
+      expiresAt: new Date("2030-01-02T00:00:00.000Z"),
+    });
+}
+
+function commitRequest(input: { readonly compressedSizeBytes: number }): Request {
+  return new Request("https://cmux.test/api/vault/sessions/commit", {
+    method: "POST",
+    headers: {
+      authorization: "Bearer access-token",
+      "x-stack-refresh-token": "refresh-token",
+      "content-type": "application/json",
+    },
+    body: JSON.stringify({
+      items: [{
+        agent: "codex",
+        agentSessionId: "session-1",
+        relPath: "sessions/session-1.jsonl.zst",
+        cwd: "/workspace",
+        sha256,
+        sizeBytes: 999,
+        compressedSizeBytes: input.compressedSizeBytes,
+      }],
+    }),
+  });
+}
+
+async function resetVaultTables(): Promise<void> {
+  await cloudDb().execute(sql`
+    truncate vault_snapshots, vault_sessions, vault_upload_grants restart identity cascade
+  `);
+}
+
+function stackUser() {
+  return {
+    id: userId,
+    displayName: null,
+    primaryEmail: "[email protected]",
+    selectedTeam: null,
+    clientReadOnlyMetadata: {},
+    listTeams: async () => [],
+  };
+}
+
+function restoreEnvValue(key: string, value: string | undefined): void {
+  if (value === undefined) {
+    delete process.env[key];
+    return;
+  }
+  process.env[key] = value;
+}

web/tests/vault-route-helpers.test.ts

@@ -9,6 +9,7 @@ import {
   withAuthedVaultApiRoute,
   withVaultApiRoute,
 } from "../services/vault/routeHelpers";
+import type { AuthedUser } from "../services/vms/auth";
 
 let exporter: InMemorySpanExporter;
 let provider: BasicTracerProvider;
@@ -88,8 +89,108 @@ describe("Vault route helper", () => {
       console.error = originalError;
     }
   });
+
+  test("blocks cross-site cookie-authenticated mutation requests before the handler", async () => {
+    const handler = mock(async () => Response.json({ ok: true }));
+
+    const response = await withAuthedVaultApiRoute(
+      new Request("https://cmux.test/api/vault/test", {
+        method: "POST",
+        headers: {
+          origin: "https://evil.test",
+          "sec-fetch-site": "cross-site",
+        },
+      }),
+      "/api/vault/test",
+      { "cmux.vault.operation": "test" },
+      "/api/vault/test failed",
+      {},
+      handler,
+      async () => testUser,
+    );
+
+    expect(response.status).toBe(403);
+    expect(await response.json()).toEqual({ error: "forbidden" });
+    expect(handler).not.toHaveBeenCalled();
+  });
+
+  test("blocks cookie-authenticated mutation requests without an origin", async () => {
+    const handler = mock(async () => Response.json({ ok: true }));
+
+    const response = await withAuthedVaultApiRoute(
+      new Request("https://cmux.test/api/vault/test", { method: "POST" }),
+      "/api/vault/test",
+      { "cmux.vault.operation": "test" },
+      "/api/vault/test failed",
+      {},
+      handler,
+      async () => testUser,
+    );
+
+    expect(response.status).toBe(403);
+    expect(await response.json()).toEqual({ error: "forbidden" });
+    expect(handler).not.toHaveBeenCalled();
+  });
+
+  test("allows same-origin cookie-authenticated mutation requests", async () => {
+    const handler = mock(async () => Response.json({ ok: true }));
+
+    const response = await withAuthedVaultApiRoute(
+      new Request("https://cmux.test/api/vault/test", {
+        method: "POST",
+        headers: { origin: "https://cmux.test" },
+      }),
+      "/api/vault/test",
+      { "cmux.vault.operation": "test" },
+      "/api/vault/test failed",
+      {},
+      handler,
+      async () => testUser,
+    );
+
+    expect(response.status).toBe(200);
+    expect(await response.json()).toEqual({ ok: true });
+    expect(handler).toHaveBeenCalledTimes(1);
+  });
+
+  test("allows bearer-authenticated mutation requests without an origin", async () => {
+    const handler = mock(async () => Response.json({ ok: true }));
+
+    const response = await withAuthedVaultApiRoute(
+      new Request("https://cmux.test/api/vault/test", {
+        method: "POST",
+        headers: {
+          authorization: "Bearer access-token",
+          "x-stack-refresh-token": "refresh-token",
+        },
+      }),
+      "/api/vault/test",
+      { "cmux.vault.operation": "test" },
+      "/api/vault/test failed",
+      { allowCookie: false },
+      handler,
+      async () => testUser,
+    );
+
+    expect(response.status).toBe(200);
+    expect(await response.json()).toEqual({ ok: true });
+    expect(handler).toHaveBeenCalledTimes(1);
+  });
 });
 
+const testUser: AuthedUser = {
+  id: "user-vault-test",
+  displayName: null,
+  primaryEmail: "[email protected]",
+  billingCustomerType: "user",
+  billingTeamId: "user-vault-test",
+  selectedTeamId: null,
+  teams: [],
+  teamIds: [],
+  userBillingPlanId: null,
+  billingPlanId: null,
+};
+
 function latestVaultTestSpan() {
   return exporter
     .getFinishedSpans()

web/tests/vault-uploads-route.test.ts

@@ -0,0 +1,607 @@
+import { afterEach, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test";
+import { eq, sql } from "drizzle-orm";
+import { cloudDb } from "../db/client";
+import { vaultSessions, vaultSnapshots, vaultUploadGrants, vaultUploadTombstones } from "../db/schema";
+
+const runDbTests = process.env.CMUX_DB_TEST === "1";
+const dbTest = runDbTests ? test : test.skip;
+const userId = "user-vault-upload-test";
+const sha256 = "a".repeat(64);
+
+const storageModule = await import("../services/vault/storage");
+const realBuildObjectKey = storageModule.buildObjectKey;
+let presignFailure: Error | null = null;
+let beforeNextPresignFailure: (() => Promise<void>) | null = null;
+let beforeNextDelete: ((key: string) => Promise<void>) | null = null;
+let presignCalls: { readonly key: string; readonly contentLength: number }[] = [];
+const presignPut = mock(async (...args: unknown[]) => {
+  const [key, contentLength] = args as [string, number];
+  presignCalls.push({ key, contentLength });
+  if (beforeNextPresignFailure) {
+    const run = beforeNextPresignFailure;
+    beforeNextPresignFailure = null;
+    await run();
+    throw new Error("transient presign failure");
+  }
+  if (presignFailure) throw presignFailure;
+  return `https://storage.test/${encodeURIComponent(key)}?contentLength=${contentLength}`;
+});
+const deleteObject = mock(async (...args: unknown[]) => {
+  const [key] = args as [string];
+  if (!beforeNextDelete) return;
+  const run = beforeNextDelete;
+  beforeNextDelete = null;
+  await run(key);
+});
+const getUser = mock(async () => stackUser());
+
+mock.module("../services/vault/storage", () => ({
+  ...storageModule,
+  presignPut,
+  deleteObject,
+}));
+
+mock.module("../app/lib/stack", () => ({
+  getStackServerApp: () => ({ getUser }),
+  isStackConfigured: () => true,
+  stackServerApp: { getUser },
+}));
+
+const { POST } = await import("../app/api/vault/uploads/route");
+
+const ORIGINAL_ENV = {
+  CMUX_VAULT_ENABLED: process.env.CMUX_VAULT_ENABLED,
+  CMUX_VAULT_S3_BUCKET: process.env.CMUX_VAULT_S3_BUCKET,
+  CMUX_VAULT_MAX_UPLOAD_BYTES: process.env.CMUX_VAULT_MAX_UPLOAD_BYTES,
+  CMUX_VAULT_MAX_USER_BYTES: process.env.CMUX_VAULT_MAX_USER_BYTES,
+};
+
+beforeAll(() => {
+  if (runDbTests && !process.env.DATABASE_URL) {
+    throw new Error("DATABASE_URL is required when CMUX_DB_TEST=1");
+  }
+});
+
+beforeEach(async () => {
+  process.env.CMUX_VAULT_ENABLED = "1";
+  process.env.CMUX_VAULT_S3_BUCKET = "test-bucket";
+  process.env.CMUX_VAULT_MAX_UPLOAD_BYTES = "1000000";
+  process.env.CMUX_VAULT_MAX_USER_BYTES = "1000000";
+  presignFailure = null;
+  beforeNextPresignFailure = null;
+  beforeNextDelete = null;
+  presignCalls = [];
+  presignPut.mockClear();
+  deleteObject.mockClear();
+  getUser.mockClear();
+  if (runDbTests) await resetVaultTables();
+});
+
+afterEach(() => {
+  restoreEnvValue("CMUX_VAULT_ENABLED", ORIGINAL_ENV.CMUX_VAULT_ENABLED);
+  restoreEnvValue("CMUX_VAULT_S3_BUCKET", ORIGINAL_ENV.CMUX_VAULT_S3_BUCKET);
+  restoreEnvValue("CMUX_VAULT_MAX_UPLOAD_BYTES", ORIGINAL_ENV.CMUX_VAULT_MAX_UPLOAD_BYTES);
+  restoreEnvValue("CMUX_VAULT_MAX_USER_BYTES", ORIGINAL_ENV.CMUX_VAULT_MAX_USER_BYTES);
+});
+
+describe("Vault uploads route", () => {
+  dbTest("restores an existing upload grant when retry presign fails", async () => {
+    const db = cloudDb();
+    const objectKey = realBuildObjectKey(userId, "codex", "session-1", sha256);
+    const previousCreatedAt = new Date("2030-01-01T00:00:00.000Z");
+    const previousExpiresAt = new Date("2030-01-02T00:00:00.000Z");
+    const [previousGrant] = await db
+      .insert(vaultUploadGrants)
+      .values({
+        userId,
+        objectKey,
+        uploadObjectKey: `${objectKey}.previous-upload`,
+        compressedSizeBytes: 123,
+        createdAt: previousCreatedAt,
+        expiresAt: previousExpiresAt,
+      })
+      .returning({ id: vaultUploadGrants.id });
+    expect(previousGrant).toBeDefined();
+
+    presignFailure = new Error("transient presign failure");
+    const response = await POST(uploadRequest({ compressedSizeBytes: 456 }));
+
+    expect(response.status).toBe(200);
+    expect(await response.json()).toEqual({
+      items: [{
+        agent: "codex",
+        agentSessionId: "session-1",
+        relPath: "sessions/session-1.jsonl.zst",
+        status: "error",
+        error: "upload_presign_failed",
+      }],
+    });
+    const rows = await db
+      .select()
+      .from(vaultUploadGrants)
+      .where(eq(vaultUploadGrants.objectKey, objectKey));
+    expect(rows).toHaveLength(1);
+    expect(rows[0].id).toBe(previousGrant!.id);
+    expect(rows[0].compressedSizeBytes).toBe(123);
+    expect(rows[0].uploadObjectKey).toBe(`${objectKey}.previous-upload`);
+    expect(rows[0].createdAt.getTime()).toBe(previousCreatedAt.getTime());
+    expect(rows[0].expiresAt.getTime()).toBe(previousExpiresAt.getTime());
+    const tombstones = await db
+      .select({ id: vaultUploadTombstones.id })
+      .from(vaultUploadTombstones)
+      .where(eq(vaultUploadTombstones.uploadObjectKey, `${objectKey}.previous-upload`));
+    expect(tombstones).toHaveLength(0);
+  });
+
+  dbTest("does not restore an older grant over a newer successful retry", async () => {
+    const db = cloudDb();
+    const objectKey = realBuildObjectKey(userId, "codex", "session-1", sha256);
+    await db
+      .insert(vaultUploadGrants)
+      .values({
+        userId,
+        objectKey,
+        uploadObjectKey: `${objectKey}.previous-upload`,
+        compressedSizeBytes: 123,
+        createdAt: new Date("2030-01-01T00:00:00.000Z"),
+        expiresAt: new Date("2030-01-02T00:00:00.000Z"),
+      });
+
+    beforeNextPresignFailure = async () => {
+      const [staleReservation] = await db
+        .select({
+          createdAt: vaultUploadGrants.createdAt,
+          expiresAt: vaultUploadGrants.expiresAt,
+        })
+        .from(vaultUploadGrants)
+        .where(eq(vaultUploadGrants.objectKey, objectKey))
+        .limit(1);
+      expect(staleReservation).toBeDefined();
+
+      const response = await POST(uploadRequest({ compressedSizeBytes: 789 }));
+      expect(response.status).toBe(200);
+      expect((await response.json()).items[0].status).toBe("upload");
+      await db
+        .update(vaultUploadGrants)
+        .set({
+          createdAt: staleReservation!.createdAt,
+          expiresAt: staleReservation!.expiresAt,
+        })
+        .where(eq(vaultUploadGrants.objectKey, objectKey));
+    };
+    const response = await POST(uploadRequest({ compressedSizeBytes: 456 }));
+
+    expect(response.status).toBe(200);
+    expect((await response.json()).items[0].error).toBe("upload_presign_failed");
+    const rows = await db
+      .select()
+      .from(vaultUploadGrants)
+      .where(eq(vaultUploadGrants.objectKey, objectKey));
+    expect(rows).toHaveLength(1);
+    expect(rows[0].compressedSizeBytes).toBe(789);
+    const tombstones = await db
+      .select({ uploadObjectKey: vaultUploadTombstones.uploadObjectKey })
+      .from(vaultUploadTombstones)
+      .where(eq(vaultUploadTombstones.uploadObjectKey, `${objectKey}.previous-upload`));
+    expect(tombstones).toHaveLength(1);
+  });
+
+  dbTest("mints a fresh staging key and tombstones the active key when retrying an existing grant", async () => {
+    const db = cloudDb();
+    const objectKey = realBuildObjectKey(userId, "codex", "session-1", sha256);
+    const uploadObjectKey = `${objectKey}.active-upload`;
+    const [originalGrant] = await db
+      .insert(vaultUploadGrants)
+      .values({
+        userId,
+        objectKey,
+        uploadObjectKey,
+        compressedSizeBytes: 123,
+        createdAt: new Date("2030-01-01T00:00:00.000Z"),
+        expiresAt: new Date(Date.now() + 60_000),
+      })
+      .returning({ reservationToken: vaultUploadGrants.reservationToken });
+
+    const response = await POST(uploadRequest({ compressedSizeBytes: 456 }));
+
+    expect(response.status).toBe(200);
+    expect((await response.json()).items[0].status).toBe("upload");
+    expect(presignPut).toHaveBeenCalledTimes(1);
+    expect(presignCalls).toHaveLength(1);
+    expect(presignCalls[0].key).not.toBe(uploadObjectKey);
+    expect(presignCalls[0].key).toContain("vault/uploads/");
+    expect(presignCalls[0].contentLength).toBe(456);
+    const rows = await db
+      .select({
+        uploadObjectKey: vaultUploadGrants.uploadObjectKey,
+        reservationToken: vaultUploadGrants.reservationToken,
+      })
+      .from(vaultUploadGrants)
+      .where(eq(vaultUploadGrants.objectKey, objectKey));
+    expect(rows).toHaveLength(1);
+    expect(rows[0].uploadObjectKey).toBe(presignCalls[0].key);
+    expect(rows[0].reservationToken).not.toBe(originalGrant!.reservationToken);
+    const tombstones = await db
+      .select({ uploadObjectKey: vaultUploadTombstones.uploadObjectKey })
+      .from(vaultUploadTombstones)
+      .where(eq(vaultUploadTombstones.uploadObjectKey, uploadObjectKey));
+    expect(tombstones).toHaveLength(1);
+  });
+
+  dbTest("mints a fresh staging key when expired staging cleanup has not completed", async () => {
+    const db = cloudDb();
+    const objectKey = realBuildObjectKey(userId, "codex", "session-1", sha256);
+    const uploadObjectKey = `${objectKey}.expired-upload`;
+    await db.insert(vaultUploadGrants).values({
+      userId,
+      objectKey,
+      uploadObjectKey,
+      compressedSizeBytes: 123,
+      createdAt: new Date("2020-01-01T00:00:00.000Z"),
+      expiresAt: new Date("2020-01-02T00:00:00.000Z"),
+    });
+    beforeNextDelete = async (key) => {
+      expect(key).toBe(uploadObjectKey);
+      throw new Error("storage cleanup failed");
+    };
+
+    const response = await POST(uploadRequest({ compressedSizeBytes: 456 }));
+
+    expect(response.status).toBe(200);
+    expect((await response.json()).items[0].status).toBe("upload");
+    expect(presignPut).toHaveBeenCalledTimes(1);
+    expect(presignCalls).toHaveLength(1);
+    expect(presignCalls[0].key).not.toBe(uploadObjectKey);
+    expect(presignCalls[0].contentLength).toBe(456);
+    const rows = await db
+      .select({ uploadObjectKey: vaultUploadGrants.uploadObjectKey })
+      .from(vaultUploadGrants)
+      .where(eq(vaultUploadGrants.objectKey, objectKey));
+    expect(rows).toHaveLength(1);
+    expect(rows[0].uploadObjectKey).toBe(presignCalls[0].key);
+    const tombstones = await db
+      .select({ uploadObjectKey: vaultUploadTombstones.uploadObjectKey })
+      .from(vaultUploadTombstones)
+      .where(eq(vaultUploadTombstones.uploadObjectKey, uploadObjectKey));
+    expect(tombstones).toHaveLength(1);
+  });
+
+  dbTest("removes a newly-created upload grant when presign fails", async () => {
+    const db = cloudDb();
+    const objectKey = realBuildObjectKey(userId, "codex", "session-1", sha256);
+
+    presignFailure = new Error("transient presign failure");
+    const response = await POST(uploadRequest({ compressedSizeBytes: 456 }));
+
+    expect(response.status).toBe(200);
+    const body = await response.json();
+    expect(body.items[0].error).toBe("upload_presign_failed");
+    const rows = await db
+      .select({ id: vaultUploadGrants.id })
+      .from(vaultUploadGrants)
+      .where(eq(vaultUploadGrants.objectKey, objectKey));
+    expect(rows).toHaveLength(0);
+  });
+
+  dbTest("rejects duplicate object keys before a later entry can overwrite the first grant", async () => {
+    const db = cloudDb();
+    const objectKey = realBuildObjectKey(userId, "codex", "session-1", sha256);
+
+    const response = await POST(duplicateUploadRequest());
+
+    expect(response.status).toBe(200);
+    const body = await response.json();
+    expect(body.items[0]).toMatchObject({
+      agent: "codex",
+      agentSessionId: "session-1",
+      relPath: "sessions/session-1.jsonl.zst",
+      status: "upload",
+      objectKey,
+    });
+    expect(body.items[0].putUrl).toContain("vault%2Fuploads%2F");
+    expect(body.items[0].putUrl).toContain("contentLength=456");
+    expect(body.items[1]).toEqual({
+      agent: "codex",
+      agentSessionId: "session-1",
+      relPath: "sessions/session-1-duplicate.jsonl.zst",
+      status: "error",
+      error: "duplicate_object_key",
+    });
+    expect(body.items).toHaveLength(2);
+    expect(presignPut).toHaveBeenCalledTimes(1);
+    const rows = await db
+      .select({ compressedSizeBytes: vaultUploadGrants.compressedSizeBytes })
+      .from(vaultUploadGrants)
+      .where(eq(vaultUploadGrants.objectKey, objectKey));
+    expect(rows).toHaveLength(1);
+    expect(rows[0].compressedSizeBytes).toBe(456);
+  });
+
+  dbTest("deletes expired staged uploads even when the final object is committed", async () => {
+    const db = cloudDb();
+    const objectKey = realBuildObjectKey(userId, "codex", "session-1", sha256);
+    const uploadObjectKey = `${objectKey}.staged`;
+    const uploadedAt = new Date("2030-01-01T00:00:00.000Z");
+    const [session] = await db
+      .insert(vaultSessions)
+      .values({
+        userId,
+        agent: "codex",
+        agentSessionId: "session-1",
+        relPath: "sessions/session-1.jsonl.zst",
+        cwd: "/workspace",
+        latestSha256: sha256,
+        latestObjectKey: objectKey,
+        sizeBytes: 999,
+        compressedSizeBytes: 456,
+        firstUploadedAt: uploadedAt,
+        lastUploadedAt: uploadedAt,
+        metadata: {},
+      })
+      .returning({ id: vaultSessions.id });
+    await db.insert(vaultSnapshots).values({
+      sessionId: session!.id,
+      sha256,
+      objectKey,
+      sizeBytes: 999,
+      compressedSizeBytes: 456,
+      uploadedAt,
+    });
+    await db.insert(vaultUploadGrants).values({
+      userId,
+      objectKey,
+      uploadObjectKey,
+      compressedSizeBytes: 456,
+      createdAt: new Date("2020-01-01T00:00:00.000Z"),
+      expiresAt: new Date("2020-01-02T00:00:00.000Z"),
+    });
+
+    const response = await POST(uploadRequest({ compressedSizeBytes: 456 }));
+
+    expect(response.status).toBe(200);
+    expect(deleteObject).toHaveBeenCalledWith(uploadObjectKey);
+    const grants = await db
+      .select({ id: vaultUploadGrants.id })
+      .from(vaultUploadGrants)
+      .where(eq(vaultUploadGrants.objectKey, objectKey));
+    expect(grants).toHaveLength(0);
+  });
+
+  dbTest("deletes expired staged uploads and uncommitted copied final objects", async () => {
+    const db = cloudDb();
+    const expiredSha = "c".repeat(64);
+    const objectKey = realBuildObjectKey(userId, "codex", "session-gc", expiredSha);
+    const uploadObjectKey = `${objectKey}.staged`;
+    await db.insert(vaultUploadGrants).values({
+      userId,
+      objectKey,
+      uploadObjectKey,
+      compressedSizeBytes: 456,
+      createdAt: new Date("2020-01-01T00:00:00.000Z"),
+      expiresAt: new Date("2020-01-02T00:00:00.000Z"),
+    });
+
+    const response = await POST(uploadRequest({ compressedSizeBytes: 456 }));
+
+    expect(response.status).toBe(200);
+    expect(deleteObject).toHaveBeenCalledWith(uploadObjectKey);
+    expect(deleteObject).toHaveBeenCalledWith(objectKey);
+    const grants = await db
+      .select({ id: vaultUploadGrants.id })
+      .from(vaultUploadGrants)
+      .where(eq(vaultUploadGrants.objectKey, objectKey));
+    expect(grants).toHaveLength(0);
+  });
+
+  dbTest("deletes expired grants for inactive users during another user's upload", async () => {
+    const db = cloudDb();
+    const inactiveUserId = "inactive-vault-upload-user";
+    const expiredSha = "e".repeat(64);
+    const objectKey = realBuildObjectKey(inactiveUserId, "codex", "inactive-session", expiredSha);
+    const uploadObjectKey = `${objectKey}.staged`;
+    await db.insert(vaultUploadGrants).values({
+      userId: inactiveUserId,
+      objectKey,
+      uploadObjectKey,
+      compressedSizeBytes: 456,
+      createdAt: new Date("2020-01-01T00:00:00.000Z"),
+      expiresAt: new Date("2020-01-02T00:00:00.000Z"),
+    });
+
+    const response = await POST(uploadRequest({ compressedSizeBytes: 456 }));
+
+    expect(response.status).toBe(200);
+    expect(deleteObject).toHaveBeenCalledWith(uploadObjectKey);
+    expect(deleteObject).toHaveBeenCalledWith(objectKey);
+    const grants = await db
+      .select({ id: vaultUploadGrants.id })
+      .from(vaultUploadGrants)
+      .where(eq(vaultUploadGrants.objectKey, objectKey));
+    expect(grants).toHaveLength(0);
+  });
+
+  dbTest("deletes expired tombstoned staged uploads without deleting committed final objects", async () => {
+    const db = cloudDb();
+    const expiredSha = "f".repeat(64);
+    const objectKey = realBuildObjectKey(userId, "codex", "session-tombstone-gc", expiredSha);
+    const uploadObjectKey = `${objectKey}.old-staged`;
+    const uploadedAt = new Date("2030-01-01T00:00:00.000Z");
+    const [session] = await db
+      .insert(vaultSessions)
+      .values({
+        userId,
+        agent: "codex",
+        agentSessionId: "session-tombstone-gc",
+        relPath: "sessions/session-tombstone-gc.jsonl.zst",
+        cwd: "/workspace",
+        latestSha256: expiredSha,
+        latestObjectKey: objectKey,
+        sizeBytes: 999,
+        compressedSizeBytes: 456,
+        firstUploadedAt: uploadedAt,
+        lastUploadedAt: uploadedAt,
+        metadata: {},
+      })
+      .returning({ id: vaultSessions.id });
+    await db.insert(vaultSnapshots).values({
+      sessionId: session!.id,
+      sha256: expiredSha,
+      objectKey,
+      sizeBytes: 999,
+      compressedSizeBytes: 456,
+      uploadedAt,
+    });
+    await db.insert(vaultUploadTombstones).values({
+      userId,
+      objectKey,
+      uploadObjectKey,
+      expiresAt: new Date("2020-01-02T00:00:00.000Z"),
+    });
+
+    const response = await POST(uploadRequest({ compressedSizeBytes: 456 }));
+
+    expect(response.status).toBe(200);
+    expect(deleteObject).toHaveBeenCalledWith(uploadObjectKey);
+    expect(deleteObject).not.toHaveBeenCalledWith(objectKey);
+    const tombstones = await db
+      .select({ id: vaultUploadTombstones.id })
+      .from(vaultUploadTombstones)
+      .where(eq(vaultUploadTombstones.uploadObjectKey, uploadObjectKey));
+    expect(tombstones).toHaveLength(0);
+  }, 10_000);
+
+  dbTest("does not delete an expired grant final object after it becomes committed", async () => {
+    const db = cloudDb();
+    const expiredSha = "d".repeat(64);
+    const objectKey = realBuildObjectKey(userId, "codex", "session-gc-race", expiredSha);
+    const uploadObjectKey = `${objectKey}.staged`;
+    await db.insert(vaultUploadGrants).values({
+      userId,
+      objectKey,
+      uploadObjectKey,
+      compressedSizeBytes: 456,
+      createdAt: new Date("2020-01-01T00:00:00.000Z"),
+      expiresAt: new Date("2020-01-02T00:00:00.000Z"),
+    });
+    beforeNextDelete = async (key) => {
+      expect(key).toBe(uploadObjectKey);
+      const uploadedAt = new Date("2030-01-01T00:00:00.000Z");
+      const [session] = await db
+        .insert(vaultSessions)
+        .values({
+          userId,
+          agent: "codex",
+          agentSessionId: "session-gc-race",
+          relPath: "sessions/session-gc-race.jsonl.zst",
+          cwd: "/workspace",
+          latestSha256: expiredSha,
+          latestObjectKey: objectKey,
+          sizeBytes: 999,
+          compressedSizeBytes: 456,
+          firstUploadedAt: uploadedAt,
+          lastUploadedAt: uploadedAt,
+          metadata: {},
+        })
+        .returning({ id: vaultSessions.id });
+      await db.insert(vaultSnapshots).values({
+        sessionId: session!.id,
+        sha256: expiredSha,
+        objectKey,
+        sizeBytes: 999,
+        compressedSizeBytes: 456,
+        uploadedAt,
+      });
+    };
+
+    const response = await POST(uploadRequest({ compressedSizeBytes: 456 }));
+
+    expect(response.status).toBe(200);
+    expect(deleteObject).toHaveBeenCalledWith(uploadObjectKey);
+    expect(deleteObject).not.toHaveBeenCalledWith(objectKey);
+    const grants = await db
+      .select({ id: vaultUploadGrants.id })
+      .from(vaultUploadGrants)
+      .where(eq(vaultUploadGrants.objectKey, objectKey));
+    expect(grants).toHaveLength(0);
+  }, 10_000);
+});
+
+function uploadRequest(input: { readonly compressedSizeBytes: number }): Request {
+  return new Request("https://cmux.test/api/vault/uploads", {
+    method: "POST",
+    headers: {
+      authorization: "Bearer access-token",
+      "x-stack-refresh-token": "refresh-token",
+      "content-type": "application/json",
+    },
+    body: JSON.stringify({
+      items: [{
+        agent: "codex",
+        agentSessionId: "session-1",
+        relPath: "sessions/session-1.jsonl.zst",
+        cwd: "/workspace",
+        sha256,
+        sizeBytes: 999,
+        compressedSizeBytes: input.compressedSizeBytes,
+      }],
+    }),
+  });
+}
+
+function duplicateUploadRequest(): Request {
+  return new Request("https://cmux.test/api/vault/uploads", {
+    method: "POST",
+    headers: {
+      authorization: "Bearer access-token",
+      "x-stack-refresh-token": "refresh-token",
+      "content-type": "application/json",
+    },
+    body: JSON.stringify({
+      items: [
+        {
+          agent: "codex",
+          agentSessionId: "session-1",
+          relPath: "sessions/session-1.jsonl.zst",
+          cwd: "/workspace",
+          sha256,
+          sizeBytes: 999,
+          compressedSizeBytes: 456,
+        },
+        {
+          agent: "codex",
+          agentSessionId: "session-1",
+          relPath: "sessions/session-1-duplicate.jsonl.zst",
+          cwd: "/workspace",
+          sha256,
+          sizeBytes: 999,
+          compressedSizeBytes: 789,
+        },
+      ],
+    }),
+  });
+}
+
+async function resetVaultTables(): Promise<void> {
+  await cloudDb().execute(sql`
+    truncate vault_snapshots, vault_sessions, vault_upload_grants, vault_upload_tombstones restart identity cascade
+  `);
+}
+
+function stackUser() {
+  return {
+    id: userId,
+    displayName: null,
+    primaryEmail: "[email protected]",
+    selectedTeam: null,
+    clientReadOnlyMetadata: {},
+    listTeams: async () => [],
+  };
+}
+
+function restoreEnvValue(key: string, value: string | undefined): void {
+  if (value === undefined) {
+    delete process.env[key];
+    return;
+  }
+  process.env[key] = value;
+}

web/tests/vault-usage.test.ts

@@ -0,0 +1,31 @@
+import { describe, expect, mock, test } from "bun:test";
+import { withVaultUserQuotaLock } from "../services/vault/usage";
+
+describe("vault usage quota locking", () => {
+  test("runs quota projection and grant reservation inside a per-user advisory transaction", async () => {
+    const tx = {
+      execute: mock(async () => undefined),
+    };
+    const db = {
+      transaction: mock(async (...args: unknown[]) => {
+        const run = args[0] as (tx: unknown) => Promise<string>;
+        return await run(tx);
+      }),
+    };
+    const run = mock(async (lockedDb: unknown) => {
+      expect(lockedDb).toBe(tx);
+      return "reserved";
+    });
+
+    const result = await withVaultUserQuotaLock(
+      db as never,
+      "user-quota-lock",
+      run as never,
+    );
+
+    expect(result).toBe("reserved");
+    expect(db.transaction).toHaveBeenCalledTimes(1);
+    expect(tx.execute).toHaveBeenCalledTimes(2);
+    expect(run).toHaveBeenCalledTimes(1);
+  });
+});

web/tests/vercel-firewall-mock.ts

@@ -0,0 +1,17 @@
+import { mock } from "bun:test";
+
+type RateLimitResult = {
+  readonly rateLimited: boolean;
+  readonly error: string | null;
+};
+
+export const checkRateLimit = mock(async (): Promise<RateLimitResult> => ({
+  rateLimited: false,
+  error: null,
+}));
+
+export function installVercelFirewallMock(): void {
+  mock.module("@vercel/firewall", () => ({
+    checkRateLimit,
+  }));
+}

web/tests/vm-freestyle-provider.test.ts

@@ -117,4 +117,50 @@ describe("FreestyleProvider attach fallback", () => {
     expect(endpoint).toEqual(endpointWithDaemon);
     expect(provider.sshCalls).toBe(0);
   });
+
+  test("keeps daemon attach when Freestyle exec probe fails but websocket admin is healthy", async () => {
+    const originalFetch = globalThis.fetch;
+    const originalApiKey = process.env.FREESTYLE_API_KEY;
+    process.env.FREESTYLE_API_KEY = "test-freestyle-api-key";
+    const urls: string[] = [];
+    globalThis.fetch = (async (input, init) => {
+      const url = input instanceof Request ? input.url : String(input);
+      urls.push(url);
+      if (url === "https://vm-1.vm.freestyle.sh/healthz") {
+        return new Response("ok", { status: 200 });
+      }
+      if (url === "https://vm-1.vm.freestyle.sh/admin/leases") {
+        expect(init?.method).toBe("POST");
+        return new Response("ok", { status: 200 });
+      }
+      return new Response(JSON.stringify({ error: "INTERNAL_ERROR", message: "Internal server error" }), {
+        status: 500,
+        headers: { "content-type": "application/json" },
+      });
+    }) as typeof fetch;
+
+    try {
+      const provider = new FreestyleProvider();
+      const endpoint = await provider.openAttach("vm-1", {
+        requireDaemon: true,
+        providerMetadata: { freestyleDaemonAdminToken: "admin-token" },
+      });
+
+      expect(endpoint.transport).toBe("websocket");
+      if (endpoint.transport !== "websocket") {
+        throw new Error("expected websocket attach endpoint");
+      }
+      expect(endpoint.url).toBe("wss://vm-1.vm.freestyle.sh/terminal");
+      expect(endpoint.daemon?.url).toBe("wss://vm-1.vm.freestyle.sh/rpc");
+      expect(urls).toContain("https://vm-1.vm.freestyle.sh/healthz");
+      expect(urls).toContain("https://vm-1.vm.freestyle.sh/admin/leases");
+    } finally {
+      globalThis.fetch = originalFetch;
+      if (originalApiKey === undefined) {
+        delete process.env.FREESTYLE_API_KEY;
+      } else {
+        process.env.FREESTYLE_API_KEY = originalApiKey;
+      }
+    }
+  });
 });

web/tests/vm-lease-cron-route.test.ts

@@ -0,0 +1,46 @@
+import { afterEach, describe, expect, test } from "bun:test";
+
+const originalCronSecret = process.env.CRON_SECRET;
+
+const route = await import("../app/api/internal/vm/leases/revoke-expired/route");
+
+afterEach(() => {
+  restoreEnv("CRON_SECRET", originalCronSecret);
+});
+
+describe("VM expired lease cron route", () => {
+  test("does not expose the secret env var name when the cron secret is missing", async () => {
+    delete process.env.CRON_SECRET;
+
+    const response = await route.POST(
+      new Request("https://cmux.test/api/internal/vm/leases/revoke-expired", {
+        method: "POST",
+      }),
+    );
+
+    expect(response.status).toBe(503);
+    expect(await response.json()).toEqual({ error: "service_unavailable" });
+  });
+
+  test("requires the configured cron secret before revoking expired leases", async () => {
+    process.env.CRON_SECRET = "cron-secret";
+
+    const response = await route.POST(
+      new Request("https://cmux.test/api/internal/vm/leases/revoke-expired", {
+        method: "POST",
+        headers: { authorization: "Bearer wrong" },
+      }),
+    );
+
+    expect(response.status).toBe(401);
+    expect(await response.json()).toEqual({ error: "unauthorized" });
+  });
+});
+
+function restoreEnv(key: string, value: string | undefined): void {
+  if (typeof value === "undefined") {
+    delete process.env[key];
+  } else {
+    process.env[key] = value;
+  }
+}

web/tests/vm-provider-errors.test.ts

@@ -0,0 +1,30 @@
+import { describe, expect, test } from "bun:test";
+import {
+  isProviderIdentityNotFoundError,
+  isProviderNotFoundError,
+} from "../services/vms/providerErrors";
+
+describe("provider error classification", () => {
+  test("keeps identity deletion errors out of VM not-found classification", () => {
+    expect(isProviderNotFoundError(new Error("identity does not exist"))).toBe(false);
+    expect(isProviderIdentityNotFoundError(new Error("identity does not exist"))).toBe(true);
+  });
+
+  test("keeps VM deletion errors in VM not-found classification", () => {
+    expect(isProviderNotFoundError(new Error("VM does not exist"))).toBe(true);
+    expect(isProviderNotFoundError(new Error("sandbox has been deleted"))).toBe(true);
+  });
+
+  test("recognizes provider identity missing errors in nested response bodies", () => {
+    const err = {
+      response: {
+        data: {
+          error: "requested credential was not found",
+        },
+      },
+    };
+
+    expect(isProviderIdentityNotFoundError(err)).toBe(true);
+    expect(isProviderNotFoundError(err)).toBe(false);
+  });
+});

web/tests/vm-route-auth.test.ts

@@ -786,6 +786,7 @@ describe("VM REST auth", () => {
     expect(getVm).toHaveBeenCalledWith({
       userId: "user-1",
       billingTeamId: "team-1",
+      teamIds: ["team-1"],
       providerVmId: "provider-vm-team-1",
     });
 
@@ -800,6 +801,7 @@ describe("VM REST auth", () => {
     expect(destroyVm).toHaveBeenCalledWith({
       userId: "user-1",
       billingTeamId: "team-1",
+      teamIds: ["team-1"],
       providerVmId: "provider-vm-team-1",
     });
 
@@ -823,6 +825,7 @@ describe("VM REST auth", () => {
     expect(openAttachEndpoint).toHaveBeenCalledWith(expect.objectContaining({
       userId: "user-1",
       billingTeamId: "team-1",
+      teamIds: ["team-1"],
       providerVmId: "provider-vm-team-1",
     }));
 
@@ -844,6 +847,7 @@ describe("VM REST auth", () => {
     expect(openSshEndpoint).toHaveBeenCalledWith({
       userId: "user-1",
       billingTeamId: "team-1",
+      teamIds: ["team-1"],
       providerVmId: "provider-vm-team-1",
     });
 
@@ -859,6 +863,7 @@ describe("VM REST auth", () => {
     expect(execVm).toHaveBeenCalledWith({
       userId: "user-1",
       billingTeamId: "team-1",
+      teamIds: ["team-1"],
       providerVmId: "provider-vm-team-1",
       command: "true",
       timeoutMs: 30_000,

web/tests/vm-workflows.test.ts

@@ -15,6 +15,8 @@ import {
   FAILED_CREATE_RETRY_WINDOW_MS,
   VmRepository,
   VmRepositoryLive,
+  type CloudVmIdentityLeaseRow,
+  type CloudVmLeaseRow,
   type CloudVmSessionRow,
   type CloudVmRow,
   type VmRepositoryShape,
@@ -37,6 +39,7 @@ import {
   openBaseVm,
   openAttachEndpoint,
   openSshEndpoint,
+  revokeExpiredIdentityLeases,
   resetBaseVm,
   restoreVm,
   reconcileVmProviderStatuses,
@@ -50,6 +53,7 @@ let sql: Sql | null = null;
 type RecordedUsageEvent = Parameters<VmRepositoryShape["recordUsageEvent"]>[0];
 type RecordedLease = Parameters<VmRepositoryShape["recordLease"]>[0];
 type ObservedStatusUpdate = Parameters<VmRepositoryShape["markProviderObservedStatus"]>[0];
+type LeaseRevocationRetry = Parameters<NonNullable<VmRepositoryShape["markLeaseRevocationRetry"]>>[0];
 
 function databaseURL() {
   const url = process.env.DIRECT_DATABASE_URL ?? process.env.DATABASE_URL;
@@ -133,6 +137,7 @@ describe("VM Effect workflows", () => {
     const result = await Effect.runPromise(
       execVm({
         userId: "user-workflow-exec-resume",
+        teamIds: ["team-workflow-exec-resume"],
         providerVmId: "provider-vm-exec-resume",
         command: "echo preflight",
         timeoutMs: 1000,
@@ -297,6 +302,407 @@ describe("VM Effect workflows", () => {
     expect(resumeCalls).toBe(1);
   });
 
+  test("does not sweep expired identity leases during user VM exec", async () => {
+    const vm = testCloudVmRow({
+      id: "00000000-0000-4000-8000-000000000116",
+      userId: "user-workflow-cleanup-owner",
+      providerVmId: "provider-vm-cleanup-owner",
+      status: "running",
+    });
+    let sweepCalls = 0;
+    const repo = testWorkflowRepo({
+      vm,
+      expiredIdentityLeases: () =>
+        Effect.sync(() => {
+          sweepCalls += 1;
+          return [];
+        }),
+    });
+    const provider: VmProviderGatewayShape = {
+      ...unusedProviderGateway(),
+      exec: () => Effect.succeed({ exitCode: 0, stdout: "", stderr: "" }),
+    };
+
+    const result = await Effect.runPromise(
+      execVm({
+        userId: "user-workflow-cleanup-owner",
+        providerVmId: "provider-vm-cleanup-owner",
+        command: "true",
+        timeoutMs: 1000,
+      }).pipe(Effect.provide(workflowLayer(repo, provider))),
+    );
+
+    expect(result).toEqual({ exitCode: 0, stdout: "", stderr: "" });
+    expect(sweepCalls).toBe(0);
+  });
+
+  test("destroyVm fails closed when active identity cleanup fails", async () => {
+    const vm = testCloudVmRow({
+      id: "00000000-0000-4000-8000-000000000130",
+      userId: "user-workflow-destroy-cleanup-bound",
+      providerVmId: "provider-vm-destroy-cleanup-bound",
+      status: "running",
+    });
+    const activeIdentityLeases: CloudVmLeaseRow[] = Array.from({ length: 8 }, (_, index) => ({
+      id: `lease-destroy-cleanup-${index}`,
+      vmId: vm.id,
+      userId: vm.userId,
+      kind: "ssh",
+      tokenHash: `destroy-cleanup-${index}`,
+      providerIdentityHandle: `identity-destroy-cleanup-${index}`,
+      sessionId: null,
+      transport: "ssh",
+      metadata: {},
+      expiresAt: new Date(Date.now() + 60_000),
+      consumedAt: null,
+      revokedAt: null,
+      createdAt: new Date(Date.now() + index),
+    }));
+    const repo = testWorkflowRepo({ vm, activeIdentityLeases });
+    let revokeCalls = 0;
+    let destroyCalls = 0;
+    const provider: VmProviderGatewayShape = {
+      ...unusedProviderGateway(),
+      revokeSSHIdentity: () => {
+        revokeCalls += 1;
+        return Effect.fail(providerOperationError("revokeSSHIdentity", "provider delete failed"));
+      },
+      destroy: () =>
+        Effect.sync(() => {
+          destroyCalls += 1;
+        }),
+    };
+
+    await expect(
+      Effect.runPromise(
+        destroyVm({
+          userId: "user-workflow-destroy-cleanup-bound",
+          providerVmId: "provider-vm-destroy-cleanup-bound",
+        }).pipe(Effect.provide(workflowLayer(repo, provider))),
+      ),
+    ).rejects.toThrow();
+
+    expect(revokeCalls).toBe(1);
+    expect(destroyCalls).toBe(0);
+  });
+
+  test("destroyVm fails closed when active identity cleanup exceeds the hot-path cap", async () => {
+    const vm = testCloudVmRow({
+      id: "00000000-0000-4000-8000-000000000132",
+      userId: "user-workflow-destroy-cleanup-cap",
+      providerVmId: "provider-vm-destroy-cleanup-cap",
+      status: "running",
+    });
+    const activeIdentityLeases: CloudVmLeaseRow[] = Array.from({ length: 9 }, (_, index) => ({
+      id: `lease-destroy-cleanup-cap-${index}`,
+      vmId: vm.id,
+      userId: vm.userId,
+      kind: "ssh",
+      tokenHash: `destroy-cleanup-cap-${index}`,
+      providerIdentityHandle: `identity-destroy-cleanup-cap-${index}`,
+      sessionId: null,
+      transport: "ssh",
+      metadata: {},
+      expiresAt: new Date(Date.now() + 60_000),
+      consumedAt: null,
+      revokedAt: null,
+      createdAt: new Date(Date.now() + index),
+    }));
+    const repo = testWorkflowRepo({ vm, activeIdentityLeases });
+    let revokeCalls = 0;
+    let destroyCalls = 0;
+    const provider: VmProviderGatewayShape = {
+      ...unusedProviderGateway(),
+      revokeSSHIdentity: () =>
+        Effect.sync(() => {
+          revokeCalls += 1;
+        }),
+      destroy: () =>
+        Effect.sync(() => {
+          destroyCalls += 1;
+        }),
+    };
+
+    await expect(
+      Effect.runPromise(
+        destroyVm({
+          userId: "user-workflow-destroy-cleanup-cap",
+          providerVmId: "provider-vm-destroy-cleanup-cap",
+        }).pipe(Effect.provide(workflowLayer(repo, provider))),
+      ),
+    ).rejects.toThrow();
+
+    expect(revokeCalls).toBe(0);
+    expect(destroyCalls).toBe(0);
+  });
+
+  test("revokeExpiredIdentityLeases uses a small default cron batch", async () => {
+    const vm = testCloudVmRow({
+      id: "00000000-0000-4000-8000-000000000131",
+      userId: "user-workflow-expired-default-limit",
+      providerVmId: "provider-vm-expired-default-limit",
+      status: "running",
+    });
+    let requestedLimit = 0;
+    const repo = testWorkflowRepo({
+      vm,
+      expiredIdentityLeases: (input) =>
+        Effect.sync(() => {
+          requestedLimit = input.limit;
+          return [];
+        }),
+    });
+
+    const revoked = await Effect.runPromise(
+      revokeExpiredIdentityLeases().pipe(
+        Effect.provide(workflowLayer(repo, unusedProviderGateway())),
+      ),
+    );
+
+    expect(revoked).toBe(0);
+    expect(requestedLimit).toBe(5);
+  });
+
+  test("marks expired identity leases revoked when the provider identity is already gone", async () => {
+    const now = new Date();
+    const vm = testCloudVmRow({
+      id: "00000000-0000-4000-8000-000000000117",
+      userId: "user-workflow-expired-identity",
+      providerVmId: "provider-vm-expired-identity",
+      status: "running",
+    });
+    const lease: CloudVmIdentityLeaseRow = {
+      id: "lease-expired-identity",
+      vmId: vm.id,
+      userId: vm.userId,
+      kind: "ssh",
+      tokenHash: "expired-token-hash",
+      providerIdentityHandle: "identity-already-gone",
+      sessionId: null,
+      transport: "ssh",
+      metadata: {},
+      expiresAt: new Date(now.getTime() - 1000),
+      consumedAt: null,
+      revokedAt: null,
+      createdAt: new Date(now.getTime() - 2000),
+      provider: "freestyle",
+    };
+    const revokedLeaseIds: string[] = [];
+    const repo = testWorkflowRepo({
+      vm,
+      expiredIdentityLeases: () => Effect.succeed([lease]),
+      revokedLeaseIds,
+    });
+    const provider: VmProviderGatewayShape = {
+      ...unusedProviderGateway(),
+      revokeSSHIdentity: () =>
+        Effect.fail(providerOperationError("revokeSSHIdentity", "identity not found")),
+    };
+
+    const revoked = await Effect.runPromise(
+      revokeExpiredIdentityLeases({ now, limit: 1 }).pipe(
+        Effect.provide(workflowLayer(repo, provider)),
+      ),
+    );
+
+    expect(revoked).toBe(1);
+    expect(revokedLeaseIds).toEqual(["lease-expired-identity"]);
+  });
+
+  test("keeps expired identity leases retryable when provider revocation fails", async () => {
+    const now = new Date();
+    const vm = testCloudVmRow({
+      id: "00000000-0000-4000-8000-000000000119",
+      userId: "user-workflow-expired-identity-failure",
+      providerVmId: "provider-vm-expired-identity-failure",
+      status: "running",
+    });
+    const lease: CloudVmIdentityLeaseRow = {
+      id: "lease-expired-identity-failure",
+      vmId: vm.id,
+      userId: vm.userId,
+      kind: "ssh",
+      tokenHash: "expired-token-hash-failure",
+      providerIdentityHandle: "identity-still-live",
+      sessionId: null,
+      transport: "ssh",
+      metadata: {},
+      expiresAt: new Date(now.getTime() - 1000),
+      consumedAt: null,
+      revokedAt: null,
+      createdAt: new Date(now.getTime() - 2000),
+      provider: "freestyle",
+    };
+    const revokedLeaseIds: string[] = [];
+    const leaseRevocationRetries: LeaseRevocationRetry[] = [];
+    const repo = testWorkflowRepo({
+      vm,
+      expiredIdentityLeases: () => Effect.succeed([lease]),
+      revokedLeaseIds,
+      leaseRevocationRetries,
+    });
+    const provider: VmProviderGatewayShape = {
+      ...unusedProviderGateway(),
+      revokeSSHIdentity: () => {
+        expect(leaseRevocationRetries).toHaveLength(1);
+        expect(leaseRevocationRetries[0]).toMatchObject({ id: lease.id, error: "revoke pending" });
+        return Effect.fail(providerOperationError("revokeSSHIdentity", "provider delete failed"));
+      },
+    };
+
+    const revoked = await Effect.runPromise(
+      revokeExpiredIdentityLeases({ now, limit: 1 }).pipe(
+        Effect.provide(workflowLayer(repo, provider)),
+      ),
+    );
+
+    expect(revoked).toBe(0);
+    expect(revokedLeaseIds).toEqual([]);
+    expect(leaseRevocationRetries).toHaveLength(1);
+  });
+
+  dbTest("backs off failed expired identity cleanup so later leases progress", async () => {
+    if (!sql) throw new Error("test database not initialized");
+    await sql`truncate cloud_vm_billing_grants, cloud_vm_usage_events, cloud_vm_leases, cloud_vms restart identity cascade`;
+    const now = new Date("2026-01-01T00:00:00.000Z");
+    const [vm] = await sql<{ id: string }[]>`
+      insert into cloud_vms (
+        user_id, provider, provider_vm_id, image_id, status, created_at, updated_at
+      )
+      values (
+        'user-expired-starvation',
+        'freestyle',
+        'provider-expired-starvation',
+        'snapshot-test',
+        'running',
+        ${new Date(now.getTime() - 60_000)},
+        ${new Date(now.getTime() - 60_000)}
+      )
+      returning id
+    `;
+    await sql`
+      insert into cloud_vm_leases (
+        vm_id, user_id, kind, token_hash, provider_identity_handle, transport, expires_at, created_at
+      )
+      values
+        (
+          ${vm.id},
+          'user-expired-starvation',
+          'ssh',
+          'expired-starvation-fail',
+          'identity-delete-fails',
+          'ssh',
+          ${new Date(now.getTime() - 2_000)},
+          ${new Date(now.getTime() - 2_000)}
+        ),
+        (
+          ${vm.id},
+          'user-expired-starvation',
+          'ssh',
+          'expired-starvation-later',
+          'identity-delete-later',
+          'ssh',
+          ${new Date(now.getTime() - 1_000)},
+          ${new Date(now.getTime() - 1_000)}
+        )
+    `;
+    const revokeCalls: string[] = [];
+    const provider: VmProviderGatewayShape = {
+      ...unusedProviderGateway(),
+      revokeSSHIdentity: (_provider, handle) => {
+        revokeCalls.push(handle);
+        if (handle === "identity-delete-fails") {
+          return Effect.fail(providerOperationError("revokeSSHIdentity", "provider delete failed"));
+        }
+        return Effect.void;
+      },
+    };
+
+    const first = await Effect.runPromise(
+      revokeExpiredIdentityLeases({ now, limit: 1 }).pipe(
+        Effect.provide(providerLayer(provider)),
+      ),
+    );
+    const second = await Effect.runPromise(
+      revokeExpiredIdentityLeases({ now, limit: 1 }).pipe(
+        Effect.provide(providerLayer(provider)),
+      ),
+    );
+
+    expect(first).toBe(0);
+    expect(second).toBe(1);
+    expect(revokeCalls).toEqual(["identity-delete-fails", "identity-delete-later"]);
+    const [failed] = await sql<{ retryAfter: string | null; attempts: string | null }[]>`
+      select
+        metadata->>'identityCleanupRetryAfter' as "retryAfter",
+        metadata->>'identityCleanupAttempts' as attempts
+      from cloud_vm_leases
+      where token_hash = 'expired-starvation-fail'
+    `;
+    const [later] = await sql<{ revokedAt: Date | null }[]>`
+      select revoked_at as "revokedAt"
+      from cloud_vm_leases
+      where token_hash = 'expired-starvation-later'
+    `;
+    expect(failed.retryAfter).toBeTruthy();
+    expect(failed.attempts).toBe("1");
+    expect(later.revokedAt).toBeInstanceOf(Date);
+  });
+
+  test("fails closed when team-scoped VM access omits team membership context", async () => {
+    const vm = testCloudVmRow({
+      id: "00000000-0000-4000-8000-000000000118",
+      userId: "user-workflow-team-context",
+      billingTeamId: "team-workflow-team-context",
+      providerVmId: "provider-vm-team-context",
+      status: "running",
+    });
+    const repo = testWorkflowRepo({ vm });
+    const provider: VmProviderGatewayShape = {
+      ...unusedProviderGateway(),
+      exec: () => Effect.succeed({ exitCode: 0, stdout: "", stderr: "" }),
+    };
+
+    const error = await Effect.runPromise(
+      execVm({
+        userId: "user-workflow-team-context",
+        billingTeamId: "team-workflow-team-context",
+        providerVmId: "provider-vm-team-context",
+        command: "true",
+        timeoutMs: 1000,
+      }).pipe(Effect.flip, Effect.provide(workflowLayer(repo, provider))),
+    );
+
+    expect(error).toBeInstanceOf(VmNotFoundError);
+  });
+
+  test("fails closed when personal-scoped access omits team membership context for a team VM", async () => {
+    const vm = testCloudVmRow({
+      id: "00000000-0000-4000-8000-000000000129",
+      userId: "user-workflow-team-context-personal",
+      billingTeamId: "team-workflow-team-context-personal",
+      providerVmId: "provider-vm-team-context-personal",
+      status: "running",
+    });
+    const repo = testWorkflowRepo({ vm });
+    const provider: VmProviderGatewayShape = {
+      ...unusedProviderGateway(),
+      exec: () => Effect.succeed({ exitCode: 0, stdout: "", stderr: "" }),
+    };
+
+    const error = await Effect.runPromise(
+      execVm({
+        userId: "user-workflow-team-context-personal",
+        billingTeamId: null,
+        providerVmId: "provider-vm-team-context-personal",
+        command: "true",
+        timeoutMs: 1000,
+      }).pipe(Effect.flip, Effect.provide(workflowLayer(repo, provider))),
+    );
+
+    expect(error).toBeInstanceOf(VmNotFoundError);
+  });
+
   test("exec failure without gateway getStatus propagates the original error", async () => {
     const vm = testCloudVmRow({
       id: "00000000-0000-4000-8000-000000000104",
@@ -485,7 +891,6 @@ describe("VM Effect workflows", () => {
     const leases: RecordedLease[] = [];
     const observedStatuses: ObservedStatusUpdate[] = [];
     const repo = testWorkflowRepo({ vm, usageEvents, leases, observedStatuses });
-    const originalError = providerOperationError("openAttach", "provider attach unavailable");
     const endpoint = testAttachEndpoint();
     let attachCalls = 0;
     let statusCalls = 0;
@@ -512,6 +917,7 @@ describe("VM Effect workflows", () => {
     const result = await Effect.runPromise(
       openAttachEndpoint({
         userId: "user-workflow-attach-resume",
+        teamIds: ["team-workflow-attach-resume"],
         providerVmId: "provider-vm-attach-resume",
         options: { requireDaemon: true },
       }).pipe(Effect.provide(workflowLayer(repo, provider))),
@@ -585,6 +991,7 @@ describe("VM Effect workflows", () => {
     const error = await Effect.runPromise(
       openAttachEndpoint({
         userId: "user-workflow-attach-mark-fails",
+        teamIds: ["team-workflow-attach-mark-fails"],
         providerVmId: "provider-vm-attach-mark-fails",
       }).pipe(Effect.flip, Effect.provide(workflowLayer(repo, provider))),
     );
@@ -646,6 +1053,7 @@ describe("VM Effect workflows", () => {
     const error = await Effect.runPromise(
       openAttachEndpoint({
         userId: "user-workflow-attach-mark-false",
+        teamIds: ["team-workflow-attach-mark-false"],
         providerVmId: "provider-vm-attach-mark-false",
       }).pipe(Effect.flip, Effect.provide(workflowLayer(repo, provider))),
     );
@@ -671,7 +1079,6 @@ describe("VM Effect workflows", () => {
     const leases: RecordedLease[] = [];
     const observedStatuses: ObservedStatusUpdate[] = [];
     const repo = testWorkflowRepo({ vm, usageEvents, leases, observedStatuses });
-    const originalError = providerOperationError("openSSH", "provider ssh unavailable");
     const endpoint = testSshEndpoint();
     let sshCalls = 0;
     let statusCalls = 0;
@@ -698,6 +1105,7 @@ describe("VM Effect workflows", () => {
     const result = await Effect.runPromise(
       openSshEndpoint({
         userId: "user-workflow-ssh-resume",
+        teamIds: ["team-workflow-ssh-resume"],
         providerVmId: "provider-vm-ssh-resume",
       }).pipe(Effect.provide(workflowLayer(repo, provider))),
     );
@@ -718,6 +1126,82 @@ describe("VM Effect workflows", () => {
     });
   });
 
+  test("openSshEndpoint does not pause a preflight-resumed VM when cleanup fails before minting", async () => {
+    const vm = testCloudVmRow({
+      id: "00000000-0000-4000-8000-000000000128",
+      userId: "user-workflow-ssh-cleanup-before-resume",
+      providerVmId: "provider-vm-ssh-cleanup-before-resume",
+      status: "paused",
+    });
+    const activeLease: CloudVmLeaseRow = {
+      id: "lease-active-cleanup-before-resume",
+      vmId: vm.id,
+      userId: vm.userId,
+      kind: "ssh",
+      tokenHash: "active-cleanup-before-resume",
+      providerIdentityHandle: "identity-cleanup-before-resume",
+      sessionId: null,
+      transport: "ssh",
+      metadata: {},
+      expiresAt: new Date(Date.now() + 60_000),
+      consumedAt: null,
+      revokedAt: null,
+      createdAt: new Date(),
+    };
+    const observedStatuses: ObservedStatusUpdate[] = [];
+    const repo = testWorkflowRepo({ vm, activeIdentityLeases: [activeLease], observedStatuses });
+    let statusCalls = 0;
+    let resumeCalls = 0;
+    let pauseCalls = 0;
+    let openCalls = 0;
+    const provider: VmProviderGatewayShape = {
+      ...unusedProviderGateway(),
+      getStatus: () => {
+        statusCalls += 1;
+        return Effect.succeed("paused");
+      },
+      resume: () => {
+        resumeCalls += 1;
+        return Effect.succeed(testVmHandle({ providerVmId: vm.providerVmId! }));
+      },
+      pause: () =>
+        Effect.sync(() => {
+          pauseCalls += 1;
+        }),
+      revokeSSHIdentity: () =>
+        Effect.fail(providerOperationError("revokeSSHIdentity", "provider delete failed")),
+      openSSH: () => {
+        openCalls += 1;
+        return Effect.succeed({
+          transport: "ssh" as const,
+          host: "vm-ssh.freestyle.sh",
+          port: 22,
+          username: "provider-vm-ssh-cleanup-before-resume+cmux",
+          publicKeyFingerprint: null,
+          credential: { kind: "password" as const, value: "secret" },
+          identityHandle: "new-identity",
+        });
+      },
+    };
+
+    await expect(
+      Effect.runPromise(
+        openSshEndpoint({
+          userId: vm.userId,
+          providerVmId: vm.providerVmId!,
+        }).pipe(Effect.provide(workflowLayer(repo, provider))),
+      ),
+    ).rejects.toThrow();
+
+    expect(statusCalls).toBe(1);
+    expect(resumeCalls).toBe(1);
+    expect(pauseCalls).toBe(0);
+    expect(openCalls).toBe(0);
+    expect(observedStatuses).toEqual([
+      { id: vm.id, providerVmId: vm.providerVmId!, status: "running" },
+    ]);
+  });
+
   test("openAttachEndpoint recovers when the VM suspends between preflight and minting", async () => {
     const vm = testCloudVmRow({
       id: "00000000-0000-4000-8000-000000000112",
@@ -759,6 +1243,7 @@ describe("VM Effect workflows", () => {
     const result = await Effect.runPromise(
       openAttachEndpoint({
         userId: "user-workflow-attach-race",
+        teamIds: ["team-workflow-attach-race"],
         providerVmId: "provider-vm-attach-race",
       }).pipe(Effect.provide(workflowLayer(repo, provider))),
     );
@@ -806,6 +1291,7 @@ describe("VM Effect workflows", () => {
     const error = await Effect.runPromise(
       openAttachEndpoint({
         userId: "user-workflow-attach-probe-fail",
+        teamIds: ["team-workflow-attach-probe-fail"],
         providerVmId: "provider-vm-attach-probe-fail",
       }).pipe(Effect.flip, Effect.provide(workflowLayer(repo, provider))),
     );
@@ -856,6 +1342,7 @@ describe("VM Effect workflows", () => {
     const result = await Effect.runPromise(
       execVm({
         userId: "user-workflow-exec-settle",
+        teamIds: ["team-workflow-exec-settle"],
         providerVmId: "provider-vm-exec-settle",
         command: "echo ok",
         timeoutMs: 1000,
@@ -907,6 +1394,7 @@ describe("VM Effect workflows", () => {
     const result = await Effect.runPromise(
       execVm({
         userId: "user-workflow-exec-concurrent",
+        teamIds: ["team-workflow-exec-concurrent"],
         providerVmId: "provider-vm-exec-concurrent",
         command: "echo ok",
         timeoutMs: 1000,
@@ -1193,6 +1681,103 @@ describe("VM Effect workflows", () => {
     ]);
   });
 
+  dbTest("reopens Base with a new generation when the active provider VM was deleted", async () => {
+    if (!sql) throw new Error("test database not initialized");
+    await sql`truncate cloud_vm_billing_grants, cloud_vm_usage_events, cloud_vm_leases, cloud_vms restart identity cascade`;
+
+    let createCalls = 0;
+    let statusCalls = 0;
+    const provider: VmProviderGatewayShape = {
+      create: () =>
+        Effect.sync(() => {
+          createCalls += 1;
+          return {
+            provider: "freestyle" as const,
+            providerVmId: `provider-vm-base-reopen-${createCalls}`,
+            status: "running" as const,
+            image: "snapshot-test",
+            createdAt: Date.now(),
+          };
+        }),
+      destroy: () => Effect.void,
+      exec: () => Effect.succeed({ exitCode: 0, stdout: "", stderr: "" }),
+      openAttach: () => Effect.fail(new Error("unused") as never),
+      openSSH: () => Effect.fail(new Error("unused") as never),
+      revokeSSHIdentity: () => Effect.void,
+      getStatus: (_provider, providerVmId) =>
+        Effect.suspend(() => {
+          statusCalls += 1;
+          const deleted = new Error(`VM_DELETED: Vm ${providerVmId} is marked as deleted but still exists in the database`);
+          deleted.name = "VmDeletedError";
+          return Effect.fail(new VmProviderOperationError({
+            provider: "freestyle",
+            operation: "getStatus",
+            cause: deleted,
+          }));
+        }),
+    };
+    const layer = providerLayer(provider);
+
+    const first = await Effect.runPromise(openBaseVm({
+      userId: "user-base-reopen-deleted",
+      billingCustomerType: "team",
+      billingTeamId: "team-base-reopen-deleted",
+      billingPlanId: "free",
+      maxActiveVms: 1,
+      provider: "freestyle",
+      image: "snapshot-test",
+      imageVersion: "test-version",
+    }).pipe(Effect.provide(layer)));
+    const reopened = await Effect.runPromise(openBaseVm({
+      userId: "user-base-reopen-deleted",
+      billingCustomerType: "team",
+      billingTeamId: "team-base-reopen-deleted",
+      billingPlanId: "free",
+      maxActiveVms: 1,
+      provider: "freestyle",
+      image: "snapshot-test",
+      imageVersion: "test-version",
+    }).pipe(Effect.provide(layer)));
+
+    expect(first.providerVmId).toBe("provider-vm-base-reopen-1");
+    expect(reopened.providerVmId).toBe("provider-vm-base-reopen-2");
+    expect(reopened.generation).toBe(2);
+    expect(createCalls).toBe(2);
+    expect(statusCalls).toBe(1);
+
+    const vms = await sql<{ providerVmId: string; status: string; destroyedAt: Date | null }[]>`
+      select provider_vm_id as "providerVmId", status, destroyed_at as "destroyedAt"
+      from cloud_vms
+      where billing_team_id = 'team-base-reopen-deleted'
+      order by provider_vm_id
+    `;
+    expect(vms[0]?.providerVmId).toBe("provider-vm-base-reopen-1");
+    expect(vms[0]?.status).toBe("destroyed");
+    expect(vms[0]?.destroyedAt).toBeInstanceOf(Date);
+    expect(vms[1]).toEqual({
+      providerVmId: "provider-vm-base-reopen-2",
+      status: "running",
+      destroyedAt: null,
+    });
+
+    const bases = await sql<{ activeProviderVmId: string; activeGeneration: number }[]>`
+      select active_provider_vm_id as "activeProviderVmId", active_generation as "activeGeneration"
+      from cloud_vm_bases
+      where scope_id = 'team-base-reopen-deleted'
+    `;
+    expect(bases).toEqual([
+      { activeProviderVmId: "provider-vm-base-reopen-2", activeGeneration: 2 },
+    ]);
+
+    const [{ destroyedUsageCount }] = await sql<{ destroyedUsageCount: string }[]>`
+      select count(*)::text as "destroyedUsageCount"
+      from cloud_vm_usage_events
+      where event_type = 'vm.destroyed'
+        and metadata->>'source' = 'base_open_provider_missing'
+    `;
+    expect(destroyedUsageCount).toBe("1");
+  });
+
   dbTest("resets Base by retaining the previous generation when capacity allows", async () => {
     if (!sql) throw new Error("test database not initialized");
     await sql`truncate cloud_vm_billing_grants, cloud_vm_usage_events, cloud_vm_leases, cloud_vms restart identity cascade`;
@@ -1523,6 +2108,128 @@ describe("VM Effect workflows", () => {
     expect(leases[1]).toMatchObject({ providerIdentityHandle: "identity-2", revokedAt: null });
   });
 
+  dbTest("does not mint a replacement SSH endpoint when active identity cleanup fails", async () => {
+    if (!sql) throw new Error("test database not initialized");
+    await sql`truncate cloud_vm_billing_grants, cloud_vm_usage_events, cloud_vm_leases, cloud_vms restart identity cascade`;
+    const [vm] = await sql<{ id: string }[]>`
+      insert into cloud_vms (user_id, provider, provider_vm_id, image_id, status)
+      values ('user-workflow-ssh-revoke-failure', 'freestyle', 'provider-vm-ssh-revoke-failure', 'snapshot-test', 'running')
+      returning id
+    `;
+
+    let mintCount = 0;
+    const provider: VmProviderGatewayShape = {
+      create: () => Effect.fail(new Error("unused") as never),
+      destroy: () => Effect.void,
+      exec: () => Effect.succeed({ exitCode: 0, stdout: "", stderr: "" }),
+      openAttach: () => Effect.fail(new Error("unused") as never),
+      openSSH: () =>
+        Effect.sync(() => {
+          mintCount += 1;
+          return {
+            transport: "ssh" as const,
+            host: "vm-ssh.freestyle.sh",
+            port: 22,
+            username: "provider-vm-ssh-revoke-failure+cmux",
+            publicKeyFingerprint: null,
+            credential: { kind: "password" as const, value: `token-${mintCount}` },
+            identityHandle: `identity-revoke-failure-${mintCount}`,
+          };
+        }),
+      revokeSSHIdentity: () =>
+        Effect.fail(providerOperationError("revokeSSHIdentity", "provider delete failed")),
+    };
+    const layer = providerLayer(provider);
+
+    await Effect.runPromise(
+      openSshEndpoint({
+        userId: "user-workflow-ssh-revoke-failure",
+        providerVmId: "provider-vm-ssh-revoke-failure",
+      }).pipe(Effect.provide(layer)),
+    );
+    await expect(
+      Effect.runPromise(
+        openSshEndpoint({
+          userId: "user-workflow-ssh-revoke-failure",
+          providerVmId: "provider-vm-ssh-revoke-failure",
+        }).pipe(Effect.provide(layer)),
+      ),
+    ).rejects.toThrow();
+    expect(mintCount).toBe(1);
+
+    const leases = await sql<{ providerIdentityHandle: string; revokedAt: Date | null }[]>`
+      select provider_identity_handle as "providerIdentityHandle", revoked_at as "revokedAt"
+      from cloud_vm_leases
+      where vm_id = ${vm.id}
+      order by provider_identity_handle
+    `;
+    expect(leases).toEqual([
+      { providerIdentityHandle: "identity-revoke-failure-1", revokedAt: null },
+    ]);
+  });
+
+  dbTest("does not revoke or mint when active identity cleanup exceeds the hot-path cap", async () => {
+    if (!sql) throw new Error("test database not initialized");
+    await sql`truncate cloud_vm_billing_grants, cloud_vm_usage_events, cloud_vm_leases, cloud_vms restart identity cascade`;
+    const [vm] = await sql<{ id: string }[]>`
+      insert into cloud_vms (user_id, provider, provider_vm_id, image_id, status)
+      values ('user-workflow-ssh-cleanup-bound', 'freestyle', 'provider-vm-ssh-cleanup-bound', 'snapshot-test', 'running')
+      returning id
+    `;
+    await sql`
+      insert into cloud_vm_leases (
+        vm_id, user_id, kind, token_hash, expires_at, provider_identity_handle, transport, metadata
+      )
+      select ${vm.id}, 'user-workflow-ssh-cleanup-bound', 'ssh', 'cleanup-bound-token-' || n,
+        now() + interval '15 minutes', 'identity-cleanup-bound-' || n, 'ssh', '{}'::jsonb
+      from generate_series(1, 9) as n
+    `;
+
+    let revokeCalls = 0;
+    let mintCalls = 0;
+    const provider: VmProviderGatewayShape = {
+      create: () => Effect.fail(new Error("unused") as never),
+      destroy: () => Effect.void,
+      exec: () => Effect.succeed({ exitCode: 0, stdout: "", stderr: "" }),
+      openAttach: () => Effect.fail(new Error("unused") as never),
+      openSSH: () =>
+        Effect.sync(() => {
+          mintCalls += 1;
+          return {
+            transport: "ssh" as const,
+            host: "vm-ssh.freestyle.sh",
+            port: 22,
+            username: "provider-vm-ssh-cleanup-bound+cmux",
+            publicKeyFingerprint: null,
+            credential: { kind: "password" as const, value: "token" },
+            identityHandle: "identity-cleanup-bound-new",
+          };
+        }),
+      revokeSSHIdentity: () =>
+        Effect.sync(() => {
+          revokeCalls += 1;
+        }),
+    };
+
+    await expect(
+      Effect.runPromise(
+        openSshEndpoint({
+          userId: "user-workflow-ssh-cleanup-bound",
+          providerVmId: "provider-vm-ssh-cleanup-bound",
+        }).pipe(Effect.provide(providerLayer(provider))),
+      ),
+    ).rejects.toThrow();
+    expect(revokeCalls).toBe(0);
+    expect(mintCalls).toBe(0);
+
+    const [{ remainingLeaseCount }] = await sql<{ remainingLeaseCount: string }[]>`
+      select count(*)::text as "remainingLeaseCount"
+      from cloud_vm_leases
+      where vm_id = ${vm.id} and revoked_at is null
+    `;
+    expect(remainingLeaseCount).toBe("9");
+  });
+
   dbTest("resumes a paused VM before minting SSH credentials", async () => {
     if (!sql) throw new Error("test database not initialized");
     await sql`truncate cloud_vm_billing_grants, cloud_vm_usage_events, cloud_vm_leases, cloud_vms restart identity cascade`;
@@ -1577,6 +2284,7 @@ describe("VM Effect workflows", () => {
       openSshEndpoint({
         userId: "user-workflow-resume-ssh",
         billingTeamId: "team-workflow-resume-ssh",
+        teamIds: ["team-workflow-resume-ssh"],
         providerVmId: "provider-vm-resume-ssh",
       }).pipe(
         Effect.provide(providerLayer(provider)),
@@ -1759,6 +2467,7 @@ describe("VM Effect workflows", () => {
         openAttachEndpoint({
           userId: "user-workflow-resume-limit",
           billingTeamId: "team-workflow-resume-limit",
+          teamIds: ["team-workflow-resume-limit"],
           providerVmId: "provider-vm-resume-paused",
         }).pipe(
           Effect.flip,
@@ -1828,6 +2537,7 @@ describe("VM Effect workflows", () => {
       openAttachEndpoint({
         userId: "user-workflow-resume-fail",
         billingTeamId: "team-workflow-resume-fail",
+        teamIds: ["team-workflow-resume-fail"],
         providerVmId: "provider-vm-resume-fail",
       }).pipe(
         Effect.flip,
@@ -2606,6 +3316,7 @@ describe("VM Effect workflows", () => {
       destroyVm({
         userId: "user-workflow-reuse-slot",
         billingTeamId: "team-workflow-reuse-slot",
+        teamIds: ["team-workflow-reuse-slot"],
         providerVmId: "provider-vm-reuse-old",
       }).pipe(
         Effect.provide(layer),
@@ -3257,6 +3968,7 @@ describe("VM Effect workflows", () => {
       openAttachEndpoint({
         userId: "user-workflow-teammate",
         billingTeamId: "team-workflow-shared",
+        teamIds: ["team-workflow-shared"],
         providerVmId: "provider-vm-shared-team",
       }).pipe(Effect.provide(layer)),
     );
@@ -3508,10 +4220,10 @@ describe("VM Effect workflows", () => {
 
 function testCloudVmRow(overrides: Partial<CloudVmRow> = {}): CloudVmRow {
   const now = new Date();
-  return {
+  const row: CloudVmRow = {
     id: "00000000-0000-4000-8000-000000000001",
     userId: "user-workflow-usage-events",
-    billingTeamId: "team-workflow-usage-events",
+    billingTeamId: "user-workflow-usage-events",
     billingPlanId: "free",
     provider: "freestyle",
     providerVmId: null,
@@ -3527,12 +4239,20 @@ function testCloudVmRow(overrides: Partial<CloudVmRow> = {}): CloudVmRow {
     providerMetadata: {},
     ...overrides,
   };
+  if (!("billingTeamId" in overrides)) {
+    return { ...row, billingTeamId: row.userId };
+  }
+  return row;
 }
 
 function testWorkflowRepo(input: {
   readonly vm: CloudVmRow;
   readonly usageEvents?: RecordedUsageEvent[];
   readonly leases?: RecordedLease[];
+  readonly activeIdentityLeases?: CloudVmLeaseRow[];
+  readonly expiredIdentityLeases?: VmRepositoryShape["expiredIdentityLeases"];
+  readonly revokedLeaseIds?: string[];
+  readonly leaseRevocationRetries?: LeaseRevocationRetry[];
   readonly observedStatuses?: ObservedStatusUpdate[];
   readonly markProviderObservedStatus?: (
     update: ObservedStatusUpdate,
@@ -3576,6 +4296,11 @@ function testWorkflowRepo(input: {
       Effect.sync(() => {
         input.leases?.push(lease);
       }),
+    expiredIdentityLeases: input.expiredIdentityLeases,
+    markLeaseRevocationRetry: (retry) =>
+      Effect.sync(() => {
+        input.leaseRevocationRetries?.push(retry);
+      }),
     listVmSessions: () => Effect.succeed([]),
     upsertVmSession: (session) =>
       Effect.sync(() => {
@@ -3602,8 +4327,16 @@ function testWorkflowRepo(input: {
           closedAt: null,
         } satisfies CloudVmSessionRow;
       }),
-    activeIdentityLeases: () => Effect.succeed([]),
-    markLeasesRevoked: () => Effect.void,
+    activeIdentityLeases: (_vmId, limit) =>
+      Effect.succeed(
+        typeof limit === "number" && limit > 0
+          ? (input.activeIdentityLeases ?? []).slice(0, limit)
+          : input.activeIdentityLeases ?? [],
+      ),
+    markLeasesRevoked: (leaseIds) =>
+      Effect.sync(() => {
+        input.revokedLeaseIds?.push(...leaseIds);
+      }),
     recordUsageEvent: (event) =>
       Effect.sync(() => {
         input.usageEvents?.push(event);

web/vercel.json

@@ -11,6 +11,10 @@
     {
       "path": "/api/cron/vm-alerts",
       "schedule": "*/5 * * * *"
+    },
+    {
+      "path": "/api/internal/vm/leases/revoke-expired",
+      "schedule": "*/10 * * * *"
     }
   ]
 }