|
|
|
@@ -5,32 +5,45 @@ import { ApiError } from '../../../common/errors/api-error';
|
|
|
|
|
import { ERROR_CODES } from '../../../common/errors/error-codes';
|
|
|
|
|
|
|
|
|
|
export interface SwapPair {
|
|
|
|
|
/** Absolute path of the live filesystem location that will be swapped out. */
|
|
|
|
|
livePath: string;
|
|
|
|
|
/** Absolute path of the staged replacement that will be swapped in. */
|
|
|
|
|
stagedPath: string;
|
|
|
|
|
/**
|
|
|
|
|
* If true, the pair participates in the manifest's "legacy"/"primary"
|
|
|
|
|
* generation tally. The database and uploads are typically `true`, while
|
|
|
|
|
* sidecar files (e.g. `-wal`, `-shm`) are NOT — they are tracked as
|
|
|
|
|
* auxiliary, so a partial sidecar swap does not mark the parent
|
|
|
|
|
* generation as "incomplete".
|
|
|
|
|
*/
|
|
|
|
|
contributesToGeneration?: boolean;
|
|
|
|
|
/**
|
|
|
|
|
* Auxiliary sidecar suffixes to move with the live path. When the live
|
|
|
|
|
* path is replaced, any matching sidecars (e.g. `<live>.wal`) are
|
|
|
|
|
* moved into the rollback dir and back. When the staged replacement
|
|
|
|
|
* arrives, stale sidecars are removed so the new file is not paired
|
|
|
|
|
* with an old WAL.
|
|
|
|
|
*/
|
|
|
|
|
sidecarSuffixes?: string[];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export interface SwapRollbackHandle {
|
|
|
|
|
rolledBack: boolean;
|
|
|
|
|
pairs: Array<{ livePath: string; rollbackPath: string; stagedPath: string }>;
|
|
|
|
|
committed: boolean;
|
|
|
|
|
/** Public identifier of the durable transaction; only present when recoverability is enabled. */
|
|
|
|
|
transactionId?: string;
|
|
|
|
|
/**
|
|
|
|
|
* The exact per-transaction manifest directory created under the system
|
|
|
|
|
* staging root. Stored on the handle so that `commit()` and `rollback()`
|
|
|
|
|
* can clean the right record without re-deriving paths.
|
|
|
|
|
*/
|
|
|
|
|
manifestDir?: string;
|
|
|
|
|
pairs: Array<{ livePath: string; rollbackPath: string; stagedPath: string; sidecarSuffixes: string[] }>;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface WorkerHooks {
|
|
|
|
|
/**
|
|
|
|
|
* Optional hook that runs once after `prepare()` (file staging) but
|
|
|
|
|
* before the live swap (e.g. database rebuild). Throw to abort the
|
|
|
|
|
* operation and immediately trigger rollback.
|
|
|
|
|
*/
|
|
|
|
|
beforeSwap?: () => Promise<void>;
|
|
|
|
|
/**
|
|
|
|
|
* Optional hook that runs after the swap is registered but before it
|
|
|
|
|
* is considered durable (e.g. final database open, verification).
|
|
|
|
|
* Throw to trigger rollback.
|
|
|
|
|
*/
|
|
|
|
|
afterSwap?: () => Promise<void>;
|
|
|
|
|
/**
|
|
|
|
|
* Optional hook that runs after finalization completes successfully.
|
|
|
|
|
* Throw to trigger rollback even from a successful operation.
|
|
|
|
|
*/
|
|
|
|
|
onCommit?: () => Promise<void>;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -41,54 +54,135 @@ export interface StageSwapInput {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Swap a set of file/directory pairs in a crash-safe way. Each pair moves
|
|
|
|
|
* the existing live path to a rollback location, then renames the staged
|
|
|
|
|
* replacement into place. If any step fails (or one of the supplied hooks
|
|
|
|
|
* throws), the rollback path is restored to the live location exactly as
|
|
|
|
|
* it was before the operation began.
|
|
|
|
|
* Transaction phase values written to the durable manifest. Recovery
|
|
|
|
|
* inspects these to decide whether the system is in a coherent state.
|
|
|
|
|
*
|
|
|
|
|
* The class assumes all live/staged paths share the same filesystem so
|
|
|
|
|
* `rename` is atomic; cross-filesystem moves fall back to copy+unlink.
|
|
|
|
|
* Phase progression under success:
|
|
|
|
|
* prepared → live-snapshotted → replacements-installed → verified → committed
|
|
|
|
|
*
|
|
|
|
|
* Phase progression under in-process failure or rollback():
|
|
|
|
|
* any → rolled-back
|
|
|
|
|
*/
|
|
|
|
|
export type TransactionPhase =
|
|
|
|
|
| 'prepared'
|
|
|
|
|
| 'live-snapshotted'
|
|
|
|
|
| 'replacements-installed'
|
|
|
|
|
| 'verified'
|
|
|
|
|
| 'committed'
|
|
|
|
|
| 'rolled-back';
|
|
|
|
|
|
|
|
|
|
export interface TransactionManifest {
|
|
|
|
|
schemaVersion: 1;
|
|
|
|
|
transactionId: string;
|
|
|
|
|
operation: string;
|
|
|
|
|
createdAt: string;
|
|
|
|
|
updatedAt: string;
|
|
|
|
|
phase: TransactionPhase;
|
|
|
|
|
pairs: Array<{
|
|
|
|
|
livePath: string;
|
|
|
|
|
rollbackPath: string;
|
|
|
|
|
stagedPath: string;
|
|
|
|
|
contributesToGeneration: boolean;
|
|
|
|
|
sidecarSuffixes: string[];
|
|
|
|
|
}>;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export interface RecoveryReport {
|
|
|
|
|
transactionsScanned: number;
|
|
|
|
|
resolved: number;
|
|
|
|
|
aborted: string[];
|
|
|
|
|
cleaned: string[];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Crash-safe filesystem swap with a durable manifest.
|
|
|
|
|
*
|
|
|
|
|
* Each `stageSwap()` writes a small JSON manifest under the configured
|
|
|
|
|
* system staging root. The manifest records the live/staged/rollback
|
|
|
|
|
* paths and the current phase. If the process is killed mid-swap, the
|
|
|
|
|
* manifest is still on disk so startup recovery can resolve the
|
|
|
|
|
* transaction deterministically:
|
|
|
|
|
*
|
|
|
|
|
* - If the old generation (live + rollback trees) is fully present,
|
|
|
|
|
* restore it.
|
|
|
|
|
* - If the new generation (staged → live) is fully present AND was
|
|
|
|
|
* marked `verified`, finalize it and remove the rollback copy.
|
|
|
|
|
* - Otherwise, fail closed: the manifest is left on disk for an
|
|
|
|
|
* operator, and startup throws a `SYSTEM_RESTORE_RECOVERY_PENDING`
|
|
|
|
|
* error.
|
|
|
|
|
*
|
|
|
|
|
* The class also sweeps stale `*.rollback-*` artifacts in the configured
|
|
|
|
|
* live paths that are NOT associated with any live manifest (e.g. from
|
|
|
|
|
* a previous Job 919 incident) so the recovery itself never silently
|
|
|
|
|
* removes unowned files.
|
|
|
|
|
*/
|
|
|
|
|
@Injectable()
|
|
|
|
|
export class FilesystemTransactionService {
|
|
|
|
|
private readonly logger = new Logger(FilesystemTransactionService.name);
|
|
|
|
|
|
|
|
|
|
/** Manifest file name inside each per-transaction directory. */
|
|
|
|
|
static MANIFEST_FILENAME = 'manifest.json';
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Apply the swap. Returns a rollback handle; call `commit()` after the
|
|
|
|
|
* rest of the operation (including database swap) is durable, or
|
|
|
|
|
* `rollback()` on any failure.
|
|
|
|
|
* Apply the swap. Returns a rollback handle.
|
|
|
|
|
*
|
|
|
|
|
* Behavior matches the original (in-memory) contract: success means
|
|
|
|
|
* the staged replacement now lives at `livePath`; failure rolls the
|
|
|
|
|
* pair back. The difference is that the swap is now also recorded in a
|
|
|
|
|
* durable manifest so a crashed process can finish or undo the
|
|
|
|
|
* operation on next startup.
|
|
|
|
|
*
|
|
|
|
|
* If the caller's environment doesn't configure a system staging root,
|
|
|
|
|
* the method falls back to the old in-memory behavior (still safe; just
|
|
|
|
|
* not recoverable across process death).
|
|
|
|
|
*/
|
|
|
|
|
async stageSwap(input: StageSwapInput): Promise<SwapRollbackHandle> {
|
|
|
|
|
async stageSwap(
|
|
|
|
|
input: StageSwapInput,
|
|
|
|
|
opts: { stagingDir?: string } = {},
|
|
|
|
|
): Promise<SwapRollbackHandle> {
|
|
|
|
|
const txDir = opts.stagingDir ? this.ensureTxDir(opts.stagingDir) : null;
|
|
|
|
|
const manifest: TransactionManifest | null = txDir
|
|
|
|
|
? this.makeInitialManifest(input, txDir)
|
|
|
|
|
: null;
|
|
|
|
|
|
|
|
|
|
const stamp = `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
|
|
|
|
|
const rolled: SwapRollbackHandle['pairs'] = [];
|
|
|
|
|
const cleanupRollbackOnly: string[] = [];
|
|
|
|
|
try {
|
|
|
|
|
// 1. Stage: snapshot each live path to a rollback location.
|
|
|
|
|
for (const pair of input.pairs) {
|
|
|
|
|
if (!pair.livePath) throw new ApiError(ERROR_CODES.SYSTEM_RESTORE_FAILED, 'Missing live path', 500);
|
|
|
|
|
if (!pair.stagedPath) throw new ApiError(ERROR_CODES.SYSTEM_RESTORE_FAILED, 'Missing staged path', 500);
|
|
|
|
|
const rollbackPath = FilesystemTransactionService.makeRollbackPath(pair.livePath, stamp);
|
|
|
|
|
if (fs.existsSync(pair.livePath)) {
|
|
|
|
|
fs.renameSync(pair.livePath, rollbackPath);
|
|
|
|
|
cleanupRollbackOnly.push(rollbackPath);
|
|
|
|
|
} else if (fs.existsSync(pair.stagedPath)) {
|
|
|
|
|
// Live path absent but staged exists — delete staged so we fail closed.
|
|
|
|
|
try {
|
|
|
|
|
fs.rmSync(pair.stagedPath, { recursive: true, force: true });
|
|
|
|
|
} catch {
|
|
|
|
|
// ignore
|
|
|
|
|
const sidecarSuffixes = pair.sidecarSuffixes ?? [];
|
|
|
|
|
|
|
|
|
|
// Move any sidecar files (e.g. -wal/-shm) into the rollback dir first so they cannot
|
|
|
|
|
// be left paired with the swapped-in live path.
|
|
|
|
|
for (const suffix of sidecarSuffixes) {
|
|
|
|
|
const liveSidecar = `${pair.livePath}${suffix}`;
|
|
|
|
|
const rollbackSidecar = `${rollbackPath}${suffix}`;
|
|
|
|
|
if (fs.existsSync(liveSidecar) && !fs.existsSync(rollbackSidecar)) {
|
|
|
|
|
try {
|
|
|
|
|
fs.renameSync(liveSidecar, rollbackSidecar);
|
|
|
|
|
} catch {
|
|
|
|
|
// Cross-device: best-effort copy+unlink.
|
|
|
|
|
FilesystemTransactionService.copyDirSync(liveSidecar, rollbackSidecar);
|
|
|
|
|
try { fs.rmSync(liveSidecar, { recursive: true, force: true }); } catch { /* ignore */ }
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
rolled.push({ livePath: pair.livePath, rollbackPath, stagedPath: pair.stagedPath });
|
|
|
|
|
|
|
|
|
|
if (fs.existsSync(pair.livePath)) {
|
|
|
|
|
fs.renameSync(pair.livePath, rollbackPath);
|
|
|
|
|
} else if (fs.existsSync(pair.stagedPath)) {
|
|
|
|
|
try { fs.rmSync(pair.stagedPath, { recursive: true, force: true }); } catch { /* ignore */ }
|
|
|
|
|
}
|
|
|
|
|
rolled.push({ livePath: pair.livePath, rollbackPath, stagedPath: pair.stagedPath, sidecarSuffixes });
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (manifest) this.updateManifest(manifest, txDir!, 'live-snapshotted');
|
|
|
|
|
|
|
|
|
|
if (input.hooks?.beforeSwap) {
|
|
|
|
|
try {
|
|
|
|
|
await input.hooks.beforeSwap();
|
|
|
|
|
} catch (err) {
|
|
|
|
|
throw err;
|
|
|
|
|
}
|
|
|
|
|
await input.hooks.beforeSwap();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 2. Swap: rename each staged path into the live location.
|
|
|
|
@@ -101,37 +195,49 @@ export class FilesystemTransactionService {
|
|
|
|
|
try {
|
|
|
|
|
fs.renameSync(r.stagedPath, r.livePath);
|
|
|
|
|
} catch (err) {
|
|
|
|
|
// Cross-device fall-back: copy + unlink.
|
|
|
|
|
FilesystemTransactionService.copyDirSync(r.stagedPath, r.livePath);
|
|
|
|
|
try {
|
|
|
|
|
fs.rmSync(r.stagedPath, { recursive: true, force: true });
|
|
|
|
|
} catch {
|
|
|
|
|
// ignore
|
|
|
|
|
try { fs.rmSync(r.stagedPath, { recursive: true, force: true }); } catch { /* ignore */ }
|
|
|
|
|
}
|
|
|
|
|
// Stale sidecars that belonged to a previous live DB must NOT survive the swap:
|
|
|
|
|
// drop anything that matches and isn't one we tracked above.
|
|
|
|
|
for (const suffix of r.sidecarSuffixes) {
|
|
|
|
|
const liveSidecar = `${r.livePath}${suffix}`;
|
|
|
|
|
if (fs.existsSync(liveSidecar)) {
|
|
|
|
|
try { fs.rmSync(liveSidecar, { recursive: true, force: true }); } catch { /* ignore */ }
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (manifest) this.updateManifest(manifest, txDir!, 'replacements-installed');
|
|
|
|
|
|
|
|
|
|
if (input.hooks?.afterSwap) {
|
|
|
|
|
await input.hooks.afterSwap();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (manifest) this.updateManifest(manifest, txDir!, 'verified');
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
rolledBack: false,
|
|
|
|
|
committed: false,
|
|
|
|
|
transactionId: manifest?.transactionId,
|
|
|
|
|
manifestDir: txDir ?? undefined,
|
|
|
|
|
pairs: rolled,
|
|
|
|
|
};
|
|
|
|
|
} catch (err) {
|
|
|
|
|
await this.rollbackLocal({ rolledBack: false, pairs: rolled });
|
|
|
|
|
await this.rollbackLocal({ rolledBack: false, committed: false, pairs: rolled, transactionId: manifest?.transactionId });
|
|
|
|
|
if (manifest && txDir) {
|
|
|
|
|
try {
|
|
|
|
|
this.updateManifest(manifest, txDir, 'rolled-back');
|
|
|
|
|
} catch {
|
|
|
|
|
// ignore: best effort
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
throw err;
|
|
|
|
|
} finally {
|
|
|
|
|
// Successful rename leaves rollback artifacts; we keep them until
|
|
|
|
|
// commit() (or rollback() explicitly deletes them).
|
|
|
|
|
void cleanupRollbackOnly;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Restore the original live paths from their rollback snapshots. Safe
|
|
|
|
|
* to call multiple times.
|
|
|
|
|
* Restore the original live paths from their rollback snapshots. Safe to call multiple times.
|
|
|
|
|
*/
|
|
|
|
|
async rollback(handle: SwapRollbackHandle): Promise<void> {
|
|
|
|
|
if (!handle || handle.rolledBack) return;
|
|
|
|
@@ -146,10 +252,11 @@ export class FilesystemTransactionService {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Mark the swap committed; the rollback snapshots are deleted.
|
|
|
|
|
* Mark the swap committed; the rollback snapshots and (any) staged
|
|
|
|
|
* leftovers are deleted, and the manifest's terminal phase is recorded.
|
|
|
|
|
*/
|
|
|
|
|
commit(handle: SwapRollbackHandle): void {
|
|
|
|
|
if (!handle || handle.rolledBack) return;
|
|
|
|
|
if (!handle || handle.rolledBack || handle.committed) return;
|
|
|
|
|
for (const r of handle.pairs) {
|
|
|
|
|
try {
|
|
|
|
|
if (fs.existsSync(r.rollbackPath)) {
|
|
|
|
@@ -158,24 +265,403 @@ export class FilesystemTransactionService {
|
|
|
|
|
if (fs.existsSync(r.stagedPath)) {
|
|
|
|
|
fs.rmSync(r.stagedPath, { recursive: true, force: true });
|
|
|
|
|
}
|
|
|
|
|
// Sidecars: a successful commit means the new live has fresh
|
|
|
|
|
// sidecars, but any leftover from the staged tree should be gone.
|
|
|
|
|
for (const suffix of r.sidecarSuffixes) {
|
|
|
|
|
const stagedSidecar = `${r.stagedPath}${suffix}`;
|
|
|
|
|
if (fs.existsSync(stagedSidecar)) {
|
|
|
|
|
try { fs.rmSync(stagedSidecar, { recursive: true, force: true }); } catch { /* ignore */ }
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch (err) {
|
|
|
|
|
this.logger.warn(
|
|
|
|
|
`Failed to clean rollback artifact ${r.rollbackPath}: ${(err as Error)?.message}`,
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (handle.manifestDir) {
|
|
|
|
|
try {
|
|
|
|
|
FilesystemTransactionService.rmSafe(handle.manifestDir);
|
|
|
|
|
} catch {
|
|
|
|
|
// ignore
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
handle.rolledBack = true;
|
|
|
|
|
handle.committed = true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Reset the swap state after a successful commit (also clears
|
|
|
|
|
* rolledBack flag so commit() is safely idempotent).
|
|
|
|
|
* Idempotent terminal step retained for backwards compatibility.
|
|
|
|
|
*/
|
|
|
|
|
finalize(handle: SwapRollbackHandle): void {
|
|
|
|
|
if (!handle) return;
|
|
|
|
|
this.commit(handle);
|
|
|
|
|
if (!handle.committed && !handle.rolledBack) this.commit(handle);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Scan a system-staging root for unfinished transactions and resolve
|
|
|
|
|
* each one to a coherent state. Intended to be called once during
|
|
|
|
|
* startup BEFORE the database / uploads are opened.
|
|
|
|
|
*
|
|
|
|
|
* Returns a summary that the caller can log. Throws if any transaction
|
|
|
|
|
* cannot be safely resolved (ambiguous manifest state). Throwing at
|
|
|
|
|
* startup is preferable to silently producing a hybrid state.
|
|
|
|
|
*/
|
|
|
|
|
recoverTransactions(stagingDir: string, opts: { now?: number; maxAgeMs?: number } = {}): RecoveryReport {
|
|
|
|
|
FilesystemTransactionService.ensureDir(stagingDir);
|
|
|
|
|
const report: RecoveryReport = { transactionsScanned: 0, resolved: 0, aborted: [], cleaned: [] };
|
|
|
|
|
|
|
|
|
|
const entries = fs.readdirSync(stagingDir, { withFileTypes: true });
|
|
|
|
|
const now = opts.now ?? Date.now();
|
|
|
|
|
const maxAgeMs = opts.maxAgeMs ?? 7 * 24 * 60 * 60_000;
|
|
|
|
|
|
|
|
|
|
for (const entry of entries) {
|
|
|
|
|
if (!entry.isDirectory()) continue;
|
|
|
|
|
const txDir = path.join(stagingDir, entry.name);
|
|
|
|
|
const manifestPath = path.join(txDir, FilesystemTransactionService.MANIFEST_FILENAME);
|
|
|
|
|
if (!fs.existsSync(manifestPath)) {
|
|
|
|
|
// Orphan tx dir without a manifest — only remove if stale (we never delete anything tied to a live manifest).
|
|
|
|
|
try {
|
|
|
|
|
const stat = fs.statSync(txDir);
|
|
|
|
|
if (now - stat.mtimeMs > maxAgeMs) {
|
|
|
|
|
FilesystemTransactionService.rmSafe(txDir);
|
|
|
|
|
report.cleaned.push(txDir);
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
// ignore
|
|
|
|
|
}
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
report.transactionsScanned += 1;
|
|
|
|
|
let manifest: TransactionManifest;
|
|
|
|
|
try {
|
|
|
|
|
manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) as TransactionManifest;
|
|
|
|
|
} catch (err) {
|
|
|
|
|
report.aborted.push(manifestPath);
|
|
|
|
|
throw new ApiError(
|
|
|
|
|
ERROR_CODES.SYSTEM_RESTORE_FAILED,
|
|
|
|
|
`Unreadable transaction manifest at ${manifestPath}: ${(err as Error).message}`,
|
|
|
|
|
500,
|
|
|
|
|
{ details: { phase: 'recovery' } },
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Terminal phases need no further action.
|
|
|
|
|
if (manifest.phase === 'committed' || manifest.phase === 'rolled-back') {
|
|
|
|
|
FilesystemTransactionService.rmSafe(txDir);
|
|
|
|
|
report.resolved += 1;
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Mid-swap recovery decision.
|
|
|
|
|
try {
|
|
|
|
|
const decision = this.resolveInterruptedManifest(manifest);
|
|
|
|
|
if (decision === 'keep-old') {
|
|
|
|
|
this.restoreOldGeneration(manifest, txDir);
|
|
|
|
|
} else if (decision === 'keep-new') {
|
|
|
|
|
this.finalizeNewGeneration(manifest, txDir);
|
|
|
|
|
} else {
|
|
|
|
|
report.aborted.push(manifestPath);
|
|
|
|
|
throw new ApiError(
|
|
|
|
|
ERROR_CODES.SYSTEM_RESTORE_FAILED,
|
|
|
|
|
`Cannot resolve interrupted transaction ${manifest.transactionId} at phase=${manifest.phase}: neither old nor new generation is fully present`,
|
|
|
|
|
500,
|
|
|
|
|
{ details: { phase: 'recovery', transactionId: manifest.transactionId, transactionPhase: manifest.phase } },
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
FilesystemTransactionService.rmSafe(txDir);
|
|
|
|
|
report.resolved += 1;
|
|
|
|
|
} catch (err) {
|
|
|
|
|
if ((err as ApiError)?.code) throw err;
|
|
|
|
|
report.aborted.push(manifestPath);
|
|
|
|
|
throw new ApiError(
|
|
|
|
|
ERROR_CODES.SYSTEM_RESTORE_FAILED,
|
|
|
|
|
`Recovery of transaction ${manifest.transactionId} failed: ${(err as Error).message}`,
|
|
|
|
|
500,
|
|
|
|
|
{ details: { phase: 'recovery', transactionId: manifest.transactionId } },
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return report;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Sweep unowned rollback / restore-stage / wipe-stage fragments that
|
|
|
|
|
* sit NEXT to a live path (database or uploads) and are not currently
|
|
|
|
|
* tracked by any manifest. These can be left behind by an interrupted
|
|
|
|
|
* swap from before this durable manifest existed, OR by an interrupted
|
|
|
|
|
* wipe-challenges snapshot. Only artifacts older than `maxAgeMs` are
|
|
|
|
|
* removed so that a perfectly healthy in-flight operation is never
|
|
|
|
|
* disturbed.
|
|
|
|
|
*/
|
|
|
|
|
sweepUnownedArtifactsNearLivePaths(
|
|
|
|
|
paths: string[],
|
|
|
|
|
opts: { now?: number; maxAgeMs?: number; stagingDir?: string } = {},
|
|
|
|
|
): string[] {
|
|
|
|
|
const cleaned: string[] = [];
|
|
|
|
|
const now = opts.now ?? Date.now();
|
|
|
|
|
const maxAgeMs = opts.maxAgeMs ?? 24 * 60 * 60_000;
|
|
|
|
|
const stagingDir = opts.stagingDir;
|
|
|
|
|
|
|
|
|
|
const candidates = (livePath: string, suffixRegex: RegExp): string[] => {
|
|
|
|
|
const dir = path.dirname(livePath);
|
|
|
|
|
const base = path.basename(livePath);
|
|
|
|
|
if (!fs.existsSync(dir)) return [];
|
|
|
|
|
const result: string[] = [];
|
|
|
|
|
for (const entry of fs.readdirSync(dir)) {
|
|
|
|
|
if (!suffixRegex.test(entry)) continue;
|
|
|
|
|
// The artifact must be derived from this specific live path (its
|
|
|
|
|
// base name must appear at the start of the artifact).
|
|
|
|
|
if (!entry.startsWith(`${base}.`) && !entry.startsWith(`.${base}.`)) continue;
|
|
|
|
|
const full = path.join(dir, entry);
|
|
|
|
|
try {
|
|
|
|
|
const stat = fs.statSync(full);
|
|
|
|
|
if (now - stat.mtimeMs < maxAgeMs) continue;
|
|
|
|
|
} catch {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
// Skip artifacts actively tracked by a manifest.
|
|
|
|
|
if (stagingDir && FilesystemTransactionService.hasManifestTracking(stagingDir, full)) continue;
|
|
|
|
|
result.push(full);
|
|
|
|
|
}
|
|
|
|
|
return result;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const restoreStageRe = /\.restore-stage-[\w-]+$/;
|
|
|
|
|
// Either a hidden (`.<base>.rollback-…`) or visible (`<base>.rollback-…`)
|
|
|
|
|
// rollback artifact derived from the live path basename.
|
|
|
|
|
const rollbackRe = /^\.?[^/]+\.rollback-[\w-]+$/;
|
|
|
|
|
const wipeStageRe = /^\.challenges\.wipe-stage-[\w-]+$/;
|
|
|
|
|
|
|
|
|
|
for (const livePath of paths) {
|
|
|
|
|
const directDir = path.dirname(livePath);
|
|
|
|
|
if (!fs.existsSync(directDir)) continue;
|
|
|
|
|
// For the uploadDir live, also sweep the `challenges/` subdirectory
|
|
|
|
|
// (used by DangerZoneService.wipeChallenges()).
|
|
|
|
|
const extraDirs: string[] = [];
|
|
|
|
|
if (fs.existsSync(livePath)) {
|
|
|
|
|
const stat = fs.statSync(livePath);
|
|
|
|
|
if (stat.isDirectory()) {
|
|
|
|
|
extraDirs.push(path.join(livePath, 'challenges'));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const allArtifacts = [
|
|
|
|
|
...candidates(livePath, restoreStageRe),
|
|
|
|
|
...candidates(livePath, rollbackRe),
|
|
|
|
|
...candidates(livePath, wipeStageRe),
|
|
|
|
|
];
|
|
|
|
|
for (const artifact of allArtifacts) {
|
|
|
|
|
FilesystemTransactionService.rmSafe(artifact);
|
|
|
|
|
cleaned.push(artifact);
|
|
|
|
|
}
|
|
|
|
|
for (const extra of extraDirs) {
|
|
|
|
|
if (!fs.existsSync(extra)) continue;
|
|
|
|
|
for (const entry of fs.readdirSync(extra)) {
|
|
|
|
|
if (!wipeStageRe.test(entry)) continue;
|
|
|
|
|
const full = path.join(extra, entry);
|
|
|
|
|
try {
|
|
|
|
|
const stat = fs.statSync(full);
|
|
|
|
|
if (now - stat.mtimeMs < maxAgeMs) continue;
|
|
|
|
|
} catch {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
if (stagingDir && FilesystemTransactionService.hasManifestTracking(stagingDir, full)) continue;
|
|
|
|
|
FilesystemTransactionService.rmSafe(full);
|
|
|
|
|
cleaned.push(full);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return cleaned;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Decide whether to restore the OLD generation (live+rollback both
|
|
|
|
|
* present) or keep the NEW generation (live already swapped + rollback
|
|
|
|
|
* present). Returns null when neither generation is complete.
|
|
|
|
|
*/
|
|
|
|
|
private resolveInterruptedManifest(manifest: TransactionManifest): 'keep-old' | 'keep-new' | null {
|
|
|
|
|
const generationPairs = manifest.pairs.filter((p) => p.contributesToGeneration);
|
|
|
|
|
if (generationPairs.length === 0) return 'keep-new';
|
|
|
|
|
|
|
|
|
|
// Old generation complete iff every livePath is currently at the
|
|
|
|
|
// rollbackPath (no swap happened for that pair). Once a swap has been
|
|
|
|
|
// recorded as `replacements-installed`, the live path is the new
|
|
|
|
|
// generation.
|
|
|
|
|
if (manifest.phase === 'prepared') {
|
|
|
|
|
// Nothing has happened yet — the live tree is still the old one and
|
|
|
|
|
// there is no rollback because the rename hasn't run. We treat this
|
|
|
|
|
// as a clean "keep-old" with no filesystem action.
|
|
|
|
|
return 'keep-old';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (manifest.phase === 'live-snapshotted') {
|
|
|
|
|
const oldComplete = generationPairs.every((p) => fs.existsSync(p.rollbackPath) && !fs.existsSync(p.livePath));
|
|
|
|
|
return oldComplete ? 'keep-old' : null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (manifest.phase === 'replacements-installed' || manifest.phase === 'verified' || manifest.phase === 'committed') {
|
|
|
|
|
const newComplete = generationPairs.every((p) => fs.existsSync(p.livePath) && fs.existsSync(p.rollbackPath));
|
|
|
|
|
return newComplete ? 'keep-new' : null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Restore the OLD generation: move each rollback path back to the live path
|
|
|
|
|
* (or, when the live path already holds the new generation due to an
|
|
|
|
|
* earlier partial swap, remove it first). The rollback is the snapshot of
|
|
|
|
|
* the pre-swap live tree.
|
|
|
|
|
*/
|
|
|
|
|
private restoreOldGeneration(manifest: TransactionManifest, txDir: string): void {
|
|
|
|
|
if (manifest.phase === 'prepared') {
|
|
|
|
|
// Nothing was moved; just record the terminal state.
|
|
|
|
|
this.updateManifest(manifest, txDir, 'rolled-back');
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
for (const p of manifest.pairs) {
|
|
|
|
|
// Remove any partial new-generation content that may already exist at livePath.
|
|
|
|
|
if (fs.existsSync(p.livePath)) {
|
|
|
|
|
FilesystemTransactionService.rmSafe(p.livePath);
|
|
|
|
|
}
|
|
|
|
|
if (fs.existsSync(p.rollbackPath)) {
|
|
|
|
|
FilesystemTransactionService.ensureDir(path.dirname(p.livePath));
|
|
|
|
|
try {
|
|
|
|
|
fs.renameSync(p.rollbackPath, p.livePath);
|
|
|
|
|
} catch {
|
|
|
|
|
FilesystemTransactionService.copyDirSync(p.rollbackPath, p.livePath);
|
|
|
|
|
try { fs.rmSync(p.rollbackPath, { recursive: true, force: true }); } catch { /* ignore */ }
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// Sidecars of the OLD generation are stored next to the rollback path
|
|
|
|
|
// (e.g. `rollbackPath-wal`); restore them too if present.
|
|
|
|
|
for (const suffix of p.sidecarSuffixes) {
|
|
|
|
|
const rollbackSidecar = `${p.rollbackPath}${suffix}`;
|
|
|
|
|
const liveSidecar = `${p.livePath}${suffix}`;
|
|
|
|
|
if (fs.existsSync(rollbackSidecar)) {
|
|
|
|
|
try { fs.rmSync(liveSidecar, { recursive: true, force: true }); } catch { /* ignore */ }
|
|
|
|
|
FilesystemTransactionService.ensureDir(path.dirname(liveSidecar));
|
|
|
|
|
try {
|
|
|
|
|
fs.renameSync(rollbackSidecar, liveSidecar);
|
|
|
|
|
} catch {
|
|
|
|
|
FilesystemTransactionService.copyDirSync(rollbackSidecar, liveSidecar);
|
|
|
|
|
try { fs.rmSync(rollbackSidecar, { recursive: true, force: true }); } catch { /* ignore */ }
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (fs.existsSync(liveSidecar) && !fs.existsSync(rollbackSidecar)) {
|
|
|
|
|
// No source sidecar and the live sidecar remains — leave it removed.
|
|
|
|
|
try { fs.rmSync(liveSidecar, { recursive: true, force: true }); } catch { /* ignore */ }
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
this.updateManifest(manifest, txDir, 'rolled-back');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Finalize the NEW generation: keep the live replacement in place,
|
|
|
|
|
* remove the rollback snapshot and any stale sidecars.
|
|
|
|
|
*/
|
|
|
|
|
private finalizeNewGeneration(manifest: TransactionManifest, txDir: string): void {
|
|
|
|
|
for (const p of manifest.pairs) {
|
|
|
|
|
// Drop the rollback copy; the new live is now durable.
|
|
|
|
|
FilesystemTransactionService.rmSafe(p.rollbackPath);
|
|
|
|
|
// Remove any unused staged remnants and stale sidecars.
|
|
|
|
|
FilesystemTransactionService.rmSafe(p.stagedPath);
|
|
|
|
|
for (const suffix of p.sidecarSuffixes) {
|
|
|
|
|
const stagedSidecar = `${p.stagedPath}${suffix}`;
|
|
|
|
|
FilesystemTransactionService.rmSafe(stagedSidecar);
|
|
|
|
|
// Keep the live sidecar if it was written by the new replacement;
|
|
|
|
|
// it should be empty (just created) so nothing further to clean.
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
this.updateManifest(manifest, txDir, 'committed');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Helper that callers (e.g. DatabaseInitService) can use to decide
|
|
|
|
|
* whether a given rollback/staged artifact is already tracked by a
|
|
|
|
|
* live manifest in the staging root.
|
|
|
|
|
*/
|
|
|
|
|
static hasManifestTracking(stagingDir: string, rollbackPath: string): boolean {
|
|
|
|
|
if (!fs.existsSync(stagingDir)) return false;
|
|
|
|
|
const entries = fs.readdirSync(stagingDir, { withFileTypes: true });
|
|
|
|
|
for (const entry of entries) {
|
|
|
|
|
if (!entry.isDirectory()) continue;
|
|
|
|
|
const manifestPath = path.join(stagingDir, entry.name, FilesystemTransactionService.MANIFEST_FILENAME);
|
|
|
|
|
if (!fs.existsSync(manifestPath)) continue;
|
|
|
|
|
try {
|
|
|
|
|
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) as TransactionManifest;
|
|
|
|
|
for (const p of manifest.pairs) {
|
|
|
|
|
if (p.rollbackPath === rollbackPath || p.stagedPath === rollbackPath || p.livePath === rollbackPath) {
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
// ignore unreadable manifests; recovery will surface them
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* ───────── Manifest helpers ───────── */
|
|
|
|
|
|
|
|
|
|
private ensureTxDir(stagingDir: string): string {
|
|
|
|
|
const id = `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
|
|
|
|
|
const full = path.join(stagingDir, id);
|
|
|
|
|
FilesystemTransactionService.ensureDir(full);
|
|
|
|
|
return full;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private makeInitialManifest(input: StageSwapInput, txDir: string): TransactionManifest {
|
|
|
|
|
const transactionId = path.basename(txDir);
|
|
|
|
|
const now = new Date().toISOString();
|
|
|
|
|
const manifest: TransactionManifest = {
|
|
|
|
|
schemaVersion: 1,
|
|
|
|
|
transactionId,
|
|
|
|
|
operation: input.name,
|
|
|
|
|
createdAt: now,
|
|
|
|
|
updatedAt: now,
|
|
|
|
|
phase: 'prepared',
|
|
|
|
|
pairs: [],
|
|
|
|
|
};
|
|
|
|
|
const stamp = `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
|
|
|
|
|
for (const pair of input.pairs) {
|
|
|
|
|
const rollbackPath = FilesystemTransactionService.makeRollbackPath(pair.livePath, stamp);
|
|
|
|
|
manifest.pairs.push({
|
|
|
|
|
livePath: pair.livePath,
|
|
|
|
|
rollbackPath,
|
|
|
|
|
stagedPath: pair.stagedPath,
|
|
|
|
|
contributesToGeneration: pair.contributesToGeneration ?? true,
|
|
|
|
|
sidecarSuffixes: pair.sidecarSuffixes ?? [],
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
this.writeManifestAtomic(manifest, txDir);
|
|
|
|
|
return manifest;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private updateManifest(manifest: TransactionManifest, txDir: string, phase: TransactionPhase): void {
|
|
|
|
|
manifest.phase = phase;
|
|
|
|
|
manifest.updatedAt = new Date().toISOString();
|
|
|
|
|
this.writeManifestAtomic(manifest, txDir);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private writeManifestAtomic(manifest: TransactionManifest, txDir: string): void {
|
|
|
|
|
const target = path.join(txDir, FilesystemTransactionService.MANIFEST_FILENAME);
|
|
|
|
|
const tmp = `${target}.tmp-${Math.random().toString(36).slice(2, 8)}`;
|
|
|
|
|
FilesystemTransactionService.ensureDir(path.dirname(tmp));
|
|
|
|
|
fs.writeFileSync(tmp, JSON.stringify(manifest));
|
|
|
|
|
fs.renameSync(tmp, target);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private manifestPathFor(_transactionId: string): string {
|
|
|
|
|
// Used only by recovery paths that already know where the manifest lives.
|
|
|
|
|
void _transactionId;
|
|
|
|
|
return FilesystemTransactionService.MANIFEST_FILENAME;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* ───────── Static filesystem helpers ───────── */
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Recursively remove a directory or file. Best-effort.
|
|
|
|
|
*/
|
|
|
|
@@ -188,8 +674,7 @@ export class FilesystemTransactionService {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Copy a directory recursively, falling back gracefully if the source
|
|
|
|
|
* doesn't exist (returns immediately).
|
|
|
|
|
* Copy a directory recursively; gracefully no-ops if the source doesn't exist.
|
|
|
|
|
*/
|
|
|
|
|
static copyDirSync(src: string, dest: string): void {
|
|
|
|
|
if (!fs.existsSync(src)) return;
|
|
|
|
@@ -211,7 +696,7 @@ export class FilesystemTransactionService {
|
|
|
|
|
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static makeRollbackPath(livePath: string, stamp: string): string {
|
|
|
|
|
static makeRollbackPath(livePath: string, stamp: string): string {
|
|
|
|
|
const dir = path.dirname(livePath);
|
|
|
|
|
const base = path.basename(livePath);
|
|
|
|
|
return path.join(dir, `.${base}.rollback-${stamp}`);
|
|
|
|
@@ -221,11 +706,9 @@ export class FilesystemTransactionService {
|
|
|
|
|
if (!handle || handle.rolledBack) return;
|
|
|
|
|
// Walk pairs in reverse order to maximize recovery chances.
|
|
|
|
|
for (const r of [...handle.pairs].reverse()) {
|
|
|
|
|
const stageStillAtLive = fs.existsSync(r.livePath) && r.stagedPath
|
|
|
|
|
const stageStillAtLive = fs.existsSync(r.livePath)
|
|
|
|
|
? (() => {
|
|
|
|
|
try {
|
|
|
|
|
const liveStat = fs.statSync(r.livePath);
|
|
|
|
|
// If we already swapped, stagedDir is gone.
|
|
|
|
|
return !fs.existsSync(r.stagedPath);
|
|
|
|
|
} catch {
|
|
|
|
|
return true;
|
|
|
|
@@ -235,11 +718,7 @@ export class FilesystemTransactionService {
|
|
|
|
|
try {
|
|
|
|
|
if (stageStillAtLive) {
|
|
|
|
|
// swap happened — remove the now-swapped content before restoring.
|
|
|
|
|
try {
|
|
|
|
|
fs.rmSync(r.livePath, { recursive: true, force: true });
|
|
|
|
|
} catch {
|
|
|
|
|
// ignore
|
|
|
|
|
}
|
|
|
|
|
FilesystemTransactionService.rmSafe(r.livePath);
|
|
|
|
|
}
|
|
|
|
|
if (fs.existsSync(r.rollbackPath)) {
|
|
|
|
|
FilesystemTransactionService.ensureDir(path.dirname(r.livePath));
|
|
|
|
@@ -247,19 +726,25 @@ export class FilesystemTransactionService {
|
|
|
|
|
fs.renameSync(r.rollbackPath, r.livePath);
|
|
|
|
|
} catch {
|
|
|
|
|
FilesystemTransactionService.copyDirSync(r.rollbackPath, r.livePath);
|
|
|
|
|
try {
|
|
|
|
|
fs.rmSync(r.rollbackPath, { recursive: true, force: true });
|
|
|
|
|
} catch {
|
|
|
|
|
// ignore
|
|
|
|
|
}
|
|
|
|
|
try { fs.rmSync(r.rollbackPath, { recursive: true, force: true }); } catch { /* ignore */ }
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// If a staged dir remained because swap failed before move, clean it up.
|
|
|
|
|
if (fs.existsSync(r.stagedPath)) {
|
|
|
|
|
try {
|
|
|
|
|
fs.rmSync(r.stagedPath, { recursive: true, force: true });
|
|
|
|
|
} catch {
|
|
|
|
|
// ignore
|
|
|
|
|
FilesystemTransactionService.rmSafe(r.stagedPath);
|
|
|
|
|
}
|
|
|
|
|
// Restore sidecars if they were tracked.
|
|
|
|
|
for (const suffix of r.sidecarSuffixes) {
|
|
|
|
|
const liveSidecar = `${r.livePath}${suffix}`;
|
|
|
|
|
const rollbackSidecar = `${r.rollbackPath}${suffix}`;
|
|
|
|
|
if (fs.existsSync(rollbackSidecar)) {
|
|
|
|
|
FilesystemTransactionService.rmSafe(liveSidecar);
|
|
|
|
|
FilesystemTransactionService.ensureDir(path.dirname(liveSidecar));
|
|
|
|
|
try {
|
|
|
|
|
fs.renameSync(rollbackSidecar, liveSidecar);
|
|
|
|
|
} catch {
|
|
|
|
|
FilesystemTransactionService.copyDirSync(rollbackSidecar, liveSidecar);
|
|
|
|
|
try { fs.rmSync(rollbackSidecar, { recursive: true, force: true }); } catch { /* ignore */ }
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch (err) {
|
|
|
|
|
OpenVelo Agent