process.env.DATABASE_PATH = ':memory:'; process.env.THEMES_DIR = './themes'; process.env.FRONTEND_DIST = './frontend/dist'; process.env.UPLOAD_SIZE_LIMIT = '5mb'; process.env.NODE_ENV = 'test'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import { FilesystemTransactionService } from '../../backend/src/modules/admin/system/filesystem-transaction.service'; import { ERROR_CODES } from '../../backend/src/common/errors/error-codes'; import { TransactionManifest } from '../../backend/src/modules/admin/system/filesystem-transaction.service'; function makeRoot(prefix: string): { root: string; cleanup: () => void } { const root = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); return { root, cleanup: () => { try { fs.rmSync(root, { recursive: true, force: true }); } catch { /* ignore */ } }, }; } function readManifest(stagingDir: string, id: string): TransactionManifest { const full = path.join(stagingDir, id, FilesystemTransactionService.MANIFEST_FILENAME); return JSON.parse(fs.readFileSync(full, 'utf8')) as TransactionManifest; } describe('Job 919: FilesystemTransactionService durable manifest', () => { let svc: FilesystemTransactionService; let dataRoot: { root: string; cleanup: () => void }; beforeEach(() => { svc = new FilesystemTransactionService(); dataRoot = makeRoot('hipctf-fs-tx-'); }); afterEach(() => { dataRoot.cleanup(); }); it('stageSwap + commit leaves the new generation in place and removes rollback/staged artifacts', async () => { const live = path.join(dataRoot.root, 'live.txt'); const staged = path.join(dataRoot.root, 'staged.txt'); fs.writeFileSync(live, 'OLD'); fs.writeFileSync(staged, 'NEW'); const handle = await svc.stageSwap( { name: 'unit-swap', pairs: [{ livePath: live, stagedPath: staged, contributesToGeneration: true }], }, { stagingDir: dataRoot.root }, ); expect(handle.transactionId).toBeTruthy(); expect(fs.readFileSync(live, 'utf8')).toBe('NEW'); // Manifest exists and is at the `verified` phase before commit const manifest = readManifest(dataRoot.root, handle.transactionId!); expect(manifest.phase).toBe('verified'); svc.commit(handle); expect(fs.existsSync(handle.manifestDir!)).toBe(false); expect(fs.readFileSync(live, 'utf8')).toBe('NEW'); }); it('stageSwap failure rolls the live path back to its prior content and marks manifest rolled-back', async () => { const dataDir = path.join(dataRoot.root, 'data'); fs.mkdirSync(dataDir, { recursive: true }); const stagingRoot = path.join(dataRoot.root, '.system-staging'); fs.mkdirSync(stagingRoot, { recursive: true }); const live = path.join(dataDir, 'live.bin'); const staged = path.join(dataDir, 'staged.bin'); fs.writeFileSync(live, 'OLD-CONTENT'); fs.writeFileSync(staged, 'NEW-CONTENT'); await expect( svc.stageSwap( { name: 'unit-swap-fail', pairs: [{ livePath: live, stagedPath: staged, contributesToGeneration: true }], hooks: { beforeSwap: async () => { throw new Error('boom'); }, }, }, { stagingDir: stagingRoot }, ), ).rejects.toMatchObject({ message: 'boom' }); expect(fs.readFileSync(live, 'utf8')).toBe('OLD-CONTENT'); const manifestDirs = fs.readdirSync(stagingRoot); expect(manifestDirs.length).toBe(1); const manifest = readManifest(stagingRoot, manifestDirs[0]); expect(manifest.phase).toBe('rolled-back'); }); it('recoverTransactions restores the OLD generation when the manifest is at live-snapshotted and both rollback dirs are intact', () => { const liveDb = path.join(dataRoot.root, 'db.sqlite'); const stagedDb = path.join(dataRoot.root, 'db.staged.sqlite'); fs.writeFileSync(liveDb, 'OLD-DB'); fs.writeFileSync(stagedDb, 'NEW-DB'); const liveUploads = path.join(dataRoot.root, 'uploads'); const stagedUploads = path.join(dataRoot.root, 'uploads.stage'); fs.mkdirSync(liveUploads, { recursive: true }); fs.mkdirSync(stagedUploads, { recursive: true }); fs.writeFileSync(path.join(liveUploads, 'keep.txt'), 'OLD-UPLOAD'); fs.writeFileSync(path.join(stagedUploads, 'replaced.txt'), 'NEW-UPLOAD'); const id = `${Date.now()}-abcdef`; const txDir = path.join(dataRoot.root, id); fs.mkdirSync(txDir, { recursive: true }); const stamp = `${Date.now()}-12345`; const manifest: TransactionManifest = { schemaVersion: 1, transactionId: id, operation: 'restore-backup', createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), phase: 'live-snapshotted', pairs: [ { livePath: liveDb, rollbackPath: `${liveDb}.rollback-${stamp}`, stagedPath: stagedDb, contributesToGeneration: true, sidecarSuffixes: ['-wal', '-shm'], }, { livePath: liveUploads, rollbackPath: `${liveUploads}.rollback-${stamp}`, stagedPath: stagedUploads, contributesToGeneration: true, sidecarSuffixes: [], }, ], }; fs.writeFileSync(path.join(txDir, FilesystemTransactionService.MANIFEST_FILENAME), JSON.stringify(manifest)); // Simulate the in-progress state where the snapshot-to-rollback has already happened. fs.renameSync(liveDb, manifest.pairs[0].rollbackPath); fs.renameSync(liveUploads, manifest.pairs[1].rollbackPath); const report = svc.recoverTransactions(dataRoot.root); expect(report.transactionsScanned).toBe(1); expect(report.resolved).toBe(1); expect(fs.readFileSync(liveDb, 'utf8')).toBe('OLD-DB'); expect(fs.readFileSync(path.join(liveUploads, 'keep.txt'), 'utf8')).toBe('OLD-UPLOAD'); // Staged trees should not have ended up at the live paths. expect(fs.existsSync(stagedDb)).toBe(true); expect(fs.existsSync(stagedUploads)).toBe(true); }); it('recoverTransactions keeps the NEW generation when both pairs swapped but manifest is at verified', () => { const liveDb = path.join(dataRoot.root, 'db.sqlite'); const liveUploads = path.join(dataRoot.root, 'uploads'); fs.writeFileSync(liveDb, 'NEW-DB'); fs.mkdirSync(liveUploads, { recursive: true }); fs.writeFileSync(path.join(liveUploads, 'replaced.txt'), 'NEW-UPLOAD'); const stamp = `${Date.now()}-99999`; const oldDbPath = `${liveDb}.rollback-${stamp}`; const oldUpPath = `${liveUploads}.rollback-${stamp}`; fs.writeFileSync(oldDbPath, 'OLD-DB'); fs.mkdirSync(oldUpPath, { recursive: true }); fs.writeFileSync(path.join(oldUpPath, 'keep.txt'), 'OLD-UPLOAD'); const id = `${Date.now()}-x9`; const txDir = path.join(dataRoot.root, id); fs.mkdirSync(txDir, { recursive: true }); const manifest: TransactionManifest = { schemaVersion: 1, transactionId: id, operation: 'restore-backup', createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), phase: 'verified', pairs: [ { livePath: liveDb, rollbackPath: oldDbPath, stagedPath: path.join(dataRoot.root, 'unused.db'), contributesToGeneration: true, sidecarSuffixes: [] }, { livePath: liveUploads, rollbackPath: oldUpPath, stagedPath: path.join(dataRoot.root, 'unused.up'), contributesToGeneration: true, sidecarSuffixes: [] }, ], }; fs.writeFileSync(path.join(txDir, FilesystemTransactionService.MANIFEST_FILENAME), JSON.stringify(manifest)); const report = svc.recoverTransactions(dataRoot.root); expect(report.resolved).toBe(1); expect(fs.readFileSync(liveDb, 'utf8')).toBe('NEW-DB'); expect(fs.readFileSync(path.join(liveUploads, 'replaced.txt'), 'utf8')).toBe('NEW-UPLOAD'); // Rollback trees for the (now finalized) new generation must be gone. expect(fs.existsSync(oldDbPath)).toBe(false); expect(fs.existsSync(oldUpPath)).toBe(false); }); it('recoverTransactions fails closed when neither generation is fully present (the Job 919 hybrid state)', () => { // Reproduce the actual reported hybrid: the swap committed the new // uploads tree to the live path and wrote a rollback of the OLD // uploads tree, but the DB live→rollback snapshot happened while the // staged DB never reached the live path. From the resolver's point // of view the DB pair looks "old-complete" (livePath missing, // rollbackPath present), the uploads pair looks "new-complete" // (livePath present, rollbackPath present). Both pairs are not // consistent; recovery must fail closed without modifying either // generation. const dataDir = path.join(dataRoot.root, 'data'); fs.mkdirSync(dataDir, { recursive: true }); const liveDb = path.join(dataDir, 'db.sqlite'); const liveUploads = path.join(dataDir, 'uploads'); fs.mkdirSync(liveUploads, { recursive: true }); fs.writeFileSync(path.join(liveUploads, 'replaced.txt'), 'NEW-ORPHAN'); // stale from a failed swap const stamp = `${Date.now()}-hybrid`; const oldDbRollback = `${liveDb}.rollback-${stamp}`; const oldUpRollback = `${liveUploads}.rollback-${stamp}`; fs.writeFileSync(oldDbRollback, 'OLD-DB-SNAPSHOT'); fs.mkdirSync(oldUpRollback, { recursive: true }); fs.writeFileSync(path.join(oldUpRollback, 'keep.txt'), 'OLD-UPLOAD-SNAPSHOT'); const id = `${Date.now()}-hybrid`; const stagingRoot = path.join(dataRoot.root, '.system-staging'); fs.mkdirSync(stagingRoot, { recursive: true }); const txDir = path.join(stagingRoot, id); fs.mkdirSync(txDir, { recursive: true }); const manifest: TransactionManifest = { schemaVersion: 1, transactionId: id, operation: 'restore-backup', createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), phase: 'verified', pairs: [ { livePath: liveDb, rollbackPath: oldDbRollback, stagedPath: path.join(dataDir, 'unused.db'), contributesToGeneration: true, sidecarSuffixes: [] }, { livePath: liveUploads, rollbackPath: oldUpRollback, stagedPath: path.join(dataDir, 'unused.up'), contributesToGeneration: true, sidecarSuffixes: [] }, ], }; fs.writeFileSync(path.join(txDir, FilesystemTransactionService.MANIFEST_FILENAME), JSON.stringify(manifest)); expect(() => svc.recoverTransactions(stagingRoot)).toThrow( expect.objectContaining({ code: ERROR_CODES.SYSTEM_RESTORE_FAILED }), ); // Fail-closed must NOT delete or move anything in either generation. expect(fs.readFileSync(oldDbRollback, 'utf8')).toBe('OLD-DB-SNAPSHOT'); expect(fs.readFileSync(path.join(liveUploads, 'replaced.txt'), 'utf8')).toBe('NEW-ORPHAN'); expect(fs.readFileSync(path.join(oldUpRollback, 'keep.txt'), 'utf8')).toBe('OLD-UPLOAD-SNAPSHOT'); // Manifest stays on disk for the operator. expect(fs.existsSync(txDir)).toBe(true); }); it('sweepUnownedArtifactsNearLivePaths cleans up stale rollback/restore-stage/wipe-stage directories', () => { const dataDir = path.join(dataRoot.root, 'data'); fs.mkdirSync(dataDir, { recursive: true }); const liveDb = path.join(dataDir, 'db.sqlite'); fs.writeFileSync(liveDb, 'OLD'); const liveUploads = path.join(dataDir, 'uploads'); fs.mkdirSync(liveUploads, { recursive: true }); const challenges = path.join(liveUploads, 'challenges'); fs.mkdirSync(challenges, { recursive: true }); const now = Date.now(); const oldStamp = now - 2 * 24 * 60 * 60_000; // 2 days old const freshStamp = now - 60_000; // 1 minute old const rollbackUploads = `${liveUploads}.rollback-${oldStamp}-abc`; const rollbackDb = path.join(dataDir, `.db.sqlite.rollback-${oldStamp}-def`); const stageUploads = `${liveUploads}.restore-stage-${oldStamp}-ghi`; const wipeChallenges = path.join(challenges, `.challenges.wipe-stage-${oldStamp}-jkl`); fs.mkdirSync(rollbackUploads, { recursive: true }); fs.writeFileSync(rollbackDb, 'orphan'); fs.mkdirSync(stageUploads, { recursive: true }); fs.mkdirSync(wipeChallenges, { recursive: true }); // A fresh rollback pattern must NOT be removed. const freshRollbackUploads = `${liveUploads}.rollback-${freshStamp}-mno`; fs.mkdirSync(freshRollbackUploads, { recursive: true }); // Backdate the old artifacts. const oldTime = new Date(oldStamp); fs.utimesSync(rollbackUploads, oldTime, oldTime); fs.utimesSync(rollbackDb, oldTime, oldTime); fs.utimesSync(stageUploads, oldTime, oldTime); fs.utimesSync(wipeChallenges, oldTime, oldTime); const cleaned = svc.sweepUnownedArtifactsNearLivePaths([liveDb, liveUploads], { now, maxAgeMs: 60 * 60_000, stagingDir: dataRoot.root, }); expect(cleaned).toEqual(expect.arrayContaining([rollbackUploads, rollbackDb, stageUploads, wipeChallenges])); expect(fs.existsSync(rollbackUploads)).toBe(false); expect(fs.existsSync(rollbackDb)).toBe(false); expect(fs.existsSync(stageUploads)).toBe(false); expect(fs.existsSync(wipeChallenges)).toBe(false); // Fresh rollback path remains. expect(fs.existsSync(freshRollbackUploads)).toBe(true); // Cleanup helper for the remaining fresh artifact. try { fs.rmSync(freshRollbackUploads, { recursive: true, force: true }); } catch { /* ignore */ } }); });