OpenVelo Agent
301 lines
13 KiB
TypeScript
301 lines
13 KiB
TypeScript
process.env.DATABASE_PATH = ':memory:';
|
|
process.env.THEMES_DIR = './themes';
|
|
process.env.FRONTEND_DIST = './frontend/dist';
|
|
process.env.UPLOAD_SIZE_LIMIT = '5mb';
|
|
process.env.NODE_ENV = 'test';
|
|
|
|
import * as fs from 'fs';
|
|
import * as os from 'os';
|
|
import * as path from 'path';
|
|
import { FilesystemTransactionService } from '../../backend/src/modules/admin/system/filesystem-transaction.service';
|
|
import { ERROR_CODES } from '../../backend/src/common/errors/error-codes';
|
|
import { TransactionManifest } from '../../backend/src/modules/admin/system/filesystem-transaction.service';
|
|
|
|
function makeRoot(prefix: string): { root: string; cleanup: () => void } {
|
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
|
return {
|
|
root,
|
|
cleanup: () => {
|
|
try { fs.rmSync(root, { recursive: true, force: true }); } catch { /* ignore */ }
|
|
},
|
|
};
|
|
}
|
|
|
|
function readManifest(stagingDir: string, id: string): TransactionManifest {
|
|
const full = path.join(stagingDir, id, FilesystemTransactionService.MANIFEST_FILENAME);
|
|
return JSON.parse(fs.readFileSync(full, 'utf8')) as TransactionManifest;
|
|
}
|
|
|
|
describe('Job 919: FilesystemTransactionService durable manifest', () => {
|
|
let svc: FilesystemTransactionService;
|
|
let dataRoot: { root: string; cleanup: () => void };
|
|
|
|
beforeEach(() => {
|
|
svc = new FilesystemTransactionService();
|
|
dataRoot = makeRoot('hipctf-fs-tx-');
|
|
});
|
|
|
|
afterEach(() => {
|
|
dataRoot.cleanup();
|
|
});
|
|
|
|
it('stageSwap + commit leaves the new generation in place and removes rollback/staged artifacts', async () => {
|
|
const live = path.join(dataRoot.root, 'live.txt');
|
|
const staged = path.join(dataRoot.root, 'staged.txt');
|
|
fs.writeFileSync(live, 'OLD');
|
|
fs.writeFileSync(staged, 'NEW');
|
|
|
|
const handle = await svc.stageSwap(
|
|
{
|
|
name: 'unit-swap',
|
|
pairs: [{ livePath: live, stagedPath: staged, contributesToGeneration: true }],
|
|
},
|
|
{ stagingDir: dataRoot.root },
|
|
);
|
|
expect(handle.transactionId).toBeTruthy();
|
|
expect(fs.readFileSync(live, 'utf8')).toBe('NEW');
|
|
// Manifest exists and is at the `verified` phase before commit
|
|
const manifest = readManifest(dataRoot.root, handle.transactionId!);
|
|
expect(manifest.phase).toBe('verified');
|
|
|
|
svc.commit(handle);
|
|
expect(fs.existsSync(handle.manifestDir!)).toBe(false);
|
|
expect(fs.readFileSync(live, 'utf8')).toBe('NEW');
|
|
});
|
|
|
|
it('stageSwap failure rolls the live path back to its prior content and marks manifest rolled-back', async () => {
|
|
const dataDir = path.join(dataRoot.root, 'data');
|
|
fs.mkdirSync(dataDir, { recursive: true });
|
|
const stagingRoot = path.join(dataRoot.root, '.system-staging');
|
|
fs.mkdirSync(stagingRoot, { recursive: true });
|
|
|
|
const live = path.join(dataDir, 'live.bin');
|
|
const staged = path.join(dataDir, 'staged.bin');
|
|
fs.writeFileSync(live, 'OLD-CONTENT');
|
|
fs.writeFileSync(staged, 'NEW-CONTENT');
|
|
|
|
await expect(
|
|
svc.stageSwap(
|
|
{
|
|
name: 'unit-swap-fail',
|
|
pairs: [{ livePath: live, stagedPath: staged, contributesToGeneration: true }],
|
|
hooks: {
|
|
beforeSwap: async () => {
|
|
throw new Error('boom');
|
|
},
|
|
},
|
|
},
|
|
{ stagingDir: stagingRoot },
|
|
),
|
|
).rejects.toMatchObject({ message: 'boom' });
|
|
|
|
expect(fs.readFileSync(live, 'utf8')).toBe('OLD-CONTENT');
|
|
const manifestDirs = fs.readdirSync(stagingRoot);
|
|
expect(manifestDirs.length).toBe(1);
|
|
const manifest = readManifest(stagingRoot, manifestDirs[0]);
|
|
expect(manifest.phase).toBe('rolled-back');
|
|
});
|
|
|
|
it('recoverTransactions restores the OLD generation when the manifest is at live-snapshotted and both rollback dirs are intact', () => {
|
|
const liveDb = path.join(dataRoot.root, 'db.sqlite');
|
|
const stagedDb = path.join(dataRoot.root, 'db.staged.sqlite');
|
|
fs.writeFileSync(liveDb, 'OLD-DB');
|
|
fs.writeFileSync(stagedDb, 'NEW-DB');
|
|
|
|
const liveUploads = path.join(dataRoot.root, 'uploads');
|
|
const stagedUploads = path.join(dataRoot.root, 'uploads.stage');
|
|
fs.mkdirSync(liveUploads, { recursive: true });
|
|
fs.mkdirSync(stagedUploads, { recursive: true });
|
|
fs.writeFileSync(path.join(liveUploads, 'keep.txt'), 'OLD-UPLOAD');
|
|
fs.writeFileSync(path.join(stagedUploads, 'replaced.txt'), 'NEW-UPLOAD');
|
|
|
|
const id = `${Date.now()}-abcdef`;
|
|
const txDir = path.join(dataRoot.root, id);
|
|
fs.mkdirSync(txDir, { recursive: true });
|
|
const stamp = `${Date.now()}-12345`;
|
|
const manifest: TransactionManifest = {
|
|
schemaVersion: 1,
|
|
transactionId: id,
|
|
operation: 'restore-backup',
|
|
createdAt: new Date().toISOString(),
|
|
updatedAt: new Date().toISOString(),
|
|
phase: 'live-snapshotted',
|
|
pairs: [
|
|
{
|
|
livePath: liveDb,
|
|
rollbackPath: `${liveDb}.rollback-${stamp}`,
|
|
stagedPath: stagedDb,
|
|
contributesToGeneration: true,
|
|
sidecarSuffixes: ['-wal', '-shm'],
|
|
},
|
|
{
|
|
livePath: liveUploads,
|
|
rollbackPath: `${liveUploads}.rollback-${stamp}`,
|
|
stagedPath: stagedUploads,
|
|
contributesToGeneration: true,
|
|
sidecarSuffixes: [],
|
|
},
|
|
],
|
|
};
|
|
fs.writeFileSync(path.join(txDir, FilesystemTransactionService.MANIFEST_FILENAME), JSON.stringify(manifest));
|
|
// Simulate the in-progress state where the snapshot-to-rollback has already happened.
|
|
fs.renameSync(liveDb, manifest.pairs[0].rollbackPath);
|
|
fs.renameSync(liveUploads, manifest.pairs[1].rollbackPath);
|
|
|
|
const report = svc.recoverTransactions(dataRoot.root);
|
|
expect(report.transactionsScanned).toBe(1);
|
|
expect(report.resolved).toBe(1);
|
|
expect(fs.readFileSync(liveDb, 'utf8')).toBe('OLD-DB');
|
|
expect(fs.readFileSync(path.join(liveUploads, 'keep.txt'), 'utf8')).toBe('OLD-UPLOAD');
|
|
// Staged trees should not have ended up at the live paths.
|
|
expect(fs.existsSync(stagedDb)).toBe(true);
|
|
expect(fs.existsSync(stagedUploads)).toBe(true);
|
|
});
|
|
|
|
it('recoverTransactions keeps the NEW generation when both pairs swapped but manifest is at verified', () => {
|
|
const liveDb = path.join(dataRoot.root, 'db.sqlite');
|
|
const liveUploads = path.join(dataRoot.root, 'uploads');
|
|
fs.writeFileSync(liveDb, 'NEW-DB');
|
|
fs.mkdirSync(liveUploads, { recursive: true });
|
|
fs.writeFileSync(path.join(liveUploads, 'replaced.txt'), 'NEW-UPLOAD');
|
|
|
|
const stamp = `${Date.now()}-99999`;
|
|
const oldDbPath = `${liveDb}.rollback-${stamp}`;
|
|
const oldUpPath = `${liveUploads}.rollback-${stamp}`;
|
|
fs.writeFileSync(oldDbPath, 'OLD-DB');
|
|
fs.mkdirSync(oldUpPath, { recursive: true });
|
|
fs.writeFileSync(path.join(oldUpPath, 'keep.txt'), 'OLD-UPLOAD');
|
|
|
|
const id = `${Date.now()}-x9`;
|
|
const txDir = path.join(dataRoot.root, id);
|
|
fs.mkdirSync(txDir, { recursive: true });
|
|
const manifest: TransactionManifest = {
|
|
schemaVersion: 1,
|
|
transactionId: id,
|
|
operation: 'restore-backup',
|
|
createdAt: new Date().toISOString(),
|
|
updatedAt: new Date().toISOString(),
|
|
phase: 'verified',
|
|
pairs: [
|
|
{ livePath: liveDb, rollbackPath: oldDbPath, stagedPath: path.join(dataRoot.root, 'unused.db'), contributesToGeneration: true, sidecarSuffixes: [] },
|
|
{ livePath: liveUploads, rollbackPath: oldUpPath, stagedPath: path.join(dataRoot.root, 'unused.up'), contributesToGeneration: true, sidecarSuffixes: [] },
|
|
],
|
|
};
|
|
fs.writeFileSync(path.join(txDir, FilesystemTransactionService.MANIFEST_FILENAME), JSON.stringify(manifest));
|
|
|
|
const report = svc.recoverTransactions(dataRoot.root);
|
|
expect(report.resolved).toBe(1);
|
|
expect(fs.readFileSync(liveDb, 'utf8')).toBe('NEW-DB');
|
|
expect(fs.readFileSync(path.join(liveUploads, 'replaced.txt'), 'utf8')).toBe('NEW-UPLOAD');
|
|
// Rollback trees for the (now finalized) new generation must be gone.
|
|
expect(fs.existsSync(oldDbPath)).toBe(false);
|
|
expect(fs.existsSync(oldUpPath)).toBe(false);
|
|
});
|
|
|
|
it('recoverTransactions fails closed when neither generation is fully present (the Job 919 hybrid state)', () => {
|
|
// Reproduce the actual reported hybrid: the swap committed the new
|
|
// uploads tree to the live path and wrote a rollback of the OLD
|
|
// uploads tree, but the DB live→rollback snapshot happened while the
|
|
// staged DB never reached the live path. From the resolver's point
|
|
// of view the DB pair looks "old-complete" (livePath missing,
|
|
// rollbackPath present), the uploads pair looks "new-complete"
|
|
// (livePath present, rollbackPath present). Both pairs are not
|
|
// consistent; recovery must fail closed without modifying either
|
|
// generation.
|
|
const dataDir = path.join(dataRoot.root, 'data');
|
|
fs.mkdirSync(dataDir, { recursive: true });
|
|
const liveDb = path.join(dataDir, 'db.sqlite');
|
|
const liveUploads = path.join(dataDir, 'uploads');
|
|
fs.mkdirSync(liveUploads, { recursive: true });
|
|
fs.writeFileSync(path.join(liveUploads, 'replaced.txt'), 'NEW-ORPHAN'); // stale from a failed swap
|
|
|
|
const stamp = `${Date.now()}-hybrid`;
|
|
const oldDbRollback = `${liveDb}.rollback-${stamp}`;
|
|
const oldUpRollback = `${liveUploads}.rollback-${stamp}`;
|
|
fs.writeFileSync(oldDbRollback, 'OLD-DB-SNAPSHOT');
|
|
fs.mkdirSync(oldUpRollback, { recursive: true });
|
|
fs.writeFileSync(path.join(oldUpRollback, 'keep.txt'), 'OLD-UPLOAD-SNAPSHOT');
|
|
|
|
const id = `${Date.now()}-hybrid`;
|
|
const stagingRoot = path.join(dataRoot.root, '.system-staging');
|
|
fs.mkdirSync(stagingRoot, { recursive: true });
|
|
const txDir = path.join(stagingRoot, id);
|
|
fs.mkdirSync(txDir, { recursive: true });
|
|
const manifest: TransactionManifest = {
|
|
schemaVersion: 1,
|
|
transactionId: id,
|
|
operation: 'restore-backup',
|
|
createdAt: new Date().toISOString(),
|
|
updatedAt: new Date().toISOString(),
|
|
phase: 'verified',
|
|
pairs: [
|
|
{ livePath: liveDb, rollbackPath: oldDbRollback, stagedPath: path.join(dataDir, 'unused.db'), contributesToGeneration: true, sidecarSuffixes: [] },
|
|
{ livePath: liveUploads, rollbackPath: oldUpRollback, stagedPath: path.join(dataDir, 'unused.up'), contributesToGeneration: true, sidecarSuffixes: [] },
|
|
],
|
|
};
|
|
fs.writeFileSync(path.join(txDir, FilesystemTransactionService.MANIFEST_FILENAME), JSON.stringify(manifest));
|
|
|
|
expect(() => svc.recoverTransactions(stagingRoot)).toThrow(
|
|
expect.objectContaining({ code: ERROR_CODES.SYSTEM_RESTORE_FAILED }),
|
|
);
|
|
|
|
// Fail-closed must NOT delete or move anything in either generation.
|
|
expect(fs.readFileSync(oldDbRollback, 'utf8')).toBe('OLD-DB-SNAPSHOT');
|
|
expect(fs.readFileSync(path.join(liveUploads, 'replaced.txt'), 'utf8')).toBe('NEW-ORPHAN');
|
|
expect(fs.readFileSync(path.join(oldUpRollback, 'keep.txt'), 'utf8')).toBe('OLD-UPLOAD-SNAPSHOT');
|
|
// Manifest stays on disk for the operator.
|
|
expect(fs.existsSync(txDir)).toBe(true);
|
|
});
|
|
|
|
it('sweepUnownedArtifactsNearLivePaths cleans up stale rollback/restore-stage/wipe-stage directories', () => {
|
|
const dataDir = path.join(dataRoot.root, 'data');
|
|
fs.mkdirSync(dataDir, { recursive: true });
|
|
const liveDb = path.join(dataDir, 'db.sqlite');
|
|
fs.writeFileSync(liveDb, 'OLD');
|
|
const liveUploads = path.join(dataDir, 'uploads');
|
|
fs.mkdirSync(liveUploads, { recursive: true });
|
|
const challenges = path.join(liveUploads, 'challenges');
|
|
fs.mkdirSync(challenges, { recursive: true });
|
|
|
|
const now = Date.now();
|
|
const oldStamp = now - 2 * 24 * 60 * 60_000; // 2 days old
|
|
const freshStamp = now - 60_000; // 1 minute old
|
|
|
|
const rollbackUploads = `${liveUploads}.rollback-${oldStamp}-abc`;
|
|
const rollbackDb = path.join(dataDir, `.db.sqlite.rollback-${oldStamp}-def`);
|
|
const stageUploads = `${liveUploads}.restore-stage-${oldStamp}-ghi`;
|
|
const wipeChallenges = path.join(challenges, `.challenges.wipe-stage-${oldStamp}-jkl`);
|
|
fs.mkdirSync(rollbackUploads, { recursive: true });
|
|
fs.writeFileSync(rollbackDb, 'orphan');
|
|
fs.mkdirSync(stageUploads, { recursive: true });
|
|
fs.mkdirSync(wipeChallenges, { recursive: true });
|
|
|
|
// A fresh rollback pattern must NOT be removed.
|
|
const freshRollbackUploads = `${liveUploads}.rollback-${freshStamp}-mno`;
|
|
fs.mkdirSync(freshRollbackUploads, { recursive: true });
|
|
|
|
// Backdate the old artifacts.
|
|
const oldTime = new Date(oldStamp);
|
|
fs.utimesSync(rollbackUploads, oldTime, oldTime);
|
|
fs.utimesSync(rollbackDb, oldTime, oldTime);
|
|
fs.utimesSync(stageUploads, oldTime, oldTime);
|
|
fs.utimesSync(wipeChallenges, oldTime, oldTime);
|
|
|
|
const cleaned = svc.sweepUnownedArtifactsNearLivePaths([liveDb, liveUploads], {
|
|
now,
|
|
maxAgeMs: 60 * 60_000,
|
|
stagingDir: dataRoot.root,
|
|
});
|
|
expect(cleaned).toEqual(expect.arrayContaining([rollbackUploads, rollbackDb, stageUploads, wipeChallenges]));
|
|
expect(fs.existsSync(rollbackUploads)).toBe(false);
|
|
expect(fs.existsSync(rollbackDb)).toBe(false);
|
|
expect(fs.existsSync(stageUploads)).toBe(false);
|
|
expect(fs.existsSync(wipeChallenges)).toBe(false);
|
|
// Fresh rollback path remains.
|
|
expect(fs.existsSync(freshRollbackUploads)).toBe(true);
|
|
|
|
// Cleanup helper for the remaining fresh artifact.
|
|
try { fs.rmSync(freshRollbackUploads, { recursive: true, force: true }); } catch { /* ignore */ }
|
|
});
|
|
});
|