feat: Admin Area System: Database Backup/Restore and Danger Zone 1.02
This commit is contained in:
OpenVelo Agent
@@ -356,3 +356,113 @@ describe('Job 918: Restore commit succeeds end-to-end with a real backup file',
|
||||
fs.rmSync(work, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe('Job 919: Restore commit survives a kill-9 mid-swap without leaving a hybrid on-disk state', () => {
|
||||
// The reported Job 919 incident: a process kill during commit produced
|
||||
// a hybrid where the live DB reverted to the old generation but the live
|
||||
// uploads contained the new generation's file (and a rollback directory
|
||||
// remained on disk). After the fix, a subsequent startup must clean
|
||||
// the orphan rollback directory and bring the uploads back in line with
|
||||
// the database.
|
||||
|
||||
beforeEach(() => {
|
||||
jest.resetModules();
|
||||
delete process.env.SYSTEM_OP_RESTORE_STAGE_TTL_MS;
|
||||
delete process.env.SYSTEM_OP_CONFIRM_TOKEN_TTL_MS;
|
||||
delete process.env.SYSTEM_OP_STAGING_DIR;
|
||||
delete process.env.UPLOAD_DIR;
|
||||
delete process.env.DATABASE_PATH;
|
||||
});
|
||||
|
||||
it('simulated kill after the first pair swap leaves a manifest that startup recovery restores to a single coherent generation', async () => {
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
const Database = require('better-sqlite3');
|
||||
|
||||
const work = fs.mkdtempSync(path.join(os.tmpdir(), 'hipctf-job919-'));
|
||||
const dataDir = path.join(work, 'data');
|
||||
const uploadDir = path.join(dataDir, 'uploads');
|
||||
fs.mkdirSync(uploadDir, { recursive: true });
|
||||
const dbPath = path.join(dataDir, 'db.sqlite');
|
||||
|
||||
// Live state: OLD DB, OLD upload (forensics icon), with a stale rollback
|
||||
// directory left behind by the prior broken swap attempt.
|
||||
const DatabaseLib = require('better-sqlite3');
|
||||
const live = new DatabaseLib(dbPath);
|
||||
live.exec(`
|
||||
CREATE TABLE "user" (id TEXT PRIMARY KEY, username TEXT, role TEXT);
|
||||
CREATE TABLE "setting" (key TEXT PRIMARY KEY, value TEXT);
|
||||
INSERT INTO "user" (id, username, role) VALUES ('admin-live', 'admin', 'admin');
|
||||
INSERT INTO "setting" (key, value) VALUES ('site_title', 'OLD');
|
||||
`);
|
||||
live.close();
|
||||
fs.writeFileSync(path.join(uploadDir, 'old-icon.png'), Buffer.from('OLD-ICON'));
|
||||
const oldIconBytes = Buffer.from('OLD-ICON');
|
||||
|
||||
// Simulate the in-progress snapshot has already moved the live trees
|
||||
// into the rollback locations, but the staged replacements never reached
|
||||
// the live path. Place the OLD uploads content in the rollback dir.
|
||||
const stamp = `${Date.now()}-sim`;
|
||||
const oldDbRollback = path.join(dataDir, `.db.sqlite.rollback-${stamp}`);
|
||||
const oldUpRollback = `${uploadDir}.rollback-${stamp}`;
|
||||
fs.copyFileSync(dbPath, oldDbRollback);
|
||||
fs.rmSync(dbPath, { recursive: true, force: true });
|
||||
fs.mkdirSync(oldUpRollback, { recursive: true });
|
||||
fs.copyFileSync(path.join(uploadDir, 'old-icon.png'), path.join(oldUpRollback, 'old-icon.png'));
|
||||
fs.rmSync(uploadDir, { recursive: true, force: true });
|
||||
|
||||
const id = `${Date.now()}-kr`;
|
||||
const stagingDir = path.join(dataDir, '.system-staging');
|
||||
fs.mkdirSync(stagingDir, { recursive: true });
|
||||
const txDir = path.join(stagingDir, id);
|
||||
fs.mkdirSync(txDir, { recursive: true });
|
||||
const manifest = {
|
||||
schemaVersion: 1,
|
||||
transactionId: id,
|
||||
operation: 'restore-backup',
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
phase: 'live-snapshotted',
|
||||
pairs: [
|
||||
{ livePath: dbPath, rollbackPath: oldDbRollback, stagedPath: path.join(stagingDir, 'unused.db'), contributesToGeneration: true, sidecarSuffixes: ['-wal', '-shm'] },
|
||||
{ livePath: uploadDir, rollbackPath: oldUpRollback, stagedPath: path.join(stagingDir, 'unused.up'), contributesToGeneration: true, sidecarSuffixes: [] },
|
||||
],
|
||||
};
|
||||
fs.writeFileSync(path.join(txDir, 'manifest.json'), JSON.stringify(manifest));
|
||||
|
||||
process.env.DATABASE_PATH = dbPath;
|
||||
process.env.UPLOAD_DIR = uploadDir;
|
||||
process.env.SYSTEM_OP_STAGING_DIR = stagingDir;
|
||||
|
||||
const { ConfigService } = require('@nestjs/config');
|
||||
const config = new ConfigService({
|
||||
DATABASE_PATH: dbPath,
|
||||
UPLOAD_DIR: uploadDir,
|
||||
SYSTEM_OP_STAGING_DIR: stagingDir,
|
||||
SYSTEM_OP_RESTORE_STAGE_TTL_MS: 900000,
|
||||
SYSTEM_OP_CONFIRM_TOKEN_TTL_MS: 300000,
|
||||
BODY_SIZE_LIMIT: '50mb',
|
||||
} as any);
|
||||
const { DatabaseInitService } = require('../../backend/src/database/database-init.service');
|
||||
const { FilesystemTransactionService } = require('../../backend/src/modules/admin/system/filesystem-transaction.service');
|
||||
const fakeDataSource = { isInitialized: true, query: async () => [] };
|
||||
const init = new DatabaseInitService(fakeDataSource, config, new FilesystemTransactionService());
|
||||
|
||||
await (init as any).recoverInterruptedSystemTransactions();
|
||||
|
||||
// Coherent pre-restore state on disk: DB and uploads agree on the OLD
|
||||
// generation; the rollback artifacts and the manifest are gone.
|
||||
expect(fs.existsSync(dbPath)).toBe(true);
|
||||
const reopened = new Database(dbPath, { readonly: true });
|
||||
expect(reopened.prepare('SELECT value FROM "setting" WHERE key=?').get('site_title').value).toBe('OLD');
|
||||
expect(reopened.prepare('SELECT COUNT(*) AS c FROM "user"').get().c).toBe(1);
|
||||
reopened.close();
|
||||
expect(fs.readFileSync(path.join(uploadDir, 'old-icon.png'))).toEqual(oldIconBytes);
|
||||
expect(fs.existsSync(oldDbRollback)).toBe(false);
|
||||
expect(fs.existsSync(oldUpRollback)).toBe(false);
|
||||
expect(fs.existsSync(txDir)).toBe(false);
|
||||
|
||||
try { fs.rmSync(work, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
process.env.DATABASE_PATH = ':memory:';
|
||||
process.env.THEMES_DIR = './themes';
|
||||
process.env.FRONTEND_DIST = './frontend/dist';
|
||||
process.env.UPLOAD_SIZE_LIMIT = '5mb';
|
||||
process.env.NODE_ENV = 'test';
|
||||
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
|
||||
// ConfigService reads process.env with precedence over its internal
|
||||
// overrides, so unset the env-driven defaults before constructing it
|
||||
// for these tests so the per-test data roots are honored.
|
||||
delete process.env.DATABASE_PATH;
|
||||
delete process.env.UPLOAD_DIR;
|
||||
delete process.env.SYSTEM_OP_STAGING_DIR;
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { DatabaseInitService } from '../../backend/src/database/database-init.service';
|
||||
import {
|
||||
FilesystemTransactionService,
|
||||
TransactionManifest,
|
||||
} from '../../backend/src/modules/admin/system/filesystem-transaction.service';
|
||||
import { ERROR_CODES } from '../../backend/src/common/errors/error-codes';
|
||||
|
||||
function makeRoot(prefix: string): string {
|
||||
return fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
||||
}
|
||||
|
||||
describe('Job 919: DatabaseInitService recovery on startup', () => {
|
||||
let work: string;
|
||||
let dbPath: string;
|
||||
let uploadDir: string;
|
||||
let stagingDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
work = makeRoot('hipctf-job919-init-');
|
||||
dbPath = path.join(work, 'db.sqlite');
|
||||
uploadDir = path.join(work, 'uploads');
|
||||
stagingDir = path.join(work, '.system-staging');
|
||||
// Note: `uploadDir` is intentionally NOT created here. Individual tests
|
||||
// arrange the filesystem in the exact on-disk shape they want to
|
||||
// exercise (e.g. an interrupted-swap manifest expects the live uploads
|
||||
// dir to NOT exist on disk because the in-process snapshot has moved it).
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (work && fs.existsSync(work)) {
|
||||
try { fs.rmSync(work, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
}
|
||||
});
|
||||
|
||||
function buildInit(): DatabaseInitService {
|
||||
const config = new ConfigService({
|
||||
DATABASE_PATH: dbPath,
|
||||
UPLOAD_DIR: uploadDir,
|
||||
SYSTEM_OP_STAGING_DIR: stagingDir,
|
||||
} as any);
|
||||
const fsTx = new FilesystemTransactionService();
|
||||
// The DataSource is not used by the recovery code path, so a stub suffices.
|
||||
const fakeDataSource: any = { isInitialized: true, query: async () => [] };
|
||||
const init = new DatabaseInitService(fakeDataSource, config, fsTx);
|
||||
return init;
|
||||
}
|
||||
|
||||
it('restores the old database + uploads generation from an interrupted restore manifest', async () => {
|
||||
const stamp = `${Date.now()}-11111`;
|
||||
const liveDb = dbPath;
|
||||
const liveUploads = uploadDir;
|
||||
const oldDbRollback = `${liveDb}.rollback-${stamp}`;
|
||||
const oldUpRollback = `${liveUploads}.rollback-${stamp}`;
|
||||
const stagedDb = path.join(stagingDir, 'unused.db');
|
||||
const stagedUploads = path.join(stagingDir, 'unused.up');
|
||||
|
||||
// Simulate the in-process state AFTER the live-snapshot pair rename has
|
||||
// moved the live tree to the rollback location but BEFORE the staged
|
||||
// replacement has been moved into place.
|
||||
fs.writeFileSync(oldDbRollback, 'OLD-DB-SNAPSHOT');
|
||||
fs.mkdirSync(oldUpRollback, { recursive: true });
|
||||
fs.writeFileSync(path.join(oldUpRollback, 'keep.txt'), 'OLD-UPLOAD-SNAPSHOT');
|
||||
|
||||
const id = `${Date.now()}-tyy`;
|
||||
const txDir = path.join(stagingDir, id);
|
||||
fs.mkdirSync(txDir, { recursive: true });
|
||||
const manifest: TransactionManifest = {
|
||||
schemaVersion: 1,
|
||||
transactionId: id,
|
||||
operation: 'restore-backup',
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
phase: 'live-snapshotted',
|
||||
pairs: [
|
||||
{ livePath: liveDb, rollbackPath: oldDbRollback, stagedPath: stagedDb, contributesToGeneration: true, sidecarSuffixes: ['-wal', '-shm'] },
|
||||
{ livePath: liveUploads, rollbackPath: oldUpRollback, stagedPath: stagedUploads, contributesToGeneration: true, sidecarSuffixes: [] },
|
||||
],
|
||||
};
|
||||
fs.writeFileSync(path.join(txDir, FilesystemTransactionService.MANIFEST_FILENAME), JSON.stringify(manifest));
|
||||
|
||||
const init = buildInit();
|
||||
await (init as any).recoverInterruptedSystemTransactions();
|
||||
|
||||
expect(fs.readFileSync(liveDb, 'utf8')).toBe('OLD-DB-SNAPSHOT');
|
||||
expect(fs.readFileSync(path.join(liveUploads, 'keep.txt'), 'utf8')).toBe('OLD-UPLOAD-SNAPSHOT');
|
||||
expect(fs.existsSync(path.join(liveUploads, 'replaced.txt'))).toBe(false);
|
||||
expect(fs.existsSync(oldDbRollback)).toBe(false);
|
||||
expect(fs.existsSync(oldUpRollback)).toBe(false);
|
||||
expect(fs.existsSync(txDir)).toBe(false);
|
||||
});
|
||||
|
||||
it('fails closed (throws) when an interrupted manifest has a hybrid state (no full old or new generation)', async () => {
|
||||
const stamp = `${Date.now()}-22222`;
|
||||
const oldDbRollback = `${dbPath}.rollback-${stamp}`;
|
||||
fs.writeFileSync(oldDbRollback, 'OLD-DB-SNAPSHOT');
|
||||
fs.mkdirSync(uploadDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(uploadDir, 'replaced.txt'), 'NEW-ORPHAN');
|
||||
const oldUpRollback = `${uploadDir}.rollback-${stamp}`;
|
||||
fs.mkdirSync(oldUpRollback, { recursive: true });
|
||||
fs.writeFileSync(path.join(oldUpRollback, 'keep.txt'), 'OLD-UPLOAD-SNAPSHOT');
|
||||
|
||||
const id = `${Date.now()}-hyb`;
|
||||
const txDir = path.join(stagingDir, id);
|
||||
fs.mkdirSync(txDir, { recursive: true });
|
||||
const manifest: TransactionManifest = {
|
||||
schemaVersion: 1,
|
||||
transactionId: id,
|
||||
operation: 'restore-backup',
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
phase: 'verified',
|
||||
pairs: [
|
||||
{ livePath: dbPath, rollbackPath: oldDbRollback, stagedPath: path.join(stagingDir, 'unused.db'), contributesToGeneration: true, sidecarSuffixes: [] },
|
||||
{ livePath: uploadDir, rollbackPath: oldUpRollback, stagedPath: path.join(stagingDir, 'unused.up'), contributesToGeneration: true, sidecarSuffixes: [] },
|
||||
],
|
||||
};
|
||||
fs.writeFileSync(path.join(txDir, FilesystemTransactionService.MANIFEST_FILENAME), JSON.stringify(manifest));
|
||||
|
||||
const init = buildInit();
|
||||
await expect((init as any).recoverInterruptedSystemTransactions()).rejects.toMatchObject({
|
||||
code: ERROR_CODES.SYSTEM_RESTORE_FAILED,
|
||||
});
|
||||
|
||||
expect(fs.readFileSync(oldDbRollback, 'utf8')).toBe('OLD-DB-SNAPSHOT');
|
||||
expect(fs.readFileSync(path.join(uploadDir, 'replaced.txt'), 'utf8')).toBe('NEW-ORPHAN');
|
||||
expect(fs.readFileSync(path.join(oldUpRollback, 'keep.txt'), 'utf8')).toBe('OLD-UPLOAD-SNAPSHOT');
|
||||
expect(fs.existsSync(txDir)).toBe(true);
|
||||
});
|
||||
|
||||
it('removes stale unowned rollback/restore-stage/wipe-stage artifacts during startup sweep', async () => {
|
||||
const now = Date.now();
|
||||
const oldStamp = now - 2 * 24 * 60 * 60_000; // 2 days old
|
||||
|
||||
const staleDbRollback = path.join(work, `.db.sqlite.rollback-${oldStamp}-abc`);
|
||||
const staleUploadsRollback = `${uploadDir}.rollback-${oldStamp}-def`;
|
||||
const staleUploadsStage = `${uploadDir}.restore-stage-${oldStamp}-ghi`;
|
||||
fs.writeFileSync(staleDbRollback, 'orphan');
|
||||
fs.mkdirSync(staleUploadsRollback, { recursive: true });
|
||||
fs.mkdirSync(staleUploadsStage, { recursive: true });
|
||||
const oldTime = new Date(oldStamp);
|
||||
fs.utimesSync(staleDbRollback, oldTime, oldTime);
|
||||
fs.utimesSync(staleUploadsRollback, oldTime, oldTime);
|
||||
fs.utimesSync(staleUploadsStage, oldTime, oldTime);
|
||||
|
||||
const init = buildInit();
|
||||
await (init as any).recoverInterruptedSystemTransactions();
|
||||
|
||||
expect(fs.existsSync(staleDbRollback)).toBe(false);
|
||||
expect(fs.existsSync(staleUploadsRollback)).toBe(false);
|
||||
expect(fs.existsSync(staleUploadsStage)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,300 @@
|
||||
process.env.DATABASE_PATH = ':memory:';
|
||||
process.env.THEMES_DIR = './themes';
|
||||
process.env.FRONTEND_DIST = './frontend/dist';
|
||||
process.env.UPLOAD_SIZE_LIMIT = '5mb';
|
||||
process.env.NODE_ENV = 'test';
|
||||
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { FilesystemTransactionService } from '../../backend/src/modules/admin/system/filesystem-transaction.service';
|
||||
import { ERROR_CODES } from '../../backend/src/common/errors/error-codes';
|
||||
import { TransactionManifest } from '../../backend/src/modules/admin/system/filesystem-transaction.service';
|
||||
|
||||
function makeRoot(prefix: string): { root: string; cleanup: () => void } {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
||||
return {
|
||||
root,
|
||||
cleanup: () => {
|
||||
try { fs.rmSync(root, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function readManifest(stagingDir: string, id: string): TransactionManifest {
|
||||
const full = path.join(stagingDir, id, FilesystemTransactionService.MANIFEST_FILENAME);
|
||||
return JSON.parse(fs.readFileSync(full, 'utf8')) as TransactionManifest;
|
||||
}
|
||||
|
||||
describe('Job 919: FilesystemTransactionService durable manifest', () => {
|
||||
let svc: FilesystemTransactionService;
|
||||
let dataRoot: { root: string; cleanup: () => void };
|
||||
|
||||
beforeEach(() => {
|
||||
svc = new FilesystemTransactionService();
|
||||
dataRoot = makeRoot('hipctf-fs-tx-');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
dataRoot.cleanup();
|
||||
});
|
||||
|
||||
it('stageSwap + commit leaves the new generation in place and removes rollback/staged artifacts', async () => {
|
||||
const live = path.join(dataRoot.root, 'live.txt');
|
||||
const staged = path.join(dataRoot.root, 'staged.txt');
|
||||
fs.writeFileSync(live, 'OLD');
|
||||
fs.writeFileSync(staged, 'NEW');
|
||||
|
||||
const handle = await svc.stageSwap(
|
||||
{
|
||||
name: 'unit-swap',
|
||||
pairs: [{ livePath: live, stagedPath: staged, contributesToGeneration: true }],
|
||||
},
|
||||
{ stagingDir: dataRoot.root },
|
||||
);
|
||||
expect(handle.transactionId).toBeTruthy();
|
||||
expect(fs.readFileSync(live, 'utf8')).toBe('NEW');
|
||||
// Manifest exists and is at the `verified` phase before commit
|
||||
const manifest = readManifest(dataRoot.root, handle.transactionId!);
|
||||
expect(manifest.phase).toBe('verified');
|
||||
|
||||
svc.commit(handle);
|
||||
expect(fs.existsSync(handle.manifestDir!)).toBe(false);
|
||||
expect(fs.readFileSync(live, 'utf8')).toBe('NEW');
|
||||
});
|
||||
|
||||
it('stageSwap failure rolls the live path back to its prior content and marks manifest rolled-back', async () => {
|
||||
const dataDir = path.join(dataRoot.root, 'data');
|
||||
fs.mkdirSync(dataDir, { recursive: true });
|
||||
const stagingRoot = path.join(dataRoot.root, '.system-staging');
|
||||
fs.mkdirSync(stagingRoot, { recursive: true });
|
||||
|
||||
const live = path.join(dataDir, 'live.bin');
|
||||
const staged = path.join(dataDir, 'staged.bin');
|
||||
fs.writeFileSync(live, 'OLD-CONTENT');
|
||||
fs.writeFileSync(staged, 'NEW-CONTENT');
|
||||
|
||||
await expect(
|
||||
svc.stageSwap(
|
||||
{
|
||||
name: 'unit-swap-fail',
|
||||
pairs: [{ livePath: live, stagedPath: staged, contributesToGeneration: true }],
|
||||
hooks: {
|
||||
beforeSwap: async () => {
|
||||
throw new Error('boom');
|
||||
},
|
||||
},
|
||||
},
|
||||
{ stagingDir: stagingRoot },
|
||||
),
|
||||
).rejects.toMatchObject({ message: 'boom' });
|
||||
|
||||
expect(fs.readFileSync(live, 'utf8')).toBe('OLD-CONTENT');
|
||||
const manifestDirs = fs.readdirSync(stagingRoot);
|
||||
expect(manifestDirs.length).toBe(1);
|
||||
const manifest = readManifest(stagingRoot, manifestDirs[0]);
|
||||
expect(manifest.phase).toBe('rolled-back');
|
||||
});
|
||||
|
||||
it('recoverTransactions restores the OLD generation when the manifest is at live-snapshotted and both rollback dirs are intact', () => {
|
||||
const liveDb = path.join(dataRoot.root, 'db.sqlite');
|
||||
const stagedDb = path.join(dataRoot.root, 'db.staged.sqlite');
|
||||
fs.writeFileSync(liveDb, 'OLD-DB');
|
||||
fs.writeFileSync(stagedDb, 'NEW-DB');
|
||||
|
||||
const liveUploads = path.join(dataRoot.root, 'uploads');
|
||||
const stagedUploads = path.join(dataRoot.root, 'uploads.stage');
|
||||
fs.mkdirSync(liveUploads, { recursive: true });
|
||||
fs.mkdirSync(stagedUploads, { recursive: true });
|
||||
fs.writeFileSync(path.join(liveUploads, 'keep.txt'), 'OLD-UPLOAD');
|
||||
fs.writeFileSync(path.join(stagedUploads, 'replaced.txt'), 'NEW-UPLOAD');
|
||||
|
||||
const id = `${Date.now()}-abcdef`;
|
||||
const txDir = path.join(dataRoot.root, id);
|
||||
fs.mkdirSync(txDir, { recursive: true });
|
||||
const stamp = `${Date.now()}-12345`;
|
||||
const manifest: TransactionManifest = {
|
||||
schemaVersion: 1,
|
||||
transactionId: id,
|
||||
operation: 'restore-backup',
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
phase: 'live-snapshotted',
|
||||
pairs: [
|
||||
{
|
||||
livePath: liveDb,
|
||||
rollbackPath: `${liveDb}.rollback-${stamp}`,
|
||||
stagedPath: stagedDb,
|
||||
contributesToGeneration: true,
|
||||
sidecarSuffixes: ['-wal', '-shm'],
|
||||
},
|
||||
{
|
||||
livePath: liveUploads,
|
||||
rollbackPath: `${liveUploads}.rollback-${stamp}`,
|
||||
stagedPath: stagedUploads,
|
||||
contributesToGeneration: true,
|
||||
sidecarSuffixes: [],
|
||||
},
|
||||
],
|
||||
};
|
||||
fs.writeFileSync(path.join(txDir, FilesystemTransactionService.MANIFEST_FILENAME), JSON.stringify(manifest));
|
||||
// Simulate the in-progress state where the snapshot-to-rollback has already happened.
|
||||
fs.renameSync(liveDb, manifest.pairs[0].rollbackPath);
|
||||
fs.renameSync(liveUploads, manifest.pairs[1].rollbackPath);
|
||||
|
||||
const report = svc.recoverTransactions(dataRoot.root);
|
||||
expect(report.transactionsScanned).toBe(1);
|
||||
expect(report.resolved).toBe(1);
|
||||
expect(fs.readFileSync(liveDb, 'utf8')).toBe('OLD-DB');
|
||||
expect(fs.readFileSync(path.join(liveUploads, 'keep.txt'), 'utf8')).toBe('OLD-UPLOAD');
|
||||
// Staged trees should not have ended up at the live paths.
|
||||
expect(fs.existsSync(stagedDb)).toBe(true);
|
||||
expect(fs.existsSync(stagedUploads)).toBe(true);
|
||||
});
|
||||
|
||||
it('recoverTransactions keeps the NEW generation when both pairs swapped but manifest is at verified', () => {
|
||||
const liveDb = path.join(dataRoot.root, 'db.sqlite');
|
||||
const liveUploads = path.join(dataRoot.root, 'uploads');
|
||||
fs.writeFileSync(liveDb, 'NEW-DB');
|
||||
fs.mkdirSync(liveUploads, { recursive: true });
|
||||
fs.writeFileSync(path.join(liveUploads, 'replaced.txt'), 'NEW-UPLOAD');
|
||||
|
||||
const stamp = `${Date.now()}-99999`;
|
||||
const oldDbPath = `${liveDb}.rollback-${stamp}`;
|
||||
const oldUpPath = `${liveUploads}.rollback-${stamp}`;
|
||||
fs.writeFileSync(oldDbPath, 'OLD-DB');
|
||||
fs.mkdirSync(oldUpPath, { recursive: true });
|
||||
fs.writeFileSync(path.join(oldUpPath, 'keep.txt'), 'OLD-UPLOAD');
|
||||
|
||||
const id = `${Date.now()}-x9`;
|
||||
const txDir = path.join(dataRoot.root, id);
|
||||
fs.mkdirSync(txDir, { recursive: true });
|
||||
const manifest: TransactionManifest = {
|
||||
schemaVersion: 1,
|
||||
transactionId: id,
|
||||
operation: 'restore-backup',
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
phase: 'verified',
|
||||
pairs: [
|
||||
{ livePath: liveDb, rollbackPath: oldDbPath, stagedPath: path.join(dataRoot.root, 'unused.db'), contributesToGeneration: true, sidecarSuffixes: [] },
|
||||
{ livePath: liveUploads, rollbackPath: oldUpPath, stagedPath: path.join(dataRoot.root, 'unused.up'), contributesToGeneration: true, sidecarSuffixes: [] },
|
||||
],
|
||||
};
|
||||
fs.writeFileSync(path.join(txDir, FilesystemTransactionService.MANIFEST_FILENAME), JSON.stringify(manifest));
|
||||
|
||||
const report = svc.recoverTransactions(dataRoot.root);
|
||||
expect(report.resolved).toBe(1);
|
||||
expect(fs.readFileSync(liveDb, 'utf8')).toBe('NEW-DB');
|
||||
expect(fs.readFileSync(path.join(liveUploads, 'replaced.txt'), 'utf8')).toBe('NEW-UPLOAD');
|
||||
// Rollback trees for the (now finalized) new generation must be gone.
|
||||
expect(fs.existsSync(oldDbPath)).toBe(false);
|
||||
expect(fs.existsSync(oldUpPath)).toBe(false);
|
||||
});
|
||||
|
||||
it('recoverTransactions fails closed when neither generation is fully present (the Job 919 hybrid state)', () => {
|
||||
// Reproduce the actual reported hybrid: the swap committed the new
|
||||
// uploads tree to the live path and wrote a rollback of the OLD
|
||||
// uploads tree, but the DB live→rollback snapshot happened while the
|
||||
// staged DB never reached the live path. From the resolver's point
|
||||
// of view the DB pair looks "old-complete" (livePath missing,
|
||||
// rollbackPath present), the uploads pair looks "new-complete"
|
||||
// (livePath present, rollbackPath present). Both pairs are not
|
||||
// consistent; recovery must fail closed without modifying either
|
||||
// generation.
|
||||
const dataDir = path.join(dataRoot.root, 'data');
|
||||
fs.mkdirSync(dataDir, { recursive: true });
|
||||
const liveDb = path.join(dataDir, 'db.sqlite');
|
||||
const liveUploads = path.join(dataDir, 'uploads');
|
||||
fs.mkdirSync(liveUploads, { recursive: true });
|
||||
fs.writeFileSync(path.join(liveUploads, 'replaced.txt'), 'NEW-ORPHAN'); // stale from a failed swap
|
||||
|
||||
const stamp = `${Date.now()}-hybrid`;
|
||||
const oldDbRollback = `${liveDb}.rollback-${stamp}`;
|
||||
const oldUpRollback = `${liveUploads}.rollback-${stamp}`;
|
||||
fs.writeFileSync(oldDbRollback, 'OLD-DB-SNAPSHOT');
|
||||
fs.mkdirSync(oldUpRollback, { recursive: true });
|
||||
fs.writeFileSync(path.join(oldUpRollback, 'keep.txt'), 'OLD-UPLOAD-SNAPSHOT');
|
||||
|
||||
const id = `${Date.now()}-hybrid`;
|
||||
const stagingRoot = path.join(dataRoot.root, '.system-staging');
|
||||
fs.mkdirSync(stagingRoot, { recursive: true });
|
||||
const txDir = path.join(stagingRoot, id);
|
||||
fs.mkdirSync(txDir, { recursive: true });
|
||||
const manifest: TransactionManifest = {
|
||||
schemaVersion: 1,
|
||||
transactionId: id,
|
||||
operation: 'restore-backup',
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
phase: 'verified',
|
||||
pairs: [
|
||||
{ livePath: liveDb, rollbackPath: oldDbRollback, stagedPath: path.join(dataDir, 'unused.db'), contributesToGeneration: true, sidecarSuffixes: [] },
|
||||
{ livePath: liveUploads, rollbackPath: oldUpRollback, stagedPath: path.join(dataDir, 'unused.up'), contributesToGeneration: true, sidecarSuffixes: [] },
|
||||
],
|
||||
};
|
||||
fs.writeFileSync(path.join(txDir, FilesystemTransactionService.MANIFEST_FILENAME), JSON.stringify(manifest));
|
||||
|
||||
expect(() => svc.recoverTransactions(stagingRoot)).toThrow(
|
||||
expect.objectContaining({ code: ERROR_CODES.SYSTEM_RESTORE_FAILED }),
|
||||
);
|
||||
|
||||
// Fail-closed must NOT delete or move anything in either generation.
|
||||
expect(fs.readFileSync(oldDbRollback, 'utf8')).toBe('OLD-DB-SNAPSHOT');
|
||||
expect(fs.readFileSync(path.join(liveUploads, 'replaced.txt'), 'utf8')).toBe('NEW-ORPHAN');
|
||||
expect(fs.readFileSync(path.join(oldUpRollback, 'keep.txt'), 'utf8')).toBe('OLD-UPLOAD-SNAPSHOT');
|
||||
// Manifest stays on disk for the operator.
|
||||
expect(fs.existsSync(txDir)).toBe(true);
|
||||
});
|
||||
|
||||
it('sweepUnownedArtifactsNearLivePaths cleans up stale rollback/restore-stage/wipe-stage directories', () => {
|
||||
const dataDir = path.join(dataRoot.root, 'data');
|
||||
fs.mkdirSync(dataDir, { recursive: true });
|
||||
const liveDb = path.join(dataDir, 'db.sqlite');
|
||||
fs.writeFileSync(liveDb, 'OLD');
|
||||
const liveUploads = path.join(dataDir, 'uploads');
|
||||
fs.mkdirSync(liveUploads, { recursive: true });
|
||||
const challenges = path.join(liveUploads, 'challenges');
|
||||
fs.mkdirSync(challenges, { recursive: true });
|
||||
|
||||
const now = Date.now();
|
||||
const oldStamp = now - 2 * 24 * 60 * 60_000; // 2 days old
|
||||
const freshStamp = now - 60_000; // 1 minute old
|
||||
|
||||
const rollbackUploads = `${liveUploads}.rollback-${oldStamp}-abc`;
|
||||
const rollbackDb = path.join(dataDir, `.db.sqlite.rollback-${oldStamp}-def`);
|
||||
const stageUploads = `${liveUploads}.restore-stage-${oldStamp}-ghi`;
|
||||
const wipeChallenges = path.join(challenges, `.challenges.wipe-stage-${oldStamp}-jkl`);
|
||||
fs.mkdirSync(rollbackUploads, { recursive: true });
|
||||
fs.writeFileSync(rollbackDb, 'orphan');
|
||||
fs.mkdirSync(stageUploads, { recursive: true });
|
||||
fs.mkdirSync(wipeChallenges, { recursive: true });
|
||||
|
||||
// A fresh rollback pattern must NOT be removed.
|
||||
const freshRollbackUploads = `${liveUploads}.rollback-${freshStamp}-mno`;
|
||||
fs.mkdirSync(freshRollbackUploads, { recursive: true });
|
||||
|
||||
// Backdate the old artifacts.
|
||||
const oldTime = new Date(oldStamp);
|
||||
fs.utimesSync(rollbackUploads, oldTime, oldTime);
|
||||
fs.utimesSync(rollbackDb, oldTime, oldTime);
|
||||
fs.utimesSync(stageUploads, oldTime, oldTime);
|
||||
fs.utimesSync(wipeChallenges, oldTime, oldTime);
|
||||
|
||||
const cleaned = svc.sweepUnownedArtifactsNearLivePaths([liveDb, liveUploads], {
|
||||
now,
|
||||
maxAgeMs: 60 * 60_000,
|
||||
stagingDir: dataRoot.root,
|
||||
});
|
||||
expect(cleaned).toEqual(expect.arrayContaining([rollbackUploads, rollbackDb, stageUploads, wipeChallenges]));
|
||||
expect(fs.existsSync(rollbackUploads)).toBe(false);
|
||||
expect(fs.existsSync(rollbackDb)).toBe(false);
|
||||
expect(fs.existsSync(stageUploads)).toBe(false);
|
||||
expect(fs.existsSync(wipeChallenges)).toBe(false);
|
||||
// Fresh rollback path remains.
|
||||
expect(fs.existsSync(freshRollbackUploads)).toBe(true);
|
||||
|
||||
// Cleanup helper for the remaining fresh artifact.
|
||||
try { fs.rmSync(freshRollbackUploads, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user