AI Implementation feature(1086): Posted-Anniversary Dedup Store (#5)
This commit was merged in pull request #5.
This commit is contained in:
+192
-18
@@ -1,34 +1,208 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Iterable
|
||||
|
||||
from .logging_setup import log_warning
|
||||
|
||||
|
||||
def posted_path(data_dir: Path) -> Path:
|
||||
return Path(data_dir) / "posted.json"
|
||||
DEFAULT_DATA_DIR = Path("./data")
|
||||
_FILE_NAME = "posted.json"
|
||||
_EMPTY_DOCUMENT: dict[str, list[str]] = {"posted": []}
|
||||
|
||||
|
||||
def load_posted(data_dir: Path) -> dict[str, Any]:
|
||||
"""Return the posted-state map, creating an empty one if missing."""
|
||||
path = posted_path(data_dir)
|
||||
if not path.exists():
|
||||
return {}
|
||||
class PostedStoreError(RuntimeError):
|
||||
"""Raised when the posted-state store cannot be used."""
|
||||
|
||||
|
||||
def posted_path(data_dir: Path | None = None) -> Path:
|
||||
"""Resolve the path of ``posted.json``.
|
||||
|
||||
When ``data_dir`` is ``None``, the directory is read from the
|
||||
``DATA_DIR`` environment variable, falling back to ``./data``
|
||||
(the host bind-mount described in the README).
|
||||
"""
|
||||
if data_dir is None:
|
||||
data_dir = Path(os.environ.get("DATA_DIR", str(DEFAULT_DATA_DIR)))
|
||||
return Path(data_dir) / _FILE_NAME
|
||||
|
||||
|
||||
def _coerce_list(value: object) -> list[str]:
|
||||
"""Filter ``value`` down to a list of strings, dropping anything else."""
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [item for item in value if isinstance(item, str)]
|
||||
|
||||
|
||||
def _read_existing(path: Path) -> dict[str, list[str]] | None:
|
||||
"""Return the parsed JSON document at ``path`` or ``None`` on errors.
|
||||
|
||||
A missing file is **not** an error: returns ``None`` so the caller
|
||||
can treat it as the empty document. Malformed JSON or unreadable
|
||||
bytes return ``None`` and a warning is emitted.
|
||||
"""
|
||||
try:
|
||||
with path.open("r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except (json.JSONDecodeError, OSError) as exc:
|
||||
log_warning("posted_store_corrupt", path=str(path), error=type(exc).__name__)
|
||||
return None
|
||||
if not isinstance(data, dict):
|
||||
return {}
|
||||
return None
|
||||
return data
|
||||
|
||||
|
||||
def save_posted(data_dir: Path, state: dict[str, Any]) -> None:
|
||||
"""Persist the posted-state map atomically-ish."""
|
||||
path = posted_path(data_dir)
|
||||
def _ensure_parent(path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
with tmp.open("w", encoding="utf-8") as fh:
|
||||
json.dump(state, fh, sort_keys=True, indent=2)
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def _write_document(path: Path, document: dict[str, list[str]]) -> None:
|
||||
"""Atomically write ``document`` to ``path`` via temp-file + rename."""
|
||||
_ensure_parent(path)
|
||||
pid = os.getpid()
|
||||
tmp = path.with_name(f"{path.name}.tmp.{pid}")
|
||||
try:
|
||||
with tmp.open("w", encoding="utf-8") as fh:
|
||||
json.dump(document, fh, sort_keys=False, indent=2)
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno())
|
||||
os.replace(tmp, path)
|
||||
except Exception:
|
||||
try:
|
||||
tmp.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def _lock_path(path: Path):
|
||||
"""Return an open file descriptor on the lock file inside the data dir.
|
||||
|
||||
The lock file lives in the same directory as ``posted.json`` so the
|
||||
flock is always on a single inode, regardless of whether the JSON
|
||||
file currently exists. The descriptor is opened in append mode so
|
||||
concurrent readers never truncate it.
|
||||
"""
|
||||
_ensure_parent(path)
|
||||
lock = path.with_name(f".{path.name}.lock")
|
||||
fd = os.open(str(lock), os.O_CREAT | os.O_RDWR, 0o644)
|
||||
return lock, fd
|
||||
|
||||
|
||||
class PostedStore:
|
||||
"""Self-contained dedup store persisted as ``posted.json``.
|
||||
|
||||
The on-disk shape is::
|
||||
|
||||
{"posted": ["2014/2014-08-04-foo.md", ...]}
|
||||
|
||||
All mutating operations acquire an exclusive :mod:`fcntl` flock on a
|
||||
sibling lock file so concurrent container runs cannot corrupt the
|
||||
JSON document. The store never raises on a missing file, an
|
||||
unreadable file, or an already-present path — it always reports
|
||||
success in a way that the orchestrator can act on.
|
||||
"""
|
||||
|
||||
def __init__(self, data_dir: Path | None = None) -> None:
|
||||
self._path = posted_path(data_dir)
|
||||
|
||||
@property
|
||||
def path(self) -> Path:
|
||||
return self._path
|
||||
|
||||
def load(self) -> list[str]:
|
||||
"""Read the posted list, creating an empty file when missing."""
|
||||
_ensure_parent(self._path)
|
||||
document = _read_existing(self._path)
|
||||
if document is None:
|
||||
if not self._path.exists():
|
||||
_write_document(self._path, dict(_EMPTY_DOCUMENT))
|
||||
return []
|
||||
return _coerce_list(document.get("posted"))
|
||||
|
||||
def is_posted(self, relative_path: str) -> bool:
|
||||
"""Return ``True`` when ``relative_path`` is already recorded."""
|
||||
posted = self.load()
|
||||
return relative_path in posted
|
||||
|
||||
def _with_lock(self, mutate):
|
||||
lock_path, fd = _lock_path(self._path)
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_EX)
|
||||
return mutate()
|
||||
finally:
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||
finally:
|
||||
os.close(fd)
|
||||
try:
|
||||
lock_path.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
def mark_posted(self, relative_path: str) -> bool:
|
||||
"""Append ``relative_path`` to the stored list. Idempotent.
|
||||
|
||||
Returns ``True`` when the path was newly added, ``False`` when
|
||||
it was already present.
|
||||
"""
|
||||
return self.mark_posted_many([relative_path]) != []
|
||||
|
||||
def mark_posted_many(self, relative_paths: Iterable[str]) -> list[str]:
|
||||
"""Append any new entries from ``relative_paths`` in one write.
|
||||
|
||||
Returns the list of paths that were newly added (possibly
|
||||
empty when the store already contained every supplied path).
|
||||
"""
|
||||
|
||||
candidates = [p for p in relative_paths if isinstance(p, str) and p]
|
||||
if not candidates:
|
||||
return []
|
||||
|
||||
def _mutate() -> list[str]:
|
||||
document = _read_existing(self._path) or dict(_EMPTY_DOCUMENT)
|
||||
existing = _coerce_list(document.get("posted"))
|
||||
added = [p for p in candidates if p not in existing]
|
||||
if not added:
|
||||
return []
|
||||
existing.extend(added)
|
||||
document["posted"] = existing
|
||||
_write_document(self._path, document)
|
||||
return added
|
||||
|
||||
return self._with_lock(_mutate)
|
||||
|
||||
|
||||
def load_posted(data_dir: Path | None = None) -> list[str]:
|
||||
return PostedStore(data_dir).load()
|
||||
|
||||
|
||||
def is_posted(relative_path: str, data_dir: Path | None = None) -> bool:
|
||||
return PostedStore(data_dir).is_posted(relative_path)
|
||||
|
||||
|
||||
def mark_posted(relative_path: str, data_dir: Path | None = None) -> bool:
|
||||
return PostedStore(data_dir).mark_posted(relative_path)
|
||||
|
||||
|
||||
def mark_posted_many(
|
||||
relative_paths: list[str], data_dir: Path | None = None
|
||||
) -> list[str]:
|
||||
return PostedStore(data_dir).mark_posted_many(relative_paths)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_DATA_DIR",
|
||||
"PostedStore",
|
||||
"PostedStoreError",
|
||||
"is_posted",
|
||||
"load_posted",
|
||||
"mark_posted",
|
||||
"mark_posted_many",
|
||||
"posted_path",
|
||||
]
|
||||
Reference in New Issue
Block a user