AI Implementation feature(1086): Posted-Anniversary Dedup Store (#5)

src/tenbackward/logging_setup.py

@@ -139,6 +139,12 @@ def log_error(event: str, *, exc: BaseException | None = None, **fields: Any) ->
     logging.getLogger(_LOGGER_NAME).error(event, extra=payload)
 
 
+def log_warning(event: str, **fields: Any) -> None:
+    """Emit a single structured warning line, redacting any secret-shaped keys."""
+    payload = _redact_dict({"event": event, **fields})
+    logging.getLogger(_LOGGER_NAME).warning(event, extra=payload)
+
+
 def log_startup(**fields: Any) -> None:
     """Emit a single structured info line at startup."""
     payload = _redact_dict({"event": "startup", **fields})
@@ -152,4 +158,5 @@ __all__ = [
     "log_error",
     "log_run_summary",
     "log_startup",
+    "log_warning",
 ]

src/tenbackward/main.py

@@ -14,7 +14,7 @@ from .logging_setup import (
     log_startup,
 )
 from .matching import iter_anniversary_paths
-from .state import load_posted, save_posted
+from .state import PostedStore
 
 
 def _iter_candidates(config: Config) -> Iterable[str]:
@@ -38,7 +38,7 @@ def _run_once(config: Config) -> tuple[int, int, int, int, list[str]]:
         max_retries=config.max_retries,
     )
 
-    already_posted = set(load_posted(config.data_dir).keys())
+    store = PostedStore(config.data_dir)
 
     scanned = 0
     matched = 0
@@ -46,20 +46,17 @@ def _run_once(config: Config) -> tuple[int, int, int, int, list[str]]:
     skipped = 0
     posted_ids: list[str] = []
 
-    state = load_posted(config.data_dir)
-
     for candidate_id in _iter_candidates(config):
         scanned += 1
         matched += 1
-        if candidate_id in already_posted:
+        if store.is_posted(candidate_id):
             skipped += 1
             continue
         posted_ids.append(candidate_id)
-        state[candidate_id] = {"posted_at": _now_iso()}
         posted += 1
 
     if posted_ids:
-        save_posted(config.data_dir, state)
+        store.mark_posted_many(posted_ids)
 
     return scanned, matched, posted, skipped, posted_ids
 

src/tenbackward/state.py

@@ -1,34 +1,208 @@
 from __future__ import annotations
 
+import fcntl
 import json
+import os
 from pathlib import Path
-from typing import Any
+from typing import Iterable
+
+from .logging_setup import log_warning
 
 
-def posted_path(data_dir: Path) -> Path:
-    return Path(data_dir) / "posted.json"
+DEFAULT_DATA_DIR = Path("./data")
+_FILE_NAME = "posted.json"
+_EMPTY_DOCUMENT: dict[str, list[str]] = {"posted": []}
 
 
-def load_posted(data_dir: Path) -> dict[str, Any]:
-    """Return the posted-state map, creating an empty one if missing."""
-    path = posted_path(data_dir)
-    if not path.exists():
-        return {}
+class PostedStoreError(RuntimeError):
+    """Raised when the posted-state store cannot be used."""
+
+
+def posted_path(data_dir: Path | None = None) -> Path:
+    """Resolve the path of ``posted.json``.
+
+    When ``data_dir`` is ``None``, the directory is read from the
+    ``DATA_DIR`` environment variable, falling back to ``./data``
+    (the host bind-mount described in the README).
+    """
+    if data_dir is None:
+        data_dir = Path(os.environ.get("DATA_DIR", str(DEFAULT_DATA_DIR)))
+    return Path(data_dir) / _FILE_NAME
+
+
+def _coerce_list(value: object) -> list[str]:
+    """Filter ``value`` down to a list of strings, dropping anything else."""
+    if not isinstance(value, list):
+        return []
+    return [item for item in value if isinstance(item, str)]
+
+
+def _read_existing(path: Path) -> dict[str, list[str]] | None:
+    """Return the parsed JSON document at ``path`` or ``None`` on errors.
+
+    A missing file is **not** an error: returns ``None`` so the caller
+    can treat it as the empty document. Malformed JSON or unreadable
+    bytes return ``None`` and a warning is emitted.
+    """
     try:
         with path.open("r", encoding="utf-8") as fh:
             data = json.load(fh)
-    except json.JSONDecodeError:
-        return {}
+    except FileNotFoundError:
+        return None
+    except (json.JSONDecodeError, OSError) as exc:
+        log_warning("posted_store_corrupt", path=str(path), error=type(exc).__name__)
+        return None
     if not isinstance(data, dict):
-        return {}
+        return None
     return data
 
 
-def save_posted(data_dir: Path, state: dict[str, Any]) -> None:
-    """Persist the posted-state map atomically-ish."""
-    path = posted_path(data_dir)
+def _ensure_parent(path: Path) -> None:
     path.parent.mkdir(parents=True, exist_ok=True)
-    tmp = path.with_suffix(path.suffix + ".tmp")
-    with tmp.open("w", encoding="utf-8") as fh:
-        json.dump(state, fh, sort_keys=True, indent=2)
-    tmp.replace(path)
+
+
+def _write_document(path: Path, document: dict[str, list[str]]) -> None:
+    """Atomically write ``document`` to ``path`` via temp-file + rename."""
+    _ensure_parent(path)
+    pid = os.getpid()
+    tmp = path.with_name(f"{path.name}.tmp.{pid}")
+    try:
+        with tmp.open("w", encoding="utf-8") as fh:
+            json.dump(document, fh, sort_keys=False, indent=2)
+            fh.flush()
+            os.fsync(fh.fileno())
+        os.replace(tmp, path)
+    except Exception:
+        try:
+            tmp.unlink()
+        except FileNotFoundError:
+            pass
+        raise
+
+
+def _lock_path(path: Path):
+    """Return an open file descriptor on the lock file inside the data dir.
+
+    The lock file lives in the same directory as ``posted.json`` so the
+    flock is always on a single inode, regardless of whether the JSON
+    file currently exists. The descriptor is opened in append mode so
+    concurrent readers never truncate it.
+    """
+    _ensure_parent(path)
+    lock = path.with_name(f".{path.name}.lock")
+    fd = os.open(str(lock), os.O_CREAT | os.O_RDWR, 0o644)
+    return lock, fd
+
+
+class PostedStore:
+    """Self-contained dedup store persisted as ``posted.json``.
+
+    The on-disk shape is::
+
+        {"posted": ["2014/2014-08-04-foo.md", ...]}
+
+    All mutating operations acquire an exclusive :mod:`fcntl` flock on a
+    sibling lock file so concurrent container runs cannot corrupt the
+    JSON document. The store never raises on a missing file, an
+    unreadable file, or an already-present path — it always reports
+    success in a way that the orchestrator can act on.
+    """
+
+    def __init__(self, data_dir: Path | None = None) -> None:
+        self._path = posted_path(data_dir)
+
+    @property
+    def path(self) -> Path:
+        return self._path
+
+    def load(self) -> list[str]:
+        """Read the posted list, creating an empty file when missing."""
+        _ensure_parent(self._path)
+        document = _read_existing(self._path)
+        if document is None:
+            if not self._path.exists():
+                _write_document(self._path, dict(_EMPTY_DOCUMENT))
+            return []
+        return _coerce_list(document.get("posted"))
+
+    def is_posted(self, relative_path: str) -> bool:
+        """Return ``True`` when ``relative_path`` is already recorded."""
+        posted = self.load()
+        return relative_path in posted
+
+    def _with_lock(self, mutate):
+        lock_path, fd = _lock_path(self._path)
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX)
+            return mutate()
+        finally:
+            try:
+                fcntl.flock(fd, fcntl.LOCK_UN)
+            finally:
+                os.close(fd)
+            try:
+                lock_path.unlink()
+            except FileNotFoundError:
+                pass
+
+    def mark_posted(self, relative_path: str) -> bool:
+        """Append ``relative_path`` to the stored list. Idempotent.
+
+        Returns ``True`` when the path was newly added, ``False`` when
+        it was already present.
+        """
+        return self.mark_posted_many([relative_path]) != []
+
+    def mark_posted_many(self, relative_paths: Iterable[str]) -> list[str]:
+        """Append any new entries from ``relative_paths`` in one write.
+
+        Returns the list of paths that were newly added (possibly
+        empty when the store already contained every supplied path).
+        """
+
+        candidates = [p for p in relative_paths if isinstance(p, str) and p]
+        if not candidates:
+            return []
+
+        def _mutate() -> list[str]:
+            document = _read_existing(self._path) or dict(_EMPTY_DOCUMENT)
+            existing = _coerce_list(document.get("posted"))
+            added = [p for p in candidates if p not in existing]
+            if not added:
+                return []
+            existing.extend(added)
+            document["posted"] = existing
+            _write_document(self._path, document)
+            return added
+
+        return self._with_lock(_mutate)
+
+
+def load_posted(data_dir: Path | None = None) -> list[str]:
+    return PostedStore(data_dir).load()
+
+
+def is_posted(relative_path: str, data_dir: Path | None = None) -> bool:
+    return PostedStore(data_dir).is_posted(relative_path)
+
+
+def mark_posted(relative_path: str, data_dir: Path | None = None) -> bool:
+    return PostedStore(data_dir).mark_posted(relative_path)
+
+
+def mark_posted_many(
+    relative_paths: list[str], data_dir: Path | None = None
+) -> list[str]:
+    return PostedStore(data_dir).mark_posted_many(relative_paths)
+
+
+__all__ = [
+    "DEFAULT_DATA_DIR",
+    "PostedStore",
+    "PostedStoreError",
+    "is_posted",
+    "load_posted",
+    "mark_posted",
+    "mark_posted_many",
+    "posted_path",
+]