From 3fe2b90f511b8a9f107634b7ad6b61b442d3c60b Mon Sep 17 00:00:00 2001 From: m0rph3us1987 Date: Tue, 4 Aug 2026 18:00:55 +0000 Subject: [PATCH] AI Implementation feature(1084): Blog Repository Clone and Pull Workflow (#3) --- .env.example | 4 + docs/architecture/config-schema.md | 20 ++- docs/architecture/system-overview.md | 4 +- setup.sh | 12 ++ src/tenbackward/blog.py | 178 ++++++++++++++++++++-- src/tenbackward/config.py | 21 ++- src/tenbackward/main.py | 19 +++ tests/test_blog.py | 213 +++++++++++++++++++++++++++ tests/test_config.py | 29 ++++ 9 files changed, 477 insertions(+), 23 deletions(-) create mode 100644 tests/test_blog.py diff --git a/.env.example b/.env.example index 603cb12..0463f41 100644 --- a/.env.example +++ b/.env.example @@ -9,3 +9,7 @@ THROWNBACK_PREFIX=Throwback: MAX_RETRIES=3 RUN_AT=09:00 TZ=Europe/Berlin + +# Optional — Jekyll blog clone source. Defaults point at the Chaospott public repo. +# BLOG_REPO_URL=https://git.chaospott.de/Chaospott/site +# BLOG_DIR=blog diff --git a/docs/architecture/config-schema.md b/docs/architecture/config-schema.md index 4e9d6cf..512e73c 100644 --- a/docs/architecture/config-schema.md +++ b/docs/architecture/config-schema.md @@ -41,13 +41,15 @@ environment for the container to boot. `DEFAULTS` in `config.py` provides fallback strings that `apply_defaults` fills into the merged map **before** validation runs: -| Default key | Default value | -|-----------------------|-------------------| -| `VISIBILITY` | `public` | -| `THROWNBACK_PREFIX` | `Throwback:` | -| `MAX_RETRIES` | `3` | -| `TZ` | `Europe/Berlin` | -| `RUN_AT` | `09:00` | +| Default key | Default value | +|-----------------------|-------------------------------------------| +| `VISIBILITY` | `public` | +| `THROWNBACK_PREFIX` | `Throwback:` | +| `MAX_RETRIES` | `3` | +| `TZ` | `Europe/Berlin` | +| `RUN_AT` | `09:00` | +| `BLOG_REPO_URL` | `https://git.chaospott.de/Chaospott/site` | +| `BLOG_DIR` | `blog` (resolved relative to `data_dir`) | Because every required key carries a default, a freshly-initialised container can still boot to validate the scaffold, but the operator @@ -68,6 +70,8 @@ ideally `SITE_URL`) to make a real run. | `run_at` | `str` | `RUN_AT` | | `tz` | `str` | `TZ` | | `data_dir` | `Path` | Constant: `Path("/app/data")` | +| `blog_repo_url` | `str` | `BLOG_REPO_URL` | +| `blog_dir` | `Path` | `BLOG_DIR` resolved against `data_dir` | # Validation Behaviour @@ -80,6 +84,8 @@ problem joined with `"; "`. Validation covers: * `MASTODON_BASE_URL` and `SITE_URL` parse via `urllib.parse.urlparse` with an `http`/`https` scheme and a non-empty `netloc`. * `VISIBILITY` is one of `{"public", "unlisted"}` (`ALLOWED_VISIBILITY`). +* `BLOG_REPO_URL` (optional, has default) parses via `urllib.parse.urlparse` + with an `http`/`https` scheme and a non-empty `netloc` when present. * `TZ` resolves via `zoneinfo.ZoneInfo`. * `MAX_RETRIES` parses as a non-negative integer (rejects bools). diff --git a/docs/architecture/system-overview.md b/docs/architecture/system-overview.md index 8b2c651..10593bf 100644 --- a/docs/architecture/system-overview.md +++ b/docs/architecture/system-overview.md @@ -34,7 +34,7 @@ foreground of the `cron` process. Each scheduled invocation calls | `tenbackward.config` | Loads `.env` + process env, applies defaults, validates schema, produces a typed `Config` dataclass. | | `tenbackward.logging_setup` | JSON formatter (one log record per line), secret redaction, and `log_startup` / `log_error` / `log_run_summary` helpers. | | `tenbackward.state` | Read/write of `posted.json` (atomic temp-file replace). | -| `tenbackward.blog` | Stubbed blog clone/update helper (lands in a follow-up job). | +| `tenbackward.blog` | GitPython `ensure_repo()` — clones the configured blog repo on first run, fast-forwards it via `pull --ff-only` thereafter, with exponential-backoff retries on transient network errors. | | `/etc/cron.d/tenbackward`| Rendered cron file. One daily line that `cd /app` and runs `python -m tenbackward`. | # Communication & Wiring @@ -80,7 +80,7 @@ foreground of the `cron` process. Each scheduled invocation calls | `/repo/src/tenbackward/config.py` | Env merging, defaults, validation, `Config` dataclass, `ConfigError`. | | `/repo/src/tenbackward/logging_setup.py` | JSON formatter, secret redaction, structured event helpers. | | `/repo/src/tenbackward/state.py` | `posted.json` read/write with atomic temp-file replace. | -| `/repo/src/tenbackward/blog.py` | Stubbed blog clone helper (not yet wired into the pipeline). | +| `/repo/src/tenbackward/blog.py` | `ensure_repo()` — GitPython clone + fast-forward pull with `2^n` retry/backoff; raises `BlogRepoError` on local modifications or exhausted retries. | | `/repo/.env.example` | Canonical list of environment variables. | | `/repo/docker-compose.yml` | Service definition; binds env vars from the host `.env`. | | `/repo/Dockerfile` | Builds the runtime image. | diff --git a/setup.sh b/setup.sh index 5d15a76..2236275 100644 --- a/setup.sh +++ b/setup.sh @@ -2,6 +2,16 @@ set -euo pipefail export DEBIAN_FRONTEND=noninteractive +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +cd "${SCRIPT_DIR}" + +if [ -w /usr/local/bin ] && [ ! -e /usr/local/bin/build.sh ]; then + ln -s "${SCRIPT_DIR}/build.sh" /usr/local/bin/build.sh || true +fi +if [ -w /usr/local/bin ] && [ ! -e /usr/local/bin/setup.sh ]; then + ln -s "${SCRIPT_DIR}/setup.sh" /usr/local/bin/setup.sh || true +fi + apt-get update apt-get install -y --no-install-recommends \ bash \ @@ -22,6 +32,8 @@ fi if [ -n "${PIP_BIN}" ]; then ${PIP_BIN} install --no-cache-dir --upgrade pip >/dev/null 2>&1 || true + ${PIP_BIN} install --no-cache-dir --break-system-packages -r /repo/requirements.txt || \ + ${PIP_BIN} install --no-cache-dir -r /repo/requirements.txt ${PIP_BIN} install --no-cache-dir --break-system-packages -r /repo/requirements-dev.txt || \ ${PIP_BIN} install --no-cache-dir -r /repo/requirements-dev.txt fi diff --git a/src/tenbackward/blog.py b/src/tenbackward/blog.py index 2fc73bd..2efd02d 100644 --- a/src/tenbackward/blog.py +++ b/src/tenbackward/blog.py @@ -1,22 +1,174 @@ from __future__ import annotations +import logging +import os +import time from pathlib import Path -from urllib.parse import urlparse +from typing import Callable + +import git +from git import GitCommandError, InvalidGitRepositoryError, NoSuchPathError, Repo -def clone_or_update(site_url: str, dest: Path) -> Path: - """Clone or update the source blog git repository under `dest`. +DEFAULT_REPO_URL = "https://git.chaospott.de/Chaospott/site" +DEFAULT_BLOG_SUBDIR = "blog" +MAX_RETRIES_DEFAULT = 5 +BACKOFF_BASE_SECONDS = 2 +REMOTE_NAME = "origin" +BRANCH_NAME = "master" - Stubbed for the scaffold (Job 1082). Future jobs will implement this - using GitPython over HTTPS. - """ - parsed = urlparse(site_url) - if parsed.scheme not in {"http", "https"}: - raise ValueError(f"unsupported SITE_URL scheme: {parsed.scheme!r}") - raise NotImplementedError( - "blog clone/update lands in a follow-up job (relies on GitPython over HTTPS)" - ) +_LOCAL_MODIFICATION_MARKERS = ( + "Your local changes", + "would be overwritten", + "Please commit your changes", + "Please move or remove them", +) + + +class BlogRepoError(RuntimeError): + """Raised when the local blog working tree cannot be ensured.""" def blog_dir(data_dir: Path) -> Path: - return Path(data_dir) / "blog" + return Path(data_dir) / DEFAULT_BLOG_SUBDIR + + +def blog_repo_url() -> str: + return os.environ.get("BLOG_REPO_URL") or DEFAULT_REPO_URL + + +def _is_dir_empty(path: Path) -> bool: + if not path.exists(): + return True + try: + next(path.iterdir()) + except StopIteration: + return True + return False + + +def _is_local_modification_error(exc: GitCommandError) -> bool: + message = str(exc) + return any(marker in message for marker in _LOCAL_MODIFICATION_MARKERS) + + +def _fetch_and_pull( + repo: Repo, + *, + fetch: Callable[[], object], + pull: Callable[[], object], + sleep: Callable[[float], None], + max_retries: int, + logger: logging.Logger | None, +) -> None: + """Fetch + fast-forward pull with retry/backoff for transient network errors. + + Local-modification errors raise ``BlogRepoError`` immediately, without retry. + """ + attempts = max(1, max_retries + 1) + last_exc: GitCommandError | None = None + + for attempt in range(1, attempts + 1): + try: + fetch() + pull() + return + except GitCommandError as exc: + last_exc = exc + if _is_local_modification_error(exc): + if logger is not None: + logger.error( + "blog_local_modifications", + extra={"event": "blog_local_modifications", "error": str(exc)}, + ) + raise BlogRepoError( + "local modifications detected in blog working tree; aborting run" + ) from exc + if logger is not None: + logger.error( + "blog_fetch_error", + extra={ + "event": "blog_fetch_error", + "attempt": attempt, + "max_attempts": attempts, + "error": str(exc), + }, + ) + if attempt < attempts: + sleep(BACKOFF_BASE_SECONDS ** attempt) + + assert last_exc is not None + raise BlogRepoError( + f"blog fetch/pull failed after {attempts} attempt(s): {last_exc}" + ) from last_exc + + +def ensure_repo( + data_dir: Path, + *, + repo_url: str | None = None, + blog_path: Path | None = None, + max_retries: int | None = None, + logger: logging.Logger | None = None, + sleep: Callable[[float], None] = time.sleep, + fetch_impl: Callable[[Repo], Callable[[], object]] | None = None, + pull_impl: Callable[[Repo], Callable[[], object]] | None = None, +) -> Repo: + """Ensure a local working clone of the blog repository exists and is up to date. + + Returns the :class:`git.Repo` pointing at an up-to-date working tree. + Raises :class:`BlogRepoError` on irrecoverable failures. + """ + url = repo_url or blog_repo_url() + target = blog_path or blog_dir(data_dir) + retries = MAX_RETRIES_DEFAULT if max_retries is None else max_retries + + target.parent.mkdir(parents=True, exist_ok=True) + + needs_clone = _is_dir_empty(target) + + if needs_clone: + if logger is not None: + logger.info( + "blog_clone_start", + extra={"event": "blog_clone_start", "url": url, "path": str(target)}, + ) + cloned = Repo.clone_from(url, str(target)) + if logger is not None: + logger.info( + "blog_clone_complete", + extra={"event": "blog_clone_complete", "path": str(target)}, + ) + return cloned + + try: + repo = Repo(str(target)) + except (InvalidGitRepositoryError, NoSuchPathError) as exc: + raise BlogRepoError( + f"{target} exists but is not a valid git repository: {exc}" + ) from exc + + fetch = (fetch_impl or _default_fetch)(repo) + pull = (pull_impl or _default_pull)(repo) + + _fetch_and_pull( + repo, + fetch=fetch, + pull=pull, + sleep=sleep, + max_retries=retries, + logger=logger, + ) + return repo + + +def _default_fetch(repo: Repo) -> Callable[[], object]: + def _do() -> object: + return repo.remotes[REMOTE_NAME].fetch() + return _do + + +def _default_pull(repo: Repo) -> Callable[[], object]: + def _do() -> object: + return repo.git.pull("--ff-only") + return _do \ No newline at end of file diff --git a/src/tenbackward/config.py b/src/tenbackward/config.py index 831b39a..5c70892 100644 --- a/src/tenbackward/config.py +++ b/src/tenbackward/config.py @@ -22,7 +22,7 @@ REQUIRED_KEYS = ( "TZ", ) -OPTIONAL_KEYS: tuple[str, ...] = () +OPTIONAL_KEYS: tuple[str, ...] = ("BLOG_REPO_URL", "BLOG_DIR") DEFAULTS = { "VISIBILITY": "public", @@ -30,6 +30,8 @@ DEFAULTS = { "MAX_RETRIES": "3", "TZ": "Europe/Berlin", "RUN_AT": "09:00", + "BLOG_REPO_URL": "https://git.chaospott.de/Chaospott/site", + "BLOG_DIR": "blog", } @@ -52,6 +54,8 @@ class Config: run_at: str tz: str data_dir: Path = field(default_factory=lambda: Path("/app/data")) + blog_repo_url: str = "https://git.chaospott.de/Chaospott/site" + blog_dir: Path = field(default_factory=lambda: Path("/app/data/blog")) extra: dict = field(default_factory=dict) @property @@ -103,6 +107,13 @@ def load_config(dotenv_path: Optional[Path] = None) -> Config: data_dir = Path(os.environ.get("DATA_DIR", "/app/data")).resolve() + blog_dir_raw = Path(merged["BLOG_DIR"]).expanduser() + blog_dir = ( + blog_dir_raw + if blog_dir_raw.is_absolute() + else (data_dir / blog_dir_raw).resolve() + ) + extra = {k: v for k, v in merged.items() if k not in REQUIRED_KEYS + OPTIONAL_KEYS} return Config( @@ -116,6 +127,8 @@ def load_config(dotenv_path: Optional[Path] = None) -> Config: run_at=merged["RUN_AT"], tz=merged["TZ"], data_dir=data_dir, + blog_repo_url=merged["BLOG_REPO_URL"], + blog_dir=blog_dir, extra=extra, ) @@ -151,6 +164,12 @@ def validate_config(values: dict[str, str]) -> None: f"VISIBILITY={visibility!r} must be one of: {sorted(ALLOWED_VISIBILITY)}" ) + blog_repo_url = values.get("BLOG_REPO_URL", "") + if blog_repo_url and not _is_valid_url(blog_repo_url): + errors.append( + f"BLOG_REPO_URL={blog_repo_url!r} must be a valid http(s) URL" + ) + tz = values.get("TZ", "") if tz and not _is_valid_tz(tz): errors.append(f"TZ={tz!r} must be a valid IANA timezone") diff --git a/src/tenbackward/main.py b/src/tenbackward/main.py index 41b6ce9..aff4ab9 100644 --- a/src/tenbackward/main.py +++ b/src/tenbackward/main.py @@ -5,6 +5,7 @@ import time from typing import Iterable from . import __version__ +from .blog import BlogRepoError, ensure_repo from .config import Config, ConfigError, load_config from .logging_setup import ( configure_json_logging, @@ -30,6 +31,13 @@ def _run_once(config: Config) -> tuple[int, int, int, int, list[str]]: Returns ``(scanned, matched, posted, skipped, posted_ids)``. """ + ensure_repo( + config.data_dir, + repo_url=config.blog_repo_url, + blog_path=config.blog_dir, + max_retries=config.max_retries, + ) + already_posted = set(load_posted(config.data_dir).keys()) scanned = 0 @@ -96,6 +104,17 @@ def main() -> int: config.data_dir.mkdir(parents=True, exist_ok=True) + try: + ensure_repo( + config.data_dir, + repo_url=config.blog_repo_url, + blog_path=config.blog_dir, + max_retries=config.max_retries, + ) + except BlogRepoError as exc: + log_error("blog_repo_error", exc=exc) + return 1 + log_startup( version=__version__, site_url=config.site_url, diff --git a/tests/test_blog.py b/tests/test_blog.py new file mode 100644 index 0000000..133b475 --- /dev/null +++ b/tests/test_blog.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +import logging +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from git import GitCommandError, InvalidGitRepositoryError, Repo + +from tenbackward.blog import ( + BACKOFF_BASE_SECONDS, + BlogRepoError, + DEFAULT_BLOG_SUBDIR, + DEFAULT_REPO_URL, + blog_dir, + blog_repo_url, + ensure_repo, +) + + +def _git_err(stderr: str = "boom") -> GitCommandError: + return GitCommandError(["git"], stderr=stderr, status=1) + + +def test_blog_dir_default_subdir(tmp_path: Path) -> None: + assert blog_dir(tmp_path) == tmp_path / DEFAULT_BLOG_SUBDIR + + +def test_blog_repo_url_default(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("BLOG_REPO_URL", raising=False) + assert blog_repo_url() == DEFAULT_REPO_URL + + +def test_blog_repo_url_env_override(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("BLOG_REPO_URL", "https://example.com/repo.git") + assert blog_repo_url() == "https://example.com/repo.git" + + +def test_ensure_repo_clones_when_missing( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + target = tmp_path / "blog" + fake_repo = MagicMock(spec=Repo) + calls: list[tuple[str, str]] = [] + + def fake_clone_from(url: str, path: str) -> Repo: + calls.append((url, path)) + target.mkdir(parents=True) + (target / ".git").mkdir() + return fake_repo + + monkeypatch.setattr("tenbackward.blog.Repo.clone_from", fake_clone_from) + sleeps: list[float] = [] + result = ensure_repo( + tmp_path, + repo_url="https://example.com/repo.git", + blog_path=target, + max_retries=2, + sleep=sleeps.append, + ) + assert result is fake_repo + assert calls == [("https://example.com/repo.git", str(target))] + assert sleeps == [] + + +def test_ensure_repo_pulls_when_clone_exists( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + target = tmp_path / "blog" + target.mkdir() + (target / ".git").mkdir() + + fake_repo = MagicMock(spec=Repo) + monkeypatch.setattr("tenbackward.blog.Repo", MagicMock(return_value=fake_repo)) + + fetch_impl = MagicMock() + pull_impl = MagicMock() + sleeps: list[float] = [] + + result = ensure_repo( + tmp_path, + repo_url="https://example.com/repo.git", + blog_path=target, + max_retries=3, + sleep=sleeps.append, + fetch_impl=lambda r: fetch_impl, + pull_impl=lambda r: pull_impl, + ) + + assert result is fake_repo + fetch_impl.assert_called_once_with() + pull_impl.assert_called_once_with() + assert sleeps == [] + + +def test_ensure_repo_aborts_on_local_modifications( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + target = tmp_path / "blog" + target.mkdir() + (target / ".git").mkdir() + + fake_repo = MagicMock(spec=Repo) + monkeypatch.setattr("tenbackward.blog.Repo", MagicMock(return_value=fake_repo)) + + fetch_impl = MagicMock() + pull_impl = MagicMock( + side_effect=_git_err("Your local changes to 'foo' would be overwritten by merge") + ) + sleeps: list[float] = [] + + with caplog.at_level(logging.ERROR): + with pytest.raises(BlogRepoError, match="local modifications"): + ensure_repo( + tmp_path, + repo_url="https://example.com/repo.git", + blog_path=target, + max_retries=5, + sleep=sleeps.append, + fetch_impl=lambda r: fetch_impl, + pull_impl=lambda r: pull_impl, + ) + + assert fetch_impl.call_count == 1 + assert pull_impl.call_count == 1 + assert sleeps == [] + + +def test_ensure_repo_retries_transient_error_then_succeeds( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + target = tmp_path / "blog" + target.mkdir() + (target / ".git").mkdir() + + fake_repo = MagicMock(spec=Repo) + monkeypatch.setattr("tenbackward.blog.Repo", MagicMock(return_value=fake_repo)) + + fetch_impl = MagicMock(side_effect=[_git_err("Could not resolve host"), None]) + pull_impl = MagicMock() + sleeps: list[float] = [] + + ensure_repo( + tmp_path, + repo_url="https://example.com/repo.git", + blog_path=target, + max_retries=5, + sleep=sleeps.append, + fetch_impl=lambda r: fetch_impl, + pull_impl=lambda r: pull_impl, + ) + + assert fetch_impl.call_count == 2 + pull_impl.assert_called_once_with() + assert sleeps == [BACKOFF_BASE_SECONDS ** 1] + + +def test_ensure_repo_raises_after_exhausted_retries( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + target = tmp_path / "blog" + target.mkdir() + (target / ".git").mkdir() + + fake_repo = MagicMock(spec=Repo) + monkeypatch.setattr("tenbackward.blog.Repo", MagicMock(return_value=fake_repo)) + + fetch_impl = MagicMock(side_effect=_git_err("Could not resolve host")) + pull_impl = MagicMock() + sleeps: list[float] = [] + + with pytest.raises(BlogRepoError, match="blog fetch/pull failed"): + ensure_repo( + tmp_path, + repo_url="https://example.com/repo.git", + blog_path=target, + max_retries=2, + sleep=sleeps.append, + fetch_impl=lambda r: fetch_impl, + pull_impl=lambda r: pull_impl, + ) + + assert fetch_impl.call_count == 3 + pull_impl.assert_not_called() + assert sleeps == [BACKOFF_BASE_SECONDS ** 1, BACKOFF_BASE_SECONDS ** 2] + + +def test_ensure_repo_invalid_clone_dir_raises( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + target = tmp_path / "blog" + target.mkdir() + (target / "not-a-repo.txt").write_text("hi") + + def fake_repo(_path: str) -> Repo: + raise InvalidGitRepositoryError(f"{_path} is not a repo") + + monkeypatch.setattr("tenbackward.blog.Repo", fake_repo) + + with pytest.raises(BlogRepoError, match="not a valid git repository"): + ensure_repo( + tmp_path, + repo_url="https://example.com/repo.git", + blog_path=target, + max_retries=0, + ) \ No newline at end of file diff --git a/tests/test_config.py b/tests/test_config.py index 7addfc2..baa7d05 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -35,6 +35,35 @@ def test_load_config_succeeds_with_complete_env(env_setup) -> None: assert config.visibility == "public" +def test_load_config_blog_defaults(env_setup) -> None: + config = load_config() + assert config.blog_repo_url == "https://git.chaospott.de/Chaospott/site" + assert config.blog_dir.name == "blog" + + +def test_load_config_blog_repo_url_override(env_setup, monkeypatch) -> None: + monkeypatch.setenv("BLOG_REPO_URL", "https://example.com/repo.git") + config = load_config() + assert config.blog_repo_url == "https://example.com/repo.git" + + +def test_validate_config_rejects_bad_blog_repo_url(env_setup) -> None: + values = { + "MASTODON_BASE_URL": "https://mastodon.example", + "MASTODON_ACCESS_TOKEN": "x", + "VISIBILITY": "public", + "SITE_URL": "https://blog.example.com", + "HASHTAGS": "#throwback", + "THROWNBACK_PREFIX": "Throwback:", + "MAX_RETRIES": "3", + "RUN_AT": "09:00", + "TZ": "Europe/Berlin", + "BLOG_REPO_URL": "ftp://bad", + } + with pytest.raises(ConfigError, match="BLOG_REPO_URL"): + validate_config(values) + + def test_validate_config_lists_every_missing_key(env_setup, monkeypatch) -> None: monkeypatch.delenv("MASTODON_ACCESS_TOKEN") monkeypatch.delenv("SITE_URL")