From e30fa780192aeeb1b3eb6602561a1723177420df Mon Sep 17 00:00:00 2001 From: m0rph3us1987 Date: Tue, 4 Aug 2026 18:57:46 +0000 Subject: [PATCH] AI Implementation feature(1087): Mastodon Post Composition and Publishing (#6) --- .env.example | 2 +- docs/architecture/config-schema.md | 2 +- docs/architecture/logging.md | 3 +- docs/architecture/mastodon-publishing.md | 174 +++++++++++++++++++++ docs/architecture/pipeline-runner.md | 43 +++-- docs/architecture/system-overview.md | 56 +++++-- docs/guides/daily-run.md | 11 +- docs/index.md | 7 +- docs/operations/environment-setup.md | 2 +- src/tenbackward/config.py | 2 +- src/tenbackward/main.py | 38 ++--- src/tenbackward/matching.py | 21 ++- src/tenbackward/publishing.py | 141 +++++++++++++++++ tests/test_config.py | 2 +- tests/test_matching.py | 3 + tests/test_publishing.py | 191 +++++++++++++++++++++++ tests/test_run_logging.py | 51 +++++- 17 files changed, 680 insertions(+), 69 deletions(-) create mode 100644 docs/architecture/mastodon-publishing.md create mode 100644 src/tenbackward/publishing.py create mode 100644 tests/test_publishing.py diff --git a/.env.example b/.env.example index 0463f41..90fd313 100644 --- a/.env.example +++ b/.env.example @@ -5,7 +5,7 @@ MASTODON_ACCESS_TOKEN=replace-me VISIBILITY=public SITE_URL=https://blog.example.com HASHTAGS=#throwback,#10backward -THROWNBACK_PREFIX=Throwback: +THROWNBACK_PREFIX=Heute vor 10 Jahren: MAX_RETRIES=3 RUN_AT=09:00 TZ=Europe/Berlin diff --git a/docs/architecture/config-schema.md b/docs/architecture/config-schema.md index 512e73c..591db8e 100644 --- a/docs/architecture/config-schema.md +++ b/docs/architecture/config-schema.md @@ -44,7 +44,7 @@ fills into the merged map **before** validation runs: | Default key | Default value | |-----------------------|-------------------------------------------| | `VISIBILITY` | `public` | -| `THROWNBACK_PREFIX` | `Throwback:` | +| `THROWNBACK_PREFIX` | `Heute vor 10 Jahren:` | | `MAX_RETRIES` | `3` | | `TZ` | `Europe/Berlin` | | `RUN_AT` | `09:00` | diff --git a/docs/architecture/logging.md b/docs/architecture/logging.md index 7d5b55e..03d93d9 100644 --- a/docs/architecture/logging.md +++ b/docs/architecture/logging.md @@ -95,7 +95,7 @@ A pass with no candidates emits **no line at all**, matching the ## Successful run with one new post ```json -{"ts":"2026-08-04T09:00:00+00:00","level":"INFO","logger":"tenbackward","message":"startup","event":"startup","version":"0.1.0","site_url":"https://blog.example.com","run_at":"09:00","tz":"Europe/Berlin","hashtags":"#throwback,#10backward","throwback_prefix":"Throwback:"} +{"ts":"2026-08-04T09:00:00+00:00","level":"INFO","logger":"tenbackward","message":"startup","event":"startup","version":"0.1.0","site_url":"https://blog.example.com","run_at":"09:00","tz":"Europe/Berlin","hashtags":"#throwback,#10backward","throwback_prefix":"Heute vor 10 Jahren:"} {"ts":"2026-08-04T09:00:01+00:00","level":"INFO","logger":"tenbackward","message":"run complete","event":"run_complete","scanned":1,"matched":1,"posted":1,"skipped":0,"posted_ids":["2025-08-04-post-slug"]} ``` @@ -123,3 +123,4 @@ formatter replaces it: * [System Architecture](/architecture/system-overview.md) * [Pipeline Runner](/architecture/pipeline-runner.md) * [Config Schema](/architecture/config-schema.md) +* [Mastodon Publishing](/architecture/mastodon-publishing.md) diff --git a/docs/architecture/mastodon-publishing.md b/docs/architecture/mastodon-publishing.md new file mode 100644 index 0000000..f7e20e8 --- /dev/null +++ b/docs/architecture/mastodon-publishing.md @@ -0,0 +1,174 @@ +--- +type: architecture +title: Mastodon Publishing +description: The single boundary between the pipeline runner and the Mastodon HTTP API — status composition, length validation, and the publish call. +tags: [publishing, mastodon, api, boundary] +timestamp: 2026-08-04T18:55:00Z +--- + +# Purpose + +`tenbackward.publishing` is the **single success boundary** between the +pipeline orchestrator and the Mastodon HTTP API. The runner calls +`publish_mastodon()` after dedupe and before state is persisted; if +the call raises, the retry loop re-attempts and `posted.json` is left +untouched. Composition and validation are pure functions that do not +touch the network so they are unit-tested without a server. + +# Public API + +| Symbol | Responsibility | +|---|---| +| `MASTODON_STATUS_LIMIT` | `500` — Mastodon's hard maximum status length. | +| `PublishError` | Raised on composition failure, length overflow, or any wrapped API exception. Subclass of `RuntimeError`. | +| `build_status_text(prefix, posts, hashtags)` | Pure composer. Returns the final status string. Raises `PublishError("empty_posts: ...")` when `posts` is empty. | +| `validate_status(status, limit=MASTODON_STATUS_LIMIT)` | Pure validator. Raises `PublishError("status_too_long: len=N limit=L")` when `len(status) > limit`. Never truncates. | +| `publish_mastodon(config, posts, *, client_factory=None)` | Composes, validates, and posts. Returns the composed status string. Wraps every third-party exception in `PublishError(f"publish_failed: {ExcType}: {exc}", ) from exc`. | +| `slugify_title(title)` | Re-exported from [matching](/architecture/anniversary-matching.md); used by callers that need to derive the same URL slug the matcher emits. | + +# Status Layout + +`build_status_text` produces exactly one Mastodon status for any number +of matching posts, in this shape: + +```text +{prefix} + +{title1} +{url1} + +{title2} +{url2} + +... + +{hashtags} +``` + +* The prefix comes from `Config.throwback_prefix` (default + `Heute vor 10 Jahren:`, overridable via `THROWNBACK_PREFIX`). +* Posts are sorted by `(date, path)` so output is deterministic + regardless of upstream order. +* `hashtags` is a comma-separated string; commas are collapsed to a + single space so each tag starts with `#` and no trailing comma + is emitted (empty tags are dropped). +* A trailing newline is always appended. + +# Composition Examples + +## Single post + +```text +Heute vor 10 Jahren: + +Mein erster Post +https://blog.example.com/2016/08/04/mein-erster-post/ + +#throwback #10backward +``` + +## Multiple posts on the same day + +When more than one Jekyll post matches the current day, they are +**merged into one status** — the runner does not issue separate API +calls per match. Ordering is `(date, path)`, so identical dates sort +by relative path: + +```text +Heute vor 10 Jahren: + +Post A +https://blog.example.com/2016/08/04/a/ + +Post B +https://blog.example.com/2016/08/04/b/ + +#throwback #10backward +``` + +## No posts + +Calling `build_status_text("...", [], "#x")` raises +`PublishError("empty_posts: cannot compose status without posts")` +before any API call. The runner only invokes the publisher when its +candidate list is non-empty, so this guard is a backstop for direct +callers and unit tests. + +# Length Validation + +Mastodon's API rejects statuses longer than 500 characters. The +spec requires **failing safely** — the bot must never silently +truncate content. `validate_status()` enforces this contract: the +full status is measured, and overflow raises +`PublishError("status_too_long: len=N limit=500")` *before* the API +call. The runner's retry loop treats this exactly like any other +publish failure. + +# Mastodon API Call + +`_post_status_via_mastodon_py()` constructs a `mastodon.Mastodon` +client with `(access_token=config.mastodon_access_token, +api_base_url=config.mastodon_base_url)` and calls +`status_post(status, visibility=config.visibility)`. + +The `client_factory` keyword argument on `publish_mastodon` lets +tests inject a fake client without monkey-patching. Production +callers leave it as `None`. + +# Failure Modes + +| Source | Exception surfaced to runner | Cause attached? | +|---|---|---| +| Empty `posts` argument | `PublishError("empty_posts: ...")` | No (no inner exc). | +| Composed status > 500 chars | `PublishError("status_too_long: ...")` | No. | +| Any other exception from inside the boundary | `PublishError("publish_failed: {ExcType}: {exc}")` | Yes, via `raise ... from exc`. | + +The runner catches every `Exception` in `_run_once`, so any of the +above becomes a `pipeline_error` log line and the retry budget +decides whether to give up. + +# Wiring + +``` +_run_once(config) # main.py + ├── ensure_repo(...) + ├── store = PostedStore(config.data_dir) + ├── candidates = list(_iter_candidates(config)) # find_anniversary_matches + ├── unposted = [m for m in candidates if not store.is_posted(m.path)] + ├── if unposted: + │ publish_mastodon(config, unposted) # <-- THIS boundary + │ store.mark_posted_many(posted_ids) # only after success +``` + +State is written **only after** `publish_mastodon` returns. A publish +failure leaves `posted.json` unchanged and lets the retry budget +re-attempt on the next iteration. + +# Configuration Surface + +| Env var | Consumed via | Effect | +|---|---|---| +| `MASTODON_BASE_URL` | `Config.mastodon_base_url` | Mastodon instance URL. | +| `MASTODON_ACCESS_TOKEN` | `Config.mastodon_access_token` | OAuth token passed to the `Mastodon` client. | +| `VISIBILITY` | `Config.visibility` | Passed as `visibility=` to `status_post`. | +| `THROWNBACK_PREFIX` | `Config.throwback_prefix` | First line of every published status. | +| `HASHTAGS` | `Config.hashtags` | Trailing hashtag line; commas become spaces. | + +See [Config Schema](/architecture/config-schema.md) for full validation +rules. + +# Key Files + +| Path | Responsibility | +|---|---| +| `/repo/src/tenbackward/publishing.py` | `MASTODON_STATUS_LIMIT`, `PublishError`, `build_status_text`, `validate_status`, `_post_status_via_mastodon_py`, `publish_mastodon`. | +| `/repo/src/tenbackward/matching.py` | `MatchedPost` dataclass and `slugify_title()` reused here. | +| `/repo/src/tenbackward/main.py` | Calls `publish_mastodon()` between dedupe and `mark_posted_many`. | +| `/repo/tests/test_publishing.py` | Composition, length validation, visibility passthrough, API failure wrapping, and the no-persistence-on-failure contract. | + +# Related + +* [Pipeline Runner](/architecture/pipeline-runner.md) +* [Anniversary Matching](/architecture/anniversary-matching.md) +* [Config Schema](/architecture/config-schema.md) +* [Daily Run Guide](/guides/daily-run.md) \ No newline at end of file diff --git a/docs/architecture/pipeline-runner.md b/docs/architecture/pipeline-runner.md index 41b1e1d..1aa782e 100644 --- a/docs/architecture/pipeline-runner.md +++ b/docs/architecture/pipeline-runner.md @@ -10,9 +10,8 @@ timestamp: 2026-08-04T17:51:00Z `tenbackward.main` is the entry point executed by cron. The pipeline first ensures that the configured blog repository is available and current, then scans Jekyll posts for today's ten-year anniversary, filters already-recorded -paths, and persists newly discovered identifiers. Job 1086 wires the -anniversary matcher into the runner; Mastodon publishing remains a future -step. +paths, publishes a single combined Mastodon status for the new matches, and +finally persists the freshly published identifiers. # Call Flow @@ -28,8 +27,9 @@ main.main() ├── result = _run_with_retry(config) │ ├── for attempt in 1 .. max_retries+1: │ │ try: return _run_once(config) - │ │ └── _run_once → ensure_repo → iter_anniversary_paths - │ │ → PostedStore deduplication → mark_posted_many + │ │ └── _run_once → ensure_repo → find_anniversary_matches + │ │ → PostedStore deduplication → publish_mastodon + │ │ → mark_posted_many # state only after publish OK │ ├── log_error("pipeline_error", exc=exc, attempt=attempt, max_attempts=attempts) │ ├── log_error("pipeline_failed", exc=last_exc, attempts=attempts) │ └── return None @@ -40,12 +40,16 @@ main.main() # Extension Point: `_iter_candidates` ```python -def _iter_candidates(config: Config) -> Iterable[str]: ... +def _iter_candidates(config: Config) -> Iterable[MatchedPost]: ... ``` -Job 1086 wires `_iter_candidates` to the anniversary matcher. The matcher -is now active; it discovers matching Jekyll posts after the repository is -synchronized. +`_iter_candidates` delegates to `find_anniversary_matches()` with +`config.blog_dir / "_posts" / "blog"` and `config.site_url`. The matcher +returns full `MatchedPost` values (relative path, title, date, canonical +URL); the runner uses `MatchedPost.path` as a stable deduplication ID and +passes the full objects to `publish_mastodon()` for status composition. +See [Anniversary Matching](/architecture/anniversary-matching.md) for the +file and front matter rules. ## Blog Synchronization @@ -56,14 +60,6 @@ fail immediately with `BlogRepoError`. The same synchronization occurs once in `main()` before `startup` and again inside `_run_once()` as the retryable pipeline boundary. - -`_iter_candidates(config)` delegates to `iter_anniversary_paths()` with -`config.blog_dir / "_posts" / "blog"` and `config.site_url`. The matcher -returns relative Markdown paths, such as -`2016/2016-08-04-example.md`, which are used as stable deduplication IDs. -See [Anniversary Matching](/architecture/anniversary-matching.md) for the -file and front matter rules. - # `_run_once` — Summary Counters `_run_once` returns a 5-tuple: @@ -86,6 +82,19 @@ The store persists the list under a `"posted"` key (`{"posted": ["2014/2014-08-04-foo.md", ...]}`) and serialises concurrent runs with an `fcntl.flock`. +# Publish Boundary + +`publish_mastodon()` is the single success boundary. The runner calls it +**after** dedupe and **before** `mark_posted_many`. If the call raises a +`PublishError`, the runner treats it like any other pipeline exception — +the retry loop in `_run_with_retry` re-attempts, and no identifier is +written to `posted.json`. Identical-day matches are published as **one** +combined status: prefix line + one `title\nurl` block per post + hashtags +line, sorted by `(date, path)` for deterministic ordering. The default +prefix is `Heute vor 10 Jahren:` (overridable via `THROWNBACK_PREFIX`). +Generated statuses are validated against `MASTODON_STATUS_LIMIT` (500 +characters); the runner fails safely (raises) rather than truncating. + # Retry Behaviour (`_run_with_retry`) * `attempts = max(1, config.max_retries + 1)` — at least one attempt diff --git a/docs/architecture/system-overview.md b/docs/architecture/system-overview.md index 79e9abd..b443191 100644 --- a/docs/architecture/system-overview.md +++ b/docs/architecture/system-overview.md @@ -3,7 +3,7 @@ type: architecture title: System Architecture description: Component map of 10Backward — how configuration, blog synchronization, anniversary matching, logging, the pipeline runner, state persistence, and container entrypoint are wired together. tags: [architecture, overview] -timestamp: 2026-08-04T17:51:00Z +timestamp: 2026-08-04T18:55:00Z --- # Overview @@ -12,8 +12,9 @@ timestamp: 2026-08-04T17:51:00Z containerised cron job. Job 1086 adds anniversary matching across Jekyll posts, backed by the blog clone/pull workflow from Job 1084 and the structured logging, retry-capable runner, and deduplication state from -Job 1083. The current pipeline identifies matching post paths and records -those identifiers; Mastodon publishing is still a future integration. +Job 1083. The current pipeline identifies matching posts, publishes a +single combined Mastodon status for the new matches, and records the +published identifiers in `posted.json`. The container boots, validates environment configuration, renders a `/etc/cron.d/tenbackward` entry that fires once per day at the @@ -25,7 +26,9 @@ foreground of the `cron` process. Each scheduled invocation calls 2. Loads and validates the `Config`. 3. Ensures the data directory exists. 4. Emits a `startup` log line. -5. Executes one pipeline pass wrapped in a retry loop. +5. Executes one pipeline pass wrapped in a retry loop: + `ensure_repo` → `find_anniversary_matches` → `PostedStore` dedupe → + `publish_mastodon` → `PostedStore.mark_posted_many`. 6. Emits a `run_complete` summary (skipped silently if there were no candidates). # Components @@ -39,6 +42,7 @@ foreground of the `cron` process. Each scheduled invocation calls | `tenbackward.state` | `PostedStore` — self-contained dedup store. `posted.json` holds `{"posted": [str, ...]}`; mutations are serialised by an `fcntl.flock` on a sibling lock file, writes go through a temp-file replace, and `load()` auto-creates an empty list when the file is missing. | | `tenbackward.blog` | GitPython `ensure_repo()` — clones the configured blog repo on first run, fast-forwards it via `pull --ff-only` thereafter, with exponential-backoff retries on transient network errors. | | `tenbackward.matching` | Walks `_posts/blog/**/*.md`, parses Jekyll front matter and filenames, and yields posts exactly ten years before the current date using Berlin-time and leap-day rules. | +| `tenbackward.publishing` | `publish_mastodon()` — the single success boundary between the runner and the Mastodon HTTP API. Composes a combined status for the day's matches, validates the 500-character limit, and posts via `mastodon.Mastodon.status_post`. Raises `PublishError` on composition, length, or API failure; the runner treats it like any other pipeline exception. | | `/etc/cron.d/tenbackward`| Rendered cron file. One daily line that `cd /app` and runs `python -m tenbackward`. | # Communication & Wiring @@ -55,16 +59,30 @@ foreground of the `cron` process. Each scheduled invocation calls | /etc/cron.d/ | -- daily fires --> | tenbackward.main.main | | tenbackward | | - configure logging | +-------------------+ | - retry-wrapped pass | - +-----------+-------------+ - | - v -+-------------------------+ - | _run_with_retry(...) | - | -> _run_once(...) | - | -> PostedStore. | - | {is_posted, | - | mark_posted_many}| - +-------------------------+ + +-----------+-------------+ + | + v + +-------------------------+ + | _run_with_retry(...) | + | -> _run_once(...) | + | -> ensure_repo(...) | + | -> matching | + | find_anniversary_| + | matches | + | -> PostedStore | + | {is_posted, | + | mark_posted_many}| + | -> publishing | + | publish_mastodon | + | -> PostedStore | + | mark_posted_many | + +-------------------------+ + | + v + +-------------------------+ + | Mastodon HTTP API | + | status_post(...) | + +-------------------------+ ``` * **Env → Config.** `load_config()` merges a `.env` file (when @@ -73,10 +91,14 @@ foreground of the `cron` process. Each scheduled invocation calls `Config` dataclass. * **Cron → main.** Each scheduled tick re-runs `python -m tenbackward`, so every run is a fresh interpreter invocation. +* **Matching → publishing.** `find_anniversary_matches` yields + `MatchedPost` objects; the runner dedupes against `PostedStore`, then + passes the unposted objects to `publish_mastodon`. See + [Mastodon Publishing](/architecture/mastodon-publishing.md). * **Pipeline state.** `_run_once` reads `posted.json` via `PostedStore.is_posted()` for dedupe, then writes it back via - `PostedStore.mark_posted_many()` only when at least one new post - was recorded. The store serialises concurrent runs with an + `PostedStore.mark_posted_many()` **only after** `publish_mastodon` + returns. The store serialises concurrent runs with an `fcntl.flock` on a sibling lock file and writes via temp-file rename. @@ -91,6 +113,7 @@ foreground of the `cron` process. Each scheduled invocation calls | `/repo/src/tenbackward/state.py` | `PostedStore` + module helpers (`load_posted`, `is_posted`, `mark_posted`, `mark_posted_many`); list-shaped JSON, `fcntl.flock`, atomic temp-file replace, env-var data dir. | | `/repo/src/tenbackward/blog.py` | `ensure_repo()` — GitPython clone + fast-forward pull with `2^n` retry/backoff; raises `BlogRepoError` on local modifications or exhausted retries. | | `/repo/src/tenbackward/matching.py` | Jekyll post discovery, front matter parsing, anniversary and leap-day matching, and canonical URL construction. | +| `/repo/src/tenbackward/publishing.py` | `publish_mastodon()` — composes a combined Mastodon status, validates the 500-character limit, posts via `mastodon.Mastodon.status_post`. Pure helpers `build_status_text` and `validate_status` keep network code out of tests. | | `/repo/src/tenbackward/__main__.py` | Module entrypoint that invokes `tenbackward.main.main()`. | | `/repo/.env.example` | Canonical list of environment variables. | | `/repo/docker-compose.yml` | Service definition; binds env vars from the host `.env`. | @@ -105,5 +128,6 @@ foreground of the `cron` process. Each scheduled invocation calls * [Logging & Run Summary](/architecture/logging.md) * [Pipeline Runner](/architecture/pipeline-runner.md) * [Anniversary Matching](/architecture/anniversary-matching.md) +* [Mastodon Publishing](/architecture/mastodon-publishing.md) * [Environment Variable Setup](/operations/environment-setup.md) * [Cron Lifecycle](/operations/cron-lifecycle.md) diff --git a/docs/guides/daily-run.md b/docs/guides/daily-run.md index 2d5f129..d170cd6 100644 --- a/docs/guides/daily-run.md +++ b/docs/guides/daily-run.md @@ -29,9 +29,12 @@ scheduled tick. Before scanning, the process ensures the configured blog repository exists under `/app/data/blog` and is fast-forwarded from `BLOG_REPO_URL`. It then walks `_posts/blog/**/*.md`, finds posts whose date is exactly ten years -before today, skips IDs already present in `posted.json`, and records new -relative paths. The current implementation records matching IDs only; -Mastodon publication is not yet wired. +before today, skips IDs already present in `posted.json`, composes a +single German-language Mastodon status (default prefix +`Heute vor 10 Jahren:`) listing each new match's title and canonical URL +followed by the configured hashtags, and — only after the Mastodon API +call succeeds — records the relative paths of the published posts in +`posted.json`. # What You Should See @@ -42,7 +45,7 @@ The bot uses a **JSON-per-line** logger that writes to should show output similar to: ```json -{"ts": "2026-08-04T09:00:00+00:00", "level": "INFO", "logger": "tenbackward", "message": "startup", "event": "startup", "version": "0.1.0", "site_url": "https://blog.example.com", "run_at": "09:00", "tz": "Europe/Berlin", "hashtags": "#throwback,#10backward", "throwback_prefix": "Throwback:"} +{"ts": "2026-08-04T09:00:00+00:00", "level": "INFO", "logger": "tenbackward", "message": "startup", "event": "startup", "version": "0.1.0", "site_url": "https://blog.example.com", "run_at": "09:00", "tz": "Europe/Berlin", "hashtags": "#throwback,#10backward", "throwback_prefix": "Heute vor 10 Jahren:"} ``` On the current scaffold (`_iter_candidates` is intentionally empty), diff --git a/docs/index.md b/docs/index.md index ec49d05..a87fcb5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,15 +3,16 @@ okf_version: "0.1" --- # Architecture -* [System Architecture](/architecture/system-overview.md) — Component map of 10Backward: entrypoint, config, logging, runner, state, and cron wiring. +* [System Architecture](/architecture/system-overview.md) — Component map of 10Backward: entrypoint, config, logging, runner, state, publishing boundary, and cron wiring. * [Config Schema](/architecture/config-schema.md) — Required/optional env vars, validation rules, and the typed Config dataclass. * [Logging & Run Summary](/architecture/logging.md) — JSON formatter, secret redaction, and structured event helpers. -* [Pipeline Runner](/architecture/pipeline-runner.md) — How a cron tick synchronizes the blog, matches anniversaries, applies deduplication, retries failures, and emits run events. +* [Pipeline Runner](/architecture/pipeline-runner.md) — How a cron tick synchronizes the blog, matches anniversaries, applies deduplication, publishes to Mastodon, retries failures, and emits run events. * [Anniversary Matching](/architecture/anniversary-matching.md) — Jekyll filename/front matter rules, ten-year date matching, leap-day handling, and candidate identifiers. +* [Mastodon Publishing](/architecture/mastodon-publishing.md) — The single success boundary: status composition, 500-character validation, and the Mastodon API call. # Operations * [Environment Variable Setup](/operations/environment-setup.md) — Required env vars, renamed keys, and how to populate `.env`. * [Cron Lifecycle](/operations/cron-lifecycle.md) — How `entrypoint.sh` renders `/etc/cron.d/tenbackward` and hands off to `cron -f`. # User Guides -* [Daily Run Guide](/guides/daily-run.md) — Tester/operator walkthrough of repository sync, anniversary matching, deduplication, and expected log/output behaviour. +* [Daily Run Guide](/guides/daily-run.md) — Tester/operator walkthrough of repository sync, anniversary matching, deduplication, Mastodon publishing, and expected log/output behaviour. diff --git a/docs/operations/environment-setup.md b/docs/operations/environment-setup.md index 7d41067..1f1385f 100644 --- a/docs/operations/environment-setup.md +++ b/docs/operations/environment-setup.md @@ -21,7 +21,7 @@ full set before the container will boot. | `VISIBILITY` | `public` | Post visibility (`public` or `unlisted`). | | `SITE_URL` | `https://blog.example.com` | Source blog URL (used by the future clone step). | | `HASHTAGS` | `#throwback,#10backward` | Hashtags appended to every throwback post. | -| `THROWNBACK_PREFIX` | `Throwback:` | Prefix prepended to every post. | +| `THROWNBACK_PREFIX` | `Heute vor 10 Jahren:` | Prefix prepended to every Mastodon status. | | `MAX_RETRIES` | `3` | Non-negative retry count for the pipeline. | | `RUN_AT` | `09:00` | Daily fire time (HH:MM, 24-hour). | | `TZ` | `Europe/Berlin` | IANA timezone for cron + container clock. | diff --git a/src/tenbackward/config.py b/src/tenbackward/config.py index 5c70892..390448f 100644 --- a/src/tenbackward/config.py +++ b/src/tenbackward/config.py @@ -26,7 +26,7 @@ OPTIONAL_KEYS: tuple[str, ...] = ("BLOG_REPO_URL", "BLOG_DIR") DEFAULTS = { "VISIBILITY": "public", - "THROWNBACK_PREFIX": "Throwback:", + "THROWNBACK_PREFIX": "Heute vor 10 Jahren:", "MAX_RETRIES": "3", "TZ": "Europe/Berlin", "RUN_AT": "09:00", diff --git a/src/tenbackward/main.py b/src/tenbackward/main.py index 16fa1f1..342e237 100644 --- a/src/tenbackward/main.py +++ b/src/tenbackward/main.py @@ -13,17 +13,17 @@ from .logging_setup import ( log_run_summary, log_startup, ) -from .matching import iter_anniversary_paths +from .matching import MatchedPost, find_anniversary_matches +from .publishing import publish_mastodon from .state import PostedStore -def _iter_candidates(config: Config) -> Iterable[str]: - """Yield candidate post identifiers (the relative path under - ``_posts/blog/``) for posts whose anniversary is exactly 10 years - before today. +def _iter_candidates(config: Config) -> Iterable[MatchedPost]: + """Yield candidate :class:`MatchedPost` objects whose anniversary is + exactly 10 years before today. """ post_root = config.blog_dir / "_posts" / "blog" - return iter_anniversary_paths(post_root, config.site_url) + return find_anniversary_matches(post_root, config.site_url) def _run_once(config: Config) -> tuple[int, int, int, int, list[str]]: @@ -40,24 +40,24 @@ def _run_once(config: Config) -> tuple[int, int, int, int, list[str]]: store = PostedStore(config.data_dir) - scanned = 0 - matched = 0 - posted = 0 - skipped = 0 - posted_ids: list[str] = [] + candidates = list(_iter_candidates(config)) + scanned = len(candidates) + matched = len(candidates) - for candidate_id in _iter_candidates(config): - scanned += 1 - matched += 1 - if store.is_posted(candidate_id): - skipped += 1 + unposted: list[MatchedPost] = [] + for match in candidates: + if store.is_posted(match.path): continue - posted_ids.append(candidate_id) - posted += 1 + unposted.append(match) - if posted_ids: + skipped = matched - len(unposted) + posted_ids = [m.path for m in unposted] + + if unposted: + publish_mastodon(config, unposted) store.mark_posted_many(posted_ids) + posted = len(unposted) return scanned, matched, posted, skipped, posted_ids diff --git a/src/tenbackward/matching.py b/src/tenbackward/matching.py index 01b9290..837870e 100644 --- a/src/tenbackward/matching.py +++ b/src/tenbackward/matching.py @@ -3,6 +3,7 @@ from __future__ import annotations import calendar import logging import re +import unicodedata from dataclasses import dataclass from datetime import date, datetime from pathlib import Path @@ -15,6 +16,8 @@ _LOGGER = logging.getLogger("tenbackward.matching") _FILENAME_PATTERN = re.compile(r"^(\d{4})-(\d{2})-(\d{2})-(.+)$") +_NON_ALNUM_RE = re.compile(r"[^a-z0-9]+") + _DEFAULT_TZ = "Europe/Berlin" @@ -26,6 +29,19 @@ class MatchedPost: url: str +def slugify_title(title: str) -> str: + """Return a URL-safe slug derived from ``title``. + + Used by the matcher to derive the slug component of + :attr:`MatchedPost.url` from the resolved post title. + """ + normalized = unicodedata.normalize("NFKD", title) + ascii_only = normalized.encode("ascii", "ignore").decode("ascii") + lowered = ascii_only.lower() + dashed = _NON_ALNUM_RE.sub("-", lowered) + return dashed.strip("-") + + def _today_in_berlin(tz_name: str = _DEFAULT_TZ) -> date: return datetime.now(ZoneInfo(tz_name)).date() @@ -152,7 +168,7 @@ def _process_file( path=rel, title=title, date=post_date, - url=_build_url(site_url, post_date, fn_slug), + url=_build_url(site_url, post_date, slugify_title(title)), ) @@ -206,4 +222,5 @@ __all__ = [ "MatchedPost", "find_anniversary_matches", "iter_anniversary_paths", -] \ No newline at end of file + "slugify_title", +] \ No newline at end of file diff --git a/src/tenbackward/publishing.py b/src/tenbackward/publishing.py new file mode 100644 index 0000000..5533741 --- /dev/null +++ b/src/tenbackward/publishing.py @@ -0,0 +1,141 @@ +"""Mastodon publishing helpers and the API success boundary. + +The :func:`publish_mastodon` function is the single boundary between the +pipeline orchestrator and the Mastodon HTTP API. Status composition and +length validation are pure and isolated from network access so they can +be tested without a server. +""" + +from __future__ import annotations + +from typing import Callable + +from mastodon import Mastodon + +from .config import Config +from .matching import MatchedPost, slugify_title + + +MASTODON_STATUS_LIMIT = 500 + + +class PublishError(RuntimeError): + """Raised when status composition, validation, or the Mastodon API + call fails. The pipeline orchestrator treats this like any other + pipeline error and lets the retry budget decide whether to give up. + """ + + +def _normalize_hashtags(hashtags: str) -> str: + parts = [token.strip() for token in (hashtags or "").split(",")] + parts = [token for token in parts if token] + return " ".join(parts) + + +def build_status_text( + prefix: str, + posts: list[MatchedPost], + hashtags: str, +) -> str: + """Compose a single Mastodon status string for ``posts``. + + The format is:: + + {prefix} + + {title1} + {url1} + + {title2} + {url2} + + ... + + {hashtags} + + Posts are sorted by ``(date, path)`` for deterministic output + regardless of the upstream ordering. + """ + if not posts: + raise PublishError("empty_posts: cannot compose status without posts") + + ordered = sorted(posts, key=lambda m: (m.date, m.path)) + blocks: list[str] = [] + for match in ordered: + blocks.append(f"{match.title}\n{match.url}") + body = "\n\n".join(blocks) + + lines: list[str] = [prefix.strip(), body] + tag_line = _normalize_hashtags(hashtags) + if tag_line: + lines.append(tag_line) + return "\n\n".join(lines) + "\n" + + +def validate_status(status: str, limit: int = MASTODON_STATUS_LIMIT) -> None: + """Raise :class:`PublishError` when ``status`` exceeds ``limit``. + + Never truncates: the spec requires failing safely rather than + shortening content. + """ + if len(status) > limit: + raise PublishError( + f"status_too_long: len={len(status)} limit={limit}" + ) + + +def _post_status_via_mastodon_py( + status: str, + *, + base_url: str, + access_token: str, + visibility: str, + client_factory: Callable[..., Mastodon] | None = None, +) -> None: + factory = client_factory if client_factory is not None else Mastodon + client = factory(access_token=access_token, api_base_url=base_url) + client.status_post(status, visibility=visibility) + + +def publish_mastodon( + config: Config, + posts: list[MatchedPost], + *, + client_factory: Callable[..., Mastodon] | None = None, +) -> str: + """Compose, validate, and publish ``posts`` to Mastodon. + + Returns the composed status text on success. Raises + :class:`PublishError` on any failure (composition, length, or API). + The original exception is chained via ``raise ... from exc`` so the + caller can inspect the underlying cause. + """ + try: + status = build_status_text( + config.throwback_prefix, posts, config.hashtags + ) + validate_status(status) + _post_status_via_mastodon_py( + status, + base_url=config.mastodon_base_url, + access_token=config.mastodon_access_token, + visibility=config.visibility, + client_factory=client_factory, + ) + except PublishError: + raise + except Exception as exc: # noqa: BLE001 — third-party boundary + raise PublishError( + f"publish_failed: {type(exc).__name__}: {exc}" + ) from exc + return status + + +__all__ = [ + "MASTODON_STATUS_LIMIT", + "PublishError", + "build_status_text", + "publish_mastodon", + "slugify_title", + "validate_status", +] diff --git a/tests/test_config.py b/tests/test_config.py index baa7d05..59ab063 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -142,7 +142,7 @@ def test_load_config_applies_optional_defaults(env_setup, monkeypatch) -> None: monkeypatch.delenv("TZ", raising=False) config = load_config() - assert config.throwback_prefix == "Throwback:" + assert config.throwback_prefix == "Heute vor 10 Jahren:" assert config.max_retries == 3 assert config.tz == "Europe/Berlin" assert config.visibility == "public" diff --git a/tests/test_matching.py b/tests/test_matching.py index 525f58c..762626b 100644 --- a/tests/test_matching.py +++ b/tests/test_matching.py @@ -11,6 +11,7 @@ from tenbackward.matching import ( MatchedPost, find_anniversary_matches, iter_anniversary_paths, + slugify_title, ) @@ -64,6 +65,7 @@ def test_match_uses_frontmatter_date_and_title(tmp_path: Path) -> None: assert match.title == "Foo" assert match.date == date(2014, 8, 4) assert match.url == "https://chaospott.de/2014/08/04/foo/" + assert match.url.endswith(slugify_title(match.title) + "/") def test_match_falls_back_to_filename_when_frontmatter_missing(tmp_path: Path) -> None: @@ -87,6 +89,7 @@ def test_match_falls_back_to_filename_when_frontmatter_missing(tmp_path: Path) - assert match.title == "no-frontmatter" assert match.date == date(2015, 3, 10) assert match.url == "https://chaospott.de/2015/03/10/no-frontmatter/" + assert match.url.endswith(slugify_title(match.title) + "/") def test_unpublished_post_is_skipped(tmp_path: Path) -> None: diff --git a/tests/test_publishing.py b/tests/test_publishing.py new file mode 100644 index 0000000..ac1ea81 --- /dev/null +++ b/tests/test_publishing.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +import os +from datetime import date +from pathlib import Path +from typing import Callable + +import pytest + +from tenbackward.config import load_config +from tenbackward.matching import MatchedPost +from tenbackward.publishing import ( + MASTODON_STATUS_LIMIT, + PublishError, + build_status_text, + publish_mastodon, + slugify_title, + validate_status, +) +from tenbackward.state import PostedStore + + +def _make_match(path: str, title: str, day: int = 4) -> MatchedPost: + return MatchedPost( + path=path, + title=title, + date=date(2016, 8, day), + url=f"https://blog.example.com/2016/08/{day:02d}/{path.split('-', 3)[-1].removesuffix('.md')}/", + ) + + +class _FakeMastodon: + """Captures ``status_post`` calls and can be configured to raise.""" + + def __init__(self, *, raise_on_post: Exception | None = None) -> None: + self.raise_on_post = raise_on_post + self.calls: list[tuple[str, dict]] = [] + + def __call__(self, *, access_token: str, api_base_url: str) -> "_FakeMastodon": + self.access_token = access_token + self.api_base_url = api_base_url + return self + + def status_post(self, status: str, **kwargs) -> None: + self.calls.append((status, kwargs)) + if self.raise_on_post is not None: + raise self.raise_on_post + + +@pytest.fixture() +def full_config(env_setup, tmp_path: Path): + os.environ["DATA_DIR"] = str(tmp_path) + return load_config() + + +def test_slugify_title_basic_ascii() -> None: + assert slugify_title("Hello, World!") == "hello-world" + + +def test_slugify_title_collapses_repeats_and_trims() -> None: + assert slugify_title("!!!Foo---Bar???") == "foo-bar" + assert slugify_title("---trim---me---") == "trim-me" + + +def test_build_status_text_single_post_layout() -> None: + match = _make_match("2016/2016-08-04-foo.md", "Foo") + status = build_status_text( + "Heute vor 10 Jahren:", [match], "#throwback,#10backward" + ) + assert status == ( + "Heute vor 10 Jahren:\n\n" + "Foo\nhttps://blog.example.com/2016/08/04/foo/\n\n" + "#throwback #10backward\n" + ) + + +def test_build_status_text_multiple_posts_combined() -> None: + match_a = _make_match("2016/2016-08-04-a.md", "Alpha") + match_b = _make_match("2016/2016-08-04-b.md", "Bravo") + status = build_status_text("P:", [match_a, match_b], "#x") + assert "Alpha" in status + assert "Bravo" in status + assert match_a.url in status + assert match_b.url in status + assert status.count("Alpha") == 1 + assert status.count("Bravo") == 1 + assert status.count(match_a.url) == 1 + assert status.count(match_b.url) == 1 + + +def test_build_status_text_empty_posts_raises() -> None: + with pytest.raises(PublishError): + build_status_text("P:", [], "#x") + + +def test_build_status_text_preserves_deterministic_order() -> None: + match_a = _make_match("2016/2016-08-05-a.md", "Alpha", day=5) + match_b = _make_match("2016/2016-08-04-b.md", "Bravo", day=4) + forward = build_status_text("P:", [match_a, match_b], "#x") + reverse = build_status_text("P:", [match_b, match_a], "#x") + assert forward == reverse + assert forward.index("Bravo") < forward.index("Alpha") + + +def test_validate_status_within_limit() -> None: + validate_status("a" * 480) + + +def test_validate_status_over_limit_raises_and_does_not_truncate() -> None: + long = "a" * (MASTODON_STATUS_LIMIT + 1) + with pytest.raises(PublishError): + validate_status(long) + assert len(long) == MASTODON_STATUS_LIMIT + 1 + + +def test_publish_mastodon_success_invokes_client_with_composed_status( + full_config, +) -> None: + match = _make_match("2016/2016-08-04-foo.md", "Foo") + fake = _FakeMastodon() + + result = publish_mastodon(full_config, [match], client_factory=fake) + + assert result.endswith("\n") + assert fake.calls, "status_post must be invoked" + posted_status, posted_kwargs = fake.calls[0] + assert posted_status == result + assert posted_kwargs == {"visibility": "public"} + assert fake.api_base_url == "https://mastodon.example" + assert fake.access_token == "test-token" + + +def test_publish_mastodon_api_failure_raises_and_does_not_persist( + full_config, tmp_path: Path +) -> None: + match = _make_match("2016/2016-08-04-foo.md", "Foo") + fake = _FakeMastodon(raise_on_post=RuntimeError("boom")) + + with pytest.raises(PublishError) as exc_info: + publish_mastodon(full_config, [match], client_factory=fake) + + assert "publish_failed" in str(exc_info.value) + assert isinstance(exc_info.value.__cause__, RuntimeError) + + store = PostedStore(tmp_path) + assert not store.is_posted(match.path) + + +def test_publish_mastodon_over_limit_raises_before_api_call( + full_config, +) -> None: + long_title = "T" * (MASTODON_STATUS_LIMIT + 1) + match = MatchedPost( + path="2016/2016-08-04-foo.md", + title=long_title, + date=date(2016, 8, 4), + url="https://blog.example.com/2016/08/04/foo/", + ) + fake = _FakeMastodon() + + with pytest.raises(PublishError): + publish_mastodon(full_config, [match], client_factory=fake) + + assert fake.calls == [] + + +def test_publish_mastodon_combined_status_for_multiple_matches(full_config) -> None: + match_a = _make_match("2016/2016-08-04-a.md", "Alpha") + match_b = _make_match("2016/2016-08-04-b.md", "Bravo") + fake = _FakeMastodon() + + publish_mastodon(full_config, [match_a, match_b], client_factory=fake) + + assert len(fake.calls) == 1 + status, _ = fake.calls[0] + assert "Alpha" in status + assert "Bravo" in status + assert match_a.url in status + assert match_b.url in status + + +def test_publish_mastodon_uses_config_visibility(env_setup, tmp_path: Path) -> None: + os.environ["VISIBILITY"] = "unlisted" + os.environ["DATA_DIR"] = str(tmp_path) + config = load_config() + match = _make_match("2016/2016-08-04-foo.md", "Foo") + fake = _FakeMastodon() + + publish_mastodon(config, [match], client_factory=fake) + + assert fake.calls[0][1] == {"visibility": "unlisted"} diff --git a/tests/test_run_logging.py b/tests/test_run_logging.py index 874e189..9c87ecb 100644 --- a/tests/test_run_logging.py +++ b/tests/test_run_logging.py @@ -8,9 +8,22 @@ from pathlib import Path import pytest +from datetime import date + from tenbackward import main as main_module from tenbackward.logging_setup import JsonFormatter from tenbackward.main import main +from tenbackward.matching import MatchedPost +from tenbackward.publishing import PublishError + + +def _match(path: str) -> MatchedPost: + return MatchedPost( + path=path, + title=path, + date=date(2016, 8, 4), + url=f"https://blog.example.com/2016/08/04/{path.split('-', 3)[-1].removesuffix('.md')}/", + ) @pytest.fixture() @@ -38,7 +51,11 @@ def _seed_state(data_dir: Path, ids: list[str]) -> None: def test_run_emits_one_info_summary_on_success(env_setup, data_dir, capture_logger, monkeypatch) -> None: monkeypatch.setenv("DATA_DIR", str(data_dir)) - monkeypatch.setattr(main_module, "_iter_candidates", lambda config: ["new-1", "already-1"]) + def _stub_publish(config, posts, **_kwargs): + return "stubbed" + + monkeypatch.setattr(main_module, "_iter_candidates", lambda config: [_match("new-1"), _match("already-1")]) + monkeypatch.setattr(main_module, "publish_mastodon", _stub_publish) _seed_state(data_dir, ["already-1"]) @@ -100,7 +117,12 @@ def test_run_distinguishes_posted_from_skipped_via_ids(env_setup, data_dir, capt monkeypatch.setattr( main_module, "_iter_candidates", - lambda config: ["alpha", "beta", "gamma"], + lambda config: [_match("alpha"), _match("beta"), _match("gamma")], + ) + monkeypatch.setattr( + main_module, + "publish_mastodon", + lambda config, posts, **_kwargs: "stubbed", ) _seed_state(data_dir, ["beta"]) @@ -114,3 +136,28 @@ def test_run_distinguishes_posted_from_skipped_via_ids(env_setup, data_dir, capt assert "beta" not in payload["posted_ids"] assert payload["skipped"] == 1 assert payload["posted"] == 2 + + +def test_run_publish_failure_leaves_state_untouched( + env_setup, data_dir, capture_logger, monkeypatch +) -> None: + monkeypatch.setenv("DATA_DIR", str(data_dir)) + + def _boom(config, posts, **_kwargs): + raise PublishError("publish_failed: stub") + + monkeypatch.setattr(main_module, "_iter_candidates", lambda config: [_match("alpha")]) + monkeypatch.setattr(main_module, "publish_mastodon", _boom) + + rc = main() + assert rc != 0 + + from tenbackward.state import load_posted + + assert load_posted(data_dir) == [] + error_lines = [ + line + for line in _run_lines(capture_logger) + if line.get("level") == "ERROR" + ] + assert any(line.get("event") == "pipeline_error" for line in error_lines)