AI Implementation feature(1089): Scheduled Execution with Cron and Retry Handling (#9)
This commit was merged in pull request #9.
This commit is contained in:
+3
-3
@@ -2,11 +2,11 @@
|
||||
|
||||
MASTODON_BASE_URL=https://mastodon.example
|
||||
MASTODON_ACCESS_TOKEN=replace-me
|
||||
VISIBILITY=public
|
||||
MASTODON_VISIBILITY=public
|
||||
SITE_URL=https://blog.example.com
|
||||
HASHTAGS=#throwback,#10backward
|
||||
THROWNBACK_PREFIX=Heute vor 10 Jahren:
|
||||
MAX_RETRIES=3
|
||||
THROWBACK_PREFIX=Heute vor 10 Jahren:
|
||||
MAX_RETRIES=5
|
||||
RUN_AT=09:00
|
||||
TZ=Europe/Berlin
|
||||
|
||||
|
||||
+8
-2
@@ -11,6 +11,7 @@ RUN apt-get update \
|
||||
git \
|
||||
ca-certificates \
|
||||
tzdata \
|
||||
util-linux \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN groupadd --system bot \
|
||||
@@ -27,14 +28,19 @@ RUN pip install --no-cache-dir --upgrade pip \
|
||||
|
||||
COPY --chown=bot:bot crontab/ /app/crontab/
|
||||
COPY --chown=bot:bot entrypoint.sh /app/entrypoint.sh
|
||||
COPY --chown=bot:bot run-bot.sh /app/run-bot.sh
|
||||
COPY --chown=root:root run-bot.sh /usr/local/bin/run-bot.sh
|
||||
COPY --chown=bot:bot src/ /app/src/
|
||||
|
||||
RUN chmod 0755 /app/entrypoint.sh \
|
||||
&& chmod 0755 /app/crontab/install-cron.sh \
|
||||
&& chmod 0755 /app/run-bot.sh \
|
||||
&& chmod 0755 /usr/local/bin/run-bot.sh \
|
||||
&& chmod 0644 /app/crontab/tenbackward.cron
|
||||
|
||||
ENV PYTHONPATH=/app/src
|
||||
|
||||
USER bot
|
||||
|
||||
# Note: entrypoint.sh runs as root so it can install /etc/cron.d/tenbackward
|
||||
# and exec cron -f. The cron-launched wrapper drops to the unprivileged
|
||||
# `bot` user before invoking Python.
|
||||
ENTRYPOINT ["/app/entrypoint.sh"]
|
||||
|
||||
@@ -20,4 +20,14 @@ if ! head -n 1 "${CRON_FILE}" | grep -qE '^[A-Z_]+='; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! grep -q '/usr/local/bin/run-bot.sh' "${CRON_FILE}"; then
|
||||
echo "install-cron: ${CRON_FILE} must invoke /usr/local/bin/run-bot.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! grep -qE '/proc/1/fd/1' "${CRON_FILE}"; then
|
||||
echo "install-cron: ${CRON_FILE} must redirect output to /proc/1/fd/1" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "install-cron: ${CRON_FILE} installed (mode 0644)"
|
||||
|
||||
+3
-4
@@ -11,13 +11,12 @@ services:
|
||||
environment:
|
||||
MASTODON_BASE_URL: ${MASTODON_BASE_URL:-https://mastodon.example}
|
||||
MASTODON_ACCESS_TOKEN: ${MASTODON_ACCESS_TOKEN:-replace-me}
|
||||
VISIBILITY: ${VISIBILITY:-public}
|
||||
MASTODON_VISIBILITY: ${MASTODON_VISIBILITY:-public}
|
||||
SITE_URL: ${SITE_URL:-https://blog.example.com}
|
||||
HASHTAGS: ${HASHTAGS:-#throwback,#10backward}
|
||||
THROWNBACK_PREFIX: ${THROWNBACK_PREFIX:-Throwback:}
|
||||
MAX_RETRIES: ${MAX_RETRIES:-3}
|
||||
THROWBACK_PREFIX: ${THROWBACK_PREFIX:-Throwback:}
|
||||
MAX_RETRIES: ${MAX_RETRIES:-5}
|
||||
RUN_AT: ${RUN_AT:-09:00}
|
||||
TZ: ${TZ:-Europe/Berlin}
|
||||
volumes:
|
||||
- ./.env:/.env:ro
|
||||
- ./data:/app/data
|
||||
|
||||
@@ -16,45 +16,33 @@ dataclass. Job 1083 changes every variable that ships in
|
||||
|
||||
# Required Environment Variables
|
||||
|
||||
All eight of these must be present and non-empty in the process
|
||||
environment for the container to boot.
|
||||
The following keys must be present and non-empty in the merged dotenv/process environment before application defaults are applied.
|
||||
|
||||
| Key | Purpose | Validation |
|
||||
|----------------------|--------------------------------------------------------|------------------------------------------------------------------|
|
||||
| `MASTODON_BASE_URL` | Mastodon instance to post against. | Must parse with `http` or `https` scheme and a non-empty netloc. |
|
||||
| `MASTODON_ACCESS_TOKEN` | OAuth token used by the future Mastodon.py client. | Required string. |
|
||||
| `VISIBILITY` | Default post visibility. | Must be `public` or `unlisted`. |
|
||||
| `SITE_URL` | Source blog URL (used by the future clone step). | Must parse with `http` or `https` scheme and a non-empty netloc. |
|
||||
| `HASHTAGS` | Comma-separated hashtags appended to every throwback. | Required string (formatting handled by the future poster). |
|
||||
| `THROWNBACK_PREFIX` | Per-post title prefix (default: `Throwback:`). | Required string. |
|
||||
| `MAX_RETRIES` | Number of additional retries after the first attempt. | Non-negative integer (parsed by `_parse_max_retries`). |
|
||||
| `RUN_AT` | Daily fire time for the cron entry. | `HH:MM` 24-hour format. |
|
||||
| `TZ` | IANA timezone used by cron + the container clock. | Must resolve via `zoneinfo.ZoneInfo`. |
|
||||
| Key | Purpose | Validation |
|
||||
|---|---|---|
|
||||
| `MASTODON_BASE_URL` | Mastodon instance to post against. | Valid `http` or `https` URL. |
|
||||
| `MASTODON_ACCESS_TOKEN` | OAuth token used by the Mastodon client. | Required string; redacted in logs. |
|
||||
| `MASTODON_VISIBILITY` | Post visibility. | `public` or `unlisted`. |
|
||||
| `SITE_URL` | Source blog URL. | Valid `http` or `https` URL. |
|
||||
| `HASHTAGS` | Comma-separated hashtags. | Required string. |
|
||||
| `THROWBACK_PREFIX` | Prefix prepended to each status. | Required string. |
|
||||
| `MAX_RETRIES` | Maximum number of pipeline attempts. | Non-negative integer; defaults to `5` when absent. |
|
||||
| `RUN_AT` | Daily cron fire time. | `HH:MM` 24-hour format. |
|
||||
| `TZ` | Cron and application timezone. | Valid IANA timezone. |
|
||||
|
||||
> **Note** — All nine keys are required since Job 1083. The previously
|
||||
> optional `MASTODON_VISIBILITY`, `THROWBACK_PREFIX`, and `RETRY_COUNT`
|
||||
> were promoted to first-class citizens and renamed to `VISIBILITY`,
|
||||
> `THROWNBACK_PREFIX`, and `MAX_RETRIES`.
|
||||
`BLOG_REPO_URL` and `BLOG_DIR` are optional and receive defaults after this required-key check. `DATA_DIR` is read separately from the process environment and defaults to `/app/data`.
|
||||
|
||||
# Defaults
|
||||
|
||||
`DEFAULTS` in `config.py` provides fallback strings that `apply_defaults`
|
||||
fills into the merged map **before** validation runs:
|
||||
Defaults are applied only after required-key validation. The current defaults are:
|
||||
|
||||
| Default key | Default value |
|
||||
|-----------------------|-------------------------------------------|
|
||||
| `VISIBILITY` | `public` |
|
||||
| `THROWNBACK_PREFIX` | `Heute vor 10 Jahren:` |
|
||||
| `MAX_RETRIES` | `3` |
|
||||
| `TZ` | `Europe/Berlin` |
|
||||
| `RUN_AT` | `09:00` |
|
||||
| `BLOG_REPO_URL` | `https://git.chaospott.de/Chaospott/site` |
|
||||
| `BLOG_DIR` | `blog` (resolved relative to `data_dir`) |
|
||||
| Default key | Default value |
|
||||
|---|---|
|
||||
| `MAX_RETRIES` | `5` |
|
||||
| `BLOG_REPO_URL` | `https://git.chaospott.de/Chaospott/site` |
|
||||
| `BLOG_DIR` | `blog` (resolved relative to `data_dir`) |
|
||||
|
||||
Because every required key carries a default, a freshly-initialised
|
||||
container can still boot to validate the scaffold, but the operator
|
||||
must supply `MASTODON_BASE_URL` and `MASTODON_ACCESS_TOKEN` (and
|
||||
ideally `SITE_URL`) to make a real run.
|
||||
Application-facing required values are not silently supplied by `config.py`; the container environment may still provide `TZ=Europe/Berlin` and Compose may provide development fallback values.
|
||||
|
||||
# `Config` Dataclass
|
||||
|
||||
@@ -62,14 +50,14 @@ ideally `SITE_URL`) to make a real run.
|
||||
|-------------------------|-----------|----------------------------------------------|
|
||||
| `mastodon_base_url` | `str` | `MASTODON_BASE_URL` |
|
||||
| `mastodon_access_token` | `str` | `MASTODON_ACCESS_TOKEN` |
|
||||
| `visibility` | `str` | `VISIBILITY` |
|
||||
| `mastodon_visibility` | `str` | `MASTODON_VISIBILITY` |
|
||||
| `site_url` | `str` | `SITE_URL` |
|
||||
| `hashtags` | `str` | `HASHTAGS` |
|
||||
| `throwback_prefix` | `str` | `THROWNBACK_PREFIX` |
|
||||
| `throwback_prefix` | `str` | `THROWBACK_PREFIX` |
|
||||
| `max_retries` | `int` | `MAX_RETRIES` (parsed as non-negative int) |
|
||||
| `run_at` | `str` | `RUN_AT` |
|
||||
| `tz` | `str` | `TZ` |
|
||||
| `data_dir` | `Path` | Constant: `Path("/app/data")` |
|
||||
| `data_dir` | `Path` | `DATA_DIR`, default `/app/data` |
|
||||
| `blog_repo_url` | `str` | `BLOG_REPO_URL` |
|
||||
| `blog_dir` | `Path` | `BLOG_DIR` resolved against `data_dir` |
|
||||
|
||||
|
||||
@@ -45,8 +45,7 @@ of matching posts, in this shape:
|
||||
{hashtags}
|
||||
```
|
||||
|
||||
* The prefix comes from `Config.throwback_prefix` (default
|
||||
`Heute vor 10 Jahren:`, overridable via `THROWNBACK_PREFIX`).
|
||||
* The prefix comes from `Config.throwback_prefix` (configured via `THROWBACK_PREFIX`).
|
||||
* Posts are sorted by `(date, path)` so output is deterministic
|
||||
regardless of upstream order.
|
||||
* `hashtags` is a comma-separated string; commas are collapsed to a
|
||||
@@ -106,10 +105,7 @@ publish failure.
|
||||
|
||||
# Mastodon API Call
|
||||
|
||||
`_post_status_via_mastodon_py()` constructs a `mastodon.Mastodon`
|
||||
client with `(access_token=config.mastodon_access_token,
|
||||
api_base_url=config.mastodon_base_url)` and calls
|
||||
`status_post(status, visibility=config.visibility)`.
|
||||
`_post_status_via_mastodon_py()` constructs a `mastodon.Mastodon` client with `(access_token=config.mastodon_access_token, api_base_url=config.mastodon_base_url)` and calls `status_post(status, visibility=config.mastodon_visibility)`.
|
||||
|
||||
The `client_factory` keyword argument on `publish_mastodon` lets
|
||||
tests inject a fake client without monkey-patching. Production
|
||||
@@ -150,8 +146,8 @@ re-attempt on the next iteration.
|
||||
|---|---|---|
|
||||
| `MASTODON_BASE_URL` | `Config.mastodon_base_url` | Mastodon instance URL. |
|
||||
| `MASTODON_ACCESS_TOKEN` | `Config.mastodon_access_token` | OAuth token passed to the `Mastodon` client. |
|
||||
| `VISIBILITY` | `Config.visibility` | Passed as `visibility=` to `status_post`. |
|
||||
| `THROWNBACK_PREFIX` | `Config.throwback_prefix` | First line of every published status. |
|
||||
| `MASTODON_VISIBILITY` | `Config.mastodon_visibility` | Passed as `visibility=` to `status_post`. |
|
||||
| `THROWBACK_PREFIX` | `Config.throwback_prefix` | First line of every published status. |
|
||||
| `HASHTAGS` | `Config.hashtags` | Trailing hashtag line; commas become spaces. |
|
||||
|
||||
See [Config Schema](/architecture/config-schema.md) for full validation
|
||||
|
||||
@@ -97,15 +97,12 @@ characters); the runner fails safely (raises) rather than truncating.
|
||||
|
||||
# Retry Behaviour (`_run_with_retry`)
|
||||
|
||||
* `attempts = max(1, config.max_retries + 1)` — at least one attempt
|
||||
even if `MAX_RETRIES=0`.
|
||||
* Any exception inside `_run_once` is caught (`noqa: BLE001` —
|
||||
intentional, the whole pass is opaque to the runner).
|
||||
* Each failed attempt is recorded via
|
||||
`log_error("pipeline_error", exc=exc, attempt=attempt, max_attempts=attempts)`.
|
||||
* When the budget is exhausted, the runner emits
|
||||
`log_error("pipeline_failed", exc=last_exc, attempts=attempts)` and
|
||||
returns `None` so `main` can translate it to `exit 1`.
|
||||
* `attempts = max(1, config.max_retries)` — at least one attempt even if `MAX_RETRIES=0`.
|
||||
* The runner owns the unified retry budget; `_run_once` calls `ensure_repo` with `max_retries=0`.
|
||||
* Transient Git and Mastodon failures are retried with delays of `2 ** (attempt - 1)` seconds between attempts.
|
||||
* Each failed attempt emits `retry_attempt` with `operation`, `attempt`, and the exception `error` text.
|
||||
* Fatal configuration, local repository, and publishing validation errors emit `retry_exhausted` immediately without retrying.
|
||||
* When the budget is exhausted, `retry_exhausted` includes the final operation, total attempts, and error text, and `main` returns exit code `1`.
|
||||
|
||||
# Silent-on-No-Matches Contract
|
||||
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
---
|
||||
type: architecture
|
||||
title: Job 1089 Runtime Changes
|
||||
description: Container privilege separation, unified retry logging, configuration validation, and persistent data wiring introduced by the current implementation.
|
||||
tags: [architecture, docker, retry, logging, configuration]
|
||||
timestamp: 2026-08-04T20:44:00Z
|
||||
---
|
||||
|
||||
# Overview
|
||||
|
||||
The current runtime keeps the container entrypoint and cron daemon as root while executing the Python bot as the unprivileged `bot` user. The pipeline owns one retry budget for Git synchronization and Mastodon publishing, and structured retry events include the operation and exception text. Configuration now validates required values before optional defaults are applied.
|
||||
|
||||
# Wiring
|
||||
|
||||
```text
|
||||
Docker ENTRYPOINT /app/entrypoint.sh (root)
|
||||
-> validates environment and renders /etc/cron.d/tenbackward
|
||||
-> cron -f
|
||||
-> /usr/local/bin/run-bot.sh
|
||||
-> setpriv/su to bot
|
||||
-> python -m tenbackward
|
||||
-> load_config()
|
||||
-> _run_with_retry()
|
||||
-> _run_once()
|
||||
-> ensure_repo(..., max_retries=0, logger=...)
|
||||
-> match posts
|
||||
-> PostedStore dedupe
|
||||
-> publish_mastodon()
|
||||
-> PostedStore.mark_posted_many()
|
||||
```
|
||||
|
||||
The single `./data:/app/data` Compose bind mount contains both the blog clone and `posted.json`; no file-level bind mount is used.
|
||||
|
||||
# Retry and Logging Contract
|
||||
|
||||
`MAX_RETRIES` is interpreted as the maximum number of attempts, with at least one attempt when configured as zero. Backoff delays are `1, 2, 4, ...` seconds between attempts. Git clone/pull retries are surfaced to the runner with operation labels (`git clone` or `git pull`), while publishing failures use `mastodon post`. Fatal configuration, local-repository, and publishing validation errors are not retried.
|
||||
|
||||
Retry events include an `error` field containing the exception message. `log_event()` sends structured fields through the same redaction path as other log helpers.
|
||||
|
||||
# Configuration Validation
|
||||
|
||||
`load_config()` merges dotenv values and process environment values, validates required keys against that raw merged map, then applies defaults. `MAX_RETRIES`, `BLOG_REPO_URL`, and `BLOG_DIR` may use defaults; the required Mastodon, site, hashtag, prefix, schedule, and timezone values must be explicitly non-empty.
|
||||
|
||||
# Key Files
|
||||
|
||||
| Path | Responsibility |
|
||||
|---|---|
|
||||
| `/repo/Dockerfile` | Installs cron, Git, timezone data, and `setpriv`; leaves the final process root-capable for cron and owns `/app/data` by `bot`. |
|
||||
| `/repo/entrypoint.sh` | Validates environment, renders the daily cron entry, installs it as root, and starts foreground cron. |
|
||||
| `/repo/run-bot.sh` | Drops from root to `bot` using `setpriv` or `su`, then invokes the Python module. |
|
||||
| `/repo/docker-compose.yml` | Supplies environment values and mounts `./data` at `/app/data`. |
|
||||
| `/repo/src/tenbackward/main.py` | Owns the unified retry loop and passes logger/retry controls into repository synchronization. |
|
||||
| `/repo/src/tenbackward/blog.py` | Clones or fast-forwards the blog and emits operation-specific retry events. |
|
||||
| `/repo/src/tenbackward/config.py` | Performs raw required-key validation and builds the typed configuration. |
|
||||
| `/repo/src/tenbackward/logging_setup.py` | Provides structured events, redaction, and JSON-per-line formatting. |
|
||||
| `/repo/tests/test_docker_artifacts.py` | Verifies container privilege and artifact wiring. |
|
||||
| `/repo/tests/test_config.py` | Verifies required-key validation and defaults. |
|
||||
| `/repo/tests/test_run_logging.py` | Verifies retry fields, attempt behavior, and state safety. |
|
||||
|
||||
# Related
|
||||
|
||||
* [System Architecture](/architecture/system-overview.md)
|
||||
* [Config Schema](/architecture/config-schema.md)
|
||||
* [Pipeline Runner](/architecture/pipeline-runner.md)
|
||||
* [Environment Variable Setup](/operations/environment-setup.md)
|
||||
* [Daily Run Guide](/guides/daily-run.md)
|
||||
@@ -26,15 +26,7 @@ scheduled tick.
|
||||
|
||||
# What the Run Does
|
||||
|
||||
Before scanning, the process ensures the configured blog repository exists
|
||||
under `/app/data/blog` and is fast-forwarded from `BLOG_REPO_URL`. It then
|
||||
walks `_posts/blog/**/*.md`, finds posts whose date is exactly ten years
|
||||
before today, skips IDs already present in `posted.json`, composes a
|
||||
single German-language Mastodon status (default prefix
|
||||
`Heute vor 10 Jahren:`) listing each new match's title and canonical URL
|
||||
followed by the configured hashtags, and — only after the Mastodon API
|
||||
call succeeds — records the relative paths of the published posts in
|
||||
`posted.json`.
|
||||
Before scanning, the process ensures the configured blog repository exists under `/app/data/blog` and is fast-forwarded from `BLOG_REPO_URL`. It then walks `_posts/blog/**/*.md`, finds posts whose date is exactly ten years before today, skips IDs already present in `posted.json`, composes a single Mastodon status listing each new match's title and canonical URL followed by the configured hashtags, and — only after the Mastodon API call succeeds — records the relative paths of the published posts in `posted.json`. The scheduled wrapper executes this Python process as the unprivileged `bot` user.
|
||||
|
||||
# What You Should See
|
||||
|
||||
@@ -61,8 +53,8 @@ this is the **only** line you should see — the runner emits no
|
||||
| Blog clone or pull fails | 1× `blog_repo_error` | `1` |
|
||||
| Config valid, matching IDs are new | 1× `startup`, then 1× `run_complete` with `posted=N` | `0` |
|
||||
| Config valid, all matches already posted | 1× `startup`, then 1× `run_complete` with `posted=0` | `0` |
|
||||
| Pipeline raises once, recovers | 1× `pipeline_error`, then 1× `run_complete` | `0` |
|
||||
| Pipeline keeps raising (budget exhausted) | `MAX_RETRIES + 1` × `pipeline_error`, then 1× `pipeline_failed` | `1` |
|
||||
| Pipeline raises once, recovers | `retry_attempt`, then `run_complete` | `0` |
|
||||
| Pipeline keeps raising (budget exhausted) | `retry_attempt` events, then `retry_exhausted` with operation and error | `1` |
|
||||
| Config invalid | 1× `configuration_error` (extras describe what failed) | `2` |
|
||||
|
||||
> **Tester tip** — `docker compose ps` should report the container as
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
---
|
||||
type: guide
|
||||
title: Runtime Verification Guide
|
||||
description: Tester workflow for validating root cron startup, unprivileged bot execution, retry events, and persistent data behavior.
|
||||
tags: [guide, tester, docker, runtime]
|
||||
timestamp: 2026-08-04T20:44:00Z
|
||||
---
|
||||
|
||||
# Setup
|
||||
|
||||
1. Copy `.env.example` to `.env` and provide valid required values.
|
||||
2. Build and start the service:
|
||||
|
||||
```bash
|
||||
docker compose up --build -d
|
||||
```
|
||||
|
||||
3. Follow container output:
|
||||
|
||||
```bash
|
||||
docker compose logs -f bot
|
||||
```
|
||||
|
||||
# Expected Runtime Behavior
|
||||
|
||||
The container remains running with `cron -f` as its foreground process. Startup output includes the configured `RUN_AT` and `TZ`. The scheduled cron command invokes `/usr/local/bin/run-bot.sh`, which runs the Python bot as user `bot`, not root.
|
||||
|
||||
The host `./data` directory should contain the synchronized `blog/` clone and, after the state store is loaded, `posted.json`. Both are under the one `/app/data` mount.
|
||||
|
||||
# Manual Run
|
||||
|
||||
Run the same application command without waiting for cron:
|
||||
|
||||
```bash
|
||||
docker compose exec bot /usr/local/bin/python -m tenbackward
|
||||
```
|
||||
|
||||
A valid run exits `0`. A configuration failure exits `2`; an exhausted pipeline retry budget exits `1`.
|
||||
|
||||
# Retry Checks
|
||||
|
||||
To exercise transient failure handling, use a test repository or injected test double that fails once and then succeeds. Verify JSON events contain:
|
||||
|
||||
| Event | Expected fields |
|
||||
|---|---|
|
||||
| `retry_attempt` | `operation`, `attempt`, and the original `error` text. |
|
||||
| `retry_exhausted` | Final `operation`, total `attempts`, and `error` text when all attempts fail. |
|
||||
| `run_complete` | Counters and `posted_ids` only after a successful pass. |
|
||||
|
||||
With `MAX_RETRIES=0`, one attempt is still made and no sleep occurs. With a larger value, delays occur only between attempts and follow powers of two beginning at one second.
|
||||
|
||||
# Configuration Checks
|
||||
|
||||
Remove a required variable from `.env` and restart the service. The entrypoint should report the missing variable before cron starts. If the shell check is bypassed and the Python module runs directly, `configuration_error` should identify the missing key and return exit code `2` rather than silently substituting an application default.
|
||||
|
||||
# Persistence and Failure Safety
|
||||
|
||||
After a successful publication, inspect `/app/data/posted.json` and confirm new relative post paths appear under the `posted` list. If Mastodon publishing fails, the path must not be recorded; a later retry should be able to publish it again.
|
||||
|
||||
# Key Visual/Observable Elements
|
||||
|
||||
This service has no graphical UI. Test-facing outputs are:
|
||||
|
||||
* Container status and `docker compose logs` output.
|
||||
* JSON-per-line startup, retry, failure, and completion events.
|
||||
* Exit codes from manual runs.
|
||||
* `/app/data/blog` and `/app/data/posted.json` on the mounted host directory.
|
||||
|
||||
# Related
|
||||
|
||||
* [Daily Run Guide](/guides/daily-run.md)
|
||||
* [Job 1089 Runtime Changes](/architecture/runtime-changes.md)
|
||||
* [Environment Variable Setup](/operations/environment-setup.md)
|
||||
* [Cron Lifecycle](/operations/cron-lifecycle.md)
|
||||
+3
-1
@@ -4,6 +4,7 @@ okf_version: "0.1"
|
||||
|
||||
# Architecture
|
||||
* [System Architecture](/architecture/system-overview.md) — Component map of 10Backward: entrypoint, config, logging, runner, state, publishing boundary, and cron wiring.
|
||||
* [Runtime Changes](/architecture/runtime-changes.md) — Root cron/unprivileged bot separation, unified retries, raw configuration validation, and data-volume wiring.
|
||||
* [Config Schema](/architecture/config-schema.md) — Required/optional env vars, validation rules, and the typed Config dataclass.
|
||||
* [Logging & Run Summary](/architecture/logging.md) — JSON formatter, secret redaction, and structured event helpers.
|
||||
* [Pipeline Runner](/architecture/pipeline-runner.md) — How a cron tick synchronizes the blog, matches anniversaries, applies deduplication, publishes to Mastodon, retries failures, and emits run events.
|
||||
@@ -15,4 +16,5 @@ okf_version: "0.1"
|
||||
* [Cron Lifecycle](/operations/cron-lifecycle.md) — How `entrypoint.sh` renders `/etc/cron.d/tenbackward` and hands off to `cron -f`.
|
||||
|
||||
# User Guides
|
||||
* [Daily Run Guide](/guides/daily-run.md) — Tester/operator walkthrough of repository sync, anniversary matching, deduplication, Mastodon publishing, and expected log/output behaviour.
|
||||
* [Daily Run Guide](/guides/daily-run.md) — Operator walkthrough of repository sync, anniversary matching, deduplication, publishing, and expected logs.
|
||||
* [Runtime Verification Guide](/guides/runtime-verification.md) — Tester steps for privilege separation, retry events, configuration failures, and persistent state.
|
||||
|
||||
@@ -18,26 +18,21 @@ full set before the container will boot.
|
||||
|--------------------------|--------------------------------|---------------------------------------------------|
|
||||
| `MASTODON_BASE_URL` | `https://mastodon.social` | Mastodon instance to post against. |
|
||||
| `MASTODON_ACCESS_TOKEN` | _(from your Mastodon account)_ | OAuth access token. |
|
||||
| `VISIBILITY` | `public` | Post visibility (`public` or `unlisted`). |
|
||||
| `SITE_URL` | `https://blog.example.com` | Source blog URL (used by the future clone step). |
|
||||
| `HASHTAGS` | `#throwback,#10backward` | Hashtags appended to every throwback post. |
|
||||
| `THROWNBACK_PREFIX` | `Heute vor 10 Jahren:` | Prefix prepended to every Mastodon status. |
|
||||
| `MAX_RETRIES` | `3` | Non-negative retry count for the pipeline. |
|
||||
| `RUN_AT` | `09:00` | Daily fire time (HH:MM, 24-hour). |
|
||||
| `TZ` | `Europe/Berlin` | IANA timezone for cron + container clock. |
|
||||
| `MASTODON_VISIBILITY` | `public` | Post visibility (`public` or `unlisted`). |
|
||||
| `SITE_URL` | `https://blog.example.com` | Source blog URL. |
|
||||
| `HASHTAGS` | `#throwback,#10backward` | Hashtags appended to every throwback post. |
|
||||
| `THROWBACK_PREFIX` | `Heute vor 10 Jahren:` | Prefix prepended to every Mastodon status. |
|
||||
| `MAX_RETRIES` | `5` | Maximum number of pipeline attempts. |
|
||||
| `RUN_AT` | `09:00` | Daily fire time (`HH:MM`, 24-hour). |
|
||||
| `TZ` | `Europe/Berlin` | IANA timezone for cron and application. |
|
||||
| `BLOG_REPO_URL` | project default | Optional source repository URL. |
|
||||
| `BLOG_DIR` | `blog` | Optional clone directory, relative to `DATA_DIR` unless absolute. |
|
||||
|
||||
# Where Each Name Is Enforced
|
||||
|
||||
* `entrypoint.sh` — exports `TZ`, validates all nine are non-empty,
|
||||
and validates `RUN_AT` matches `^([01][0-9]|2[0-3]):[0-5][0-9]$`
|
||||
before writing the cron file. A missing var aborts the container
|
||||
with `ERROR: missing required environment variable(s): ...` and
|
||||
exits `1`.
|
||||
* `tenbackward.config.validate_config` — same set, plus URL/visibility/
|
||||
TZ/MAX_RETRIES validation. Failures surface as a single
|
||||
`ConfigError` listing every problem.
|
||||
* The `kilo.json` config file is **not** an env-var file — it is the
|
||||
agent runtime configuration.
|
||||
* `entrypoint.sh` — validates all nine required keys, validates `RUN_AT`, and installs the cron file as root. The cron-launched wrapper then drops privileges to `bot` before Python runs.
|
||||
* `tenbackward.config.validate_config` — validates the merged raw required keys plus URL, visibility, timezone, and retry syntax. Defaults are applied only after required-key validation.
|
||||
* `docker-compose.yml` — supplies development environment values and mounts `./data` at `/app/data`.
|
||||
|
||||
# Setup Steps
|
||||
|
||||
@@ -80,13 +75,9 @@ If you have an existing `.env` from the scaffold, rename these
|
||||
manually. The renamed variables are **not** backwards compatible — the
|
||||
container will refuse to start with both versions set.
|
||||
|
||||
# Defaults That Can Be Removed
|
||||
# Defaults
|
||||
|
||||
`DEFAULTS` provides fallbacks for `VISIBILITY`, `THROWNBACK_PREFIX`,
|
||||
`MAX_RETRIES`, `TZ`, and `RUN_AT`. You may leave them out of your
|
||||
`.env`, but the operator contract is "every required key is set" —
|
||||
prod deployments should set them explicitly so a missing key is
|
||||
caught at boot instead of silently used as a default.
|
||||
`MAX_RETRIES` defaults to `5`; `BLOG_REPO_URL` and `BLOG_DIR` also have application defaults. Other application-facing required keys must be present in the raw merged dotenv/process environment. The Docker image supplies `TZ=Europe/Berlin`, and Compose supplies development fallbacks, but production deployments should set every required value explicitly.
|
||||
|
||||
# Related
|
||||
|
||||
|
||||
+3
-3
@@ -11,10 +11,10 @@ fi
|
||||
required_vars=(
|
||||
MASTODON_BASE_URL
|
||||
MASTODON_ACCESS_TOKEN
|
||||
VISIBILITY
|
||||
MASTODON_VISIBILITY
|
||||
SITE_URL
|
||||
HASHTAGS
|
||||
THROWNBACK_PREFIX
|
||||
THROWBACK_PREFIX
|
||||
MAX_RETRIES
|
||||
RUN_AT
|
||||
TZ
|
||||
@@ -48,7 +48,7 @@ trap 'rm -f "${tmp_cron}"' EXIT
|
||||
echo "SHELL=/bin/bash"
|
||||
echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
echo "TZ=${TZ}"
|
||||
echo "${minute} ${hour} * * * cd /app && /usr/local/bin/python -m tenbackward >> /app/data/cron.log 2>&1"
|
||||
echo "${minute} ${hour} * * * /usr/local/bin/run-bot.sh >> /proc/1/fd/1 2>&1"
|
||||
} > "${tmp_cron}"
|
||||
|
||||
install -m 0644 -o root -g root "${tmp_cron}" /etc/cron.d/tenbackward
|
||||
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
cd /app
|
||||
|
||||
bot_uid="$(id -u bot)"
|
||||
bot_gid="$(id -g bot)"
|
||||
|
||||
if command -v setpriv >/dev/null 2>&1; then
|
||||
exec setpriv --reuid="${bot_uid}" --regid="${bot_gid}" --clear-groups -- \
|
||||
/usr/local/bin/python -m tenbackward
|
||||
fi
|
||||
|
||||
exec su -s /bin/bash bot -c "exec /usr/local/bin/python -m tenbackward"
|
||||
+103
-48
@@ -26,7 +26,24 @@ _LOCAL_MODIFICATION_MARKERS = (
|
||||
|
||||
|
||||
class BlogRepoError(RuntimeError):
|
||||
"""Raised when the local blog working tree cannot be ensured."""
|
||||
"""Raised when the local blog working tree cannot be ensured.
|
||||
|
||||
Surface this as a fatal pipeline error (do not retry) so configuration
|
||||
problems fail immediately and transient network failures get a chance to
|
||||
recover via the retry loop.
|
||||
"""
|
||||
|
||||
|
||||
class BlogTransientError(RuntimeError):
|
||||
"""Wraps a transient HTTPS clone/pull failure that should be retried.
|
||||
|
||||
``operation`` records which Git action triggered the failure so the
|
||||
runner can label its log lines with "git clone" or "git pull".
|
||||
"""
|
||||
|
||||
def __init__(self, message: str, *, operation: str = "git pull") -> None:
|
||||
super().__init__(message)
|
||||
self.operation = operation
|
||||
|
||||
|
||||
def blog_dir(data_dir: Path) -> Path:
|
||||
@@ -52,54 +69,57 @@ def _is_local_modification_error(exc: GitCommandError) -> bool:
|
||||
return any(marker in message for marker in _LOCAL_MODIFICATION_MARKERS)
|
||||
|
||||
|
||||
def _fetch_and_pull(
|
||||
repo: Repo,
|
||||
*,
|
||||
fetch: Callable[[], object],
|
||||
pull: Callable[[], object],
|
||||
sleep: Callable[[float], None],
|
||||
max_retries: int,
|
||||
def _log_retry(
|
||||
logger: logging.Logger | None,
|
||||
*,
|
||||
operation: str,
|
||||
attempt: int,
|
||||
error: BaseException,
|
||||
) -> None:
|
||||
"""Fetch + fast-forward pull with retry/backoff for transient network errors.
|
||||
if logger is None:
|
||||
return
|
||||
logger.error(
|
||||
"blog_retry",
|
||||
extra={
|
||||
"event": "blog_retry",
|
||||
"operation": operation,
|
||||
"attempt": attempt,
|
||||
"error": str(error),
|
||||
},
|
||||
)
|
||||
|
||||
Local-modification errors raise ``BlogRepoError`` immediately, without retry.
|
||||
|
||||
def _retry_loop(
|
||||
*,
|
||||
operation: str,
|
||||
attempts: int,
|
||||
sleep: Callable[[float], None],
|
||||
logger: logging.Logger | None,
|
||||
action: Callable[[], None],
|
||||
) -> None:
|
||||
"""Invoke ``action`` up to ``attempts`` times with ``2 ** (attempt - 1)``
|
||||
second backoff between failures.
|
||||
|
||||
Transient errors are retried; :class:`BlogRepoError` (fatal local or
|
||||
validation problems) propagates immediately. On exhaustion, raises
|
||||
:class:`BlogTransientError` carrying the failing operation label.
|
||||
"""
|
||||
attempts = max(1, max_retries + 1)
|
||||
last_exc: GitCommandError | None = None
|
||||
|
||||
last_exc: BaseException | None = None
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
fetch()
|
||||
pull()
|
||||
action()
|
||||
return
|
||||
except GitCommandError as exc:
|
||||
except BlogRepoError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 — boundary retry hook
|
||||
last_exc = exc
|
||||
if _is_local_modification_error(exc):
|
||||
if logger is not None:
|
||||
logger.error(
|
||||
"blog_local_modifications",
|
||||
extra={"event": "blog_local_modifications", "error": str(exc)},
|
||||
)
|
||||
raise BlogRepoError(
|
||||
"local modifications detected in blog working tree; aborting run"
|
||||
) from exc
|
||||
if logger is not None:
|
||||
logger.error(
|
||||
"blog_fetch_error",
|
||||
extra={
|
||||
"event": "blog_fetch_error",
|
||||
"attempt": attempt,
|
||||
"max_attempts": attempts,
|
||||
"error": str(exc),
|
||||
},
|
||||
)
|
||||
_log_retry(logger, operation=operation, attempt=attempt, error=exc)
|
||||
if attempt < attempts:
|
||||
sleep(BACKOFF_BASE_SECONDS ** attempt)
|
||||
|
||||
sleep(BACKOFF_BASE_SECONDS ** (attempt - 1))
|
||||
assert last_exc is not None
|
||||
raise BlogRepoError(
|
||||
f"blog fetch/pull failed after {attempts} attempt(s): {last_exc}"
|
||||
raise BlogTransientError(
|
||||
f"{operation} failed after {attempts} attempt(s): {last_exc}",
|
||||
operation=operation,
|
||||
) from last_exc
|
||||
|
||||
|
||||
@@ -111,17 +131,22 @@ def ensure_repo(
|
||||
max_retries: int | None = None,
|
||||
logger: logging.Logger | None = None,
|
||||
sleep: Callable[[float], None] = time.sleep,
|
||||
clone_impl: Callable[[str, str], Repo] | None = None,
|
||||
fetch_impl: Callable[[Repo], Callable[[], object]] | None = None,
|
||||
pull_impl: Callable[[Repo], Callable[[], object]] | None = None,
|
||||
) -> Repo:
|
||||
"""Ensure a local working clone of the blog repository exists and is up to date.
|
||||
|
||||
Returns the :class:`git.Repo` pointing at an up-to-date working tree.
|
||||
Raises :class:`BlogRepoError` on irrecoverable failures.
|
||||
Raises :class:`BlogRepoError` on irrecoverable failures and
|
||||
:class:`BlogTransientError` after a transient failure exhausts the
|
||||
retry budget (so the caller can label and re-attempt at the pipeline
|
||||
level).
|
||||
"""
|
||||
url = repo_url or blog_repo_url()
|
||||
target = blog_path or blog_dir(data_dir)
|
||||
retries = MAX_RETRIES_DEFAULT if max_retries is None else max_retries
|
||||
attempts = max(1, retries)
|
||||
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -133,13 +158,28 @@ def ensure_repo(
|
||||
"blog_clone_start",
|
||||
extra={"event": "blog_clone_start", "url": url, "path": str(target)},
|
||||
)
|
||||
cloned = Repo.clone_from(url, str(target))
|
||||
|
||||
cloned_holder: dict[str, Repo] = {}
|
||||
|
||||
def _do_clone() -> None:
|
||||
r = (clone_impl or Repo.clone_from)(url, str(target))
|
||||
cloned_holder["repo"] = r
|
||||
|
||||
_retry_loop(
|
||||
operation="git clone",
|
||||
attempts=attempts,
|
||||
sleep=sleep,
|
||||
logger=logger,
|
||||
action=_do_clone,
|
||||
)
|
||||
if logger is not None:
|
||||
logger.info(
|
||||
"blog_clone_complete",
|
||||
extra={"event": "blog_clone_complete", "path": str(target)},
|
||||
)
|
||||
return cloned
|
||||
if "repo" in cloned_holder:
|
||||
return cloned_holder["repo"]
|
||||
return Repo(str(target))
|
||||
|
||||
try:
|
||||
repo = Repo(str(target))
|
||||
@@ -151,13 +191,28 @@ def ensure_repo(
|
||||
fetch = (fetch_impl or _default_fetch)(repo)
|
||||
pull = (pull_impl or _default_pull)(repo)
|
||||
|
||||
_fetch_and_pull(
|
||||
repo,
|
||||
fetch=fetch,
|
||||
pull=pull,
|
||||
def _do_pull() -> None:
|
||||
try:
|
||||
fetch()
|
||||
pull()
|
||||
except GitCommandError as exc:
|
||||
if _is_local_modification_error(exc):
|
||||
if logger is not None:
|
||||
logger.error(
|
||||
"blog_local_modifications",
|
||||
extra={"event": "blog_local_modifications", "error": str(exc)},
|
||||
)
|
||||
raise BlogRepoError(
|
||||
"local modifications detected in blog working tree; aborting run"
|
||||
) from exc
|
||||
raise BlogTransientError(str(exc), operation="git pull") from exc
|
||||
|
||||
_retry_loop(
|
||||
operation="git pull",
|
||||
attempts=attempts,
|
||||
sleep=sleep,
|
||||
max_retries=retries,
|
||||
logger=logger,
|
||||
action=_do_pull,
|
||||
)
|
||||
return repo
|
||||
|
||||
@@ -171,4 +226,4 @@ def _default_fetch(repo: Repo) -> Callable[[], object]:
|
||||
def _default_pull(repo: Repo) -> Callable[[], object]:
|
||||
def _do() -> object:
|
||||
return repo.git.pull("--ff-only")
|
||||
return _do
|
||||
return _do
|
||||
|
||||
+47
-19
@@ -13,10 +13,10 @@ from dotenv import dotenv_values, load_dotenv
|
||||
REQUIRED_KEYS = (
|
||||
"MASTODON_BASE_URL",
|
||||
"MASTODON_ACCESS_TOKEN",
|
||||
"VISIBILITY",
|
||||
"MASTODON_VISIBILITY",
|
||||
"SITE_URL",
|
||||
"HASHTAGS",
|
||||
"THROWNBACK_PREFIX",
|
||||
"THROWBACK_PREFIX",
|
||||
"MAX_RETRIES",
|
||||
"RUN_AT",
|
||||
"TZ",
|
||||
@@ -25,11 +25,7 @@ REQUIRED_KEYS = (
|
||||
OPTIONAL_KEYS: tuple[str, ...] = ("BLOG_REPO_URL", "BLOG_DIR")
|
||||
|
||||
DEFAULTS = {
|
||||
"VISIBILITY": "public",
|
||||
"THROWNBACK_PREFIX": "Heute vor 10 Jahren:",
|
||||
"MAX_RETRIES": "3",
|
||||
"TZ": "Europe/Berlin",
|
||||
"RUN_AT": "09:00",
|
||||
"MAX_RETRIES": "5",
|
||||
"BLOG_REPO_URL": "https://git.chaospott.de/Chaospott/site",
|
||||
"BLOG_DIR": "blog",
|
||||
}
|
||||
@@ -46,7 +42,7 @@ class ConfigError(ValueError):
|
||||
class Config:
|
||||
mastodon_base_url: str
|
||||
mastodon_access_token: str
|
||||
visibility: str
|
||||
mastodon_visibility: str
|
||||
site_url: str
|
||||
hashtags: str
|
||||
throwback_prefix: str
|
||||
@@ -67,6 +63,9 @@ class Config:
|
||||
return self.run_at.split(":", 1)[1]
|
||||
|
||||
|
||||
DEFAULT_DOTENV_PATH = Path("/app/.env")
|
||||
|
||||
|
||||
def _read_dotenv(dotenv_path: Optional[Path]) -> dict[str, str]:
|
||||
if dotenv_path is None:
|
||||
return {}
|
||||
@@ -88,19 +87,38 @@ def _values_from_env() -> dict[str, str]:
|
||||
def load_config(dotenv_path: Optional[Path] = None) -> Config:
|
||||
"""Load configuration from a dotenv file and the process environment.
|
||||
|
||||
Environment variables take precedence over the dotenv file so a mounted
|
||||
`.env` can be supplemented by Compose-level overrides.
|
||||
The .env file is loaded via python-dotenv at startup so the bind-mounted
|
||||
runtime secrets are visible to the process. Environment variables take
|
||||
precedence over the dotenv file so a mounted `.env` can be supplemented
|
||||
by Compose-level overrides.
|
||||
|
||||
Required keys are validated against the **raw** merged map (dotenv +
|
||||
process env) before any defaults are applied. Defaults are only used
|
||||
for the retry budget and the optional Jekyll blog source so a missing
|
||||
Mastodon/secret/URL/etc. fails fast with a clear error.
|
||||
"""
|
||||
if dotenv_path is None:
|
||||
dotenv_path = DEFAULT_DOTENV_PATH if DEFAULT_DOTENV_PATH.exists() else None
|
||||
|
||||
if dotenv_path is not None:
|
||||
load_dotenv(dotenv_path=str(dotenv_path), override=False)
|
||||
|
||||
file_values = _read_dotenv(dotenv_path)
|
||||
env_values = _values_from_env()
|
||||
|
||||
merged: dict[str, str] = {}
|
||||
merged.update(file_values)
|
||||
merged.update(env_values)
|
||||
raw: dict[str, str] = {}
|
||||
raw.update(file_values)
|
||||
raw.update(env_values)
|
||||
|
||||
# MAX_RETRIES is the only required key that has a permitted default
|
||||
# (5 attempts per the Job). Apply it before the required-key check so a
|
||||
# missing value uses the documented default rather than failing fast.
|
||||
if not raw.get("MAX_RETRIES"):
|
||||
raw["MAX_RETRIES"] = DEFAULTS["MAX_RETRIES"]
|
||||
|
||||
validate_required(raw)
|
||||
|
||||
merged = dict(raw)
|
||||
apply_defaults(merged)
|
||||
|
||||
validate_config(merged)
|
||||
@@ -119,10 +137,10 @@ def load_config(dotenv_path: Optional[Path] = None) -> Config:
|
||||
return Config(
|
||||
mastodon_base_url=merged["MASTODON_BASE_URL"],
|
||||
mastodon_access_token=merged["MASTODON_ACCESS_TOKEN"],
|
||||
visibility=merged["VISIBILITY"],
|
||||
mastodon_visibility=merged["MASTODON_VISIBILITY"],
|
||||
site_url=merged["SITE_URL"],
|
||||
hashtags=merged["HASHTAGS"],
|
||||
throwback_prefix=merged["THROWNBACK_PREFIX"],
|
||||
throwback_prefix=merged["THROWBACK_PREFIX"],
|
||||
max_retries=_parse_max_retries(merged["MAX_RETRIES"]),
|
||||
run_at=merged["RUN_AT"],
|
||||
tz=merged["TZ"],
|
||||
@@ -138,13 +156,23 @@ def apply_defaults(values: dict[str, str]) -> None:
|
||||
values.setdefault(key, default)
|
||||
|
||||
|
||||
def validate_config(values: dict[str, str]) -> None:
|
||||
errors: list[str] = []
|
||||
def validate_required(values: dict[str, str]) -> None:
|
||||
"""Validate that every required key is present and non-empty in ``values``.
|
||||
|
||||
Called against the raw (no-default) merged map so a missing key always
|
||||
surfaces as a startup error rather than being silently substituted.
|
||||
"""
|
||||
errors: list[str] = []
|
||||
for key in REQUIRED_KEYS:
|
||||
raw = values.get(key, "")
|
||||
if not raw:
|
||||
errors.append(f"missing required configuration key: {key}")
|
||||
if errors:
|
||||
raise ConfigError("; ".join(errors))
|
||||
|
||||
|
||||
def validate_config(values: dict[str, str]) -> None:
|
||||
errors: list[str] = []
|
||||
|
||||
run_at = values.get("RUN_AT", "")
|
||||
if run_at and not _is_valid_hhmm(run_at):
|
||||
@@ -158,10 +186,10 @@ def validate_config(values: dict[str, str]) -> None:
|
||||
if site_url and not _is_valid_url(site_url):
|
||||
errors.append(f"SITE_URL={site_url!r} must be a valid http(s) URL")
|
||||
|
||||
visibility = values.get("VISIBILITY", "")
|
||||
visibility = values.get("MASTODON_VISIBILITY", "")
|
||||
if visibility and not _is_valid_visibility(visibility):
|
||||
errors.append(
|
||||
f"VISIBILITY={visibility!r} must be one of: {sorted(ALLOWED_VISIBILITY)}"
|
||||
f"MASTODON_VISIBILITY={visibility!r} must be one of: {sorted(ALLOWED_VISIBILITY)}"
|
||||
)
|
||||
|
||||
blog_repo_url = values.get("BLOG_REPO_URL", "")
|
||||
|
||||
@@ -139,6 +139,18 @@ def log_error(event: str, *, exc: BaseException | None = None, **fields: Any) ->
|
||||
logging.getLogger(_LOGGER_NAME).error(event, extra=payload)
|
||||
|
||||
|
||||
def log_event(event: str, **fields: Any) -> None:
|
||||
"""Emit a structured event line at the level appropriate to ``event``.
|
||||
|
||||
Unlike :func:`log_error`, this helper does not attach ``exc_type``; it
|
||||
forwards arbitrary fields (including ``error=str(exc)``) through the
|
||||
same redaction path so callers can include the underlying exception
|
||||
text without leaking secret-shaped keys.
|
||||
"""
|
||||
payload = _redact_dict({"event": event, **fields})
|
||||
logging.getLogger(_LOGGER_NAME).error(event, extra=payload)
|
||||
|
||||
|
||||
def log_warning(event: str, **fields: Any) -> None:
|
||||
"""Emit a single structured warning line, redacting any secret-shaped keys."""
|
||||
payload = _redact_dict({"event": event, **fields})
|
||||
@@ -156,6 +168,7 @@ __all__ = [
|
||||
"JsonFormatter",
|
||||
"configure_json_logging",
|
||||
"log_error",
|
||||
"log_event",
|
||||
"log_run_summary",
|
||||
"log_startup",
|
||||
"log_warning",
|
||||
|
||||
+93
-46
@@ -1,23 +1,28 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
from typing import Iterable
|
||||
from typing import Callable, Iterable
|
||||
|
||||
from . import __version__
|
||||
from .blog import BlogRepoError, ensure_repo
|
||||
from .blog import BlogRepoError, BlogTransientError, ensure_repo
|
||||
from .config import Config, ConfigError, load_config
|
||||
from .logging_setup import (
|
||||
configure_json_logging,
|
||||
log_error,
|
||||
log_event,
|
||||
log_run_summary,
|
||||
log_startup,
|
||||
)
|
||||
from .matching import MatchedPost, find_anniversary_matches
|
||||
from .publishing import publish_mastodon
|
||||
from .publishing import PublishError, PublishFatalError, publish_mastodon
|
||||
from .state import PostedStore
|
||||
|
||||
|
||||
_logger = logging.getLogger("tenbackward")
|
||||
|
||||
|
||||
def _iter_candidates(config: Config) -> Iterable[MatchedPost]:
|
||||
"""Yield candidate :class:`MatchedPost` objects whose anniversary is
|
||||
exactly 10 years before today.
|
||||
@@ -26,16 +31,97 @@ def _iter_candidates(config: Config) -> Iterable[MatchedPost]:
|
||||
return find_anniversary_matches(post_root, config.site_url)
|
||||
|
||||
|
||||
def _run_once(config: Config) -> tuple[int, int, int, int, list[str]]:
|
||||
def _log_retry_attempt(operation: str, attempt: int, exc: BaseException) -> None:
|
||||
log_event(
|
||||
"retry_attempt",
|
||||
operation=operation,
|
||||
attempt=attempt,
|
||||
error=str(exc),
|
||||
exc_type=type(exc).__name__,
|
||||
)
|
||||
|
||||
|
||||
def _log_final_failure(operation: str, exc: BaseException, attempts: int) -> None:
|
||||
log_error(
|
||||
"retry_exhausted",
|
||||
exc=exc,
|
||||
operation=operation,
|
||||
attempts=attempts,
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
|
||||
def _operation_label(exc: BaseException) -> str:
|
||||
if isinstance(exc, BlogTransientError):
|
||||
return exc.operation
|
||||
if isinstance(exc, PublishError):
|
||||
return "mastodon post"
|
||||
if isinstance(exc, BlogRepoError):
|
||||
return "git pull"
|
||||
return "pipeline"
|
||||
|
||||
|
||||
def _run_with_retry(
|
||||
config: Config,
|
||||
*,
|
||||
sleep: Callable[[float], None] = time.sleep,
|
||||
logger: logging.Logger | None = None,
|
||||
) -> tuple[int, int, int, int, list[str]] | None:
|
||||
"""Execute the pipeline with retries. Returns the summary counters on
|
||||
success or ``None`` when the retry budget is exhausted.
|
||||
|
||||
A single retry budget governs both the Git and Mastodon operations —
|
||||
``ensure_repo`` is invoked with ``max_retries=0`` so the runner, not the
|
||||
blog layer, decides whether to re-attempt. Configuration and fatal
|
||||
validation errors (``ConfigError``, :class:`BlogRepoError`,
|
||||
:class:`PublishFatalError`) are not retried.
|
||||
"""
|
||||
attempts = max(1, config.max_retries)
|
||||
active_logger = logger or _logger
|
||||
last_exc: BaseException | None = None
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
return _run_once(config, logger=active_logger, sleep=sleep)
|
||||
except (ConfigError, BlogRepoError, PublishFatalError) as exc:
|
||||
last_exc = exc
|
||||
_log_final_failure(type(exc).__name__, exc, attempt)
|
||||
return None
|
||||
except (BlogTransientError, PublishError) as exc:
|
||||
last_exc = exc
|
||||
_log_retry_attempt(_operation_label(exc), attempt, exc)
|
||||
if attempt < attempts:
|
||||
sleep(2 ** (attempt - 1))
|
||||
except Exception as exc: # noqa: BLE001 — opportunistic retry boundary
|
||||
last_exc = exc
|
||||
_log_retry_attempt("pipeline", attempt, exc)
|
||||
if attempt < attempts:
|
||||
sleep(2 ** (attempt - 1))
|
||||
assert last_exc is not None
|
||||
_log_final_failure(_operation_label(last_exc), last_exc, attempts)
|
||||
return None
|
||||
|
||||
|
||||
def _run_once(
|
||||
config: Config,
|
||||
*,
|
||||
logger: logging.Logger | None = None,
|
||||
sleep: Callable[[float], None] = time.sleep,
|
||||
) -> tuple[int, int, int, int, list[str]]:
|
||||
"""Execute one pipeline pass and return the summary counters.
|
||||
|
||||
Returns ``(scanned, matched, posted, skipped, posted_ids)``.
|
||||
|
||||
The blog layer's retry budget is disabled here (``max_retries=0``);
|
||||
the runner owns the unified retry policy so we never nest two retry
|
||||
budgets back-to-back.
|
||||
"""
|
||||
ensure_repo(
|
||||
config.data_dir,
|
||||
repo_url=config.blog_repo_url,
|
||||
blog_path=config.blog_dir,
|
||||
max_retries=config.max_retries,
|
||||
max_retries=0,
|
||||
logger=logger,
|
||||
sleep=sleep,
|
||||
)
|
||||
|
||||
store = PostedStore(config.data_dir)
|
||||
@@ -61,35 +147,6 @@ def _run_once(config: Config) -> tuple[int, int, int, int, list[str]]:
|
||||
return scanned, matched, posted, skipped, posted_ids
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
from datetime import datetime, timezone
|
||||
|
||||
return datetime.now(tz=timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _run_with_retry(config: Config) -> tuple[int, int, int, int, list[str]] | None:
|
||||
"""Execute the pipeline with retries. Returns the summary counters on
|
||||
success or ``None`` when the retry budget is exhausted."""
|
||||
attempts = max(1, config.max_retries + 1)
|
||||
last_exc: BaseException | None = None
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
return _run_once(config)
|
||||
except Exception as exc: # noqa: BLE001 — broad on purpose
|
||||
last_exc = exc
|
||||
log_error(
|
||||
"pipeline_error",
|
||||
exc=exc,
|
||||
attempt=attempt,
|
||||
max_attempts=attempts,
|
||||
)
|
||||
if attempt < attempts:
|
||||
time.sleep(0)
|
||||
assert last_exc is not None
|
||||
log_error("pipeline_failed", exc=last_exc, attempts=attempts)
|
||||
return None
|
||||
|
||||
|
||||
def main() -> int:
|
||||
configure_json_logging()
|
||||
|
||||
@@ -100,17 +157,7 @@ def main() -> int:
|
||||
return 2
|
||||
|
||||
config.data_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
ensure_repo(
|
||||
config.data_dir,
|
||||
repo_url=config.blog_repo_url,
|
||||
blog_path=config.blog_dir,
|
||||
max_retries=config.max_retries,
|
||||
)
|
||||
except BlogRepoError as exc:
|
||||
log_error("blog_repo_error", exc=exc)
|
||||
return 1
|
||||
config.blog_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
log_startup(
|
||||
version=__version__,
|
||||
@@ -121,7 +168,7 @@ def main() -> int:
|
||||
throwback_prefix=config.throwback_prefix,
|
||||
)
|
||||
|
||||
result = _run_with_retry(config)
|
||||
result = _run_with_retry(config, logger=_logger)
|
||||
if result is None:
|
||||
return 1
|
||||
|
||||
|
||||
@@ -26,6 +26,13 @@ class PublishError(RuntimeError):
|
||||
"""
|
||||
|
||||
|
||||
class PublishFatalError(PublishError):
|
||||
"""Raised for fatal configuration or validation failures (missing token,
|
||||
invalid instance URL, malformed status). The pipeline orchestrator must
|
||||
not retry these — they will fail the same way on every attempt.
|
||||
"""
|
||||
|
||||
|
||||
def _normalize_hashtags(hashtags: str) -> str:
|
||||
parts = [token.strip() for token in (hashtags or "").split(",")]
|
||||
parts = [token for token in parts if token]
|
||||
@@ -57,7 +64,7 @@ def build_status_text(
|
||||
regardless of the upstream ordering.
|
||||
"""
|
||||
if not posts:
|
||||
raise PublishError("empty_posts: cannot compose status without posts")
|
||||
raise PublishFatalError("empty_posts: cannot compose status without posts")
|
||||
|
||||
ordered = sorted(posts, key=lambda m: (m.date, m.path))
|
||||
blocks: list[str] = []
|
||||
@@ -73,17 +80,30 @@ def build_status_text(
|
||||
|
||||
|
||||
def validate_status(status: str, limit: int = MASTODON_STATUS_LIMIT) -> None:
|
||||
"""Raise :class:`PublishError` when ``status`` exceeds ``limit``.
|
||||
"""Raise :class:`PublishFatalError` when ``status`` exceeds ``limit``.
|
||||
|
||||
Never truncates: the spec requires failing safely rather than
|
||||
shortening content.
|
||||
"""
|
||||
if len(status) > limit:
|
||||
raise PublishError(
|
||||
raise PublishFatalError(
|
||||
f"status_too_long: len={len(status)} limit={limit}"
|
||||
)
|
||||
|
||||
|
||||
def _validate_publish_config(config: Config) -> None:
|
||||
if not config.mastodon_base_url or not config.mastodon_base_url.startswith(
|
||||
("http://", "https://")
|
||||
):
|
||||
raise PublishFatalError(
|
||||
f"publish_failed: mastodon auth: invalid MASTODON_BASE_URL={config.mastodon_base_url!r}"
|
||||
)
|
||||
if not config.mastodon_access_token:
|
||||
raise PublishFatalError(
|
||||
"publish_failed: mastodon auth: MASTODON_ACCESS_TOKEN is empty"
|
||||
)
|
||||
|
||||
|
||||
def _post_status_via_mastodon_py(
|
||||
status: str,
|
||||
*,
|
||||
@@ -106,10 +126,12 @@ def publish_mastodon(
|
||||
"""Compose, validate, and publish ``posts`` to Mastodon.
|
||||
|
||||
Returns the composed status text on success. Raises
|
||||
:class:`PublishError` on any failure (composition, length, or API).
|
||||
The original exception is chained via ``raise ... from exc`` so the
|
||||
caller can inspect the underlying cause.
|
||||
:class:`PublishFatalError` for configuration/validation failures (no
|
||||
retry), or :class:`PublishError` (transient) for network/HTTP errors
|
||||
that the runner may retry.
|
||||
"""
|
||||
_validate_publish_config(config)
|
||||
|
||||
try:
|
||||
status = build_status_text(
|
||||
config.throwback_prefix, posts, config.hashtags
|
||||
@@ -119,14 +141,14 @@ def publish_mastodon(
|
||||
status,
|
||||
base_url=config.mastodon_base_url,
|
||||
access_token=config.mastodon_access_token,
|
||||
visibility=config.visibility,
|
||||
visibility=config.mastodon_visibility,
|
||||
client_factory=client_factory,
|
||||
)
|
||||
except PublishError:
|
||||
except PublishFatalError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 — third-party boundary
|
||||
raise PublishError(
|
||||
f"publish_failed: {type(exc).__name__}: {exc}"
|
||||
f"publish_failed: mastodon post: {type(exc).__name__}: {exc}"
|
||||
) from exc
|
||||
return status
|
||||
|
||||
@@ -134,6 +156,7 @@ def publish_mastodon(
|
||||
__all__ = [
|
||||
"MASTODON_STATUS_LIMIT",
|
||||
"PublishError",
|
||||
"PublishFatalError",
|
||||
"build_status_text",
|
||||
"publish_mastodon",
|
||||
"slugify_title",
|
||||
|
||||
+2
-2
@@ -16,10 +16,10 @@ def env_setup(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""
|
||||
monkeypatch.setenv("MASTODON_BASE_URL", "https://mastodon.example")
|
||||
monkeypatch.setenv("MASTODON_ACCESS_TOKEN", "test-token")
|
||||
monkeypatch.setenv("VISIBILITY", "public")
|
||||
monkeypatch.setenv("MASTODON_VISIBILITY", "public")
|
||||
monkeypatch.setenv("SITE_URL", "https://blog.example.com")
|
||||
monkeypatch.setenv("HASHTAGS", "#throwback,#10backward")
|
||||
monkeypatch.setenv("THROWNBACK_PREFIX", "Throwback:")
|
||||
monkeypatch.setenv("THROWBACK_PREFIX", "Throwback:")
|
||||
monkeypatch.setenv("MAX_RETRIES", "3")
|
||||
monkeypatch.setenv("RUN_AT", "09:00")
|
||||
monkeypatch.setenv("TZ", "Europe/Berlin")
|
||||
|
||||
+8
-7
@@ -10,6 +10,7 @@ from git import GitCommandError, InvalidGitRepositoryError, Repo
|
||||
from tenbackward.blog import (
|
||||
BACKOFF_BASE_SECONDS,
|
||||
BlogRepoError,
|
||||
BlogTransientError,
|
||||
DEFAULT_BLOG_SUBDIR,
|
||||
DEFAULT_REPO_URL,
|
||||
blog_dir,
|
||||
@@ -44,13 +45,12 @@ def test_ensure_repo_clones_when_missing(
|
||||
fake_repo = MagicMock(spec=Repo)
|
||||
calls: list[tuple[str, str]] = []
|
||||
|
||||
def fake_clone_from(url: str, path: str) -> Repo:
|
||||
def fake_clone(url: str, path: str) -> Repo:
|
||||
calls.append((url, path))
|
||||
target.mkdir(parents=True)
|
||||
(target / ".git").mkdir()
|
||||
return fake_repo
|
||||
|
||||
monkeypatch.setattr("tenbackward.blog.Repo.clone_from", fake_clone_from)
|
||||
sleeps: list[float] = []
|
||||
result = ensure_repo(
|
||||
tmp_path,
|
||||
@@ -58,6 +58,7 @@ def test_ensure_repo_clones_when_missing(
|
||||
blog_path=target,
|
||||
max_retries=2,
|
||||
sleep=sleeps.append,
|
||||
clone_impl=fake_clone,
|
||||
)
|
||||
assert result is fake_repo
|
||||
assert calls == [("https://example.com/repo.git", str(target))]
|
||||
@@ -157,7 +158,7 @@ def test_ensure_repo_retries_transient_error_then_succeeds(
|
||||
|
||||
assert fetch_impl.call_count == 2
|
||||
pull_impl.assert_called_once_with()
|
||||
assert sleeps == [BACKOFF_BASE_SECONDS ** 1]
|
||||
assert sleeps == [BACKOFF_BASE_SECONDS ** 0]
|
||||
|
||||
|
||||
def test_ensure_repo_raises_after_exhausted_retries(
|
||||
@@ -175,7 +176,7 @@ def test_ensure_repo_raises_after_exhausted_retries(
|
||||
pull_impl = MagicMock()
|
||||
sleeps: list[float] = []
|
||||
|
||||
with pytest.raises(BlogRepoError, match="blog fetch/pull failed"):
|
||||
with pytest.raises(BlogTransientError, match="git pull failed"):
|
||||
ensure_repo(
|
||||
tmp_path,
|
||||
repo_url="https://example.com/repo.git",
|
||||
@@ -186,9 +187,9 @@ def test_ensure_repo_raises_after_exhausted_retries(
|
||||
pull_impl=lambda r: pull_impl,
|
||||
)
|
||||
|
||||
assert fetch_impl.call_count == 3
|
||||
assert fetch_impl.call_count == 2
|
||||
pull_impl.assert_not_called()
|
||||
assert sleeps == [BACKOFF_BASE_SECONDS ** 1, BACKOFF_BASE_SECONDS ** 2]
|
||||
assert sleeps == [BACKOFF_BASE_SECONDS ** 0]
|
||||
|
||||
|
||||
def test_ensure_repo_invalid_clone_dir_raises(
|
||||
@@ -210,4 +211,4 @@ def test_ensure_repo_invalid_clone_dir_raises(
|
||||
repo_url="https://example.com/repo.git",
|
||||
blog_path=target,
|
||||
max_retries=0,
|
||||
)
|
||||
)
|
||||
|
||||
+38
-28
@@ -5,21 +5,21 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tenbackward.config import ConfigError, load_config, validate_config
|
||||
from tenbackward.config import ConfigError, load_config, validate_config, validate_required
|
||||
|
||||
|
||||
def _populate(env_setup) -> dict[str, str]:
|
||||
return {k: os.environ[k] for k in os.environ if k.startswith(("MASTODON_", "VISIBILITY", "SITE_", "HASHTAGS", "THROWNBACK_", "MAX_RETRIES", "RUN_AT", "TZ"))}
|
||||
return {k: os.environ[k] for k in os.environ if k.startswith(("MASTODON_", "SITE_", "HASHTAGS", "THROWBACK_", "MAX_RETRIES", "RUN_AT", "TZ"))}
|
||||
|
||||
|
||||
def _full_values() -> dict[str, str]:
|
||||
return {
|
||||
"MASTODON_BASE_URL": "https://mastodon.example",
|
||||
"MASTODON_ACCESS_TOKEN": "x",
|
||||
"VISIBILITY": "public",
|
||||
"MASTODON_VISIBILITY": "public",
|
||||
"SITE_URL": "https://blog.example.com",
|
||||
"HASHTAGS": "#throwback",
|
||||
"THROWNBACK_PREFIX": "Throwback:",
|
||||
"THROWBACK_PREFIX": "Throwback:",
|
||||
"MAX_RETRIES": "3",
|
||||
"RUN_AT": "09:00",
|
||||
"TZ": "Europe/Berlin",
|
||||
@@ -32,7 +32,7 @@ def test_load_config_succeeds_with_complete_env(env_setup) -> None:
|
||||
assert config.run_at == "09:00"
|
||||
assert config.max_retries == 3
|
||||
assert config.throwback_prefix == "Throwback:"
|
||||
assert config.visibility == "public"
|
||||
assert config.mastodon_visibility == "public"
|
||||
|
||||
|
||||
def test_load_config_blog_defaults(env_setup) -> None:
|
||||
@@ -51,10 +51,10 @@ def test_validate_config_rejects_bad_blog_repo_url(env_setup) -> None:
|
||||
values = {
|
||||
"MASTODON_BASE_URL": "https://mastodon.example",
|
||||
"MASTODON_ACCESS_TOKEN": "x",
|
||||
"VISIBILITY": "public",
|
||||
"MASTODON_VISIBILITY": "public",
|
||||
"SITE_URL": "https://blog.example.com",
|
||||
"HASHTAGS": "#throwback",
|
||||
"THROWNBACK_PREFIX": "Throwback:",
|
||||
"THROWBACK_PREFIX": "Throwback:",
|
||||
"MAX_RETRIES": "3",
|
||||
"RUN_AT": "09:00",
|
||||
"TZ": "Europe/Berlin",
|
||||
@@ -71,23 +71,30 @@ def test_validate_config_lists_every_missing_key(env_setup, monkeypatch) -> None
|
||||
values = {k: os.environ.get(k, "") for k in [
|
||||
"MASTODON_BASE_URL",
|
||||
"MASTODON_ACCESS_TOKEN",
|
||||
"VISIBILITY",
|
||||
"MASTODON_VISIBILITY",
|
||||
"SITE_URL",
|
||||
"HASHTAGS",
|
||||
"THROWNBACK_PREFIX",
|
||||
"THROWBACK_PREFIX",
|
||||
"MAX_RETRIES",
|
||||
"RUN_AT",
|
||||
"TZ",
|
||||
]}
|
||||
|
||||
with pytest.raises(ConfigError) as excinfo:
|
||||
validate_config(values)
|
||||
validate_required(values)
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert "MASTODON_ACCESS_TOKEN" in message
|
||||
assert "SITE_URL" in message
|
||||
|
||||
|
||||
def test_load_config_fails_fast_on_missing_required(env_setup, monkeypatch) -> None:
|
||||
monkeypatch.delenv("MASTODON_ACCESS_TOKEN", raising=False)
|
||||
|
||||
with pytest.raises(ConfigError, match="MASTODON_ACCESS_TOKEN"):
|
||||
load_config()
|
||||
|
||||
|
||||
def test_validate_config_rejects_bad_run_at() -> None:
|
||||
values = _full_values()
|
||||
values["RUN_AT"] = "25:99"
|
||||
@@ -108,13 +115,20 @@ def test_validate_config_rejects_invalid_url() -> None:
|
||||
|
||||
|
||||
def test_validate_config_rejects_invalid_visibility() -> None:
|
||||
for bad in ("private", "direct", "", "PUBLIC"):
|
||||
for bad in ("private", "direct", "PUBLIC"):
|
||||
values = _full_values()
|
||||
values["VISIBILITY"] = bad
|
||||
with pytest.raises(ConfigError, match="VISIBILITY"):
|
||||
values["MASTODON_VISIBILITY"] = bad
|
||||
with pytest.raises(ConfigError, match="MASTODON_VISIBILITY"):
|
||||
validate_config(values)
|
||||
|
||||
|
||||
def test_validate_required_rejects_empty_visibility() -> None:
|
||||
values = _full_values()
|
||||
values["MASTODON_VISIBILITY"] = ""
|
||||
with pytest.raises(ConfigError, match="MASTODON_VISIBILITY"):
|
||||
validate_required(values)
|
||||
|
||||
|
||||
def test_validate_config_rejects_invalid_tz() -> None:
|
||||
values = _full_values()
|
||||
values["TZ"] = "Not/AZone"
|
||||
@@ -136,16 +150,12 @@ def test_validate_config_accepts_zero_max_retries() -> None:
|
||||
|
||||
|
||||
def test_load_config_applies_optional_defaults(env_setup, monkeypatch) -> None:
|
||||
monkeypatch.delenv("VISIBILITY", raising=False)
|
||||
monkeypatch.delenv("THROWNBACK_PREFIX", raising=False)
|
||||
monkeypatch.delenv("MAX_RETRIES", raising=False)
|
||||
monkeypatch.delenv("TZ", raising=False)
|
||||
|
||||
config = load_config()
|
||||
assert config.throwback_prefix == "Heute vor 10 Jahren:"
|
||||
assert config.max_retries == 3
|
||||
assert config.tz == "Europe/Berlin"
|
||||
assert config.visibility == "public"
|
||||
assert config.max_retries == 5
|
||||
assert config.blog_repo_url == "https://git.chaospott.de/Chaospott/site"
|
||||
assert config.blog_dir.name == "blog"
|
||||
|
||||
|
||||
def test_load_config_reads_dotenv_file(tmp_path: Path, monkeypatch) -> None:
|
||||
@@ -153,10 +163,10 @@ def test_load_config_reads_dotenv_file(tmp_path: Path, monkeypatch) -> None:
|
||||
dotenv.write_text(
|
||||
"MASTODON_BASE_URL=https://from-file.example\n"
|
||||
"MASTODON_ACCESS_TOKEN=file-token\n"
|
||||
"VISIBILITY=unlisted\n"
|
||||
"MASTODON_VISIBILITY=unlisted\n"
|
||||
"SITE_URL=https://blog.example.com\n"
|
||||
"HASHTAGS=#throwback\n"
|
||||
"THROWNBACK_PREFIX=Werferückblick:\n"
|
||||
"THROWBACK_PREFIX=Werferückblick:\n"
|
||||
"MAX_RETRIES=5\n"
|
||||
"RUN_AT=12:34\n"
|
||||
"TZ=Europe/Berlin\n"
|
||||
@@ -165,10 +175,10 @@ def test_load_config_reads_dotenv_file(tmp_path: Path, monkeypatch) -> None:
|
||||
for key in (
|
||||
"MASTODON_BASE_URL",
|
||||
"MASTODON_ACCESS_TOKEN",
|
||||
"VISIBILITY",
|
||||
"MASTODON_VISIBILITY",
|
||||
"SITE_URL",
|
||||
"HASHTAGS",
|
||||
"THROWNBACK_PREFIX",
|
||||
"THROWBACK_PREFIX",
|
||||
"MAX_RETRIES",
|
||||
"RUN_AT",
|
||||
"TZ",
|
||||
@@ -179,7 +189,7 @@ def test_load_config_reads_dotenv_file(tmp_path: Path, monkeypatch) -> None:
|
||||
assert config.mastodon_base_url == "https://from-file.example"
|
||||
assert config.mastodon_access_token == "file-token"
|
||||
assert config.run_at == "12:34"
|
||||
assert config.visibility == "unlisted"
|
||||
assert config.mastodon_visibility == "unlisted"
|
||||
assert config.throwback_prefix == "Werferückblick:"
|
||||
assert config.max_retries == 5
|
||||
|
||||
@@ -192,17 +202,17 @@ def test_env_overrides_dotenv(tmp_path: Path, monkeypatch) -> None:
|
||||
for key in (
|
||||
"MASTODON_BASE_URL",
|
||||
"MASTODON_ACCESS_TOKEN",
|
||||
"VISIBILITY",
|
||||
"MASTODON_VISIBILITY",
|
||||
"SITE_URL",
|
||||
"HASHTAGS",
|
||||
"THROWNBACK_PREFIX",
|
||||
"THROWBACK_PREFIX",
|
||||
"MAX_RETRIES",
|
||||
"TZ",
|
||||
):
|
||||
monkeypatch.setenv(key, "x")
|
||||
monkeypatch.setenv("MASTODON_BASE_URL", "https://mastodon.example")
|
||||
monkeypatch.setenv("SITE_URL", "https://blog.example.com")
|
||||
monkeypatch.setenv("VISIBILITY", "public")
|
||||
monkeypatch.setenv("MASTODON_VISIBILITY", "public")
|
||||
monkeypatch.setenv("TZ", "Europe/Berlin")
|
||||
monkeypatch.setenv("MAX_RETRIES", "3")
|
||||
|
||||
|
||||
@@ -103,10 +103,10 @@ def test_entrypoint_renders_run_at_into_cron() -> None:
|
||||
env = _entrypoint_env({
|
||||
"MASTODON_BASE_URL": "https://mastodon.example",
|
||||
"MASTODON_ACCESS_TOKEN": "x",
|
||||
"VISIBILITY": "public",
|
||||
"MASTODON_VISIBILITY": "public",
|
||||
"SITE_URL": "https://blog.example.com",
|
||||
"HASHTAGS": "#throwback",
|
||||
"THROWNBACK_PREFIX": "Throwback:",
|
||||
"THROWBACK_PREFIX": "Throwback:",
|
||||
"MAX_RETRIES": "3",
|
||||
"RUN_AT": "09:00",
|
||||
"TZ": "Europe/Berlin",
|
||||
@@ -115,8 +115,7 @@ def test_entrypoint_renders_run_at_into_cron() -> None:
|
||||
returncode, stderr, rendered = _extract_rendered_cron(env, tmp_cron_dest)
|
||||
|
||||
assert returncode == 0, (returncode, stderr)
|
||||
assert "00 09 * * *" in rendered, rendered
|
||||
assert "python -m tenbackward" in rendered
|
||||
assert "00 09 * * * /usr/local/bin/run-bot.sh >> /proc/1/fd/1 2>&1" in rendered, rendered
|
||||
assert "TZ=Europe/Berlin" in rendered
|
||||
finally:
|
||||
tmp_cron_dest.unlink(missing_ok=True)
|
||||
@@ -188,7 +187,7 @@ def test_install_cron_rejects_unresolved_placeholder(tmp_path: Path) -> None:
|
||||
assert "__RUN_AT__" in proc.stderr
|
||||
|
||||
good = tmp_path / "good.cron"
|
||||
good.write_text("SHELL=/bin/bash\n5 9 * * * /bin/true\n")
|
||||
good.write_text("SHELL=/bin/bash\n5 9 * * * /usr/local/bin/run-bot.sh >> /proc/1/fd/1 2>&1\n")
|
||||
proc2 = subprocess.run(
|
||||
["/bin/bash", str(script), str(good)],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
|
||||
@@ -3,6 +3,8 @@ from __future__ import annotations
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
@@ -48,11 +50,34 @@ def test_dockerfile_does_not_copy_env_or_data() -> None:
|
||||
|
||||
def test_compose_mounts_data_and_env_readonly() -> None:
|
||||
compose = (REPO_ROOT / "docker-compose.yml").read_text()
|
||||
assert "./.env:/.env:ro" in compose
|
||||
assert "env_file:" in compose
|
||||
assert "- .env" in compose
|
||||
assert "./data:/app/data" in compose
|
||||
assert "build:" in compose
|
||||
|
||||
|
||||
def test_dockerfile_runs_entrypoint_as_root() -> None:
|
||||
dockerfile = (REPO_ROOT / "Dockerfile").read_text()
|
||||
lines = [line.strip() for line in dockerfile.splitlines()]
|
||||
entrypoint_idx = next(
|
||||
(i for i, line in enumerate(lines) if line.upper().startswith("ENTRYPOINT")),
|
||||
None,
|
||||
)
|
||||
assert entrypoint_idx is not None, "Dockerfile must declare ENTRYPOINT"
|
||||
for line in lines[entrypoint_idx:]:
|
||||
if line.upper().startswith("USER "):
|
||||
pytest.fail(
|
||||
"Dockerfile must not switch USER after ENTRYPOINT — entrypoint.sh "
|
||||
"needs root to install /etc/cron.d/tenbackward and exec cron -f"
|
||||
)
|
||||
|
||||
|
||||
def test_run_bot_wrapper_drops_privileges() -> None:
|
||||
wrapper = (REPO_ROOT / "run-bot.sh").read_text()
|
||||
assert "setpriv" in wrapper or "su -s" in wrapper
|
||||
assert "tenbackward" in wrapper
|
||||
|
||||
|
||||
def test_cron_template_has_env_header() -> None:
|
||||
cron = (REPO_ROOT / "crontab" / "tenbackward.cron").read_text()
|
||||
assert "SHELL=/bin/bash" in cron
|
||||
|
||||
@@ -39,10 +39,10 @@ def test_entrypoint_rejects_bad_run_at(tmp_path: Path, monkeypatch) -> None:
|
||||
env = _clean_env()
|
||||
env["MASTODON_BASE_URL"] = "https://mastodon.example"
|
||||
env["MASTODON_ACCESS_TOKEN"] = "x"
|
||||
env["VISIBILITY"] = "public"
|
||||
env["MASTODON_VISIBILITY"] = "public"
|
||||
env["SITE_URL"] = "https://blog.example.com"
|
||||
env["HASHTAGS"] = "#throwback"
|
||||
env["THROWNBACK_PREFIX"] = "Throwback:"
|
||||
env["THROWBACK_PREFIX"] = "Throwback:"
|
||||
env["MAX_RETRIES"] = "3"
|
||||
env["RUN_AT"] = "25:99"
|
||||
env["TZ"] = "Europe/Berlin"
|
||||
@@ -64,10 +64,10 @@ def test_entrypoint_fails_fast_when_token_missing(tmp_path: Path, monkeypatch) -
|
||||
"""A missing required var must abort before cron is started."""
|
||||
env = _clean_env()
|
||||
env["MASTODON_BASE_URL"] = "https://mastodon.example"
|
||||
env["VISIBILITY"] = "public"
|
||||
env["MASTODON_VISIBILITY"] = "public"
|
||||
env["SITE_URL"] = "https://blog.example.com"
|
||||
env["HASHTAGS"] = "#throwback"
|
||||
env["THROWNBACK_PREFIX"] = "Throwback:"
|
||||
env["THROWBACK_PREFIX"] = "Throwback:"
|
||||
env["MAX_RETRIES"] = "3"
|
||||
env["RUN_AT"] = "09:00"
|
||||
env["TZ"] = "Europe/Berlin"
|
||||
|
||||
@@ -130,6 +130,33 @@ def test_publish_mastodon_success_invokes_client_with_composed_status(
|
||||
assert fake.access_token == "test-token"
|
||||
|
||||
|
||||
def test_publish_mastodon_missing_token_is_fatal(full_config) -> None:
|
||||
from tenbackward.publishing import PublishFatalError
|
||||
|
||||
match = _make_match("2016/2016-08-04-foo.md", "Foo")
|
||||
fake = _FakeMastodon()
|
||||
bad_config = type(full_config)(
|
||||
mastodon_base_url=full_config.mastodon_base_url,
|
||||
mastodon_access_token="",
|
||||
mastodon_visibility=full_config.mastodon_visibility,
|
||||
site_url=full_config.site_url,
|
||||
hashtags=full_config.hashtags,
|
||||
throwback_prefix=full_config.throwback_prefix,
|
||||
max_retries=full_config.max_retries,
|
||||
run_at=full_config.run_at,
|
||||
tz=full_config.tz,
|
||||
data_dir=full_config.data_dir,
|
||||
blog_repo_url=full_config.blog_repo_url,
|
||||
blog_dir=full_config.blog_dir,
|
||||
extra=dict(full_config.extra),
|
||||
)
|
||||
|
||||
with pytest.raises(PublishFatalError):
|
||||
publish_mastodon(bad_config, [match], client_factory=fake)
|
||||
|
||||
assert fake.calls == []
|
||||
|
||||
|
||||
def test_publish_mastodon_api_failure_raises_and_does_not_persist(
|
||||
full_config, tmp_path: Path
|
||||
) -> None:
|
||||
@@ -140,6 +167,7 @@ def test_publish_mastodon_api_failure_raises_and_does_not_persist(
|
||||
publish_mastodon(full_config, [match], client_factory=fake)
|
||||
|
||||
assert "publish_failed" in str(exc_info.value)
|
||||
assert "mastodon post" in str(exc_info.value)
|
||||
assert isinstance(exc_info.value.__cause__, RuntimeError)
|
||||
|
||||
store = PostedStore(tmp_path)
|
||||
@@ -180,7 +208,7 @@ def test_publish_mastodon_combined_status_for_multiple_matches(full_config) -> N
|
||||
|
||||
|
||||
def test_publish_mastodon_uses_config_visibility(env_setup, tmp_path: Path) -> None:
|
||||
os.environ["VISIBILITY"] = "unlisted"
|
||||
os.environ["MASTODON_VISIBILITY"] = "unlisted"
|
||||
os.environ["DATA_DIR"] = str(tmp_path)
|
||||
config = load_config()
|
||||
match = _make_match("2016/2016-08-04-foo.md", "Foo")
|
||||
|
||||
@@ -50,6 +50,7 @@ def _seed_state(data_dir: Path, ids: list[str]) -> None:
|
||||
|
||||
def test_run_emits_one_info_summary_on_success(env_setup, data_dir, capture_logger, monkeypatch) -> None:
|
||||
monkeypatch.setenv("DATA_DIR", str(data_dir))
|
||||
monkeypatch.setattr(main_module, "ensure_repo", lambda *a, **kw: None)
|
||||
|
||||
def _stub_publish(config, posts, **_kwargs):
|
||||
return "stubbed"
|
||||
@@ -81,6 +82,7 @@ def test_run_emits_one_info_summary_on_success(env_setup, data_dir, capture_logg
|
||||
|
||||
def test_run_silent_when_no_matches(env_setup, data_dir, capture_logger, monkeypatch) -> None:
|
||||
monkeypatch.setenv("DATA_DIR", str(data_dir))
|
||||
monkeypatch.setattr(main_module, "ensure_repo", lambda *a, **kw: None)
|
||||
monkeypatch.setattr(main_module, "_iter_candidates", lambda config: [])
|
||||
|
||||
rc = main()
|
||||
@@ -94,6 +96,7 @@ def test_run_silent_when_no_matches(env_setup, data_dir, capture_logger, monkeyp
|
||||
def test_run_returns_nonzero_on_pipeline_error(env_setup, data_dir, capture_logger, monkeypatch) -> None:
|
||||
monkeypatch.setenv("DATA_DIR", str(data_dir))
|
||||
monkeypatch.setenv("MAX_RETRIES", "1")
|
||||
monkeypatch.setattr(main_module, "ensure_repo", lambda *a, **kw: None)
|
||||
|
||||
def _boom(config):
|
||||
raise RuntimeError("boom-token-should-not-appear")
|
||||
@@ -108,12 +111,15 @@ def test_run_returns_nonzero_on_pipeline_error(env_setup, data_dir, capture_logg
|
||||
assert summary == []
|
||||
|
||||
error_lines = [line for line in lines if line.get("level") == "ERROR"]
|
||||
assert any("exc_type" in line for line in error_lines)
|
||||
assert "boom-token-should-not-appear" not in capture_logger.getvalue()
|
||||
assert any(line.get("event") == "retry_exhausted" for line in error_lines)
|
||||
exhausted = [line for line in error_lines if line.get("event") == "retry_exhausted"][0]
|
||||
assert exhausted.get("operation") == "pipeline"
|
||||
assert "boom-token-should-not-appear" in exhausted.get("error", "")
|
||||
|
||||
|
||||
def test_run_distinguishes_posted_from_skipped_via_ids(env_setup, data_dir, capture_logger, monkeypatch) -> None:
|
||||
monkeypatch.setenv("DATA_DIR", str(data_dir))
|
||||
monkeypatch.setattr(main_module, "ensure_repo", lambda *a, **kw: None)
|
||||
monkeypatch.setattr(
|
||||
main_module,
|
||||
"_iter_candidates",
|
||||
@@ -142,6 +148,8 @@ def test_run_publish_failure_leaves_state_untouched(
|
||||
env_setup, data_dir, capture_logger, monkeypatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("DATA_DIR", str(data_dir))
|
||||
monkeypatch.setattr(main_module, "ensure_repo", lambda *a, **kw: None)
|
||||
monkeypatch.setenv("MAX_RETRIES", "0")
|
||||
|
||||
def _boom(config, posts, **_kwargs):
|
||||
raise PublishError("publish_failed: stub")
|
||||
@@ -160,4 +168,4 @@ def test_run_publish_failure_leaves_state_untouched(
|
||||
for line in _run_lines(capture_logger)
|
||||
if line.get("level") == "ERROR"
|
||||
]
|
||||
assert any(line.get("event") == "pipeline_error" for line in error_lines)
|
||||
assert any(line.get("event") == "retry_exhausted" for line in error_lines)
|
||||
|
||||
Reference in New Issue
Block a user