Compare commits

9 Commits

48 changed files with 4768 additions and 0 deletions
+22
View File
@@ -0,0 +1,22 @@
.env
.env.example
data/
.git/
.venv/
venv/
__pycache__/
*.pyc
*.pyo
*.egg-info/
.pytest_cache/
.coverage
tests/
.kilo/
kilo.json
setup.sh
README.md
.dockerignore
.gitignore
.idea/
.vscode/
*.log
+15
View File
@@ -0,0 +1,15 @@
# 10Backward configuration — placeholder values only. NEVER use real credentials here.
MASTODON_BASE_URL=https://mastodon.example
MASTODON_ACCESS_TOKEN=replace-me
MASTODON_VISIBILITY=public
SITE_URL=https://blog.example.com
HASHTAGS=#throwback,#10backward
THROWBACK_PREFIX=Heute vor 10 Jahren:
MAX_RETRIES=5
RUN_AT=09:00
TZ=Europe/Berlin
# Optional — Jekyll blog clone source. Defaults point at the Chaospott public repo.
# BLOG_REPO_URL=https://git.chaospott.de/Chaospott/site
# BLOG_DIR=blog
+18
View File
@@ -0,0 +1,18 @@
.env
.env.*
!/.env.example
data/
__pycache__/
*.pyc
*.pyo
*.egg-info/
.venv/
venv/
.pytest_cache/
.coverage
htmlcov/
*.log
.DS_Store
.idea/
.vscode/
.kilo/
+46
View File
@@ -0,0 +1,46 @@
FROM python:3.11-slim
ENV DEBIAN_FRONTEND=noninteractive \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
TZ=Europe/Berlin
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
cron \
git \
ca-certificates \
tzdata \
util-linux \
&& rm -rf /var/lib/apt/lists/*
RUN groupadd --system bot \
&& useradd --system --gid bot --home /app --shell /usr/sbin/nologin bot
WORKDIR /app
RUN mkdir -p /app/data \
&& chown -R bot:bot /app
COPY requirements.txt /app/requirements.txt
RUN pip install --no-cache-dir --upgrade pip \
&& pip install --no-cache-dir -r /app/requirements.txt
COPY --chown=bot:bot crontab/ /app/crontab/
COPY --chown=bot:bot entrypoint.sh /app/entrypoint.sh
COPY --chown=bot:bot run-bot.sh /app/run-bot.sh
COPY --chown=root:root run-bot.sh /usr/local/bin/run-bot.sh
COPY --chown=bot:bot src/ /app/src/
RUN chmod 0755 /app/entrypoint.sh \
&& chmod 0755 /app/crontab/install-cron.sh \
&& chmod 0755 /app/run-bot.sh \
&& chmod 0755 /usr/local/bin/run-bot.sh \
&& chmod 0644 /app/crontab/tenbackward.cron
ENV PYTHONPATH=/app/src
# Note: entrypoint.sh runs as root so it can install /etc/cron.d/tenbackward
# and exec cron -f. The cron-launched wrapper drops to the unprivileged
# `bot` user before invoking Python.
ENTRYPOINT ["/app/entrypoint.sh"]
+48
View File
@@ -1,2 +1,50 @@
# 10Backward
Daily Mastodon throwback bot scaffolded with Docker Compose.
The container runs a cron daemon that invokes the Python entrypoint once per
day at the configured `RUN_AT` (Europe/Berlin default). The actual blog
clone / post pipeline is intentionally stubbed in this scaffold and is to be
implemented in a follow-up job — this repository currently ships:
- Pinned Python dependency manifest.
- `Dockerfile` (python:3.11-slim + cron + git + ca-certificates + tzdata).
- `docker-compose.yml` building locally, mounting `.env` read-only and
persisting `./data` for the blog clone and `posted.json` state.
- `entrypoint.sh` that validates required env vars and fails clearly on
startup before launching `cron -f` in the foreground.
- Safe `.env.example` with placeholder credentials only.
## Build & run
```bash
cp .env.example .env # fill in real credentials locally
docker compose build
docker compose up -d
docker compose logs -f bot
```
`./data` is persisted across restarts on the host. The blog clone (added in a
follow-up job) will live in `./data/blog/`, and posted-state in
`./data/posted.json`.
## Configuration
| Variable | Required | Default | Notes |
| --- | --- | --- | --- |
| `MASTODON_BASE_URL` | yes | — | e.g. `https://mastodon.social` |
| `MASTODON_ACCESS_TOKEN` | yes | — | never commit |
| `SITE_URL` | yes | — | source blog URL (HTTPS) |
| `HASHTAGS` | yes | — | comma-separated, e.g. `#throwback,#10backward` |
| `RUN_AT` | yes | `09:00` | `HH:MM` in `TZ` |
| `MASTODON_VISIBILITY` | no | `public` | `public`, `unlisted`, `private`, `direct` |
| `THROWBACK_PREFIX` | no | `Throwback:` | German-localizable |
| `RETRY_COUNT` | no | `3` | post retries |
| `TZ` | no | `Europe/Berlin` | any IANA timezone |
## Tests
```bash
pip install -r requirements-dev.txt
pytest
```
Executable
+37
View File
@@ -0,0 +1,37 @@
#!/bin/bash
set -euo pipefail
export DEBIAN_FRONTEND=noninteractive
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
cd "${SCRIPT_DIR}"
if [ -w /usr/local/bin ] && [ ! -e /usr/local/bin/build.sh ]; then
ln -s "${SCRIPT_DIR}/build.sh" /usr/local/bin/build.sh || true
fi
apt-get update
apt-get install -y --no-install-recommends \
bash \
git \
ca-certificates \
python3 \
python3-pip \
python3-venv \
tzdata
rm -rf /var/lib/apt/lists/*
PYTHON_BIN="$(command -v python3)"
PIP_BIN="$(command -v pip3 || true)"
if [ -z "${PIP_BIN}" ] && [ -n "${PYTHON_BIN}" ]; then
PIP_BIN="${PYTHON_BIN} -m pip"
fi
if [ -n "${PIP_BIN}" ]; then
${PIP_BIN} install --no-cache-dir --upgrade pip >/dev/null 2>&1 || true
${PIP_BIN} install --no-cache-dir --break-system-packages -r /repo/requirements-dev.txt || \
${PIP_BIN} install --no-cache-dir -r /repo/requirements-dev.txt
fi
cd /repo
PYTHONPATH=/repo/src pytest -q
+33
View File
@@ -0,0 +1,33 @@
#!/bin/bash
set -euo pipefail
CRON_FILE="${1:-/etc/cron.d/tenbackward}"
if [ ! -f "${CRON_FILE}" ]; then
echo "install-cron: cron file ${CRON_FILE} does not exist" >&2
exit 1
fi
if grep -q '__RUN_AT__' "${CRON_FILE}"; then
echo "install-cron: refusing to install cron file with unresolved __RUN_AT__ placeholder" >&2
exit 1
fi
chmod 0644 "${CRON_FILE}"
if ! head -n 1 "${CRON_FILE}" | grep -qE '^[A-Z_]+='; then
echo "install-cron: ${CRON_FILE} must start with a cron env line (e.g. SHELL=/bin/bash)" >&2
exit 1
fi
if ! grep -q '/usr/local/bin/run-bot.sh' "${CRON_FILE}"; then
echo "install-cron: ${CRON_FILE} must invoke /usr/local/bin/run-bot.sh" >&2
exit 1
fi
if ! grep -qE '/proc/1/fd/1' "${CRON_FILE}"; then
echo "install-cron: ${CRON_FILE} must redirect output to /proc/1/fd/1" >&2
exit 1
fi
echo "install-cron: ${CRON_FILE} installed (mode 0644)"
+2
View File
@@ -0,0 +1,2 @@
SHELL=/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+22
View File
@@ -0,0 +1,22 @@
services:
bot:
build:
context: .
dockerfile: Dockerfile
image: tenbackward-bot:latest
container_name: tenbackward-bot
restart: unless-stopped
env_file:
- .env
environment:
MASTODON_BASE_URL: ${MASTODON_BASE_URL:-https://mastodon.example}
MASTODON_ACCESS_TOKEN: ${MASTODON_ACCESS_TOKEN:-replace-me}
MASTODON_VISIBILITY: ${MASTODON_VISIBILITY:-public}
SITE_URL: ${SITE_URL:-https://blog.example.com}
HASHTAGS: ${HASHTAGS:-#throwback,#10backward}
THROWBACK_PREFIX: ${THROWBACK_PREFIX:-Throwback:}
MAX_RETRIES: ${MAX_RETRIES:-5}
RUN_AT: ${RUN_AT:-09:00}
TZ: ${TZ:-Europe/Berlin}
volumes:
- ./data:/app/data
+81
View File
@@ -0,0 +1,81 @@
---
type: architecture
title: Anniversary Matching
description: Rules and file boundaries used to discover Jekyll posts whose publication anniversary is exactly ten years before the current date.
tags: [matching, jekyll, anniversary, posts]
timestamp: 2026-08-04T19:45:00Z
---
# Purpose
`tenbackward.matching` scans the synchronized blog at
`<blog_dir>/_posts/blog` and returns posts whose date is exactly ten years
before the current date. The current date is evaluated in the configured
IANA timezone, defaulting to `Europe/Berlin` in the matcher.
# Discovery Rules
1. Walk every Markdown file below the configured post root recursively.
2. Parse YAML front matter when present.
3. Require a filename stem shaped as `YYYY-MM-DD-slug` with a valid calendar
date and non-empty slug.
4. Prefer a valid front matter `date`; otherwise use the date in the filename.
5. Match the post year to `today.year - 10` and the month/day to today.
6. Skip posts with `published: false`.
7. Return a `MatchedPost` containing the relative path, title, date, and
canonical site URL. If front matter has no title, use the filename slug.
# Leap-Day Behavior
A February 29 post matches February 29 when the current year is a leap year.
In a non-leap year it matches March 1, but only when the post's year is
exactly ten years before the current year (`today.year 10`). Because
`today.year 10` is never itself a leap year whenever today is a leap year
(`10 mod 4 == 2`), the Feb 29 ↔ Feb 29 branch is reached independently of
the year-equality check: any Feb 29 post from a prior leap year is a
candidate, and its canonical URL keeps the original `YYYY/02/29/<slug>/`
path.
# Output and Integration
`find_anniversary_matches()` returns `MatchedPost` values for callers that
need metadata. `iter_anniversary_paths()` yields only relative paths; the
pipeline uses those paths as deduplication identifiers in the
[system state store](/architecture/system-overview.md) and persists new IDs in
`posted.json`.
Unreadable files, malformed front matter, invalid filenames, unpublished
posts, and unexpected per-file errors are skipped with structured warning
events rather than aborting the full scan.
# Key Files
| Path | Responsibility |
|---|---|
| `/repo/src/tenbackward/matching.py` | Date parsing, Jekyll front matter handling, anniversary rules, URL construction, and recursive scanning. |
| `/repo/src/tenbackward/main.py` | Supplies the blog post root and site URL, then applies state deduplication. |
| `/repo/tests/test_matching.py` | Covers filename/front matter parsing, date matching, unpublished posts, malformed files, URL output, and leap-day behavior. |
| `/repo/src/tenbackward/blog.py` | Ensures the source repository is cloned or fast-forwarded before matching. |
# Examples
## Matching post
On 2026-08-04, a file named
`_posts/blog/2016/2016-08-04-release.md` is eligible. Its identifier is
`2016/2016-08-04-release.md` and its generated URL is:
```text
https://example.com/2016/08/04/release/
```
## Skipped post
A file with `published: false`, an invalid date, or a non-matching
month/day is not yielded and does not enter `posted.json`.
# Related
* [Pipeline Runner](/architecture/pipeline-runner.md)
* [System Architecture](/architecture/system-overview.md)
* [Daily Run Guide](/guides/daily-run.md)
+97
View File
@@ -0,0 +1,97 @@
---
type: api
title: Config Schema
description: Required/optional environment variables, validation rules, and the typed Config dataclass returned by tenbackward.config.load_config.
tags: [config, env, schema]
timestamp: 2026-08-04T17:51:00Z
---
# Purpose
`tenbackward.config.load_config()` is the single boundary that turns
process environment + optional `.env` file into a typed `Config`
dataclass. Job 1083 changes every variable that ships in
`.env.example` so that it is **explicitly required** at every layer
(`entrypoint.sh`, `config.validate_config`, and the rendered cron).
# Required Environment Variables
The following keys must be present and non-empty in the merged dotenv/process environment before application defaults are applied.
| Key | Purpose | Validation |
|---|---|---|
| `MASTODON_BASE_URL` | Mastodon instance to post against. | Valid `http` or `https` URL. |
| `MASTODON_ACCESS_TOKEN` | OAuth token used by the Mastodon client. | Required string; redacted in logs. |
| `MASTODON_VISIBILITY` | Post visibility. | `public` or `unlisted`. |
| `SITE_URL` | Source blog URL. | Valid `http` or `https` URL. |
| `HASHTAGS` | Comma-separated hashtags. | Required string. |
| `THROWBACK_PREFIX` | Prefix prepended to each status. | Required string. |
| `MAX_RETRIES` | Maximum number of pipeline attempts. | Non-negative integer; defaults to `5` when absent. |
| `RUN_AT` | Daily cron fire time. | `HH:MM` 24-hour format. |
| `TZ` | Cron and application timezone. | Valid IANA timezone. |
`BLOG_REPO_URL` and `BLOG_DIR` are optional and receive defaults after this required-key check. `DATA_DIR` is read separately from the process environment and defaults to `/app/data`.
# Defaults
Defaults are applied only after required-key validation. The current defaults are:
| Default key | Default value |
|---|---|
| `MAX_RETRIES` | `5` |
| `BLOG_REPO_URL` | `https://git.chaospott.de/Chaospott/site` |
| `BLOG_DIR` | `blog` (resolved relative to `data_dir`) |
Application-facing required values are not silently supplied by `config.py`; the container environment may still provide `TZ=Europe/Berlin` and Compose may provide development fallback values.
# `Config` Dataclass
| Field | Type | Source |
|-------------------------|-----------|----------------------------------------------|
| `mastodon_base_url` | `str` | `MASTODON_BASE_URL` |
| `mastodon_access_token` | `str` | `MASTODON_ACCESS_TOKEN` |
| `mastodon_visibility` | `str` | `MASTODON_VISIBILITY` |
| `site_url` | `str` | `SITE_URL` |
| `hashtags` | `str` | `HASHTAGS` |
| `throwback_prefix` | `str` | `THROWBACK_PREFIX` |
| `max_retries` | `int` | `MAX_RETRIES` (parsed as non-negative int) |
| `run_at` | `str` | `RUN_AT` |
| `tz` | `str` | `TZ` |
| `data_dir` | `Path` | `DATA_DIR`, default `/app/data` |
| `blog_repo_url` | `str` | `BLOG_REPO_URL` |
| `blog_dir` | `Path` | `BLOG_DIR` resolved against `data_dir` |
# Validation Behaviour
`validate_config(values)` collects all failures before raising so the
operator sees every problem at once. A single `ConfigError` lists each
problem joined with `"; "`. Validation covers:
* Required keys are non-empty.
* `RUN_AT` parses as `HH:MM`.
* `MASTODON_BASE_URL` and `SITE_URL` parse via `urllib.parse.urlparse`
with an `http`/`https` scheme and a non-empty `netloc`.
* `VISIBILITY` is one of `{"public", "unlisted"}` (`ALLOWED_VISIBILITY`).
* `BLOG_REPO_URL` (optional, has default) parses via `urllib.parse.urlparse`
with an `http`/`https` scheme and a non-empty `netloc` when present.
* `TZ` resolves via `zoneinfo.ZoneInfo`.
* `MAX_RETRIES` parses as a non-negative integer (rejects bools).
# Exit Codes
| Code | Source | Meaning |
|------|-----------------------------------------|----------------------------------------------------------------------|
| `1` | `main.main()` (retry budget exhausted) | Pipeline failed on every attempt after `MAX_RETRIES + 1` tries. |
| `2` | `main.main()` (`ConfigError`) | Configuration was present but invalid; details are logged. |
# Citations
* [1] `src/tenbackward/config.py``REQUIRED_KEYS`, `DEFAULTS`, `ALLOWED_VISIBILITY`, `Config`, `load_config`, `validate_config`.
* [2] `.env.example` — canonical env-var list.
* [3] `entrypoint.sh` — shell-side mirror of the required keys.
# Related
* [System Architecture](/architecture/system-overview.md)
* [Logging & Run Summary](/architecture/logging.md)
* [Environment Variable Setup](/operations/environment-setup.md)
+126
View File
@@ -0,0 +1,126 @@
---
type: architecture
title: Logging & Run Summary
description: Structured JSON logging in 10Backward — JsonFormatter, secret redaction, and the startup / error / run_complete event helpers.
tags: [logging, observability, json]
timestamp: 2026-08-04T17:51:00Z
---
# Purpose
`tenbackward.logging_setup` replaces the scaffold-era `basicConfig`
text logger with a **single-line JSON formatter** that downstream
log shippers can index. It also owns the three helpers the runner
uses to record structured events: `log_startup`, `log_error`,
`log_run_summary`.
# JsonFormatter
`JsonFormatter.format(record)` returns one JSON object per log record:
| Key | Source |
|------------|---------------------------------------------------------------------|
| `ts` | UTC `datetime.fromtimestamp(record.created).isoformat()`. |
| `level` | `record.levelname` (e.g. `INFO`, `ERROR`). |
| `logger` | `record.name` (always `tenbackward`). |
| `message` | `record.getMessage()`. |
| _extras_ | Any keyword passed via `logging.info(..., extra={...})` that is not a reserved `LogRecord` attribute and whose key is **not** in `_SECRET_KEYS`. |
| `exc_type` | When `record.exc_info` is set, the exception class name. |
Reserved `LogRecord` attributes and any key starting with `_` are
dropped before serialization, so the payload contains only the values
that callers intentionally attached.
## Secret Redaction
`_SECRET_KEYS` is the canonical denylist applied in two places:
1. **At format time** for keys passed via `extra={...}`.
2. **At helper time** by `_redact_dict(...)` so values reaching
`log_startup` / `log_error` never expose known-sensitive fields
even before they hit the formatter.
Deny-list contents (case-insensitive):
```
token, access_token, mastodon_access_token, password, secret,
authorization, api_key
```
Any matching key is replaced with the string `[REDACTED]`.
# Helper Functions
| Helper | When it fires | Logger / Level | `event` field |
|---------------------------------------|-----------------------------------------------------------------------|----------------|--------------------|
| `configure_json_logging(level=INFO)` | First line of `main.main()`. Idempotent — clears existing handlers. | n/a | n/a |
| `log_startup(**fields)` | After config load + `data_dir.mkdir`, before the pipeline pass. | `INFO` | `startup` |
| `log_error(event, *, exc=None, **fields)` | When a pipeline attempt raises, or when the retry budget is exhausted. | `ERROR` | the supplied event name; `exc_type` added when `exc` is provided. |
| `log_run_summary(scanned, matched, posted, skipped, posted_ids)` | Once per successful pass. **Silently skipped** when all counters are zero and no ids were posted. | `INFO` | `run_complete` |
## Startup payload
`log_startup` is invoked with:
```
version, site_url, run_at, tz, hashtags, throwback_prefix
```
Each is added to the `startup` event payload under its own key.
## Run-summary payload
`log_run_summary` is the canonical "what did the bot do today?" record:
```json
{
"ts": "2026-08-04T09:00:00+00:00",
"level": "INFO",
"logger": "tenbackward",
"message": "run complete",
"event": "run_complete",
"scanned": 0,
"matched": 0,
"posted": 0,
"skipped": 0,
"posted_ids": []
}
```
A pass with no candidates emits **no line at all**, matching the
"silent on no matches" contract.
# Examples
## Successful run with one new post
```json
{"ts":"2026-08-04T09:00:00+00:00","level":"INFO","logger":"tenbackward","message":"startup","event":"startup","version":"0.1.0","site_url":"https://blog.example.com","run_at":"09:00","tz":"Europe/Berlin","hashtags":"#throwback,#10backward","throwback_prefix":"Heute vor 10 Jahren:"}
{"ts":"2026-08-04T09:00:01+00:00","level":"INFO","logger":"tenbackward","message":"run complete","event":"run_complete","scanned":1,"matched":1,"posted":1,"skipped":0,"posted_ids":["2025-08-04-post-slug"]}
```
## Retry budget exhausted
```json
{"ts":"...","level":"ERROR","logger":"tenbackward","message":"pipeline_error","event":"pipeline_error","attempt":1,"max_attempts":4,"exc_type":"ConnectionError"}
{"ts":"...","level":"ERROR","logger":"tenbackward","message":"pipeline_error","event":"pipeline_error","attempt":2,"max_attempts":4,"exc_type":"ConnectionError"}
{"ts":"...","level":"ERROR","logger":"tenbackward","message":"pipeline_failed","event":"pipeline_failed","attempts":4,"exc_type":"ConnectionError"}
```
The container then exits with status `1`.
## Token accidentally logged
Even if a future call wrote `extra={"access_token": "..."}`, the
formatter replaces it:
```json
{ "...": "...", "access_token": "[REDACTED]" }
```
# Related
* [System Architecture](/architecture/system-overview.md)
* [Pipeline Runner](/architecture/pipeline-runner.md)
* [Config Schema](/architecture/config-schema.md)
* [Mastodon Publishing](/architecture/mastodon-publishing.md)
+170
View File
@@ -0,0 +1,170 @@
---
type: architecture
title: Mastodon Publishing
description: The single boundary between the pipeline runner and the Mastodon HTTP API — status composition, length validation, and the publish call.
tags: [publishing, mastodon, api, boundary]
timestamp: 2026-08-04T18:55:00Z
---
# Purpose
`tenbackward.publishing` is the **single success boundary** between the
pipeline orchestrator and the Mastodon HTTP API. The runner calls
`publish_mastodon()` after dedupe and before state is persisted; if
the call raises, the retry loop re-attempts and `posted.json` is left
untouched. Composition and validation are pure functions that do not
touch the network so they are unit-tested without a server.
# Public API
| Symbol | Responsibility |
|---|---|
| `MASTODON_STATUS_LIMIT` | `500` — Mastodon's hard maximum status length. |
| `PublishError` | Raised on composition failure, length overflow, or any wrapped API exception. Subclass of `RuntimeError`. |
| `build_status_text(prefix, posts, hashtags)` | Pure composer. Returns the final status string. Raises `PublishError("empty_posts: ...")` when `posts` is empty. |
| `validate_status(status, limit=MASTODON_STATUS_LIMIT)` | Pure validator. Raises `PublishError("status_too_long: len=N limit=L")` when `len(status) > limit`. Never truncates. |
| `publish_mastodon(config, posts, *, client_factory=None)` | Composes, validates, and posts. Returns the composed status string. Wraps every third-party exception in `PublishError(f"publish_failed: {ExcType}: {exc}", ) from exc`. |
| `slugify_title(title)` | Re-exported from [matching](/architecture/anniversary-matching.md); used by callers that need to derive the same URL slug the matcher emits. |
# Status Layout
`build_status_text` produces exactly one Mastodon status for any number
of matching posts, in this shape:
```text
{prefix}
{title1}
{url1}
{title2}
{url2}
...
{hashtags}
```
* The prefix comes from `Config.throwback_prefix` (configured via `THROWBACK_PREFIX`).
* Posts are sorted by `(date, path)` so output is deterministic
regardless of upstream order.
* `hashtags` is a comma-separated string; commas are collapsed to a
single space so each tag starts with `#` and no trailing comma
is emitted (empty tags are dropped).
* A trailing newline is always appended.
# Composition Examples
## Single post
```text
Heute vor 10 Jahren:
Mein erster Post
https://blog.example.com/2016/08/04/mein-erster-post/
#throwback #10backward
```
## Multiple posts on the same day
When more than one Jekyll post matches the current day, they are
**merged into one status** — the runner does not issue separate API
calls per match. Ordering is `(date, path)`, so identical dates sort
by relative path:
```text
Heute vor 10 Jahren:
Post A
https://blog.example.com/2016/08/04/a/
Post B
https://blog.example.com/2016/08/04/b/
#throwback #10backward
```
## No posts
Calling `build_status_text("...", [], "#x")` raises
`PublishError("empty_posts: cannot compose status without posts")`
before any API call. The runner only invokes the publisher when its
candidate list is non-empty, so this guard is a backstop for direct
callers and unit tests.
# Length Validation
Mastodon's API rejects statuses longer than 500 characters. The
spec requires **failing safely** — the bot must never silently
truncate content. `validate_status()` enforces this contract: the
full status is measured, and overflow raises
`PublishError("status_too_long: len=N limit=500")` *before* the API
call. The runner's retry loop treats this exactly like any other
publish failure.
# Mastodon API Call
`_post_status_via_mastodon_py()` constructs a `mastodon.Mastodon` client with `(access_token=config.mastodon_access_token, api_base_url=config.mastodon_base_url)` and calls `status_post(status, visibility=config.mastodon_visibility)`.
The `client_factory` keyword argument on `publish_mastodon` lets
tests inject a fake client without monkey-patching. Production
callers leave it as `None`.
# Failure Modes
| Source | Exception surfaced to runner | Cause attached? |
|---|---|---|
| Empty `posts` argument | `PublishError("empty_posts: ...")` | No (no inner exc). |
| Composed status > 500 chars | `PublishError("status_too_long: ...")` | No. |
| Any other exception from inside the boundary | `PublishError("publish_failed: {ExcType}: {exc}")` | Yes, via `raise ... from exc`. |
The runner catches every `Exception` in `_run_once`, so any of the
above becomes a `pipeline_error` log line and the retry budget
decides whether to give up.
# Wiring
```
_run_once(config) # main.py
├── ensure_repo(...)
├── store = PostedStore(config.data_dir)
├── candidates = list(_iter_candidates(config)) # find_anniversary_matches
├── unposted = [m for m in candidates if not store.is_posted(m.path)]
├── if unposted:
│ publish_mastodon(config, unposted) # <-- THIS boundary
│ store.mark_posted_many(posted_ids) # only after success
```
State is written **only after** `publish_mastodon` returns. A publish
failure leaves `posted.json` unchanged and lets the retry budget
re-attempt on the next iteration.
# Configuration Surface
| Env var | Consumed via | Effect |
|---|---|---|
| `MASTODON_BASE_URL` | `Config.mastodon_base_url` | Mastodon instance URL. |
| `MASTODON_ACCESS_TOKEN` | `Config.mastodon_access_token` | OAuth token passed to the `Mastodon` client. |
| `MASTODON_VISIBILITY` | `Config.mastodon_visibility` | Passed as `visibility=` to `status_post`. |
| `THROWBACK_PREFIX` | `Config.throwback_prefix` | First line of every published status. |
| `HASHTAGS` | `Config.hashtags` | Trailing hashtag line; commas become spaces. |
See [Config Schema](/architecture/config-schema.md) for full validation
rules.
# Key Files
| Path | Responsibility |
|---|---|
| `/repo/src/tenbackward/publishing.py` | `MASTODON_STATUS_LIMIT`, `PublishError`, `build_status_text`, `validate_status`, `_post_status_via_mastodon_py`, `publish_mastodon`. |
| `/repo/src/tenbackward/matching.py` | `MatchedPost` dataclass and `slugify_title()` reused here. |
| `/repo/src/tenbackward/main.py` | Calls `publish_mastodon()` between dedupe and `mark_posted_many`. |
| `/repo/tests/test_publishing.py` | Composition, length validation, visibility passthrough, API failure wrapping, and the no-persistence-on-failure contract. |
# Related
* [Pipeline Runner](/architecture/pipeline-runner.md)
* [Anniversary Matching](/architecture/anniversary-matching.md)
* [Config Schema](/architecture/config-schema.md)
* [Daily Run Guide](/guides/daily-run.md)
+134
View File
@@ -0,0 +1,134 @@
---
type: architecture
title: Pipeline Runner
description: How tenbackward.main synchronizes the blog, discovers anniversary candidates, applies deduplication, retries failures, and emits a run summary.
tags: [pipeline, runner, retry]
timestamp: 2026-08-04T17:51:00Z
---
# Purpose
`tenbackward.main` is the entry point executed by cron. The pipeline first
ensures that the configured blog repository is available and current, then
scans Jekyll posts for today's ten-year anniversary, filters already-recorded
paths, publishes a single combined Mastodon status for the new matches, and
finally persists the freshly published identifiers.
# Call Flow
```
main.main()
├── configure_json_logging()
├── load_config() ── raises ConfigError → exit 2
├── config.data_dir.mkdir(parents=True, exist_ok=True)
├── ensure_repo(config.data_dir, blog_repo_url, blog_dir, max_retries)
│ └── BlogRepoError → log blog_repo_error → exit 1
├── log_startup(version, site_url, run_at, tz, hashtags, throwback_prefix)
├── result = _run_with_retry(config)
│ ├── for attempt in 1 .. max_retries+1:
│ │ try: return _run_once(config)
│ │ └── _run_once → ensure_repo → find_anniversary_matches
│ │ → PostedStore deduplication → publish_mastodon
│ │ → mark_posted_many # state only after publish OK
│ ├── log_error("pipeline_error", exc=exc, attempt=attempt, max_attempts=attempts)
│ ├── log_error("pipeline_failed", exc=last_exc, attempts=attempts)
│ └── return None
├── if result is None: return 1
└── log_run_summary(scanned, matched, posted, skipped, posted_ids)
```
# Extension Point: `_iter_candidates`
```python
def _iter_candidates(config: Config) -> Iterable[MatchedPost]: ...
```
`_iter_candidates` delegates to `find_anniversary_matches()` with
`config.blog_dir / "_posts" / "blog"` and `config.site_url`. The matcher
returns full `MatchedPost` values (relative path, title, date, canonical
URL); the runner uses `MatchedPost.path` as a stable deduplication ID and
passes the full objects to `publish_mastodon()` for status composition.
See [Anniversary Matching](/architecture/anniversary-matching.md) for the
file and front matter rules.
## Blog Synchronization
`ensure_repo()` runs before candidate discovery. An empty target is cloned
from `BLOG_REPO_URL`; an existing repository is fetched and fast-forwarded.
Transient Git failures use exponential backoff, while local modifications
fail immediately with `BlogRepoError`. The same synchronization occurs once
in `main()` before `startup` and again inside `_run_once()` as the retryable
pipeline boundary.
# `_run_once` — Summary Counters
`_run_once` returns a 5-tuple:
```python
(scanned, matched, posted, skipped, posted_ids)
```
| Counter | Meaning |
|-------------|----------------------------------------------------------------------|
| `scanned` | Items yielded by `_iter_candidates`. |
| `matched` | Items that survived the dedupe-vs-`already_posted` check. |
| `posted` | Items newly recorded in `posted.json` during this run. |
| `skipped` | Items that matched but were already in `posted.json` (deduped). |
| `posted_ids`| The list of post identifiers written to state. |
`PostedStore.mark_posted_many` is called **only** when `posted_ids`
is non-empty, so a no-op run does not touch `posted.json` on disk.
The store persists the list under a `"posted"` key
(`{"posted": ["2014/2014-08-04-foo.md", ...]}`) and serialises
concurrent runs with an `fcntl.flock`.
# Publish Boundary
`publish_mastodon()` is the single success boundary. The runner calls it
**after** dedupe and **before** `mark_posted_many`. If the call raises a
`PublishError`, the runner treats it like any other pipeline exception —
the retry loop in `_run_with_retry` re-attempts, and no identifier is
written to `posted.json`. Identical-day matches are published as **one**
combined status: prefix line + one `title\nurl` block per post + hashtags
line, sorted by `(date, path)` for deterministic ordering. The default
prefix is `Heute vor 10 Jahren:` (overridable via `THROWNBACK_PREFIX`).
Generated statuses are validated against `MASTODON_STATUS_LIMIT` (500
characters); the runner fails safely (raises) rather than truncating.
# Retry Behaviour (`_run_with_retry`)
* `attempts = max(1, config.max_retries)` — at least one attempt even if `MAX_RETRIES=0`.
* The runner owns the unified retry budget; `_run_once` calls `ensure_repo` with `max_retries=0`.
* Transient Git and Mastodon failures are retried with delays of `2 ** (attempt - 1)` seconds between attempts.
* Each failed attempt emits `retry_attempt` with `operation`, `attempt`, and the exception `error` text.
* Fatal configuration, local repository, and publishing validation errors emit `retry_exhausted` immediately without retrying.
* When the budget is exhausted, `retry_exhausted` includes the final operation, total attempts, and error text, and `main` returns exit code `1`.
# Silent-on-No-Matches Contract
* `run_complete` is emitted for a successful pass, including when all discovered candidates were already posted.
* A run with zero candidates emits only `startup`; no `run_complete` line is emitted.
# Example Sequence (Healthy Run)
1. `startup` line at INFO.
2. `_run_with_retry` returns `(0, 0, 0, 0, [])` (no candidates yet
because `_iter_candidates` is empty) → `main` exits `0`
**without** a `run_complete` line.
3. Next-day tick repeats the cycle.
# Example Sequence (Future Blog Step in Place)
1. `startup` line at INFO.
2. `_iter_candidates` yields `[id-1, id-2]`. `id-1` is already in
`posted.json`, so `skipped += 1`. `id-2` is new and appended to
`posted_ids` and persisted.
3. `run_complete` line at INFO with
`scanned=2, matched=2, posted=1, skipped=1, posted_ids=["id-2"]`.
# Related
* [Anniversary Matching](/architecture/anniversary-matching.md)
* [Logging & Run Summary](/architecture/logging.md)
* [Config Schema](/architecture/config-schema.md)
* [Cron Lifecycle](/operations/cron-lifecycle.md)
+66
View File
@@ -0,0 +1,66 @@
---
type: architecture
title: Job 1089 Runtime Changes
description: Container privilege separation, unified retry logging, configuration validation, and persistent data wiring introduced by the current implementation.
tags: [architecture, docker, retry, logging, configuration]
timestamp: 2026-08-04T20:44:00Z
---
# Overview
The current runtime keeps the container entrypoint and cron daemon as root while executing the Python bot as the unprivileged `bot` user. The pipeline owns one retry budget for Git synchronization and Mastodon publishing, and structured retry events include the operation and exception text. Configuration now validates required values before optional defaults are applied.
# Wiring
```text
Docker ENTRYPOINT /app/entrypoint.sh (root)
-> validates environment and renders /etc/cron.d/tenbackward
-> cron -f
-> /usr/local/bin/run-bot.sh
-> setpriv/su to bot
-> python -m tenbackward
-> load_config()
-> _run_with_retry()
-> _run_once()
-> ensure_repo(..., max_retries=0, logger=...)
-> match posts
-> PostedStore dedupe
-> publish_mastodon()
-> PostedStore.mark_posted_many()
```
The single `./data:/app/data` Compose bind mount contains both the blog clone and `posted.json`; no file-level bind mount is used.
# Retry and Logging Contract
`MAX_RETRIES` is interpreted as the maximum number of attempts, with at least one attempt when configured as zero. Backoff delays are `1, 2, 4, ...` seconds between attempts. Git clone/pull retries are surfaced to the runner with operation labels (`git clone` or `git pull`), while publishing failures use `mastodon post`. Fatal configuration, local-repository, and publishing validation errors are not retried.
Retry events include an `error` field containing the exception message. `log_event()` sends structured fields through the same redaction path as other log helpers.
# Configuration Validation
`load_config()` merges dotenv values and process environment values, validates required keys against that raw merged map, then applies defaults. `MAX_RETRIES`, `BLOG_REPO_URL`, and `BLOG_DIR` may use defaults; the required Mastodon, site, hashtag, prefix, schedule, and timezone values must be explicitly non-empty.
# Key Files
| Path | Responsibility |
|---|---|
| `/repo/Dockerfile` | Installs cron, Git, timezone data, and `setpriv`; leaves the final process root-capable for cron and owns `/app/data` by `bot`. |
| `/repo/entrypoint.sh` | Validates environment, renders the daily cron entry, installs it as root, and starts foreground cron. |
| `/repo/run-bot.sh` | Drops from root to `bot` using `setpriv` or `su`, then invokes the Python module. |
| `/repo/docker-compose.yml` | Supplies environment values and mounts `./data` at `/app/data`. |
| `/repo/src/tenbackward/main.py` | Owns the unified retry loop and passes logger/retry controls into repository synchronization. |
| `/repo/src/tenbackward/blog.py` | Clones or fast-forwards the blog and emits operation-specific retry events. |
| `/repo/src/tenbackward/config.py` | Performs raw required-key validation and builds the typed configuration. |
| `/repo/src/tenbackward/logging_setup.py` | Provides structured events, redaction, and JSON-per-line formatting. |
| `/repo/tests/test_docker_artifacts.py` | Verifies container privilege and artifact wiring. |
| `/repo/tests/test_config.py` | Verifies required-key validation and defaults. |
| `/repo/tests/test_run_logging.py` | Verifies retry fields, attempt behavior, and state safety. |
# Related
* [System Architecture](/architecture/system-overview.md)
* [Config Schema](/architecture/config-schema.md)
* [Pipeline Runner](/architecture/pipeline-runner.md)
* [Environment Variable Setup](/operations/environment-setup.md)
* [Daily Run Guide](/guides/daily-run.md)
+133
View File
@@ -0,0 +1,133 @@
---
type: architecture
title: System Architecture
description: Component map of 10Backward — how configuration, blog synchronization, anniversary matching, logging, the pipeline runner, state persistence, and container entrypoint are wired together.
tags: [architecture, overview]
timestamp: 2026-08-04T18:55:00Z
---
# Overview
`10Backward` is a Mastodon daily-throwback bot that runs as a single
containerised cron job. Job 1086 adds anniversary matching across Jekyll
posts, backed by the blog clone/pull workflow from Job 1084 and the
structured logging, retry-capable runner, and deduplication state from
Job 1083. The current pipeline identifies matching posts, publishes a
single combined Mastodon status for the new matches, and records the
published identifiers in `posted.json`.
The container boots, validates environment configuration, renders a
`/etc/cron.d/tenbackward` entry that fires once per day at the
configured `RUN_AT`, and then runs `python -m tenbackward` in the
foreground of the `cron` process. Each scheduled invocation calls
`tenbackward.main.main()`, which:
1. Installs the JSON logging formatter on the root logger.
2. Loads and validates the `Config`.
3. Ensures the data directory exists.
4. Emits a `startup` log line.
5. Executes one pipeline pass wrapped in a retry loop:
`ensure_repo``find_anniversary_matches``PostedStore` dedupe →
`publish_mastodon``PostedStore.mark_posted_many`.
6. Emits a `run_complete` summary (skipped silently if there were no candidates).
# Components
| Component | Responsibility |
|--------------------------|------------------------------------------------------------------------------------------------------------|
| `entrypoint.sh` | Validates required env vars, renders the cron file from `RUN_AT`/`TZ`, then `exec cron -f`. |
| `tenbackward.main` | CLI entry point; orchestrates startup logging, retry-wrapped pipeline pass, and run summary. |
| `tenbackward.config` | Loads `.env` + process env, applies defaults, validates schema, produces a typed `Config` dataclass. |
| `tenbackward.logging_setup` | JSON formatter (one log record per line), secret redaction, and `log_startup` / `log_error` / `log_run_summary` helpers. |
| `tenbackward.state` | `PostedStore` — self-contained dedup store. `posted.json` holds `{"posted": [str, ...]}`; mutations are serialised by an `fcntl.flock` on a sibling lock file, writes go through a temp-file replace, and `load()` auto-creates an empty list when the file is missing. |
| `tenbackward.blog` | GitPython `ensure_repo()` — clones the configured blog repo on first run, fast-forwards it via `pull --ff-only` thereafter, with exponential-backoff retries on transient network errors. |
| `tenbackward.matching` | Walks `_posts/blog/**/*.md`, parses Jekyll front matter and filenames, and yields posts exactly ten years before the current date using Berlin-time and leap-day rules. |
| `tenbackward.publishing` | `publish_mastodon()` — the single success boundary between the runner and the Mastodon HTTP API. Composes a combined status for the day's matches, validates the 500-character limit, and posts via `mastodon.Mastodon.status_post`. Raises `PublishError` on composition, length, or API failure; the runner treats it like any other pipeline exception. |
| `/etc/cron.d/tenbackward`| Rendered cron file. One daily line that `cd /app` and runs `python -m tenbackward`. |
# Communication & Wiring
```
+-------------------+ env vars +-------------------------+
| entrypoint | -------------------> | config.load_config |
| (.sh) | | (Config dataclass) |
+---------+---------+ +-----------+-------------+
| |
| installs cron file | feeds
v v
+-------------------+ +-------------------------+
| /etc/cron.d/ | -- daily fires --> | tenbackward.main.main |
| tenbackward | | - configure logging |
+-------------------+ | - retry-wrapped pass |
+-----------+-------------+
|
v
+-------------------------+
| _run_with_retry(...) |
| -> _run_once(...) |
| -> ensure_repo(...) |
| -> matching |
| find_anniversary_|
| matches |
| -> PostedStore |
| {is_posted, |
| mark_posted_many}|
| -> publishing |
| publish_mastodon |
| -> PostedStore |
| mark_posted_many |
+-------------------------+
|
v
+-------------------------+
| Mastodon HTTP API |
| status_post(...) |
+-------------------------+
```
* **Env → Config.** `load_config()` merges a `.env` file (when
present) under the process environment via `dotenv_values` + `load_dotenv`,
applies defaults, then validates the merged map before producing the
`Config` dataclass.
* **Cron → main.** Each scheduled tick re-runs `python -m tenbackward`,
so every run is a fresh interpreter invocation.
* **Matching → publishing.** `find_anniversary_matches` yields
`MatchedPost` objects; the runner dedupes against `PostedStore`, then
passes the unposted objects to `publish_mastodon`. See
[Mastodon Publishing](/architecture/mastodon-publishing.md).
* **Pipeline state.** `_run_once` reads `posted.json` via
`PostedStore.is_posted()` for dedupe, then writes it back via
`PostedStore.mark_posted_many()` **only after** `publish_mastodon`
returns. The store serialises concurrent runs with an
`fcntl.flock` on a sibling lock file and writes via temp-file
rename.
# Key Files
| Path | Responsibility |
|-------------------------------------|---------------------------------------------------------------------------------|
| `/repo/entrypoint.sh` | Bootstraps cron; validates required env vars; renders `/etc/cron.d/tenbackward`. |
| `/repo/src/tenbackward/main.py` | CLI entry, retry wrapper, pipeline counters. |
| `/repo/src/tenbackward/config.py` | Env merging, defaults, validation, `Config` dataclass, `ConfigError`. |
| `/repo/src/tenbackward/logging_setup.py` | JSON formatter, secret redaction, structured event helpers. |
| `/repo/src/tenbackward/state.py` | `PostedStore` + module helpers (`load_posted`, `is_posted`, `mark_posted`, `mark_posted_many`); list-shaped JSON, `fcntl.flock`, atomic temp-file replace, env-var data dir. |
| `/repo/src/tenbackward/blog.py` | `ensure_repo()` — GitPython clone + fast-forward pull with `2^n` retry/backoff; raises `BlogRepoError` on local modifications or exhausted retries. |
| `/repo/src/tenbackward/matching.py` | Jekyll post discovery, front matter parsing, anniversary and leap-day matching, and canonical URL construction. |
| `/repo/src/tenbackward/publishing.py` | `publish_mastodon()` — composes a combined Mastodon status, validates the 500-character limit, posts via `mastodon.Mastodon.status_post`. Pure helpers `build_status_text` and `validate_status` keep network code out of tests. |
| `/repo/src/tenbackward/__main__.py` | Module entrypoint that invokes `tenbackward.main.main()`. |
| `/repo/.env.example` | Canonical list of environment variables. |
| `/repo/docker-compose.yml` | Service definition; binds env vars from the host `.env`. |
| `/repo/Dockerfile` | Builds the runtime image. |
| `/repo/crontab/tenbackward.cron` | Cron template shipped in the image. |
| `/repo/crontab/install-cron.sh` | Installs the rendered cron file with mode 0644; refuses placeholder leftovers. |
| `/repo/tests/` | Pytest suite covering config, logging, runner, cron, and entrypoint validation. |
# Related
* [Config Schema](/architecture/config-schema.md)
* [Logging & Run Summary](/architecture/logging.md)
* [Pipeline Runner](/architecture/pipeline-runner.md)
* [Anniversary Matching](/architecture/anniversary-matching.md)
* [Mastodon Publishing](/architecture/mastodon-publishing.md)
* [Environment Variable Setup](/operations/environment-setup.md)
* [Cron Lifecycle](/operations/cron-lifecycle.md)
+125
View File
@@ -0,0 +1,125 @@
---
type: guide
title: Daily Run Guide
description: Operator- and tester-focused walkthrough of one daily cron tick — repository synchronization, anniversary matching, deduplication, logging, and retry behaviour.
tags: [guide, run, daily, tester]
timestamp: 2026-08-04T17:51:00Z
---
# Purpose
A tester or operator needs to know **what should happen** each time
the cron tick fires. This guide describes the contract from the
outside of the container, in observable terms: log lines, exit codes,
files appearing on the volume, and the "silent on no matches"
behaviour that the runner enforces.
# Triggering a Run Manually
```bash
docker compose exec tenbackward /usr/local/bin/python -m tenbackward
```
This is the same command the container's cron entry issues once per
day at `RUN_AT`. Use it to verify behaviour without waiting for the
scheduled tick.
# What the Run Does
Before scanning, the process ensures the configured blog repository exists under `/app/data/blog` and is fast-forwarded from `BLOG_REPO_URL`. It then walks `_posts/blog/**/*.md`, finds posts whose date is exactly ten years before today, skips IDs already present in `posted.json`, composes a single Mastodon status listing each new match's title and canonical URL followed by the configured hashtags, and — only after the Mastodon API call succeeds — records the relative paths of the published posts in `posted.json`. The scheduled wrapper executes this Python process as the unprivileged `bot` user.
# What You Should See
The bot uses a **JSON-per-line** logger that writes to
`/app/data/cron.log` (because the cron line redirects with
`>> /app/data/cron.log 2>&1`). Inspecting the file with
`docker compose exec tenbackward tail -n 100 /app/data/cron.log`
should show output similar to:
```json
{"ts": "2026-08-04T09:00:00+00:00", "level": "INFO", "logger": "tenbackward", "message": "startup", "event": "startup", "version": "0.1.0", "site_url": "https://blog.example.com", "run_at": "09:00", "tz": "Europe/Berlin", "hashtags": "#throwback,#10backward", "throwback_prefix": "Heute vor 10 Jahren:"}
```
On the current scaffold (`_iter_candidates` is intentionally empty),
this is the **only** line you should see — the runner emits no
`run_complete` when no candidates were found. This is the
"silent on no matches" contract.
# Expected Container Behaviour
| Scenario | Lines in `cron.log` | Container exit |
|-------------------------------------------|----------------------------------------------------------------|------------------|
| Config valid, no anniversary matches | 1× `startup` | `0` |
| Blog clone or pull fails | 1× `blog_repo_error` | `1` |
| Config valid, matching IDs are new | 1× `startup`, then 1× `run_complete` with `posted=N` | `0` |
| Config valid, all matches already posted | 1× `startup`, then 1× `run_complete` with `posted=0` | `0` |
| Pipeline raises once, recovers | `retry_attempt`, then `run_complete` | `0` |
| Pipeline keeps raising (budget exhausted) | `retry_attempt` events, then `retry_exhausted` with operation and error | `1` |
| Config invalid | 1× `configuration_error` (extras describe what failed) | `2` |
> **Tester tip** — `docker compose ps` should report the container as
> `running` after a `pipeline_failed` exit *only* if cron has not yet
> fired again. A single failed pipeline tick does not kill the container
> itself; cron re-runs it the next day.
# Files the Tester Should Look For
| Path (inside container) | When it appears |
|-------------------------|--------------------------------------------|
| `/app/data/cron.log` | Always (cron appends stdout/stderr here). |
| `/app/data/posted.json` | Created when the matcher or a state check first loads the store; rewritten only when a new ID is posted. The file contains relative Jekyll paths under a `posted` list. |
To verify these from the host:
```bash
docker compose exec tenbackward ls -la /app/data
docker compose exec tenbackward cat /app/data/cron.log
```
# Visual Elements
This is a server-side bot, so there is no UI. The "UI" consists of:
* **Log lines on stdout** — one JSON object per `INFO`/`ERROR` event.
* **Exit code** — `0` (healthy), `1` (pipeline exhausted), or `2` (config).
* **`posted.json`** — JSON state file; its mtime changes whenever a new id is persisted.
* **`/app/data/blog`** — synchronized local clone containing the Jekyll posts scanned for anniversaries.
# Examples
## Healthy scaffold tick
```bash
$ docker compose exec tenbackward python -m tenbackward
{"ts":"...","event":"startup",...}
$ echo $?
0
```
## Bad `RUN_AT`
```bash
$ RUN_AT=25:99 docker compose exec tenbackward python -m tenbackward
ERROR: RUN_AT='25:99' must be in HH:MM (24-hour) format
$ docker compose ps
tenbackward Exit 1
```
(The error message lands on stderr in the entrypoint path and is
captured by `cron.log`.)
## Missing required env var
Unset `MAX_RETRIES`, restart the container:
```bash
$ docker compose up --build
tenbackward | ERROR: missing required environment variable(s): MAX_RETRIES
tenbackward exited with code 1
```
# Related
* [Environment Variable Setup](/operations/environment-setup.md)
* [Pipeline Runner](/architecture/pipeline-runner.md)
* [Logging & Run Summary](/architecture/logging.md)
+74
View File
@@ -0,0 +1,74 @@
---
type: guide
title: Runtime Verification Guide
description: Tester workflow for validating root cron startup, unprivileged bot execution, retry events, and persistent data behavior.
tags: [guide, tester, docker, runtime]
timestamp: 2026-08-04T20:44:00Z
---
# Setup
1. Copy `.env.example` to `.env` and provide valid required values.
2. Build and start the service:
```bash
docker compose up --build -d
```
3. Follow container output:
```bash
docker compose logs -f bot
```
# Expected Runtime Behavior
The container remains running with `cron -f` as its foreground process. Startup output includes the configured `RUN_AT` and `TZ`. The scheduled cron command invokes `/usr/local/bin/run-bot.sh`, which runs the Python bot as user `bot`, not root.
The host `./data` directory should contain the synchronized `blog/` clone and, after the state store is loaded, `posted.json`. Both are under the one `/app/data` mount.
# Manual Run
Run the same application command without waiting for cron:
```bash
docker compose exec bot /usr/local/bin/python -m tenbackward
```
A valid run exits `0`. A configuration failure exits `2`; an exhausted pipeline retry budget exits `1`.
# Retry Checks
To exercise transient failure handling, use a test repository or injected test double that fails once and then succeeds. Verify JSON events contain:
| Event | Expected fields |
|---|---|
| `retry_attempt` | `operation`, `attempt`, and the original `error` text. |
| `retry_exhausted` | Final `operation`, total `attempts`, and `error` text when all attempts fail. |
| `run_complete` | Counters and `posted_ids` only after a successful pass. |
With `MAX_RETRIES=0`, one attempt is still made and no sleep occurs. With a larger value, delays occur only between attempts and follow powers of two beginning at one second.
# Configuration Checks
Remove a required variable from `.env` and restart the service. The entrypoint should report the missing variable before cron starts. If the shell check is bypassed and the Python module runs directly, `configuration_error` should identify the missing key and return exit code `2` rather than silently substituting an application default.
# Persistence and Failure Safety
After a successful publication, inspect `/app/data/posted.json` and confirm new relative post paths appear under the `posted` list. If Mastodon publishing fails, the path must not be recorded; a later retry should be able to publish it again.
# Key Visual/Observable Elements
This service has no graphical UI. Test-facing outputs are:
* Container status and `docker compose logs` output.
* JSON-per-line startup, retry, failure, and completion events.
* Exit codes from manual runs.
* `/app/data/blog` and `/app/data/posted.json` on the mounted host directory.
# Related
* [Daily Run Guide](/guides/daily-run.md)
* [Job 1089 Runtime Changes](/architecture/runtime-changes.md)
* [Environment Variable Setup](/operations/environment-setup.md)
* [Cron Lifecycle](/operations/cron-lifecycle.md)
+20
View File
@@ -0,0 +1,20 @@
---
okf_version: "0.1"
---
# Architecture
* [System Architecture](/architecture/system-overview.md) — Component map of 10Backward: entrypoint, config, logging, runner, state, publishing boundary, and cron wiring.
* [Runtime Changes](/architecture/runtime-changes.md) — Root cron/unprivileged bot separation, unified retries, raw configuration validation, and data-volume wiring.
* [Config Schema](/architecture/config-schema.md) — Required/optional env vars, validation rules, and the typed Config dataclass.
* [Logging & Run Summary](/architecture/logging.md) — JSON formatter, secret redaction, and structured event helpers.
* [Pipeline Runner](/architecture/pipeline-runner.md) — How a cron tick synchronizes the blog, matches anniversaries, applies deduplication, publishes to Mastodon, retries failures, and emits run events.
* [Anniversary Matching](/architecture/anniversary-matching.md) — Jekyll filename/front matter rules, ten-year date matching, leap-day handling, and candidate identifiers.
* [Mastodon Publishing](/architecture/mastodon-publishing.md) — The single success boundary: status composition, 500-character validation, and the Mastodon API call.
# Operations
* [Environment Variable Setup](/operations/environment-setup.md) — Required env vars, renamed keys, and how to populate `.env`.
* [Cron Lifecycle](/operations/cron-lifecycle.md) — How `entrypoint.sh` renders `/etc/cron.d/tenbackward` and hands off to `cron -f`.
# User Guides
* [Daily Run Guide](/guides/daily-run.md) — Operator walkthrough of repository sync, anniversary matching, deduplication, publishing, and expected logs.
* [Runtime Verification Guide](/guides/runtime-verification.md) — Tester steps for privilege separation, retry events, configuration failures, and persistent state.
+96
View File
@@ -0,0 +1,96 @@
---
type: operations
title: Cron Lifecycle
description: How entrypoint.sh validates env, renders /etc/cron.d/tenbackward from RUN_AT/TZ, and hands off to cron -f.
tags: [cron, entrypoint, ops]
timestamp: 2026-08-04T17:51:00Z
---
# Purpose
`entrypoint.sh` is the first process inside the container. Its job is
to refuse to boot when something is wrong with env config and to
materialise a cron file from the templated values.
# Sequence
1. Set `DEBIAN_FRONTEND=noninteractive`.
2. Set `TZ` (default `Europe/Berlin`) and link `/etc/localtime` if
`/usr/share/zoneinfo/${TZ}` exists and `/etc/localtime` does not.
3. Iterate over `required_vars`. Any empty variable aborts with
`ERROR: missing required environment variable(s): ...` and exits
`1`. The current required set is listed in
[Environment Variable Setup](/operations/environment-setup.md).
4. Validate `RUN_AT` against
`^([01][0-9]|2[0-3]):[0-5][0-9]$`. Invalid input exits `1` with
`ERROR: RUN_AT='X' must be in HH:MM (24-hour) format`.
5. Render `/etc/cron.d/tenbackward` into a `mktemp` file with:
```cron
SHELL=/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
TZ=${TZ}
${minute} ${hour} * * * cd /app && /usr/local/bin/python -m tenbackward >> /app/data/cron.log 2>&1
```
6. Install the rendered file with mode `0644`, root:root.
7. Run `crontab/install-cron.sh /etc/cron.d/tenbackward`. The helper
refuses files that still contain the `__RUN_AT__` placeholder and
files that do not start with an `^[A-Z_]+=` cron env line.
8. Print a one-line confirmation: `tenbackward: starting cron
(RUN_AT=..., TZ=...)`.
9. `exec cron -f`.
# Output Files
| Path | Owner / Mode | Purpose |
|------------------------------|--------------|------------------------------------------------------|
| `/etc/cron.d/tenbackward` | root / 0644 | The rendered cron file. |
| `/app/data/cron.log` | container user / append | `python -m tenbackward` stdout+stderr. |
| `/app/data/posted.json` | container user / rw | Persisted state from `tenbackward.state`. |
# Cron Helpers
| File | Purpose |
|---------------------------------|------------------------------------------------------------------------|
| `crontab/tenbackward.cron` | Static template (env + PATH) shipped in the image. |
| `crontab/install-cron.sh` | Validates and installs a rendered cron file with mode 0644. |
# Operator Recipes
* **Trigger a manual run** without waiting for the cron tick:
```bash
docker compose exec tenbackward /usr/local/bin/python -m tenbackward
```
* **Inspect the rendered cron file** (the file the container actually
installed):
```bash
docker compose exec tenbackward cat /etc/cron.d/tenbackward
```
* **Tail the structured logs** produced by the bot:
```bash
docker compose exec tenbackward tail -n 100 /app/data/cron.log
```
Expect one JSON object per line (`startup`, optional `run_complete`,
or `pipeline_error` / `pipeline_failed`).
# Build Script Notes
`build.sh` now `cd`s into its own script directory (so it works
no matter where it is invoked from) and, when possible, symlinks
itself to `/usr/local/bin/build.sh` so the build can also be run as a
plain `build.sh` command inside the build context. The symlink is a
best-effort `|| true`, so a read-only filesystem will not break the
build.
# Related
* [Environment Variable Setup](/operations/environment-setup.md)
* [Pipeline Runner](/architecture/pipeline-runner.md)
* [Logging & Run Summary](/architecture/logging.md)
+86
View File
@@ -0,0 +1,86 @@
---
type: operations
title: Environment Variable Setup
description: How to provide and validate the required env vars for the 10Backward container — .env, docker-compose, and the runtime check in entrypoint.sh.
tags: [env, ops, setup]
timestamp: 2026-08-04T17:51:00Z
---
# Purpose
Job 1083 promoted every previously-optional knob in `.env.example`
to required, and renamed three of them. Operators must populate the
full set before the container will boot.
# Required Variables
| Variable | Example value | Purpose |
|--------------------------|--------------------------------|---------------------------------------------------|
| `MASTODON_BASE_URL` | `https://mastodon.social` | Mastodon instance to post against. |
| `MASTODON_ACCESS_TOKEN` | _(from your Mastodon account)_ | OAuth access token. |
| `MASTODON_VISIBILITY` | `public` | Post visibility (`public` or `unlisted`). |
| `SITE_URL` | `https://blog.example.com` | Source blog URL. |
| `HASHTAGS` | `#throwback,#10backward` | Hashtags appended to every throwback post. |
| `THROWBACK_PREFIX` | `Heute vor 10 Jahren:` | Prefix prepended to every Mastodon status. |
| `MAX_RETRIES` | `5` | Maximum number of pipeline attempts. |
| `RUN_AT` | `09:00` | Daily fire time (`HH:MM`, 24-hour). |
| `TZ` | `Europe/Berlin` | IANA timezone for cron and application. |
| `BLOG_REPO_URL` | project default | Optional source repository URL. |
| `BLOG_DIR` | `blog` | Optional clone directory, relative to `DATA_DIR` unless absolute. |
# Where Each Name Is Enforced
* `entrypoint.sh` — validates all nine required keys, validates `RUN_AT`, and installs the cron file as root. The cron-launched wrapper then drops privileges to `bot` before Python runs.
* `tenbackward.config.validate_config` — validates the merged raw required keys plus URL, visibility, timezone, and retry syntax. Defaults are applied only after required-key validation.
* `docker-compose.yml` — supplies development environment values and mounts `./data` at `/app/data`.
# Setup Steps
1. Copy the template:
```bash
cp .env.example .env
```
2. Replace placeholder values in `.env` (the example ships with
`MASTODON_ACCESS_TOKEN=replace-me` and `https://mastodon.example`).
3. Verify the file parses by running the validator directly:
```bash
python -m tenbackward
```
A successful validation run emits one `startup` JSON log line and
exits `0`. Missing/invalid config exits `2`.
4. Bring up the container:
```bash
docker compose up --build
```
The required-var check in `entrypoint.sh` runs first; if any
variable is empty, the container exits before `cron` starts.
# Renames vs. Job 1082
| Old key | New key |
|------------------------|--------------------------|
| `MASTODON_VISIBILITY` | `VISIBILITY` |
| `THROWBACK_PREFIX` | `THROWNBACK_PREFIX` |
| `RETRY_COUNT` | `MAX_RETRIES` |
If you have an existing `.env` from the scaffold, rename these
manually. The renamed variables are **not** backwards compatible — the
container will refuse to start with both versions set.
# Defaults
`MAX_RETRIES` defaults to `5`; `BLOG_REPO_URL` and `BLOG_DIR` also have application defaults. Other application-facing required keys must be present in the raw merged dotenv/process environment. The Docker image supplies `TZ=Europe/Berlin`, and Compose supplies development fallbacks, but production deployments should set every required value explicitly.
# Related
* [Config Schema](/architecture/config-schema.md)
* [System Architecture](/architecture/system-overview.md)
* [Cron Lifecycle](/operations/cron-lifecycle.md)
Executable
+59
View File
@@ -0,0 +1,59 @@
#!/bin/bash
set -euo pipefail
export DEBIAN_FRONTEND="${DEBIAN_FRONTEND:-noninteractive}"
export TZ="${TZ:-Europe/Berlin}"
if [ -f /usr/share/zoneinfo/"${TZ}" ] && [ ! -f /etc/localtime ]; then
ln -snf /usr/share/zoneinfo/"${TZ}" /etc/localtime || true
fi
required_vars=(
MASTODON_BASE_URL
MASTODON_ACCESS_TOKEN
MASTODON_VISIBILITY
SITE_URL
HASHTAGS
THROWBACK_PREFIX
MAX_RETRIES
RUN_AT
TZ
)
missing=()
for var in "${required_vars[@]}"; do
if [ -z "${!var:-}" ]; then
missing+=("${var}")
fi
done
if [ "${#missing[@]}" -gt 0 ]; then
echo "ERROR: missing required environment variable(s): ${missing[*]}" >&2
exit 1
fi
run_at="${RUN_AT}"
if ! [[ "${run_at}" =~ ^([01][0-9]|2[0-3]):[0-5][0-9]$ ]]; then
echo "ERROR: RUN_AT='${run_at}' must be in HH:MM (24-hour) format" >&2
exit 1
fi
hour="${run_at%%:*}"
minute="${run_at##*:}"
tmp_cron="$(mktemp)"
trap 'rm -f "${tmp_cron}"' EXIT
{
echo "SHELL=/bin/bash"
echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
echo "TZ=${TZ}"
echo "${minute} ${hour} * * * /usr/local/bin/run-bot.sh >> /proc/1/fd/1 2>&1"
} > "${tmp_cron}"
install -m 0644 -o root -g root "${tmp_cron}" /etc/cron.d/tenbackward
/app/crontab/install-cron.sh /etc/cron.d/tenbackward
echo "tenbackward: starting cron (RUN_AT=${run_at}, TZ=${TZ})"
exec cron -f
+3
View File
@@ -0,0 +1,3 @@
[pytest]
testpaths = tests
pythonpath = src
+1
View File
@@ -0,0 +1 @@
pytest==8.3.3
+4
View File
@@ -0,0 +1,4 @@
python-dotenv==1.0.1
python-frontmatter==1.1.0
GitPython==3.1.43
Mastodon.py==1.8.0
+14
View File
@@ -0,0 +1,14 @@
#!/bin/bash
set -euo pipefail
cd /app
bot_uid="$(id -u bot)"
bot_gid="$(id -g bot)"
if command -v setpriv >/dev/null 2>&1; then
exec setpriv --reuid="${bot_uid}" --regid="${bot_gid}" --clear-groups -- \
/usr/local/bin/python -m tenbackward
fi
exec su -s /bin/bash bot -c "exec /usr/local/bin/python -m tenbackward"
+42
View File
@@ -0,0 +1,42 @@
#!/bin/bash
set -euo pipefail
export DEBIAN_FRONTEND=noninteractive
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
cd "${SCRIPT_DIR}"
if [ -w /usr/local/bin ] && [ ! -e /usr/local/bin/build.sh ]; then
ln -s "${SCRIPT_DIR}/build.sh" /usr/local/bin/build.sh || true
fi
if [ -w /usr/local/bin ] && [ ! -e /usr/local/bin/setup.sh ]; then
ln -s "${SCRIPT_DIR}/setup.sh" /usr/local/bin/setup.sh || true
fi
apt-get update
apt-get install -y --no-install-recommends \
bash \
git \
ca-certificates \
python3 \
python3-pip \
python3-venv \
tzdata
rm -rf /var/lib/apt/lists/*
PYTHON_BIN="$(command -v python3)"
PIP_BIN="$(command -v pip3 || true)"
if [ -z "${PIP_BIN}" ] && [ -n "${PYTHON_BIN}" ]; then
PIP_BIN="${PYTHON_BIN} -m pip"
fi
if [ -n "${PIP_BIN}" ]; then
${PIP_BIN} install --no-cache-dir --upgrade pip >/dev/null 2>&1 || true
${PIP_BIN} install --no-cache-dir --break-system-packages -r /repo/requirements.txt || \
${PIP_BIN} install --no-cache-dir -r /repo/requirements.txt
${PIP_BIN} install --no-cache-dir --break-system-packages -r /repo/requirements-dev.txt || \
${PIP_BIN} install --no-cache-dir -r /repo/requirements-dev.txt
fi
cd /repo
PYTHONPATH=/repo/src pytest -q
+3
View File
@@ -0,0 +1,3 @@
"""10Backward — Mastodon daily-throwback bot."""
__version__ = "0.1.0"
+3
View File
@@ -0,0 +1,3 @@
from .main import main
raise SystemExit(main())
+229
View File
@@ -0,0 +1,229 @@
from __future__ import annotations
import logging
import os
import time
from pathlib import Path
from typing import Callable
import git
from git import GitCommandError, InvalidGitRepositoryError, NoSuchPathError, Repo
DEFAULT_REPO_URL = "https://git.chaospott.de/Chaospott/site"
DEFAULT_BLOG_SUBDIR = "blog"
MAX_RETRIES_DEFAULT = 5
BACKOFF_BASE_SECONDS = 2
REMOTE_NAME = "origin"
BRANCH_NAME = "master"
_LOCAL_MODIFICATION_MARKERS = (
"Your local changes",
"would be overwritten",
"Please commit your changes",
"Please move or remove them",
)
class BlogRepoError(RuntimeError):
"""Raised when the local blog working tree cannot be ensured.
Surface this as a fatal pipeline error (do not retry) so configuration
problems fail immediately and transient network failures get a chance to
recover via the retry loop.
"""
class BlogTransientError(RuntimeError):
"""Wraps a transient HTTPS clone/pull failure that should be retried.
``operation`` records which Git action triggered the failure so the
runner can label its log lines with "git clone" or "git pull".
"""
def __init__(self, message: str, *, operation: str = "git pull") -> None:
super().__init__(message)
self.operation = operation
def blog_dir(data_dir: Path) -> Path:
return Path(data_dir) / DEFAULT_BLOG_SUBDIR
def blog_repo_url() -> str:
return os.environ.get("BLOG_REPO_URL") or DEFAULT_REPO_URL
def _is_dir_empty(path: Path) -> bool:
if not path.exists():
return True
try:
next(path.iterdir())
except StopIteration:
return True
return False
def _is_local_modification_error(exc: GitCommandError) -> bool:
message = str(exc)
return any(marker in message for marker in _LOCAL_MODIFICATION_MARKERS)
def _log_retry(
logger: logging.Logger | None,
*,
operation: str,
attempt: int,
error: BaseException,
) -> None:
if logger is None:
return
logger.error(
"blog_retry",
extra={
"event": "blog_retry",
"operation": operation,
"attempt": attempt,
"error": str(error),
},
)
def _retry_loop(
*,
operation: str,
attempts: int,
sleep: Callable[[float], None],
logger: logging.Logger | None,
action: Callable[[], None],
) -> None:
"""Invoke ``action`` up to ``attempts`` times with ``2 ** (attempt - 1)``
second backoff between failures.
Transient errors are retried; :class:`BlogRepoError` (fatal local or
validation problems) propagates immediately. On exhaustion, raises
:class:`BlogTransientError` carrying the failing operation label.
"""
last_exc: BaseException | None = None
for attempt in range(1, attempts + 1):
try:
action()
return
except BlogRepoError:
raise
except Exception as exc: # noqa: BLE001 — boundary retry hook
last_exc = exc
_log_retry(logger, operation=operation, attempt=attempt, error=exc)
if attempt < attempts:
sleep(BACKOFF_BASE_SECONDS ** (attempt - 1))
assert last_exc is not None
raise BlogTransientError(
f"{operation} failed after {attempts} attempt(s): {last_exc}",
operation=operation,
) from last_exc
def ensure_repo(
data_dir: Path,
*,
repo_url: str | None = None,
blog_path: Path | None = None,
max_retries: int | None = None,
logger: logging.Logger | None = None,
sleep: Callable[[float], None] = time.sleep,
clone_impl: Callable[[str, str], Repo] | None = None,
fetch_impl: Callable[[Repo], Callable[[], object]] | None = None,
pull_impl: Callable[[Repo], Callable[[], object]] | None = None,
) -> Repo:
"""Ensure a local working clone of the blog repository exists and is up to date.
Returns the :class:`git.Repo` pointing at an up-to-date working tree.
Raises :class:`BlogRepoError` on irrecoverable failures and
:class:`BlogTransientError` after a transient failure exhausts the
retry budget (so the caller can label and re-attempt at the pipeline
level).
"""
url = repo_url or blog_repo_url()
target = blog_path or blog_dir(data_dir)
retries = MAX_RETRIES_DEFAULT if max_retries is None else max_retries
attempts = max(1, retries)
target.parent.mkdir(parents=True, exist_ok=True)
needs_clone = _is_dir_empty(target)
if needs_clone:
if logger is not None:
logger.info(
"blog_clone_start",
extra={"event": "blog_clone_start", "url": url, "path": str(target)},
)
cloned_holder: dict[str, Repo] = {}
def _do_clone() -> None:
r = (clone_impl or Repo.clone_from)(url, str(target))
cloned_holder["repo"] = r
_retry_loop(
operation="git clone",
attempts=attempts,
sleep=sleep,
logger=logger,
action=_do_clone,
)
if logger is not None:
logger.info(
"blog_clone_complete",
extra={"event": "blog_clone_complete", "path": str(target)},
)
if "repo" in cloned_holder:
return cloned_holder["repo"]
return Repo(str(target))
try:
repo = Repo(str(target))
except (InvalidGitRepositoryError, NoSuchPathError) as exc:
raise BlogRepoError(
f"{target} exists but is not a valid git repository: {exc}"
) from exc
fetch = (fetch_impl or _default_fetch)(repo)
pull = (pull_impl or _default_pull)(repo)
def _do_pull() -> None:
try:
fetch()
pull()
except GitCommandError as exc:
if _is_local_modification_error(exc):
if logger is not None:
logger.error(
"blog_local_modifications",
extra={"event": "blog_local_modifications", "error": str(exc)},
)
raise BlogRepoError(
"local modifications detected in blog working tree; aborting run"
) from exc
raise BlogTransientError(str(exc), operation="git pull") from exc
_retry_loop(
operation="git pull",
attempts=attempts,
sleep=sleep,
logger=logger,
action=_do_pull,
)
return repo
def _default_fetch(repo: Repo) -> Callable[[], object]:
def _do() -> object:
return repo.remotes[REMOTE_NAME].fetch()
return _do
def _default_pull(repo: Repo) -> Callable[[], object]:
def _do() -> object:
return repo.git.pull("--ff-only")
return _do
+267
View File
@@ -0,0 +1,267 @@
from __future__ import annotations
import os
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional
from urllib.parse import urlparse
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from dotenv import dotenv_values, load_dotenv
REQUIRED_KEYS = (
"MASTODON_BASE_URL",
"MASTODON_ACCESS_TOKEN",
"MASTODON_VISIBILITY",
"SITE_URL",
"HASHTAGS",
"THROWBACK_PREFIX",
"MAX_RETRIES",
"RUN_AT",
"TZ",
)
OPTIONAL_KEYS: tuple[str, ...] = ("BLOG_REPO_URL", "BLOG_DIR")
DEFAULTS = {
"MAX_RETRIES": "5",
"BLOG_REPO_URL": "https://git.chaospott.de/Chaospott/site",
"BLOG_DIR": "blog",
}
ALLOWED_VISIBILITY = frozenset({"public", "unlisted"})
class ConfigError(ValueError):
"""Raised when required configuration is missing or invalid."""
@dataclass(frozen=True)
class Config:
mastodon_base_url: str
mastodon_access_token: str
mastodon_visibility: str
site_url: str
hashtags: str
throwback_prefix: str
max_retries: int
run_at: str
tz: str
data_dir: Path = field(default_factory=lambda: Path("/app/data"))
blog_repo_url: str = "https://git.chaospott.de/Chaospott/site"
blog_dir: Path = field(default_factory=lambda: Path("/app/data/blog"))
extra: dict = field(default_factory=dict)
@property
def cron_minute(self) -> str:
return self.run_at.split(":", 1)[0]
@property
def cron_hour(self) -> str:
return self.run_at.split(":", 1)[1]
DEFAULT_DOTENV_PATH = Path("/app/.env")
def _read_dotenv(dotenv_path: Optional[Path]) -> dict[str, str]:
if dotenv_path is None:
return {}
if not dotenv_path.exists():
return {}
values = dotenv_values(dotenv_path=str(dotenv_path))
return {k: v for k, v in values.items() if v is not None}
def _values_from_env() -> dict[str, str]:
values: dict[str, str] = {}
for key in REQUIRED_KEYS + OPTIONAL_KEYS:
raw = os.environ.get(key)
if raw is not None and raw != "":
values[key] = raw
return values
def load_config(dotenv_path: Optional[Path] = None) -> Config:
"""Load configuration from a dotenv file and the process environment.
The .env file is loaded via python-dotenv at startup so the bind-mounted
runtime secrets are visible to the process. Environment variables take
precedence over the dotenv file so a mounted `.env` can be supplemented
by Compose-level overrides.
Required keys are validated against the **raw** merged map (dotenv +
process env) before any defaults are applied. Defaults are only used
for the retry budget and the optional Jekyll blog source so a missing
Mastodon/secret/URL/etc. fails fast with a clear error.
"""
if dotenv_path is None:
dotenv_path = DEFAULT_DOTENV_PATH if DEFAULT_DOTENV_PATH.exists() else None
if dotenv_path is not None:
load_dotenv(dotenv_path=str(dotenv_path), override=False)
file_values = _read_dotenv(dotenv_path)
env_values = _values_from_env()
raw: dict[str, str] = {}
raw.update(file_values)
raw.update(env_values)
# MAX_RETRIES is the only required key that has a permitted default
# (5 attempts per the Job). Apply it before the required-key check so a
# missing value uses the documented default rather than failing fast.
if not raw.get("MAX_RETRIES"):
raw["MAX_RETRIES"] = DEFAULTS["MAX_RETRIES"]
validate_required(raw)
merged = dict(raw)
apply_defaults(merged)
validate_config(merged)
data_dir = Path(os.environ.get("DATA_DIR", "/app/data")).resolve()
blog_dir_raw = Path(merged["BLOG_DIR"]).expanduser()
blog_dir = (
blog_dir_raw
if blog_dir_raw.is_absolute()
else (data_dir / blog_dir_raw).resolve()
)
extra = {k: v for k, v in merged.items() if k not in REQUIRED_KEYS + OPTIONAL_KEYS}
return Config(
mastodon_base_url=merged["MASTODON_BASE_URL"],
mastodon_access_token=merged["MASTODON_ACCESS_TOKEN"],
mastodon_visibility=merged["MASTODON_VISIBILITY"],
site_url=merged["SITE_URL"],
hashtags=merged["HASHTAGS"],
throwback_prefix=merged["THROWBACK_PREFIX"],
max_retries=_parse_max_retries(merged["MAX_RETRIES"]),
run_at=merged["RUN_AT"],
tz=merged["TZ"],
data_dir=data_dir,
blog_repo_url=merged["BLOG_REPO_URL"],
blog_dir=blog_dir,
extra=extra,
)
def apply_defaults(values: dict[str, str]) -> None:
for key, default in DEFAULTS.items():
values.setdefault(key, default)
def validate_required(values: dict[str, str]) -> None:
"""Validate that every required key is present and non-empty in ``values``.
Called against the raw (no-default) merged map so a missing key always
surfaces as a startup error rather than being silently substituted.
"""
errors: list[str] = []
for key in REQUIRED_KEYS:
raw = values.get(key, "")
if not raw:
errors.append(f"missing required configuration key: {key}")
if errors:
raise ConfigError("; ".join(errors))
def validate_config(values: dict[str, str]) -> None:
errors: list[str] = []
run_at = values.get("RUN_AT", "")
if run_at and not _is_valid_hhmm(run_at):
errors.append(f"RUN_AT={run_at!r} must be in HH:MM (24-hour) format")
base_url = values.get("MASTODON_BASE_URL", "")
if base_url and not _is_valid_url(base_url):
errors.append(f"MASTODON_BASE_URL={base_url!r} must be a valid http(s) URL")
site_url = values.get("SITE_URL", "")
if site_url and not _is_valid_url(site_url):
errors.append(f"SITE_URL={site_url!r} must be a valid http(s) URL")
visibility = values.get("MASTODON_VISIBILITY", "")
if visibility and not _is_valid_visibility(visibility):
errors.append(
f"MASTODON_VISIBILITY={visibility!r} must be one of: {sorted(ALLOWED_VISIBILITY)}"
)
blog_repo_url = values.get("BLOG_REPO_URL", "")
if blog_repo_url and not _is_valid_url(blog_repo_url):
errors.append(
f"BLOG_REPO_URL={blog_repo_url!r} must be a valid http(s) URL"
)
tz = values.get("TZ", "")
if tz and not _is_valid_tz(tz):
errors.append(f"TZ={tz!r} must be a valid IANA timezone")
max_retries = values.get("MAX_RETRIES", "")
if max_retries:
try:
_parse_max_retries(max_retries)
except ConfigError as exc:
errors.append(str(exc))
if errors:
raise ConfigError("; ".join(errors))
def _is_valid_hhmm(value) -> bool:
if not isinstance(value, str):
return False
parts = value.split(":")
if len(parts) != 2:
return False
hour, minute = parts
if len(hour) != 2 or len(minute) != 2:
return False
if not hour.isdigit() or not minute.isdigit():
return False
h = int(hour)
m = int(minute)
return 0 <= h <= 23 and 0 <= m <= 59
def _is_valid_url(value: str) -> bool:
if not isinstance(value, str) or not value:
return False
try:
parsed = urlparse(value)
except (TypeError, ValueError):
return False
return parsed.scheme in {"http", "https"} and bool(parsed.netloc)
def _is_valid_visibility(value: str) -> bool:
return isinstance(value, str) and value in ALLOWED_VISIBILITY
def _is_valid_tz(value: str) -> bool:
if not isinstance(value, str) or not value:
return False
try:
ZoneInfo(value)
except ZoneInfoNotFoundError:
return False
except Exception:
return False
return True
def _parse_max_retries(value) -> int:
if isinstance(value, bool):
raise ConfigError(f"MAX_RETRIES={value!r} must be a non-negative integer")
try:
count = int(value)
except (TypeError, ValueError):
raise ConfigError(f"MAX_RETRIES={value!r} must be a non-negative integer")
if count < 0:
raise ConfigError(f"MAX_RETRIES={value!r} must be >= 0")
return count
+175
View File
@@ -0,0 +1,175 @@
from __future__ import annotations
import json
import logging
from datetime import datetime, timezone
from typing import Any
from .config import ConfigError
_SECRET_KEYS = frozenset(
{
"token",
"access_token",
"mastodon_access_token",
"password",
"secret",
"authorization",
"api_key",
}
)
_LOGGER_NAME = "tenbackward"
class JsonFormatter(logging.Formatter):
"""Emit one JSON object per log record.
Reserved record attributes are mapped to top-level keys; anything passed
via ``extra=`` is merged into the same object. Keys whose name appears in
``_SECRET_KEYS`` are redacted before formatting.
"""
_RESERVED = {
"name",
"msg",
"args",
"levelname",
"levelno",
"pathname",
"filename",
"module",
"exc_info",
"exc_text",
"stack_info",
"lineno",
"funcName",
"created",
"msecs",
"relativeCreated",
"thread",
"threadName",
"processName",
"process",
"message",
"asctime",
}
def format(self, record: logging.LogRecord) -> str: # noqa: A003
message = record.getMessage()
payload: dict[str, Any] = {
"ts": datetime.fromtimestamp(record.created, tz=timezone.utc).isoformat(),
"level": record.levelname,
"logger": record.name,
"message": message,
}
for key, value in record.__dict__.items():
if key in self._RESERVED or key.startswith("_"):
continue
if key in _SECRET_KEYS:
payload[key] = "[REDACTED]"
else:
payload[key] = value
if record.exc_info:
payload["exc_type"] = record.exc_info[0].__name__ if record.exc_info[0] else None
return json.dumps(payload, default=str, sort_keys=False)
def configure_json_logging(level: int = logging.INFO) -> None:
"""Install the JSON formatter on the root logger.
Idempotent: safe to call multiple times (replaces any existing handler).
"""
handler = logging.StreamHandler()
handler.setFormatter(JsonFormatter())
handler.setLevel(level)
root = logging.getLogger()
root.setLevel(level)
for existing in list(root.handlers):
root.removeHandler(existing)
root.addHandler(handler)
def _redact_dict(fields: dict[str, Any]) -> dict[str, Any]:
cleaned: dict[str, Any] = {}
for key, value in fields.items():
if key.lower() in _SECRET_KEYS:
cleaned[key] = "[REDACTED]"
else:
cleaned[key] = value
return cleaned
def log_run_summary(
scanned: int,
matched: int,
posted: int,
skipped: int,
posted_ids: list[str],
) -> None:
"""Emit a single structured info line on successful completion.
A run with no matching posts is silent — no log line is emitted.
"""
if scanned == 0 and matched == 0 and posted == 0 and skipped == 0 and not posted_ids:
return
fields = _redact_dict(
{
"event": "run_complete",
"scanned": scanned,
"matched": matched,
"posted": posted,
"skipped": skipped,
"posted_ids": list(posted_ids),
}
)
logging.getLogger(_LOGGER_NAME).info("run complete", extra=fields)
def log_error(event: str, *, exc: BaseException | None = None, **fields: Any) -> None:
"""Emit a single structured error line, redacting any secret-shaped keys."""
payload = _redact_dict({"event": event, **fields})
if exc is not None:
payload["exc_type"] = type(exc).__name__
logging.getLogger(_LOGGER_NAME).error(event, extra=payload)
def log_event(event: str, **fields: Any) -> None:
"""Emit a structured event line at the level appropriate to ``event``.
Unlike :func:`log_error`, this helper does not attach ``exc_type``; it
forwards arbitrary fields (including ``error=str(exc)``) through the
same redaction path so callers can include the underlying exception
text without leaking secret-shaped keys.
"""
payload = _redact_dict({"event": event, **fields})
logging.getLogger(_LOGGER_NAME).error(event, extra=payload)
def log_warning(event: str, **fields: Any) -> None:
"""Emit a single structured warning line, redacting any secret-shaped keys."""
payload = _redact_dict({"event": event, **fields})
logging.getLogger(_LOGGER_NAME).warning(event, extra=payload)
def log_startup(**fields: Any) -> None:
"""Emit a single structured info line at startup."""
payload = _redact_dict({"event": "startup", **fields})
logging.getLogger(_LOGGER_NAME).info("startup", extra=payload)
__all__ = [
"ConfigError",
"JsonFormatter",
"configure_json_logging",
"log_error",
"log_event",
"log_run_summary",
"log_startup",
"log_warning",
]
+181
View File
@@ -0,0 +1,181 @@
from __future__ import annotations
import logging
import sys
import time
from typing import Callable, Iterable
from . import __version__
from .blog import BlogRepoError, BlogTransientError, ensure_repo
from .config import Config, ConfigError, load_config
from .logging_setup import (
configure_json_logging,
log_error,
log_event,
log_run_summary,
log_startup,
)
from .matching import MatchedPost, find_anniversary_matches
from .publishing import PublishError, PublishFatalError, publish_mastodon
from .state import PostedStore
_logger = logging.getLogger("tenbackward")
def _iter_candidates(config: Config) -> Iterable[MatchedPost]:
"""Yield candidate :class:`MatchedPost` objects whose anniversary is
exactly 10 years before today.
"""
post_root = config.blog_dir / "_posts" / "blog"
return find_anniversary_matches(post_root, config.site_url)
def _log_retry_attempt(operation: str, attempt: int, exc: BaseException) -> None:
log_event(
"retry_attempt",
operation=operation,
attempt=attempt,
error=str(exc),
exc_type=type(exc).__name__,
)
def _log_final_failure(operation: str, exc: BaseException, attempts: int) -> None:
log_error(
"retry_exhausted",
exc=exc,
operation=operation,
attempts=attempts,
error=str(exc),
)
def _operation_label(exc: BaseException) -> str:
if isinstance(exc, BlogTransientError):
return exc.operation
if isinstance(exc, PublishError):
return "mastodon post"
if isinstance(exc, BlogRepoError):
return "git pull"
return "pipeline"
def _run_with_retry(
config: Config,
*,
sleep: Callable[[float], None] = time.sleep,
logger: logging.Logger | None = None,
) -> tuple[int, int, int, int, list[str]] | None:
"""Execute the pipeline with retries. Returns the summary counters on
success or ``None`` when the retry budget is exhausted.
A single retry budget governs both the Git and Mastodon operations —
``ensure_repo`` is invoked with ``max_retries=0`` so the runner, not the
blog layer, decides whether to re-attempt. Configuration and fatal
validation errors (``ConfigError``, :class:`BlogRepoError`,
:class:`PublishFatalError`) are not retried.
"""
attempts = max(1, config.max_retries)
active_logger = logger or _logger
last_exc: BaseException | None = None
for attempt in range(1, attempts + 1):
try:
return _run_once(config, logger=active_logger, sleep=sleep)
except (ConfigError, BlogRepoError, PublishFatalError) as exc:
last_exc = exc
_log_final_failure(type(exc).__name__, exc, attempt)
return None
except (BlogTransientError, PublishError) as exc:
last_exc = exc
_log_retry_attempt(_operation_label(exc), attempt, exc)
if attempt < attempts:
sleep(2 ** (attempt - 1))
except Exception as exc: # noqa: BLE001 — opportunistic retry boundary
last_exc = exc
_log_retry_attempt("pipeline", attempt, exc)
if attempt < attempts:
sleep(2 ** (attempt - 1))
assert last_exc is not None
_log_final_failure(_operation_label(last_exc), last_exc, attempts)
return None
def _run_once(
config: Config,
*,
logger: logging.Logger | None = None,
sleep: Callable[[float], None] = time.sleep,
) -> tuple[int, int, int, int, list[str]]:
"""Execute one pipeline pass and return the summary counters.
Returns ``(scanned, matched, posted, skipped, posted_ids)``.
The blog layer's retry budget is disabled here (``max_retries=0``);
the runner owns the unified retry policy so we never nest two retry
budgets back-to-back.
"""
ensure_repo(
config.data_dir,
repo_url=config.blog_repo_url,
blog_path=config.blog_dir,
max_retries=0,
logger=logger,
sleep=sleep,
)
store = PostedStore(config.data_dir)
candidates = list(_iter_candidates(config))
scanned = len(candidates)
matched = len(candidates)
unposted: list[MatchedPost] = []
for match in candidates:
if store.is_posted(match.path):
continue
unposted.append(match)
skipped = matched - len(unposted)
posted_ids = [m.path for m in unposted]
if unposted:
publish_mastodon(config, unposted)
store.mark_posted_many(posted_ids)
posted = len(unposted)
return scanned, matched, posted, skipped, posted_ids
def main() -> int:
configure_json_logging()
try:
config = load_config()
except ConfigError as exc:
log_error("configuration_error", exc=exc)
return 2
config.data_dir.mkdir(parents=True, exist_ok=True)
config.blog_dir.mkdir(parents=True, exist_ok=True)
log_startup(
version=__version__,
site_url=config.site_url,
run_at=config.run_at,
tz=config.tz,
hashtags=config.hashtags,
throwback_prefix=config.throwback_prefix,
)
result = _run_with_retry(config, logger=_logger)
if result is None:
return 1
scanned, matched, posted, skipped, posted_ids = result
log_run_summary(scanned, matched, posted, skipped, posted_ids)
return 0
if __name__ == "__main__":
sys.exit(main())
+230
View File
@@ -0,0 +1,230 @@
from __future__ import annotations
import calendar
import logging
import re
import unicodedata
from dataclasses import dataclass
from datetime import date, datetime
from pathlib import Path
from zoneinfo import ZoneInfo
import frontmatter
_LOGGER = logging.getLogger("tenbackward.matching")
_FILENAME_PATTERN = re.compile(r"^(\d{4})-(\d{2})-(\d{2})-(.+)$")
_NON_ALNUM_RE = re.compile(r"[^a-z0-9]+")
_DEFAULT_TZ = "Europe/Berlin"
@dataclass(frozen=True)
class MatchedPost:
path: str
title: str
date: date
url: str
def slugify_title(title: str) -> str:
"""Return a URL-safe slug derived from ``title``.
Used by the matcher to derive the slug component of
:attr:`MatchedPost.url` from the resolved post title.
"""
normalized = unicodedata.normalize("NFKD", title)
ascii_only = normalized.encode("ascii", "ignore").decode("ascii")
lowered = ascii_only.lower()
dashed = _NON_ALNUM_RE.sub("-", lowered)
return dashed.strip("-")
def _today_in_berlin(tz_name: str = _DEFAULT_TZ) -> date:
return datetime.now(ZoneInfo(tz_name)).date()
def _is_leap_year(year: int) -> bool:
return calendar.isleap(year)
def _try_parse_date(value: object) -> date | None:
if isinstance(value, datetime):
return value.date()
if isinstance(value, date):
return value
return None
def _safe_filename_match(stem: str) -> tuple[int, int, int, str] | None:
match = _FILENAME_PATTERN.match(stem)
if match is None:
return None
year_str, month_str, day_str, slug = match.groups()
try:
year = int(year_str)
month = int(month_str)
day = int(day_str)
except ValueError:
return None
try:
date(year, month, day)
except ValueError:
return None
if not slug:
return None
return year, month, day, slug
def _build_url(site_url: str, post_date: date, slug: str) -> str:
base = site_url.rstrip("/")
return f"{base}/{post_date.year:04d}/{post_date.month:02d}/{post_date.day:02d}/{slug}/"
def _is_unpublished(metadata: dict) -> bool:
if "published" not in metadata:
return False
return metadata["published"] is False
def _matches_anniversary(post_date: date, target_year: int, today: date) -> bool:
if post_date.month == 2 and post_date.day == 29:
if _is_leap_year(today.year):
return today.month == 2 and today.day == 29
return (
post_date.year == target_year
and today.month == 3
and today.day == 1
)
if post_date.year != target_year:
return False
return post_date.month == today.month and post_date.day == today.day
def _process_file(
md_path: Path,
post_root: Path,
site_url: str,
target_year: int,
today: date,
) -> MatchedPost | None:
rel = md_path.relative_to(post_root).as_posix()
try:
with md_path.open("r", encoding="utf-8") as fh:
text = fh.read()
except (OSError, UnicodeDecodeError) as exc:
_LOGGER.warning(
"skipping unreadable post file",
extra={"event": "post_unreadable", "path": rel, "error": type(exc).__name__},
)
return None
title: str | None = None
post_date: date | None = None
unpublished = False
frontmatter_parsed = False
try:
parsed = frontmatter.loads(text)
frontmatter_parsed = True
metadata = parsed.metadata if isinstance(parsed.metadata, dict) else {}
if isinstance(metadata.get("title"), str) and metadata["title"].strip():
title = metadata["title"]
post_date = _try_parse_date(metadata.get("date"))
unpublished = _is_unpublished(metadata)
except Exception as exc: # noqa: BLE001 — frontmatter is opaque
_LOGGER.warning(
"frontmatter parse failed",
extra={"event": "post_frontmatter_error", "path": rel, "error": type(exc).__name__},
)
slug_match = _safe_filename_match(md_path.stem)
if slug_match is None:
if not frontmatter_parsed:
return None
_LOGGER.warning(
"filename missing required YYYY-MM-DD-slug pattern",
extra={"event": "post_filename_invalid", "path": rel},
)
return None
fn_year, fn_month, fn_day, fn_slug = slug_match
if post_date is None:
post_date = date(fn_year, fn_month, fn_day)
if not _matches_anniversary(post_date, target_year, today):
return None
if unpublished:
_LOGGER.warning(
"skipping unpublished post",
extra={"event": "post_unpublished", "path": rel},
)
return None
if title is None:
title = fn_slug
return MatchedPost(
path=rel,
title=title,
date=post_date,
url=_build_url(site_url, post_date, slugify_title(title)),
)
def find_anniversary_matches(
post_root: Path,
site_url: str,
*,
today: date | None = None,
) -> list[MatchedPost]:
"""Walk the Jekyll post directory and return posts whose anniversary
date is exactly 10 years before ``today`` (per the leap-day rules).
"""
if today is None:
today = _today_in_berlin()
target_year = today.year - 10
if not post_root.exists():
return []
matches: list[MatchedPost] = []
for md_path in sorted(post_root.rglob("*.md")):
if not md_path.is_file():
continue
try:
result = _process_file(md_path, post_root, site_url, target_year, today)
except Exception as exc: # noqa: BLE001 — never abort the scan
rel = md_path.relative_to(post_root).as_posix()
_LOGGER.warning(
"unexpected error processing post",
extra={
"event": "post_processing_error",
"path": rel,
"error": type(exc).__name__,
},
)
continue
if result is not None:
matches.append(result)
return matches
def iter_anniversary_paths(post_root: Path, site_url: str, *, today: date | None = None):
"""Yield each matched post's relative ``path`` (the dedupe identifier)."""
for match in find_anniversary_matches(post_root, site_url, today=today):
yield match.path
__all__ = [
"MatchedPost",
"find_anniversary_matches",
"iter_anniversary_paths",
"slugify_title",
]
+164
View File
@@ -0,0 +1,164 @@
"""Mastodon publishing helpers and the API success boundary.
The :func:`publish_mastodon` function is the single boundary between the
pipeline orchestrator and the Mastodon HTTP API. Status composition and
length validation are pure and isolated from network access so they can
be tested without a server.
"""
from __future__ import annotations
from typing import Callable
from mastodon import Mastodon
from .config import Config
from .matching import MatchedPost, slugify_title
MASTODON_STATUS_LIMIT = 500
class PublishError(RuntimeError):
"""Raised when status composition, validation, or the Mastodon API
call fails. The pipeline orchestrator treats this like any other
pipeline error and lets the retry budget decide whether to give up.
"""
class PublishFatalError(PublishError):
"""Raised for fatal configuration or validation failures (missing token,
invalid instance URL, malformed status). The pipeline orchestrator must
not retry these — they will fail the same way on every attempt.
"""
def _normalize_hashtags(hashtags: str) -> str:
parts = [token.strip() for token in (hashtags or "").split(",")]
parts = [token for token in parts if token]
return " ".join(parts)
def build_status_text(
prefix: str,
posts: list[MatchedPost],
hashtags: str,
) -> str:
"""Compose a single Mastodon status string for ``posts``.
The format is::
{prefix}
{title1}
{url1}
{title2}
{url2}
...
{hashtags}
Posts are sorted by ``(date, path)`` for deterministic output
regardless of the upstream ordering.
"""
if not posts:
raise PublishFatalError("empty_posts: cannot compose status without posts")
ordered = sorted(posts, key=lambda m: (m.date, m.path))
blocks: list[str] = []
for match in ordered:
blocks.append(f"{match.title}\n{match.url}")
body = "\n\n".join(blocks)
lines: list[str] = [prefix.strip(), body]
tag_line = _normalize_hashtags(hashtags)
if tag_line:
lines.append(tag_line)
return "\n\n".join(lines) + "\n"
def validate_status(status: str, limit: int = MASTODON_STATUS_LIMIT) -> None:
"""Raise :class:`PublishFatalError` when ``status`` exceeds ``limit``.
Never truncates: the spec requires failing safely rather than
shortening content.
"""
if len(status) > limit:
raise PublishFatalError(
f"status_too_long: len={len(status)} limit={limit}"
)
def _validate_publish_config(config: Config) -> None:
if not config.mastodon_base_url or not config.mastodon_base_url.startswith(
("http://", "https://")
):
raise PublishFatalError(
f"publish_failed: mastodon auth: invalid MASTODON_BASE_URL={config.mastodon_base_url!r}"
)
if not config.mastodon_access_token:
raise PublishFatalError(
"publish_failed: mastodon auth: MASTODON_ACCESS_TOKEN is empty"
)
def _post_status_via_mastodon_py(
status: str,
*,
base_url: str,
access_token: str,
visibility: str,
client_factory: Callable[..., Mastodon] | None = None,
) -> None:
factory = client_factory if client_factory is not None else Mastodon
client = factory(access_token=access_token, api_base_url=base_url)
client.status_post(status, visibility=visibility)
def publish_mastodon(
config: Config,
posts: list[MatchedPost],
*,
client_factory: Callable[..., Mastodon] | None = None,
) -> str:
"""Compose, validate, and publish ``posts`` to Mastodon.
Returns the composed status text on success. Raises
:class:`PublishFatalError` for configuration/validation failures (no
retry), or :class:`PublishError` (transient) for network/HTTP errors
that the runner may retry.
"""
_validate_publish_config(config)
try:
status = build_status_text(
config.throwback_prefix, posts, config.hashtags
)
validate_status(status)
_post_status_via_mastodon_py(
status,
base_url=config.mastodon_base_url,
access_token=config.mastodon_access_token,
visibility=config.mastodon_visibility,
client_factory=client_factory,
)
except PublishFatalError:
raise
except Exception as exc: # noqa: BLE001 — third-party boundary
raise PublishError(
f"publish_failed: mastodon post: {type(exc).__name__}: {exc}"
) from exc
return status
__all__ = [
"MASTODON_STATUS_LIMIT",
"PublishError",
"PublishFatalError",
"build_status_text",
"publish_mastodon",
"slugify_title",
"validate_status",
]
+208
View File
@@ -0,0 +1,208 @@
from __future__ import annotations
import fcntl
import json
import os
from pathlib import Path
from typing import Iterable
from .logging_setup import log_warning
DEFAULT_DATA_DIR = Path("./data")
_FILE_NAME = "posted.json"
_EMPTY_DOCUMENT: dict[str, list[str]] = {"posted": []}
class PostedStoreError(RuntimeError):
"""Raised when the posted-state store cannot be used."""
def posted_path(data_dir: Path | None = None) -> Path:
"""Resolve the path of ``posted.json``.
When ``data_dir`` is ``None``, the directory is read from the
``DATA_DIR`` environment variable, falling back to ``./data``
(the host bind-mount described in the README).
"""
if data_dir is None:
data_dir = Path(os.environ.get("DATA_DIR", str(DEFAULT_DATA_DIR)))
return Path(data_dir) / _FILE_NAME
def _coerce_list(value: object) -> list[str]:
"""Filter ``value`` down to a list of strings, dropping anything else."""
if not isinstance(value, list):
return []
return [item for item in value if isinstance(item, str)]
def _read_existing(path: Path) -> dict[str, list[str]] | None:
"""Return the parsed JSON document at ``path`` or ``None`` on errors.
A missing file is **not** an error: returns ``None`` so the caller
can treat it as the empty document. Malformed JSON or unreadable
bytes return ``None`` and a warning is emitted.
"""
try:
with path.open("r", encoding="utf-8") as fh:
data = json.load(fh)
except FileNotFoundError:
return None
except (json.JSONDecodeError, OSError) as exc:
log_warning("posted_store_corrupt", path=str(path), error=type(exc).__name__)
return None
if not isinstance(data, dict):
return None
return data
def _ensure_parent(path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
def _write_document(path: Path, document: dict[str, list[str]]) -> None:
"""Atomically write ``document`` to ``path`` via temp-file + rename."""
_ensure_parent(path)
pid = os.getpid()
tmp = path.with_name(f"{path.name}.tmp.{pid}")
try:
with tmp.open("w", encoding="utf-8") as fh:
json.dump(document, fh, sort_keys=False, indent=2)
fh.flush()
os.fsync(fh.fileno())
os.replace(tmp, path)
except Exception:
try:
tmp.unlink()
except FileNotFoundError:
pass
raise
def _lock_path(path: Path):
"""Return an open file descriptor on the lock file inside the data dir.
The lock file lives in the same directory as ``posted.json`` so the
flock is always on a single inode, regardless of whether the JSON
file currently exists. The descriptor is opened in append mode so
concurrent readers never truncate it.
"""
_ensure_parent(path)
lock = path.with_name(f".{path.name}.lock")
fd = os.open(str(lock), os.O_CREAT | os.O_RDWR, 0o644)
return lock, fd
class PostedStore:
"""Self-contained dedup store persisted as ``posted.json``.
The on-disk shape is::
{"posted": ["2014/2014-08-04-foo.md", ...]}
All mutating operations acquire an exclusive :mod:`fcntl` flock on a
sibling lock file so concurrent container runs cannot corrupt the
JSON document. The store never raises on a missing file, an
unreadable file, or an already-present path — it always reports
success in a way that the orchestrator can act on.
"""
def __init__(self, data_dir: Path | None = None) -> None:
self._path = posted_path(data_dir)
@property
def path(self) -> Path:
return self._path
def load(self) -> list[str]:
"""Read the posted list, creating an empty file when missing."""
_ensure_parent(self._path)
document = _read_existing(self._path)
if document is None:
if not self._path.exists():
_write_document(self._path, dict(_EMPTY_DOCUMENT))
return []
return _coerce_list(document.get("posted"))
def is_posted(self, relative_path: str) -> bool:
"""Return ``True`` when ``relative_path`` is already recorded."""
posted = self.load()
return relative_path in posted
def _with_lock(self, mutate):
lock_path, fd = _lock_path(self._path)
try:
fcntl.flock(fd, fcntl.LOCK_EX)
return mutate()
finally:
try:
fcntl.flock(fd, fcntl.LOCK_UN)
finally:
os.close(fd)
try:
lock_path.unlink()
except FileNotFoundError:
pass
def mark_posted(self, relative_path: str) -> bool:
"""Append ``relative_path`` to the stored list. Idempotent.
Returns ``True`` when the path was newly added, ``False`` when
it was already present.
"""
return self.mark_posted_many([relative_path]) != []
def mark_posted_many(self, relative_paths: Iterable[str]) -> list[str]:
"""Append any new entries from ``relative_paths`` in one write.
Returns the list of paths that were newly added (possibly
empty when the store already contained every supplied path).
"""
candidates = [p for p in relative_paths if isinstance(p, str) and p]
if not candidates:
return []
def _mutate() -> list[str]:
document = _read_existing(self._path) or dict(_EMPTY_DOCUMENT)
existing = _coerce_list(document.get("posted"))
added = [p for p in candidates if p not in existing]
if not added:
return []
existing.extend(added)
document["posted"] = existing
_write_document(self._path, document)
return added
return self._with_lock(_mutate)
def load_posted(data_dir: Path | None = None) -> list[str]:
return PostedStore(data_dir).load()
def is_posted(relative_path: str, data_dir: Path | None = None) -> bool:
return PostedStore(data_dir).is_posted(relative_path)
def mark_posted(relative_path: str, data_dir: Path | None = None) -> bool:
return PostedStore(data_dir).mark_posted(relative_path)
def mark_posted_many(
relative_paths: list[str], data_dir: Path | None = None
) -> list[str]:
return PostedStore(data_dir).mark_posted_many(relative_paths)
__all__ = [
"DEFAULT_DATA_DIR",
"PostedStore",
"PostedStoreError",
"is_posted",
"load_posted",
"mark_posted",
"mark_posted_many",
"posted_path",
]
View File
+41
View File
@@ -0,0 +1,41 @@
from __future__ import annotations
from pathlib import Path
import pytest
from tenbackward.config import Config
@pytest.fixture()
def env_setup(monkeypatch: pytest.MonkeyPatch) -> None:
"""Provide a fully populated environment for Config success paths.
Tests that exercise the missing-required-key path should call
`monkeypatch.delenv(...)` explicitly.
"""
monkeypatch.setenv("MASTODON_BASE_URL", "https://mastodon.example")
monkeypatch.setenv("MASTODON_ACCESS_TOKEN", "test-token")
monkeypatch.setenv("MASTODON_VISIBILITY", "public")
monkeypatch.setenv("SITE_URL", "https://blog.example.com")
monkeypatch.setenv("HASHTAGS", "#throwback,#10backward")
monkeypatch.setenv("THROWBACK_PREFIX", "Throwback:")
monkeypatch.setenv("MAX_RETRIES", "3")
monkeypatch.setenv("RUN_AT", "09:00")
monkeypatch.setenv("TZ", "Europe/Berlin")
@pytest.fixture()
def data_dir(tmp_path: Path) -> Path:
return tmp_path / "data"
@pytest.fixture()
def full_config(env_setup, data_dir) -> Config:
"""Build a Config object that points at an isolated data dir."""
from tenbackward.config import load_config
import os
os.environ["DATA_DIR"] = str(data_dir)
return load_config()
+214
View File
@@ -0,0 +1,214 @@
from __future__ import annotations
import logging
from pathlib import Path
from unittest.mock import MagicMock
import pytest
from git import GitCommandError, InvalidGitRepositoryError, Repo
from tenbackward.blog import (
BACKOFF_BASE_SECONDS,
BlogRepoError,
BlogTransientError,
DEFAULT_BLOG_SUBDIR,
DEFAULT_REPO_URL,
blog_dir,
blog_repo_url,
ensure_repo,
)
def _git_err(stderr: str = "boom") -> GitCommandError:
return GitCommandError(["git"], stderr=stderr, status=1)
def test_blog_dir_default_subdir(tmp_path: Path) -> None:
assert blog_dir(tmp_path) == tmp_path / DEFAULT_BLOG_SUBDIR
def test_blog_repo_url_default(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("BLOG_REPO_URL", raising=False)
assert blog_repo_url() == DEFAULT_REPO_URL
def test_blog_repo_url_env_override(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("BLOG_REPO_URL", "https://example.com/repo.git")
assert blog_repo_url() == "https://example.com/repo.git"
def test_ensure_repo_clones_when_missing(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
target = tmp_path / "blog"
fake_repo = MagicMock(spec=Repo)
calls: list[tuple[str, str]] = []
def fake_clone(url: str, path: str) -> Repo:
calls.append((url, path))
target.mkdir(parents=True)
(target / ".git").mkdir()
return fake_repo
sleeps: list[float] = []
result = ensure_repo(
tmp_path,
repo_url="https://example.com/repo.git",
blog_path=target,
max_retries=2,
sleep=sleeps.append,
clone_impl=fake_clone,
)
assert result is fake_repo
assert calls == [("https://example.com/repo.git", str(target))]
assert sleeps == []
def test_ensure_repo_pulls_when_clone_exists(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
target = tmp_path / "blog"
target.mkdir()
(target / ".git").mkdir()
fake_repo = MagicMock(spec=Repo)
monkeypatch.setattr("tenbackward.blog.Repo", MagicMock(return_value=fake_repo))
fetch_impl = MagicMock()
pull_impl = MagicMock()
sleeps: list[float] = []
result = ensure_repo(
tmp_path,
repo_url="https://example.com/repo.git",
blog_path=target,
max_retries=3,
sleep=sleeps.append,
fetch_impl=lambda r: fetch_impl,
pull_impl=lambda r: pull_impl,
)
assert result is fake_repo
fetch_impl.assert_called_once_with()
pull_impl.assert_called_once_with()
assert sleeps == []
def test_ensure_repo_aborts_on_local_modifications(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
) -> None:
target = tmp_path / "blog"
target.mkdir()
(target / ".git").mkdir()
fake_repo = MagicMock(spec=Repo)
monkeypatch.setattr("tenbackward.blog.Repo", MagicMock(return_value=fake_repo))
fetch_impl = MagicMock()
pull_impl = MagicMock(
side_effect=_git_err("Your local changes to 'foo' would be overwritten by merge")
)
sleeps: list[float] = []
with caplog.at_level(logging.ERROR):
with pytest.raises(BlogRepoError, match="local modifications"):
ensure_repo(
tmp_path,
repo_url="https://example.com/repo.git",
blog_path=target,
max_retries=5,
sleep=sleeps.append,
fetch_impl=lambda r: fetch_impl,
pull_impl=lambda r: pull_impl,
)
assert fetch_impl.call_count == 1
assert pull_impl.call_count == 1
assert sleeps == []
def test_ensure_repo_retries_transient_error_then_succeeds(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
target = tmp_path / "blog"
target.mkdir()
(target / ".git").mkdir()
fake_repo = MagicMock(spec=Repo)
monkeypatch.setattr("tenbackward.blog.Repo", MagicMock(return_value=fake_repo))
fetch_impl = MagicMock(side_effect=[_git_err("Could not resolve host"), None])
pull_impl = MagicMock()
sleeps: list[float] = []
ensure_repo(
tmp_path,
repo_url="https://example.com/repo.git",
blog_path=target,
max_retries=5,
sleep=sleeps.append,
fetch_impl=lambda r: fetch_impl,
pull_impl=lambda r: pull_impl,
)
assert fetch_impl.call_count == 2
pull_impl.assert_called_once_with()
assert sleeps == [BACKOFF_BASE_SECONDS ** 0]
def test_ensure_repo_raises_after_exhausted_retries(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
target = tmp_path / "blog"
target.mkdir()
(target / ".git").mkdir()
fake_repo = MagicMock(spec=Repo)
monkeypatch.setattr("tenbackward.blog.Repo", MagicMock(return_value=fake_repo))
fetch_impl = MagicMock(side_effect=_git_err("Could not resolve host"))
pull_impl = MagicMock()
sleeps: list[float] = []
with pytest.raises(BlogTransientError, match="git pull failed"):
ensure_repo(
tmp_path,
repo_url="https://example.com/repo.git",
blog_path=target,
max_retries=2,
sleep=sleeps.append,
fetch_impl=lambda r: fetch_impl,
pull_impl=lambda r: pull_impl,
)
assert fetch_impl.call_count == 2
pull_impl.assert_not_called()
assert sleeps == [BACKOFF_BASE_SECONDS ** 0]
def test_ensure_repo_invalid_clone_dir_raises(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
target = tmp_path / "blog"
target.mkdir()
(target / "not-a-repo.txt").write_text("hi")
def fake_repo(_path: str) -> Repo:
raise InvalidGitRepositoryError(f"{_path} is not a repo")
monkeypatch.setattr("tenbackward.blog.Repo", fake_repo)
with pytest.raises(BlogRepoError, match="not a valid git repository"):
ensure_repo(
tmp_path,
repo_url="https://example.com/repo.git",
blog_path=target,
max_retries=0,
)
+220
View File
@@ -0,0 +1,220 @@
from __future__ import annotations
import os
from pathlib import Path
import pytest
from tenbackward.config import ConfigError, load_config, validate_config, validate_required
def _populate(env_setup) -> dict[str, str]:
return {k: os.environ[k] for k in os.environ if k.startswith(("MASTODON_", "SITE_", "HASHTAGS", "THROWBACK_", "MAX_RETRIES", "RUN_AT", "TZ"))}
def _full_values() -> dict[str, str]:
return {
"MASTODON_BASE_URL": "https://mastodon.example",
"MASTODON_ACCESS_TOKEN": "x",
"MASTODON_VISIBILITY": "public",
"SITE_URL": "https://blog.example.com",
"HASHTAGS": "#throwback",
"THROWBACK_PREFIX": "Throwback:",
"MAX_RETRIES": "3",
"RUN_AT": "09:00",
"TZ": "Europe/Berlin",
}
def test_load_config_succeeds_with_complete_env(env_setup) -> None:
config = load_config()
assert config.mastodon_base_url == "https://mastodon.example"
assert config.run_at == "09:00"
assert config.max_retries == 3
assert config.throwback_prefix == "Throwback:"
assert config.mastodon_visibility == "public"
def test_load_config_blog_defaults(env_setup) -> None:
config = load_config()
assert config.blog_repo_url == "https://git.chaospott.de/Chaospott/site"
assert config.blog_dir.name == "blog"
def test_load_config_blog_repo_url_override(env_setup, monkeypatch) -> None:
monkeypatch.setenv("BLOG_REPO_URL", "https://example.com/repo.git")
config = load_config()
assert config.blog_repo_url == "https://example.com/repo.git"
def test_validate_config_rejects_bad_blog_repo_url(env_setup) -> None:
values = {
"MASTODON_BASE_URL": "https://mastodon.example",
"MASTODON_ACCESS_TOKEN": "x",
"MASTODON_VISIBILITY": "public",
"SITE_URL": "https://blog.example.com",
"HASHTAGS": "#throwback",
"THROWBACK_PREFIX": "Throwback:",
"MAX_RETRIES": "3",
"RUN_AT": "09:00",
"TZ": "Europe/Berlin",
"BLOG_REPO_URL": "ftp://bad",
}
with pytest.raises(ConfigError, match="BLOG_REPO_URL"):
validate_config(values)
def test_validate_config_lists_every_missing_key(env_setup, monkeypatch) -> None:
monkeypatch.delenv("MASTODON_ACCESS_TOKEN")
monkeypatch.delenv("SITE_URL")
values = {k: os.environ.get(k, "") for k in [
"MASTODON_BASE_URL",
"MASTODON_ACCESS_TOKEN",
"MASTODON_VISIBILITY",
"SITE_URL",
"HASHTAGS",
"THROWBACK_PREFIX",
"MAX_RETRIES",
"RUN_AT",
"TZ",
]}
with pytest.raises(ConfigError) as excinfo:
validate_required(values)
message = str(excinfo.value)
assert "MASTODON_ACCESS_TOKEN" in message
assert "SITE_URL" in message
def test_load_config_fails_fast_on_missing_required(env_setup, monkeypatch) -> None:
monkeypatch.delenv("MASTODON_ACCESS_TOKEN", raising=False)
with pytest.raises(ConfigError, match="MASTODON_ACCESS_TOKEN"):
load_config()
def test_validate_config_rejects_bad_run_at() -> None:
values = _full_values()
values["RUN_AT"] = "25:99"
with pytest.raises(ConfigError):
validate_config(values)
def test_validate_config_rejects_invalid_url() -> None:
values = _full_values()
values["SITE_URL"] = "not-a-url"
with pytest.raises(ConfigError, match="SITE_URL"):
validate_config(values)
values = _full_values()
values["MASTODON_BASE_URL"] = "ftp://mastodon.example"
with pytest.raises(ConfigError, match="MASTODON_BASE_URL"):
validate_config(values)
def test_validate_config_rejects_invalid_visibility() -> None:
for bad in ("private", "direct", "PUBLIC"):
values = _full_values()
values["MASTODON_VISIBILITY"] = bad
with pytest.raises(ConfigError, match="MASTODON_VISIBILITY"):
validate_config(values)
def test_validate_required_rejects_empty_visibility() -> None:
values = _full_values()
values["MASTODON_VISIBILITY"] = ""
with pytest.raises(ConfigError, match="MASTODON_VISIBILITY"):
validate_required(values)
def test_validate_config_rejects_invalid_tz() -> None:
values = _full_values()
values["TZ"] = "Not/AZone"
with pytest.raises(ConfigError, match="TZ"):
validate_config(values)
def test_validate_config_rejects_negative_max_retries() -> None:
values = _full_values()
values["MAX_RETRIES"] = "-1"
with pytest.raises(ConfigError, match="MAX_RETRIES"):
validate_config(values)
def test_validate_config_accepts_zero_max_retries() -> None:
values = _full_values()
values["MAX_RETRIES"] = "0"
validate_config(values)
def test_load_config_applies_optional_defaults(env_setup, monkeypatch) -> None:
monkeypatch.delenv("MAX_RETRIES", raising=False)
config = load_config()
assert config.max_retries == 5
assert config.blog_repo_url == "https://git.chaospott.de/Chaospott/site"
assert config.blog_dir.name == "blog"
def test_load_config_reads_dotenv_file(tmp_path: Path, monkeypatch) -> None:
dotenv = tmp_path / ".env"
dotenv.write_text(
"MASTODON_BASE_URL=https://from-file.example\n"
"MASTODON_ACCESS_TOKEN=file-token\n"
"MASTODON_VISIBILITY=unlisted\n"
"SITE_URL=https://blog.example.com\n"
"HASHTAGS=#throwback\n"
"THROWBACK_PREFIX=Werferückblick:\n"
"MAX_RETRIES=5\n"
"RUN_AT=12:34\n"
"TZ=Europe/Berlin\n"
)
for key in (
"MASTODON_BASE_URL",
"MASTODON_ACCESS_TOKEN",
"MASTODON_VISIBILITY",
"SITE_URL",
"HASHTAGS",
"THROWBACK_PREFIX",
"MAX_RETRIES",
"RUN_AT",
"TZ",
):
monkeypatch.delenv(key, raising=False)
config = load_config(dotenv_path=dotenv)
assert config.mastodon_base_url == "https://from-file.example"
assert config.mastodon_access_token == "file-token"
assert config.run_at == "12:34"
assert config.mastodon_visibility == "unlisted"
assert config.throwback_prefix == "Werferückblick:"
assert config.max_retries == 5
def test_env_overrides_dotenv(tmp_path: Path, monkeypatch) -> None:
dotenv = tmp_path / ".env"
dotenv.write_text("RUN_AT=01:00\n")
monkeypatch.setenv("RUN_AT", "23:00")
for key in (
"MASTODON_BASE_URL",
"MASTODON_ACCESS_TOKEN",
"MASTODON_VISIBILITY",
"SITE_URL",
"HASHTAGS",
"THROWBACK_PREFIX",
"MAX_RETRIES",
"TZ",
):
monkeypatch.setenv(key, "x")
monkeypatch.setenv("MASTODON_BASE_URL", "https://mastodon.example")
monkeypatch.setenv("SITE_URL", "https://blog.example.com")
monkeypatch.setenv("MASTODON_VISIBILITY", "public")
monkeypatch.setenv("TZ", "Europe/Berlin")
monkeypatch.setenv("MAX_RETRIES", "3")
config = load_config(dotenv_path=dotenv)
assert config.run_at == "23:00"
+195
View File
@@ -0,0 +1,195 @@
from __future__ import annotations
import shutil
import subprocess
from pathlib import Path
import pytest
from tenbackward.config import _is_valid_hhmm
REPO_ROOT = Path(__file__).resolve().parents[1]
ENTRYPOINT = REPO_ROOT / "entrypoint.sh"
def test_valid_hhmm() -> None:
assert _is_valid_hhmm("00:00")
assert _is_valid_hhmm("09:30")
assert _is_valid_hhmm("23:59")
@pytest.mark.parametrize("value", ["24:00", "9:00", "09:60", "9", "9:00:00", "ab:cd", "", None])
def test_invalid_hhmm(value) -> None:
assert not _is_valid_hhmm(value) # type: ignore[arg-type]
def _entrypoint_env(extra: dict[str, str]) -> dict[str, str]:
env = {
"PATH": "/usr/bin:/bin:/usr/sbin:/sbin",
"DEBIAN_FRONTEND": "noninteractive",
"HOME": "/tmp",
"LANG": "C.UTF-8",
}
env.update(extra)
return env
def _run_entrypoint_cron_render(extra: dict[str, str]) -> str:
raise NotImplementedError
def _extract_rendered_cron(env: dict[str, str]) -> tuple[int, str, str]:
"""Run the entrypoint with a fake `install` that records the cron file,
and a stub `cron` that exits immediately so the entrypoint doesn't hang.
Returns (returncode, stderr, installed-cron-contents).
"""
import tempfile
with tempfile.TemporaryDirectory() as fake_bin_dir, tempfile.TemporaryDirectory() as tmp:
fake_cron = Path(tmp) / "tenbackward.cron"
# Fake `install`: real syntax is `install [-m mode] [-o owner] [-g group] SRC DST`.
# We extract the last positional arg as DST and the last existing source before it.
fake_install = Path(fake_bin_dir) / "install"
fake_install.write_text(
"#!/bin/bash\n"
"src=\"\"\n"
"dst=\"\"\n"
"while [ $# -gt 0 ]; do\n"
" case \"$1\" in\n"
" -m|-o|-g) shift; shift;;\n"
" *) src=\"${src:-$1}\"; dst=\"$1\"; shift;;\n"
" esac\n"
"done\n"
"cp \"$src\" \"" + str(fake_cron) + "\"\n"
)
fake_install.chmod(0o755)
# Fake `cron` exits immediately so we never hang waiting on cron.
fake_cron_bin = Path(fake_bin_dir) / "cron"
fake_cron_bin.write_text("#!/bin/bash\nexit 0\n")
fake_cron_bin.chmod(0o755)
env = dict(env)
env["PATH"] = f"{fake_bin_dir}:/usr/bin:/bin"
proc = subprocess.run(
["/bin/bash", str(ENTRYPOINT)],
capture_output=True,
text=True,
env=env,
timeout=10,
check=False,
)
rendered = fake_cron.read_text() if fake_cron.exists() else ""
return proc.returncode, proc.stderr, rendered
@pytest.mark.skipif(not shutil.which("bash"), reason="bash not available")
def test_entrypoint_renders_run_at_into_cron() -> None:
"""Drive the entrypoint with a hermetic PATH so we can observe the rendered cron line.
The entrypoint hard-codes `/app/crontab/install-cron.sh` (correct in the
Docker image). For the host-side test we override PATH so `install` and
`cron` are stubbed, but we still need `/app/crontab/install-cron.sh` — we
bind-mount it via a bind-mount emulation by overriding the env var
`PATH` and having the install-cron.sh check fall back: instead, we patch
the entrypoint with an in-place substitution for the test.
"""
tmp_cron_dest = _write_substituted_entrypoint()
try:
env = _entrypoint_env({
"MASTODON_BASE_URL": "https://mastodon.example",
"MASTODON_ACCESS_TOKEN": "x",
"MASTODON_VISIBILITY": "public",
"SITE_URL": "https://blog.example.com",
"HASHTAGS": "#throwback",
"THROWBACK_PREFIX": "Throwback:",
"MAX_RETRIES": "3",
"RUN_AT": "09:00",
"TZ": "Europe/Berlin",
})
returncode, stderr, rendered = _extract_rendered_cron(env, tmp_cron_dest)
assert returncode == 0, (returncode, stderr)
assert "00 09 * * * /usr/local/bin/run-bot.sh >> /proc/1/fd/1 2>&1" in rendered, rendered
assert "TZ=Europe/Berlin" in rendered
finally:
tmp_cron_dest.unlink(missing_ok=True)
def _write_substituted_entrypoint():
"""Copy entrypoint.sh to a temp file with `/app/crontab/install-cron.sh`
rewritten to the repo-relative path so the host can execute it."""
import tempfile
src = ENTRYPOINT.read_text()
sub = src.replace("/app/crontab/install-cron.sh", str(REPO_ROOT / "crontab" / "install-cron.sh"))
tmp = Path(tempfile.mkstemp(prefix="entrypoint-", suffix=".sh")[1])
tmp.write_text(sub)
tmp.chmod(0o755)
return tmp
def _extract_rendered_cron(env: dict[str, str], entrypoint_path: Path) -> tuple[int, str, str]:
"""Run the (substituted) entrypoint with stubs for `install` and `cron`."""
import tempfile
with tempfile.TemporaryDirectory() as fake_bin_dir, tempfile.TemporaryDirectory() as tmp:
sidecar = Path(tmp) / "tenbackward.cron"
fake_install = Path(fake_bin_dir) / "install"
fake_install.write_text(
"#!/bin/bash\n"
"src=\"\"; dst=\"\"\n"
"while [ $# -gt 0 ]; do\n"
" case \"$1\" in -m|-o|-g) shift; shift;; *) src=\"${src:-$1}\"; dst=\"$1\"; shift;; esac\n"
"done\n"
"cp \"$src\" \"" + str(sidecar) + "\"\n"
"mkdir -p \"$(dirname \"$dst\")\" 2>/dev/null\n"
"cp \"$src\" \"$dst\" 2>/dev/null || true\n"
)
fake_install.chmod(0o755)
fake_cron_bin = Path(fake_bin_dir) / "cron"
fake_cron_bin.write_text("#!/bin/bash\nexit 0\n")
fake_cron_bin.chmod(0o755)
env = dict(env)
env["PATH"] = f"{fake_bin_dir}:/usr/bin:/bin"
proc = subprocess.run(
["/bin/bash", str(entrypoint_path)],
capture_output=True,
text=True,
env=env,
timeout=10,
check=False,
)
rendered = sidecar.read_text() if sidecar.exists() else ""
return proc.returncode, proc.stderr, rendered
@pytest.mark.skipif(not shutil.which("bash"), reason="bash not available")
def test_install_cron_rejects_unresolved_placeholder(tmp_path: Path) -> None:
script = REPO_ROOT / "crontab" / "install-cron.sh"
bad = tmp_path / "bad.cron"
bad.write_text("__RUN_AT__ * * * * /bin/false\n")
proc = subprocess.run(
["/bin/bash", str(script), str(bad)],
capture_output=True, text=True, timeout=10,
)
assert proc.returncode != 0
assert "__RUN_AT__" in proc.stderr
good = tmp_path / "good.cron"
good.write_text("SHELL=/bin/bash\n5 9 * * * /usr/local/bin/run-bot.sh >> /proc/1/fd/1 2>&1\n")
proc2 = subprocess.run(
["/bin/bash", str(script), str(good)],
capture_output=True, text=True, timeout=10,
)
assert proc2.returncode == 0, proc2.stderr
+84
View File
@@ -0,0 +1,84 @@
from __future__ import annotations
import re
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parents[1]
def test_dockerignore_blocks_secrets_and_data() -> None:
dockerignore = (REPO_ROOT / ".dockerignore").read_text().splitlines()
normalized = {line.strip() for line in dockerignore if line.strip() and not line.startswith("#")}
for required in (".env", "data/", ".git/", "tests/", ".kilo/", ".venv/"):
assert required in normalized, f"missing {required} from .dockerignore"
def test_gitignore_blocks_dotenv_and_data() -> None:
gitignore = (REPO_ROOT / ".gitignore").read_text().splitlines()
normalized = {line.strip() for line in gitignore if line.strip() and not line.startswith("#")}
assert ".env" in normalized
assert "data/" in normalized
assert ".kilo/" in normalized
def test_env_example_has_no_real_tokens() -> None:
text = (REPO_ROOT / ".env.example").read_text()
assert "MASTODON_ACCESS_TOKEN=replace-me" in text
for token in ("ghp_", "gho_", "xoxb-", "Bearer ey", "AKIA"):
assert token not in text, f"found suspicious token prefix {token!r} in .env.example"
mastodon_line = next(
line for line in text.splitlines() if line.startswith("MASTODON_ACCESS_TOKEN=")
)
value = mastodon_line.split("=", 1)[1].strip()
assert not re.fullmatch(r"[A-Za-z0-9_\-]{40,}", value), (
"MASTODON_ACCESS_TOKEN placeholder should not look like a real token"
)
def test_dockerfile_does_not_copy_env_or_data() -> None:
dockerfile = (REPO_ROOT / "Dockerfile").read_text()
for blocked in (".env", "data/", "./data"):
for line in dockerfile.splitlines():
if line.lstrip().upper().startswith("COPY"):
assert blocked not in line, f"Dockerfile COPY must not reference {blocked}: {line!r}"
def test_compose_mounts_data_and_env_readonly() -> None:
compose = (REPO_ROOT / "docker-compose.yml").read_text()
assert "env_file:" in compose
assert "- .env" in compose
assert "./data:/app/data" in compose
assert "build:" in compose
def test_dockerfile_runs_entrypoint_as_root() -> None:
dockerfile = (REPO_ROOT / "Dockerfile").read_text()
lines = [line.strip() for line in dockerfile.splitlines()]
entrypoint_idx = next(
(i for i, line in enumerate(lines) if line.upper().startswith("ENTRYPOINT")),
None,
)
assert entrypoint_idx is not None, "Dockerfile must declare ENTRYPOINT"
for line in lines[entrypoint_idx:]:
if line.upper().startswith("USER "):
pytest.fail(
"Dockerfile must not switch USER after ENTRYPOINT — entrypoint.sh "
"needs root to install /etc/cron.d/tenbackward and exec cron -f"
)
def test_run_bot_wrapper_drops_privileges() -> None:
wrapper = (REPO_ROOT / "run-bot.sh").read_text()
assert "setpriv" in wrapper or "su -s" in wrapper
assert "tenbackward" in wrapper
def test_cron_template_has_env_header() -> None:
cron = (REPO_ROOT / "crontab" / "tenbackward.cron").read_text()
assert "SHELL=/bin/bash" in cron
assert "PATH=" in cron
+94
View File
@@ -0,0 +1,94 @@
from __future__ import annotations
import re
import shutil
import subprocess
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parents[1]
ENTRYPOINT = REPO_ROOT / "entrypoint.sh"
def _has_bash() -> bool:
return shutil.which("bash") is not None
def _minimal_path() -> str:
parts = ["/usr/bin", "/bin", "/usr/sbin", "/sbin"]
for p in parts:
if Path(p).exists():
return ":".join(parts)
return os.environ.get("PATH", "")
def _clean_env() -> dict[str, str]:
env = {
"PATH": _minimal_path(),
"DEBIAN_FRONTEND": "noninteractive",
"HOME": "/tmp",
"LANG": "C.UTF-8",
}
return env
@pytest.mark.skipif(not _has_bash(), reason="bash not available")
def test_entrypoint_rejects_bad_run_at(tmp_path: Path, monkeypatch) -> None:
"""The entrypoint must exit non-zero with a clear message on a bad RUN_AT."""
env = _clean_env()
env["MASTODON_BASE_URL"] = "https://mastodon.example"
env["MASTODON_ACCESS_TOKEN"] = "x"
env["MASTODON_VISIBILITY"] = "public"
env["SITE_URL"] = "https://blog.example.com"
env["HASHTAGS"] = "#throwback"
env["THROWBACK_PREFIX"] = "Throwback:"
env["MAX_RETRIES"] = "3"
env["RUN_AT"] = "25:99"
env["TZ"] = "Europe/Berlin"
proc = subprocess.run(
["/bin/bash", str(ENTRYPOINT)],
capture_output=True,
text=True,
env=env,
timeout=10,
check=False,
)
assert proc.returncode != 0
assert "RUN_AT" in (proc.stdout + proc.stderr)
@pytest.mark.skipif(not _has_bash(), reason="bash not available")
def test_entrypoint_fails_fast_when_token_missing(tmp_path: Path, monkeypatch) -> None:
"""A missing required var must abort before cron is started."""
env = _clean_env()
env["MASTODON_BASE_URL"] = "https://mastodon.example"
env["MASTODON_VISIBILITY"] = "public"
env["SITE_URL"] = "https://blog.example.com"
env["HASHTAGS"] = "#throwback"
env["THROWBACK_PREFIX"] = "Throwback:"
env["MAX_RETRIES"] = "3"
env["RUN_AT"] = "09:00"
env["TZ"] = "Europe/Berlin"
env.pop("MASTODON_ACCESS_TOKEN", None)
proc = subprocess.run(
["/bin/bash", str(ENTRYPOINT)],
capture_output=True,
text=True,
env=env,
timeout=10,
check=False,
)
assert proc.returncode != 0
combined = (proc.stdout + proc.stderr).lower()
assert "missing" in combined
assert "mastodon_access_token" in combined.lower()
def test_entrypoint_references_cron_in_foreground() -> None:
text = ENTRYPOINT.read_text()
assert re.search(r"exec\s+cron\s+-f", text), "entrypoint must exec cron in the foreground"
cron = (REPO_ROOT / "crontab" / "tenbackward.cron").read_text()
assert "SHELL=/bin/bash" in cron
+93
View File
@@ -0,0 +1,93 @@
from __future__ import annotations
import io
import json
import logging
from tenbackward.logging_setup import (
JsonFormatter,
configure_json_logging,
log_error,
log_run_summary,
log_startup,
)
def _capture(logger: logging.Logger) -> io.StringIO:
buf = io.StringIO()
handler = logging.StreamHandler(buf)
handler.setFormatter(JsonFormatter())
logger.handlers = [handler]
logger.setLevel(logging.INFO)
logger.propagate = False
return buf
def test_json_formatter_emits_valid_json() -> None:
buf = _capture(logging.getLogger("tenbackward.test"))
logging.getLogger("tenbackward.test").info("hello", extra={"foo": 1})
line = buf.getvalue().strip()
payload = json.loads(line)
assert payload["message"] == "hello"
assert payload["level"] == "INFO"
assert payload["logger"] == "tenbackward.test"
assert payload["foo"] == 1
assert "ts" in payload
def test_json_formatter_redacts_secret_keys() -> None:
buf = _capture(logging.getLogger("tenbackward.test"))
logging.getLogger("tenbackward.test").info(
"msg",
extra={"access_token": "secret-value", "site_url": "https://x"},
)
payload = json.loads(buf.getvalue().strip())
assert payload["access_token"] == "[REDACTED]"
assert payload["site_url"] == "https://x"
def test_log_run_summary_emits_one_line_with_counters() -> None:
buf = _capture(logging.getLogger("tenbackward"))
log_run_summary(3, 2, 1, 1, ["x"])
lines = [line for line in buf.getvalue().splitlines() if line.strip()]
assert len(lines) == 1
payload = json.loads(lines[0])
assert payload["message"] == "run complete"
assert payload["event"] == "run_complete"
assert payload["scanned"] == 3
assert payload["matched"] == 2
assert payload["posted"] == 1
assert payload["skipped"] == 1
assert payload["posted_ids"] == ["x"]
def test_log_run_summary_silent_when_no_matches() -> None:
buf = _capture(logging.getLogger("tenbackward"))
log_run_summary(0, 0, 0, 0, [])
assert buf.getvalue() == ""
def test_log_startup_emits_single_info_line() -> None:
buf = _capture(logging.getLogger("tenbackward"))
log_startup(version="0.1.0", site_url="https://x", run_at="09:00", tz="Europe/Berlin")
payload = json.loads(buf.getvalue().strip())
assert payload["event"] == "startup"
assert payload["version"] == "0.1.0"
def test_log_error_redacts_secret_keys_and_exposes_exc_type() -> None:
buf = _capture(logging.getLogger("tenbackward"))
log_error("boom", exc=RuntimeError("x"), access_token="abc")
payload = json.loads(buf.getvalue().strip())
assert payload["level"] == "ERROR"
assert payload["event"] == "boom"
assert payload["access_token"] == "[REDACTED]"
assert payload["exc_type"] == "RuntimeError"
def test_configure_json_logging_is_idempotent() -> None:
configure_json_logging()
configure_json_logging()
root = logging.getLogger()
json_handlers = [h for h in root.handlers if isinstance(h.formatter, JsonFormatter)]
assert len(json_handlers) == 1
+236
View File
@@ -0,0 +1,236 @@
from __future__ import annotations
import logging
from datetime import date
from pathlib import Path
from typing import Callable
import pytest
from tenbackward.matching import (
MatchedPost,
find_anniversary_matches,
iter_anniversary_paths,
slugify_title,
)
SITE_URL = "https://chaospott.de"
_MATCHING_LOGGER = "tenbackward.matching"
def _write_post(root: Path, rel_path: str, *, body: str = "post") -> None:
full = root / rel_path
full.parent.mkdir(parents=True, exist_ok=True)
full.write_text(body, encoding="utf-8")
def _capture_matching_warnings(call: Callable[[], object]) -> list[logging.LogRecord]:
records: list[logging.LogRecord] = []
handler = logging.Handler()
handler.setLevel(logging.WARNING)
handler.emit = records.append # type: ignore[assignment]
logger = logging.getLogger(_MATCHING_LOGGER)
previous_level = logger.level
logger.setLevel(logging.WARNING)
logger.addHandler(handler)
try:
call()
finally:
logger.removeHandler(handler)
logger.setLevel(previous_level)
return records
def _event_names(records: list[logging.LogRecord]) -> list[str]:
return [getattr(record, "event", None) for record in records]
def test_match_uses_frontmatter_date_and_title(tmp_path: Path) -> None:
root = tmp_path / "_posts" / "blog"
_write_post(
root,
"2014/2014-08-04-foo.md",
body="---\ntitle: Foo\ndate: 2014-08-04\n---\nbody\n",
)
today = date(2024, 8, 4)
matches = find_anniversary_matches(root, SITE_URL, today=today)
assert len(matches) == 1
match = matches[0]
assert isinstance(match, MatchedPost)
assert match.path == "2014/2014-08-04-foo.md"
assert match.title == "Foo"
assert match.date == date(2014, 8, 4)
assert match.url == "https://chaospott.de/2014/08/04/foo/"
assert match.url.endswith(slugify_title(match.title) + "/")
def test_match_falls_back_to_filename_when_frontmatter_missing(tmp_path: Path) -> None:
root = tmp_path / "_posts" / "blog"
_write_post(root, "2015/2015-03-10-no-frontmatter.md", body="no frontmatter at all\n")
_write_post(
root, "2015/2015-03-11-broken-frontmatter.md", body="---\ndate: not-a-date\n: :\n"
)
today = date(2025, 3, 10)
matches: list[MatchedPost] = []
records = _capture_matching_warnings(
lambda: matches.extend(find_anniversary_matches(root, SITE_URL, today=today))
)
paths = {m.path for m in matches}
assert "2015/2015-03-10-no-frontmatter.md" in paths
assert "2015/2015-03-11-broken-frontmatter.md" not in paths
match = next(m for m in matches if m.path == "2015/2015-03-10-no-frontmatter.md")
assert match.title == "no-frontmatter"
assert match.date == date(2015, 3, 10)
assert match.url == "https://chaospott.de/2015/03/10/no-frontmatter/"
assert match.url.endswith(slugify_title(match.title) + "/")
def test_unpublished_post_is_skipped(tmp_path: Path) -> None:
root = tmp_path / "_posts" / "blog"
_write_post(
root,
"2014/2014-08-04-draft.md",
body="---\ntitle: Draft\ndate: 2014-08-04\npublished: false\n---\n",
)
today = date(2024, 8, 4)
matches: list[MatchedPost] = []
records = _capture_matching_warnings(
lambda: matches.extend(find_anniversary_matches(root, SITE_URL, today=today))
)
assert matches == []
assert _event_names(records) == ["post_unpublished"]
def test_iter_anniversary_paths_yields_only_path(tmp_path: Path) -> None:
root = tmp_path / "_posts" / "blog"
_write_post(
root,
"2014/2014-08-04-foo.md",
body="---\ntitle: Foo\ndate: 2014-08-04\n---\n",
)
paths = list(iter_anniversary_paths(root, SITE_URL, today=date(2024, 8, 4)))
assert paths == ["2014/2014-08-04-foo.md"]
@pytest.mark.parametrize(
"today, source_iso, expected_match",
[
# Normal exact match (Mar 1 source, Mar 1 today) -> match
(date(2025, 3, 1), "2015-03-01", True),
# Mar 1 source (2016, the day after Feb 29) in a non-leap current year -> no match
(date(2025, 3, 1), "2016-03-01", False),
# Feb 29 source in a non-leap current year, today is Mar 1: source year
# (2016) does not equal target_year (2015), so it does NOT match under
# the strict year-equality rule.
(date(2025, 3, 1), "2016-02-29", False),
# Feb 29 source (2016) in a leap current year (2028), today is Feb 29:
# target_year (2018) is not a leap year, but the Feb 29 -> Feb 29
# leap-year branch accepts any prior Feb 29 post.
(date(2028, 2, 29), "2016-02-29", True),
# Feb 29 source (2020) in a leap current year (2028), today is Feb 29:
# same branch, also matches.
(date(2028, 2, 29), "2020-02-29", True),
],
)
def test_leap_day_matching(
tmp_path: Path, today: date, source_iso: str, expected_match: bool
) -> None:
root = tmp_path / "_posts" / "blog"
target_year = today.year - 10
post_year = int(source_iso.split("-", 1)[0])
rel = f"{post_year}/{source_iso}-leap.md"
_write_post(
root,
rel,
body=f"---\ntitle: Leap\ndate: {source_iso}\n---\n",
)
matches = find_anniversary_matches(root, SITE_URL, today=today)
matched_paths = [m.path for m in matches]
if expected_match:
assert rel in matched_paths
else:
assert rel not in matched_paths
def test_leap_day_feb29_matches_with_canonical_url(tmp_path: Path) -> None:
root = tmp_path / "_posts" / "blog"
rel = "2016/2016-02-29-ten-years-before-2026.md"
_write_post(
root,
rel,
body=(
"---\n"
"title: Would Be Ten Years Ago Today In 2026 Mar 1\n"
"date: 2016-02-29 08:00:00\n"
"---\n"
"body\n"
),
)
today = date(2028, 2, 29)
matches = find_anniversary_matches(root, SITE_URL, today=today)
assert len(matches) == 1
match = matches[0]
assert match.path == rel
assert match.date == date(2016, 2, 29)
assert match.title == "Would Be Ten Years Ago Today In 2026 Mar 1"
assert match.url == "https://chaospott.de/2016/02/29/would-be-ten-years-ago-today-in-2026-mar-1/"
def test_match_uses_datetime_frontmatter_date(tmp_path: Path) -> None:
root = tmp_path / "_posts" / "blog"
_write_post(
root,
"2014/2014-08-04-with-time.md",
body="---\ntitle: WithTime\ndate: 2014-08-04T12:30:00\n---\n",
)
today = date(2024, 8, 4)
matches = find_anniversary_matches(root, SITE_URL, today=today)
assert len(matches) == 1
assert matches[0].date == date(2014, 8, 4)
def test_match_returns_results_in_deterministic_order(tmp_path: Path) -> None:
root = tmp_path / "_posts" / "blog"
for slug, day in [("alpha", 1), ("beta", 2), ("gamma", 3)]:
_write_post(
root,
f"2014/2014-08-0{day}-{slug}.md",
body=f"---\ntitle: {slug}\ndate: 2014-08-0{day}\n---\n",
)
today = date(2024, 8, 2)
matches = find_anniversary_matches(root, SITE_URL, today=today)
assert [m.path for m in matches] == ["2014/2014-08-02-beta.md"]
def test_missing_post_root_returns_empty(tmp_path: Path) -> None:
today = date(2024, 8, 4)
assert find_anniversary_matches(tmp_path / "does-not-exist", SITE_URL, today=today) == []
def test_invalid_filename_is_skipped(tmp_path: Path) -> None:
root = tmp_path / "_posts" / "blog"
_write_post(root, "2014/not-a-date.md", body="---\ntitle: Bad\n---\n")
today = date(2024, 8, 4)
matches: list[MatchedPost] = []
records = _capture_matching_warnings(
lambda: matches.extend(find_anniversary_matches(root, SITE_URL, today=today))
)
assert matches == []
assert "post_filename_invalid" in _event_names(records)
+219
View File
@@ -0,0 +1,219 @@
from __future__ import annotations
import os
from datetime import date
from pathlib import Path
from typing import Callable
import pytest
from tenbackward.config import load_config
from tenbackward.matching import MatchedPost
from tenbackward.publishing import (
MASTODON_STATUS_LIMIT,
PublishError,
build_status_text,
publish_mastodon,
slugify_title,
validate_status,
)
from tenbackward.state import PostedStore
def _make_match(path: str, title: str, day: int = 4) -> MatchedPost:
return MatchedPost(
path=path,
title=title,
date=date(2016, 8, day),
url=f"https://blog.example.com/2016/08/{day:02d}/{path.split('-', 3)[-1].removesuffix('.md')}/",
)
class _FakeMastodon:
"""Captures ``status_post`` calls and can be configured to raise."""
def __init__(self, *, raise_on_post: Exception | None = None) -> None:
self.raise_on_post = raise_on_post
self.calls: list[tuple[str, dict]] = []
def __call__(self, *, access_token: str, api_base_url: str) -> "_FakeMastodon":
self.access_token = access_token
self.api_base_url = api_base_url
return self
def status_post(self, status: str, **kwargs) -> None:
self.calls.append((status, kwargs))
if self.raise_on_post is not None:
raise self.raise_on_post
@pytest.fixture()
def full_config(env_setup, tmp_path: Path):
os.environ["DATA_DIR"] = str(tmp_path)
return load_config()
def test_slugify_title_basic_ascii() -> None:
assert slugify_title("Hello, World!") == "hello-world"
def test_slugify_title_collapses_repeats_and_trims() -> None:
assert slugify_title("!!!Foo---Bar???") == "foo-bar"
assert slugify_title("---trim---me---") == "trim-me"
def test_build_status_text_single_post_layout() -> None:
match = _make_match("2016/2016-08-04-foo.md", "Foo")
status = build_status_text(
"Heute vor 10 Jahren:", [match], "#throwback,#10backward"
)
assert status == (
"Heute vor 10 Jahren:\n\n"
"Foo\nhttps://blog.example.com/2016/08/04/foo/\n\n"
"#throwback #10backward\n"
)
def test_build_status_text_multiple_posts_combined() -> None:
match_a = _make_match("2016/2016-08-04-a.md", "Alpha")
match_b = _make_match("2016/2016-08-04-b.md", "Bravo")
status = build_status_text("P:", [match_a, match_b], "#x")
assert "Alpha" in status
assert "Bravo" in status
assert match_a.url in status
assert match_b.url in status
assert status.count("Alpha") == 1
assert status.count("Bravo") == 1
assert status.count(match_a.url) == 1
assert status.count(match_b.url) == 1
def test_build_status_text_empty_posts_raises() -> None:
with pytest.raises(PublishError):
build_status_text("P:", [], "#x")
def test_build_status_text_preserves_deterministic_order() -> None:
match_a = _make_match("2016/2016-08-05-a.md", "Alpha", day=5)
match_b = _make_match("2016/2016-08-04-b.md", "Bravo", day=4)
forward = build_status_text("P:", [match_a, match_b], "#x")
reverse = build_status_text("P:", [match_b, match_a], "#x")
assert forward == reverse
assert forward.index("Bravo") < forward.index("Alpha")
def test_validate_status_within_limit() -> None:
validate_status("a" * 480)
def test_validate_status_over_limit_raises_and_does_not_truncate() -> None:
long = "a" * (MASTODON_STATUS_LIMIT + 1)
with pytest.raises(PublishError):
validate_status(long)
assert len(long) == MASTODON_STATUS_LIMIT + 1
def test_publish_mastodon_success_invokes_client_with_composed_status(
full_config,
) -> None:
match = _make_match("2016/2016-08-04-foo.md", "Foo")
fake = _FakeMastodon()
result = publish_mastodon(full_config, [match], client_factory=fake)
assert result.endswith("\n")
assert fake.calls, "status_post must be invoked"
posted_status, posted_kwargs = fake.calls[0]
assert posted_status == result
assert posted_kwargs == {"visibility": "public"}
assert fake.api_base_url == "https://mastodon.example"
assert fake.access_token == "test-token"
def test_publish_mastodon_missing_token_is_fatal(full_config) -> None:
from tenbackward.publishing import PublishFatalError
match = _make_match("2016/2016-08-04-foo.md", "Foo")
fake = _FakeMastodon()
bad_config = type(full_config)(
mastodon_base_url=full_config.mastodon_base_url,
mastodon_access_token="",
mastodon_visibility=full_config.mastodon_visibility,
site_url=full_config.site_url,
hashtags=full_config.hashtags,
throwback_prefix=full_config.throwback_prefix,
max_retries=full_config.max_retries,
run_at=full_config.run_at,
tz=full_config.tz,
data_dir=full_config.data_dir,
blog_repo_url=full_config.blog_repo_url,
blog_dir=full_config.blog_dir,
extra=dict(full_config.extra),
)
with pytest.raises(PublishFatalError):
publish_mastodon(bad_config, [match], client_factory=fake)
assert fake.calls == []
def test_publish_mastodon_api_failure_raises_and_does_not_persist(
full_config, tmp_path: Path
) -> None:
match = _make_match("2016/2016-08-04-foo.md", "Foo")
fake = _FakeMastodon(raise_on_post=RuntimeError("boom"))
with pytest.raises(PublishError) as exc_info:
publish_mastodon(full_config, [match], client_factory=fake)
assert "publish_failed" in str(exc_info.value)
assert "mastodon post" in str(exc_info.value)
assert isinstance(exc_info.value.__cause__, RuntimeError)
store = PostedStore(tmp_path)
assert not store.is_posted(match.path)
def test_publish_mastodon_over_limit_raises_before_api_call(
full_config,
) -> None:
long_title = "T" * (MASTODON_STATUS_LIMIT + 1)
match = MatchedPost(
path="2016/2016-08-04-foo.md",
title=long_title,
date=date(2016, 8, 4),
url="https://blog.example.com/2016/08/04/foo/",
)
fake = _FakeMastodon()
with pytest.raises(PublishError):
publish_mastodon(full_config, [match], client_factory=fake)
assert fake.calls == []
def test_publish_mastodon_combined_status_for_multiple_matches(full_config) -> None:
match_a = _make_match("2016/2016-08-04-a.md", "Alpha")
match_b = _make_match("2016/2016-08-04-b.md", "Bravo")
fake = _FakeMastodon()
publish_mastodon(full_config, [match_a, match_b], client_factory=fake)
assert len(fake.calls) == 1
status, _ = fake.calls[0]
assert "Alpha" in status
assert "Bravo" in status
assert match_a.url in status
assert match_b.url in status
def test_publish_mastodon_uses_config_visibility(env_setup, tmp_path: Path) -> None:
os.environ["MASTODON_VISIBILITY"] = "unlisted"
os.environ["DATA_DIR"] = str(tmp_path)
config = load_config()
match = _make_match("2016/2016-08-04-foo.md", "Foo")
fake = _FakeMastodon()
publish_mastodon(config, [match], client_factory=fake)
assert fake.calls[0][1] == {"visibility": "unlisted"}
+171
View File
@@ -0,0 +1,171 @@
from __future__ import annotations
import io
import json
import logging
import os
from pathlib import Path
import pytest
from datetime import date
from tenbackward import main as main_module
from tenbackward.logging_setup import JsonFormatter
from tenbackward.main import main
from tenbackward.matching import MatchedPost
from tenbackward.publishing import PublishError
def _match(path: str) -> MatchedPost:
return MatchedPost(
path=path,
title=path,
date=date(2016, 8, 4),
url=f"https://blog.example.com/2016/08/04/{path.split('-', 3)[-1].removesuffix('.md')}/",
)
@pytest.fixture()
def capture_logger() -> io.StringIO:
buf = io.StringIO()
handler = logging.StreamHandler(buf)
handler.setFormatter(JsonFormatter())
target = logging.getLogger("tenbackward")
target.handlers = [handler]
target.setLevel(logging.INFO)
target.propagate = False
return buf
def _run_lines(buf: io.StringIO) -> list[dict]:
return [json.loads(line) for line in buf.getvalue().splitlines() if line.strip()]
def _seed_state(data_dir: Path, ids: list[str]) -> None:
from tenbackward.state import mark_posted_many
mark_posted_many(list(ids), data_dir)
def test_run_emits_one_info_summary_on_success(env_setup, data_dir, capture_logger, monkeypatch) -> None:
monkeypatch.setenv("DATA_DIR", str(data_dir))
monkeypatch.setattr(main_module, "ensure_repo", lambda *a, **kw: None)
def _stub_publish(config, posts, **_kwargs):
return "stubbed"
monkeypatch.setattr(main_module, "_iter_candidates", lambda config: [_match("new-1"), _match("already-1")])
monkeypatch.setattr(main_module, "publish_mastodon", _stub_publish)
_seed_state(data_dir, ["already-1"])
rc = main()
assert rc == 0
lines = _run_lines(capture_logger)
summary = [line for line in lines if line.get("event") == "run_complete"]
assert len(summary) == 1
payload = summary[0]
assert payload["scanned"] == 2
assert payload["matched"] == 2
assert payload["posted"] == 1
assert payload["skipped"] == 1
assert payload["posted_ids"] == ["new-1"]
from tenbackward.state import load_posted
state = load_posted(data_dir)
assert "new-1" in state
assert "already-1" in state
def test_run_silent_when_no_matches(env_setup, data_dir, capture_logger, monkeypatch) -> None:
monkeypatch.setenv("DATA_DIR", str(data_dir))
monkeypatch.setattr(main_module, "ensure_repo", lambda *a, **kw: None)
monkeypatch.setattr(main_module, "_iter_candidates", lambda config: [])
rc = main()
assert rc == 0
lines = _run_lines(capture_logger)
summary = [line for line in lines if line.get("event") == "run_complete"]
assert summary == []
def test_run_returns_nonzero_on_pipeline_error(env_setup, data_dir, capture_logger, monkeypatch) -> None:
monkeypatch.setenv("DATA_DIR", str(data_dir))
monkeypatch.setenv("MAX_RETRIES", "1")
monkeypatch.setattr(main_module, "ensure_repo", lambda *a, **kw: None)
def _boom(config):
raise RuntimeError("boom-token-should-not-appear")
monkeypatch.setattr(main_module, "_iter_candidates", _boom)
rc = main()
assert rc == 1
lines = _run_lines(capture_logger)
summary = [line for line in lines if line.get("event") == "run_complete"]
assert summary == []
error_lines = [line for line in lines if line.get("level") == "ERROR"]
assert any(line.get("event") == "retry_exhausted" for line in error_lines)
exhausted = [line for line in error_lines if line.get("event") == "retry_exhausted"][0]
assert exhausted.get("operation") == "pipeline"
assert "boom-token-should-not-appear" in exhausted.get("error", "")
def test_run_distinguishes_posted_from_skipped_via_ids(env_setup, data_dir, capture_logger, monkeypatch) -> None:
monkeypatch.setenv("DATA_DIR", str(data_dir))
monkeypatch.setattr(main_module, "ensure_repo", lambda *a, **kw: None)
monkeypatch.setattr(
main_module,
"_iter_candidates",
lambda config: [_match("alpha"), _match("beta"), _match("gamma")],
)
monkeypatch.setattr(
main_module,
"publish_mastodon",
lambda config, posts, **_kwargs: "stubbed",
)
_seed_state(data_dir, ["beta"])
rc = main()
assert rc == 0
summary = [line for line in _run_lines(capture_logger) if line.get("event") == "run_complete"]
assert len(summary) == 1
payload = summary[0]
assert set(payload["posted_ids"]) == {"alpha", "gamma"}
assert "beta" not in payload["posted_ids"]
assert payload["skipped"] == 1
assert payload["posted"] == 2
def test_run_publish_failure_leaves_state_untouched(
env_setup, data_dir, capture_logger, monkeypatch
) -> None:
monkeypatch.setenv("DATA_DIR", str(data_dir))
monkeypatch.setattr(main_module, "ensure_repo", lambda *a, **kw: None)
monkeypatch.setenv("MAX_RETRIES", "0")
def _boom(config, posts, **_kwargs):
raise PublishError("publish_failed: stub")
monkeypatch.setattr(main_module, "_iter_candidates", lambda config: [_match("alpha")])
monkeypatch.setattr(main_module, "publish_mastodon", _boom)
rc = main()
assert rc != 0
from tenbackward.state import load_posted
assert load_posted(data_dir) == []
error_lines = [
line
for line in _run_lines(capture_logger)
if line.get("level") == "ERROR"
]
assert any(line.get("event") == "retry_exhausted" for line in error_lines)
+167
View File
@@ -0,0 +1,167 @@
from __future__ import annotations
import io
import json
import logging
import os
from pathlib import Path
from tenbackward.logging_setup import JsonFormatter
from tenbackward.state import (
PostedStore,
is_posted,
load_posted,
mark_posted,
mark_posted_many,
posted_path,
)
def _store(tmp_path: Path) -> PostedStore:
return PostedStore(tmp_path)
def test_load_creates_empty_file_when_missing(tmp_path: Path) -> None:
store = _store(tmp_path)
result = store.load()
assert result == []
assert posted_path(tmp_path).exists()
document = json.loads(posted_path(tmp_path).read_text(encoding="utf-8"))
assert document == {"posted": []}
def test_load_returns_empty_when_file_missing_no_crash(tmp_path: Path) -> None:
assert _store(tmp_path).load() == []
def test_load_warns_and_returns_empty_on_malformed_json(
tmp_path: Path,
) -> None:
buf = io.StringIO()
handler = logging.StreamHandler(buf)
handler.setFormatter(JsonFormatter())
logger = logging.getLogger("tenbackward")
original_handlers = list(logger.handlers)
original_propagate = logger.propagate
logger.handlers = [handler]
logger.setLevel(logging.WARNING)
logger.propagate = False
try:
posted_path(tmp_path).parent.mkdir(parents=True, exist_ok=True)
posted_path(tmp_path).write_text("not-json", encoding="utf-8")
result = _store(tmp_path).load()
finally:
logger.handlers = original_handlers
logger.propagate = original_propagate
assert result == []
lines = [
json.loads(line) for line in buf.getvalue().splitlines() if line.strip()
]
events = [line.get("event") for line in lines]
assert "posted_store_corrupt" in events
def test_mark_posted_writes_relative_path_to_file(tmp_path: Path) -> None:
_store(tmp_path).mark_posted("2014/2014-08-04-foo.md")
document = json.loads(posted_path(tmp_path).read_text(encoding="utf-8"))
assert document == {"posted": ["2014/2014-08-04-foo.md"]}
def test_is_posted_round_trip(tmp_path: Path) -> None:
store = _store(tmp_path)
store.mark_posted("2014/2014-08-04-foo.md")
assert store.is_posted("2014/2014-08-04-foo.md") is True
assert store.is_posted("2015/2015-01-01-bar.md") is False
def test_mark_posted_is_idempotent(tmp_path: Path) -> None:
store = _store(tmp_path)
first = store.mark_posted("2014/2014-08-04-foo.md")
second = store.mark_posted("2014/2014-08-04-foo.md")
assert first is True
assert second is False
document = json.loads(posted_path(tmp_path).read_text(encoding="utf-8"))
assert document == {"posted": ["2014/2014-08-04-foo.md"]}
def test_mark_posted_many_persists_in_one_write(
tmp_path: Path, monkeypatch
) -> None:
store = _store(tmp_path)
replaces: list[tuple[object, object]] = []
real_replace = os.replace
def counting_replace(src, dst) -> None:
replaces.append((src, dst))
real_replace(src, dst)
monkeypatch.setattr("tenbackward.state.os.replace", counting_replace)
added = store.mark_posted_many(
["2014/2014-08-04-foo.md", "2015/2015-01-01-bar.md"]
)
assert added == ["2014/2014-08-04-foo.md", "2015/2015-01-01-bar.md"]
assert len(replaces) == 1
assert Path(replaces[0][1]) == posted_path(tmp_path)
def test_mark_posted_leaves_no_temp_files(tmp_path: Path) -> None:
_store(tmp_path).mark_posted("2014/2014-08-04-foo.md")
leftovers = [
p for p in tmp_path.iterdir() if p.name.startswith("posted.json.tmp.")
]
assert leftovers == []
def test_mark_posted_many_against_existing_is_no_op(tmp_path: Path) -> None:
store = _store(tmp_path)
store.mark_posted("2014/2014-08-04-foo.md")
added = store.mark_posted_many(
["2014/2014-08-04-foo.md", "2015/2015-01-01-bar.md"]
)
assert added == ["2015/2015-01-01-bar.md"]
document = json.loads(posted_path(tmp_path).read_text(encoding="utf-8"))
assert document == {
"posted": ["2014/2014-08-04-foo.md", "2015/2015-01-01-bar.md"]
}
def test_posted_path_defaults_via_DATA_DIR(
monkeypatch, tmp_path: Path
) -> None:
monkeypatch.setenv("DATA_DIR", str(tmp_path))
assert posted_path(None) == tmp_path / "posted.json"
def test_load_filters_non_string_entries(tmp_path: Path) -> None:
posted_path(tmp_path).parent.mkdir(parents=True, exist_ok=True)
posted_path(tmp_path).write_text(
json.dumps({"posted": ["good", 42, None, "also-good"]}),
encoding="utf-8",
)
assert _store(tmp_path).load() == ["good", "also-good"]
def test_module_helpers_round_trip(tmp_path: Path) -> None:
assert load_posted(tmp_path) == []
assert is_posted("2014/2014-08-04-foo.md", tmp_path) is False
assert mark_posted("2014/2014-08-04-foo.md", tmp_path) is True
added = mark_posted_many(
["2014/2014-08-04-foo.md", "2015/2015-01-01-bar.md"], tmp_path
)
assert added == ["2015/2015-01-01-bar.md"]
assert load_posted(tmp_path) == [
"2014/2014-08-04-foo.md",
"2015/2015-01-01-bar.md",
]