Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5f6ddf76a4 | |||
| 3545b81546 | |||
| 42258b9a96 | |||
| e30fa78019 | |||
| 7d39f3a339 | |||
| 72c2369160 | |||
| 3fe2b90f51 | |||
| 207e2e6bbb | |||
| 33c6c76ce9 |
@@ -0,0 +1,22 @@
|
||||
.env
|
||||
.env.example
|
||||
data/
|
||||
.git/
|
||||
.venv/
|
||||
venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.egg-info/
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
tests/
|
||||
.kilo/
|
||||
kilo.json
|
||||
setup.sh
|
||||
README.md
|
||||
.dockerignore
|
||||
.gitignore
|
||||
.idea/
|
||||
.vscode/
|
||||
*.log
|
||||
@@ -0,0 +1,15 @@
|
||||
# 10Backward configuration — placeholder values only. NEVER use real credentials here.
|
||||
|
||||
MASTODON_BASE_URL=https://mastodon.example
|
||||
MASTODON_ACCESS_TOKEN=replace-me
|
||||
MASTODON_VISIBILITY=public
|
||||
SITE_URL=https://blog.example.com
|
||||
HASHTAGS=#throwback,#10backward
|
||||
THROWBACK_PREFIX=Heute vor 10 Jahren:
|
||||
MAX_RETRIES=5
|
||||
RUN_AT=09:00
|
||||
TZ=Europe/Berlin
|
||||
|
||||
# Optional — Jekyll blog clone source. Defaults point at the Chaospott public repo.
|
||||
# BLOG_REPO_URL=https://git.chaospott.de/Chaospott/site
|
||||
# BLOG_DIR=blog
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
.env
|
||||
.env.*
|
||||
!/.env.example
|
||||
data/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.egg-info/
|
||||
.venv/
|
||||
venv/
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
htmlcov/
|
||||
*.log
|
||||
.DS_Store
|
||||
.idea/
|
||||
.vscode/
|
||||
.kilo/
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
TZ=Europe/Berlin
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
cron \
|
||||
git \
|
||||
ca-certificates \
|
||||
tzdata \
|
||||
util-linux \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN groupadd --system bot \
|
||||
&& useradd --system --gid bot --home /app --shell /usr/sbin/nologin bot
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN mkdir -p /app/data \
|
||||
&& chown -R bot:bot /app
|
||||
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
RUN pip install --no-cache-dir --upgrade pip \
|
||||
&& pip install --no-cache-dir -r /app/requirements.txt
|
||||
|
||||
COPY --chown=bot:bot crontab/ /app/crontab/
|
||||
COPY --chown=bot:bot entrypoint.sh /app/entrypoint.sh
|
||||
COPY --chown=bot:bot run-bot.sh /app/run-bot.sh
|
||||
COPY --chown=root:root run-bot.sh /usr/local/bin/run-bot.sh
|
||||
COPY --chown=bot:bot src/ /app/src/
|
||||
|
||||
RUN chmod 0755 /app/entrypoint.sh \
|
||||
&& chmod 0755 /app/crontab/install-cron.sh \
|
||||
&& chmod 0755 /app/run-bot.sh \
|
||||
&& chmod 0755 /usr/local/bin/run-bot.sh \
|
||||
&& chmod 0644 /app/crontab/tenbackward.cron
|
||||
|
||||
ENV PYTHONPATH=/app/src
|
||||
|
||||
# Note: entrypoint.sh runs as root so it can install /etc/cron.d/tenbackward
|
||||
# and exec cron -f. The cron-launched wrapper drops to the unprivileged
|
||||
# `bot` user before invoking Python.
|
||||
ENTRYPOINT ["/app/entrypoint.sh"]
|
||||
@@ -1,2 +1,50 @@
|
||||
# 10Backward
|
||||
|
||||
Daily Mastodon throwback bot scaffolded with Docker Compose.
|
||||
|
||||
The container runs a cron daemon that invokes the Python entrypoint once per
|
||||
day at the configured `RUN_AT` (Europe/Berlin default). The actual blog
|
||||
clone / post pipeline is intentionally stubbed in this scaffold and is to be
|
||||
implemented in a follow-up job — this repository currently ships:
|
||||
|
||||
- Pinned Python dependency manifest.
|
||||
- `Dockerfile` (python:3.11-slim + cron + git + ca-certificates + tzdata).
|
||||
- `docker-compose.yml` building locally, mounting `.env` read-only and
|
||||
persisting `./data` for the blog clone and `posted.json` state.
|
||||
- `entrypoint.sh` that validates required env vars and fails clearly on
|
||||
startup before launching `cron -f` in the foreground.
|
||||
- Safe `.env.example` with placeholder credentials only.
|
||||
|
||||
## Build & run
|
||||
|
||||
```bash
|
||||
cp .env.example .env # fill in real credentials locally
|
||||
docker compose build
|
||||
docker compose up -d
|
||||
docker compose logs -f bot
|
||||
```
|
||||
|
||||
`./data` is persisted across restarts on the host. The blog clone (added in a
|
||||
follow-up job) will live in `./data/blog/`, and posted-state in
|
||||
`./data/posted.json`.
|
||||
|
||||
## Configuration
|
||||
|
||||
| Variable | Required | Default | Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| `MASTODON_BASE_URL` | yes | — | e.g. `https://mastodon.social` |
|
||||
| `MASTODON_ACCESS_TOKEN` | yes | — | never commit |
|
||||
| `SITE_URL` | yes | — | source blog URL (HTTPS) |
|
||||
| `HASHTAGS` | yes | — | comma-separated, e.g. `#throwback,#10backward` |
|
||||
| `RUN_AT` | yes | `09:00` | `HH:MM` in `TZ` |
|
||||
| `MASTODON_VISIBILITY` | no | `public` | `public`, `unlisted`, `private`, `direct` |
|
||||
| `THROWBACK_PREFIX` | no | `Throwback:` | German-localizable |
|
||||
| `RETRY_COUNT` | no | `3` | post retries |
|
||||
| `TZ` | no | `Europe/Berlin` | any IANA timezone |
|
||||
|
||||
## Tests
|
||||
|
||||
```bash
|
||||
pip install -r requirements-dev.txt
|
||||
pytest
|
||||
```
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "${SCRIPT_DIR}"
|
||||
|
||||
if [ -w /usr/local/bin ] && [ ! -e /usr/local/bin/build.sh ]; then
|
||||
ln -s "${SCRIPT_DIR}/build.sh" /usr/local/bin/build.sh || true
|
||||
fi
|
||||
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends \
|
||||
bash \
|
||||
git \
|
||||
ca-certificates \
|
||||
python3 \
|
||||
python3-pip \
|
||||
python3-venv \
|
||||
tzdata
|
||||
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
PYTHON_BIN="$(command -v python3)"
|
||||
PIP_BIN="$(command -v pip3 || true)"
|
||||
if [ -z "${PIP_BIN}" ] && [ -n "${PYTHON_BIN}" ]; then
|
||||
PIP_BIN="${PYTHON_BIN} -m pip"
|
||||
fi
|
||||
|
||||
if [ -n "${PIP_BIN}" ]; then
|
||||
${PIP_BIN} install --no-cache-dir --upgrade pip >/dev/null 2>&1 || true
|
||||
${PIP_BIN} install --no-cache-dir --break-system-packages -r /repo/requirements-dev.txt || \
|
||||
${PIP_BIN} install --no-cache-dir -r /repo/requirements-dev.txt
|
||||
fi
|
||||
|
||||
cd /repo
|
||||
PYTHONPATH=/repo/src pytest -q
|
||||
Executable
+33
@@ -0,0 +1,33 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
CRON_FILE="${1:-/etc/cron.d/tenbackward}"
|
||||
|
||||
if [ ! -f "${CRON_FILE}" ]; then
|
||||
echo "install-cron: cron file ${CRON_FILE} does not exist" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if grep -q '__RUN_AT__' "${CRON_FILE}"; then
|
||||
echo "install-cron: refusing to install cron file with unresolved __RUN_AT__ placeholder" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
chmod 0644 "${CRON_FILE}"
|
||||
|
||||
if ! head -n 1 "${CRON_FILE}" | grep -qE '^[A-Z_]+='; then
|
||||
echo "install-cron: ${CRON_FILE} must start with a cron env line (e.g. SHELL=/bin/bash)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! grep -q '/usr/local/bin/run-bot.sh' "${CRON_FILE}"; then
|
||||
echo "install-cron: ${CRON_FILE} must invoke /usr/local/bin/run-bot.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! grep -qE '/proc/1/fd/1' "${CRON_FILE}"; then
|
||||
echo "install-cron: ${CRON_FILE} must redirect output to /proc/1/fd/1" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "install-cron: ${CRON_FILE} installed (mode 0644)"
|
||||
@@ -0,0 +1,2 @@
|
||||
SHELL=/bin/bash
|
||||
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
@@ -0,0 +1,22 @@
|
||||
services:
|
||||
bot:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: tenbackward-bot:latest
|
||||
container_name: tenbackward-bot
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- .env
|
||||
environment:
|
||||
MASTODON_BASE_URL: ${MASTODON_BASE_URL:-https://mastodon.example}
|
||||
MASTODON_ACCESS_TOKEN: ${MASTODON_ACCESS_TOKEN:-replace-me}
|
||||
MASTODON_VISIBILITY: ${MASTODON_VISIBILITY:-public}
|
||||
SITE_URL: ${SITE_URL:-https://blog.example.com}
|
||||
HASHTAGS: ${HASHTAGS:-#throwback,#10backward}
|
||||
THROWBACK_PREFIX: ${THROWBACK_PREFIX:-Throwback:}
|
||||
MAX_RETRIES: ${MAX_RETRIES:-5}
|
||||
RUN_AT: ${RUN_AT:-09:00}
|
||||
TZ: ${TZ:-Europe/Berlin}
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
@@ -0,0 +1,81 @@
|
||||
---
|
||||
type: architecture
|
||||
title: Anniversary Matching
|
||||
description: Rules and file boundaries used to discover Jekyll posts whose publication anniversary is exactly ten years before the current date.
|
||||
tags: [matching, jekyll, anniversary, posts]
|
||||
timestamp: 2026-08-04T19:45:00Z
|
||||
---
|
||||
|
||||
# Purpose
|
||||
|
||||
`tenbackward.matching` scans the synchronized blog at
|
||||
`<blog_dir>/_posts/blog` and returns posts whose date is exactly ten years
|
||||
before the current date. The current date is evaluated in the configured
|
||||
IANA timezone, defaulting to `Europe/Berlin` in the matcher.
|
||||
|
||||
# Discovery Rules
|
||||
|
||||
1. Walk every Markdown file below the configured post root recursively.
|
||||
2. Parse YAML front matter when present.
|
||||
3. Require a filename stem shaped as `YYYY-MM-DD-slug` with a valid calendar
|
||||
date and non-empty slug.
|
||||
4. Prefer a valid front matter `date`; otherwise use the date in the filename.
|
||||
5. Match the post year to `today.year - 10` and the month/day to today.
|
||||
6. Skip posts with `published: false`.
|
||||
7. Return a `MatchedPost` containing the relative path, title, date, and
|
||||
canonical site URL. If front matter has no title, use the filename slug.
|
||||
|
||||
# Leap-Day Behavior
|
||||
|
||||
A February 29 post matches February 29 when the current year is a leap year.
|
||||
In a non-leap year it matches March 1, but only when the post's year is
|
||||
exactly ten years before the current year (`today.year − 10`). Because
|
||||
`today.year − 10` is never itself a leap year whenever today is a leap year
|
||||
(`10 mod 4 == 2`), the Feb 29 ↔ Feb 29 branch is reached independently of
|
||||
the year-equality check: any Feb 29 post from a prior leap year is a
|
||||
candidate, and its canonical URL keeps the original `YYYY/02/29/<slug>/`
|
||||
path.
|
||||
|
||||
# Output and Integration
|
||||
|
||||
`find_anniversary_matches()` returns `MatchedPost` values for callers that
|
||||
need metadata. `iter_anniversary_paths()` yields only relative paths; the
|
||||
pipeline uses those paths as deduplication identifiers in the
|
||||
[system state store](/architecture/system-overview.md) and persists new IDs in
|
||||
`posted.json`.
|
||||
|
||||
Unreadable files, malformed front matter, invalid filenames, unpublished
|
||||
posts, and unexpected per-file errors are skipped with structured warning
|
||||
events rather than aborting the full scan.
|
||||
|
||||
# Key Files
|
||||
|
||||
| Path | Responsibility |
|
||||
|---|---|
|
||||
| `/repo/src/tenbackward/matching.py` | Date parsing, Jekyll front matter handling, anniversary rules, URL construction, and recursive scanning. |
|
||||
| `/repo/src/tenbackward/main.py` | Supplies the blog post root and site URL, then applies state deduplication. |
|
||||
| `/repo/tests/test_matching.py` | Covers filename/front matter parsing, date matching, unpublished posts, malformed files, URL output, and leap-day behavior. |
|
||||
| `/repo/src/tenbackward/blog.py` | Ensures the source repository is cloned or fast-forwarded before matching. |
|
||||
|
||||
# Examples
|
||||
|
||||
## Matching post
|
||||
|
||||
On 2026-08-04, a file named
|
||||
`_posts/blog/2016/2016-08-04-release.md` is eligible. Its identifier is
|
||||
`2016/2016-08-04-release.md` and its generated URL is:
|
||||
|
||||
```text
|
||||
https://example.com/2016/08/04/release/
|
||||
```
|
||||
|
||||
## Skipped post
|
||||
|
||||
A file with `published: false`, an invalid date, or a non-matching
|
||||
month/day is not yielded and does not enter `posted.json`.
|
||||
|
||||
# Related
|
||||
|
||||
* [Pipeline Runner](/architecture/pipeline-runner.md)
|
||||
* [System Architecture](/architecture/system-overview.md)
|
||||
* [Daily Run Guide](/guides/daily-run.md)
|
||||
@@ -0,0 +1,97 @@
|
||||
---
|
||||
type: api
|
||||
title: Config Schema
|
||||
description: Required/optional environment variables, validation rules, and the typed Config dataclass returned by tenbackward.config.load_config.
|
||||
tags: [config, env, schema]
|
||||
timestamp: 2026-08-04T17:51:00Z
|
||||
---
|
||||
|
||||
# Purpose
|
||||
|
||||
`tenbackward.config.load_config()` is the single boundary that turns
|
||||
process environment + optional `.env` file into a typed `Config`
|
||||
dataclass. Job 1083 changes every variable that ships in
|
||||
`.env.example` so that it is **explicitly required** at every layer
|
||||
(`entrypoint.sh`, `config.validate_config`, and the rendered cron).
|
||||
|
||||
# Required Environment Variables
|
||||
|
||||
The following keys must be present and non-empty in the merged dotenv/process environment before application defaults are applied.
|
||||
|
||||
| Key | Purpose | Validation |
|
||||
|---|---|---|
|
||||
| `MASTODON_BASE_URL` | Mastodon instance to post against. | Valid `http` or `https` URL. |
|
||||
| `MASTODON_ACCESS_TOKEN` | OAuth token used by the Mastodon client. | Required string; redacted in logs. |
|
||||
| `MASTODON_VISIBILITY` | Post visibility. | `public` or `unlisted`. |
|
||||
| `SITE_URL` | Source blog URL. | Valid `http` or `https` URL. |
|
||||
| `HASHTAGS` | Comma-separated hashtags. | Required string. |
|
||||
| `THROWBACK_PREFIX` | Prefix prepended to each status. | Required string. |
|
||||
| `MAX_RETRIES` | Maximum number of pipeline attempts. | Non-negative integer; defaults to `5` when absent. |
|
||||
| `RUN_AT` | Daily cron fire time. | `HH:MM` 24-hour format. |
|
||||
| `TZ` | Cron and application timezone. | Valid IANA timezone. |
|
||||
|
||||
`BLOG_REPO_URL` and `BLOG_DIR` are optional and receive defaults after this required-key check. `DATA_DIR` is read separately from the process environment and defaults to `/app/data`.
|
||||
|
||||
# Defaults
|
||||
|
||||
Defaults are applied only after required-key validation. The current defaults are:
|
||||
|
||||
| Default key | Default value |
|
||||
|---|---|
|
||||
| `MAX_RETRIES` | `5` |
|
||||
| `BLOG_REPO_URL` | `https://git.chaospott.de/Chaospott/site` |
|
||||
| `BLOG_DIR` | `blog` (resolved relative to `data_dir`) |
|
||||
|
||||
Application-facing required values are not silently supplied by `config.py`; the container environment may still provide `TZ=Europe/Berlin` and Compose may provide development fallback values.
|
||||
|
||||
# `Config` Dataclass
|
||||
|
||||
| Field | Type | Source |
|
||||
|-------------------------|-----------|----------------------------------------------|
|
||||
| `mastodon_base_url` | `str` | `MASTODON_BASE_URL` |
|
||||
| `mastodon_access_token` | `str` | `MASTODON_ACCESS_TOKEN` |
|
||||
| `mastodon_visibility` | `str` | `MASTODON_VISIBILITY` |
|
||||
| `site_url` | `str` | `SITE_URL` |
|
||||
| `hashtags` | `str` | `HASHTAGS` |
|
||||
| `throwback_prefix` | `str` | `THROWBACK_PREFIX` |
|
||||
| `max_retries` | `int` | `MAX_RETRIES` (parsed as non-negative int) |
|
||||
| `run_at` | `str` | `RUN_AT` |
|
||||
| `tz` | `str` | `TZ` |
|
||||
| `data_dir` | `Path` | `DATA_DIR`, default `/app/data` |
|
||||
| `blog_repo_url` | `str` | `BLOG_REPO_URL` |
|
||||
| `blog_dir` | `Path` | `BLOG_DIR` resolved against `data_dir` |
|
||||
|
||||
# Validation Behaviour
|
||||
|
||||
`validate_config(values)` collects all failures before raising so the
|
||||
operator sees every problem at once. A single `ConfigError` lists each
|
||||
problem joined with `"; "`. Validation covers:
|
||||
|
||||
* Required keys are non-empty.
|
||||
* `RUN_AT` parses as `HH:MM`.
|
||||
* `MASTODON_BASE_URL` and `SITE_URL` parse via `urllib.parse.urlparse`
|
||||
with an `http`/`https` scheme and a non-empty `netloc`.
|
||||
* `VISIBILITY` is one of `{"public", "unlisted"}` (`ALLOWED_VISIBILITY`).
|
||||
* `BLOG_REPO_URL` (optional, has default) parses via `urllib.parse.urlparse`
|
||||
with an `http`/`https` scheme and a non-empty `netloc` when present.
|
||||
* `TZ` resolves via `zoneinfo.ZoneInfo`.
|
||||
* `MAX_RETRIES` parses as a non-negative integer (rejects bools).
|
||||
|
||||
# Exit Codes
|
||||
|
||||
| Code | Source | Meaning |
|
||||
|------|-----------------------------------------|----------------------------------------------------------------------|
|
||||
| `1` | `main.main()` (retry budget exhausted) | Pipeline failed on every attempt after `MAX_RETRIES + 1` tries. |
|
||||
| `2` | `main.main()` (`ConfigError`) | Configuration was present but invalid; details are logged. |
|
||||
|
||||
# Citations
|
||||
|
||||
* [1] `src/tenbackward/config.py` — `REQUIRED_KEYS`, `DEFAULTS`, `ALLOWED_VISIBILITY`, `Config`, `load_config`, `validate_config`.
|
||||
* [2] `.env.example` — canonical env-var list.
|
||||
* [3] `entrypoint.sh` — shell-side mirror of the required keys.
|
||||
|
||||
# Related
|
||||
|
||||
* [System Architecture](/architecture/system-overview.md)
|
||||
* [Logging & Run Summary](/architecture/logging.md)
|
||||
* [Environment Variable Setup](/operations/environment-setup.md)
|
||||
@@ -0,0 +1,126 @@
|
||||
---
|
||||
type: architecture
|
||||
title: Logging & Run Summary
|
||||
description: Structured JSON logging in 10Backward — JsonFormatter, secret redaction, and the startup / error / run_complete event helpers.
|
||||
tags: [logging, observability, json]
|
||||
timestamp: 2026-08-04T17:51:00Z
|
||||
---
|
||||
|
||||
# Purpose
|
||||
|
||||
`tenbackward.logging_setup` replaces the scaffold-era `basicConfig`
|
||||
text logger with a **single-line JSON formatter** that downstream
|
||||
log shippers can index. It also owns the three helpers the runner
|
||||
uses to record structured events: `log_startup`, `log_error`,
|
||||
`log_run_summary`.
|
||||
|
||||
# JsonFormatter
|
||||
|
||||
`JsonFormatter.format(record)` returns one JSON object per log record:
|
||||
|
||||
| Key | Source |
|
||||
|------------|---------------------------------------------------------------------|
|
||||
| `ts` | UTC `datetime.fromtimestamp(record.created).isoformat()`. |
|
||||
| `level` | `record.levelname` (e.g. `INFO`, `ERROR`). |
|
||||
| `logger` | `record.name` (always `tenbackward`). |
|
||||
| `message` | `record.getMessage()`. |
|
||||
| _extras_ | Any keyword passed via `logging.info(..., extra={...})` that is not a reserved `LogRecord` attribute and whose key is **not** in `_SECRET_KEYS`. |
|
||||
| `exc_type` | When `record.exc_info` is set, the exception class name. |
|
||||
|
||||
Reserved `LogRecord` attributes and any key starting with `_` are
|
||||
dropped before serialization, so the payload contains only the values
|
||||
that callers intentionally attached.
|
||||
|
||||
## Secret Redaction
|
||||
|
||||
`_SECRET_KEYS` is the canonical denylist applied in two places:
|
||||
|
||||
1. **At format time** for keys passed via `extra={...}`.
|
||||
2. **At helper time** by `_redact_dict(...)` so values reaching
|
||||
`log_startup` / `log_error` never expose known-sensitive fields
|
||||
even before they hit the formatter.
|
||||
|
||||
Deny-list contents (case-insensitive):
|
||||
|
||||
```
|
||||
token, access_token, mastodon_access_token, password, secret,
|
||||
authorization, api_key
|
||||
```
|
||||
|
||||
Any matching key is replaced with the string `[REDACTED]`.
|
||||
|
||||
# Helper Functions
|
||||
|
||||
| Helper | When it fires | Logger / Level | `event` field |
|
||||
|---------------------------------------|-----------------------------------------------------------------------|----------------|--------------------|
|
||||
| `configure_json_logging(level=INFO)` | First line of `main.main()`. Idempotent — clears existing handlers. | n/a | n/a |
|
||||
| `log_startup(**fields)` | After config load + `data_dir.mkdir`, before the pipeline pass. | `INFO` | `startup` |
|
||||
| `log_error(event, *, exc=None, **fields)` | When a pipeline attempt raises, or when the retry budget is exhausted. | `ERROR` | the supplied event name; `exc_type` added when `exc` is provided. |
|
||||
| `log_run_summary(scanned, matched, posted, skipped, posted_ids)` | Once per successful pass. **Silently skipped** when all counters are zero and no ids were posted. | `INFO` | `run_complete` |
|
||||
|
||||
## Startup payload
|
||||
|
||||
`log_startup` is invoked with:
|
||||
|
||||
```
|
||||
version, site_url, run_at, tz, hashtags, throwback_prefix
|
||||
```
|
||||
|
||||
Each is added to the `startup` event payload under its own key.
|
||||
|
||||
## Run-summary payload
|
||||
|
||||
`log_run_summary` is the canonical "what did the bot do today?" record:
|
||||
|
||||
```json
|
||||
{
|
||||
"ts": "2026-08-04T09:00:00+00:00",
|
||||
"level": "INFO",
|
||||
"logger": "tenbackward",
|
||||
"message": "run complete",
|
||||
"event": "run_complete",
|
||||
"scanned": 0,
|
||||
"matched": 0,
|
||||
"posted": 0,
|
||||
"skipped": 0,
|
||||
"posted_ids": []
|
||||
}
|
||||
```
|
||||
|
||||
A pass with no candidates emits **no line at all**, matching the
|
||||
"silent on no matches" contract.
|
||||
|
||||
# Examples
|
||||
|
||||
## Successful run with one new post
|
||||
|
||||
```json
|
||||
{"ts":"2026-08-04T09:00:00+00:00","level":"INFO","logger":"tenbackward","message":"startup","event":"startup","version":"0.1.0","site_url":"https://blog.example.com","run_at":"09:00","tz":"Europe/Berlin","hashtags":"#throwback,#10backward","throwback_prefix":"Heute vor 10 Jahren:"}
|
||||
{"ts":"2026-08-04T09:00:01+00:00","level":"INFO","logger":"tenbackward","message":"run complete","event":"run_complete","scanned":1,"matched":1,"posted":1,"skipped":0,"posted_ids":["2025-08-04-post-slug"]}
|
||||
```
|
||||
|
||||
## Retry budget exhausted
|
||||
|
||||
```json
|
||||
{"ts":"...","level":"ERROR","logger":"tenbackward","message":"pipeline_error","event":"pipeline_error","attempt":1,"max_attempts":4,"exc_type":"ConnectionError"}
|
||||
{"ts":"...","level":"ERROR","logger":"tenbackward","message":"pipeline_error","event":"pipeline_error","attempt":2,"max_attempts":4,"exc_type":"ConnectionError"}
|
||||
{"ts":"...","level":"ERROR","logger":"tenbackward","message":"pipeline_failed","event":"pipeline_failed","attempts":4,"exc_type":"ConnectionError"}
|
||||
```
|
||||
|
||||
The container then exits with status `1`.
|
||||
|
||||
## Token accidentally logged
|
||||
|
||||
Even if a future call wrote `extra={"access_token": "..."}`, the
|
||||
formatter replaces it:
|
||||
|
||||
```json
|
||||
{ "...": "...", "access_token": "[REDACTED]" }
|
||||
```
|
||||
|
||||
# Related
|
||||
|
||||
* [System Architecture](/architecture/system-overview.md)
|
||||
* [Pipeline Runner](/architecture/pipeline-runner.md)
|
||||
* [Config Schema](/architecture/config-schema.md)
|
||||
* [Mastodon Publishing](/architecture/mastodon-publishing.md)
|
||||
@@ -0,0 +1,170 @@
|
||||
---
|
||||
type: architecture
|
||||
title: Mastodon Publishing
|
||||
description: The single boundary between the pipeline runner and the Mastodon HTTP API — status composition, length validation, and the publish call.
|
||||
tags: [publishing, mastodon, api, boundary]
|
||||
timestamp: 2026-08-04T18:55:00Z
|
||||
---
|
||||
|
||||
# Purpose
|
||||
|
||||
`tenbackward.publishing` is the **single success boundary** between the
|
||||
pipeline orchestrator and the Mastodon HTTP API. The runner calls
|
||||
`publish_mastodon()` after dedupe and before state is persisted; if
|
||||
the call raises, the retry loop re-attempts and `posted.json` is left
|
||||
untouched. Composition and validation are pure functions that do not
|
||||
touch the network so they are unit-tested without a server.
|
||||
|
||||
# Public API
|
||||
|
||||
| Symbol | Responsibility |
|
||||
|---|---|
|
||||
| `MASTODON_STATUS_LIMIT` | `500` — Mastodon's hard maximum status length. |
|
||||
| `PublishError` | Raised on composition failure, length overflow, or any wrapped API exception. Subclass of `RuntimeError`. |
|
||||
| `build_status_text(prefix, posts, hashtags)` | Pure composer. Returns the final status string. Raises `PublishError("empty_posts: ...")` when `posts` is empty. |
|
||||
| `validate_status(status, limit=MASTODON_STATUS_LIMIT)` | Pure validator. Raises `PublishError("status_too_long: len=N limit=L")` when `len(status) > limit`. Never truncates. |
|
||||
| `publish_mastodon(config, posts, *, client_factory=None)` | Composes, validates, and posts. Returns the composed status string. Wraps every third-party exception in `PublishError(f"publish_failed: {ExcType}: {exc}", ) from exc`. |
|
||||
| `slugify_title(title)` | Re-exported from [matching](/architecture/anniversary-matching.md); used by callers that need to derive the same URL slug the matcher emits. |
|
||||
|
||||
# Status Layout
|
||||
|
||||
`build_status_text` produces exactly one Mastodon status for any number
|
||||
of matching posts, in this shape:
|
||||
|
||||
```text
|
||||
{prefix}
|
||||
|
||||
{title1}
|
||||
{url1}
|
||||
|
||||
{title2}
|
||||
{url2}
|
||||
|
||||
...
|
||||
|
||||
{hashtags}
|
||||
```
|
||||
|
||||
* The prefix comes from `Config.throwback_prefix` (configured via `THROWBACK_PREFIX`).
|
||||
* Posts are sorted by `(date, path)` so output is deterministic
|
||||
regardless of upstream order.
|
||||
* `hashtags` is a comma-separated string; commas are collapsed to a
|
||||
single space so each tag starts with `#` and no trailing comma
|
||||
is emitted (empty tags are dropped).
|
||||
* A trailing newline is always appended.
|
||||
|
||||
# Composition Examples
|
||||
|
||||
## Single post
|
||||
|
||||
```text
|
||||
Heute vor 10 Jahren:
|
||||
|
||||
Mein erster Post
|
||||
https://blog.example.com/2016/08/04/mein-erster-post/
|
||||
|
||||
#throwback #10backward
|
||||
```
|
||||
|
||||
## Multiple posts on the same day
|
||||
|
||||
When more than one Jekyll post matches the current day, they are
|
||||
**merged into one status** — the runner does not issue separate API
|
||||
calls per match. Ordering is `(date, path)`, so identical dates sort
|
||||
by relative path:
|
||||
|
||||
```text
|
||||
Heute vor 10 Jahren:
|
||||
|
||||
Post A
|
||||
https://blog.example.com/2016/08/04/a/
|
||||
|
||||
Post B
|
||||
https://blog.example.com/2016/08/04/b/
|
||||
|
||||
#throwback #10backward
|
||||
```
|
||||
|
||||
## No posts
|
||||
|
||||
Calling `build_status_text("...", [], "#x")` raises
|
||||
`PublishError("empty_posts: cannot compose status without posts")`
|
||||
before any API call. The runner only invokes the publisher when its
|
||||
candidate list is non-empty, so this guard is a backstop for direct
|
||||
callers and unit tests.
|
||||
|
||||
# Length Validation
|
||||
|
||||
Mastodon's API rejects statuses longer than 500 characters. The
|
||||
spec requires **failing safely** — the bot must never silently
|
||||
truncate content. `validate_status()` enforces this contract: the
|
||||
full status is measured, and overflow raises
|
||||
`PublishError("status_too_long: len=N limit=500")` *before* the API
|
||||
call. The runner's retry loop treats this exactly like any other
|
||||
publish failure.
|
||||
|
||||
# Mastodon API Call
|
||||
|
||||
`_post_status_via_mastodon_py()` constructs a `mastodon.Mastodon` client with `(access_token=config.mastodon_access_token, api_base_url=config.mastodon_base_url)` and calls `status_post(status, visibility=config.mastodon_visibility)`.
|
||||
|
||||
The `client_factory` keyword argument on `publish_mastodon` lets
|
||||
tests inject a fake client without monkey-patching. Production
|
||||
callers leave it as `None`.
|
||||
|
||||
# Failure Modes
|
||||
|
||||
| Source | Exception surfaced to runner | Cause attached? |
|
||||
|---|---|---|
|
||||
| Empty `posts` argument | `PublishError("empty_posts: ...")` | No (no inner exc). |
|
||||
| Composed status > 500 chars | `PublishError("status_too_long: ...")` | No. |
|
||||
| Any other exception from inside the boundary | `PublishError("publish_failed: {ExcType}: {exc}")` | Yes, via `raise ... from exc`. |
|
||||
|
||||
The runner catches every `Exception` in `_run_once`, so any of the
|
||||
above becomes a `pipeline_error` log line and the retry budget
|
||||
decides whether to give up.
|
||||
|
||||
# Wiring
|
||||
|
||||
```
|
||||
_run_once(config) # main.py
|
||||
├── ensure_repo(...)
|
||||
├── store = PostedStore(config.data_dir)
|
||||
├── candidates = list(_iter_candidates(config)) # find_anniversary_matches
|
||||
├── unposted = [m for m in candidates if not store.is_posted(m.path)]
|
||||
├── if unposted:
|
||||
│ publish_mastodon(config, unposted) # <-- THIS boundary
|
||||
│ store.mark_posted_many(posted_ids) # only after success
|
||||
```
|
||||
|
||||
State is written **only after** `publish_mastodon` returns. A publish
|
||||
failure leaves `posted.json` unchanged and lets the retry budget
|
||||
re-attempt on the next iteration.
|
||||
|
||||
# Configuration Surface
|
||||
|
||||
| Env var | Consumed via | Effect |
|
||||
|---|---|---|
|
||||
| `MASTODON_BASE_URL` | `Config.mastodon_base_url` | Mastodon instance URL. |
|
||||
| `MASTODON_ACCESS_TOKEN` | `Config.mastodon_access_token` | OAuth token passed to the `Mastodon` client. |
|
||||
| `MASTODON_VISIBILITY` | `Config.mastodon_visibility` | Passed as `visibility=` to `status_post`. |
|
||||
| `THROWBACK_PREFIX` | `Config.throwback_prefix` | First line of every published status. |
|
||||
| `HASHTAGS` | `Config.hashtags` | Trailing hashtag line; commas become spaces. |
|
||||
|
||||
See [Config Schema](/architecture/config-schema.md) for full validation
|
||||
rules.
|
||||
|
||||
# Key Files
|
||||
|
||||
| Path | Responsibility |
|
||||
|---|---|
|
||||
| `/repo/src/tenbackward/publishing.py` | `MASTODON_STATUS_LIMIT`, `PublishError`, `build_status_text`, `validate_status`, `_post_status_via_mastodon_py`, `publish_mastodon`. |
|
||||
| `/repo/src/tenbackward/matching.py` | `MatchedPost` dataclass and `slugify_title()` reused here. |
|
||||
| `/repo/src/tenbackward/main.py` | Calls `publish_mastodon()` between dedupe and `mark_posted_many`. |
|
||||
| `/repo/tests/test_publishing.py` | Composition, length validation, visibility passthrough, API failure wrapping, and the no-persistence-on-failure contract. |
|
||||
|
||||
# Related
|
||||
|
||||
* [Pipeline Runner](/architecture/pipeline-runner.md)
|
||||
* [Anniversary Matching](/architecture/anniversary-matching.md)
|
||||
* [Config Schema](/architecture/config-schema.md)
|
||||
* [Daily Run Guide](/guides/daily-run.md)
|
||||
@@ -0,0 +1,134 @@
|
||||
---
|
||||
type: architecture
|
||||
title: Pipeline Runner
|
||||
description: How tenbackward.main synchronizes the blog, discovers anniversary candidates, applies deduplication, retries failures, and emits a run summary.
|
||||
tags: [pipeline, runner, retry]
|
||||
timestamp: 2026-08-04T17:51:00Z
|
||||
---
|
||||
|
||||
# Purpose
|
||||
`tenbackward.main` is the entry point executed by cron. The pipeline first
|
||||
ensures that the configured blog repository is available and current, then
|
||||
scans Jekyll posts for today's ten-year anniversary, filters already-recorded
|
||||
paths, publishes a single combined Mastodon status for the new matches, and
|
||||
finally persists the freshly published identifiers.
|
||||
|
||||
|
||||
# Call Flow
|
||||
|
||||
```
|
||||
main.main()
|
||||
├── configure_json_logging()
|
||||
├── load_config() ── raises ConfigError → exit 2
|
||||
├── config.data_dir.mkdir(parents=True, exist_ok=True)
|
||||
├── ensure_repo(config.data_dir, blog_repo_url, blog_dir, max_retries)
|
||||
│ └── BlogRepoError → log blog_repo_error → exit 1
|
||||
├── log_startup(version, site_url, run_at, tz, hashtags, throwback_prefix)
|
||||
├── result = _run_with_retry(config)
|
||||
│ ├── for attempt in 1 .. max_retries+1:
|
||||
│ │ try: return _run_once(config)
|
||||
│ │ └── _run_once → ensure_repo → find_anniversary_matches
|
||||
│ │ → PostedStore deduplication → publish_mastodon
|
||||
│ │ → mark_posted_many # state only after publish OK
|
||||
│ ├── log_error("pipeline_error", exc=exc, attempt=attempt, max_attempts=attempts)
|
||||
│ ├── log_error("pipeline_failed", exc=last_exc, attempts=attempts)
|
||||
│ └── return None
|
||||
├── if result is None: return 1
|
||||
└── log_run_summary(scanned, matched, posted, skipped, posted_ids)
|
||||
```
|
||||
|
||||
# Extension Point: `_iter_candidates`
|
||||
|
||||
```python
|
||||
def _iter_candidates(config: Config) -> Iterable[MatchedPost]: ...
|
||||
```
|
||||
|
||||
`_iter_candidates` delegates to `find_anniversary_matches()` with
|
||||
`config.blog_dir / "_posts" / "blog"` and `config.site_url`. The matcher
|
||||
returns full `MatchedPost` values (relative path, title, date, canonical
|
||||
URL); the runner uses `MatchedPost.path` as a stable deduplication ID and
|
||||
passes the full objects to `publish_mastodon()` for status composition.
|
||||
See [Anniversary Matching](/architecture/anniversary-matching.md) for the
|
||||
file and front matter rules.
|
||||
|
||||
## Blog Synchronization
|
||||
|
||||
`ensure_repo()` runs before candidate discovery. An empty target is cloned
|
||||
from `BLOG_REPO_URL`; an existing repository is fetched and fast-forwarded.
|
||||
Transient Git failures use exponential backoff, while local modifications
|
||||
fail immediately with `BlogRepoError`. The same synchronization occurs once
|
||||
in `main()` before `startup` and again inside `_run_once()` as the retryable
|
||||
pipeline boundary.
|
||||
|
||||
# `_run_once` — Summary Counters
|
||||
|
||||
`_run_once` returns a 5-tuple:
|
||||
|
||||
```python
|
||||
(scanned, matched, posted, skipped, posted_ids)
|
||||
```
|
||||
|
||||
| Counter | Meaning |
|
||||
|-------------|----------------------------------------------------------------------|
|
||||
| `scanned` | Items yielded by `_iter_candidates`. |
|
||||
| `matched` | Items that survived the dedupe-vs-`already_posted` check. |
|
||||
| `posted` | Items newly recorded in `posted.json` during this run. |
|
||||
| `skipped` | Items that matched but were already in `posted.json` (deduped). |
|
||||
| `posted_ids`| The list of post identifiers written to state. |
|
||||
|
||||
`PostedStore.mark_posted_many` is called **only** when `posted_ids`
|
||||
is non-empty, so a no-op run does not touch `posted.json` on disk.
|
||||
The store persists the list under a `"posted"` key
|
||||
(`{"posted": ["2014/2014-08-04-foo.md", ...]}`) and serialises
|
||||
concurrent runs with an `fcntl.flock`.
|
||||
|
||||
# Publish Boundary
|
||||
|
||||
`publish_mastodon()` is the single success boundary. The runner calls it
|
||||
**after** dedupe and **before** `mark_posted_many`. If the call raises a
|
||||
`PublishError`, the runner treats it like any other pipeline exception —
|
||||
the retry loop in `_run_with_retry` re-attempts, and no identifier is
|
||||
written to `posted.json`. Identical-day matches are published as **one**
|
||||
combined status: prefix line + one `title\nurl` block per post + hashtags
|
||||
line, sorted by `(date, path)` for deterministic ordering. The default
|
||||
prefix is `Heute vor 10 Jahren:` (overridable via `THROWNBACK_PREFIX`).
|
||||
Generated statuses are validated against `MASTODON_STATUS_LIMIT` (500
|
||||
characters); the runner fails safely (raises) rather than truncating.
|
||||
|
||||
# Retry Behaviour (`_run_with_retry`)
|
||||
|
||||
* `attempts = max(1, config.max_retries)` — at least one attempt even if `MAX_RETRIES=0`.
|
||||
* The runner owns the unified retry budget; `_run_once` calls `ensure_repo` with `max_retries=0`.
|
||||
* Transient Git and Mastodon failures are retried with delays of `2 ** (attempt - 1)` seconds between attempts.
|
||||
* Each failed attempt emits `retry_attempt` with `operation`, `attempt`, and the exception `error` text.
|
||||
* Fatal configuration, local repository, and publishing validation errors emit `retry_exhausted` immediately without retrying.
|
||||
* When the budget is exhausted, `retry_exhausted` includes the final operation, total attempts, and error text, and `main` returns exit code `1`.
|
||||
|
||||
# Silent-on-No-Matches Contract
|
||||
|
||||
* `run_complete` is emitted for a successful pass, including when all discovered candidates were already posted.
|
||||
* A run with zero candidates emits only `startup`; no `run_complete` line is emitted.
|
||||
|
||||
# Example Sequence (Healthy Run)
|
||||
|
||||
1. `startup` line at INFO.
|
||||
2. `_run_with_retry` returns `(0, 0, 0, 0, [])` (no candidates yet
|
||||
because `_iter_candidates` is empty) → `main` exits `0`
|
||||
**without** a `run_complete` line.
|
||||
3. Next-day tick repeats the cycle.
|
||||
|
||||
# Example Sequence (Future Blog Step in Place)
|
||||
|
||||
1. `startup` line at INFO.
|
||||
2. `_iter_candidates` yields `[id-1, id-2]`. `id-1` is already in
|
||||
`posted.json`, so `skipped += 1`. `id-2` is new and appended to
|
||||
`posted_ids` and persisted.
|
||||
3. `run_complete` line at INFO with
|
||||
`scanned=2, matched=2, posted=1, skipped=1, posted_ids=["id-2"]`.
|
||||
|
||||
# Related
|
||||
|
||||
* [Anniversary Matching](/architecture/anniversary-matching.md)
|
||||
* [Logging & Run Summary](/architecture/logging.md)
|
||||
* [Config Schema](/architecture/config-schema.md)
|
||||
* [Cron Lifecycle](/operations/cron-lifecycle.md)
|
||||
@@ -0,0 +1,66 @@
|
||||
---
|
||||
type: architecture
|
||||
title: Job 1089 Runtime Changes
|
||||
description: Container privilege separation, unified retry logging, configuration validation, and persistent data wiring introduced by the current implementation.
|
||||
tags: [architecture, docker, retry, logging, configuration]
|
||||
timestamp: 2026-08-04T20:44:00Z
|
||||
---
|
||||
|
||||
# Overview
|
||||
|
||||
The current runtime keeps the container entrypoint and cron daemon as root while executing the Python bot as the unprivileged `bot` user. The pipeline owns one retry budget for Git synchronization and Mastodon publishing, and structured retry events include the operation and exception text. Configuration now validates required values before optional defaults are applied.
|
||||
|
||||
# Wiring
|
||||
|
||||
```text
|
||||
Docker ENTRYPOINT /app/entrypoint.sh (root)
|
||||
-> validates environment and renders /etc/cron.d/tenbackward
|
||||
-> cron -f
|
||||
-> /usr/local/bin/run-bot.sh
|
||||
-> setpriv/su to bot
|
||||
-> python -m tenbackward
|
||||
-> load_config()
|
||||
-> _run_with_retry()
|
||||
-> _run_once()
|
||||
-> ensure_repo(..., max_retries=0, logger=...)
|
||||
-> match posts
|
||||
-> PostedStore dedupe
|
||||
-> publish_mastodon()
|
||||
-> PostedStore.mark_posted_many()
|
||||
```
|
||||
|
||||
The single `./data:/app/data` Compose bind mount contains both the blog clone and `posted.json`; no file-level bind mount is used.
|
||||
|
||||
# Retry and Logging Contract
|
||||
|
||||
`MAX_RETRIES` is interpreted as the maximum number of attempts, with at least one attempt when configured as zero. Backoff delays are `1, 2, 4, ...` seconds between attempts. Git clone/pull retries are surfaced to the runner with operation labels (`git clone` or `git pull`), while publishing failures use `mastodon post`. Fatal configuration, local-repository, and publishing validation errors are not retried.
|
||||
|
||||
Retry events include an `error` field containing the exception message. `log_event()` sends structured fields through the same redaction path as other log helpers.
|
||||
|
||||
# Configuration Validation
|
||||
|
||||
`load_config()` merges dotenv values and process environment values, validates required keys against that raw merged map, then applies defaults. `MAX_RETRIES`, `BLOG_REPO_URL`, and `BLOG_DIR` may use defaults; the required Mastodon, site, hashtag, prefix, schedule, and timezone values must be explicitly non-empty.
|
||||
|
||||
# Key Files
|
||||
|
||||
| Path | Responsibility |
|
||||
|---|---|
|
||||
| `/repo/Dockerfile` | Installs cron, Git, timezone data, and `setpriv`; leaves the final process root-capable for cron and owns `/app/data` by `bot`. |
|
||||
| `/repo/entrypoint.sh` | Validates environment, renders the daily cron entry, installs it as root, and starts foreground cron. |
|
||||
| `/repo/run-bot.sh` | Drops from root to `bot` using `setpriv` or `su`, then invokes the Python module. |
|
||||
| `/repo/docker-compose.yml` | Supplies environment values and mounts `./data` at `/app/data`. |
|
||||
| `/repo/src/tenbackward/main.py` | Owns the unified retry loop and passes logger/retry controls into repository synchronization. |
|
||||
| `/repo/src/tenbackward/blog.py` | Clones or fast-forwards the blog and emits operation-specific retry events. |
|
||||
| `/repo/src/tenbackward/config.py` | Performs raw required-key validation and builds the typed configuration. |
|
||||
| `/repo/src/tenbackward/logging_setup.py` | Provides structured events, redaction, and JSON-per-line formatting. |
|
||||
| `/repo/tests/test_docker_artifacts.py` | Verifies container privilege and artifact wiring. |
|
||||
| `/repo/tests/test_config.py` | Verifies required-key validation and defaults. |
|
||||
| `/repo/tests/test_run_logging.py` | Verifies retry fields, attempt behavior, and state safety. |
|
||||
|
||||
# Related
|
||||
|
||||
* [System Architecture](/architecture/system-overview.md)
|
||||
* [Config Schema](/architecture/config-schema.md)
|
||||
* [Pipeline Runner](/architecture/pipeline-runner.md)
|
||||
* [Environment Variable Setup](/operations/environment-setup.md)
|
||||
* [Daily Run Guide](/guides/daily-run.md)
|
||||
@@ -0,0 +1,133 @@
|
||||
---
|
||||
type: architecture
|
||||
title: System Architecture
|
||||
description: Component map of 10Backward — how configuration, blog synchronization, anniversary matching, logging, the pipeline runner, state persistence, and container entrypoint are wired together.
|
||||
tags: [architecture, overview]
|
||||
timestamp: 2026-08-04T18:55:00Z
|
||||
---
|
||||
|
||||
# Overview
|
||||
|
||||
`10Backward` is a Mastodon daily-throwback bot that runs as a single
|
||||
containerised cron job. Job 1086 adds anniversary matching across Jekyll
|
||||
posts, backed by the blog clone/pull workflow from Job 1084 and the
|
||||
structured logging, retry-capable runner, and deduplication state from
|
||||
Job 1083. The current pipeline identifies matching posts, publishes a
|
||||
single combined Mastodon status for the new matches, and records the
|
||||
published identifiers in `posted.json`.
|
||||
|
||||
The container boots, validates environment configuration, renders a
|
||||
`/etc/cron.d/tenbackward` entry that fires once per day at the
|
||||
configured `RUN_AT`, and then runs `python -m tenbackward` in the
|
||||
foreground of the `cron` process. Each scheduled invocation calls
|
||||
`tenbackward.main.main()`, which:
|
||||
|
||||
1. Installs the JSON logging formatter on the root logger.
|
||||
2. Loads and validates the `Config`.
|
||||
3. Ensures the data directory exists.
|
||||
4. Emits a `startup` log line.
|
||||
5. Executes one pipeline pass wrapped in a retry loop:
|
||||
`ensure_repo` → `find_anniversary_matches` → `PostedStore` dedupe →
|
||||
`publish_mastodon` → `PostedStore.mark_posted_many`.
|
||||
6. Emits a `run_complete` summary (skipped silently if there were no candidates).
|
||||
|
||||
# Components
|
||||
|
||||
| Component | Responsibility |
|
||||
|--------------------------|------------------------------------------------------------------------------------------------------------|
|
||||
| `entrypoint.sh` | Validates required env vars, renders the cron file from `RUN_AT`/`TZ`, then `exec cron -f`. |
|
||||
| `tenbackward.main` | CLI entry point; orchestrates startup logging, retry-wrapped pipeline pass, and run summary. |
|
||||
| `tenbackward.config` | Loads `.env` + process env, applies defaults, validates schema, produces a typed `Config` dataclass. |
|
||||
| `tenbackward.logging_setup` | JSON formatter (one log record per line), secret redaction, and `log_startup` / `log_error` / `log_run_summary` helpers. |
|
||||
| `tenbackward.state` | `PostedStore` — self-contained dedup store. `posted.json` holds `{"posted": [str, ...]}`; mutations are serialised by an `fcntl.flock` on a sibling lock file, writes go through a temp-file replace, and `load()` auto-creates an empty list when the file is missing. |
|
||||
| `tenbackward.blog` | GitPython `ensure_repo()` — clones the configured blog repo on first run, fast-forwards it via `pull --ff-only` thereafter, with exponential-backoff retries on transient network errors. |
|
||||
| `tenbackward.matching` | Walks `_posts/blog/**/*.md`, parses Jekyll front matter and filenames, and yields posts exactly ten years before the current date using Berlin-time and leap-day rules. |
|
||||
| `tenbackward.publishing` | `publish_mastodon()` — the single success boundary between the runner and the Mastodon HTTP API. Composes a combined status for the day's matches, validates the 500-character limit, and posts via `mastodon.Mastodon.status_post`. Raises `PublishError` on composition, length, or API failure; the runner treats it like any other pipeline exception. |
|
||||
| `/etc/cron.d/tenbackward`| Rendered cron file. One daily line that `cd /app` and runs `python -m tenbackward`. |
|
||||
|
||||
# Communication & Wiring
|
||||
|
||||
```
|
||||
+-------------------+ env vars +-------------------------+
|
||||
| entrypoint | -------------------> | config.load_config |
|
||||
| (.sh) | | (Config dataclass) |
|
||||
+---------+---------+ +-----------+-------------+
|
||||
| |
|
||||
| installs cron file | feeds
|
||||
v v
|
||||
+-------------------+ +-------------------------+
|
||||
| /etc/cron.d/ | -- daily fires --> | tenbackward.main.main |
|
||||
| tenbackward | | - configure logging |
|
||||
+-------------------+ | - retry-wrapped pass |
|
||||
+-----------+-------------+
|
||||
|
|
||||
v
|
||||
+-------------------------+
|
||||
| _run_with_retry(...) |
|
||||
| -> _run_once(...) |
|
||||
| -> ensure_repo(...) |
|
||||
| -> matching |
|
||||
| find_anniversary_|
|
||||
| matches |
|
||||
| -> PostedStore |
|
||||
| {is_posted, |
|
||||
| mark_posted_many}|
|
||||
| -> publishing |
|
||||
| publish_mastodon |
|
||||
| -> PostedStore |
|
||||
| mark_posted_many |
|
||||
+-------------------------+
|
||||
|
|
||||
v
|
||||
+-------------------------+
|
||||
| Mastodon HTTP API |
|
||||
| status_post(...) |
|
||||
+-------------------------+
|
||||
```
|
||||
|
||||
* **Env → Config.** `load_config()` merges a `.env` file (when
|
||||
present) under the process environment via `dotenv_values` + `load_dotenv`,
|
||||
applies defaults, then validates the merged map before producing the
|
||||
`Config` dataclass.
|
||||
* **Cron → main.** Each scheduled tick re-runs `python -m tenbackward`,
|
||||
so every run is a fresh interpreter invocation.
|
||||
* **Matching → publishing.** `find_anniversary_matches` yields
|
||||
`MatchedPost` objects; the runner dedupes against `PostedStore`, then
|
||||
passes the unposted objects to `publish_mastodon`. See
|
||||
[Mastodon Publishing](/architecture/mastodon-publishing.md).
|
||||
* **Pipeline state.** `_run_once` reads `posted.json` via
|
||||
`PostedStore.is_posted()` for dedupe, then writes it back via
|
||||
`PostedStore.mark_posted_many()` **only after** `publish_mastodon`
|
||||
returns. The store serialises concurrent runs with an
|
||||
`fcntl.flock` on a sibling lock file and writes via temp-file
|
||||
rename.
|
||||
|
||||
# Key Files
|
||||
|
||||
| Path | Responsibility |
|
||||
|-------------------------------------|---------------------------------------------------------------------------------|
|
||||
| `/repo/entrypoint.sh` | Bootstraps cron; validates required env vars; renders `/etc/cron.d/tenbackward`. |
|
||||
| `/repo/src/tenbackward/main.py` | CLI entry, retry wrapper, pipeline counters. |
|
||||
| `/repo/src/tenbackward/config.py` | Env merging, defaults, validation, `Config` dataclass, `ConfigError`. |
|
||||
| `/repo/src/tenbackward/logging_setup.py` | JSON formatter, secret redaction, structured event helpers. |
|
||||
| `/repo/src/tenbackward/state.py` | `PostedStore` + module helpers (`load_posted`, `is_posted`, `mark_posted`, `mark_posted_many`); list-shaped JSON, `fcntl.flock`, atomic temp-file replace, env-var data dir. |
|
||||
| `/repo/src/tenbackward/blog.py` | `ensure_repo()` — GitPython clone + fast-forward pull with `2^n` retry/backoff; raises `BlogRepoError` on local modifications or exhausted retries. |
|
||||
| `/repo/src/tenbackward/matching.py` | Jekyll post discovery, front matter parsing, anniversary and leap-day matching, and canonical URL construction. |
|
||||
| `/repo/src/tenbackward/publishing.py` | `publish_mastodon()` — composes a combined Mastodon status, validates the 500-character limit, posts via `mastodon.Mastodon.status_post`. Pure helpers `build_status_text` and `validate_status` keep network code out of tests. |
|
||||
| `/repo/src/tenbackward/__main__.py` | Module entrypoint that invokes `tenbackward.main.main()`. |
|
||||
| `/repo/.env.example` | Canonical list of environment variables. |
|
||||
| `/repo/docker-compose.yml` | Service definition; binds env vars from the host `.env`. |
|
||||
| `/repo/Dockerfile` | Builds the runtime image. |
|
||||
| `/repo/crontab/tenbackward.cron` | Cron template shipped in the image. |
|
||||
| `/repo/crontab/install-cron.sh` | Installs the rendered cron file with mode 0644; refuses placeholder leftovers. |
|
||||
| `/repo/tests/` | Pytest suite covering config, logging, runner, cron, and entrypoint validation. |
|
||||
|
||||
# Related
|
||||
|
||||
* [Config Schema](/architecture/config-schema.md)
|
||||
* [Logging & Run Summary](/architecture/logging.md)
|
||||
* [Pipeline Runner](/architecture/pipeline-runner.md)
|
||||
* [Anniversary Matching](/architecture/anniversary-matching.md)
|
||||
* [Mastodon Publishing](/architecture/mastodon-publishing.md)
|
||||
* [Environment Variable Setup](/operations/environment-setup.md)
|
||||
* [Cron Lifecycle](/operations/cron-lifecycle.md)
|
||||
@@ -0,0 +1,125 @@
|
||||
---
|
||||
type: guide
|
||||
title: Daily Run Guide
|
||||
description: Operator- and tester-focused walkthrough of one daily cron tick — repository synchronization, anniversary matching, deduplication, logging, and retry behaviour.
|
||||
tags: [guide, run, daily, tester]
|
||||
timestamp: 2026-08-04T17:51:00Z
|
||||
---
|
||||
|
||||
# Purpose
|
||||
|
||||
A tester or operator needs to know **what should happen** each time
|
||||
the cron tick fires. This guide describes the contract from the
|
||||
outside of the container, in observable terms: log lines, exit codes,
|
||||
files appearing on the volume, and the "silent on no matches"
|
||||
behaviour that the runner enforces.
|
||||
|
||||
# Triggering a Run Manually
|
||||
|
||||
```bash
|
||||
docker compose exec tenbackward /usr/local/bin/python -m tenbackward
|
||||
```
|
||||
|
||||
This is the same command the container's cron entry issues once per
|
||||
day at `RUN_AT`. Use it to verify behaviour without waiting for the
|
||||
scheduled tick.
|
||||
|
||||
# What the Run Does
|
||||
|
||||
Before scanning, the process ensures the configured blog repository exists under `/app/data/blog` and is fast-forwarded from `BLOG_REPO_URL`. It then walks `_posts/blog/**/*.md`, finds posts whose date is exactly ten years before today, skips IDs already present in `posted.json`, composes a single Mastodon status listing each new match's title and canonical URL followed by the configured hashtags, and — only after the Mastodon API call succeeds — records the relative paths of the published posts in `posted.json`. The scheduled wrapper executes this Python process as the unprivileged `bot` user.
|
||||
|
||||
# What You Should See
|
||||
|
||||
The bot uses a **JSON-per-line** logger that writes to
|
||||
`/app/data/cron.log` (because the cron line redirects with
|
||||
`>> /app/data/cron.log 2>&1`). Inspecting the file with
|
||||
`docker compose exec tenbackward tail -n 100 /app/data/cron.log`
|
||||
should show output similar to:
|
||||
|
||||
```json
|
||||
{"ts": "2026-08-04T09:00:00+00:00", "level": "INFO", "logger": "tenbackward", "message": "startup", "event": "startup", "version": "0.1.0", "site_url": "https://blog.example.com", "run_at": "09:00", "tz": "Europe/Berlin", "hashtags": "#throwback,#10backward", "throwback_prefix": "Heute vor 10 Jahren:"}
|
||||
```
|
||||
|
||||
On the current scaffold (`_iter_candidates` is intentionally empty),
|
||||
this is the **only** line you should see — the runner emits no
|
||||
`run_complete` when no candidates were found. This is the
|
||||
"silent on no matches" contract.
|
||||
|
||||
# Expected Container Behaviour
|
||||
|
||||
| Scenario | Lines in `cron.log` | Container exit |
|
||||
|-------------------------------------------|----------------------------------------------------------------|------------------|
|
||||
| Config valid, no anniversary matches | 1× `startup` | `0` |
|
||||
| Blog clone or pull fails | 1× `blog_repo_error` | `1` |
|
||||
| Config valid, matching IDs are new | 1× `startup`, then 1× `run_complete` with `posted=N` | `0` |
|
||||
| Config valid, all matches already posted | 1× `startup`, then 1× `run_complete` with `posted=0` | `0` |
|
||||
| Pipeline raises once, recovers | `retry_attempt`, then `run_complete` | `0` |
|
||||
| Pipeline keeps raising (budget exhausted) | `retry_attempt` events, then `retry_exhausted` with operation and error | `1` |
|
||||
| Config invalid | 1× `configuration_error` (extras describe what failed) | `2` |
|
||||
|
||||
> **Tester tip** — `docker compose ps` should report the container as
|
||||
> `running` after a `pipeline_failed` exit *only* if cron has not yet
|
||||
> fired again. A single failed pipeline tick does not kill the container
|
||||
> itself; cron re-runs it the next day.
|
||||
|
||||
# Files the Tester Should Look For
|
||||
|
||||
| Path (inside container) | When it appears |
|
||||
|-------------------------|--------------------------------------------|
|
||||
| `/app/data/cron.log` | Always (cron appends stdout/stderr here). |
|
||||
| `/app/data/posted.json` | Created when the matcher or a state check first loads the store; rewritten only when a new ID is posted. The file contains relative Jekyll paths under a `posted` list. |
|
||||
|
||||
To verify these from the host:
|
||||
|
||||
```bash
|
||||
docker compose exec tenbackward ls -la /app/data
|
||||
docker compose exec tenbackward cat /app/data/cron.log
|
||||
```
|
||||
|
||||
# Visual Elements
|
||||
|
||||
This is a server-side bot, so there is no UI. The "UI" consists of:
|
||||
|
||||
* **Log lines on stdout** — one JSON object per `INFO`/`ERROR` event.
|
||||
* **Exit code** — `0` (healthy), `1` (pipeline exhausted), or `2` (config).
|
||||
* **`posted.json`** — JSON state file; its mtime changes whenever a new id is persisted.
|
||||
* **`/app/data/blog`** — synchronized local clone containing the Jekyll posts scanned for anniversaries.
|
||||
|
||||
# Examples
|
||||
|
||||
## Healthy scaffold tick
|
||||
|
||||
```bash
|
||||
$ docker compose exec tenbackward python -m tenbackward
|
||||
{"ts":"...","event":"startup",...}
|
||||
$ echo $?
|
||||
0
|
||||
```
|
||||
|
||||
## Bad `RUN_AT`
|
||||
|
||||
```bash
|
||||
$ RUN_AT=25:99 docker compose exec tenbackward python -m tenbackward
|
||||
ERROR: RUN_AT='25:99' must be in HH:MM (24-hour) format
|
||||
$ docker compose ps
|
||||
tenbackward Exit 1
|
||||
```
|
||||
|
||||
(The error message lands on stderr in the entrypoint path and is
|
||||
captured by `cron.log`.)
|
||||
|
||||
## Missing required env var
|
||||
|
||||
Unset `MAX_RETRIES`, restart the container:
|
||||
|
||||
```bash
|
||||
$ docker compose up --build
|
||||
tenbackward | ERROR: missing required environment variable(s): MAX_RETRIES
|
||||
tenbackward exited with code 1
|
||||
```
|
||||
|
||||
# Related
|
||||
|
||||
* [Environment Variable Setup](/operations/environment-setup.md)
|
||||
* [Pipeline Runner](/architecture/pipeline-runner.md)
|
||||
* [Logging & Run Summary](/architecture/logging.md)
|
||||
@@ -0,0 +1,74 @@
|
||||
---
|
||||
type: guide
|
||||
title: Runtime Verification Guide
|
||||
description: Tester workflow for validating root cron startup, unprivileged bot execution, retry events, and persistent data behavior.
|
||||
tags: [guide, tester, docker, runtime]
|
||||
timestamp: 2026-08-04T20:44:00Z
|
||||
---
|
||||
|
||||
# Setup
|
||||
|
||||
1. Copy `.env.example` to `.env` and provide valid required values.
|
||||
2. Build and start the service:
|
||||
|
||||
```bash
|
||||
docker compose up --build -d
|
||||
```
|
||||
|
||||
3. Follow container output:
|
||||
|
||||
```bash
|
||||
docker compose logs -f bot
|
||||
```
|
||||
|
||||
# Expected Runtime Behavior
|
||||
|
||||
The container remains running with `cron -f` as its foreground process. Startup output includes the configured `RUN_AT` and `TZ`. The scheduled cron command invokes `/usr/local/bin/run-bot.sh`, which runs the Python bot as user `bot`, not root.
|
||||
|
||||
The host `./data` directory should contain the synchronized `blog/` clone and, after the state store is loaded, `posted.json`. Both are under the one `/app/data` mount.
|
||||
|
||||
# Manual Run
|
||||
|
||||
Run the same application command without waiting for cron:
|
||||
|
||||
```bash
|
||||
docker compose exec bot /usr/local/bin/python -m tenbackward
|
||||
```
|
||||
|
||||
A valid run exits `0`. A configuration failure exits `2`; an exhausted pipeline retry budget exits `1`.
|
||||
|
||||
# Retry Checks
|
||||
|
||||
To exercise transient failure handling, use a test repository or injected test double that fails once and then succeeds. Verify JSON events contain:
|
||||
|
||||
| Event | Expected fields |
|
||||
|---|---|
|
||||
| `retry_attempt` | `operation`, `attempt`, and the original `error` text. |
|
||||
| `retry_exhausted` | Final `operation`, total `attempts`, and `error` text when all attempts fail. |
|
||||
| `run_complete` | Counters and `posted_ids` only after a successful pass. |
|
||||
|
||||
With `MAX_RETRIES=0`, one attempt is still made and no sleep occurs. With a larger value, delays occur only between attempts and follow powers of two beginning at one second.
|
||||
|
||||
# Configuration Checks
|
||||
|
||||
Remove a required variable from `.env` and restart the service. The entrypoint should report the missing variable before cron starts. If the shell check is bypassed and the Python module runs directly, `configuration_error` should identify the missing key and return exit code `2` rather than silently substituting an application default.
|
||||
|
||||
# Persistence and Failure Safety
|
||||
|
||||
After a successful publication, inspect `/app/data/posted.json` and confirm new relative post paths appear under the `posted` list. If Mastodon publishing fails, the path must not be recorded; a later retry should be able to publish it again.
|
||||
|
||||
# Key Visual/Observable Elements
|
||||
|
||||
This service has no graphical UI. Test-facing outputs are:
|
||||
|
||||
* Container status and `docker compose logs` output.
|
||||
* JSON-per-line startup, retry, failure, and completion events.
|
||||
* Exit codes from manual runs.
|
||||
* `/app/data/blog` and `/app/data/posted.json` on the mounted host directory.
|
||||
|
||||
# Related
|
||||
|
||||
* [Daily Run Guide](/guides/daily-run.md)
|
||||
* [Job 1089 Runtime Changes](/architecture/runtime-changes.md)
|
||||
* [Environment Variable Setup](/operations/environment-setup.md)
|
||||
* [Cron Lifecycle](/operations/cron-lifecycle.md)
|
||||
@@ -0,0 +1,20 @@
|
||||
---
|
||||
okf_version: "0.1"
|
||||
---
|
||||
|
||||
# Architecture
|
||||
* [System Architecture](/architecture/system-overview.md) — Component map of 10Backward: entrypoint, config, logging, runner, state, publishing boundary, and cron wiring.
|
||||
* [Runtime Changes](/architecture/runtime-changes.md) — Root cron/unprivileged bot separation, unified retries, raw configuration validation, and data-volume wiring.
|
||||
* [Config Schema](/architecture/config-schema.md) — Required/optional env vars, validation rules, and the typed Config dataclass.
|
||||
* [Logging & Run Summary](/architecture/logging.md) — JSON formatter, secret redaction, and structured event helpers.
|
||||
* [Pipeline Runner](/architecture/pipeline-runner.md) — How a cron tick synchronizes the blog, matches anniversaries, applies deduplication, publishes to Mastodon, retries failures, and emits run events.
|
||||
* [Anniversary Matching](/architecture/anniversary-matching.md) — Jekyll filename/front matter rules, ten-year date matching, leap-day handling, and candidate identifiers.
|
||||
* [Mastodon Publishing](/architecture/mastodon-publishing.md) — The single success boundary: status composition, 500-character validation, and the Mastodon API call.
|
||||
|
||||
# Operations
|
||||
* [Environment Variable Setup](/operations/environment-setup.md) — Required env vars, renamed keys, and how to populate `.env`.
|
||||
* [Cron Lifecycle](/operations/cron-lifecycle.md) — How `entrypoint.sh` renders `/etc/cron.d/tenbackward` and hands off to `cron -f`.
|
||||
|
||||
# User Guides
|
||||
* [Daily Run Guide](/guides/daily-run.md) — Operator walkthrough of repository sync, anniversary matching, deduplication, publishing, and expected logs.
|
||||
* [Runtime Verification Guide](/guides/runtime-verification.md) — Tester steps for privilege separation, retry events, configuration failures, and persistent state.
|
||||
@@ -0,0 +1,96 @@
|
||||
---
|
||||
type: operations
|
||||
title: Cron Lifecycle
|
||||
description: How entrypoint.sh validates env, renders /etc/cron.d/tenbackward from RUN_AT/TZ, and hands off to cron -f.
|
||||
tags: [cron, entrypoint, ops]
|
||||
timestamp: 2026-08-04T17:51:00Z
|
||||
---
|
||||
|
||||
# Purpose
|
||||
|
||||
`entrypoint.sh` is the first process inside the container. Its job is
|
||||
to refuse to boot when something is wrong with env config and to
|
||||
materialise a cron file from the templated values.
|
||||
|
||||
# Sequence
|
||||
|
||||
1. Set `DEBIAN_FRONTEND=noninteractive`.
|
||||
2. Set `TZ` (default `Europe/Berlin`) and link `/etc/localtime` if
|
||||
`/usr/share/zoneinfo/${TZ}` exists and `/etc/localtime` does not.
|
||||
3. Iterate over `required_vars`. Any empty variable aborts with
|
||||
`ERROR: missing required environment variable(s): ...` and exits
|
||||
`1`. The current required set is listed in
|
||||
[Environment Variable Setup](/operations/environment-setup.md).
|
||||
4. Validate `RUN_AT` against
|
||||
`^([01][0-9]|2[0-3]):[0-5][0-9]$`. Invalid input exits `1` with
|
||||
`ERROR: RUN_AT='X' must be in HH:MM (24-hour) format`.
|
||||
5. Render `/etc/cron.d/tenbackward` into a `mktemp` file with:
|
||||
|
||||
```cron
|
||||
SHELL=/bin/bash
|
||||
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
TZ=${TZ}
|
||||
${minute} ${hour} * * * cd /app && /usr/local/bin/python -m tenbackward >> /app/data/cron.log 2>&1
|
||||
```
|
||||
|
||||
6. Install the rendered file with mode `0644`, root:root.
|
||||
7. Run `crontab/install-cron.sh /etc/cron.d/tenbackward`. The helper
|
||||
refuses files that still contain the `__RUN_AT__` placeholder and
|
||||
files that do not start with an `^[A-Z_]+=` cron env line.
|
||||
8. Print a one-line confirmation: `tenbackward: starting cron
|
||||
(RUN_AT=..., TZ=...)`.
|
||||
9. `exec cron -f`.
|
||||
|
||||
# Output Files
|
||||
|
||||
| Path | Owner / Mode | Purpose |
|
||||
|------------------------------|--------------|------------------------------------------------------|
|
||||
| `/etc/cron.d/tenbackward` | root / 0644 | The rendered cron file. |
|
||||
| `/app/data/cron.log` | container user / append | `python -m tenbackward` stdout+stderr. |
|
||||
| `/app/data/posted.json` | container user / rw | Persisted state from `tenbackward.state`. |
|
||||
|
||||
# Cron Helpers
|
||||
|
||||
| File | Purpose |
|
||||
|---------------------------------|------------------------------------------------------------------------|
|
||||
| `crontab/tenbackward.cron` | Static template (env + PATH) shipped in the image. |
|
||||
| `crontab/install-cron.sh` | Validates and installs a rendered cron file with mode 0644. |
|
||||
|
||||
# Operator Recipes
|
||||
|
||||
* **Trigger a manual run** without waiting for the cron tick:
|
||||
|
||||
```bash
|
||||
docker compose exec tenbackward /usr/local/bin/python -m tenbackward
|
||||
```
|
||||
|
||||
* **Inspect the rendered cron file** (the file the container actually
|
||||
installed):
|
||||
|
||||
```bash
|
||||
docker compose exec tenbackward cat /etc/cron.d/tenbackward
|
||||
```
|
||||
|
||||
* **Tail the structured logs** produced by the bot:
|
||||
|
||||
```bash
|
||||
docker compose exec tenbackward tail -n 100 /app/data/cron.log
|
||||
```
|
||||
|
||||
Expect one JSON object per line (`startup`, optional `run_complete`,
|
||||
or `pipeline_error` / `pipeline_failed`).
|
||||
|
||||
# Build Script Notes
|
||||
|
||||
`build.sh` now `cd`s into its own script directory (so it works
|
||||
no matter where it is invoked from) and, when possible, symlinks
|
||||
itself to `/usr/local/bin/build.sh` so the build can also be run as a
|
||||
plain `build.sh` command inside the build context. The symlink is a
|
||||
best-effort `|| true`, so a read-only filesystem will not break the
|
||||
build.
|
||||
|
||||
# Related
|
||||
|
||||
* [Environment Variable Setup](/operations/environment-setup.md)
|
||||
* [Pipeline Runner](/architecture/pipeline-runner.md)
|
||||
* [Logging & Run Summary](/architecture/logging.md)
|
||||
@@ -0,0 +1,86 @@
|
||||
---
|
||||
type: operations
|
||||
title: Environment Variable Setup
|
||||
description: How to provide and validate the required env vars for the 10Backward container — .env, docker-compose, and the runtime check in entrypoint.sh.
|
||||
tags: [env, ops, setup]
|
||||
timestamp: 2026-08-04T17:51:00Z
|
||||
---
|
||||
|
||||
# Purpose
|
||||
|
||||
Job 1083 promoted every previously-optional knob in `.env.example`
|
||||
to required, and renamed three of them. Operators must populate the
|
||||
full set before the container will boot.
|
||||
|
||||
# Required Variables
|
||||
|
||||
| Variable | Example value | Purpose |
|
||||
|--------------------------|--------------------------------|---------------------------------------------------|
|
||||
| `MASTODON_BASE_URL` | `https://mastodon.social` | Mastodon instance to post against. |
|
||||
| `MASTODON_ACCESS_TOKEN` | _(from your Mastodon account)_ | OAuth access token. |
|
||||
| `MASTODON_VISIBILITY` | `public` | Post visibility (`public` or `unlisted`). |
|
||||
| `SITE_URL` | `https://blog.example.com` | Source blog URL. |
|
||||
| `HASHTAGS` | `#throwback,#10backward` | Hashtags appended to every throwback post. |
|
||||
| `THROWBACK_PREFIX` | `Heute vor 10 Jahren:` | Prefix prepended to every Mastodon status. |
|
||||
| `MAX_RETRIES` | `5` | Maximum number of pipeline attempts. |
|
||||
| `RUN_AT` | `09:00` | Daily fire time (`HH:MM`, 24-hour). |
|
||||
| `TZ` | `Europe/Berlin` | IANA timezone for cron and application. |
|
||||
| `BLOG_REPO_URL` | project default | Optional source repository URL. |
|
||||
| `BLOG_DIR` | `blog` | Optional clone directory, relative to `DATA_DIR` unless absolute. |
|
||||
|
||||
# Where Each Name Is Enforced
|
||||
|
||||
* `entrypoint.sh` — validates all nine required keys, validates `RUN_AT`, and installs the cron file as root. The cron-launched wrapper then drops privileges to `bot` before Python runs.
|
||||
* `tenbackward.config.validate_config` — validates the merged raw required keys plus URL, visibility, timezone, and retry syntax. Defaults are applied only after required-key validation.
|
||||
* `docker-compose.yml` — supplies development environment values and mounts `./data` at `/app/data`.
|
||||
|
||||
# Setup Steps
|
||||
|
||||
1. Copy the template:
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
2. Replace placeholder values in `.env` (the example ships with
|
||||
`MASTODON_ACCESS_TOKEN=replace-me` and `https://mastodon.example`).
|
||||
|
||||
3. Verify the file parses by running the validator directly:
|
||||
|
||||
```bash
|
||||
python -m tenbackward
|
||||
```
|
||||
|
||||
A successful validation run emits one `startup` JSON log line and
|
||||
exits `0`. Missing/invalid config exits `2`.
|
||||
|
||||
4. Bring up the container:
|
||||
|
||||
```bash
|
||||
docker compose up --build
|
||||
```
|
||||
|
||||
The required-var check in `entrypoint.sh` runs first; if any
|
||||
variable is empty, the container exits before `cron` starts.
|
||||
|
||||
# Renames vs. Job 1082
|
||||
|
||||
| Old key | New key |
|
||||
|------------------------|--------------------------|
|
||||
| `MASTODON_VISIBILITY` | `VISIBILITY` |
|
||||
| `THROWBACK_PREFIX` | `THROWNBACK_PREFIX` |
|
||||
| `RETRY_COUNT` | `MAX_RETRIES` |
|
||||
|
||||
If you have an existing `.env` from the scaffold, rename these
|
||||
manually. The renamed variables are **not** backwards compatible — the
|
||||
container will refuse to start with both versions set.
|
||||
|
||||
# Defaults
|
||||
|
||||
`MAX_RETRIES` defaults to `5`; `BLOG_REPO_URL` and `BLOG_DIR` also have application defaults. Other application-facing required keys must be present in the raw merged dotenv/process environment. The Docker image supplies `TZ=Europe/Berlin`, and Compose supplies development fallbacks, but production deployments should set every required value explicitly.
|
||||
|
||||
# Related
|
||||
|
||||
* [Config Schema](/architecture/config-schema.md)
|
||||
* [System Architecture](/architecture/system-overview.md)
|
||||
* [Cron Lifecycle](/operations/cron-lifecycle.md)
|
||||
Executable
+59
@@ -0,0 +1,59 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
export DEBIAN_FRONTEND="${DEBIAN_FRONTEND:-noninteractive}"
|
||||
export TZ="${TZ:-Europe/Berlin}"
|
||||
|
||||
if [ -f /usr/share/zoneinfo/"${TZ}" ] && [ ! -f /etc/localtime ]; then
|
||||
ln -snf /usr/share/zoneinfo/"${TZ}" /etc/localtime || true
|
||||
fi
|
||||
|
||||
required_vars=(
|
||||
MASTODON_BASE_URL
|
||||
MASTODON_ACCESS_TOKEN
|
||||
MASTODON_VISIBILITY
|
||||
SITE_URL
|
||||
HASHTAGS
|
||||
THROWBACK_PREFIX
|
||||
MAX_RETRIES
|
||||
RUN_AT
|
||||
TZ
|
||||
)
|
||||
|
||||
missing=()
|
||||
for var in "${required_vars[@]}"; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
missing+=("${var}")
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "${#missing[@]}" -gt 0 ]; then
|
||||
echo "ERROR: missing required environment variable(s): ${missing[*]}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
run_at="${RUN_AT}"
|
||||
if ! [[ "${run_at}" =~ ^([01][0-9]|2[0-3]):[0-5][0-9]$ ]]; then
|
||||
echo "ERROR: RUN_AT='${run_at}' must be in HH:MM (24-hour) format" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
hour="${run_at%%:*}"
|
||||
minute="${run_at##*:}"
|
||||
|
||||
tmp_cron="$(mktemp)"
|
||||
trap 'rm -f "${tmp_cron}"' EXIT
|
||||
|
||||
{
|
||||
echo "SHELL=/bin/bash"
|
||||
echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
echo "TZ=${TZ}"
|
||||
echo "${minute} ${hour} * * * /usr/local/bin/run-bot.sh >> /proc/1/fd/1 2>&1"
|
||||
} > "${tmp_cron}"
|
||||
|
||||
install -m 0644 -o root -g root "${tmp_cron}" /etc/cron.d/tenbackward
|
||||
|
||||
/app/crontab/install-cron.sh /etc/cron.d/tenbackward
|
||||
|
||||
echo "tenbackward: starting cron (RUN_AT=${run_at}, TZ=${TZ})"
|
||||
exec cron -f
|
||||
@@ -0,0 +1,3 @@
|
||||
[pytest]
|
||||
testpaths = tests
|
||||
pythonpath = src
|
||||
@@ -0,0 +1 @@
|
||||
pytest==8.3.3
|
||||
@@ -0,0 +1,4 @@
|
||||
python-dotenv==1.0.1
|
||||
python-frontmatter==1.1.0
|
||||
GitPython==3.1.43
|
||||
Mastodon.py==1.8.0
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
cd /app
|
||||
|
||||
bot_uid="$(id -u bot)"
|
||||
bot_gid="$(id -g bot)"
|
||||
|
||||
if command -v setpriv >/dev/null 2>&1; then
|
||||
exec setpriv --reuid="${bot_uid}" --regid="${bot_gid}" --clear-groups -- \
|
||||
/usr/local/bin/python -m tenbackward
|
||||
fi
|
||||
|
||||
exec su -s /bin/bash bot -c "exec /usr/local/bin/python -m tenbackward"
|
||||
@@ -0,0 +1,42 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "${SCRIPT_DIR}"
|
||||
|
||||
if [ -w /usr/local/bin ] && [ ! -e /usr/local/bin/build.sh ]; then
|
||||
ln -s "${SCRIPT_DIR}/build.sh" /usr/local/bin/build.sh || true
|
||||
fi
|
||||
if [ -w /usr/local/bin ] && [ ! -e /usr/local/bin/setup.sh ]; then
|
||||
ln -s "${SCRIPT_DIR}/setup.sh" /usr/local/bin/setup.sh || true
|
||||
fi
|
||||
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends \
|
||||
bash \
|
||||
git \
|
||||
ca-certificates \
|
||||
python3 \
|
||||
python3-pip \
|
||||
python3-venv \
|
||||
tzdata
|
||||
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
PYTHON_BIN="$(command -v python3)"
|
||||
PIP_BIN="$(command -v pip3 || true)"
|
||||
if [ -z "${PIP_BIN}" ] && [ -n "${PYTHON_BIN}" ]; then
|
||||
PIP_BIN="${PYTHON_BIN} -m pip"
|
||||
fi
|
||||
|
||||
if [ -n "${PIP_BIN}" ]; then
|
||||
${PIP_BIN} install --no-cache-dir --upgrade pip >/dev/null 2>&1 || true
|
||||
${PIP_BIN} install --no-cache-dir --break-system-packages -r /repo/requirements.txt || \
|
||||
${PIP_BIN} install --no-cache-dir -r /repo/requirements.txt
|
||||
${PIP_BIN} install --no-cache-dir --break-system-packages -r /repo/requirements-dev.txt || \
|
||||
${PIP_BIN} install --no-cache-dir -r /repo/requirements-dev.txt
|
||||
fi
|
||||
|
||||
cd /repo
|
||||
PYTHONPATH=/repo/src pytest -q
|
||||
@@ -0,0 +1,3 @@
|
||||
"""10Backward — Mastodon daily-throwback bot."""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,3 @@
|
||||
from .main import main
|
||||
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,229 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
import git
|
||||
from git import GitCommandError, InvalidGitRepositoryError, NoSuchPathError, Repo
|
||||
|
||||
|
||||
DEFAULT_REPO_URL = "https://git.chaospott.de/Chaospott/site"
|
||||
DEFAULT_BLOG_SUBDIR = "blog"
|
||||
MAX_RETRIES_DEFAULT = 5
|
||||
BACKOFF_BASE_SECONDS = 2
|
||||
REMOTE_NAME = "origin"
|
||||
BRANCH_NAME = "master"
|
||||
|
||||
_LOCAL_MODIFICATION_MARKERS = (
|
||||
"Your local changes",
|
||||
"would be overwritten",
|
||||
"Please commit your changes",
|
||||
"Please move or remove them",
|
||||
)
|
||||
|
||||
|
||||
class BlogRepoError(RuntimeError):
|
||||
"""Raised when the local blog working tree cannot be ensured.
|
||||
|
||||
Surface this as a fatal pipeline error (do not retry) so configuration
|
||||
problems fail immediately and transient network failures get a chance to
|
||||
recover via the retry loop.
|
||||
"""
|
||||
|
||||
|
||||
class BlogTransientError(RuntimeError):
|
||||
"""Wraps a transient HTTPS clone/pull failure that should be retried.
|
||||
|
||||
``operation`` records which Git action triggered the failure so the
|
||||
runner can label its log lines with "git clone" or "git pull".
|
||||
"""
|
||||
|
||||
def __init__(self, message: str, *, operation: str = "git pull") -> None:
|
||||
super().__init__(message)
|
||||
self.operation = operation
|
||||
|
||||
|
||||
def blog_dir(data_dir: Path) -> Path:
|
||||
return Path(data_dir) / DEFAULT_BLOG_SUBDIR
|
||||
|
||||
|
||||
def blog_repo_url() -> str:
|
||||
return os.environ.get("BLOG_REPO_URL") or DEFAULT_REPO_URL
|
||||
|
||||
|
||||
def _is_dir_empty(path: Path) -> bool:
|
||||
if not path.exists():
|
||||
return True
|
||||
try:
|
||||
next(path.iterdir())
|
||||
except StopIteration:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _is_local_modification_error(exc: GitCommandError) -> bool:
|
||||
message = str(exc)
|
||||
return any(marker in message for marker in _LOCAL_MODIFICATION_MARKERS)
|
||||
|
||||
|
||||
def _log_retry(
|
||||
logger: logging.Logger | None,
|
||||
*,
|
||||
operation: str,
|
||||
attempt: int,
|
||||
error: BaseException,
|
||||
) -> None:
|
||||
if logger is None:
|
||||
return
|
||||
logger.error(
|
||||
"blog_retry",
|
||||
extra={
|
||||
"event": "blog_retry",
|
||||
"operation": operation,
|
||||
"attempt": attempt,
|
||||
"error": str(error),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _retry_loop(
|
||||
*,
|
||||
operation: str,
|
||||
attempts: int,
|
||||
sleep: Callable[[float], None],
|
||||
logger: logging.Logger | None,
|
||||
action: Callable[[], None],
|
||||
) -> None:
|
||||
"""Invoke ``action`` up to ``attempts`` times with ``2 ** (attempt - 1)``
|
||||
second backoff between failures.
|
||||
|
||||
Transient errors are retried; :class:`BlogRepoError` (fatal local or
|
||||
validation problems) propagates immediately. On exhaustion, raises
|
||||
:class:`BlogTransientError` carrying the failing operation label.
|
||||
"""
|
||||
last_exc: BaseException | None = None
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
action()
|
||||
return
|
||||
except BlogRepoError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 — boundary retry hook
|
||||
last_exc = exc
|
||||
_log_retry(logger, operation=operation, attempt=attempt, error=exc)
|
||||
if attempt < attempts:
|
||||
sleep(BACKOFF_BASE_SECONDS ** (attempt - 1))
|
||||
assert last_exc is not None
|
||||
raise BlogTransientError(
|
||||
f"{operation} failed after {attempts} attempt(s): {last_exc}",
|
||||
operation=operation,
|
||||
) from last_exc
|
||||
|
||||
|
||||
def ensure_repo(
|
||||
data_dir: Path,
|
||||
*,
|
||||
repo_url: str | None = None,
|
||||
blog_path: Path | None = None,
|
||||
max_retries: int | None = None,
|
||||
logger: logging.Logger | None = None,
|
||||
sleep: Callable[[float], None] = time.sleep,
|
||||
clone_impl: Callable[[str, str], Repo] | None = None,
|
||||
fetch_impl: Callable[[Repo], Callable[[], object]] | None = None,
|
||||
pull_impl: Callable[[Repo], Callable[[], object]] | None = None,
|
||||
) -> Repo:
|
||||
"""Ensure a local working clone of the blog repository exists and is up to date.
|
||||
|
||||
Returns the :class:`git.Repo` pointing at an up-to-date working tree.
|
||||
Raises :class:`BlogRepoError` on irrecoverable failures and
|
||||
:class:`BlogTransientError` after a transient failure exhausts the
|
||||
retry budget (so the caller can label and re-attempt at the pipeline
|
||||
level).
|
||||
"""
|
||||
url = repo_url or blog_repo_url()
|
||||
target = blog_path or blog_dir(data_dir)
|
||||
retries = MAX_RETRIES_DEFAULT if max_retries is None else max_retries
|
||||
attempts = max(1, retries)
|
||||
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
needs_clone = _is_dir_empty(target)
|
||||
|
||||
if needs_clone:
|
||||
if logger is not None:
|
||||
logger.info(
|
||||
"blog_clone_start",
|
||||
extra={"event": "blog_clone_start", "url": url, "path": str(target)},
|
||||
)
|
||||
|
||||
cloned_holder: dict[str, Repo] = {}
|
||||
|
||||
def _do_clone() -> None:
|
||||
r = (clone_impl or Repo.clone_from)(url, str(target))
|
||||
cloned_holder["repo"] = r
|
||||
|
||||
_retry_loop(
|
||||
operation="git clone",
|
||||
attempts=attempts,
|
||||
sleep=sleep,
|
||||
logger=logger,
|
||||
action=_do_clone,
|
||||
)
|
||||
if logger is not None:
|
||||
logger.info(
|
||||
"blog_clone_complete",
|
||||
extra={"event": "blog_clone_complete", "path": str(target)},
|
||||
)
|
||||
if "repo" in cloned_holder:
|
||||
return cloned_holder["repo"]
|
||||
return Repo(str(target))
|
||||
|
||||
try:
|
||||
repo = Repo(str(target))
|
||||
except (InvalidGitRepositoryError, NoSuchPathError) as exc:
|
||||
raise BlogRepoError(
|
||||
f"{target} exists but is not a valid git repository: {exc}"
|
||||
) from exc
|
||||
|
||||
fetch = (fetch_impl or _default_fetch)(repo)
|
||||
pull = (pull_impl or _default_pull)(repo)
|
||||
|
||||
def _do_pull() -> None:
|
||||
try:
|
||||
fetch()
|
||||
pull()
|
||||
except GitCommandError as exc:
|
||||
if _is_local_modification_error(exc):
|
||||
if logger is not None:
|
||||
logger.error(
|
||||
"blog_local_modifications",
|
||||
extra={"event": "blog_local_modifications", "error": str(exc)},
|
||||
)
|
||||
raise BlogRepoError(
|
||||
"local modifications detected in blog working tree; aborting run"
|
||||
) from exc
|
||||
raise BlogTransientError(str(exc), operation="git pull") from exc
|
||||
|
||||
_retry_loop(
|
||||
operation="git pull",
|
||||
attempts=attempts,
|
||||
sleep=sleep,
|
||||
logger=logger,
|
||||
action=_do_pull,
|
||||
)
|
||||
return repo
|
||||
|
||||
|
||||
def _default_fetch(repo: Repo) -> Callable[[], object]:
|
||||
def _do() -> object:
|
||||
return repo.remotes[REMOTE_NAME].fetch()
|
||||
return _do
|
||||
|
||||
|
||||
def _default_pull(repo: Repo) -> Callable[[], object]:
|
||||
def _do() -> object:
|
||||
return repo.git.pull("--ff-only")
|
||||
return _do
|
||||
@@ -0,0 +1,267 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from urllib.parse import urlparse
|
||||
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
||||
|
||||
from dotenv import dotenv_values, load_dotenv
|
||||
|
||||
|
||||
REQUIRED_KEYS = (
|
||||
"MASTODON_BASE_URL",
|
||||
"MASTODON_ACCESS_TOKEN",
|
||||
"MASTODON_VISIBILITY",
|
||||
"SITE_URL",
|
||||
"HASHTAGS",
|
||||
"THROWBACK_PREFIX",
|
||||
"MAX_RETRIES",
|
||||
"RUN_AT",
|
||||
"TZ",
|
||||
)
|
||||
|
||||
OPTIONAL_KEYS: tuple[str, ...] = ("BLOG_REPO_URL", "BLOG_DIR")
|
||||
|
||||
DEFAULTS = {
|
||||
"MAX_RETRIES": "5",
|
||||
"BLOG_REPO_URL": "https://git.chaospott.de/Chaospott/site",
|
||||
"BLOG_DIR": "blog",
|
||||
}
|
||||
|
||||
|
||||
ALLOWED_VISIBILITY = frozenset({"public", "unlisted"})
|
||||
|
||||
|
||||
class ConfigError(ValueError):
|
||||
"""Raised when required configuration is missing or invalid."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
mastodon_base_url: str
|
||||
mastodon_access_token: str
|
||||
mastodon_visibility: str
|
||||
site_url: str
|
||||
hashtags: str
|
||||
throwback_prefix: str
|
||||
max_retries: int
|
||||
run_at: str
|
||||
tz: str
|
||||
data_dir: Path = field(default_factory=lambda: Path("/app/data"))
|
||||
blog_repo_url: str = "https://git.chaospott.de/Chaospott/site"
|
||||
blog_dir: Path = field(default_factory=lambda: Path("/app/data/blog"))
|
||||
extra: dict = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def cron_minute(self) -> str:
|
||||
return self.run_at.split(":", 1)[0]
|
||||
|
||||
@property
|
||||
def cron_hour(self) -> str:
|
||||
return self.run_at.split(":", 1)[1]
|
||||
|
||||
|
||||
DEFAULT_DOTENV_PATH = Path("/app/.env")
|
||||
|
||||
|
||||
def _read_dotenv(dotenv_path: Optional[Path]) -> dict[str, str]:
|
||||
if dotenv_path is None:
|
||||
return {}
|
||||
if not dotenv_path.exists():
|
||||
return {}
|
||||
values = dotenv_values(dotenv_path=str(dotenv_path))
|
||||
return {k: v for k, v in values.items() if v is not None}
|
||||
|
||||
|
||||
def _values_from_env() -> dict[str, str]:
|
||||
values: dict[str, str] = {}
|
||||
for key in REQUIRED_KEYS + OPTIONAL_KEYS:
|
||||
raw = os.environ.get(key)
|
||||
if raw is not None and raw != "":
|
||||
values[key] = raw
|
||||
return values
|
||||
|
||||
|
||||
def load_config(dotenv_path: Optional[Path] = None) -> Config:
|
||||
"""Load configuration from a dotenv file and the process environment.
|
||||
|
||||
The .env file is loaded via python-dotenv at startup so the bind-mounted
|
||||
runtime secrets are visible to the process. Environment variables take
|
||||
precedence over the dotenv file so a mounted `.env` can be supplemented
|
||||
by Compose-level overrides.
|
||||
|
||||
Required keys are validated against the **raw** merged map (dotenv +
|
||||
process env) before any defaults are applied. Defaults are only used
|
||||
for the retry budget and the optional Jekyll blog source so a missing
|
||||
Mastodon/secret/URL/etc. fails fast with a clear error.
|
||||
"""
|
||||
if dotenv_path is None:
|
||||
dotenv_path = DEFAULT_DOTENV_PATH if DEFAULT_DOTENV_PATH.exists() else None
|
||||
|
||||
if dotenv_path is not None:
|
||||
load_dotenv(dotenv_path=str(dotenv_path), override=False)
|
||||
|
||||
file_values = _read_dotenv(dotenv_path)
|
||||
env_values = _values_from_env()
|
||||
|
||||
raw: dict[str, str] = {}
|
||||
raw.update(file_values)
|
||||
raw.update(env_values)
|
||||
|
||||
# MAX_RETRIES is the only required key that has a permitted default
|
||||
# (5 attempts per the Job). Apply it before the required-key check so a
|
||||
# missing value uses the documented default rather than failing fast.
|
||||
if not raw.get("MAX_RETRIES"):
|
||||
raw["MAX_RETRIES"] = DEFAULTS["MAX_RETRIES"]
|
||||
|
||||
validate_required(raw)
|
||||
|
||||
merged = dict(raw)
|
||||
apply_defaults(merged)
|
||||
|
||||
validate_config(merged)
|
||||
|
||||
data_dir = Path(os.environ.get("DATA_DIR", "/app/data")).resolve()
|
||||
|
||||
blog_dir_raw = Path(merged["BLOG_DIR"]).expanduser()
|
||||
blog_dir = (
|
||||
blog_dir_raw
|
||||
if blog_dir_raw.is_absolute()
|
||||
else (data_dir / blog_dir_raw).resolve()
|
||||
)
|
||||
|
||||
extra = {k: v for k, v in merged.items() if k not in REQUIRED_KEYS + OPTIONAL_KEYS}
|
||||
|
||||
return Config(
|
||||
mastodon_base_url=merged["MASTODON_BASE_URL"],
|
||||
mastodon_access_token=merged["MASTODON_ACCESS_TOKEN"],
|
||||
mastodon_visibility=merged["MASTODON_VISIBILITY"],
|
||||
site_url=merged["SITE_URL"],
|
||||
hashtags=merged["HASHTAGS"],
|
||||
throwback_prefix=merged["THROWBACK_PREFIX"],
|
||||
max_retries=_parse_max_retries(merged["MAX_RETRIES"]),
|
||||
run_at=merged["RUN_AT"],
|
||||
tz=merged["TZ"],
|
||||
data_dir=data_dir,
|
||||
blog_repo_url=merged["BLOG_REPO_URL"],
|
||||
blog_dir=blog_dir,
|
||||
extra=extra,
|
||||
)
|
||||
|
||||
|
||||
def apply_defaults(values: dict[str, str]) -> None:
|
||||
for key, default in DEFAULTS.items():
|
||||
values.setdefault(key, default)
|
||||
|
||||
|
||||
def validate_required(values: dict[str, str]) -> None:
|
||||
"""Validate that every required key is present and non-empty in ``values``.
|
||||
|
||||
Called against the raw (no-default) merged map so a missing key always
|
||||
surfaces as a startup error rather than being silently substituted.
|
||||
"""
|
||||
errors: list[str] = []
|
||||
for key in REQUIRED_KEYS:
|
||||
raw = values.get(key, "")
|
||||
if not raw:
|
||||
errors.append(f"missing required configuration key: {key}")
|
||||
if errors:
|
||||
raise ConfigError("; ".join(errors))
|
||||
|
||||
|
||||
def validate_config(values: dict[str, str]) -> None:
|
||||
errors: list[str] = []
|
||||
|
||||
run_at = values.get("RUN_AT", "")
|
||||
if run_at and not _is_valid_hhmm(run_at):
|
||||
errors.append(f"RUN_AT={run_at!r} must be in HH:MM (24-hour) format")
|
||||
|
||||
base_url = values.get("MASTODON_BASE_URL", "")
|
||||
if base_url and not _is_valid_url(base_url):
|
||||
errors.append(f"MASTODON_BASE_URL={base_url!r} must be a valid http(s) URL")
|
||||
|
||||
site_url = values.get("SITE_URL", "")
|
||||
if site_url and not _is_valid_url(site_url):
|
||||
errors.append(f"SITE_URL={site_url!r} must be a valid http(s) URL")
|
||||
|
||||
visibility = values.get("MASTODON_VISIBILITY", "")
|
||||
if visibility and not _is_valid_visibility(visibility):
|
||||
errors.append(
|
||||
f"MASTODON_VISIBILITY={visibility!r} must be one of: {sorted(ALLOWED_VISIBILITY)}"
|
||||
)
|
||||
|
||||
blog_repo_url = values.get("BLOG_REPO_URL", "")
|
||||
if blog_repo_url and not _is_valid_url(blog_repo_url):
|
||||
errors.append(
|
||||
f"BLOG_REPO_URL={blog_repo_url!r} must be a valid http(s) URL"
|
||||
)
|
||||
|
||||
tz = values.get("TZ", "")
|
||||
if tz and not _is_valid_tz(tz):
|
||||
errors.append(f"TZ={tz!r} must be a valid IANA timezone")
|
||||
|
||||
max_retries = values.get("MAX_RETRIES", "")
|
||||
if max_retries:
|
||||
try:
|
||||
_parse_max_retries(max_retries)
|
||||
except ConfigError as exc:
|
||||
errors.append(str(exc))
|
||||
|
||||
if errors:
|
||||
raise ConfigError("; ".join(errors))
|
||||
|
||||
|
||||
def _is_valid_hhmm(value) -> bool:
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
parts = value.split(":")
|
||||
if len(parts) != 2:
|
||||
return False
|
||||
hour, minute = parts
|
||||
if len(hour) != 2 or len(minute) != 2:
|
||||
return False
|
||||
if not hour.isdigit() or not minute.isdigit():
|
||||
return False
|
||||
h = int(hour)
|
||||
m = int(minute)
|
||||
return 0 <= h <= 23 and 0 <= m <= 59
|
||||
|
||||
|
||||
def _is_valid_url(value: str) -> bool:
|
||||
if not isinstance(value, str) or not value:
|
||||
return False
|
||||
try:
|
||||
parsed = urlparse(value)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return parsed.scheme in {"http", "https"} and bool(parsed.netloc)
|
||||
|
||||
|
||||
def _is_valid_visibility(value: str) -> bool:
|
||||
return isinstance(value, str) and value in ALLOWED_VISIBILITY
|
||||
|
||||
|
||||
def _is_valid_tz(value: str) -> bool:
|
||||
if not isinstance(value, str) or not value:
|
||||
return False
|
||||
try:
|
||||
ZoneInfo(value)
|
||||
except ZoneInfoNotFoundError:
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _parse_max_retries(value) -> int:
|
||||
if isinstance(value, bool):
|
||||
raise ConfigError(f"MAX_RETRIES={value!r} must be a non-negative integer")
|
||||
try:
|
||||
count = int(value)
|
||||
except (TypeError, ValueError):
|
||||
raise ConfigError(f"MAX_RETRIES={value!r} must be a non-negative integer")
|
||||
if count < 0:
|
||||
raise ConfigError(f"MAX_RETRIES={value!r} must be >= 0")
|
||||
return count
|
||||
@@ -0,0 +1,175 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from .config import ConfigError
|
||||
|
||||
_SECRET_KEYS = frozenset(
|
||||
{
|
||||
"token",
|
||||
"access_token",
|
||||
"mastodon_access_token",
|
||||
"password",
|
||||
"secret",
|
||||
"authorization",
|
||||
"api_key",
|
||||
}
|
||||
)
|
||||
|
||||
_LOGGER_NAME = "tenbackward"
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""Emit one JSON object per log record.
|
||||
|
||||
Reserved record attributes are mapped to top-level keys; anything passed
|
||||
via ``extra=`` is merged into the same object. Keys whose name appears in
|
||||
``_SECRET_KEYS`` are redacted before formatting.
|
||||
"""
|
||||
|
||||
_RESERVED = {
|
||||
"name",
|
||||
"msg",
|
||||
"args",
|
||||
"levelname",
|
||||
"levelno",
|
||||
"pathname",
|
||||
"filename",
|
||||
"module",
|
||||
"exc_info",
|
||||
"exc_text",
|
||||
"stack_info",
|
||||
"lineno",
|
||||
"funcName",
|
||||
"created",
|
||||
"msecs",
|
||||
"relativeCreated",
|
||||
"thread",
|
||||
"threadName",
|
||||
"processName",
|
||||
"process",
|
||||
"message",
|
||||
"asctime",
|
||||
}
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str: # noqa: A003
|
||||
message = record.getMessage()
|
||||
payload: dict[str, Any] = {
|
||||
"ts": datetime.fromtimestamp(record.created, tz=timezone.utc).isoformat(),
|
||||
"level": record.levelname,
|
||||
"logger": record.name,
|
||||
"message": message,
|
||||
}
|
||||
|
||||
for key, value in record.__dict__.items():
|
||||
if key in self._RESERVED or key.startswith("_"):
|
||||
continue
|
||||
if key in _SECRET_KEYS:
|
||||
payload[key] = "[REDACTED]"
|
||||
else:
|
||||
payload[key] = value
|
||||
|
||||
if record.exc_info:
|
||||
payload["exc_type"] = record.exc_info[0].__name__ if record.exc_info[0] else None
|
||||
|
||||
return json.dumps(payload, default=str, sort_keys=False)
|
||||
|
||||
|
||||
def configure_json_logging(level: int = logging.INFO) -> None:
|
||||
"""Install the JSON formatter on the root logger.
|
||||
|
||||
Idempotent: safe to call multiple times (replaces any existing handler).
|
||||
"""
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(JsonFormatter())
|
||||
handler.setLevel(level)
|
||||
|
||||
root = logging.getLogger()
|
||||
root.setLevel(level)
|
||||
for existing in list(root.handlers):
|
||||
root.removeHandler(existing)
|
||||
root.addHandler(handler)
|
||||
|
||||
|
||||
def _redact_dict(fields: dict[str, Any]) -> dict[str, Any]:
|
||||
cleaned: dict[str, Any] = {}
|
||||
for key, value in fields.items():
|
||||
if key.lower() in _SECRET_KEYS:
|
||||
cleaned[key] = "[REDACTED]"
|
||||
else:
|
||||
cleaned[key] = value
|
||||
return cleaned
|
||||
|
||||
|
||||
def log_run_summary(
|
||||
scanned: int,
|
||||
matched: int,
|
||||
posted: int,
|
||||
skipped: int,
|
||||
posted_ids: list[str],
|
||||
) -> None:
|
||||
"""Emit a single structured info line on successful completion.
|
||||
|
||||
A run with no matching posts is silent — no log line is emitted.
|
||||
"""
|
||||
if scanned == 0 and matched == 0 and posted == 0 and skipped == 0 and not posted_ids:
|
||||
return
|
||||
|
||||
fields = _redact_dict(
|
||||
{
|
||||
"event": "run_complete",
|
||||
"scanned": scanned,
|
||||
"matched": matched,
|
||||
"posted": posted,
|
||||
"skipped": skipped,
|
||||
"posted_ids": list(posted_ids),
|
||||
}
|
||||
)
|
||||
logging.getLogger(_LOGGER_NAME).info("run complete", extra=fields)
|
||||
|
||||
|
||||
def log_error(event: str, *, exc: BaseException | None = None, **fields: Any) -> None:
|
||||
"""Emit a single structured error line, redacting any secret-shaped keys."""
|
||||
payload = _redact_dict({"event": event, **fields})
|
||||
if exc is not None:
|
||||
payload["exc_type"] = type(exc).__name__
|
||||
logging.getLogger(_LOGGER_NAME).error(event, extra=payload)
|
||||
|
||||
|
||||
def log_event(event: str, **fields: Any) -> None:
|
||||
"""Emit a structured event line at the level appropriate to ``event``.
|
||||
|
||||
Unlike :func:`log_error`, this helper does not attach ``exc_type``; it
|
||||
forwards arbitrary fields (including ``error=str(exc)``) through the
|
||||
same redaction path so callers can include the underlying exception
|
||||
text without leaking secret-shaped keys.
|
||||
"""
|
||||
payload = _redact_dict({"event": event, **fields})
|
||||
logging.getLogger(_LOGGER_NAME).error(event, extra=payload)
|
||||
|
||||
|
||||
def log_warning(event: str, **fields: Any) -> None:
|
||||
"""Emit a single structured warning line, redacting any secret-shaped keys."""
|
||||
payload = _redact_dict({"event": event, **fields})
|
||||
logging.getLogger(_LOGGER_NAME).warning(event, extra=payload)
|
||||
|
||||
|
||||
def log_startup(**fields: Any) -> None:
|
||||
"""Emit a single structured info line at startup."""
|
||||
payload = _redact_dict({"event": "startup", **fields})
|
||||
logging.getLogger(_LOGGER_NAME).info("startup", extra=payload)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ConfigError",
|
||||
"JsonFormatter",
|
||||
"configure_json_logging",
|
||||
"log_error",
|
||||
"log_event",
|
||||
"log_run_summary",
|
||||
"log_startup",
|
||||
"log_warning",
|
||||
]
|
||||
@@ -0,0 +1,181 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
from typing import Callable, Iterable
|
||||
|
||||
from . import __version__
|
||||
from .blog import BlogRepoError, BlogTransientError, ensure_repo
|
||||
from .config import Config, ConfigError, load_config
|
||||
from .logging_setup import (
|
||||
configure_json_logging,
|
||||
log_error,
|
||||
log_event,
|
||||
log_run_summary,
|
||||
log_startup,
|
||||
)
|
||||
from .matching import MatchedPost, find_anniversary_matches
|
||||
from .publishing import PublishError, PublishFatalError, publish_mastodon
|
||||
from .state import PostedStore
|
||||
|
||||
|
||||
_logger = logging.getLogger("tenbackward")
|
||||
|
||||
|
||||
def _iter_candidates(config: Config) -> Iterable[MatchedPost]:
|
||||
"""Yield candidate :class:`MatchedPost` objects whose anniversary is
|
||||
exactly 10 years before today.
|
||||
"""
|
||||
post_root = config.blog_dir / "_posts" / "blog"
|
||||
return find_anniversary_matches(post_root, config.site_url)
|
||||
|
||||
|
||||
def _log_retry_attempt(operation: str, attempt: int, exc: BaseException) -> None:
|
||||
log_event(
|
||||
"retry_attempt",
|
||||
operation=operation,
|
||||
attempt=attempt,
|
||||
error=str(exc),
|
||||
exc_type=type(exc).__name__,
|
||||
)
|
||||
|
||||
|
||||
def _log_final_failure(operation: str, exc: BaseException, attempts: int) -> None:
|
||||
log_error(
|
||||
"retry_exhausted",
|
||||
exc=exc,
|
||||
operation=operation,
|
||||
attempts=attempts,
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
|
||||
def _operation_label(exc: BaseException) -> str:
|
||||
if isinstance(exc, BlogTransientError):
|
||||
return exc.operation
|
||||
if isinstance(exc, PublishError):
|
||||
return "mastodon post"
|
||||
if isinstance(exc, BlogRepoError):
|
||||
return "git pull"
|
||||
return "pipeline"
|
||||
|
||||
|
||||
def _run_with_retry(
|
||||
config: Config,
|
||||
*,
|
||||
sleep: Callable[[float], None] = time.sleep,
|
||||
logger: logging.Logger | None = None,
|
||||
) -> tuple[int, int, int, int, list[str]] | None:
|
||||
"""Execute the pipeline with retries. Returns the summary counters on
|
||||
success or ``None`` when the retry budget is exhausted.
|
||||
|
||||
A single retry budget governs both the Git and Mastodon operations —
|
||||
``ensure_repo`` is invoked with ``max_retries=0`` so the runner, not the
|
||||
blog layer, decides whether to re-attempt. Configuration and fatal
|
||||
validation errors (``ConfigError``, :class:`BlogRepoError`,
|
||||
:class:`PublishFatalError`) are not retried.
|
||||
"""
|
||||
attempts = max(1, config.max_retries)
|
||||
active_logger = logger or _logger
|
||||
last_exc: BaseException | None = None
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
return _run_once(config, logger=active_logger, sleep=sleep)
|
||||
except (ConfigError, BlogRepoError, PublishFatalError) as exc:
|
||||
last_exc = exc
|
||||
_log_final_failure(type(exc).__name__, exc, attempt)
|
||||
return None
|
||||
except (BlogTransientError, PublishError) as exc:
|
||||
last_exc = exc
|
||||
_log_retry_attempt(_operation_label(exc), attempt, exc)
|
||||
if attempt < attempts:
|
||||
sleep(2 ** (attempt - 1))
|
||||
except Exception as exc: # noqa: BLE001 — opportunistic retry boundary
|
||||
last_exc = exc
|
||||
_log_retry_attempt("pipeline", attempt, exc)
|
||||
if attempt < attempts:
|
||||
sleep(2 ** (attempt - 1))
|
||||
assert last_exc is not None
|
||||
_log_final_failure(_operation_label(last_exc), last_exc, attempts)
|
||||
return None
|
||||
|
||||
|
||||
def _run_once(
|
||||
config: Config,
|
||||
*,
|
||||
logger: logging.Logger | None = None,
|
||||
sleep: Callable[[float], None] = time.sleep,
|
||||
) -> tuple[int, int, int, int, list[str]]:
|
||||
"""Execute one pipeline pass and return the summary counters.
|
||||
|
||||
Returns ``(scanned, matched, posted, skipped, posted_ids)``.
|
||||
|
||||
The blog layer's retry budget is disabled here (``max_retries=0``);
|
||||
the runner owns the unified retry policy so we never nest two retry
|
||||
budgets back-to-back.
|
||||
"""
|
||||
ensure_repo(
|
||||
config.data_dir,
|
||||
repo_url=config.blog_repo_url,
|
||||
blog_path=config.blog_dir,
|
||||
max_retries=0,
|
||||
logger=logger,
|
||||
sleep=sleep,
|
||||
)
|
||||
|
||||
store = PostedStore(config.data_dir)
|
||||
|
||||
candidates = list(_iter_candidates(config))
|
||||
scanned = len(candidates)
|
||||
matched = len(candidates)
|
||||
|
||||
unposted: list[MatchedPost] = []
|
||||
for match in candidates:
|
||||
if store.is_posted(match.path):
|
||||
continue
|
||||
unposted.append(match)
|
||||
|
||||
skipped = matched - len(unposted)
|
||||
posted_ids = [m.path for m in unposted]
|
||||
|
||||
if unposted:
|
||||
publish_mastodon(config, unposted)
|
||||
store.mark_posted_many(posted_ids)
|
||||
|
||||
posted = len(unposted)
|
||||
return scanned, matched, posted, skipped, posted_ids
|
||||
|
||||
|
||||
def main() -> int:
|
||||
configure_json_logging()
|
||||
|
||||
try:
|
||||
config = load_config()
|
||||
except ConfigError as exc:
|
||||
log_error("configuration_error", exc=exc)
|
||||
return 2
|
||||
|
||||
config.data_dir.mkdir(parents=True, exist_ok=True)
|
||||
config.blog_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
log_startup(
|
||||
version=__version__,
|
||||
site_url=config.site_url,
|
||||
run_at=config.run_at,
|
||||
tz=config.tz,
|
||||
hashtags=config.hashtags,
|
||||
throwback_prefix=config.throwback_prefix,
|
||||
)
|
||||
|
||||
result = _run_with_retry(config, logger=_logger)
|
||||
if result is None:
|
||||
return 1
|
||||
|
||||
scanned, matched, posted, skipped, posted_ids = result
|
||||
log_run_summary(scanned, matched, posted, skipped, posted_ids)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,230 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import calendar
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import frontmatter
|
||||
|
||||
|
||||
_LOGGER = logging.getLogger("tenbackward.matching")
|
||||
|
||||
_FILENAME_PATTERN = re.compile(r"^(\d{4})-(\d{2})-(\d{2})-(.+)$")
|
||||
|
||||
_NON_ALNUM_RE = re.compile(r"[^a-z0-9]+")
|
||||
|
||||
_DEFAULT_TZ = "Europe/Berlin"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MatchedPost:
|
||||
path: str
|
||||
title: str
|
||||
date: date
|
||||
url: str
|
||||
|
||||
|
||||
def slugify_title(title: str) -> str:
|
||||
"""Return a URL-safe slug derived from ``title``.
|
||||
|
||||
Used by the matcher to derive the slug component of
|
||||
:attr:`MatchedPost.url` from the resolved post title.
|
||||
"""
|
||||
normalized = unicodedata.normalize("NFKD", title)
|
||||
ascii_only = normalized.encode("ascii", "ignore").decode("ascii")
|
||||
lowered = ascii_only.lower()
|
||||
dashed = _NON_ALNUM_RE.sub("-", lowered)
|
||||
return dashed.strip("-")
|
||||
|
||||
|
||||
def _today_in_berlin(tz_name: str = _DEFAULT_TZ) -> date:
|
||||
return datetime.now(ZoneInfo(tz_name)).date()
|
||||
|
||||
|
||||
def _is_leap_year(year: int) -> bool:
|
||||
return calendar.isleap(year)
|
||||
|
||||
|
||||
def _try_parse_date(value: object) -> date | None:
|
||||
if isinstance(value, datetime):
|
||||
return value.date()
|
||||
if isinstance(value, date):
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _safe_filename_match(stem: str) -> tuple[int, int, int, str] | None:
|
||||
match = _FILENAME_PATTERN.match(stem)
|
||||
if match is None:
|
||||
return None
|
||||
year_str, month_str, day_str, slug = match.groups()
|
||||
try:
|
||||
year = int(year_str)
|
||||
month = int(month_str)
|
||||
day = int(day_str)
|
||||
except ValueError:
|
||||
return None
|
||||
try:
|
||||
date(year, month, day)
|
||||
except ValueError:
|
||||
return None
|
||||
if not slug:
|
||||
return None
|
||||
return year, month, day, slug
|
||||
|
||||
|
||||
def _build_url(site_url: str, post_date: date, slug: str) -> str:
|
||||
base = site_url.rstrip("/")
|
||||
return f"{base}/{post_date.year:04d}/{post_date.month:02d}/{post_date.day:02d}/{slug}/"
|
||||
|
||||
|
||||
def _is_unpublished(metadata: dict) -> bool:
|
||||
if "published" not in metadata:
|
||||
return False
|
||||
return metadata["published"] is False
|
||||
|
||||
|
||||
def _matches_anniversary(post_date: date, target_year: int, today: date) -> bool:
|
||||
if post_date.month == 2 and post_date.day == 29:
|
||||
if _is_leap_year(today.year):
|
||||
return today.month == 2 and today.day == 29
|
||||
return (
|
||||
post_date.year == target_year
|
||||
and today.month == 3
|
||||
and today.day == 1
|
||||
)
|
||||
if post_date.year != target_year:
|
||||
return False
|
||||
return post_date.month == today.month and post_date.day == today.day
|
||||
|
||||
|
||||
def _process_file(
|
||||
md_path: Path,
|
||||
post_root: Path,
|
||||
site_url: str,
|
||||
target_year: int,
|
||||
today: date,
|
||||
) -> MatchedPost | None:
|
||||
rel = md_path.relative_to(post_root).as_posix()
|
||||
|
||||
try:
|
||||
with md_path.open("r", encoding="utf-8") as fh:
|
||||
text = fh.read()
|
||||
except (OSError, UnicodeDecodeError) as exc:
|
||||
_LOGGER.warning(
|
||||
"skipping unreadable post file",
|
||||
extra={"event": "post_unreadable", "path": rel, "error": type(exc).__name__},
|
||||
)
|
||||
return None
|
||||
|
||||
title: str | None = None
|
||||
post_date: date | None = None
|
||||
unpublished = False
|
||||
frontmatter_parsed = False
|
||||
|
||||
try:
|
||||
parsed = frontmatter.loads(text)
|
||||
frontmatter_parsed = True
|
||||
metadata = parsed.metadata if isinstance(parsed.metadata, dict) else {}
|
||||
if isinstance(metadata.get("title"), str) and metadata["title"].strip():
|
||||
title = metadata["title"]
|
||||
post_date = _try_parse_date(metadata.get("date"))
|
||||
unpublished = _is_unpublished(metadata)
|
||||
except Exception as exc: # noqa: BLE001 — frontmatter is opaque
|
||||
_LOGGER.warning(
|
||||
"frontmatter parse failed",
|
||||
extra={"event": "post_frontmatter_error", "path": rel, "error": type(exc).__name__},
|
||||
)
|
||||
|
||||
slug_match = _safe_filename_match(md_path.stem)
|
||||
if slug_match is None:
|
||||
if not frontmatter_parsed:
|
||||
return None
|
||||
_LOGGER.warning(
|
||||
"filename missing required YYYY-MM-DD-slug pattern",
|
||||
extra={"event": "post_filename_invalid", "path": rel},
|
||||
)
|
||||
return None
|
||||
|
||||
fn_year, fn_month, fn_day, fn_slug = slug_match
|
||||
if post_date is None:
|
||||
post_date = date(fn_year, fn_month, fn_day)
|
||||
|
||||
if not _matches_anniversary(post_date, target_year, today):
|
||||
return None
|
||||
|
||||
if unpublished:
|
||||
_LOGGER.warning(
|
||||
"skipping unpublished post",
|
||||
extra={"event": "post_unpublished", "path": rel},
|
||||
)
|
||||
return None
|
||||
|
||||
if title is None:
|
||||
title = fn_slug
|
||||
|
||||
return MatchedPost(
|
||||
path=rel,
|
||||
title=title,
|
||||
date=post_date,
|
||||
url=_build_url(site_url, post_date, slugify_title(title)),
|
||||
)
|
||||
|
||||
|
||||
def find_anniversary_matches(
|
||||
post_root: Path,
|
||||
site_url: str,
|
||||
*,
|
||||
today: date | None = None,
|
||||
) -> list[MatchedPost]:
|
||||
"""Walk the Jekyll post directory and return posts whose anniversary
|
||||
date is exactly 10 years before ``today`` (per the leap-day rules).
|
||||
"""
|
||||
if today is None:
|
||||
today = _today_in_berlin()
|
||||
|
||||
target_year = today.year - 10
|
||||
|
||||
if not post_root.exists():
|
||||
return []
|
||||
|
||||
matches: list[MatchedPost] = []
|
||||
for md_path in sorted(post_root.rglob("*.md")):
|
||||
if not md_path.is_file():
|
||||
continue
|
||||
try:
|
||||
result = _process_file(md_path, post_root, site_url, target_year, today)
|
||||
except Exception as exc: # noqa: BLE001 — never abort the scan
|
||||
rel = md_path.relative_to(post_root).as_posix()
|
||||
_LOGGER.warning(
|
||||
"unexpected error processing post",
|
||||
extra={
|
||||
"event": "post_processing_error",
|
||||
"path": rel,
|
||||
"error": type(exc).__name__,
|
||||
},
|
||||
)
|
||||
continue
|
||||
if result is not None:
|
||||
matches.append(result)
|
||||
|
||||
return matches
|
||||
|
||||
|
||||
def iter_anniversary_paths(post_root: Path, site_url: str, *, today: date | None = None):
|
||||
"""Yield each matched post's relative ``path`` (the dedupe identifier)."""
|
||||
for match in find_anniversary_matches(post_root, site_url, today=today):
|
||||
yield match.path
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MatchedPost",
|
||||
"find_anniversary_matches",
|
||||
"iter_anniversary_paths",
|
||||
"slugify_title",
|
||||
]
|
||||
@@ -0,0 +1,164 @@
|
||||
"""Mastodon publishing helpers and the API success boundary.
|
||||
|
||||
The :func:`publish_mastodon` function is the single boundary between the
|
||||
pipeline orchestrator and the Mastodon HTTP API. Status composition and
|
||||
length validation are pure and isolated from network access so they can
|
||||
be tested without a server.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Callable
|
||||
|
||||
from mastodon import Mastodon
|
||||
|
||||
from .config import Config
|
||||
from .matching import MatchedPost, slugify_title
|
||||
|
||||
|
||||
MASTODON_STATUS_LIMIT = 500
|
||||
|
||||
|
||||
class PublishError(RuntimeError):
|
||||
"""Raised when status composition, validation, or the Mastodon API
|
||||
call fails. The pipeline orchestrator treats this like any other
|
||||
pipeline error and lets the retry budget decide whether to give up.
|
||||
"""
|
||||
|
||||
|
||||
class PublishFatalError(PublishError):
|
||||
"""Raised for fatal configuration or validation failures (missing token,
|
||||
invalid instance URL, malformed status). The pipeline orchestrator must
|
||||
not retry these — they will fail the same way on every attempt.
|
||||
"""
|
||||
|
||||
|
||||
def _normalize_hashtags(hashtags: str) -> str:
|
||||
parts = [token.strip() for token in (hashtags or "").split(",")]
|
||||
parts = [token for token in parts if token]
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
def build_status_text(
|
||||
prefix: str,
|
||||
posts: list[MatchedPost],
|
||||
hashtags: str,
|
||||
) -> str:
|
||||
"""Compose a single Mastodon status string for ``posts``.
|
||||
|
||||
The format is::
|
||||
|
||||
{prefix}
|
||||
|
||||
{title1}
|
||||
{url1}
|
||||
|
||||
{title2}
|
||||
{url2}
|
||||
|
||||
...
|
||||
|
||||
{hashtags}
|
||||
|
||||
Posts are sorted by ``(date, path)`` for deterministic output
|
||||
regardless of the upstream ordering.
|
||||
"""
|
||||
if not posts:
|
||||
raise PublishFatalError("empty_posts: cannot compose status without posts")
|
||||
|
||||
ordered = sorted(posts, key=lambda m: (m.date, m.path))
|
||||
blocks: list[str] = []
|
||||
for match in ordered:
|
||||
blocks.append(f"{match.title}\n{match.url}")
|
||||
body = "\n\n".join(blocks)
|
||||
|
||||
lines: list[str] = [prefix.strip(), body]
|
||||
tag_line = _normalize_hashtags(hashtags)
|
||||
if tag_line:
|
||||
lines.append(tag_line)
|
||||
return "\n\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def validate_status(status: str, limit: int = MASTODON_STATUS_LIMIT) -> None:
|
||||
"""Raise :class:`PublishFatalError` when ``status`` exceeds ``limit``.
|
||||
|
||||
Never truncates: the spec requires failing safely rather than
|
||||
shortening content.
|
||||
"""
|
||||
if len(status) > limit:
|
||||
raise PublishFatalError(
|
||||
f"status_too_long: len={len(status)} limit={limit}"
|
||||
)
|
||||
|
||||
|
||||
def _validate_publish_config(config: Config) -> None:
|
||||
if not config.mastodon_base_url or not config.mastodon_base_url.startswith(
|
||||
("http://", "https://")
|
||||
):
|
||||
raise PublishFatalError(
|
||||
f"publish_failed: mastodon auth: invalid MASTODON_BASE_URL={config.mastodon_base_url!r}"
|
||||
)
|
||||
if not config.mastodon_access_token:
|
||||
raise PublishFatalError(
|
||||
"publish_failed: mastodon auth: MASTODON_ACCESS_TOKEN is empty"
|
||||
)
|
||||
|
||||
|
||||
def _post_status_via_mastodon_py(
|
||||
status: str,
|
||||
*,
|
||||
base_url: str,
|
||||
access_token: str,
|
||||
visibility: str,
|
||||
client_factory: Callable[..., Mastodon] | None = None,
|
||||
) -> None:
|
||||
factory = client_factory if client_factory is not None else Mastodon
|
||||
client = factory(access_token=access_token, api_base_url=base_url)
|
||||
client.status_post(status, visibility=visibility)
|
||||
|
||||
|
||||
def publish_mastodon(
|
||||
config: Config,
|
||||
posts: list[MatchedPost],
|
||||
*,
|
||||
client_factory: Callable[..., Mastodon] | None = None,
|
||||
) -> str:
|
||||
"""Compose, validate, and publish ``posts`` to Mastodon.
|
||||
|
||||
Returns the composed status text on success. Raises
|
||||
:class:`PublishFatalError` for configuration/validation failures (no
|
||||
retry), or :class:`PublishError` (transient) for network/HTTP errors
|
||||
that the runner may retry.
|
||||
"""
|
||||
_validate_publish_config(config)
|
||||
|
||||
try:
|
||||
status = build_status_text(
|
||||
config.throwback_prefix, posts, config.hashtags
|
||||
)
|
||||
validate_status(status)
|
||||
_post_status_via_mastodon_py(
|
||||
status,
|
||||
base_url=config.mastodon_base_url,
|
||||
access_token=config.mastodon_access_token,
|
||||
visibility=config.mastodon_visibility,
|
||||
client_factory=client_factory,
|
||||
)
|
||||
except PublishFatalError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 — third-party boundary
|
||||
raise PublishError(
|
||||
f"publish_failed: mastodon post: {type(exc).__name__}: {exc}"
|
||||
) from exc
|
||||
return status
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MASTODON_STATUS_LIMIT",
|
||||
"PublishError",
|
||||
"PublishFatalError",
|
||||
"build_status_text",
|
||||
"publish_mastodon",
|
||||
"slugify_title",
|
||||
"validate_status",
|
||||
]
|
||||
@@ -0,0 +1,208 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from .logging_setup import log_warning
|
||||
|
||||
|
||||
DEFAULT_DATA_DIR = Path("./data")
|
||||
_FILE_NAME = "posted.json"
|
||||
_EMPTY_DOCUMENT: dict[str, list[str]] = {"posted": []}
|
||||
|
||||
|
||||
class PostedStoreError(RuntimeError):
|
||||
"""Raised when the posted-state store cannot be used."""
|
||||
|
||||
|
||||
def posted_path(data_dir: Path | None = None) -> Path:
|
||||
"""Resolve the path of ``posted.json``.
|
||||
|
||||
When ``data_dir`` is ``None``, the directory is read from the
|
||||
``DATA_DIR`` environment variable, falling back to ``./data``
|
||||
(the host bind-mount described in the README).
|
||||
"""
|
||||
if data_dir is None:
|
||||
data_dir = Path(os.environ.get("DATA_DIR", str(DEFAULT_DATA_DIR)))
|
||||
return Path(data_dir) / _FILE_NAME
|
||||
|
||||
|
||||
def _coerce_list(value: object) -> list[str]:
|
||||
"""Filter ``value`` down to a list of strings, dropping anything else."""
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [item for item in value if isinstance(item, str)]
|
||||
|
||||
|
||||
def _read_existing(path: Path) -> dict[str, list[str]] | None:
|
||||
"""Return the parsed JSON document at ``path`` or ``None`` on errors.
|
||||
|
||||
A missing file is **not** an error: returns ``None`` so the caller
|
||||
can treat it as the empty document. Malformed JSON or unreadable
|
||||
bytes return ``None`` and a warning is emitted.
|
||||
"""
|
||||
try:
|
||||
with path.open("r", encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except (json.JSONDecodeError, OSError) as exc:
|
||||
log_warning("posted_store_corrupt", path=str(path), error=type(exc).__name__)
|
||||
return None
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
return data
|
||||
|
||||
|
||||
def _ensure_parent(path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def _write_document(path: Path, document: dict[str, list[str]]) -> None:
|
||||
"""Atomically write ``document`` to ``path`` via temp-file + rename."""
|
||||
_ensure_parent(path)
|
||||
pid = os.getpid()
|
||||
tmp = path.with_name(f"{path.name}.tmp.{pid}")
|
||||
try:
|
||||
with tmp.open("w", encoding="utf-8") as fh:
|
||||
json.dump(document, fh, sort_keys=False, indent=2)
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno())
|
||||
os.replace(tmp, path)
|
||||
except Exception:
|
||||
try:
|
||||
tmp.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def _lock_path(path: Path):
|
||||
"""Return an open file descriptor on the lock file inside the data dir.
|
||||
|
||||
The lock file lives in the same directory as ``posted.json`` so the
|
||||
flock is always on a single inode, regardless of whether the JSON
|
||||
file currently exists. The descriptor is opened in append mode so
|
||||
concurrent readers never truncate it.
|
||||
"""
|
||||
_ensure_parent(path)
|
||||
lock = path.with_name(f".{path.name}.lock")
|
||||
fd = os.open(str(lock), os.O_CREAT | os.O_RDWR, 0o644)
|
||||
return lock, fd
|
||||
|
||||
|
||||
class PostedStore:
|
||||
"""Self-contained dedup store persisted as ``posted.json``.
|
||||
|
||||
The on-disk shape is::
|
||||
|
||||
{"posted": ["2014/2014-08-04-foo.md", ...]}
|
||||
|
||||
All mutating operations acquire an exclusive :mod:`fcntl` flock on a
|
||||
sibling lock file so concurrent container runs cannot corrupt the
|
||||
JSON document. The store never raises on a missing file, an
|
||||
unreadable file, or an already-present path — it always reports
|
||||
success in a way that the orchestrator can act on.
|
||||
"""
|
||||
|
||||
def __init__(self, data_dir: Path | None = None) -> None:
|
||||
self._path = posted_path(data_dir)
|
||||
|
||||
@property
|
||||
def path(self) -> Path:
|
||||
return self._path
|
||||
|
||||
def load(self) -> list[str]:
|
||||
"""Read the posted list, creating an empty file when missing."""
|
||||
_ensure_parent(self._path)
|
||||
document = _read_existing(self._path)
|
||||
if document is None:
|
||||
if not self._path.exists():
|
||||
_write_document(self._path, dict(_EMPTY_DOCUMENT))
|
||||
return []
|
||||
return _coerce_list(document.get("posted"))
|
||||
|
||||
def is_posted(self, relative_path: str) -> bool:
|
||||
"""Return ``True`` when ``relative_path`` is already recorded."""
|
||||
posted = self.load()
|
||||
return relative_path in posted
|
||||
|
||||
def _with_lock(self, mutate):
|
||||
lock_path, fd = _lock_path(self._path)
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_EX)
|
||||
return mutate()
|
||||
finally:
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||
finally:
|
||||
os.close(fd)
|
||||
try:
|
||||
lock_path.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
def mark_posted(self, relative_path: str) -> bool:
|
||||
"""Append ``relative_path`` to the stored list. Idempotent.
|
||||
|
||||
Returns ``True`` when the path was newly added, ``False`` when
|
||||
it was already present.
|
||||
"""
|
||||
return self.mark_posted_many([relative_path]) != []
|
||||
|
||||
def mark_posted_many(self, relative_paths: Iterable[str]) -> list[str]:
|
||||
"""Append any new entries from ``relative_paths`` in one write.
|
||||
|
||||
Returns the list of paths that were newly added (possibly
|
||||
empty when the store already contained every supplied path).
|
||||
"""
|
||||
|
||||
candidates = [p for p in relative_paths if isinstance(p, str) and p]
|
||||
if not candidates:
|
||||
return []
|
||||
|
||||
def _mutate() -> list[str]:
|
||||
document = _read_existing(self._path) or dict(_EMPTY_DOCUMENT)
|
||||
existing = _coerce_list(document.get("posted"))
|
||||
added = [p for p in candidates if p not in existing]
|
||||
if not added:
|
||||
return []
|
||||
existing.extend(added)
|
||||
document["posted"] = existing
|
||||
_write_document(self._path, document)
|
||||
return added
|
||||
|
||||
return self._with_lock(_mutate)
|
||||
|
||||
|
||||
def load_posted(data_dir: Path | None = None) -> list[str]:
|
||||
return PostedStore(data_dir).load()
|
||||
|
||||
|
||||
def is_posted(relative_path: str, data_dir: Path | None = None) -> bool:
|
||||
return PostedStore(data_dir).is_posted(relative_path)
|
||||
|
||||
|
||||
def mark_posted(relative_path: str, data_dir: Path | None = None) -> bool:
|
||||
return PostedStore(data_dir).mark_posted(relative_path)
|
||||
|
||||
|
||||
def mark_posted_many(
|
||||
relative_paths: list[str], data_dir: Path | None = None
|
||||
) -> list[str]:
|
||||
return PostedStore(data_dir).mark_posted_many(relative_paths)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_DATA_DIR",
|
||||
"PostedStore",
|
||||
"PostedStoreError",
|
||||
"is_posted",
|
||||
"load_posted",
|
||||
"mark_posted",
|
||||
"mark_posted_many",
|
||||
"posted_path",
|
||||
]
|
||||
@@ -0,0 +1,41 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tenbackward.config import Config
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def env_setup(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Provide a fully populated environment for Config success paths.
|
||||
|
||||
Tests that exercise the missing-required-key path should call
|
||||
`monkeypatch.delenv(...)` explicitly.
|
||||
"""
|
||||
monkeypatch.setenv("MASTODON_BASE_URL", "https://mastodon.example")
|
||||
monkeypatch.setenv("MASTODON_ACCESS_TOKEN", "test-token")
|
||||
monkeypatch.setenv("MASTODON_VISIBILITY", "public")
|
||||
monkeypatch.setenv("SITE_URL", "https://blog.example.com")
|
||||
monkeypatch.setenv("HASHTAGS", "#throwback,#10backward")
|
||||
monkeypatch.setenv("THROWBACK_PREFIX", "Throwback:")
|
||||
monkeypatch.setenv("MAX_RETRIES", "3")
|
||||
monkeypatch.setenv("RUN_AT", "09:00")
|
||||
monkeypatch.setenv("TZ", "Europe/Berlin")
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def data_dir(tmp_path: Path) -> Path:
|
||||
return tmp_path / "data"
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def full_config(env_setup, data_dir) -> Config:
|
||||
"""Build a Config object that points at an isolated data dir."""
|
||||
from tenbackward.config import load_config
|
||||
|
||||
import os
|
||||
|
||||
os.environ["DATA_DIR"] = str(data_dir)
|
||||
return load_config()
|
||||
@@ -0,0 +1,214 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from git import GitCommandError, InvalidGitRepositoryError, Repo
|
||||
|
||||
from tenbackward.blog import (
|
||||
BACKOFF_BASE_SECONDS,
|
||||
BlogRepoError,
|
||||
BlogTransientError,
|
||||
DEFAULT_BLOG_SUBDIR,
|
||||
DEFAULT_REPO_URL,
|
||||
blog_dir,
|
||||
blog_repo_url,
|
||||
ensure_repo,
|
||||
)
|
||||
|
||||
|
||||
def _git_err(stderr: str = "boom") -> GitCommandError:
|
||||
return GitCommandError(["git"], stderr=stderr, status=1)
|
||||
|
||||
|
||||
def test_blog_dir_default_subdir(tmp_path: Path) -> None:
|
||||
assert blog_dir(tmp_path) == tmp_path / DEFAULT_BLOG_SUBDIR
|
||||
|
||||
|
||||
def test_blog_repo_url_default(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("BLOG_REPO_URL", raising=False)
|
||||
assert blog_repo_url() == DEFAULT_REPO_URL
|
||||
|
||||
|
||||
def test_blog_repo_url_env_override(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("BLOG_REPO_URL", "https://example.com/repo.git")
|
||||
assert blog_repo_url() == "https://example.com/repo.git"
|
||||
|
||||
|
||||
def test_ensure_repo_clones_when_missing(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
target = tmp_path / "blog"
|
||||
fake_repo = MagicMock(spec=Repo)
|
||||
calls: list[tuple[str, str]] = []
|
||||
|
||||
def fake_clone(url: str, path: str) -> Repo:
|
||||
calls.append((url, path))
|
||||
target.mkdir(parents=True)
|
||||
(target / ".git").mkdir()
|
||||
return fake_repo
|
||||
|
||||
sleeps: list[float] = []
|
||||
result = ensure_repo(
|
||||
tmp_path,
|
||||
repo_url="https://example.com/repo.git",
|
||||
blog_path=target,
|
||||
max_retries=2,
|
||||
sleep=sleeps.append,
|
||||
clone_impl=fake_clone,
|
||||
)
|
||||
assert result is fake_repo
|
||||
assert calls == [("https://example.com/repo.git", str(target))]
|
||||
assert sleeps == []
|
||||
|
||||
|
||||
def test_ensure_repo_pulls_when_clone_exists(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
target = tmp_path / "blog"
|
||||
target.mkdir()
|
||||
(target / ".git").mkdir()
|
||||
|
||||
fake_repo = MagicMock(spec=Repo)
|
||||
monkeypatch.setattr("tenbackward.blog.Repo", MagicMock(return_value=fake_repo))
|
||||
|
||||
fetch_impl = MagicMock()
|
||||
pull_impl = MagicMock()
|
||||
sleeps: list[float] = []
|
||||
|
||||
result = ensure_repo(
|
||||
tmp_path,
|
||||
repo_url="https://example.com/repo.git",
|
||||
blog_path=target,
|
||||
max_retries=3,
|
||||
sleep=sleeps.append,
|
||||
fetch_impl=lambda r: fetch_impl,
|
||||
pull_impl=lambda r: pull_impl,
|
||||
)
|
||||
|
||||
assert result is fake_repo
|
||||
fetch_impl.assert_called_once_with()
|
||||
pull_impl.assert_called_once_with()
|
||||
assert sleeps == []
|
||||
|
||||
|
||||
def test_ensure_repo_aborts_on_local_modifications(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
target = tmp_path / "blog"
|
||||
target.mkdir()
|
||||
(target / ".git").mkdir()
|
||||
|
||||
fake_repo = MagicMock(spec=Repo)
|
||||
monkeypatch.setattr("tenbackward.blog.Repo", MagicMock(return_value=fake_repo))
|
||||
|
||||
fetch_impl = MagicMock()
|
||||
pull_impl = MagicMock(
|
||||
side_effect=_git_err("Your local changes to 'foo' would be overwritten by merge")
|
||||
)
|
||||
sleeps: list[float] = []
|
||||
|
||||
with caplog.at_level(logging.ERROR):
|
||||
with pytest.raises(BlogRepoError, match="local modifications"):
|
||||
ensure_repo(
|
||||
tmp_path,
|
||||
repo_url="https://example.com/repo.git",
|
||||
blog_path=target,
|
||||
max_retries=5,
|
||||
sleep=sleeps.append,
|
||||
fetch_impl=lambda r: fetch_impl,
|
||||
pull_impl=lambda r: pull_impl,
|
||||
)
|
||||
|
||||
assert fetch_impl.call_count == 1
|
||||
assert pull_impl.call_count == 1
|
||||
assert sleeps == []
|
||||
|
||||
|
||||
def test_ensure_repo_retries_transient_error_then_succeeds(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
target = tmp_path / "blog"
|
||||
target.mkdir()
|
||||
(target / ".git").mkdir()
|
||||
|
||||
fake_repo = MagicMock(spec=Repo)
|
||||
monkeypatch.setattr("tenbackward.blog.Repo", MagicMock(return_value=fake_repo))
|
||||
|
||||
fetch_impl = MagicMock(side_effect=[_git_err("Could not resolve host"), None])
|
||||
pull_impl = MagicMock()
|
||||
sleeps: list[float] = []
|
||||
|
||||
ensure_repo(
|
||||
tmp_path,
|
||||
repo_url="https://example.com/repo.git",
|
||||
blog_path=target,
|
||||
max_retries=5,
|
||||
sleep=sleeps.append,
|
||||
fetch_impl=lambda r: fetch_impl,
|
||||
pull_impl=lambda r: pull_impl,
|
||||
)
|
||||
|
||||
assert fetch_impl.call_count == 2
|
||||
pull_impl.assert_called_once_with()
|
||||
assert sleeps == [BACKOFF_BASE_SECONDS ** 0]
|
||||
|
||||
|
||||
def test_ensure_repo_raises_after_exhausted_retries(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
target = tmp_path / "blog"
|
||||
target.mkdir()
|
||||
(target / ".git").mkdir()
|
||||
|
||||
fake_repo = MagicMock(spec=Repo)
|
||||
monkeypatch.setattr("tenbackward.blog.Repo", MagicMock(return_value=fake_repo))
|
||||
|
||||
fetch_impl = MagicMock(side_effect=_git_err("Could not resolve host"))
|
||||
pull_impl = MagicMock()
|
||||
sleeps: list[float] = []
|
||||
|
||||
with pytest.raises(BlogTransientError, match="git pull failed"):
|
||||
ensure_repo(
|
||||
tmp_path,
|
||||
repo_url="https://example.com/repo.git",
|
||||
blog_path=target,
|
||||
max_retries=2,
|
||||
sleep=sleeps.append,
|
||||
fetch_impl=lambda r: fetch_impl,
|
||||
pull_impl=lambda r: pull_impl,
|
||||
)
|
||||
|
||||
assert fetch_impl.call_count == 2
|
||||
pull_impl.assert_not_called()
|
||||
assert sleeps == [BACKOFF_BASE_SECONDS ** 0]
|
||||
|
||||
|
||||
def test_ensure_repo_invalid_clone_dir_raises(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
target = tmp_path / "blog"
|
||||
target.mkdir()
|
||||
(target / "not-a-repo.txt").write_text("hi")
|
||||
|
||||
def fake_repo(_path: str) -> Repo:
|
||||
raise InvalidGitRepositoryError(f"{_path} is not a repo")
|
||||
|
||||
monkeypatch.setattr("tenbackward.blog.Repo", fake_repo)
|
||||
|
||||
with pytest.raises(BlogRepoError, match="not a valid git repository"):
|
||||
ensure_repo(
|
||||
tmp_path,
|
||||
repo_url="https://example.com/repo.git",
|
||||
blog_path=target,
|
||||
max_retries=0,
|
||||
)
|
||||
@@ -0,0 +1,220 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tenbackward.config import ConfigError, load_config, validate_config, validate_required
|
||||
|
||||
|
||||
def _populate(env_setup) -> dict[str, str]:
|
||||
return {k: os.environ[k] for k in os.environ if k.startswith(("MASTODON_", "SITE_", "HASHTAGS", "THROWBACK_", "MAX_RETRIES", "RUN_AT", "TZ"))}
|
||||
|
||||
|
||||
def _full_values() -> dict[str, str]:
|
||||
return {
|
||||
"MASTODON_BASE_URL": "https://mastodon.example",
|
||||
"MASTODON_ACCESS_TOKEN": "x",
|
||||
"MASTODON_VISIBILITY": "public",
|
||||
"SITE_URL": "https://blog.example.com",
|
||||
"HASHTAGS": "#throwback",
|
||||
"THROWBACK_PREFIX": "Throwback:",
|
||||
"MAX_RETRIES": "3",
|
||||
"RUN_AT": "09:00",
|
||||
"TZ": "Europe/Berlin",
|
||||
}
|
||||
|
||||
|
||||
def test_load_config_succeeds_with_complete_env(env_setup) -> None:
|
||||
config = load_config()
|
||||
assert config.mastodon_base_url == "https://mastodon.example"
|
||||
assert config.run_at == "09:00"
|
||||
assert config.max_retries == 3
|
||||
assert config.throwback_prefix == "Throwback:"
|
||||
assert config.mastodon_visibility == "public"
|
||||
|
||||
|
||||
def test_load_config_blog_defaults(env_setup) -> None:
|
||||
config = load_config()
|
||||
assert config.blog_repo_url == "https://git.chaospott.de/Chaospott/site"
|
||||
assert config.blog_dir.name == "blog"
|
||||
|
||||
|
||||
def test_load_config_blog_repo_url_override(env_setup, monkeypatch) -> None:
|
||||
monkeypatch.setenv("BLOG_REPO_URL", "https://example.com/repo.git")
|
||||
config = load_config()
|
||||
assert config.blog_repo_url == "https://example.com/repo.git"
|
||||
|
||||
|
||||
def test_validate_config_rejects_bad_blog_repo_url(env_setup) -> None:
|
||||
values = {
|
||||
"MASTODON_BASE_URL": "https://mastodon.example",
|
||||
"MASTODON_ACCESS_TOKEN": "x",
|
||||
"MASTODON_VISIBILITY": "public",
|
||||
"SITE_URL": "https://blog.example.com",
|
||||
"HASHTAGS": "#throwback",
|
||||
"THROWBACK_PREFIX": "Throwback:",
|
||||
"MAX_RETRIES": "3",
|
||||
"RUN_AT": "09:00",
|
||||
"TZ": "Europe/Berlin",
|
||||
"BLOG_REPO_URL": "ftp://bad",
|
||||
}
|
||||
with pytest.raises(ConfigError, match="BLOG_REPO_URL"):
|
||||
validate_config(values)
|
||||
|
||||
|
||||
def test_validate_config_lists_every_missing_key(env_setup, monkeypatch) -> None:
|
||||
monkeypatch.delenv("MASTODON_ACCESS_TOKEN")
|
||||
monkeypatch.delenv("SITE_URL")
|
||||
|
||||
values = {k: os.environ.get(k, "") for k in [
|
||||
"MASTODON_BASE_URL",
|
||||
"MASTODON_ACCESS_TOKEN",
|
||||
"MASTODON_VISIBILITY",
|
||||
"SITE_URL",
|
||||
"HASHTAGS",
|
||||
"THROWBACK_PREFIX",
|
||||
"MAX_RETRIES",
|
||||
"RUN_AT",
|
||||
"TZ",
|
||||
]}
|
||||
|
||||
with pytest.raises(ConfigError) as excinfo:
|
||||
validate_required(values)
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert "MASTODON_ACCESS_TOKEN" in message
|
||||
assert "SITE_URL" in message
|
||||
|
||||
|
||||
def test_load_config_fails_fast_on_missing_required(env_setup, monkeypatch) -> None:
|
||||
monkeypatch.delenv("MASTODON_ACCESS_TOKEN", raising=False)
|
||||
|
||||
with pytest.raises(ConfigError, match="MASTODON_ACCESS_TOKEN"):
|
||||
load_config()
|
||||
|
||||
|
||||
def test_validate_config_rejects_bad_run_at() -> None:
|
||||
values = _full_values()
|
||||
values["RUN_AT"] = "25:99"
|
||||
with pytest.raises(ConfigError):
|
||||
validate_config(values)
|
||||
|
||||
|
||||
def test_validate_config_rejects_invalid_url() -> None:
|
||||
values = _full_values()
|
||||
values["SITE_URL"] = "not-a-url"
|
||||
with pytest.raises(ConfigError, match="SITE_URL"):
|
||||
validate_config(values)
|
||||
|
||||
values = _full_values()
|
||||
values["MASTODON_BASE_URL"] = "ftp://mastodon.example"
|
||||
with pytest.raises(ConfigError, match="MASTODON_BASE_URL"):
|
||||
validate_config(values)
|
||||
|
||||
|
||||
def test_validate_config_rejects_invalid_visibility() -> None:
|
||||
for bad in ("private", "direct", "PUBLIC"):
|
||||
values = _full_values()
|
||||
values["MASTODON_VISIBILITY"] = bad
|
||||
with pytest.raises(ConfigError, match="MASTODON_VISIBILITY"):
|
||||
validate_config(values)
|
||||
|
||||
|
||||
def test_validate_required_rejects_empty_visibility() -> None:
|
||||
values = _full_values()
|
||||
values["MASTODON_VISIBILITY"] = ""
|
||||
with pytest.raises(ConfigError, match="MASTODON_VISIBILITY"):
|
||||
validate_required(values)
|
||||
|
||||
|
||||
def test_validate_config_rejects_invalid_tz() -> None:
|
||||
values = _full_values()
|
||||
values["TZ"] = "Not/AZone"
|
||||
with pytest.raises(ConfigError, match="TZ"):
|
||||
validate_config(values)
|
||||
|
||||
|
||||
def test_validate_config_rejects_negative_max_retries() -> None:
|
||||
values = _full_values()
|
||||
values["MAX_RETRIES"] = "-1"
|
||||
with pytest.raises(ConfigError, match="MAX_RETRIES"):
|
||||
validate_config(values)
|
||||
|
||||
|
||||
def test_validate_config_accepts_zero_max_retries() -> None:
|
||||
values = _full_values()
|
||||
values["MAX_RETRIES"] = "0"
|
||||
validate_config(values)
|
||||
|
||||
|
||||
def test_load_config_applies_optional_defaults(env_setup, monkeypatch) -> None:
|
||||
monkeypatch.delenv("MAX_RETRIES", raising=False)
|
||||
|
||||
config = load_config()
|
||||
assert config.max_retries == 5
|
||||
assert config.blog_repo_url == "https://git.chaospott.de/Chaospott/site"
|
||||
assert config.blog_dir.name == "blog"
|
||||
|
||||
|
||||
def test_load_config_reads_dotenv_file(tmp_path: Path, monkeypatch) -> None:
|
||||
dotenv = tmp_path / ".env"
|
||||
dotenv.write_text(
|
||||
"MASTODON_BASE_URL=https://from-file.example\n"
|
||||
"MASTODON_ACCESS_TOKEN=file-token\n"
|
||||
"MASTODON_VISIBILITY=unlisted\n"
|
||||
"SITE_URL=https://blog.example.com\n"
|
||||
"HASHTAGS=#throwback\n"
|
||||
"THROWBACK_PREFIX=Werferückblick:\n"
|
||||
"MAX_RETRIES=5\n"
|
||||
"RUN_AT=12:34\n"
|
||||
"TZ=Europe/Berlin\n"
|
||||
)
|
||||
|
||||
for key in (
|
||||
"MASTODON_BASE_URL",
|
||||
"MASTODON_ACCESS_TOKEN",
|
||||
"MASTODON_VISIBILITY",
|
||||
"SITE_URL",
|
||||
"HASHTAGS",
|
||||
"THROWBACK_PREFIX",
|
||||
"MAX_RETRIES",
|
||||
"RUN_AT",
|
||||
"TZ",
|
||||
):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
config = load_config(dotenv_path=dotenv)
|
||||
assert config.mastodon_base_url == "https://from-file.example"
|
||||
assert config.mastodon_access_token == "file-token"
|
||||
assert config.run_at == "12:34"
|
||||
assert config.mastodon_visibility == "unlisted"
|
||||
assert config.throwback_prefix == "Werferückblick:"
|
||||
assert config.max_retries == 5
|
||||
|
||||
|
||||
def test_env_overrides_dotenv(tmp_path: Path, monkeypatch) -> None:
|
||||
dotenv = tmp_path / ".env"
|
||||
dotenv.write_text("RUN_AT=01:00\n")
|
||||
|
||||
monkeypatch.setenv("RUN_AT", "23:00")
|
||||
for key in (
|
||||
"MASTODON_BASE_URL",
|
||||
"MASTODON_ACCESS_TOKEN",
|
||||
"MASTODON_VISIBILITY",
|
||||
"SITE_URL",
|
||||
"HASHTAGS",
|
||||
"THROWBACK_PREFIX",
|
||||
"MAX_RETRIES",
|
||||
"TZ",
|
||||
):
|
||||
monkeypatch.setenv(key, "x")
|
||||
monkeypatch.setenv("MASTODON_BASE_URL", "https://mastodon.example")
|
||||
monkeypatch.setenv("SITE_URL", "https://blog.example.com")
|
||||
monkeypatch.setenv("MASTODON_VISIBILITY", "public")
|
||||
monkeypatch.setenv("TZ", "Europe/Berlin")
|
||||
monkeypatch.setenv("MAX_RETRIES", "3")
|
||||
|
||||
config = load_config(dotenv_path=dotenv)
|
||||
assert config.run_at == "23:00"
|
||||
@@ -0,0 +1,195 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tenbackward.config import _is_valid_hhmm
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
ENTRYPOINT = REPO_ROOT / "entrypoint.sh"
|
||||
|
||||
|
||||
def test_valid_hhmm() -> None:
|
||||
assert _is_valid_hhmm("00:00")
|
||||
assert _is_valid_hhmm("09:30")
|
||||
assert _is_valid_hhmm("23:59")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["24:00", "9:00", "09:60", "9", "9:00:00", "ab:cd", "", None])
|
||||
def test_invalid_hhmm(value) -> None:
|
||||
assert not _is_valid_hhmm(value) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _entrypoint_env(extra: dict[str, str]) -> dict[str, str]:
|
||||
env = {
|
||||
"PATH": "/usr/bin:/bin:/usr/sbin:/sbin",
|
||||
"DEBIAN_FRONTEND": "noninteractive",
|
||||
"HOME": "/tmp",
|
||||
"LANG": "C.UTF-8",
|
||||
}
|
||||
env.update(extra)
|
||||
return env
|
||||
|
||||
|
||||
def _run_entrypoint_cron_render(extra: dict[str, str]) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def _extract_rendered_cron(env: dict[str, str]) -> tuple[int, str, str]:
|
||||
"""Run the entrypoint with a fake `install` that records the cron file,
|
||||
and a stub `cron` that exits immediately so the entrypoint doesn't hang.
|
||||
Returns (returncode, stderr, installed-cron-contents).
|
||||
"""
|
||||
import tempfile
|
||||
|
||||
with tempfile.TemporaryDirectory() as fake_bin_dir, tempfile.TemporaryDirectory() as tmp:
|
||||
fake_cron = Path(tmp) / "tenbackward.cron"
|
||||
|
||||
# Fake `install`: real syntax is `install [-m mode] [-o owner] [-g group] SRC DST`.
|
||||
# We extract the last positional arg as DST and the last existing source before it.
|
||||
fake_install = Path(fake_bin_dir) / "install"
|
||||
fake_install.write_text(
|
||||
"#!/bin/bash\n"
|
||||
"src=\"\"\n"
|
||||
"dst=\"\"\n"
|
||||
"while [ $# -gt 0 ]; do\n"
|
||||
" case \"$1\" in\n"
|
||||
" -m|-o|-g) shift; shift;;\n"
|
||||
" *) src=\"${src:-$1}\"; dst=\"$1\"; shift;;\n"
|
||||
" esac\n"
|
||||
"done\n"
|
||||
"cp \"$src\" \"" + str(fake_cron) + "\"\n"
|
||||
)
|
||||
fake_install.chmod(0o755)
|
||||
|
||||
# Fake `cron` exits immediately so we never hang waiting on cron.
|
||||
fake_cron_bin = Path(fake_bin_dir) / "cron"
|
||||
fake_cron_bin.write_text("#!/bin/bash\nexit 0\n")
|
||||
fake_cron_bin.chmod(0o755)
|
||||
|
||||
env = dict(env)
|
||||
env["PATH"] = f"{fake_bin_dir}:/usr/bin:/bin"
|
||||
|
||||
proc = subprocess.run(
|
||||
["/bin/bash", str(ENTRYPOINT)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
|
||||
rendered = fake_cron.read_text() if fake_cron.exists() else ""
|
||||
return proc.returncode, proc.stderr, rendered
|
||||
|
||||
|
||||
@pytest.mark.skipif(not shutil.which("bash"), reason="bash not available")
|
||||
def test_entrypoint_renders_run_at_into_cron() -> None:
|
||||
"""Drive the entrypoint with a hermetic PATH so we can observe the rendered cron line.
|
||||
|
||||
The entrypoint hard-codes `/app/crontab/install-cron.sh` (correct in the
|
||||
Docker image). For the host-side test we override PATH so `install` and
|
||||
`cron` are stubbed, but we still need `/app/crontab/install-cron.sh` — we
|
||||
bind-mount it via a bind-mount emulation by overriding the env var
|
||||
`PATH` and having the install-cron.sh check fall back: instead, we patch
|
||||
the entrypoint with an in-place substitution for the test.
|
||||
"""
|
||||
tmp_cron_dest = _write_substituted_entrypoint()
|
||||
try:
|
||||
env = _entrypoint_env({
|
||||
"MASTODON_BASE_URL": "https://mastodon.example",
|
||||
"MASTODON_ACCESS_TOKEN": "x",
|
||||
"MASTODON_VISIBILITY": "public",
|
||||
"SITE_URL": "https://blog.example.com",
|
||||
"HASHTAGS": "#throwback",
|
||||
"THROWBACK_PREFIX": "Throwback:",
|
||||
"MAX_RETRIES": "3",
|
||||
"RUN_AT": "09:00",
|
||||
"TZ": "Europe/Berlin",
|
||||
})
|
||||
|
||||
returncode, stderr, rendered = _extract_rendered_cron(env, tmp_cron_dest)
|
||||
|
||||
assert returncode == 0, (returncode, stderr)
|
||||
assert "00 09 * * * /usr/local/bin/run-bot.sh >> /proc/1/fd/1 2>&1" in rendered, rendered
|
||||
assert "TZ=Europe/Berlin" in rendered
|
||||
finally:
|
||||
tmp_cron_dest.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _write_substituted_entrypoint():
|
||||
"""Copy entrypoint.sh to a temp file with `/app/crontab/install-cron.sh`
|
||||
rewritten to the repo-relative path so the host can execute it."""
|
||||
import tempfile
|
||||
src = ENTRYPOINT.read_text()
|
||||
sub = src.replace("/app/crontab/install-cron.sh", str(REPO_ROOT / "crontab" / "install-cron.sh"))
|
||||
tmp = Path(tempfile.mkstemp(prefix="entrypoint-", suffix=".sh")[1])
|
||||
tmp.write_text(sub)
|
||||
tmp.chmod(0o755)
|
||||
return tmp
|
||||
|
||||
|
||||
def _extract_rendered_cron(env: dict[str, str], entrypoint_path: Path) -> tuple[int, str, str]:
|
||||
"""Run the (substituted) entrypoint with stubs for `install` and `cron`."""
|
||||
import tempfile
|
||||
|
||||
with tempfile.TemporaryDirectory() as fake_bin_dir, tempfile.TemporaryDirectory() as tmp:
|
||||
sidecar = Path(tmp) / "tenbackward.cron"
|
||||
|
||||
fake_install = Path(fake_bin_dir) / "install"
|
||||
fake_install.write_text(
|
||||
"#!/bin/bash\n"
|
||||
"src=\"\"; dst=\"\"\n"
|
||||
"while [ $# -gt 0 ]; do\n"
|
||||
" case \"$1\" in -m|-o|-g) shift; shift;; *) src=\"${src:-$1}\"; dst=\"$1\"; shift;; esac\n"
|
||||
"done\n"
|
||||
"cp \"$src\" \"" + str(sidecar) + "\"\n"
|
||||
"mkdir -p \"$(dirname \"$dst\")\" 2>/dev/null\n"
|
||||
"cp \"$src\" \"$dst\" 2>/dev/null || true\n"
|
||||
)
|
||||
fake_install.chmod(0o755)
|
||||
|
||||
fake_cron_bin = Path(fake_bin_dir) / "cron"
|
||||
fake_cron_bin.write_text("#!/bin/bash\nexit 0\n")
|
||||
fake_cron_bin.chmod(0o755)
|
||||
|
||||
env = dict(env)
|
||||
env["PATH"] = f"{fake_bin_dir}:/usr/bin:/bin"
|
||||
|
||||
proc = subprocess.run(
|
||||
["/bin/bash", str(entrypoint_path)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
|
||||
rendered = sidecar.read_text() if sidecar.exists() else ""
|
||||
return proc.returncode, proc.stderr, rendered
|
||||
|
||||
|
||||
@pytest.mark.skipif(not shutil.which("bash"), reason="bash not available")
|
||||
def test_install_cron_rejects_unresolved_placeholder(tmp_path: Path) -> None:
|
||||
script = REPO_ROOT / "crontab" / "install-cron.sh"
|
||||
|
||||
bad = tmp_path / "bad.cron"
|
||||
bad.write_text("__RUN_AT__ * * * * /bin/false\n")
|
||||
proc = subprocess.run(
|
||||
["/bin/bash", str(script), str(bad)],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
assert proc.returncode != 0
|
||||
assert "__RUN_AT__" in proc.stderr
|
||||
|
||||
good = tmp_path / "good.cron"
|
||||
good.write_text("SHELL=/bin/bash\n5 9 * * * /usr/local/bin/run-bot.sh >> /proc/1/fd/1 2>&1\n")
|
||||
proc2 = subprocess.run(
|
||||
["/bin/bash", str(script), str(good)],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
assert proc2.returncode == 0, proc2.stderr
|
||||
@@ -0,0 +1,84 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def test_dockerignore_blocks_secrets_and_data() -> None:
|
||||
dockerignore = (REPO_ROOT / ".dockerignore").read_text().splitlines()
|
||||
normalized = {line.strip() for line in dockerignore if line.strip() and not line.startswith("#")}
|
||||
|
||||
for required in (".env", "data/", ".git/", "tests/", ".kilo/", ".venv/"):
|
||||
assert required in normalized, f"missing {required} from .dockerignore"
|
||||
|
||||
|
||||
def test_gitignore_blocks_dotenv_and_data() -> None:
|
||||
gitignore = (REPO_ROOT / ".gitignore").read_text().splitlines()
|
||||
normalized = {line.strip() for line in gitignore if line.strip() and not line.startswith("#")}
|
||||
assert ".env" in normalized
|
||||
assert "data/" in normalized
|
||||
assert ".kilo/" in normalized
|
||||
|
||||
|
||||
def test_env_example_has_no_real_tokens() -> None:
|
||||
text = (REPO_ROOT / ".env.example").read_text()
|
||||
assert "MASTODON_ACCESS_TOKEN=replace-me" in text
|
||||
|
||||
for token in ("ghp_", "gho_", "xoxb-", "Bearer ey", "AKIA"):
|
||||
assert token not in text, f"found suspicious token prefix {token!r} in .env.example"
|
||||
|
||||
mastodon_line = next(
|
||||
line for line in text.splitlines() if line.startswith("MASTODON_ACCESS_TOKEN=")
|
||||
)
|
||||
value = mastodon_line.split("=", 1)[1].strip()
|
||||
assert not re.fullmatch(r"[A-Za-z0-9_\-]{40,}", value), (
|
||||
"MASTODON_ACCESS_TOKEN placeholder should not look like a real token"
|
||||
)
|
||||
|
||||
|
||||
def test_dockerfile_does_not_copy_env_or_data() -> None:
|
||||
dockerfile = (REPO_ROOT / "Dockerfile").read_text()
|
||||
for blocked in (".env", "data/", "./data"):
|
||||
for line in dockerfile.splitlines():
|
||||
if line.lstrip().upper().startswith("COPY"):
|
||||
assert blocked not in line, f"Dockerfile COPY must not reference {blocked}: {line!r}"
|
||||
|
||||
|
||||
def test_compose_mounts_data_and_env_readonly() -> None:
|
||||
compose = (REPO_ROOT / "docker-compose.yml").read_text()
|
||||
assert "env_file:" in compose
|
||||
assert "- .env" in compose
|
||||
assert "./data:/app/data" in compose
|
||||
assert "build:" in compose
|
||||
|
||||
|
||||
def test_dockerfile_runs_entrypoint_as_root() -> None:
|
||||
dockerfile = (REPO_ROOT / "Dockerfile").read_text()
|
||||
lines = [line.strip() for line in dockerfile.splitlines()]
|
||||
entrypoint_idx = next(
|
||||
(i for i, line in enumerate(lines) if line.upper().startswith("ENTRYPOINT")),
|
||||
None,
|
||||
)
|
||||
assert entrypoint_idx is not None, "Dockerfile must declare ENTRYPOINT"
|
||||
for line in lines[entrypoint_idx:]:
|
||||
if line.upper().startswith("USER "):
|
||||
pytest.fail(
|
||||
"Dockerfile must not switch USER after ENTRYPOINT — entrypoint.sh "
|
||||
"needs root to install /etc/cron.d/tenbackward and exec cron -f"
|
||||
)
|
||||
|
||||
|
||||
def test_run_bot_wrapper_drops_privileges() -> None:
|
||||
wrapper = (REPO_ROOT / "run-bot.sh").read_text()
|
||||
assert "setpriv" in wrapper or "su -s" in wrapper
|
||||
assert "tenbackward" in wrapper
|
||||
|
||||
|
||||
def test_cron_template_has_env_header() -> None:
|
||||
cron = (REPO_ROOT / "crontab" / "tenbackward.cron").read_text()
|
||||
assert "SHELL=/bin/bash" in cron
|
||||
assert "PATH=" in cron
|
||||
@@ -0,0 +1,94 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
ENTRYPOINT = REPO_ROOT / "entrypoint.sh"
|
||||
|
||||
|
||||
def _has_bash() -> bool:
|
||||
return shutil.which("bash") is not None
|
||||
|
||||
|
||||
def _minimal_path() -> str:
|
||||
parts = ["/usr/bin", "/bin", "/usr/sbin", "/sbin"]
|
||||
for p in parts:
|
||||
if Path(p).exists():
|
||||
return ":".join(parts)
|
||||
return os.environ.get("PATH", "")
|
||||
|
||||
|
||||
def _clean_env() -> dict[str, str]:
|
||||
env = {
|
||||
"PATH": _minimal_path(),
|
||||
"DEBIAN_FRONTEND": "noninteractive",
|
||||
"HOME": "/tmp",
|
||||
"LANG": "C.UTF-8",
|
||||
}
|
||||
return env
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _has_bash(), reason="bash not available")
|
||||
def test_entrypoint_rejects_bad_run_at(tmp_path: Path, monkeypatch) -> None:
|
||||
"""The entrypoint must exit non-zero with a clear message on a bad RUN_AT."""
|
||||
env = _clean_env()
|
||||
env["MASTODON_BASE_URL"] = "https://mastodon.example"
|
||||
env["MASTODON_ACCESS_TOKEN"] = "x"
|
||||
env["MASTODON_VISIBILITY"] = "public"
|
||||
env["SITE_URL"] = "https://blog.example.com"
|
||||
env["HASHTAGS"] = "#throwback"
|
||||
env["THROWBACK_PREFIX"] = "Throwback:"
|
||||
env["MAX_RETRIES"] = "3"
|
||||
env["RUN_AT"] = "25:99"
|
||||
env["TZ"] = "Europe/Berlin"
|
||||
|
||||
proc = subprocess.run(
|
||||
["/bin/bash", str(ENTRYPOINT)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
assert proc.returncode != 0
|
||||
assert "RUN_AT" in (proc.stdout + proc.stderr)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _has_bash(), reason="bash not available")
|
||||
def test_entrypoint_fails_fast_when_token_missing(tmp_path: Path, monkeypatch) -> None:
|
||||
"""A missing required var must abort before cron is started."""
|
||||
env = _clean_env()
|
||||
env["MASTODON_BASE_URL"] = "https://mastodon.example"
|
||||
env["MASTODON_VISIBILITY"] = "public"
|
||||
env["SITE_URL"] = "https://blog.example.com"
|
||||
env["HASHTAGS"] = "#throwback"
|
||||
env["THROWBACK_PREFIX"] = "Throwback:"
|
||||
env["MAX_RETRIES"] = "3"
|
||||
env["RUN_AT"] = "09:00"
|
||||
env["TZ"] = "Europe/Berlin"
|
||||
env.pop("MASTODON_ACCESS_TOKEN", None)
|
||||
|
||||
proc = subprocess.run(
|
||||
["/bin/bash", str(ENTRYPOINT)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
assert proc.returncode != 0
|
||||
combined = (proc.stdout + proc.stderr).lower()
|
||||
assert "missing" in combined
|
||||
assert "mastodon_access_token" in combined.lower()
|
||||
|
||||
|
||||
def test_entrypoint_references_cron_in_foreground() -> None:
|
||||
text = ENTRYPOINT.read_text()
|
||||
assert re.search(r"exec\s+cron\s+-f", text), "entrypoint must exec cron in the foreground"
|
||||
cron = (REPO_ROOT / "crontab" / "tenbackward.cron").read_text()
|
||||
assert "SHELL=/bin/bash" in cron
|
||||
@@ -0,0 +1,93 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
|
||||
from tenbackward.logging_setup import (
|
||||
JsonFormatter,
|
||||
configure_json_logging,
|
||||
log_error,
|
||||
log_run_summary,
|
||||
log_startup,
|
||||
)
|
||||
|
||||
|
||||
def _capture(logger: logging.Logger) -> io.StringIO:
|
||||
buf = io.StringIO()
|
||||
handler = logging.StreamHandler(buf)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
logger.handlers = [handler]
|
||||
logger.setLevel(logging.INFO)
|
||||
logger.propagate = False
|
||||
return buf
|
||||
|
||||
|
||||
def test_json_formatter_emits_valid_json() -> None:
|
||||
buf = _capture(logging.getLogger("tenbackward.test"))
|
||||
logging.getLogger("tenbackward.test").info("hello", extra={"foo": 1})
|
||||
line = buf.getvalue().strip()
|
||||
payload = json.loads(line)
|
||||
assert payload["message"] == "hello"
|
||||
assert payload["level"] == "INFO"
|
||||
assert payload["logger"] == "tenbackward.test"
|
||||
assert payload["foo"] == 1
|
||||
assert "ts" in payload
|
||||
|
||||
|
||||
def test_json_formatter_redacts_secret_keys() -> None:
|
||||
buf = _capture(logging.getLogger("tenbackward.test"))
|
||||
logging.getLogger("tenbackward.test").info(
|
||||
"msg",
|
||||
extra={"access_token": "secret-value", "site_url": "https://x"},
|
||||
)
|
||||
payload = json.loads(buf.getvalue().strip())
|
||||
assert payload["access_token"] == "[REDACTED]"
|
||||
assert payload["site_url"] == "https://x"
|
||||
|
||||
|
||||
def test_log_run_summary_emits_one_line_with_counters() -> None:
|
||||
buf = _capture(logging.getLogger("tenbackward"))
|
||||
log_run_summary(3, 2, 1, 1, ["x"])
|
||||
lines = [line for line in buf.getvalue().splitlines() if line.strip()]
|
||||
assert len(lines) == 1
|
||||
payload = json.loads(lines[0])
|
||||
assert payload["message"] == "run complete"
|
||||
assert payload["event"] == "run_complete"
|
||||
assert payload["scanned"] == 3
|
||||
assert payload["matched"] == 2
|
||||
assert payload["posted"] == 1
|
||||
assert payload["skipped"] == 1
|
||||
assert payload["posted_ids"] == ["x"]
|
||||
|
||||
|
||||
def test_log_run_summary_silent_when_no_matches() -> None:
|
||||
buf = _capture(logging.getLogger("tenbackward"))
|
||||
log_run_summary(0, 0, 0, 0, [])
|
||||
assert buf.getvalue() == ""
|
||||
|
||||
|
||||
def test_log_startup_emits_single_info_line() -> None:
|
||||
buf = _capture(logging.getLogger("tenbackward"))
|
||||
log_startup(version="0.1.0", site_url="https://x", run_at="09:00", tz="Europe/Berlin")
|
||||
payload = json.loads(buf.getvalue().strip())
|
||||
assert payload["event"] == "startup"
|
||||
assert payload["version"] == "0.1.0"
|
||||
|
||||
|
||||
def test_log_error_redacts_secret_keys_and_exposes_exc_type() -> None:
|
||||
buf = _capture(logging.getLogger("tenbackward"))
|
||||
log_error("boom", exc=RuntimeError("x"), access_token="abc")
|
||||
payload = json.loads(buf.getvalue().strip())
|
||||
assert payload["level"] == "ERROR"
|
||||
assert payload["event"] == "boom"
|
||||
assert payload["access_token"] == "[REDACTED]"
|
||||
assert payload["exc_type"] == "RuntimeError"
|
||||
|
||||
|
||||
def test_configure_json_logging_is_idempotent() -> None:
|
||||
configure_json_logging()
|
||||
configure_json_logging()
|
||||
root = logging.getLogger()
|
||||
json_handlers = [h for h in root.handlers if isinstance(h.formatter, JsonFormatter)]
|
||||
assert len(json_handlers) == 1
|
||||
@@ -0,0 +1,236 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
import pytest
|
||||
|
||||
from tenbackward.matching import (
|
||||
MatchedPost,
|
||||
find_anniversary_matches,
|
||||
iter_anniversary_paths,
|
||||
slugify_title,
|
||||
)
|
||||
|
||||
|
||||
SITE_URL = "https://chaospott.de"
|
||||
|
||||
_MATCHING_LOGGER = "tenbackward.matching"
|
||||
|
||||
|
||||
def _write_post(root: Path, rel_path: str, *, body: str = "post") -> None:
|
||||
full = root / rel_path
|
||||
full.parent.mkdir(parents=True, exist_ok=True)
|
||||
full.write_text(body, encoding="utf-8")
|
||||
|
||||
|
||||
def _capture_matching_warnings(call: Callable[[], object]) -> list[logging.LogRecord]:
|
||||
records: list[logging.LogRecord] = []
|
||||
handler = logging.Handler()
|
||||
handler.setLevel(logging.WARNING)
|
||||
handler.emit = records.append # type: ignore[assignment]
|
||||
logger = logging.getLogger(_MATCHING_LOGGER)
|
||||
previous_level = logger.level
|
||||
logger.setLevel(logging.WARNING)
|
||||
logger.addHandler(handler)
|
||||
try:
|
||||
call()
|
||||
finally:
|
||||
logger.removeHandler(handler)
|
||||
logger.setLevel(previous_level)
|
||||
return records
|
||||
|
||||
|
||||
def _event_names(records: list[logging.LogRecord]) -> list[str]:
|
||||
return [getattr(record, "event", None) for record in records]
|
||||
|
||||
|
||||
def test_match_uses_frontmatter_date_and_title(tmp_path: Path) -> None:
|
||||
root = tmp_path / "_posts" / "blog"
|
||||
_write_post(
|
||||
root,
|
||||
"2014/2014-08-04-foo.md",
|
||||
body="---\ntitle: Foo\ndate: 2014-08-04\n---\nbody\n",
|
||||
)
|
||||
|
||||
today = date(2024, 8, 4)
|
||||
matches = find_anniversary_matches(root, SITE_URL, today=today)
|
||||
|
||||
assert len(matches) == 1
|
||||
match = matches[0]
|
||||
assert isinstance(match, MatchedPost)
|
||||
assert match.path == "2014/2014-08-04-foo.md"
|
||||
assert match.title == "Foo"
|
||||
assert match.date == date(2014, 8, 4)
|
||||
assert match.url == "https://chaospott.de/2014/08/04/foo/"
|
||||
assert match.url.endswith(slugify_title(match.title) + "/")
|
||||
|
||||
|
||||
def test_match_falls_back_to_filename_when_frontmatter_missing(tmp_path: Path) -> None:
|
||||
root = tmp_path / "_posts" / "blog"
|
||||
_write_post(root, "2015/2015-03-10-no-frontmatter.md", body="no frontmatter at all\n")
|
||||
_write_post(
|
||||
root, "2015/2015-03-11-broken-frontmatter.md", body="---\ndate: not-a-date\n: :\n"
|
||||
)
|
||||
|
||||
today = date(2025, 3, 10)
|
||||
matches: list[MatchedPost] = []
|
||||
records = _capture_matching_warnings(
|
||||
lambda: matches.extend(find_anniversary_matches(root, SITE_URL, today=today))
|
||||
)
|
||||
|
||||
paths = {m.path for m in matches}
|
||||
assert "2015/2015-03-10-no-frontmatter.md" in paths
|
||||
assert "2015/2015-03-11-broken-frontmatter.md" not in paths
|
||||
|
||||
match = next(m for m in matches if m.path == "2015/2015-03-10-no-frontmatter.md")
|
||||
assert match.title == "no-frontmatter"
|
||||
assert match.date == date(2015, 3, 10)
|
||||
assert match.url == "https://chaospott.de/2015/03/10/no-frontmatter/"
|
||||
assert match.url.endswith(slugify_title(match.title) + "/")
|
||||
|
||||
|
||||
def test_unpublished_post_is_skipped(tmp_path: Path) -> None:
|
||||
root = tmp_path / "_posts" / "blog"
|
||||
_write_post(
|
||||
root,
|
||||
"2014/2014-08-04-draft.md",
|
||||
body="---\ntitle: Draft\ndate: 2014-08-04\npublished: false\n---\n",
|
||||
)
|
||||
|
||||
today = date(2024, 8, 4)
|
||||
matches: list[MatchedPost] = []
|
||||
records = _capture_matching_warnings(
|
||||
lambda: matches.extend(find_anniversary_matches(root, SITE_URL, today=today))
|
||||
)
|
||||
|
||||
assert matches == []
|
||||
assert _event_names(records) == ["post_unpublished"]
|
||||
|
||||
|
||||
def test_iter_anniversary_paths_yields_only_path(tmp_path: Path) -> None:
|
||||
root = tmp_path / "_posts" / "blog"
|
||||
_write_post(
|
||||
root,
|
||||
"2014/2014-08-04-foo.md",
|
||||
body="---\ntitle: Foo\ndate: 2014-08-04\n---\n",
|
||||
)
|
||||
|
||||
paths = list(iter_anniversary_paths(root, SITE_URL, today=date(2024, 8, 4)))
|
||||
assert paths == ["2014/2014-08-04-foo.md"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"today, source_iso, expected_match",
|
||||
[
|
||||
# Normal exact match (Mar 1 source, Mar 1 today) -> match
|
||||
(date(2025, 3, 1), "2015-03-01", True),
|
||||
# Mar 1 source (2016, the day after Feb 29) in a non-leap current year -> no match
|
||||
(date(2025, 3, 1), "2016-03-01", False),
|
||||
# Feb 29 source in a non-leap current year, today is Mar 1: source year
|
||||
# (2016) does not equal target_year (2015), so it does NOT match under
|
||||
# the strict year-equality rule.
|
||||
(date(2025, 3, 1), "2016-02-29", False),
|
||||
# Feb 29 source (2016) in a leap current year (2028), today is Feb 29:
|
||||
# target_year (2018) is not a leap year, but the Feb 29 -> Feb 29
|
||||
# leap-year branch accepts any prior Feb 29 post.
|
||||
(date(2028, 2, 29), "2016-02-29", True),
|
||||
# Feb 29 source (2020) in a leap current year (2028), today is Feb 29:
|
||||
# same branch, also matches.
|
||||
(date(2028, 2, 29), "2020-02-29", True),
|
||||
],
|
||||
)
|
||||
def test_leap_day_matching(
|
||||
tmp_path: Path, today: date, source_iso: str, expected_match: bool
|
||||
) -> None:
|
||||
root = tmp_path / "_posts" / "blog"
|
||||
target_year = today.year - 10
|
||||
post_year = int(source_iso.split("-", 1)[0])
|
||||
rel = f"{post_year}/{source_iso}-leap.md"
|
||||
_write_post(
|
||||
root,
|
||||
rel,
|
||||
body=f"---\ntitle: Leap\ndate: {source_iso}\n---\n",
|
||||
)
|
||||
|
||||
matches = find_anniversary_matches(root, SITE_URL, today=today)
|
||||
matched_paths = [m.path for m in matches]
|
||||
if expected_match:
|
||||
assert rel in matched_paths
|
||||
else:
|
||||
assert rel not in matched_paths
|
||||
|
||||
|
||||
def test_leap_day_feb29_matches_with_canonical_url(tmp_path: Path) -> None:
|
||||
root = tmp_path / "_posts" / "blog"
|
||||
rel = "2016/2016-02-29-ten-years-before-2026.md"
|
||||
_write_post(
|
||||
root,
|
||||
rel,
|
||||
body=(
|
||||
"---\n"
|
||||
"title: Would Be Ten Years Ago Today In 2026 Mar 1\n"
|
||||
"date: 2016-02-29 08:00:00\n"
|
||||
"---\n"
|
||||
"body\n"
|
||||
),
|
||||
)
|
||||
|
||||
today = date(2028, 2, 29)
|
||||
matches = find_anniversary_matches(root, SITE_URL, today=today)
|
||||
|
||||
assert len(matches) == 1
|
||||
match = matches[0]
|
||||
assert match.path == rel
|
||||
assert match.date == date(2016, 2, 29)
|
||||
assert match.title == "Would Be Ten Years Ago Today In 2026 Mar 1"
|
||||
assert match.url == "https://chaospott.de/2016/02/29/would-be-ten-years-ago-today-in-2026-mar-1/"
|
||||
|
||||
|
||||
def test_match_uses_datetime_frontmatter_date(tmp_path: Path) -> None:
|
||||
root = tmp_path / "_posts" / "blog"
|
||||
_write_post(
|
||||
root,
|
||||
"2014/2014-08-04-with-time.md",
|
||||
body="---\ntitle: WithTime\ndate: 2014-08-04T12:30:00\n---\n",
|
||||
)
|
||||
|
||||
today = date(2024, 8, 4)
|
||||
matches = find_anniversary_matches(root, SITE_URL, today=today)
|
||||
assert len(matches) == 1
|
||||
assert matches[0].date == date(2014, 8, 4)
|
||||
|
||||
|
||||
def test_match_returns_results_in_deterministic_order(tmp_path: Path) -> None:
|
||||
root = tmp_path / "_posts" / "blog"
|
||||
for slug, day in [("alpha", 1), ("beta", 2), ("gamma", 3)]:
|
||||
_write_post(
|
||||
root,
|
||||
f"2014/2014-08-0{day}-{slug}.md",
|
||||
body=f"---\ntitle: {slug}\ndate: 2014-08-0{day}\n---\n",
|
||||
)
|
||||
|
||||
today = date(2024, 8, 2)
|
||||
matches = find_anniversary_matches(root, SITE_URL, today=today)
|
||||
assert [m.path for m in matches] == ["2014/2014-08-02-beta.md"]
|
||||
|
||||
|
||||
def test_missing_post_root_returns_empty(tmp_path: Path) -> None:
|
||||
today = date(2024, 8, 4)
|
||||
assert find_anniversary_matches(tmp_path / "does-not-exist", SITE_URL, today=today) == []
|
||||
|
||||
|
||||
def test_invalid_filename_is_skipped(tmp_path: Path) -> None:
|
||||
root = tmp_path / "_posts" / "blog"
|
||||
_write_post(root, "2014/not-a-date.md", body="---\ntitle: Bad\n---\n")
|
||||
|
||||
today = date(2024, 8, 4)
|
||||
matches: list[MatchedPost] = []
|
||||
records = _capture_matching_warnings(
|
||||
lambda: matches.extend(find_anniversary_matches(root, SITE_URL, today=today))
|
||||
)
|
||||
|
||||
assert matches == []
|
||||
assert "post_filename_invalid" in _event_names(records)
|
||||
@@ -0,0 +1,219 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
import pytest
|
||||
|
||||
from tenbackward.config import load_config
|
||||
from tenbackward.matching import MatchedPost
|
||||
from tenbackward.publishing import (
|
||||
MASTODON_STATUS_LIMIT,
|
||||
PublishError,
|
||||
build_status_text,
|
||||
publish_mastodon,
|
||||
slugify_title,
|
||||
validate_status,
|
||||
)
|
||||
from tenbackward.state import PostedStore
|
||||
|
||||
|
||||
def _make_match(path: str, title: str, day: int = 4) -> MatchedPost:
|
||||
return MatchedPost(
|
||||
path=path,
|
||||
title=title,
|
||||
date=date(2016, 8, day),
|
||||
url=f"https://blog.example.com/2016/08/{day:02d}/{path.split('-', 3)[-1].removesuffix('.md')}/",
|
||||
)
|
||||
|
||||
|
||||
class _FakeMastodon:
|
||||
"""Captures ``status_post`` calls and can be configured to raise."""
|
||||
|
||||
def __init__(self, *, raise_on_post: Exception | None = None) -> None:
|
||||
self.raise_on_post = raise_on_post
|
||||
self.calls: list[tuple[str, dict]] = []
|
||||
|
||||
def __call__(self, *, access_token: str, api_base_url: str) -> "_FakeMastodon":
|
||||
self.access_token = access_token
|
||||
self.api_base_url = api_base_url
|
||||
return self
|
||||
|
||||
def status_post(self, status: str, **kwargs) -> None:
|
||||
self.calls.append((status, kwargs))
|
||||
if self.raise_on_post is not None:
|
||||
raise self.raise_on_post
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def full_config(env_setup, tmp_path: Path):
|
||||
os.environ["DATA_DIR"] = str(tmp_path)
|
||||
return load_config()
|
||||
|
||||
|
||||
def test_slugify_title_basic_ascii() -> None:
|
||||
assert slugify_title("Hello, World!") == "hello-world"
|
||||
|
||||
|
||||
def test_slugify_title_collapses_repeats_and_trims() -> None:
|
||||
assert slugify_title("!!!Foo---Bar???") == "foo-bar"
|
||||
assert slugify_title("---trim---me---") == "trim-me"
|
||||
|
||||
|
||||
def test_build_status_text_single_post_layout() -> None:
|
||||
match = _make_match("2016/2016-08-04-foo.md", "Foo")
|
||||
status = build_status_text(
|
||||
"Heute vor 10 Jahren:", [match], "#throwback,#10backward"
|
||||
)
|
||||
assert status == (
|
||||
"Heute vor 10 Jahren:\n\n"
|
||||
"Foo\nhttps://blog.example.com/2016/08/04/foo/\n\n"
|
||||
"#throwback #10backward\n"
|
||||
)
|
||||
|
||||
|
||||
def test_build_status_text_multiple_posts_combined() -> None:
|
||||
match_a = _make_match("2016/2016-08-04-a.md", "Alpha")
|
||||
match_b = _make_match("2016/2016-08-04-b.md", "Bravo")
|
||||
status = build_status_text("P:", [match_a, match_b], "#x")
|
||||
assert "Alpha" in status
|
||||
assert "Bravo" in status
|
||||
assert match_a.url in status
|
||||
assert match_b.url in status
|
||||
assert status.count("Alpha") == 1
|
||||
assert status.count("Bravo") == 1
|
||||
assert status.count(match_a.url) == 1
|
||||
assert status.count(match_b.url) == 1
|
||||
|
||||
|
||||
def test_build_status_text_empty_posts_raises() -> None:
|
||||
with pytest.raises(PublishError):
|
||||
build_status_text("P:", [], "#x")
|
||||
|
||||
|
||||
def test_build_status_text_preserves_deterministic_order() -> None:
|
||||
match_a = _make_match("2016/2016-08-05-a.md", "Alpha", day=5)
|
||||
match_b = _make_match("2016/2016-08-04-b.md", "Bravo", day=4)
|
||||
forward = build_status_text("P:", [match_a, match_b], "#x")
|
||||
reverse = build_status_text("P:", [match_b, match_a], "#x")
|
||||
assert forward == reverse
|
||||
assert forward.index("Bravo") < forward.index("Alpha")
|
||||
|
||||
|
||||
def test_validate_status_within_limit() -> None:
|
||||
validate_status("a" * 480)
|
||||
|
||||
|
||||
def test_validate_status_over_limit_raises_and_does_not_truncate() -> None:
|
||||
long = "a" * (MASTODON_STATUS_LIMIT + 1)
|
||||
with pytest.raises(PublishError):
|
||||
validate_status(long)
|
||||
assert len(long) == MASTODON_STATUS_LIMIT + 1
|
||||
|
||||
|
||||
def test_publish_mastodon_success_invokes_client_with_composed_status(
|
||||
full_config,
|
||||
) -> None:
|
||||
match = _make_match("2016/2016-08-04-foo.md", "Foo")
|
||||
fake = _FakeMastodon()
|
||||
|
||||
result = publish_mastodon(full_config, [match], client_factory=fake)
|
||||
|
||||
assert result.endswith("\n")
|
||||
assert fake.calls, "status_post must be invoked"
|
||||
posted_status, posted_kwargs = fake.calls[0]
|
||||
assert posted_status == result
|
||||
assert posted_kwargs == {"visibility": "public"}
|
||||
assert fake.api_base_url == "https://mastodon.example"
|
||||
assert fake.access_token == "test-token"
|
||||
|
||||
|
||||
def test_publish_mastodon_missing_token_is_fatal(full_config) -> None:
|
||||
from tenbackward.publishing import PublishFatalError
|
||||
|
||||
match = _make_match("2016/2016-08-04-foo.md", "Foo")
|
||||
fake = _FakeMastodon()
|
||||
bad_config = type(full_config)(
|
||||
mastodon_base_url=full_config.mastodon_base_url,
|
||||
mastodon_access_token="",
|
||||
mastodon_visibility=full_config.mastodon_visibility,
|
||||
site_url=full_config.site_url,
|
||||
hashtags=full_config.hashtags,
|
||||
throwback_prefix=full_config.throwback_prefix,
|
||||
max_retries=full_config.max_retries,
|
||||
run_at=full_config.run_at,
|
||||
tz=full_config.tz,
|
||||
data_dir=full_config.data_dir,
|
||||
blog_repo_url=full_config.blog_repo_url,
|
||||
blog_dir=full_config.blog_dir,
|
||||
extra=dict(full_config.extra),
|
||||
)
|
||||
|
||||
with pytest.raises(PublishFatalError):
|
||||
publish_mastodon(bad_config, [match], client_factory=fake)
|
||||
|
||||
assert fake.calls == []
|
||||
|
||||
|
||||
def test_publish_mastodon_api_failure_raises_and_does_not_persist(
|
||||
full_config, tmp_path: Path
|
||||
) -> None:
|
||||
match = _make_match("2016/2016-08-04-foo.md", "Foo")
|
||||
fake = _FakeMastodon(raise_on_post=RuntimeError("boom"))
|
||||
|
||||
with pytest.raises(PublishError) as exc_info:
|
||||
publish_mastodon(full_config, [match], client_factory=fake)
|
||||
|
||||
assert "publish_failed" in str(exc_info.value)
|
||||
assert "mastodon post" in str(exc_info.value)
|
||||
assert isinstance(exc_info.value.__cause__, RuntimeError)
|
||||
|
||||
store = PostedStore(tmp_path)
|
||||
assert not store.is_posted(match.path)
|
||||
|
||||
|
||||
def test_publish_mastodon_over_limit_raises_before_api_call(
|
||||
full_config,
|
||||
) -> None:
|
||||
long_title = "T" * (MASTODON_STATUS_LIMIT + 1)
|
||||
match = MatchedPost(
|
||||
path="2016/2016-08-04-foo.md",
|
||||
title=long_title,
|
||||
date=date(2016, 8, 4),
|
||||
url="https://blog.example.com/2016/08/04/foo/",
|
||||
)
|
||||
fake = _FakeMastodon()
|
||||
|
||||
with pytest.raises(PublishError):
|
||||
publish_mastodon(full_config, [match], client_factory=fake)
|
||||
|
||||
assert fake.calls == []
|
||||
|
||||
|
||||
def test_publish_mastodon_combined_status_for_multiple_matches(full_config) -> None:
|
||||
match_a = _make_match("2016/2016-08-04-a.md", "Alpha")
|
||||
match_b = _make_match("2016/2016-08-04-b.md", "Bravo")
|
||||
fake = _FakeMastodon()
|
||||
|
||||
publish_mastodon(full_config, [match_a, match_b], client_factory=fake)
|
||||
|
||||
assert len(fake.calls) == 1
|
||||
status, _ = fake.calls[0]
|
||||
assert "Alpha" in status
|
||||
assert "Bravo" in status
|
||||
assert match_a.url in status
|
||||
assert match_b.url in status
|
||||
|
||||
|
||||
def test_publish_mastodon_uses_config_visibility(env_setup, tmp_path: Path) -> None:
|
||||
os.environ["MASTODON_VISIBILITY"] = "unlisted"
|
||||
os.environ["DATA_DIR"] = str(tmp_path)
|
||||
config = load_config()
|
||||
match = _make_match("2016/2016-08-04-foo.md", "Foo")
|
||||
fake = _FakeMastodon()
|
||||
|
||||
publish_mastodon(config, [match], client_factory=fake)
|
||||
|
||||
assert fake.calls[0][1] == {"visibility": "unlisted"}
|
||||
@@ -0,0 +1,171 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from datetime import date
|
||||
|
||||
from tenbackward import main as main_module
|
||||
from tenbackward.logging_setup import JsonFormatter
|
||||
from tenbackward.main import main
|
||||
from tenbackward.matching import MatchedPost
|
||||
from tenbackward.publishing import PublishError
|
||||
|
||||
|
||||
def _match(path: str) -> MatchedPost:
|
||||
return MatchedPost(
|
||||
path=path,
|
||||
title=path,
|
||||
date=date(2016, 8, 4),
|
||||
url=f"https://blog.example.com/2016/08/04/{path.split('-', 3)[-1].removesuffix('.md')}/",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def capture_logger() -> io.StringIO:
|
||||
buf = io.StringIO()
|
||||
handler = logging.StreamHandler(buf)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
target = logging.getLogger("tenbackward")
|
||||
target.handlers = [handler]
|
||||
target.setLevel(logging.INFO)
|
||||
target.propagate = False
|
||||
return buf
|
||||
|
||||
|
||||
def _run_lines(buf: io.StringIO) -> list[dict]:
|
||||
return [json.loads(line) for line in buf.getvalue().splitlines() if line.strip()]
|
||||
|
||||
|
||||
def _seed_state(data_dir: Path, ids: list[str]) -> None:
|
||||
from tenbackward.state import mark_posted_many
|
||||
|
||||
mark_posted_many(list(ids), data_dir)
|
||||
|
||||
|
||||
def test_run_emits_one_info_summary_on_success(env_setup, data_dir, capture_logger, monkeypatch) -> None:
|
||||
monkeypatch.setenv("DATA_DIR", str(data_dir))
|
||||
monkeypatch.setattr(main_module, "ensure_repo", lambda *a, **kw: None)
|
||||
|
||||
def _stub_publish(config, posts, **_kwargs):
|
||||
return "stubbed"
|
||||
|
||||
monkeypatch.setattr(main_module, "_iter_candidates", lambda config: [_match("new-1"), _match("already-1")])
|
||||
monkeypatch.setattr(main_module, "publish_mastodon", _stub_publish)
|
||||
|
||||
_seed_state(data_dir, ["already-1"])
|
||||
|
||||
rc = main()
|
||||
assert rc == 0
|
||||
|
||||
lines = _run_lines(capture_logger)
|
||||
summary = [line for line in lines if line.get("event") == "run_complete"]
|
||||
assert len(summary) == 1
|
||||
payload = summary[0]
|
||||
assert payload["scanned"] == 2
|
||||
assert payload["matched"] == 2
|
||||
assert payload["posted"] == 1
|
||||
assert payload["skipped"] == 1
|
||||
assert payload["posted_ids"] == ["new-1"]
|
||||
|
||||
from tenbackward.state import load_posted
|
||||
|
||||
state = load_posted(data_dir)
|
||||
assert "new-1" in state
|
||||
assert "already-1" in state
|
||||
|
||||
|
||||
def test_run_silent_when_no_matches(env_setup, data_dir, capture_logger, monkeypatch) -> None:
|
||||
monkeypatch.setenv("DATA_DIR", str(data_dir))
|
||||
monkeypatch.setattr(main_module, "ensure_repo", lambda *a, **kw: None)
|
||||
monkeypatch.setattr(main_module, "_iter_candidates", lambda config: [])
|
||||
|
||||
rc = main()
|
||||
assert rc == 0
|
||||
|
||||
lines = _run_lines(capture_logger)
|
||||
summary = [line for line in lines if line.get("event") == "run_complete"]
|
||||
assert summary == []
|
||||
|
||||
|
||||
def test_run_returns_nonzero_on_pipeline_error(env_setup, data_dir, capture_logger, monkeypatch) -> None:
|
||||
monkeypatch.setenv("DATA_DIR", str(data_dir))
|
||||
monkeypatch.setenv("MAX_RETRIES", "1")
|
||||
monkeypatch.setattr(main_module, "ensure_repo", lambda *a, **kw: None)
|
||||
|
||||
def _boom(config):
|
||||
raise RuntimeError("boom-token-should-not-appear")
|
||||
|
||||
monkeypatch.setattr(main_module, "_iter_candidates", _boom)
|
||||
|
||||
rc = main()
|
||||
assert rc == 1
|
||||
|
||||
lines = _run_lines(capture_logger)
|
||||
summary = [line for line in lines if line.get("event") == "run_complete"]
|
||||
assert summary == []
|
||||
|
||||
error_lines = [line for line in lines if line.get("level") == "ERROR"]
|
||||
assert any(line.get("event") == "retry_exhausted" for line in error_lines)
|
||||
exhausted = [line for line in error_lines if line.get("event") == "retry_exhausted"][0]
|
||||
assert exhausted.get("operation") == "pipeline"
|
||||
assert "boom-token-should-not-appear" in exhausted.get("error", "")
|
||||
|
||||
|
||||
def test_run_distinguishes_posted_from_skipped_via_ids(env_setup, data_dir, capture_logger, monkeypatch) -> None:
|
||||
monkeypatch.setenv("DATA_DIR", str(data_dir))
|
||||
monkeypatch.setattr(main_module, "ensure_repo", lambda *a, **kw: None)
|
||||
monkeypatch.setattr(
|
||||
main_module,
|
||||
"_iter_candidates",
|
||||
lambda config: [_match("alpha"), _match("beta"), _match("gamma")],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
main_module,
|
||||
"publish_mastodon",
|
||||
lambda config, posts, **_kwargs: "stubbed",
|
||||
)
|
||||
_seed_state(data_dir, ["beta"])
|
||||
|
||||
rc = main()
|
||||
assert rc == 0
|
||||
|
||||
summary = [line for line in _run_lines(capture_logger) if line.get("event") == "run_complete"]
|
||||
assert len(summary) == 1
|
||||
payload = summary[0]
|
||||
assert set(payload["posted_ids"]) == {"alpha", "gamma"}
|
||||
assert "beta" not in payload["posted_ids"]
|
||||
assert payload["skipped"] == 1
|
||||
assert payload["posted"] == 2
|
||||
|
||||
|
||||
def test_run_publish_failure_leaves_state_untouched(
|
||||
env_setup, data_dir, capture_logger, monkeypatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("DATA_DIR", str(data_dir))
|
||||
monkeypatch.setattr(main_module, "ensure_repo", lambda *a, **kw: None)
|
||||
monkeypatch.setenv("MAX_RETRIES", "0")
|
||||
|
||||
def _boom(config, posts, **_kwargs):
|
||||
raise PublishError("publish_failed: stub")
|
||||
|
||||
monkeypatch.setattr(main_module, "_iter_candidates", lambda config: [_match("alpha")])
|
||||
monkeypatch.setattr(main_module, "publish_mastodon", _boom)
|
||||
|
||||
rc = main()
|
||||
assert rc != 0
|
||||
|
||||
from tenbackward.state import load_posted
|
||||
|
||||
assert load_posted(data_dir) == []
|
||||
error_lines = [
|
||||
line
|
||||
for line in _run_lines(capture_logger)
|
||||
if line.get("level") == "ERROR"
|
||||
]
|
||||
assert any(line.get("event") == "retry_exhausted" for line in error_lines)
|
||||
@@ -0,0 +1,167 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from tenbackward.logging_setup import JsonFormatter
|
||||
from tenbackward.state import (
|
||||
PostedStore,
|
||||
is_posted,
|
||||
load_posted,
|
||||
mark_posted,
|
||||
mark_posted_many,
|
||||
posted_path,
|
||||
)
|
||||
|
||||
|
||||
def _store(tmp_path: Path) -> PostedStore:
|
||||
return PostedStore(tmp_path)
|
||||
|
||||
|
||||
def test_load_creates_empty_file_when_missing(tmp_path: Path) -> None:
|
||||
store = _store(tmp_path)
|
||||
result = store.load()
|
||||
|
||||
assert result == []
|
||||
assert posted_path(tmp_path).exists()
|
||||
document = json.loads(posted_path(tmp_path).read_text(encoding="utf-8"))
|
||||
assert document == {"posted": []}
|
||||
|
||||
|
||||
def test_load_returns_empty_when_file_missing_no_crash(tmp_path: Path) -> None:
|
||||
assert _store(tmp_path).load() == []
|
||||
|
||||
|
||||
def test_load_warns_and_returns_empty_on_malformed_json(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
buf = io.StringIO()
|
||||
handler = logging.StreamHandler(buf)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
logger = logging.getLogger("tenbackward")
|
||||
original_handlers = list(logger.handlers)
|
||||
original_propagate = logger.propagate
|
||||
logger.handlers = [handler]
|
||||
logger.setLevel(logging.WARNING)
|
||||
logger.propagate = False
|
||||
try:
|
||||
posted_path(tmp_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
posted_path(tmp_path).write_text("not-json", encoding="utf-8")
|
||||
result = _store(tmp_path).load()
|
||||
finally:
|
||||
logger.handlers = original_handlers
|
||||
logger.propagate = original_propagate
|
||||
|
||||
assert result == []
|
||||
lines = [
|
||||
json.loads(line) for line in buf.getvalue().splitlines() if line.strip()
|
||||
]
|
||||
events = [line.get("event") for line in lines]
|
||||
assert "posted_store_corrupt" in events
|
||||
|
||||
|
||||
def test_mark_posted_writes_relative_path_to_file(tmp_path: Path) -> None:
|
||||
_store(tmp_path).mark_posted("2014/2014-08-04-foo.md")
|
||||
|
||||
document = json.loads(posted_path(tmp_path).read_text(encoding="utf-8"))
|
||||
assert document == {"posted": ["2014/2014-08-04-foo.md"]}
|
||||
|
||||
|
||||
def test_is_posted_round_trip(tmp_path: Path) -> None:
|
||||
store = _store(tmp_path)
|
||||
store.mark_posted("2014/2014-08-04-foo.md")
|
||||
|
||||
assert store.is_posted("2014/2014-08-04-foo.md") is True
|
||||
assert store.is_posted("2015/2015-01-01-bar.md") is False
|
||||
|
||||
|
||||
def test_mark_posted_is_idempotent(tmp_path: Path) -> None:
|
||||
store = _store(tmp_path)
|
||||
first = store.mark_posted("2014/2014-08-04-foo.md")
|
||||
second = store.mark_posted("2014/2014-08-04-foo.md")
|
||||
|
||||
assert first is True
|
||||
assert second is False
|
||||
|
||||
document = json.loads(posted_path(tmp_path).read_text(encoding="utf-8"))
|
||||
assert document == {"posted": ["2014/2014-08-04-foo.md"]}
|
||||
|
||||
|
||||
def test_mark_posted_many_persists_in_one_write(
|
||||
tmp_path: Path, monkeypatch
|
||||
) -> None:
|
||||
store = _store(tmp_path)
|
||||
replaces: list[tuple[object, object]] = []
|
||||
real_replace = os.replace
|
||||
|
||||
def counting_replace(src, dst) -> None:
|
||||
replaces.append((src, dst))
|
||||
real_replace(src, dst)
|
||||
|
||||
monkeypatch.setattr("tenbackward.state.os.replace", counting_replace)
|
||||
|
||||
added = store.mark_posted_many(
|
||||
["2014/2014-08-04-foo.md", "2015/2015-01-01-bar.md"]
|
||||
)
|
||||
|
||||
assert added == ["2014/2014-08-04-foo.md", "2015/2015-01-01-bar.md"]
|
||||
assert len(replaces) == 1
|
||||
assert Path(replaces[0][1]) == posted_path(tmp_path)
|
||||
|
||||
|
||||
def test_mark_posted_leaves_no_temp_files(tmp_path: Path) -> None:
|
||||
_store(tmp_path).mark_posted("2014/2014-08-04-foo.md")
|
||||
|
||||
leftovers = [
|
||||
p for p in tmp_path.iterdir() if p.name.startswith("posted.json.tmp.")
|
||||
]
|
||||
assert leftovers == []
|
||||
|
||||
|
||||
def test_mark_posted_many_against_existing_is_no_op(tmp_path: Path) -> None:
|
||||
store = _store(tmp_path)
|
||||
store.mark_posted("2014/2014-08-04-foo.md")
|
||||
|
||||
added = store.mark_posted_many(
|
||||
["2014/2014-08-04-foo.md", "2015/2015-01-01-bar.md"]
|
||||
)
|
||||
|
||||
assert added == ["2015/2015-01-01-bar.md"]
|
||||
document = json.loads(posted_path(tmp_path).read_text(encoding="utf-8"))
|
||||
assert document == {
|
||||
"posted": ["2014/2014-08-04-foo.md", "2015/2015-01-01-bar.md"]
|
||||
}
|
||||
|
||||
|
||||
def test_posted_path_defaults_via_DATA_DIR(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
monkeypatch.setenv("DATA_DIR", str(tmp_path))
|
||||
assert posted_path(None) == tmp_path / "posted.json"
|
||||
|
||||
|
||||
def test_load_filters_non_string_entries(tmp_path: Path) -> None:
|
||||
posted_path(tmp_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
posted_path(tmp_path).write_text(
|
||||
json.dumps({"posted": ["good", 42, None, "also-good"]}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
assert _store(tmp_path).load() == ["good", "also-good"]
|
||||
|
||||
|
||||
def test_module_helpers_round_trip(tmp_path: Path) -> None:
|
||||
assert load_posted(tmp_path) == []
|
||||
assert is_posted("2014/2014-08-04-foo.md", tmp_path) is False
|
||||
assert mark_posted("2014/2014-08-04-foo.md", tmp_path) is True
|
||||
added = mark_posted_many(
|
||||
["2014/2014-08-04-foo.md", "2015/2015-01-01-bar.md"], tmp_path
|
||||
)
|
||||
assert added == ["2015/2015-01-01-bar.md"]
|
||||
assert load_posted(tmp_path) == [
|
||||
"2014/2014-08-04-foo.md",
|
||||
"2015/2015-01-01-bar.md",
|
||||
]
|
||||
Reference in New Issue
Block a user