feat: Environment Configuration and Logging Baseline

This commit is contained in:
OpenVelo Agent
2026-08-04 17:53:17 +00:00
parent bbad70bed1
commit a8b87179f0
13 changed files with 667 additions and 77 deletions
+3 -3
View File
@@ -2,10 +2,10 @@
MASTODON_BASE_URL=https://mastodon.example MASTODON_BASE_URL=https://mastodon.example
MASTODON_ACCESS_TOKEN=replace-me MASTODON_ACCESS_TOKEN=replace-me
MASTODON_VISIBILITY=public VISIBILITY=public
SITE_URL=https://blog.example.com SITE_URL=https://blog.example.com
HASHTAGS=#throwback,#10backward HASHTAGS=#throwback,#10backward
THROWBACK_PREFIX=Throwback: THROWNBACK_PREFIX=Throwback:
RETRY_COUNT=3 MAX_RETRIES=3
RUN_AT=09:00 RUN_AT=09:00
TZ=Europe/Berlin TZ=Europe/Berlin
+7
View File
@@ -2,6 +2,13 @@
set -euo pipefail set -euo pipefail
export DEBIAN_FRONTEND=noninteractive export DEBIAN_FRONTEND=noninteractive
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
cd "${SCRIPT_DIR}"
if [ -w /usr/local/bin ] && [ ! -e /usr/local/bin/build.sh ]; then
ln -s "${SCRIPT_DIR}/build.sh" /usr/local/bin/build.sh || true
fi
apt-get update apt-get update
apt-get install -y --no-install-recommends \ apt-get install -y --no-install-recommends \
bash \ bash \
+3 -3
View File
@@ -11,11 +11,11 @@ services:
environment: environment:
MASTODON_BASE_URL: ${MASTODON_BASE_URL:-https://mastodon.example} MASTODON_BASE_URL: ${MASTODON_BASE_URL:-https://mastodon.example}
MASTODON_ACCESS_TOKEN: ${MASTODON_ACCESS_TOKEN:-replace-me} MASTODON_ACCESS_TOKEN: ${MASTODON_ACCESS_TOKEN:-replace-me}
MASTODON_VISIBILITY: ${MASTODON_VISIBILITY:-public} VISIBILITY: ${VISIBILITY:-public}
SITE_URL: ${SITE_URL:-https://blog.example.com} SITE_URL: ${SITE_URL:-https://blog.example.com}
HASHTAGS: ${HASHTAGS:-#throwback,#10backward} HASHTAGS: ${HASHTAGS:-#throwback,#10backward}
THROWBACK_PREFIX: ${THROWBACK_PREFIX:-Throwback:} THROWNBACK_PREFIX: ${THROWNBACK_PREFIX:-Throwback:}
RETRY_COUNT: ${RETRY_COUNT:-3} MAX_RETRIES: ${MAX_RETRIES:-3}
RUN_AT: ${RUN_AT:-09:00} RUN_AT: ${RUN_AT:-09:00}
TZ: ${TZ:-Europe/Berlin} TZ: ${TZ:-Europe/Berlin}
volumes: volumes:
+4
View File
@@ -11,9 +11,13 @@ fi
required_vars=( required_vars=(
MASTODON_BASE_URL MASTODON_BASE_URL
MASTODON_ACCESS_TOKEN MASTODON_ACCESS_TOKEN
VISIBILITY
SITE_URL SITE_URL
HASHTAGS HASHTAGS
THROWNBACK_PREFIX
MAX_RETRIES
RUN_AT RUN_AT
TZ
) )
missing=() missing=()
+84 -30
View File
@@ -4,6 +4,8 @@ import os
from dataclasses import dataclass, field from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
from typing import Optional from typing import Optional
from urllib.parse import urlparse
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from dotenv import dotenv_values, load_dotenv from dotenv import dotenv_values, load_dotenv
@@ -11,27 +13,29 @@ from dotenv import dotenv_values, load_dotenv
REQUIRED_KEYS = ( REQUIRED_KEYS = (
"MASTODON_BASE_URL", "MASTODON_BASE_URL",
"MASTODON_ACCESS_TOKEN", "MASTODON_ACCESS_TOKEN",
"VISIBILITY",
"SITE_URL", "SITE_URL",
"HASHTAGS", "HASHTAGS",
"THROWNBACK_PREFIX",
"MAX_RETRIES",
"RUN_AT", "RUN_AT",
)
OPTIONAL_KEYS = (
"MASTODON_VISIBILITY",
"THROWBACK_PREFIX",
"RETRY_COUNT",
"TZ", "TZ",
) )
OPTIONAL_KEYS: tuple[str, ...] = ()
DEFAULTS = { DEFAULTS = {
"MASTODON_VISIBILITY": "public", "VISIBILITY": "public",
"THROWBACK_PREFIX": "Throwback:", "THROWNBACK_PREFIX": "Throwback:",
"RETRY_COUNT": "3", "MAX_RETRIES": "3",
"TZ": "Europe/Berlin", "TZ": "Europe/Berlin",
"RUN_AT": "09:00", "RUN_AT": "09:00",
} }
ALLOWED_VISIBILITY = frozenset({"public", "unlisted"})
class ConfigError(ValueError): class ConfigError(ValueError):
"""Raised when required configuration is missing or invalid.""" """Raised when required configuration is missing or invalid."""
@@ -40,11 +44,11 @@ class ConfigError(ValueError):
class Config: class Config:
mastodon_base_url: str mastodon_base_url: str
mastodon_access_token: str mastodon_access_token: str
mastodon_visibility: str visibility: str
site_url: str site_url: str
hashtags: str hashtags: str
throwback_prefix: str throwback_prefix: str
retry_count: int max_retries: int
run_at: str run_at: str
tz: str tz: str
data_dir: Path = field(default_factory=lambda: Path("/app/data")) data_dir: Path = field(default_factory=lambda: Path("/app/data"))
@@ -104,11 +108,11 @@ def load_config(dotenv_path: Optional[Path] = None) -> Config:
return Config( return Config(
mastodon_base_url=merged["MASTODON_BASE_URL"], mastodon_base_url=merged["MASTODON_BASE_URL"],
mastodon_access_token=merged["MASTODON_ACCESS_TOKEN"], mastodon_access_token=merged["MASTODON_ACCESS_TOKEN"],
mastodon_visibility=merged["MASTODON_VISIBILITY"], visibility=merged["VISIBILITY"],
site_url=merged["SITE_URL"], site_url=merged["SITE_URL"],
hashtags=merged["HASHTAGS"], hashtags=merged["HASHTAGS"],
throwback_prefix=merged["THROWBACK_PREFIX"], throwback_prefix=merged["THROWNBACK_PREFIX"],
retry_count=_parse_retry_count(merged["RETRY_COUNT"]), max_retries=_parse_max_retries(merged["MAX_RETRIES"]),
run_at=merged["RUN_AT"], run_at=merged["RUN_AT"],
tz=merged["TZ"], tz=merged["TZ"],
data_dir=data_dir, data_dir=data_dir,
@@ -122,25 +126,47 @@ def apply_defaults(values: dict[str, str]) -> None:
def validate_config(values: dict[str, str]) -> None: def validate_config(values: dict[str, str]) -> None:
missing = [k for k in REQUIRED_KEYS if not values.get(k)] errors: list[str] = []
if missing:
raise ConfigError( for key in REQUIRED_KEYS:
"missing required configuration key(s): " + ", ".join(missing) raw = values.get(key, "")
) if not raw:
errors.append(f"missing required configuration key: {key}")
run_at = values.get("RUN_AT", "") run_at = values.get("RUN_AT", "")
if not _is_valid_hhmm(run_at): if run_at and not _is_valid_hhmm(run_at):
raise ConfigError( errors.append(f"RUN_AT={run_at!r} must be in HH:MM (24-hour) format")
f"RUN_AT={run_at!r} must be in HH:MM (24-hour) format"
base_url = values.get("MASTODON_BASE_URL", "")
if base_url and not _is_valid_url(base_url):
errors.append(f"MASTODON_BASE_URL={base_url!r} must be a valid http(s) URL")
site_url = values.get("SITE_URL", "")
if site_url and not _is_valid_url(site_url):
errors.append(f"SITE_URL={site_url!r} must be a valid http(s) URL")
visibility = values.get("VISIBILITY", "")
if visibility and not _is_valid_visibility(visibility):
errors.append(
f"VISIBILITY={visibility!r} must be one of: {sorted(ALLOWED_VISIBILITY)}"
) )
try: tz = values.get("TZ", "")
_parse_retry_count(values.get("RETRY_COUNT", "")) if tz and not _is_valid_tz(tz):
except ConfigError as exc: errors.append(f"TZ={tz!r} must be a valid IANA timezone")
raise ConfigError(str(exc)) from exc
max_retries = values.get("MAX_RETRIES", "")
if max_retries:
try:
_parse_max_retries(max_retries)
except ConfigError as exc:
errors.append(str(exc))
if errors:
raise ConfigError("; ".join(errors))
def _is_valid_hhmm(value: str) -> bool: def _is_valid_hhmm(value) -> bool:
if not isinstance(value, str): if not isinstance(value, str):
return False return False
parts = value.split(":") parts = value.split(":")
@@ -156,11 +182,39 @@ def _is_valid_hhmm(value: str) -> bool:
return 0 <= h <= 23 and 0 <= m <= 59 return 0 <= h <= 23 and 0 <= m <= 59
def _parse_retry_count(value: str) -> int: def _is_valid_url(value: str) -> bool:
if not isinstance(value, str) or not value:
return False
try:
parsed = urlparse(value)
except (TypeError, ValueError):
return False
return parsed.scheme in {"http", "https"} and bool(parsed.netloc)
def _is_valid_visibility(value: str) -> bool:
return isinstance(value, str) and value in ALLOWED_VISIBILITY
def _is_valid_tz(value: str) -> bool:
if not isinstance(value, str) or not value:
return False
try:
ZoneInfo(value)
except ZoneInfoNotFoundError:
return False
except Exception:
return False
return True
def _parse_max_retries(value) -> int:
if isinstance(value, bool):
raise ConfigError(f"MAX_RETRIES={value!r} must be a non-negative integer")
try: try:
count = int(value) count = int(value)
except (TypeError, ValueError): except (TypeError, ValueError):
raise ConfigError(f"RETRY_COUNT={value!r} must be a positive integer") raise ConfigError(f"MAX_RETRIES={value!r} must be a non-negative integer")
if count < 0: if count < 0:
raise ConfigError(f"RETRY_COUNT={value!r} must be >= 0") raise ConfigError(f"MAX_RETRIES={value!r} must be >= 0")
return count return count
+155
View File
@@ -0,0 +1,155 @@
from __future__ import annotations
import json
import logging
from datetime import datetime, timezone
from typing import Any
from .config import ConfigError
_SECRET_KEYS = frozenset(
{
"token",
"access_token",
"mastodon_access_token",
"password",
"secret",
"authorization",
"api_key",
}
)
_LOGGER_NAME = "tenbackward"
class JsonFormatter(logging.Formatter):
"""Emit one JSON object per log record.
Reserved record attributes are mapped to top-level keys; anything passed
via ``extra=`` is merged into the same object. Keys whose name appears in
``_SECRET_KEYS`` are redacted before formatting.
"""
_RESERVED = {
"name",
"msg",
"args",
"levelname",
"levelno",
"pathname",
"filename",
"module",
"exc_info",
"exc_text",
"stack_info",
"lineno",
"funcName",
"created",
"msecs",
"relativeCreated",
"thread",
"threadName",
"processName",
"process",
"message",
"asctime",
}
def format(self, record: logging.LogRecord) -> str: # noqa: A003
message = record.getMessage()
payload: dict[str, Any] = {
"ts": datetime.fromtimestamp(record.created, tz=timezone.utc).isoformat(),
"level": record.levelname,
"logger": record.name,
"message": message,
}
for key, value in record.__dict__.items():
if key in self._RESERVED or key.startswith("_"):
continue
if key in _SECRET_KEYS:
payload[key] = "[REDACTED]"
else:
payload[key] = value
if record.exc_info:
payload["exc_type"] = record.exc_info[0].__name__ if record.exc_info[0] else None
return json.dumps(payload, default=str, sort_keys=False)
def configure_json_logging(level: int = logging.INFO) -> None:
"""Install the JSON formatter on the root logger.
Idempotent: safe to call multiple times (replaces any existing handler).
"""
handler = logging.StreamHandler()
handler.setFormatter(JsonFormatter())
handler.setLevel(level)
root = logging.getLogger()
root.setLevel(level)
for existing in list(root.handlers):
root.removeHandler(existing)
root.addHandler(handler)
def _redact_dict(fields: dict[str, Any]) -> dict[str, Any]:
cleaned: dict[str, Any] = {}
for key, value in fields.items():
if key.lower() in _SECRET_KEYS:
cleaned[key] = "[REDACTED]"
else:
cleaned[key] = value
return cleaned
def log_run_summary(
scanned: int,
matched: int,
posted: int,
skipped: int,
posted_ids: list[str],
) -> None:
"""Emit a single structured info line on successful completion.
A run with no matching posts is silent — no log line is emitted.
"""
if scanned == 0 and matched == 0 and posted == 0 and skipped == 0 and not posted_ids:
return
fields = _redact_dict(
{
"event": "run_complete",
"scanned": scanned,
"matched": matched,
"posted": posted,
"skipped": skipped,
"posted_ids": list(posted_ids),
}
)
logging.getLogger(_LOGGER_NAME).info("run complete", extra=fields)
def log_error(event: str, *, exc: BaseException | None = None, **fields: Any) -> None:
"""Emit a single structured error line, redacting any secret-shaped keys."""
payload = _redact_dict({"event": event, **fields})
if exc is not None:
payload["exc_type"] = type(exc).__name__
logging.getLogger(_LOGGER_NAME).error(event, extra=payload)
def log_startup(**fields: Any) -> None:
"""Emit a single structured info line at startup."""
payload = _redact_dict({"event": "startup", **fields})
logging.getLogger(_LOGGER_NAME).info("startup", extra=payload)
__all__ = [
"ConfigError",
"JsonFormatter",
"configure_json_logging",
"log_error",
"log_run_summary",
"log_startup",
]
+92 -23
View File
@@ -1,47 +1,116 @@
from __future__ import annotations from __future__ import annotations
import logging
import sys import sys
import time
from typing import Iterable
from . import __version__ from . import __version__
from .config import ConfigError, load_config from .config import Config, ConfigError, load_config
from .logging_setup import (
configure_json_logging,
log_error,
log_run_summary,
log_startup,
)
from .state import load_posted, save_posted from .state import load_posted, save_posted
log = logging.getLogger("tenbackward") def _iter_candidates(config: Config) -> Iterable[str]:
"""Yield candidate post identifiers.
This is the extension seam for the future blog-clone + matching
pipeline. Job 1083 leaves it empty so the silent-on-no-matches
contract is the default behaviour.
"""
return []
def _run_once(config: Config) -> tuple[int, int, int, int, list[str]]:
"""Execute one pipeline pass and return the summary counters.
Returns ``(scanned, matched, posted, skipped, posted_ids)``.
"""
already_posted = set(load_posted(config.data_dir).keys())
scanned = 0
matched = 0
posted = 0
skipped = 0
posted_ids: list[str] = []
state = load_posted(config.data_dir)
for candidate_id in _iter_candidates(config):
scanned += 1
matched += 1
if candidate_id in already_posted:
skipped += 1
continue
posted_ids.append(candidate_id)
state[candidate_id] = {"posted_at": _now_iso()}
posted += 1
if posted_ids:
save_posted(config.data_dir, state)
return scanned, matched, posted, skipped, posted_ids
def _now_iso() -> str:
from datetime import datetime, timezone
return datetime.now(tz=timezone.utc).isoformat()
def _run_with_retry(config: Config) -> tuple[int, int, int, int, list[str]] | None:
"""Execute the pipeline with retries. Returns the summary counters on
success or ``None`` when the retry budget is exhausted."""
attempts = max(1, config.max_retries + 1)
last_exc: BaseException | None = None
for attempt in range(1, attempts + 1):
try:
return _run_once(config)
except Exception as exc: # noqa: BLE001 — broad on purpose
last_exc = exc
log_error(
"pipeline_error",
exc=exc,
attempt=attempt,
max_attempts=attempts,
)
if attempt < attempts:
time.sleep(0)
assert last_exc is not None
log_error("pipeline_failed", exc=last_exc, attempts=attempts)
return None
def main() -> int: def main() -> int:
logging.basicConfig( configure_json_logging()
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
)
try: try:
config = load_config() config = load_config()
except ConfigError as exc: except ConfigError as exc:
log.error("configuration error: %s", exc) log_error("configuration_error", exc=exc)
return 2 return 2
config.data_dir.mkdir(parents=True, exist_ok=True) config.data_dir.mkdir(parents=True, exist_ok=True)
state = load_posted(config.data_dir) log_startup(
save_posted(config.data_dir, state) version=__version__,
site_url=config.site_url,
log.info( run_at=config.run_at,
"10backward v%s ready (site=%s, run_at=%s, tz=%s, hashtags=%s, prefix=%r)", tz=config.tz,
__version__, hashtags=config.hashtags,
config.site_url, throwback_prefix=config.throwback_prefix,
config.run_at,
config.tz,
config.hashtags,
config.throwback_prefix,
) )
log.warning( result = _run_with_retry(config)
"post pipeline is not yet implemented; this run only validates the scaffold. " if result is None:
"Future job will clone the blog and post a throwback via Mastodon.py." return 1
)
scanned, matched, posted, skipped, posted_ids = result
log_run_summary(scanned, matched, posted, skipped, posted_ids)
return 0 return 0
+4
View File
@@ -16,9 +16,13 @@ def env_setup(monkeypatch: pytest.MonkeyPatch) -> None:
""" """
monkeypatch.setenv("MASTODON_BASE_URL", "https://mastodon.example") monkeypatch.setenv("MASTODON_BASE_URL", "https://mastodon.example")
monkeypatch.setenv("MASTODON_ACCESS_TOKEN", "test-token") monkeypatch.setenv("MASTODON_ACCESS_TOKEN", "test-token")
monkeypatch.setenv("VISIBILITY", "public")
monkeypatch.setenv("SITE_URL", "https://blog.example.com") monkeypatch.setenv("SITE_URL", "https://blog.example.com")
monkeypatch.setenv("HASHTAGS", "#throwback,#10backward") monkeypatch.setenv("HASHTAGS", "#throwback,#10backward")
monkeypatch.setenv("THROWNBACK_PREFIX", "Throwback:")
monkeypatch.setenv("MAX_RETRIES", "3")
monkeypatch.setenv("RUN_AT", "09:00") monkeypatch.setenv("RUN_AT", "09:00")
monkeypatch.setenv("TZ", "Europe/Berlin")
@pytest.fixture() @pytest.fixture()
+96 -18
View File
@@ -9,15 +9,30 @@ from tenbackward.config import ConfigError, load_config, validate_config
def _populate(env_setup) -> dict[str, str]: def _populate(env_setup) -> dict[str, str]:
return {k: os.environ[k] for k in os.environ if k.startswith(("MASTODON_", "SITE_", "HASHTAGS", "RUN_AT", "THROWBACK_", "RETRY_", "TZ"))} return {k: os.environ[k] for k in os.environ if k.startswith(("MASTODON_", "VISIBILITY", "SITE_", "HASHTAGS", "THROWNBACK_", "MAX_RETRIES", "RUN_AT", "TZ"))}
def _full_values() -> dict[str, str]:
return {
"MASTODON_BASE_URL": "https://mastodon.example",
"MASTODON_ACCESS_TOKEN": "x",
"VISIBILITY": "public",
"SITE_URL": "https://blog.example.com",
"HASHTAGS": "#throwback",
"THROWNBACK_PREFIX": "Throwback:",
"MAX_RETRIES": "3",
"RUN_AT": "09:00",
"TZ": "Europe/Berlin",
}
def test_load_config_succeeds_with_complete_env(env_setup) -> None: def test_load_config_succeeds_with_complete_env(env_setup) -> None:
config = load_config() config = load_config()
assert config.mastodon_base_url == "https://mastodon.example" assert config.mastodon_base_url == "https://mastodon.example"
assert config.run_at == "09:00" assert config.run_at == "09:00"
assert config.retry_count == 3 assert config.max_retries == 3
assert config.throwback_prefix == "Throwback:" assert config.throwback_prefix == "Throwback:"
assert config.visibility == "public"
def test_validate_config_lists_every_missing_key(env_setup, monkeypatch) -> None: def test_validate_config_lists_every_missing_key(env_setup, monkeypatch) -> None:
@@ -27,9 +42,13 @@ def test_validate_config_lists_every_missing_key(env_setup, monkeypatch) -> None
values = {k: os.environ.get(k, "") for k in [ values = {k: os.environ.get(k, "") for k in [
"MASTODON_BASE_URL", "MASTODON_BASE_URL",
"MASTODON_ACCESS_TOKEN", "MASTODON_ACCESS_TOKEN",
"VISIBILITY",
"SITE_URL", "SITE_URL",
"HASHTAGS", "HASHTAGS",
"THROWNBACK_PREFIX",
"MAX_RETRIES",
"RUN_AT", "RUN_AT",
"TZ",
]} ]}
with pytest.raises(ConfigError) as excinfo: with pytest.raises(ConfigError) as excinfo:
@@ -41,30 +60,63 @@ def test_validate_config_lists_every_missing_key(env_setup, monkeypatch) -> None
def test_validate_config_rejects_bad_run_at() -> None: def test_validate_config_rejects_bad_run_at() -> None:
values = { values = _full_values()
"MASTODON_BASE_URL": "https://mastodon.example", values["RUN_AT"] = "25:99"
"MASTODON_ACCESS_TOKEN": "x",
"SITE_URL": "https://blog.example.com",
"HASHTAGS": "#x",
"RUN_AT": "25:99",
"MASTODON_VISIBILITY": "public",
"THROWBACK_PREFIX": "Throwback:",
"RETRY_COUNT": "3",
"TZ": "Europe/Berlin",
}
with pytest.raises(ConfigError): with pytest.raises(ConfigError):
validate_config(values) validate_config(values)
def test_validate_config_rejects_invalid_url() -> None:
values = _full_values()
values["SITE_URL"] = "not-a-url"
with pytest.raises(ConfigError, match="SITE_URL"):
validate_config(values)
values = _full_values()
values["MASTODON_BASE_URL"] = "ftp://mastodon.example"
with pytest.raises(ConfigError, match="MASTODON_BASE_URL"):
validate_config(values)
def test_validate_config_rejects_invalid_visibility() -> None:
for bad in ("private", "direct", "", "PUBLIC"):
values = _full_values()
values["VISIBILITY"] = bad
with pytest.raises(ConfigError, match="VISIBILITY"):
validate_config(values)
def test_validate_config_rejects_invalid_tz() -> None:
values = _full_values()
values["TZ"] = "Not/AZone"
with pytest.raises(ConfigError, match="TZ"):
validate_config(values)
def test_validate_config_rejects_negative_max_retries() -> None:
values = _full_values()
values["MAX_RETRIES"] = "-1"
with pytest.raises(ConfigError, match="MAX_RETRIES"):
validate_config(values)
def test_validate_config_accepts_zero_max_retries() -> None:
values = _full_values()
values["MAX_RETRIES"] = "0"
validate_config(values)
def test_load_config_applies_optional_defaults(env_setup, monkeypatch) -> None: def test_load_config_applies_optional_defaults(env_setup, monkeypatch) -> None:
monkeypatch.delenv("THROWBACK_PREFIX", raising=False) monkeypatch.delenv("VISIBILITY", raising=False)
monkeypatch.delenv("RETRY_COUNT", raising=False) monkeypatch.delenv("THROWNBACK_PREFIX", raising=False)
monkeypatch.delenv("MAX_RETRIES", raising=False)
monkeypatch.delenv("TZ", raising=False)
config = load_config() config = load_config()
assert config.throwback_prefix == "Throwback:" assert config.throwback_prefix == "Throwback:"
assert config.retry_count == 3 assert config.max_retries == 3
assert config.tz == "Europe/Berlin" assert config.tz == "Europe/Berlin"
assert config.mastodon_visibility == "public" assert config.visibility == "public"
def test_load_config_reads_dotenv_file(tmp_path: Path, monkeypatch) -> None: def test_load_config_reads_dotenv_file(tmp_path: Path, monkeypatch) -> None:
@@ -72,18 +124,35 @@ def test_load_config_reads_dotenv_file(tmp_path: Path, monkeypatch) -> None:
dotenv.write_text( dotenv.write_text(
"MASTODON_BASE_URL=https://from-file.example\n" "MASTODON_BASE_URL=https://from-file.example\n"
"MASTODON_ACCESS_TOKEN=file-token\n" "MASTODON_ACCESS_TOKEN=file-token\n"
"VISIBILITY=unlisted\n"
"SITE_URL=https://blog.example.com\n" "SITE_URL=https://blog.example.com\n"
"HASHTAGS=#throwback\n" "HASHTAGS=#throwback\n"
"THROWNBACK_PREFIX=Werferückblick:\n"
"MAX_RETRIES=5\n"
"RUN_AT=12:34\n" "RUN_AT=12:34\n"
"TZ=Europe/Berlin\n"
) )
for key in ("MASTODON_BASE_URL", "MASTODON_ACCESS_TOKEN", "SITE_URL", "HASHTAGS", "RUN_AT"): for key in (
"MASTODON_BASE_URL",
"MASTODON_ACCESS_TOKEN",
"VISIBILITY",
"SITE_URL",
"HASHTAGS",
"THROWNBACK_PREFIX",
"MAX_RETRIES",
"RUN_AT",
"TZ",
):
monkeypatch.delenv(key, raising=False) monkeypatch.delenv(key, raising=False)
config = load_config(dotenv_path=dotenv) config = load_config(dotenv_path=dotenv)
assert config.mastodon_base_url == "https://from-file.example" assert config.mastodon_base_url == "https://from-file.example"
assert config.mastodon_access_token == "file-token" assert config.mastodon_access_token == "file-token"
assert config.run_at == "12:34" assert config.run_at == "12:34"
assert config.visibility == "unlisted"
assert config.throwback_prefix == "Werferückblick:"
assert config.max_retries == 5
def test_env_overrides_dotenv(tmp_path: Path, monkeypatch) -> None: def test_env_overrides_dotenv(tmp_path: Path, monkeypatch) -> None:
@@ -94,10 +163,19 @@ def test_env_overrides_dotenv(tmp_path: Path, monkeypatch) -> None:
for key in ( for key in (
"MASTODON_BASE_URL", "MASTODON_BASE_URL",
"MASTODON_ACCESS_TOKEN", "MASTODON_ACCESS_TOKEN",
"VISIBILITY",
"SITE_URL", "SITE_URL",
"HASHTAGS", "HASHTAGS",
"THROWNBACK_PREFIX",
"MAX_RETRIES",
"TZ",
): ):
monkeypatch.setenv(key, "x") monkeypatch.setenv(key, "x")
monkeypatch.setenv("MASTODON_BASE_URL", "https://mastodon.example")
monkeypatch.setenv("SITE_URL", "https://blog.example.com")
monkeypatch.setenv("VISIBILITY", "public")
monkeypatch.setenv("TZ", "Europe/Berlin")
monkeypatch.setenv("MAX_RETRIES", "3")
config = load_config(dotenv_path=dotenv) config = load_config(dotenv_path=dotenv)
assert config.run_at == "23:00" assert config.run_at == "23:00"
+3
View File
@@ -103,8 +103,11 @@ def test_entrypoint_renders_run_at_into_cron() -> None:
env = _entrypoint_env({ env = _entrypoint_env({
"MASTODON_BASE_URL": "https://mastodon.example", "MASTODON_BASE_URL": "https://mastodon.example",
"MASTODON_ACCESS_TOKEN": "x", "MASTODON_ACCESS_TOKEN": "x",
"VISIBILITY": "public",
"SITE_URL": "https://blog.example.com", "SITE_URL": "https://blog.example.com",
"HASHTAGS": "#throwback", "HASHTAGS": "#throwback",
"THROWNBACK_PREFIX": "Throwback:",
"MAX_RETRIES": "3",
"RUN_AT": "09:00", "RUN_AT": "09:00",
"TZ": "Europe/Berlin", "TZ": "Europe/Berlin",
}) })
+6
View File
@@ -39,8 +39,11 @@ def test_entrypoint_rejects_bad_run_at(tmp_path: Path, monkeypatch) -> None:
env = _clean_env() env = _clean_env()
env["MASTODON_BASE_URL"] = "https://mastodon.example" env["MASTODON_BASE_URL"] = "https://mastodon.example"
env["MASTODON_ACCESS_TOKEN"] = "x" env["MASTODON_ACCESS_TOKEN"] = "x"
env["VISIBILITY"] = "public"
env["SITE_URL"] = "https://blog.example.com" env["SITE_URL"] = "https://blog.example.com"
env["HASHTAGS"] = "#throwback" env["HASHTAGS"] = "#throwback"
env["THROWNBACK_PREFIX"] = "Throwback:"
env["MAX_RETRIES"] = "3"
env["RUN_AT"] = "25:99" env["RUN_AT"] = "25:99"
env["TZ"] = "Europe/Berlin" env["TZ"] = "Europe/Berlin"
@@ -61,8 +64,11 @@ def test_entrypoint_fails_fast_when_token_missing(tmp_path: Path, monkeypatch) -
"""A missing required var must abort before cron is started.""" """A missing required var must abort before cron is started."""
env = _clean_env() env = _clean_env()
env["MASTODON_BASE_URL"] = "https://mastodon.example" env["MASTODON_BASE_URL"] = "https://mastodon.example"
env["VISIBILITY"] = "public"
env["SITE_URL"] = "https://blog.example.com" env["SITE_URL"] = "https://blog.example.com"
env["HASHTAGS"] = "#throwback" env["HASHTAGS"] = "#throwback"
env["THROWNBACK_PREFIX"] = "Throwback:"
env["MAX_RETRIES"] = "3"
env["RUN_AT"] = "09:00" env["RUN_AT"] = "09:00"
env["TZ"] = "Europe/Berlin" env["TZ"] = "Europe/Berlin"
env.pop("MASTODON_ACCESS_TOKEN", None) env.pop("MASTODON_ACCESS_TOKEN", None)
+93
View File
@@ -0,0 +1,93 @@
from __future__ import annotations
import io
import json
import logging
from tenbackward.logging_setup import (
JsonFormatter,
configure_json_logging,
log_error,
log_run_summary,
log_startup,
)
def _capture(logger: logging.Logger) -> io.StringIO:
buf = io.StringIO()
handler = logging.StreamHandler(buf)
handler.setFormatter(JsonFormatter())
logger.handlers = [handler]
logger.setLevel(logging.INFO)
logger.propagate = False
return buf
def test_json_formatter_emits_valid_json() -> None:
buf = _capture(logging.getLogger("tenbackward.test"))
logging.getLogger("tenbackward.test").info("hello", extra={"foo": 1})
line = buf.getvalue().strip()
payload = json.loads(line)
assert payload["message"] == "hello"
assert payload["level"] == "INFO"
assert payload["logger"] == "tenbackward.test"
assert payload["foo"] == 1
assert "ts" in payload
def test_json_formatter_redacts_secret_keys() -> None:
buf = _capture(logging.getLogger("tenbackward.test"))
logging.getLogger("tenbackward.test").info(
"msg",
extra={"access_token": "secret-value", "site_url": "https://x"},
)
payload = json.loads(buf.getvalue().strip())
assert payload["access_token"] == "[REDACTED]"
assert payload["site_url"] == "https://x"
def test_log_run_summary_emits_one_line_with_counters() -> None:
buf = _capture(logging.getLogger("tenbackward"))
log_run_summary(3, 2, 1, 1, ["x"])
lines = [line for line in buf.getvalue().splitlines() if line.strip()]
assert len(lines) == 1
payload = json.loads(lines[0])
assert payload["message"] == "run complete"
assert payload["event"] == "run_complete"
assert payload["scanned"] == 3
assert payload["matched"] == 2
assert payload["posted"] == 1
assert payload["skipped"] == 1
assert payload["posted_ids"] == ["x"]
def test_log_run_summary_silent_when_no_matches() -> None:
buf = _capture(logging.getLogger("tenbackward"))
log_run_summary(0, 0, 0, 0, [])
assert buf.getvalue() == ""
def test_log_startup_emits_single_info_line() -> None:
buf = _capture(logging.getLogger("tenbackward"))
log_startup(version="0.1.0", site_url="https://x", run_at="09:00", tz="Europe/Berlin")
payload = json.loads(buf.getvalue().strip())
assert payload["event"] == "startup"
assert payload["version"] == "0.1.0"
def test_log_error_redacts_secret_keys_and_exposes_exc_type() -> None:
buf = _capture(logging.getLogger("tenbackward"))
log_error("boom", exc=RuntimeError("x"), access_token="abc")
payload = json.loads(buf.getvalue().strip())
assert payload["level"] == "ERROR"
assert payload["event"] == "boom"
assert payload["access_token"] == "[REDACTED]"
assert payload["exc_type"] == "RuntimeError"
def test_configure_json_logging_is_idempotent() -> None:
configure_json_logging()
configure_json_logging()
root = logging.getLogger()
json_handlers = [h for h in root.handlers if isinstance(h.formatter, JsonFormatter)]
assert len(json_handlers) == 1
+117
View File
@@ -0,0 +1,117 @@
from __future__ import annotations
import io
import json
import logging
import os
from pathlib import Path
import pytest
from tenbackward import main as main_module
from tenbackward.logging_setup import JsonFormatter
from tenbackward.main import main
@pytest.fixture()
def capture_logger() -> io.StringIO:
buf = io.StringIO()
handler = logging.StreamHandler(buf)
handler.setFormatter(JsonFormatter())
target = logging.getLogger("tenbackward")
target.handlers = [handler]
target.setLevel(logging.INFO)
target.propagate = False
return buf
def _run_lines(buf: io.StringIO) -> list[dict]:
return [json.loads(line) for line in buf.getvalue().splitlines() if line.strip()]
def _seed_state(data_dir: Path, ids: list[str]) -> None:
from tenbackward.state import save_posted
state = {pid: {"posted_at": "2024-01-01T00:00:00+00:00"} for pid in ids}
save_posted(data_dir, state)
def test_run_emits_one_info_summary_on_success(env_setup, data_dir, capture_logger, monkeypatch) -> None:
monkeypatch.setenv("DATA_DIR", str(data_dir))
monkeypatch.setattr(main_module, "_iter_candidates", lambda config: ["new-1", "already-1"])
_seed_state(data_dir, ["already-1"])
rc = main()
assert rc == 0
lines = _run_lines(capture_logger)
summary = [line for line in lines if line.get("event") == "run_complete"]
assert len(summary) == 1
payload = summary[0]
assert payload["scanned"] == 2
assert payload["matched"] == 2
assert payload["posted"] == 1
assert payload["skipped"] == 1
assert payload["posted_ids"] == ["new-1"]
from tenbackward.state import load_posted
state = load_posted(data_dir)
assert "new-1" in state
assert "already-1" in state
def test_run_silent_when_no_matches(env_setup, data_dir, capture_logger, monkeypatch) -> None:
monkeypatch.setenv("DATA_DIR", str(data_dir))
monkeypatch.setattr(main_module, "_iter_candidates", lambda config: [])
rc = main()
assert rc == 0
lines = _run_lines(capture_logger)
summary = [line for line in lines if line.get("event") == "run_complete"]
assert summary == []
def test_run_returns_nonzero_on_pipeline_error(env_setup, data_dir, capture_logger, monkeypatch) -> None:
monkeypatch.setenv("DATA_DIR", str(data_dir))
monkeypatch.setenv("MAX_RETRIES", "1")
def _boom(config):
raise RuntimeError("boom-token-should-not-appear")
monkeypatch.setattr(main_module, "_iter_candidates", _boom)
rc = main()
assert rc == 1
lines = _run_lines(capture_logger)
summary = [line for line in lines if line.get("event") == "run_complete"]
assert summary == []
error_lines = [line for line in lines if line.get("level") == "ERROR"]
assert any("exc_type" in line for line in error_lines)
assert "boom-token-should-not-appear" not in capture_logger.getvalue()
def test_run_distinguishes_posted_from_skipped_via_ids(env_setup, data_dir, capture_logger, monkeypatch) -> None:
monkeypatch.setenv("DATA_DIR", str(data_dir))
monkeypatch.setattr(
main_module,
"_iter_candidates",
lambda config: ["alpha", "beta", "gamma"],
)
_seed_state(data_dir, ["beta"])
rc = main()
assert rc == 0
summary = [line for line in _run_lines(capture_logger) if line.get("event") == "run_complete"]
assert len(summary) == 1
payload = summary[0]
assert set(payload["posted_ids"]) == {"alpha", "gamma"}
assert "beta" not in payload["posted_ids"]
assert payload["skipped"] == 1
assert payload["posted"] == 2