feat: Environment Configuration and Logging Baseline
This commit is contained in:
+3
-3
@@ -2,10 +2,10 @@
|
|||||||
|
|
||||||
MASTODON_BASE_URL=https://mastodon.example
|
MASTODON_BASE_URL=https://mastodon.example
|
||||||
MASTODON_ACCESS_TOKEN=replace-me
|
MASTODON_ACCESS_TOKEN=replace-me
|
||||||
MASTODON_VISIBILITY=public
|
VISIBILITY=public
|
||||||
SITE_URL=https://blog.example.com
|
SITE_URL=https://blog.example.com
|
||||||
HASHTAGS=#throwback,#10backward
|
HASHTAGS=#throwback,#10backward
|
||||||
THROWBACK_PREFIX=Throwback:
|
THROWNBACK_PREFIX=Throwback:
|
||||||
RETRY_COUNT=3
|
MAX_RETRIES=3
|
||||||
RUN_AT=09:00
|
RUN_AT=09:00
|
||||||
TZ=Europe/Berlin
|
TZ=Europe/Berlin
|
||||||
|
|||||||
@@ -2,6 +2,13 @@
|
|||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
export DEBIAN_FRONTEND=noninteractive
|
export DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
cd "${SCRIPT_DIR}"
|
||||||
|
|
||||||
|
if [ -w /usr/local/bin ] && [ ! -e /usr/local/bin/build.sh ]; then
|
||||||
|
ln -s "${SCRIPT_DIR}/build.sh" /usr/local/bin/build.sh || true
|
||||||
|
fi
|
||||||
|
|
||||||
apt-get update
|
apt-get update
|
||||||
apt-get install -y --no-install-recommends \
|
apt-get install -y --no-install-recommends \
|
||||||
bash \
|
bash \
|
||||||
|
|||||||
+3
-3
@@ -11,11 +11,11 @@ services:
|
|||||||
environment:
|
environment:
|
||||||
MASTODON_BASE_URL: ${MASTODON_BASE_URL:-https://mastodon.example}
|
MASTODON_BASE_URL: ${MASTODON_BASE_URL:-https://mastodon.example}
|
||||||
MASTODON_ACCESS_TOKEN: ${MASTODON_ACCESS_TOKEN:-replace-me}
|
MASTODON_ACCESS_TOKEN: ${MASTODON_ACCESS_TOKEN:-replace-me}
|
||||||
MASTODON_VISIBILITY: ${MASTODON_VISIBILITY:-public}
|
VISIBILITY: ${VISIBILITY:-public}
|
||||||
SITE_URL: ${SITE_URL:-https://blog.example.com}
|
SITE_URL: ${SITE_URL:-https://blog.example.com}
|
||||||
HASHTAGS: ${HASHTAGS:-#throwback,#10backward}
|
HASHTAGS: ${HASHTAGS:-#throwback,#10backward}
|
||||||
THROWBACK_PREFIX: ${THROWBACK_PREFIX:-Throwback:}
|
THROWNBACK_PREFIX: ${THROWNBACK_PREFIX:-Throwback:}
|
||||||
RETRY_COUNT: ${RETRY_COUNT:-3}
|
MAX_RETRIES: ${MAX_RETRIES:-3}
|
||||||
RUN_AT: ${RUN_AT:-09:00}
|
RUN_AT: ${RUN_AT:-09:00}
|
||||||
TZ: ${TZ:-Europe/Berlin}
|
TZ: ${TZ:-Europe/Berlin}
|
||||||
volumes:
|
volumes:
|
||||||
|
|||||||
@@ -11,9 +11,13 @@ fi
|
|||||||
required_vars=(
|
required_vars=(
|
||||||
MASTODON_BASE_URL
|
MASTODON_BASE_URL
|
||||||
MASTODON_ACCESS_TOKEN
|
MASTODON_ACCESS_TOKEN
|
||||||
|
VISIBILITY
|
||||||
SITE_URL
|
SITE_URL
|
||||||
HASHTAGS
|
HASHTAGS
|
||||||
|
THROWNBACK_PREFIX
|
||||||
|
MAX_RETRIES
|
||||||
RUN_AT
|
RUN_AT
|
||||||
|
TZ
|
||||||
)
|
)
|
||||||
|
|
||||||
missing=()
|
missing=()
|
||||||
|
|||||||
+82
-28
@@ -4,6 +4,8 @@ import os
|
|||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
||||||
|
|
||||||
from dotenv import dotenv_values, load_dotenv
|
from dotenv import dotenv_values, load_dotenv
|
||||||
|
|
||||||
@@ -11,27 +13,29 @@ from dotenv import dotenv_values, load_dotenv
|
|||||||
REQUIRED_KEYS = (
|
REQUIRED_KEYS = (
|
||||||
"MASTODON_BASE_URL",
|
"MASTODON_BASE_URL",
|
||||||
"MASTODON_ACCESS_TOKEN",
|
"MASTODON_ACCESS_TOKEN",
|
||||||
|
"VISIBILITY",
|
||||||
"SITE_URL",
|
"SITE_URL",
|
||||||
"HASHTAGS",
|
"HASHTAGS",
|
||||||
|
"THROWNBACK_PREFIX",
|
||||||
|
"MAX_RETRIES",
|
||||||
"RUN_AT",
|
"RUN_AT",
|
||||||
)
|
|
||||||
|
|
||||||
OPTIONAL_KEYS = (
|
|
||||||
"MASTODON_VISIBILITY",
|
|
||||||
"THROWBACK_PREFIX",
|
|
||||||
"RETRY_COUNT",
|
|
||||||
"TZ",
|
"TZ",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
OPTIONAL_KEYS: tuple[str, ...] = ()
|
||||||
|
|
||||||
DEFAULTS = {
|
DEFAULTS = {
|
||||||
"MASTODON_VISIBILITY": "public",
|
"VISIBILITY": "public",
|
||||||
"THROWBACK_PREFIX": "Throwback:",
|
"THROWNBACK_PREFIX": "Throwback:",
|
||||||
"RETRY_COUNT": "3",
|
"MAX_RETRIES": "3",
|
||||||
"TZ": "Europe/Berlin",
|
"TZ": "Europe/Berlin",
|
||||||
"RUN_AT": "09:00",
|
"RUN_AT": "09:00",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
ALLOWED_VISIBILITY = frozenset({"public", "unlisted"})
|
||||||
|
|
||||||
|
|
||||||
class ConfigError(ValueError):
|
class ConfigError(ValueError):
|
||||||
"""Raised when required configuration is missing or invalid."""
|
"""Raised when required configuration is missing or invalid."""
|
||||||
|
|
||||||
@@ -40,11 +44,11 @@ class ConfigError(ValueError):
|
|||||||
class Config:
|
class Config:
|
||||||
mastodon_base_url: str
|
mastodon_base_url: str
|
||||||
mastodon_access_token: str
|
mastodon_access_token: str
|
||||||
mastodon_visibility: str
|
visibility: str
|
||||||
site_url: str
|
site_url: str
|
||||||
hashtags: str
|
hashtags: str
|
||||||
throwback_prefix: str
|
throwback_prefix: str
|
||||||
retry_count: int
|
max_retries: int
|
||||||
run_at: str
|
run_at: str
|
||||||
tz: str
|
tz: str
|
||||||
data_dir: Path = field(default_factory=lambda: Path("/app/data"))
|
data_dir: Path = field(default_factory=lambda: Path("/app/data"))
|
||||||
@@ -104,11 +108,11 @@ def load_config(dotenv_path: Optional[Path] = None) -> Config:
|
|||||||
return Config(
|
return Config(
|
||||||
mastodon_base_url=merged["MASTODON_BASE_URL"],
|
mastodon_base_url=merged["MASTODON_BASE_URL"],
|
||||||
mastodon_access_token=merged["MASTODON_ACCESS_TOKEN"],
|
mastodon_access_token=merged["MASTODON_ACCESS_TOKEN"],
|
||||||
mastodon_visibility=merged["MASTODON_VISIBILITY"],
|
visibility=merged["VISIBILITY"],
|
||||||
site_url=merged["SITE_URL"],
|
site_url=merged["SITE_URL"],
|
||||||
hashtags=merged["HASHTAGS"],
|
hashtags=merged["HASHTAGS"],
|
||||||
throwback_prefix=merged["THROWBACK_PREFIX"],
|
throwback_prefix=merged["THROWNBACK_PREFIX"],
|
||||||
retry_count=_parse_retry_count(merged["RETRY_COUNT"]),
|
max_retries=_parse_max_retries(merged["MAX_RETRIES"]),
|
||||||
run_at=merged["RUN_AT"],
|
run_at=merged["RUN_AT"],
|
||||||
tz=merged["TZ"],
|
tz=merged["TZ"],
|
||||||
data_dir=data_dir,
|
data_dir=data_dir,
|
||||||
@@ -122,25 +126,47 @@ def apply_defaults(values: dict[str, str]) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def validate_config(values: dict[str, str]) -> None:
|
def validate_config(values: dict[str, str]) -> None:
|
||||||
missing = [k for k in REQUIRED_KEYS if not values.get(k)]
|
errors: list[str] = []
|
||||||
if missing:
|
|
||||||
raise ConfigError(
|
for key in REQUIRED_KEYS:
|
||||||
"missing required configuration key(s): " + ", ".join(missing)
|
raw = values.get(key, "")
|
||||||
)
|
if not raw:
|
||||||
|
errors.append(f"missing required configuration key: {key}")
|
||||||
|
|
||||||
run_at = values.get("RUN_AT", "")
|
run_at = values.get("RUN_AT", "")
|
||||||
if not _is_valid_hhmm(run_at):
|
if run_at and not _is_valid_hhmm(run_at):
|
||||||
raise ConfigError(
|
errors.append(f"RUN_AT={run_at!r} must be in HH:MM (24-hour) format")
|
||||||
f"RUN_AT={run_at!r} must be in HH:MM (24-hour) format"
|
|
||||||
|
base_url = values.get("MASTODON_BASE_URL", "")
|
||||||
|
if base_url and not _is_valid_url(base_url):
|
||||||
|
errors.append(f"MASTODON_BASE_URL={base_url!r} must be a valid http(s) URL")
|
||||||
|
|
||||||
|
site_url = values.get("SITE_URL", "")
|
||||||
|
if site_url and not _is_valid_url(site_url):
|
||||||
|
errors.append(f"SITE_URL={site_url!r} must be a valid http(s) URL")
|
||||||
|
|
||||||
|
visibility = values.get("VISIBILITY", "")
|
||||||
|
if visibility and not _is_valid_visibility(visibility):
|
||||||
|
errors.append(
|
||||||
|
f"VISIBILITY={visibility!r} must be one of: {sorted(ALLOWED_VISIBILITY)}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
tz = values.get("TZ", "")
|
||||||
|
if tz and not _is_valid_tz(tz):
|
||||||
|
errors.append(f"TZ={tz!r} must be a valid IANA timezone")
|
||||||
|
|
||||||
|
max_retries = values.get("MAX_RETRIES", "")
|
||||||
|
if max_retries:
|
||||||
try:
|
try:
|
||||||
_parse_retry_count(values.get("RETRY_COUNT", ""))
|
_parse_max_retries(max_retries)
|
||||||
except ConfigError as exc:
|
except ConfigError as exc:
|
||||||
raise ConfigError(str(exc)) from exc
|
errors.append(str(exc))
|
||||||
|
|
||||||
|
if errors:
|
||||||
|
raise ConfigError("; ".join(errors))
|
||||||
|
|
||||||
|
|
||||||
def _is_valid_hhmm(value: str) -> bool:
|
def _is_valid_hhmm(value) -> bool:
|
||||||
if not isinstance(value, str):
|
if not isinstance(value, str):
|
||||||
return False
|
return False
|
||||||
parts = value.split(":")
|
parts = value.split(":")
|
||||||
@@ -156,11 +182,39 @@ def _is_valid_hhmm(value: str) -> bool:
|
|||||||
return 0 <= h <= 23 and 0 <= m <= 59
|
return 0 <= h <= 23 and 0 <= m <= 59
|
||||||
|
|
||||||
|
|
||||||
def _parse_retry_count(value: str) -> int:
|
def _is_valid_url(value: str) -> bool:
|
||||||
|
if not isinstance(value, str) or not value:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
parsed = urlparse(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return False
|
||||||
|
return parsed.scheme in {"http", "https"} and bool(parsed.netloc)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_valid_visibility(value: str) -> bool:
|
||||||
|
return isinstance(value, str) and value in ALLOWED_VISIBILITY
|
||||||
|
|
||||||
|
|
||||||
|
def _is_valid_tz(value: str) -> bool:
|
||||||
|
if not isinstance(value, str) or not value:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
ZoneInfo(value)
|
||||||
|
except ZoneInfoNotFoundError:
|
||||||
|
return False
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_max_retries(value) -> int:
|
||||||
|
if isinstance(value, bool):
|
||||||
|
raise ConfigError(f"MAX_RETRIES={value!r} must be a non-negative integer")
|
||||||
try:
|
try:
|
||||||
count = int(value)
|
count = int(value)
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
raise ConfigError(f"RETRY_COUNT={value!r} must be a positive integer")
|
raise ConfigError(f"MAX_RETRIES={value!r} must be a non-negative integer")
|
||||||
if count < 0:
|
if count < 0:
|
||||||
raise ConfigError(f"RETRY_COUNT={value!r} must be >= 0")
|
raise ConfigError(f"MAX_RETRIES={value!r} must be >= 0")
|
||||||
return count
|
return count
|
||||||
|
|||||||
@@ -0,0 +1,155 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from .config import ConfigError
|
||||||
|
|
||||||
|
_SECRET_KEYS = frozenset(
|
||||||
|
{
|
||||||
|
"token",
|
||||||
|
"access_token",
|
||||||
|
"mastodon_access_token",
|
||||||
|
"password",
|
||||||
|
"secret",
|
||||||
|
"authorization",
|
||||||
|
"api_key",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
_LOGGER_NAME = "tenbackward"
|
||||||
|
|
||||||
|
|
||||||
|
class JsonFormatter(logging.Formatter):
|
||||||
|
"""Emit one JSON object per log record.
|
||||||
|
|
||||||
|
Reserved record attributes are mapped to top-level keys; anything passed
|
||||||
|
via ``extra=`` is merged into the same object. Keys whose name appears in
|
||||||
|
``_SECRET_KEYS`` are redacted before formatting.
|
||||||
|
"""
|
||||||
|
|
||||||
|
_RESERVED = {
|
||||||
|
"name",
|
||||||
|
"msg",
|
||||||
|
"args",
|
||||||
|
"levelname",
|
||||||
|
"levelno",
|
||||||
|
"pathname",
|
||||||
|
"filename",
|
||||||
|
"module",
|
||||||
|
"exc_info",
|
||||||
|
"exc_text",
|
||||||
|
"stack_info",
|
||||||
|
"lineno",
|
||||||
|
"funcName",
|
||||||
|
"created",
|
||||||
|
"msecs",
|
||||||
|
"relativeCreated",
|
||||||
|
"thread",
|
||||||
|
"threadName",
|
||||||
|
"processName",
|
||||||
|
"process",
|
||||||
|
"message",
|
||||||
|
"asctime",
|
||||||
|
}
|
||||||
|
|
||||||
|
def format(self, record: logging.LogRecord) -> str: # noqa: A003
|
||||||
|
message = record.getMessage()
|
||||||
|
payload: dict[str, Any] = {
|
||||||
|
"ts": datetime.fromtimestamp(record.created, tz=timezone.utc).isoformat(),
|
||||||
|
"level": record.levelname,
|
||||||
|
"logger": record.name,
|
||||||
|
"message": message,
|
||||||
|
}
|
||||||
|
|
||||||
|
for key, value in record.__dict__.items():
|
||||||
|
if key in self._RESERVED or key.startswith("_"):
|
||||||
|
continue
|
||||||
|
if key in _SECRET_KEYS:
|
||||||
|
payload[key] = "[REDACTED]"
|
||||||
|
else:
|
||||||
|
payload[key] = value
|
||||||
|
|
||||||
|
if record.exc_info:
|
||||||
|
payload["exc_type"] = record.exc_info[0].__name__ if record.exc_info[0] else None
|
||||||
|
|
||||||
|
return json.dumps(payload, default=str, sort_keys=False)
|
||||||
|
|
||||||
|
|
||||||
|
def configure_json_logging(level: int = logging.INFO) -> None:
|
||||||
|
"""Install the JSON formatter on the root logger.
|
||||||
|
|
||||||
|
Idempotent: safe to call multiple times (replaces any existing handler).
|
||||||
|
"""
|
||||||
|
handler = logging.StreamHandler()
|
||||||
|
handler.setFormatter(JsonFormatter())
|
||||||
|
handler.setLevel(level)
|
||||||
|
|
||||||
|
root = logging.getLogger()
|
||||||
|
root.setLevel(level)
|
||||||
|
for existing in list(root.handlers):
|
||||||
|
root.removeHandler(existing)
|
||||||
|
root.addHandler(handler)
|
||||||
|
|
||||||
|
|
||||||
|
def _redact_dict(fields: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
cleaned: dict[str, Any] = {}
|
||||||
|
for key, value in fields.items():
|
||||||
|
if key.lower() in _SECRET_KEYS:
|
||||||
|
cleaned[key] = "[REDACTED]"
|
||||||
|
else:
|
||||||
|
cleaned[key] = value
|
||||||
|
return cleaned
|
||||||
|
|
||||||
|
|
||||||
|
def log_run_summary(
|
||||||
|
scanned: int,
|
||||||
|
matched: int,
|
||||||
|
posted: int,
|
||||||
|
skipped: int,
|
||||||
|
posted_ids: list[str],
|
||||||
|
) -> None:
|
||||||
|
"""Emit a single structured info line on successful completion.
|
||||||
|
|
||||||
|
A run with no matching posts is silent — no log line is emitted.
|
||||||
|
"""
|
||||||
|
if scanned == 0 and matched == 0 and posted == 0 and skipped == 0 and not posted_ids:
|
||||||
|
return
|
||||||
|
|
||||||
|
fields = _redact_dict(
|
||||||
|
{
|
||||||
|
"event": "run_complete",
|
||||||
|
"scanned": scanned,
|
||||||
|
"matched": matched,
|
||||||
|
"posted": posted,
|
||||||
|
"skipped": skipped,
|
||||||
|
"posted_ids": list(posted_ids),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
logging.getLogger(_LOGGER_NAME).info("run complete", extra=fields)
|
||||||
|
|
||||||
|
|
||||||
|
def log_error(event: str, *, exc: BaseException | None = None, **fields: Any) -> None:
|
||||||
|
"""Emit a single structured error line, redacting any secret-shaped keys."""
|
||||||
|
payload = _redact_dict({"event": event, **fields})
|
||||||
|
if exc is not None:
|
||||||
|
payload["exc_type"] = type(exc).__name__
|
||||||
|
logging.getLogger(_LOGGER_NAME).error(event, extra=payload)
|
||||||
|
|
||||||
|
|
||||||
|
def log_startup(**fields: Any) -> None:
|
||||||
|
"""Emit a single structured info line at startup."""
|
||||||
|
payload = _redact_dict({"event": "startup", **fields})
|
||||||
|
logging.getLogger(_LOGGER_NAME).info("startup", extra=payload)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"ConfigError",
|
||||||
|
"JsonFormatter",
|
||||||
|
"configure_json_logging",
|
||||||
|
"log_error",
|
||||||
|
"log_run_summary",
|
||||||
|
"log_startup",
|
||||||
|
]
|
||||||
+92
-23
@@ -1,47 +1,116 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
|
||||||
import sys
|
import sys
|
||||||
|
import time
|
||||||
|
from typing import Iterable
|
||||||
|
|
||||||
from . import __version__
|
from . import __version__
|
||||||
from .config import ConfigError, load_config
|
from .config import Config, ConfigError, load_config
|
||||||
|
from .logging_setup import (
|
||||||
|
configure_json_logging,
|
||||||
|
log_error,
|
||||||
|
log_run_summary,
|
||||||
|
log_startup,
|
||||||
|
)
|
||||||
from .state import load_posted, save_posted
|
from .state import load_posted, save_posted
|
||||||
|
|
||||||
|
|
||||||
log = logging.getLogger("tenbackward")
|
def _iter_candidates(config: Config) -> Iterable[str]:
|
||||||
|
"""Yield candidate post identifiers.
|
||||||
|
|
||||||
|
This is the extension seam for the future blog-clone + matching
|
||||||
|
pipeline. Job 1083 leaves it empty so the silent-on-no-matches
|
||||||
|
contract is the default behaviour.
|
||||||
|
"""
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _run_once(config: Config) -> tuple[int, int, int, int, list[str]]:
|
||||||
|
"""Execute one pipeline pass and return the summary counters.
|
||||||
|
|
||||||
|
Returns ``(scanned, matched, posted, skipped, posted_ids)``.
|
||||||
|
"""
|
||||||
|
already_posted = set(load_posted(config.data_dir).keys())
|
||||||
|
|
||||||
|
scanned = 0
|
||||||
|
matched = 0
|
||||||
|
posted = 0
|
||||||
|
skipped = 0
|
||||||
|
posted_ids: list[str] = []
|
||||||
|
|
||||||
|
state = load_posted(config.data_dir)
|
||||||
|
|
||||||
|
for candidate_id in _iter_candidates(config):
|
||||||
|
scanned += 1
|
||||||
|
matched += 1
|
||||||
|
if candidate_id in already_posted:
|
||||||
|
skipped += 1
|
||||||
|
continue
|
||||||
|
posted_ids.append(candidate_id)
|
||||||
|
state[candidate_id] = {"posted_at": _now_iso()}
|
||||||
|
posted += 1
|
||||||
|
|
||||||
|
if posted_ids:
|
||||||
|
save_posted(config.data_dir, state)
|
||||||
|
|
||||||
|
return scanned, matched, posted, skipped, posted_ids
|
||||||
|
|
||||||
|
|
||||||
|
def _now_iso() -> str:
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
return datetime.now(tz=timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def _run_with_retry(config: Config) -> tuple[int, int, int, int, list[str]] | None:
|
||||||
|
"""Execute the pipeline with retries. Returns the summary counters on
|
||||||
|
success or ``None`` when the retry budget is exhausted."""
|
||||||
|
attempts = max(1, config.max_retries + 1)
|
||||||
|
last_exc: BaseException | None = None
|
||||||
|
for attempt in range(1, attempts + 1):
|
||||||
|
try:
|
||||||
|
return _run_once(config)
|
||||||
|
except Exception as exc: # noqa: BLE001 — broad on purpose
|
||||||
|
last_exc = exc
|
||||||
|
log_error(
|
||||||
|
"pipeline_error",
|
||||||
|
exc=exc,
|
||||||
|
attempt=attempt,
|
||||||
|
max_attempts=attempts,
|
||||||
|
)
|
||||||
|
if attempt < attempts:
|
||||||
|
time.sleep(0)
|
||||||
|
assert last_exc is not None
|
||||||
|
log_error("pipeline_failed", exc=last_exc, attempts=attempts)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
logging.basicConfig(
|
configure_json_logging()
|
||||||
level=logging.INFO,
|
|
||||||
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
config = load_config()
|
config = load_config()
|
||||||
except ConfigError as exc:
|
except ConfigError as exc:
|
||||||
log.error("configuration error: %s", exc)
|
log_error("configuration_error", exc=exc)
|
||||||
return 2
|
return 2
|
||||||
|
|
||||||
config.data_dir.mkdir(parents=True, exist_ok=True)
|
config.data_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
state = load_posted(config.data_dir)
|
log_startup(
|
||||||
save_posted(config.data_dir, state)
|
version=__version__,
|
||||||
|
site_url=config.site_url,
|
||||||
log.info(
|
run_at=config.run_at,
|
||||||
"10backward v%s ready (site=%s, run_at=%s, tz=%s, hashtags=%s, prefix=%r)",
|
tz=config.tz,
|
||||||
__version__,
|
hashtags=config.hashtags,
|
||||||
config.site_url,
|
throwback_prefix=config.throwback_prefix,
|
||||||
config.run_at,
|
|
||||||
config.tz,
|
|
||||||
config.hashtags,
|
|
||||||
config.throwback_prefix,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
log.warning(
|
result = _run_with_retry(config)
|
||||||
"post pipeline is not yet implemented; this run only validates the scaffold. "
|
if result is None:
|
||||||
"Future job will clone the blog and post a throwback via Mastodon.py."
|
return 1
|
||||||
)
|
|
||||||
|
scanned, matched, posted, skipped, posted_ids = result
|
||||||
|
log_run_summary(scanned, matched, posted, skipped, posted_ids)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -16,9 +16,13 @@ def env_setup(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||||||
"""
|
"""
|
||||||
monkeypatch.setenv("MASTODON_BASE_URL", "https://mastodon.example")
|
monkeypatch.setenv("MASTODON_BASE_URL", "https://mastodon.example")
|
||||||
monkeypatch.setenv("MASTODON_ACCESS_TOKEN", "test-token")
|
monkeypatch.setenv("MASTODON_ACCESS_TOKEN", "test-token")
|
||||||
|
monkeypatch.setenv("VISIBILITY", "public")
|
||||||
monkeypatch.setenv("SITE_URL", "https://blog.example.com")
|
monkeypatch.setenv("SITE_URL", "https://blog.example.com")
|
||||||
monkeypatch.setenv("HASHTAGS", "#throwback,#10backward")
|
monkeypatch.setenv("HASHTAGS", "#throwback,#10backward")
|
||||||
|
monkeypatch.setenv("THROWNBACK_PREFIX", "Throwback:")
|
||||||
|
monkeypatch.setenv("MAX_RETRIES", "3")
|
||||||
monkeypatch.setenv("RUN_AT", "09:00")
|
monkeypatch.setenv("RUN_AT", "09:00")
|
||||||
|
monkeypatch.setenv("TZ", "Europe/Berlin")
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture()
|
@pytest.fixture()
|
||||||
|
|||||||
+96
-18
@@ -9,15 +9,30 @@ from tenbackward.config import ConfigError, load_config, validate_config
|
|||||||
|
|
||||||
|
|
||||||
def _populate(env_setup) -> dict[str, str]:
|
def _populate(env_setup) -> dict[str, str]:
|
||||||
return {k: os.environ[k] for k in os.environ if k.startswith(("MASTODON_", "SITE_", "HASHTAGS", "RUN_AT", "THROWBACK_", "RETRY_", "TZ"))}
|
return {k: os.environ[k] for k in os.environ if k.startswith(("MASTODON_", "VISIBILITY", "SITE_", "HASHTAGS", "THROWNBACK_", "MAX_RETRIES", "RUN_AT", "TZ"))}
|
||||||
|
|
||||||
|
|
||||||
|
def _full_values() -> dict[str, str]:
|
||||||
|
return {
|
||||||
|
"MASTODON_BASE_URL": "https://mastodon.example",
|
||||||
|
"MASTODON_ACCESS_TOKEN": "x",
|
||||||
|
"VISIBILITY": "public",
|
||||||
|
"SITE_URL": "https://blog.example.com",
|
||||||
|
"HASHTAGS": "#throwback",
|
||||||
|
"THROWNBACK_PREFIX": "Throwback:",
|
||||||
|
"MAX_RETRIES": "3",
|
||||||
|
"RUN_AT": "09:00",
|
||||||
|
"TZ": "Europe/Berlin",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def test_load_config_succeeds_with_complete_env(env_setup) -> None:
|
def test_load_config_succeeds_with_complete_env(env_setup) -> None:
|
||||||
config = load_config()
|
config = load_config()
|
||||||
assert config.mastodon_base_url == "https://mastodon.example"
|
assert config.mastodon_base_url == "https://mastodon.example"
|
||||||
assert config.run_at == "09:00"
|
assert config.run_at == "09:00"
|
||||||
assert config.retry_count == 3
|
assert config.max_retries == 3
|
||||||
assert config.throwback_prefix == "Throwback:"
|
assert config.throwback_prefix == "Throwback:"
|
||||||
|
assert config.visibility == "public"
|
||||||
|
|
||||||
|
|
||||||
def test_validate_config_lists_every_missing_key(env_setup, monkeypatch) -> None:
|
def test_validate_config_lists_every_missing_key(env_setup, monkeypatch) -> None:
|
||||||
@@ -27,9 +42,13 @@ def test_validate_config_lists_every_missing_key(env_setup, monkeypatch) -> None
|
|||||||
values = {k: os.environ.get(k, "") for k in [
|
values = {k: os.environ.get(k, "") for k in [
|
||||||
"MASTODON_BASE_URL",
|
"MASTODON_BASE_URL",
|
||||||
"MASTODON_ACCESS_TOKEN",
|
"MASTODON_ACCESS_TOKEN",
|
||||||
|
"VISIBILITY",
|
||||||
"SITE_URL",
|
"SITE_URL",
|
||||||
"HASHTAGS",
|
"HASHTAGS",
|
||||||
|
"THROWNBACK_PREFIX",
|
||||||
|
"MAX_RETRIES",
|
||||||
"RUN_AT",
|
"RUN_AT",
|
||||||
|
"TZ",
|
||||||
]}
|
]}
|
||||||
|
|
||||||
with pytest.raises(ConfigError) as excinfo:
|
with pytest.raises(ConfigError) as excinfo:
|
||||||
@@ -41,30 +60,63 @@ def test_validate_config_lists_every_missing_key(env_setup, monkeypatch) -> None
|
|||||||
|
|
||||||
|
|
||||||
def test_validate_config_rejects_bad_run_at() -> None:
|
def test_validate_config_rejects_bad_run_at() -> None:
|
||||||
values = {
|
values = _full_values()
|
||||||
"MASTODON_BASE_URL": "https://mastodon.example",
|
values["RUN_AT"] = "25:99"
|
||||||
"MASTODON_ACCESS_TOKEN": "x",
|
|
||||||
"SITE_URL": "https://blog.example.com",
|
|
||||||
"HASHTAGS": "#x",
|
|
||||||
"RUN_AT": "25:99",
|
|
||||||
"MASTODON_VISIBILITY": "public",
|
|
||||||
"THROWBACK_PREFIX": "Throwback:",
|
|
||||||
"RETRY_COUNT": "3",
|
|
||||||
"TZ": "Europe/Berlin",
|
|
||||||
}
|
|
||||||
with pytest.raises(ConfigError):
|
with pytest.raises(ConfigError):
|
||||||
validate_config(values)
|
validate_config(values)
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_config_rejects_invalid_url() -> None:
|
||||||
|
values = _full_values()
|
||||||
|
values["SITE_URL"] = "not-a-url"
|
||||||
|
with pytest.raises(ConfigError, match="SITE_URL"):
|
||||||
|
validate_config(values)
|
||||||
|
|
||||||
|
values = _full_values()
|
||||||
|
values["MASTODON_BASE_URL"] = "ftp://mastodon.example"
|
||||||
|
with pytest.raises(ConfigError, match="MASTODON_BASE_URL"):
|
||||||
|
validate_config(values)
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_config_rejects_invalid_visibility() -> None:
|
||||||
|
for bad in ("private", "direct", "", "PUBLIC"):
|
||||||
|
values = _full_values()
|
||||||
|
values["VISIBILITY"] = bad
|
||||||
|
with pytest.raises(ConfigError, match="VISIBILITY"):
|
||||||
|
validate_config(values)
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_config_rejects_invalid_tz() -> None:
|
||||||
|
values = _full_values()
|
||||||
|
values["TZ"] = "Not/AZone"
|
||||||
|
with pytest.raises(ConfigError, match="TZ"):
|
||||||
|
validate_config(values)
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_config_rejects_negative_max_retries() -> None:
|
||||||
|
values = _full_values()
|
||||||
|
values["MAX_RETRIES"] = "-1"
|
||||||
|
with pytest.raises(ConfigError, match="MAX_RETRIES"):
|
||||||
|
validate_config(values)
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_config_accepts_zero_max_retries() -> None:
|
||||||
|
values = _full_values()
|
||||||
|
values["MAX_RETRIES"] = "0"
|
||||||
|
validate_config(values)
|
||||||
|
|
||||||
|
|
||||||
def test_load_config_applies_optional_defaults(env_setup, monkeypatch) -> None:
|
def test_load_config_applies_optional_defaults(env_setup, monkeypatch) -> None:
|
||||||
monkeypatch.delenv("THROWBACK_PREFIX", raising=False)
|
monkeypatch.delenv("VISIBILITY", raising=False)
|
||||||
monkeypatch.delenv("RETRY_COUNT", raising=False)
|
monkeypatch.delenv("THROWNBACK_PREFIX", raising=False)
|
||||||
|
monkeypatch.delenv("MAX_RETRIES", raising=False)
|
||||||
|
monkeypatch.delenv("TZ", raising=False)
|
||||||
|
|
||||||
config = load_config()
|
config = load_config()
|
||||||
assert config.throwback_prefix == "Throwback:"
|
assert config.throwback_prefix == "Throwback:"
|
||||||
assert config.retry_count == 3
|
assert config.max_retries == 3
|
||||||
assert config.tz == "Europe/Berlin"
|
assert config.tz == "Europe/Berlin"
|
||||||
assert config.mastodon_visibility == "public"
|
assert config.visibility == "public"
|
||||||
|
|
||||||
|
|
||||||
def test_load_config_reads_dotenv_file(tmp_path: Path, monkeypatch) -> None:
|
def test_load_config_reads_dotenv_file(tmp_path: Path, monkeypatch) -> None:
|
||||||
@@ -72,18 +124,35 @@ def test_load_config_reads_dotenv_file(tmp_path: Path, monkeypatch) -> None:
|
|||||||
dotenv.write_text(
|
dotenv.write_text(
|
||||||
"MASTODON_BASE_URL=https://from-file.example\n"
|
"MASTODON_BASE_URL=https://from-file.example\n"
|
||||||
"MASTODON_ACCESS_TOKEN=file-token\n"
|
"MASTODON_ACCESS_TOKEN=file-token\n"
|
||||||
|
"VISIBILITY=unlisted\n"
|
||||||
"SITE_URL=https://blog.example.com\n"
|
"SITE_URL=https://blog.example.com\n"
|
||||||
"HASHTAGS=#throwback\n"
|
"HASHTAGS=#throwback\n"
|
||||||
|
"THROWNBACK_PREFIX=Werferückblick:\n"
|
||||||
|
"MAX_RETRIES=5\n"
|
||||||
"RUN_AT=12:34\n"
|
"RUN_AT=12:34\n"
|
||||||
|
"TZ=Europe/Berlin\n"
|
||||||
)
|
)
|
||||||
|
|
||||||
for key in ("MASTODON_BASE_URL", "MASTODON_ACCESS_TOKEN", "SITE_URL", "HASHTAGS", "RUN_AT"):
|
for key in (
|
||||||
|
"MASTODON_BASE_URL",
|
||||||
|
"MASTODON_ACCESS_TOKEN",
|
||||||
|
"VISIBILITY",
|
||||||
|
"SITE_URL",
|
||||||
|
"HASHTAGS",
|
||||||
|
"THROWNBACK_PREFIX",
|
||||||
|
"MAX_RETRIES",
|
||||||
|
"RUN_AT",
|
||||||
|
"TZ",
|
||||||
|
):
|
||||||
monkeypatch.delenv(key, raising=False)
|
monkeypatch.delenv(key, raising=False)
|
||||||
|
|
||||||
config = load_config(dotenv_path=dotenv)
|
config = load_config(dotenv_path=dotenv)
|
||||||
assert config.mastodon_base_url == "https://from-file.example"
|
assert config.mastodon_base_url == "https://from-file.example"
|
||||||
assert config.mastodon_access_token == "file-token"
|
assert config.mastodon_access_token == "file-token"
|
||||||
assert config.run_at == "12:34"
|
assert config.run_at == "12:34"
|
||||||
|
assert config.visibility == "unlisted"
|
||||||
|
assert config.throwback_prefix == "Werferückblick:"
|
||||||
|
assert config.max_retries == 5
|
||||||
|
|
||||||
|
|
||||||
def test_env_overrides_dotenv(tmp_path: Path, monkeypatch) -> None:
|
def test_env_overrides_dotenv(tmp_path: Path, monkeypatch) -> None:
|
||||||
@@ -94,10 +163,19 @@ def test_env_overrides_dotenv(tmp_path: Path, monkeypatch) -> None:
|
|||||||
for key in (
|
for key in (
|
||||||
"MASTODON_BASE_URL",
|
"MASTODON_BASE_URL",
|
||||||
"MASTODON_ACCESS_TOKEN",
|
"MASTODON_ACCESS_TOKEN",
|
||||||
|
"VISIBILITY",
|
||||||
"SITE_URL",
|
"SITE_URL",
|
||||||
"HASHTAGS",
|
"HASHTAGS",
|
||||||
|
"THROWNBACK_PREFIX",
|
||||||
|
"MAX_RETRIES",
|
||||||
|
"TZ",
|
||||||
):
|
):
|
||||||
monkeypatch.setenv(key, "x")
|
monkeypatch.setenv(key, "x")
|
||||||
|
monkeypatch.setenv("MASTODON_BASE_URL", "https://mastodon.example")
|
||||||
|
monkeypatch.setenv("SITE_URL", "https://blog.example.com")
|
||||||
|
monkeypatch.setenv("VISIBILITY", "public")
|
||||||
|
monkeypatch.setenv("TZ", "Europe/Berlin")
|
||||||
|
monkeypatch.setenv("MAX_RETRIES", "3")
|
||||||
|
|
||||||
config = load_config(dotenv_path=dotenv)
|
config = load_config(dotenv_path=dotenv)
|
||||||
assert config.run_at == "23:00"
|
assert config.run_at == "23:00"
|
||||||
|
|||||||
@@ -103,8 +103,11 @@ def test_entrypoint_renders_run_at_into_cron() -> None:
|
|||||||
env = _entrypoint_env({
|
env = _entrypoint_env({
|
||||||
"MASTODON_BASE_URL": "https://mastodon.example",
|
"MASTODON_BASE_URL": "https://mastodon.example",
|
||||||
"MASTODON_ACCESS_TOKEN": "x",
|
"MASTODON_ACCESS_TOKEN": "x",
|
||||||
|
"VISIBILITY": "public",
|
||||||
"SITE_URL": "https://blog.example.com",
|
"SITE_URL": "https://blog.example.com",
|
||||||
"HASHTAGS": "#throwback",
|
"HASHTAGS": "#throwback",
|
||||||
|
"THROWNBACK_PREFIX": "Throwback:",
|
||||||
|
"MAX_RETRIES": "3",
|
||||||
"RUN_AT": "09:00",
|
"RUN_AT": "09:00",
|
||||||
"TZ": "Europe/Berlin",
|
"TZ": "Europe/Berlin",
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -39,8 +39,11 @@ def test_entrypoint_rejects_bad_run_at(tmp_path: Path, monkeypatch) -> None:
|
|||||||
env = _clean_env()
|
env = _clean_env()
|
||||||
env["MASTODON_BASE_URL"] = "https://mastodon.example"
|
env["MASTODON_BASE_URL"] = "https://mastodon.example"
|
||||||
env["MASTODON_ACCESS_TOKEN"] = "x"
|
env["MASTODON_ACCESS_TOKEN"] = "x"
|
||||||
|
env["VISIBILITY"] = "public"
|
||||||
env["SITE_URL"] = "https://blog.example.com"
|
env["SITE_URL"] = "https://blog.example.com"
|
||||||
env["HASHTAGS"] = "#throwback"
|
env["HASHTAGS"] = "#throwback"
|
||||||
|
env["THROWNBACK_PREFIX"] = "Throwback:"
|
||||||
|
env["MAX_RETRIES"] = "3"
|
||||||
env["RUN_AT"] = "25:99"
|
env["RUN_AT"] = "25:99"
|
||||||
env["TZ"] = "Europe/Berlin"
|
env["TZ"] = "Europe/Berlin"
|
||||||
|
|
||||||
@@ -61,8 +64,11 @@ def test_entrypoint_fails_fast_when_token_missing(tmp_path: Path, monkeypatch) -
|
|||||||
"""A missing required var must abort before cron is started."""
|
"""A missing required var must abort before cron is started."""
|
||||||
env = _clean_env()
|
env = _clean_env()
|
||||||
env["MASTODON_BASE_URL"] = "https://mastodon.example"
|
env["MASTODON_BASE_URL"] = "https://mastodon.example"
|
||||||
|
env["VISIBILITY"] = "public"
|
||||||
env["SITE_URL"] = "https://blog.example.com"
|
env["SITE_URL"] = "https://blog.example.com"
|
||||||
env["HASHTAGS"] = "#throwback"
|
env["HASHTAGS"] = "#throwback"
|
||||||
|
env["THROWNBACK_PREFIX"] = "Throwback:"
|
||||||
|
env["MAX_RETRIES"] = "3"
|
||||||
env["RUN_AT"] = "09:00"
|
env["RUN_AT"] = "09:00"
|
||||||
env["TZ"] = "Europe/Berlin"
|
env["TZ"] = "Europe/Berlin"
|
||||||
env.pop("MASTODON_ACCESS_TOKEN", None)
|
env.pop("MASTODON_ACCESS_TOKEN", None)
|
||||||
|
|||||||
@@ -0,0 +1,93 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from tenbackward.logging_setup import (
|
||||||
|
JsonFormatter,
|
||||||
|
configure_json_logging,
|
||||||
|
log_error,
|
||||||
|
log_run_summary,
|
||||||
|
log_startup,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _capture(logger: logging.Logger) -> io.StringIO:
|
||||||
|
buf = io.StringIO()
|
||||||
|
handler = logging.StreamHandler(buf)
|
||||||
|
handler.setFormatter(JsonFormatter())
|
||||||
|
logger.handlers = [handler]
|
||||||
|
logger.setLevel(logging.INFO)
|
||||||
|
logger.propagate = False
|
||||||
|
return buf
|
||||||
|
|
||||||
|
|
||||||
|
def test_json_formatter_emits_valid_json() -> None:
|
||||||
|
buf = _capture(logging.getLogger("tenbackward.test"))
|
||||||
|
logging.getLogger("tenbackward.test").info("hello", extra={"foo": 1})
|
||||||
|
line = buf.getvalue().strip()
|
||||||
|
payload = json.loads(line)
|
||||||
|
assert payload["message"] == "hello"
|
||||||
|
assert payload["level"] == "INFO"
|
||||||
|
assert payload["logger"] == "tenbackward.test"
|
||||||
|
assert payload["foo"] == 1
|
||||||
|
assert "ts" in payload
|
||||||
|
|
||||||
|
|
||||||
|
def test_json_formatter_redacts_secret_keys() -> None:
|
||||||
|
buf = _capture(logging.getLogger("tenbackward.test"))
|
||||||
|
logging.getLogger("tenbackward.test").info(
|
||||||
|
"msg",
|
||||||
|
extra={"access_token": "secret-value", "site_url": "https://x"},
|
||||||
|
)
|
||||||
|
payload = json.loads(buf.getvalue().strip())
|
||||||
|
assert payload["access_token"] == "[REDACTED]"
|
||||||
|
assert payload["site_url"] == "https://x"
|
||||||
|
|
||||||
|
|
||||||
|
def test_log_run_summary_emits_one_line_with_counters() -> None:
|
||||||
|
buf = _capture(logging.getLogger("tenbackward"))
|
||||||
|
log_run_summary(3, 2, 1, 1, ["x"])
|
||||||
|
lines = [line for line in buf.getvalue().splitlines() if line.strip()]
|
||||||
|
assert len(lines) == 1
|
||||||
|
payload = json.loads(lines[0])
|
||||||
|
assert payload["message"] == "run complete"
|
||||||
|
assert payload["event"] == "run_complete"
|
||||||
|
assert payload["scanned"] == 3
|
||||||
|
assert payload["matched"] == 2
|
||||||
|
assert payload["posted"] == 1
|
||||||
|
assert payload["skipped"] == 1
|
||||||
|
assert payload["posted_ids"] == ["x"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_log_run_summary_silent_when_no_matches() -> None:
|
||||||
|
buf = _capture(logging.getLogger("tenbackward"))
|
||||||
|
log_run_summary(0, 0, 0, 0, [])
|
||||||
|
assert buf.getvalue() == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_log_startup_emits_single_info_line() -> None:
|
||||||
|
buf = _capture(logging.getLogger("tenbackward"))
|
||||||
|
log_startup(version="0.1.0", site_url="https://x", run_at="09:00", tz="Europe/Berlin")
|
||||||
|
payload = json.loads(buf.getvalue().strip())
|
||||||
|
assert payload["event"] == "startup"
|
||||||
|
assert payload["version"] == "0.1.0"
|
||||||
|
|
||||||
|
|
||||||
|
def test_log_error_redacts_secret_keys_and_exposes_exc_type() -> None:
|
||||||
|
buf = _capture(logging.getLogger("tenbackward"))
|
||||||
|
log_error("boom", exc=RuntimeError("x"), access_token="abc")
|
||||||
|
payload = json.loads(buf.getvalue().strip())
|
||||||
|
assert payload["level"] == "ERROR"
|
||||||
|
assert payload["event"] == "boom"
|
||||||
|
assert payload["access_token"] == "[REDACTED]"
|
||||||
|
assert payload["exc_type"] == "RuntimeError"
|
||||||
|
|
||||||
|
|
||||||
|
def test_configure_json_logging_is_idempotent() -> None:
|
||||||
|
configure_json_logging()
|
||||||
|
configure_json_logging()
|
||||||
|
root = logging.getLogger()
|
||||||
|
json_handlers = [h for h in root.handlers if isinstance(h.formatter, JsonFormatter)]
|
||||||
|
assert len(json_handlers) == 1
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from tenbackward import main as main_module
|
||||||
|
from tenbackward.logging_setup import JsonFormatter
|
||||||
|
from tenbackward.main import main
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def capture_logger() -> io.StringIO:
|
||||||
|
buf = io.StringIO()
|
||||||
|
handler = logging.StreamHandler(buf)
|
||||||
|
handler.setFormatter(JsonFormatter())
|
||||||
|
target = logging.getLogger("tenbackward")
|
||||||
|
target.handlers = [handler]
|
||||||
|
target.setLevel(logging.INFO)
|
||||||
|
target.propagate = False
|
||||||
|
return buf
|
||||||
|
|
||||||
|
|
||||||
|
def _run_lines(buf: io.StringIO) -> list[dict]:
|
||||||
|
return [json.loads(line) for line in buf.getvalue().splitlines() if line.strip()]
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_state(data_dir: Path, ids: list[str]) -> None:
|
||||||
|
from tenbackward.state import save_posted
|
||||||
|
|
||||||
|
state = {pid: {"posted_at": "2024-01-01T00:00:00+00:00"} for pid in ids}
|
||||||
|
save_posted(data_dir, state)
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_emits_one_info_summary_on_success(env_setup, data_dir, capture_logger, monkeypatch) -> None:
|
||||||
|
monkeypatch.setenv("DATA_DIR", str(data_dir))
|
||||||
|
|
||||||
|
monkeypatch.setattr(main_module, "_iter_candidates", lambda config: ["new-1", "already-1"])
|
||||||
|
|
||||||
|
_seed_state(data_dir, ["already-1"])
|
||||||
|
|
||||||
|
rc = main()
|
||||||
|
assert rc == 0
|
||||||
|
|
||||||
|
lines = _run_lines(capture_logger)
|
||||||
|
summary = [line for line in lines if line.get("event") == "run_complete"]
|
||||||
|
assert len(summary) == 1
|
||||||
|
payload = summary[0]
|
||||||
|
assert payload["scanned"] == 2
|
||||||
|
assert payload["matched"] == 2
|
||||||
|
assert payload["posted"] == 1
|
||||||
|
assert payload["skipped"] == 1
|
||||||
|
assert payload["posted_ids"] == ["new-1"]
|
||||||
|
|
||||||
|
from tenbackward.state import load_posted
|
||||||
|
|
||||||
|
state = load_posted(data_dir)
|
||||||
|
assert "new-1" in state
|
||||||
|
assert "already-1" in state
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_silent_when_no_matches(env_setup, data_dir, capture_logger, monkeypatch) -> None:
|
||||||
|
monkeypatch.setenv("DATA_DIR", str(data_dir))
|
||||||
|
monkeypatch.setattr(main_module, "_iter_candidates", lambda config: [])
|
||||||
|
|
||||||
|
rc = main()
|
||||||
|
assert rc == 0
|
||||||
|
|
||||||
|
lines = _run_lines(capture_logger)
|
||||||
|
summary = [line for line in lines if line.get("event") == "run_complete"]
|
||||||
|
assert summary == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_returns_nonzero_on_pipeline_error(env_setup, data_dir, capture_logger, monkeypatch) -> None:
|
||||||
|
monkeypatch.setenv("DATA_DIR", str(data_dir))
|
||||||
|
monkeypatch.setenv("MAX_RETRIES", "1")
|
||||||
|
|
||||||
|
def _boom(config):
|
||||||
|
raise RuntimeError("boom-token-should-not-appear")
|
||||||
|
|
||||||
|
monkeypatch.setattr(main_module, "_iter_candidates", _boom)
|
||||||
|
|
||||||
|
rc = main()
|
||||||
|
assert rc == 1
|
||||||
|
|
||||||
|
lines = _run_lines(capture_logger)
|
||||||
|
summary = [line for line in lines if line.get("event") == "run_complete"]
|
||||||
|
assert summary == []
|
||||||
|
|
||||||
|
error_lines = [line for line in lines if line.get("level") == "ERROR"]
|
||||||
|
assert any("exc_type" in line for line in error_lines)
|
||||||
|
assert "boom-token-should-not-appear" not in capture_logger.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_distinguishes_posted_from_skipped_via_ids(env_setup, data_dir, capture_logger, monkeypatch) -> None:
|
||||||
|
monkeypatch.setenv("DATA_DIR", str(data_dir))
|
||||||
|
monkeypatch.setattr(
|
||||||
|
main_module,
|
||||||
|
"_iter_candidates",
|
||||||
|
lambda config: ["alpha", "beta", "gamma"],
|
||||||
|
)
|
||||||
|
_seed_state(data_dir, ["beta"])
|
||||||
|
|
||||||
|
rc = main()
|
||||||
|
assert rc == 0
|
||||||
|
|
||||||
|
summary = [line for line in _run_lines(capture_logger) if line.get("event") == "run_complete"]
|
||||||
|
assert len(summary) == 1
|
||||||
|
payload = summary[0]
|
||||||
|
assert set(payload["posted_ids"]) == {"alpha", "gamma"}
|
||||||
|
assert "beta" not in payload["posted_ids"]
|
||||||
|
assert payload["skipped"] == 1
|
||||||
|
assert payload["posted"] == 2
|
||||||
Reference in New Issue
Block a user