- blocklist_repo.py: CRUD for blocklist_sources table - import_log_repo.py: add/list/get-last log entries - blocklist_service.py: source CRUD, preview, import (download/validate/ban), import_all, schedule get/set/info - blocklist_import.py: APScheduler task (hourly/daily/weekly schedule triggers) - blocklist.py router: 9 endpoints (list/create/update/delete/preview/import/ schedule-get+put/log) - blocklist.py models: ScheduleFrequency (StrEnum), ScheduleConfig, ScheduleInfo, ImportSourceResult, ImportRunResult, PreviewResponse - 59 new tests (18 repo + 19 service + 22 router); 374 total pass - ruff clean, mypy clean for Stage 10 files - types/blocklist.ts, api/blocklist.ts, hooks/useBlocklist.ts - BlocklistsPage.tsx: source management, schedule picker, import log table - Frontend tsc + ESLint clean
154 lines
4.9 KiB
Python
154 lines
4.9 KiB
Python
"""External blocklist import background task.
|
|
|
|
Registers an APScheduler job that downloads all enabled blocklist sources,
|
|
validates their entries, and applies bans via fail2ban on a configurable
|
|
schedule. The default schedule is daily at 03:00 UTC; it is stored in the
|
|
application :class:`~app.models.blocklist.ScheduleConfig` settings and can
|
|
be updated at runtime through the blocklist router.
|
|
|
|
The scheduler job ID is ``"blocklist_import"`` — using a stable id means
|
|
re-registering the job (e.g. after a schedule update) safely replaces the
|
|
existing entry without creating duplicates.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
import structlog
|
|
|
|
from app.models.blocklist import ScheduleFrequency
|
|
from app.services import blocklist_service
|
|
|
|
if TYPE_CHECKING:
|
|
from fastapi import FastAPI
|
|
|
|
log: structlog.stdlib.BoundLogger = structlog.get_logger()
|
|
|
|
#: Stable APScheduler job id so the job can be replaced without duplicates.
|
|
JOB_ID: str = "blocklist_import"
|
|
|
|
|
|
async def _run_import(app: Any) -> None:
|
|
"""APScheduler callback that imports all enabled blocklist sources.
|
|
|
|
Reads shared resources from ``app.state`` and delegates to
|
|
:func:`~app.services.blocklist_service.import_all`.
|
|
|
|
Args:
|
|
app: The :class:`fastapi.FastAPI` application instance passed via
|
|
APScheduler ``kwargs``.
|
|
"""
|
|
db = app.state.db
|
|
http_session = app.state.http_session
|
|
socket_path: str = app.state.settings.fail2ban_socket
|
|
|
|
log.info("blocklist_import_starting")
|
|
try:
|
|
result = await blocklist_service.import_all(db, http_session, socket_path)
|
|
log.info(
|
|
"blocklist_import_finished",
|
|
total_imported=result.total_imported,
|
|
total_skipped=result.total_skipped,
|
|
errors=result.errors_count,
|
|
)
|
|
except Exception:
|
|
log.exception("blocklist_import_unexpected_error")
|
|
|
|
|
|
def register(app: FastAPI) -> None:
|
|
"""Add (or replace) the blocklist import job in the application scheduler.
|
|
|
|
Reads the persisted :class:`~app.models.blocklist.ScheduleConfig` from
|
|
the database and translates it into the appropriate APScheduler trigger.
|
|
|
|
Should be called inside the lifespan handler after the scheduler and
|
|
database have been initialised.
|
|
|
|
Args:
|
|
app: The :class:`fastapi.FastAPI` application instance whose
|
|
``app.state.scheduler`` will receive the job.
|
|
"""
|
|
import asyncio # noqa: PLC0415
|
|
|
|
async def _do_register() -> None:
|
|
config = await blocklist_service.get_schedule(app.state.db)
|
|
_apply_schedule(app, config)
|
|
|
|
# APScheduler is synchronous at registration time; use asyncio to read
|
|
# the stored schedule from the DB before registering.
|
|
try:
|
|
loop = asyncio.get_event_loop()
|
|
loop.run_until_complete(_do_register())
|
|
except RuntimeError:
|
|
# If the current thread already has a running loop (uvicorn), schedule
|
|
# the registration as a coroutine.
|
|
asyncio.ensure_future(_do_register())
|
|
|
|
|
|
def reschedule(app: FastAPI) -> None:
|
|
"""Re-register the blocklist import job with the latest schedule config.
|
|
|
|
Called by the blocklist router after a schedule update so changes take
|
|
effect immediately without a server restart.
|
|
|
|
Args:
|
|
app: The :class:`fastapi.FastAPI` application instance.
|
|
"""
|
|
import asyncio # noqa: PLC0415
|
|
|
|
async def _do_reschedule() -> None:
|
|
config = await blocklist_service.get_schedule(app.state.db)
|
|
_apply_schedule(app, config)
|
|
|
|
asyncio.ensure_future(_do_reschedule())
|
|
|
|
|
|
def _apply_schedule(app: FastAPI, config: Any) -> None: # type: ignore[override]
|
|
"""Add or replace the APScheduler cron/interval job for the given config.
|
|
|
|
Args:
|
|
app: FastAPI application instance.
|
|
config: :class:`~app.models.blocklist.ScheduleConfig` to apply.
|
|
"""
|
|
scheduler = app.state.scheduler
|
|
|
|
kwargs: dict[str, Any] = {"app": app}
|
|
trigger_type: str
|
|
trigger_kwargs: dict[str, Any]
|
|
|
|
if config.frequency == ScheduleFrequency.hourly:
|
|
trigger_type = "interval"
|
|
trigger_kwargs = {"hours": config.interval_hours}
|
|
elif config.frequency == ScheduleFrequency.weekly:
|
|
trigger_type = "cron"
|
|
trigger_kwargs = {
|
|
"day_of_week": config.day_of_week,
|
|
"hour": config.hour,
|
|
"minute": config.minute,
|
|
}
|
|
else: # daily (default)
|
|
trigger_type = "cron"
|
|
trigger_kwargs = {
|
|
"hour": config.hour,
|
|
"minute": config.minute,
|
|
}
|
|
|
|
# Remove existing job if it exists, then add new one.
|
|
if scheduler.get_job(JOB_ID):
|
|
scheduler.remove_job(JOB_ID)
|
|
|
|
scheduler.add_job(
|
|
_run_import,
|
|
trigger=trigger_type,
|
|
id=JOB_ID,
|
|
kwargs=kwargs,
|
|
**trigger_kwargs,
|
|
)
|
|
log.info(
|
|
"blocklist_import_scheduled",
|
|
frequency=config.frequency,
|
|
trigger=trigger_type,
|
|
trigger_kwargs=trigger_kwargs,
|
|
)
|