Stage 10: external blocklist importer — backend + frontend
- blocklist_repo.py: CRUD for blocklist_sources table - import_log_repo.py: add/list/get-last log entries - blocklist_service.py: source CRUD, preview, import (download/validate/ban), import_all, schedule get/set/info - blocklist_import.py: APScheduler task (hourly/daily/weekly schedule triggers) - blocklist.py router: 9 endpoints (list/create/update/delete/preview/import/ schedule-get+put/log) - blocklist.py models: ScheduleFrequency (StrEnum), ScheduleConfig, ScheduleInfo, ImportSourceResult, ImportRunResult, PreviewResponse - 59 new tests (18 repo + 19 service + 22 router); 374 total pass - ruff clean, mypy clean for Stage 10 files - types/blocklist.ts, api/blocklist.ts, hooks/useBlocklist.ts - BlocklistsPage.tsx: source management, schedule picker, import log table - Frontend tsc + ESLint clean
This commit is contained in:
153
backend/app/tasks/blocklist_import.py
Normal file
153
backend/app/tasks/blocklist_import.py
Normal file
@@ -0,0 +1,153 @@
|
||||
"""External blocklist import background task.
|
||||
|
||||
Registers an APScheduler job that downloads all enabled blocklist sources,
|
||||
validates their entries, and applies bans via fail2ban on a configurable
|
||||
schedule. The default schedule is daily at 03:00 UTC; it is stored in the
|
||||
application :class:`~app.models.blocklist.ScheduleConfig` settings and can
|
||||
be updated at runtime through the blocklist router.
|
||||
|
||||
The scheduler job ID is ``"blocklist_import"`` — using a stable id means
|
||||
re-registering the job (e.g. after a schedule update) safely replaces the
|
||||
existing entry without creating duplicates.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import structlog
|
||||
|
||||
from app.models.blocklist import ScheduleFrequency
|
||||
from app.services import blocklist_service
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import FastAPI
|
||||
|
||||
log: structlog.stdlib.BoundLogger = structlog.get_logger()
|
||||
|
||||
#: Stable APScheduler job id so the job can be replaced without duplicates.
|
||||
JOB_ID: str = "blocklist_import"
|
||||
|
||||
|
||||
async def _run_import(app: Any) -> None:
|
||||
"""APScheduler callback that imports all enabled blocklist sources.
|
||||
|
||||
Reads shared resources from ``app.state`` and delegates to
|
||||
:func:`~app.services.blocklist_service.import_all`.
|
||||
|
||||
Args:
|
||||
app: The :class:`fastapi.FastAPI` application instance passed via
|
||||
APScheduler ``kwargs``.
|
||||
"""
|
||||
db = app.state.db
|
||||
http_session = app.state.http_session
|
||||
socket_path: str = app.state.settings.fail2ban_socket
|
||||
|
||||
log.info("blocklist_import_starting")
|
||||
try:
|
||||
result = await blocklist_service.import_all(db, http_session, socket_path)
|
||||
log.info(
|
||||
"blocklist_import_finished",
|
||||
total_imported=result.total_imported,
|
||||
total_skipped=result.total_skipped,
|
||||
errors=result.errors_count,
|
||||
)
|
||||
except Exception:
|
||||
log.exception("blocklist_import_unexpected_error")
|
||||
|
||||
|
||||
def register(app: FastAPI) -> None:
|
||||
"""Add (or replace) the blocklist import job in the application scheduler.
|
||||
|
||||
Reads the persisted :class:`~app.models.blocklist.ScheduleConfig` from
|
||||
the database and translates it into the appropriate APScheduler trigger.
|
||||
|
||||
Should be called inside the lifespan handler after the scheduler and
|
||||
database have been initialised.
|
||||
|
||||
Args:
|
||||
app: The :class:`fastapi.FastAPI` application instance whose
|
||||
``app.state.scheduler`` will receive the job.
|
||||
"""
|
||||
import asyncio # noqa: PLC0415
|
||||
|
||||
async def _do_register() -> None:
|
||||
config = await blocklist_service.get_schedule(app.state.db)
|
||||
_apply_schedule(app, config)
|
||||
|
||||
# APScheduler is synchronous at registration time; use asyncio to read
|
||||
# the stored schedule from the DB before registering.
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(_do_register())
|
||||
except RuntimeError:
|
||||
# If the current thread already has a running loop (uvicorn), schedule
|
||||
# the registration as a coroutine.
|
||||
asyncio.ensure_future(_do_register())
|
||||
|
||||
|
||||
def reschedule(app: FastAPI) -> None:
|
||||
"""Re-register the blocklist import job with the latest schedule config.
|
||||
|
||||
Called by the blocklist router after a schedule update so changes take
|
||||
effect immediately without a server restart.
|
||||
|
||||
Args:
|
||||
app: The :class:`fastapi.FastAPI` application instance.
|
||||
"""
|
||||
import asyncio # noqa: PLC0415
|
||||
|
||||
async def _do_reschedule() -> None:
|
||||
config = await blocklist_service.get_schedule(app.state.db)
|
||||
_apply_schedule(app, config)
|
||||
|
||||
asyncio.ensure_future(_do_reschedule())
|
||||
|
||||
|
||||
def _apply_schedule(app: FastAPI, config: Any) -> None: # type: ignore[override]
|
||||
"""Add or replace the APScheduler cron/interval job for the given config.
|
||||
|
||||
Args:
|
||||
app: FastAPI application instance.
|
||||
config: :class:`~app.models.blocklist.ScheduleConfig` to apply.
|
||||
"""
|
||||
scheduler = app.state.scheduler
|
||||
|
||||
kwargs: dict[str, Any] = {"app": app}
|
||||
trigger_type: str
|
||||
trigger_kwargs: dict[str, Any]
|
||||
|
||||
if config.frequency == ScheduleFrequency.hourly:
|
||||
trigger_type = "interval"
|
||||
trigger_kwargs = {"hours": config.interval_hours}
|
||||
elif config.frequency == ScheduleFrequency.weekly:
|
||||
trigger_type = "cron"
|
||||
trigger_kwargs = {
|
||||
"day_of_week": config.day_of_week,
|
||||
"hour": config.hour,
|
||||
"minute": config.minute,
|
||||
}
|
||||
else: # daily (default)
|
||||
trigger_type = "cron"
|
||||
trigger_kwargs = {
|
||||
"hour": config.hour,
|
||||
"minute": config.minute,
|
||||
}
|
||||
|
||||
# Remove existing job if it exists, then add new one.
|
||||
if scheduler.get_job(JOB_ID):
|
||||
scheduler.remove_job(JOB_ID)
|
||||
|
||||
scheduler.add_job(
|
||||
_run_import,
|
||||
trigger=trigger_type,
|
||||
id=JOB_ID,
|
||||
kwargs=kwargs,
|
||||
**trigger_kwargs,
|
||||
)
|
||||
log.info(
|
||||
"blocklist_import_scheduled",
|
||||
frequency=config.frequency,
|
||||
trigger=trigger_type,
|
||||
trigger_kwargs=trigger_kwargs,
|
||||
)
|
||||
Reference in New Issue
Block a user