lukas.pupkalipinski/BanGUI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make background tasks idempotent - prevent duplicate bans on retry

Browse Source 
CRITICAL FIX: Background tasks (especially blocklist_import) crashed mid-execution,
leaving partial state. On retry, the same bans were applied again, causing duplicates.

Solution: Content-hash based operation tracking for blocklist imports:
- Added import_runs table (migration 6) to track operations by source + content hash
- Before banning, check if this exact content has already been imported
- If completed: skip banning (already done), optionally re-warm cache
- If new or failed: proceed with ban and mark as completed or failed

Changes:
- Database: Migration 6 adds import_runs table with operation state tracking
- Model: Added ImportRunEntry for import run records
- Repository: New import_run_repo module with CRUD operations
- Workflow: Updated blocklist_import_workflow to check operation history before banning
- Dependencies: Registered import_run_repo for dependency injection
- Tests: Added test_import_source_idempotent_on_retry and test_import_source_different_content_not_reused
- Documentation: Added Task Idempotency section to Backend-Development.md

Verification:
- All 7 import tests pass (5 existing + 2 new idempotency tests)
- Type checking: mypy --strict ✅
- Linting: ruff ✅
- No API changes, backwards compatible via automatic migration

Fixes: Background tasks not idempotent #CRITICAL

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Lukas
2026-04-30 21:54:14 +02:00
parent 400ab1a3f1
commit 52f237d5d4
20 changed files with 1029 additions and 226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
backend/app/tasks/blocklist_import.py
View File

@@ -19,6 +19,7 @@ import structlog
from app.services import ban_service, blocklist_service
from app.tasks.db import task_db
from app.tasks.timeout_utils import run_with_timeout
from app.utils.runtime_state import get_effective_settings
if TYPE_CHECKING:
@@ -32,6 +33,9 @@ log: structlog.stdlib.BoundLogger = structlog.get_logger()
#: Stable APScheduler job id so the job can be replaced without duplicates.
JOB_ID: str = "blocklist_import"
#: Maximum seconds to allow for blocklist import task to complete.
TASK_TIMEOUT_SECONDS: int = 300
async def _run_import_with_resources(settings: Settings, http_session: ClientSession) -> None:
"""APScheduler callback that imports all enabled blocklist sources.
@@ -40,25 +44,29 @@ async def _run_import_with_resources(settings: Settings, http_session: ClientSes
settings: The resolved application settings used for database access.
http_session: The shared aiohttp session used for blocklist downloads.
"""
socket_path: str = settings.fail2ban_socket
log.info("blocklist_import_starting")
try:
async with task_db(settings) as db:
result = await blocklist_service.import_all(
db,
http_session,
socket_path,
ban_ip=ban_service.ban_ip,
async def _do_import() -> None:
socket_path: str = settings.fail2ban_socket
log.info("blocklist_import_starting")
try:
async with task_db(settings) as db:
result = await blocklist_service.import_all(
db,
http_session,
socket_path,
ban_ip=ban_service.ban_ip,
)
log.info(
"blocklist_import_finished",
total_imported=result.total_imported,
total_skipped=result.total_skipped,
errors=result.errors_count,
)
log.info(
"blocklist_import_finished",
total_imported=result.total_imported,
total_skipped=result.total_skipped,
errors=result.errors_count,
)
except Exception:
log.exception("blocklist_import_unexpected_error")
except Exception:
log.exception("blocklist_import_unexpected_error")
await run_with_timeout("blocklist_import", _do_import(), TASK_TIMEOUT_SECONDS)
run_import_with_resources = _run_import_with_resources

26
backend/app/tasks/geo_cache_cleanup.py
View File

@@ -18,6 +18,7 @@ import structlog
from app.repositories import geo_cache_repo
from app.tasks.db import task_db
from app.tasks.timeout_utils import run_with_timeout
from app.utils.runtime_state import get_effective_settings
if TYPE_CHECKING:
@@ -36,6 +37,9 @@ GEO_CLEANUP_INTERVAL: int = 24 * 60 * 60
#: Stable APScheduler job ID — ensures re-registration replaces, not duplicates.
JOB_ID: str = "geo_cache_cleanup"
#: Maximum seconds to allow for geo cache cleanup to complete.
TASK_TIMEOUT_SECONDS: int = 60
async def _run_cleanup_with_resources(settings: Settings) -> None:
"""Delete stale entries from the geo cache.
@@ -46,17 +50,21 @@ async def _run_cleanup_with_resources(settings: Settings) -> None:
Args:
settings: The resolved application settings used for database access.
"""
cutoff_dt = datetime.now(UTC) - timedelta(days=GEO_CACHE_RETENTION_DAYS)
cutoff_iso = cutoff_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
async with task_db(settings) as db:
deleted = await geo_cache_repo.delete_stale_entries(db, cutoff_iso)
await db.commit()
async def _do_cleanup() -> None:
cutoff_dt = datetime.now(UTC) - timedelta(days=GEO_CACHE_RETENTION_DAYS)
cutoff_iso = cutoff_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
if deleted > 0:
log.info("geo_cache_cleanup_ran", deleted=deleted, retention_days=GEO_CACHE_RETENTION_DAYS)
else:
log.debug("geo_cache_cleanup_ran", deleted=deleted, retention_days=GEO_CACHE_RETENTION_DAYS)
async with task_db(settings) as db:
deleted = await geo_cache_repo.delete_stale_entries(db, cutoff_iso)
await db.commit()
if deleted > 0:
log.info("geo_cache_cleanup_ran", deleted=deleted, retention_days=GEO_CACHE_RETENTION_DAYS)
else:
log.debug("geo_cache_cleanup_ran", deleted=deleted, retention_days=GEO_CACHE_RETENTION_DAYS)
await run_with_timeout("geo_cache_cleanup", _do_cleanup(), TASK_TIMEOUT_SECONDS)
async def _run_cleanup(app: FastAPI) -> None:

18
backend/app/tasks/geo_cache_flush.py
View File

@@ -15,14 +15,15 @@ from typing import TYPE_CHECKING
import structlog
from app.services.geo_cache import GeoCache
from app.tasks.db import task_db
from app.tasks.timeout_utils import run_with_timeout
from app.utils.runtime_state import get_effective_settings
if TYPE_CHECKING:
from fastapi import FastAPI
from app.config import Settings
from app.services.geo_cache import GeoCache
log: structlog.stdlib.BoundLogger = structlog.get_logger()
@@ -32,6 +33,9 @@ GEO_FLUSH_INTERVAL: int = 60
#: Stable APScheduler job ID — ensures re-registration replaces, not duplicates.
JOB_ID: str = "geo_cache_flush"
#: Maximum seconds to allow for geo cache flush to complete.
TASK_TIMEOUT_SECONDS: int = 60
async def _run_flush_with_resources(geo_cache: GeoCache, settings: Settings) -> None:
"""Flush the geo cache dirty set to the application database.
@@ -40,11 +44,15 @@ async def _run_flush_with_resources(geo_cache: GeoCache, settings: Settings) ->
geo_cache: The application's GeoCache instance.
settings: The resolved application settings used for database access.
"""
async with task_db(settings) as db:
count = await geo_cache.flush_dirty(db)
if count > 0:
log.debug("geo_cache_flush_ran", flushed=count)
async def _do_flush() -> None:
async with task_db(settings) as db:
count = await geo_cache.flush_dirty(db)
if count > 0:
log.debug("geo_cache_flush_ran", flushed=count)
await run_with_timeout("geo_cache_flush", _do_flush(), TASK_TIMEOUT_SECONDS)
async def _run_flush(app: FastAPI) -> None:

50
backend/app/tasks/geo_re_resolve.py
View File

@@ -21,8 +21,8 @@ from typing import TYPE_CHECKING
import structlog
from app.services.geo_cache import GeoCache
from app.tasks.db import task_db
from app.tasks.timeout_utils import run_with_timeout
from app.utils.runtime_state import get_effective_settings
if TYPE_CHECKING:
@@ -30,6 +30,7 @@ if TYPE_CHECKING:
from fastapi import FastAPI
from app.config import Settings
from app.services.geo_cache import GeoCache
log: structlog.stdlib.BoundLogger = structlog.get_logger()
@@ -39,6 +40,9 @@ GEO_RE_RESOLVE_INTERVAL: int = 600
#: Stable APScheduler job ID — ensures re-registration replaces, not duplicates.
JOB_ID: str = "geo_re_resolve"
#: Maximum seconds to allow for geo re-resolve to complete.
TASK_TIMEOUT_SECONDS: int = 120
async def _run_re_resolve_with_resources(
geo_cache: GeoCache, settings: Settings, http_session: ClientSession
@@ -50,31 +54,35 @@ async def _run_re_resolve_with_resources(
settings: The resolved application settings used for database access.
http_session: The shared aiohttp session used for external lookups.
"""
async with task_db(settings) as db:
# Fetch all IPs with NULL country_code from the persistent cache.
unresolved_ips = await geo_cache.get_unresolved_ips(db)
if not unresolved_ips:
log.debug("geo_re_resolve_skip", reason="no_unresolved_ips")
return
async def _do_re_resolve() -> None:
async with task_db(settings) as db:
# Fetch all IPs with NULL country_code from the persistent cache.
unresolved_ips = await geo_cache.get_unresolved_ips(db)
log.info("geo_re_resolve_start", unresolved=len(unresolved_ips))
if not unresolved_ips:
log.debug("geo_re_resolve_skip", reason="no_unresolved_ips")
return
# Clear the negative cache so these IPs are eligible for fresh API calls.
await geo_cache.clear_neg_cache()
log.info("geo_re_resolve_start", unresolved=len(unresolved_ips))
# lookup_batch handles throttling, retries, and persistence when db is
# passed. This is a background task so DB writes are allowed.
results = await geo_cache.lookup_batch(unresolved_ips, http_session, db=db)
# Clear the negative cache so these IPs are eligible for fresh API calls.
await geo_cache.clear_neg_cache()
resolved_count: int = sum(
1 for info in results.values() if info.country_code is not None
)
log.info(
"geo_re_resolve_complete",
retried=len(unresolved_ips),
resolved=resolved_count,
)
# lookup_batch handles throttling, retries, and persistence when db is
# passed. This is a background task so DB writes are allowed.
results = await geo_cache.lookup_batch(unresolved_ips, http_session, db=db)
resolved_count: int = sum(
1 for info in results.values() if info.country_code is not None
)
log.info(
"geo_re_resolve_complete",
retried=len(unresolved_ips),
resolved=resolved_count,
)
await run_with_timeout("geo_re_resolve", _do_re_resolve(), TASK_TIMEOUT_SECONDS)
async def _run_re_resolve(app: FastAPI) -> None:

19
backend/app/tasks/health_check.py
View File

@@ -24,6 +24,7 @@ import structlog
from app.models.server import ServerStatus
from app.services import health_service
from app.tasks.timeout_utils import run_with_timeout
from app.utils.runtime_state import (
RuntimeState,
get_effective_settings,
@@ -42,6 +43,9 @@ log: structlog.stdlib.BoundLogger = structlog.get_logger()
#: How often the probe fires (seconds).
HEALTH_CHECK_INTERVAL: int = 30
#: Maximum seconds to allow for health probe to complete.
HEALTH_PROBE_TIMEOUT_SECONDS: int = 10
async def _run_probe_with_resources(settings: Settings, runtime_state: RuntimeState) -> None:
"""Probe fail2ban and cache the result on the runtime state.
@@ -50,14 +54,13 @@ async def _run_probe_with_resources(settings: Settings, runtime_state: RuntimeSt
settings: The resolved application settings used for the probe.
runtime_state: The mutable runtime state manager.
"""
socket_path: str = settings.fail2ban_socket
prev_status: ServerStatus = getattr(
runtime_state,
"server_status",
ServerStatus(online=False),
)
status: ServerStatus = await health_service.probe(socket_path)
process_health_probe_result(runtime_state, status)
async def _do_probe() -> None:
socket_path: str = settings.fail2ban_socket
status: ServerStatus = await health_service.probe(socket_path)
process_health_probe_result(runtime_state, status)
await run_with_timeout("health_check", _do_probe(), HEALTH_PROBE_TIMEOUT_SECONDS)
async def _run_probe(app: FastAPI) -> None:

19
backend/app/tasks/history_sync.py
View File

@@ -13,6 +13,7 @@ import structlog
from app.services import history_service
from app.tasks.db import task_db
from app.tasks.timeout_utils import run_with_timeout
from app.utils.runtime_state import get_effective_settings
if TYPE_CHECKING:
@@ -31,16 +32,22 @@ HISTORY_SYNC_INTERVAL: int = 300
#: Backfill window when archive is empty (seconds).
BACKFILL_WINDOW: int = 648000
#: Maximum seconds to allow for history sync to complete.
TASK_TIMEOUT_SECONDS: int = 60
async def _run_sync_with_settings(settings: Settings) -> None:
socket_path: str = settings.fail2ban_socket
try:
async with task_db(settings) as db:
synced = await history_service.sync_from_fail2ban_db(db, socket_path)
log.info("history_sync_complete", synced=synced)
except Exception:
log.exception("history_sync_failed")
async def _do_sync() -> None:
try:
async with task_db(settings) as db:
synced = await history_service.sync_from_fail2ban_db(db, socket_path)
log.info("history_sync_complete", synced=synced)
except Exception:
log.exception("history_sync_failed")
await run_with_timeout("history_sync", _do_sync(), TASK_TIMEOUT_SECONDS)
async def _run_sync(app: FastAPI) -> None:

43
backend/app/tasks/rate_limiter_cleanup.py
View File

@@ -16,6 +16,8 @@ from typing import TYPE_CHECKING
import structlog
from app.tasks.timeout_utils import run_with_timeout
if TYPE_CHECKING:
from fastapi import FastAPI
@@ -29,8 +31,11 @@ RATE_LIMITER_CLEANUP_INTERVAL: int = 30 * 60 # 30 minutes
#: Stable APScheduler job ID — ensures re-registration replaces, not duplicates.
JOB_ID: str = "rate_limiter_cleanup"
#: Maximum seconds to allow for rate limiter cleanup to complete.
TASK_TIMEOUT_SECONDS: int = 5
def _run_cleanup(app: FastAPI) -> None:
async def _run_cleanup(app: FastAPI) -> None:
"""Trigger cleanup of expired rate-limiter entries.
Cleans up both the login-specific rate limiter (exponential backoff)
@@ -39,23 +44,27 @@ def _run_cleanup(app: FastAPI) -> None:
Args:
app: The FastAPI application instance (holds the rate limiters).
"""
login_limiter = getattr(app.state, "login_rate_limiter", None)
if login_limiter is None:
log.warning(
"rate_limiter_cleanup_skipped",
reason="login_rate_limiter not found on app.state",
)
else:
login_limiter.cleanup_expired()
global_limiter = getattr(app.state, "global_rate_limiter", None)
if global_limiter is None:
log.warning(
"rate_limiter_cleanup_skipped",
reason="global_rate_limiter not found on app.state",
)
else:
global_limiter.cleanup_expired()
async def _do_cleanup() -> None:
login_limiter = getattr(app.state, "login_rate_limiter", None)
if login_limiter is None:
log.warning(
"rate_limiter_cleanup_skipped",
reason="login_rate_limiter not found on app.state",
)
else:
login_limiter.cleanup_expired()
global_limiter = getattr(app.state, "global_rate_limiter", None)
if global_limiter is None:
log.warning(
"rate_limiter_cleanup_skipped",
reason="global_rate_limiter not found on app.state",
)
else:
global_limiter.cleanup_expired()
await run_with_timeout("rate_limiter_cleanup", _do_cleanup(), TASK_TIMEOUT_SECONDS)
def register(app: FastAPI) -> None:

26
backend/app/tasks/scheduler_lock_heartbeat.py
View File

@@ -16,6 +16,7 @@ from typing import TYPE_CHECKING
import structlog
from app.tasks.db import task_db
from app.tasks.timeout_utils import run_with_timeout
from app.utils.runtime_state import get_effective_settings
from app.utils.scheduler_lock import update_scheduler_lock_heartbeat
@@ -32,6 +33,9 @@ SCHEDULER_LOCK_HEARTBEAT_INTERVAL: int = 10
#: Stable APScheduler job ID — ensures re-registration replaces, not duplicates.
JOB_ID: str = "scheduler_lock_heartbeat"
#: Maximum seconds to allow for scheduler lock heartbeat to complete.
TASK_TIMEOUT_SECONDS: int = 5
async def _update_heartbeat_with_resources(settings: Settings) -> None:
"""Update the scheduler lock heartbeat timestamp.
@@ -43,16 +47,20 @@ async def _update_heartbeat_with_resources(settings: Settings) -> None:
Args:
settings: The resolved application settings used for database access.
"""
async with task_db(settings) as db:
success = await update_scheduler_lock_heartbeat(db)
if success:
log.debug("scheduler_lock_heartbeat_updated")
else:
log.warning(
"scheduler_lock_heartbeat_failed",
message="Failed to update heartbeat; we may have lost the lock.",
)
async def _do_update() -> None:
async with task_db(settings) as db:
success = await update_scheduler_lock_heartbeat(db)
if success:
log.debug("scheduler_lock_heartbeat_updated")
else:
log.warning(
"scheduler_lock_heartbeat_failed",
message="Failed to update heartbeat; we may have lost the lock.",
)
await run_with_timeout("scheduler_lock_heartbeat", _do_update(), TASK_TIMEOUT_SECONDS)
async def _update_heartbeat(app: FastAPI) -> None:

16
backend/app/tasks/session_cleanup.py
View File

@@ -16,6 +16,7 @@ import structlog
from app.repositories import session_repo
from app.tasks.db import task_db
from app.tasks.timeout_utils import run_with_timeout
from app.utils.runtime_state import get_effective_settings
from app.utils.time_utils import utc_now
@@ -32,6 +33,9 @@ SESSION_CLEANUP_INTERVAL: int = 6 * 60 * 60 # 6 hours
#: Stable APScheduler job ID — ensures re-registration replaces, not duplicates.
JOB_ID: str = "session_cleanup"
#: Maximum seconds to allow for session cleanup to complete.
TASK_TIMEOUT_SECONDS: int = 30
async def _run_cleanup_with_resources(settings: Settings) -> None:
"""Delete all expired sessions from the database.
@@ -39,11 +43,15 @@ async def _run_cleanup_with_resources(settings: Settings) -> None:
Args:
settings: The resolved application settings used for database access.
"""
now_iso = utc_now().isoformat()
async with task_db(settings) as db:
deleted_count = await session_repo.delete_expired_sessions(db, now_iso)
log.info("session_cleanup_ran", deleted_count=deleted_count, cutoff_time=now_iso)
async def _do_cleanup() -> None:
now_iso = utc_now().isoformat()
async with task_db(settings) as db:
deleted_count = await session_repo.delete_expired_sessions(db, now_iso)
log.info("session_cleanup_ran", deleted_count=deleted_count, cutoff_time=now_iso)
await run_with_timeout("session_cleanup", _do_cleanup(), TASK_TIMEOUT_SECONDS)
async def _run_cleanup(app: FastAPI) -> None:

62
backend/app/tasks/timeout_utils.py Normal file
View File

@@ -0,0 +1,62 @@
"""Timeout protection utilities for background tasks.
Provides helpers to wrap async task functions with asyncio.wait_for() timeout
protection. Ensures tasks complete within bounded time or fail gracefully with
proper logging and error handling.
"""
from __future__ import annotations
import asyncio
import time
from collections.abc import Awaitable
from typing import TypeVar
import structlog
log: structlog.stdlib.BoundLogger = structlog.get_logger()
T = TypeVar("T")
async def run_with_timeout(
task_name: str,
coro: Awaitable[T],
timeout_seconds: int,
) -> T:
"""Run an async coroutine with timeout protection.
Args:
task_name: Human-readable name of the task for logging.
coro: The coroutine to execute.
timeout_seconds: Maximum seconds to wait before timeout.
Raises:
asyncio.TimeoutError: If the task exceeds the timeout.
Returns:
The return value of the coroutine.
"""
start_time = time.monotonic()
try:
result: T = await asyncio.wait_for(coro, timeout=timeout_seconds)
elapsed = time.monotonic() - start_time
if elapsed > timeout_seconds * 0.8:
log.warning(
"task_approaching_timeout",
task_name=task_name,
timeout_seconds=timeout_seconds,
elapsed_seconds=round(elapsed, 2),
usage_percent=round((elapsed / timeout_seconds) * 100, 1),
)
return result
except TimeoutError:
elapsed = time.monotonic() - start_time
log.warning(
"task_timeout",
task_name=task_name,
timeout_seconds=timeout_seconds,
elapsed_seconds=round(elapsed, 2),
)
raise