Optimise geo lookup and aggregation for 10k+ IPs

- Add persistent geo_cache SQLite table (db.py)
- Rewrite geo_service: batch API (100 IPs/call), two-tier cache,
  no caching of failed lookups so they are retried
- Pre-warm geo cache from DB on startup (main.py lifespan)
- Rewrite bans_by_country: SQL GROUP BY ip aggregation + lookup_batch
  instead of 2000-row fetch + asyncio.gather individual calls
- Pre-warm geo cache after blocklist import (blocklist_service)
- Add 300ms debounce to useMapData hook to cancel stale requests
- Add perf benchmark asserting <2s for 10k bans
- Add seed_10k_bans.py script for manual perf testing
This commit is contained in:
2026-03-07 20:28:51 +01:00
parent 53d664de4f
commit ddfc8a0b02
13 changed files with 917 additions and 90 deletions

View File

@@ -245,6 +245,10 @@ async def import_source(
fail2ban requires individual addresses. Any error encountered during
download is recorded and the result is returned without raising.
After a successful import the geo cache is pre-warmed by batch-resolving
all newly banned IPs. This ensures the dashboard and map show country
data immediately after import rather than facing cold-cache lookups.
Args:
source: The :class:`~app.models.blocklist.BlocklistSource` to import.
http_session: Shared :class:`aiohttp.ClientSession`.
@@ -287,6 +291,7 @@ async def import_source(
imported = 0
skipped = 0
ban_error: str | None = None
imported_ips: list[str] = []
# Import jail_service here to avoid circular import at module level.
from app.services import jail_service # noqa: PLC0415
@@ -304,6 +309,7 @@ async def import_source(
try:
await jail_service.ban_ip(socket_path, BLOCKLIST_JAIL, stripped)
imported += 1
imported_ips.append(stripped)
except jail_service.JailNotFoundError as exc:
# The target jail does not exist in fail2ban — there is no point
# continuing because every subsequent ban would also fail.
@@ -329,6 +335,25 @@ async def import_source(
skipped=skipped,
error=ban_error,
)
# --- Pre-warm geo cache for newly imported IPs ---
if imported_ips:
from app.services import geo_service # noqa: PLC0415
try:
await geo_service.lookup_batch(imported_ips, http_session, db=db)
log.info(
"blocklist_geo_prewarm_complete",
source_id=source.id,
count=len(imported_ips),
)
except Exception as exc: # noqa: BLE001
log.warning(
"blocklist_geo_prewarm_failed",
source_id=source.id,
error=str(exc),
)
return ImportSourceResult(
source_id=source.id,
source_url=source.url,