Optimise geo lookup and aggregation for 10k+ IPs
- Add persistent geo_cache SQLite table (db.py) - Rewrite geo_service: batch API (100 IPs/call), two-tier cache, no caching of failed lookups so they are retried - Pre-warm geo cache from DB on startup (main.py lifespan) - Rewrite bans_by_country: SQL GROUP BY ip aggregation + lookup_batch instead of 2000-row fetch + asyncio.gather individual calls - Pre-warm geo cache after blocklist import (blocklist_service) - Add 300ms debounce to useMapData hook to cancel stale requests - Add perf benchmark asserting <2s for 10k bans - Add seed_10k_bans.py script for manual perf testing
This commit is contained in:
@@ -245,6 +245,10 @@ async def import_source(
|
||||
fail2ban requires individual addresses. Any error encountered during
|
||||
download is recorded and the result is returned without raising.
|
||||
|
||||
After a successful import the geo cache is pre-warmed by batch-resolving
|
||||
all newly banned IPs. This ensures the dashboard and map show country
|
||||
data immediately after import rather than facing cold-cache lookups.
|
||||
|
||||
Args:
|
||||
source: The :class:`~app.models.blocklist.BlocklistSource` to import.
|
||||
http_session: Shared :class:`aiohttp.ClientSession`.
|
||||
@@ -287,6 +291,7 @@ async def import_source(
|
||||
imported = 0
|
||||
skipped = 0
|
||||
ban_error: str | None = None
|
||||
imported_ips: list[str] = []
|
||||
|
||||
# Import jail_service here to avoid circular import at module level.
|
||||
from app.services import jail_service # noqa: PLC0415
|
||||
@@ -304,6 +309,7 @@ async def import_source(
|
||||
try:
|
||||
await jail_service.ban_ip(socket_path, BLOCKLIST_JAIL, stripped)
|
||||
imported += 1
|
||||
imported_ips.append(stripped)
|
||||
except jail_service.JailNotFoundError as exc:
|
||||
# The target jail does not exist in fail2ban — there is no point
|
||||
# continuing because every subsequent ban would also fail.
|
||||
@@ -329,6 +335,25 @@ async def import_source(
|
||||
skipped=skipped,
|
||||
error=ban_error,
|
||||
)
|
||||
|
||||
# --- Pre-warm geo cache for newly imported IPs ---
|
||||
if imported_ips:
|
||||
from app.services import geo_service # noqa: PLC0415
|
||||
|
||||
try:
|
||||
await geo_service.lookup_batch(imported_ips, http_session, db=db)
|
||||
log.info(
|
||||
"blocklist_geo_prewarm_complete",
|
||||
source_id=source.id,
|
||||
count=len(imported_ips),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning(
|
||||
"blocklist_geo_prewarm_failed",
|
||||
source_id=source.id,
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
return ImportSourceResult(
|
||||
source_id=source.id,
|
||||
source_url=source.url,
|
||||
|
||||
Reference in New Issue
Block a user