Optimise geo lookup and aggregation for 10k+ IPs

- Add persistent geo_cache SQLite table (db.py)
- Rewrite geo_service: batch API (100 IPs/call), two-tier cache,
  no caching of failed lookups so they are retried
- Pre-warm geo cache from DB on startup (main.py lifespan)
- Rewrite bans_by_country: SQL GROUP BY ip aggregation + lookup_batch
  instead of 2000-row fetch + asyncio.gather individual calls
- Pre-warm geo cache after blocklist import (blocklist_service)
- Add 300ms debounce to useMapData hook to cancel stale requests
- Add perf benchmark asserting <2s for 10k bans
- Add seed_10k_bans.py script for manual perf testing
This commit is contained in:
2026-03-07 20:28:51 +01:00
parent 53d664de4f
commit ddfc8a0b02
13 changed files with 917 additions and 90 deletions

View File

@@ -9,14 +9,16 @@ Also provides ``GET /api/dashboard/bans`` for the dashboard ban-list table.
from __future__ import annotations
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Annotated
import aiosqlite
if TYPE_CHECKING:
import aiohttp
from fastapi import APIRouter, Query, Request
from fastapi import APIRouter, Depends, Query, Request
from app.dependencies import AuthDep
from app.dependencies import AuthDep, get_db
from app.models.ban import (
BanOrigin,
BansByCountryResponse,
@@ -75,6 +77,7 @@ async def get_server_status(
async def get_dashboard_bans(
request: Request,
_auth: AuthDep,
db: Annotated[aiosqlite.Connection, Depends(get_db)],
range: TimeRange = Query(default=_DEFAULT_RANGE, description="Time-range preset."),
page: int = Query(default=1, ge=1, description="1-based page number."),
page_size: int = Query(default=_DEFAULT_PAGE_SIZE, ge=1, le=500, description="Items per page."),
@@ -92,6 +95,7 @@ async def get_dashboard_bans(
Args:
request: The incoming request (used to access ``app.state``).
_auth: Validated session dependency.
db: BanGUI application database (for persistent geo cache writes).
range: Time-range preset — ``"24h"``, ``"7d"``, ``"30d"``, or
``"365d"``.
page: 1-based page number.
@@ -106,7 +110,7 @@ async def get_dashboard_bans(
http_session: aiohttp.ClientSession = request.app.state.http_session
async def _enricher(ip: str) -> geo_service.GeoInfo | None:
return await geo_service.lookup(ip, http_session)
return await geo_service.lookup(ip, http_session, db=db)
return await ban_service.list_bans(
socket_path,
@@ -126,6 +130,7 @@ async def get_dashboard_bans(
async def get_bans_by_country(
request: Request,
_auth: AuthDep,
db: Annotated[aiosqlite.Connection, Depends(get_db)],
range: TimeRange = Query(default=_DEFAULT_RANGE, description="Time-range preset."),
origin: BanOrigin | None = Query(
default=None,
@@ -134,30 +139,29 @@ async def get_bans_by_country(
) -> BansByCountryResponse:
"""Return ban counts aggregated by ISO country code.
Fetches up to 2 000 ban records in the selected time window, enriches
every record with geo data, and returns a ``{country_code: count}`` map
plus the full enriched ban list for the companion access table.
Uses SQL aggregation (``GROUP BY ip``) and batch geo-resolution to handle
10 000+ banned IPs efficiently. Returns a ``{country_code: count}`` map
and the 200 most recent raw ban rows for the companion access table.
Args:
request: The incoming request.
_auth: Validated session dependency.
db: BanGUI application database (for persistent geo cache writes).
range: Time-range preset.
origin: Optional filter by ban origin.
Returns:
:class:`~app.models.ban.BansByCountryResponse` with per-country
aggregation and the full ban list.
aggregation and the companion ban list.
"""
socket_path: str = request.app.state.settings.fail2ban_socket
http_session: aiohttp.ClientSession = request.app.state.http_session
async def _enricher(ip: str) -> geo_service.GeoInfo | None:
return await geo_service.lookup(ip, http_session)
return await ban_service.bans_by_country(
socket_path,
range,
geo_enricher=_enricher,
http_session=http_session,
app_db=db,
origin=origin,
)