Optimise geo lookup and aggregation for 10k+ IPs
- Add persistent geo_cache SQLite table (db.py) - Rewrite geo_service: batch API (100 IPs/call), two-tier cache, no caching of failed lookups so they are retried - Pre-warm geo cache from DB on startup (main.py lifespan) - Rewrite bans_by_country: SQL GROUP BY ip aggregation + lookup_batch instead of 2000-row fetch + asyncio.gather individual calls - Pre-warm geo cache after blocklist import (blocklist_service) - Add 300ms debounce to useMapData hook to cancel stale requests - Add perf benchmark asserting <2s for 10k bans - Add seed_10k_bans.py script for manual perf testing
This commit is contained in:
@@ -9,14 +9,16 @@ Also provides ``GET /api/dashboard/bans`` for the dashboard ban-list table.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Annotated
|
||||
|
||||
import aiosqlite
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import aiohttp
|
||||
|
||||
from fastapi import APIRouter, Query, Request
|
||||
from fastapi import APIRouter, Depends, Query, Request
|
||||
|
||||
from app.dependencies import AuthDep
|
||||
from app.dependencies import AuthDep, get_db
|
||||
from app.models.ban import (
|
||||
BanOrigin,
|
||||
BansByCountryResponse,
|
||||
@@ -75,6 +77,7 @@ async def get_server_status(
|
||||
async def get_dashboard_bans(
|
||||
request: Request,
|
||||
_auth: AuthDep,
|
||||
db: Annotated[aiosqlite.Connection, Depends(get_db)],
|
||||
range: TimeRange = Query(default=_DEFAULT_RANGE, description="Time-range preset."),
|
||||
page: int = Query(default=1, ge=1, description="1-based page number."),
|
||||
page_size: int = Query(default=_DEFAULT_PAGE_SIZE, ge=1, le=500, description="Items per page."),
|
||||
@@ -92,6 +95,7 @@ async def get_dashboard_bans(
|
||||
Args:
|
||||
request: The incoming request (used to access ``app.state``).
|
||||
_auth: Validated session dependency.
|
||||
db: BanGUI application database (for persistent geo cache writes).
|
||||
range: Time-range preset — ``"24h"``, ``"7d"``, ``"30d"``, or
|
||||
``"365d"``.
|
||||
page: 1-based page number.
|
||||
@@ -106,7 +110,7 @@ async def get_dashboard_bans(
|
||||
http_session: aiohttp.ClientSession = request.app.state.http_session
|
||||
|
||||
async def _enricher(ip: str) -> geo_service.GeoInfo | None:
|
||||
return await geo_service.lookup(ip, http_session)
|
||||
return await geo_service.lookup(ip, http_session, db=db)
|
||||
|
||||
return await ban_service.list_bans(
|
||||
socket_path,
|
||||
@@ -126,6 +130,7 @@ async def get_dashboard_bans(
|
||||
async def get_bans_by_country(
|
||||
request: Request,
|
||||
_auth: AuthDep,
|
||||
db: Annotated[aiosqlite.Connection, Depends(get_db)],
|
||||
range: TimeRange = Query(default=_DEFAULT_RANGE, description="Time-range preset."),
|
||||
origin: BanOrigin | None = Query(
|
||||
default=None,
|
||||
@@ -134,30 +139,29 @@ async def get_bans_by_country(
|
||||
) -> BansByCountryResponse:
|
||||
"""Return ban counts aggregated by ISO country code.
|
||||
|
||||
Fetches up to 2 000 ban records in the selected time window, enriches
|
||||
every record with geo data, and returns a ``{country_code: count}`` map
|
||||
plus the full enriched ban list for the companion access table.
|
||||
Uses SQL aggregation (``GROUP BY ip``) and batch geo-resolution to handle
|
||||
10 000+ banned IPs efficiently. Returns a ``{country_code: count}`` map
|
||||
and the 200 most recent raw ban rows for the companion access table.
|
||||
|
||||
Args:
|
||||
request: The incoming request.
|
||||
_auth: Validated session dependency.
|
||||
db: BanGUI application database (for persistent geo cache writes).
|
||||
range: Time-range preset.
|
||||
origin: Optional filter by ban origin.
|
||||
|
||||
Returns:
|
||||
:class:`~app.models.ban.BansByCountryResponse` with per-country
|
||||
aggregation and the full ban list.
|
||||
aggregation and the companion ban list.
|
||||
"""
|
||||
socket_path: str = request.app.state.settings.fail2ban_socket
|
||||
http_session: aiohttp.ClientSession = request.app.state.http_session
|
||||
|
||||
async def _enricher(ip: str) -> geo_service.GeoInfo | None:
|
||||
return await geo_service.lookup(ip, http_session)
|
||||
|
||||
return await ban_service.bans_by_country(
|
||||
socket_path,
|
||||
range,
|
||||
geo_enricher=_enricher,
|
||||
http_session=http_session,
|
||||
app_db=db,
|
||||
origin=origin,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user