Optimise geo lookup and aggregation for 10k+ IPs

- Add persistent geo_cache SQLite table (db.py)
- Rewrite geo_service: batch API (100 IPs/call), two-tier cache,
  no caching of failed lookups so they are retried
- Pre-warm geo cache from DB on startup (main.py lifespan)
- Rewrite bans_by_country: SQL GROUP BY ip aggregation + lookup_batch
  instead of 2000-row fetch + asyncio.gather individual calls
- Pre-warm geo cache after blocklist import (blocklist_service)
- Add 300ms debounce to useMapData hook to cancel stale requests
- Add perf benchmark asserting <2s for 10k bans
- Add seed_10k_bans.py script for manual perf testing
This commit is contained in:
2026-03-07 20:28:51 +01:00
parent 53d664de4f
commit ddfc8a0b02
13 changed files with 917 additions and 90 deletions

View File

@@ -7,6 +7,13 @@ import { fetchBansByCountry } from "../api/map";
import type { BansByCountryResponse, MapBanItem, TimeRange } from "../types/map";
import type { BanOriginFilter } from "../types/ban";
// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------
/** Debounce delay in milliseconds before firing a fetch after filter changes. */
const DEBOUNCE_MS = 300;
// ---------------------------------------------------------------------------
// Return type
// ---------------------------------------------------------------------------
@@ -40,30 +47,43 @@ export function useMapData(
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const abortRef = useRef<AbortController | null>(null);
const debounceRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const load = useCallback((): void => {
abortRef.current?.abort();
abortRef.current = new AbortController();
// Cancel any pending debounce timer.
if (debounceRef.current != null) {
clearTimeout(debounceRef.current);
}
// Show loading immediately so the skeleton / spinner appears.
setLoading(true);
setError(null);
fetchBansByCountry(range, origin)
.then((resp) => {
setData(resp);
})
.catch((err: unknown) => {
if (err instanceof Error && err.name !== "AbortError") {
setError(err.message);
}
})
.finally((): void => {
setLoading(false);
});
debounceRef.current = setTimeout((): void => {
// Abort any in-flight request from a previous filter selection.
abortRef.current?.abort();
abortRef.current = new AbortController();
fetchBansByCountry(range, origin)
.then((resp) => {
setData(resp);
})
.catch((err: unknown) => {
if (err instanceof Error && err.name !== "AbortError") {
setError(err.message);
}
})
.finally((): void => {
setLoading(false);
});
}, DEBOUNCE_MS);
}, [range, origin]);
useEffect((): (() => void) => {
load();
return (): void => {
if (debounceRef.current != null) {
clearTimeout(debounceRef.current);
}
abortRef.current?.abort();
};
}, [load]);