feat(geo): add cache hit/miss metrics and prewarm support

- Add _hits/_misses counters to GeoCache for cache hit/miss ratio tracking
- Reset counters on clear()
- Count hits before misses in lookup_batch() to avoid interleaving
- Add synchronous prewarm() using asyncio.create_task for fire-and-forget
- Add hits/misses fields to GeoCacheStatsResponse model
- Add TestCacheMetrics (5 tests), TestPrewarm (3 tests), TestLargeBanList (2 tests)
- Fix _make_async_db() mock: db.execute is not async, returns ctx manager
- Move collections.abc to TYPE_CHECKING block (TC003)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
2026-05-03 00:35:47 +02:00
parent b587c6e850
commit bd6170722a
4 changed files with 280 additions and 152 deletions

View File

@@ -75,6 +75,8 @@ class GeoCacheStatsResponse(BanGuiBaseModel):
unresolved: int = Field(..., description="Number of geo_cache rows with country_code IS NULL.")
neg_cache_size: int = Field(..., description="Number of entries in the in-memory negative cache.")
dirty_size: int = Field(..., description="Number of newly resolved entries not yet flushed to disk.")
hits: int = Field(default=0, description="Number of cache hits since last clear.")
misses: int = Field(default=0, description="Number of cache misses since last clear.")
class GeoReResolveResponse(BanGuiBaseModel):
"""Response for ``POST /api/geo/re-resolve``.

View File

@@ -16,6 +16,7 @@ An instance should be created once at startup and stored on ``app.state.geo_cach
from __future__ import annotations
import asyncio
import collections.abc
import time
from typing import TYPE_CHECKING
@@ -26,6 +27,8 @@ from app.models.geo import GeoInfo
from app.repositories import geo_cache_repo
if TYPE_CHECKING:
import collections.abc
import aiosqlite
import geoip2.database
import geoip2.errors
@@ -91,6 +94,8 @@ class GeoCache:
_geoip_initialized: Indicates whether init_geoip() has been called.
_allow_http_fallback: Whether to use ip-api.com as fallback.
_cache_lock: Async lock protecting cache mutations.
_hits: Cache hit counter (increments on hit, resets on clear).
_misses: Cache miss counter (increments on miss, resets on clear).
"""
def __init__(self, allow_http_fallback: bool = False) -> None:
@@ -108,6 +113,8 @@ class GeoCache:
self._geoip_initialized: bool = False
self._allow_http_fallback: bool = allow_http_fallback
self._cache_lock: asyncio.Lock = asyncio.Lock()
self._hits: int = 0
self._misses: int = 0
async def clear(self) -> None:
"""Flush both the positive and negative lookup caches.
@@ -119,6 +126,8 @@ class GeoCache:
self._cache.clear()
self._neg_cache.clear()
self._dirty.clear()
self._hits = 0
self._misses = 0
async def clear_neg_cache(self) -> None:
"""Flush only the negative (failed-lookups) cache.
@@ -162,6 +171,8 @@ class GeoCache:
"unresolved": unresolved,
"neg_cache_size": len(self._neg_cache),
"dirty_size": len(self._dirty),
"hits": self._hits,
"misses": self._misses,
}
async def count_unresolved(self, db: aiosqlite.Connection) -> int:
@@ -365,6 +376,7 @@ class GeoCache:
(e.g. network timeout).
"""
if ip in self._cache:
self._hits += 1
return self._cache[ip]
# Negative cache: skip IPs that recently failed to avoid hammering the API.
@@ -396,6 +408,7 @@ class GeoCache:
log.debug("geo_lookup_failed_no_http_fallback", ip=ip)
async with self._cache_lock:
self._neg_cache[ip] = time.monotonic()
self._misses += 1
if db is not None:
try:
await geo_cache_repo.upsert_neg_entry_and_commit(db=db, ip=ip)
@@ -447,6 +460,7 @@ class GeoCache:
# Both resolvers failed — record in negative cache to avoid hammering.
async with self._cache_lock:
self._neg_cache[ip] = time.monotonic()
self._misses += 1
if db is not None:
try:
await geo_cache_repo.upsert_neg_entry_and_commit(db=db, ip=ip)
@@ -538,9 +552,15 @@ class GeoCache:
else:
uncached.append(ip)
# Count cache hits (IPs found in cache) before miss counting.
cache_hits = sum(1 for ip in unique_ips if ip in self._cache)
self._hits += cache_hits
if not uncached:
return geo_result
self._misses += len(uncached)
log.info("geo_batch_lookup_start", total=len(uncached))
# PRIMARY: Try local MaxMind database for all uncached IPs.
@@ -824,3 +844,20 @@ class GeoCache:
log.info("geo_flush_dirty_complete", count=len(rows))
return len(rows)
def prewarm(
self,
ips: collections.abc.Iterable[str],
http_session: aiohttp.ClientSession,
) -> None:
"""Pre-load geo data for *ips* without blocking the caller.
Fires off :meth:`lookup_batch` as a background task. Useful at startup
or when warming the cache with commonly-seen IPs before they appear in
bans.
Args:
ips: Iterable of IP address strings to pre-warm.
http_session: Shared :class:`aiohttp.ClientSession` for HTTP fallback.
"""
asyncio.create_task(self.lookup_batch(list(ips), http_session))