Fix country not shown in ban list due to geo rate limiting
list_bans() was calling geo_service.lookup() once per IP on the page (e.g. 100 sequential HTTP requests), hitting the ip-api.com free-tier single-IP limit of 45 req/min. IPs beyond the ~45th were added to the in-process negative cache (5 min TTL) and showed as no country until the TTL expired. The map endpoint never had this problem because it used lookup_batch (100 IPs per POST). Add http_session and app_db params to list_bans(). When http_session is provided (production path), the entire page is resolved in one lookup_batch() call instead of N individual ones. The legacy geo_enricher callback is kept for test compatibility. Update the dashboard router to use the batch path directly. Adds 3 tests covering the batch geo path, failure resilience, and http_session priority over geo_enricher.
This commit is contained in:
@@ -171,6 +171,8 @@ async def list_bans(
|
||||
*,
|
||||
page: int = 1,
|
||||
page_size: int = _DEFAULT_PAGE_SIZE,
|
||||
http_session: aiohttp.ClientSession | None = None,
|
||||
app_db: aiosqlite.Connection | None = None,
|
||||
geo_enricher: Any | None = None,
|
||||
origin: BanOrigin | None = None,
|
||||
) -> DashboardBanListResponse:
|
||||
@@ -180,6 +182,15 @@ async def list_bans(
|
||||
``timeofban`` falls within the specified *range_*. Results are ordered
|
||||
newest-first.
|
||||
|
||||
Geo enrichment strategy (highest priority first):
|
||||
|
||||
1. When *http_session* is provided the entire page of IPs is resolved in
|
||||
one :func:`~app.services.geo_service.lookup_batch` call (up to 100 IPs
|
||||
per HTTP request). This avoids the 45 req/min rate limit of the
|
||||
single-IP endpoint and is the preferred production path.
|
||||
2. When only *geo_enricher* is provided (legacy / test path) each IP is
|
||||
resolved individually via the supplied async callable.
|
||||
|
||||
Args:
|
||||
socket_path: Path to the fail2ban Unix domain socket.
|
||||
range_: Time-range preset (``"24h"``, ``"7d"``, ``"30d"``, or
|
||||
@@ -187,8 +198,13 @@ async def list_bans(
|
||||
page: 1-based page number (default: ``1``).
|
||||
page_size: Maximum items per page, capped at ``_MAX_PAGE_SIZE``
|
||||
(default: ``100``).
|
||||
http_session: Optional shared :class:`aiohttp.ClientSession`. When
|
||||
provided, :func:`~app.services.geo_service.lookup_batch` is used
|
||||
for efficient bulk geo resolution.
|
||||
app_db: Optional BanGUI application database used to persist newly
|
||||
resolved geo entries and to read back cached results.
|
||||
geo_enricher: Optional async callable ``(ip: str) -> GeoInfo | None``.
|
||||
When supplied every result is enriched with country and ASN data.
|
||||
Used as a fallback when *http_session* is ``None`` (e.g. tests).
|
||||
origin: Optional origin filter — ``"blocklist"`` restricts results to
|
||||
the ``blocklist-import`` jail, ``"selfblock"`` excludes it.
|
||||
|
||||
@@ -196,6 +212,8 @@ async def list_bans(
|
||||
:class:`~app.models.ban.DashboardBanListResponse` containing the
|
||||
paginated items and total count.
|
||||
"""
|
||||
from app.services import geo_service # noqa: PLC0415
|
||||
|
||||
since: int = _since_unix(range_)
|
||||
effective_page_size: int = min(page_size, _MAX_PAGE_SIZE)
|
||||
offset: int = (page - 1) * effective_page_size
|
||||
@@ -231,6 +249,17 @@ async def list_bans(
|
||||
) as cur:
|
||||
rows = await cur.fetchall()
|
||||
|
||||
# Batch-resolve geo data for all IPs on this page in a single API call.
|
||||
# This avoids hitting the 45 req/min single-IP rate limit when the
|
||||
# page contains many bans (e.g. after a large blocklist import).
|
||||
geo_map: dict[str, Any] = {}
|
||||
if http_session is not None and rows:
|
||||
page_ips: list[str] = [str(r["ip"]) for r in rows]
|
||||
try:
|
||||
geo_map = await geo_service.lookup_batch(page_ips, http_session, db=app_db)
|
||||
except Exception: # noqa: BLE001
|
||||
log.warning("ban_service_batch_geo_failed_list_bans")
|
||||
|
||||
items: list[DashboardBanItem] = []
|
||||
for row in rows:
|
||||
jail: str = str(row["jail"])
|
||||
@@ -245,7 +274,14 @@ async def list_bans(
|
||||
asn: str | None = None
|
||||
org: str | None = None
|
||||
|
||||
if geo_enricher is not None:
|
||||
if geo_map:
|
||||
geo = geo_map.get(ip)
|
||||
if geo is not None:
|
||||
country_code = geo.country_code
|
||||
country_name = geo.country_name
|
||||
asn = geo.asn
|
||||
org = geo.org
|
||||
elif geo_enricher is not None:
|
||||
try:
|
||||
geo = await geo_enricher(ip)
|
||||
if geo is not None:
|
||||
|
||||
Reference in New Issue
Block a user