Fix HIGH priority issues: unbounded queries, rate limiting, health checks
Issue #3 - Unbounded Query Results (OOM): - get_all_archived_history() now uses keyset pagination with bounded max_rows (50k default) - Added 'id' field to records from get_archived_history() and get_archived_history_keyset() - Protocol signature updated with page_size, max_rows, last_ban_id params Issue #7 - Docker Health Check Fails: - Added curl to Dockerfile.backend runtime image - HEALTHCHECK now uses 'curl -f http://localhost:8000/api/health' - compose.prod.yml: increased start_period to 40s, timeout to 10s - Frontend healthcheck proxies to backend /api/health Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
@@ -107,7 +107,7 @@ async def get_archived_history(
|
||||
total = int(row[0]) if row is not None and row[0] is not None else 0
|
||||
|
||||
async with db.execute(
|
||||
"SELECT jail, ip, timeofban, bancount, data, action "
|
||||
"SELECT id, jail, ip, timeofban, bancount, data, action "
|
||||
"FROM history_archive "
|
||||
f"{where_sql} "
|
||||
"ORDER BY timeofban DESC LIMIT ? OFFSET ?",
|
||||
@@ -117,12 +117,13 @@ async def get_archived_history(
|
||||
|
||||
records = [
|
||||
{
|
||||
"jail": str(r[0]),
|
||||
"ip": str(r[1]),
|
||||
"timeofban": int(r[2]),
|
||||
"bancount": int(r[3]),
|
||||
"data": str(r[4]),
|
||||
"action": str(r[5]),
|
||||
"id": int(r[0]),
|
||||
"jail": str(r[1]),
|
||||
"ip": str(r[2]),
|
||||
"timeofban": int(r[3]),
|
||||
"bancount": int(r[4]),
|
||||
"data": str(r[5]),
|
||||
"action": str(r[6]),
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
@@ -137,29 +138,59 @@ async def get_all_archived_history(
|
||||
ip_filter: str | list[str] | None = None,
|
||||
origin: BanOrigin | None = None,
|
||||
action: str | None = None,
|
||||
page_size: int = 1000,
|
||||
max_rows: int = 50_000,
|
||||
last_ban_id: int | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return all archived history rows for the given filters."""
|
||||
page: int = 1
|
||||
page_size: int = 500
|
||||
"""Return archived history rows for the given filters, bounded to *max_rows*.
|
||||
|
||||
Uses keyset pagination internally for constant-time performance regardless
|
||||
of how deep into the result set we go. The caller must provide *last_ban_id*
|
||||
from the previous call to continue pagination; ``None`` starts fresh.
|
||||
|
||||
Args:
|
||||
page_size: Number of rows to fetch per internal batch (default 1000).
|
||||
max_rows: Hard cap on total rows returned (default 50 000). When
|
||||
reached the function returns even if more rows exist. Pass ``0``
|
||||
to request zero rows (useful for count-only callers).
|
||||
last_ban_id: Cursor from the previous call. ``None`` for the first
|
||||
call — the result set will start from the newest row.
|
||||
"""
|
||||
if max_rows <= 0:
|
||||
return []
|
||||
|
||||
all_rows: list[dict[str, Any]] = []
|
||||
current_last_ban_id: int | None = last_ban_id
|
||||
|
||||
while True:
|
||||
rows, total = await get_archived_history(
|
||||
batch, has_more = await get_archived_history_keyset(
|
||||
db=db,
|
||||
since=since,
|
||||
jail=jail,
|
||||
ip_filter=ip_filter,
|
||||
origin=origin,
|
||||
action=action,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
last_ban_id=current_last_ban_id,
|
||||
)
|
||||
all_rows.extend(rows)
|
||||
if len(rows) < page_size:
|
||||
if not batch:
|
||||
break
|
||||
all_rows.extend(batch)
|
||||
if len(all_rows) >= max_rows:
|
||||
break
|
||||
if not has_more:
|
||||
break
|
||||
# Use the id of the last row in the batch as the next cursor.
|
||||
# Rows are ordered id DESC, so the last row has the smallest id
|
||||
# seen in this batch and is the correct keyset anchor.
|
||||
last_row = batch[-1]
|
||||
current_last_ban_id = last_row.get("id")
|
||||
if current_last_ban_id is None:
|
||||
# Fallback: determine id from the WHERE clause of the previous query.
|
||||
# If we somehow cannot determine the id, stop to avoid an infinite loop.
|
||||
break
|
||||
page += 1
|
||||
|
||||
return all_rows
|
||||
return all_rows[:max_rows]
|
||||
|
||||
|
||||
async def purge_archived_history(db: aiosqlite.Connection, age_seconds: int) -> int:
|
||||
@@ -266,6 +297,7 @@ async def get_archived_history_keyset(
|
||||
|
||||
records = [
|
||||
{
|
||||
"id": int(r[0]),
|
||||
"jail": str(r[1]),
|
||||
"ip": str(r[2]),
|
||||
"timeofban": int(r[3]),
|
||||
|
||||
@@ -292,6 +292,9 @@ class HistoryArchiveRepository(Protocol):
|
||||
ip_filter: str | list[str] | None = None,
|
||||
origin: BanOrigin | None = None,
|
||||
action: str | None = None,
|
||||
page_size: int = 1000,
|
||||
max_rows: int = 50_000,
|
||||
last_ban_id: int | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
...
|
||||
|
||||
|
||||
Reference in New Issue
Block a user