- Add /api/v1/health endpoint with component-level checks - Verify DB connectivity, fail2ban socket, scheduler, session cache - Add SQLite WAL cleanup on startup (orphan crash files) - Migration 8: import_log.timestamp → INTEGER UNIX epoch - Align import_log timestamps with history_archive (already UNIX int) - Add unit tests for DB cleanup and health router Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
234 lines
6.8 KiB
Python
234 lines
6.8 KiB
Python
"""Import log repository.
|
|
|
|
Persists and queries blocklist import run records in the ``import_log``
|
|
table. All methods are plain async functions that accept a
|
|
:class:`aiosqlite.Connection`.
|
|
|
|
Supports both offset-based and cursor-based pagination:
|
|
|
|
- **Offset pagination** (legacy): ``list_logs(page=2, page_size=50)`` - query-efficient
|
|
but degrades on large offsets.
|
|
|
|
- **Cursor pagination** (recommended): ``list_logs_keyset(page_size=50, last_log_id=None)``
|
|
- constant-time performance regardless of dataset size.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
from typing import TYPE_CHECKING, cast
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Mapping
|
|
|
|
import aiosqlite
|
|
|
|
from app.models.blocklist import ImportLogEntry
|
|
|
|
# Alias for backward compatibility with protocols
|
|
ImportLogRow = ImportLogEntry
|
|
async def add_log(
|
|
db: aiosqlite.Connection,
|
|
*,
|
|
source_id: int | None,
|
|
source_url: str,
|
|
ips_imported: int,
|
|
ips_skipped: int,
|
|
errors: str | None,
|
|
) -> int:
|
|
"""Insert a new import log entry and return its id.
|
|
|
|
Args:
|
|
db: Active aiosqlite connection.
|
|
source_id: FK to ``blocklist_sources.id``, or ``None`` if the source
|
|
has been deleted since the import ran.
|
|
source_url: URL that was downloaded.
|
|
ips_imported: Number of IPs successfully applied as bans.
|
|
ips_skipped: Number of lines that were skipped (invalid or CIDR).
|
|
errors: Error message string, or ``None`` if the import succeeded.
|
|
|
|
Returns:
|
|
Primary key of the inserted row.
|
|
"""
|
|
import time
|
|
|
|
timestamp_unix: int = int(time.time())
|
|
cursor = await db.execute(
|
|
"""
|
|
INSERT INTO import_log (source_id, source_url, timestamp, ips_imported, ips_skipped, errors)
|
|
VALUES (?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(source_id, source_url, timestamp_unix, ips_imported, ips_skipped, errors),
|
|
)
|
|
await db.commit()
|
|
return int(cursor.lastrowid) # type: ignore[arg-type]
|
|
|
|
|
|
async def list_logs(
|
|
db: aiosqlite.Connection,
|
|
*,
|
|
source_id: int | None = None,
|
|
page: int = 1,
|
|
page_size: int = 50,
|
|
) -> tuple[list[ImportLogRow], int]:
|
|
"""Return a paginated list of import log entries.
|
|
|
|
Args:
|
|
db: Active aiosqlite connection.
|
|
source_id: If given, filter to logs for this source only.
|
|
page: 1-based page index.
|
|
page_size: Number of items per page.
|
|
|
|
Returns:
|
|
A 2-tuple ``(items, total)`` where *items* is a list of dicts and
|
|
*total* is the count of all matching rows (ignoring pagination).
|
|
"""
|
|
where = ""
|
|
params_count: list[object] = []
|
|
params_rows: list[object] = []
|
|
|
|
if source_id is not None:
|
|
where = " WHERE source_id = ?"
|
|
params_count.append(source_id)
|
|
params_rows.append(source_id)
|
|
|
|
# Total count
|
|
async with db.execute(
|
|
f"SELECT COUNT(*) FROM import_log{where}", # noqa: S608
|
|
params_count,
|
|
) as cursor:
|
|
count_row = await cursor.fetchone()
|
|
total: int = int(count_row[0]) if count_row else 0
|
|
|
|
offset = (page - 1) * page_size
|
|
params_rows.extend([page_size, offset])
|
|
|
|
async with db.execute(
|
|
f"""
|
|
SELECT id, source_id, source_url, timestamp, ips_imported, ips_skipped, errors
|
|
FROM import_log{where}
|
|
ORDER BY id DESC
|
|
LIMIT ? OFFSET ?
|
|
""", # noqa: S608
|
|
params_rows,
|
|
) as cursor:
|
|
rows = await cursor.fetchall()
|
|
items = [_row_to_dict(r) for r in rows]
|
|
|
|
return items, total
|
|
|
|
|
|
async def get_last_log(db: aiosqlite.Connection) -> ImportLogRow | None:
|
|
"""Return the most recent import log entry across all sources.
|
|
|
|
Args:
|
|
db: Active aiosqlite connection.
|
|
|
|
Returns:
|
|
The latest log entry as a dict, or ``None`` if no logs exist.
|
|
"""
|
|
async with db.execute(
|
|
"""
|
|
SELECT id, source_id, source_url, timestamp, ips_imported, ips_skipped, errors
|
|
FROM import_log
|
|
ORDER BY id DESC
|
|
LIMIT 1
|
|
"""
|
|
) as cursor:
|
|
row = await cursor.fetchone()
|
|
return _row_to_dict(row) if row is not None else None
|
|
|
|
|
|
def compute_total_pages(total: int, page_size: int) -> int:
|
|
"""Return the total number of pages for a given total and page size.
|
|
|
|
Args:
|
|
total: Total number of items.
|
|
page_size: Items per page.
|
|
|
|
Returns:
|
|
Number of pages (minimum 1).
|
|
"""
|
|
if total == 0:
|
|
return 1
|
|
return math.ceil(total / page_size)
|
|
|
|
|
|
async def list_logs_keyset(
|
|
db: aiosqlite.Connection,
|
|
*,
|
|
source_id: int | None = None,
|
|
page_size: int = 50,
|
|
last_log_id: int | None = None,
|
|
) -> tuple[list[ImportLogRow], bool]:
|
|
"""Return a cursor-paginated list of import log entries.
|
|
|
|
Uses keyset pagination (WHERE id < last_id) for constant-time performance
|
|
regardless of result set size. This is the recommended pagination method
|
|
for large result sets.
|
|
|
|
Args:
|
|
db: Active aiosqlite connection.
|
|
source_id: If given, filter to logs for this source only.
|
|
page_size: Number of items per page (max returned is page_size + 1 to detect overflow).
|
|
last_log_id: The ID of the last item from the previous page (for cursor).
|
|
None for the first page.
|
|
|
|
Returns:
|
|
A 2-tuple ``(items, has_more)`` where:
|
|
- *items* is a list of up to page_size ImportLogEntry objects
|
|
- *has_more* is True if there are additional pages beyond this one
|
|
"""
|
|
where = ""
|
|
params: list[object] = []
|
|
|
|
if source_id is not None:
|
|
where = " WHERE source_id = ?"
|
|
params.append(source_id)
|
|
|
|
if last_log_id is not None:
|
|
if where:
|
|
where += " AND id < ?"
|
|
else:
|
|
where = " WHERE id < ?"
|
|
params.append(last_log_id)
|
|
|
|
# Fetch page_size + 1 to detect if there are more pages
|
|
fetch_limit = page_size + 1
|
|
params.append(fetch_limit)
|
|
|
|
async with db.execute(
|
|
f"""
|
|
SELECT id, source_id, source_url, timestamp, ips_imported, ips_skipped, errors
|
|
FROM import_log{where}
|
|
ORDER BY id DESC
|
|
LIMIT ?
|
|
""", # noqa: S608
|
|
params,
|
|
) as cursor:
|
|
rows_iterable = await cursor.fetchall()
|
|
rows = list(rows_iterable)
|
|
items = [_row_to_dict(r) for r in rows[:page_size]]
|
|
has_more = len(rows) > page_size
|
|
|
|
return items, has_more
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Internal helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _row_to_dict(row: object) -> ImportLogRow:
|
|
"""Convert an aiosqlite row to an ImportLogEntry Pydantic model.
|
|
|
|
Args:
|
|
row: An :class:`aiosqlite.Row` or similar mapping returned by a cursor.
|
|
|
|
Returns:
|
|
ImportLogEntry Pydantic model instance.
|
|
"""
|
|
from typing import Any as AnyType
|
|
mapping = cast("Mapping[str, AnyType]", row)
|
|
return ImportLogEntry.model_validate(dict(mapping))
|