Refactor pagination with cursor-based support and standardized response format
- Implement cursor-based pagination in pagination.py - Update response models to standardize pagination structure - Add cursor pagination utilities for repositories - Update HistoryArchiveRepository and ImportLogRepository with new pagination - Add comprehensive tests for cursor pagination - Update documentation for backend development and task tracking Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
@@ -3,6 +3,14 @@
|
||||
Persists and queries blocklist import run records in the ``import_log``
|
||||
table. All methods are plain async functions that accept a
|
||||
:class:`aiosqlite.Connection`.
|
||||
|
||||
Supports both offset-based and cursor-based pagination:
|
||||
|
||||
- **Offset pagination** (legacy): ``list_logs(page=2, page_size=50)`` - query-efficient
|
||||
but degrades on large offsets.
|
||||
|
||||
- **Cursor pagination** (recommended): ``list_logs_keyset(page_size=50, last_log_id=None)``
|
||||
- constant-time performance regardless of dataset size.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -17,7 +25,6 @@ if TYPE_CHECKING:
|
||||
|
||||
from app.models.blocklist import ImportLogEntry
|
||||
|
||||
|
||||
# Alias for backward compatibility with protocols
|
||||
ImportLogRow = ImportLogEntry
|
||||
async def add_log(
|
||||
@@ -144,6 +151,66 @@ def compute_total_pages(total: int, page_size: int) -> int:
|
||||
return math.ceil(total / page_size)
|
||||
|
||||
|
||||
async def list_logs_keyset(
|
||||
db: aiosqlite.Connection,
|
||||
*,
|
||||
source_id: int | None = None,
|
||||
page_size: int = 50,
|
||||
last_log_id: int | None = None,
|
||||
) -> tuple[list[ImportLogRow], bool]:
|
||||
"""Return a cursor-paginated list of import log entries.
|
||||
|
||||
Uses keyset pagination (WHERE id < last_id) for constant-time performance
|
||||
regardless of result set size. This is the recommended pagination method
|
||||
for large result sets.
|
||||
|
||||
Args:
|
||||
db: Active aiosqlite connection.
|
||||
source_id: If given, filter to logs for this source only.
|
||||
page_size: Number of items per page (max returned is page_size + 1 to detect overflow).
|
||||
last_log_id: The ID of the last item from the previous page (for cursor).
|
||||
None for the first page.
|
||||
|
||||
Returns:
|
||||
A 2-tuple ``(items, has_more)`` where:
|
||||
- *items* is a list of up to page_size ImportLogEntry objects
|
||||
- *has_more* is True if there are additional pages beyond this one
|
||||
"""
|
||||
where = ""
|
||||
params: list[object] = []
|
||||
|
||||
if source_id is not None:
|
||||
where = " WHERE source_id = ?"
|
||||
params.append(source_id)
|
||||
|
||||
if last_log_id is not None:
|
||||
if where:
|
||||
where += " AND id < ?"
|
||||
else:
|
||||
where = " WHERE id < ?"
|
||||
params.append(last_log_id)
|
||||
|
||||
# Fetch page_size + 1 to detect if there are more pages
|
||||
fetch_limit = page_size + 1
|
||||
params.append(fetch_limit)
|
||||
|
||||
async with db.execute(
|
||||
f"""
|
||||
SELECT id, source_id, source_url, timestamp, ips_imported, ips_skipped, errors
|
||||
FROM import_log{where}
|
||||
ORDER BY id DESC
|
||||
LIMIT ?
|
||||
""", # noqa: S608
|
||||
params,
|
||||
) as cursor:
|
||||
rows_iterable = await cursor.fetchall()
|
||||
rows = list(rows_iterable)
|
||||
items = [_row_to_dict(r) for r in rows[:page_size]]
|
||||
has_more = len(rows) > page_size
|
||||
|
||||
return items, has_more
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -158,5 +225,6 @@ def _row_to_dict(row: object) -> ImportLogRow:
|
||||
Returns:
|
||||
ImportLogEntry Pydantic model instance.
|
||||
"""
|
||||
mapping = cast("Mapping[str, object]", row)
|
||||
return ImportLogEntry(**mapping)
|
||||
from typing import Any as AnyType
|
||||
mapping = cast("Mapping[str, AnyType]", row)
|
||||
return ImportLogEntry.model_validate(dict(mapping))
|
||||
|
||||
Reference in New Issue
Block a user