Stage 10: external blocklist importer — backend + frontend
- blocklist_repo.py: CRUD for blocklist_sources table - import_log_repo.py: add/list/get-last log entries - blocklist_service.py: source CRUD, preview, import (download/validate/ban), import_all, schedule get/set/info - blocklist_import.py: APScheduler task (hourly/daily/weekly schedule triggers) - blocklist.py router: 9 endpoints (list/create/update/delete/preview/import/ schedule-get+put/log) - blocklist.py models: ScheduleFrequency (StrEnum), ScheduleConfig, ScheduleInfo, ImportSourceResult, ImportRunResult, PreviewResponse - 59 new tests (18 repo + 19 service + 22 router); 374 total pass - ruff clean, mypy clean for Stage 10 files - types/blocklist.ts, api/blocklist.ts, hooks/useBlocklist.ts - BlocklistsPage.tsx: source management, schedule picker, import log table - Frontend tsc + ESLint clean
This commit is contained in:
187
backend/app/repositories/blocklist_repo.py
Normal file
187
backend/app/repositories/blocklist_repo.py
Normal file
@@ -0,0 +1,187 @@
|
||||
"""Blocklist sources repository.
|
||||
|
||||
CRUD operations for the ``blocklist_sources`` table in the application
|
||||
SQLite database. All methods accept a :class:`aiosqlite.Connection` — no
|
||||
ORM, no HTTP exceptions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import aiosqlite
|
||||
|
||||
|
||||
async def create_source(
|
||||
db: aiosqlite.Connection,
|
||||
name: str,
|
||||
url: str,
|
||||
*,
|
||||
enabled: bool = True,
|
||||
) -> int:
|
||||
"""Insert a new blocklist source and return its generated id.
|
||||
|
||||
Args:
|
||||
db: Active aiosqlite connection.
|
||||
name: Human-readable display name.
|
||||
url: URL of the blocklist text file.
|
||||
enabled: Whether the source is active. Defaults to ``True``.
|
||||
|
||||
Returns:
|
||||
The ``ROWID`` / primary key of the new row.
|
||||
"""
|
||||
cursor = await db.execute(
|
||||
"""
|
||||
INSERT INTO blocklist_sources (name, url, enabled)
|
||||
VALUES (?, ?, ?)
|
||||
""",
|
||||
(name, url, int(enabled)),
|
||||
)
|
||||
await db.commit()
|
||||
return int(cursor.lastrowid) # type: ignore[arg-type]
|
||||
|
||||
|
||||
async def get_source(
|
||||
db: aiosqlite.Connection,
|
||||
source_id: int,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Return a single blocklist source row as a plain dict, or ``None``.
|
||||
|
||||
Args:
|
||||
db: Active aiosqlite connection.
|
||||
source_id: Primary key of the source to retrieve.
|
||||
|
||||
Returns:
|
||||
A dict with keys matching the ``blocklist_sources`` columns, or
|
||||
``None`` if no row with that id exists.
|
||||
"""
|
||||
async with db.execute(
|
||||
"SELECT id, name, url, enabled, created_at, updated_at FROM blocklist_sources WHERE id = ?",
|
||||
(source_id,),
|
||||
) as cursor:
|
||||
row = await cursor.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return _row_to_dict(row)
|
||||
|
||||
|
||||
async def list_sources(db: aiosqlite.Connection) -> list[dict[str, Any]]:
|
||||
"""Return all blocklist sources ordered by id ascending.
|
||||
|
||||
Args:
|
||||
db: Active aiosqlite connection.
|
||||
|
||||
Returns:
|
||||
List of dicts, one per row in ``blocklist_sources``.
|
||||
"""
|
||||
async with db.execute(
|
||||
"SELECT id, name, url, enabled, created_at, updated_at FROM blocklist_sources ORDER BY id"
|
||||
) as cursor:
|
||||
rows = await cursor.fetchall()
|
||||
return [_row_to_dict(r) for r in rows]
|
||||
|
||||
|
||||
async def list_enabled_sources(db: aiosqlite.Connection) -> list[dict[str, Any]]:
|
||||
"""Return only enabled blocklist sources ordered by id.
|
||||
|
||||
Args:
|
||||
db: Active aiosqlite connection.
|
||||
|
||||
Returns:
|
||||
List of dicts for rows where ``enabled = 1``.
|
||||
"""
|
||||
async with db.execute(
|
||||
"SELECT id, name, url, enabled, created_at, updated_at FROM blocklist_sources WHERE enabled = 1 ORDER BY id"
|
||||
) as cursor:
|
||||
rows = await cursor.fetchall()
|
||||
return [_row_to_dict(r) for r in rows]
|
||||
|
||||
|
||||
async def update_source(
|
||||
db: aiosqlite.Connection,
|
||||
source_id: int,
|
||||
*,
|
||||
name: str | None = None,
|
||||
url: str | None = None,
|
||||
enabled: bool | None = None,
|
||||
) -> bool:
|
||||
"""Update one or more fields on a blocklist source.
|
||||
|
||||
Only the keyword arguments that are not ``None`` are included in the
|
||||
``UPDATE`` statement.
|
||||
|
||||
Args:
|
||||
db: Active aiosqlite connection.
|
||||
source_id: Primary key of the source to update.
|
||||
name: New display name, or ``None`` to leave unchanged.
|
||||
url: New URL, or ``None`` to leave unchanged.
|
||||
enabled: New enabled flag, or ``None`` to leave unchanged.
|
||||
|
||||
Returns:
|
||||
``True`` if a row was updated, ``False`` if the id does not exist.
|
||||
"""
|
||||
fields: list[str] = []
|
||||
params: list[Any] = []
|
||||
|
||||
if name is not None:
|
||||
fields.append("name = ?")
|
||||
params.append(name)
|
||||
if url is not None:
|
||||
fields.append("url = ?")
|
||||
params.append(url)
|
||||
if enabled is not None:
|
||||
fields.append("enabled = ?")
|
||||
params.append(int(enabled))
|
||||
|
||||
if not fields:
|
||||
# Nothing to update — treat as success only if the row exists.
|
||||
return await get_source(db, source_id) is not None
|
||||
|
||||
fields.append("updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')")
|
||||
params.append(source_id)
|
||||
|
||||
cursor = await db.execute(
|
||||
f"UPDATE blocklist_sources SET {', '.join(fields)} WHERE id = ?", # noqa: S608
|
||||
params,
|
||||
)
|
||||
await db.commit()
|
||||
return cursor.rowcount > 0
|
||||
|
||||
|
||||
async def delete_source(db: aiosqlite.Connection, source_id: int) -> bool:
|
||||
"""Delete a blocklist source by id.
|
||||
|
||||
Args:
|
||||
db: Active aiosqlite connection.
|
||||
source_id: Primary key of the source to remove.
|
||||
|
||||
Returns:
|
||||
``True`` if a row was deleted, ``False`` if the id did not exist.
|
||||
"""
|
||||
cursor = await db.execute(
|
||||
"DELETE FROM blocklist_sources WHERE id = ?",
|
||||
(source_id,),
|
||||
)
|
||||
await db.commit()
|
||||
return cursor.rowcount > 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _row_to_dict(row: Any) -> dict[str, Any]:
|
||||
"""Convert an aiosqlite row to a plain Python dict.
|
||||
|
||||
Args:
|
||||
row: An :class:`aiosqlite.Row` or sequence returned by a cursor.
|
||||
|
||||
Returns:
|
||||
``dict`` mapping column names to values with ``enabled`` cast to
|
||||
``bool``.
|
||||
"""
|
||||
d: dict[str, Any] = dict(row)
|
||||
d["enabled"] = bool(d["enabled"])
|
||||
return d
|
||||
155
backend/app/repositories/import_log_repo.py
Normal file
155
backend/app/repositories/import_log_repo.py
Normal file
@@ -0,0 +1,155 @@
|
||||
"""Import log repository.
|
||||
|
||||
Persists and queries blocklist import run records in the ``import_log``
|
||||
table. All methods are plain async functions that accept a
|
||||
:class:`aiosqlite.Connection`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import aiosqlite
|
||||
|
||||
|
||||
async def add_log(
|
||||
db: aiosqlite.Connection,
|
||||
*,
|
||||
source_id: int | None,
|
||||
source_url: str,
|
||||
ips_imported: int,
|
||||
ips_skipped: int,
|
||||
errors: str | None,
|
||||
) -> int:
|
||||
"""Insert a new import log entry and return its id.
|
||||
|
||||
Args:
|
||||
db: Active aiosqlite connection.
|
||||
source_id: FK to ``blocklist_sources.id``, or ``None`` if the source
|
||||
has been deleted since the import ran.
|
||||
source_url: URL that was downloaded.
|
||||
ips_imported: Number of IPs successfully applied as bans.
|
||||
ips_skipped: Number of lines that were skipped (invalid or CIDR).
|
||||
errors: Error message string, or ``None`` if the import succeeded.
|
||||
|
||||
Returns:
|
||||
Primary key of the inserted row.
|
||||
"""
|
||||
cursor = await db.execute(
|
||||
"""
|
||||
INSERT INTO import_log (source_id, source_url, ips_imported, ips_skipped, errors)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""",
|
||||
(source_id, source_url, ips_imported, ips_skipped, errors),
|
||||
)
|
||||
await db.commit()
|
||||
return int(cursor.lastrowid) # type: ignore[arg-type]
|
||||
|
||||
|
||||
async def list_logs(
|
||||
db: aiosqlite.Connection,
|
||||
*,
|
||||
source_id: int | None = None,
|
||||
page: int = 1,
|
||||
page_size: int = 50,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
"""Return a paginated list of import log entries.
|
||||
|
||||
Args:
|
||||
db: Active aiosqlite connection.
|
||||
source_id: If given, filter to logs for this source only.
|
||||
page: 1-based page index.
|
||||
page_size: Number of items per page.
|
||||
|
||||
Returns:
|
||||
A 2-tuple ``(items, total)`` where *items* is a list of dicts and
|
||||
*total* is the count of all matching rows (ignoring pagination).
|
||||
"""
|
||||
where = ""
|
||||
params_count: list[Any] = []
|
||||
params_rows: list[Any] = []
|
||||
|
||||
if source_id is not None:
|
||||
where = " WHERE source_id = ?"
|
||||
params_count.append(source_id)
|
||||
params_rows.append(source_id)
|
||||
|
||||
# Total count
|
||||
async with db.execute(
|
||||
f"SELECT COUNT(*) FROM import_log{where}", # noqa: S608
|
||||
params_count,
|
||||
) as cursor:
|
||||
count_row = await cursor.fetchone()
|
||||
total: int = int(count_row[0]) if count_row else 0
|
||||
|
||||
offset = (page - 1) * page_size
|
||||
params_rows.extend([page_size, offset])
|
||||
|
||||
async with db.execute(
|
||||
f"""
|
||||
SELECT id, source_id, source_url, timestamp, ips_imported, ips_skipped, errors
|
||||
FROM import_log{where}
|
||||
ORDER BY id DESC
|
||||
LIMIT ? OFFSET ?
|
||||
""", # noqa: S608
|
||||
params_rows,
|
||||
) as cursor:
|
||||
rows = await cursor.fetchall()
|
||||
items = [_row_to_dict(r) for r in rows]
|
||||
|
||||
return items, total
|
||||
|
||||
|
||||
async def get_last_log(db: aiosqlite.Connection) -> dict[str, Any] | None:
|
||||
"""Return the most recent import log entry across all sources.
|
||||
|
||||
Args:
|
||||
db: Active aiosqlite connection.
|
||||
|
||||
Returns:
|
||||
The latest log entry as a dict, or ``None`` if no logs exist.
|
||||
"""
|
||||
async with db.execute(
|
||||
"""
|
||||
SELECT id, source_id, source_url, timestamp, ips_imported, ips_skipped, errors
|
||||
FROM import_log
|
||||
ORDER BY id DESC
|
||||
LIMIT 1
|
||||
"""
|
||||
) as cursor:
|
||||
row = await cursor.fetchone()
|
||||
return _row_to_dict(row) if row is not None else None
|
||||
|
||||
|
||||
def compute_total_pages(total: int, page_size: int) -> int:
|
||||
"""Return the total number of pages for a given total and page size.
|
||||
|
||||
Args:
|
||||
total: Total number of items.
|
||||
page_size: Items per page.
|
||||
|
||||
Returns:
|
||||
Number of pages (minimum 1).
|
||||
"""
|
||||
if total == 0:
|
||||
return 1
|
||||
return math.ceil(total / page_size)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _row_to_dict(row: Any) -> dict[str, Any]:
|
||||
"""Convert an aiosqlite row to a plain Python dict.
|
||||
|
||||
Args:
|
||||
row: An :class:`aiosqlite.Row` or sequence returned by a cursor.
|
||||
|
||||
Returns:
|
||||
Dict mapping column names to Python values.
|
||||
"""
|
||||
return dict(row)
|
||||
Reference in New Issue
Block a user