lukas.pupkalipinski/BanGUI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Complete history archive support for dashboard/map data and mark task finished

Browse Source 
Add source=archive option for dashboard endpoints and history service; update Docs/Tasks.md; include archive branch for list_bans, bans_by_country, ban_trend, bans_by_jail; tests for archive paths.
This commit is contained in:
Lukas
2026-03-28 12:39:47 +01:00
parent 876af46955
commit 9f05da2d4d
13 changed files with 714 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
backend/app/db.py
View File

@@ -75,6 +75,20 @@ CREATE TABLE IF NOT EXISTS geo_cache (
);
"""
_CREATE_HISTORY_ARCHIVE: str = """
CREATE TABLE IF NOT EXISTS history_archive (
id INTEGER PRIMARY KEY AUTOINCREMENT,
jail TEXT NOT NULL,
ip TEXT NOT NULL,
timeofban INTEGER NOT NULL,
bancount INTEGER NOT NULL,
data TEXT NOT NULL,
action TEXT NOT NULL CHECK(action IN ('ban', 'unban')),
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
UNIQUE(ip, jail, action, timeofban)
);
"""
# Ordered list of DDL statements to execute on initialisation.
_SCHEMA_STATEMENTS: list[str] = [
_CREATE_SETTINGS,
@@ -83,6 +97,7 @@ _SCHEMA_STATEMENTS: list[str] = [
_CREATE_BLOCKLIST_SOURCES,
_CREATE_IMPORT_LOG,
_CREATE_GEO_CACHE,
_CREATE_HISTORY_ARCHIVE,
]

5
backend/app/main.py
View File

@@ -48,7 +48,7 @@ from app.routers import (
server,
setup,
)
from app.tasks import blocklist_import, geo_cache_flush, geo_re_resolve, health_check
from app.tasks import blocklist_import, geo_cache_flush, geo_re_resolve, health_check, history_sync
from app.utils.fail2ban_client import Fail2BanConnectionError, Fail2BanProtocolError
from app.utils.jail_config import ensure_jail_configs
@@ -183,6 +183,9 @@ async def _lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
# --- Periodic re-resolve of NULL-country geo entries ---
geo_re_resolve.register(app)
# --- Periodic history sync from fail2ban into BanGUI archive ---
history_sync.register(app)
log.info("bangui_started")
try:

47
backend/app/repositories/history_archive_repo.py
View File

@@ -9,6 +9,8 @@ from __future__ import annotations
import datetime
from typing import TYPE_CHECKING
from app.models.ban import BLOCKLIST_JAIL, BanOrigin
if TYPE_CHECKING:
import aiosqlite
@@ -39,6 +41,7 @@ async def get_archived_history(
since: int | None = None,
jail: str | None = None,
ip_filter: str | None = None,
origin: BanOrigin | None = None,
action: str | None = None,
page: int = 1,
page_size: int = 100,
@@ -59,6 +62,13 @@ async def get_archived_history(
wheres.append("ip LIKE ?")
params.append(f"{ip_filter}%")
if origin == "blocklist":
wheres.append("jail = ?")
params.append(BLOCKLIST_JAIL)
elif origin == "selfblock":
wheres.append("jail != ?")
params.append(BLOCKLIST_JAIL)
if action is not None:
wheres.append("action = ?")
params.append(action)
@@ -71,7 +81,10 @@ async def get_archived_history(
total = int(row[0]) if row is not None and row[0] is not None else 0
async with db.execute(
f"SELECT jail, ip, timeofban, bancount, data, action FROM history_archive {where_sql} ORDER BY timeofban DESC LIMIT ? OFFSET ?",
"SELECT jail, ip, timeofban, bancount, data, action "
"FROM history_archive "
f"{where_sql} "
"ORDER BY timeofban DESC LIMIT ? OFFSET ?",
[*params, page_size, offset],
) as cur:
rows = await cur.fetchall()
@@ -91,6 +104,38 @@ async def get_archived_history(
return records, total
async def get_all_archived_history(
db: aiosqlite.Connection,
since: int | None = None,
jail: str | None = None,
ip_filter: str | None = None,
origin: BanOrigin | None = None,
action: str | None = None,
) -> list[dict]:
"""Return all archived history rows for the given filters."""
page: int = 1
page_size: int = 500
all_rows: list[dict] = []
while True:
rows, total = await get_archived_history(
db=db,
since=since,
jail=jail,
ip_filter=ip_filter,
origin=origin,
action=action,
page=page,
page_size=page_size,
)
all_rows.extend(rows)
if len(rows) < page_size:
break
page += 1
return all_rows
async def purge_archived_history(db: aiosqlite.Connection, age_seconds: int) -> int:
"""Purge archived entries older than *age_seconds*; return rows deleted."""
threshold = int(datetime.datetime.now(datetime.UTC).timestamp()) - age_seconds

26
backend/app/routers/dashboard.py
View File

@@ -83,6 +83,7 @@ async def get_dashboard_bans(
request: Request,
_auth: AuthDep,
range: TimeRange = Query(default=_DEFAULT_RANGE, description="Time-range preset."),
source: str = Query(default="fail2ban", description="Data source: 'fail2ban' or 'archive'."),
page: int = Query(default=1, ge=1, description="1-based page number."),
page_size: int = Query(default=_DEFAULT_PAGE_SIZE, ge=1, le=500, description="Items per page."),
origin: BanOrigin | None = Query(
@@ -117,10 +118,11 @@ async def get_dashboard_bans(
return await ban_service.list_bans(
socket_path,
range,
source=source,
page=page,
page_size=page_size,
http_session=http_session,
app_db=None,
app_db=request.app.state.db,
geo_batch_lookup=geo_service.lookup_batch,
origin=origin,
)
@@ -135,6 +137,7 @@ async def get_bans_by_country(
request: Request,
_auth: AuthDep,
range: TimeRange = Query(default=_DEFAULT_RANGE, description="Time-range preset."),
source: str = Query(default="fail2ban", description="Data source: 'fail2ban' or 'archive'."),
origin: BanOrigin | None = Query(
default=None,
description="Filter by ban origin: 'blocklist' or 'selfblock'. Omit for all.",
@@ -164,10 +167,11 @@ async def get_bans_by_country(
return await ban_service.bans_by_country(
socket_path,
range,
source=source,
http_session=http_session,
geo_cache_lookup=geo_service.lookup_cached_only,
geo_batch_lookup=geo_service.lookup_batch,
app_db=None,
app_db=request.app.state.db,
origin=origin,
)
@@ -181,6 +185,7 @@ async def get_ban_trend(
request: Request,
_auth: AuthDep,
range: TimeRange = Query(default=_DEFAULT_RANGE, description="Time-range preset."),
source: str = Query(default="fail2ban", description="Data source: 'fail2ban' or 'archive'."),
origin: BanOrigin | None = Query(
default=None,
description="Filter by ban origin: 'blocklist' or 'selfblock'. Omit for all.",
@@ -212,7 +217,13 @@ async def get_ban_trend(
"""
socket_path: str = request.app.state.settings.fail2ban_socket
return await ban_service.ban_trend(socket_path, range, origin=origin)
return await ban_service.ban_trend(
socket_path,
range,
source=source,
app_db=request.app.state.db,
origin=origin,
)
@router.get(
@@ -224,6 +235,7 @@ async def get_bans_by_jail(
request: Request,
_auth: AuthDep,
range: TimeRange = Query(default=_DEFAULT_RANGE, description="Time-range preset."),
source: str = Query(default="fail2ban", description="Data source: 'fail2ban' or 'archive'."),
origin: BanOrigin | None = Query(
default=None,
description="Filter by ban origin: 'blocklist' or 'selfblock'. Omit for all.",
@@ -248,4 +260,10 @@ async def get_bans_by_jail(
"""
socket_path: str = request.app.state.settings.fail2ban_socket
return await ban_service.bans_by_jail(socket_path, range, origin=origin)
return await ban_service.bans_by_jail(
socket_path,
range,
source=source,
app_db=request.app.state.db,
origin=origin,
)

36
backend/app/routers/history.py
View File

@@ -186,39 +186,3 @@ async def get_ip_history(
raise HTTPException(status_code=404, detail=f"No history found for IP {ip!r}.")
return detail
@router.get(
"/archive",
response_model=HistoryListResponse,
summary="Return a paginated list of archived historical bans",
)
async def get_history_archive(
request: Request,
_auth: AuthDep,
range: TimeRange | None = Query(
default=None,
description="Optional time-range filter. Omit for all-time.",
),
jail: str | None = Query(default=None, description="Restrict results to this jail name."),
ip: str | None = Query(default=None, description="Restrict results to IPs matching this prefix."),
page: int = Query(default=1, ge=1, description="1-based page number."),
page_size: int = Query(default=_DEFAULT_PAGE_SIZE, ge=1, le=500, description="Items per page (max 500)."),
) -> HistoryListResponse:
socket_path: str = request.app.state.settings.fail2ban_socket
http_session: aiohttp.ClientSession = request.app.state.http_session
async def _enricher(addr: str) -> geo_service.GeoInfo | None:
return await geo_service.lookup(addr, http_session)
return await history_service.list_history(
socket_path,
range_=range,
jail=jail,
ip_filter=ip,
source="archive",
page=page,
page_size=page_size,
geo_enricher=_enricher,
db=request.app.state.db,
)

386
backend/app/services/ban_service.py
View File

@@ -112,6 +112,7 @@ async def list_bans(
socket_path: str,
range_: TimeRange,
*,
source: str = "fail2ban",
page: int = 1,
page_size: int = _DEFAULT_PAGE_SIZE,
http_session: aiohttp.ClientSession | None = None,
@@ -160,24 +161,41 @@ async def list_bans(
since: int = _since_unix(range_)
effective_page_size: int = min(page_size, _MAX_PAGE_SIZE)
offset: int = (page - 1) * effective_page_size
origin_clause, origin_params = _origin_sql_filter(origin)
db_path: str = await get_fail2ban_db_path(socket_path)
log.info(
"ban_service_list_bans",
db_path=db_path,
since=since,
range=range_,
origin=origin,
)
if source not in ("fail2ban", "archive"):
raise ValueError(f"Unsupported source: {source!r}")
rows, total = await fail2ban_db_repo.get_currently_banned(
db_path=db_path,
since=since,
origin=origin,
limit=effective_page_size,
offset=offset,
)
if source == "archive":
if app_db is None:
raise ValueError("app_db must be provided when source is 'archive'")
from app.repositories.history_archive_repo import get_archived_history
rows, total = await get_archived_history(
db=app_db,
since=since,
origin=origin,
action="ban",
page=page,
page_size=effective_page_size,
)
else:
db_path: str = await get_fail2ban_db_path(socket_path)
log.info(
"ban_service_list_bans",
db_path=db_path,
since=since,
range=range_,
origin=origin,
)
rows, total = await fail2ban_db_repo.get_currently_banned(
db_path=db_path,
since=since,
origin=origin,
limit=effective_page_size,
offset=offset,
)
# Batch-resolve geo data for all IPs on this page in a single API call.
# This avoids hitting the 45 req/min single-IP rate limit when the
@@ -192,11 +210,19 @@ async def list_bans(
items: list[DashboardBanItem] = []
for row in rows:
jail: str = row.jail
ip: str = row.ip
banned_at: str = ts_to_iso(row.timeofban)
ban_count: int = row.bancount
matches, _ = parse_data_json(row.data)
if source == "archive":
jail = str(row["jail"])
ip = str(row["ip"])
banned_at = ts_to_iso(int(row["timeofban"]))
ban_count = int(row["bancount"])
matches, _ = parse_data_json(row["data"])
else:
jail = row.jail
ip = row.ip
banned_at = ts_to_iso(row.timeofban)
ban_count = row.bancount
matches, _ = parse_data_json(row.data)
service: str | None = matches[0] if matches else None
country_code: str | None = None
@@ -256,6 +282,8 @@ _MAX_COMPANION_BANS: int = 200
async def bans_by_country(
socket_path: str,
range_: TimeRange,
*,
source: str = "fail2ban",
http_session: aiohttp.ClientSession | None = None,
geo_cache_lookup: GeoCacheLookup | None = None,
geo_batch_lookup: GeoBatchLookup | None = None,
@@ -300,41 +328,80 @@ async def bans_by_country(
"""
since: int = _since_unix(range_)
origin_clause, origin_params = _origin_sql_filter(origin)
db_path: str = await get_fail2ban_db_path(socket_path)
log.info(
"ban_service_bans_by_country",
db_path=db_path,
since=since,
range=range_,
origin=origin,
)
# Total count and companion rows reuse the same SQL query logic.
# Passing limit=0 returns only the total from the count query.
_, total = await fail2ban_db_repo.get_currently_banned(
db_path=db_path,
since=since,
origin=origin,
limit=0,
offset=0,
)
if source not in ("fail2ban", "archive"):
raise ValueError(f"Unsupported source: {source!r}")
agg_rows = await fail2ban_db_repo.get_ban_event_counts(
db_path=db_path,
since=since,
origin=origin,
)
if source == "archive":
if app_db is None:
raise ValueError("app_db must be provided when source is 'archive'")
companion_rows, _ = await fail2ban_db_repo.get_currently_banned(
db_path=db_path,
since=since,
origin=origin,
limit=_MAX_COMPANION_BANS,
offset=0,
)
from app.repositories.history_archive_repo import (
get_all_archived_history,
get_archived_history,
)
unique_ips: list[str] = [r.ip for r in agg_rows]
all_rows = await get_all_archived_history(
db=app_db,
since=since,
origin=origin,
action="ban",
)
total = len(all_rows)
# companion rows for the table should be most recent
companion_rows, _ = await get_archived_history(
db=app_db,
since=since,
origin=origin,
action="ban",
page=1,
page_size=_MAX_COMPANION_BANS,
)
agg_rows = {}
for row in all_rows:
ip = str(row["ip"])
agg_rows[ip] = agg_rows.get(ip, 0) + 1
unique_ips = list(agg_rows.keys())
else:
origin_clause, origin_params = _origin_sql_filter(origin)
db_path: str = await get_fail2ban_db_path(socket_path)
log.info(
"ban_service_bans_by_country",
db_path=db_path,
since=since,
range=range_,
origin=origin,
)
# Total count and companion rows reuse the same SQL query logic.
# Passing limit=0 returns only the total from the count query.
_, total = await fail2ban_db_repo.get_currently_banned(
db_path=db_path,
since=since,
origin=origin,
limit=0,
offset=0,
)
agg_rows = await fail2ban_db_repo.get_ban_event_counts(
db_path=db_path,
since=since,
origin=origin,
)
companion_rows, _ = await fail2ban_db_repo.get_currently_banned(
db_path=db_path,
since=since,
origin=origin,
limit=_MAX_COMPANION_BANS,
offset=0,
)
unique_ips = [r.ip for r in agg_rows]
geo_map: dict[str, GeoInfo] = {}
if http_session is not None and unique_ips and geo_cache_lookup is not None:
@@ -371,12 +438,28 @@ async def bans_by_country(
countries: dict[str, int] = {}
country_names: dict[str, str] = {}
for agg_row in agg_rows:
ip: str = agg_row.ip
if source == "archive":
agg_items = [
{
"ip": ip,
"event_count": count,
}
for ip, count in agg_rows.items()
]
else:
agg_items = agg_rows
for agg_row in agg_items:
if source == "archive":
ip = agg_row["ip"]
event_count = agg_row["event_count"]
else:
ip = agg_row.ip
event_count = agg_row.event_count
geo = geo_map.get(ip)
cc: str | None = geo.country_code if geo else None
cn: str | None = geo.country_name if geo else None
event_count: int = agg_row.event_count
if cc:
countries[cc] = countries.get(cc, 0) + event_count
@@ -386,26 +469,38 @@ async def bans_by_country(
# Build companion table from recent rows (geo already cached from batch step).
bans: list[DashboardBanItem] = []
for companion_row in companion_rows:
ip = companion_row.ip
if source == "archive":
ip = companion_row["ip"]
jail = companion_row["jail"]
banned_at = ts_to_iso(int(companion_row["timeofban"]))
ban_count = int(companion_row["bancount"])
service = None
else:
ip = companion_row.ip
jail = companion_row.jail
banned_at = ts_to_iso(companion_row.timeofban)
ban_count = companion_row.bancount
matches, _ = parse_data_json(companion_row.data)
service = matches[0] if matches else None
geo = geo_map.get(ip)
cc = geo.country_code if geo else None
cn = geo.country_name if geo else None
asn: str | None = geo.asn if geo else None
org: str | None = geo.org if geo else None
matches, _ = parse_data_json(companion_row.data)
bans.append(
DashboardBanItem(
ip=ip,
jail=companion_row.jail,
banned_at=ts_to_iso(companion_row.timeofban),
service=matches[0] if matches else None,
jail=jail,
banned_at=banned_at,
service=service,
country_code=cc,
country_name=cn,
asn=asn,
org=org,
ban_count=companion_row.bancount,
origin=_derive_origin(companion_row.jail),
ban_count=ban_count,
origin=_derive_origin(jail),
)
)
@@ -426,6 +521,8 @@ async def ban_trend(
socket_path: str,
range_: TimeRange,
*,
source: str = "fail2ban",
app_db: aiosqlite.Connection | None = None,
origin: BanOrigin | None = None,
) -> BanTrendResponse:
"""Return ban counts aggregated into equal-width time buckets.
@@ -457,26 +554,58 @@ async def ban_trend(
since: int = _since_unix(range_)
bucket_secs: int = BUCKET_SECONDS[range_]
num_buckets: int = bucket_count(range_)
origin_clause, origin_params = _origin_sql_filter(origin)
db_path: str = await get_fail2ban_db_path(socket_path)
log.info(
"ban_service_ban_trend",
db_path=db_path,
since=since,
range=range_,
origin=origin,
bucket_secs=bucket_secs,
num_buckets=num_buckets,
)
if source not in ("fail2ban", "archive"):
raise ValueError(f"Unsupported source: {source!r}")
counts = await fail2ban_db_repo.get_ban_counts_by_bucket(
db_path=db_path,
since=since,
bucket_secs=bucket_secs,
num_buckets=num_buckets,
origin=origin,
)
if source == "archive":
if app_db is None:
raise ValueError("app_db must be provided when source is 'archive'")
from app.repositories.history_archive_repo import get_all_archived_history
all_rows = await get_all_archived_history(
db=app_db,
since=since,
origin=origin,
action="ban",
)
counts: list[int] = [0] * num_buckets
for row in all_rows:
timeofban = int(row["timeofban"])
bucket_index = int((timeofban - since) / bucket_secs)
if 0 <= bucket_index < num_buckets:
counts[bucket_index] += 1
log.info(
"ban_service_ban_trend",
source=source,
since=since,
range=range_,
origin=origin,
bucket_secs=bucket_secs,
num_buckets=num_buckets,
)
else:
db_path: str = await get_fail2ban_db_path(socket_path)
log.info(
"ban_service_ban_trend",
db_path=db_path,
since=since,
range=range_,
origin=origin,
bucket_secs=bucket_secs,
num_buckets=num_buckets,
)
counts = await fail2ban_db_repo.get_ban_counts_by_bucket(
db_path=db_path,
since=since,
bucket_secs=bucket_secs,
num_buckets=num_buckets,
origin=origin,
)
buckets: list[BanTrendBucket] = [
BanTrendBucket(
@@ -501,6 +630,8 @@ async def bans_by_jail(
socket_path: str,
range_: TimeRange,
*,
source: str = "fail2ban",
app_db: aiosqlite.Connection | None = None,
origin: BanOrigin | None = None,
) -> BansByJailResponse:
"""Return ban counts aggregated per jail for the selected time window.
@@ -522,38 +653,75 @@ async def bans_by_jail(
sorted descending and the total ban count.
"""
since: int = _since_unix(range_)
origin_clause, origin_params = _origin_sql_filter(origin)
db_path: str = await get_fail2ban_db_path(socket_path)
log.debug(
"ban_service_bans_by_jail",
db_path=db_path,
since=since,
since_iso=ts_to_iso(since),
range=range_,
origin=origin,
)
if source not in ("fail2ban", "archive"):
raise ValueError(f"Unsupported source: {source!r}")
total, jail_counts = await fail2ban_db_repo.get_bans_by_jail(
db_path=db_path,
since=since,
origin=origin,
)
if source == "archive":
if app_db is None:
raise ValueError("app_db must be provided when source is 'archive'")
# Diagnostic guard: if zero results were returned, check whether the table
# has *any* rows and log a warning with min/max timeofban so operators can
# diagnose timezone or filter mismatches from logs.
if total == 0:
table_row_count, min_timeofban, max_timeofban = await fail2ban_db_repo.get_bans_table_summary(db_path)
if table_row_count > 0:
log.warning(
"ban_service_bans_by_jail_empty_despite_data",
table_row_count=table_row_count,
min_timeofban=min_timeofban,
max_timeofban=max_timeofban,
since=since,
range=range_,
)
from app.repositories.history_archive_repo import get_all_archived_history
all_rows = await get_all_archived_history(
db=app_db,
since=since,
origin=origin,
action="ban",
)
jail_counter: dict[str, int] = {}
for row in all_rows:
jail_name = str(row["jail"])
jail_counter[jail_name] = jail_counter.get(jail_name, 0) + 1
total = sum(jail_counter.values())
jail_counts = [
JailBanCountModel(jail=jail_name, count=count)
for jail_name, count in sorted(jail_counter.items(), key=lambda x: x[1], reverse=True)
]
log.debug(
"ban_service_bans_by_jail",
source=source,
since=since,
since_iso=ts_to_iso(since),
range=range_,
origin=origin,
)
else:
origin_clause, origin_params = _origin_sql_filter(origin)
db_path: str = await get_fail2ban_db_path(socket_path)
log.debug(
"ban_service_bans_by_jail",
db_path=db_path,
since=since,
since_iso=ts_to_iso(since),
range=range_,
origin=origin,
)
total, jail_counts = await fail2ban_db_repo.get_bans_by_jail(
db_path=db_path,
since=since,
origin=origin,
)
# Diagnostic guard: if zero results were returned, check whether the table
# has *any* rows and log a warning with min/max timeofban so operators can
# diagnose timezone or filter mismatches from logs.
if total == 0:
table_row_count, min_timeofban, max_timeofban = await fail2ban_db_repo.get_bans_table_summary(db_path)
if table_row_count > 0:
log.warning(
"ban_service_bans_by_jail_empty_despite_data",
table_row_count=table_row_count,
min_timeofban=min_timeofban,
max_timeofban=max_timeofban,
since=since,
range=range_,
)
log.debug(
"ban_service_bans_by_jail_result",

146
backend/app/services/history_service.py
View File

@@ -16,6 +16,8 @@ from typing import TYPE_CHECKING
import structlog
if TYPE_CHECKING:
import aiosqlite
from app.models.geo import GeoEnricher
from app.models.ban import TIME_RANGE_SECONDS, BanOrigin, TimeRange
@@ -63,9 +65,11 @@ async def list_history(
jail: str | None = None,
ip_filter: str | None = None,
origin: BanOrigin | None = None,
source: str = "fail2ban",
page: int = 1,
page_size: int = _DEFAULT_PAGE_SIZE,
geo_enricher: GeoEnricher | None = None,
db: aiosqlite.Connection | None = None,
) -> HistoryListResponse:
"""Return a paginated list of historical ban records with optional filters.
@@ -104,55 +108,111 @@ async def list_history(
page=page,
)
rows, total = await fail2ban_db_repo.get_history_page(
db_path=db_path,
since=since,
jail=jail,
ip_filter=ip_filter,
origin=origin,
page=page,
page_size=effective_page_size,
)
items: list[HistoryBanItem] = []
for row in rows:
jail_name: str = row.jail
ip: str = row.ip
banned_at: str = ts_to_iso(row.timeofban)
ban_count: int = row.bancount
matches, failures = parse_data_json(row.data)
total: int
country_code: str | None = None
country_name: str | None = None
asn: str | None = None
org: str | None = None
if source == "archive":
if db is None:
raise ValueError("db must be provided when source is 'archive'")
if geo_enricher is not None:
try:
geo = await geo_enricher(ip)
if geo is not None:
country_code = geo.country_code
country_name = geo.country_name
asn = geo.asn
org = geo.org
except Exception: # noqa: BLE001
log.warning("history_service_geo_lookup_failed", ip=ip)
from app.repositories.history_archive_repo import get_archived_history
items.append(
HistoryBanItem(
ip=ip,
jail=jail_name,
banned_at=banned_at,
ban_count=ban_count,
failures=failures,
matches=matches,
country_code=country_code,
country_name=country_name,
asn=asn,
org=org,
)
archived_rows, total = await get_archived_history(
db=db,
since=since,
jail=jail,
ip_filter=ip_filter,
page=page,
page_size=effective_page_size,
)
for row in archived_rows:
jail_name = row["jail"]
ip = row["ip"]
banned_at = ts_to_iso(int(row["timeofban"]))
ban_count = int(row["bancount"])
matches, failures = parse_data_json(row["data"])
# archive records may include actions; we treat all as history
country_code = None
country_name = None
asn = None
org = None
if geo_enricher is not None:
try:
geo = await geo_enricher(ip)
if geo is not None:
country_code = geo.country_code
country_name = geo.country_name
asn = geo.asn
org = geo.org
except Exception: # noqa: BLE001
log.warning("history_service_geo_lookup_failed", ip=ip)
items.append(
HistoryBanItem(
ip=ip,
jail=jail_name,
banned_at=banned_at,
ban_count=ban_count,
failures=failures,
matches=matches,
country_code=country_code,
country_name=country_name,
asn=asn,
org=org,
)
)
else:
rows, total = await fail2ban_db_repo.get_history_page(
db_path=db_path,
since=since,
jail=jail,
ip_filter=ip_filter,
origin=origin,
page=page,
page_size=effective_page_size,
)
for row in rows:
jail_name: str = row.jail
ip: str = row.ip
banned_at: str = ts_to_iso(row.timeofban)
ban_count: int = row.bancount
matches, failures = parse_data_json(row.data)
country_code: str | None = None
country_name: str | None = None
asn: str | None = None
org: str | None = None
if geo_enricher is not None:
try:
geo = await geo_enricher(ip)
if geo is not None:
country_code = geo.country_code
country_name = geo.country_name
asn = geo.asn
org = geo.org
except Exception: # noqa: BLE001
log.warning("history_service_geo_lookup_failed", ip=ip)
items.append(
HistoryBanItem(
ip=ip,
jail=jail_name,
banned_at=banned_at,
ban_count=ban_count,
failures=failures,
matches=matches,
country_code=country_code,
country_name=country_name,
asn=asn,
org=org,
)
)
return HistoryListResponse(
items=items,
total=total,

109
backend/app/tasks/history_sync.py Normal file
View File

@@ -0,0 +1,109 @@
"""History sync background task.
Periodically copies new records from the fail2ban sqlite database into the
BanGUI application archive table to prevent gaps when fail2ban purges old rows.
"""
from __future__ import annotations
import datetime
from typing import TYPE_CHECKING
import structlog
from app.repositories import fail2ban_db_repo
from app.utils.fail2ban_db_utils import get_fail2ban_db_path
if TYPE_CHECKING: # pragma: no cover
from fastapi import FastAPI
log: structlog.stdlib.BoundLogger = structlog.get_logger()
#: Stable APScheduler job id.
JOB_ID: str = "history_sync"
#: Interval in seconds between sync runs.
HISTORY_SYNC_INTERVAL: int = 300
#: Backfill window when archive is empty (seconds).
BACKFILL_WINDOW: int = 7 * 86400
async def _get_last_archive_ts(db) -> int | None:
async with db.execute("SELECT MAX(timeofban) FROM history_archive") as cur:
row = await cur.fetchone()
if row is None or row[0] is None:
return None
return int(row[0])
async def _run_sync(app: FastAPI) -> None:
db = app.state.db
socket_path: str = app.state.settings.fail2ban_socket
try:
last_ts = await _get_last_archive_ts(db)
now_ts = int(datetime.datetime.now(datetime.UTC).timestamp())
if last_ts is None:
last_ts = now_ts - BACKFILL_WINDOW
log.info("history_sync_backfill", window_seconds=BACKFILL_WINDOW)
per_page = 500
next_since = last_ts
total_synced = 0
while True:
fail2ban_db_path = await get_fail2ban_db_path(socket_path)
rows, total = await fail2ban_db_repo.get_history_page(
db_path=fail2ban_db_path,
since=next_since,
page=1,
page_size=per_page,
)
if not rows:
break
from app.repositories.history_archive_repo import archive_ban_event
for row in rows:
await archive_ban_event(
db=db,
jail=row.jail,
ip=row.ip,
timeofban=row.timeofban,
bancount=row.bancount,
data=row.data,
action="ban",
)
total_synced += 1
# Continue where we left off by max timeofban + 1.
max_time = max(row.timeofban for row in rows)
next_since = max_time + 1
if len(rows) < per_page:
break
log.info("history_sync_complete", synced=total_synced)
except Exception:
log.exception("history_sync_failed")
def register(app: FastAPI) -> None:
"""Register the history sync periodic job.
Should be called after scheduler startup, from the lifespan handler.
"""
app.state.scheduler.add_job(
_run_sync,
trigger="interval",
seconds=HISTORY_SYNC_INTERVAL,
kwargs={"app": app},
id=JOB_ID,
replace_existing=True,
next_run_time=datetime.datetime.now(tz=datetime.UTC),
)
log.info("history_sync_scheduled", interval_seconds=HISTORY_SYNC_INTERVAL)