lukas.pupkalipinski/BanGUI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Complete history archive support for dashboard/map data and mark task finished

Browse Source 
Add source=archive option for dashboard endpoints and history service; update Docs/Tasks.md; include archive branch for list_bans, bans_by_country, ban_trend, bans_by_jail; tests for archive paths.
This commit is contained in:
Lukas
2026-03-28 12:39:47 +01:00
parent 876af46955
commit 9f05da2d4d
13 changed files with 714 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

386
backend/app/services/ban_service.py
View File

@@ -112,6 +112,7 @@ async def list_bans(
socket_path: str,
range_: TimeRange,
*,
source: str = "fail2ban",
page: int = 1,
page_size: int = _DEFAULT_PAGE_SIZE,
http_session: aiohttp.ClientSession | None = None,
@@ -160,24 +161,41 @@ async def list_bans(
since: int = _since_unix(range_)
effective_page_size: int = min(page_size, _MAX_PAGE_SIZE)
offset: int = (page - 1) * effective_page_size
origin_clause, origin_params = _origin_sql_filter(origin)
db_path: str = await get_fail2ban_db_path(socket_path)
log.info(
"ban_service_list_bans",
db_path=db_path,
since=since,
range=range_,
origin=origin,
)
if source not in ("fail2ban", "archive"):
raise ValueError(f"Unsupported source: {source!r}")
rows, total = await fail2ban_db_repo.get_currently_banned(
db_path=db_path,
since=since,
origin=origin,
limit=effective_page_size,
offset=offset,
)
if source == "archive":
if app_db is None:
raise ValueError("app_db must be provided when source is 'archive'")
from app.repositories.history_archive_repo import get_archived_history
rows, total = await get_archived_history(
db=app_db,
since=since,
origin=origin,
action="ban",
page=page,
page_size=effective_page_size,
)
else:
db_path: str = await get_fail2ban_db_path(socket_path)
log.info(
"ban_service_list_bans",
db_path=db_path,
since=since,
range=range_,
origin=origin,
)
rows, total = await fail2ban_db_repo.get_currently_banned(
db_path=db_path,
since=since,
origin=origin,
limit=effective_page_size,
offset=offset,
)
# Batch-resolve geo data for all IPs on this page in a single API call.
# This avoids hitting the 45 req/min single-IP rate limit when the
@@ -192,11 +210,19 @@ async def list_bans(
items: list[DashboardBanItem] = []
for row in rows:
jail: str = row.jail
ip: str = row.ip
banned_at: str = ts_to_iso(row.timeofban)
ban_count: int = row.bancount
matches, _ = parse_data_json(row.data)
if source == "archive":
jail = str(row["jail"])
ip = str(row["ip"])
banned_at = ts_to_iso(int(row["timeofban"]))
ban_count = int(row["bancount"])
matches, _ = parse_data_json(row["data"])
else:
jail = row.jail
ip = row.ip
banned_at = ts_to_iso(row.timeofban)
ban_count = row.bancount
matches, _ = parse_data_json(row.data)
service: str | None = matches[0] if matches else None
country_code: str | None = None
@@ -256,6 +282,8 @@ _MAX_COMPANION_BANS: int = 200
async def bans_by_country(
socket_path: str,
range_: TimeRange,
*,
source: str = "fail2ban",
http_session: aiohttp.ClientSession | None = None,
geo_cache_lookup: GeoCacheLookup | None = None,
geo_batch_lookup: GeoBatchLookup | None = None,
@@ -300,41 +328,80 @@ async def bans_by_country(
"""
since: int = _since_unix(range_)
origin_clause, origin_params = _origin_sql_filter(origin)
db_path: str = await get_fail2ban_db_path(socket_path)
log.info(
"ban_service_bans_by_country",
db_path=db_path,
since=since,
range=range_,
origin=origin,
)
# Total count and companion rows reuse the same SQL query logic.
# Passing limit=0 returns only the total from the count query.
_, total = await fail2ban_db_repo.get_currently_banned(
db_path=db_path,
since=since,
origin=origin,
limit=0,
offset=0,
)
if source not in ("fail2ban", "archive"):
raise ValueError(f"Unsupported source: {source!r}")
agg_rows = await fail2ban_db_repo.get_ban_event_counts(
db_path=db_path,
since=since,
origin=origin,
)
if source == "archive":
if app_db is None:
raise ValueError("app_db must be provided when source is 'archive'")
companion_rows, _ = await fail2ban_db_repo.get_currently_banned(
db_path=db_path,
since=since,
origin=origin,
limit=_MAX_COMPANION_BANS,
offset=0,
)
from app.repositories.history_archive_repo import (
get_all_archived_history,
get_archived_history,
)
unique_ips: list[str] = [r.ip for r in agg_rows]
all_rows = await get_all_archived_history(
db=app_db,
since=since,
origin=origin,
action="ban",
)
total = len(all_rows)
# companion rows for the table should be most recent
companion_rows, _ = await get_archived_history(
db=app_db,
since=since,
origin=origin,
action="ban",
page=1,
page_size=_MAX_COMPANION_BANS,
)
agg_rows = {}
for row in all_rows:
ip = str(row["ip"])
agg_rows[ip] = agg_rows.get(ip, 0) + 1
unique_ips = list(agg_rows.keys())
else:
origin_clause, origin_params = _origin_sql_filter(origin)
db_path: str = await get_fail2ban_db_path(socket_path)
log.info(
"ban_service_bans_by_country",
db_path=db_path,
since=since,
range=range_,
origin=origin,
)
# Total count and companion rows reuse the same SQL query logic.
# Passing limit=0 returns only the total from the count query.
_, total = await fail2ban_db_repo.get_currently_banned(
db_path=db_path,
since=since,
origin=origin,
limit=0,
offset=0,
)
agg_rows = await fail2ban_db_repo.get_ban_event_counts(
db_path=db_path,
since=since,
origin=origin,
)
companion_rows, _ = await fail2ban_db_repo.get_currently_banned(
db_path=db_path,
since=since,
origin=origin,
limit=_MAX_COMPANION_BANS,
offset=0,
)
unique_ips = [r.ip for r in agg_rows]
geo_map: dict[str, GeoInfo] = {}
if http_session is not None and unique_ips and geo_cache_lookup is not None:
@@ -371,12 +438,28 @@ async def bans_by_country(
countries: dict[str, int] = {}
country_names: dict[str, str] = {}
for agg_row in agg_rows:
ip: str = agg_row.ip
if source == "archive":
agg_items = [
{
"ip": ip,
"event_count": count,
}
for ip, count in agg_rows.items()
]
else:
agg_items = agg_rows
for agg_row in agg_items:
if source == "archive":
ip = agg_row["ip"]
event_count = agg_row["event_count"]
else:
ip = agg_row.ip
event_count = agg_row.event_count
geo = geo_map.get(ip)
cc: str | None = geo.country_code if geo else None
cn: str | None = geo.country_name if geo else None
event_count: int = agg_row.event_count
if cc:
countries[cc] = countries.get(cc, 0) + event_count
@@ -386,26 +469,38 @@ async def bans_by_country(
# Build companion table from recent rows (geo already cached from batch step).
bans: list[DashboardBanItem] = []
for companion_row in companion_rows:
ip = companion_row.ip
if source == "archive":
ip = companion_row["ip"]
jail = companion_row["jail"]
banned_at = ts_to_iso(int(companion_row["timeofban"]))
ban_count = int(companion_row["bancount"])
service = None
else:
ip = companion_row.ip
jail = companion_row.jail
banned_at = ts_to_iso(companion_row.timeofban)
ban_count = companion_row.bancount
matches, _ = parse_data_json(companion_row.data)
service = matches[0] if matches else None
geo = geo_map.get(ip)
cc = geo.country_code if geo else None
cn = geo.country_name if geo else None
asn: str | None = geo.asn if geo else None
org: str | None = geo.org if geo else None
matches, _ = parse_data_json(companion_row.data)
bans.append(
DashboardBanItem(
ip=ip,
jail=companion_row.jail,
banned_at=ts_to_iso(companion_row.timeofban),
service=matches[0] if matches else None,
jail=jail,
banned_at=banned_at,
service=service,
country_code=cc,
country_name=cn,
asn=asn,
org=org,
ban_count=companion_row.bancount,
origin=_derive_origin(companion_row.jail),
ban_count=ban_count,
origin=_derive_origin(jail),
)
)
@@ -426,6 +521,8 @@ async def ban_trend(
socket_path: str,
range_: TimeRange,
*,
source: str = "fail2ban",
app_db: aiosqlite.Connection | None = None,
origin: BanOrigin | None = None,
) -> BanTrendResponse:
"""Return ban counts aggregated into equal-width time buckets.
@@ -457,26 +554,58 @@ async def ban_trend(
since: int = _since_unix(range_)
bucket_secs: int = BUCKET_SECONDS[range_]
num_buckets: int = bucket_count(range_)
origin_clause, origin_params = _origin_sql_filter(origin)
db_path: str = await get_fail2ban_db_path(socket_path)
log.info(
"ban_service_ban_trend",
db_path=db_path,
since=since,
range=range_,
origin=origin,
bucket_secs=bucket_secs,
num_buckets=num_buckets,
)
if source not in ("fail2ban", "archive"):
raise ValueError(f"Unsupported source: {source!r}")
counts = await fail2ban_db_repo.get_ban_counts_by_bucket(
db_path=db_path,
since=since,
bucket_secs=bucket_secs,
num_buckets=num_buckets,
origin=origin,
)
if source == "archive":
if app_db is None:
raise ValueError("app_db must be provided when source is 'archive'")
from app.repositories.history_archive_repo import get_all_archived_history
all_rows = await get_all_archived_history(
db=app_db,
since=since,
origin=origin,
action="ban",
)
counts: list[int] = [0] * num_buckets
for row in all_rows:
timeofban = int(row["timeofban"])
bucket_index = int((timeofban - since) / bucket_secs)
if 0 <= bucket_index < num_buckets:
counts[bucket_index] += 1
log.info(
"ban_service_ban_trend",
source=source,
since=since,
range=range_,
origin=origin,
bucket_secs=bucket_secs,
num_buckets=num_buckets,
)
else:
db_path: str = await get_fail2ban_db_path(socket_path)
log.info(
"ban_service_ban_trend",
db_path=db_path,
since=since,
range=range_,
origin=origin,
bucket_secs=bucket_secs,
num_buckets=num_buckets,
)
counts = await fail2ban_db_repo.get_ban_counts_by_bucket(
db_path=db_path,
since=since,
bucket_secs=bucket_secs,
num_buckets=num_buckets,
origin=origin,
)
buckets: list[BanTrendBucket] = [
BanTrendBucket(
@@ -501,6 +630,8 @@ async def bans_by_jail(
socket_path: str,
range_: TimeRange,
*,
source: str = "fail2ban",
app_db: aiosqlite.Connection | None = None,
origin: BanOrigin | None = None,
) -> BansByJailResponse:
"""Return ban counts aggregated per jail for the selected time window.
@@ -522,38 +653,75 @@ async def bans_by_jail(
sorted descending and the total ban count.
"""
since: int = _since_unix(range_)
origin_clause, origin_params = _origin_sql_filter(origin)
db_path: str = await get_fail2ban_db_path(socket_path)
log.debug(
"ban_service_bans_by_jail",
db_path=db_path,
since=since,
since_iso=ts_to_iso(since),
range=range_,
origin=origin,
)
if source not in ("fail2ban", "archive"):
raise ValueError(f"Unsupported source: {source!r}")
total, jail_counts = await fail2ban_db_repo.get_bans_by_jail(
db_path=db_path,
since=since,
origin=origin,
)
if source == "archive":
if app_db is None:
raise ValueError("app_db must be provided when source is 'archive'")
# Diagnostic guard: if zero results were returned, check whether the table
# has *any* rows and log a warning with min/max timeofban so operators can
# diagnose timezone or filter mismatches from logs.
if total == 0:
table_row_count, min_timeofban, max_timeofban = await fail2ban_db_repo.get_bans_table_summary(db_path)
if table_row_count > 0:
log.warning(
"ban_service_bans_by_jail_empty_despite_data",
table_row_count=table_row_count,
min_timeofban=min_timeofban,
max_timeofban=max_timeofban,
since=since,
range=range_,
)
from app.repositories.history_archive_repo import get_all_archived_history
all_rows = await get_all_archived_history(
db=app_db,
since=since,
origin=origin,
action="ban",
)
jail_counter: dict[str, int] = {}
for row in all_rows:
jail_name = str(row["jail"])
jail_counter[jail_name] = jail_counter.get(jail_name, 0) + 1
total = sum(jail_counter.values())
jail_counts = [
JailBanCountModel(jail=jail_name, count=count)
for jail_name, count in sorted(jail_counter.items(), key=lambda x: x[1], reverse=True)
]
log.debug(
"ban_service_bans_by_jail",
source=source,
since=since,
since_iso=ts_to_iso(since),
range=range_,
origin=origin,
)
else:
origin_clause, origin_params = _origin_sql_filter(origin)
db_path: str = await get_fail2ban_db_path(socket_path)
log.debug(
"ban_service_bans_by_jail",
db_path=db_path,
since=since,
since_iso=ts_to_iso(since),
range=range_,
origin=origin,
)
total, jail_counts = await fail2ban_db_repo.get_bans_by_jail(
db_path=db_path,
since=since,
origin=origin,
)
# Diagnostic guard: if zero results were returned, check whether the table
# has *any* rows and log a warning with min/max timeofban so operators can
# diagnose timezone or filter mismatches from logs.
if total == 0:
table_row_count, min_timeofban, max_timeofban = await fail2ban_db_repo.get_bans_table_summary(db_path)
if table_row_count > 0:
log.warning(
"ban_service_bans_by_jail_empty_despite_data",
table_row_count=table_row_count,
min_timeofban=min_timeofban,
max_timeofban=max_timeofban,
since=since,
range=range_,
)
log.debug(
"ban_service_bans_by_jail_result",