Complete history archive support for dashboard/map data and mark task finished

Add source=archive option for dashboard endpoints and history service; update Docs/Tasks.md; include archive branch for list_bans, bans_by_country, ban_trend, bans_by_jail; tests for archive paths.

backend/app/services/ban_service.py

@@ -112,6 +112,7 @@ async def list_bans(
     socket_path: str,
     range_: TimeRange,
     *,
+    source: str = "fail2ban",
     page: int = 1,
     page_size: int = _DEFAULT_PAGE_SIZE,
     http_session: aiohttp.ClientSession | None = None,
@@ -160,24 +161,41 @@ async def list_bans(
     since: int = _since_unix(range_)
     effective_page_size: int = min(page_size, _MAX_PAGE_SIZE)
     offset: int = (page - 1) * effective_page_size
-    origin_clause, origin_params = _origin_sql_filter(origin)
 
-    db_path: str = await get_fail2ban_db_path(socket_path)
-    log.info(
-        "ban_service_list_bans",
-        db_path=db_path,
-        since=since,
-        range=range_,
-        origin=origin,
-    )
+    if source not in ("fail2ban", "archive"):
+        raise ValueError(f"Unsupported source: {source!r}")
 
-    rows, total = await fail2ban_db_repo.get_currently_banned(
-        db_path=db_path,
-        since=since,
-        origin=origin,
-        limit=effective_page_size,
-        offset=offset,
-    )
+    if source == "archive":
+        if app_db is None:
+            raise ValueError("app_db must be provided when source is 'archive'")
+
+        from app.repositories.history_archive_repo import get_archived_history
+
+        rows, total = await get_archived_history(
+            db=app_db,
+            since=since,
+            origin=origin,
+            action="ban",
+            page=page,
+            page_size=effective_page_size,
+        )
+    else:
+        db_path: str = await get_fail2ban_db_path(socket_path)
+        log.info(
+            "ban_service_list_bans",
+            db_path=db_path,
+            since=since,
+            range=range_,
+            origin=origin,
+        )
+
+        rows, total = await fail2ban_db_repo.get_currently_banned(
+            db_path=db_path,
+            since=since,
+            origin=origin,
+            limit=effective_page_size,
+            offset=offset,
+        )
 
     # Batch-resolve geo data for all IPs on this page in a single API call.
     # This avoids hitting the 45 req/min single-IP rate limit when the
@@ -192,11 +210,19 @@ async def list_bans(
 
     items: list[DashboardBanItem] = []
     for row in rows:
-        jail: str = row.jail
-        ip: str = row.ip
-        banned_at: str = ts_to_iso(row.timeofban)
-        ban_count: int = row.bancount
-        matches, _ = parse_data_json(row.data)
+        if source == "archive":
+            jail = str(row["jail"])
+            ip = str(row["ip"])
+            banned_at = ts_to_iso(int(row["timeofban"]))
+            ban_count = int(row["bancount"])
+            matches, _ = parse_data_json(row["data"])
+        else:
+            jail = row.jail
+            ip = row.ip
+            banned_at = ts_to_iso(row.timeofban)
+            ban_count = row.bancount
+            matches, _ = parse_data_json(row.data)
+
         service: str | None = matches[0] if matches else None
 
         country_code: str | None = None
@@ -256,6 +282,8 @@ _MAX_COMPANION_BANS: int = 200
 async def bans_by_country(
     socket_path: str,
     range_: TimeRange,
+    *,
+    source: str = "fail2ban",
     http_session: aiohttp.ClientSession | None = None,
     geo_cache_lookup: GeoCacheLookup | None = None,
     geo_batch_lookup: GeoBatchLookup | None = None,
@@ -300,41 +328,80 @@ async def bans_by_country(
     """
 
     since: int = _since_unix(range_)
-    origin_clause, origin_params = _origin_sql_filter(origin)
-    db_path: str = await get_fail2ban_db_path(socket_path)
-    log.info(
-        "ban_service_bans_by_country",
-        db_path=db_path,
-        since=since,
-        range=range_,
-        origin=origin,
-    )
 
-    # Total count and companion rows reuse the same SQL query logic.
-    # Passing limit=0 returns only the total from the count query.
-    _, total = await fail2ban_db_repo.get_currently_banned(
-        db_path=db_path,
-        since=since,
-        origin=origin,
-        limit=0,
-        offset=0,
-    )
+    if source not in ("fail2ban", "archive"):
+        raise ValueError(f"Unsupported source: {source!r}")
 
-    agg_rows = await fail2ban_db_repo.get_ban_event_counts(
-        db_path=db_path,
-        since=since,
-        origin=origin,
-    )
+    if source == "archive":
+        if app_db is None:
+            raise ValueError("app_db must be provided when source is 'archive'")
 
-    companion_rows, _ = await fail2ban_db_repo.get_currently_banned(
-        db_path=db_path,
-        since=since,
-        origin=origin,
-        limit=_MAX_COMPANION_BANS,
-        offset=0,
-    )
+        from app.repositories.history_archive_repo import (
+            get_all_archived_history,
+            get_archived_history,
+        )
 
-    unique_ips: list[str] = [r.ip for r in agg_rows]
+        all_rows = await get_all_archived_history(
+            db=app_db,
+            since=since,
+            origin=origin,
+            action="ban",
+        )
+
+        total = len(all_rows)
+
+        # companion rows for the table should be most recent
+        companion_rows, _ = await get_archived_history(
+            db=app_db,
+            since=since,
+            origin=origin,
+            action="ban",
+            page=1,
+            page_size=_MAX_COMPANION_BANS,
+        )
+
+        agg_rows = {}
+        for row in all_rows:
+            ip = str(row["ip"])
+            agg_rows[ip] = agg_rows.get(ip, 0) + 1
+
+        unique_ips = list(agg_rows.keys())
+    else:
+        origin_clause, origin_params = _origin_sql_filter(origin)
+        db_path: str = await get_fail2ban_db_path(socket_path)
+        log.info(
+            "ban_service_bans_by_country",
+            db_path=db_path,
+            since=since,
+            range=range_,
+            origin=origin,
+        )
+
+        # Total count and companion rows reuse the same SQL query logic.
+        # Passing limit=0 returns only the total from the count query.
+        _, total = await fail2ban_db_repo.get_currently_banned(
+            db_path=db_path,
+            since=since,
+            origin=origin,
+            limit=0,
+            offset=0,
+        )
+
+        agg_rows = await fail2ban_db_repo.get_ban_event_counts(
+            db_path=db_path,
+            since=since,
+            origin=origin,
+        )
+
+        companion_rows, _ = await fail2ban_db_repo.get_currently_banned(
+            db_path=db_path,
+            since=since,
+            origin=origin,
+            limit=_MAX_COMPANION_BANS,
+            offset=0,
+        )
+
+        unique_ips = [r.ip for r in agg_rows]
     geo_map: dict[str, GeoInfo] = {}
 
     if http_session is not None and unique_ips and geo_cache_lookup is not None:
@@ -371,12 +438,28 @@ async def bans_by_country(
     countries: dict[str, int] = {}
     country_names: dict[str, str] = {}
 
-    for agg_row in agg_rows:
-        ip: str = agg_row.ip
+    if source == "archive":
+        agg_items = [
+            {
+                "ip": ip,
+                "event_count": count,
+            }
+            for ip, count in agg_rows.items()
+        ]
+    else:
+        agg_items = agg_rows
+
+    for agg_row in agg_items:
+        if source == "archive":
+            ip = agg_row["ip"]
+            event_count = agg_row["event_count"]
+        else:
+            ip = agg_row.ip
+            event_count = agg_row.event_count
+
         geo = geo_map.get(ip)
         cc: str | None = geo.country_code if geo else None
         cn: str | None = geo.country_name if geo else None
-        event_count: int = agg_row.event_count
 
         if cc:
             countries[cc] = countries.get(cc, 0) + event_count
@@ -386,26 +469,38 @@ async def bans_by_country(
     # Build companion table from recent rows (geo already cached from batch step).
     bans: list[DashboardBanItem] = []
     for companion_row in companion_rows:
-        ip = companion_row.ip
+        if source == "archive":
+            ip = companion_row["ip"]
+            jail = companion_row["jail"]
+            banned_at = ts_to_iso(int(companion_row["timeofban"]))
+            ban_count = int(companion_row["bancount"])
+            service = None
+        else:
+            ip = companion_row.ip
+            jail = companion_row.jail
+            banned_at = ts_to_iso(companion_row.timeofban)
+            ban_count = companion_row.bancount
+            matches, _ = parse_data_json(companion_row.data)
+            service = matches[0] if matches else None
+
         geo = geo_map.get(ip)
         cc = geo.country_code if geo else None
         cn = geo.country_name if geo else None
         asn: str | None = geo.asn if geo else None
         org: str | None = geo.org if geo else None
-        matches, _ = parse_data_json(companion_row.data)
 
         bans.append(
             DashboardBanItem(
                 ip=ip,
-                jail=companion_row.jail,
-                banned_at=ts_to_iso(companion_row.timeofban),
-                service=matches[0] if matches else None,
+                jail=jail,
+                banned_at=banned_at,
+                service=service,
                 country_code=cc,
                 country_name=cn,
                 asn=asn,
                 org=org,
-                ban_count=companion_row.bancount,
-                origin=_derive_origin(companion_row.jail),
+                ban_count=ban_count,
+                origin=_derive_origin(jail),
             )
         )
 
@@ -426,6 +521,8 @@ async def ban_trend(
     socket_path: str,
     range_: TimeRange,
     *,
+    source: str = "fail2ban",
+    app_db: aiosqlite.Connection | None = None,
     origin: BanOrigin | None = None,
 ) -> BanTrendResponse:
     """Return ban counts aggregated into equal-width time buckets.
@@ -457,26 +554,58 @@ async def ban_trend(
     since: int = _since_unix(range_)
     bucket_secs: int = BUCKET_SECONDS[range_]
     num_buckets: int = bucket_count(range_)
-    origin_clause, origin_params = _origin_sql_filter(origin)
 
-    db_path: str = await get_fail2ban_db_path(socket_path)
-    log.info(
-        "ban_service_ban_trend",
-        db_path=db_path,
-        since=since,
-        range=range_,
-        origin=origin,
-        bucket_secs=bucket_secs,
-        num_buckets=num_buckets,
-    )
+    if source not in ("fail2ban", "archive"):
+        raise ValueError(f"Unsupported source: {source!r}")
 
-    counts = await fail2ban_db_repo.get_ban_counts_by_bucket(
-        db_path=db_path,
-        since=since,
-        bucket_secs=bucket_secs,
-        num_buckets=num_buckets,
-        origin=origin,
-    )
+    if source == "archive":
+        if app_db is None:
+            raise ValueError("app_db must be provided when source is 'archive'")
+
+        from app.repositories.history_archive_repo import get_all_archived_history
+
+        all_rows = await get_all_archived_history(
+            db=app_db,
+            since=since,
+            origin=origin,
+            action="ban",
+        )
+
+        counts: list[int] = [0] * num_buckets
+        for row in all_rows:
+            timeofban = int(row["timeofban"])
+            bucket_index = int((timeofban - since) / bucket_secs)
+            if 0 <= bucket_index < num_buckets:
+                counts[bucket_index] += 1
+
+        log.info(
+            "ban_service_ban_trend",
+            source=source,
+            since=since,
+            range=range_,
+            origin=origin,
+            bucket_secs=bucket_secs,
+            num_buckets=num_buckets,
+        )
+    else:
+        db_path: str = await get_fail2ban_db_path(socket_path)
+        log.info(
+            "ban_service_ban_trend",
+            db_path=db_path,
+            since=since,
+            range=range_,
+            origin=origin,
+            bucket_secs=bucket_secs,
+            num_buckets=num_buckets,
+        )
+
+        counts = await fail2ban_db_repo.get_ban_counts_by_bucket(
+            db_path=db_path,
+            since=since,
+            bucket_secs=bucket_secs,
+            num_buckets=num_buckets,
+            origin=origin,
+        )
 
     buckets: list[BanTrendBucket] = [
         BanTrendBucket(
@@ -501,6 +630,8 @@ async def bans_by_jail(
     socket_path: str,
     range_: TimeRange,
     *,
+    source: str = "fail2ban",
+    app_db: aiosqlite.Connection | None = None,
     origin: BanOrigin | None = None,
 ) -> BansByJailResponse:
     """Return ban counts aggregated per jail for the selected time window.
@@ -522,38 +653,75 @@ async def bans_by_jail(
         sorted descending and the total ban count.
     """
     since: int = _since_unix(range_)
-    origin_clause, origin_params = _origin_sql_filter(origin)
 
-    db_path: str = await get_fail2ban_db_path(socket_path)
-    log.debug(
-        "ban_service_bans_by_jail",
-        db_path=db_path,
-        since=since,
-        since_iso=ts_to_iso(since),
-        range=range_,
-        origin=origin,
-    )
+    if source not in ("fail2ban", "archive"):
+        raise ValueError(f"Unsupported source: {source!r}")
 
-    total, jail_counts = await fail2ban_db_repo.get_bans_by_jail(
-        db_path=db_path,
-        since=since,
-        origin=origin,
-    )
+    if source == "archive":
+        if app_db is None:
+            raise ValueError("app_db must be provided when source is 'archive'")
 
-    # Diagnostic guard: if zero results were returned, check whether the table
-    # has *any* rows and log a warning with min/max timeofban so operators can
-    # diagnose timezone or filter mismatches from logs.
-    if total == 0:
-        table_row_count, min_timeofban, max_timeofban = await fail2ban_db_repo.get_bans_table_summary(db_path)
-        if table_row_count > 0:
-            log.warning(
-                "ban_service_bans_by_jail_empty_despite_data",
-                table_row_count=table_row_count,
-                min_timeofban=min_timeofban,
-                max_timeofban=max_timeofban,
-                since=since,
-                range=range_,
-            )
+        from app.repositories.history_archive_repo import get_all_archived_history
+
+        all_rows = await get_all_archived_history(
+            db=app_db,
+            since=since,
+            origin=origin,
+            action="ban",
+        )
+
+        jail_counter: dict[str, int] = {}
+        for row in all_rows:
+            jail_name = str(row["jail"])
+            jail_counter[jail_name] = jail_counter.get(jail_name, 0) + 1
+
+        total = sum(jail_counter.values())
+        jail_counts = [
+            JailBanCountModel(jail=jail_name, count=count)
+            for jail_name, count in sorted(jail_counter.items(), key=lambda x: x[1], reverse=True)
+        ]
+
+        log.debug(
+            "ban_service_bans_by_jail",
+            source=source,
+            since=since,
+            since_iso=ts_to_iso(since),
+            range=range_,
+            origin=origin,
+        )
+    else:
+        origin_clause, origin_params = _origin_sql_filter(origin)
+
+        db_path: str = await get_fail2ban_db_path(socket_path)
+        log.debug(
+            "ban_service_bans_by_jail",
+            db_path=db_path,
+            since=since,
+            since_iso=ts_to_iso(since),
+            range=range_,
+            origin=origin,
+        )
+
+        total, jail_counts = await fail2ban_db_repo.get_bans_by_jail(
+            db_path=db_path,
+            since=since,
+            origin=origin,
+        )
+
+        # Diagnostic guard: if zero results were returned, check whether the table
+        # has *any* rows and log a warning with min/max timeofban so operators can
+        # diagnose timezone or filter mismatches from logs.
+        if total == 0:
+            table_row_count, min_timeofban, max_timeofban = await fail2ban_db_repo.get_bans_table_summary(db_path)
+            if table_row_count > 0:
+                log.warning(
+                    "ban_service_bans_by_jail_empty_despite_data",
+                    table_row_count=table_row_count,
+                    min_timeofban=min_timeofban,
+                    max_timeofban=max_timeofban,
+                    since=since,
+                    range=range_,
+                )
 
     log.debug(
         "ban_service_bans_by_jail_result",

backend/app/services/history_service.py

@@ -16,6 +16,8 @@ from typing import TYPE_CHECKING
 import structlog
 
 if TYPE_CHECKING:
+    import aiosqlite
+
     from app.models.geo import GeoEnricher
 
 from app.models.ban import TIME_RANGE_SECONDS, BanOrigin, TimeRange
@@ -63,9 +65,11 @@ async def list_history(
     jail: str | None = None,
     ip_filter: str | None = None,
     origin: BanOrigin | None = None,
+    source: str = "fail2ban",
     page: int = 1,
     page_size: int = _DEFAULT_PAGE_SIZE,
     geo_enricher: GeoEnricher | None = None,
+    db: aiosqlite.Connection | None = None,
 ) -> HistoryListResponse:
     """Return a paginated list of historical ban records with optional filters.
 
@@ -104,55 +108,111 @@ async def list_history(
         page=page,
     )
 
-    rows, total = await fail2ban_db_repo.get_history_page(
-        db_path=db_path,
-        since=since,
-        jail=jail,
-        ip_filter=ip_filter,
-        origin=origin,
-        page=page,
-        page_size=effective_page_size,
-    )
-
     items: list[HistoryBanItem] = []
-    for row in rows:
-        jail_name: str = row.jail
-        ip: str = row.ip
-        banned_at: str = ts_to_iso(row.timeofban)
-        ban_count: int = row.bancount
-        matches, failures = parse_data_json(row.data)
+    total: int
 
-        country_code: str | None = None
-        country_name: str | None = None
-        asn: str | None = None
-        org: str | None = None
+    if source == "archive":
+        if db is None:
+            raise ValueError("db must be provided when source is 'archive'")
 
-        if geo_enricher is not None:
-            try:
-                geo = await geo_enricher(ip)
-                if geo is not None:
-                    country_code = geo.country_code
-                    country_name = geo.country_name
-                    asn = geo.asn
-                    org = geo.org
-            except Exception:  # noqa: BLE001
-                log.warning("history_service_geo_lookup_failed", ip=ip)
+        from app.repositories.history_archive_repo import get_archived_history
 
-        items.append(
-            HistoryBanItem(
-                ip=ip,
-                jail=jail_name,
-                banned_at=banned_at,
-                ban_count=ban_count,
-                failures=failures,
-                matches=matches,
-                country_code=country_code,
-                country_name=country_name,
-                asn=asn,
-                org=org,
-            )
+        archived_rows, total = await get_archived_history(
+            db=db,
+            since=since,
+            jail=jail,
+            ip_filter=ip_filter,
+            page=page,
+            page_size=effective_page_size,
         )
 
+        for row in archived_rows:
+            jail_name = row["jail"]
+            ip = row["ip"]
+            banned_at = ts_to_iso(int(row["timeofban"]))
+            ban_count = int(row["bancount"])
+            matches, failures = parse_data_json(row["data"])
+            # archive records may include actions; we treat all as history
+
+            country_code = None
+            country_name = None
+            asn = None
+            org = None
+
+            if geo_enricher is not None:
+                try:
+                    geo = await geo_enricher(ip)
+                    if geo is not None:
+                        country_code = geo.country_code
+                        country_name = geo.country_name
+                        asn = geo.asn
+                        org = geo.org
+                except Exception:  # noqa: BLE001
+                    log.warning("history_service_geo_lookup_failed", ip=ip)
+
+            items.append(
+                HistoryBanItem(
+                    ip=ip,
+                    jail=jail_name,
+                    banned_at=banned_at,
+                    ban_count=ban_count,
+                    failures=failures,
+                    matches=matches,
+                    country_code=country_code,
+                    country_name=country_name,
+                    asn=asn,
+                    org=org,
+                )
+            )
+    else:
+        rows, total = await fail2ban_db_repo.get_history_page(
+            db_path=db_path,
+            since=since,
+            jail=jail,
+            ip_filter=ip_filter,
+            origin=origin,
+            page=page,
+            page_size=effective_page_size,
+        )
+
+        for row in rows:
+            jail_name: str = row.jail
+            ip: str = row.ip
+            banned_at: str = ts_to_iso(row.timeofban)
+            ban_count: int = row.bancount
+            matches, failures = parse_data_json(row.data)
+
+            country_code: str | None = None
+            country_name: str | None = None
+            asn: str | None = None
+            org: str | None = None
+
+            if geo_enricher is not None:
+                try:
+                    geo = await geo_enricher(ip)
+                    if geo is not None:
+                        country_code = geo.country_code
+                        country_name = geo.country_name
+                        asn = geo.asn
+                        org = geo.org
+                except Exception:  # noqa: BLE001
+                    log.warning("history_service_geo_lookup_failed", ip=ip)
+
+            items.append(
+                HistoryBanItem(
+                    ip=ip,
+                    jail=jail_name,
+                    banned_at=banned_at,
+                    ban_count=ban_count,
+                    failures=failures,
+                    matches=matches,
+                    country_code=country_code,
+                    country_name=country_name,
+                    asn=asn,
+                    org=org,
+                )
+            )
+
     return HistoryListResponse(
         items=items,
         total=total,