Switch dashboard/map/history views to archive source for long-term data

Update fail2ban dbpurgeage to 648000 and history sync backfill/pagination for archive-based 7.5 day history.

Docker/fail2ban-dev-config/README.md

@@ -78,6 +78,11 @@ Chains steps 1–3 automatically with appropriate sleep intervals.
 Inside the container the log file is mounted at `/remotelogs/bangui/auth.log`
 (see `fail2ban/paths-lsio.conf` — `remote_logs_path = /remotelogs`).
 
+BanGUI also extends fail2ban history retention for archive backfill. In
+the development config `fail2ban/fail2ban.conf` the database purge age is
+set to `648000` seconds (7.5 days) so the first archive sync can recover a
+full 7-day window before fail2ban purges old rows.
+
 To change sensitivity, edit `fail2ban/jail.d/manual-Jail.conf`:
 
 ```ini

backend/app/routers/dashboard.py

@@ -12,7 +12,7 @@ Also provides ``GET /api/dashboard/bans`` for the dashboard ban-list table,
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Literal
 
 if TYPE_CHECKING:
     import aiohttp
@@ -83,7 +83,7 @@ async def get_dashboard_bans(
     request: Request,
     _auth: AuthDep,
     range: TimeRange = Query(default=_DEFAULT_RANGE, description="Time-range preset."),
-    source: str = Query(default="fail2ban", description="Data source: 'fail2ban' or 'archive'."),
+    source: Literal["fail2ban", "archive"] = Query(default="fail2ban", description="Data source: 'fail2ban' or 'archive'."),
     page: int = Query(default=1, ge=1, description="1-based page number."),
     page_size: int = Query(default=_DEFAULT_PAGE_SIZE, ge=1, le=500, description="Items per page."),
     origin: BanOrigin | None = Query(
@@ -137,7 +137,7 @@ async def get_bans_by_country(
     request: Request,
     _auth: AuthDep,
     range: TimeRange = Query(default=_DEFAULT_RANGE, description="Time-range preset."),
-    source: str = Query(default="fail2ban", description="Data source: 'fail2ban' or 'archive'."),
+    source: Literal["fail2ban", "archive"] = Query(default="fail2ban", description="Data source: 'fail2ban' or 'archive'."),
     origin: BanOrigin | None = Query(
         default=None,
         description="Filter by ban origin: 'blocklist' or 'selfblock'. Omit for all.",
@@ -185,7 +185,7 @@ async def get_ban_trend(
     request: Request,
     _auth: AuthDep,
     range: TimeRange = Query(default=_DEFAULT_RANGE, description="Time-range preset."),
-    source: str = Query(default="fail2ban", description="Data source: 'fail2ban' or 'archive'."),
+    source: Literal["fail2ban", "archive"] = Query(default="fail2ban", description="Data source: 'fail2ban' or 'archive'."),
     origin: BanOrigin | None = Query(
         default=None,
         description="Filter by ban origin: 'blocklist' or 'selfblock'. Omit for all.",
@@ -235,7 +235,7 @@ async def get_bans_by_jail(
     request: Request,
     _auth: AuthDep,
     range: TimeRange = Query(default=_DEFAULT_RANGE, description="Time-range preset."),
-    source: str = Query(default="fail2ban", description="Data source: 'fail2ban' or 'archive'."),
+    source: Literal["fail2ban", "archive"] = Query(default="fail2ban", description="Data source: 'fail2ban' or 'archive'."),
     origin: BanOrigin | None = Query(
         default=None,
         description="Filter by ban origin: 'blocklist' or 'selfblock'. Omit for all.",

backend/app/routers/history.py

@@ -15,7 +15,7 @@ Routes
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Literal
 
 if TYPE_CHECKING:
     import aiohttp
@@ -56,7 +56,7 @@ async def get_history(
         default=None,
         description="Filter by ban origin: 'blocklist' or 'selfblock'. Omit for all.",
     ),
-    source: str = Query(
+    source: Literal["fail2ban", "archive"] = Query(
         default="fail2ban",
         description="Data source: 'fail2ban' or 'archive'.",
     ),

backend/app/tasks/history_sync.py

@@ -26,7 +26,7 @@ JOB_ID: str = "history_sync"
 HISTORY_SYNC_INTERVAL: int = 300
 
 #: Backfill window when archive is empty (seconds).
-BACKFILL_WINDOW: int = 7 * 86400
+BACKFILL_WINDOW: int = 648000
 
 
 async def _get_last_archive_ts(db) -> int | None:
@@ -50,7 +50,7 @@ async def _run_sync(app: FastAPI) -> None:
             log.info("history_sync_backfill", window_seconds=BACKFILL_WINDOW)
 
         per_page = 500
-        next_since = last_ts
+        next_since = last_ts + 1
         total_synced = 0
 
         while True:

backend/tests/test_routers/test_dashboard.py

@@ -428,6 +428,15 @@ class TestBansByCountry:
         called_range = mock_fn.call_args[0][1]
         assert called_range == "7d"
 
+    async def test_invalid_source_returns_422(
+        self, dashboard_client: AsyncClient
+    ) -> None:
+        """An invalid source value returns HTTP 422."""
+        response = await dashboard_client.get(
+            "/api/dashboard/bans/by-country?source=invalid"
+        )
+        assert response.status_code == 422
+
     async def test_empty_window_returns_empty_response(
         self, dashboard_client: AsyncClient
     ) -> None:
@@ -722,6 +731,15 @@ class TestBanTrend:
         )
         assert response.status_code == 422
 
+    async def test_invalid_source_returns_422(
+        self, dashboard_client: AsyncClient
+    ) -> None:
+        """An invalid source value returns HTTP 422."""
+        response = await dashboard_client.get(
+            "/api/dashboard/bans/trend?source=invalid"
+        )
+        assert response.status_code == 422
+
     async def test_empty_buckets_response(self, dashboard_client: AsyncClient) -> None:
         """Empty bucket list is serialised correctly."""
         from app.models.ban import BanTrendResponse
@@ -857,6 +875,15 @@ class TestBansByJail:
         )
         assert response.status_code == 422
 
+    async def test_invalid_source_returns_422(
+        self, dashboard_client: AsyncClient
+    ) -> None:
+        """An invalid source value returns HTTP 422."""
+        response = await dashboard_client.get(
+            "/api/dashboard/bans/by-jail?source=invalid"
+        )
+        assert response.status_code == 422
+
     async def test_empty_jails_response(self, dashboard_client: AsyncClient) -> None:
         """Empty jails list is serialised correctly."""
         from app.models.ban import BansByJailResponse

backend/tests/test_tasks/test_history_sync.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from unittest.mock import MagicMock
+from unittest.mock import AsyncMock, MagicMock, patch
 
 from app.tasks import history_sync
 
@@ -27,3 +27,33 @@ class TestHistorySyncTask:
         called_args, called_kwargs = fake_scheduler.add_job.call_args
         assert called_kwargs["id"] == history_sync.JOB_ID
         assert called_kwargs["kwargs"]["app"] == app
+
+    async def test_backfill_window_is_7_5_days(self) -> None:
+        assert history_sync.BACKFILL_WINDOW == 648000
+
+    async def test_sync_uses_strict_since_after_restart(self) -> None:
+        fake_app = type("FakeApp", (), {})()
+        fake_app.state = type("FakeState", (), {})()
+        fake_app.state.settings = type("FakeSettings", (), {})()
+        fake_app.state.settings.fail2ban_socket = "/tmp/fake.sock"
+
+        fake_app.state.db = MagicMock()
+
+        async def fake_get_history_page(*, db_path: str, since: int, page: int, page_size: int, **kwargs):
+            assert since == 1001
+            return [], 0
+
+        async def fake_get_fail2ban_db_path(socket_path: str) -> str:
+            return "/tmp/fake.sqlite3"
+
+        with patch(
+            "app.tasks.history_sync._get_last_archive_ts",
+            new=AsyncMock(return_value=1000),
+        ), patch(
+            "app.tasks.history_sync.get_fail2ban_db_path",
+            new=fake_get_fail2ban_db_path,
+        ), patch(
+            "app.tasks.history_sync.fail2ban_db_repo.get_history_page",
+            new=fake_get_history_page,
+        ):
+            await history_sync._run_sync(fake_app)

fail2ban-master/config/fail2ban.conf

@@ -72,7 +72,7 @@ dbfile = /var/lib/fail2ban/fail2ban.sqlite3
 # Options: dbpurgeage
 # Notes.: Sets age at which bans should be purged from the database
 # Values: [ SECONDS ] Default: 86400 (24hours)
-dbpurgeage = 1d
+dbpurgeage = 648000
 
 # Options: dbmaxmatches
 # Notes.: Number of matches stored in database per ticket (resolvable via 

frontend/src/api/dashboard.ts

@@ -42,6 +42,7 @@ export async function fetchBans(
   page = 1,
   pageSize = 100,
   origin: BanOriginFilter = "all",
+  source: "fail2ban" | "archive" = "fail2ban",
 ): Promise<DashboardBanListResponse> {
   const params = new URLSearchParams({
     range,
@@ -51,6 +52,9 @@ export async function fetchBans(
   if (origin !== "all") {
     params.set("origin", origin);
   }
+  if (source !== "fail2ban") {
+    params.set("source", source);
+  }
   return get<DashboardBanListResponse>(`${ENDPOINTS.dashboardBans}?${params.toString()}`);
 }
 
@@ -66,11 +70,15 @@ export async function fetchBans(
 export async function fetchBanTrend(
   range: TimeRange,
   origin: BanOriginFilter = "all",
+  source: "fail2ban" | "archive" = "fail2ban",
 ): Promise<BanTrendResponse> {
   const params = new URLSearchParams({ range });
   if (origin !== "all") {
     params.set("origin", origin);
   }
+  if (source !== "fail2ban") {
+    params.set("source", source);
+  }
   return get<BanTrendResponse>(`${ENDPOINTS.dashboardBansTrend}?${params.toString()}`);
 }
 
@@ -86,10 +94,14 @@ export async function fetchBanTrend(
 export async function fetchBansByJail(
   range: TimeRange,
   origin: BanOriginFilter = "all",
+  source: "fail2ban" | "archive" = "fail2ban",
 ): Promise<BansByJailResponse> {
   const params = new URLSearchParams({ range });
   if (origin !== "all") {
     params.set("origin", origin);
   }
+  if (source !== "fail2ban") {
+    params.set("source", source);
+  }
   return get<BansByJailResponse>(`${ENDPOINTS.dashboardBansByJail}?${params.toString()}`);
 }

frontend/src/api/history.ts

@@ -21,6 +21,7 @@ export async function fetchHistory(
   if (query.origin) params.set("origin", query.origin);
   if (query.jail) params.set("jail", query.jail);
   if (query.ip) params.set("ip", query.ip);
+  if (query.source) params.set("source", query.source);
   if (query.page !== undefined) params.set("page", String(query.page));
   if (query.page_size !== undefined)
     params.set("page_size", String(query.page_size));

frontend/src/api/map.ts

@@ -17,10 +17,14 @@ import type { BanOriginFilter } from "../types/ban";
 export async function fetchBansByCountry(
   range: TimeRange = "24h",
   origin: BanOriginFilter = "all",
+  source: "fail2ban" | "archive" = "fail2ban",
 ): Promise<BansByCountryResponse> {
   const params = new URLSearchParams({ range });
   if (origin !== "all") {
     params.set("origin", origin);
   }
+  if (source !== "fail2ban") {
+    params.set("source", source);
+  }
   return get<BansByCountryResponse>(`${ENDPOINTS.dashboardBansByCountry}?${params.toString()}`);
 }

frontend/src/components/BanTable.tsx

@@ -46,6 +46,10 @@ interface BanTableProps {
    * Changing this value triggers a re-fetch and resets to page 1.
    */
   origin?: BanOriginFilter;
+  /**
+   * Data source used for the table query.
+   */
+  source?: "fail2ban" | "archive";
 }
 
 // ---------------------------------------------------------------------------
@@ -186,9 +190,9 @@ function buildBanColumns(styles: ReturnType<typeof useStyles>): TableColumnDefin
  * @param props.timeRange - Active time-range preset from the parent page.
  * @param props.origin    - Active origin filter from the parent page.
  */
-export function BanTable({ timeRange, origin = "all" }: BanTableProps): React.JSX.Element {
+export function BanTable({ timeRange, origin = "all", source = "fail2ban" }: BanTableProps): React.JSX.Element {
   const styles = useStyles();
-  const { banItems, total, page, setPage, loading, error, refresh } = useBans(timeRange, origin);
+  const { banItems, total, page, setPage, loading, error, refresh } = useBans(timeRange, origin, source);
 
   const banColumns = buildBanColumns(styles);
 

frontend/src/components/BanTrendChart.tsx

@@ -53,6 +53,8 @@ interface BanTrendChartProps {
   timeRange: TimeRange;
   /** Origin filter controlling which bans are included. */
   origin: BanOriginFilter;
+  /** Data source used for the chart. */
+  source?: "fail2ban" | "archive";
 }
 
 /** Internal chart data point shape. */
@@ -188,9 +190,10 @@ function TrendTooltip(props: TooltipContentProps): React.JSX.Element | null {
 export function BanTrendChart({
   timeRange,
   origin,
+  source = "fail2ban",
 }: BanTrendChartProps): React.JSX.Element {
   const styles = useStyles();
-  const { buckets, isLoading, error, reload } = useBanTrend(timeRange, origin);
+  const { buckets, isLoading, error, reload } = useBanTrend(timeRange, origin, source);
 
   const isEmpty = buckets.every((b) => b.count === 0);
   const entries = buildEntries(buckets, timeRange);

frontend/src/hooks/useBanTrend.ts

@@ -42,6 +42,7 @@ export interface UseBanTrendResult {
 export function useBanTrend(
   timeRange: TimeRange,
   origin: BanOriginFilter,
+  source: "fail2ban" | "archive" = "fail2ban",
 ): UseBanTrendResult {
   const [buckets, setBuckets] = useState<BanTrendBucket[]>([]);
   const [bucketSize, setBucketSize] = useState<string>("1h");
@@ -58,7 +59,7 @@ export function useBanTrend(
     setIsLoading(true);
     setError(null);
 
-    fetchBanTrend(timeRange, origin)
+    fetchBanTrend(timeRange, origin, source)
       .then((data) => {
         if (controller.signal.aborted) return;
         setBuckets(data.buckets);
@@ -73,7 +74,7 @@ export function useBanTrend(
           setIsLoading(false);
         }
       });
-  }, [timeRange, origin]);
+  }, [timeRange, origin, source]);
 
   useEffect(() => {
     load();

frontend/src/hooks/useBans.ts

@@ -44,6 +44,7 @@ export interface UseBansResult {
 export function useBans(
   timeRange: TimeRange,
   origin: BanOriginFilter = "all",
+  source: "fail2ban" | "archive" = "fail2ban",
 ): UseBansResult {
   const [banItems, setBanItems] = useState<DashboardBanItem[]>([]);
   const [total, setTotal] = useState<number>(0);
@@ -51,16 +52,16 @@ export function useBans(
   const [loading, setLoading] = useState<boolean>(true);
   const [error, setError] = useState<string | null>(null);
 
-  // Reset page when time range or origin filter changes.
+  // Reset page when time range, origin filter, or source changes.
   useEffect(() => {
     setPage(1);
-  }, [timeRange, origin]);
+  }, [timeRange, origin, source]);
 
   const doFetch = useCallback(async (): Promise<void> => {
     setLoading(true);
     setError(null);
     try {
-      const data = await fetchBans(timeRange, page, PAGE_SIZE, origin);
+      const data = await fetchBans(timeRange, page, PAGE_SIZE, origin, source);
       setBanItems(data.items);
       setTotal(data.total);
     } catch (err: unknown) {
@@ -68,7 +69,7 @@ export function useBans(
     } finally {
       setLoading(false);
     }
-  }, [timeRange, page, origin]);
+  }, [timeRange, page, origin, source]);
 
   // Stable ref to the latest doFetch so the refresh callback is always current.
   const doFetchRef = useRef(doFetch);

frontend/src/hooks/useDashboardCountryData.ts

@@ -48,6 +48,7 @@ export interface UseDashboardCountryDataResult {
 export function useDashboardCountryData(
   timeRange: TimeRange,
   origin: BanOriginFilter,
+  source: "fail2ban" | "archive" = "fail2ban",
 ): UseDashboardCountryDataResult {
   const [countries, setCountries] = useState<Record<string, number>>({});
   const [countryNames, setCountryNames] = useState<Record<string, string>>({});
@@ -67,7 +68,7 @@ export function useDashboardCountryData(
     setIsLoading(true);
     setError(null);
 
-    fetchBansByCountry(timeRange, origin)
+    fetchBansByCountry(timeRange, origin, source)
       .then((data) => {
         if (controller.signal.aborted) return;
         setCountries(data.countries);
@@ -85,7 +86,7 @@ export function useDashboardCountryData(
           setIsLoading(false);
         }
       });
-  }, [timeRange, origin]);
+  }, [timeRange, origin, source]);
 
   useEffect(() => {
     load();

frontend/src/hooks/useMapData.ts

@@ -43,6 +43,7 @@ export interface UseMapDataResult {
 export function useMapData(
   range: TimeRange = "24h",
   origin: BanOriginFilter = "all",
+  source: "fail2ban" | "archive" = "fail2ban",
 ): UseMapDataResult {
   const [data, setData] = useState<BansByCountryResponse | null>(null);
   const [loading, setLoading] = useState(true);
@@ -64,7 +65,7 @@ export function useMapData(
       abortRef.current?.abort();
       abortRef.current = new AbortController();
 
-      fetchBansByCountry(range, origin)
+      fetchBansByCountry(range, origin, source)
         .then((resp) => {
           setData(resp);
         })
@@ -75,7 +76,7 @@ export function useMapData(
           setLoading(false);
         });
     }, DEBOUNCE_MS);
-  }, [range, origin]);
+  }, [range, origin, source]);
 
   useEffect((): (() => void) => {
     load();

frontend/src/pages/DashboardPage.tsx

@@ -7,7 +7,7 @@
  */
 
 import { useState } from "react";
-import { Text, makeStyles, tokens } from "@fluentui/react-components";
+import { Badge, Text, makeStyles, tokens } from "@fluentui/react-components";
 import { BanTable } from "../components/BanTable";
 import { BanTrendChart } from "../components/BanTrendChart";
 import { ChartStateWrapper } from "../components/ChartStateWrapper";
@@ -71,8 +71,10 @@ export function DashboardPage(): React.JSX.Element {
   const [timeRange, setTimeRange] = useState<TimeRange>("24h");
   const [originFilter, setOriginFilter] = useState<BanOriginFilter>("all");
 
+  const source = timeRange === "24h" ? "fail2ban" : "archive";
+
   const { countries, countryNames, isLoading: countryLoading, error: countryError, reload: reloadCountry } =
-    useDashboardCountryData(timeRange, originFilter);
+    useDashboardCountryData(timeRange, originFilter, source);
 
   const sectionStyles = useCommonSectionStyles();
 
@@ -86,12 +88,17 @@ export function DashboardPage(): React.JSX.Element {
       {/* ------------------------------------------------------------------ */}
       {/* Global filter bar                                                  */}
       {/* ------------------------------------------------------------------ */}
-      <DashboardFilterBar
+      <div style={{ display: "flex", alignItems: "center", gap: tokens.spacingHorizontalM, flexWrap: "wrap" }}>
-        timeRange={timeRange}
+        <DashboardFilterBar
-        onTimeRangeChange={setTimeRange}
+          timeRange={timeRange}
-        originFilter={originFilter}
+          onTimeRangeChange={setTimeRange}
-        onOriginFilterChange={setOriginFilter}
+          originFilter={originFilter}
-      />
+          onOriginFilterChange={setOriginFilter}
+        />
+        <Badge appearance="filled" color={source === "archive" ? "brand" : "success"}>
+          {source === "archive" ? "Archive (BanGUI DB)" : "Live (fail2ban DB)"}
+        </Badge>
+      </div>
 
       {/* ------------------------------------------------------------------ */}
       {/* Ban Trend section                                                   */}
@@ -103,7 +110,7 @@ export function DashboardPage(): React.JSX.Element {
           </Text>
         </div>
         <div className={styles.tabContent}>
-          <BanTrendChart timeRange={timeRange} origin={originFilter} />
+          <BanTrendChart timeRange={timeRange} origin={originFilter} source={source} />
         </div>
       </div>
 
@@ -154,7 +161,7 @@ export function DashboardPage(): React.JSX.Element {
 
         {/* Ban table */}
         <div className={styles.tabContent}>
-          <BanTable timeRange={timeRange} origin={originFilter} />
+          <BanTable timeRange={timeRange} origin={originFilter} source={source} />
         </div>
       </div>
     </div>

frontend/src/pages/HistoryPage.tsx

@@ -143,6 +143,7 @@ function areHistoryQueriesEqual(
     a.origin === b.origin &&
     a.jail === b.jail &&
     a.ip === b.ip &&
+    a.source === b.source &&
     a.page === b.page &&
     a.page_size === b.page_size
   );
@@ -386,11 +387,12 @@ export function HistoryPage(): React.JSX.Element {
   const styles = useStyles();
 
   // Filter state
-  const [range, setRange] = useState<TimeRange>("24h");
+  const [range, setRange] = useState<TimeRange>("7d");
   const [originFilter, setOriginFilter] = useState<BanOriginFilter>("all");
   const [jailFilter, setJailFilter] = useState("");
   const [ipFilter, setIpFilter] = useState("");
   const [appliedQuery, setAppliedQuery] = useState<HistoryQuery>({
+    source: "archive",
     page_size: PAGE_SIZE,
   });
 
@@ -400,12 +402,15 @@ export function HistoryPage(): React.JSX.Element {
   const { items, total, page, loading, error, setPage, refresh } =
     useHistory(appliedQuery);
 
+  const sourceLabel = "Archive (BanGUI DB)";
+
   useEffect((): void => {
     const nextQuery: HistoryQuery = {
       range,
       origin: originFilter !== "all" ? originFilter : undefined,
       jail: jailFilter.trim() || undefined,
       ip: ipFilter.trim() || undefined,
+      source: "archive",
       page: 1,
       page_size: PAGE_SIZE,
     };
@@ -485,6 +490,9 @@ export function HistoryPage(): React.JSX.Element {
             setIpFilter(value);
           }}
         />
+        <Badge appearance="filled" color="brand" style={{ alignSelf: "flex-start" }}>
+          {sourceLabel}
+        </Badge>
       </div>
 
       {/* ---------------------------------------------------------------- */}

frontend/src/pages/MapPage.tsx

@@ -98,8 +98,10 @@ export function MapPage(): React.JSX.Element {
 
   const PAGE_SIZE_OPTIONS = [25, 50, 100] as const;
 
+  const source = range === "24h" ? "fail2ban" : "archive";
+
   const { countries, countryNames, bans, total, loading, error, refresh } =
-    useMapData(range, originFilter);
+    useMapData(range, originFilter, source);
 
   const {
     thresholds: mapThresholds,
@@ -163,6 +165,9 @@ export function MapPage(): React.JSX.Element {
               setSelectedCountry(null);
             }}
           />
+          <Badge appearance="filled" color={source === "archive" ? "brand" : "success"}>
+            {source === "archive" ? "Archive (BanGUI DB)" : "Live (fail2ban DB)"}
+          </Badge>
           <Button
             icon={<ArrowCounterclockwiseRegular />}
             onClick={(): void => {

frontend/src/pages/__tests__/HistoryPage.test.tsx

@@ -50,7 +50,8 @@ describe("HistoryPage", () => {
     // Initial load should include the auto-applied default query.
     await waitFor(() => {
       expect(lastQuery).toEqual({
-        range: "24h",
+        range: "7d",
+        source: "archive",
         origin: undefined,
         jail: undefined,
         ip: undefined,

frontend/src/types/history.ts

@@ -57,6 +57,7 @@ export interface HistoryQuery {
   origin?: BanOriginFilter;
   jail?: string;
   ip?: string;
+  source?: "fail2ban" | "archive";
   page?: number;
   page_size?: number;
 }