diff --git a/Docker/fail2ban-dev-config/README.md b/Docker/fail2ban-dev-config/README.md index 6ecaf56..6422e00 100644 --- a/Docker/fail2ban-dev-config/README.md +++ b/Docker/fail2ban-dev-config/README.md @@ -78,6 +78,11 @@ Chains steps 1–3 automatically with appropriate sleep intervals. Inside the container the log file is mounted at `/remotelogs/bangui/auth.log` (see `fail2ban/paths-lsio.conf` — `remote_logs_path = /remotelogs`). +BanGUI also extends fail2ban history retention for archive backfill. In +the development config `fail2ban/fail2ban.conf` the database purge age is +set to `648000` seconds (7.5 days) so the first archive sync can recover a +full 7-day window before fail2ban purges old rows. + To change sensitivity, edit `fail2ban/jail.d/manual-Jail.conf`: ```ini diff --git a/backend/app/routers/dashboard.py b/backend/app/routers/dashboard.py index 2179ac5..af90287 100644 --- a/backend/app/routers/dashboard.py +++ b/backend/app/routers/dashboard.py @@ -12,7 +12,7 @@ Also provides ``GET /api/dashboard/bans`` for the dashboard ban-list table, from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal if TYPE_CHECKING: import aiohttp @@ -83,7 +83,7 @@ async def get_dashboard_bans( request: Request, _auth: AuthDep, range: TimeRange = Query(default=_DEFAULT_RANGE, description="Time-range preset."), - source: str = Query(default="fail2ban", description="Data source: 'fail2ban' or 'archive'."), + source: Literal["fail2ban", "archive"] = Query(default="fail2ban", description="Data source: 'fail2ban' or 'archive'."), page: int = Query(default=1, ge=1, description="1-based page number."), page_size: int = Query(default=_DEFAULT_PAGE_SIZE, ge=1, le=500, description="Items per page."), origin: BanOrigin | None = Query( @@ -137,7 +137,7 @@ async def get_bans_by_country( request: Request, _auth: AuthDep, range: TimeRange = Query(default=_DEFAULT_RANGE, description="Time-range preset."), - source: str = Query(default="fail2ban", description="Data source: 'fail2ban' or 'archive'."), + source: Literal["fail2ban", "archive"] = Query(default="fail2ban", description="Data source: 'fail2ban' or 'archive'."), origin: BanOrigin | None = Query( default=None, description="Filter by ban origin: 'blocklist' or 'selfblock'. Omit for all.", @@ -185,7 +185,7 @@ async def get_ban_trend( request: Request, _auth: AuthDep, range: TimeRange = Query(default=_DEFAULT_RANGE, description="Time-range preset."), - source: str = Query(default="fail2ban", description="Data source: 'fail2ban' or 'archive'."), + source: Literal["fail2ban", "archive"] = Query(default="fail2ban", description="Data source: 'fail2ban' or 'archive'."), origin: BanOrigin | None = Query( default=None, description="Filter by ban origin: 'blocklist' or 'selfblock'. Omit for all.", @@ -235,7 +235,7 @@ async def get_bans_by_jail( request: Request, _auth: AuthDep, range: TimeRange = Query(default=_DEFAULT_RANGE, description="Time-range preset."), - source: str = Query(default="fail2ban", description="Data source: 'fail2ban' or 'archive'."), + source: Literal["fail2ban", "archive"] = Query(default="fail2ban", description="Data source: 'fail2ban' or 'archive'."), origin: BanOrigin | None = Query( default=None, description="Filter by ban origin: 'blocklist' or 'selfblock'. Omit for all.", diff --git a/backend/app/routers/history.py b/backend/app/routers/history.py index 74228fe..bc0f214 100644 --- a/backend/app/routers/history.py +++ b/backend/app/routers/history.py @@ -15,7 +15,7 @@ Routes from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal if TYPE_CHECKING: import aiohttp @@ -56,7 +56,7 @@ async def get_history( default=None, description="Filter by ban origin: 'blocklist' or 'selfblock'. Omit for all.", ), - source: str = Query( + source: Literal["fail2ban", "archive"] = Query( default="fail2ban", description="Data source: 'fail2ban' or 'archive'.", ), diff --git a/backend/app/tasks/history_sync.py b/backend/app/tasks/history_sync.py index b6ea3d3..17d48fd 100644 --- a/backend/app/tasks/history_sync.py +++ b/backend/app/tasks/history_sync.py @@ -26,7 +26,7 @@ JOB_ID: str = "history_sync" HISTORY_SYNC_INTERVAL: int = 300 #: Backfill window when archive is empty (seconds). -BACKFILL_WINDOW: int = 7 * 86400 +BACKFILL_WINDOW: int = 648000 async def _get_last_archive_ts(db) -> int | None: @@ -50,7 +50,7 @@ async def _run_sync(app: FastAPI) -> None: log.info("history_sync_backfill", window_seconds=BACKFILL_WINDOW) per_page = 500 - next_since = last_ts + next_since = last_ts + 1 total_synced = 0 while True: diff --git a/backend/tests/test_routers/test_dashboard.py b/backend/tests/test_routers/test_dashboard.py index 20bcade..30a8c89 100644 --- a/backend/tests/test_routers/test_dashboard.py +++ b/backend/tests/test_routers/test_dashboard.py @@ -428,6 +428,15 @@ class TestBansByCountry: called_range = mock_fn.call_args[0][1] assert called_range == "7d" + async def test_invalid_source_returns_422( + self, dashboard_client: AsyncClient + ) -> None: + """An invalid source value returns HTTP 422.""" + response = await dashboard_client.get( + "/api/dashboard/bans/by-country?source=invalid" + ) + assert response.status_code == 422 + async def test_empty_window_returns_empty_response( self, dashboard_client: AsyncClient ) -> None: @@ -722,6 +731,15 @@ class TestBanTrend: ) assert response.status_code == 422 + async def test_invalid_source_returns_422( + self, dashboard_client: AsyncClient + ) -> None: + """An invalid source value returns HTTP 422.""" + response = await dashboard_client.get( + "/api/dashboard/bans/trend?source=invalid" + ) + assert response.status_code == 422 + async def test_empty_buckets_response(self, dashboard_client: AsyncClient) -> None: """Empty bucket list is serialised correctly.""" from app.models.ban import BanTrendResponse @@ -857,6 +875,15 @@ class TestBansByJail: ) assert response.status_code == 422 + async def test_invalid_source_returns_422( + self, dashboard_client: AsyncClient + ) -> None: + """An invalid source value returns HTTP 422.""" + response = await dashboard_client.get( + "/api/dashboard/bans/by-jail?source=invalid" + ) + assert response.status_code == 422 + async def test_empty_jails_response(self, dashboard_client: AsyncClient) -> None: """Empty jails list is serialised correctly.""" from app.models.ban import BansByJailResponse diff --git a/backend/tests/test_tasks/test_history_sync.py b/backend/tests/test_tasks/test_history_sync.py index c9e1c44..de167d6 100644 --- a/backend/tests/test_tasks/test_history_sync.py +++ b/backend/tests/test_tasks/test_history_sync.py @@ -2,7 +2,7 @@ from __future__ import annotations -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock, patch from app.tasks import history_sync @@ -27,3 +27,33 @@ class TestHistorySyncTask: called_args, called_kwargs = fake_scheduler.add_job.call_args assert called_kwargs["id"] == history_sync.JOB_ID assert called_kwargs["kwargs"]["app"] == app + + async def test_backfill_window_is_7_5_days(self) -> None: + assert history_sync.BACKFILL_WINDOW == 648000 + + async def test_sync_uses_strict_since_after_restart(self) -> None: + fake_app = type("FakeApp", (), {})() + fake_app.state = type("FakeState", (), {})() + fake_app.state.settings = type("FakeSettings", (), {})() + fake_app.state.settings.fail2ban_socket = "/tmp/fake.sock" + + fake_app.state.db = MagicMock() + + async def fake_get_history_page(*, db_path: str, since: int, page: int, page_size: int, **kwargs): + assert since == 1001 + return [], 0 + + async def fake_get_fail2ban_db_path(socket_path: str) -> str: + return "/tmp/fake.sqlite3" + + with patch( + "app.tasks.history_sync._get_last_archive_ts", + new=AsyncMock(return_value=1000), + ), patch( + "app.tasks.history_sync.get_fail2ban_db_path", + new=fake_get_fail2ban_db_path, + ), patch( + "app.tasks.history_sync.fail2ban_db_repo.get_history_page", + new=fake_get_history_page, + ): + await history_sync._run_sync(fake_app) diff --git a/fail2ban-master/config/fail2ban.conf b/fail2ban-master/config/fail2ban.conf index fd6baeb..080031c 100644 --- a/fail2ban-master/config/fail2ban.conf +++ b/fail2ban-master/config/fail2ban.conf @@ -72,7 +72,7 @@ dbfile = /var/lib/fail2ban/fail2ban.sqlite3 # Options: dbpurgeage # Notes.: Sets age at which bans should be purged from the database # Values: [ SECONDS ] Default: 86400 (24hours) -dbpurgeage = 1d +dbpurgeage = 648000 # Options: dbmaxmatches # Notes.: Number of matches stored in database per ticket (resolvable via diff --git a/frontend/src/api/dashboard.ts b/frontend/src/api/dashboard.ts index 90309d8..61e429e 100644 --- a/frontend/src/api/dashboard.ts +++ b/frontend/src/api/dashboard.ts @@ -42,6 +42,7 @@ export async function fetchBans( page = 1, pageSize = 100, origin: BanOriginFilter = "all", + source: "fail2ban" | "archive" = "fail2ban", ): Promise { const params = new URLSearchParams({ range, @@ -51,6 +52,9 @@ export async function fetchBans( if (origin !== "all") { params.set("origin", origin); } + if (source !== "fail2ban") { + params.set("source", source); + } return get(`${ENDPOINTS.dashboardBans}?${params.toString()}`); } @@ -66,11 +70,15 @@ export async function fetchBans( export async function fetchBanTrend( range: TimeRange, origin: BanOriginFilter = "all", + source: "fail2ban" | "archive" = "fail2ban", ): Promise { const params = new URLSearchParams({ range }); if (origin !== "all") { params.set("origin", origin); } + if (source !== "fail2ban") { + params.set("source", source); + } return get(`${ENDPOINTS.dashboardBansTrend}?${params.toString()}`); } @@ -86,10 +94,14 @@ export async function fetchBanTrend( export async function fetchBansByJail( range: TimeRange, origin: BanOriginFilter = "all", + source: "fail2ban" | "archive" = "fail2ban", ): Promise { const params = new URLSearchParams({ range }); if (origin !== "all") { params.set("origin", origin); } + if (source !== "fail2ban") { + params.set("source", source); + } return get(`${ENDPOINTS.dashboardBansByJail}?${params.toString()}`); } diff --git a/frontend/src/api/history.ts b/frontend/src/api/history.ts index e318a5a..f239317 100644 --- a/frontend/src/api/history.ts +++ b/frontend/src/api/history.ts @@ -21,6 +21,7 @@ export async function fetchHistory( if (query.origin) params.set("origin", query.origin); if (query.jail) params.set("jail", query.jail); if (query.ip) params.set("ip", query.ip); + if (query.source) params.set("source", query.source); if (query.page !== undefined) params.set("page", String(query.page)); if (query.page_size !== undefined) params.set("page_size", String(query.page_size)); diff --git a/frontend/src/api/map.ts b/frontend/src/api/map.ts index e6b8eda..5405995 100644 --- a/frontend/src/api/map.ts +++ b/frontend/src/api/map.ts @@ -17,10 +17,14 @@ import type { BanOriginFilter } from "../types/ban"; export async function fetchBansByCountry( range: TimeRange = "24h", origin: BanOriginFilter = "all", + source: "fail2ban" | "archive" = "fail2ban", ): Promise { const params = new URLSearchParams({ range }); if (origin !== "all") { params.set("origin", origin); } + if (source !== "fail2ban") { + params.set("source", source); + } return get(`${ENDPOINTS.dashboardBansByCountry}?${params.toString()}`); } diff --git a/frontend/src/components/BanTable.tsx b/frontend/src/components/BanTable.tsx index bff6164..85879ac 100644 --- a/frontend/src/components/BanTable.tsx +++ b/frontend/src/components/BanTable.tsx @@ -46,6 +46,10 @@ interface BanTableProps { * Changing this value triggers a re-fetch and resets to page 1. */ origin?: BanOriginFilter; + /** + * Data source used for the table query. + */ + source?: "fail2ban" | "archive"; } // --------------------------------------------------------------------------- @@ -186,9 +190,9 @@ function buildBanColumns(styles: ReturnType): TableColumnDefin * @param props.timeRange - Active time-range preset from the parent page. * @param props.origin - Active origin filter from the parent page. */ -export function BanTable({ timeRange, origin = "all" }: BanTableProps): React.JSX.Element { +export function BanTable({ timeRange, origin = "all", source = "fail2ban" }: BanTableProps): React.JSX.Element { const styles = useStyles(); - const { banItems, total, page, setPage, loading, error, refresh } = useBans(timeRange, origin); + const { banItems, total, page, setPage, loading, error, refresh } = useBans(timeRange, origin, source); const banColumns = buildBanColumns(styles); diff --git a/frontend/src/components/BanTrendChart.tsx b/frontend/src/components/BanTrendChart.tsx index 9f56586..32dbfaa 100644 --- a/frontend/src/components/BanTrendChart.tsx +++ b/frontend/src/components/BanTrendChart.tsx @@ -53,6 +53,8 @@ interface BanTrendChartProps { timeRange: TimeRange; /** Origin filter controlling which bans are included. */ origin: BanOriginFilter; + /** Data source used for the chart. */ + source?: "fail2ban" | "archive"; } /** Internal chart data point shape. */ @@ -188,9 +190,10 @@ function TrendTooltip(props: TooltipContentProps): React.JSX.Element | null { export function BanTrendChart({ timeRange, origin, + source = "fail2ban", }: BanTrendChartProps): React.JSX.Element { const styles = useStyles(); - const { buckets, isLoading, error, reload } = useBanTrend(timeRange, origin); + const { buckets, isLoading, error, reload } = useBanTrend(timeRange, origin, source); const isEmpty = buckets.every((b) => b.count === 0); const entries = buildEntries(buckets, timeRange); diff --git a/frontend/src/hooks/useBanTrend.ts b/frontend/src/hooks/useBanTrend.ts index 2e45660..b258081 100644 --- a/frontend/src/hooks/useBanTrend.ts +++ b/frontend/src/hooks/useBanTrend.ts @@ -42,6 +42,7 @@ export interface UseBanTrendResult { export function useBanTrend( timeRange: TimeRange, origin: BanOriginFilter, + source: "fail2ban" | "archive" = "fail2ban", ): UseBanTrendResult { const [buckets, setBuckets] = useState([]); const [bucketSize, setBucketSize] = useState("1h"); @@ -58,7 +59,7 @@ export function useBanTrend( setIsLoading(true); setError(null); - fetchBanTrend(timeRange, origin) + fetchBanTrend(timeRange, origin, source) .then((data) => { if (controller.signal.aborted) return; setBuckets(data.buckets); @@ -73,7 +74,7 @@ export function useBanTrend( setIsLoading(false); } }); - }, [timeRange, origin]); + }, [timeRange, origin, source]); useEffect(() => { load(); diff --git a/frontend/src/hooks/useBans.ts b/frontend/src/hooks/useBans.ts index 9e36f45..c51471c 100644 --- a/frontend/src/hooks/useBans.ts +++ b/frontend/src/hooks/useBans.ts @@ -44,6 +44,7 @@ export interface UseBansResult { export function useBans( timeRange: TimeRange, origin: BanOriginFilter = "all", + source: "fail2ban" | "archive" = "fail2ban", ): UseBansResult { const [banItems, setBanItems] = useState([]); const [total, setTotal] = useState(0); @@ -51,16 +52,16 @@ export function useBans( const [loading, setLoading] = useState(true); const [error, setError] = useState(null); - // Reset page when time range or origin filter changes. + // Reset page when time range, origin filter, or source changes. useEffect(() => { setPage(1); - }, [timeRange, origin]); + }, [timeRange, origin, source]); const doFetch = useCallback(async (): Promise => { setLoading(true); setError(null); try { - const data = await fetchBans(timeRange, page, PAGE_SIZE, origin); + const data = await fetchBans(timeRange, page, PAGE_SIZE, origin, source); setBanItems(data.items); setTotal(data.total); } catch (err: unknown) { @@ -68,7 +69,7 @@ export function useBans( } finally { setLoading(false); } - }, [timeRange, page, origin]); + }, [timeRange, page, origin, source]); // Stable ref to the latest doFetch so the refresh callback is always current. const doFetchRef = useRef(doFetch); diff --git a/frontend/src/hooks/useDashboardCountryData.ts b/frontend/src/hooks/useDashboardCountryData.ts index 250fcff..3c9e625 100644 --- a/frontend/src/hooks/useDashboardCountryData.ts +++ b/frontend/src/hooks/useDashboardCountryData.ts @@ -48,6 +48,7 @@ export interface UseDashboardCountryDataResult { export function useDashboardCountryData( timeRange: TimeRange, origin: BanOriginFilter, + source: "fail2ban" | "archive" = "fail2ban", ): UseDashboardCountryDataResult { const [countries, setCountries] = useState>({}); const [countryNames, setCountryNames] = useState>({}); @@ -67,7 +68,7 @@ export function useDashboardCountryData( setIsLoading(true); setError(null); - fetchBansByCountry(timeRange, origin) + fetchBansByCountry(timeRange, origin, source) .then((data) => { if (controller.signal.aborted) return; setCountries(data.countries); @@ -85,7 +86,7 @@ export function useDashboardCountryData( setIsLoading(false); } }); - }, [timeRange, origin]); + }, [timeRange, origin, source]); useEffect(() => { load(); diff --git a/frontend/src/hooks/useMapData.ts b/frontend/src/hooks/useMapData.ts index d4537e6..f1d2f76 100644 --- a/frontend/src/hooks/useMapData.ts +++ b/frontend/src/hooks/useMapData.ts @@ -43,6 +43,7 @@ export interface UseMapDataResult { export function useMapData( range: TimeRange = "24h", origin: BanOriginFilter = "all", + source: "fail2ban" | "archive" = "fail2ban", ): UseMapDataResult { const [data, setData] = useState(null); const [loading, setLoading] = useState(true); @@ -64,7 +65,7 @@ export function useMapData( abortRef.current?.abort(); abortRef.current = new AbortController(); - fetchBansByCountry(range, origin) + fetchBansByCountry(range, origin, source) .then((resp) => { setData(resp); }) @@ -75,7 +76,7 @@ export function useMapData( setLoading(false); }); }, DEBOUNCE_MS); - }, [range, origin]); + }, [range, origin, source]); useEffect((): (() => void) => { load(); diff --git a/frontend/src/pages/DashboardPage.tsx b/frontend/src/pages/DashboardPage.tsx index c22abcc..60921fa 100644 --- a/frontend/src/pages/DashboardPage.tsx +++ b/frontend/src/pages/DashboardPage.tsx @@ -7,7 +7,7 @@ */ import { useState } from "react"; -import { Text, makeStyles, tokens } from "@fluentui/react-components"; +import { Badge, Text, makeStyles, tokens } from "@fluentui/react-components"; import { BanTable } from "../components/BanTable"; import { BanTrendChart } from "../components/BanTrendChart"; import { ChartStateWrapper } from "../components/ChartStateWrapper"; @@ -71,8 +71,10 @@ export function DashboardPage(): React.JSX.Element { const [timeRange, setTimeRange] = useState("24h"); const [originFilter, setOriginFilter] = useState("all"); + const source = timeRange === "24h" ? "fail2ban" : "archive"; + const { countries, countryNames, isLoading: countryLoading, error: countryError, reload: reloadCountry } = - useDashboardCountryData(timeRange, originFilter); + useDashboardCountryData(timeRange, originFilter, source); const sectionStyles = useCommonSectionStyles(); @@ -86,12 +88,17 @@ export function DashboardPage(): React.JSX.Element { {/* ------------------------------------------------------------------ */} {/* Global filter bar */} {/* ------------------------------------------------------------------ */} - +
+ + + {source === "archive" ? "Archive (BanGUI DB)" : "Live (fail2ban DB)"} + +
{/* ------------------------------------------------------------------ */} {/* Ban Trend section */} @@ -103,7 +110,7 @@ export function DashboardPage(): React.JSX.Element {
- +
@@ -154,7 +161,7 @@ export function DashboardPage(): React.JSX.Element { {/* Ban table */}
- +
diff --git a/frontend/src/pages/HistoryPage.tsx b/frontend/src/pages/HistoryPage.tsx index debdb85..83de3fc 100644 --- a/frontend/src/pages/HistoryPage.tsx +++ b/frontend/src/pages/HistoryPage.tsx @@ -143,6 +143,7 @@ function areHistoryQueriesEqual( a.origin === b.origin && a.jail === b.jail && a.ip === b.ip && + a.source === b.source && a.page === b.page && a.page_size === b.page_size ); @@ -386,11 +387,12 @@ export function HistoryPage(): React.JSX.Element { const styles = useStyles(); // Filter state - const [range, setRange] = useState("24h"); + const [range, setRange] = useState("7d"); const [originFilter, setOriginFilter] = useState("all"); const [jailFilter, setJailFilter] = useState(""); const [ipFilter, setIpFilter] = useState(""); const [appliedQuery, setAppliedQuery] = useState({ + source: "archive", page_size: PAGE_SIZE, }); @@ -400,12 +402,15 @@ export function HistoryPage(): React.JSX.Element { const { items, total, page, loading, error, setPage, refresh } = useHistory(appliedQuery); + const sourceLabel = "Archive (BanGUI DB)"; + useEffect((): void => { const nextQuery: HistoryQuery = { range, origin: originFilter !== "all" ? originFilter : undefined, jail: jailFilter.trim() || undefined, ip: ipFilter.trim() || undefined, + source: "archive", page: 1, page_size: PAGE_SIZE, }; @@ -485,6 +490,9 @@ export function HistoryPage(): React.JSX.Element { setIpFilter(value); }} /> + + {sourceLabel} + {/* ---------------------------------------------------------------- */} diff --git a/frontend/src/pages/MapPage.tsx b/frontend/src/pages/MapPage.tsx index 1b3fbdc..60ce55f 100644 --- a/frontend/src/pages/MapPage.tsx +++ b/frontend/src/pages/MapPage.tsx @@ -98,8 +98,10 @@ export function MapPage(): React.JSX.Element { const PAGE_SIZE_OPTIONS = [25, 50, 100] as const; + const source = range === "24h" ? "fail2ban" : "archive"; + const { countries, countryNames, bans, total, loading, error, refresh } = - useMapData(range, originFilter); + useMapData(range, originFilter, source); const { thresholds: mapThresholds, @@ -163,6 +165,9 @@ export function MapPage(): React.JSX.Element { setSelectedCountry(null); }} /> + + {source === "archive" ? "Archive (BanGUI DB)" : "Live (fail2ban DB)"} +