lukas.pupkalipinski/BanGUI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TASK-032: Implement geo_cache retention policy and cleanup

Browse Source 
Add automatic cleanup of stale geolocation cache entries to prevent
unbounded database growth. Resolves the issue where unique IP addresses
accumulated indefinitely in the geo_cache table, degrading query performance.

## Changes

### Database Schema (Migration 3)
- Add 'last_seen' column to geo_cache table tracking last reference time
- Existing entries default to current timestamp

### Repository Layer (geo_cache_repo.py)
- Update upsert_entry() to set/refresh last_seen on insert/update
- Update upsert_neg_entry() to set/refresh last_seen on negative cache hits
- Update bulk_upsert_entries() to set/refresh last_seen in batch operations
- Add delete_stale_entries(db, cutoff_iso) -> int for purging old entries

### Background Task (geo_cache_cleanup.py)
- New APScheduler task that runs nightly (24-hour interval)
- Calculates cutoff as 90 days ago from current time (UTC)
- Deletes all entries with last_seen older than cutoff
- Logs operation results (info when deleted > 0, debug when 0 deleted)
- Configurable retention period via GEO_CACHE_RETENTION_DAYS constant

### Application Startup (startup.py)
- Register geo_cache_cleanup task in scheduler during app startup
- Placed after geo_cache_flush in task registration order

### Tests
- Add delete_stale_entries test cases covering:
  * Removal of old entries beyond cutoff
  * No deletion when all entries are recent
  * Empty table edge case
- Update existing test fixtures to include last_seen column
- Add full test suite for cleanup task registration and execution

### Documentation
- Architekture.md: Document cleanup task, update schema/diagram
- Backend-Development.md: Add retention policy documentation

## Behavior

When an IP is accessed, its last_seen is refreshed. After 90 days of no
access, an IP is purged by the nightly cleanup. On next encounter, the IP
is re-resolved from MaxMind MMDB or ip-api.com (if configured).

This is acceptable because:
1. Stale geolocation data may become inaccurate over time
2. Re-resolution cost is minimal compared to unbounded storage growth
3. Active IPs maintain fresh data through their last_seen updates

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Lukas
2026-04-26 19:24:34 +02:00
parent 32aad186c3
commit e2560f5db0
9 changed files with 405 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
backend/tests/test_repositories/test_geo_cache_repo.py
View File

@@ -17,7 +17,8 @@ async def _create_geo_cache_table(db: aiosqlite.Connection) -> None:
country_name TEXT,
asn TEXT,
org TEXT,
cached_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
cached_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
last_seen TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
)
"""
)
@@ -183,3 +184,82 @@ async def test_bulk_upsert_entries_and_neg_entries(tmp_path: Path) -> None:
row = await cur.fetchone()
assert row is not None
assert int(row[0]) == 4
@pytest.mark.asyncio
async def test_delete_stale_entries_removes_old_entries(tmp_path: Path) -> None:
db_path = str(tmp_path / "geo_cache.db")
async with aiosqlite.connect(db_path) as db:
await _create_geo_cache_table(db)
# Insert entries with various last_seen times
await db.execute(
"INSERT INTO geo_cache (ip, country_code, last_seen) VALUES (?, ?, ?)",
("1.1.1.1", "US", "2020-01-01T00:00:00Z"),
)
await db.execute(
"INSERT INTO geo_cache (ip, country_code, last_seen) VALUES (?, ?, ?)",
("2.2.2.2", "DE", "2024-12-01T00:00:00Z"),
)
await db.execute(
"INSERT INTO geo_cache (ip, country_code, last_seen) VALUES (?, ?, ?)",
("3.3.3.3", "FR", "2025-01-01T00:00:00Z"),
)
await db.commit()
async with aiosqlite.connect(db_path) as db:
# Delete entries older than 2024-06-01
deleted = await geo_cache_repo.delete_stale_entries(db, "2024-06-01T00:00:00Z")
await db.commit()
assert deleted == 1
# Verify the correct entry was deleted
async with aiosqlite.connect(db_path) as db, db.execute("SELECT ip FROM geo_cache ORDER BY ip") as cur:
rows = await cur.fetchall()
ips = [row[0] for row in rows]
assert sorted(ips) == ["2.2.2.2", "3.3.3.3"]
@pytest.mark.asyncio
async def test_delete_stale_entries_returns_zero_when_none_stale(tmp_path: Path) -> None:
db_path = str(tmp_path / "geo_cache.db")
async with aiosqlite.connect(db_path) as db:
await _create_geo_cache_table(db)
# Insert entries with recent last_seen times
await db.execute(
"INSERT INTO geo_cache (ip, country_code, last_seen) VALUES (?, ?, ?)",
("1.1.1.1", "US", "2025-01-01T00:00:00Z"),
)
await db.execute(
"INSERT INTO geo_cache (ip, country_code, last_seen) VALUES (?, ?, ?)",
("2.2.2.2", "DE", "2025-01-02T00:00:00Z"),
)
await db.commit()
async with aiosqlite.connect(db_path) as db:
# Try to delete entries older than 2020-01-01 (all are newer)
deleted = await geo_cache_repo.delete_stale_entries(db, "2020-01-01T00:00:00Z")
await db.commit()
assert deleted == 0
# Verify no entries were deleted
async with aiosqlite.connect(db_path) as db, db.execute("SELECT COUNT(*) FROM geo_cache") as cur:
row = await cur.fetchone()
assert row is not None
assert int(row[0]) == 2
@pytest.mark.asyncio
async def test_delete_stale_entries_with_empty_table(tmp_path: Path) -> None:
db_path = str(tmp_path / "geo_cache.db")
async with aiosqlite.connect(db_path) as db:
await _create_geo_cache_table(db)
async with aiosqlite.connect(db_path) as db:
deleted = await geo_cache_repo.delete_stale_entries(db, "2024-01-01T00:00:00Z")
await db.commit()
assert deleted == 0

175
backend/tests/test_tasks/test_geo_cache_cleanup.py Normal file
View File

@@ -0,0 +1,175 @@
"""Tests for the geo cache cleanup background task.
Validates that :func:`~app.tasks.geo_cache_cleanup._run_cleanup_with_resources` correctly
calculates the cutoff timestamp, calls :func:`~app.repositories.geo_cache_repo.delete_stale_entries`,
and logs appropriately based on the number of deleted entries, and that
:func:`~app.tasks.geo_cache_cleanup.register` configures the APScheduler job with the correct
interval and stable job ID.
"""
from __future__ import annotations
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from app.tasks.geo_cache_cleanup import GEO_CLEANUP_INTERVAL, JOB_ID, register
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _make_app() -> MagicMock:
"""Build a minimal mock ``app`` for geo cache cleanup task tests.
Returns:
A :class:`unittest.mock.MagicMock` that mimics ``fastapi.FastAPI``.
"""
app = MagicMock()
app.state.scheduler = MagicMock()
app.state.settings = MagicMock(database_path="/tmp/fake.db")
app.state.runtime_settings = None
return app
# ---------------------------------------------------------------------------
# Tests for _run_cleanup
# ---------------------------------------------------------------------------
class TestRunCleanup:
"""Tests for :func:`~app.tasks.geo_cache_cleanup._run_cleanup_with_resources`."""
@pytest.mark.asyncio
async def test_run_cleanup_calls_delete_stale_entries(self) -> None:
"""``_run_cleanup_with_resources`` must call ``delete_stale_entries`` with a cutoff date."""
settings = MagicMock(database_path="/tmp/fake.db")
with patch(
"app.tasks.db.task_db",
MagicMock(
return_value=AsyncMock(
__aenter__=AsyncMock(return_value=MagicMock()),
__aexit__=AsyncMock(return_value=False),
)
),
), patch(
"app.tasks.geo_cache_cleanup.geo_cache_repo.delete_stale_entries",
new_callable=AsyncMock,
return_value=0,
) as mock_delete:
from app.tasks.geo_cache_cleanup import _run_cleanup_with_resources
await _run_cleanup_with_resources(settings)
mock_delete.assert_awaited_once()
# Verify the cutoff timestamp is in ISO format
call_args = mock_delete.call_args
assert call_args is not None
cutoff_iso = call_args[0][1] # Second positional arg
assert isinstance(cutoff_iso, str)
assert "T" in cutoff_iso and "Z" in cutoff_iso
@pytest.mark.asyncio
async def test_run_cleanup_logs_when_entries_deleted(self) -> None:
"""``_run_cleanup_with_resources`` must emit an info log when entries are deleted."""
settings = MagicMock(database_path="/tmp/fake.db")
with patch(
"app.tasks.db.task_db",
MagicMock(
return_value=AsyncMock(
__aenter__=AsyncMock(return_value=MagicMock()),
__aexit__=AsyncMock(return_value=False),
)
),
), patch(
"app.tasks.geo_cache_cleanup.geo_cache_repo.delete_stale_entries",
new_callable=AsyncMock,
return_value=42,
), patch(
"app.tasks.geo_cache_cleanup.log"
) as mock_log:
from app.tasks.geo_cache_cleanup import _run_cleanup_with_resources
await _run_cleanup_with_resources(settings)
info_calls = [c for c in mock_log.info.call_args_list if c[0][0] == "geo_cache_cleanup_ran"]
assert len(info_calls) == 1
assert info_calls[0][1]["deleted"] == 42
@pytest.mark.asyncio
async def test_run_cleanup_logs_debug_when_nothing_deleted(self) -> None:
"""``_run_cleanup_with_resources`` must emit a debug log when 0 entries are deleted."""
settings = MagicMock(database_path="/tmp/fake.db")
with patch(
"app.tasks.db.task_db",
MagicMock(
return_value=AsyncMock(
__aenter__=AsyncMock(return_value=MagicMock()),
__aexit__=AsyncMock(return_value=False),
)
),
), patch(
"app.tasks.geo_cache_cleanup.geo_cache_repo.delete_stale_entries",
new_callable=AsyncMock,
return_value=0,
), patch(
"app.tasks.geo_cache_cleanup.log"
) as mock_log:
from app.tasks.geo_cache_cleanup import _run_cleanup_with_resources
await _run_cleanup_with_resources(settings)
debug_calls = [c for c in mock_log.debug.call_args_list if c[0][0] == "geo_cache_cleanup_ran"]
assert len(debug_calls) == 1
assert debug_calls[0][1]["deleted"] == 0
# ---------------------------------------------------------------------------
# Tests for register
# ---------------------------------------------------------------------------
class TestRegister:
"""Tests for :func:`~app.tasks.geo_cache_cleanup.register`."""
def test_register_adds_interval_job_to_scheduler(self) -> None:
"""``register`` must add a job with an ``"interval"`` trigger."""
app = _make_app()
register(app)
app.state.scheduler.add_job.assert_called_once()
_, kwargs = app.state.scheduler.add_job.call_args
assert kwargs["trigger"] == "interval"
assert kwargs["seconds"] == GEO_CLEANUP_INTERVAL
def test_register_uses_stable_job_id(self) -> None:
"""``register`` must use the module-level ``JOB_ID`` constant."""
app = _make_app()
register(app)
_, kwargs = app.state.scheduler.add_job.call_args
assert kwargs["id"] == JOB_ID
def test_register_sets_replace_existing(self) -> None:
"""``register`` must use ``replace_existing=True`` to avoid duplicate jobs."""
app = _make_app()
register(app)
_, kwargs = app.state.scheduler.add_job.call_args
assert kwargs["replace_existing"] is True
def test_register_passes_settings_in_kwargs(self) -> None:
"""The scheduled job must receive settings as kwargs instead of app."""
app = _make_app()
register(app)
_, kwargs = app.state.scheduler.add_job.call_args
assert "settings" in kwargs["kwargs"]