TASK-032: Implement geo_cache retention policy and cleanup
Add automatic cleanup of stale geolocation cache entries to prevent unbounded database growth. Resolves the issue where unique IP addresses accumulated indefinitely in the geo_cache table, degrading query performance. ## Changes ### Database Schema (Migration 3) - Add 'last_seen' column to geo_cache table tracking last reference time - Existing entries default to current timestamp ### Repository Layer (geo_cache_repo.py) - Update upsert_entry() to set/refresh last_seen on insert/update - Update upsert_neg_entry() to set/refresh last_seen on negative cache hits - Update bulk_upsert_entries() to set/refresh last_seen in batch operations - Add delete_stale_entries(db, cutoff_iso) -> int for purging old entries ### Background Task (geo_cache_cleanup.py) - New APScheduler task that runs nightly (24-hour interval) - Calculates cutoff as 90 days ago from current time (UTC) - Deletes all entries with last_seen older than cutoff - Logs operation results (info when deleted > 0, debug when 0 deleted) - Configurable retention period via GEO_CACHE_RETENTION_DAYS constant ### Application Startup (startup.py) - Register geo_cache_cleanup task in scheduler during app startup - Placed after geo_cache_flush in task registration order ### Tests - Add delete_stale_entries test cases covering: * Removal of old entries beyond cutoff * No deletion when all entries are recent * Empty table edge case - Update existing test fixtures to include last_seen column - Add full test suite for cleanup task registration and execution ### Documentation - Architekture.md: Document cleanup task, update schema/diagram - Backend-Development.md: Add retention policy documentation ## Behavior When an IP is accessed, its last_seen is refreshed. After 90 days of no access, an IP is purged by the nightly cleanup. On next encounter, the IP is re-resolved from MaxMind MMDB or ip-api.com (if configured). This is acceptable because: 1. Stale geolocation data may become inaccurate over time 2. Re-resolution cost is minimal compared to unbounded storage growth 3. Active IPs maintain fresh data through their last_seen updates Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
175
backend/tests/test_tasks/test_geo_cache_cleanup.py
Normal file
175
backend/tests/test_tasks/test_geo_cache_cleanup.py
Normal file
@@ -0,0 +1,175 @@
|
||||
"""Tests for the geo cache cleanup background task.
|
||||
|
||||
Validates that :func:`~app.tasks.geo_cache_cleanup._run_cleanup_with_resources` correctly
|
||||
calculates the cutoff timestamp, calls :func:`~app.repositories.geo_cache_repo.delete_stale_entries`,
|
||||
and logs appropriately based on the number of deleted entries, and that
|
||||
:func:`~app.tasks.geo_cache_cleanup.register` configures the APScheduler job with the correct
|
||||
interval and stable job ID.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.tasks.geo_cache_cleanup import GEO_CLEANUP_INTERVAL, JOB_ID, register
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_app() -> MagicMock:
|
||||
"""Build a minimal mock ``app`` for geo cache cleanup task tests.
|
||||
|
||||
Returns:
|
||||
A :class:`unittest.mock.MagicMock` that mimics ``fastapi.FastAPI``.
|
||||
"""
|
||||
app = MagicMock()
|
||||
app.state.scheduler = MagicMock()
|
||||
app.state.settings = MagicMock(database_path="/tmp/fake.db")
|
||||
app.state.runtime_settings = None
|
||||
return app
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for _run_cleanup
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRunCleanup:
|
||||
"""Tests for :func:`~app.tasks.geo_cache_cleanup._run_cleanup_with_resources`."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_cleanup_calls_delete_stale_entries(self) -> None:
|
||||
"""``_run_cleanup_with_resources`` must call ``delete_stale_entries`` with a cutoff date."""
|
||||
settings = MagicMock(database_path="/tmp/fake.db")
|
||||
|
||||
with patch(
|
||||
"app.tasks.db.task_db",
|
||||
MagicMock(
|
||||
return_value=AsyncMock(
|
||||
__aenter__=AsyncMock(return_value=MagicMock()),
|
||||
__aexit__=AsyncMock(return_value=False),
|
||||
)
|
||||
),
|
||||
), patch(
|
||||
"app.tasks.geo_cache_cleanup.geo_cache_repo.delete_stale_entries",
|
||||
new_callable=AsyncMock,
|
||||
return_value=0,
|
||||
) as mock_delete:
|
||||
from app.tasks.geo_cache_cleanup import _run_cleanup_with_resources
|
||||
|
||||
await _run_cleanup_with_resources(settings)
|
||||
|
||||
mock_delete.assert_awaited_once()
|
||||
# Verify the cutoff timestamp is in ISO format
|
||||
call_args = mock_delete.call_args
|
||||
assert call_args is not None
|
||||
cutoff_iso = call_args[0][1] # Second positional arg
|
||||
assert isinstance(cutoff_iso, str)
|
||||
assert "T" in cutoff_iso and "Z" in cutoff_iso
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_cleanup_logs_when_entries_deleted(self) -> None:
|
||||
"""``_run_cleanup_with_resources`` must emit an info log when entries are deleted."""
|
||||
settings = MagicMock(database_path="/tmp/fake.db")
|
||||
|
||||
with patch(
|
||||
"app.tasks.db.task_db",
|
||||
MagicMock(
|
||||
return_value=AsyncMock(
|
||||
__aenter__=AsyncMock(return_value=MagicMock()),
|
||||
__aexit__=AsyncMock(return_value=False),
|
||||
)
|
||||
),
|
||||
), patch(
|
||||
"app.tasks.geo_cache_cleanup.geo_cache_repo.delete_stale_entries",
|
||||
new_callable=AsyncMock,
|
||||
return_value=42,
|
||||
), patch(
|
||||
"app.tasks.geo_cache_cleanup.log"
|
||||
) as mock_log:
|
||||
from app.tasks.geo_cache_cleanup import _run_cleanup_with_resources
|
||||
|
||||
await _run_cleanup_with_resources(settings)
|
||||
|
||||
info_calls = [c for c in mock_log.info.call_args_list if c[0][0] == "geo_cache_cleanup_ran"]
|
||||
assert len(info_calls) == 1
|
||||
assert info_calls[0][1]["deleted"] == 42
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_cleanup_logs_debug_when_nothing_deleted(self) -> None:
|
||||
"""``_run_cleanup_with_resources`` must emit a debug log when 0 entries are deleted."""
|
||||
settings = MagicMock(database_path="/tmp/fake.db")
|
||||
|
||||
with patch(
|
||||
"app.tasks.db.task_db",
|
||||
MagicMock(
|
||||
return_value=AsyncMock(
|
||||
__aenter__=AsyncMock(return_value=MagicMock()),
|
||||
__aexit__=AsyncMock(return_value=False),
|
||||
)
|
||||
),
|
||||
), patch(
|
||||
"app.tasks.geo_cache_cleanup.geo_cache_repo.delete_stale_entries",
|
||||
new_callable=AsyncMock,
|
||||
return_value=0,
|
||||
), patch(
|
||||
"app.tasks.geo_cache_cleanup.log"
|
||||
) as mock_log:
|
||||
from app.tasks.geo_cache_cleanup import _run_cleanup_with_resources
|
||||
|
||||
await _run_cleanup_with_resources(settings)
|
||||
|
||||
debug_calls = [c for c in mock_log.debug.call_args_list if c[0][0] == "geo_cache_cleanup_ran"]
|
||||
assert len(debug_calls) == 1
|
||||
assert debug_calls[0][1]["deleted"] == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for register
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRegister:
|
||||
"""Tests for :func:`~app.tasks.geo_cache_cleanup.register`."""
|
||||
|
||||
def test_register_adds_interval_job_to_scheduler(self) -> None:
|
||||
"""``register`` must add a job with an ``"interval"`` trigger."""
|
||||
app = _make_app()
|
||||
|
||||
register(app)
|
||||
|
||||
app.state.scheduler.add_job.assert_called_once()
|
||||
_, kwargs = app.state.scheduler.add_job.call_args
|
||||
assert kwargs["trigger"] == "interval"
|
||||
assert kwargs["seconds"] == GEO_CLEANUP_INTERVAL
|
||||
|
||||
def test_register_uses_stable_job_id(self) -> None:
|
||||
"""``register`` must use the module-level ``JOB_ID`` constant."""
|
||||
app = _make_app()
|
||||
|
||||
register(app)
|
||||
|
||||
_, kwargs = app.state.scheduler.add_job.call_args
|
||||
assert kwargs["id"] == JOB_ID
|
||||
|
||||
def test_register_sets_replace_existing(self) -> None:
|
||||
"""``register`` must use ``replace_existing=True`` to avoid duplicate jobs."""
|
||||
app = _make_app()
|
||||
|
||||
register(app)
|
||||
|
||||
_, kwargs = app.state.scheduler.add_job.call_args
|
||||
assert kwargs["replace_existing"] is True
|
||||
|
||||
def test_register_passes_settings_in_kwargs(self) -> None:
|
||||
"""The scheduled job must receive settings as kwargs instead of app."""
|
||||
app = _make_app()
|
||||
|
||||
register(app)
|
||||
|
||||
_, kwargs = app.state.scheduler.add_job.call_args
|
||||
assert "settings" in kwargs["kwargs"]
|
||||
Reference in New Issue
Block a user