Add Kubernetes liveness/readiness probes and middleware order validation

- Split /health into /health/live (liveness) and /health/ready (readiness)
  following Kubernetes conventions. Combined /health retained for backward
  compatibility with existing Docker HEALTHCHECK definitions.
- Add ReadyCheck and ReadyResponse models for structured readiness output.
- Add _assert_middleware_order() startup check enforcing:
  RateLimit → Csrf → CorrelationId middleware chain.
- Register CorrelationIdMiddleware, CsrfMiddleware, RateLimitMiddleware
  in create_app() with documented required order (reverse of processing).
- Add correlation.py, csrf.py, rate_limit.py middleware modules.
- Add health probe tests in test_health_probes.py.
- Update test_main.py with middleware order assertion tests.
- Update frontend useFetchData hook tests.
- Docs: update Deployment.md with Kubernetes probe config examples.

backend/tests/test_routers/test_health_probes.py

@@ -0,0 +1,130 @@
+"""Tests for the health-check router — liveness and readiness probes."""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+from httpx import AsyncClient
+
+from app.models.server import ServerStatus
+from app.models.response import ReadyCheck
+
+
+# ---------------------------------------------------------------------------
+# GET /health/live — liveness probe
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_liveness_returns_200(client: AsyncClient) -> None:
+    """``GET /api/v1/health/live`` must always return HTTP 200."""
+    response = await client.get("/api/v1/health/live")
+    assert response.status_code == 200
+
+
+@pytest.mark.asyncio
+async def test_liveness_body_is_ready_response(client: AsyncClient) -> None:
+    """Response body must be a ReadyResponse."""
+    response = await client.get("/api/v1/health/live")
+    data: dict[str, object] = response.json()
+    assert data["status"] == "ok"
+    assert data["failed_count"] == 0
+    assert "checks" in data
+    assert isinstance(data["checks"], list)
+
+
+@pytest.mark.asyncio
+async def test_liveness_includes_process_check(client: AsyncClient) -> None:
+    """Liveness response must include a 'process' check."""
+    response = await client.get("/api/v1/health/live")
+    data: dict[str, object] = response.json()
+    checks: list[dict[str, object]] = data["checks"]  # type: ignore[assignment]
+    assert any(c.get("name") == "process" and c.get("healthy") is True for c in checks)
+
+
+# ---------------------------------------------------------------------------
+# GET /health/ready — readiness probe
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_readiness_returns_200_when_all_pass(client: AsyncClient) -> None:
+    """``GET /api/v1/health/ready`` must return 200 when all subsystems pass."""
+    with patch("app.routers.health._run_check", side_effect=lambda n, c, e: ReadyCheck(name=n, healthy=True)):
+        response = await client.get("/api/v1/health/ready")
+    assert response.status_code == 200
+
+
+@pytest.mark.asyncio
+async def test_readiness_returns_503_when_subsystem_fails(client: AsyncClient) -> None:
+    """``GET /api/v1/health/ready`` must return 503 when at least one check fails."""
+    # Force fail2ban offline
+    client._transport.app.state.server_status = ServerStatus(online=False)
+    response = await client.get("/api/v1/health/ready")
+    assert response.status_code == 503
+
+
+@pytest.mark.asyncio
+async def test_readiness_body_is_ready_response(client: AsyncClient) -> None:
+    """Response body must be a ReadyResponse."""
+    response = await client.get("/api/v1/health/ready")
+    data: dict[str, object] = response.json()
+    assert data["status"] in ("ok", "error")
+    assert "failed_count" in data
+    assert "checks" in data
+    assert isinstance(data["checks"], list)
+
+
+@pytest.mark.asyncio
+async def test_readiness_includes_all_subsystems(client: AsyncClient) -> None:
+    """Readiness response must include checks for all four subsystems."""
+    response = await client.get("/api/v1/health/ready")
+    data: dict[str, object] = response.json()
+    checks: list[dict[str, object]] = data["checks"]  # type: ignore[assignment]
+    names = {c["name"] for c in checks}
+    assert names == {"database", "fail2ban", "config_dir", "scheduler"}
+
+
+@pytest.mark.asyncio
+async def test_readiness_status_ok_when_all_healthy(client: AsyncClient) -> None:
+    """``status`` must be 'ok' when all checks pass."""
+    with patch("app.routers.health._run_check", side_effect=lambda n, c, e: ReadyCheck(name=n, healthy=True)):
+        response = await client.get("/api/v1/health/ready")
+    data: dict[str, object] = response.json()
+    assert data["status"] == "ok"
+    assert data["failed_count"] == 0
+
+
+@pytest.mark.asyncio
+async def test_readiness_status_error_when_fail2ban_offline(client: AsyncClient) -> None:
+    """``status`` must be 'error' when fail2ban is offline."""
+    client._transport.app.state.server_status = ServerStatus(online=False)
+    response = await client.get("/api/v1/health/ready")
+    data: dict[str, object] = response.json()
+    assert data["status"] == "error"
+    assert data["failed_count"] > 0
+
+
+@pytest.mark.asyncio
+async def test_readiness_includes_failed_subsystem_detail(client: AsyncClient) -> None:
+    """When fail2ban is offline the fail2ban check must include an error message."""
+    client._transport.app.state.server_status = ServerStatus(online=False)
+    response = await client.get("/api/v1/health/ready")
+    data: dict[str, object] = response.json()
+    checks: list[dict[str, object]] = data["checks"]  # type: ignore[assignment]
+    f2b = next(c for c in checks if c["name"] == "fail2ban")
+    assert f2b["healthy"] is False
+    assert f2b["message"] is not None
+
+
+@pytest.mark.asyncio
+async def test_readiness_content_type_is_json(client: AsyncClient) -> None:
+    """``/api/v1/health/ready`` must set the ``Content-Type`` header to JSON."""
+    response = await client.get("/api/v1/health/ready")
+    assert "application/json" in response.headers.get("content-type", "")
+
+
+@pytest.mark.asyncio
+async def test_readiness_live_content_type_is_json(client: AsyncClient) -> None:
+    """``/api/v1/health/live`` must set the ``Content-Type`` header to JSON."""
+    response = await client.get("/api/v1/health/live")
+    assert "application/json" in response.headers.get("content-type", "")