Extract health-check crash-detection logic into runtime state helper

backend/app/utils/runtime_state.py

@@ -15,14 +15,22 @@ from typing import TYPE_CHECKING, Any
 
 from starlette.datastructures import State
 
+import structlog
+
 from app.models.config import PendingRecovery
 from app.models.server import ServerStatus
 
 if TYPE_CHECKING:  # pragma: no cover
     from app.config import Settings
 
+log: structlog.stdlib.BoundLogger = structlog.get_logger()
+
 ActivationRecord = dict[str, datetime.datetime]
 
+# Maximum seconds since an activation for a subsequent crash to be
+# attributed to that activation.
+_ACTIVATION_CRASH_WINDOW: int = 60
+
 _RUNTIME_ATTRIBUTES: frozenset[str] = frozenset(
     {
         "setup_complete_cached",
@@ -151,3 +159,67 @@ def clear_pending_recovery(app: Any) -> None:
 def clear_activation_record(app: Any) -> None:
     """Clear the current activation tracking record."""
     get_runtime_state(app).last_activation = None
+
+
+def process_health_probe_result(
+    runtime_state: RuntimeState,
+    status: ServerStatus,
+    now: datetime.datetime | None = None,
+) -> None:
+    """Process a new health probe result and update runtime state.
+
+    This function tracks fail2ban transitions and creates or resolves
+    pending recovery records when the daemon goes offline shortly after a
+    jail activation.
+
+    Args:
+        runtime_state: The mutable runtime state manager.
+        status: The latest fail2ban server status.
+        now: The current timestamp used for time-based decisions.
+    """
+    prev_status = getattr(runtime_state, "server_status", ServerStatus(online=False))
+    runtime_state.server_status = status
+    now = now if now is not None else datetime.datetime.now(tz=datetime.UTC)
+
+    if status.online and not prev_status.online:
+        log.info("fail2ban_came_online", version=status.version)
+        existing = runtime_state.pending_recovery
+        if existing is not None and not existing.recovered:
+            runtime_state.pending_recovery = PendingRecovery(
+                jail_name=existing.jail_name,
+                activated_at=existing.activated_at,
+                detected_at=existing.detected_at,
+                recovered=True,
+            )
+            log.info(
+                "pending_recovery_resolved",
+                jail=existing.jail_name,
+            )
+
+    elif not status.online and prev_status.online:
+        log.warning("fail2ban_went_offline")
+        last_activation = runtime_state.last_activation
+        if last_activation is not None:
+            activated_at = last_activation["at"]
+            seconds_since = (now - activated_at).total_seconds()
+            if seconds_since <= _ACTIVATION_CRASH_WINDOW:
+                jail_name = last_activation["jail_name"]
+                current = runtime_state.pending_recovery
+                if current is None or current.recovered:
+                    runtime_state.pending_recovery = PendingRecovery(
+                        jail_name=jail_name,
+                        activated_at=activated_at,
+                        detected_at=now,
+                    )
+                    log.warning(
+                        "activation_crash_detected",
+                        jail=jail_name,
+                        seconds_since_activation=seconds_since,
+                    )
+
+    log.debug(
+        "health_check_complete",
+        online=status.online,
+        version=status.version,
+        active_jails=status.active_jails,
+    )