Refactor scheduler lock implementation with heartbeat mechanism

- Add heartbeat-based lock renewal in scheduler_lock_heartbeat.py
- Update scheduler_lock.py with improved lock management
- Add comprehensive tests for scheduler lock functionality
- Update deployment and task documentation

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

backend/tests/test_scheduler_lock.py

@@ -2,7 +2,7 @@
 
 These tests verify that the database-backed scheduler lock correctly enforces
 single-executor safety across multiple startup attempts, including stale lock
-cleanup and heartbeat updates.
+cleanup, heartbeat updates, and multi-process race condition prevention.
 """
 
 from __future__ import annotations
@@ -15,6 +15,7 @@ import aiosqlite
 import pytest
 
 from app.utils.scheduler_lock import (
+    SCHEDULER_LOCK_HEARTBEAT_INTERVAL_SECONDS,
     SCHEDULER_LOCK_TTL_SECONDS,
     acquire_scheduler_lock,
     get_scheduler_lock_info,
@@ -220,3 +221,75 @@ async def test_scheduler_lock_full_lifecycle(
     await release_scheduler_lock(lock_db)
     info = await get_scheduler_lock_info(lock_db)
     assert info is None
+
+
+@pytest.mark.asyncio
+async def test_scheduler_lock_heartbeat_interval_sanity(
+    lock_db: aiosqlite.Connection,
+) -> None:
+    """Verify heartbeat interval is less than TTL to prevent premature expiry.
+
+    With a 5-second heartbeat interval and 60-second TTL, the lock can survive
+    ~12 missed heartbeats before expiring. This provides robust protection against
+    temporary delays or high load that could cause a single missed heartbeat.
+    """
+    # Verify the configuration ratio is safe (interval < TTL)
+    assert SCHEDULER_LOCK_HEARTBEAT_INTERVAL_SECONDS < SCHEDULER_LOCK_TTL_SECONDS
+
+    # With this ratio, the lock can survive at least 12 missed heartbeats
+    # (60s TTL / 5s interval = 12 intervals between heartbeats before expiry)
+    safe_ratio = SCHEDULER_LOCK_TTL_SECONDS / SCHEDULER_LOCK_HEARTBEAT_INTERVAL_SECONDS
+    assert safe_ratio >= 12, (
+        f"Heartbeat interval too long: lock can only survive {safe_ratio:.1f} missed heartbeats. "
+        f"Should be at least 12 for safety."
+    )
+
+
+@pytest.mark.asyncio
+async def test_scheduler_lock_race_condition_prevention(
+    lock_db: aiosqlite.Connection,
+) -> None:
+    """Test that the lock prevents concurrent execution (race condition).
+
+    Scenario: Process A acquires the lock and starts working. Process B starts
+    up and tries to acquire the lock. Even if Process A's heartbeat fails
+    momentarily, Process B should not acquire the lock immediately.
+
+    This test verifies:
+    1. Only one process can hold the lock at a time
+    2. The lock cannot be stolen while being actively maintained (via heartbeat)
+    3. Stale locks are only cleaned after TTL expires
+    """
+    # Process A acquires the lock
+    result_a = await acquire_scheduler_lock(lock_db)
+    assert result_a is True
+
+    # Get the lock info
+    info_a = await get_scheduler_lock_info(lock_db)
+    assert info_a is not None
+    lock_heartbeat_a = info_a["heartbeat_at"]
+
+    # Process B tries to acquire — should fail
+    result_b = await acquire_scheduler_lock(lock_db)
+    assert result_b is False
+
+    # Process A updates its heartbeat (simulating ongoing work)
+    time.sleep(0.01)
+    result_heartbeat = await update_scheduler_lock_heartbeat(lock_db)
+    assert result_heartbeat is True
+
+    # Verify heartbeat was updated
+    info_a_updated = await get_scheduler_lock_info(lock_db)
+    assert info_a_updated is not None
+    assert info_a_updated["heartbeat_at"] > lock_heartbeat_a
+
+    # Process B still cannot acquire the lock (it's active and well-maintained)
+    result_b_retry = await acquire_scheduler_lock(lock_db)
+    assert result_b_retry is False
+
+    # Process A releases the lock
+    await release_scheduler_lock(lock_db)
+
+    # Now Process B can acquire it
+    result_b_final = await acquire_scheduler_lock(lock_db)
+    assert result_b_final is True