fix: handle lifespan errors gracefully

- Add error tracking in lifespan context manager
- Only cleanup services that were successfully initialized
- Properly handle startup errors without breaking async context
- Fixes RuntimeError: generator didn't stop after athrow()
This commit is contained in:
2026-01-23 17:13:30 +01:00
parent 314f535446
commit 611798b786
4 changed files with 409 additions and 349 deletions

View File

@@ -113,7 +113,15 @@ async def lifespan(_application: FastAPI):
# Setup logging first with INFO level
logger = setup_logging(log_level="INFO")
# Track successful initialization steps
initialized = {
'database': False,
'services': False,
'background_loader': False
}
# Startup
startup_error = None
try:
logger.info("Starting FastAPI application...")
@@ -121,9 +129,11 @@ async def lifespan(_application: FastAPI):
try:
from src.server.database.connection import init_db
await init_db()
initialized['database'] = True
logger.info("Database initialized successfully")
except Exception as e:
logger.error("Failed to initialize database: %s", e, exc_info=True)
startup_error = e
raise # Database is required, fail startup if it fails
# Load configuration from config.json and sync with settings
@@ -216,6 +226,7 @@ async def lifespan(_application: FastAPI):
from src.server.utils.dependencies import get_download_service
download_service = get_download_service()
await download_service.initialize()
initialized['services'] = True
logger.info("Download service initialized and queue restored")
# Initialize background loader service
@@ -231,6 +242,7 @@ async def lifespan(_application: FastAPI):
series_app=series_app_instance
)
await background_loader.start()
initialized['background_loader'] = True
logger.info("Background loader service started")
# Run media scan only on first run
@@ -251,14 +263,34 @@ async def lifespan(_application: FastAPI):
)
except Exception as e:
logger.error("Error during startup: %s", e, exc_info=True)
raise # Re-raise to prevent app from starting in broken state
startup_error = e
# Don't re-raise here, let the finally/cleanup handle shutdown
# Yield control to the application
yield
# Yield control to the application (or immediately go to cleanup on error)
if startup_error is None:
try:
yield
except Exception as e:
logger.error("Error during application runtime: %s", e, exc_info=True)
else:
# Startup failed, but we still need to yield to satisfy the protocol
# The app won't actually run since we'll raise after cleanup
try:
yield
finally:
# After cleanup, re-raise the startup error
pass
# Shutdown - execute in proper order with timeout protection
logger.info("FastAPI application shutting down (graceful shutdown initiated)")
# Only cleanup what was successfully initialized
if not initialized['database']:
logger.info("Database was not initialized, skipping all cleanup")
if startup_error:
raise startup_error
return
# Define shutdown timeout (total time allowed for all shutdown operations)
SHUTDOWN_TIMEOUT = 30.0
@@ -270,20 +302,21 @@ async def lifespan(_application: FastAPI):
elapsed = time.monotonic() - shutdown_start
return max(0.0, SHUTDOWN_TIMEOUT - elapsed)
# 1. Stop background loader service
try:
from src.server.utils.dependencies import _background_loader_service
if _background_loader_service is not None:
logger.info("Stopping background loader service...")
await asyncio.wait_for(
_background_loader_service.stop(),
timeout=min(10.0, remaining_time())
)
logger.info("Background loader service stopped")
except asyncio.TimeoutError:
logger.warning("Background loader service shutdown timed out")
except Exception as e: # pylint: disable=broad-exception-caught
logger.error("Error stopping background loader service: %s", e, exc_info=True)
# 1. Stop background loader service (only if initialized)
if initialized['background_loader']:
try:
from src.server.utils.dependencies import _background_loader_service
if _background_loader_service is not None:
logger.info("Stopping background loader service...")
await asyncio.wait_for(
_background_loader_service.stop(),
timeout=min(10.0, remaining_time())
)
logger.info("Background loader service stopped")
except asyncio.TimeoutError:
logger.warning("Background loader service shutdown timed out")
except Exception as e: # pylint: disable=broad-exception-caught
logger.error("Error stopping background loader service: %s", e, exc_info=True)
# 2. Broadcast shutdown notification via WebSocket
try:
@@ -353,6 +386,10 @@ async def lifespan(_application: FastAPI):
"FastAPI application shutdown complete (took %.2fs)",
elapsed_total
)
# Re-raise startup error if it occurred
if startup_error:
raise startup_error
# Initialize FastAPI app with lifespan