fix: handle lifespan errors gracefully
- Add error tracking in lifespan context manager - Only cleanup services that were successfully initialized - Properly handle startup errors without breaking async context - Fixes RuntimeError: generator didn't stop after athrow()
This commit is contained in:
@@ -113,7 +113,15 @@ async def lifespan(_application: FastAPI):
|
||||
# Setup logging first with INFO level
|
||||
logger = setup_logging(log_level="INFO")
|
||||
|
||||
# Track successful initialization steps
|
||||
initialized = {
|
||||
'database': False,
|
||||
'services': False,
|
||||
'background_loader': False
|
||||
}
|
||||
|
||||
# Startup
|
||||
startup_error = None
|
||||
try:
|
||||
logger.info("Starting FastAPI application...")
|
||||
|
||||
@@ -121,9 +129,11 @@ async def lifespan(_application: FastAPI):
|
||||
try:
|
||||
from src.server.database.connection import init_db
|
||||
await init_db()
|
||||
initialized['database'] = True
|
||||
logger.info("Database initialized successfully")
|
||||
except Exception as e:
|
||||
logger.error("Failed to initialize database: %s", e, exc_info=True)
|
||||
startup_error = e
|
||||
raise # Database is required, fail startup if it fails
|
||||
|
||||
# Load configuration from config.json and sync with settings
|
||||
@@ -216,6 +226,7 @@ async def lifespan(_application: FastAPI):
|
||||
from src.server.utils.dependencies import get_download_service
|
||||
download_service = get_download_service()
|
||||
await download_service.initialize()
|
||||
initialized['services'] = True
|
||||
logger.info("Download service initialized and queue restored")
|
||||
|
||||
# Initialize background loader service
|
||||
@@ -231,6 +242,7 @@ async def lifespan(_application: FastAPI):
|
||||
series_app=series_app_instance
|
||||
)
|
||||
await background_loader.start()
|
||||
initialized['background_loader'] = True
|
||||
logger.info("Background loader service started")
|
||||
|
||||
# Run media scan only on first run
|
||||
@@ -251,14 +263,34 @@ async def lifespan(_application: FastAPI):
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Error during startup: %s", e, exc_info=True)
|
||||
raise # Re-raise to prevent app from starting in broken state
|
||||
startup_error = e
|
||||
# Don't re-raise here, let the finally/cleanup handle shutdown
|
||||
|
||||
# Yield control to the application
|
||||
yield
|
||||
# Yield control to the application (or immediately go to cleanup on error)
|
||||
if startup_error is None:
|
||||
try:
|
||||
yield
|
||||
except Exception as e:
|
||||
logger.error("Error during application runtime: %s", e, exc_info=True)
|
||||
else:
|
||||
# Startup failed, but we still need to yield to satisfy the protocol
|
||||
# The app won't actually run since we'll raise after cleanup
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
# After cleanup, re-raise the startup error
|
||||
pass
|
||||
|
||||
# Shutdown - execute in proper order with timeout protection
|
||||
logger.info("FastAPI application shutting down (graceful shutdown initiated)")
|
||||
|
||||
# Only cleanup what was successfully initialized
|
||||
if not initialized['database']:
|
||||
logger.info("Database was not initialized, skipping all cleanup")
|
||||
if startup_error:
|
||||
raise startup_error
|
||||
return
|
||||
|
||||
# Define shutdown timeout (total time allowed for all shutdown operations)
|
||||
SHUTDOWN_TIMEOUT = 30.0
|
||||
|
||||
@@ -270,20 +302,21 @@ async def lifespan(_application: FastAPI):
|
||||
elapsed = time.monotonic() - shutdown_start
|
||||
return max(0.0, SHUTDOWN_TIMEOUT - elapsed)
|
||||
|
||||
# 1. Stop background loader service
|
||||
try:
|
||||
from src.server.utils.dependencies import _background_loader_service
|
||||
if _background_loader_service is not None:
|
||||
logger.info("Stopping background loader service...")
|
||||
await asyncio.wait_for(
|
||||
_background_loader_service.stop(),
|
||||
timeout=min(10.0, remaining_time())
|
||||
)
|
||||
logger.info("Background loader service stopped")
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning("Background loader service shutdown timed out")
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error stopping background loader service: %s", e, exc_info=True)
|
||||
# 1. Stop background loader service (only if initialized)
|
||||
if initialized['background_loader']:
|
||||
try:
|
||||
from src.server.utils.dependencies import _background_loader_service
|
||||
if _background_loader_service is not None:
|
||||
logger.info("Stopping background loader service...")
|
||||
await asyncio.wait_for(
|
||||
_background_loader_service.stop(),
|
||||
timeout=min(10.0, remaining_time())
|
||||
)
|
||||
logger.info("Background loader service stopped")
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning("Background loader service shutdown timed out")
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error stopping background loader service: %s", e, exc_info=True)
|
||||
|
||||
# 2. Broadcast shutdown notification via WebSocket
|
||||
try:
|
||||
@@ -353,6 +386,10 @@ async def lifespan(_application: FastAPI):
|
||||
"FastAPI application shutdown complete (took %.2fs)",
|
||||
elapsed_total
|
||||
)
|
||||
|
||||
# Re-raise startup error if it occurred
|
||||
if startup_error:
|
||||
raise startup_error
|
||||
|
||||
|
||||
# Initialize FastAPI app with lifespan
|
||||
|
||||
Reference in New Issue
Block a user