Implement async series data loading with background processing
- Add loading status fields to AnimeSeries model
- Create BackgroundLoaderService for async task processing
- Update POST /api/anime/add to return 202 Accepted immediately
- Add GET /api/anime/{key}/loading-status endpoint
- Integrate background loader with startup/shutdown lifecycle
- Create database migration script for loading status fields
- Add unit tests for BackgroundLoaderService (10 tests, all passing)
- Update AnimeSeriesService.create() to accept loading status fields
Architecture follows clean separation with no code duplication:
- BackgroundLoader orchestrates, doesn't reimplement
- Reuses existing AnimeService, NFOService, WebSocket patterns
- Database-backed status survives restarts
This commit is contained in:
@@ -44,6 +44,66 @@ from src.server.services.websocket_service import get_websocket_service
|
||||
# module-level globals. This makes testing and multi-instance hosting safer.
|
||||
|
||||
|
||||
async def _check_incomplete_series_on_startup(background_loader) -> None:
|
||||
"""Check for incomplete series on startup and queue background loading.
|
||||
|
||||
Args:
|
||||
background_loader: BackgroundLoaderService instance
|
||||
"""
|
||||
logger = setup_logging(log_level="INFO")
|
||||
|
||||
try:
|
||||
from src.server.database.connection import get_db_session
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
|
||||
async for db in get_db_session():
|
||||
try:
|
||||
# Get all series from database
|
||||
series_list = await AnimeSeriesService.get_all(db)
|
||||
|
||||
incomplete_series = []
|
||||
|
||||
for series in series_list:
|
||||
# Check if series has incomplete loading
|
||||
if series.loading_status != "completed":
|
||||
incomplete_series.append(series)
|
||||
# Or check if specific data is missing
|
||||
elif (not series.episodes_loaded or
|
||||
not series.has_nfo or
|
||||
not series.logo_loaded or
|
||||
not series.images_loaded):
|
||||
incomplete_series.append(series)
|
||||
|
||||
if incomplete_series:
|
||||
logger.info(
|
||||
f"Found {len(incomplete_series)} series with missing data. "
|
||||
f"Queuing for background loading..."
|
||||
)
|
||||
|
||||
for series in incomplete_series:
|
||||
await background_loader.add_series_loading_task(
|
||||
key=series.key,
|
||||
folder=series.folder,
|
||||
name=series.name,
|
||||
year=series.year
|
||||
)
|
||||
logger.debug(
|
||||
f"Queued background loading for series: {series.key}"
|
||||
)
|
||||
|
||||
logger.info("All incomplete series queued for background loading")
|
||||
else:
|
||||
logger.info("All series data is complete. No background loading needed.")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking incomplete series: {e}", exc_info=True)
|
||||
|
||||
break # Exit after first iteration
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to check incomplete series on startup: {e}", exc_info=True)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_application: FastAPI):
|
||||
"""Manage application lifespan (startup and shutdown).
|
||||
@@ -156,6 +216,15 @@ async def lifespan(_application: FastAPI):
|
||||
download_service = get_download_service()
|
||||
await download_service.initialize()
|
||||
logger.info("Download service initialized and queue restored")
|
||||
|
||||
# Initialize background loader service
|
||||
from src.server.utils.dependencies import get_background_loader_service
|
||||
background_loader = get_background_loader_service()
|
||||
await background_loader.start()
|
||||
logger.info("Background loader service started")
|
||||
|
||||
# Check for incomplete series and queue background loading
|
||||
await _check_incomplete_series_on_startup(background_loader)
|
||||
else:
|
||||
logger.info(
|
||||
"Download service initialization skipped - "
|
||||
@@ -191,7 +260,22 @@ async def lifespan(_application: FastAPI):
|
||||
elapsed = time.monotonic() - shutdown_start
|
||||
return max(0.0, SHUTDOWN_TIMEOUT - elapsed)
|
||||
|
||||
# 1. Broadcast shutdown notification via WebSocket
|
||||
# 1. Stop background loader service
|
||||
try:
|
||||
from src.server.utils.dependencies import _background_loader_service
|
||||
if _background_loader_service is not None:
|
||||
logger.info("Stopping background loader service...")
|
||||
await asyncio.wait_for(
|
||||
_background_loader_service.stop(),
|
||||
timeout=min(10.0, remaining_time())
|
||||
)
|
||||
logger.info("Background loader service stopped")
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning("Background loader service shutdown timed out")
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error stopping background loader service: %s", e, exc_info=True)
|
||||
|
||||
# 2. Broadcast shutdown notification via WebSocket
|
||||
try:
|
||||
ws_service = get_websocket_service()
|
||||
logger.info("Broadcasting shutdown notification to WebSocket clients...")
|
||||
@@ -205,7 +289,7 @@ async def lifespan(_application: FastAPI):
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error during WebSocket shutdown: %s", e, exc_info=True)
|
||||
|
||||
# 2. Shutdown download service and persist active downloads
|
||||
# 3. Shutdown download service and persist active downloads
|
||||
try:
|
||||
from src.server.services.download_service import ( # noqa: E501
|
||||
_download_service_instance,
|
||||
@@ -218,7 +302,7 @@ async def lifespan(_application: FastAPI):
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error stopping download service: %s", e, exc_info=True)
|
||||
|
||||
# 3. Shutdown SeriesApp and cleanup thread pool
|
||||
# 4. Shutdown SeriesApp and cleanup thread pool
|
||||
try:
|
||||
from src.server.utils.dependencies import _series_app
|
||||
if _series_app is not None:
|
||||
@@ -228,7 +312,7 @@ async def lifespan(_application: FastAPI):
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error during SeriesApp shutdown: %s", e, exc_info=True)
|
||||
|
||||
# 4. Cleanup progress service
|
||||
# 5. Cleanup progress service
|
||||
try:
|
||||
progress_service = get_progress_service()
|
||||
logger.info("Cleaning up progress service...")
|
||||
|
||||
Reference in New Issue
Block a user