Implement async series data loading with background processing

- Add loading status fields to AnimeSeries model
- Create BackgroundLoaderService for async task processing
- Update POST /api/anime/add to return 202 Accepted immediately
- Add GET /api/anime/{key}/loading-status endpoint
- Integrate background loader with startup/shutdown lifecycle
- Create database migration script for loading status fields
- Add unit tests for BackgroundLoaderService (10 tests, all passing)
- Update AnimeSeriesService.create() to accept loading status fields

Architecture follows clean separation with no code duplication:
- BackgroundLoader orchestrates, doesn't reimplement
- Reuses existing AnimeService, NFOService, WebSocket patterns
- Database-backed status survives restarts
This commit is contained in:
2026-01-19 07:14:55 +01:00
parent df19f8ad95
commit f18c31a035
12 changed files with 3463 additions and 141 deletions

View File

@@ -44,6 +44,66 @@ from src.server.services.websocket_service import get_websocket_service
# module-level globals. This makes testing and multi-instance hosting safer.
async def _check_incomplete_series_on_startup(background_loader) -> None:
"""Check for incomplete series on startup and queue background loading.
Args:
background_loader: BackgroundLoaderService instance
"""
logger = setup_logging(log_level="INFO")
try:
from src.server.database.connection import get_db_session
from src.server.database.service import AnimeSeriesService
async for db in get_db_session():
try:
# Get all series from database
series_list = await AnimeSeriesService.get_all(db)
incomplete_series = []
for series in series_list:
# Check if series has incomplete loading
if series.loading_status != "completed":
incomplete_series.append(series)
# Or check if specific data is missing
elif (not series.episodes_loaded or
not series.has_nfo or
not series.logo_loaded or
not series.images_loaded):
incomplete_series.append(series)
if incomplete_series:
logger.info(
f"Found {len(incomplete_series)} series with missing data. "
f"Queuing for background loading..."
)
for series in incomplete_series:
await background_loader.add_series_loading_task(
key=series.key,
folder=series.folder,
name=series.name,
year=series.year
)
logger.debug(
f"Queued background loading for series: {series.key}"
)
logger.info("All incomplete series queued for background loading")
else:
logger.info("All series data is complete. No background loading needed.")
except Exception as e:
logger.error(f"Error checking incomplete series: {e}", exc_info=True)
break # Exit after first iteration
except Exception as e:
logger.error(f"Failed to check incomplete series on startup: {e}", exc_info=True)
@asynccontextmanager
async def lifespan(_application: FastAPI):
"""Manage application lifespan (startup and shutdown).
@@ -156,6 +216,15 @@ async def lifespan(_application: FastAPI):
download_service = get_download_service()
await download_service.initialize()
logger.info("Download service initialized and queue restored")
# Initialize background loader service
from src.server.utils.dependencies import get_background_loader_service
background_loader = get_background_loader_service()
await background_loader.start()
logger.info("Background loader service started")
# Check for incomplete series and queue background loading
await _check_incomplete_series_on_startup(background_loader)
else:
logger.info(
"Download service initialization skipped - "
@@ -191,7 +260,22 @@ async def lifespan(_application: FastAPI):
elapsed = time.monotonic() - shutdown_start
return max(0.0, SHUTDOWN_TIMEOUT - elapsed)
# 1. Broadcast shutdown notification via WebSocket
# 1. Stop background loader service
try:
from src.server.utils.dependencies import _background_loader_service
if _background_loader_service is not None:
logger.info("Stopping background loader service...")
await asyncio.wait_for(
_background_loader_service.stop(),
timeout=min(10.0, remaining_time())
)
logger.info("Background loader service stopped")
except asyncio.TimeoutError:
logger.warning("Background loader service shutdown timed out")
except Exception as e: # pylint: disable=broad-exception-caught
logger.error("Error stopping background loader service: %s", e, exc_info=True)
# 2. Broadcast shutdown notification via WebSocket
try:
ws_service = get_websocket_service()
logger.info("Broadcasting shutdown notification to WebSocket clients...")
@@ -205,7 +289,7 @@ async def lifespan(_application: FastAPI):
except Exception as e: # pylint: disable=broad-exception-caught
logger.error("Error during WebSocket shutdown: %s", e, exc_info=True)
# 2. Shutdown download service and persist active downloads
# 3. Shutdown download service and persist active downloads
try:
from src.server.services.download_service import ( # noqa: E501
_download_service_instance,
@@ -218,7 +302,7 @@ async def lifespan(_application: FastAPI):
except Exception as e: # pylint: disable=broad-exception-caught
logger.error("Error stopping download service: %s", e, exc_info=True)
# 3. Shutdown SeriesApp and cleanup thread pool
# 4. Shutdown SeriesApp and cleanup thread pool
try:
from src.server.utils.dependencies import _series_app
if _series_app is not None:
@@ -228,7 +312,7 @@ async def lifespan(_application: FastAPI):
except Exception as e: # pylint: disable=broad-exception-caught
logger.error("Error during SeriesApp shutdown: %s", e, exc_info=True)
# 4. Cleanup progress service
# 5. Cleanup progress service
try:
progress_service = get_progress_service()
logger.info("Cleaning up progress service...")