lukas.pupkalipinski/Aniworld
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement async series data loading with background processing

Browse Source 
- Add loading status fields to AnimeSeries model
- Create BackgroundLoaderService for async task processing
- Update POST /api/anime/add to return 202 Accepted immediately
- Add GET /api/anime/{key}/loading-status endpoint
- Integrate background loader with startup/shutdown lifecycle
- Create database migration script for loading status fields
- Add unit tests for BackgroundLoaderService (10 tests, all passing)
- Update AnimeSeriesService.create() to accept loading status fields

Architecture follows clean separation with no code duplication:
- BackgroundLoader orchestrates, doesn't reimplement
- Reuses existing AnimeService, NFOService, WebSocket patterns
- Database-backed status survives restarts
This commit is contained in:
Lukas
2026-01-19 07:14:55 +01:00
parent df19f8ad95
commit f18c31a035
12 changed files with 3463 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

221
src/server/api/anime.py
View File

@@ -1,5 +1,4 @@
import logging
import os
import warnings
from typing import Any, List, Optional
@@ -16,6 +15,10 @@ from src.server.exceptions import (
ValidationError,
)
from src.server.services.anime_service import AnimeService, AnimeServiceError
from src.server.services.background_loader_service import (
BackgroundLoaderService,
get_background_loader_service,
)
from src.server.utils.dependencies import (
get_anime_service,
get_optional_database_session,
@@ -688,23 +691,27 @@ async def _perform_search(
) from exc
@router.post("/add")
@router.post("/add", status_code=status.HTTP_202_ACCEPTED)
async def add_series(
request: AddSeriesRequest,
_auth: dict = Depends(require_auth),
series_app: Any = Depends(get_series_app),
db: Optional[AsyncSession] = Depends(get_optional_database_session),
anime_service: AnimeService = Depends(get_anime_service),
background_loader: BackgroundLoaderService = Depends(get_background_loader_service),
) -> dict:
"""Add a new series to the library with full initialization.
"""Add a new series to the library with asynchronous data loading.
This endpoint performs the complete series addition flow:
This endpoint performs immediate series addition and queues background loading:
1. Validates inputs and extracts the series key from the link URL
2. Creates a sanitized folder name from the display name
3. Saves the series to the database (if available)
3. Saves the series to the database with loading_status="pending"
4. Creates the folder on disk with the sanitized name
5. Triggers a targeted scan for missing episodes (only this series)
5. Queues background loading task for episodes, NFO, and images
6. Returns immediately (202 Accepted) without waiting for data loading
Data loading happens asynchronously in the background, with real-time
status updates via WebSocket.
The `key` is the URL-safe identifier used for all lookups.
The `name` is stored as display metadata and used to derive
the filesystem folder name (sanitized for filesystem safety).
@@ -716,7 +723,7 @@ async def add_series(
_auth: Ensures the caller is authenticated (value unused)
series_app: Core `SeriesApp` instance provided via dependency
db: Optional database session for async operations
anime_service: AnimeService for scanning operations
background_loader: BackgroundLoaderService for async data loading
Returns:
Dict[str, Any]: Status payload with:
@@ -725,8 +732,8 @@ async def add_series(
- key: Series unique identifier
- folder: Created folder path
- db_id: Database ID (if saved to DB)
- missing_episodes: Dict of missing episodes by season
- total_missing: Total count of missing episodes
- loading_status: Current loading status
- loading_progress: Dict of what data is being loaded
Raises:
HTTPException: If adding the series fails or link is invalid
@@ -792,8 +799,6 @@ async def add_series(
)
db_id = None
missing_episodes: dict = {}
scan_error: Optional[str] = None
# Step C: Save to database if available
if db is not None:
@@ -806,11 +811,16 @@ async def add_series(
"key": key,
"folder": existing.folder,
"db_id": existing.id,
"missing_episodes": {},
"total_missing": 0
"loading_status": existing.loading_status,
"loading_progress": {
"episodes": existing.episodes_loaded,
"nfo": existing.has_nfo,
"logo": existing.logo_loaded,
"images": existing.images_loaded
}
}
# Save to database using AnimeSeriesService
# Save to database using AnimeSeriesService with loading status
anime_series = await AnimeSeriesService.create(
db=db,
key=key,
@@ -818,11 +828,16 @@ async def add_series(
site="aniworld.to",
folder=folder,
year=year,
loading_status="pending",
episodes_loaded=False,
logo_loaded=False,
images_loaded=False,
loading_started_at=None,
)
db_id = anime_series.id
logger.info(
"Added series to database: %s (key=%s, db_id=%d, year=%s)",
"Added series to database: %s (key=%s, db_id=%d, year=%s, loading=pending)",
name,
key,
db_id,
@@ -851,80 +866,43 @@ async def add_series(
year
)
# Step E: Trigger targeted scan for missing episodes
# Step E: Queue background loading task for episodes, NFO, and images
try:
if series_app and hasattr(series_app, "serie_scanner"):
missing_episodes = series_app.serie_scanner.scan_single_series(
key=key,
folder=folder
)
logger.info(
"Targeted scan completed for %s: found %d missing episodes",
key,
sum(len(eps) for eps in missing_episodes.values())
)
# Update the serie in keyDict with the missing episodes
if hasattr(series_app, "list") and hasattr(series_app.list, "keyDict"):
if key in series_app.list.keyDict:
series_app.list.keyDict[key].episodeDict = missing_episodes
# Save missing episodes to database
if db is not None and missing_episodes:
from src.server.database.service import EpisodeService
for season, episode_numbers in missing_episodes.items():
for episode_number in episode_numbers:
await EpisodeService.create(
db=db,
series_id=db_id,
season=season,
episode_number=episode_number,
)
logger.info(
"Saved %d missing episodes to database for %s",
sum(len(eps) for eps in missing_episodes.values()),
key
)
else:
# Scanner not available - this shouldn't happen in normal operation
logger.warning(
"Scanner not available for targeted scan of %s",
key
)
await background_loader.add_series_loading_task(
key=key,
folder=folder,
name=name,
year=year
)
logger.info(
"Queued background loading for %s (key=%s)",
name,
key
)
except Exception as e:
# Scan failure is not critical - series was still added
scan_error = str(e)
# Background loading queue failure is not critical - series was still added
logger.warning(
"Targeted scan failed for %s: %s (series still added)",
"Failed to queue background loading for %s: %s",
key,
e
)
# Convert missing episodes keys to strings for JSON serialization
missing_episodes_serializable = {
str(season): episodes
for season, episodes in missing_episodes.items()
}
# Calculate total missing
total_missing = sum(len(eps) for eps in missing_episodes.values())
# Step F: Return response
# Step F: Return immediate response (202 Accepted)
response = {
"status": "success",
"message": f"Successfully added series: {name}",
"message": f"Series added successfully: {name}. Data will be loaded in background.",
"key": key,
"folder": folder,
"db_id": db_id,
"missing_episodes": missing_episodes_serializable,
"total_missing": total_missing
"loading_status": "pending",
"loading_progress": {
"episodes": False,
"nfo": False,
"logo": False,
"images": False
}
}
if scan_error:
response["scan_warning"] = f"Scan partially failed: {scan_error}"
return response
except HTTPException:
@@ -941,6 +919,97 @@ async def add_series(
) from exc
@router.get("/{anime_key}/loading-status")
async def get_loading_status(
anime_key: str,
_auth: dict = Depends(require_auth),
db: Optional[AsyncSession] = Depends(get_optional_database_session),
) -> dict:
"""Get current loading status for a series.
Returns the current background loading status including what data
has been loaded and what is still pending.
Args:
anime_key: Series unique identifier (key)
_auth: Ensures the caller is authenticated
db: Optional database session
Returns:
Dict with loading status information:
- key: Series identifier
- loading_status: Current status (pending, loading_*, completed, failed)
- progress: Dict of what data is loaded
- started_at: When loading started
- completed_at: When loading completed (if done)
- message: Human-readable status message
- error: Error message if failed
Raises:
HTTPException: If series not found or database unavailable
"""
if db is None:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="Database not available"
)
try:
from src.server.database.service import AnimeSeriesService
# Get series from database
series = await AnimeSeriesService.get_by_key(db, anime_key)
if not series:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=f"Series not found: {anime_key}"
)
# Build status message
message = ""
if series.loading_status == "pending":
message = "Queued for loading..."
elif series.loading_status == "loading_episodes":
message = "Loading episodes..."
elif series.loading_status == "loading_nfo":
message = "Generating NFO file..."
elif series.loading_status == "loading_logo":
message = "Downloading logo..."
elif series.loading_status == "loading_images":
message = "Downloading images..."
elif series.loading_status == "completed":
message = "All data loaded successfully"
elif series.loading_status == "failed":
message = f"Loading failed: {series.loading_error}"
else:
message = "Loading..."
return {
"key": series.key,
"loading_status": series.loading_status,
"progress": {
"episodes": series.episodes_loaded,
"nfo": series.has_nfo,
"logo": series.logo_loaded,
"images": series.images_loaded
},
"started_at": series.loading_started_at.isoformat() if series.loading_started_at else None,
"completed_at": series.loading_completed_at.isoformat() if series.loading_completed_at else None,
"message": message,
"error": series.loading_error
}
except HTTPException:
raise
except Exception as exc:
logger.error("Failed to get loading status: %s", exc, exc_info=True)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Failed to get loading status: {str(exc)}"
) from exc
@router.get("/{anime_id}", response_model=AnimeDetail)
async def get_anime(
anime_id: str,