Migrate download queue from JSON to SQLite database

- Created QueueRepository adapter in src/server/services/queue_repository.py
- Refactored DownloadService to use repository pattern instead of JSON
- Updated application startup to initialize download service from database
- Updated all test fixtures to use MockQueueRepository
- All 1104 tests passing
This commit is contained in:
2025-12-02 16:01:25 +01:00
parent 48daeba012
commit b0f3b643c7
18 changed files with 1393 additions and 330 deletions

View File

@@ -0,0 +1,753 @@
"""Queue repository adapter for database-backed download queue operations.
This module provides a repository adapter that wraps the DownloadQueueService
and provides the interface needed by DownloadService for queue persistence.
The repository pattern abstracts the database operations from the business logic,
allowing the DownloadService to work with domain models (DownloadItem) while
the repository handles conversion to/from database models (DownloadQueueItem).
"""
from __future__ import annotations
import logging
from datetime import datetime, timezone
from typing import Callable, List, Optional
from sqlalchemy.ext.asyncio import AsyncSession
from src.server.database.models import AnimeSeries
from src.server.database.models import DownloadPriority as DBDownloadPriority
from src.server.database.models import DownloadQueueItem as DBDownloadQueueItem
from src.server.database.models import DownloadStatus as DBDownloadStatus
from src.server.database.service import AnimeSeriesService, DownloadQueueService
from src.server.models.download import (
DownloadItem,
DownloadPriority,
DownloadProgress,
DownloadStatus,
EpisodeIdentifier,
)
logger = logging.getLogger(__name__)
class QueueRepositoryError(Exception):
"""Repository-level exception for queue operations."""
class QueueRepository:
"""Repository adapter for database-backed download queue operations.
Provides clean interface for queue operations while handling
model conversion between Pydantic (DownloadItem) and SQLAlchemy
(DownloadQueueItem) models.
Attributes:
_db_session_factory: Factory function to create database sessions
"""
def __init__(
self,
db_session_factory: Callable[[], AsyncSession],
) -> None:
"""Initialize the queue repository.
Args:
db_session_factory: Factory function that returns AsyncSession instances
"""
self._db_session_factory = db_session_factory
logger.info("QueueRepository initialized")
# =========================================================================
# Model Conversion Methods
# =========================================================================
def _status_to_db(self, status: DownloadStatus) -> DBDownloadStatus:
"""Convert Pydantic DownloadStatus to SQLAlchemy DownloadStatus.
Args:
status: Pydantic status enum
Returns:
SQLAlchemy status enum
"""
return DBDownloadStatus(status.value)
def _status_from_db(self, status: DBDownloadStatus) -> DownloadStatus:
"""Convert SQLAlchemy DownloadStatus to Pydantic DownloadStatus.
Args:
status: SQLAlchemy status enum
Returns:
Pydantic status enum
"""
return DownloadStatus(status.value)
def _priority_to_db(self, priority: DownloadPriority) -> DBDownloadPriority:
"""Convert Pydantic DownloadPriority to SQLAlchemy DownloadPriority.
Args:
priority: Pydantic priority enum
Returns:
SQLAlchemy priority enum
"""
# Handle case differences (Pydantic uses uppercase, DB uses lowercase)
return DBDownloadPriority(priority.value.lower())
def _priority_from_db(self, priority: DBDownloadPriority) -> DownloadPriority:
"""Convert SQLAlchemy DownloadPriority to Pydantic DownloadPriority.
Args:
priority: SQLAlchemy priority enum
Returns:
Pydantic priority enum
"""
# Handle case differences (DB uses lowercase, Pydantic uses uppercase)
return DownloadPriority(priority.value.upper())
def _to_db_model(
self,
item: DownloadItem,
series_id: int,
) -> DBDownloadQueueItem:
"""Convert DownloadItem to database model.
Args:
item: Pydantic download item
series_id: Database series ID (foreign key)
Returns:
SQLAlchemy download queue item model
"""
return DBDownloadQueueItem(
series_id=series_id,
season=item.episode.season,
episode_number=item.episode.episode,
status=self._status_to_db(item.status),
priority=self._priority_to_db(item.priority),
progress_percent=item.progress.percent if item.progress else 0.0,
downloaded_bytes=int(
item.progress.downloaded_mb * 1024 * 1024
) if item.progress else 0,
total_bytes=int(
item.progress.total_mb * 1024 * 1024
) if item.progress and item.progress.total_mb else None,
download_speed=(
item.progress.speed_mbps * 1024 * 1024
) if item.progress and item.progress.speed_mbps else None,
error_message=item.error,
retry_count=item.retry_count,
download_url=str(item.source_url) if item.source_url else None,
started_at=item.started_at,
completed_at=item.completed_at,
)
def _from_db_model(
self,
db_item: DBDownloadQueueItem,
item_id: Optional[str] = None,
) -> DownloadItem:
"""Convert database model to DownloadItem.
Args:
db_item: SQLAlchemy download queue item
item_id: Optional override for item ID (uses db ID if not provided)
Returns:
Pydantic download item
"""
# Build progress object if there's progress data
progress = None
if db_item.progress_percent > 0 or db_item.downloaded_bytes > 0:
progress = DownloadProgress(
percent=db_item.progress_percent,
downloaded_mb=db_item.downloaded_bytes / (1024 * 1024),
total_mb=(
db_item.total_bytes / (1024 * 1024)
if db_item.total_bytes else None
),
speed_mbps=(
db_item.download_speed / (1024 * 1024)
if db_item.download_speed else None
),
)
return DownloadItem(
id=item_id or str(db_item.id),
serie_id=db_item.series.key if db_item.series else "",
serie_folder=db_item.series.folder if db_item.series else "",
serie_name=db_item.series.name if db_item.series else "",
episode=EpisodeIdentifier(
season=db_item.season,
episode=db_item.episode_number,
),
status=self._status_from_db(db_item.status),
priority=self._priority_from_db(db_item.priority),
added_at=db_item.created_at or datetime.now(timezone.utc),
started_at=db_item.started_at,
completed_at=db_item.completed_at,
progress=progress,
error=db_item.error_message,
retry_count=db_item.retry_count,
source_url=db_item.download_url,
)
# =========================================================================
# CRUD Operations
# =========================================================================
async def save_item(
self,
item: DownloadItem,
db: Optional[AsyncSession] = None,
) -> DownloadItem:
"""Save a download item to the database.
Creates a new record if the item doesn't exist in the database.
Args:
item: Download item to save
db: Optional existing database session
Returns:
Saved download item with database ID
Raises:
QueueRepositoryError: If save operation fails
"""
session = db or self._db_session_factory()
manage_session = db is None
try:
# Find series by key
series = await AnimeSeriesService.get_by_key(session, item.serie_id)
if not series:
# Create series if it doesn't exist
series = await AnimeSeriesService.create(
db=session,
key=item.serie_id,
name=item.serie_name,
site="", # Will be updated later if needed
folder=item.serie_folder,
)
logger.info(
"Created new series for queue item",
key=item.serie_id,
name=item.serie_name,
)
# Create queue item
db_item = await DownloadQueueService.create(
db=session,
series_id=series.id,
season=item.episode.season,
episode_number=item.episode.episode,
priority=self._priority_to_db(item.priority),
download_url=str(item.source_url) if item.source_url else None,
)
if manage_session:
await session.commit()
# Update the item ID with the database ID
item.id = str(db_item.id)
logger.debug(
"Saved queue item to database",
item_id=item.id,
serie_key=item.serie_id,
)
return item
except Exception as e:
if manage_session:
await session.rollback()
logger.error("Failed to save queue item", error=str(e))
raise QueueRepositoryError(f"Failed to save item: {str(e)}") from e
finally:
if manage_session:
await session.close()
async def get_item(
self,
item_id: str,
db: Optional[AsyncSession] = None,
) -> Optional[DownloadItem]:
"""Get a download item by ID.
Args:
item_id: Download item ID (database ID as string)
db: Optional existing database session
Returns:
Download item or None if not found
Raises:
QueueRepositoryError: If query fails
"""
session = db or self._db_session_factory()
manage_session = db is None
try:
db_item = await DownloadQueueService.get_by_id(
session, int(item_id)
)
if not db_item:
return None
return self._from_db_model(db_item, item_id)
except ValueError:
# Invalid ID format
return None
except Exception as e:
logger.error("Failed to get queue item", error=str(e))
raise QueueRepositoryError(f"Failed to get item: {str(e)}") from e
finally:
if manage_session:
await session.close()
async def get_pending_items(
self,
limit: Optional[int] = None,
db: Optional[AsyncSession] = None,
) -> List[DownloadItem]:
"""Get pending download items ordered by priority.
Args:
limit: Optional maximum number of items to return
db: Optional existing database session
Returns:
List of pending download items
Raises:
QueueRepositoryError: If query fails
"""
session = db or self._db_session_factory()
manage_session = db is None
try:
db_items = await DownloadQueueService.get_pending(session, limit)
return [self._from_db_model(item) for item in db_items]
except Exception as e:
logger.error("Failed to get pending items", error=str(e))
raise QueueRepositoryError(
f"Failed to get pending items: {str(e)}"
) from e
finally:
if manage_session:
await session.close()
async def get_active_item(
self,
db: Optional[AsyncSession] = None,
) -> Optional[DownloadItem]:
"""Get the currently active (downloading) item.
Args:
db: Optional existing database session
Returns:
Active download item or None if none active
Raises:
QueueRepositoryError: If query fails
"""
session = db or self._db_session_factory()
manage_session = db is None
try:
db_items = await DownloadQueueService.get_active(session)
if not db_items:
return None
# Return first active item (should only be one)
return self._from_db_model(db_items[0])
except Exception as e:
logger.error("Failed to get active item", error=str(e))
raise QueueRepositoryError(
f"Failed to get active item: {str(e)}"
) from e
finally:
if manage_session:
await session.close()
async def get_completed_items(
self,
limit: int = 100,
db: Optional[AsyncSession] = None,
) -> List[DownloadItem]:
"""Get completed download items.
Args:
limit: Maximum number of items to return
db: Optional existing database session
Returns:
List of completed download items
Raises:
QueueRepositoryError: If query fails
"""
session = db or self._db_session_factory()
manage_session = db is None
try:
db_items = await DownloadQueueService.get_by_status(
session, DBDownloadStatus.COMPLETED, limit
)
return [self._from_db_model(item) for item in db_items]
except Exception as e:
logger.error("Failed to get completed items", error=str(e))
raise QueueRepositoryError(
f"Failed to get completed items: {str(e)}"
) from e
finally:
if manage_session:
await session.close()
async def get_failed_items(
self,
limit: int = 50,
db: Optional[AsyncSession] = None,
) -> List[DownloadItem]:
"""Get failed download items.
Args:
limit: Maximum number of items to return
db: Optional existing database session
Returns:
List of failed download items
Raises:
QueueRepositoryError: If query fails
"""
session = db or self._db_session_factory()
manage_session = db is None
try:
db_items = await DownloadQueueService.get_by_status(
session, DBDownloadStatus.FAILED, limit
)
return [self._from_db_model(item) for item in db_items]
except Exception as e:
logger.error("Failed to get failed items", error=str(e))
raise QueueRepositoryError(
f"Failed to get failed items: {str(e)}"
) from e
finally:
if manage_session:
await session.close()
async def update_status(
self,
item_id: str,
status: DownloadStatus,
error: Optional[str] = None,
db: Optional[AsyncSession] = None,
) -> bool:
"""Update the status of a download item.
Args:
item_id: Download item ID
status: New download status
error: Optional error message for failed status
db: Optional existing database session
Returns:
True if update succeeded, False if item not found
Raises:
QueueRepositoryError: If update fails
"""
session = db or self._db_session_factory()
manage_session = db is None
try:
result = await DownloadQueueService.update_status(
session,
int(item_id),
self._status_to_db(status),
error,
)
if manage_session:
await session.commit()
success = result is not None
if success:
logger.debug(
"Updated queue item status",
item_id=item_id,
status=status.value,
)
return success
except ValueError:
return False
except Exception as e:
if manage_session:
await session.rollback()
logger.error("Failed to update status", error=str(e))
raise QueueRepositoryError(
f"Failed to update status: {str(e)}"
) from e
finally:
if manage_session:
await session.close()
async def update_progress(
self,
item_id: str,
progress: float,
downloaded: int,
total: Optional[int],
speed: Optional[float],
db: Optional[AsyncSession] = None,
) -> bool:
"""Update download progress for an item.
Args:
item_id: Download item ID
progress: Progress percentage (0-100)
downloaded: Downloaded bytes
total: Total bytes (optional)
speed: Download speed in bytes/second (optional)
db: Optional existing database session
Returns:
True if update succeeded, False if item not found
Raises:
QueueRepositoryError: If update fails
"""
session = db or self._db_session_factory()
manage_session = db is None
try:
result = await DownloadQueueService.update_progress(
session,
int(item_id),
progress,
downloaded,
total,
speed,
)
if manage_session:
await session.commit()
return result is not None
except ValueError:
return False
except Exception as e:
if manage_session:
await session.rollback()
logger.error("Failed to update progress", error=str(e))
raise QueueRepositoryError(
f"Failed to update progress: {str(e)}"
) from e
finally:
if manage_session:
await session.close()
async def delete_item(
self,
item_id: str,
db: Optional[AsyncSession] = None,
) -> bool:
"""Delete a download item from the database.
Args:
item_id: Download item ID
db: Optional existing database session
Returns:
True if item was deleted, False if not found
Raises:
QueueRepositoryError: If delete fails
"""
session = db or self._db_session_factory()
manage_session = db is None
try:
result = await DownloadQueueService.delete(session, int(item_id))
if manage_session:
await session.commit()
if result:
logger.debug("Deleted queue item", item_id=item_id)
return result
except ValueError:
return False
except Exception as e:
if manage_session:
await session.rollback()
logger.error("Failed to delete item", error=str(e))
raise QueueRepositoryError(
f"Failed to delete item: {str(e)}"
) from e
finally:
if manage_session:
await session.close()
async def clear_completed(
self,
db: Optional[AsyncSession] = None,
) -> int:
"""Clear all completed download items.
Args:
db: Optional existing database session
Returns:
Number of items cleared
Raises:
QueueRepositoryError: If operation fails
"""
session = db or self._db_session_factory()
manage_session = db is None
try:
count = await DownloadQueueService.clear_completed(session)
if manage_session:
await session.commit()
logger.info("Cleared completed items from queue", count=count)
return count
except Exception as e:
if manage_session:
await session.rollback()
logger.error("Failed to clear completed items", error=str(e))
raise QueueRepositoryError(
f"Failed to clear completed: {str(e)}"
) from e
finally:
if manage_session:
await session.close()
async def get_all_items(
self,
db: Optional[AsyncSession] = None,
) -> List[DownloadItem]:
"""Get all download items regardless of status.
Args:
db: Optional existing database session
Returns:
List of all download items
Raises:
QueueRepositoryError: If query fails
"""
session = db or self._db_session_factory()
manage_session = db is None
try:
db_items = await DownloadQueueService.get_all(
session, with_series=True
)
return [self._from_db_model(item) for item in db_items]
except Exception as e:
logger.error("Failed to get all items", error=str(e))
raise QueueRepositoryError(
f"Failed to get all items: {str(e)}"
) from e
finally:
if manage_session:
await session.close()
async def retry_failed_items(
self,
max_retries: int = 3,
db: Optional[AsyncSession] = None,
) -> List[DownloadItem]:
"""Retry failed downloads that haven't exceeded max retries.
Args:
max_retries: Maximum number of retry attempts
db: Optional existing database session
Returns:
List of items marked for retry
Raises:
QueueRepositoryError: If operation fails
"""
session = db or self._db_session_factory()
manage_session = db is None
try:
db_items = await DownloadQueueService.retry_failed(
session, max_retries
)
if manage_session:
await session.commit()
return [self._from_db_model(item) for item in db_items]
except Exception as e:
if manage_session:
await session.rollback()
logger.error("Failed to retry failed items", error=str(e))
raise QueueRepositoryError(
f"Failed to retry failed items: {str(e)}"
) from e
finally:
if manage_session:
await session.close()
# Singleton instance
_queue_repository_instance: Optional[QueueRepository] = None
def get_queue_repository(
db_session_factory: Optional[Callable[[], AsyncSession]] = None,
) -> QueueRepository:
"""Get or create the QueueRepository singleton.
Args:
db_session_factory: Optional factory function for database sessions.
If not provided, uses default from connection module.
Returns:
QueueRepository singleton instance
"""
global _queue_repository_instance
if _queue_repository_instance is None:
if db_session_factory is None:
# Use default session factory
from src.server.database.connection import get_async_session_factory
db_session_factory = get_async_session_factory
_queue_repository_instance = QueueRepository(db_session_factory)
return _queue_repository_instance