lukas.pupkalipinski/Aniworld
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate download queue from JSON to SQLite database

Browse Source 
- Created QueueRepository adapter in src/server/services/queue_repository.py
- Refactored DownloadService to use repository pattern instead of JSON
- Updated application startup to initialize download service from database
- Updated all test fixtures to use MockQueueRepository
- All 1104 tests passing
This commit is contained in:
Lukas
2025-12-02 16:01:25 +01:00
parent 48daeba012
commit b0f3b643c7
18 changed files with 1393 additions and 330 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tests/integration/test_download_progress_integration.py
View File

@@ -72,11 +72,14 @@ async def anime_service(mock_series_app, progress_service):
@pytest.fixture
async def download_service(anime_service, progress_service):
"""Create a DownloadService."""
"""Create a DownloadService with mock queue repository."""
from tests.unit.test_download_service import MockQueueRepository
mock_repo = MockQueueRepository()
service = DownloadService(
anime_service=anime_service,
progress_service=progress_service,
persistence_path="/tmp/test_integration_progress_queue.json",
queue_repository=mock_repo,
)
yield service
await service.stop()

28
tests/integration/test_identifier_consistency.py
View File

@@ -88,9 +88,10 @@ def progress_service():
@pytest.fixture
async def download_service(mock_series_app, progress_service, tmp_path):
"""Create a DownloadService with dependencies."""
import uuid
persistence_path = tmp_path / f"test_queue_{uuid.uuid4()}.json"
"""Create a DownloadService with mock repository for testing."""
from tests.unit.test_download_service import MockQueueRepository
mock_repo = MockQueueRepository()
anime_service = AnimeService(
series_app=mock_series_app,
@@ -101,7 +102,7 @@ async def download_service(mock_series_app, progress_service, tmp_path):
service = DownloadService(
anime_service=anime_service,
progress_service=progress_service,
persistence_path=str(persistence_path),
queue_repository=mock_repo,
)
yield service
await service.stop()
@@ -319,8 +320,6 @@ class TestServiceIdentifierConsistency:
- Persisted data contains serie_id (key)
- Data can be restored with correct identifiers
"""
import json
# Add item to queue
await download_service.add_to_queue(
serie_id="jujutsu-kaisen",
@@ -330,18 +329,13 @@ class TestServiceIdentifierConsistency:
priority=DownloadPriority.NORMAL,
)
# Read persisted data
persistence_path = download_service._persistence_path
with open(persistence_path, "r") as f:
data = json.load(f)
# Verify item is in pending queue (in-memory cache synced with DB)
pending_items = list(download_service._pending_queue)
assert len(pending_items) == 1
# Verify persisted data structure
assert "pending" in data
assert len(data["pending"]) == 1
persisted_item = data["pending"][0]
assert persisted_item["serie_id"] == "jujutsu-kaisen"
assert persisted_item["serie_folder"] == "Jujutsu Kaisen (2020)"
persisted_item = pending_items[0]
assert persisted_item.serie_id == "jujutsu-kaisen"
assert persisted_item.serie_folder == "Jujutsu Kaisen (2020)"
class TestWebSocketIdentifierConsistency:

11
tests/integration/test_websocket_integration.py
View File

@@ -69,16 +69,17 @@ async def anime_service(mock_series_app, progress_service):
@pytest.fixture
async def download_service(anime_service, progress_service, tmp_path):
"""Create a DownloadService with dependencies.
"""Create a DownloadService with mock repository for testing.
Uses tmp_path to ensure each test has isolated queue storage.
Uses mock repository to ensure each test has isolated queue storage.
"""
import uuid
persistence_path = tmp_path / f"test_queue_{uuid.uuid4()}.json"
from tests.unit.test_download_service import MockQueueRepository
mock_repo = MockQueueRepository()
service = DownloadService(
anime_service=anime_service,
progress_service=progress_service,
persistence_path=str(persistence_path),
queue_repository=mock_repo,
)
yield service, progress_service
await service.stop()

33
tests/performance/test_download_stress.py
View File

@@ -28,12 +28,13 @@ class TestDownloadQueueStress:
@pytest.fixture
def download_service(self, mock_anime_service, tmp_path):
"""Create download service with mock."""
persistence_path = str(tmp_path / "test_queue.json")
"""Create download service with mock repository."""
from tests.unit.test_download_service import MockQueueRepository
mock_repo = MockQueueRepository()
service = DownloadService(
anime_service=mock_anime_service,
max_retries=3,
persistence_path=persistence_path,
queue_repository=mock_repo,
)
return service
@@ -176,12 +177,13 @@ class TestDownloadMemoryUsage:
@pytest.fixture
def download_service(self, mock_anime_service, tmp_path):
"""Create download service with mock."""
persistence_path = str(tmp_path / "test_queue.json")
"""Create download service with mock repository."""
from tests.unit.test_download_service import MockQueueRepository
mock_repo = MockQueueRepository()
service = DownloadService(
anime_service=mock_anime_service,
max_retries=3,
persistence_path=persistence_path,
queue_repository=mock_repo,
)
return service
@@ -232,12 +234,13 @@ class TestDownloadConcurrency:
@pytest.fixture
def download_service(self, mock_anime_service, tmp_path):
"""Create download service with mock."""
persistence_path = str(tmp_path / "test_queue.json")
"""Create download service with mock repository."""
from tests.unit.test_download_service import MockQueueRepository
mock_repo = MockQueueRepository()
service = DownloadService(
anime_service=mock_anime_service,
max_retries=3,
persistence_path=persistence_path,
queue_repository=mock_repo,
)
return service
@@ -321,11 +324,12 @@ class TestDownloadErrorHandling:
self, mock_failing_anime_service, tmp_path
):
"""Create download service with failing mock."""
persistence_path = str(tmp_path / "test_queue.json")
from tests.unit.test_download_service import MockQueueRepository
mock_repo = MockQueueRepository()
service = DownloadService(
anime_service=mock_failing_anime_service,
max_retries=3,
persistence_path=persistence_path,
queue_repository=mock_repo,
)
return service
@@ -338,12 +342,13 @@ class TestDownloadErrorHandling:
@pytest.fixture
def download_service(self, mock_anime_service, tmp_path):
"""Create download service with mock."""
persistence_path = str(tmp_path / "test_queue.json")
"""Create download service with mock repository."""
from tests.unit.test_download_service import MockQueueRepository
mock_repo = MockQueueRepository()
service = DownloadService(
anime_service=mock_anime_service,
max_retries=3,
persistence_path=persistence_path,
queue_repository=mock_repo,
)
return service

15
tests/unit/test_download_progress_websocket.py
View File

@@ -102,27 +102,20 @@ async def anime_service(mock_series_app, progress_service):
@pytest.fixture
async def download_service(anime_service, progress_service):
"""Create a DownloadService with dependencies."""
import os
persistence_path = "/tmp/test_download_progress_queue.json"
"""Create a DownloadService with mock repository for testing."""
from tests.unit.test_download_service import MockQueueRepository
# Remove any existing queue file
if os.path.exists(persistence_path):
os.remove(persistence_path)
mock_repo = MockQueueRepository()
service = DownloadService(
anime_service=anime_service,
progress_service=progress_service,
persistence_path=persistence_path,
queue_repository=mock_repo,
)
yield service, progress_service
await service.stop()
# Clean up after test
if os.path.exists(persistence_path):
os.remove(persistence_path)
class TestDownloadProgressWebSocket:

224
tests/unit/test_download_service.py
View File

@@ -1,14 +1,13 @@
"""Unit tests for the download queue service.
Tests cover queue management, manual download control, persistence,
Tests cover queue management, manual download control, database persistence,
and error scenarios for the simplified download service.
"""
from __future__ import annotations
import asyncio
import json
from datetime import datetime, timezone
from pathlib import Path
from typing import Dict, List, Optional
from unittest.mock import AsyncMock, MagicMock
import pytest
@@ -20,7 +19,125 @@ from src.server.models.download import (
EpisodeIdentifier,
)
from src.server.services.anime_service import AnimeService
from src.server.services.download_service import DownloadService, DownloadServiceError
from src.server.services.download_service import (
DownloadService,
DownloadServiceError,
)
class MockQueueRepository:
"""Mock implementation of QueueRepository for testing.
This provides an in-memory storage that mimics the database repository
behavior without requiring actual database connections.
"""
def __init__(self):
"""Initialize mock repository with in-memory storage."""
self._items: Dict[str, DownloadItem] = {}
async def save_item(self, item: DownloadItem) -> DownloadItem:
"""Save item to in-memory storage."""
self._items[item.id] = item
return item
async def get_item(self, item_id: str) -> Optional[DownloadItem]:
"""Get item by ID from in-memory storage."""
return self._items.get(item_id)
async def get_pending_items(self) -> List[DownloadItem]:
"""Get all pending items."""
return [
item for item in self._items.values()
if item.status == DownloadStatus.PENDING
]
async def get_active_item(self) -> Optional[DownloadItem]:
"""Get the currently active item."""
for item in self._items.values():
if item.status == DownloadStatus.DOWNLOADING:
return item
return None
async def get_completed_items(
self, limit: int = 100
) -> List[DownloadItem]:
"""Get completed items."""
completed = [
item for item in self._items.values()
if item.status == DownloadStatus.COMPLETED
]
return completed[:limit]
async def get_failed_items(self, limit: int = 50) -> List[DownloadItem]:
"""Get failed items."""
failed = [
item for item in self._items.values()
if item.status == DownloadStatus.FAILED
]
return failed[:limit]
async def update_status(
self,
item_id: str,
status: DownloadStatus,
error: Optional[str] = None
) -> bool:
"""Update item status."""
if item_id not in self._items:
return False
self._items[item_id].status = status
if error:
self._items[item_id].error = error
if status == DownloadStatus.COMPLETED:
self._items[item_id].completed_at = datetime.now(timezone.utc)
elif status == DownloadStatus.DOWNLOADING:
self._items[item_id].started_at = datetime.now(timezone.utc)
return True
async def update_progress(
self,
item_id: str,
progress: float,
downloaded: int,
total: int,
speed: float
) -> bool:
"""Update download progress."""
if item_id not in self._items:
return False
item = self._items[item_id]
if item.progress is None:
from src.server.models.download import DownloadProgress
item.progress = DownloadProgress(
percent=progress,
downloaded_bytes=downloaded,
total_bytes=total,
speed_bps=speed
)
else:
item.progress.percent = progress
item.progress.downloaded_bytes = downloaded
item.progress.total_bytes = total
item.progress.speed_bps = speed
return True
async def delete_item(self, item_id: str) -> bool:
"""Delete item from storage."""
if item_id in self._items:
del self._items[item_id]
return True
return False
async def clear_completed(self) -> int:
"""Clear all completed items."""
completed_ids = [
item_id for item_id, item in self._items.items()
if item.status == DownloadStatus.COMPLETED
]
for item_id in completed_ids:
del self._items[item_id]
return len(completed_ids)
@pytest.fixture
@@ -32,18 +149,18 @@ def mock_anime_service():
@pytest.fixture
def temp_persistence_path(tmp_path):
"""Create a temporary persistence path."""
return str(tmp_path / "test_queue.json")
def mock_queue_repository():
"""Create a mock QueueRepository for testing."""
return MockQueueRepository()
@pytest.fixture
def download_service(mock_anime_service, temp_persistence_path):
def download_service(mock_anime_service, mock_queue_repository):
"""Create a DownloadService instance for testing."""
return DownloadService(
anime_service=mock_anime_service,
queue_repository=mock_queue_repository,
max_retries=3,
persistence_path=temp_persistence_path,
)
@@ -51,12 +168,12 @@ class TestDownloadServiceInitialization:
"""Test download service initialization."""
def test_initialization_creates_queues(
self, mock_anime_service, temp_persistence_path
self, mock_anime_service, mock_queue_repository
):
"""Test that initialization creates empty queues."""
service = DownloadService(
anime_service=mock_anime_service,
persistence_path=temp_persistence_path,
queue_repository=mock_queue_repository,
)
assert len(service._pending_queue) == 0
@@ -65,45 +182,30 @@ class TestDownloadServiceInitialization:
assert len(service._failed_items) == 0
assert service._is_stopped is True
def test_initialization_loads_persisted_queue(
self, mock_anime_service, temp_persistence_path
@pytest.mark.asyncio
async def test_initialization_loads_persisted_queue(
self, mock_anime_service, mock_queue_repository
):
"""Test that initialization loads persisted queue state."""
# Create a persisted queue file
persistence_file = Path(temp_persistence_path)
persistence_file.parent.mkdir(parents=True, exist_ok=True)
test_data = {
"pending": [
{
"id": "test-id-1",
"serie_id": "series-1",
"serie_folder": "test-series", # Added missing field
"serie_name": "Test Series",
"episode": {"season": 1, "episode": 1, "title": None},
"status": "pending",
"priority": "NORMAL", # Must be uppercase
"added_at": datetime.now(timezone.utc).isoformat(),
"started_at": None,
"completed_at": None,
"progress": None,
"error": None,
"retry_count": 0,
"source_url": None,
}
],
"active": [],
"failed": [],
"timestamp": datetime.now(timezone.utc).isoformat(),
}
with open(persistence_file, "w", encoding="utf-8") as f:
json.dump(test_data, f)
"""Test that initialization loads persisted queue from database."""
# Pre-populate the mock repository with a pending item
test_item = DownloadItem(
id="test-id-1",
serie_id="series-1",
serie_folder="test-series",
serie_name="Test Series",
episode=EpisodeIdentifier(season=1, episode=1),
status=DownloadStatus.PENDING,
priority=DownloadPriority.NORMAL,
added_at=datetime.now(timezone.utc),
)
await mock_queue_repository.save_item(test_item)
# Create service and initialize from database
service = DownloadService(
anime_service=mock_anime_service,
persistence_path=temp_persistence_path,
queue_repository=mock_queue_repository,
)
await service.initialize()
assert len(service._pending_queue) == 1
assert service._pending_queue[0].id == "test-id-1"
@@ -391,11 +493,13 @@ class TestQueueControl:
class TestPersistence:
"""Test queue persistence functionality."""
"""Test queue persistence functionality with database backend."""
@pytest.mark.asyncio
async def test_queue_persistence(self, download_service):
"""Test that queue state is persisted to disk."""
async def test_queue_persistence(
self, download_service, mock_queue_repository
):
"""Test that queue state is persisted to database."""
await download_service.add_to_queue(
serie_id="series-1",
serie_folder="series",
@@ -403,26 +507,20 @@ class TestPersistence:
episodes=[EpisodeIdentifier(season=1, episode=1)],
)
# Persistence file should exist
persistence_path = Path(download_service._persistence_path)
assert persistence_path.exists()
# Check file contents
with open(persistence_path, "r") as f:
data = json.load(f)
assert len(data["pending"]) == 1
assert data["pending"][0]["serie_id"] == "series-1"
# Item should be saved in mock repository
pending_items = await mock_queue_repository.get_pending_items()
assert len(pending_items) == 1
assert pending_items[0].serie_id == "series-1"
@pytest.mark.asyncio
async def test_queue_recovery_after_restart(
self, mock_anime_service, temp_persistence_path
self, mock_anime_service, mock_queue_repository
):
"""Test that queue is recovered after service restart."""
# Create and populate first service
service1 = DownloadService(
anime_service=mock_anime_service,
persistence_path=temp_persistence_path,
queue_repository=mock_queue_repository,
)
await service1.add_to_queue(
@@ -435,11 +533,13 @@ class TestPersistence:
],
)
# Create new service with same persistence path
# Create new service with same repository (simulating restart)
service2 = DownloadService(
anime_service=mock_anime_service,
persistence_path=temp_persistence_path,
queue_repository=mock_queue_repository,
)
# Initialize to load from database to recover state
await service2.initialize()
# Should recover pending items
assert len(service2._pending_queue) == 2