Lukas
17754a86f0
- Create DataMigrationService for migrating data files to SQLite - Add sync database methods to AnimeSeriesService - Update SerieScanner to save to database with file fallback - Update anime API endpoints to use database with fallback - Add delete endpoint for anime series - Add automatic migration on startup in fastapi_app.py lifespan - Add 28 unit tests for migration service - Add 14 integration tests for migration flow - Update infrastructure.md and database README docs Migration runs automatically on startup, legacy data files preserved.
720 lines
22 KiB
Python
720 lines
22 KiB
Python
"""Unit tests for data migration service.
|
|
|
|
Tests cover:
|
|
- Detection of legacy data files
|
|
- Migration of data files to database
|
|
- Error handling for corrupted files
|
|
- Backup functionality
|
|
- Migration status reporting
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import tempfile
|
|
|
|
import pytest
|
|
from sqlalchemy.ext.asyncio import create_async_engine
|
|
from sqlalchemy.pool import StaticPool
|
|
|
|
from src.server.database.base import Base
|
|
from src.server.database.service import AnimeSeriesService
|
|
from src.server.services.data_migration_service import (
|
|
DataMigrationService,
|
|
MigrationResult,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
async def test_engine():
|
|
"""Create in-memory SQLite engine for testing."""
|
|
engine = create_async_engine(
|
|
"sqlite+aiosqlite:///:memory:",
|
|
echo=False,
|
|
poolclass=StaticPool,
|
|
)
|
|
async with engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
yield engine
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.fixture
|
|
async def test_session(test_engine):
|
|
"""Create async session for testing."""
|
|
from sqlalchemy.ext.asyncio import async_sessionmaker
|
|
|
|
async_session = async_sessionmaker(
|
|
test_engine,
|
|
expire_on_commit=False,
|
|
)
|
|
async with async_session() as session:
|
|
yield session
|
|
|
|
|
|
@pytest.fixture
|
|
def temp_anime_dir():
|
|
"""Create temporary directory for testing anime folders."""
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
yield tmp_dir
|
|
|
|
|
|
@pytest.fixture
|
|
def migration_service():
|
|
"""Create DataMigrationService instance."""
|
|
return DataMigrationService()
|
|
|
|
|
|
def create_test_data_file(
|
|
base_dir: str,
|
|
folder_name: str,
|
|
key: str,
|
|
name: str,
|
|
) -> str:
|
|
"""Create a test data file in the specified folder.
|
|
|
|
Args:
|
|
base_dir: Base anime directory
|
|
folder_name: Folder name to create
|
|
key: Series key
|
|
name: Series name
|
|
|
|
Returns:
|
|
Path to the created data file
|
|
"""
|
|
folder_path = os.path.join(base_dir, folder_name)
|
|
os.makedirs(folder_path, exist_ok=True)
|
|
|
|
data_path = os.path.join(folder_path, "data")
|
|
data = {
|
|
"key": key,
|
|
"name": name,
|
|
"site": "aniworld.to",
|
|
"folder": folder_name,
|
|
"episodeDict": {"1": [1, 2, 3], "2": [1, 2]},
|
|
}
|
|
|
|
with open(data_path, "w", encoding="utf-8") as f:
|
|
json.dump(data, f, indent=4)
|
|
|
|
return data_path
|
|
|
|
|
|
# =============================================================================
|
|
# MigrationResult Tests
|
|
# =============================================================================
|
|
|
|
|
|
class TestMigrationResult:
|
|
"""Test cases for MigrationResult dataclass."""
|
|
|
|
def test_migration_result_defaults(self):
|
|
"""Test MigrationResult default values."""
|
|
result = MigrationResult()
|
|
|
|
assert result.total_found == 0
|
|
assert result.migrated == 0
|
|
assert result.failed == 0
|
|
assert result.skipped == 0
|
|
assert result.errors == []
|
|
|
|
def test_migration_result_with_values(self):
|
|
"""Test MigrationResult with custom values."""
|
|
result = MigrationResult(
|
|
total_found=10,
|
|
migrated=7,
|
|
failed=1,
|
|
skipped=2,
|
|
errors=["Error 1"],
|
|
)
|
|
|
|
assert result.total_found == 10
|
|
assert result.migrated == 7
|
|
assert result.failed == 1
|
|
assert result.skipped == 2
|
|
assert result.errors == ["Error 1"]
|
|
|
|
def test_migration_result_str(self):
|
|
"""Test MigrationResult string representation."""
|
|
result = MigrationResult(
|
|
total_found=10,
|
|
migrated=7,
|
|
failed=1,
|
|
skipped=2,
|
|
)
|
|
|
|
result_str = str(result)
|
|
assert "7 migrated" in result_str
|
|
assert "2 skipped" in result_str
|
|
assert "1 failed" in result_str
|
|
assert "10" in result_str
|
|
|
|
|
|
# =============================================================================
|
|
# Check for Legacy Data Files Tests
|
|
# =============================================================================
|
|
|
|
|
|
class TestCheckForLegacyDataFiles:
|
|
"""Test cases for check_for_legacy_data_files method."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_check_empty_directory(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
temp_anime_dir: str,
|
|
):
|
|
"""Test scanning empty directory returns empty list."""
|
|
files = await migration_service.check_for_legacy_data_files(
|
|
temp_anime_dir
|
|
)
|
|
assert files == []
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_check_nonexistent_directory(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
):
|
|
"""Test scanning nonexistent directory returns empty list."""
|
|
files = await migration_service.check_for_legacy_data_files(
|
|
"/nonexistent/path"
|
|
)
|
|
assert files == []
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_check_none_directory(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
):
|
|
"""Test scanning None directory returns empty list."""
|
|
files = await migration_service.check_for_legacy_data_files(None)
|
|
assert files == []
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_check_empty_string_directory(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
):
|
|
"""Test scanning empty string directory returns empty list."""
|
|
files = await migration_service.check_for_legacy_data_files("")
|
|
assert files == []
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_find_single_data_file(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
temp_anime_dir: str,
|
|
):
|
|
"""Test finding a single data file."""
|
|
create_test_data_file(
|
|
temp_anime_dir,
|
|
"Test Anime",
|
|
"test-anime",
|
|
"Test Anime",
|
|
)
|
|
|
|
files = await migration_service.check_for_legacy_data_files(
|
|
temp_anime_dir
|
|
)
|
|
|
|
assert len(files) == 1
|
|
assert files[0].endswith("data")
|
|
assert "Test Anime" in files[0]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_find_multiple_data_files(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
temp_anime_dir: str,
|
|
):
|
|
"""Test finding multiple data files."""
|
|
create_test_data_file(
|
|
temp_anime_dir, "Anime 1", "anime-1", "Anime 1"
|
|
)
|
|
create_test_data_file(
|
|
temp_anime_dir, "Anime 2", "anime-2", "Anime 2"
|
|
)
|
|
create_test_data_file(
|
|
temp_anime_dir, "Anime 3", "anime-3", "Anime 3"
|
|
)
|
|
|
|
files = await migration_service.check_for_legacy_data_files(
|
|
temp_anime_dir
|
|
)
|
|
|
|
assert len(files) == 3
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_skip_folders_without_data_file(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
temp_anime_dir: str,
|
|
):
|
|
"""Test that folders without data files are skipped."""
|
|
# Create folder with data file
|
|
create_test_data_file(
|
|
temp_anime_dir, "With Data", "with-data", "With Data"
|
|
)
|
|
|
|
# Create folder without data file
|
|
empty_folder = os.path.join(temp_anime_dir, "Without Data")
|
|
os.makedirs(empty_folder, exist_ok=True)
|
|
|
|
files = await migration_service.check_for_legacy_data_files(
|
|
temp_anime_dir
|
|
)
|
|
|
|
assert len(files) == 1
|
|
assert "With Data" in files[0]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_skip_non_directories(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
temp_anime_dir: str,
|
|
):
|
|
"""Test that non-directory entries are skipped."""
|
|
create_test_data_file(
|
|
temp_anime_dir, "Anime", "anime", "Anime"
|
|
)
|
|
|
|
# Create a file (not directory) in anime dir
|
|
file_path = os.path.join(temp_anime_dir, "some_file.txt")
|
|
with open(file_path, "w") as f:
|
|
f.write("test")
|
|
|
|
files = await migration_service.check_for_legacy_data_files(
|
|
temp_anime_dir
|
|
)
|
|
|
|
assert len(files) == 1
|
|
|
|
|
|
# =============================================================================
|
|
# Migrate Data File to DB Tests
|
|
# =============================================================================
|
|
|
|
|
|
class TestMigrateDataFileToDb:
|
|
"""Test cases for migrate_data_file_to_db method."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_migrate_valid_data_file(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
temp_anime_dir: str,
|
|
test_session,
|
|
):
|
|
"""Test migrating a valid data file creates database entry."""
|
|
data_path = create_test_data_file(
|
|
temp_anime_dir,
|
|
"Test Anime",
|
|
"test-anime",
|
|
"Test Anime",
|
|
)
|
|
|
|
result = await migration_service.migrate_data_file_to_db(
|
|
data_path, test_session
|
|
)
|
|
|
|
assert result is True
|
|
|
|
# Verify database entry
|
|
series = await AnimeSeriesService.get_by_key(
|
|
test_session, "test-anime"
|
|
)
|
|
assert series is not None
|
|
assert series.name == "Test Anime"
|
|
assert series.site == "aniworld.to"
|
|
assert series.folder == "Test Anime"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_migrate_nonexistent_file_raises_error(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
test_session,
|
|
):
|
|
"""Test migrating nonexistent file raises FileNotFoundError."""
|
|
with pytest.raises(FileNotFoundError):
|
|
await migration_service.migrate_data_file_to_db(
|
|
"/nonexistent/path/data",
|
|
test_session,
|
|
)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_migrate_invalid_json_raises_error(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
temp_anime_dir: str,
|
|
test_session,
|
|
):
|
|
"""Test migrating invalid JSON file raises ValueError."""
|
|
# Create folder with invalid data file
|
|
folder_path = os.path.join(temp_anime_dir, "Invalid")
|
|
os.makedirs(folder_path, exist_ok=True)
|
|
|
|
data_path = os.path.join(folder_path, "data")
|
|
with open(data_path, "w") as f:
|
|
f.write("not valid json")
|
|
|
|
with pytest.raises(ValueError):
|
|
await migration_service.migrate_data_file_to_db(
|
|
data_path, test_session
|
|
)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_migrate_skips_existing_series(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
temp_anime_dir: str,
|
|
test_session,
|
|
):
|
|
"""Test migration returns False for existing series in DB."""
|
|
# Create series in database first
|
|
await AnimeSeriesService.create(
|
|
test_session,
|
|
key="existing-anime",
|
|
name="Existing Anime",
|
|
site="aniworld.to",
|
|
folder="Existing Anime",
|
|
)
|
|
await test_session.commit()
|
|
|
|
# Create data file with same key
|
|
data_path = create_test_data_file(
|
|
temp_anime_dir,
|
|
"Existing Anime",
|
|
"existing-anime",
|
|
"Existing Anime",
|
|
)
|
|
|
|
result = await migration_service.migrate_data_file_to_db(
|
|
data_path, test_session
|
|
)
|
|
|
|
assert result is False
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_migrate_preserves_episode_dict(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
temp_anime_dir: str,
|
|
test_session,
|
|
):
|
|
"""Test migration preserves episode dictionary correctly."""
|
|
data_path = create_test_data_file(
|
|
temp_anime_dir,
|
|
"With Episodes",
|
|
"with-episodes",
|
|
"With Episodes",
|
|
)
|
|
|
|
await migration_service.migrate_data_file_to_db(
|
|
data_path, test_session
|
|
)
|
|
|
|
series = await AnimeSeriesService.get_by_key(
|
|
test_session, "with-episodes"
|
|
)
|
|
assert series is not None
|
|
assert series.episode_dict is not None
|
|
# Note: JSON keys become strings, so check string keys
|
|
assert "1" in series.episode_dict or 1 in series.episode_dict
|
|
|
|
|
|
# =============================================================================
|
|
# Migrate All Legacy Data Tests
|
|
# =============================================================================
|
|
|
|
|
|
class TestMigrateAllLegacyData:
|
|
"""Test cases for migrate_all_legacy_data method."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_migrate_empty_directory(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
temp_anime_dir: str,
|
|
test_session,
|
|
):
|
|
"""Test migrating empty directory returns zero counts."""
|
|
result = await migration_service.migrate_all_legacy_data(
|
|
temp_anime_dir, test_session
|
|
)
|
|
|
|
assert result.total_found == 0
|
|
assert result.migrated == 0
|
|
assert result.failed == 0
|
|
assert result.skipped == 0
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_migrate_multiple_files(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
temp_anime_dir: str,
|
|
test_session,
|
|
):
|
|
"""Test migrating multiple files successfully."""
|
|
create_test_data_file(
|
|
temp_anime_dir, "Anime 1", "anime-1", "Anime 1"
|
|
)
|
|
create_test_data_file(
|
|
temp_anime_dir, "Anime 2", "anime-2", "Anime 2"
|
|
)
|
|
|
|
result = await migration_service.migrate_all_legacy_data(
|
|
temp_anime_dir, test_session
|
|
)
|
|
|
|
assert result.total_found == 2
|
|
assert result.migrated == 2
|
|
assert result.failed == 0
|
|
assert result.skipped == 0
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_migrate_with_existing_entries(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
temp_anime_dir: str,
|
|
test_session,
|
|
):
|
|
"""Test migration skips existing database entries."""
|
|
# Create one in DB already
|
|
await AnimeSeriesService.create(
|
|
test_session,
|
|
key="anime-1",
|
|
name="Anime 1",
|
|
site="aniworld.to",
|
|
folder="Anime 1",
|
|
)
|
|
await test_session.commit()
|
|
|
|
# Create data files
|
|
create_test_data_file(
|
|
temp_anime_dir, "Anime 1", "anime-1", "Anime 1"
|
|
)
|
|
create_test_data_file(
|
|
temp_anime_dir, "Anime 2", "anime-2", "Anime 2"
|
|
)
|
|
|
|
result = await migration_service.migrate_all_legacy_data(
|
|
temp_anime_dir, test_session
|
|
)
|
|
|
|
assert result.total_found == 2
|
|
assert result.migrated == 1
|
|
assert result.skipped == 1
|
|
assert result.failed == 0
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_migrate_with_invalid_file(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
temp_anime_dir: str,
|
|
test_session,
|
|
):
|
|
"""Test migration continues after encountering invalid file."""
|
|
# Create valid data file
|
|
create_test_data_file(
|
|
temp_anime_dir, "Valid Anime", "valid-anime", "Valid Anime"
|
|
)
|
|
|
|
# Create invalid data file
|
|
invalid_folder = os.path.join(temp_anime_dir, "Invalid")
|
|
os.makedirs(invalid_folder, exist_ok=True)
|
|
invalid_path = os.path.join(invalid_folder, "data")
|
|
with open(invalid_path, "w") as f:
|
|
f.write("invalid json")
|
|
|
|
result = await migration_service.migrate_all_legacy_data(
|
|
temp_anime_dir, test_session
|
|
)
|
|
|
|
assert result.total_found == 2
|
|
assert result.migrated == 1
|
|
assert result.failed == 1
|
|
assert len(result.errors) == 1
|
|
|
|
|
|
# =============================================================================
|
|
# Cleanup Migrated Files Tests
|
|
# =============================================================================
|
|
|
|
|
|
class TestCleanupMigratedFiles:
|
|
"""Test cases for cleanup_migrated_files method."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cleanup_empty_list(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
):
|
|
"""Test cleanup with empty list does nothing."""
|
|
# Should not raise
|
|
await migration_service.cleanup_migrated_files([])
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cleanup_creates_backup(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
temp_anime_dir: str,
|
|
):
|
|
"""Test cleanup creates backup before removal."""
|
|
data_path = create_test_data_file(
|
|
temp_anime_dir, "Test", "test", "Test"
|
|
)
|
|
|
|
await migration_service.cleanup_migrated_files(
|
|
[data_path], backup=True
|
|
)
|
|
|
|
# Original file should be gone
|
|
assert not os.path.exists(data_path)
|
|
|
|
# Backup should exist
|
|
folder_path = os.path.dirname(data_path)
|
|
backup_files = [
|
|
f for f in os.listdir(folder_path) if f.startswith("data.backup")
|
|
]
|
|
assert len(backup_files) == 1
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cleanup_without_backup(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
temp_anime_dir: str,
|
|
):
|
|
"""Test cleanup without backup just removes file."""
|
|
data_path = create_test_data_file(
|
|
temp_anime_dir, "Test", "test", "Test"
|
|
)
|
|
|
|
await migration_service.cleanup_migrated_files(
|
|
[data_path], backup=False
|
|
)
|
|
|
|
# Original file should be gone
|
|
assert not os.path.exists(data_path)
|
|
|
|
# No backup should exist
|
|
folder_path = os.path.dirname(data_path)
|
|
backup_files = [
|
|
f for f in os.listdir(folder_path) if f.startswith("data.backup")
|
|
]
|
|
assert len(backup_files) == 0
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cleanup_handles_missing_file(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
):
|
|
"""Test cleanup handles already-deleted files gracefully."""
|
|
# Should not raise
|
|
await migration_service.cleanup_migrated_files(
|
|
["/nonexistent/path/data"]
|
|
)
|
|
|
|
|
|
# =============================================================================
|
|
# Migration Status Tests
|
|
# =============================================================================
|
|
|
|
|
|
class TestGetMigrationStatus:
|
|
"""Test cases for get_migration_status method."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_status_empty_directory_empty_db(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
temp_anime_dir: str,
|
|
test_session,
|
|
):
|
|
"""Test status with empty directory and database."""
|
|
status = await migration_service.get_migration_status(
|
|
temp_anime_dir, test_session
|
|
)
|
|
|
|
assert status["legacy_files_count"] == 0
|
|
assert status["database_entries_count"] == 0
|
|
assert status["only_in_files"] == []
|
|
assert status["only_in_database"] == []
|
|
assert status["in_both"] == []
|
|
assert status["migration_complete"] is True
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_status_with_pending_migrations(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
temp_anime_dir: str,
|
|
test_session,
|
|
):
|
|
"""Test status when files need migration."""
|
|
create_test_data_file(
|
|
temp_anime_dir, "Anime 1", "anime-1", "Anime 1"
|
|
)
|
|
create_test_data_file(
|
|
temp_anime_dir, "Anime 2", "anime-2", "Anime 2"
|
|
)
|
|
|
|
status = await migration_service.get_migration_status(
|
|
temp_anime_dir, test_session
|
|
)
|
|
|
|
assert status["legacy_files_count"] == 2
|
|
assert status["database_entries_count"] == 0
|
|
assert len(status["only_in_files"]) == 2
|
|
assert status["migration_complete"] is False
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_status_after_complete_migration(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
temp_anime_dir: str,
|
|
test_session,
|
|
):
|
|
"""Test status after all files migrated."""
|
|
create_test_data_file(
|
|
temp_anime_dir, "Anime 1", "anime-1", "Anime 1"
|
|
)
|
|
|
|
# Migrate
|
|
await migration_service.migrate_all_legacy_data(
|
|
temp_anime_dir, test_session
|
|
)
|
|
|
|
status = await migration_service.get_migration_status(
|
|
temp_anime_dir, test_session
|
|
)
|
|
|
|
assert status["legacy_files_count"] == 1
|
|
assert status["database_entries_count"] == 1
|
|
assert status["in_both"] == ["anime-1"]
|
|
assert status["migration_complete"] is True
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_status_with_only_db_entries(
|
|
self,
|
|
migration_service: DataMigrationService,
|
|
temp_anime_dir: str,
|
|
test_session,
|
|
):
|
|
"""Test status when database has entries but no files."""
|
|
await AnimeSeriesService.create(
|
|
test_session,
|
|
key="db-only-anime",
|
|
name="DB Only Anime",
|
|
site="aniworld.to",
|
|
folder="DB Only",
|
|
)
|
|
await test_session.commit()
|
|
|
|
status = await migration_service.get_migration_status(
|
|
temp_anime_dir, test_session
|
|
)
|
|
|
|
assert status["legacy_files_count"] == 0
|
|
assert status["database_entries_count"] == 1
|
|
assert status["only_in_database"] == ["db-only-anime"]
|
|
assert status["migration_complete"] is True
|