- Add migration_legacy_files_completed flag to SystemSettings model - Create legacy_file_migration service to migrate series from key/data files - Integrate legacy migration into initialization_service startup flow - Add integration tests for legacy file migration - Update DATABASE.md documentation with migration details - Fix various test and service issues (nfo_repair, tmdb_client, download_service) - Add test_database_schema unit tests
336 lines
13 KiB
Python
336 lines
13 KiB
Python
"""Integration tests for legacy key/data file migration.
|
|
|
|
Tests the one-time migration safety net that imports series from
|
|
legacy key and data files into the database.
|
|
"""
|
|
import json
|
|
import os
|
|
import tempfile
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from src.server.services.legacy_file_migration import (
|
|
_load_data_file,
|
|
_load_key_file,
|
|
migrate_series_from_files_to_db,
|
|
)
|
|
|
|
|
|
class TestLoadLegacyFiles:
|
|
"""Test helper functions for loading legacy files."""
|
|
|
|
def test_load_data_file_valid_json(self):
|
|
"""Test loading a valid JSON data file."""
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
data_file = os.path.join(tmp_dir, "data")
|
|
test_data = {
|
|
"key": "test-anime",
|
|
"name": "Test Anime",
|
|
"site": "https://aniworld.to",
|
|
"folder": "Test Anime",
|
|
"episodeDict": {"1": [1, 2, 3]}
|
|
}
|
|
with open(data_file, "w", encoding="utf-8") as f:
|
|
json.dump(test_data, f)
|
|
|
|
result = _load_data_file(data_file)
|
|
|
|
assert result is not None
|
|
assert result["key"] == "test-anime"
|
|
assert result["name"] == "Test Anime"
|
|
# episodeDict keys should be converted to int
|
|
assert 1 in result["episodeDict"]
|
|
|
|
def test_load_data_file_invalid_json(self):
|
|
"""Test handling of corrupt JSON data file."""
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
data_file = os.path.join(tmp_dir, "data")
|
|
with open(data_file, "w", encoding="utf-8") as f:
|
|
f.write("this is not valid json {{{")
|
|
|
|
result = _load_data_file(data_file)
|
|
|
|
assert result is None
|
|
|
|
def test_load_data_file_not_dict(self):
|
|
"""Test handling of JSON file that is not a dict."""
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
data_file = os.path.join(tmp_dir, "data")
|
|
with open(data_file, "w", encoding="utf-8") as f:
|
|
json.dump(["not", "a", "dict"], f)
|
|
|
|
result = _load_data_file(data_file)
|
|
|
|
assert result is None
|
|
|
|
def test_load_key_file_valid(self):
|
|
"""Test loading a key file with valid content."""
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
key_file = os.path.join(tmp_dir, "key")
|
|
with open(key_file, "w", encoding="utf-8") as f:
|
|
f.write("my-anime-key")
|
|
|
|
result = _load_key_file(key_file, "My Anime")
|
|
|
|
assert result is not None
|
|
assert result["key"] == "my-anime-key"
|
|
assert result["name"] == "My Anime"
|
|
assert result["site"] == "https://aniworld.to"
|
|
assert result["episodeDict"] == {}
|
|
|
|
def test_load_key_file_empty(self):
|
|
"""Test handling of empty key file."""
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
key_file = os.path.join(tmp_dir, "key")
|
|
with open(key_file, "w", encoding="utf-8") as f:
|
|
f.write("")
|
|
|
|
result = _load_key_file(key_file, "My Anime")
|
|
|
|
assert result is None
|
|
|
|
|
|
class TestMigrateLegacyFiles:
|
|
"""Test the main migration function with database."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_migrate_series_from_files_to_db_no_files(self):
|
|
"""Test migration with empty directory returns 0."""
|
|
mock_db = AsyncMock()
|
|
mock_db.execute = AsyncMock()
|
|
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
count = await migrate_series_from_files_to_db(tmp_dir, mock_db)
|
|
assert count == 0
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_migrate_data_file_to_db(self):
|
|
"""Test migration of a legacy data file."""
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
# Create a folder with a data file
|
|
anime_folder = os.path.join(tmp_dir, "Test Anime")
|
|
os.makedirs(anime_folder, exist_ok=True)
|
|
|
|
data_file = os.path.join(anime_folder, "data")
|
|
test_data = {
|
|
"key": "migrate-test-anime",
|
|
"name": "Migrate Test Anime",
|
|
"site": "https://aniworld.to",
|
|
"folder": "Test Anime",
|
|
"episodeDict": {"1": [1, 2]}
|
|
}
|
|
with open(data_file, "w", encoding="utf-8") as f:
|
|
json.dump(test_data, f)
|
|
|
|
# Mock the DB session and services
|
|
mock_db = AsyncMock()
|
|
mock_series_service = AsyncMock()
|
|
mock_episode_service = AsyncMock()
|
|
|
|
# Mock get_by_key returning None (not in DB)
|
|
mock_series_service.get_by_key = AsyncMock(return_value=None)
|
|
|
|
# Mock AnimeSeriesService.create returning a mock with id=1
|
|
mock_created_series = MagicMock()
|
|
mock_created_series.id = 1
|
|
mock_series_service.create = AsyncMock(return_value=mock_created_series)
|
|
|
|
with patch.dict('sys.modules', {
|
|
'src.server.database.service': MagicMock(
|
|
AnimeSeriesService=mock_series_service,
|
|
EpisodeService=mock_episode_service
|
|
)
|
|
}):
|
|
count = await migrate_series_from_files_to_db(tmp_dir, mock_db)
|
|
assert count == 1
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_migrate_key_file_to_db(self):
|
|
"""Test migration of a legacy key file."""
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
# Create a folder with only a key file
|
|
anime_folder = os.path.join(tmp_dir, "Key Only Anime")
|
|
os.makedirs(anime_folder, exist_ok=True)
|
|
|
|
key_file = os.path.join(anime_folder, "key")
|
|
with open(key_file, "w", encoding="utf-8") as f:
|
|
f.write("key-only-anime")
|
|
|
|
# Mock the DB session and services
|
|
mock_db = AsyncMock()
|
|
mock_series_service = AsyncMock()
|
|
mock_episode_service = AsyncMock()
|
|
|
|
# Mock get_by_key returning None (not in DB)
|
|
mock_series_service.get_by_key = AsyncMock(return_value=None)
|
|
|
|
# Mock AnimeSeriesService.create returning a mock with id=1
|
|
mock_created_series = MagicMock()
|
|
mock_created_series.id = 1
|
|
mock_series_service.create = AsyncMock(return_value=mock_created_series)
|
|
|
|
with patch.dict('sys.modules', {
|
|
'src.server.database.service': MagicMock(
|
|
AnimeSeriesService=mock_series_service,
|
|
EpisodeService=mock_episode_service
|
|
)
|
|
}):
|
|
count = await migrate_series_from_files_to_db(tmp_dir, mock_db)
|
|
assert count == 1
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_migration_skips_already_migrated(self):
|
|
"""Test that migration skips series already in DB."""
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
# Create a folder with a data file
|
|
anime_folder = os.path.join(tmp_dir, "Already Migrated")
|
|
os.makedirs(anime_folder, exist_ok=True)
|
|
|
|
data_file = os.path.join(anime_folder, "data")
|
|
test_data = {
|
|
"key": "already-migrated",
|
|
"name": "Already Migrated",
|
|
"site": "https://aniworld.to",
|
|
"folder": "Already Migrated",
|
|
"episodeDict": {"1": [1]}
|
|
}
|
|
with open(data_file, "w", encoding="utf-8") as f:
|
|
json.dump(test_data, f)
|
|
|
|
# Mock the DB session and services
|
|
mock_db = AsyncMock()
|
|
mock_series_service = AsyncMock()
|
|
mock_episode_service = AsyncMock()
|
|
|
|
# Mock get_by_key returning existing series (already migrated)
|
|
mock_existing_series = MagicMock()
|
|
mock_existing_series.name = "Modified Name"
|
|
mock_series_service.get_by_key = AsyncMock(return_value=mock_existing_series)
|
|
|
|
with patch.dict('sys.modules', {
|
|
'src.server.database.service': MagicMock(
|
|
AnimeSeriesService=mock_series_service,
|
|
EpisodeService=mock_episode_service
|
|
)
|
|
}):
|
|
count = await migrate_series_from_files_to_db(tmp_dir, mock_db)
|
|
assert count == 0 # No new series migrated
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_migration_handles_corrupt_data_file(self):
|
|
"""Test that corrupt data files don't crash migration."""
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
# Create a folder with a corrupt data file
|
|
corrupt_folder = os.path.join(tmp_dir, "Corrupt Anime")
|
|
os.makedirs(corrupt_folder, exist_ok=True)
|
|
|
|
corrupt_file = os.path.join(corrupt_folder, "data")
|
|
with open(corrupt_file, "w", encoding="utf-8") as f:
|
|
f.write("not valid json {{{")
|
|
|
|
# Create a valid folder
|
|
valid_folder = os.path.join(tmp_dir, "Valid Anime")
|
|
os.makedirs(valid_folder, exist_ok=True)
|
|
|
|
valid_file = os.path.join(valid_folder, "data")
|
|
valid_data = {
|
|
"key": "valid-anime",
|
|
"name": "Valid Anime",
|
|
"site": "https://aniworld.to",
|
|
"folder": "Valid Anime",
|
|
"episodeDict": {"1": [1]}
|
|
}
|
|
with open(valid_file, "w", encoding="utf-8") as f:
|
|
json.dump(valid_data, f)
|
|
|
|
# Mock the DB session and services
|
|
mock_db = AsyncMock()
|
|
mock_series_service = AsyncMock()
|
|
mock_episode_service = AsyncMock()
|
|
|
|
# Mock get_by_key returning None (not in DB)
|
|
mock_series_service.get_by_key = AsyncMock(return_value=None)
|
|
|
|
# Mock AnimeSeriesService.create returning a mock with id=1
|
|
mock_created_series = MagicMock()
|
|
mock_created_series.id = 1
|
|
mock_series_service.create = AsyncMock(return_value=mock_created_series)
|
|
|
|
with patch.dict('sys.modules', {
|
|
'src.server.database.service': MagicMock(
|
|
AnimeSeriesService=mock_series_service,
|
|
EpisodeService=mock_episode_service
|
|
)
|
|
}):
|
|
# Migration should succeed despite corrupt file
|
|
count = await migrate_series_from_files_to_db(tmp_dir, mock_db)
|
|
assert count == 1 # Only the valid one
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_migration_idempotent(self):
|
|
"""Test that running migration twice doesn't change DB state."""
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
# Create a folder with a data file
|
|
anime_folder = os.path.join(tmp_dir, "Idempotent Test")
|
|
os.makedirs(anime_folder, exist_ok=True)
|
|
|
|
data_file = os.path.join(anime_folder, "data")
|
|
test_data = {
|
|
"key": "idempotent-test",
|
|
"name": "Idempotent Test",
|
|
"site": "https://aniworld.to",
|
|
"folder": "Idempotent Test",
|
|
"episodeDict": {"1": [1, 2]}
|
|
}
|
|
with open(data_file, "w", encoding="utf-8") as f:
|
|
json.dump(test_data, f)
|
|
|
|
# Mock the DB session and services
|
|
mock_db = AsyncMock()
|
|
mock_series_service = AsyncMock()
|
|
mock_episode_service = AsyncMock()
|
|
|
|
# First call returns None (not in DB), second call returns the series
|
|
mock_existing_series = MagicMock()
|
|
mock_existing_series.id = 1
|
|
mock_series_service.get_by_key = AsyncMock(side_effect=[None, mock_existing_series])
|
|
|
|
# Mock AnimeSeriesService.create returning a mock with id=1
|
|
mock_created_series = MagicMock()
|
|
mock_created_series.id = 1
|
|
mock_series_service.create = AsyncMock(return_value=mock_created_series)
|
|
|
|
with patch.dict('sys.modules', {
|
|
'src.server.database.service': MagicMock(
|
|
AnimeSeriesService=mock_series_service,
|
|
EpisodeService=mock_episode_service
|
|
)
|
|
}):
|
|
# First migration
|
|
count1 = await migrate_series_from_files_to_db(tmp_dir, mock_db)
|
|
assert count1 == 1
|
|
|
|
# Second migration
|
|
count2 = await migrate_series_from_files_to_db(tmp_dir, mock_db)
|
|
assert count2 == 0 # Already migrated
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_migration_skips_folders_without_files(self):
|
|
"""Test that folders without key/data files are skipped."""
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
# Create an empty folder (no key or data file)
|
|
empty_folder = os.path.join(tmp_dir, "Empty Folder")
|
|
os.makedirs(empty_folder, exist_ok=True)
|
|
|
|
# Create a folder with only a video file
|
|
video_folder = os.path.join(tmp_dir, "Video Folder")
|
|
os.makedirs(video_folder, exist_ok=True)
|
|
with open(os.path.join(video_folder, "episode1.mp4"), "w") as f:
|
|
f.write("fake video content")
|
|
|
|
mock_db = AsyncMock()
|
|
|
|
count = await migrate_series_from_files_to_db(tmp_dir, mock_db)
|
|
assert count == 0
|