Add data file to database sync functionality
- Add get_all_series_from_data_files() to SeriesApp - Sync series from data files to DB on startup - Add unit tests for new SeriesApp method - Add integration tests for sync functionality - Update documentation
This commit is contained in:
350
tests/integration/test_data_file_db_sync.py
Normal file
350
tests/integration/test_data_file_db_sync.py
Normal file
@@ -0,0 +1,350 @@
|
||||
"""Integration tests for data file to database synchronization.
|
||||
|
||||
This module verifies that the data file to database sync functionality
|
||||
works correctly, including:
|
||||
- Loading series from data files
|
||||
- Adding series to the database
|
||||
- Preventing duplicate entries
|
||||
- Handling corrupt or missing files gracefully
|
||||
- End-to-end startup sync behavior
|
||||
|
||||
The sync functionality allows existing series metadata stored in
|
||||
data files to be automatically imported into the database during
|
||||
application startup.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from src.core.entities.SerieList import SerieList
|
||||
from src.core.entities.series import Serie
|
||||
from src.core.SeriesApp import SeriesApp
|
||||
|
||||
|
||||
class TestGetAllSeriesFromDataFiles:
|
||||
"""Test SeriesApp.get_all_series_from_data_files() method."""
|
||||
|
||||
def test_returns_empty_list_for_empty_directory(self):
|
||||
"""Test that empty directory returns empty list."""
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
with patch('src.core.SeriesApp.Loaders'), \
|
||||
patch('src.core.SeriesApp.SerieScanner'):
|
||||
app = SeriesApp(tmp_dir)
|
||||
result = app.get_all_series_from_data_files()
|
||||
|
||||
assert isinstance(result, list)
|
||||
assert len(result) == 0
|
||||
|
||||
def test_returns_series_from_data_files(self):
|
||||
"""Test that valid data files are loaded correctly."""
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
# Create test data files
|
||||
_create_test_data_file(
|
||||
tmp_dir,
|
||||
folder="Anime Test 1",
|
||||
key="anime-test-1",
|
||||
name="Anime Test 1",
|
||||
episodes={1: [1, 2, 3]}
|
||||
)
|
||||
_create_test_data_file(
|
||||
tmp_dir,
|
||||
folder="Anime Test 2",
|
||||
key="anime-test-2",
|
||||
name="Anime Test 2",
|
||||
episodes={1: [1]}
|
||||
)
|
||||
|
||||
with patch('src.core.SeriesApp.Loaders'), \
|
||||
patch('src.core.SeriesApp.SerieScanner'):
|
||||
app = SeriesApp(tmp_dir)
|
||||
result = app.get_all_series_from_data_files()
|
||||
|
||||
assert isinstance(result, list)
|
||||
assert len(result) == 2
|
||||
keys = {s.key for s in result}
|
||||
assert "anime-test-1" in keys
|
||||
assert "anime-test-2" in keys
|
||||
|
||||
def test_handles_corrupt_data_files_gracefully(self):
|
||||
"""Test that corrupt data files don't crash the sync."""
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
# Create a valid data file
|
||||
_create_test_data_file(
|
||||
tmp_dir,
|
||||
folder="Valid Anime",
|
||||
key="valid-anime",
|
||||
name="Valid Anime",
|
||||
episodes={1: [1]}
|
||||
)
|
||||
|
||||
# Create a corrupt data file (invalid JSON)
|
||||
corrupt_dir = os.path.join(tmp_dir, "Corrupt Anime")
|
||||
os.makedirs(corrupt_dir, exist_ok=True)
|
||||
with open(os.path.join(corrupt_dir, "data"), "w") as f:
|
||||
f.write("this is not valid json {{{")
|
||||
|
||||
with patch('src.core.SeriesApp.Loaders'), \
|
||||
patch('src.core.SeriesApp.SerieScanner'):
|
||||
app = SeriesApp(tmp_dir)
|
||||
result = app.get_all_series_from_data_files()
|
||||
|
||||
# Should still return the valid series
|
||||
assert isinstance(result, list)
|
||||
assert len(result) >= 1
|
||||
# The valid anime should be loaded
|
||||
keys = {s.key for s in result}
|
||||
assert "valid-anime" in keys
|
||||
|
||||
def test_handles_missing_directory_gracefully(self):
|
||||
"""Test that non-existent directory returns empty list."""
|
||||
non_existent_dir = "/non/existent/directory/path"
|
||||
|
||||
with patch('src.core.SeriesApp.Loaders'), \
|
||||
patch('src.core.SeriesApp.SerieScanner'):
|
||||
app = SeriesApp(non_existent_dir)
|
||||
result = app.get_all_series_from_data_files()
|
||||
|
||||
assert isinstance(result, list)
|
||||
assert len(result) == 0
|
||||
|
||||
|
||||
class TestSerieListAddToDb:
|
||||
"""Test SerieList.add_to_db() method for database insertion."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_to_db_creates_record(self):
|
||||
"""Test that add_to_db creates a database record."""
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
serie = Serie(
|
||||
key="new-anime",
|
||||
name="New Anime",
|
||||
site="https://aniworld.to",
|
||||
folder="New Anime (2024)",
|
||||
episodeDict={1: [1, 2, 3], 2: [1, 2]}
|
||||
)
|
||||
|
||||
# Mock database session and services
|
||||
mock_db = AsyncMock()
|
||||
mock_anime_series = Mock()
|
||||
mock_anime_series.id = 1
|
||||
mock_anime_series.key = "new-anime"
|
||||
mock_anime_series.name = "New Anime"
|
||||
|
||||
with patch(
|
||||
'src.server.database.service.AnimeSeriesService'
|
||||
) as mock_service, patch(
|
||||
'src.server.database.service.EpisodeService'
|
||||
) as mock_episode_service:
|
||||
# Setup mocks
|
||||
mock_service.get_by_key = AsyncMock(return_value=None)
|
||||
mock_service.create = AsyncMock(return_value=mock_anime_series)
|
||||
mock_episode_service.create = AsyncMock()
|
||||
|
||||
serie_list = SerieList(tmp_dir, skip_load=True)
|
||||
result = await serie_list.add_to_db(serie, mock_db)
|
||||
|
||||
# Verify series was created
|
||||
assert result is not None
|
||||
mock_service.create.assert_called_once()
|
||||
|
||||
# Verify episodes were created (5 total: 3 + 2)
|
||||
assert mock_episode_service.create.call_count == 5
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_to_db_skips_existing_series(self):
|
||||
"""Test that add_to_db skips existing series."""
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
serie = Serie(
|
||||
key="existing-anime",
|
||||
name="Existing Anime",
|
||||
site="https://aniworld.to",
|
||||
folder="Existing Anime (2023)",
|
||||
episodeDict={1: [1]}
|
||||
)
|
||||
|
||||
mock_db = AsyncMock()
|
||||
mock_existing = Mock()
|
||||
mock_existing.id = 99
|
||||
mock_existing.key = "existing-anime"
|
||||
|
||||
with patch(
|
||||
'src.server.database.service.AnimeSeriesService'
|
||||
) as mock_service:
|
||||
# Return existing series
|
||||
mock_service.get_by_key = AsyncMock(return_value=mock_existing)
|
||||
mock_service.create = AsyncMock()
|
||||
|
||||
serie_list = SerieList(tmp_dir, skip_load=True)
|
||||
result = await serie_list.add_to_db(serie, mock_db)
|
||||
|
||||
# Verify None returned (already exists)
|
||||
assert result is None
|
||||
# Verify create was NOT called
|
||||
mock_service.create.assert_not_called()
|
||||
|
||||
|
||||
class TestSyncSeriesToDatabase:
|
||||
"""Test _sync_series_to_database function from fastapi_app."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_with_empty_directory(self):
|
||||
"""Test sync with empty anime directory."""
|
||||
from src.server.fastapi_app import _sync_series_to_database
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
mock_logger = Mock()
|
||||
|
||||
with patch('src.core.SeriesApp.Loaders'), \
|
||||
patch('src.core.SeriesApp.SerieScanner'):
|
||||
count = await _sync_series_to_database(tmp_dir, mock_logger)
|
||||
|
||||
assert count == 0
|
||||
# Should log that no series were found
|
||||
mock_logger.info.assert_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_adds_new_series_to_database(self):
|
||||
"""Test that sync adds new series to database.
|
||||
|
||||
This is a more realistic test that verifies series data is loaded
|
||||
from files and the sync function attempts to add them to the DB.
|
||||
The actual DB interaction is tested in test_add_to_db_creates_record.
|
||||
"""
|
||||
from src.server.fastapi_app import _sync_series_to_database
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
# Create test data files
|
||||
_create_test_data_file(
|
||||
tmp_dir,
|
||||
folder="Sync Test Anime",
|
||||
key="sync-test-anime",
|
||||
name="Sync Test Anime",
|
||||
episodes={1: [1, 2]}
|
||||
)
|
||||
|
||||
mock_logger = Mock()
|
||||
|
||||
# First verify that we can load the series from files
|
||||
with patch('src.core.SeriesApp.Loaders'), \
|
||||
patch('src.core.SeriesApp.SerieScanner'):
|
||||
app = SeriesApp(tmp_dir)
|
||||
series = app.get_all_series_from_data_files()
|
||||
assert len(series) == 1
|
||||
assert series[0].key == "sync-test-anime"
|
||||
|
||||
# Now test that the sync function loads series and handles DB
|
||||
# gracefully (even if DB operations fail, it should not crash)
|
||||
with patch('src.core.SeriesApp.Loaders'), \
|
||||
patch('src.core.SeriesApp.SerieScanner'):
|
||||
# The function should return 0 because DB isn't available
|
||||
# but should not crash
|
||||
count = await _sync_series_to_database(tmp_dir, mock_logger)
|
||||
|
||||
# Since no real DB, it will fail gracefully
|
||||
assert isinstance(count, int)
|
||||
# Should have logged something
|
||||
assert mock_logger.info.called or mock_logger.warning.called
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_handles_exceptions_gracefully(self):
|
||||
"""Test that sync handles exceptions without crashing."""
|
||||
from src.server.fastapi_app import _sync_series_to_database
|
||||
|
||||
mock_logger = Mock()
|
||||
|
||||
# Make SeriesApp raise an exception during initialization
|
||||
with patch('src.core.SeriesApp.Loaders'), \
|
||||
patch('src.core.SeriesApp.SerieScanner'), \
|
||||
patch(
|
||||
'src.core.SeriesApp.SerieList',
|
||||
side_effect=Exception("Test error")
|
||||
):
|
||||
count = await _sync_series_to_database(
|
||||
"/fake/path", mock_logger
|
||||
)
|
||||
|
||||
assert count == 0
|
||||
# Should log the warning
|
||||
mock_logger.warning.assert_called()
|
||||
|
||||
|
||||
class TestEndToEndSync:
|
||||
"""End-to-end tests for the sync functionality."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_startup_sync_integration(self):
|
||||
"""Test end-to-end startup sync behavior."""
|
||||
# This test verifies the integration of all components
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
# Create test data
|
||||
_create_test_data_file(
|
||||
tmp_dir,
|
||||
folder="E2E Test Anime 1",
|
||||
key="e2e-test-anime-1",
|
||||
name="E2E Test Anime 1",
|
||||
episodes={1: [1, 2, 3]}
|
||||
)
|
||||
_create_test_data_file(
|
||||
tmp_dir,
|
||||
folder="E2E Test Anime 2",
|
||||
key="e2e-test-anime-2",
|
||||
name="E2E Test Anime 2",
|
||||
episodes={1: [1], 2: [1, 2]}
|
||||
)
|
||||
|
||||
# Use SeriesApp to load series from files
|
||||
with patch('src.core.SeriesApp.Loaders'), \
|
||||
patch('src.core.SeriesApp.SerieScanner'):
|
||||
app = SeriesApp(tmp_dir)
|
||||
all_series = app.get_all_series_from_data_files()
|
||||
|
||||
# Verify all series were loaded
|
||||
assert len(all_series) == 2
|
||||
|
||||
# Verify series data is correct
|
||||
series_by_key = {s.key: s for s in all_series}
|
||||
assert "e2e-test-anime-1" in series_by_key
|
||||
assert "e2e-test-anime-2" in series_by_key
|
||||
|
||||
# Verify episode data
|
||||
anime1 = series_by_key["e2e-test-anime-1"]
|
||||
assert anime1.episodeDict == {1: [1, 2, 3]}
|
||||
|
||||
anime2 = series_by_key["e2e-test-anime-2"]
|
||||
assert anime2.episodeDict == {1: [1], 2: [1, 2]}
|
||||
|
||||
|
||||
def _create_test_data_file(
|
||||
base_dir: str,
|
||||
folder: str,
|
||||
key: str,
|
||||
name: str,
|
||||
episodes: dict
|
||||
) -> None:
|
||||
"""
|
||||
Create a test data file in the anime directory.
|
||||
|
||||
Args:
|
||||
base_dir: Base directory for anime folders
|
||||
folder: Folder name for the anime
|
||||
key: Unique key for the series
|
||||
name: Display name of the series
|
||||
episodes: Dictionary mapping season to list of episode numbers
|
||||
"""
|
||||
anime_dir = os.path.join(base_dir, folder)
|
||||
os.makedirs(anime_dir, exist_ok=True)
|
||||
|
||||
data = {
|
||||
"key": key,
|
||||
"name": name,
|
||||
"site": "https://aniworld.to",
|
||||
"folder": folder,
|
||||
"episodeDict": {str(k): v for k, v in episodes.items()}
|
||||
}
|
||||
|
||||
data_file = os.path.join(anime_dir, "data")
|
||||
with open(data_file, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
@@ -559,3 +559,196 @@ class TestSeriesAppAsyncDbInit:
|
||||
assert len(w) == 1
|
||||
assert "without db_session" in str(w[0].message)
|
||||
|
||||
|
||||
class TestSeriesAppGetAllSeriesFromDataFiles:
|
||||
"""Test get_all_series_from_data_files() functionality."""
|
||||
|
||||
@patch('src.core.SeriesApp.Loaders')
|
||||
@patch('src.core.SeriesApp.SerieScanner')
|
||||
@patch('src.core.SeriesApp.SerieList')
|
||||
def test_returns_list_of_series(
|
||||
self, mock_serie_list_class, mock_scanner, mock_loaders
|
||||
):
|
||||
"""Test that get_all_series_from_data_files returns a list of Serie."""
|
||||
from src.core.entities.series import Serie
|
||||
|
||||
test_dir = "/test/anime"
|
||||
|
||||
# Mock series to return
|
||||
mock_series = [
|
||||
Serie(
|
||||
key="anime1",
|
||||
name="Anime 1",
|
||||
site="https://aniworld.to",
|
||||
folder="Anime 1 (2020)",
|
||||
episodeDict={1: [1, 2, 3]}
|
||||
),
|
||||
Serie(
|
||||
key="anime2",
|
||||
name="Anime 2",
|
||||
site="https://aniworld.to",
|
||||
folder="Anime 2 (2021)",
|
||||
episodeDict={1: [1]}
|
||||
),
|
||||
]
|
||||
|
||||
# Setup mock for the main SerieList instance (constructor call)
|
||||
mock_main_list = Mock()
|
||||
mock_main_list.GetMissingEpisode.return_value = []
|
||||
|
||||
# Setup mock for temporary SerieList in get_all_series_from_data_files
|
||||
mock_temp_list = Mock()
|
||||
mock_temp_list.get_all.return_value = mock_series
|
||||
|
||||
# Return different mocks for the two calls
|
||||
mock_serie_list_class.side_effect = [mock_main_list, mock_temp_list]
|
||||
|
||||
# Create app
|
||||
app = SeriesApp(test_dir)
|
||||
|
||||
# Call the method
|
||||
result = app.get_all_series_from_data_files()
|
||||
|
||||
# Verify result is a list of Serie
|
||||
assert isinstance(result, list)
|
||||
assert len(result) == 2
|
||||
assert all(isinstance(s, Serie) for s in result)
|
||||
assert result[0].key == "anime1"
|
||||
assert result[1].key == "anime2"
|
||||
|
||||
@patch('src.core.SeriesApp.Loaders')
|
||||
@patch('src.core.SeriesApp.SerieScanner')
|
||||
@patch('src.core.SeriesApp.SerieList')
|
||||
def test_returns_empty_list_when_no_data_files(
|
||||
self, mock_serie_list_class, mock_scanner, mock_loaders
|
||||
):
|
||||
"""Test that empty list is returned when no data files exist."""
|
||||
test_dir = "/test/anime"
|
||||
|
||||
# Setup mock for the main SerieList instance
|
||||
mock_main_list = Mock()
|
||||
mock_main_list.GetMissingEpisode.return_value = []
|
||||
|
||||
# Setup mock for the temporary SerieList (empty directory)
|
||||
mock_temp_list = Mock()
|
||||
mock_temp_list.get_all.return_value = []
|
||||
|
||||
mock_serie_list_class.side_effect = [mock_main_list, mock_temp_list]
|
||||
|
||||
# Create app
|
||||
app = SeriesApp(test_dir)
|
||||
|
||||
# Call the method
|
||||
result = app.get_all_series_from_data_files()
|
||||
|
||||
# Verify empty list is returned
|
||||
assert isinstance(result, list)
|
||||
assert len(result) == 0
|
||||
|
||||
@patch('src.core.SeriesApp.Loaders')
|
||||
@patch('src.core.SeriesApp.SerieScanner')
|
||||
@patch('src.core.SeriesApp.SerieList')
|
||||
def test_handles_exception_gracefully(
|
||||
self, mock_serie_list_class, mock_scanner, mock_loaders
|
||||
):
|
||||
"""Test exceptions are handled gracefully and empty list returned."""
|
||||
test_dir = "/test/anime"
|
||||
|
||||
# Setup mock for the main SerieList instance
|
||||
mock_main_list = Mock()
|
||||
mock_main_list.GetMissingEpisode.return_value = []
|
||||
|
||||
# Make the second SerieList constructor raise an exception
|
||||
mock_serie_list_class.side_effect = [
|
||||
mock_main_list,
|
||||
OSError("Directory not found")
|
||||
]
|
||||
|
||||
# Create app
|
||||
app = SeriesApp(test_dir)
|
||||
|
||||
# Call the method - should not raise
|
||||
result = app.get_all_series_from_data_files()
|
||||
|
||||
# Verify empty list is returned on error
|
||||
assert isinstance(result, list)
|
||||
assert len(result) == 0
|
||||
|
||||
@patch('src.core.SeriesApp.Loaders')
|
||||
@patch('src.core.SeriesApp.SerieScanner')
|
||||
@patch('src.core.SeriesApp.SerieList')
|
||||
def test_uses_file_based_loading(
|
||||
self, mock_serie_list_class, mock_scanner, mock_loaders
|
||||
):
|
||||
"""Test that method uses file-based loading (no db_session)."""
|
||||
test_dir = "/test/anime"
|
||||
|
||||
# Setup mock for the main SerieList instance
|
||||
mock_main_list = Mock()
|
||||
mock_main_list.GetMissingEpisode.return_value = []
|
||||
|
||||
# Setup mock for the temporary SerieList
|
||||
mock_temp_list = Mock()
|
||||
mock_temp_list.get_all.return_value = []
|
||||
|
||||
mock_serie_list_class.side_effect = [mock_main_list, mock_temp_list]
|
||||
|
||||
# Create app
|
||||
app = SeriesApp(test_dir)
|
||||
|
||||
# Call the method
|
||||
app.get_all_series_from_data_files()
|
||||
|
||||
# Verify the second SerieList was created with correct params
|
||||
# (file-based loading: db_session=None, skip_load=False)
|
||||
calls = mock_serie_list_class.call_args_list
|
||||
assert len(calls) == 2
|
||||
|
||||
# Check the second call (for get_all_series_from_data_files)
|
||||
second_call = calls[1]
|
||||
assert second_call.kwargs.get('db_session') is None
|
||||
assert second_call.kwargs.get('skip_load') is False
|
||||
|
||||
@patch('src.core.SeriesApp.Loaders')
|
||||
@patch('src.core.SeriesApp.SerieScanner')
|
||||
@patch('src.core.SeriesApp.SerieList')
|
||||
def test_does_not_modify_main_list(
|
||||
self, mock_serie_list_class, mock_scanner, mock_loaders
|
||||
):
|
||||
"""Test that method does not modify the main SerieList instance."""
|
||||
from src.core.entities.series import Serie
|
||||
|
||||
test_dir = "/test/anime"
|
||||
|
||||
# Setup mock for the main SerieList instance
|
||||
mock_main_list = Mock()
|
||||
mock_main_list.GetMissingEpisode.return_value = []
|
||||
mock_main_list.get_all.return_value = []
|
||||
|
||||
# Setup mock for the temporary SerieList
|
||||
mock_temp_list = Mock()
|
||||
mock_temp_list.get_all.return_value = [
|
||||
Serie(
|
||||
key="anime1",
|
||||
name="Anime 1",
|
||||
site="https://aniworld.to",
|
||||
folder="Anime 1",
|
||||
episodeDict={}
|
||||
)
|
||||
]
|
||||
|
||||
mock_serie_list_class.side_effect = [mock_main_list, mock_temp_list]
|
||||
|
||||
# Create app
|
||||
app = SeriesApp(test_dir)
|
||||
|
||||
# Store reference to original list
|
||||
original_list = app.list
|
||||
|
||||
# Call the method
|
||||
app.get_all_series_from_data_files()
|
||||
|
||||
# Verify main list is unchanged
|
||||
assert app.list is original_list
|
||||
# Verify the main list's get_all was not called
|
||||
mock_main_list.get_all.assert_not_called()
|
||||
|
||||
Reference in New Issue
Block a user