feat(SerieScanner): DB lookup primary, deprecate key file fallback
DB now source of truth for folder -> Serie resolution. Changes: - AnimeSeriesService.get_by_folder(): new async lookup by folder name - SerieScanner.__read_data_from_file(): query DB first, then provider callback, then legacy key file (temporary, removed v3.0.0) - Serie: reconstruct from DB record with episode dict - Key file: warn on use, scheduled removal v3.0.0 Add unit tests for DB hit/miss/callback/fallback edge cases Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
@@ -23,6 +23,9 @@ from src.core.entities.series import Serie
|
|||||||
from src.core.exceptions.Exceptions import MatchNotFoundError, NoKeyFoundException
|
from src.core.exceptions.Exceptions import MatchNotFoundError, NoKeyFoundException
|
||||||
from src.core.providers.base_provider import Loader
|
from src.core.providers.base_provider import Loader
|
||||||
|
|
||||||
|
from src.server.database.connection import get_sync_session
|
||||||
|
from src.server.database.service import AnimeSeriesService
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
error_logger = logging.getLogger("error")
|
error_logger = logging.getLogger("error")
|
||||||
no_key_found_logger = logging.getLogger("series.nokey")
|
no_key_found_logger = logging.getLogger("series.nokey")
|
||||||
@@ -278,25 +281,6 @@ class SerieScanner:
|
|||||||
)
|
)
|
||||||
|
|
||||||
serie = self.__read_data_from_file(folder)
|
serie = self.__read_data_from_file(folder)
|
||||||
if serie is None or not serie.key or not serie.key.strip():
|
|
||||||
# Fallback: ask the database for a matching series
|
|
||||||
if self._db_lookup is not None:
|
|
||||||
try:
|
|
||||||
serie = self._db_lookup(folder)
|
|
||||||
if serie:
|
|
||||||
logger.info(
|
|
||||||
"DB lookup resolved folder '%s' -> key='%s'",
|
|
||||||
folder,
|
|
||||||
serie.key,
|
|
||||||
)
|
|
||||||
except Exception as exc:
|
|
||||||
logger.warning(
|
|
||||||
"DB lookup failed for folder '%s': %s",
|
|
||||||
folder,
|
|
||||||
exc,
|
|
||||||
)
|
|
||||||
serie = None
|
|
||||||
|
|
||||||
if serie is None or not serie.key or not serie.key.strip():
|
if serie is None or not serie.key or not serie.key.strip():
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"No key or data file found for folder '%s', skipping",
|
"No key or data file found for folder '%s', skipping",
|
||||||
@@ -470,25 +454,83 @@ class SerieScanner:
|
|||||||
yield anime_name, mp4_files if has_files else []
|
yield anime_name, mp4_files if has_files else []
|
||||||
|
|
||||||
def __read_data_from_file(self, folder_name: str) -> Optional[Serie]:
|
def __read_data_from_file(self, folder_name: str) -> Optional[Serie]:
|
||||||
"""Read serie data from file or key file.
|
"""Load or discover a Serie for the given folder.
|
||||||
|
|
||||||
|
Strategy:
|
||||||
|
1. Query DB by folder name
|
||||||
|
2. If found, return cached Serie object
|
||||||
|
3. If not in DB, fall back to provider search via _db_lookup callback
|
||||||
|
4. (Legacy) If still not found, try reading 'key' file as last resort
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
folder_name: Filesystem folder name
|
folder_name: Filesystem folder name
|
||||||
(used only to locate data files)
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Serie object with valid key if found, None otherwise
|
Serie object with valid key if found, None otherwise
|
||||||
|
|
||||||
Note:
|
|
||||||
The returned Serie will have its 'key' as the primary identifier.
|
|
||||||
The 'folder' field is metadata only.
|
|
||||||
"""
|
|
||||||
folder_path = os.path.join(self.directory, folder_name)
|
|
||||||
key = None
|
|
||||||
key_file = os.path.join(folder_path, 'key')
|
|
||||||
serie_file = os.path.join(folder_path, 'data')
|
|
||||||
|
|
||||||
|
Note:
|
||||||
|
DB is the source of truth. File-based lookups (key/data files)
|
||||||
|
are temporary backward compatibility for deployments with old data.
|
||||||
|
Will be removed in v3.0.0.
|
||||||
|
"""
|
||||||
|
# Step 1: Try DB lookup by folder name
|
||||||
|
try:
|
||||||
|
session = get_sync_session()
|
||||||
|
try:
|
||||||
|
anime_series = AnimeSeriesService.get_by_folder_sync(session, folder_name)
|
||||||
|
if anime_series:
|
||||||
|
# Reconstruct Serie from DB record
|
||||||
|
episode_dict: dict[int, list[int]] = {}
|
||||||
|
if anime_series.episodes:
|
||||||
|
for ep in anime_series.episodes:
|
||||||
|
season = ep.season or 1
|
||||||
|
if season not in episode_dict:
|
||||||
|
episode_dict[season] = []
|
||||||
|
episode_dict[season].append(ep.episode_number or ep.number or 0)
|
||||||
|
return Serie(
|
||||||
|
key=anime_series.key,
|
||||||
|
name=anime_series.name,
|
||||||
|
site=anime_series.site,
|
||||||
|
folder=anime_series.folder,
|
||||||
|
episodeDict=episode_dict,
|
||||||
|
year=anime_series.year
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(
|
||||||
|
"DB lookup failed for folder '%s': %s",
|
||||||
|
folder_name,
|
||||||
|
exc
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 2: Fall back to provider search callback
|
||||||
|
if self._db_lookup is not None:
|
||||||
|
try:
|
||||||
|
serie = self._db_lookup(folder_name)
|
||||||
|
if serie and serie.key and serie.key.strip():
|
||||||
|
logger.info(
|
||||||
|
"Provider lookup resolved folder '%s' -> key='%s'",
|
||||||
|
folder_name,
|
||||||
|
serie.key
|
||||||
|
)
|
||||||
|
return serie
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(
|
||||||
|
"Provider lookup failed for folder '%s': %s",
|
||||||
|
folder_name,
|
||||||
|
exc
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 3: Legacy fallback - TEMPORARY (remove in v3.0.0)
|
||||||
|
folder_path = os.path.join(self.directory, folder_name)
|
||||||
|
key_file = os.path.join(folder_path, 'key')
|
||||||
if os.path.exists(key_file):
|
if os.path.exists(key_file):
|
||||||
|
logger.warning(
|
||||||
|
"Using legacy 'key' file for '%s' - this fallback is deprecated "
|
||||||
|
"and will be removed in v3.0.0",
|
||||||
|
folder_name
|
||||||
|
)
|
||||||
with open(key_file, 'r', encoding='utf-8') as file:
|
with open(key_file, 'r', encoding='utf-8') as file:
|
||||||
key = file.read().strip()
|
key = file.read().strip()
|
||||||
logger.info(
|
logger.info(
|
||||||
@@ -499,6 +541,7 @@ class SerieScanner:
|
|||||||
year_from_folder = self._extract_year_from_folder_name(folder_name)
|
year_from_folder = self._extract_year_from_folder_name(folder_name)
|
||||||
return Serie(key, "", "aniworld.to", folder_name, dict(), year=year_from_folder)
|
return Serie(key, "", "aniworld.to", folder_name, dict(), year=year_from_folder)
|
||||||
|
|
||||||
|
serie_file = os.path.join(folder_path, 'data')
|
||||||
if os.path.exists(serie_file):
|
if os.path.exists(serie_file):
|
||||||
with open(serie_file, "rb") as file:
|
with open(serie_file, "rb") as file:
|
||||||
logger.info(
|
logger.info(
|
||||||
|
|||||||
@@ -169,6 +169,26 @@ class AnimeSeriesService:
|
|||||||
)
|
)
|
||||||
return result.scalar_one_or_none()
|
return result.scalar_one_or_none()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def get_by_folder(db: AsyncSession, folder: str) -> Optional[AnimeSeries]:
|
||||||
|
"""Look up an anime series by its filesystem folder name (async).
|
||||||
|
|
||||||
|
Intended as primary lookup for ``SerieScanner`` when scanning
|
||||||
|
directories, replacing the legacy file-based lookups (key/data files).
|
||||||
|
|
||||||
|
Args:
|
||||||
|
db: Async database session.
|
||||||
|
folder: Filesystem folder name to match (e.g.
|
||||||
|
``"Rooster Fighter (2026)"``).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
``AnimeSeries`` instance or ``None`` if not found.
|
||||||
|
"""
|
||||||
|
result = await db.execute(
|
||||||
|
select(AnimeSeries).where(AnimeSeries.folder == folder)
|
||||||
|
)
|
||||||
|
return result.scalar_one_or_none()
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def get_all(
|
async def get_all(
|
||||||
db: AsyncSession,
|
db: AsyncSession,
|
||||||
|
|||||||
124
tests/unit/test_serie_scanner_db_lookup.py
Normal file
124
tests/unit/test_serie_scanner_db_lookup.py
Normal file
@@ -0,0 +1,124 @@
|
|||||||
|
"""Tests for SerieScanner DB lookup functionality."""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from src.core.entities.series import Serie
|
||||||
|
from src.core.SerieScanner import SerieScanner
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def mock_loader():
|
||||||
|
"""Create a mock Loader instance."""
|
||||||
|
loader = MagicMock()
|
||||||
|
loader.get_season_episode_count = MagicMock(return_value={1: 12})
|
||||||
|
loader.is_language = MagicMock(return_value=True)
|
||||||
|
loader.get_year = MagicMock(return_value=2026)
|
||||||
|
return loader
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def temp_directory():
|
||||||
|
"""Create a temporary directory with subdirectories for testing."""
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
anime_folder = os.path.join(tmpdir, "Rooster Fighter (2026)")
|
||||||
|
os.makedirs(anime_folder, exist_ok=True)
|
||||||
|
mp4_path = os.path.join(anime_folder, "S01E001.mp4")
|
||||||
|
with open(mp4_path, "w") as f:
|
||||||
|
f.write("dummy mp4")
|
||||||
|
yield tmpdir
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetSerieFromFolderDbLookup:
|
||||||
|
"""Test __read_data_from_file DB lookup behavior."""
|
||||||
|
|
||||||
|
def test_db_hit_returns_serie_from_db(self, temp_directory, mock_loader):
|
||||||
|
"""DB lookup resolves folder -> Serie returned."""
|
||||||
|
from src.server.database.models import AnimeSeries
|
||||||
|
from src.server.database import service as anime_series_service
|
||||||
|
|
||||||
|
mock_session = MagicMock()
|
||||||
|
mock_anime_series = MagicMock()
|
||||||
|
mock_anime_series.key = "rooster-fighter"
|
||||||
|
mock_anime_series.name = "Rooster Fighter"
|
||||||
|
mock_anime_series.site = "aniworld.to"
|
||||||
|
mock_anime_series.folder = "Rooster Fighter (2026)"
|
||||||
|
mock_anime_series.year = 2026
|
||||||
|
mock_anime_series.episodes = []
|
||||||
|
mock_session.execute.return_value.scalar_one_or_none.return_value = mock_anime_series
|
||||||
|
|
||||||
|
with patch("src.core.SerieScanner.get_sync_session", return_value=mock_session):
|
||||||
|
scanner = SerieScanner(temp_directory, mock_loader)
|
||||||
|
result = scanner._SerieScanner__read_data_from_file("Rooster Fighter (2026)")
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert result.key == "rooster-fighter"
|
||||||
|
assert result.name == "Rooster Fighter"
|
||||||
|
assert result.year == 2026
|
||||||
|
|
||||||
|
def test_db_miss_falls_back_to_provider_callback(self, temp_directory, mock_loader):
|
||||||
|
"""DB miss -> _db_lookup callback called."""
|
||||||
|
lookup = MagicMock(return_value=Serie(
|
||||||
|
key="rooster-fighter",
|
||||||
|
name="Rooster Fighter",
|
||||||
|
site="aniworld.to",
|
||||||
|
folder="Rooster Fighter (2026)",
|
||||||
|
episodeDict={},
|
||||||
|
))
|
||||||
|
scanner = SerieScanner(temp_directory, mock_loader, db_lookup=lookup)
|
||||||
|
|
||||||
|
result = scanner._SerieScanner__read_data_from_file("Rooster Fighter (2026)")
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert result.key == "rooster-fighter"
|
||||||
|
lookup.assert_called_once_with("Rooster Fighter (2026)")
|
||||||
|
|
||||||
|
def test_legacy_key_file_as_last_resort(self, temp_directory, mock_loader):
|
||||||
|
"""No DB, no callback -> legacy 'key' file used with deprecation warning."""
|
||||||
|
folder = os.path.join(temp_directory, "Legacy Series")
|
||||||
|
os.makedirs(folder, exist_ok=True)
|
||||||
|
with open(os.path.join(folder, "key"), "w") as f:
|
||||||
|
f.write("legacy-key")
|
||||||
|
|
||||||
|
scanner = SerieScanner(temp_directory, mock_loader)
|
||||||
|
|
||||||
|
with patch.object(logging.getLogger("src.core.SerieScanner"), "warning") as mock_warning:
|
||||||
|
result = scanner._SerieScanner__read_data_from_file("Legacy Series")
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert result.key == "legacy-key"
|
||||||
|
mock_warning.assert_called()
|
||||||
|
warning_calls = [str(c) for c in mock_warning.call_args_list]
|
||||||
|
assert any("deprecated" in c or "v3.0.0" in c for c in warning_calls)
|
||||||
|
|
||||||
|
def test_db_lookup_exception_caught_and_logged(self, temp_directory, mock_loader):
|
||||||
|
"""DB exception -> fallback to provider callback."""
|
||||||
|
def bad_lookup(folder):
|
||||||
|
raise RuntimeError("DB connection failed")
|
||||||
|
|
||||||
|
scanner = SerieScanner(temp_directory, mock_loader, db_lookup=bad_lookup)
|
||||||
|
|
||||||
|
with patch.object(logging.getLogger("src.core.SerieScanner"), "warning") as mock_warning:
|
||||||
|
result = scanner._SerieScanner__read_data_from_file("Rooster Fighter (2026)")
|
||||||
|
mock_warning.assert_called()
|
||||||
|
assert any("DB lookup failed" in str(c) for c in mock_warning.call_args_list)
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetSerieFromFolderEdgeCases:
|
||||||
|
"""Edge case tests for __read_data_from_file."""
|
||||||
|
|
||||||
|
def test_empty_folder_name_returns_none(self, temp_directory, mock_loader):
|
||||||
|
"""Empty folder name -> returns None (no DB lookup attempted)."""
|
||||||
|
scanner = SerieScanner(temp_directory, mock_loader)
|
||||||
|
result = scanner._SerieScanner__read_data_from_file("")
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_nonexistent_folder_no_exception(self, temp_directory, mock_loader):
|
||||||
|
"""Folder doesn't exist -> returns None without raising."""
|
||||||
|
scanner = SerieScanner(temp_directory, mock_loader)
|
||||||
|
result = scanner._SerieScanner__read_data_from_file("Nonexistent Folder")
|
||||||
|
assert result is None
|
||||||
Reference in New Issue
Block a user