From 4e6afa31b597ac63952a23473943dd2b1d2c6df6 Mon Sep 17 00:00:00 2001 From: Lukas Date: Thu, 28 May 2026 22:01:37 +0200 Subject: [PATCH] Remove legacy key file support after DB migration - SerieScanner: Remove key file fallback, keep data file fallback - SystemSettings: Add legacy_key_cleanup_completed flag - initialization_service: Add cleanup task to remove key files from folders with DB entries - Tests updated to reflect key file removal from legacy path Key files caused duplicate key errors on folder rename. DB is now sole source of truth. --- src/core/SerieScanner.py | 27 +---- src/server/database/models.py | 4 + .../database/system_settings_service.py | 32 ++++++ src/server/services/initialization_service.py | 102 ++++++++++++++++++ tests/unit/test_serie_scanner.py | 17 +-- tests/unit/test_serie_scanner_db_lookup.py | 18 ++-- tests/unit/test_system_settings_service.py | 10 ++ 7 files changed, 161 insertions(+), 49 deletions(-) diff --git a/src/core/SerieScanner.py b/src/core/SerieScanner.py index 6e45d90..b4d3d12 100644 --- a/src/core/SerieScanner.py +++ b/src/core/SerieScanner.py @@ -618,7 +618,8 @@ class SerieScanner: 1. Query DB by folder name 2. If found, return cached Serie object 3. If not in DB, fall back to provider search via _db_lookup callback - 4. (Legacy) If still not found, try reading 'key' file as last resort + 4. If still not found, try reading 'data' file for legacy deployments + 5. Generate key from folder name as last resort Args: folder_name: Filesystem folder name @@ -627,9 +628,8 @@ class SerieScanner: Serie object with valid key if found, None otherwise Note: - DB is the source of truth. File-based lookups (key/data files) - are temporary backward compatibility for deployments with old data. - Will be removed in v3.0.0. + DB is the source of truth. File-based lookups (data files) + are temporary backward compatibility for CLI-only deployments. """ # Step 1: Try DB lookup by folder name try: @@ -680,25 +680,8 @@ class SerieScanner: exc ) - # Step 3: Legacy fallback - TEMPORARY (remove in v3.0.0) + # Step 3: Legacy data file fallback (CLI-only deployments) folder_path = os.path.join(self.directory, folder_name) - key_file = os.path.join(folder_path, 'key') - if os.path.exists(key_file): - logger.warning( - "Using legacy 'key' file for '%s' - this fallback is deprecated " - "and will be removed in v3.0.0", - folder_name - ) - with open(key_file, 'r', encoding='utf-8') as file: - key = file.read().strip() - logger.info( - "Key found for folder '%s': %s", - folder_name, - key - ) - year_from_folder = self._extract_year_from_folder_name(folder_name) - return Serie(key, "", "aniworld.to", folder_name, dict(), year=year_from_folder) - serie_file = os.path.join(folder_path, 'data') if os.path.exists(serie_file): with open(serie_file, "rb") as file: diff --git a/src/server/database/models.py b/src/server/database/models.py index 58f43e7..12df1d0 100644 --- a/src/server/database/models.py +++ b/src/server/database/models.py @@ -617,6 +617,10 @@ class SystemSettings(Base, TimestampMixin): Boolean, nullable=False, default=False, server_default="0", doc="Whether legacy key/data file migration has been completed" ) + legacy_key_cleanup_completed: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=False, server_default="0", + doc="Whether legacy key file cleanup has been completed" + ) last_scan_timestamp: Mapped[Optional[datetime]] = mapped_column( DateTime(timezone=True), nullable=True, doc="Timestamp of the last completed scan" diff --git a/src/server/database/system_settings_service.py b/src/server/database/system_settings_service.py index c6e1c06..330dfb3 100644 --- a/src/server/database/system_settings_service.py +++ b/src/server/database/system_settings_service.py @@ -155,6 +155,36 @@ class SystemSettingsService: await db.commit() logger.info("Marked legacy files migration as completed") + @staticmethod + async def is_legacy_key_cleanup_completed(db: AsyncSession) -> bool: + """Check if legacy key file cleanup has been completed. + + Args: + db: Database session + + Returns: + True if cleanup is completed, False otherwise + """ + settings = await SystemSettingsService.get_or_create(db) + return settings.legacy_key_cleanup_completed + + @staticmethod + async def mark_legacy_key_cleanup_completed( + db: AsyncSession, + timestamp: Optional[datetime] = None + ) -> None: + """Mark the legacy key file cleanup as completed. + + Args: + db: Database session + timestamp: Optional timestamp to set, defaults to current time + """ + settings = await SystemSettingsService.get_or_create(db) + settings.legacy_key_cleanup_completed = True + settings.last_scan_timestamp = timestamp or datetime.now(timezone.utc) + await db.commit() + logger.info("Marked legacy key file cleanup as completed") + @staticmethod async def mark_initial_media_scan_completed( db: AsyncSession, @@ -184,6 +214,8 @@ class SystemSettingsService: settings.initial_scan_completed = False settings.initial_nfo_scan_completed = False settings.initial_media_scan_completed = False + settings.migration_legacy_files_completed = False + settings.legacy_key_cleanup_completed = False settings.last_scan_timestamp = None await db.commit() logger.info("Reset all scan completion flags") diff --git a/src/server/services/initialization_service.py b/src/server/services/initialization_service.py index 566e967..40e30ba 100644 --- a/src/server/services/initialization_service.py +++ b/src/server/services/initialization_service.py @@ -1,5 +1,6 @@ """Centralized initialization service for application startup and setup.""" import asyncio +import os from pathlib import Path from typing import Callable, Optional @@ -122,6 +123,28 @@ async def _mark_legacy_migration_completed() -> None: ) +async def _check_legacy_key_cleanup_status() -> bool: + """Check if legacy key file cleanup has been completed. + + Returns: + bool: True if cleanup was completed, False otherwise + """ + return await _check_scan_status( + check_method=lambda svc, db: svc.is_legacy_key_cleanup_completed(db), + scan_type="legacy_key_cleanup", + log_completed_msg="Legacy key file cleanup already completed, skipping", + log_not_completed_msg="Legacy key file cleanup not yet run, will clean up key files" + ) + + +async def _mark_legacy_key_cleanup_completed() -> None: + """Mark the legacy key file cleanup as completed in system settings.""" + await _mark_scan_completed( + mark_method=lambda svc, db: svc.mark_legacy_key_cleanup_completed(db), + scan_type="legacy_key_cleanup" + ) + + async def _migrate_legacy_files() -> int: """Migrate series from legacy key/data files to database. @@ -151,6 +174,78 @@ async def _migrate_legacy_files() -> int: return 0 +async def _cleanup_legacy_key_files() -> int: + """Remove legacy key files from folders that already have DB entries. + + This is a one-time cleanup task that runs at startup after legacy migration. + It removes deprecated 'key' files that cause duplicate key errors when + folders are renamed, since the DB is now the source of truth. + + Returns: + int: Number of key files deleted + """ + from src.server.database.connection import get_db_session + from src.server.database.service import AnimeSeriesService + + logger.info("Checking for legacy key files to clean up...") + + if not settings.anime_directory or not os.path.isdir(settings.anime_directory): + logger.warning( + "Anime directory not configured or does not exist, skipping legacy key cleanup" + ) + return 0 + + deleted_count = 0 + scanned_count = 0 + + try: + async with get_db_session() as db: + # Get all series from DB to know which folders should have key files removed + all_series = await AnimeSeriesService.get_all(db) + + # Build a set of known folder names from DB + db_folders: set[str] = {series.folder for series in all_series if series.folder} + + for folder_name in db_folders: + folder_path = settings.anime_directory / folder_name + key_file = folder_path / "key" + + if not key_file.exists(): + continue + + scanned_count += 1 + try: + key_file.unlink() + deleted_count += 1 + logger.info( + "Removed legacy key file", + folder=folder_name, + key_file=str(key_file) + ) + except OSError as exc: + logger.warning( + "Could not remove legacy key file", + folder=folder_name, + key_file=str(key_file), + error=str(exc) + ) + + except Exception as e: + logger.error( + "Legacy key file cleanup failed", + error=str(e), + exc_info=True + ) + return deleted_count + + logger.info( + "Legacy key file cleanup complete", + scanned=scanned_count, + deleted=deleted_count + ) + return deleted_count + + async def _sync_anime_folders(progress_service=None) -> int: """Scan anime folders and sync series to database. @@ -287,6 +382,13 @@ async def perform_initial_setup(progress_service=None): # Sync series from anime folders to database await _sync_anime_folders(progress_service) + # Clean up legacy key files from folders that now have DB entries + # This runs after migration/sync to ensure DB entries exist before deletion + is_key_cleanup_done = await _check_legacy_key_cleanup_status() + if not is_key_cleanup_done: + await _cleanup_legacy_key_files() + await _mark_legacy_key_cleanup_completed() + # Mark the initial scan as completed await _mark_initial_scan_completed() diff --git a/tests/unit/test_serie_scanner.py b/tests/unit/test_serie_scanner.py index 2ad5dff..56c1530 100644 --- a/tests/unit/test_serie_scanner.py +++ b/tests/unit/test_serie_scanner.py @@ -519,23 +519,8 @@ class TestFindMp4Files: class TestReadDataFromFile: """Test __read_data_from_file method.""" - def test_reads_key_file(self, mock_loader): - """Should read key from 'key' file.""" - import tempfile - - with tempfile.TemporaryDirectory() as tmpdir: - anime_folder = os.path.join(tmpdir, "SomeAnime") - os.makedirs(anime_folder) - with open(os.path.join(anime_folder, "key"), "w") as f: - f.write("some-key") - - scanner = SerieScanner(tmpdir, mock_loader) - result = scanner._SerieScanner__read_data_from_file("SomeAnime") - assert result is not None - assert result.key == "some-key" - def test_reads_data_file(self, mock_loader): - """Should read Serie from 'data' file when no 'key' file.""" + """Should read Serie from 'data' file when no DB entry exists.""" import tempfile with tempfile.TemporaryDirectory() as tmpdir: diff --git a/tests/unit/test_serie_scanner_db_lookup.py b/tests/unit/test_serie_scanner_db_lookup.py index be7d44a..8cf44a0 100644 --- a/tests/unit/test_serie_scanner_db_lookup.py +++ b/tests/unit/test_serie_scanner_db_lookup.py @@ -77,23 +77,19 @@ class TestGetSerieFromFolderDbLookup: assert result.key == "rooster-fighter" lookup.assert_called_once_with("Rooster Fighter (2026)") - def test_legacy_key_file_as_last_resort(self, temp_directory, mock_loader): - """No DB, no callback -> legacy 'key' file used with deprecation warning.""" + def test_no_db_no_callback_generates_key_from_folder_name(self, temp_directory, mock_loader): + """No DB entry, no callback -> key generated from folder name.""" folder = os.path.join(temp_directory, "Legacy Series") os.makedirs(folder, exist_ok=True) - with open(os.path.join(folder, "key"), "w") as f: - f.write("legacy-key") + # No key file, no data file - should fall through to Step 4 (key generation) scanner = SerieScanner(temp_directory, mock_loader) - with patch.object(logging.getLogger("src.core.SerieScanner"), "warning") as mock_warning: - result = scanner._SerieScanner__read_data_from_file("Legacy Series") + result = scanner._SerieScanner__read_data_from_file("Legacy Series") - assert result is not None - assert result.key == "legacy-key" - mock_warning.assert_called() - warning_calls = [str(c) for c in mock_warning.call_args_list] - assert any("deprecated" in c or "v3.0.0" in c for c in warning_calls) + assert result is not None + assert result.key == "legacy-series" + assert result.folder == "Legacy Series" def test_db_lookup_exception_caught_and_logged(self, temp_directory, mock_loader): """DB exception -> fallback to provider callback.""" diff --git a/tests/unit/test_system_settings_service.py b/tests/unit/test_system_settings_service.py index ce93857..6d71d53 100644 --- a/tests/unit/test_system_settings_service.py +++ b/tests/unit/test_system_settings_service.py @@ -23,6 +23,8 @@ async def test_system_settings_integration(): assert settings.initial_scan_completed is False assert settings.initial_nfo_scan_completed is False assert settings.initial_media_scan_completed is False + assert settings.migration_legacy_files_completed is False + assert settings.legacy_key_cleanup_completed is False # Test checking individual flags async with get_db_session() as db: @@ -34,6 +36,12 @@ async def test_system_settings_integration(): is_media_done = await SystemSettingsService.is_initial_media_scan_completed(db) assert is_media_done is False + + is_migration_done = await SystemSettingsService.is_migration_legacy_files_completed(db) + assert is_migration_done is False + + is_key_cleanup_done = await SystemSettingsService.is_legacy_key_cleanup_completed(db) + assert is_key_cleanup_done is False # Test marking scans as completed async with get_db_session() as db: @@ -56,6 +64,8 @@ async def test_system_settings_integration(): assert settings.initial_scan_completed is False assert settings.initial_nfo_scan_completed is False assert settings.initial_media_scan_completed is False + assert settings.migration_legacy_files_completed is False + assert settings.legacy_key_cleanup_completed is False if __name__ == "__main__":