lukas.pupkalipinski/Aniworld
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove legacy key file support after DB migration

Browse Source 
- SerieScanner: Remove key file fallback, keep data file fallback
- SystemSettings: Add legacy_key_cleanup_completed flag
- initialization_service: Add cleanup task to remove key files from folders with DB entries
- Tests updated to reflect key file removal from legacy path

Key files caused duplicate key errors on folder rename. DB is now sole source of truth.
This commit is contained in:
Lukas
2026-05-28 22:01:37 +02:00
parent 1ef59c5283
commit 4e6afa31b5
7 changed files with 161 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
src/core/SerieScanner.py
View File

@@ -618,7 +618,8 @@ class SerieScanner:
1. Query DB by folder name
2. If found, return cached Serie object
3. If not in DB, fall back to provider search via _db_lookup callback
4. (Legacy) If still not found, try reading 'key' file as last resort
4. If still not found, try reading 'data' file for legacy deployments
5. Generate key from folder name as last resort
Args:
folder_name: Filesystem folder name
@@ -627,9 +628,8 @@ class SerieScanner:
Serie object with valid key if found, None otherwise
Note:
DB is the source of truth. File-based lookups (key/data files)
are temporary backward compatibility for deployments with old data.
Will be removed in v3.0.0.
DB is the source of truth. File-based lookups (data files)
are temporary backward compatibility for CLI-only deployments.
"""
# Step 1: Try DB lookup by folder name
try:
@@ -680,25 +680,8 @@ class SerieScanner:
exc
)
# Step 3: Legacy fallback - TEMPORARY (remove in v3.0.0)
# Step 3: Legacy data file fallback (CLI-only deployments)
folder_path = os.path.join(self.directory, folder_name)
key_file = os.path.join(folder_path, 'key')
if os.path.exists(key_file):
logger.warning(
"Using legacy 'key' file for '%s' - this fallback is deprecated "
"and will be removed in v3.0.0",
folder_name
)
with open(key_file, 'r', encoding='utf-8') as file:
key = file.read().strip()
logger.info(
"Key found for folder '%s': %s",
folder_name,
key
)
year_from_folder = self._extract_year_from_folder_name(folder_name)
return Serie(key, "", "aniworld.to", folder_name, dict(), year=year_from_folder)
serie_file = os.path.join(folder_path, 'data')
if os.path.exists(serie_file):
with open(serie_file, "rb") as file:

4
src/server/database/models.py
View File

@@ -617,6 +617,10 @@ class SystemSettings(Base, TimestampMixin):
Boolean, nullable=False, default=False, server_default="0",
doc="Whether legacy key/data file migration has been completed"
)
legacy_key_cleanup_completed: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default="0",
doc="Whether legacy key file cleanup has been completed"
)
last_scan_timestamp: Mapped[Optional[datetime]] = mapped_column(
DateTime(timezone=True), nullable=True,
doc="Timestamp of the last completed scan"

32
src/server/database/system_settings_service.py
View File

@@ -155,6 +155,36 @@ class SystemSettingsService:
await db.commit()
logger.info("Marked legacy files migration as completed")
@staticmethod
async def is_legacy_key_cleanup_completed(db: AsyncSession) -> bool:
"""Check if legacy key file cleanup has been completed.
Args:
db: Database session
Returns:
True if cleanup is completed, False otherwise
"""
settings = await SystemSettingsService.get_or_create(db)
return settings.legacy_key_cleanup_completed
@staticmethod
async def mark_legacy_key_cleanup_completed(
db: AsyncSession,
timestamp: Optional[datetime] = None
) -> None:
"""Mark the legacy key file cleanup as completed.
Args:
db: Database session
timestamp: Optional timestamp to set, defaults to current time
"""
settings = await SystemSettingsService.get_or_create(db)
settings.legacy_key_cleanup_completed = True
settings.last_scan_timestamp = timestamp or datetime.now(timezone.utc)
await db.commit()
logger.info("Marked legacy key file cleanup as completed")
@staticmethod
async def mark_initial_media_scan_completed(
db: AsyncSession,
@@ -184,6 +214,8 @@ class SystemSettingsService:
settings.initial_scan_completed = False
settings.initial_nfo_scan_completed = False
settings.initial_media_scan_completed = False
settings.migration_legacy_files_completed = False
settings.legacy_key_cleanup_completed = False
settings.last_scan_timestamp = None
await db.commit()
logger.info("Reset all scan completion flags")

102
src/server/services/initialization_service.py
View File

@@ -1,5 +1,6 @@
"""Centralized initialization service for application startup and setup."""
import asyncio
import os
from pathlib import Path
from typing import Callable, Optional
@@ -122,6 +123,28 @@ async def _mark_legacy_migration_completed() -> None:
)
async def _check_legacy_key_cleanup_status() -> bool:
"""Check if legacy key file cleanup has been completed.
Returns:
bool: True if cleanup was completed, False otherwise
"""
return await _check_scan_status(
check_method=lambda svc, db: svc.is_legacy_key_cleanup_completed(db),
scan_type="legacy_key_cleanup",
log_completed_msg="Legacy key file cleanup already completed, skipping",
log_not_completed_msg="Legacy key file cleanup not yet run, will clean up key files"
)
async def _mark_legacy_key_cleanup_completed() -> None:
"""Mark the legacy key file cleanup as completed in system settings."""
await _mark_scan_completed(
mark_method=lambda svc, db: svc.mark_legacy_key_cleanup_completed(db),
scan_type="legacy_key_cleanup"
)
async def _migrate_legacy_files() -> int:
"""Migrate series from legacy key/data files to database.
@@ -151,6 +174,78 @@ async def _migrate_legacy_files() -> int:
return 0
async def _cleanup_legacy_key_files() -> int:
"""Remove legacy key files from folders that already have DB entries.
This is a one-time cleanup task that runs at startup after legacy migration.
It removes deprecated 'key' files that cause duplicate key errors when
folders are renamed, since the DB is now the source of truth.
Returns:
int: Number of key files deleted
"""
from src.server.database.connection import get_db_session
from src.server.database.service import AnimeSeriesService
logger.info("Checking for legacy key files to clean up...")
if not settings.anime_directory or not os.path.isdir(settings.anime_directory):
logger.warning(
"Anime directory not configured or does not exist, skipping legacy key cleanup"
)
return 0
deleted_count = 0
scanned_count = 0
try:
async with get_db_session() as db:
# Get all series from DB to know which folders should have key files removed
all_series = await AnimeSeriesService.get_all(db)
# Build a set of known folder names from DB
db_folders: set[str] = {series.folder for series in all_series if series.folder}
for folder_name in db_folders:
folder_path = settings.anime_directory / folder_name
key_file = folder_path / "key"
if not key_file.exists():
continue
scanned_count += 1
try:
key_file.unlink()
deleted_count += 1
logger.info(
"Removed legacy key file",
folder=folder_name,
key_file=str(key_file)
)
except OSError as exc:
logger.warning(
"Could not remove legacy key file",
folder=folder_name,
key_file=str(key_file),
error=str(exc)
)
except Exception as e:
logger.error(
"Legacy key file cleanup failed",
error=str(e),
exc_info=True
)
return deleted_count
logger.info(
"Legacy key file cleanup complete",
scanned=scanned_count,
deleted=deleted_count
)
return deleted_count
async def _sync_anime_folders(progress_service=None) -> int:
"""Scan anime folders and sync series to database.
@@ -287,6 +382,13 @@ async def perform_initial_setup(progress_service=None):
# Sync series from anime folders to database
await _sync_anime_folders(progress_service)
# Clean up legacy key files from folders that now have DB entries
# This runs after migration/sync to ensure DB entries exist before deletion
is_key_cleanup_done = await _check_legacy_key_cleanup_status()
if not is_key_cleanup_done:
await _cleanup_legacy_key_files()
await _mark_legacy_key_cleanup_completed()
# Mark the initial scan as completed
await _mark_initial_scan_completed()

17
tests/unit/test_serie_scanner.py
View File

@@ -519,23 +519,8 @@ class TestFindMp4Files:
class TestReadDataFromFile:
"""Test __read_data_from_file method."""
def test_reads_key_file(self, mock_loader):
"""Should read key from 'key' file."""
import tempfile
with tempfile.TemporaryDirectory() as tmpdir:
anime_folder = os.path.join(tmpdir, "SomeAnime")
os.makedirs(anime_folder)
with open(os.path.join(anime_folder, "key"), "w") as f:
f.write("some-key")
scanner = SerieScanner(tmpdir, mock_loader)
result = scanner._SerieScanner__read_data_from_file("SomeAnime")
assert result is not None
assert result.key == "some-key"
def test_reads_data_file(self, mock_loader):
"""Should read Serie from 'data' file when no 'key' file."""
"""Should read Serie from 'data' file when no DB entry exists."""
import tempfile
with tempfile.TemporaryDirectory() as tmpdir:

18
tests/unit/test_serie_scanner_db_lookup.py
View File

@@ -77,23 +77,19 @@ class TestGetSerieFromFolderDbLookup:
assert result.key == "rooster-fighter"
lookup.assert_called_once_with("Rooster Fighter (2026)")
def test_legacy_key_file_as_last_resort(self, temp_directory, mock_loader):
"""No DB, no callback -> legacy 'key' file used with deprecation warning."""
def test_no_db_no_callback_generates_key_from_folder_name(self, temp_directory, mock_loader):
"""No DB entry, no callback -> key generated from folder name."""
folder = os.path.join(temp_directory, "Legacy Series")
os.makedirs(folder, exist_ok=True)
with open(os.path.join(folder, "key"), "w") as f:
f.write("legacy-key")
# No key file, no data file - should fall through to Step 4 (key generation)
scanner = SerieScanner(temp_directory, mock_loader)
with patch.object(logging.getLogger("src.core.SerieScanner"), "warning") as mock_warning:
result = scanner._SerieScanner__read_data_from_file("Legacy Series")
result = scanner._SerieScanner__read_data_from_file("Legacy Series")
assert result is not None
assert result.key == "legacy-key"
mock_warning.assert_called()
warning_calls = [str(c) for c in mock_warning.call_args_list]
assert any("deprecated" in c or "v3.0.0" in c for c in warning_calls)
assert result is not None
assert result.key == "legacy-series"
assert result.folder == "Legacy Series"
def test_db_lookup_exception_caught_and_logged(self, temp_directory, mock_loader):
"""DB exception -> fallback to provider callback."""

10
tests/unit/test_system_settings_service.py
View File

@@ -23,6 +23,8 @@ async def test_system_settings_integration():
assert settings.initial_scan_completed is False
assert settings.initial_nfo_scan_completed is False
assert settings.initial_media_scan_completed is False
assert settings.migration_legacy_files_completed is False
assert settings.legacy_key_cleanup_completed is False
# Test checking individual flags
async with get_db_session() as db:
@@ -34,6 +36,12 @@ async def test_system_settings_integration():
is_media_done = await SystemSettingsService.is_initial_media_scan_completed(db)
assert is_media_done is False
is_migration_done = await SystemSettingsService.is_migration_legacy_files_completed(db)
assert is_migration_done is False
is_key_cleanup_done = await SystemSettingsService.is_legacy_key_cleanup_completed(db)
assert is_key_cleanup_done is False
# Test marking scans as completed
async with get_db_session() as db:
@@ -56,6 +64,8 @@ async def test_system_settings_integration():
assert settings.initial_scan_completed is False
assert settings.initial_nfo_scan_completed is False
assert settings.initial_media_scan_completed is False
assert settings.migration_legacy_files_completed is False
assert settings.legacy_key_cleanup_completed is False
if __name__ == "__main__":