migration removed
This commit is contained in:
@@ -1,436 +0,0 @@
|
||||
"""Data migration service for migrating file-based storage to database.
|
||||
|
||||
This module provides functionality to migrate anime series data from
|
||||
legacy file-based storage (data files without .json extension) to the
|
||||
SQLite database using the AnimeSeries model.
|
||||
|
||||
The migration service:
|
||||
- Scans anime directories for existing data files
|
||||
- Reads Serie objects from data files
|
||||
- Migrates them to the database using AnimeSeriesService
|
||||
- Handles errors gracefully without stopping the migration
|
||||
- Provides detailed migration results
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.core.entities.series import Serie
|
||||
from src.server.database.service import AnimeSeriesService, EpisodeService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MigrationResult:
|
||||
"""Result of a data file migration operation.
|
||||
|
||||
Attributes:
|
||||
total_found: Total number of data files found
|
||||
migrated: Number of files successfully migrated
|
||||
skipped: Number of files skipped (already in database)
|
||||
failed: Number of files that failed to migrate
|
||||
errors: List of error messages encountered
|
||||
"""
|
||||
total_found: int = 0
|
||||
migrated: int = 0
|
||||
skipped: int = 0
|
||||
failed: int = 0
|
||||
errors: List[str] = field(default_factory=list)
|
||||
|
||||
def __post_init__(self):
|
||||
"""Ensure errors is always a list."""
|
||||
if self.errors is None:
|
||||
self.errors = []
|
||||
|
||||
|
||||
class DataMigrationError(Exception):
|
||||
"""Base exception for data migration errors."""
|
||||
|
||||
|
||||
class DataFileReadError(DataMigrationError):
|
||||
"""Raised when a data file cannot be read."""
|
||||
|
||||
|
||||
class DataMigrationService:
|
||||
"""Service for migrating data files to database.
|
||||
|
||||
This service handles the migration of anime series data from
|
||||
file-based storage to the database. It scans directories for
|
||||
data files, reads Serie objects, and creates AnimeSeries records.
|
||||
|
||||
Example:
|
||||
```python
|
||||
service = DataMigrationService()
|
||||
|
||||
# Check if migration is needed
|
||||
if await service.is_migration_needed("/path/to/anime"):
|
||||
async with get_db_session() as db:
|
||||
result = await service.migrate_all("/path/to/anime", db)
|
||||
print(f"Migrated {result.migrated} series")
|
||||
```
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialize the data migration service."""
|
||||
pass
|
||||
|
||||
def scan_for_data_files(self, anime_directory: str) -> List[Path]:
|
||||
"""Scan for data files in the anime directory.
|
||||
|
||||
Finds all 'data' files (JSON format without extension) in
|
||||
the anime directory structure. Each series folder may contain
|
||||
a 'data' file with series metadata.
|
||||
|
||||
Args:
|
||||
anime_directory: Path to the anime directory containing
|
||||
series folders
|
||||
|
||||
Returns:
|
||||
List of Path objects pointing to data files
|
||||
|
||||
Raises:
|
||||
ValueError: If anime_directory is invalid
|
||||
"""
|
||||
if not anime_directory or not anime_directory.strip():
|
||||
logger.warning("Empty anime directory provided")
|
||||
return []
|
||||
|
||||
base_path = Path(anime_directory)
|
||||
|
||||
if not base_path.exists():
|
||||
logger.warning(
|
||||
"Anime directory does not exist: %s",
|
||||
anime_directory
|
||||
)
|
||||
return []
|
||||
|
||||
if not base_path.is_dir():
|
||||
logger.warning(
|
||||
"Anime directory is not a directory: %s",
|
||||
anime_directory
|
||||
)
|
||||
return []
|
||||
|
||||
data_files: List[Path] = []
|
||||
|
||||
try:
|
||||
# Iterate through all subdirectories (series folders)
|
||||
for folder in base_path.iterdir():
|
||||
if not folder.is_dir():
|
||||
continue
|
||||
|
||||
# Check for 'data' file in each series folder
|
||||
data_file = folder / "data"
|
||||
if data_file.exists() and data_file.is_file():
|
||||
data_files.append(data_file)
|
||||
logger.debug("Found data file: %s", data_file)
|
||||
|
||||
except PermissionError as e:
|
||||
logger.error(
|
||||
"Permission denied scanning directory %s: %s",
|
||||
anime_directory,
|
||||
e
|
||||
)
|
||||
except OSError as e:
|
||||
logger.error(
|
||||
"OS error scanning directory %s: %s",
|
||||
anime_directory,
|
||||
e
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Found %d data files in %s",
|
||||
len(data_files),
|
||||
anime_directory
|
||||
)
|
||||
return data_files
|
||||
|
||||
def _read_data_file(self, data_path: Path) -> Optional[Serie]:
|
||||
"""Read a Serie object from a data file.
|
||||
|
||||
Args:
|
||||
data_path: Path to the data file
|
||||
|
||||
Returns:
|
||||
Serie object if successfully read, None otherwise
|
||||
|
||||
Raises:
|
||||
DataFileReadError: If the file cannot be read or parsed
|
||||
"""
|
||||
try:
|
||||
serie = Serie.load_from_file(str(data_path))
|
||||
|
||||
# Validate the serie has required fields
|
||||
if not serie.key or not serie.key.strip():
|
||||
raise DataFileReadError(
|
||||
f"Data file {data_path} has empty or missing key"
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"Successfully read serie '%s' from %s",
|
||||
serie.key,
|
||||
data_path
|
||||
)
|
||||
return serie
|
||||
|
||||
except FileNotFoundError as e:
|
||||
raise DataFileReadError(
|
||||
f"Data file not found: {data_path}"
|
||||
) from e
|
||||
except PermissionError as e:
|
||||
raise DataFileReadError(
|
||||
f"Permission denied reading data file: {data_path}"
|
||||
) from e
|
||||
except (ValueError, KeyError, TypeError) as e:
|
||||
raise DataFileReadError(
|
||||
f"Invalid data in file {data_path}: {e}"
|
||||
) from e
|
||||
except Exception as e:
|
||||
raise DataFileReadError(
|
||||
f"Error reading data file {data_path}: {e}"
|
||||
) from e
|
||||
|
||||
async def migrate_data_file(
|
||||
self,
|
||||
data_path: Path,
|
||||
db: AsyncSession
|
||||
) -> bool:
|
||||
"""Migrate a single data file to the database.
|
||||
|
||||
Reads the data file, checks if the series already exists in the
|
||||
database, and creates a new record if it doesn't exist. If the
|
||||
series exists, optionally updates the episodes if changed.
|
||||
|
||||
Args:
|
||||
data_path: Path to the data file
|
||||
db: Async database session
|
||||
|
||||
Returns:
|
||||
True if the series was migrated (created or updated),
|
||||
False if skipped (already exists with same data)
|
||||
|
||||
Raises:
|
||||
DataFileReadError: If the file cannot be read
|
||||
DataMigrationError: If database operation fails
|
||||
"""
|
||||
# Read the data file
|
||||
serie = self._read_data_file(data_path)
|
||||
if serie is None:
|
||||
raise DataFileReadError(f"Could not read data file: {data_path}")
|
||||
|
||||
# Check if series already exists in database
|
||||
existing = await AnimeSeriesService.get_by_key(db, serie.key)
|
||||
|
||||
if existing is not None:
|
||||
# Build episode dict from existing episodes for comparison
|
||||
existing_dict: dict[int, list[int]] = {}
|
||||
episodes = await EpisodeService.get_by_series(db, existing.id)
|
||||
for ep in episodes:
|
||||
if ep.season not in existing_dict:
|
||||
existing_dict[ep.season] = []
|
||||
existing_dict[ep.season].append(ep.episode_number)
|
||||
for season in existing_dict:
|
||||
existing_dict[season].sort()
|
||||
|
||||
new_dict = serie.episodeDict or {}
|
||||
|
||||
if existing_dict == new_dict:
|
||||
logger.debug(
|
||||
"Series '%s' already exists with same data, skipping",
|
||||
serie.key
|
||||
)
|
||||
return False
|
||||
|
||||
# Update episodes if different - add new episodes
|
||||
for season, episode_numbers in new_dict.items():
|
||||
existing_eps = set(existing_dict.get(season, []))
|
||||
for ep_num in episode_numbers:
|
||||
if ep_num not in existing_eps:
|
||||
await EpisodeService.create(
|
||||
db=db,
|
||||
series_id=existing.id,
|
||||
season=season,
|
||||
episode_number=ep_num,
|
||||
)
|
||||
logger.info(
|
||||
"Updated episodes for existing series '%s'",
|
||||
serie.key
|
||||
)
|
||||
return True
|
||||
|
||||
# Create new series in database
|
||||
try:
|
||||
# Use folder as fallback name if name is empty
|
||||
series_name = serie.name
|
||||
if not series_name or not series_name.strip():
|
||||
series_name = serie.folder
|
||||
logger.debug(
|
||||
"Using folder '%s' as name for series '%s'",
|
||||
series_name,
|
||||
serie.key
|
||||
)
|
||||
|
||||
anime_series = await AnimeSeriesService.create(
|
||||
db,
|
||||
key=serie.key,
|
||||
name=series_name,
|
||||
site=serie.site,
|
||||
folder=serie.folder,
|
||||
)
|
||||
|
||||
# Create Episode records for each episode in episodeDict
|
||||
if serie.episodeDict:
|
||||
for season, episode_numbers in serie.episodeDict.items():
|
||||
for episode_number in episode_numbers:
|
||||
await EpisodeService.create(
|
||||
db=db,
|
||||
series_id=anime_series.id,
|
||||
season=season,
|
||||
episode_number=episode_number,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Migrated series '%s' to database",
|
||||
serie.key
|
||||
)
|
||||
return True
|
||||
|
||||
except IntegrityError as e:
|
||||
# Race condition - series was created by another process
|
||||
logger.warning(
|
||||
"Series '%s' was already created (race condition): %s",
|
||||
serie.key,
|
||||
e
|
||||
)
|
||||
return False
|
||||
except Exception as e:
|
||||
raise DataMigrationError(
|
||||
f"Failed to create series '{serie.key}' in database: {e}"
|
||||
) from e
|
||||
|
||||
async def migrate_all(
|
||||
self,
|
||||
anime_directory: str,
|
||||
db: AsyncSession
|
||||
) -> MigrationResult:
|
||||
"""Migrate all data files from anime directory to database.
|
||||
|
||||
Scans the anime directory for data files and migrates each one
|
||||
to the database. Errors are logged but do not stop the migration.
|
||||
|
||||
Args:
|
||||
anime_directory: Path to the anime directory
|
||||
db: Async database session
|
||||
|
||||
Returns:
|
||||
MigrationResult with counts and error messages
|
||||
"""
|
||||
result = MigrationResult()
|
||||
|
||||
# Scan for data files
|
||||
data_files = self.scan_for_data_files(anime_directory)
|
||||
result.total_found = len(data_files)
|
||||
|
||||
if result.total_found == 0:
|
||||
logger.info("No data files found to migrate")
|
||||
return result
|
||||
|
||||
logger.info(
|
||||
"Starting migration of %d data files",
|
||||
result.total_found
|
||||
)
|
||||
|
||||
# Migrate each file
|
||||
for data_path in data_files:
|
||||
try:
|
||||
migrated = await self.migrate_data_file(data_path, db)
|
||||
|
||||
if migrated:
|
||||
result.migrated += 1
|
||||
else:
|
||||
result.skipped += 1
|
||||
|
||||
except DataFileReadError as e:
|
||||
result.failed += 1
|
||||
error_msg = f"Failed to read {data_path}: {e}"
|
||||
result.errors.append(error_msg)
|
||||
logger.error(error_msg)
|
||||
|
||||
except DataMigrationError as e:
|
||||
result.failed += 1
|
||||
error_msg = f"Failed to migrate {data_path}: {e}"
|
||||
result.errors.append(error_msg)
|
||||
logger.error(error_msg)
|
||||
|
||||
except Exception as e:
|
||||
result.failed += 1
|
||||
error_msg = f"Unexpected error migrating {data_path}: {e}"
|
||||
result.errors.append(error_msg)
|
||||
logger.exception(error_msg)
|
||||
|
||||
# Commit all changes
|
||||
try:
|
||||
await db.commit()
|
||||
except Exception as e:
|
||||
logger.error("Failed to commit migration: %s", e)
|
||||
result.errors.append(f"Failed to commit migration: {e}")
|
||||
|
||||
logger.info(
|
||||
"Migration complete: %d migrated, %d skipped, %d failed",
|
||||
result.migrated,
|
||||
result.skipped,
|
||||
result.failed
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
def is_migration_needed(self, anime_directory: str) -> bool:
|
||||
"""Check if there are data files to migrate.
|
||||
|
||||
Args:
|
||||
anime_directory: Path to the anime directory
|
||||
|
||||
Returns:
|
||||
True if data files exist, False otherwise
|
||||
"""
|
||||
data_files = self.scan_for_data_files(anime_directory)
|
||||
needs_migration = len(data_files) > 0
|
||||
|
||||
if needs_migration:
|
||||
logger.info(
|
||||
"Migration needed: found %d data files",
|
||||
len(data_files)
|
||||
)
|
||||
else:
|
||||
logger.debug("No migration needed: no data files found")
|
||||
|
||||
return needs_migration
|
||||
|
||||
|
||||
# Singleton instance for the service
|
||||
_data_migration_service: Optional[DataMigrationService] = None
|
||||
|
||||
|
||||
def get_data_migration_service() -> DataMigrationService:
|
||||
"""Get the singleton data migration service instance.
|
||||
|
||||
Returns:
|
||||
DataMigrationService instance
|
||||
"""
|
||||
global _data_migration_service
|
||||
if _data_migration_service is None:
|
||||
_data_migration_service = DataMigrationService()
|
||||
return _data_migration_service
|
||||
|
||||
|
||||
def reset_data_migration_service() -> None:
|
||||
"""Reset the singleton service instance (for testing)."""
|
||||
global _data_migration_service
|
||||
_data_migration_service = None
|
||||
@@ -1,309 +0,0 @@
|
||||
"""Startup migration runner for data file to database migration.
|
||||
|
||||
This module provides functions to run the data file migration automatically
|
||||
during application startup. The migration checks for existing data files
|
||||
in the anime directory and migrates them to the database.
|
||||
|
||||
Usage:
|
||||
This module is intended to be called from the FastAPI lifespan context.
|
||||
|
||||
Example:
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
# ... initialization ...
|
||||
await ensure_migration_on_startup()
|
||||
yield
|
||||
# ... cleanup ...
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from src.server.database.connection import get_db_session
|
||||
from src.server.services.auth_service import auth_service
|
||||
from src.server.services.config_service import ConfigService
|
||||
from src.server.services.data_migration_service import (
|
||||
MigrationResult,
|
||||
get_data_migration_service,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def run_startup_migration(anime_directory: str) -> MigrationResult:
|
||||
"""Run data file migration for the given anime directory.
|
||||
|
||||
Checks if there are data files to migrate and runs the migration
|
||||
if needed. This function is idempotent - running it multiple times
|
||||
will only migrate files that haven't been migrated yet.
|
||||
|
||||
Args:
|
||||
anime_directory: Path to the anime directory containing
|
||||
series folders with data files
|
||||
|
||||
Returns:
|
||||
MigrationResult: Results of the migration operation,
|
||||
including counts of migrated, skipped, and failed items
|
||||
|
||||
Note:
|
||||
This function creates its own database session and commits
|
||||
the transaction at the end of the migration.
|
||||
"""
|
||||
service = get_data_migration_service()
|
||||
|
||||
# Check if migration is needed
|
||||
if not service.is_migration_needed(anime_directory):
|
||||
logger.info(
|
||||
"No data files found to migrate in: %s",
|
||||
anime_directory
|
||||
)
|
||||
return MigrationResult(total_found=0)
|
||||
|
||||
logger.info(
|
||||
"Starting data file migration from: %s",
|
||||
anime_directory
|
||||
)
|
||||
|
||||
# Get database session and run migration
|
||||
async with get_db_session() as db:
|
||||
result = await service.migrate_all(anime_directory, db)
|
||||
|
||||
# Log results
|
||||
if result.migrated > 0 or result.failed > 0:
|
||||
logger.info(
|
||||
"Migration complete: %d migrated, %d skipped, %d failed",
|
||||
result.migrated,
|
||||
result.skipped,
|
||||
result.failed
|
||||
)
|
||||
|
||||
if result.errors:
|
||||
for error in result.errors:
|
||||
logger.warning("Migration error: %s", error)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _get_anime_directory_from_config() -> Optional[str]:
|
||||
"""Get anime directory from application configuration.
|
||||
|
||||
Attempts to load the configuration file and extract the
|
||||
anime_directory setting from the 'other' config section.
|
||||
|
||||
Returns:
|
||||
Anime directory path if configured, None otherwise
|
||||
"""
|
||||
try:
|
||||
config_service = ConfigService()
|
||||
config = config_service.load_config()
|
||||
|
||||
# anime_directory is stored in the 'other' dict
|
||||
anime_dir = config.other.get("anime_directory")
|
||||
|
||||
if anime_dir:
|
||||
anime_dir = str(anime_dir).strip()
|
||||
if anime_dir:
|
||||
return anime_dir
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Could not load anime directory from config: %s",
|
||||
e
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _is_setup_complete() -> bool:
|
||||
"""Check if the application setup is complete.
|
||||
|
||||
Setup is complete when:
|
||||
1. Master password is configured
|
||||
2. Configuration file exists and is valid
|
||||
|
||||
Returns:
|
||||
True if setup is complete, False otherwise
|
||||
"""
|
||||
# Check if master password is configured
|
||||
if not auth_service.is_configured():
|
||||
return False
|
||||
|
||||
# Check if config exists and is valid
|
||||
try:
|
||||
config_service = ConfigService()
|
||||
config = config_service.load_config()
|
||||
|
||||
# Validate the loaded config
|
||||
validation = config.validate()
|
||||
if not validation.valid:
|
||||
return False
|
||||
|
||||
except Exception:
|
||||
# If we can't load or validate config, setup is not complete
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
async def ensure_migration_on_startup() -> Optional[MigrationResult]:
|
||||
"""Ensure data file migration runs during application startup.
|
||||
|
||||
This function should be called during FastAPI application startup.
|
||||
It loads the anime directory from configuration and runs the
|
||||
migration if the directory is configured and contains data files.
|
||||
|
||||
Migration will only run if setup is complete (master password
|
||||
configured and valid configuration exists).
|
||||
|
||||
Returns:
|
||||
MigrationResult if migration was run, None if skipped
|
||||
(e.g., when no anime directory is configured)
|
||||
|
||||
Behavior:
|
||||
- Returns None if anime_directory is not configured (first run)
|
||||
- Returns None if anime_directory does not exist
|
||||
- Returns MigrationResult with total_found=0 if no data files exist
|
||||
- Returns MigrationResult with migration counts if migration ran
|
||||
|
||||
Note:
|
||||
This function catches and logs all exceptions without re-raising,
|
||||
ensuring that startup migration failures don't block application
|
||||
startup. Check the logs for any migration errors.
|
||||
|
||||
Example:
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
await init_db()
|
||||
|
||||
try:
|
||||
result = await ensure_migration_on_startup()
|
||||
if result:
|
||||
logger.info(
|
||||
"Migration: %d migrated, %d failed",
|
||||
result.migrated,
|
||||
result.failed
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Migration failed: %s", e)
|
||||
|
||||
yield
|
||||
await close_db()
|
||||
"""
|
||||
# Check if setup is complete before running migration
|
||||
if not _is_setup_complete():
|
||||
logger.debug(
|
||||
"Setup not complete, skipping startup migration"
|
||||
)
|
||||
return None
|
||||
|
||||
# Get anime directory from config
|
||||
anime_directory = _get_anime_directory_from_config()
|
||||
|
||||
if not anime_directory:
|
||||
logger.debug(
|
||||
"No anime directory configured, skipping migration"
|
||||
)
|
||||
return None
|
||||
|
||||
# Validate directory exists
|
||||
anime_path = Path(anime_directory)
|
||||
if not anime_path.exists():
|
||||
logger.warning(
|
||||
"Anime directory does not exist: %s, skipping migration",
|
||||
anime_directory
|
||||
)
|
||||
return None
|
||||
|
||||
if not anime_path.is_dir():
|
||||
logger.warning(
|
||||
"Anime directory path is not a directory: %s, skipping migration",
|
||||
anime_directory
|
||||
)
|
||||
return None
|
||||
|
||||
logger.info(
|
||||
"Checking for data files to migrate in: %s",
|
||||
anime_directory
|
||||
)
|
||||
|
||||
try:
|
||||
result = await run_startup_migration(anime_directory)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Data file migration failed: %s",
|
||||
e,
|
||||
exc_info=True
|
||||
)
|
||||
# Return empty result rather than None to indicate we attempted
|
||||
return MigrationResult(
|
||||
total_found=0,
|
||||
failed=1,
|
||||
errors=[f"Migration failed: {str(e)}"]
|
||||
)
|
||||
|
||||
|
||||
async def run_migration_for_directory(
|
||||
anime_directory: str
|
||||
) -> Optional[MigrationResult]:
|
||||
"""Run data file migration for a specific directory.
|
||||
|
||||
This function can be called after setup is complete to migrate
|
||||
data files from the specified anime directory to the database.
|
||||
Unlike ensure_migration_on_startup, this does not check setup
|
||||
status as it's intended to be called after setup is complete.
|
||||
|
||||
Args:
|
||||
anime_directory: Path to the anime directory containing
|
||||
series folders with data files
|
||||
|
||||
Returns:
|
||||
MigrationResult if migration was run, None if directory invalid
|
||||
"""
|
||||
if not anime_directory or not anime_directory.strip():
|
||||
logger.debug("Empty anime directory provided, skipping migration")
|
||||
return None
|
||||
|
||||
anime_directory = anime_directory.strip()
|
||||
|
||||
# Validate directory exists
|
||||
anime_path = Path(anime_directory)
|
||||
if not anime_path.exists():
|
||||
logger.warning(
|
||||
"Anime directory does not exist: %s, skipping migration",
|
||||
anime_directory
|
||||
)
|
||||
return None
|
||||
|
||||
if not anime_path.is_dir():
|
||||
logger.warning(
|
||||
"Anime directory path is not a directory: %s",
|
||||
anime_directory
|
||||
)
|
||||
return None
|
||||
|
||||
logger.info(
|
||||
"Running migration for directory: %s",
|
||||
anime_directory
|
||||
)
|
||||
|
||||
try:
|
||||
result = await run_startup_migration(anime_directory)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Data file migration failed for %s: %s",
|
||||
anime_directory,
|
||||
e,
|
||||
exc_info=True
|
||||
)
|
||||
return MigrationResult(
|
||||
total_found=0,
|
||||
failed=1,
|
||||
errors=[f"Migration failed: {str(e)}"]
|
||||
)
|
||||
Reference in New Issue
Block a user