From bf3cfa00d5f779055445e49813824a5502ef9a24 Mon Sep 17 00:00:00 2001 From: Lukas Date: Wed, 21 Jan 2026 19:22:50 +0100 Subject: [PATCH] Implement initial scan tracking for one-time setup - Add SystemSettings model to track setup completion status - Create SystemSettingsService for managing setup flags - Modify fastapi_app startup to check and set initial_scan_completed flag - Anime folder scanning now only runs on first startup - Update DATABASE.md with new system_settings table documentation - Add unit test for SystemSettingsService functionality This ensures expensive one-time operations like scanning the entire anime directory only occur during initial setup, not on every application restart. --- docs/DATABASE.md | 67 +++++--- docs/instructions.md | 27 ++- src/server/database/__init__.py | 4 + src/server/database/init.py | 1 + src/server/database/models.py | 49 ++++++ .../database/system_settings_service.py | 159 ++++++++++++++++++ src/server/fastapi_app.py | 63 ++++++- tests/unit/test_system_settings_service.py | 58 +++++++ 8 files changed, 383 insertions(+), 45 deletions(-) create mode 100644 src/server/database/system_settings_service.py create mode 100644 tests/unit/test_system_settings_service.py diff --git a/docs/DATABASE.md b/docs/DATABASE.md index a9852fc..6ea6bea 100644 --- a/docs/DATABASE.md +++ b/docs/DATABASE.md @@ -33,31 +33,55 @@ Source: [src/server/database/connection.py](../src/server/database/connection.py ## 2. Entity Relationship Diagram ``` -+-------------------+ +-------------------+ +------------------------+ -| anime_series | | episodes | | download_queue_item | -+-------------------+ +-------------------+ +------------------------+ -| id (PK) |<--+ | id (PK) | +-->| id (PK, VARCHAR) | -| key (UNIQUE) | | | series_id (FK)----+---+ | series_id (FK)---------+ -| name | +---| | | status | -| site | | season | | priority | -| folder | | episode_number | | season | -| created_at | | title | | episode | -| updated_at | | file_path | | progress_percent | -+-------------------+ | is_downloaded | | error_message | - | created_at | | retry_count | - | updated_at | | added_at | - +-------------------+ | started_at | - | completed_at | - | created_at | - | updated_at | - +------------------------+ ++---------------------+ +-------------------+ +-------------------+ +------------------------+ +| system_settings | | anime_series | | episodes | | download_queue_item | ++---------------------+ +-------------------+ +-------------------+ +------------------------+ +| id (PK) | | id (PK) |<--+ | id (PK) | +-->| id (PK, VARCHAR) | +| initial_scan_... | | key (UNIQUE) | | | series_id (FK)----+---+ | series_id (FK)---------+ +| initial_nfo_scan... | | name | +---| | | status | +| initial_media_... | | site | | season | | priority | +| last_scan_timestamp | | folder | | episode_number | | season | +| created_at | | created_at | | title | | episode | +| updated_at | | updated_at | | file_path | | progress_percent | ++---------------------+ +-------------------+ | is_downloaded | | error_message | + | created_at | | retry_count | + | updated_at | | added_at | + +-------------------+ | started_at | + | completed_at | + | created_at | + | updated_at | + +------------------------+ ``` --- ## 3. Table Schemas -### 3.1 anime_series +### 3.1 system_settings + +Stores application-wide system settings and initialization state. + +| Column | Type | Constraints | Description | +| ------------------------------ | -------- | -------------------------- | --------------------------------------------- | +| `id` | INTEGER | PRIMARY KEY, AUTOINCREMENT | Internal database ID (only one row) | +| `initial_scan_completed` | BOOLEAN | NOT NULL, DEFAULT FALSE | Whether initial anime folder scan is complete | +| `initial_nfo_scan_completed` | BOOLEAN | NOT NULL, DEFAULT FALSE | Whether initial NFO scan is complete | +| `initial_media_scan_completed` | BOOLEAN | NOT NULL, DEFAULT FALSE | Whether initial media scan is complete | +| `last_scan_timestamp` | DATETIME | NULLABLE | Timestamp of last completed scan | +| `created_at` | DATETIME | NOT NULL, DEFAULT NOW | Record creation timestamp | +| `updated_at` | DATETIME | NOT NULL, ON UPDATE NOW | Last update timestamp | + +**Purpose:** + +This table tracks the initialization status of the application to ensure that expensive one-time setup operations (like scanning the entire anime directory) only run on the first startup, not on every restart. + +- Only one row exists in this table +- The `initial_scan_completed` flag prevents redundant full directory scans on each startup +- The NFO and media scan flags similarly track completion of those setup tasks + +Source: [src/server/database/models.py](../src/server/database/models.py), [src/server/database/system_settings_service.py](../src/server/database/system_settings_service.py) + +### 3.2 anime_series Stores anime series metadata. @@ -79,7 +103,7 @@ Stores anime series metadata. Source: [src/server/database/models.py](../src/server/database/models.py#L23-L87) -### 3.2 episodes +### 3.3 episodes Stores **missing episodes** that need to be downloaded. Episodes are automatically managed during scans: @@ -105,7 +129,7 @@ Stores **missing episodes** that need to be downloaded. Episodes are automatical Source: [src/server/database/models.py](../src/server/database/models.py#L122-L181) -### 3.3 download_queue_item +### 3.4 download_queue_item Stores download queue items with status tracking. @@ -143,6 +167,7 @@ Source: [src/server/database/models.py](../src/server/database/models.py#L200-L3 | Table | Index Name | Columns | Purpose | | --------------------- | ----------------------- | ----------- | --------------------------------- | +| `system_settings` | N/A (single row) | N/A | Only one row, no indexes needed | | `anime_series` | `ix_anime_series_key` | `key` | Fast lookup by primary identifier | | `anime_series` | `ix_anime_series_name` | `name` | Search by name | | `episodes` | `ix_episodes_series_id` | `series_id` | Join with series | diff --git a/docs/instructions.md b/docs/instructions.md index be8b5db..8b87358 100644 --- a/docs/instructions.md +++ b/docs/instructions.md @@ -119,23 +119,18 @@ For each task completed: ## TODO List: -✅ **FIXED:** Anime list endpoint now correctly returns anime data after server startup. +Make sure you do not produce doublicate code. the function below is mostly implemented. +make sure you maintain the function on one location -**Root Cause:** The anime list was empty because: -1. The `SeriesApp.list` was initialized with `skip_load=True` to avoid loading from filesystem during initialization -2. Series data is synced from filesystem data files to the database during server startup -3. Series are then loaded from the database into `SeriesApp` memory via `anime_service._load_series_from_db()` -4. The server needed to be restarted to complete this initialization process +1. scanning anime from folder + make sure that scanning anime from folder only runs on setup and not on each start -**Solution:** The existing startup process in [fastapi_app.py](../src/server/fastapi_app.py) correctly: -- Syncs series from data files to database via `sync_series_from_data_files()` -- Loads series from database into memory via `anime_service._load_series_from_db()` +2. Nfo scan + make sure nfo scan runs only on setup and not on each start -The issue was resolved by restarting the server to allow the full initialization process to complete. +3. nfo data + during nfo scan read tmdb id from nfo file and write it in db. + during nfo scan read tvdb id from nfo file and write it in db. -**Verified:** GET `/api/anime` now returns 192 anime series with complete metadata including: -- Unique key (primary identifier) -- Name and folder -- Missing episodes tracking -- NFO metadata status -- TMDB/TVDB IDs when available +4. Media scan + make sure media scan runs only on setup and not on each start diff --git a/src/server/database/__init__.py b/src/server/database/__init__.py index afbdccc..9a24fe0 100644 --- a/src/server/database/__init__.py +++ b/src/server/database/__init__.py @@ -39,6 +39,7 @@ from src.server.database.models import ( AnimeSeries, DownloadQueueItem, Episode, + SystemSettings, UserSession, ) from src.server.database.service import ( @@ -47,6 +48,7 @@ from src.server.database.service import ( EpisodeService, UserSessionService, ) +from src.server.database.system_settings_service import SystemSettingsService __all__ = [ # Base and connection @@ -69,10 +71,12 @@ __all__ = [ "AnimeSeries", "Episode", "DownloadQueueItem", + "SystemSettings", "UserSession", # Services "AnimeSeriesService", "EpisodeService", "DownloadQueueService", + "SystemSettingsService", "UserSessionService", ] diff --git a/src/server/database/init.py b/src/server/database/init.py index b330de9..2c12738 100644 --- a/src/server/database/init.py +++ b/src/server/database/init.py @@ -36,6 +36,7 @@ EXPECTED_TABLES = { "episodes", "download_queue", "user_sessions", + "system_settings", } # Expected indexes for performance diff --git a/src/server/database/models.py b/src/server/database/models.py index cb7d6bd..3f133b9 100644 --- a/src/server/database/models.py +++ b/src/server/database/models.py @@ -543,3 +543,52 @@ class UserSession(Base, TimestampMixin): def revoke(self) -> None: """Revoke this session.""" self.is_active = False + + +class SystemSettings(Base, TimestampMixin): + """SQLAlchemy model for system-wide settings and state. + + Stores application-level configuration and state flags that persist + across restarts. Used to track initialization status and setup completion. + + Attributes: + id: Primary key (single row expected) + initial_scan_completed: Whether the initial anime folder scan has been completed + initial_nfo_scan_completed: Whether the initial NFO scan has been completed + initial_media_scan_completed: Whether the initial media scan has been completed + last_scan_timestamp: Timestamp of the last completed scan + created_at: Creation timestamp (from TimestampMixin) + updated_at: Last update timestamp (from TimestampMixin) + """ + __tablename__ = "system_settings" + + # Primary key (only one row should exist) + id: Mapped[int] = mapped_column( + Integer, primary_key=True, autoincrement=True + ) + + # Setup/initialization tracking + initial_scan_completed: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=False, server_default="0", + doc="Whether the initial anime folder scan has been completed" + ) + initial_nfo_scan_completed: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=False, server_default="0", + doc="Whether the initial NFO scan has been completed" + ) + initial_media_scan_completed: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=False, server_default="0", + doc="Whether the initial media scan has been completed" + ) + last_scan_timestamp: Mapped[Optional[datetime]] = mapped_column( + DateTime(timezone=True), nullable=True, + doc="Timestamp of the last completed scan" + ) + + def __repr__(self) -> str: + return ( + f"" + ) diff --git a/src/server/database/system_settings_service.py b/src/server/database/system_settings_service.py new file mode 100644 index 0000000..eeacac0 --- /dev/null +++ b/src/server/database/system_settings_service.py @@ -0,0 +1,159 @@ +"""System settings service for managing application-level configuration. + +This module provides services for managing system-wide settings and state, +including tracking initial setup completion status. +""" +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Optional + +import structlog +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from src.server.database.models import SystemSettings + +logger = structlog.get_logger(__name__) + + +class SystemSettingsService: + """Service for managing system settings.""" + + @staticmethod + async def get_or_create(db: AsyncSession) -> SystemSettings: + """Get the system settings record, creating it if it doesn't exist. + + Only one system settings record should exist in the database. + + Args: + db: Database session + + Returns: + SystemSettings instance + """ + # Try to get existing settings + stmt = select(SystemSettings).limit(1) + result = await db.execute(stmt) + settings = result.scalar_one_or_none() + + if settings is None: + # Create new settings with defaults + settings = SystemSettings( + initial_scan_completed=False, + initial_nfo_scan_completed=False, + initial_media_scan_completed=False, + ) + db.add(settings) + await db.commit() + await db.refresh(settings) + logger.info("Created new system settings record") + + return settings + + @staticmethod + async def is_initial_scan_completed(db: AsyncSession) -> bool: + """Check if the initial anime folder scan has been completed. + + Args: + db: Database session + + Returns: + True if initial scan is completed, False otherwise + """ + settings = await SystemSettingsService.get_or_create(db) + return settings.initial_scan_completed + + @staticmethod + async def mark_initial_scan_completed( + db: AsyncSession, + timestamp: Optional[datetime] = None + ) -> None: + """Mark the initial anime folder scan as completed. + + Args: + db: Database session + timestamp: Optional timestamp to set, defaults to current time + """ + settings = await SystemSettingsService.get_or_create(db) + settings.initial_scan_completed = True + settings.last_scan_timestamp = timestamp or datetime.now(timezone.utc) + await db.commit() + logger.info("Marked initial scan as completed") + + @staticmethod + async def is_initial_nfo_scan_completed(db: AsyncSession) -> bool: + """Check if the initial NFO scan has been completed. + + Args: + db: Database session + + Returns: + True if initial NFO scan is completed, False otherwise + """ + settings = await SystemSettingsService.get_or_create(db) + return settings.initial_nfo_scan_completed + + @staticmethod + async def mark_initial_nfo_scan_completed( + db: AsyncSession, + timestamp: Optional[datetime] = None + ) -> None: + """Mark the initial NFO scan as completed. + + Args: + db: Database session + timestamp: Optional timestamp to set, defaults to current time + """ + settings = await SystemSettingsService.get_or_create(db) + settings.initial_nfo_scan_completed = True + if timestamp: + settings.last_scan_timestamp = timestamp + await db.commit() + logger.info("Marked initial NFO scan as completed") + + @staticmethod + async def is_initial_media_scan_completed(db: AsyncSession) -> bool: + """Check if the initial media scan has been completed. + + Args: + db: Database session + + Returns: + True if initial media scan is completed, False otherwise + """ + settings = await SystemSettingsService.get_or_create(db) + return settings.initial_media_scan_completed + + @staticmethod + async def mark_initial_media_scan_completed( + db: AsyncSession, + timestamp: Optional[datetime] = None + ) -> None: + """Mark the initial media scan as completed. + + Args: + db: Database session + timestamp: Optional timestamp to set, defaults to current time + """ + settings = await SystemSettingsService.get_or_create(db) + settings.initial_media_scan_completed = True + if timestamp: + settings.last_scan_timestamp = timestamp + await db.commit() + logger.info("Marked initial media scan as completed") + + @staticmethod + async def reset_all_scans(db: AsyncSession) -> None: + """Reset all scan completion flags (for testing or re-setup). + + Args: + db: Database session + """ + settings = await SystemSettingsService.get_or_create(db) + settings.initial_scan_completed = False + settings.initial_nfo_scan_completed = False + settings.initial_media_scan_completed = False + settings.last_scan_timestamp = None + await db.commit() + logger.info("Reset all scan completion flags") diff --git a/src/server/fastapi_app.py b/src/server/fastapi_app.py index 07265a9..7b21bf9 100644 --- a/src/server/fastapi_app.py +++ b/src/server/fastapi_app.py @@ -186,7 +186,34 @@ async def lifespan(_application: FastAPI): # Subscribe to progress events progress_service.subscribe("progress_updated", progress_event_handler) - # Sync series from data files to database FIRST (one-time setup) + # Check if initial setup has been completed + try: + from src.server.database.connection import get_db_session + from src.server.database.system_settings_service import ( + SystemSettingsService, + ) + + async with get_db_session() as db: + is_initial_scan_done = ( + await SystemSettingsService.is_initial_scan_completed(db) + ) + + if is_initial_scan_done: + logger.info( + "Initial scan already completed, skipping data file sync" + ) + else: + logger.info( + "Initial scan not completed, " + "performing first-time setup" + ) + except Exception as e: + logger.warning( + "Failed to check system settings: %s, assuming first run", e + ) + is_initial_scan_done = False + + # Sync series from data files to database (only on first run) # This must happen before SeriesApp initialization try: logger.info( @@ -195,13 +222,33 @@ async def lifespan(_application: FastAPI): ) if settings.anime_directory: - # Sync series from data files to database (one-time setup) - sync_count = await sync_series_from_data_files( - settings.anime_directory - ) - logger.info( - "Data file sync complete. Added %d series.", sync_count - ) + # Only sync from data files on first run + if not is_initial_scan_done: + logger.info("Performing initial anime folder scan...") + sync_count = await sync_series_from_data_files( + settings.anime_directory + ) + logger.info( + "Data file sync complete. Added %d series.", sync_count + ) + + # Mark initial scan as completed + try: + async with get_db_session() as db: + await ( + SystemSettingsService + .mark_initial_scan_completed(db) + ) + logger.info("Marked initial scan as completed") + except Exception as e: + logger.warning( + "Failed to mark initial scan as completed: %s", e + ) + else: + logger.info( + "Skipping initial scan - " + "already completed on previous run" + ) # Load series from database into SeriesApp's in-memory cache from src.server.utils.dependencies import get_anime_service diff --git a/tests/unit/test_system_settings_service.py b/tests/unit/test_system_settings_service.py new file mode 100644 index 0000000..ab272b6 --- /dev/null +++ b/tests/unit/test_system_settings_service.py @@ -0,0 +1,58 @@ +"""Test the system settings service integration.""" +import pytest + +from src.server.database.connection import get_db_session, init_db +from src.server.database.system_settings_service import SystemSettingsService + + +@pytest.mark.asyncio +async def test_system_settings_integration(): + """Test SystemSettings service with actual database operations.""" + # Initialize database + await init_db() + + # Test get_or_create (should create on first call) + async with get_db_session() as db: + settings = await SystemSettingsService.get_or_create(db) + assert settings is not None + assert settings.id is not None + assert settings.initial_scan_completed is False + assert settings.initial_nfo_scan_completed is False + assert settings.initial_media_scan_completed is False + + # Test checking individual flags + async with get_db_session() as db: + is_scan_done = await SystemSettingsService.is_initial_scan_completed(db) + assert is_scan_done is False + + is_nfo_done = await SystemSettingsService.is_initial_nfo_scan_completed(db) + assert is_nfo_done is False + + is_media_done = await SystemSettingsService.is_initial_media_scan_completed(db) + assert is_media_done is False + + # Test marking scans as completed + async with get_db_session() as db: + await SystemSettingsService.mark_initial_scan_completed(db) + + async with get_db_session() as db: + is_scan_done = await SystemSettingsService.is_initial_scan_completed(db) + assert is_scan_done is True + + # Others should still be False + is_nfo_done = await SystemSettingsService.is_initial_nfo_scan_completed(db) + assert is_nfo_done is False + + # Test reset + async with get_db_session() as db: + await SystemSettingsService.reset_all_scans(db) + + async with get_db_session() as db: + settings = await SystemSettingsService.get_or_create(db) + assert settings.initial_scan_completed is False + assert settings.initial_nfo_scan_completed is False + assert settings.initial_media_scan_completed is False + + +if __name__ == "__main__": + pytest.main([__file__, "-v"])