Implement async series data loading with background processing
- Add loading status fields to AnimeSeries model
- Create BackgroundLoaderService for async task processing
- Update POST /api/anime/add to return 202 Accepted immediately
- Add GET /api/anime/{key}/loading-status endpoint
- Integrate background loader with startup/shutdown lifecycle
- Create database migration script for loading status fields
- Add unit tests for BackgroundLoaderService (10 tests, all passing)
- Update AnimeSeriesService.create() to accept loading status fields
Architecture follows clean separation with no code duplication:
- BackgroundLoader orchestrates, doesn't reimplement
- Reuses existing AnimeService, NFOService, WebSocket patterns
- Database-backed status survives restarts
This commit is contained in:
97
scripts/migrate_loading_status.py
Normal file
97
scripts/migrate_loading_status.py
Normal file
@@ -0,0 +1,97 @@
|
||||
"""Database migration utility for adding loading status fields.
|
||||
|
||||
This script adds the loading status fields to existing anime_series tables
|
||||
without Alembic. For new databases, these fields are created automatically
|
||||
via create_all().
|
||||
|
||||
Run this after updating the models.py file.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to Python path
|
||||
project_root = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
from src.config.settings import settings
|
||||
from src.server.database.connection import get_engine, init_db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
|
||||
async def migrate_add_loading_status_fields():
|
||||
"""Add loading status fields to anime_series table if they don't exist."""
|
||||
|
||||
# Initialize database connection
|
||||
await init_db()
|
||||
engine = get_engine()
|
||||
|
||||
if not engine:
|
||||
logger.error("Failed to get database engine")
|
||||
return
|
||||
|
||||
# Define the migrations
|
||||
migrations = [
|
||||
("loading_status", "ALTER TABLE anime_series ADD COLUMN loading_status VARCHAR(50) NOT NULL DEFAULT 'completed'"),
|
||||
("episodes_loaded", "ALTER TABLE anime_series ADD COLUMN episodes_loaded BOOLEAN NOT NULL DEFAULT 1"),
|
||||
("logo_loaded", "ALTER TABLE anime_series ADD COLUMN logo_loaded BOOLEAN NOT NULL DEFAULT 0"),
|
||||
("images_loaded", "ALTER TABLE anime_series ADD COLUMN images_loaded BOOLEAN NOT NULL DEFAULT 0"),
|
||||
("loading_started_at", "ALTER TABLE anime_series ADD COLUMN loading_started_at TIMESTAMP"),
|
||||
("loading_completed_at", "ALTER TABLE anime_series ADD COLUMN loading_completed_at TIMESTAMP"),
|
||||
("loading_error", "ALTER TABLE anime_series ADD COLUMN loading_error VARCHAR(1000)"),
|
||||
]
|
||||
|
||||
async with engine.begin() as conn:
|
||||
for column_name, sql in migrations:
|
||||
try:
|
||||
logger.info(f"Adding column: {column_name}")
|
||||
await conn.execute(text(sql))
|
||||
logger.info(f"✅ Successfully added column: {column_name}")
|
||||
except OperationalError as e:
|
||||
if "duplicate column name" in str(e).lower() or "already exists" in str(e).lower():
|
||||
logger.info(f"⏭️ Column {column_name} already exists, skipping")
|
||||
else:
|
||||
logger.error(f"❌ Error adding column {column_name}: {e}")
|
||||
raise
|
||||
|
||||
logger.info("Migration completed successfully!")
|
||||
logger.info("All loading status fields are now available in anime_series table")
|
||||
|
||||
|
||||
async def rollback_loading_status_fields():
|
||||
"""Remove loading status fields from anime_series table."""
|
||||
|
||||
await init_db()
|
||||
engine = get_engine()
|
||||
|
||||
if not engine:
|
||||
logger.error("Failed to get database engine")
|
||||
return
|
||||
|
||||
# SQLite doesn't support DROP COLUMN easily, so we'd need to recreate the table
|
||||
# For now, just log a warning
|
||||
logger.warning("Rollback not implemented for SQLite")
|
||||
logger.warning("To rollback, you would need to:")
|
||||
logger.warning("1. Create a new table without the loading fields")
|
||||
logger.warning("2. Copy data from old table")
|
||||
logger.warning("3. Drop old table and rename new table")
|
||||
|
||||
|
||||
def main():
|
||||
"""Run the migration."""
|
||||
import sys
|
||||
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "rollback":
|
||||
asyncio.run(rollback_loading_status_fields())
|
||||
else:
|
||||
asyncio.run(migrate_add_loading_status_fields())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user