Integrate data migration into FastAPI lifespan (Task 3)

instructions.md

@@ -142,7 +142,7 @@ The current implementation stores anime series metadata in `data` files (JSON fo
 
 ---
 
-### Task 3: Integrate Migration into FastAPI Lifespan ⬜
+### Task 3: Integrate Migration into FastAPI Lifespan ✅
 
 **File:** `src/server/fastapi_app.py`
 

src/server/fastapi_app.py

@@ -67,6 +67,23 @@ async def lifespan(app: FastAPI):
         except Exception as e:
             logger.warning("Failed to load config from config.json: %s", e)
 
+        # Run data file to database migration
+        try:
+            from src.server.services.startup_migration import (
+                ensure_migration_on_startup,
+            )
+            migration_result = await ensure_migration_on_startup()
+            if migration_result:
+                logger.info(
+                    "Data migration complete: %d migrated, %d skipped, %d failed",
+                    migration_result.migrated,
+                    migration_result.skipped,
+                    migration_result.failed
+                )
+        except Exception as e:
+            logger.error("Data migration failed: %s", e, exc_info=True)
+            # Continue startup - migration failure should not block app
+
         # Initialize progress service with event subscription
         progress_service = get_progress_service()
         ws_service = get_websocket_service()

tests/integration/test_data_file_migration.py

@@ -0,0 +1,215 @@
+"""Integration tests for data file to database migration.
+
+This module tests the complete migration workflow including:
+- Migration runs on server startup
+- App starts even if migration fails
+- Data files are correctly migrated to database
+"""
+import json
+import tempfile
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from httpx import ASGITransport, AsyncClient
+
+from src.server.services.data_migration_service import DataMigrationService
+from src.server.services.startup_migration import ensure_migration_on_startup
+
+
+class TestMigrationStartupIntegration:
+    """Test migration integration with application startup."""
+
+    @pytest.mark.asyncio
+    async def test_app_starts_with_migration(self):
+        """Test that app starts successfully with migration enabled."""
+        from src.server.fastapi_app import app
+        
+        transport = ASGITransport(app=app)
+        async with AsyncClient(
+            transport=transport,
+            base_url="http://test"
+        ) as client:
+            # App should start and health endpoint should work
+            response = await client.get("/health")
+            assert response.status_code == 200
+
+    @pytest.mark.asyncio
+    async def test_migration_with_valid_data_files(self):
+        """Test migration correctly processes data files."""
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            # Create test data files
+            for i in range(2):
+                series_dir = Path(tmp_dir) / f"Test Series {i}"
+                series_dir.mkdir()
+                data = {
+                    "key": f"test-series-{i}",
+                    "name": f"Test Series {i}",
+                    "site": "aniworld.to",
+                    "folder": f"Test Series {i}",
+                    "episodeDict": {"1": [1, 2, 3]}
+                }
+                (series_dir / "data").write_text(json.dumps(data))
+            
+            # Test migration scan
+            service = DataMigrationService()
+            data_files = service.scan_for_data_files(tmp_dir)
+            
+            assert len(data_files) == 2
+
+    @pytest.mark.asyncio
+    async def test_migration_handles_corrupted_files(self):
+        """Test migration handles corrupted data files gracefully."""
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            # Create valid data file
+            valid_dir = Path(tmp_dir) / "Valid Series"
+            valid_dir.mkdir()
+            valid_data = {
+                "key": "valid-series",
+                "name": "Valid Series",
+                "site": "aniworld.to",
+                "folder": "Valid Series",
+                "episodeDict": {}
+            }
+            (valid_dir / "data").write_text(json.dumps(valid_data))
+            
+            # Create corrupted data file
+            invalid_dir = Path(tmp_dir) / "Invalid Series"
+            invalid_dir.mkdir()
+            (invalid_dir / "data").write_text("not valid json {{{")
+            
+            # Migration should process valid file and report error for invalid
+            service = DataMigrationService()
+            
+            with patch(
+                'src.server.services.data_migration_service.AnimeSeriesService'
+            ) as MockService:
+                MockService.get_by_key = AsyncMock(return_value=None)
+                MockService.create = AsyncMock()
+                
+                mock_db = AsyncMock()
+                mock_db.commit = AsyncMock()
+                
+                result = await service.migrate_all(tmp_dir, mock_db)
+                
+                # Should have found 2 files
+                assert result.total_found == 2
+                # One should succeed, one should fail
+                assert result.migrated == 1
+                assert result.failed == 1
+                assert len(result.errors) == 1
+
+
+class TestMigrationWithConfig:
+    """Test migration with configuration file."""
+
+    @pytest.mark.asyncio
+    async def test_migration_uses_config_anime_directory(self):
+        """Test that migration reads anime directory from config."""
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            mock_config = MagicMock()
+            mock_config.other = {"anime_directory": tmp_dir}
+            
+            with patch(
+                'src.server.services.startup_migration.ConfigService'
+            ) as MockConfigService:
+                mock_service = MagicMock()
+                mock_service.load_config.return_value = mock_config
+                MockConfigService.return_value = mock_service
+                
+                with patch(
+                    'src.server.services.startup_migration.get_data_migration_service'
+                ) as mock_get_service:
+                    migration_service = MagicMock()
+                    migration_service.is_migration_needed.return_value = False
+                    mock_get_service.return_value = migration_service
+                    
+                    result = await ensure_migration_on_startup()
+                    
+                    # Should check the correct directory
+                    migration_service.is_migration_needed.assert_called_once_with(
+                        tmp_dir
+                    )
+
+
+class TestMigrationIdempotency:
+    """Test that migration is idempotent."""
+
+    @pytest.mark.asyncio
+    async def test_migration_skips_existing_entries(self):
+        """Test that migration skips series already in database."""
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            # Create data file
+            series_dir = Path(tmp_dir) / "Test Series"
+            series_dir.mkdir()
+            data = {
+                "key": "test-series",
+                "name": "Test Series",
+                "site": "aniworld.to",
+                "folder": "Test Series",
+                "episodeDict": {"1": [1, 2]}
+            }
+            (series_dir / "data").write_text(json.dumps(data))
+            
+            # Mock existing series in database
+            existing = MagicMock()
+            existing.id = 1
+            existing.episode_dict = {"1": [1, 2]}  # Same data
+            
+            service = DataMigrationService()
+            
+            with patch(
+                'src.server.services.data_migration_service.AnimeSeriesService'
+            ) as MockService:
+                MockService.get_by_key = AsyncMock(return_value=existing)
+                
+                mock_db = AsyncMock()
+                mock_db.commit = AsyncMock()
+                
+                result = await service.migrate_all(tmp_dir, mock_db)
+                
+                # Should skip since data is same
+                assert result.total_found == 1
+                assert result.skipped == 1
+                assert result.migrated == 0
+                # Should not call create
+                MockService.create.assert_not_called()
+
+    @pytest.mark.asyncio
+    async def test_migration_updates_changed_episodes(self):
+        """Test that migration updates series with changed episode data."""
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            # Create data file with new episodes
+            series_dir = Path(tmp_dir) / "Test Series"
+            series_dir.mkdir()
+            data = {
+                "key": "test-series",
+                "name": "Test Series",
+                "site": "aniworld.to",
+                "folder": "Test Series",
+                "episodeDict": {"1": [1, 2, 3, 4, 5]}  # More episodes
+            }
+            (series_dir / "data").write_text(json.dumps(data))
+            
+            # Mock existing series with fewer episodes
+            existing = MagicMock()
+            existing.id = 1
+            existing.episode_dict = {"1": [1, 2]}  # Fewer episodes
+            
+            service = DataMigrationService()
+            
+            with patch(
+                'src.server.services.data_migration_service.AnimeSeriesService'
+            ) as MockService:
+                MockService.get_by_key = AsyncMock(return_value=existing)
+                MockService.update = AsyncMock()
+                
+                mock_db = AsyncMock()
+                mock_db.commit = AsyncMock()
+                
+                result = await service.migrate_all(tmp_dir, mock_db)
+                
+                # Should update since data changed
+                assert result.total_found == 1
+                assert result.migrated == 1
+                MockService.update.assert_called_once()