diff --git a/docs/task3_status.md b/docs/task3_status.md index 47981d3..1082367 100644 --- a/docs/task3_status.md +++ b/docs/task3_status.md @@ -1,115 +1,131 @@ # Task 3: NFO Metadata Integration - Status Report ## Summary + Task 3 focuses on creating tvshow.nfo files and downloading media (poster/logo/fanart) using TMDB API, adapted from the scraper repository. ## ✅ Completed (80%) ### 1. Core Infrastructure (100%) -- ✅ **TMDB API Client** (`src/core/services/tmdb_client.py` - 270 lines) - - Async HTTP client using aiohttp - - Search TV shows by name and year - - Get detailed show information with external IDs - - Get show images (posters, backdrops, logos) - - Download images from TMDB - - Response caching to reduce API calls - - Rate limit handling (429 status) - - Retry logic with exponential backoff - - Proper error handling (401, 404, 500) - - Context manager support -- ✅ **NFO XML Generator** (`src/core/utils/nfo_generator.py` - 180 lines) - - Generate Kodi/XBMC XML from TVShowNFO models - - Handle all standard Kodi fields - - Support ratings, actors, images, unique IDs - - XML validation function - - Proper encoding (UTF-8) - - Handle special characters and Unicode +- ✅ **TMDB API Client** (`src/core/services/tmdb_client.py` - 270 lines) -- ✅ **Image Downloader** (`src/core/utils/image_downloader.py` - 296 lines) - - Download images from URLs - - Validate images using PIL (format, size) - - Retry logic with exponential backoff - - Skip existing files option - - Min file size checking (1KB) - - Download specific types: poster.jpg, logo.png, fanart.jpg - - Concurrent downloads via download_all_media() - - Proper error handling + - Async HTTP client using aiohttp + - Search TV shows by name and year + - Get detailed show information with external IDs + - Get show images (posters, backdrops, logos) + - Download images from TMDB + - Response caching to reduce API calls + - Rate limit handling (429 status) + - Retry logic with exponential backoff + - Proper error handling (401, 404, 500) + - Context manager support -- ✅ **NFO Service** (`src/core/services/nfo_service.py` - 390 lines) - - Orchestrates TMDB client, NFO generator, and image downloader - - check_nfo_exists() - Check if tvshow.nfo exists - - create_tvshow_nfo() - Create NFO by scraping TMDB - - _find_best_match() - Match search results with year filter - - _tmdb_to_nfo_model() - Convert TMDB data to TVShowNFO model - - _download_media_files() - Download poster/logo/fanart - - Handle search ambiguity - - Proper error handling and logging +- ✅ **NFO XML Generator** (`src/core/utils/nfo_generator.py` - 180 lines) + + - Generate Kodi/XBMC XML from TVShowNFO models + - Handle all standard Kodi fields + - Support ratings, actors, images, unique IDs + - XML validation function + - Proper encoding (UTF-8) + - Handle special characters and Unicode + +- ✅ **Image Downloader** (`src/core/utils/image_downloader.py` - 296 lines) + + - Download images from URLs + - Validate images using PIL (format, size) + - Retry logic with exponential backoff + - Skip existing files option + - Min file size checking (1KB) + - Download specific types: poster.jpg, logo.png, fanart.jpg + - Concurrent downloads via download_all_media() + - Proper error handling + +- ✅ **NFO Service** (`src/core/services/nfo_service.py` - 390 lines) + - Orchestrates TMDB client, NFO generator, and image downloader + - check_nfo_exists() - Check if tvshow.nfo exists + - create_tvshow_nfo() - Create NFO by scraping TMDB + - \_find_best_match() - Match search results with year filter + - \_tmdb_to_nfo_model() - Convert TMDB data to TVShowNFO model + - \_download_media_files() - Download poster/logo/fanart + - Handle search ambiguity + - Proper error handling and logging ### 2. Configuration (100%) -- ✅ Added NFO settings to `src/config/settings.py`: - - TMDB_API_KEY: API key for TMDB access - - NFO_AUTO_CREATE: Auto-create NFOs when scanning (default: False) - - NFO_UPDATE_ON_SCAN: Update existing NFOs (default: False) - - NFO_DOWNLOAD_POSTER: Download poster.jpg (default: True) - - NFO_DOWNLOAD_LOGO: Download logo.png (default: True) - - NFO_DOWNLOAD_FANART: Download fanart.jpg (default: True) - - NFO_IMAGE_SIZE: Image size to download (default: "original") + +- ✅ Added NFO settings to `src/config/settings.py`: + - TMDB_API_KEY: API key for TMDB access + - NFO_AUTO_CREATE: Auto-create NFOs when scanning (default: False) + - NFO_UPDATE_ON_SCAN: Update existing NFOs (default: False) + - NFO_DOWNLOAD_POSTER: Download poster.jpg (default: True) + - NFO_DOWNLOAD_LOGO: Download logo.png (default: True) + - NFO_DOWNLOAD_FANART: Download fanart.jpg (default: True) + - NFO_IMAGE_SIZE: Image size to download (default: "original") ### 3. Dependencies (100%) -- ✅ Updated `requirements.txt`: - - aiohttp>=3.9.0 (async HTTP client) - - lxml>=5.0.0 (XML generation/validation) - - pillow>=10.0.0 (image validation) -- ✅ Installed in conda environment + +- ✅ Updated `requirements.txt`: + - aiohttp>=3.9.0 (async HTTP client) + - lxml>=5.0.0 (XML generation/validation) + - pillow>=10.0.0 (image validation) +- ✅ Installed in conda environment ## ⚠️ Needs Refinement (20%) ### 1. Unit Tests (40% complete, needs major updates) **Current Status:** -- ✅ Test files created for all modules: - - `tests/unit/test_tmdb_client.py` (16 tests, all failing) - - `tests/unit/test_nfo_generator.py` (21 tests, 18 passing, 3 failing) - - `tests/unit/test_image_downloader.py` (23 tests, all failing) + +- ✅ Test files created for all modules: + - `tests/unit/test_tmdb_client.py` (16 tests, all failing) + - `tests/unit/test_nfo_generator.py` (21 tests, 18 passing, 3 failing) + - `tests/unit/test_image_downloader.py` (23 tests, all failing) **Issues:** The tests were written based on assumptions about the API that don't match the actual implementation: 1. **ImageDownloader Issues:** - - Tests assume context manager (`__aenter__`), but not implemented - - Tests assume `_validate_image()` method, actual is `validate_image()` (no underscore) - - Tests assume `session` attribute, but ImageDownloader creates sessions internally - - Tests try to mock `session.get()`, but implementation uses `aiohttp.ClientSession()` directly in method - - Tests assume `ImageValidationError` exception, but only `ImageDownloadError` exists + + - Tests assume context manager (`__aenter__`), but not implemented + - Tests assume `_validate_image()` method, actual is `validate_image()` (no underscore) + - Tests assume `session` attribute, but ImageDownloader creates sessions internally + - Tests try to mock `session.get()`, but implementation uses `aiohttp.ClientSession()` directly in method + - Tests assume `ImageValidationError` exception, but only `ImageDownloadError` exists 2. **NFO Generator Issues:** - - 3 tests failing due to XML validation logic differences - - Need to review actual lxml etree behavior + + - 3 tests failing due to XML validation logic differences + - Need to review actual lxml etree behavior 3. **TMDB Client Issues:** - - Tests assume `session` attribute for mocking, need to check actual implementation - - Tests assume `_make_request()` method, need to verify API + - Tests assume `session` attribute for mocking, need to check actual implementation + - Tests assume `_make_request()` method, need to verify API **Refactoring Needed:** -- Review actual implementation APIs -- Update test mocks to match implementation -- Consider adding context manager support to ImageDownloader -- Simplify test approach - use @patch on aiohttp.ClientSession instead of internal mocking -- Add integration tests with real API calls (optional, for manual verification) + +- ❗ **Critical Challenge**: aiohttp mocking is complex due to nested async context managers +- **Alternative Approach Recommended**: + 1. Create integration test script with real API calls (see below) + 2. Focus unit tests on business logic (NFO conversion, file operations) + 3. Mock at higher level (mock `download_image` method, not aiohttp internals) + 4. Consider adding dependency injection to make testing easier +- Or: Simplify implementation to use requests library (sync) for easier testing +- Add integration tests with real API calls (optional, for manual verification) ### 2. Integration with SerieList (Not started) **Needs Implementation:** -- Integrate NFOService into SerieList scan process -- Add auto-create logic based on NFO_AUTO_CREATE setting -- Add update logic based on NFO_UPDATE_ON_SCAN setting -- Test end-to-end NFO creation flow + +- Integrate NFOService into SerieList scan process +- Add auto-create logic based on NFO_AUTO_CREATE setting +- Add update logic based on NFO_UPDATE_ON_SCAN setting +- Test end-to-end NFO creation flow ### 3. CLI Commands (Not started) **Optional Enhancement:** Add CLI commands for NFO management: + ```bash # Create NFO for specific series python src/cli/Main.py nfo create "Attack on Titan" --year 2013 @@ -127,104 +143,116 @@ python src/cli/Main.py nfo status ## 📊 Coverage Status **Current:** -- `src/core/services/tmdb_client.py`: 0% (tests failing) -- `src/core/utils/nfo_generator.py`: 0% (tests failing) -- `src/core/utils/image_downloader.py`: 0% (tests failing) -- `src/core/services/nfo_service.py`: Not tested yet + +- `src/core/services/tmdb_client.py`: 0% (tests failing) +- `src/core/utils/nfo_generator.py`: 0% (tests failing) +- `src/core/utils/image_downloader.py`: 0% (tests failing) +- `src/core/services/nfo_service.py`: Not tested yet **Target:** -- All modules: > 85% coverage + +- All modules: > 85% coverage ## 🔧 Next Steps (Priority Order) ### High Priority + 1. **Fix Unit Tests** (2-3 hours) - - Update test_image_downloader.py to match actual API - - Fix test_nfo_generator.py validation tests - - Update test_tmdb_client.py mocking strategy - - Add test_nfo_service.py with comprehensive tests - - Run tests and achieve > 85% coverage + + - Update test_image_downloader.py to match actual API + - Fix test_nfo_generator.py validation tests + - Update test_tmdb_client.py mocking strategy + - Add test_nfo_service.py with comprehensive tests + - Run tests and achieve > 85% coverage 2. **Manual Integration Testing** (1 hour) - - Create test script to verify TMDB client with real API - - Test NFO generation with sample data - - Test image downloads - - Verify generated NFO is valid Kodi format + - Create test script to verify TMDB client with real API + - Test NFO generation with sample data + - Test image downloads + - Verify generated NFO is valid Kodi format ### Medium Priority + 3. **Integrate with SerieList** (1-2 hours) - - Add NFOService to SerieList.load_series() - - Implement auto-create logic - - Implement update logic - - Add logging for NFO operations - - Test with existing series folders + + - Add NFOService to SerieList.load_series() + - Implement auto-create logic + - Implement update logic + - Add logging for NFO operations + - Test with existing series folders 4. **CLI Commands** (1-2 hours, optional) - - Create nfo_commands.py module - - Implement create, update, status commands - - Add to CLI menu - - Test commands + - Create nfo_commands.py module + - Implement create, update, status commands + - Add to CLI menu + - Test commands ### Low Priority + 5. **Documentation** (30 minutes) - - Document TMDB API setup (getting API key) - - Document NFO file format and Kodi compatibility - - Add examples to README - - Update ARCHITECTURE.md with NFO components + + - Document TMDB API setup (getting API key) + - Document NFO file format and Kodi compatibility + - Add examples to README + - Update ARCHITECTURE.md with NFO components 6. **API Endpoints** (Future, separate task) - - POST /api/series/{id}/nfo - Create/update NFO - - GET /api/series/{id}/nfo - Get NFO status - - DELETE /api/series/{id}/nfo - Delete NFO + - POST /api/series/{id}/nfo - Create/update NFO + - GET /api/series/{id}/nfo - Get NFO status + - DELETE /api/series/{id}/nfo - Delete NFO ## 🐛 Known Issues 1. **NFOService.update_tvshow_nfo()** - Not implemented - - Marked with `raise NotImplementedError` - - Need to parse existing NFO to extract TMDB ID - - Then refetch and regenerate + + - Marked with `raise NotImplementedError` + - Need to parse existing NFO to extract TMDB ID + - Then refetch and regenerate 2. **Test Failures** - See "Unit Tests" section above 3. **No Error Recovery** - If TMDB API fails during scan - - Need to handle gracefully - - Don't block scan if NFO creation fails - - Log errors but continue + - Need to handle gracefully + - Don't block scan if NFO creation fails + - Log errors but continue ## 📝 Testing Checklist Once tests are fixed, verify: -- [ ] TMDBClient can search for shows -- [ ] TMDBClient handles year filtering -- [ ] TMDBClient gets detailed show info -- [ ] TMDBClient downloads images -- [ ] TMDBClient handles rate limits -- [ ] TMDBClient handles API errors -- [ ] NFO generator creates valid XML -- [ ] NFO generator handles Unicode -- [ ] NFO generator escapes special chars -- [ ] ImageDownloader validates images -- [ ] ImageDownloader retries on failure -- [ ] ImageDownloader skips existing files -- [ ] NFOService creates complete NFO -- [ ] NFOService downloads all media -- [ ] NFOService handles missing images -- [ ] All tests pass with > 85% coverage +- [ ] TMDBClient can search for shows +- [ ] TMDBClient handles year filtering +- [ ] TMDBClient gets detailed show info +- [ ] TMDBClient downloads images +- [ ] TMDBClient handles rate limits +- [ ] TMDBClient handles API errors +- [ ] NFO generator creates valid XML +- [ ] NFO generator handles Unicode +- [ ] NFO generator escapes special chars +- [ ] ImageDownloader validates images +- [ ] ImageDownloader retries on failure +- [ ] ImageDownloader skips existing files +- [ ] NFOService creates complete NFO +- [ ] NFOService downloads all media +- [ ] NFOService handles missing images +- [ ] All tests pass with > 85% coverage ## 💡 Recommendations ### Immediate Actions + 1. Invest time in fixing tests - they provide essential validation 2. Add simple integration test script for manual verification 3. Test with a few real anime series to validate Kodi compatibility ### Architecture Improvements + 1. Consider adding context manager to ImageDownloader for consistency 2. Add more detailed logging in NFOService for debugging 3. Consider caching TMDB results more aggressively ### Future Enhancements + 1. Support for episode-level NFO files (episodedetails) 2. Support for season-level NFO files 3. Background task for bulk NFO creation @@ -235,19 +263,21 @@ Once tests are fixed, verify: ## 🎯 Completion Criteria Task 3 will be considered complete when: -- ✅ All core components implemented (DONE) -- ✅ Configuration added (DONE) -- ✅ Dependencies installed (DONE) -- ⚠️ Unit tests pass with > 85% coverage (PENDING) -- ⚠️ Integration with SerieList (PENDING) -- ⚠️ Manual testing validates Kodi compatibility (PENDING) -- ⚠️ Documentation updated (PENDING) + +- ✅ All core components implemented (DONE) +- ✅ Configuration added (DONE) +- ✅ Dependencies installed (DONE) +- ⚠️ Unit tests pass with > 85% coverage (PENDING) +- ⚠️ Integration with SerieList (PENDING) +- ⚠️ Manual testing validates Kodi compatibility (PENDING) +- ⚠️ Documentation updated (PENDING) ## ⏱️ Estimated Time to Complete -- Fix tests: 2-3 hours -- Integration: 1-2 hours -- Documentation: 30 minutes -- **Total: 4-6 hours** + +- Fix tests: 2-3 hours +- Integration: 1-2 hours +- Documentation: 30 minutes +- **Total: 4-6 hours** --- diff --git a/scripts/test_nfo_integration.py b/scripts/test_nfo_integration.py new file mode 100644 index 0000000..81ec0bf --- /dev/null +++ b/scripts/test_nfo_integration.py @@ -0,0 +1,289 @@ +"""Manual integration test for NFO functionality. + +This script tests the complete NFO generation workflow with real TMDB API calls. +It's intended for manual verification, not automated testing. + +Usage: + 1. Set TMDB_API_KEY environment variable + 2. Run: python scripts/test_nfo_integration.py + 3. Check output in test_output/ directory + +Requirements: + - Valid TMDB API key (get from https://www.themoviedb.org/settings/api) + - Internet connection + - Write permissions for test_output/ directory +""" + +import asyncio +import os +import sys +from pathlib import Path + +# Add src to path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from src.core.services.tmdb_client import TMDBClient, TMDBAPIError +from src.core.services.nfo_service import NFOService +from src.core.entities.nfo_models import TVShowNFO +from src.core.utils.nfo_generator import generate_tvshow_nfo, validate_nfo_xml + + +async def test_tmdb_client(): + """Test TMDB client basic functionality.""" + print("\n=== Testing TMDB Client ===") + + api_key = os.getenv("TMDB_API_KEY") + if not api_key: + print("❌ TMDB_API_KEY environment variable not set") + print(" Get your API key from: https://www.themoviedb.org/settings/api") + return False + + try: + async with TMDBClient(api_key=api_key) as client: + # Test 1: Search for a show + print("\n1. Searching for 'Attack on Titan'...") + results = await client.search_tv_show("Attack on Titan") + + if results and results.get("results"): + show = results["results"][0] + print(f" ✅ Found: {show['name']} (ID: {show['id']})") + show_id = show["id"] + else: + print(" ❌ No results found") + return False + + # Test 2: Get show details + print(f"\n2. Getting details for show ID {show_id}...") + details = await client.get_tv_show_details( + show_id, + append_to_response="credits,external_ids,images" + ) + print(f" ✅ Title: {details['name']}") + print(f" ✅ First Air Date: {details.get('first_air_date', 'N/A')}") + print(f" ✅ Rating: {details.get('vote_average', 'N/A')}/10") + + # Test 3: Get external IDs + if "external_ids" in details: + ext_ids = details["external_ids"] + print(f" ✅ IMDB ID: {ext_ids.get('imdb_id', 'N/A')}") + print(f" ✅ TVDB ID: {ext_ids.get('tvdb_id', 'N/A')}") + + # Test 4: Get images + if "images" in details: + images = details["images"] + print(f" ✅ Posters: {len(images.get('posters', []))}") + print(f" ✅ Backdrops: {len(images.get('backdrops', []))}") + print(f" ✅ Logos: {len(images.get('logos', []))}") + + # Test 5: Get image URL + if details.get("poster_path"): + url = client.get_image_url(details["poster_path"], "w500") + print(f" ✅ Poster URL: {url[:60]}...") + + return True + + except TMDBAPIError as e: + print(f" ❌ TMDB API Error: {e}") + return False + except Exception as e: + print(f" ❌ Unexpected Error: {e}") + import traceback + traceback.print_exc() + return False + + +async def test_nfo_generation(): + """Test NFO XML generation.""" + print("\n=== Testing NFO Generation ===") + + try: + # Create a sample NFO model + print("\n1. Creating sample TVShowNFO model...") + from src.core.entities.nfo_models import RatingInfo, ActorInfo, ImageInfo, UniqueID + + nfo = TVShowNFO( + title="Test Show", + originaltitle="Test Show Original", + year=2020, + plot="This is a test show for NFO generation validation.", + runtime=45, + premiered="2020-01-15", + status="Continuing", + genre=["Action", "Drama", "Animation"], + studio=["Test Studio"], + country=["Japan"], + ratings=[RatingInfo( + name="themoviedb", + value=8.5, + votes=1000, + max_rating=10, + default=True + )], + actors=[ + ActorInfo(name="Test Actor 1", role="Main Character"), + ActorInfo(name="Test Actor 2", role="Villain") + ], + thumb=[ImageInfo(url="https://image.tmdb.org/t/p/w500/poster.jpg")], + fanart=[ImageInfo(url="https://image.tmdb.org/t/p/original/fanart.jpg")], + uniqueid=[ + UniqueID(type="tmdb", value="12345", default=False), + UniqueID(type="tvdb", value="67890", default=True) + ], + tmdbid=12345, + tvdbid=67890, + imdbid="tt1234567" + ) + print(" ✅ TVShowNFO model created") + + # Test 2: Generate XML + print("\n2. Generating XML...") + xml_string = generate_tvshow_nfo(nfo) + print(f" ✅ Generated {len(xml_string)} characters") + + # Test 3: Validate XML + print("\n3. Validating XML...") + validate_nfo_xml(xml_string) + print(" ✅ XML is valid") + + # Test 4: Save to file + output_dir = Path("test_output") + output_dir.mkdir(exist_ok=True) + nfo_path = output_dir / "test_tvshow.nfo" + nfo_path.write_text(xml_string, encoding="utf-8") + print(f" ✅ Saved to: {nfo_path}") + + # Test 5: Show sample + print("\n4. Sample XML (first 500 chars):") + print(" " + xml_string[:500].replace("\n", "\n ")) + + return True + + except Exception as e: + print(f" ❌ Error: {e}") + import traceback + traceback.print_exc() + return False + + +async def test_nfo_service(): + """Test complete NFO service workflow.""" + print("\n=== Testing NFO Service ===") + + api_key = os.getenv("TMDB_API_KEY") + if not api_key: + print("❌ TMDB_API_KEY environment variable not set") + return False + + try: + # Create test output directory + output_dir = Path("test_output") + output_dir.mkdir(exist_ok=True) + + # Create a test series folder + test_series = output_dir / "Attack_on_Titan" + test_series.mkdir(exist_ok=True) + + print(f"\n1. Creating NFO for 'Attack on Titan'...") + print(f" Output directory: {test_series}") + + # Initialize NFO service + nfo_service = NFOService( + tmdb_api_key=api_key, + anime_directory=str(output_dir), + image_size="w500" + ) + + # Create NFO + nfo_path = await nfo_service.create_tvshow_nfo( + serie_name="Attack on Titan", + serie_folder="Attack_on_Titan", + year=2013, + download_poster=True, + download_logo=True, + download_fanart=True + ) + + print(f" ✅ NFO created: {nfo_path}") + + # Check if files were created + print("\n2. Checking created files...") + files_created = { + "tvshow.nfo": (test_series / "tvshow.nfo").exists(), + "poster.jpg": (test_series / "poster.jpg").exists(), + "logo.png": (test_series / "logo.png").exists(), + "fanart.jpg": (test_series / "fanart.jpg").exists(), + } + + for filename, exists in files_created.items(): + status = "✅" if exists else "❌" + size = "" + if exists: + file_path = test_series / filename + size = f" ({file_path.stat().st_size:,} bytes)" + print(f" {status} {filename}{size}") + + # Read and validate NFO + if files_created["tvshow.nfo"]: + print("\n3. Validating generated NFO...") + nfo_content = nfo_path.read_text(encoding="utf-8") + validate_nfo_xml(nfo_content) + print(" ✅ NFO is valid XML") + + # Show sample + print("\n4. NFO Content (first 800 chars):") + print(" " + nfo_content[:800].replace("\n", "\n ")) + + return all(files_created.values()) + + except Exception as e: + print(f" ❌ Error: {e}") + import traceback + traceback.print_exc() + return False + + +async def main(): + """Run all integration tests.""" + print("=" * 70) + print("NFO Functionality Integration Tests") + print("=" * 70) + print("\nNOTE: This requires a valid TMDB API key set as environment variable.") + print("Get your API key from: https://www.themoviedb.org/settings/api") + print("Set it with: export TMDB_API_KEY='your_api_key_here'") + + results = [] + + # Test 1: TMDB Client + results.append(("TMDB Client", await test_tmdb_client())) + + # Test 2: NFO Generation + results.append(("NFO Generation", await test_nfo_generation())) + + # Test 3: NFO Service (full workflow) + results.append(("NFO Service", await test_nfo_service())) + + # Summary + print("\n" + "=" * 70) + print("SUMMARY") + print("=" * 70) + + for test_name, passed in results: + status = "✅ PASSED" if passed else "❌ FAILED" + print(f"{test_name:.<50} {status}") + + all_passed = all(result for _, result in results) + + if all_passed: + print("\n🎉 All tests passed!") + print("\nGenerated files are in the 'test_output/' directory.") + print("You can import tvshow.nfo into Kodi/Plex/Jellyfin to verify compatibility.") + else: + print("\n⚠️ Some tests failed. Check the output above for details.") + return 1 + + return 0 + + +if __name__ == "__main__": + exit_code = asyncio.run(main()) + sys.exit(exit_code)