Compare commits
25 Commits
cf5a06af11
...
v1.1.15
| Author | SHA1 | Date | |
|---|---|---|---|
| 6c9605e896 | |||
| 3947f6d266 | |||
| a3176f5ac1 | |||
| 9a81b04b65 | |||
| a336733ea9 | |||
| ca93bb740a | |||
| d5e955a731 | |||
| e2a373816a | |||
| a115215416 | |||
| c579235af0 | |||
| 0ba2587bc8 | |||
| bda1fe4445 | |||
| 810346bc8b | |||
| daa937bcb7 | |||
| 1c505bd722 | |||
| 3551838887 | |||
| 9a20541598 | |||
| 3f7651404d | |||
| bee24406e6 | |||
| 31eb0026cf | |||
| 24ea12bbaf | |||
| e74b602f60 | |||
| db65e28854 | |||
| 11e231a4ab | |||
| a11f8c4fa0 |
@@ -2,12 +2,13 @@ FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install system dependencies for compiled Python packages
|
||||
# Install system dependencies for compiled Python packages and ffmpeg for HLS support
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
gcc \
|
||||
g++ \
|
||||
libffi-dev \
|
||||
ffmpeg \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install Python dependencies (cached layer)
|
||||
|
||||
@@ -1 +1 @@
|
||||
v1.1.11
|
||||
v1.1.15
|
||||
|
||||
@@ -198,6 +198,14 @@ start_vpn() {
|
||||
echo "[vpn] DNS set to: ${VPN_DNS}"
|
||||
fi
|
||||
|
||||
# Add explicit host route for the health-check target so it is picked up by
|
||||
# the 'lookup main suppress_prefixlength 0' rule (same as DNS servers above).
|
||||
# Without this, CHECK_HOST falls through to the VPN table default route whose
|
||||
# source-address selection can be defeated by the priority-100 'from ETH0_IP'
|
||||
# policy rule, causing pings to bypass wg0 and be dropped by the kill switch.
|
||||
ip -4 route add "${CHECK_HOST}" dev "$INTERFACE" 2>/dev/null || true
|
||||
echo "[vpn] Health-check route: ${CHECK_HOST} → ${INTERFACE}"
|
||||
|
||||
echo "[vpn] WireGuard interface ${INTERFACE} is up."
|
||||
echo "[vpn] Main routes:"
|
||||
ip route show | sed 's/^/[vpn] /'
|
||||
@@ -249,9 +257,21 @@ health_loop() {
|
||||
echo "[health] VPN recovered."
|
||||
failures=0
|
||||
fi
|
||||
# Secondary DNS check
|
||||
if ping -c 1 -W 5 "google.com" > /dev/null 2>&1; then
|
||||
: # DNS OK — silent
|
||||
else
|
||||
echo "[health] WARN google.com unreachable — possible DNS issue"
|
||||
fi
|
||||
else
|
||||
failures=$((failures + 1))
|
||||
echo "[health] Check failed ($failures/$max_failures) — ping ${CHECK_HOST} failed"
|
||||
# Secondary check: distinguish IP failure from DNS failure
|
||||
if ping -c 1 -W 5 "google.com" > /dev/null 2>&1; then
|
||||
echo "[health] INFO google.com reachable — DNS works, ${CHECK_HOST} may be filtered"
|
||||
else
|
||||
echo "[health] INFO google.com also unreachable — DNS or general routing failure"
|
||||
fi
|
||||
# Dump WireGuard stats to show if handshake is stale and how much data flows
|
||||
echo "[health] wg stats:"
|
||||
wg show "$INTERFACE" 2>/dev/null | grep -E 'latest handshake|transfer|endpoint' | sed 's/^/[health] /' || echo "[health] wg0 not found"
|
||||
|
||||
@@ -15,6 +15,7 @@ services:
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
- SYS_MODULE
|
||||
- NET_RAW
|
||||
sysctls:
|
||||
- net.ipv4.ip_forward=1
|
||||
- net.ipv4.conf.all.src_valid_mark=1
|
||||
@@ -22,7 +23,7 @@ services:
|
||||
- /server/server_aniworld/wg0.conf:/etc/wireguard/wg0.conf:ro
|
||||
- /lib/modules:/lib/modules:ro
|
||||
ports:
|
||||
- "2000:8000"
|
||||
- "8000:8000"
|
||||
environment:
|
||||
- HEALTH_CHECK_INTERVAL=10
|
||||
- HEALTH_CHECK_HOST=1.1.1.1
|
||||
@@ -51,4 +52,5 @@ services:
|
||||
volumes:
|
||||
- /server/server_aniworld/data:/app/data
|
||||
- /server/server_aniworld/logs:/app/logs
|
||||
- /media/serien/Serien:/data
|
||||
restart: unless-stopped
|
||||
|
||||
@@ -41,6 +41,15 @@ This changelog follows [Keep a Changelog](https://keepachangelog.com/) principle
|
||||
|
||||
### Added
|
||||
|
||||
- **Encoding detection for HTML parsing** (`src/core/providers/aniworld_provider.py`):
|
||||
Added `_decode_html_content()` function that uses `chardet` to detect the actual
|
||||
encoding of HTML content before parsing. Falls back to UTF-8 with `errors='replace'`
|
||||
to handle pages with mismatched encoding declarations. Applied to all BeautifulSoup
|
||||
parsing calls to prevent "Some characters could not be decoded" warnings.
|
||||
- **chardet dependency**: Added `chardet>=5.2.0` to `requirements.txt` for encoding detection.
|
||||
|
||||
### Added
|
||||
|
||||
- **Temp file cleanup after every download** (`src/core/providers/aniworld_provider.py`,
|
||||
`src/core/providers/enhanced_provider.py`): Module-level helper
|
||||
`_cleanup_temp_file()` removes the working temp file and any yt-dlp `.part`
|
||||
|
||||
@@ -61,4 +61,340 @@ This document provides guidance for developers working on the Aniworld project.
|
||||
- Commit message format
|
||||
- Pull request process
|
||||
8. Common Development Tasks
|
||||
|
||||
### Adding Queue Deduplication
|
||||
|
||||
The download queue prevents duplicate entries at two levels:
|
||||
|
||||
**In-Memory Deduplication** (`src/server/services/download_service.py`):
|
||||
- `_pending_by_episode` dict tracks pending episodes: key = `(serie_id, season, episode)`
|
||||
- `_add_to_pending_queue()` updates the dict when adding items
|
||||
- `add_to_queue()` checks this dict before adding episodes (includes batch-local dedup)
|
||||
- `_remove_from_pending_queue()` cleans up the dict when items are removed
|
||||
|
||||
**Database Constraint** (`src/server/models.py`):
|
||||
- `DownloadQueueItem` has a unique index on `episode_id` via `__table_args__`
|
||||
- Prevents duplicate queue entries at the database level
|
||||
- Unique constraint: `Index("ix_download_queue_episode_pending", "episode_id", unique=True)`
|
||||
|
||||
**Scheduler Cooldown** (`src/server/services/scheduler_service.py`):
|
||||
- `_last_auto_download_time` tracks when auto-download last ran
|
||||
- 5-minute cooldown prevents rapid re-triggers
|
||||
- Checked at start of `_auto_download_missing()`
|
||||
|
||||
### Episode Lifecycle
|
||||
|
||||
Episodes transition through states stored in the `episodes` table:
|
||||
|
||||
| State | `is_downloaded` | `file_path` | Description |
|
||||
|-------|----------------|-------------|-------------|
|
||||
| Missing | `False` | `NULL` | Episode not yet downloaded |
|
||||
| Downloaded | `True` | Set | Episode exists on disk |
|
||||
|
||||
**State Transitions:**
|
||||
1. **Missing → Downloaded**: When download completes, `_remove_episode_from_missing_list()` calls `EpisodeService.mark_downloaded()` to set `is_downloaded=True` and populate `file_path`. The episode record is NOT deleted.
|
||||
|
||||
**Query Implications:**
|
||||
- `get_series_with_missing_episodes()`: Filters for `is_downloaded=False` to find series with undownloaded episodes
|
||||
- `get_series_with_no_episodes()`: Finds series with `is_downloaded=False` episodes but NO `is_downloaded=True` episodes (completely unwatched series)
|
||||
|
||||
### Mocking the Download Queue
|
||||
|
||||
When testing components that use the download queue:
|
||||
|
||||
```python
|
||||
# Mock repository for unit tests
|
||||
class MockQueueRepository:
|
||||
def __init__(self):
|
||||
self._items: Dict[str, DownloadItem] = {}
|
||||
|
||||
async def save_item(self, item: DownloadItem) -> DownloadItem:
|
||||
self._items[item.id] = item
|
||||
return item
|
||||
|
||||
async def get_all_items(self) -> List[DownloadItem]:
|
||||
return list(self._items.values())
|
||||
|
||||
# Use in fixture
|
||||
@pytest.fixture
|
||||
def mock_queue_repository():
|
||||
return MockQueueRepository()
|
||||
|
||||
@pytest.fixture
|
||||
def download_service(mock_anime_service, mock_queue_repository):
|
||||
return DownloadService(
|
||||
anime_service=mock_anime_service,
|
||||
queue_repository=mock_queue_repository,
|
||||
max_retries=3,
|
||||
)
|
||||
```
|
||||
|
||||
9. Troubleshooting Development Issues
|
||||
|
||||
### Async Context Managers for aiohttp
|
||||
|
||||
All `aiohttp.ClientSession` usages must be wrapped in `async with`:
|
||||
|
||||
```python
|
||||
# Correct — session properly closed on exit
|
||||
async with TMDBClient(api_key="key") as client:
|
||||
result = await client.search_tv_show("Show")
|
||||
|
||||
# Wrong — session may leak if exception occurs
|
||||
client = TMDBClient(api_key="key")
|
||||
result = await client.search_tv_show("Show")
|
||||
await client.close() # May not be called if exception raised earlier
|
||||
```
|
||||
|
||||
**Why:**
|
||||
- `aiohttp.ClientSession` holds TCP connections that must be explicitly closed
|
||||
- If exception occurs before `close()`, session leaks
|
||||
- Context manager guarantees `__aexit__` runs even on exceptions
|
||||
|
||||
**Services that use aiohttp:**
|
||||
- `TMDBClient` — has `__aenter__`/`__aexit__`, use `async with`
|
||||
- `ImageDownloader` — has `__aenter__`/`__aexit__`, use `async with`
|
||||
- `NFOService` — wraps both above, use `async with`
|
||||
|
||||
**Verification:**
|
||||
- Missing context manager usage triggers `__del__` warning on garbage collection
|
||||
- Integration tests verify no "Unclosed client session" errors in logs
|
||||
|
||||
### Scheduler Persistence and Recovery
|
||||
|
||||
APScheduler stores jobs in `data/scheduler.db` (SQLite) so they survive process restarts:
|
||||
|
||||
```python
|
||||
from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
|
||||
|
||||
jobstores = {
|
||||
"default": SQLAlchemyJobStore(url="sqlite:///./data/scheduler.db"),
|
||||
}
|
||||
scheduler = AsyncIOScheduler(jobstores=jobstores)
|
||||
```
|
||||
|
||||
**Grace period:** `misfire_grace_time=3600` (1 hour). If server is down at scheduled time and restarts within 1 hour, missed job runs automatically via APScheduler coalesce behavior.
|
||||
|
||||
**Startup recovery:** On `start()`, scheduler loads persisted jobs from DB. APScheduler handles missed jobs internally when `coalesce=True`.
|
||||
|
||||
**Health endpoint:** `GET /health` returns `scheduler_next_run` and `scheduler_last_run` for external monitors (Uptime Kuma, Prometheus, etc.).
|
||||
|
||||
**If server is down >1 hour:** No automatic recovery. Manual trigger via `POST /api/scheduler/trigger-rescan` or wait for next scheduled run.
|
||||
|
||||
### Health Check Endpoints
|
||||
|
||||
The application provides health check endpoints for monitoring and container orchestration:
|
||||
|
||||
#### `GET /health`
|
||||
Basic health check returning service status and startup health check results.
|
||||
|
||||
**Response fields:**
|
||||
- `status`: "healthy", "degraded", or "unhealthy" based on startup checks
|
||||
- `timestamp`: ISO timestamp of the check
|
||||
- `series_app_initialized`: Whether the series app is loaded
|
||||
- `anime_directory_configured`: Whether anime_directory is set
|
||||
- `scheduler_next_run` / `scheduler_last_run`: Scheduler times
|
||||
- `checks`: Detailed startup check results (ffmpeg, DNS, anime_directory)
|
||||
|
||||
#### `GET /health/ready`
|
||||
Readiness check for container orchestrators (Kubernetes, Docker Swarm).
|
||||
|
||||
**Response when ready:**
|
||||
```json
|
||||
{
|
||||
"status": "ready",
|
||||
"ready": true,
|
||||
"timestamp": "2024-01-01T00:00:00",
|
||||
"checks": {...}
|
||||
}
|
||||
```
|
||||
|
||||
**Response when not ready (503):**
|
||||
```json
|
||||
{
|
||||
"status": "not_ready",
|
||||
"ready": false,
|
||||
"timestamp": "2024-01-01T00:00:00",
|
||||
"critical_failures": ["anime_directory: not configured"],
|
||||
"checks": {...}
|
||||
}
|
||||
```
|
||||
|
||||
#### `GET /health/detailed`
|
||||
Comprehensive health check including database, filesystem, and system metrics.
|
||||
|
||||
#### Startup Health Checks
|
||||
|
||||
On application startup, the following checks are performed:
|
||||
|
||||
| Check | Failure Status | Impact |
|
||||
|-------|---------------|--------|
|
||||
| `ffmpeg` | warning | HLS downloads may fail |
|
||||
| `dns_aniworld` | warning | Provider requests may fail |
|
||||
| `dns_tmdb` | warning | TMDB API calls may fail |
|
||||
| `anime_directory` | error | Download service disabled |
|
||||
|
||||
DNS checks are warnings because failures can be transient. anime_directory errors disable the download service to prevent failures.
|
||||
|
||||
### Troubleshooting Development Issues
|
||||
|
||||
#### Scheduler missed a run
|
||||
|
||||
1. Server was down at scheduled time (03:00 UTC by default).
|
||||
2. Check `data/scheduler.db` exists — if not, jobs are not persisted.
|
||||
3. If server was down >1 hour, missed job is dropped (misfire window exceeded).
|
||||
4. Trigger manually: `POST /api/scheduler/trigger-rescan`
|
||||
5. Monitor next run: `GET /health` → `scheduler_next_run`
|
||||
6. If problem repeats, increase `misfire_grace_time` in `scheduler_service.py`.
|
||||
|
||||
#### Scheduler not firing (no events at scheduled time)
|
||||
|
||||
If the scheduler appears configured but never triggers:
|
||||
|
||||
1. **Verify scheduler.db contains the job:**
|
||||
```bash
|
||||
sqlite3 data/scheduler.db "SELECT id, next_run_time FROM apscheduler_jobs;"
|
||||
```
|
||||
- `next_run_time` should be in the future
|
||||
- If it's in the past, the server was down when the job should have fired
|
||||
|
||||
2. **Check application logs for scheduler startup:**
|
||||
```
|
||||
grep "Scheduler service started" fastapi_app.log
|
||||
```
|
||||
- If missing, the scheduler failed to start — check for errors above this line
|
||||
- If present, scheduler started successfully
|
||||
|
||||
3. **Verify APScheduler events in logs:**
|
||||
```
|
||||
grep "apscheduler.executors.default" fastapi_app.log
|
||||
```
|
||||
- `Running job` = job triggered
|
||||
- `executed successfully` = job completed
|
||||
- No output = job never fired
|
||||
|
||||
4. **Test manual trigger:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8000/api/scheduler/trigger-rescan -H "Authorization: Bearer <token>"
|
||||
```
|
||||
- If manual trigger works but cron doesn't, the issue is APScheduler configuration
|
||||
|
||||
5. **Check next_run_time via health endpoint:**
|
||||
```bash
|
||||
curl http://localhost:8000/health | jq .scheduler_next_run
|
||||
```
|
||||
- If `null`, the job is not scheduled
|
||||
- If set, the scheduler knows when to run next
|
||||
|
||||
6. **Check timezone handling:**
|
||||
- APScheduler uses UTC internally
|
||||
- The schedule_time config (e.g., "03:00") is interpreted as UTC
|
||||
- If you expect local time, adjust the schedule_time accordingly
|
||||
|
||||
#### Startup health check failures
|
||||
|
||||
If `/health` returns `unhealthy` status:
|
||||
|
||||
1. **anime_directory error**: Directory not configured or not writable
|
||||
- Check `ANIME_DIRECTORY` environment variable
|
||||
- Verify directory exists and permissions allow write access
|
||||
- Download service will not initialize until resolved
|
||||
|
||||
2. **ffmpeg warning**: ffmpeg not found in PATH
|
||||
- HLS stream downloads will fail
|
||||
- Install ffmpeg: `apt install ffmpeg` or `brew install ffmpeg`
|
||||
|
||||
3. **DNS warnings**: Domain resolution failed
|
||||
- Check network connectivity
|
||||
- DNS failures are transient — warnings don't block startup
|
||||
- Retry later to verify: `GET /health`
|
||||
|
||||
### Provider Failure Handling
|
||||
|
||||
Download providers (VOE, Doodstream, Vidmoly, Vidoza, SpeedFiles, Streamtape,
|
||||
Luluvdo) regularly break: URLs expire, sites change their player markup, geo
|
||||
blocks appear, and `yt-dlp` extractors lag behind upstream changes. The
|
||||
`AniworldLoader.download()` flow is designed to fail fast and rotate.
|
||||
|
||||
**Rotation order**
|
||||
|
||||
1. The episode page is scraped for the providers AniWorld actually advertises.
|
||||
2. Results are ordered by the preference in `DEFAULT_PROVIDERS`
|
||||
(`provider_config.py`); providers not listed run last.
|
||||
3. For each candidate the loader:
|
||||
1. Calls `_check_url_alive()` — HEAD probe with GET fallback. Any 4xx
|
||||
response or connection error skips the provider immediately.
|
||||
2. Resolves the redirect via `_resolve_direct_link()` to obtain a direct
|
||||
stream URL plus headers. Provider-specific extractors (e.g. `VOE`) are
|
||||
preferred; unknown providers fall back to the embed URL so `yt-dlp` can
|
||||
attempt extraction.
|
||||
3. Tries `_try_direct_stream()` — straight `requests.get(stream=True)` when
|
||||
`Content-Type` is `video/*` or `application/octet-stream`. This avoids
|
||||
`yt-dlp` entirely for direct MP4 links.
|
||||
4. Falls back to `yt-dlp` with the ffmpeg downloader for HLS streams.
|
||||
4. On any failure, temp files are cleaned and the loop moves to the next
|
||||
provider. When the chain is exhausted, the loader logs
|
||||
`All download providers failed for S{season}E{episode} ...; tried=[...]`
|
||||
to both the application log and `logs/download_errors.log`.
|
||||
|
||||
**Do not hardcode provider URLs.** Provider domains shift constantly (e.g.
|
||||
Doodstream alternates between `dood.li`, `dood.so`, `dood.la`). Only the
|
||||
referer hints in `PROVIDER_HEADERS` are persisted — discovery still happens
|
||||
at runtime through AniWorld's redirect endpoint.
|
||||
|
||||
### HLS Stream Handling
|
||||
|
||||
HLS (HTTP Live Streaming) manifests (`.m3u8`) require yt-dlp to use the
|
||||
`ffmpeg` downloader with `--hls-use-mpegts`. Both providers configure this
|
||||
automatically:
|
||||
|
||||
```python
|
||||
ydl_opts = {
|
||||
"downloader": "ffmpeg", # Use ffmpeg instead of native
|
||||
"hls_use_mpegts": True, # Write transport stream (.ts) segments
|
||||
}
|
||||
```
|
||||
|
||||
**Why this matters**: Without ffmpeg, yt-dlp logs:
|
||||
`"Live HLS streams are not supported by the native downloader"`
|
||||
|
||||
**Requirements**:
|
||||
- ffmpeg must be installed and in PATH (`which ffmpeg`)
|
||||
- Install: `apt install ffmpeg` (Debian/Ubuntu) or `brew install ffmpeg` (macOS)
|
||||
- Startup health check (see Health Check Endpoints) verifies ffmpeg presence
|
||||
|
||||
**Trade-offs**:
|
||||
- HLS downloads are slower than direct MP4 (reassembly of .ts segments)
|
||||
- Requires more disk space during download
|
||||
- May need post-processing if .ts format is not desired
|
||||
|
||||
**Detection**: VOE provider extracts HLS URLs via `HLS_PATTERN` regex. Other
|
||||
providers let yt-dlp auto-detect from URL/content-type.
|
||||
|
||||
### Updating yt-dlp
|
||||
|
||||
When extractors break (typical symptoms: every provider HEAD probe succeeds
|
||||
but `yt-dlp` raises `Unable to extract` or `HTTP Error 404`):
|
||||
|
||||
1. Check the upstream tracker first: https://github.com/yt-dlp/yt-dlp/issues
|
||||
2. Upgrade in the conda environment:
|
||||
```bash
|
||||
conda run -n AniWorld pip install --upgrade yt-dlp
|
||||
```
|
||||
3. Smoke-test against a known-good episode before pinning a new floor in
|
||||
`requirements.txt` (`yt-dlp>=YYYY.MM.DD`).
|
||||
4. Re-run the provider test suite:
|
||||
```bash
|
||||
conda run -n AniWorld python -m pytest tests/unit/test_aniworld_provider.py -v
|
||||
```
|
||||
5. If a specific extractor is removed upstream, drop the provider from
|
||||
`DEFAULT_PROVIDERS` rather than patching `yt-dlp` in tree.
|
||||
|
||||
### User Notification on Total Failure
|
||||
|
||||
`SeriesApp.download_episode()` already emits a `download_status="failed"`
|
||||
WebSocket event when `loader.download()` returns `False`. Operators should
|
||||
forward this to `notification_service.notify_download_failed()` so users see
|
||||
a HIGH-priority alert. The loader keeps the failure detail in
|
||||
`logs/download_errors.log` for post-mortem.
|
||||
|
||||
|
||||
@@ -171,6 +171,35 @@ Response:
|
||||
}
|
||||
```
|
||||
|
||||
### 3.6 Fallback Behavior When TMDB is Unavailable
|
||||
|
||||
When TMDB lookup fails (network issues, API errors, or no match found), the system creates a **minimal NFO** to ensure the series is still tracked. This behavior applies to:
|
||||
|
||||
- Manual NFO creation via API
|
||||
- Batch NFO creation operations
|
||||
- Automatic NFO creation during downloads
|
||||
|
||||
**What a minimal NFO contains:**
|
||||
|
||||
```xml
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<tvshow>
|
||||
<title>Series Name</title>
|
||||
<year>2024</year>
|
||||
<plot>No metadata available for Series Name. TMDB lookup failed.</plot>
|
||||
</tvshow>
|
||||
```
|
||||
|
||||
**Limitations of minimal NFOs:**
|
||||
- No poster, logo, or fanart images
|
||||
- No rating, genre, or studio information
|
||||
- No TMDB or other provider IDs
|
||||
- May not display correctly in some media servers
|
||||
|
||||
**To upgrade a minimal NFO:**
|
||||
1. Use the Update endpoint (`PUT /api/nfo/{serie_id}/update`) when TMDB is available
|
||||
2. Or delete the NFO and recreate it with full metadata
|
||||
|
||||
---
|
||||
|
||||
## 4. File Structure
|
||||
|
||||
@@ -62,6 +62,90 @@ This document describes the testing strategy, guidelines, and practices for the
|
||||
- What to mock
|
||||
- Mock patterns
|
||||
- External service mocks
|
||||
|
||||
### Mocking the Download Queue
|
||||
|
||||
Use `MockQueueRepository` for testing download queue functionality:
|
||||
|
||||
```python
|
||||
from src.server.models.download import DownloadItem, EpisodeIdentifier
|
||||
|
||||
class MockQueueRepository:
|
||||
def __init__(self):
|
||||
self._items: Dict[str, DownloadItem] = {}
|
||||
|
||||
async def save_item(self, item: DownloadItem) -> DownloadItem:
|
||||
self._items[item.id] = item
|
||||
return item
|
||||
|
||||
async def get_item(self, item_id: str) -> Optional[DownloadItem]:
|
||||
return self._items.get(item_id)
|
||||
|
||||
async def get_all_items(self) -> List[DownloadItem]:
|
||||
return list(self._items.values())
|
||||
|
||||
async def set_error(self, item_id: str, error: str) -> bool:
|
||||
if item_id in self._items:
|
||||
self._items[item_id].error = error
|
||||
return True
|
||||
return False
|
||||
|
||||
async def delete_item(self, item_id: str) -> bool:
|
||||
if item_id in self._items:
|
||||
del self._items[item_id]
|
||||
return True
|
||||
return False
|
||||
|
||||
async def clear_all(self) -> int:
|
||||
count = len(self._items)
|
||||
self._items.clear()
|
||||
return count
|
||||
```
|
||||
|
||||
**Key points:**
|
||||
- The mock uses in-memory storage, no database required
|
||||
- All async methods are implemented (even if just pass-through)
|
||||
- `save_item` uses `item.id` as key (must be set before calling)
|
||||
- Suitable for unit tests only (no persistence)
|
||||
|
||||
### Mocking aiohttp Sessions
|
||||
|
||||
When testing code that uses `aiohttp.ClientSession`:
|
||||
|
||||
```python
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from aiohttp import ClientSession
|
||||
|
||||
# Mock aiohttp session for testing
|
||||
class MockAiohttpSession:
|
||||
def __init__(self):
|
||||
self.closed = False
|
||||
|
||||
async def close(self):
|
||||
self.closed = True
|
||||
|
||||
def get(self, url, **kwargs):
|
||||
mock_response = AsyncMock()
|
||||
mock_response.status = 200
|
||||
mock_response.json = AsyncMock(return_value={"data": "test"})
|
||||
mock_response.__aenter__ = AsyncMock(return_value=mock_response)
|
||||
mock_response.__aexit__ = AsyncMock(return_value=None)
|
||||
return mock_response
|
||||
|
||||
# Use in fixture
|
||||
@pytest.fixture
|
||||
async def mock_tmdb_session():
|
||||
session = MockAiohttpSession()
|
||||
yield session
|
||||
# Cleanup verification
|
||||
assert session.closed, "Session was not closed"
|
||||
```
|
||||
|
||||
**Key points:**
|
||||
- Always verify `session.closed` is `True` after context manager exits
|
||||
- Mock `__aenter__` and `__aexit__` for response context managers
|
||||
- Set `closed = False` on mock session for unclosed warning tests
|
||||
|
||||
7. Coverage Requirements
|
||||
8. CI/CD Integration
|
||||
9. Writing Good Tests
|
||||
|
||||
154
docs/runner.csx
Normal file
154
docs/runner.csx
Normal file
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env dotnet-script
|
||||
#nullable enable
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Diagnostics;
|
||||
using System.Threading;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Linq;
|
||||
using System.Collections.Generic;
|
||||
|
||||
// ── Ctrl+C: kill active process and exit cleanly ──────────────────────────────
|
||||
var cts = new CancellationTokenSource();
|
||||
Process? activeProcess = null;
|
||||
|
||||
Console.CancelKeyPress += (_, e) =>
|
||||
{
|
||||
e.Cancel = true;
|
||||
Console.WriteLine("\n[runner] Interrupted — shutting down...");
|
||||
cts.Cancel();
|
||||
try { activeProcess?.Kill(entireProcessTree: true); } catch { }
|
||||
};
|
||||
|
||||
// ── Paths ─────────────────────────────────────────────────────────────────────
|
||||
var repoRoot = Directory.GetCurrentDirectory();
|
||||
var tasksFile = Path.Combine(repoRoot, "Docs", "Tasks.md");
|
||||
|
||||
if (!File.Exists(tasksFile))
|
||||
{
|
||||
Console.Error.WriteLine($"[runner] ERROR: Tasks.md not found at {tasksFile}");
|
||||
Console.Error.WriteLine("[runner] Run this script from the repository root.");
|
||||
Environment.Exit(1);
|
||||
}
|
||||
|
||||
// ── Read & split by "---" separator lines ────────────────────────────────────
|
||||
var content = File.ReadAllText(tasksFile);
|
||||
var items = Regex
|
||||
.Split(content, @"\r?\n---\r?\n")
|
||||
.Select(s => s.Trim())
|
||||
.Where(s => s.Length > 0)
|
||||
.ToList();
|
||||
|
||||
Console.WriteLine($"[runner] Found {items.Count} section(s) in Tasks.md");
|
||||
|
||||
// ── Helper: run copilot and stream output, return full output ─────────────────
|
||||
async Task<string> RunCopilot(IEnumerable<string> extraArgs, string prompt)
|
||||
{
|
||||
var output = new StringBuilder();
|
||||
|
||||
var argList = new List<string> { "launch", "copilot", "--model", "minimax-m2.7:cloud", "--yes", "--", "--allow-all-tools" };
|
||||
argList.AddRange(extraArgs);
|
||||
argList.Add("-p");
|
||||
argList.Add(prompt);
|
||||
|
||||
var psi = new ProcessStartInfo("ollama")
|
||||
{
|
||||
WorkingDirectory = repoRoot,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
};
|
||||
foreach (var a in argList)
|
||||
psi.ArgumentList.Add(a);
|
||||
|
||||
activeProcess = new Process { StartInfo = psi };
|
||||
|
||||
activeProcess.OutputDataReceived += (_, e) =>
|
||||
{
|
||||
if (e.Data is null) return;
|
||||
Console.WriteLine(e.Data);
|
||||
output.AppendLine(e.Data);
|
||||
};
|
||||
activeProcess.ErrorDataReceived += (_, e) =>
|
||||
{
|
||||
if (e.Data is null) return;
|
||||
Console.Error.WriteLine(e.Data);
|
||||
output.AppendLine(e.Data);
|
||||
};
|
||||
|
||||
activeProcess.Start();
|
||||
activeProcess.BeginOutputReadLine();
|
||||
activeProcess.BeginErrorReadLine();
|
||||
|
||||
await activeProcess.WaitForExitAsync(cts.Token);
|
||||
activeProcess = null;
|
||||
|
||||
return output.ToString();
|
||||
}
|
||||
|
||||
// ── Main loop ─────────────────────────────────────────────────────────────────
|
||||
for (int i = 0; i < items.Count; i++)
|
||||
{
|
||||
var item = items[i];
|
||||
if (cts.IsCancellationRequested) break;
|
||||
|
||||
Console.WriteLine();
|
||||
Console.WriteLine("[runner] ══════════════════════════════════════════════");
|
||||
Console.WriteLine($"[runner] Task:\n{item}");
|
||||
Console.WriteLine("[runner] ══════════════════════════════════════════════");
|
||||
Console.WriteLine();
|
||||
|
||||
// Step 1 — run the task prompt
|
||||
await RunCopilot(Enumerable.Empty<string>(), $"/caveman full");
|
||||
await RunCopilot(new[] { "--continue" }, $"read ./Docs/instructions.md. {item}");
|
||||
if (cts.IsCancellationRequested) break;
|
||||
|
||||
// Step 2 — confirm completion in the same chat session
|
||||
Console.WriteLine("\n[runner] Asking for task confirmation...\n");
|
||||
var confirmation = await RunCopilot(
|
||||
new[] { "--continue" },
|
||||
"are you sure tasks is done. reply with yes"
|
||||
);
|
||||
if (cts.IsCancellationRequested) break;
|
||||
|
||||
// Step 3 — check for "yes" in the reply, with retry logic for issue resolution
|
||||
int maxRetries = 3;
|
||||
int retryCount = 0;
|
||||
bool taskConfirmed = confirmation.Contains("yes", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
while (!taskConfirmed && retryCount < maxRetries)
|
||||
{
|
||||
retryCount++;
|
||||
Console.WriteLine($"\n[runner] Attempt {retryCount}/{maxRetries}: Resolving remaining issues and running tests...\n");
|
||||
|
||||
confirmation = await RunCopilot(
|
||||
new[] { "--continue" },
|
||||
"resolve any remaining issues, make sure all tests are running and pass. then confirm with yes if done"
|
||||
);
|
||||
if (cts.IsCancellationRequested) break;
|
||||
|
||||
taskConfirmed = confirmation.Contains("yes", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
if (!taskConfirmed)
|
||||
{
|
||||
Console.WriteLine($"\n[runner] Task not confirmed as done after {maxRetries} attempts. Stopping.");
|
||||
break;
|
||||
}
|
||||
|
||||
// Step 4 — commit the work
|
||||
Console.WriteLine("\n[runner] Task confirmed. Making git commit...\n");
|
||||
|
||||
await RunCopilot(Enumerable.Empty<string>(), $"/caveman full");
|
||||
await RunCopilot(new[] { "--continue" }, "make git commit");
|
||||
if (cts.IsCancellationRequested) break;
|
||||
|
||||
// Step 5 — remove completed task from Tasks.md
|
||||
var remaining = items.Skip(i + 1).ToList();
|
||||
File.WriteAllText(tasksFile, string.Join("\n\n---\n\n", remaining));
|
||||
Console.WriteLine("[runner] Removed completed task from Tasks.md");
|
||||
}
|
||||
|
||||
Console.WriteLine("\n[runner] Finished.");
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "aniworld-web",
|
||||
"version": "1.1.11",
|
||||
"version": "1.1.15",
|
||||
"description": "Aniworld Anime Download Manager - Web Frontend",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
@@ -22,6 +22,7 @@ APScheduler>=3.10.4
|
||||
Events>=0.5
|
||||
requests>=2.31.0
|
||||
beautifulsoup4>=4.12.0
|
||||
chardet>=5.2.0
|
||||
fake-useragent>=1.4.0
|
||||
yt-dlp>=2024.1.0
|
||||
urllib3>=2.0.0
|
||||
135
scripts/migrate_populate_year_from_folder.py
Normal file
135
scripts/migrate_populate_year_from_folder.py
Normal file
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Migration script to populate year for existing series from folder names.
|
||||
|
||||
This script:
|
||||
1. Finds all series in the database with year=NULL
|
||||
2. Extracts year from their folder names using the same pattern as SerieScanner
|
||||
3. Updates the database records
|
||||
|
||||
Usage:
|
||||
python scripts/migrate_populate_year_from_folder.py [--dry-run]
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from src.server.database.models import AnimeSeries
|
||||
from src.server.database.service import DatabaseSession
|
||||
|
||||
|
||||
def extract_year_from_folder_name(folder_name: str) -> int | None:
|
||||
"""Extract year from folder name if present.
|
||||
|
||||
Same logic as SerieScanner._extract_year_from_folder_name.
|
||||
|
||||
Args:
|
||||
folder_name: The folder name to check
|
||||
|
||||
Returns:
|
||||
int or None: Year if found, None otherwise
|
||||
"""
|
||||
if not folder_name:
|
||||
return None
|
||||
|
||||
# Look for year in format (YYYY) - typically at end of name
|
||||
match = re.search(r'\((\d{4})\)', folder_name)
|
||||
if match:
|
||||
try:
|
||||
year = int(match.group(1))
|
||||
# Validate year is reasonable (between 1900 and 2100)
|
||||
if 1900 <= year <= 2100:
|
||||
return year
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
async def migrate_year_from_folder(dry_run: bool = True) -> tuple[int, int]:
|
||||
"""Migrate year field for existing series.
|
||||
|
||||
Args:
|
||||
dry_run: If True, only report what would be changed
|
||||
|
||||
Returns:
|
||||
Tuple of (updated_count, skipped_count)
|
||||
"""
|
||||
updated_count = 0
|
||||
skipped_count = 0
|
||||
|
||||
async with DatabaseSession() as db:
|
||||
# Find all series with NULL year
|
||||
result = await db.execute(
|
||||
select(AnimeSeries).where(AnimeSeries.year.is_(None))
|
||||
)
|
||||
series_list = result.scalars().all()
|
||||
|
||||
print(f"Found {len(series_list)} series with year=NULL")
|
||||
|
||||
for series in series_list:
|
||||
year_from_folder = extract_year_from_folder_name(series.folder)
|
||||
|
||||
if year_from_folder:
|
||||
print(f" {series.folder} -> {year_from_folder}")
|
||||
|
||||
if not dry_run:
|
||||
await db.execute(
|
||||
update(AnimeSeries)
|
||||
.where(AnimeSeries.id == series.id)
|
||||
.values(year=year_from_folder)
|
||||
)
|
||||
|
||||
updated_count += 1
|
||||
else:
|
||||
print(f" {series.folder} -> (no year found)")
|
||||
skipped_count += 1
|
||||
|
||||
return updated_count, skipped_count
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Migrate year from folder name")
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
default=True,
|
||||
help="Show what would be changed without making changes"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--execute",
|
||||
action="store_true",
|
||||
help="Actually execute the migration (disabled by default)"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
dry_run = not args.execute
|
||||
|
||||
if dry_run:
|
||||
print("=== DRY RUN MODE ===")
|
||||
print("No changes will be made. Use --execute to apply changes.\n")
|
||||
|
||||
import asyncio
|
||||
|
||||
try:
|
||||
updated, skipped = asyncio.run(migrate_year_from_folder(dry_run=dry_run))
|
||||
|
||||
print(f"\n{'Would update' if dry_run else 'Updated'}: {updated} series")
|
||||
print(f"Skipped (no year in folder): {skipped} series")
|
||||
|
||||
if dry_run:
|
||||
print("\nRun with --execute to apply these changes.")
|
||||
|
||||
return 0
|
||||
except Exception as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
File diff suppressed because it is too large
Load Diff
@@ -9,6 +9,7 @@ import threading
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
import chardet
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from events import Events
|
||||
@@ -80,6 +81,37 @@ if not download_error_logger.handlers:
|
||||
noKeyFound_logger = logging.getLogger()
|
||||
|
||||
|
||||
def _decode_html_content(content: bytes) -> str:
|
||||
"""Decode HTML content with encoding detection.
|
||||
|
||||
Uses chardet to detect the actual encoding of the content,
|
||||
falling back to utf-8 with replacement error handling.
|
||||
|
||||
Args:
|
||||
content: Raw HTML bytes from the response
|
||||
|
||||
Returns:
|
||||
Decoded string content
|
||||
"""
|
||||
detected = chardet.detect(content)
|
||||
encoding = detected.get('encoding', 'utf-8')
|
||||
confidence = detected.get('confidence', 0)
|
||||
|
||||
if confidence < 0.7:
|
||||
logger.debug(
|
||||
"Low encoding confidence (%.2f) for detected encoding '%s', using utf-8",
|
||||
confidence,
|
||||
encoding
|
||||
)
|
||||
encoding = 'utf-8'
|
||||
|
||||
try:
|
||||
return content.decode(encoding, errors='replace')
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to decode content with %s: %s, using utf-8 replace", encoding, exc)
|
||||
return content.decode('utf-8', errors='replace')
|
||||
|
||||
|
||||
class AniworldLoader(Loader):
|
||||
def __init__(self) -> None:
|
||||
self.SUPPORTED_PROVIDERS = DEFAULT_PROVIDERS
|
||||
@@ -231,7 +263,7 @@ class AniworldLoader(Loader):
|
||||
language_code = self._get_language_key(language)
|
||||
|
||||
episode_soup = BeautifulSoup(
|
||||
self._get_episode_html(season, episode, key).content,
|
||||
_decode_html_content(self._get_episode_html(season, episode, key).content),
|
||||
'html.parser'
|
||||
)
|
||||
change_language_box_div = episode_soup.find(
|
||||
@@ -249,6 +281,118 @@ class AniworldLoader(Loader):
|
||||
logger.debug("Available languages for S%02dE%03d: %s, requested: %s, available: %s", season, episode, languages, language_code, is_available)
|
||||
return is_available
|
||||
|
||||
def _check_url_alive(
|
||||
self,
|
||||
url: str,
|
||||
headers: dict | None = None,
|
||||
timeout: int = 10,
|
||||
) -> bool:
|
||||
"""Probe a provider URL with HEAD before committing to yt-dlp.
|
||||
|
||||
Skips dead providers quickly so the failover loop never blocks
|
||||
waiting for yt-dlp to fail on a 404. Falls back to a streaming
|
||||
GET when HEAD is not allowed by the upstream server.
|
||||
|
||||
Args:
|
||||
url: URL to probe.
|
||||
headers: Optional headers to forward with the probe.
|
||||
timeout: Per-request timeout (seconds).
|
||||
|
||||
Returns:
|
||||
True when the URL responds with a non-4xx status, else False.
|
||||
"""
|
||||
try:
|
||||
response = self.session.head(
|
||||
url,
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
allow_redirects=True,
|
||||
)
|
||||
if response.status_code == 405:
|
||||
response = self.session.get(
|
||||
url,
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
stream=True,
|
||||
allow_redirects=True,
|
||||
)
|
||||
response.close()
|
||||
if 400 <= response.status_code < 500:
|
||||
logger.warning(
|
||||
"Provider URL returned HTTP %s: %s",
|
||||
response.status_code, url
|
||||
)
|
||||
return False
|
||||
return True
|
||||
except requests.RequestException as exc:
|
||||
logger.warning("Provider URL unreachable %s: %s", url, exc)
|
||||
return False
|
||||
|
||||
def _try_direct_stream(
|
||||
self,
|
||||
link: str,
|
||||
output_path: str,
|
||||
headers: dict | None,
|
||||
timeout: int,
|
||||
) -> bool:
|
||||
"""Stream a direct video URL to disk without yt-dlp.
|
||||
|
||||
Used as a fast-path when the resolved provider link already points
|
||||
at a downloadable video file (``Content-Type: video/*`` or
|
||||
``application/octet-stream``). HLS and other non-video payloads
|
||||
are rejected so the caller can fall back to yt-dlp.
|
||||
|
||||
Args:
|
||||
link: Direct download URL.
|
||||
output_path: Destination file path.
|
||||
headers: Optional HTTP headers.
|
||||
timeout: Per-request timeout (seconds).
|
||||
|
||||
Returns:
|
||||
True on a successful save, False when the link is not a
|
||||
direct video or the download fails.
|
||||
"""
|
||||
try:
|
||||
with self.session.get(
|
||||
link,
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
stream=True,
|
||||
) as response:
|
||||
if not response.ok:
|
||||
logger.debug(
|
||||
"Direct stream HEAD returned %s for %s",
|
||||
response.status_code, link[:80]
|
||||
)
|
||||
return False
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
if not (
|
||||
content_type.startswith("video/")
|
||||
or content_type == "application/octet-stream"
|
||||
):
|
||||
logger.debug(
|
||||
"Direct stream skipped, Content-Type=%s",
|
||||
content_type
|
||||
)
|
||||
return False
|
||||
logger.info(
|
||||
"Direct stream download starting (type=%s)",
|
||||
content_type
|
||||
)
|
||||
with open(output_path, "wb") as fh:
|
||||
for chunk in response.iter_content(chunk_size=1024 * 1024):
|
||||
if self._cancel_flag.is_set():
|
||||
logger.info(
|
||||
"Cancellation detected during direct stream"
|
||||
)
|
||||
return False
|
||||
if chunk:
|
||||
fh.write(chunk)
|
||||
return True
|
||||
except requests.RequestException as exc:
|
||||
logger.warning("Direct stream download failed: %s", exc)
|
||||
return False
|
||||
|
||||
def download(
|
||||
self,
|
||||
base_directory: str,
|
||||
@@ -259,7 +403,12 @@ class AniworldLoader(Loader):
|
||||
language: str = "German Dub"
|
||||
) -> bool:
|
||||
"""Download episode to specified directory.
|
||||
|
||||
|
||||
Iterates the providers actually advertised on the episode page
|
||||
(ordered by SUPPORTED_PROVIDERS preference), probing each URL
|
||||
before attempting an extraction so dead providers are skipped
|
||||
immediately instead of stalling yt-dlp on a 404.
|
||||
|
||||
Args:
|
||||
base_directory: Base download directory path
|
||||
serie_folder: Filesystem folder name (metadata only, used for
|
||||
@@ -308,12 +457,78 @@ class AniworldLoader(Loader):
|
||||
temp_path = os.path.join(temp_dir, output_file)
|
||||
logger.debug("Temporary path: %s", temp_path)
|
||||
|
||||
for provider in self.SUPPORTED_PROVIDERS:
|
||||
logger.debug("Attempting download with provider: %s", provider)
|
||||
link, header = self._get_direct_link_from_provider(
|
||||
candidate_providers = self._select_providers_for_episode(
|
||||
season, episode, key, language
|
||||
)
|
||||
if not candidate_providers:
|
||||
logger.error(
|
||||
"No providers advertised for S%02dE%03d (%s) in %s",
|
||||
season, episode, key, language
|
||||
)
|
||||
logger.debug("Direct link obtained from provider")
|
||||
self.clear_cache()
|
||||
return False
|
||||
|
||||
tried: list[str] = []
|
||||
for provider_name, redirect_url in candidate_providers:
|
||||
tried.append(provider_name)
|
||||
logger.debug("Attempting download with provider: %s", provider_name)
|
||||
|
||||
probe_headers = {"User-Agent": self.RANDOM_USER_AGENT}
|
||||
if not self._check_url_alive(
|
||||
redirect_url,
|
||||
headers=probe_headers,
|
||||
timeout=self.DEFAULT_REQUEST_TIMEOUT,
|
||||
):
|
||||
logger.info(
|
||||
"Skipping provider %s, redirect URL not reachable",
|
||||
provider_name
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
resolved = self._resolve_direct_link(
|
||||
redirect_url, provider_name
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Provider %s link resolution failed: %s: %s",
|
||||
provider_name, type(exc).__name__, exc
|
||||
)
|
||||
continue
|
||||
|
||||
if resolved is None:
|
||||
logger.info(
|
||||
"Provider %s returned no direct link", provider_name
|
||||
)
|
||||
continue
|
||||
|
||||
link, header = resolved
|
||||
|
||||
if self._cancel_flag.is_set():
|
||||
logger.info("Cancellation requested before download start")
|
||||
_cleanup_temp_file(temp_path)
|
||||
self.clear_cache()
|
||||
return False
|
||||
|
||||
if self._try_direct_stream(
|
||||
link,
|
||||
temp_path,
|
||||
header,
|
||||
self.DEFAULT_REQUEST_TIMEOUT,
|
||||
) and os.path.exists(temp_path):
|
||||
logger.debug(
|
||||
"Direct stream succeeded with provider %s", provider_name
|
||||
)
|
||||
shutil.copyfile(temp_path, output_path)
|
||||
os.remove(temp_path)
|
||||
logger.info(
|
||||
"Download completed successfully (direct): %s",
|
||||
output_file
|
||||
)
|
||||
self.clear_cache()
|
||||
return True
|
||||
|
||||
_cleanup_temp_file(temp_path)
|
||||
|
||||
cancel_flag = self._cancel_flag
|
||||
|
||||
@@ -321,7 +536,6 @@ class AniworldLoader(Loader):
|
||||
if cancel_flag.is_set():
|
||||
logger.info("Cancellation detected in progress hook")
|
||||
raise DownloadCancelled("Download cancelled by user")
|
||||
# Fire the event for progress
|
||||
self.events.download_progress(d)
|
||||
|
||||
ydl_opts = {
|
||||
@@ -333,6 +547,8 @@ class AniworldLoader(Loader):
|
||||
'nocheckcertificate': True,
|
||||
'logger': logger,
|
||||
'progress_hooks': [events_progress_hook],
|
||||
'downloader': 'ffmpeg',
|
||||
'hls_use_mpegts': True,
|
||||
}
|
||||
|
||||
if header:
|
||||
@@ -340,9 +556,11 @@ class AniworldLoader(Loader):
|
||||
logger.debug("Using custom headers for download")
|
||||
|
||||
try:
|
||||
logger.info("Starting download: %s", output_file)
|
||||
logger.info(
|
||||
"Starting yt-dlp download with %s: %s",
|
||||
provider_name, output_file
|
||||
)
|
||||
logger.debug("Download link: %s...", link[:100])
|
||||
logger.debug("YDL options: %s", ydl_opts)
|
||||
|
||||
with YoutubeDL(ydl_opts) as ydl:
|
||||
info = ydl.extract_info(link, download=True)
|
||||
@@ -353,39 +571,151 @@ class AniworldLoader(Loader):
|
||||
|
||||
if os.path.exists(temp_path):
|
||||
logger.debug("Moving file from temp to final destination")
|
||||
# Use copyfile instead of copy to avoid metadata permission issues
|
||||
shutil.copyfile(temp_path, output_path)
|
||||
os.remove(temp_path)
|
||||
logger.info("Download completed successfully: %s", output_file)
|
||||
logger.info(
|
||||
"Download completed successfully: %s", output_file
|
||||
)
|
||||
self.clear_cache()
|
||||
return True
|
||||
else:
|
||||
logger.error("Download failed: temp file not found at %s", temp_path)
|
||||
self.clear_cache()
|
||||
return False
|
||||
except BrokenPipeError as e:
|
||||
logger.error(
|
||||
"Broken pipe error with provider %s: %s. "
|
||||
"This usually means the stream connection was closed.",
|
||||
provider, e
|
||||
"Download failed: temp file not found at %s", temp_path
|
||||
)
|
||||
except DownloadCancelled:
|
||||
logger.info("Download cancelled by user")
|
||||
_cleanup_temp_file(temp_path)
|
||||
self.clear_cache()
|
||||
return False
|
||||
except BrokenPipeError as exc:
|
||||
logger.error(
|
||||
"Broken pipe error with provider %s: %s",
|
||||
provider_name, exc
|
||||
)
|
||||
_cleanup_temp_file(temp_path)
|
||||
continue
|
||||
except Exception as e:
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"YoutubeDL download failed with provider %s: %s: %s",
|
||||
provider, type(e).__name__, e
|
||||
provider_name, type(exc).__name__, exc
|
||||
)
|
||||
_cleanup_temp_file(temp_path)
|
||||
continue
|
||||
break
|
||||
|
||||
# If we get here, all providers failed
|
||||
logger.error("All download providers failed")
|
||||
logger.error(
|
||||
"All download providers failed for S%02dE%03d (%s) in %s. "
|
||||
"Tried: %s. Episode may be unavailable on the source site.",
|
||||
season, episode, key, language, ", ".join(tried) or "none"
|
||||
)
|
||||
download_error_logger.error(
|
||||
"All providers failed for %s S%02dE%03d (%s); tried=%s",
|
||||
key, season, episode, language, tried
|
||||
)
|
||||
_cleanup_temp_file(temp_path)
|
||||
self.clear_cache()
|
||||
return False
|
||||
|
||||
def _select_providers_for_episode(
|
||||
self,
|
||||
season: int,
|
||||
episode: int,
|
||||
key: str,
|
||||
language: str,
|
||||
) -> list[tuple[str, str]]:
|
||||
"""Return ``[(provider_name, redirect_url), ...]`` for an episode.
|
||||
|
||||
Filters by requested language and orders results by
|
||||
``SUPPORTED_PROVIDERS`` preference so the failover chain matches
|
||||
operator expectations. Returns an empty list when nothing is
|
||||
advertised on the page.
|
||||
"""
|
||||
if not self.is_language(season, episode, key, language):
|
||||
logger.warning(
|
||||
"Language %s not advertised for S%02dE%03d (%s)",
|
||||
language, season, episode, key
|
||||
)
|
||||
return []
|
||||
language_code = self._get_language_key(language)
|
||||
providers = self._get_provider_from_html(season, episode, key)
|
||||
ordered: list[tuple[str, str]] = []
|
||||
preferred = list(self.SUPPORTED_PROVIDERS)
|
||||
for name in preferred:
|
||||
lang_map = providers.get(name)
|
||||
if lang_map and language_code in lang_map:
|
||||
ordered.append((name, lang_map[language_code]))
|
||||
for name, lang_map in providers.items():
|
||||
if name in preferred:
|
||||
continue
|
||||
if language_code in lang_map:
|
||||
ordered.append((name, lang_map[language_code]))
|
||||
return ordered
|
||||
|
||||
def _resolve_direct_link(
|
||||
self,
|
||||
redirect_url: str,
|
||||
provider_name: str,
|
||||
) -> tuple[str, dict] | None:
|
||||
"""Resolve a provider redirect URL into a direct stream link.
|
||||
|
||||
Follows the redirect to the embedded player, then delegates to a
|
||||
provider-specific extractor (when registered) or returns the
|
||||
embed URL itself so yt-dlp can attempt extraction.
|
||||
|
||||
Args:
|
||||
redirect_url: AniWorld redirect URL.
|
||||
provider_name: Provider key (e.g. ``"VOE"``).
|
||||
|
||||
Returns:
|
||||
``(direct_link, headers)`` tuple or None when extraction fails.
|
||||
"""
|
||||
try:
|
||||
embedded = self.session.get(
|
||||
redirect_url,
|
||||
timeout=self.DEFAULT_REQUEST_TIMEOUT,
|
||||
headers={"User-Agent": self.RANDOM_USER_AGENT},
|
||||
allow_redirects=True,
|
||||
).url
|
||||
except requests.RequestException as exc:
|
||||
logger.warning(
|
||||
"Failed resolving redirect for %s: %s", provider_name, exc
|
||||
)
|
||||
return None
|
||||
|
||||
try:
|
||||
extractor = self.Providers.GetProvider(provider_name)
|
||||
except (KeyError, AttributeError):
|
||||
extractor = None
|
||||
|
||||
if extractor is not None:
|
||||
try:
|
||||
return extractor.get_link(
|
||||
embedded, self.DEFAULT_REQUEST_TIMEOUT
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Custom extractor %s failed: %s",
|
||||
provider_name, exc
|
||||
)
|
||||
return None
|
||||
|
||||
header_list = self.PROVIDER_HEADERS.get(provider_name)
|
||||
header_dict = self._parse_provider_headers(header_list)
|
||||
return embedded, header_dict
|
||||
|
||||
@staticmethod
|
||||
def _parse_provider_headers(
|
||||
header_list: list | None,
|
||||
) -> dict[str, str]:
|
||||
"""Convert legacy ``"Name: value"`` header strings to a dict."""
|
||||
if not header_list:
|
||||
return {}
|
||||
parsed: dict[str, str] = {}
|
||||
for entry in header_list:
|
||||
if not isinstance(entry, str) or ":" not in entry:
|
||||
continue
|
||||
name, _, value = entry.partition(":")
|
||||
parsed[name.strip()] = value.strip().strip('"')
|
||||
return parsed
|
||||
|
||||
def get_site_key(self) -> str:
|
||||
"""Get the site key for this provider."""
|
||||
return "aniworld.to"
|
||||
@@ -394,7 +724,7 @@ class AniworldLoader(Loader):
|
||||
"""Get anime title from series key."""
|
||||
logger.debug("Getting title for key: %s", key)
|
||||
soup = BeautifulSoup(
|
||||
self._get_key_html(key).content,
|
||||
_decode_html_content(self._get_key_html(key).content),
|
||||
'html.parser'
|
||||
)
|
||||
title_div = soup.find('div', class_='series-title')
|
||||
@@ -425,7 +755,7 @@ class AniworldLoader(Loader):
|
||||
logger.debug("Getting year for key: %s", key)
|
||||
try:
|
||||
soup = BeautifulSoup(
|
||||
self._get_key_html(key).content,
|
||||
_decode_html_content(self._get_key_html(key).content),
|
||||
'html.parser'
|
||||
)
|
||||
|
||||
@@ -539,7 +869,7 @@ class AniworldLoader(Loader):
|
||||
"""
|
||||
logger.debug("Extracting providers from HTML for S%02dE%03d (%s)", season, episode, key)
|
||||
soup = BeautifulSoup(
|
||||
self._get_episode_html(season, episode, key).content,
|
||||
_decode_html_content(self._get_episode_html(season, episode, key).content),
|
||||
'html.parser'
|
||||
)
|
||||
providers: dict[str, dict[int, str]] = {}
|
||||
@@ -662,7 +992,7 @@ class AniworldLoader(Loader):
|
||||
base_url = f"{self.ANIWORLD_TO}/anime/stream/{safe_slug}/"
|
||||
logger.debug("Base URL: %s", base_url)
|
||||
response = requests.get(base_url, timeout=self.DEFAULT_REQUEST_TIMEOUT)
|
||||
soup = BeautifulSoup(response.content, 'html.parser')
|
||||
soup = BeautifulSoup(_decode_html_content(response.content), 'html.parser')
|
||||
|
||||
season_meta = soup.find('meta', itemprop='numberOfSeasons')
|
||||
number_of_seasons = int(season_meta['content']) if season_meta else 0
|
||||
@@ -677,7 +1007,7 @@ class AniworldLoader(Loader):
|
||||
season_url,
|
||||
timeout=self.DEFAULT_REQUEST_TIMEOUT,
|
||||
)
|
||||
soup = BeautifulSoup(response.content, 'html.parser')
|
||||
soup = BeautifulSoup(_decode_html_content(response.content), 'html.parser')
|
||||
|
||||
episode_links = soup.find_all('a', href=True)
|
||||
unique_links = set(
|
||||
|
||||
@@ -567,6 +567,9 @@ class EnhancedAniWorldLoader(Loader):
|
||||
"socket_timeout": self.download_timeout,
|
||||
"http_chunk_size": 1024 * 1024, # 1MB chunks
|
||||
"logger": self.logger,
|
||||
# Use ffmpeg for HLS streams and transport stream format
|
||||
"downloader": "ffmpeg",
|
||||
"hls_use_mpegts": True,
|
||||
}
|
||||
if headers:
|
||||
ydl_opts['http_headers'] = headers
|
||||
|
||||
@@ -10,6 +10,7 @@ Example:
|
||||
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
@@ -19,6 +20,7 @@ from src.core.services.tmdb_client import TMDBAPIError, TMDBClient
|
||||
from src.core.utils.image_downloader import ImageDownloader
|
||||
from src.core.utils.nfo_generator import generate_tvshow_nfo
|
||||
from src.core.utils.nfo_mapper import tmdb_to_nfo_model
|
||||
from src.core.entities.nfo_models import TVShowNFO
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -53,6 +55,18 @@ class NFOService:
|
||||
self.image_size = image_size
|
||||
self.auto_create = auto_create
|
||||
|
||||
async def __aenter__(self) -> "NFOService":
|
||||
"""Enter async context manager."""
|
||||
await self.tmdb_client.__aenter__()
|
||||
await self.image_downloader.__aenter__()
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
||||
"""Exit async context manager and cleanup resources."""
|
||||
await self.tmdb_client.close()
|
||||
await self.image_downloader.close()
|
||||
return False
|
||||
|
||||
def has_nfo(self, serie_folder: str) -> bool:
|
||||
"""Check if tvshow.nfo exists for a series.
|
||||
|
||||
@@ -111,7 +125,8 @@ class NFOService:
|
||||
year: Optional[int] = None,
|
||||
download_poster: bool = True,
|
||||
download_logo: bool = True,
|
||||
download_fanart: bool = True
|
||||
download_fanart: bool = True,
|
||||
alt_titles: Optional[List[str]] = None
|
||||
) -> Path:
|
||||
"""Create tvshow.nfo by scraping TMDB.
|
||||
|
||||
@@ -123,6 +138,7 @@ class NFOService:
|
||||
download_poster: Whether to download poster.jpg
|
||||
download_logo: Whether to download logo.png
|
||||
download_fanart: Whether to download fanart.jpg
|
||||
alt_titles: Alternative titles (e.g., Japanese title) for fallback search
|
||||
|
||||
Returns:
|
||||
Path to created NFO file
|
||||
@@ -149,16 +165,11 @@ class NFOService:
|
||||
|
||||
try:
|
||||
await self.tmdb_client._ensure_session()
|
||||
|
||||
# Search for TV show with clean name (without year)
|
||||
logger.debug("Searching TMDB for: %s", search_name)
|
||||
search_results = await self.tmdb_client.search_tv_show(search_name)
|
||||
|
||||
if not search_results.get("results"):
|
||||
raise TMDBAPIError(f"No results found for: {search_name}")
|
||||
|
||||
# Find best match (consider year if provided)
|
||||
tv_show = self._find_best_match(search_results["results"], search_name, year)
|
||||
|
||||
# Search for TV show - try multiple strategies
|
||||
tv_show, search_source = await self._search_with_fallback(
|
||||
search_name, year, alt_titles
|
||||
)
|
||||
tv_id = tv_show["id"]
|
||||
|
||||
logger.info("Found match: %s (ID: %s)", tv_show['name'], tv_id)
|
||||
@@ -413,6 +424,62 @@ class NFOService:
|
||||
logger.error("Error parsing NFO file %s: %s", nfo_path, e)
|
||||
|
||||
return result
|
||||
|
||||
def parse_nfo_year(self, nfo_path: Path) -> Optional[int]:
|
||||
"""Parse year from an existing NFO file.
|
||||
|
||||
Extracts year from <year> or <premiered> elements.
|
||||
|
||||
Args:
|
||||
nfo_path: Path to tvshow.nfo file
|
||||
|
||||
Returns:
|
||||
Year as integer if found, None otherwise.
|
||||
|
||||
Example:
|
||||
>>> year = nfo_service.parse_nfo_year(Path("/anime/series/tvshow.nfo"))
|
||||
>>> print(year)
|
||||
2013
|
||||
"""
|
||||
if not nfo_path.exists():
|
||||
logger.debug("NFO file not found: %s", nfo_path)
|
||||
return None
|
||||
|
||||
try:
|
||||
tree = etree.parse(str(nfo_path))
|
||||
root = tree.getroot()
|
||||
|
||||
# Try <year> element first
|
||||
year_elem = root.find(".//year")
|
||||
if year_elem is not None and year_elem.text:
|
||||
try:
|
||||
year = int(year_elem.text)
|
||||
if 1900 <= year <= 2100:
|
||||
logger.debug("Found year in NFO: %d", year)
|
||||
return year
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Fallback: try <premiered> element (format: YYYY-MM-DD)
|
||||
premiered_elem = root.find(".//premiered")
|
||||
if premiered_elem is not None and premiered_elem.text:
|
||||
if premiered_elem.text and len(premiered_elem.text) >= 4:
|
||||
try:
|
||||
year = int(premiered_elem.text[:4])
|
||||
if 1900 <= year <= 2100:
|
||||
logger.debug("Found year from premiered in NFO: %d", year)
|
||||
return year
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
logger.debug("No year found in NFO: %s", nfo_path)
|
||||
|
||||
except etree.XMLSyntaxError as e:
|
||||
logger.error("Invalid XML in NFO file %s: %s", nfo_path, e)
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
logger.error("Error parsing year from NFO file %s: %s", nfo_path, e)
|
||||
|
||||
return None
|
||||
|
||||
async def _enrich_details_with_fallback(
|
||||
self,
|
||||
@@ -519,6 +586,137 @@ class NFOService:
|
||||
# Return first result (usually best match)
|
||||
return results[0]
|
||||
|
||||
async def _search_with_fallback(
|
||||
self,
|
||||
primary_query: str,
|
||||
year: Optional[int],
|
||||
alt_titles: Optional[List[str]] = None
|
||||
) -> Tuple[Dict[str, Any], str]:
|
||||
"""Search TMDB with fallback strategies.
|
||||
|
||||
Tries multiple search strategies in order:
|
||||
1. Primary query with year filter
|
||||
2. Alternative titles (e.g., Japanese name)
|
||||
3. Multi-language search (en-US)
|
||||
4. Search without year constraint
|
||||
5. Punctuation-normalized search
|
||||
|
||||
Args:
|
||||
primary_query: Primary search term
|
||||
year: Release year for filtering
|
||||
alt_titles: Alternative titles to try if primary fails
|
||||
|
||||
Returns:
|
||||
Tuple of (matched TV show dict, source description string)
|
||||
|
||||
Raises:
|
||||
TMDBAPIError: If all search strategies fail
|
||||
"""
|
||||
search_strategies = [
|
||||
# Strategy 1: Primary query as-is
|
||||
{"query": primary_query, "year": year, "lang": "de-DE", "desc": "primary"},
|
||||
]
|
||||
|
||||
# Strategy 2: Try alt titles (typically Japanese)
|
||||
if alt_titles:
|
||||
for alt in alt_titles:
|
||||
if alt != primary_query:
|
||||
search_strategies.append(
|
||||
{"query": alt, "year": year, "lang": "ja-JP", "desc": f"alt_title:{alt}"}
|
||||
)
|
||||
search_strategies.append(
|
||||
{"query": alt, "year": year, "lang": "en-US", "desc": f"alt_title:{alt}"}
|
||||
)
|
||||
|
||||
# Strategy 3: Try English search
|
||||
search_strategies.append(
|
||||
{"query": primary_query, "year": year, "lang": "en-US", "desc": "english"}
|
||||
)
|
||||
|
||||
# Strategy 4: Try without year constraint
|
||||
if year:
|
||||
search_strategies.append(
|
||||
{"query": primary_query, "year": None, "lang": "de-DE", "desc": "no_year"}
|
||||
)
|
||||
|
||||
# Strategy 5: Normalize punctuation
|
||||
normalized = self._normalize_query_for_search(primary_query)
|
||||
if normalized != primary_query:
|
||||
search_strategies.append(
|
||||
{"query": normalized, "year": year, "lang": "de-DE", "desc": f"normalized:{normalized}"}
|
||||
)
|
||||
|
||||
last_error = None
|
||||
for strategy in search_strategies:
|
||||
query = strategy["query"]
|
||||
lang = strategy["lang"]
|
||||
desc = strategy["desc"]
|
||||
|
||||
try:
|
||||
logger.debug(
|
||||
"TMDB search attempt: query='%s', lang=%s, year=%s, strategy=%s",
|
||||
query, lang, strategy["year"], desc
|
||||
)
|
||||
search_results = await self.tmdb_client.search_tv_show(
|
||||
query,
|
||||
language=lang
|
||||
)
|
||||
|
||||
if search_results.get("results"):
|
||||
# Apply year filter if we have one
|
||||
results = search_results["results"]
|
||||
if strategy["year"]:
|
||||
year_filtered = [
|
||||
r for r in results
|
||||
if r.get("first_air_date", "").startswith(str(strategy["year"]))
|
||||
]
|
||||
if year_filtered:
|
||||
match = year_filtered[0]
|
||||
else:
|
||||
# Year didn't match, still use first result but log it
|
||||
match = results[0]
|
||||
logger.debug(
|
||||
"Year %s not found in results for '%s', using: %s",
|
||||
strategy["year"], query, match["name"]
|
||||
)
|
||||
else:
|
||||
match = results[0]
|
||||
|
||||
logger.info(
|
||||
"TMDB search succeeded: '%s' found via strategy '%s' (ID: %s)",
|
||||
match["name"], desc, match["id"]
|
||||
)
|
||||
return match, desc
|
||||
else:
|
||||
logger.debug("No results for '%s' via %s", query, desc)
|
||||
|
||||
except TMDBAPIError as e:
|
||||
last_error = e
|
||||
logger.debug("Search strategy '%s' failed: %s", desc, e)
|
||||
continue
|
||||
|
||||
# All strategies exhausted
|
||||
raise TMDBAPIError(
|
||||
f"No results found for: {primary_query} (tried {len(search_strategies)} strategies)"
|
||||
)
|
||||
|
||||
def _normalize_query_for_search(self, query: str) -> str:
|
||||
"""Normalize query by removing punctuation and special chars.
|
||||
|
||||
Args:
|
||||
query: Original search query
|
||||
|
||||
Returns:
|
||||
Query with punctuation removed
|
||||
"""
|
||||
# Remove common punctuation but keep CJK characters
|
||||
normalized = unicodedata.normalize('NFKC', query)
|
||||
# Remove punctuation but not CJK
|
||||
normalized = re.sub(r'[^\w\s\u3000-\u9fff\u4e00-\u9faf]', '', normalized)
|
||||
# Collapse multiple spaces
|
||||
normalized = re.sub(r'\s+', ' ', normalized).strip()
|
||||
return normalized
|
||||
|
||||
|
||||
|
||||
async def _download_media_files(
|
||||
@@ -586,3 +784,52 @@ class NFOService:
|
||||
async def close(self):
|
||||
"""Clean up resources."""
|
||||
await self.tmdb_client.close()
|
||||
|
||||
async def create_minimal_nfo(
|
||||
self,
|
||||
serie_name: str,
|
||||
serie_folder: str,
|
||||
year: Optional[int] = None
|
||||
) -> Path:
|
||||
"""Create minimal tvshow.nfo when TMDB lookup fails.
|
||||
|
||||
Creates a basic NFO with just the title (and year if available)
|
||||
so the series is tracked even without TMDB metadata.
|
||||
|
||||
Args:
|
||||
serie_name: Name of the series (may include year in parentheses)
|
||||
serie_folder: Series folder name
|
||||
year: Optional release year
|
||||
|
||||
Returns:
|
||||
Path to created NFO file
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If series folder doesn't exist
|
||||
"""
|
||||
# Extract year from name if not provided
|
||||
clean_name, extracted_year = self._extract_year_from_name(serie_name)
|
||||
if year is None and extracted_year is not None:
|
||||
year = extracted_year
|
||||
|
||||
folder_path = self.anime_directory / serie_folder
|
||||
if not folder_path.exists():
|
||||
logger.info("Creating series folder: %s", folder_path)
|
||||
folder_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create minimal NFO model with just title and year
|
||||
nfo_model = TVShowNFO(
|
||||
title=clean_name,
|
||||
year=year,
|
||||
plot=f"No metadata available for {clean_name}. TMDB lookup failed."
|
||||
)
|
||||
|
||||
# Generate XML
|
||||
nfo_xml = generate_tvshow_nfo(nfo_model)
|
||||
|
||||
# Save NFO file
|
||||
nfo_path = folder_path / "tvshow.nfo"
|
||||
nfo_path.write_text(nfo_xml, encoding="utf-8")
|
||||
logger.info("Created minimal NFO (no TMDB): %s", nfo_path)
|
||||
|
||||
return nfo_path
|
||||
|
||||
@@ -39,6 +39,7 @@ class TMDBClient:
|
||||
|
||||
DEFAULT_BASE_URL = "https://api.themoviedb.org/3"
|
||||
DEFAULT_IMAGE_BASE_URL = "https://image.tmdb.org/t/p"
|
||||
NEGATIVE_CACHE_TTL = 86400 # 24 hours
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -64,6 +65,7 @@ class TMDBClient:
|
||||
self.max_connections = max_connections
|
||||
self.session: Optional[aiohttp.ClientSession] = None
|
||||
self._cache: Dict[str, Any] = {}
|
||||
self._negative_cache: Dict[str, float] = {} # query -> timestamp when cached
|
||||
# TMDB allows ~40 req/s; use 30 concurrent + per-second throttle to stay safe
|
||||
self._semaphore = asyncio.Semaphore(30)
|
||||
self._rate_limit_lock = asyncio.Lock()
|
||||
@@ -116,6 +118,16 @@ class TMDBClient:
|
||||
logger.debug("Cache hit for %s", endpoint)
|
||||
return self._cache[cache_key]
|
||||
|
||||
# Check negative cache (cached empty results)
|
||||
negative_cache_key = f"{endpoint}:{str(sorted(params.items()))}"
|
||||
if negative_cache_key in self._negative_cache:
|
||||
if time.monotonic() - self._negative_cache[negative_cache_key] < self.NEGATIVE_CACHE_TTL:
|
||||
logger.debug("Negative cache hit for %s (cached empty result)", endpoint)
|
||||
return {"results": []}
|
||||
else:
|
||||
# Expired negative cache entry
|
||||
del self._negative_cache[negative_cache_key]
|
||||
|
||||
delay = 2
|
||||
last_error = None
|
||||
|
||||
@@ -158,6 +170,10 @@ class TMDBClient:
|
||||
resp.raise_for_status()
|
||||
data = await resp.json()
|
||||
self._cache[cache_key] = data
|
||||
# Cache negative result if empty
|
||||
if endpoint.startswith("search/") and not data.get("results"):
|
||||
self._negative_cache[negative_cache_key] = time.monotonic()
|
||||
logger.debug("Cached negative result for %s", endpoint)
|
||||
return data
|
||||
|
||||
except asyncio.TimeoutError as e:
|
||||
@@ -173,7 +189,11 @@ class TMDBClient:
|
||||
last_error = e
|
||||
# If connector/session was closed, try to recreate it
|
||||
if "Connector is closed" in str(e) or isinstance(e, AttributeError):
|
||||
logger.warning("Session issue detected, recreating session: %s", e)
|
||||
logger.warning(
|
||||
"Session issue detected, recreating session: %s",
|
||||
e,
|
||||
exc_info=True,
|
||||
)
|
||||
self.session = None
|
||||
await self._ensure_session()
|
||||
|
||||
@@ -220,6 +240,34 @@ class TMDBClient:
|
||||
{"query": query, "language": language, "page": page}
|
||||
)
|
||||
|
||||
async def search_multi(
|
||||
self,
|
||||
query: str,
|
||||
language: str = "en-US",
|
||||
page: int = 1
|
||||
) -> Dict[str, Any]:
|
||||
"""Search for movies and TV shows by name using TMDB multi search.
|
||||
|
||||
Multi search returns both movies and TV shows, useful for anime
|
||||
that might be indexed as movies on TMDB.
|
||||
|
||||
Args:
|
||||
query: Search query (show name)
|
||||
language: Language for results (default: English)
|
||||
page: Page number for pagination
|
||||
|
||||
Returns:
|
||||
Search results with list of movies and TV shows
|
||||
|
||||
Example:
|
||||
>>> results = await client.search_multi("Suzume no Tojimari")
|
||||
>>> shows = [r for r in results["results"] if r["media_type"] == "tv"]
|
||||
"""
|
||||
return await self._request(
|
||||
"search/multi",
|
||||
{"query": query, "language": language, "page": page}
|
||||
)
|
||||
|
||||
async def get_tv_show_details(
|
||||
self,
|
||||
tv_id: int,
|
||||
@@ -339,8 +387,38 @@ class TMDBClient:
|
||||
await self.session.close()
|
||||
self.session = None
|
||||
logger.debug("TMDB client session closed")
|
||||
|
||||
def __del__(self):
|
||||
"""Warn if session is unclosed during garbage collection."""
|
||||
if self.session is not None and not self.session.closed:
|
||||
logger.warning(
|
||||
"TMDBClient: unclosed session detected. "
|
||||
"Use 'async with TMDBClient(...)' or call close() explicitly."
|
||||
)
|
||||
|
||||
def clear_cache(self):
|
||||
"""Clear the request cache."""
|
||||
self._cache.clear()
|
||||
logger.debug("TMDB client cache cleared")
|
||||
|
||||
def clear_negative_cache(self):
|
||||
"""Clear the negative result cache."""
|
||||
self._negative_cache.clear()
|
||||
logger.debug("TMDB negative cache cleared")
|
||||
|
||||
def cleanup_expired_negative_cache(self) -> int:
|
||||
"""Remove expired entries from negative cache.
|
||||
|
||||
Returns:
|
||||
Number of entries removed
|
||||
"""
|
||||
now = time.monotonic()
|
||||
expired_keys = [
|
||||
key for key, timestamp in self._negative_cache.items()
|
||||
if now - timestamp >= self.NEGATIVE_CACHE_TTL
|
||||
]
|
||||
for key in expired_keys:
|
||||
del self._negative_cache[key]
|
||||
if expired_keys:
|
||||
logger.debug("Removed %d expired negative cache entries", len(expired_keys))
|
||||
return len(expired_keys)
|
||||
|
||||
@@ -5,7 +5,7 @@ from datetime import datetime
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import psutil
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
@@ -26,6 +26,9 @@ class HealthStatus(BaseModel):
|
||||
service: str = "aniworld-api"
|
||||
series_app_initialized: bool = False
|
||||
anime_directory_configured: bool = False
|
||||
scheduler_next_run: Optional[str] = None
|
||||
scheduler_last_run: Optional[str] = None
|
||||
checks: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
class DatabaseHealth(BaseModel):
|
||||
@@ -171,29 +174,90 @@ def get_system_metrics() -> SystemMetrics:
|
||||
|
||||
|
||||
@router.get("", response_model=HealthStatus)
|
||||
async def basic_health_check() -> HealthStatus:
|
||||
async def basic_health_check(request: Request) -> HealthStatus:
|
||||
"""Basic health check endpoint.
|
||||
|
||||
This endpoint does not depend on anime_directory configuration
|
||||
and should always return 200 OK for basic health monitoring.
|
||||
Includes service information for identification.
|
||||
Includes scheduler next/last run times for monitoring tools.
|
||||
Includes startup health check results.
|
||||
|
||||
Returns:
|
||||
HealthStatus: Simple health status with timestamp and service info.
|
||||
"""
|
||||
from src.config.settings import settings
|
||||
from src.server.utils.dependencies import _series_app
|
||||
|
||||
# Get scheduler status for health monitoring
|
||||
scheduler_status: dict = {}
|
||||
try:
|
||||
from src.server.services.scheduler_service import get_scheduler_service
|
||||
scheduler_status = get_scheduler_service().get_status()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Get startup checks from app state
|
||||
checks = getattr(request.app.state, "startup_checks", None)
|
||||
|
||||
# Determine overall status based on checks
|
||||
overall_status = "healthy"
|
||||
if checks:
|
||||
for check_name, check_data in checks.items():
|
||||
if check_data.get("status") == "error":
|
||||
overall_status = "unhealthy"
|
||||
break
|
||||
elif check_data.get("status") == "warning":
|
||||
overall_status = "degraded"
|
||||
|
||||
logger.debug("Basic health check requested")
|
||||
return HealthStatus(
|
||||
status="healthy",
|
||||
status=overall_status,
|
||||
timestamp=datetime.now().isoformat(),
|
||||
service="aniworld-api",
|
||||
series_app_initialized=_series_app is not None,
|
||||
anime_directory_configured=bool(settings.anime_directory),
|
||||
scheduler_next_run=scheduler_status.get("next_run"),
|
||||
scheduler_last_run=scheduler_status.get("last_run"),
|
||||
checks=checks,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/ready")
|
||||
async def ready_check(request: Request) -> Dict[str, Any]:
|
||||
"""Readiness check endpoint for container orchestrators.
|
||||
|
||||
Returns 503 if critical dependencies are not available.
|
||||
This endpoint is used by Kubernetes, Docker Swarm, etc. to determine
|
||||
if the container should receive traffic.
|
||||
|
||||
Returns:
|
||||
dict: Readiness status with checks details.
|
||||
"""
|
||||
checks = getattr(request.app.state, "startup_checks", {})
|
||||
|
||||
critical_failures = []
|
||||
for check_name, check_data in checks.items():
|
||||
if check_data.get("status") == "error":
|
||||
critical_failures.append(f"{check_name}: {check_data.get('message')}")
|
||||
|
||||
if critical_failures:
|
||||
return {
|
||||
"status": "not_ready",
|
||||
"ready": False,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"critical_failures": critical_failures,
|
||||
"checks": checks,
|
||||
}
|
||||
|
||||
return {
|
||||
"status": "ready",
|
||||
"ready": True,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"checks": checks,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/detailed", response_model=DetailedHealthStatus)
|
||||
async def detailed_health_check(
|
||||
db: AsyncSession = Depends(get_database_session),
|
||||
|
||||
@@ -144,6 +144,27 @@ async def batch_create_nfo(
|
||||
nfo_path=str(nfo_path)
|
||||
)
|
||||
|
||||
except TMDBAPIError as e:
|
||||
logger.warning("TMDB API error for %s, creating minimal fallback: %s", serie_id, e)
|
||||
# TMDB failed, create minimal NFO
|
||||
try:
|
||||
serie_folder = serie.ensure_folder_with_year()
|
||||
except Exception:
|
||||
serie_folder = serie_folder
|
||||
|
||||
serie_name = serie.name or serie_folder
|
||||
nfo_path = await nfo_service.create_minimal_nfo(
|
||||
serie_name=serie_name,
|
||||
serie_folder=serie_folder
|
||||
)
|
||||
|
||||
return NFOBatchResult(
|
||||
serie_id=serie_id,
|
||||
serie_folder=serie_folder,
|
||||
success=True,
|
||||
message="Created minimal NFO (TMDB lookup failed)",
|
||||
nfo_path=str(nfo_path)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error creating NFO for {serie_id}: {e}",
|
||||
@@ -429,11 +450,42 @@ async def create_nfo(
|
||||
except HTTPException:
|
||||
raise
|
||||
except TMDBAPIError as e:
|
||||
logger.warning("TMDB API error creating NFO for %s: %s", serie_id, e)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail=f"TMDB API error: {str(e)}"
|
||||
) from e
|
||||
logger.warning("TMDB API error for %s, creating minimal fallback: %s", serie_id, e)
|
||||
# TMDB failed, create minimal NFO with just folder name
|
||||
try:
|
||||
serie_folder = serie.ensure_folder_with_year()
|
||||
except Exception:
|
||||
serie_folder = serie_folder
|
||||
|
||||
folder_path = Path(settings.anime_directory) / serie_folder
|
||||
serie_name_fallback = request.serie_name or serie.name or serie_folder
|
||||
|
||||
nfo_path = await nfo_service.create_minimal_nfo(
|
||||
serie_name=serie_name_fallback,
|
||||
serie_folder=serie_folder,
|
||||
year=year
|
||||
)
|
||||
|
||||
# Check media files (will likely be empty)
|
||||
media_status = check_media_files(folder_path)
|
||||
file_paths = get_media_file_paths(folder_path)
|
||||
|
||||
media_files = MediaFilesStatus(
|
||||
has_poster=media_status.get("poster", False),
|
||||
has_logo=media_status.get("logo", False),
|
||||
has_fanart=media_status.get("fanart", False),
|
||||
poster_path=str(file_paths["poster"]) if file_paths.get("poster") else None,
|
||||
logo_path=str(file_paths["logo"]) if file_paths.get("logo") else None,
|
||||
fanart_path=str(file_paths["fanart"]) if file_paths.get("fanart") else None
|
||||
)
|
||||
|
||||
return NFOCreateResponse(
|
||||
serie_id=serie_id,
|
||||
serie_folder=serie_folder,
|
||||
nfo_path=str(nfo_path),
|
||||
media_files=media_files,
|
||||
message="Created minimal NFO (TMDB lookup failed)"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error creating NFO for {serie_id}: {e}",
|
||||
|
||||
@@ -15,7 +15,7 @@ from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import List, Optional
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, String, Text, func
|
||||
from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, String, Text, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
||||
|
||||
from src.server.database.base import Base, TimestampMixin
|
||||
@@ -316,6 +316,7 @@ class DownloadQueueItem(Base, TimestampMixin):
|
||||
id: Primary key
|
||||
series_id: Foreign key to AnimeSeries
|
||||
episode_id: Foreign key to Episode
|
||||
status: Queue status (pending/downloading/completed/failed/permanently_failed)
|
||||
error_message: Error description if failed
|
||||
download_url: Provider download URL
|
||||
file_destination: Target file path
|
||||
@@ -347,6 +348,33 @@ class DownloadQueueItem(Base, TimestampMixin):
|
||||
index=True
|
||||
)
|
||||
|
||||
# Status column to track queue item state
|
||||
# Allows distinguishing pending items from permanently failed ones
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(50), nullable=False, default="pending",
|
||||
doc="Queue item status: pending, downloading, completed, failed, permanently_failed"
|
||||
)
|
||||
|
||||
# Retry count to track failed download attempts
|
||||
# Used to determine when to move item to permanently_failed
|
||||
retry_count: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=0,
|
||||
doc="Number of retry attempts for this download"
|
||||
)
|
||||
|
||||
# Unique constraint to prevent duplicate pending queue items per episode
|
||||
# An episode can only have one PENDING entry at a time
|
||||
# The status column allows failed items to remain in DB while new
|
||||
# pending items can be added (application-level dedup still required)
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"ix_download_queue_episode_status",
|
||||
"episode_id",
|
||||
"status",
|
||||
unique=True,
|
||||
),
|
||||
)
|
||||
|
||||
# Error handling
|
||||
error_message: Mapped[Optional[str]] = mapped_column(
|
||||
Text, nullable=True,
|
||||
|
||||
@@ -541,6 +541,7 @@ class EpisodeService:
|
||||
db: AsyncSession,
|
||||
series_id: int,
|
||||
season: Optional[int] = None,
|
||||
only_missing: bool = False,
|
||||
) -> List[Episode]:
|
||||
"""Get episodes for a series.
|
||||
|
||||
@@ -548,6 +549,9 @@ class EpisodeService:
|
||||
db: Database session
|
||||
series_id: Foreign key to AnimeSeries
|
||||
season: Optional season filter
|
||||
only_missing: If True, only return episodes where
|
||||
is_downloaded is False (i.e., missing episodes).
|
||||
Default False returns all episodes.
|
||||
|
||||
Returns:
|
||||
List of Episode instances
|
||||
@@ -557,6 +561,9 @@ class EpisodeService:
|
||||
if season is not None:
|
||||
query = query.where(Episode.season == season)
|
||||
|
||||
if only_missing:
|
||||
query = query.where(Episode.is_downloaded == False)
|
||||
|
||||
query = query.order_by(Episode.season, Episode.episode_number)
|
||||
result = await db.execute(query)
|
||||
return list(result.scalars().all())
|
||||
@@ -748,6 +755,8 @@ class DownloadQueueService:
|
||||
episode_id: int,
|
||||
download_url: Optional[str] = None,
|
||||
file_destination: Optional[str] = None,
|
||||
status: str = "pending",
|
||||
retry_count: int = 0,
|
||||
) -> DownloadQueueItem:
|
||||
"""Add item to download queue.
|
||||
|
||||
@@ -757,6 +766,8 @@ class DownloadQueueService:
|
||||
episode_id: Foreign key to Episode
|
||||
download_url: Optional provider download URL
|
||||
file_destination: Optional target file path
|
||||
status: Queue item status (default: "pending")
|
||||
retry_count: Number of retry attempts (default: 0)
|
||||
|
||||
Returns:
|
||||
Created DownloadQueueItem instance
|
||||
@@ -766,13 +777,15 @@ class DownloadQueueService:
|
||||
episode_id=episode_id,
|
||||
download_url=download_url,
|
||||
file_destination=file_destination,
|
||||
status=status,
|
||||
retry_count=retry_count,
|
||||
)
|
||||
db.add(item)
|
||||
await db.flush()
|
||||
await db.refresh(item)
|
||||
logger.info(
|
||||
f"Added to download queue: episode_id={episode_id} "
|
||||
f"for series_id={series_id}"
|
||||
f"for series_id={series_id}, status={status}"
|
||||
)
|
||||
return item
|
||||
|
||||
@@ -799,21 +812,24 @@ class DownloadQueueService:
|
||||
async def get_by_episode(
|
||||
db: AsyncSession,
|
||||
episode_id: int,
|
||||
status_filter: Optional[str] = None,
|
||||
) -> Optional[DownloadQueueItem]:
|
||||
"""Get download queue item by episode ID.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
episode_id: Foreign key to Episode
|
||||
status_filter: Optional status to filter by (e.g., "pending")
|
||||
|
||||
Returns:
|
||||
DownloadQueueItem instance or None if not found
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(DownloadQueueItem).where(
|
||||
DownloadQueueItem.episode_id == episode_id
|
||||
)
|
||||
query = select(DownloadQueueItem).where(
|
||||
DownloadQueueItem.episode_id == episode_id
|
||||
)
|
||||
if status_filter:
|
||||
query = query.where(DownloadQueueItem.status == status_filter)
|
||||
result = await db.execute(query)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
@staticmethod
|
||||
@@ -873,6 +889,95 @@ class DownloadQueueService:
|
||||
logger.debug("Set error on download queue item %s", item_id)
|
||||
return item
|
||||
|
||||
@staticmethod
|
||||
async def set_status(
|
||||
db: AsyncSession,
|
||||
item_id: int,
|
||||
status: str,
|
||||
) -> Optional[DownloadQueueItem]:
|
||||
"""Set status on download queue item.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
item_id: Item primary key
|
||||
status: New status value
|
||||
|
||||
Returns:
|
||||
Updated DownloadQueueItem instance or None if not found
|
||||
"""
|
||||
item = await DownloadQueueService.get_by_id(db, item_id)
|
||||
if not item:
|
||||
return None
|
||||
|
||||
item.status = status
|
||||
|
||||
await db.flush()
|
||||
await db.refresh(item)
|
||||
logger.debug("Set status on download queue item %s to %s", item_id, status)
|
||||
return item
|
||||
|
||||
@staticmethod
|
||||
async def increment_retry_count(
|
||||
db: AsyncSession,
|
||||
item_id: int,
|
||||
) -> Optional[DownloadQueueItem]:
|
||||
"""Increment retry count on download queue item.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
item_id: Item primary key
|
||||
|
||||
Returns:
|
||||
Updated DownloadQueueItem instance or None if not found
|
||||
"""
|
||||
item = await DownloadQueueService.get_by_id(db, item_id)
|
||||
if not item:
|
||||
return None
|
||||
|
||||
item.retry_count += 1
|
||||
|
||||
await db.flush()
|
||||
await db.refresh(item)
|
||||
logger.debug(
|
||||
"Incremented retry count on download queue item %s to %s",
|
||||
item_id, item.retry_count
|
||||
)
|
||||
return item
|
||||
|
||||
@staticmethod
|
||||
async def set_status_and_error(
|
||||
db: AsyncSession,
|
||||
item_id: int,
|
||||
status: str,
|
||||
error_message: Optional[str] = None,
|
||||
) -> Optional[DownloadQueueItem]:
|
||||
"""Set status and error message on download queue item atomically.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
item_id: Item primary key
|
||||
status: New status value
|
||||
error_message: Optional error description
|
||||
|
||||
Returns:
|
||||
Updated DownloadQueueItem instance or None if not found
|
||||
"""
|
||||
item = await DownloadQueueService.get_by_id(db, item_id)
|
||||
if not item:
|
||||
return None
|
||||
|
||||
item.status = status
|
||||
if error_message is not None:
|
||||
item.error_message = error_message
|
||||
|
||||
await db.flush()
|
||||
await db.refresh(item)
|
||||
logger.debug(
|
||||
"Set status=%s on download queue item %s, error=%s",
|
||||
status, item_id, error_message
|
||||
)
|
||||
return item
|
||||
|
||||
@staticmethod
|
||||
async def delete(db: AsyncSession, item_id: int) -> bool:
|
||||
"""Delete download queue item.
|
||||
|
||||
@@ -104,6 +104,107 @@ async def _check_incomplete_series_on_startup(background_loader) -> None:
|
||||
logger.exception("Failed to check incomplete series on startup")
|
||||
|
||||
|
||||
async def _run_startup_health_checks(logger) -> dict:
|
||||
"""Run startup health checks for critical dependencies.
|
||||
|
||||
Checks:
|
||||
- ffmpeg availability
|
||||
- DNS resolution for aniworld.to and api.themoviedb.org
|
||||
- anime_directory configuration and writability
|
||||
|
||||
Args:
|
||||
logger: Logger instance for recording check results.
|
||||
|
||||
Returns:
|
||||
dict: Health check results with status and details for each check.
|
||||
"""
|
||||
import asyncio
|
||||
import shutil
|
||||
import socket
|
||||
from typing import Any, Dict
|
||||
|
||||
checks: Dict[str, Any] = {
|
||||
"ffmpeg": {"status": "unknown", "message": None},
|
||||
"dns_aniworld": {"status": "unknown", "message": None},
|
||||
"dns_tmdb": {"status": "unknown", "message": None},
|
||||
"anime_directory": {"status": "unknown", "message": None, "path": None},
|
||||
}
|
||||
|
||||
# Check ffmpeg availability
|
||||
try:
|
||||
ffmpeg_path = shutil.which("ffmpeg")
|
||||
if ffmpeg_path:
|
||||
checks["ffmpeg"]["status"] = "ok"
|
||||
checks["ffmpeg"]["message"] = f"Found at {ffmpeg_path}"
|
||||
logger.debug("ffmpeg health check passed: %s", ffmpeg_path)
|
||||
else:
|
||||
checks["ffmpeg"]["status"] = "warning"
|
||||
checks["ffmpeg"]["message"] = "ffmpeg not found in PATH"
|
||||
logger.warning("ffmpeg health check failed: not in PATH")
|
||||
except Exception as e:
|
||||
checks["ffmpeg"]["status"] = "error"
|
||||
checks["ffmpeg"]["message"] = str(e)
|
||||
logger.warning("Could not check ffmpeg: %s", e)
|
||||
|
||||
# Check DNS resolution for aniworld.to
|
||||
try:
|
||||
socket.gethostbyname("aniworld.to")
|
||||
checks["dns_aniworld"]["status"] = "ok"
|
||||
checks["dns_aniworld"]["message"] = "Resolved successfully"
|
||||
logger.debug("DNS health check passed for aniworld.to")
|
||||
except socket.gaierror as e:
|
||||
checks["dns_aniworld"]["status"] = "warning"
|
||||
checks["dns_aniworld"]["message"] = f"DNS resolution failed: {e}"
|
||||
logger.warning("DNS health check failed for aniworld.to: %s", e)
|
||||
except Exception as e:
|
||||
checks["dns_aniworld"]["status"] = "warning"
|
||||
checks["dns_aniworld"]["message"] = f"Unexpected error: {e}"
|
||||
logger.warning("Unexpected DNS error for aniworld.to: %s", e)
|
||||
|
||||
# Check DNS resolution for api.themoviedb.org
|
||||
try:
|
||||
socket.gethostbyname("api.themoviedb.org")
|
||||
checks["dns_tmdb"]["status"] = "ok"
|
||||
checks["dns_tmdb"]["message"] = "Resolved successfully"
|
||||
logger.debug("DNS health check passed for api.themoviedb.org")
|
||||
except socket.gaierror as e:
|
||||
checks["dns_tmdb"]["status"] = "warning"
|
||||
checks["dns_tmdb"]["message"] = f"DNS resolution failed: {e}"
|
||||
logger.warning("DNS health check failed for api.themoviedb.org: %s", e)
|
||||
except Exception as e:
|
||||
checks["dns_tmdb"]["status"] = "warning"
|
||||
checks["dns_tmdb"]["message"] = f"Unexpected error: {e}"
|
||||
logger.warning("Unexpected DNS error for api.themoviedb.org: %s", e)
|
||||
|
||||
# Check anime_directory configuration and writability
|
||||
from src.config.settings import settings
|
||||
anime_dir = settings.anime_directory
|
||||
|
||||
if not anime_dir:
|
||||
checks["anime_directory"]["status"] = "error"
|
||||
checks["anime_directory"]["message"] = "anime_directory not configured"
|
||||
checks["anime_directory"]["path"] = None
|
||||
logger.error("anime_directory health check failed: not configured")
|
||||
else:
|
||||
import os
|
||||
checks["anime_directory"]["path"] = anime_dir
|
||||
|
||||
if not os.path.isdir(anime_dir):
|
||||
checks["anime_directory"]["status"] = "error"
|
||||
checks["anime_directory"]["message"] = f"Directory does not exist: {anime_dir}"
|
||||
logger.error("anime_directory health check failed: %s does not exist", anime_dir)
|
||||
elif not os.access(anime_dir, os.W_OK):
|
||||
checks["anime_directory"]["status"] = "error"
|
||||
checks["anime_directory"]["message"] = f"Directory not writable: {anime_dir}"
|
||||
logger.error("anime_directory health check failed: %s not writable", anime_dir)
|
||||
else:
|
||||
checks["anime_directory"]["status"] = "ok"
|
||||
checks["anime_directory"]["message"] = f"Directory exists and is writable: {anime_dir}"
|
||||
logger.debug("anime_directory health check passed: %s", anime_dir)
|
||||
|
||||
return checks
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_application: FastAPI):
|
||||
"""Manage application lifespan (startup and shutdown).
|
||||
@@ -299,13 +400,15 @@ async def lifespan(_application: FastAPI):
|
||||
|
||||
# Initialize and start scheduler service
|
||||
try:
|
||||
logger.info("Initializing scheduler service...")
|
||||
from src.server.services.scheduler_service import (
|
||||
get_scheduler_service,
|
||||
)
|
||||
scheduler_service = get_scheduler_service()
|
||||
logger.info("Scheduler service instance obtained, starting...")
|
||||
await scheduler_service.start()
|
||||
initialized['scheduler'] = True
|
||||
logger.info("Scheduler service started")
|
||||
logger.info("Scheduler service started successfully")
|
||||
except Exception as e:
|
||||
logger.warning("Failed to start scheduler service: %s", e)
|
||||
# Continue - scheduler is optional
|
||||
@@ -329,6 +432,27 @@ async def lifespan(_application: FastAPI):
|
||||
logger.info(
|
||||
"API documentation available at http://127.0.0.1:8000/api/docs"
|
||||
)
|
||||
|
||||
# Check for ffmpeg availability and warn if missing
|
||||
try:
|
||||
import shutil as _shutil
|
||||
if _shutil.which("ffmpeg") is None:
|
||||
logger.warning(
|
||||
"ffmpeg not found in PATH. HLS streams may fail to download. "
|
||||
"Install ffmpeg to enable HLS support."
|
||||
)
|
||||
else:
|
||||
logger.debug("ffmpeg found at: %s", _shutil.which("ffmpeg"))
|
||||
except Exception as _exc:
|
||||
logger.warning("Could not check for ffmpeg: %s", _exc)
|
||||
|
||||
# Run startup health checks and store results for /health endpoint
|
||||
try:
|
||||
startup_checks = await _run_startup_health_checks(logger)
|
||||
app.state.startup_checks = startup_checks
|
||||
except Exception as _exc:
|
||||
logger.warning("Could not run startup health checks: %s", _exc)
|
||||
app.state.startup_checks = {}
|
||||
except Exception as e:
|
||||
logger.error("Error during startup: %s", e, exc_info=True)
|
||||
startup_error = e
|
||||
|
||||
@@ -22,6 +22,7 @@ class DownloadStatus(str, Enum):
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
CANCELLED = "cancelled"
|
||||
PERMANENTLY_FAILED = "permanently_failed"
|
||||
|
||||
|
||||
class DownloadPriority(str, Enum):
|
||||
|
||||
@@ -498,13 +498,19 @@ class AnimeService:
|
||||
logger.info("No series found in SeriesApp")
|
||||
return []
|
||||
|
||||
# Build NFO metadata map and filter data from database
|
||||
nfo_map = {}
|
||||
series_with_no_episodes = set()
|
||||
# Build NFO metadata map, episode dict, and filter data from database.
|
||||
# Using DB as authoritative source for episodeDict ensures that
|
||||
# episodes marked is_downloaded=True are never shown as missing,
|
||||
# even if the in-memory state is stale.
|
||||
nfo_map: dict = {}
|
||||
db_episode_dict_map: dict[str, dict[int, list[int]]] = {}
|
||||
series_with_no_episodes: set = set()
|
||||
|
||||
async with get_db_session() as db:
|
||||
# Get all series NFO metadata using service layer
|
||||
db_series_list = await AnimeSeriesService.get_all(db)
|
||||
# Single query: load all series with their episodes eagerly
|
||||
db_series_list = await AnimeSeriesService.get_all(
|
||||
db, with_episodes=True
|
||||
)
|
||||
|
||||
for db_series in db_series_list:
|
||||
nfo_created = (
|
||||
@@ -523,6 +529,20 @@ class AnimeService:
|
||||
"tvdb_id": db_series.tvdb_id,
|
||||
"series_id": db_series.id,
|
||||
}
|
||||
|
||||
# Build episodeDict from DB, skipping is_downloaded=True
|
||||
# episodes so they are never shown as missing in the UI.
|
||||
ep_dict: dict[int, list[int]] = {}
|
||||
if db_series.episodes:
|
||||
for ep in db_series.episodes:
|
||||
if ep.is_downloaded:
|
||||
continue
|
||||
if ep.season not in ep_dict:
|
||||
ep_dict[ep.season] = []
|
||||
ep_dict[ep.season].append(ep.episode_number)
|
||||
for s in ep_dict:
|
||||
ep_dict[s].sort()
|
||||
db_episode_dict_map[db_series.folder] = ep_dict
|
||||
|
||||
# If filter is "missing_episodes", get series with any missing episodes
|
||||
if filter_type == "missing_episodes":
|
||||
@@ -545,7 +565,12 @@ class AnimeService:
|
||||
name = getattr(serie, "name", "")
|
||||
site = getattr(serie, "site", "")
|
||||
folder = getattr(serie, "folder", "")
|
||||
episode_dict = getattr(serie, "episodeDict", {}) or {}
|
||||
# Use DB-backed episodeDict (is_downloaded=True already filtered out)
|
||||
# with in-memory episodeDict as fallback if the series isn't in DB yet.
|
||||
episode_dict = db_episode_dict_map.get(
|
||||
folder,
|
||||
getattr(serie, "episodeDict", {}) or {}
|
||||
)
|
||||
|
||||
# Apply filter if specified
|
||||
if filter_type == "missing_episodes":
|
||||
@@ -815,18 +840,24 @@ class AnimeService:
|
||||
- Adds new missing episodes that are not in the database
|
||||
- Removes episodes from database that are no longer missing
|
||||
(i.e., the file has been added to the filesystem)
|
||||
- Preserves episodes marked as downloaded (is_downloaded=True)
|
||||
so download history is not lost
|
||||
"""
|
||||
from src.server.database.service import AnimeSeriesService, EpisodeService
|
||||
|
||||
# Get existing episodes from database
|
||||
# Get existing episodes from database (all episodes, including downloaded)
|
||||
existing_episodes = await EpisodeService.get_by_series(db, existing.id)
|
||||
|
||||
# Build dict of existing episodes: {season: {ep_num: episode_id}}
|
||||
# and track which ones are already downloaded
|
||||
existing_dict: dict[int, dict[int, int]] = {}
|
||||
downloaded_set: set[tuple[int, int]] = set()
|
||||
for ep in existing_episodes:
|
||||
if ep.season not in existing_dict:
|
||||
existing_dict[ep.season] = {}
|
||||
existing_dict[ep.season][ep.episode_number] = ep.id
|
||||
if ep.is_downloaded:
|
||||
downloaded_set.add((ep.season, ep.episode_number))
|
||||
|
||||
# Get new missing episodes from scan
|
||||
new_dict = serie.episodeDict or {}
|
||||
@@ -857,9 +888,22 @@ class AnimeService:
|
||||
|
||||
# Remove episodes from database that are no longer missing
|
||||
# (i.e., the episode file now exists on the filesystem)
|
||||
# BUT: preserve episodes that are already downloaded (is_downloaded=True)
|
||||
# so we don't lose download history
|
||||
for season, eps_dict in existing_dict.items():
|
||||
for ep_num, episode_id in eps_dict.items():
|
||||
if (season, ep_num) not in new_missing_set:
|
||||
# Skip already-downloaded episodes — they should stay in DB
|
||||
# with is_downloaded=True to preserve download history
|
||||
if (season, ep_num) in downloaded_set:
|
||||
logger.debug(
|
||||
"Preserving downloaded episode in database: "
|
||||
"%s S%02dE%02d",
|
||||
serie.key,
|
||||
season,
|
||||
ep_num
|
||||
)
|
||||
continue
|
||||
await EpisodeService.delete(db, episode_id)
|
||||
logger.info(
|
||||
"Removed episode from database (no longer missing): "
|
||||
@@ -889,6 +933,10 @@ class AnimeService:
|
||||
|
||||
This method is called during initialization and after rescans
|
||||
to ensure the in-memory series list is in sync with the database.
|
||||
|
||||
Only episodes where is_downloaded=False are loaded into the
|
||||
in-memory episodeDict, so downloaded episodes are not shown
|
||||
as missing.
|
||||
"""
|
||||
from src.core.entities.series import Serie
|
||||
from src.server.database.connection import get_db_session
|
||||
@@ -903,9 +951,14 @@ class AnimeService:
|
||||
series_list = []
|
||||
for anime_series in anime_series_list:
|
||||
# Build episode_dict from episodes relationship
|
||||
# Only include episodes that are NOT downloaded (is_downloaded=False)
|
||||
# so the missing-episode list stays accurate
|
||||
episode_dict: dict[int, list[int]] = {}
|
||||
if anime_series.episodes:
|
||||
for episode in anime_series.episodes:
|
||||
# Skip downloaded episodes — they are not missing
|
||||
if episode.is_downloaded:
|
||||
continue
|
||||
season = episode.season
|
||||
if season not in episode_dict:
|
||||
episode_dict[season] = []
|
||||
@@ -919,7 +972,8 @@ class AnimeService:
|
||||
name=anime_series.name,
|
||||
site=anime_series.site,
|
||||
folder=anime_series.folder,
|
||||
episodeDict=episode_dict
|
||||
episodeDict=episode_dict,
|
||||
year=anime_series.year
|
||||
)
|
||||
series_list.append(serie)
|
||||
|
||||
@@ -962,23 +1016,39 @@ class AnimeService:
|
||||
logger.warning("Series not found in database: %s", series_key)
|
||||
return 0
|
||||
|
||||
# Get existing episodes from database
|
||||
# Get existing episodes from database (all, including downloaded)
|
||||
existing_episodes = await EpisodeService.get_by_series(db, series_db.id)
|
||||
|
||||
# Build dict of existing episodes: {season: {ep_num: episode_id}}
|
||||
# and track which ones are already downloaded
|
||||
existing_dict: dict[int, dict[int, int]] = {}
|
||||
downloaded_set: set[tuple[int, int]] = set()
|
||||
for ep in existing_episodes:
|
||||
if ep.season not in existing_dict:
|
||||
existing_dict[ep.season] = {}
|
||||
existing_dict[ep.season][ep.episode_number] = ep.id
|
||||
if ep.is_downloaded:
|
||||
downloaded_set.add((ep.season, ep.episode_number))
|
||||
|
||||
# Get new missing episodes from in-memory serie
|
||||
new_dict = serie.episodeDict or {}
|
||||
|
||||
# Add new missing episodes that are not in the database
|
||||
# Skip episodes that are already downloaded (is_downloaded=True)
|
||||
# so we don't re-add them as missing after they've been downloaded
|
||||
for season, episode_numbers in new_dict.items():
|
||||
existing_season_eps = existing_dict.get(season, {})
|
||||
for ep_num in episode_numbers:
|
||||
# Skip if already downloaded — don't re-add as missing
|
||||
if (season, ep_num) in downloaded_set:
|
||||
logger.debug(
|
||||
"Skipping already-downloaded episode: "
|
||||
"%s S%02dE%02d",
|
||||
series_key,
|
||||
season,
|
||||
ep_num,
|
||||
)
|
||||
continue
|
||||
if ep_num not in existing_season_eps:
|
||||
await EpisodeService.create(
|
||||
db=db,
|
||||
@@ -1014,20 +1084,23 @@ class AnimeService:
|
||||
if hasattr(self._app, 'list') and hasattr(self._app.list, 'keyDict'):
|
||||
serie = self._app.list.keyDict.get(series_key)
|
||||
if serie:
|
||||
# Convert episode dict keys to strings for JSON
|
||||
missing_episodes = {str(k): v for k, v in (serie.episodeDict or {}).items()}
|
||||
total_missing = sum(len(eps) for eps in missing_episodes.values())
|
||||
|
||||
# Fetch NFO metadata from database
|
||||
# Fetch NFO metadata and episodes from database.
|
||||
# Using DB as the authoritative source for missing_episodes
|
||||
# ensures that episodes marked is_downloaded=True are never
|
||||
# broadcast as missing, even if in-memory state is stale.
|
||||
has_nfo = False
|
||||
nfo_created_at = None
|
||||
nfo_updated_at = None
|
||||
tmdb_id = None
|
||||
tvdb_id = None
|
||||
missing_episodes: dict[str, list] = {}
|
||||
|
||||
try:
|
||||
from src.server.database.connection import get_db_session
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
from src.server.database.service import (
|
||||
AnimeSeriesService,
|
||||
EpisodeService,
|
||||
)
|
||||
|
||||
async with get_db_session() as db:
|
||||
db_series = await AnimeSeriesService.get_by_key(db, series_key)
|
||||
@@ -1043,12 +1116,31 @@ class AnimeService:
|
||||
)
|
||||
tmdb_id = db_series.tmdb_id
|
||||
tvdb_id = db_series.tvdb_id
|
||||
|
||||
# Build missing_episodes from DB, skipping is_downloaded=True
|
||||
db_episodes = await EpisodeService.get_by_series(
|
||||
db, db_series.id, only_missing=True
|
||||
)
|
||||
for ep in db_episodes:
|
||||
key_str = str(ep.season)
|
||||
if key_str not in missing_episodes:
|
||||
missing_episodes[key_str] = []
|
||||
missing_episodes[key_str].append(ep.episode_number)
|
||||
for s in missing_episodes:
|
||||
missing_episodes[s].sort()
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Could not fetch NFO data for %s: %s",
|
||||
"Could not fetch series data for %s from DB: %s",
|
||||
series_key,
|
||||
str(e)
|
||||
)
|
||||
# Fallback to in-memory state
|
||||
missing_episodes = {
|
||||
str(k): v
|
||||
for k, v in (serie.episodeDict or {}).items()
|
||||
}
|
||||
|
||||
total_missing = sum(len(eps) for eps in missing_episodes.values())
|
||||
|
||||
series_data = {
|
||||
"key": serie.key,
|
||||
@@ -1550,6 +1642,7 @@ async def sync_series_from_data_files(
|
||||
name=serie.name,
|
||||
site=serie.site,
|
||||
folder=serie.folder,
|
||||
year=serie.year if hasattr(serie, 'year') else None,
|
||||
)
|
||||
|
||||
# Create Episode records for each episode in episodeDict
|
||||
|
||||
@@ -14,6 +14,7 @@ import uuid
|
||||
from collections import deque
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Dict, List, Optional
|
||||
|
||||
import structlog
|
||||
@@ -68,6 +69,7 @@ class DownloadService:
|
||||
progress_service: Optional progress service for tracking
|
||||
"""
|
||||
self._anime_service = anime_service
|
||||
self._directory = anime_service._directory
|
||||
self._max_retries = max_retries
|
||||
self._progress_service = progress_service or get_progress_service()
|
||||
|
||||
@@ -79,6 +81,9 @@ class DownloadService:
|
||||
self._pending_queue: deque[DownloadItem] = deque()
|
||||
# Helper dict for O(1) lookup of pending items by ID
|
||||
self._pending_items_by_id: Dict[str, DownloadItem] = {}
|
||||
# Helper dict for O(1) lookup of pending items by episode identity
|
||||
# Key: (serie_id, season, episode), Value: item ID
|
||||
self._pending_by_episode: Dict[tuple, str] = {}
|
||||
self._active_download: Optional[DownloadItem] = None
|
||||
self._completed_items: deque[DownloadItem] = deque(maxlen=100)
|
||||
self._failed_items: deque[DownloadItem] = deque(maxlen=50)
|
||||
@@ -165,6 +170,27 @@ class DownloadService:
|
||||
logger.error("Failed to save item to database: %s", e)
|
||||
return item
|
||||
|
||||
async def _set_status_in_database(
|
||||
self,
|
||||
item_id: str,
|
||||
status: str,
|
||||
) -> bool:
|
||||
"""Set status on an item in the database.
|
||||
|
||||
Args:
|
||||
item_id: Download item ID
|
||||
status: New status value
|
||||
|
||||
Returns:
|
||||
True if update succeeded
|
||||
"""
|
||||
try:
|
||||
repository = self._get_repository()
|
||||
return await repository.set_status(item_id, status)
|
||||
except Exception as e:
|
||||
logger.error("Failed to set status in database: %s", e)
|
||||
return False
|
||||
|
||||
async def _set_error_in_database(
|
||||
self,
|
||||
item_id: str,
|
||||
@@ -186,6 +212,25 @@ class DownloadService:
|
||||
logger.error("Failed to set error in database: %s", e)
|
||||
return False
|
||||
|
||||
async def _increment_retry_in_database(
|
||||
self,
|
||||
item_id: str,
|
||||
) -> bool:
|
||||
"""Increment retry count on an item in the database.
|
||||
|
||||
Args:
|
||||
item_id: Download item ID
|
||||
|
||||
Returns:
|
||||
True if update succeeded
|
||||
"""
|
||||
try:
|
||||
repository = self._get_repository()
|
||||
return await repository.increment_retry(item_id)
|
||||
except Exception as e:
|
||||
logger.error("Failed to increment retry in database: %s", e)
|
||||
return False
|
||||
|
||||
async def _delete_from_database(self, item_id: str) -> bool:
|
||||
"""Delete an item from the database.
|
||||
|
||||
@@ -207,30 +252,33 @@ class DownloadService:
|
||||
series_key: str,
|
||||
season: int,
|
||||
episode: int,
|
||||
serie_folder: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""Remove a downloaded episode from the missing episodes list.
|
||||
"""Mark a downloaded episode as downloaded instead of deleting it.
|
||||
|
||||
Called when a download completes successfully to update both:
|
||||
1. The database (Episode record deleted)
|
||||
1. The database (Episode record marked is_downloaded=True)
|
||||
2. The in-memory Serie.episodeDict and series_list cache
|
||||
|
||||
This ensures the episode no longer appears as missing in both
|
||||
the API responses and the UI immediately after download.
|
||||
the API responses and the UI immediately after download,
|
||||
while preserving the download history.
|
||||
|
||||
Args:
|
||||
series_key: Unique provider key for the series
|
||||
season: Season number
|
||||
episode: Episode number within season
|
||||
serie_folder: Series folder name (required for file_path)
|
||||
|
||||
Returns:
|
||||
True if episode was removed, False otherwise
|
||||
True if episode was updated, False otherwise
|
||||
"""
|
||||
try:
|
||||
from src.server.database.connection import get_db_session
|
||||
from src.server.database.service import EpisodeService
|
||||
from src.server.database.service import AnimeSeriesService, EpisodeService
|
||||
|
||||
logger.info(
|
||||
"Attempting to remove missing episode from DB: "
|
||||
"Attempting to mark episode as downloaded in DB: "
|
||||
"%s S%02dE%02d",
|
||||
series_key,
|
||||
season,
|
||||
@@ -238,28 +286,63 @@ class DownloadService:
|
||||
)
|
||||
|
||||
async with get_db_session() as db:
|
||||
deleted = await EpisodeService.delete_by_series_and_episode(
|
||||
# Get series by key to find series_id
|
||||
series = await AnimeSeriesService.get_by_key(db, series_key)
|
||||
if not series:
|
||||
logger.warning(
|
||||
"Series not found for key: %s", series_key
|
||||
)
|
||||
return False
|
||||
|
||||
# Get episode by series_id, season, episode_number
|
||||
ep = await EpisodeService.get_by_episode(
|
||||
db=db,
|
||||
series_key=series_key,
|
||||
series_id=series.id,
|
||||
season=season,
|
||||
episode_number=episode,
|
||||
)
|
||||
if deleted:
|
||||
if not ep:
|
||||
logger.warning(
|
||||
"Episode not found in DB: %s S%02dE%02d",
|
||||
series_key,
|
||||
season,
|
||||
episode,
|
||||
)
|
||||
return False
|
||||
|
||||
# Construct file_path if serie_folder provided
|
||||
file_path = None
|
||||
if serie_folder:
|
||||
season_folder = f"Season {season}"
|
||||
file_path = str(
|
||||
Path(self._directory) / serie_folder / season_folder
|
||||
)
|
||||
|
||||
# Mark episode as downloaded instead of deleting
|
||||
updated = await EpisodeService.mark_downloaded(
|
||||
db=db,
|
||||
episode_id=ep.id,
|
||||
file_path=file_path or "",
|
||||
)
|
||||
|
||||
if updated:
|
||||
logger.info(
|
||||
"Successfully removed episode from DB missing list: "
|
||||
"Marked episode as downloaded in DB: "
|
||||
"%s S%02dE%02d, file_path=%s",
|
||||
series_key,
|
||||
season,
|
||||
episode,
|
||||
file_path,
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Failed to mark episode as downloaded: "
|
||||
"%s S%02dE%02d",
|
||||
series_key,
|
||||
season,
|
||||
episode,
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Episode not found in DB missing list "
|
||||
"(may already be removed): %s S%02dE%02d",
|
||||
series_key,
|
||||
season,
|
||||
episode,
|
||||
)
|
||||
return False
|
||||
|
||||
# Update in-memory Serie.episodeDict so list_missing is
|
||||
# immediately consistent without a full DB reload
|
||||
@@ -270,8 +353,8 @@ class DownloadService:
|
||||
try:
|
||||
self._anime_service._cached_list_missing.cache_clear()
|
||||
logger.debug(
|
||||
"Cleared list_missing cache after removing "
|
||||
"%s S%02dE%02d",
|
||||
"Cleared list_missing cache after marking "
|
||||
"%s S%02dE%02d as downloaded",
|
||||
series_key,
|
||||
season,
|
||||
episode,
|
||||
@@ -279,10 +362,35 @@ class DownloadService:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return deleted
|
||||
# Broadcast real-time update to frontend so the series card
|
||||
# immediately reflects the new downloaded state (no longer
|
||||
# shows the episode as missing) without waiting for a full
|
||||
# reload on DOWNLOAD_COMPLETED.
|
||||
try:
|
||||
await self._anime_service._broadcast_series_updated(
|
||||
series_key
|
||||
)
|
||||
logger.debug(
|
||||
"Broadcast series_updated after marking "
|
||||
"%s S%02dE%02d as downloaded",
|
||||
series_key,
|
||||
season,
|
||||
episode,
|
||||
)
|
||||
except Exception as broadcast_exc:
|
||||
logger.warning(
|
||||
"Failed to broadcast series update after marking "
|
||||
"%s S%02dE%02d as downloaded: %s",
|
||||
series_key,
|
||||
season,
|
||||
episode,
|
||||
broadcast_exc,
|
||||
)
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Failed to remove episode from missing list: "
|
||||
"Failed to mark episode as downloaded: "
|
||||
"%s S%02dE%02d - %s",
|
||||
series_key,
|
||||
season,
|
||||
@@ -409,7 +517,7 @@ class DownloadService:
|
||||
def _add_to_pending_queue(
|
||||
self, item: DownloadItem, front: bool = False
|
||||
) -> None:
|
||||
"""Add item to pending queue and update helper dict.
|
||||
"""Add item to pending queue and update helper dicts.
|
||||
|
||||
Args:
|
||||
item: Download item to add
|
||||
@@ -420,9 +528,12 @@ class DownloadService:
|
||||
else:
|
||||
self._pending_queue.append(item)
|
||||
self._pending_items_by_id[item.id] = item
|
||||
# Track by episode identity for deduplication
|
||||
ep_key = (item.serie_id, item.episode.season, item.episode.episode)
|
||||
self._pending_by_episode[ep_key] = item.id
|
||||
|
||||
def _remove_from_pending_queue(self, item_or_id: str) -> Optional[DownloadItem]: # noqa: E501
|
||||
"""Remove item from pending queue and update helper dict.
|
||||
"""Remove item from pending queue and update helper dicts.
|
||||
|
||||
Args:
|
||||
item_or_id: Item ID to remove
|
||||
@@ -442,6 +553,10 @@ class DownloadService:
|
||||
try:
|
||||
self._pending_queue.remove(item)
|
||||
del self._pending_items_by_id[item_id]
|
||||
# Clean up episode tracking
|
||||
ep_key = (item.serie_id, item.episode.season, item.episode.episode)
|
||||
if self._pending_by_episode.get(ep_key) == item_id:
|
||||
del self._pending_by_episode[ep_key]
|
||||
return item
|
||||
except (ValueError, KeyError):
|
||||
return None
|
||||
@@ -481,10 +596,35 @@ class DownloadService:
|
||||
# Initialize queue progress tracking if not already done
|
||||
await self._init_queue_progress()
|
||||
|
||||
# Filter out episodes already in pending queue
|
||||
episodes_to_add = []
|
||||
skipped_count = 0
|
||||
seen_in_batch: set = set() # Track duplicates within this batch
|
||||
for ep in episodes:
|
||||
ep_key = (serie_id, ep.season, ep.episode)
|
||||
if ep_key in self._pending_by_episode or ep_key in seen_in_batch:
|
||||
logger.debug(
|
||||
"Skipping duplicate episode in queue",
|
||||
serie_key=serie_id,
|
||||
season=ep.season,
|
||||
episode=ep.episode,
|
||||
)
|
||||
skipped_count += 1
|
||||
continue
|
||||
seen_in_batch.add(ep_key)
|
||||
episodes_to_add.append(ep)
|
||||
|
||||
if skipped_count > 0:
|
||||
logger.info(
|
||||
"Skipped %d duplicate episodes in queue",
|
||||
skipped_count,
|
||||
serie_key=serie_id,
|
||||
)
|
||||
|
||||
created_ids = []
|
||||
|
||||
try:
|
||||
for episode in episodes:
|
||||
for episode in episodes_to_add:
|
||||
item = DownloadItem(
|
||||
id=self._generate_item_id(),
|
||||
serie_id=serie_id,
|
||||
@@ -976,17 +1116,15 @@ class DownloadService:
|
||||
if item.retry_count >= self._max_retries:
|
||||
continue
|
||||
|
||||
# Move back to pending
|
||||
# Move back to pending (retry_count will be incremented
|
||||
# by _process_download when the item fails again)
|
||||
self._failed_items.remove(item)
|
||||
item.status = DownloadStatus.PENDING
|
||||
item.retry_count += 1
|
||||
item.error = None
|
||||
item.progress = None
|
||||
self._add_to_pending_queue(item)
|
||||
retried_ids.append(item.id)
|
||||
|
||||
# Status is now managed in-memory only
|
||||
|
||||
logger.info(
|
||||
"Retrying failed item: item_id=%s, retry_count=%d",
|
||||
item.id,
|
||||
@@ -994,18 +1132,23 @@ class DownloadService:
|
||||
)
|
||||
|
||||
if retried_ids:
|
||||
# Notify via progress service
|
||||
queue_status = await self.get_queue_status()
|
||||
await self._progress_service.update_progress(
|
||||
progress_id="download_queue",
|
||||
message=f"Retried {len(retried_ids)} failed items",
|
||||
metadata={
|
||||
"action": "items_retried",
|
||||
"retried_ids": retried_ids,
|
||||
"queue_status": queue_status.model_dump(mode="json"),
|
||||
},
|
||||
force_broadcast=True,
|
||||
)
|
||||
# Notify via progress service if available
|
||||
try:
|
||||
queue_status = await self.get_queue_status()
|
||||
await self._progress_service.update_progress(
|
||||
progress_id="download_queue",
|
||||
message=f"Retried {len(retried_ids)} failed items",
|
||||
metadata={
|
||||
"action": "items_retried",
|
||||
"retried_ids": retried_ids,
|
||||
"queue_status": queue_status.model_dump(mode="json"),
|
||||
},
|
||||
force_broadcast=True,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Failed to broadcast retry progress: %s", e
|
||||
)
|
||||
|
||||
return retried_ids
|
||||
|
||||
@@ -1084,12 +1227,13 @@ class DownloadService:
|
||||
# Delete completed item from download queue database
|
||||
await self._delete_from_database(item.id)
|
||||
|
||||
# Remove episode from missing episodes list
|
||||
# Mark episode as downloaded in missing episodes list
|
||||
# (both database and in-memory)
|
||||
removed = await self._remove_episode_from_missing_list(
|
||||
series_key=item.serie_id,
|
||||
season=item.episode.season,
|
||||
episode=item.episode.episode,
|
||||
serie_folder=item.serie_folder,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
@@ -1144,17 +1288,35 @@ class DownloadService:
|
||||
item.status = DownloadStatus.FAILED
|
||||
item.completed_at = datetime.now(timezone.utc)
|
||||
item.error = str(e)
|
||||
|
||||
# Increment retry count in memory and database
|
||||
item.retry_count += 1
|
||||
await self._increment_retry_in_database(item.id)
|
||||
|
||||
self._failed_items.append(item)
|
||||
|
||||
# Set error in database
|
||||
await self._set_error_in_database(item.id, str(e))
|
||||
|
||||
logger.error(
|
||||
"Download failed: item_id=%s, error=%s, retry_count=%d",
|
||||
item.id,
|
||||
str(e),
|
||||
item.retry_count,
|
||||
)
|
||||
# Check if max retries exceeded - move to dead-letter
|
||||
if item.retry_count >= self._max_retries:
|
||||
await self._set_status_in_database(
|
||||
item.id, DownloadStatus.PERMANENTLY_FAILED.value
|
||||
)
|
||||
logger.error(
|
||||
"Download permanently failed after max retries: "
|
||||
"item_id=%s, error=%s, retry_count=%d",
|
||||
item.id,
|
||||
str(e),
|
||||
item.retry_count,
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
"Download failed: item_id=%s, error=%s, retry_count=%d",
|
||||
item.id,
|
||||
str(e),
|
||||
item.retry_count,
|
||||
)
|
||||
# Note: Failure is already broadcast by AnimeService
|
||||
# via ProgressService when SeriesApp fires failed event
|
||||
|
||||
|
||||
@@ -28,6 +28,36 @@ _POSTER_DOWNLOAD_SEMAPHORE: asyncio.Semaphore = asyncio.Semaphore(3)
|
||||
_NFO_REPAIR_SEMAPHORE: asyncio.Semaphore = asyncio.Semaphore(3)
|
||||
|
||||
|
||||
async def _create_missing_nfo(series_dir: Path, series_name: str) -> None:
|
||||
"""Create minimal NFO for series without one.
|
||||
|
||||
Creates a fresh :class:`NFOService` per invocation so concurrent
|
||||
tasks cannot interfere with each other.
|
||||
|
||||
A module-level semaphore limits concurrent TMDB operations to 3.
|
||||
|
||||
Args:
|
||||
series_dir: Absolute path to the series folder.
|
||||
series_name: Human-readable series name for log messages.
|
||||
"""
|
||||
from src.core.services.nfo_factory import NFOServiceFactory
|
||||
|
||||
async with _NFO_REPAIR_SEMAPHORE:
|
||||
try:
|
||||
factory = NFOServiceFactory()
|
||||
nfo_service = factory.create()
|
||||
await nfo_service.create_minimal_nfo(
|
||||
serie_name=series_name,
|
||||
serie_folder=series_dir.name,
|
||||
)
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
logger.error(
|
||||
"NFO creation failed for %s: %s",
|
||||
series_name,
|
||||
exc,
|
||||
)
|
||||
|
||||
|
||||
async def _repair_one_series(series_dir: Path, series_name: str) -> None:
|
||||
"""Repair a single series NFO in isolation.
|
||||
|
||||
@@ -63,12 +93,13 @@ async def _repair_one_series(series_dir: Path, series_name: str) -> None:
|
||||
|
||||
|
||||
async def perform_nfo_repair_scan(background_loader=None) -> None:
|
||||
"""Scan all series folders and repair incomplete tvshow.nfo files.
|
||||
"""Scan all series folders, repair incomplete and create missing NFO files.
|
||||
|
||||
Called from ``FolderScanService.run_folder_scan()`` during the scheduled
|
||||
daily folder scan (not on every startup). Checks each subfolder of
|
||||
``settings.anime_directory`` for a ``tvshow.nfo`` and calls
|
||||
``_repair_one_series`` for every file with absent or empty required tags.
|
||||
``settings.anime_directory`` for a ``tvshow.nfo``:
|
||||
- Missing NFOs: creates minimal NFO via ``_create_missing_nfo``
|
||||
- Incomplete NFOs: repairs via ``_repair_one_series``
|
||||
|
||||
Each repair task creates its own isolated :class:`NFOService` /
|
||||
:class:`TMDBClient` so concurrent tasks never share an ``aiohttp``
|
||||
@@ -97,26 +128,33 @@ async def perform_nfo_repair_scan(background_loader=None) -> None:
|
||||
|
||||
queued = 0
|
||||
total = 0
|
||||
missing_nfo_count = 0
|
||||
for series_dir in sorted(anime_dir.iterdir()):
|
||||
if not series_dir.is_dir():
|
||||
continue
|
||||
nfo_path = series_dir / "tvshow.nfo"
|
||||
series_name = series_dir.name
|
||||
if not nfo_path.exists():
|
||||
# Create minimal NFO for series without one
|
||||
missing_nfo_count += 1
|
||||
asyncio.create_task(
|
||||
_create_missing_nfo(series_dir, series_name),
|
||||
name=f"nfo_create:{series_name}",
|
||||
)
|
||||
continue
|
||||
total += 1
|
||||
series_name = series_dir.name
|
||||
if nfo_needs_repair(nfo_path):
|
||||
queued += 1
|
||||
# Each task creates its own NFOService so connectors are isolated.
|
||||
asyncio.create_task(
|
||||
_repair_one_series(series_dir, series_name),
|
||||
name=f"nfo_repair:{series_name}",
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"NFO repair scan complete: %d of %d series queued for repair",
|
||||
"NFO repair scan complete: %d of %d series queued for repair, %d missing NFOs queued for creation",
|
||||
queued,
|
||||
total,
|
||||
missing_nfo_count,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -83,15 +83,12 @@ class QueueRepository:
|
||||
) -> DownloadItem:
|
||||
"""Convert database model to DownloadItem.
|
||||
|
||||
Note: Since the database model is simplified, status, priority,
|
||||
progress, and retry_count default to initial values.
|
||||
|
||||
Args:
|
||||
db_item: SQLAlchemy download queue item
|
||||
item_id: Optional override for item ID
|
||||
|
||||
Returns:
|
||||
Pydantic download item with default status/priority
|
||||
Pydantic download item with status/retry_count from database
|
||||
"""
|
||||
# Get episode info from the related Episode object
|
||||
episode = db_item.episode
|
||||
@@ -109,14 +106,14 @@ class QueueRepository:
|
||||
serie_folder=series.folder if series else "",
|
||||
serie_name=series.name if series else "",
|
||||
episode=episode_identifier,
|
||||
status=DownloadStatus.PENDING, # Default - managed in-memory
|
||||
priority=DownloadPriority.NORMAL, # Default - managed in-memory
|
||||
status=DownloadStatus(db_item.status), # From database
|
||||
priority=DownloadPriority.NORMAL, # Managed in-memory
|
||||
added_at=db_item.created_at or datetime.now(timezone.utc),
|
||||
started_at=db_item.started_at,
|
||||
completed_at=db_item.completed_at,
|
||||
progress=None, # Managed in-memory
|
||||
error=db_item.error_message,
|
||||
retry_count=0, # Managed in-memory
|
||||
retry_count=db_item.retry_count, # From database
|
||||
source_url=db_item.download_url,
|
||||
)
|
||||
|
||||
@@ -350,6 +347,110 @@ class QueueRepository:
|
||||
finally:
|
||||
if manage_session:
|
||||
await session.close()
|
||||
|
||||
async def set_status(
|
||||
self,
|
||||
item_id: str,
|
||||
status: str,
|
||||
db: Optional[AsyncSession] = None,
|
||||
) -> bool:
|
||||
"""Set status on a download item.
|
||||
|
||||
Args:
|
||||
item_id: Download item ID
|
||||
status: New status value
|
||||
db: Optional existing database session
|
||||
|
||||
Returns:
|
||||
True if update succeeded, False if item not found
|
||||
|
||||
Raises:
|
||||
QueueRepositoryError: If update fails
|
||||
"""
|
||||
session = db or self._db_session_factory()
|
||||
manage_session = db is None
|
||||
|
||||
try:
|
||||
result = await DownloadQueueService.set_status(
|
||||
session,
|
||||
int(item_id),
|
||||
status,
|
||||
)
|
||||
|
||||
if manage_session:
|
||||
await session.commit()
|
||||
|
||||
success = result is not None
|
||||
|
||||
if success:
|
||||
logger.debug(
|
||||
"Set status on queue item: item_id=%s, status=%s",
|
||||
item_id,
|
||||
status,
|
||||
)
|
||||
|
||||
return success
|
||||
|
||||
except ValueError:
|
||||
return False
|
||||
except Exception as e:
|
||||
if manage_session:
|
||||
await session.rollback()
|
||||
logger.error("Failed to set status: %s", e)
|
||||
raise QueueRepositoryError(f"Failed to set status: {e}") from e
|
||||
finally:
|
||||
if manage_session:
|
||||
await session.close()
|
||||
|
||||
async def increment_retry(
|
||||
self,
|
||||
item_id: str,
|
||||
db: Optional[AsyncSession] = None,
|
||||
) -> bool:
|
||||
"""Increment retry count on a download item.
|
||||
|
||||
Args:
|
||||
item_id: Download item ID
|
||||
db: Optional existing database session
|
||||
|
||||
Returns:
|
||||
True if update succeeded, False if item not found
|
||||
|
||||
Raises:
|
||||
QueueRepositoryError: If update fails
|
||||
"""
|
||||
session = db or self._db_session_factory()
|
||||
manage_session = db is None
|
||||
|
||||
try:
|
||||
result = await DownloadQueueService.increment_retry_count(
|
||||
session,
|
||||
int(item_id),
|
||||
)
|
||||
|
||||
if manage_session:
|
||||
await session.commit()
|
||||
|
||||
success = result is not None
|
||||
|
||||
if success:
|
||||
logger.debug(
|
||||
"Incremented retry count on queue item: item_id=%s",
|
||||
item_id,
|
||||
)
|
||||
|
||||
return success
|
||||
|
||||
except ValueError:
|
||||
return False
|
||||
except Exception as e:
|
||||
if manage_session:
|
||||
await session.rollback()
|
||||
logger.error("Failed to increment retry: %s", e)
|
||||
raise QueueRepositoryError(f"Failed to increment retry: {e}") from e
|
||||
finally:
|
||||
if manage_session:
|
||||
await session.close()
|
||||
|
||||
async def delete_item(
|
||||
self,
|
||||
|
||||
@@ -3,23 +3,32 @@
|
||||
Uses APScheduler's AsyncIOScheduler with CronTrigger for precise
|
||||
cron-based scheduling. The legacy interval-based loop has been removed
|
||||
in favour of the cron approach.
|
||||
|
||||
Jobs are persisted to a SQLite database so they survive process restarts.
|
||||
On startup, if the last scheduled run was missed (server was down at the
|
||||
cron time), the job is triggered immediately within a grace period.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import List, Optional
|
||||
|
||||
import structlog
|
||||
from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
|
||||
from src.server.models.config import SchedulerConfig
|
||||
from src.server.services.config_service import ConfigServiceError, get_config_service
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_JOB_ID = "scheduled_rescan"
|
||||
|
||||
# Grace period for missed jobs (1 hour — handles server downtime between
|
||||
# scheduled time and startup).
|
||||
_MISFIRE_GRACE_SECONDS = 3600
|
||||
|
||||
|
||||
class SchedulerServiceError(Exception):
|
||||
"""Service-level exception for scheduler operations."""
|
||||
@@ -44,6 +53,9 @@ class SchedulerService:
|
||||
self._config: Optional[SchedulerConfig] = None
|
||||
self._last_scan_time: Optional[datetime] = None
|
||||
self._scan_in_progress: bool = False
|
||||
# Cooldown tracking for auto-download to prevent rapid re-triggers
|
||||
self._last_auto_download_time: Optional[datetime] = None
|
||||
self._auto_download_cooldown_seconds: int = 300 # 5 minutes default
|
||||
logger.info("SchedulerService initialised")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
@@ -57,24 +69,39 @@ class SchedulerService:
|
||||
SchedulerServiceError: If the scheduler is already running or
|
||||
config cannot be loaded.
|
||||
"""
|
||||
logger.info("SchedulerService.start() called")
|
||||
if self._is_running:
|
||||
logger.warning("Scheduler start called but already running")
|
||||
raise SchedulerServiceError("Scheduler is already running")
|
||||
|
||||
try:
|
||||
config_service = get_config_service()
|
||||
config = config_service.load_config()
|
||||
self._config = config.scheduler
|
||||
logger.info("Scheduler config loaded successfully")
|
||||
except ConfigServiceError as exc:
|
||||
logger.error("Failed to load scheduler configuration", error=str(exc))
|
||||
logger.error("Failed to load scheduler configuration: %s", exc)
|
||||
raise SchedulerServiceError(f"Failed to load config: {exc}") from exc
|
||||
|
||||
self._scheduler = AsyncIOScheduler()
|
||||
jobstores = {
|
||||
"default": SQLAlchemyJobStore(url="sqlite:///./data/scheduler.db"),
|
||||
}
|
||||
self._scheduler = AsyncIOScheduler(jobstores=jobstores)
|
||||
|
||||
if not self._config.enabled:
|
||||
logger.info("Scheduler is disabled in configuration — not adding jobs")
|
||||
self._is_running = True
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"Scheduler config loaded: enabled=%s time=%s days=%s auto_download=%s folder_scan=%s",
|
||||
self._config.enabled,
|
||||
self._config.schedule_time,
|
||||
self._config.schedule_days,
|
||||
self._config.auto_download_after_rescan,
|
||||
self._config.folder_scan_enabled,
|
||||
)
|
||||
|
||||
trigger = self._build_cron_trigger()
|
||||
if trigger is None:
|
||||
logger.warning(
|
||||
@@ -82,23 +109,37 @@ class SchedulerService:
|
||||
)
|
||||
else:
|
||||
self._scheduler.add_job(
|
||||
self._perform_rescan,
|
||||
_run_rescan_job,
|
||||
trigger=trigger,
|
||||
id=_JOB_ID,
|
||||
replace_existing=True,
|
||||
misfire_grace_time=300,
|
||||
misfire_grace_time=_MISFIRE_GRACE_SECONDS,
|
||||
coalesce=True,
|
||||
)
|
||||
logger.info(
|
||||
"Scheduler started with cron trigger",
|
||||
schedule_time=self._config.schedule_time,
|
||||
schedule_days=self._config.schedule_days,
|
||||
"Scheduler started with cron trigger: time=%s days=%s",
|
||||
self._config.schedule_time,
|
||||
self._config.schedule_days,
|
||||
)
|
||||
|
||||
self._scheduler.start()
|
||||
self._is_running = True
|
||||
|
||||
# Startup recovery: if the server was down at the scheduled time and
|
||||
# the job is within the misfire window, APScheduler will run it
|
||||
# automatically. Log the scheduled time for visibility.
|
||||
# Note: next_run_time is only available AFTER scheduler.start()
|
||||
job = self._scheduler.get_job(_JOB_ID)
|
||||
if job:
|
||||
next_run = job.next_run_time
|
||||
logger.info(
|
||||
"Scheduler next run: %s",
|
||||
next_run.isoformat() if next_run else None,
|
||||
)
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""Stop the APScheduler gracefully."""
|
||||
logger.info("SchedulerService.stop() called")
|
||||
if not self._is_running:
|
||||
logger.debug("Scheduler stop called but not running")
|
||||
return
|
||||
@@ -106,8 +147,11 @@ class SchedulerService:
|
||||
if self._scheduler and self._scheduler.running:
|
||||
self._scheduler.shutdown(wait=False)
|
||||
logger.info("Scheduler stopped")
|
||||
else:
|
||||
logger.info("Scheduler stop: scheduler was not running")
|
||||
|
||||
self._is_running = False
|
||||
logger.info("SchedulerService stopped successfully")
|
||||
|
||||
async def trigger_rescan(self) -> bool:
|
||||
"""Manually trigger a library rescan.
|
||||
@@ -140,12 +184,12 @@ class SchedulerService:
|
||||
"""
|
||||
self._config = config
|
||||
logger.info(
|
||||
"Scheduler config reloaded",
|
||||
enabled=config.enabled,
|
||||
schedule_time=config.schedule_time,
|
||||
schedule_days=config.schedule_days,
|
||||
auto_download=config.auto_download_after_rescan,
|
||||
folder_scan=config.folder_scan_enabled,
|
||||
"Scheduler config reloaded: enabled=%s time=%s days=%s auto_download=%s folder_scan=%s",
|
||||
config.enabled,
|
||||
config.schedule_time,
|
||||
config.schedule_days,
|
||||
config.auto_download_after_rescan,
|
||||
config.folder_scan_enabled,
|
||||
)
|
||||
|
||||
if not self._scheduler or not self._scheduler.running:
|
||||
@@ -166,22 +210,23 @@ class SchedulerService:
|
||||
if self._scheduler.get_job(_JOB_ID):
|
||||
self._scheduler.reschedule_job(_JOB_ID, trigger=trigger)
|
||||
logger.info(
|
||||
"Scheduler rescheduled with cron trigger",
|
||||
schedule_time=config.schedule_time,
|
||||
schedule_days=config.schedule_days,
|
||||
"Scheduler rescheduled with cron trigger: time=%s days=%s",
|
||||
config.schedule_time,
|
||||
config.schedule_days,
|
||||
)
|
||||
else:
|
||||
self._scheduler.add_job(
|
||||
self._perform_rescan,
|
||||
_run_rescan_job,
|
||||
trigger=trigger,
|
||||
id=_JOB_ID,
|
||||
replace_existing=True,
|
||||
misfire_grace_time=300,
|
||||
misfire_grace_time=_MISFIRE_GRACE_SECONDS,
|
||||
coalesce=True,
|
||||
)
|
||||
logger.info(
|
||||
"Scheduler job added with cron trigger",
|
||||
schedule_time=config.schedule_time,
|
||||
schedule_days=config.schedule_days,
|
||||
"Scheduler job added with cron trigger: time=%s days=%s",
|
||||
config.schedule_time,
|
||||
config.schedule_days,
|
||||
)
|
||||
|
||||
def get_status(self) -> dict:
|
||||
@@ -235,10 +280,10 @@ class SchedulerService:
|
||||
day_of_week=day_of_week,
|
||||
)
|
||||
logger.debug(
|
||||
"CronTrigger built",
|
||||
hour=hour_str,
|
||||
minute=minute_str,
|
||||
day_of_week=day_of_week,
|
||||
"CronTrigger built: hour=%s minute=%s day_of_week=%s",
|
||||
hour_str,
|
||||
minute_str,
|
||||
day_of_week,
|
||||
)
|
||||
return trigger
|
||||
|
||||
@@ -252,16 +297,30 @@ class SchedulerService:
|
||||
ws_service = get_websocket_service()
|
||||
await ws_service.manager.broadcast({"type": event_type, "data": data})
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.warning("WebSocket broadcast failed", event=event_type, error=str(exc))
|
||||
logger.warning("WebSocket broadcast failed: event=%s error=%s", event_type, exc)
|
||||
|
||||
async def _auto_download_missing(self) -> None:
|
||||
"""Queue and start downloads for all series with missing episodes."""
|
||||
from datetime import timedelta # noqa: PLC0415
|
||||
|
||||
from src.server.models.download import EpisodeIdentifier # noqa: PLC0415
|
||||
from src.server.utils.dependencies import ( # noqa: PLC0415
|
||||
get_anime_service,
|
||||
get_download_service,
|
||||
)
|
||||
|
||||
# Check cooldown to prevent rapid re-triggers
|
||||
now = datetime.now(timezone.utc)
|
||||
if self._last_auto_download_time is not None:
|
||||
elapsed = now - self._last_auto_download_time
|
||||
if elapsed < timedelta(seconds=self._auto_download_cooldown_seconds):
|
||||
logger.debug(
|
||||
"Auto-download skipped: cooldown active (elapsed=%.1fs cooldown=%ds)",
|
||||
elapsed.total_seconds(),
|
||||
self._auto_download_cooldown_seconds,
|
||||
)
|
||||
return
|
||||
|
||||
anime_service = get_anime_service()
|
||||
download_service = get_download_service()
|
||||
|
||||
@@ -291,26 +350,31 @@ class SchedulerService:
|
||||
)
|
||||
queued_count += len(episodes)
|
||||
logger.info(
|
||||
"Auto-download queued episodes",
|
||||
series=series.get("key"),
|
||||
count=len(episodes),
|
||||
"Auto-download queued episodes for series=%s count=%d",
|
||||
series.get("key"),
|
||||
len(episodes),
|
||||
)
|
||||
|
||||
if queued_count:
|
||||
await download_service.start_queue_processing()
|
||||
logger.info("Auto-download queue processing started", queued=queued_count)
|
||||
logger.info("Auto-download queue processing started: queued=%d", queued_count)
|
||||
|
||||
await self._broadcast("auto_download_started", {"queued_count": queued_count})
|
||||
logger.info("Auto-download completed", queued_count=queued_count)
|
||||
logger.info("Auto-download completed: queued_count=%d", queued_count)
|
||||
|
||||
# Update cooldown timestamp after successful auto-download
|
||||
self._last_auto_download_time = datetime.now(timezone.utc)
|
||||
|
||||
async def _perform_rescan(self) -> None:
|
||||
"""Execute a library rescan and optionally trigger auto-download."""
|
||||
logger.info("Scheduler _perform_rescan entered: scan_in_progress=%s", self._scan_in_progress)
|
||||
if self._scan_in_progress:
|
||||
logger.warning("Skipping rescan: previous scan still in progress")
|
||||
return
|
||||
|
||||
self._scan_in_progress = True
|
||||
scan_start = datetime.now(timezone.utc)
|
||||
logger.info("Scheduled rescan started at %s", scan_start.isoformat())
|
||||
|
||||
try:
|
||||
logger.info("Starting scheduled library rescan")
|
||||
@@ -318,18 +382,20 @@ class SchedulerService:
|
||||
from src.server.utils.dependencies import get_anime_service # noqa: PLC0415
|
||||
|
||||
anime_service = get_anime_service()
|
||||
logger.info("Anime service obtained for rescan")
|
||||
|
||||
await self._broadcast(
|
||||
"scheduled_rescan_started",
|
||||
{"timestamp": scan_start.isoformat()},
|
||||
)
|
||||
|
||||
logger.info("Calling anime_service.rescan()...")
|
||||
await anime_service.rescan()
|
||||
|
||||
self._last_scan_time = datetime.now(timezone.utc)
|
||||
duration = (self._last_scan_time - scan_start).total_seconds()
|
||||
|
||||
logger.info("Scheduled library rescan completed", duration_seconds=duration)
|
||||
logger.info("Scheduled library rescan completed: duration=%.2fs", duration)
|
||||
|
||||
await self._broadcast(
|
||||
"scheduled_rescan_completed",
|
||||
@@ -346,8 +412,8 @@ class SchedulerService:
|
||||
await self._auto_download_missing()
|
||||
except Exception as dl_exc: # pylint: disable=broad-exception-caught
|
||||
logger.error(
|
||||
"Auto-download after rescan failed",
|
||||
error=str(dl_exc),
|
||||
"Auto-download after rescan failed: %s",
|
||||
dl_exc,
|
||||
exc_info=True,
|
||||
)
|
||||
await self._broadcast(
|
||||
@@ -366,10 +432,11 @@ class SchedulerService:
|
||||
|
||||
folder_scan_service = FolderScanService()
|
||||
await folder_scan_service.run_folder_scan()
|
||||
logger.info("Folder scan completed successfully")
|
||||
except Exception as fs_exc: # pylint: disable=broad-exception-caught
|
||||
logger.error(
|
||||
"Folder scan failed",
|
||||
error=str(fs_exc),
|
||||
"Folder scan failed: %s",
|
||||
fs_exc,
|
||||
exc_info=True,
|
||||
)
|
||||
await self._broadcast(
|
||||
@@ -379,7 +446,7 @@ class SchedulerService:
|
||||
logger.debug("Folder scan is disabled — skipping")
|
||||
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.error("Scheduled rescan failed", error=str(exc), exc_info=True)
|
||||
logger.error("Scheduled rescan failed: %s", exc, exc_info=True)
|
||||
await self._broadcast(
|
||||
"scheduled_rescan_error",
|
||||
{"error": str(exc), "timestamp": datetime.now(timezone.utc).isoformat()},
|
||||
@@ -387,6 +454,27 @@ class SchedulerService:
|
||||
|
||||
finally:
|
||||
self._scan_in_progress = False
|
||||
logger.info("Scheduled rescan finished: scan_in_progress reset to False")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module-level job runner
|
||||
#
|
||||
# APScheduler cannot serialize bound methods (SchedulerService instance
|
||||
# contains a reference to the scheduler itself, creating a circular pickle
|
||||
# error). Using a module-level function avoids this.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def _run_rescan_job() -> None:
|
||||
"""Module-level job entry point — delegates to the current service."""
|
||||
logger.info("=" * 60)
|
||||
logger.info("APScheduler triggered _run_rescan_job")
|
||||
logger.info("Getting scheduler service singleton...")
|
||||
svc = get_scheduler_service()
|
||||
logger.info("Scheduler service obtained, calling _perform_rescan()")
|
||||
await svc._perform_rescan()
|
||||
logger.info("_run_rescan_job completed")
|
||||
logger.info("=" * 60)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -400,7 +488,10 @@ def get_scheduler_service() -> SchedulerService:
|
||||
"""Return the singleton SchedulerService instance."""
|
||||
global _scheduler_service
|
||||
if _scheduler_service is None:
|
||||
logger.info("Creating new SchedulerService singleton")
|
||||
_scheduler_service = SchedulerService()
|
||||
else:
|
||||
logger.debug("Returning existing SchedulerService singleton")
|
||||
return _scheduler_service
|
||||
|
||||
|
||||
|
||||
@@ -203,6 +203,17 @@ AniWorld.SeriesManager = (function() {
|
||||
function applyFiltersAndSort() {
|
||||
let filtered = seriesData.slice();
|
||||
|
||||
// Apply client-side filter so that real-time WebSocket updates
|
||||
// (e.g. an episode being marked downloaded) are immediately
|
||||
// reflected without a full server reload.
|
||||
if (filterMode === 'missing_episodes') {
|
||||
filtered = filtered.filter(function(s) {
|
||||
return s.missing_episodes > 0;
|
||||
});
|
||||
}
|
||||
// 'no_episodes' filter state is maintained server-side;
|
||||
// don't try to replicate it client-side here.
|
||||
|
||||
// Sort based on the current sorting mode
|
||||
filtered.sort(function(a, b) {
|
||||
if (sortAlphabetical) {
|
||||
@@ -233,8 +244,12 @@ AniWorld.SeriesManager = (function() {
|
||||
*/
|
||||
function renderSeries() {
|
||||
const grid = document.getElementById('series-grid');
|
||||
const dataToRender = filteredSeriesData.length > 0 ? filteredSeriesData :
|
||||
(seriesData.length > 0 ? seriesData : []);
|
||||
// Always use filteredSeriesData — applyFiltersAndSort() is always
|
||||
// called before renderSeries(), so filteredSeriesData is current.
|
||||
// The old fallback to seriesData was incorrect: when a filter is
|
||||
// active and filteredSeriesData is empty it must show the empty-state
|
||||
// message, not fall through to unfiltered seriesData.
|
||||
const dataToRender = filteredSeriesData;
|
||||
|
||||
if (dataToRender.length === 0) {
|
||||
let message;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Pytest configuration and shared fixtures for all tests."""
|
||||
|
||||
import logging
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
@@ -149,3 +150,44 @@ def mock_series_app_download(monkeypatch):
|
||||
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_logging_state():
|
||||
"""Reset logging handlers and propagate flags before and after each test.
|
||||
|
||||
Tests that call setup_logging() or logging.config.dictConfig() may leave
|
||||
FileHandlers and propagate=False on various loggers. This fixture clears
|
||||
handlers and resets propagate for all relevant loggers before/after tests.
|
||||
"""
|
||||
# All loggers that might have handlers or propagate changes from test setup
|
||||
logger_names = (
|
||||
"aniworld", "uvicorn", "uvicorn.access", "uvicorn.error",
|
||||
"watchfiles.main"
|
||||
)
|
||||
|
||||
def clear_logger_state(logger_name):
|
||||
logger = logging.getLogger(logger_name)
|
||||
for handler in logger.handlers[:]:
|
||||
logger.removeHandler(handler)
|
||||
handler.close()
|
||||
# Reset propagate to default (True) for child loggers
|
||||
# Root logger propagate is always True by default
|
||||
if logger_name != "root":
|
||||
logger.propagate = True
|
||||
|
||||
# Clear state BEFORE test
|
||||
for name in logger_names:
|
||||
clear_logger_state(name)
|
||||
|
||||
yield
|
||||
|
||||
# Clear state AFTER test
|
||||
for name in logger_names:
|
||||
clear_logger_state(name)
|
||||
|
||||
# Also clear root handlers
|
||||
root = logging.getLogger()
|
||||
for handler in root.handlers[:]:
|
||||
root.removeHandler(handler)
|
||||
handler.close()
|
||||
|
||||
|
||||
@@ -392,23 +392,33 @@ class TestAddSeriesWithEpisodes:
|
||||
nfo_created_at=datetime(2024, 1, 1, 12, 0, 0),
|
||||
nfo_updated_at=datetime(2024, 1, 2, 12, 0, 0)
|
||||
)
|
||||
|
||||
mock_db_series.id = 1
|
||||
|
||||
# Create service with mocked WebSocket
|
||||
anime_service = AnimeService(mock_series_app)
|
||||
mock_websocket = AsyncMock()
|
||||
anime_service._websocket_service = mock_websocket
|
||||
|
||||
|
||||
# Mock database session and service
|
||||
mock_db_session = AsyncMock()
|
||||
mock_db_session.__aenter__ = AsyncMock(return_value=mock_db_session)
|
||||
mock_db_session.__aexit__ = AsyncMock()
|
||||
|
||||
|
||||
# Mock episodes that match the in-memory episodeDict
|
||||
mock_episodes = [
|
||||
MagicMock(season=1, episode_number=1),
|
||||
MagicMock(season=1, episode_number=2),
|
||||
MagicMock(season=1, episode_number=3),
|
||||
]
|
||||
|
||||
with patch('src.server.database.connection.get_db_session', return_value=mock_db_session):
|
||||
with patch('src.server.database.service.AnimeSeriesService') as MockAnimeSeriesService:
|
||||
MockAnimeSeriesService.get_by_key = AsyncMock(return_value=mock_db_series)
|
||||
|
||||
# Act
|
||||
await anime_service._broadcast_series_updated(key)
|
||||
with patch('src.server.database.service.EpisodeService') as MockEpisodeService:
|
||||
MockEpisodeService.get_by_series = AsyncMock(return_value=mock_episodes)
|
||||
|
||||
# Act
|
||||
await anime_service._broadcast_series_updated(key)
|
||||
|
||||
# Assert
|
||||
mock_websocket.broadcast.assert_called_once()
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
"""Unit tests for aniworld_provider.py - Anime catalog scraping, episode listing, streaming link extraction."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from src.core.providers.aniworld_provider import AniworldLoader
|
||||
|
||||
@@ -472,3 +474,284 @@ class TestAniworldEvents:
|
||||
# Fire event - handler should NOT be called
|
||||
loader.events.download_progress({"status": "downloading"})
|
||||
handler.assert_not_called()
|
||||
|
||||
|
||||
class TestAniworldHealthCheck:
|
||||
"""Tests for the _check_url_alive HEAD probe."""
|
||||
|
||||
def test_returns_true_on_200(self, loader):
|
||||
loader.session.head.return_value = MagicMock(status_code=200)
|
||||
assert loader._check_url_alive("https://provider/x") is True
|
||||
|
||||
def test_returns_false_on_404(self, loader):
|
||||
loader.session.head.return_value = MagicMock(status_code=404)
|
||||
assert loader._check_url_alive("https://provider/x") is False
|
||||
|
||||
def test_returns_false_on_403(self, loader):
|
||||
loader.session.head.return_value = MagicMock(status_code=403)
|
||||
assert loader._check_url_alive("https://provider/x") is False
|
||||
|
||||
def test_falls_back_to_get_when_head_disallowed(self, loader):
|
||||
loader.session.head.return_value = MagicMock(status_code=405)
|
||||
get_resp = MagicMock(status_code=200)
|
||||
get_resp.close = MagicMock()
|
||||
loader.session.get.return_value = get_resp
|
||||
assert loader._check_url_alive("https://provider/x") is True
|
||||
loader.session.get.assert_called_once()
|
||||
|
||||
def test_returns_false_on_connection_error(self, loader):
|
||||
loader.session.head.side_effect = requests.ConnectionError("boom")
|
||||
assert loader._check_url_alive("https://provider/x") is False
|
||||
|
||||
|
||||
class TestAniworldDirectStream:
|
||||
"""Tests for the _try_direct_stream fast-path."""
|
||||
|
||||
def _build_response(self, status, content_type, body=b""):
|
||||
resp = MagicMock()
|
||||
resp.ok = status < 400
|
||||
resp.status_code = status
|
||||
resp.headers = {"Content-Type": content_type}
|
||||
resp.iter_content = MagicMock(return_value=[body])
|
||||
resp.__enter__ = MagicMock(return_value=resp)
|
||||
resp.__exit__ = MagicMock(return_value=False)
|
||||
return resp
|
||||
|
||||
def test_skips_non_video_content(self, loader, tmp_path):
|
||||
target = tmp_path / "out.mp4"
|
||||
loader.session.get.return_value = self._build_response(
|
||||
200, "text/html"
|
||||
)
|
||||
assert loader._try_direct_stream(
|
||||
"https://x", str(target), None, 10
|
||||
) is False
|
||||
assert not target.exists()
|
||||
|
||||
def test_writes_video_content(self, loader, tmp_path):
|
||||
target = tmp_path / "out.mp4"
|
||||
loader.session.get.return_value = self._build_response(
|
||||
200, "video/mp4", body=b"abc123"
|
||||
)
|
||||
assert loader._try_direct_stream(
|
||||
"https://x", str(target), None, 10
|
||||
) is True
|
||||
assert target.read_bytes() == b"abc123"
|
||||
|
||||
def test_returns_false_on_http_error(self, loader, tmp_path):
|
||||
target = tmp_path / "out.mp4"
|
||||
loader.session.get.return_value = self._build_response(
|
||||
404, "video/mp4"
|
||||
)
|
||||
assert loader._try_direct_stream(
|
||||
"https://x", str(target), None, 10
|
||||
) is False
|
||||
|
||||
def test_returns_false_on_request_exception(self, loader, tmp_path):
|
||||
loader.session.get.side_effect = requests.RequestException("nope")
|
||||
assert loader._try_direct_stream(
|
||||
"https://x", str(tmp_path / "out.mp4"), None, 10
|
||||
) is False
|
||||
|
||||
|
||||
class TestAniworldProviderSelection:
|
||||
"""Tests for _select_providers_for_episode ordering and filtering."""
|
||||
|
||||
def test_orders_by_supported_preference(self, loader):
|
||||
loader.is_language = MagicMock(return_value=True)
|
||||
loader._get_provider_from_html = MagicMock(return_value={
|
||||
"Vidoza": {1: "https://aniworld.to/redirect/2"},
|
||||
"VOE": {1: "https://aniworld.to/redirect/1"},
|
||||
})
|
||||
result = loader._select_providers_for_episode(1, 1, "k", "German Dub")
|
||||
assert [name for name, _ in result] == ["VOE", "Vidoza"]
|
||||
|
||||
def test_filters_by_language(self, loader):
|
||||
loader.is_language = MagicMock(return_value=True)
|
||||
loader._get_provider_from_html = MagicMock(return_value={
|
||||
"VOE": {2: "https://aniworld.to/redirect/1"}, # English only
|
||||
})
|
||||
result = loader._select_providers_for_episode(1, 1, "k", "German Dub")
|
||||
assert result == []
|
||||
|
||||
def test_returns_empty_when_language_unavailable(self, loader):
|
||||
loader.is_language = MagicMock(return_value=False)
|
||||
loader._get_provider_from_html = MagicMock()
|
||||
result = loader._select_providers_for_episode(1, 1, "k", "German Dub")
|
||||
assert result == []
|
||||
loader._get_provider_from_html.assert_not_called()
|
||||
|
||||
|
||||
class TestAniworldDownloadFailover:
|
||||
"""Tests for the failover rotation in download()."""
|
||||
|
||||
@pytest.fixture
|
||||
def patched_loader(self, loader, tmp_path):
|
||||
"""Loader with side-effect heavy methods stubbed."""
|
||||
loader.get_title = MagicMock(return_value="Anime")
|
||||
loader._select_providers_for_episode = MagicMock(return_value=[
|
||||
("VOE", "https://aniworld.to/redirect/1"),
|
||||
("Doodstream", "https://aniworld.to/redirect/2"),
|
||||
])
|
||||
loader._check_url_alive = MagicMock(return_value=True)
|
||||
loader._try_direct_stream = MagicMock(return_value=False)
|
||||
loader.clear_cache = MagicMock()
|
||||
loader._resolve_direct_link = MagicMock(
|
||||
return_value=("https://cdn/video.m3u8", {"Referer": "https://x"})
|
||||
)
|
||||
return loader
|
||||
|
||||
def test_skips_provider_when_url_dead(self, patched_loader, tmp_path):
|
||||
# First provider URL fails health check, second succeeds and downloads
|
||||
patched_loader._check_url_alive.side_effect = [False, True]
|
||||
|
||||
def fake_ytdl(opts):
|
||||
outpath = opts["outtmpl"]
|
||||
os.makedirs(os.path.dirname(outpath), exist_ok=True)
|
||||
with open(outpath, "wb") as fh:
|
||||
fh.write(b"data")
|
||||
ydl = MagicMock()
|
||||
ydl.__enter__ = MagicMock(return_value=ydl)
|
||||
ydl.__exit__ = MagicMock(return_value=False)
|
||||
ydl.extract_info = MagicMock(return_value={"title": "t"})
|
||||
return ydl
|
||||
|
||||
with patch(
|
||||
"src.core.providers.aniworld_provider.YoutubeDL",
|
||||
side_effect=fake_ytdl,
|
||||
):
|
||||
result = patched_loader.download(
|
||||
str(tmp_path), "Anime", 1, 1, "k", "German Dub"
|
||||
)
|
||||
assert result is True
|
||||
assert patched_loader._check_url_alive.call_count == 2
|
||||
# Only second provider (Doodstream) attempted resolve
|
||||
patched_loader._resolve_direct_link.assert_called_once_with(
|
||||
"https://aniworld.to/redirect/2", "Doodstream"
|
||||
)
|
||||
|
||||
def test_falls_back_to_next_provider_on_ytdl_error(
|
||||
self, patched_loader, tmp_path
|
||||
):
|
||||
calls = {"n": 0}
|
||||
|
||||
def fake_ytdl(opts):
|
||||
calls["n"] += 1
|
||||
if calls["n"] == 1:
|
||||
raise Exception("HTTP 404 from VOE")
|
||||
outpath = opts["outtmpl"]
|
||||
os.makedirs(os.path.dirname(outpath), exist_ok=True)
|
||||
with open(outpath, "wb") as fh:
|
||||
fh.write(b"ok")
|
||||
ydl = MagicMock()
|
||||
ydl.__enter__ = MagicMock(return_value=ydl)
|
||||
ydl.__exit__ = MagicMock(return_value=False)
|
||||
ydl.extract_info = MagicMock(return_value={"title": "t"})
|
||||
return ydl
|
||||
|
||||
with patch(
|
||||
"src.core.providers.aniworld_provider.YoutubeDL",
|
||||
side_effect=fake_ytdl,
|
||||
):
|
||||
result = patched_loader.download(
|
||||
str(tmp_path), "Anime", 1, 1, "k", "German Dub"
|
||||
)
|
||||
assert result is True
|
||||
assert calls["n"] == 2
|
||||
|
||||
def test_uses_direct_stream_when_available(
|
||||
self, patched_loader, tmp_path
|
||||
):
|
||||
def write_direct(link, output, headers, timeout):
|
||||
os.makedirs(os.path.dirname(output), exist_ok=True)
|
||||
with open(output, "wb") as fh:
|
||||
fh.write(b"vid")
|
||||
return True
|
||||
|
||||
patched_loader._try_direct_stream.side_effect = write_direct
|
||||
|
||||
with patch(
|
||||
"src.core.providers.aniworld_provider.YoutubeDL"
|
||||
) as mock_ydl:
|
||||
result = patched_loader.download(
|
||||
str(tmp_path), "Anime", 1, 1, "k", "German Dub"
|
||||
)
|
||||
assert result is True
|
||||
mock_ydl.assert_not_called()
|
||||
|
||||
def test_returns_false_when_all_providers_fail(
|
||||
self, patched_loader, tmp_path, caplog
|
||||
):
|
||||
with patch(
|
||||
"src.core.providers.aniworld_provider.YoutubeDL",
|
||||
side_effect=Exception("HTTP 404"),
|
||||
):
|
||||
result = patched_loader.download(
|
||||
str(tmp_path), "Anime", 1, 1, "k", "German Dub"
|
||||
)
|
||||
assert result is False
|
||||
assert "All download providers failed" in caplog.text
|
||||
# Both providers attempted
|
||||
assert patched_loader._resolve_direct_link.call_count == 2
|
||||
|
||||
def test_returns_false_when_no_providers_advertised(
|
||||
self, patched_loader, tmp_path, caplog
|
||||
):
|
||||
patched_loader._select_providers_for_episode.return_value = []
|
||||
result = patched_loader.download(
|
||||
str(tmp_path), "Anime", 1, 1, "k", "German Dub"
|
||||
)
|
||||
assert result is False
|
||||
assert "No providers advertised" in caplog.text
|
||||
|
||||
|
||||
class TestAniworldHeaderParsing:
|
||||
"""_parse_provider_headers normalizes legacy strings to dict."""
|
||||
|
||||
def test_parses_referer(self):
|
||||
result = AniworldLoader._parse_provider_headers(
|
||||
['Referer: "https://vidmoly.to"']
|
||||
)
|
||||
assert result == {"Referer": "https://vidmoly.to"}
|
||||
|
||||
def test_handles_none(self):
|
||||
assert AniworldLoader._parse_provider_headers(None) == {}
|
||||
|
||||
def test_skips_malformed_entries(self):
|
||||
result = AniworldLoader._parse_provider_headers(
|
||||
["not-a-header", "Key: value"]
|
||||
)
|
||||
assert result == {"Key": "value"}
|
||||
|
||||
|
||||
class TestDecodeHtmlContent:
|
||||
"""Test _decode_html_content function."""
|
||||
|
||||
def test_decodes_utf8_content(self):
|
||||
"""Should correctly decode UTF-8 content."""
|
||||
from src.core.providers.aniworld_provider import _decode_html_content
|
||||
html = '<html><body><h1>Titel mit Ümläüten</h1></body></html>'
|
||||
content = html.encode('utf-8')
|
||||
result = _decode_html_content(content)
|
||||
assert 'Titel mit Ümläüten' in result
|
||||
|
||||
def test_decodes_latin1_content(self):
|
||||
"""Should correctly decode Latin-1 content when chardet detects it."""
|
||||
from src.core.providers.aniworld_provider import _decode_html_content
|
||||
# Longer content for more reliable chardet detection
|
||||
html = '<html><body><h1>CafÉ and more text here</h1></body></html>'
|
||||
content = html.encode('latin-1')
|
||||
result = _decode_html_content(content)
|
||||
assert 'Caf' in result # Decoded content contains expected substring
|
||||
|
||||
def test_replaces_invalid_bytes(self):
|
||||
"""Should replace invalid bytes with replacement character."""
|
||||
from src.core.providers.aniworld_provider import _decode_html_content
|
||||
content = b'\xff\xfe Invalid \x80\x81'
|
||||
result = _decode_html_content(content)
|
||||
assert isinstance(result, str)
|
||||
|
||||
def test_handles_empty_content(self):
|
||||
"""Should handle empty content gracefully."""
|
||||
from src.core.providers.aniworld_provider import _decode_html_content
|
||||
result = _decode_html_content(b'')
|
||||
assert result == ''
|
||||
|
||||
@@ -60,6 +60,27 @@ class MockQueueRepository:
|
||||
self._items[item_id].error = error
|
||||
return True
|
||||
|
||||
async def set_status(
|
||||
self,
|
||||
item_id: str,
|
||||
status: str,
|
||||
) -> bool:
|
||||
"""Set status on an item."""
|
||||
if item_id not in self._items:
|
||||
return False
|
||||
self._items[item_id].status = DownloadStatus(status)
|
||||
return True
|
||||
|
||||
async def increment_retry(
|
||||
self,
|
||||
item_id: str,
|
||||
) -> bool:
|
||||
"""Increment retry count on an item."""
|
||||
if item_id not in self._items:
|
||||
return False
|
||||
self._items[item_id].retry_count += 1
|
||||
return True
|
||||
|
||||
async def delete_item(self, item_id: str) -> bool:
|
||||
"""Delete item from storage."""
|
||||
if item_id in self._items:
|
||||
@@ -79,6 +100,8 @@ def mock_anime_service():
|
||||
"""Create a mock AnimeService."""
|
||||
service = MagicMock(spec=AnimeService)
|
||||
service.download = AsyncMock(return_value=True)
|
||||
service._directory = "/mock/anime/directory"
|
||||
service._broadcast_series_updated = AsyncMock(return_value=None)
|
||||
return service
|
||||
|
||||
|
||||
@@ -503,7 +526,9 @@ class TestRetryLogic:
|
||||
assert len(retried_ids) == 1
|
||||
assert len(download_service._failed_items) == 0
|
||||
assert len(download_service._pending_queue) == 1
|
||||
assert download_service._pending_queue[0].retry_count == 1
|
||||
# retry_count stays same when retrying; incremented only on failure
|
||||
assert download_service._pending_queue[0].retry_count == 0
|
||||
assert download_service._pending_queue[0].status == DownloadStatus.PENDING
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_max_retries_not_exceeded(self, download_service):
|
||||
@@ -526,6 +551,45 @@ class TestRetryLogic:
|
||||
assert len(retried_ids) == 0
|
||||
assert len(download_service._failed_items) == 1
|
||||
assert len(download_service._pending_queue) == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_permanently_failed_after_max_retries(self, download_service):
|
||||
"""Test that item is marked permanently_failed after max retries."""
|
||||
# Mock download to fail
|
||||
download_service._anime_service.download = AsyncMock(
|
||||
side_effect=Exception("Download failed")
|
||||
)
|
||||
|
||||
# Create item with max_retries - 1 already used
|
||||
item = DownloadItem(
|
||||
id="perm-failed-1",
|
||||
serie_id="series-1",
|
||||
serie_folder="Test Series (2023)",
|
||||
serie_name="Test Series",
|
||||
episode=EpisodeIdentifier(season=1, episode=1),
|
||||
status=DownloadStatus.PENDING,
|
||||
retry_count=2, # Already 2 retries, max is 3
|
||||
error=None,
|
||||
)
|
||||
download_service._pending_queue.append(item)
|
||||
|
||||
# Process download - will fail and reach max retries
|
||||
await download_service._process_download(item)
|
||||
|
||||
# Item should be in failed_items with permanently_failed status
|
||||
assert len(download_service._failed_items) == 1
|
||||
assert download_service._failed_items[0].retry_count == 3
|
||||
|
||||
|
||||
class TestDeadLetterQueue:
|
||||
"""Test dead-letter queue behavior for permanently failed items."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_requeue_permanently_failed_item(self, download_service):
|
||||
"""Test that a permanently failed item can be re-queued."""
|
||||
# The unique constraint now includes status, so a permanently_failed
|
||||
# item doesn't block re-queuing the same episode
|
||||
pass # Implementation depends on UI/API behavior
|
||||
|
||||
|
||||
class TestBroadcastCallbacks:
|
||||
@@ -731,13 +795,22 @@ class TestRemoveEpisodeFromMissingList:
|
||||
download_service._anime_service._app = mock_app
|
||||
download_service._anime_service._cached_list_missing = MagicMock()
|
||||
|
||||
# Mock DB call
|
||||
# Mock DB session
|
||||
mock_db_session = AsyncMock()
|
||||
mock_delete = AsyncMock(return_value=True)
|
||||
|
||||
# Mock series returned by get_by_key
|
||||
mock_series = MagicMock()
|
||||
mock_series.id = 1
|
||||
|
||||
# Mock episode returned by get_by_episode
|
||||
mock_episode = MagicMock()
|
||||
mock_episode.id = 100
|
||||
|
||||
with patch(
|
||||
"src.server.database.connection.get_db_session"
|
||||
) as mock_get_db, patch(
|
||||
"src.server.database.service.AnimeSeriesService"
|
||||
) as mock_series_svc, patch(
|
||||
"src.server.database.service.EpisodeService"
|
||||
) as mock_ep_svc:
|
||||
mock_get_db.return_value.__aenter__ = AsyncMock(
|
||||
@@ -746,26 +819,40 @@ class TestRemoveEpisodeFromMissingList:
|
||||
mock_get_db.return_value.__aexit__ = AsyncMock(
|
||||
return_value=False
|
||||
)
|
||||
mock_ep_svc.delete_by_series_and_episode = mock_delete
|
||||
|
||||
# Mock get_by_key to return series
|
||||
mock_series_svc.get_by_key = AsyncMock(return_value=mock_series)
|
||||
|
||||
# Mock get_by_episode to return episode
|
||||
mock_ep_svc.get_by_episode = AsyncMock(return_value=mock_episode)
|
||||
|
||||
# Mock mark_downloaded to succeed
|
||||
mock_ep_svc.mark_downloaded = AsyncMock(return_value=mock_episode)
|
||||
|
||||
result = await download_service._remove_episode_from_missing_list(
|
||||
series_key="test-series",
|
||||
season=1,
|
||||
episode=2,
|
||||
serie_folder="Test Series (2024)",
|
||||
)
|
||||
|
||||
# DB deletion was called
|
||||
mock_delete.assert_awaited_once_with(
|
||||
# mark_downloaded was called instead of delete
|
||||
mock_ep_svc.mark_downloaded.assert_awaited_once_with(
|
||||
db=mock_db_session,
|
||||
series_key="test-series",
|
||||
season=1,
|
||||
episode_number=2,
|
||||
episode_id=100,
|
||||
file_path=(
|
||||
f"{download_service._directory}/Test Series (2024)/Season 1"
|
||||
),
|
||||
)
|
||||
# In-memory update happened
|
||||
assert 2 not in serie.episodeDict[1]
|
||||
assert serie.episodeDict[1] == [1, 3]
|
||||
# Cache was cleared
|
||||
download_service._anime_service._cached_list_missing.cache_clear.assert_called()
|
||||
# Broadcast was sent so frontend gets real-time update
|
||||
download_service._anime_service._broadcast_series_updated.assert_awaited_once_with(
|
||||
"test-series"
|
||||
)
|
||||
assert result is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -807,11 +894,20 @@ class TestRemoveEpisodeFromMissingList:
|
||||
|
||||
# Mock DB calls
|
||||
mock_db_session = AsyncMock()
|
||||
mock_delete = AsyncMock(return_value=True)
|
||||
|
||||
# Mock series returned by get_by_key
|
||||
mock_series = MagicMock()
|
||||
mock_series.id = 1
|
||||
|
||||
# Mock episode returned by get_by_episode
|
||||
mock_episode = MagicMock()
|
||||
mock_episode.id = 100
|
||||
|
||||
with patch(
|
||||
"src.server.database.connection.get_db_session"
|
||||
) as mock_get_db, patch(
|
||||
"src.server.database.service.AnimeSeriesService"
|
||||
) as mock_series_svc, patch(
|
||||
"src.server.database.service.EpisodeService"
|
||||
) as mock_ep_svc:
|
||||
mock_get_db.return_value.__aenter__ = AsyncMock(
|
||||
@@ -820,7 +916,15 @@ class TestRemoveEpisodeFromMissingList:
|
||||
mock_get_db.return_value.__aexit__ = AsyncMock(
|
||||
return_value=False
|
||||
)
|
||||
mock_ep_svc.delete_by_series_and_episode = mock_delete
|
||||
|
||||
# Mock get_by_key to return series
|
||||
mock_series_svc.get_by_key = AsyncMock(return_value=mock_series)
|
||||
|
||||
# Mock get_by_episode to return episode
|
||||
mock_ep_svc.get_by_episode = AsyncMock(return_value=mock_episode)
|
||||
|
||||
# Mock mark_downloaded to succeed
|
||||
mock_ep_svc.mark_downloaded = AsyncMock(return_value=mock_episode)
|
||||
|
||||
# Process the download
|
||||
item = download_service._pending_queue.popleft()
|
||||
@@ -834,3 +938,111 @@ class TestRemoveEpisodeFromMissingList:
|
||||
# Episode 2 should be removed from in-memory missing list
|
||||
assert 2 not in serie.episodeDict[1]
|
||||
assert serie.episodeDict[1] == [1, 3]
|
||||
|
||||
|
||||
class TestQueueDeduplication:
|
||||
"""Test queue deduplication to prevent duplicate entries."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_same_episode_twice_creates_only_one_entry(
|
||||
self, download_service
|
||||
):
|
||||
"""Test that adding the same episode twice only creates one queue entry."""
|
||||
episodes = [EpisodeIdentifier(season=1, episode=1)]
|
||||
|
||||
# Add same episode twice
|
||||
ids1 = await download_service.add_to_queue(
|
||||
serie_id="series-1",
|
||||
serie_folder="series",
|
||||
serie_name="Test Series",
|
||||
episodes=episodes,
|
||||
)
|
||||
ids2 = await download_service.add_to_queue(
|
||||
serie_id="series-1",
|
||||
serie_folder="series",
|
||||
serie_name="Test Series",
|
||||
episodes=episodes,
|
||||
)
|
||||
|
||||
# Should only have one entry
|
||||
assert len(download_service._pending_queue) == 1
|
||||
# First call creates one ID
|
||||
assert len(ids1) == 1
|
||||
# Second call creates zero IDs (deduplicated)
|
||||
assert len(ids2) == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_different_episodes_creates_separate_entries(
|
||||
self, download_service
|
||||
):
|
||||
"""Test that different episodes create separate queue entries."""
|
||||
episodes1 = [EpisodeIdentifier(season=1, episode=1)]
|
||||
episodes2 = [EpisodeIdentifier(season=1, episode=2)]
|
||||
|
||||
ids1 = await download_service.add_to_queue(
|
||||
serie_id="series-1",
|
||||
serie_folder="series",
|
||||
serie_name="Test Series",
|
||||
episodes=episodes1,
|
||||
)
|
||||
ids2 = await download_service.add_to_queue(
|
||||
serie_id="series-1",
|
||||
serie_folder="series",
|
||||
serie_name="Test Series",
|
||||
episodes=episodes2,
|
||||
)
|
||||
|
||||
# Should have two separate entries
|
||||
assert len(download_service._pending_queue) == 2
|
||||
assert len(ids1) == 1
|
||||
assert len(ids2) == 1
|
||||
# IDs should be different
|
||||
assert ids1[0] != ids2[0]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_same_episode_different_series_creates_entries(
|
||||
self, download_service
|
||||
):
|
||||
"""Test that same episode in different series creates separate entries."""
|
||||
episodes = [EpisodeIdentifier(season=1, episode=1)]
|
||||
|
||||
ids1 = await download_service.add_to_queue(
|
||||
serie_id="series-1",
|
||||
serie_folder="series1",
|
||||
serie_name="Test Series 1",
|
||||
episodes=episodes,
|
||||
)
|
||||
ids2 = await download_service.add_to_queue(
|
||||
serie_id="series-2",
|
||||
serie_folder="series2",
|
||||
serie_name="Test Series 2",
|
||||
episodes=episodes,
|
||||
)
|
||||
|
||||
# Should have two separate entries (different series)
|
||||
assert len(download_service._pending_queue) == 2
|
||||
assert len(ids1) == 1
|
||||
assert len(ids2) == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_multiple_episodes_with_duplicates_filters_correctly(
|
||||
self, download_service
|
||||
):
|
||||
"""Test that adding multiple episodes with some duplicates filters correctly."""
|
||||
episodes = [
|
||||
EpisodeIdentifier(season=1, episode=1),
|
||||
EpisodeIdentifier(season=1, episode=2),
|
||||
EpisodeIdentifier(season=1, episode=1), # duplicate
|
||||
EpisodeIdentifier(season=1, episode=3),
|
||||
]
|
||||
|
||||
ids1 = await download_service.add_to_queue(
|
||||
serie_id="series-1",
|
||||
serie_folder="series",
|
||||
serie_name="Test Series",
|
||||
episodes=episodes,
|
||||
)
|
||||
|
||||
# Should only have 3 entries (1, 2, 3) - one filtered out
|
||||
assert len(download_service._pending_queue) == 3
|
||||
assert len(ids1) == 3
|
||||
|
||||
@@ -917,4 +917,97 @@ class TestAniworldLoaderCompat:
|
||||
"""AniworldLoader should extend EnhancedAniWorldLoader."""
|
||||
from src.core.providers.enhanced_provider import AniworldLoader
|
||||
|
||||
assert issubclass(AniworldLoader, EnhancedAniWorldLoader)
|
||||
assert issubclass(AniworldLoader, EnhancedAniWorldLoader)
|
||||
|
||||
class TestFfmpegHlsOptions:
|
||||
"""Test that yt-dlp is configured with ffmpeg for HLS streams."""
|
||||
|
||||
def test_ytdl_opts_include_ffmpeg_for_hls(self, enhanced_loader, tmp_path):
|
||||
"""yt-dlp options should include ffmpeg downloader and hls-use-mpegts."""
|
||||
temp_path = str(tmp_path / "temp.mp4")
|
||||
output_path = str(tmp_path / "output.mp4")
|
||||
|
||||
captured_opts = {}
|
||||
|
||||
def capture_ytdl_download(self, temp_path, ydl_opts, link):
|
||||
captured_opts.update(ydl_opts)
|
||||
with open(temp_path, "wb") as f:
|
||||
f.write(b"fake-video-data")
|
||||
return True
|
||||
|
||||
with patch(
|
||||
"src.core.providers.enhanced_provider.recovery_strategies"
|
||||
) as mock_rs, patch(
|
||||
"src.core.providers.enhanced_provider.file_corruption_detector"
|
||||
) as mock_fcd, patch(
|
||||
"src.core.providers.enhanced_provider.get_integrity_manager"
|
||||
) as mock_im:
|
||||
mock_rs.handle_network_failure.return_value = (
|
||||
"https://direct.example.com/v.mp4",
|
||||
[],
|
||||
)
|
||||
mock_rs.handle_download_failure.side_effect = capture_ytdl_download
|
||||
mock_fcd.is_valid_video_file.return_value = True
|
||||
mock_im.return_value.store_checksum.return_value = "abc123"
|
||||
|
||||
enhanced_loader._download_with_recovery(
|
||||
1, 1, "test", "German Dub",
|
||||
temp_path, output_path, None,
|
||||
)
|
||||
|
||||
assert captured_opts.get("downloader") == "ffmpeg", (
|
||||
f"Expected downloader='ffmpeg', got {captured_opts.get('downloader')}"
|
||||
)
|
||||
assert captured_opts.get("hls_use_mpegts") is True, (
|
||||
f"Expected hls_use_mpegts=True, got {captured_opts.get('hls_use_mpegts')}"
|
||||
)
|
||||
|
||||
|
||||
class TestHlsUrlDetection:
|
||||
"""Test HLS URL detection patterns."""
|
||||
|
||||
def test_voe_hls_pattern_extracts_hls_url(self):
|
||||
"""HLS_PATTERN should extract HLS URL from VOE embedded player HTML."""
|
||||
import re
|
||||
from src.core.providers.streaming.voe import HLS_PATTERN
|
||||
|
||||
html_with_hls = """
|
||||
var playerConfig = {
|
||||
'hls': 'aHR0cHM6Ly92b2Uuc3YvZS9hYmMuaGxtMTNobG0xNm0zNDU2Nzg5MGE0MzIxLm0zdTg=',
|
||||
'source': 'direct_mp4_url'
|
||||
};
|
||||
"""
|
||||
match = HLS_PATTERN.search(html_with_hls)
|
||||
assert match is not None
|
||||
assert match.group("hls") == "aHR0cHM6Ly92b2Uuc3YvZS9hYmMuaGxtMTNobG0xNm0zNDU2Nzg5MGE0MzIxLm0zdTg="
|
||||
|
||||
def test_voe_hls_pattern_returns_none_when_no_hls(self):
|
||||
"""HLS_PATTERN should return None when no HLS URL in HTML."""
|
||||
import re
|
||||
from src.core.providers.streaming.voe import HLS_PATTERN
|
||||
|
||||
html_no_hls = """
|
||||
var playerConfig = {
|
||||
'source': 'https://direct.example.com/video.mp4'
|
||||
};
|
||||
"""
|
||||
match = HLS_PATTERN.search(html_no_hls)
|
||||
assert match is None
|
||||
|
||||
def test_hls_url_detection_in_provider_flow(self, enhanced_loader, tmp_path):
|
||||
"""Provider should detect and handle HLS URLs from VOE extractor."""
|
||||
import re
|
||||
from src.core.providers.streaming.voe import HLS_PATTERN
|
||||
|
||||
# Simulate VOE returning an HLS URL (base64 encoded .m3u8)
|
||||
encoded_hls = "aHR0cHM6Ly9leGFtcGxlLmNvbS92aWRlby5tM3U4"
|
||||
expected_hls = "https://example.com/video.m3u8"
|
||||
|
||||
html = f"var playerConfig = {{'hls': '{encoded_hls}'}};"
|
||||
|
||||
# Verify pattern correctly decodes to an m3u8 URL
|
||||
match = HLS_PATTERN.search(html)
|
||||
assert match is not None
|
||||
decoded = match.group("hls")
|
||||
# Note: this is just the base64 encoding of the URL, not actual decoding in pattern
|
||||
assert decoded == encoded_hls
|
||||
|
||||
54
tests/unit/test_ffmpeg_health_check.py
Normal file
54
tests/unit/test_ffmpeg_health_check.py
Normal file
@@ -0,0 +1,54 @@
|
||||
"""Unit tests for ffmpeg health check in fastapi_app.py."""
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestFfmpegHealthCheck:
|
||||
"""Test ffmpeg health check warns when not in PATH."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ffmpeg_missing_warns(self):
|
||||
"""Should log warning when ffmpeg not found in PATH."""
|
||||
with patch("shutil.which", return_value=None):
|
||||
with patch("src.server.fastapi_app.setup_logging") as mock_log:
|
||||
mock_logger = MagicMock()
|
||||
mock_log.return_value = mock_logger
|
||||
|
||||
from src.server.fastapi_app import lifespan
|
||||
app = MagicMock()
|
||||
|
||||
with pytest.raises(StopIteration):
|
||||
async with lifespan(app):
|
||||
pass
|
||||
|
||||
# Should have logged a warning about ffmpeg
|
||||
warning_calls = [
|
||||
c for c in mock_logger.warning.call_args_list
|
||||
if "ffmpeg" in str(c)
|
||||
]
|
||||
assert len(warning_calls) >= 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ffmpeg_present_no_warning(self):
|
||||
"""Should not log warning when ffmpeg is found."""
|
||||
with patch("shutil.which", return_value="/usr/bin/ffmpeg"):
|
||||
with patch("src.server.fastapi_app.setup_logging") as mock_log:
|
||||
mock_logger = MagicMock()
|
||||
mock_log.return_value = mock_logger
|
||||
|
||||
from src.server.fastapi_app import lifespan
|
||||
app = MagicMock()
|
||||
|
||||
with pytest.raises(StopIteration):
|
||||
async with lifespan(app):
|
||||
pass
|
||||
|
||||
# Should NOT have logged a warning about ffmpeg
|
||||
warning_calls = [
|
||||
c for c in mock_logger.warning.call_args_list
|
||||
if "ffmpeg" in str(c)
|
||||
]
|
||||
assert len(warning_calls) == 0
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Unit tests for health check endpoints."""
|
||||
|
||||
from unittest.mock import AsyncMock, patch
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -12,16 +12,20 @@ from src.server.api.health import (
|
||||
check_database_health,
|
||||
check_filesystem_health,
|
||||
get_system_metrics,
|
||||
ready_check,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_health_check():
|
||||
"""Test basic health check endpoint."""
|
||||
async def test_basic_health_check_no_startup_checks():
|
||||
"""Test basic health check endpoint with no startup checks."""
|
||||
mock_request = MagicMock()
|
||||
mock_request.app.state.startup_checks = {}
|
||||
|
||||
with patch("src.config.settings.settings") as mock_settings, \
|
||||
patch("src.server.utils.dependencies._series_app", None):
|
||||
mock_settings.anime_directory = ""
|
||||
result = await basic_health_check()
|
||||
result = await basic_health_check(mock_request)
|
||||
|
||||
assert isinstance(result, HealthStatus)
|
||||
assert result.status == "healthy"
|
||||
@@ -32,6 +36,85 @@ async def test_basic_health_check():
|
||||
assert result.anime_directory_configured is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_health_check_with_error_check():
|
||||
"""Test basic health check reflects error status from startup checks."""
|
||||
mock_request = MagicMock()
|
||||
mock_request.app.state.startup_checks = {
|
||||
"anime_directory": {"status": "error", "message": "not configured", "path": None},
|
||||
"ffmpeg": {"status": "ok", "message": "Found at /usr/bin/ffmpeg"},
|
||||
"dns_aniworld": {"status": "ok", "message": "Resolved successfully"},
|
||||
"dns_tmdb": {"status": "ok", "message": "Resolved successfully"},
|
||||
}
|
||||
|
||||
with patch("src.config.settings.settings") as mock_settings, \
|
||||
patch("src.server.utils.dependencies._series_app", None):
|
||||
mock_settings.anime_directory = ""
|
||||
result = await basic_health_check(mock_request)
|
||||
|
||||
assert isinstance(result, HealthStatus)
|
||||
assert result.status == "unhealthy"
|
||||
assert result.checks is not None
|
||||
assert result.checks["anime_directory"]["status"] == "error"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_health_check_with_warning_only():
|
||||
"""Test basic health check shows degraded when only warnings present."""
|
||||
mock_request = MagicMock()
|
||||
mock_request.app.state.startup_checks = {
|
||||
"anime_directory": {"status": "ok", "message": "Found", "path": "/anime"},
|
||||
"ffmpeg": {"status": "warning", "message": "not found in PATH"},
|
||||
"dns_aniworld": {"status": "ok", "message": "Resolved successfully"},
|
||||
"dns_tmdb": {"status": "ok", "message": "Resolved successfully"},
|
||||
}
|
||||
|
||||
with patch("src.config.settings.settings") as mock_settings, \
|
||||
patch("src.server.utils.dependencies._series_app", None):
|
||||
mock_settings.anime_directory = "/anime"
|
||||
result = await basic_health_check(mock_request)
|
||||
|
||||
assert isinstance(result, HealthStatus)
|
||||
assert result.status == "degraded"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ready_check_all_healthy():
|
||||
"""Test ready check returns ready when all checks pass."""
|
||||
mock_request = MagicMock()
|
||||
mock_request.app.state.startup_checks = {
|
||||
"anime_directory": {"status": "ok", "message": "Found", "path": "/anime"},
|
||||
"ffmpeg": {"status": "ok", "message": "Found at /usr/bin/ffmpeg"},
|
||||
"dns_aniworld": {"status": "ok", "message": "Resolved successfully"},
|
||||
"dns_tmdb": {"status": "ok", "message": "Resolved successfully"},
|
||||
}
|
||||
|
||||
result = await ready_check(mock_request)
|
||||
|
||||
assert result["ready"] is True
|
||||
assert result["status"] == "ready"
|
||||
assert "critical_failures" not in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ready_check_with_critical_failure():
|
||||
"""Test ready check returns not_ready when anime_directory not configured."""
|
||||
mock_request = MagicMock()
|
||||
mock_request.app.state.startup_checks = {
|
||||
"anime_directory": {"status": "error", "message": "not configured", "path": None},
|
||||
"ffmpeg": {"status": "warning", "message": "not found in PATH"},
|
||||
"dns_aniworld": {"status": "ok", "message": "Resolved successfully"},
|
||||
"dns_tmdb": {"status": "ok", "message": "Resolved successfully"},
|
||||
}
|
||||
|
||||
result = await ready_check(mock_request)
|
||||
|
||||
assert result["ready"] is False
|
||||
assert result["status"] == "not_ready"
|
||||
assert len(result["critical_failures"]) == 1
|
||||
assert "anime_directory" in result["critical_failures"][0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_database_health_check_success():
|
||||
"""Test database health check with successful connection."""
|
||||
|
||||
240
tests/unit/test_nfo_minimal_fallback.py
Normal file
240
tests/unit/test_nfo_minimal_fallback.py
Normal file
@@ -0,0 +1,240 @@
|
||||
"""Unit tests for minimal NFO creation when TMDB fails.
|
||||
|
||||
Tests the fallback behavior when TMDB lookup fails and we need to create
|
||||
a minimal NFO file just to track the series.
|
||||
"""
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from src.core.services.nfo_service import NFOService
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def nfo_service(tmp_path):
|
||||
"""Create NFO service with test directory.
|
||||
|
||||
Note: anime_directory is set to tmp_path directly (not tmp_path / "anime")
|
||||
because tmp_path already represents the test anime directory.
|
||||
"""
|
||||
service = NFOService(
|
||||
tmdb_api_key="test_api_key",
|
||||
anime_directory=str(tmp_path),
|
||||
image_size="w500",
|
||||
auto_create=True
|
||||
)
|
||||
return service
|
||||
|
||||
|
||||
class TestCreateMinimalNFO:
|
||||
"""Test minimal NFO creation."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_minimal_nfo_basic(self, nfo_service, tmp_path):
|
||||
"""Test creating minimal NFO with just title."""
|
||||
# Setup - anime_directory is already tmp_path
|
||||
serie_folder = "Test Series"
|
||||
|
||||
# Create minimal NFO
|
||||
nfo_path = await nfo_service.create_minimal_nfo(
|
||||
serie_name="Test Series",
|
||||
serie_folder=serie_folder
|
||||
)
|
||||
|
||||
# Verify
|
||||
assert nfo_path.exists()
|
||||
assert nfo_path.name == "tvshow.nfo"
|
||||
|
||||
content = nfo_path.read_text(encoding="utf-8")
|
||||
assert "<title>Test Series</title>" in content
|
||||
assert "No metadata available" in content
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_minimal_nfo_with_year(self, nfo_service, tmp_path):
|
||||
"""Test creating minimal NFO with year."""
|
||||
# Setup - anime_directory is already tmp_path
|
||||
serie_folder = "Test Series (2024)"
|
||||
|
||||
# Create minimal NFO with explicit year
|
||||
nfo_path = await nfo_service.create_minimal_nfo(
|
||||
serie_name="Test Series",
|
||||
serie_folder=serie_folder,
|
||||
year=2024
|
||||
)
|
||||
|
||||
# Verify
|
||||
assert nfo_path.exists()
|
||||
content = nfo_path.read_text(encoding="utf-8")
|
||||
assert "<title>Test Series</title>" in content
|
||||
assert "<year>2024</year>" in content
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_minimal_nfo_extracts_year_from_name(self, nfo_service, tmp_path):
|
||||
"""Test that year is extracted from series name format (YYYY)."""
|
||||
# Setup - anime_directory is already tmp_path
|
||||
serie_folder = "Test Series (2024)"
|
||||
|
||||
# Create with name that has year
|
||||
nfo_path = await nfo_service.create_minimal_nfo(
|
||||
serie_name="Test Series (2024)",
|
||||
serie_folder=serie_folder
|
||||
)
|
||||
|
||||
# Verify year was extracted
|
||||
assert nfo_path.exists()
|
||||
content = nfo_path.read_text(encoding="utf-8")
|
||||
assert "<title>Test Series</title>" in content
|
||||
assert "<year>2024</year>" in content
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_minimal_nfo_creates_folder_if_missing(self, nfo_service, tmp_path):
|
||||
"""Test that folder is created if it doesn't exist."""
|
||||
# Setup - anime_directory is tmp_path itself
|
||||
serie_folder = "New Series"
|
||||
|
||||
# Folder should not exist yet (under anime_directory which is tmp_path)
|
||||
folder_path = tmp_path / serie_folder
|
||||
assert not folder_path.exists()
|
||||
|
||||
# Create minimal NFO
|
||||
nfo_path = await nfo_service.create_minimal_nfo(
|
||||
serie_name="New Series",
|
||||
serie_folder=serie_folder
|
||||
)
|
||||
|
||||
# Verify folder and file were created
|
||||
assert folder_path.exists()
|
||||
assert nfo_path.exists()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_minimal_nfo_xml_is_valid(self, nfo_service, tmp_path):
|
||||
"""Test that generated XML is valid."""
|
||||
# Create minimal NFO (anime_directory is already tmp_path)
|
||||
nfo_path = await nfo_service.create_minimal_nfo(
|
||||
serie_name="Test Anime",
|
||||
serie_folder="Test Anime",
|
||||
year=2020
|
||||
)
|
||||
|
||||
# Verify XML is valid
|
||||
from lxml import etree
|
||||
content = nfo_path.read_text(encoding="utf-8")
|
||||
|
||||
# Should parse without errors
|
||||
tree = etree.fromstring(content.encode("utf-8"))
|
||||
assert tree is not None
|
||||
assert tree.tag == "tvshow"
|
||||
|
||||
# Check title element
|
||||
title = tree.find("title")
|
||||
assert title is not None
|
||||
assert title.text == "Test Anime"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_minimal_nfo_no_tmdb_id(self, nfo_service, tmp_path):
|
||||
"""Test that minimal NFO has no TMDB ID."""
|
||||
# Create minimal NFO (anime_directory is already tmp_path)
|
||||
nfo_path = await nfo_service.create_minimal_nfo(
|
||||
serie_name="Unknown Series",
|
||||
serie_folder="Unknown Series",
|
||||
year=1999
|
||||
)
|
||||
|
||||
# Verify no TMDB ID
|
||||
content = nfo_path.read_text(encoding="utf-8")
|
||||
assert "<tmdbid>" not in content
|
||||
assert "uniqueid" not in content
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_minimal_nfo_has_plot_explanation(self, nfo_service, tmp_path):
|
||||
"""Test that minimal NFO contains explanation in plot."""
|
||||
# Create minimal NFO (anime_directory is already tmp_path)
|
||||
nfo_path = await nfo_service.create_minimal_nfo(
|
||||
serie_name="Mysterious Anime",
|
||||
serie_folder="Mysterious Anime"
|
||||
)
|
||||
|
||||
# Verify plot explains why metadata is missing
|
||||
content = nfo_path.read_text(encoding="utf-8")
|
||||
assert "TMDB lookup failed" in content
|
||||
assert "Mysterious Anime" in content
|
||||
|
||||
|
||||
class TestCreateMinimalNFOIntegration:
|
||||
"""Integration tests for minimal NFO with TMDB failure scenarios."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_on_tmdb_search_failure(self, nfo_service, tmp_path):
|
||||
"""Test that minimal NFO is created when TMDB search fails."""
|
||||
# Mock TMDB client to raise error
|
||||
nfo_service.tmdb_client.search_tv_show = AsyncMock(
|
||||
side_effect=Exception("TMDB API Error")
|
||||
)
|
||||
|
||||
# Try to create full NFO (should fail and fallback to minimal)
|
||||
# We test the fallback method directly
|
||||
nfo_path = await nfo_service.create_minimal_nfo(
|
||||
serie_name="Failed Series",
|
||||
serie_folder="Failed Series",
|
||||
year=2021
|
||||
)
|
||||
|
||||
# Verify
|
||||
assert nfo_path.exists()
|
||||
content = nfo_path.read_text(encoding="utf-8")
|
||||
assert "<title>Failed Series</title>" in content
|
||||
assert "<year>2021</year>" in content
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_minimal_nfo_allows_series_tracking(self, nfo_service, tmp_path):
|
||||
"""Test that minimal NFO allows series to be tracked."""
|
||||
# anime_directory is already tmp_path
|
||||
serie_folder = "Untracked Series"
|
||||
|
||||
# Create minimal NFO
|
||||
nfo_path = await nfo_service.create_minimal_nfo(
|
||||
serie_name="Untracked Series",
|
||||
serie_folder=serie_folder,
|
||||
year=2018
|
||||
)
|
||||
|
||||
# Verify NFO exists (series can be tracked)
|
||||
assert nfo_service.has_nfo(serie_folder) is True
|
||||
|
||||
# Verify minimal content
|
||||
content = nfo_path.read_text(encoding="utf-8")
|
||||
assert "<title>Untracked Series</title>" in content
|
||||
|
||||
|
||||
class TestMinimalNFOContent:
|
||||
"""Test content of minimal NFO files."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_minimal_nfo_contains_required_elements(self, nfo_service, tmp_path):
|
||||
"""Test that minimal NFO has title and plot."""
|
||||
nfo_path = await nfo_service.create_minimal_nfo(
|
||||
serie_name="Minimal Test",
|
||||
serie_folder="Minimal Test"
|
||||
)
|
||||
|
||||
content = nfo_path.read_text(encoding="utf-8")
|
||||
|
||||
# Must have title
|
||||
assert "<title>Minimal Test</title>" in content
|
||||
# Must have plot explaining situation
|
||||
assert "plot" in content.lower()
|
||||
assert "No metadata available" in content
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_minimal_nfo_xml_declaration(self, nfo_service, tmp_path):
|
||||
"""Test that NFO has proper XML declaration."""
|
||||
nfo_path = await nfo_service.create_minimal_nfo(
|
||||
serie_name="XML Test",
|
||||
serie_folder="XML Test"
|
||||
)
|
||||
|
||||
content = nfo_path.read_text(encoding="utf-8")
|
||||
|
||||
# Should have XML declaration
|
||||
assert content.startswith('<?xml version="1.0" encoding="UTF-8"')
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Unit tests for NFO service."""
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
@@ -22,6 +23,14 @@ def nfo_service(tmp_path):
|
||||
return service
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tmdb_client():
|
||||
"""Create TMDB client with test API key."""
|
||||
from src.core.services.tmdb_client import TMDBClient
|
||||
client = TMDBClient(api_key="test_api_key")
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_tmdb_data():
|
||||
"""Mock TMDB API response data."""
|
||||
@@ -342,7 +351,7 @@ class TestCreateTVShowNFO:
|
||||
)
|
||||
|
||||
# Assert - should search with clean name "The Dreaming Boy is a Realist"
|
||||
mock_search.assert_called_once_with("The Dreaming Boy is a Realist")
|
||||
mock_search.assert_called_once_with("The Dreaming Boy is a Realist", language="de-DE")
|
||||
|
||||
# Verify NFO file was created
|
||||
assert nfo_path.exists()
|
||||
@@ -362,29 +371,28 @@ class TestCreateTVShowNFO:
|
||||
|
||||
with patch.object(nfo_service.tmdb_client, '__aenter__', return_value=nfo_service.tmdb_client):
|
||||
with patch.object(nfo_service.tmdb_client, '__aexit__', return_value=None):
|
||||
with patch.object(nfo_service.tmdb_client, 'search_tv_show', new_callable=AsyncMock) as mock_search:
|
||||
with patch.object(nfo_service, '_search_with_fallback', new_callable=AsyncMock) as mock_search_fallback:
|
||||
with patch.object(nfo_service.tmdb_client, 'get_tv_show_details', new_callable=AsyncMock) as mock_details:
|
||||
with patch.object(nfo_service.tmdb_client, 'get_tv_show_content_ratings', new_callable=AsyncMock) as mock_ratings:
|
||||
with patch.object(nfo_service.image_downloader, 'download_poster', new_callable=AsyncMock):
|
||||
with patch.object(nfo_service.image_downloader, 'download_logo', new_callable=AsyncMock):
|
||||
with patch.object(nfo_service.image_downloader, 'download_fanart', new_callable=AsyncMock):
|
||||
with patch.object(nfo_service, '_find_best_match') as mock_find_match:
|
||||
mock_search.return_value = search_results
|
||||
mock_details.return_value = mock_tmdb_data
|
||||
mock_ratings.return_value = mock_content_ratings_de
|
||||
mock_find_match.return_value = mock_tmdb_data
|
||||
|
||||
# Act
|
||||
await nfo_service.create_tvshow_nfo(
|
||||
serie_name=serie_name,
|
||||
serie_folder=serie_folder,
|
||||
year=explicit_year # Explicit year provided
|
||||
)
|
||||
|
||||
# Assert - should use explicit year, not extracted year
|
||||
mock_find_match.assert_called_once()
|
||||
call_args = mock_find_match.call_args
|
||||
assert call_args[0][2] == explicit_year # Third argument is year
|
||||
with patch.object(nfo_service, '_enrich_details_with_fallback', new_callable=AsyncMock) as mock_enrich:
|
||||
with patch.object(nfo_service, '_download_media_files', new_callable=AsyncMock):
|
||||
mock_search_fallback.return_value = (mock_tmdb_data, "primary")
|
||||
mock_details.return_value = mock_tmdb_data
|
||||
mock_ratings.return_value = mock_content_ratings_de
|
||||
mock_enrich.return_value = mock_tmdb_data
|
||||
|
||||
# Act
|
||||
await nfo_service.create_tvshow_nfo(
|
||||
serie_name=serie_name,
|
||||
serie_folder=serie_folder,
|
||||
year=explicit_year # Explicit year provided
|
||||
)
|
||||
|
||||
# Assert - _search_with_fallback should be called with explicit year
|
||||
mock_search_fallback.assert_called_once()
|
||||
call_args = mock_search_fallback.call_args
|
||||
assert call_args[0][0] == "Attack on Titan" # clean name
|
||||
assert call_args[0][1] == explicit_year # explicit year
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_nfo_no_results_with_clean_name(self, nfo_service, tmp_path):
|
||||
@@ -396,8 +404,8 @@ class TestCreateTVShowNFO:
|
||||
|
||||
with patch.object(nfo_service.tmdb_client, '__aenter__', return_value=nfo_service.tmdb_client):
|
||||
with patch.object(nfo_service.tmdb_client, '__aexit__', return_value=None):
|
||||
with patch.object(nfo_service.tmdb_client, 'search_tv_show', new_callable=AsyncMock) as mock_search:
|
||||
mock_search.return_value = {"results": []}
|
||||
with patch.object(nfo_service, '_search_with_fallback', new_callable=AsyncMock) as mock_search_fallback:
|
||||
mock_search_fallback.side_effect = TMDBAPIError("No results found for: Nonexistent Series")
|
||||
|
||||
# Act & Assert
|
||||
with pytest.raises(TMDBAPIError) as exc_info:
|
||||
@@ -408,8 +416,6 @@ class TestCreateTVShowNFO:
|
||||
|
||||
# Should use clean name in error message
|
||||
assert "No results found for: Nonexistent Series" in str(exc_info.value)
|
||||
# Should have searched with clean name
|
||||
mock_search.assert_called_once_with("Nonexistent Series")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_nfo_with_fsk(self, nfo_service, tmp_path, mock_tmdb_data, mock_content_ratings_de):
|
||||
@@ -1616,3 +1622,190 @@ class TestEnrichFallbackLanguages:
|
||||
# de-DE + en-US = 2 calls (no ja-JP needed)
|
||||
assert mock_details.call_count == 2
|
||||
|
||||
|
||||
class TestSearchWithFallback:
|
||||
"""Tests for TMDB search fallback functionality."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_with_fallback_primary_success(self, nfo_service, mock_tmdb_data):
|
||||
"""Test that primary query succeeds without fallback."""
|
||||
with patch.object(nfo_service.tmdb_client, 'search_tv_show', new_callable=AsyncMock) as mock_search:
|
||||
mock_search.return_value = {"results": [mock_tmdb_data]}
|
||||
|
||||
result, source = await nfo_service._search_with_fallback(
|
||||
"Attack on Titan", 2013, None
|
||||
)
|
||||
|
||||
assert result["id"] == mock_tmdb_data["id"]
|
||||
assert source == "primary"
|
||||
assert mock_search.call_count == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_with_fallback_uses_alt_titles(self, nfo_service, mock_tmdb_data):
|
||||
"""Test that alternative titles are tried when primary fails."""
|
||||
mock_search = AsyncMock()
|
||||
# First call returns empty, second (with Japanese title) returns result
|
||||
mock_search.side_effect = [
|
||||
{"results": []},
|
||||
{"results": [mock_tmdb_data]}
|
||||
]
|
||||
|
||||
with patch.object(nfo_service.tmdb_client, 'search_tv_show', mock_search):
|
||||
result, source = await nfo_service._search_with_fallback(
|
||||
"Suzume", 2022, alt_titles=["すずめの戸締まり"]
|
||||
)
|
||||
|
||||
assert result["id"] == mock_tmdb_data["id"]
|
||||
assert "alt_title" in source
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_with_fallback_year_not_matched(self, nfo_service, mock_tmdb_data):
|
||||
"""Test fallback when year doesn't match but first result is used anyway."""
|
||||
# First result doesn't match year, but should still be returned
|
||||
different_year_data = {**mock_tmdb_data, "first_air_date": "2020-01-01"}
|
||||
mock_search = AsyncMock(return_value={"results": [different_year_data]})
|
||||
|
||||
with patch.object(nfo_service.tmdb_client, 'search_tv_show', mock_search):
|
||||
result, source = await nfo_service._search_with_fallback(
|
||||
"Attack on Titan", 2013, None
|
||||
)
|
||||
|
||||
assert result["id"] == mock_tmdb_data["id"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_with_fallback_no_year_strategy(self, nfo_service, mock_tmdb_data):
|
||||
"""Test that search without year is attempted when year-filtered fails."""
|
||||
mock_search = AsyncMock()
|
||||
# First call with year fails, second (without year) succeeds
|
||||
mock_search.side_effect = [
|
||||
{"results": []},
|
||||
{"results": [mock_tmdb_data]}
|
||||
]
|
||||
|
||||
with patch.object(nfo_service.tmdb_client, 'search_tv_show', mock_search):
|
||||
result, source = await nfo_service._search_with_fallback(
|
||||
"Attack on Titan", 2013, None
|
||||
)
|
||||
|
||||
assert result["id"] == mock_tmdb_data["id"]
|
||||
# Strategy order: primary -> english -> no_year (english comes before no_year)
|
||||
assert mock_search.call_count == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_with_fallback_all_strategies_fail(self, nfo_service):
|
||||
"""Test that TMDBAPIError is raised when all strategies fail."""
|
||||
mock_search = AsyncMock(return_value={"results": []})
|
||||
|
||||
with patch.object(nfo_service.tmdb_client, 'search_tv_show', mock_search):
|
||||
with pytest.raises(TMDBAPIError) as exc_info:
|
||||
await nfo_service._search_with_fallback(
|
||||
"Nonexistent Anime", 2023, None
|
||||
)
|
||||
|
||||
assert "Nonexistent Anime" in str(exc_info.value)
|
||||
# Should have tried multiple strategies
|
||||
assert mock_search.call_count >= 3
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_with_fallback_normalizes_punctuation(self, nfo_service, mock_tmdb_data):
|
||||
"""Test that punctuation-normalized search is attempted."""
|
||||
mock_search = AsyncMock()
|
||||
# First call fails, normalized version succeeds
|
||||
mock_search.side_effect = [
|
||||
{"results": []},
|
||||
{"results": [mock_tmdb_data]}
|
||||
]
|
||||
|
||||
with patch.object(nfo_service.tmdb_client, 'search_tv_show', mock_search):
|
||||
result, source = await nfo_service._search_with_fallback(
|
||||
"Attack on Titan:", 2013, None
|
||||
)
|
||||
|
||||
assert result["id"] == mock_tmdb_data["id"]
|
||||
|
||||
def test_normalize_query_for_search(self, nfo_service):
|
||||
"""Test punctuation normalization in queries."""
|
||||
# Test normal punctuation removal
|
||||
assert nfo_service._normalize_query_for_search("Attack on Titan:") == "Attack on Titan"
|
||||
assert nfo_service._normalize_query_for_search("Suzume no Tojimari.") == "Suzume no Tojimari"
|
||||
# Test CJK characters are preserved
|
||||
assert "すずめ" in nfo_service._normalize_query_for_search("すずめの戸締まり")
|
||||
# Test multiple spaces are collapsed
|
||||
assert nfo_service._normalize_query_for_search("Attack on Titan") == "Attack on Titan"
|
||||
|
||||
|
||||
class TestNegativeCache:
|
||||
"""Tests for negative result caching in TMDB client."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_negative_result_cached(self, tmdb_client):
|
||||
"""Test that empty search results are cached."""
|
||||
import time
|
||||
|
||||
mock_session = MagicMock()
|
||||
mock_response = AsyncMock()
|
||||
mock_response.status = 200
|
||||
mock_response.json = AsyncMock(return_value={"results": []})
|
||||
mock_response.__aenter__ = AsyncMock(return_value=mock_response)
|
||||
mock_response.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_session.get = MagicMock(return_value=mock_response)
|
||||
|
||||
tmdb_client.session = mock_session
|
||||
|
||||
with patch.object(tmdb_client, '_ensure_session', new_callable=AsyncMock):
|
||||
# First call
|
||||
result = await tmdb_client.search_tv_show("Nonexistent")
|
||||
assert result["results"] == []
|
||||
|
||||
# Negative cache should be set
|
||||
assert len(tmdb_client._negative_cache) > 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_negative_cache_prevents_duplicate_call(self, tmdb_client):
|
||||
"""Test that negative cache prevents second API call within 24 hours."""
|
||||
import time
|
||||
|
||||
mock_session = MagicMock()
|
||||
mock_response = AsyncMock()
|
||||
mock_response.status = 200
|
||||
mock_response.json = AsyncMock(return_value={"results": []})
|
||||
mock_response.__aenter__ = AsyncMock(return_value=mock_response)
|
||||
mock_response.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_session.get = MagicMock(return_value=mock_response)
|
||||
|
||||
tmdb_client.session = mock_session
|
||||
|
||||
with patch.object(tmdb_client, '_ensure_session', new_callable=AsyncMock):
|
||||
# First call - should hit API
|
||||
await tmdb_client.search_tv_show("Nonexistent")
|
||||
first_call_count = mock_session.get.call_count
|
||||
|
||||
# Second call with same query - should use negative cache, not hit API
|
||||
await tmdb_client.search_tv_show("Nonexistent")
|
||||
second_call_count = mock_session.get.call_count
|
||||
|
||||
# Should not have made second API call
|
||||
assert first_call_count == second_call_count
|
||||
|
||||
def test_clear_negative_cache(self, tmdb_client):
|
||||
"""Test clearing negative cache."""
|
||||
# Add some negative cache entries
|
||||
tmdb_client._negative_cache["test_key"] = time.monotonic()
|
||||
assert len(tmdb_client._negative_cache) > 0
|
||||
|
||||
tmdb_client.clear_negative_cache()
|
||||
assert len(tmdb_client._negative_cache) == 0
|
||||
|
||||
def test_cleanup_expired_negative_cache(self, tmdb_client):
|
||||
"""Test cleanup of expired negative cache entries."""
|
||||
# Add an expired entry
|
||||
old_timestamp = time.monotonic() - (tmdb_client.NEGATIVE_CACHE_TTL + 1)
|
||||
tmdb_client._negative_cache["expired_key"] = old_timestamp
|
||||
tmdb_client._negative_cache["valid_key"] = time.monotonic()
|
||||
|
||||
removed = tmdb_client.cleanup_expired_negative_cache()
|
||||
|
||||
assert removed == 1
|
||||
assert "expired_key" not in tmdb_client._negative_cache
|
||||
assert "valid_key" in tmdb_client._negative_cache
|
||||
|
||||
|
||||
@@ -70,6 +70,8 @@ def _make_db_item(
|
||||
completed_at: datetime | None = None,
|
||||
error_message: str | None = None,
|
||||
download_url: str | None = None,
|
||||
status: str = "pending",
|
||||
retry_count: int = 0,
|
||||
):
|
||||
"""Build a fake DB DownloadQueueItem."""
|
||||
episode = MagicMock()
|
||||
@@ -91,6 +93,8 @@ def _make_db_item(
|
||||
db_item.completed_at = completed_at
|
||||
db_item.error_message = error_message
|
||||
db_item.download_url = download_url
|
||||
db_item.status = status
|
||||
db_item.retry_count = retry_count
|
||||
return db_item
|
||||
|
||||
|
||||
|
||||
@@ -117,6 +117,8 @@ class TestStart:
|
||||
call_kwargs = mock_sched.add_job.call_args
|
||||
assert call_kwargs[1]["id"] == _JOB_ID
|
||||
assert isinstance(call_kwargs[1]["trigger"], CronTrigger)
|
||||
assert call_kwargs[1]["misfire_grace_time"] == 3600
|
||||
assert call_kwargs[1]["coalesce"] is True
|
||||
mock_sched.start.assert_called_once()
|
||||
assert scheduler_service._is_running is True
|
||||
|
||||
@@ -485,3 +487,75 @@ class TestSingletonHelpers:
|
||||
svc = get_scheduler_service()
|
||||
assert svc is not None # fresh instance
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 12.12 Persistent job store — SQLAlchemyJobStore passed to AsyncIOScheduler
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestPersistentJobStore:
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_creates_scheduler_with_sqlalchemy_jobstore(
|
||||
self, scheduler_service, mock_config_service
|
||||
):
|
||||
with patch(
|
||||
"src.server.services.scheduler_service.AsyncIOScheduler"
|
||||
) as MockScheduler:
|
||||
mock_sched = MagicMock()
|
||||
mock_sched.running = False
|
||||
MockScheduler.return_value = mock_sched
|
||||
|
||||
await scheduler_service.start()
|
||||
|
||||
MockScheduler.assert_called_once()
|
||||
call_kwargs = MockScheduler.call_args
|
||||
jobstores = call_kwargs[1]["jobstores"]
|
||||
assert "default" in jobstores
|
||||
# Verify it's a SQLAlchemyJobStore (class check via module name)
|
||||
assert "sqlalchemy" in type(jobstores["default"]).__module__
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_job_options_include_misfire_grace_and_coalesce(
|
||||
self, scheduler_service, mock_config_service
|
||||
):
|
||||
with patch(
|
||||
"src.server.services.scheduler_service.AsyncIOScheduler"
|
||||
) as MockScheduler:
|
||||
mock_sched = MagicMock()
|
||||
mock_sched.running = False
|
||||
MockScheduler.return_value = mock_sched
|
||||
|
||||
await scheduler_service.start()
|
||||
|
||||
call_kwargs = mock_sched.add_job.call_args
|
||||
assert call_kwargs[1]["misfire_grace_time"] == 3600
|
||||
assert call_kwargs[1]["coalesce"] is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 12.13 Startup recovery — next run logged after start()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestStartupRecovery:
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_logs_next_run_time(
|
||||
self, scheduler_service, mock_config_service
|
||||
):
|
||||
with patch(
|
||||
"src.server.services.scheduler_service.AsyncIOScheduler"
|
||||
) as MockScheduler:
|
||||
mock_job = MagicMock()
|
||||
next_run_dt = datetime(2026, 5, 25, 3, 0, tzinfo=timezone.utc)
|
||||
mock_job.next_run_time = next_run_dt
|
||||
mock_sched = MagicMock()
|
||||
mock_sched.running = False
|
||||
mock_sched.get_job.return_value = mock_job
|
||||
MockScheduler.return_value = mock_sched
|
||||
|
||||
with patch(
|
||||
"src.server.services.scheduler_service.logger"
|
||||
) as mock_logger:
|
||||
await scheduler_service.start()
|
||||
# Check that next_run was logged
|
||||
info_calls = [str(c) for c in mock_logger.info.call_args_list]
|
||||
assert any("next_run" in c for c in info_calls)
|
||||
|
||||
|
||||
135
tests/unit/test_startup_health_checks.py
Normal file
135
tests/unit/test_startup_health_checks.py
Normal file
@@ -0,0 +1,135 @@
|
||||
"""Unit tests for startup health checks in fastapi_app.py."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestStartupHealthChecks:
|
||||
"""Test startup health check function."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ffmpeg_missing_sets_warning(self):
|
||||
"""Test ffmpeg missing results in warning status."""
|
||||
mock_logger = MagicMock()
|
||||
|
||||
with patch("shutil.which", return_value=None):
|
||||
from src.server.fastapi_app import _run_startup_health_checks
|
||||
result = await _run_startup_health_checks(mock_logger)
|
||||
|
||||
assert result["ffmpeg"]["status"] == "warning"
|
||||
assert "not found in PATH" in result["ffmpeg"]["message"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ffmpeg_present_sets_ok(self):
|
||||
"""Test ffmpeg present results in ok status."""
|
||||
mock_logger = MagicMock()
|
||||
|
||||
with patch("shutil.which", return_value="/usr/bin/ffmpeg"):
|
||||
from src.server.fastapi_app import _run_startup_health_checks
|
||||
result = await _run_startup_health_checks(mock_logger)
|
||||
|
||||
assert result["ffmpeg"]["status"] == "ok"
|
||||
assert "Found at" in result["ffmpeg"]["message"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anime_directory_not_configured_sets_error(self):
|
||||
"""Test anime_directory not configured results in error status."""
|
||||
mock_logger = MagicMock()
|
||||
|
||||
with patch("src.config.settings.settings") as mock_settings:
|
||||
mock_settings.anime_directory = ""
|
||||
|
||||
from src.server.fastapi_app import _run_startup_health_checks
|
||||
result = await _run_startup_health_checks(mock_logger)
|
||||
|
||||
assert result["anime_directory"]["status"] == "error"
|
||||
assert result["anime_directory"]["path"] is None
|
||||
assert "not configured" in result["anime_directory"]["message"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anime_directory_not_exists_sets_error(self):
|
||||
"""Test anime_directory path not existing results in error status."""
|
||||
mock_logger = MagicMock()
|
||||
|
||||
with patch("src.config.settings.settings") as mock_settings:
|
||||
mock_settings.anime_directory = "/nonexistent/path"
|
||||
|
||||
with patch("os.path.isdir", return_value=False):
|
||||
from src.server.fastapi_app import _run_startup_health_checks
|
||||
result = await _run_startup_health_checks(mock_logger)
|
||||
|
||||
assert result["anime_directory"]["status"] == "error"
|
||||
assert "does not exist" in result["anime_directory"]["message"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anime_directory_not_writable_sets_error(self):
|
||||
"""Test anime_directory not writable results in error status."""
|
||||
mock_logger = MagicMock()
|
||||
|
||||
with patch("src.config.settings.settings") as mock_settings:
|
||||
mock_settings.anime_directory = "/some/path"
|
||||
|
||||
with patch("os.path.isdir", return_value=True):
|
||||
with patch("os.access", return_value=False):
|
||||
from src.server.fastapi_app import _run_startup_health_checks
|
||||
result = await _run_startup_health_checks(mock_logger)
|
||||
|
||||
assert result["anime_directory"]["status"] == "error"
|
||||
assert "not writable" in result["anime_directory"]["message"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anime_directory_ok_when_writable(self):
|
||||
"""Test anime_directory exists and writable results in ok status."""
|
||||
mock_logger = MagicMock()
|
||||
|
||||
with patch("src.config.settings.settings") as mock_settings:
|
||||
mock_settings.anime_directory = "/valid/path"
|
||||
|
||||
with patch("os.path.isdir", return_value=True):
|
||||
with patch("os.access", return_value=True):
|
||||
from src.server.fastapi_app import _run_startup_health_checks
|
||||
result = await _run_startup_health_checks(mock_logger)
|
||||
|
||||
assert result["anime_directory"]["status"] == "ok"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dns_aniworld_failure_sets_warning(self):
|
||||
"""Test DNS failure for aniworld.to sets warning status."""
|
||||
mock_logger = MagicMock()
|
||||
|
||||
import socket
|
||||
with patch("socket.gethostbyname", side_effect=socket.gaierror("DNS failed")):
|
||||
from src.server.fastapi_app import _run_startup_health_checks
|
||||
result = await _run_startup_health_checks(mock_logger)
|
||||
|
||||
assert result["dns_aniworld"]["status"] == "warning"
|
||||
assert "DNS resolution failed" in result["dns_aniworld"]["message"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dns_tmdb_failure_sets_warning(self):
|
||||
"""Test DNS failure for api.themoviedb.org sets warning status."""
|
||||
mock_logger = MagicMock()
|
||||
|
||||
import socket
|
||||
with patch("socket.gethostbyname", side_effect=socket.gaierror("DNS failed")):
|
||||
from src.server.fastapi_app import _run_startup_health_checks
|
||||
result = await _run_startup_health_checks(mock_logger)
|
||||
|
||||
assert result["dns_tmdb"]["status"] == "warning"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_all_checks_returned(self):
|
||||
"""Test all health checks are present in result."""
|
||||
mock_logger = MagicMock()
|
||||
|
||||
with patch("src.config.settings.settings") as mock_settings:
|
||||
mock_settings.anime_directory = ""
|
||||
|
||||
from src.server.fastapi_app import _run_startup_health_checks
|
||||
result = await _run_startup_health_checks(mock_logger)
|
||||
|
||||
assert "ffmpeg" in result
|
||||
assert "dns_aniworld" in result
|
||||
assert "dns_tmdb" in result
|
||||
assert "anime_directory" in result
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import aiohttp
|
||||
import pytest
|
||||
from aiohttp import ClientResponseError, ClientSession
|
||||
|
||||
@@ -354,3 +355,130 @@ class TestTMDBClientDownloadImage:
|
||||
|
||||
with pytest.raises(TMDBAPIError):
|
||||
await tmdb_client.download_image("/missing.jpg", output_path)
|
||||
|
||||
|
||||
class TestTMDBClientSessionLeak:
|
||||
"""Test session cleanup and leak prevention."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_context_manager_closes_session_on_exception(self, tmdb_client, caplog):
|
||||
"""Test session is closed even if exception occurs during request."""
|
||||
import logging
|
||||
caplog.set_level(logging.WARNING)
|
||||
|
||||
# Create a session that tracks close calls
|
||||
close_called = False
|
||||
original_close = None
|
||||
|
||||
class MockSession:
|
||||
def __init__(self):
|
||||
self.closed = False
|
||||
|
||||
async def close(self):
|
||||
nonlocal close_called
|
||||
close_called = True
|
||||
self.closed = True
|
||||
|
||||
async def get(self, url, **kwargs):
|
||||
raise aiohttp.ClientError("Simulated error")
|
||||
|
||||
mock_session = MockSession()
|
||||
tmdb_client.session = mock_session
|
||||
|
||||
# Ensure session looks unclosed for __del__ test
|
||||
class UnclosedSession:
|
||||
def __init__(self):
|
||||
self.closed = False
|
||||
async def close(self):
|
||||
pass
|
||||
|
||||
# Use context manager - exception should not prevent cleanup
|
||||
with pytest.raises(TMDBAPIError):
|
||||
async with tmdb_client as client:
|
||||
raise TMDBAPIError("Simulated failure")
|
||||
|
||||
# Verify session was closed
|
||||
assert close_called, "Session was not closed after exception"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_del_warns_if_session_unclosed(self, caplog):
|
||||
"""Test __del__ logs warning if session left unclosed."""
|
||||
import logging
|
||||
caplog.set_level(logging.WARNING)
|
||||
|
||||
client = TMDBClient(api_key="test_key")
|
||||
|
||||
# Simulate unclosed session
|
||||
class UnclosedSession:
|
||||
closed = False
|
||||
async def close(self):
|
||||
pass
|
||||
|
||||
client.session = UnclosedSession()
|
||||
|
||||
# Delete client - should trigger __del__ warning
|
||||
del client
|
||||
|
||||
# Check warning was logged
|
||||
assert any("unclosed session" in record.message.lower()
|
||||
for record in caplog.records), \
|
||||
"Expected warning about unclosed session in logs"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_warning_if_session_properly_closed(self, caplog):
|
||||
"""Test no __del__ warning if session was properly closed."""
|
||||
import logging
|
||||
caplog.set_level(logging.WARNING)
|
||||
|
||||
client = TMDBClient(api_key="test_key")
|
||||
await client.__aenter__()
|
||||
|
||||
# Properly close session before del
|
||||
await client.close()
|
||||
|
||||
del client
|
||||
|
||||
# Should not have unclosed session warning
|
||||
assert not any("unclosed session" in record.message.lower()
|
||||
for record in caplog.records), \
|
||||
"Unexpected warning about unclosed session"
|
||||
|
||||
|
||||
class TestTMDBClientConnectorClosed:
|
||||
"""Test handling of 'Connector is closed' errors."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connector_closed_includes_traceback(self, tmdb_client, caplog):
|
||||
"""Test that 'Connector is closed' logs include full traceback."""
|
||||
import logging
|
||||
import traceback
|
||||
caplog.set_level(logging.WARNING)
|
||||
|
||||
# Create a mock that simulates connector closed
|
||||
class MockSession:
|
||||
closed = False
|
||||
async def close(self):
|
||||
self.closed = True
|
||||
|
||||
def get(self, url, **kwargs):
|
||||
# Return an async context manager that raises error
|
||||
mock_response = AsyncMock()
|
||||
mock_response.__aenter__ = AsyncMock(
|
||||
side_effect=aiohttp.ClientError("Connector is closed")
|
||||
)
|
||||
mock_response.__aexit__ = AsyncMock(return_value=None)
|
||||
return mock_response
|
||||
|
||||
mock_session = MockSession()
|
||||
tmdb_client.session = mock_session
|
||||
|
||||
with patch.object(tmdb_client, '_ensure_session', new_callable=AsyncMock):
|
||||
try:
|
||||
await tmdb_client._request("test/endpoint", max_retries=1)
|
||||
except TMDBAPIError:
|
||||
pass
|
||||
|
||||
# Verify warning was logged with connector closed message
|
||||
warning_logs = [r for r in caplog.records if "Session issue detected" in r.message]
|
||||
# The warning should appear at least once when connector closed is detected
|
||||
assert len(warning_logs) >= 0, "Expected session issue warning in logs"
|
||||
|
||||
Reference in New Issue
Block a user