Compare commits
411 Commits
ccbd9768a2
...
v1.4.11
| Author | SHA1 | Date | |
|---|---|---|---|
| fd3ec5df83 | |||
| 275aeb4544 | |||
| be7b210959 | |||
| 486c5440f2 | |||
| 4076b9dd43 | |||
| df93e8a81f | |||
| 576d9f7a7b | |||
| af93daeddc | |||
| a05795bb35 | |||
| d22df947e4 | |||
| 8bb8c6aa64 | |||
| 109d3c8ac9 | |||
| 6a934db8ac | |||
| ac7302b1dd | |||
| ac5ee3bb27 | |||
| a9084202e3 | |||
| be9f2a4c0c | |||
| 53fe09351f | |||
| dc7d9ee5f7 | |||
| da3cae2812 | |||
| 2876cef24b | |||
| 6a402623c4 | |||
| ebfbec1225 | |||
| 01e4dec8d7 | |||
| ecef21eec4 | |||
| d9738ffb78 | |||
| 6aec2a1733 | |||
| 84487d7571 | |||
| e02d65778f | |||
| 45d259bab2 | |||
| 7b8de8d988 | |||
| 18d10b44b5 | |||
| 5c2be3f7c4 | |||
| 2c47713339 | |||
| e74b04c1ee | |||
| 8b21f1243f | |||
| 3d33626546 | |||
| 7d9f80a0c6 | |||
| 25dc66fec3 | |||
| 2be7b692b9 | |||
| 2b5c969a83 | |||
| 830f6b4c93 | |||
| 5526ab884a | |||
| 09d454d4c0 | |||
| 13504c3172 | |||
| 82493d41ea | |||
| 274f773988 | |||
| 21af502184 | |||
| 97caaf0d18 | |||
| dc5d6506bc | |||
| dbaf80e941 | |||
| 4fc597c5de | |||
| a77bb371df | |||
| 420d10bb34 | |||
| e29918488c | |||
| 9c3f03d610 | |||
| 9d64241230 | |||
| 49cd84f3e5 | |||
| e46759347e | |||
| 75f743e6cc | |||
| 4dc5ffa19e | |||
| 1649a22418 | |||
| 246752e2fc | |||
| 84b24ed79e | |||
| bf3954587a | |||
| ed8f5cae10 | |||
| a54c285994 | |||
| c58b42dfa5 | |||
| 6dfb24de7e | |||
| 6021cdef28 | |||
| 5517ccbab0 | |||
| 94ed013172 | |||
| 76b849fc91 | |||
| 00b26c8cbc | |||
| a6f2399aca | |||
| cf001563b3 | |||
| 38c12638a4 | |||
| 765e43c684 | |||
| 5190d32665 | |||
| 4e6afa31b5 | |||
| 1ef59c5283 | |||
| 239341629c | |||
| 51b7f349f8 | |||
| 14b8ef7f06 | |||
| 7abba0dae2 | |||
| 30858f441c | |||
| 33f63ca304 | |||
| fe9284b80e | |||
| 12e5526991 | |||
| bc87bee416 | |||
| 7ded5a6e4d | |||
| d596902ca3 | |||
| d358a07290 | |||
| b9c55f9e7a | |||
| fc4e52f1a2 | |||
| 6d30747f25 | |||
| ceb6a2aeb4 | |||
| 53d6da5dac | |||
| 102d83e947 | |||
| 841368bf85 | |||
| cbd53ef2a0 | |||
| 50a77976d5 | |||
| dfc28b8e66 | |||
| 6c9605e896 | |||
| 3947f6d266 | |||
| a3176f5ac1 | |||
| 9a81b04b65 | |||
| a336733ea9 | |||
| ca93bb740a | |||
| d5e955a731 | |||
| e2a373816a | |||
| a115215416 | |||
| c579235af0 | |||
| 0ba2587bc8 | |||
| bda1fe4445 | |||
| 810346bc8b | |||
| daa937bcb7 | |||
| 1c505bd722 | |||
| 3551838887 | |||
| 9a20541598 | |||
| 3f7651404d | |||
| bee24406e6 | |||
| 31eb0026cf | |||
| 24ea12bbaf | |||
| e74b602f60 | |||
| db65e28854 | |||
| 11e231a4ab | |||
| a11f8c4fa0 | |||
| cf5a06af11 | |||
| e07f75432e | |||
| 1696d5c65b | |||
| c8b386f47a | |||
| 3888da352a | |||
| 06e104db42 | |||
| d4594bd1d9 | |||
| d866e836f6 | |||
| 195dae13cb | |||
| 51be777e7d | |||
| 7930e49701 | |||
| 75c22fe296 | |||
| 7bcd0600d5 | |||
| a333329ae2 | |||
| 363f7899f8 | |||
| a08a8f7408 | |||
| 54ac5e9ab7 | |||
| c93ac3e7b8 | |||
| 68c4335348 | |||
| be87f2e230 | |||
| c56e0f507d | |||
| cb0a36ccc2 | |||
| 3644b16447 | |||
| d5116e378e | |||
| 50a7083ce5 | |||
| 52c0ff2337 | |||
| a5fd88e224 | |||
| 98d4edad14 | |||
| bc8059b453 | |||
| 815a4f1520 | |||
| e3509f5c8f | |||
| 69c2fd01f9 | |||
| 0f36afd88c | |||
| ceac22fc34 | |||
| 9c0f7ce08d | |||
| 756731cd5d | |||
| eb0e6e8ccb | |||
| eb2fc3c5ab | |||
| c39ae9d0fc | |||
| 079f1f99e3 | |||
| 9373f500d3 | |||
| 2274403899 | |||
| 6ad14c03b5 | |||
| b10cce0489 | |||
| 2aa184c870 | |||
| 92bd55ada1 | |||
| e5fae0a0a2 | |||
| 151a08e033 | |||
| e44a8190d0 | |||
| 94720f2d61 | |||
| 0ec120e08f | |||
| db58ea9396 | |||
| 69b409f42d | |||
| b34ee59bca | |||
| 624c0db16e | |||
| e6d9f9f342 | |||
| fc8cdc538d | |||
| d8248be67d | |||
| 6c7dc66c5d | |||
| d951963d87 | |||
| f6000b1fff | |||
| ddf10327c7 | |||
| 747e1acc21 | |||
| 1885fed4bd | |||
| dd45494717 | |||
| 4ac51a789a | |||
| 1712dfd776 | |||
| ddcac5a96d | |||
| c186e0d4f7 | |||
| 759cd09ded | |||
| bbf0a0815a | |||
| 87bf0d71cd | |||
| 8e262c947c | |||
| adea1e2ede | |||
| d71feb64dd | |||
| 3e5ad8a4a6 | |||
| e1abf90c81 | |||
| 228964e928 | |||
| dee2601bda | |||
| 61f35632b9 | |||
| eed75ff08b | |||
| 0265ae2a70 | |||
| ac7e15e1eb | |||
| 850207d9a8 | |||
| 1c39dd5c6a | |||
| 76f02ec822 | |||
| e84a220f55 | |||
| d7ab689fe1 | |||
| 0d2ce07ad7 | |||
| e4d328bb45 | |||
| f283e581d6 | |||
| 88043ed749 | |||
| 7effc02f33 | |||
| 60e5b5ccda | |||
| 88f3219126 | |||
| c6da967893 | |||
| 9275747b6d | |||
| 5b3fbf36b9 | |||
| 46dab1dbc1 | |||
| d1d30dde9e | |||
| 4b35cb63d1 | |||
| af208882f5 | |||
| cf754860f1 | |||
| 53b628efd9 | |||
| 06fb6630ea | |||
| d72b8cb1ab | |||
| d74c181556 | |||
| c757123429 | |||
| 436dc8b338 | |||
| f8122099c3 | |||
| 8174cf73c4 | |||
| 6208cae5c7 | |||
| 708bf42f89 | |||
| 27c6087d88 | |||
| 9157c4b274 | |||
| 700415af57 | |||
| 7f21d3236f | |||
| 253750ad45 | |||
| b1d9714123 | |||
| 562fcdc811 | |||
| 212b971bba | |||
| 08123d40e4 | |||
| 30ff7c7a93 | |||
| bd5538be59 | |||
| a92340aa8b | |||
| 9ab96398b0 | |||
| aceaba5849 | |||
| a345f9b4e9 | |||
| e3de8a4c9a | |||
| aa601daf88 | |||
| 7100b3c968 | |||
| ab40cdcf2c | |||
| 26532ea592 | |||
| 1f551a3fbe | |||
| eb0f6cdc85 | |||
| 63da2daa53 | |||
| 0ab9adbd04 | |||
| 1a4fce16d6 | |||
| c693c6572b | |||
| f409b81aa2 | |||
| f5a42f269e | |||
| cf4e698454 | |||
| 58fb9fdd3e | |||
| dc6c113707 | |||
| 3b1ab36786 | |||
| cc6f190cb6 | |||
| 954d571a80 | |||
| 7693828621 | |||
| 10246df78b | |||
| 846176f114 | |||
| 732181b709 | |||
| 6854d72d56 | |||
| ab1836575e | |||
| 0ffcfac674 | |||
| 797bba4151 | |||
| 458fc483e4 | |||
| 3f2e15669d | |||
| 7c1242a122 | |||
| fb8f0bdbd2 | |||
| 52d82ab6bc | |||
| 8647da8474 | |||
| 46271a9845 | |||
| 4abaf8def7 | |||
| c4080e4e57 | |||
| ed3882991f | |||
| 35a7aeac9e | |||
| b89da0d7a0 | |||
| 14dce41de8 | |||
| 6d0259d4b4 | |||
| f7cc296aa7 | |||
| 8ff558cb07 | |||
| 04f26d5cfc | |||
| 5af72c33b8 | |||
| c7bf232fe1 | |||
| 2b904fd01e | |||
| e09bb0451c | |||
| 800790fc8f | |||
| 0e58a49cdd | |||
| fed6162452 | |||
| 611798b786 | |||
| 314f535446 | |||
| a8011eb6a3 | |||
| ba6429bb2f | |||
| 168b4c5ac4 | |||
| 925f408699 | |||
| 9fb93794e6 | |||
| faac14346f | |||
| f8634bf605 | |||
| 7bf02ac8f8 | |||
| 026e96b66c | |||
| c586e9f69d | |||
| f89649fe20 | |||
| 33406fef1a | |||
| 5e233bcba0 | |||
| 48a2fd0f2a | |||
| 77ffdac84b | |||
| 92c8d42c4d | |||
| ae162d9a6d | |||
| 4c606faa0e | |||
| 50e0b21669 | |||
| 8e8487b7b7 | |||
| 61c86dc698 | |||
| 88c00b761c | |||
| 125892abe5 | |||
| 050db40af3 | |||
| 9f1158b9af | |||
| db7e21a14c | |||
| bf3cfa00d5 | |||
| 35c82e68b7 | |||
| b2379e05cf | |||
| f9e4970615 | |||
| 5aba36c40a | |||
| d425d711bd | |||
| 6215477eef | |||
| 0b580f2fab | |||
| bfbae88ade | |||
| 01f828c799 | |||
| 6d40ddbfe5 | |||
| d6a82f4329 | |||
| 7d95c180a9 | |||
| 62bdcf35cb | |||
| c97da7db2e | |||
| 09a5eccea7 | |||
| 265d7fe435 | |||
| 0bbdd46fc7 | |||
| 8b0a4abca9 | |||
| 5ca6a27573 | |||
| 9d5bd12ec8 | |||
| 0b4fb10d65 | |||
| f18c31a035 | |||
| df19f8ad95 | |||
| 2495b07fc4 | |||
| ea9e959a7b | |||
| 7a77dff194 | |||
| 1b4526d050 | |||
| 491daa2e50 | |||
| 03901a8c2d | |||
| c92e2d340e | |||
| e502dcb8bd | |||
| 4e56093ff9 | |||
| 9877f9400c | |||
| db1e7fa54b | |||
| 4408874d37 | |||
| 390cafc0dc | |||
| a06abaa2e5 | |||
| c6919ac124 | |||
| 22a41ba93f | |||
| fd5e85d5ea | |||
| 4e29c4ed80 | |||
| d676cb7dca | |||
| d1a966cc0d | |||
| c88e2d2b7b | |||
| 2f04b2a862 | |||
| 120b26b9f7 | |||
| ecfa8d3c10 | |||
| d642234814 | |||
| 56b4975d10 | |||
| 94f4cc69c4 | |||
| b27cd5fb82 | |||
| 45a37a8c08 | |||
| c5dbc9a22b | |||
| 6f2a8f26e1 | |||
| 9078a6f3dc | |||
| a1865a41c6 | |||
| 99a5086158 | |||
| 4b636979f9 | |||
| 9f6606f1e1 | |||
| b9f3149679 | |||
| 1c476003d6 | |||
| e32098fb94 | |||
| 67119d0627 | |||
| a62cec2090 | |||
| 6901df11c4 | |||
| 36e663c556 | |||
| 2f00c3feac | |||
| c163b076a0 | |||
| 3a0243da1f | |||
| 641fa09251 | |||
| 4895e487c0 | |||
| 5e8815d143 | |||
| 65b116c39f | |||
| 9a1c9b39ee | |||
| 40ffb99c97 |
37
.dockerignore
Normal file
37
.dockerignore
Normal file
@@ -0,0 +1,37 @@
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.egg-info/
|
||||
.git/
|
||||
.github/
|
||||
.gitignore
|
||||
.vscode/
|
||||
.vs/
|
||||
.idea/
|
||||
.mypy_cache/
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
.env
|
||||
*.log
|
||||
|
||||
# Docker files (not needed inside the image)
|
||||
Docker/
|
||||
|
||||
# Exception: VERSION is needed by Dockerfile.app
|
||||
!Docker/VERSION
|
||||
|
||||
# Test and dev files
|
||||
tests/
|
||||
Temp/
|
||||
test_data/
|
||||
docs/
|
||||
diagrams/
|
||||
|
||||
# Runtime data (mounted as volumes)
|
||||
data/aniworld.db
|
||||
data/config_backups/
|
||||
logs/
|
||||
|
||||
# Frontend tooling
|
||||
node_modules/
|
||||
package.json
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@@ -4,6 +4,7 @@
|
||||
/src/__pycache__/*
|
||||
/src/__pycache__/
|
||||
/.vs/*
|
||||
/.venv/*
|
||||
/src/Temp/*
|
||||
/src/Loaders/__pycache__/*
|
||||
/src/Loaders/provider/__pycache__/*
|
||||
@@ -81,3 +82,5 @@ Temp/
|
||||
temp/
|
||||
tmp/
|
||||
*.tmp
|
||||
.coverage
|
||||
.venv/bin/dotenv
|
||||
|
||||
4
.gitmodules
vendored
4
.gitmodules
vendored
@@ -1,4 +0,0 @@
|
||||
[submodule "src/AniWorld-Downloader"]
|
||||
path = src/AniWorld-Downloader
|
||||
url = https://github.com/lukaspupkalipinski/AniWorld-Downloader.git
|
||||
branch = next
|
||||
7
.vscode/settings.json
vendored
7
.vscode/settings.json
vendored
@@ -1,8 +1,11 @@
|
||||
{
|
||||
"python.defaultInterpreterPath": "C:\\Users\\lukas\\anaconda3\\envs\\AniWorld\\python.exe",
|
||||
"python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python",
|
||||
"python.terminal.activateEnvironment": true,
|
||||
"python.condaPath": "C:\\Users\\lukas\\anaconda3\\Scripts\\conda.exe",
|
||||
"python.terminal.activateEnvInCurrentTerminal": true,
|
||||
"terminal.integrated.env.linux": {
|
||||
"VIRTUAL_ENV": "${workspaceFolder}/.venv",
|
||||
"PATH": "${workspaceFolder}/.venv/bin:${env:PATH}"
|
||||
},
|
||||
"python.linting.enabled": true,
|
||||
"python.linting.flake8Enabled": true,
|
||||
"python.linting.pylintEnabled": true,
|
||||
|
||||
25
Docker/Containerfile
Normal file
25
Docker/Containerfile
Normal file
@@ -0,0 +1,25 @@
|
||||
FROM alpine:3.19
|
||||
|
||||
RUN apk add --no-cache \
|
||||
wireguard-tools \
|
||||
iptables \
|
||||
ip6tables \
|
||||
bash \
|
||||
curl \
|
||||
iputils-ping \
|
||||
iproute2 \
|
||||
openresolv
|
||||
|
||||
# Create wireguard config directory (config is mounted at runtime)
|
||||
RUN mkdir -p /etc/wireguard
|
||||
|
||||
# Copy version file and entrypoint
|
||||
COPY VERSION /etc/wireguard/VERSION
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
# Health check: can we reach the internet through the VPN?
|
||||
HEALTHCHECK --interval=30s --timeout=10s --retries=5 \
|
||||
CMD curl -sf --max-time 5 http://1.1.1.1 || exit 1
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
35
Docker/Dockerfile.app
Normal file
35
Docker/Dockerfile.app
Normal file
@@ -0,0 +1,35 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install system dependencies for compiled Python packages and ffmpeg for HLS support
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
gcc \
|
||||
g++ \
|
||||
libffi-dev \
|
||||
ffmpeg \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install Python dependencies (cached layer)
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Copy the full application
|
||||
COPY src/ ./src/
|
||||
COPY run_server.py .
|
||||
COPY pyproject.toml .
|
||||
COPY data/config.json ./data/config.json
|
||||
COPY Docker/VERSION ./Docker/VERSION
|
||||
|
||||
# Create runtime directories
|
||||
RUN mkdir -p /app/data/config_backups /app/logs
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV PYTHONPATH=/app
|
||||
|
||||
# Bind to 0.0.0.0 so the app is reachable from the VPN container's network
|
||||
CMD ["python", "-m", "uvicorn", "src.server.fastapi_app:app", \
|
||||
"--host", "0.0.0.0", "--port", "8000"]
|
||||
1
Docker/VERSION
Normal file
1
Docker/VERSION
Normal file
@@ -0,0 +1 @@
|
||||
v1.4.11
|
||||
91
Docker/dispatcher.d-99-wg-routes.sh
Normal file
91
Docker/dispatcher.d-99-wg-routes.sh
Normal file
@@ -0,0 +1,91 @@
|
||||
#!/bin/bash
|
||||
|
||||
# === Configuration ===
|
||||
LOGFILE="/tmp/dispatcher.log"
|
||||
BACKUP="/tmp/dispatcher.log.1"
|
||||
MAXSIZE=$((1024 * 1024)) # 1 MB
|
||||
VPN_IFACE="nl"
|
||||
GATEWAY="192.168.178.1"
|
||||
LOCAL_IFACE="wlp4s0f0"
|
||||
ROUTE1="185.183.34.149"
|
||||
ROUTE2="192.168.178.0/24"
|
||||
|
||||
# === Log Rotation ===
|
||||
if [ -f "$LOGFILE" ] && [ "$(stat -c%s "$LOGFILE")" -ge "$MAXSIZE" ]; then
|
||||
echo "[$(date)] Log file exceeded 1MB, rotating..." >> "$LOGFILE"
|
||||
mv "$LOGFILE" "$BACKUP"
|
||||
touch "$LOGFILE"
|
||||
fi
|
||||
|
||||
# === Logging Setup ===
|
||||
exec >> "$LOGFILE" 2>&1
|
||||
echo "[$(date)] Running dispatcher for $1 with status $2"
|
||||
|
||||
IFACE="$1"
|
||||
STATUS="$2"
|
||||
|
||||
log_and_run() {
|
||||
echo "[$(date)] Executing: $*"
|
||||
if ! output=$("$@" 2>&1); then
|
||||
echo "[$(date)] ERROR: Command failed: $*"
|
||||
echo "[$(date)] Output: $output"
|
||||
else
|
||||
echo "[$(date)] Success: $*"
|
||||
fi
|
||||
}
|
||||
|
||||
# === VPN Routing Logic ===
|
||||
if [ "$IFACE" = "$VPN_IFACE" ]; then
|
||||
case "$STATUS" in
|
||||
up)
|
||||
echo "[$(date)] VPN interface is up. Preparing routes..."
|
||||
|
||||
# === Wait for local interface and gateway ===
|
||||
echo "[$(date)] Waiting for $LOCAL_IFACE (state UP) and gateway $GATEWAY (reachable)..."
|
||||
until ip link show "$LOCAL_IFACE" | grep -q "state UP" && ip route get "$GATEWAY" &>/dev/null; do
|
||||
echo "[$(date)] Waiting for $LOCAL_IFACE and $GATEWAY..."
|
||||
sleep 1
|
||||
done
|
||||
echo "[$(date)] Local interface and gateway are ready."
|
||||
# === End Wait ===
|
||||
|
||||
# === APPLY ROUTES (Corrected Order) ===
|
||||
|
||||
# 1. Add the route for the local network FIRST
|
||||
log_and_run /sbin/ip route replace "$ROUTE2" dev "$LOCAL_IFACE"
|
||||
|
||||
# 2. Add the route to the VPN endpoint via the gateway SECOND
|
||||
log_and_run /sbin/ip route replace "$ROUTE1" via "$GATEWAY" dev "$LOCAL_IFACE"
|
||||
|
||||
# === END APPLY ROUTES ===
|
||||
|
||||
# Log interface and WireGuard status
|
||||
echo "[$(date)] --- ip addr show $VPN_IFACE ---"
|
||||
ip addr show "$VPN_IFACE"
|
||||
echo "[$(date)] --- wg show $VPN_IFACE ---"
|
||||
wg show "$VPN_IFACE"
|
||||
|
||||
;;
|
||||
|
||||
down)
|
||||
echo "[$(date)] VPN interface is down. Verifying before removing routes..."
|
||||
|
||||
# Log interface and WireGuard status
|
||||
echo "[$(date)] --- ip addr show $VPN_IFACE ---"
|
||||
ip addr show "$VPN_IFACE"
|
||||
echo "[$(date)] --- wg show $VPN_IFACE ---"
|
||||
wg show "$VPN_IFACE"
|
||||
|
||||
# Delay and confirm interface is still down
|
||||
sleep 5
|
||||
if ip link show "$VPN_IFACE" | grep -q "state UP"; then
|
||||
echo "[$(date)] VPN interface is still up. Skipping route removal."
|
||||
else
|
||||
echo "[$(date)] Confirmed VPN is down. Removing routes..."
|
||||
# It's good practice to remove them in reverse order, too.
|
||||
log_and_run /sbin/ip route del "$ROUTE1" via "$GATEWAY" dev "$LOCAL_IFACE"
|
||||
log_and_run /sbin/ip route del "$ROUTE2" dev "$LOCAL_IFACE"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
387
Docker/entrypoint.sh
Normal file
387
Docker/entrypoint.sh
Normal file
@@ -0,0 +1,387 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
VERSION_FILE="/etc/wireguard/VERSION"
|
||||
if [ -f "$VERSION_FILE" ]; then
|
||||
VERSION=$(cat "$VERSION_FILE")
|
||||
else
|
||||
VERSION="unknown"
|
||||
fi
|
||||
echo "[init] VPN Container Entrypoint ${VERSION}"
|
||||
|
||||
INTERFACE="wg0"
|
||||
MOUNT_CONFIG="/etc/wireguard/${INTERFACE}.conf"
|
||||
CONFIG_DIR="/run/wireguard"
|
||||
CONFIG_FILE="${CONFIG_DIR}/${INTERFACE}.conf"
|
||||
CHECK_INTERVAL="${HEALTH_CHECK_INTERVAL:-10}"
|
||||
CHECK_HOST="${HEALTH_CHECK_HOST:-1.1.1.1}"
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Validate config exists, copy to writable location
|
||||
# ──────────────────────────────────────────────
|
||||
if [ ! -f "$MOUNT_CONFIG" ]; then
|
||||
echo "[error] WireGuard config not found at ${MOUNT_CONFIG}"
|
||||
echo "[error] Mount your config file: -v /path/to/your.conf:/etc/wireguard/wg0.conf:ro"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$CONFIG_DIR"
|
||||
cp "$MOUNT_CONFIG" "$CONFIG_FILE"
|
||||
chmod 600 "$CONFIG_FILE"
|
||||
|
||||
# Extract endpoint IP and port from the config
|
||||
VPN_ENDPOINT=$(grep -i '^Endpoint' "$CONFIG_FILE" | head -1 | sed 's/.*= *//;s/:.*//;s/ //g')
|
||||
VPN_PORT=$(grep -i '^Endpoint' "$CONFIG_FILE" | head -1 | sed 's/.*://;s/ //g')
|
||||
# Extract address
|
||||
VPN_ADDRESS=$(grep -i '^Address' "$CONFIG_FILE" | head -1 | sed 's/.*= *//;s/ //g')
|
||||
|
||||
if [ -z "$VPN_ENDPOINT" ] || [ -z "$VPN_PORT" ]; then
|
||||
echo "[error] Could not parse Endpoint from ${CONFIG_FILE}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[init] Config: ${CONFIG_FILE}"
|
||||
echo "[init] Endpoint: ${VPN_ENDPOINT}:${VPN_PORT}"
|
||||
echo "[init] Address: ${VPN_ADDRESS}"
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Kill switch: only allow traffic through wg0
|
||||
# ──────────────────────────────────────────────
|
||||
setup_killswitch() {
|
||||
echo "[killswitch] Setting up iptables kill switch..."
|
||||
|
||||
# Flush existing rules
|
||||
iptables -F
|
||||
iptables -X
|
||||
iptables -t nat -F
|
||||
|
||||
# Default policy: DROP everything
|
||||
iptables -P INPUT DROP
|
||||
iptables -P FORWARD DROP
|
||||
iptables -P OUTPUT DROP
|
||||
|
||||
# Allow loopback
|
||||
iptables -A INPUT -i lo -j ACCEPT
|
||||
iptables -A OUTPUT -o lo -j ACCEPT
|
||||
|
||||
# Allow traffic to/from VPN endpoint (needed to establish tunnel)
|
||||
iptables -A OUTPUT -d "$VPN_ENDPOINT" -p udp --dport "$VPN_PORT" -j ACCEPT
|
||||
iptables -A INPUT -s "$VPN_ENDPOINT" -p udp --sport "$VPN_PORT" -j ACCEPT
|
||||
|
||||
# Allow all traffic through the WireGuard interface
|
||||
iptables -A INPUT -i "$INTERFACE" -j ACCEPT
|
||||
iptables -A OUTPUT -o "$INTERFACE" -j ACCEPT
|
||||
|
||||
# Allow DNS (VPN DNS servers are routed through wg0; allow before routing decision)
|
||||
iptables -A OUTPUT -p udp --dport 53 -j ACCEPT
|
||||
iptables -A OUTPUT -p tcp --dport 53 -j ACCEPT
|
||||
iptables -A INPUT -p udp --sport 53 -j ACCEPT
|
||||
iptables -A INPUT -p tcp --sport 53 -j ACCEPT
|
||||
|
||||
# Allow DHCP (for container networking)
|
||||
iptables -A OUTPUT -p udp --dport 67:68 -j ACCEPT
|
||||
iptables -A INPUT -p udp --sport 67:68 -j ACCEPT
|
||||
|
||||
# Allow established/related connections
|
||||
iptables -A INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
|
||||
iptables -A OUTPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
|
||||
|
||||
# ── Allow incoming connections to exposed service ports (e.g. app on 8000) ──
|
||||
# LOCAL_PORTS can be set as env var, e.g. "8000,8080,3000"
|
||||
if [ -n "${LOCAL_PORTS:-}" ]; then
|
||||
for port in $(echo "$LOCAL_PORTS" | tr ',' ' '); do
|
||||
echo "[killswitch] Allowing incoming traffic on port ${port}"
|
||||
iptables -A INPUT -p tcp --dport "$port" -j ACCEPT
|
||||
iptables -A OUTPUT -p tcp --sport "$port" -j ACCEPT
|
||||
done
|
||||
fi
|
||||
|
||||
# ── FORWARDING (so other containers can use this VPN) ──
|
||||
iptables -A FORWARD -i eth0 -o "$INTERFACE" -j ACCEPT
|
||||
iptables -A FORWARD -i "$INTERFACE" -o eth0 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
|
||||
|
||||
# NAT: masquerade traffic from other containers going out through wg0
|
||||
iptables -t nat -A POSTROUTING -o "$INTERFACE" -j MASQUERADE
|
||||
|
||||
echo "[killswitch] Kill switch active. Traffic blocked if VPN drops."
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Enable IP forwarding so other containers can route through us
|
||||
# ──────────────────────────────────────────────
|
||||
enable_forwarding() {
|
||||
echo "[init] Enabling IP forwarding..."
|
||||
if cat /proc/sys/net/ipv4/ip_forward 2>/dev/null | grep -q 1; then
|
||||
echo "[init] IP forwarding already enabled."
|
||||
elif echo 1 > /proc/sys/net/ipv4/ip_forward 2>/dev/null; then
|
||||
echo "[init] IP forwarding enabled via /proc."
|
||||
else
|
||||
echo "[init] /proc read-only — relying on --sysctl net.ipv4.ip_forward=1"
|
||||
fi
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Start WireGuard manually (no wg-quick, avoids sysctl issues)
|
||||
# ──────────────────────────────────────────────
|
||||
start_vpn() {
|
||||
echo "[vpn] Starting WireGuard interface ${INTERFACE}..."
|
||||
|
||||
# Create the interface
|
||||
ip link add "$INTERFACE" type wireguard
|
||||
|
||||
# Apply the WireGuard config (keys, peer, endpoint)
|
||||
# Filter out wg-quick directives that wg setconf doesn't understand
|
||||
wg setconf "$INTERFACE" <(grep -v -i '^\(Address\|DNS\|MTU\|Table\|PreUp\|PostUp\|PreDown\|PostDown\|SaveConfig\)' "$CONFIG_FILE")
|
||||
|
||||
# Log public key so it can be verified against the server's peer list
|
||||
local PUBKEY
|
||||
PUBKEY=$(wg show "$INTERFACE" public-key 2>/dev/null || echo "unknown")
|
||||
echo "[vpn] Public key: ${PUBKEY}"
|
||||
|
||||
# Assign the address
|
||||
ip -4 address add "$VPN_ADDRESS" dev "$INTERFACE"
|
||||
|
||||
# Set MTU and bring up
|
||||
ip link set mtu 1420 up dev "$INTERFACE"
|
||||
|
||||
# ── fwmark-based routing (mirrors wg-quick behavior) ──
|
||||
# WireGuard marks its own encapsulated UDP packets with this fwmark.
|
||||
# Policy rules then ensure:
|
||||
# - Normal packets (no mark) → VPN routing table → wg0
|
||||
# - WireGuard-encapsulated packets (marked) → main table → eth0
|
||||
local FW_MARK=51820
|
||||
local FW_TABLE=51820
|
||||
wg set "$INTERFACE" fwmark "$FW_MARK"
|
||||
|
||||
# Remove any auto-created default route on wg0
|
||||
ip route del default dev "$INTERFACE" 2>/dev/null || true
|
||||
|
||||
# VPN routing table: send everything through the tunnel
|
||||
ip -4 route add default dev "$INTERFACE" table "$FW_TABLE"
|
||||
|
||||
# Policy rules:
|
||||
# 1. Packets NOT marked by WireGuard use the VPN table (→ wg0)
|
||||
# 2. suppress_prefixlength 0: ignore bare default routes in main table,
|
||||
# but keep more-specific routes (e.g. LAN, endpoint) working
|
||||
ip -4 rule add not fwmark "$FW_MARK" table "$FW_TABLE"
|
||||
ip -4 rule add table main suppress_prefixlength 0
|
||||
|
||||
# Find default gateway/interface
|
||||
DEFAULT_GW=$(ip route | grep '^default' | head -1 | awk '{print $3}')
|
||||
DEFAULT_IF=$(ip route | grep '^default' | head -1 | awk '{print $5}')
|
||||
|
||||
# ── Policy routing: ensure responses to incoming LAN traffic go back via eth0 ──
|
||||
if [ -n "$DEFAULT_GW" ] && [ -n "$DEFAULT_IF" ]; then
|
||||
# Get the container's eth0 IP address (BusyBox-compatible, no grep -P)
|
||||
ETH0_IP=$(ip -4 addr show "$DEFAULT_IF" | awk '/inet / {split($2, a, "/"); print a[1]}' | head -1)
|
||||
ETH0_SUBNET=$(ip -4 route show dev "$DEFAULT_IF" | grep -v default | head -1 | awk '{print $1}')
|
||||
if [ -n "$ETH0_IP" ] && [ -n "$ETH0_SUBNET" ]; then
|
||||
echo "[vpn] Setting up policy routing for incoming traffic (${ETH0_IP} on ${DEFAULT_IF})"
|
||||
ip route add default via "$DEFAULT_GW" dev "$DEFAULT_IF" table 100 2>/dev/null || true
|
||||
ip route add "$ETH0_SUBNET" dev "$DEFAULT_IF" table 100 2>/dev/null || true
|
||||
ip rule add from "$ETH0_IP" table 100 priority 100 2>/dev/null || true
|
||||
echo "[vpn] Policy routing active — incoming connections will be routed back via ${DEFAULT_IF}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Set up DNS (handle comma-separated DNS servers)
|
||||
VPN_DNS=$(grep -i '^DNS' "$CONFIG_FILE" | head -1 | sed 's/.*= *//;s/ //g')
|
||||
if [ -n "$VPN_DNS" ]; then
|
||||
# Clear resolv.conf and add each DNS server on its own line
|
||||
> /etc/resolv.conf
|
||||
for dns in $(echo "$VPN_DNS" | tr ',' ' '); do
|
||||
echo "nameserver $dns" >> /etc/resolv.conf
|
||||
# Add explicit route to DNS server through wg0 so it's found in main table
|
||||
# (suppress_prefixlength 0 ignores default routes but allows host routes)
|
||||
ip -4 route add "$dns" dev "$INTERFACE" 2>/dev/null || true
|
||||
done
|
||||
echo "[vpn] DNS set to: ${VPN_DNS}"
|
||||
fi
|
||||
|
||||
# Add explicit host route for the health-check target so it is picked up by
|
||||
# the 'lookup main suppress_prefixlength 0' rule (same as DNS servers above).
|
||||
# Without this, CHECK_HOST falls through to the VPN table default route whose
|
||||
# source-address selection can be defeated by the priority-100 'from ETH0_IP'
|
||||
# policy rule, causing pings to bypass wg0 and be dropped by the kill switch.
|
||||
ip -4 route add "${CHECK_HOST}" dev "$INTERFACE" 2>/dev/null || true
|
||||
echo "[vpn] Health-check route: ${CHECK_HOST} → ${INTERFACE}"
|
||||
|
||||
echo "[vpn] WireGuard interface ${INTERFACE} is up."
|
||||
echo "[vpn] Main routes:"
|
||||
ip route show | sed 's/^/[vpn] /'
|
||||
echo "[vpn] VPN table ($FW_TABLE):"
|
||||
ip route show table "$FW_TABLE" 2>/dev/null | sed 's/^/[vpn] /'
|
||||
echo "[vpn] Policy rules:"
|
||||
ip rule show | sed 's/^/[vpn] /'
|
||||
echo "[vpn] WireGuard status:"
|
||||
wg show "$INTERFACE" 2>/dev/null | sed 's/^/[vpn] /'
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Stop WireGuard manually
|
||||
# ──────────────────────────────────────────────
|
||||
stop_vpn() {
|
||||
echo "[vpn] Stopping WireGuard interface ${INTERFACE}..."
|
||||
|
||||
local FW_MARK=51820
|
||||
local FW_TABLE=51820
|
||||
|
||||
# Remove fwmark-based policy rules
|
||||
ip -4 rule del not fwmark "$FW_MARK" table "$FW_TABLE" 2>/dev/null || true
|
||||
ip -4 rule del table main suppress_prefixlength 0 2>/dev/null || true
|
||||
|
||||
# Flush VPN routing table
|
||||
ip -4 route flush table "$FW_TABLE" 2>/dev/null || true
|
||||
|
||||
# Remove LAN policy routing
|
||||
ip -4 rule del table 100 2>/dev/null || true
|
||||
ip -4 route flush table 100 2>/dev/null || true
|
||||
|
||||
ip link del "$INTERFACE" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Health check loop — restarts VPN if tunnel dies
|
||||
# ──────────────────────────────────────────────
|
||||
health_loop() {
|
||||
local failures=0
|
||||
local max_failures=3
|
||||
|
||||
echo "[health] Starting health check (every ${CHECK_INTERVAL}s, target ${CHECK_HOST})..."
|
||||
|
||||
while true; do
|
||||
sleep "$CHECK_INTERVAL"
|
||||
|
||||
if ping -c 1 -W 5 "$CHECK_HOST" > /dev/null 2>&1; then
|
||||
if [ "$failures" -gt 0 ]; then
|
||||
echo "[health] VPN recovered."
|
||||
failures=0
|
||||
fi
|
||||
# Secondary DNS check
|
||||
if ping -c 1 -W 5 "google.com" > /dev/null 2>&1; then
|
||||
: # DNS OK — silent
|
||||
else
|
||||
echo "[health] WARN google.com unreachable — possible DNS issue"
|
||||
fi
|
||||
else
|
||||
failures=$((failures + 1))
|
||||
echo "[health] Check failed ($failures/$max_failures) — ping ${CHECK_HOST} failed"
|
||||
# Secondary check: distinguish IP failure from DNS failure
|
||||
if ping -c 1 -W 5 "google.com" > /dev/null 2>&1; then
|
||||
echo "[health] INFO google.com reachable — DNS works, ${CHECK_HOST} may be filtered"
|
||||
else
|
||||
echo "[health] INFO google.com also unreachable — DNS or general routing failure"
|
||||
fi
|
||||
# Dump WireGuard stats to show if handshake is stale and how much data flows
|
||||
echo "[health] wg stats:"
|
||||
wg show "$INTERFACE" 2>/dev/null | grep -E 'latest handshake|transfer|endpoint' | sed 's/^/[health] /' || echo "[health] wg0 not found"
|
||||
echo "[health] routes:"
|
||||
ip route show | grep -E 'wg0|default' | sed 's/^/[health] /'
|
||||
|
||||
if [ "$failures" -ge "$max_failures" ]; then
|
||||
echo "[health] VPN appears down. Restarting WireGuard..."
|
||||
stop_vpn
|
||||
sleep 2
|
||||
start_vpn
|
||||
failures=0
|
||||
echo "[health] WireGuard restarted."
|
||||
fi
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Graceful shutdown
|
||||
# ──────────────────────────────────────────────
|
||||
cleanup() {
|
||||
echo "[shutdown] Stopping WireGuard..."
|
||||
stop_vpn
|
||||
echo "[shutdown] Flushing iptables..."
|
||||
iptables -F
|
||||
iptables -t nat -F
|
||||
echo "[shutdown] Done."
|
||||
exit 0
|
||||
}
|
||||
|
||||
trap cleanup SIGTERM SIGINT
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Startup connectivity checks — diagnose issues early
|
||||
# ──────────────────────────────────────────────
|
||||
check_vpn_connectivity() {
|
||||
echo "[check] ── Startup connectivity checks ──"
|
||||
|
||||
# 1. Wait for WireGuard handshake (up to 15s)
|
||||
local elapsed=0
|
||||
local handshake_ts=0
|
||||
echo "[check] Waiting for WireGuard handshake (up to 15s)..."
|
||||
while [ "$elapsed" -lt 15 ]; do
|
||||
handshake_ts=$(wg show "$INTERFACE" latest-handshakes 2>/dev/null | awk '{print $2}' | head -1)
|
||||
if [ -n "$handshake_ts" ] && [ "$handshake_ts" != "0" ]; then
|
||||
local age=$(( $(date +%s) - handshake_ts ))
|
||||
echo "[check] OK Handshake established ${age}s ago"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
elapsed=$((elapsed + 1))
|
||||
done
|
||||
if [ "$elapsed" -ge 15 ]; then
|
||||
echo "[check] FAIL No WireGuard handshake after 15s — tunnel is not up"
|
||||
echo "[check] This container's public key (must be on the server):"
|
||||
echo "[check] PublicKey = $(wg show "$INTERFACE" public-key 2>/dev/null || echo 'unknown')"
|
||||
echo "[check] AllowedIPs = ${VPN_ADDRESS}"
|
||||
echo "[check] Verify on server: wg show"
|
||||
fi
|
||||
|
||||
# 2. Check whether traffic actually flows through the tunnel
|
||||
echo "[check] Testing traffic through tunnel (ping ${CHECK_HOST})..."
|
||||
local rx_before
|
||||
rx_before=$(wg show "$INTERFACE" transfer 2>/dev/null | awk '{print $2}' | head -1)
|
||||
|
||||
if ping -c 1 -W 8 "${CHECK_HOST}" > /dev/null 2>&1; then
|
||||
echo "[check] OK Traffic flows — tunnel is fully working"
|
||||
else
|
||||
local rx_after
|
||||
rx_after=$(wg show "$INTERFACE" transfer 2>/dev/null | awk '{print $2}' | head -1)
|
||||
echo "[check] FAIL ping ${CHECK_HOST} unreachable through tunnel"
|
||||
|
||||
if [ -n "$rx_before" ] && [ -n "$rx_after" ]; then
|
||||
if [ "$rx_after" -le "$rx_before" ]; then
|
||||
echo "[check] RX bytes unchanged (${rx_before} → ${rx_after})"
|
||||
echo "[check] Server receives packets but does NOT route them back"
|
||||
echo "[check] Fix on VPN server (${VPN_ENDPOINT}):"
|
||||
echo "[check] sysctl net.ipv4.ip_forward # must output 1"
|
||||
echo "[check] iptables -t nat -L POSTROUTING -v -n # must have MASQUERADE"
|
||||
echo "[check] wg show # check peer + AllowedIPs"
|
||||
else
|
||||
echo "[check] RX increased (${rx_before} → ${rx_after}) — tunnel passes data"
|
||||
echo "[check] Issue may be specific to ${CHECK_HOST} or DNS"
|
||||
fi
|
||||
fi
|
||||
|
||||
local transfer
|
||||
transfer=$(wg show "$INTERFACE" transfer 2>/dev/null | awk '{printf "rx=%s tx=%s", $2, $3}')
|
||||
echo "[check] wg transfer: ${transfer}"
|
||||
fi
|
||||
|
||||
# 3. DNS check
|
||||
echo "[check] Testing DNS resolution..."
|
||||
if nslookup 1.1.1.1 > /dev/null 2>&1 || nslookup google.com > /dev/null 2>&1; then
|
||||
echo "[check] OK DNS resolves"
|
||||
else
|
||||
echo "[check] FAIL DNS resolution failed"
|
||||
echo "[check] resolv.conf: $(tr '\n' ' ' < /etc/resolv.conf)"
|
||||
echo "[check] Check that DNS servers are reachable through wg0"
|
||||
echo "[check] ── End of checks ──"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[check] ── End of checks ──"
|
||||
}
|
||||
|
||||
# ── Main ──
|
||||
enable_forwarding
|
||||
setup_killswitch
|
||||
start_vpn
|
||||
check_vpn_connectivity
|
||||
health_loop
|
||||
16
Docker/nl.conf
Normal file
16
Docker/nl.conf
Normal file
@@ -0,0 +1,16 @@
|
||||
[Interface]
|
||||
PrivateKey = EPRa2f/v72LvIXAY4yqIRJifsSb+nCcYHqC2rwA94UI=
|
||||
Address = 100.64.244.78/32
|
||||
DNS = 198.18.0.1,198.18.0.2
|
||||
|
||||
# Route zum VPN-Server direkt über dein lokales Netz
|
||||
PostUp = ip route add 91.148.236.64 via 192.168.178.1 dev wlp4s0f0
|
||||
PostUp = ip route add 192.168.178.0/24 via 192.168.178.1 dev wlp4s0f0
|
||||
PostDown = ip route del 91.148.236.64 via 192.168.178.1 dev wlp4s0f0
|
||||
PostDown = ip route del 192.168.178.0/24 via 192.168.178.1 dev wlp4s0f0
|
||||
|
||||
[Peer]
|
||||
PublicKey = KgTUh3KLijVluDvNpzDCJJfrJ7EyLzYLmdHCksG4sRg=
|
||||
AllowedIPs = 0.0.0.0/0
|
||||
Endpoint = 91.148.236.64:51820
|
||||
|
||||
56
Docker/podman-compose.prod.yml
Normal file
56
Docker/podman-compose.prod.yml
Normal file
@@ -0,0 +1,56 @@
|
||||
# Production compose — pulls pre-built images from Gitea registry.
|
||||
#
|
||||
# Usage:
|
||||
# podman login git.lpl-mind.de
|
||||
# podman-compose -f podman-compose.prod.yml pull
|
||||
# podman-compose -f podman-compose.prod.yml up -d
|
||||
#
|
||||
# Required files:
|
||||
# - wg0.conf (WireGuard configuration in the same directory)
|
||||
|
||||
services:
|
||||
vpn:
|
||||
image: git.lpl-mind.de/lukas.pupkalipinski/aniworld/vpn:latest
|
||||
container_name: vpn-wireguard
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
- SYS_MODULE
|
||||
- NET_RAW
|
||||
sysctls:
|
||||
- net.ipv4.ip_forward=1
|
||||
- net.ipv4.conf.all.src_valid_mark=1
|
||||
volumes:
|
||||
- /server/server_aniworld/wg0.conf:/etc/wireguard/wg0.conf:ro
|
||||
- /lib/modules:/lib/modules:ro
|
||||
ports:
|
||||
- "8000:8000"
|
||||
environment:
|
||||
- HEALTH_CHECK_INTERVAL=10
|
||||
- HEALTH_CHECK_HOST=1.1.1.1
|
||||
- LOCAL_PORTS=8000
|
||||
- PUID=1013
|
||||
- PGID=1001
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-sf", "--max-time", "5", "http://1.1.1.1"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 60s
|
||||
|
||||
app:
|
||||
image: git.lpl-mind.de/lukas.pupkalipinski/aniworld/app:latest
|
||||
container_name: aniworld-app
|
||||
network_mode: "service:vpn"
|
||||
depends_on:
|
||||
vpn:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
- PYTHONUNBUFFERED=1
|
||||
- PUID=1013
|
||||
- PGID=1001
|
||||
volumes:
|
||||
- /server/server_aniworld/data:/app/data
|
||||
- /server/server_aniworld/logs:/app/logs
|
||||
- /media/serien/Serien:/data
|
||||
restart: unless-stopped
|
||||
49
Docker/podman-compose.yml
Normal file
49
Docker/podman-compose.yml
Normal file
@@ -0,0 +1,49 @@
|
||||
services:
|
||||
vpn:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Containerfile
|
||||
container_name: vpn-wireguard
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
- SYS_MODULE
|
||||
- NET_RAW
|
||||
sysctls:
|
||||
- net.ipv4.ip_forward=1
|
||||
- net.ipv4.conf.all.src_valid_mark=1
|
||||
volumes:
|
||||
- ./wg0.conf:/etc/wireguard/wg0.conf:ro
|
||||
- /lib/modules:/lib/modules:ro
|
||||
ports:
|
||||
- "8000:8000"
|
||||
environment:
|
||||
- HEALTH_CHECK_INTERVAL=10
|
||||
- HEALTH_CHECK_HOST=1.1.1.1
|
||||
- LOCAL_PORTS=8000
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "ping", "-c", "1", "-W", "5", "1.1.1.1"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
|
||||
app:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: Docker/Dockerfile.app
|
||||
container_name: aniworld-app
|
||||
network_mode: "service:vpn"
|
||||
depends_on:
|
||||
vpn:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
- PYTHONUNBUFFERED=1
|
||||
- LOG_LEVEL=DEBUG
|
||||
volumes:
|
||||
- app-data:/app/data
|
||||
- app-logs:/app/logs
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
app-data:
|
||||
app-logs:
|
||||
140
Docker/push.sh
Normal file
140
Docker/push.sh
Normal file
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Build and push AniWorld container images to the Gitea registry.
|
||||
#
|
||||
# Usage:
|
||||
# ./push.sh # builds & pushes app with tag "latest"
|
||||
# ./push.sh app # builds & pushes app image
|
||||
# ./push.sh vpn # builds & pushes vpn image
|
||||
# ./push.sh all # builds & pushes both images
|
||||
# ./push.sh app v1.2.3 # builds & pushes app with tag "v1.2.3"
|
||||
# ./push.sh vpn v1.2.3 # builds & pushes vpn with tag "v1.2.3"
|
||||
# ./push.sh all v1.2.3 # builds & pushes both images
|
||||
# ./push.sh app v1.2.3 --no-build # pushes existing image only
|
||||
#
|
||||
# Prerequisites:
|
||||
# podman login git.lpl-mind.de (or: docker login git.lpl-mind.de)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
REGISTRY="git.lpl-mind.de"
|
||||
NAMESPACE="lukas.pupkalipinski"
|
||||
PROJECT="aniworld"
|
||||
|
||||
APP_IMAGE="${REGISTRY}/${NAMESPACE}/${PROJECT}/app"
|
||||
VPN_IMAGE="${REGISTRY}/${NAMESPACE}/${PROJECT}/vpn"
|
||||
|
||||
# Parse arguments
|
||||
TARGET="${1:-app}"
|
||||
TAG="${2:-latest}"
|
||||
SKIP_BUILD=false
|
||||
if [[ "${3:-}" == "--no-build" ]]; then
|
||||
SKIP_BUILD=true
|
||||
fi
|
||||
|
||||
# Validate target
|
||||
if [[ "${TARGET}" != "app" && "${TARGET}" != "vpn" && "${TARGET}" != "all" ]]; then
|
||||
echo "ERROR: Invalid target '${TARGET}'. Must be one of: app, vpn, all" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
log() { echo -e "\n>>> $*"; }
|
||||
err() { echo -e "\nERROR: $*" >&2; exit 1; }
|
||||
|
||||
# Detect container engine (podman preferred, docker fallback)
|
||||
if command -v podman &>/dev/null; then
|
||||
ENGINE="podman"
|
||||
elif command -v docker &>/dev/null; then
|
||||
ENGINE="docker"
|
||||
else
|
||||
err "Neither podman nor docker is installed."
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pre-flight checks
|
||||
# ---------------------------------------------------------------------------
|
||||
echo "============================================"
|
||||
echo " AniWorld — Build & Push"
|
||||
echo " Engine : ${ENGINE}"
|
||||
echo " Registry : ${REGISTRY}"
|
||||
echo " Target : ${TARGET}"
|
||||
echo " Tag : ${TAG}"
|
||||
echo "============================================"
|
||||
|
||||
log "Logging in to ${REGISTRY}"
|
||||
"${ENGINE}" login "${REGISTRY}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Build
|
||||
# ---------------------------------------------------------------------------
|
||||
build_app() {
|
||||
log "Building app image → ${APP_IMAGE}:${TAG}"
|
||||
"${ENGINE}" build \
|
||||
-t "${APP_IMAGE}:${TAG}" \
|
||||
-f "${SCRIPT_DIR}/Dockerfile.app" \
|
||||
"${PROJECT_ROOT}"
|
||||
}
|
||||
|
||||
build_vpn() {
|
||||
log "Building vpn image → ${VPN_IMAGE}:${TAG}"
|
||||
"${ENGINE}" build \
|
||||
-t "${VPN_IMAGE}:${TAG}" \
|
||||
-f "${SCRIPT_DIR}/Containerfile" \
|
||||
"${SCRIPT_DIR}"
|
||||
}
|
||||
|
||||
if [[ "${SKIP_BUILD}" == false ]]; then
|
||||
case "${TARGET}" in
|
||||
app) build_app ;;
|
||||
vpn) build_vpn ;;
|
||||
all) build_app; build_vpn ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Push
|
||||
# ---------------------------------------------------------------------------
|
||||
push_app() {
|
||||
log "Pushing ${APP_IMAGE}:${TAG}"
|
||||
"${ENGINE}" push "${APP_IMAGE}:${TAG}"
|
||||
}
|
||||
|
||||
push_vpn() {
|
||||
log "Pushing ${VPN_IMAGE}:${TAG}"
|
||||
"${ENGINE}" push "${VPN_IMAGE}:${TAG}"
|
||||
}
|
||||
|
||||
case "${TARGET}" in
|
||||
app) push_app ;;
|
||||
vpn) push_vpn ;;
|
||||
all) push_app; push_vpn ;;
|
||||
esac
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summary
|
||||
# ---------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "============================================"
|
||||
echo " Push complete!"
|
||||
echo ""
|
||||
echo " Images:"
|
||||
case "${TARGET}" in
|
||||
app) echo " ${APP_IMAGE}:${TAG}" ;;
|
||||
vpn) echo " ${VPN_IMAGE}:${TAG}" ;;
|
||||
all) echo " ${APP_IMAGE}:${TAG}"; echo " ${VPN_IMAGE}:${TAG}" ;;
|
||||
esac
|
||||
echo ""
|
||||
echo " Deploy on server:"
|
||||
echo " ${ENGINE} login ${REGISTRY}"
|
||||
echo " ${ENGINE} compose -f Docker/podman-compose.prod.yml pull"
|
||||
echo " ${ENGINE} compose -f Docker/podman-compose.prod.yml up -d"
|
||||
echo "============================================"
|
||||
129
Docker/release.sh
Normal file
129
Docker/release.sh
Normal file
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Bump the project version and push images to the registry.
|
||||
#
|
||||
# Usage:
|
||||
# ./release.sh
|
||||
#
|
||||
# The current version is stored in VERSION (next to this script).
|
||||
# You will be asked whether to bump major, minor, or patch.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
VERSION_FILE="${SCRIPT_DIR}/VERSION"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Read current version
|
||||
# ---------------------------------------------------------------------------
|
||||
if [[ ! -f "${VERSION_FILE}" ]]; then
|
||||
echo "0.0.0" > "${VERSION_FILE}"
|
||||
fi
|
||||
|
||||
CURRENT="$(cat "${VERSION_FILE}")"
|
||||
# Strip leading 'v' for arithmetic
|
||||
VERSION="${CURRENT#v}"
|
||||
|
||||
IFS='.' read -r MAJOR MINOR PATCH <<< "${VERSION}"
|
||||
|
||||
echo "============================================"
|
||||
echo " AniWorld — Release"
|
||||
echo " Current version: v${MAJOR}.${MINOR}.${PATCH}"
|
||||
echo "============================================"
|
||||
echo ""
|
||||
echo "Which image(s) would you like to release?"
|
||||
echo " 1) app (Dockerfile.app)"
|
||||
echo " 2) vpn (Containerfile)"
|
||||
echo " 3) all (both images)"
|
||||
echo ""
|
||||
read -rp "Enter choice [1/2/3]: " TARGET_CHOICE
|
||||
|
||||
case "${TARGET_CHOICE}" in
|
||||
1) TARGET="app" ;;
|
||||
2) TARGET="vpn" ;;
|
||||
3) TARGET="all" ;;
|
||||
*)
|
||||
echo "Invalid choice. Aborting." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo ""
|
||||
echo "How would you like to bump the version?"
|
||||
echo " 1) patch (v${MAJOR}.${MINOR}.${PATCH} → v${MAJOR}.${MINOR}.$((PATCH + 1)))"
|
||||
echo " 2) minor (v${MAJOR}.${MINOR}.${PATCH} → v${MAJOR}.$((MINOR + 1)).0)"
|
||||
echo " 3) major (v${MAJOR}.${MINOR}.${PATCH} → v$((MAJOR + 1)).0.0)"
|
||||
echo ""
|
||||
read -rp "Enter choice [1/2/3]: " CHOICE
|
||||
|
||||
case "${CHOICE}" in
|
||||
1) NEW_TAG="v${MAJOR}.${MINOR}.$((PATCH + 1))" ;;
|
||||
2) NEW_TAG="v${MAJOR}.$((MINOR + 1)).0" ;;
|
||||
3) NEW_TAG="v$((MAJOR + 1)).0.0" ;;
|
||||
*)
|
||||
echo "Invalid choice. Aborting." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo ""
|
||||
echo "New version: ${NEW_TAG}"
|
||||
echo "Target: ${TARGET}"
|
||||
read -rp "Confirm? [y/N]: " CONFIRM
|
||||
if [[ ! "${CONFIRM}" =~ ^[yY]$ ]]; then
|
||||
echo "Aborted."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Write new version
|
||||
# ---------------------------------------------------------------------------
|
||||
echo "${NEW_TAG}" > "${VERSION_FILE}"
|
||||
echo "Version file updated → ${VERSION_FILE}"
|
||||
|
||||
# Keep root package.json in sync.
|
||||
FRONT_VERSION="${NEW_TAG#v}"
|
||||
FRONT_PKG="${SCRIPT_DIR}/../package.json"
|
||||
if [[ -f "${FRONT_PKG}" ]]; then
|
||||
sed -i "s/\"version\": \"[^\"]*\"/\"version\": \"${FRONT_VERSION}\"/" "${FRONT_PKG}"
|
||||
echo "package.json version updated → ${FRONT_VERSION}"
|
||||
else
|
||||
echo "Warning: package.json not found, skipping package.json version sync" >&2
|
||||
fi
|
||||
|
||||
# Keep root pyproject.toml in sync.
|
||||
BACKEND_PYPROJECT="${SCRIPT_DIR}/../pyproject.toml"
|
||||
if [[ -f "${BACKEND_PYPROJECT}" ]]; then
|
||||
# Update version under [project] section if present
|
||||
if grep -q '^\[project\]' "${BACKEND_PYPROJECT}"; then
|
||||
sed -i "/^\[project\]/,/^\[/ s/^version = \".*\"/version = \"${FRONT_VERSION}\"/" "${BACKEND_PYPROJECT}"
|
||||
else
|
||||
sed -i "s/^version = \".*\"/version = \"${FRONT_VERSION}\"/" "${BACKEND_PYPROJECT}"
|
||||
fi
|
||||
echo "pyproject.toml version updated → ${FRONT_VERSION}"
|
||||
else
|
||||
echo "Warning: pyproject.toml not found, skipping pyproject.toml version sync" >&2
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Push containers
|
||||
# ---------------------------------------------------------------------------
|
||||
bash "${SCRIPT_DIR}/push.sh" "${TARGET}" "${NEW_TAG}"
|
||||
bash "${SCRIPT_DIR}/push.sh" "${TARGET}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Git tag (local only; push after container build)
|
||||
# ---------------------------------------------------------------------------
|
||||
cd "${SCRIPT_DIR}/.."
|
||||
git add Docker/VERSION package.json pyproject.toml
|
||||
git commit -m "chore: bump version"
|
||||
git tag -a "${NEW_TAG}" -m "Release ${NEW_TAG}"
|
||||
echo "Local git commit + tag ${NEW_TAG} created."
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Push git commits & tag
|
||||
# ---------------------------------------------------------------------------
|
||||
git push origin HEAD
|
||||
git push origin "${NEW_TAG}"
|
||||
echo "Git commit and tag ${NEW_TAG} pushed."
|
||||
253
Docker/test_vpn.py
Normal file
253
Docker/test_vpn.py
Normal file
@@ -0,0 +1,253 @@
|
||||
"""
|
||||
Integration test for the WireGuard VPN Podman image.
|
||||
|
||||
Verifies:
|
||||
1. The image builds successfully.
|
||||
2. The container starts and becomes healthy.
|
||||
3. The public IP inside the VPN differs from the host IP.
|
||||
4. Kill switch blocks traffic when WireGuard is down.
|
||||
5. AllowedIPs routes are set dynamically from the config.
|
||||
|
||||
Requirements:
|
||||
- podman installed
|
||||
- Root/sudo (NET_ADMIN capability) for container runtime tests
|
||||
- A valid WireGuard config at ./wg0.conf (or ./nl.conf)
|
||||
|
||||
Usage:
|
||||
# Build-only test (no sudo needed):
|
||||
python3 -m pytest test_vpn.py::TestVPNImage::test_00_build_image -v
|
||||
|
||||
# Full integration test (requires sudo):
|
||||
sudo python3 -m pytest test_vpn.py -v
|
||||
# or
|
||||
sudo python3 test_vpn.py
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
IMAGE_NAME = "vpn-wireguard-test"
|
||||
CONTAINER_NAME = "vpn-test-container"
|
||||
CONFIG_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "wg0.conf")
|
||||
BUILD_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
IP_CHECK_URL = "https://ifconfig.me"
|
||||
STARTUP_TIMEOUT = 30 # seconds to wait for VPN to come up
|
||||
HEALTH_POLL_INTERVAL = 2 # seconds between health checks
|
||||
|
||||
|
||||
def is_root() -> bool:
|
||||
"""Check if running as root."""
|
||||
return os.geteuid() == 0
|
||||
|
||||
|
||||
def run(cmd: list[str], timeout: int = 30, check: bool = True) -> subprocess.CompletedProcess:
|
||||
"""Run a command and return the result."""
|
||||
return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, check=check)
|
||||
|
||||
|
||||
def get_host_ip() -> str:
|
||||
"""Get the public IP of the host machine."""
|
||||
result = run(["curl", "-s", "--max-time", "10", IP_CHECK_URL])
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def podman_exec(container: str, cmd: list[str], timeout: int = 15) -> subprocess.CompletedProcess:
|
||||
"""Execute a command inside a running container."""
|
||||
return run(["podman", "exec", container] + cmd, timeout=timeout, check=False)
|
||||
|
||||
|
||||
class TestVPNImage(unittest.TestCase):
|
||||
"""Test suite for the WireGuard VPN container."""
|
||||
|
||||
host_ip: str = ""
|
||||
container_id: str = ""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""Build image, get host IP, start container, wait for VPN."""
|
||||
# Clean up any leftover container from a previous run
|
||||
subprocess.run(
|
||||
["podman", "rm", "-f", CONTAINER_NAME],
|
||||
capture_output=True, check=False,
|
||||
)
|
||||
|
||||
# ── 1. Get host public IP before VPN ──
|
||||
logger.info("Fetching host public IP...")
|
||||
cls.host_ip = get_host_ip()
|
||||
logger.info("Host public IP: %s", cls.host_ip)
|
||||
assert cls.host_ip, "Could not determine host public IP"
|
||||
|
||||
# ── 2. Build the image ──
|
||||
logger.info("Building image '%s'...", IMAGE_NAME)
|
||||
result = run(
|
||||
["podman", "build", "-t", IMAGE_NAME, BUILD_DIR],
|
||||
timeout=180,
|
||||
)
|
||||
logger.debug(
|
||||
"Build output: %s",
|
||||
result.stdout[-500:] if len(result.stdout) > 500 else result.stdout,
|
||||
)
|
||||
assert result.returncode == 0, f"Build failed:\n{result.stderr}"
|
||||
logger.info("Image built successfully.")
|
||||
|
||||
# Skip container runtime tests if not root
|
||||
if not is_root():
|
||||
logger.warning("Not running as root — skipping container runtime tests.")
|
||||
cls.container_id = ""
|
||||
return
|
||||
|
||||
# ── 3. Start the container ──
|
||||
logger.info("Starting container '%s'...", CONTAINER_NAME)
|
||||
result = run(
|
||||
[
|
||||
"podman", "run", "-d",
|
||||
"--name", CONTAINER_NAME,
|
||||
"--cap-add=NET_ADMIN",
|
||||
"--cap-add=SYS_MODULE",
|
||||
"--sysctl", "net.ipv4.ip_forward=1",
|
||||
"-v", f"{CONFIG_FILE}:/etc/wireguard/wg0.conf:ro",
|
||||
"-v", "/lib/modules:/lib/modules:ro",
|
||||
IMAGE_NAME,
|
||||
],
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, f"Container failed to start:\n{result.stderr}"
|
||||
cls.container_id = result.stdout.strip()
|
||||
logger.info("Container started: %s", cls.container_id[:12])
|
||||
|
||||
# Verify it's running
|
||||
inspect = run(
|
||||
["podman", "inspect", "-f", "{{.State.Running}}", CONTAINER_NAME],
|
||||
check=False,
|
||||
)
|
||||
assert inspect.stdout.strip() == "true", "Container is not running"
|
||||
|
||||
# ── 4. Wait for VPN to come up ──
|
||||
logger.info("Waiting up to %d seconds for VPN tunnel...", STARTUP_TIMEOUT)
|
||||
vpn_up = cls._wait_for_vpn_cls(STARTUP_TIMEOUT)
|
||||
assert vpn_up, f"VPN did not come up within {STARTUP_TIMEOUT}s"
|
||||
logger.info("VPN tunnel is up. Running tests.")
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
"""Stop and remove the container."""
|
||||
if not is_root():
|
||||
return
|
||||
logger.info("Cleaning up test container...")
|
||||
subprocess.run(["podman", "rm", "-f", CONTAINER_NAME], capture_output=True, check=False)
|
||||
logger.info("Cleanup complete.")
|
||||
|
||||
@classmethod
|
||||
def _wait_for_vpn_cls(cls, timeout: int = STARTUP_TIMEOUT) -> bool:
|
||||
"""Wait until the VPN tunnel is up (can reach the internet)."""
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
result = podman_exec(CONTAINER_NAME, ["ping", "-c", "1", "-W", "3", "1.1.1.1"])
|
||||
if result.returncode == 0:
|
||||
return True
|
||||
time.sleep(HEALTH_POLL_INTERVAL)
|
||||
return False
|
||||
|
||||
def _get_vpn_ip(self) -> str:
|
||||
"""Get the public IP as seen from inside the container."""
|
||||
result = podman_exec(
|
||||
CONTAINER_NAME,
|
||||
["curl", "-s", "--max-time", "10", IP_CHECK_URL],
|
||||
timeout=20,
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
def _skip_if_not_root(self):
|
||||
"""Skip test if not running as root."""
|
||||
if not is_root():
|
||||
self.skipTest("This test requires root/sudo privileges")
|
||||
|
||||
# ── Tests ────────────────────────────────────────────────
|
||||
|
||||
def test_00_build_image(self):
|
||||
"""The image builds successfully."""
|
||||
# This is already verified in setUpClass, just confirm here
|
||||
result = run(["podman", "images", "--format", "{{.Repository}}:{{.Tag}}"])
|
||||
self.assertIn(IMAGE_NAME, result.stdout, "Image was not built")
|
||||
|
||||
def test_01_ip_differs_from_host(self):
|
||||
"""Public IP inside VPN is different from host IP."""
|
||||
self._skip_if_not_root()
|
||||
vpn_ip = self._get_vpn_ip()
|
||||
logger.info("VPN public IP: %s", vpn_ip)
|
||||
logger.info("Host public IP: %s", self.host_ip)
|
||||
|
||||
self.assertTrue(vpn_ip, "Could not fetch IP from inside the container")
|
||||
self.assertNotEqual(
|
||||
vpn_ip,
|
||||
self.host_ip,
|
||||
f"VPN IP ({vpn_ip}) is the same as host IP — VPN is not working!",
|
||||
)
|
||||
|
||||
def test_02_wireguard_interface_exists(self):
|
||||
"""The wg0 interface is present in the container."""
|
||||
self._skip_if_not_root()
|
||||
result = podman_exec(CONTAINER_NAME, ["wg", "show", "wg0"])
|
||||
self.assertEqual(result.returncode, 0, f"wg show failed:\n{result.stderr}")
|
||||
self.assertIn("peer", result.stdout.lower(), "No peer information in wg show output")
|
||||
# AllowedIPs should be present in wg show output
|
||||
self.assertIn("allowed ips", result.stdout.lower(), "AllowedIPs not found in wg show output")
|
||||
|
||||
def test_03_allowedips_routes_set(self):
|
||||
"""Routes are set dynamically based on AllowedIPs from config."""
|
||||
self._skip_if_not_root()
|
||||
# Check that routes exist for the AllowedIPs
|
||||
result = podman_exec(CONTAINER_NAME, ["ip", "route", "show", "dev", "wg0"])
|
||||
self.assertEqual(result.returncode, 0, f"ip route show failed:\n{result.stderr}")
|
||||
# The config has AllowedIPs = 0.0.0.0/0, which should result in:
|
||||
# 0.0.0.0/1 dev wg0 and 128.0.0.0/1 dev wg0
|
||||
self.assertIn("0.0.0.0/1", result.stdout, "Route 0.0.0.0/1 not found")
|
||||
self.assertIn("128.0.0.0/1", result.stdout, "Route 128.0.0.0/1 not found")
|
||||
# Make sure there is NO default route through wg0 (Table = off should prevent this)
|
||||
self.assertNotIn("default dev wg0", result.stdout, "Default route through wg0 found — Table = off not working!")
|
||||
logger.info("AllowedIPs routes verified: %s", result.stdout.strip())
|
||||
|
||||
def test_03b_dns_configured(self):
|
||||
"""DNS is configured correctly with multiple nameserver lines."""
|
||||
self._skip_if_not_root()
|
||||
result = podman_exec(CONTAINER_NAME, ["cat", "/etc/resolv.conf"])
|
||||
self.assertEqual(result.returncode, 0, f"cat /etc/resolv.conf failed:\n{result.stderr}")
|
||||
# Should have two separate nameserver lines, not one with commas
|
||||
self.assertIn("nameserver 198.18.0.1", result.stdout, "DNS 198.18.0.1 not found")
|
||||
self.assertIn("nameserver 198.18.0.2", result.stdout, "DNS 198.18.0.2 not found")
|
||||
# Make sure there are no commas in nameserver lines
|
||||
self.assertNotIn("nameserver 198.18.0.1,198.18.0.2", result.stdout, "DNS servers written on one line with comma!")
|
||||
logger.info("DNS config verified: %s", result.stdout.strip())
|
||||
|
||||
def test_04_kill_switch_blocks_traffic(self):
|
||||
"""When WireGuard is down, traffic is blocked (kill switch)."""
|
||||
self._skip_if_not_root()
|
||||
# Bring down the WireGuard interface by deleting it
|
||||
down_result = podman_exec(CONTAINER_NAME, ["ip", "link", "del", "wg0"], timeout=10)
|
||||
self.assertEqual(down_result.returncode, 0, f"ip link del wg0 failed:\n{down_result.stderr}")
|
||||
|
||||
# Give iptables a moment
|
||||
time.sleep(2)
|
||||
|
||||
# Try to reach the internet — should fail due to kill switch
|
||||
result = podman_exec(
|
||||
CONTAINER_NAME,
|
||||
["curl", "-s", "--max-time", "5", IP_CHECK_URL],
|
||||
timeout=10,
|
||||
)
|
||||
self.assertNotEqual(
|
||||
result.returncode, 0,
|
||||
"Traffic went through even with WireGuard down — kill switch is NOT working!",
|
||||
)
|
||||
logger.info("Kill switch confirmed: traffic blocked with VPN down")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
18
Docker/wg0.conf
Normal file
18
Docker/wg0.conf
Normal file
@@ -0,0 +1,18 @@
|
||||
[Interface]
|
||||
PrivateKey = EPRa2f/v72LvIXAY4yqIRJifsSb+nCcYHqC2rwA94UI=
|
||||
Address = 100.64.244.78/32
|
||||
#DNS = 198.18.0.1,198.18.0.2
|
||||
DNS = 8.8.8.8
|
||||
|
||||
# Route zum VPN-Server direkt über dein lokales Netz
|
||||
PostUp = ip route add 91.148.236.64 via 192.168.178.1 dev wlp4s0f0
|
||||
PostUp = ip route add 192.168.178.0/24 via 192.168.178.1 dev wlp4s0f0
|
||||
PostDown = ip route del 91.148.236.64 via 192.168.178.1 dev wlp4s0f0
|
||||
PostDown = ip route del 192.168.178.0/24 via 192.168.178.1 dev wlp4s0f0
|
||||
|
||||
[Peer]
|
||||
PublicKey = KgTUh3KLijVluDvNpzDCJJfrJ7EyLzYLmdHCksG4sRg=
|
||||
AllowedIPs = 0.0.0.0/0
|
||||
Endpoint = 91.148.236.64:51820
|
||||
PersistentKeepalive = 25
|
||||
|
||||
@@ -34,11 +34,11 @@ Authorization: Bearer <jwt_token>
|
||||
|
||||
**Public Endpoints (no authentication required):**
|
||||
|
||||
- `/api/auth/*` - Authentication endpoints
|
||||
- `/api/health` - Health check endpoints
|
||||
- `/api/docs`, `/api/redoc` - API documentation
|
||||
- `/static/*` - Static files
|
||||
- `/`, `/login`, `/setup`, `/queue` - UI pages
|
||||
- `/api/auth/*` - Authentication endpoints
|
||||
- `/api/health` - Health check endpoints
|
||||
- `/api/docs`, `/api/redoc` - API documentation
|
||||
- `/static/*` - Static files
|
||||
- `/`, `/login`, `/setup`, `/queue` - UI pages
|
||||
|
||||
Source: [src/server/middleware/auth.py](../src/server/middleware/auth.py#L39-L52)
|
||||
|
||||
@@ -91,7 +91,7 @@ Initial setup endpoint to configure the master password. Can only be called once
|
||||
|
||||
**Errors:**
|
||||
|
||||
- `400 Bad Request` - Master password already configured or invalid password
|
||||
- `400 Bad Request` - Master password already configured or invalid password
|
||||
|
||||
Source: [src/server/api/auth.py](../src/server/api/auth.py#L28-L90)
|
||||
|
||||
@@ -120,8 +120,8 @@ Validate master password and return JWT token.
|
||||
|
||||
**Errors:**
|
||||
|
||||
- `401 Unauthorized` - Invalid credentials
|
||||
- `429 Too Many Requests` - Account locked due to failed attempts
|
||||
- `401 Unauthorized` - Invalid credentials
|
||||
- `429 Too Many Requests` - Account locked due to failed attempts
|
||||
|
||||
Source: [src/server/api/auth.py](../src/server/api/auth.py#L93-L124)
|
||||
|
||||
@@ -203,7 +203,14 @@ List library series that have missing episodes.
|
||||
| `page` | int | 1 | Page number (must be positive) |
|
||||
| `per_page` | int | 20 | Items per page (max 1000) |
|
||||
| `sort_by` | string | null | Sort field: `title`, `id`, `name`, `missing_episodes` |
|
||||
| `filter` | string | null | Filter term |
|
||||
| `filter` | string | null | Filter: `missing_episodes` (shows series with any missing episodes), `no_episodes` (shows series with zero downloaded episodes) |
|
||||
|
||||
**Filter Details:**
|
||||
|
||||
- `missing_episodes`: Returns series that have at least one missing episode recorded in the database (`is_downloaded=False`)
|
||||
- `no_episodes`: Returns series that have missing episodes and no downloaded episodes (i.e., only missing episodes exist in the database)
|
||||
- Episodes in the database represent MISSING episodes (from episodeDict during scanning)
|
||||
- `is_downloaded=False` means the episode file was not found in the folder
|
||||
|
||||
**Response (200 OK):**
|
||||
|
||||
@@ -220,6 +227,12 @@ List library series that have missing episodes.
|
||||
]
|
||||
```
|
||||
|
||||
**Example with filter:**
|
||||
|
||||
```bash
|
||||
GET /api/anime?filter=no_episodes
|
||||
```
|
||||
|
||||
Source: [src/server/api/anime.py](../src/server/api/anime.py#L155-L303)
|
||||
|
||||
### GET /api/anime/search
|
||||
@@ -306,10 +319,10 @@ Add a new series to the library with automatic database persistence, folder crea
|
||||
|
||||
**Folder Name Sanitization:**
|
||||
|
||||
- Removes invalid filesystem characters: `< > : " / \ | ? *`
|
||||
- Trims leading/trailing whitespace and dots
|
||||
- Preserves Unicode characters (for Japanese titles)
|
||||
- Example: `"Attack on Titan: Final Season"` → `"Attack on Titan Final Season"`
|
||||
- Removes invalid filesystem characters: `< > : " / \ | ? *`
|
||||
- Trims leading/trailing whitespace and dots
|
||||
- Preserves Unicode characters (for Japanese titles)
|
||||
- Example: `"Attack on Titan: Final Season"` → `"Attack on Titan Final Season"`
|
||||
|
||||
Source: [src/server/api/anime.py](../src/server/api/anime.py#L604-L710)
|
||||
|
||||
@@ -647,7 +660,10 @@ Return current application configuration.
|
||||
"data_dir": "data",
|
||||
"scheduler": {
|
||||
"enabled": true,
|
||||
"interval_minutes": 60
|
||||
"interval_minutes": 60,
|
||||
"schedule_time": "03:00",
|
||||
"schedule_days": ["mon", "tue", "wed", "thu", "fri", "sat", "sun"],
|
||||
"auto_download_after_rescan": false
|
||||
},
|
||||
"logging": {
|
||||
"level": "INFO",
|
||||
@@ -678,7 +694,9 @@ Apply an update to the configuration.
|
||||
{
|
||||
"scheduler": {
|
||||
"enabled": true,
|
||||
"interval_minutes": 30
|
||||
"interval_minutes": 60,
|
||||
"schedule_time": "06:30",
|
||||
"schedule_days": ["mon", "wed", "fri"]
|
||||
},
|
||||
"logging": {
|
||||
"level": "DEBUG"
|
||||
@@ -804,32 +822,260 @@ Source: [src/server/api/config.py](../src/server/api/config.py#L189-L247)
|
||||
|
||||
---
|
||||
|
||||
## 6. Scheduler Endpoints
|
||||
## 6. NFO Management Endpoints
|
||||
|
||||
Prefix: `/api/scheduler`
|
||||
Prefix: `/api/nfo`
|
||||
|
||||
Source: [src/server/api/scheduler.py](../src/server/api/scheduler.py#L1-L122)
|
||||
Source: [src/server/api/nfo.py](../src/server/api/nfo.py#L1-L684)
|
||||
|
||||
### GET /api/scheduler/config
|
||||
These endpoints manage tvshow.nfo metadata files and associated media (poster, logo, fanart) for anime series. NFO files use Kodi/XBMC format and are scraped from TMDB API.
|
||||
|
||||
Get current scheduler configuration.
|
||||
**Prerequisites:**
|
||||
|
||||
- TMDB API key must be configured in settings
|
||||
- NFO service returns 503 if API key not configured
|
||||
|
||||
### GET /api/nfo/{serie_id}/check
|
||||
|
||||
Check if NFO file and media files exist for a series.
|
||||
|
||||
**Authentication:** Required
|
||||
|
||||
**Path Parameters:**
|
||||
|
||||
- `serie_id` (string): Series identifier
|
||||
|
||||
**Response (200 OK):**
|
||||
|
||||
```json
|
||||
{
|
||||
"enabled": true,
|
||||
"interval_minutes": 60
|
||||
"serie_id": "one-piece",
|
||||
"serie_folder": "One Piece (1999)",
|
||||
"has_nfo": true,
|
||||
"nfo_path": "/path/to/anime/One Piece (1999)/tvshow.nfo",
|
||||
"media_files": {
|
||||
"has_poster": true,
|
||||
"has_logo": false,
|
||||
"has_fanart": true,
|
||||
"poster_path": "/path/to/anime/One Piece (1999)/poster.jpg",
|
||||
"logo_path": null,
|
||||
"fanart_path": "/path/to/anime/One Piece (1999)/fanart.jpg"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Source: [src/server/api/scheduler.py](../src/server/api/scheduler.py#L22-L42)
|
||||
**Errors:**
|
||||
|
||||
### POST /api/scheduler/config
|
||||
- `401 Unauthorized` - Not authenticated
|
||||
- `404 Not Found` - Series not found
|
||||
- `503 Service Unavailable` - TMDB API key not configured
|
||||
|
||||
Update scheduler configuration.
|
||||
Source: [src/server/api/nfo.py](../src/server/api/nfo.py#L90-L147)
|
||||
|
||||
### POST /api/nfo/{serie_id}/create
|
||||
|
||||
Create NFO file and download media for a series.
|
||||
|
||||
**Authentication:** Required
|
||||
|
||||
**Path Parameters:**
|
||||
|
||||
- `serie_id` (string): Series identifier
|
||||
|
||||
**Request Body:**
|
||||
|
||||
```json
|
||||
{
|
||||
"serie_name": "One Piece",
|
||||
"year": 1999,
|
||||
"download_poster": true,
|
||||
"download_logo": true,
|
||||
"download_fanart": true,
|
||||
"overwrite_existing": false
|
||||
}
|
||||
```
|
||||
|
||||
**Fields:**
|
||||
|
||||
- `serie_name` (string, optional): Series name for TMDB search (defaults to folder name)
|
||||
- `year` (integer, optional): Series year to help narrow TMDB search
|
||||
- `download_poster` (boolean, default: true): Download poster.jpg
|
||||
- `download_logo` (boolean, default: true): Download logo.png
|
||||
- `download_fanart` (boolean, default: true): Download fanart.jpg
|
||||
- `overwrite_existing` (boolean, default: false): Overwrite existing NFO
|
||||
|
||||
**Response (200 OK):**
|
||||
|
||||
```json
|
||||
{
|
||||
"serie_id": "one-piece",
|
||||
"serie_folder": "One Piece (1999)",
|
||||
"nfo_path": "/path/to/anime/One Piece (1999)/tvshow.nfo",
|
||||
"media_files": {
|
||||
"has_poster": true,
|
||||
"has_logo": true,
|
||||
"has_fanart": true,
|
||||
"poster_path": "/path/to/anime/One Piece (1999)/poster.jpg",
|
||||
"logo_path": "/path/to/anime/One Piece (1999)/logo.png",
|
||||
"fanart_path": "/path/to/anime/One Piece (1999)/fanart.jpg"
|
||||
},
|
||||
"message": "NFO and media files created successfully"
|
||||
}
|
||||
```
|
||||
|
||||
**Errors:**
|
||||
|
||||
- `401 Unauthorized` - Not authenticated
|
||||
- `404 Not Found` - Series not found
|
||||
- `409 Conflict` - NFO already exists (use `overwrite_existing: true`)
|
||||
- `503 Service Unavailable` - TMDB API error or key not configured
|
||||
|
||||
Source: [src/server/api/nfo.py](../src/server/api/nfo.py#L150-L240)
|
||||
|
||||
### PUT /api/nfo/{serie_id}/update
|
||||
|
||||
Update existing NFO file with fresh TMDB data.
|
||||
|
||||
**Authentication:** Required
|
||||
|
||||
**Path Parameters:**
|
||||
|
||||
- `serie_id` (string): Series identifier
|
||||
|
||||
**Query Parameters:**
|
||||
|
||||
- `download_media` (boolean, default: true): Re-download media files
|
||||
|
||||
**Response (200 OK):**
|
||||
|
||||
```json
|
||||
{
|
||||
"serie_id": "one-piece",
|
||||
"serie_folder": "One Piece (1999)",
|
||||
"nfo_path": "/path/to/anime/One Piece (1999)/tvshow.nfo",
|
||||
"media_files": {
|
||||
"has_poster": true,
|
||||
"has_logo": true,
|
||||
"has_fanart": true,
|
||||
"poster_path": "/path/to/anime/One Piece (1999)/poster.jpg",
|
||||
"logo_path": "/path/to/anime/One Piece (1999)/logo.png",
|
||||
"fanart_path": "/path/to/anime/One Piece (1999)/fanart.jpg"
|
||||
},
|
||||
"message": "NFO updated successfully"
|
||||
}
|
||||
```
|
||||
|
||||
**Errors:**
|
||||
|
||||
- `401 Unauthorized` - Not authenticated
|
||||
- `404 Not Found` - Series or NFO not found (use create endpoint)
|
||||
- `503 Service Unavailable` - TMDB API error
|
||||
|
||||
Source: [src/server/api/nfo.py](../src/server/api/nfo.py#L243-L325)
|
||||
|
||||
### GET /api/nfo/{serie_id}/content
|
||||
|
||||
Get NFO file XML content for a series.
|
||||
|
||||
**Authentication:** Required
|
||||
|
||||
**Path Parameters:**
|
||||
|
||||
- `serie_id` (string): Series identifier
|
||||
|
||||
**Response (200 OK):**
|
||||
|
||||
```json
|
||||
{
|
||||
"serie_id": "one-piece",
|
||||
"serie_folder": "One Piece (1999)",
|
||||
"content": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<tvshow>...</tvshow>",
|
||||
"file_size": 2048,
|
||||
"last_modified": "2026-01-15T10:30:00"
|
||||
}
|
||||
```
|
||||
|
||||
**Errors:**
|
||||
|
||||
- `401 Unauthorized` - Not authenticated
|
||||
- `404 Not Found` - Series or NFO not found
|
||||
|
||||
Source: [src/server/api/nfo.py](../src/server/api/nfo.py#L328-L397)
|
||||
|
||||
### GET /api/nfo/{serie_id}/media/status
|
||||
|
||||
Get media files status for a series.
|
||||
|
||||
**Authentication:** Required
|
||||
|
||||
**Path Parameters:**
|
||||
|
||||
- `serie_id` (string): Series identifier
|
||||
|
||||
**Response (200 OK):**
|
||||
|
||||
```json
|
||||
{
|
||||
"has_poster": true,
|
||||
"has_logo": false,
|
||||
"has_fanart": true,
|
||||
"poster_path": "/path/to/anime/One Piece (1999)/poster.jpg",
|
||||
"logo_path": null,
|
||||
"fanart_path": "/path/to/anime/One Piece (1999)/fanart.jpg"
|
||||
}
|
||||
```
|
||||
|
||||
**Errors:**
|
||||
|
||||
- `401 Unauthorized` - Not authenticated
|
||||
- `404 Not Found` - Series not found
|
||||
|
||||
Source: [src/server/api/nfo.py](../src/server/api/nfo.py#L400-L447)
|
||||
|
||||
### POST /api/nfo/{serie_id}/media/download
|
||||
|
||||
Download missing media files for a series.
|
||||
|
||||
**Authentication:** Required
|
||||
|
||||
**Path Parameters:**
|
||||
|
||||
- `serie_id` (string): Series identifier
|
||||
|
||||
**Request Body:**
|
||||
|
||||
```json
|
||||
{
|
||||
"download_poster": true,
|
||||
"download_logo": true,
|
||||
"download_fanart": true
|
||||
}
|
||||
```
|
||||
|
||||
**Response (200 OK):**
|
||||
|
||||
```json
|
||||
{
|
||||
"has_poster": true,
|
||||
"has_logo": true,
|
||||
"has_fanart": true,
|
||||
"poster_path": "/path/to/anime/One Piece (1999)/poster.jpg",
|
||||
"logo_path": "/path/to/anime/One Piece (1999)/logo.png",
|
||||
"fanart_path": "/path/to/anime/One Piece (1999)/fanart.jpg"
|
||||
}
|
||||
```
|
||||
|
||||
**Errors:**
|
||||
|
||||
- `401 Unauthorized` - Not authenticated
|
||||
- `404 Not Found` - Series or NFO not found (NFO required for TMDB ID)
|
||||
- `503 Service Unavailable` - TMDB API error
|
||||
|
||||
Source: [src/server/api/nfo.py](../src/server/api/nfo.py#L450-L519)
|
||||
|
||||
### POST /api/nfo/batch/create
|
||||
|
||||
Batch create NFO files for multiple series.
|
||||
|
||||
**Authentication:** Required
|
||||
|
||||
@@ -837,18 +1083,120 @@ Update scheduler configuration.
|
||||
|
||||
```json
|
||||
{
|
||||
"enabled": true,
|
||||
"interval_minutes": 30
|
||||
"serie_ids": ["one-piece", "naruto", "bleach"],
|
||||
"download_media": true,
|
||||
"skip_existing": true,
|
||||
"max_concurrent": 3
|
||||
}
|
||||
```
|
||||
|
||||
**Response (200 OK):** Updated scheduler configuration
|
||||
**Fields:**
|
||||
|
||||
Source: [src/server/api/scheduler.py](../src/server/api/scheduler.py#L45-L75)
|
||||
- `serie_ids` (array of strings): Series identifiers to process
|
||||
- `download_media` (boolean, default: true): Download media files
|
||||
- `skip_existing` (boolean, default: true): Skip series with existing NFOs
|
||||
- `max_concurrent` (integer, 1-10, default: 3): Number of concurrent operations
|
||||
|
||||
### POST /api/scheduler/trigger-rescan
|
||||
**Response (200 OK):**
|
||||
|
||||
Manually trigger a library rescan.
|
||||
```json
|
||||
{
|
||||
"total": 3,
|
||||
"successful": 2,
|
||||
"failed": 0,
|
||||
"skipped": 1,
|
||||
"results": [
|
||||
{
|
||||
"serie_id": "one-piece",
|
||||
"serie_folder": "One Piece (1999)",
|
||||
"success": true,
|
||||
"message": "NFO created successfully",
|
||||
"nfo_path": "/path/to/anime/One Piece (1999)/tvshow.nfo"
|
||||
},
|
||||
{
|
||||
"serie_id": "naruto",
|
||||
"serie_folder": "Naruto (2002)",
|
||||
"success": false,
|
||||
"message": "Skipped - NFO already exists",
|
||||
"nfo_path": null
|
||||
},
|
||||
{
|
||||
"serie_id": "bleach",
|
||||
"serie_folder": "Bleach (2004)",
|
||||
"success": true,
|
||||
"message": "NFO created successfully",
|
||||
"nfo_path": "/path/to/anime/Bleach (2004)/tvshow.nfo"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Errors:**
|
||||
|
||||
- `401 Unauthorized` - Not authenticated
|
||||
- `503 Service Unavailable` - TMDB API key not configured
|
||||
|
||||
Source: [src/server/api/nfo.py](../src/server/api/nfo.py#L522-L634)
|
||||
|
||||
### GET /api/nfo/missing
|
||||
|
||||
Get list of series without NFO files.
|
||||
|
||||
**Authentication:** Required
|
||||
|
||||
**Response (200 OK):**
|
||||
|
||||
```json
|
||||
{
|
||||
"total_series": 150,
|
||||
"missing_nfo_count": 23,
|
||||
"series": [
|
||||
{
|
||||
"serie_id": "dragon-ball",
|
||||
"serie_folder": "Dragon Ball (1986)",
|
||||
"serie_name": "Dragon Ball",
|
||||
"has_media": false,
|
||||
"media_files": {
|
||||
"has_poster": false,
|
||||
"has_logo": false,
|
||||
"has_fanart": false,
|
||||
"poster_path": null,
|
||||
"logo_path": null,
|
||||
"fanart_path": null
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**Errors:**
|
||||
|
||||
- `401 Unauthorized` - Not authenticated
|
||||
- `503 Service Unavailable` - TMDB API key not configured
|
||||
|
||||
Source: [src/server/api/nfo.py](../src/server/api/nfo.py#L637-L684)
|
||||
|
||||
---
|
||||
|
||||
## 7. Scheduler Endpoints
|
||||
|
||||
Prefix: `/api/scheduler`
|
||||
|
||||
All GET/POST config responses share the same envelope:
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"config": { ... },
|
||||
"status": { ... }
|
||||
}
|
||||
```
|
||||
|
||||
Source: [src/server/api/scheduler.py](../src/server/api/scheduler.py)
|
||||
|
||||
### GET /api/scheduler/config
|
||||
|
||||
Get current scheduler configuration and runtime status.
|
||||
|
||||
**Authentication:** Required
|
||||
|
||||
@@ -857,15 +1205,69 @@ Manually trigger a library rescan.
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"config": {
|
||||
"enabled": true,
|
||||
"interval_minutes": 60,
|
||||
"schedule_time": "03:00",
|
||||
"schedule_days": ["mon", "tue", "wed", "thu", "fri", "sat", "sun"],
|
||||
"auto_download_after_rescan": false
|
||||
},
|
||||
"status": {
|
||||
"is_running": true,
|
||||
"next_run": "2025-07-15T03:00:00+00:00",
|
||||
"last_run": null,
|
||||
"scan_in_progress": false
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### POST /api/scheduler/config
|
||||
|
||||
Update scheduler configuration and apply changes immediately.
|
||||
|
||||
**Authentication:** Required
|
||||
|
||||
**Request Body (all fields optional, uses model defaults):**
|
||||
|
||||
```json
|
||||
{
|
||||
"enabled": true,
|
||||
"schedule_time": "06:30",
|
||||
"schedule_days": ["mon", "wed", "fri"],
|
||||
"auto_download_after_rescan": true
|
||||
}
|
||||
```
|
||||
|
||||
**Response (200 OK):** Same envelope as GET, reflecting saved values.
|
||||
|
||||
**Validation errors (422):**
|
||||
|
||||
- `schedule_time` must match `HH:MM` (00:00–23:59)
|
||||
- `schedule_days` entries must be one of `mon tue wed thu fri sat sun`
|
||||
- `interval_minutes` must be ≥ 1
|
||||
|
||||
### POST /api/scheduler/trigger-rescan
|
||||
|
||||
Manually trigger a library rescan (and auto-download if configured).
|
||||
|
||||
**Authentication:** Required
|
||||
|
||||
**Response (200 OK):**
|
||||
|
||||
```json
|
||||
{
|
||||
"message": "Rescan started successfully"
|
||||
}
|
||||
```
|
||||
|
||||
Source: [src/server/api/scheduler.py](../src/server/api/scheduler.py#L78-L122)
|
||||
**Error responses:**
|
||||
|
||||
- `503` — SeriesApp not yet initialised
|
||||
- `500` — Rescan failed unexpectedly
|
||||
|
||||
---
|
||||
|
||||
## 7. Health Check Endpoints
|
||||
## 8. Health Check Endpoints
|
||||
|
||||
Prefix: `/health`
|
||||
|
||||
@@ -883,7 +1285,7 @@ Basic health check endpoint.
|
||||
{
|
||||
"status": "healthy",
|
||||
"timestamp": "2025-12-13T10:30:00.000Z",
|
||||
"version": "1.0.0"
|
||||
"version": "1.0.1"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -901,7 +1303,7 @@ Comprehensive health check with database, filesystem, and system metrics.
|
||||
{
|
||||
"status": "healthy",
|
||||
"timestamp": "2025-12-13T10:30:00.000Z",
|
||||
"version": "1.0.0",
|
||||
"version": "1.0.1",
|
||||
"dependencies": {
|
||||
"database": {
|
||||
"status": "healthy",
|
||||
@@ -930,7 +1332,7 @@ Source: [src/server/api/health.py](../src/server/api/health.py#L164-L200)
|
||||
|
||||
---
|
||||
|
||||
## 8. WebSocket Protocol
|
||||
## 9. WebSocket Protocol
|
||||
|
||||
Endpoint: `/ws/connect`
|
||||
|
||||
@@ -991,7 +1393,7 @@ Clients can join/leave rooms to receive specific updates.
|
||||
|
||||
**Available Rooms:**
|
||||
|
||||
- `downloads` - Download progress and status updates
|
||||
- `downloads` - Download progress and status updates
|
||||
|
||||
### Server Message Format
|
||||
|
||||
@@ -1039,7 +1441,7 @@ Source: [src/server/api/websocket.py](../src/server/api/websocket.py#L238-L260)
|
||||
|
||||
---
|
||||
|
||||
## 9. Data Models
|
||||
## 10. Data Models
|
||||
|
||||
### Download Item
|
||||
|
||||
@@ -1100,7 +1502,7 @@ Source: [src/server/models/download.py](../src/server/models/download.py#L44-L60
|
||||
|
||||
---
|
||||
|
||||
## 10. Error Handling
|
||||
## 11. Error Handling
|
||||
|
||||
### HTTP Status Codes
|
||||
|
||||
@@ -1146,7 +1548,7 @@ Source: [src/server/middleware/error_handler.py](../src/server/middleware/error_
|
||||
|
||||
---
|
||||
|
||||
## 11. Rate Limiting
|
||||
## 12. Rate Limiting
|
||||
|
||||
### Authentication Endpoints
|
||||
|
||||
@@ -1175,7 +1577,7 @@ HTTP Status: 429 Too Many Requests
|
||||
|
||||
---
|
||||
|
||||
## 12. Pagination
|
||||
## 13. Pagination
|
||||
|
||||
The anime list endpoint supports pagination.
|
||||
|
||||
@@ -31,8 +31,10 @@ Aniworld is a web-based anime download manager built with Python, FastAPI, and S
|
||||
+--------v---------+ +--------v---------+
|
||||
| | | |
|
||||
| SQLite DB | | File System |
|
||||
| (aniworld.db) | | (data/*.json) |
|
||||
| | | |
|
||||
| (aniworld.db) | | (anime/*/) |
|
||||
| - Series data | | - Video files |
|
||||
| - Episodes | | - NFO files |
|
||||
| - Queue state | | - Media files |
|
||||
+------------------+ +------------------+
|
||||
```
|
||||
|
||||
@@ -63,6 +65,7 @@ src/server/
|
||||
| +-- config.py # /api/config/* endpoints
|
||||
| +-- download.py # /api/queue/* endpoints
|
||||
| +-- scheduler.py # /api/scheduler/* endpoints
|
||||
| +-- nfo.py # /api/nfo/* endpoints
|
||||
| +-- websocket.py # /ws/* WebSocket handlers
|
||||
| +-- health.py # /health/* endpoints
|
||||
+-- controllers/ # Page controllers for HTML rendering
|
||||
@@ -77,6 +80,9 @@ src/server/
|
||||
| +-- progress_service.py # Progress tracking
|
||||
| +-- websocket_service.py# WebSocket broadcasting
|
||||
| +-- queue_repository.py # Database persistence
|
||||
| +-- nfo_service.py # NFO metadata management
|
||||
| +-- setup_service.py # Series key resolution from folder names
|
||||
| +-- folder_scan_service.py # Daily folder maintenance scan
|
||||
+-- models/ # Pydantic models
|
||||
| +-- auth.py # Auth request/response models
|
||||
| +-- config.py # Configuration models
|
||||
@@ -200,6 +206,17 @@ src/core/
|
||||
+-- entities/ # Domain entities
|
||||
| +-- series.py # Serie class with sanitized_folder property
|
||||
| +-- SerieList.py # SerieList collection with sanitized folder support
|
||||
| +-- nfo_models.py # Pydantic models for tvshow.nfo (TVShowNFO, ActorInfo…)
|
||||
+-- services/ # Domain services
|
||||
| +-- nfo_service.py # NFO lifecycle: create / update tvshow.nfo
|
||||
| +-- nfo_repair_service.py # Detect & repair incomplete tvshow.nfo files
|
||||
| | # (parse_nfo_tags, find_missing_tags, NfoRepairService)
|
||||
| +-- tmdb_client.py # Async TMDB API client
|
||||
+-- utils/ # Utility helpers (no side-effects)
|
||||
| +-- nfo_generator.py # TVShowNFO → XML serialiser
|
||||
| +-- nfo_mapper.py # TMDB API dict → TVShowNFO (tmdb_to_nfo_model,
|
||||
| | # _extract_rating_by_country, _extract_fsk_rating)
|
||||
| +-- image_downloader.py # TMDB image downloader
|
||||
+-- providers/ # External provider adapters
|
||||
| +-- base_provider.py # Loader interface
|
||||
| +-- provider_factory.py # Provider registry
|
||||
@@ -218,6 +235,10 @@ src/core/
|
||||
| `Serie` | Domain entity with `sanitized_folder` property for filesystem-safe names |
|
||||
| `SerieList` | Collection management with automatic folder creation using sanitized names |
|
||||
|
||||
**Initialization:**
|
||||
|
||||
`SeriesApp` is initialized with `skip_load=True` passed to `SerieList`, preventing automatic loading of series from data files on every instantiation. Series data is loaded once during application setup via `sync_series_from_data_files()` in the FastAPI lifespan, which reads data files and syncs them to the database. Subsequent operations load series from the database through the service layer.
|
||||
|
||||
Source: [src/core/](../src/core/)
|
||||
|
||||
### 2.4 Infrastructure Layer (`src/infrastructure/`)
|
||||
@@ -242,6 +263,48 @@ Source: [src/config/settings.py](../src/config/settings.py#L1-L96)
|
||||
|
||||
---
|
||||
|
||||
## 12. Startup Sequence
|
||||
|
||||
The FastAPI lifespan function (`src/server/fastapi_app.py`) runs the following steps on every server start.
|
||||
|
||||
### 12.1 Startup Order
|
||||
|
||||
```
|
||||
1. Logging configured
|
||||
|
||||
2. Temp folder purged ← cleans leftover partial download files
|
||||
+-- Iterate ./Temp/ and delete every file and sub-directory
|
||||
+-- Create ./Temp/ if it does not exist
|
||||
+-- Errors are logged as warnings; startup continues regardless
|
||||
|
||||
3. Database initialised (required – abort on failure)
|
||||
+-- SQLite file created / migrated via init_db()
|
||||
|
||||
4. Configuration loaded from data/config.json
|
||||
+-- Synced to settings (ENV vars take precedence)
|
||||
|
||||
5. Progress & WebSocket services wired up
|
||||
|
||||
6. Series loaded from database into memory
|
||||
|
||||
7. Download service initialised (queue restored from DB)
|
||||
|
||||
8. Background loader service started
|
||||
|
||||
9. Scheduler service started
|
||||
+-- Cron-based library rescans configured
|
||||
+-- Optional: auto-download missing episodes after rescan
|
||||
+-- Optional: folder maintenance (NFO repair, key resolution, renaming, poster checks) during scheduled runs
|
||||
```
|
||||
|
||||
### 12.2 Temp Folder Guarantee
|
||||
|
||||
Every server start begins with a clean `./Temp/` directory. This ensures that partial `.part` files or stale temp videos from a crashed or force-killed previous session are never left behind before new downloads start.
|
||||
|
||||
Source: [src/server/fastapi_app.py](../src/server/fastapi_app.py)
|
||||
|
||||
---
|
||||
|
||||
## 11. Graceful Shutdown
|
||||
|
||||
The application implements a comprehensive graceful shutdown mechanism that ensures data integrity and proper cleanup when the server is stopped via Ctrl+C (SIGINT) or SIGTERM.
|
||||
@@ -344,12 +407,29 @@ Source: [src/server/middleware/auth.py](../src/server/middleware/auth.py#L1-L209
|
||||
+-- WebSocketService broadcasts to clients
|
||||
|
||||
3. During download:
|
||||
+-- Provider writes to ./Temp/<filename> (+ ./Temp/<filename>.part fragments)
|
||||
+-- ProgressService.emit("progress_updated")
|
||||
+-- WebSocketService.broadcast_to_room()
|
||||
+-- Client receives WebSocket message
|
||||
|
||||
4. After download attempt (success OR failure):
|
||||
+-- _cleanup_temp_file() removes ./Temp/<filename> and all .part fragments
|
||||
+-- On success: file was already moved to final destination before cleanup
|
||||
+-- On failure / exception: no partial files remain in ./Temp/
|
||||
```
|
||||
|
||||
Source: [src/server/services/download_service.py](../src/server/services/download_service.py#L1-L150)
|
||||
#### Temp Directory Contract
|
||||
|
||||
| Situation | Outcome |
|
||||
| -------------------------------- | ------------------------------------------------------------------- |
|
||||
| Server start | Entire `./Temp/` directory is purged before any service initialises |
|
||||
| Successful download | Temp file moved to destination, then removed from `./Temp/` |
|
||||
| Failed download (provider error) | Temp + `.part` fragments removed by `_cleanup_temp_file()` |
|
||||
| Exception / cancellation | Temp + `.part` fragments removed in `except` block |
|
||||
|
||||
Source: [src/server/services/download_service.py](../src/server/services/download_service.py#L1-L150),
|
||||
[src/core/providers/aniworld_provider.py](../src/core/providers/aniworld_provider.py),
|
||||
[src/core/providers/enhanced_provider.py](../src/core/providers/enhanced_provider.py)
|
||||
|
||||
### 3.3 WebSocket Event Flow
|
||||
|
||||
@@ -368,19 +448,54 @@ Source: [src/server/api/websocket.py](../src/server/api/websocket.py#L1-L260)
|
||||
|
||||
## 4. Design Patterns
|
||||
|
||||
### 4.1 Repository Pattern
|
||||
### 4.1 Repository Pattern (Service Layer as Repository)
|
||||
|
||||
Database access is abstracted through repository classes.
|
||||
**Architecture Decision**: The Service Layer serves as the Repository layer for database access.
|
||||
|
||||
Database access is abstracted through service classes in `src/server/database/service.py` that provide CRUD operations and act as the repository layer. This eliminates the need for a separate repository layer while maintaining clean separation of concerns.
|
||||
|
||||
**Service Layer Classes** (acting as repositories):
|
||||
|
||||
- `AnimeSeriesService` - CRUD operations for anime series
|
||||
- `EpisodeService` - CRUD operations for episodes
|
||||
- `DownloadQueueService` - CRUD operations for download queue
|
||||
- `UserSessionService` - CRUD operations for user sessions
|
||||
- `SystemSettingsService` - CRUD operations for system settings
|
||||
|
||||
**Key Principles**:
|
||||
|
||||
1. **No Direct Database Queries**: Controllers and business logic services MUST use service layer methods
|
||||
2. **Service Layer Encapsulation**: All SQLAlchemy queries are encapsulated in service methods
|
||||
3. **Consistent Interface**: Services provide consistent async methods for all database operations
|
||||
4. **Single Responsibility**: Each service manages one entity type
|
||||
|
||||
**Example Usage**:
|
||||
|
||||
```python
|
||||
# QueueRepository provides CRUD for download items
|
||||
# CORRECT: Use service layer
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
|
||||
async with get_db_session() as db:
|
||||
series = await AnimeSeriesService.get_by_key(db, "attack-on-titan")
|
||||
await AnimeSeriesService.update(db, series.id, has_nfo=True)
|
||||
|
||||
# INCORRECT: Direct database query
|
||||
result = await db.execute(select(AnimeSeries).filter(...)) # ❌ Never do this
|
||||
```
|
||||
|
||||
**Special Case - Queue Repository Adapter**:
|
||||
|
||||
The `QueueRepository` in `src/server/services/queue_repository.py` is an adapter that wraps `DownloadQueueService` to provide domain model conversion between Pydantic models and SQLAlchemy models:
|
||||
|
||||
```python
|
||||
# QueueRepository provides CRUD with model conversion
|
||||
class QueueRepository:
|
||||
async def save_item(self, item: DownloadItem) -> None: ...
|
||||
async def get_all_items(self) -> List[DownloadItem]: ...
|
||||
async def save_item(self, item: DownloadItem) -> None: ... # Converts Pydantic → SQLAlchemy
|
||||
async def get_all_items(self) -> List[DownloadItem]: ... # Converts SQLAlchemy → Pydantic
|
||||
async def delete_item(self, item_id: str) -> bool: ...
|
||||
```
|
||||
|
||||
Source: [src/server/services/queue_repository.py](../src/server/services/queue_repository.py)
|
||||
Source: [src/server/database/service.py](../src/server/database/service.py), [src/server/services/queue_repository.py](../src/server/services/queue_repository.py)
|
||||
|
||||
### 4.2 Dependency Injection
|
||||
|
||||
@@ -425,6 +540,78 @@ def get_download_service() -> DownloadService:
|
||||
return _download_service_instance
|
||||
```
|
||||
|
||||
### 4.5 Error Handling Pattern
|
||||
|
||||
**Architecture Decision**: Dual error handling approach based on exception source.
|
||||
|
||||
The application uses two complementary error handling mechanisms:
|
||||
|
||||
1. **FastAPI HTTPException** - For simple validation and HTTP-level errors
|
||||
2. **Custom Exception Hierarchy** - For business logic and service-level errors with rich context
|
||||
|
||||
#### Exception Hierarchy
|
||||
|
||||
```python
|
||||
# Base exception with HTTP status mapping
|
||||
AniWorldAPIException(message, status_code, error_code, details)
|
||||
├── AuthenticationError (401)
|
||||
├── AuthorizationError (403)
|
||||
├── ValidationError (422)
|
||||
├── NotFoundError (404)
|
||||
├── ConflictError (409)
|
||||
├── BadRequestError (400)
|
||||
├── RateLimitError (429)
|
||||
└── ServerError (500)
|
||||
├── DownloadError
|
||||
├── ConfigurationError
|
||||
├── ProviderError
|
||||
└── DatabaseError
|
||||
```
|
||||
|
||||
#### When to Use Each
|
||||
|
||||
**Use HTTPException for:**
|
||||
|
||||
- Simple parameter validation (missing fields, wrong type)
|
||||
- Direct HTTP-level errors (401, 403, 404 without business context)
|
||||
- Quick endpoint-specific failures
|
||||
|
||||
**Use Custom Exceptions for:**
|
||||
|
||||
- Service-layer business logic errors (AnimeServiceError, ConfigServiceError)
|
||||
- Errors needing rich context (details dict, error codes)
|
||||
- Errors that should be logged with specific categorization
|
||||
- Cross-cutting concerns (authentication, authorization, rate limiting)
|
||||
|
||||
**Example:**
|
||||
|
||||
```python
|
||||
# Simple validation - Use HTTPException
|
||||
if not series_key:
|
||||
raise HTTPException(status_code=400, detail="series_key required")
|
||||
|
||||
# Business logic error - Use custom exception
|
||||
try:
|
||||
await anime_service.add_series(series_key)
|
||||
except AnimeServiceError as e:
|
||||
raise ServerError(
|
||||
message=f"Failed to add series: {e}",
|
||||
error_code="ANIME_ADD_FAILED",
|
||||
details={"series_key": series_key}
|
||||
)
|
||||
```
|
||||
|
||||
#### Global Exception Handlers
|
||||
|
||||
All custom exceptions are automatically handled by global middleware that:
|
||||
|
||||
- Converts exceptions to structured JSON responses
|
||||
- Logs errors with appropriate severity
|
||||
- Includes request ID for tracking
|
||||
- Provides consistent error format
|
||||
|
||||
**Source**: [src/server/exceptions/\_\_init\_\_.py](../src/server/exceptions/__init__.py), [src/server/middleware/error_handler.py](../src/server/middleware/error_handler.py)
|
||||
|
||||
Source: [src/server/services/download_service.py](../src/server/services/download_service.py)
|
||||
|
||||
---
|
||||
@@ -470,7 +657,12 @@ Configuration is stored in `data/config.json`:
|
||||
{
|
||||
"name": "Aniworld",
|
||||
"data_dir": "data",
|
||||
"scheduler": { "enabled": true, "interval_minutes": 60 },
|
||||
"scheduler": {
|
||||
"enabled": true,
|
||||
"schedule_time": "03:00",
|
||||
"schedule_days": ["mon", "tue", "wed", "thu", "fri", "sat", "sun"],
|
||||
"auto_download_after_rescan": false
|
||||
},
|
||||
"logging": { "level": "INFO" },
|
||||
"backup": { "enabled": false, "path": "data/backups" },
|
||||
"other": {
|
||||
@@ -574,10 +766,10 @@ Source: [src/server/services/auth_service.py](../src/server/services/auth_servic
|
||||
|
||||
### 9.2 Password Requirements
|
||||
|
||||
- Minimum 8 characters
|
||||
- Mixed case (upper and lower)
|
||||
- At least one number
|
||||
- At least one special character
|
||||
- Minimum 8 characters
|
||||
- Mixed case (upper and lower)
|
||||
- At least one number
|
||||
- At least one special character
|
||||
|
||||
Source: [src/server/services/auth_service.py](../src/server/services/auth_service.py#L97-L125)
|
||||
|
||||
243
Docs/CHANGELOG.md
Normal file
243
Docs/CHANGELOG.md
Normal file
@@ -0,0 +1,243 @@
|
||||
# Changelog
|
||||
|
||||
## Document Purpose
|
||||
|
||||
This document tracks all notable changes to the Aniworld project.
|
||||
|
||||
### What This Document Contains
|
||||
|
||||
- **Version History**: All released versions with dates
|
||||
- **Added Features**: New functionality in each release
|
||||
- **Changed Features**: Modifications to existing features
|
||||
- **Deprecated Features**: Features marked for removal
|
||||
- **Removed Features**: Features removed from the codebase
|
||||
- **Fixed Bugs**: Bug fixes with issue references
|
||||
- **Security Fixes**: Security-related changes
|
||||
- **Breaking Changes**: Changes requiring user action
|
||||
|
||||
### What This Document Does NOT Contain
|
||||
|
||||
- Internal refactoring details (unless user-facing)
|
||||
- Commit-level changes
|
||||
- Work-in-progress features
|
||||
- Roadmap or planned features
|
||||
|
||||
### Target Audience
|
||||
|
||||
- All users and stakeholders
|
||||
- Operators planning upgrades
|
||||
- Developers tracking changes
|
||||
- Support personnel
|
||||
|
||||
---
|
||||
|
||||
## Format
|
||||
|
||||
This changelog follows [Keep a Changelog](https://keepachangelog.com/) principles and adheres to [Semantic Versioning](https://semver.org/).
|
||||
|
||||
---
|
||||
|
||||
## [Unreleased] - 2026-06-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Folder scan series key resolution**: Fixed "Could not resolve series key for folder, skipping" warnings during library setup. `_resolve_key_via_search()` now uses fuzzy title matching instead of exact string comparison.
|
||||
- Added `_normalize_title()` to strip anime suffixes: `(TV)`, `(Anime)`, `(OAD)`, `(OVA)`, `(Special)`, `(Movie)`, `(Spin-Off)`
|
||||
- Added `_titles_match()` using `difflib.SequenceMatcher` with 0.85 similarity threshold for tolerance of minor title variations
|
||||
- Added debug logging for title mismatches and multiple search results
|
||||
|
||||
---
|
||||
|
||||
## [1.3.1] - 2026-02-22
|
||||
|
||||
### Added
|
||||
|
||||
- **Encoding detection for HTML parsing** (`src/core/providers/aniworld_provider.py`):
|
||||
Added `_decode_html_content()` function that uses `chardet` to detect the actual
|
||||
encoding of HTML content before parsing. Falls back to UTF-8 with `errors='replace'`
|
||||
to handle pages with mismatched encoding declarations. Applied to all BeautifulSoup
|
||||
parsing calls to prevent "Some characters could not be decoded" warnings.
|
||||
- **chardet dependency**: Added `chardet>=5.2.0` to `requirements.txt` for encoding detection.
|
||||
|
||||
### Added
|
||||
|
||||
- **Temp file cleanup after every download** (`src/core/providers/aniworld_provider.py`,
|
||||
`src/core/providers/enhanced_provider.py`): Module-level helper
|
||||
`_cleanup_temp_file()` removes the working temp file and any yt-dlp `.part`
|
||||
fragments after each download attempt — on success, on failure, and on
|
||||
exceptions (including `BrokenPipeError` and cancellation). Ensures that no
|
||||
partial files accumulate in `./Temp/` across multiple runs.
|
||||
- **Temp folder purge on server start** (`src/server/fastapi_app.py`): The
|
||||
FastAPI lifespan startup now iterates `./Temp/` and deletes every file and
|
||||
sub-directory before the rest of the initialisation sequence runs. If the
|
||||
folder does not exist it is created. Errors are caught and logged as warnings
|
||||
so that they never abort startup.
|
||||
|
||||
---
|
||||
|
||||
## [1.3.0] - 2026-02-22
|
||||
|
||||
### Added
|
||||
|
||||
- **NFO tag completeness (`nfo_mapper.py`)**: All 17 required NFO tags are now
|
||||
explicitly populated during creation: `originaltitle`, `sorttitle`, `year`,
|
||||
`plot`, `outline`, `tagline`, `runtime`, `premiered`, `status`, `imdbid`,
|
||||
`genre`, `studio`, `country`, `actor`, `watched`, `dateadded`, `mpaa`.
|
||||
- **`src/core/utils/nfo_mapper.py`**: New module containing
|
||||
`tmdb_to_nfo_model()`, `_extract_rating_by_country()`, and
|
||||
`_extract_fsk_rating()`. Extracted from `NFOService` to keep files under
|
||||
500 lines and isolate pure mapping logic.
|
||||
- **US MPAA rating**: `_extract_rating_by_country(ratings, "US")` now maps the
|
||||
US TMDB content rating to the `<mpaa>` NFO tag.
|
||||
- **`NfoRepairService` (`src/core/services/nfo_repair_service.py`)**: New service
|
||||
that detects incomplete `tvshow.nfo` files and triggers TMDB re-fetch.
|
||||
Provides `parse_nfo_tags()`, `find_missing_tags()`, `nfo_needs_repair()`, and
|
||||
`NfoRepairService.repair_series()`. 13 required tags are checked.
|
||||
- **`perform_nfo_repair_scan()`
|
||||
(`src/server/services/folder_scan_service.py`)**: New async function
|
||||
that iterates every series directory, checks whether `tvshow.nfo` is missing
|
||||
required tags using `nfo_needs_repair()`, and queues the series for background
|
||||
reload via `asyncio.create_task`. Skips gracefully when `tmdb_api_key` or
|
||||
`anime_directory` is not configured.
|
||||
- **NFO repair wired into scheduled folder scan (`src/server/services/folder_scan_service.py`)**:
|
||||
`perform_nfo_repair_scan(background_loader=None)` is called during the
|
||||
scheduled daily folder scan, keeping startup fast while ensuring regular
|
||||
maintenance.
|
||||
|
||||
### Changed
|
||||
|
||||
- `NFOService._tmdb_to_nfo_model()` and `NFOService._extract_fsk_rating()` moved
|
||||
to `src/core/utils/nfo_mapper.py` as module-level functions
|
||||
`tmdb_to_nfo_model()` and `_extract_fsk_rating()`.
|
||||
- `src/core/services/nfo_service.py` reduced from 640 → 471 lines.
|
||||
|
||||
---
|
||||
|
||||
## [Unreleased] - 2026-01-18
|
||||
|
||||
### Added
|
||||
|
||||
- **Cron-based Scheduler**: Replaced the asyncio sleep-loop with APScheduler's `AsyncIOScheduler + CronTrigger`
|
||||
- Schedule rescans at a specific **time of day** (`HH:MM`) on selected **days of the week**
|
||||
- New `SchedulerConfig` fields: `schedule_time` (default `"03:00"`), `schedule_days` (default all 7), `auto_download_after_rescan` (default `false`)
|
||||
- Old `interval_minutes` field retained for backward compatibility
|
||||
- **Auto-download after rescan**: When `auto_download_after_rescan` is enabled, missing episodes are automatically queued for download after each scheduled rescan
|
||||
- **Day-of-week UI**: New day-of-week pill toggles (Mon–Sun) in the Settings → Scheduler section
|
||||
- **Live config reload**: POST `/api/scheduler/config` reschedules the APScheduler job without restarting the application
|
||||
- **Enriched API response**: GET/POST `/api/scheduler/config` now returns `{"success", "config", "status"}` envelope including `next_run`, `last_run`, and `scan_in_progress`
|
||||
|
||||
### Changed
|
||||
|
||||
- Scheduler API response format: previously returned flat config; now returns `{"success": true, "config": {...}, "status": {...}}`
|
||||
- `reload_config()` is now a synchronous method accepting a `SchedulerConfig` argument (previously async, no arguments)
|
||||
- Dependencies: added `APScheduler>=3.10.4` to `requirements.txt`
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Series Visibility**: Fixed issue where series added to the database weren't appearing in the API/UI
|
||||
- Series are now loaded from database into SeriesApp's in-memory cache on startup
|
||||
- Added `_load_series_from_db()` call after initial database sync in FastAPI lifespan
|
||||
- **Episode Tracking**: Fixed missing episodes not being saved to database when adding new series
|
||||
- Missing episodes are now persisted to the `episodes` table after the targeted scan
|
||||
- Episodes are properly synced during rescan operations (added/removed based on filesystem state)
|
||||
- **Database Synchronization**: Improved data consistency between database and in-memory cache
|
||||
- Rescan process properly updates episodes: adds new missing episodes, removes downloaded ones
|
||||
- All series operations now maintain database and cache synchronization
|
||||
|
||||
### Technical Details
|
||||
|
||||
- Modified `src/server/fastapi_app.py` to load series from database after sync
|
||||
- Modified `src/server/api/anime.py` to save scanned episodes to database
|
||||
- Episodes table properly tracks missing episodes with automatic cleanup
|
||||
|
||||
### Deprecated
|
||||
|
||||
- **Legacy Series Files (key/data)**: File-based series storage is deprecated. `key` and `data` files in anime folders will be removed in v3.0.0. Database storage is now the primary method. See [docs/MIGRATION_GUIDE.md](docs/MIGRATION_GUIDE.md) for details.
|
||||
|
||||
---
|
||||
|
||||
## Sections for Each Release
|
||||
|
||||
```markdown
|
||||
## [Version] - YYYY-MM-DD
|
||||
|
||||
### Added
|
||||
|
||||
- New features
|
||||
|
||||
### Changed
|
||||
|
||||
- Changes to existing functionality
|
||||
|
||||
### Deprecated
|
||||
|
||||
- Features that will be removed in future versions
|
||||
|
||||
### Removed
|
||||
|
||||
- Features removed in this release
|
||||
|
||||
### Fixed
|
||||
|
||||
- Bug fixes
|
||||
|
||||
### Security
|
||||
|
||||
- Security-related fixes
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Unreleased
|
||||
|
||||
_Changes that are in development but not yet released._
|
||||
|
||||
### Added
|
||||
|
||||
- **Comprehensive Test Suite**: Created 1,070+ tests across 4 priority tiers
|
||||
- **TIER 1 (Critical)**: 159 tests - Scheduler, NFO batch operations, download queue, persistence
|
||||
- **TIER 2 (High Priority)**: 390 tests - JavaScript framework, dark mode, setup page, settings modal, WebSocket, queue UI
|
||||
- **TIER 3 (Medium Priority)**: 156 tests - WebSocket load, concurrent operations, retry logic, NFO performance, series parsing, TMDB integration
|
||||
- **TIER 4 (Polish)**: 426 tests - Internationalization (89), user preferences (68), accessibility (250+), media server compatibility (19)
|
||||
- **Frontend Testing Infrastructure**: Vitest for unit tests, Playwright for E2E tests
|
||||
- **Security Test Coverage**: Complete testing for authentication, authorization, CSRF, XSS, SQL injection
|
||||
- **Performance Validation**: WebSocket load (200+ concurrent clients), batch operations, concurrent access
|
||||
- **Accessibility Tests**: WCAG 2.1 AA compliance testing (keyboard navigation, ARIA labels, screen readers)
|
||||
- **Media Server Compatibility**: NFO format validation for Kodi, Plex, Jellyfin, and Emby
|
||||
|
||||
### Changed
|
||||
|
||||
- Updated testing documentation (TESTING_COMPLETE.md, instructions.md) to reflect 100% completion of all test tiers
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Enhanced Anime Add Flow**: Automatic database persistence, targeted episode scanning, and folder creation with sanitized names
|
||||
- Filesystem utility module (`src/server/utils/filesystem.py`) with `sanitize_folder_name()`, `is_safe_path()`, and `create_safe_folder()` functions
|
||||
- `Serie.sanitized_folder` property for generating filesystem-safe folder names from display names
|
||||
- `SerieScanner.scan_single_series()` method for targeted scanning of individual anime without full library rescan
|
||||
- Add series API response now includes `missing_episodes` list and `total_missing` count
|
||||
- Database transaction support with `@transactional` decorator and `atomic()` context manager
|
||||
- Transaction propagation modes (REQUIRED, REQUIRES_NEW, NESTED) for fine-grained control
|
||||
- Savepoint support for nested transactions with partial rollback capability
|
||||
- `TransactionManager` helper class for manual transaction control
|
||||
- Bulk operations: `bulk_mark_downloaded`, `bulk_delete`, `clear_all` for batch processing
|
||||
- `rotate_session` atomic operation for secure session rotation
|
||||
- Transaction utilities: `is_session_in_transaction`, `get_session_transaction_depth`
|
||||
- `get_transactional_session` for sessions without auto-commit
|
||||
|
||||
### Changed
|
||||
|
||||
- `QueueRepository.save_item()` now uses atomic transactions for data consistency
|
||||
- `QueueRepository.clear_all()` now uses atomic transactions for all-or-nothing behavior
|
||||
- Service layer documentation updated to reflect transaction-aware design
|
||||
|
||||
### Fixed
|
||||
|
||||
- Scan status indicator now correctly shows running state after page reload during active scan
|
||||
- Improved reliability of process status updates in the UI header
|
||||
|
||||
---
|
||||
|
||||
## Version History
|
||||
|
||||
_To be documented as versions are released._
|
||||
@@ -10,24 +10,43 @@ This document provides a comprehensive reference for all configuration options i
|
||||
|
||||
### Configuration Sources
|
||||
|
||||
Aniworld uses a layered configuration system:
|
||||
Aniworld uses a layered configuration system with **explicit precedence rules**:
|
||||
|
||||
1. **Environment Variables** (highest priority)
|
||||
2. **`.env` file** in project root
|
||||
3. **`data/config.json`** file
|
||||
4. **Default values** (lowest priority)
|
||||
1. **Environment Variables** (highest priority) - Takes precedence over all other sources
|
||||
2. **`.env` file** in project root - Loaded as environment variables
|
||||
3. **`data/config.json`** file - Persistent file-based configuration
|
||||
4. **Default values** (lowest priority) - Built-in fallback values
|
||||
|
||||
### Precedence Rules
|
||||
|
||||
**Critical Principle**: `ENV VARS > config.json > defaults`
|
||||
|
||||
- **Environment variables always win**: If a value is set via environment variable, it will NOT be overridden by config.json
|
||||
- **config.json as fallback**: If an ENV var is not set (or is empty/default), the value from config.json is used
|
||||
- **Defaults as last resort**: Built-in default values are used only if neither ENV var nor config.json provide a value
|
||||
|
||||
### Loading Mechanism
|
||||
|
||||
Configuration is loaded at application startup via Pydantic Settings.
|
||||
Configuration is loaded at application startup in `src/server/fastapi_app.py`:
|
||||
|
||||
```python
|
||||
# src/config/settings.py
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
|
||||
1. **Pydantic Settings** loads ENV vars and .env file with defaults
|
||||
2. **config.json** is loaded via `ConfigService`
|
||||
3. **Selective sync**: config.json values sync to settings **only if** ENV var not set
|
||||
4. **Runtime access**: Code uses `settings` object (which has final merged values)
|
||||
|
||||
**Example**:
|
||||
|
||||
```bash
|
||||
# If ENV var is set:
|
||||
ANIME_DIRECTORY=/env/path # This takes precedence
|
||||
|
||||
# config.json has:
|
||||
{"other": {"anime_directory": "/config/path"}} # This is ignored
|
||||
|
||||
# Result: settings.anime_directory = "/env/path"
|
||||
```
|
||||
|
||||
Source: [src/config/settings.py](../src/config/settings.py#L1-L96)
|
||||
**Source**: [src/config/settings.py](../src/config/settings.py#L1-L96), [src/server/fastapi_app.py](../src/server/fastapi_app.py#L139-L185)
|
||||
|
||||
---
|
||||
|
||||
@@ -67,6 +86,20 @@ Source: [src/config/settings.py](../src/config/settings.py#L43-L68)
|
||||
|
||||
Source: [src/config/settings.py](../src/config/settings.py#L69-L79)
|
||||
|
||||
### NFO Settings
|
||||
|
||||
| Variable | Type | Default | Description |
|
||||
| --------------------- | ------ | -------- | -------------------------------------------------- |
|
||||
| `TMDB_API_KEY` | string | `""` | The Movie Database (TMDB) API key for metadata. |
|
||||
| `NFO_AUTO_CREATE` | bool | `true` | Automatically create NFO files during downloads. |
|
||||
| `NFO_UPDATE_ON_SCAN` | bool | `false` | Update existing NFO files when scanning library. |
|
||||
| `NFO_DOWNLOAD_POSTER` | bool | `true` | Download poster images along with NFO files. |
|
||||
| `NFO_DOWNLOAD_LOGO` | bool | `false` | Download logo images along with NFO files. |
|
||||
| `NFO_DOWNLOAD_FANART` | bool | `false` | Download fanart images along with NFO files. |
|
||||
| `NFO_IMAGE_SIZE` | string | `"w500"` | Image size for TMDB images (w500, w780, original). |
|
||||
|
||||
Source: [src/server/models/config.py](../src/server/models/config.py#L109-L132)
|
||||
|
||||
---
|
||||
|
||||
## 3. Configuration File (config.json)
|
||||
@@ -81,7 +114,11 @@ Location: `data/config.json`
|
||||
"data_dir": "data",
|
||||
"scheduler": {
|
||||
"enabled": true,
|
||||
"interval_minutes": 60
|
||||
"interval_minutes": 60,
|
||||
"schedule_time": "03:00",
|
||||
"schedule_days": ["mon", "tue", "wed", "thu", "fri", "sat", "sun"],
|
||||
"auto_download_after_rescan": false,
|
||||
"folder_scan_enabled": false
|
||||
},
|
||||
"logging": {
|
||||
"level": "INFO",
|
||||
@@ -94,11 +131,20 @@ Location: `data/config.json`
|
||||
"path": "data/backups",
|
||||
"keep_days": 30
|
||||
},
|
||||
"nfo": {
|
||||
"tmdb_api_key": "",
|
||||
"auto_create": true,
|
||||
"update_on_scan": false,
|
||||
"download_poster": true,
|
||||
"download_logo": false,
|
||||
"download_fanart": false,
|
||||
"image_size": "w500"
|
||||
},
|
||||
"other": {
|
||||
"master_password_hash": "$pbkdf2-sha256$...",
|
||||
"anime_directory": "/path/to/anime"
|
||||
},
|
||||
"version": "1.0.0"
|
||||
"version": "1.0.1"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -119,12 +165,18 @@ Source: [src/server/models/config.py](../src/server/models/config.py#L62-L66)
|
||||
|
||||
### 4.2 Scheduler Settings
|
||||
|
||||
Controls automatic library rescanning.
|
||||
Controls automatic cron-based library rescanning (powered by APScheduler).
|
||||
|
||||
| Field | Type | Default | Description |
|
||||
| ---------------------------- | ---- | ------- | -------------------------------------------- |
|
||||
| `scheduler.enabled` | bool | `true` | Enable/disable automatic scans. |
|
||||
| `scheduler.interval_minutes` | int | `60` | Minutes between automatic scans. Minimum: 1. |
|
||||
| Field | Type | Default | Description |
|
||||
| -------------------------------------- | ------------ | --------------------------------------------- | -------------------------------------------------------------------- |
|
||||
| `scheduler.enabled` | bool | `true` | Enable/disable automatic scans. |
|
||||
| `scheduler.interval_minutes` | int | `60` | Legacy field kept for backward compatibility. Minimum: 1. |
|
||||
| `scheduler.schedule_time` | string | `"03:00"` | Daily run time in 24-h `HH:MM` format. |
|
||||
| `scheduler.schedule_days` | list[string] | `["mon","tue","wed","thu","fri","sat","sun"]` | Days of the week to run the scan. Empty list disables the cron job. |
|
||||
| `scheduler.auto_download_after_rescan` | bool | `false` | Automatically queue missing episodes for download after each rescan. |
|
||||
| `scheduler.folder_scan_enabled` | bool | `false` | Run folder maintenance (NFO repair, folder renaming, poster checks) during scheduled runs. **When enabled, series folders are automatically renamed to match the `<title> (<year>)` convention derived from their `tvshow.nfo` files.** |
|
||||
|
||||
Valid day abbreviations: `mon`, `tue`, `wed`, `thu`, `fri`, `sat`, `sun`.
|
||||
|
||||
Source: [src/server/models/config.py](../src/server/models/config.py#L5-L12)
|
||||
|
||||
@@ -149,7 +201,31 @@ Source: [src/server/models/config.py](../src/server/models/config.py#L27-L46)
|
||||
|
||||
Source: [src/server/models/config.py](../src/server/models/config.py#L15-L24)
|
||||
|
||||
### 4.5 Other Settings (Dynamic)
|
||||
### 4.5 NFO Settings
|
||||
|
||||
| Field | Type | Default | Description |
|
||||
| --------------------- | ------ | -------- | ------------------------------------------------------------- |
|
||||
| `nfo.tmdb_api_key` | string | `""` | The Movie Database (TMDB) API key for fetching metadata. |
|
||||
| `nfo.auto_create` | bool | `true` | Automatically create NFO files when downloading episodes. |
|
||||
| `nfo.update_on_scan` | bool | `false` | Update existing NFO files during library scan operations. |
|
||||
| `nfo.download_poster` | bool | `true` | Download poster images (poster.jpg) along with NFO files. |
|
||||
| `nfo.download_logo` | bool | `false` | Download logo images (logo.png) along with NFO files. |
|
||||
| `nfo.download_fanart` | bool | `false` | Download fanart images (fanart.jpg) along with NFO files. |
|
||||
| `nfo.image_size` | string | `"w500"` | TMDB image size: `w500` (recommended), `w780`, or `original`. |
|
||||
|
||||
**Notes:**
|
||||
|
||||
- Obtain a TMDB API key from https://www.themoviedb.org/settings/api
|
||||
- `auto_create` creates NFO files during the download process
|
||||
- `update_on_scan` refreshes metadata when scanning existing anime
|
||||
- `download_poster` also controls whether the scheduled folder scan checks for and re-downloads missing or corrupted `poster.jpg` files (see [NFO_GUIDE.md](NFO_GUIDE.md#6-poster-check))
|
||||
- Image downloads require valid `tmdb_api_key`
|
||||
- `TMDB_API_KEY` environment variable is optional when `nfo.tmdb_api_key` is configured in `data/config.json`
|
||||
- Larger image sizes (`w780`, `original`) consume more storage space
|
||||
|
||||
Source: [src/server/models/config.py](../src/server/models/config.py#L109-L132)
|
||||
|
||||
### 4.6 Other Settings (Dynamic)
|
||||
|
||||
The `other` field stores arbitrary settings.
|
||||
|
||||
@@ -178,11 +254,11 @@ Settings are resolved in this order (first match wins):
|
||||
|
||||
Master password must meet all criteria:
|
||||
|
||||
- Minimum 8 characters
|
||||
- At least one uppercase letter
|
||||
- At least one lowercase letter
|
||||
- At least one digit
|
||||
- At least one special character
|
||||
- Minimum 8 characters
|
||||
- At least one uppercase letter
|
||||
- At least one lowercase letter
|
||||
- At least one digit
|
||||
- At least one special character
|
||||
|
||||
Source: [src/server/services/auth_service.py](../src/server/services/auth_service.py#L97-L125)
|
||||
|
||||
@@ -293,6 +369,6 @@ Source: [src/server/api/config.py](../src/server/api/config.py#L67-L142)
|
||||
|
||||
## 10. Related Documentation
|
||||
|
||||
- [API.md](API.md) - Configuration API endpoints
|
||||
- [DEVELOPMENT.md](DEVELOPMENT.md) - Development environment setup
|
||||
- [ARCHITECTURE.md](ARCHITECTURE.md) - Configuration service architecture
|
||||
- [API.md](API.md) - Configuration API endpoints
|
||||
- [DEVELOPMENT.md](DEVELOPMENT.md) - Development environment setup
|
||||
- [ARCHITECTURE.md](ARCHITECTURE.md) - Configuration service architecture
|
||||
642
Docs/DATABASE.md
Normal file
642
Docs/DATABASE.md
Normal file
@@ -0,0 +1,642 @@
|
||||
# Database Documentation
|
||||
|
||||
## Document Purpose
|
||||
|
||||
This document describes the database schema, models, and data layer of the Aniworld application.
|
||||
|
||||
---
|
||||
|
||||
## 1. Database Overview
|
||||
|
||||
### Technology
|
||||
|
||||
- **Database Engine**: SQLite 3 (default), PostgreSQL supported
|
||||
- **ORM**: SQLAlchemy 2.0 with async support (aiosqlite)
|
||||
- **Location**: `data/aniworld.db` (configurable via `DATABASE_URL`)
|
||||
|
||||
Source: [src/config/settings.py](../src/config/settings.py#L53-L55)
|
||||
|
||||
### Connection Configuration
|
||||
|
||||
```python
|
||||
# Default connection string
|
||||
DATABASE_URL = "sqlite+aiosqlite:///./data/aniworld.db"
|
||||
|
||||
# PostgreSQL alternative
|
||||
DATABASE_URL = "postgresql+asyncpg://user:pass@localhost/aniworld"
|
||||
```
|
||||
|
||||
Source: [src/server/database/connection.py](../src/server/database/connection.py)
|
||||
|
||||
---
|
||||
|
||||
## 2. Entity Relationship Diagram
|
||||
|
||||
```
|
||||
+---------------------+ +-------------------+ +-------------------+ +------------------------+
|
||||
| system_settings | | anime_series | | episodes | | download_queue_item |
|
||||
+---------------------+ +-------------------+ +-------------------+ +------------------------+
|
||||
| id (PK) | | id (PK) |<--+ | id (PK) | +-->| id (PK, VARCHAR) |
|
||||
| initial_scan_... | | key (UNIQUE) | | | series_id (FK)----+---+ | series_id (FK)---------+
|
||||
| initial_nfo_scan... | | name | +---| | | status |
|
||||
| initial_media_... | | site | | season | | priority |
|
||||
| last_scan_timestamp | | folder | | episode_number | | season |
|
||||
| created_at | | created_at | | title | | episode |
|
||||
| updated_at | | updated_at | | file_path | | progress_percent |
|
||||
+---------------------+ +-------------------+ | is_downloaded | | error_message |
|
||||
| created_at | | retry_count |
|
||||
| updated_at | | added_at |
|
||||
+-------------------+ | started_at |
|
||||
| completed_at |
|
||||
| created_at |
|
||||
| updated_at |
|
||||
+------------------------+
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Table Schemas
|
||||
|
||||
### 3.1 system_settings
|
||||
|
||||
Stores application-wide system settings and initialization state.
|
||||
|
||||
| Column | Type | Constraints | Description |
|
||||
| ------------------------------ | -------- | -------------------------- | --------------------------------------------- |
|
||||
| `id` | INTEGER | PRIMARY KEY, AUTOINCREMENT | Internal database ID (only one row) |
|
||||
| `initial_scan_completed` | BOOLEAN | NOT NULL, DEFAULT FALSE | Whether initial anime folder scan is complete |
|
||||
| `initial_nfo_scan_completed` | BOOLEAN | NOT NULL, DEFAULT FALSE | Whether initial NFO scan is complete |
|
||||
| `initial_media_scan_completed` | BOOLEAN | NOT NULL, DEFAULT FALSE | Whether initial media scan is complete |
|
||||
| `last_scan_timestamp` | DATETIME | NULLABLE | Timestamp of last completed scan |
|
||||
| `created_at` | DATETIME | NOT NULL, DEFAULT NOW | Record creation timestamp |
|
||||
| `updated_at` | DATETIME | NOT NULL, ON UPDATE NOW | Last update timestamp |
|
||||
|
||||
**Purpose:**
|
||||
|
||||
This table tracks the initialization status of the application to ensure that expensive one-time setup operations (like scanning the entire anime directory) only run on the first startup, not on every restart.
|
||||
|
||||
- Only one row exists in this table
|
||||
- The `initial_scan_completed` flag prevents redundant full directory scans on each startup
|
||||
- The NFO and media scan flags similarly track completion of those setup tasks
|
||||
|
||||
Source: [src/server/database/models.py](../src/server/database/models.py), [src/server/database/system_settings_service.py](../src/server/database/system_settings_service.py)
|
||||
|
||||
### 3.2 anime_series
|
||||
|
||||
Stores anime series metadata. Corresponds to the core `Serie` class.
|
||||
|
||||
| Column | Type | Constraints | Description |
|
||||
| ---------------- | ------------- | -------------------------- | ------------------------------------------------------- |
|
||||
| `id` | INTEGER | PRIMARY KEY, AUTOINCREMENT | Internal database ID |
|
||||
| `key` | VARCHAR(255) | UNIQUE, NOT NULL, INDEX | **Primary identifier** - provider-assigned URL-safe key |
|
||||
| `name` | VARCHAR(500) | NOT NULL, INDEX | Display name of the series |
|
||||
| `site` | VARCHAR(500) | NOT NULL | Provider site URL |
|
||||
| `folder` | VARCHAR(1000) | NOT NULL | Filesystem folder name (metadata only) |
|
||||
| `year` | INTEGER | NULLABLE | Release year of the series |
|
||||
| `nfo_path` | VARCHAR(1000) | NULLABLE | Path to tvshow.nfo metadata file |
|
||||
| `tmdb_id` | INTEGER | NULLABLE, INDEX | TMDB (The Movie Database) ID for metadata |
|
||||
| `tvdb_id` | INTEGER | NULLABLE, INDEX | TVDB (TheTVDB) ID for metadata |
|
||||
| `has_nfo` | BOOLEAN | NOT NULL, DEFAULT FALSE | Whether tvshow.nfo exists |
|
||||
| `loading_status` | VARCHAR(50) | NOT NULL, DEFAULT 'completed' | Status: pending, loading_episodes, loading_nfo, completed, failed |
|
||||
| `created_at` | DATETIME | NOT NULL, DEFAULT NOW | Record creation timestamp |
|
||||
| `updated_at` | DATETIME | NOT NULL, ON UPDATE NOW | Last update timestamp |
|
||||
|
||||
**Identifier Convention:**
|
||||
|
||||
- `key` is the **primary identifier** for all operations (e.g., `"attack-on-titan"`)
|
||||
- `folder` is **metadata only** for filesystem operations (e.g., `"Attack on Titan (2013)"`)
|
||||
- `id` is used only for database relationships
|
||||
|
||||
**EpisodeDict Mapping:**
|
||||
|
||||
The `episodeDict` (season → episode numbers mapping) is stored as individual `Episode` records:
|
||||
- Each `Episode` has `season` and `episode_number` columns
|
||||
- Relationship: `AnimeSeries.episodes` returns all Episode records for that series
|
||||
|
||||
Source: [src/server/database/models.py](../src/server/database/models.py#L23-L150)
|
||||
|
||||
### 3.3 episodes
|
||||
|
||||
Stores **missing episodes** that need to be downloaded. Episodes are automatically managed during scans:
|
||||
|
||||
- New missing episodes are added to the database
|
||||
- Episodes that are no longer missing (files now exist) are removed from the database
|
||||
- When an episode is downloaded, it can be marked with `is_downloaded=True` or removed from tracking
|
||||
|
||||
| Column | Type | Constraints | Description |
|
||||
| ---------------- | ------------- | ---------------------------- | ----------------------------- |
|
||||
| `id` | INTEGER | PRIMARY KEY, AUTOINCREMENT | Internal database ID |
|
||||
| `series_id` | INTEGER | FOREIGN KEY, NOT NULL, INDEX | Reference to anime_series.id |
|
||||
| `season` | INTEGER | NOT NULL | Season number (1-based) |
|
||||
| `episode_number` | INTEGER | NOT NULL | Episode number within season |
|
||||
| `title` | VARCHAR(500) | NULLABLE | Episode title if known |
|
||||
| `file_path` | VARCHAR(1000) | NULLABLE | Local file path if downloaded |
|
||||
| `is_downloaded` | BOOLEAN | NOT NULL, DEFAULT FALSE | Download status flag |
|
||||
| `created_at` | DATETIME | NOT NULL, DEFAULT NOW | Record creation timestamp |
|
||||
| `updated_at` | DATETIME | NOT NULL, ON UPDATE NOW | Last update timestamp |
|
||||
|
||||
**Foreign Key:**
|
||||
|
||||
- `series_id` -> `anime_series.id` (ON DELETE CASCADE)
|
||||
|
||||
Source: [src/server/database/models.py](../src/server/database/models.py#L122-L181)
|
||||
|
||||
### 3.4 download_queue_item
|
||||
|
||||
Stores download queue items with status tracking.
|
||||
|
||||
| Column | Type | Constraints | Description |
|
||||
| ------------------ | ------------- | --------------------------- | ------------------------------ |
|
||||
| `id` | VARCHAR(36) | PRIMARY KEY | UUID identifier |
|
||||
| `series_id` | INTEGER | FOREIGN KEY, NOT NULL | Reference to anime_series.id |
|
||||
| `season` | INTEGER | NOT NULL | Season number |
|
||||
| `episode` | INTEGER | NOT NULL | Episode number |
|
||||
| `status` | VARCHAR(20) | NOT NULL, DEFAULT 'pending' | Download status |
|
||||
| `priority` | VARCHAR(10) | NOT NULL, DEFAULT 'NORMAL' | Queue priority |
|
||||
| `progress_percent` | FLOAT | NULLABLE | Download progress (0-100) |
|
||||
| `error_message` | TEXT | NULLABLE | Error description if failed |
|
||||
| `retry_count` | INTEGER | NOT NULL, DEFAULT 0 | Number of retry attempts |
|
||||
| `source_url` | VARCHAR(2000) | NULLABLE | Download source URL |
|
||||
| `added_at` | DATETIME | NOT NULL, DEFAULT NOW | When added to queue |
|
||||
| `started_at` | DATETIME | NULLABLE | When download started |
|
||||
| `completed_at` | DATETIME | NULLABLE | When download completed/failed |
|
||||
| `created_at` | DATETIME | NOT NULL, DEFAULT NOW | Record creation timestamp |
|
||||
| `updated_at` | DATETIME | NOT NULL, ON UPDATE NOW | Last update timestamp |
|
||||
|
||||
**Status Values:** `pending`, `downloading`, `paused`, `completed`, `failed`, `cancelled`
|
||||
|
||||
**Priority Values:** `LOW`, `NORMAL`, `HIGH`
|
||||
|
||||
**Foreign Key:**
|
||||
|
||||
- `series_id` -> `anime_series.id` (ON DELETE CASCADE)
|
||||
|
||||
Source: [src/server/database/models.py](../src/server/database/models.py#L200-L300)
|
||||
|
||||
---
|
||||
|
||||
## 4. Indexes
|
||||
|
||||
| Table | Index Name | Columns | Purpose |
|
||||
| --------------------- | ----------------------- | ----------- | --------------------------------- |
|
||||
| `system_settings` | N/A (single row) | N/A | Only one row, no indexes needed |
|
||||
| `anime_series` | `ix_anime_series_key` | `key` | Fast lookup by primary identifier |
|
||||
| `anime_series` | `ix_anime_series_name` | `name` | Search by name |
|
||||
| `episodes` | `ix_episodes_series_id` | `series_id` | Join with series |
|
||||
| `download_queue_item` | `ix_download_series_id` | `series_id` | Filter by series |
|
||||
| `download_queue_item` | `ix_download_status` | `status` | Filter by status |
|
||||
|
||||
---
|
||||
|
||||
## 5. Model Layer
|
||||
|
||||
### 5.1 SQLAlchemy ORM Models
|
||||
|
||||
```python
|
||||
# src/server/database/models.py
|
||||
|
||||
class AnimeSeries(Base, TimestampMixin):
|
||||
__tablename__ = "anime_series"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
key: Mapped[str] = mapped_column(String(255), unique=True, index=True)
|
||||
name: Mapped[str] = mapped_column(String(500), index=True)
|
||||
site: Mapped[str] = mapped_column(String(500))
|
||||
folder: Mapped[str] = mapped_column(String(1000))
|
||||
|
||||
episodes: Mapped[List["Episode"]] = relationship(
|
||||
"Episode", back_populates="series", cascade="all, delete-orphan"
|
||||
)
|
||||
```
|
||||
|
||||
Source: [src/server/database/models.py](../src/server/database/models.py#L23-L87)
|
||||
|
||||
### 5.2 Pydantic API Models
|
||||
|
||||
```python
|
||||
# src/server/models/download.py
|
||||
|
||||
class DownloadItem(BaseModel):
|
||||
id: str
|
||||
serie_id: str # Maps to anime_series.key
|
||||
serie_folder: str # Metadata only
|
||||
serie_name: str
|
||||
episode: EpisodeIdentifier
|
||||
status: DownloadStatus
|
||||
priority: DownloadPriority
|
||||
```
|
||||
|
||||
Source: [src/server/models/download.py](../src/server/models/download.py#L63-L118)
|
||||
|
||||
### 5.3 Model Mapping
|
||||
|
||||
| API Field | Database Column | Notes |
|
||||
| -------------- | --------------------- | ------------------ |
|
||||
| `serie_id` | `anime_series.key` | Primary identifier |
|
||||
| `serie_folder` | `anime_series.folder` | Metadata only |
|
||||
| `serie_name` | `anime_series.name` | Display name |
|
||||
|
||||
---
|
||||
|
||||
## 6. Transaction Support
|
||||
|
||||
### 6.1 Overview
|
||||
|
||||
The database layer provides comprehensive transaction support to ensure data consistency across compound operations. All write operations can be wrapped in explicit transactions.
|
||||
|
||||
Source: [src/server/database/transaction.py](../src/server/database/transaction.py)
|
||||
|
||||
### 6.2 Transaction Utilities
|
||||
|
||||
| Component | Type | Description |
|
||||
| ------------------------- | ----------------- | ---------------------------------------- |
|
||||
| `@transactional` | Decorator | Wraps function in transaction boundary |
|
||||
| `atomic()` | Async context mgr | Provides atomic operation block |
|
||||
| `atomic_sync()` | Sync context mgr | Sync version of atomic() |
|
||||
| `TransactionContext` | Class | Explicit sync transaction control |
|
||||
| `AsyncTransactionContext` | Class | Explicit async transaction control |
|
||||
| `TransactionManager` | Class | Helper for manual transaction management |
|
||||
|
||||
### 6.3 Transaction Propagation Modes
|
||||
|
||||
| Mode | Behavior |
|
||||
| -------------- | ------------------------------------------------ |
|
||||
| `REQUIRED` | Use existing transaction or create new (default) |
|
||||
| `REQUIRES_NEW` | Always create new transaction |
|
||||
| `NESTED` | Create savepoint within existing transaction |
|
||||
|
||||
### 6.4 Usage Examples
|
||||
|
||||
**Using @transactional decorator:**
|
||||
|
||||
```python
|
||||
from src.server.database.transaction import transactional
|
||||
|
||||
@transactional()
|
||||
async def compound_operation(db: AsyncSession, data: dict):
|
||||
# All operations commit together or rollback on error
|
||||
series = await AnimeSeriesService.create(db, ...)
|
||||
episode = await EpisodeService.create(db, series_id=series.id, ...)
|
||||
return series, episode
|
||||
```
|
||||
|
||||
**Using atomic() context manager:**
|
||||
|
||||
```python
|
||||
from src.server.database.transaction import atomic
|
||||
|
||||
async def some_function(db: AsyncSession):
|
||||
async with atomic(db) as tx:
|
||||
await operation1(db)
|
||||
await operation2(db)
|
||||
# Auto-commits on success, rolls back on exception
|
||||
```
|
||||
|
||||
**Using savepoints for partial rollback:**
|
||||
|
||||
```python
|
||||
async with atomic(db) as tx:
|
||||
await outer_operation(db)
|
||||
|
||||
async with tx.savepoint() as sp:
|
||||
await risky_operation(db)
|
||||
if error_condition:
|
||||
await sp.rollback() # Only rollback nested ops
|
||||
|
||||
await final_operation(db) # Still executes
|
||||
```
|
||||
|
||||
Source: [src/server/database/transaction.py](../src/server/database/transaction.py)
|
||||
|
||||
### 6.5 Connection Module Additions
|
||||
|
||||
| Function | Description |
|
||||
| ------------------------------- | -------------------------------------------- |
|
||||
| `get_transactional_session` | Session without auto-commit for transactions |
|
||||
| `TransactionManager` | Helper class for manual transaction control |
|
||||
| `is_session_in_transaction` | Check if session is in active transaction |
|
||||
| `get_session_transaction_depth` | Get nesting depth of transactions |
|
||||
|
||||
Source: [src/server/database/connection.py](../src/server/database/connection.py)
|
||||
|
||||
---
|
||||
|
||||
## 7. Repository Pattern
|
||||
|
||||
The `QueueRepository` class provides data access abstraction.
|
||||
|
||||
```python
|
||||
class QueueRepository:
|
||||
async def save_item(self, item: DownloadItem) -> None:
|
||||
"""Save or update a download item (atomic operation)."""
|
||||
|
||||
async def get_all_items(self) -> List[DownloadItem]:
|
||||
"""Get all items from database."""
|
||||
|
||||
async def delete_item(self, item_id: str) -> bool:
|
||||
"""Delete item by ID."""
|
||||
|
||||
async def clear_all(self) -> int:
|
||||
"""Clear all items (atomic operation)."""
|
||||
```
|
||||
|
||||
Note: Compound operations (`save_item`, `clear_all`) are wrapped in `atomic()` transactions.
|
||||
|
||||
Source: [src/server/services/queue_repository.py](../src/server/services/queue_repository.py)
|
||||
|
||||
---
|
||||
|
||||
## 8. Database Service
|
||||
|
||||
The `AnimeSeriesService` provides async CRUD operations.
|
||||
|
||||
```python
|
||||
class AnimeSeriesService:
|
||||
@staticmethod
|
||||
async def create(
|
||||
db: AsyncSession,
|
||||
key: str,
|
||||
name: str,
|
||||
site: str,
|
||||
folder: str
|
||||
) -> AnimeSeries:
|
||||
"""Create a new anime series."""
|
||||
|
||||
@staticmethod
|
||||
async def get_by_key(
|
||||
db: AsyncSession,
|
||||
key: str
|
||||
) -> Optional[AnimeSeries]:
|
||||
"""Get series by primary key identifier."""
|
||||
```
|
||||
|
||||
### Bulk Operations
|
||||
|
||||
Services provide bulk operations for transaction-safe batch processing:
|
||||
|
||||
| Service | Method | Description |
|
||||
| ---------------------- | ---------------------- | ------------------------------ |
|
||||
| `EpisodeService` | `bulk_mark_downloaded` | Mark multiple episodes at once |
|
||||
| `DownloadQueueService` | `bulk_delete` | Delete multiple queue items |
|
||||
| `DownloadQueueService` | `clear_all` | Clear entire queue |
|
||||
| `UserSessionService` | `rotate_session` | Revoke old + create new atomic |
|
||||
| `UserSessionService` | `cleanup_expired` | Bulk delete expired sessions |
|
||||
|
||||
Source: [src/server/database/service.py](../src/server/database/service.py)
|
||||
|
||||
---
|
||||
|
||||
## 9. Data Integrity Rules
|
||||
|
||||
### Validation Constraints
|
||||
|
||||
| Field | Rule | Error Message |
|
||||
| ------------------------- | ------------------------ | ------------------------------------- |
|
||||
| `anime_series.key` | Non-empty, max 255 chars | "Series key cannot be empty" |
|
||||
| `anime_series.name` | Non-empty, max 500 chars | "Series name cannot be empty" |
|
||||
| `episodes.season` | 0-1000 | "Season number must be non-negative" |
|
||||
| `episodes.episode_number` | 0-10000 | "Episode number must be non-negative" |
|
||||
|
||||
Source: [src/server/database/models.py](../src/server/database/models.py#L89-L119)
|
||||
|
||||
### Cascade Rules
|
||||
|
||||
- Deleting `anime_series` deletes all related `episodes` and `download_queue_item`
|
||||
|
||||
---
|
||||
|
||||
## 10. Migration Strategy
|
||||
|
||||
Currently, SQLAlchemy's `create_all()` is used for schema creation.
|
||||
|
||||
```python
|
||||
# src/server/database/connection.py
|
||||
async def init_db():
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
```
|
||||
|
||||
For production migrations, Alembic is recommended but not yet implemented.
|
||||
|
||||
Source: [src/server/database/connection.py](../src/server/database/connection.py)
|
||||
|
||||
---
|
||||
|
||||
## 11. Common Query Patterns
|
||||
|
||||
### Get all series with missing episodes
|
||||
|
||||
```python
|
||||
series = await db.execute(
|
||||
select(AnimeSeries).options(selectinload(AnimeSeries.episodes))
|
||||
)
|
||||
for serie in series.scalars():
|
||||
downloaded = [e for e in serie.episodes if e.is_downloaded]
|
||||
```
|
||||
|
||||
### Get pending downloads ordered by priority
|
||||
|
||||
```python
|
||||
items = await db.execute(
|
||||
select(DownloadQueueItem)
|
||||
.where(DownloadQueueItem.status == "pending")
|
||||
.order_by(
|
||||
case(
|
||||
(DownloadQueueItem.priority == "HIGH", 1),
|
||||
(DownloadQueueItem.priority == "NORMAL", 2),
|
||||
(DownloadQueueItem.priority == "LOW", 3),
|
||||
),
|
||||
DownloadQueueItem.added_at
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 12. Series Storage: Database vs Files (Deprecated)
|
||||
|
||||
### File-Based Storage (Removed in v2.0)
|
||||
|
||||
Prior to v2.0, series metadata was stored in two files per anime folder:
|
||||
|
||||
| File | Contents |
|
||||
| -------- | ------------------------------------------------------- |
|
||||
| `key` | Series provider key (e.g., `"attack-on-titan"`) |
|
||||
| `data` | JSON serialization of `Serie` object |
|
||||
|
||||
File structure example:
|
||||
```
|
||||
/anime/Attack on Titan (2013)/
|
||||
├── key # Contains: attack-on-titan
|
||||
├── data # Contains: {"key": "...", "name": "...", "episodeDict": {...}}
|
||||
├── Season 1/
|
||||
│ └── ...
|
||||
```
|
||||
|
||||
### Database Storage (Current)
|
||||
|
||||
Since v2.0, all series metadata is stored in the `anime_series` table with `Episode` records for episode tracking. This provides:
|
||||
|
||||
- **ACID transactions** for data consistency
|
||||
- **Foreign key constraints** (cascade delete)
|
||||
- **Indexed queries** for fast lookups
|
||||
- **No filesystem dependency** for metadata
|
||||
|
||||
### Migration from Files to Database
|
||||
|
||||
The `Serie.save_to_file()` and `Serie.load_from_file()` methods are deprecated but still functional for backward compatibility during migration:
|
||||
|
||||
```python
|
||||
from src.core.entities.series import Serie
|
||||
|
||||
# Old file-based loading (deprecated)
|
||||
serie = Serie.load_from_file("/anime/Attack on Titan (2013)/data")
|
||||
|
||||
# New database-based loading
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
serie = await AnimeSeriesService.get_by_key(db, "attack-on-titan")
|
||||
```
|
||||
|
||||
### Removing File Dependencies
|
||||
|
||||
After verifying database schema supports all fields, file-based storage can be removed:
|
||||
|
||||
1. ✅ Schema verified: All `Serie` fields have corresponding DB columns
|
||||
2. ✅ Migration complete: All existing series migrated to database
|
||||
3. ❌ File cleanup: Remove `key` and `data` files (pending)
|
||||
|
||||
**Note:** The `save_to_file()` and `load_from_file()` methods will be removed in v3.0.0.
|
||||
|
||||
---
|
||||
|
||||
## 12. Series Persistence Flow
|
||||
|
||||
When a directory scan discovers or updates series, the scanner persists data to the database instead of writing to disk files.
|
||||
|
||||
### Scan Flow
|
||||
|
||||
```
|
||||
Scan Directory
|
||||
│
|
||||
▼
|
||||
Find MP4 Files → Extract Serie Key
|
||||
│
|
||||
▼
|
||||
Check DB for Existing Series (by key)
|
||||
│
|
||||
├─── EXISTS ──────────────────────► Update Series Metadata
|
||||
│ │
|
||||
│ ▼
|
||||
│ Sync Episodes to DB
|
||||
│ │
|
||||
│◄──────────────────────────────────────┘
|
||||
│
|
||||
└─── NEW ───────────────────────────► Create New Series Record
|
||||
│
|
||||
▼
|
||||
Create Episode Records
|
||||
│
|
||||
▼
|
||||
Return to Scan Loop
|
||||
```
|
||||
|
||||
### Key Methods
|
||||
|
||||
**SerieScanner._persist_serie_to_db()**
|
||||
- Called after `get_missing_episodes_and_season()` computes episodeDict
|
||||
- Uses `AnimeSeriesService.get_by_key()` to check if series exists
|
||||
- If exists: calls `AnimeSeriesService.update()` + `_sync_episodes_to_db()`
|
||||
- If new: calls `AnimeSeriesService.create()` + creates episodes
|
||||
|
||||
**SerieScanner._sync_episodes_to_db()**
|
||||
- Gets existing episodes from DB via `EpisodeService.get_by_series()`
|
||||
- Compares with new episodeDict
|
||||
- Removes episodes no longer missing (unless `is_downloaded=True`)
|
||||
- Adds new missing episodes
|
||||
- Preserves `is_downloaded=True` episodes when removing missing ones
|
||||
|
||||
**SerieList.add_to_db()**
|
||||
- Used when adding a new discovered series via API
|
||||
- Creates filesystem folder + database record + episode records
|
||||
|
||||
### Episode Sync Logic
|
||||
|
||||
```python
|
||||
# For each episode in DB but not in new episodeDict:
|
||||
if episode.is_downloaded:
|
||||
# Keep - file exists, don't remove
|
||||
pass
|
||||
else:
|
||||
# Remove - no longer missing
|
||||
EpisodeService.delete()
|
||||
|
||||
# For each episode in new episodeDict but not in DB:
|
||||
# Add as new missing episode
|
||||
EpisodeService.create(is_downloaded=False)
|
||||
```
|
||||
|
||||
### Transaction Handling
|
||||
|
||||
- DB operations use their own session with commit/rollback
|
||||
- If DB write fails, error is logged and scan continues
|
||||
- File-based `save_to_file()` no longer called during scan
|
||||
|
||||
### Migration Path
|
||||
|
||||
1. v2.x: Scanner writes to both DB (primary) and files (fallback)
|
||||
2. v3.0: Scanner writes only to DB, file methods removed
|
||||
|
||||
---
|
||||
|
||||
## 13. Series Persistence
|
||||
|
||||
### Schema
|
||||
|
||||
**AnimeSeries Table**: Stores series metadata (key, name, site, folder, year)
|
||||
|
||||
| Column | Type | Constraints | Description |
|
||||
|-----------|--------------|---------------------------|----------------------|
|
||||
| `id` | INTEGER | PRIMARY KEY | Auto-increment |
|
||||
| `key` | VARCHAR(255) | UNIQUE, NOT NULL | Series provider key |
|
||||
| `name` | VARCHAR(500) | NOT NULL | Display name |
|
||||
| `site` | VARCHAR(500) | | Provider site URL |
|
||||
| `folder` | VARCHAR(1000)| | Filesystem folder |
|
||||
|
||||
**Episode Table**: Stores per-episode metadata (season, episode_number, is_downloaded)
|
||||
|
||||
| Column | Type | Constraints | Description |
|
||||
|-----------------|--------------|---------------------------|----------------------|
|
||||
| `id` | INTEGER | PRIMARY KEY | Auto-increment |
|
||||
| `series_id` | INTEGER | FOREIGN KEY → anime_series| Parent series |
|
||||
| `season` | INTEGER | NOT NULL | Season number |
|
||||
| `episode_number`| INTEGER | NOT NULL | Episode number |
|
||||
| `is_downloaded` | BOOLEAN | DEFAULT FALSE | Download status |
|
||||
|
||||
### Relationships
|
||||
|
||||
- `AnimeSeries.episodes` → List of Episode objects (one-to-many)
|
||||
- `Episode.series` → Parent AnimeSeries (many-to-one)
|
||||
- Cascade delete: Deleting a series removes all its episodes
|
||||
|
||||
### Queries
|
||||
|
||||
```python
|
||||
# Get all series with episodes
|
||||
AnimeSeriesService.get_all(db, with_episodes=True)
|
||||
|
||||
# Get by provider key
|
||||
AnimeSeriesService.get_by_key(db, key)
|
||||
|
||||
# Get by folder path
|
||||
AnimeSeriesService.get_by_folder(db, folder)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 14. Database Location
|
||||
|
||||
| Environment | Default Location |
|
||||
| ----------- | ------------------------------------------------- |
|
||||
| Development | `./data/aniworld.db` |
|
||||
| Production | Via `DATABASE_URL` environment variable |
|
||||
| Testing | In-memory SQLite (`sqlite+aiosqlite:///:memory:`) |
|
||||
436
Docs/DEVELOPMENT.md
Normal file
436
Docs/DEVELOPMENT.md
Normal file
@@ -0,0 +1,436 @@
|
||||
# Development Guide
|
||||
|
||||
## Document Purpose
|
||||
|
||||
This document provides guidance for developers working on the Aniworld project.
|
||||
|
||||
### What This Document Contains
|
||||
|
||||
- **Prerequisites**: Required software and tools
|
||||
- **Environment Setup**: Step-by-step local development setup
|
||||
- **Project Structure**: Source code organization explanation
|
||||
- **Development Workflow**: Branch strategy, commit conventions
|
||||
- **Coding Standards**: Style guide, linting, formatting
|
||||
- **Running the Application**: Development server, CLI usage
|
||||
- **Debugging Tips**: Common debugging approaches
|
||||
- **IDE Configuration**: VS Code settings, recommended extensions
|
||||
- **Contributing Guidelines**: How to submit changes
|
||||
- **Code Review Process**: Review checklist and expectations
|
||||
|
||||
### What This Document Does NOT Contain
|
||||
|
||||
- Production deployment (see [DEPLOYMENT.md](DEPLOYMENT.md))
|
||||
- API reference (see [API.md](API.md))
|
||||
- Architecture decisions (see [ARCHITECTURE.md](ARCHITECTURE.md))
|
||||
- Test writing guides (see [TESTING.md](TESTING.md))
|
||||
- Security guidelines (see [SECURITY.md](SECURITY.md))
|
||||
|
||||
### Target Audience
|
||||
|
||||
- New Developers joining the project
|
||||
- Contributors (internal and external)
|
||||
- Anyone setting up a development environment
|
||||
|
||||
---
|
||||
|
||||
## Sections to Document
|
||||
|
||||
1. Prerequisites
|
||||
- Python version
|
||||
- Conda environment
|
||||
- Node.js (if applicable)
|
||||
- Git
|
||||
2. Getting Started
|
||||
- Clone repository
|
||||
- Setup conda environment
|
||||
- Install dependencies
|
||||
- Configuration setup
|
||||
3. Project Structure Overview
|
||||
4. Development Server
|
||||
- Starting FastAPI server
|
||||
- Hot reload configuration
|
||||
- Debug mode
|
||||
5. CLI Development
|
||||
6. Code Style
|
||||
- PEP 8 compliance
|
||||
- Type hints requirements
|
||||
- Docstring format
|
||||
- Import organization
|
||||
7. Git Workflow
|
||||
- Branch naming
|
||||
- Commit message format
|
||||
- Pull request process
|
||||
8. Common Development Tasks
|
||||
|
||||
### Adding Queue Deduplication
|
||||
|
||||
The download queue prevents duplicate entries at two levels:
|
||||
|
||||
**In-Memory Deduplication** (`src/server/services/download_service.py`):
|
||||
- `_pending_by_episode` dict tracks pending episodes: key = `(serie_id, season, episode)`
|
||||
- `_add_to_pending_queue()` updates the dict when adding items
|
||||
- `add_to_queue()` checks this dict before adding episodes (includes batch-local dedup)
|
||||
- `_remove_from_pending_queue()` cleans up the dict when items are removed
|
||||
|
||||
**Database Constraint** (`src/server/models.py`):
|
||||
- `DownloadQueueItem` has a unique index on `episode_id` via `__table_args__`
|
||||
- Prevents duplicate queue entries at the database level
|
||||
- Unique constraint: `Index("ix_download_queue_episode_pending", "episode_id", unique=True)`
|
||||
|
||||
**Scheduler Cooldown** (`src/server/services/scheduler_service.py`):
|
||||
- `_last_auto_download_time` tracks when auto-download last ran
|
||||
- 5-minute cooldown prevents rapid re-triggers
|
||||
- Checked at start of `_auto_download_missing()`
|
||||
|
||||
### Episode Lifecycle
|
||||
|
||||
Episodes transition through states stored in the `episodes` table:
|
||||
|
||||
| State | `is_downloaded` | `file_path` | Description |
|
||||
|-------|----------------|-------------|-------------|
|
||||
| Missing | `False` | `NULL` | Episode not yet downloaded |
|
||||
| Downloaded | `True` | Set | Episode exists on disk |
|
||||
|
||||
**State Transitions:**
|
||||
1. **Missing → Downloaded**: When download completes, `_remove_episode_from_missing_list()` calls `EpisodeService.mark_downloaded()` to set `is_downloaded=True` and populate `file_path`. The episode record is NOT deleted.
|
||||
|
||||
**Query Implications:**
|
||||
- `get_series_with_missing_episodes()`: Filters for `is_downloaded=False` to find series with undownloaded episodes
|
||||
- `get_series_with_no_episodes()`: Finds series with `is_downloaded=False` episodes but NO `is_downloaded=True` episodes (completely unwatched series)
|
||||
|
||||
### Mocking the Download Queue
|
||||
|
||||
When testing components that use the download queue:
|
||||
|
||||
```python
|
||||
# Mock repository for unit tests
|
||||
class MockQueueRepository:
|
||||
def __init__(self):
|
||||
self._items: Dict[str, DownloadItem] = {}
|
||||
|
||||
async def save_item(self, item: DownloadItem) -> DownloadItem:
|
||||
self._items[item.id] = item
|
||||
return item
|
||||
|
||||
async def get_all_items(self) -> List[DownloadItem]:
|
||||
return list(self._items.values())
|
||||
|
||||
# Use in fixture
|
||||
@pytest.fixture
|
||||
def mock_queue_repository():
|
||||
return MockQueueRepository()
|
||||
|
||||
@pytest.fixture
|
||||
def download_service(mock_anime_service, mock_queue_repository):
|
||||
return DownloadService(
|
||||
anime_service=mock_anime_service,
|
||||
queue_repository=mock_queue_repository,
|
||||
max_retries=3,
|
||||
)
|
||||
```
|
||||
|
||||
9. Troubleshooting Development Issues
|
||||
|
||||
### Async Context Managers for aiohttp
|
||||
|
||||
All `aiohttp.ClientSession` usages must be wrapped in `async with`:
|
||||
|
||||
```python
|
||||
# Correct — session properly closed on exit
|
||||
async with TMDBClient(api_key="key") as client:
|
||||
result = await client.search_tv_show("Show")
|
||||
|
||||
# Wrong — session may leak if exception occurs
|
||||
client = TMDBClient(api_key="key")
|
||||
result = await client.search_tv_show("Show")
|
||||
await client.close() # May not be called if exception raised earlier
|
||||
```
|
||||
|
||||
**Why:**
|
||||
- `aiohttp.ClientSession` holds TCP connections that must be explicitly closed
|
||||
- If exception occurs before `close()`, session leaks
|
||||
- Context manager guarantees `__aexit__` runs even on exceptions
|
||||
|
||||
**Services that use aiohttp:**
|
||||
- `TMDBClient` — has `__aenter__`/`__aexit__`, use `async with`
|
||||
- `ImageDownloader` — has `__aenter__`/`__aexit__`, use `async with`
|
||||
- `NFOService` — wraps both above, use `async with`
|
||||
|
||||
**Verification:**
|
||||
- Missing context manager usage triggers `__del__` warning on garbage collection
|
||||
- Integration tests verify no "Unclosed client session" errors in logs
|
||||
|
||||
### Scheduler Persistence and Recovery
|
||||
|
||||
The scheduler uses APScheduler's in-memory job store. Jobs are reconstructed from `config.json` on every startup — no separate database is needed.
|
||||
|
||||
```python
|
||||
# Jobs are built from config on startup — no persistence DB required
|
||||
scheduler = AsyncIOScheduler() # default MemoryJobStore
|
||||
scheduler.add_job(..., replace_existing=True)
|
||||
```
|
||||
|
||||
**Startup misfire recovery:** On `start()`, the scheduler checks `system_settings.last_scan_timestamp` in `aniworld.db`. If the last scan is overdue (>23h but <25h ago), an immediate rescan is triggered. This replaces APScheduler's built-in misfire handling which required a separate SQLite database.
|
||||
|
||||
**Grace period:** If the server was down for more than 25 hours, no automatic recovery occurs to avoid surprise rescans after long downtime.
|
||||
|
||||
**Health endpoint:** `GET /health` returns `scheduler_next_run` and `scheduler_last_run` for external monitors (Uptime Kuma, Prometheus, etc.).
|
||||
|
||||
**If server is down too long:** Manual trigger via `POST /api/scheduler/trigger-rescan` or wait for next scheduled run.
|
||||
|
||||
### Database Session Management
|
||||
|
||||
`get_async_session_factory()` returns a **new AsyncSession instance** directly (not a factory). The function name is historical — callers receive the session immediately:
|
||||
|
||||
```python
|
||||
# Correct usage:
|
||||
db = get_async_session_factory() # db IS the session
|
||||
await db.execute(...)
|
||||
await db.commit()
|
||||
await db.close()
|
||||
```
|
||||
|
||||
Do NOT call the result again with `()` — that tries to call an `AsyncSession` object, causing `'AsyncSession' object is not callable`.
|
||||
|
||||
For context manager usage, prefer `get_db_session()` (auto-commits) or `get_transactional_session()` (manual commit).
|
||||
|
||||
### Health Check Endpoints
|
||||
|
||||
The application provides health check endpoints for monitoring and container orchestration:
|
||||
|
||||
#### `GET /health`
|
||||
Basic health check returning service status and startup health check results.
|
||||
|
||||
**Response fields:**
|
||||
- `status`: "healthy", "degraded", or "unhealthy" based on startup checks
|
||||
- `timestamp`: ISO timestamp of the check
|
||||
- `series_app_initialized`: Whether the series app is loaded
|
||||
- `anime_directory_configured`: Whether anime_directory is set
|
||||
- `scheduler_next_run` / `scheduler_last_run`: Scheduler times
|
||||
- `checks`: Detailed startup check results (ffmpeg, DNS, anime_directory)
|
||||
|
||||
#### `GET /health/ready`
|
||||
Readiness check for container orchestrators (Kubernetes, Docker Swarm).
|
||||
|
||||
**Response when ready:**
|
||||
```json
|
||||
{
|
||||
"status": "ready",
|
||||
"ready": true,
|
||||
"timestamp": "2024-01-01T00:00:00",
|
||||
"checks": {...}
|
||||
}
|
||||
```
|
||||
|
||||
**Response when not ready (503):**
|
||||
```json
|
||||
{
|
||||
"status": "not_ready",
|
||||
"ready": false,
|
||||
"timestamp": "2024-01-01T00:00:00",
|
||||
"critical_failures": ["anime_directory: not configured"],
|
||||
"checks": {...}
|
||||
}
|
||||
```
|
||||
|
||||
#### `GET /health/detailed`
|
||||
Comprehensive health check including database, filesystem, and system metrics.
|
||||
|
||||
#### Startup Health Checks
|
||||
|
||||
On application startup, the following checks are performed:
|
||||
|
||||
| Check | Failure Status | Impact |
|
||||
|-------|---------------|--------|
|
||||
| `ffmpeg` | warning | HLS downloads may fail |
|
||||
| `dns_aniworld` | warning | Provider requests may fail |
|
||||
| `dns_tmdb` | warning | TMDB API calls may fail |
|
||||
| `anime_directory` | error | Download service disabled |
|
||||
|
||||
DNS checks are warnings because failures can be transient. anime_directory errors disable the download service to prevent failures.
|
||||
|
||||
### Troubleshooting Development Issues
|
||||
|
||||
#### Scheduler missed a run
|
||||
|
||||
1. Server was down at scheduled time (03:00 UTC by default).
|
||||
2. On restart, the scheduler checks `last_scan_timestamp` — if overdue by 23-25h, it triggers immediately.
|
||||
3. If server was down >25 hours, missed job is skipped to avoid surprise rescans.
|
||||
4. Trigger manually: `POST /api/scheduler/trigger-rescan`
|
||||
5. Monitor next run: `GET /health` → `scheduler_next_run`
|
||||
|
||||
#### Scheduler not firing (no events at scheduled time)
|
||||
|
||||
If the scheduler appears configured but never triggers:
|
||||
|
||||
1. **Check application logs for scheduler startup:**
|
||||
```
|
||||
grep "Scheduler service started" fastapi_app.log
|
||||
```
|
||||
- If missing, the scheduler failed to start — check for errors above this line
|
||||
- If present, scheduler started successfully
|
||||
|
||||
2. **Verify the job is registered:**
|
||||
```
|
||||
grep "Scheduler started with cron trigger" fastapi_app.log
|
||||
```
|
||||
|
||||
3. **Verify APScheduler events in logs:**
|
||||
```
|
||||
grep "apscheduler.executors.default" fastapi_app.log
|
||||
```
|
||||
- `Running job` = job triggered
|
||||
- `executed successfully` = job completed
|
||||
- No output = job never fired
|
||||
|
||||
4. **Test manual trigger:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8000/api/scheduler/trigger-rescan -H "Authorization: Bearer <token>"
|
||||
```
|
||||
- If manual trigger works but cron doesn't, the issue is APScheduler configuration
|
||||
|
||||
5. **Check next_run_time via health endpoint:**
|
||||
```bash
|
||||
curl http://localhost:8000/health | jq .scheduler_next_run
|
||||
```
|
||||
- If `null`, the job is not scheduled
|
||||
- If set, the scheduler knows when to run next
|
||||
|
||||
6. **Check timezone handling:**
|
||||
- APScheduler uses UTC internally
|
||||
- The schedule_time config (e.g., "03:00") is interpreted as UTC
|
||||
- If you expect local time, adjust the schedule_time accordingly
|
||||
|
||||
#### Startup health check failures
|
||||
|
||||
If `/health` returns `unhealthy` status:
|
||||
|
||||
1. **anime_directory error**: Directory not configured or not writable
|
||||
- Check `ANIME_DIRECTORY` environment variable
|
||||
- Verify directory exists and permissions allow write access
|
||||
- Download service will not initialize until resolved
|
||||
|
||||
2. **ffmpeg warning**: ffmpeg not found in PATH
|
||||
- HLS stream downloads will fail
|
||||
- Install ffmpeg: `apt install ffmpeg` or `brew install ffmpeg`
|
||||
|
||||
3. **DNS warnings**: Domain resolution failed
|
||||
- Check network connectivity
|
||||
- DNS failures are transient — warnings don't block startup
|
||||
- Retry later to verify: `GET /health`
|
||||
|
||||
### Provider Failure Handling
|
||||
|
||||
Download providers (VOE, Doodstream, Vidmoly, Vidoza, SpeedFiles, Streamtape,
|
||||
Luluvdo) regularly break: URLs expire, sites change their player markup, geo
|
||||
blocks appear, and `yt-dlp` extractors lag behind upstream changes. The
|
||||
`AniworldLoader.download()` flow is designed to fail fast and rotate.
|
||||
|
||||
**Rotation order**
|
||||
|
||||
1. The episode page is scraped for the providers AniWorld actually advertises.
|
||||
2. Results are ordered by the preference in `DEFAULT_PROVIDERS`
|
||||
(`provider_config.py`); providers not listed run last.
|
||||
3. For each candidate the loader:
|
||||
1. Calls `_check_url_alive()` — HEAD probe with GET fallback. Any 4xx
|
||||
response or connection error skips the provider immediately.
|
||||
2. Resolves the redirect via `_resolve_direct_link()` to obtain a direct
|
||||
stream URL plus headers. Provider-specific extractors (e.g. `VOE`) are
|
||||
preferred; unknown providers fall back to the embed URL so `yt-dlp` can
|
||||
attempt extraction.
|
||||
3. Tries `_try_direct_stream()` — straight `requests.get(stream=True)` when
|
||||
`Content-Type` is `video/*` or `application/octet-stream`. This avoids
|
||||
`yt-dlp` entirely for direct MP4 links.
|
||||
4. Falls back to `yt-dlp` with the ffmpeg downloader for HLS streams.
|
||||
4. On any failure, temp files are cleaned and the loop moves to the next
|
||||
provider. When the chain is exhausted, the loader logs
|
||||
`All download providers failed for S{season}E{episode} ...; tried=[...]`
|
||||
to both the application log and `logs/download_errors.log`.
|
||||
|
||||
**Do not hardcode provider URLs.** Provider domains shift constantly (e.g.
|
||||
Doodstream alternates between `dood.li`, `dood.so`, `dood.la`). Only the
|
||||
referer hints in `PROVIDER_HEADERS` are persisted — discovery still happens
|
||||
at runtime through AniWorld's redirect endpoint.
|
||||
|
||||
### HLS Stream Handling
|
||||
|
||||
HLS (HTTP Live Streaming) manifests (`.m3u8`) require yt-dlp to use the
|
||||
`ffmpeg` downloader with `--hls-use-mpegts`. Both providers configure this
|
||||
automatically:
|
||||
|
||||
```python
|
||||
ydl_opts = {
|
||||
"downloader": "ffmpeg", # Use ffmpeg instead of native
|
||||
"hls_use_mpegts": True, # Write transport stream (.ts) segments
|
||||
}
|
||||
```
|
||||
|
||||
**Why this matters**: Without ffmpeg, yt-dlp logs:
|
||||
`"Live HLS streams are not supported by the native downloader"`
|
||||
|
||||
**Requirements**:
|
||||
- ffmpeg must be installed and in PATH (`which ffmpeg`)
|
||||
- Install: `apt install ffmpeg` (Debian/Ubuntu) or `brew install ffmpeg` (macOS)
|
||||
- Startup health check (see Health Check Endpoints) verifies ffmpeg presence
|
||||
|
||||
**Trade-offs**:
|
||||
- HLS downloads are slower than direct MP4 (reassembly of .ts segments)
|
||||
- Requires more disk space during download
|
||||
- May need post-processing if .ts format is not desired
|
||||
|
||||
**Detection**: VOE provider extracts HLS URLs via `HLS_PATTERN` regex. Other
|
||||
providers let yt-dlp auto-detect from URL/content-type.
|
||||
|
||||
### Updating yt-dlp
|
||||
|
||||
When extractors break (typical symptoms: every provider HEAD probe succeeds
|
||||
but `yt-dlp` raises `Unable to extract` or `HTTP Error 404`):
|
||||
|
||||
1. Check the upstream tracker first: https://github.com/yt-dlp/yt-dlp/issues
|
||||
2. Upgrade in the conda environment:
|
||||
```bash
|
||||
conda run -n AniWorld pip install --upgrade yt-dlp
|
||||
```
|
||||
3. Smoke-test against a known-good episode before pinning a new floor in
|
||||
`requirements.txt` (`yt-dlp>=YYYY.MM.DD`).
|
||||
4. Re-run the provider test suite:
|
||||
```bash
|
||||
conda run -n AniWorld python -m pytest tests/unit/test_aniworld_provider.py -v
|
||||
```
|
||||
5. If a specific extractor is removed upstream, drop the provider from
|
||||
`DEFAULT_PROVIDERS` rather than patching `yt-dlp` in tree.
|
||||
|
||||
### User Notification on Total Failure
|
||||
|
||||
`SeriesApp.download_episode()` already emits a `download_status="failed"`
|
||||
WebSocket event when `loader.download()` returns `False`. Operators should
|
||||
forward this to `notification_service.notify_download_failed()` so users see
|
||||
a HIGH-priority alert. The loader keeps the failure detail in
|
||||
`logs/download_errors.log` for post-mortem.
|
||||
|
||||
## Series Storage
|
||||
|
||||
### Overview
|
||||
|
||||
Series metadata now stored in the database (SQLAlchemy ORM).
|
||||
Legacy files (`key` and `data` per folder) are deprecated but preserved
|
||||
for backward compatibility.
|
||||
|
||||
### Architecture
|
||||
|
||||
- **Database**: Single source of truth for all series metadata
|
||||
- **In-Memory Cache**: SeriesApp maintains a cache for performance
|
||||
- **Filesystem**: Only used for episode files themselves, not metadata
|
||||
|
||||
### Migration
|
||||
|
||||
First startup after upgrade automatically imports any legacy
|
||||
series files into the database.
|
||||
|
||||
### Legacy Files
|
||||
|
||||
- `key` file: Contains series provider key (deprecated)
|
||||
- `data` file: Contains Serie JSON object (deprecated)
|
||||
|
||||
Both are safe to delete after migration; not needed for normal operation.
|
||||
|
||||
94
Docs/InstructionsLogging.md
Normal file
94
Docs/InstructionsLogging.md
Normal file
@@ -0,0 +1,94 @@
|
||||
# Logging Instructions
|
||||
|
||||
This document describes how to write and refactor logging across the AniWorld codebase to make logs **human-readable**, **debug-friendly**, and **noise-free**.
|
||||
|
||||
> ✅ Goal: Logs should help a developer understand what happened, why it happened, and what to inspect next — without overwhelming them with duplicates or irrelevant details.
|
||||
|
||||
---
|
||||
|
||||
## 1. Principles for Great Logs
|
||||
|
||||
### 1.1 Use the Right Log Level
|
||||
|
||||
- `DEBUG`: Detailed internal state useful when debugging a specific issue (e.g., decision points, returned values, request/response payloads). Not for normal operation.
|
||||
- `INFO`: High-level events that represent what the system is doing (e.g., "Import started", "New series added", "Config reloaded"). Use sparingly.
|
||||
- `WARNING`: Something unexpected happened, but the system can continue (e.g., missing optional file, fallback behavior).
|
||||
- `ERROR`: An operation failed and needs attention (e.g., exception caught, failed database write).
|
||||
- `CRITICAL`: The system is in an unusable state (e.g., config corruption, failed startup).
|
||||
|
||||
### 1.2 Keep Logs Human-Readable
|
||||
|
||||
- Write messages in a clear, descriptive sentence-style format.
|
||||
- Avoid cryptic codes or single-word log messages.
|
||||
- Prefer `logger.debug("... %s", value)`-style formatting over f-strings to avoid unnecessary work when the log level is disabled.
|
||||
|
||||
### 1.3 Avoid Log Spam
|
||||
|
||||
- Don’t log inside hot loops unless you explicitly aggregate and log a summary (e.g., "Processed 124 files, 3 failures").
|
||||
- Avoid repeated/logging the same event at the same level (e.g., do not log "Retrying" 10 times at INFO; log once at INFO and then use DEBUG for each retry).
|
||||
- Use rate limiting or debounce patterns for logs that can fire rapidly (e.g., external service health checks).
|
||||
- Prefer a single higher-level log with context rather than many low-level logs that clutter output.
|
||||
|
||||
### 1.4 Log Objects Usefully
|
||||
|
||||
- When logging objects, log the minimal useful representation (e.g., ID, name, status) rather than the full object or its memory address.
|
||||
- If an object has a `.dict()`, `.to_dict()`, or `.as_dict()` helper (common in Pydantic models), log that rather than relying on `repr()`.
|
||||
- Add a `__repr__` or `__str__` implementation to domain models that returns a helpful, concise string with key identifiers.
|
||||
- Use structured logging (e.g., `logger.info("Series added", extra={"series_id": series.id, "title": series.title})`) where supported.
|
||||
- For exceptions, prefer `logger.exception("Failed to ...")` to capture stack traces.
|
||||
|
||||
---
|
||||
|
||||
## 2. Refactoring Existing Logs
|
||||
|
||||
When improving or refactoring existing log statements, aim to make them:
|
||||
|
||||
- **Actionable**: A developer reading the log should know what happened and what to check next.
|
||||
- **Non-redundant**: Remove duplicates and ensure only one log records the same high-level event at a given level.
|
||||
- **Context-rich**: Include identifiers (e.g., `series_id`, `file_path`, `user_id`) and key state that explains why a decision was made.
|
||||
- **Level-appropriate**: Downgrade noisy INFO logs to DEBUG, and elevate critical failures to ERROR/CRITICAL.
|
||||
|
||||
### 2.1 Refactor Checklist
|
||||
|
||||
1. **Locate noisy logs**: Search for repeated messages (e.g., "Start", "Done") and determine whether they should be DEBUG or removed.
|
||||
2. **Replace ad-hoc prints**: Remove `print()` statements or `print(obj)` and replace with `logger.*` calls.
|
||||
3. **Use structured context**: If a function logs multiple related messages, include the same context in each (e.g., `extra={"series_id": series.id}`) or use a context manager that attaches it.
|
||||
4. **Validate object output**: Ensure any logged object produces a useful representation (add methods or translate to dict). If not, log the key fields explicitly.
|
||||
5. **Batch repetitive events**: If a loop logs per item, consider collecting stats and logging a summary at the end.
|
||||
|
||||
## 3. Adding New Logs
|
||||
|
||||
When adding logs to new code paths:
|
||||
|
||||
- Log **important state transitions** (e.g., "Queue started", "Download completed", "Config reloaded").
|
||||
- For error paths, include what failed and why (e.g., "Could not load config from X: {exc}").
|
||||
- Prefer logging at the boundaries of operations, not deep inside utility functions unless it aids debugging.
|
||||
- Write logs in full sentences, with a clear subject, verb, and object.
|
||||
|
||||
---
|
||||
|
||||
## 4. Example Patterns
|
||||
|
||||
```python
|
||||
logger.info("Import completed", extra={"series_id": series.id, "count": len(imported)})
|
||||
|
||||
logger.debug(
|
||||
"Fetched feed items",
|
||||
extra={"feed_url": feed.url, "item_count": len(items)},
|
||||
)
|
||||
|
||||
try:
|
||||
result = download_episode(episode)
|
||||
except Exception:
|
||||
logger.exception("Failed to download episode %s", episode.id)
|
||||
```
|
||||
|
||||
> 💡 When in doubt, favor **fewer, richer logs** over many noisy logs.
|
||||
|
||||
---
|
||||
|
||||
## 5. Logging Audit Task List
|
||||
|
||||
For a guided checklist of files and logging improvements, see **`docs/tasks.md`**. This is where we track which files have been reviewed and which logging items still need attention.
|
||||
|
||||
> ✅ After applying the guidelines above, update `docs/tasks.md` to indicate which tasks are complete.
|
||||
111
Docs/MIGRATION_GUIDE.md
Normal file
111
Docs/MIGRATION_GUIDE.md
Normal file
@@ -0,0 +1,111 @@
|
||||
# Migration Guide: File-Based to Database Storage
|
||||
|
||||
## Overview
|
||||
|
||||
This guide covers the transition from file-based series metadata storage to the new database-backed system introduced in v2.0.
|
||||
|
||||
## What Changed
|
||||
|
||||
**Before v2.0**: Series metadata stored in `key` and `data` files alongside anime folders.
|
||||
|
||||
**After v2.0**: All metadata stored in SQLite database (`aniworld.db`). Files are deprecated but still supported for backward compatibility during migration.
|
||||
|
||||
## Automated Migration
|
||||
|
||||
The application automatically migrates on first startup:
|
||||
|
||||
1. Scans anime directory for `key` and `data` files
|
||||
2. Parses legacy files into `AnimeSeries` and `Episode` records
|
||||
3. Loads series into in-memory cache
|
||||
4. Logs migration results
|
||||
|
||||
**No manual action required.**
|
||||
|
||||
## Manual Verification
|
||||
|
||||
After first startup with the new version:
|
||||
|
||||
1. **Check logs** for: `"Migrated X series from files to DB"`
|
||||
2. **Verify series count**: UI shows same number of series as before
|
||||
3. **Confirm episodes**: Episode counts match expected totals
|
||||
|
||||
```bash
|
||||
# Check migration log
|
||||
grep "Migrated" logs/app.log
|
||||
|
||||
# Verify series via API
|
||||
curl http://localhost:8000/api/anime | jq '.total'
|
||||
```
|
||||
|
||||
## After Migration
|
||||
|
||||
### Safe to Delete
|
||||
|
||||
Once verified, these files can be removed:
|
||||
|
||||
```
|
||||
<anime_folder>/
|
||||
├── Attack on Titan (2013)/
|
||||
│ ├── key # ❌ Can delete
|
||||
│ ├── data # ❌ Can delete
|
||||
│ └── Season 1/
|
||||
│ └── ...
|
||||
```
|
||||
|
||||
**Deleting these files does not affect the database.** The metadata now lives in `aniworld.db`.
|
||||
|
||||
### Backup (Recommended)
|
||||
|
||||
Before deleting, backup the files:
|
||||
|
||||
```bash
|
||||
# Create backup directory
|
||||
mkdir -p backup/legacy_series_files
|
||||
|
||||
# Copy all key and data files
|
||||
find /path/to/anime -name "key" -o -name "data" | while read f; do
|
||||
cp "$f" "backup/legacy_series_files/"
|
||||
done
|
||||
```
|
||||
|
||||
## Reverting (Not Recommended)
|
||||
|
||||
If you must revert to file-based storage:
|
||||
|
||||
1. **Restore from database backup** (if available)
|
||||
2. **Export manually** (no export script exists)
|
||||
|
||||
**Warning**: File-based storage is deprecated and will be removed in v3.0.0.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Series Not Appearing After Migration
|
||||
|
||||
1. Check logs for migration errors: `grep -i error logs/app.log`
|
||||
2. Verify `key` and `data` files exist and are readable
|
||||
3. Manually trigger rescan: `POST /api/scheduler/trigger-rescan`
|
||||
|
||||
### Duplicate Series
|
||||
|
||||
1. Check for duplicate `key` files (same series in multiple folders)
|
||||
2. Verify series key uniqueness in database:
|
||||
|
||||
```bash
|
||||
sqlite3 aniworld.db "SELECT key, COUNT(*) FROM anime_series GROUP BY key HAVING COUNT(*) > 1;"
|
||||
```
|
||||
|
||||
### Missing Episodes
|
||||
|
||||
1. Trigger targeted scan for affected series
|
||||
2. Check episode sync logs
|
||||
3. Verify file permissions on anime directory
|
||||
|
||||
## Deprecation Timeline
|
||||
|
||||
| Version | Status |
|
||||
|---------|--------|
|
||||
| v2.0.x | Legacy files supported, migration automated |
|
||||
| v2.1.x | Legacy files still supported, warnings in logs |
|
||||
| v3.0.0 | **Legacy files removed** - database only |
|
||||
|
||||
Upgrade to v3.0.0 before legacy file support ends.
|
||||
206
Docs/NAVIGATION.md
Normal file
206
Docs/NAVIGATION.md
Normal file
@@ -0,0 +1,206 @@
|
||||
# Navigation & Redirect Logic
|
||||
|
||||
This document describes the setup flow navigation, covering how users progress from initial setup through to the main application.
|
||||
|
||||
## Overview
|
||||
|
||||
The application uses a middleware-based redirect system to ensure users complete setup before accessing the main app. The flow involves multiple pages handling setup completion, unresolved folder detection, and initialization.
|
||||
|
||||
## Setup Flow
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────┐
|
||||
│ SETUP FLOW │
|
||||
├─────────────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ /setup ──► /loading ──► /setup/unresolved ──► /loading ──► /login │
|
||||
│ │ │ │ │ │
|
||||
│ │ │ │ │ │
|
||||
│ ▼ ▼ ▼ ▼ │
|
||||
│ (first (Series Scan + (has folders) (all resolved) │
|
||||
│ time) NFO Scan) │ │
|
||||
│ │ │ │
|
||||
│ │ │ │
|
||||
│ │ ▼ │
|
||||
│ │ [Done button] ──► marks complete │
|
||||
│ │ │ │
|
||||
│ │ ▼ │
|
||||
│ │ /loading (NFO phase runs again) │
|
||||
│ │ │ │
|
||||
│ └────────┴─────────────────────────────────────┘
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**New Navigation Order:**
|
||||
1. `/setup` → Initial configuration
|
||||
2. `/loading` → Series scan + NFO scan
|
||||
3. `/setup/unresolved` → Resolve folders (if any)
|
||||
4. `/loading` → NFO scan runs again
|
||||
5. `/login` → Authentication
|
||||
|
||||
**Key Changes:**
|
||||
- After `/setup/unresolved`, the "Done" button marks the phase as complete
|
||||
- Revisiting `/setup/unresolved` after completion → redirects to `/loading`
|
||||
- `/loading` always goes to `/setup/unresolved` if unresolved folders exist
|
||||
- NFO scan runs as a separate phase after series sync during initialization
|
||||
|
||||
## Middleware: SetupRedirectMiddleware
|
||||
|
||||
**File:** `src/server/middleware/setup_redirect.py`
|
||||
|
||||
The middleware intercepts all requests and redirects to `/setup` if:
|
||||
- No master password is configured
|
||||
- Configuration file is missing or invalid
|
||||
|
||||
### Exempt Paths (always accessible)
|
||||
|
||||
| Path | Purpose |
|
||||
|------|---------|
|
||||
| `/setup` | Initial setup page |
|
||||
| `/setup/unresolved` | Unresolved folder resolution |
|
||||
| `/loading` | Initialization progress page |
|
||||
| `/login` | Authentication |
|
||||
| `/api/auth/*` | Auth endpoints |
|
||||
| `/api/config/*` | Config API |
|
||||
| `/api/health` | Health check |
|
||||
| `/static/*` | Static assets |
|
||||
|
||||
### Middleware Logic
|
||||
|
||||
1. **Setup incomplete** → Redirect to `/setup`
|
||||
2. **Setup complete, accessing `/setup`** → Redirect to `/login`
|
||||
3. **Setup complete, accessing `/loading`** → Allow access (page handles its own redirect)
|
||||
4. **Setup complete, accessing `/setup/unresolved`**:
|
||||
- If `unresolved_completed` flag is set → Redirect to `/loading`
|
||||
- Otherwise → Allow access
|
||||
5. **API requests during setup** → Return 503 with `setup_url`
|
||||
|
||||
## Pages
|
||||
|
||||
### 1. Setup Page (`/setup`)
|
||||
|
||||
**File:** `src/server/web/templates/setup.html`
|
||||
|
||||
Handles initial configuration:
|
||||
- Master password creation
|
||||
- Anime directory selection
|
||||
- Database initialization
|
||||
|
||||
**Post-completion flow:**
|
||||
- Redirects to `/loading` to begin initialization
|
||||
|
||||
### 2. Loading Page (`/loading`)
|
||||
|
||||
**File:** `src/server/web/templates/loading.html`
|
||||
|
||||
Shows initialization progress via WebSocket:
|
||||
- Series scanning
|
||||
- Database population
|
||||
- Logo/image loading
|
||||
|
||||
**Post-initialization flow:**
|
||||
```javascript
|
||||
async function checkUnresolvedAndProceed() {
|
||||
// Fetch unresolved folders via API
|
||||
const res = await fetch('/api/setup/unresolved', {
|
||||
headers: { 'Authorization': `Bearer ${token}` }
|
||||
});
|
||||
const folders = await res.json();
|
||||
|
||||
if (folders.length > 0) {
|
||||
// Has unresolved folders → go to resolution page
|
||||
window.location.href = '/setup/unresolved';
|
||||
} else {
|
||||
// No unresolved folders → go to login
|
||||
window.location.href = '/login';
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Unresolved Folders Page (`/setup/unresolved`)
|
||||
|
||||
**File:** `src/server/web/templates/unresolved.html`
|
||||
|
||||
Allows manual resolution of folders that couldn't be auto-matched:
|
||||
- Shows list of unresolved folders
|
||||
- Provides search suggestions
|
||||
- Input field for entering provider key
|
||||
- Resolve/delete actions
|
||||
- **Done button** at top to complete the phase without resolving all folders
|
||||
|
||||
**Post-resolution flow:**
|
||||
```javascript
|
||||
// After clicking "Done" button
|
||||
async function handleDone() {
|
||||
// Call API to mark phase as complete
|
||||
await fetch('/api/setup/unresolved/done', { method: 'POST' });
|
||||
// Redirect to loading for final NFO scan
|
||||
window.location.href = '/loading';
|
||||
}
|
||||
```
|
||||
|
||||
**Done button behavior:**
|
||||
- Marks all remaining folders as handled
|
||||
- Sets `unresolved_completed` flag in config
|
||||
- Redirects to `/loading` to run final NFO scan
|
||||
- After completion, `/setup/unresolved` becomes inaccessible (redirects to `/loading`)
|
||||
|
||||
### 4. Login Page (`/login`)
|
||||
|
||||
**File:** `src/server/web/templates/login.html`
|
||||
|
||||
Authentication page. After successful login → redirect to `/` (main app).
|
||||
|
||||
## API Endpoints
|
||||
|
||||
### Unresolved Folders API
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| `GET` | `/api/setup/unresolved` | List all unresolved folders |
|
||||
| `GET` | `/api/setup/unresolved/{folder_name}` | Get specific folder details |
|
||||
| `POST` | `/api/setup/unresolved/{folder_name}/resolve` | Resolve with provider key |
|
||||
| `POST` | `/api/setup/unresolved/{folder_name}/search` | Re-search for matches |
|
||||
| `DELETE` | `/api/setup/unresolved/{folder_name}` | Remove folder from tracking |
|
||||
| `POST` | `/api/setup/unresolved/done` | Mark unresolved phase as complete |
|
||||
|
||||
### Auth API
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| `POST` | `/api/auth/setup` | Create master password |
|
||||
| `POST` | `/api/auth/login` | Authenticate |
|
||||
| `POST` | `/api/auth/logout` | End session |
|
||||
|
||||
## Key Files
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `src/server/middleware/setup_redirect.py` | Redirect middleware |
|
||||
| `src/server/controllers/page_controller.py` | Page route handlers |
|
||||
| `src/server/web/templates/setup.html` | Setup template |
|
||||
| `src/server/web/templates/loading.html` | Loading template |
|
||||
| `src/server/web/templates/unresolved.html` | Unresolved folders template |
|
||||
| `src/server/api/setup_endpoints.py` | Unresolved folders API |
|
||||
| `src/server/database/service.py` | UnresolvedFolderService |
|
||||
|
||||
## Common Issues
|
||||
|
||||
### Redirect Loop
|
||||
|
||||
**Symptom:** Browser keeps redirecting between pages.
|
||||
|
||||
**Causes:**
|
||||
1. `loading.html` always redirected to `/setup/unresolved` without checking if any exist
|
||||
2. `unresolved.html` redirected to `/` which middleware redirected back to `/login`
|
||||
|
||||
**Fix:** See the navigation logic updates in loading.html and unresolved.html.
|
||||
|
||||
### Can't Access Unresolved Page After Setup
|
||||
|
||||
**Symptom:** Middleware redirects to `/login` instead of allowing access to `/setup/unresolved`.
|
||||
|
||||
**Cause:** `/setup/unresolved` is in the exempt paths but the request may not be reaching it due to completion check timing.
|
||||
|
||||
**Fix:** The middleware allows access to `/loading` which handles the redirect to `/setup/unresolved` after initialization.
|
||||
905
Docs/NFO_GUIDE.md
Normal file
905
Docs/NFO_GUIDE.md
Normal file
@@ -0,0 +1,905 @@
|
||||
# NFO Metadata Guide
|
||||
|
||||
## Document Purpose
|
||||
|
||||
This guide explains how to use the NFO metadata feature to enrich your anime library with TMDB metadata and artwork for Plex, Jellyfin, Emby, and Kodi.
|
||||
|
||||
---
|
||||
|
||||
## 1. Overview
|
||||
|
||||
### What are NFO Files?
|
||||
|
||||
NFO files are XML documents that contain metadata about TV shows and episodes. Media servers like Plex, Jellyfin, Emby, and Kodi use these files to display information about your library without needing to scrape external sources.
|
||||
|
||||
### Features
|
||||
|
||||
- **Automatic NFO Creation**: Generate NFO files during downloads
|
||||
- **TMDB Integration**: Fetch metadata from The Movie Database
|
||||
- **Image Downloads**: Poster, fanart, and logo images
|
||||
- **Batch Operations**: Create/update NFO files for multiple anime
|
||||
- **Web UI**: Manage NFO settings and operations
|
||||
- **API Access**: Programmatic NFO management
|
||||
|
||||
---
|
||||
|
||||
## 2. Getting Started
|
||||
|
||||
### 2.1 Obtain TMDB API Key
|
||||
|
||||
1. Create a free account at https://www.themoviedb.org
|
||||
2. Navigate to https://www.themoviedb.org/settings/api
|
||||
3. Request an API key (select "Developer" option)
|
||||
4. Copy your API key (v3 auth)
|
||||
|
||||
### 2.2 Configure NFO Settings
|
||||
|
||||
#### Via Web Interface
|
||||
|
||||
1. Open http://127.0.0.1:8000
|
||||
2. Click **Configuration** button
|
||||
3. Scroll to **NFO Settings** section
|
||||
4. Enter your TMDB API key
|
||||
5. Click **Test Connection** to verify
|
||||
6. Configure options:
|
||||
- **Auto-create during downloads**: Enable to create NFO files automatically
|
||||
- **Update on library scan**: Enable to refresh existing NFO files
|
||||
- **Download poster**: Episode and show poster images (poster.jpg)
|
||||
- **Download logo**: Show logo images (logo.png)
|
||||
- **Download fanart**: Background artwork (fanart.jpg)
|
||||
- **Image size**: Select w500 (recommended), w780, or original
|
||||
7. Click **Save**
|
||||
|
||||
#### Via Environment Variables
|
||||
|
||||
Add to your `.env` file:
|
||||
|
||||
```bash
|
||||
TMDB_API_KEY=your_api_key_here
|
||||
NFO_AUTO_CREATE=true
|
||||
NFO_UPDATE_ON_SCAN=false
|
||||
NFO_DOWNLOAD_POSTER=true
|
||||
NFO_DOWNLOAD_LOGO=false
|
||||
NFO_DOWNLOAD_FANART=false
|
||||
NFO_IMAGE_SIZE=w500
|
||||
```
|
||||
|
||||
#### Via config.json
|
||||
|
||||
Edit `data/config.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"nfo": {
|
||||
"tmdb_api_key": "your_api_key_here",
|
||||
"auto_create": true,
|
||||
"update_on_scan": false,
|
||||
"download_poster": true,
|
||||
"download_logo": false,
|
||||
"download_fanart": false,
|
||||
"image_size": "w500"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Using NFO Features
|
||||
|
||||
### 3.1 Automatic NFO Creation
|
||||
|
||||
With `auto_create` enabled, NFO files are created automatically when downloading episodes:
|
||||
|
||||
1. Add episodes to download queue
|
||||
2. Start queue processing
|
||||
3. NFO files are created after successful downloads
|
||||
4. Images are downloaded based on configuration
|
||||
|
||||
### 3.2 Manual NFO Creation
|
||||
|
||||
#### Via Web Interface
|
||||
|
||||
1. Navigate to the main page
|
||||
2. Click **Create NFO** button next to an anime
|
||||
3. Wait for completion notification
|
||||
|
||||
#### Via API
|
||||
|
||||
```bash
|
||||
curl -X POST "http://127.0.0.1:8000/api/nfo/create" \
|
||||
-H "Authorization: Bearer YOUR_JWT_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"anime_id": 123,
|
||||
"folder_path": "/path/to/anime/Attack on Titan"
|
||||
}'
|
||||
```
|
||||
|
||||
### 3.3 Batch NFO Creation
|
||||
|
||||
Create NFO files for multiple anime at once:
|
||||
|
||||
```bash
|
||||
curl -X POST "http://127.0.0.1:8000/api/nfo/batch/create" \
|
||||
-H "Authorization: Bearer YOUR_JWT_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"anime_ids": [123, 456, 789]
|
||||
}'
|
||||
```
|
||||
|
||||
### 3.4 Update Existing NFO Files
|
||||
|
||||
Update NFO files with latest TMDB metadata:
|
||||
|
||||
```bash
|
||||
curl -X POST "http://127.0.0.1:8000/api/nfo/update" \
|
||||
-H "Authorization: Bearer YOUR_JWT_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"anime_id": 123,
|
||||
"folder_path": "/path/to/anime/Attack on Titan",
|
||||
"force": true
|
||||
}'
|
||||
```
|
||||
|
||||
### 3.5 Check NFO Status
|
||||
|
||||
Check which anime have NFO files:
|
||||
|
||||
```bash
|
||||
curl -X GET "http://127.0.0.1:8000/api/nfo/check?folder_path=/path/to/anime" \
|
||||
-H "Authorization: Bearer YOUR_JWT_TOKEN"
|
||||
```
|
||||
|
||||
Response:
|
||||
|
||||
```json
|
||||
{
|
||||
"has_tvshow_nfo": true,
|
||||
"episode_nfos": [
|
||||
{
|
||||
"season": 1,
|
||||
"episode": 1,
|
||||
"has_nfo": true,
|
||||
"file_path": "/path/to/anime/Season 1/S01E01.nfo"
|
||||
}
|
||||
],
|
||||
"missing_episodes": [],
|
||||
"total_episodes": 25,
|
||||
"nfo_count": 25
|
||||
}
|
||||
```
|
||||
|
||||
### 3.6 Fallback Behavior When TMDB is Unavailable
|
||||
|
||||
When TMDB lookup fails (network issues, API errors, or no match found), the system creates a **minimal NFO** to ensure the series is still tracked. This behavior applies to:
|
||||
|
||||
- Manual NFO creation via API
|
||||
- Batch NFO creation operations
|
||||
- Automatic NFO creation during downloads
|
||||
|
||||
**What a minimal NFO contains:**
|
||||
|
||||
```xml
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<tvshow>
|
||||
<title>Series Name</title>
|
||||
<year>2024</year>
|
||||
<plot>No metadata available for Series Name. TMDB lookup failed.</plot>
|
||||
</tvshow>
|
||||
```
|
||||
|
||||
**Limitations of minimal NFOs:**
|
||||
- No poster, logo, or fanart images
|
||||
- No rating, genre, or studio information
|
||||
- No TMDB or other provider IDs
|
||||
- May not display correctly in some media servers
|
||||
|
||||
**To upgrade a minimal NFO:**
|
||||
1. Use the Update endpoint (`PUT /api/nfo/{serie_id}/update`) when TMDB is available
|
||||
2. Or delete the NFO and recreate it with full metadata
|
||||
|
||||
---
|
||||
|
||||
## 4. File Structure
|
||||
|
||||
### 4.1 NFO File Locations
|
||||
|
||||
NFO files are created in the anime directory:
|
||||
|
||||
```
|
||||
/path/to/anime/Attack on Titan/
|
||||
├── tvshow.nfo # Show metadata
|
||||
├── poster.jpg # Show poster (optional)
|
||||
├── logo.png # Show logo (optional)
|
||||
├── fanart.jpg # Show fanart (optional)
|
||||
├── Season 1/
|
||||
│ ├── S01E01.mkv
|
||||
│ ├── S01E01.nfo # Episode metadata
|
||||
│ ├── S01E01-thumb.jpg # Episode thumbnail (optional)
|
||||
│ ├── S01E02.mkv
|
||||
│ └── S01E02.nfo
|
||||
└── Season 2/
|
||||
├── S02E01.mkv
|
||||
└── S02E01.nfo
|
||||
```
|
||||
|
||||
### 4.2 tvshow.nfo Format
|
||||
|
||||
```xml
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<tvshow>
|
||||
<title>Attack on Titan</title>
|
||||
<originaltitle>進撃の巨人</originaltitle>
|
||||
<showtitle>Attack on Titan</showtitle>
|
||||
<sorttitle>Attack on Titan</sorttitle>
|
||||
<rating>8.5</rating>
|
||||
<year>2013</year>
|
||||
<plot>Humans are nearly exterminated by giant creatures...</plot>
|
||||
<runtime>24</runtime>
|
||||
<mpaa>TV-MA</mpaa>
|
||||
<premiered>2013-04-07</premiered>
|
||||
<status>Ended</status>
|
||||
<studio>Wit Studio</studio>
|
||||
<genre>Animation</genre>
|
||||
<genre>Action</genre>
|
||||
<genre>Sci-Fi & Fantasy</genre>
|
||||
<uniqueid type="tmdb">1429</uniqueid>
|
||||
<tmdbid>1429</tmdbid>
|
||||
<thumb aspect="poster">https://image.tmdb.org/t/p/w500/...</thumb>
|
||||
<fanart>
|
||||
<thumb>https://image.tmdb.org/t/p/original/...</thumb>
|
||||
</fanart>
|
||||
</tvshow>
|
||||
```
|
||||
|
||||
**Manual TMDB ID Override**: To skip TMDB search and use a specific ID directly, include `<tmdbid>YOUR_ID</tmdbid>` in the NFO. This is useful when:
|
||||
- TMDB search fails for your series (e.g., new or obscure anime)
|
||||
- You already know the correct TMDB ID
|
||||
- You want to avoid rate limiting from repeated searches
|
||||
|
||||
Aniworld reads `<tmdbid>` element and `<uniqueid type="tmdb">` first. If found, it uses the ID directly instead of searching.
|
||||
|
||||
### 4.3 Episode NFO Format
|
||||
|
||||
```xml
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<episodedetails>
|
||||
<title>To You, in 2000 Years: The Fall of Shiganshina, Part 1</title>
|
||||
<showtitle>Attack on Titan</showtitle>
|
||||
<season>1</season>
|
||||
<episode>1</episode>
|
||||
<displayseason>1</displayseason>
|
||||
<displayepisode>1</displayepisode>
|
||||
<plot>After a hundred years of peace...</plot>
|
||||
<runtime>24</runtime>
|
||||
<aired>2013-04-07</aired>
|
||||
<rating>8.2</rating>
|
||||
<uniqueid type="tmdb">63056</uniqueid>
|
||||
<thumb>https://image.tmdb.org/t/p/w500/...</thumb>
|
||||
</episodedetails>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. Folder Naming Convention
|
||||
|
||||
### 5.1 Expected Format
|
||||
|
||||
After the daily folder scan (when **Update on library scan** is enabled), Aniworld validates every series folder against its `tvshow.nfo` metadata. If the folder name does not match the expected convention, it is automatically renamed.
|
||||
|
||||
**Format:**
|
||||
|
||||
```
|
||||
{title} ({year})
|
||||
```
|
||||
|
||||
**Examples:**
|
||||
|
||||
| NFO `<title>` | NFO `<year>` | Expected Folder Name |
|
||||
|---------------|--------------|----------------------|
|
||||
| `Attack on Titan` | `2013` | `Attack on Titan (2013)` |
|
||||
| `One Piece` | `1999` | `One Piece (1999)` |
|
||||
| `Demon Slayer: Kimetsu no Yaiba` | `2019` | `Demon Slayer Kimetsu no Yaiba (2019)` |
|
||||
|
||||
### 5.2 Sanitization Rules
|
||||
|
||||
Illegal filesystem characters are removed or replaced to ensure cross-platform compatibility:
|
||||
|
||||
- Removed: `< > : " / \ | ? *` and null bytes
|
||||
- Control characters stripped
|
||||
- Multiple spaces collapsed to one
|
||||
- Leading/trailing dots and whitespace trimmed
|
||||
- Maximum length: 200 characters (truncated at word boundary if possible)
|
||||
|
||||
### 5.3 Skip Conditions
|
||||
|
||||
A folder is **not** renamed when any of the following apply:
|
||||
|
||||
- `tvshow.nfo` is missing `<title>` or `<year>` (or they are empty)
|
||||
- The series has an **active or pending download**
|
||||
- The target folder name already exists (duplicate)
|
||||
- The resulting path would exceed the OS path-length limit
|
||||
- The app lacks write permission to the anime directory
|
||||
|
||||
All skipped and renamed actions are logged.
|
||||
|
||||
---
|
||||
|
||||
## 6. Poster Check
|
||||
|
||||
### 6.1 Overview
|
||||
|
||||
During the daily folder scan, Aniworld checks every series folder for a valid `poster.jpg`. If the file is missing or smaller than 1 KB, the application attempts to re-download it from the URL stored in the series' `tvshow.nfo` file.
|
||||
|
||||
### 6.2 How It Works
|
||||
|
||||
1. **Scan** — After folder renaming, the scan iterates over all series folders that contain a `tvshow.nfo`.
|
||||
2. **Validate** — For each folder, it checks whether `poster.jpg` exists and is at least 1 KB.
|
||||
3. **Parse NFO** — If the poster is missing or too small, the scan reads `tvshow.nfo` and looks for a `<thumb aspect="poster">` (or any `<thumb>`) URL.
|
||||
4. **Download** — If a URL is found, the poster is downloaded using `ImageDownloader` with a concurrency limit of 3 simultaneous downloads.
|
||||
5. **Validate Download** — The downloaded image is validated with PIL to ensure it is not corrupted.
|
||||
|
||||
### 6.3 Skip Conditions
|
||||
|
||||
A folder is **not** processed for poster download when any of the following apply:
|
||||
|
||||
- `tvshow.nfo` does not exist in the folder.
|
||||
- `poster.jpg` already exists and is ≥ 1 KB.
|
||||
- No `<thumb>` URL is found in the NFO (the NFO may have been created before thumb tags were added).
|
||||
- The `nfo.download_poster` setting is `false` (poster checks are still performed, but downloads are skipped if the setting is disabled; see [CONFIGURATION.md](CONFIGURATION.md)).
|
||||
|
||||
### 6.4 Logging
|
||||
|
||||
Every poster check action is logged:
|
||||
|
||||
- **INFO** — When a poster is successfully downloaded.
|
||||
- **WARNING** — When a download fails or no URL is found.
|
||||
- **ERROR** — When an unexpected exception occurs during download.
|
||||
|
||||
---
|
||||
|
||||
## 7. API Reference
|
||||
|
||||
### 5.1 Check NFO Status
|
||||
|
||||
**Endpoint**: `GET /api/nfo/check`
|
||||
|
||||
**Query Parameters**:
|
||||
|
||||
- `folder_path` (required): Absolute path to anime directory
|
||||
|
||||
**Response**:
|
||||
|
||||
```json
|
||||
{
|
||||
"has_tvshow_nfo": true,
|
||||
"episode_nfos": [
|
||||
{
|
||||
"season": 1,
|
||||
"episode": 1,
|
||||
"has_nfo": true,
|
||||
"file_path": "/path/to/S01E01.nfo"
|
||||
}
|
||||
],
|
||||
"missing_episodes": [],
|
||||
"total_episodes": 25,
|
||||
"nfo_count": 25
|
||||
}
|
||||
```
|
||||
|
||||
### 5.2 Create NFO Files
|
||||
|
||||
**Endpoint**: `POST /api/nfo/create`
|
||||
|
||||
**Request Body**:
|
||||
|
||||
```json
|
||||
{
|
||||
"anime_id": 123,
|
||||
"folder_path": "/path/to/anime/Attack on Titan"
|
||||
}
|
||||
```
|
||||
|
||||
**Response**:
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"message": "NFO files created successfully",
|
||||
"files_created": ["tvshow.nfo", "S01E01.nfo", "S01E02.nfo"],
|
||||
"images_downloaded": ["poster.jpg", "S01E01-thumb.jpg"]
|
||||
}
|
||||
```
|
||||
|
||||
### 5.3 Update NFO Files
|
||||
|
||||
**Endpoint**: `POST /api/nfo/update`
|
||||
|
||||
**Request Body**:
|
||||
|
||||
```json
|
||||
{
|
||||
"anime_id": 123,
|
||||
"folder_path": "/path/to/anime",
|
||||
"force": false
|
||||
}
|
||||
```
|
||||
|
||||
**Response**:
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"message": "NFO files updated successfully",
|
||||
"files_updated": ["tvshow.nfo", "S01E01.nfo"]
|
||||
}
|
||||
```
|
||||
|
||||
### 5.4 View NFO Content
|
||||
|
||||
**Endpoint**: `GET /api/nfo/view`
|
||||
|
||||
**Query Parameters**:
|
||||
|
||||
- `file_path` (required): Absolute path to NFO file
|
||||
|
||||
**Response**:
|
||||
|
||||
```json
|
||||
{
|
||||
"content": "<?xml version=\"1.0\"...?>",
|
||||
"file_path": "/path/to/tvshow.nfo",
|
||||
"exists": true
|
||||
}
|
||||
```
|
||||
|
||||
### 5.5 Get Media Status
|
||||
|
||||
**Endpoint**: `GET /api/nfo/media/status`
|
||||
|
||||
**Query Parameters**:
|
||||
|
||||
- `folder_path` (required): Absolute path to anime directory
|
||||
|
||||
**Response**:
|
||||
|
||||
```json
|
||||
{
|
||||
"poster_exists": true,
|
||||
"poster_path": "/path/to/poster.jpg",
|
||||
"logo_exists": false,
|
||||
"logo_path": null,
|
||||
"fanart_exists": true,
|
||||
"fanart_path": "/path/to/fanart.jpg",
|
||||
"episode_thumbs": [
|
||||
{
|
||||
"season": 1,
|
||||
"episode": 1,
|
||||
"exists": true,
|
||||
"path": "/path/to/S01E01-thumb.jpg"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### 5.6 Download Media
|
||||
|
||||
**Endpoint**: `POST /api/nfo/media/download`
|
||||
|
||||
**Request Body**:
|
||||
|
||||
```json
|
||||
{
|
||||
"folder_path": "/path/to/anime",
|
||||
"anime_id": 123,
|
||||
"download_poster": true,
|
||||
"download_logo": false,
|
||||
"download_fanart": false,
|
||||
"image_size": "w500"
|
||||
}
|
||||
```
|
||||
|
||||
**Response**:
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"message": "Media downloaded successfully",
|
||||
"downloaded": ["poster.jpg", "S01E01-thumb.jpg"]
|
||||
}
|
||||
```
|
||||
|
||||
### 5.7 Batch Create NFO
|
||||
|
||||
**Endpoint**: `POST /api/nfo/batch/create`
|
||||
|
||||
**Request Body**:
|
||||
|
||||
```json
|
||||
{
|
||||
"anime_ids": [123, 456, 789]
|
||||
}
|
||||
```
|
||||
|
||||
**Response**:
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"results": [
|
||||
{
|
||||
"anime_id": 123,
|
||||
"success": true,
|
||||
"message": "Created successfully"
|
||||
},
|
||||
{
|
||||
"anime_id": 456,
|
||||
"success": false,
|
||||
"error": "Folder not found"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### 5.8 Find Missing NFOs
|
||||
|
||||
**Endpoint**: `GET /api/nfo/missing`
|
||||
|
||||
**Response**:
|
||||
|
||||
```json
|
||||
{
|
||||
"anime_list": [
|
||||
{
|
||||
"anime_id": 123,
|
||||
"title": "Attack on Titan",
|
||||
"folder_path": "/path/to/anime/Attack on Titan",
|
||||
"missing_tvshow_nfo": false,
|
||||
"missing_episode_count": 3,
|
||||
"total_episodes": 25
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Troubleshooting
|
||||
|
||||
### 6.1 NFO Files Not Created
|
||||
|
||||
**Problem**: NFO files are not being created during downloads.
|
||||
|
||||
**Solutions**:
|
||||
|
||||
1. Verify TMDB API key is configured correctly
|
||||
2. Check `auto_create` is enabled in settings
|
||||
3. Ensure anime directory has write permissions
|
||||
4. Check logs for error messages
|
||||
5. Test TMDB connection using "Test Connection" button
|
||||
|
||||
### 6.2 Invalid TMDB API Key
|
||||
|
||||
**Problem**: TMDB validation fails with "Invalid API key".
|
||||
|
||||
**Solutions**:
|
||||
|
||||
1. Verify API key is copied correctly (no extra spaces)
|
||||
2. Ensure you're using the v3 API key (not v4)
|
||||
3. Check API key is active on TMDB website
|
||||
4. Try regenerating API key on TMDB
|
||||
|
||||
### 6.3 Images Not Downloading
|
||||
|
||||
**Problem**: NFO files are created but images are missing.
|
||||
|
||||
**Solutions**:
|
||||
|
||||
1. Enable image downloads in settings (poster/logo/fanart)
|
||||
2. Verify TMDB API key is valid
|
||||
3. Check network connectivity to TMDB servers
|
||||
4. Ensure sufficient disk space
|
||||
5. Check file permissions in anime directory
|
||||
|
||||
### 6.4 Incorrect Metadata
|
||||
|
||||
**Problem**: NFO contains wrong show information.
|
||||
|
||||
**Solutions**:
|
||||
|
||||
1. Verify anime title matches TMDB exactly
|
||||
2. Use TMDB ID if available for accurate matching
|
||||
3. Update NFO files with `force=true` to refresh metadata
|
||||
4. Check TMDB website for correct show information
|
||||
|
||||
### 6.5 Permission Errors
|
||||
|
||||
**Problem**: "Permission denied" when creating NFO files.
|
||||
|
||||
**Solutions**:
|
||||
|
||||
1. Check anime directory permissions: `chmod 755 /path/to/anime`
|
||||
2. Ensure application user has write access
|
||||
3. Verify directory ownership: `chown -R user:group /path/to/anime`
|
||||
4. Check parent directories are accessible
|
||||
|
||||
### 6.6 Slow NFO Creation
|
||||
|
||||
**Problem**: NFO creation takes a long time.
|
||||
|
||||
**Solutions**:
|
||||
|
||||
1. Reduce image size (use w500 instead of original)
|
||||
2. Disable unnecessary images (logo, fanart)
|
||||
3. Create NFOs in batches during off-peak hours
|
||||
4. Check network speed to TMDB servers
|
||||
5. Verify disk I/O performance
|
||||
|
||||
### 6.7 TMDB Lookup Fails for My Series
|
||||
|
||||
**Problem**: TMDB search fails with "No results found" for a valid series.
|
||||
|
||||
**Solutions**:
|
||||
|
||||
1. **Check if series exists on TMDB**: Visit https://www.themoviedb.org and search for your series
|
||||
2. **Use manual ID override**: Add TMDB ID directly to `tvshow.nfo`:
|
||||
```xml
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<tvshow>
|
||||
<title>Your Series Name</title>
|
||||
<tmdbid>12345</tmdbid>
|
||||
<uniqueid type="tmdb">12345</uniqueid>
|
||||
</tvshow>
|
||||
```
|
||||
Aniworld will use this ID directly instead of searching.
|
||||
|
||||
3. **Try alternative titles**: Some anime have different titles (Japanese, romaji, English). If you have access to the folder, rename it to match the TMDB title.
|
||||
|
||||
4. **Add to existing NFO**: If `tvshow.nfo` exists but has no TMDB ID, edit it to add:
|
||||
```xml
|
||||
<tmdbid>YOUR_TMDB_ID</tmdbid>
|
||||
```
|
||||
Then use the Update endpoint to refresh metadata.
|
||||
|
||||
5. **Check for rate limiting**: If many lookups fail at once, you may be hitting TMDB rate limits. Wait and retry later.
|
||||
|
||||
6. **Verify API key**: Ensure your TMDB API key is valid and has not exceeded usage limits.
|
||||
|
||||
---
|
||||
|
||||
## 7. Best Practices
|
||||
|
||||
### 7.1 Configuration Recommendations
|
||||
|
||||
- **Image Size**: Use `w500` for optimal balance of quality and storage
|
||||
- **Auto-create**: Enable for new downloads
|
||||
- **Update on scan**: Disable to avoid unnecessary TMDB API calls
|
||||
- **Poster**: Always enable for show and episode thumbnails
|
||||
- **Logo/Fanart**: Enable only if your media server supports them
|
||||
|
||||
### 7.2 Maintenance
|
||||
|
||||
- **Regular Updates**: Update NFO files quarterly to get latest metadata
|
||||
- **Backup**: Include NFO files in your backup strategy
|
||||
- **Validation**: Periodically check missing NFOs using `/api/nfo/missing`
|
||||
- **API Rate Limits**: Be mindful of TMDB API rate limits when batch processing
|
||||
|
||||
### 7.3 Performance
|
||||
|
||||
- **Batch Operations**: Use batch endpoints for multiple anime
|
||||
- **Off-Peak Processing**: Create NFOs during low-activity periods
|
||||
- **Image Optimization**: Use smaller image sizes for large libraries
|
||||
- **Selective Updates**: Only update NFOs when metadata changes
|
||||
|
||||
### 7.4 Media Server Integration
|
||||
|
||||
#### Plex
|
||||
|
||||
- Use "Personal Media Shows" agent
|
||||
- Enable "Local Media Assets" scanner
|
||||
- Place NFO files in anime directories
|
||||
- Refresh metadata after creating NFOs
|
||||
|
||||
#### Jellyfin
|
||||
|
||||
- Use "NFO" metadata provider
|
||||
- Enable in Library settings
|
||||
- Order providers: NFO first, then online sources
|
||||
- Scan library after NFO creation
|
||||
|
||||
#### Emby
|
||||
|
||||
- Enable "NFO" metadata reader
|
||||
- Configure in Library advanced settings
|
||||
- Use "Prefer embedded metadata" option
|
||||
- Refresh metadata after updates
|
||||
|
||||
#### Kodi
|
||||
|
||||
- NFO files are automatically detected
|
||||
- No additional configuration needed
|
||||
- Update library to see changes
|
||||
|
||||
---
|
||||
|
||||
## 8. Advanced Usage
|
||||
|
||||
### 8.1 Custom NFO Templates
|
||||
|
||||
You can customize NFO generation by modifying the NFO service:
|
||||
|
||||
```python
|
||||
# src/core/services/nfo_creator.py
|
||||
def generate_tvshow_nfo(self, metadata: dict) -> str:
|
||||
# Add custom fields or modify structure
|
||||
pass
|
||||
```
|
||||
|
||||
### 8.2 Bulk Operations
|
||||
|
||||
Create NFOs for entire library:
|
||||
|
||||
```bash
|
||||
# Get all anime without NFOs
|
||||
curl -X GET "http://127.0.0.1:8000/api/nfo/missing" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
| jq -r '.anime_list[].anime_id' \
|
||||
| xargs -I{} curl -X POST "http://127.0.0.1:8000/api/nfo/batch/create" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"anime_ids": [{}]}'
|
||||
```
|
||||
|
||||
### 8.3 Scheduled Updates
|
||||
|
||||
Use the scheduler API to refresh NFOs automatically:
|
||||
|
||||
```bash
|
||||
# Schedule weekly NFO updates (rescan runs Sunday at 03:00)
|
||||
curl -X POST "http://127.0.0.1:8000/api/scheduler/config" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"enabled": true,
|
||||
"schedule_time": "03:00",
|
||||
"schedule_days": ["sun"],
|
||||
"auto_download_after_rescan": false
|
||||
}'
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 9. Related Documentation
|
||||
|
||||
- [API.md](API.md) - Complete API reference
|
||||
- [CONFIGURATION.md](CONFIGURATION.md) - All configuration options
|
||||
- [ARCHITECTURE.md](ARCHITECTURE.md) - System architecture
|
||||
- [DEVELOPMENT.md](DEVELOPMENT.md) - Development guide
|
||||
|
||||
---
|
||||
|
||||
## 10. Tag Reference
|
||||
|
||||
The table below lists every XML tag written to `tvshow.nfo` and its source in
|
||||
the TMDB API response. All tags are written whenever the NFO is created or
|
||||
updated via `create_tvshow_nfo()` / `update_tvshow_nfo()`.
|
||||
|
||||
| NFO tag | TMDB source field | Required |
|
||||
| --------------- | ----------------------------------------------------- | -------- |
|
||||
| `title` | `name` | ✅ |
|
||||
| `originaltitle` | `original_name` | ✅ |
|
||||
| `showtitle` | `name` (same as `title`) | ✅ |
|
||||
| `sorttitle` | `name` (same as `title`) | ✅ |
|
||||
| `year` | First 4 chars of `first_air_date` | ✅ |
|
||||
| `plot` | `overview` | ✅ |
|
||||
| `outline` | `overview` (same as `plot`) | ✅ |
|
||||
| `tagline` | `tagline` | optional |
|
||||
| `runtime` | `episode_run_time[0]` | ✅ |
|
||||
| `premiered` | `first_air_date` | ✅ |
|
||||
| `status` | `status` | ✅ |
|
||||
| `mpaa` | US content rating from `content_ratings.results` | optional |
|
||||
| `fsk` | DE content rating (written as `mpaa` when preferred) | optional |
|
||||
| `imdbid` | `external_ids.imdb_id` | ✅ |
|
||||
| `tmdbid` | `id` | ✅ |
|
||||
| `tvdbid` | `external_ids.tvdb_id` | optional |
|
||||
| `genre` | `genres[].name` (one element per genre) | ✅ |
|
||||
| `studio` | `networks[].name` (one element per network) | ✅ |
|
||||
| `country` | `origin_country[]` or `production_countries[].name` | ✅ |
|
||||
| `actor` | `credits.cast[]` (top 10, with name/role/thumb) | ✅ |
|
||||
| `watched` | Always `false` on creation | ✅ |
|
||||
| `dateadded` | System clock at creation time (`YYYY-MM-DD HH:MM:SS`) | ✅ |
|
||||
|
||||
The mapping logic lives in `src/core/utils/nfo_mapper.py` (`tmdb_to_nfo_model`).
|
||||
The XML serialisation lives in `src/core/utils/nfo_generator.py`
|
||||
(`generate_tvshow_nfo`).
|
||||
|
||||
---
|
||||
|
||||
## 11. Automatic NFO Repair
|
||||
|
||||
NFO repair now runs as part of the scheduled daily folder scan rather than on every
|
||||
startup. When the scheduler triggers `FolderScanService.run_folder_scan()`, the first
|
||||
step is `perform_nfo_repair_scan(background_loader=None)`. Each incomplete NFO is
|
||||
queued as a background `asyncio` task, so the scan returns quickly while repairs
|
||||
continue asynchronously.
|
||||
|
||||
### How It Works
|
||||
|
||||
1. **Scan** — `perform_nfo_repair_scan()` in
|
||||
`src/server/services/initialization_service.py` is called from
|
||||
`FolderScanService.run_folder_scan()` (`src/server/services/folder_scan_service.py`).
|
||||
2. **Detect** — `nfo_needs_repair(nfo_path)` from
|
||||
`src/core/services/nfo_repair_service.py` parses each `tvshow.nfo` with
|
||||
`lxml` and checks for the 13 required tags listed below.
|
||||
3. **Repair** — Series whose NFO is incomplete are queued for background reload
|
||||
via `asyncio.create_task`. Each task creates its own isolated
|
||||
:class:`NFOService` / :class:`TMDBClient` so concurrent tasks never share an
|
||||
``aiohttp`` session — this prevents "Connector is closed" errors when many repairs
|
||||
run in parallel. A semaphore caps TMDB concurrency at 3 to stay within rate limits.
|
||||
|
||||
### Tags Checked (13 required)
|
||||
|
||||
| XPath | Tag name |
|
||||
| ----------------- | --------------- |
|
||||
| `./title` | `title` |
|
||||
| `./originaltitle` | `originaltitle` |
|
||||
| `./year` | `year` |
|
||||
| `./plot` | `plot` |
|
||||
| `./runtime` | `runtime` |
|
||||
| `./premiered` | `premiered` |
|
||||
| `./status` | `status` |
|
||||
| `./imdbid` | `imdbid` |
|
||||
| `./genre` | `genre` |
|
||||
| `./studio` | `studio` |
|
||||
| `./country` | `country` |
|
||||
| `./actor/name` | `actor/name` |
|
||||
| `./watched` | `watched` |
|
||||
|
||||
### Log Messages
|
||||
|
||||
| Message | Meaning |
|
||||
| ----------------------------------------------------------- | ------------------------------------------------- |
|
||||
| `NFO repair scan complete: 0 of N series queued for repair` | All NFOs are complete — no action needed |
|
||||
| `NFO repair scan complete: X of N series queued for repair` | X series had incomplete NFOs and have been queued |
|
||||
| `NFO repair scan skipped: TMDB API key not configured` | Set `tmdb_api_key` in `data/config.json` |
|
||||
| `NFO repair scan skipped: anime directory not configured` | Set `anime_directory` in `data/config.json` |
|
||||
|
||||
### Triggering a Manual Repair
|
||||
|
||||
You can also repair a single series on demand via the API:
|
||||
|
||||
```http
|
||||
POST /api/nfo/update/{series_key}
|
||||
```
|
||||
|
||||
This calls `NFOService.update_tvshow_nfo()` directly and overwrites the existing
|
||||
`tvshow.nfo` with fresh data from TMDB.
|
||||
|
||||
### Source Files
|
||||
|
||||
| File | Purpose |
|
||||
| ----------------------------------------------- | ---------------------------------------------------------------------------------------------- |
|
||||
| `src/core/services/nfo_repair_service.py` | `REQUIRED_TAGS`, `parse_nfo_tags`, `find_missing_tags`, `nfo_needs_repair`, `NfoRepairService` |
|
||||
| `src/server/services/folder_scan_service.py` | `perform_nfo_repair_scan` — invoked during the scheduled daily folder scan |
|
||||
|
||||
---
|
||||
|
||||
## 12. Support
|
||||
|
||||
### Getting Help
|
||||
|
||||
- Check logs in `logs/` directory for error details
|
||||
- Review [TESTING.md](TESTING.md) for test coverage
|
||||
- Consult [DATABASE.md](DATABASE.md) for NFO status schema
|
||||
|
||||
### Common Issues
|
||||
|
||||
See section 6 (Troubleshooting) for solutions to common problems.
|
||||
|
||||
### TMDB Resources
|
||||
|
||||
- TMDB API Documentation: https://developers.themoviedb.org/3
|
||||
- TMDB Support: https://www.themoviedb.org/talk
|
||||
- TMDB API Status: https://status.themoviedb.org/
|
||||
146
Docs/TESTING.md
Normal file
146
Docs/TESTING.md
Normal file
@@ -0,0 +1,146 @@
|
||||
# Testing Documentation
|
||||
|
||||
## Document Purpose
|
||||
|
||||
This document describes the testing strategy, guidelines, and practices for the Aniworld project.
|
||||
|
||||
### What This Document Contains
|
||||
|
||||
- **Testing Strategy**: Overall approach to quality assurance
|
||||
- **Test Categories**: Unit, integration, API, performance, security tests
|
||||
- **Test Structure**: Organization of test files and directories
|
||||
- **Writing Tests**: Guidelines for writing effective tests
|
||||
- **Fixtures and Mocking**: Shared test utilities and mock patterns
|
||||
- **Running Tests**: Commands and configurations
|
||||
- **Coverage Requirements**: Minimum coverage thresholds
|
||||
- **CI/CD Integration**: How tests run in automation
|
||||
- **Test Data Management**: Managing test fixtures and data
|
||||
- **Best Practices**: Do's and don'ts for testing
|
||||
|
||||
### What This Document Does NOT Contain
|
||||
|
||||
- Production deployment (see [DEPLOYMENT.md](DEPLOYMENT.md))
|
||||
- Security audit procedures (see [SECURITY.md](SECURITY.md))
|
||||
- Bug tracking and issue management
|
||||
- Performance benchmarking results
|
||||
|
||||
### Target Audience
|
||||
|
||||
- Developers writing tests
|
||||
- QA Engineers
|
||||
- CI/CD Engineers
|
||||
- Code reviewers
|
||||
|
||||
---
|
||||
|
||||
## Sections to Document
|
||||
|
||||
1. Testing Philosophy
|
||||
- Test pyramid approach
|
||||
- Quality gates
|
||||
2. Test Categories
|
||||
- Unit Tests (`tests/unit/`)
|
||||
- Integration Tests (`tests/integration/`)
|
||||
- API Tests (`tests/api/`)
|
||||
- Frontend Tests (`tests/frontend/`)
|
||||
- Performance Tests (`tests/performance/`)
|
||||
- Security Tests (`tests/security/`)
|
||||
3. Test Structure and Naming
|
||||
- File naming conventions
|
||||
- Test function naming
|
||||
- Test class organization
|
||||
4. Running Tests
|
||||
- pytest commands
|
||||
- Running specific tests
|
||||
- Verbose output
|
||||
- Coverage reports
|
||||
5. Fixtures and Conftest
|
||||
- Shared fixtures
|
||||
- Database fixtures
|
||||
- Mock services
|
||||
6. Mocking Guidelines
|
||||
- What to mock
|
||||
- Mock patterns
|
||||
- External service mocks
|
||||
|
||||
### Mocking the Download Queue
|
||||
|
||||
Use `MockQueueRepository` for testing download queue functionality:
|
||||
|
||||
```python
|
||||
from src.server.models.download import DownloadItem, EpisodeIdentifier
|
||||
|
||||
class MockQueueRepository:
|
||||
def __init__(self):
|
||||
self._items: Dict[str, DownloadItem] = {}
|
||||
```
|
||||
|
||||
### Testing SetupService
|
||||
|
||||
SetupService handles series key resolution from folder names during library setup. Test file: `tests/unit/test_setup_service.py`.
|
||||
|
||||
Key methods tested:
|
||||
- `_extract_year_from_folder_name()` — parses `(YYYY)` suffix
|
||||
- `_extract_title_from_folder_name()` — strips year suffix
|
||||
- `_resolve_key_via_search()` — resolves provider key via fuzzy title matching
|
||||
|
||||
```python
|
||||
@pytest.mark.asyncio
|
||||
async def test_returns_key_when_single_exact_match(self):
|
||||
"""Search returns 1 result with same name → returns key."""
|
||||
mock_series_app = AsyncMock()
|
||||
mock_series_app.search.return_value = [
|
||||
{'title': 'Attack on Titan', 'link': '/anime/stream/attack-on-titan'}
|
||||
]
|
||||
|
||||
with patch('src.server.services.setup_service.get_series_app', return_value=mock_series_app):
|
||||
result = await SetupService._resolve_key_via_search("Attack on Titan")
|
||||
|
||||
assert result == 'attack-on-titan'
|
||||
```
|
||||
|
||||
### Mocking aiohttp Sessions
|
||||
|
||||
When testing code that uses `aiohttp.ClientSession`:
|
||||
|
||||
```python
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from aiohttp import ClientSession
|
||||
|
||||
# Mock aiohttp session for testing
|
||||
class MockAiohttpSession:
|
||||
def __init__(self):
|
||||
self.closed = False
|
||||
|
||||
async def close(self):
|
||||
self.closed = True
|
||||
|
||||
def get(self, url, **kwargs):
|
||||
mock_response = AsyncMock()
|
||||
mock_response.status = 200
|
||||
mock_response.json = AsyncMock(return_value={"data": "test"})
|
||||
mock_response.__aenter__ = AsyncMock(return_value=mock_response)
|
||||
mock_response.__aexit__ = AsyncMock(return_value=None)
|
||||
return mock_response
|
||||
|
||||
# Use in fixture
|
||||
@pytest.fixture
|
||||
async def mock_tmdb_session():
|
||||
session = MockAiohttpSession()
|
||||
yield session
|
||||
# Cleanup verification
|
||||
assert session.closed, "Session was not closed"
|
||||
```
|
||||
|
||||
**Key points:**
|
||||
- Always verify `session.closed` is `True` after context manager exits
|
||||
- Mock `__aenter__` and `__aexit__` for response context managers
|
||||
- Set `closed = False` on mock session for unclosed warning tests
|
||||
|
||||
7. Coverage Requirements
|
||||
8. CI/CD Integration
|
||||
9. Writing Good Tests
|
||||
- Arrange-Act-Assert pattern
|
||||
- Test isolation
|
||||
- Edge cases
|
||||
10. Common Pitfalls to Avoid
|
||||
@@ -31,14 +31,16 @@ flowchart TB
|
||||
|
||||
subgraph Core["Core Layer"]
|
||||
SeriesApp["SeriesApp"]
|
||||
SeriesCache["SeriesCache<br/>(In-Memory)"]
|
||||
SerieScanner["SerieScanner"]
|
||||
SerieList["SerieList"]
|
||||
end
|
||||
|
||||
subgraph Data["Data Layer"]
|
||||
SQLite[(SQLite<br/>aniworld.db)]
|
||||
SQLite[("SQLite<br/>aniworld.db")]
|
||||
ConfigJSON[(config.json)]
|
||||
FileSystem[(File System<br/>Anime Directory)]
|
||||
FileSystem[(File System<br/>Anime Episodes)]
|
||||
LegacyFiles[("Legacy Files<br/>key/data<br/>(Deprecated)")]
|
||||
end
|
||||
|
||||
subgraph External["External"]
|
||||
@@ -71,9 +73,13 @@ flowchart TB
|
||||
AnimeService --> SQLite
|
||||
|
||||
%% Core to Data
|
||||
SeriesApp --> SeriesCache
|
||||
SeriesCache -.->|Cached Series| SQLite
|
||||
SeriesApp --> SerieScanner
|
||||
SeriesApp --> SerieList
|
||||
SerieScanner --> FileSystem
|
||||
SerieScanner -->|Scan Episodes| FileSystem
|
||||
SerieScanner -->|Detect Series| SQLite
|
||||
SerieScanner -->|Migrate Legacy| LegacyFiles
|
||||
SerieScanner --> Provider
|
||||
|
||||
%% Event flow
|
||||
118
Docs/features.md
Normal file
118
Docs/features.md
Normal file
@@ -0,0 +1,118 @@
|
||||
# Aniworld Web Application Features
|
||||
|
||||
## Recent Updates
|
||||
|
||||
### Enhanced Setup and Settings Pages (Latest)
|
||||
|
||||
The application now features a comprehensive configuration system that allows users to configure all settings during initial setup or modify them later through the settings modal:
|
||||
|
||||
**Setup Page Enhancements:**
|
||||
|
||||
- Single-page setup with all configuration options organized into clear sections
|
||||
- Real-time password strength indicator for security
|
||||
- Form validation with helpful error messages
|
||||
- Comprehensive settings including: general, security, scheduler, logging, backup, and NFO metadata
|
||||
|
||||
**Settings Modal Enhancements:**
|
||||
|
||||
- All configuration fields are now editable through the main application's config modal
|
||||
- Organized into logical sections with clear labels and help text
|
||||
- Real-time saving with immediate feedback
|
||||
- Configuration validation to prevent invalid settings
|
||||
- Full control over cron-based scheduler (time, days of week, auto-download), logging options, and backup settings
|
||||
|
||||
---
|
||||
|
||||
## Authentication & Security
|
||||
|
||||
- **Master Password Login**: Secure access to the application with a master password system
|
||||
- **JWT Token Sessions**: Stateless authentication with JSON Web Tokens
|
||||
- **Rate Limiting**: Built-in protection against brute force attacks
|
||||
|
||||
## Configuration Management
|
||||
|
||||
- **Enhanced Setup Page**: Comprehensive initial configuration interface with all settings in one place:
|
||||
- General Settings: Application name and data directory configuration
|
||||
- Security Settings: Master password setup with strength indicator
|
||||
- Anime Directory: Primary directory path for anime storage
|
||||
- Scheduler Settings: Enable/disable scheduler, configure daily run time, select days of week, and optionally auto-download missing episodes after rescan
|
||||
- Logging Settings: Configure log level, file path, file size limits, and backup count
|
||||
- Backup Settings: Enable automatic backups with configurable path and retention period
|
||||
- NFO Settings: TMDB API key, auto-creation options, and media file download preferences
|
||||
- **Enhanced Settings/Config Modal**: Comprehensive configuration interface accessible from main page:
|
||||
- General Settings: Edit application name and data directory
|
||||
- Anime Directory: Modify anime storage location with browse functionality
|
||||
- Scheduler Configuration: Enable/disable, set cron run time (`HH:MM`), select active days of the week, and toggle auto-download after rescan
|
||||
- Logging Configuration: Full control over logging level, file rotation, and backup count
|
||||
- Backup Configuration: Configure automatic backup settings including path and retention
|
||||
- NFO Settings: Complete control over TMDB integration and media file downloads
|
||||
- Configuration Validation: Validate configuration for errors before saving
|
||||
- Backup Management: Create, restore, and manage configuration backups
|
||||
- Export/Import: Export configuration for backup or transfer to another instance
|
||||
|
||||
## User Interface
|
||||
|
||||
- **Dark Mode**: Toggle between light and dark themes for better user experience
|
||||
- **Responsive Design**: Mobile-friendly interface with touch support
|
||||
- **Real-time Updates**: WebSocket-based live notifications and progress tracking
|
||||
|
||||
## Anime Management
|
||||
|
||||
- **Anime Library Page**: Display list of anime series with missing episodes
|
||||
- **Library Filters**:
|
||||
- "Missing Episodes Only" (shows only series with missing episodes, including series that currently have no downloaded episodes)
|
||||
- "No Episodes" (shows series that are present in the library but have zero downloaded episodes)
|
||||
- "Show All Series" (overrides other filters to show every series)
|
||||
- **Database-Backed Series Storage**: All series metadata and missing episodes stored in SQLite database
|
||||
- **Automatic Database Synchronization**: Series loaded from database on startup, stays in sync with filesystem
|
||||
- **Series Selection**: Select individual anime series and add episodes to download queue
|
||||
- **Anime Search**: Search for anime series using integrated providers
|
||||
- **Library Scanning**: Automated scanning for missing episodes with database persistence
|
||||
- **Episode Tracking**: Missing episodes tracked in database, automatically updated during scans
|
||||
- **NFO Status Indicators**: Visual badges showing NFO and media file status for each series
|
||||
|
||||
## NFO Metadata Management
|
||||
|
||||
- **TMDB Integration**: Automatic metadata fetching from The Movie Database (TMDB)
|
||||
- **Auto-Create NFO Files**: Automatically generate tvshow.nfo files during downloads
|
||||
- **Media File Downloads**: Automatic download of poster.jpg, logo.png, and fanart.jpg
|
||||
- **NFO Status Tracking**: Database tracking of NFO creation and update timestamps
|
||||
- **Manual NFO Creation**: Create NFO files and download media for existing anime
|
||||
- **NFO Updates**: Update existing NFO files with latest TMDB metadata
|
||||
- **Batch Operations**: Create NFO files for multiple anime at once
|
||||
- **NFO Content Viewing**: View generated NFO file content in the UI
|
||||
- **Media Server Compatibility**: Kodi, Plex, Jellyfin, and Emby compatible format
|
||||
- **Configuration Options**: Customize which media files to download and image quality
|
||||
|
||||
## Download Management
|
||||
|
||||
- **Download Queue Page**: View and manage the current download queue with organized sections
|
||||
- **Queue Organization**: Displays downloads organized by status (pending, active, completed, failed)
|
||||
- **NFO Integration**: Automatic NFO and media file creation before episode downloads
|
||||
- **Manual Start/Stop Control**: User manually starts downloads one at a time with Start/Stop buttons
|
||||
- **FIFO Queue Processing**: First-in, first-out queue order (no priority or reordering)
|
||||
- **Single Download Mode**: Only one download active at a time, new downloads must be manually started
|
||||
- **Download Status Display**: Real-time status updates and progress of current download
|
||||
- **Queue Operations**: Add and remove items from the pending queue
|
||||
- **Completed Downloads List**: Separate section for completed downloads with clear button
|
||||
- **Failed Downloads List**: Separate section for failed downloads with retry and clear options
|
||||
- **Retry Failed Downloads**: Automatically retry failed downloads with configurable limits
|
||||
- **Clear Completed**: Remove completed downloads from the queue
|
||||
- **Clear Failed**: Remove failed downloads from the queue
|
||||
- **Queue Statistics**: Real-time counters for pending, active, completed, and failed items
|
||||
|
||||
## Real-time Communication
|
||||
|
||||
- **WebSocket Support**: Real-time notifications for download progress and queue updates
|
||||
- **Progress Tracking**: Live progress updates for downloads and scans
|
||||
- **System Notifications**: Real-time system messages and alerts
|
||||
|
||||
## Folder Management
|
||||
|
||||
- **Fuzzy Series Key Resolution**: Automatic series key resolution from folder names using fuzzy title matching — tolerates title variations like `(TV)`, `(OVA)`, `(Movie)` suffixes and uses similarity matching to resolve provider keys during library setup
|
||||
|
||||
## Core Functionality Overview
|
||||
|
||||
The web application provides a complete interface for managing anime downloads with user-friendly pages for configuration, library management, search capabilities, and download monitoring. All operations are tracked in real-time with comprehensive progress reporting and error handling.
|
||||
|
||||
**NFO Metadata Features**: The application now includes full support for generating Kodi/Plex/Jellyfin/Emby compatible metadata files (tvshow.nfo) with automatic TMDB integration. NFO files are created automatically during downloads or can be managed manually through the UI. The system tracks NFO status in the database and provides comprehensive API endpoints for programmatic access. Media files (poster, logo, fanart) are automatically downloaded based on configuration settings.
|
||||
@@ -8,38 +8,47 @@ The goal is to create a FastAPI-based web application that provides a modern int
|
||||
|
||||
## Architecture Principles
|
||||
|
||||
- **Single Responsibility**: Each file/class has one clear purpose
|
||||
- **Dependency Injection**: Use FastAPI's dependency system
|
||||
- **Clean Separation**: Web layer calls core logic, never the reverse
|
||||
- **File Size Limit**: Maximum 500 lines per file
|
||||
- **Type Hints**: Use comprehensive type annotations
|
||||
- **Error Handling**: Proper exception handling and logging
|
||||
- **Single Responsibility**: Each file/class has one clear purpose
|
||||
- **Dependency Injection**: Use FastAPI's dependency system
|
||||
- **Clean Separation**: Web layer calls core logic, never the reverse
|
||||
- **File Size Limit**: Maximum 500 lines per file
|
||||
- **Type Hints**: Use comprehensive type annotations
|
||||
- **Error Handling**: Proper exception handling and logging
|
||||
|
||||
## Additional Implementation Guidelines
|
||||
|
||||
### Code Style and Standards
|
||||
|
||||
- **Type Hints**: Use comprehensive type annotations throughout all modules
|
||||
- **Docstrings**: Follow PEP 257 for function and class documentation
|
||||
- **Error Handling**: Implement custom exception classes with meaningful messages
|
||||
- **Logging**: Use structured logging with appropriate log levels
|
||||
- **Security**: Validate all inputs and sanitize outputs
|
||||
- **Performance**: Use async/await patterns for I/O operations
|
||||
- **Type Hints**: Use comprehensive type annotations throughout all modules
|
||||
- **Docstrings**: Follow PEP 257 for function and class documentation
|
||||
- **Error Handling**: Implement custom exception classes with meaningful messages
|
||||
- **Logging**: Use structured logging with appropriate log levels
|
||||
- **Security**: Validate all inputs and sanitize outputs
|
||||
- **Performance**: Use async/await patterns for I/O operations
|
||||
|
||||
## 📞 Escalation
|
||||
|
||||
If you encounter:
|
||||
|
||||
- Architecture issues requiring design decisions
|
||||
- Tests that conflict with documented requirements
|
||||
- Breaking changes needed
|
||||
- Unclear requirements or expectations
|
||||
- Architecture issues requiring design decisions
|
||||
- Tests that conflict with documented requirements
|
||||
- Breaking changes needed
|
||||
- Unclear requirements or expectations
|
||||
|
||||
**Document the issue and escalate rather than guessing.**
|
||||
|
||||
---
|
||||
|
||||
## 📚 Helpful Commands
|
||||
## <EFBFBD> Credentials
|
||||
|
||||
**Admin Login:**
|
||||
|
||||
- Username: `admin`
|
||||
- Password: `Hallo123!`
|
||||
|
||||
---
|
||||
|
||||
## <20>📚 Helpful Commands
|
||||
|
||||
```bash
|
||||
# Run all tests
|
||||
@@ -86,24 +95,25 @@ conda run -n AniWorld python -m uvicorn src.server.fastapi_app:app --host 127.0.
|
||||
7. **Monitoring**: Implement comprehensive monitoring and alerting
|
||||
8. **Maintenance**: Plan for regular maintenance and updates
|
||||
|
||||
---
|
||||
|
||||
## Task Completion Checklist
|
||||
|
||||
For each task completed:
|
||||
|
||||
- [ ] Implementation follows coding standards
|
||||
- [ ] Unit tests written and passing
|
||||
- [ ] Integration tests passing
|
||||
- [ ] Documentation updated
|
||||
- [ ] Error handling implemented
|
||||
- [ ] Logging added
|
||||
- [ ] Security considerations addressed
|
||||
- [ ] Performance validated
|
||||
- [ ] Code reviewed
|
||||
- [ ] Task marked as complete in instructions.md
|
||||
- [ ] Infrastructure.md updated and other docs
|
||||
- [ ] Changes committed to git; keep your messages in git short and clear
|
||||
- [ ] Take the next task
|
||||
- [ ] Implementation follows coding standards
|
||||
- [ ] Unit tests written and passing
|
||||
- [ ] Integration tests passing
|
||||
- [ ] Documentation updated
|
||||
- [ ] Error handling implemented
|
||||
- [ ] Logging added
|
||||
- [ ] Security considerations addressed
|
||||
- [ ] Performance validated
|
||||
- [ ] Code reviewed
|
||||
- [ ] Task marked as complete in instructions.md
|
||||
- [ ] Infrastructure.md updated and other docs
|
||||
- [ ] Changes committed to git; keep your messages in git short and clear
|
||||
- [ ] Take the next task
|
||||
|
||||
---
|
||||
|
||||
## TODO List:
|
||||
3
Docs/key
Normal file
3
Docs/key
Normal file
@@ -0,0 +1,3 @@
|
||||
API key : 299ae8f630a31bda814263c551361448
|
||||
9bc3e547caff878615cbdba2cc421d37
|
||||
|
||||
154
Docs/runner.csx
Normal file
154
Docs/runner.csx
Normal file
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env dotnet-script
|
||||
#nullable enable
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Diagnostics;
|
||||
using System.Threading;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Linq;
|
||||
using System.Collections.Generic;
|
||||
|
||||
// ── Ctrl+C: kill active process and exit cleanly ──────────────────────────────
|
||||
var cts = new CancellationTokenSource();
|
||||
Process? activeProcess = null;
|
||||
|
||||
Console.CancelKeyPress += (_, e) =>
|
||||
{
|
||||
e.Cancel = true;
|
||||
Console.WriteLine("\n[runner] Interrupted — shutting down...");
|
||||
cts.Cancel();
|
||||
try { activeProcess?.Kill(entireProcessTree: true); } catch { }
|
||||
};
|
||||
|
||||
// ── Paths ─────────────────────────────────────────────────────────────────────
|
||||
var repoRoot = Directory.GetCurrentDirectory();
|
||||
var tasksFile = Path.Combine(repoRoot, "Docs", "Tasks.md");
|
||||
|
||||
if (!File.Exists(tasksFile))
|
||||
{
|
||||
Console.Error.WriteLine($"[runner] ERROR: Tasks.md not found at {tasksFile}");
|
||||
Console.Error.WriteLine("[runner] Run this script from the repository root.");
|
||||
Environment.Exit(1);
|
||||
}
|
||||
|
||||
// ── Read & split by "---" separator lines ────────────────────────────────────
|
||||
var content = File.ReadAllText(tasksFile);
|
||||
var items = Regex
|
||||
.Split(content, @"\r?\n---\r?\n")
|
||||
.Select(s => s.Trim())
|
||||
.Where(s => s.Length > 0)
|
||||
.ToList();
|
||||
|
||||
Console.WriteLine($"[runner] Found {items.Count} section(s) in Tasks.md");
|
||||
|
||||
// ── Helper: run copilot and stream output, return full output ─────────────────
|
||||
async Task<string> RunCopilot(IEnumerable<string> extraArgs, string prompt)
|
||||
{
|
||||
var output = new StringBuilder();
|
||||
|
||||
var argList = new List<string> { "launch", "copilot", "--model", "minimax-m2.7:cloud", "--yes", "--", "--allow-all-tools" };
|
||||
argList.AddRange(extraArgs);
|
||||
argList.Add("-p");
|
||||
argList.Add(prompt);
|
||||
|
||||
var psi = new ProcessStartInfo("ollama")
|
||||
{
|
||||
WorkingDirectory = repoRoot,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
};
|
||||
foreach (var a in argList)
|
||||
psi.ArgumentList.Add(a);
|
||||
|
||||
activeProcess = new Process { StartInfo = psi };
|
||||
|
||||
activeProcess.OutputDataReceived += (_, e) =>
|
||||
{
|
||||
if (e.Data is null) return;
|
||||
Console.WriteLine(e.Data);
|
||||
output.AppendLine(e.Data);
|
||||
};
|
||||
activeProcess.ErrorDataReceived += (_, e) =>
|
||||
{
|
||||
if (e.Data is null) return;
|
||||
Console.Error.WriteLine(e.Data);
|
||||
output.AppendLine(e.Data);
|
||||
};
|
||||
|
||||
activeProcess.Start();
|
||||
activeProcess.BeginOutputReadLine();
|
||||
activeProcess.BeginErrorReadLine();
|
||||
|
||||
await activeProcess.WaitForExitAsync(cts.Token);
|
||||
activeProcess = null;
|
||||
|
||||
return output.ToString();
|
||||
}
|
||||
|
||||
// ── Main loop ─────────────────────────────────────────────────────────────────
|
||||
for (int i = 0; i < items.Count; i++)
|
||||
{
|
||||
var item = items[i];
|
||||
if (cts.IsCancellationRequested) break;
|
||||
|
||||
Console.WriteLine();
|
||||
Console.WriteLine("[runner] ══════════════════════════════════════════════");
|
||||
Console.WriteLine($"[runner] Task:\n{item}");
|
||||
Console.WriteLine("[runner] ══════════════════════════════════════════════");
|
||||
Console.WriteLine();
|
||||
|
||||
// Step 1 — run the task prompt
|
||||
await RunCopilot(Enumerable.Empty<string>(), $"/caveman full");
|
||||
await RunCopilot(new[] { "--continue" }, $"read ./Docs/instructions.md. {item}");
|
||||
if (cts.IsCancellationRequested) break;
|
||||
|
||||
// Step 2 — confirm completion in the same chat session
|
||||
Console.WriteLine("\n[runner] Asking for task confirmation...\n");
|
||||
var confirmation = await RunCopilot(
|
||||
new[] { "--continue" },
|
||||
"are you sure tasks is done. reply with yes"
|
||||
);
|
||||
if (cts.IsCancellationRequested) break;
|
||||
|
||||
// Step 3 — check for "yes" in the reply, with retry logic for issue resolution
|
||||
int maxRetries = 3;
|
||||
int retryCount = 0;
|
||||
bool taskConfirmed = confirmation.Contains("yes", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
while (!taskConfirmed && retryCount < maxRetries)
|
||||
{
|
||||
retryCount++;
|
||||
Console.WriteLine($"\n[runner] Attempt {retryCount}/{maxRetries}: Resolving remaining issues and running tests...\n");
|
||||
|
||||
confirmation = await RunCopilot(
|
||||
new[] { "--continue" },
|
||||
"resolve any remaining issues, make sure all tests are running and pass. then confirm with yes if done"
|
||||
);
|
||||
if (cts.IsCancellationRequested) break;
|
||||
|
||||
taskConfirmed = confirmation.Contains("yes", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
if (!taskConfirmed)
|
||||
{
|
||||
Console.WriteLine($"\n[runner] Task not confirmed as done after {maxRetries} attempts. Stopping.");
|
||||
break;
|
||||
}
|
||||
|
||||
// Step 4 — commit the work
|
||||
Console.WriteLine("\n[runner] Task confirmed. Making git commit...\n");
|
||||
|
||||
await RunCopilot(Enumerable.Empty<string>(), $"/caveman full");
|
||||
await RunCopilot(new[] { "--continue" }, "make git commit");
|
||||
if (cts.IsCancellationRequested) break;
|
||||
|
||||
// Step 5 — remove completed task from Tasks.md
|
||||
var remaining = items.Skip(i + 1).ToList();
|
||||
File.WriteAllText(tasksFile, string.Join("\n\n---\n\n", remaining));
|
||||
Console.WriteLine("[runner] Removed completed task from Tasks.md");
|
||||
}
|
||||
|
||||
Console.WriteLine("\n[runner] Finished.");
|
||||
178
Docs/tasks.md
Normal file
178
Docs/tasks.md
Normal file
@@ -0,0 +1,178 @@
|
||||
# Tasks
|
||||
|
||||
## 1. Scheduled Folder Scan
|
||||
|
||||
### Task 1.1: Add folder scan scheduler configuration
|
||||
|
||||
**Where is that found**
|
||||
- `src/server/models/config.py` (`SchedulerConfig`)
|
||||
- `data/config.json` (example/default config)
|
||||
- `src/server/web/templates/setup.html` (setup UI)
|
||||
- `src/server/api/auth.py` (config save endpoint, if it validates scheduler fields)
|
||||
|
||||
**Goal. How it should be**
|
||||
Add a new boolean field `folder_scan_enabled` (default `false`) to `SchedulerConfig`. When `true`, the scheduler will execute the folder maintenance routine during its scheduled run. Add the field to the setup page as a checkbox. Ensure existing configs without this field load successfully (Pydantic default handles this).
|
||||
|
||||
**Possible traps and issues**
|
||||
- Backward compatibility: old `data/config.json` files must load without errors. Pydantic defaults solve this, but verify by loading an old config.
|
||||
- The setup page JavaScript must include the new field in the payload sent to `/api/config`.
|
||||
- Do not confuse this with `auto_download_after_rescan` — this is a separate toggle.
|
||||
|
||||
**Docs changes needed**
|
||||
- `docs/CONFIGURATION.md`: Document the new `scheduler.folder_scan_enabled` option.
|
||||
- `docs/ARCHITECTURE.md`: Mention folder scan in the scheduler section.
|
||||
|
||||
**Why this is needed**
|
||||
Users need an opt-in toggle to enable automatic daily folder maintenance (NFO repair, folder renaming, poster checks) without forcing it on everyone.
|
||||
|
||||
---
|
||||
|
||||
### Task 1.2: Create FolderScanService skeleton
|
||||
|
||||
**Where is that found**
|
||||
- New file: `src/server/services/folder_scan_service.py`
|
||||
- `src/server/services/scheduler_service.py` (to call it)
|
||||
|
||||
**Goal. How it should be**
|
||||
Create a new `FolderScanService` class with a single async entry point `async def run_folder_scan(self) -> None`. The method should:
|
||||
1. Log start/completion with structlog.
|
||||
2. Check prerequisites (`settings.anime_directory` exists, `settings.tmdb_api_key` is set).
|
||||
3. Skip gracefully with a warning log if prerequisites are missing.
|
||||
4. Use a module-level semaphore (similar to `_NFO_REPAIR_SEMAPHORE`) to limit concurrent TMDB operations to 3.
|
||||
|
||||
Keep the implementation empty for the sub-tasks (1.3–1.5) to fill in. Just add the skeleton and the semaphore.
|
||||
|
||||
**Possible traps and issues**
|
||||
- Circular imports: `folder_scan_service.py` will import from `initialization_service`, `config.settings`, etc. Keep imports inside methods or at the bottom if circular issues arise.
|
||||
- The service should follow the singleton pattern like `SchedulerService` and `DownloadService` if it holds state, or be stateless. For simplicity, make it a plain class instantiated per call or a module-level function set.
|
||||
- Exception handling: any unhandled exception in the scheduled task should be caught and logged so it doesn't crash the scheduler.
|
||||
|
||||
**Docs changes needed**
|
||||
- `docs/ARCHITECTURE.md`: Add `folder_scan_service.py` to the services list.
|
||||
|
||||
**Why this is needed**
|
||||
Encapsulates the new daily maintenance logic in its own module, keeping `scheduler_service.py` clean and allowing the folder scan to be tested independently.
|
||||
|
||||
---
|
||||
|
||||
### Task 1.3: Integrate NFO repair into folder scan
|
||||
|
||||
**Where is that found**
|
||||
- `src/server/services/folder_scan_service.py`
|
||||
- `src/server/services/initialization_service.py` (`perform_nfo_repair_scan`)
|
||||
|
||||
**Goal. How it should be**
|
||||
Inside `FolderScanService.run_folder_scan()`, call `perform_nfo_repair_scan(background_loader=None)` as the first step. Reuse the existing function exactly — do not copy its logic. Log a message before and after the call.
|
||||
|
||||
**Possible traps and issues**
|
||||
- `perform_nfo_repair_scan` spawns `asyncio.create_task` for each repair. When called from the scheduler, these background tasks will still run after `run_folder_scan` returns. This is fine, but log that repairs are queued.
|
||||
- The function already handles missing `tmdb_api_key` and `anime_directory`, so the caller doesn't need to double-check, but the skeleton from Task 1.2 already checks prerequisites.
|
||||
- `perform_nfo_repair_scan` imports `nfo_needs_repair` and `NfoRepairService` inside the function, so no heavy import-time dependencies.
|
||||
|
||||
**Docs changes needed**
|
||||
- `docs/NFO_GUIDE.md`: Update the "Automatic NFO Repair" section to state that repair now runs as part of the scheduled folder scan instead of every startup.
|
||||
|
||||
**Why this is needed**
|
||||
Reuses the existing, tested NFO repair logic. Moves NFO repair from startup blocking to scheduled background maintenance.
|
||||
|
||||
---
|
||||
|
||||
### Task 1.4: Validate and rename series folders
|
||||
|
||||
**Where is that found**
|
||||
- `src/server/services/folder_scan_service.py`
|
||||
- `src/core/services/nfo_repair_service.py` (for `parse_nfo_tags` or similar NFO parsing)
|
||||
- `src/server/database/models.py` / `src/server/database/system_settings_service.py` (if folder paths are stored in DB)
|
||||
|
||||
**Goal. How it should be**
|
||||
After NFO repair, iterate over every subfolder in `settings.anime_directory` that contains a `tvshow.nfo`. For each folder:
|
||||
1. Parse the NFO to extract `<title>` and `<year>` text values.
|
||||
2. Compute the expected folder name: `f"{title} ({year})"`.
|
||||
3. Sanitize the expected name for filesystem safety (remove/replace illegal characters like `/`, `\`, `:`, etc.).
|
||||
4. Compare with the current folder name (`series_dir.name`).
|
||||
5. If different, rename the folder using `series_dir.rename(expected_path)`.
|
||||
6. If the series path is stored in the database (check `anime_service` or DB models), update the database record to point to the new path.
|
||||
|
||||
Skip folders where title or year is missing/empty. Log every rename action.
|
||||
|
||||
**Possible traps and issues**
|
||||
- **Database path consistency**: If `Series` or `Episode` models store absolute or relative paths, renaming the folder on disk without updating the DB will break downloads, NFO updates, and the web UI. Must verify whether paths are stored in the DB and update them.
|
||||
- **Active downloads**: A series currently being downloaded should not be renamed. Check the download queue or lock status before renaming. If no lock mechanism exists, this is a major trap — document it.
|
||||
- **Filesystem permissions**: The app may not have write permission to the anime directory. Catch `PermissionError` and `OSError` and log gracefully.
|
||||
- **Special characters**: Titles like `"A / B"` or `"Show: Subtitle"` contain characters illegal in folder names. Define a sanitization function (e.g., replace `/` with `-`, remove trailing dots on Windows, etc.).
|
||||
- **Duplicate names**: Two different series could sanitize to the same name. Check if target path already exists before renaming.
|
||||
- **Path length limits**: Very long titles might exceed OS path limits.
|
||||
|
||||
**Docs changes needed**
|
||||
- `docs/NFO_GUIDE.md`: Add a section "Folder Naming Convention" explaining the `<title> (<year>)` format.
|
||||
- `docs/CONFIGURATION.md`: Mention that enabling folder scan will rename folders.
|
||||
|
||||
**Why this is needed**
|
||||
Enforces a consistent, predictable folder naming scheme across the library, making it easier for media center apps (Kodi, Jellyfin, Plex) to match metadata.
|
||||
|
||||
---
|
||||
|
||||
### Task 1.5: Check and download missing poster.jpg
|
||||
|
||||
**Where is that found**
|
||||
- `src/server/services/folder_scan_service.py`
|
||||
- `src/core/utils/image_downloader.py` (`ImageDownloader`)
|
||||
- `src/core/services/nfo_service.py` or `src/core/services/nfo_repair_service.py` (to get poster URL from NFO or TMDB)
|
||||
|
||||
**Goal. How it should be**
|
||||
After folder renaming, iterate over series folders again (or combine with Task 1.4 loop). For each folder:
|
||||
1. Check if `poster.jpg` exists and has a size ≥ `ImageDownloader.min_file_size` (1 KB by default).
|
||||
2. If missing or too small:
|
||||
a. Parse `tvshow.nfo` for `<thumb aspect="poster">` or `<thumb>` URL.
|
||||
b. If no URL in NFO, skip (do not query TMDB again to keep tasks small; the NFO should already have it after repair).
|
||||
c. Use `ImageDownloader` (with context manager) to download the image to `series_dir / "poster.jpg"`.
|
||||
d. Validate the downloaded image with `ImageDownloader._validate_image` (or similar existing validation).
|
||||
3. Use the existing `_NFO_REPAIR_SEMAPHORE` or a new `POSTER_DOWNLOAD_SEMAPHORE` to limit concurrent downloads to 3.
|
||||
|
||||
**Possible traps and issues**
|
||||
- **TMDB rate limiting**: Even downloading images hits TMDB CDN. The semaphore limits concurrency.
|
||||
- **Invalid images**: A download might produce a 0-byte or corrupted file. `ImageDownloader` already validates with PIL; reuse that.
|
||||
- **NFO without thumb URL**: If the NFO was created before thumb tags were added, there may be no URL. In that case, skip and log. A future task could query TMDB directly.
|
||||
- **Write permissions**: Same as Task 1.4.
|
||||
- **Async session sharing**: `ImageDownloader` manages its own `aiohttp` session. Use `async with ImageDownloader() as downloader:` to ensure cleanup.
|
||||
|
||||
**Docs changes needed**
|
||||
- `docs/NFO_GUIDE.md`: Add "Poster Check" subsection under folder scan.
|
||||
- `docs/CONFIGURATION.md`: Mention that `nfo.download_poster` setting also affects scheduled poster checks.
|
||||
|
||||
**Why this is needed**
|
||||
Ensures every series has artwork, which is required by most media center front-ends for a polished library view.
|
||||
|
||||
---
|
||||
|
||||
## 2. Remove startup NFO repair
|
||||
|
||||
### Task 2.1: Remove perform_nfo_repair_scan from startup lifespan
|
||||
|
||||
**Where is that found**
|
||||
- `src/server/fastapi_app.py` (lifespan startup block, lines ~245 and ~319)
|
||||
- `src/server/services/initialization_service.py` (keep the function, just remove the call site)
|
||||
- `tests/integration/test_nfo_repair_startup.py`
|
||||
- `tests/unit/test_initialization_service.py` (tests that call `perform_nfo_repair_scan` directly can stay, but integration tests verifying startup wiring must change)
|
||||
|
||||
**Goal. How it should be**
|
||||
1. In `src/server/fastapi_app.py`, remove the import of `perform_nfo_repair_scan` from the `initialization_service` import block.
|
||||
2. Remove the line `await perform_nfo_repair_scan(background_loader)` from the lifespan startup sequence.
|
||||
3. Update `tests/integration/test_nfo_repair_startup.py`:
|
||||
- Remove or modify `test_perform_nfo_repair_scan_imported_in_lifespan` and `test_perform_nfo_repair_scan_called_after_media_scan` since the startup wiring is gone.
|
||||
- Replace with a test that verifies `perform_nfo_repair_scan` is NOT called during startup (or simply delete the file if it has no other purpose).
|
||||
4. `tests/unit/test_initialization_service.py` tests for `perform_nfo_repair_scan` can remain because they test the function itself, not the startup wiring.
|
||||
|
||||
**Possible traps and issues**
|
||||
- **Test failures**: `test_nfo_repair_startup.py` will fail immediately after the code change. It must be updated in the same PR.
|
||||
- **Documentation drift**: `docs/NFO_GUIDE.md`, `docs/CHANGELOG.md`, and `docs/ARCHITECTURE.md` all describe the startup NFO repair behavior. If docs are not updated, users will expect repair on every start.
|
||||
- **Background loader parameter**: The `background_loader` variable was created partly for `perform_nfo_repair_scan`. After removal, check if `background_loader` is still needed for other startup steps (yes — `perform_media_scan_if_needed` uses it). Do not remove `background_loader` entirely.
|
||||
- **Import cleanup**: Ensure no unused imports remain in `fastapi_app.py` after removal.
|
||||
|
||||
**Docs changes needed**
|
||||
- `docs/NFO_GUIDE.md`: Update section 11 "Automatic NFO Repair" to remove startup references and state it runs via scheduler.
|
||||
- `docs/CHANGELOG.md`: Add an entry under "Changed" or "Removed" noting that startup NFO repair is replaced by scheduled folder scan.
|
||||
- `docs/ARCHITECTURE.md`: Update the startup sequence description.
|
||||
|
||||
**Why this is needed**
|
||||
Running `perform_nfo_repair_scan` on every startup slows down server restarts, especially for large libraries. Moving it to a scheduled task keeps startup fast while still ensuring regular maintenance.
|
||||
108
README.md
108
README.md
@@ -4,20 +4,23 @@ A web-based anime download manager with REST API, WebSocket real-time updates, a
|
||||
|
||||
## Features
|
||||
|
||||
- Web interface for managing anime library
|
||||
- REST API for programmatic access
|
||||
- WebSocket real-time progress updates
|
||||
- Download queue with priority management
|
||||
- Automatic library scanning for missing episodes
|
||||
- JWT-based authentication
|
||||
- SQLite database for persistence
|
||||
- Web interface for managing anime library
|
||||
- REST API for programmatic access
|
||||
- WebSocket real-time progress updates
|
||||
- Download queue with priority management
|
||||
- Automatic library scanning for missing episodes
|
||||
- **NFO metadata management with TMDB integration**
|
||||
- **Automatic poster/fanart/logo downloads**
|
||||
- JWT-based authentication
|
||||
- SQLite database for persistence
|
||||
- **Comprehensive test coverage** (1,070+ tests, 91.3% coverage)
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Python 3.10+
|
||||
- Conda (recommended) or virtualenv
|
||||
- Python 3.10+
|
||||
- Conda (recommended) or virtualenv
|
||||
|
||||
### Installation
|
||||
|
||||
@@ -54,7 +57,18 @@ python -m uvicorn src.server.fastapi_app:app --host 127.0.0.1 --port 8000
|
||||
1. Navigate to http://127.0.0.1:8000/setup
|
||||
2. Set a master password (minimum 8 characters, mixed case, number, special character)
|
||||
3. Configure your anime directory path
|
||||
4. Login with your master password
|
||||
4. **(Optional)** Configure NFO settings with your TMDB API key
|
||||
5. Login with your master password
|
||||
|
||||
### NFO Metadata Setup (Optional)
|
||||
|
||||
For automatic NFO file generation with metadata and images:
|
||||
|
||||
1. Get a free TMDB API key from https://www.themoviedb.org/settings/api
|
||||
2. Go to Configuration → NFO Settings in the web interface
|
||||
3. Enter your TMDB API key and click "Test Connection"
|
||||
4. Enable auto-creation and select which images to download
|
||||
5. NFO files will be created automatically during downloads
|
||||
|
||||
## Documentation
|
||||
|
||||
@@ -96,6 +110,8 @@ src/
|
||||
| `POST /api/queue/add` | Add episodes to download queue |
|
||||
| `POST /api/queue/start` | Start queue processing |
|
||||
| `GET /api/queue/status` | Get queue status |
|
||||
| `GET /api/nfo/check` | Check NFO status for anime |
|
||||
| `POST /api/nfo/create` | Create NFO files |
|
||||
| `WS /ws/connect` | WebSocket for real-time updates |
|
||||
|
||||
See [docs/API.md](docs/API.md) for complete API reference.
|
||||
@@ -104,19 +120,22 @@ See [docs/API.md](docs/API.md) for complete API reference.
|
||||
|
||||
Environment variables (via `.env` file):
|
||||
|
||||
| Variable | Default | Description |
|
||||
| ----------------- | ------------------------------ | ---------------------- |
|
||||
| `JWT_SECRET_KEY` | (random) | Secret for JWT signing |
|
||||
| `DATABASE_URL` | `sqlite:///./data/aniworld.db` | Database connection |
|
||||
| `ANIME_DIRECTORY` | (empty) | Path to anime library |
|
||||
| `LOG_LEVEL` | `INFO` | Logging level |
|
||||
| Variable | Default | Description |
|
||||
| ----------------- | ------------------------------ | ------------------------- |
|
||||
| `JWT_SECRET_KEY` | (random) | Secret for JWT signing |
|
||||
| `DATABASE_URL` | `sqlite:///./data/aniworld.db` | Database connection |
|
||||
| `ANIME_DIRECTORY` | (empty) | Path to anime library |
|
||||
| `TMDB_API_KEY` | (empty) | TMDB API key for metadata |
|
||||
| `LOG_LEVEL` | `INFO` | Logging level |
|
||||
|
||||
See [docs/CONFIGURATION.md](docs/CONFIGURATION.md) for all options.
|
||||
|
||||
## Running Tests
|
||||
|
||||
The project includes a comprehensive test suite with **1,070+ tests** and **91.3% coverage** across all critical systems:
|
||||
|
||||
```bash
|
||||
# Run all tests
|
||||
# Run all Python tests
|
||||
conda run -n AniWorld python -m pytest tests/ -v
|
||||
|
||||
# Run unit tests only
|
||||
@@ -124,16 +143,59 @@ conda run -n AniWorld python -m pytest tests/unit/ -v
|
||||
|
||||
# Run integration tests
|
||||
conda run -n AniWorld python -m pytest tests/integration/ -v
|
||||
|
||||
# Run with coverage report
|
||||
conda run -n AniWorld python -m pytest tests/ --cov --cov-report=html
|
||||
|
||||
# Run JavaScript/E2E tests (requires Node.js)
|
||||
npm test # Unit tests (Vitest)
|
||||
npm run test:e2e # E2E tests (Playwright)
|
||||
```
|
||||
|
||||
**Test Coverage:**
|
||||
|
||||
- ✅ 1,070+ tests across 4 priority tiers (644 Python tests passing, 426 JavaScript/E2E tests)
|
||||
- ✅ 91.3% code coverage
|
||||
- ✅ **TIER 1 Critical**: 159/159 tests - Scheduler, NFO batch, download queue, persistence
|
||||
- ✅ **TIER 2 High Priority**: 390/390 tests - Frontend UI, WebSocket, dark mode, settings
|
||||
- ✅ **TIER 3 Medium Priority**: 95/156 tests - Performance, edge cases (core scenarios complete)
|
||||
- ✅ **TIER 4 Polish**: 426 tests - Internationalization, accessibility, media server compatibility
|
||||
- ✅ Security: Complete coverage (authentication, authorization, CSRF, XSS, SQL injection)
|
||||
- ✅ Performance: Validated (200+ concurrent WebSocket clients, batch operations)
|
||||
|
||||
See [docs/TESTING_COMPLETE.md](docs/TESTING_COMPLETE.md) for comprehensive testing documentation.
|
||||
|
||||
## Technology Stack
|
||||
|
||||
- **Web Framework**: FastAPI 0.104.1
|
||||
- **Database**: SQLite + SQLAlchemy 2.0
|
||||
- **Auth**: JWT (python-jose) + passlib
|
||||
- **Validation**: Pydantic 2.5
|
||||
- **Logging**: structlog
|
||||
- **Testing**: pytest + pytest-asyncio
|
||||
- **Web Framework**: FastAPI 0.104.1
|
||||
- **Database**: SQLite + SQLAlchemy 2.0
|
||||
- **Auth**: JWT (python-jose) + passlib
|
||||
- **Validation**: Pydantic 2.5
|
||||
- **Logging**: structlog
|
||||
- **Testing**: pytest + pytest-asyncio
|
||||
|
||||
## Application Lifecycle
|
||||
|
||||
### Initialization
|
||||
|
||||
On first startup, the application performs a one-time sync of series from data files to the database:
|
||||
|
||||
1. FastAPI lifespan starts
|
||||
2. Database is initialized
|
||||
3. `sync_series_from_data_files()` reads all data files from the anime directory (creates temporary SeriesApp)
|
||||
4. Series metadata is synced to the database
|
||||
5. DownloadService initializes (triggers main `SeriesApp` creation)
|
||||
6. `SeriesApp` loads series from database via service layer (not from files)
|
||||
|
||||
On subsequent startups, the same flow applies but the sync finds no new series. `SeriesApp` always initializes with an empty series list (`skip_load=True`) and loads data from the database on demand, avoiding redundant file system scans.
|
||||
|
||||
### Adding New Series
|
||||
|
||||
When adding a new series:
|
||||
|
||||
1. Series is added to the database via `AnimeService`
|
||||
2. Data file is created in the anime directory
|
||||
3. In-memory `SerieList` is updated via `load_series_from_list()`
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -1,105 +0,0 @@
|
||||
# Changelog
|
||||
|
||||
## Document Purpose
|
||||
|
||||
This document tracks all notable changes to the Aniworld project.
|
||||
|
||||
### What This Document Contains
|
||||
|
||||
- **Version History**: All released versions with dates
|
||||
- **Added Features**: New functionality in each release
|
||||
- **Changed Features**: Modifications to existing features
|
||||
- **Deprecated Features**: Features marked for removal
|
||||
- **Removed Features**: Features removed from the codebase
|
||||
- **Fixed Bugs**: Bug fixes with issue references
|
||||
- **Security Fixes**: Security-related changes
|
||||
- **Breaking Changes**: Changes requiring user action
|
||||
|
||||
### What This Document Does NOT Contain
|
||||
|
||||
- Internal refactoring details (unless user-facing)
|
||||
- Commit-level changes
|
||||
- Work-in-progress features
|
||||
- Roadmap or planned features
|
||||
|
||||
### Target Audience
|
||||
|
||||
- All users and stakeholders
|
||||
- Operators planning upgrades
|
||||
- Developers tracking changes
|
||||
- Support personnel
|
||||
|
||||
---
|
||||
|
||||
## Format
|
||||
|
||||
This changelog follows [Keep a Changelog](https://keepachangelog.com/) principles and adheres to [Semantic Versioning](https://semver.org/).
|
||||
|
||||
## Sections for Each Release
|
||||
|
||||
```markdown
|
||||
## [Version] - YYYY-MM-DD
|
||||
|
||||
### Added
|
||||
|
||||
- New features
|
||||
|
||||
### Changed
|
||||
|
||||
- Changes to existing functionality
|
||||
|
||||
### Deprecated
|
||||
|
||||
- Features that will be removed in future versions
|
||||
|
||||
### Removed
|
||||
|
||||
- Features removed in this release
|
||||
|
||||
### Fixed
|
||||
|
||||
- Bug fixes
|
||||
|
||||
### Security
|
||||
|
||||
- Security-related fixes
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Unreleased
|
||||
|
||||
_Changes that are in development but not yet released._
|
||||
|
||||
### Added
|
||||
|
||||
- **Enhanced Anime Add Flow**: Automatic database persistence, targeted episode scanning, and folder creation with sanitized names
|
||||
- Filesystem utility module (`src/server/utils/filesystem.py`) with `sanitize_folder_name()`, `is_safe_path()`, and `create_safe_folder()` functions
|
||||
- `Serie.sanitized_folder` property for generating filesystem-safe folder names from display names
|
||||
- `SerieScanner.scan_single_series()` method for targeted scanning of individual anime without full library rescan
|
||||
- Add series API response now includes `missing_episodes` list and `total_missing` count
|
||||
- Database transaction support with `@transactional` decorator and `atomic()` context manager
|
||||
- Transaction propagation modes (REQUIRED, REQUIRES_NEW, NESTED) for fine-grained control
|
||||
- Savepoint support for nested transactions with partial rollback capability
|
||||
- `TransactionManager` helper class for manual transaction control
|
||||
- Bulk operations: `bulk_mark_downloaded`, `bulk_delete`, `clear_all` for batch processing
|
||||
- `rotate_session` atomic operation for secure session rotation
|
||||
- Transaction utilities: `is_session_in_transaction`, `get_session_transaction_depth`
|
||||
- `get_transactional_session` for sessions without auto-commit
|
||||
|
||||
### Changed
|
||||
|
||||
- `QueueRepository.save_item()` now uses atomic transactions for data consistency
|
||||
- `QueueRepository.clear_all()` now uses atomic transactions for all-or-nothing behavior
|
||||
- Service layer documentation updated to reflect transaction-aware design
|
||||
|
||||
### Fixed
|
||||
|
||||
- Scan status indicator now correctly shows running state after page reload during active scan
|
||||
- Improved reliability of process status updates in the UI header
|
||||
|
||||
---
|
||||
|
||||
## Version History
|
||||
|
||||
_To be documented as versions are released._
|
||||
421
docs/DATABASE.md
421
docs/DATABASE.md
@@ -1,421 +0,0 @@
|
||||
# Database Documentation
|
||||
|
||||
## Document Purpose
|
||||
|
||||
This document describes the database schema, models, and data layer of the Aniworld application.
|
||||
|
||||
---
|
||||
|
||||
## 1. Database Overview
|
||||
|
||||
### Technology
|
||||
|
||||
- **Database Engine**: SQLite 3 (default), PostgreSQL supported
|
||||
- **ORM**: SQLAlchemy 2.0 with async support (aiosqlite)
|
||||
- **Location**: `data/aniworld.db` (configurable via `DATABASE_URL`)
|
||||
|
||||
Source: [src/config/settings.py](../src/config/settings.py#L53-L55)
|
||||
|
||||
### Connection Configuration
|
||||
|
||||
```python
|
||||
# Default connection string
|
||||
DATABASE_URL = "sqlite+aiosqlite:///./data/aniworld.db"
|
||||
|
||||
# PostgreSQL alternative
|
||||
DATABASE_URL = "postgresql+asyncpg://user:pass@localhost/aniworld"
|
||||
```
|
||||
|
||||
Source: [src/server/database/connection.py](../src/server/database/connection.py)
|
||||
|
||||
---
|
||||
|
||||
## 2. Entity Relationship Diagram
|
||||
|
||||
```
|
||||
+-------------------+ +-------------------+ +------------------------+
|
||||
| anime_series | | episodes | | download_queue_item |
|
||||
+-------------------+ +-------------------+ +------------------------+
|
||||
| id (PK) |<--+ | id (PK) | +-->| id (PK, VARCHAR) |
|
||||
| key (UNIQUE) | | | series_id (FK)----+---+ | series_id (FK)---------+
|
||||
| name | +---| | | status |
|
||||
| site | | season | | priority |
|
||||
| folder | | episode_number | | season |
|
||||
| created_at | | title | | episode |
|
||||
| updated_at | | file_path | | progress_percent |
|
||||
+-------------------+ | is_downloaded | | error_message |
|
||||
| created_at | | retry_count |
|
||||
| updated_at | | added_at |
|
||||
+-------------------+ | started_at |
|
||||
| completed_at |
|
||||
| created_at |
|
||||
| updated_at |
|
||||
+------------------------+
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Table Schemas
|
||||
|
||||
### 3.1 anime_series
|
||||
|
||||
Stores anime series metadata.
|
||||
|
||||
| Column | Type | Constraints | Description |
|
||||
| ------------ | ------------- | -------------------------- | ------------------------------------------------------- |
|
||||
| `id` | INTEGER | PRIMARY KEY, AUTOINCREMENT | Internal database ID |
|
||||
| `key` | VARCHAR(255) | UNIQUE, NOT NULL, INDEX | **Primary identifier** - provider-assigned URL-safe key |
|
||||
| `name` | VARCHAR(500) | NOT NULL, INDEX | Display name of the series |
|
||||
| `site` | VARCHAR(500) | NOT NULL | Provider site URL |
|
||||
| `folder` | VARCHAR(1000) | NOT NULL | Filesystem folder name (metadata only) |
|
||||
| `created_at` | DATETIME | NOT NULL, DEFAULT NOW | Record creation timestamp |
|
||||
| `updated_at` | DATETIME | NOT NULL, ON UPDATE NOW | Last update timestamp |
|
||||
|
||||
**Identifier Convention:**
|
||||
|
||||
- `key` is the **primary identifier** for all operations (e.g., `"attack-on-titan"`)
|
||||
- `folder` is **metadata only** for filesystem operations (e.g., `"Attack on Titan (2013)"`)
|
||||
- `id` is used only for database relationships
|
||||
|
||||
Source: [src/server/database/models.py](../src/server/database/models.py#L23-L87)
|
||||
|
||||
### 3.2 episodes
|
||||
|
||||
Stores individual episode information.
|
||||
|
||||
| Column | Type | Constraints | Description |
|
||||
| ---------------- | ------------- | ---------------------------- | ----------------------------- |
|
||||
| `id` | INTEGER | PRIMARY KEY, AUTOINCREMENT | Internal database ID |
|
||||
| `series_id` | INTEGER | FOREIGN KEY, NOT NULL, INDEX | Reference to anime_series.id |
|
||||
| `season` | INTEGER | NOT NULL | Season number (1-based) |
|
||||
| `episode_number` | INTEGER | NOT NULL | Episode number within season |
|
||||
| `title` | VARCHAR(500) | NULLABLE | Episode title if known |
|
||||
| `file_path` | VARCHAR(1000) | NULLABLE | Local file path if downloaded |
|
||||
| `is_downloaded` | BOOLEAN | NOT NULL, DEFAULT FALSE | Download status flag |
|
||||
| `created_at` | DATETIME | NOT NULL, DEFAULT NOW | Record creation timestamp |
|
||||
| `updated_at` | DATETIME | NOT NULL, ON UPDATE NOW | Last update timestamp |
|
||||
|
||||
**Foreign Key:**
|
||||
|
||||
- `series_id` -> `anime_series.id` (ON DELETE CASCADE)
|
||||
|
||||
Source: [src/server/database/models.py](../src/server/database/models.py#L122-L181)
|
||||
|
||||
### 3.3 download_queue_item
|
||||
|
||||
Stores download queue items with status tracking.
|
||||
|
||||
| Column | Type | Constraints | Description |
|
||||
| ------------------ | ------------- | --------------------------- | ------------------------------ |
|
||||
| `id` | VARCHAR(36) | PRIMARY KEY | UUID identifier |
|
||||
| `series_id` | INTEGER | FOREIGN KEY, NOT NULL | Reference to anime_series.id |
|
||||
| `season` | INTEGER | NOT NULL | Season number |
|
||||
| `episode` | INTEGER | NOT NULL | Episode number |
|
||||
| `status` | VARCHAR(20) | NOT NULL, DEFAULT 'pending' | Download status |
|
||||
| `priority` | VARCHAR(10) | NOT NULL, DEFAULT 'NORMAL' | Queue priority |
|
||||
| `progress_percent` | FLOAT | NULLABLE | Download progress (0-100) |
|
||||
| `error_message` | TEXT | NULLABLE | Error description if failed |
|
||||
| `retry_count` | INTEGER | NOT NULL, DEFAULT 0 | Number of retry attempts |
|
||||
| `source_url` | VARCHAR(2000) | NULLABLE | Download source URL |
|
||||
| `added_at` | DATETIME | NOT NULL, DEFAULT NOW | When added to queue |
|
||||
| `started_at` | DATETIME | NULLABLE | When download started |
|
||||
| `completed_at` | DATETIME | NULLABLE | When download completed/failed |
|
||||
| `created_at` | DATETIME | NOT NULL, DEFAULT NOW | Record creation timestamp |
|
||||
| `updated_at` | DATETIME | NOT NULL, ON UPDATE NOW | Last update timestamp |
|
||||
|
||||
**Status Values:** `pending`, `downloading`, `paused`, `completed`, `failed`, `cancelled`
|
||||
|
||||
**Priority Values:** `LOW`, `NORMAL`, `HIGH`
|
||||
|
||||
**Foreign Key:**
|
||||
|
||||
- `series_id` -> `anime_series.id` (ON DELETE CASCADE)
|
||||
|
||||
Source: [src/server/database/models.py](../src/server/database/models.py#L200-L300)
|
||||
|
||||
---
|
||||
|
||||
## 4. Indexes
|
||||
|
||||
| Table | Index Name | Columns | Purpose |
|
||||
| --------------------- | ----------------------- | ----------- | --------------------------------- |
|
||||
| `anime_series` | `ix_anime_series_key` | `key` | Fast lookup by primary identifier |
|
||||
| `anime_series` | `ix_anime_series_name` | `name` | Search by name |
|
||||
| `episodes` | `ix_episodes_series_id` | `series_id` | Join with series |
|
||||
| `download_queue_item` | `ix_download_series_id` | `series_id` | Filter by series |
|
||||
| `download_queue_item` | `ix_download_status` | `status` | Filter by status |
|
||||
|
||||
---
|
||||
|
||||
## 5. Model Layer
|
||||
|
||||
### 5.1 SQLAlchemy ORM Models
|
||||
|
||||
```python
|
||||
# src/server/database/models.py
|
||||
|
||||
class AnimeSeries(Base, TimestampMixin):
|
||||
__tablename__ = "anime_series"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
key: Mapped[str] = mapped_column(String(255), unique=True, index=True)
|
||||
name: Mapped[str] = mapped_column(String(500), index=True)
|
||||
site: Mapped[str] = mapped_column(String(500))
|
||||
folder: Mapped[str] = mapped_column(String(1000))
|
||||
|
||||
episodes: Mapped[List["Episode"]] = relationship(
|
||||
"Episode", back_populates="series", cascade="all, delete-orphan"
|
||||
)
|
||||
```
|
||||
|
||||
Source: [src/server/database/models.py](../src/server/database/models.py#L23-L87)
|
||||
|
||||
### 5.2 Pydantic API Models
|
||||
|
||||
```python
|
||||
# src/server/models/download.py
|
||||
|
||||
class DownloadItem(BaseModel):
|
||||
id: str
|
||||
serie_id: str # Maps to anime_series.key
|
||||
serie_folder: str # Metadata only
|
||||
serie_name: str
|
||||
episode: EpisodeIdentifier
|
||||
status: DownloadStatus
|
||||
priority: DownloadPriority
|
||||
```
|
||||
|
||||
Source: [src/server/models/download.py](../src/server/models/download.py#L63-L118)
|
||||
|
||||
### 5.3 Model Mapping
|
||||
|
||||
| API Field | Database Column | Notes |
|
||||
| -------------- | --------------------- | ------------------ |
|
||||
| `serie_id` | `anime_series.key` | Primary identifier |
|
||||
| `serie_folder` | `anime_series.folder` | Metadata only |
|
||||
| `serie_name` | `anime_series.name` | Display name |
|
||||
|
||||
---
|
||||
|
||||
## 6. Transaction Support
|
||||
|
||||
### 6.1 Overview
|
||||
|
||||
The database layer provides comprehensive transaction support to ensure data consistency across compound operations. All write operations can be wrapped in explicit transactions.
|
||||
|
||||
Source: [src/server/database/transaction.py](../src/server/database/transaction.py)
|
||||
|
||||
### 6.2 Transaction Utilities
|
||||
|
||||
| Component | Type | Description |
|
||||
| ------------------------- | ----------------- | ---------------------------------------- |
|
||||
| `@transactional` | Decorator | Wraps function in transaction boundary |
|
||||
| `atomic()` | Async context mgr | Provides atomic operation block |
|
||||
| `atomic_sync()` | Sync context mgr | Sync version of atomic() |
|
||||
| `TransactionContext` | Class | Explicit sync transaction control |
|
||||
| `AsyncTransactionContext` | Class | Explicit async transaction control |
|
||||
| `TransactionManager` | Class | Helper for manual transaction management |
|
||||
|
||||
### 6.3 Transaction Propagation Modes
|
||||
|
||||
| Mode | Behavior |
|
||||
| -------------- | ------------------------------------------------ |
|
||||
| `REQUIRED` | Use existing transaction or create new (default) |
|
||||
| `REQUIRES_NEW` | Always create new transaction |
|
||||
| `NESTED` | Create savepoint within existing transaction |
|
||||
|
||||
### 6.4 Usage Examples
|
||||
|
||||
**Using @transactional decorator:**
|
||||
|
||||
```python
|
||||
from src.server.database.transaction import transactional
|
||||
|
||||
@transactional()
|
||||
async def compound_operation(db: AsyncSession, data: dict):
|
||||
# All operations commit together or rollback on error
|
||||
series = await AnimeSeriesService.create(db, ...)
|
||||
episode = await EpisodeService.create(db, series_id=series.id, ...)
|
||||
return series, episode
|
||||
```
|
||||
|
||||
**Using atomic() context manager:**
|
||||
|
||||
```python
|
||||
from src.server.database.transaction import atomic
|
||||
|
||||
async def some_function(db: AsyncSession):
|
||||
async with atomic(db) as tx:
|
||||
await operation1(db)
|
||||
await operation2(db)
|
||||
# Auto-commits on success, rolls back on exception
|
||||
```
|
||||
|
||||
**Using savepoints for partial rollback:**
|
||||
|
||||
```python
|
||||
async with atomic(db) as tx:
|
||||
await outer_operation(db)
|
||||
|
||||
async with tx.savepoint() as sp:
|
||||
await risky_operation(db)
|
||||
if error_condition:
|
||||
await sp.rollback() # Only rollback nested ops
|
||||
|
||||
await final_operation(db) # Still executes
|
||||
```
|
||||
|
||||
Source: [src/server/database/transaction.py](../src/server/database/transaction.py)
|
||||
|
||||
### 6.5 Connection Module Additions
|
||||
|
||||
| Function | Description |
|
||||
| ------------------------------- | -------------------------------------------- |
|
||||
| `get_transactional_session` | Session without auto-commit for transactions |
|
||||
| `TransactionManager` | Helper class for manual transaction control |
|
||||
| `is_session_in_transaction` | Check if session is in active transaction |
|
||||
| `get_session_transaction_depth` | Get nesting depth of transactions |
|
||||
|
||||
Source: [src/server/database/connection.py](../src/server/database/connection.py)
|
||||
|
||||
---
|
||||
|
||||
## 7. Repository Pattern
|
||||
|
||||
The `QueueRepository` class provides data access abstraction.
|
||||
|
||||
```python
|
||||
class QueueRepository:
|
||||
async def save_item(self, item: DownloadItem) -> None:
|
||||
"""Save or update a download item (atomic operation)."""
|
||||
|
||||
async def get_all_items(self) -> List[DownloadItem]:
|
||||
"""Get all items from database."""
|
||||
|
||||
async def delete_item(self, item_id: str) -> bool:
|
||||
"""Delete item by ID."""
|
||||
|
||||
async def clear_all(self) -> int:
|
||||
"""Clear all items (atomic operation)."""
|
||||
```
|
||||
|
||||
Note: Compound operations (`save_item`, `clear_all`) are wrapped in `atomic()` transactions.
|
||||
|
||||
Source: [src/server/services/queue_repository.py](../src/server/services/queue_repository.py)
|
||||
|
||||
---
|
||||
|
||||
## 8. Database Service
|
||||
|
||||
The `AnimeSeriesService` provides async CRUD operations.
|
||||
|
||||
```python
|
||||
class AnimeSeriesService:
|
||||
@staticmethod
|
||||
async def create(
|
||||
db: AsyncSession,
|
||||
key: str,
|
||||
name: str,
|
||||
site: str,
|
||||
folder: str
|
||||
) -> AnimeSeries:
|
||||
"""Create a new anime series."""
|
||||
|
||||
@staticmethod
|
||||
async def get_by_key(
|
||||
db: AsyncSession,
|
||||
key: str
|
||||
) -> Optional[AnimeSeries]:
|
||||
"""Get series by primary key identifier."""
|
||||
```
|
||||
|
||||
### Bulk Operations
|
||||
|
||||
Services provide bulk operations for transaction-safe batch processing:
|
||||
|
||||
| Service | Method | Description |
|
||||
| ---------------------- | ---------------------- | ------------------------------ |
|
||||
| `EpisodeService` | `bulk_mark_downloaded` | Mark multiple episodes at once |
|
||||
| `DownloadQueueService` | `bulk_delete` | Delete multiple queue items |
|
||||
| `DownloadQueueService` | `clear_all` | Clear entire queue |
|
||||
| `UserSessionService` | `rotate_session` | Revoke old + create new atomic |
|
||||
| `UserSessionService` | `cleanup_expired` | Bulk delete expired sessions |
|
||||
|
||||
Source: [src/server/database/service.py](../src/server/database/service.py)
|
||||
|
||||
---
|
||||
|
||||
## 9. Data Integrity Rules
|
||||
|
||||
### Validation Constraints
|
||||
|
||||
| Field | Rule | Error Message |
|
||||
| ------------------------- | ------------------------ | ------------------------------------- |
|
||||
| `anime_series.key` | Non-empty, max 255 chars | "Series key cannot be empty" |
|
||||
| `anime_series.name` | Non-empty, max 500 chars | "Series name cannot be empty" |
|
||||
| `episodes.season` | 0-1000 | "Season number must be non-negative" |
|
||||
| `episodes.episode_number` | 0-10000 | "Episode number must be non-negative" |
|
||||
|
||||
Source: [src/server/database/models.py](../src/server/database/models.py#L89-L119)
|
||||
|
||||
### Cascade Rules
|
||||
|
||||
- Deleting `anime_series` deletes all related `episodes` and `download_queue_item`
|
||||
|
||||
---
|
||||
|
||||
## 10. Migration Strategy
|
||||
|
||||
Currently, SQLAlchemy's `create_all()` is used for schema creation.
|
||||
|
||||
```python
|
||||
# src/server/database/connection.py
|
||||
async def init_db():
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
```
|
||||
|
||||
For production migrations, Alembic is recommended but not yet implemented.
|
||||
|
||||
Source: [src/server/database/connection.py](../src/server/database/connection.py)
|
||||
|
||||
---
|
||||
|
||||
## 11. Common Query Patterns
|
||||
|
||||
### Get all series with missing episodes
|
||||
|
||||
```python
|
||||
series = await db.execute(
|
||||
select(AnimeSeries).options(selectinload(AnimeSeries.episodes))
|
||||
)
|
||||
for serie in series.scalars():
|
||||
downloaded = [e for e in serie.episodes if e.is_downloaded]
|
||||
```
|
||||
|
||||
### Get pending downloads ordered by priority
|
||||
|
||||
```python
|
||||
items = await db.execute(
|
||||
select(DownloadQueueItem)
|
||||
.where(DownloadQueueItem.status == "pending")
|
||||
.order_by(
|
||||
case(
|
||||
(DownloadQueueItem.priority == "HIGH", 1),
|
||||
(DownloadQueueItem.priority == "NORMAL", 2),
|
||||
(DownloadQueueItem.priority == "LOW", 3),
|
||||
),
|
||||
DownloadQueueItem.added_at
|
||||
)
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 12. Database Location
|
||||
|
||||
| Environment | Default Location |
|
||||
| ----------- | ------------------------------------------------- |
|
||||
| Development | `./data/aniworld.db` |
|
||||
| Production | Via `DATABASE_URL` environment variable |
|
||||
| Testing | In-memory SQLite (`sqlite+aiosqlite:///:memory:`) |
|
||||
@@ -1,64 +0,0 @@
|
||||
# Development Guide
|
||||
|
||||
## Document Purpose
|
||||
|
||||
This document provides guidance for developers working on the Aniworld project.
|
||||
|
||||
### What This Document Contains
|
||||
|
||||
- **Prerequisites**: Required software and tools
|
||||
- **Environment Setup**: Step-by-step local development setup
|
||||
- **Project Structure**: Source code organization explanation
|
||||
- **Development Workflow**: Branch strategy, commit conventions
|
||||
- **Coding Standards**: Style guide, linting, formatting
|
||||
- **Running the Application**: Development server, CLI usage
|
||||
- **Debugging Tips**: Common debugging approaches
|
||||
- **IDE Configuration**: VS Code settings, recommended extensions
|
||||
- **Contributing Guidelines**: How to submit changes
|
||||
- **Code Review Process**: Review checklist and expectations
|
||||
|
||||
### What This Document Does NOT Contain
|
||||
|
||||
- Production deployment (see [DEPLOYMENT.md](DEPLOYMENT.md))
|
||||
- API reference (see [API.md](API.md))
|
||||
- Architecture decisions (see [ARCHITECTURE.md](ARCHITECTURE.md))
|
||||
- Test writing guides (see [TESTING.md](TESTING.md))
|
||||
- Security guidelines (see [SECURITY.md](SECURITY.md))
|
||||
|
||||
### Target Audience
|
||||
|
||||
- New Developers joining the project
|
||||
- Contributors (internal and external)
|
||||
- Anyone setting up a development environment
|
||||
|
||||
---
|
||||
|
||||
## Sections to Document
|
||||
|
||||
1. Prerequisites
|
||||
- Python version
|
||||
- Conda environment
|
||||
- Node.js (if applicable)
|
||||
- Git
|
||||
2. Getting Started
|
||||
- Clone repository
|
||||
- Setup conda environment
|
||||
- Install dependencies
|
||||
- Configuration setup
|
||||
3. Project Structure Overview
|
||||
4. Development Server
|
||||
- Starting FastAPI server
|
||||
- Hot reload configuration
|
||||
- Debug mode
|
||||
5. CLI Development
|
||||
6. Code Style
|
||||
- PEP 8 compliance
|
||||
- Type hints requirements
|
||||
- Docstring format
|
||||
- Import organization
|
||||
7. Git Workflow
|
||||
- Branch naming
|
||||
- Commit message format
|
||||
- Pull request process
|
||||
8. Common Development Tasks
|
||||
9. Troubleshooting Development Issues
|
||||
@@ -1,71 +0,0 @@
|
||||
# Testing Documentation
|
||||
|
||||
## Document Purpose
|
||||
|
||||
This document describes the testing strategy, guidelines, and practices for the Aniworld project.
|
||||
|
||||
### What This Document Contains
|
||||
|
||||
- **Testing Strategy**: Overall approach to quality assurance
|
||||
- **Test Categories**: Unit, integration, API, performance, security tests
|
||||
- **Test Structure**: Organization of test files and directories
|
||||
- **Writing Tests**: Guidelines for writing effective tests
|
||||
- **Fixtures and Mocking**: Shared test utilities and mock patterns
|
||||
- **Running Tests**: Commands and configurations
|
||||
- **Coverage Requirements**: Minimum coverage thresholds
|
||||
- **CI/CD Integration**: How tests run in automation
|
||||
- **Test Data Management**: Managing test fixtures and data
|
||||
- **Best Practices**: Do's and don'ts for testing
|
||||
|
||||
### What This Document Does NOT Contain
|
||||
|
||||
- Production deployment (see [DEPLOYMENT.md](DEPLOYMENT.md))
|
||||
- Security audit procedures (see [SECURITY.md](SECURITY.md))
|
||||
- Bug tracking and issue management
|
||||
- Performance benchmarking results
|
||||
|
||||
### Target Audience
|
||||
|
||||
- Developers writing tests
|
||||
- QA Engineers
|
||||
- CI/CD Engineers
|
||||
- Code reviewers
|
||||
|
||||
---
|
||||
|
||||
## Sections to Document
|
||||
|
||||
1. Testing Philosophy
|
||||
- Test pyramid approach
|
||||
- Quality gates
|
||||
2. Test Categories
|
||||
- Unit Tests (`tests/unit/`)
|
||||
- Integration Tests (`tests/integration/`)
|
||||
- API Tests (`tests/api/`)
|
||||
- Frontend Tests (`tests/frontend/`)
|
||||
- Performance Tests (`tests/performance/`)
|
||||
- Security Tests (`tests/security/`)
|
||||
3. Test Structure and Naming
|
||||
- File naming conventions
|
||||
- Test function naming
|
||||
- Test class organization
|
||||
4. Running Tests
|
||||
- pytest commands
|
||||
- Running specific tests
|
||||
- Verbose output
|
||||
- Coverage reports
|
||||
5. Fixtures and Conftest
|
||||
- Shared fixtures
|
||||
- Database fixtures
|
||||
- Mock services
|
||||
6. Mocking Guidelines
|
||||
- What to mock
|
||||
- Mock patterns
|
||||
- External service mocks
|
||||
7. Coverage Requirements
|
||||
8. CI/CD Integration
|
||||
9. Writing Good Tests
|
||||
- Arrange-Act-Assert pattern
|
||||
- Test isolation
|
||||
- Edge cases
|
||||
10. Common Pitfalls to Avoid
|
||||
@@ -1,53 +0,0 @@
|
||||
# Aniworld Web Application Features
|
||||
|
||||
## Authentication & Security
|
||||
|
||||
- **Master Password Login**: Secure access to the application with a master password system
|
||||
- **JWT Token Sessions**: Stateless authentication with JSON Web Tokens
|
||||
- **Rate Limiting**: Built-in protection against brute force attacks
|
||||
|
||||
## Configuration Management
|
||||
|
||||
- **Setup Page**: Initial configuration interface for server setup and basic settings
|
||||
- **Config Page**: View and modify application configuration settings
|
||||
- **Scheduler Configuration**: Configure automated rescan schedules
|
||||
- **Backup Management**: Create, restore, and manage configuration backups
|
||||
|
||||
## User Interface
|
||||
|
||||
- **Dark Mode**: Toggle between light and dark themes for better user experience
|
||||
- **Responsive Design**: Mobile-friendly interface with touch support
|
||||
- **Real-time Updates**: WebSocket-based live notifications and progress tracking
|
||||
|
||||
## Anime Management
|
||||
|
||||
- **Anime Library Page**: Display list of anime series with missing episodes
|
||||
- **Series Selection**: Select individual anime series and add episodes to download queue
|
||||
- **Anime Search**: Search for anime series using integrated providers
|
||||
- **Library Scanning**: Automated scanning for missing episodes
|
||||
|
||||
## Download Management
|
||||
|
||||
- **Download Queue Page**: View and manage the current download queue with organized sections
|
||||
- **Queue Organization**: Displays downloads organized by status (pending, active, completed, failed)
|
||||
- **Manual Start/Stop Control**: User manually starts downloads one at a time with Start/Stop buttons
|
||||
- **FIFO Queue Processing**: First-in, first-out queue order (no priority or reordering)
|
||||
- **Single Download Mode**: Only one download active at a time, new downloads must be manually started
|
||||
- **Download Status Display**: Real-time status updates and progress of current download
|
||||
- **Queue Operations**: Add and remove items from the pending queue
|
||||
- **Completed Downloads List**: Separate section for completed downloads with clear button
|
||||
- **Failed Downloads List**: Separate section for failed downloads with retry and clear options
|
||||
- **Retry Failed Downloads**: Automatically retry failed downloads with configurable limits
|
||||
- **Clear Completed**: Remove completed downloads from the queue
|
||||
- **Clear Failed**: Remove failed downloads from the queue
|
||||
- **Queue Statistics**: Real-time counters for pending, active, completed, and failed items
|
||||
|
||||
## Real-time Communication
|
||||
|
||||
- **WebSocket Support**: Real-time notifications for download progress and queue updates
|
||||
- **Progress Tracking**: Live progress updates for downloads and scans
|
||||
- **System Notifications**: Real-time system messages and alerts
|
||||
|
||||
## Core Functionality Overview
|
||||
|
||||
The web application provides a complete interface for managing anime downloads with user-friendly pages for configuration, library management, search capabilities, and download monitoring. All operations are tracked in real-time with comprehensive progress reporting and error handling.
|
||||
@@ -1,131 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Script to fix test files that use old set_broadcast_callback pattern."""
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def fix_file(filepath: Path) -> bool:
|
||||
"""Fix a single test file.
|
||||
|
||||
Args:
|
||||
filepath: Path to the test file
|
||||
|
||||
Returns:
|
||||
True if file was modified, False otherwise
|
||||
"""
|
||||
content = filepath.read_text()
|
||||
original = content
|
||||
|
||||
# Pattern 1: Replace set_broadcast_callback calls
|
||||
# Old: service.set_broadcast_callback(mock_broadcast)
|
||||
# New: progress_service.subscribe("progress_updated", mock_event_handler)
|
||||
|
||||
# Pattern 2: Fix download_service fixture to return tuple
|
||||
if "async def download_service(" in content and "yield service" in content:
|
||||
content = re.sub(
|
||||
r'(async def download_service\([^)]+\):.*?)(yield service)',
|
||||
r'\1yield service, progress_service',
|
||||
content,
|
||||
flags=re.DOTALL
|
||||
)
|
||||
|
||||
#Pattern 3: Unpack download_service in tests
|
||||
if "def test_" in content or "async def test_" in content:
|
||||
# Find tests that use download_service but don't unpack it
|
||||
content = re.sub(
|
||||
r'(async def test_[^\(]+\([^)]*download_service[^)]*\):.*?""".*?""")\s*broadcasts',
|
||||
r'\1\n download_svc, progress_svc = download_service\n broadcasts',
|
||||
content,
|
||||
flags=re.DOTALL,
|
||||
count=1 # Only first occurrence in each test
|
||||
)
|
||||
|
||||
# Pattern 4: Replace set_broadcast_callback with subscribe
|
||||
content = re.sub(
|
||||
r'(\w+)\.set_broadcast_callback\((\w+)\)',
|
||||
r'progress_service.subscribe("progress_updated", \2)',
|
||||
content
|
||||
)
|
||||
|
||||
# Pattern 5: Fix event handler signatures
|
||||
# Old: async def mock_broadcast(message_type: str, room: str, data: dict):
|
||||
# New: async def mock_event_handler(event):
|
||||
content = re.sub(
|
||||
r'async def (mock_broadcast\w*)\([^)]+\):(\s+"""[^"]*""")?(\s+)broadcasts\.append',
|
||||
r'async def mock_event_handler(event):\2\3broadcasts.append',
|
||||
content
|
||||
)
|
||||
|
||||
# Pattern 6: Fix broadcast append calls
|
||||
# Old: broadcasts.append({"type": message_type, "data": data})
|
||||
# New: broadcasts.append({"type": event.event_type, "data": event.progress.to_dict()})
|
||||
content = re.sub(
|
||||
r'broadcasts\.append\(\{[^}]*"type":\s*message_type[^}]*\}\)',
|
||||
'broadcasts.append({"type": event.event_type, "data": event.progress.to_dict()})',
|
||||
content
|
||||
)
|
||||
|
||||
# Pattern 7: Update download_service usage in tests to use unpacked version
|
||||
content = re.sub(
|
||||
r'await download_service\.add_to_queue\(',
|
||||
r'await download_svc.add_to_queue(',
|
||||
content
|
||||
)
|
||||
content = re.sub(
|
||||
r'await download_service\.start',
|
||||
r'await download_svc.start',
|
||||
content
|
||||
)
|
||||
content = re.sub(
|
||||
r'await download_service\.stop',
|
||||
r'await download_svc.stop',
|
||||
content
|
||||
)
|
||||
content = re.sub(
|
||||
r'await download_service\.get_queue_status\(',
|
||||
r'await download_svc.get_queue_status(',
|
||||
content
|
||||
)
|
||||
content = re.sub(
|
||||
r'await download_service\.remove_from_queue\(',
|
||||
r'await download_svc.remove_from_queue(',
|
||||
content
|
||||
)
|
||||
content = re.sub(
|
||||
r'await download_service\.clear_completed\(',
|
||||
r'await download_svc.clear_completed(',
|
||||
content
|
||||
)
|
||||
|
||||
if content != original:
|
||||
filepath.write_text(content)
|
||||
print(f"✓ Fixed {filepath}")
|
||||
return True
|
||||
else:
|
||||
print(f" Skipped {filepath} (no changes needed)")
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function to fix all test files."""
|
||||
test_dir = Path(__file__).parent / "tests"
|
||||
|
||||
# Find all test files that might need fixing
|
||||
test_files = list(test_dir.rglob("test_*.py"))
|
||||
|
||||
print(f"Found {len(test_files)} test files")
|
||||
print("Fixing test files...")
|
||||
|
||||
fixed_count = 0
|
||||
for test_file in test_files:
|
||||
if fix_file(test_file):
|
||||
fixed_count += 1
|
||||
|
||||
print(f"\nFixed {fixed_count}/{len(test_files)} files")
|
||||
return 0 if fixed_count > 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
104
fix_tests.py
104
fix_tests.py
@@ -1,104 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Script to batch fix common test issues after API changes."""
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def fix_add_to_queue_calls(content: str) -> str:
|
||||
"""Add serie_folder parameter to add_to_queue calls."""
|
||||
# Pattern: add_to_queue(\n serie_id="...",
|
||||
# Add: serie_folder="...",
|
||||
pattern = r'(add_to_queue\(\s+serie_id="([^"]+)",)'
|
||||
|
||||
def replace_func(match):
|
||||
serie_id = match.group(2)
|
||||
# Extract just the series name without number if present
|
||||
serie_folder = serie_id.split('-')[0] if '-' in serie_id else serie_id
|
||||
return f'{match.group(1)}\n serie_folder="{serie_folder}",'
|
||||
|
||||
return re.sub(pattern, replace_func, content)
|
||||
|
||||
|
||||
def fix_queue_status_response(content: str) -> str:
|
||||
"""Fix queue status response structure - remove nested 'status' key."""
|
||||
# Replace data["status"]["pending"] with data["pending_queue"]
|
||||
content = re.sub(r'data\["status"\]\["pending"\]', 'data["pending_queue"]', content)
|
||||
content = re.sub(r'data\["status"\]\["active"\]', 'data["active_downloads"]', content)
|
||||
content = re.sub(r'data\["status"\]\["completed"\]', 'data["completed_downloads"]', content)
|
||||
content = re.sub(r'data\["status"\]\["failed"\]', 'data["failed_downloads"]', content)
|
||||
content = re.sub(r'data\["status"\]\["is_running"\]', 'data["is_running"]', content)
|
||||
content = re.sub(r'data\["status"\]\["is_paused"\]', 'data["is_paused"]', content)
|
||||
|
||||
# Also fix response.json()["status"]["..."]
|
||||
content = re.sub(r'response\.json\(\)\["status"\]\["pending"\]', 'response.json()["pending_queue"]', content)
|
||||
content = re.sub(r'response\.json\(\)\["status"\]\["is_running"\]', 'response.json()["is_running"]', content)
|
||||
content = re.sub(r'status\.json\(\)\["status"\]\["is_running"\]', 'status.json()["is_running"]', content)
|
||||
content = re.sub(r'status\.json\(\)\["status"\]\["failed"\]', 'status.json()["failed_downloads"]', content)
|
||||
content = re.sub(r'status\.json\(\)\["status"\]\["completed"\]', 'status.json()["completed_downloads"]', content)
|
||||
|
||||
# Fix assert "status" in data
|
||||
content = re.sub(r'assert "status" in data', 'assert "is_running" in data', content)
|
||||
|
||||
return content
|
||||
|
||||
|
||||
def fix_anime_service_init(content: str) -> str:
|
||||
"""Fix AnimeService initialization in test fixtures."""
|
||||
# This one is complex, so we'll just note files that need manual review
|
||||
if 'AnimeService(' in content and 'directory=' in content:
|
||||
print(" ⚠️ Contains AnimeService with directory= parameter - needs manual review")
|
||||
return content
|
||||
|
||||
|
||||
def main():
|
||||
test_dir = Path(__file__).parent / "tests"
|
||||
|
||||
if not test_dir.exists():
|
||||
print(f"Error: {test_dir} not found")
|
||||
sys.exit(1)
|
||||
|
||||
files_to_fix = [
|
||||
# Download service tests
|
||||
"unit/test_download_service.py",
|
||||
"unit/test_download_progress_websocket.py",
|
||||
"integration/test_download_progress_integration.py",
|
||||
"integration/test_websocket_integration.py",
|
||||
# API tests with queue status
|
||||
"api/test_queue_features.py",
|
||||
"api/test_download_endpoints.py",
|
||||
"frontend/test_existing_ui_integration.py",
|
||||
]
|
||||
|
||||
for file_path in files_to_fix:
|
||||
full_path = test_dir / file_path
|
||||
if not full_path.exists():
|
||||
print(f"Skipping {file_path} (not found)")
|
||||
continue
|
||||
|
||||
print(f"Processing {file_path}...")
|
||||
|
||||
# Read content
|
||||
content = full_path.read_text()
|
||||
original_content = content
|
||||
|
||||
# Apply fixes
|
||||
if 'add_to_queue(' in content:
|
||||
content = fix_add_to_queue_calls(content)
|
||||
|
||||
if 'data["status"]' in content or 'response.json()["status"]' in content:
|
||||
content = fix_queue_status_response(content)
|
||||
|
||||
content = fix_anime_service_init(content)
|
||||
|
||||
# Write back if changed
|
||||
if content != original_content:
|
||||
full_path.write_text(content)
|
||||
print(f" ✓ Updated {file_path}")
|
||||
else:
|
||||
print(f" - No changes needed for {file_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
27
package.json
Normal file
27
package.json
Normal file
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"name": "aniworld-web",
|
||||
"version": "1.4.11",
|
||||
"description": "Aniworld Anime Download Manager - Web Frontend",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest",
|
||||
"test:ui": "vitest --ui",
|
||||
"test:coverage": "vitest run --coverage",
|
||||
"test:e2e": "playwright test",
|
||||
"test:e2e:ui": "playwright test --ui",
|
||||
"test:e2e:headed": "playwright test --headed",
|
||||
"test:e2e:debug": "playwright test --debug",
|
||||
"playwright:install": "playwright install --with-deps chromium"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@playwright/test": "^1.41.0",
|
||||
"@vitest/coverage-v8": "^1.2.0",
|
||||
"@vitest/ui": "^1.2.0",
|
||||
"happy-dom": "^13.3.5",
|
||||
"vitest": "^1.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
}
|
||||
@@ -14,4 +14,15 @@ pytest==7.4.3
|
||||
pytest-asyncio==0.21.1
|
||||
httpx==0.25.2
|
||||
sqlalchemy>=2.0.35
|
||||
aiosqlite>=0.19.0
|
||||
aiosqlite>=0.19.0
|
||||
aiohttp>=3.9.0
|
||||
lxml>=5.0.0
|
||||
pillow>=10.0.0
|
||||
APScheduler>=3.10.4
|
||||
Events>=0.5
|
||||
requests>=2.31.0
|
||||
beautifulsoup4>=4.12.0
|
||||
chardet>=5.2.0
|
||||
fake-useragent>=1.4.0
|
||||
yt-dlp>=2024.1.0
|
||||
urllib3>=2.0.0
|
||||
421
scripts/setup.py
421
scripts/setup.py
@@ -1,421 +0,0 @@
|
||||
"""
|
||||
Aniworld Application Setup Script
|
||||
|
||||
This script handles initial setup, dependency installation, database
|
||||
initialization, and configuration for the Aniworld application.
|
||||
|
||||
Usage:
|
||||
python setup.py [--environment {development|production}] [--no-deps]
|
||||
python setup.py --help
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class SetupManager:
|
||||
"""Manages application setup and initialization."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
environment: str = "development",
|
||||
skip_deps: bool = False
|
||||
):
|
||||
"""
|
||||
Initialize setup manager.
|
||||
|
||||
Args:
|
||||
environment: Environment mode (development or production)
|
||||
skip_deps: Skip dependency installation
|
||||
"""
|
||||
self.environment = environment
|
||||
self.skip_deps = skip_deps
|
||||
self.project_root = Path(__file__).parent.parent
|
||||
self.conda_env = "AniWorld"
|
||||
|
||||
# ============================================================================
|
||||
# Logging
|
||||
# ============================================================================
|
||||
|
||||
@staticmethod
|
||||
def log_info(message: str) -> None:
|
||||
"""Log info message."""
|
||||
print(f"\033[34m[INFO]\033[0m {message}")
|
||||
|
||||
@staticmethod
|
||||
def log_success(message: str) -> None:
|
||||
"""Log success message."""
|
||||
print(f"\033[32m[SUCCESS]\033[0m {message}")
|
||||
|
||||
@staticmethod
|
||||
def log_warning(message: str) -> None:
|
||||
"""Log warning message."""
|
||||
print(f"\033[33m[WARNING]\033[0m {message}")
|
||||
|
||||
@staticmethod
|
||||
def log_error(message: str) -> None:
|
||||
"""Log error message."""
|
||||
print(f"\033[31m[ERROR]\033[0m {message}")
|
||||
|
||||
# ============================================================================
|
||||
# Validation
|
||||
# ============================================================================
|
||||
|
||||
def validate_environment(self) -> bool:
|
||||
"""
|
||||
Validate environment parameter.
|
||||
|
||||
Returns:
|
||||
True if valid, False otherwise
|
||||
"""
|
||||
valid_envs = {"development", "production", "testing"}
|
||||
if self.environment not in valid_envs:
|
||||
self.log_error(
|
||||
f"Invalid environment: {self.environment}. "
|
||||
f"Must be one of: {valid_envs}"
|
||||
)
|
||||
return False
|
||||
self.log_success(f"Environment: {self.environment}")
|
||||
return True
|
||||
|
||||
def check_conda_env(self) -> bool:
|
||||
"""
|
||||
Check if conda environment exists.
|
||||
|
||||
Returns:
|
||||
True if exists, False otherwise
|
||||
"""
|
||||
result = subprocess.run(
|
||||
["conda", "env", "list"],
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
if self.conda_env in result.stdout:
|
||||
self.log_success(f"Conda environment '{self.conda_env}' found")
|
||||
return True
|
||||
self.log_error(
|
||||
f"Conda environment '{self.conda_env}' not found. "
|
||||
f"Create with: conda create -n {self.conda_env} python=3.11"
|
||||
)
|
||||
return False
|
||||
|
||||
def check_python_version(self) -> bool:
|
||||
"""
|
||||
Check Python version.
|
||||
|
||||
Returns:
|
||||
True if version >= 3.9, False otherwise
|
||||
"""
|
||||
if sys.version_info < (3, 9):
|
||||
self.log_error(
|
||||
f"Python 3.9+ required. Current: {sys.version_info.major}."
|
||||
f"{sys.version_info.minor}"
|
||||
)
|
||||
return False
|
||||
self.log_success(
|
||||
f"Python version: {sys.version_info.major}."
|
||||
f"{sys.version_info.minor}"
|
||||
)
|
||||
return True
|
||||
|
||||
# ============================================================================
|
||||
# Directory Setup
|
||||
# ============================================================================
|
||||
|
||||
def create_directories(self) -> bool:
|
||||
"""
|
||||
Create necessary directories.
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
try:
|
||||
directories = [
|
||||
"logs",
|
||||
"data",
|
||||
"data/config_backups",
|
||||
"Temp",
|
||||
"tests",
|
||||
"scripts",
|
||||
]
|
||||
self.log_info("Creating directories...")
|
||||
for directory in directories:
|
||||
dir_path = self.project_root / directory
|
||||
dir_path.mkdir(parents=True, exist_ok=True)
|
||||
self.log_success("Directories created")
|
||||
return True
|
||||
except Exception as e:
|
||||
self.log_error(f"Failed to create directories: {e}")
|
||||
return False
|
||||
|
||||
# ============================================================================
|
||||
# Dependency Installation
|
||||
# ============================================================================
|
||||
|
||||
def install_dependencies(self) -> bool:
|
||||
"""
|
||||
Install Python dependencies.
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
if self.skip_deps:
|
||||
self.log_warning("Skipping dependency installation")
|
||||
return True
|
||||
|
||||
try:
|
||||
requirements_file = self.project_root / "requirements.txt"
|
||||
if not requirements_file.exists():
|
||||
self.log_error(
|
||||
f"requirements.txt not found at {requirements_file}"
|
||||
)
|
||||
return False
|
||||
|
||||
self.log_info("Installing dependencies...")
|
||||
subprocess.run(
|
||||
["conda", "run", "-n", self.conda_env,
|
||||
"pip", "install", "-q", "-r", str(requirements_file)],
|
||||
check=True
|
||||
)
|
||||
self.log_success("Dependencies installed")
|
||||
return True
|
||||
except subprocess.CalledProcessError as e:
|
||||
self.log_error(f"Failed to install dependencies: {e}")
|
||||
return False
|
||||
|
||||
# ============================================================================
|
||||
# Environment Configuration
|
||||
# ============================================================================
|
||||
|
||||
def create_env_files(self) -> bool:
|
||||
"""
|
||||
Create environment configuration files.
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
try:
|
||||
self.log_info("Creating environment configuration files...")
|
||||
|
||||
env_file = self.project_root / f".env.{self.environment}"
|
||||
if env_file.exists():
|
||||
self.log_warning(f"{env_file.name} already exists")
|
||||
return True
|
||||
|
||||
# Create environment file with defaults
|
||||
env_content = self._get_env_template()
|
||||
env_file.write_text(env_content)
|
||||
self.log_success(f"Created {env_file.name}")
|
||||
return True
|
||||
except Exception as e:
|
||||
self.log_error(f"Failed to create env files: {e}")
|
||||
return False
|
||||
|
||||
def _get_env_template(self) -> str:
|
||||
"""
|
||||
Get environment file template.
|
||||
|
||||
Returns:
|
||||
Environment file content
|
||||
"""
|
||||
if self.environment == "production":
|
||||
return """# Aniworld Production Configuration
|
||||
# IMPORTANT: Set these values before running in production
|
||||
|
||||
# Security (REQUIRED - generate new values)
|
||||
JWT_SECRET_KEY=change-this-to-a-secure-random-key
|
||||
PASSWORD_SALT=change-this-to-a-secure-random-salt
|
||||
MASTER_PASSWORD_HASH=change-this-to-hashed-password
|
||||
|
||||
# Database (REQUIRED - use PostgreSQL or MySQL in production)
|
||||
DATABASE_URL=postgresql://user:password@localhost/aniworld
|
||||
DATABASE_POOL_SIZE=20
|
||||
DATABASE_MAX_OVERFLOW=10
|
||||
|
||||
# Application
|
||||
ENVIRONMENT=production
|
||||
ANIME_DIRECTORY=/var/lib/aniworld
|
||||
TEMP_DIRECTORY=/tmp/aniworld
|
||||
|
||||
# Server
|
||||
HOST=0.0.0.0
|
||||
PORT=8000
|
||||
WORKERS=4
|
||||
|
||||
# Security
|
||||
CORS_ORIGINS=https://yourdomain.com
|
||||
ALLOWED_HOSTS=yourdomain.com
|
||||
|
||||
# Logging
|
||||
LOG_LEVEL=WARNING
|
||||
LOG_FILE=logs/production.log
|
||||
LOG_ROTATION_SIZE=10485760
|
||||
LOG_RETENTION_DAYS=30
|
||||
|
||||
# Performance
|
||||
API_RATE_LIMIT=60
|
||||
SESSION_TIMEOUT_HOURS=24
|
||||
MAX_CONCURRENT_DOWNLOADS=3
|
||||
"""
|
||||
else: # development
|
||||
return """# Aniworld Development Configuration
|
||||
|
||||
# Security (Development defaults - NOT for production)
|
||||
JWT_SECRET_KEY=dev-secret-key-change-in-production
|
||||
PASSWORD_SALT=dev-salt-change-in-production
|
||||
MASTER_PASSWORD_HASH=$2b$12$wP0KBVbJKVAb8CdSSXw0NeGTKCkbw4fSAFXIqR2/wDqPSEBn9w7lS
|
||||
MASTER_PASSWORD=password
|
||||
|
||||
# Database
|
||||
DATABASE_URL=sqlite:///./data/aniworld_dev.db
|
||||
|
||||
# Application
|
||||
ENVIRONMENT=development
|
||||
ANIME_DIRECTORY=/tmp/aniworld_dev
|
||||
TEMP_DIRECTORY=/tmp/aniworld_dev/temp
|
||||
|
||||
# Server
|
||||
HOST=127.0.0.1
|
||||
PORT=8000
|
||||
WORKERS=1
|
||||
|
||||
# Security
|
||||
CORS_ORIGINS=*
|
||||
|
||||
# Logging
|
||||
LOG_LEVEL=DEBUG
|
||||
LOG_FILE=logs/development.log
|
||||
|
||||
# Performance
|
||||
API_RATE_LIMIT=1000
|
||||
SESSION_TIMEOUT_HOURS=168
|
||||
MAX_CONCURRENT_DOWNLOADS=1
|
||||
"""
|
||||
|
||||
# ============================================================================
|
||||
# Database Initialization
|
||||
# ============================================================================
|
||||
|
||||
async def init_database(self) -> bool:
|
||||
"""
|
||||
Initialize database.
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
try:
|
||||
self.log_info("Initializing database...")
|
||||
# Import and run database initialization
|
||||
os.chdir(self.project_root)
|
||||
from src.server.database import init_db
|
||||
await init_db()
|
||||
self.log_success("Database initialized")
|
||||
return True
|
||||
except Exception as e:
|
||||
self.log_error(f"Failed to initialize database: {e}")
|
||||
return False
|
||||
|
||||
# ============================================================================
|
||||
# Summary
|
||||
# ============================================================================
|
||||
|
||||
def print_summary(self) -> None:
|
||||
"""Print setup summary."""
|
||||
self.log_info("=" * 50)
|
||||
self.log_info("Setup Summary")
|
||||
self.log_info("=" * 50)
|
||||
self.log_info(f"Environment: {self.environment}")
|
||||
self.log_info(f"Conda Environment: {self.conda_env}")
|
||||
self.log_info(f"Project Root: {self.project_root}")
|
||||
self.log_info("")
|
||||
self.log_success("Setup complete!")
|
||||
self.log_info("")
|
||||
self.log_info("Next steps:")
|
||||
self.log_info("1. Configure .env files with your settings")
|
||||
if self.environment == "production":
|
||||
self.log_info("2. Set up database (PostgreSQL/MySQL)")
|
||||
self.log_info("3. Configure security settings")
|
||||
self.log_info("4. Run: ./scripts/start.sh production")
|
||||
else:
|
||||
self.log_info("2. Run: ./scripts/start.sh development")
|
||||
self.log_info("")
|
||||
|
||||
# ============================================================================
|
||||
# Main Setup
|
||||
# ============================================================================
|
||||
|
||||
async def run(self) -> int:
|
||||
"""
|
||||
Run setup process.
|
||||
|
||||
Returns:
|
||||
0 if successful, 1 otherwise
|
||||
"""
|
||||
print("\033[34m" + "=" * 50 + "\033[0m")
|
||||
print("\033[34mAniworld Application Setup\033[0m")
|
||||
print("\033[34m" + "=" * 50 + "\033[0m")
|
||||
print()
|
||||
|
||||
# Validation
|
||||
if not self.validate_environment():
|
||||
return 1
|
||||
if not self.check_python_version():
|
||||
return 1
|
||||
if not self.check_conda_env():
|
||||
return 1
|
||||
|
||||
# Setup
|
||||
if not self.create_directories():
|
||||
return 1
|
||||
if not self.create_env_files():
|
||||
return 1
|
||||
if not self.install_dependencies():
|
||||
return 1
|
||||
|
||||
# Initialize database
|
||||
if not await self.init_database():
|
||||
return 1
|
||||
|
||||
# Summary
|
||||
self.print_summary()
|
||||
return 0
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
"""
|
||||
Main entry point.
|
||||
|
||||
Returns:
|
||||
Exit code
|
||||
"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Aniworld Application Setup"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--environment",
|
||||
choices=["development", "production", "testing"],
|
||||
default="development",
|
||||
help="Environment to set up (default: development)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-deps",
|
||||
action="store_true",
|
||||
help="Skip dependency installation"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
setup = SetupManager(
|
||||
environment=args.environment,
|
||||
skip_deps=args.no_deps
|
||||
)
|
||||
return await setup.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = asyncio.run(main())
|
||||
sys.exit(exit_code)
|
||||
225
scripts/start.sh
225
scripts/start.sh
@@ -1,225 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
################################################################################
|
||||
# Aniworld Application Startup Script
|
||||
#
|
||||
# This script initializes the development or production environment,
|
||||
# installs dependencies, sets up the database, and starts the application.
|
||||
#
|
||||
# Usage:
|
||||
# ./start.sh [development|production] [--no-install]
|
||||
#
|
||||
# Environment Variables:
|
||||
# ENVIRONMENT: 'development' or 'production' (default: development)
|
||||
# CONDA_ENV: Conda environment name (default: AniWorld)
|
||||
# PORT: Server port (default: 8000)
|
||||
# HOST: Server host (default: 127.0.0.1)
|
||||
#
|
||||
################################################################################
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ============================================================================
|
||||
# Configuration
|
||||
# ============================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
|
||||
CONDA_ENV="${CONDA_ENV:-AniWorld}"
|
||||
ENVIRONMENT="${1:-development}"
|
||||
INSTALL_DEPS="${INSTALL_DEPS:-true}"
|
||||
PORT="${PORT:-8000}"
|
||||
HOST="${HOST:-127.0.0.1}"
|
||||
|
||||
# ============================================================================
|
||||
# Color Output
|
||||
# ============================================================================
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# ============================================================================
|
||||
# Functions
|
||||
# ============================================================================
|
||||
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
# Check if conda environment exists
|
||||
check_conda_env() {
|
||||
if ! conda env list | grep -q "^$CONDA_ENV "; then
|
||||
log_error "Conda environment '$CONDA_ENV' not found."
|
||||
log_info "Create it with: conda create -n $CONDA_ENV python=3.11"
|
||||
exit 1
|
||||
fi
|
||||
log_success "Conda environment '$CONDA_ENV' found."
|
||||
}
|
||||
|
||||
# Validate environment parameter
|
||||
validate_environment() {
|
||||
if [[ ! "$ENVIRONMENT" =~ ^(development|production|testing)$ ]]; then
|
||||
log_error "Invalid environment: $ENVIRONMENT"
|
||||
log_info "Valid options: development, production, testing"
|
||||
exit 1
|
||||
fi
|
||||
log_success "Environment set to: $ENVIRONMENT"
|
||||
}
|
||||
|
||||
# Create necessary directories
|
||||
create_directories() {
|
||||
log_info "Creating necessary directories..."
|
||||
mkdir -p "$PROJECT_ROOT/logs"
|
||||
mkdir -p "$PROJECT_ROOT/data"
|
||||
mkdir -p "$PROJECT_ROOT/data/config_backups"
|
||||
mkdir -p "$PROJECT_ROOT/Temp"
|
||||
log_success "Directories created."
|
||||
}
|
||||
|
||||
# Install dependencies
|
||||
install_dependencies() {
|
||||
if [[ "$INSTALL_DEPS" != "true" ]]; then
|
||||
log_warning "Skipping dependency installation."
|
||||
return
|
||||
fi
|
||||
|
||||
log_info "Installing dependencies..."
|
||||
conda run -n "$CONDA_ENV" pip install -q -r "$PROJECT_ROOT/requirements.txt"
|
||||
log_success "Dependencies installed."
|
||||
}
|
||||
|
||||
# Initialize database
|
||||
init_database() {
|
||||
log_info "Initializing database..."
|
||||
cd "$PROJECT_ROOT"
|
||||
conda run -n "$CONDA_ENV" \
|
||||
python -c "from src.server.database import init_db; import asyncio; asyncio.run(init_db())"
|
||||
log_success "Database initialized."
|
||||
}
|
||||
|
||||
# Create environment file if it doesn't exist
|
||||
create_env_file() {
|
||||
ENV_FILE="$PROJECT_ROOT/.env.$ENVIRONMENT"
|
||||
if [[ ! -f "$ENV_FILE" ]]; then
|
||||
log_warning "Creating $ENV_FILE with defaults..."
|
||||
cat > "$ENV_FILE" << EOF
|
||||
# Aniworld Configuration for $ENVIRONMENT
|
||||
|
||||
# Security Settings
|
||||
JWT_SECRET_KEY=your-secret-key-here
|
||||
PASSWORD_SALT=your-salt-here
|
||||
MASTER_PASSWORD_HASH=\$2b\$12\$wP0KBVbJKVAb8CdSSXw0NeGTKCkbw4fSAFXIqR2/wDqPSEBn9w7lS
|
||||
|
||||
# Database
|
||||
DATABASE_URL=sqlite:///./data/aniworld_${ENVIRONMENT}.db
|
||||
|
||||
# Application
|
||||
ENVIRONMENT=${ENVIRONMENT}
|
||||
ANIME_DIRECTORY=/path/to/anime
|
||||
|
||||
# Server
|
||||
PORT=${PORT}
|
||||
HOST=${HOST}
|
||||
|
||||
# Logging
|
||||
LOG_LEVEL=$([ "$ENVIRONMENT" = "production" ] && echo "WARNING" || echo "DEBUG")
|
||||
|
||||
# Features (development only)
|
||||
$([ "$ENVIRONMENT" = "development" ] && echo "DEBUG=true" || echo "DEBUG=false")
|
||||
EOF
|
||||
log_success "Created $ENV_FILE - please configure with your settings"
|
||||
fi
|
||||
}
|
||||
|
||||
# Start the application
|
||||
start_application() {
|
||||
log_info "Starting Aniworld application..."
|
||||
log_info "Environment: $ENVIRONMENT"
|
||||
log_info "Conda Environment: $CONDA_ENV"
|
||||
log_info "Server: http://$HOST:$PORT"
|
||||
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
case "$ENVIRONMENT" in
|
||||
development)
|
||||
log_info "Starting in development mode with auto-reload..."
|
||||
conda run -n "$CONDA_ENV" \
|
||||
python -m uvicorn \
|
||||
src.server.fastapi_app:app \
|
||||
--host "$HOST" \
|
||||
--port "$PORT" \
|
||||
--reload
|
||||
;;
|
||||
production)
|
||||
WORKERS="${WORKERS:-4}"
|
||||
log_info "Starting in production mode with $WORKERS workers..."
|
||||
conda run -n "$CONDA_ENV" \
|
||||
python -m uvicorn \
|
||||
src.server.fastapi_app:app \
|
||||
--host "$HOST" \
|
||||
--port "$PORT" \
|
||||
--workers "$WORKERS" \
|
||||
--worker-class "uvicorn.workers.UvicornWorker"
|
||||
;;
|
||||
testing)
|
||||
log_warning "Starting in testing mode..."
|
||||
# Testing mode typically runs tests instead of starting server
|
||||
conda run -n "$CONDA_ENV" \
|
||||
python -m pytest tests/ -v --tb=short
|
||||
;;
|
||||
*)
|
||||
log_error "Unknown environment: $ENVIRONMENT"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Main Script
|
||||
# ============================================================================
|
||||
|
||||
main() {
|
||||
log_info "=========================================="
|
||||
log_info "Aniworld Application Startup"
|
||||
log_info "=========================================="
|
||||
|
||||
# Parse command-line options
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--no-install)
|
||||
INSTALL_DEPS="false"
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
ENVIRONMENT="$1"
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
validate_environment
|
||||
check_conda_env
|
||||
create_directories
|
||||
create_env_file
|
||||
install_dependencies
|
||||
init_database
|
||||
start_application
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main "$@"
|
||||
205
src/cli/nfo_cli.py
Normal file
205
src/cli/nfo_cli.py
Normal file
@@ -0,0 +1,205 @@
|
||||
"""CLI command for NFO management.
|
||||
|
||||
Note: NFO service has been removed. This CLI is no longer functional.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add src to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def scan_and_create_nfo():
|
||||
"""Scan all series and create missing NFO files."""
|
||||
logger.info("%s", "=" * 70)
|
||||
logger.info("NFO Auto-Creation Tool")
|
||||
logger.info("%s", "=" * 70)
|
||||
|
||||
if not settings.tmdb_api_key:
|
||||
logger.error("TMDB_API_KEY not configured")
|
||||
logger.error("Set TMDB_API_KEY in .env file or environment")
|
||||
logger.error("Get API key from: https://www.themoviedb.org/settings/api")
|
||||
return 1
|
||||
|
||||
if not settings.anime_directory:
|
||||
logger.error("ANIME_DIRECTORY not configured")
|
||||
return 1
|
||||
|
||||
logger.info("Anime Directory: %s", settings.anime_directory)
|
||||
logger.info("Auto-create NFO: %s", settings.nfo_auto_create)
|
||||
logger.info("Update on scan: %s", settings.nfo_update_on_scan)
|
||||
logger.info("Download poster: %s", settings.nfo_download_poster)
|
||||
logger.info("Download logo: %s", settings.nfo_download_logo)
|
||||
logger.info("Download fanart: %s", settings.nfo_download_fanart)
|
||||
|
||||
if not settings.nfo_auto_create:
|
||||
logger.warning("NFO_AUTO_CREATE is set to False")
|
||||
logger.warning("Enable it in .env to auto-create NFO files")
|
||||
logger.info("Continuing anyway to demonstrate functionality...")
|
||||
# Override for demonstration
|
||||
settings.nfo_auto_create = True
|
||||
|
||||
logger.info("Initializing series manager...")
|
||||
manager = SeriesManagerService.from_settings()
|
||||
|
||||
# Get series list first
|
||||
serie_list = manager.get_serie_list()
|
||||
all_series = serie_list.get_all()
|
||||
|
||||
logger.info("Found %d series in directory", len(all_series))
|
||||
|
||||
if not all_series:
|
||||
logger.warning("No series found. Add some anime series first.")
|
||||
return 0
|
||||
|
||||
# Show series without NFO
|
||||
series_without_nfo = []
|
||||
for serie in all_series:
|
||||
if not serie.has_nfo():
|
||||
series_without_nfo.append(serie)
|
||||
|
||||
if series_without_nfo:
|
||||
logger.info("Series without NFO: %d", len(series_without_nfo))
|
||||
for serie in series_without_nfo[:5]: # Show first 5
|
||||
logger.debug("Missing NFO: %s (%s)", serie.name, serie.folder)
|
||||
if len(series_without_nfo) > 5:
|
||||
logger.info("... and %d more", len(series_without_nfo) - 5)
|
||||
else:
|
||||
logger.info("All series already have NFO files")
|
||||
|
||||
if not settings.nfo_update_on_scan:
|
||||
logger.info("Nothing to do. Enable NFO_UPDATE_ON_SCAN to update existing NFOs.")
|
||||
return 0
|
||||
|
||||
logger.info("Processing NFO files...")
|
||||
logger.info("This may take a while depending on the number of series")
|
||||
|
||||
try:
|
||||
await manager.scan_and_process_nfo()
|
||||
logger.info("NFO processing complete")
|
||||
|
||||
# Show updated stats
|
||||
serie_list.load_series() # Reload to get updated stats
|
||||
all_series = serie_list.get_all()
|
||||
series_with_nfo = [s for s in all_series if s.has_nfo()]
|
||||
series_with_poster = [s for s in all_series if s.has_poster()]
|
||||
series_with_logo = [s for s in all_series if s.has_logo()]
|
||||
series_with_fanart = [s for s in all_series if s.has_fanart()]
|
||||
|
||||
logger.info("Final statistics", extra={
|
||||
"total_series": len(all_series),
|
||||
"with_nfo": len(series_with_nfo),
|
||||
"with_poster": len(series_with_poster),
|
||||
"with_logo": len(series_with_logo),
|
||||
"with_fanart": len(series_with_fanart),
|
||||
})
|
||||
|
||||
except Exception:
|
||||
logger.exception("Failed to process NFO files")
|
||||
return 1
|
||||
finally:
|
||||
await manager.close()
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
async def check_nfo_status():
|
||||
"""Check NFO status for all series."""
|
||||
logger.info("%s", "=" * 70)
|
||||
logger.info("NFO Status Check")
|
||||
logger.info("%s", "=" * 70)
|
||||
|
||||
if not settings.anime_directory:
|
||||
logger.error("ANIME_DIRECTORY not configured")
|
||||
return 1
|
||||
|
||||
logger.info("Anime Directory: %s", settings.anime_directory)
|
||||
|
||||
# Create series list (no NFO service needed for status check)
|
||||
from src.server.database.SerieList import SerieList
|
||||
serie_list = SerieList(settings.anime_directory)
|
||||
all_series = serie_list.get_all()
|
||||
|
||||
if not all_series:
|
||||
logger.warning("No series found")
|
||||
return 0
|
||||
|
||||
logger.info("Total series: %d", len(all_series))
|
||||
|
||||
# Categorize series
|
||||
with_nfo = []
|
||||
without_nfo = []
|
||||
|
||||
for serie in all_series:
|
||||
if serie.has_nfo():
|
||||
with_nfo.append(serie)
|
||||
else:
|
||||
without_nfo.append(serie)
|
||||
|
||||
logger.info(
|
||||
"Series NFO coverage",
|
||||
extra={
|
||||
"with_nfo": len(with_nfo),
|
||||
"without_nfo": len(without_nfo),
|
||||
"total": len(all_series),
|
||||
},
|
||||
)
|
||||
|
||||
if without_nfo:
|
||||
logger.info("Series missing NFO: %d", len(without_nfo))
|
||||
for serie in without_nfo[:10]:
|
||||
logger.debug("Missing NFO: %s (%s)", serie.name, serie.folder)
|
||||
if len(without_nfo) > 10:
|
||||
logger.info("... and %d more", len(without_nfo) - 10)
|
||||
|
||||
# Media file statistics
|
||||
with_poster = sum(1 for s in all_series if s.has_poster())
|
||||
with_logo = sum(1 for s in all_series if s.has_logo())
|
||||
with_fanart = sum(1 for s in all_series if s.has_fanart())
|
||||
|
||||
logger.info(
|
||||
"Media file coverage",
|
||||
extra={
|
||||
"posters": with_poster,
|
||||
"logos": with_logo,
|
||||
"fanart": with_fanart,
|
||||
"total": len(all_series),
|
||||
},
|
||||
)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
"""Main CLI entry point."""
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
logger.info("NFO Management Tool")
|
||||
logger.info("\nUsage:")
|
||||
logger.info(" python -m src.cli.nfo_cli scan # Scan and create missing NFO files")
|
||||
logger.info(" python -m src.cli.nfo_cli status # Check NFO status for all series")
|
||||
logger.info("\nConfiguration:")
|
||||
logger.info(" Set TMDB_API_KEY in .env file")
|
||||
logger.info(" Set NFO_AUTO_CREATE=true to enable auto-creation")
|
||||
logger.info(" Set NFO_UPDATE_ON_SCAN=true to update existing NFOs during scan")
|
||||
return 1
|
||||
|
||||
command = sys.argv[1].lower()
|
||||
|
||||
if command == "scan":
|
||||
return asyncio.run(scan_and_create_nfo())
|
||||
elif command == "status":
|
||||
return asyncio.run(check_nfo_status())
|
||||
else:
|
||||
logger.error("Unknown command: %s", command)
|
||||
logger.info("Use 'scan' or 'status'")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,3 +1,4 @@
|
||||
import re
|
||||
import secrets
|
||||
from typing import Optional
|
||||
|
||||
@@ -72,6 +73,82 @@ class Settings(BaseSettings):
|
||||
default=3,
|
||||
validation_alias="RETRY_ATTEMPTS"
|
||||
)
|
||||
|
||||
# NFO / TMDB Settings
|
||||
tmdb_api_key: Optional[str] = Field(
|
||||
default=None,
|
||||
validation_alias="TMDB_API_KEY",
|
||||
description="TMDB API key for scraping TV show metadata"
|
||||
)
|
||||
nfo_auto_create: bool = Field(
|
||||
default=False,
|
||||
validation_alias="NFO_AUTO_CREATE",
|
||||
description="Automatically create NFO files when scanning series"
|
||||
)
|
||||
nfo_update_on_scan: bool = Field(
|
||||
default=False,
|
||||
validation_alias="NFO_UPDATE_ON_SCAN",
|
||||
description="Update existing NFO files when scanning series"
|
||||
)
|
||||
nfo_download_poster: bool = Field(
|
||||
default=True,
|
||||
validation_alias="NFO_DOWNLOAD_POSTER",
|
||||
description="Download poster.jpg when creating NFO"
|
||||
)
|
||||
nfo_download_logo: bool = Field(
|
||||
default=True,
|
||||
validation_alias="NFO_DOWNLOAD_LOGO",
|
||||
description="Download logo.png when creating NFO"
|
||||
)
|
||||
nfo_download_fanart: bool = Field(
|
||||
default=True,
|
||||
validation_alias="NFO_DOWNLOAD_FANART",
|
||||
description="Download fanart.jpg when creating NFO"
|
||||
)
|
||||
nfo_image_size: str = Field(
|
||||
default="original",
|
||||
validation_alias="NFO_IMAGE_SIZE",
|
||||
description="Image size to download (original, w500, etc.)"
|
||||
)
|
||||
nfo_prefer_fsk_rating: bool = Field(
|
||||
default=True,
|
||||
validation_alias="NFO_PREFER_FSK_RATING",
|
||||
description="Prefer German FSK rating over MPAA rating in NFO files"
|
||||
)
|
||||
nfo_folder_ignore_patterns: str = Field(
|
||||
default="The Last of Us|Loki|Chernobyl|Star Trek Discovery|Marvel|Matrix|Fast & Furious|Jurassic|James Bond|Mission: Impossible|Bourne|Hunger Games|Die Hard|John Wick|Pacific Rim|Guardians of the Galaxy|Avengers|Batman|Superman|Wonder Woman|Spider-Man|X-Men|Fantastic Four|Terminator|Predator|Rambo|Rocky|Expendables|Tomb Raider|Jumanji|Jurassic Park|Pirates of the Caribbean|Harry Potter|Lord of the Rings|Hobbit|Game of Thrones|Westworld|Stranger Things|Breaking Bad|Better Call Saul|Sherlock|Downton Abbey|The Crown|Bridgerton|Sex Education|Normal People|Emily in Paris|The Witcher|Servant|Lucifer|Dark|Shadow and Bone|Grimm|Fairytale",
|
||||
validation_alias="NFO_FOLDER_IGNORE_PATTERNS",
|
||||
description="Regex patterns for folder names to skip during scan (pipe-separated)"
|
||||
)
|
||||
|
||||
@property
|
||||
def folder_ignore_patterns(self) -> list[str]:
|
||||
"""Parse ignore patterns from comma-separated string into list.
|
||||
|
||||
Returns:
|
||||
List of regex patterns to skip during folder scanning.
|
||||
"""
|
||||
if not self.nfo_folder_ignore_patterns:
|
||||
return []
|
||||
return [
|
||||
pattern.strip()
|
||||
for pattern in self.nfo_folder_ignore_patterns.split("|")
|
||||
if pattern.strip()
|
||||
]
|
||||
|
||||
def should_ignore_folder(self, folder_name: str) -> bool:
|
||||
"""Check if folder should be ignored based on ignore patterns.
|
||||
|
||||
Args:
|
||||
folder_name: Name of folder to check.
|
||||
|
||||
Returns:
|
||||
True if folder matches any ignore pattern, False otherwise.
|
||||
"""
|
||||
for pattern in self.folder_ignore_patterns:
|
||||
if re.search(pattern, folder_name, re.IGNORECASE):
|
||||
return True
|
||||
return False
|
||||
|
||||
@property
|
||||
def allowed_origins(self) -> list[str]:
|
||||
@@ -92,5 +169,23 @@ class Settings(BaseSettings):
|
||||
]
|
||||
return [origin.strip() for origin in raw.split(",") if origin.strip()]
|
||||
|
||||
@property
|
||||
def scan_key_overrides(self) -> dict[str, str]:
|
||||
"""Return scan key overrides from config.json.
|
||||
|
||||
Maps folder names to provider keys for cases where auto-generated
|
||||
keys from folder names are incorrect.
|
||||
|
||||
Returns:
|
||||
Dict mapping folder names to provider keys.
|
||||
"""
|
||||
from src.server.services.config_service import ConfigService
|
||||
try:
|
||||
config_service = ConfigService()
|
||||
config = config_service.load_config()
|
||||
return config.scan_key_overrides or {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
settings = Settings()
|
||||
|
||||
@@ -1,233 +0,0 @@
|
||||
"""Utilities for loading and managing stored anime series metadata.
|
||||
|
||||
This module provides the SerieList class for managing collections of anime
|
||||
series metadata. It uses file-based storage only.
|
||||
|
||||
Note:
|
||||
This module is part of the core domain layer and has no database
|
||||
dependencies. All database operations are handled by the service layer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import warnings
|
||||
from json import JSONDecodeError
|
||||
from typing import Dict, Iterable, List, Optional
|
||||
|
||||
from src.core.entities.series import Serie
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SerieList:
|
||||
"""
|
||||
Represents the collection of cached series stored on disk.
|
||||
|
||||
Series are identified by their unique 'key' (provider identifier).
|
||||
The 'folder' is metadata only and not used for lookups.
|
||||
|
||||
This class manages in-memory series data loaded from filesystem.
|
||||
It has no database dependencies - all persistence is handled by
|
||||
the service layer.
|
||||
|
||||
Example:
|
||||
# File-based mode
|
||||
serie_list = SerieList("/path/to/anime")
|
||||
series = serie_list.get_all()
|
||||
|
||||
Attributes:
|
||||
directory: Path to the anime directory
|
||||
keyDict: Internal dictionary mapping serie.key to Serie objects
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_path: str,
|
||||
skip_load: bool = False
|
||||
) -> None:
|
||||
"""Initialize the SerieList.
|
||||
|
||||
Args:
|
||||
base_path: Path to the anime directory
|
||||
skip_load: If True, skip automatic loading of series from files.
|
||||
Useful when planning to load from database instead.
|
||||
"""
|
||||
self.directory: str = base_path
|
||||
# Internal storage using serie.key as the dictionary key
|
||||
self.keyDict: Dict[str, Serie] = {}
|
||||
|
||||
# Only auto-load from files if not skipping
|
||||
if not skip_load:
|
||||
self.load_series()
|
||||
|
||||
def add(self, serie: Serie, use_sanitized_folder: bool = True) -> str:
|
||||
"""
|
||||
Persist a new series if it is not already present (file-based mode).
|
||||
|
||||
Uses serie.key for identification. Creates the filesystem folder
|
||||
using either the sanitized display name (default) or the existing
|
||||
folder property.
|
||||
|
||||
Args:
|
||||
serie: The Serie instance to add
|
||||
use_sanitized_folder: If True (default), use serie.sanitized_folder
|
||||
for the filesystem folder name based on display name.
|
||||
If False, use serie.folder as-is for backward compatibility.
|
||||
|
||||
Returns:
|
||||
str: The folder path that was created/used
|
||||
|
||||
Note:
|
||||
This method creates data files on disk. For database storage,
|
||||
use add_to_db() instead.
|
||||
"""
|
||||
if self.contains(serie.key):
|
||||
# Return existing folder path
|
||||
existing = self.keyDict[serie.key]
|
||||
return os.path.join(self.directory, existing.folder)
|
||||
|
||||
# Determine folder name to use
|
||||
if use_sanitized_folder:
|
||||
folder_name = serie.sanitized_folder
|
||||
# Update the serie's folder property to match what we create
|
||||
serie.folder = folder_name
|
||||
else:
|
||||
folder_name = serie.folder
|
||||
|
||||
data_path = os.path.join(self.directory, folder_name, "data")
|
||||
anime_path = os.path.join(self.directory, folder_name)
|
||||
os.makedirs(anime_path, exist_ok=True)
|
||||
if not os.path.isfile(data_path):
|
||||
serie.save_to_file(data_path)
|
||||
# Store by key, not folder
|
||||
self.keyDict[serie.key] = serie
|
||||
|
||||
return anime_path
|
||||
|
||||
def contains(self, key: str) -> bool:
|
||||
"""
|
||||
Return True when a series identified by ``key`` already exists.
|
||||
|
||||
Args:
|
||||
key: The unique provider identifier for the series
|
||||
|
||||
Returns:
|
||||
True if the series exists in the collection
|
||||
"""
|
||||
return key in self.keyDict
|
||||
|
||||
def load_series(self) -> None:
|
||||
"""Populate the in-memory map with metadata discovered on disk."""
|
||||
|
||||
logging.info("Scanning anime folders in %s", self.directory)
|
||||
try:
|
||||
entries: Iterable[str] = os.listdir(self.directory)
|
||||
except OSError as error:
|
||||
logging.error(
|
||||
"Unable to scan directory %s: %s",
|
||||
self.directory,
|
||||
error,
|
||||
)
|
||||
return
|
||||
|
||||
for anime_folder in entries:
|
||||
anime_path = os.path.join(self.directory, anime_folder, "data")
|
||||
if os.path.isfile(anime_path):
|
||||
logging.debug("Found data file for folder %s", anime_folder)
|
||||
self._load_data(anime_folder, anime_path)
|
||||
continue
|
||||
|
||||
logging.warning(
|
||||
"Skipping folder %s because no metadata file was found",
|
||||
anime_folder,
|
||||
)
|
||||
|
||||
def _load_data(self, anime_folder: str, data_path: str) -> None:
|
||||
"""
|
||||
Load a single series metadata file into the in-memory collection.
|
||||
|
||||
Args:
|
||||
anime_folder: The folder name (for logging only)
|
||||
data_path: Path to the metadata file
|
||||
"""
|
||||
try:
|
||||
serie = Serie.load_from_file(data_path)
|
||||
# Store by key, not folder
|
||||
self.keyDict[serie.key] = serie
|
||||
logging.debug(
|
||||
"Successfully loaded metadata for %s (key: %s)",
|
||||
anime_folder,
|
||||
serie.key
|
||||
)
|
||||
except (OSError, JSONDecodeError, KeyError, ValueError) as error:
|
||||
logging.error(
|
||||
"Failed to load metadata for folder %s from %s: %s",
|
||||
anime_folder,
|
||||
data_path,
|
||||
error,
|
||||
)
|
||||
|
||||
def GetMissingEpisode(self) -> List[Serie]:
|
||||
"""Return all series that still contain missing episodes."""
|
||||
return [
|
||||
serie
|
||||
for serie in self.keyDict.values()
|
||||
if serie.episodeDict
|
||||
]
|
||||
|
||||
def get_missing_episodes(self) -> List[Serie]:
|
||||
"""PEP8-friendly alias for :meth:`GetMissingEpisode`."""
|
||||
return self.GetMissingEpisode()
|
||||
|
||||
def GetList(self) -> List[Serie]:
|
||||
"""Return all series instances stored in the list."""
|
||||
return list(self.keyDict.values())
|
||||
|
||||
def get_all(self) -> List[Serie]:
|
||||
"""PEP8-friendly alias for :meth:`GetList`."""
|
||||
return self.GetList()
|
||||
|
||||
def get_by_key(self, key: str) -> Optional[Serie]:
|
||||
"""
|
||||
Get a series by its unique provider key.
|
||||
|
||||
This is the primary method for series lookup.
|
||||
|
||||
Args:
|
||||
key: The unique provider identifier (e.g., "attack-on-titan")
|
||||
|
||||
Returns:
|
||||
The Serie instance if found, None otherwise
|
||||
"""
|
||||
return self.keyDict.get(key)
|
||||
|
||||
def get_by_folder(self, folder: str) -> Optional[Serie]:
|
||||
"""
|
||||
Get a series by its folder name.
|
||||
|
||||
.. deprecated:: 2.0.0
|
||||
Use :meth:`get_by_key` instead. Folder-based lookups will be
|
||||
removed in version 3.0.0. The `folder` field is metadata only
|
||||
and should not be used for identification.
|
||||
|
||||
This method is provided for backward compatibility only.
|
||||
Prefer using get_by_key() for new code.
|
||||
|
||||
Args:
|
||||
folder: The filesystem folder name (e.g., "Attack on Titan (2013)")
|
||||
|
||||
Returns:
|
||||
The Serie instance if found, None otherwise
|
||||
"""
|
||||
warnings.warn(
|
||||
"get_by_folder() is deprecated and will be removed in v3.0.0. "
|
||||
"Use get_by_key() instead. The 'folder' field is metadata only.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
for serie in self.keyDict.values():
|
||||
if serie.folder == folder:
|
||||
return serie
|
||||
return None
|
||||
@@ -1,231 +0,0 @@
|
||||
import json
|
||||
import warnings
|
||||
|
||||
from src.server.utils.filesystem import sanitize_folder_name
|
||||
|
||||
|
||||
class Serie:
|
||||
"""
|
||||
Represents an anime series with metadata and episode information.
|
||||
|
||||
The `key` property is the unique identifier for the series
|
||||
(provider-assigned, URL-safe).
|
||||
The `folder` property is the filesystem folder name
|
||||
(metadata only, not used for lookups).
|
||||
|
||||
Args:
|
||||
key: Unique series identifier from provider
|
||||
(e.g., "attack-on-titan"). Cannot be empty.
|
||||
name: Display name of the series
|
||||
site: Provider site URL
|
||||
folder: Filesystem folder name (metadata only,
|
||||
e.g., "Attack on Titan (2013)")
|
||||
episodeDict: Dictionary mapping season numbers to
|
||||
lists of episode numbers
|
||||
|
||||
Raises:
|
||||
ValueError: If key is None or empty string
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
key: str,
|
||||
name: str,
|
||||
site: str,
|
||||
folder: str,
|
||||
episodeDict: dict[int, list[int]]
|
||||
):
|
||||
if not key or not key.strip():
|
||||
raise ValueError("Serie key cannot be None or empty")
|
||||
|
||||
self._key = key.strip()
|
||||
self._name = name
|
||||
self._site = site
|
||||
self._folder = folder
|
||||
self._episodeDict = episodeDict
|
||||
|
||||
def __str__(self):
|
||||
"""String representation of Serie object"""
|
||||
return (
|
||||
f"Serie(key='{self.key}', name='{self.name}', "
|
||||
f"site='{self.site}', folder='{self.folder}', "
|
||||
f"episodeDict={self.episodeDict})"
|
||||
)
|
||||
|
||||
@property
|
||||
def key(self) -> str:
|
||||
"""
|
||||
Unique series identifier (primary identifier for all lookups).
|
||||
|
||||
This is the provider-assigned, URL-safe identifier used
|
||||
throughout the application for series identification,
|
||||
lookups, and operations.
|
||||
|
||||
Returns:
|
||||
str: The unique series key
|
||||
"""
|
||||
return self._key
|
||||
|
||||
@key.setter
|
||||
def key(self, value: str):
|
||||
"""
|
||||
Set the unique series identifier.
|
||||
|
||||
Args:
|
||||
value: New key value
|
||||
|
||||
Raises:
|
||||
ValueError: If value is None or empty string
|
||||
"""
|
||||
if not value or not value.strip():
|
||||
raise ValueError("Serie key cannot be None or empty")
|
||||
self._key = value.strip()
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._name
|
||||
|
||||
@name.setter
|
||||
def name(self, value: str):
|
||||
self._name = value
|
||||
|
||||
@property
|
||||
def site(self) -> str:
|
||||
return self._site
|
||||
|
||||
@site.setter
|
||||
def site(self, value: str):
|
||||
self._site = value
|
||||
|
||||
@property
|
||||
def folder(self) -> str:
|
||||
"""
|
||||
Filesystem folder name (metadata only, not used for lookups).
|
||||
|
||||
This property contains the local directory name where the series
|
||||
files are stored. It should NOT be used as an identifier for
|
||||
series lookups - use `key` instead.
|
||||
|
||||
Returns:
|
||||
str: The filesystem folder name
|
||||
"""
|
||||
return self._folder
|
||||
|
||||
@folder.setter
|
||||
def folder(self, value: str):
|
||||
"""
|
||||
Set the filesystem folder name.
|
||||
|
||||
Args:
|
||||
value: Folder name for the series
|
||||
"""
|
||||
self._folder = value
|
||||
|
||||
@property
|
||||
def episodeDict(self) -> dict[int, list[int]]:
|
||||
return self._episodeDict
|
||||
|
||||
@episodeDict.setter
|
||||
def episodeDict(self, value: dict[int, list[int]]):
|
||||
self._episodeDict = value
|
||||
|
||||
@property
|
||||
def sanitized_folder(self) -> str:
|
||||
"""
|
||||
Get a filesystem-safe folder name derived from the display name.
|
||||
|
||||
This property returns a sanitized version of the series name
|
||||
suitable for use as a filesystem folder name. It removes/replaces
|
||||
characters that are invalid for filesystems while preserving
|
||||
Unicode characters.
|
||||
|
||||
Use this property when creating folders for the series on disk.
|
||||
The `folder` property stores the actual folder name used.
|
||||
|
||||
Returns:
|
||||
str: Filesystem-safe folder name based on display name
|
||||
|
||||
Example:
|
||||
>>> serie = Serie("attack-on-titan", "Attack on Titan: Final", ...)
|
||||
>>> serie.sanitized_folder
|
||||
'Attack on Titan Final'
|
||||
"""
|
||||
# Use name if available, fall back to folder, then key
|
||||
name_to_sanitize = self._name or self._folder or self._key
|
||||
try:
|
||||
return sanitize_folder_name(name_to_sanitize)
|
||||
except ValueError:
|
||||
# Fallback to key if name cannot be sanitized
|
||||
return sanitize_folder_name(self._key)
|
||||
|
||||
def to_dict(self):
|
||||
"""Convert Serie object to dictionary for JSON serialization."""
|
||||
return {
|
||||
"key": self.key,
|
||||
"name": self.name,
|
||||
"site": self.site,
|
||||
"folder": self.folder,
|
||||
"episodeDict": {
|
||||
str(k): list(v) for k, v in self.episodeDict.items()
|
||||
}
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def from_dict(data: dict):
|
||||
"""Create a Serie object from dictionary."""
|
||||
# Convert keys to int
|
||||
episode_dict = {
|
||||
int(k): v for k, v in data["episodeDict"].items()
|
||||
}
|
||||
return Serie(
|
||||
data["key"],
|
||||
data["name"],
|
||||
data["site"],
|
||||
data["folder"],
|
||||
episode_dict
|
||||
)
|
||||
|
||||
def save_to_file(self, filename: str):
|
||||
"""Save Serie object to JSON file.
|
||||
|
||||
.. deprecated::
|
||||
File-based storage is deprecated. Use database storage via
|
||||
`AnimeSeriesService.create()` instead. This method will be
|
||||
removed in v3.0.0.
|
||||
|
||||
Args:
|
||||
filename: Path to save the JSON file
|
||||
"""
|
||||
warnings.warn(
|
||||
"save_to_file() is deprecated and will be removed in v3.0.0. "
|
||||
"Use database storage via AnimeSeriesService.create() instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
with open(filename, "w", encoding="utf-8") as file:
|
||||
json.dump(self.to_dict(), file, indent=4)
|
||||
|
||||
@classmethod
|
||||
def load_from_file(cls, filename: str) -> "Serie":
|
||||
"""Load Serie object from JSON file.
|
||||
|
||||
.. deprecated::
|
||||
File-based storage is deprecated. Use database storage via
|
||||
`AnimeSeriesService.get_by_key()` instead. This method will be
|
||||
removed in v3.0.0.
|
||||
|
||||
Args:
|
||||
filename: Path to load the JSON file from
|
||||
|
||||
Returns:
|
||||
Serie: The loaded Serie object
|
||||
"""
|
||||
warnings.warn(
|
||||
"load_from_file() is deprecated and will be removed in v3.0.0. "
|
||||
"Use database storage via AnimeSeriesService instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
with open(filename, "r", encoding="utf-8") as file:
|
||||
data = json.load(file)
|
||||
return cls.from_dict(data)
|
||||
@@ -1,149 +0,0 @@
|
||||
"""
|
||||
Error handling and recovery strategies for core providers.
|
||||
|
||||
This module provides custom exceptions and decorators for handling
|
||||
errors in provider operations with automatic retry mechanisms.
|
||||
"""
|
||||
|
||||
import functools
|
||||
import logging
|
||||
from typing import Any, Callable, TypeVar
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Type variable for decorator
|
||||
F = TypeVar("F", bound=Callable[..., Any])
|
||||
|
||||
|
||||
class RetryableError(Exception):
|
||||
"""Exception that indicates an operation can be safely retried."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class NonRetryableError(Exception):
|
||||
"""Exception that indicates an operation should not be retried."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class NetworkError(Exception):
|
||||
"""Exception for network-related errors."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class DownloadError(Exception):
|
||||
"""Exception for download-related errors."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class RecoveryStrategies:
|
||||
"""Strategies for handling errors and recovering from failures."""
|
||||
|
||||
@staticmethod
|
||||
def handle_network_failure(
|
||||
func: Callable, *args: Any, **kwargs: Any
|
||||
) -> Any:
|
||||
"""Handle network failures with basic retry logic."""
|
||||
max_retries = 3
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except (NetworkError, ConnectionError):
|
||||
if attempt == max_retries - 1:
|
||||
raise
|
||||
logger.warning(
|
||||
f"Network error on attempt {attempt + 1}, retrying..."
|
||||
)
|
||||
continue
|
||||
|
||||
@staticmethod
|
||||
def handle_download_failure(
|
||||
func: Callable, *args: Any, **kwargs: Any
|
||||
) -> Any:
|
||||
"""Handle download failures with retry logic."""
|
||||
max_retries = 2
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except DownloadError:
|
||||
if attempt == max_retries - 1:
|
||||
raise
|
||||
logger.warning(
|
||||
f"Download error on attempt {attempt + 1}, retrying..."
|
||||
)
|
||||
continue
|
||||
|
||||
|
||||
class FileCorruptionDetector:
|
||||
"""Detector for corrupted files."""
|
||||
|
||||
@staticmethod
|
||||
def is_valid_video_file(filepath: str) -> bool:
|
||||
"""Check if a video file is valid and not corrupted."""
|
||||
try:
|
||||
import os
|
||||
if not os.path.exists(filepath):
|
||||
return False
|
||||
|
||||
file_size = os.path.getsize(filepath)
|
||||
# Video files should be at least 1MB
|
||||
return file_size > 1024 * 1024
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking file validity: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def with_error_recovery(
|
||||
max_retries: int = 3, context: str = ""
|
||||
) -> Callable[[F], F]:
|
||||
"""
|
||||
Decorator for adding error recovery to functions.
|
||||
|
||||
Args:
|
||||
max_retries: Maximum number of retry attempts
|
||||
context: Context string for logging
|
||||
|
||||
Returns:
|
||||
Decorated function with retry logic
|
||||
"""
|
||||
|
||||
def decorator(func: F) -> F:
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
||||
last_error = None
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except NonRetryableError:
|
||||
raise
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
if attempt < max_retries - 1:
|
||||
logger.warning(
|
||||
f"Error in {context} (attempt {attempt + 1}/"
|
||||
f"{max_retries}): {e}, retrying..."
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
f"Error in {context} failed after {max_retries} "
|
||||
f"attempts: {e}"
|
||||
)
|
||||
|
||||
if last_error:
|
||||
raise last_error
|
||||
|
||||
raise RuntimeError(
|
||||
f"Unexpected error in {context} after {max_retries} attempts"
|
||||
)
|
||||
|
||||
return wrapper # type: ignore
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
# Create module-level instances for use in provider code
|
||||
recovery_strategies = RecoveryStrategies()
|
||||
file_corruption_detector = FileCorruptionDetector()
|
||||
@@ -1,615 +0,0 @@
|
||||
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from events import Events
|
||||
from fake_useragent import UserAgent
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
from yt_dlp import YoutubeDL
|
||||
from yt_dlp.utils import DownloadCancelled
|
||||
|
||||
from ..interfaces.providers import Providers
|
||||
from .base_provider import Loader
|
||||
|
||||
# Imported shared provider configuration
|
||||
from .provider_config import (
|
||||
ANIWORLD_HEADERS,
|
||||
DEFAULT_DOWNLOAD_TIMEOUT,
|
||||
DEFAULT_PROVIDERS,
|
||||
INVALID_PATH_CHARS,
|
||||
LULUVDO_USER_AGENT,
|
||||
ProviderType,
|
||||
)
|
||||
|
||||
# Configure persistent loggers but don't add duplicate handlers when module
|
||||
# is imported multiple times (common in test environments).
|
||||
# Use absolute paths for log files to prevent security issues
|
||||
|
||||
# Determine project root (assuming this file is in src/core/providers/)
|
||||
_module_dir = Path(__file__).parent
|
||||
_project_root = _module_dir.parent.parent.parent
|
||||
_logs_dir = _project_root / "logs"
|
||||
|
||||
# Ensure logs directory exists
|
||||
_logs_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
download_error_logger = logging.getLogger("DownloadErrors")
|
||||
if not download_error_logger.handlers:
|
||||
log_path = _logs_dir / "download_errors.log"
|
||||
download_error_handler = logging.FileHandler(str(log_path))
|
||||
download_error_handler.setLevel(logging.ERROR)
|
||||
download_error_logger.addHandler(download_error_handler)
|
||||
|
||||
noKeyFound_logger = logging.getLogger()
|
||||
|
||||
|
||||
class AniworldLoader(Loader):
|
||||
def __init__(self) -> None:
|
||||
self.SUPPORTED_PROVIDERS = DEFAULT_PROVIDERS
|
||||
# Copy default AniWorld headers so modifications remain local
|
||||
self.AniworldHeaders = dict(ANIWORLD_HEADERS)
|
||||
self.INVALID_PATH_CHARS = INVALID_PATH_CHARS
|
||||
self.RANDOM_USER_AGENT = UserAgent().random
|
||||
self.LULUVDO_USER_AGENT = LULUVDO_USER_AGENT
|
||||
self.PROVIDER_HEADERS = {
|
||||
ProviderType.VIDMOLY.value: ['Referer: "https://vidmoly.to"'],
|
||||
ProviderType.DOODSTREAM.value: ['Referer: "https://dood.li/"'],
|
||||
ProviderType.VOE.value: [f"User-Agent: {self.RANDOM_USER_AGENT}"],
|
||||
ProviderType.LULUVDO.value: [
|
||||
f"User-Agent: {self.LULUVDO_USER_AGENT}",
|
||||
"Accept-Language: de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7",
|
||||
'Origin: "https://luluvdo.com"',
|
||||
'Referer: "https://luluvdo.com/"',
|
||||
],
|
||||
}
|
||||
self.ANIWORLD_TO = "https://aniworld.to"
|
||||
self.session = requests.Session()
|
||||
|
||||
# Cancellation flag for graceful shutdown
|
||||
self._cancel_flag = threading.Event()
|
||||
|
||||
# Configure retries with backoff
|
||||
retries = Retry(
|
||||
total=5, # Number of retries
|
||||
backoff_factor=1, # Delay multiplier (1s, 2s, 4s, ...)
|
||||
status_forcelist=[500, 502, 503, 504],
|
||||
allowed_methods=["GET"]
|
||||
)
|
||||
|
||||
adapter = HTTPAdapter(max_retries=retries)
|
||||
self.session.mount("https://", adapter)
|
||||
# Default HTTP request timeout used for requests.Session calls.
|
||||
# Allows overriding via DOWNLOAD_TIMEOUT env var at runtime.
|
||||
self.DEFAULT_REQUEST_TIMEOUT = int(
|
||||
os.getenv("DOWNLOAD_TIMEOUT") or DEFAULT_DOWNLOAD_TIMEOUT
|
||||
)
|
||||
|
||||
self._KeyHTMLDict = {}
|
||||
self._EpisodeHTMLDict = {}
|
||||
self.Providers = Providers()
|
||||
|
||||
# Events: download_progress is triggered with progress dict
|
||||
self.events = Events()
|
||||
|
||||
def subscribe_download_progress(self, handler):
|
||||
"""Subscribe a handler to the download_progress event.
|
||||
Args:
|
||||
handler: Callable to be called with progress dict.
|
||||
"""
|
||||
self.events.download_progress += handler
|
||||
|
||||
def unsubscribe_download_progress(self, handler):
|
||||
"""Unsubscribe a handler from the download_progress event.
|
||||
Args:
|
||||
handler: Callable previously subscribed.
|
||||
"""
|
||||
self.events.download_progress -= handler
|
||||
|
||||
def clear_cache(self):
|
||||
"""Clear the cached HTML data."""
|
||||
logging.debug("Clearing HTML cache")
|
||||
self._KeyHTMLDict = {}
|
||||
self._EpisodeHTMLDict = {}
|
||||
logging.debug("HTML cache cleared successfully")
|
||||
|
||||
def remove_from_cache(self):
|
||||
"""Remove episode HTML from cache."""
|
||||
logging.debug("Removing episode HTML from cache")
|
||||
self._EpisodeHTMLDict = {}
|
||||
logging.debug("Episode HTML cache cleared")
|
||||
|
||||
def search(self, word: str) -> list:
|
||||
"""Search for anime series.
|
||||
|
||||
Args:
|
||||
word: Search term
|
||||
|
||||
Returns:
|
||||
List of found series
|
||||
"""
|
||||
logging.info(f"Searching for anime with keyword: '{word}'")
|
||||
search_url = (
|
||||
f"{self.ANIWORLD_TO}/ajax/seriesSearch?keyword={quote(word)}"
|
||||
)
|
||||
logging.debug(f"Search URL: {search_url}")
|
||||
anime_list = self.fetch_anime_list(search_url)
|
||||
logging.info(f"Found {len(anime_list)} anime series for keyword '{word}'")
|
||||
|
||||
return anime_list
|
||||
|
||||
def fetch_anime_list(self, url: str) -> list:
|
||||
logging.debug(f"Fetching anime list from URL: {url}")
|
||||
response = self.session.get(url, timeout=self.DEFAULT_REQUEST_TIMEOUT)
|
||||
response.raise_for_status()
|
||||
logging.debug(f"Response status code: {response.status_code}")
|
||||
|
||||
clean_text = response.text.strip()
|
||||
|
||||
try:
|
||||
decoded_data = json.loads(html.unescape(clean_text))
|
||||
logging.debug(f"Successfully decoded JSON data on first attempt")
|
||||
return decoded_data if isinstance(decoded_data, list) else []
|
||||
except json.JSONDecodeError:
|
||||
logging.warning("Initial JSON decode failed, attempting cleanup")
|
||||
try:
|
||||
# Remove BOM and problematic characters
|
||||
clean_text = clean_text.encode('utf-8').decode('utf-8-sig')
|
||||
# Remove problematic characters
|
||||
clean_text = re.sub(r'[\x00-\x1F\x7F-\x9F]', '', clean_text)
|
||||
# Parse the new text
|
||||
decoded_data = json.loads(clean_text)
|
||||
logging.debug("Successfully decoded JSON after cleanup")
|
||||
return decoded_data if isinstance(decoded_data, list) else []
|
||||
except (requests.RequestException, json.JSONDecodeError) as exc:
|
||||
logging.error(f"Failed to decode anime list from {url}: {exc}")
|
||||
raise ValueError("Could not get valid anime: ") from exc
|
||||
|
||||
def _get_language_key(self, language: str) -> int:
|
||||
"""Convert language name to language code.
|
||||
|
||||
Language Codes:
|
||||
1: German Dub
|
||||
2: English Sub
|
||||
3: German Sub
|
||||
"""
|
||||
language_code = 0
|
||||
if language == "German Dub":
|
||||
language_code = 1
|
||||
if language == "English Sub":
|
||||
language_code = 2
|
||||
if language == "German Sub":
|
||||
language_code = 3
|
||||
logging.debug(f"Converted language '{language}' to code {language_code}")
|
||||
return language_code
|
||||
|
||||
def is_language(
|
||||
self,
|
||||
season: int,
|
||||
episode: int,
|
||||
key: str,
|
||||
language: str = "German Dub"
|
||||
) -> bool:
|
||||
"""Check if episode is available in specified language."""
|
||||
logging.debug(f"Checking if S{season:02}E{episode:03} ({key}) is available in {language}")
|
||||
language_code = self._get_language_key(language)
|
||||
|
||||
episode_soup = BeautifulSoup(
|
||||
self._get_episode_html(season, episode, key).content,
|
||||
'html.parser'
|
||||
)
|
||||
change_language_box_div = episode_soup.find(
|
||||
'div', class_='changeLanguageBox')
|
||||
languages = []
|
||||
|
||||
if change_language_box_div:
|
||||
img_tags = change_language_box_div.find_all('img')
|
||||
for img in img_tags:
|
||||
lang_key = img.get('data-lang-key')
|
||||
if lang_key and lang_key.isdigit():
|
||||
languages.append(int(lang_key))
|
||||
|
||||
is_available = language_code in languages
|
||||
logging.debug(f"Available languages for S{season:02}E{episode:03}: {languages}, requested: {language_code}, available: {is_available}")
|
||||
return is_available
|
||||
|
||||
def download(
|
||||
self,
|
||||
base_directory: str,
|
||||
serie_folder: str,
|
||||
season: int,
|
||||
episode: int,
|
||||
key: str,
|
||||
language: str = "German Dub"
|
||||
) -> bool:
|
||||
"""Download episode to specified directory.
|
||||
|
||||
Args:
|
||||
base_directory: Base download directory path
|
||||
serie_folder: Filesystem folder name (metadata only, used for
|
||||
file path construction)
|
||||
season: Season number
|
||||
episode: Episode number
|
||||
key: Series unique identifier from provider (used for
|
||||
identification and API calls)
|
||||
language: Audio language preference (default: German Dub)
|
||||
Returns:
|
||||
bool: True if download succeeded, False otherwise
|
||||
"""
|
||||
logging.info(
|
||||
f"Starting download for S{season:02}E{episode:03} "
|
||||
f"({key}) in {language}"
|
||||
)
|
||||
sanitized_anime_title = ''.join(
|
||||
char for char in self.get_title(key)
|
||||
if char not in self.INVALID_PATH_CHARS
|
||||
)
|
||||
logging.debug(f"Sanitized anime title: {sanitized_anime_title}")
|
||||
|
||||
if season == 0:
|
||||
output_file = (
|
||||
f"{sanitized_anime_title} - "
|
||||
f"Movie {episode:02} - "
|
||||
f"({language}).mp4"
|
||||
)
|
||||
else:
|
||||
output_file = (
|
||||
f"{sanitized_anime_title} - "
|
||||
f"S{season:02}E{episode:03} - "
|
||||
f"({language}).mp4"
|
||||
)
|
||||
|
||||
folder_path = os.path.join(
|
||||
os.path.join(base_directory, serie_folder),
|
||||
f"Season {season}"
|
||||
)
|
||||
output_path = os.path.join(folder_path, output_file)
|
||||
logging.debug(f"Output path: {output_path}")
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
|
||||
temp_dir = "./Temp/"
|
||||
os.makedirs(os.path.dirname(temp_dir), exist_ok=True)
|
||||
temp_path = os.path.join(temp_dir, output_file)
|
||||
logging.debug(f"Temporary path: {temp_path}")
|
||||
|
||||
for provider in self.SUPPORTED_PROVIDERS:
|
||||
logging.debug(f"Attempting download with provider: {provider}")
|
||||
link, header = self._get_direct_link_from_provider(
|
||||
season, episode, key, language
|
||||
)
|
||||
logging.debug("Direct link obtained from provider")
|
||||
|
||||
cancel_flag = self._cancel_flag
|
||||
|
||||
def events_progress_hook(d):
|
||||
if cancel_flag.is_set():
|
||||
logging.info("Cancellation detected in progress hook")
|
||||
raise DownloadCancelled("Download cancelled by user")
|
||||
# Fire the event for progress
|
||||
self.events.download_progress(d)
|
||||
|
||||
ydl_opts = {
|
||||
'fragment_retries': float('inf'),
|
||||
'outtmpl': temp_path,
|
||||
'quiet': True,
|
||||
'no_warnings': True,
|
||||
'progress_with_newline': False,
|
||||
'nocheckcertificate': True,
|
||||
'progress_hooks': [events_progress_hook],
|
||||
}
|
||||
|
||||
if header:
|
||||
ydl_opts['http_headers'] = header
|
||||
logging.debug("Using custom headers for download")
|
||||
|
||||
try:
|
||||
logging.debug("Starting YoutubeDL download")
|
||||
logging.debug(f"Download link: {link[:100]}...")
|
||||
logging.debug(f"YDL options: {ydl_opts}")
|
||||
|
||||
with YoutubeDL(ydl_opts) as ydl:
|
||||
info = ydl.extract_info(link, download=True)
|
||||
logging.debug(
|
||||
f"Download info: "
|
||||
f"title={info.get('title')}, "
|
||||
f"filesize={info.get('filesize')}"
|
||||
)
|
||||
|
||||
if os.path.exists(temp_path):
|
||||
logging.debug("Moving file from temp to final destination")
|
||||
# Use copyfile instead of copy to avoid metadata permission issues
|
||||
shutil.copyfile(temp_path, output_path)
|
||||
os.remove(temp_path)
|
||||
logging.info(
|
||||
f"Download completed successfully: {output_file}"
|
||||
)
|
||||
self.clear_cache()
|
||||
return True
|
||||
else:
|
||||
logging.error(
|
||||
f"Download failed: temp file not found at {temp_path}"
|
||||
)
|
||||
self.clear_cache()
|
||||
return False
|
||||
except BrokenPipeError as e:
|
||||
logging.error(
|
||||
f"Broken pipe error with provider {provider}: {e}. "
|
||||
f"This usually means the stream connection was closed."
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
logging.error(
|
||||
f"YoutubeDL download failed with provider {provider}: "
|
||||
f"{type(e).__name__}: {e}"
|
||||
)
|
||||
continue
|
||||
break
|
||||
|
||||
# If we get here, all providers failed
|
||||
logging.error("All download providers failed")
|
||||
self.clear_cache()
|
||||
return False
|
||||
|
||||
def get_site_key(self) -> str:
|
||||
"""Get the site key for this provider."""
|
||||
return "aniworld.to"
|
||||
|
||||
def get_title(self, key: str) -> str:
|
||||
"""Get anime title from series key."""
|
||||
logging.debug(f"Getting title for key: {key}")
|
||||
soup = BeautifulSoup(
|
||||
self._get_key_html(key).content,
|
||||
'html.parser'
|
||||
)
|
||||
title_div = soup.find('div', class_='series-title')
|
||||
|
||||
if title_div:
|
||||
title = title_div.find('h1').find('span').text
|
||||
logging.debug(f"Found title: {title}")
|
||||
return title
|
||||
|
||||
logging.warning(f"No title found for key: {key}")
|
||||
return ""
|
||||
|
||||
def _get_key_html(self, key: str):
|
||||
"""Get cached HTML for series key.
|
||||
|
||||
Args:
|
||||
key: Series identifier (will be URL-encoded for safety)
|
||||
|
||||
Returns:
|
||||
Cached or fetched HTML response
|
||||
"""
|
||||
if key in self._KeyHTMLDict:
|
||||
logging.debug(f"Using cached HTML for key: {key}")
|
||||
return self._KeyHTMLDict[key]
|
||||
|
||||
# Sanitize key parameter for URL
|
||||
safe_key = quote(key, safe='')
|
||||
url = f"{self.ANIWORLD_TO}/anime/stream/{safe_key}"
|
||||
logging.debug(f"Fetching HTML for key: {key} from {url}")
|
||||
self._KeyHTMLDict[key] = self.session.get(
|
||||
url,
|
||||
timeout=self.DEFAULT_REQUEST_TIMEOUT
|
||||
)
|
||||
logging.debug(f"Cached HTML for key: {key}")
|
||||
return self._KeyHTMLDict[key]
|
||||
|
||||
def _get_episode_html(self, season: int, episode: int, key: str):
|
||||
"""Get cached HTML for episode.
|
||||
|
||||
Args:
|
||||
season: Season number (validated to be positive)
|
||||
episode: Episode number (validated to be positive)
|
||||
key: Series identifier (will be URL-encoded for safety)
|
||||
|
||||
Returns:
|
||||
Cached or fetched HTML response
|
||||
|
||||
Raises:
|
||||
ValueError: If season or episode are invalid
|
||||
"""
|
||||
# Validate season and episode numbers
|
||||
if season < 1 or season > 999:
|
||||
logging.error(f"Invalid season number: {season}")
|
||||
raise ValueError(f"Invalid season number: {season}")
|
||||
if episode < 1 or episode > 9999:
|
||||
logging.error(f"Invalid episode number: {episode}")
|
||||
raise ValueError(f"Invalid episode number: {episode}")
|
||||
|
||||
if key in self._EpisodeHTMLDict:
|
||||
logging.debug(f"Using cached HTML for S{season:02}E{episode:03} ({key})")
|
||||
return self._EpisodeHTMLDict[(key, season, episode)]
|
||||
|
||||
# Sanitize key parameter for URL
|
||||
safe_key = quote(key, safe='')
|
||||
link = (
|
||||
f"{self.ANIWORLD_TO}/anime/stream/{safe_key}/"
|
||||
f"staffel-{season}/episode-{episode}"
|
||||
)
|
||||
logging.debug(f"Fetching episode HTML from: {link}")
|
||||
html = self.session.get(link, timeout=self.DEFAULT_REQUEST_TIMEOUT)
|
||||
self._EpisodeHTMLDict[(key, season, episode)] = html
|
||||
logging.debug(f"Cached episode HTML for S{season:02}E{episode:03} ({key})")
|
||||
return self._EpisodeHTMLDict[(key, season, episode)]
|
||||
|
||||
def _get_provider_from_html(
|
||||
self,
|
||||
season: int,
|
||||
episode: int,
|
||||
key: str
|
||||
) -> dict:
|
||||
"""Parse HTML content to extract streaming providers.
|
||||
|
||||
Returns a dictionary with provider names as keys
|
||||
and language key-to-redirect URL mappings as values.
|
||||
|
||||
Example:
|
||||
{
|
||||
'VOE': {1: 'https://aniworld.to/redirect/1766412',
|
||||
2: 'https://aniworld.to/redirect/1766405'},
|
||||
}
|
||||
"""
|
||||
logging.debug(f"Extracting providers from HTML for S{season:02}E{episode:03} ({key})")
|
||||
soup = BeautifulSoup(
|
||||
self._get_episode_html(season, episode, key).content,
|
||||
'html.parser'
|
||||
)
|
||||
providers: dict[str, dict[int, str]] = {}
|
||||
|
||||
episode_links = soup.find_all(
|
||||
'li', class_=lambda x: x and x.startswith('episodeLink')
|
||||
)
|
||||
|
||||
if not episode_links:
|
||||
logging.warning(f"No episode links found for S{season:02}E{episode:03} ({key})")
|
||||
return providers
|
||||
|
||||
for link in episode_links:
|
||||
provider_name_tag = link.find('h4')
|
||||
provider_name = (
|
||||
provider_name_tag.text.strip()
|
||||
if provider_name_tag else None
|
||||
)
|
||||
|
||||
redirect_link_tag = link.find('a', class_='watchEpisode')
|
||||
redirect_link = (
|
||||
redirect_link_tag['href']
|
||||
if redirect_link_tag else None
|
||||
)
|
||||
|
||||
lang_key = link.get('data-lang-key')
|
||||
lang_key = (
|
||||
int(lang_key)
|
||||
if lang_key and lang_key.isdigit() else None
|
||||
)
|
||||
|
||||
if provider_name and redirect_link and lang_key:
|
||||
if provider_name not in providers:
|
||||
providers[provider_name] = {}
|
||||
providers[provider_name][lang_key] = (
|
||||
f"{self.ANIWORLD_TO}{redirect_link}"
|
||||
)
|
||||
logging.debug(f"Found provider: {provider_name}, lang_key: {lang_key}")
|
||||
|
||||
logging.debug(f"Total providers found: {len(providers)}")
|
||||
return providers
|
||||
|
||||
def _get_redirect_link(
|
||||
self,
|
||||
season: int,
|
||||
episode: int,
|
||||
key: str,
|
||||
language: str = "German Dub"
|
||||
):
|
||||
"""Get redirect link for episode in specified language."""
|
||||
logging.debug(f"Getting redirect link for S{season:02}E{episode:03} ({key}) in {language}")
|
||||
language_code = self._get_language_key(language)
|
||||
if self.is_language(season, episode, key, language):
|
||||
for (provider_name, lang_dict) in (
|
||||
self._get_provider_from_html(
|
||||
season, episode, key
|
||||
).items()
|
||||
):
|
||||
if language_code in lang_dict:
|
||||
logging.debug(f"Found redirect link with provider: {provider_name}")
|
||||
return (lang_dict[language_code], provider_name)
|
||||
logging.warning(f"No redirect link found for S{season:02}E{episode:03} ({key}) in {language}")
|
||||
return None
|
||||
|
||||
def _get_embeded_link(
|
||||
self,
|
||||
season: int,
|
||||
episode: int,
|
||||
key: str,
|
||||
language: str = "German Dub"
|
||||
):
|
||||
"""Get embedded link from redirect link."""
|
||||
logging.debug(f"Getting embedded link for S{season:02}E{episode:03} ({key}) in {language}")
|
||||
redirect_link, provider_name = (
|
||||
self._get_redirect_link(season, episode, key, language)
|
||||
)
|
||||
logging.debug(f"Redirect link: {redirect_link}, provider: {provider_name}")
|
||||
|
||||
embeded_link = self.session.get(
|
||||
redirect_link,
|
||||
timeout=self.DEFAULT_REQUEST_TIMEOUT,
|
||||
headers={'User-Agent': self.RANDOM_USER_AGENT}
|
||||
).url
|
||||
logging.debug(f"Embedded link: {embeded_link}")
|
||||
return embeded_link
|
||||
|
||||
def _get_direct_link_from_provider(
|
||||
self,
|
||||
season: int,
|
||||
episode: int,
|
||||
key: str,
|
||||
language: str = "German Dub"
|
||||
):
|
||||
"""Get direct download link from streaming provider."""
|
||||
logging.debug(f"Getting direct link from provider for S{season:02}E{episode:03} ({key}) in {language}")
|
||||
embeded_link = self._get_embeded_link(
|
||||
season, episode, key, language
|
||||
)
|
||||
if embeded_link is None:
|
||||
logging.error(f"No embedded link found for S{season:02}E{episode:03} ({key})")
|
||||
return None
|
||||
|
||||
logging.debug(f"Using VOE provider to extract direct link")
|
||||
return self.Providers.GetProvider(
|
||||
"VOE"
|
||||
).get_link(embeded_link, self.DEFAULT_REQUEST_TIMEOUT)
|
||||
|
||||
def get_season_episode_count(self, slug: str) -> dict:
|
||||
"""Get episode count for each season.
|
||||
|
||||
Args:
|
||||
slug: Series identifier (will be URL-encoded for safety)
|
||||
|
||||
Returns:
|
||||
Dictionary mapping season numbers to episode counts
|
||||
"""
|
||||
logging.info(f"Getting season and episode count for slug: {slug}")
|
||||
# Sanitize slug parameter for URL
|
||||
safe_slug = quote(slug, safe='')
|
||||
base_url = f"{self.ANIWORLD_TO}/anime/stream/{safe_slug}/"
|
||||
logging.debug(f"Base URL: {base_url}")
|
||||
response = requests.get(base_url, timeout=self.DEFAULT_REQUEST_TIMEOUT)
|
||||
soup = BeautifulSoup(response.content, 'html.parser')
|
||||
|
||||
season_meta = soup.find('meta', itemprop='numberOfSeasons')
|
||||
number_of_seasons = int(season_meta['content']) if season_meta else 0
|
||||
logging.info(f"Found {number_of_seasons} seasons for '{slug}'")
|
||||
|
||||
episode_counts = {}
|
||||
|
||||
for season in range(1, number_of_seasons + 1):
|
||||
season_url = f"{base_url}staffel-{season}"
|
||||
logging.debug(f"Fetching episodes for season {season} from: {season_url}")
|
||||
response = requests.get(
|
||||
season_url,
|
||||
timeout=self.DEFAULT_REQUEST_TIMEOUT,
|
||||
)
|
||||
soup = BeautifulSoup(response.content, 'html.parser')
|
||||
|
||||
episode_links = soup.find_all('a', href=True)
|
||||
unique_links = set(
|
||||
link['href']
|
||||
for link in episode_links
|
||||
if f"staffel-{season}/episode-" in link['href']
|
||||
)
|
||||
|
||||
episode_counts[season] = len(unique_links)
|
||||
logging.debug(f"Season {season} has {episode_counts[season]} episodes")
|
||||
|
||||
logging.info(f"Episode count retrieval complete for '{slug}': {episode_counts}")
|
||||
return episode_counts
|
||||
Binary file not shown.
Binary file not shown.
@@ -1,88 +0,0 @@
|
||||
"""Resolve Doodstream embed players into direct download URLs."""
|
||||
|
||||
import random
|
||||
import re
|
||||
import string
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from fake_useragent import UserAgent
|
||||
|
||||
from .Provider import Provider
|
||||
|
||||
# Precompiled regex patterns to extract the ``pass_md5`` endpoint and the
|
||||
# session token embedded in the obfuscated player script. Compiling once keeps
|
||||
# repeated invocations fast and documents the parsing intent.
|
||||
PASS_MD5_PATTERN = re.compile(r"\$\.get\('([^']*/pass_md5/[^']*)'")
|
||||
TOKEN_PATTERN = re.compile(r"token=([a-zA-Z0-9]+)")
|
||||
|
||||
|
||||
class Doodstream(Provider):
|
||||
"""Doodstream video provider implementation."""
|
||||
|
||||
def __init__(self):
|
||||
self.RANDOM_USER_AGENT = UserAgent().random
|
||||
|
||||
def get_link(
|
||||
self, embedded_link: str, timeout: int
|
||||
) -> tuple[str, dict[str, Any]]:
|
||||
"""
|
||||
Extract direct download link from Doodstream embedded player.
|
||||
|
||||
Args:
|
||||
embedded_link: URL of the embedded Doodstream player
|
||||
timeout: Request timeout in seconds
|
||||
|
||||
Returns:
|
||||
Tuple of (direct_link, headers)
|
||||
"""
|
||||
headers = {
|
||||
"User-Agent": self.RANDOM_USER_AGENT,
|
||||
"Referer": "https://dood.li/",
|
||||
}
|
||||
|
||||
def extract_data(pattern: re.Pattern[str], content: str) -> str | None:
|
||||
"""Extract data using a compiled regex pattern."""
|
||||
match = pattern.search(content)
|
||||
return match.group(1) if match else None
|
||||
|
||||
def generate_random_string(length: int = 10) -> str:
|
||||
"""Generate random alphanumeric string."""
|
||||
charset = string.ascii_letters + string.digits
|
||||
return "".join(random.choices(charset, k=length))
|
||||
|
||||
# WARNING: SSL verification disabled for doodstream compatibility
|
||||
# This is a known limitation with this streaming provider
|
||||
response = requests.get(
|
||||
embedded_link,
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
verify=True, # Changed from False for security
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
pass_md5_url = extract_data(PASS_MD5_PATTERN, response.text)
|
||||
if not pass_md5_url:
|
||||
raise ValueError(f"pass_md5 URL not found using {embedded_link}.")
|
||||
|
||||
full_md5_url = f"https://dood.li{pass_md5_url}"
|
||||
|
||||
token = extract_data(TOKEN_PATTERN, response.text)
|
||||
if not token:
|
||||
raise ValueError(f"Token not found using {embedded_link}.")
|
||||
|
||||
md5_response = requests.get(
|
||||
full_md5_url, headers=headers, timeout=timeout, verify=True
|
||||
)
|
||||
md5_response.raise_for_status()
|
||||
video_base_url = md5_response.text.strip()
|
||||
|
||||
random_string = generate_random_string(10)
|
||||
expiry = int(time.time())
|
||||
|
||||
direct_link = (
|
||||
f"{video_base_url}{random_string}?token={token}&expiry={expiry}"
|
||||
)
|
||||
|
||||
return direct_link, headers
|
||||
@@ -1,59 +0,0 @@
|
||||
"""Resolve Filemoon embed pages into direct streaming asset URLs."""
|
||||
|
||||
import re
|
||||
|
||||
import requests
|
||||
from aniworld import config
|
||||
|
||||
# import jsbeautifier.unpackers.packer as packer
|
||||
|
||||
|
||||
# Match the embedded ``iframe`` pointing to the actual Filemoon player.
|
||||
REDIRECT_REGEX = re.compile(
|
||||
r'<iframe *(?:[^>]+ )?src=(?:\'([^\']+)\'|"([^"]+)")[^>]*>')
|
||||
# The player HTML hides an ``eval`` wrapped script with ``data-cfasync``
|
||||
# disabled; capture the entire script body for unpacking.
|
||||
SCRIPT_REGEX = re.compile(
|
||||
r'(?s)<script\s+[^>]*?data-cfasync=["\']?false["\']?[^>]*>(.+?)</script>')
|
||||
# Extract the direct ``file:"<m3u8>"`` URL once the script is unpacked.
|
||||
VIDEO_URL_REGEX = re.compile(r'file:\s*"([^"]+\.m3u8[^"]*)"')
|
||||
|
||||
# TODO Implement this script fully
|
||||
|
||||
|
||||
def get_direct_link_from_filemoon(embeded_filemoon_link: str):
|
||||
session = requests.Session()
|
||||
session.verify = False
|
||||
|
||||
headers = {
|
||||
"User-Agent": config.RANDOM_USER_AGENT,
|
||||
"Referer": embeded_filemoon_link,
|
||||
}
|
||||
|
||||
response = session.get(embeded_filemoon_link, headers=headers)
|
||||
source = response.text
|
||||
|
||||
match = REDIRECT_REGEX.search(source)
|
||||
if match:
|
||||
redirect_url = match.group(1) or match.group(2)
|
||||
response = session.get(redirect_url, headers=headers)
|
||||
source = response.text
|
||||
|
||||
for script_match in SCRIPT_REGEX.finditer(source):
|
||||
script_content = script_match.group(1).strip()
|
||||
|
||||
if not script_content.startswith("eval("):
|
||||
continue
|
||||
|
||||
if packer.detect(script_content):
|
||||
unpacked = packer.unpack(script_content)
|
||||
video_match = VIDEO_URL_REGEX.search(unpacked)
|
||||
if video_match:
|
||||
return video_match.group(1)
|
||||
|
||||
raise Exception("No Video link found!")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
url = input("Enter Filemoon Link: ")
|
||||
print(get_direct_link_from_filemoon(url))
|
||||
@@ -1,95 +0,0 @@
|
||||
"""Helpers for extracting direct stream URLs from hanime.tv pages."""
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
import requests
|
||||
from aniworld.config import DEFAULT_REQUEST_TIMEOUT
|
||||
|
||||
|
||||
def fetch_page_content(url):
|
||||
try:
|
||||
response = requests.get(url, timeout=DEFAULT_REQUEST_TIMEOUT)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"Failed to fetch the page content: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def extract_video_data(page_content):
|
||||
# ``videos_manifest`` lines embed a JSON blob with the stream metadata
|
||||
# inside a larger script tag; grab that entire line for further parsing.
|
||||
match = re.search(r'^.*videos_manifest.*$', page_content, re.MULTILINE)
|
||||
if not match:
|
||||
raise ValueError("Failed to extract video manifest from the response.")
|
||||
|
||||
json_str = match.group(0)[match.group(0).find(
|
||||
'{'):match.group(0).rfind('}') + 1]
|
||||
return json.loads(json_str)
|
||||
|
||||
|
||||
def get_streams(url):
|
||||
page_content = fetch_page_content(url)
|
||||
data = extract_video_data(page_content)
|
||||
video_info = data['state']['data']['video']
|
||||
name = video_info['hentai_video']['name']
|
||||
streams = video_info['videos_manifest']['servers'][0]['streams']
|
||||
|
||||
return {"name": name, "streams": streams}
|
||||
|
||||
|
||||
def display_streams(streams):
|
||||
if not streams:
|
||||
print("No streams available.")
|
||||
return
|
||||
|
||||
print("Available qualities:")
|
||||
for i, stream in enumerate(streams, 1):
|
||||
premium_tag = "(Premium)" if not stream['is_guest_allowed'] else ""
|
||||
print(
|
||||
f"{i}. {stream['width']}x{stream['height']}\t"
|
||||
f"({stream['filesize_mbs']}MB) {premium_tag}")
|
||||
|
||||
|
||||
def get_user_selection(streams):
|
||||
try:
|
||||
selected_index = int(input("Select a stream: ").strip()) - 1
|
||||
if 0 <= selected_index < len(streams):
|
||||
return selected_index
|
||||
|
||||
print("Invalid selection.")
|
||||
return None
|
||||
except ValueError:
|
||||
print("Invalid input.")
|
||||
return None
|
||||
|
||||
|
||||
def get_direct_link_from_hanime(url=None):
|
||||
try:
|
||||
if url is None:
|
||||
if len(sys.argv) > 1:
|
||||
url = sys.argv[1]
|
||||
else:
|
||||
url = input("Please enter the hanime.tv video URL: ").strip()
|
||||
|
||||
try:
|
||||
video_data = get_streams(url)
|
||||
print(f"Video: {video_data['name']}")
|
||||
print('*' * 40)
|
||||
display_streams(video_data['streams'])
|
||||
|
||||
selected_index = None
|
||||
while selected_index is None:
|
||||
selected_index = get_user_selection(video_data['streams'])
|
||||
|
||||
print(f"M3U8 URL: {video_data['streams'][selected_index]['url']}")
|
||||
except ValueError as e:
|
||||
print(f"Error: {e}")
|
||||
except KeyboardInterrupt:
|
||||
print("\nOperation cancelled by user.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
get_direct_link_from_hanime()
|
||||
@@ -1,59 +0,0 @@
|
||||
import json
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
|
||||
# TODO Doesn't work on download yet and has to be implemented
|
||||
|
||||
|
||||
def get_direct_link_from_loadx(embeded_loadx_link: str):
|
||||
"""Extract direct download link from LoadX streaming provider.
|
||||
|
||||
Args:
|
||||
embeded_loadx_link: Embedded LoadX link
|
||||
|
||||
Returns:
|
||||
str: Direct video URL
|
||||
|
||||
Raises:
|
||||
ValueError: If link extraction fails
|
||||
"""
|
||||
# Default timeout for network requests
|
||||
timeout = 30
|
||||
|
||||
response = requests.head(
|
||||
embeded_loadx_link,
|
||||
allow_redirects=True,
|
||||
verify=True,
|
||||
timeout=timeout
|
||||
)
|
||||
|
||||
parsed_url = urlparse(response.url)
|
||||
path_parts = parsed_url.path.split("/")
|
||||
if len(path_parts) < 3:
|
||||
raise ValueError("Invalid path!")
|
||||
|
||||
id_hash = path_parts[2]
|
||||
host = parsed_url.netloc
|
||||
|
||||
post_url = f"https://{host}/player/index.php?data={id_hash}&do=getVideo"
|
||||
headers = {"X-Requested-With": "XMLHttpRequest"}
|
||||
response = requests.post(
|
||||
post_url,
|
||||
headers=headers,
|
||||
verify=True,
|
||||
timeout=timeout
|
||||
)
|
||||
|
||||
data = json.loads(response.text)
|
||||
print(data)
|
||||
video_url = data.get("videoSource")
|
||||
if not video_url:
|
||||
raise ValueError("No Video link found!")
|
||||
|
||||
return video_url
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
url = input("Enter Loadx Link: ")
|
||||
print(get_direct_link_from_loadx(url))
|
||||
@@ -1,40 +0,0 @@
|
||||
import re
|
||||
|
||||
import requests
|
||||
from aniworld import config
|
||||
|
||||
|
||||
def get_direct_link_from_luluvdo(embeded_luluvdo_link, arguments=None):
|
||||
luluvdo_id = embeded_luluvdo_link.split('/')[-1]
|
||||
filelink = (
|
||||
f"https://luluvdo.com/dl?op=embed&file_code={luluvdo_id}&embed=1&referer=luluvdo.com&adb=0"
|
||||
)
|
||||
|
||||
# The User-Agent needs to be the same as the direct-link ones to work
|
||||
headers = {
|
||||
"Origin": "https://luluvdo.com",
|
||||
"Referer": "https://luluvdo.com/",
|
||||
"User-Agent": config.LULUVDO_USER_AGENT
|
||||
}
|
||||
|
||||
if arguments.action == "Download":
|
||||
headers["Accept-Language"] = "de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7"
|
||||
|
||||
response = requests.get(filelink, headers=headers,
|
||||
timeout=config.DEFAULT_REQUEST_TIMEOUT)
|
||||
|
||||
if response.status_code == 200:
|
||||
# Capture the ``file:"<url>"`` assignment embedded in the player
|
||||
# configuration so we can return the stream URL.
|
||||
pattern = r'file:\s*"([^"]+)"'
|
||||
matches = re.findall(pattern, str(response.text))
|
||||
|
||||
if matches:
|
||||
return matches[0]
|
||||
|
||||
raise ValueError("No match found")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
url = input("Enter Luluvdo Link: ")
|
||||
print(get_direct_link_from_luluvdo(url))
|
||||
@@ -1,45 +0,0 @@
|
||||
import base64
|
||||
import re
|
||||
|
||||
import requests
|
||||
from aniworld.config import DEFAULT_REQUEST_TIMEOUT, RANDOM_USER_AGENT
|
||||
|
||||
# Capture the base64 payload hidden inside the obfuscated ``_0x5opu234``
|
||||
# assignment. The named group lets us pull out the encoded blob directly.
|
||||
SPEEDFILES_PATTERN = re.compile(r'var _0x5opu234 = "(?P<encoded_data>.*?)";')
|
||||
|
||||
|
||||
def get_direct_link_from_speedfiles(embeded_speedfiles_link):
|
||||
response = requests.get(
|
||||
embeded_speedfiles_link,
|
||||
timeout=DEFAULT_REQUEST_TIMEOUT,
|
||||
headers={'User-Agent': RANDOM_USER_AGENT}
|
||||
)
|
||||
|
||||
if "<span class=\"inline-block\">Web server is down</span>" in response.text:
|
||||
raise ValueError(
|
||||
"The SpeedFiles server is currently down.\n"
|
||||
"Please try again later or choose a different hoster."
|
||||
)
|
||||
|
||||
match = SPEEDFILES_PATTERN.search(response.text)
|
||||
|
||||
if not match:
|
||||
raise ValueError("Pattern not found in the response.")
|
||||
|
||||
encoded_data = match.group("encoded_data")
|
||||
decoded = base64.b64decode(encoded_data).decode()
|
||||
decoded = decoded.swapcase()[::-1]
|
||||
decoded = base64.b64decode(decoded).decode()[::-1]
|
||||
decoded_hex = ''.join(chr(int(decoded[i:i + 2], 16))
|
||||
for i in range(0, len(decoded), 2))
|
||||
shifted = ''.join(chr(ord(char) - 3) for char in decoded_hex)
|
||||
result = base64.b64decode(shifted.swapcase()[::-1]).decode()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
speedfiles_link = input("Enter Speedfiles Link: ")
|
||||
print(get_direct_link_from_speedfiles(
|
||||
embeded_speedfiles_link=speedfiles_link))
|
||||
@@ -1,2 +0,0 @@
|
||||
def get_direct_link_from_streamtape(embeded_streamtape_link: str) -> str:
|
||||
pass
|
||||
@@ -1,35 +0,0 @@
|
||||
import re
|
||||
|
||||
import requests
|
||||
from aniworld.config import DEFAULT_REQUEST_TIMEOUT, RANDOM_USER_AGENT
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
|
||||
def get_direct_link_from_vidmoly(embeded_vidmoly_link: str):
|
||||
response = requests.get(
|
||||
embeded_vidmoly_link,
|
||||
headers={'User-Agent': RANDOM_USER_AGENT},
|
||||
timeout=DEFAULT_REQUEST_TIMEOUT
|
||||
)
|
||||
html_content = response.text
|
||||
soup = BeautifulSoup(html_content, 'html.parser')
|
||||
scripts = soup.find_all('script')
|
||||
|
||||
# Match the ``file:"<url>"`` assignment inside the obfuscated player
|
||||
# script so we can recover the direct MP4 source URL.
|
||||
file_link_pattern = r'file:\s*"(https?://.*?)"'
|
||||
|
||||
for script in scripts:
|
||||
if script.string:
|
||||
match = re.search(file_link_pattern, script.string)
|
||||
if match:
|
||||
file_link = match.group(1)
|
||||
return file_link
|
||||
|
||||
raise ValueError("No direct link found.")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
link = input("Enter Vidmoly Link: ")
|
||||
print('Note: --referer "https://vidmoly.to"')
|
||||
print(get_direct_link_from_vidmoly(embeded_vidmoly_link=link))
|
||||
@@ -1,30 +0,0 @@
|
||||
import re
|
||||
|
||||
import requests
|
||||
from aniworld.config import DEFAULT_REQUEST_TIMEOUT, RANDOM_USER_AGENT
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
|
||||
def get_direct_link_from_vidoza(embeded_vidoza_link: str) -> str:
|
||||
response = requests.get(
|
||||
embeded_vidoza_link,
|
||||
headers={'User-Agent': RANDOM_USER_AGENT},
|
||||
timeout=DEFAULT_REQUEST_TIMEOUT
|
||||
)
|
||||
|
||||
soup = BeautifulSoup(response.content, "html.parser")
|
||||
|
||||
for tag in soup.find_all('script'):
|
||||
if 'sourcesCode:' in tag.text:
|
||||
# Script blocks contain a ``sourcesCode`` object with ``src``
|
||||
# assignments; extract the first URL between the quotes.
|
||||
match = re.search(r'src: "(.*?)"', tag.text)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
raise ValueError("No direct link found.")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
link = input("Enter Vidoza Link: ")
|
||||
print(get_direct_link_from_vidoza(embeded_vidoza_link=link))
|
||||
Binary file not shown.
Binary file not shown.
@@ -36,10 +36,10 @@ class ConfigEncryption:
|
||||
def _ensure_key_exists(self) -> None:
|
||||
"""Ensure encryption key exists or create one."""
|
||||
if not self.key_file.exists():
|
||||
logger.info(f"Creating new encryption key at {self.key_file}")
|
||||
logger.info("Creating new encryption key at %s", self.key_file)
|
||||
self._generate_new_key()
|
||||
else:
|
||||
logger.info(f"Using existing encryption key from {self.key_file}")
|
||||
logger.info("Using existing encryption key from %s", self.key_file)
|
||||
|
||||
def _generate_new_key(self) -> None:
|
||||
"""Generate and store a new encryption key."""
|
||||
@@ -56,7 +56,7 @@ class ConfigEncryption:
|
||||
logger.info("Generated new encryption key")
|
||||
|
||||
except IOError as e:
|
||||
logger.error(f"Failed to generate encryption key: {e}")
|
||||
logger.error("Failed to generate encryption key: %s", e)
|
||||
raise
|
||||
|
||||
def _load_key(self) -> bytes:
|
||||
@@ -77,7 +77,7 @@ class ConfigEncryption:
|
||||
key = self.key_file.read_bytes()
|
||||
return key
|
||||
except IOError as e:
|
||||
logger.error(f"Failed to load encryption key: {e}")
|
||||
logger.error("Failed to load encryption key: %s", e)
|
||||
raise
|
||||
|
||||
def _get_cipher(self) -> Fernet:
|
||||
@@ -117,7 +117,7 @@ class ConfigEncryption:
|
||||
return encrypted_str
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to encrypt value: {e}")
|
||||
logger.error("Failed to encrypt value: %s", e)
|
||||
raise
|
||||
|
||||
def decrypt_value(self, encrypted_value: str) -> str:
|
||||
@@ -149,7 +149,7 @@ class ConfigEncryption:
|
||||
return decrypted_str
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to decrypt value: {e}")
|
||||
logger.error("Failed to decrypt value: %s", e)
|
||||
raise
|
||||
|
||||
def encrypt_config(self, config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
@@ -191,9 +191,9 @@ class ConfigEncryption:
|
||||
'encrypted': True,
|
||||
'value': self.encrypt_value(value)
|
||||
}
|
||||
logger.debug(f"Encrypted config field: {key}")
|
||||
logger.debug("Encrypted config field: %s", key)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to encrypt {key}: {e}")
|
||||
logger.warning("Failed to encrypt %s: %s", key, e)
|
||||
encrypted_config[key] = value
|
||||
else:
|
||||
encrypted_config[key] = value
|
||||
@@ -222,9 +222,9 @@ class ConfigEncryption:
|
||||
decrypted_config[key] = self.decrypt_value(
|
||||
value['value']
|
||||
)
|
||||
logger.debug(f"Decrypted config field: {key}")
|
||||
logger.debug("Decrypted config field: %s", key)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to decrypt {key}: {e}")
|
||||
logger.error("Failed to decrypt %s: %s", key, e)
|
||||
decrypted_config[key] = None
|
||||
else:
|
||||
decrypted_config[key] = value
|
||||
@@ -248,7 +248,7 @@ class ConfigEncryption:
|
||||
if self.key_file.exists():
|
||||
backup_path = self.key_file.with_suffix('.key.bak')
|
||||
self.key_file.rename(backup_path)
|
||||
logger.info(f"Backed up old key to {backup_path}")
|
||||
logger.info("Backed up old key to %s", backup_path)
|
||||
|
||||
# Generate new key
|
||||
if new_key_file:
|
||||
|
||||
@@ -276,13 +276,13 @@ class DatabaseIntegrityChecker:
|
||||
removed += 1
|
||||
|
||||
self.session.commit()
|
||||
logger.info(f"Removed {removed} orphaned records")
|
||||
logger.info("Removed %s orphaned records", removed)
|
||||
|
||||
return removed
|
||||
|
||||
except Exception as e:
|
||||
self.session.rollback()
|
||||
logger.error(f"Error removing orphaned records: {e}")
|
||||
logger.error("Error removing orphaned records: %s", e)
|
||||
raise
|
||||
|
||||
|
||||
|
||||
@@ -39,13 +39,15 @@ class FileIntegrityManager:
|
||||
self.checksums = json.load(f)
|
||||
count = len(self.checksums)
|
||||
logger.info(
|
||||
f"Loaded {count} checksums from {self.checksum_file}"
|
||||
"Loaded %d checksums from %s",
|
||||
count,
|
||||
self.checksum_file,
|
||||
)
|
||||
except (json.JSONDecodeError, IOError) as e:
|
||||
logger.error(f"Failed to load checksums: {e}")
|
||||
logger.error("Failed to load checksums: %s", e)
|
||||
self.checksums = {}
|
||||
else:
|
||||
logger.info(f"Checksum file does not exist: {self.checksum_file}")
|
||||
logger.info("Checksum file does not exist: %s", self.checksum_file)
|
||||
self.checksums = {}
|
||||
|
||||
def _save_checksums(self) -> None:
|
||||
@@ -56,10 +58,12 @@ class FileIntegrityManager:
|
||||
json.dump(self.checksums, f, indent=2)
|
||||
count = len(self.checksums)
|
||||
logger.debug(
|
||||
f"Saved {count} checksums to {self.checksum_file}"
|
||||
"Saved %d checksums to %s",
|
||||
count,
|
||||
self.checksum_file,
|
||||
)
|
||||
except IOError as e:
|
||||
logger.error(f"Failed to save checksums: {e}")
|
||||
logger.error("Failed to save checksums: %s", e)
|
||||
|
||||
def calculate_checksum(
|
||||
self, file_path: Path, algorithm: str = "sha256"
|
||||
@@ -94,12 +98,15 @@ class FileIntegrityManager:
|
||||
checksum = hash_obj.hexdigest()
|
||||
filename = file_path.name
|
||||
logger.debug(
|
||||
f"Calculated {algorithm} checksum for {filename}: {checksum}"
|
||||
"Calculated %s checksum for %s: %s",
|
||||
algorithm,
|
||||
filename,
|
||||
checksum,
|
||||
)
|
||||
return checksum
|
||||
|
||||
except IOError as e:
|
||||
logger.error(f"Failed to read file {file_path}: {e}")
|
||||
logger.error("Failed to read file %s: %s", file_path, e)
|
||||
raise
|
||||
|
||||
def store_checksum(
|
||||
@@ -126,7 +133,7 @@ class FileIntegrityManager:
|
||||
self.checksums[key] = checksum
|
||||
self._save_checksums()
|
||||
|
||||
logger.info(f"Stored checksum for {file_path.name}")
|
||||
logger.info("Stored checksum for %s", file_path.name)
|
||||
return checksum
|
||||
|
||||
def verify_checksum(
|
||||
@@ -197,10 +204,10 @@ class FileIntegrityManager:
|
||||
if key in self.checksums:
|
||||
del self.checksums[key]
|
||||
self._save_checksums()
|
||||
logger.info(f"Removed checksum for {file_path.name}")
|
||||
logger.info("Removed checksum for %s", file_path.name)
|
||||
return True
|
||||
else:
|
||||
logger.debug(f"No checksum found to remove for {file_path.name}")
|
||||
logger.debug("No checksum found to remove for %s", file_path.name)
|
||||
return False
|
||||
|
||||
def has_checksum(self, file_path: Path) -> bool:
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -14,14 +14,15 @@ import asyncio
|
||||
import logging
|
||||
import os
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from events import Events
|
||||
|
||||
from src.core.entities.SerieList import SerieList
|
||||
from src.core.entities.series import Serie
|
||||
from src.core.providers.provider_factory import Loaders
|
||||
from src.core.SerieScanner import SerieScanner
|
||||
from src.config.settings import settings
|
||||
from src.server.database.SerieList import SerieList
|
||||
from src.server.database.models import AnimeSeries
|
||||
from src.server.providers.provider_factory import Loaders
|
||||
from src.server.SerieScanner import SerieScanner
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -159,17 +160,21 @@ class SeriesApp:
|
||||
self.loaders = Loaders()
|
||||
self.loader = self.loaders.GetLoader(key="aniworld.to")
|
||||
self.serie_scanner = SerieScanner(
|
||||
directory_to_search, self.loader
|
||||
directory_to_search,
|
||||
self.loader,
|
||||
)
|
||||
# Series will be loaded from database by the service layer during application setup
|
||||
self.list = SerieList(self.directory_to_search)
|
||||
self.series_list: List[Any] = []
|
||||
# Synchronous init used during constructor to avoid awaiting
|
||||
# in __init__
|
||||
self._init_list_sync()
|
||||
|
||||
# Initialize empty list - series loaded later via load_series_from_list()
|
||||
# No need to call _init_list_sync() anymore
|
||||
|
||||
# NFO service removed - metadata handling moved to server layer
|
||||
self.nfo_service = None
|
||||
|
||||
logger.info(
|
||||
"SeriesApp initialized for directory: %s",
|
||||
directory_to_search
|
||||
directory_to_search,
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -222,26 +227,6 @@ class SeriesApp:
|
||||
len(self.series_list)
|
||||
)
|
||||
|
||||
def _init_list_sync(self) -> None:
|
||||
"""Synchronous initialization helper for constructor."""
|
||||
self.series_list = self.list.GetMissingEpisode()
|
||||
logger.debug(
|
||||
"Loaded %d series with missing episodes",
|
||||
len(self.series_list)
|
||||
)
|
||||
|
||||
async def _init_list(self) -> None:
|
||||
"""Initialize the series list with missing episodes (async)."""
|
||||
loop = asyncio.get_running_loop()
|
||||
self.series_list = await loop.run_in_executor(
|
||||
self.executor,
|
||||
self.list.GetMissingEpisode
|
||||
)
|
||||
logger.debug(
|
||||
"Loaded %d series with missing episodes",
|
||||
len(self.series_list)
|
||||
)
|
||||
|
||||
async def search(self, words: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Search for anime series (async).
|
||||
@@ -351,9 +336,12 @@ class SeriesApp:
|
||||
try:
|
||||
def download_progress_handler(progress_info):
|
||||
"""Handle download progress events from loader."""
|
||||
logger.debug(
|
||||
"download_progress_handler called with: %s", progress_info
|
||||
)
|
||||
# Throttle progress logging to avoid spam
|
||||
status = progress_info.get("status", "")
|
||||
if status in ("downloading", "finished"):
|
||||
logger.debug(
|
||||
"download_progress_handler called with: %s", progress_info
|
||||
)
|
||||
|
||||
downloaded = progress_info.get('downloaded_bytes', 0)
|
||||
total_bytes = (
|
||||
@@ -665,7 +653,7 @@ class SeriesApp:
|
||||
"""
|
||||
await self._init_list()
|
||||
|
||||
def _get_serie_by_key(self, key: str) -> Optional[Serie]:
|
||||
def _get_serie_by_key(self, key: str) -> Optional[AnimeSeries]:
|
||||
"""
|
||||
Get a series by its unique provider key.
|
||||
|
||||
@@ -676,7 +664,7 @@ class SeriesApp:
|
||||
"attack-on-titan")
|
||||
|
||||
Returns:
|
||||
The Serie instance if found, None otherwise
|
||||
The AnimeSeries instance if found, None otherwise
|
||||
|
||||
Note:
|
||||
This method uses the SerieList.get_by_key() method which
|
||||
@@ -684,39 +672,40 @@ class SeriesApp:
|
||||
"""
|
||||
return self.list.get_by_key(key)
|
||||
|
||||
def get_all_series_from_data_files(self) -> List[Serie]:
|
||||
def get_all_series_from_data_files(self) -> List[AnimeSeries]:
|
||||
"""
|
||||
Get all series from data files in the anime directory.
|
||||
|
||||
Scans the directory_to_search for all 'data' files and loads
|
||||
the Serie metadata from each file. This method is synchronous
|
||||
the AnimeSeries metadata from each file. This method is synchronous
|
||||
and can be wrapped with asyncio.to_thread if needed for async
|
||||
contexts.
|
||||
|
||||
Returns:
|
||||
List of Serie objects found in data files. Returns an empty
|
||||
List of AnimeSeries objects found in data files. Returns an empty
|
||||
list if no data files are found or if the directory doesn't
|
||||
exist.
|
||||
|
||||
Example:
|
||||
series_app = SeriesApp("/path/to/anime")
|
||||
all_series = series_app.get_all_series_from_data_files()
|
||||
for serie in all_series:
|
||||
print(f"Found: {serie.name} (key={serie.key})")
|
||||
for anime in all_series:
|
||||
print(f"Found: {anime.name} (key={anime.key})")
|
||||
"""
|
||||
logger.info(
|
||||
"Scanning for data files in directory: %s",
|
||||
self.directory_to_search
|
||||
)
|
||||
|
||||
# Create a fresh SerieList instance for file-based loading
|
||||
# This ensures we get all series from data files without
|
||||
# interfering with the main instance's state
|
||||
all_series: List[AnimeSeries] = []
|
||||
|
||||
try:
|
||||
temp_list = SerieList(
|
||||
self.directory_to_search,
|
||||
skip_load=False # Allow automatic loading
|
||||
)
|
||||
if not os.path.isdir(self.directory_to_search):
|
||||
logger.warning(
|
||||
"Directory does not exist: %s",
|
||||
self.directory_to_search
|
||||
)
|
||||
return []
|
||||
except (OSError, ValueError) as e:
|
||||
logger.error(
|
||||
"Failed to scan directory for data files: %s",
|
||||
@@ -725,8 +714,53 @@ class SeriesApp:
|
||||
)
|
||||
return []
|
||||
|
||||
# Get all series from the temporary list
|
||||
all_series = temp_list.get_all()
|
||||
try:
|
||||
for folder_name in os.listdir(self.directory_to_search):
|
||||
folder_path = os.path.join(
|
||||
self.directory_to_search, folder_name
|
||||
)
|
||||
if not os.path.isdir(folder_path):
|
||||
continue
|
||||
|
||||
data_file = os.path.join(folder_path, "data")
|
||||
if not os.path.isfile(data_file):
|
||||
continue
|
||||
|
||||
series_data = _load_data_file(data_file)
|
||||
if series_data is None:
|
||||
continue
|
||||
|
||||
key = series_data.get("key")
|
||||
if not key:
|
||||
logger.warning(
|
||||
"Data file missing key, skipping: %s",
|
||||
data_file
|
||||
)
|
||||
continue
|
||||
|
||||
anime = AnimeSeries(
|
||||
key=key,
|
||||
name=series_data.get("name") or folder_name,
|
||||
site=series_data.get("site", "https://aniworld.to"),
|
||||
folder=series_data.get("folder", folder_name),
|
||||
year=series_data.get("year"),
|
||||
)
|
||||
|
||||
episode_dict = series_data.get("episodeDict", {})
|
||||
if episode_dict:
|
||||
anime._episode_dict_cache = {
|
||||
int(season): episodes
|
||||
for season, episodes in episode_dict.items()
|
||||
}
|
||||
|
||||
all_series.append(anime)
|
||||
except (OSError, ValueError) as e:
|
||||
logger.error(
|
||||
"Failed to scan directory for data files: %s",
|
||||
str(e),
|
||||
exc_info=True
|
||||
)
|
||||
return []
|
||||
|
||||
logger.info(
|
||||
"Found %d series from data files in %s",
|
||||
@@ -746,3 +780,38 @@ class SeriesApp:
|
||||
if hasattr(self, 'executor'):
|
||||
self.executor.shutdown(wait=True)
|
||||
logger.info("ThreadPoolExecutor shut down successfully")
|
||||
|
||||
|
||||
def _load_data_file(data_file_path: str) -> Optional[dict]:
|
||||
"""Load and parse a legacy 'data' file (JSON).
|
||||
|
||||
Args:
|
||||
data_file_path: Path to the data file
|
||||
|
||||
Returns:
|
||||
Parsed data dict or None if parsing fails
|
||||
"""
|
||||
import json
|
||||
|
||||
try:
|
||||
with open(data_file_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
if not isinstance(data, dict):
|
||||
logger.warning("Data file is not a dictionary: %s", data_file_path)
|
||||
return None
|
||||
|
||||
return data
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning(
|
||||
"Failed to parse legacy data file (JSON error): %s - %s",
|
||||
data_file_path, str(e)
|
||||
)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Failed to read legacy data file: %s - %s",
|
||||
data_file_path, str(e)
|
||||
)
|
||||
return None
|
||||
@@ -1,5 +1,5 @@
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import warnings
|
||||
from typing import Any, List, Optional
|
||||
|
||||
@@ -7,7 +7,8 @@ from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.core.entities.series import Serie
|
||||
from src.config.settings import settings
|
||||
from src.server.database.models import AnimeSeries
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
from src.server.exceptions import (
|
||||
BadRequestError,
|
||||
@@ -15,20 +16,51 @@ from src.server.exceptions import (
|
||||
ServerError,
|
||||
ValidationError,
|
||||
)
|
||||
from src.server.models.anime import AnimeMetadataUpdate
|
||||
from src.server.services.anime_service import AnimeService, AnimeServiceError
|
||||
from src.server.services.background_loader_service import BackgroundLoaderService
|
||||
from src.server.utils.dependencies import (
|
||||
get_anime_service,
|
||||
get_background_loader_service,
|
||||
get_database_session,
|
||||
get_optional_database_session,
|
||||
get_series_app,
|
||||
require_auth,
|
||||
)
|
||||
from src.server.utils.filesystem import sanitize_folder_name
|
||||
from src.server.utils.key_utils import generate_key_from_folder, is_valid_key
|
||||
from src.server.utils.validators import validate_filter_value, validate_search_query
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/anime", tags=["anime"])
|
||||
|
||||
|
||||
def _compute_folder_name(name: str, year: Optional[int]) -> str:
|
||||
"""Compute sanitized folder name from display name and year.
|
||||
|
||||
If year is provided, strips any existing year in (YYYY) format to avoid
|
||||
duplicates, then appends the new year. If year is None, preserves the
|
||||
original name (with any existing year).
|
||||
|
||||
Args:
|
||||
name: Display name of the series
|
||||
year: Release year from provider, or None
|
||||
|
||||
Returns:
|
||||
Sanitized folder name in format "Name (YYYY)" or just "Name"
|
||||
"""
|
||||
if year:
|
||||
# Strip any existing year in (YYYY) format before adding new year
|
||||
clean_name = re.sub(r'\s*\(\d{4}\)\s*$', '', name).strip()
|
||||
folder_name_with_year = f"{clean_name} ({year})"
|
||||
else:
|
||||
# No new year provided, preserve original name (with any existing year)
|
||||
folder_name_with_year = name
|
||||
|
||||
return sanitize_folder_name(folder_name_with_year)
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def get_anime_status(
|
||||
_auth: dict = Depends(require_auth),
|
||||
@@ -68,6 +100,37 @@ async def get_anime_status(
|
||||
) from exc
|
||||
|
||||
|
||||
class DuplicateFolderGroup(BaseModel):
|
||||
"""Placeholder - duplicates functionality removed."""
|
||||
key: str = Field(..., description="Series key (unique identifier)")
|
||||
folders: List[str] = Field(..., description="List of duplicate folder names")
|
||||
folder_count: int = Field(..., description="Number of duplicate folders")
|
||||
|
||||
|
||||
class DuplicateFoldersResponse(BaseModel):
|
||||
"""Placeholder - duplicates functionality removed."""
|
||||
total_groups: int = Field(..., description="Total number of duplicate groups")
|
||||
duplicate_groups: List[DuplicateFolderGroup] = Field(
|
||||
..., description="List of duplicate folder groups"
|
||||
)
|
||||
message: str = Field(..., description="Human-readable summary")
|
||||
|
||||
|
||||
@router.get("/duplicate-folders", response_model=DuplicateFoldersResponse)
|
||||
async def get_duplicate_folders(
|
||||
_auth: dict = Depends(require_auth),
|
||||
) -> DuplicateFoldersResponse:
|
||||
"""List all pre-existing duplicate folder groups.
|
||||
|
||||
Note: Duplicate folder scanning has been removed. Returns empty response.
|
||||
"""
|
||||
return DuplicateFoldersResponse(
|
||||
total_groups=0,
|
||||
duplicate_groups=[],
|
||||
message="Duplicate folder scanning has been removed.",
|
||||
)
|
||||
|
||||
|
||||
class AnimeSummary(BaseModel):
|
||||
"""Summary of an anime series with missing episodes.
|
||||
|
||||
@@ -85,6 +148,11 @@ class AnimeSummary(BaseModel):
|
||||
missing_episodes: Episode dictionary mapping seasons to episode numbers
|
||||
has_missing: Boolean flag indicating if series has missing episodes
|
||||
link: Optional link to the series page (used when adding new series)
|
||||
has_nfo: Whether the series has NFO metadata
|
||||
nfo_created_at: ISO timestamp when NFO was created
|
||||
nfo_updated_at: ISO timestamp when NFO was last updated
|
||||
tmdb_id: The Movie Database (TMDB) ID
|
||||
tvdb_id: TheTVDB ID
|
||||
"""
|
||||
key: str = Field(
|
||||
...,
|
||||
@@ -114,6 +182,26 @@ class AnimeSummary(BaseModel):
|
||||
default="",
|
||||
description="Link to the series page (for adding new series)"
|
||||
)
|
||||
has_nfo: bool = Field(
|
||||
default=False,
|
||||
description="Whether the series has NFO metadata"
|
||||
)
|
||||
nfo_created_at: Optional[str] = Field(
|
||||
default=None,
|
||||
description="ISO timestamp when NFO was created"
|
||||
)
|
||||
nfo_updated_at: Optional[str] = Field(
|
||||
default=None,
|
||||
description="ISO timestamp when NFO was last updated"
|
||||
)
|
||||
tmdb_id: Optional[int] = Field(
|
||||
default=None,
|
||||
description="The Movie Database (TMDB) ID"
|
||||
)
|
||||
tvdb_id: Optional[int] = Field(
|
||||
default=None,
|
||||
description="TheTVDB ID"
|
||||
)
|
||||
|
||||
class Config:
|
||||
"""Pydantic model configuration."""
|
||||
@@ -125,7 +213,12 @@ class AnimeSummary(BaseModel):
|
||||
"folder": "beheneko the elf girls cat (2025)",
|
||||
"missing_episodes": {"1": [1, 2, 3, 4]},
|
||||
"has_missing": True,
|
||||
"link": "https://aniworld.to/anime/stream/beheneko"
|
||||
"link": "https://aniworld.to/anime/stream/beheneko",
|
||||
"has_nfo": True,
|
||||
"nfo_created_at": "2025-01-15T10:30:00Z",
|
||||
"nfo_updated_at": "2025-01-15T10:30:00Z",
|
||||
"tmdb_id": 12345,
|
||||
"tvdb_id": 67890
|
||||
}
|
||||
}
|
||||
|
||||
@@ -187,7 +280,7 @@ async def list_anime(
|
||||
sort_by: Optional[str] = None,
|
||||
filter: Optional[str] = None,
|
||||
_auth: dict = Depends(require_auth),
|
||||
series_app: Any = Depends(get_series_app),
|
||||
anime_service: AnimeService = Depends(get_anime_service),
|
||||
) -> List[AnimeSummary]:
|
||||
"""List all library series with their missing episodes status.
|
||||
|
||||
@@ -203,9 +296,11 @@ async def list_anime(
|
||||
per_page: Items per page (must be positive, max 1000)
|
||||
sort_by: Optional sorting parameter. Allowed: title, id, name,
|
||||
missing_episodes
|
||||
filter: Optional filter parameter (validated for security)
|
||||
filter: Optional filter parameter. Allowed values:
|
||||
- "missing_episodes": Show only series that have any missing episodes
|
||||
- "no_episodes": Show only series that have no downloaded episodes
|
||||
_auth: Ensures the caller is authenticated (value unused)
|
||||
series_app: Core SeriesApp instance provided via dependency.
|
||||
anime_service: AnimeService instance provided via dependency
|
||||
|
||||
Returns:
|
||||
List[AnimeSummary]: Summary entries with `key` as primary identifier.
|
||||
@@ -263,34 +358,22 @@ async def list_anime(
|
||||
|
||||
# Validate filter parameter
|
||||
if filter:
|
||||
# Check for dangerous patterns in filter
|
||||
dangerous_patterns = [
|
||||
";", "--", "/*", "*/",
|
||||
"drop", "delete", "insert", "update"
|
||||
]
|
||||
lower_filter = filter.lower()
|
||||
for pattern in dangerous_patterns:
|
||||
if pattern in lower_filter:
|
||||
raise ValidationError(
|
||||
message="Invalid filter parameter"
|
||||
)
|
||||
try:
|
||||
allowed_filters = ["missing_episodes", "no_episodes"]
|
||||
validate_filter_value(filter, allowed_filters)
|
||||
except ValueError as e:
|
||||
raise ValidationError(message=str(e))
|
||||
|
||||
try:
|
||||
# Get all series from series app
|
||||
if not hasattr(series_app, "list"):
|
||||
return []
|
||||
# Use AnimeService to get series with metadata from database
|
||||
series_list = await anime_service.list_series_with_filters(
|
||||
filter_type=filter
|
||||
)
|
||||
|
||||
series = series_app.list.GetList()
|
||||
summaries: List[AnimeSummary] = []
|
||||
for serie in series:
|
||||
# Get all properties from the serie object
|
||||
key = getattr(serie, "key", "")
|
||||
name = getattr(serie, "name", "")
|
||||
site = getattr(serie, "site", "")
|
||||
folder = getattr(serie, "folder", "")
|
||||
episode_dict = getattr(serie, "episodeDict", {}) or {}
|
||||
|
||||
for series_dict in series_list:
|
||||
# Convert episode dict keys to strings for JSON serialization
|
||||
episode_dict = series_dict.get("episodeDict", {}) or {}
|
||||
missing_episodes = {str(k): v for k, v in episode_dict.items()}
|
||||
|
||||
# Determine if series has missing episodes
|
||||
@@ -298,12 +381,17 @@ async def list_anime(
|
||||
|
||||
summaries.append(
|
||||
AnimeSummary(
|
||||
key=key,
|
||||
name=name,
|
||||
site=site,
|
||||
folder=folder,
|
||||
key=series_dict["key"],
|
||||
name=series_dict["name"],
|
||||
site=series_dict["site"],
|
||||
folder=series_dict["folder"],
|
||||
missing_episodes=missing_episodes,
|
||||
has_missing=has_missing,
|
||||
has_nfo=series_dict.get("has_nfo", False),
|
||||
nfo_created_at=series_dict.get("nfo_created_at"),
|
||||
nfo_updated_at=series_dict.get("nfo_updated_at"),
|
||||
tmdb_id=series_dict.get("tmdb_id"),
|
||||
tvdb_id=series_dict.get("tvdb_id"),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -400,8 +488,8 @@ class AddSeriesRequest(BaseModel):
|
||||
name: str
|
||||
|
||||
|
||||
def validate_search_query(query: str) -> str:
|
||||
"""Validate and sanitize search query.
|
||||
def _validate_search_query_extended(query: str) -> str:
|
||||
"""Validate and sanitize search query with additional checks.
|
||||
|
||||
Args:
|
||||
query: The search query string
|
||||
@@ -432,25 +520,16 @@ def validate_search_query(query: str) -> str:
|
||||
detail="Search query too long (max 200 characters)"
|
||||
)
|
||||
|
||||
# Strip and normalize whitespace
|
||||
normalized = " ".join(query.strip().split())
|
||||
|
||||
# Prevent SQL-like injection patterns
|
||||
dangerous_patterns = [
|
||||
"--", "/*", "*/", "xp_", "sp_", "exec", "execute",
|
||||
"union", "select", "insert", "update", "delete", "drop",
|
||||
"create", "alter", "truncate", "sleep", "waitfor", "benchmark",
|
||||
" or ", "||", " and ", "&&"
|
||||
]
|
||||
lower_query = normalized.lower()
|
||||
for pattern in dangerous_patterns:
|
||||
if pattern in lower_query:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="Invalid character sequence detected"
|
||||
)
|
||||
|
||||
return normalized
|
||||
# Validate and normalize the search query using utility function
|
||||
try:
|
||||
normalized = validate_search_query(query)
|
||||
return normalized
|
||||
except ValueError as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail=str(e)
|
||||
)
|
||||
|
||||
|
||||
|
||||
class SearchAnimeRequest(BaseModel):
|
||||
@@ -539,7 +618,7 @@ async def _perform_search(
|
||||
"""
|
||||
try:
|
||||
# Validate and sanitize the query
|
||||
validated_query = validate_search_query(query)
|
||||
validated_query = _validate_search_query_extended(query)
|
||||
|
||||
# Check if series_app is available
|
||||
if not series_app:
|
||||
@@ -616,23 +695,28 @@ async def _perform_search(
|
||||
) from exc
|
||||
|
||||
|
||||
@router.post("/add")
|
||||
@router.post("/add", status_code=status.HTTP_202_ACCEPTED)
|
||||
async def add_series(
|
||||
request: AddSeriesRequest,
|
||||
_auth: dict = Depends(require_auth),
|
||||
series_app: Any = Depends(get_series_app),
|
||||
db: Optional[AsyncSession] = Depends(get_optional_database_session),
|
||||
anime_service: AnimeService = Depends(get_anime_service),
|
||||
db: Optional[AsyncSession] = Depends(get_optional_database_session),
|
||||
background_loader: BackgroundLoaderService = Depends(get_background_loader_service),
|
||||
) -> dict:
|
||||
"""Add a new series to the library with full initialization.
|
||||
"""Add a new series to the library with asynchronous data loading.
|
||||
|
||||
This endpoint performs the complete series addition flow:
|
||||
This endpoint performs immediate series addition and queues background loading:
|
||||
1. Validates inputs and extracts the series key from the link URL
|
||||
2. Creates a sanitized folder name from the display name
|
||||
3. Saves the series to the database (if available)
|
||||
3. Saves the series to the database with loading_status="pending"
|
||||
4. Creates the folder on disk with the sanitized name
|
||||
5. Triggers a targeted scan for missing episodes (only this series)
|
||||
5. Queues background loading task for episodes, NFO, and images
|
||||
6. Returns immediately (202 Accepted) without waiting for data loading
|
||||
|
||||
Data loading happens asynchronously in the background, with real-time
|
||||
status updates via WebSocket.
|
||||
|
||||
The `key` is the URL-safe identifier used for all lookups.
|
||||
The `name` is stored as display metadata and used to derive
|
||||
the filesystem folder name (sanitized for filesystem safety).
|
||||
@@ -644,7 +728,7 @@ async def add_series(
|
||||
_auth: Ensures the caller is authenticated (value unused)
|
||||
series_app: Core `SeriesApp` instance provided via dependency
|
||||
db: Optional database session for async operations
|
||||
anime_service: AnimeService for scanning operations
|
||||
background_loader: BackgroundLoaderService for async data loading
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: Status payload with:
|
||||
@@ -653,8 +737,8 @@ async def add_series(
|
||||
- key: Series unique identifier
|
||||
- folder: Created folder path
|
||||
- db_id: Database ID (if saved to DB)
|
||||
- missing_episodes: Dict of missing episodes by season
|
||||
- total_missing: Total count of missing episodes
|
||||
- loading_status: Current loading status
|
||||
- loading_progress: Dict of what data is being loaded
|
||||
|
||||
Raises:
|
||||
HTTPException: If adding the series fails or link is invalid
|
||||
@@ -693,10 +777,21 @@ async def add_series(
|
||||
detail="Could not extract series key from link",
|
||||
)
|
||||
|
||||
# Step B: Create sanitized folder name from display name
|
||||
# Step B: Fetch year from provider and create folder name with year
|
||||
name = request.name.strip()
|
||||
|
||||
# Fetch year from provider
|
||||
year = None
|
||||
if series_app and hasattr(series_app, 'loader'):
|
||||
try:
|
||||
year = series_app.loader.get_year(key)
|
||||
logger.info("Fetched year for %s: %s", key, year)
|
||||
except Exception as e:
|
||||
logger.warning("Could not fetch year for %s: %s", key, e)
|
||||
|
||||
# Step B: Compute sanitized folder name with year (deduplicates if year already in name)
|
||||
try:
|
||||
folder = sanitize_folder_name(name)
|
||||
folder = _compute_folder_name(name, year)
|
||||
except ValueError as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
@@ -704,10 +799,38 @@ async def add_series(
|
||||
)
|
||||
|
||||
db_id = None
|
||||
missing_episodes: dict = {}
|
||||
scan_error: Optional[str] = None
|
||||
|
||||
# Step C: Save to database if available
|
||||
# Step C: Create folder on disk if it doesn't exist, and rename if needed
|
||||
# Determine the anime directory path
|
||||
anime_dir = settings.anime_directory if hasattr(settings, 'anime_directory') else None
|
||||
current_folder_on_disk = None
|
||||
|
||||
if anime_dir:
|
||||
import os
|
||||
anime_path = os.path.join(anime_dir, folder)
|
||||
|
||||
# Check if an existing folder (without year) needs renaming
|
||||
# Look for folder that matches name without year
|
||||
if year:
|
||||
potential_old_name = sanitize_folder_name(name)
|
||||
potential_old_path = os.path.join(anime_dir, potential_old_name)
|
||||
if potential_old_path != anime_path and os.path.exists(potential_old_path):
|
||||
current_folder_on_disk = potential_old_name
|
||||
logger.info(
|
||||
"Found existing folder without year for %s: %s, renaming to %s",
|
||||
key,
|
||||
potential_old_name,
|
||||
folder
|
||||
)
|
||||
elif not os.path.exists(anime_path):
|
||||
# No existing folder to rename, create new one
|
||||
os.makedirs(anime_path, exist_ok=True)
|
||||
else:
|
||||
# No year, just ensure folder exists
|
||||
if not os.path.exists(anime_path):
|
||||
os.makedirs(anime_path, exist_ok=True)
|
||||
|
||||
# Step D: Save to database if available
|
||||
if db is not None:
|
||||
# Check if series already exists in database
|
||||
existing = await AnimeSeriesService.get_by_key(db, key)
|
||||
@@ -718,102 +841,159 @@ async def add_series(
|
||||
"key": key,
|
||||
"folder": existing.folder,
|
||||
"db_id": existing.id,
|
||||
"missing_episodes": {},
|
||||
"total_missing": 0
|
||||
"loading_status": existing.loading_status,
|
||||
"loading_progress": {
|
||||
"episodes": existing.episodes_loaded,
|
||||
"nfo": existing.has_nfo,
|
||||
"logo": existing.logo_loaded,
|
||||
"images": existing.images_loaded
|
||||
}
|
||||
}
|
||||
|
||||
# Save to database using AnimeSeriesService
|
||||
# Save to database using AnimeSeriesService with loading status
|
||||
anime_series = await AnimeSeriesService.create(
|
||||
db=db,
|
||||
key=key,
|
||||
name=name,
|
||||
site="aniworld.to",
|
||||
folder=folder,
|
||||
year=year,
|
||||
loading_status="pending",
|
||||
episodes_loaded=False,
|
||||
logo_loaded=False,
|
||||
images_loaded=False,
|
||||
loading_started_at=None,
|
||||
)
|
||||
db_id = anime_series.id
|
||||
|
||||
logger.info(
|
||||
"Added series to database: %s (key=%s, db_id=%d)",
|
||||
"Added series to database: %s (key=%s, db_id=%d, year=%s, loading=pending)",
|
||||
name,
|
||||
key,
|
||||
db_id
|
||||
db_id,
|
||||
year
|
||||
)
|
||||
|
||||
# Step D: Add to SerieList (in-memory only, no folder creation)
|
||||
if series_app and hasattr(series_app, "list"):
|
||||
serie = Serie(
|
||||
from src.server.database.models import AnimeSeries
|
||||
anime = AnimeSeries(
|
||||
key=key,
|
||||
name=name,
|
||||
site="aniworld.to",
|
||||
folder=folder,
|
||||
episodeDict={}
|
||||
year=year
|
||||
)
|
||||
|
||||
|
||||
# Add to in-memory cache without creating folder on disk
|
||||
if hasattr(series_app.list, 'keyDict'):
|
||||
series_app.list.keyDict[key] = serie
|
||||
series_app.list.keyDict[key] = anime
|
||||
logger.info(
|
||||
"Added series to in-memory cache: %s (key=%s, folder=%s)",
|
||||
"Added series to in-memory cache: %s (key=%s, folder=%s, year=%s)",
|
||||
name,
|
||||
key,
|
||||
folder
|
||||
folder,
|
||||
year
|
||||
)
|
||||
|
||||
# Step E: Trigger targeted scan for missing episodes
|
||||
# Step E: Rename existing folder if needed (e.g., folder existed without year)
|
||||
if current_folder_on_disk:
|
||||
try:
|
||||
renamed = await anime_service.rename_folder_if_needed(
|
||||
key=key,
|
||||
current_folder=current_folder_on_disk,
|
||||
target_folder=folder,
|
||||
db=db
|
||||
)
|
||||
if renamed:
|
||||
logger.info(
|
||||
"Successfully renamed folder for %s: %s -> %s",
|
||||
key,
|
||||
current_folder_on_disk,
|
||||
folder
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Failed to rename folder for %s: %s -> %s: %s",
|
||||
key,
|
||||
current_folder_on_disk,
|
||||
folder,
|
||||
e
|
||||
)
|
||||
|
||||
# Step F: Queue background loading task for episodes, NFO, and images
|
||||
try:
|
||||
if series_app and hasattr(series_app, "serie_scanner"):
|
||||
await background_loader.add_series_loading_task(
|
||||
key=key,
|
||||
folder=folder,
|
||||
name=name,
|
||||
year=year
|
||||
)
|
||||
logger.info(
|
||||
"Queued background loading for %s (key=%s)",
|
||||
name,
|
||||
key
|
||||
)
|
||||
except Exception as e:
|
||||
# Background loading queue failure is not critical - series was still added
|
||||
logger.warning(
|
||||
"Failed to queue background loading for %s: %s",
|
||||
key,
|
||||
e
|
||||
)
|
||||
|
||||
# Step G: Scan missing episodes immediately if background loader is not running
|
||||
# Uses existing SerieScanner and AnimeService sync to avoid duplicates
|
||||
try:
|
||||
loader_running = bool(
|
||||
background_loader.worker_tasks
|
||||
and any(not t.done() for t in background_loader.worker_tasks)
|
||||
)
|
||||
if (
|
||||
not loader_running
|
||||
and series_app
|
||||
and hasattr(series_app, "serie_scanner")
|
||||
):
|
||||
missing_episodes = series_app.serie_scanner.scan_single_series(
|
||||
key=key,
|
||||
folder=folder
|
||||
)
|
||||
logger.info(
|
||||
"Targeted scan completed for %s: found %d missing episodes",
|
||||
key,
|
||||
sum(len(eps) for eps in missing_episodes.values())
|
||||
total_missing = sum(
|
||||
len(eps) for eps in missing_episodes.values()
|
||||
)
|
||||
|
||||
# Update the serie in keyDict with the missing episodes
|
||||
if hasattr(series_app, "list") and hasattr(series_app.list, "keyDict"):
|
||||
if key in series_app.list.keyDict:
|
||||
series_app.list.keyDict[key].episodeDict = missing_episodes
|
||||
else:
|
||||
# Scanner not available - this shouldn't happen in normal operation
|
||||
logger.warning(
|
||||
"Scanner not available for targeted scan of %s",
|
||||
logger.info(
|
||||
"Scanned %d missing episodes for %s",
|
||||
total_missing,
|
||||
key
|
||||
)
|
||||
|
||||
# Persist scan results to database (includes episodes)
|
||||
# scan_single_series updates serie_scanner.keyDict with episodeDict
|
||||
# sync_single_series_after_scan retrieves from there and saves to DB
|
||||
await anime_service.sync_single_series_after_scan(key)
|
||||
except Exception as e:
|
||||
# Scan failure is not critical - series was still added
|
||||
scan_error = str(e)
|
||||
logger.warning(
|
||||
"Targeted scan failed for %s: %s (series still added)",
|
||||
"Failed to scan missing episodes for %s: %s",
|
||||
key,
|
||||
e
|
||||
)
|
||||
|
||||
# Convert missing episodes keys to strings for JSON serialization
|
||||
missing_episodes_serializable = {
|
||||
str(season): episodes
|
||||
for season, episodes in missing_episodes.items()
|
||||
}
|
||||
|
||||
# Calculate total missing
|
||||
total_missing = sum(len(eps) for eps in missing_episodes.values())
|
||||
|
||||
# Step F: Return response
|
||||
# Step G: Return immediate response (202 Accepted)
|
||||
response = {
|
||||
"status": "success",
|
||||
"message": f"Successfully added series: {name}",
|
||||
"message": f"Series added successfully: {name}. Data will be loaded in background.",
|
||||
"key": key,
|
||||
"folder": folder,
|
||||
"db_id": db_id,
|
||||
"missing_episodes": missing_episodes_serializable,
|
||||
"total_missing": total_missing
|
||||
"loading_status": "pending",
|
||||
"loading_progress": {
|
||||
"episodes": False,
|
||||
"nfo": False,
|
||||
"logo": False,
|
||||
"images": False
|
||||
}
|
||||
}
|
||||
|
||||
if scan_error:
|
||||
response["scan_warning"] = f"Scan partially failed: {scan_error}"
|
||||
|
||||
return response
|
||||
|
||||
except HTTPException:
|
||||
@@ -830,6 +1010,97 @@ async def add_series(
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get("/{anime_key}/loading-status")
|
||||
async def get_loading_status(
|
||||
anime_key: str,
|
||||
_auth: dict = Depends(require_auth),
|
||||
db: Optional[AsyncSession] = Depends(get_optional_database_session),
|
||||
) -> dict:
|
||||
"""Get current loading status for a series.
|
||||
|
||||
Returns the current background loading status including what data
|
||||
has been loaded and what is still pending.
|
||||
|
||||
Args:
|
||||
anime_key: Series unique identifier (key)
|
||||
_auth: Ensures the caller is authenticated
|
||||
db: Optional database session
|
||||
|
||||
Returns:
|
||||
Dict with loading status information:
|
||||
- key: Series identifier
|
||||
- loading_status: Current status (pending, loading_*, completed, failed)
|
||||
- progress: Dict of what data is loaded
|
||||
- started_at: When loading started
|
||||
- completed_at: When loading completed (if done)
|
||||
- message: Human-readable status message
|
||||
- error: Error message if failed
|
||||
|
||||
Raises:
|
||||
HTTPException: If series not found or database unavailable
|
||||
"""
|
||||
if db is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="Database not available"
|
||||
)
|
||||
|
||||
try:
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
|
||||
# Get series from database
|
||||
series = await AnimeSeriesService.get_by_key(db, anime_key)
|
||||
|
||||
if not series:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Series not found: {anime_key}"
|
||||
)
|
||||
|
||||
# Build status message
|
||||
message = ""
|
||||
if series.loading_status == "pending":
|
||||
message = "Queued for loading..."
|
||||
elif series.loading_status == "loading_episodes":
|
||||
message = "Loading episodes..."
|
||||
elif series.loading_status == "loading_nfo":
|
||||
message = "Generating NFO file..."
|
||||
elif series.loading_status == "loading_logo":
|
||||
message = "Downloading logo..."
|
||||
elif series.loading_status == "loading_images":
|
||||
message = "Downloading images..."
|
||||
elif series.loading_status == "completed":
|
||||
message = "All data loaded successfully"
|
||||
elif series.loading_status == "failed":
|
||||
message = f"Loading failed: {series.loading_error}"
|
||||
else:
|
||||
message = "Loading..."
|
||||
|
||||
return {
|
||||
"key": series.key,
|
||||
"loading_status": series.loading_status,
|
||||
"progress": {
|
||||
"episodes": series.episodes_loaded,
|
||||
"nfo": series.has_nfo,
|
||||
"logo": series.logo_loaded,
|
||||
"images": series.images_loaded
|
||||
},
|
||||
"started_at": series.loading_started_at.isoformat() if series.loading_started_at else None,
|
||||
"completed_at": series.loading_completed_at.isoformat() if series.loading_completed_at else None,
|
||||
"message": message,
|
||||
"error": series.loading_error
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.error("Failed to get loading status: %s", exc, exc_info=True)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to get loading status: {str(exc)}"
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get("/{anime_id}", response_model=AnimeDetail)
|
||||
async def get_anime(
|
||||
anime_id: str,
|
||||
@@ -924,3 +1195,75 @@ async def get_anime(
|
||||
# Maximum allowed input size for security
|
||||
MAX_INPUT_LENGTH = 100000 # 100KB
|
||||
|
||||
|
||||
@router.put("/{anime_key}")
|
||||
async def update_anime_metadata(
|
||||
anime_key: str,
|
||||
body: AnimeMetadataUpdate,
|
||||
_auth: dict = Depends(require_auth),
|
||||
db: AsyncSession = Depends(get_database_session),
|
||||
) -> dict:
|
||||
"""Update anime metadata (key, tmdb_id, tvdb_id).
|
||||
|
||||
Args:
|
||||
anime_key: Current series key to update
|
||||
body: Fields to update (all optional)
|
||||
_auth: Authentication dependency
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Updated series metadata
|
||||
|
||||
Raises:
|
||||
HTTPException 404: Series not found
|
||||
HTTPException 409: Key conflict (new key already exists)
|
||||
HTTPException 422: Validation error
|
||||
"""
|
||||
series = await AnimeSeriesService.get_by_key(db, anime_key)
|
||||
if not series:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Series with key '{anime_key}' not found",
|
||||
)
|
||||
|
||||
updates = {}
|
||||
|
||||
if body.key is not None and body.key != anime_key:
|
||||
existing = await AnimeSeriesService.get_by_key(db, body.key)
|
||||
if existing:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"A series with key '{body.key}' already exists",
|
||||
)
|
||||
updates["key"] = body.key
|
||||
|
||||
if body.tmdb_id is not None:
|
||||
updates["tmdb_id"] = body.tmdb_id
|
||||
|
||||
if body.tvdb_id is not None:
|
||||
updates["tvdb_id"] = body.tvdb_id
|
||||
|
||||
if not updates:
|
||||
return {
|
||||
"key": series.key,
|
||||
"tmdb_id": series.tmdb_id,
|
||||
"tvdb_id": series.tvdb_id,
|
||||
"message": "No changes",
|
||||
}
|
||||
|
||||
updated = await AnimeSeriesService.update(db, series.id, **updates)
|
||||
await db.commit()
|
||||
|
||||
logger.info(
|
||||
"Updated metadata for '%s': %s",
|
||||
anime_key,
|
||||
updates,
|
||||
)
|
||||
|
||||
return {
|
||||
"key": updated.key,
|
||||
"tmdb_id": updated.tmdb_id,
|
||||
"tvdb_id": updated.tvdb_id,
|
||||
"message": "Metadata updated successfully",
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Authentication API endpoints for Aniworld."""
|
||||
from typing import Optional
|
||||
|
||||
import structlog
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import status as http_status
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||
@@ -16,6 +17,8 @@ from src.server.models.config import AppConfig
|
||||
from src.server.services.auth_service import AuthError, LockedOutError, auth_service
|
||||
from src.server.services.config_service import get_config_service
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
# NOTE: import dependencies (optional_auth, security) lazily inside handlers
|
||||
# to avoid importing heavyweight modules (e.g. sqlalchemy) at import time.
|
||||
|
||||
@@ -29,8 +32,9 @@ optional_bearer = HTTPBearer(auto_error=False)
|
||||
async def setup_auth(req: SetupRequest):
|
||||
"""Initial setup endpoint to configure the master password.
|
||||
|
||||
This endpoint also initializes the configuration with default values
|
||||
and saves the anime directory and master password hash.
|
||||
This endpoint also initializes the configuration with all provided values
|
||||
and saves them to config.json. It triggers background initialization
|
||||
and redirects to a loading page that shows real-time progress.
|
||||
"""
|
||||
if auth_service.is_configured():
|
||||
raise HTTPException(
|
||||
@@ -44,51 +48,171 @@ async def setup_auth(req: SetupRequest):
|
||||
req.master_password
|
||||
)
|
||||
|
||||
# Initialize or update config with master password hash
|
||||
# and anime directory
|
||||
# Initialize or update config with all provided values
|
||||
config_service = get_config_service()
|
||||
try:
|
||||
config = config_service.load_config()
|
||||
except Exception:
|
||||
# If config doesn't exist, create default
|
||||
from src.server.models.config import (
|
||||
BackupConfig,
|
||||
LoggingConfig,
|
||||
NFOConfig,
|
||||
SchedulerConfig,
|
||||
)
|
||||
config = AppConfig()
|
||||
|
||||
# Update basic settings
|
||||
if req.name:
|
||||
config.name = req.name
|
||||
if req.data_dir:
|
||||
config.data_dir = req.data_dir
|
||||
|
||||
# Update scheduler configuration
|
||||
if req.scheduler_enabled is not None:
|
||||
config.scheduler.enabled = req.scheduler_enabled
|
||||
if req.scheduler_interval_minutes is not None:
|
||||
config.scheduler.interval_minutes = req.scheduler_interval_minutes
|
||||
if req.scheduler_schedule_time is not None:
|
||||
config.scheduler.schedule_time = req.scheduler_schedule_time
|
||||
if req.scheduler_schedule_days is not None:
|
||||
config.scheduler.schedule_days = req.scheduler_schedule_days
|
||||
if req.scheduler_auto_download_after_rescan is not None:
|
||||
config.scheduler.auto_download_after_rescan = req.scheduler_auto_download_after_rescan
|
||||
|
||||
# Update logging configuration
|
||||
if req.logging_level:
|
||||
config.logging.level = req.logging_level.upper()
|
||||
if req.logging_file is not None:
|
||||
config.logging.file = req.logging_file
|
||||
if req.logging_max_bytes is not None:
|
||||
config.logging.max_bytes = req.logging_max_bytes
|
||||
if req.logging_backup_count is not None:
|
||||
config.logging.backup_count = req.logging_backup_count
|
||||
|
||||
# Update backup configuration
|
||||
if req.backup_enabled is not None:
|
||||
config.backup.enabled = req.backup_enabled
|
||||
if req.backup_path:
|
||||
config.backup.path = req.backup_path
|
||||
if req.backup_keep_days is not None:
|
||||
config.backup.keep_days = req.backup_keep_days
|
||||
|
||||
# Update NFO configuration
|
||||
if req.nfo_tmdb_api_key is not None:
|
||||
config.nfo.tmdb_api_key = req.nfo_tmdb_api_key
|
||||
if req.nfo_auto_create is not None:
|
||||
config.nfo.auto_create = req.nfo_auto_create
|
||||
if req.nfo_update_on_scan is not None:
|
||||
config.nfo.update_on_scan = req.nfo_update_on_scan
|
||||
if req.nfo_download_poster is not None:
|
||||
config.nfo.download_poster = req.nfo_download_poster
|
||||
if req.nfo_download_logo is not None:
|
||||
config.nfo.download_logo = req.nfo_download_logo
|
||||
if req.nfo_download_fanart is not None:
|
||||
config.nfo.download_fanart = req.nfo_download_fanart
|
||||
if req.nfo_image_size:
|
||||
config.nfo.image_size = req.nfo_image_size.lower()
|
||||
|
||||
# Store master password hash in config's other field
|
||||
config.other['master_password_hash'] = password_hash
|
||||
|
||||
# Store anime directory in config's other field if provided
|
||||
anime_directory = None
|
||||
if hasattr(req, 'anime_directory') and req.anime_directory:
|
||||
if req.anime_directory:
|
||||
anime_directory = req.anime_directory.strip()
|
||||
if anime_directory:
|
||||
config.other['anime_directory'] = anime_directory
|
||||
|
||||
# Save the config with the password hash and anime directory
|
||||
# Save the config with all updates
|
||||
config_service.save_config(config, create_backup=False)
|
||||
|
||||
# Sync series from data files to database if anime directory is set
|
||||
if anime_directory:
|
||||
try:
|
||||
import structlog
|
||||
# Sync config.json values to settings object
|
||||
# (mirroring the logic in fastapi_app.py lifespan)
|
||||
from src.config.settings import settings
|
||||
other_settings = dict(config.other) if config.other else {}
|
||||
if other_settings.get("anime_directory"):
|
||||
settings.anime_directory = str(other_settings["anime_directory"])
|
||||
|
||||
if config.nfo:
|
||||
if config.nfo.tmdb_api_key:
|
||||
settings.tmdb_api_key = config.nfo.tmdb_api_key
|
||||
settings.nfo_auto_create = config.nfo.auto_create
|
||||
settings.nfo_update_on_scan = config.nfo.update_on_scan
|
||||
settings.nfo_download_poster = config.nfo.download_poster
|
||||
settings.nfo_download_logo = config.nfo.download_logo
|
||||
settings.nfo_download_fanart = config.nfo.download_fanart
|
||||
settings.nfo_image_size = config.nfo.image_size
|
||||
|
||||
# Trigger initialization in background task
|
||||
import asyncio
|
||||
|
||||
from src.server.services.anime_service import (
|
||||
sync_series_from_data_files,
|
||||
from src.server.services.initialization_service import perform_initial_setup
|
||||
from src.server.services.progress_service import get_progress_service
|
||||
|
||||
progress_service = get_progress_service()
|
||||
|
||||
async def run_initialization():
|
||||
"""Run initialization steps with progress updates."""
|
||||
try:
|
||||
# Perform the initial series sync and mark as completed
|
||||
await perform_initial_setup(progress_service)
|
||||
|
||||
# Start scheduler if anime_directory is now set
|
||||
try:
|
||||
from src.server.services.scheduler.scheduler_service import (
|
||||
get_scheduler_service,
|
||||
)
|
||||
|
||||
scheduler_svc = get_scheduler_service()
|
||||
logger.info("Starting scheduler after initialization")
|
||||
await scheduler_svc.ensure_started()
|
||||
logger.info("Scheduler started successfully during setup")
|
||||
except Exception as sched_exc:
|
||||
logger.warning(
|
||||
"Failed to start scheduler during setup: %s", sched_exc
|
||||
)
|
||||
# Continue — scheduler failure should not break initialization
|
||||
|
||||
# Send completion event
|
||||
await progress_service.start_progress(
|
||||
progress_id="initialization_complete",
|
||||
progress_type=ProgressType.SYSTEM,
|
||||
title="Initialization Complete",
|
||||
total=100,
|
||||
message="All initialization tasks completed successfully",
|
||||
metadata={"initialization_complete": True}
|
||||
)
|
||||
logger = structlog.get_logger(__name__)
|
||||
sync_count = await sync_series_from_data_files(
|
||||
anime_directory, logger
|
||||
)
|
||||
logger.info(
|
||||
"Setup complete: synced series from data files",
|
||||
count=sync_count
|
||||
await progress_service.complete_progress(
|
||||
progress_id="initialization_complete",
|
||||
message="All initialization tasks completed successfully",
|
||||
metadata={"initialization_complete": True}
|
||||
)
|
||||
except Exception as e:
|
||||
# Log but don't fail setup if sync fails
|
||||
import structlog
|
||||
structlog.get_logger(__name__).warning(
|
||||
"Failed to sync series after setup",
|
||||
error=str(e)
|
||||
# Send error event
|
||||
from src.server.services.progress_service import ProgressType
|
||||
await progress_service.start_progress(
|
||||
progress_id="initialization_error",
|
||||
progress_type=ProgressType.ERROR,
|
||||
title="Initialization Failed",
|
||||
total=100,
|
||||
message=str(e),
|
||||
metadata={"initialization_complete": True, "error": str(e)}
|
||||
)
|
||||
await progress_service.fail_progress(
|
||||
progress_id="initialization_error",
|
||||
error_message=str(e),
|
||||
metadata={"initialization_complete": True, "error": str(e)}
|
||||
)
|
||||
|
||||
# Start initialization in background
|
||||
asyncio.create_task(run_initialization())
|
||||
|
||||
# Return redirect to loading page
|
||||
return {"status": "ok", "redirect": "/loading"}
|
||||
# Note: Media scan is skipped during setup as it requires
|
||||
# background_loader service which is only available during
|
||||
# application lifespan. It will run on first application startup.
|
||||
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from src.server.models.config import AppConfig, ConfigUpdate, ValidationResult
|
||||
from src.server.services.config_service import (
|
||||
ConfigBackupError,
|
||||
@@ -28,16 +31,53 @@ def get_config(auth: Optional[dict] = Depends(require_auth)) -> AppConfig:
|
||||
|
||||
|
||||
@router.put("", response_model=AppConfig)
|
||||
def update_config(
|
||||
async def update_config(
|
||||
update: ConfigUpdate, auth: dict = Depends(require_auth)
|
||||
) -> AppConfig:
|
||||
"""Apply an update to the configuration and persist it.
|
||||
|
||||
Creates automatic backup before applying changes.
|
||||
Creates automatic backup before applying changes. If anime_directory
|
||||
is configured, starts the scheduler service.
|
||||
"""
|
||||
try:
|
||||
config_service = get_config_service()
|
||||
return config_service.update_config(update)
|
||||
updated_config = config_service.update_config(update)
|
||||
|
||||
# Sync anime_directory to settings if it was updated
|
||||
from src.config.settings import settings as app_settings
|
||||
|
||||
anime_dir_changed = False
|
||||
if update.other and update.other.get("anime_directory"):
|
||||
anime_dir = update.other.get("anime_directory")
|
||||
if anime_dir and not app_settings.anime_directory:
|
||||
app_settings.anime_directory = str(anime_dir)
|
||||
anime_dir_changed = True
|
||||
logger.info("Synced anime_directory from config: %s", anime_dir)
|
||||
|
||||
# Start scheduler if anime_directory was just configured
|
||||
if anime_dir_changed:
|
||||
try:
|
||||
from src.server.services.scheduler.scheduler_service import (
|
||||
get_scheduler_service,
|
||||
)
|
||||
|
||||
scheduler_svc = get_scheduler_service()
|
||||
logger.info(
|
||||
"Starting scheduler after anime_directory configuration"
|
||||
)
|
||||
await scheduler_svc.ensure_started()
|
||||
logger.info(
|
||||
"Scheduler started successfully after config update"
|
||||
)
|
||||
except Exception as sched_exc:
|
||||
logger.warning(
|
||||
"Failed to start scheduler after config update: %s",
|
||||
sched_exc,
|
||||
)
|
||||
# Config was already saved, don't fail the request
|
||||
|
||||
return updated_config
|
||||
|
||||
except ConfigValidationError as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
@@ -244,9 +284,9 @@ async def update_directory(
|
||||
try:
|
||||
import structlog
|
||||
|
||||
from src.server.services.anime_service import sync_series_from_data_files
|
||||
from src.server.services.anime_service import sync_legacy_series_to_db
|
||||
logger = structlog.get_logger(__name__)
|
||||
sync_count = await sync_series_from_data_files(directory, logger)
|
||||
sync_count = await sync_legacy_series_to_db(directory, logger)
|
||||
logger.info(
|
||||
"Directory updated: synced series from data files",
|
||||
directory=directory,
|
||||
@@ -371,3 +411,59 @@ def reset_config(
|
||||
detail=f"Failed to reset config: {e}"
|
||||
) from e
|
||||
|
||||
|
||||
@router.post("/tmdb/validate", response_model=Dict[str, Any])
|
||||
async def validate_tmdb_key(
|
||||
api_key_data: Dict[str, str], auth: dict = Depends(require_auth)
|
||||
) -> Dict[str, Any]:
|
||||
"""Validate TMDB API key by making a test request.
|
||||
|
||||
Args:
|
||||
api_key_data: Dictionary with 'api_key' field
|
||||
auth: Authentication token (required)
|
||||
|
||||
Returns:
|
||||
Validation result with success status and message
|
||||
"""
|
||||
import aiohttp
|
||||
|
||||
api_key = api_key_data.get("api_key", "").strip()
|
||||
|
||||
if not api_key:
|
||||
return {
|
||||
"valid": False,
|
||||
"message": "API key is required"
|
||||
}
|
||||
|
||||
try:
|
||||
# Test the API key with a simple configuration request
|
||||
url = f"https://api.themoviedb.org/3/configuration?api_key={api_key}"
|
||||
|
||||
timeout = aiohttp.ClientTimeout(total=10)
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(url, timeout=timeout) as response:
|
||||
if response.status == 200:
|
||||
return {
|
||||
"valid": True,
|
||||
"message": "TMDB API key is valid"
|
||||
}
|
||||
elif response.status == 401:
|
||||
return {
|
||||
"valid": False,
|
||||
"message": "Invalid API key"
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"valid": False,
|
||||
"message": f"TMDB API error: {response.status}"
|
||||
}
|
||||
except aiohttp.ClientError as e:
|
||||
return {
|
||||
"valid": False,
|
||||
"message": f"Connection error: {str(e)}"
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"valid": False,
|
||||
"message": f"Validation error: {str(e)}"
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ from datetime import datetime
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import psutil
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
@@ -17,15 +17,21 @@ logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/health", tags=["health"])
|
||||
|
||||
|
||||
from src.server.utils.version import APP_VERSION
|
||||
|
||||
|
||||
class HealthStatus(BaseModel):
|
||||
"""Basic health status response."""
|
||||
|
||||
status: str
|
||||
timestamp: str
|
||||
version: str = "1.0.0"
|
||||
version: str = APP_VERSION
|
||||
service: str = "aniworld-api"
|
||||
series_app_initialized: bool = False
|
||||
anime_directory_configured: bool = False
|
||||
scheduler_next_run: Optional[str] = None
|
||||
scheduler_last_run: Optional[str] = None
|
||||
checks: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
class DatabaseHealth(BaseModel):
|
||||
@@ -60,7 +66,7 @@ class DetailedHealthStatus(BaseModel):
|
||||
|
||||
status: str
|
||||
timestamp: str
|
||||
version: str = "1.0.0"
|
||||
version: str = APP_VERSION
|
||||
dependencies: DependencyHealth
|
||||
startup_time: datetime
|
||||
|
||||
@@ -91,7 +97,7 @@ async def check_database_health(db: AsyncSession) -> DatabaseHealth:
|
||||
message="Database connection successful",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Database health check failed: {e}")
|
||||
logger.error("Database health check failed: %s", e)
|
||||
return DatabaseHealth(
|
||||
status="unhealthy",
|
||||
connection_time_ms=0,
|
||||
@@ -121,7 +127,7 @@ async def check_filesystem_health() -> Dict[str, Any]:
|
||||
"message": "Filesystem check completed",
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Filesystem health check failed: {e}")
|
||||
logger.error("Filesystem health check failed: %s", e)
|
||||
return {
|
||||
"status": "unhealthy",
|
||||
"message": f"Filesystem check failed: {str(e)}",
|
||||
@@ -164,36 +170,99 @@ def get_system_metrics() -> SystemMetrics:
|
||||
uptime_seconds=uptime_seconds,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"System metrics collection failed: {e}")
|
||||
logger.error("System metrics collection failed: %s", e)
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Failed to collect system metrics: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
@router.get("", response_model=HealthStatus)
|
||||
async def basic_health_check() -> HealthStatus:
|
||||
async def basic_health_check(request: Request) -> HealthStatus:
|
||||
"""Basic health check endpoint.
|
||||
|
||||
This endpoint does not depend on anime_directory configuration
|
||||
and should always return 200 OK for basic health monitoring.
|
||||
Includes service information for identification.
|
||||
Includes scheduler next/last run times for monitoring tools.
|
||||
Includes startup health check results.
|
||||
|
||||
Returns:
|
||||
HealthStatus: Simple health status with timestamp and service info.
|
||||
"""
|
||||
from src.config.settings import settings
|
||||
from src.server.utils.dependencies import _series_app
|
||||
|
||||
# Get scheduler status for health monitoring
|
||||
scheduler_status: dict = {}
|
||||
try:
|
||||
from src.server.services.scheduler.scheduler_service import (
|
||||
get_scheduler_service,
|
||||
)
|
||||
scheduler_status = get_scheduler_service().get_status()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Get startup checks from app state
|
||||
checks = getattr(request.app.state, "startup_checks", None)
|
||||
|
||||
# Determine overall status based on checks
|
||||
overall_status = "healthy"
|
||||
if checks:
|
||||
for check_name, check_data in checks.items():
|
||||
if check_data.get("status") == "error":
|
||||
overall_status = "unhealthy"
|
||||
break
|
||||
elif check_data.get("status") == "warning":
|
||||
overall_status = "degraded"
|
||||
|
||||
logger.debug("Basic health check requested")
|
||||
return HealthStatus(
|
||||
status="healthy",
|
||||
status=overall_status,
|
||||
timestamp=datetime.now().isoformat(),
|
||||
service="aniworld-api",
|
||||
series_app_initialized=_series_app is not None,
|
||||
anime_directory_configured=bool(settings.anime_directory),
|
||||
scheduler_next_run=scheduler_status.get("next_run"),
|
||||
scheduler_last_run=scheduler_status.get("last_run"),
|
||||
checks=checks,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/ready")
|
||||
async def ready_check(request: Request) -> Dict[str, Any]:
|
||||
"""Readiness check endpoint for container orchestrators.
|
||||
|
||||
Returns 503 if critical dependencies are not available.
|
||||
This endpoint is used by Kubernetes, Docker Swarm, etc. to determine
|
||||
if the container should receive traffic.
|
||||
|
||||
Returns:
|
||||
dict: Readiness status with checks details.
|
||||
"""
|
||||
checks = getattr(request.app.state, "startup_checks", {})
|
||||
|
||||
critical_failures = []
|
||||
for check_name, check_data in checks.items():
|
||||
if check_data.get("status") == "error":
|
||||
critical_failures.append(f"{check_name}: {check_data.get('message')}")
|
||||
|
||||
if critical_failures:
|
||||
return {
|
||||
"status": "not_ready",
|
||||
"ready": False,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"critical_failures": critical_failures,
|
||||
"checks": checks,
|
||||
}
|
||||
|
||||
return {
|
||||
"status": "ready",
|
||||
"ready": True,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"checks": checks,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/detailed", response_model=DetailedHealthStatus)
|
||||
async def detailed_health_check(
|
||||
db: AsyncSession = Depends(get_database_session),
|
||||
@@ -236,7 +305,7 @@ async def detailed_health_check(
|
||||
startup_time=startup_time,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Detailed health check failed: {e}")
|
||||
logger.error("Detailed health check failed: %s", e)
|
||||
raise HTTPException(status_code=500, detail="Health check failed")
|
||||
|
||||
|
||||
|
||||
230
src/server/api/logging.py
Normal file
230
src/server/api/logging.py
Normal file
@@ -0,0 +1,230 @@
|
||||
"""Logging API endpoints for AniWorld.
|
||||
|
||||
Provides endpoints for reading log configuration, listing log files,
|
||||
tailing/downloading individual log files, testing logging, and cleanup.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from src.server.services.config_service import get_config_service
|
||||
from src.server.utils.dependencies import require_auth
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/logging", tags=["logging"])
|
||||
|
||||
_LOG_DIR = Path("logs")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _log_dir() -> Path:
|
||||
"""Return the log directory, creating it if necessary."""
|
||||
_LOG_DIR.mkdir(exist_ok=True)
|
||||
return _LOG_DIR
|
||||
|
||||
|
||||
def _list_log_files() -> List[Dict[str, Any]]:
|
||||
"""Return metadata for all .log files in the log directory."""
|
||||
result: List[Dict[str, Any]] = []
|
||||
log_dir = _log_dir()
|
||||
for entry in sorted(log_dir.iterdir()):
|
||||
if entry.is_file() and entry.suffix in {".log", ".txt"}:
|
||||
stat = entry.stat()
|
||||
result.append(
|
||||
{
|
||||
"name": entry.name,
|
||||
"size_mb": round(stat.st_size / (1024 * 1024), 2),
|
||||
"modified": stat.st_mtime,
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/config")
|
||||
def get_logging_config(
|
||||
auth: Optional[dict] = Depends(require_auth),
|
||||
) -> Dict[str, Any]:
|
||||
"""Return current logging configuration as used by the frontend.
|
||||
|
||||
Maps the internal ``LoggingConfig`` model fields to the shape expected
|
||||
by ``logging-config.js``.
|
||||
"""
|
||||
try:
|
||||
config_service = get_config_service()
|
||||
app_config = config_service.load_config()
|
||||
lc = app_config.logging
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"config": {
|
||||
# Primary fields (match the model)
|
||||
"log_level": lc.level,
|
||||
"log_file": lc.file,
|
||||
"max_bytes": lc.max_bytes,
|
||||
"backup_count": lc.backup_count,
|
||||
# UI-only flags – defaults; not yet persisted in the model
|
||||
"enable_console_logging": True,
|
||||
"enable_console_progress": False,
|
||||
"enable_fail2ban_logging": False,
|
||||
},
|
||||
}
|
||||
except Exception as exc:
|
||||
logger.exception("Failed to read logging config")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to read logging config: {exc}",
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get("/files")
|
||||
def list_files(
|
||||
auth: Optional[dict] = Depends(require_auth),
|
||||
) -> Dict[str, Any]:
|
||||
"""List all available log files with metadata."""
|
||||
try:
|
||||
return {"success": True, "files": _list_log_files()}
|
||||
except Exception as exc:
|
||||
logger.exception("Failed to list log files")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to list log files: {exc}",
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get("/files/{filename}/tail")
|
||||
def tail_file(
|
||||
filename: str,
|
||||
lines: int = 100,
|
||||
auth: Optional[dict] = Depends(require_auth),
|
||||
) -> Dict[str, Any]:
|
||||
"""Return the last *lines* lines of a log file.
|
||||
|
||||
Args:
|
||||
filename: Name of the log file (no path traversal).
|
||||
lines: Number of lines to return (default 100).
|
||||
|
||||
Returns:
|
||||
Dict with ``success``, ``lines``, ``showing_lines``, ``total_lines``.
|
||||
"""
|
||||
# Prevent path traversal
|
||||
safe_name = Path(filename).name
|
||||
file_path = _log_dir() / safe_name
|
||||
if not file_path.exists():
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Log file not found: {safe_name}",
|
||||
)
|
||||
|
||||
try:
|
||||
all_lines = file_path.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
tail = all_lines[-lines:] if len(all_lines) > lines else all_lines
|
||||
return {
|
||||
"success": True,
|
||||
"lines": tail,
|
||||
"showing_lines": len(tail),
|
||||
"total_lines": len(all_lines),
|
||||
}
|
||||
except Exception as exc:
|
||||
logger.exception("Failed to tail log file %s", safe_name)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to read log file: {exc}",
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get("/files/{filename}/download")
|
||||
def download_file(
|
||||
filename: str,
|
||||
auth: Optional[dict] = Depends(require_auth),
|
||||
) -> FileResponse:
|
||||
"""Download a log file as an attachment."""
|
||||
safe_name = Path(filename).name
|
||||
file_path = _log_dir() / safe_name
|
||||
if not file_path.exists():
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Log file not found: {safe_name}",
|
||||
)
|
||||
return FileResponse(
|
||||
path=str(file_path),
|
||||
filename=safe_name,
|
||||
media_type="text/plain",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/test")
|
||||
def test_logging(
|
||||
auth: dict = Depends(require_auth),
|
||||
) -> Dict[str, Any]:
|
||||
"""Write test log messages at all levels."""
|
||||
logging.getLogger("aniworld.test").debug("Test DEBUG message")
|
||||
logging.getLogger("aniworld.test").info("Test INFO message")
|
||||
logging.getLogger("aniworld.test").warning("Test WARNING message")
|
||||
logging.getLogger("aniworld.test").error("Test ERROR message")
|
||||
return {"success": True, "message": "Test messages written to log"}
|
||||
|
||||
|
||||
@router.post("/cleanup")
|
||||
def cleanup_logs(
|
||||
payload: Dict[str, Any],
|
||||
auth: dict = Depends(require_auth),
|
||||
) -> Dict[str, Any]:
|
||||
"""Delete log files older than *days* days.
|
||||
|
||||
Args:
|
||||
payload: JSON body with ``days`` (int) field.
|
||||
|
||||
Returns:
|
||||
Dict with ``success`` and ``message`` describing what was deleted.
|
||||
"""
|
||||
import time
|
||||
|
||||
days = payload.get("days", 30)
|
||||
try:
|
||||
days = int(days)
|
||||
if days < 1:
|
||||
raise ValueError("days must be >= 1")
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid days value: {exc}",
|
||||
) from exc
|
||||
|
||||
cutoff = time.time() - days * 86400
|
||||
removed: List[str] = []
|
||||
errors: List[str] = []
|
||||
|
||||
for entry in _log_dir().iterdir():
|
||||
if entry.is_file() and entry.suffix in {".log", ".txt"}:
|
||||
if entry.stat().st_mtime < cutoff:
|
||||
try:
|
||||
entry.unlink()
|
||||
removed.append(entry.name)
|
||||
except OSError as exc:
|
||||
errors.append(f"{entry.name}: {exc}")
|
||||
|
||||
message = f"Removed {len(removed)} file(s) older than {days} days."
|
||||
if errors:
|
||||
message += f" Errors: {'; '.join(errors)}"
|
||||
|
||||
logger.info(
|
||||
"Log cleanup by %s: removed=%s days=%s",
|
||||
auth.get("username", "unknown"),
|
||||
removed,
|
||||
days,
|
||||
)
|
||||
return {"success": True, "message": message, "removed": removed}
|
||||
70
src/server/api/nfo.py
Normal file
70
src/server/api/nfo.py
Normal file
@@ -0,0 +1,70 @@
|
||||
"""NFO Management API endpoints.
|
||||
|
||||
Note: NFO service has been removed. All NFO endpoints return 503.
|
||||
"""
|
||||
from fastapi import APIRouter, HTTPException, status
|
||||
|
||||
router = APIRouter(prefix="/api/nfo", tags=["nfo"])
|
||||
|
||||
|
||||
@router.get("/disabled")
|
||||
async def nfo_disabled():
|
||||
"""NFO endpoints disabled - NFO service removed."""
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="NFO service has been removed. Use series management endpoints instead."
|
||||
)
|
||||
|
||||
|
||||
@router.post("/batch/create")
|
||||
async def batch_create_nfo():
|
||||
"""NFO endpoints disabled - NFO service removed."""
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="NFO service has been removed. Use series management endpoints instead."
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{serie_id}/create")
|
||||
async def create_nfo(serie_id: str):
|
||||
"""NFO endpoints disabled - NFO service removed."""
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="NFO service has been removed. Use series management endpoints instead."
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{serie_id}/status")
|
||||
async def get_nfo_status(serie_id: str):
|
||||
"""NFO endpoints disabled - NFO service removed."""
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="NFO service has been removed. Use series management endpoints instead."
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{serie_id}/delete")
|
||||
async def delete_nfo(serie_id: str):
|
||||
"""NFO endpoints disabled - NFO service removed."""
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="NFO service has been removed. Use series management endpoints instead."
|
||||
)
|
||||
|
||||
|
||||
@router.get("/poster/{serie_id}")
|
||||
async def get_nfo_poster(serie_id: str):
|
||||
"""NFO endpoints disabled - NFO service removed."""
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="NFO service has been removed. Use series management endpoints instead."
|
||||
)
|
||||
|
||||
|
||||
@router.get("/fanart/{serie_id}")
|
||||
async def get_nfo_fanart(serie_id: str):
|
||||
"""NFO endpoints disabled - NFO service removed."""
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="NFO service has been removed. Use series management endpoints instead."
|
||||
)
|
||||
@@ -4,12 +4,13 @@ This module provides endpoints for managing scheduled tasks such as
|
||||
automatic anime library rescans.
|
||||
"""
|
||||
import logging
|
||||
from typing import Dict, Optional
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
|
||||
from src.server.models.config import SchedulerConfig
|
||||
from src.server.services.config_service import ConfigServiceError, get_config_service
|
||||
from src.server.services.scheduler.scheduler_service import get_scheduler_service
|
||||
from src.server.utils.dependencies import require_auth
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -17,78 +18,105 @@ logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/scheduler", tags=["scheduler"])
|
||||
|
||||
|
||||
@router.get("/config", response_model=SchedulerConfig)
|
||||
def get_scheduler_config(
|
||||
auth: Optional[dict] = Depends(require_auth)
|
||||
) -> SchedulerConfig:
|
||||
"""Get current scheduler configuration.
|
||||
def _build_response(config: SchedulerConfig) -> Dict[str, Any]:
|
||||
"""Build a standardised GET/POST response combining config + runtime status."""
|
||||
scheduler_service = get_scheduler_service()
|
||||
runtime = scheduler_service.get_status()
|
||||
|
||||
Args:
|
||||
auth: Authentication token (optional for read operations)
|
||||
return {
|
||||
"success": True,
|
||||
"config": {
|
||||
"enabled": config.enabled,
|
||||
"interval_minutes": config.interval_minutes,
|
||||
"schedule_time": config.schedule_time,
|
||||
"schedule_days": config.schedule_days,
|
||||
"auto_download_after_rescan": config.auto_download_after_rescan,
|
||||
},
|
||||
"status": {
|
||||
"is_running": runtime.get("is_running", False),
|
||||
"next_run": runtime.get("next_run"),
|
||||
"last_run": runtime.get("last_run"),
|
||||
"scan_in_progress": runtime.get("scan_in_progress", False),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/config")
|
||||
def get_scheduler_config(
|
||||
auth: Optional[dict] = Depends(require_auth),
|
||||
) -> Dict[str, Any]:
|
||||
"""Get current scheduler configuration along with runtime status.
|
||||
|
||||
Returns:
|
||||
SchedulerConfig: Current scheduler configuration
|
||||
Combined config and status response.
|
||||
|
||||
Raises:
|
||||
HTTPException: If configuration cannot be loaded
|
||||
HTTPException: 500 if configuration cannot be loaded.
|
||||
"""
|
||||
try:
|
||||
config_service = get_config_service()
|
||||
app_config = config_service.load_config()
|
||||
return app_config.scheduler
|
||||
except ConfigServiceError as e:
|
||||
logger.error(f"Failed to load scheduler config: {e}")
|
||||
return _build_response(app_config.scheduler)
|
||||
except ConfigServiceError as exc:
|
||||
logger.error("Failed to load scheduler config: %s", exc)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to load scheduler configuration: {e}",
|
||||
) from e
|
||||
detail=f"Failed to load scheduler configuration: {exc}",
|
||||
) from exc
|
||||
|
||||
|
||||
@router.post("/config", response_model=SchedulerConfig)
|
||||
@router.post("/config")
|
||||
def update_scheduler_config(
|
||||
scheduler_config: SchedulerConfig,
|
||||
auth: dict = Depends(require_auth),
|
||||
) -> SchedulerConfig:
|
||||
"""Update scheduler configuration.
|
||||
) -> Dict[str, Any]:
|
||||
"""Update scheduler configuration and apply changes immediately.
|
||||
|
||||
Args:
|
||||
scheduler_config: New scheduler configuration
|
||||
auth: Authentication token (required)
|
||||
Accepts the full SchedulerConfig body; any fields not supplied default
|
||||
to their model defaults (backward compatible).
|
||||
|
||||
Returns:
|
||||
SchedulerConfig: Updated scheduler configuration
|
||||
Combined config and status response reflecting the saved config.
|
||||
|
||||
Raises:
|
||||
HTTPException: If configuration update fails
|
||||
HTTPException: 422 on validation errors (handled by FastAPI/Pydantic),
|
||||
500 on save or scheduler failure.
|
||||
"""
|
||||
try:
|
||||
config_service = get_config_service()
|
||||
app_config = config_service.load_config()
|
||||
|
||||
# Update scheduler section
|
||||
app_config.scheduler = scheduler_config
|
||||
|
||||
# Save and return
|
||||
config_service.save_config(app_config)
|
||||
|
||||
logger.info(
|
||||
f"Scheduler config updated by {auth.get('username', 'unknown')}"
|
||||
"Scheduler config updated by %s: time=%s days=%s auto_dl=%s",
|
||||
auth.get("username", "unknown"),
|
||||
scheduler_config.schedule_time,
|
||||
scheduler_config.schedule_days,
|
||||
scheduler_config.auto_download_after_rescan,
|
||||
)
|
||||
|
||||
return scheduler_config
|
||||
except ConfigServiceError as e:
|
||||
logger.error(f"Failed to update scheduler config: {e}")
|
||||
# Apply changes to the running scheduler without restart
|
||||
try:
|
||||
sched_svc = get_scheduler_service()
|
||||
sched_svc.reload_config(scheduler_config)
|
||||
except Exception as sched_exc: # pylint: disable=broad-exception-caught
|
||||
logger.error("Scheduler reload after config update failed: %s", sched_exc)
|
||||
# Config was saved — don't fail the request, just warn
|
||||
|
||||
return _build_response(scheduler_config)
|
||||
|
||||
except ConfigServiceError as exc:
|
||||
logger.error("Failed to update scheduler config: %s", exc)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to update scheduler configuration: {e}",
|
||||
) from e
|
||||
detail=f"Failed to update scheduler configuration: {exc}",
|
||||
) from exc
|
||||
|
||||
|
||||
@router.post("/trigger-rescan", response_model=Dict[str, str])
|
||||
async def trigger_rescan(auth: dict = Depends(require_auth)) -> Dict[str, str]:
|
||||
"""Manually trigger a library rescan.
|
||||
|
||||
This endpoint triggers an immediate anime library rescan, bypassing
|
||||
the scheduler interval.
|
||||
"""Manually trigger a library rescan (and auto-download if configured).
|
||||
|
||||
Args:
|
||||
auth: Authentication token (required)
|
||||
@@ -100,8 +128,7 @@ async def trigger_rescan(auth: dict = Depends(require_auth)) -> Dict[str, str]:
|
||||
HTTPException: If rescan cannot be triggered
|
||||
"""
|
||||
try:
|
||||
# Import here to avoid circular dependency
|
||||
from src.server.fastapi_app import get_series_app
|
||||
from src.server.utils.dependencies import get_series_app # noqa: PLC0415
|
||||
|
||||
series_app = get_series_app()
|
||||
if not series_app:
|
||||
@@ -110,21 +137,19 @@ async def trigger_rescan(auth: dict = Depends(require_auth)) -> Dict[str, str]:
|
||||
detail="SeriesApp not initialized",
|
||||
)
|
||||
|
||||
# Trigger the rescan
|
||||
logger.info(
|
||||
f"Manual rescan triggered by {auth.get('username', 'unknown')}"
|
||||
"Manual rescan triggered by %s", auth.get("username", "unknown")
|
||||
)
|
||||
|
||||
# Use existing rescan logic from anime API
|
||||
from src.server.api.anime import trigger_rescan as do_rescan
|
||||
from src.server.api.anime import trigger_rescan as do_rescan # noqa: PLC0415
|
||||
|
||||
return await do_rescan()
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
except Exception as exc:
|
||||
logger.exception("Failed to trigger manual rescan")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to trigger rescan: {str(e)}",
|
||||
) from e
|
||||
detail=f"Failed to trigger rescan: {exc}",
|
||||
) from exc
|
||||
|
||||
376
src/server/api/setup_endpoints.py
Normal file
376
src/server/api/setup_endpoints.py
Normal file
@@ -0,0 +1,376 @@
|
||||
"""API endpoints for setup and unresolved folder management.
|
||||
|
||||
Provides endpoints to:
|
||||
- List unresolved folders that couldn't be auto-resolved during setup
|
||||
- Get suggestions/search results for an unresolved folder
|
||||
- Resolve an unresolved folder by providing a provider key
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from src.server.database.connection import get_db_session
|
||||
from src.server.database.service import AnimeSeriesService, UnresolvedFolderService
|
||||
from src.server.utils.dependencies import (
|
||||
get_database_session,
|
||||
get_series_app,
|
||||
require_auth,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/setup", tags=["setup"])
|
||||
|
||||
|
||||
class UnresolvedFolderResponse(BaseModel):
|
||||
"""Response model for an unresolved folder."""
|
||||
|
||||
folder_name: str = Field(..., description="Original filesystem folder name")
|
||||
title: str = Field(..., description="Extracted title from folder name")
|
||||
year: Optional[int] = Field(None, description="Extracted release year")
|
||||
search_attempts: int = Field(..., description="Number of search attempts made")
|
||||
search_suggestions: list[dict[str, Any]] = Field(
|
||||
default_factory=list,
|
||||
description="Cached search results for potential matches"
|
||||
)
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class ResolveFolderRequest(BaseModel):
|
||||
"""Request model for resolving an unresolved folder."""
|
||||
|
||||
provider_key: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
max_length=255,
|
||||
description="Provider key to associate with this folder"
|
||||
)
|
||||
|
||||
|
||||
class ResolveFolderResponse(BaseModel):
|
||||
"""Response model for resolving an unresolved folder."""
|
||||
|
||||
status: str = Field(..., description="Operation status")
|
||||
message: str = Field(..., description="Human-readable message")
|
||||
folder_name: str = Field(..., description="Folder name that was resolved")
|
||||
key: str = Field(..., description="Provider key that was used")
|
||||
series_id: int = Field(..., description="Database ID of the created series")
|
||||
|
||||
|
||||
@router.get("/unresolved", response_model=list[UnresolvedFolderResponse])
|
||||
async def list_unresolved_folders(
|
||||
db=Depends(get_database_session),
|
||||
) -> list[UnresolvedFolderResponse]:
|
||||
"""List all unresolved folders that need manual key resolution.
|
||||
|
||||
Returns folders that couldn't be auto-resolved during setup,
|
||||
including cached search suggestions when available.
|
||||
|
||||
Returns:
|
||||
List of UnresolvedFolderResponse objects
|
||||
"""
|
||||
folders = await UnresolvedFolderService.get_all_unresolved(db)
|
||||
|
||||
result = []
|
||||
for folder in folders:
|
||||
suggestions = []
|
||||
if folder.last_search_result:
|
||||
try:
|
||||
suggestions = json.loads(folder.last_search_result)
|
||||
except json.JSONDecodeError:
|
||||
logger.warning(
|
||||
"Failed to parse search result for folder: %s",
|
||||
folder.folder_name
|
||||
)
|
||||
|
||||
result.append(UnresolvedFolderResponse(
|
||||
folder_name=folder.folder_name,
|
||||
title=folder.title,
|
||||
year=folder.year,
|
||||
search_attempts=folder.search_attempts,
|
||||
search_suggestions=suggestions,
|
||||
))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/unresolved/{folder_name}", response_model=UnresolvedFolderResponse)
|
||||
async def get_unresolved_folder(
|
||||
folder_name: str,
|
||||
db=Depends(get_database_session),
|
||||
) -> UnresolvedFolderResponse:
|
||||
"""Get details for a specific unresolved folder.
|
||||
|
||||
Args:
|
||||
folder_name: URL-encoded folder name to look up
|
||||
|
||||
Returns:
|
||||
UnresolvedFolderResponse for the specified folder
|
||||
|
||||
Raises:
|
||||
HTTPException: 404 if folder not found or already resolved
|
||||
"""
|
||||
folder = await UnresolvedFolderService.get_by_folder_name(db, folder_name)
|
||||
|
||||
if not folder:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Unresolved folder not found: {folder_name}"
|
||||
)
|
||||
|
||||
if folder.is_resolved:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Folder already resolved: {folder_name}"
|
||||
)
|
||||
|
||||
suggestions = []
|
||||
if folder.last_search_result:
|
||||
try:
|
||||
suggestions = json.loads(folder.last_search_result)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
return UnresolvedFolderResponse(
|
||||
folder_name=folder.folder_name,
|
||||
title=folder.title,
|
||||
year=folder.year,
|
||||
search_attempts=folder.search_attempts,
|
||||
search_suggestions=suggestions,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/unresolved/{folder_name}/resolve", response_model=ResolveFolderResponse)
|
||||
async def resolve_unresolved_folder(
|
||||
folder_name: str,
|
||||
request: ResolveFolderRequest,
|
||||
db=Depends(get_database_session),
|
||||
) -> ResolveFolderResponse:
|
||||
"""Resolve an unresolved folder by providing the correct provider key.
|
||||
|
||||
This endpoint:
|
||||
1. Validates the provider key format
|
||||
2. Updates the UnresolvedFolder record as resolved
|
||||
3. Creates the AnimeSeries record in the database
|
||||
4. Returns the created series information
|
||||
|
||||
Args:
|
||||
folder_name: URL-encoded folder name to resolve
|
||||
request: ResolveFolderRequest with the provider_key
|
||||
|
||||
Returns:
|
||||
ResolveFolderResponse with created series details
|
||||
|
||||
Raises:
|
||||
HTTPException: 404 if folder not found
|
||||
HTTPException: 400 if key is invalid or series already exists
|
||||
"""
|
||||
# Check if folder exists and is unresolved
|
||||
unresolved = await UnresolvedFolderService.get_by_folder_name(db, folder_name)
|
||||
|
||||
if not unresolved:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Unresolved folder not found: {folder_name}"
|
||||
)
|
||||
|
||||
if unresolved.is_resolved:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Folder already resolved: {folder_name}"
|
||||
)
|
||||
|
||||
# Check if a series with this key already exists
|
||||
existing_series = await AnimeSeriesService.get_by_key(db, request.provider_key)
|
||||
if existing_series:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Series with key '{request.provider_key}' already exists"
|
||||
)
|
||||
|
||||
# Mark as resolved
|
||||
await UnresolvedFolderService.resolve(db, folder_name, request.provider_key)
|
||||
|
||||
# Create the AnimeSeries record
|
||||
series = await AnimeSeriesService.create(
|
||||
db=db,
|
||||
key=request.provider_key,
|
||||
name=unresolved.title,
|
||||
site="https://aniworld.to",
|
||||
folder=folder_name,
|
||||
year=unresolved.year,
|
||||
loading_status="pending",
|
||||
episodes_loaded=False,
|
||||
logo_loaded=False,
|
||||
images_loaded=False,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Resolved unresolved folder via API: %s -> key=%s (series_id=%d)",
|
||||
folder_name, request.provider_key, series.id
|
||||
)
|
||||
|
||||
return ResolveFolderResponse(
|
||||
status="success",
|
||||
message=f"Successfully resolved and added series: {unresolved.title}",
|
||||
folder_name=folder_name,
|
||||
key=request.provider_key,
|
||||
series_id=series.id,
|
||||
)
|
||||
|
||||
|
||||
class SearchFolderRequest(BaseModel):
|
||||
"""Request model for searching an unresolved folder with custom query."""
|
||||
query: Optional[str] = Field(None, description="Custom search query override")
|
||||
|
||||
|
||||
@router.post("/unresolved/{folder_name}/search", response_model=UnresolvedFolderResponse)
|
||||
async def search_unresolved_folder(
|
||||
folder_name: str,
|
||||
request: Optional[SearchFolderRequest] = None,
|
||||
db=Depends(get_database_session),
|
||||
) -> UnresolvedFolderResponse:
|
||||
"""Re-search for a specific unresolved folder to get fresh suggestions.
|
||||
|
||||
Performs a new search using the folder's title or a custom query.
|
||||
Caches the results for subsequent display.
|
||||
|
||||
Args:
|
||||
folder_name: URL-encoded folder name to search for
|
||||
request: Optional SearchFolderRequest with custom query override
|
||||
|
||||
Returns:
|
||||
UnresolvedFolderResponse with updated search suggestions
|
||||
|
||||
Raises:
|
||||
HTTPException: 404 if folder not found or already resolved
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
folder = await UnresolvedFolderService.get_by_folder_name(db, folder_name)
|
||||
|
||||
if not folder:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Unresolved folder not found: {folder_name}"
|
||||
)
|
||||
|
||||
if folder.is_resolved:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Folder already resolved: {folder_name}"
|
||||
)
|
||||
|
||||
# Use custom query if provided, otherwise fall back to folder title
|
||||
search_query = request.query if request and request.query else folder.title
|
||||
|
||||
# Perform search
|
||||
series_app = get_series_app()
|
||||
try:
|
||||
results = await series_app.search(search_query)
|
||||
search_result_json = json.dumps(results) if results else "[]"
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Search failed for unresolved folder: %s, error: %s",
|
||||
folder_name, str(e)
|
||||
)
|
||||
search_result_json = "[]"
|
||||
results = []
|
||||
|
||||
# Update the folder with new search results
|
||||
await UnresolvedFolderService.update_search_result(db, folder_name, search_result_json)
|
||||
|
||||
return UnresolvedFolderResponse(
|
||||
folder_name=folder.folder_name,
|
||||
title=folder.title,
|
||||
year=folder.year,
|
||||
search_attempts=folder.search_attempts + 1,
|
||||
search_suggestions=results,
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/unresolved/{folder_name}")
|
||||
async def delete_unresolved_folder(
|
||||
folder_name: str,
|
||||
db=Depends(get_database_session),
|
||||
) -> dict[str, str]:
|
||||
"""Delete an unresolved folder tracking record.
|
||||
|
||||
Use this when you've manually added the series outside of this flow
|
||||
(e.g., via POST /api/anime/add) to clean up the unresolved tracker.
|
||||
|
||||
Args:
|
||||
folder_name: URL-encoded folder name to delete
|
||||
|
||||
Returns:
|
||||
Dict with status message
|
||||
|
||||
Raises:
|
||||
HTTPException: 404 if folder not found
|
||||
"""
|
||||
deleted = await UnresolvedFolderService.delete(db, folder_name)
|
||||
|
||||
if not deleted:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Unresolved folder not found: {folder_name}"
|
||||
)
|
||||
|
||||
return {"status": "success", "message": f"Deleted unresolved folder: {folder_name}"}
|
||||
|
||||
|
||||
class DoneResponse(BaseModel):
|
||||
"""Response model for completing unresolved folders."""
|
||||
status: str = Field(..., description="Operation status")
|
||||
message: str = Field(..., description="Human-readable message")
|
||||
count: int = Field(..., description="Number of folders marked as done")
|
||||
|
||||
|
||||
@router.post("/unresolved/done", response_model=DoneResponse)
|
||||
async def complete_unresolved_folders(
|
||||
db=Depends(get_database_session),
|
||||
) -> DoneResponse:
|
||||
"""Mark all unresolved folders as handled and complete the unresolved phase.
|
||||
|
||||
This endpoint:
|
||||
1. Marks the unresolved phase as completed in config
|
||||
2. Returns the count of folders that were handled
|
||||
|
||||
After this, /setup/unresolved will redirect to /loading.
|
||||
|
||||
Returns:
|
||||
DoneResponse with status and count of handled folders
|
||||
"""
|
||||
from src.server.services.config_service import get_config_service
|
||||
|
||||
# Get all unresolved folders
|
||||
folders = await UnresolvedFolderService.get_all_unresolved(db)
|
||||
count = len(folders)
|
||||
|
||||
# Mark unresolved as completed in config
|
||||
config_service = get_config_service()
|
||||
try:
|
||||
config = config_service.load_config()
|
||||
if config.other is None:
|
||||
config.other = {}
|
||||
config.other['unresolved_completed'] = True
|
||||
config_service.save_config(config, create_backup=False)
|
||||
logger.info("Marked unresolved phase as completed")
|
||||
except Exception as e:
|
||||
logger.warning("Failed to save unresolved_completed flag: %s", e)
|
||||
|
||||
logger.info(
|
||||
"Completed unresolved phase: %d folders handled",
|
||||
count
|
||||
)
|
||||
|
||||
return DoneResponse(
|
||||
status="success",
|
||||
message=f"Marked {count} folders as handled. Unresolved phase completed.",
|
||||
count=count,
|
||||
)
|
||||
@@ -39,7 +39,7 @@ def get_settings() -> Union[DevelopmentSettings, ProductionSettings]:
|
||||
Example:
|
||||
>>> settings = get_settings()
|
||||
>>> print(settings.log_level)
|
||||
DEBUG
|
||||
INFO
|
||||
"""
|
||||
if ENVIRONMENT in {"development", "testing"}:
|
||||
return get_development_settings()
|
||||
|
||||
@@ -215,7 +215,7 @@ class DevelopmentSettings(BaseSettings):
|
||||
@property
|
||||
def debug_enabled(self) -> bool:
|
||||
"""Check if debug mode is enabled."""
|
||||
return True
|
||||
return False
|
||||
|
||||
@property
|
||||
def reload_enabled(self) -> bool:
|
||||
|
||||
@@ -95,10 +95,10 @@ def setup_logging() -> Dict[str, logging.Logger]:
|
||||
# Log initial setup
|
||||
root_logger.info("=" * 80)
|
||||
root_logger.info("FastAPI Server Logging Initialized")
|
||||
root_logger.info(f"Log Level: {settings.log_level.upper()}")
|
||||
root_logger.info(f"Server Log: {server_log_file.absolute()}")
|
||||
root_logger.info(f"Error Log: {error_log_file.absolute()}")
|
||||
root_logger.info(f"Access Log: {access_log_file.absolute()}")
|
||||
root_logger.info("Log Level: %s", settings.log_level.upper())
|
||||
root_logger.info("Server Log: %s", server_log_file.absolute())
|
||||
root_logger.info("Error Log: %s", error_log_file.absolute())
|
||||
root_logger.info("Access Log: %s", access_log_file.absolute())
|
||||
root_logger.info("=" * 80)
|
||||
|
||||
return {
|
||||
|
||||
@@ -49,3 +49,23 @@ async def queue_page(request: Request):
|
||||
request,
|
||||
title="Download Queue - Aniworld"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/loading", response_class=HTMLResponse)
|
||||
async def loading_page(request: Request):
|
||||
"""Serve the initialization loading page."""
|
||||
return render_template(
|
||||
"loading.html",
|
||||
request,
|
||||
title="Initializing - Aniworld"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/setup/unresolved", response_class=HTMLResponse)
|
||||
async def unresolved_page(request: Request):
|
||||
"""Serve the unresolved folders resolution page."""
|
||||
return render_template(
|
||||
"unresolved.html",
|
||||
request,
|
||||
title="Resolve Series - Aniworld"
|
||||
)
|
||||
|
||||
288
src/server/database/SerieList.py
Normal file
288
src/server/database/SerieList.py
Normal file
@@ -0,0 +1,288 @@
|
||||
"""Utilities for loading and managing stored anime series metadata.
|
||||
|
||||
This module provides the SerieList class for managing collections of anime
|
||||
series metadata loaded from the database.
|
||||
|
||||
Note:
|
||||
This module is part of the server database layer. All persistence
|
||||
is handled by the service layer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from src.server.database.models import AnimeSeries
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SerieList:
|
||||
"""
|
||||
Represents the collection of cached series loaded from database.
|
||||
|
||||
Series are identified by their unique 'key' (provider identifier).
|
||||
The 'folder' is metadata only and not used for lookups.
|
||||
|
||||
This class manages in-memory series data loaded from database.
|
||||
|
||||
Example:
|
||||
# Load from database
|
||||
serie_list = SerieList("/path/to/anime")
|
||||
await serie_list.load_all_from_db()
|
||||
series = serie_list.get_all()
|
||||
|
||||
Attributes:
|
||||
directory: Path to the anime directory
|
||||
keyDict: Internal dictionary mapping serie.key to AnimeSeries objects
|
||||
"""
|
||||
|
||||
def __init__(self, base_path: str) -> None:
|
||||
"""Initialize the SerieList.
|
||||
|
||||
Args:
|
||||
base_path: Path to the anime directory
|
||||
"""
|
||||
self.directory: str = base_path
|
||||
# Internal storage using serie.key as the dictionary key
|
||||
self.keyDict: Dict[str, AnimeSeries] = {}
|
||||
|
||||
async def add_to_db(self, anime: AnimeSeries) -> bool:
|
||||
"""Persist a new series to the database.
|
||||
|
||||
Creates the filesystem folder using anime.folder, then persists
|
||||
the series metadata to the database.
|
||||
|
||||
Args:
|
||||
anime: The AnimeSeries instance to add
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
try:
|
||||
from src.server.database.connection import get_async_session_factory
|
||||
from src.server.database.service import AnimeSeriesService, EpisodeService
|
||||
|
||||
folder_name = anime.folder
|
||||
anime_path = self.directory + "/" + folder_name
|
||||
import os
|
||||
os.makedirs(anime_path, exist_ok=True)
|
||||
|
||||
session_factory = get_async_session_factory()
|
||||
db = session_factory()
|
||||
try:
|
||||
existing = await AnimeSeriesService.get_by_key(db, anime.key)
|
||||
if existing:
|
||||
logger.debug(
|
||||
"Series '%s' (key=%s) already exists in DB, skipping",
|
||||
anime.name, anime.key
|
||||
)
|
||||
return True
|
||||
|
||||
db_anime_series = await AnimeSeriesService.create(
|
||||
db=db,
|
||||
key=anime.key,
|
||||
name=anime.name,
|
||||
site=anime.site,
|
||||
folder=folder_name,
|
||||
year=anime.year
|
||||
)
|
||||
for ep in anime.episodes:
|
||||
await EpisodeService.create(
|
||||
db=db,
|
||||
series_id=db_anime_series.id,
|
||||
season=ep.season,
|
||||
episode_number=ep.episode_number
|
||||
)
|
||||
await db.commit()
|
||||
self.keyDict[anime.key] = anime
|
||||
logger.info(
|
||||
"Persisted series '%s' to database",
|
||||
anime.name
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
logger.error(
|
||||
"Failed to persist series '%s' to DB: %s",
|
||||
anime.key, e, exc_info=True
|
||||
)
|
||||
return False
|
||||
finally:
|
||||
await db.close()
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Could not add series '%s' to DB (DB unavailable?): %s",
|
||||
anime.key, e
|
||||
)
|
||||
return False
|
||||
|
||||
def contains(self, key: str) -> bool:
|
||||
"""
|
||||
Return True when a series identified by ``key`` already exists.
|
||||
|
||||
Args:
|
||||
key: The unique provider identifier for the series
|
||||
|
||||
Returns:
|
||||
True if the series exists in the collection
|
||||
"""
|
||||
return key in self.keyDict
|
||||
|
||||
def GetMissingEpisode(self) -> List[AnimeSeries]:
|
||||
"""Return all series that still contain missing episodes."""
|
||||
return [
|
||||
anime for anime in self.keyDict.values()
|
||||
if anime.episodeDict
|
||||
]
|
||||
|
||||
def get_missing_episodes(self) -> List[AnimeSeries]:
|
||||
"""PEP8-friendly alias for :meth:`GetMissingEpisode`."""
|
||||
return self.GetMissingEpisode()
|
||||
|
||||
def GetList(self) -> List[AnimeSeries]:
|
||||
"""Return all series instances stored in the list."""
|
||||
return list(self.keyDict.values())
|
||||
|
||||
def get_all(self) -> List[AnimeSeries]:
|
||||
"""PEP8-friendly alias for :meth:`GetList`."""
|
||||
return self.GetList()
|
||||
|
||||
def get_by_key(self, key: str) -> Optional[AnimeSeries]:
|
||||
"""
|
||||
Get a series by its unique provider key.
|
||||
|
||||
This is the primary method for series lookup.
|
||||
|
||||
Args:
|
||||
key: The unique provider identifier (e.g., "attack-on-titan")
|
||||
|
||||
Returns:
|
||||
The AnimeSeries instance if found, None otherwise
|
||||
"""
|
||||
return self.keyDict.get(key)
|
||||
|
||||
def get_by_folder(self, folder: str) -> Optional[AnimeSeries]:
|
||||
"""
|
||||
Get a series by its folder name.
|
||||
|
||||
.. deprecated:: 2.0.0
|
||||
Use :meth:`get_by_key` instead. Folder-based lookups will be
|
||||
removed in version 3.0.0. The `folder` field is metadata only
|
||||
and should not be used for identification.
|
||||
|
||||
This method is provided for backward compatibility only.
|
||||
Prefer using get_by_key() for new code.
|
||||
|
||||
Args:
|
||||
folder: The filesystem folder name (e.g., "Attack on Titan (2013)")
|
||||
|
||||
Returns:
|
||||
The AnimeSeries instance if found, None otherwise
|
||||
"""
|
||||
import warnings
|
||||
warnings.warn(
|
||||
"get_by_folder() is deprecated and will be removed in v3.0.0. "
|
||||
"Use get_by_key() instead. The 'folder' field is metadata only.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
for anime in self.keyDict.values():
|
||||
if anime.folder == folder:
|
||||
return anime
|
||||
return None
|
||||
|
||||
async def load_all_from_db(self) -> int:
|
||||
"""Load all series from database into in-memory cache.
|
||||
|
||||
Retrieves all anime series from the database with their episodes
|
||||
and populates the in-memory keyDict for fast access.
|
||||
|
||||
Returns:
|
||||
int: Number of series loaded into cache
|
||||
"""
|
||||
from src.server.database.connection import get_async_session_factory
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
|
||||
try:
|
||||
session_factory = get_async_session_factory()
|
||||
db = session_factory()
|
||||
try:
|
||||
anime_series_list = await AnimeSeriesService.get_all(
|
||||
db, with_episodes=True
|
||||
)
|
||||
|
||||
count = 0
|
||||
for anime_series in anime_series_list:
|
||||
self.keyDict[anime_series.key] = anime_series
|
||||
count += 1
|
||||
|
||||
logger.info(
|
||||
"Loaded %d series from database into in-memory cache",
|
||||
count
|
||||
)
|
||||
return count
|
||||
finally:
|
||||
await db.close()
|
||||
except RuntimeError:
|
||||
logger.warning(
|
||||
"Database not available, skipping DB load"
|
||||
)
|
||||
return 0
|
||||
|
||||
async def _load_single_series_from_db(
|
||||
self,
|
||||
anime_folder: str
|
||||
) -> Optional[AnimeSeries]:
|
||||
"""Load a single series from database by folder name.
|
||||
|
||||
Looks up a series in the database by its folder name and adds
|
||||
it to the in-memory cache.
|
||||
|
||||
Args:
|
||||
anime_folder: The filesystem folder name to look up
|
||||
|
||||
Returns:
|
||||
AnimeSeries if found and loaded, None otherwise
|
||||
"""
|
||||
from src.server.database.connection import get_async_session_factory
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
|
||||
try:
|
||||
session_factory = get_async_session_factory()
|
||||
db = session_factory()
|
||||
try:
|
||||
anime_series = await AnimeSeriesService.get_by_folder(
|
||||
db, anime_folder
|
||||
)
|
||||
if not anime_series:
|
||||
logger.debug(
|
||||
"Series with folder '%s' not found in DB",
|
||||
anime_folder
|
||||
)
|
||||
return None
|
||||
|
||||
self.keyDict[anime_series.key] = anime_series
|
||||
logger.debug(
|
||||
"Loaded series '%s' (key=%s) from DB",
|
||||
anime_series.name, anime_series.key
|
||||
)
|
||||
return anime_series
|
||||
finally:
|
||||
await db.close()
|
||||
except RuntimeError:
|
||||
logger.warning(
|
||||
"Database not available, cannot load series '%s'",
|
||||
anime_folder
|
||||
)
|
||||
return None
|
||||
|
||||
def invalidate_cache(self) -> None:
|
||||
"""Clear the in-memory cache.
|
||||
|
||||
Use after database modifications to force reload from DB
|
||||
on next access.
|
||||
"""
|
||||
self.keyDict.clear()
|
||||
logger.debug("SerieList in-memory cache invalidated")
|
||||
@@ -39,6 +39,7 @@ from src.server.database.models import (
|
||||
AnimeSeries,
|
||||
DownloadQueueItem,
|
||||
Episode,
|
||||
SystemSettings,
|
||||
UserSession,
|
||||
)
|
||||
from src.server.database.service import (
|
||||
@@ -47,6 +48,8 @@ from src.server.database.service import (
|
||||
EpisodeService,
|
||||
UserSessionService,
|
||||
)
|
||||
from src.server.database.SerieList import SerieList
|
||||
from src.server.database.system_settings_service import SystemSettingsService
|
||||
|
||||
__all__ = [
|
||||
# Base and connection
|
||||
@@ -69,10 +72,14 @@ __all__ = [
|
||||
"AnimeSeries",
|
||||
"Episode",
|
||||
"DownloadQueueItem",
|
||||
"SystemSettings",
|
||||
"UserSession",
|
||||
# Services
|
||||
"AnimeSeriesService",
|
||||
"EpisodeService",
|
||||
"DownloadQueueService",
|
||||
"SystemSettingsService",
|
||||
"UserSessionService",
|
||||
# SerieList
|
||||
"SerieList",
|
||||
]
|
||||
|
||||
@@ -88,7 +88,7 @@ async def init_db() -> None:
|
||||
try:
|
||||
# Get database URL
|
||||
db_url = _get_database_url()
|
||||
logger.info(f"Initializing database: {db_url}")
|
||||
logger.info("Initializing database: %s", db_url)
|
||||
|
||||
# Build engine kwargs based on database type
|
||||
is_sqlite = "sqlite" in db_url
|
||||
@@ -143,7 +143,7 @@ async def init_db() -> None:
|
||||
logger.info("Database initialization complete")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize database: {e}")
|
||||
logger.error("Failed to initialize database: %s", e)
|
||||
raise
|
||||
|
||||
|
||||
@@ -171,7 +171,7 @@ async def close_db() -> None:
|
||||
conn.commit()
|
||||
logger.info("SQLite WAL checkpoint completed")
|
||||
except Exception as e:
|
||||
logger.warning(f"WAL checkpoint failed (non-critical): {e}")
|
||||
logger.warning("WAL checkpoint failed (non-critical): %s", e)
|
||||
|
||||
if _engine:
|
||||
logger.info("Closing async database engine...")
|
||||
@@ -188,7 +188,7 @@ async def close_db() -> None:
|
||||
logger.info("Database connections closed")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error closing database: {e}")
|
||||
logger.error("Error closing database: %s", e)
|
||||
|
||||
|
||||
def get_engine() -> AsyncEngine:
|
||||
|
||||
@@ -27,7 +27,7 @@ logger = logging.getLogger(__name__)
|
||||
# Schema Version Constants
|
||||
# =============================================================================
|
||||
|
||||
CURRENT_SCHEMA_VERSION = "1.0.0"
|
||||
CURRENT_SCHEMA_VERSION = "1.0.1"
|
||||
SCHEMA_VERSION_TABLE = "schema_version"
|
||||
|
||||
# Expected tables in the current schema
|
||||
@@ -36,6 +36,7 @@ EXPECTED_TABLES = {
|
||||
"episodes",
|
||||
"download_queue",
|
||||
"user_sessions",
|
||||
"system_settings",
|
||||
}
|
||||
|
||||
# Expected indexes for performance
|
||||
@@ -97,7 +98,7 @@ async def initialize_database(
|
||||
seed_data=True
|
||||
)
|
||||
if result["success"]:
|
||||
logger.info(f"Database initialized: {result['schema_version']}")
|
||||
logger.info("Database initialized: %s", result['schema_version'])
|
||||
"""
|
||||
if engine is None:
|
||||
engine = get_engine()
|
||||
@@ -116,7 +117,12 @@ async def initialize_database(
|
||||
if create_schema:
|
||||
tables = await create_database_schema(engine)
|
||||
result["tables_created"] = tables
|
||||
logger.info(f"Created {len(tables)} tables")
|
||||
logger.info("Created %s tables", len(tables))
|
||||
|
||||
# Migrate schema if needed (add missing columns to existing tables)
|
||||
migrations = await migrate_schema_if_needed(engine)
|
||||
if migrations:
|
||||
logger.info("Applied %s schema migrations", len(migrations))
|
||||
|
||||
# Validate schema if requested
|
||||
if validate_schema:
|
||||
@@ -147,7 +153,7 @@ async def initialize_database(
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Database initialization failed: {e}", exc_info=True)
|
||||
logger.exception("Database initialization failed: %s", e)
|
||||
raise RuntimeError(f"Failed to initialize database: {e}") from e
|
||||
|
||||
|
||||
@@ -193,14 +199,14 @@ async def create_database_schema(
|
||||
created_tables = [t for t in new_tables if t not in existing_tables]
|
||||
|
||||
if created_tables:
|
||||
logger.info(f"Created tables: {', '.join(created_tables)}")
|
||||
logger.info("Created tables: %s", ', '.join(created_tables))
|
||||
else:
|
||||
logger.info("All tables already exist")
|
||||
|
||||
return new_tables
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create schema: {e}", exc_info=True)
|
||||
logger.exception("Failed to create schema: %s", e)
|
||||
raise RuntimeError(f"Schema creation failed: {e}") from e
|
||||
|
||||
|
||||
@@ -294,7 +300,7 @@ async def validate_database_schema(
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Schema validation failed: {e}", exc_info=True)
|
||||
logger.exception("Schema validation failed: %s", e)
|
||||
return {
|
||||
"valid": False,
|
||||
"missing_tables": [],
|
||||
@@ -304,6 +310,66 @@ async def validate_database_schema(
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Schema Migration
|
||||
# =============================================================================
|
||||
|
||||
|
||||
async def migrate_schema_if_needed(
|
||||
engine: Optional[AsyncEngine] = None
|
||||
) -> List[str]:
|
||||
"""Migrate database schema to current version if needed.
|
||||
|
||||
Handles adding missing columns to existing tables for backward
|
||||
compatibility with older database schemas.
|
||||
|
||||
Args:
|
||||
engine: Optional database engine (uses default if not provided)
|
||||
|
||||
Returns:
|
||||
List of migration operations performed
|
||||
"""
|
||||
if engine is None:
|
||||
engine = get_engine()
|
||||
|
||||
migrations_applied = []
|
||||
|
||||
try:
|
||||
async with engine.connect() as conn:
|
||||
# Get existing columns in system_settings table
|
||||
existing_columns = await conn.run_sync(
|
||||
lambda sync_conn: [
|
||||
col["name"]
|
||||
for col in inspect(sync_conn).get_columns("system_settings")
|
||||
]
|
||||
)
|
||||
|
||||
# Migration: Add legacy_key_cleanup_completed column if missing
|
||||
if "legacy_key_cleanup_completed" not in existing_columns:
|
||||
logger.info(
|
||||
"Migrating system_settings table: "
|
||||
"adding legacy_key_cleanup_completed column"
|
||||
)
|
||||
await conn.execute(
|
||||
text("""
|
||||
ALTER TABLE system_settings
|
||||
ADD COLUMN legacy_key_cleanup_completed BOOLEAN
|
||||
NOT NULL DEFAULT 0
|
||||
""")
|
||||
)
|
||||
migrations_applied.append("added legacy_key_cleanup_completed")
|
||||
logger.info(
|
||||
"Migration complete: added legacy_key_cleanup_completed column"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Schema migration failed: %s", e)
|
||||
# Don't raise - migration failures shouldn't block startup
|
||||
# The missing column will be handled gracefully by the application
|
||||
|
||||
return migrations_applied
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Schema Version Management
|
||||
# =============================================================================
|
||||
@@ -318,7 +384,7 @@ async def get_schema_version(engine: Optional[AsyncEngine] = None) -> str:
|
||||
engine: Optional database engine (uses default if not provided)
|
||||
|
||||
Returns:
|
||||
Schema version string (e.g., "1.0.0", "empty", "unknown")
|
||||
Schema version string (e.g., "1.0.1", "empty", "unknown")
|
||||
"""
|
||||
if engine is None:
|
||||
engine = get_engine()
|
||||
@@ -341,7 +407,7 @@ async def get_schema_version(engine: Optional[AsyncEngine] = None) -> str:
|
||||
return "unknown"
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get schema version: {e}")
|
||||
logger.error("Failed to get schema version: %s", e)
|
||||
return "error"
|
||||
|
||||
|
||||
@@ -408,7 +474,7 @@ async def seed_initial_data(engine: Optional[AsyncEngine] = None) -> None:
|
||||
logger.info("Data will be populated via normal application usage")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to seed initial data: {e}", exc_info=True)
|
||||
logger.exception("Failed to seed initial data: %s", e)
|
||||
raise
|
||||
|
||||
|
||||
@@ -483,12 +549,12 @@ async def check_database_health(
|
||||
f"(connectivity: {result['connectivity_ms']}ms)"
|
||||
)
|
||||
else:
|
||||
logger.warning(f"Database health issues: {result['issues']}")
|
||||
logger.warning("Database health issues: %s", result['issues'])
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Database health check failed: {e}")
|
||||
logger.error("Database health check failed: %s", e)
|
||||
return {
|
||||
"healthy": False,
|
||||
"accessible": False,
|
||||
@@ -546,13 +612,13 @@ async def create_database_backup(
|
||||
backup_path = backup_dir / f"aniworld_{timestamp}.db"
|
||||
|
||||
try:
|
||||
logger.info(f"Creating database backup: {backup_path}")
|
||||
logger.info("Creating database backup: %s", backup_path)
|
||||
shutil.copy2(db_path, backup_path)
|
||||
logger.info(f"Backup created successfully: {backup_path}")
|
||||
logger.info("Backup created successfully: %s", backup_path)
|
||||
return backup_path
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create backup: {e}", exc_info=True)
|
||||
logger.exception("Failed to create backup: %s", e)
|
||||
raise RuntimeError(f"Backup creation failed: {e}") from e
|
||||
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import List, Optional
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, String, Text, func
|
||||
from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, String, Text, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
||||
|
||||
from src.server.database.base import Base, TimestampMixin
|
||||
@@ -73,6 +73,68 @@ class AnimeSeries(Base, TimestampMixin):
|
||||
String(1000), nullable=False,
|
||||
doc="Filesystem folder name - METADATA ONLY, not for lookups"
|
||||
)
|
||||
year: Mapped[Optional[int]] = mapped_column(
|
||||
Integer, nullable=True,
|
||||
doc="Release year of the series"
|
||||
)
|
||||
|
||||
# NFO metadata tracking
|
||||
has_nfo: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="0",
|
||||
doc="Whether tvshow.nfo file exists for this series"
|
||||
)
|
||||
nfo_path: Mapped[Optional[str]] = mapped_column(
|
||||
String(1000), nullable=True,
|
||||
doc="Path to the tvshow.nfo metadata file"
|
||||
)
|
||||
nfo_created_at: Mapped[Optional[datetime]] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True,
|
||||
doc="Timestamp when NFO was first created"
|
||||
)
|
||||
nfo_updated_at: Mapped[Optional[datetime]] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True,
|
||||
doc="Timestamp when NFO was last updated"
|
||||
)
|
||||
# TMDB (The Movie Database) ID for series metadata
|
||||
tmdb_id: Mapped[Optional[int]] = mapped_column(
|
||||
Integer, nullable=True, index=True,
|
||||
doc="TMDB (The Movie Database) ID for series metadata"
|
||||
)
|
||||
tvdb_id: Mapped[Optional[int]] = mapped_column(
|
||||
Integer, nullable=True, index=True,
|
||||
doc="TVDB (TheTVDB) ID for series metadata"
|
||||
)
|
||||
|
||||
# Loading status fields for asynchronous data loading
|
||||
loading_status: Mapped[str] = mapped_column(
|
||||
String(50), nullable=False, default="completed", server_default="completed",
|
||||
doc="Loading status: pending, loading_episodes, loading_nfo, loading_logo, "
|
||||
"loading_images, completed, failed"
|
||||
)
|
||||
episodes_loaded: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True, server_default="1",
|
||||
doc="Whether episodes have been scanned and loaded"
|
||||
)
|
||||
logo_loaded: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="0",
|
||||
doc="Whether logo.png has been downloaded"
|
||||
)
|
||||
images_loaded: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="0",
|
||||
doc="Whether poster/fanart images have been downloaded"
|
||||
)
|
||||
loading_started_at: Mapped[Optional[datetime]] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True,
|
||||
doc="Timestamp when background loading started"
|
||||
)
|
||||
loading_completed_at: Mapped[Optional[datetime]] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True,
|
||||
doc="Timestamp when background loading completed"
|
||||
)
|
||||
loading_error: Mapped[Optional[str]] = mapped_column(
|
||||
String(1000), nullable=True,
|
||||
doc="Error message if loading failed"
|
||||
)
|
||||
|
||||
# Relationships
|
||||
episodes: Mapped[List["Episode"]] = relationship(
|
||||
@@ -123,7 +185,58 @@ class AnimeSeries(Base, TimestampMixin):
|
||||
return value.strip()
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<AnimeSeries(id={self.id}, key='{self.key}', name='{self.name}')>"
|
||||
return (
|
||||
f"<AnimeSeries(id={self.id}, key='{self.key}', "
|
||||
f"name='{self.name}')>"
|
||||
)
|
||||
|
||||
@property
|
||||
def episodeDict(self) -> dict[int, list[int]]:
|
||||
"""Build episode dictionary from episodes relationship or private cache.
|
||||
|
||||
Returns:
|
||||
Dictionary mapping season numbers to lists of episode numbers
|
||||
"""
|
||||
# Check for private cache first (set when loading from JSON without DB)
|
||||
if hasattr(self, '_episode_dict_cache') and self._episode_dict_cache is not None:
|
||||
return self._episode_dict_cache
|
||||
|
||||
episode_dict: dict[int, list[int]] = {}
|
||||
if self.episodes:
|
||||
for ep in self.episodes:
|
||||
season = ep.season or 1
|
||||
if season not in episode_dict:
|
||||
episode_dict[season] = []
|
||||
episode_dict[season].append(ep.episode_number or 0)
|
||||
return episode_dict
|
||||
|
||||
@property
|
||||
def name_with_year(self) -> str:
|
||||
"""Get series name with year appended if available.
|
||||
|
||||
Returns:
|
||||
Name in format "Name (Year)" if year is available, else just name
|
||||
"""
|
||||
if self.year:
|
||||
import re
|
||||
year_suffix = f" ({self.year})"
|
||||
clean_name = re.sub(r'(\s*\(\d{4}\))+\s*$', '', self.name or '').strip()
|
||||
return f"{clean_name}{year_suffix}"
|
||||
return self.name or ''
|
||||
|
||||
@property
|
||||
def sanitized_folder(self) -> str:
|
||||
"""Get filesystem-safe folder name from display name with year.
|
||||
|
||||
Returns:
|
||||
Sanitized folder name based on display name with year
|
||||
"""
|
||||
from src.server.utils.filesystem import sanitize_folder_name
|
||||
name_to_sanitize = self.name_with_year or self.folder or self.key
|
||||
try:
|
||||
return sanitize_folder_name(name_to_sanitize)
|
||||
except ValueError:
|
||||
return sanitize_folder_name(self.key)
|
||||
|
||||
|
||||
class Episode(Base, TimestampMixin):
|
||||
@@ -256,6 +369,7 @@ class DownloadQueueItem(Base, TimestampMixin):
|
||||
id: Primary key
|
||||
series_id: Foreign key to AnimeSeries
|
||||
episode_id: Foreign key to Episode
|
||||
status: Queue status (pending/downloading/completed/failed/permanently_failed)
|
||||
error_message: Error description if failed
|
||||
download_url: Provider download URL
|
||||
file_destination: Target file path
|
||||
@@ -287,6 +401,33 @@ class DownloadQueueItem(Base, TimestampMixin):
|
||||
index=True
|
||||
)
|
||||
|
||||
# Status column to track queue item state
|
||||
# Allows distinguishing pending items from permanently failed ones
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(50), nullable=False, default="pending",
|
||||
doc="Queue item status: pending, downloading, completed, failed, permanently_failed"
|
||||
)
|
||||
|
||||
# Retry count to track failed download attempts
|
||||
# Used to determine when to move item to permanently_failed
|
||||
retry_count: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=0,
|
||||
doc="Number of retry attempts for this download"
|
||||
)
|
||||
|
||||
# Unique constraint to prevent duplicate pending queue items per episode
|
||||
# An episode can only have one PENDING entry at a time
|
||||
# The status column allows failed items to remain in DB while new
|
||||
# pending items can be added (application-level dedup still required)
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"ix_download_queue_episode_status",
|
||||
"episode_id",
|
||||
"status",
|
||||
unique=True,
|
||||
),
|
||||
)
|
||||
|
||||
# Error handling
|
||||
error_message: Mapped[Optional[str]] = mapped_column(
|
||||
Text, nullable=True,
|
||||
@@ -483,3 +624,150 @@ class UserSession(Base, TimestampMixin):
|
||||
def revoke(self) -> None:
|
||||
"""Revoke this session."""
|
||||
self.is_active = False
|
||||
|
||||
|
||||
class UnresolvedFolder(Base, TimestampMixin):
|
||||
"""SQLAlchemy model for folders that couldn't be resolved during setup.
|
||||
|
||||
Tracks anime folders whose provider key couldn't be auto-resolved
|
||||
during the initial setup scan. Users can provide the correct key
|
||||
via the API to complete the series registration.
|
||||
|
||||
Attributes:
|
||||
id: Primary key
|
||||
folder_name: Original filesystem folder name
|
||||
title: Extracted title from folder name
|
||||
year: Extracted release year (optional)
|
||||
provider_key: User-provided provider key to resolve this folder
|
||||
search_attempts: Number of auto-search attempts made
|
||||
last_search_result: Cached search results (JSON string) for UI suggestions
|
||||
resolved_at: Timestamp when provider_key was provided
|
||||
created_at: Creation timestamp (from TimestampMixin)
|
||||
updated_at: Last update timestamp (from TimestampMixin)
|
||||
"""
|
||||
__tablename__ = "unresolved_folders"
|
||||
|
||||
# Primary key
|
||||
id: Mapped[int] = mapped_column(
|
||||
Integer, primary_key=True, autoincrement=True
|
||||
)
|
||||
|
||||
# Folder metadata
|
||||
folder_name: Mapped[str] = mapped_column(
|
||||
String(1000), unique=True, nullable=False, index=True,
|
||||
doc="Original filesystem folder name"
|
||||
)
|
||||
title: Mapped[str] = mapped_column(
|
||||
String(500), nullable=False,
|
||||
doc="Extracted title from folder name"
|
||||
)
|
||||
year: Mapped[Optional[int]] = mapped_column(
|
||||
Integer, nullable=True,
|
||||
doc="Extracted release year"
|
||||
)
|
||||
|
||||
# Resolution data
|
||||
provider_key: Mapped[Optional[str]] = mapped_column(
|
||||
String(255), nullable=True,
|
||||
doc="User-provided provider key to resolve this folder"
|
||||
)
|
||||
search_attempts: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=0, server_default="0",
|
||||
doc="Number of auto-search attempts made"
|
||||
)
|
||||
last_search_result: Mapped[Optional[str]] = mapped_column(
|
||||
Text, nullable=True,
|
||||
doc="Cached search results (JSON) for UI display"
|
||||
)
|
||||
resolved_at: Mapped[Optional[datetime]] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True,
|
||||
doc="Timestamp when this folder was resolved"
|
||||
)
|
||||
|
||||
@validates('folder_name')
|
||||
def validate_folder_name(self, key: str, value: str) -> str:
|
||||
"""Validate folder name is not empty."""
|
||||
if not value or not value.strip():
|
||||
raise ValueError("Folder name cannot be empty")
|
||||
if len(value) > 1000:
|
||||
raise ValueError("Folder name must be 1000 characters or less")
|
||||
return value.strip()
|
||||
|
||||
@validates('title')
|
||||
def validate_title(self, key: str, value: str) -> str:
|
||||
"""Validate title is not empty."""
|
||||
if not value or not value.strip():
|
||||
raise ValueError("Title cannot be empty")
|
||||
if len(value) > 500:
|
||||
raise ValueError("Title must be 500 characters or less")
|
||||
return value.strip()
|
||||
|
||||
@property
|
||||
def is_resolved(self) -> bool:
|
||||
"""Check if this folder has been resolved with a provider key."""
|
||||
return self.provider_key is not None and self.resolved_at is not None
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"<UnresolvedFolder(id={self.id}, "
|
||||
f"folder_name='{self.folder_name}', "
|
||||
f"title='{self.title}', "
|
||||
f"resolved={self.is_resolved})>"
|
||||
)
|
||||
|
||||
|
||||
class SystemSettings(Base, TimestampMixin):
|
||||
"""SQLAlchemy model for system-wide settings and state.
|
||||
|
||||
Stores application-level configuration and state flags that persist
|
||||
across restarts. Used to track initialization status and setup completion.
|
||||
|
||||
Attributes:
|
||||
id: Primary key (single row expected)
|
||||
initial_scan_completed: Whether the initial anime folder scan has been completed
|
||||
initial_nfo_scan_completed: Whether the initial NFO scan has been completed
|
||||
initial_media_scan_completed: Whether the initial media scan has been completed
|
||||
last_scan_timestamp: Timestamp of the last completed scan
|
||||
created_at: Creation timestamp (from TimestampMixin)
|
||||
updated_at: Last update timestamp (from TimestampMixin)
|
||||
"""
|
||||
__tablename__ = "system_settings"
|
||||
|
||||
# Primary key (only one row should exist)
|
||||
id: Mapped[int] = mapped_column(
|
||||
Integer, primary_key=True, autoincrement=True
|
||||
)
|
||||
|
||||
# Setup/initialization tracking
|
||||
initial_scan_completed: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="0",
|
||||
doc="Whether the initial anime folder scan has been completed"
|
||||
)
|
||||
initial_nfo_scan_completed: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="0",
|
||||
doc="Whether the initial NFO scan has been completed"
|
||||
)
|
||||
initial_media_scan_completed: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="0",
|
||||
doc="Whether the initial media scan has been completed"
|
||||
)
|
||||
migration_legacy_files_completed: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="0",
|
||||
doc="Whether legacy key/data file migration has been completed"
|
||||
)
|
||||
legacy_key_cleanup_completed: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="0",
|
||||
doc="Whether legacy key file cleanup has been completed"
|
||||
)
|
||||
last_scan_timestamp: Mapped[Optional[datetime]] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True,
|
||||
doc="Timestamp of the last completed scan"
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"<SystemSettings(id={self.id}, "
|
||||
f"initial_scan_completed={self.initial_scan_completed}, "
|
||||
f"initial_nfo_scan_completed={self.initial_nfo_scan_completed}, "
|
||||
f"initial_media_scan_completed={self.initial_media_scan_completed})>"
|
||||
)
|
||||
|
||||
@@ -26,7 +26,7 @@ import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import List, Optional
|
||||
|
||||
from sqlalchemy import delete, select, update
|
||||
from sqlalchemy import Integer, delete, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import Session, selectinload
|
||||
|
||||
@@ -34,6 +34,7 @@ from src.server.database.models import (
|
||||
AnimeSeries,
|
||||
DownloadQueueItem,
|
||||
Episode,
|
||||
UnresolvedFolder,
|
||||
UserSession,
|
||||
)
|
||||
|
||||
@@ -64,6 +65,16 @@ class AnimeSeriesService:
|
||||
name: str,
|
||||
site: str,
|
||||
folder: str,
|
||||
year: int | None = None,
|
||||
loading_status: str = "completed",
|
||||
episodes_loaded: bool = True,
|
||||
logo_loaded: bool = False,
|
||||
images_loaded: bool = False,
|
||||
loading_started_at: datetime | None = None,
|
||||
has_nfo: bool = False,
|
||||
nfo_path: str | None = None,
|
||||
nfo_created_at: datetime | None = None,
|
||||
nfo_updated_at: datetime | None = None,
|
||||
) -> AnimeSeries:
|
||||
"""Create a new anime series.
|
||||
|
||||
@@ -73,6 +84,16 @@ class AnimeSeriesService:
|
||||
name: Series name
|
||||
site: Provider site URL
|
||||
folder: Local filesystem path
|
||||
year: Release year (optional)
|
||||
loading_status: Initial loading status (default: "completed")
|
||||
episodes_loaded: Whether episodes are loaded (default: True for backward compat)
|
||||
logo_loaded: Whether logo is loaded (default: False)
|
||||
images_loaded: Whether images are loaded (default: False)
|
||||
loading_started_at: When loading started (optional)
|
||||
has_nfo: Whether tvshow.nfo exists (default: False)
|
||||
nfo_path: Path to tvshow.nfo file (optional)
|
||||
nfo_created_at: When NFO file was created (optional)
|
||||
nfo_updated_at: When NFO file was last updated (optional)
|
||||
|
||||
Returns:
|
||||
Created AnimeSeries instance
|
||||
@@ -85,11 +106,21 @@ class AnimeSeriesService:
|
||||
name=name,
|
||||
site=site,
|
||||
folder=folder,
|
||||
year=year,
|
||||
loading_status=loading_status,
|
||||
episodes_loaded=episodes_loaded,
|
||||
logo_loaded=logo_loaded,
|
||||
images_loaded=images_loaded,
|
||||
loading_started_at=loading_started_at,
|
||||
has_nfo=has_nfo,
|
||||
nfo_path=nfo_path,
|
||||
nfo_created_at=nfo_created_at,
|
||||
nfo_updated_at=nfo_updated_at,
|
||||
)
|
||||
db.add(series)
|
||||
await db.flush()
|
||||
await db.refresh(series)
|
||||
logger.info(f"Created anime series: {series.name} (key={series.key})")
|
||||
logger.info("Created anime series: %s (key=%s, year=%s)", series.name, series.key, year)
|
||||
return series
|
||||
|
||||
@staticmethod
|
||||
@@ -130,7 +161,47 @@ class AnimeSeriesService:
|
||||
select(AnimeSeries).where(AnimeSeries.key == key)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
@staticmethod
|
||||
def get_by_folder_sync(db: Session, folder: str) -> Optional[AnimeSeries]:
|
||||
"""Look up an anime series by its filesystem folder name (sync).
|
||||
|
||||
Intended as a fallback for ``SerieScanner`` when neither a ``key``
|
||||
file nor a ``data`` file exists on disk for a given folder.
|
||||
|
||||
Args:
|
||||
db: Synchronous database session (from ``get_sync_session``).
|
||||
folder: Filesystem folder name to match (e.g.
|
||||
``"Rooster Fighter (2026)"``).
|
||||
|
||||
Returns:
|
||||
``AnimeSeries`` instance or ``None`` if not found.
|
||||
"""
|
||||
result = db.execute(
|
||||
select(AnimeSeries).where(AnimeSeries.folder == folder)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
@staticmethod
|
||||
async def get_by_folder(db: AsyncSession, folder: str) -> Optional[AnimeSeries]:
|
||||
"""Look up an anime series by its filesystem folder name (async).
|
||||
|
||||
Intended as primary lookup for ``SerieScanner`` when scanning
|
||||
directories, replacing the legacy file-based lookups (key/data files).
|
||||
|
||||
Args:
|
||||
db: Async database session.
|
||||
folder: Filesystem folder name to match (e.g.
|
||||
``"Rooster Fighter (2026)"``).
|
||||
|
||||
Returns:
|
||||
``AnimeSeries`` instance or ``None`` if not found.
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(AnimeSeries).where(AnimeSeries.folder == folder)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
@staticmethod
|
||||
async def get_all(
|
||||
db: AsyncSession,
|
||||
@@ -187,7 +258,7 @@ class AnimeSeriesService:
|
||||
|
||||
await db.flush()
|
||||
await db.refresh(series)
|
||||
logger.info(f"Updated anime series: {series.name} (id={series_id})")
|
||||
logger.info("Updated anime series: %s (id=%s)", series.name, series_id)
|
||||
return series
|
||||
|
||||
@staticmethod
|
||||
@@ -208,7 +279,7 @@ class AnimeSeriesService:
|
||||
)
|
||||
deleted = result.rowcount > 0
|
||||
if deleted:
|
||||
logger.info(f"Deleted anime series with id={series_id}")
|
||||
logger.info("Deleted anime series with id=%s", series_id)
|
||||
return deleted
|
||||
|
||||
@staticmethod
|
||||
@@ -233,6 +304,201 @@ class AnimeSeriesService:
|
||||
.limit(limit)
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
@staticmethod
|
||||
async def get_series_with_missing_episodes(
|
||||
db: AsyncSession,
|
||||
limit: Optional[int] = None,
|
||||
offset: int = 0,
|
||||
) -> List[AnimeSeries]:
|
||||
"""Get anime series that currently have missing episodes.
|
||||
|
||||
Episodes in the database represent missing episodes (from episodeDict).
|
||||
This returns series that have at least one missing episode recorded in
|
||||
the database (is_downloaded=False).
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
limit: Optional limit for results
|
||||
offset: Offset for pagination
|
||||
|
||||
Returns:
|
||||
List of AnimeSeries that have missing episodes.
|
||||
"""
|
||||
# Subquery to find series IDs with at least one missing episode
|
||||
missing_series_ids = (
|
||||
select(Episode.series_id)
|
||||
.where(Episode.is_downloaded == False)
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
|
||||
query = (
|
||||
select(AnimeSeries)
|
||||
.where(AnimeSeries.id.in_(select(missing_series_ids.c.series_id)))
|
||||
.order_by(AnimeSeries.name)
|
||||
.offset(offset)
|
||||
)
|
||||
|
||||
if limit:
|
||||
query = query.limit(limit)
|
||||
|
||||
result = await db.execute(query)
|
||||
return list(result.scalars().all())
|
||||
|
||||
@staticmethod
|
||||
async def get_series_with_no_episodes(
|
||||
db: AsyncSession,
|
||||
limit: Optional[int] = None,
|
||||
offset: int = 0,
|
||||
) -> List[AnimeSeries]:
|
||||
"""Get anime series that have no downloaded episodes.
|
||||
|
||||
A series has "no episodes" if it has at least one missing episode
|
||||
(is_downloaded=False) and no downloaded episodes (is_downloaded=True).
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
limit: Optional limit for results
|
||||
offset: Offset for pagination
|
||||
|
||||
Returns:
|
||||
List of AnimeSeries where no episodes are downloaded.
|
||||
"""
|
||||
# Series with missing episodes
|
||||
missing_series_ids = (
|
||||
select(Episode.series_id)
|
||||
.where(Episode.is_downloaded == False)
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
|
||||
# Series with any downloaded episodes
|
||||
downloaded_series_ids = (
|
||||
select(Episode.series_id)
|
||||
.where(Episode.is_downloaded == True)
|
||||
.distinct()
|
||||
.subquery()
|
||||
)
|
||||
|
||||
query = (
|
||||
select(AnimeSeries)
|
||||
.where(AnimeSeries.id.in_(select(missing_series_ids.c.series_id)))
|
||||
.where(~AnimeSeries.id.in_(select(downloaded_series_ids.c.series_id)))
|
||||
.order_by(AnimeSeries.name)
|
||||
.offset(offset)
|
||||
)
|
||||
|
||||
if limit:
|
||||
query = query.limit(limit)
|
||||
|
||||
result = await db.execute(query)
|
||||
return list(result.scalars().all())
|
||||
|
||||
@staticmethod
|
||||
async def get_series_without_nfo(
|
||||
db: AsyncSession,
|
||||
limit: Optional[int] = None,
|
||||
offset: int = 0,
|
||||
) -> List[AnimeSeries]:
|
||||
"""Get anime series without NFO files.
|
||||
|
||||
Returns series where has_nfo is False.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
limit: Optional limit for results
|
||||
offset: Offset for pagination
|
||||
|
||||
Returns:
|
||||
List of AnimeSeries without NFO files
|
||||
"""
|
||||
query = (
|
||||
select(AnimeSeries)
|
||||
.where(AnimeSeries.has_nfo == False) # noqa: E712
|
||||
.order_by(AnimeSeries.name)
|
||||
.offset(offset)
|
||||
)
|
||||
|
||||
if limit:
|
||||
query = query.limit(limit)
|
||||
|
||||
result = await db.execute(query)
|
||||
return list(result.scalars().all())
|
||||
|
||||
@staticmethod
|
||||
async def count_all(db: AsyncSession) -> int:
|
||||
"""Count total number of anime series.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Total count of series
|
||||
"""
|
||||
from sqlalchemy import func
|
||||
|
||||
result = await db.execute(
|
||||
select(func.count()).select_from(AnimeSeries)
|
||||
)
|
||||
return result.scalar() or 0
|
||||
|
||||
@staticmethod
|
||||
async def count_with_nfo(db: AsyncSession) -> int:
|
||||
"""Count anime series with NFO files.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Count of series with has_nfo=True
|
||||
"""
|
||||
from sqlalchemy import func
|
||||
|
||||
result = await db.execute(
|
||||
select(func.count())
|
||||
.select_from(AnimeSeries)
|
||||
.where(AnimeSeries.has_nfo == True) # noqa: E712
|
||||
)
|
||||
return result.scalar() or 0
|
||||
|
||||
@staticmethod
|
||||
async def count_with_tmdb_id(db: AsyncSession) -> int:
|
||||
"""Count anime series with TMDB ID.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Count of series with tmdb_id set
|
||||
"""
|
||||
from sqlalchemy import func
|
||||
|
||||
result = await db.execute(
|
||||
select(func.count())
|
||||
.select_from(AnimeSeries)
|
||||
.where(AnimeSeries.tmdb_id.isnot(None))
|
||||
)
|
||||
return result.scalar() or 0
|
||||
|
||||
@staticmethod
|
||||
async def count_with_tvdb_id(db: AsyncSession) -> int:
|
||||
"""Count anime series with TVDB ID.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Count of series with tvdb_id set
|
||||
"""
|
||||
from sqlalchemy import func
|
||||
|
||||
result = await db.execute(
|
||||
select(func.count())
|
||||
.select_from(AnimeSeries)
|
||||
.where(AnimeSeries.tvdb_id.isnot(None))
|
||||
)
|
||||
return result.scalar() or 0
|
||||
|
||||
|
||||
# ============================================================================
|
||||
@@ -308,6 +574,7 @@ class EpisodeService:
|
||||
db: AsyncSession,
|
||||
series_id: int,
|
||||
season: Optional[int] = None,
|
||||
only_missing: bool = False,
|
||||
) -> List[Episode]:
|
||||
"""Get episodes for a series.
|
||||
|
||||
@@ -315,6 +582,9 @@ class EpisodeService:
|
||||
db: Database session
|
||||
series_id: Foreign key to AnimeSeries
|
||||
season: Optional season filter
|
||||
only_missing: If True, only return episodes where
|
||||
is_downloaded is False (i.e., missing episodes).
|
||||
Default False returns all episodes.
|
||||
|
||||
Returns:
|
||||
List of Episode instances
|
||||
@@ -324,6 +594,9 @@ class EpisodeService:
|
||||
if season is not None:
|
||||
query = query.where(Episode.season == season)
|
||||
|
||||
if only_missing:
|
||||
query = query.where(Episode.is_downloaded == False)
|
||||
|
||||
query = query.order_by(Episode.season, Episode.episode_number)
|
||||
result = await db.execute(query)
|
||||
return list(result.scalars().all())
|
||||
@@ -389,11 +662,11 @@ class EpisodeService:
|
||||
@staticmethod
|
||||
async def delete(db: AsyncSession, episode_id: int) -> bool:
|
||||
"""Delete episode.
|
||||
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
episode_id: Episode primary key
|
||||
|
||||
|
||||
Returns:
|
||||
True if deleted, False if not found
|
||||
"""
|
||||
@@ -402,6 +675,33 @@ class EpisodeService:
|
||||
)
|
||||
return result.rowcount > 0
|
||||
|
||||
@staticmethod
|
||||
async def delete_by_series(
|
||||
db: AsyncSession,
|
||||
series_id: int,
|
||||
season: int,
|
||||
episode_number: int,
|
||||
) -> bool:
|
||||
"""Delete episode by series ID, season, and episode number.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
series_id: Foreign key to AnimeSeries
|
||||
season: Season number
|
||||
episode_number: Episode number within season
|
||||
|
||||
Returns:
|
||||
True if deleted, False if not found
|
||||
"""
|
||||
result = await db.execute(
|
||||
delete(Episode).where(
|
||||
Episode.series_id == series_id,
|
||||
Episode.season == season,
|
||||
Episode.episode_number == episode_number,
|
||||
)
|
||||
)
|
||||
return result.rowcount > 0
|
||||
|
||||
@staticmethod
|
||||
async def delete_by_series_and_episode(
|
||||
db: AsyncSession,
|
||||
@@ -488,7 +788,7 @@ class EpisodeService:
|
||||
updated_count += 1
|
||||
|
||||
await db.flush()
|
||||
logger.info(f"Bulk marked {updated_count} episodes as downloaded")
|
||||
logger.info("Bulk marked %s episodes as downloaded", updated_count)
|
||||
|
||||
return updated_count
|
||||
|
||||
@@ -515,6 +815,8 @@ class DownloadQueueService:
|
||||
episode_id: int,
|
||||
download_url: Optional[str] = None,
|
||||
file_destination: Optional[str] = None,
|
||||
status: str = "pending",
|
||||
retry_count: int = 0,
|
||||
) -> DownloadQueueItem:
|
||||
"""Add item to download queue.
|
||||
|
||||
@@ -524,6 +826,8 @@ class DownloadQueueService:
|
||||
episode_id: Foreign key to Episode
|
||||
download_url: Optional provider download URL
|
||||
file_destination: Optional target file path
|
||||
status: Queue item status (default: "pending")
|
||||
retry_count: Number of retry attempts (default: 0)
|
||||
|
||||
Returns:
|
||||
Created DownloadQueueItem instance
|
||||
@@ -533,13 +837,15 @@ class DownloadQueueService:
|
||||
episode_id=episode_id,
|
||||
download_url=download_url,
|
||||
file_destination=file_destination,
|
||||
status=status,
|
||||
retry_count=retry_count,
|
||||
)
|
||||
db.add(item)
|
||||
await db.flush()
|
||||
await db.refresh(item)
|
||||
logger.info(
|
||||
f"Added to download queue: episode_id={episode_id} "
|
||||
f"for series_id={series_id}"
|
||||
f"for series_id={series_id}, status={status}"
|
||||
)
|
||||
return item
|
||||
|
||||
@@ -566,21 +872,24 @@ class DownloadQueueService:
|
||||
async def get_by_episode(
|
||||
db: AsyncSession,
|
||||
episode_id: int,
|
||||
status_filter: Optional[str] = None,
|
||||
) -> Optional[DownloadQueueItem]:
|
||||
"""Get download queue item by episode ID.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
episode_id: Foreign key to Episode
|
||||
status_filter: Optional status to filter by (e.g., "pending")
|
||||
|
||||
Returns:
|
||||
DownloadQueueItem instance or None if not found
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(DownloadQueueItem).where(
|
||||
DownloadQueueItem.episode_id == episode_id
|
||||
)
|
||||
query = select(DownloadQueueItem).where(
|
||||
DownloadQueueItem.episode_id == episode_id
|
||||
)
|
||||
if status_filter:
|
||||
query = query.where(DownloadQueueItem.status == status_filter)
|
||||
result = await db.execute(query)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
@staticmethod
|
||||
@@ -592,7 +901,7 @@ class DownloadQueueService:
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
with_series: Whether to eagerly load series data
|
||||
with_series: Whether to eagerly load series and episode data
|
||||
|
||||
Returns:
|
||||
List of all DownloadQueueItem instances
|
||||
@@ -600,7 +909,11 @@ class DownloadQueueService:
|
||||
query = select(DownloadQueueItem)
|
||||
|
||||
if with_series:
|
||||
query = query.options(selectinload(DownloadQueueItem.series))
|
||||
# Eagerly load both series and episode relationships
|
||||
query = query.options(
|
||||
selectinload(DownloadQueueItem.series),
|
||||
selectinload(DownloadQueueItem.episode)
|
||||
)
|
||||
|
||||
query = query.order_by(
|
||||
DownloadQueueItem.created_at.asc(),
|
||||
@@ -633,7 +946,96 @@ class DownloadQueueService:
|
||||
|
||||
await db.flush()
|
||||
await db.refresh(item)
|
||||
logger.debug(f"Set error on download queue item {item_id}")
|
||||
logger.debug("Set error on download queue item %s", item_id)
|
||||
return item
|
||||
|
||||
@staticmethod
|
||||
async def set_status(
|
||||
db: AsyncSession,
|
||||
item_id: int,
|
||||
status: str,
|
||||
) -> Optional[DownloadQueueItem]:
|
||||
"""Set status on download queue item.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
item_id: Item primary key
|
||||
status: New status value
|
||||
|
||||
Returns:
|
||||
Updated DownloadQueueItem instance or None if not found
|
||||
"""
|
||||
item = await DownloadQueueService.get_by_id(db, item_id)
|
||||
if not item:
|
||||
return None
|
||||
|
||||
item.status = status
|
||||
|
||||
await db.flush()
|
||||
await db.refresh(item)
|
||||
logger.debug("Set status on download queue item %s to %s", item_id, status)
|
||||
return item
|
||||
|
||||
@staticmethod
|
||||
async def increment_retry_count(
|
||||
db: AsyncSession,
|
||||
item_id: int,
|
||||
) -> Optional[DownloadQueueItem]:
|
||||
"""Increment retry count on download queue item.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
item_id: Item primary key
|
||||
|
||||
Returns:
|
||||
Updated DownloadQueueItem instance or None if not found
|
||||
"""
|
||||
item = await DownloadQueueService.get_by_id(db, item_id)
|
||||
if not item:
|
||||
return None
|
||||
|
||||
item.retry_count += 1
|
||||
|
||||
await db.flush()
|
||||
await db.refresh(item)
|
||||
logger.debug(
|
||||
"Incremented retry count on download queue item %s to %s",
|
||||
item_id, item.retry_count
|
||||
)
|
||||
return item
|
||||
|
||||
@staticmethod
|
||||
async def set_status_and_error(
|
||||
db: AsyncSession,
|
||||
item_id: int,
|
||||
status: str,
|
||||
error_message: Optional[str] = None,
|
||||
) -> Optional[DownloadQueueItem]:
|
||||
"""Set status and error message on download queue item atomically.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
item_id: Item primary key
|
||||
status: New status value
|
||||
error_message: Optional error description
|
||||
|
||||
Returns:
|
||||
Updated DownloadQueueItem instance or None if not found
|
||||
"""
|
||||
item = await DownloadQueueService.get_by_id(db, item_id)
|
||||
if not item:
|
||||
return None
|
||||
|
||||
item.status = status
|
||||
if error_message is not None:
|
||||
item.error_message = error_message
|
||||
|
||||
await db.flush()
|
||||
await db.refresh(item)
|
||||
logger.debug(
|
||||
"Set status=%s on download queue item %s, error=%s",
|
||||
status, item_id, error_message
|
||||
)
|
||||
return item
|
||||
|
||||
@staticmethod
|
||||
@@ -652,7 +1054,7 @@ class DownloadQueueService:
|
||||
)
|
||||
deleted = result.rowcount > 0
|
||||
if deleted:
|
||||
logger.info(f"Deleted download queue item with id={item_id}")
|
||||
logger.info("Deleted download queue item with id=%s", item_id)
|
||||
return deleted
|
||||
|
||||
@staticmethod
|
||||
@@ -714,7 +1116,7 @@ class DownloadQueueService:
|
||||
)
|
||||
|
||||
count = result.rowcount
|
||||
logger.info(f"Bulk deleted {count} download queue items")
|
||||
logger.info("Bulk deleted %s download queue items", count)
|
||||
|
||||
return count
|
||||
|
||||
@@ -735,7 +1137,7 @@ class DownloadQueueService:
|
||||
"""
|
||||
result = await db.execute(delete(DownloadQueueItem))
|
||||
count = result.rowcount
|
||||
logger.info(f"Cleared all {count} download queue items")
|
||||
logger.info("Cleared all %s download queue items", count)
|
||||
return count
|
||||
|
||||
|
||||
@@ -789,7 +1191,7 @@ class UserSessionService:
|
||||
db.add(session)
|
||||
await db.flush()
|
||||
await db.refresh(session)
|
||||
logger.info(f"Created user session: {session_id}")
|
||||
logger.info("Created user session: %s", session_id)
|
||||
return session
|
||||
|
||||
@staticmethod
|
||||
@@ -876,7 +1278,7 @@ class UserSessionService:
|
||||
|
||||
session.revoke()
|
||||
await db.flush()
|
||||
logger.info(f"Revoked user session: {session_id}")
|
||||
logger.info("Revoked user session: %s", session_id)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
@@ -898,7 +1300,7 @@ class UserSessionService:
|
||||
)
|
||||
)
|
||||
count = result.rowcount
|
||||
logger.info(f"Cleaned up {count} expired sessions")
|
||||
logger.info("Cleaned up %s expired sessions", count)
|
||||
return count
|
||||
|
||||
@staticmethod
|
||||
@@ -963,3 +1365,176 @@ class UserSessionService:
|
||||
|
||||
return new_session
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Unresolved Folder Service
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class UnresolvedFolderService:
|
||||
"""Service for tracking and resolving folders that couldn't be auto-resolved.
|
||||
|
||||
During initial setup, some folders may not resolve to a provider key
|
||||
(no search match or multiple ambiguous matches). These are tracked as
|
||||
UnresolvedFolder records and can later be resolved by the user providing
|
||||
the correct provider key.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
async def create(
|
||||
db: AsyncSession,
|
||||
folder_name: str,
|
||||
title: str,
|
||||
year: int | None = None,
|
||||
search_attempts: int = 1,
|
||||
last_search_result: str | None = None,
|
||||
) -> UnresolvedFolder:
|
||||
"""Create a new unresolved folder tracking record.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
folder_name: Original filesystem folder name
|
||||
title: Extracted title from folder name
|
||||
year: Extracted release year (optional)
|
||||
search_attempts: Number of search attempts made (default: 1)
|
||||
last_search_result: JSON string of search results for UI (optional)
|
||||
|
||||
Returns:
|
||||
Created UnresolvedFolder instance
|
||||
"""
|
||||
folder = UnresolvedFolder(
|
||||
folder_name=folder_name,
|
||||
title=title,
|
||||
year=year,
|
||||
search_attempts=search_attempts,
|
||||
last_search_result=last_search_result,
|
||||
)
|
||||
db.add(folder)
|
||||
await db.flush()
|
||||
await db.refresh(folder)
|
||||
logger.info(
|
||||
"Created unresolved folder tracking: %s (title=%s, year=%s)",
|
||||
folder_name, title, year
|
||||
)
|
||||
return folder
|
||||
|
||||
@staticmethod
|
||||
async def get_by_folder_name(
|
||||
db: AsyncSession,
|
||||
folder_name: str,
|
||||
) -> Optional[UnresolvedFolder]:
|
||||
"""Get unresolved folder by folder name.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
folder_name: Filesystem folder name to look up
|
||||
|
||||
Returns:
|
||||
UnresolvedFolder instance or None if not found
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(UnresolvedFolder).where(
|
||||
UnresolvedFolder.folder_name == folder_name
|
||||
)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
@staticmethod
|
||||
async def get_all_unresolved(
|
||||
db: AsyncSession,
|
||||
) -> list[UnresolvedFolder]:
|
||||
"""Get all unresolved folders that haven't been resolved yet.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
List of unresolved UnresolvedFolder instances
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(UnresolvedFolder)
|
||||
.where(UnresolvedFolder.provider_key.is_(None))
|
||||
.order_by(UnresolvedFolder.created_at)
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
@staticmethod
|
||||
async def resolve(
|
||||
db: AsyncSession,
|
||||
folder_name: str,
|
||||
provider_key: str,
|
||||
) -> Optional[UnresolvedFolder]:
|
||||
"""Mark an unresolved folder as resolved with the given provider key.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
folder_name: Filesystem folder name to resolve
|
||||
provider_key: Provider key to associate with this folder
|
||||
|
||||
Returns:
|
||||
Updated UnresolvedFolder instance or None if not found
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
folder = await UnresolvedFolderService.get_by_folder_name(db, folder_name)
|
||||
if not folder:
|
||||
return None
|
||||
|
||||
folder.provider_key = provider_key
|
||||
folder.resolved_at = datetime.now(timezone.utc)
|
||||
await db.flush()
|
||||
await db.refresh(folder)
|
||||
logger.info(
|
||||
"Resolved unresolved folder: %s -> key=%s",
|
||||
folder_name, provider_key
|
||||
)
|
||||
return folder
|
||||
|
||||
@staticmethod
|
||||
async def delete(
|
||||
db: AsyncSession,
|
||||
folder_name: str,
|
||||
) -> bool:
|
||||
"""Delete an unresolved folder record (e.g., after manual add).
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
folder_name: Filesystem folder name to delete
|
||||
|
||||
Returns:
|
||||
True if deleted, False if not found
|
||||
"""
|
||||
folder = await UnresolvedFolderService.get_by_folder_name(db, folder_name)
|
||||
if not folder:
|
||||
return False
|
||||
|
||||
await db.delete(folder)
|
||||
await db.flush()
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
async def update_search_result(
|
||||
db: AsyncSession,
|
||||
folder_name: str,
|
||||
search_result: str,
|
||||
) -> Optional[UnresolvedFolder]:
|
||||
"""Update the cached search result for an unresolved folder.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
folder_name: Filesystem folder name to update
|
||||
search_result: JSON string of search results
|
||||
|
||||
Returns:
|
||||
Updated UnresolvedFolder instance or None if not found
|
||||
"""
|
||||
folder = await UnresolvedFolderService.get_by_folder_name(db, folder_name)
|
||||
if not folder:
|
||||
return None
|
||||
|
||||
folder.search_attempts += 1
|
||||
folder.last_search_result = search_result
|
||||
await db.flush()
|
||||
await db.refresh(folder)
|
||||
return folder
|
||||
|
||||
|
||||
221
src/server/database/system_settings_service.py
Normal file
221
src/server/database/system_settings_service.py
Normal file
@@ -0,0 +1,221 @@
|
||||
"""System settings service for managing application-level configuration.
|
||||
|
||||
This module provides services for managing system-wide settings and state,
|
||||
including tracking initial setup completion status.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
import structlog
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.server.database.models import SystemSettings
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
class SystemSettingsService:
|
||||
"""Service for managing system settings."""
|
||||
|
||||
@staticmethod
|
||||
async def get_or_create(db: AsyncSession) -> SystemSettings:
|
||||
"""Get the system settings record, creating it if it doesn't exist.
|
||||
|
||||
Only one system settings record should exist in the database.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
SystemSettings instance
|
||||
"""
|
||||
# Try to get existing settings
|
||||
stmt = select(SystemSettings).limit(1)
|
||||
result = await db.execute(stmt)
|
||||
settings = result.scalar_one_or_none()
|
||||
|
||||
if settings is None:
|
||||
# Create new settings with defaults
|
||||
settings = SystemSettings(
|
||||
initial_scan_completed=False,
|
||||
initial_nfo_scan_completed=False,
|
||||
initial_media_scan_completed=False,
|
||||
)
|
||||
db.add(settings)
|
||||
await db.commit()
|
||||
await db.refresh(settings)
|
||||
logger.info("Created new system settings record")
|
||||
|
||||
return settings
|
||||
|
||||
@staticmethod
|
||||
async def is_initial_scan_completed(db: AsyncSession) -> bool:
|
||||
"""Check if the initial anime folder scan has been completed.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
True if initial scan is completed, False otherwise
|
||||
"""
|
||||
settings = await SystemSettingsService.get_or_create(db)
|
||||
return settings.initial_scan_completed
|
||||
|
||||
@staticmethod
|
||||
async def mark_initial_scan_completed(
|
||||
db: AsyncSession,
|
||||
timestamp: Optional[datetime] = None
|
||||
) -> None:
|
||||
"""Mark the initial anime folder scan as completed.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
timestamp: Optional timestamp to set, defaults to current time
|
||||
"""
|
||||
settings = await SystemSettingsService.get_or_create(db)
|
||||
settings.initial_scan_completed = True
|
||||
settings.last_scan_timestamp = timestamp or datetime.now(timezone.utc)
|
||||
await db.commit()
|
||||
logger.info("Marked initial scan as completed")
|
||||
|
||||
@staticmethod
|
||||
async def is_initial_nfo_scan_completed(db: AsyncSession) -> bool:
|
||||
"""Check if the initial NFO scan has been completed.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
True if initial NFO scan is completed, False otherwise
|
||||
"""
|
||||
settings = await SystemSettingsService.get_or_create(db)
|
||||
return settings.initial_nfo_scan_completed
|
||||
|
||||
@staticmethod
|
||||
async def mark_initial_nfo_scan_completed(
|
||||
db: AsyncSession,
|
||||
timestamp: Optional[datetime] = None
|
||||
) -> None:
|
||||
"""Mark the initial NFO scan as completed.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
timestamp: Optional timestamp to set, defaults to current time
|
||||
"""
|
||||
settings = await SystemSettingsService.get_or_create(db)
|
||||
settings.initial_nfo_scan_completed = True
|
||||
if timestamp:
|
||||
settings.last_scan_timestamp = timestamp
|
||||
await db.commit()
|
||||
logger.info("Marked initial NFO scan as completed")
|
||||
|
||||
@staticmethod
|
||||
async def is_initial_media_scan_completed(db: AsyncSession) -> bool:
|
||||
"""Check if the initial media scan has been completed.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
True if initial media scan is completed, False otherwise
|
||||
"""
|
||||
settings = await SystemSettingsService.get_or_create(db)
|
||||
return settings.initial_media_scan_completed
|
||||
|
||||
@staticmethod
|
||||
async def is_migration_legacy_files_completed(db: AsyncSession) -> bool:
|
||||
"""Check if legacy key/data file migration has been completed.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
True if legacy migration is completed, False otherwise
|
||||
"""
|
||||
settings = await SystemSettingsService.get_or_create(db)
|
||||
return settings.migration_legacy_files_completed
|
||||
|
||||
@staticmethod
|
||||
async def mark_migration_legacy_files_completed(
|
||||
db: AsyncSession,
|
||||
timestamp: Optional[datetime] = None
|
||||
) -> None:
|
||||
"""Mark the legacy key/data file migration as completed.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
timestamp: Optional timestamp to set, defaults to current time
|
||||
"""
|
||||
settings = await SystemSettingsService.get_or_create(db)
|
||||
settings.migration_legacy_files_completed = True
|
||||
settings.last_scan_timestamp = timestamp or datetime.now(timezone.utc)
|
||||
await db.commit()
|
||||
logger.info("Marked legacy files migration as completed")
|
||||
|
||||
@staticmethod
|
||||
async def is_legacy_key_cleanup_completed(db: AsyncSession) -> bool:
|
||||
"""Check if legacy key file cleanup has been completed.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
True if cleanup is completed, False otherwise
|
||||
"""
|
||||
settings = await SystemSettingsService.get_or_create(db)
|
||||
return settings.legacy_key_cleanup_completed
|
||||
|
||||
@staticmethod
|
||||
async def mark_legacy_key_cleanup_completed(
|
||||
db: AsyncSession,
|
||||
timestamp: Optional[datetime] = None
|
||||
) -> None:
|
||||
"""Mark the legacy key file cleanup as completed.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
timestamp: Optional timestamp to set, defaults to current time
|
||||
"""
|
||||
settings = await SystemSettingsService.get_or_create(db)
|
||||
settings.legacy_key_cleanup_completed = True
|
||||
settings.last_scan_timestamp = timestamp or datetime.now(timezone.utc)
|
||||
await db.commit()
|
||||
logger.info("Marked legacy key file cleanup as completed")
|
||||
|
||||
@staticmethod
|
||||
async def mark_initial_media_scan_completed(
|
||||
db: AsyncSession,
|
||||
timestamp: Optional[datetime] = None
|
||||
) -> None:
|
||||
"""Mark the initial media scan as completed.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
timestamp: Optional timestamp to set, defaults to current time
|
||||
"""
|
||||
settings = await SystemSettingsService.get_or_create(db)
|
||||
settings.initial_media_scan_completed = True
|
||||
if timestamp:
|
||||
settings.last_scan_timestamp = timestamp
|
||||
await db.commit()
|
||||
logger.info("Marked initial media scan as completed")
|
||||
|
||||
@staticmethod
|
||||
async def reset_all_scans(db: AsyncSession) -> None:
|
||||
"""Reset all scan completion flags (for testing or re-setup).
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
"""
|
||||
settings = await SystemSettingsService.get_or_create(db)
|
||||
settings.initial_scan_completed = False
|
||||
settings.initial_nfo_scan_completed = False
|
||||
settings.initial_media_scan_completed = False
|
||||
settings.migration_legacy_files_completed = False
|
||||
settings.legacy_key_cleanup_completed = False
|
||||
settings.last_scan_timestamp = None
|
||||
await db.commit()
|
||||
logger.info("Reset all scan completion flags")
|
||||
335
src/server/entities/nfo_models.py
Normal file
335
src/server/entities/nfo_models.py
Normal file
@@ -0,0 +1,335 @@
|
||||
"""Pydantic models for NFO metadata based on Kodi/XBMC standard.
|
||||
|
||||
This module provides data models for tvshow.nfo files that are compatible
|
||||
with media center applications like Kodi, Plex, and Jellyfin.
|
||||
|
||||
Example:
|
||||
>>> nfo = TVShowNFO(
|
||||
... title="Attack on Titan",
|
||||
... year=2013,
|
||||
... tmdbid=1429
|
||||
... )
|
||||
>>> nfo.premiered = "2013-04-07"
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
|
||||
from pydantic import BaseModel, Field, HttpUrl, field_validator
|
||||
|
||||
|
||||
class RatingInfo(BaseModel):
|
||||
"""Rating information from various sources.
|
||||
|
||||
Attributes:
|
||||
name: Source of the rating (e.g., 'themoviedb', 'imdb')
|
||||
value: Rating value (typically 0-10)
|
||||
votes: Number of votes
|
||||
max_rating: Maximum possible rating (default: 10)
|
||||
default: Whether this is the default rating to display
|
||||
"""
|
||||
|
||||
name: str = Field(..., description="Rating source name")
|
||||
value: float = Field(..., ge=0, description="Rating value")
|
||||
votes: Optional[int] = Field(None, ge=0, description="Number of votes")
|
||||
max_rating: int = Field(10, ge=1, description="Maximum rating value")
|
||||
default: bool = Field(False, description="Is this the default rating")
|
||||
|
||||
@field_validator('value')
|
||||
@classmethod
|
||||
def validate_value(cls, v: float, info) -> float:
|
||||
"""Ensure rating value doesn't exceed max_rating."""
|
||||
# Note: max_rating is not available yet during validation,
|
||||
# so we use a reasonable default check
|
||||
if v > 10:
|
||||
raise ValueError("Rating value cannot exceed 10")
|
||||
return v
|
||||
|
||||
|
||||
class ActorInfo(BaseModel):
|
||||
"""Actor/cast member information.
|
||||
|
||||
Attributes:
|
||||
name: Actor's name
|
||||
role: Character name/role
|
||||
thumb: URL to actor's photo
|
||||
profile: URL to actor's profile page
|
||||
tmdbid: TMDB ID for the actor
|
||||
"""
|
||||
|
||||
name: str = Field(..., description="Actor's name")
|
||||
role: Optional[str] = Field(None, description="Character role")
|
||||
thumb: Optional[HttpUrl] = Field(None, description="Actor photo URL")
|
||||
profile: Optional[HttpUrl] = Field(None, description="Actor profile URL")
|
||||
tmdbid: Optional[int] = Field(None, description="TMDB actor ID")
|
||||
|
||||
|
||||
class ImageInfo(BaseModel):
|
||||
"""Image information for posters, fanart, and logos.
|
||||
|
||||
Attributes:
|
||||
url: URL to the image
|
||||
aspect: Image aspect/type (e.g., 'poster', 'clearlogo', 'logo')
|
||||
season: Season number for season-specific images
|
||||
type: Image type (e.g., 'season')
|
||||
"""
|
||||
|
||||
url: HttpUrl = Field(..., description="Image URL")
|
||||
aspect: Optional[str] = Field(
|
||||
None,
|
||||
description="Image aspect (poster, clearlogo, logo)"
|
||||
)
|
||||
season: Optional[int] = Field(None, ge=-1, description="Season number")
|
||||
type: Optional[str] = Field(None, description="Image type")
|
||||
|
||||
|
||||
class NamedSeason(BaseModel):
|
||||
"""Named season information.
|
||||
|
||||
Attributes:
|
||||
number: Season number
|
||||
name: Season name/title
|
||||
"""
|
||||
|
||||
number: int = Field(..., ge=0, description="Season number")
|
||||
name: str = Field(..., description="Season name")
|
||||
|
||||
|
||||
class UniqueID(BaseModel):
|
||||
"""Unique identifier from various sources.
|
||||
|
||||
Attributes:
|
||||
type: ID source type (tmdb, imdb, tvdb)
|
||||
value: The ID value
|
||||
default: Whether this is the default ID
|
||||
"""
|
||||
|
||||
type: str = Field(..., description="ID type (tmdb, imdb, tvdb)")
|
||||
value: str = Field(..., description="ID value")
|
||||
default: bool = Field(False, description="Is default ID")
|
||||
|
||||
|
||||
class TVShowNFO(BaseModel):
|
||||
"""Main tvshow.nfo structure following Kodi/XBMC standard.
|
||||
|
||||
This model represents the complete metadata for a TV show that can be
|
||||
serialized to XML for use with media center applications.
|
||||
|
||||
Attributes:
|
||||
title: Main title of the show
|
||||
originaltitle: Original title (e.g., in original language)
|
||||
showtitle: Show title (often same as title)
|
||||
sorttitle: Title used for sorting
|
||||
year: Release year
|
||||
plot: Full plot description
|
||||
outline: Short plot summary
|
||||
tagline: Show tagline/slogan
|
||||
runtime: Episode runtime in minutes
|
||||
mpaa: Content rating (e.g., TV-14, TV-MA)
|
||||
certification: Additional certification info
|
||||
premiered: Premiere date (YYYY-MM-DD format)
|
||||
status: Show status (e.g., 'Continuing', 'Ended')
|
||||
studio: List of production studios
|
||||
genre: List of genres
|
||||
country: List of countries
|
||||
tag: List of tags/keywords
|
||||
ratings: List of ratings from various sources
|
||||
userrating: User's personal rating
|
||||
watched: Whether the show has been watched
|
||||
playcount: Number of times watched
|
||||
tmdbid: TMDB ID
|
||||
imdbid: IMDB ID
|
||||
tvdbid: TVDB ID
|
||||
uniqueid: List of unique IDs
|
||||
thumb: List of thumbnail/poster images
|
||||
fanart: List of fanart/backdrop images
|
||||
actors: List of cast members
|
||||
namedseason: List of named seasons
|
||||
trailer: Trailer URL
|
||||
dateadded: Date when added to library
|
||||
"""
|
||||
|
||||
# Required fields
|
||||
title: str = Field(..., description="Show title", min_length=1)
|
||||
|
||||
# Basic information (optional)
|
||||
originaltitle: Optional[str] = Field(None, description="Original title")
|
||||
showtitle: Optional[str] = Field(None, description="Show title")
|
||||
sorttitle: Optional[str] = Field(None, description="Sort title")
|
||||
year: Optional[int] = Field(
|
||||
None,
|
||||
ge=1900,
|
||||
le=2100,
|
||||
description="Release year"
|
||||
)
|
||||
|
||||
# Plot and description
|
||||
plot: Optional[str] = Field(None, description="Full plot description")
|
||||
outline: Optional[str] = Field(None, description="Short plot summary")
|
||||
tagline: Optional[str] = Field(None, description="Show tagline")
|
||||
|
||||
# Technical details
|
||||
runtime: Optional[int] = Field(
|
||||
None,
|
||||
ge=0,
|
||||
description="Episode runtime in minutes"
|
||||
)
|
||||
mpaa: Optional[str] = Field(None, description="Content rating")
|
||||
fsk: Optional[str] = Field(
|
||||
None,
|
||||
description="German FSK rating (e.g., 'FSK 12', 'FSK 16')"
|
||||
)
|
||||
certification: Optional[str] = Field(
|
||||
None,
|
||||
description="Certification info"
|
||||
)
|
||||
|
||||
# Status and dates
|
||||
premiered: Optional[str] = Field(
|
||||
None,
|
||||
description="Premiere date (YYYY-MM-DD)"
|
||||
)
|
||||
status: Optional[str] = Field(None, description="Show status")
|
||||
dateadded: Optional[str] = Field(
|
||||
None,
|
||||
description="Date added to library"
|
||||
)
|
||||
|
||||
# Multi-value fields
|
||||
studio: List[str] = Field(
|
||||
default_factory=list,
|
||||
description="Production studios"
|
||||
)
|
||||
genre: List[str] = Field(
|
||||
default_factory=list,
|
||||
description="Genres"
|
||||
)
|
||||
country: List[str] = Field(
|
||||
default_factory=list,
|
||||
description="Countries"
|
||||
)
|
||||
tag: List[str] = Field(
|
||||
default_factory=list,
|
||||
description="Tags/keywords"
|
||||
)
|
||||
|
||||
# IDs
|
||||
tmdbid: Optional[int] = Field(None, description="TMDB ID")
|
||||
imdbid: Optional[str] = Field(None, description="IMDB ID")
|
||||
tvdbid: Optional[int] = Field(None, description="TVDB ID")
|
||||
uniqueid: List[UniqueID] = Field(
|
||||
default_factory=list,
|
||||
description="Unique IDs"
|
||||
)
|
||||
|
||||
# Ratings and viewing info
|
||||
ratings: List[RatingInfo] = Field(
|
||||
default_factory=list,
|
||||
description="Ratings"
|
||||
)
|
||||
userrating: Optional[float] = Field(
|
||||
None,
|
||||
ge=0,
|
||||
le=10,
|
||||
description="User rating"
|
||||
)
|
||||
watched: bool = Field(False, description="Watched status")
|
||||
playcount: Optional[int] = Field(
|
||||
None,
|
||||
ge=0,
|
||||
description="Play count"
|
||||
)
|
||||
|
||||
# Media
|
||||
thumb: List[ImageInfo] = Field(
|
||||
default_factory=list,
|
||||
description="Thumbnail images"
|
||||
)
|
||||
fanart: List[ImageInfo] = Field(
|
||||
default_factory=list,
|
||||
description="Fanart images"
|
||||
)
|
||||
|
||||
# Cast and crew
|
||||
actors: List[ActorInfo] = Field(
|
||||
default_factory=list,
|
||||
description="Cast members"
|
||||
)
|
||||
|
||||
# Seasons
|
||||
namedseason: List[NamedSeason] = Field(
|
||||
default_factory=list,
|
||||
description="Named seasons"
|
||||
)
|
||||
|
||||
# Additional
|
||||
trailer: Optional[HttpUrl] = Field(None, description="Trailer URL")
|
||||
|
||||
@field_validator('premiered')
|
||||
@classmethod
|
||||
def validate_premiered_date(cls, v: Optional[str]) -> Optional[str]:
|
||||
"""Validate premiered date format (YYYY-MM-DD)."""
|
||||
if v is None:
|
||||
return v
|
||||
|
||||
# Check format strictly: YYYY-MM-DD
|
||||
if len(v) != 10 or v[4] != '-' or v[7] != '-':
|
||||
raise ValueError(
|
||||
"Premiered date must be in YYYY-MM-DD format"
|
||||
)
|
||||
|
||||
try:
|
||||
datetime.strptime(v, '%Y-%m-%d')
|
||||
except ValueError as exc:
|
||||
raise ValueError(
|
||||
"Premiered date must be in YYYY-MM-DD format"
|
||||
) from exc
|
||||
|
||||
return v
|
||||
|
||||
@field_validator('dateadded')
|
||||
@classmethod
|
||||
def validate_dateadded(cls, v: Optional[str]) -> Optional[str]:
|
||||
"""Validate dateadded format (YYYY-MM-DD HH:MM:SS)."""
|
||||
if v is None:
|
||||
return v
|
||||
|
||||
# Check format strictly: YYYY-MM-DD HH:MM:SS
|
||||
if len(v) != 19 or v[4] != '-' or v[7] != '-' or v[10] != ' ' or v[13] != ':' or v[16] != ':':
|
||||
raise ValueError(
|
||||
"Dateadded must be in YYYY-MM-DD HH:MM:SS format"
|
||||
)
|
||||
|
||||
try:
|
||||
datetime.strptime(v, '%Y-%m-%d %H:%M:%S')
|
||||
except ValueError as exc:
|
||||
raise ValueError(
|
||||
"Dateadded must be in YYYY-MM-DD HH:MM:SS format"
|
||||
) from exc
|
||||
|
||||
return v
|
||||
|
||||
@field_validator('imdbid')
|
||||
@classmethod
|
||||
def validate_imdbid(cls, v: Optional[str]) -> Optional[str]:
|
||||
"""Validate IMDB ID format (should start with 'tt')."""
|
||||
if v is None:
|
||||
return v
|
||||
|
||||
if not v.startswith('tt'):
|
||||
raise ValueError("IMDB ID must start with 'tt'")
|
||||
|
||||
if not v[2:].isdigit():
|
||||
raise ValueError("IMDB ID must be 'tt' followed by digits")
|
||||
|
||||
return v
|
||||
|
||||
def model_post_init(self, __context) -> None:
|
||||
"""Set default values after initialization."""
|
||||
# Set showtitle to title if not provided
|
||||
if self.showtitle is None:
|
||||
self.showtitle = self.title
|
||||
|
||||
# Set originaltitle to title if not provided
|
||||
if self.originaltitle is None:
|
||||
self.originaltitle = self.title
|
||||
212
src/server/error_handler.py
Normal file
212
src/server/error_handler.py
Normal file
@@ -0,0 +1,212 @@
|
||||
"""
|
||||
Error handling and recovery strategies for core providers.
|
||||
|
||||
This module provides custom exceptions and decorators for handling
|
||||
errors in provider operations with automatic retry mechanisms.
|
||||
"""
|
||||
|
||||
import functools
|
||||
import logging
|
||||
from typing import Any, Callable, Optional, TypeVar
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Type variable for decorator
|
||||
F = TypeVar("F", bound=Callable[..., Any])
|
||||
|
||||
|
||||
class RetryableError(Exception):
|
||||
"""Exception that indicates an operation can be safely retried."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class NonRetryableError(Exception):
|
||||
"""Exception that indicates an operation should not be retried."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class NetworkError(Exception):
|
||||
"""Exception for network-related errors."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class DownloadError(Exception):
|
||||
"""Exception for download-related errors."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class RecoveryStrategies:
|
||||
"""Strategies for handling errors and recovering from failures."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_retries: int = 3,
|
||||
base_delay: float = 1.0,
|
||||
max_delay: float = 60.0,
|
||||
exponential_base: float = 2.0,
|
||||
) -> None:
|
||||
"""Initialize recovery strategies.
|
||||
|
||||
Args:
|
||||
max_retries: Maximum number of retry attempts.
|
||||
base_delay: Initial delay between retries in seconds.
|
||||
max_delay: Maximum delay between retries in seconds.
|
||||
exponential_base: Base for exponential backoff multiplier.
|
||||
"""
|
||||
self.max_retries = max_retries
|
||||
self.base_delay = base_delay
|
||||
self.max_delay = max_delay
|
||||
self.exponential_base = exponential_base
|
||||
|
||||
def _calculate_delay(self, attempt: int) -> float:
|
||||
"""Calculate delay for given retry attempt using exponential backoff.
|
||||
|
||||
Args:
|
||||
attempt: Zero-based retry attempt number.
|
||||
|
||||
Returns:
|
||||
Delay in seconds before next retry.
|
||||
"""
|
||||
delay = self.base_delay * (self.exponential_base ** attempt)
|
||||
return min(delay, self.max_delay)
|
||||
|
||||
@staticmethod
|
||||
def handle_network_failure(
|
||||
func: Callable, *args: Any, **kwargs: Any
|
||||
) -> Any:
|
||||
"""Handle network failures with exponential backoff retry logic."""
|
||||
last_error: Optional[Exception] = None
|
||||
max_retries = 3
|
||||
base_delay = 1.0
|
||||
max_delay = 60.0
|
||||
exponential_base = 2.0
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except (NetworkError, ConnectionError, TimeoutError) as exc:
|
||||
last_error = exc
|
||||
if attempt < max_retries - 1:
|
||||
delay = base_delay * (exponential_base ** attempt)
|
||||
delay = min(delay, max_delay)
|
||||
logger.warning(
|
||||
"Network error on attempt %d/%d, retrying in %.1fs: %s",
|
||||
attempt + 1, max_retries, delay, exc
|
||||
)
|
||||
import time
|
||||
time.sleep(delay)
|
||||
continue
|
||||
if last_error:
|
||||
raise last_error
|
||||
raise NetworkError("Network failure after retries")
|
||||
|
||||
@staticmethod
|
||||
def handle_download_failure(
|
||||
func: Callable, *args: Any, **kwargs: Any
|
||||
) -> Any:
|
||||
"""Handle download failures with exponential backoff retry logic."""
|
||||
last_error: Optional[Exception] = None
|
||||
max_retries = 2
|
||||
base_delay = 1.0
|
||||
max_delay = 60.0
|
||||
exponential_base = 2.0
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except DownloadError as exc:
|
||||
last_error = exc
|
||||
if attempt < max_retries - 1:
|
||||
delay = base_delay * (exponential_base ** attempt)
|
||||
delay = min(delay, max_delay)
|
||||
logger.warning(
|
||||
"Download error on attempt %d/%d, retrying in %.1fs: %s",
|
||||
attempt + 1, max_retries, delay, exc
|
||||
)
|
||||
import time
|
||||
time.sleep(delay)
|
||||
continue
|
||||
if last_error:
|
||||
raise last_error
|
||||
raise DownloadError("Download failed after retries")
|
||||
|
||||
|
||||
class FileCorruptionDetector:
|
||||
"""Detector for corrupted files."""
|
||||
|
||||
@staticmethod
|
||||
def is_valid_video_file(filepath: str) -> bool:
|
||||
"""Check if a video file is valid and not corrupted."""
|
||||
try:
|
||||
import os
|
||||
if not os.path.exists(filepath):
|
||||
return False
|
||||
|
||||
file_size = os.path.getsize(filepath)
|
||||
# Video files should be at least 1MB
|
||||
return file_size > 1024 * 1024
|
||||
except Exception as e:
|
||||
logger.error("Error checking file validity: %s", e)
|
||||
return False
|
||||
|
||||
|
||||
def with_error_recovery(
|
||||
max_retries: int = 3, context: str = ""
|
||||
) -> Callable[[F], F]:
|
||||
"""
|
||||
Decorator for adding error recovery to functions.
|
||||
|
||||
Args:
|
||||
max_retries: Maximum number of retry attempts
|
||||
context: Context string for logging
|
||||
|
||||
Returns:
|
||||
Decorated function with retry logic
|
||||
"""
|
||||
|
||||
def decorator(func: F) -> F:
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
||||
last_error = None
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except NonRetryableError:
|
||||
raise
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
if attempt < max_retries - 1:
|
||||
logger.warning(
|
||||
"Error in %s (attempt %d/%d): %s, retrying...",
|
||||
context,
|
||||
attempt + 1,
|
||||
max_retries,
|
||||
e,
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
"Error in %s failed after %d attempts: %s",
|
||||
context,
|
||||
max_retries,
|
||||
e,
|
||||
)
|
||||
|
||||
if last_error:
|
||||
raise last_error
|
||||
|
||||
raise RuntimeError(
|
||||
f"Unexpected error in {context} after {max_retries} attempts"
|
||||
)
|
||||
|
||||
return wrapper # type: ignore
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
# Create module-level instances for use in provider code
|
||||
recovery_strategies = RecoveryStrategies()
|
||||
file_corruption_detector = FileCorruptionDetector()
|
||||
3
src/server/exceptions/exceptions/__init__.py
Normal file
3
src/server/exceptions/exceptions/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
from src.server.exceptions.exceptions.Exceptions import MatchNotFoundError, NoKeyFoundException
|
||||
|
||||
__all__ = ["MatchNotFoundError", "NoKeyFoundException"]
|
||||
@@ -6,6 +6,7 @@ configuration, middleware setup, static file serving, and Jinja2 template
|
||||
integration.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
@@ -23,7 +24,10 @@ from src.server.api.auth import router as auth_router
|
||||
from src.server.api.config import router as config_router
|
||||
from src.server.api.download import router as download_router
|
||||
from src.server.api.health import router as health_router
|
||||
from src.server.api.logging import router as logging_router
|
||||
from src.server.api.nfo import router as nfo_router
|
||||
from src.server.api.scheduler import router as scheduler_router
|
||||
from src.server.api.setup_endpoints import router as setup_router
|
||||
from src.server.api.websocket import router as websocket_router
|
||||
from src.server.controllers.error_controller import (
|
||||
not_found_handler,
|
||||
@@ -35,7 +39,6 @@ from src.server.controllers.page_controller import router as page_router
|
||||
from src.server.middleware.auth import AuthMiddleware
|
||||
from src.server.middleware.error_handler import register_exception_handlers
|
||||
from src.server.middleware.setup_redirect import SetupRedirectMiddleware
|
||||
from src.server.services.anime_service import sync_series_from_data_files
|
||||
from src.server.services.progress_service import get_progress_service
|
||||
from src.server.services.websocket_service import get_websocket_service
|
||||
|
||||
@@ -43,6 +46,165 @@ from src.server.services.websocket_service import get_websocket_service
|
||||
# module-level globals. This makes testing and multi-instance hosting safer.
|
||||
|
||||
|
||||
async def _check_incomplete_series_on_startup(background_loader) -> None:
|
||||
"""Check for incomplete series on startup and queue background loading.
|
||||
|
||||
Args:
|
||||
background_loader: BackgroundLoaderService instance
|
||||
"""
|
||||
logger = logging.getLogger("aniworld")
|
||||
|
||||
try:
|
||||
from src.server.database.connection import get_db_session
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
|
||||
async with get_db_session() as db:
|
||||
try:
|
||||
# Get all series from database
|
||||
series_list = await AnimeSeriesService.get_all(db)
|
||||
|
||||
incomplete_series = []
|
||||
|
||||
for series in series_list:
|
||||
# Check if series has incomplete loading
|
||||
if series.loading_status != "completed":
|
||||
incomplete_series.append(series)
|
||||
# Or check if specific data is missing
|
||||
elif (not series.episodes_loaded or
|
||||
not series.has_nfo or
|
||||
not series.logo_loaded or
|
||||
not series.images_loaded):
|
||||
incomplete_series.append(series)
|
||||
|
||||
if incomplete_series:
|
||||
logger.info(
|
||||
f"Found {len(incomplete_series)} series with missing data. "
|
||||
f"Queuing for background loading..."
|
||||
)
|
||||
|
||||
for series in incomplete_series:
|
||||
await background_loader.add_series_loading_task(
|
||||
key=series.key,
|
||||
folder=series.folder,
|
||||
name=series.name,
|
||||
year=series.year
|
||||
)
|
||||
logger.debug(
|
||||
f"Queued background loading for series: {series.key}"
|
||||
)
|
||||
|
||||
logger.info("All incomplete series queued for background loading")
|
||||
else:
|
||||
logger.info("All series data is complete. No background loading needed.")
|
||||
|
||||
except Exception:
|
||||
logger.exception("Error checking incomplete series")
|
||||
|
||||
except Exception:
|
||||
logger.exception("Failed to check incomplete series on startup")
|
||||
|
||||
|
||||
async def _run_startup_health_checks(logger) -> dict:
|
||||
"""Run startup health checks for critical dependencies.
|
||||
|
||||
Checks:
|
||||
- ffmpeg availability
|
||||
- DNS resolution for aniworld.to and api.themoviedb.org
|
||||
- anime_directory configuration and writability
|
||||
|
||||
Args:
|
||||
logger: Logger instance for recording check results.
|
||||
|
||||
Returns:
|
||||
dict: Health check results with status and details for each check.
|
||||
"""
|
||||
import asyncio
|
||||
import shutil
|
||||
import socket
|
||||
from typing import Any, Dict
|
||||
|
||||
checks: Dict[str, Any] = {
|
||||
"ffmpeg": {"status": "unknown", "message": None},
|
||||
"dns_aniworld": {"status": "unknown", "message": None},
|
||||
"dns_tmdb": {"status": "unknown", "message": None},
|
||||
"anime_directory": {"status": "unknown", "message": None, "path": None},
|
||||
}
|
||||
|
||||
# Check ffmpeg availability
|
||||
try:
|
||||
ffmpeg_path = shutil.which("ffmpeg")
|
||||
if ffmpeg_path:
|
||||
checks["ffmpeg"]["status"] = "ok"
|
||||
checks["ffmpeg"]["message"] = f"Found at {ffmpeg_path}"
|
||||
logger.debug("ffmpeg health check passed: %s", ffmpeg_path)
|
||||
else:
|
||||
checks["ffmpeg"]["status"] = "warning"
|
||||
checks["ffmpeg"]["message"] = "ffmpeg not found in PATH"
|
||||
logger.warning("ffmpeg health check failed: not in PATH")
|
||||
except Exception as e:
|
||||
checks["ffmpeg"]["status"] = "error"
|
||||
checks["ffmpeg"]["message"] = str(e)
|
||||
logger.warning("Could not check ffmpeg: %s", e)
|
||||
|
||||
# Check DNS resolution for aniworld.to
|
||||
try:
|
||||
socket.gethostbyname("aniworld.to")
|
||||
checks["dns_aniworld"]["status"] = "ok"
|
||||
checks["dns_aniworld"]["message"] = "Resolved successfully"
|
||||
logger.debug("DNS health check passed for aniworld.to")
|
||||
except socket.gaierror as e:
|
||||
checks["dns_aniworld"]["status"] = "warning"
|
||||
checks["dns_aniworld"]["message"] = f"DNS resolution failed: {e}"
|
||||
logger.warning("DNS health check failed for aniworld.to: %s", e)
|
||||
except Exception as e:
|
||||
checks["dns_aniworld"]["status"] = "warning"
|
||||
checks["dns_aniworld"]["message"] = f"Unexpected error: {e}"
|
||||
logger.warning("Unexpected DNS error for aniworld.to: %s", e)
|
||||
|
||||
# Check DNS resolution for api.themoviedb.org
|
||||
try:
|
||||
socket.gethostbyname("api.themoviedb.org")
|
||||
checks["dns_tmdb"]["status"] = "ok"
|
||||
checks["dns_tmdb"]["message"] = "Resolved successfully"
|
||||
logger.debug("DNS health check passed for api.themoviedb.org")
|
||||
except socket.gaierror as e:
|
||||
checks["dns_tmdb"]["status"] = "warning"
|
||||
checks["dns_tmdb"]["message"] = f"DNS resolution failed: {e}"
|
||||
logger.warning("DNS health check failed for api.themoviedb.org: %s", e)
|
||||
except Exception as e:
|
||||
checks["dns_tmdb"]["status"] = "warning"
|
||||
checks["dns_tmdb"]["message"] = f"Unexpected error: {e}"
|
||||
logger.warning("Unexpected DNS error for api.themoviedb.org: %s", e)
|
||||
|
||||
# Check anime_directory configuration and writability
|
||||
from src.config.settings import settings
|
||||
anime_dir = settings.anime_directory
|
||||
|
||||
if not anime_dir:
|
||||
checks["anime_directory"]["status"] = "error"
|
||||
checks["anime_directory"]["message"] = "anime_directory not configured"
|
||||
checks["anime_directory"]["path"] = None
|
||||
logger.error("anime_directory health check failed: not configured")
|
||||
else:
|
||||
import os
|
||||
checks["anime_directory"]["path"] = anime_dir
|
||||
|
||||
if not os.path.isdir(anime_dir):
|
||||
checks["anime_directory"]["status"] = "error"
|
||||
checks["anime_directory"]["message"] = f"Directory does not exist: {anime_dir}"
|
||||
logger.error("anime_directory health check failed: %s does not exist", anime_dir)
|
||||
elif not os.access(anime_dir, os.W_OK):
|
||||
checks["anime_directory"]["status"] = "error"
|
||||
checks["anime_directory"]["message"] = f"Directory not writable: {anime_dir}"
|
||||
logger.error("anime_directory health check failed: %s not writable", anime_dir)
|
||||
else:
|
||||
checks["anime_directory"]["status"] = "ok"
|
||||
checks["anime_directory"]["message"] = f"Directory exists and is writable: {anime_dir}"
|
||||
logger.debug("anime_directory health check passed: %s", anime_dir)
|
||||
|
||||
return checks
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_application: FastAPI):
|
||||
"""Manage application lifespan (startup and shutdown).
|
||||
@@ -53,21 +215,57 @@ async def lifespan(_application: FastAPI):
|
||||
"""
|
||||
# Setup logging first with INFO level
|
||||
logger = setup_logging(log_level="INFO")
|
||||
logger.info("Starting FastAPI application v%s", APP_VERSION)
|
||||
|
||||
# Track successful initialization steps
|
||||
initialized = {
|
||||
'database': False,
|
||||
'services': False,
|
||||
'background_loader': False,
|
||||
'scheduler': False
|
||||
}
|
||||
|
||||
# Startup
|
||||
startup_error = None
|
||||
try:
|
||||
logger.info("Starting FastAPI application...")
|
||||
|
||||
|
||||
# Clean up any leftover temp download files from a previous run
|
||||
try:
|
||||
import shutil as _shutil
|
||||
_temp_dir = Path(__file__).resolve().parents[2] / "Temp"
|
||||
if _temp_dir.exists():
|
||||
_removed = 0
|
||||
for _item in _temp_dir.iterdir():
|
||||
try:
|
||||
if _item.is_file():
|
||||
_item.unlink()
|
||||
elif _item.is_dir():
|
||||
_shutil.rmtree(_item)
|
||||
_removed += 1
|
||||
except OSError as _exc:
|
||||
logger.warning("Could not remove temp item %s: %s", _item, _exc)
|
||||
logger.info("Cleaned %d item(s) from Temp folder on startup", _removed)
|
||||
else:
|
||||
_temp_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.debug("Created Temp folder: %s", _temp_dir)
|
||||
except Exception as _exc:
|
||||
logger.warning("Failed to clean Temp folder on startup: %s", _exc)
|
||||
|
||||
# Initialize database first (required for other services)
|
||||
try:
|
||||
from src.server.database.connection import init_db
|
||||
await init_db()
|
||||
initialized['database'] = True
|
||||
logger.info("Database initialized successfully")
|
||||
except Exception as e:
|
||||
logger.error("Failed to initialize database: %s", e, exc_info=True)
|
||||
startup_error = e
|
||||
raise # Database is required, fail startup if it fails
|
||||
|
||||
# Load configuration from config.json and sync with settings
|
||||
# Precedence: ENV vars > config.json > defaults
|
||||
# Only sync from config.json if setting is at default value
|
||||
try:
|
||||
from src.server.services.config_service import get_config_service
|
||||
config_service = get_config_service()
|
||||
@@ -78,19 +276,46 @@ async def lifespan(_application: FastAPI):
|
||||
)
|
||||
|
||||
# Sync anime_directory from config.json to settings
|
||||
# config.other is Dict[str, object] - pylint doesn't infer this
|
||||
# Only if not already set via ENV var (i.e., still empty)
|
||||
other_settings = dict(config.other) if config.other else {}
|
||||
if other_settings.get("anime_directory"):
|
||||
anime_dir = other_settings["anime_directory"]
|
||||
settings.anime_directory = str(anime_dir)
|
||||
logger.info(
|
||||
"Loaded anime_directory from config: %s",
|
||||
settings.anime_directory
|
||||
)
|
||||
# Only override if settings.anime_directory is empty (default)
|
||||
if not settings.anime_directory:
|
||||
settings.anime_directory = str(anime_dir)
|
||||
logger.info(
|
||||
"Loaded anime_directory from config.json: %s",
|
||||
settings.anime_directory
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"anime_directory from ENV var takes precedence: %s",
|
||||
settings.anime_directory
|
||||
)
|
||||
else:
|
||||
logger.debug(
|
||||
"anime_directory not found in config.other"
|
||||
)
|
||||
|
||||
# Sync NFO settings from config.json to settings
|
||||
# Only if not already set via ENV var
|
||||
if config.nfo:
|
||||
# TMDB API key: ENV takes precedence
|
||||
if config.nfo.tmdb_api_key and not settings.tmdb_api_key:
|
||||
settings.tmdb_api_key = config.nfo.tmdb_api_key
|
||||
logger.info("Loaded TMDB API key from config.json")
|
||||
elif settings.tmdb_api_key:
|
||||
logger.info("Using TMDB API key from ENV var")
|
||||
|
||||
# NFO boolean flags: Sync from config.json
|
||||
# (These have proper defaults, so we can sync them)
|
||||
settings.nfo_auto_create = config.nfo.auto_create
|
||||
settings.nfo_update_on_scan = config.nfo.update_on_scan
|
||||
settings.nfo_download_poster = config.nfo.download_poster
|
||||
settings.nfo_download_logo = config.nfo.download_logo
|
||||
settings.nfo_download_fanart = config.nfo.download_fanart
|
||||
settings.nfo_image_size = config.nfo.image_size
|
||||
logger.debug("Synced NFO settings from config.json")
|
||||
except (OSError, ValueError, KeyError) as e:
|
||||
logger.warning("Failed to load config from config.json: %s", e)
|
||||
|
||||
@@ -113,52 +338,149 @@ async def lifespan(_application: FastAPI):
|
||||
# Subscribe to progress events
|
||||
progress_service.subscribe("progress_updated", progress_event_handler)
|
||||
|
||||
# Initialize download service and restore queue from database
|
||||
# Only if anime directory is configured
|
||||
# Perform initial setup (series sync and marking as completed)
|
||||
# This is centralized in initialization_service and also called
|
||||
# from the setup endpoint
|
||||
from src.server.services.initialization_service import (
|
||||
perform_initial_setup,
|
||||
perform_media_scan_if_needed,
|
||||
)
|
||||
|
||||
try:
|
||||
from src.server.utils.dependencies import get_download_service
|
||||
|
||||
logger.info(
|
||||
"Checking anime_directory setting: '%s'",
|
||||
settings.anime_directory
|
||||
)
|
||||
|
||||
if settings.anime_directory:
|
||||
download_service = get_download_service()
|
||||
await download_service.initialize()
|
||||
logger.info("Download service initialized and queue restored")
|
||||
|
||||
# Sync series from data files to database
|
||||
sync_count = await sync_series_from_data_files(
|
||||
settings.anime_directory
|
||||
)
|
||||
logger.info(
|
||||
"Data file sync complete. Added %d series.", sync_count
|
||||
)
|
||||
# Perform initial setup if needed
|
||||
await perform_initial_setup()
|
||||
|
||||
# Get anime service and load series — isolated so a missing
|
||||
# directory doesn't abort the rest of the startup sequence
|
||||
try:
|
||||
from src.server.utils.dependencies import get_anime_service
|
||||
anime_service = get_anime_service()
|
||||
|
||||
# Always load series from database into memory on startup
|
||||
logger.info("Loading series from database into memory...")
|
||||
await anime_service._load_series_from_db()
|
||||
logger.info("Series loaded from database into memory")
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Could not load series into memory (directory may not "
|
||||
"exist yet): %s", e
|
||||
)
|
||||
|
||||
# Initialize download service
|
||||
try:
|
||||
from src.server.utils.dependencies import get_download_service
|
||||
download_service = get_download_service()
|
||||
await download_service.initialize()
|
||||
initialized['services'] = True
|
||||
logger.info("Download service initialized and queue restored")
|
||||
except Exception as e:
|
||||
logger.warning("Failed to initialize download service: %s", e)
|
||||
|
||||
# Initialize background loader service
|
||||
background_loader = None
|
||||
try:
|
||||
from src.server.utils.dependencies import (
|
||||
get_background_loader_service,
|
||||
)
|
||||
background_loader = get_background_loader_service()
|
||||
await background_loader.start()
|
||||
initialized['background_loader'] = True
|
||||
logger.info("Background loader service started")
|
||||
except Exception as e:
|
||||
logger.warning("Failed to start background loader service: %s", e)
|
||||
|
||||
# Run media scan only on first run
|
||||
await perform_media_scan_if_needed(background_loader)
|
||||
else:
|
||||
logger.info(
|
||||
"Download service initialization skipped - "
|
||||
"anime directory not configured"
|
||||
)
|
||||
|
||||
# Initialize and start scheduler service (independent of anime_directory)
|
||||
# The scheduler loads its own config from config.json and the
|
||||
# anime_directory may be configured there even if the env var is empty.
|
||||
try:
|
||||
logger.info("Initializing scheduler service...")
|
||||
from src.server.services.scheduler.scheduler_service import (
|
||||
get_scheduler_service,
|
||||
)
|
||||
scheduler_service = get_scheduler_service()
|
||||
logger.info("Scheduler service instance obtained, starting...")
|
||||
await scheduler_service.start()
|
||||
initialized['scheduler'] = True
|
||||
logger.info("Scheduler service started successfully")
|
||||
except Exception as e:
|
||||
logger.warning("Failed to start scheduler service: %s", e)
|
||||
except (OSError, RuntimeError, ValueError) as e:
|
||||
logger.warning("Failed to initialize download service: %s", e)
|
||||
# Continue startup - download service can be initialized later
|
||||
logger.warning("Failed to initialize services: %s", e)
|
||||
# Continue startup - services can be initialized later
|
||||
except Exception as e:
|
||||
logger.warning("Unexpected error during startup initialization: %s", e)
|
||||
# Continue startup - services can be configured/initialized later
|
||||
|
||||
logger.info("FastAPI application started successfully")
|
||||
logger.info("Server running on http://127.0.0.1:8000")
|
||||
logger.info(
|
||||
"API documentation available at http://127.0.0.1:8000/api/docs"
|
||||
)
|
||||
|
||||
# Check for ffmpeg availability and warn if missing
|
||||
try:
|
||||
import shutil as _shutil
|
||||
if _shutil.which("ffmpeg") is None:
|
||||
logger.warning(
|
||||
"ffmpeg not found in PATH. HLS streams may fail to download. "
|
||||
"Install ffmpeg to enable HLS support."
|
||||
)
|
||||
else:
|
||||
logger.debug("ffmpeg found at: %s", _shutil.which("ffmpeg"))
|
||||
except Exception as _exc:
|
||||
logger.warning("Could not check for ffmpeg: %s", _exc)
|
||||
|
||||
# Run startup health checks and store results for /health endpoint
|
||||
try:
|
||||
startup_checks = await _run_startup_health_checks(logger)
|
||||
app.state.startup_checks = startup_checks
|
||||
except Exception as _exc:
|
||||
logger.warning("Could not run startup health checks: %s", _exc)
|
||||
app.state.startup_checks = {}
|
||||
except Exception as e:
|
||||
logger.error("Error during startup: %s", e, exc_info=True)
|
||||
raise # Re-raise to prevent app from starting in broken state
|
||||
startup_error = e
|
||||
# Don't re-raise here, let the finally/cleanup handle shutdown
|
||||
|
||||
# Yield control to the application
|
||||
yield
|
||||
# Yield control to the application (or immediately go to cleanup on error)
|
||||
if startup_error is None:
|
||||
try:
|
||||
yield
|
||||
except Exception as e:
|
||||
logger.error("Error during application runtime: %s", e, exc_info=True)
|
||||
else:
|
||||
# Startup failed, but we still need to yield to satisfy the protocol
|
||||
# The app won't actually run since we'll raise after cleanup
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
# After cleanup, re-raise the startup error
|
||||
pass
|
||||
|
||||
# Shutdown - execute in proper order with timeout protection
|
||||
logger.info("FastAPI application shutting down (graceful shutdown initiated)")
|
||||
|
||||
# Only cleanup what was successfully initialized
|
||||
if not initialized['database']:
|
||||
logger.info("Database was not initialized, skipping all cleanup")
|
||||
if startup_error:
|
||||
raise startup_error
|
||||
return
|
||||
|
||||
# Define shutdown timeout (total time allowed for all shutdown operations)
|
||||
SHUTDOWN_TIMEOUT = 30.0
|
||||
|
||||
@@ -170,7 +492,41 @@ async def lifespan(_application: FastAPI):
|
||||
elapsed = time.monotonic() - shutdown_start
|
||||
return max(0.0, SHUTDOWN_TIMEOUT - elapsed)
|
||||
|
||||
# 1. Broadcast shutdown notification via WebSocket
|
||||
# 1. Stop scheduler service (only if initialized)
|
||||
if initialized['scheduler']:
|
||||
try:
|
||||
from src.server.services.scheduler.scheduler_service import (
|
||||
get_scheduler_service,
|
||||
)
|
||||
scheduler_service = get_scheduler_service()
|
||||
logger.info("Stopping scheduler service...")
|
||||
await asyncio.wait_for(
|
||||
scheduler_service.stop(),
|
||||
timeout=min(5.0, remaining_time())
|
||||
)
|
||||
logger.info("Scheduler service stopped")
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning("Scheduler service shutdown timed out")
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error stopping scheduler service: %s", e, exc_info=True)
|
||||
|
||||
# 2. Stop background loader service (only if initialized)
|
||||
if initialized['background_loader']:
|
||||
try:
|
||||
from src.server.utils.dependencies import _background_loader_service
|
||||
if _background_loader_service is not None:
|
||||
logger.info("Stopping background loader service...")
|
||||
await asyncio.wait_for(
|
||||
_background_loader_service.stop(),
|
||||
timeout=min(10.0, remaining_time())
|
||||
)
|
||||
logger.info("Background loader service stopped")
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning("Background loader service shutdown timed out")
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error stopping background loader service: %s", e, exc_info=True)
|
||||
|
||||
# 3. Broadcast shutdown notification via WebSocket
|
||||
try:
|
||||
ws_service = get_websocket_service()
|
||||
logger.info("Broadcasting shutdown notification to WebSocket clients...")
|
||||
@@ -184,10 +540,10 @@ async def lifespan(_application: FastAPI):
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error during WebSocket shutdown: %s", e, exc_info=True)
|
||||
|
||||
# 2. Shutdown download service and persist active downloads
|
||||
# 4. Shutdown download service and persist active downloads
|
||||
try:
|
||||
from src.server.services.download_service import ( # noqa: E501
|
||||
_download_service_instance,
|
||||
from src.server.services.download_service import (
|
||||
_download_service_instance, # noqa: E501
|
||||
)
|
||||
if _download_service_instance is not None:
|
||||
logger.info("Stopping download service...")
|
||||
@@ -197,7 +553,7 @@ async def lifespan(_application: FastAPI):
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error stopping download service: %s", e, exc_info=True)
|
||||
|
||||
# 3. Shutdown SeriesApp and cleanup thread pool
|
||||
# 4. Shutdown SeriesApp and cleanup thread pool
|
||||
try:
|
||||
from src.server.utils.dependencies import _series_app
|
||||
if _series_app is not None:
|
||||
@@ -207,7 +563,7 @@ async def lifespan(_application: FastAPI):
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error("Error during SeriesApp shutdown: %s", e, exc_info=True)
|
||||
|
||||
# 4. Cleanup progress service
|
||||
# 5. Cleanup progress service
|
||||
try:
|
||||
progress_service = get_progress_service()
|
||||
logger.info("Cleaning up progress service...")
|
||||
@@ -238,13 +594,19 @@ async def lifespan(_application: FastAPI):
|
||||
"FastAPI application shutdown complete (took %.2fs)",
|
||||
elapsed_total
|
||||
)
|
||||
|
||||
# Re-raise startup error if it occurred
|
||||
if startup_error:
|
||||
raise startup_error
|
||||
|
||||
|
||||
from src.server.utils.version import APP_VERSION
|
||||
|
||||
# Initialize FastAPI app with lifespan
|
||||
app = FastAPI(
|
||||
title="Aniworld Download Manager",
|
||||
description="Modern web interface for Aniworld anime download management",
|
||||
version="1.0.0",
|
||||
version=APP_VERSION,
|
||||
docs_url="/api/docs",
|
||||
redoc_url="/api/redoc",
|
||||
lifespan=lifespan
|
||||
@@ -282,6 +644,9 @@ app.include_router(config_router)
|
||||
app.include_router(scheduler_router)
|
||||
app.include_router(anime_router)
|
||||
app.include_router(download_router)
|
||||
app.include_router(nfo_router)
|
||||
app.include_router(setup_router)
|
||||
app.include_router(logging_router)
|
||||
app.include_router(websocket_router)
|
||||
|
||||
# Register exception handlers
|
||||
|
||||
@@ -12,6 +12,8 @@ from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OperationType(str, Enum):
|
||||
"""Types of operations that can report progress."""
|
||||
@@ -313,7 +315,7 @@ class CallbackManager:
|
||||
callback.on_progress(context)
|
||||
except Exception as e:
|
||||
# Log but don't let callback errors break the operation
|
||||
logging.error(
|
||||
logger.error(
|
||||
"Error in progress callback %s: %s",
|
||||
callback,
|
||||
e,
|
||||
@@ -332,7 +334,7 @@ class CallbackManager:
|
||||
callback.on_error(context)
|
||||
except Exception as e:
|
||||
# Log but don't let callback errors break the operation
|
||||
logging.error(
|
||||
logger.error(
|
||||
"Error in error callback %s: %s",
|
||||
callback,
|
||||
e,
|
||||
@@ -351,7 +353,7 @@ class CallbackManager:
|
||||
callback.on_completion(context)
|
||||
except Exception as e:
|
||||
# Log but don't let callback errors break the operation
|
||||
logging.error(
|
||||
logger.error(
|
||||
"Error in completion callback %s: %s",
|
||||
callback,
|
||||
e,
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user