Compare commits
155 Commits
079f1f99e3
...
v1.4.7
| Author | SHA1 | Date | |
|---|---|---|---|
| 6a934db8ac | |||
| ac7302b1dd | |||
| ac5ee3bb27 | |||
| a9084202e3 | |||
| be9f2a4c0c | |||
| 53fe09351f | |||
| dc7d9ee5f7 | |||
| da3cae2812 | |||
| 2876cef24b | |||
| 6a402623c4 | |||
| ebfbec1225 | |||
| 01e4dec8d7 | |||
| ecef21eec4 | |||
| d9738ffb78 | |||
| 6aec2a1733 | |||
| 84487d7571 | |||
| e02d65778f | |||
| 45d259bab2 | |||
| 7b8de8d988 | |||
| 18d10b44b5 | |||
| 5c2be3f7c4 | |||
| 2c47713339 | |||
| e74b04c1ee | |||
| 8b21f1243f | |||
| 3d33626546 | |||
| 7d9f80a0c6 | |||
| 25dc66fec3 | |||
| 2be7b692b9 | |||
| 2b5c969a83 | |||
| 830f6b4c93 | |||
| 5526ab884a | |||
| 09d454d4c0 | |||
| 13504c3172 | |||
| 82493d41ea | |||
| 274f773988 | |||
| 21af502184 | |||
| 97caaf0d18 | |||
| dc5d6506bc | |||
| dbaf80e941 | |||
| 4fc597c5de | |||
| a77bb371df | |||
| 420d10bb34 | |||
| e29918488c | |||
| 9c3f03d610 | |||
| 9d64241230 | |||
| 49cd84f3e5 | |||
| e46759347e | |||
| 75f743e6cc | |||
| 4dc5ffa19e | |||
| 1649a22418 | |||
| 246752e2fc | |||
| 84b24ed79e | |||
| bf3954587a | |||
| ed8f5cae10 | |||
| a54c285994 | |||
| c58b42dfa5 | |||
| 6dfb24de7e | |||
| 6021cdef28 | |||
| 5517ccbab0 | |||
| 94ed013172 | |||
| 76b849fc91 | |||
| 00b26c8cbc | |||
| a6f2399aca | |||
| cf001563b3 | |||
| 38c12638a4 | |||
| 765e43c684 | |||
| 5190d32665 | |||
| 4e6afa31b5 | |||
| 1ef59c5283 | |||
| 239341629c | |||
| 51b7f349f8 | |||
| 14b8ef7f06 | |||
| 7abba0dae2 | |||
| 30858f441c | |||
| 33f63ca304 | |||
| fe9284b80e | |||
| 12e5526991 | |||
| bc87bee416 | |||
| 7ded5a6e4d | |||
| d596902ca3 | |||
| d358a07290 | |||
| b9c55f9e7a | |||
| fc4e52f1a2 | |||
| 6d30747f25 | |||
| ceb6a2aeb4 | |||
| 53d6da5dac | |||
| 102d83e947 | |||
| 841368bf85 | |||
| cbd53ef2a0 | |||
| 50a77976d5 | |||
| dfc28b8e66 | |||
| 6c9605e896 | |||
| 3947f6d266 | |||
| a3176f5ac1 | |||
| 9a81b04b65 | |||
| a336733ea9 | |||
| ca93bb740a | |||
| d5e955a731 | |||
| e2a373816a | |||
| a115215416 | |||
| c579235af0 | |||
| 0ba2587bc8 | |||
| bda1fe4445 | |||
| 810346bc8b | |||
| daa937bcb7 | |||
| 1c505bd722 | |||
| 3551838887 | |||
| 9a20541598 | |||
| 3f7651404d | |||
| bee24406e6 | |||
| 31eb0026cf | |||
| 24ea12bbaf | |||
| e74b602f60 | |||
| db65e28854 | |||
| 11e231a4ab | |||
| a11f8c4fa0 | |||
| cf5a06af11 | |||
| e07f75432e | |||
| 1696d5c65b | |||
| c8b386f47a | |||
| 3888da352a | |||
| 06e104db42 | |||
| d4594bd1d9 | |||
| d866e836f6 | |||
| 195dae13cb | |||
| 51be777e7d | |||
| 7930e49701 | |||
| 75c22fe296 | |||
| 7bcd0600d5 | |||
| a333329ae2 | |||
| 363f7899f8 | |||
| a08a8f7408 | |||
| 54ac5e9ab7 | |||
| c93ac3e7b8 | |||
| 68c4335348 | |||
| be87f2e230 | |||
| c56e0f507d | |||
| cb0a36ccc2 | |||
| 3644b16447 | |||
| d5116e378e | |||
| 50a7083ce5 | |||
| 52c0ff2337 | |||
| a5fd88e224 | |||
| 98d4edad14 | |||
| bc8059b453 | |||
| 815a4f1520 | |||
| e3509f5c8f | |||
| 69c2fd01f9 | |||
| 0f36afd88c | |||
| ceac22fc34 | |||
| 9c0f7ce08d | |||
| 756731cd5d | |||
| eb0e6e8ccb | |||
| eb2fc3c5ab | |||
| c39ae9d0fc |
@@ -17,6 +17,9 @@ __pycache__/
|
||||
# Docker files (not needed inside the image)
|
||||
Docker/
|
||||
|
||||
# Exception: VERSION is needed by Dockerfile.app
|
||||
!Docker/VERSION
|
||||
|
||||
# Test and dev files
|
||||
tests/
|
||||
Temp/
|
||||
|
||||
@@ -13,7 +13,8 @@ RUN apk add --no-cache \
|
||||
# Create wireguard config directory (config is mounted at runtime)
|
||||
RUN mkdir -p /etc/wireguard
|
||||
|
||||
# Copy entrypoint
|
||||
# Copy version file and entrypoint
|
||||
COPY VERSION /etc/wireguard/VERSION
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
|
||||
@@ -2,12 +2,13 @@ FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install system dependencies for compiled Python packages
|
||||
# Install system dependencies for compiled Python packages and ffmpeg for HLS support
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
gcc \
|
||||
g++ \
|
||||
libffi-dev \
|
||||
ffmpeg \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install Python dependencies (cached layer)
|
||||
@@ -19,6 +20,7 @@ COPY src/ ./src/
|
||||
COPY run_server.py .
|
||||
COPY pyproject.toml .
|
||||
COPY data/config.json ./data/config.json
|
||||
COPY Docker/VERSION ./Docker/VERSION
|
||||
|
||||
# Create runtime directories
|
||||
RUN mkdir -p /app/data/config_backups /app/logs
|
||||
|
||||
1
Docker/VERSION
Normal file
1
Docker/VERSION
Normal file
@@ -0,0 +1 @@
|
||||
v1.4.7
|
||||
@@ -1,6 +1,14 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
VERSION_FILE="/etc/wireguard/VERSION"
|
||||
if [ -f "$VERSION_FILE" ]; then
|
||||
VERSION=$(cat "$VERSION_FILE")
|
||||
else
|
||||
VERSION="unknown"
|
||||
fi
|
||||
echo "[init] VPN Container Entrypoint ${VERSION}"
|
||||
|
||||
INTERFACE="wg0"
|
||||
MOUNT_CONFIG="/etc/wireguard/${INTERFACE}.conf"
|
||||
CONFIG_DIR="/run/wireguard"
|
||||
@@ -64,9 +72,11 @@ setup_killswitch() {
|
||||
iptables -A INPUT -i "$INTERFACE" -j ACCEPT
|
||||
iptables -A OUTPUT -o "$INTERFACE" -j ACCEPT
|
||||
|
||||
# Allow DNS to the VPN DNS server (through wg0)
|
||||
iptables -A OUTPUT -o "$INTERFACE" -p udp --dport 53 -j ACCEPT
|
||||
iptables -A OUTPUT -o "$INTERFACE" -p tcp --dport 53 -j ACCEPT
|
||||
# Allow DNS (VPN DNS servers are routed through wg0; allow before routing decision)
|
||||
iptables -A OUTPUT -p udp --dport 53 -j ACCEPT
|
||||
iptables -A OUTPUT -p tcp --dport 53 -j ACCEPT
|
||||
iptables -A INPUT -p udp --sport 53 -j ACCEPT
|
||||
iptables -A INPUT -p tcp --sport 53 -j ACCEPT
|
||||
|
||||
# Allow DHCP (for container networking)
|
||||
iptables -A OUTPUT -p udp --dport 67:68 -j ACCEPT
|
||||
@@ -120,27 +130,46 @@ start_vpn() {
|
||||
ip link add "$INTERFACE" type wireguard
|
||||
|
||||
# Apply the WireGuard config (keys, peer, endpoint)
|
||||
# Filter out wg-quick directives that wg setconf doesn't understand
|
||||
wg setconf "$INTERFACE" <(grep -v -i '^\(Address\|DNS\|MTU\|Table\|PreUp\|PostUp\|PreDown\|PostDown\|SaveConfig\)' "$CONFIG_FILE")
|
||||
|
||||
# Log public key so it can be verified against the server's peer list
|
||||
local PUBKEY
|
||||
PUBKEY=$(wg show "$INTERFACE" public-key 2>/dev/null || echo "unknown")
|
||||
echo "[vpn] Public key: ${PUBKEY}"
|
||||
|
||||
# Assign the address
|
||||
ip -4 address add "$VPN_ADDRESS" dev "$INTERFACE"
|
||||
|
||||
# Set MTU
|
||||
# Set MTU and bring up
|
||||
ip link set mtu 1420 up dev "$INTERFACE"
|
||||
|
||||
# Find default gateway/interface for the endpoint route
|
||||
# ── fwmark-based routing (mirrors wg-quick behavior) ──
|
||||
# WireGuard marks its own encapsulated UDP packets with this fwmark.
|
||||
# Policy rules then ensure:
|
||||
# - Normal packets (no mark) → VPN routing table → wg0
|
||||
# - WireGuard-encapsulated packets (marked) → main table → eth0
|
||||
local FW_MARK=51820
|
||||
local FW_TABLE=51820
|
||||
wg set "$INTERFACE" fwmark "$FW_MARK"
|
||||
|
||||
# Remove any auto-created default route on wg0
|
||||
ip route del default dev "$INTERFACE" 2>/dev/null || true
|
||||
|
||||
# VPN routing table: send everything through the tunnel
|
||||
ip -4 route add default dev "$INTERFACE" table "$FW_TABLE"
|
||||
|
||||
# Policy rules:
|
||||
# 1. Packets NOT marked by WireGuard use the VPN table (→ wg0)
|
||||
# 2. suppress_prefixlength 0: ignore bare default routes in main table,
|
||||
# but keep more-specific routes (e.g. LAN, endpoint) working
|
||||
ip -4 rule add not fwmark "$FW_MARK" table "$FW_TABLE"
|
||||
ip -4 rule add table main suppress_prefixlength 0
|
||||
|
||||
# Find default gateway/interface
|
||||
DEFAULT_GW=$(ip route | grep '^default' | head -1 | awk '{print $3}')
|
||||
DEFAULT_IF=$(ip route | grep '^default' | head -1 | awk '{print $5}')
|
||||
|
||||
# Route VPN endpoint through the container's default gateway
|
||||
if [ -n "$DEFAULT_GW" ] && [ -n "$DEFAULT_IF" ]; then
|
||||
ip route add "$VPN_ENDPOINT/32" via "$DEFAULT_GW" dev "$DEFAULT_IF" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# Route all traffic through the WireGuard tunnel
|
||||
ip route add 0.0.0.0/1 dev "$INTERFACE"
|
||||
ip route add 128.0.0.0/1 dev "$INTERFACE"
|
||||
|
||||
# ── Policy routing: ensure responses to incoming LAN traffic go back via eth0 ──
|
||||
if [ -n "$DEFAULT_GW" ] && [ -n "$DEFAULT_IF" ]; then
|
||||
# Get the container's eth0 IP address (BusyBox-compatible, no grep -P)
|
||||
@@ -155,14 +184,37 @@ start_vpn() {
|
||||
fi
|
||||
fi
|
||||
|
||||
# Set up DNS
|
||||
# Set up DNS (handle comma-separated DNS servers)
|
||||
VPN_DNS=$(grep -i '^DNS' "$CONFIG_FILE" | head -1 | sed 's/.*= *//;s/ //g')
|
||||
if [ -n "$VPN_DNS" ]; then
|
||||
echo "nameserver $VPN_DNS" > /etc/resolv.conf
|
||||
echo "[vpn] DNS set to ${VPN_DNS}"
|
||||
# Clear resolv.conf and add each DNS server on its own line
|
||||
> /etc/resolv.conf
|
||||
for dns in $(echo "$VPN_DNS" | tr ',' ' '); do
|
||||
echo "nameserver $dns" >> /etc/resolv.conf
|
||||
# Add explicit route to DNS server through wg0 so it's found in main table
|
||||
# (suppress_prefixlength 0 ignores default routes but allows host routes)
|
||||
ip -4 route add "$dns" dev "$INTERFACE" 2>/dev/null || true
|
||||
done
|
||||
echo "[vpn] DNS set to: ${VPN_DNS}"
|
||||
fi
|
||||
|
||||
# Add explicit host route for the health-check target so it is picked up by
|
||||
# the 'lookup main suppress_prefixlength 0' rule (same as DNS servers above).
|
||||
# Without this, CHECK_HOST falls through to the VPN table default route whose
|
||||
# source-address selection can be defeated by the priority-100 'from ETH0_IP'
|
||||
# policy rule, causing pings to bypass wg0 and be dropped by the kill switch.
|
||||
ip -4 route add "${CHECK_HOST}" dev "$INTERFACE" 2>/dev/null || true
|
||||
echo "[vpn] Health-check route: ${CHECK_HOST} → ${INTERFACE}"
|
||||
|
||||
echo "[vpn] WireGuard interface ${INTERFACE} is up."
|
||||
echo "[vpn] Main routes:"
|
||||
ip route show | sed 's/^/[vpn] /'
|
||||
echo "[vpn] VPN table ($FW_TABLE):"
|
||||
ip route show table "$FW_TABLE" 2>/dev/null | sed 's/^/[vpn] /'
|
||||
echo "[vpn] Policy rules:"
|
||||
ip rule show | sed 's/^/[vpn] /'
|
||||
echo "[vpn] WireGuard status:"
|
||||
wg show "$INTERFACE" 2>/dev/null | sed 's/^/[vpn] /'
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
@@ -170,6 +222,21 @@ start_vpn() {
|
||||
# ──────────────────────────────────────────────
|
||||
stop_vpn() {
|
||||
echo "[vpn] Stopping WireGuard interface ${INTERFACE}..."
|
||||
|
||||
local FW_MARK=51820
|
||||
local FW_TABLE=51820
|
||||
|
||||
# Remove fwmark-based policy rules
|
||||
ip -4 rule del not fwmark "$FW_MARK" table "$FW_TABLE" 2>/dev/null || true
|
||||
ip -4 rule del table main suppress_prefixlength 0 2>/dev/null || true
|
||||
|
||||
# Flush VPN routing table
|
||||
ip -4 route flush table "$FW_TABLE" 2>/dev/null || true
|
||||
|
||||
# Remove LAN policy routing
|
||||
ip -4 rule del table 100 2>/dev/null || true
|
||||
ip -4 route flush table 100 2>/dev/null || true
|
||||
|
||||
ip link del "$INTERFACE" 2>/dev/null || true
|
||||
}
|
||||
|
||||
@@ -185,14 +252,31 @@ health_loop() {
|
||||
while true; do
|
||||
sleep "$CHECK_INTERVAL"
|
||||
|
||||
if curl -sf --max-time 5 "http://$CHECK_HOST" > /dev/null 2>&1; then
|
||||
if ping -c 1 -W 5 "$CHECK_HOST" > /dev/null 2>&1; then
|
||||
if [ "$failures" -gt 0 ]; then
|
||||
echo "[health] VPN recovered."
|
||||
failures=0
|
||||
fi
|
||||
# Secondary DNS check
|
||||
if ping -c 1 -W 5 "google.com" > /dev/null 2>&1; then
|
||||
: # DNS OK — silent
|
||||
else
|
||||
echo "[health] WARN google.com unreachable — possible DNS issue"
|
||||
fi
|
||||
else
|
||||
failures=$((failures + 1))
|
||||
echo "[health] Ping failed ($failures/$max_failures)"
|
||||
echo "[health] Check failed ($failures/$max_failures) — ping ${CHECK_HOST} failed"
|
||||
# Secondary check: distinguish IP failure from DNS failure
|
||||
if ping -c 1 -W 5 "google.com" > /dev/null 2>&1; then
|
||||
echo "[health] INFO google.com reachable — DNS works, ${CHECK_HOST} may be filtered"
|
||||
else
|
||||
echo "[health] INFO google.com also unreachable — DNS or general routing failure"
|
||||
fi
|
||||
# Dump WireGuard stats to show if handshake is stale and how much data flows
|
||||
echo "[health] wg stats:"
|
||||
wg show "$INTERFACE" 2>/dev/null | grep -E 'latest handshake|transfer|endpoint' | sed 's/^/[health] /' || echo "[health] wg0 not found"
|
||||
echo "[health] routes:"
|
||||
ip route show | grep -E 'wg0|default' | sed 's/^/[health] /'
|
||||
|
||||
if [ "$failures" -ge "$max_failures" ]; then
|
||||
echo "[health] VPN appears down. Restarting WireGuard..."
|
||||
@@ -221,8 +305,83 @@ cleanup() {
|
||||
|
||||
trap cleanup SIGTERM SIGINT
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# Startup connectivity checks — diagnose issues early
|
||||
# ──────────────────────────────────────────────
|
||||
check_vpn_connectivity() {
|
||||
echo "[check] ── Startup connectivity checks ──"
|
||||
|
||||
# 1. Wait for WireGuard handshake (up to 15s)
|
||||
local elapsed=0
|
||||
local handshake_ts=0
|
||||
echo "[check] Waiting for WireGuard handshake (up to 15s)..."
|
||||
while [ "$elapsed" -lt 15 ]; do
|
||||
handshake_ts=$(wg show "$INTERFACE" latest-handshakes 2>/dev/null | awk '{print $2}' | head -1)
|
||||
if [ -n "$handshake_ts" ] && [ "$handshake_ts" != "0" ]; then
|
||||
local age=$(( $(date +%s) - handshake_ts ))
|
||||
echo "[check] OK Handshake established ${age}s ago"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
elapsed=$((elapsed + 1))
|
||||
done
|
||||
if [ "$elapsed" -ge 15 ]; then
|
||||
echo "[check] FAIL No WireGuard handshake after 15s — tunnel is not up"
|
||||
echo "[check] This container's public key (must be on the server):"
|
||||
echo "[check] PublicKey = $(wg show "$INTERFACE" public-key 2>/dev/null || echo 'unknown')"
|
||||
echo "[check] AllowedIPs = ${VPN_ADDRESS}"
|
||||
echo "[check] Verify on server: wg show"
|
||||
fi
|
||||
|
||||
# 2. Check whether traffic actually flows through the tunnel
|
||||
echo "[check] Testing traffic through tunnel (ping ${CHECK_HOST})..."
|
||||
local rx_before
|
||||
rx_before=$(wg show "$INTERFACE" transfer 2>/dev/null | awk '{print $2}' | head -1)
|
||||
|
||||
if ping -c 1 -W 8 "${CHECK_HOST}" > /dev/null 2>&1; then
|
||||
echo "[check] OK Traffic flows — tunnel is fully working"
|
||||
else
|
||||
local rx_after
|
||||
rx_after=$(wg show "$INTERFACE" transfer 2>/dev/null | awk '{print $2}' | head -1)
|
||||
echo "[check] FAIL ping ${CHECK_HOST} unreachable through tunnel"
|
||||
|
||||
if [ -n "$rx_before" ] && [ -n "$rx_after" ]; then
|
||||
if [ "$rx_after" -le "$rx_before" ]; then
|
||||
echo "[check] RX bytes unchanged (${rx_before} → ${rx_after})"
|
||||
echo "[check] Server receives packets but does NOT route them back"
|
||||
echo "[check] Fix on VPN server (${VPN_ENDPOINT}):"
|
||||
echo "[check] sysctl net.ipv4.ip_forward # must output 1"
|
||||
echo "[check] iptables -t nat -L POSTROUTING -v -n # must have MASQUERADE"
|
||||
echo "[check] wg show # check peer + AllowedIPs"
|
||||
else
|
||||
echo "[check] RX increased (${rx_before} → ${rx_after}) — tunnel passes data"
|
||||
echo "[check] Issue may be specific to ${CHECK_HOST} or DNS"
|
||||
fi
|
||||
fi
|
||||
|
||||
local transfer
|
||||
transfer=$(wg show "$INTERFACE" transfer 2>/dev/null | awk '{printf "rx=%s tx=%s", $2, $3}')
|
||||
echo "[check] wg transfer: ${transfer}"
|
||||
fi
|
||||
|
||||
# 3. DNS check
|
||||
echo "[check] Testing DNS resolution..."
|
||||
if nslookup 1.1.1.1 > /dev/null 2>&1 || nslookup google.com > /dev/null 2>&1; then
|
||||
echo "[check] OK DNS resolves"
|
||||
else
|
||||
echo "[check] FAIL DNS resolution failed"
|
||||
echo "[check] resolv.conf: $(tr '\n' ' ' < /etc/resolv.conf)"
|
||||
echo "[check] Check that DNS servers are reachable through wg0"
|
||||
echo "[check] ── End of checks ──"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[check] ── End of checks ──"
|
||||
}
|
||||
|
||||
# ── Main ──
|
||||
enable_forwarding
|
||||
setup_killswitch
|
||||
start_vpn
|
||||
check_vpn_connectivity
|
||||
health_loop
|
||||
|
||||
@@ -1,17 +1,16 @@
|
||||
[Interface]
|
||||
PrivateKey = iO5spIue/6ciwUoR95hYtuxdtQxV/Q9EOoQ/jHe18kM=
|
||||
Address = 10.2.0.2/32
|
||||
DNS = 10.2.0.1
|
||||
PrivateKey = EPRa2f/v72LvIXAY4yqIRJifsSb+nCcYHqC2rwA94UI=
|
||||
Address = 100.64.244.78/32
|
||||
DNS = 198.18.0.1,198.18.0.2
|
||||
|
||||
# Route zum VPN-Server direkt über dein lokales Netz
|
||||
PostUp = ip route add 185.183.34.149 via 192.168.178.1 dev wlp4s0f0
|
||||
PostUp = ip route add 91.148.236.64 via 192.168.178.1 dev wlp4s0f0
|
||||
PostUp = ip route add 192.168.178.0/24 via 192.168.178.1 dev wlp4s0f0
|
||||
PostDown = ip route del 185.183.34.149 via 192.168.178.1 dev wlp4s0f0
|
||||
PostDown = ip route del 91.148.236.64 via 192.168.178.1 dev wlp4s0f0
|
||||
PostDown = ip route del 192.168.178.0/24 via 192.168.178.1 dev wlp4s0f0
|
||||
|
||||
[Peer]
|
||||
PublicKey = J4XVdtoBVc/EoI2Yk673Oes97WMnQSH5KfamZNjtM2s=
|
||||
AllowedIPs = 0.0.0.0/1, 128.0.0.0/1
|
||||
Endpoint = 185.183.34.149:51820
|
||||
|
||||
PublicKey = KgTUh3KLijVluDvNpzDCJJfrJ7EyLzYLmdHCksG4sRg=
|
||||
AllowedIPs = 0.0.0.0/0
|
||||
Endpoint = 91.148.236.64:51820
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ services:
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
- SYS_MODULE
|
||||
- NET_RAW
|
||||
sysctls:
|
||||
- net.ipv4.ip_forward=1
|
||||
- net.ipv4.conf.all.src_valid_mark=1
|
||||
@@ -22,7 +23,7 @@ services:
|
||||
- /server/server_aniworld/wg0.conf:/etc/wireguard/wg0.conf:ro
|
||||
- /lib/modules:/lib/modules:ro
|
||||
ports:
|
||||
- "2000:8000"
|
||||
- "8000:8000"
|
||||
environment:
|
||||
- HEALTH_CHECK_INTERVAL=10
|
||||
- HEALTH_CHECK_HOST=1.1.1.1
|
||||
@@ -51,4 +52,5 @@ services:
|
||||
volumes:
|
||||
- /server/server_aniworld/data:/app/data
|
||||
- /server/server_aniworld/logs:/app/logs
|
||||
- /media/serien/Serien:/data
|
||||
restart: unless-stopped
|
||||
|
||||
@@ -7,6 +7,7 @@ services:
|
||||
cap_add:
|
||||
- NET_ADMIN
|
||||
- SYS_MODULE
|
||||
- NET_RAW
|
||||
sysctls:
|
||||
- net.ipv4.ip_forward=1
|
||||
- net.ipv4.conf.all.src_valid_mark=1
|
||||
@@ -37,6 +38,7 @@ services:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
- PYTHONUNBUFFERED=1
|
||||
- LOG_LEVEL=DEBUG
|
||||
volumes:
|
||||
- app-data:/app/data
|
||||
- app-logs:/app/logs
|
||||
|
||||
101
Docker/push.sh
101
Docker/push.sh
@@ -1,15 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
# filepath: /home/lukas/Volume/repo/Aniworld/Docker/push.sh
|
||||
#
|
||||
# Build and push Aniworld container images to the Gitea registry.
|
||||
# Build and push AniWorld container images to the Gitea registry.
|
||||
#
|
||||
# Usage:
|
||||
# ./push.sh # builds & pushes with tag "latest"
|
||||
# ./push.sh v1.2.3 # builds & pushes with tag "v1.2.3"
|
||||
# ./push.sh v1.2.3 --no-build # pushes existing images only
|
||||
# ./push.sh # builds & pushes app with tag "latest"
|
||||
# ./push.sh app # builds & pushes app image
|
||||
# ./push.sh vpn # builds & pushes vpn image
|
||||
# ./push.sh all # builds & pushes both images
|
||||
# ./push.sh app v1.2.3 # builds & pushes app with tag "v1.2.3"
|
||||
# ./push.sh vpn v1.2.3 # builds & pushes vpn with tag "v1.2.3"
|
||||
# ./push.sh all v1.2.3 # builds & pushes both images
|
||||
# ./push.sh app v1.2.3 --no-build # pushes existing image only
|
||||
#
|
||||
# Prerequisites:
|
||||
# podman login git.lpl-mind.de
|
||||
# podman login git.lpl-mind.de (or: docker login git.lpl-mind.de)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
@@ -23,12 +27,20 @@ PROJECT="aniworld"
|
||||
APP_IMAGE="${REGISTRY}/${NAMESPACE}/${PROJECT}/app"
|
||||
VPN_IMAGE="${REGISTRY}/${NAMESPACE}/${PROJECT}/vpn"
|
||||
|
||||
TAG="${1:-latest}"
|
||||
# Parse arguments
|
||||
TARGET="${1:-app}"
|
||||
TAG="${2:-latest}"
|
||||
SKIP_BUILD=false
|
||||
if [[ "${2:-}" == "--no-build" ]]; then
|
||||
if [[ "${3:-}" == "--no-build" ]]; then
|
||||
SKIP_BUILD=true
|
||||
fi
|
||||
|
||||
# Validate target
|
||||
if [[ "${TARGET}" != "app" && "${TARGET}" != "vpn" && "${TARGET}" != "all" ]]; then
|
||||
echo "ERROR: Invalid target '${TARGET}'. Must be one of: app, vpn, all" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
@@ -36,62 +48,93 @@ PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
log() { echo -e "\n>>> $*"; }
|
||||
err() { echo -e "\n❌ ERROR: $*" >&2; exit 1; }
|
||||
err() { echo -e "\nERROR: $*" >&2; exit 1; }
|
||||
|
||||
# Detect container engine (podman preferred, docker fallback)
|
||||
if command -v podman &>/dev/null; then
|
||||
ENGINE="podman"
|
||||
elif command -v docker &>/dev/null; then
|
||||
ENGINE="docker"
|
||||
else
|
||||
err "Neither podman nor docker is installed."
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pre-flight checks
|
||||
# ---------------------------------------------------------------------------
|
||||
echo "============================================"
|
||||
echo " Aniworld — Build & Push"
|
||||
echo " AniWorld — Build & Push"
|
||||
echo " Engine : ${ENGINE}"
|
||||
echo " Registry : ${REGISTRY}"
|
||||
echo " Target : ${TARGET}"
|
||||
echo " Tag : ${TAG}"
|
||||
echo "============================================"
|
||||
|
||||
command -v podman &>/dev/null || err "podman is not installed."
|
||||
|
||||
if ! podman login --get-login "${REGISTRY}" &>/dev/null; then
|
||||
err "Not logged in. Run:\n podman login ${REGISTRY}"
|
||||
fi
|
||||
log "Logging in to ${REGISTRY}"
|
||||
"${ENGINE}" login "${REGISTRY}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Build
|
||||
# ---------------------------------------------------------------------------
|
||||
if [[ "${SKIP_BUILD}" == false ]]; then
|
||||
build_app() {
|
||||
log "Building app image → ${APP_IMAGE}:${TAG}"
|
||||
podman build \
|
||||
"${ENGINE}" build \
|
||||
-t "${APP_IMAGE}:${TAG}" \
|
||||
-f "${SCRIPT_DIR}/Dockerfile.app" \
|
||||
"${PROJECT_ROOT}"
|
||||
}
|
||||
|
||||
log "Building VPN image → ${VPN_IMAGE}:${TAG}"
|
||||
podman build \
|
||||
build_vpn() {
|
||||
log "Building vpn image → ${VPN_IMAGE}:${TAG}"
|
||||
"${ENGINE}" build \
|
||||
-t "${VPN_IMAGE}:${TAG}" \
|
||||
-f "${SCRIPT_DIR}/Containerfile" \
|
||||
"${SCRIPT_DIR}"
|
||||
}
|
||||
|
||||
if [[ "${SKIP_BUILD}" == false ]]; then
|
||||
case "${TARGET}" in
|
||||
app) build_app ;;
|
||||
vpn) build_vpn ;;
|
||||
all) build_app; build_vpn ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Push
|
||||
# ---------------------------------------------------------------------------
|
||||
log "Pushing ${APP_IMAGE}:${TAG}"
|
||||
podman push "${APP_IMAGE}:${TAG}"
|
||||
push_app() {
|
||||
log "Pushing ${APP_IMAGE}:${TAG}"
|
||||
"${ENGINE}" push "${APP_IMAGE}:${TAG}"
|
||||
}
|
||||
|
||||
log "Pushing ${VPN_IMAGE}:${TAG}"
|
||||
podman push "${VPN_IMAGE}:${TAG}"
|
||||
push_vpn() {
|
||||
log "Pushing ${VPN_IMAGE}:${TAG}"
|
||||
"${ENGINE}" push "${VPN_IMAGE}:${TAG}"
|
||||
}
|
||||
|
||||
case "${TARGET}" in
|
||||
app) push_app ;;
|
||||
vpn) push_vpn ;;
|
||||
all) push_app; push_vpn ;;
|
||||
esac
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summary
|
||||
# ---------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "============================================"
|
||||
echo " ✅ Push complete!"
|
||||
echo " Push complete!"
|
||||
echo ""
|
||||
echo " Images:"
|
||||
echo " ${APP_IMAGE}:${TAG}"
|
||||
echo " ${VPN_IMAGE}:${TAG}"
|
||||
case "${TARGET}" in
|
||||
app) echo " ${APP_IMAGE}:${TAG}" ;;
|
||||
vpn) echo " ${VPN_IMAGE}:${TAG}" ;;
|
||||
all) echo " ${APP_IMAGE}:${TAG}"; echo " ${VPN_IMAGE}:${TAG}" ;;
|
||||
esac
|
||||
echo ""
|
||||
echo " Deploy on server:"
|
||||
echo " podman login ${REGISTRY}"
|
||||
echo " podman-compose -f podman-compose.prod.yml pull"
|
||||
echo " podman-compose -f podman-compose.prod.yml up -d"
|
||||
echo " ${ENGINE} login ${REGISTRY}"
|
||||
echo " ${ENGINE} compose -f Docker/podman-compose.prod.yml pull"
|
||||
echo " ${ENGINE} compose -f Docker/podman-compose.prod.yml up -d"
|
||||
echo "============================================"
|
||||
129
Docker/release.sh
Normal file
129
Docker/release.sh
Normal file
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Bump the project version and push images to the registry.
|
||||
#
|
||||
# Usage:
|
||||
# ./release.sh
|
||||
#
|
||||
# The current version is stored in VERSION (next to this script).
|
||||
# You will be asked whether to bump major, minor, or patch.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
VERSION_FILE="${SCRIPT_DIR}/VERSION"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Read current version
|
||||
# ---------------------------------------------------------------------------
|
||||
if [[ ! -f "${VERSION_FILE}" ]]; then
|
||||
echo "0.0.0" > "${VERSION_FILE}"
|
||||
fi
|
||||
|
||||
CURRENT="$(cat "${VERSION_FILE}")"
|
||||
# Strip leading 'v' for arithmetic
|
||||
VERSION="${CURRENT#v}"
|
||||
|
||||
IFS='.' read -r MAJOR MINOR PATCH <<< "${VERSION}"
|
||||
|
||||
echo "============================================"
|
||||
echo " AniWorld — Release"
|
||||
echo " Current version: v${MAJOR}.${MINOR}.${PATCH}"
|
||||
echo "============================================"
|
||||
echo ""
|
||||
echo "Which image(s) would you like to release?"
|
||||
echo " 1) app (Dockerfile.app)"
|
||||
echo " 2) vpn (Containerfile)"
|
||||
echo " 3) all (both images)"
|
||||
echo ""
|
||||
read -rp "Enter choice [1/2/3]: " TARGET_CHOICE
|
||||
|
||||
case "${TARGET_CHOICE}" in
|
||||
1) TARGET="app" ;;
|
||||
2) TARGET="vpn" ;;
|
||||
3) TARGET="all" ;;
|
||||
*)
|
||||
echo "Invalid choice. Aborting." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo ""
|
||||
echo "How would you like to bump the version?"
|
||||
echo " 1) patch (v${MAJOR}.${MINOR}.${PATCH} → v${MAJOR}.${MINOR}.$((PATCH + 1)))"
|
||||
echo " 2) minor (v${MAJOR}.${MINOR}.${PATCH} → v${MAJOR}.$((MINOR + 1)).0)"
|
||||
echo " 3) major (v${MAJOR}.${MINOR}.${PATCH} → v$((MAJOR + 1)).0.0)"
|
||||
echo ""
|
||||
read -rp "Enter choice [1/2/3]: " CHOICE
|
||||
|
||||
case "${CHOICE}" in
|
||||
1) NEW_TAG="v${MAJOR}.${MINOR}.$((PATCH + 1))" ;;
|
||||
2) NEW_TAG="v${MAJOR}.$((MINOR + 1)).0" ;;
|
||||
3) NEW_TAG="v$((MAJOR + 1)).0.0" ;;
|
||||
*)
|
||||
echo "Invalid choice. Aborting." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo ""
|
||||
echo "New version: ${NEW_TAG}"
|
||||
echo "Target: ${TARGET}"
|
||||
read -rp "Confirm? [y/N]: " CONFIRM
|
||||
if [[ ! "${CONFIRM}" =~ ^[yY]$ ]]; then
|
||||
echo "Aborted."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Write new version
|
||||
# ---------------------------------------------------------------------------
|
||||
echo "${NEW_TAG}" > "${VERSION_FILE}"
|
||||
echo "Version file updated → ${VERSION_FILE}"
|
||||
|
||||
# Keep root package.json in sync.
|
||||
FRONT_VERSION="${NEW_TAG#v}"
|
||||
FRONT_PKG="${SCRIPT_DIR}/../package.json"
|
||||
if [[ -f "${FRONT_PKG}" ]]; then
|
||||
sed -i "s/\"version\": \"[^\"]*\"/\"version\": \"${FRONT_VERSION}\"/" "${FRONT_PKG}"
|
||||
echo "package.json version updated → ${FRONT_VERSION}"
|
||||
else
|
||||
echo "Warning: package.json not found, skipping package.json version sync" >&2
|
||||
fi
|
||||
|
||||
# Keep root pyproject.toml in sync.
|
||||
BACKEND_PYPROJECT="${SCRIPT_DIR}/../pyproject.toml"
|
||||
if [[ -f "${BACKEND_PYPROJECT}" ]]; then
|
||||
# Update version under [project] section if present
|
||||
if grep -q '^\[project\]' "${BACKEND_PYPROJECT}"; then
|
||||
sed -i "/^\[project\]/,/^\[/ s/^version = \".*\"/version = \"${FRONT_VERSION}\"/" "${BACKEND_PYPROJECT}"
|
||||
else
|
||||
sed -i "s/^version = \".*\"/version = \"${FRONT_VERSION}\"/" "${BACKEND_PYPROJECT}"
|
||||
fi
|
||||
echo "pyproject.toml version updated → ${FRONT_VERSION}"
|
||||
else
|
||||
echo "Warning: pyproject.toml not found, skipping pyproject.toml version sync" >&2
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Push containers
|
||||
# ---------------------------------------------------------------------------
|
||||
bash "${SCRIPT_DIR}/push.sh" "${TARGET}" "${NEW_TAG}"
|
||||
bash "${SCRIPT_DIR}/push.sh" "${TARGET}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Git tag (local only; push after container build)
|
||||
# ---------------------------------------------------------------------------
|
||||
cd "${SCRIPT_DIR}/.."
|
||||
git add Docker/VERSION package.json pyproject.toml
|
||||
git commit -m "chore: bump version"
|
||||
git tag -a "${NEW_TAG}" -m "Release ${NEW_TAG}"
|
||||
echo "Local git commit + tag ${NEW_TAG} created."
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Push git commits & tag
|
||||
# ---------------------------------------------------------------------------
|
||||
git push origin HEAD
|
||||
git push origin "${NEW_TAG}"
|
||||
echo "Git commit and tag ${NEW_TAG} pushed."
|
||||
@@ -6,23 +6,29 @@ Verifies:
|
||||
2. The container starts and becomes healthy.
|
||||
3. The public IP inside the VPN differs from the host IP.
|
||||
4. Kill switch blocks traffic when WireGuard is down.
|
||||
5. AllowedIPs routes are set dynamically from the config.
|
||||
|
||||
Requirements:
|
||||
- podman installed
|
||||
- Root/sudo (NET_ADMIN capability)
|
||||
- Root/sudo (NET_ADMIN capability) for container runtime tests
|
||||
- A valid WireGuard config at ./wg0.conf (or ./nl.conf)
|
||||
|
||||
Usage:
|
||||
# Build-only test (no sudo needed):
|
||||
python3 -m pytest test_vpn.py::TestVPNImage::test_00_build_image -v
|
||||
|
||||
# Full integration test (requires sudo):
|
||||
sudo python3 -m pytest test_vpn.py -v
|
||||
# or
|
||||
sudo python3 test_vpn.py
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
import os
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -35,6 +41,11 @@ STARTUP_TIMEOUT = 30 # seconds to wait for VPN to come up
|
||||
HEALTH_POLL_INTERVAL = 2 # seconds between health checks
|
||||
|
||||
|
||||
def is_root() -> bool:
|
||||
"""Check if running as root."""
|
||||
return os.geteuid() == 0
|
||||
|
||||
|
||||
def run(cmd: list[str], timeout: int = 30, check: bool = True) -> subprocess.CompletedProcess:
|
||||
"""Run a command and return the result."""
|
||||
return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, check=check)
|
||||
@@ -55,6 +66,7 @@ class TestVPNImage(unittest.TestCase):
|
||||
"""Test suite for the WireGuard VPN container."""
|
||||
|
||||
host_ip: str = ""
|
||||
container_id: str = ""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
@@ -84,6 +96,12 @@ class TestVPNImage(unittest.TestCase):
|
||||
assert result.returncode == 0, f"Build failed:\n{result.stderr}"
|
||||
logger.info("Image built successfully.")
|
||||
|
||||
# Skip container runtime tests if not root
|
||||
if not is_root():
|
||||
logger.warning("Not running as root — skipping container runtime tests.")
|
||||
cls.container_id = ""
|
||||
return
|
||||
|
||||
# ── 3. Start the container ──
|
||||
logger.info("Starting container '%s'...", CONTAINER_NAME)
|
||||
result = run(
|
||||
@@ -120,6 +138,8 @@ class TestVPNImage(unittest.TestCase):
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
"""Stop and remove the container."""
|
||||
if not is_root():
|
||||
return
|
||||
logger.info("Cleaning up test container...")
|
||||
subprocess.run(["podman", "rm", "-f", CONTAINER_NAME], capture_output=True, check=False)
|
||||
logger.info("Cleanup complete.")
|
||||
@@ -144,10 +164,22 @@ class TestVPNImage(unittest.TestCase):
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
def _skip_if_not_root(self):
|
||||
"""Skip test if not running as root."""
|
||||
if not is_root():
|
||||
self.skipTest("This test requires root/sudo privileges")
|
||||
|
||||
# ── Tests ────────────────────────────────────────────────
|
||||
|
||||
def test_00_build_image(self):
|
||||
"""The image builds successfully."""
|
||||
# This is already verified in setUpClass, just confirm here
|
||||
result = run(["podman", "images", "--format", "{{.Repository}}:{{.Tag}}"])
|
||||
self.assertIn(IMAGE_NAME, result.stdout, "Image was not built")
|
||||
|
||||
def test_01_ip_differs_from_host(self):
|
||||
"""Public IP inside VPN is different from host IP."""
|
||||
self._skip_if_not_root()
|
||||
vpn_ip = self._get_vpn_ip()
|
||||
logger.info("VPN public IP: %s", vpn_ip)
|
||||
logger.info("Host public IP: %s", self.host_ip)
|
||||
@@ -161,12 +193,42 @@ class TestVPNImage(unittest.TestCase):
|
||||
|
||||
def test_02_wireguard_interface_exists(self):
|
||||
"""The wg0 interface is present in the container."""
|
||||
self._skip_if_not_root()
|
||||
result = podman_exec(CONTAINER_NAME, ["wg", "show", "wg0"])
|
||||
self.assertEqual(result.returncode, 0, f"wg show failed:\n{result.stderr}")
|
||||
self.assertIn("peer", result.stdout.lower(), "No peer information in wg show output")
|
||||
# AllowedIPs should be present in wg show output
|
||||
self.assertIn("allowed ips", result.stdout.lower(), "AllowedIPs not found in wg show output")
|
||||
|
||||
def test_03_kill_switch_blocks_traffic(self):
|
||||
def test_03_allowedips_routes_set(self):
|
||||
"""Routes are set dynamically based on AllowedIPs from config."""
|
||||
self._skip_if_not_root()
|
||||
# Check that routes exist for the AllowedIPs
|
||||
result = podman_exec(CONTAINER_NAME, ["ip", "route", "show", "dev", "wg0"])
|
||||
self.assertEqual(result.returncode, 0, f"ip route show failed:\n{result.stderr}")
|
||||
# The config has AllowedIPs = 0.0.0.0/0, which should result in:
|
||||
# 0.0.0.0/1 dev wg0 and 128.0.0.0/1 dev wg0
|
||||
self.assertIn("0.0.0.0/1", result.stdout, "Route 0.0.0.0/1 not found")
|
||||
self.assertIn("128.0.0.0/1", result.stdout, "Route 128.0.0.0/1 not found")
|
||||
# Make sure there is NO default route through wg0 (Table = off should prevent this)
|
||||
self.assertNotIn("default dev wg0", result.stdout, "Default route through wg0 found — Table = off not working!")
|
||||
logger.info("AllowedIPs routes verified: %s", result.stdout.strip())
|
||||
|
||||
def test_03b_dns_configured(self):
|
||||
"""DNS is configured correctly with multiple nameserver lines."""
|
||||
self._skip_if_not_root()
|
||||
result = podman_exec(CONTAINER_NAME, ["cat", "/etc/resolv.conf"])
|
||||
self.assertEqual(result.returncode, 0, f"cat /etc/resolv.conf failed:\n{result.stderr}")
|
||||
# Should have two separate nameserver lines, not one with commas
|
||||
self.assertIn("nameserver 198.18.0.1", result.stdout, "DNS 198.18.0.1 not found")
|
||||
self.assertIn("nameserver 198.18.0.2", result.stdout, "DNS 198.18.0.2 not found")
|
||||
# Make sure there are no commas in nameserver lines
|
||||
self.assertNotIn("nameserver 198.18.0.1,198.18.0.2", result.stdout, "DNS servers written on one line with comma!")
|
||||
logger.info("DNS config verified: %s", result.stdout.strip())
|
||||
|
||||
def test_04_kill_switch_blocks_traffic(self):
|
||||
"""When WireGuard is down, traffic is blocked (kill switch)."""
|
||||
self._skip_if_not_root()
|
||||
# Bring down the WireGuard interface by deleting it
|
||||
down_result = podman_exec(CONTAINER_NAME, ["ip", "link", "del", "wg0"], timeout=10)
|
||||
self.assertEqual(down_result.returncode, 0, f"ip link del wg0 failed:\n{down_result.stderr}")
|
||||
|
||||
@@ -1,10 +1,18 @@
|
||||
[Interface]
|
||||
PrivateKey = iO5spIue/6ciwUoR95hYtuxdtQxV/Q9EOoQ/jHe18kM=
|
||||
Address = 10.2.0.2/32
|
||||
DNS = 10.2.0.1
|
||||
PrivateKey = EPRa2f/v72LvIXAY4yqIRJifsSb+nCcYHqC2rwA94UI=
|
||||
Address = 100.64.244.78/32
|
||||
#DNS = 198.18.0.1,198.18.0.2
|
||||
DNS = 8.8.8.8
|
||||
|
||||
# Route zum VPN-Server direkt über dein lokales Netz
|
||||
PostUp = ip route add 91.148.236.64 via 192.168.178.1 dev wlp4s0f0
|
||||
PostUp = ip route add 192.168.178.0/24 via 192.168.178.1 dev wlp4s0f0
|
||||
PostDown = ip route del 91.148.236.64 via 192.168.178.1 dev wlp4s0f0
|
||||
PostDown = ip route del 192.168.178.0/24 via 192.168.178.1 dev wlp4s0f0
|
||||
|
||||
[Peer]
|
||||
PublicKey = J4XVdtoBVc/EoI2Yk673Oes97WMnQSH5KfamZNjtM2s=
|
||||
PublicKey = KgTUh3KLijVluDvNpzDCJJfrJ7EyLzYLmdHCksG4sRg=
|
||||
AllowedIPs = 0.0.0.0/0
|
||||
Endpoint = 185.183.34.149:51820
|
||||
Endpoint = 91.148.236.64:51820
|
||||
PersistentKeepalive = 25
|
||||
|
||||
|
||||
@@ -1285,7 +1285,7 @@ Basic health check endpoint.
|
||||
{
|
||||
"status": "healthy",
|
||||
"timestamp": "2025-12-13T10:30:00.000Z",
|
||||
"version": "1.0.0"
|
||||
"version": "1.0.1"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -1303,7 +1303,7 @@ Comprehensive health check with database, filesystem, and system metrics.
|
||||
{
|
||||
"status": "healthy",
|
||||
"timestamp": "2025-12-13T10:30:00.000Z",
|
||||
"version": "1.0.0",
|
||||
"version": "1.0.1",
|
||||
"dependencies": {
|
||||
"database": {
|
||||
"status": "healthy",
|
||||
|
||||
@@ -81,6 +81,8 @@ src/server/
|
||||
| +-- websocket_service.py# WebSocket broadcasting
|
||||
| +-- queue_repository.py # Database persistence
|
||||
| +-- nfo_service.py # NFO metadata management
|
||||
| +-- setup_service.py # Series key resolution from folder names
|
||||
| +-- folder_scan_service.py # Daily folder maintenance scan
|
||||
+-- models/ # Pydantic models
|
||||
| +-- auth.py # Auth request/response models
|
||||
| +-- config.py # Configuration models
|
||||
@@ -290,8 +292,9 @@ The FastAPI lifespan function (`src/server/fastapi_app.py`) runs the following s
|
||||
8. Background loader service started
|
||||
|
||||
9. Scheduler service started
|
||||
|
||||
10. NFO repair scan (queue incomplete tvshow.nfo files for background reload)
|
||||
+-- Cron-based library rescans configured
|
||||
+-- Optional: auto-download missing episodes after rescan
|
||||
+-- Optional: folder maintenance (NFO repair, key resolution, renaming, poster checks) during scheduled runs
|
||||
```
|
||||
|
||||
### 12.2 Temp Folder Guarantee
|
||||
|
||||
@@ -37,10 +37,30 @@ This changelog follows [Keep a Changelog](https://keepachangelog.com/) principle
|
||||
|
||||
---
|
||||
|
||||
## [Unreleased] - 2026-06-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Folder scan series key resolution**: Fixed "Could not resolve series key for folder, skipping" warnings during library setup. `_resolve_key_via_search()` now uses fuzzy title matching instead of exact string comparison.
|
||||
- Added `_normalize_title()` to strip anime suffixes: `(TV)`, `(Anime)`, `(OAD)`, `(OVA)`, `(Special)`, `(Movie)`, `(Spin-Off)`
|
||||
- Added `_titles_match()` using `difflib.SequenceMatcher` with 0.85 similarity threshold for tolerance of minor title variations
|
||||
- Added debug logging for title mismatches and multiple search results
|
||||
|
||||
---
|
||||
|
||||
## [1.3.1] - 2026-02-22
|
||||
|
||||
### Added
|
||||
|
||||
- **Encoding detection for HTML parsing** (`src/core/providers/aniworld_provider.py`):
|
||||
Added `_decode_html_content()` function that uses `chardet` to detect the actual
|
||||
encoding of HTML content before parsing. Falls back to UTF-8 with `errors='replace'`
|
||||
to handle pages with mismatched encoding declarations. Applied to all BeautifulSoup
|
||||
parsing calls to prevent "Some characters could not be decoded" warnings.
|
||||
- **chardet dependency**: Added `chardet>=5.2.0` to `requirements.txt` for encoding detection.
|
||||
|
||||
### Added
|
||||
|
||||
- **Temp file cleanup after every download** (`src/core/providers/aniworld_provider.py`,
|
||||
`src/core/providers/enhanced_provider.py`): Module-level helper
|
||||
`_cleanup_temp_file()` removes the working temp file and any yt-dlp `.part`
|
||||
@@ -73,17 +93,16 @@ This changelog follows [Keep a Changelog](https://keepachangelog.com/) principle
|
||||
that detects incomplete `tvshow.nfo` files and triggers TMDB re-fetch.
|
||||
Provides `parse_nfo_tags()`, `find_missing_tags()`, `nfo_needs_repair()`, and
|
||||
`NfoRepairService.repair_series()`. 13 required tags are checked.
|
||||
- **`perform_nfo_repair_scan()` startup hook
|
||||
(`src/server/services/initialization_service.py`)**: New async function
|
||||
called during application startup. Iterates every series directory, checks
|
||||
whether `tvshow.nfo` is missing required tags using `nfo_needs_repair()`, and
|
||||
either queues the series for background reload (when a `background_loader` is
|
||||
provided) or calls `NfoRepairService.repair_series()` directly. Skips
|
||||
gracefully when `tmdb_api_key` or `anime_directory` is not configured.
|
||||
- **NFO repair wired into startup lifespan (`src/server/fastapi_app.py`)**:
|
||||
`perform_nfo_repair_scan(background_loader)` is called at the end of the
|
||||
FastAPI lifespan startup, after `perform_media_scan_if_needed`, ensuring
|
||||
every existing series NFO is checked and repaired on each server start.
|
||||
- **`perform_nfo_repair_scan()`
|
||||
(`src/server/services/folder_scan_service.py`)**: New async function
|
||||
that iterates every series directory, checks whether `tvshow.nfo` is missing
|
||||
required tags using `nfo_needs_repair()`, and queues the series for background
|
||||
reload via `asyncio.create_task`. Skips gracefully when `tmdb_api_key` or
|
||||
`anime_directory` is not configured.
|
||||
- **NFO repair wired into scheduled folder scan (`src/server/services/folder_scan_service.py`)**:
|
||||
`perform_nfo_repair_scan(background_loader=None)` is called during the
|
||||
scheduled daily folder scan, keeping startup fast while ensuring regular
|
||||
maintenance.
|
||||
|
||||
### Changed
|
||||
|
||||
@@ -131,6 +150,10 @@ This changelog follows [Keep a Changelog](https://keepachangelog.com/) principle
|
||||
- Modified `src/server/api/anime.py` to save scanned episodes to database
|
||||
- Episodes table properly tracks missing episodes with automatic cleanup
|
||||
|
||||
### Deprecated
|
||||
|
||||
- **Legacy Series Files (key/data)**: File-based series storage is deprecated. `key` and `data` files in anime folders will be removed in v3.0.0. Database storage is now the primary method. See [docs/MIGRATION_GUIDE.md](docs/MIGRATION_GUIDE.md) for details.
|
||||
|
||||
---
|
||||
|
||||
## Sections for Each Release
|
||||
|
||||
@@ -117,7 +117,8 @@ Location: `data/config.json`
|
||||
"interval_minutes": 60,
|
||||
"schedule_time": "03:00",
|
||||
"schedule_days": ["mon", "tue", "wed", "thu", "fri", "sat", "sun"],
|
||||
"auto_download_after_rescan": false
|
||||
"auto_download_after_rescan": false,
|
||||
"folder_scan_enabled": false
|
||||
},
|
||||
"logging": {
|
||||
"level": "INFO",
|
||||
@@ -143,7 +144,7 @@ Location: `data/config.json`
|
||||
"master_password_hash": "$pbkdf2-sha256$...",
|
||||
"anime_directory": "/path/to/anime"
|
||||
},
|
||||
"version": "1.0.0"
|
||||
"version": "1.0.1"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -173,6 +174,7 @@ Controls automatic cron-based library rescanning (powered by APScheduler).
|
||||
| `scheduler.schedule_time` | string | `"03:00"` | Daily run time in 24-h `HH:MM` format. |
|
||||
| `scheduler.schedule_days` | list[string] | `["mon","tue","wed","thu","fri","sat","sun"]` | Days of the week to run the scan. Empty list disables the cron job. |
|
||||
| `scheduler.auto_download_after_rescan` | bool | `false` | Automatically queue missing episodes for download after each rescan. |
|
||||
| `scheduler.folder_scan_enabled` | bool | `false` | Run folder maintenance (NFO repair, folder renaming, poster checks) during scheduled runs. **When enabled, series folders are automatically renamed to match the `<title> (<year>)` convention derived from their `tvshow.nfo` files.** |
|
||||
|
||||
Valid day abbreviations: `mon`, `tue`, `wed`, `thu`, `fri`, `sat`, `sun`.
|
||||
|
||||
@@ -216,6 +218,7 @@ Source: [src/server/models/config.py](../src/server/models/config.py#L15-L24)
|
||||
- Obtain a TMDB API key from https://www.themoviedb.org/settings/api
|
||||
- `auto_create` creates NFO files during the download process
|
||||
- `update_on_scan` refreshes metadata when scanning existing anime
|
||||
- `download_poster` also controls whether the scheduled folder scan checks for and re-downloads missing or corrupted `poster.jpg` files (see [NFO_GUIDE.md](NFO_GUIDE.md#6-poster-check))
|
||||
- Image downloads require valid `tmdb_api_key`
|
||||
- `TMDB_API_KEY` environment variable is optional when `nfo.tmdb_api_key` is configured in `data/config.json`
|
||||
- Larger image sizes (`w780`, `original`) consume more storage space
|
||||
|
||||
216
docs/DATABASE.md
216
docs/DATABASE.md
@@ -83,17 +83,23 @@ Source: [src/server/database/models.py](../src/server/database/models.py), [src/
|
||||
|
||||
### 3.2 anime_series
|
||||
|
||||
Stores anime series metadata.
|
||||
Stores anime series metadata. Corresponds to the core `Serie` class.
|
||||
|
||||
| Column | Type | Constraints | Description |
|
||||
| ------------ | ------------- | -------------------------- | ------------------------------------------------------- |
|
||||
| `id` | INTEGER | PRIMARY KEY, AUTOINCREMENT | Internal database ID |
|
||||
| `key` | VARCHAR(255) | UNIQUE, NOT NULL, INDEX | **Primary identifier** - provider-assigned URL-safe key |
|
||||
| `name` | VARCHAR(500) | NOT NULL, INDEX | Display name of the series |
|
||||
| `site` | VARCHAR(500) | NOT NULL | Provider site URL |
|
||||
| `folder` | VARCHAR(1000) | NOT NULL | Filesystem folder name (metadata only) |
|
||||
| `created_at` | DATETIME | NOT NULL, DEFAULT NOW | Record creation timestamp |
|
||||
| `updated_at` | DATETIME | NOT NULL, ON UPDATE NOW | Last update timestamp |
|
||||
| Column | Type | Constraints | Description |
|
||||
| ---------------- | ------------- | -------------------------- | ------------------------------------------------------- |
|
||||
| `id` | INTEGER | PRIMARY KEY, AUTOINCREMENT | Internal database ID |
|
||||
| `key` | VARCHAR(255) | UNIQUE, NOT NULL, INDEX | **Primary identifier** - provider-assigned URL-safe key |
|
||||
| `name` | VARCHAR(500) | NOT NULL, INDEX | Display name of the series |
|
||||
| `site` | VARCHAR(500) | NOT NULL | Provider site URL |
|
||||
| `folder` | VARCHAR(1000) | NOT NULL | Filesystem folder name (metadata only) |
|
||||
| `year` | INTEGER | NULLABLE | Release year of the series |
|
||||
| `nfo_path` | VARCHAR(1000) | NULLABLE | Path to tvshow.nfo metadata file |
|
||||
| `tmdb_id` | INTEGER | NULLABLE, INDEX | TMDB (The Movie Database) ID for metadata |
|
||||
| `tvdb_id` | INTEGER | NULLABLE, INDEX | TVDB (TheTVDB) ID for metadata |
|
||||
| `has_nfo` | BOOLEAN | NOT NULL, DEFAULT FALSE | Whether tvshow.nfo exists |
|
||||
| `loading_status` | VARCHAR(50) | NOT NULL, DEFAULT 'completed' | Status: pending, loading_episodes, loading_nfo, completed, failed |
|
||||
| `created_at` | DATETIME | NOT NULL, DEFAULT NOW | Record creation timestamp |
|
||||
| `updated_at` | DATETIME | NOT NULL, ON UPDATE NOW | Last update timestamp |
|
||||
|
||||
**Identifier Convention:**
|
||||
|
||||
@@ -101,7 +107,13 @@ Stores anime series metadata.
|
||||
- `folder` is **metadata only** for filesystem operations (e.g., `"Attack on Titan (2013)"`)
|
||||
- `id` is used only for database relationships
|
||||
|
||||
Source: [src/server/database/models.py](../src/server/database/models.py#L23-L87)
|
||||
**EpisodeDict Mapping:**
|
||||
|
||||
The `episodeDict` (season → episode numbers mapping) is stored as individual `Episode` records:
|
||||
- Each `Episode` has `season` and `episode_number` columns
|
||||
- Relationship: `AnimeSeries.episodes` returns all Episode records for that series
|
||||
|
||||
Source: [src/server/database/models.py](../src/server/database/models.py#L23-L150)
|
||||
|
||||
### 3.3 episodes
|
||||
|
||||
@@ -441,7 +453,187 @@ items = await db.execute(
|
||||
|
||||
---
|
||||
|
||||
## 12. Database Location
|
||||
## 12. Series Storage: Database vs Files (Deprecated)
|
||||
|
||||
### File-Based Storage (Removed in v2.0)
|
||||
|
||||
Prior to v2.0, series metadata was stored in two files per anime folder:
|
||||
|
||||
| File | Contents |
|
||||
| -------- | ------------------------------------------------------- |
|
||||
| `key` | Series provider key (e.g., `"attack-on-titan"`) |
|
||||
| `data` | JSON serialization of `Serie` object |
|
||||
|
||||
File structure example:
|
||||
```
|
||||
/anime/Attack on Titan (2013)/
|
||||
├── key # Contains: attack-on-titan
|
||||
├── data # Contains: {"key": "...", "name": "...", "episodeDict": {...}}
|
||||
├── Season 1/
|
||||
│ └── ...
|
||||
```
|
||||
|
||||
### Database Storage (Current)
|
||||
|
||||
Since v2.0, all series metadata is stored in the `anime_series` table with `Episode` records for episode tracking. This provides:
|
||||
|
||||
- **ACID transactions** for data consistency
|
||||
- **Foreign key constraints** (cascade delete)
|
||||
- **Indexed queries** for fast lookups
|
||||
- **No filesystem dependency** for metadata
|
||||
|
||||
### Migration from Files to Database
|
||||
|
||||
The `Serie.save_to_file()` and `Serie.load_from_file()` methods are deprecated but still functional for backward compatibility during migration:
|
||||
|
||||
```python
|
||||
from src.core.entities.series import Serie
|
||||
|
||||
# Old file-based loading (deprecated)
|
||||
serie = Serie.load_from_file("/anime/Attack on Titan (2013)/data")
|
||||
|
||||
# New database-based loading
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
serie = await AnimeSeriesService.get_by_key(db, "attack-on-titan")
|
||||
```
|
||||
|
||||
### Removing File Dependencies
|
||||
|
||||
After verifying database schema supports all fields, file-based storage can be removed:
|
||||
|
||||
1. ✅ Schema verified: All `Serie` fields have corresponding DB columns
|
||||
2. ✅ Migration complete: All existing series migrated to database
|
||||
3. ❌ File cleanup: Remove `key` and `data` files (pending)
|
||||
|
||||
**Note:** The `save_to_file()` and `load_from_file()` methods will be removed in v3.0.0.
|
||||
|
||||
---
|
||||
|
||||
## 12. Series Persistence Flow
|
||||
|
||||
When a directory scan discovers or updates series, the scanner persists data to the database instead of writing to disk files.
|
||||
|
||||
### Scan Flow
|
||||
|
||||
```
|
||||
Scan Directory
|
||||
│
|
||||
▼
|
||||
Find MP4 Files → Extract Serie Key
|
||||
│
|
||||
▼
|
||||
Check DB for Existing Series (by key)
|
||||
│
|
||||
├─── EXISTS ──────────────────────► Update Series Metadata
|
||||
│ │
|
||||
│ ▼
|
||||
│ Sync Episodes to DB
|
||||
│ │
|
||||
│◄──────────────────────────────────────┘
|
||||
│
|
||||
└─── NEW ───────────────────────────► Create New Series Record
|
||||
│
|
||||
▼
|
||||
Create Episode Records
|
||||
│
|
||||
▼
|
||||
Return to Scan Loop
|
||||
```
|
||||
|
||||
### Key Methods
|
||||
|
||||
**SerieScanner._persist_serie_to_db()**
|
||||
- Called after `get_missing_episodes_and_season()` computes episodeDict
|
||||
- Uses `AnimeSeriesService.get_by_key()` to check if series exists
|
||||
- If exists: calls `AnimeSeriesService.update()` + `_sync_episodes_to_db()`
|
||||
- If new: calls `AnimeSeriesService.create()` + creates episodes
|
||||
|
||||
**SerieScanner._sync_episodes_to_db()**
|
||||
- Gets existing episodes from DB via `EpisodeService.get_by_series()`
|
||||
- Compares with new episodeDict
|
||||
- Removes episodes no longer missing (unless `is_downloaded=True`)
|
||||
- Adds new missing episodes
|
||||
- Preserves `is_downloaded=True` episodes when removing missing ones
|
||||
|
||||
**SerieList.add_to_db()**
|
||||
- Used when adding a new discovered series via API
|
||||
- Creates filesystem folder + database record + episode records
|
||||
|
||||
### Episode Sync Logic
|
||||
|
||||
```python
|
||||
# For each episode in DB but not in new episodeDict:
|
||||
if episode.is_downloaded:
|
||||
# Keep - file exists, don't remove
|
||||
pass
|
||||
else:
|
||||
# Remove - no longer missing
|
||||
EpisodeService.delete()
|
||||
|
||||
# For each episode in new episodeDict but not in DB:
|
||||
# Add as new missing episode
|
||||
EpisodeService.create(is_downloaded=False)
|
||||
```
|
||||
|
||||
### Transaction Handling
|
||||
|
||||
- DB operations use their own session with commit/rollback
|
||||
- If DB write fails, error is logged and scan continues
|
||||
- File-based `save_to_file()` no longer called during scan
|
||||
|
||||
### Migration Path
|
||||
|
||||
1. v2.x: Scanner writes to both DB (primary) and files (fallback)
|
||||
2. v3.0: Scanner writes only to DB, file methods removed
|
||||
|
||||
---
|
||||
|
||||
## 13. Series Persistence
|
||||
|
||||
### Schema
|
||||
|
||||
**AnimeSeries Table**: Stores series metadata (key, name, site, folder, year)
|
||||
|
||||
| Column | Type | Constraints | Description |
|
||||
|-----------|--------------|---------------------------|----------------------|
|
||||
| `id` | INTEGER | PRIMARY KEY | Auto-increment |
|
||||
| `key` | VARCHAR(255) | UNIQUE, NOT NULL | Series provider key |
|
||||
| `name` | VARCHAR(500) | NOT NULL | Display name |
|
||||
| `site` | VARCHAR(500) | | Provider site URL |
|
||||
| `folder` | VARCHAR(1000)| | Filesystem folder |
|
||||
|
||||
**Episode Table**: Stores per-episode metadata (season, episode_number, is_downloaded)
|
||||
|
||||
| Column | Type | Constraints | Description |
|
||||
|-----------------|--------------|---------------------------|----------------------|
|
||||
| `id` | INTEGER | PRIMARY KEY | Auto-increment |
|
||||
| `series_id` | INTEGER | FOREIGN KEY → anime_series| Parent series |
|
||||
| `season` | INTEGER | NOT NULL | Season number |
|
||||
| `episode_number`| INTEGER | NOT NULL | Episode number |
|
||||
| `is_downloaded` | BOOLEAN | DEFAULT FALSE | Download status |
|
||||
|
||||
### Relationships
|
||||
|
||||
- `AnimeSeries.episodes` → List of Episode objects (one-to-many)
|
||||
- `Episode.series` → Parent AnimeSeries (many-to-one)
|
||||
- Cascade delete: Deleting a series removes all its episodes
|
||||
|
||||
### Queries
|
||||
|
||||
```python
|
||||
# Get all series with episodes
|
||||
AnimeSeriesService.get_all(db, with_episodes=True)
|
||||
|
||||
# Get by provider key
|
||||
AnimeSeriesService.get_by_key(db, key)
|
||||
|
||||
# Get by folder path
|
||||
AnimeSeriesService.get_by_folder(db, folder)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 14. Database Location
|
||||
|
||||
| Environment | Default Location |
|
||||
| ----------- | ------------------------------------------------- |
|
||||
|
||||
@@ -61,4 +61,376 @@ This document provides guidance for developers working on the Aniworld project.
|
||||
- Commit message format
|
||||
- Pull request process
|
||||
8. Common Development Tasks
|
||||
|
||||
### Adding Queue Deduplication
|
||||
|
||||
The download queue prevents duplicate entries at two levels:
|
||||
|
||||
**In-Memory Deduplication** (`src/server/services/download_service.py`):
|
||||
- `_pending_by_episode` dict tracks pending episodes: key = `(serie_id, season, episode)`
|
||||
- `_add_to_pending_queue()` updates the dict when adding items
|
||||
- `add_to_queue()` checks this dict before adding episodes (includes batch-local dedup)
|
||||
- `_remove_from_pending_queue()` cleans up the dict when items are removed
|
||||
|
||||
**Database Constraint** (`src/server/models.py`):
|
||||
- `DownloadQueueItem` has a unique index on `episode_id` via `__table_args__`
|
||||
- Prevents duplicate queue entries at the database level
|
||||
- Unique constraint: `Index("ix_download_queue_episode_pending", "episode_id", unique=True)`
|
||||
|
||||
**Scheduler Cooldown** (`src/server/services/scheduler_service.py`):
|
||||
- `_last_auto_download_time` tracks when auto-download last ran
|
||||
- 5-minute cooldown prevents rapid re-triggers
|
||||
- Checked at start of `_auto_download_missing()`
|
||||
|
||||
### Episode Lifecycle
|
||||
|
||||
Episodes transition through states stored in the `episodes` table:
|
||||
|
||||
| State | `is_downloaded` | `file_path` | Description |
|
||||
|-------|----------------|-------------|-------------|
|
||||
| Missing | `False` | `NULL` | Episode not yet downloaded |
|
||||
| Downloaded | `True` | Set | Episode exists on disk |
|
||||
|
||||
**State Transitions:**
|
||||
1. **Missing → Downloaded**: When download completes, `_remove_episode_from_missing_list()` calls `EpisodeService.mark_downloaded()` to set `is_downloaded=True` and populate `file_path`. The episode record is NOT deleted.
|
||||
|
||||
**Query Implications:**
|
||||
- `get_series_with_missing_episodes()`: Filters for `is_downloaded=False` to find series with undownloaded episodes
|
||||
- `get_series_with_no_episodes()`: Finds series with `is_downloaded=False` episodes but NO `is_downloaded=True` episodes (completely unwatched series)
|
||||
|
||||
### Mocking the Download Queue
|
||||
|
||||
When testing components that use the download queue:
|
||||
|
||||
```python
|
||||
# Mock repository for unit tests
|
||||
class MockQueueRepository:
|
||||
def __init__(self):
|
||||
self._items: Dict[str, DownloadItem] = {}
|
||||
|
||||
async def save_item(self, item: DownloadItem) -> DownloadItem:
|
||||
self._items[item.id] = item
|
||||
return item
|
||||
|
||||
async def get_all_items(self) -> List[DownloadItem]:
|
||||
return list(self._items.values())
|
||||
|
||||
# Use in fixture
|
||||
@pytest.fixture
|
||||
def mock_queue_repository():
|
||||
return MockQueueRepository()
|
||||
|
||||
@pytest.fixture
|
||||
def download_service(mock_anime_service, mock_queue_repository):
|
||||
return DownloadService(
|
||||
anime_service=mock_anime_service,
|
||||
queue_repository=mock_queue_repository,
|
||||
max_retries=3,
|
||||
)
|
||||
```
|
||||
|
||||
9. Troubleshooting Development Issues
|
||||
|
||||
### Async Context Managers for aiohttp
|
||||
|
||||
All `aiohttp.ClientSession` usages must be wrapped in `async with`:
|
||||
|
||||
```python
|
||||
# Correct — session properly closed on exit
|
||||
async with TMDBClient(api_key="key") as client:
|
||||
result = await client.search_tv_show("Show")
|
||||
|
||||
# Wrong — session may leak if exception occurs
|
||||
client = TMDBClient(api_key="key")
|
||||
result = await client.search_tv_show("Show")
|
||||
await client.close() # May not be called if exception raised earlier
|
||||
```
|
||||
|
||||
**Why:**
|
||||
- `aiohttp.ClientSession` holds TCP connections that must be explicitly closed
|
||||
- If exception occurs before `close()`, session leaks
|
||||
- Context manager guarantees `__aexit__` runs even on exceptions
|
||||
|
||||
**Services that use aiohttp:**
|
||||
- `TMDBClient` — has `__aenter__`/`__aexit__`, use `async with`
|
||||
- `ImageDownloader` — has `__aenter__`/`__aexit__`, use `async with`
|
||||
- `NFOService` — wraps both above, use `async with`
|
||||
|
||||
**Verification:**
|
||||
- Missing context manager usage triggers `__del__` warning on garbage collection
|
||||
- Integration tests verify no "Unclosed client session" errors in logs
|
||||
|
||||
### Scheduler Persistence and Recovery
|
||||
|
||||
The scheduler uses APScheduler's in-memory job store. Jobs are reconstructed from `config.json` on every startup — no separate database is needed.
|
||||
|
||||
```python
|
||||
# Jobs are built from config on startup — no persistence DB required
|
||||
scheduler = AsyncIOScheduler() # default MemoryJobStore
|
||||
scheduler.add_job(..., replace_existing=True)
|
||||
```
|
||||
|
||||
**Startup misfire recovery:** On `start()`, the scheduler checks `system_settings.last_scan_timestamp` in `aniworld.db`. If the last scan is overdue (>23h but <25h ago), an immediate rescan is triggered. This replaces APScheduler's built-in misfire handling which required a separate SQLite database.
|
||||
|
||||
**Grace period:** If the server was down for more than 25 hours, no automatic recovery occurs to avoid surprise rescans after long downtime.
|
||||
|
||||
**Health endpoint:** `GET /health` returns `scheduler_next_run` and `scheduler_last_run` for external monitors (Uptime Kuma, Prometheus, etc.).
|
||||
|
||||
**If server is down too long:** Manual trigger via `POST /api/scheduler/trigger-rescan` or wait for next scheduled run.
|
||||
|
||||
### Database Session Management
|
||||
|
||||
`get_async_session_factory()` returns a **new AsyncSession instance** directly (not a factory). The function name is historical — callers receive the session immediately:
|
||||
|
||||
```python
|
||||
# Correct usage:
|
||||
db = get_async_session_factory() # db IS the session
|
||||
await db.execute(...)
|
||||
await db.commit()
|
||||
await db.close()
|
||||
```
|
||||
|
||||
Do NOT call the result again with `()` — that tries to call an `AsyncSession` object, causing `'AsyncSession' object is not callable`.
|
||||
|
||||
For context manager usage, prefer `get_db_session()` (auto-commits) or `get_transactional_session()` (manual commit).
|
||||
|
||||
### Health Check Endpoints
|
||||
|
||||
The application provides health check endpoints for monitoring and container orchestration:
|
||||
|
||||
#### `GET /health`
|
||||
Basic health check returning service status and startup health check results.
|
||||
|
||||
**Response fields:**
|
||||
- `status`: "healthy", "degraded", or "unhealthy" based on startup checks
|
||||
- `timestamp`: ISO timestamp of the check
|
||||
- `series_app_initialized`: Whether the series app is loaded
|
||||
- `anime_directory_configured`: Whether anime_directory is set
|
||||
- `scheduler_next_run` / `scheduler_last_run`: Scheduler times
|
||||
- `checks`: Detailed startup check results (ffmpeg, DNS, anime_directory)
|
||||
|
||||
#### `GET /health/ready`
|
||||
Readiness check for container orchestrators (Kubernetes, Docker Swarm).
|
||||
|
||||
**Response when ready:**
|
||||
```json
|
||||
{
|
||||
"status": "ready",
|
||||
"ready": true,
|
||||
"timestamp": "2024-01-01T00:00:00",
|
||||
"checks": {...}
|
||||
}
|
||||
```
|
||||
|
||||
**Response when not ready (503):**
|
||||
```json
|
||||
{
|
||||
"status": "not_ready",
|
||||
"ready": false,
|
||||
"timestamp": "2024-01-01T00:00:00",
|
||||
"critical_failures": ["anime_directory: not configured"],
|
||||
"checks": {...}
|
||||
}
|
||||
```
|
||||
|
||||
#### `GET /health/detailed`
|
||||
Comprehensive health check including database, filesystem, and system metrics.
|
||||
|
||||
#### Startup Health Checks
|
||||
|
||||
On application startup, the following checks are performed:
|
||||
|
||||
| Check | Failure Status | Impact |
|
||||
|-------|---------------|--------|
|
||||
| `ffmpeg` | warning | HLS downloads may fail |
|
||||
| `dns_aniworld` | warning | Provider requests may fail |
|
||||
| `dns_tmdb` | warning | TMDB API calls may fail |
|
||||
| `anime_directory` | error | Download service disabled |
|
||||
|
||||
DNS checks are warnings because failures can be transient. anime_directory errors disable the download service to prevent failures.
|
||||
|
||||
### Troubleshooting Development Issues
|
||||
|
||||
#### Scheduler missed a run
|
||||
|
||||
1. Server was down at scheduled time (03:00 UTC by default).
|
||||
2. On restart, the scheduler checks `last_scan_timestamp` — if overdue by 23-25h, it triggers immediately.
|
||||
3. If server was down >25 hours, missed job is skipped to avoid surprise rescans.
|
||||
4. Trigger manually: `POST /api/scheduler/trigger-rescan`
|
||||
5. Monitor next run: `GET /health` → `scheduler_next_run`
|
||||
|
||||
#### Scheduler not firing (no events at scheduled time)
|
||||
|
||||
If the scheduler appears configured but never triggers:
|
||||
|
||||
1. **Check application logs for scheduler startup:**
|
||||
```
|
||||
grep "Scheduler service started" fastapi_app.log
|
||||
```
|
||||
- If missing, the scheduler failed to start — check for errors above this line
|
||||
- If present, scheduler started successfully
|
||||
|
||||
2. **Verify the job is registered:**
|
||||
```
|
||||
grep "Scheduler started with cron trigger" fastapi_app.log
|
||||
```
|
||||
|
||||
3. **Verify APScheduler events in logs:**
|
||||
```
|
||||
grep "apscheduler.executors.default" fastapi_app.log
|
||||
```
|
||||
- `Running job` = job triggered
|
||||
- `executed successfully` = job completed
|
||||
- No output = job never fired
|
||||
|
||||
4. **Test manual trigger:**
|
||||
```bash
|
||||
curl -X POST http://localhost:8000/api/scheduler/trigger-rescan -H "Authorization: Bearer <token>"
|
||||
```
|
||||
- If manual trigger works but cron doesn't, the issue is APScheduler configuration
|
||||
|
||||
5. **Check next_run_time via health endpoint:**
|
||||
```bash
|
||||
curl http://localhost:8000/health | jq .scheduler_next_run
|
||||
```
|
||||
- If `null`, the job is not scheduled
|
||||
- If set, the scheduler knows when to run next
|
||||
|
||||
6. **Check timezone handling:**
|
||||
- APScheduler uses UTC internally
|
||||
- The schedule_time config (e.g., "03:00") is interpreted as UTC
|
||||
- If you expect local time, adjust the schedule_time accordingly
|
||||
|
||||
#### Startup health check failures
|
||||
|
||||
If `/health` returns `unhealthy` status:
|
||||
|
||||
1. **anime_directory error**: Directory not configured or not writable
|
||||
- Check `ANIME_DIRECTORY` environment variable
|
||||
- Verify directory exists and permissions allow write access
|
||||
- Download service will not initialize until resolved
|
||||
|
||||
2. **ffmpeg warning**: ffmpeg not found in PATH
|
||||
- HLS stream downloads will fail
|
||||
- Install ffmpeg: `apt install ffmpeg` or `brew install ffmpeg`
|
||||
|
||||
3. **DNS warnings**: Domain resolution failed
|
||||
- Check network connectivity
|
||||
- DNS failures are transient — warnings don't block startup
|
||||
- Retry later to verify: `GET /health`
|
||||
|
||||
### Provider Failure Handling
|
||||
|
||||
Download providers (VOE, Doodstream, Vidmoly, Vidoza, SpeedFiles, Streamtape,
|
||||
Luluvdo) regularly break: URLs expire, sites change their player markup, geo
|
||||
blocks appear, and `yt-dlp` extractors lag behind upstream changes. The
|
||||
`AniworldLoader.download()` flow is designed to fail fast and rotate.
|
||||
|
||||
**Rotation order**
|
||||
|
||||
1. The episode page is scraped for the providers AniWorld actually advertises.
|
||||
2. Results are ordered by the preference in `DEFAULT_PROVIDERS`
|
||||
(`provider_config.py`); providers not listed run last.
|
||||
3. For each candidate the loader:
|
||||
1. Calls `_check_url_alive()` — HEAD probe with GET fallback. Any 4xx
|
||||
response or connection error skips the provider immediately.
|
||||
2. Resolves the redirect via `_resolve_direct_link()` to obtain a direct
|
||||
stream URL plus headers. Provider-specific extractors (e.g. `VOE`) are
|
||||
preferred; unknown providers fall back to the embed URL so `yt-dlp` can
|
||||
attempt extraction.
|
||||
3. Tries `_try_direct_stream()` — straight `requests.get(stream=True)` when
|
||||
`Content-Type` is `video/*` or `application/octet-stream`. This avoids
|
||||
`yt-dlp` entirely for direct MP4 links.
|
||||
4. Falls back to `yt-dlp` with the ffmpeg downloader for HLS streams.
|
||||
4. On any failure, temp files are cleaned and the loop moves to the next
|
||||
provider. When the chain is exhausted, the loader logs
|
||||
`All download providers failed for S{season}E{episode} ...; tried=[...]`
|
||||
to both the application log and `logs/download_errors.log`.
|
||||
|
||||
**Do not hardcode provider URLs.** Provider domains shift constantly (e.g.
|
||||
Doodstream alternates between `dood.li`, `dood.so`, `dood.la`). Only the
|
||||
referer hints in `PROVIDER_HEADERS` are persisted — discovery still happens
|
||||
at runtime through AniWorld's redirect endpoint.
|
||||
|
||||
### HLS Stream Handling
|
||||
|
||||
HLS (HTTP Live Streaming) manifests (`.m3u8`) require yt-dlp to use the
|
||||
`ffmpeg` downloader with `--hls-use-mpegts`. Both providers configure this
|
||||
automatically:
|
||||
|
||||
```python
|
||||
ydl_opts = {
|
||||
"downloader": "ffmpeg", # Use ffmpeg instead of native
|
||||
"hls_use_mpegts": True, # Write transport stream (.ts) segments
|
||||
}
|
||||
```
|
||||
|
||||
**Why this matters**: Without ffmpeg, yt-dlp logs:
|
||||
`"Live HLS streams are not supported by the native downloader"`
|
||||
|
||||
**Requirements**:
|
||||
- ffmpeg must be installed and in PATH (`which ffmpeg`)
|
||||
- Install: `apt install ffmpeg` (Debian/Ubuntu) or `brew install ffmpeg` (macOS)
|
||||
- Startup health check (see Health Check Endpoints) verifies ffmpeg presence
|
||||
|
||||
**Trade-offs**:
|
||||
- HLS downloads are slower than direct MP4 (reassembly of .ts segments)
|
||||
- Requires more disk space during download
|
||||
- May need post-processing if .ts format is not desired
|
||||
|
||||
**Detection**: VOE provider extracts HLS URLs via `HLS_PATTERN` regex. Other
|
||||
providers let yt-dlp auto-detect from URL/content-type.
|
||||
|
||||
### Updating yt-dlp
|
||||
|
||||
When extractors break (typical symptoms: every provider HEAD probe succeeds
|
||||
but `yt-dlp` raises `Unable to extract` or `HTTP Error 404`):
|
||||
|
||||
1. Check the upstream tracker first: https://github.com/yt-dlp/yt-dlp/issues
|
||||
2. Upgrade in the conda environment:
|
||||
```bash
|
||||
conda run -n AniWorld pip install --upgrade yt-dlp
|
||||
```
|
||||
3. Smoke-test against a known-good episode before pinning a new floor in
|
||||
`requirements.txt` (`yt-dlp>=YYYY.MM.DD`).
|
||||
4. Re-run the provider test suite:
|
||||
```bash
|
||||
conda run -n AniWorld python -m pytest tests/unit/test_aniworld_provider.py -v
|
||||
```
|
||||
5. If a specific extractor is removed upstream, drop the provider from
|
||||
`DEFAULT_PROVIDERS` rather than patching `yt-dlp` in tree.
|
||||
|
||||
### User Notification on Total Failure
|
||||
|
||||
`SeriesApp.download_episode()` already emits a `download_status="failed"`
|
||||
WebSocket event when `loader.download()` returns `False`. Operators should
|
||||
forward this to `notification_service.notify_download_failed()` so users see
|
||||
a HIGH-priority alert. The loader keeps the failure detail in
|
||||
`logs/download_errors.log` for post-mortem.
|
||||
|
||||
## Series Storage
|
||||
|
||||
### Overview
|
||||
|
||||
Series metadata now stored in the database (SQLAlchemy ORM).
|
||||
Legacy files (`key` and `data` per folder) are deprecated but preserved
|
||||
for backward compatibility.
|
||||
|
||||
### Architecture
|
||||
|
||||
- **Database**: Single source of truth for all series metadata
|
||||
- **In-Memory Cache**: SeriesApp maintains a cache for performance
|
||||
- **Filesystem**: Only used for episode files themselves, not metadata
|
||||
|
||||
### Migration
|
||||
|
||||
First startup after upgrade automatically imports any legacy
|
||||
series files into the database.
|
||||
|
||||
### Legacy Files
|
||||
|
||||
- `key` file: Contains series provider key (deprecated)
|
||||
- `data` file: Contains Serie JSON object (deprecated)
|
||||
|
||||
Both are safe to delete after migration; not needed for normal operation.
|
||||
|
||||
|
||||
111
docs/MIGRATION_GUIDE.md
Normal file
111
docs/MIGRATION_GUIDE.md
Normal file
@@ -0,0 +1,111 @@
|
||||
# Migration Guide: File-Based to Database Storage
|
||||
|
||||
## Overview
|
||||
|
||||
This guide covers the transition from file-based series metadata storage to the new database-backed system introduced in v2.0.
|
||||
|
||||
## What Changed
|
||||
|
||||
**Before v2.0**: Series metadata stored in `key` and `data` files alongside anime folders.
|
||||
|
||||
**After v2.0**: All metadata stored in SQLite database (`aniworld.db`). Files are deprecated but still supported for backward compatibility during migration.
|
||||
|
||||
## Automated Migration
|
||||
|
||||
The application automatically migrates on first startup:
|
||||
|
||||
1. Scans anime directory for `key` and `data` files
|
||||
2. Parses legacy files into `AnimeSeries` and `Episode` records
|
||||
3. Loads series into in-memory cache
|
||||
4. Logs migration results
|
||||
|
||||
**No manual action required.**
|
||||
|
||||
## Manual Verification
|
||||
|
||||
After first startup with the new version:
|
||||
|
||||
1. **Check logs** for: `"Migrated X series from files to DB"`
|
||||
2. **Verify series count**: UI shows same number of series as before
|
||||
3. **Confirm episodes**: Episode counts match expected totals
|
||||
|
||||
```bash
|
||||
# Check migration log
|
||||
grep "Migrated" logs/app.log
|
||||
|
||||
# Verify series via API
|
||||
curl http://localhost:8000/api/anime | jq '.total'
|
||||
```
|
||||
|
||||
## After Migration
|
||||
|
||||
### Safe to Delete
|
||||
|
||||
Once verified, these files can be removed:
|
||||
|
||||
```
|
||||
<anime_folder>/
|
||||
├── Attack on Titan (2013)/
|
||||
│ ├── key # ❌ Can delete
|
||||
│ ├── data # ❌ Can delete
|
||||
│ └── Season 1/
|
||||
│ └── ...
|
||||
```
|
||||
|
||||
**Deleting these files does not affect the database.** The metadata now lives in `aniworld.db`.
|
||||
|
||||
### Backup (Recommended)
|
||||
|
||||
Before deleting, backup the files:
|
||||
|
||||
```bash
|
||||
# Create backup directory
|
||||
mkdir -p backup/legacy_series_files
|
||||
|
||||
# Copy all key and data files
|
||||
find /path/to/anime -name "key" -o -name "data" | while read f; do
|
||||
cp "$f" "backup/legacy_series_files/"
|
||||
done
|
||||
```
|
||||
|
||||
## Reverting (Not Recommended)
|
||||
|
||||
If you must revert to file-based storage:
|
||||
|
||||
1. **Restore from database backup** (if available)
|
||||
2. **Export manually** (no export script exists)
|
||||
|
||||
**Warning**: File-based storage is deprecated and will be removed in v3.0.0.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Series Not Appearing After Migration
|
||||
|
||||
1. Check logs for migration errors: `grep -i error logs/app.log`
|
||||
2. Verify `key` and `data` files exist and are readable
|
||||
3. Manually trigger rescan: `POST /api/scheduler/trigger-rescan`
|
||||
|
||||
### Duplicate Series
|
||||
|
||||
1. Check for duplicate `key` files (same series in multiple folders)
|
||||
2. Verify series key uniqueness in database:
|
||||
|
||||
```bash
|
||||
sqlite3 aniworld.db "SELECT key, COUNT(*) FROM anime_series GROUP BY key HAVING COUNT(*) > 1;"
|
||||
```
|
||||
|
||||
### Missing Episodes
|
||||
|
||||
1. Trigger targeted scan for affected series
|
||||
2. Check episode sync logs
|
||||
3. Verify file permissions on anime directory
|
||||
|
||||
## Deprecation Timeline
|
||||
|
||||
| Version | Status |
|
||||
|---------|--------|
|
||||
| v2.0.x | Legacy files supported, migration automated |
|
||||
| v2.1.x | Legacy files still supported, warnings in logs |
|
||||
| v3.0.0 | **Legacy files removed** - database only |
|
||||
|
||||
Upgrade to v3.0.0 before legacy file support ends.
|
||||
@@ -171,6 +171,35 @@ Response:
|
||||
}
|
||||
```
|
||||
|
||||
### 3.6 Fallback Behavior When TMDB is Unavailable
|
||||
|
||||
When TMDB lookup fails (network issues, API errors, or no match found), the system creates a **minimal NFO** to ensure the series is still tracked. This behavior applies to:
|
||||
|
||||
- Manual NFO creation via API
|
||||
- Batch NFO creation operations
|
||||
- Automatic NFO creation during downloads
|
||||
|
||||
**What a minimal NFO contains:**
|
||||
|
||||
```xml
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<tvshow>
|
||||
<title>Series Name</title>
|
||||
<year>2024</year>
|
||||
<plot>No metadata available for Series Name. TMDB lookup failed.</plot>
|
||||
</tvshow>
|
||||
```
|
||||
|
||||
**Limitations of minimal NFOs:**
|
||||
- No poster, logo, or fanart images
|
||||
- No rating, genre, or studio information
|
||||
- No TMDB or other provider IDs
|
||||
- May not display correctly in some media servers
|
||||
|
||||
**To upgrade a minimal NFO:**
|
||||
1. Use the Update endpoint (`PUT /api/nfo/{serie_id}/update`) when TMDB is available
|
||||
2. Or delete the NFO and recreate it with full metadata
|
||||
|
||||
---
|
||||
|
||||
## 4. File Structure
|
||||
@@ -217,6 +246,7 @@ NFO files are created in the anime directory:
|
||||
<genre>Action</genre>
|
||||
<genre>Sci-Fi & Fantasy</genre>
|
||||
<uniqueid type="tmdb">1429</uniqueid>
|
||||
<tmdbid>1429</tmdbid>
|
||||
<thumb aspect="poster">https://image.tmdb.org/t/p/w500/...</thumb>
|
||||
<fanart>
|
||||
<thumb>https://image.tmdb.org/t/p/original/...</thumb>
|
||||
@@ -224,6 +254,13 @@ NFO files are created in the anime directory:
|
||||
</tvshow>
|
||||
```
|
||||
|
||||
**Manual TMDB ID Override**: To skip TMDB search and use a specific ID directly, include `<tmdbid>YOUR_ID</tmdbid>` in the NFO. This is useful when:
|
||||
- TMDB search fails for your series (e.g., new or obscure anime)
|
||||
- You already know the correct TMDB ID
|
||||
- You want to avoid rate limiting from repeated searches
|
||||
|
||||
Aniworld reads `<tmdbid>` element and `<uniqueid type="tmdb">` first. If found, it uses the ID directly instead of searching.
|
||||
|
||||
### 4.3 Episode NFO Format
|
||||
|
||||
```xml
|
||||
@@ -246,7 +283,84 @@ NFO files are created in the anime directory:
|
||||
|
||||
---
|
||||
|
||||
## 5. API Reference
|
||||
## 5. Folder Naming Convention
|
||||
|
||||
### 5.1 Expected Format
|
||||
|
||||
After the daily folder scan (when **Update on library scan** is enabled), Aniworld validates every series folder against its `tvshow.nfo` metadata. If the folder name does not match the expected convention, it is automatically renamed.
|
||||
|
||||
**Format:**
|
||||
|
||||
```
|
||||
{title} ({year})
|
||||
```
|
||||
|
||||
**Examples:**
|
||||
|
||||
| NFO `<title>` | NFO `<year>` | Expected Folder Name |
|
||||
|---------------|--------------|----------------------|
|
||||
| `Attack on Titan` | `2013` | `Attack on Titan (2013)` |
|
||||
| `One Piece` | `1999` | `One Piece (1999)` |
|
||||
| `Demon Slayer: Kimetsu no Yaiba` | `2019` | `Demon Slayer Kimetsu no Yaiba (2019)` |
|
||||
|
||||
### 5.2 Sanitization Rules
|
||||
|
||||
Illegal filesystem characters are removed or replaced to ensure cross-platform compatibility:
|
||||
|
||||
- Removed: `< > : " / \ | ? *` and null bytes
|
||||
- Control characters stripped
|
||||
- Multiple spaces collapsed to one
|
||||
- Leading/trailing dots and whitespace trimmed
|
||||
- Maximum length: 200 characters (truncated at word boundary if possible)
|
||||
|
||||
### 5.3 Skip Conditions
|
||||
|
||||
A folder is **not** renamed when any of the following apply:
|
||||
|
||||
- `tvshow.nfo` is missing `<title>` or `<year>` (or they are empty)
|
||||
- The series has an **active or pending download**
|
||||
- The target folder name already exists (duplicate)
|
||||
- The resulting path would exceed the OS path-length limit
|
||||
- The app lacks write permission to the anime directory
|
||||
|
||||
All skipped and renamed actions are logged.
|
||||
|
||||
---
|
||||
|
||||
## 6. Poster Check
|
||||
|
||||
### 6.1 Overview
|
||||
|
||||
During the daily folder scan, Aniworld checks every series folder for a valid `poster.jpg`. If the file is missing or smaller than 1 KB, the application attempts to re-download it from the URL stored in the series' `tvshow.nfo` file.
|
||||
|
||||
### 6.2 How It Works
|
||||
|
||||
1. **Scan** — After folder renaming, the scan iterates over all series folders that contain a `tvshow.nfo`.
|
||||
2. **Validate** — For each folder, it checks whether `poster.jpg` exists and is at least 1 KB.
|
||||
3. **Parse NFO** — If the poster is missing or too small, the scan reads `tvshow.nfo` and looks for a `<thumb aspect="poster">` (or any `<thumb>`) URL.
|
||||
4. **Download** — If a URL is found, the poster is downloaded using `ImageDownloader` with a concurrency limit of 3 simultaneous downloads.
|
||||
5. **Validate Download** — The downloaded image is validated with PIL to ensure it is not corrupted.
|
||||
|
||||
### 6.3 Skip Conditions
|
||||
|
||||
A folder is **not** processed for poster download when any of the following apply:
|
||||
|
||||
- `tvshow.nfo` does not exist in the folder.
|
||||
- `poster.jpg` already exists and is ≥ 1 KB.
|
||||
- No `<thumb>` URL is found in the NFO (the NFO may have been created before thumb tags were added).
|
||||
- The `nfo.download_poster` setting is `false` (poster checks are still performed, but downloads are skipped if the setting is disabled; see [CONFIGURATION.md](CONFIGURATION.md)).
|
||||
|
||||
### 6.4 Logging
|
||||
|
||||
Every poster check action is logged:
|
||||
|
||||
- **INFO** — When a poster is successfully downloaded.
|
||||
- **WARNING** — When a download fails or no URL is found.
|
||||
- **ERROR** — When an unexpected exception occurs during download.
|
||||
|
||||
---
|
||||
|
||||
## 7. API Reference
|
||||
|
||||
### 5.1 Check NFO Status
|
||||
|
||||
@@ -523,6 +637,36 @@ NFO files are created in the anime directory:
|
||||
4. Check network speed to TMDB servers
|
||||
5. Verify disk I/O performance
|
||||
|
||||
### 6.7 TMDB Lookup Fails for My Series
|
||||
|
||||
**Problem**: TMDB search fails with "No results found" for a valid series.
|
||||
|
||||
**Solutions**:
|
||||
|
||||
1. **Check if series exists on TMDB**: Visit https://www.themoviedb.org and search for your series
|
||||
2. **Use manual ID override**: Add TMDB ID directly to `tvshow.nfo`:
|
||||
```xml
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<tvshow>
|
||||
<title>Your Series Name</title>
|
||||
<tmdbid>12345</tmdbid>
|
||||
<uniqueid type="tmdb">12345</uniqueid>
|
||||
</tvshow>
|
||||
```
|
||||
Aniworld will use this ID directly instead of searching.
|
||||
|
||||
3. **Try alternative titles**: Some anime have different titles (Japanese, romaji, English). If you have access to the folder, rename it to match the TMDB title.
|
||||
|
||||
4. **Add to existing NFO**: If `tvshow.nfo` exists but has no TMDB ID, edit it to add:
|
||||
```xml
|
||||
<tmdbid>YOUR_TMDB_ID</tmdbid>
|
||||
```
|
||||
Then use the Update endpoint to refresh metadata.
|
||||
|
||||
5. **Check for rate limiting**: If many lookups fail at once, you may be hitting TMDB rate limits. Wait and retry later.
|
||||
|
||||
6. **Verify API key**: Ensure your TMDB API key is valid and has not exceeded usage limits.
|
||||
|
||||
---
|
||||
|
||||
## 7. Best Practices
|
||||
@@ -675,21 +819,25 @@ The XML serialisation lives in `src/core/utils/nfo_generator.py`
|
||||
|
||||
## 11. Automatic NFO Repair
|
||||
|
||||
Every time the server starts, Aniworld scans all existing `tvshow.nfo` files and
|
||||
automatically repairs any that are missing required tags.
|
||||
NFO repair now runs as part of the scheduled daily folder scan rather than on every
|
||||
startup. When the scheduler triggers `FolderScanService.run_folder_scan()`, the first
|
||||
step is `perform_nfo_repair_scan(background_loader=None)`. Each incomplete NFO is
|
||||
queued as a background `asyncio` task, so the scan returns quickly while repairs
|
||||
continue asynchronously.
|
||||
|
||||
### How It Works
|
||||
|
||||
1. **Scan** — `perform_nfo_repair_scan()` in
|
||||
`src/server/services/initialization_service.py` is called from the FastAPI
|
||||
lifespan after `perform_media_scan_if_needed()`.
|
||||
`src/server/services/initialization_service.py` is called from
|
||||
`FolderScanService.run_folder_scan()` (`src/server/services/folder_scan_service.py`).
|
||||
2. **Detect** — `nfo_needs_repair(nfo_path)` from
|
||||
`src/core/services/nfo_repair_service.py` parses each `tvshow.nfo` with
|
||||
`lxml` and checks for the 13 required tags listed below.
|
||||
3. **Repair** — Series whose NFO is incomplete are queued for background reload
|
||||
via `BackgroundLoaderService.add_series_loading_task()`. The background
|
||||
loader re-fetches metadata from TMDB and rewrites the NFO with all tags
|
||||
populated.
|
||||
via `asyncio.create_task`. Each task creates its own isolated
|
||||
:class:`NFOService` / :class:`TMDBClient` so concurrent tasks never share an
|
||||
``aiohttp`` session — this prevents "Connector is closed" errors when many repairs
|
||||
run in parallel. A semaphore caps TMDB concurrency at 3 to stay within rate limits.
|
||||
|
||||
### Tags Checked (13 required)
|
||||
|
||||
@@ -734,8 +882,7 @@ This calls `NFOService.update_tvshow_nfo()` directly and overwrites the existing
|
||||
| File | Purpose |
|
||||
| ----------------------------------------------- | ---------------------------------------------------------------------------------------------- |
|
||||
| `src/core/services/nfo_repair_service.py` | `REQUIRED_TAGS`, `parse_nfo_tags`, `find_missing_tags`, `nfo_needs_repair`, `NfoRepairService` |
|
||||
| `src/server/services/initialization_service.py` | `perform_nfo_repair_scan` startup hook |
|
||||
| `src/server/fastapi_app.py` | Wires `perform_nfo_repair_scan` into the lifespan |
|
||||
| `src/server/services/folder_scan_service.py` | `perform_nfo_repair_scan` — invoked during the scheduled daily folder scan |
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -62,6 +62,81 @@ This document describes the testing strategy, guidelines, and practices for the
|
||||
- What to mock
|
||||
- Mock patterns
|
||||
- External service mocks
|
||||
|
||||
### Mocking the Download Queue
|
||||
|
||||
Use `MockQueueRepository` for testing download queue functionality:
|
||||
|
||||
```python
|
||||
from src.server.models.download import DownloadItem, EpisodeIdentifier
|
||||
|
||||
class MockQueueRepository:
|
||||
def __init__(self):
|
||||
self._items: Dict[str, DownloadItem] = {}
|
||||
```
|
||||
|
||||
### Testing SetupService
|
||||
|
||||
SetupService handles series key resolution from folder names during library setup. Test file: `tests/unit/test_setup_service.py`.
|
||||
|
||||
Key methods tested:
|
||||
- `_extract_year_from_folder_name()` — parses `(YYYY)` suffix
|
||||
- `_extract_title_from_folder_name()` — strips year suffix
|
||||
- `_resolve_key_via_search()` — resolves provider key via fuzzy title matching
|
||||
|
||||
```python
|
||||
@pytest.mark.asyncio
|
||||
async def test_returns_key_when_single_exact_match(self):
|
||||
"""Search returns 1 result with same name → returns key."""
|
||||
mock_series_app = AsyncMock()
|
||||
mock_series_app.search.return_value = [
|
||||
{'title': 'Attack on Titan', 'link': '/anime/stream/attack-on-titan'}
|
||||
]
|
||||
|
||||
with patch('src.server.services.setup_service.get_series_app', return_value=mock_series_app):
|
||||
result = await SetupService._resolve_key_via_search("Attack on Titan")
|
||||
|
||||
assert result == 'attack-on-titan'
|
||||
```
|
||||
|
||||
### Mocking aiohttp Sessions
|
||||
|
||||
When testing code that uses `aiohttp.ClientSession`:
|
||||
|
||||
```python
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from aiohttp import ClientSession
|
||||
|
||||
# Mock aiohttp session for testing
|
||||
class MockAiohttpSession:
|
||||
def __init__(self):
|
||||
self.closed = False
|
||||
|
||||
async def close(self):
|
||||
self.closed = True
|
||||
|
||||
def get(self, url, **kwargs):
|
||||
mock_response = AsyncMock()
|
||||
mock_response.status = 200
|
||||
mock_response.json = AsyncMock(return_value={"data": "test"})
|
||||
mock_response.__aenter__ = AsyncMock(return_value=mock_response)
|
||||
mock_response.__aexit__ = AsyncMock(return_value=None)
|
||||
return mock_response
|
||||
|
||||
# Use in fixture
|
||||
@pytest.fixture
|
||||
async def mock_tmdb_session():
|
||||
session = MockAiohttpSession()
|
||||
yield session
|
||||
# Cleanup verification
|
||||
assert session.closed, "Session was not closed"
|
||||
```
|
||||
|
||||
**Key points:**
|
||||
- Always verify `session.closed` is `True` after context manager exits
|
||||
- Mock `__aenter__` and `__aexit__` for response context managers
|
||||
- Set `closed = False` on mock session for unclosed warning tests
|
||||
|
||||
7. Coverage Requirements
|
||||
8. CI/CD Integration
|
||||
9. Writing Good Tests
|
||||
|
||||
10
docs/bla
10
docs/bla
@@ -1,10 +0,0 @@
|
||||
review frontend code and check for architektre issues
|
||||
|
||||
write the tasks in Task.md
|
||||
for each task add the following informations
|
||||
|
||||
where is that found
|
||||
goal. how it should be
|
||||
possibale traps and issues
|
||||
docs changes needed
|
||||
why this is needed
|
||||
@@ -31,14 +31,16 @@ flowchart TB
|
||||
|
||||
subgraph Core["Core Layer"]
|
||||
SeriesApp["SeriesApp"]
|
||||
SeriesCache["SeriesCache<br/>(In-Memory)"]
|
||||
SerieScanner["SerieScanner"]
|
||||
SerieList["SerieList"]
|
||||
end
|
||||
|
||||
subgraph Data["Data Layer"]
|
||||
SQLite[(SQLite<br/>aniworld.db)]
|
||||
SQLite[("SQLite<br/>aniworld.db")]
|
||||
ConfigJSON[(config.json)]
|
||||
FileSystem[(File System<br/>Anime Directory)]
|
||||
FileSystem[(File System<br/>Anime Episodes)]
|
||||
LegacyFiles[("Legacy Files<br/>key/data<br/>(Deprecated)")]
|
||||
end
|
||||
|
||||
subgraph External["External"]
|
||||
@@ -71,9 +73,13 @@ flowchart TB
|
||||
AnimeService --> SQLite
|
||||
|
||||
%% Core to Data
|
||||
SeriesApp --> SeriesCache
|
||||
SeriesCache -.->|Cached Series| SQLite
|
||||
SeriesApp --> SerieScanner
|
||||
SeriesApp --> SerieList
|
||||
SerieScanner --> FileSystem
|
||||
SerieScanner -->|Scan Episodes| FileSystem
|
||||
SerieScanner -->|Detect Series| SQLite
|
||||
SerieScanner -->|Migrate Legacy| LegacyFiles
|
||||
SerieScanner --> Provider
|
||||
|
||||
%% Event flow
|
||||
|
||||
@@ -107,6 +107,10 @@ The application now features a comprehensive configuration system that allows us
|
||||
- **Progress Tracking**: Live progress updates for downloads and scans
|
||||
- **System Notifications**: Real-time system messages and alerts
|
||||
|
||||
## Folder Management
|
||||
|
||||
- **Fuzzy Series Key Resolution**: Automatic series key resolution from folder names using fuzzy title matching — tolerates title variations like `(TV)`, `(OVA)`, `(Movie)` suffixes and uses similarity matching to resolve provider keys during library setup
|
||||
|
||||
## Core Functionality Overview
|
||||
|
||||
The web application provides a complete interface for managing anime downloads with user-friendly pages for configuration, library management, search capabilities, and download monitoring. All operations are tracked in real-time with comprehensive progress reporting and error handling.
|
||||
|
||||
47
docs/key
47
docs/key
@@ -2,3 +2,50 @@ API key : 299ae8f630a31bda814263c551361448
|
||||
|
||||
/mnt/server/serien/Serien/
|
||||
|
||||
{
|
||||
"name": "Aniworld",
|
||||
"data_dir": "data",
|
||||
"scheduler": {
|
||||
"enabled": true,
|
||||
"interval_minutes": 60,
|
||||
"schedule_time": "03:00",
|
||||
"schedule_days": [
|
||||
"mon",
|
||||
"tue",
|
||||
"wed",
|
||||
"thu",
|
||||
"fri",
|
||||
"sat",
|
||||
"sun"
|
||||
],
|
||||
"auto_download_after_rescan": true,
|
||||
"folder_scan_enabled": true
|
||||
},
|
||||
"logging": {
|
||||
"level": "INFO",
|
||||
"file": null,
|
||||
"max_bytes": null,
|
||||
"backup_count": 3
|
||||
},
|
||||
"backup": {
|
||||
"enabled": false,
|
||||
"path": "data/backups",
|
||||
"keep_days": 30
|
||||
},
|
||||
"nfo": {
|
||||
"tmdb_api_key": "9bc3e547caff878615cbdba2cc421d37",
|
||||
"auto_create": true,
|
||||
"update_on_scan": true,
|
||||
"download_poster": true,
|
||||
"download_logo": true,
|
||||
"download_fanart": true,
|
||||
"image_size": "original"
|
||||
},
|
||||
"other": {
|
||||
"master_password_hash": "$pbkdf2-sha256$29000$HQNASKk1xpgTAgAgJGRMaQ$73TOCCM0UEZONyNXQEPa3SmIoXeG6C1l5mMFDNgYfMQ",
|
||||
"anime_directory": "/data"
|
||||
},
|
||||
"version": "1.0.0"
|
||||
}
|
||||
|
||||
|
||||
|
||||
154
docs/runner.csx
Normal file
154
docs/runner.csx
Normal file
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env dotnet-script
|
||||
#nullable enable
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Diagnostics;
|
||||
using System.Threading;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Linq;
|
||||
using System.Collections.Generic;
|
||||
|
||||
// ── Ctrl+C: kill active process and exit cleanly ──────────────────────────────
|
||||
var cts = new CancellationTokenSource();
|
||||
Process? activeProcess = null;
|
||||
|
||||
Console.CancelKeyPress += (_, e) =>
|
||||
{
|
||||
e.Cancel = true;
|
||||
Console.WriteLine("\n[runner] Interrupted — shutting down...");
|
||||
cts.Cancel();
|
||||
try { activeProcess?.Kill(entireProcessTree: true); } catch { }
|
||||
};
|
||||
|
||||
// ── Paths ─────────────────────────────────────────────────────────────────────
|
||||
var repoRoot = Directory.GetCurrentDirectory();
|
||||
var tasksFile = Path.Combine(repoRoot, "Docs", "Tasks.md");
|
||||
|
||||
if (!File.Exists(tasksFile))
|
||||
{
|
||||
Console.Error.WriteLine($"[runner] ERROR: Tasks.md not found at {tasksFile}");
|
||||
Console.Error.WriteLine("[runner] Run this script from the repository root.");
|
||||
Environment.Exit(1);
|
||||
}
|
||||
|
||||
// ── Read & split by "---" separator lines ────────────────────────────────────
|
||||
var content = File.ReadAllText(tasksFile);
|
||||
var items = Regex
|
||||
.Split(content, @"\r?\n---\r?\n")
|
||||
.Select(s => s.Trim())
|
||||
.Where(s => s.Length > 0)
|
||||
.ToList();
|
||||
|
||||
Console.WriteLine($"[runner] Found {items.Count} section(s) in Tasks.md");
|
||||
|
||||
// ── Helper: run copilot and stream output, return full output ─────────────────
|
||||
async Task<string> RunCopilot(IEnumerable<string> extraArgs, string prompt)
|
||||
{
|
||||
var output = new StringBuilder();
|
||||
|
||||
var argList = new List<string> { "launch", "copilot", "--model", "minimax-m2.7:cloud", "--yes", "--", "--allow-all-tools" };
|
||||
argList.AddRange(extraArgs);
|
||||
argList.Add("-p");
|
||||
argList.Add(prompt);
|
||||
|
||||
var psi = new ProcessStartInfo("ollama")
|
||||
{
|
||||
WorkingDirectory = repoRoot,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
UseShellExecute = false,
|
||||
};
|
||||
foreach (var a in argList)
|
||||
psi.ArgumentList.Add(a);
|
||||
|
||||
activeProcess = new Process { StartInfo = psi };
|
||||
|
||||
activeProcess.OutputDataReceived += (_, e) =>
|
||||
{
|
||||
if (e.Data is null) return;
|
||||
Console.WriteLine(e.Data);
|
||||
output.AppendLine(e.Data);
|
||||
};
|
||||
activeProcess.ErrorDataReceived += (_, e) =>
|
||||
{
|
||||
if (e.Data is null) return;
|
||||
Console.Error.WriteLine(e.Data);
|
||||
output.AppendLine(e.Data);
|
||||
};
|
||||
|
||||
activeProcess.Start();
|
||||
activeProcess.BeginOutputReadLine();
|
||||
activeProcess.BeginErrorReadLine();
|
||||
|
||||
await activeProcess.WaitForExitAsync(cts.Token);
|
||||
activeProcess = null;
|
||||
|
||||
return output.ToString();
|
||||
}
|
||||
|
||||
// ── Main loop ─────────────────────────────────────────────────────────────────
|
||||
for (int i = 0; i < items.Count; i++)
|
||||
{
|
||||
var item = items[i];
|
||||
if (cts.IsCancellationRequested) break;
|
||||
|
||||
Console.WriteLine();
|
||||
Console.WriteLine("[runner] ══════════════════════════════════════════════");
|
||||
Console.WriteLine($"[runner] Task:\n{item}");
|
||||
Console.WriteLine("[runner] ══════════════════════════════════════════════");
|
||||
Console.WriteLine();
|
||||
|
||||
// Step 1 — run the task prompt
|
||||
await RunCopilot(Enumerable.Empty<string>(), $"/caveman full");
|
||||
await RunCopilot(new[] { "--continue" }, $"read ./Docs/instructions.md. {item}");
|
||||
if (cts.IsCancellationRequested) break;
|
||||
|
||||
// Step 2 — confirm completion in the same chat session
|
||||
Console.WriteLine("\n[runner] Asking for task confirmation...\n");
|
||||
var confirmation = await RunCopilot(
|
||||
new[] { "--continue" },
|
||||
"are you sure tasks is done. reply with yes"
|
||||
);
|
||||
if (cts.IsCancellationRequested) break;
|
||||
|
||||
// Step 3 — check for "yes" in the reply, with retry logic for issue resolution
|
||||
int maxRetries = 3;
|
||||
int retryCount = 0;
|
||||
bool taskConfirmed = confirmation.Contains("yes", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
while (!taskConfirmed && retryCount < maxRetries)
|
||||
{
|
||||
retryCount++;
|
||||
Console.WriteLine($"\n[runner] Attempt {retryCount}/{maxRetries}: Resolving remaining issues and running tests...\n");
|
||||
|
||||
confirmation = await RunCopilot(
|
||||
new[] { "--continue" },
|
||||
"resolve any remaining issues, make sure all tests are running and pass. then confirm with yes if done"
|
||||
);
|
||||
if (cts.IsCancellationRequested) break;
|
||||
|
||||
taskConfirmed = confirmation.Contains("yes", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
if (!taskConfirmed)
|
||||
{
|
||||
Console.WriteLine($"\n[runner] Task not confirmed as done after {maxRetries} attempts. Stopping.");
|
||||
break;
|
||||
}
|
||||
|
||||
// Step 4 — commit the work
|
||||
Console.WriteLine("\n[runner] Task confirmed. Making git commit...\n");
|
||||
|
||||
await RunCopilot(Enumerable.Empty<string>(), $"/caveman full");
|
||||
await RunCopilot(new[] { "--continue" }, "make git commit");
|
||||
if (cts.IsCancellationRequested) break;
|
||||
|
||||
// Step 5 — remove completed task from Tasks.md
|
||||
var remaining = items.Skip(i + 1).ToList();
|
||||
File.WriteAllText(tasksFile, string.Join("\n\n---\n\n", remaining));
|
||||
Console.WriteLine("[runner] Removed completed task from Tasks.md");
|
||||
}
|
||||
|
||||
Console.WriteLine("\n[runner] Finished.");
|
||||
278
docs/tasks.md
278
docs/tasks.md
@@ -1,174 +1,178 @@
|
||||
# Tasks — NFO Plot Missing Bug
|
||||
# Tasks
|
||||
|
||||
These tasks fix the root causes of `<plot>` being empty in `tvshow.nfo` after adding a series via the web UI.
|
||||
The bug does **not** appear after a server restart because the repair scan uses a different, correctly isolated code path.
|
||||
## 1. Scheduled Folder Scan
|
||||
|
||||
### Task 1.1: Add folder scan scheduler configuration
|
||||
|
||||
**Where is that found**
|
||||
- `src/server/models/config.py` (`SchedulerConfig`)
|
||||
- `data/config.json` (example/default config)
|
||||
- `src/server/web/templates/setup.html` (setup UI)
|
||||
- `src/server/api/auth.py` (config save endpoint, if it validates scheduler fields)
|
||||
|
||||
**Goal. How it should be**
|
||||
Add a new boolean field `folder_scan_enabled` (default `false`) to `SchedulerConfig`. When `true`, the scheduler will execute the folder maintenance routine during its scheduled run. Add the field to the setup page as a checkbox. Ensure existing configs without this field load successfully (Pydantic default handles this).
|
||||
|
||||
**Possible traps and issues**
|
||||
- Backward compatibility: old `data/config.json` files must load without errors. Pydantic defaults solve this, but verify by loading an old config.
|
||||
- The setup page JavaScript must include the new field in the payload sent to `/api/config`.
|
||||
- Do not confuse this with `auto_download_after_rescan` — this is a separate toggle.
|
||||
|
||||
**Docs changes needed**
|
||||
- `docs/CONFIGURATION.md`: Document the new `scheduler.folder_scan_enabled` option.
|
||||
- `docs/ARCHITECTURE.md`: Mention folder scan in the scheduler section.
|
||||
|
||||
**Why this is needed**
|
||||
Users need an opt-in toggle to enable automatic daily folder maintenance (NFO repair, folder renaming, poster checks) without forcing it on everyone.
|
||||
|
||||
---
|
||||
|
||||
## Task 1 — Replace shared NFOService in BackgroundLoaderService with per-task instances
|
||||
### Task 1.2: Create FolderScanService skeleton
|
||||
|
||||
- [x] Completed
|
||||
**Where is that found**
|
||||
- New file: `src/server/services/folder_scan_service.py`
|
||||
- `src/server/services/scheduler_service.py` (to call it)
|
||||
|
||||
### Where
|
||||
`src/server/services/background_loader_service.py` — method `_load_nfo_and_images` (~line 555)
|
||||
**Goal. How it should be**
|
||||
Create a new `FolderScanService` class with a single async entry point `async def run_folder_scan(self) -> None`. The method should:
|
||||
1. Log start/completion with structlog.
|
||||
2. Check prerequisites (`settings.anime_directory` exists, `settings.tmdb_api_key` is set).
|
||||
3. Skip gracefully with a warning log if prerequisites are missing.
|
||||
4. Use a module-level semaphore (similar to `_NFO_REPAIR_SEMAPHORE`) to limit concurrent TMDB operations to 3.
|
||||
|
||||
```python
|
||||
nfo_path = await self.series_app.nfo_service.create_tvshow_nfo(
|
||||
serie_name=task.name,
|
||||
serie_folder=task.folder,
|
||||
year=task.year,
|
||||
...
|
||||
)
|
||||
```
|
||||
Keep the implementation empty for the sub-tasks (1.3–1.5) to fill in. Just add the skeleton and the semaphore.
|
||||
|
||||
### Goal
|
||||
Create a fresh, isolated `NFOService` (with its own `TMDBClient` and `aiohttp` session) for every background loading task, exactly the same way `_repair_one_series` in `initialization_service.py` already does it.
|
||||
Each task must own its client so that closing the session at the end of one task never kills an in-flight request inside another task.
|
||||
**Possible traps and issues**
|
||||
- Circular imports: `folder_scan_service.py` will import from `initialization_service`, `config.settings`, etc. Keep imports inside methods or at the bottom if circular issues arise.
|
||||
- The service should follow the singleton pattern like `SchedulerService` and `DownloadService` if it holds state, or be stateless. For simplicity, make it a plain class instantiated per call or a module-level function set.
|
||||
- Exception handling: any unhandled exception in the scheduled task should be caught and logged so it doesn't crash the scheduler.
|
||||
|
||||
### How it should look
|
||||
```python
|
||||
from src.core.services.nfo_factory import NFOServiceFactory
|
||||
**Docs changes needed**
|
||||
- `docs/ARCHITECTURE.md`: Add `folder_scan_service.py` to the services list.
|
||||
|
||||
factory = NFOServiceFactory()
|
||||
nfo_service = factory.create()
|
||||
nfo_path = await nfo_service.create_tvshow_nfo(
|
||||
serie_name=task.name,
|
||||
serie_folder=task.folder,
|
||||
year=task.year,
|
||||
...
|
||||
)
|
||||
```
|
||||
|
||||
### Possible traps and issues
|
||||
- `NFOServiceFactory.create()` raises `ValueError` if no TMDB API key is available. Wrap in try/except and fall back gracefully (same behaviour as now when `nfo_service` is `None`).
|
||||
- The factory reads the API key from `settings` first, then from `config.json`. Do not pass the key explicitly so the fallback chain stays intact.
|
||||
- Each new `NFOService` opens its own `aiohttp` connector. Make sure to call `await nfo_service.close()` in a `finally` block to avoid connector leaks.
|
||||
|
||||
### Docs changes needed
|
||||
None — this is an internal implementation detail.
|
||||
|
||||
### Why this is needed
|
||||
Up to 5 background workers share one `NFOService`/`TMDBClient` instance. The `async with self.tmdb_client:` context manager inside `create_tvshow_nfo` calls `close()` on `__aexit__`, setting `session = None`. When Worker B exits its context while Worker A is still inside `_enrich_details_with_fallback` trying the `en-US` fallback request, that request throws "Connector is closed". The exception is silently swallowed, both `en-US` and `ja-JP` fallbacks fail, `details["overview"]` stays empty, and `plot` is written as an empty element.
|
||||
**Why this is needed**
|
||||
Encapsulates the new daily maintenance logic in its own module, keeping `scheduler_service.py` clean and allowing the folder scan to be tested independently.
|
||||
|
||||
---
|
||||
|
||||
## Task 2 — Guard NFOService init in SeriesApp on factory fallback, not just env var
|
||||
### Task 1.3: Integrate NFO repair into folder scan
|
||||
|
||||
- [x] Completed
|
||||
**Where is that found**
|
||||
- `src/server/services/folder_scan_service.py`
|
||||
- `src/server/services/initialization_service.py` (`perform_nfo_repair_scan`)
|
||||
|
||||
### Where
|
||||
`src/core/SeriesApp.py` — `__init__` method (~line 175)
|
||||
**Goal. How it should be**
|
||||
Inside `FolderScanService.run_folder_scan()`, call `perform_nfo_repair_scan(background_loader=None)` as the first step. Reuse the existing function exactly — do not copy its logic. Log a message before and after the call.
|
||||
|
||||
```python
|
||||
self.nfo_service: Optional[NFOService] = None
|
||||
if settings.tmdb_api_key: # ← checks env var ONLY
|
||||
factory = get_nfo_factory()
|
||||
self.nfo_service = factory.create()
|
||||
```
|
||||
**Possible traps and issues**
|
||||
- `perform_nfo_repair_scan` spawns `asyncio.create_task` for each repair. When called from the scheduler, these background tasks will still run after `run_folder_scan` returns. This is fine, but log that repairs are queued.
|
||||
- The function already handles missing `tmdb_api_key` and `anime_directory`, so the caller doesn't need to double-check, but the skeleton from Task 1.2 already checks prerequisites.
|
||||
- `perform_nfo_repair_scan` imports `nfo_needs_repair` and `NfoRepairService` inside the function, so no heavy import-time dependencies.
|
||||
|
||||
### Goal
|
||||
The guard condition should be equivalent to what `NFOServiceFactory.create()` itself checks: whether the key is available from *any* source (env var or `config.json`). Replace the guard with a try/create pattern so that `nfo_service` is initialised whenever the factory would succeed.
|
||||
**Docs changes needed**
|
||||
- `docs/NFO_GUIDE.md`: Update the "Automatic NFO Repair" section to state that repair now runs as part of the scheduled folder scan instead of every startup.
|
||||
|
||||
### How it should look
|
||||
```python
|
||||
self.nfo_service: Optional[NFOService] = None
|
||||
try:
|
||||
from src.core.services.nfo_factory import get_nfo_factory
|
||||
factory = get_nfo_factory()
|
||||
self.nfo_service = factory.create()
|
||||
logger.info("NFO service initialized successfully")
|
||||
except ValueError:
|
||||
logger.info("NFO service not available — TMDB API key not configured")
|
||||
except Exception as e:
|
||||
logger.warning("Failed to initialize NFO service: %s", e)
|
||||
```
|
||||
|
||||
### Possible traps and issues
|
||||
- This changes the condition from "env var set" to "factory can produce a service". The factory already has a safe fallback and raises `ValueError` when no key exists — so the `except ValueError` path is the normal "not configured" case, not an error.
|
||||
- `SeriesApp` is used in tests with `settings.tmdb_api_key = None`. Those tests must not be affected; the `except ValueError` branch keeps behaviour identical.
|
||||
- `series_app.nfo_service` is still `None` when not configured — downstream code that checks `if self.series_app.nfo_service:` remains correct.
|
||||
|
||||
### Docs changes needed
|
||||
`docs/CONFIGURATION.md` — note that `TMDB_API_KEY` env var is not required if `nfo.tmdb_api_key` is set in `config.json`.
|
||||
|
||||
### Why this is needed
|
||||
If the TMDB API key is configured only via `config.json` (not the `TMDB_API_KEY` env var), `settings.tmdb_api_key` is `None` and the guard prevents `nfo_service` from ever being created. The background loader then skips NFO creation completely (`nfo_service` is `None`). The repair scan at startup uses `NFOServiceFactory` directly (reads config.json) so it does create the NFO — which is exactly why restart works but add does not.
|
||||
**Why this is needed**
|
||||
Reuses the existing, tested NFO repair logic. Moves NFO repair from startup blocking to scheduled background maintenance.
|
||||
|
||||
---
|
||||
|
||||
## Task 3 — Remove non-reentrant `async with self.tmdb_client:` from NFOService public methods
|
||||
### Task 1.4: Validate and rename series folders
|
||||
|
||||
- [x] Completed
|
||||
**Where is that found**
|
||||
- `src/server/services/folder_scan_service.py`
|
||||
- `src/core/services/nfo_repair_service.py` (for `parse_nfo_tags` or similar NFO parsing)
|
||||
- `src/server/database/models.py` / `src/server/database/system_settings_service.py` (if folder paths are stored in DB)
|
||||
|
||||
### Where
|
||||
`src/core/services/nfo_service.py` — `create_tvshow_nfo` (~line 151) and `update_tvshow_nfo` (~line 265)
|
||||
**Goal. How it should be**
|
||||
After NFO repair, iterate over every subfolder in `settings.anime_directory` that contains a `tvshow.nfo`. For each folder:
|
||||
1. Parse the NFO to extract `<title>` and `<year>` text values.
|
||||
2. Compute the expected folder name: `f"{title} ({year})"`.
|
||||
3. Sanitize the expected name for filesystem safety (remove/replace illegal characters like `/`, `\`, `:`, etc.).
|
||||
4. Compare with the current folder name (`series_dir.name`).
|
||||
5. If different, rename the folder using `series_dir.rename(expected_path)`.
|
||||
6. If the series path is stored in the database (check `anime_service` or DB models), update the database record to point to the new path.
|
||||
|
||||
```python
|
||||
async with self.tmdb_client:
|
||||
details = await self.tmdb_client.get_tv_show_details(...)
|
||||
...
|
||||
```
|
||||
Skip folders where title or year is missing/empty. Log every rename action.
|
||||
|
||||
### Goal
|
||||
The `TMDBClient.__aenter__` / `__aexit__` open and **close** the session, making any concurrent call to the same client instance fail. Because Task 1 creates a fresh instance per call, this context manager becomes redundant. Change both methods to use `_ensure_session()` at the start and `close()` in a `finally` block, or simply call `await self.tmdb_client._ensure_session()` once and close after all requests. This makes the lifetime explicit and prevents double-close if the caller already manages it.
|
||||
**Possible traps and issues**
|
||||
- **Database path consistency**: If `Series` or `Episode` models store absolute or relative paths, renaming the folder on disk without updating the DB will break downloads, NFO updates, and the web UI. Must verify whether paths are stored in the DB and update them.
|
||||
- **Active downloads**: A series currently being downloaded should not be renamed. Check the download queue or lock status before renaming. If no lock mechanism exists, this is a major trap — document it.
|
||||
- **Filesystem permissions**: The app may not have write permission to the anime directory. Catch `PermissionError` and `OSError` and log gracefully.
|
||||
- **Special characters**: Titles like `"A / B"` or `"Show: Subtitle"` contain characters illegal in folder names. Define a sanitization function (e.g., replace `/` with `-`, remove trailing dots on Windows, etc.).
|
||||
- **Duplicate names**: Two different series could sanitize to the same name. Check if target path already exists before renaming.
|
||||
- **Path length limits**: Very long titles might exceed OS path limits.
|
||||
|
||||
### How it should look
|
||||
```python
|
||||
async def create_tvshow_nfo(self, ...) -> Path:
|
||||
try:
|
||||
await self.tmdb_client._ensure_session()
|
||||
search_results = await self.tmdb_client.search_tv_show(search_name)
|
||||
...
|
||||
finally:
|
||||
await self.tmdb_client.close()
|
||||
```
|
||||
**Docs changes needed**
|
||||
- `docs/NFO_GUIDE.md`: Add a section "Folder Naming Convention" explaining the `<title> (<year>)` format.
|
||||
- `docs/CONFIGURATION.md`: Mention that enabling folder scan will rename folders.
|
||||
|
||||
### Possible traps and issues
|
||||
- `TMDBClient.close()` is idempotent (checks `session.closed` before closing), so calling it in `finally` is safe even if the try block never opened a session.
|
||||
- After Task 1 every `NFOService` is short-lived (one call), so `finally: close()` effectively replaces the context manager with no behaviour change.
|
||||
- Do not remove the `__aenter__`/`__aexit__` from `TMDBClient` itself — other callers (e.g. tests, CLI) may still use it as a context manager.
|
||||
- `update_tvshow_nfo` has the same pattern; fix both methods.
|
||||
|
||||
### Docs changes needed
|
||||
None — internal implementation detail.
|
||||
|
||||
### Why this is needed
|
||||
Even after Task 1 fixes the shared-instance problem, the `async with self.tmdb_client:` pattern is fragile by design: `__aexit__` calls `close()`, which would break any hypothetical future reuse. Removing the implicit close makes the session lifetime explicit and eliminates the root mechanism that caused the original bug.
|
||||
**Why this is needed**
|
||||
Enforces a consistent, predictable folder naming scheme across the library, making it easier for media center apps (Kodi, Jellyfin, Plex) to match metadata.
|
||||
|
||||
---
|
||||
|
||||
## Task 4 — Add `en-US` search fallback so `search_overview` is never empty
|
||||
### Task 1.5: Check and download missing poster.jpg
|
||||
|
||||
### Where
|
||||
`src/core/services/nfo_service.py` — `create_tvshow_nfo` (~line 178) and `_enrich_details_with_fallback` (~line 395)
|
||||
**Where is that found**
|
||||
- `src/server/services/folder_scan_service.py`
|
||||
- `src/core/utils/image_downloader.py` (`ImageDownloader`)
|
||||
- `src/core/services/nfo_service.py` or `src/core/services/nfo_repair_service.py` (to get poster URL from NFO or TMDB)
|
||||
|
||||
```python
|
||||
search_overview = tv_show.get("overview") or None # always None for anime — de-DE search returns ""
|
||||
```
|
||||
**Goal. How it should be**
|
||||
After folder renaming, iterate over series folders again (or combine with Task 1.4 loop). For each folder:
|
||||
1. Check if `poster.jpg` exists and has a size ≥ `ImageDownloader.min_file_size` (1 KB by default).
|
||||
2. If missing or too small:
|
||||
a. Parse `tvshow.nfo` for `<thumb aspect="poster">` or `<thumb>` URL.
|
||||
b. If no URL in NFO, skip (do not query TMDB again to keep tasks small; the NFO should already have it after repair).
|
||||
c. Use `ImageDownloader` (with context manager) to download the image to `series_dir / "poster.jpg"`.
|
||||
d. Validate the downloaded image with `ImageDownloader._validate_image` (or similar existing validation).
|
||||
3. Use the existing `_NFO_REPAIR_SEMAPHORE` or a new `POSTER_DOWNLOAD_SEMAPHORE` to limit concurrent downloads to 3.
|
||||
|
||||
### Goal
|
||||
When the German `search_tv_show` result has an empty `overview`, perform a second search in `en-US` to obtain a non-empty overview as the last-resort fallback text. Store this as `search_overview` so `_enrich_details_with_fallback` can use it even if all language-specific detail requests fail.
|
||||
**Possible traps and issues**
|
||||
- **TMDB rate limiting**: Even downloading images hits TMDB CDN. The semaphore limits concurrency.
|
||||
- **Invalid images**: A download might produce a 0-byte or corrupted file. `ImageDownloader` already validates with PIL; reuse that.
|
||||
- **NFO without thumb URL**: If the NFO was created before thumb tags were added, there may be no URL. In that case, skip and log. A future task could query TMDB directly.
|
||||
- **Write permissions**: Same as Task 1.4.
|
||||
- **Async session sharing**: `ImageDownloader` manages its own `aiohttp` session. Use `async with ImageDownloader() as downloader:` to ensure cleanup.
|
||||
|
||||
### How it should look
|
||||
```python
|
||||
search_overview = tv_show.get("overview") or None
|
||||
if not search_overview:
|
||||
try:
|
||||
en_results = await self.tmdb_client.search_tv_show(search_name, language="en-US")
|
||||
en_match = self._find_best_match(en_results.get("results", []), search_name, year)
|
||||
search_overview = en_match.get("overview") or None
|
||||
except Exception:
|
||||
pass # best-effort only
|
||||
```
|
||||
**Docs changes needed**
|
||||
- `docs/NFO_GUIDE.md`: Add "Poster Check" subsection under folder scan.
|
||||
- `docs/CONFIGURATION.md`: Mention that `nfo.download_poster` setting also affects scheduled poster checks.
|
||||
|
||||
### Possible traps and issues
|
||||
- This adds one extra TMDB request per series when the German overview is empty. It is best-effort and must be wrapped in a broad `except` so it never blocks NFO creation.
|
||||
- The TMDB search endpoint rate-limit is generous; one extra request per add is negligible.
|
||||
- `_find_best_match` can raise `TMDBAPIError` if the result list is empty — catch both `TMDBAPIError` and `Exception`.
|
||||
- `update_tvshow_nfo` calls `_enrich_details_with_fallback` without `search_overview`. This is acceptable because the detail request with `en-US` fallback covers it; the search overview is only a last resort for the create path.
|
||||
**Why this is needed**
|
||||
Ensures every series has artwork, which is required by most media center front-ends for a polished library view.
|
||||
|
||||
### Docs changes needed
|
||||
None — transparent improvement.
|
||||
---
|
||||
|
||||
### Why this is needed
|
||||
Most anime have no German translation on TMDB. The `de-DE` search result returns `overview: ""`. The current code stores this as `search_overview = None` so the last-resort fallback in `_enrich_details_with_fallback` never fires. Combined with session contention (Task 1), the detail-level `en-US` fallback also fails, leaving `plot` empty. This task ensures that at least the search-level `en-US` overview is available as a safety net.
|
||||
## 2. Remove startup NFO repair
|
||||
|
||||
### Task 2.1: Remove perform_nfo_repair_scan from startup lifespan
|
||||
|
||||
**Where is that found**
|
||||
- `src/server/fastapi_app.py` (lifespan startup block, lines ~245 and ~319)
|
||||
- `src/server/services/initialization_service.py` (keep the function, just remove the call site)
|
||||
- `tests/integration/test_nfo_repair_startup.py`
|
||||
- `tests/unit/test_initialization_service.py` (tests that call `perform_nfo_repair_scan` directly can stay, but integration tests verifying startup wiring must change)
|
||||
|
||||
**Goal. How it should be**
|
||||
1. In `src/server/fastapi_app.py`, remove the import of `perform_nfo_repair_scan` from the `initialization_service` import block.
|
||||
2. Remove the line `await perform_nfo_repair_scan(background_loader)` from the lifespan startup sequence.
|
||||
3. Update `tests/integration/test_nfo_repair_startup.py`:
|
||||
- Remove or modify `test_perform_nfo_repair_scan_imported_in_lifespan` and `test_perform_nfo_repair_scan_called_after_media_scan` since the startup wiring is gone.
|
||||
- Replace with a test that verifies `perform_nfo_repair_scan` is NOT called during startup (or simply delete the file if it has no other purpose).
|
||||
4. `tests/unit/test_initialization_service.py` tests for `perform_nfo_repair_scan` can remain because they test the function itself, not the startup wiring.
|
||||
|
||||
**Possible traps and issues**
|
||||
- **Test failures**: `test_nfo_repair_startup.py` will fail immediately after the code change. It must be updated in the same PR.
|
||||
- **Documentation drift**: `docs/NFO_GUIDE.md`, `docs/CHANGELOG.md`, and `docs/ARCHITECTURE.md` all describe the startup NFO repair behavior. If docs are not updated, users will expect repair on every start.
|
||||
- **Background loader parameter**: The `background_loader` variable was created partly for `perform_nfo_repair_scan`. After removal, check if `background_loader` is still needed for other startup steps (yes — `perform_media_scan_if_needed` uses it). Do not remove `background_loader` entirely.
|
||||
- **Import cleanup**: Ensure no unused imports remain in `fastapi_app.py` after removal.
|
||||
|
||||
**Docs changes needed**
|
||||
- `docs/NFO_GUIDE.md`: Update section 11 "Automatic NFO Repair" to remove startup references and state it runs via scheduler.
|
||||
- `docs/CHANGELOG.md`: Add an entry under "Changed" or "Removed" noting that startup NFO repair is replaced by scheduled folder scan.
|
||||
- `docs/ARCHITECTURE.md`: Update the startup sequence description.
|
||||
|
||||
**Why this is needed**
|
||||
Running `perform_nfo_repair_scan` on every startup slows down server restarts, especially for large libraries. Moving it to a scheduled task keeps startup fast while still ensuring regular maintenance.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "aniworld-web",
|
||||
"version": "1.0.0",
|
||||
"version": "1.4.7",
|
||||
"description": "Aniworld Anime Download Manager - Web Frontend",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
@@ -22,6 +22,7 @@ APScheduler>=3.10.4
|
||||
Events>=0.5
|
||||
requests>=2.31.0
|
||||
beautifulsoup4>=4.12.0
|
||||
chardet>=5.2.0
|
||||
fake-useragent>=1.4.0
|
||||
yt-dlp>=2024.1.0
|
||||
urllib3>=2.0.0
|
||||
@@ -1,10 +1,8 @@
|
||||
"""CLI command for NFO management.
|
||||
|
||||
This script provides command-line interface for creating, updating,
|
||||
and checking NFO metadata files.
|
||||
Note: NFO service has been removed. This CLI is no longer functional.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
@@ -12,9 +10,6 @@ from pathlib import Path
|
||||
# Add src to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
||||
|
||||
from src.config.settings import settings
|
||||
from src.core.services.series_manager_service import SeriesManagerService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -125,7 +120,7 @@ async def check_nfo_status():
|
||||
logger.info("Anime Directory: %s", settings.anime_directory)
|
||||
|
||||
# Create series list (no NFO service needed for status check)
|
||||
from src.core.entities.SerieList import SerieList
|
||||
from src.server.database.SerieList import SerieList
|
||||
serie_list = SerieList(settings.anime_directory)
|
||||
all_series = serie_list.get_all()
|
||||
|
||||
@@ -179,91 +174,6 @@ async def check_nfo_status():
|
||||
return 0
|
||||
|
||||
|
||||
async def update_nfo_files():
|
||||
"""Update existing NFO files with fresh data from TMDB."""
|
||||
logger.info("%s", "=" * 70)
|
||||
logger.info("NFO Update Tool")
|
||||
logger.info("%s", "=" * 70)
|
||||
|
||||
if not settings.tmdb_api_key:
|
||||
logger.error("TMDB_API_KEY not configured")
|
||||
logger.error("Set TMDB_API_KEY in .env file or environment")
|
||||
logger.error("Get API key from: https://www.themoviedb.org/settings/api")
|
||||
return 1
|
||||
|
||||
if not settings.anime_directory:
|
||||
logger.error("ANIME_DIRECTORY not configured")
|
||||
return 1
|
||||
|
||||
logger.info("Anime Directory: %s", settings.anime_directory)
|
||||
logger.info(
|
||||
"Download media: %s",
|
||||
settings.nfo_download_poster or settings.nfo_download_logo or settings.nfo_download_fanart,
|
||||
)
|
||||
|
||||
# Get series with NFO
|
||||
from src.core.entities.SerieList import SerieList
|
||||
serie_list = SerieList(settings.anime_directory)
|
||||
all_series = serie_list.get_all()
|
||||
series_with_nfo = [s for s in all_series if s.has_nfo()]
|
||||
|
||||
if not series_with_nfo:
|
||||
logger.warning("No series with NFO files found")
|
||||
logger.info("Run 'scan' command first to create NFO files")
|
||||
return 0
|
||||
|
||||
logger.info("Found %d series with NFO files", len(series_with_nfo))
|
||||
logger.info("Updating NFO files with fresh data from TMDB...")
|
||||
logger.info("This may take a while")
|
||||
|
||||
# Initialize NFO service using factory
|
||||
from src.core.services.nfo_factory import create_nfo_service
|
||||
try:
|
||||
nfo_service = create_nfo_service()
|
||||
except ValueError as e:
|
||||
logger.error("Error creating NFO service: %s", e)
|
||||
return 1
|
||||
|
||||
success_count = 0
|
||||
error_count = 0
|
||||
|
||||
try:
|
||||
for i, serie in enumerate(series_with_nfo, 1):
|
||||
logger.info("[%d/%d] Updating: %s", i, len(series_with_nfo), serie.name)
|
||||
|
||||
try:
|
||||
await nfo_service.update_tvshow_nfo(
|
||||
serie_folder=serie.folder,
|
||||
download_media=(
|
||||
settings.nfo_download_poster or
|
||||
settings.nfo_download_logo or
|
||||
settings.nfo_download_fanart
|
||||
),
|
||||
)
|
||||
logger.info("Updated successfully: %s", serie.name)
|
||||
success_count += 1
|
||||
|
||||
# Small delay to respect API rate limits
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Failed to update NFO for %s", serie.name)
|
||||
error_count += 1
|
||||
|
||||
logger.info("%s", "=" * 70)
|
||||
logger.info("Update complete")
|
||||
logger.info("Success: %d", success_count)
|
||||
logger.info("Errors: %d", error_count)
|
||||
|
||||
except Exception:
|
||||
logger.exception("Fatal error during NFO update")
|
||||
return 1
|
||||
finally:
|
||||
await nfo_service.close()
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
"""Main CLI entry point."""
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
@@ -273,7 +183,6 @@ def main():
|
||||
logger.info("\nUsage:")
|
||||
logger.info(" python -m src.cli.nfo_cli scan # Scan and create missing NFO files")
|
||||
logger.info(" python -m src.cli.nfo_cli status # Check NFO status for all series")
|
||||
logger.info(" python -m src.cli.nfo_cli update # Update existing NFO files with fresh data")
|
||||
logger.info("\nConfiguration:")
|
||||
logger.info(" Set TMDB_API_KEY in .env file")
|
||||
logger.info(" Set NFO_AUTO_CREATE=true to enable auto-creation")
|
||||
@@ -286,11 +195,9 @@ def main():
|
||||
return asyncio.run(scan_and_create_nfo())
|
||||
elif command == "status":
|
||||
return asyncio.run(check_nfo_status())
|
||||
elif command == "update":
|
||||
return asyncio.run(update_nfo_files())
|
||||
else:
|
||||
logger.error("Unknown command: %s", command)
|
||||
logger.info("Use 'scan', 'status', or 'update'")
|
||||
logger.info("Use 'scan' or 'status'")
|
||||
return 1
|
||||
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import re
|
||||
import secrets
|
||||
from typing import Optional
|
||||
|
||||
@@ -114,6 +115,40 @@ class Settings(BaseSettings):
|
||||
validation_alias="NFO_PREFER_FSK_RATING",
|
||||
description="Prefer German FSK rating over MPAA rating in NFO files"
|
||||
)
|
||||
nfo_folder_ignore_patterns: str = Field(
|
||||
default="The Last of Us|Loki|Chernobyl|Star Trek Discovery|Marvel|Matrix|Fast & Furious|Jurassic|James Bond|Mission: Impossible|Bourne|Hunger Games|Die Hard|John Wick|Pacific Rim|Guardians of the Galaxy|Avengers|Batman|Superman|Wonder Woman|Spider-Man|X-Men|Fantastic Four|Terminator|Predator|Rambo|Rocky|Expendables|Tomb Raider|Jumanji|Jurassic Park|Pirates of the Caribbean|Harry Potter|Lord of the Rings|Hobbit|Game of Thrones|Westworld|Stranger Things|Breaking Bad|Better Call Saul|Sherlock|Downton Abbey|The Crown|Bridgerton|Sex Education|Normal People|Emily in Paris|The Witcher|Servant|Lucifer|Dark|Shadow and Bone|Grimm|Fairytale",
|
||||
validation_alias="NFO_FOLDER_IGNORE_PATTERNS",
|
||||
description="Regex patterns for folder names to skip during scan (pipe-separated)"
|
||||
)
|
||||
|
||||
@property
|
||||
def folder_ignore_patterns(self) -> list[str]:
|
||||
"""Parse ignore patterns from comma-separated string into list.
|
||||
|
||||
Returns:
|
||||
List of regex patterns to skip during folder scanning.
|
||||
"""
|
||||
if not self.nfo_folder_ignore_patterns:
|
||||
return []
|
||||
return [
|
||||
pattern.strip()
|
||||
for pattern in self.nfo_folder_ignore_patterns.split("|")
|
||||
if pattern.strip()
|
||||
]
|
||||
|
||||
def should_ignore_folder(self, folder_name: str) -> bool:
|
||||
"""Check if folder should be ignored based on ignore patterns.
|
||||
|
||||
Args:
|
||||
folder_name: Name of folder to check.
|
||||
|
||||
Returns:
|
||||
True if folder matches any ignore pattern, False otherwise.
|
||||
"""
|
||||
for pattern in self.folder_ignore_patterns:
|
||||
if re.search(pattern, folder_name, re.IGNORECASE):
|
||||
return True
|
||||
return False
|
||||
|
||||
@property
|
||||
def allowed_origins(self) -> list[str]:
|
||||
@@ -134,5 +169,23 @@ class Settings(BaseSettings):
|
||||
]
|
||||
return [origin.strip() for origin in raw.split(",") if origin.strip()]
|
||||
|
||||
@property
|
||||
def scan_key_overrides(self) -> dict[str, str]:
|
||||
"""Return scan key overrides from config.json.
|
||||
|
||||
Maps folder names to provider keys for cases where auto-generated
|
||||
keys from folder names are incorrect.
|
||||
|
||||
Returns:
|
||||
Dict mapping folder names to provider keys.
|
||||
"""
|
||||
from src.server.services.config_service import ConfigService
|
||||
try:
|
||||
config_service = ConfigService()
|
||||
config = config_service.load_config()
|
||||
return config.scan_key_overrides or {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
settings = Settings()
|
||||
|
||||
@@ -1,320 +0,0 @@
|
||||
"""Utilities for loading and managing stored anime series metadata.
|
||||
|
||||
This module provides the SerieList class for managing collections of anime
|
||||
series metadata. It uses file-based storage only.
|
||||
|
||||
Note:
|
||||
This module is part of the core domain layer and has no database
|
||||
dependencies. All database operations are handled by the service layer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import warnings
|
||||
from json import JSONDecodeError
|
||||
from typing import Dict, Iterable, List, Optional
|
||||
|
||||
from src.core.entities.series import Serie
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SerieList:
|
||||
"""
|
||||
Represents the collection of cached series stored on disk.
|
||||
|
||||
Series are identified by their unique 'key' (provider identifier).
|
||||
The 'folder' is metadata only and not used for lookups.
|
||||
|
||||
This class manages in-memory series data loaded from filesystem.
|
||||
It has no database dependencies - all persistence is handled by
|
||||
the service layer.
|
||||
|
||||
Example:
|
||||
# File-based mode
|
||||
serie_list = SerieList("/path/to/anime")
|
||||
series = serie_list.get_all()
|
||||
|
||||
Attributes:
|
||||
directory: Path to the anime directory
|
||||
keyDict: Internal dictionary mapping serie.key to Serie objects
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_path: str,
|
||||
skip_load: bool = False
|
||||
) -> None:
|
||||
"""Initialize the SerieList.
|
||||
|
||||
Args:
|
||||
base_path: Path to the anime directory
|
||||
skip_load: If True, skip automatic loading of series from files.
|
||||
Useful when planning to load from database instead.
|
||||
"""
|
||||
self.directory: str = base_path
|
||||
# Internal storage using serie.key as the dictionary key
|
||||
self.keyDict: Dict[str, Serie] = {}
|
||||
|
||||
# Only auto-load from files if not skipping
|
||||
if not skip_load:
|
||||
self.load_series()
|
||||
|
||||
def add(self, serie: Serie, use_sanitized_folder: bool = True) -> str:
|
||||
"""
|
||||
Persist a new series if it is not already present (file-based mode).
|
||||
|
||||
Uses serie.key for identification. Creates the filesystem folder
|
||||
using either the sanitized display name (default) or the existing
|
||||
folder property.
|
||||
|
||||
Args:
|
||||
serie: The Serie instance to add
|
||||
use_sanitized_folder: If True (default), use serie.sanitized_folder
|
||||
for the filesystem folder name based on display name.
|
||||
If False, use serie.folder as-is for backward compatibility.
|
||||
|
||||
Returns:
|
||||
str: The folder path that was created/used
|
||||
|
||||
Note:
|
||||
This method creates data files on disk. For database storage,
|
||||
use add_to_db() instead.
|
||||
"""
|
||||
if self.contains(serie.key):
|
||||
# Return existing folder path
|
||||
existing = self.keyDict[serie.key]
|
||||
return os.path.join(self.directory, existing.folder)
|
||||
|
||||
# Determine folder name to use
|
||||
if use_sanitized_folder:
|
||||
folder_name = serie.sanitized_folder
|
||||
# Update the serie's folder property to match what we create
|
||||
serie.folder = folder_name
|
||||
else:
|
||||
folder_name = serie.folder
|
||||
|
||||
data_path = os.path.join(self.directory, folder_name, "data")
|
||||
anime_path = os.path.join(self.directory, folder_name)
|
||||
os.makedirs(anime_path, exist_ok=True)
|
||||
if not os.path.isfile(data_path):
|
||||
serie.save_to_file(data_path)
|
||||
# Store by key, not folder
|
||||
self.keyDict[serie.key] = serie
|
||||
|
||||
return anime_path
|
||||
|
||||
def contains(self, key: str) -> bool:
|
||||
"""
|
||||
Return True when a series identified by ``key`` already exists.
|
||||
|
||||
Args:
|
||||
key: The unique provider identifier for the series
|
||||
|
||||
Returns:
|
||||
True if the series exists in the collection
|
||||
"""
|
||||
return key in self.keyDict
|
||||
|
||||
def load_series(self) -> None:
|
||||
"""Populate the in-memory map with metadata discovered on disk."""
|
||||
|
||||
logger.info("Scanning anime folders in %s", self.directory)
|
||||
try:
|
||||
entries: Iterable[str] = os.listdir(self.directory)
|
||||
except OSError as error:
|
||||
logger.error(
|
||||
"Unable to scan directory %s: %s",
|
||||
self.directory,
|
||||
error,
|
||||
)
|
||||
return
|
||||
|
||||
nfo_stats = {"total": 0, "with_nfo": 0, "without_nfo": 0}
|
||||
media_stats = {
|
||||
"with_poster": 0,
|
||||
"without_poster": 0,
|
||||
"with_logo": 0,
|
||||
"without_logo": 0,
|
||||
"with_fanart": 0,
|
||||
"without_fanart": 0
|
||||
}
|
||||
|
||||
for anime_folder in entries:
|
||||
anime_path = os.path.join(self.directory, anime_folder, "data")
|
||||
if os.path.isfile(anime_path):
|
||||
logger.debug("Found data file for folder %s", anime_folder)
|
||||
serie = self._load_data(anime_folder, anime_path)
|
||||
|
||||
if serie:
|
||||
nfo_stats["total"] += 1
|
||||
# Check for NFO file
|
||||
nfo_file_path = os.path.join(
|
||||
self.directory, anime_folder, "tvshow.nfo"
|
||||
)
|
||||
if os.path.isfile(nfo_file_path):
|
||||
serie.nfo_path = nfo_file_path
|
||||
nfo_stats["with_nfo"] += 1
|
||||
else:
|
||||
nfo_stats["without_nfo"] += 1
|
||||
logger.debug(
|
||||
"Series '%s' (key: %s) is missing tvshow.nfo",
|
||||
serie.name,
|
||||
serie.key
|
||||
)
|
||||
|
||||
# Check for media files
|
||||
folder_path = os.path.join(self.directory, anime_folder)
|
||||
|
||||
poster_path = os.path.join(folder_path, "poster.jpg")
|
||||
if os.path.isfile(poster_path):
|
||||
media_stats["with_poster"] += 1
|
||||
else:
|
||||
media_stats["without_poster"] += 1
|
||||
logger.debug(
|
||||
"Series '%s' (key: %s) is missing poster.jpg",
|
||||
serie.name,
|
||||
serie.key
|
||||
)
|
||||
|
||||
logo_path = os.path.join(folder_path, "logo.png")
|
||||
if os.path.isfile(logo_path):
|
||||
media_stats["with_logo"] += 1
|
||||
else:
|
||||
media_stats["without_logo"] += 1
|
||||
logger.debug(
|
||||
"Series '%s' (key: %s) is missing logo.png",
|
||||
serie.name,
|
||||
serie.key
|
||||
)
|
||||
|
||||
fanart_path = os.path.join(folder_path, "fanart.jpg")
|
||||
if os.path.isfile(fanart_path):
|
||||
media_stats["with_fanart"] += 1
|
||||
else:
|
||||
media_stats["without_fanart"] += 1
|
||||
logger.debug(
|
||||
"Series '%s' (key: %s) is missing fanart.jpg",
|
||||
serie.name,
|
||||
serie.key
|
||||
)
|
||||
|
||||
continue
|
||||
|
||||
logger.warning(
|
||||
"Skipping folder %s because no metadata file was found",
|
||||
anime_folder,
|
||||
)
|
||||
|
||||
# Log summary statistics
|
||||
if nfo_stats["total"] > 0:
|
||||
logger.info(
|
||||
"NFO scan complete: %d series total, %d with NFO, %d without NFO",
|
||||
nfo_stats["total"],
|
||||
nfo_stats["with_nfo"],
|
||||
nfo_stats["without_nfo"]
|
||||
)
|
||||
logger.info(
|
||||
"Media scan complete: Poster (%d/%d), Logo (%d/%d), Fanart (%d/%d)",
|
||||
media_stats["with_poster"],
|
||||
nfo_stats["total"],
|
||||
media_stats["with_logo"],
|
||||
nfo_stats["total"],
|
||||
media_stats["with_fanart"],
|
||||
nfo_stats["total"]
|
||||
)
|
||||
|
||||
def _load_data(self, anime_folder: str, data_path: str) -> Optional[Serie]:
|
||||
"""
|
||||
Load a single series metadata file into the in-memory collection.
|
||||
|
||||
Args:
|
||||
anime_folder: The folder name (for logging only)
|
||||
data_path: Path to the metadata file
|
||||
|
||||
Returns:
|
||||
Serie: The loaded Serie object, or None if loading failed
|
||||
"""
|
||||
try:
|
||||
serie = Serie.load_from_file(data_path)
|
||||
# Store by key, not folder
|
||||
self.keyDict[serie.key] = serie
|
||||
logger.debug(
|
||||
"Successfully loaded metadata for %s (key: %s)",
|
||||
anime_folder,
|
||||
serie.key
|
||||
)
|
||||
return serie
|
||||
except (OSError, JSONDecodeError, KeyError, ValueError) as error:
|
||||
logger.error(
|
||||
"Failed to load metadata for folder %s from %s: %s",
|
||||
anime_folder,
|
||||
data_path,
|
||||
error,
|
||||
)
|
||||
return None
|
||||
|
||||
def GetMissingEpisode(self) -> List[Serie]:
|
||||
"""Return all series that still contain missing episodes."""
|
||||
return [
|
||||
serie
|
||||
for serie in self.keyDict.values()
|
||||
if serie.episodeDict
|
||||
]
|
||||
|
||||
def get_missing_episodes(self) -> List[Serie]:
|
||||
"""PEP8-friendly alias for :meth:`GetMissingEpisode`."""
|
||||
return self.GetMissingEpisode()
|
||||
|
||||
def GetList(self) -> List[Serie]:
|
||||
"""Return all series instances stored in the list."""
|
||||
return list(self.keyDict.values())
|
||||
|
||||
def get_all(self) -> List[Serie]:
|
||||
"""PEP8-friendly alias for :meth:`GetList`."""
|
||||
return self.GetList()
|
||||
|
||||
def get_by_key(self, key: str) -> Optional[Serie]:
|
||||
"""
|
||||
Get a series by its unique provider key.
|
||||
|
||||
This is the primary method for series lookup.
|
||||
|
||||
Args:
|
||||
key: The unique provider identifier (e.g., "attack-on-titan")
|
||||
|
||||
Returns:
|
||||
The Serie instance if found, None otherwise
|
||||
"""
|
||||
return self.keyDict.get(key)
|
||||
|
||||
def get_by_folder(self, folder: str) -> Optional[Serie]:
|
||||
"""
|
||||
Get a series by its folder name.
|
||||
|
||||
.. deprecated:: 2.0.0
|
||||
Use :meth:`get_by_key` instead. Folder-based lookups will be
|
||||
removed in version 3.0.0. The `folder` field is metadata only
|
||||
and should not be used for identification.
|
||||
|
||||
This method is provided for backward compatibility only.
|
||||
Prefer using get_by_key() for new code.
|
||||
|
||||
Args:
|
||||
folder: The filesystem folder name (e.g., "Attack on Titan (2013)")
|
||||
|
||||
Returns:
|
||||
The Serie instance if found, None otherwise
|
||||
"""
|
||||
warnings.warn(
|
||||
"get_by_folder() is deprecated and will be removed in v3.0.0. "
|
||||
"Use get_by_key() instead. The 'folder' field is metadata only.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
for serie in self.keyDict.values():
|
||||
if serie.folder == folder:
|
||||
return serie
|
||||
return None
|
||||
@@ -1,410 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from src.server.utils.filesystem import sanitize_folder_name
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Serie:
|
||||
"""
|
||||
Represents an anime series with metadata and episode information.
|
||||
|
||||
The `key` property is the unique identifier for the series
|
||||
(provider-assigned, URL-safe).
|
||||
The `folder` property is the filesystem folder name
|
||||
(metadata only, not used for lookups).
|
||||
|
||||
Args:
|
||||
key: Unique series identifier from provider
|
||||
(e.g., "attack-on-titan"). Cannot be empty.
|
||||
name: Display name of the series
|
||||
site: Provider site URL
|
||||
folder: Filesystem folder name (metadata only,
|
||||
e.g., "Attack on Titan (2013)")
|
||||
episodeDict: Dictionary mapping season numbers to
|
||||
lists of episode numbers
|
||||
year: Release year of the series (optional)
|
||||
|
||||
Raises:
|
||||
ValueError: If key is None or empty string
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
key: str,
|
||||
name: str,
|
||||
site: str,
|
||||
folder: str,
|
||||
episodeDict: dict[int, list[int]],
|
||||
year: int | None = None,
|
||||
nfo_path: Optional[str] = None
|
||||
):
|
||||
if not key or not key.strip():
|
||||
raise ValueError("Serie key cannot be None or empty")
|
||||
|
||||
self._key = key.strip()
|
||||
self._name = name
|
||||
self._site = site
|
||||
self._folder = folder
|
||||
self._episodeDict = episodeDict
|
||||
self._year = year
|
||||
self._nfo_path = nfo_path
|
||||
|
||||
def __str__(self):
|
||||
"""String representation of Serie object"""
|
||||
year_str = f", year={self.year}" if self.year else ""
|
||||
return (
|
||||
f"Serie(key='{self.key}', name='{self.name}', "
|
||||
f"site='{self.site}', folder='{self.folder}', "
|
||||
f"episodeDict={self.episodeDict}{year_str})"
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
"""Concise developer representation of Serie object."""
|
||||
season_count = len(self.episodeDict)
|
||||
episode_count = sum(len(eps) for eps in self.episodeDict.values())
|
||||
year_str = f", year={self.year}" if self.year else ""
|
||||
return (
|
||||
f"Serie(key={self.key!r}, name={self.name!r}"
|
||||
f"{year_str}, seasons={season_count}, episodes={episode_count})"
|
||||
)
|
||||
|
||||
@property
|
||||
def key(self) -> str:
|
||||
"""
|
||||
Unique series identifier (primary identifier for all lookups).
|
||||
|
||||
This is the provider-assigned, URL-safe identifier used
|
||||
throughout the application for series identification,
|
||||
lookups, and operations.
|
||||
|
||||
Returns:
|
||||
str: The unique series key
|
||||
"""
|
||||
return self._key
|
||||
|
||||
@key.setter
|
||||
def key(self, value: str):
|
||||
"""
|
||||
Set the unique series identifier.
|
||||
|
||||
Args:
|
||||
value: New key value
|
||||
|
||||
Raises:
|
||||
ValueError: If value is None or empty string
|
||||
"""
|
||||
if not value or not value.strip():
|
||||
raise ValueError("Serie key cannot be None or empty")
|
||||
self._key = value.strip()
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._name
|
||||
|
||||
@name.setter
|
||||
def name(self, value: str):
|
||||
self._name = value
|
||||
|
||||
@property
|
||||
def site(self) -> str:
|
||||
return self._site
|
||||
|
||||
@site.setter
|
||||
def site(self, value: str):
|
||||
self._site = value
|
||||
|
||||
@property
|
||||
def folder(self) -> str:
|
||||
"""
|
||||
Filesystem folder name (metadata only, not used for lookups).
|
||||
|
||||
This property contains the local directory name where the series
|
||||
files are stored. It should NOT be used as an identifier for
|
||||
series lookups - use `key` instead.
|
||||
|
||||
Returns:
|
||||
str: The filesystem folder name
|
||||
"""
|
||||
return self._folder
|
||||
|
||||
@folder.setter
|
||||
def folder(self, value: str):
|
||||
"""
|
||||
Set the filesystem folder name.
|
||||
|
||||
Args:
|
||||
value: Folder name for the series
|
||||
"""
|
||||
self._folder = value
|
||||
|
||||
@property
|
||||
def episodeDict(self) -> dict[int, list[int]]:
|
||||
return self._episodeDict
|
||||
|
||||
@episodeDict.setter
|
||||
def episodeDict(self, value: dict[int, list[int]]):
|
||||
self._episodeDict = value
|
||||
|
||||
@property
|
||||
def year(self) -> int | None:
|
||||
"""
|
||||
Release year of the series.
|
||||
|
||||
Returns:
|
||||
int or None: The year the series was released, or None if unknown
|
||||
"""
|
||||
return self._year
|
||||
|
||||
@year.setter
|
||||
def year(self, value: int | None):
|
||||
"""Set the release year of the series."""
|
||||
self._year = value
|
||||
|
||||
@property
|
||||
def nfo_path(self) -> Optional[str]:
|
||||
"""
|
||||
Path to the tvshow.nfo metadata file.
|
||||
|
||||
Returns:
|
||||
str or None: Path to the NFO file, or None if not set
|
||||
"""
|
||||
return self._nfo_path
|
||||
|
||||
@nfo_path.setter
|
||||
def nfo_path(self, value: Optional[str]):
|
||||
"""Set the path to the NFO file."""
|
||||
self._nfo_path = value
|
||||
|
||||
def has_nfo(self, base_directory: Optional[str] = None) -> bool:
|
||||
"""
|
||||
Check if tvshow.nfo file exists for this series.
|
||||
|
||||
Args:
|
||||
base_directory: Base anime directory path. If provided, checks
|
||||
relative to base_directory/folder/tvshow.nfo. If not provided,
|
||||
uses nfo_path directly.
|
||||
|
||||
Returns:
|
||||
bool: True if tvshow.nfo exists, False otherwise
|
||||
"""
|
||||
if base_directory:
|
||||
nfo_file = Path(base_directory) / self.folder / "tvshow.nfo"
|
||||
elif self._nfo_path:
|
||||
nfo_file = Path(self._nfo_path)
|
||||
else:
|
||||
return False
|
||||
|
||||
return nfo_file.exists() and nfo_file.is_file()
|
||||
|
||||
def has_poster(self, base_directory: Optional[str] = None) -> bool:
|
||||
"""
|
||||
Check if poster.jpg file exists for this series.
|
||||
|
||||
Args:
|
||||
base_directory: Base anime directory path. If provided, checks
|
||||
relative to base_directory/folder/poster.jpg.
|
||||
|
||||
Returns:
|
||||
bool: True if poster.jpg exists, False otherwise
|
||||
"""
|
||||
if not base_directory:
|
||||
return False
|
||||
|
||||
poster_file = Path(base_directory) / self.folder / "poster.jpg"
|
||||
return poster_file.exists() and poster_file.is_file()
|
||||
|
||||
def has_logo(self, base_directory: Optional[str] = None) -> bool:
|
||||
"""
|
||||
Check if logo.png file exists for this series.
|
||||
|
||||
Args:
|
||||
base_directory: Base anime directory path. If provided, checks
|
||||
relative to base_directory/folder/logo.png.
|
||||
|
||||
Returns:
|
||||
bool: True if logo.png exists, False otherwise
|
||||
"""
|
||||
if not base_directory:
|
||||
return False
|
||||
|
||||
logo_file = Path(base_directory) / self.folder / "logo.png"
|
||||
return logo_file.exists() and logo_file.is_file()
|
||||
|
||||
def has_fanart(self, base_directory: Optional[str] = None) -> bool:
|
||||
"""
|
||||
Check if fanart.jpg file exists for this series.
|
||||
|
||||
Args:
|
||||
base_directory: Base anime directory path. If provided, checks
|
||||
relative to base_directory/folder/fanart.jpg.
|
||||
|
||||
Returns:
|
||||
bool: True if fanart.jpg exists, False otherwise
|
||||
"""
|
||||
if not base_directory:
|
||||
return False
|
||||
|
||||
fanart_file = Path(base_directory) / self.folder / "fanart.jpg"
|
||||
return fanart_file.exists() and fanart_file.is_file()
|
||||
|
||||
@property
|
||||
def name_with_year(self) -> str:
|
||||
"""
|
||||
Get the series name with year appended if available.
|
||||
|
||||
Returns a name in the format "Name (Year)" if year is available,
|
||||
otherwise returns just the name. This should be used for creating
|
||||
filesystem folders to distinguish series with the same name.
|
||||
|
||||
Returns:
|
||||
str: Name with year in format "Name (Year)", or just name if no year
|
||||
|
||||
Example:
|
||||
>>> serie = Serie("dororo", "Dororo", ..., year=2025)
|
||||
>>> serie.name_with_year
|
||||
'Dororo (2025)'
|
||||
"""
|
||||
if self._year:
|
||||
return f"{self._name} ({self._year})"
|
||||
return self._name
|
||||
|
||||
@property
|
||||
def sanitized_folder(self) -> str:
|
||||
"""
|
||||
Get a filesystem-safe folder name derived from the display name with year.
|
||||
|
||||
This property returns a sanitized version of the series name with year
|
||||
(if available) suitable for use as a filesystem folder name. It removes/
|
||||
replaces characters that are invalid for filesystems while preserving
|
||||
Unicode characters.
|
||||
|
||||
Use this property when creating folders for the series on disk.
|
||||
The `folder` property stores the actual folder name used.
|
||||
|
||||
Returns:
|
||||
str: Filesystem-safe folder name based on display name with year
|
||||
|
||||
Example:
|
||||
>>> serie = Serie("attack-on-titan", "Attack on Titan: Final", ..., year=2025)
|
||||
>>> serie.sanitized_folder
|
||||
'Attack on Titan Final (2025)'
|
||||
"""
|
||||
# Use name_with_year if available, fall back to folder, then key
|
||||
name_to_sanitize = self.name_with_year or self._folder or self._key
|
||||
try:
|
||||
return sanitize_folder_name(name_to_sanitize)
|
||||
except ValueError:
|
||||
# Fallback to key if name cannot be sanitized
|
||||
return sanitize_folder_name(self._key)
|
||||
|
||||
def ensure_folder_with_year(self) -> str:
|
||||
"""Ensure folder name includes year if available.
|
||||
|
||||
If the serie has a year and the current folder name doesn't include it,
|
||||
updates the folder name to include the year in format "Name (Year)".
|
||||
|
||||
This method should be called before creating folders or NFO files to
|
||||
ensure consistent naming across the application.
|
||||
|
||||
Returns:
|
||||
str: The folder name (updated if needed)
|
||||
|
||||
Example:
|
||||
>>> serie = Serie("perfect-blue", "Perfect Blue", ..., folder="Perfect Blue", year=1997)
|
||||
>>> serie.ensure_folder_with_year()
|
||||
'Perfect Blue (1997)'
|
||||
>>> serie.folder # folder property is updated
|
||||
'Perfect Blue (1997)'
|
||||
"""
|
||||
if self._year:
|
||||
# Check if folder already has year format
|
||||
year_pattern = f"({self._year})"
|
||||
if year_pattern not in self._folder:
|
||||
# Update folder to include year
|
||||
self._folder = self.sanitized_folder
|
||||
logger.info(
|
||||
f"Updated folder name for '{self._key}' to include year: {self._folder}"
|
||||
)
|
||||
return self._folder
|
||||
|
||||
def to_dict(self):
|
||||
"""Convert Serie object to dictionary for JSON serialization."""
|
||||
return {
|
||||
"key": self.key,
|
||||
"name": self.name,
|
||||
"site": self.site,
|
||||
"folder": self.folder,
|
||||
"episodeDict": {
|
||||
str(k): list(v) for k, v in self.episodeDict.items()
|
||||
},
|
||||
"year": self.year,
|
||||
"nfo_path": self.nfo_path
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def from_dict(data: dict):
|
||||
"""Create a Serie object from dictionary."""
|
||||
# Convert keys to int
|
||||
episode_dict = {
|
||||
int(k): v for k, v in data["episodeDict"].items()
|
||||
}
|
||||
return Serie(
|
||||
data["key"],
|
||||
data["name"],
|
||||
data["site"],
|
||||
data["folder"],
|
||||
episode_dict,
|
||||
data.get("year"), # Optional year field for backward compatibility
|
||||
data.get("nfo_path") # Optional nfo_path field
|
||||
)
|
||||
|
||||
def save_to_file(self, filename: str):
|
||||
"""Save Serie object to JSON file.
|
||||
|
||||
.. deprecated::
|
||||
File-based storage is deprecated. Use database storage via
|
||||
`AnimeSeriesService.create()` instead. This method will be
|
||||
removed in v3.0.0.
|
||||
|
||||
Args:
|
||||
filename: Path to save the JSON file
|
||||
"""
|
||||
warnings.warn(
|
||||
"save_to_file() is deprecated and will be removed in v3.0.0. "
|
||||
"Use database storage via AnimeSeriesService.create() instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
with open(filename, "w", encoding="utf-8") as file:
|
||||
json.dump(self.to_dict(), file, indent=4)
|
||||
|
||||
@classmethod
|
||||
def load_from_file(cls, filename: str) -> "Serie":
|
||||
"""Load Serie object from JSON file.
|
||||
|
||||
.. deprecated::
|
||||
File-based storage is deprecated. Use database storage via
|
||||
`AnimeSeriesService.get_by_key()` instead. This method will be
|
||||
removed in v3.0.0.
|
||||
|
||||
Args:
|
||||
filename: Path to load the JSON file from
|
||||
|
||||
Returns:
|
||||
Serie: The loaded Serie object
|
||||
"""
|
||||
warnings.warn(
|
||||
"load_from_file() is deprecated and will be removed in v3.0.0. "
|
||||
"Use database storage via AnimeSeriesService instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
with open(filename, "r", encoding="utf-8") as file:
|
||||
data = json.load(file)
|
||||
return cls.from_dict(data)
|
||||
@@ -1,237 +0,0 @@
|
||||
"""NFO Service Factory Module.
|
||||
|
||||
This module provides a centralized factory for creating NFOService instances
|
||||
with consistent configuration and initialization logic.
|
||||
|
||||
The factory supports both direct instantiation and FastAPI dependency injection,
|
||||
while remaining testable through optional dependency overrides.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from src.config.settings import settings
|
||||
from src.core.services.nfo_service import NFOService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class NFOServiceFactory:
|
||||
"""Factory for creating NFOService instances with consistent configuration.
|
||||
|
||||
This factory centralizes NFO service initialization logic that was previously
|
||||
duplicated across multiple modules (SeriesApp, SeriesManagerService, API endpoints).
|
||||
|
||||
The factory follows these precedence rules for configuration:
|
||||
1. Explicit parameters (highest priority)
|
||||
2. Environment variables via settings
|
||||
3. config.json via ConfigService (fallback)
|
||||
4. Raise error if TMDB API key unavailable
|
||||
|
||||
Example:
|
||||
>>> factory = NFOServiceFactory()
|
||||
>>> nfo_service = factory.create()
|
||||
>>> # Or with custom settings:
|
||||
>>> nfo_service = factory.create(tmdb_api_key="custom_key")
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the NFO service factory."""
|
||||
self._config_service = None
|
||||
|
||||
def create(
|
||||
self,
|
||||
tmdb_api_key: Optional[str] = None,
|
||||
anime_directory: Optional[str] = None,
|
||||
image_size: Optional[str] = None,
|
||||
auto_create: Optional[bool] = None
|
||||
) -> NFOService:
|
||||
"""Create an NFOService instance with proper configuration.
|
||||
|
||||
This method implements the configuration precedence:
|
||||
1. Use explicit parameters if provided
|
||||
2. Fall back to settings (from ENV vars)
|
||||
3. Fall back to config.json (only if ENV not set)
|
||||
4. Raise ValueError if TMDB API key still unavailable
|
||||
|
||||
Args:
|
||||
tmdb_api_key: TMDB API key (optional, falls back to settings/config)
|
||||
anime_directory: Anime directory path (optional, defaults to settings)
|
||||
image_size: Image size for downloads (optional, defaults to settings)
|
||||
auto_create: Whether to auto-create NFO files (optional, defaults to settings)
|
||||
|
||||
Returns:
|
||||
NFOService: Configured NFO service instance
|
||||
|
||||
Raises:
|
||||
ValueError: If TMDB API key cannot be determined from any source
|
||||
|
||||
Example:
|
||||
>>> factory = NFOServiceFactory()
|
||||
>>> # Use all defaults from settings
|
||||
>>> service = factory.create()
|
||||
>>> # Override specific settings
|
||||
>>> service = factory.create(auto_create=False)
|
||||
"""
|
||||
# Step 1: Determine TMDB API key with fallback logic
|
||||
api_key = tmdb_api_key or settings.tmdb_api_key
|
||||
|
||||
# Step 2: If no API key in settings, try config.json as fallback
|
||||
if not api_key:
|
||||
api_key = self._get_api_key_from_config()
|
||||
|
||||
# Step 3: Validate API key is available
|
||||
if not api_key:
|
||||
raise ValueError(
|
||||
"TMDB API key not configured. Set TMDB_API_KEY environment "
|
||||
"variable or configure in config.json (nfo.tmdb_api_key)."
|
||||
)
|
||||
|
||||
# Step 4: Use provided values or fall back to settings
|
||||
directory = anime_directory or settings.anime_directory
|
||||
size = image_size or settings.nfo_image_size
|
||||
auto = auto_create if auto_create is not None else settings.nfo_auto_create
|
||||
|
||||
# Step 5: Create and return the service
|
||||
logger.debug(
|
||||
"Creating NFOService: directory=%s, size=%s, auto_create=%s",
|
||||
directory, size, auto
|
||||
)
|
||||
|
||||
return NFOService(
|
||||
tmdb_api_key=api_key,
|
||||
anime_directory=directory,
|
||||
image_size=size,
|
||||
auto_create=auto
|
||||
)
|
||||
|
||||
def create_optional(
|
||||
self,
|
||||
tmdb_api_key: Optional[str] = None,
|
||||
anime_directory: Optional[str] = None,
|
||||
image_size: Optional[str] = None,
|
||||
auto_create: Optional[bool] = None
|
||||
) -> Optional[NFOService]:
|
||||
"""Create an NFOService instance, returning None if configuration unavailable.
|
||||
|
||||
This is a convenience method for cases where NFO service is optional.
|
||||
Unlike create(), this returns None instead of raising ValueError when
|
||||
the TMDB API key is not configured.
|
||||
|
||||
Args:
|
||||
tmdb_api_key: TMDB API key (optional)
|
||||
anime_directory: Anime directory path (optional)
|
||||
image_size: Image size for downloads (optional)
|
||||
auto_create: Whether to auto-create NFO files (optional)
|
||||
|
||||
Returns:
|
||||
Optional[NFOService]: Configured service or None if key unavailable
|
||||
|
||||
Example:
|
||||
>>> factory = NFOServiceFactory()
|
||||
>>> service = factory.create_optional()
|
||||
>>> if service:
|
||||
... service.create_tvshow_nfo(...)
|
||||
"""
|
||||
try:
|
||||
return self.create(
|
||||
tmdb_api_key=tmdb_api_key,
|
||||
anime_directory=anime_directory,
|
||||
image_size=image_size,
|
||||
auto_create=auto_create
|
||||
)
|
||||
except ValueError as e:
|
||||
logger.debug("NFO service not available: %s", e)
|
||||
return None
|
||||
|
||||
def _get_api_key_from_config(self) -> Optional[str]:
|
||||
"""Get TMDB API key from config.json as fallback.
|
||||
|
||||
This method is only called when the API key is not in settings
|
||||
(i.e., not set via environment variable). It provides backward
|
||||
compatibility with config.json configuration.
|
||||
|
||||
Returns:
|
||||
Optional[str]: API key from config.json, or None if unavailable
|
||||
"""
|
||||
try:
|
||||
# Lazy import to avoid circular dependencies
|
||||
from src.server.services.config_service import get_config_service
|
||||
|
||||
if self._config_service is None:
|
||||
self._config_service = get_config_service()
|
||||
|
||||
config = self._config_service.load_config()
|
||||
|
||||
if config.nfo and config.nfo.tmdb_api_key:
|
||||
logger.debug("Using TMDB API key from config.json")
|
||||
return config.nfo.tmdb_api_key
|
||||
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
logger.debug("Could not load API key from config.json: %s", e)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# Global factory instance for convenience
|
||||
_factory_instance: Optional[NFOServiceFactory] = None
|
||||
|
||||
|
||||
def get_nfo_factory() -> NFOServiceFactory:
|
||||
"""Get the global NFO service factory instance.
|
||||
|
||||
This function provides a singleton factory instance for the application.
|
||||
The singleton pattern here is for the factory itself (which is stateless),
|
||||
not for the NFO service instances it creates.
|
||||
|
||||
Returns:
|
||||
NFOServiceFactory: The global factory instance
|
||||
|
||||
Example:
|
||||
>>> factory = get_nfo_factory()
|
||||
>>> service = factory.create()
|
||||
"""
|
||||
global _factory_instance
|
||||
|
||||
if _factory_instance is None:
|
||||
_factory_instance = NFOServiceFactory()
|
||||
|
||||
return _factory_instance
|
||||
|
||||
|
||||
def create_nfo_service(
|
||||
tmdb_api_key: Optional[str] = None,
|
||||
anime_directory: Optional[str] = None,
|
||||
image_size: Optional[str] = None,
|
||||
auto_create: Optional[bool] = None
|
||||
) -> NFOService:
|
||||
"""Convenience function to create an NFOService instance.
|
||||
|
||||
This is a shorthand for get_nfo_factory().create() that can be used
|
||||
when you need a quick NFO service instance without interacting with
|
||||
the factory directly.
|
||||
|
||||
Args:
|
||||
tmdb_api_key: TMDB API key (optional)
|
||||
anime_directory: Anime directory path (optional)
|
||||
image_size: Image size for downloads (optional)
|
||||
auto_create: Whether to auto-create NFO files (optional)
|
||||
|
||||
Returns:
|
||||
NFOService: Configured NFO service instance
|
||||
|
||||
Raises:
|
||||
ValueError: If TMDB API key cannot be determined
|
||||
|
||||
Example:
|
||||
>>> service = create_nfo_service()
|
||||
>>> # Or with custom settings:
|
||||
>>> service = create_nfo_service(auto_create=False)
|
||||
"""
|
||||
factory = get_nfo_factory()
|
||||
return factory.create(
|
||||
tmdb_api_key=tmdb_api_key,
|
||||
anime_directory=anime_directory,
|
||||
image_size=image_size,
|
||||
auto_create=auto_create
|
||||
)
|
||||
@@ -1,180 +0,0 @@
|
||||
"""NFO repair service for detecting and fixing incomplete tvshow.nfo files.
|
||||
|
||||
This module provides utilities to check whether an existing ``tvshow.nfo``
|
||||
contains all required tags and to trigger a repair (re-fetch from TMDB) when
|
||||
needed.
|
||||
|
||||
Example:
|
||||
>>> service = NfoRepairService(nfo_service)
|
||||
>>> repaired = await service.repair_series(Path("/anime/Attack on Titan"), "Attack on Titan")
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
from lxml import etree
|
||||
|
||||
from src.core.services.nfo_service import NFOService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# XPath relative to <tvshow> root → human-readable label
|
||||
REQUIRED_TAGS: Dict[str, str] = {
|
||||
"./title": "title",
|
||||
"./originaltitle": "originaltitle",
|
||||
"./year": "year",
|
||||
"./plot": "plot",
|
||||
"./runtime": "runtime",
|
||||
"./premiered": "premiered",
|
||||
"./status": "status",
|
||||
"./imdbid": "imdbid",
|
||||
"./genre": "genre",
|
||||
"./studio": "studio",
|
||||
"./country": "country",
|
||||
"./actor/name": "actor/name",
|
||||
"./watched": "watched",
|
||||
}
|
||||
|
||||
|
||||
def parse_nfo_tags(nfo_path: Path) -> Dict[str, List[str]]:
|
||||
"""Parse an existing tvshow.nfo and return present tag values.
|
||||
|
||||
Evaluates every XPath in :data:`REQUIRED_TAGS` against the document root
|
||||
and collects all non-empty text values.
|
||||
|
||||
Args:
|
||||
nfo_path: Absolute path to the ``tvshow.nfo`` file.
|
||||
|
||||
Returns:
|
||||
Mapping of XPath expression → list of non-empty text strings found in
|
||||
the document. Returns an empty dict on any error (missing file,
|
||||
invalid XML, permission error).
|
||||
|
||||
Example:
|
||||
>>> tags = parse_nfo_tags(Path("/anime/Attack on Titan/tvshow.nfo"))
|
||||
>>> tags.get("./title")
|
||||
['Attack on Titan']
|
||||
"""
|
||||
if not nfo_path.exists():
|
||||
logger.debug("NFO file not found: %s", nfo_path)
|
||||
return {}
|
||||
|
||||
try:
|
||||
tree = etree.parse(str(nfo_path))
|
||||
root = tree.getroot()
|
||||
|
||||
result: Dict[str, List[str]] = {}
|
||||
for xpath in REQUIRED_TAGS:
|
||||
elements = root.findall(xpath)
|
||||
result[xpath] = [e.text for e in elements if e.text]
|
||||
|
||||
return result
|
||||
|
||||
except etree.XMLSyntaxError as exc:
|
||||
logger.warning("Malformed XML in %s: %s", nfo_path, exc)
|
||||
return {}
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
logger.warning("Unexpected error parsing %s: %s", nfo_path, exc)
|
||||
return {}
|
||||
|
||||
|
||||
def find_missing_tags(nfo_path: Path) -> List[str]:
|
||||
"""Return tags that are absent or empty in the NFO.
|
||||
|
||||
Args:
|
||||
nfo_path: Absolute path to the ``tvshow.nfo`` file.
|
||||
|
||||
Returns:
|
||||
List of human-readable tag labels (values from :data:`REQUIRED_TAGS`)
|
||||
whose XPath matched no elements or only elements with empty text.
|
||||
An empty list means the NFO is complete.
|
||||
|
||||
Example:
|
||||
>>> missing = find_missing_tags(Path("/anime/series/tvshow.nfo"))
|
||||
>>> if missing:
|
||||
... print("Missing:", missing)
|
||||
"""
|
||||
parsed = parse_nfo_tags(nfo_path)
|
||||
missing: List[str] = []
|
||||
for xpath, label in REQUIRED_TAGS.items():
|
||||
if not parsed.get(xpath):
|
||||
missing.append(label)
|
||||
return missing
|
||||
|
||||
|
||||
def nfo_needs_repair(nfo_path: Path) -> bool:
|
||||
"""Return ``True`` if the NFO is missing any required tag.
|
||||
|
||||
Args:
|
||||
nfo_path: Absolute path to the ``tvshow.nfo`` file.
|
||||
|
||||
Returns:
|
||||
True if :func:`find_missing_tags` returns a non-empty list.
|
||||
|
||||
Example:
|
||||
>>> if nfo_needs_repair(Path("/anime/series/tvshow.nfo")):
|
||||
... await service.repair_series(series_path, series_name)
|
||||
"""
|
||||
return bool(find_missing_tags(nfo_path))
|
||||
|
||||
|
||||
class NfoRepairService:
|
||||
"""Service that detects and repairs incomplete tvshow.nfo files.
|
||||
|
||||
Wraps the module-level helpers with structured logging and delegates
|
||||
the actual TMDB re-fetch to an injected :class:`NFOService` instance.
|
||||
|
||||
Attributes:
|
||||
_nfo_service: The underlying NFOService used to update NFOs.
|
||||
"""
|
||||
|
||||
def __init__(self, nfo_service: NFOService) -> None:
|
||||
"""Initialise the repair service.
|
||||
|
||||
Args:
|
||||
nfo_service: Configured :class:`NFOService` instance.
|
||||
"""
|
||||
self._nfo_service = nfo_service
|
||||
|
||||
async def repair_series(self, series_path: Path, series_name: str) -> bool:
|
||||
"""Repair an NFO file if required tags are missing.
|
||||
|
||||
Checks ``{series_path}/tvshow.nfo`` for completeness. If tags are
|
||||
missing, logs them and calls
|
||||
``NFOService.update_tvshow_nfo(series_name)`` to re-fetch metadata
|
||||
from TMDB.
|
||||
|
||||
Args:
|
||||
series_path: Absolute path to the series folder.
|
||||
series_name: Series folder name used as the identifier for
|
||||
:meth:`NFOService.update_tvshow_nfo`.
|
||||
|
||||
Returns:
|
||||
``True`` if a repair was triggered, ``False`` if the NFO was
|
||||
already complete (or did not exist).
|
||||
"""
|
||||
nfo_path = series_path / "tvshow.nfo"
|
||||
missing = find_missing_tags(nfo_path)
|
||||
|
||||
if not missing:
|
||||
logger.info(
|
||||
"NFO repair skipped — complete: %s",
|
||||
series_name,
|
||||
)
|
||||
return False
|
||||
|
||||
logger.info(
|
||||
"NFO repair triggered for %s — missing tags: %s",
|
||||
series_name,
|
||||
", ".join(missing),
|
||||
)
|
||||
|
||||
await self._nfo_service.update_tvshow_nfo(
|
||||
series_name,
|
||||
download_media=False,
|
||||
)
|
||||
|
||||
logger.info("NFO repair completed: %s", series_name)
|
||||
return True
|
||||
@@ -1,587 +0,0 @@
|
||||
"""NFO service for creating and managing tvshow.nfo files.
|
||||
|
||||
This service orchestrates TMDB API calls, XML generation, and media downloads
|
||||
to create complete NFO metadata for TV series.
|
||||
|
||||
Example:
|
||||
>>> nfo_service = NFOService(tmdb_api_key="key", anime_directory="/anime")
|
||||
>>> await nfo_service.create_tvshow_nfo("Attack on Titan", "/anime/aot", 2013)
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from lxml import etree
|
||||
|
||||
from src.core.services.tmdb_client import TMDBAPIError, TMDBClient
|
||||
from src.core.utils.image_downloader import ImageDownloader
|
||||
from src.core.utils.nfo_generator import generate_tvshow_nfo
|
||||
from src.core.utils.nfo_mapper import tmdb_to_nfo_model
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class NFOService:
|
||||
"""Service for creating and managing tvshow.nfo files.
|
||||
|
||||
Attributes:
|
||||
tmdb_client: TMDB API client
|
||||
image_downloader: Image downloader utility
|
||||
anime_directory: Base directory for anime series
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tmdb_api_key: str,
|
||||
anime_directory: str,
|
||||
image_size: str = "original",
|
||||
auto_create: bool = True
|
||||
):
|
||||
"""Initialize NFO service.
|
||||
|
||||
Args:
|
||||
tmdb_api_key: TMDB API key
|
||||
anime_directory: Base anime directory path
|
||||
image_size: Image size to download (original, w500, etc.)
|
||||
auto_create: Whether to auto-create NFOs
|
||||
"""
|
||||
self.tmdb_client = TMDBClient(api_key=tmdb_api_key)
|
||||
self.image_downloader = ImageDownloader()
|
||||
self.anime_directory = Path(anime_directory)
|
||||
self.image_size = image_size
|
||||
self.auto_create = auto_create
|
||||
|
||||
def has_nfo(self, serie_folder: str) -> bool:
|
||||
"""Check if tvshow.nfo exists for a series.
|
||||
|
||||
Args:
|
||||
serie_folder: Series folder name
|
||||
|
||||
Returns:
|
||||
True if NFO file exists
|
||||
"""
|
||||
nfo_path = self.anime_directory / serie_folder / "tvshow.nfo"
|
||||
return nfo_path.exists()
|
||||
|
||||
@staticmethod
|
||||
def _extract_year_from_name(serie_name: str) -> Tuple[str, Optional[int]]:
|
||||
"""Extract year from series name if present in format 'Name (YYYY)'.
|
||||
|
||||
Args:
|
||||
serie_name: Series name, possibly with year in parentheses
|
||||
|
||||
Returns:
|
||||
Tuple of (clean_name, year)
|
||||
- clean_name: Series name without year
|
||||
- year: Extracted year or None
|
||||
|
||||
Examples:
|
||||
>>> _extract_year_from_name("Attack on Titan (2013)")
|
||||
("Attack on Titan", 2013)
|
||||
>>> _extract_year_from_name("Attack on Titan")
|
||||
("Attack on Titan", None)
|
||||
"""
|
||||
# Match year in parentheses at the end: (YYYY)
|
||||
match = re.search(r'\((\d{4})\)\s*$', serie_name)
|
||||
if match:
|
||||
year = int(match.group(1))
|
||||
clean_name = serie_name[:match.start()].strip()
|
||||
return clean_name, year
|
||||
return serie_name, None
|
||||
|
||||
async def check_nfo_exists(self, serie_folder: str) -> bool:
|
||||
"""Check if tvshow.nfo exists for a series.
|
||||
|
||||
Args:
|
||||
serie_folder: Series folder name
|
||||
|
||||
Returns:
|
||||
True if tvshow.nfo exists
|
||||
"""
|
||||
nfo_path = self.anime_directory / serie_folder / "tvshow.nfo"
|
||||
return nfo_path.exists()
|
||||
|
||||
async def create_tvshow_nfo(
|
||||
self,
|
||||
serie_name: str,
|
||||
serie_folder: str,
|
||||
year: Optional[int] = None,
|
||||
download_poster: bool = True,
|
||||
download_logo: bool = True,
|
||||
download_fanart: bool = True
|
||||
) -> Path:
|
||||
"""Create tvshow.nfo by scraping TMDB.
|
||||
|
||||
Args:
|
||||
serie_name: Name of the series to search (may include year in parentheses)
|
||||
serie_folder: Series folder name
|
||||
year: Release year (helps narrow search). If None and name contains year,
|
||||
year will be auto-extracted
|
||||
download_poster: Whether to download poster.jpg
|
||||
download_logo: Whether to download logo.png
|
||||
download_fanart: Whether to download fanart.jpg
|
||||
|
||||
Returns:
|
||||
Path to created NFO file
|
||||
|
||||
Raises:
|
||||
TMDBAPIError: If TMDB API fails
|
||||
FileNotFoundError: If series folder doesn't exist
|
||||
"""
|
||||
# Extract year from name if not provided
|
||||
clean_name, extracted_year = self._extract_year_from_name(serie_name)
|
||||
if year is None and extracted_year is not None:
|
||||
year = extracted_year
|
||||
logger.info("Extracted year %s from series name", year)
|
||||
|
||||
# Use clean name for search
|
||||
search_name = clean_name
|
||||
|
||||
logger.info("Creating NFO for %s (year: %s)", search_name, year)
|
||||
|
||||
folder_path = self.anime_directory / serie_folder
|
||||
if not folder_path.exists():
|
||||
logger.info("Creating series folder: %s", folder_path)
|
||||
folder_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
await self.tmdb_client._ensure_session()
|
||||
|
||||
# Search for TV show with clean name (without year)
|
||||
logger.debug("Searching TMDB for: %s", search_name)
|
||||
search_results = await self.tmdb_client.search_tv_show(search_name)
|
||||
|
||||
if not search_results.get("results"):
|
||||
raise TMDBAPIError(f"No results found for: {search_name}")
|
||||
|
||||
# Find best match (consider year if provided)
|
||||
tv_show = self._find_best_match(search_results["results"], search_name, year)
|
||||
tv_id = tv_show["id"]
|
||||
|
||||
logger.info("Found match: %s (ID: %s)", tv_show['name'], tv_id)
|
||||
|
||||
# Get detailed information with multi-language image support
|
||||
details = await self.tmdb_client.get_tv_show_details(
|
||||
tv_id,
|
||||
append_to_response="credits,external_ids,images"
|
||||
)
|
||||
|
||||
# Get content ratings for FSK
|
||||
content_ratings = await self.tmdb_client.get_tv_show_content_ratings(tv_id)
|
||||
|
||||
# Enrich with fallback languages for empty overview/tagline
|
||||
# Pass search result overview as last resort fallback
|
||||
search_overview = tv_show.get("overview") or None
|
||||
if not search_overview:
|
||||
try:
|
||||
logger.debug(
|
||||
"No overview in German search result, trying en-US search fallback for: %s",
|
||||
search_name,
|
||||
)
|
||||
en_search_results = await self.tmdb_client.search_tv_show(
|
||||
search_name,
|
||||
language="en-US",
|
||||
)
|
||||
if en_search_results.get("results"):
|
||||
en_match = self._find_best_match(
|
||||
en_search_results["results"], search_name, year
|
||||
)
|
||||
search_overview = en_match.get("overview") or None
|
||||
if search_overview:
|
||||
logger.info(
|
||||
"Using en-US search overview fallback for %s",
|
||||
search_name,
|
||||
)
|
||||
except (TMDBAPIError, Exception) as exc:
|
||||
logger.warning(
|
||||
"Failed en-US search fallback for overview: %s",
|
||||
exc,
|
||||
)
|
||||
|
||||
details = await self._enrich_details_with_fallback(
|
||||
details, search_overview=search_overview
|
||||
)
|
||||
|
||||
# Convert TMDB data to TVShowNFO model
|
||||
nfo_model = tmdb_to_nfo_model(
|
||||
details,
|
||||
content_ratings,
|
||||
self.tmdb_client.get_image_url,
|
||||
self.image_size,
|
||||
)
|
||||
|
||||
# Generate XML
|
||||
nfo_xml = generate_tvshow_nfo(nfo_model)
|
||||
|
||||
# Save NFO file
|
||||
nfo_path = folder_path / "tvshow.nfo"
|
||||
nfo_path.write_text(nfo_xml, encoding="utf-8")
|
||||
logger.info("Created NFO: %s", nfo_path)
|
||||
|
||||
# Download media files
|
||||
await self._download_media_files(
|
||||
details,
|
||||
folder_path,
|
||||
download_poster=download_poster,
|
||||
download_logo=download_logo,
|
||||
download_fanart=download_fanart
|
||||
)
|
||||
|
||||
return nfo_path
|
||||
finally:
|
||||
await self.tmdb_client.close()
|
||||
|
||||
async def update_tvshow_nfo(
|
||||
self,
|
||||
serie_folder: str,
|
||||
download_media: bool = True
|
||||
) -> Path:
|
||||
"""Update existing tvshow.nfo with fresh data from TMDB.
|
||||
|
||||
Args:
|
||||
serie_folder: Series folder name
|
||||
download_media: Whether to re-download media files
|
||||
|
||||
Returns:
|
||||
Path to updated NFO file
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If NFO file doesn't exist
|
||||
TMDBAPIError: If TMDB API fails or no TMDB ID found in NFO
|
||||
"""
|
||||
folder_path = self.anime_directory / serie_folder
|
||||
nfo_path = folder_path / "tvshow.nfo"
|
||||
|
||||
if not nfo_path.exists():
|
||||
raise FileNotFoundError(f"NFO file not found: {nfo_path}")
|
||||
|
||||
logger.info("Updating NFO for %s", serie_folder)
|
||||
|
||||
# Parse existing NFO to extract TMDB ID
|
||||
try:
|
||||
tree = etree.parse(str(nfo_path))
|
||||
root = tree.getroot()
|
||||
|
||||
# Try to find TMDB ID from uniqueid elements
|
||||
tmdb_id = None
|
||||
for uniqueid in root.findall(".//uniqueid"):
|
||||
if uniqueid.get("type") == "tmdb":
|
||||
tmdb_id = int(uniqueid.text)
|
||||
break
|
||||
|
||||
# Fallback: check for tmdbid element
|
||||
if tmdb_id is None:
|
||||
tmdbid_elem = root.find(".//tmdbid")
|
||||
if tmdbid_elem is not None and tmdbid_elem.text:
|
||||
tmdb_id = int(tmdbid_elem.text)
|
||||
|
||||
if tmdb_id is None:
|
||||
raise TMDBAPIError(
|
||||
f"No TMDB ID found in existing NFO. "
|
||||
f"Delete the NFO and create a new one instead."
|
||||
)
|
||||
|
||||
logger.debug("Found TMDB ID: %s", tmdb_id)
|
||||
|
||||
except etree.XMLSyntaxError as e:
|
||||
raise TMDBAPIError(f"Invalid XML in NFO file: {e}")
|
||||
except ValueError as e:
|
||||
raise TMDBAPIError(f"Invalid TMDB ID format in NFO: {e}")
|
||||
|
||||
try:
|
||||
await self.tmdb_client._ensure_session()
|
||||
logger.debug("Fetching fresh data for TMDB ID: %s", tmdb_id)
|
||||
details = await self.tmdb_client.get_tv_show_details(
|
||||
tmdb_id,
|
||||
append_to_response="credits,external_ids,images"
|
||||
)
|
||||
|
||||
# Get content ratings for FSK
|
||||
content_ratings = await self.tmdb_client.get_tv_show_content_ratings(tmdb_id)
|
||||
|
||||
# Enrich with fallback languages for empty overview/tagline
|
||||
details = await self._enrich_details_with_fallback(details)
|
||||
# Convert TMDB data to TVShowNFO model
|
||||
nfo_model = tmdb_to_nfo_model(
|
||||
details,
|
||||
content_ratings,
|
||||
self.tmdb_client.get_image_url,
|
||||
self.image_size,
|
||||
)
|
||||
|
||||
# Generate XML
|
||||
nfo_xml = generate_tvshow_nfo(nfo_model)
|
||||
|
||||
# Save updated NFO file
|
||||
nfo_path.write_text(nfo_xml, encoding="utf-8")
|
||||
logger.info("Updated NFO: %s", nfo_path)
|
||||
|
||||
# Re-download media files if requested
|
||||
if download_media:
|
||||
await self._download_media_files(
|
||||
details,
|
||||
folder_path,
|
||||
download_poster=True,
|
||||
download_logo=True,
|
||||
download_fanart=True
|
||||
)
|
||||
|
||||
return nfo_path
|
||||
finally:
|
||||
await self.tmdb_client.close()
|
||||
|
||||
def parse_nfo_ids(self, nfo_path: Path) -> Dict[str, Optional[int]]:
|
||||
"""Parse TMDB ID and TVDB ID from an existing NFO file.
|
||||
|
||||
Args:
|
||||
nfo_path: Path to tvshow.nfo file
|
||||
|
||||
Returns:
|
||||
Dictionary with 'tmdb_id' and 'tvdb_id' keys.
|
||||
Values are integers if found, None otherwise.
|
||||
|
||||
Example:
|
||||
>>> ids = nfo_service.parse_nfo_ids(Path("/anime/series/tvshow.nfo"))
|
||||
>>> print(ids)
|
||||
{'tmdb_id': 1429, 'tvdb_id': 79168}
|
||||
"""
|
||||
result = {"tmdb_id": None, "tvdb_id": None}
|
||||
|
||||
if not nfo_path.exists():
|
||||
logger.debug("NFO file not found: %s", nfo_path)
|
||||
return result
|
||||
|
||||
try:
|
||||
tree = etree.parse(str(nfo_path))
|
||||
root = tree.getroot()
|
||||
|
||||
# Try to find TMDB ID from uniqueid elements first
|
||||
for uniqueid in root.findall(".//uniqueid"):
|
||||
uid_type = uniqueid.get("type")
|
||||
uid_text = uniqueid.text
|
||||
|
||||
if uid_type == "tmdb" and uid_text:
|
||||
try:
|
||||
result["tmdb_id"] = int(uid_text)
|
||||
except ValueError:
|
||||
logger.warning(
|
||||
f"Invalid TMDB ID format in NFO: {uid_text}"
|
||||
)
|
||||
|
||||
elif uid_type == "tvdb" and uid_text:
|
||||
try:
|
||||
result["tvdb_id"] = int(uid_text)
|
||||
except ValueError:
|
||||
logger.warning(
|
||||
f"Invalid TVDB ID format in NFO: {uid_text}"
|
||||
)
|
||||
|
||||
# Fallback: check for dedicated tmdbid/tvdbid elements
|
||||
if result["tmdb_id"] is None:
|
||||
tmdbid_elem = root.find(".//tmdbid")
|
||||
if tmdbid_elem is not None and tmdbid_elem.text:
|
||||
try:
|
||||
result["tmdb_id"] = int(tmdbid_elem.text)
|
||||
except ValueError:
|
||||
logger.warning(
|
||||
f"Invalid TMDB ID format in tmdbid element: "
|
||||
f"{tmdbid_elem.text}"
|
||||
)
|
||||
|
||||
if result["tvdb_id"] is None:
|
||||
tvdbid_elem = root.find(".//tvdbid")
|
||||
if tvdbid_elem is not None and tvdbid_elem.text:
|
||||
try:
|
||||
result["tvdb_id"] = int(tvdbid_elem.text)
|
||||
except ValueError:
|
||||
logger.warning(
|
||||
f"Invalid TVDB ID format in tvdbid element: "
|
||||
f"{tvdbid_elem.text}"
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f"Parsed IDs from NFO: {nfo_path.name} - "
|
||||
f"TMDB: {result['tmdb_id']}, TVDB: {result['tvdb_id']}"
|
||||
)
|
||||
|
||||
except etree.XMLSyntaxError as e:
|
||||
logger.error("Invalid XML in NFO file %s: %s", nfo_path, e)
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
logger.error("Error parsing NFO file %s: %s", nfo_path, e)
|
||||
|
||||
return result
|
||||
|
||||
async def _enrich_details_with_fallback(
|
||||
self,
|
||||
details: Dict[str, Any],
|
||||
search_overview: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Enrich TMDB details with fallback languages for empty fields.
|
||||
|
||||
When requesting details in ``de-DE``, some anime have an empty
|
||||
``overview`` (and potentially other translatable fields). This
|
||||
method detects empty values and fills them from alternative
|
||||
languages (``en-US``, then ``ja-JP``) so that NFO files always
|
||||
contain a ``plot`` regardless of whether the German translation
|
||||
exists. As a last resort, the overview from the search result
|
||||
is used.
|
||||
|
||||
Args:
|
||||
details: TMDB TV show details (language ``de-DE``).
|
||||
search_overview: Overview text from the TMDB search result,
|
||||
used as a final fallback if all language-specific
|
||||
requests fail or return empty overviews.
|
||||
|
||||
Returns:
|
||||
The *same* dict, mutated in-place with fallback values
|
||||
where needed.
|
||||
"""
|
||||
overview = details.get("overview") or ""
|
||||
|
||||
if overview:
|
||||
# Overview already populated – nothing to do.
|
||||
return details
|
||||
|
||||
tmdb_id = details.get("id")
|
||||
fallback_languages = ["en-US", "ja-JP"]
|
||||
|
||||
for lang in fallback_languages:
|
||||
if details.get("overview"):
|
||||
break
|
||||
|
||||
logger.debug(
|
||||
"Trying %s fallback for TMDB ID %s",
|
||||
lang, tmdb_id,
|
||||
)
|
||||
|
||||
try:
|
||||
lang_details = await self.tmdb_client.get_tv_show_details(
|
||||
tmdb_id,
|
||||
language=lang,
|
||||
)
|
||||
|
||||
if not details.get("overview") and lang_details.get("overview"):
|
||||
details["overview"] = lang_details["overview"]
|
||||
logger.info(
|
||||
"Used %s overview fallback for TMDB ID %s",
|
||||
lang, tmdb_id,
|
||||
)
|
||||
|
||||
# Also fill tagline if missing
|
||||
if not details.get("tagline") and lang_details.get("tagline"):
|
||||
details["tagline"] = lang_details["tagline"]
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
logger.warning(
|
||||
"Failed to fetch %s fallback for TMDB ID %s: %s",
|
||||
lang, tmdb_id, exc,
|
||||
)
|
||||
|
||||
# Last resort: use search result overview
|
||||
if not details.get("overview") and search_overview:
|
||||
details["overview"] = search_overview
|
||||
logger.info(
|
||||
"Used search result overview fallback for TMDB ID %s",
|
||||
tmdb_id,
|
||||
)
|
||||
|
||||
return details
|
||||
|
||||
def _find_best_match(
|
||||
self,
|
||||
results: List[Dict[str, Any]],
|
||||
query: str,
|
||||
year: Optional[int] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""Find best matching TV show from search results.
|
||||
|
||||
Args:
|
||||
results: TMDB search results
|
||||
query: Original search query
|
||||
year: Expected release year
|
||||
|
||||
Returns:
|
||||
Best matching TV show data
|
||||
"""
|
||||
if not results:
|
||||
raise TMDBAPIError("No search results to match")
|
||||
|
||||
# If year is provided, try to find exact match
|
||||
if year:
|
||||
for result in results:
|
||||
first_air_date = result.get("first_air_date", "")
|
||||
if first_air_date.startswith(str(year)):
|
||||
logger.debug("Found year match: %s (%s)", result['name'], first_air_date)
|
||||
return result
|
||||
|
||||
# Return first result (usually best match)
|
||||
return results[0]
|
||||
|
||||
|
||||
|
||||
async def _download_media_files(
|
||||
self,
|
||||
tmdb_data: Dict[str, Any],
|
||||
folder_path: Path,
|
||||
download_poster: bool = True,
|
||||
download_logo: bool = True,
|
||||
download_fanart: bool = True
|
||||
) -> Dict[str, bool]:
|
||||
"""Download media files (poster, logo, fanart).
|
||||
|
||||
Args:
|
||||
tmdb_data: TMDB TV show details
|
||||
folder_path: Series folder path
|
||||
download_poster: Download poster.jpg
|
||||
download_logo: Download logo.png
|
||||
download_fanart: Download fanart.jpg
|
||||
|
||||
Returns:
|
||||
Dictionary with download status for each file
|
||||
"""
|
||||
poster_url = None
|
||||
logo_url = None
|
||||
fanart_url = None
|
||||
|
||||
# Get poster URL
|
||||
if download_poster and tmdb_data.get("poster_path"):
|
||||
poster_url = self.tmdb_client.get_image_url(
|
||||
tmdb_data["poster_path"],
|
||||
self.image_size
|
||||
)
|
||||
|
||||
# Get fanart URL
|
||||
if download_fanart and tmdb_data.get("backdrop_path"):
|
||||
fanart_url = self.tmdb_client.get_image_url(
|
||||
tmdb_data["backdrop_path"],
|
||||
"original" # Always use original for fanart
|
||||
)
|
||||
|
||||
# Get logo URL
|
||||
if download_logo:
|
||||
images_data = tmdb_data.get("images", {})
|
||||
logos = images_data.get("logos", [])
|
||||
if logos:
|
||||
logo_url = self.tmdb_client.get_image_url(
|
||||
logos[0]["file_path"],
|
||||
"original" # Logos should be original size
|
||||
)
|
||||
|
||||
# Download all media concurrently
|
||||
results = await self.image_downloader.download_all_media(
|
||||
folder_path,
|
||||
poster_url=poster_url,
|
||||
logo_url=logo_url,
|
||||
fanart_url=fanart_url,
|
||||
skip_existing=True
|
||||
)
|
||||
|
||||
logger.info("Media download results: %s", results)
|
||||
return results
|
||||
|
||||
|
||||
|
||||
async def close(self):
|
||||
"""Clean up resources."""
|
||||
await self.tmdb_client.close()
|
||||
@@ -1,279 +0,0 @@
|
||||
"""Service for managing series with NFO metadata support.
|
||||
|
||||
This service layer component orchestrates SerieList (core entity) with
|
||||
NFOService to provide automatic NFO creation and updates during series scans.
|
||||
|
||||
This follows clean architecture principles by keeping the core entities
|
||||
independent of external services like TMDB API.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from src.config.settings import settings
|
||||
from src.core.entities.SerieList import SerieList
|
||||
from src.core.services.nfo_service import NFOService
|
||||
from src.core.services.tmdb_client import TMDBAPIError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SeriesManagerService:
|
||||
"""Service for managing series with optional NFO metadata support.
|
||||
|
||||
This service wraps SerieList and adds NFO creation/update capabilities
|
||||
based on configuration settings. It maintains clean separation between
|
||||
core entities and external services.
|
||||
|
||||
Attributes:
|
||||
serie_list: SerieList instance for series management
|
||||
nfo_service: Optional NFOService for metadata management
|
||||
auto_create_nfo: Whether to auto-create NFO files
|
||||
update_on_scan: Whether to update existing NFO files
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
anime_directory: str,
|
||||
tmdb_api_key: Optional[str] = None,
|
||||
auto_create_nfo: bool = False,
|
||||
update_on_scan: bool = False,
|
||||
download_poster: bool = True,
|
||||
download_logo: bool = True,
|
||||
download_fanart: bool = True,
|
||||
image_size: str = "original"
|
||||
):
|
||||
"""Initialize series manager service.
|
||||
|
||||
Args:
|
||||
anime_directory: Base directory for anime series
|
||||
tmdb_api_key: TMDB API key (optional, required for NFO features)
|
||||
auto_create_nfo: Automatically create NFO files when scanning
|
||||
update_on_scan: Update existing NFO files when scanning
|
||||
download_poster: Download poster.jpg
|
||||
download_logo: Download logo.png
|
||||
download_fanart: Download fanart.jpg
|
||||
image_size: Image size to download
|
||||
"""
|
||||
self.anime_directory = anime_directory
|
||||
# Skip automatic folder scanning - we load from database instead
|
||||
self.serie_list = SerieList(anime_directory, skip_load=True)
|
||||
|
||||
# NFO configuration
|
||||
self.auto_create_nfo = auto_create_nfo
|
||||
self.update_on_scan = update_on_scan
|
||||
self.download_poster = download_poster
|
||||
self.download_logo = download_logo
|
||||
self.download_fanart = download_fanart
|
||||
|
||||
# Initialize NFO service if API key provided and NFO features enabled
|
||||
self.nfo_service: Optional[NFOService] = None
|
||||
if tmdb_api_key and (auto_create_nfo or update_on_scan):
|
||||
try:
|
||||
from src.core.services.nfo_factory import get_nfo_factory
|
||||
factory = get_nfo_factory()
|
||||
self.nfo_service = factory.create(
|
||||
tmdb_api_key=tmdb_api_key,
|
||||
anime_directory=anime_directory,
|
||||
image_size=image_size,
|
||||
auto_create=auto_create_nfo
|
||||
)
|
||||
logger.info("NFO service initialized (auto_create=%s, update=%s)",
|
||||
auto_create_nfo, update_on_scan)
|
||||
except (ValueError, Exception) as e: # pylint: disable=broad-except
|
||||
logger.warning(
|
||||
"Failed to initialize NFO service: %s", str(e)
|
||||
)
|
||||
self.nfo_service = None
|
||||
elif auto_create_nfo or update_on_scan:
|
||||
logger.warning(
|
||||
"NFO features requested but TMDB_API_KEY not provided. "
|
||||
"NFO creation/updates will be skipped."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls) -> "SeriesManagerService":
|
||||
"""Create SeriesManagerService from application settings.
|
||||
|
||||
Returns:
|
||||
Configured SeriesManagerService instance
|
||||
"""
|
||||
return cls(
|
||||
anime_directory=settings.anime_directory,
|
||||
tmdb_api_key=settings.tmdb_api_key,
|
||||
auto_create_nfo=settings.nfo_auto_create,
|
||||
update_on_scan=settings.nfo_update_on_scan,
|
||||
download_poster=settings.nfo_download_poster,
|
||||
download_logo=settings.nfo_download_logo,
|
||||
download_fanart=settings.nfo_download_fanart,
|
||||
image_size=settings.nfo_image_size
|
||||
)
|
||||
|
||||
async def process_nfo_for_series(
|
||||
self,
|
||||
serie_folder: str,
|
||||
serie_name: str,
|
||||
serie_key: str,
|
||||
year: Optional[int] = None
|
||||
):
|
||||
"""Process NFO file for a series (create or update).
|
||||
|
||||
Args:
|
||||
serie_folder: Series folder name
|
||||
serie_name: Series display name
|
||||
serie_key: Series unique identifier for database updates
|
||||
year: Release year (helps with TMDB matching)
|
||||
"""
|
||||
if not self.nfo_service:
|
||||
return
|
||||
|
||||
try:
|
||||
folder_path = Path(self.anime_directory) / serie_folder
|
||||
nfo_path = folder_path / "tvshow.nfo"
|
||||
nfo_exists = await self.nfo_service.check_nfo_exists(serie_folder)
|
||||
|
||||
# If NFO exists, parse IDs and update database
|
||||
if nfo_exists:
|
||||
logger.debug("Parsing IDs from existing NFO for '%s'", serie_name)
|
||||
ids = self.nfo_service.parse_nfo_ids(nfo_path)
|
||||
|
||||
if ids["tmdb_id"] or ids["tvdb_id"]:
|
||||
# Update database using service layer
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from src.server.database.connection import get_db_session
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
|
||||
async with get_db_session() as db:
|
||||
series = await AnimeSeriesService.get_by_key(db, serie_key)
|
||||
|
||||
if series:
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
# Prepare update fields
|
||||
update_fields = {
|
||||
"has_nfo": True,
|
||||
"nfo_updated_at": now,
|
||||
}
|
||||
|
||||
if series.nfo_created_at is None:
|
||||
update_fields["nfo_created_at"] = now
|
||||
|
||||
if ids["tmdb_id"] is not None:
|
||||
update_fields["tmdb_id"] = ids["tmdb_id"]
|
||||
logger.debug(
|
||||
f"Updated TMDB ID for '{serie_name}': "
|
||||
f"{ids['tmdb_id']}"
|
||||
)
|
||||
|
||||
if ids["tvdb_id"] is not None:
|
||||
update_fields["tvdb_id"] = ids["tvdb_id"]
|
||||
logger.debug(
|
||||
f"Updated TVDB ID for '{serie_name}': "
|
||||
f"{ids['tvdb_id']}"
|
||||
)
|
||||
|
||||
# Use service layer for update
|
||||
await AnimeSeriesService.update(db, series.id, **update_fields)
|
||||
await db.commit()
|
||||
|
||||
logger.info(
|
||||
f"Updated database with IDs from NFO for "
|
||||
f"'{serie_name}' - TMDB: {ids['tmdb_id']}, "
|
||||
f"TVDB: {ids['tvdb_id']}"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"Series not found in database for NFO ID "
|
||||
f"update: {serie_key}"
|
||||
)
|
||||
|
||||
# Create NFO file only if it doesn't exist and auto_create enabled
|
||||
if not nfo_exists and self.auto_create_nfo:
|
||||
logger.info(
|
||||
f"Creating NFO for '{serie_name}' ({serie_folder})"
|
||||
)
|
||||
await self.nfo_service.create_tvshow_nfo(
|
||||
serie_name=serie_name,
|
||||
serie_folder=serie_folder,
|
||||
year=year,
|
||||
download_poster=self.download_poster,
|
||||
download_logo=self.download_logo,
|
||||
download_fanart=self.download_fanart
|
||||
)
|
||||
logger.info("Successfully created NFO for '%s'", serie_name)
|
||||
elif nfo_exists:
|
||||
logger.debug(
|
||||
f"NFO exists for '{serie_name}', skipping download"
|
||||
)
|
||||
|
||||
except TMDBAPIError as e:
|
||||
logger.error("TMDB API error processing '%s': %s", serie_name, e)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Unexpected error processing NFO for '{serie_name}': {e}",
|
||||
exc_info=True
|
||||
)
|
||||
|
||||
async def scan_and_process_nfo(self):
|
||||
"""Scan all series and process NFO files based on configuration.
|
||||
|
||||
This method:
|
||||
1. Loads series from database (avoiding filesystem scan)
|
||||
2. For each series with existing NFO, reads TMDB/TVDB IDs
|
||||
and updates database
|
||||
3. For each series without NFO (if auto_create=True), creates one
|
||||
4. For each series with NFO (if update_on_scan=True), updates it
|
||||
5. Runs operations concurrently for better performance
|
||||
"""
|
||||
if not self.nfo_service:
|
||||
logger.info("NFO service not enabled, skipping NFO processing")
|
||||
return
|
||||
|
||||
# Import database dependencies
|
||||
from src.server.database.connection import get_db_session
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
|
||||
# Load series from database (not from filesystem)
|
||||
async with get_db_session() as db:
|
||||
anime_series_list = await AnimeSeriesService.get_all(
|
||||
db, with_episodes=False
|
||||
)
|
||||
|
||||
if not anime_series_list:
|
||||
logger.info("No series found in database to process")
|
||||
return
|
||||
|
||||
logger.info("Processing NFO for %s series...", len(anime_series_list))
|
||||
|
||||
# Create tasks for concurrent processing
|
||||
# Each task creates its own database session
|
||||
tasks = []
|
||||
for anime_series in anime_series_list:
|
||||
# Extract year if available
|
||||
year = getattr(anime_series, 'year', None)
|
||||
|
||||
task = self.process_nfo_for_series(
|
||||
serie_folder=anime_series.folder,
|
||||
serie_name=anime_series.name,
|
||||
serie_key=anime_series.key,
|
||||
year=year
|
||||
)
|
||||
tasks.append(task)
|
||||
|
||||
# Process in batches to avoid overwhelming TMDB API
|
||||
batch_size = 5
|
||||
for i in range(0, len(tasks), batch_size):
|
||||
batch = tasks[i:i + batch_size]
|
||||
await asyncio.gather(*batch, return_exceptions=True)
|
||||
|
||||
# Small delay between batches to respect rate limits
|
||||
if i + batch_size < len(tasks):
|
||||
await asyncio.sleep(2)
|
||||
|
||||
async def close(self):
|
||||
"""Clean up resources."""
|
||||
if self.nfo_service:
|
||||
await self.nfo_service.close()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -14,17 +14,15 @@ import asyncio
|
||||
import logging
|
||||
import os
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from events import Events
|
||||
|
||||
from src.config.settings import settings
|
||||
from src.core.entities.SerieList import SerieList
|
||||
from src.core.entities.series import Serie
|
||||
from src.core.providers.provider_factory import Loaders
|
||||
from src.core.SerieScanner import SerieScanner
|
||||
from src.core.services.nfo_service import NFOService
|
||||
from src.core.services.tmdb_client import TMDBAPIError
|
||||
from src.server.database.SerieList import SerieList
|
||||
from src.server.database.models import AnimeSeries
|
||||
from src.server.providers.provider_factory import Loaders
|
||||
from src.server.SerieScanner import SerieScanner
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -162,32 +160,18 @@ class SeriesApp:
|
||||
self.loaders = Loaders()
|
||||
self.loader = self.loaders.GetLoader(key="aniworld.to")
|
||||
self.serie_scanner = SerieScanner(
|
||||
directory_to_search, self.loader
|
||||
directory_to_search,
|
||||
self.loader,
|
||||
)
|
||||
# Skip automatic loading from data files - series will be loaded
|
||||
# from database by the service layer during application setup
|
||||
self.list = SerieList(self.directory_to_search, skip_load=True)
|
||||
# Series will be loaded from database by the service layer during application setup
|
||||
self.list = SerieList(self.directory_to_search)
|
||||
self.series_list: List[Any] = []
|
||||
# Initialize empty list - series loaded later via load_series_from_list()
|
||||
# No need to call _init_list_sync() anymore
|
||||
|
||||
# Initialize NFO service if a TMDB API key is configured
|
||||
self.nfo_service: Optional[NFOService] = None
|
||||
try:
|
||||
from src.core.services.nfo_factory import get_nfo_factory
|
||||
|
||||
factory = get_nfo_factory()
|
||||
self.nfo_service = factory.create()
|
||||
logger.info("NFO service initialized successfully")
|
||||
except ValueError:
|
||||
logger.info(
|
||||
"NFO service not available — TMDB API key not configured"
|
||||
)
|
||||
self.nfo_service = None
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
logger.warning("Failed to initialize NFO service: %s", str(e))
|
||||
self.nfo_service = None
|
||||
|
||||
# NFO service removed - metadata handling moved to server layer
|
||||
self.nfo_service = None
|
||||
|
||||
logger.info(
|
||||
"SeriesApp initialized for directory: %s",
|
||||
directory_to_search,
|
||||
@@ -349,101 +333,15 @@ class SeriesApp:
|
||||
)
|
||||
return False
|
||||
|
||||
# Check and create NFO files if needed
|
||||
if self.nfo_service and settings.nfo_auto_create:
|
||||
try:
|
||||
# Check if NFO exists
|
||||
nfo_exists = await self.nfo_service.check_nfo_exists(
|
||||
serie_folder
|
||||
)
|
||||
|
||||
if not nfo_exists:
|
||||
logger.info(
|
||||
"NFO not found for %s, creating metadata...",
|
||||
serie_folder
|
||||
)
|
||||
|
||||
# Fire NFO creation started event
|
||||
self._events.download_status(
|
||||
DownloadStatusEventArgs(
|
||||
serie_folder=serie_folder,
|
||||
key=key,
|
||||
season=season,
|
||||
episode=episode,
|
||||
status="nfo_creating",
|
||||
message="Creating NFO metadata...",
|
||||
item_id=item_id,
|
||||
)
|
||||
)
|
||||
|
||||
# Create NFO and download media files
|
||||
try:
|
||||
# Use folder name as series name
|
||||
await self.nfo_service.create_tvshow_nfo(
|
||||
serie_name=serie_folder,
|
||||
serie_folder=serie_folder,
|
||||
download_poster=settings.nfo_download_poster,
|
||||
download_logo=settings.nfo_download_logo,
|
||||
download_fanart=settings.nfo_download_fanart
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"NFO and media files created for %s",
|
||||
serie_folder
|
||||
)
|
||||
|
||||
# Fire NFO creation completed event
|
||||
self._events.download_status(
|
||||
DownloadStatusEventArgs(
|
||||
serie_folder=serie_folder,
|
||||
key=key,
|
||||
season=season,
|
||||
episode=episode,
|
||||
status="nfo_completed",
|
||||
message="NFO metadata created",
|
||||
item_id=item_id,
|
||||
)
|
||||
)
|
||||
|
||||
except TMDBAPIError as tmdb_error:
|
||||
logger.warning(
|
||||
"Failed to create NFO for %s: %s",
|
||||
serie_folder,
|
||||
str(tmdb_error)
|
||||
)
|
||||
# Fire failed event (but continue with download)
|
||||
self._events.download_status(
|
||||
DownloadStatusEventArgs(
|
||||
serie_folder=serie_folder,
|
||||
key=key,
|
||||
season=season,
|
||||
episode=episode,
|
||||
status="nfo_failed",
|
||||
message=(
|
||||
f"NFO creation failed: "
|
||||
f"{str(tmdb_error)}"
|
||||
),
|
||||
item_id=item_id,
|
||||
)
|
||||
)
|
||||
else:
|
||||
logger.debug("NFO already exists for %s", serie_folder)
|
||||
|
||||
except Exception as nfo_error: # pylint: disable=broad-except
|
||||
logger.error(
|
||||
"Error checking/creating NFO for %s: %s",
|
||||
serie_folder,
|
||||
str(nfo_error),
|
||||
exc_info=True
|
||||
)
|
||||
# Don't fail the download if NFO creation fails
|
||||
|
||||
try:
|
||||
def download_progress_handler(progress_info):
|
||||
"""Handle download progress events from loader."""
|
||||
logger.debug(
|
||||
"download_progress_handler called with: %s", progress_info
|
||||
)
|
||||
# Throttle progress logging to avoid spam
|
||||
status = progress_info.get("status", "")
|
||||
if status in ("downloading", "finished"):
|
||||
logger.debug(
|
||||
"download_progress_handler called with: %s", progress_info
|
||||
)
|
||||
|
||||
downloaded = progress_info.get('downloaded_bytes', 0)
|
||||
total_bytes = (
|
||||
@@ -755,7 +653,7 @@ class SeriesApp:
|
||||
"""
|
||||
await self._init_list()
|
||||
|
||||
def _get_serie_by_key(self, key: str) -> Optional[Serie]:
|
||||
def _get_serie_by_key(self, key: str) -> Optional[AnimeSeries]:
|
||||
"""
|
||||
Get a series by its unique provider key.
|
||||
|
||||
@@ -766,7 +664,7 @@ class SeriesApp:
|
||||
"attack-on-titan")
|
||||
|
||||
Returns:
|
||||
The Serie instance if found, None otherwise
|
||||
The AnimeSeries instance if found, None otherwise
|
||||
|
||||
Note:
|
||||
This method uses the SerieList.get_by_key() method which
|
||||
@@ -774,39 +672,40 @@ class SeriesApp:
|
||||
"""
|
||||
return self.list.get_by_key(key)
|
||||
|
||||
def get_all_series_from_data_files(self) -> List[Serie]:
|
||||
def get_all_series_from_data_files(self) -> List[AnimeSeries]:
|
||||
"""
|
||||
Get all series from data files in the anime directory.
|
||||
|
||||
Scans the directory_to_search for all 'data' files and loads
|
||||
the Serie metadata from each file. This method is synchronous
|
||||
the AnimeSeries metadata from each file. This method is synchronous
|
||||
and can be wrapped with asyncio.to_thread if needed for async
|
||||
contexts.
|
||||
|
||||
Returns:
|
||||
List of Serie objects found in data files. Returns an empty
|
||||
List of AnimeSeries objects found in data files. Returns an empty
|
||||
list if no data files are found or if the directory doesn't
|
||||
exist.
|
||||
|
||||
Example:
|
||||
series_app = SeriesApp("/path/to/anime")
|
||||
all_series = series_app.get_all_series_from_data_files()
|
||||
for serie in all_series:
|
||||
print(f"Found: {serie.name} (key={serie.key})")
|
||||
for anime in all_series:
|
||||
print(f"Found: {anime.name} (key={anime.key})")
|
||||
"""
|
||||
logger.info(
|
||||
"Scanning for data files in directory: %s",
|
||||
self.directory_to_search
|
||||
)
|
||||
|
||||
# Create a fresh SerieList instance for file-based loading
|
||||
# This ensures we get all series from data files without
|
||||
# interfering with the main instance's state
|
||||
all_series: List[AnimeSeries] = []
|
||||
|
||||
try:
|
||||
temp_list = SerieList(
|
||||
self.directory_to_search,
|
||||
skip_load=False # Allow automatic loading
|
||||
)
|
||||
if not os.path.isdir(self.directory_to_search):
|
||||
logger.warning(
|
||||
"Directory does not exist: %s",
|
||||
self.directory_to_search
|
||||
)
|
||||
return []
|
||||
except (OSError, ValueError) as e:
|
||||
logger.error(
|
||||
"Failed to scan directory for data files: %s",
|
||||
@@ -815,8 +714,53 @@ class SeriesApp:
|
||||
)
|
||||
return []
|
||||
|
||||
# Get all series from the temporary list
|
||||
all_series = temp_list.get_all()
|
||||
try:
|
||||
for folder_name in os.listdir(self.directory_to_search):
|
||||
folder_path = os.path.join(
|
||||
self.directory_to_search, folder_name
|
||||
)
|
||||
if not os.path.isdir(folder_path):
|
||||
continue
|
||||
|
||||
data_file = os.path.join(folder_path, "data")
|
||||
if not os.path.isfile(data_file):
|
||||
continue
|
||||
|
||||
series_data = _load_data_file(data_file)
|
||||
if series_data is None:
|
||||
continue
|
||||
|
||||
key = series_data.get("key")
|
||||
if not key:
|
||||
logger.warning(
|
||||
"Data file missing key, skipping: %s",
|
||||
data_file
|
||||
)
|
||||
continue
|
||||
|
||||
anime = AnimeSeries(
|
||||
key=key,
|
||||
name=series_data.get("name") or folder_name,
|
||||
site=series_data.get("site", "https://aniworld.to"),
|
||||
folder=series_data.get("folder", folder_name),
|
||||
year=series_data.get("year"),
|
||||
)
|
||||
|
||||
episode_dict = series_data.get("episodeDict", {})
|
||||
if episode_dict:
|
||||
anime._episode_dict_cache = {
|
||||
int(season): episodes
|
||||
for season, episodes in episode_dict.items()
|
||||
}
|
||||
|
||||
all_series.append(anime)
|
||||
except (OSError, ValueError) as e:
|
||||
logger.error(
|
||||
"Failed to scan directory for data files: %s",
|
||||
str(e),
|
||||
exc_info=True
|
||||
)
|
||||
return []
|
||||
|
||||
logger.info(
|
||||
"Found %d series from data files in %s",
|
||||
@@ -836,3 +780,38 @@ class SeriesApp:
|
||||
if hasattr(self, 'executor'):
|
||||
self.executor.shutdown(wait=True)
|
||||
logger.info("ThreadPoolExecutor shut down successfully")
|
||||
|
||||
|
||||
def _load_data_file(data_file_path: str) -> Optional[dict]:
|
||||
"""Load and parse a legacy 'data' file (JSON).
|
||||
|
||||
Args:
|
||||
data_file_path: Path to the data file
|
||||
|
||||
Returns:
|
||||
Parsed data dict or None if parsing fails
|
||||
"""
|
||||
import json
|
||||
|
||||
try:
|
||||
with open(data_file_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
if not isinstance(data, dict):
|
||||
logger.warning("Data file is not a dictionary: %s", data_file_path)
|
||||
return None
|
||||
|
||||
return data
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.warning(
|
||||
"Failed to parse legacy data file (JSON error): %s - %s",
|
||||
data_file_path, str(e)
|
||||
)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Failed to read legacy data file: %s - %s",
|
||||
data_file_path, str(e)
|
||||
)
|
||||
return None
|
||||
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
import re
|
||||
import warnings
|
||||
from typing import Any, List, Optional
|
||||
|
||||
@@ -6,7 +7,8 @@ from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.core.entities.series import Serie
|
||||
from src.config.settings import settings
|
||||
from src.server.database.models import AnimeSeries
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
from src.server.exceptions import (
|
||||
BadRequestError,
|
||||
@@ -14,16 +16,19 @@ from src.server.exceptions import (
|
||||
ServerError,
|
||||
ValidationError,
|
||||
)
|
||||
from src.server.models.anime import AnimeMetadataUpdate
|
||||
from src.server.services.anime_service import AnimeService, AnimeServiceError
|
||||
from src.server.services.background_loader_service import BackgroundLoaderService
|
||||
from src.server.utils.dependencies import (
|
||||
get_anime_service,
|
||||
get_background_loader_service,
|
||||
get_database_session,
|
||||
get_optional_database_session,
|
||||
get_series_app,
|
||||
require_auth,
|
||||
)
|
||||
from src.server.utils.filesystem import sanitize_folder_name
|
||||
from src.server.utils.key_utils import generate_key_from_folder, is_valid_key
|
||||
from src.server.utils.validators import validate_filter_value, validate_search_query
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -31,6 +36,31 @@ logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/anime", tags=["anime"])
|
||||
|
||||
|
||||
def _compute_folder_name(name: str, year: Optional[int]) -> str:
|
||||
"""Compute sanitized folder name from display name and year.
|
||||
|
||||
If year is provided, strips any existing year in (YYYY) format to avoid
|
||||
duplicates, then appends the new year. If year is None, preserves the
|
||||
original name (with any existing year).
|
||||
|
||||
Args:
|
||||
name: Display name of the series
|
||||
year: Release year from provider, or None
|
||||
|
||||
Returns:
|
||||
Sanitized folder name in format "Name (YYYY)" or just "Name"
|
||||
"""
|
||||
if year:
|
||||
# Strip any existing year in (YYYY) format before adding new year
|
||||
clean_name = re.sub(r'\s*\(\d{4}\)\s*$', '', name).strip()
|
||||
folder_name_with_year = f"{clean_name} ({year})"
|
||||
else:
|
||||
# No new year provided, preserve original name (with any existing year)
|
||||
folder_name_with_year = name
|
||||
|
||||
return sanitize_folder_name(folder_name_with_year)
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def get_anime_status(
|
||||
_auth: dict = Depends(require_auth),
|
||||
@@ -70,6 +100,37 @@ async def get_anime_status(
|
||||
) from exc
|
||||
|
||||
|
||||
class DuplicateFolderGroup(BaseModel):
|
||||
"""Placeholder - duplicates functionality removed."""
|
||||
key: str = Field(..., description="Series key (unique identifier)")
|
||||
folders: List[str] = Field(..., description="List of duplicate folder names")
|
||||
folder_count: int = Field(..., description="Number of duplicate folders")
|
||||
|
||||
|
||||
class DuplicateFoldersResponse(BaseModel):
|
||||
"""Placeholder - duplicates functionality removed."""
|
||||
total_groups: int = Field(..., description="Total number of duplicate groups")
|
||||
duplicate_groups: List[DuplicateFolderGroup] = Field(
|
||||
..., description="List of duplicate folder groups"
|
||||
)
|
||||
message: str = Field(..., description="Human-readable summary")
|
||||
|
||||
|
||||
@router.get("/duplicate-folders", response_model=DuplicateFoldersResponse)
|
||||
async def get_duplicate_folders(
|
||||
_auth: dict = Depends(require_auth),
|
||||
) -> DuplicateFoldersResponse:
|
||||
"""List all pre-existing duplicate folder groups.
|
||||
|
||||
Note: Duplicate folder scanning has been removed. Returns empty response.
|
||||
"""
|
||||
return DuplicateFoldersResponse(
|
||||
total_groups=0,
|
||||
duplicate_groups=[],
|
||||
message="Duplicate folder scanning has been removed.",
|
||||
)
|
||||
|
||||
|
||||
class AnimeSummary(BaseModel):
|
||||
"""Summary of an anime series with missing episodes.
|
||||
|
||||
@@ -728,14 +789,9 @@ async def add_series(
|
||||
except Exception as e:
|
||||
logger.warning("Could not fetch year for %s: %s", key, e)
|
||||
|
||||
# Create folder name with year if available
|
||||
if year:
|
||||
folder_name_with_year = f"{name} ({year})"
|
||||
else:
|
||||
folder_name_with_year = name
|
||||
|
||||
# Step B: Compute sanitized folder name with year (deduplicates if year already in name)
|
||||
try:
|
||||
folder = sanitize_folder_name(folder_name_with_year)
|
||||
folder = _compute_folder_name(name, year)
|
||||
except ValueError as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
@@ -744,7 +800,37 @@ async def add_series(
|
||||
|
||||
db_id = None
|
||||
|
||||
# Step C: Save to database if available
|
||||
# Step C: Create folder on disk if it doesn't exist, and rename if needed
|
||||
# Determine the anime directory path
|
||||
anime_dir = settings.anime_directory if hasattr(settings, 'anime_directory') else None
|
||||
current_folder_on_disk = None
|
||||
|
||||
if anime_dir:
|
||||
import os
|
||||
anime_path = os.path.join(anime_dir, folder)
|
||||
|
||||
# Check if an existing folder (without year) needs renaming
|
||||
# Look for folder that matches name without year
|
||||
if year:
|
||||
potential_old_name = sanitize_folder_name(name)
|
||||
potential_old_path = os.path.join(anime_dir, potential_old_name)
|
||||
if potential_old_path != anime_path and os.path.exists(potential_old_path):
|
||||
current_folder_on_disk = potential_old_name
|
||||
logger.info(
|
||||
"Found existing folder without year for %s: %s, renaming to %s",
|
||||
key,
|
||||
potential_old_name,
|
||||
folder
|
||||
)
|
||||
elif not os.path.exists(anime_path):
|
||||
# No existing folder to rename, create new one
|
||||
os.makedirs(anime_path, exist_ok=True)
|
||||
else:
|
||||
# No year, just ensure folder exists
|
||||
if not os.path.exists(anime_path):
|
||||
os.makedirs(anime_path, exist_ok=True)
|
||||
|
||||
# Step D: Save to database if available
|
||||
if db is not None:
|
||||
# Check if series already exists in database
|
||||
existing = await AnimeSeriesService.get_by_key(db, key)
|
||||
@@ -790,18 +876,18 @@ async def add_series(
|
||||
|
||||
# Step D: Add to SerieList (in-memory only, no folder creation)
|
||||
if series_app and hasattr(series_app, "list"):
|
||||
serie = Serie(
|
||||
from src.server.database.models import AnimeSeries
|
||||
anime = AnimeSeries(
|
||||
key=key,
|
||||
name=name,
|
||||
site="aniworld.to",
|
||||
folder=folder,
|
||||
episodeDict={},
|
||||
year=year
|
||||
)
|
||||
|
||||
|
||||
# Add to in-memory cache without creating folder on disk
|
||||
if hasattr(series_app.list, 'keyDict'):
|
||||
series_app.list.keyDict[key] = serie
|
||||
series_app.list.keyDict[key] = anime
|
||||
logger.info(
|
||||
"Added series to in-memory cache: %s (key=%s, folder=%s, year=%s)",
|
||||
name,
|
||||
@@ -810,7 +896,32 @@ async def add_series(
|
||||
year
|
||||
)
|
||||
|
||||
# Step E: Queue background loading task for episodes, NFO, and images
|
||||
# Step E: Rename existing folder if needed (e.g., folder existed without year)
|
||||
if current_folder_on_disk:
|
||||
try:
|
||||
renamed = await anime_service.rename_folder_if_needed(
|
||||
key=key,
|
||||
current_folder=current_folder_on_disk,
|
||||
target_folder=folder,
|
||||
db=db
|
||||
)
|
||||
if renamed:
|
||||
logger.info(
|
||||
"Successfully renamed folder for %s: %s -> %s",
|
||||
key,
|
||||
current_folder_on_disk,
|
||||
folder
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Failed to rename folder for %s: %s -> %s: %s",
|
||||
key,
|
||||
current_folder_on_disk,
|
||||
folder,
|
||||
e
|
||||
)
|
||||
|
||||
# Step F: Queue background loading task for episodes, NFO, and images
|
||||
try:
|
||||
await background_loader.add_series_loading_task(
|
||||
key=key,
|
||||
@@ -831,7 +942,7 @@ async def add_series(
|
||||
e
|
||||
)
|
||||
|
||||
# Step F: Scan missing episodes immediately if background loader is not running
|
||||
# Step G: Scan missing episodes immediately if background loader is not running
|
||||
# Uses existing SerieScanner and AnimeService sync to avoid duplicates
|
||||
try:
|
||||
loader_running = bool(
|
||||
@@ -1084,3 +1195,75 @@ async def get_anime(
|
||||
# Maximum allowed input size for security
|
||||
MAX_INPUT_LENGTH = 100000 # 100KB
|
||||
|
||||
|
||||
@router.put("/{anime_key}")
|
||||
async def update_anime_metadata(
|
||||
anime_key: str,
|
||||
body: AnimeMetadataUpdate,
|
||||
_auth: dict = Depends(require_auth),
|
||||
db: AsyncSession = Depends(get_database_session),
|
||||
) -> dict:
|
||||
"""Update anime metadata (key, tmdb_id, tvdb_id).
|
||||
|
||||
Args:
|
||||
anime_key: Current series key to update
|
||||
body: Fields to update (all optional)
|
||||
_auth: Authentication dependency
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
Updated series metadata
|
||||
|
||||
Raises:
|
||||
HTTPException 404: Series not found
|
||||
HTTPException 409: Key conflict (new key already exists)
|
||||
HTTPException 422: Validation error
|
||||
"""
|
||||
series = await AnimeSeriesService.get_by_key(db, anime_key)
|
||||
if not series:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Series with key '{anime_key}' not found",
|
||||
)
|
||||
|
||||
updates = {}
|
||||
|
||||
if body.key is not None and body.key != anime_key:
|
||||
existing = await AnimeSeriesService.get_by_key(db, body.key)
|
||||
if existing:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"A series with key '{body.key}' already exists",
|
||||
)
|
||||
updates["key"] = body.key
|
||||
|
||||
if body.tmdb_id is not None:
|
||||
updates["tmdb_id"] = body.tmdb_id
|
||||
|
||||
if body.tvdb_id is not None:
|
||||
updates["tvdb_id"] = body.tvdb_id
|
||||
|
||||
if not updates:
|
||||
return {
|
||||
"key": series.key,
|
||||
"tmdb_id": series.tmdb_id,
|
||||
"tvdb_id": series.tvdb_id,
|
||||
"message": "No changes",
|
||||
}
|
||||
|
||||
updated = await AnimeSeriesService.update(db, series.id, **updates)
|
||||
await db.commit()
|
||||
|
||||
logger.info(
|
||||
"Updated metadata for '%s': %s",
|
||||
anime_key,
|
||||
updates,
|
||||
)
|
||||
|
||||
return {
|
||||
"key": updated.key,
|
||||
"tmdb_id": updated.tmdb_id,
|
||||
"tvdb_id": updated.tvdb_id,
|
||||
"message": "Metadata updated successfully",
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Authentication API endpoints for Aniworld."""
|
||||
from typing import Optional
|
||||
|
||||
import structlog
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import status as http_status
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||
@@ -16,6 +17,8 @@ from src.server.models.config import AppConfig
|
||||
from src.server.services.auth_service import AuthError, LockedOutError, auth_service
|
||||
from src.server.services.config_service import get_config_service
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
# NOTE: import dependencies (optional_auth, security) lazily inside handlers
|
||||
# to avoid importing heavyweight modules (e.g. sqlalchemy) at import time.
|
||||
|
||||
@@ -161,6 +164,22 @@ async def setup_auth(req: SetupRequest):
|
||||
# Perform NFO scan if configured
|
||||
await perform_nfo_scan_if_needed(progress_service)
|
||||
|
||||
# Start scheduler if anime_directory is now set
|
||||
try:
|
||||
from src.server.services.scheduler.scheduler_service import (
|
||||
get_scheduler_service,
|
||||
)
|
||||
|
||||
scheduler_svc = get_scheduler_service()
|
||||
logger.info("Starting scheduler after initialization")
|
||||
await scheduler_svc.ensure_started()
|
||||
logger.info("Scheduler started successfully during setup")
|
||||
except Exception as sched_exc:
|
||||
logger.warning(
|
||||
"Failed to start scheduler during setup: %s", sched_exc
|
||||
)
|
||||
# Continue — scheduler failure should not break initialization
|
||||
|
||||
# Send completion event
|
||||
from src.server.services.progress_service import ProgressType
|
||||
await progress_service.start_progress(
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from src.server.models.config import AppConfig, ConfigUpdate, ValidationResult
|
||||
from src.server.services.config_service import (
|
||||
ConfigBackupError,
|
||||
@@ -28,16 +31,53 @@ def get_config(auth: Optional[dict] = Depends(require_auth)) -> AppConfig:
|
||||
|
||||
|
||||
@router.put("", response_model=AppConfig)
|
||||
def update_config(
|
||||
async def update_config(
|
||||
update: ConfigUpdate, auth: dict = Depends(require_auth)
|
||||
) -> AppConfig:
|
||||
"""Apply an update to the configuration and persist it.
|
||||
|
||||
Creates automatic backup before applying changes.
|
||||
Creates automatic backup before applying changes. If anime_directory
|
||||
is configured, starts the scheduler service.
|
||||
"""
|
||||
try:
|
||||
config_service = get_config_service()
|
||||
return config_service.update_config(update)
|
||||
updated_config = config_service.update_config(update)
|
||||
|
||||
# Sync anime_directory to settings if it was updated
|
||||
from src.config.settings import settings as app_settings
|
||||
|
||||
anime_dir_changed = False
|
||||
if update.other and update.other.get("anime_directory"):
|
||||
anime_dir = update.other.get("anime_directory")
|
||||
if anime_dir and not app_settings.anime_directory:
|
||||
app_settings.anime_directory = str(anime_dir)
|
||||
anime_dir_changed = True
|
||||
logger.info("Synced anime_directory from config: %s", anime_dir)
|
||||
|
||||
# Start scheduler if anime_directory was just configured
|
||||
if anime_dir_changed:
|
||||
try:
|
||||
from src.server.services.scheduler.scheduler_service import (
|
||||
get_scheduler_service,
|
||||
)
|
||||
|
||||
scheduler_svc = get_scheduler_service()
|
||||
logger.info(
|
||||
"Starting scheduler after anime_directory configuration"
|
||||
)
|
||||
await scheduler_svc.ensure_started()
|
||||
logger.info(
|
||||
"Scheduler started successfully after config update"
|
||||
)
|
||||
except Exception as sched_exc:
|
||||
logger.warning(
|
||||
"Failed to start scheduler after config update: %s",
|
||||
sched_exc,
|
||||
)
|
||||
# Config was already saved, don't fail the request
|
||||
|
||||
return updated_config
|
||||
|
||||
except ConfigValidationError as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
@@ -244,9 +284,9 @@ async def update_directory(
|
||||
try:
|
||||
import structlog
|
||||
|
||||
from src.server.services.anime_service import sync_series_from_data_files
|
||||
from src.server.services.anime_service import sync_legacy_series_to_db
|
||||
logger = structlog.get_logger(__name__)
|
||||
sync_count = await sync_series_from_data_files(directory, logger)
|
||||
sync_count = await sync_legacy_series_to_db(directory, logger)
|
||||
logger.info(
|
||||
"Directory updated: synced series from data files",
|
||||
directory=directory,
|
||||
|
||||
@@ -5,7 +5,7 @@ from datetime import datetime
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import psutil
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
@@ -17,15 +17,21 @@ logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/health", tags=["health"])
|
||||
|
||||
|
||||
from src.server.utils.version import APP_VERSION
|
||||
|
||||
|
||||
class HealthStatus(BaseModel):
|
||||
"""Basic health status response."""
|
||||
|
||||
status: str
|
||||
timestamp: str
|
||||
version: str = "1.0.0"
|
||||
version: str = APP_VERSION
|
||||
service: str = "aniworld-api"
|
||||
series_app_initialized: bool = False
|
||||
anime_directory_configured: bool = False
|
||||
scheduler_next_run: Optional[str] = None
|
||||
scheduler_last_run: Optional[str] = None
|
||||
checks: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
class DatabaseHealth(BaseModel):
|
||||
@@ -60,7 +66,7 @@ class DetailedHealthStatus(BaseModel):
|
||||
|
||||
status: str
|
||||
timestamp: str
|
||||
version: str = "1.0.0"
|
||||
version: str = APP_VERSION
|
||||
dependencies: DependencyHealth
|
||||
startup_time: datetime
|
||||
|
||||
@@ -171,29 +177,92 @@ def get_system_metrics() -> SystemMetrics:
|
||||
|
||||
|
||||
@router.get("", response_model=HealthStatus)
|
||||
async def basic_health_check() -> HealthStatus:
|
||||
async def basic_health_check(request: Request) -> HealthStatus:
|
||||
"""Basic health check endpoint.
|
||||
|
||||
This endpoint does not depend on anime_directory configuration
|
||||
and should always return 200 OK for basic health monitoring.
|
||||
Includes service information for identification.
|
||||
Includes scheduler next/last run times for monitoring tools.
|
||||
Includes startup health check results.
|
||||
|
||||
Returns:
|
||||
HealthStatus: Simple health status with timestamp and service info.
|
||||
"""
|
||||
from src.config.settings import settings
|
||||
from src.server.utils.dependencies import _series_app
|
||||
|
||||
# Get scheduler status for health monitoring
|
||||
scheduler_status: dict = {}
|
||||
try:
|
||||
from src.server.services.scheduler.scheduler_service import (
|
||||
get_scheduler_service,
|
||||
)
|
||||
scheduler_status = get_scheduler_service().get_status()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Get startup checks from app state
|
||||
checks = getattr(request.app.state, "startup_checks", None)
|
||||
|
||||
# Determine overall status based on checks
|
||||
overall_status = "healthy"
|
||||
if checks:
|
||||
for check_name, check_data in checks.items():
|
||||
if check_data.get("status") == "error":
|
||||
overall_status = "unhealthy"
|
||||
break
|
||||
elif check_data.get("status") == "warning":
|
||||
overall_status = "degraded"
|
||||
|
||||
logger.debug("Basic health check requested")
|
||||
return HealthStatus(
|
||||
status="healthy",
|
||||
status=overall_status,
|
||||
timestamp=datetime.now().isoformat(),
|
||||
service="aniworld-api",
|
||||
series_app_initialized=_series_app is not None,
|
||||
anime_directory_configured=bool(settings.anime_directory),
|
||||
scheduler_next_run=scheduler_status.get("next_run"),
|
||||
scheduler_last_run=scheduler_status.get("last_run"),
|
||||
checks=checks,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/ready")
|
||||
async def ready_check(request: Request) -> Dict[str, Any]:
|
||||
"""Readiness check endpoint for container orchestrators.
|
||||
|
||||
Returns 503 if critical dependencies are not available.
|
||||
This endpoint is used by Kubernetes, Docker Swarm, etc. to determine
|
||||
if the container should receive traffic.
|
||||
|
||||
Returns:
|
||||
dict: Readiness status with checks details.
|
||||
"""
|
||||
checks = getattr(request.app.state, "startup_checks", {})
|
||||
|
||||
critical_failures = []
|
||||
for check_name, check_data in checks.items():
|
||||
if check_data.get("status") == "error":
|
||||
critical_failures.append(f"{check_name}: {check_data.get('message')}")
|
||||
|
||||
if critical_failures:
|
||||
return {
|
||||
"status": "not_ready",
|
||||
"ready": False,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"critical_failures": critical_failures,
|
||||
"checks": checks,
|
||||
}
|
||||
|
||||
return {
|
||||
"status": "ready",
|
||||
"ready": True,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"checks": checks,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/detailed", response_model=DetailedHealthStatus)
|
||||
async def detailed_health_check(
|
||||
db: AsyncSession = Depends(get_database_session),
|
||||
|
||||
@@ -1,758 +1,70 @@
|
||||
"""NFO Management API endpoints.
|
||||
|
||||
This module provides REST API endpoints for managing tvshow.nfo files
|
||||
and associated media (poster, logo, fanart).
|
||||
Note: NFO service has been removed. All NFO endpoints return 503.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
|
||||
from src.config.settings import settings
|
||||
from src.core.entities.series import Serie
|
||||
from src.core.SeriesApp import SeriesApp
|
||||
from src.core.services.nfo_factory import get_nfo_factory
|
||||
from src.core.services.nfo_service import NFOService
|
||||
from src.core.services.tmdb_client import TMDBAPIError
|
||||
from src.server.models.nfo import (
|
||||
MediaDownloadRequest,
|
||||
MediaFilesStatus,
|
||||
NFOBatchCreateRequest,
|
||||
NFOBatchCreateResponse,
|
||||
NFOBatchResult,
|
||||
NFOCheckResponse,
|
||||
NFOContentResponse,
|
||||
NFOCreateRequest,
|
||||
NFOCreateResponse,
|
||||
NFOMissingResponse,
|
||||
NFOMissingSeries,
|
||||
)
|
||||
from src.server.utils.dependencies import get_series_app, require_auth
|
||||
from src.server.utils.media import check_media_files, get_media_file_paths
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from fastapi import APIRouter, HTTPException, status
|
||||
|
||||
router = APIRouter(prefix="/api/nfo", tags=["nfo"])
|
||||
|
||||
|
||||
async def get_nfo_service() -> NFOService:
|
||||
"""Get NFO service dependency.
|
||||
|
||||
Returns:
|
||||
NFOService instance
|
||||
|
||||
Raises:
|
||||
HTTPException: If NFO service not configured
|
||||
"""
|
||||
try:
|
||||
# Use centralized factory for consistent initialization
|
||||
factory = get_nfo_factory()
|
||||
return factory.create()
|
||||
except ValueError as e:
|
||||
# Factory raises ValueError if API key not configured
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail=str(e)
|
||||
) from e
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# IMPORTANT: Literal path routes must be defined BEFORE path parameter routes
|
||||
# to avoid route matching conflicts. For example, /batch/create must come
|
||||
# before /{serie_id}/create, otherwise "batch" is treated as a serie_id.
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@router.post("/batch/create", response_model=NFOBatchCreateResponse)
|
||||
async def batch_create_nfo(
|
||||
request: NFOBatchCreateRequest,
|
||||
_auth: dict = Depends(require_auth),
|
||||
series_app: SeriesApp = Depends(get_series_app),
|
||||
nfo_service: NFOService = Depends(get_nfo_service)
|
||||
) -> NFOBatchCreateResponse:
|
||||
"""Batch create NFO files for multiple series.
|
||||
|
||||
Args:
|
||||
request: Batch creation options
|
||||
_auth: Authentication dependency
|
||||
series_app: Series app dependency
|
||||
nfo_service: NFO service dependency
|
||||
|
||||
Returns:
|
||||
NFOBatchCreateResponse with results
|
||||
"""
|
||||
results: List[NFOBatchResult] = []
|
||||
successful = 0
|
||||
failed = 0
|
||||
skipped = 0
|
||||
|
||||
# Get all series
|
||||
series_list = series_app.list.GetList()
|
||||
series_map = {
|
||||
getattr(s, 'key', None): s
|
||||
for s in series_list
|
||||
if getattr(s, 'key', None)
|
||||
}
|
||||
|
||||
# Process each series
|
||||
semaphore = asyncio.Semaphore(request.max_concurrent)
|
||||
|
||||
async def process_serie(serie_id: str) -> NFOBatchResult:
|
||||
"""Process a single series."""
|
||||
async with semaphore:
|
||||
try:
|
||||
serie = series_map.get(serie_id)
|
||||
if not serie:
|
||||
return NFOBatchResult(
|
||||
serie_id=serie_id,
|
||||
serie_folder="",
|
||||
success=False,
|
||||
message="Series not found"
|
||||
)
|
||||
|
||||
# Ensure folder name includes year if available
|
||||
serie_folder = serie.ensure_folder_with_year()
|
||||
|
||||
# Check if NFO exists
|
||||
if request.skip_existing:
|
||||
has_nfo = await nfo_service.check_nfo_exists(serie_folder)
|
||||
if has_nfo:
|
||||
return NFOBatchResult(
|
||||
serie_id=serie_id,
|
||||
serie_folder=serie_folder,
|
||||
success=False,
|
||||
message="Skipped - NFO already exists"
|
||||
)
|
||||
|
||||
# Create NFO
|
||||
nfo_path = await nfo_service.create_tvshow_nfo(
|
||||
serie_name=serie.name or serie_folder,
|
||||
serie_folder=serie_folder,
|
||||
download_poster=request.download_media,
|
||||
download_logo=request.download_media,
|
||||
download_fanart=request.download_media
|
||||
)
|
||||
|
||||
return NFOBatchResult(
|
||||
serie_id=serie_id,
|
||||
serie_folder=serie_folder,
|
||||
success=True,
|
||||
message="NFO created successfully",
|
||||
nfo_path=str(nfo_path)
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error creating NFO for {serie_id}: {e}",
|
||||
exc_info=True
|
||||
)
|
||||
return NFOBatchResult(
|
||||
serie_id=serie_id,
|
||||
serie_folder=serie.folder if serie else "",
|
||||
success=False,
|
||||
message=f"Error: {str(e)}"
|
||||
)
|
||||
|
||||
# Process all series concurrently
|
||||
tasks = [process_serie(sid) for sid in request.serie_ids]
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
# Count results
|
||||
for result in results:
|
||||
if result.success:
|
||||
successful += 1
|
||||
elif "Skipped" in result.message:
|
||||
skipped += 1
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
return NFOBatchCreateResponse(
|
||||
total=len(request.serie_ids),
|
||||
successful=successful,
|
||||
failed=failed,
|
||||
skipped=skipped,
|
||||
results=list(results)
|
||||
@router.get("/disabled")
|
||||
async def nfo_disabled():
|
||||
"""NFO endpoints disabled - NFO service removed."""
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="NFO service has been removed. Use series management endpoints instead."
|
||||
)
|
||||
|
||||
|
||||
@router.get("/missing", response_model=NFOMissingResponse)
|
||||
async def get_missing_nfo(
|
||||
_auth: dict = Depends(require_auth),
|
||||
series_app: SeriesApp = Depends(get_series_app),
|
||||
nfo_service: NFOService = Depends(get_nfo_service)
|
||||
) -> NFOMissingResponse:
|
||||
"""Get list of series without NFO files.
|
||||
|
||||
Args:
|
||||
_auth: Authentication dependency
|
||||
series_app: Series app dependency
|
||||
nfo_service: NFO service dependency
|
||||
|
||||
Returns:
|
||||
NFOMissingResponse with series list
|
||||
"""
|
||||
try:
|
||||
series_list = series_app.list.GetList()
|
||||
missing_series: List[NFOMissingSeries] = []
|
||||
|
||||
for serie in series_list:
|
||||
serie_id = getattr(serie, 'key', None)
|
||||
if not serie_id:
|
||||
continue
|
||||
|
||||
# Ensure folder name includes year if available
|
||||
serie_folder = serie.ensure_folder_with_year()
|
||||
has_nfo = await nfo_service.check_nfo_exists(serie_folder)
|
||||
|
||||
if not has_nfo:
|
||||
# Build full path and check media files
|
||||
folder_path = Path(settings.anime_directory) / serie_folder
|
||||
media_status = check_media_files(folder_path)
|
||||
file_paths = get_media_file_paths(folder_path)
|
||||
|
||||
media_files = MediaFilesStatus(
|
||||
has_poster=media_status.get("poster", False),
|
||||
has_logo=media_status.get("logo", False),
|
||||
has_fanart=media_status.get("fanart", False),
|
||||
poster_path=str(file_paths["poster"]) if file_paths.get("poster") else None,
|
||||
logo_path=str(file_paths["logo"]) if file_paths.get("logo") else None,
|
||||
fanart_path=str(file_paths["fanart"]) if file_paths.get("fanart") else None
|
||||
)
|
||||
|
||||
has_media = (
|
||||
media_files.has_poster
|
||||
or media_files.has_logo
|
||||
or media_files.has_fanart
|
||||
)
|
||||
|
||||
missing_series.append(NFOMissingSeries(
|
||||
serie_id=serie_id,
|
||||
serie_folder=serie_folder,
|
||||
serie_name=serie.name or serie_folder,
|
||||
has_media=has_media,
|
||||
media_files=media_files
|
||||
))
|
||||
|
||||
return NFOMissingResponse(
|
||||
total_series=len(series_list),
|
||||
missing_nfo_count=len(missing_series),
|
||||
series=missing_series
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Error getting missing NFOs: %s", e)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to get missing NFOs: {str(e)}"
|
||||
) from e
|
||||
@router.post("/batch/create")
|
||||
async def batch_create_nfo():
|
||||
"""NFO endpoints disabled - NFO service removed."""
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="NFO service has been removed. Use series management endpoints instead."
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Series-specific endpoints (with {serie_id} path parameter)
|
||||
# These must come AFTER literal path routes like /batch/create and /missing
|
||||
# =============================================================================
|
||||
@router.post("/{serie_id}/create")
|
||||
async def create_nfo(serie_id: str):
|
||||
"""NFO endpoints disabled - NFO service removed."""
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="NFO service has been removed. Use series management endpoints instead."
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{serie_id}/check", response_model=NFOCheckResponse)
|
||||
async def check_nfo(
|
||||
serie_id: str,
|
||||
_auth: dict = Depends(require_auth),
|
||||
series_app: SeriesApp = Depends(get_series_app),
|
||||
nfo_service: NFOService = Depends(get_nfo_service)
|
||||
) -> NFOCheckResponse:
|
||||
"""Check if NFO and media files exist for a series.
|
||||
|
||||
Args:
|
||||
serie_id: Series identifier
|
||||
_auth: Authentication dependency
|
||||
series_app: Series app dependency
|
||||
nfo_service: NFO service dependency
|
||||
|
||||
Returns:
|
||||
NFOCheckResponse with NFO and media status
|
||||
|
||||
Raises:
|
||||
HTTPException: If series not found
|
||||
"""
|
||||
try:
|
||||
# Get series info
|
||||
series_list = series_app.list.GetList()
|
||||
serie = next(
|
||||
(s for s in series_list if getattr(s, 'key', None) == serie_id),
|
||||
None
|
||||
)
|
||||
|
||||
if not serie:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Series not found: {serie_id}"
|
||||
)
|
||||
|
||||
# Ensure folder name includes year if available
|
||||
serie_folder = serie.ensure_folder_with_year()
|
||||
folder_path = Path(settings.anime_directory) / serie_folder
|
||||
|
||||
# Check NFO
|
||||
has_nfo = await nfo_service.check_nfo_exists(serie_folder)
|
||||
nfo_path = None
|
||||
if has_nfo:
|
||||
nfo_path = str(folder_path / "tvshow.nfo")
|
||||
|
||||
# Check media files using utility function
|
||||
media_status = check_media_files(
|
||||
folder_path,
|
||||
check_poster=True,
|
||||
check_logo=True,
|
||||
check_fanart=True,
|
||||
check_nfo=False # Already checked above
|
||||
)
|
||||
|
||||
# Get file paths
|
||||
file_paths = get_media_file_paths(folder_path)
|
||||
|
||||
# Build MediaFilesStatus model
|
||||
media_files = MediaFilesStatus(
|
||||
has_poster=media_status.get("poster", False),
|
||||
has_logo=media_status.get("logo", False),
|
||||
has_fanart=media_status.get("fanart", False),
|
||||
poster_path=str(file_paths["poster"]) if file_paths["poster"] else None,
|
||||
logo_path=str(file_paths["logo"]) if file_paths["logo"] else None,
|
||||
fanart_path=str(file_paths["fanart"]) if file_paths["fanart"] else None
|
||||
)
|
||||
|
||||
return NFOCheckResponse(
|
||||
serie_id=serie_id,
|
||||
serie_folder=serie_folder,
|
||||
has_nfo=has_nfo,
|
||||
nfo_path=nfo_path,
|
||||
media_files=media_files
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception("Error checking NFO for %s: %s", serie_id, e)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to check NFO: {str(e)}"
|
||||
) from e
|
||||
@router.get("/{serie_id}/status")
|
||||
async def get_nfo_status(serie_id: str):
|
||||
"""NFO endpoints disabled - NFO service removed."""
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="NFO service has been removed. Use series management endpoints instead."
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{serie_id}/create", response_model=NFOCreateResponse)
|
||||
async def create_nfo(
|
||||
serie_id: str,
|
||||
request: NFOCreateRequest,
|
||||
_auth: dict = Depends(require_auth),
|
||||
series_app: SeriesApp = Depends(get_series_app),
|
||||
nfo_service: NFOService = Depends(get_nfo_service)
|
||||
) -> NFOCreateResponse:
|
||||
"""Create NFO file and download media for a series.
|
||||
|
||||
Args:
|
||||
serie_id: Series identifier
|
||||
request: NFO creation options
|
||||
_auth: Authentication dependency
|
||||
series_app: Series app dependency
|
||||
nfo_service: NFO service dependency
|
||||
|
||||
Returns:
|
||||
NFOCreateResponse with creation result
|
||||
|
||||
Raises:
|
||||
HTTPException: If series not found or creation fails
|
||||
"""
|
||||
try:
|
||||
# Get series info
|
||||
series_list = series_app.list.GetList()
|
||||
serie = next(
|
||||
(s for s in series_list if getattr(s, 'key', None) == serie_id),
|
||||
None
|
||||
)
|
||||
|
||||
if not serie:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Series not found: {serie_id}"
|
||||
)
|
||||
|
||||
# Ensure folder name includes year if available
|
||||
serie_folder = serie.ensure_folder_with_year()
|
||||
|
||||
# If year not provided in request but serie has year, use it
|
||||
year = request.year or serie.year
|
||||
|
||||
# Check if NFO already exists
|
||||
if not request.overwrite_existing:
|
||||
has_nfo = await nfo_service.check_nfo_exists(serie_folder)
|
||||
if has_nfo:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail="NFO already exists. Use overwrite_existing=true"
|
||||
)
|
||||
|
||||
# Create NFO
|
||||
serie_name = request.serie_name or serie.name or serie_folder
|
||||
nfo_path = await nfo_service.create_tvshow_nfo(
|
||||
serie_name=serie_name,
|
||||
serie_folder=serie_folder,
|
||||
year=year,
|
||||
download_poster=request.download_poster,
|
||||
download_logo=request.download_logo,
|
||||
download_fanart=request.download_fanart
|
||||
)
|
||||
|
||||
# Check media files
|
||||
folder_path = Path(settings.anime_directory) / serie_folder
|
||||
media_status = check_media_files(folder_path)
|
||||
file_paths = get_media_file_paths(folder_path)
|
||||
|
||||
media_files = MediaFilesStatus(
|
||||
has_poster=media_status.get("poster", False),
|
||||
has_logo=media_status.get("logo", False),
|
||||
has_fanart=media_status.get("fanart", False),
|
||||
poster_path=str(file_paths["poster"]) if file_paths.get("poster") else None,
|
||||
logo_path=str(file_paths["logo"]) if file_paths.get("logo") else None,
|
||||
fanart_path=str(file_paths["fanart"]) if file_paths.get("fanart") else None
|
||||
)
|
||||
|
||||
return NFOCreateResponse(
|
||||
serie_id=serie_id,
|
||||
serie_folder=serie_folder,
|
||||
nfo_path=str(nfo_path),
|
||||
media_files=media_files,
|
||||
message="NFO and media files created successfully"
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except TMDBAPIError as e:
|
||||
logger.warning("TMDB API error creating NFO for %s: %s", serie_id, e)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail=f"TMDB API error: {str(e)}"
|
||||
) from e
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error creating NFO for {serie_id}: {e}",
|
||||
exc_info=True
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to create NFO: {str(e)}"
|
||||
) from e
|
||||
@router.delete("/{serie_id}/delete")
|
||||
async def delete_nfo(serie_id: str):
|
||||
"""NFO endpoints disabled - NFO service removed."""
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="NFO service has been removed. Use series management endpoints instead."
|
||||
)
|
||||
|
||||
|
||||
@router.put("/{serie_id}/update", response_model=NFOCreateResponse)
|
||||
async def update_nfo(
|
||||
serie_id: str,
|
||||
download_media: bool = True,
|
||||
_auth: dict = Depends(require_auth),
|
||||
series_app: SeriesApp = Depends(get_series_app),
|
||||
nfo_service: NFOService = Depends(get_nfo_service)
|
||||
) -> NFOCreateResponse:
|
||||
"""Update existing NFO file with fresh TMDB data.
|
||||
|
||||
Args:
|
||||
serie_id: Series identifier
|
||||
download_media: Whether to re-download media files
|
||||
_auth: Authentication dependency
|
||||
series_app: Series app dependency
|
||||
nfo_service: NFO service dependency
|
||||
|
||||
Returns:
|
||||
NFOCreateResponse with update result
|
||||
|
||||
Raises:
|
||||
HTTPException: If series or NFO not found
|
||||
"""
|
||||
try:
|
||||
# Get series info
|
||||
series_list = series_app.list.GetList()
|
||||
serie = next(
|
||||
(s for s in series_list if getattr(s, 'key', None) == serie_id),
|
||||
None
|
||||
)
|
||||
|
||||
if not serie:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Series not found: {serie_id}"
|
||||
)
|
||||
|
||||
# Ensure folder name includes year if available
|
||||
serie_folder = serie.ensure_folder_with_year()
|
||||
|
||||
# Check if NFO exists
|
||||
has_nfo = await nfo_service.check_nfo_exists(serie_folder)
|
||||
if not has_nfo:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="NFO file not found. Use create endpoint instead."
|
||||
)
|
||||
|
||||
# Update NFO
|
||||
nfo_path = await nfo_service.update_tvshow_nfo(
|
||||
serie_folder=serie_folder,
|
||||
download_media=download_media
|
||||
)
|
||||
|
||||
# Check media files
|
||||
folder_path = Path(settings.anime_directory) / serie_folder
|
||||
media_status = check_media_files(folder_path)
|
||||
file_paths = get_media_file_paths(folder_path)
|
||||
|
||||
media_files = MediaFilesStatus(
|
||||
has_poster=media_status.get("poster", False),
|
||||
has_logo=media_status.get("logo", False),
|
||||
has_fanart=media_status.get("fanart", False),
|
||||
poster_path=str(file_paths["poster"]) if file_paths.get("poster") else None,
|
||||
logo_path=str(file_paths["logo"]) if file_paths.get("logo") else None,
|
||||
fanart_path=str(file_paths["fanart"]) if file_paths.get("fanart") else None
|
||||
)
|
||||
|
||||
return NFOCreateResponse(
|
||||
serie_id=serie_id,
|
||||
serie_folder=serie_folder,
|
||||
nfo_path=str(nfo_path),
|
||||
media_files=media_files,
|
||||
message="NFO updated successfully"
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except TMDBAPIError as e:
|
||||
logger.warning("TMDB API error updating NFO for %s: %s", serie_id, e)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail=f"TMDB API error: {str(e)}"
|
||||
) from e
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error updating NFO for {serie_id}: {e}",
|
||||
exc_info=True
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to update NFO: {str(e)}"
|
||||
) from e
|
||||
@router.get("/poster/{serie_id}")
|
||||
async def get_nfo_poster(serie_id: str):
|
||||
"""NFO endpoints disabled - NFO service removed."""
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="NFO service has been removed. Use series management endpoints instead."
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{serie_id}/content", response_model=NFOContentResponse)
|
||||
async def get_nfo_content(
|
||||
serie_id: str,
|
||||
_auth: dict = Depends(require_auth),
|
||||
series_app: SeriesApp = Depends(get_series_app),
|
||||
nfo_service: NFOService = Depends(get_nfo_service)
|
||||
) -> NFOContentResponse:
|
||||
"""Get NFO file content for a series.
|
||||
|
||||
Args:
|
||||
serie_id: Series identifier
|
||||
_auth: Authentication dependency
|
||||
series_app: Series app dependency
|
||||
nfo_service: NFO service dependency
|
||||
|
||||
Returns:
|
||||
NFOContentResponse with NFO content
|
||||
|
||||
Raises:
|
||||
HTTPException: If series or NFO not found
|
||||
"""
|
||||
try:
|
||||
# Get series info
|
||||
series_list = series_app.list.GetList()
|
||||
serie = next(
|
||||
(s for s in series_list if getattr(s, 'key', None) == serie_id),
|
||||
None
|
||||
)
|
||||
|
||||
if not serie:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Series not found: {serie_id}"
|
||||
)
|
||||
|
||||
# Ensure folder name includes year if available
|
||||
serie_folder = serie.ensure_folder_with_year()
|
||||
|
||||
# Check if NFO exists
|
||||
nfo_path = (
|
||||
Path(settings.anime_directory) / serie_folder / "tvshow.nfo"
|
||||
)
|
||||
if not nfo_path.exists():
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="NFO file not found"
|
||||
)
|
||||
|
||||
# Read NFO content
|
||||
content = nfo_path.read_text(encoding="utf-8")
|
||||
file_size = nfo_path.stat().st_size
|
||||
last_modified = datetime.fromtimestamp(nfo_path.stat().st_mtime)
|
||||
|
||||
return NFOContentResponse(
|
||||
serie_id=serie_id,
|
||||
serie_folder=serie_folder,
|
||||
content=content,
|
||||
file_size=file_size,
|
||||
last_modified=last_modified
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error reading NFO content for {serie_id}: {e}",
|
||||
exc_info=True
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to read NFO content: {str(e)}"
|
||||
) from e
|
||||
|
||||
|
||||
@router.get("/{serie_id}/media/status", response_model=MediaFilesStatus)
|
||||
async def get_media_status(
|
||||
serie_id: str,
|
||||
_auth: dict = Depends(require_auth),
|
||||
series_app: SeriesApp = Depends(get_series_app)
|
||||
) -> MediaFilesStatus:
|
||||
"""Get media files status for a series.
|
||||
|
||||
Args:
|
||||
serie_id: Series identifier
|
||||
_auth: Authentication dependency
|
||||
series_app: Series app dependency
|
||||
|
||||
Returns:
|
||||
MediaFilesStatus with file existence info
|
||||
|
||||
Raises:
|
||||
HTTPException: If series not found
|
||||
"""
|
||||
try:
|
||||
# Get series info
|
||||
series_list = series_app.list.GetList()
|
||||
serie = next(
|
||||
(s for s in series_list if getattr(s, 'key', None) == serie_id),
|
||||
None
|
||||
)
|
||||
|
||||
if not serie:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Series not found: {serie_id}"
|
||||
)
|
||||
|
||||
# Build full path and check media files
|
||||
folder_path = Path(settings.anime_directory) / serie.folder
|
||||
media_status = check_media_files(folder_path)
|
||||
file_paths = get_media_file_paths(folder_path)
|
||||
|
||||
return MediaFilesStatus(
|
||||
has_poster=media_status.get("poster", False),
|
||||
has_logo=media_status.get("logo", False),
|
||||
has_fanart=media_status.get("fanart", False),
|
||||
poster_path=str(file_paths["poster"]) if file_paths.get("poster") else None,
|
||||
logo_path=str(file_paths["logo"]) if file_paths.get("logo") else None,
|
||||
fanart_path=str(file_paths["fanart"]) if file_paths.get("fanart") else None
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error checking media status for {serie_id}: {e}",
|
||||
exc_info=True
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to check media status: {str(e)}"
|
||||
) from e
|
||||
|
||||
|
||||
@router.post("/{serie_id}/media/download", response_model=MediaFilesStatus)
|
||||
async def download_media(
|
||||
serie_id: str,
|
||||
request: MediaDownloadRequest,
|
||||
_auth: dict = Depends(require_auth),
|
||||
series_app: SeriesApp = Depends(get_series_app),
|
||||
nfo_service: NFOService = Depends(get_nfo_service)
|
||||
) -> MediaFilesStatus:
|
||||
"""Download missing media files for a series.
|
||||
|
||||
Args:
|
||||
serie_id: Series identifier
|
||||
request: Media download options
|
||||
_auth: Authentication dependency
|
||||
series_app: Series app dependency
|
||||
nfo_service: NFO service dependency
|
||||
|
||||
Returns:
|
||||
MediaFilesStatus after download attempt
|
||||
|
||||
Raises:
|
||||
HTTPException: If series or NFO not found
|
||||
"""
|
||||
try:
|
||||
# Get series info
|
||||
series_list = series_app.list.GetList()
|
||||
serie = next(
|
||||
(s for s in series_list if getattr(s, 'key', None) == serie_id),
|
||||
None
|
||||
)
|
||||
|
||||
if not serie:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Series not found: {serie_id}"
|
||||
)
|
||||
|
||||
# Ensure folder name includes year if available
|
||||
serie_folder = serie.ensure_folder_with_year()
|
||||
|
||||
# Check if NFO exists (needed for TMDB ID)
|
||||
has_nfo = await nfo_service.check_nfo_exists(serie_folder)
|
||||
if not has_nfo:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="NFO required for media download. Create NFO first."
|
||||
)
|
||||
|
||||
# For now, update NFO which will re-download media
|
||||
# In future, could add standalone media download
|
||||
if (request.download_poster or request.download_logo
|
||||
or request.download_fanart):
|
||||
await nfo_service.update_tvshow_nfo(
|
||||
serie_folder=serie_folder,
|
||||
download_media=True
|
||||
)
|
||||
|
||||
# Build full path and check media files
|
||||
folder_path = Path(settings.anime_directory) / serie_folder
|
||||
media_status = check_media_files(folder_path)
|
||||
file_paths = get_media_file_paths(folder_path)
|
||||
|
||||
return MediaFilesStatus(
|
||||
has_poster=media_status.get("poster", False),
|
||||
has_logo=media_status.get("logo", False),
|
||||
has_fanart=media_status.get("fanart", False),
|
||||
poster_path=str(file_paths["poster"]) if file_paths.get("poster") else None,
|
||||
logo_path=str(file_paths["logo"]) if file_paths.get("logo") else None,
|
||||
fanart_path=str(file_paths["fanart"]) if file_paths.get("fanart") else None
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error downloading media for {serie_id}: {e}",
|
||||
exc_info=True
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to download media: {str(e)}"
|
||||
) from e
|
||||
@router.get("/fanart/{serie_id}")
|
||||
async def get_nfo_fanart(serie_id: str):
|
||||
"""NFO endpoints disabled - NFO service removed."""
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="NFO service has been removed. Use series management endpoints instead."
|
||||
)
|
||||
@@ -10,7 +10,7 @@ from fastapi import APIRouter, Depends, HTTPException, status
|
||||
|
||||
from src.server.models.config import SchedulerConfig
|
||||
from src.server.services.config_service import ConfigServiceError, get_config_service
|
||||
from src.server.services.scheduler_service import get_scheduler_service
|
||||
from src.server.services.scheduler.scheduler_service import get_scheduler_service
|
||||
from src.server.utils.dependencies import require_auth
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
313
src/server/api/setup_endpoints.py
Normal file
313
src/server/api/setup_endpoints.py
Normal file
@@ -0,0 +1,313 @@
|
||||
"""API endpoints for setup and unresolved folder management.
|
||||
|
||||
Provides endpoints to:
|
||||
- List unresolved folders that couldn't be auto-resolved during setup
|
||||
- Get suggestions/search results for an unresolved folder
|
||||
- Resolve an unresolved folder by providing a provider key
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from src.server.database.connection import get_db_session
|
||||
from src.server.database.service import AnimeSeriesService, UnresolvedFolderService
|
||||
from src.server.utils.dependencies import (
|
||||
get_database_session,
|
||||
get_series_app,
|
||||
require_auth,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/setup", tags=["setup"])
|
||||
|
||||
|
||||
class UnresolvedFolderResponse(BaseModel):
|
||||
"""Response model for an unresolved folder."""
|
||||
|
||||
folder_name: str = Field(..., description="Original filesystem folder name")
|
||||
title: str = Field(..., description="Extracted title from folder name")
|
||||
year: Optional[int] = Field(None, description="Extracted release year")
|
||||
search_attempts: int = Field(..., description="Number of search attempts made")
|
||||
search_suggestions: list[dict[str, Any]] = Field(
|
||||
default_factory=list,
|
||||
description="Cached search results for potential matches"
|
||||
)
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class ResolveFolderRequest(BaseModel):
|
||||
"""Request model for resolving an unresolved folder."""
|
||||
|
||||
provider_key: str = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
max_length=255,
|
||||
description="Provider key to associate with this folder"
|
||||
)
|
||||
|
||||
|
||||
class ResolveFolderResponse(BaseModel):
|
||||
"""Response model for resolving an unresolved folder."""
|
||||
|
||||
status: str = Field(..., description="Operation status")
|
||||
message: str = Field(..., description="Human-readable message")
|
||||
folder_name: str = Field(..., description="Folder name that was resolved")
|
||||
key: str = Field(..., description="Provider key that was used")
|
||||
series_id: int = Field(..., description="Database ID of the created series")
|
||||
|
||||
|
||||
@router.get("/unresolved", response_model=list[UnresolvedFolderResponse])
|
||||
async def list_unresolved_folders(
|
||||
db=Depends(get_database_session),
|
||||
) -> list[UnresolvedFolderResponse]:
|
||||
"""List all unresolved folders that need manual key resolution.
|
||||
|
||||
Returns folders that couldn't be auto-resolved during setup,
|
||||
including cached search suggestions when available.
|
||||
|
||||
Returns:
|
||||
List of UnresolvedFolderResponse objects
|
||||
"""
|
||||
folders = await UnresolvedFolderService.get_all_unresolved(db)
|
||||
|
||||
result = []
|
||||
for folder in folders:
|
||||
suggestions = []
|
||||
if folder.last_search_result:
|
||||
try:
|
||||
suggestions = json.loads(folder.last_search_result)
|
||||
except json.JSONDecodeError:
|
||||
logger.warning(
|
||||
"Failed to parse search result for folder: %s",
|
||||
folder.folder_name
|
||||
)
|
||||
|
||||
result.append(UnresolvedFolderResponse(
|
||||
folder_name=folder.folder_name,
|
||||
title=folder.title,
|
||||
year=folder.year,
|
||||
search_attempts=folder.search_attempts,
|
||||
search_suggestions=suggestions,
|
||||
))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/unresolved/{folder_name}", response_model=UnresolvedFolderResponse)
|
||||
async def get_unresolved_folder(
|
||||
folder_name: str,
|
||||
db=Depends(get_database_session),
|
||||
) -> UnresolvedFolderResponse:
|
||||
"""Get details for a specific unresolved folder.
|
||||
|
||||
Args:
|
||||
folder_name: URL-encoded folder name to look up
|
||||
|
||||
Returns:
|
||||
UnresolvedFolderResponse for the specified folder
|
||||
|
||||
Raises:
|
||||
HTTPException: 404 if folder not found or already resolved
|
||||
"""
|
||||
folder = await UnresolvedFolderService.get_by_folder_name(db, folder_name)
|
||||
|
||||
if not folder:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Unresolved folder not found: {folder_name}"
|
||||
)
|
||||
|
||||
if folder.is_resolved:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Folder already resolved: {folder_name}"
|
||||
)
|
||||
|
||||
suggestions = []
|
||||
if folder.last_search_result:
|
||||
try:
|
||||
suggestions = json.loads(folder.last_search_result)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
return UnresolvedFolderResponse(
|
||||
folder_name=folder.folder_name,
|
||||
title=folder.title,
|
||||
year=folder.year,
|
||||
search_attempts=folder.search_attempts,
|
||||
search_suggestions=suggestions,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/unresolved/{folder_name}/resolve", response_model=ResolveFolderResponse)
|
||||
async def resolve_unresolved_folder(
|
||||
folder_name: str,
|
||||
request: ResolveFolderRequest,
|
||||
db=Depends(get_database_session),
|
||||
) -> ResolveFolderResponse:
|
||||
"""Resolve an unresolved folder by providing the correct provider key.
|
||||
|
||||
This endpoint:
|
||||
1. Validates the provider key format
|
||||
2. Updates the UnresolvedFolder record as resolved
|
||||
3. Creates the AnimeSeries record in the database
|
||||
4. Returns the created series information
|
||||
|
||||
Args:
|
||||
folder_name: URL-encoded folder name to resolve
|
||||
request: ResolveFolderRequest with the provider_key
|
||||
|
||||
Returns:
|
||||
ResolveFolderResponse with created series details
|
||||
|
||||
Raises:
|
||||
HTTPException: 404 if folder not found
|
||||
HTTPException: 400 if key is invalid or series already exists
|
||||
"""
|
||||
# Check if folder exists and is unresolved
|
||||
unresolved = await UnresolvedFolderService.get_by_folder_name(db, folder_name)
|
||||
|
||||
if not unresolved:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Unresolved folder not found: {folder_name}"
|
||||
)
|
||||
|
||||
if unresolved.is_resolved:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Folder already resolved: {folder_name}"
|
||||
)
|
||||
|
||||
# Check if a series with this key already exists
|
||||
existing_series = await AnimeSeriesService.get_by_key(db, request.provider_key)
|
||||
if existing_series:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Series with key '{request.provider_key}' already exists"
|
||||
)
|
||||
|
||||
# Mark as resolved
|
||||
await UnresolvedFolderService.resolve(db, folder_name, request.provider_key)
|
||||
|
||||
# Create the AnimeSeries record
|
||||
series = await AnimeSeriesService.create(
|
||||
db=db,
|
||||
key=request.provider_key,
|
||||
name=unresolved.title,
|
||||
site="https://aniworld.to",
|
||||
folder=folder_name,
|
||||
year=unresolved.year,
|
||||
loading_status="pending",
|
||||
episodes_loaded=False,
|
||||
logo_loaded=False,
|
||||
images_loaded=False,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Resolved unresolved folder via API: %s -> key=%s (series_id=%d)",
|
||||
folder_name, request.provider_key, series.id
|
||||
)
|
||||
|
||||
return ResolveFolderResponse(
|
||||
status="success",
|
||||
message=f"Successfully resolved and added series: {unresolved.title}",
|
||||
folder_name=folder_name,
|
||||
key=request.provider_key,
|
||||
series_id=series.id,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/unresolved/{folder_name}/search", response_model=UnresolvedFolderResponse)
|
||||
async def search_unresolved_folder(
|
||||
folder_name: str,
|
||||
db=Depends(get_database_session),
|
||||
) -> UnresolvedFolderResponse:
|
||||
"""Re-search for a specific unresolved folder to get fresh suggestions.
|
||||
|
||||
Performs a new search using the folder's title and caches the results.
|
||||
|
||||
Args:
|
||||
folder_name: URL-encoded folder name to search for
|
||||
|
||||
Returns:
|
||||
UnresolvedFolderResponse with updated search suggestions
|
||||
|
||||
Raises:
|
||||
HTTPException: 404 if folder not found or already resolved
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
folder = await UnresolvedFolderService.get_by_folder_name(db, folder_name)
|
||||
|
||||
if not folder:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Unresolved folder not found: {folder_name}"
|
||||
)
|
||||
|
||||
if folder.is_resolved:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Folder already resolved: {folder_name}"
|
||||
)
|
||||
|
||||
# Perform search
|
||||
series_app = get_series_app()
|
||||
try:
|
||||
results = await series_app.search(folder.title)
|
||||
search_result_json = json.dumps(results) if results else "[]"
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Search failed for unresolved folder: %s, error: %s",
|
||||
folder_name, str(e)
|
||||
)
|
||||
search_result_json = "[]"
|
||||
results = []
|
||||
|
||||
# Update the folder with new search results
|
||||
await UnresolvedFolderService.update_search_result(db, folder_name, search_result_json)
|
||||
|
||||
return UnresolvedFolderResponse(
|
||||
folder_name=folder.folder_name,
|
||||
title=folder.title,
|
||||
year=folder.year,
|
||||
search_attempts=folder.search_attempts,
|
||||
search_suggestions=results,
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/unresolved/{folder_name}")
|
||||
async def delete_unresolved_folder(
|
||||
folder_name: str,
|
||||
db=Depends(get_database_session),
|
||||
) -> dict[str, str]:
|
||||
"""Delete an unresolved folder tracking record.
|
||||
|
||||
Use this when you've manually added the series outside of this flow
|
||||
(e.g., via POST /api/anime/add) to clean up the unresolved tracker.
|
||||
|
||||
Args:
|
||||
folder_name: URL-encoded folder name to delete
|
||||
|
||||
Returns:
|
||||
Dict with status message
|
||||
|
||||
Raises:
|
||||
HTTPException: 404 if folder not found
|
||||
"""
|
||||
deleted = await UnresolvedFolderService.delete(db, folder_name)
|
||||
|
||||
if not deleted:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Unresolved folder not found: {folder_name}"
|
||||
)
|
||||
|
||||
return {"status": "success", "message": f"Deleted unresolved folder: {folder_name}"}
|
||||
@@ -39,7 +39,7 @@ def get_settings() -> Union[DevelopmentSettings, ProductionSettings]:
|
||||
Example:
|
||||
>>> settings = get_settings()
|
||||
>>> print(settings.log_level)
|
||||
DEBUG
|
||||
INFO
|
||||
"""
|
||||
if ENVIRONMENT in {"development", "testing"}:
|
||||
return get_development_settings()
|
||||
|
||||
@@ -215,7 +215,7 @@ class DevelopmentSettings(BaseSettings):
|
||||
@property
|
||||
def debug_enabled(self) -> bool:
|
||||
"""Check if debug mode is enabled."""
|
||||
return True
|
||||
return False
|
||||
|
||||
@property
|
||||
def reload_enabled(self) -> bool:
|
||||
|
||||
@@ -59,3 +59,13 @@ async def loading_page(request: Request):
|
||||
request,
|
||||
title="Initializing - Aniworld"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/setup/unresolved", response_class=HTMLResponse)
|
||||
async def unresolved_page(request: Request):
|
||||
"""Serve the unresolved folders resolution page."""
|
||||
return render_template(
|
||||
"unresolved.html",
|
||||
request,
|
||||
title="Resolve Series - Aniworld"
|
||||
)
|
||||
|
||||
288
src/server/database/SerieList.py
Normal file
288
src/server/database/SerieList.py
Normal file
@@ -0,0 +1,288 @@
|
||||
"""Utilities for loading and managing stored anime series metadata.
|
||||
|
||||
This module provides the SerieList class for managing collections of anime
|
||||
series metadata loaded from the database.
|
||||
|
||||
Note:
|
||||
This module is part of the server database layer. All persistence
|
||||
is handled by the service layer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from src.server.database.models import AnimeSeries
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SerieList:
|
||||
"""
|
||||
Represents the collection of cached series loaded from database.
|
||||
|
||||
Series are identified by their unique 'key' (provider identifier).
|
||||
The 'folder' is metadata only and not used for lookups.
|
||||
|
||||
This class manages in-memory series data loaded from database.
|
||||
|
||||
Example:
|
||||
# Load from database
|
||||
serie_list = SerieList("/path/to/anime")
|
||||
await serie_list.load_all_from_db()
|
||||
series = serie_list.get_all()
|
||||
|
||||
Attributes:
|
||||
directory: Path to the anime directory
|
||||
keyDict: Internal dictionary mapping serie.key to AnimeSeries objects
|
||||
"""
|
||||
|
||||
def __init__(self, base_path: str) -> None:
|
||||
"""Initialize the SerieList.
|
||||
|
||||
Args:
|
||||
base_path: Path to the anime directory
|
||||
"""
|
||||
self.directory: str = base_path
|
||||
# Internal storage using serie.key as the dictionary key
|
||||
self.keyDict: Dict[str, AnimeSeries] = {}
|
||||
|
||||
async def add_to_db(self, anime: AnimeSeries) -> bool:
|
||||
"""Persist a new series to the database.
|
||||
|
||||
Creates the filesystem folder using anime.folder, then persists
|
||||
the series metadata to the database.
|
||||
|
||||
Args:
|
||||
anime: The AnimeSeries instance to add
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
try:
|
||||
from src.server.database.connection import get_async_session_factory
|
||||
from src.server.database.service import AnimeSeriesService, EpisodeService
|
||||
|
||||
folder_name = anime.folder
|
||||
anime_path = self.directory + "/" + folder_name
|
||||
import os
|
||||
os.makedirs(anime_path, exist_ok=True)
|
||||
|
||||
session_factory = get_async_session_factory()
|
||||
db = session_factory()
|
||||
try:
|
||||
existing = await AnimeSeriesService.get_by_key(db, anime.key)
|
||||
if existing:
|
||||
logger.debug(
|
||||
"Series '%s' (key=%s) already exists in DB, skipping",
|
||||
anime.name, anime.key
|
||||
)
|
||||
return True
|
||||
|
||||
db_anime_series = await AnimeSeriesService.create(
|
||||
db=db,
|
||||
key=anime.key,
|
||||
name=anime.name,
|
||||
site=anime.site,
|
||||
folder=folder_name,
|
||||
year=anime.year
|
||||
)
|
||||
for ep in anime.episodes:
|
||||
await EpisodeService.create(
|
||||
db=db,
|
||||
series_id=db_anime_series.id,
|
||||
season=ep.season,
|
||||
episode_number=ep.episode_number
|
||||
)
|
||||
await db.commit()
|
||||
self.keyDict[anime.key] = anime
|
||||
logger.info(
|
||||
"Persisted series '%s' to database",
|
||||
anime.name
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
logger.error(
|
||||
"Failed to persist series '%s' to DB: %s",
|
||||
anime.key, e, exc_info=True
|
||||
)
|
||||
return False
|
||||
finally:
|
||||
await db.close()
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Could not add series '%s' to DB (DB unavailable?): %s",
|
||||
anime.key, e
|
||||
)
|
||||
return False
|
||||
|
||||
def contains(self, key: str) -> bool:
|
||||
"""
|
||||
Return True when a series identified by ``key`` already exists.
|
||||
|
||||
Args:
|
||||
key: The unique provider identifier for the series
|
||||
|
||||
Returns:
|
||||
True if the series exists in the collection
|
||||
"""
|
||||
return key in self.keyDict
|
||||
|
||||
def GetMissingEpisode(self) -> List[AnimeSeries]:
|
||||
"""Return all series that still contain missing episodes."""
|
||||
return [
|
||||
anime for anime in self.keyDict.values()
|
||||
if anime.episodeDict
|
||||
]
|
||||
|
||||
def get_missing_episodes(self) -> List[AnimeSeries]:
|
||||
"""PEP8-friendly alias for :meth:`GetMissingEpisode`."""
|
||||
return self.GetMissingEpisode()
|
||||
|
||||
def GetList(self) -> List[AnimeSeries]:
|
||||
"""Return all series instances stored in the list."""
|
||||
return list(self.keyDict.values())
|
||||
|
||||
def get_all(self) -> List[AnimeSeries]:
|
||||
"""PEP8-friendly alias for :meth:`GetList`."""
|
||||
return self.GetList()
|
||||
|
||||
def get_by_key(self, key: str) -> Optional[AnimeSeries]:
|
||||
"""
|
||||
Get a series by its unique provider key.
|
||||
|
||||
This is the primary method for series lookup.
|
||||
|
||||
Args:
|
||||
key: The unique provider identifier (e.g., "attack-on-titan")
|
||||
|
||||
Returns:
|
||||
The AnimeSeries instance if found, None otherwise
|
||||
"""
|
||||
return self.keyDict.get(key)
|
||||
|
||||
def get_by_folder(self, folder: str) -> Optional[AnimeSeries]:
|
||||
"""
|
||||
Get a series by its folder name.
|
||||
|
||||
.. deprecated:: 2.0.0
|
||||
Use :meth:`get_by_key` instead. Folder-based lookups will be
|
||||
removed in version 3.0.0. The `folder` field is metadata only
|
||||
and should not be used for identification.
|
||||
|
||||
This method is provided for backward compatibility only.
|
||||
Prefer using get_by_key() for new code.
|
||||
|
||||
Args:
|
||||
folder: The filesystem folder name (e.g., "Attack on Titan (2013)")
|
||||
|
||||
Returns:
|
||||
The AnimeSeries instance if found, None otherwise
|
||||
"""
|
||||
import warnings
|
||||
warnings.warn(
|
||||
"get_by_folder() is deprecated and will be removed in v3.0.0. "
|
||||
"Use get_by_key() instead. The 'folder' field is metadata only.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
for anime in self.keyDict.values():
|
||||
if anime.folder == folder:
|
||||
return anime
|
||||
return None
|
||||
|
||||
async def load_all_from_db(self) -> int:
|
||||
"""Load all series from database into in-memory cache.
|
||||
|
||||
Retrieves all anime series from the database with their episodes
|
||||
and populates the in-memory keyDict for fast access.
|
||||
|
||||
Returns:
|
||||
int: Number of series loaded into cache
|
||||
"""
|
||||
from src.server.database.connection import get_async_session_factory
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
|
||||
try:
|
||||
session_factory = get_async_session_factory()
|
||||
db = session_factory()
|
||||
try:
|
||||
anime_series_list = await AnimeSeriesService.get_all(
|
||||
db, with_episodes=True
|
||||
)
|
||||
|
||||
count = 0
|
||||
for anime_series in anime_series_list:
|
||||
self.keyDict[anime_series.key] = anime_series
|
||||
count += 1
|
||||
|
||||
logger.info(
|
||||
"Loaded %d series from database into in-memory cache",
|
||||
count
|
||||
)
|
||||
return count
|
||||
finally:
|
||||
await db.close()
|
||||
except RuntimeError:
|
||||
logger.warning(
|
||||
"Database not available, skipping DB load"
|
||||
)
|
||||
return 0
|
||||
|
||||
async def _load_single_series_from_db(
|
||||
self,
|
||||
anime_folder: str
|
||||
) -> Optional[AnimeSeries]:
|
||||
"""Load a single series from database by folder name.
|
||||
|
||||
Looks up a series in the database by its folder name and adds
|
||||
it to the in-memory cache.
|
||||
|
||||
Args:
|
||||
anime_folder: The filesystem folder name to look up
|
||||
|
||||
Returns:
|
||||
AnimeSeries if found and loaded, None otherwise
|
||||
"""
|
||||
from src.server.database.connection import get_async_session_factory
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
|
||||
try:
|
||||
session_factory = get_async_session_factory()
|
||||
db = session_factory()
|
||||
try:
|
||||
anime_series = await AnimeSeriesService.get_by_folder(
|
||||
db, anime_folder
|
||||
)
|
||||
if not anime_series:
|
||||
logger.debug(
|
||||
"Series with folder '%s' not found in DB",
|
||||
anime_folder
|
||||
)
|
||||
return None
|
||||
|
||||
self.keyDict[anime_series.key] = anime_series
|
||||
logger.debug(
|
||||
"Loaded series '%s' (key=%s) from DB",
|
||||
anime_series.name, anime_series.key
|
||||
)
|
||||
return anime_series
|
||||
finally:
|
||||
await db.close()
|
||||
except RuntimeError:
|
||||
logger.warning(
|
||||
"Database not available, cannot load series '%s'",
|
||||
anime_folder
|
||||
)
|
||||
return None
|
||||
|
||||
def invalidate_cache(self) -> None:
|
||||
"""Clear the in-memory cache.
|
||||
|
||||
Use after database modifications to force reload from DB
|
||||
on next access.
|
||||
"""
|
||||
self.keyDict.clear()
|
||||
logger.debug("SerieList in-memory cache invalidated")
|
||||
@@ -48,6 +48,7 @@ from src.server.database.service import (
|
||||
EpisodeService,
|
||||
UserSessionService,
|
||||
)
|
||||
from src.server.database.SerieList import SerieList
|
||||
from src.server.database.system_settings_service import SystemSettingsService
|
||||
|
||||
__all__ = [
|
||||
@@ -79,4 +80,6 @@ __all__ = [
|
||||
"DownloadQueueService",
|
||||
"SystemSettingsService",
|
||||
"UserSessionService",
|
||||
# SerieList
|
||||
"SerieList",
|
||||
]
|
||||
|
||||
@@ -27,7 +27,7 @@ logger = logging.getLogger(__name__)
|
||||
# Schema Version Constants
|
||||
# =============================================================================
|
||||
|
||||
CURRENT_SCHEMA_VERSION = "1.0.0"
|
||||
CURRENT_SCHEMA_VERSION = "1.0.1"
|
||||
SCHEMA_VERSION_TABLE = "schema_version"
|
||||
|
||||
# Expected tables in the current schema
|
||||
@@ -119,6 +119,11 @@ async def initialize_database(
|
||||
result["tables_created"] = tables
|
||||
logger.info("Created %s tables", len(tables))
|
||||
|
||||
# Migrate schema if needed (add missing columns to existing tables)
|
||||
migrations = await migrate_schema_if_needed(engine)
|
||||
if migrations:
|
||||
logger.info("Applied %s schema migrations", len(migrations))
|
||||
|
||||
# Validate schema if requested
|
||||
if validate_schema:
|
||||
validation = await validate_database_schema(engine)
|
||||
@@ -305,6 +310,66 @@ async def validate_database_schema(
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Schema Migration
|
||||
# =============================================================================
|
||||
|
||||
|
||||
async def migrate_schema_if_needed(
|
||||
engine: Optional[AsyncEngine] = None
|
||||
) -> List[str]:
|
||||
"""Migrate database schema to current version if needed.
|
||||
|
||||
Handles adding missing columns to existing tables for backward
|
||||
compatibility with older database schemas.
|
||||
|
||||
Args:
|
||||
engine: Optional database engine (uses default if not provided)
|
||||
|
||||
Returns:
|
||||
List of migration operations performed
|
||||
"""
|
||||
if engine is None:
|
||||
engine = get_engine()
|
||||
|
||||
migrations_applied = []
|
||||
|
||||
try:
|
||||
async with engine.connect() as conn:
|
||||
# Get existing columns in system_settings table
|
||||
existing_columns = await conn.run_sync(
|
||||
lambda sync_conn: [
|
||||
col["name"]
|
||||
for col in inspect(sync_conn).get_columns("system_settings")
|
||||
]
|
||||
)
|
||||
|
||||
# Migration: Add legacy_key_cleanup_completed column if missing
|
||||
if "legacy_key_cleanup_completed" not in existing_columns:
|
||||
logger.info(
|
||||
"Migrating system_settings table: "
|
||||
"adding legacy_key_cleanup_completed column"
|
||||
)
|
||||
await conn.execute(
|
||||
text("""
|
||||
ALTER TABLE system_settings
|
||||
ADD COLUMN legacy_key_cleanup_completed BOOLEAN
|
||||
NOT NULL DEFAULT 0
|
||||
""")
|
||||
)
|
||||
migrations_applied.append("added legacy_key_cleanup_completed")
|
||||
logger.info(
|
||||
"Migration complete: added legacy_key_cleanup_completed column"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Schema migration failed: %s", e)
|
||||
# Don't raise - migration failures shouldn't block startup
|
||||
# The missing column will be handled gracefully by the application
|
||||
|
||||
return migrations_applied
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Schema Version Management
|
||||
# =============================================================================
|
||||
@@ -319,7 +384,7 @@ async def get_schema_version(engine: Optional[AsyncEngine] = None) -> str:
|
||||
engine: Optional database engine (uses default if not provided)
|
||||
|
||||
Returns:
|
||||
Schema version string (e.g., "1.0.0", "empty", "unknown")
|
||||
Schema version string (e.g., "1.0.1", "empty", "unknown")
|
||||
"""
|
||||
if engine is None:
|
||||
engine = get_engine()
|
||||
|
||||
@@ -15,7 +15,7 @@ from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import List, Optional
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, String, Text, func
|
||||
from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, String, Text, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
||||
|
||||
from src.server.database.base import Base, TimestampMixin
|
||||
@@ -83,6 +83,10 @@ class AnimeSeries(Base, TimestampMixin):
|
||||
Boolean, nullable=False, default=False, server_default="0",
|
||||
doc="Whether tvshow.nfo file exists for this series"
|
||||
)
|
||||
nfo_path: Mapped[Optional[str]] = mapped_column(
|
||||
String(1000), nullable=True,
|
||||
doc="Path to the tvshow.nfo metadata file"
|
||||
)
|
||||
nfo_created_at: Mapped[Optional[datetime]] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True,
|
||||
doc="Timestamp when NFO was first created"
|
||||
@@ -91,6 +95,7 @@ class AnimeSeries(Base, TimestampMixin):
|
||||
DateTime(timezone=True), nullable=True,
|
||||
doc="Timestamp when NFO was last updated"
|
||||
)
|
||||
# TMDB (The Movie Database) ID for series metadata
|
||||
tmdb_id: Mapped[Optional[int]] = mapped_column(
|
||||
Integer, nullable=True, index=True,
|
||||
doc="TMDB (The Movie Database) ID for series metadata"
|
||||
@@ -185,6 +190,54 @@ class AnimeSeries(Base, TimestampMixin):
|
||||
f"name='{self.name}')>"
|
||||
)
|
||||
|
||||
@property
|
||||
def episodeDict(self) -> dict[int, list[int]]:
|
||||
"""Build episode dictionary from episodes relationship or private cache.
|
||||
|
||||
Returns:
|
||||
Dictionary mapping season numbers to lists of episode numbers
|
||||
"""
|
||||
# Check for private cache first (set when loading from JSON without DB)
|
||||
if hasattr(self, '_episode_dict_cache') and self._episode_dict_cache is not None:
|
||||
return self._episode_dict_cache
|
||||
|
||||
episode_dict: dict[int, list[int]] = {}
|
||||
if self.episodes:
|
||||
for ep in self.episodes:
|
||||
season = ep.season or 1
|
||||
if season not in episode_dict:
|
||||
episode_dict[season] = []
|
||||
episode_dict[season].append(ep.episode_number or 0)
|
||||
return episode_dict
|
||||
|
||||
@property
|
||||
def name_with_year(self) -> str:
|
||||
"""Get series name with year appended if available.
|
||||
|
||||
Returns:
|
||||
Name in format "Name (Year)" if year is available, else just name
|
||||
"""
|
||||
if self.year:
|
||||
import re
|
||||
year_suffix = f" ({self.year})"
|
||||
clean_name = re.sub(r'(\s*\(\d{4}\))+\s*$', '', self.name or '').strip()
|
||||
return f"{clean_name}{year_suffix}"
|
||||
return self.name or ''
|
||||
|
||||
@property
|
||||
def sanitized_folder(self) -> str:
|
||||
"""Get filesystem-safe folder name from display name with year.
|
||||
|
||||
Returns:
|
||||
Sanitized folder name based on display name with year
|
||||
"""
|
||||
from src.server.utils.filesystem import sanitize_folder_name
|
||||
name_to_sanitize = self.name_with_year or self.folder or self.key
|
||||
try:
|
||||
return sanitize_folder_name(name_to_sanitize)
|
||||
except ValueError:
|
||||
return sanitize_folder_name(self.key)
|
||||
|
||||
|
||||
class Episode(Base, TimestampMixin):
|
||||
"""SQLAlchemy model for anime episodes.
|
||||
@@ -316,6 +369,7 @@ class DownloadQueueItem(Base, TimestampMixin):
|
||||
id: Primary key
|
||||
series_id: Foreign key to AnimeSeries
|
||||
episode_id: Foreign key to Episode
|
||||
status: Queue status (pending/downloading/completed/failed/permanently_failed)
|
||||
error_message: Error description if failed
|
||||
download_url: Provider download URL
|
||||
file_destination: Target file path
|
||||
@@ -347,6 +401,33 @@ class DownloadQueueItem(Base, TimestampMixin):
|
||||
index=True
|
||||
)
|
||||
|
||||
# Status column to track queue item state
|
||||
# Allows distinguishing pending items from permanently failed ones
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(50), nullable=False, default="pending",
|
||||
doc="Queue item status: pending, downloading, completed, failed, permanently_failed"
|
||||
)
|
||||
|
||||
# Retry count to track failed download attempts
|
||||
# Used to determine when to move item to permanently_failed
|
||||
retry_count: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=0,
|
||||
doc="Number of retry attempts for this download"
|
||||
)
|
||||
|
||||
# Unique constraint to prevent duplicate pending queue items per episode
|
||||
# An episode can only have one PENDING entry at a time
|
||||
# The status column allows failed items to remain in DB while new
|
||||
# pending items can be added (application-level dedup still required)
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"ix_download_queue_episode_status",
|
||||
"episode_id",
|
||||
"status",
|
||||
unique=True,
|
||||
),
|
||||
)
|
||||
|
||||
# Error handling
|
||||
error_message: Mapped[Optional[str]] = mapped_column(
|
||||
Text, nullable=True,
|
||||
@@ -545,6 +626,96 @@ class UserSession(Base, TimestampMixin):
|
||||
self.is_active = False
|
||||
|
||||
|
||||
class UnresolvedFolder(Base, TimestampMixin):
|
||||
"""SQLAlchemy model for folders that couldn't be resolved during setup.
|
||||
|
||||
Tracks anime folders whose provider key couldn't be auto-resolved
|
||||
during the initial setup scan. Users can provide the correct key
|
||||
via the API to complete the series registration.
|
||||
|
||||
Attributes:
|
||||
id: Primary key
|
||||
folder_name: Original filesystem folder name
|
||||
title: Extracted title from folder name
|
||||
year: Extracted release year (optional)
|
||||
provider_key: User-provided provider key to resolve this folder
|
||||
search_attempts: Number of auto-search attempts made
|
||||
last_search_result: Cached search results (JSON string) for UI suggestions
|
||||
resolved_at: Timestamp when provider_key was provided
|
||||
created_at: Creation timestamp (from TimestampMixin)
|
||||
updated_at: Last update timestamp (from TimestampMixin)
|
||||
"""
|
||||
__tablename__ = "unresolved_folders"
|
||||
|
||||
# Primary key
|
||||
id: Mapped[int] = mapped_column(
|
||||
Integer, primary_key=True, autoincrement=True
|
||||
)
|
||||
|
||||
# Folder metadata
|
||||
folder_name: Mapped[str] = mapped_column(
|
||||
String(1000), unique=True, nullable=False, index=True,
|
||||
doc="Original filesystem folder name"
|
||||
)
|
||||
title: Mapped[str] = mapped_column(
|
||||
String(500), nullable=False,
|
||||
doc="Extracted title from folder name"
|
||||
)
|
||||
year: Mapped[Optional[int]] = mapped_column(
|
||||
Integer, nullable=True,
|
||||
doc="Extracted release year"
|
||||
)
|
||||
|
||||
# Resolution data
|
||||
provider_key: Mapped[Optional[str]] = mapped_column(
|
||||
String(255), nullable=True,
|
||||
doc="User-provided provider key to resolve this folder"
|
||||
)
|
||||
search_attempts: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=0, server_default="0",
|
||||
doc="Number of auto-search attempts made"
|
||||
)
|
||||
last_search_result: Mapped[Optional[str]] = mapped_column(
|
||||
Text, nullable=True,
|
||||
doc="Cached search results (JSON) for UI display"
|
||||
)
|
||||
resolved_at: Mapped[Optional[datetime]] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True,
|
||||
doc="Timestamp when this folder was resolved"
|
||||
)
|
||||
|
||||
@validates('folder_name')
|
||||
def validate_folder_name(self, key: str, value: str) -> str:
|
||||
"""Validate folder name is not empty."""
|
||||
if not value or not value.strip():
|
||||
raise ValueError("Folder name cannot be empty")
|
||||
if len(value) > 1000:
|
||||
raise ValueError("Folder name must be 1000 characters or less")
|
||||
return value.strip()
|
||||
|
||||
@validates('title')
|
||||
def validate_title(self, key: str, value: str) -> str:
|
||||
"""Validate title is not empty."""
|
||||
if not value or not value.strip():
|
||||
raise ValueError("Title cannot be empty")
|
||||
if len(value) > 500:
|
||||
raise ValueError("Title must be 500 characters or less")
|
||||
return value.strip()
|
||||
|
||||
@property
|
||||
def is_resolved(self) -> bool:
|
||||
"""Check if this folder has been resolved with a provider key."""
|
||||
return self.provider_key is not None and self.resolved_at is not None
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"<UnresolvedFolder(id={self.id}, "
|
||||
f"folder_name='{self.folder_name}', "
|
||||
f"title='{self.title}', "
|
||||
f"resolved={self.is_resolved})>"
|
||||
)
|
||||
|
||||
|
||||
class SystemSettings(Base, TimestampMixin):
|
||||
"""SQLAlchemy model for system-wide settings and state.
|
||||
|
||||
@@ -580,6 +751,14 @@ class SystemSettings(Base, TimestampMixin):
|
||||
Boolean, nullable=False, default=False, server_default="0",
|
||||
doc="Whether the initial media scan has been completed"
|
||||
)
|
||||
migration_legacy_files_completed: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="0",
|
||||
doc="Whether legacy key/data file migration has been completed"
|
||||
)
|
||||
legacy_key_cleanup_completed: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="0",
|
||||
doc="Whether legacy key file cleanup has been completed"
|
||||
)
|
||||
last_scan_timestamp: Mapped[Optional[datetime]] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True,
|
||||
doc="Timestamp of the last completed scan"
|
||||
|
||||
@@ -34,6 +34,7 @@ from src.server.database.models import (
|
||||
AnimeSeries,
|
||||
DownloadQueueItem,
|
||||
Episode,
|
||||
UnresolvedFolder,
|
||||
UserSession,
|
||||
)
|
||||
|
||||
@@ -70,6 +71,10 @@ class AnimeSeriesService:
|
||||
logo_loaded: bool = False,
|
||||
images_loaded: bool = False,
|
||||
loading_started_at: datetime | None = None,
|
||||
has_nfo: bool = False,
|
||||
nfo_path: str | None = None,
|
||||
nfo_created_at: datetime | None = None,
|
||||
nfo_updated_at: datetime | None = None,
|
||||
) -> AnimeSeries:
|
||||
"""Create a new anime series.
|
||||
|
||||
@@ -85,6 +90,10 @@ class AnimeSeriesService:
|
||||
logo_loaded: Whether logo is loaded (default: False)
|
||||
images_loaded: Whether images are loaded (default: False)
|
||||
loading_started_at: When loading started (optional)
|
||||
has_nfo: Whether tvshow.nfo exists (default: False)
|
||||
nfo_path: Path to tvshow.nfo file (optional)
|
||||
nfo_created_at: When NFO file was created (optional)
|
||||
nfo_updated_at: When NFO file was last updated (optional)
|
||||
|
||||
Returns:
|
||||
Created AnimeSeries instance
|
||||
@@ -103,6 +112,10 @@ class AnimeSeriesService:
|
||||
logo_loaded=logo_loaded,
|
||||
images_loaded=images_loaded,
|
||||
loading_started_at=loading_started_at,
|
||||
has_nfo=has_nfo,
|
||||
nfo_path=nfo_path,
|
||||
nfo_created_at=nfo_created_at,
|
||||
nfo_updated_at=nfo_updated_at,
|
||||
)
|
||||
db.add(series)
|
||||
await db.flush()
|
||||
@@ -148,7 +161,47 @@ class AnimeSeriesService:
|
||||
select(AnimeSeries).where(AnimeSeries.key == key)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
@staticmethod
|
||||
def get_by_folder_sync(db: Session, folder: str) -> Optional[AnimeSeries]:
|
||||
"""Look up an anime series by its filesystem folder name (sync).
|
||||
|
||||
Intended as a fallback for ``SerieScanner`` when neither a ``key``
|
||||
file nor a ``data`` file exists on disk for a given folder.
|
||||
|
||||
Args:
|
||||
db: Synchronous database session (from ``get_sync_session``).
|
||||
folder: Filesystem folder name to match (e.g.
|
||||
``"Rooster Fighter (2026)"``).
|
||||
|
||||
Returns:
|
||||
``AnimeSeries`` instance or ``None`` if not found.
|
||||
"""
|
||||
result = db.execute(
|
||||
select(AnimeSeries).where(AnimeSeries.folder == folder)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
@staticmethod
|
||||
async def get_by_folder(db: AsyncSession, folder: str) -> Optional[AnimeSeries]:
|
||||
"""Look up an anime series by its filesystem folder name (async).
|
||||
|
||||
Intended as primary lookup for ``SerieScanner`` when scanning
|
||||
directories, replacing the legacy file-based lookups (key/data files).
|
||||
|
||||
Args:
|
||||
db: Async database session.
|
||||
folder: Filesystem folder name to match (e.g.
|
||||
``"Rooster Fighter (2026)"``).
|
||||
|
||||
Returns:
|
||||
``AnimeSeries`` instance or ``None`` if not found.
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(AnimeSeries).where(AnimeSeries.folder == folder)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
@staticmethod
|
||||
async def get_all(
|
||||
db: AsyncSession,
|
||||
@@ -521,6 +574,7 @@ class EpisodeService:
|
||||
db: AsyncSession,
|
||||
series_id: int,
|
||||
season: Optional[int] = None,
|
||||
only_missing: bool = False,
|
||||
) -> List[Episode]:
|
||||
"""Get episodes for a series.
|
||||
|
||||
@@ -528,6 +582,9 @@ class EpisodeService:
|
||||
db: Database session
|
||||
series_id: Foreign key to AnimeSeries
|
||||
season: Optional season filter
|
||||
only_missing: If True, only return episodes where
|
||||
is_downloaded is False (i.e., missing episodes).
|
||||
Default False returns all episodes.
|
||||
|
||||
Returns:
|
||||
List of Episode instances
|
||||
@@ -537,6 +594,9 @@ class EpisodeService:
|
||||
if season is not None:
|
||||
query = query.where(Episode.season == season)
|
||||
|
||||
if only_missing:
|
||||
query = query.where(Episode.is_downloaded == False)
|
||||
|
||||
query = query.order_by(Episode.season, Episode.episode_number)
|
||||
result = await db.execute(query)
|
||||
return list(result.scalars().all())
|
||||
@@ -602,11 +662,11 @@ class EpisodeService:
|
||||
@staticmethod
|
||||
async def delete(db: AsyncSession, episode_id: int) -> bool:
|
||||
"""Delete episode.
|
||||
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
episode_id: Episode primary key
|
||||
|
||||
|
||||
Returns:
|
||||
True if deleted, False if not found
|
||||
"""
|
||||
@@ -615,6 +675,33 @@ class EpisodeService:
|
||||
)
|
||||
return result.rowcount > 0
|
||||
|
||||
@staticmethod
|
||||
async def delete_by_series(
|
||||
db: AsyncSession,
|
||||
series_id: int,
|
||||
season: int,
|
||||
episode_number: int,
|
||||
) -> bool:
|
||||
"""Delete episode by series ID, season, and episode number.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
series_id: Foreign key to AnimeSeries
|
||||
season: Season number
|
||||
episode_number: Episode number within season
|
||||
|
||||
Returns:
|
||||
True if deleted, False if not found
|
||||
"""
|
||||
result = await db.execute(
|
||||
delete(Episode).where(
|
||||
Episode.series_id == series_id,
|
||||
Episode.season == season,
|
||||
Episode.episode_number == episode_number,
|
||||
)
|
||||
)
|
||||
return result.rowcount > 0
|
||||
|
||||
@staticmethod
|
||||
async def delete_by_series_and_episode(
|
||||
db: AsyncSession,
|
||||
@@ -728,6 +815,8 @@ class DownloadQueueService:
|
||||
episode_id: int,
|
||||
download_url: Optional[str] = None,
|
||||
file_destination: Optional[str] = None,
|
||||
status: str = "pending",
|
||||
retry_count: int = 0,
|
||||
) -> DownloadQueueItem:
|
||||
"""Add item to download queue.
|
||||
|
||||
@@ -737,6 +826,8 @@ class DownloadQueueService:
|
||||
episode_id: Foreign key to Episode
|
||||
download_url: Optional provider download URL
|
||||
file_destination: Optional target file path
|
||||
status: Queue item status (default: "pending")
|
||||
retry_count: Number of retry attempts (default: 0)
|
||||
|
||||
Returns:
|
||||
Created DownloadQueueItem instance
|
||||
@@ -746,13 +837,15 @@ class DownloadQueueService:
|
||||
episode_id=episode_id,
|
||||
download_url=download_url,
|
||||
file_destination=file_destination,
|
||||
status=status,
|
||||
retry_count=retry_count,
|
||||
)
|
||||
db.add(item)
|
||||
await db.flush()
|
||||
await db.refresh(item)
|
||||
logger.info(
|
||||
f"Added to download queue: episode_id={episode_id} "
|
||||
f"for series_id={series_id}"
|
||||
f"for series_id={series_id}, status={status}"
|
||||
)
|
||||
return item
|
||||
|
||||
@@ -779,21 +872,24 @@ class DownloadQueueService:
|
||||
async def get_by_episode(
|
||||
db: AsyncSession,
|
||||
episode_id: int,
|
||||
status_filter: Optional[str] = None,
|
||||
) -> Optional[DownloadQueueItem]:
|
||||
"""Get download queue item by episode ID.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
episode_id: Foreign key to Episode
|
||||
status_filter: Optional status to filter by (e.g., "pending")
|
||||
|
||||
Returns:
|
||||
DownloadQueueItem instance or None if not found
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(DownloadQueueItem).where(
|
||||
DownloadQueueItem.episode_id == episode_id
|
||||
)
|
||||
query = select(DownloadQueueItem).where(
|
||||
DownloadQueueItem.episode_id == episode_id
|
||||
)
|
||||
if status_filter:
|
||||
query = query.where(DownloadQueueItem.status == status_filter)
|
||||
result = await db.execute(query)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
@staticmethod
|
||||
@@ -853,6 +949,95 @@ class DownloadQueueService:
|
||||
logger.debug("Set error on download queue item %s", item_id)
|
||||
return item
|
||||
|
||||
@staticmethod
|
||||
async def set_status(
|
||||
db: AsyncSession,
|
||||
item_id: int,
|
||||
status: str,
|
||||
) -> Optional[DownloadQueueItem]:
|
||||
"""Set status on download queue item.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
item_id: Item primary key
|
||||
status: New status value
|
||||
|
||||
Returns:
|
||||
Updated DownloadQueueItem instance or None if not found
|
||||
"""
|
||||
item = await DownloadQueueService.get_by_id(db, item_id)
|
||||
if not item:
|
||||
return None
|
||||
|
||||
item.status = status
|
||||
|
||||
await db.flush()
|
||||
await db.refresh(item)
|
||||
logger.debug("Set status on download queue item %s to %s", item_id, status)
|
||||
return item
|
||||
|
||||
@staticmethod
|
||||
async def increment_retry_count(
|
||||
db: AsyncSession,
|
||||
item_id: int,
|
||||
) -> Optional[DownloadQueueItem]:
|
||||
"""Increment retry count on download queue item.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
item_id: Item primary key
|
||||
|
||||
Returns:
|
||||
Updated DownloadQueueItem instance or None if not found
|
||||
"""
|
||||
item = await DownloadQueueService.get_by_id(db, item_id)
|
||||
if not item:
|
||||
return None
|
||||
|
||||
item.retry_count += 1
|
||||
|
||||
await db.flush()
|
||||
await db.refresh(item)
|
||||
logger.debug(
|
||||
"Incremented retry count on download queue item %s to %s",
|
||||
item_id, item.retry_count
|
||||
)
|
||||
return item
|
||||
|
||||
@staticmethod
|
||||
async def set_status_and_error(
|
||||
db: AsyncSession,
|
||||
item_id: int,
|
||||
status: str,
|
||||
error_message: Optional[str] = None,
|
||||
) -> Optional[DownloadQueueItem]:
|
||||
"""Set status and error message on download queue item atomically.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
item_id: Item primary key
|
||||
status: New status value
|
||||
error_message: Optional error description
|
||||
|
||||
Returns:
|
||||
Updated DownloadQueueItem instance or None if not found
|
||||
"""
|
||||
item = await DownloadQueueService.get_by_id(db, item_id)
|
||||
if not item:
|
||||
return None
|
||||
|
||||
item.status = status
|
||||
if error_message is not None:
|
||||
item.error_message = error_message
|
||||
|
||||
await db.flush()
|
||||
await db.refresh(item)
|
||||
logger.debug(
|
||||
"Set status=%s on download queue item %s, error=%s",
|
||||
status, item_id, error_message
|
||||
)
|
||||
return item
|
||||
|
||||
@staticmethod
|
||||
async def delete(db: AsyncSession, item_id: int) -> bool:
|
||||
"""Delete download queue item.
|
||||
@@ -1180,3 +1365,176 @@ class UserSessionService:
|
||||
|
||||
return new_session
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Unresolved Folder Service
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class UnresolvedFolderService:
|
||||
"""Service for tracking and resolving folders that couldn't be auto-resolved.
|
||||
|
||||
During initial setup, some folders may not resolve to a provider key
|
||||
(no search match or multiple ambiguous matches). These are tracked as
|
||||
UnresolvedFolder records and can later be resolved by the user providing
|
||||
the correct provider key.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
async def create(
|
||||
db: AsyncSession,
|
||||
folder_name: str,
|
||||
title: str,
|
||||
year: int | None = None,
|
||||
search_attempts: int = 1,
|
||||
last_search_result: str | None = None,
|
||||
) -> UnresolvedFolder:
|
||||
"""Create a new unresolved folder tracking record.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
folder_name: Original filesystem folder name
|
||||
title: Extracted title from folder name
|
||||
year: Extracted release year (optional)
|
||||
search_attempts: Number of search attempts made (default: 1)
|
||||
last_search_result: JSON string of search results for UI (optional)
|
||||
|
||||
Returns:
|
||||
Created UnresolvedFolder instance
|
||||
"""
|
||||
folder = UnresolvedFolder(
|
||||
folder_name=folder_name,
|
||||
title=title,
|
||||
year=year,
|
||||
search_attempts=search_attempts,
|
||||
last_search_result=last_search_result,
|
||||
)
|
||||
db.add(folder)
|
||||
await db.flush()
|
||||
await db.refresh(folder)
|
||||
logger.info(
|
||||
"Created unresolved folder tracking: %s (title=%s, year=%s)",
|
||||
folder_name, title, year
|
||||
)
|
||||
return folder
|
||||
|
||||
@staticmethod
|
||||
async def get_by_folder_name(
|
||||
db: AsyncSession,
|
||||
folder_name: str,
|
||||
) -> Optional[UnresolvedFolder]:
|
||||
"""Get unresolved folder by folder name.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
folder_name: Filesystem folder name to look up
|
||||
|
||||
Returns:
|
||||
UnresolvedFolder instance or None if not found
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(UnresolvedFolder).where(
|
||||
UnresolvedFolder.folder_name == folder_name
|
||||
)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
@staticmethod
|
||||
async def get_all_unresolved(
|
||||
db: AsyncSession,
|
||||
) -> list[UnresolvedFolder]:
|
||||
"""Get all unresolved folders that haven't been resolved yet.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
List of unresolved UnresolvedFolder instances
|
||||
"""
|
||||
result = await db.execute(
|
||||
select(UnresolvedFolder)
|
||||
.where(UnresolvedFolder.provider_key.is_(None))
|
||||
.order_by(UnresolvedFolder.created_at)
|
||||
)
|
||||
return list(result.scalars().all())
|
||||
|
||||
@staticmethod
|
||||
async def resolve(
|
||||
db: AsyncSession,
|
||||
folder_name: str,
|
||||
provider_key: str,
|
||||
) -> Optional[UnresolvedFolder]:
|
||||
"""Mark an unresolved folder as resolved with the given provider key.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
folder_name: Filesystem folder name to resolve
|
||||
provider_key: Provider key to associate with this folder
|
||||
|
||||
Returns:
|
||||
Updated UnresolvedFolder instance or None if not found
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
folder = await UnresolvedFolderService.get_by_folder_name(db, folder_name)
|
||||
if not folder:
|
||||
return None
|
||||
|
||||
folder.provider_key = provider_key
|
||||
folder.resolved_at = datetime.now(timezone.utc)
|
||||
await db.flush()
|
||||
await db.refresh(folder)
|
||||
logger.info(
|
||||
"Resolved unresolved folder: %s -> key=%s",
|
||||
folder_name, provider_key
|
||||
)
|
||||
return folder
|
||||
|
||||
@staticmethod
|
||||
async def delete(
|
||||
db: AsyncSession,
|
||||
folder_name: str,
|
||||
) -> bool:
|
||||
"""Delete an unresolved folder record (e.g., after manual add).
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
folder_name: Filesystem folder name to delete
|
||||
|
||||
Returns:
|
||||
True if deleted, False if not found
|
||||
"""
|
||||
folder = await UnresolvedFolderService.get_by_folder_name(db, folder_name)
|
||||
if not folder:
|
||||
return False
|
||||
|
||||
await db.delete(folder)
|
||||
await db.flush()
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
async def update_search_result(
|
||||
db: AsyncSession,
|
||||
folder_name: str,
|
||||
search_result: str,
|
||||
) -> Optional[UnresolvedFolder]:
|
||||
"""Update the cached search result for an unresolved folder.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
folder_name: Filesystem folder name to update
|
||||
search_result: JSON string of search results
|
||||
|
||||
Returns:
|
||||
Updated UnresolvedFolder instance or None if not found
|
||||
"""
|
||||
folder = await UnresolvedFolderService.get_by_folder_name(db, folder_name)
|
||||
if not folder:
|
||||
return None
|
||||
|
||||
folder.search_attempts += 1
|
||||
folder.last_search_result = search_result
|
||||
await db.flush()
|
||||
await db.refresh(folder)
|
||||
return folder
|
||||
|
||||
|
||||
@@ -125,6 +125,66 @@ class SystemSettingsService:
|
||||
settings = await SystemSettingsService.get_or_create(db)
|
||||
return settings.initial_media_scan_completed
|
||||
|
||||
@staticmethod
|
||||
async def is_migration_legacy_files_completed(db: AsyncSession) -> bool:
|
||||
"""Check if legacy key/data file migration has been completed.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
True if legacy migration is completed, False otherwise
|
||||
"""
|
||||
settings = await SystemSettingsService.get_or_create(db)
|
||||
return settings.migration_legacy_files_completed
|
||||
|
||||
@staticmethod
|
||||
async def mark_migration_legacy_files_completed(
|
||||
db: AsyncSession,
|
||||
timestamp: Optional[datetime] = None
|
||||
) -> None:
|
||||
"""Mark the legacy key/data file migration as completed.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
timestamp: Optional timestamp to set, defaults to current time
|
||||
"""
|
||||
settings = await SystemSettingsService.get_or_create(db)
|
||||
settings.migration_legacy_files_completed = True
|
||||
settings.last_scan_timestamp = timestamp or datetime.now(timezone.utc)
|
||||
await db.commit()
|
||||
logger.info("Marked legacy files migration as completed")
|
||||
|
||||
@staticmethod
|
||||
async def is_legacy_key_cleanup_completed(db: AsyncSession) -> bool:
|
||||
"""Check if legacy key file cleanup has been completed.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
True if cleanup is completed, False otherwise
|
||||
"""
|
||||
settings = await SystemSettingsService.get_or_create(db)
|
||||
return settings.legacy_key_cleanup_completed
|
||||
|
||||
@staticmethod
|
||||
async def mark_legacy_key_cleanup_completed(
|
||||
db: AsyncSession,
|
||||
timestamp: Optional[datetime] = None
|
||||
) -> None:
|
||||
"""Mark the legacy key file cleanup as completed.
|
||||
|
||||
Args:
|
||||
db: Database session
|
||||
timestamp: Optional timestamp to set, defaults to current time
|
||||
"""
|
||||
settings = await SystemSettingsService.get_or_create(db)
|
||||
settings.legacy_key_cleanup_completed = True
|
||||
settings.last_scan_timestamp = timestamp or datetime.now(timezone.utc)
|
||||
await db.commit()
|
||||
logger.info("Marked legacy key file cleanup as completed")
|
||||
|
||||
@staticmethod
|
||||
async def mark_initial_media_scan_completed(
|
||||
db: AsyncSession,
|
||||
@@ -154,6 +214,8 @@ class SystemSettingsService:
|
||||
settings.initial_scan_completed = False
|
||||
settings.initial_nfo_scan_completed = False
|
||||
settings.initial_media_scan_completed = False
|
||||
settings.migration_legacy_files_completed = False
|
||||
settings.legacy_key_cleanup_completed = False
|
||||
settings.last_scan_timestamp = None
|
||||
await db.commit()
|
||||
logger.info("Reset all scan completion flags")
|
||||
|
||||
@@ -7,7 +7,7 @@ errors in provider operations with automatic retry mechanisms.
|
||||
|
||||
import functools
|
||||
import logging
|
||||
from typing import Any, Callable, TypeVar
|
||||
from typing import Any, Callable, Optional, TypeVar
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -42,41 +42,97 @@ class DownloadError(Exception):
|
||||
class RecoveryStrategies:
|
||||
"""Strategies for handling errors and recovering from failures."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_retries: int = 3,
|
||||
base_delay: float = 1.0,
|
||||
max_delay: float = 60.0,
|
||||
exponential_base: float = 2.0,
|
||||
) -> None:
|
||||
"""Initialize recovery strategies.
|
||||
|
||||
Args:
|
||||
max_retries: Maximum number of retry attempts.
|
||||
base_delay: Initial delay between retries in seconds.
|
||||
max_delay: Maximum delay between retries in seconds.
|
||||
exponential_base: Base for exponential backoff multiplier.
|
||||
"""
|
||||
self.max_retries = max_retries
|
||||
self.base_delay = base_delay
|
||||
self.max_delay = max_delay
|
||||
self.exponential_base = exponential_base
|
||||
|
||||
def _calculate_delay(self, attempt: int) -> float:
|
||||
"""Calculate delay for given retry attempt using exponential backoff.
|
||||
|
||||
Args:
|
||||
attempt: Zero-based retry attempt number.
|
||||
|
||||
Returns:
|
||||
Delay in seconds before next retry.
|
||||
"""
|
||||
delay = self.base_delay * (self.exponential_base ** attempt)
|
||||
return min(delay, self.max_delay)
|
||||
|
||||
@staticmethod
|
||||
def handle_network_failure(
|
||||
func: Callable, *args: Any, **kwargs: Any
|
||||
) -> Any:
|
||||
"""Handle network failures with basic retry logic."""
|
||||
"""Handle network failures with exponential backoff retry logic."""
|
||||
last_error: Optional[Exception] = None
|
||||
max_retries = 3
|
||||
base_delay = 1.0
|
||||
max_delay = 60.0
|
||||
exponential_base = 2.0
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except (NetworkError, ConnectionError):
|
||||
if attempt == max_retries - 1:
|
||||
raise
|
||||
logger.warning(
|
||||
"Network error on attempt %d, retrying...",
|
||||
attempt + 1,
|
||||
)
|
||||
continue
|
||||
except (NetworkError, ConnectionError, TimeoutError) as exc:
|
||||
last_error = exc
|
||||
if attempt < max_retries - 1:
|
||||
delay = base_delay * (exponential_base ** attempt)
|
||||
delay = min(delay, max_delay)
|
||||
logger.warning(
|
||||
"Network error on attempt %d/%d, retrying in %.1fs: %s",
|
||||
attempt + 1, max_retries, delay, exc
|
||||
)
|
||||
import time
|
||||
time.sleep(delay)
|
||||
continue
|
||||
if last_error:
|
||||
raise last_error
|
||||
raise NetworkError("Network failure after retries")
|
||||
|
||||
@staticmethod
|
||||
def handle_download_failure(
|
||||
func: Callable, *args: Any, **kwargs: Any
|
||||
) -> Any:
|
||||
"""Handle download failures with retry logic."""
|
||||
"""Handle download failures with exponential backoff retry logic."""
|
||||
last_error: Optional[Exception] = None
|
||||
max_retries = 2
|
||||
base_delay = 1.0
|
||||
max_delay = 60.0
|
||||
exponential_base = 2.0
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except DownloadError:
|
||||
if attempt == max_retries - 1:
|
||||
raise
|
||||
logger.warning(
|
||||
"Download error on attempt %d, retrying...",
|
||||
attempt + 1,
|
||||
)
|
||||
continue
|
||||
except DownloadError as exc:
|
||||
last_error = exc
|
||||
if attempt < max_retries - 1:
|
||||
delay = base_delay * (exponential_base ** attempt)
|
||||
delay = min(delay, max_delay)
|
||||
logger.warning(
|
||||
"Download error on attempt %d/%d, retrying in %.1fs: %s",
|
||||
attempt + 1, max_retries, delay, exc
|
||||
)
|
||||
import time
|
||||
time.sleep(delay)
|
||||
continue
|
||||
if last_error:
|
||||
raise last_error
|
||||
raise DownloadError("Download failed after retries")
|
||||
|
||||
|
||||
class FileCorruptionDetector:
|
||||
3
src/server/exceptions/exceptions/__init__.py
Normal file
3
src/server/exceptions/exceptions/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
from src.server.exceptions.exceptions.Exceptions import MatchNotFoundError, NoKeyFoundException
|
||||
|
||||
__all__ = ["MatchNotFoundError", "NoKeyFoundException"]
|
||||
@@ -27,6 +27,7 @@ from src.server.api.health import router as health_router
|
||||
from src.server.api.logging import router as logging_router
|
||||
from src.server.api.nfo import router as nfo_router
|
||||
from src.server.api.scheduler import router as scheduler_router
|
||||
from src.server.api.setup_endpoints import router as setup_router
|
||||
from src.server.api.websocket import router as websocket_router
|
||||
from src.server.controllers.error_controller import (
|
||||
not_found_handler,
|
||||
@@ -38,7 +39,6 @@ from src.server.controllers.page_controller import router as page_router
|
||||
from src.server.middleware.auth import AuthMiddleware
|
||||
from src.server.middleware.error_handler import register_exception_handlers
|
||||
from src.server.middleware.setup_redirect import SetupRedirectMiddleware
|
||||
from src.server.services.anime_service import sync_series_from_data_files
|
||||
from src.server.services.progress_service import get_progress_service
|
||||
from src.server.services.websocket_service import get_websocket_service
|
||||
|
||||
@@ -104,6 +104,107 @@ async def _check_incomplete_series_on_startup(background_loader) -> None:
|
||||
logger.exception("Failed to check incomplete series on startup")
|
||||
|
||||
|
||||
async def _run_startup_health_checks(logger) -> dict:
|
||||
"""Run startup health checks for critical dependencies.
|
||||
|
||||
Checks:
|
||||
- ffmpeg availability
|
||||
- DNS resolution for aniworld.to and api.themoviedb.org
|
||||
- anime_directory configuration and writability
|
||||
|
||||
Args:
|
||||
logger: Logger instance for recording check results.
|
||||
|
||||
Returns:
|
||||
dict: Health check results with status and details for each check.
|
||||
"""
|
||||
import asyncio
|
||||
import shutil
|
||||
import socket
|
||||
from typing import Any, Dict
|
||||
|
||||
checks: Dict[str, Any] = {
|
||||
"ffmpeg": {"status": "unknown", "message": None},
|
||||
"dns_aniworld": {"status": "unknown", "message": None},
|
||||
"dns_tmdb": {"status": "unknown", "message": None},
|
||||
"anime_directory": {"status": "unknown", "message": None, "path": None},
|
||||
}
|
||||
|
||||
# Check ffmpeg availability
|
||||
try:
|
||||
ffmpeg_path = shutil.which("ffmpeg")
|
||||
if ffmpeg_path:
|
||||
checks["ffmpeg"]["status"] = "ok"
|
||||
checks["ffmpeg"]["message"] = f"Found at {ffmpeg_path}"
|
||||
logger.debug("ffmpeg health check passed: %s", ffmpeg_path)
|
||||
else:
|
||||
checks["ffmpeg"]["status"] = "warning"
|
||||
checks["ffmpeg"]["message"] = "ffmpeg not found in PATH"
|
||||
logger.warning("ffmpeg health check failed: not in PATH")
|
||||
except Exception as e:
|
||||
checks["ffmpeg"]["status"] = "error"
|
||||
checks["ffmpeg"]["message"] = str(e)
|
||||
logger.warning("Could not check ffmpeg: %s", e)
|
||||
|
||||
# Check DNS resolution for aniworld.to
|
||||
try:
|
||||
socket.gethostbyname("aniworld.to")
|
||||
checks["dns_aniworld"]["status"] = "ok"
|
||||
checks["dns_aniworld"]["message"] = "Resolved successfully"
|
||||
logger.debug("DNS health check passed for aniworld.to")
|
||||
except socket.gaierror as e:
|
||||
checks["dns_aniworld"]["status"] = "warning"
|
||||
checks["dns_aniworld"]["message"] = f"DNS resolution failed: {e}"
|
||||
logger.warning("DNS health check failed for aniworld.to: %s", e)
|
||||
except Exception as e:
|
||||
checks["dns_aniworld"]["status"] = "warning"
|
||||
checks["dns_aniworld"]["message"] = f"Unexpected error: {e}"
|
||||
logger.warning("Unexpected DNS error for aniworld.to: %s", e)
|
||||
|
||||
# Check DNS resolution for api.themoviedb.org
|
||||
try:
|
||||
socket.gethostbyname("api.themoviedb.org")
|
||||
checks["dns_tmdb"]["status"] = "ok"
|
||||
checks["dns_tmdb"]["message"] = "Resolved successfully"
|
||||
logger.debug("DNS health check passed for api.themoviedb.org")
|
||||
except socket.gaierror as e:
|
||||
checks["dns_tmdb"]["status"] = "warning"
|
||||
checks["dns_tmdb"]["message"] = f"DNS resolution failed: {e}"
|
||||
logger.warning("DNS health check failed for api.themoviedb.org: %s", e)
|
||||
except Exception as e:
|
||||
checks["dns_tmdb"]["status"] = "warning"
|
||||
checks["dns_tmdb"]["message"] = f"Unexpected error: {e}"
|
||||
logger.warning("Unexpected DNS error for api.themoviedb.org: %s", e)
|
||||
|
||||
# Check anime_directory configuration and writability
|
||||
from src.config.settings import settings
|
||||
anime_dir = settings.anime_directory
|
||||
|
||||
if not anime_dir:
|
||||
checks["anime_directory"]["status"] = "error"
|
||||
checks["anime_directory"]["message"] = "anime_directory not configured"
|
||||
checks["anime_directory"]["path"] = None
|
||||
logger.error("anime_directory health check failed: not configured")
|
||||
else:
|
||||
import os
|
||||
checks["anime_directory"]["path"] = anime_dir
|
||||
|
||||
if not os.path.isdir(anime_dir):
|
||||
checks["anime_directory"]["status"] = "error"
|
||||
checks["anime_directory"]["message"] = f"Directory does not exist: {anime_dir}"
|
||||
logger.error("anime_directory health check failed: %s does not exist", anime_dir)
|
||||
elif not os.access(anime_dir, os.W_OK):
|
||||
checks["anime_directory"]["status"] = "error"
|
||||
checks["anime_directory"]["message"] = f"Directory not writable: {anime_dir}"
|
||||
logger.error("anime_directory health check failed: %s not writable", anime_dir)
|
||||
else:
|
||||
checks["anime_directory"]["status"] = "ok"
|
||||
checks["anime_directory"]["message"] = f"Directory exists and is writable: {anime_dir}"
|
||||
logger.debug("anime_directory health check passed: %s", anime_dir)
|
||||
|
||||
return checks
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_application: FastAPI):
|
||||
"""Manage application lifespan (startup and shutdown).
|
||||
@@ -114,6 +215,7 @@ async def lifespan(_application: FastAPI):
|
||||
"""
|
||||
# Setup logging first with INFO level
|
||||
logger = setup_logging(log_level="INFO")
|
||||
logger.info("Starting FastAPI application v%s", APP_VERSION)
|
||||
|
||||
# Track successful initialization steps
|
||||
initialized = {
|
||||
@@ -242,7 +344,6 @@ async def lifespan(_application: FastAPI):
|
||||
from src.server.services.initialization_service import (
|
||||
perform_initial_setup,
|
||||
perform_media_scan_if_needed,
|
||||
perform_nfo_repair_scan,
|
||||
perform_nfo_scan_if_needed,
|
||||
)
|
||||
|
||||
@@ -298,30 +399,29 @@ async def lifespan(_application: FastAPI):
|
||||
except Exception as e:
|
||||
logger.warning("Failed to start background loader service: %s", e)
|
||||
|
||||
# Initialize and start scheduler service
|
||||
try:
|
||||
from src.server.services.scheduler_service import (
|
||||
get_scheduler_service,
|
||||
)
|
||||
scheduler_service = get_scheduler_service()
|
||||
await scheduler_service.start()
|
||||
initialized['scheduler'] = True
|
||||
logger.info("Scheduler service started")
|
||||
except Exception as e:
|
||||
logger.warning("Failed to start scheduler service: %s", e)
|
||||
# Continue - scheduler is optional
|
||||
|
||||
# Run media scan only on first run
|
||||
await perform_media_scan_if_needed(background_loader)
|
||||
|
||||
# Scan every series NFO on startup and repair any that are
|
||||
# missing required tags by queuing them for background reload
|
||||
await perform_nfo_repair_scan(background_loader)
|
||||
else:
|
||||
logger.info(
|
||||
"Download service initialization skipped - "
|
||||
"anime directory not configured"
|
||||
)
|
||||
|
||||
# Initialize and start scheduler service (independent of anime_directory)
|
||||
# The scheduler loads its own config from config.json and the
|
||||
# anime_directory may be configured there even if the env var is empty.
|
||||
try:
|
||||
logger.info("Initializing scheduler service...")
|
||||
from src.server.services.scheduler.scheduler_service import (
|
||||
get_scheduler_service,
|
||||
)
|
||||
scheduler_service = get_scheduler_service()
|
||||
logger.info("Scheduler service instance obtained, starting...")
|
||||
await scheduler_service.start()
|
||||
initialized['scheduler'] = True
|
||||
logger.info("Scheduler service started successfully")
|
||||
except Exception as e:
|
||||
logger.warning("Failed to start scheduler service: %s", e)
|
||||
except (OSError, RuntimeError, ValueError) as e:
|
||||
logger.warning("Failed to initialize services: %s", e)
|
||||
# Continue startup - services can be initialized later
|
||||
@@ -334,6 +434,27 @@ async def lifespan(_application: FastAPI):
|
||||
logger.info(
|
||||
"API documentation available at http://127.0.0.1:8000/api/docs"
|
||||
)
|
||||
|
||||
# Check for ffmpeg availability and warn if missing
|
||||
try:
|
||||
import shutil as _shutil
|
||||
if _shutil.which("ffmpeg") is None:
|
||||
logger.warning(
|
||||
"ffmpeg not found in PATH. HLS streams may fail to download. "
|
||||
"Install ffmpeg to enable HLS support."
|
||||
)
|
||||
else:
|
||||
logger.debug("ffmpeg found at: %s", _shutil.which("ffmpeg"))
|
||||
except Exception as _exc:
|
||||
logger.warning("Could not check for ffmpeg: %s", _exc)
|
||||
|
||||
# Run startup health checks and store results for /health endpoint
|
||||
try:
|
||||
startup_checks = await _run_startup_health_checks(logger)
|
||||
app.state.startup_checks = startup_checks
|
||||
except Exception as _exc:
|
||||
logger.warning("Could not run startup health checks: %s", _exc)
|
||||
app.state.startup_checks = {}
|
||||
except Exception as e:
|
||||
logger.error("Error during startup: %s", e, exc_info=True)
|
||||
startup_error = e
|
||||
@@ -378,7 +499,9 @@ async def lifespan(_application: FastAPI):
|
||||
# 1. Stop scheduler service (only if initialized)
|
||||
if initialized['scheduler']:
|
||||
try:
|
||||
from src.server.services.scheduler_service import get_scheduler_service
|
||||
from src.server.services.scheduler.scheduler_service import (
|
||||
get_scheduler_service,
|
||||
)
|
||||
scheduler_service = get_scheduler_service()
|
||||
logger.info("Stopping scheduler service...")
|
||||
await asyncio.wait_for(
|
||||
@@ -423,8 +546,8 @@ async def lifespan(_application: FastAPI):
|
||||
|
||||
# 4. Shutdown download service and persist active downloads
|
||||
try:
|
||||
from src.server.services.download_service import ( # noqa: E501
|
||||
_download_service_instance,
|
||||
from src.server.services.download_service import (
|
||||
_download_service_instance, # noqa: E501
|
||||
)
|
||||
if _download_service_instance is not None:
|
||||
logger.info("Stopping download service...")
|
||||
@@ -481,11 +604,13 @@ async def lifespan(_application: FastAPI):
|
||||
raise startup_error
|
||||
|
||||
|
||||
from src.server.utils.version import APP_VERSION
|
||||
|
||||
# Initialize FastAPI app with lifespan
|
||||
app = FastAPI(
|
||||
title="Aniworld Download Manager",
|
||||
description="Modern web interface for Aniworld anime download management",
|
||||
version="1.0.0",
|
||||
version=APP_VERSION,
|
||||
docs_url="/api/docs",
|
||||
redoc_url="/api/redoc",
|
||||
lifespan=lifespan
|
||||
@@ -524,6 +649,7 @@ app.include_router(scheduler_router)
|
||||
app.include_router(anime_router)
|
||||
app.include_router(download_router)
|
||||
app.include_router(nfo_router)
|
||||
app.include_router(setup_router)
|
||||
app.include_router(logging_router)
|
||||
app.include_router(websocket_router)
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ class SetupRedirectMiddleware(BaseHTTPMiddleware):
|
||||
# Paths that should always be accessible, even without setup
|
||||
EXEMPT_PATHS = {
|
||||
"/setup", # Setup page itself
|
||||
"/setup/unresolved", # Unresolved folders page (after setup)
|
||||
"/loading", # Loading page (initialization progress)
|
||||
"/login", # Login page (needs to be accessible after setup)
|
||||
"/queue", # Queue page (for initial load)
|
||||
|
||||
@@ -83,6 +83,30 @@ class AnimeSeriesResponse(BaseModel):
|
||||
return v
|
||||
|
||||
|
||||
class AnimeMetadataUpdate(BaseModel):
|
||||
"""Request model for updating anime metadata (key, tmdb_id, tvdb_id)."""
|
||||
|
||||
key: Optional[str] = Field(None, description="New series key (URL-safe, lowercase)")
|
||||
tmdb_id: Optional[int] = Field(None, ge=1, description="TMDB ID (positive integer)")
|
||||
tvdb_id: Optional[int] = Field(None, ge=1, description="TVDB ID (positive integer)")
|
||||
|
||||
@field_validator('key', mode='before')
|
||||
@classmethod
|
||||
def validate_key_format(cls, v: Optional[str]) -> Optional[str]:
|
||||
"""Validate key is URL-safe lowercase with hyphens only."""
|
||||
if v is None:
|
||||
return v
|
||||
v = v.strip().lower()
|
||||
if not v:
|
||||
raise ValueError("Key cannot be empty")
|
||||
if not KEY_PATTERN.match(v):
|
||||
raise ValueError(
|
||||
"Key must contain only lowercase letters, numbers, and hyphens. "
|
||||
"Cannot start or end with a hyphen."
|
||||
)
|
||||
return v
|
||||
|
||||
|
||||
class SearchRequest(BaseModel):
|
||||
"""Request payload for searching series."""
|
||||
|
||||
|
||||
@@ -39,6 +39,21 @@ class SchedulerConfig(BaseModel):
|
||||
description="Automatically queue and start downloads for all missing "
|
||||
"episodes after a scheduled rescan completes.",
|
||||
)
|
||||
nfo_scan_after_rescan: bool = Field(
|
||||
default=True,
|
||||
description="Run NFO validation and creation after a scheduled rescan "
|
||||
"completes. Checks each series folder for tvshow.nfo and "
|
||||
"creates or fills missing properties.",
|
||||
)
|
||||
# Legacy alias fields — read via Pydantic alias
|
||||
auto_download: Optional[bool] = Field(default=None, alias="auto_download")
|
||||
|
||||
def __init__(self, **data):
|
||||
super().__init__(**data)
|
||||
# Map legacy keys to primary fields only when primary key absent from data.
|
||||
# "key in data" checks for explicit presence (even False/None), not just truthiness.
|
||||
if self.auto_download is not None and "auto_download_after_rescan" not in data:
|
||||
object.__setattr__(self, "auto_download_after_rescan", self.auto_download)
|
||||
|
||||
@field_validator("schedule_time")
|
||||
@classmethod
|
||||
@@ -64,6 +79,22 @@ class SchedulerConfig(BaseModel):
|
||||
)
|
||||
return v
|
||||
|
||||
def model_dump(self, **kwargs) -> Dict[str, object]:
|
||||
"""Serialize, excluding legacy alias fields when they are None.
|
||||
|
||||
The alias fields (auto_download, folder_scan) must not be written to
|
||||
config.json as null entries, otherwise a roundtrip load sees the key
|
||||
present (哪怕 value is None) and skips the alias-to-primary mapping.
|
||||
"""
|
||||
data = super().model_dump(**kwargs)
|
||||
# Drop None alias fields so they don't pollute config.json.
|
||||
# They are still settable via the constructor for backward compatibility.
|
||||
if data.get("auto_download") is None:
|
||||
data.pop("auto_download", None)
|
||||
if data.get("folder_scan") is None:
|
||||
data.pop("folder_scan", None)
|
||||
return data
|
||||
|
||||
|
||||
class BackupConfig(BaseModel):
|
||||
"""Configuration for automatic backups of application data."""
|
||||
@@ -166,6 +197,12 @@ class AppConfig(BaseModel):
|
||||
logging: LoggingConfig = Field(default_factory=LoggingConfig)
|
||||
backup: BackupConfig = Field(default_factory=BackupConfig)
|
||||
nfo: NFOConfig = Field(default_factory=NFOConfig)
|
||||
scan_key_overrides: Dict[str, str] = Field(
|
||||
default_factory=dict,
|
||||
description="Map of folder names to provider keys for scan overrides. "
|
||||
"Used when auto-generated keys from folder names are incorrect. "
|
||||
"Format: {\"Folder Name\": \"actual-provider-key\"}"
|
||||
)
|
||||
other: Dict[str, object] = Field(
|
||||
default_factory=dict, description="Arbitrary other settings"
|
||||
)
|
||||
@@ -204,6 +241,7 @@ class ConfigUpdate(BaseModel):
|
||||
logging: Optional[LoggingConfig] = None
|
||||
backup: Optional[BackupConfig] = None
|
||||
nfo: Optional[NFOConfig] = None
|
||||
scan_key_overrides: Optional[Dict[str, str]] = None
|
||||
other: Optional[Dict[str, object]] = None
|
||||
|
||||
def apply_to(self, current: AppConfig) -> AppConfig:
|
||||
@@ -220,6 +258,8 @@ class ConfigUpdate(BaseModel):
|
||||
data["backup"] = self.backup.model_dump()
|
||||
if self.nfo is not None:
|
||||
data["nfo"] = self.nfo.model_dump()
|
||||
if self.scan_key_overrides is not None:
|
||||
data["scan_key_overrides"] = self.scan_key_overrides
|
||||
if self.other is not None:
|
||||
merged = dict(current.other or {})
|
||||
merged.update(self.other)
|
||||
|
||||
@@ -22,6 +22,7 @@ class DownloadStatus(str, Enum):
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
CANCELLED = "cancelled"
|
||||
PERMANENTLY_FAILED = "permanently_failed"
|
||||
|
||||
|
||||
class DownloadPriority(str, Enum):
|
||||
|
||||
@@ -355,3 +355,29 @@ class NFOMissingResponse(BaseModel):
|
||||
...,
|
||||
description="List of series missing NFO"
|
||||
)
|
||||
|
||||
|
||||
class NfoDiagnosticsResponse(BaseModel):
|
||||
"""Response for NFO diagnostics showing missing required tags."""
|
||||
|
||||
has_nfo: bool = Field(..., description="Whether tvshow.nfo exists")
|
||||
nfo_path: Optional[str] = Field(None, description="Path to NFO file if exists")
|
||||
missing_tags: List[str] = Field(
|
||||
default_factory=list,
|
||||
description="List of missing required tag names"
|
||||
)
|
||||
required_tags: List[str] = Field(
|
||||
default_factory=list,
|
||||
description="All required tag names for reference"
|
||||
)
|
||||
|
||||
|
||||
class NfoRepairResponse(BaseModel):
|
||||
"""Response after NFO repair attempt."""
|
||||
|
||||
success: bool = Field(..., description="Whether repair succeeded")
|
||||
message: str = Field(..., description="Human-readable result message")
|
||||
repaired_tags: List[str] = Field(
|
||||
default_factory=list,
|
||||
description="Tags that were missing before repair"
|
||||
)
|
||||
|
||||
9
src/server/nfo/__init__.py
Normal file
9
src/server/nfo/__init__.py
Normal file
@@ -0,0 +1,9 @@
|
||||
"""NFO package - TV show metadata generation for Kodi/XBMC.
|
||||
|
||||
Re-exports the public API for the nfo package.
|
||||
"""
|
||||
|
||||
from src.server.nfo.nfo_models import TVShowNFO
|
||||
from src.server.nfo.tmdb_client import TMDBClient, TMDBAPIError
|
||||
from src.server.nfo.nfo_generator import generate_tvshow_nfo
|
||||
from src.server.nfo.nfo_mapper import tmdb_to_nfo_model
|
||||
@@ -4,7 +4,7 @@ This module provides functions to generate tvshow.nfo XML files from
|
||||
TVShowNFO Pydantic models, adapted from the scraper project.
|
||||
|
||||
Example:
|
||||
>>> from src.core.entities.nfo_models import TVShowNFO
|
||||
>>> from src.server.nfo.nfo_models import TVShowNFO
|
||||
>>> nfo = TVShowNFO(title="Test Show", year=2020, tmdbid=12345)
|
||||
>>> xml_string = generate_tvshow_nfo(nfo)
|
||||
"""
|
||||
@@ -15,7 +15,7 @@ from typing import Optional
|
||||
from lxml import etree
|
||||
|
||||
from src.config.settings import settings
|
||||
from src.core.entities.nfo_models import TVShowNFO
|
||||
from src.server.nfo.nfo_models import TVShowNFO
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -11,7 +11,7 @@ import logging
|
||||
from datetime import datetime
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from src.core.entities.nfo_models import (
|
||||
from src.server.nfo.nfo_models import (
|
||||
ActorInfo,
|
||||
ImageInfo,
|
||||
NamedSeason,
|
||||
335
src/server/nfo/nfo_models.py
Normal file
335
src/server/nfo/nfo_models.py
Normal file
@@ -0,0 +1,335 @@
|
||||
"""Pydantic models for NFO metadata based on Kodi/XBMC standard.
|
||||
|
||||
This module provides data models for tvshow.nfo files that are compatible
|
||||
with media center applications like Kodi, Plex, and Jellyfin.
|
||||
|
||||
Example:
|
||||
>>> nfo = TVShowNFO(
|
||||
... title="Attack on Titan",
|
||||
... year=2013,
|
||||
... tmdbid=1429
|
||||
... )
|
||||
>>> nfo.premiered = "2013-04-07"
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
|
||||
from pydantic import BaseModel, Field, HttpUrl, field_validator
|
||||
|
||||
|
||||
class RatingInfo(BaseModel):
|
||||
"""Rating information from various sources.
|
||||
|
||||
Attributes:
|
||||
name: Source of the rating (e.g., 'themoviedb', 'imdb')
|
||||
value: Rating value (typically 0-10)
|
||||
votes: Number of votes
|
||||
max_rating: Maximum possible rating (default: 10)
|
||||
default: Whether this is the default rating to display
|
||||
"""
|
||||
|
||||
name: str = Field(..., description="Rating source name")
|
||||
value: float = Field(..., ge=0, description="Rating value")
|
||||
votes: Optional[int] = Field(None, ge=0, description="Number of votes")
|
||||
max_rating: int = Field(10, ge=1, description="Maximum rating value")
|
||||
default: bool = Field(False, description="Is this the default rating")
|
||||
|
||||
@field_validator('value')
|
||||
@classmethod
|
||||
def validate_value(cls, v: float, info) -> float:
|
||||
"""Ensure rating value doesn't exceed max_rating."""
|
||||
# Note: max_rating is not available yet during validation,
|
||||
# so we use a reasonable default check
|
||||
if v > 10:
|
||||
raise ValueError("Rating value cannot exceed 10")
|
||||
return v
|
||||
|
||||
|
||||
class ActorInfo(BaseModel):
|
||||
"""Actor/cast member information.
|
||||
|
||||
Attributes:
|
||||
name: Actor's name
|
||||
role: Character name/role
|
||||
thumb: URL to actor's photo
|
||||
profile: URL to actor's profile page
|
||||
tmdbid: TMDB ID for the actor
|
||||
"""
|
||||
|
||||
name: str = Field(..., description="Actor's name")
|
||||
role: Optional[str] = Field(None, description="Character role")
|
||||
thumb: Optional[HttpUrl] = Field(None, description="Actor photo URL")
|
||||
profile: Optional[HttpUrl] = Field(None, description="Actor profile URL")
|
||||
tmdbid: Optional[int] = Field(None, description="TMDB actor ID")
|
||||
|
||||
|
||||
class ImageInfo(BaseModel):
|
||||
"""Image information for posters, fanart, and logos.
|
||||
|
||||
Attributes:
|
||||
url: URL to the image
|
||||
aspect: Image aspect/type (e.g., 'poster', 'clearlogo', 'logo')
|
||||
season: Season number for season-specific images
|
||||
type: Image type (e.g., 'season')
|
||||
"""
|
||||
|
||||
url: HttpUrl = Field(..., description="Image URL")
|
||||
aspect: Optional[str] = Field(
|
||||
None,
|
||||
description="Image aspect (poster, clearlogo, logo)"
|
||||
)
|
||||
season: Optional[int] = Field(None, ge=-1, description="Season number")
|
||||
type: Optional[str] = Field(None, description="Image type")
|
||||
|
||||
|
||||
class NamedSeason(BaseModel):
|
||||
"""Named season information.
|
||||
|
||||
Attributes:
|
||||
number: Season number
|
||||
name: Season name/title
|
||||
"""
|
||||
|
||||
number: int = Field(..., ge=0, description="Season number")
|
||||
name: str = Field(..., description="Season name")
|
||||
|
||||
|
||||
class UniqueID(BaseModel):
|
||||
"""Unique identifier from various sources.
|
||||
|
||||
Attributes:
|
||||
type: ID source type (tmdb, imdb, tvdb)
|
||||
value: The ID value
|
||||
default: Whether this is the default ID
|
||||
"""
|
||||
|
||||
type: str = Field(..., description="ID type (tmdb, imdb, tvdb)")
|
||||
value: str = Field(..., description="ID value")
|
||||
default: bool = Field(False, description="Is default ID")
|
||||
|
||||
|
||||
class TVShowNFO(BaseModel):
|
||||
"""Main tvshow.nfo structure following Kodi/XBMC standard.
|
||||
|
||||
This model represents the complete metadata for a TV show that can be
|
||||
serialized to XML for use with media center applications.
|
||||
|
||||
Attributes:
|
||||
title: Main title of the show
|
||||
originaltitle: Original title (e.g., in original language)
|
||||
showtitle: Show title (often same as title)
|
||||
sorttitle: Title used for sorting
|
||||
year: Release year
|
||||
plot: Full plot description
|
||||
outline: Short plot summary
|
||||
tagline: Show tagline/slogan
|
||||
runtime: Episode runtime in minutes
|
||||
mpaa: Content rating (e.g., TV-14, TV-MA)
|
||||
certification: Additional certification info
|
||||
premiered: Premiere date (YYYY-MM-DD format)
|
||||
status: Show status (e.g., 'Continuing', 'Ended')
|
||||
studio: List of production studios
|
||||
genre: List of genres
|
||||
country: List of countries
|
||||
tag: List of tags/keywords
|
||||
ratings: List of ratings from various sources
|
||||
userrating: User's personal rating
|
||||
watched: Whether the show has been watched
|
||||
playcount: Number of times watched
|
||||
tmdbid: TMDB ID
|
||||
imdbid: IMDB ID
|
||||
tvdbid: TVDB ID
|
||||
uniqueid: List of unique IDs
|
||||
thumb: List of thumbnail/poster images
|
||||
fanart: List of fanart/backdrop images
|
||||
actors: List of cast members
|
||||
namedseason: List of named seasons
|
||||
trailer: Trailer URL
|
||||
dateadded: Date when added to library
|
||||
"""
|
||||
|
||||
# Required fields
|
||||
title: str = Field(..., description="Show title", min_length=1)
|
||||
|
||||
# Basic information (optional)
|
||||
originaltitle: Optional[str] = Field(None, description="Original title")
|
||||
showtitle: Optional[str] = Field(None, description="Show title")
|
||||
sorttitle: Optional[str] = Field(None, description="Sort title")
|
||||
year: Optional[int] = Field(
|
||||
None,
|
||||
ge=1900,
|
||||
le=2100,
|
||||
description="Release year"
|
||||
)
|
||||
|
||||
# Plot and description
|
||||
plot: Optional[str] = Field(None, description="Full plot description")
|
||||
outline: Optional[str] = Field(None, description="Short plot summary")
|
||||
tagline: Optional[str] = Field(None, description="Show tagline")
|
||||
|
||||
# Technical details
|
||||
runtime: Optional[int] = Field(
|
||||
None,
|
||||
ge=0,
|
||||
description="Episode runtime in minutes"
|
||||
)
|
||||
mpaa: Optional[str] = Field(None, description="Content rating")
|
||||
fsk: Optional[str] = Field(
|
||||
None,
|
||||
description="German FSK rating (e.g., 'FSK 12', 'FSK 16')"
|
||||
)
|
||||
certification: Optional[str] = Field(
|
||||
None,
|
||||
description="Certification info"
|
||||
)
|
||||
|
||||
# Status and dates
|
||||
premiered: Optional[str] = Field(
|
||||
None,
|
||||
description="Premiere date (YYYY-MM-DD)"
|
||||
)
|
||||
status: Optional[str] = Field(None, description="Show status")
|
||||
dateadded: Optional[str] = Field(
|
||||
None,
|
||||
description="Date added to library"
|
||||
)
|
||||
|
||||
# Multi-value fields
|
||||
studio: List[str] = Field(
|
||||
default_factory=list,
|
||||
description="Production studios"
|
||||
)
|
||||
genre: List[str] = Field(
|
||||
default_factory=list,
|
||||
description="Genres"
|
||||
)
|
||||
country: List[str] = Field(
|
||||
default_factory=list,
|
||||
description="Countries"
|
||||
)
|
||||
tag: List[str] = Field(
|
||||
default_factory=list,
|
||||
description="Tags/keywords"
|
||||
)
|
||||
|
||||
# IDs
|
||||
tmdbid: Optional[int] = Field(None, description="TMDB ID")
|
||||
imdbid: Optional[str] = Field(None, description="IMDB ID")
|
||||
tvdbid: Optional[int] = Field(None, description="TVDB ID")
|
||||
uniqueid: List[UniqueID] = Field(
|
||||
default_factory=list,
|
||||
description="Unique IDs"
|
||||
)
|
||||
|
||||
# Ratings and viewing info
|
||||
ratings: List[RatingInfo] = Field(
|
||||
default_factory=list,
|
||||
description="Ratings"
|
||||
)
|
||||
userrating: Optional[float] = Field(
|
||||
None,
|
||||
ge=0,
|
||||
le=10,
|
||||
description="User rating"
|
||||
)
|
||||
watched: bool = Field(False, description="Watched status")
|
||||
playcount: Optional[int] = Field(
|
||||
None,
|
||||
ge=0,
|
||||
description="Play count"
|
||||
)
|
||||
|
||||
# Media
|
||||
thumb: List[ImageInfo] = Field(
|
||||
default_factory=list,
|
||||
description="Thumbnail images"
|
||||
)
|
||||
fanart: List[ImageInfo] = Field(
|
||||
default_factory=list,
|
||||
description="Fanart images"
|
||||
)
|
||||
|
||||
# Cast and crew
|
||||
actors: List[ActorInfo] = Field(
|
||||
default_factory=list,
|
||||
description="Cast members"
|
||||
)
|
||||
|
||||
# Seasons
|
||||
namedseason: List[NamedSeason] = Field(
|
||||
default_factory=list,
|
||||
description="Named seasons"
|
||||
)
|
||||
|
||||
# Additional
|
||||
trailer: Optional[HttpUrl] = Field(None, description="Trailer URL")
|
||||
|
||||
@field_validator('premiered')
|
||||
@classmethod
|
||||
def validate_premiered_date(cls, v: Optional[str]) -> Optional[str]:
|
||||
"""Validate premiered date format (YYYY-MM-DD)."""
|
||||
if v is None:
|
||||
return v
|
||||
|
||||
# Check format strictly: YYYY-MM-DD
|
||||
if len(v) != 10 or v[4] != '-' or v[7] != '-':
|
||||
raise ValueError(
|
||||
"Premiered date must be in YYYY-MM-DD format"
|
||||
)
|
||||
|
||||
try:
|
||||
datetime.strptime(v, '%Y-%m-%d')
|
||||
except ValueError as exc:
|
||||
raise ValueError(
|
||||
"Premiered date must be in YYYY-MM-DD format"
|
||||
) from exc
|
||||
|
||||
return v
|
||||
|
||||
@field_validator('dateadded')
|
||||
@classmethod
|
||||
def validate_dateadded(cls, v: Optional[str]) -> Optional[str]:
|
||||
"""Validate dateadded format (YYYY-MM-DD HH:MM:SS)."""
|
||||
if v is None:
|
||||
return v
|
||||
|
||||
# Check format strictly: YYYY-MM-DD HH:MM:SS
|
||||
if len(v) != 19 or v[4] != '-' or v[7] != '-' or v[10] != ' ' or v[13] != ':' or v[16] != ':':
|
||||
raise ValueError(
|
||||
"Dateadded must be in YYYY-MM-DD HH:MM:SS format"
|
||||
)
|
||||
|
||||
try:
|
||||
datetime.strptime(v, '%Y-%m-%d %H:%M:%S')
|
||||
except ValueError as exc:
|
||||
raise ValueError(
|
||||
"Dateadded must be in YYYY-MM-DD HH:MM:SS format"
|
||||
) from exc
|
||||
|
||||
return v
|
||||
|
||||
@field_validator('imdbid')
|
||||
@classmethod
|
||||
def validate_imdbid(cls, v: Optional[str]) -> Optional[str]:
|
||||
"""Validate IMDB ID format (should start with 'tt')."""
|
||||
if v is None:
|
||||
return v
|
||||
|
||||
if not v.startswith('tt'):
|
||||
raise ValueError("IMDB ID must start with 'tt'")
|
||||
|
||||
if not v[2:].isdigit():
|
||||
raise ValueError("IMDB ID must be 'tt' followed by digits")
|
||||
|
||||
return v
|
||||
|
||||
def model_post_init(self, __context) -> None:
|
||||
"""Set default values after initialization."""
|
||||
# Set showtitle to title if not provided
|
||||
if self.showtitle is None:
|
||||
self.showtitle = self.title
|
||||
|
||||
# Set originaltitle to title if not provided
|
||||
if self.originaltitle is None:
|
||||
self.originaltitle = self.title
|
||||
@@ -12,6 +12,7 @@ Example:
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
@@ -38,6 +39,7 @@ class TMDBClient:
|
||||
|
||||
DEFAULT_BASE_URL = "https://api.themoviedb.org/3"
|
||||
DEFAULT_IMAGE_BASE_URL = "https://image.tmdb.org/t/p"
|
||||
NEGATIVE_CACHE_TTL = 86400 # 24 hours
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -63,6 +65,12 @@ class TMDBClient:
|
||||
self.max_connections = max_connections
|
||||
self.session: Optional[aiohttp.ClientSession] = None
|
||||
self._cache: Dict[str, Any] = {}
|
||||
self._negative_cache: Dict[str, float] = {} # query -> timestamp when cached
|
||||
# TMDB allows ~40 req/s; use 30 concurrent + per-second throttle to stay safe
|
||||
self._semaphore = asyncio.Semaphore(30)
|
||||
self._rate_limit_lock = asyncio.Lock()
|
||||
self._request_timestamps: List[float] = []
|
||||
self._max_requests_per_second = 35 # Stay under TMDB's ~40/s limit
|
||||
|
||||
async def __aenter__(self):
|
||||
"""Async context manager entry."""
|
||||
@@ -83,7 +91,7 @@ class TMDBClient:
|
||||
self,
|
||||
endpoint: str,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
max_retries: int = 3
|
||||
max_retries: int = 5
|
||||
) -> Dict[str, Any]:
|
||||
"""Make an async request to TMDB API with retries.
|
||||
|
||||
@@ -110,58 +118,100 @@ class TMDBClient:
|
||||
logger.debug("Cache hit for %s", endpoint)
|
||||
return self._cache[cache_key]
|
||||
|
||||
# Check negative cache (cached empty results)
|
||||
negative_cache_key = f"{endpoint}:{str(sorted(params.items()))}"
|
||||
if negative_cache_key in self._negative_cache:
|
||||
if time.monotonic() - self._negative_cache[negative_cache_key] < self.NEGATIVE_CACHE_TTL:
|
||||
logger.debug("Negative cache hit for %s (cached empty result)", endpoint)
|
||||
return {"results": []}
|
||||
else:
|
||||
# Expired negative cache entry
|
||||
del self._negative_cache[negative_cache_key]
|
||||
|
||||
delay = 1
|
||||
last_error = None
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
# Re-ensure session before each attempt in case it was closed
|
||||
await self._ensure_session()
|
||||
|
||||
if self.session is None:
|
||||
raise TMDBAPIError("Session is not available")
|
||||
|
||||
logger.debug("TMDB API request: %s (attempt %s)", endpoint, attempt + 1)
|
||||
async with self.session.get(url, params=params, timeout=aiohttp.ClientTimeout(total=60)) as resp:
|
||||
if resp.status == 401:
|
||||
raise TMDBAPIError("Invalid TMDB API key")
|
||||
elif resp.status == 404:
|
||||
raise TMDBAPIError(f"Resource not found: {endpoint}")
|
||||
elif resp.status == 429:
|
||||
# Rate limit - wait longer
|
||||
retry_after = int(resp.headers.get('Retry-After', delay * 2))
|
||||
logger.warning("Rate limited, waiting %ss", retry_after)
|
||||
await asyncio.sleep(retry_after)
|
||||
continue
|
||||
|
||||
resp.raise_for_status()
|
||||
data = await resp.json()
|
||||
self._cache[cache_key] = data
|
||||
return data
|
||||
|
||||
except asyncio.TimeoutError as e:
|
||||
last_error = e
|
||||
if attempt < max_retries - 1:
|
||||
logger.warning("Request timeout (attempt %s), retrying in %ss", attempt + 1, delay)
|
||||
await asyncio.sleep(delay)
|
||||
delay *= 2
|
||||
else:
|
||||
logger.error("Request timed out after %s attempts", max_retries)
|
||||
|
||||
except (aiohttp.ClientError, AttributeError) as e:
|
||||
last_error = e
|
||||
# If connector/session was closed, try to recreate it
|
||||
if "Connector is closed" in str(e) or isinstance(e, AttributeError):
|
||||
logger.warning("Session issue detected, recreating session: %s", e)
|
||||
self.session = None
|
||||
# Rate limiting: ensure we don't exceed ~35 requests/second
|
||||
async with self._rate_limit_lock:
|
||||
now = time.monotonic()
|
||||
# Remove timestamps older than 1 second
|
||||
self._request_timestamps = [
|
||||
ts for ts in self._request_timestamps if now - ts < 1.0
|
||||
]
|
||||
if len(self._request_timestamps) >= self._max_requests_per_second:
|
||||
sleep_time = 1.0 - (now - self._request_timestamps[0])
|
||||
if sleep_time > 0:
|
||||
logger.debug("Rate throttling: waiting %.2fs", sleep_time)
|
||||
await asyncio.sleep(sleep_time)
|
||||
self._request_timestamps.append(time.monotonic())
|
||||
|
||||
async with self._semaphore:
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
# Re-ensure session before each attempt in case it was closed
|
||||
await self._ensure_session()
|
||||
|
||||
if attempt < max_retries - 1:
|
||||
logger.warning("Request failed (attempt %s): %s, retrying in %ss", attempt + 1, e, delay)
|
||||
await asyncio.sleep(delay)
|
||||
delay *= 2
|
||||
else:
|
||||
logger.error("Request failed after %s attempts: %s", max_retries, e)
|
||||
|
||||
if self.session is None:
|
||||
raise TMDBAPIError("Session is not available")
|
||||
|
||||
logger.debug("TMDB API request: %s (attempt %s)", endpoint, attempt + 1)
|
||||
async with self.session.get(url, params=params, timeout=aiohttp.ClientTimeout(total=60)) as resp:
|
||||
if resp.status == 401:
|
||||
raise TMDBAPIError("Invalid TMDB API key")
|
||||
elif resp.status == 404:
|
||||
raise TMDBAPIError(f"Resource not found: {endpoint}")
|
||||
elif resp.status == 429:
|
||||
# Rate limit - wait longer with exponential backoff
|
||||
retry_after = int(resp.headers.get('Retry-After', max(delay * 2, 2)))
|
||||
logger.warning("Rate limited, waiting %ss", retry_after)
|
||||
await asyncio.sleep(retry_after)
|
||||
continue
|
||||
|
||||
resp.raise_for_status()
|
||||
data = await resp.json()
|
||||
self._cache[cache_key] = data
|
||||
# Cache negative result if empty
|
||||
if endpoint.startswith("search/") and not data.get("results"):
|
||||
self._negative_cache[negative_cache_key] = time.monotonic()
|
||||
logger.debug("Cached negative result for %s", endpoint)
|
||||
return data
|
||||
|
||||
except asyncio.TimeoutError as e:
|
||||
last_error = e
|
||||
if attempt < max_retries - 1:
|
||||
logger.warning("Request timeout (attempt %s), retrying in %ss", attempt + 1, delay)
|
||||
await asyncio.sleep(delay)
|
||||
delay *= 2
|
||||
else:
|
||||
logger.error("Request timed out after %s attempts", max_retries)
|
||||
|
||||
except (aiohttp.ClientError, AttributeError) as e:
|
||||
last_error = e
|
||||
# If connector/session was closed, try to recreate it
|
||||
if "Connector is closed" in str(e) or isinstance(e, AttributeError):
|
||||
logger.warning(
|
||||
"Session issue detected, recreating session: %s",
|
||||
e,
|
||||
exc_info=True,
|
||||
)
|
||||
self.session = None
|
||||
await self._ensure_session()
|
||||
|
||||
# DNS / host-unreachable errors are not transient — abort immediately
|
||||
error_str = str(e)
|
||||
if "name resolution" in error_str.lower() or (
|
||||
isinstance(e, aiohttp.ClientConnectorError) and
|
||||
"Cannot connect to host" in error_str
|
||||
):
|
||||
logger.error("Non-transient connection error, aborting retries: %s", e)
|
||||
raise TMDBAPIError(f"Request failed after {attempt + 1} attempts: {e}") from e
|
||||
|
||||
if attempt < max_retries - 1:
|
||||
logger.warning("Request failed (attempt %s): %s, retrying in %ss", attempt + 1, e, delay)
|
||||
await asyncio.sleep(delay)
|
||||
delay *= 2
|
||||
else:
|
||||
logger.error("Request failed after %s attempts: %s", max_retries, e)
|
||||
|
||||
raise TMDBAPIError(f"Request failed after {max_retries} attempts: {last_error}")
|
||||
|
||||
@@ -190,6 +240,34 @@ class TMDBClient:
|
||||
{"query": query, "language": language, "page": page}
|
||||
)
|
||||
|
||||
async def search_multi(
|
||||
self,
|
||||
query: str,
|
||||
language: str = "en-US",
|
||||
page: int = 1
|
||||
) -> Dict[str, Any]:
|
||||
"""Search for movies and TV shows by name using TMDB multi search.
|
||||
|
||||
Multi search returns both movies and TV shows, useful for anime
|
||||
that might be indexed as movies on TMDB.
|
||||
|
||||
Args:
|
||||
query: Search query (show name)
|
||||
language: Language for results (default: English)
|
||||
page: Page number for pagination
|
||||
|
||||
Returns:
|
||||
Search results with list of movies and TV shows
|
||||
|
||||
Example:
|
||||
>>> results = await client.search_multi("Suzume no Tojimari")
|
||||
>>> shows = [r for r in results["results"] if r["media_type"] == "tv"]
|
||||
"""
|
||||
return await self._request(
|
||||
"search/multi",
|
||||
{"query": query, "language": language, "page": page}
|
||||
)
|
||||
|
||||
async def get_tv_show_details(
|
||||
self,
|
||||
tv_id: int,
|
||||
@@ -309,8 +387,38 @@ class TMDBClient:
|
||||
await self.session.close()
|
||||
self.session = None
|
||||
logger.debug("TMDB client session closed")
|
||||
|
||||
def __del__(self):
|
||||
"""Warn if session is unclosed during garbage collection."""
|
||||
if self.session is not None and not self.session.closed:
|
||||
logger.warning(
|
||||
"TMDBClient: unclosed session detected. "
|
||||
"Use 'async with TMDBClient(...)' or call close() explicitly."
|
||||
)
|
||||
|
||||
def clear_cache(self):
|
||||
"""Clear the request cache."""
|
||||
self._cache.clear()
|
||||
logger.debug("TMDB client cache cleared")
|
||||
|
||||
def clear_negative_cache(self):
|
||||
"""Clear the negative result cache."""
|
||||
self._negative_cache.clear()
|
||||
logger.debug("TMDB negative cache cleared")
|
||||
|
||||
def cleanup_expired_negative_cache(self) -> int:
|
||||
"""Remove expired entries from negative cache.
|
||||
|
||||
Returns:
|
||||
Number of entries removed
|
||||
"""
|
||||
now = time.monotonic()
|
||||
expired_keys = [
|
||||
key for key, timestamp in self._negative_cache.items()
|
||||
if now - timestamp >= self.NEGATIVE_CACHE_TTL
|
||||
]
|
||||
for key in expired_keys:
|
||||
del self._negative_cache[key]
|
||||
if expired_keys:
|
||||
logger.debug("Removed %d expired negative cache entries", len(expired_keys))
|
||||
return len(expired_keys)
|
||||
0
src/server/providers/__init__.py
Normal file
0
src/server/providers/__init__.py
Normal file
@@ -9,6 +9,7 @@ import threading
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
import chardet
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from events import Events
|
||||
@@ -80,6 +81,37 @@ if not download_error_logger.handlers:
|
||||
noKeyFound_logger = logging.getLogger()
|
||||
|
||||
|
||||
def _decode_html_content(content: bytes) -> str:
|
||||
"""Decode HTML content with encoding detection.
|
||||
|
||||
Uses chardet to detect the actual encoding of the content,
|
||||
falling back to utf-8 with replacement error handling.
|
||||
|
||||
Args:
|
||||
content: Raw HTML bytes from the response
|
||||
|
||||
Returns:
|
||||
Decoded string content
|
||||
"""
|
||||
detected = chardet.detect(content)
|
||||
encoding = detected.get('encoding', 'utf-8')
|
||||
confidence = detected.get('confidence', 0)
|
||||
|
||||
if confidence < 0.7:
|
||||
logger.debug(
|
||||
"Low encoding confidence (%.2f) for detected encoding '%s', using utf-8",
|
||||
confidence,
|
||||
encoding
|
||||
)
|
||||
encoding = 'utf-8'
|
||||
|
||||
try:
|
||||
return content.decode(encoding, errors='replace')
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to decode content with %s: %s, using utf-8 replace", encoding, exc)
|
||||
return content.decode('utf-8', errors='replace')
|
||||
|
||||
|
||||
class AniworldLoader(Loader):
|
||||
def __init__(self) -> None:
|
||||
self.SUPPORTED_PROVIDERS = DEFAULT_PROVIDERS
|
||||
@@ -90,7 +122,10 @@ class AniworldLoader(Loader):
|
||||
self.LULUVDO_USER_AGENT = LULUVDO_USER_AGENT
|
||||
self.PROVIDER_HEADERS = {
|
||||
ProviderType.VIDMOLY.value: ['Referer: "https://vidmoly.to"'],
|
||||
ProviderType.DOODSTREAM.value: ['Referer: "https://dood.li/"'],
|
||||
ProviderType.DOODSTREAM.value: [
|
||||
'Referer: "https://dood.li/"',
|
||||
'Referer: "https://playmogo.com/"',
|
||||
],
|
||||
ProviderType.VOE.value: [f"User-Agent: {self.RANDOM_USER_AGENT}"],
|
||||
ProviderType.LULUVDO.value: [
|
||||
f"User-Agent: {self.LULUVDO_USER_AGENT}",
|
||||
@@ -123,6 +158,7 @@ class AniworldLoader(Loader):
|
||||
|
||||
self._KeyHTMLDict = {}
|
||||
self._EpisodeHTMLDict = {}
|
||||
self._YearDict = {}
|
||||
self.Providers = Providers()
|
||||
|
||||
# Events: download_progress is triggered with progress dict
|
||||
@@ -231,7 +267,7 @@ class AniworldLoader(Loader):
|
||||
language_code = self._get_language_key(language)
|
||||
|
||||
episode_soup = BeautifulSoup(
|
||||
self._get_episode_html(season, episode, key).content,
|
||||
_decode_html_content(self._get_episode_html(season, episode, key).content),
|
||||
'html.parser'
|
||||
)
|
||||
change_language_box_div = episode_soup.find(
|
||||
@@ -249,6 +285,118 @@ class AniworldLoader(Loader):
|
||||
logger.debug("Available languages for S%02dE%03d: %s, requested: %s, available: %s", season, episode, languages, language_code, is_available)
|
||||
return is_available
|
||||
|
||||
def _check_url_alive(
|
||||
self,
|
||||
url: str,
|
||||
headers: dict | None = None,
|
||||
timeout: int = 10,
|
||||
) -> bool:
|
||||
"""Probe a provider URL with HEAD before committing to yt-dlp.
|
||||
|
||||
Skips dead providers quickly so the failover loop never blocks
|
||||
waiting for yt-dlp to fail on a 404. Falls back to a streaming
|
||||
GET when HEAD is not allowed by the upstream server.
|
||||
|
||||
Args:
|
||||
url: URL to probe.
|
||||
headers: Optional headers to forward with the probe.
|
||||
timeout: Per-request timeout (seconds).
|
||||
|
||||
Returns:
|
||||
True when the URL responds with a non-4xx status, else False.
|
||||
"""
|
||||
try:
|
||||
response = self.session.head(
|
||||
url,
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
allow_redirects=True,
|
||||
)
|
||||
if response.status_code == 405:
|
||||
response = self.session.get(
|
||||
url,
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
stream=True,
|
||||
allow_redirects=True,
|
||||
)
|
||||
response.close()
|
||||
if 400 <= response.status_code < 500:
|
||||
logger.warning(
|
||||
"Provider URL returned HTTP %s: %s",
|
||||
response.status_code, url
|
||||
)
|
||||
return False
|
||||
return True
|
||||
except requests.RequestException as exc:
|
||||
logger.warning("Provider URL unreachable %s: %s", url, exc)
|
||||
return False
|
||||
|
||||
def _try_direct_stream(
|
||||
self,
|
||||
link: str,
|
||||
output_path: str,
|
||||
headers: dict | None,
|
||||
timeout: int,
|
||||
) -> bool:
|
||||
"""Stream a direct video URL to disk without yt-dlp.
|
||||
|
||||
Used as a fast-path when the resolved provider link already points
|
||||
at a downloadable video file (``Content-Type: video/*`` or
|
||||
``application/octet-stream``). HLS and other non-video payloads
|
||||
are rejected so the caller can fall back to yt-dlp.
|
||||
|
||||
Args:
|
||||
link: Direct download URL.
|
||||
output_path: Destination file path.
|
||||
headers: Optional HTTP headers.
|
||||
timeout: Per-request timeout (seconds).
|
||||
|
||||
Returns:
|
||||
True on a successful save, False when the link is not a
|
||||
direct video or the download fails.
|
||||
"""
|
||||
try:
|
||||
with self.session.get(
|
||||
link,
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
stream=True,
|
||||
) as response:
|
||||
if not response.ok:
|
||||
logger.debug(
|
||||
"Direct stream HEAD returned %s for %s",
|
||||
response.status_code, link[:80]
|
||||
)
|
||||
return False
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
if not (
|
||||
content_type.startswith("video/")
|
||||
or content_type == "application/octet-stream"
|
||||
):
|
||||
logger.debug(
|
||||
"Direct stream skipped, Content-Type=%s",
|
||||
content_type
|
||||
)
|
||||
return False
|
||||
logger.info(
|
||||
"Direct stream download starting (type=%s)",
|
||||
content_type
|
||||
)
|
||||
with open(output_path, "wb") as fh:
|
||||
for chunk in response.iter_content(chunk_size=1024 * 1024):
|
||||
if self._cancel_flag.is_set():
|
||||
logger.info(
|
||||
"Cancellation detected during direct stream"
|
||||
)
|
||||
return False
|
||||
if chunk:
|
||||
fh.write(chunk)
|
||||
return True
|
||||
except requests.RequestException as exc:
|
||||
logger.warning("Direct stream download failed: %s", exc)
|
||||
return False
|
||||
|
||||
def download(
|
||||
self,
|
||||
base_directory: str,
|
||||
@@ -259,7 +407,12 @@ class AniworldLoader(Loader):
|
||||
language: str = "German Dub"
|
||||
) -> bool:
|
||||
"""Download episode to specified directory.
|
||||
|
||||
|
||||
Iterates the providers actually advertised on the episode page
|
||||
(ordered by SUPPORTED_PROVIDERS preference), probing each URL
|
||||
before attempting an extraction so dead providers are skipped
|
||||
immediately instead of stalling yt-dlp on a 404.
|
||||
|
||||
Args:
|
||||
base_directory: Base download directory path
|
||||
serie_folder: Filesystem folder name (metadata only, used for
|
||||
@@ -308,12 +461,78 @@ class AniworldLoader(Loader):
|
||||
temp_path = os.path.join(temp_dir, output_file)
|
||||
logger.debug("Temporary path: %s", temp_path)
|
||||
|
||||
for provider in self.SUPPORTED_PROVIDERS:
|
||||
logger.debug("Attempting download with provider: %s", provider)
|
||||
link, header = self._get_direct_link_from_provider(
|
||||
candidate_providers = self._select_providers_for_episode(
|
||||
season, episode, key, language
|
||||
)
|
||||
if not candidate_providers:
|
||||
logger.error(
|
||||
"No providers advertised for S%02dE%03d (%s) in %s",
|
||||
season, episode, key, language
|
||||
)
|
||||
logger.debug("Direct link obtained from provider")
|
||||
self.clear_cache()
|
||||
return False
|
||||
|
||||
tried: list[str] = []
|
||||
for provider_name, redirect_url in candidate_providers:
|
||||
tried.append(provider_name)
|
||||
logger.debug("Attempting download with provider: %s", provider_name)
|
||||
|
||||
probe_headers = {"User-Agent": self.RANDOM_USER_AGENT}
|
||||
if not self._check_url_alive(
|
||||
redirect_url,
|
||||
headers=probe_headers,
|
||||
timeout=self.DEFAULT_REQUEST_TIMEOUT,
|
||||
):
|
||||
logger.info(
|
||||
"Skipping provider %s, redirect URL not reachable",
|
||||
provider_name
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
resolved = self._resolve_direct_link(
|
||||
redirect_url, provider_name
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Provider %s link resolution failed: %s: %s",
|
||||
provider_name, type(exc).__name__, exc
|
||||
)
|
||||
continue
|
||||
|
||||
if resolved is None:
|
||||
logger.info(
|
||||
"Provider %s returned no direct link", provider_name
|
||||
)
|
||||
continue
|
||||
|
||||
link, header = resolved
|
||||
|
||||
if self._cancel_flag.is_set():
|
||||
logger.info("Cancellation requested before download start")
|
||||
_cleanup_temp_file(temp_path)
|
||||
self.clear_cache()
|
||||
return False
|
||||
|
||||
if self._try_direct_stream(
|
||||
link,
|
||||
temp_path,
|
||||
header,
|
||||
self.DEFAULT_REQUEST_TIMEOUT,
|
||||
) and os.path.exists(temp_path):
|
||||
logger.debug(
|
||||
"Direct stream succeeded with provider %s", provider_name
|
||||
)
|
||||
shutil.copyfile(temp_path, output_path)
|
||||
os.remove(temp_path)
|
||||
logger.info(
|
||||
"Download completed successfully (direct): %s",
|
||||
output_file
|
||||
)
|
||||
self.clear_cache()
|
||||
return True
|
||||
|
||||
_cleanup_temp_file(temp_path)
|
||||
|
||||
cancel_flag = self._cancel_flag
|
||||
|
||||
@@ -321,7 +540,6 @@ class AniworldLoader(Loader):
|
||||
if cancel_flag.is_set():
|
||||
logger.info("Cancellation detected in progress hook")
|
||||
raise DownloadCancelled("Download cancelled by user")
|
||||
# Fire the event for progress
|
||||
self.events.download_progress(d)
|
||||
|
||||
ydl_opts = {
|
||||
@@ -331,7 +549,12 @@ class AniworldLoader(Loader):
|
||||
'no_warnings': True,
|
||||
'progress_with_newline': False,
|
||||
'nocheckcertificate': True,
|
||||
'logger': logger,
|
||||
'progress_hooks': [events_progress_hook],
|
||||
# yt-dlp defaults to native HLS downloader which warns about
|
||||
# "Live HLS streams are not supported" - disable to go
|
||||
# straight to ffmpeg, avoiding the warning
|
||||
'hls_prefer_native': False,
|
||||
}
|
||||
|
||||
if header:
|
||||
@@ -339,9 +562,11 @@ class AniworldLoader(Loader):
|
||||
logger.debug("Using custom headers for download")
|
||||
|
||||
try:
|
||||
logger.debug("Starting YoutubeDL download")
|
||||
logger.info(
|
||||
"Starting yt-dlp download with %s: %s",
|
||||
provider_name, output_file
|
||||
)
|
||||
logger.debug("Download link: %s...", link[:100])
|
||||
logger.debug("YDL options: %s", ydl_opts)
|
||||
|
||||
with YoutubeDL(ydl_opts) as ydl:
|
||||
info = ydl.extract_info(link, download=True)
|
||||
@@ -352,39 +577,185 @@ class AniworldLoader(Loader):
|
||||
|
||||
if os.path.exists(temp_path):
|
||||
logger.debug("Moving file from temp to final destination")
|
||||
# Use copyfile instead of copy to avoid metadata permission issues
|
||||
shutil.copyfile(temp_path, output_path)
|
||||
os.remove(temp_path)
|
||||
logger.info("Download completed successfully: %s", output_file)
|
||||
logger.info(
|
||||
"Download completed successfully: %s", output_file
|
||||
)
|
||||
self.clear_cache()
|
||||
return True
|
||||
else:
|
||||
logger.error("Download failed: temp file not found at %s", temp_path)
|
||||
self.clear_cache()
|
||||
return False
|
||||
except BrokenPipeError as e:
|
||||
logger.error(
|
||||
"Broken pipe error with provider %s: %s. "
|
||||
"This usually means the stream connection was closed.",
|
||||
provider, e
|
||||
"Download failed: temp file not found at %s", temp_path
|
||||
)
|
||||
except DownloadCancelled:
|
||||
logger.info("Download cancelled by user")
|
||||
_cleanup_temp_file(temp_path)
|
||||
self.clear_cache()
|
||||
return False
|
||||
except BrokenPipeError as exc:
|
||||
logger.error(
|
||||
"Broken pipe error with provider %s: %s",
|
||||
provider_name, exc
|
||||
)
|
||||
_cleanup_temp_file(temp_path)
|
||||
continue
|
||||
except Exception as e:
|
||||
except Exception as exc:
|
||||
# Check if this is an HLS-related failure that might succeed
|
||||
# with additional ffmpeg options
|
||||
exc_str = str(exc).lower()
|
||||
is_hls_related = (
|
||||
'hls' in exc_str or
|
||||
'live' in exc_str or
|
||||
'native downloader' in exc_str
|
||||
)
|
||||
if is_hls_related and 'ffmpeg' not in str(ydl_opts.get('downloader', '')):
|
||||
logger.info(
|
||||
"HLS stream detected, retrying with ffmpeg options: %s",
|
||||
output_file
|
||||
)
|
||||
# Retry with ffmpeg explicitly set
|
||||
retry_opts = ydl_opts.copy()
|
||||
retry_opts['downloader'] = 'ffmpeg'
|
||||
retry_opts['hls_use_mpegts'] = True
|
||||
try:
|
||||
with YoutubeDL(retry_opts) as ydl:
|
||||
info = ydl.extract_info(link, download=True)
|
||||
if os.path.exists(temp_path):
|
||||
shutil.copyfile(temp_path, output_path)
|
||||
os.remove(temp_path)
|
||||
logger.info(
|
||||
"Download completed successfully (retry): %s",
|
||||
output_file
|
||||
)
|
||||
self.clear_cache()
|
||||
return True
|
||||
except Exception:
|
||||
_cleanup_temp_file(temp_path)
|
||||
# Continue to next provider if retry also fails
|
||||
continue
|
||||
|
||||
logger.error(
|
||||
"YoutubeDL download failed with provider %s: %s: %s",
|
||||
provider, type(e).__name__, e
|
||||
provider_name, type(exc).__name__, exc
|
||||
)
|
||||
_cleanup_temp_file(temp_path)
|
||||
continue
|
||||
break
|
||||
|
||||
# If we get here, all providers failed
|
||||
logger.error("All download providers failed")
|
||||
logger.error(
|
||||
"All download providers failed for S%02dE%03d (%s) in %s. "
|
||||
"Tried: %s. Episode may be unavailable on the source site.",
|
||||
season, episode, key, language, ", ".join(tried) or "none"
|
||||
)
|
||||
download_error_logger.error(
|
||||
"All providers failed for %s S%02dE%03d (%s); tried=%s",
|
||||
key, season, episode, language, tried
|
||||
)
|
||||
_cleanup_temp_file(temp_path)
|
||||
self.clear_cache()
|
||||
return False
|
||||
|
||||
def _select_providers_for_episode(
|
||||
self,
|
||||
season: int,
|
||||
episode: int,
|
||||
key: str,
|
||||
language: str,
|
||||
) -> list[tuple[str, str]]:
|
||||
"""Return ``[(provider_name, redirect_url), ...]`` for an episode.
|
||||
|
||||
Filters by requested language and orders results by
|
||||
``SUPPORTED_PROVIDERS`` preference so the failover chain matches
|
||||
operator expectations. Returns an empty list when nothing is
|
||||
advertised on the page.
|
||||
"""
|
||||
if not self.is_language(season, episode, key, language):
|
||||
logger.warning(
|
||||
"Language %s not advertised for S%02dE%03d (%s)",
|
||||
language, season, episode, key
|
||||
)
|
||||
return []
|
||||
language_code = self._get_language_key(language)
|
||||
providers = self._get_provider_from_html(season, episode, key)
|
||||
ordered: list[tuple[str, str]] = []
|
||||
preferred = list(self.SUPPORTED_PROVIDERS)
|
||||
for name in preferred:
|
||||
lang_map = providers.get(name)
|
||||
if lang_map and language_code in lang_map:
|
||||
ordered.append((name, lang_map[language_code]))
|
||||
for name, lang_map in providers.items():
|
||||
if name in preferred:
|
||||
continue
|
||||
if language_code in lang_map:
|
||||
ordered.append((name, lang_map[language_code]))
|
||||
return ordered
|
||||
|
||||
def _resolve_direct_link(
|
||||
self,
|
||||
redirect_url: str,
|
||||
provider_name: str,
|
||||
) -> tuple[str, dict] | None:
|
||||
"""Resolve a provider redirect URL into a direct stream link.
|
||||
|
||||
Follows the redirect to the embedded player, then delegates to a
|
||||
provider-specific extractor (when registered) or returns the
|
||||
embed URL itself so yt-dlp can attempt extraction.
|
||||
|
||||
Args:
|
||||
redirect_url: AniWorld redirect URL.
|
||||
provider_name: Provider key (e.g. ``"VOE"``).
|
||||
|
||||
Returns:
|
||||
``(direct_link, headers)`` tuple or None when extraction fails.
|
||||
"""
|
||||
try:
|
||||
embedded = self.session.get(
|
||||
redirect_url,
|
||||
timeout=self.DEFAULT_REQUEST_TIMEOUT,
|
||||
headers={"User-Agent": self.RANDOM_USER_AGENT},
|
||||
allow_redirects=True,
|
||||
).url
|
||||
except requests.RequestException as exc:
|
||||
logger.warning(
|
||||
"Failed resolving redirect for %s: %s", provider_name, exc
|
||||
)
|
||||
return None
|
||||
|
||||
try:
|
||||
extractor = self.Providers.GetProvider(provider_name)
|
||||
except (KeyError, AttributeError):
|
||||
extractor = None
|
||||
|
||||
if extractor is not None:
|
||||
try:
|
||||
return extractor.get_link(
|
||||
embedded, self.DEFAULT_REQUEST_TIMEOUT
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Custom extractor %s failed: %s",
|
||||
provider_name, exc
|
||||
)
|
||||
return None
|
||||
|
||||
header_list = self.PROVIDER_HEADERS.get(provider_name)
|
||||
header_dict = self._parse_provider_headers(header_list)
|
||||
return embedded, header_dict
|
||||
|
||||
@staticmethod
|
||||
def _parse_provider_headers(
|
||||
header_list: list | None,
|
||||
) -> dict[str, str]:
|
||||
"""Convert legacy ``"Name: value"`` header strings to a dict."""
|
||||
if not header_list:
|
||||
return {}
|
||||
parsed: dict[str, str] = {}
|
||||
for entry in header_list:
|
||||
if not isinstance(entry, str) or ":" not in entry:
|
||||
continue
|
||||
name, _, value = entry.partition(":")
|
||||
parsed[name.strip()] = value.strip().strip('"')
|
||||
return parsed
|
||||
|
||||
def get_site_key(self) -> str:
|
||||
"""Get the site key for this provider."""
|
||||
return "aniworld.to"
|
||||
@@ -393,7 +764,7 @@ class AniworldLoader(Loader):
|
||||
"""Get anime title from series key."""
|
||||
logger.debug("Getting title for key: %s", key)
|
||||
soup = BeautifulSoup(
|
||||
self._get_key_html(key).content,
|
||||
_decode_html_content(self._get_key_html(key).content),
|
||||
'html.parser'
|
||||
)
|
||||
title_div = soup.find('div', class_='series-title')
|
||||
@@ -404,55 +775,81 @@ class AniworldLoader(Loader):
|
||||
if span_tag:
|
||||
title = span_tag.text
|
||||
logger.debug("Found title: %s", title)
|
||||
|
||||
# Also try to extract year from sibling p tag "Jahr: {year}"
|
||||
# Year is typically right after title in the HTML structure
|
||||
year = self._extract_year_from_soup(soup)
|
||||
if year is not None:
|
||||
self._YearDict[key] = year
|
||||
logger.debug("Cached year %d for key: %s", year, key)
|
||||
|
||||
return title
|
||||
|
||||
logger.warning("No title found for key: %s", key)
|
||||
return ""
|
||||
|
||||
def _extract_year_from_soup(self, soup: BeautifulSoup) -> int | None:
|
||||
"""Extract year from BeautifulSoup object.
|
||||
|
||||
Looks for 'Jahr: {year}' pattern in p tags adjacent to series-title.
|
||||
|
||||
Args:
|
||||
soup: Parsed BeautifulSoup object
|
||||
|
||||
Returns:
|
||||
Year as int or None if not found
|
||||
"""
|
||||
# Try to find year in metadata
|
||||
for p_tag in soup.find_all('p'):
|
||||
text = p_tag.get_text()
|
||||
if 'Jahr:' in text or 'Year:' in text:
|
||||
match = re.search(r'(\d{4})', text)
|
||||
if match:
|
||||
return int(match.group(1))
|
||||
|
||||
# Fallback: look in series-info div
|
||||
info_div = soup.find('div', class_='series-info')
|
||||
if info_div:
|
||||
text = info_div.get_text()
|
||||
match = re.search(r'\b(19\d{2}|20\d{2})\b', text)
|
||||
if match:
|
||||
return int(match.group(1))
|
||||
|
||||
return None
|
||||
|
||||
def get_year(self, key: str) -> int | None:
|
||||
"""Get anime release year from series key.
|
||||
|
||||
Attempts to extract the year from the series page metadata.
|
||||
Returns None if year cannot be determined.
|
||||
|
||||
|
||||
Uses cached year from get_title if available,
|
||||
otherwise extracts and caches it.
|
||||
|
||||
Args:
|
||||
key: Series identifier
|
||||
|
||||
|
||||
Returns:
|
||||
int or None: Release year if found, None otherwise
|
||||
Release year or None if not found
|
||||
"""
|
||||
logger.debug("Getting year for key: %s", key)
|
||||
|
||||
# Check cache first
|
||||
if key in self._YearDict:
|
||||
logger.debug("Using cached year %d for key: %s", self._YearDict[key], key)
|
||||
return self._YearDict[key]
|
||||
|
||||
# Not cached - extract from HTML
|
||||
try:
|
||||
soup = BeautifulSoup(
|
||||
self._get_key_html(key).content,
|
||||
_decode_html_content(self._get_key_html(key).content),
|
||||
'html.parser'
|
||||
)
|
||||
|
||||
# Try to find year in metadata
|
||||
# Check for "Jahr:" or similar metadata fields
|
||||
for p_tag in soup.find_all('p'):
|
||||
text = p_tag.get_text()
|
||||
if 'Jahr:' in text or 'Year:' in text:
|
||||
# Extract year from text like "Jahr: 2025"
|
||||
match = re.search(r'(\d{4})', text)
|
||||
if match:
|
||||
year = int(match.group(1))
|
||||
logger.debug("Found year in metadata: %s", year)
|
||||
return year
|
||||
|
||||
# Try alternative: look for year in genre/info section
|
||||
info_div = soup.find('div', class_='series-info')
|
||||
if info_div:
|
||||
text = info_div.get_text()
|
||||
match = re.search(r'\b(19\d{2}|20\d{2})\b', text)
|
||||
if match:
|
||||
year = int(match.group(1))
|
||||
logger.debug("Found year in info section: %s", year)
|
||||
return year
|
||||
|
||||
logger.debug("No year found for key: %s", key)
|
||||
return None
|
||||
|
||||
|
||||
year = self._extract_year_from_soup(soup)
|
||||
if year is not None:
|
||||
self._YearDict[key] = year
|
||||
logger.debug("Found and cached year %d for key: %s", year, key)
|
||||
|
||||
return year
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("Error extracting year for key %s: %s", key, e)
|
||||
return None
|
||||
@@ -538,7 +935,7 @@ class AniworldLoader(Loader):
|
||||
"""
|
||||
logger.debug("Extracting providers from HTML for S%02dE%03d (%s)", season, episode, key)
|
||||
soup = BeautifulSoup(
|
||||
self._get_episode_html(season, episode, key).content,
|
||||
_decode_html_content(self._get_episode_html(season, episode, key).content),
|
||||
'html.parser'
|
||||
)
|
||||
providers: dict[str, dict[int, str]] = {}
|
||||
@@ -661,7 +1058,7 @@ class AniworldLoader(Loader):
|
||||
base_url = f"{self.ANIWORLD_TO}/anime/stream/{safe_slug}/"
|
||||
logger.debug("Base URL: %s", base_url)
|
||||
response = requests.get(base_url, timeout=self.DEFAULT_REQUEST_TIMEOUT)
|
||||
soup = BeautifulSoup(response.content, 'html.parser')
|
||||
soup = BeautifulSoup(_decode_html_content(response.content), 'html.parser')
|
||||
|
||||
season_meta = soup.find('meta', itemprop='numberOfSeasons')
|
||||
number_of_seasons = int(season_meta['content']) if season_meta else 0
|
||||
@@ -676,7 +1073,7 @@ class AniworldLoader(Loader):
|
||||
season_url,
|
||||
timeout=self.DEFAULT_REQUEST_TIMEOUT,
|
||||
)
|
||||
soup = BeautifulSoup(response.content, 'html.parser')
|
||||
soup = BeautifulSoup(_decode_html_content(response.content), 'html.parser')
|
||||
|
||||
episode_links = soup.find_all('a', href=True)
|
||||
unique_links = set(
|
||||
@@ -91,6 +91,17 @@ class Loader(ABC):
|
||||
Series title string
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_year(self, key: str) -> int | None:
|
||||
"""Get the release year of a series.
|
||||
|
||||
Args:
|
||||
key: Unique series identifier/key
|
||||
|
||||
Returns:
|
||||
Release year as integer, or None if year cannot be determined
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_season_episode_count(self, slug: str) -> Dict[int, int]:
|
||||
"""Get season and episode counts for a series.
|
||||
@@ -88,7 +88,10 @@ class EnhancedAniWorldLoader(Loader):
|
||||
|
||||
self.PROVIDER_HEADERS = {
|
||||
ProviderType.VIDMOLY.value: ['Referer: "https://vidmoly.to"'],
|
||||
ProviderType.DOODSTREAM.value: ['Referer: "https://dood.li/"'],
|
||||
ProviderType.DOODSTREAM.value: [
|
||||
'Referer: "https://dood.li/"',
|
||||
'Referer: "https://playmogo.com/"',
|
||||
],
|
||||
ProviderType.VOE.value: [f'User-Agent: {self.RANDOM_USER_AGENT}'],
|
||||
ProviderType.LULUVDO.value: [
|
||||
f'User-Agent: {self.LULUVDO_USER_AGENT}',
|
||||
@@ -107,6 +110,7 @@ class EnhancedAniWorldLoader(Loader):
|
||||
# Cache dictionaries
|
||||
self._KeyHTMLDict = {}
|
||||
self._EpisodeHTMLDict = {}
|
||||
self._YearDict = {}
|
||||
|
||||
# Provider manager
|
||||
self.Providers = Providers()
|
||||
@@ -566,6 +570,10 @@ class EnhancedAniWorldLoader(Loader):
|
||||
"nocheckcertificate": True,
|
||||
"socket_timeout": self.download_timeout,
|
||||
"http_chunk_size": 1024 * 1024, # 1MB chunks
|
||||
"logger": self.logger,
|
||||
# Use ffmpeg for HLS streams and transport stream format
|
||||
"downloader": "ffmpeg",
|
||||
"hls_use_mpegts": True,
|
||||
}
|
||||
if headers:
|
||||
ydl_opts['http_headers'] = headers
|
||||
@@ -659,6 +667,10 @@ class EnhancedAniWorldLoader(Loader):
|
||||
if title_span:
|
||||
span = title_span.find('span')
|
||||
if span:
|
||||
# Extract and cache year from soup if available
|
||||
year = self._ExtractYearFromSoup(soup)
|
||||
if year is not None:
|
||||
self._YearDict[key] = year
|
||||
return span.text.strip()
|
||||
|
||||
self.logger.warning("Could not extract title for key: %s", key)
|
||||
@@ -667,7 +679,62 @@ class EnhancedAniWorldLoader(Loader):
|
||||
except Exception as e:
|
||||
self.logger.error("Failed to get title for key %s: %s", key, e)
|
||||
raise RetryableError(f"Title extraction failed: {e}") from e
|
||||
|
||||
|
||||
def _ExtractYearFromSoup(self, soup: BeautifulSoup) -> int | None:
|
||||
"""Extract year from parsed BeautifulSoup.
|
||||
|
||||
Looks for 'Jahr: {year}' pattern in p tags.
|
||||
|
||||
Args:
|
||||
soup: Parsed BeautifulSoup object
|
||||
|
||||
Returns:
|
||||
Year as int or None if not found
|
||||
"""
|
||||
for p_tag in soup.find_all('p'):
|
||||
text = p_tag.get_text()
|
||||
if 'Jahr:' in text or 'Year:' in text:
|
||||
match = re.search(r'(\d{4})', text)
|
||||
if match:
|
||||
return int(match.group(1))
|
||||
|
||||
info_div = soup.find('div', class_='series-info')
|
||||
if info_div:
|
||||
text = info_div.get_text()
|
||||
match = re.search(r'\b(19\d{2}|20\d{2})\b', text)
|
||||
if match:
|
||||
return int(match.group(1))
|
||||
|
||||
return None
|
||||
|
||||
def GetYear(self, key: str) -> int | None:
|
||||
"""Get anime release year from series key.
|
||||
|
||||
Uses cached year from GetTitle if available,
|
||||
otherwise extracts and caches it.
|
||||
|
||||
Args:
|
||||
key: Series identifier
|
||||
|
||||
Returns:
|
||||
Release year or None if not found
|
||||
"""
|
||||
# Check cache first
|
||||
if key in self._YearDict:
|
||||
return self._YearDict[key]
|
||||
|
||||
# Not cached - extract from HTML
|
||||
try:
|
||||
soup = BeautifulSoup(self._GetKeyHTML(key).content, 'html.parser')
|
||||
year = self._ExtractYearFromSoup(soup)
|
||||
if year is not None:
|
||||
self._YearDict[key] = year
|
||||
return year
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning("Error extracting year for key %s: %s", key, e)
|
||||
return None
|
||||
|
||||
def GetSiteKey(self) -> str:
|
||||
"""Get site identifier."""
|
||||
return "aniworld.to"
|
||||
@@ -8,8 +8,8 @@ import asyncio
|
||||
import logging
|
||||
from typing import Any, Callable, Dict, List, Optional, TypeVar
|
||||
|
||||
from src.core.providers.health_monitor import get_health_monitor
|
||||
from src.core.providers.provider_config import DEFAULT_PROVIDERS
|
||||
from src.server.providers.health_monitor import get_health_monitor
|
||||
from src.server.providers.provider_config import DEFAULT_PROVIDERS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -7,8 +7,8 @@ import logging
|
||||
import time
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
from src.core.providers.base_provider import Loader
|
||||
from src.core.providers.health_monitor import get_health_monitor
|
||||
from src.server.providers.base_provider import Loader
|
||||
from src.server.providers.health_monitor import get_health_monitor
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -8,7 +8,7 @@ from typing import Optional
|
||||
|
||||
import structlog
|
||||
|
||||
from src.core.SeriesApp import SeriesApp
|
||||
from src.server.SeriesApp import SeriesApp
|
||||
from src.server.services.progress_service import (
|
||||
ProgressService,
|
||||
ProgressType,
|
||||
@@ -498,13 +498,19 @@ class AnimeService:
|
||||
logger.info("No series found in SeriesApp")
|
||||
return []
|
||||
|
||||
# Build NFO metadata map and filter data from database
|
||||
nfo_map = {}
|
||||
series_with_no_episodes = set()
|
||||
# Build NFO metadata map, episode dict, and filter data from database.
|
||||
# Using DB as authoritative source for episodeDict ensures that
|
||||
# episodes marked is_downloaded=True are never shown as missing,
|
||||
# even if the in-memory state is stale.
|
||||
nfo_map: dict = {}
|
||||
db_episode_dict_map: dict[str, dict[int, list[int]]] = {}
|
||||
series_with_no_episodes: set = set()
|
||||
|
||||
async with get_db_session() as db:
|
||||
# Get all series NFO metadata using service layer
|
||||
db_series_list = await AnimeSeriesService.get_all(db)
|
||||
# Single query: load all series with their episodes eagerly
|
||||
db_series_list = await AnimeSeriesService.get_all(
|
||||
db, with_episodes=True
|
||||
)
|
||||
|
||||
for db_series in db_series_list:
|
||||
nfo_created = (
|
||||
@@ -522,7 +528,23 @@ class AnimeService:
|
||||
"tmdb_id": db_series.tmdb_id,
|
||||
"tvdb_id": db_series.tvdb_id,
|
||||
"series_id": db_series.id,
|
||||
"loading_status": db_series.loading_status,
|
||||
"loading_error": db_series.loading_error,
|
||||
}
|
||||
|
||||
# Build episodeDict from DB, skipping is_downloaded=True
|
||||
# episodes so they are never shown as missing in the UI.
|
||||
ep_dict: dict[int, list[int]] = {}
|
||||
if db_series.episodes:
|
||||
for ep in db_series.episodes:
|
||||
if ep.is_downloaded:
|
||||
continue
|
||||
if ep.season not in ep_dict:
|
||||
ep_dict[ep.season] = []
|
||||
ep_dict[ep.season].append(ep.episode_number)
|
||||
for s in ep_dict:
|
||||
ep_dict[s].sort()
|
||||
db_episode_dict_map[db_series.folder] = ep_dict
|
||||
|
||||
# If filter is "missing_episodes", get series with any missing episodes
|
||||
if filter_type == "missing_episodes":
|
||||
@@ -545,7 +567,12 @@ class AnimeService:
|
||||
name = getattr(serie, "name", "")
|
||||
site = getattr(serie, "site", "")
|
||||
folder = getattr(serie, "folder", "")
|
||||
episode_dict = getattr(serie, "episodeDict", {}) or {}
|
||||
# Use DB-backed episodeDict (is_downloaded=True already filtered out)
|
||||
# with in-memory episodeDict as fallback if the series isn't in DB yet.
|
||||
episode_dict = db_episode_dict_map.get(
|
||||
folder,
|
||||
getattr(serie, "episodeDict", {}) or {}
|
||||
)
|
||||
|
||||
# Apply filter if specified
|
||||
if filter_type == "missing_episodes":
|
||||
@@ -571,6 +598,8 @@ class AnimeService:
|
||||
"tmdb_id": nfo_data.get("tmdb_id"),
|
||||
"tvdb_id": nfo_data.get("tvdb_id"),
|
||||
"series_id": nfo_data.get("series_id"),
|
||||
"loading_status": nfo_data.get("loading_status"),
|
||||
"loading_error": nfo_data.get("loading_error"),
|
||||
}
|
||||
result_list.append(series_dict)
|
||||
|
||||
@@ -815,18 +844,24 @@ class AnimeService:
|
||||
- Adds new missing episodes that are not in the database
|
||||
- Removes episodes from database that are no longer missing
|
||||
(i.e., the file has been added to the filesystem)
|
||||
- Preserves episodes marked as downloaded (is_downloaded=True)
|
||||
so download history is not lost
|
||||
"""
|
||||
from src.server.database.service import AnimeSeriesService, EpisodeService
|
||||
|
||||
# Get existing episodes from database
|
||||
# Get existing episodes from database (all episodes, including downloaded)
|
||||
existing_episodes = await EpisodeService.get_by_series(db, existing.id)
|
||||
|
||||
# Build dict of existing episodes: {season: {ep_num: episode_id}}
|
||||
# and track which ones are already downloaded
|
||||
existing_dict: dict[int, dict[int, int]] = {}
|
||||
downloaded_set: set[tuple[int, int]] = set()
|
||||
for ep in existing_episodes:
|
||||
if ep.season not in existing_dict:
|
||||
existing_dict[ep.season] = {}
|
||||
existing_dict[ep.season][ep.episode_number] = ep.id
|
||||
if ep.is_downloaded:
|
||||
downloaded_set.add((ep.season, ep.episode_number))
|
||||
|
||||
# Get new missing episodes from scan
|
||||
new_dict = serie.episodeDict or {}
|
||||
@@ -857,9 +892,22 @@ class AnimeService:
|
||||
|
||||
# Remove episodes from database that are no longer missing
|
||||
# (i.e., the episode file now exists on the filesystem)
|
||||
# BUT: preserve episodes that are already downloaded (is_downloaded=True)
|
||||
# so we don't lose download history
|
||||
for season, eps_dict in existing_dict.items():
|
||||
for ep_num, episode_id in eps_dict.items():
|
||||
if (season, ep_num) not in new_missing_set:
|
||||
# Skip already-downloaded episodes — they should stay in DB
|
||||
# with is_downloaded=True to preserve download history
|
||||
if (season, ep_num) in downloaded_set:
|
||||
logger.debug(
|
||||
"Preserving downloaded episode in database: "
|
||||
"%s S%02dE%02d",
|
||||
serie.key,
|
||||
season,
|
||||
ep_num
|
||||
)
|
||||
continue
|
||||
await EpisodeService.delete(db, episode_id)
|
||||
logger.info(
|
||||
"Removed episode from database (no longer missing): "
|
||||
@@ -889,42 +937,21 @@ class AnimeService:
|
||||
|
||||
This method is called during initialization and after rescans
|
||||
to ensure the in-memory series list is in sync with the database.
|
||||
|
||||
Only episodes where is_downloaded=False are loaded into the
|
||||
in-memory episodeDict, so downloaded episodes are not shown
|
||||
as missing.
|
||||
"""
|
||||
from src.core.entities.series import Serie
|
||||
from src.server.database.connection import get_db_session
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
|
||||
|
||||
async with get_db_session() as db:
|
||||
anime_series_list = await AnimeSeriesService.get_all(
|
||||
db, with_episodes=True
|
||||
)
|
||||
|
||||
# Convert to Serie objects
|
||||
series_list = []
|
||||
for anime_series in anime_series_list:
|
||||
# Build episode_dict from episodes relationship
|
||||
episode_dict: dict[int, list[int]] = {}
|
||||
if anime_series.episodes:
|
||||
for episode in anime_series.episodes:
|
||||
season = episode.season
|
||||
if season not in episode_dict:
|
||||
episode_dict[season] = []
|
||||
episode_dict[season].append(episode.episode_number)
|
||||
# Sort episode numbers
|
||||
for season in episode_dict:
|
||||
episode_dict[season].sort()
|
||||
|
||||
serie = Serie(
|
||||
key=anime_series.key,
|
||||
name=anime_series.name,
|
||||
site=anime_series.site,
|
||||
folder=anime_series.folder,
|
||||
episodeDict=episode_dict
|
||||
)
|
||||
series_list.append(serie)
|
||||
|
||||
# Load into SeriesApp
|
||||
self._app.load_series_from_list(series_list)
|
||||
|
||||
# Load AnimeSeries objects directly into SeriesApp
|
||||
self._app.load_series_from_list(anime_series_list)
|
||||
|
||||
async def sync_episodes_to_db(self, series_key: str) -> int:
|
||||
"""
|
||||
@@ -962,23 +989,39 @@ class AnimeService:
|
||||
logger.warning("Series not found in database: %s", series_key)
|
||||
return 0
|
||||
|
||||
# Get existing episodes from database
|
||||
# Get existing episodes from database (all, including downloaded)
|
||||
existing_episodes = await EpisodeService.get_by_series(db, series_db.id)
|
||||
|
||||
# Build dict of existing episodes: {season: {ep_num: episode_id}}
|
||||
# and track which ones are already downloaded
|
||||
existing_dict: dict[int, dict[int, int]] = {}
|
||||
downloaded_set: set[tuple[int, int]] = set()
|
||||
for ep in existing_episodes:
|
||||
if ep.season not in existing_dict:
|
||||
existing_dict[ep.season] = {}
|
||||
existing_dict[ep.season][ep.episode_number] = ep.id
|
||||
if ep.is_downloaded:
|
||||
downloaded_set.add((ep.season, ep.episode_number))
|
||||
|
||||
# Get new missing episodes from in-memory serie
|
||||
new_dict = serie.episodeDict or {}
|
||||
|
||||
# Add new missing episodes that are not in the database
|
||||
# Skip episodes that are already downloaded (is_downloaded=True)
|
||||
# so we don't re-add them as missing after they've been downloaded
|
||||
for season, episode_numbers in new_dict.items():
|
||||
existing_season_eps = existing_dict.get(season, {})
|
||||
for ep_num in episode_numbers:
|
||||
# Skip if already downloaded — don't re-add as missing
|
||||
if (season, ep_num) in downloaded_set:
|
||||
logger.debug(
|
||||
"Skipping already-downloaded episode: "
|
||||
"%s S%02dE%02d",
|
||||
series_key,
|
||||
season,
|
||||
ep_num,
|
||||
)
|
||||
continue
|
||||
if ep_num not in existing_season_eps:
|
||||
await EpisodeService.create(
|
||||
db=db,
|
||||
@@ -1014,20 +1057,23 @@ class AnimeService:
|
||||
if hasattr(self._app, 'list') and hasattr(self._app.list, 'keyDict'):
|
||||
serie = self._app.list.keyDict.get(series_key)
|
||||
if serie:
|
||||
# Convert episode dict keys to strings for JSON
|
||||
missing_episodes = {str(k): v for k, v in (serie.episodeDict or {}).items()}
|
||||
total_missing = sum(len(eps) for eps in missing_episodes.values())
|
||||
|
||||
# Fetch NFO metadata from database
|
||||
# Fetch NFO metadata and episodes from database.
|
||||
# Using DB as the authoritative source for missing_episodes
|
||||
# ensures that episodes marked is_downloaded=True are never
|
||||
# broadcast as missing, even if in-memory state is stale.
|
||||
has_nfo = False
|
||||
nfo_created_at = None
|
||||
nfo_updated_at = None
|
||||
tmdb_id = None
|
||||
tvdb_id = None
|
||||
missing_episodes: dict[str, list] = {}
|
||||
|
||||
try:
|
||||
from src.server.database.connection import get_db_session
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
from src.server.database.service import (
|
||||
AnimeSeriesService,
|
||||
EpisodeService,
|
||||
)
|
||||
|
||||
async with get_db_session() as db:
|
||||
db_series = await AnimeSeriesService.get_by_key(db, series_key)
|
||||
@@ -1043,12 +1089,31 @@ class AnimeService:
|
||||
)
|
||||
tmdb_id = db_series.tmdb_id
|
||||
tvdb_id = db_series.tvdb_id
|
||||
|
||||
# Build missing_episodes from DB, skipping is_downloaded=True
|
||||
db_episodes = await EpisodeService.get_by_series(
|
||||
db, db_series.id, only_missing=True
|
||||
)
|
||||
for ep in db_episodes:
|
||||
key_str = str(ep.season)
|
||||
if key_str not in missing_episodes:
|
||||
missing_episodes[key_str] = []
|
||||
missing_episodes[key_str].append(ep.episode_number)
|
||||
for s in missing_episodes:
|
||||
missing_episodes[s].sort()
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Could not fetch NFO data for %s: %s",
|
||||
"Could not fetch series data for %s from DB: %s",
|
||||
series_key,
|
||||
str(e)
|
||||
)
|
||||
# Fallback to in-memory state
|
||||
missing_episodes = {
|
||||
str(k): v
|
||||
for k, v in (serie.episodeDict or {}).items()
|
||||
}
|
||||
|
||||
total_missing = sum(len(eps) for eps in missing_episodes.values())
|
||||
|
||||
series_data = {
|
||||
"key": serie.key,
|
||||
@@ -1082,17 +1147,17 @@ class AnimeService:
|
||||
|
||||
async def add_series_to_db(
|
||||
self,
|
||||
serie,
|
||||
anime,
|
||||
db
|
||||
):
|
||||
"""
|
||||
Add a series to the database if it doesn't already exist.
|
||||
|
||||
Uses serie.key for identification. Creates a new AnimeSeries
|
||||
Uses anime.key for identification. Creates a new AnimeSeries
|
||||
record in the database if it doesn't already exist.
|
||||
|
||||
Args:
|
||||
serie: The Serie instance to add
|
||||
anime: The AnimeSeries instance to add
|
||||
db: Database session for async operations
|
||||
|
||||
Returns:
|
||||
@@ -1101,45 +1166,138 @@ class AnimeService:
|
||||
from src.server.database.service import AnimeSeriesService, EpisodeService
|
||||
|
||||
# Check if series already exists in DB
|
||||
existing = await AnimeSeriesService.get_by_key(db, serie.key)
|
||||
existing = await AnimeSeriesService.get_by_key(db, anime.key)
|
||||
if existing:
|
||||
logger.debug(
|
||||
"Series already exists in database: %s (key=%s)",
|
||||
serie.name,
|
||||
serie.key
|
||||
anime.name,
|
||||
anime.key
|
||||
)
|
||||
return None
|
||||
|
||||
# Create new series in database
|
||||
anime_series = await AnimeSeriesService.create(
|
||||
db=db,
|
||||
key=serie.key,
|
||||
name=serie.name,
|
||||
site=serie.site,
|
||||
folder=serie.folder,
|
||||
year=serie.year if hasattr(serie, 'year') else None,
|
||||
key=anime.key,
|
||||
name=anime.name,
|
||||
site=anime.site,
|
||||
folder=anime.folder,
|
||||
year=anime.year if hasattr(anime, 'year') else None,
|
||||
)
|
||||
|
||||
# Create Episode records for each episode in episodeDict
|
||||
if serie.episodeDict:
|
||||
for season, episode_numbers in serie.episodeDict.items():
|
||||
for episode_number in episode_numbers:
|
||||
await EpisodeService.create(
|
||||
db=db,
|
||||
series_id=anime_series.id,
|
||||
season=season,
|
||||
episode_number=episode_number,
|
||||
)
|
||||
# Create Episode records for each episode in episodes relationship
|
||||
if anime.episodes:
|
||||
for episode in anime.episodes:
|
||||
await EpisodeService.create(
|
||||
db=db,
|
||||
series_id=anime_series.id,
|
||||
season=episode.season,
|
||||
episode_number=episode.episode_number,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Added series to database: %s (key=%s, year=%s)",
|
||||
serie.name,
|
||||
serie.key,
|
||||
serie.year if hasattr(serie, 'year') else None
|
||||
anime.name,
|
||||
anime.key,
|
||||
anime.year if hasattr(anime, 'year') else None
|
||||
)
|
||||
|
||||
return anime_series
|
||||
|
||||
async def rename_folder_if_needed(
|
||||
self,
|
||||
key: str,
|
||||
current_folder: str,
|
||||
target_folder: str,
|
||||
db: Optional[AsyncSession] = None,
|
||||
) -> bool:
|
||||
"""Rename anime folder if current and target folders differ.
|
||||
|
||||
Compares current_folder with target_folder, and if different,
|
||||
renames the folder on disk using shutil.move. Updates the DB
|
||||
record and in-memory cache if rename succeeds.
|
||||
|
||||
Args:
|
||||
key: Series unique identifier
|
||||
current_folder: Current folder name (metadata from DB)
|
||||
target_folder: Desired folder name (computed with year)
|
||||
db: Optional database session for updating DB record
|
||||
|
||||
Returns:
|
||||
True if rename was performed, False if no rename needed or failed
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
|
||||
if current_folder == target_folder:
|
||||
logger.debug(
|
||||
"Folder rename not needed for %s: same folder name",
|
||||
key
|
||||
)
|
||||
return False
|
||||
|
||||
current_path = self._directory / current_folder
|
||||
target_path = self._directory / target_folder
|
||||
|
||||
if not current_path.exists():
|
||||
logger.debug(
|
||||
"Folder rename not needed for %s: current folder does not exist on disk",
|
||||
key
|
||||
)
|
||||
return False
|
||||
|
||||
if target_path.exists():
|
||||
logger.warning(
|
||||
"Cannot rename folder for %s: target path already exists: %s",
|
||||
key,
|
||||
target_path
|
||||
)
|
||||
return False
|
||||
|
||||
try:
|
||||
# Rename folder on disk
|
||||
shutil.move(str(current_path), str(target_path))
|
||||
logger.info(
|
||||
"Renamed folder for %s: %s -> %s",
|
||||
key,
|
||||
current_folder,
|
||||
target_folder
|
||||
)
|
||||
|
||||
# Update in-memory cache
|
||||
if key in self._app.list.keyDict:
|
||||
self._app.list.keyDict[key].folder = target_folder
|
||||
logger.debug(
|
||||
"Updated in-memory cache folder for %s: %s",
|
||||
key,
|
||||
target_folder
|
||||
)
|
||||
|
||||
# Update database if session provided
|
||||
if db is not None:
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
|
||||
# Look up series by key to get database ID
|
||||
series = await AnimeSeriesService.get_by_key(db, key)
|
||||
if series:
|
||||
await AnimeSeriesService.update(db, series_id=series.id, folder=target_folder)
|
||||
logger.debug(
|
||||
"Updated DB folder for %s: %s",
|
||||
key,
|
||||
target_folder
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
"Failed to rename folder for %s: %s -> %s",
|
||||
key,
|
||||
current_folder,
|
||||
target_folder
|
||||
)
|
||||
return False
|
||||
|
||||
async def contains_in_db(self, key: str, db) -> bool:
|
||||
"""
|
||||
Check if a series with the given key exists in the database.
|
||||
@@ -1462,19 +1620,17 @@ def get_anime_service(series_app: SeriesApp) -> AnimeService:
|
||||
return AnimeService(series_app)
|
||||
|
||||
|
||||
async def sync_series_from_data_files(
|
||||
async def sync_legacy_series_to_db(
|
||||
anime_directory: str,
|
||||
log_instance=None # pylint: disable=unused-argument
|
||||
) -> int:
|
||||
"""
|
||||
Sync series from data files to the database.
|
||||
One-time legacy sync: import any series from 'data' files
|
||||
not already in the database.
|
||||
|
||||
Scans the anime directory for data files and adds any new series
|
||||
to the database. Existing series are skipped (no duplicates).
|
||||
|
||||
This function is typically called during application startup to ensure
|
||||
series metadata stored in filesystem data files is available in the
|
||||
database.
|
||||
Deprecated: Series are now loaded directly from the database.
|
||||
This function remains for backwards compatibility with legacy
|
||||
file-based data during migration.
|
||||
|
||||
Args:
|
||||
anime_directory: Path to the anime directory with data files
|
||||
@@ -1486,6 +1642,11 @@ async def sync_series_from_data_files(
|
||||
"""
|
||||
# Always use structlog for structured logging with keyword arguments
|
||||
log = structlog.get_logger(__name__)
|
||||
|
||||
log.warning(
|
||||
"sync_legacy_series_to_db is deprecated. "
|
||||
"Series are now loaded directly from database."
|
||||
)
|
||||
|
||||
try:
|
||||
from src.server.database.connection import get_db_session
|
||||
@@ -1550,6 +1711,7 @@ async def sync_series_from_data_files(
|
||||
name=serie.name,
|
||||
site=serie.site,
|
||||
folder=serie.folder,
|
||||
year=serie.year if hasattr(serie, 'year') else None,
|
||||
)
|
||||
|
||||
# Create Episode records for each episode in episodeDict
|
||||
|
||||
@@ -22,7 +22,6 @@ from typing import Any, Dict, List, Optional
|
||||
|
||||
import structlog
|
||||
|
||||
from src.core.services.nfo_factory import get_nfo_factory
|
||||
from src.server.services.websocket_service import WebSocketService
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
@@ -497,112 +496,25 @@ class BackgroundLoaderService:
|
||||
raise
|
||||
|
||||
async def _load_nfo_and_images(self, task: SeriesLoadingTask, db: Any) -> bool:
|
||||
"""Load NFO file and images for a series by reusing NFOService.
|
||||
"""Load NFO file and images for a series.
|
||||
|
||||
Note: NFO service has been removed. This method now just marks
|
||||
progress as False since NFO handling moved to server layer.
|
||||
|
||||
Args:
|
||||
task: The loading task
|
||||
db: Database session
|
||||
|
||||
Returns:
|
||||
bool: True if NFO was created, False if it already existed or failed
|
||||
bool: Always False since NFO service removed
|
||||
"""
|
||||
task.status = LoadingStatus.LOADING_NFO
|
||||
await self._broadcast_status(task, "Checking NFO file...")
|
||||
await self._broadcast_status(task, "NFO loading disabled...")
|
||||
|
||||
try:
|
||||
# Check if NFOService is available
|
||||
if not self.series_app.nfo_service:
|
||||
logger.warning(
|
||||
f"NFOService not available, skipping NFO/images for {task.key}"
|
||||
)
|
||||
task.progress["nfo"] = False
|
||||
task.progress["logo"] = False
|
||||
task.progress["images"] = False
|
||||
return False
|
||||
|
||||
# Check if NFO already exists
|
||||
if self.series_app.nfo_service.has_nfo(task.folder):
|
||||
logger.info("NFO already exists for %s, skipping creation", task.key)
|
||||
|
||||
# Update task progress
|
||||
task.progress["nfo"] = True
|
||||
task.progress["logo"] = True # Assume logo exists if NFO exists
|
||||
task.progress["images"] = True # Assume images exist if NFO exists
|
||||
|
||||
# Update database with existing NFO info
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
series_db = await AnimeSeriesService.get_by_key(db, task.key)
|
||||
if series_db:
|
||||
# Only update if not already marked
|
||||
if not series_db.has_nfo:
|
||||
series_db.has_nfo = True
|
||||
series_db.nfo_created_at = datetime.now(timezone.utc)
|
||||
logger.info("Updated database with existing NFO for %s", task.key)
|
||||
if not series_db.logo_loaded:
|
||||
series_db.logo_loaded = True
|
||||
if not series_db.images_loaded:
|
||||
series_db.images_loaded = True
|
||||
await db.commit()
|
||||
|
||||
logger.info("Existing NFO found and database updated for series: %s", task.key)
|
||||
return False
|
||||
|
||||
# NFO doesn't exist, create it
|
||||
await self._broadcast_status(task, "Generating NFO file...")
|
||||
logger.info("Creating new NFO for %s", task.key)
|
||||
|
||||
# Create a fresh NFOService for this task to avoid shared TMDB session closure
|
||||
try:
|
||||
factory = get_nfo_factory()
|
||||
nfo_service = factory.create()
|
||||
except ValueError:
|
||||
logger.warning(
|
||||
"NFOService unavailable for %s, skipping NFO/images",
|
||||
task.key
|
||||
)
|
||||
task.progress["nfo"] = False
|
||||
task.progress["logo"] = False
|
||||
task.progress["images"] = False
|
||||
return False
|
||||
|
||||
try:
|
||||
nfo_path = await nfo_service.create_tvshow_nfo(
|
||||
serie_name=task.name,
|
||||
serie_folder=task.folder,
|
||||
year=task.year,
|
||||
download_poster=True,
|
||||
download_logo=True,
|
||||
download_fanart=True
|
||||
)
|
||||
finally:
|
||||
await nfo_service.close()
|
||||
|
||||
# Update task progress
|
||||
task.progress["nfo"] = True
|
||||
task.progress["logo"] = True
|
||||
task.progress["images"] = True
|
||||
|
||||
# Update database
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
series_db = await AnimeSeriesService.get_by_key(db, task.key)
|
||||
if series_db:
|
||||
series_db.has_nfo = True
|
||||
series_db.nfo_created_at = datetime.now(timezone.utc)
|
||||
series_db.logo_loaded = True
|
||||
series_db.images_loaded = True
|
||||
series_db.loading_status = "loading_nfo"
|
||||
await db.commit()
|
||||
|
||||
logger.info("NFO and images created and loaded for series: %s", task.key)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Failed to load NFO/images for %s: %s", task.key, e)
|
||||
# Don't fail the entire task if NFO fails
|
||||
task.progress["nfo"] = False
|
||||
task.progress["logo"] = False
|
||||
task.progress["images"] = False
|
||||
return False
|
||||
task.progress["nfo"] = False
|
||||
task.progress["logo"] = False
|
||||
task.progress["images"] = False
|
||||
return False
|
||||
|
||||
async def _scan_missing_episodes(self, task: SeriesLoadingTask, db: Any) -> None:
|
||||
"""Scan for missing episodes after NFO creation.
|
||||
|
||||
@@ -44,7 +44,7 @@ class ConfigService:
|
||||
"""
|
||||
|
||||
# Current configuration schema version
|
||||
CONFIG_VERSION = "1.0.0"
|
||||
CONFIG_VERSION = "1.0.1"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -144,7 +144,13 @@ class ConfigService:
|
||||
# Save configuration with version
|
||||
data = config.model_dump()
|
||||
data["version"] = self.CONFIG_VERSION
|
||||
|
||||
|
||||
# Re-serialize SchedulerConfig through its overridden model_dump so
|
||||
# that None legacy alias fields are stripped before writing to disk.
|
||||
# Pydantic converts nested models to plain dicts in model_dump() output,
|
||||
# so we call the override explicitly on the scheduler field.
|
||||
data["scheduler"] = config.scheduler.model_dump()
|
||||
|
||||
# Write to temporary file first for atomic operation
|
||||
temp_path = self.config_path.with_suffix(".tmp")
|
||||
try:
|
||||
|
||||
@@ -14,6 +14,7 @@ import uuid
|
||||
from collections import deque
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Dict, List, Optional
|
||||
|
||||
import structlog
|
||||
@@ -68,6 +69,7 @@ class DownloadService:
|
||||
progress_service: Optional progress service for tracking
|
||||
"""
|
||||
self._anime_service = anime_service
|
||||
self._directory = anime_service._directory
|
||||
self._max_retries = max_retries
|
||||
self._progress_service = progress_service or get_progress_service()
|
||||
|
||||
@@ -79,6 +81,9 @@ class DownloadService:
|
||||
self._pending_queue: deque[DownloadItem] = deque()
|
||||
# Helper dict for O(1) lookup of pending items by ID
|
||||
self._pending_items_by_id: Dict[str, DownloadItem] = {}
|
||||
# Helper dict for O(1) lookup of pending items by episode identity
|
||||
# Key: (serie_id, season, episode), Value: item ID
|
||||
self._pending_by_episode: Dict[tuple, str] = {}
|
||||
self._active_download: Optional[DownloadItem] = None
|
||||
self._completed_items: deque[DownloadItem] = deque(maxlen=100)
|
||||
self._failed_items: deque[DownloadItem] = deque(maxlen=50)
|
||||
@@ -165,6 +170,27 @@ class DownloadService:
|
||||
logger.error("Failed to save item to database: %s", e)
|
||||
return item
|
||||
|
||||
async def _set_status_in_database(
|
||||
self,
|
||||
item_id: str,
|
||||
status: str,
|
||||
) -> bool:
|
||||
"""Set status on an item in the database.
|
||||
|
||||
Args:
|
||||
item_id: Download item ID
|
||||
status: New status value
|
||||
|
||||
Returns:
|
||||
True if update succeeded
|
||||
"""
|
||||
try:
|
||||
repository = self._get_repository()
|
||||
return await repository.set_status(item_id, status)
|
||||
except Exception as e:
|
||||
logger.error("Failed to set status in database: %s", e)
|
||||
return False
|
||||
|
||||
async def _set_error_in_database(
|
||||
self,
|
||||
item_id: str,
|
||||
@@ -186,6 +212,25 @@ class DownloadService:
|
||||
logger.error("Failed to set error in database: %s", e)
|
||||
return False
|
||||
|
||||
async def _increment_retry_in_database(
|
||||
self,
|
||||
item_id: str,
|
||||
) -> bool:
|
||||
"""Increment retry count on an item in the database.
|
||||
|
||||
Args:
|
||||
item_id: Download item ID
|
||||
|
||||
Returns:
|
||||
True if update succeeded
|
||||
"""
|
||||
try:
|
||||
repository = self._get_repository()
|
||||
return await repository.increment_retry(item_id)
|
||||
except Exception as e:
|
||||
logger.error("Failed to increment retry in database: %s", e)
|
||||
return False
|
||||
|
||||
async def _delete_from_database(self, item_id: str) -> bool:
|
||||
"""Delete an item from the database.
|
||||
|
||||
@@ -207,30 +252,33 @@ class DownloadService:
|
||||
series_key: str,
|
||||
season: int,
|
||||
episode: int,
|
||||
serie_folder: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""Remove a downloaded episode from the missing episodes list.
|
||||
"""Mark a downloaded episode as downloaded instead of deleting it.
|
||||
|
||||
Called when a download completes successfully to update both:
|
||||
1. The database (Episode record deleted)
|
||||
1. The database (Episode record marked is_downloaded=True)
|
||||
2. The in-memory Serie.episodeDict and series_list cache
|
||||
|
||||
This ensures the episode no longer appears as missing in both
|
||||
the API responses and the UI immediately after download.
|
||||
the API responses and the UI immediately after download,
|
||||
while preserving the download history.
|
||||
|
||||
Args:
|
||||
series_key: Unique provider key for the series
|
||||
season: Season number
|
||||
episode: Episode number within season
|
||||
serie_folder: Series folder name (required for file_path)
|
||||
|
||||
Returns:
|
||||
True if episode was removed, False otherwise
|
||||
True if episode was updated, False otherwise
|
||||
"""
|
||||
try:
|
||||
from src.server.database.connection import get_db_session
|
||||
from src.server.database.service import EpisodeService
|
||||
from src.server.database.service import AnimeSeriesService, EpisodeService
|
||||
|
||||
logger.info(
|
||||
"Attempting to remove missing episode from DB: "
|
||||
"Attempting to mark episode as downloaded in DB: "
|
||||
"%s S%02dE%02d",
|
||||
series_key,
|
||||
season,
|
||||
@@ -238,28 +286,63 @@ class DownloadService:
|
||||
)
|
||||
|
||||
async with get_db_session() as db:
|
||||
deleted = await EpisodeService.delete_by_series_and_episode(
|
||||
# Get series by key to find series_id
|
||||
series = await AnimeSeriesService.get_by_key(db, series_key)
|
||||
if not series:
|
||||
logger.warning(
|
||||
"Series not found for key: %s", series_key
|
||||
)
|
||||
return False
|
||||
|
||||
# Get episode by series_id, season, episode_number
|
||||
ep = await EpisodeService.get_by_episode(
|
||||
db=db,
|
||||
series_key=series_key,
|
||||
series_id=series.id,
|
||||
season=season,
|
||||
episode_number=episode,
|
||||
)
|
||||
if deleted:
|
||||
if not ep:
|
||||
logger.warning(
|
||||
"Episode not found in DB: %s S%02dE%02d",
|
||||
series_key,
|
||||
season,
|
||||
episode,
|
||||
)
|
||||
return False
|
||||
|
||||
# Construct file_path if serie_folder provided
|
||||
file_path = None
|
||||
if serie_folder:
|
||||
season_folder = f"Season {season}"
|
||||
file_path = str(
|
||||
Path(self._directory) / serie_folder / season_folder
|
||||
)
|
||||
|
||||
# Mark episode as downloaded instead of deleting
|
||||
updated = await EpisodeService.mark_downloaded(
|
||||
db=db,
|
||||
episode_id=ep.id,
|
||||
file_path=file_path or "",
|
||||
)
|
||||
|
||||
if updated:
|
||||
logger.info(
|
||||
"Successfully removed episode from DB missing list: "
|
||||
"Marked episode as downloaded in DB: "
|
||||
"%s S%02dE%02d, file_path=%s",
|
||||
series_key,
|
||||
season,
|
||||
episode,
|
||||
file_path,
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Failed to mark episode as downloaded: "
|
||||
"%s S%02dE%02d",
|
||||
series_key,
|
||||
season,
|
||||
episode,
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Episode not found in DB missing list "
|
||||
"(may already be removed): %s S%02dE%02d",
|
||||
series_key,
|
||||
season,
|
||||
episode,
|
||||
)
|
||||
return False
|
||||
|
||||
# Update in-memory Serie.episodeDict so list_missing is
|
||||
# immediately consistent without a full DB reload
|
||||
@@ -270,8 +353,8 @@ class DownloadService:
|
||||
try:
|
||||
self._anime_service._cached_list_missing.cache_clear()
|
||||
logger.debug(
|
||||
"Cleared list_missing cache after removing "
|
||||
"%s S%02dE%02d",
|
||||
"Cleared list_missing cache after marking "
|
||||
"%s S%02dE%02d as downloaded",
|
||||
series_key,
|
||||
season,
|
||||
episode,
|
||||
@@ -279,10 +362,35 @@ class DownloadService:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return deleted
|
||||
# Broadcast real-time update to frontend so the series card
|
||||
# immediately reflects the new downloaded state (no longer
|
||||
# shows the episode as missing) without waiting for a full
|
||||
# reload on DOWNLOAD_COMPLETED.
|
||||
try:
|
||||
await self._anime_service._broadcast_series_updated(
|
||||
series_key
|
||||
)
|
||||
logger.debug(
|
||||
"Broadcast series_updated after marking "
|
||||
"%s S%02dE%02d as downloaded",
|
||||
series_key,
|
||||
season,
|
||||
episode,
|
||||
)
|
||||
except Exception as broadcast_exc:
|
||||
logger.warning(
|
||||
"Failed to broadcast series update after marking "
|
||||
"%s S%02dE%02d as downloaded: %s",
|
||||
series_key,
|
||||
season,
|
||||
episode,
|
||||
broadcast_exc,
|
||||
)
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Failed to remove episode from missing list: "
|
||||
"Failed to mark episode as downloaded: "
|
||||
"%s S%02dE%02d - %s",
|
||||
series_key,
|
||||
season,
|
||||
@@ -358,6 +466,27 @@ class DownloadService:
|
||||
"missing episodes remaining",
|
||||
len(app.series_list),
|
||||
)
|
||||
|
||||
# Update deprecated data file if it exists
|
||||
# DB is authoritative; data file is optional backup
|
||||
serie_folder = serie.folder
|
||||
data_path = Path(self._directory) / serie_folder / "data"
|
||||
if data_path.exists():
|
||||
try:
|
||||
import warnings
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", DeprecationWarning)
|
||||
serie.save_to_file(str(data_path))
|
||||
logger.debug(
|
||||
"Updated data file after download: %s",
|
||||
data_path,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Failed to update data file %s: %s",
|
||||
data_path,
|
||||
e,
|
||||
)
|
||||
else:
|
||||
logger.debug(
|
||||
"Episode %d not in season %d for %s, "
|
||||
@@ -409,7 +538,7 @@ class DownloadService:
|
||||
def _add_to_pending_queue(
|
||||
self, item: DownloadItem, front: bool = False
|
||||
) -> None:
|
||||
"""Add item to pending queue and update helper dict.
|
||||
"""Add item to pending queue and update helper dicts.
|
||||
|
||||
Args:
|
||||
item: Download item to add
|
||||
@@ -420,9 +549,12 @@ class DownloadService:
|
||||
else:
|
||||
self._pending_queue.append(item)
|
||||
self._pending_items_by_id[item.id] = item
|
||||
# Track by episode identity for deduplication
|
||||
ep_key = (item.serie_id, item.episode.season, item.episode.episode)
|
||||
self._pending_by_episode[ep_key] = item.id
|
||||
|
||||
def _remove_from_pending_queue(self, item_or_id: str) -> Optional[DownloadItem]: # noqa: E501
|
||||
"""Remove item from pending queue and update helper dict.
|
||||
"""Remove item from pending queue and update helper dicts.
|
||||
|
||||
Args:
|
||||
item_or_id: Item ID to remove
|
||||
@@ -442,6 +574,10 @@ class DownloadService:
|
||||
try:
|
||||
self._pending_queue.remove(item)
|
||||
del self._pending_items_by_id[item_id]
|
||||
# Clean up episode tracking
|
||||
ep_key = (item.serie_id, item.episode.season, item.episode.episode)
|
||||
if self._pending_by_episode.get(ep_key) == item_id:
|
||||
del self._pending_by_episode[ep_key]
|
||||
return item
|
||||
except (ValueError, KeyError):
|
||||
return None
|
||||
@@ -481,10 +617,35 @@ class DownloadService:
|
||||
# Initialize queue progress tracking if not already done
|
||||
await self._init_queue_progress()
|
||||
|
||||
# Filter out episodes already in pending queue
|
||||
episodes_to_add = []
|
||||
skipped_count = 0
|
||||
seen_in_batch: set = set() # Track duplicates within this batch
|
||||
for ep in episodes:
|
||||
ep_key = (serie_id, ep.season, ep.episode)
|
||||
if ep_key in self._pending_by_episode or ep_key in seen_in_batch:
|
||||
logger.debug(
|
||||
"Skipping duplicate episode in queue",
|
||||
serie_key=serie_id,
|
||||
season=ep.season,
|
||||
episode=ep.episode,
|
||||
)
|
||||
skipped_count += 1
|
||||
continue
|
||||
seen_in_batch.add(ep_key)
|
||||
episodes_to_add.append(ep)
|
||||
|
||||
if skipped_count > 0:
|
||||
logger.info(
|
||||
"Skipped %d duplicate episodes in queue",
|
||||
skipped_count,
|
||||
serie_key=serie_id,
|
||||
)
|
||||
|
||||
created_ids = []
|
||||
|
||||
try:
|
||||
for episode in episodes:
|
||||
for episode in episodes_to_add:
|
||||
item = DownloadItem(
|
||||
id=self._generate_item_id(),
|
||||
serie_id=serie_id,
|
||||
@@ -976,17 +1137,16 @@ class DownloadService:
|
||||
if item.retry_count >= self._max_retries:
|
||||
continue
|
||||
|
||||
# Move back to pending
|
||||
# Move back to pending (retry_count will be incremented
|
||||
# by _process_download when the item fails again)
|
||||
self._failed_items.remove(item)
|
||||
item.status = DownloadStatus.PENDING
|
||||
item.retry_count += 1
|
||||
item.error = None
|
||||
item.progress = None
|
||||
item.retry_count += 1
|
||||
self._add_to_pending_queue(item)
|
||||
retried_ids.append(item.id)
|
||||
|
||||
# Status is now managed in-memory only
|
||||
|
||||
logger.info(
|
||||
"Retrying failed item: item_id=%s, retry_count=%d",
|
||||
item.id,
|
||||
@@ -994,18 +1154,23 @@ class DownloadService:
|
||||
)
|
||||
|
||||
if retried_ids:
|
||||
# Notify via progress service
|
||||
queue_status = await self.get_queue_status()
|
||||
await self._progress_service.update_progress(
|
||||
progress_id="download_queue",
|
||||
message=f"Retried {len(retried_ids)} failed items",
|
||||
metadata={
|
||||
"action": "items_retried",
|
||||
"retried_ids": retried_ids,
|
||||
"queue_status": queue_status.model_dump(mode="json"),
|
||||
},
|
||||
force_broadcast=True,
|
||||
)
|
||||
# Notify via progress service if available
|
||||
try:
|
||||
queue_status = await self.get_queue_status()
|
||||
await self._progress_service.update_progress(
|
||||
progress_id="download_queue",
|
||||
message=f"Retried {len(retried_ids)} failed items",
|
||||
metadata={
|
||||
"action": "items_retried",
|
||||
"retried_ids": retried_ids,
|
||||
"queue_status": queue_status.model_dump(mode="json"),
|
||||
},
|
||||
force_broadcast=True,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Failed to broadcast retry progress: %s", e
|
||||
)
|
||||
|
||||
return retried_ids
|
||||
|
||||
@@ -1084,12 +1249,13 @@ class DownloadService:
|
||||
# Delete completed item from download queue database
|
||||
await self._delete_from_database(item.id)
|
||||
|
||||
# Remove episode from missing episodes list
|
||||
# Mark episode as downloaded in missing episodes list
|
||||
# (both database and in-memory)
|
||||
removed = await self._remove_episode_from_missing_list(
|
||||
series_key=item.serie_id,
|
||||
season=item.episode.season,
|
||||
episode=item.episode.episode,
|
||||
serie_folder=item.serie_folder,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
@@ -1144,17 +1310,35 @@ class DownloadService:
|
||||
item.status = DownloadStatus.FAILED
|
||||
item.completed_at = datetime.now(timezone.utc)
|
||||
item.error = str(e)
|
||||
|
||||
# Increment retry count in memory and database
|
||||
item.retry_count += 1
|
||||
await self._increment_retry_in_database(item.id)
|
||||
|
||||
self._failed_items.append(item)
|
||||
|
||||
# Set error in database
|
||||
await self._set_error_in_database(item.id, str(e))
|
||||
|
||||
logger.error(
|
||||
"Download failed: item_id=%s, error=%s, retry_count=%d",
|
||||
item.id,
|
||||
str(e),
|
||||
item.retry_count,
|
||||
)
|
||||
# Check if max retries exceeded - move to dead-letter
|
||||
if item.retry_count >= self._max_retries:
|
||||
await self._set_status_in_database(
|
||||
item.id, DownloadStatus.PERMANENTLY_FAILED.value
|
||||
)
|
||||
logger.error(
|
||||
"Download permanently failed after max retries: "
|
||||
"item_id=%s, error=%s, retry_count=%d",
|
||||
item.id,
|
||||
str(e),
|
||||
item.retry_count,
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
"Download failed: item_id=%s, error=%s, retry_count=%d",
|
||||
item.id,
|
||||
str(e),
|
||||
item.retry_count,
|
||||
)
|
||||
# Note: Failure is already broadcast by AnimeService
|
||||
# via ProgressService when SeriesApp fires failed event
|
||||
|
||||
|
||||
@@ -1,15 +1,22 @@
|
||||
"""Centralized initialization service for application startup and setup."""
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional
|
||||
|
||||
import structlog
|
||||
|
||||
from src.config.settings import settings
|
||||
from src.server.services.anime_service import sync_series_from_data_files
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
from src.server.services.anime_service import sync_legacy_series_to_db
|
||||
from src.server.services.setup_service import SetupService
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
# Provider site URL constant
|
||||
ANIMEWORLD_URL = "https://aniworld.to"
|
||||
|
||||
|
||||
async def _check_scan_status(
|
||||
check_method: Callable,
|
||||
@@ -99,6 +106,104 @@ async def _mark_initial_scan_completed() -> None:
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
async def _check_legacy_key_cleanup_status() -> bool:
|
||||
"""Check if legacy key file cleanup has been completed.
|
||||
|
||||
Returns:
|
||||
bool: True if cleanup was completed, False otherwise
|
||||
"""
|
||||
return await _check_scan_status(
|
||||
check_method=lambda svc, db: svc.is_legacy_key_cleanup_completed(db),
|
||||
scan_type="legacy_key_cleanup",
|
||||
log_completed_msg="Legacy key file cleanup already completed, skipping",
|
||||
log_not_completed_msg="Legacy key file cleanup not yet run, will clean up key files"
|
||||
)
|
||||
|
||||
|
||||
async def _mark_legacy_key_cleanup_completed() -> None:
|
||||
"""Mark the legacy key file cleanup as completed in system settings."""
|
||||
await _mark_scan_completed(
|
||||
mark_method=lambda svc, db: svc.mark_legacy_key_cleanup_completed(db),
|
||||
scan_type="legacy_key_cleanup"
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
async def _cleanup_legacy_key_files() -> int:
|
||||
"""Remove legacy key files from folders that already have DB entries.
|
||||
|
||||
This is a one-time cleanup task that runs at startup after legacy migration.
|
||||
It removes deprecated 'key' files that cause duplicate key errors when
|
||||
folders are renamed, since the DB is now the source of truth.
|
||||
|
||||
Returns:
|
||||
int: Number of key files deleted
|
||||
"""
|
||||
from src.server.database.connection import get_db_session
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
|
||||
logger.info("Checking for legacy key files to clean up...")
|
||||
|
||||
if not settings.anime_directory or not os.path.isdir(settings.anime_directory):
|
||||
logger.warning(
|
||||
"Anime directory not configured or does not exist, skipping legacy key cleanup"
|
||||
)
|
||||
return 0
|
||||
|
||||
deleted_count = 0
|
||||
scanned_count = 0
|
||||
|
||||
try:
|
||||
async with get_db_session() as db:
|
||||
# Get all series from DB to know which folders should have key files removed
|
||||
all_series = await AnimeSeriesService.get_all(db)
|
||||
|
||||
# Build a set of known folder names from DB
|
||||
db_folders: set[str] = {series.folder for series in all_series if series.folder}
|
||||
|
||||
for folder_name in db_folders:
|
||||
folder_path = Path(settings.anime_directory) / folder_name
|
||||
key_file = folder_path / "key"
|
||||
|
||||
if not key_file.exists():
|
||||
continue
|
||||
|
||||
scanned_count += 1
|
||||
try:
|
||||
key_file.unlink()
|
||||
deleted_count += 1
|
||||
logger.info(
|
||||
"Removed legacy key file",
|
||||
folder=folder_name,
|
||||
key_file=str(key_file)
|
||||
)
|
||||
except OSError as exc:
|
||||
logger.warning(
|
||||
"Could not remove legacy key file",
|
||||
folder=folder_name,
|
||||
key_file=str(key_file),
|
||||
error=str(exc)
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Legacy key file cleanup failed",
|
||||
error=str(e),
|
||||
exc_info=True
|
||||
)
|
||||
return deleted_count
|
||||
|
||||
logger.info(
|
||||
"Legacy key file cleanup complete",
|
||||
scanned=scanned_count,
|
||||
deleted=deleted_count
|
||||
)
|
||||
return deleted_count
|
||||
|
||||
|
||||
async def _sync_anime_folders(progress_service=None) -> int:
|
||||
"""Scan anime folders and sync series to database.
|
||||
|
||||
@@ -118,7 +223,7 @@ async def _sync_anime_folders(progress_service=None) -> int:
|
||||
metadata={"step_id": "series_sync"}
|
||||
)
|
||||
|
||||
sync_count = await sync_series_from_data_files(settings.anime_directory)
|
||||
sync_count = await sync_legacy_series_to_db(settings.anime_directory)
|
||||
logger.info("Data file sync complete. Added %d series.", sync_count)
|
||||
|
||||
if progress_service:
|
||||
@@ -152,6 +257,40 @@ async def _load_series_into_memory(progress_service=None) -> None:
|
||||
)
|
||||
|
||||
|
||||
async def _scan_folders_to_database(progress_service=None) -> int:
|
||||
"""Scan anime folders and create AnimeSeries DB records.
|
||||
|
||||
This function runs during initial setup only. It delegates to
|
||||
SetupService.run() which handles:
|
||||
1. Iterates subdirectories of anime_directory
|
||||
2. Extracts title/year from folder names (year via (YYYY) pattern)
|
||||
3. Uses provider search to resolve key field when single match found
|
||||
4. Creates AnimeSeries records for new folders
|
||||
|
||||
Args:
|
||||
progress_service: Optional ProgressService for progress updates
|
||||
|
||||
Returns:
|
||||
int: Number of new series created
|
||||
"""
|
||||
logger.info("Scanning anime folders for new series...")
|
||||
|
||||
if not settings.anime_directory or not os.path.isdir(settings.anime_directory):
|
||||
logger.info(
|
||||
"Anime directory not configured or does not exist, skipping folder scan"
|
||||
)
|
||||
return 0
|
||||
|
||||
# Use SetupService to handle the scanning and creation
|
||||
created_count = await SetupService.run()
|
||||
|
||||
logger.info(
|
||||
"Folder scan complete",
|
||||
created=created_count
|
||||
)
|
||||
return created_count
|
||||
|
||||
|
||||
async def _validate_anime_directory(progress_service=None) -> bool:
|
||||
"""Validate that anime directory is configured.
|
||||
|
||||
@@ -181,18 +320,19 @@ async def _validate_anime_directory(progress_service=None) -> bool:
|
||||
|
||||
async def perform_initial_setup(progress_service=None):
|
||||
"""Perform initial setup including series sync and scan completion marking.
|
||||
|
||||
|
||||
This function is called both during application lifespan startup
|
||||
and when the setup endpoint is completed. It ensures that:
|
||||
1. Series are synced from data files to database
|
||||
2. Initial scan is marked as completed
|
||||
3. Series are loaded into memory
|
||||
4. NFO scan is performed if configured
|
||||
5. Media scan is performed
|
||||
|
||||
1. Legacy key/data files are migrated to database (one-time)
|
||||
2. Series are synced from data files to database
|
||||
3. Initial scan is marked as completed
|
||||
4. Series are loaded into memory
|
||||
5. NFO scan is performed if configured
|
||||
6. Media scan is performed
|
||||
|
||||
Args:
|
||||
progress_service: Optional ProgressService for emitting updates
|
||||
|
||||
|
||||
Returns:
|
||||
bool: True if initialization was performed, False if skipped
|
||||
"""
|
||||
@@ -225,17 +365,29 @@ async def perform_initial_setup(progress_service=None):
|
||||
|
||||
# Perform the actual initialization
|
||||
try:
|
||||
# Scan folders and create AnimeSeries records first
|
||||
folder_scan_count = await _scan_folders_to_database(progress_service)
|
||||
if folder_scan_count > 0:
|
||||
logger.info("Created %d series from anime folders", folder_scan_count)
|
||||
|
||||
# Sync series from anime folders to database
|
||||
await _sync_anime_folders(progress_service)
|
||||
|
||||
|
||||
# Clean up legacy key files from folders that now have DB entries
|
||||
# This runs after migration/sync to ensure DB entries exist before deletion
|
||||
is_key_cleanup_done = await _check_legacy_key_cleanup_status()
|
||||
if not is_key_cleanup_done:
|
||||
await _cleanup_legacy_key_files()
|
||||
await _mark_legacy_key_cleanup_completed()
|
||||
|
||||
# Mark the initial scan as completed
|
||||
await _mark_initial_scan_completed()
|
||||
|
||||
|
||||
# Load series into memory from database
|
||||
await _load_series_into_memory(progress_service)
|
||||
|
||||
|
||||
return True
|
||||
|
||||
|
||||
except (OSError, RuntimeError, ValueError) as e:
|
||||
logger.warning("Failed to perform initial setup: %s", e)
|
||||
return False
|
||||
@@ -275,44 +427,13 @@ async def _is_nfo_scan_configured() -> bool:
|
||||
async def _execute_nfo_scan(progress_service=None) -> None:
|
||||
"""Execute the actual NFO scan with TMDB data.
|
||||
|
||||
Note: NFO service removed. This function is now a no-op stub.
|
||||
|
||||
Args:
|
||||
progress_service: Optional ProgressService for progress updates
|
||||
|
||||
Raises:
|
||||
Exception: If NFO scan fails
|
||||
progress_service: Unused. Kept to avoid breaking call-sites.
|
||||
"""
|
||||
from src.core.services.series_manager_service import SeriesManagerService
|
||||
|
||||
logger.info("Performing initial NFO scan...")
|
||||
|
||||
if progress_service:
|
||||
await progress_service.update_progress(
|
||||
progress_id="nfo_scan",
|
||||
current=25,
|
||||
message="Scanning series for NFO files...",
|
||||
metadata={"step_id": "nfo_scan"}
|
||||
)
|
||||
|
||||
manager = SeriesManagerService.from_settings()
|
||||
|
||||
if progress_service:
|
||||
await progress_service.update_progress(
|
||||
progress_id="nfo_scan",
|
||||
current=50,
|
||||
message="Processing NFO files with TMDB data...",
|
||||
metadata={"step_id": "nfo_scan"}
|
||||
)
|
||||
|
||||
await manager.scan_and_process_nfo()
|
||||
await manager.close()
|
||||
logger.info("Initial NFO scan completed")
|
||||
|
||||
if progress_service:
|
||||
await progress_service.complete_progress(
|
||||
progress_id="nfo_scan",
|
||||
message="NFO scan completed successfully",
|
||||
metadata={"step_id": "nfo_scan"}
|
||||
)
|
||||
logger.info("NFO scan skipped — NFO service removed")
|
||||
return
|
||||
|
||||
|
||||
async def perform_nfo_scan_if_needed(progress_service=None):
|
||||
@@ -377,101 +498,6 @@ async def perform_nfo_scan_if_needed(progress_service=None):
|
||||
)
|
||||
|
||||
|
||||
_NFO_REPAIR_SEMAPHORE: asyncio.Semaphore = asyncio.Semaphore(3)
|
||||
|
||||
|
||||
async def _repair_one_series(series_dir: Path, series_name: str) -> None:
|
||||
"""Repair a single series NFO in isolation.
|
||||
|
||||
Creates a fresh :class:`NFOService` and :class:`NfoRepairService` per
|
||||
invocation so that each repair owns its own ``aiohttp`` session/connector
|
||||
and concurrent tasks cannot interfere with each other.
|
||||
|
||||
A module-level semaphore (``_NFO_REPAIR_SEMAPHORE``) limits the number of
|
||||
simultaneous TMDB requests to avoid rate-limiting.
|
||||
|
||||
Any exception is caught and logged so the asyncio task never silently
|
||||
drops an unhandled error.
|
||||
|
||||
Args:
|
||||
series_dir: Absolute path to the series folder.
|
||||
series_name: Human-readable series name for log messages.
|
||||
"""
|
||||
from src.core.services.nfo_factory import NFOServiceFactory
|
||||
from src.core.services.nfo_repair_service import NfoRepairService
|
||||
|
||||
async with _NFO_REPAIR_SEMAPHORE:
|
||||
try:
|
||||
factory = NFOServiceFactory()
|
||||
nfo_service = factory.create()
|
||||
repair_service = NfoRepairService(nfo_service)
|
||||
await repair_service.repair_series(series_dir, series_name)
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
logger.error(
|
||||
"NFO repair failed for %s: %s",
|
||||
series_name,
|
||||
exc,
|
||||
)
|
||||
|
||||
|
||||
async def perform_nfo_repair_scan(background_loader=None) -> None:
|
||||
"""Scan all series folders and repair incomplete tvshow.nfo files.
|
||||
|
||||
Runs on every application startup (not guarded by a run-once DB flag).
|
||||
Checks each subfolder of ``settings.anime_directory`` for a ``tvshow.nfo``
|
||||
and calls ``_repair_one_series`` for every file with absent or empty
|
||||
required tags.
|
||||
|
||||
Each repair task creates its own isolated :class:`NFOService` /
|
||||
:class:`TMDBClient` so concurrent tasks never share an ``aiohttp``
|
||||
session — this prevents "Connector is closed" errors when many repairs
|
||||
run in parallel. A semaphore caps TMDB concurrency at 3 to stay within
|
||||
rate limits.
|
||||
|
||||
The ``background_loader`` parameter is accepted for backwards-compatibility
|
||||
but is no longer used.
|
||||
|
||||
Args:
|
||||
background_loader: Unused. Kept to avoid breaking call-sites.
|
||||
"""
|
||||
from src.core.services.nfo_repair_service import nfo_needs_repair
|
||||
|
||||
if not settings.tmdb_api_key:
|
||||
logger.warning("NFO repair scan skipped — TMDB API key not configured")
|
||||
return
|
||||
if not settings.anime_directory:
|
||||
logger.warning("NFO repair scan skipped — anime directory not configured")
|
||||
return
|
||||
anime_dir = Path(settings.anime_directory)
|
||||
if not anime_dir.is_dir():
|
||||
logger.warning("NFO repair scan skipped — anime directory not found: %s", anime_dir)
|
||||
return
|
||||
|
||||
queued = 0
|
||||
total = 0
|
||||
for series_dir in sorted(anime_dir.iterdir()):
|
||||
if not series_dir.is_dir():
|
||||
continue
|
||||
nfo_path = series_dir / "tvshow.nfo"
|
||||
if not nfo_path.exists():
|
||||
continue
|
||||
total += 1
|
||||
series_name = series_dir.name
|
||||
if nfo_needs_repair(nfo_path):
|
||||
queued += 1
|
||||
# Each task creates its own NFOService so connectors are isolated.
|
||||
asyncio.create_task(
|
||||
_repair_one_series(series_dir, series_name),
|
||||
name=f"nfo_repair:{series_name}",
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"NFO repair scan complete: %d of %d series queued for repair",
|
||||
queued,
|
||||
total,
|
||||
)
|
||||
|
||||
|
||||
async def _check_media_scan_status() -> bool:
|
||||
"""Check if initial media scan has been completed.
|
||||
|
||||
|
||||
651
src/server/services/nfo_scan_service.py
Normal file
651
src/server/services/nfo_scan_service.py
Normal file
@@ -0,0 +1,651 @@
|
||||
"""NFO scan service for validating and creating tvshow.nfo files.
|
||||
|
||||
This module provides a service layer for scanning the anime library,
|
||||
checking whether each series has a valid tvshow.nfo file, creating
|
||||
missing files, and filling in missing properties from TMDB metadata.
|
||||
|
||||
All series are identified by 'key' (provider-assigned, URL-safe
|
||||
identifier). 'folder' is used as metadata only for filesystem paths.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
import structlog
|
||||
|
||||
from src.config.settings import settings
|
||||
from src.server.nfo.nfo_generator import generate_tvshow_nfo
|
||||
from src.server.nfo.nfo_mapper import tmdb_to_nfo_model
|
||||
from src.server.nfo.nfo_models import TVShowNFO
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
class NfoScanServiceError(Exception):
|
||||
"""Service-level exception for NFO scan operations."""
|
||||
|
||||
|
||||
class NfoScanProgress:
|
||||
"""Tracks the current state of an NFO scan operation.
|
||||
|
||||
Attributes:
|
||||
scan_id: Unique identifier for this scan
|
||||
status: Current status (started, in_progress, completed, failed, cancelled)
|
||||
total: Total number of series to scan
|
||||
current: Number of series processed
|
||||
percentage: Completion percentage
|
||||
message: Human-readable progress message
|
||||
key: Current series key being processed (metadata only)
|
||||
folder: Current series folder being processed (metadata only)
|
||||
created: Number of NFO files created
|
||||
updated: Number of NFO files updated
|
||||
errors: List of error messages encountered
|
||||
started_at: When the scan started
|
||||
updated_at: When progress was last updated
|
||||
"""
|
||||
|
||||
def __init__(self, scan_id: str):
|
||||
self.scan_id = scan_id
|
||||
self.status = "started"
|
||||
self.total = 0
|
||||
self.current = 0
|
||||
self.percentage = 0.0
|
||||
self.message = "Initializing NFO scan..."
|
||||
self.key: Optional[str] = None
|
||||
self.folder: Optional[str] = None
|
||||
self.started_at = datetime.now(timezone.utc)
|
||||
self.updated_at = datetime.now(timezone.utc)
|
||||
self.created = 0
|
||||
self.updated = 0
|
||||
self.errors: List[str] = []
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
result = {
|
||||
"scan_id": self.scan_id,
|
||||
"status": self.status,
|
||||
"total": self.total,
|
||||
"current": self.current,
|
||||
"percentage": round(self.percentage, 2),
|
||||
"message": self.message,
|
||||
"started_at": self.started_at.isoformat(),
|
||||
"updated_at": self.updated_at.isoformat(),
|
||||
"created": self.created,
|
||||
"updated": self.updated,
|
||||
"errors": self.errors,
|
||||
}
|
||||
if self.key is not None:
|
||||
result["key"] = self.key
|
||||
if self.folder is not None:
|
||||
result["folder"] = self.folder
|
||||
return result
|
||||
|
||||
|
||||
class NfoScanService:
|
||||
"""Manages NFO validation and creation for anime series.
|
||||
|
||||
Scans the anime library directory, checks each series folder for
|
||||
a tvshow.nfo file, creates missing files, and fills in missing
|
||||
or empty properties from TMDB metadata.
|
||||
|
||||
Uses 'key' as the primary series identifier and 'folder' as
|
||||
metadata only for filesystem operations.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._current_scan: Optional[NfoScanProgress] = None
|
||||
self._is_scanning = False
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
# Event handlers for scan events
|
||||
self._scan_event_handlers: List[Callable[[Dict[str, Any]], None]] = []
|
||||
|
||||
logger.info("NfoScanService initialized")
|
||||
|
||||
def subscribe_to_scan_events(
|
||||
self,
|
||||
handler: Callable[[Dict[str, Any]], None],
|
||||
) -> None:
|
||||
"""Subscribe to NFO scan events."""
|
||||
self._scan_event_handlers.append(handler)
|
||||
|
||||
def unsubscribe_from_scan_events(
|
||||
self,
|
||||
handler: Callable[[Dict[str, Any]], None],
|
||||
) -> None:
|
||||
"""Unsubscribe from NFO scan events."""
|
||||
try:
|
||||
self._scan_event_handlers.remove(handler)
|
||||
except ValueError:
|
||||
logger.warning("Handler not found for unsubscribe")
|
||||
|
||||
async def _emit_scan_event(self, event_data: Dict[str, Any]) -> None:
|
||||
"""Emit scan event to all subscribers."""
|
||||
for handler in self._scan_event_handlers:
|
||||
try:
|
||||
if asyncio.iscoroutinefunction(handler):
|
||||
await handler(event_data)
|
||||
else:
|
||||
handler(event_data)
|
||||
except Exception as e:
|
||||
logger.error("NFO scan event handler error", error=str(e))
|
||||
|
||||
@property
|
||||
def is_scanning(self) -> bool:
|
||||
return self._is_scanning
|
||||
|
||||
@property
|
||||
def current_scan(self) -> Optional[NfoScanProgress]:
|
||||
return self._current_scan
|
||||
|
||||
async def scan_all(
|
||||
self,
|
||||
anime_service: Any, # AnimeService instance
|
||||
) -> Dict[str, Any]:
|
||||
"""Run NFO validation and creation across all series.
|
||||
|
||||
Args:
|
||||
anime_service: AnimeService instance for accessing series data.
|
||||
|
||||
Returns:
|
||||
Summary dict with keys: total, created, updated, errors_count,
|
||||
scan_id, and duration_seconds.
|
||||
|
||||
Raises:
|
||||
NfoScanServiceError: If a scan is already in progress.
|
||||
"""
|
||||
async with self._lock:
|
||||
if self._is_scanning:
|
||||
raise NfoScanServiceError("An NFO scan is already in progress")
|
||||
self._is_scanning = True
|
||||
|
||||
scan_id = f"nfo_scan_{id(self)}"
|
||||
scan_progress = NfoScanProgress(scan_id)
|
||||
self._current_scan = scan_progress
|
||||
|
||||
logger.info("Starting NFO scan")
|
||||
|
||||
# Emit scan started
|
||||
await self._emit_scan_event({
|
||||
"type": "nfo_scan_started",
|
||||
"scan_id": scan_id,
|
||||
"message": "NFO scan started",
|
||||
})
|
||||
|
||||
# Get all series from AnimeService
|
||||
try:
|
||||
series_list = await anime_service.list_series_with_filters()
|
||||
except Exception as exc:
|
||||
logger.error("Failed to get series list: %s", exc)
|
||||
async with self._lock:
|
||||
self._is_scanning = False
|
||||
raise NfoScanServiceError(f"Failed to get series list: {exc}") from exc
|
||||
|
||||
if not series_list:
|
||||
logger.info("No series found — NFO scan complete")
|
||||
scan_progress.status = "completed"
|
||||
scan_progress.message = "No series found"
|
||||
scan_progress.percentage = 100.0
|
||||
scan_progress.updated_at = datetime.now(timezone.utc)
|
||||
|
||||
async with self._lock:
|
||||
self._is_scanning = False
|
||||
|
||||
await self._emit_scan_event({
|
||||
"type": "nfo_scan_completed",
|
||||
"scan_id": scan_id,
|
||||
"success": True,
|
||||
"message": "No series found",
|
||||
"data": scan_progress.to_dict(),
|
||||
})
|
||||
return {
|
||||
"total": 0,
|
||||
"created": 0,
|
||||
"updated": 0,
|
||||
"errors_count": 0,
|
||||
"scan_id": scan_id,
|
||||
"duration_seconds": 0.0,
|
||||
}
|
||||
|
||||
scan_progress.total = len(series_list)
|
||||
scan_progress.status = "in_progress"
|
||||
scan_progress.message = f"Scanning {scan_progress.total} series..."
|
||||
scan_progress.updated_at = datetime.now(timezone.utc)
|
||||
|
||||
start_time = datetime.now(timezone.utc)
|
||||
errors: List[str] = []
|
||||
|
||||
for idx, series in enumerate(series_list):
|
||||
key = series.get("key", "")
|
||||
folder = series.get("folder", "")
|
||||
name = series.get("name", "")
|
||||
|
||||
scan_progress.key = key
|
||||
scan_progress.folder = folder
|
||||
scan_progress.message = f"Scanning: {name}"
|
||||
scan_progress.updated_at = datetime.now(timezone.utc)
|
||||
|
||||
await self._emit_scan_event({
|
||||
"type": "nfo_scan_progress",
|
||||
"data": scan_progress.to_dict(),
|
||||
})
|
||||
|
||||
try:
|
||||
result = await self._scan_series(key, folder, series)
|
||||
if result == "created":
|
||||
scan_progress.created += 1
|
||||
elif result == "updated":
|
||||
scan_progress.updated += 1
|
||||
except Exception as exc:
|
||||
error_msg = f"NFO scan failed for {key}: {exc}"
|
||||
logger.warning(error_msg)
|
||||
errors.append(error_msg)
|
||||
scan_progress.errors.append(error_msg)
|
||||
|
||||
scan_progress.current = idx + 1
|
||||
scan_progress.percentage = round(
|
||||
(scan_progress.current / scan_progress.total) * 100, 2
|
||||
)
|
||||
scan_progress.updated_at = datetime.now(timezone.utc)
|
||||
|
||||
end_time = datetime.now(timezone.utc)
|
||||
duration = (end_time - start_time).total_seconds()
|
||||
scan_progress.status = "completed"
|
||||
scan_progress.message = (
|
||||
f"NFO scan completed: {scan_progress.created} created, "
|
||||
f"{scan_progress.updated} updated, {len(errors)} errors"
|
||||
)
|
||||
scan_progress.percentage = 100.0
|
||||
scan_progress.updated_at = end_time
|
||||
|
||||
async with self._lock:
|
||||
self._is_scanning = False
|
||||
|
||||
logger.info(
|
||||
"NFO scan completed: total=%d created=%d updated=%d errors=%d duration=%.2fs",
|
||||
scan_progress.total,
|
||||
scan_progress.created,
|
||||
scan_progress.updated,
|
||||
len(errors),
|
||||
duration,
|
||||
)
|
||||
|
||||
await self._emit_scan_event({
|
||||
"type": "nfo_scan_completed",
|
||||
"scan_id": scan_id,
|
||||
"success": True,
|
||||
"message": scan_progress.message,
|
||||
"data": scan_progress.to_dict(),
|
||||
"statistics": {
|
||||
"total": scan_progress.total,
|
||||
"created": scan_progress.created,
|
||||
"updated": scan_progress.updated,
|
||||
"errors_count": len(errors),
|
||||
},
|
||||
})
|
||||
|
||||
return {
|
||||
"total": scan_progress.total,
|
||||
"created": scan_progress.created,
|
||||
"updated": scan_progress.updated,
|
||||
"errors_count": len(errors),
|
||||
"scan_id": scan_id,
|
||||
"duration_seconds": round(duration, 2),
|
||||
}
|
||||
|
||||
async def _scan_series(
|
||||
self,
|
||||
key: str,
|
||||
folder: str,
|
||||
series_data: Dict[str, Any],
|
||||
) -> Optional[str]:
|
||||
"""Scan and update NFO for a single series.
|
||||
|
||||
Args:
|
||||
key: Series key (primary identifier)
|
||||
folder: Series folder name (metadata for filesystem path)
|
||||
series_data: Series data dict from anime_service
|
||||
|
||||
Returns:
|
||||
"created" if new NFO was created, "updated" if existing was
|
||||
modified, None if no change needed or error occurred.
|
||||
"""
|
||||
if not folder:
|
||||
logger.debug("Skipping series with no folder: key=%s", key)
|
||||
return None
|
||||
|
||||
anime_dir = getattr(settings, "anime_directory", None)
|
||||
if not anime_dir:
|
||||
logger.warning("anime_directory not configured — skipping NFO scan")
|
||||
return None
|
||||
|
||||
series_path = os.path.join(anime_dir, folder)
|
||||
nfo_path = os.path.join(series_path, "tvshow.nfo")
|
||||
|
||||
nfo_exists = os.path.isfile(nfo_path)
|
||||
|
||||
if not nfo_exists:
|
||||
# Create new NFO
|
||||
logger.info("Creating NFO for series: %s (%s)", key, folder)
|
||||
await self._create_nfo(key, folder, series_data, nfo_path)
|
||||
await self._update_series_nfo_flag(key, has_nfo=True, nfo_path=nfo_path)
|
||||
return "created"
|
||||
|
||||
# NFO exists — check if it needs updating
|
||||
updated = await self._update_nfo_if_needed(key, folder, series_data, nfo_path)
|
||||
if updated:
|
||||
await self._update_series_nfo_flag(key, has_nfo=True, nfo_path=nfo_path)
|
||||
return "updated"
|
||||
|
||||
return None
|
||||
|
||||
async def _create_nfo(
|
||||
self,
|
||||
key: str,
|
||||
folder: str,
|
||||
series_data: Dict[str, Any],
|
||||
nfo_path: str,
|
||||
) -> None:
|
||||
"""Create a new tvshow.nfo file from TMDB metadata.
|
||||
|
||||
Args:
|
||||
key: Series key
|
||||
folder: Series folder name
|
||||
series_data: Series data from anime_service
|
||||
nfo_path: Full path to the NFO file to create
|
||||
"""
|
||||
tmdb_id = series_data.get("tmdb_id")
|
||||
|
||||
if not tmdb_id:
|
||||
logger.warning(
|
||||
"Cannot create NFO for %s: no tmdb_id available",
|
||||
key,
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
tmdb_data = await self._fetch_tmdb_data(tmdb_id)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to fetch TMDB data for %s: %s", key, exc)
|
||||
return
|
||||
|
||||
if not tmdb_data:
|
||||
logger.warning("No TMDB data for %s", key)
|
||||
return
|
||||
|
||||
nfo_model = tmdb_to_nfo_model(
|
||||
tmdb_data,
|
||||
content_ratings=None,
|
||||
get_image_url=self._make_tmdb_image_url(tmdb_id),
|
||||
image_size="original",
|
||||
)
|
||||
|
||||
xml_content = generate_tvshow_nfo(nfo_model)
|
||||
|
||||
# Ensure directory exists
|
||||
os.makedirs(os.path.dirname(nfo_path), exist_ok=True)
|
||||
|
||||
with open(nfo_path, "w", encoding="utf-8") as f:
|
||||
f.write(xml_content)
|
||||
|
||||
logger.info("Created tvshow.nfo for %s at %s", key, nfo_path)
|
||||
|
||||
await self._emit_scan_event({
|
||||
"type": "nfo_created",
|
||||
"key": key,
|
||||
"folder": folder,
|
||||
"path": nfo_path,
|
||||
})
|
||||
|
||||
async def _update_nfo_if_needed(
|
||||
self,
|
||||
key: str,
|
||||
folder: str,
|
||||
series_data: Dict[str, Any],
|
||||
nfo_path: str,
|
||||
) -> bool:
|
||||
"""Load existing NFO, check for missing fields, fill and rewrite.
|
||||
|
||||
Args:
|
||||
key: Series key
|
||||
folder: Series folder name
|
||||
series_data: Series data from anime_service
|
||||
nfo_path: Full path to the existing NFO file
|
||||
|
||||
Returns:
|
||||
True if NFO was updated, False if no changes were needed.
|
||||
"""
|
||||
try:
|
||||
from lxml import etree
|
||||
except ImportError:
|
||||
logger.warning("lxml not available — cannot update existing NFO files")
|
||||
return False
|
||||
|
||||
try:
|
||||
tree = etree.parse(nfo_path)
|
||||
root = tree.getroot()
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to parse existing NFO for %s: %s — will regenerate", key, exc)
|
||||
# Corrupt or unreadable NFO — regenerate from TMDB
|
||||
return await self._regenerate_nfo(key, folder, series_data, nfo_path)
|
||||
|
||||
# Check for missing or empty critical fields
|
||||
critical_fields = ["title", "plot", "premiered", "tmdbid"]
|
||||
missing_fields: List[str] = []
|
||||
|
||||
for field in critical_fields:
|
||||
elem = root.find(field)
|
||||
if elem is None or not elem.text or elem.text.strip() == "":
|
||||
missing_fields.append(field)
|
||||
|
||||
if not missing_fields:
|
||||
logger.debug("NFO for %s is complete — no update needed", key)
|
||||
return False
|
||||
|
||||
logger.info(
|
||||
"NFO for %s is missing fields %s — attempting to fill from TMDB",
|
||||
key,
|
||||
missing_fields,
|
||||
)
|
||||
|
||||
# Try to fill missing fields from TMDB
|
||||
tmdb_id = series_data.get("tmdb_id")
|
||||
if not tmdb_id:
|
||||
logger.warning("Cannot update NFO for %s: no tmdb_id", key)
|
||||
return False
|
||||
|
||||
try:
|
||||
tmdb_data = await self._fetch_tmdb_data(tmdb_id)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to fetch TMDB data for %s: %s", key, exc)
|
||||
return False
|
||||
|
||||
if not tmdb_data:
|
||||
return False
|
||||
|
||||
nfo_model = tmdb_to_nfo_model(
|
||||
tmdb_data,
|
||||
content_ratings=None,
|
||||
get_image_url=self._make_tmdb_image_url(tmdb_id),
|
||||
image_size="original",
|
||||
)
|
||||
|
||||
# Serialize updated model to XML and write
|
||||
xml_content = generate_tvshow_nfo(nfo_model)
|
||||
|
||||
with open(nfo_path, "w", encoding="utf-8") as f:
|
||||
f.write(xml_content)
|
||||
|
||||
logger.info("Updated NFO for %s (filled %d fields)", key, len(missing_fields))
|
||||
|
||||
await self._emit_scan_event({
|
||||
"type": "nfo_updated",
|
||||
"key": key,
|
||||
"folder": folder,
|
||||
"path": nfo_path,
|
||||
"missing_fields": missing_fields,
|
||||
})
|
||||
|
||||
return True
|
||||
|
||||
async def _regenerate_nfo(
|
||||
self,
|
||||
key: str,
|
||||
folder: str,
|
||||
series_data: Dict[str, Any],
|
||||
nfo_path: str,
|
||||
) -> bool:
|
||||
"""Regenerate NFO from scratch when existing file is corrupt."""
|
||||
tmdb_id = series_data.get("tmdb_id")
|
||||
if not tmdb_id:
|
||||
return False
|
||||
|
||||
try:
|
||||
tmdb_data = await self._fetch_tmdb_data(tmdb_id)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to fetch TMDB data for %s during regeneration: %s", key, exc)
|
||||
return False
|
||||
|
||||
if not tmdb_data:
|
||||
return False
|
||||
|
||||
nfo_model = tmdb_to_nfo_model(
|
||||
tmdb_data,
|
||||
content_ratings=None,
|
||||
get_image_url=self._make_tmdb_image_url(tmdb_id),
|
||||
image_size="original",
|
||||
)
|
||||
|
||||
xml_content = generate_tvshow_nfo(nfo_model)
|
||||
|
||||
with open(nfo_path, "w", encoding="utf-8") as f:
|
||||
f.write(xml_content)
|
||||
|
||||
logger.info("Regenerated NFO for %s", key)
|
||||
return True
|
||||
|
||||
async def _fetch_tmdb_data(self, tmdb_id: int) -> Optional[Dict[str, Any]]:
|
||||
"""Fetch series metadata from TMDB API.
|
||||
|
||||
Args:
|
||||
tmdb_id: TMDB series ID
|
||||
|
||||
Returns:
|
||||
TMDB response dict or None on failure.
|
||||
"""
|
||||
try:
|
||||
from src.server.nfo.tmdb_client import get_tmdb_client
|
||||
|
||||
client = get_tmdb_client()
|
||||
data = await client.get_series_details(tmdb_id)
|
||||
return data
|
||||
except Exception as exc:
|
||||
logger.warning("TMDB fetch failed for TMDB ID %s: %s", tmdb_id, exc)
|
||||
return None
|
||||
|
||||
def _make_tmdb_image_url(self, tmdb_id: int) -> Callable[[str, str], str]:
|
||||
"""Create a get_image_url closure bound to a TMDB account."""
|
||||
from src.server.nfo.tmdb_client import get_tmdb_image_base_url
|
||||
|
||||
base = get_tmdb_image_base_url(tmdb_id)
|
||||
|
||||
def get_image_url(path: str, size: str = "original") -> str:
|
||||
if not path:
|
||||
return ""
|
||||
return f"{base}{size}{path}"
|
||||
|
||||
return get_image_url
|
||||
|
||||
async def _update_series_nfo_flag(
|
||||
self,
|
||||
key: str,
|
||||
has_nfo: bool,
|
||||
nfo_path: str,
|
||||
) -> None:
|
||||
"""Update the has_nfo flag and nfo_path in the database.
|
||||
|
||||
Args:
|
||||
key: Series key (primary identifier)
|
||||
has_nfo: Whether the series now has an NFO file
|
||||
nfo_path: Path to the NFO file
|
||||
"""
|
||||
try:
|
||||
from src.server.database.connection import get_db_session
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
|
||||
async with get_db_session() as db:
|
||||
series = await AnimeSeriesService.get_by_key(db, key)
|
||||
if series:
|
||||
now = datetime.now(timezone.utc)
|
||||
series.has_nfo = has_nfo
|
||||
series.nfo_path = nfo_path
|
||||
if series.nfo_created_at is None:
|
||||
series.nfo_created_at = now
|
||||
series.nfo_updated_at = now
|
||||
await db.flush()
|
||||
logger.debug("Updated NFO flag for series: %s", key)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to update NFO flag for %s: %s", key, exc)
|
||||
|
||||
async def cancel_scan(self) -> bool:
|
||||
"""Cancel the current NFO scan if one is in progress.
|
||||
|
||||
Returns:
|
||||
True if scan was cancelled, False if no scan in progress.
|
||||
"""
|
||||
async with self._lock:
|
||||
if not self._is_scanning:
|
||||
return False
|
||||
|
||||
self._is_scanning = False
|
||||
|
||||
if self._current_scan:
|
||||
self._current_scan.status = "cancelled"
|
||||
self._current_scan.message = "NFO scan cancelled by user"
|
||||
self._current_scan.updated_at = datetime.now(timezone.utc)
|
||||
|
||||
if self._current_scan:
|
||||
await self._emit_scan_event({
|
||||
"type": "nfo_scan_cancelled",
|
||||
"scan_id": self._current_scan.scan_id,
|
||||
"message": "NFO scan cancelled by user",
|
||||
})
|
||||
|
||||
logger.info("NFO scan cancelled")
|
||||
return True
|
||||
|
||||
async def get_scan_status(self) -> Dict[str, Any]:
|
||||
"""Get the current NFO scan status.
|
||||
|
||||
Returns:
|
||||
Dict with is_scanning and current_scan data.
|
||||
"""
|
||||
return {
|
||||
"is_scanning": self._is_scanning,
|
||||
"current_scan": (
|
||||
self._current_scan.to_dict() if self._current_scan else None
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module-level singleton
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_nfo_scan_service: Optional[NfoScanService] = None
|
||||
|
||||
|
||||
def get_nfo_scan_service() -> NfoScanService:
|
||||
"""Return the singleton NfoScanService instance."""
|
||||
global _nfo_scan_service
|
||||
if _nfo_scan_service is None:
|
||||
_nfo_scan_service = NfoScanService()
|
||||
return _nfo_scan_service
|
||||
|
||||
|
||||
def reset_nfo_scan_service() -> None:
|
||||
"""Reset the singleton NfoScanService instance (for testing)."""
|
||||
global _nfo_scan_service
|
||||
_nfo_scan_service = None
|
||||
@@ -83,15 +83,12 @@ class QueueRepository:
|
||||
) -> DownloadItem:
|
||||
"""Convert database model to DownloadItem.
|
||||
|
||||
Note: Since the database model is simplified, status, priority,
|
||||
progress, and retry_count default to initial values.
|
||||
|
||||
Args:
|
||||
db_item: SQLAlchemy download queue item
|
||||
item_id: Optional override for item ID
|
||||
|
||||
Returns:
|
||||
Pydantic download item with default status/priority
|
||||
Pydantic download item with status/retry_count from database
|
||||
"""
|
||||
# Get episode info from the related Episode object
|
||||
episode = db_item.episode
|
||||
@@ -109,14 +106,14 @@ class QueueRepository:
|
||||
serie_folder=series.folder if series else "",
|
||||
serie_name=series.name if series else "",
|
||||
episode=episode_identifier,
|
||||
status=DownloadStatus.PENDING, # Default - managed in-memory
|
||||
priority=DownloadPriority.NORMAL, # Default - managed in-memory
|
||||
status=DownloadStatus(db_item.status), # From database
|
||||
priority=DownloadPriority.NORMAL, # Managed in-memory
|
||||
added_at=db_item.created_at or datetime.now(timezone.utc),
|
||||
started_at=db_item.started_at,
|
||||
completed_at=db_item.completed_at,
|
||||
progress=None, # Managed in-memory
|
||||
error=db_item.error_message,
|
||||
retry_count=0, # Managed in-memory
|
||||
retry_count=db_item.retry_count, # From database
|
||||
source_url=db_item.download_url,
|
||||
)
|
||||
|
||||
@@ -350,6 +347,110 @@ class QueueRepository:
|
||||
finally:
|
||||
if manage_session:
|
||||
await session.close()
|
||||
|
||||
async def set_status(
|
||||
self,
|
||||
item_id: str,
|
||||
status: str,
|
||||
db: Optional[AsyncSession] = None,
|
||||
) -> bool:
|
||||
"""Set status on a download item.
|
||||
|
||||
Args:
|
||||
item_id: Download item ID
|
||||
status: New status value
|
||||
db: Optional existing database session
|
||||
|
||||
Returns:
|
||||
True if update succeeded, False if item not found
|
||||
|
||||
Raises:
|
||||
QueueRepositoryError: If update fails
|
||||
"""
|
||||
session = db or self._db_session_factory()
|
||||
manage_session = db is None
|
||||
|
||||
try:
|
||||
result = await DownloadQueueService.set_status(
|
||||
session,
|
||||
int(item_id),
|
||||
status,
|
||||
)
|
||||
|
||||
if manage_session:
|
||||
await session.commit()
|
||||
|
||||
success = result is not None
|
||||
|
||||
if success:
|
||||
logger.debug(
|
||||
"Set status on queue item: item_id=%s, status=%s",
|
||||
item_id,
|
||||
status,
|
||||
)
|
||||
|
||||
return success
|
||||
|
||||
except ValueError:
|
||||
return False
|
||||
except Exception as e:
|
||||
if manage_session:
|
||||
await session.rollback()
|
||||
logger.error("Failed to set status: %s", e)
|
||||
raise QueueRepositoryError(f"Failed to set status: {e}") from e
|
||||
finally:
|
||||
if manage_session:
|
||||
await session.close()
|
||||
|
||||
async def increment_retry(
|
||||
self,
|
||||
item_id: str,
|
||||
db: Optional[AsyncSession] = None,
|
||||
) -> bool:
|
||||
"""Increment retry count on a download item.
|
||||
|
||||
Args:
|
||||
item_id: Download item ID
|
||||
db: Optional existing database session
|
||||
|
||||
Returns:
|
||||
True if update succeeded, False if item not found
|
||||
|
||||
Raises:
|
||||
QueueRepositoryError: If update fails
|
||||
"""
|
||||
session = db or self._db_session_factory()
|
||||
manage_session = db is None
|
||||
|
||||
try:
|
||||
result = await DownloadQueueService.increment_retry_count(
|
||||
session,
|
||||
int(item_id),
|
||||
)
|
||||
|
||||
if manage_session:
|
||||
await session.commit()
|
||||
|
||||
success = result is not None
|
||||
|
||||
if success:
|
||||
logger.debug(
|
||||
"Incremented retry count on queue item: item_id=%s",
|
||||
item_id,
|
||||
)
|
||||
|
||||
return success
|
||||
|
||||
except ValueError:
|
||||
return False
|
||||
except Exception as e:
|
||||
if manage_session:
|
||||
await session.rollback()
|
||||
logger.error("Failed to increment retry: %s", e)
|
||||
raise QueueRepositoryError(f"Failed to increment retry: {e}") from e
|
||||
finally:
|
||||
if manage_session:
|
||||
await session.close()
|
||||
|
||||
async def delete_item(
|
||||
self,
|
||||
|
||||
22
src/server/services/scheduler/__init__.py
Normal file
22
src/server/services/scheduler/__init__.py
Normal file
@@ -0,0 +1,22 @@
|
||||
"""Scheduler services package.
|
||||
|
||||
Contains scheduler orchestration:
|
||||
|
||||
- scheduler_service: Cron-based scheduler using APScheduler
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from src.server.services.scheduler.scheduler_service import (
|
||||
SchedulerService,
|
||||
SchedulerServiceError,
|
||||
get_scheduler_service,
|
||||
reset_scheduler_service,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# Scheduler
|
||||
"SchedulerService",
|
||||
"SchedulerServiceError",
|
||||
"get_scheduler_service",
|
||||
"reset_scheduler_service",
|
||||
]
|
||||
571
src/server/services/scheduler/scheduler_service.py
Normal file
571
src/server/services/scheduler/scheduler_service.py
Normal file
@@ -0,0 +1,571 @@
|
||||
"""Scheduler service for automatic library rescans.
|
||||
|
||||
Uses APScheduler's AsyncIOScheduler with CronTrigger for precise
|
||||
cron-based scheduling.
|
||||
|
||||
Jobs are held in memory (no separate scheduler database). On startup,
|
||||
if the last scan timestamp indicates a missed run (server was down at the
|
||||
scheduled cron time), a rescan is triggered immediately.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
|
||||
from src.server.models.config import SchedulerConfig
|
||||
from src.server.services.config_service import ConfigServiceError, get_config_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_JOB_ID = "scheduled_rescan"
|
||||
|
||||
# Grace period for missed jobs (1 hour — handles server downtime between
|
||||
# scheduled time and startup).
|
||||
_MISFIRE_GRACE_SECONDS = 3600
|
||||
|
||||
_AUTO_DOWNLOAD_COOLDOWN_SECONDS = 300 # 5 minutes
|
||||
|
||||
|
||||
class SchedulerServiceError(Exception):
|
||||
"""Service-level exception for scheduler operations."""
|
||||
|
||||
|
||||
class SchedulerService:
|
||||
"""Manages automatic library rescans on a cron-based schedule.
|
||||
|
||||
Uses APScheduler's AsyncIOScheduler so scheduling integrates cleanly
|
||||
with the running asyncio event loop. Supports:
|
||||
|
||||
- Cron-based scheduling (time of day + days of week)
|
||||
- Immediate manual trigger
|
||||
- Live config reloading without app restart
|
||||
|
||||
Actual rescan/folder-scan/auto-download work is handled inline.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialise the scheduler service."""
|
||||
self._is_running: bool = False
|
||||
self._scheduler: Optional[AsyncIOScheduler] = None
|
||||
self._config: Optional[SchedulerConfig] = None
|
||||
self._scan_in_progress: bool = False
|
||||
self._last_scan_time: Optional[datetime] = None
|
||||
self._last_auto_download_time: Optional[datetime] = None
|
||||
logger.info("SchedulerService initialised")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public lifecycle methods
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def start(self) -> None:
|
||||
"""Start the APScheduler with the configured cron trigger.
|
||||
|
||||
Raises:
|
||||
SchedulerServiceError: If the scheduler is already running or
|
||||
config cannot be loaded.
|
||||
"""
|
||||
logger.info("SchedulerService.start() called")
|
||||
if self._is_running:
|
||||
logger.warning("Scheduler start called but already running")
|
||||
raise SchedulerServiceError("Scheduler is already running")
|
||||
|
||||
try:
|
||||
config_service = get_config_service()
|
||||
config = config_service.load_config()
|
||||
self._config = config.scheduler
|
||||
logger.info("Scheduler config loaded successfully")
|
||||
except ConfigServiceError as exc:
|
||||
logger.error("Failed to load scheduler configuration: %s", exc)
|
||||
raise SchedulerServiceError(f"Failed to load config: {exc}") from exc
|
||||
|
||||
self._scheduler = AsyncIOScheduler()
|
||||
|
||||
if not self._config.enabled:
|
||||
logger.info("Scheduler is disabled in configuration — not adding jobs")
|
||||
self._is_running = True
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"Scheduler config loaded: enabled=%s time=%s days=%s auto_download=%s",
|
||||
self._config.enabled,
|
||||
self._config.schedule_time,
|
||||
self._config.schedule_days,
|
||||
self._config.auto_download_after_rescan,
|
||||
)
|
||||
|
||||
trigger = self._build_cron_trigger()
|
||||
if trigger is None:
|
||||
logger.warning(
|
||||
"schedule_days is empty — scheduler started but no job scheduled"
|
||||
)
|
||||
else:
|
||||
self._scheduler.add_job(
|
||||
_run_rescan_job,
|
||||
trigger=trigger,
|
||||
id=_JOB_ID,
|
||||
replace_existing=True,
|
||||
misfire_grace_time=_MISFIRE_GRACE_SECONDS,
|
||||
coalesce=True,
|
||||
)
|
||||
logger.info(
|
||||
"Scheduler started with cron trigger: time=%s days=%s",
|
||||
self._config.schedule_time,
|
||||
self._config.schedule_days,
|
||||
)
|
||||
|
||||
self._scheduler.start()
|
||||
self._is_running = True
|
||||
|
||||
# Log next scheduled run for visibility.
|
||||
job = self._scheduler.get_job(_JOB_ID)
|
||||
if job:
|
||||
next_run = job.next_run_time
|
||||
logger.info(
|
||||
"Scheduler next run: %s",
|
||||
next_run.isoformat() if next_run else None,
|
||||
)
|
||||
|
||||
# Startup misfire recovery: check if the last scan was missed while
|
||||
# the server was down.
|
||||
await self._check_missed_run()
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""Stop the APScheduler gracefully."""
|
||||
logger.info("SchedulerService.stop() called")
|
||||
if not self._is_running:
|
||||
logger.debug("Scheduler stop called but not running")
|
||||
return
|
||||
|
||||
if self._scheduler and self._scheduler.running:
|
||||
self._scheduler.shutdown(wait=False)
|
||||
logger.info("Scheduler stopped")
|
||||
else:
|
||||
logger.info("Scheduler stop: scheduler was not running")
|
||||
|
||||
self._is_running = False
|
||||
logger.info("SchedulerService stopped successfully")
|
||||
|
||||
async def ensure_started(self) -> None:
|
||||
"""Ensure the scheduler is running (idempotent).
|
||||
|
||||
If already running, returns immediately. Otherwise, starts the scheduler.
|
||||
This method is safe to call multiple times and from multiple callers.
|
||||
|
||||
Raises:
|
||||
SchedulerServiceError: If startup fails (except for already running).
|
||||
"""
|
||||
if self._is_running:
|
||||
logger.debug("Scheduler ensure_started called but already running")
|
||||
return
|
||||
|
||||
logger.info("Scheduler ensure_started: starting scheduler")
|
||||
await self.start()
|
||||
|
||||
async def trigger_rescan(self) -> bool:
|
||||
"""Manually trigger a library rescan.
|
||||
|
||||
Returns:
|
||||
True if rescan was started; False if a scan is already running.
|
||||
|
||||
Raises:
|
||||
SchedulerServiceError: If the scheduler service is not started.
|
||||
"""
|
||||
if not self._is_running:
|
||||
raise SchedulerServiceError("Scheduler is not running")
|
||||
|
||||
if self._scan_in_progress:
|
||||
logger.warning("Cannot trigger rescan: scan already in progress")
|
||||
return False
|
||||
|
||||
logger.info("Manual rescan triggered")
|
||||
await self._perform_rescan()
|
||||
return True
|
||||
|
||||
def reload_config(self, config: SchedulerConfig) -> None:
|
||||
"""Apply a new SchedulerConfig immediately.
|
||||
|
||||
If the scheduler is already running the job is rescheduled (or
|
||||
removed) without stopping the scheduler.
|
||||
|
||||
Args:
|
||||
config: New scheduler configuration to apply.
|
||||
"""
|
||||
self._config = config
|
||||
logger.info(
|
||||
"Scheduler config reloaded: enabled=%s time=%s days=%s auto_download=%s",
|
||||
config.enabled,
|
||||
config.schedule_time,
|
||||
config.schedule_days,
|
||||
config.auto_download_after_rescan,
|
||||
)
|
||||
|
||||
if not self._scheduler or not self._scheduler.running:
|
||||
return
|
||||
|
||||
if not config.enabled:
|
||||
if self._scheduler.get_job(_JOB_ID):
|
||||
self._scheduler.remove_job(_JOB_ID)
|
||||
logger.info("Scheduler job removed (disabled)")
|
||||
return
|
||||
|
||||
trigger = self._build_cron_trigger()
|
||||
if trigger is None:
|
||||
if self._scheduler.get_job(_JOB_ID):
|
||||
self._scheduler.remove_job(_JOB_ID)
|
||||
logger.warning("Scheduler job removed — schedule_days is empty")
|
||||
else:
|
||||
if self._scheduler.get_job(_JOB_ID):
|
||||
self._scheduler.reschedule_job(_JOB_ID, trigger=trigger)
|
||||
logger.info(
|
||||
"Scheduler rescheduled with cron trigger: time=%s days=%s",
|
||||
config.schedule_time,
|
||||
config.schedule_days,
|
||||
)
|
||||
else:
|
||||
self._scheduler.add_job(
|
||||
_run_rescan_job,
|
||||
trigger=trigger,
|
||||
id=_JOB_ID,
|
||||
replace_existing=True,
|
||||
misfire_grace_time=_MISFIRE_GRACE_SECONDS,
|
||||
coalesce=True,
|
||||
)
|
||||
logger.info(
|
||||
"Scheduler job added with cron trigger: time=%s days=%s",
|
||||
config.schedule_time,
|
||||
config.schedule_days,
|
||||
)
|
||||
|
||||
def get_status(self) -> dict:
|
||||
"""Return current scheduler status including cron configuration.
|
||||
|
||||
Returns:
|
||||
Dict containing scheduler state and config fields.
|
||||
"""
|
||||
next_run: Optional[str] = None
|
||||
if self._scheduler and self._scheduler.running:
|
||||
job = self._scheduler.get_job(_JOB_ID)
|
||||
if job and job.next_run_time:
|
||||
next_run = job.next_run_time.isoformat()
|
||||
|
||||
return {
|
||||
"is_running": self._is_running,
|
||||
"enabled": self._config.enabled if self._config else False,
|
||||
"interval_minutes": self._config.interval_minutes if self._config else None,
|
||||
"schedule_time": self._config.schedule_time if self._config else None,
|
||||
"schedule_days": self._config.schedule_days if self._config else [],
|
||||
"auto_download_after_rescan": (
|
||||
self._config.auto_download_after_rescan if self._config else False
|
||||
),
|
||||
"nfo_scan_after_rescan": (
|
||||
self._config.nfo_scan_after_rescan if self._config else True
|
||||
),
|
||||
"last_run": (
|
||||
self._last_scan_time.isoformat()
|
||||
if self._last_scan_time
|
||||
else None
|
||||
),
|
||||
"next_run": next_run,
|
||||
"scan_in_progress": self._scan_in_progress,
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Private helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _build_cron_trigger(self) -> Optional[CronTrigger]:
|
||||
"""Convert config fields into an APScheduler CronTrigger.
|
||||
|
||||
Returns:
|
||||
CronTrigger instance or None if schedule_days is empty.
|
||||
"""
|
||||
if not self._config or not self._config.schedule_days:
|
||||
return None
|
||||
|
||||
hour_str, minute_str = self._config.schedule_time.split(":")
|
||||
day_of_week = ",".join(self._config.schedule_days)
|
||||
|
||||
trigger = CronTrigger(
|
||||
hour=int(hour_str),
|
||||
minute=int(minute_str),
|
||||
day_of_week=day_of_week,
|
||||
)
|
||||
logger.debug(
|
||||
"CronTrigger built: hour=%s minute=%s day_of_week=%s",
|
||||
hour_str,
|
||||
minute_str,
|
||||
day_of_week,
|
||||
)
|
||||
return trigger
|
||||
|
||||
async def _check_missed_run(self) -> None:
|
||||
"""Check if a scheduled rescan was missed while the server was down.
|
||||
|
||||
Compares system_settings.last_scan_timestamp against the expected
|
||||
schedule. If the last scan is overdue (more than 24h ago for a daily
|
||||
schedule) but within the grace period, triggers an immediate rescan.
|
||||
"""
|
||||
if not self._config or not self._config.enabled:
|
||||
return
|
||||
if not self._config.schedule_days:
|
||||
return
|
||||
|
||||
try:
|
||||
from src.server.database.connection import get_db_session
|
||||
from src.server.database.system_settings_service import (
|
||||
SystemSettingsService,
|
||||
)
|
||||
|
||||
async with get_db_session() as db:
|
||||
settings = await SystemSettingsService.get_or_create(db)
|
||||
last_scan = settings.last_scan_timestamp
|
||||
|
||||
if last_scan is None:
|
||||
# Never scanned before — trigger immediately
|
||||
logger.info("No previous scan recorded — triggering immediate rescan")
|
||||
await self._perform_rescan()
|
||||
return
|
||||
|
||||
# Ensure timezone-aware comparison
|
||||
if last_scan.tzinfo is None:
|
||||
last_scan = last_scan.replace(tzinfo=timezone.utc)
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
elapsed = now - last_scan
|
||||
|
||||
# If last scan was more than 24h + grace period ago, don't trigger
|
||||
# (avoids surprise rescans after long downtime).
|
||||
max_overdue = timedelta(hours=24, seconds=_MISFIRE_GRACE_SECONDS)
|
||||
if elapsed > max_overdue:
|
||||
logger.info(
|
||||
"Last scan was %s ago (> %s) — skipping missed-run recovery",
|
||||
elapsed,
|
||||
max_overdue,
|
||||
)
|
||||
return
|
||||
|
||||
# Check if a run should have happened between last_scan and now.
|
||||
if elapsed > timedelta(hours=23):
|
||||
logger.info(
|
||||
"Missed scheduled rescan detected (last scan %s ago) — triggering now",
|
||||
elapsed,
|
||||
)
|
||||
await self._perform_rescan()
|
||||
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.warning("Missed-run check failed (non-fatal): %s", exc)
|
||||
|
||||
async def _perform_rescan(self) -> None:
|
||||
"""Execute a library rescan with auto-download and folder scan."""
|
||||
logger.info(
|
||||
"Scheduler _perform_rescan entered: scan_in_progress=%s",
|
||||
self._scan_in_progress,
|
||||
)
|
||||
if self._scan_in_progress:
|
||||
logger.warning("Skipping rescan: previous scan still in progress")
|
||||
return
|
||||
|
||||
self._scan_in_progress = True
|
||||
scan_start = datetime.now(timezone.utc)
|
||||
|
||||
try:
|
||||
await self._broadcast("scheduled_rescan_started", {"timestamp": scan_start.isoformat()})
|
||||
|
||||
# 1. Main library rescan
|
||||
await self._run_rescan()
|
||||
|
||||
# 2. NFO scan (if enabled)
|
||||
if self._config and self._config.nfo_scan_after_rescan:
|
||||
try:
|
||||
nfo_result = await self._run_nfo_scan()
|
||||
await self._broadcast("nfo_scan_started", {
|
||||
"created": nfo_result.get("created", 0),
|
||||
"updated": nfo_result.get("updated", 0),
|
||||
})
|
||||
except Exception as exc:
|
||||
logger.error("NFO scan failed: %s", exc, exc_info=True)
|
||||
await self._broadcast("nfo_scan_error", {"error": str(exc)})
|
||||
|
||||
# 3. Auto-download (if enabled)
|
||||
if self._config and self._config.auto_download_after_rescan:
|
||||
try:
|
||||
queued = await self._run_auto_download()
|
||||
await self._broadcast("auto_download_started", {"queued_count": queued})
|
||||
except Exception as exc:
|
||||
logger.error("Auto-download failed: %s", exc, exc_info=True)
|
||||
await self._broadcast("auto_download_error", {"error": str(exc)})
|
||||
|
||||
self._last_scan_time = datetime.now(timezone.utc)
|
||||
duration = (self._last_scan_time - scan_start).total_seconds()
|
||||
|
||||
await self._broadcast(
|
||||
"scheduled_rescan_completed",
|
||||
{
|
||||
"timestamp": self._last_scan_time.isoformat(),
|
||||
"duration_seconds": duration,
|
||||
},
|
||||
)
|
||||
logger.info(
|
||||
"Scheduled library rescan completed: duration=%.2fs",
|
||||
duration,
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Scheduled rescan failed: %s", exc, exc_info=True)
|
||||
await self._broadcast(
|
||||
"scheduled_rescan_error",
|
||||
{"error": str(exc), "timestamp": datetime.now(timezone.utc).isoformat()},
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
self._scan_in_progress = False
|
||||
logger.info("Scheduled rescan finished: scan_in_progress reset to False")
|
||||
|
||||
async def _run_rescan(self) -> None:
|
||||
"""Run the anime service rescan."""
|
||||
from src.server.utils.dependencies import get_anime_service
|
||||
|
||||
anime_service = get_anime_service()
|
||||
logger.info("Anime service obtained, calling anime_service.rescan()...")
|
||||
await anime_service.rescan()
|
||||
logger.info("anime_service.rescan() completed")
|
||||
|
||||
async def _run_nfo_scan(self) -> Dict[str, Any]:
|
||||
"""Run NFO validation and creation across all series."""
|
||||
from src.server.services.nfo_scan_service import get_nfo_scan_service
|
||||
from src.server.utils.dependencies import get_anime_service
|
||||
|
||||
anime_service = get_anime_service()
|
||||
nfo_scan_service = get_nfo_scan_service()
|
||||
|
||||
logger.info("Starting NFO scan...")
|
||||
result = await nfo_scan_service.scan_all(anime_service)
|
||||
logger.info(
|
||||
"NFO scan completed: created=%d updated=%d errors=%d",
|
||||
result.get("created", 0),
|
||||
result.get("updated", 0),
|
||||
result.get("errors_count", 0),
|
||||
)
|
||||
return result
|
||||
|
||||
async def _run_auto_download(self) -> int:
|
||||
"""Queue and start downloads for all series with missing episodes."""
|
||||
from src.server.models.download import EpisodeIdentifier
|
||||
from src.server.utils.dependencies import (
|
||||
get_anime_service,
|
||||
get_download_service,
|
||||
)
|
||||
|
||||
# Cooldown check
|
||||
now = datetime.now(timezone.utc)
|
||||
if self._last_auto_download_time is not None:
|
||||
elapsed = now - self._last_auto_download_time
|
||||
if elapsed < timedelta(seconds=_AUTO_DOWNLOAD_COOLDOWN_SECONDS):
|
||||
logger.debug(
|
||||
"Auto-download skipped: cooldown active (elapsed=%.1fs cooldown=%ds)",
|
||||
elapsed.total_seconds(),
|
||||
_AUTO_DOWNLOAD_COOLDOWN_SECONDS,
|
||||
)
|
||||
return 0
|
||||
|
||||
anime_service = get_anime_service()
|
||||
download_service = get_download_service()
|
||||
|
||||
series_list = anime_service._cached_list_missing()
|
||||
queued_count = 0
|
||||
|
||||
for series in series_list:
|
||||
episode_dict: dict = series.get("episodeDict") or {}
|
||||
if not episode_dict:
|
||||
continue
|
||||
|
||||
episodes: List[EpisodeIdentifier] = []
|
||||
for season_str, ep_numbers in episode_dict.items():
|
||||
for ep_num in ep_numbers:
|
||||
episodes.append(
|
||||
EpisodeIdentifier(season=int(season_str), episode=int(ep_num))
|
||||
)
|
||||
|
||||
if not episodes:
|
||||
continue
|
||||
|
||||
await download_service.add_to_queue(
|
||||
serie_id=series.get("key", ""),
|
||||
serie_folder=series.get("folder", series.get("name", "")),
|
||||
serie_name=series.get("name", ""),
|
||||
episodes=episodes,
|
||||
)
|
||||
queued_count += len(episodes)
|
||||
logger.info(
|
||||
"Auto-download queued episodes for series=%s count=%d",
|
||||
series.get("key"),
|
||||
len(episodes),
|
||||
)
|
||||
|
||||
if queued_count:
|
||||
await download_service.start_queue_processing()
|
||||
logger.info("Auto-download queue processing started: queued=%d", queued_count)
|
||||
|
||||
self._last_auto_download_time = datetime.now(timezone.utc)
|
||||
logger.info("Auto-download completed: queued_count=%d", queued_count)
|
||||
return queued_count
|
||||
|
||||
async def _broadcast(self, event_type: str, data: dict) -> None:
|
||||
"""Broadcast a WebSocket event to all connected clients."""
|
||||
try:
|
||||
from src.server.services.websocket_service import get_websocket_service
|
||||
|
||||
ws_service = get_websocket_service()
|
||||
await ws_service.manager.broadcast({"type": event_type, "data": data})
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"WebSocket broadcast failed: event=%s error=%s", event_type, exc
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module-level job runner
|
||||
#
|
||||
# APScheduler cannot serialize bound methods (SchedulerService instance
|
||||
# contains a reference to the scheduler itself, creating a circular pickle
|
||||
# error). Using a module-level function avoids this.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def _run_rescan_job() -> None:
|
||||
"""Module-level job entry point — delegates to the current service."""
|
||||
logger.info("=" * 60)
|
||||
logger.info("APScheduler triggered _run_rescan_job")
|
||||
logger.info("Getting scheduler service singleton...")
|
||||
svc = get_scheduler_service()
|
||||
logger.info("Scheduler service obtained, calling _perform_rescan()")
|
||||
await svc._perform_rescan()
|
||||
logger.info("_run_rescan_job completed")
|
||||
logger.info("=" * 60)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module-level singleton
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_scheduler_service: Optional[SchedulerService] = None
|
||||
|
||||
|
||||
def get_scheduler_service() -> SchedulerService:
|
||||
"""Return the singleton SchedulerService instance."""
|
||||
global _scheduler_service
|
||||
if _scheduler_service is None:
|
||||
logger.info("Creating new SchedulerService singleton")
|
||||
_scheduler_service = SchedulerService()
|
||||
else:
|
||||
logger.debug("Returning existing SchedulerService singleton")
|
||||
return _scheduler_service
|
||||
|
||||
|
||||
def reset_scheduler_service() -> None:
|
||||
"""Reset the singleton (used in tests)."""
|
||||
global _scheduler_service
|
||||
_scheduler_service = None
|
||||
@@ -1,384 +0,0 @@
|
||||
"""Scheduler service for automatic library rescans.
|
||||
|
||||
Uses APScheduler's AsyncIOScheduler with CronTrigger for precise
|
||||
cron-based scheduling. The legacy interval-based loop has been removed
|
||||
in favour of the cron approach.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import List, Optional
|
||||
|
||||
import structlog
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
|
||||
from src.server.models.config import SchedulerConfig
|
||||
from src.server.services.config_service import ConfigServiceError, get_config_service
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
_JOB_ID = "scheduled_rescan"
|
||||
|
||||
|
||||
class SchedulerServiceError(Exception):
|
||||
"""Service-level exception for scheduler operations."""
|
||||
|
||||
|
||||
class SchedulerService:
|
||||
"""Manages automatic library rescans on a cron-based schedule.
|
||||
|
||||
Uses APScheduler's AsyncIOScheduler so scheduling integrates cleanly
|
||||
with the running asyncio event loop. Supports:
|
||||
|
||||
- Cron-based scheduling (time of day + days of week)
|
||||
- Immediate manual trigger
|
||||
- Live config reloading without app restart
|
||||
- Auto-queueing downloads of missing episodes after rescan
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialise the scheduler service."""
|
||||
self._is_running: bool = False
|
||||
self._scheduler: Optional[AsyncIOScheduler] = None
|
||||
self._config: Optional[SchedulerConfig] = None
|
||||
self._last_scan_time: Optional[datetime] = None
|
||||
self._scan_in_progress: bool = False
|
||||
logger.info("SchedulerService initialised")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public lifecycle methods
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def start(self) -> None:
|
||||
"""Start the APScheduler with the configured cron trigger.
|
||||
|
||||
Raises:
|
||||
SchedulerServiceError: If the scheduler is already running or
|
||||
config cannot be loaded.
|
||||
"""
|
||||
if self._is_running:
|
||||
raise SchedulerServiceError("Scheduler is already running")
|
||||
|
||||
try:
|
||||
config_service = get_config_service()
|
||||
config = config_service.load_config()
|
||||
self._config = config.scheduler
|
||||
except ConfigServiceError as exc:
|
||||
logger.error("Failed to load scheduler configuration", error=str(exc))
|
||||
raise SchedulerServiceError(f"Failed to load config: {exc}") from exc
|
||||
|
||||
self._scheduler = AsyncIOScheduler()
|
||||
|
||||
if not self._config.enabled:
|
||||
logger.info("Scheduler is disabled in configuration — not adding jobs")
|
||||
self._is_running = True
|
||||
return
|
||||
|
||||
trigger = self._build_cron_trigger()
|
||||
if trigger is None:
|
||||
logger.warning(
|
||||
"schedule_days is empty — scheduler started but no job scheduled"
|
||||
)
|
||||
else:
|
||||
self._scheduler.add_job(
|
||||
self._perform_rescan,
|
||||
trigger=trigger,
|
||||
id=_JOB_ID,
|
||||
replace_existing=True,
|
||||
misfire_grace_time=300,
|
||||
)
|
||||
logger.info(
|
||||
"Scheduler started with cron trigger",
|
||||
schedule_time=self._config.schedule_time,
|
||||
schedule_days=self._config.schedule_days,
|
||||
)
|
||||
|
||||
self._scheduler.start()
|
||||
self._is_running = True
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""Stop the APScheduler gracefully."""
|
||||
if not self._is_running:
|
||||
logger.debug("Scheduler stop called but not running")
|
||||
return
|
||||
|
||||
if self._scheduler and self._scheduler.running:
|
||||
self._scheduler.shutdown(wait=False)
|
||||
logger.info("Scheduler stopped")
|
||||
|
||||
self._is_running = False
|
||||
|
||||
async def trigger_rescan(self) -> bool:
|
||||
"""Manually trigger a library rescan.
|
||||
|
||||
Returns:
|
||||
True if rescan was started; False if a scan is already running.
|
||||
|
||||
Raises:
|
||||
SchedulerServiceError: If the scheduler service is not started.
|
||||
"""
|
||||
if not self._is_running:
|
||||
raise SchedulerServiceError("Scheduler is not running")
|
||||
|
||||
if self._scan_in_progress:
|
||||
logger.warning("Cannot trigger rescan: scan already in progress")
|
||||
return False
|
||||
|
||||
logger.info("Manual rescan triggered")
|
||||
await self._perform_rescan()
|
||||
return True
|
||||
|
||||
def reload_config(self, config: SchedulerConfig) -> None:
|
||||
"""Apply a new SchedulerConfig immediately.
|
||||
|
||||
If the scheduler is already running the job is rescheduled (or
|
||||
removed) without stopping the scheduler.
|
||||
|
||||
Args:
|
||||
config: New scheduler configuration to apply.
|
||||
"""
|
||||
self._config = config
|
||||
logger.info(
|
||||
"Scheduler config reloaded",
|
||||
enabled=config.enabled,
|
||||
schedule_time=config.schedule_time,
|
||||
schedule_days=config.schedule_days,
|
||||
auto_download=config.auto_download_after_rescan,
|
||||
)
|
||||
|
||||
if not self._scheduler or not self._scheduler.running:
|
||||
return
|
||||
|
||||
if not config.enabled:
|
||||
if self._scheduler.get_job(_JOB_ID):
|
||||
self._scheduler.remove_job(_JOB_ID)
|
||||
logger.info("Scheduler job removed (disabled)")
|
||||
return
|
||||
|
||||
trigger = self._build_cron_trigger()
|
||||
if trigger is None:
|
||||
if self._scheduler.get_job(_JOB_ID):
|
||||
self._scheduler.remove_job(_JOB_ID)
|
||||
logger.warning("Scheduler job removed — schedule_days is empty")
|
||||
else:
|
||||
if self._scheduler.get_job(_JOB_ID):
|
||||
self._scheduler.reschedule_job(_JOB_ID, trigger=trigger)
|
||||
logger.info(
|
||||
"Scheduler rescheduled with cron trigger",
|
||||
schedule_time=config.schedule_time,
|
||||
schedule_days=config.schedule_days,
|
||||
)
|
||||
else:
|
||||
self._scheduler.add_job(
|
||||
self._perform_rescan,
|
||||
trigger=trigger,
|
||||
id=_JOB_ID,
|
||||
replace_existing=True,
|
||||
misfire_grace_time=300,
|
||||
)
|
||||
logger.info(
|
||||
"Scheduler job added with cron trigger",
|
||||
schedule_time=config.schedule_time,
|
||||
schedule_days=config.schedule_days,
|
||||
)
|
||||
|
||||
def get_status(self) -> dict:
|
||||
"""Return current scheduler status including cron configuration.
|
||||
|
||||
Returns:
|
||||
Dict containing scheduler state and config fields.
|
||||
"""
|
||||
next_run: Optional[str] = None
|
||||
if self._scheduler and self._scheduler.running:
|
||||
job = self._scheduler.get_job(_JOB_ID)
|
||||
if job and job.next_run_time:
|
||||
next_run = job.next_run_time.isoformat()
|
||||
|
||||
return {
|
||||
"is_running": self._is_running,
|
||||
"enabled": self._config.enabled if self._config else False,
|
||||
"interval_minutes": self._config.interval_minutes if self._config else None,
|
||||
"schedule_time": self._config.schedule_time if self._config else None,
|
||||
"schedule_days": self._config.schedule_days if self._config else [],
|
||||
"auto_download_after_rescan": (
|
||||
self._config.auto_download_after_rescan if self._config else False
|
||||
),
|
||||
"last_run": self._last_scan_time.isoformat() if self._last_scan_time else None,
|
||||
"next_run": next_run,
|
||||
"scan_in_progress": self._scan_in_progress,
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Private helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _build_cron_trigger(self) -> Optional[CronTrigger]:
|
||||
"""Convert config fields into an APScheduler CronTrigger.
|
||||
|
||||
Returns:
|
||||
CronTrigger instance or None if schedule_days is empty.
|
||||
"""
|
||||
if not self._config or not self._config.schedule_days:
|
||||
return None
|
||||
|
||||
hour_str, minute_str = self._config.schedule_time.split(":")
|
||||
day_of_week = ",".join(self._config.schedule_days)
|
||||
|
||||
trigger = CronTrigger(
|
||||
hour=int(hour_str),
|
||||
minute=int(minute_str),
|
||||
day_of_week=day_of_week,
|
||||
)
|
||||
logger.debug(
|
||||
"CronTrigger built",
|
||||
hour=hour_str,
|
||||
minute=minute_str,
|
||||
day_of_week=day_of_week,
|
||||
)
|
||||
return trigger
|
||||
|
||||
async def _broadcast(self, event_type: str, data: dict) -> None:
|
||||
"""Broadcast a WebSocket event to all connected clients."""
|
||||
try:
|
||||
from src.server.services.websocket_service import ( # noqa: PLC0415
|
||||
get_websocket_service,
|
||||
)
|
||||
|
||||
ws_service = get_websocket_service()
|
||||
await ws_service.manager.broadcast({"type": event_type, "data": data})
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.warning("WebSocket broadcast failed", event=event_type, error=str(exc))
|
||||
|
||||
async def _auto_download_missing(self) -> None:
|
||||
"""Queue and start downloads for all series with missing episodes."""
|
||||
from src.server.models.download import EpisodeIdentifier # noqa: PLC0415
|
||||
from src.server.utils.dependencies import ( # noqa: PLC0415
|
||||
get_anime_service,
|
||||
get_download_service,
|
||||
)
|
||||
|
||||
anime_service = get_anime_service()
|
||||
download_service = get_download_service()
|
||||
|
||||
series_list = anime_service._cached_list_missing()
|
||||
queued_count = 0
|
||||
|
||||
for series in series_list:
|
||||
episode_dict: dict = series.get("episodeDict") or {}
|
||||
if not episode_dict:
|
||||
continue
|
||||
|
||||
episodes: List[EpisodeIdentifier] = []
|
||||
for season_str, ep_numbers in episode_dict.items():
|
||||
for ep_num in ep_numbers:
|
||||
episodes.append(
|
||||
EpisodeIdentifier(season=int(season_str), episode=int(ep_num))
|
||||
)
|
||||
|
||||
if not episodes:
|
||||
continue
|
||||
|
||||
await download_service.add_to_queue(
|
||||
serie_id=series.get("key", ""),
|
||||
serie_folder=series.get("folder", series.get("name", "")),
|
||||
serie_name=series.get("name", ""),
|
||||
episodes=episodes,
|
||||
)
|
||||
queued_count += len(episodes)
|
||||
logger.info(
|
||||
"Auto-download queued episodes",
|
||||
series=series.get("key"),
|
||||
count=len(episodes),
|
||||
)
|
||||
|
||||
if queued_count:
|
||||
await download_service.start_queue_processing()
|
||||
logger.info("Auto-download queue processing started", queued=queued_count)
|
||||
|
||||
await self._broadcast("auto_download_started", {"queued_count": queued_count})
|
||||
logger.info("Auto-download completed", queued_count=queued_count)
|
||||
|
||||
async def _perform_rescan(self) -> None:
|
||||
"""Execute a library rescan and optionally trigger auto-download."""
|
||||
if self._scan_in_progress:
|
||||
logger.warning("Skipping rescan: previous scan still in progress")
|
||||
return
|
||||
|
||||
self._scan_in_progress = True
|
||||
scan_start = datetime.now(timezone.utc)
|
||||
|
||||
try:
|
||||
logger.info("Starting scheduled library rescan")
|
||||
|
||||
from src.server.utils.dependencies import get_anime_service # noqa: PLC0415
|
||||
|
||||
anime_service = get_anime_service()
|
||||
|
||||
await self._broadcast(
|
||||
"scheduled_rescan_started",
|
||||
{"timestamp": scan_start.isoformat()},
|
||||
)
|
||||
|
||||
await anime_service.rescan()
|
||||
|
||||
self._last_scan_time = datetime.now(timezone.utc)
|
||||
duration = (self._last_scan_time - scan_start).total_seconds()
|
||||
|
||||
logger.info("Scheduled library rescan completed", duration_seconds=duration)
|
||||
|
||||
await self._broadcast(
|
||||
"scheduled_rescan_completed",
|
||||
{
|
||||
"timestamp": self._last_scan_time.isoformat(),
|
||||
"duration_seconds": duration,
|
||||
},
|
||||
)
|
||||
|
||||
# Auto-download after rescan
|
||||
if self._config and self._config.auto_download_after_rescan:
|
||||
logger.info("Auto-download after rescan is enabled — starting")
|
||||
try:
|
||||
await self._auto_download_missing()
|
||||
except Exception as dl_exc: # pylint: disable=broad-exception-caught
|
||||
logger.error(
|
||||
"Auto-download after rescan failed",
|
||||
error=str(dl_exc),
|
||||
exc_info=True,
|
||||
)
|
||||
await self._broadcast(
|
||||
"auto_download_error", {"error": str(dl_exc)}
|
||||
)
|
||||
else:
|
||||
logger.debug("Auto-download after rescan is disabled — skipping")
|
||||
|
||||
except Exception as exc: # pylint: disable=broad-exception-caught
|
||||
logger.error("Scheduled rescan failed", error=str(exc), exc_info=True)
|
||||
await self._broadcast(
|
||||
"scheduled_rescan_error",
|
||||
{"error": str(exc), "timestamp": datetime.now(timezone.utc).isoformat()},
|
||||
)
|
||||
|
||||
finally:
|
||||
self._scan_in_progress = False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module-level singleton
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_scheduler_service: Optional[SchedulerService] = None
|
||||
|
||||
|
||||
def get_scheduler_service() -> SchedulerService:
|
||||
"""Return the singleton SchedulerService instance."""
|
||||
global _scheduler_service
|
||||
if _scheduler_service is None:
|
||||
_scheduler_service = SchedulerService()
|
||||
return _scheduler_service
|
||||
|
||||
|
||||
def reset_scheduler_service() -> None:
|
||||
"""Reset the singleton (used in tests)."""
|
||||
global _scheduler_service
|
||||
_scheduler_service = None
|
||||
441
src/server/services/setup_service.py
Normal file
441
src/server/services/setup_service.py
Normal file
@@ -0,0 +1,441 @@
|
||||
"""Setup service for first-time database initialization.
|
||||
|
||||
This service runs during initial application setup to:
|
||||
1. Scan anime folders in the data directory
|
||||
2. Extract title and year from folder names
|
||||
3. Create AnimeSeries records in the database
|
||||
4. Resolve provider keys via search (if single match found)
|
||||
|
||||
The run_once logic is handled by the caller (perform_initial_setup)
|
||||
via _check_initial_scan_status, not by this service itself.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import structlog
|
||||
|
||||
from src.config.settings import settings
|
||||
from src.server.database.connection import get_db_session
|
||||
from src.server.database.service import AnimeSeriesService, UnresolvedFolderService
|
||||
from src.server.utils.dependencies import get_series_app
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SeriesProperties:
|
||||
"""Filesystem-derived properties for an AnimeSeries."""
|
||||
has_nfo: bool = False
|
||||
nfo_path: Optional[str] = None
|
||||
nfo_created_at: Optional[datetime] = None
|
||||
nfo_updated_at: Optional[datetime] = None
|
||||
logo_loaded: bool = False
|
||||
images_loaded: bool = False
|
||||
|
||||
|
||||
class SetupService:
|
||||
"""Service for setup operations during application initialization."""
|
||||
|
||||
@staticmethod
|
||||
def _extract_year_from_folder_name(folder_name: str) -> Optional[int]:
|
||||
"""Extract year from folder name if present.
|
||||
|
||||
Looks for year in format "(YYYY)" at the end of folder name.
|
||||
|
||||
Args:
|
||||
folder_name: The folder name to parse
|
||||
|
||||
Returns:
|
||||
Year as integer if found, None otherwise
|
||||
"""
|
||||
if not folder_name:
|
||||
return None
|
||||
|
||||
match = re.search(r'\((\d{4})\)', folder_name)
|
||||
if match:
|
||||
year = int(match.group(1))
|
||||
if 1900 <= year <= 2100:
|
||||
return year
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_title_from_folder_name(folder_name: str) -> str:
|
||||
"""Extract title from folder name by removing year suffix.
|
||||
|
||||
Args:
|
||||
folder_name: The folder name to parse
|
||||
|
||||
Returns:
|
||||
Title with year suffix and surrounding whitespace removed
|
||||
"""
|
||||
return re.sub(r'\s*\(\d{4}\)\s*$', '', folder_name).strip()
|
||||
|
||||
@staticmethod
|
||||
def _normalize_title(title: str) -> str:
|
||||
"""Normalize title for fuzzy matching.
|
||||
|
||||
Strips common suffixes and lowercases for comparison.
|
||||
|
||||
Args:
|
||||
title: The title to normalize
|
||||
|
||||
Returns:
|
||||
Normalized title string
|
||||
"""
|
||||
# Remove common anime suffixes (case-insensitive)
|
||||
suffixes = [
|
||||
r'\s*\(TV\)\s*$',
|
||||
r'\s*\(Anime\)\s*$',
|
||||
r'\s*\(OAD\)\s*$',
|
||||
r'\s*\(OVA\)\s*$',
|
||||
r'\s*\(Special\)\s*$',
|
||||
r'\s*\(Movie\)\s*$',
|
||||
r'\s*\(Spin-Off\)\s*$',
|
||||
]
|
||||
normalized = title.lower().strip()
|
||||
for suffix_pattern in suffixes:
|
||||
normalized = re.sub(suffix_pattern, '', normalized, flags=re.IGNORECASE).strip()
|
||||
return normalized
|
||||
|
||||
@staticmethod
|
||||
def _titles_match(title1: str, title2: str, threshold: float = 0.85) -> bool:
|
||||
"""Check if two titles match using fuzzy comparison.
|
||||
|
||||
Args:
|
||||
title1: First title
|
||||
title2: Second title
|
||||
threshold: Similarity threshold (0.0 to 1.0)
|
||||
|
||||
Returns:
|
||||
True if titles match within threshold
|
||||
"""
|
||||
norm1 = SetupService._normalize_title(title1)
|
||||
norm2 = SetupService._normalize_title(title2)
|
||||
|
||||
# Direct match after normalization
|
||||
if norm1 == norm2:
|
||||
return True
|
||||
|
||||
# Containment check (e.g., "Attack on Titan" in "Attack on Titan (TV)")
|
||||
if norm1 in norm2 or norm2 in norm1:
|
||||
return True
|
||||
|
||||
# Similarity ratio check using SequenceMatcher
|
||||
from difflib import SequenceMatcher
|
||||
ratio = SequenceMatcher(None, norm1, norm2).ratio()
|
||||
return ratio >= threshold
|
||||
|
||||
@staticmethod
|
||||
async def _resolve_key_via_search(title: str) -> str:
|
||||
"""Resolve provider key by searching for the title.
|
||||
|
||||
Args:
|
||||
title: The title to search for
|
||||
|
||||
Returns:
|
||||
Provider key if exactly one match with same name found,
|
||||
empty string otherwise
|
||||
"""
|
||||
if not title:
|
||||
return ""
|
||||
|
||||
try:
|
||||
series_app = get_series_app()
|
||||
results = await series_app.search(title)
|
||||
|
||||
if len(results) == 1:
|
||||
result_name = results[0].get('title', '')
|
||||
result_link = results[0].get('link', '')
|
||||
|
||||
if SetupService._titles_match(result_name, title):
|
||||
if result_link and '/anime/stream/' in result_link:
|
||||
return result_link.split('/anime/stream/')[-1].split('/')[0]
|
||||
elif result_link:
|
||||
# Link is already the key (e.g., "shinobi-no-ittoki")
|
||||
return result_link
|
||||
else:
|
||||
logger.debug(
|
||||
"Series key resolved but link format unexpected",
|
||||
folder_title=title,
|
||||
result_title=result_name,
|
||||
link=result_link
|
||||
)
|
||||
else:
|
||||
logger.debug(
|
||||
"Series search result title mismatch",
|
||||
folder_title=title,
|
||||
result_title=result_name,
|
||||
link=result_link
|
||||
)
|
||||
elif len(results) > 1:
|
||||
logger.debug(
|
||||
"Multiple search results for title, skipping fuzzy match",
|
||||
title=title,
|
||||
result_count=len(results)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Provider search failed for folder",
|
||||
title=title,
|
||||
error=str(e)
|
||||
)
|
||||
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _check_nfo_file(folder_path: Path) -> tuple[bool, Optional[str], Optional[datetime], Optional[datetime]]:
|
||||
"""Check if tvshow.nfo exists and return its metadata.
|
||||
|
||||
Args:
|
||||
folder_path: Path to the series folder
|
||||
|
||||
Returns:
|
||||
Tuple of (has_nfo, nfo_path, nfo_created_at, nfo_updated_at)
|
||||
"""
|
||||
nfo_path = folder_path / "tvshow.nfo"
|
||||
if nfo_path.is_file():
|
||||
stat = nfo_path.stat()
|
||||
created = datetime.fromtimestamp(stat.st_ctime, tz=timezone.utc)
|
||||
updated = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)
|
||||
return True, str(nfo_path), created, updated
|
||||
return False, None, None, None
|
||||
|
||||
@staticmethod
|
||||
def _check_logo_file(folder_path: Path) -> bool:
|
||||
"""Check if logo.png exists.
|
||||
|
||||
Args:
|
||||
folder_path: Path to the series folder
|
||||
|
||||
Returns:
|
||||
True if logo.png exists, False otherwise
|
||||
"""
|
||||
return (folder_path / "logo.png").is_file()
|
||||
|
||||
@staticmethod
|
||||
def _check_image_files(folder_path: Path) -> bool:
|
||||
"""Check if any image files (poster, fanart) exist.
|
||||
|
||||
Args:
|
||||
folder_path: Path to the series folder
|
||||
|
||||
Returns:
|
||||
True if any poster.jpg/jpeg/png or fanart.jpg/jpeg/png exists
|
||||
"""
|
||||
image_extensions = {'.jpg', '.jpeg', '.png'}
|
||||
for child in folder_path.iterdir():
|
||||
if child.is_file():
|
||||
name_lower = child.name.lower()
|
||||
if name_lower.startswith(('poster', 'fanart')) and child.suffix.lower() in image_extensions:
|
||||
return True
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def _get_series_properties(cls, folder_path: Path) -> SeriesProperties:
|
||||
"""Get all filesystem-derived properties for a series folder.
|
||||
|
||||
Args:
|
||||
folder_path: Path to the series folder
|
||||
|
||||
Returns:
|
||||
SeriesProperties with all detected values
|
||||
"""
|
||||
has_nfo, nfo_path, nfo_created_at, nfo_updated_at = cls._check_nfo_file(folder_path)
|
||||
logo_loaded = cls._check_logo_file(folder_path)
|
||||
images_loaded = cls._check_image_files(folder_path)
|
||||
|
||||
return SeriesProperties(
|
||||
has_nfo=has_nfo,
|
||||
nfo_path=nfo_path,
|
||||
nfo_created_at=nfo_created_at,
|
||||
nfo_updated_at=nfo_updated_at,
|
||||
logo_loaded=logo_loaded,
|
||||
images_loaded=images_loaded,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
async def run(cls) -> int:
|
||||
"""Run the setup service.
|
||||
|
||||
Scans anime folders, creates AnimeSeries records, and resolves
|
||||
provider keys via search. Should only be called after checking
|
||||
that initial scan hasn't been completed yet (via _check_initial_scan_status).
|
||||
|
||||
Returns:
|
||||
Number of new series created
|
||||
"""
|
||||
if not settings.anime_directory:
|
||||
logger.info("Anime directory not configured, skipping setup")
|
||||
return 0
|
||||
|
||||
anime_dir = Path(settings.anime_directory)
|
||||
if not anime_dir.is_dir():
|
||||
logger.info(
|
||||
"Anime directory does not exist, skipping setup: %s",
|
||||
anime_dir
|
||||
)
|
||||
return 0
|
||||
|
||||
logger.info("Running setup service...")
|
||||
|
||||
created_count = 0
|
||||
skipped_existing = 0
|
||||
unresolved_count = 0
|
||||
|
||||
try:
|
||||
series_app = get_series_app()
|
||||
|
||||
async with get_db_session() as db:
|
||||
for folder in anime_dir.iterdir():
|
||||
if not folder.is_dir():
|
||||
continue
|
||||
|
||||
folder_name = folder.name
|
||||
|
||||
# Check if series already exists in DB
|
||||
existing = await AnimeSeriesService.get_by_folder(
|
||||
db, folder_name
|
||||
)
|
||||
if existing:
|
||||
skipped_existing += 1
|
||||
continue
|
||||
|
||||
# Check if already tracked as unresolved
|
||||
existing_unresolved = await UnresolvedFolderService.get_by_folder_name(
|
||||
db, folder_name
|
||||
)
|
||||
if existing_unresolved and existing_unresolved.is_resolved:
|
||||
# Was previously unresolved but now resolved - create the series
|
||||
resolved_key = existing_unresolved.provider_key
|
||||
year = cls._extract_year_from_folder_name(folder_name)
|
||||
title = cls._extract_title_from_folder_name(folder_name)
|
||||
props = cls._get_series_properties(folder)
|
||||
|
||||
series = await AnimeSeriesService.create(
|
||||
db=db,
|
||||
key=resolved_key,
|
||||
name=title,
|
||||
site="https://aniworld.to",
|
||||
folder=folder_name,
|
||||
year=year,
|
||||
loading_status="completed",
|
||||
episodes_loaded=True,
|
||||
logo_loaded=props.logo_loaded,
|
||||
images_loaded=props.images_loaded,
|
||||
has_nfo=props.has_nfo,
|
||||
nfo_path=props.nfo_path,
|
||||
nfo_created_at=props.nfo_created_at,
|
||||
nfo_updated_at=props.nfo_updated_at,
|
||||
)
|
||||
created_count += 1
|
||||
|
||||
# Delete the unresolved tracking now that series is created
|
||||
await UnresolvedFolderService.delete(db, folder_name)
|
||||
continue
|
||||
elif existing_unresolved:
|
||||
# Already tracked as unresolved, skip
|
||||
unresolved_count += 1
|
||||
continue
|
||||
|
||||
# Extract title and year from folder name
|
||||
year = cls._extract_year_from_folder_name(folder_name)
|
||||
title = cls._extract_title_from_folder_name(folder_name)
|
||||
|
||||
if not title:
|
||||
logger.warning(
|
||||
"Could not extract title from folder: %s",
|
||||
folder_name
|
||||
)
|
||||
continue
|
||||
|
||||
# Resolve key via provider search
|
||||
resolved_key = await cls._resolve_key_via_search(title)
|
||||
|
||||
if not resolved_key:
|
||||
# Track unresolved folder for later manual resolution
|
||||
import json
|
||||
try:
|
||||
series_results = await series_app.search(title)
|
||||
search_result_json = json.dumps(series_results) if series_results else None
|
||||
except Exception:
|
||||
search_result_json = None
|
||||
|
||||
await UnresolvedFolderService.create(
|
||||
db=db,
|
||||
folder_name=folder_name,
|
||||
title=title,
|
||||
year=year,
|
||||
search_attempts=1,
|
||||
last_search_result=search_result_json,
|
||||
)
|
||||
logger.warning(
|
||||
"Could not resolve series key for folder, tracking as unresolved: %s",
|
||||
folder_name
|
||||
)
|
||||
continue
|
||||
|
||||
# Also check if a series with this key already exists (different folder, same anime)
|
||||
existing_by_key = await AnimeSeriesService.get_by_key(db, resolved_key)
|
||||
if existing_by_key:
|
||||
logger.debug(
|
||||
"Series with key already exists, skipping",
|
||||
folder=folder_name,
|
||||
key=resolved_key,
|
||||
existing_folder=existing_by_key.folder
|
||||
)
|
||||
skipped_existing += 1
|
||||
continue
|
||||
|
||||
# Check filesystem properties
|
||||
props = cls._get_series_properties(folder)
|
||||
|
||||
# Create AnimeSeries record
|
||||
series = await AnimeSeriesService.create(
|
||||
db=db,
|
||||
key=resolved_key,
|
||||
name=title,
|
||||
site="https://aniworld.to",
|
||||
folder=folder_name,
|
||||
year=year,
|
||||
loading_status="completed",
|
||||
episodes_loaded=True,
|
||||
logo_loaded=props.logo_loaded,
|
||||
images_loaded=props.images_loaded,
|
||||
has_nfo=props.has_nfo,
|
||||
nfo_path=props.nfo_path,
|
||||
nfo_created_at=props.nfo_created_at,
|
||||
nfo_updated_at=props.nfo_updated_at,
|
||||
)
|
||||
created_count += 1
|
||||
|
||||
logger.debug(
|
||||
"Created series from folder",
|
||||
folder=folder_name,
|
||||
title=title,
|
||||
year=year,
|
||||
key=resolved_key or "(unresolved)",
|
||||
has_nfo=props.has_nfo,
|
||||
logo_loaded=props.logo_loaded,
|
||||
images_loaded=props.images_loaded,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Setup complete",
|
||||
created=created_count,
|
||||
skipped_existing=skipped_existing,
|
||||
unresolved=unresolved_count
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Setup failed",
|
||||
error=str(e),
|
||||
exc_info=True
|
||||
)
|
||||
return created_count
|
||||
|
||||
return created_count
|
||||
@@ -20,7 +20,7 @@ except Exception: # pragma: no cover - optional dependency
|
||||
AsyncSession = object
|
||||
|
||||
from src.config.settings import settings
|
||||
from src.core.SeriesApp import SeriesApp
|
||||
from src.server.SeriesApp import SeriesApp
|
||||
from src.server.services.auth_service import AuthError, auth_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -57,6 +57,35 @@ _rate_limit_lock = Lock()
|
||||
_RATE_LIMIT_WINDOW_SECONDS = 60.0
|
||||
|
||||
|
||||
def _make_db_lookup():
|
||||
"""Build a synchronous ``(folder) -> AnimeSeries | None`` callable for SerieScanner.
|
||||
|
||||
The returned function opens a short-lived sync DB session, queries for a
|
||||
series whose ``folder`` column matches the given name, and returns the
|
||||
AnimeSeries ORM object. Returns ``None`` when the DB is not yet initialised
|
||||
or no matching row is found.
|
||||
"""
|
||||
from src.server.database.models import AnimeSeries
|
||||
|
||||
def _lookup(folder: str) -> Optional["AnimeSeries"]:
|
||||
try:
|
||||
from src.server.database.connection import get_sync_session
|
||||
from src.server.database.service import AnimeSeriesService
|
||||
|
||||
db = get_sync_session()
|
||||
try:
|
||||
row = AnimeSeriesService.get_by_folder_sync(db, folder)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
return row
|
||||
except RuntimeError:
|
||||
# DB not initialised yet (e.g. first boot before init_db())
|
||||
return None
|
||||
|
||||
return _lookup
|
||||
|
||||
|
||||
def get_series_app() -> SeriesApp:
|
||||
"""
|
||||
Dependency to get SeriesApp instance.
|
||||
|
||||
248
src/server/utils/key_utils.py
Normal file
248
src/server/utils/key_utils.py
Normal file
@@ -0,0 +1,248 @@
|
||||
"""Utility functions for generating URL-safe keys from folder names.
|
||||
|
||||
This module provides key generation and normalization for anime series,
|
||||
handling edge cases like non-Latin characters and special symbols.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
import uuid
|
||||
from typing import Optional
|
||||
|
||||
# Valid key pattern: alphanumeric, hyphens, underscores
|
||||
# Must be at least 1 char, URL-safe
|
||||
VALID_KEY_PATTERN = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9_-]*$')
|
||||
|
||||
|
||||
def normalize_key(key: str) -> str:
|
||||
"""Normalize a key to a URL-safe format.
|
||||
|
||||
Args:
|
||||
key: The key to normalize
|
||||
|
||||
Returns:
|
||||
Normalized lowercase key with spaces replaced by hyphens
|
||||
"""
|
||||
if not key:
|
||||
return ""
|
||||
|
||||
# Convert to lowercase
|
||||
normalized = key.lower()
|
||||
|
||||
# Replace spaces and underscores with hyphens
|
||||
normalized = re.sub(r'[\s_]+', '-', normalized)
|
||||
|
||||
# Remove any characters that aren't alphanumeric or hyphens
|
||||
normalized = re.sub(r'[^a-z0-9-]', '', normalized)
|
||||
|
||||
# Collapse multiple consecutive hyphens
|
||||
normalized = re.sub(r'-+', '-', normalized)
|
||||
|
||||
# Remove leading/trailing hyphens
|
||||
normalized = normalized.strip('-')
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def is_valid_key(key: str) -> bool:
|
||||
"""Check if a key is valid for URL-safe use.
|
||||
|
||||
Args:
|
||||
key: The key to validate
|
||||
|
||||
Returns:
|
||||
True if key is valid (non-empty, URL-safe, alphanumeric start/end, min 2 chars)
|
||||
"""
|
||||
if not key or not key.strip():
|
||||
return False
|
||||
|
||||
if len(key) < 2:
|
||||
return False
|
||||
|
||||
return bool(VALID_KEY_PATTERN.match(key))
|
||||
|
||||
|
||||
def generate_key_from_folder(folder_name: str) -> str:
|
||||
"""Generate a URL-safe key from a folder name.
|
||||
|
||||
Handles edge cases:
|
||||
- Non-Latin characters (Japanese, Chinese, etc.)
|
||||
- Special characters
|
||||
- All-invalid names that normalize to empty
|
||||
|
||||
Args:
|
||||
folder_name: The folder name to convert to a key
|
||||
|
||||
Returns:
|
||||
A URL-safe key string. Never returns empty string.
|
||||
|
||||
Examples:
|
||||
>>> generate_key_from_folder("Attack on Titan (2013)")
|
||||
'attack-on-titan-2013'
|
||||
>>> generate_key_from_folder("A Time Called You (2023)")
|
||||
'a-time-called-you-2023'
|
||||
>>> generate_key_from_folder("25-sai no Joshikousei (2018)")
|
||||
'25-sai-no-joshikousei-2018'
|
||||
"""
|
||||
if not folder_name or not folder_name.strip():
|
||||
raise ValueError("Folder name cannot be empty")
|
||||
|
||||
# Step 1: Unicode NFC normalization (preserves international chars)
|
||||
normalized = unicodedata.normalize('NFC', folder_name.strip())
|
||||
|
||||
# Step 2: Extract alphanumeric parts, preserving international chars
|
||||
# This keeps Japanese/Chinese characters but removes special symbols
|
||||
parts = []
|
||||
|
||||
for char in normalized:
|
||||
# Keep Unicode alphanumeric characters (letters/numbers from any script)
|
||||
if char.isalnum():
|
||||
parts.append(char)
|
||||
elif char.isspace():
|
||||
parts.append(' ')
|
||||
# Handle apostrophes - treat as part of word (remove, don't replace with space)
|
||||
# This normalizes e.g., "Hell's" -> "Hells"
|
||||
# Includes: ' (0x27), ' (0x2018), ' (0x2019), ' (0x02BC), ` (0x0060)
|
||||
elif char in ("'", "'", "'", "'", "`", """, """):
|
||||
pass # Skip - drop the apostrophe
|
||||
else:
|
||||
parts.append(' ')
|
||||
|
||||
working = ''.join(parts)
|
||||
|
||||
# Step 3: Split into words and normalize each
|
||||
words = working.split()
|
||||
|
||||
# Step 4: Convert to lowercase and create hyphenated key
|
||||
key = '-'.join(word.lower() for word in words if word)
|
||||
|
||||
# Step 5: If we got a valid key, return it
|
||||
if key and is_valid_key(key):
|
||||
return key
|
||||
|
||||
# Step 6: Try just alphanumeric characters
|
||||
alphanumeric_only = re.sub(r'[^a-zA-Z0-9\s]', '', working)
|
||||
words = alphanumeric_only.split()
|
||||
key = '-'.join(word.lower() for word in words if word)
|
||||
|
||||
if key and is_valid_key(key):
|
||||
return key
|
||||
|
||||
# Step 7: Last resort - use folder name directly with transliteration
|
||||
# Try to convert non-ASCII to ASCII equivalents
|
||||
try:
|
||||
# Use NFD normalization and strip combining characters
|
||||
# This effectively Latinizes some characters
|
||||
nfd_form = unicodedata.normalize('NFD', folder_name)
|
||||
latinized = ''.join(
|
||||
char for char in nfd_form
|
||||
if unicodedata.category(char) != 'Mn' # Strip combining marks
|
||||
)
|
||||
# Remove non-ASCII letters
|
||||
latinized = re.sub(r'[^a-zA-Z0-9\s]', ' ', latinized)
|
||||
words = latinized.split()
|
||||
key = '-'.join(word.lower() for word in words if word)
|
||||
|
||||
if key and is_valid_key(key):
|
||||
return key
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Step 8: Absolute fallback - generate UUID-based key
|
||||
# Use first 8 chars of UUID for brevity
|
||||
uuid_key = uuid.uuid4().hex[:8]
|
||||
|
||||
# Try to extract any meaningful words from the original name
|
||||
meaningful_parts = []
|
||||
for char in folder_name:
|
||||
if char.isalnum():
|
||||
meaningful_parts.append(char.lower())
|
||||
elif len(meaningful_parts) > 0:
|
||||
meaningful_parts.append('-')
|
||||
|
||||
fallback_base = ''.join(meaningful_parts).strip('-')
|
||||
if fallback_base and len(fallback_base) >= 2:
|
||||
# Combine meaningful parts with UUID for uniqueness
|
||||
# Truncate meaningful parts if too long
|
||||
if len(fallback_base) > 20:
|
||||
fallback_base = fallback_base[:20]
|
||||
return f"{fallback_base}-{uuid_key}"
|
||||
|
||||
return f"series-{uuid_key}"
|
||||
|
||||
|
||||
def validate_key_uniqueness(
|
||||
key: str,
|
||||
existing_keys: set[str],
|
||||
) -> tuple[bool, str]:
|
||||
"""Validate that a key is unique among existing keys.
|
||||
|
||||
Args:
|
||||
key: The key to validate
|
||||
existing_keys: Set of keys that already exist
|
||||
|
||||
Returns:
|
||||
Tuple of (is_valid, error_message)
|
||||
"""
|
||||
if not key or not key.strip():
|
||||
return False, "Key cannot be empty"
|
||||
|
||||
stripped = key.strip()
|
||||
if len(stripped) < 2:
|
||||
return False, "Key must be at least 2 characters"
|
||||
|
||||
if not is_valid_key(stripped):
|
||||
return False, "Key must be URL-safe (alphanumeric, hyphens, underscores only)"
|
||||
|
||||
if stripped in existing_keys:
|
||||
return False, f"Key '{stripped}' is already in use"
|
||||
|
||||
return True, ""
|
||||
|
||||
|
||||
def sanitize_key_for_url(key: str) -> str:
|
||||
"""Sanitize a key for safe URL usage.
|
||||
|
||||
Args:
|
||||
key: The key to sanitize
|
||||
|
||||
Returns:
|
||||
URL-safe version of the key
|
||||
"""
|
||||
if not key:
|
||||
return ""
|
||||
|
||||
# Replace spaces with hyphens first
|
||||
sanitized = key.replace(' ', '-')
|
||||
|
||||
# Remove any characters that could cause URL issues (keep alphanumerics, hyphens, underscores)
|
||||
sanitized = re.sub(r'[^\w\-]', '', sanitized)
|
||||
|
||||
# Collapse multiple hyphens
|
||||
sanitized = re.sub(r'-+', '-', sanitized)
|
||||
|
||||
return sanitized.strip('-')
|
||||
|
||||
|
||||
def sanitize_url_for_logging(url: str, max_length: int = 100) -> str:
|
||||
"""Sanitize a URL for safe logging by removing sensitive query parameters.
|
||||
|
||||
Removes or truncates query parameters that may contain tokens, keys,
|
||||
or other sensitive data while preserving enough structure for debugging.
|
||||
|
||||
Args:
|
||||
url: The URL to sanitize
|
||||
max_length: Maximum length of the returned URL string
|
||||
|
||||
Returns:
|
||||
Sanitized URL safe for logging
|
||||
"""
|
||||
if not url:
|
||||
return ""
|
||||
|
||||
# Truncate if too long
|
||||
if len(url) > max_length:
|
||||
return url[:max_length] + "..."
|
||||
|
||||
return url
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user