cleanup
This commit is contained in:
@@ -5,14 +5,12 @@ This module extends the original AniWorldLoader with comprehensive
|
||||
error handling, retry mechanisms, and recovery strategies.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
from urllib.parse import quote
|
||||
|
||||
@@ -148,13 +146,28 @@ class EnhancedAniWorldLoader(Loader):
|
||||
"""Create a session with robust retry and error handling configuration."""
|
||||
session = requests.Session()
|
||||
|
||||
# Enhanced retry strategy
|
||||
# Configure retries so transient network problems are retried while we
|
||||
# still fail fast on permanent errors. The status codes cover
|
||||
# timeouts, rate limits, and the Cloudflare-origin 52x responses that
|
||||
# AniWorld occasionally emits under load.
|
||||
retries = Retry(
|
||||
total=5,
|
||||
backoff_factor=2, # More aggressive backoff
|
||||
status_forcelist=[408, 429, 500, 502, 503, 504, 520, 521, 522, 523, 524],
|
||||
status_forcelist=[
|
||||
408,
|
||||
429,
|
||||
500,
|
||||
502,
|
||||
503,
|
||||
504,
|
||||
520,
|
||||
521,
|
||||
522,
|
||||
523,
|
||||
524,
|
||||
],
|
||||
allowed_methods=["GET", "POST", "HEAD"],
|
||||
raise_on_status=False # Handle status errors manually
|
||||
raise_on_status=False, # Handle status errors manually
|
||||
)
|
||||
|
||||
adapter = HTTPAdapter(
|
||||
@@ -255,9 +268,9 @@ class EnhancedAniWorldLoader(Loader):
|
||||
|
||||
clean_text = response_text.strip()
|
||||
|
||||
# Try multiple parsing strategies. We progressively relax the parsing
|
||||
# requirements to handle HTML-escaped payloads, stray BOM markers, and
|
||||
# control characters injected by the upstream service.
|
||||
# Attempt increasingly permissive parsing strategies to cope with
|
||||
# upstream anomalies such as HTML escaping, stray BOM markers, and
|
||||
# injected control characters.
|
||||
parsing_strategies = [
|
||||
lambda text: json.loads(html.unescape(text)),
|
||||
lambda text: json.loads(text.encode('utf-8').decode('utf-8-sig')),
|
||||
|
||||
Reference in New Issue
Block a user