lukas.pupkalipinski/Aniworld
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: lukas.pupkalipinski:32cffc3079ddad77b4e8ccb578cb8d8b44f76f3b
Branches Tags
lukas.pupkalipinski:main
..
compare: lukas.pupkalipinski:4dae1d037f5d6f356b70300111f914139e6891e4
Branches Tags
lukas.pupkalipinski:main

No commits in common. "32cffc3079ddad77b4e8ccb578cb8d8b44f76f3b" and "4dae1d037f5d6f356b70300111f914139e6891e4" have entirely different histories.
32cffc3079 ... 4dae1d037f

2 changed files with 59 additions and 84 deletions
Show Stats Expand all files Collapse all files

21
Loader.py
View File

@ -7,14 +7,16 @@ from aniworld.models import Anime
from aniworld.config import PROVIDER_HEADERS, INVALID_PATH_CHARS from aniworld.config import PROVIDER_HEADERS, INVALID_PATH_CHARS
from aniworld.parser import arguments from aniworld.parser import arguments
# Configure logging
logging.basicConfig(
filename="download.log",
level=logging.INFO,
format="%(asctime)s - %(levelname)s - %(message)s"
)
# Read timeout from environment variable, default to 600 seconds (10 minutes) # Read timeout from environment variable, default to 600 seconds (10 minutes)
timeout = int(os.getenv("DOWNLOAD_TIMEOUT", 600)) timeout = int(os.getenv("DOWNLOAD_TIMEOUT", 600))
download_error_logger = logging.getLogger("DownloadErrors")
download_error_handler = logging.FileHandler("download_errors.log")
download_error_handler.setLevel(logging.ERROR)
download_error_logger.addHandler(download_error_handler)
def download(anime: Anime): # pylint: disable=too-many-branches def download(anime: Anime): # pylint: disable=too-many-branches
for episode in anime: for episode in anime:
sanitized_anime_title = ''.join( sanitized_anime_title = ''.join(
@ -36,6 +38,9 @@ def download(anime: Anime): # pylint: disable=too-many-branches
output_path = os.path.join(anime.output_directory, output_file) output_path = os.path.join(anime.output_directory, output_file)
os.makedirs(os.path.dirname(output_path), exist_ok=True) os.makedirs(os.path.dirname(output_path), exist_ok=True)
logging.info(f"Preparing to download: {output_path}")
command = [ command = [
"yt-dlp", "yt-dlp",
episode.get_direct_link(anime.provider, anime.language), episode.get_direct_link(anime.provider, anime.language),
@ -43,7 +48,8 @@ def download(anime: Anime): # pylint: disable=too-many-branches
"--concurrent-fragments", "4", "--concurrent-fragments", "4",
"-o", output_path, "-o", output_path,
"--quiet", "--quiet",
"--no-warnings" "--no-warnings",
"--progress"
] ]
if anime.provider in PROVIDER_HEADERS: if anime.provider in PROVIDER_HEADERS:
@ -58,7 +64,10 @@ def download(anime: Anime): # pylint: disable=too-many-branches
continue continue
try: try:
logging.info(f"Starting download to {output_path}...")
subprocess.run(command, check=True, timeout=timeout) subprocess.run(command, check=True, timeout=timeout)
logging.info(f"Download completed: {output_path}")
except subprocess.TimeoutExpired: except subprocess.TimeoutExpired:
logging.error(f"Download timed out after {timeout} seconds: {' '.join(str(item) for item in command)}") logging.error(f"Download timed out after {timeout} seconds: {' '.join(str(item) for item in command)}")
except subprocess.CalledProcessError: except subprocess.CalledProcessError:

122
main.py
View File

@ -1,33 +1,20 @@
import sys
import os import os
import traceback
import re import re
import logging import logging
from concurrent.futures import ThreadPoolExecutor
from collections import defaultdict
from aniworld.models import Anime, Episode from aniworld.models import Anime, Episode
from aniworld.common import get_season_episode_count, get_movie_episode_count from aniworld.common import get_season_episode_count, get_movie_episode_count
from aniworld.search import search_anime from aniworld.search import search_anime
from Loader import download from Loader import download
# Configure logging # Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') logging.basicConfig(
filename="loader.log",
level=logging.DEBUG,
format="%(asctime)s - %(levelname)s - %(message)s"
)
error_logger = logging.getLogger("ErrorLog")
error_handler = logging.FileHandler("errors.log")
error_handler.setLevel(logging.ERROR)
error_logger.addHandler(error_handler)
noKeyFound_logger = logging.getLogger("NoKeyFound")
noKeyFound_handler = logging.FileHandler("NoKeyFound.log")
noKeyFound_handler.setLevel(logging.ERROR)
noKeyFound_logger.addHandler(noKeyFound_handler)
class NoKeyFoundException(Exception):
"""Exception raised when an anime key cannot be found."""
pass
class MatchNotFoundError(Exception): class MatchNotFoundError(Exception):
"""Exception raised when an anime key cannot be found.""" """Custom exception raised when the pattern match is not found."""
pass pass
class Loader: class Loader:
@ -37,23 +24,16 @@ class Loader:
def __find_mp4_files(self): def __find_mp4_files(self):
logging.info("Scanning for .mp4 files") logging.info("Scanning for .mp4 files")
for root, dirs, files in os.walk(self.directory):
mp4_files = [file for file in files if file.endswith('.mp4')]
for root_folder_name in os.listdir(self.directory): if mp4_files:
relative_path = os.path.relpath(root, self.directory)
folder_data = defaultdict(list) # Dictionary to store MP4 files per folder root_folder_name = relative_path.split(os.sep)[0]
folder = os.path.join(self.directory, root_folder_name) logging.debug(f"Found {len(mp4_files)} .mp4 files in {root_folder_name}")
yield root_folder_name, mp4_files
logging.info(f"Processing folder: {root_folder_name}")
# First pass: Scan all folders and collect MP4 file data
for root, dirs, files in os.walk(folder):
mp4_files = [file for file in files if file.endswith('.mp4')]
if mp4_files:
folder_data[root_folder_name].extend(mp4_files)
yield root_folder_name, folder_data[root_folder_name]
for dir in self.__find_empty_folders(): for dir in self.__find_empty_folders():
logging.info(f"Found no .mp4 files in {dir}") logging.debug(f"Found no .mp4 files in {dir}")
yield dir, [] yield dir, []
def __find_empty_folders(self): def __find_empty_folders(self):
@ -79,24 +59,17 @@ class Loader:
if os.path.exists(key_file): if os.path.exists(key_file):
with open(key_file, 'r') as file: with open(key_file, 'r') as file:
key = file.read().strip() key = file.read().strip()
logging.info(f"Key found for folder '{folder_name}': {key}") logging.debug(f"Key found for folder '{folder_name}': {key}")
return key return key
else: else:
try: key = search_anime(self.__remove_year(folder_name))
key = search_anime(folder_name, True) with open(key_file, 'w') as file:
if key: file.write(key)
key = key[0]['link'] logging.info(f"Generated new key for folder '{folder_name}': {key}")
with open(key_file, 'w') as file: return key
file.write(key)
logging.info(f"Generated new key for folder '{folder_name}': {key}")
return key
else:
raise NoKeyFoundException(f"No key found for folder '{folder_name}'")
except Exception as e:
raise NoKeyFoundException(f"Failed to retrieve key for folder '{folder_name}'") from e
def __GetEpisodeAndSeason(self, filename: str): def __GetEpisodeAndSeason(self, filename: str):
pattern = r'S(\d+)E(\d+)' pattern = r'S(\d{2})E(\d{2})'
match = re.search(pattern, filename) match = re.search(pattern, filename)
if match: if match:
season = match.group(1) season = match.group(1)
@ -131,43 +104,36 @@ class Loader:
if missing_episodes: if missing_episodes:
yield season, missing_episodes yield season, missing_episodes
def LoadMissing(self): def LoadMissing(self):
logging.warning("Starting process to load missing episodes") logging.info("Starting process to load missing episodes")
result = self.__find_mp4_files() result = self.__find_mp4_files()
def download_episode(folder, season, episode, key): for folder, mp4_files in result:
"""Helper function to download an individual episode."""
try: try:
folder_path = os.path.join(self.directory, folder, f"Season {season}") key = self.__check_and_generate_key(folder)
anime = Anime( missings = self.__GetMissingEpisodesAndSeason(key, mp4_files)
episode_list=[Episode(slug=key, season=season, episode=episode)],
language="German Dub",
output_directory=folder_path
)
logging.warning(f"Downloading episode {episode} of season {season} for anime {key}")
download(anime)
except Exception as e:
logging.error(f"Error downloading episode {episode} of season {season} for anime {key}: {e}")
# Using ThreadPoolExecutor to run downloads in parallel for season, missing_episodes in missings:
with ThreadPoolExecutor(max_workers=5) as executor: # Adjust number of workers as needed folder_path = os.path.join(self.directory, folder, f"Season {season}")
for folder, mp4_files in result:
try: for episode in missing_episodes:
key = self.__check_and_generate_key(folder) anime = Anime(
missings = self.__GetMissingEpisodesAndSeason(key, mp4_files) episode_list=[
logging.info("Missing episodes for {key} \n" + "\n".join(f"Season {str(k)}: {', '.join(str(v))}" for k, v in missings)) Episode(slug=key, season=season, episode=episode)
for season, missing_episodes in missings: ],
for episode in missing_episodes: language="German Dub",
executor.submit(download_episode, folder, season, episode, key) output_directory=folder_path
except NoKeyFoundException as nkfe: )
noKeyFound_logger.error(f"Error processing folder '{folder}': {nkfe}") logging.info(f"Downloading episode {episode} of season {season} for anime {key}")
except Exception as e: download(anime)
error_logger.error(f"Unexpected error processing folder '{folder}': {e} \n {traceback.format_exc()}") except Exception as e:
continue logging.error(f"Error processing folder '{folder}': {e}")
continue
# Read the base directory from an environment variable # Read the base directory from an environment variable
directory_to_search = os.getenv("ANIME_DIRECTORY", "\\\\sshfs.r\\ubuntu@192.168.178.43\\media\\serien\\Serien") #directory_to_search = os.getenv("ANIME_DIRECTORY", "\\\\sshfs.r\\ubuntu@192.168.178.43\\media\\serien\\Serien")
#directory_to_search = os.getenv("ANIME_DIRECTORY", "D:\sss") directory_to_search = os.getenv("ANIME_DIRECTORY", "D:\sss")
loader = Loader(directory_to_search) loader = Loader(directory_to_search)
loader.LoadMissing() loader.LoadMissing()