chore: better logging

Loader.py

@@ -7,16 +7,14 @@ from aniworld.models import Anime
 from aniworld.config import PROVIDER_HEADERS, INVALID_PATH_CHARS
 from aniworld.parser import arguments
 
-# Configure logging
-logging.basicConfig(
-    filename="download.log",
-    level=logging.INFO,
-    format="%(asctime)s - %(levelname)s - %(message)s"
-)
-
 # Read timeout from environment variable, default to 600 seconds (10 minutes)
 timeout = int(os.getenv("DOWNLOAD_TIMEOUT", 600))
 
+download_error_logger = logging.getLogger("DownloadErrors")
+download_error_handler = logging.FileHandler("download_errors.log")
+download_error_handler.setLevel(logging.ERROR)
+download_error_logger.addHandler(download_error_handler)
+
 def download(anime: Anime):  # pylint: disable=too-many-branches
     for episode in anime:
         sanitized_anime_title = ''.join(
@@ -38,9 +36,6 @@ def download(anime: Anime):  # pylint: disable=too-many-branches
 
         output_path = os.path.join(anime.output_directory, output_file)
         os.makedirs(os.path.dirname(output_path), exist_ok=True)
-
-        logging.info(f"Preparing to download: {output_path}")
-
         command = [
             "yt-dlp",
             episode.get_direct_link(anime.provider, anime.language),
@@ -48,8 +43,7 @@ def download(anime: Anime):  # pylint: disable=too-many-branches
             "--concurrent-fragments", "4",
             "-o", output_path,
             "--quiet",
-            "--no-warnings",
+            "--no-warnings"
-            "--progress"
         ]
 
         if anime.provider in PROVIDER_HEADERS:
@@ -64,10 +58,7 @@ def download(anime: Anime):  # pylint: disable=too-many-branches
             continue
 
         try:
-
-            logging.info(f"Starting download to {output_path}...")
             subprocess.run(command, check=True, timeout=timeout)
-            logging.info(f"Download completed: {output_path}")
         except subprocess.TimeoutExpired:
             logging.error(f"Download timed out after {timeout} seconds: {' '.join(str(item) for item in command)}")
         except subprocess.CalledProcessError:

main.py

@@ -1,20 +1,33 @@
+import sys
 import os
+import traceback
 import re
 import logging
+from concurrent.futures import ThreadPoolExecutor
+from collections import defaultdict
 from aniworld.models import Anime, Episode
 from aniworld.common import get_season_episode_count, get_movie_episode_count
 from aniworld.search import search_anime
 from Loader import download
 
 # Configure logging
-logging.basicConfig(
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
-    filename="loader.log",
-    level=logging.DEBUG,
-    format="%(asctime)s - %(levelname)s - %(message)s"
-)
 
+error_logger = logging.getLogger("ErrorLog")
+error_handler = logging.FileHandler("errors.log")
+error_handler.setLevel(logging.ERROR)
+error_logger.addHandler(error_handler)
+
+noKeyFound_logger = logging.getLogger("NoKeyFound")
+noKeyFound_handler = logging.FileHandler("NoKeyFound.log")
+noKeyFound_handler.setLevel(logging.ERROR)
+noKeyFound_logger.addHandler(noKeyFound_handler)
+
+class NoKeyFoundException(Exception):
+    """Exception raised when an anime key cannot be found."""
+    pass
 class MatchNotFoundError(Exception):
-    """Custom exception raised when the pattern match is not found."""
+    """Exception raised when an anime key cannot be found."""
     pass
 
 class Loader:
@@ -24,16 +37,23 @@ class Loader:
 
     def __find_mp4_files(self):
         logging.info("Scanning for .mp4 files")
-        for root, dirs, files in os.walk(self.directory):
-            mp4_files = [file for file in files if file.endswith('.mp4')]
 
+        for root_folder_name in os.listdir(self.directory):
+
+            folder_data = defaultdict(list)  # Dictionary to store MP4 files per folder
+            folder = os.path.join(self.directory, root_folder_name)
+
+            logging.info(f"Processing folder: {root_folder_name}")
+
+            # First pass: Scan all folders and collect MP4 file data
+            for root, dirs, files in os.walk(folder):
+                mp4_files = [file for file in files if file.endswith('.mp4')]
                 if mp4_files:
-                relative_path = os.path.relpath(root, self.directory)
+                    folder_data[root_folder_name].extend(mp4_files)
-                root_folder_name = relative_path.split(os.sep)[0]
+            yield root_folder_name, folder_data[root_folder_name]
-                logging.debug(f"Found {len(mp4_files)} .mp4 files in {root_folder_name}")
+
-                yield root_folder_name, mp4_files
         for dir in self.__find_empty_folders():
-            logging.debug(f"Found no .mp4 files in {dir}")
+            logging.info(f"Found no .mp4 files in {dir}")
             yield dir, []
 
     def __find_empty_folders(self):
@@ -59,17 +79,24 @@ class Loader:
         if os.path.exists(key_file):
             with open(key_file, 'r') as file:
                 key = file.read().strip()
-                logging.debug(f"Key found for folder '{folder_name}': {key}")
+                logging.info(f"Key found for folder '{folder_name}': {key}")
                 return key
         else:
-            key = search_anime(self.__remove_year(folder_name))
+            try:
+                key = search_anime(folder_name, True)
+                if key:
+                    key = key[0]['link']
                     with open(key_file, 'w') as file:
                         file.write(key)
                     logging.info(f"Generated new key for folder '{folder_name}': {key}")
                     return key
+                else:
+                    raise NoKeyFoundException(f"No key found for folder '{folder_name}'")
+            except Exception as e:
+                raise NoKeyFoundException(f"Failed to retrieve key for folder '{folder_name}'") from e
 
     def __GetEpisodeAndSeason(self, filename: str):
-        pattern = r'S(\d{2})E(\d{2})'
+        pattern = r'S(\d+)E(\d+)'
         match = re.search(pattern, filename)
         if match:
             season = match.group(1)
@@ -104,36 +131,43 @@ class Loader:
             if missing_episodes:
                 yield season, missing_episodes
 
-
     def LoadMissing(self):
-        logging.info("Starting process to load missing episodes")
+        logging.warning("Starting process to load missing episodes")
         result = self.__find_mp4_files()
 
+        def download_episode(folder, season, episode, key):
+            """Helper function to download an individual episode."""
+            try:
+                folder_path = os.path.join(self.directory, folder, f"Season {season}")
+                anime = Anime(
+                    episode_list=[Episode(slug=key, season=season, episode=episode)],
+                    language="German Dub",
+                    output_directory=folder_path
+                )
+                logging.warning(f"Downloading episode {episode} of season {season} for anime {key}")
+                download(anime)
+            except Exception as e:
+                logging.error(f"Error downloading episode {episode} of season {season} for anime {key}: {e}")
+
+        # Using ThreadPoolExecutor to run downloads in parallel
+        with ThreadPoolExecutor(max_workers=5) as executor:  # Adjust number of workers as needed
             for folder, mp4_files in result:
                 try:
                     key = self.__check_and_generate_key(folder)
                     missings = self.__GetMissingEpisodesAndSeason(key, mp4_files)
-
+                    logging.info("Missing episodes for {key} \n" + "\n".join(f"Season {str(k)}: {', '.join(str(v))}" for k, v in missings))
                     for season, missing_episodes in missings:
-                    folder_path = os.path.join(self.directory, folder, f"Season {season}")
-
                         for episode in missing_episodes:
-                        anime = Anime(
+                            executor.submit(download_episode, folder, season, episode, key)
-                            episode_list=[
+                except NoKeyFoundException as nkfe:
-                                Episode(slug=key, season=season, episode=episode)
+                    noKeyFound_logger.error(f"Error processing folder '{folder}': {nkfe}")
-                            ],
-                            language="German Dub",
-                            output_directory=folder_path
-                        )
-                        logging.info(f"Downloading episode {episode} of season {season} for anime {key}")
-                        download(anime)
                 except Exception as e:
-                logging.error(f"Error processing folder '{folder}': {e}")
+                    error_logger.error(f"Unexpected error processing folder '{folder}': {e} \n {traceback.format_exc()}")
                     continue
 
 # Read the base directory from an environment variable
-#directory_to_search = os.getenv("ANIME_DIRECTORY", "\\\\sshfs.r\\ubuntu@192.168.178.43\\media\\serien\\Serien")
+directory_to_search = os.getenv("ANIME_DIRECTORY", "\\\\sshfs.r\\ubuntu@192.168.178.43\\media\\serien\\Serien")
-directory_to_search = os.getenv("ANIME_DIRECTORY", "D:\sss")
+#directory_to_search = os.getenv("ANIME_DIRECTORY", "D:\sss")
 
 loader = Loader(directory_to_search)
 loader.LoadMissing()