Files
Aniworld/tests/unit/test_serie_scanner_scan_dedup.py
Lukas c3aca9217d fix(episodes): prevent duplicate-row accumulation at schema + write sites
Followup to commit c84f968 (read-boundary dedup) and commit f75d591
(cleanup CLI). The read boundary filters duplicates out of the
in-memory episodeDict and the CLI cleans up historical duplicates
in the DB, but the underlying pathology — duplicate rows being
created in the first place — was still active on every rescan.

Two layered prevention fixes:

1. Schema-level guard: add UNIQUE(series_id, season, episode_number)
   to the episodes table. SQLite's CREATE UNIQUE INDEX requires
   no existing duplicates, but the cleanup CLI from f75d591 has
   already been run (or is a one-shot prerequisite for users on
   older DBs). Future duplicate rows are rejected at the DB layer.

2. Write-site guard: SerieScanner.scan_single_series used to
   `extend` the in-memory episodeDict on every rescan of a
   series already in keyDict — across N rescans, the same missing
   list was appended N times, growing the dict with duplicates that
   then flowed through _update_series_in_db into the episodes
   table. The fix replaces the cache with the latest scan result
   instead of extending, and dedupes within a single call as
   defense in depth against a buggy upstream loader.

Defensive dedup is layered three deep:
  - schema constraint (this commit, primary)
  - scan_single_series replace-not-extend (this commit, secondary)
  - episodeDict property read-boundary dedup (commit c84f968,
    tertiary — covers legacy DBs that predate the constraint)

Tests:
  - Updated test_serie_scanner.test_scan_single_series_existing_entry
    to assert the new replace-not-merge behavior (the old assertion
    encoded the buggy extend behavior).
  - New test_serie_scanner_scan_dedup.py covers the regression
    directly: two rescans of the same series with the same missing
    list must yield a canonical dict, not an accumulated one.
  - test_database_models and test_clean_duplicate_episodes_cli now
    use a legacy_engine fixture that drops the UNIQUE constraint,
    so the duplicate-row scenarios they exercise (the read-boundary
    dedup and the cleanup tool, both meant to defend against
    pre-migration state) can still be tested under the new schema.

Verified manually: clean_duplicate_episodes --apply on the user's
backup DB still removes all 633 duplicate rows under the new
schema (the CLI doesn't depend on the UNIQUE constraint — it
operates on whatever rows already exist).
2026-09-15 20:58:30 +02:00

179 lines
6.7 KiB
Python

"""Regression test for ``SerieScanner.scan_single_series``.
The previous implementation ``extend````ed the in-memory
``episodeDict`` for series already present in the scanner's
``keyDict`` — every rescan of the same series appended the new
missing-episode list on top of the existing one, so the dict grew
with duplicates across rescans. Those duplicates then propagated
through ``_update_series_in_db`` into the ``episodes`` table.
This test exercises the real ``scan_single_series`` method (not a
mock) to assert that two rescans of the same series produce a
canonical (deduplicated) ``episodeDict``, not an accumulated one.
"""
from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
import pytest
from src.server.SerieScanner import SerieScanner
class _StubLoader:
"""A minimal loader that returns whatever ``missing_episodes``
the test wants. Replaces the real loader on a
``SerieScanner`` instance via ``monkeypatch.setattr``."""
def __init__(self, missing_per_call: list[dict]):
self._missing_per_call = list(missing_per_call)
self._call_index = 0
def get_season_episode_count(self, key):
# The real loader returns the total episode count per
# season; ``scan_single_series`` doesn't actually use it
# (it calls ``__get_missing_episodes_and_season`` directly),
# but the stub keeps the call site safe.
max_seen = 0
for m in self._missing_per_call:
for season, eps in m.items():
if eps:
max_seen = max(max_seen, max(eps))
return {1: max_seen or 1}
def is_language(self, season, ep, key):
return True
def next_missing(self) -> dict:
if self._call_index >= len(self._missing_per_call):
return {}
result = self._missing_per_call[self._call_index]
self._call_index += 1
return result
def _make_scanner(
tmp_path: Path,
missing_per_call: list[dict],
) -> tuple[SerieScanner, _StubLoader]:
"""Build a real ``SerieScanner`` with the private
``__get_missing_episodes_and_season`` method replaced by a
stub that returns the next ``missing_episodes`` dict on each
call. Everything else (events, directory) is stubbed so the
test runs without filesystem or scheduler setup."""
scanner = SerieScanner.__new__(SerieScanner)
loader = _StubLoader(missing_per_call)
scanner.loader = loader # type: ignore[assignment]
scanner.keyDict = {}
# ``self.directory`` is the attribute ``scan_single_series``
# reads at line 734. ``scan_single_series`` checks
# ``os.path.isdir(folder_path)`` and, if the folder does not
# exist, treats the scan as "no MP4 files on disk". Use a path
# whose subdirectories do not exist so the scan takes the
# empty-mp4-files branch without us having to populate any
# filesystem state.
scanner.directory = str(tmp_path)
scanner.directory_to_search = tmp_path
scanner.events = SimpleNamespace(
on_progress=lambda *a, **k: None,
on_completion=lambda *a, **k: None,
on_error=lambda *a, **k: None,
)
def fake_get_missing_episodes_and_season(key, mp4_files):
return loader.next_missing(), "aniworld.to"
scanner._SerieScanner__get_missing_episodes_and_season = ( # type: ignore[attr-defined]
fake_get_missing_episodes_and_season
)
return scanner, loader
def test_scan_single_series_replaces_not_extends(tmp_path):
"""Two rescans of the same series with the same missing
episodes must produce a canonical (deduplicated) episodeDict —
not a list with duplicates from accumulation."""
canonical = {1: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]}
scanner, _ = _make_scanner(
tmp_path,
# Same missing-episode list reported on each scan.
missing_per_call=[canonical, canonical],
)
# First scan: key not in keyDict, so the else branch fires
# and the cache is set to the missing-episode list.
scanner.keyDict.clear()
result_first = scanner.scan_single_series(
key="erased", folder="Erased"
)
assert result_first == canonical
cached = scanner.keyDict["erased"].episodeDict
assert cached == canonical
# Second scan: key IS in keyDict, so the if branch fires.
# Before the fix, this would extend the cached list with
# [1..12] again, producing {1: [1..12, 1..12]}. After the
# fix, the cache is replaced, not extended.
result_second = scanner.scan_single_series(
key="erased", folder="Erased"
)
assert result_second == canonical
cached_second = scanner.keyDict["erased"].episodeDict
assert cached_second == canonical, (
"second scan extended the dict instead of replacing it: "
f"{cached_second}"
)
# Defensive: the dict has no duplicates within any season.
for season, eps in cached_second.items():
assert len(eps) == len(set(eps)), (
f"season {season} has duplicate episode numbers: {eps}"
)
def test_scan_single_series_resets_when_becomes_complete(tmp_path):
"""When a rescan finds the series is complete (no missing
episodes), the episodeDict must be empty — extending the
previous dict would leave stale entries behind."""
scanner, _ = _make_scanner(
tmp_path,
# First scan: missing 1..12. Second scan: nothing missing.
missing_per_call=[
{1: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]},
{},
],
)
scanner.keyDict.clear()
scanner.scan_single_series(key="complete-me", folder="Complete Me")
assert scanner.keyDict["complete-me"].episodeDict == {
1: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
}
scanner.scan_single_series(key="complete-me", folder="Complete Me")
assert scanner.keyDict["complete-me"].episodeDict == {}, (
"rescan with no missing episodes must reset the dict, "
f"got: {scanner.keyDict['complete-me'].episodeDict}"
)
def test_scan_single_series_dedupes_within_a_single_call(tmp_path):
"""If the loader itself returns duplicate episode numbers in
``missing_episodes`` (a buggy upstream loader), the scanner
must still produce a canonical dict — defense in depth on top
of the loader and the read-boundary dedup in
``AnimeSeries.episodeDict``."""
scanner, _ = _make_scanner(
tmp_path,
# The loader returns the same episode numbers multiple
# times within a single call.
missing_per_call=[{1: [1, 1, 2, 2, 3, 3, 3, 4]}],
)
scanner.keyDict.clear()
scanner.scan_single_series(key="buggy", folder="Buggy")
cached = scanner.keyDict["buggy"].episodeDict
assert cached == {1: [1, 2, 3, 4]}, (
f"single-scan dedup failed: {cached}"
)