Followup to commitc84f968(read-boundary dedup) and commitf75d591(cleanup CLI). The read boundary filters duplicates out of the in-memory episodeDict and the CLI cleans up historical duplicates in the DB, but the underlying pathology — duplicate rows being created in the first place — was still active on every rescan. Two layered prevention fixes: 1. Schema-level guard: add UNIQUE(series_id, season, episode_number) to the episodes table. SQLite's CREATE UNIQUE INDEX requires no existing duplicates, but the cleanup CLI fromf75d591has already been run (or is a one-shot prerequisite for users on older DBs). Future duplicate rows are rejected at the DB layer. 2. Write-site guard: SerieScanner.scan_single_series used to `extend` the in-memory episodeDict on every rescan of a series already in keyDict — across N rescans, the same missing list was appended N times, growing the dict with duplicates that then flowed through _update_series_in_db into the episodes table. The fix replaces the cache with the latest scan result instead of extending, and dedupes within a single call as defense in depth against a buggy upstream loader. Defensive dedup is layered three deep: - schema constraint (this commit, primary) - scan_single_series replace-not-extend (this commit, secondary) - episodeDict property read-boundary dedup (commitc84f968, tertiary — covers legacy DBs that predate the constraint) Tests: - Updated test_serie_scanner.test_scan_single_series_existing_entry to assert the new replace-not-merge behavior (the old assertion encoded the buggy extend behavior). - New test_serie_scanner_scan_dedup.py covers the regression directly: two rescans of the same series with the same missing list must yield a canonical dict, not an accumulated one. - test_database_models and test_clean_duplicate_episodes_cli now use a legacy_engine fixture that drops the UNIQUE constraint, so the duplicate-row scenarios they exercise (the read-boundary dedup and the cleanup tool, both meant to defend against pre-migration state) can still be tested under the new schema. Verified manually: clean_duplicate_episodes --apply on the user's backup DB still removes all 633 duplicate rows under the new schema (the CLI doesn't depend on the UNIQUE constraint — it operates on whatever rows already exist).
179 lines
6.7 KiB
Python
179 lines
6.7 KiB
Python
"""Regression test for ``SerieScanner.scan_single_series``.
|
|
|
|
The previous implementation ``extend````ed the in-memory
|
|
``episodeDict`` for series already present in the scanner's
|
|
``keyDict`` — every rescan of the same series appended the new
|
|
missing-episode list on top of the existing one, so the dict grew
|
|
with duplicates across rescans. Those duplicates then propagated
|
|
through ``_update_series_in_db`` into the ``episodes`` table.
|
|
|
|
This test exercises the real ``scan_single_series`` method (not a
|
|
mock) to assert that two rescans of the same series produce a
|
|
canonical (deduplicated) ``episodeDict``, not an accumulated one.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from src.server.SerieScanner import SerieScanner
|
|
|
|
|
|
class _StubLoader:
|
|
"""A minimal loader that returns whatever ``missing_episodes``
|
|
the test wants. Replaces the real loader on a
|
|
``SerieScanner`` instance via ``monkeypatch.setattr``."""
|
|
|
|
def __init__(self, missing_per_call: list[dict]):
|
|
self._missing_per_call = list(missing_per_call)
|
|
self._call_index = 0
|
|
|
|
def get_season_episode_count(self, key):
|
|
# The real loader returns the total episode count per
|
|
# season; ``scan_single_series`` doesn't actually use it
|
|
# (it calls ``__get_missing_episodes_and_season`` directly),
|
|
# but the stub keeps the call site safe.
|
|
max_seen = 0
|
|
for m in self._missing_per_call:
|
|
for season, eps in m.items():
|
|
if eps:
|
|
max_seen = max(max_seen, max(eps))
|
|
return {1: max_seen or 1}
|
|
|
|
def is_language(self, season, ep, key):
|
|
return True
|
|
|
|
def next_missing(self) -> dict:
|
|
if self._call_index >= len(self._missing_per_call):
|
|
return {}
|
|
result = self._missing_per_call[self._call_index]
|
|
self._call_index += 1
|
|
return result
|
|
|
|
|
|
def _make_scanner(
|
|
tmp_path: Path,
|
|
missing_per_call: list[dict],
|
|
) -> tuple[SerieScanner, _StubLoader]:
|
|
"""Build a real ``SerieScanner`` with the private
|
|
``__get_missing_episodes_and_season`` method replaced by a
|
|
stub that returns the next ``missing_episodes`` dict on each
|
|
call. Everything else (events, directory) is stubbed so the
|
|
test runs without filesystem or scheduler setup."""
|
|
scanner = SerieScanner.__new__(SerieScanner)
|
|
loader = _StubLoader(missing_per_call)
|
|
scanner.loader = loader # type: ignore[assignment]
|
|
scanner.keyDict = {}
|
|
# ``self.directory`` is the attribute ``scan_single_series``
|
|
# reads at line 734. ``scan_single_series`` checks
|
|
# ``os.path.isdir(folder_path)`` and, if the folder does not
|
|
# exist, treats the scan as "no MP4 files on disk". Use a path
|
|
# whose subdirectories do not exist so the scan takes the
|
|
# empty-mp4-files branch without us having to populate any
|
|
# filesystem state.
|
|
scanner.directory = str(tmp_path)
|
|
scanner.directory_to_search = tmp_path
|
|
scanner.events = SimpleNamespace(
|
|
on_progress=lambda *a, **k: None,
|
|
on_completion=lambda *a, **k: None,
|
|
on_error=lambda *a, **k: None,
|
|
)
|
|
|
|
def fake_get_missing_episodes_and_season(key, mp4_files):
|
|
return loader.next_missing(), "aniworld.to"
|
|
|
|
scanner._SerieScanner__get_missing_episodes_and_season = ( # type: ignore[attr-defined]
|
|
fake_get_missing_episodes_and_season
|
|
)
|
|
return scanner, loader
|
|
|
|
|
|
def test_scan_single_series_replaces_not_extends(tmp_path):
|
|
"""Two rescans of the same series with the same missing
|
|
episodes must produce a canonical (deduplicated) episodeDict —
|
|
not a list with duplicates from accumulation."""
|
|
canonical = {1: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]}
|
|
scanner, _ = _make_scanner(
|
|
tmp_path,
|
|
# Same missing-episode list reported on each scan.
|
|
missing_per_call=[canonical, canonical],
|
|
)
|
|
|
|
# First scan: key not in keyDict, so the else branch fires
|
|
# and the cache is set to the missing-episode list.
|
|
scanner.keyDict.clear()
|
|
result_first = scanner.scan_single_series(
|
|
key="erased", folder="Erased"
|
|
)
|
|
assert result_first == canonical
|
|
cached = scanner.keyDict["erased"].episodeDict
|
|
assert cached == canonical
|
|
|
|
# Second scan: key IS in keyDict, so the if branch fires.
|
|
# Before the fix, this would extend the cached list with
|
|
# [1..12] again, producing {1: [1..12, 1..12]}. After the
|
|
# fix, the cache is replaced, not extended.
|
|
result_second = scanner.scan_single_series(
|
|
key="erased", folder="Erased"
|
|
)
|
|
assert result_second == canonical
|
|
cached_second = scanner.keyDict["erased"].episodeDict
|
|
assert cached_second == canonical, (
|
|
"second scan extended the dict instead of replacing it: "
|
|
f"{cached_second}"
|
|
)
|
|
|
|
# Defensive: the dict has no duplicates within any season.
|
|
for season, eps in cached_second.items():
|
|
assert len(eps) == len(set(eps)), (
|
|
f"season {season} has duplicate episode numbers: {eps}"
|
|
)
|
|
|
|
|
|
def test_scan_single_series_resets_when_becomes_complete(tmp_path):
|
|
"""When a rescan finds the series is complete (no missing
|
|
episodes), the episodeDict must be empty — extending the
|
|
previous dict would leave stale entries behind."""
|
|
scanner, _ = _make_scanner(
|
|
tmp_path,
|
|
# First scan: missing 1..12. Second scan: nothing missing.
|
|
missing_per_call=[
|
|
{1: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]},
|
|
{},
|
|
],
|
|
)
|
|
|
|
scanner.keyDict.clear()
|
|
scanner.scan_single_series(key="complete-me", folder="Complete Me")
|
|
assert scanner.keyDict["complete-me"].episodeDict == {
|
|
1: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
|
|
}
|
|
|
|
scanner.scan_single_series(key="complete-me", folder="Complete Me")
|
|
assert scanner.keyDict["complete-me"].episodeDict == {}, (
|
|
"rescan with no missing episodes must reset the dict, "
|
|
f"got: {scanner.keyDict['complete-me'].episodeDict}"
|
|
)
|
|
|
|
|
|
def test_scan_single_series_dedupes_within_a_single_call(tmp_path):
|
|
"""If the loader itself returns duplicate episode numbers in
|
|
``missing_episodes`` (a buggy upstream loader), the scanner
|
|
must still produce a canonical dict — defense in depth on top
|
|
of the loader and the read-boundary dedup in
|
|
``AnimeSeries.episodeDict``."""
|
|
scanner, _ = _make_scanner(
|
|
tmp_path,
|
|
# The loader returns the same episode numbers multiple
|
|
# times within a single call.
|
|
missing_per_call=[{1: [1, 1, 2, 2, 3, 3, 3, 4]}],
|
|
)
|
|
|
|
scanner.keyDict.clear()
|
|
scanner.scan_single_series(key="buggy", folder="Buggy")
|
|
cached = scanner.keyDict["buggy"].episodeDict
|
|
assert cached == {1: [1, 2, 3, 4]}, (
|
|
f"single-scan dedup failed: {cached}"
|
|
) |