The 'episodes' table has no UNIQUE constraint on
(series_id, season, episode_number), so historical scans can leave
duplicate rows behind. Commit c84f968 added a read-boundary dedup
in AnimeSeries.episodeDict so the rest of the stack never sees
duplicates — but the duplicate rows themselves still bloat the DB
and confuse direct SQL queries.
This commit adds a standalone CLI to find and (with --apply)
delete those duplicate rows. The cleanup keeps the lowest 'id'
per tuple (the oldest insert, which is most likely to have
populated title / file_path fields) and is idempotent.
Usage:
python -m src.cli.clean_duplicate_episodes # dry-run report
python -m src.cli.clean_duplicate_episodes --apply # actually delete
python -m src.cli.clean_duplicate_episodes --max-series 5 # limit report
Override the target DB with DATABASE_URL=sqlite:///path/to.db.
Verified against the user's backup DB: 633 duplicate rows across
231 (series, season, episode) tuples removed cleanly, leaving
1221 unique rows. Re-running reports no duplicates. The cleanup
does not affect the read-boundary dedup — both layers are
defensive in depth.
280 lines
9.4 KiB
Python
280 lines
9.4 KiB
Python
"""Unit tests for the ``clean_duplicate_episodes`` CLI.
|
|
|
|
Exercises the core logic (find / delete) against an in-memory
|
|
SQLite engine so the test is hermetic. The CLI module reads the
|
|
global async session factory from ``src.server.database.connection``
|
|
— the test patches that factory with an in-memory engine.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from typing import List
|
|
|
|
import pytest
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import (
|
|
AsyncEngine,
|
|
AsyncSession,
|
|
async_sessionmaker,
|
|
create_async_engine,
|
|
)
|
|
from sqlalchemy.pool import StaticPool
|
|
|
|
from src.cli import clean_duplicate_episodes as cli
|
|
from src.server.database import connection as conn_module
|
|
from src.server.database.base import Base
|
|
from src.server.database.models import AnimeSeries, Episode
|
|
|
|
|
|
@pytest.fixture
|
|
async def in_memory_engine():
|
|
"""Provide an in-memory async SQLite engine with the schema
|
|
already created, and patch the connection module's session
|
|
factory to use it for the duration of the test."""
|
|
engine: AsyncEngine = create_async_engine(
|
|
"sqlite+aiosqlite:///:memory:",
|
|
echo=False,
|
|
poolclass=StaticPool,
|
|
)
|
|
async with engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
|
|
factory = async_sessionmaker(
|
|
bind=engine,
|
|
class_=AsyncSession,
|
|
expire_on_commit=False,
|
|
)
|
|
|
|
# Monkey-patch the global session factory.
|
|
original_factory = conn_module._session_factory
|
|
conn_module._session_factory = factory
|
|
try:
|
|
yield engine
|
|
finally:
|
|
conn_module._session_factory = original_factory
|
|
await engine.dispose()
|
|
|
|
|
|
async def _add_series(session: AsyncSession, key: str) -> int:
|
|
series = AnimeSeries(
|
|
key=key,
|
|
name=key.replace("-", " ").title(),
|
|
site="https://aniworld.to",
|
|
folder=f"/anime/{key}",
|
|
)
|
|
session.add(series)
|
|
await session.commit()
|
|
await session.refresh(series)
|
|
return int(series.id)
|
|
|
|
|
|
async def _add_episode(
|
|
session: AsyncSession,
|
|
series_id: int,
|
|
season: int,
|
|
ep_num: int,
|
|
is_downloaded: bool = False,
|
|
title: str | None = None,
|
|
) -> int:
|
|
ep = Episode(
|
|
series_id=series_id,
|
|
season=season,
|
|
episode_number=ep_num,
|
|
is_downloaded=is_downloaded,
|
|
title=title,
|
|
)
|
|
session.add(ep)
|
|
await session.commit()
|
|
await session.refresh(ep)
|
|
return int(ep.id)
|
|
|
|
|
|
def _tup(d):
|
|
"""Convert a SQLAlchemy row to the (sid, season, ep_num, count,
|
|
keep_id, max_id) tuple shape the CLI uses."""
|
|
return (
|
|
int(d.series_id),
|
|
int(d.season),
|
|
int(d.episode_number),
|
|
int(d.row_count),
|
|
int(d.keep_id),
|
|
int(d.max_id),
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_find_returns_empty_when_no_duplicates(in_memory_engine):
|
|
"""With one row per (series, season, ep_num), nothing is found."""
|
|
async with AsyncSession(in_memory_engine) as session:
|
|
sid = await _add_series(session, "no-dupes")
|
|
await _add_episode(session, sid, 1, 1)
|
|
await _add_episode(session, sid, 1, 2)
|
|
await _add_episode(session, sid, 2, 1)
|
|
|
|
duplicates = await cli.find_duplicate_episodes()
|
|
assert duplicates == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_find_groups_duplicates_per_tuple(in_memory_engine):
|
|
"""Three copies of (S1, E2) collapse to one DuplicateTuple with
|
|
count=3 and keep_id = the lowest id."""
|
|
async with AsyncSession(in_memory_engine) as session:
|
|
sid = await _add_series(session, "triple")
|
|
e1 = await _add_episode(session, sid, 1, 1)
|
|
# Three rows for (S1, E2): keep_id should be e2 (lowest)
|
|
e2 = await _add_episode(session, sid, 1, 2)
|
|
e3 = await _add_episode(session, sid, 1, 2)
|
|
e4 = await _add_episode(session, sid, 1, 2)
|
|
await _add_episode(session, sid, 1, 3)
|
|
|
|
duplicates = await cli.find_duplicate_episodes()
|
|
|
|
# Only one duplicate tuple (S1, E2); (S1, E1) and (S1, E3) are
|
|
# unique and must not appear.
|
|
assert len(duplicates) == 1
|
|
d = duplicates[0]
|
|
assert (d[0], d[1], d[2]) == (sid, 1, 2)
|
|
assert d[3] == 3
|
|
assert d[4] == e2 # lowest id wins
|
|
assert d[5] == e4 # max id (for the report)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_keeps_lowest_id_per_tuple(in_memory_engine):
|
|
"""The DELETE call preserves the lowest-id row per duplicate
|
|
tuple and removes the rest."""
|
|
async with AsyncSession(in_memory_engine) as session:
|
|
sid = await _add_series(session, "keep-lowest")
|
|
e1 = await _add_episode(session, sid, 1, 1) # unique
|
|
e2 = await _add_episode(session, sid, 1, 2) # lowest of dupes
|
|
await _add_episode(session, sid, 1, 2) # e3 - duplicate of e2
|
|
await _add_episode(session, sid, 1, 2) # e4 - duplicate of e2
|
|
e5 = await _add_episode(session, sid, 1, 3) # unique
|
|
|
|
duplicates = await cli.find_duplicate_episodes()
|
|
assert len(duplicates) == 1
|
|
|
|
deleted = await cli.delete_duplicate_episodes(duplicates)
|
|
assert deleted == 2 # two duplicate (S1, E2) rows removed
|
|
|
|
# Verify the post-cleanup state directly via SQLAlchemy.
|
|
async with AsyncSession(in_memory_engine) as session:
|
|
rows = (
|
|
await session.execute(
|
|
select(Episode).order_by(Episode.id)
|
|
)
|
|
).scalars().all()
|
|
remaining_ids = [int(r.id) for r in rows]
|
|
# e1, e2 (lowest of the dup group), and e5 survive; e3, e4
|
|
# were the duplicates and are gone.
|
|
assert remaining_ids == [e1, e2, e5]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_keeps_lowest_even_with_title_metadata(in_memory_engine):
|
|
"""When a higher-id row has populated metadata but the lowest-id
|
|
row is empty, the lowest-id row is still kept — that's the
|
|
documented behavior (oldest insert wins)."""
|
|
async with AsyncSession(in_memory_engine) as session:
|
|
sid = await _add_series(session, "metadata")
|
|
e1 = await _add_episode(session, sid, 1, 1) # no title
|
|
e2 = await _add_episode(
|
|
session, sid, 1, 1, title="Better Episode 1"
|
|
)
|
|
|
|
duplicates = await cli.find_duplicate_episodes()
|
|
assert len(duplicates) == 1
|
|
|
|
await cli.delete_duplicate_episodes(duplicates)
|
|
|
|
async with AsyncSession(in_memory_engine) as session:
|
|
rows = (
|
|
await session.execute(select(Episode))
|
|
).scalars().all()
|
|
assert len(rows) == 1
|
|
assert int(rows[0].id) == e1
|
|
assert rows[0].title is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_full_workflow_is_idempotent(in_memory_engine):
|
|
"""Running find -> delete -> find yields nothing the second
|
|
time. Mirrors the production usage pattern."""
|
|
async with AsyncSession(in_memory_engine) as session:
|
|
sid = await _add_series(session, "idempotent")
|
|
for ep_num in (1, 2, 3):
|
|
for _ in range(3): # 3 copies of each
|
|
await _add_episode(session, sid, 1, ep_num)
|
|
|
|
first = await cli.find_duplicate_episodes()
|
|
assert len(first) == 3
|
|
|
|
await cli.delete_duplicate_episodes(first)
|
|
second = await cli.find_duplicate_episodes()
|
|
assert second == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_max_series_filter_limits_report(in_memory_engine):
|
|
"""``--max-series`` caps which series appear in the report but
|
|
the underlying find still returns every duplicate for those
|
|
series."""
|
|
async with AsyncSession(in_memory_engine) as session:
|
|
s1 = await _add_series(session, "alpha")
|
|
s2 = await _add_series(session, "beta")
|
|
s3 = await _add_series(session, "gamma")
|
|
for sid in (s1, s2, s3):
|
|
for ep_num in (1, 2):
|
|
for _ in range(2): # duplicates
|
|
await _add_episode(session, sid, 1, ep_num)
|
|
|
|
# Without filter: all three series have duplicates.
|
|
full = await cli.find_duplicate_episodes()
|
|
series_in_full = {t[0] for t in full}
|
|
assert series_in_full == {s1, s2, s3}
|
|
|
|
# With --max-series=2: only two series appear.
|
|
limited = await cli.find_duplicate_episodes(max_series=2)
|
|
series_in_limited = {t[0] for t in limited}
|
|
assert len(series_in_limited) == 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_format_report_human_readable(in_memory_engine):
|
|
"""The report includes the series name, season, episode, count
|
|
and keep_id for each duplicate tuple."""
|
|
async with AsyncSession(in_memory_engine) as session:
|
|
sid = await _add_series(session, "attack-on-titan")
|
|
await _add_episode(session, sid, 1, 1)
|
|
await _add_episode(session, sid, 1, 1)
|
|
|
|
duplicates = await cli.find_duplicate_episodes()
|
|
names = await cli._load_series_names([sid])
|
|
|
|
report = cli.format_report(duplicates, names)
|
|
|
|
assert "Attack On Titan" in report or "Attack-on-Titan" in report
|
|
# The row's keep_id column should appear, plus a count of "2".
|
|
assert "2" in report
|
|
assert "found" in report.lower()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_format_report_handles_no_duplicates():
|
|
"""Empty input yields a friendly 'nothing to do' message."""
|
|
report = cli.format_report([], {})
|
|
assert "no duplicate" in report.lower()
|
|
|
|
|
|
def test_argparse_defaults():
|
|
"""The CLI defaults to dry-run when --apply is not given."""
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--apply", action="store_true")
|
|
parser.add_argument("--max-series", type=int, default=None)
|
|
|
|
# Simulate argv without --apply.
|
|
args = parser.parse_args([])
|
|
assert args.apply is False
|
|
assert args.max_series is None |