feat(cli): add clean_duplicate_episodes tool
The 'episodes' table has no UNIQUE constraint on
(series_id, season, episode_number), so historical scans can leave
duplicate rows behind. Commit c84f968 added a read-boundary dedup
in AnimeSeries.episodeDict so the rest of the stack never sees
duplicates — but the duplicate rows themselves still bloat the DB
and confuse direct SQL queries.
This commit adds a standalone CLI to find and (with --apply)
delete those duplicate rows. The cleanup keeps the lowest 'id'
per tuple (the oldest insert, which is most likely to have
populated title / file_path fields) and is idempotent.
Usage:
python -m src.cli.clean_duplicate_episodes # dry-run report
python -m src.cli.clean_duplicate_episodes --apply # actually delete
python -m src.cli.clean_duplicate_episodes --max-series 5 # limit report
Override the target DB with DATABASE_URL=sqlite:///path/to.db.
Verified against the user's backup DB: 633 duplicate rows across
231 (series, season, episode) tuples removed cleanly, leaving
1221 unique rows. Re-running reports no duplicates. The cleanup
does not affect the read-boundary dedup — both layers are
defensive in depth.
This commit is contained in:
280
tests/unit/test_clean_duplicate_episodes_cli.py
Normal file
280
tests/unit/test_clean_duplicate_episodes_cli.py
Normal file
@@ -0,0 +1,280 @@
|
||||
"""Unit tests for the ``clean_duplicate_episodes`` CLI.
|
||||
|
||||
Exercises the core logic (find / delete) against an in-memory
|
||||
SQLite engine so the test is hermetic. The CLI module reads the
|
||||
global async session factory from ``src.server.database.connection``
|
||||
— the test patches that factory with an in-memory engine.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
AsyncEngine,
|
||||
AsyncSession,
|
||||
async_sessionmaker,
|
||||
create_async_engine,
|
||||
)
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from src.cli import clean_duplicate_episodes as cli
|
||||
from src.server.database import connection as conn_module
|
||||
from src.server.database.base import Base
|
||||
from src.server.database.models import AnimeSeries, Episode
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def in_memory_engine():
|
||||
"""Provide an in-memory async SQLite engine with the schema
|
||||
already created, and patch the connection module's session
|
||||
factory to use it for the duration of the test."""
|
||||
engine: AsyncEngine = create_async_engine(
|
||||
"sqlite+aiosqlite:///:memory:",
|
||||
echo=False,
|
||||
poolclass=StaticPool,
|
||||
)
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
|
||||
factory = async_sessionmaker(
|
||||
bind=engine,
|
||||
class_=AsyncSession,
|
||||
expire_on_commit=False,
|
||||
)
|
||||
|
||||
# Monkey-patch the global session factory.
|
||||
original_factory = conn_module._session_factory
|
||||
conn_module._session_factory = factory
|
||||
try:
|
||||
yield engine
|
||||
finally:
|
||||
conn_module._session_factory = original_factory
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def _add_series(session: AsyncSession, key: str) -> int:
|
||||
series = AnimeSeries(
|
||||
key=key,
|
||||
name=key.replace("-", " ").title(),
|
||||
site="https://aniworld.to",
|
||||
folder=f"/anime/{key}",
|
||||
)
|
||||
session.add(series)
|
||||
await session.commit()
|
||||
await session.refresh(series)
|
||||
return int(series.id)
|
||||
|
||||
|
||||
async def _add_episode(
|
||||
session: AsyncSession,
|
||||
series_id: int,
|
||||
season: int,
|
||||
ep_num: int,
|
||||
is_downloaded: bool = False,
|
||||
title: str | None = None,
|
||||
) -> int:
|
||||
ep = Episode(
|
||||
series_id=series_id,
|
||||
season=season,
|
||||
episode_number=ep_num,
|
||||
is_downloaded=is_downloaded,
|
||||
title=title,
|
||||
)
|
||||
session.add(ep)
|
||||
await session.commit()
|
||||
await session.refresh(ep)
|
||||
return int(ep.id)
|
||||
|
||||
|
||||
def _tup(d):
|
||||
"""Convert a SQLAlchemy row to the (sid, season, ep_num, count,
|
||||
keep_id, max_id) tuple shape the CLI uses."""
|
||||
return (
|
||||
int(d.series_id),
|
||||
int(d.season),
|
||||
int(d.episode_number),
|
||||
int(d.row_count),
|
||||
int(d.keep_id),
|
||||
int(d.max_id),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_returns_empty_when_no_duplicates(in_memory_engine):
|
||||
"""With one row per (series, season, ep_num), nothing is found."""
|
||||
async with AsyncSession(in_memory_engine) as session:
|
||||
sid = await _add_series(session, "no-dupes")
|
||||
await _add_episode(session, sid, 1, 1)
|
||||
await _add_episode(session, sid, 1, 2)
|
||||
await _add_episode(session, sid, 2, 1)
|
||||
|
||||
duplicates = await cli.find_duplicate_episodes()
|
||||
assert duplicates == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_groups_duplicates_per_tuple(in_memory_engine):
|
||||
"""Three copies of (S1, E2) collapse to one DuplicateTuple with
|
||||
count=3 and keep_id = the lowest id."""
|
||||
async with AsyncSession(in_memory_engine) as session:
|
||||
sid = await _add_series(session, "triple")
|
||||
e1 = await _add_episode(session, sid, 1, 1)
|
||||
# Three rows for (S1, E2): keep_id should be e2 (lowest)
|
||||
e2 = await _add_episode(session, sid, 1, 2)
|
||||
e3 = await _add_episode(session, sid, 1, 2)
|
||||
e4 = await _add_episode(session, sid, 1, 2)
|
||||
await _add_episode(session, sid, 1, 3)
|
||||
|
||||
duplicates = await cli.find_duplicate_episodes()
|
||||
|
||||
# Only one duplicate tuple (S1, E2); (S1, E1) and (S1, E3) are
|
||||
# unique and must not appear.
|
||||
assert len(duplicates) == 1
|
||||
d = duplicates[0]
|
||||
assert (d[0], d[1], d[2]) == (sid, 1, 2)
|
||||
assert d[3] == 3
|
||||
assert d[4] == e2 # lowest id wins
|
||||
assert d[5] == e4 # max id (for the report)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_keeps_lowest_id_per_tuple(in_memory_engine):
|
||||
"""The DELETE call preserves the lowest-id row per duplicate
|
||||
tuple and removes the rest."""
|
||||
async with AsyncSession(in_memory_engine) as session:
|
||||
sid = await _add_series(session, "keep-lowest")
|
||||
e1 = await _add_episode(session, sid, 1, 1) # unique
|
||||
e2 = await _add_episode(session, sid, 1, 2) # lowest of dupes
|
||||
await _add_episode(session, sid, 1, 2) # e3 - duplicate of e2
|
||||
await _add_episode(session, sid, 1, 2) # e4 - duplicate of e2
|
||||
e5 = await _add_episode(session, sid, 1, 3) # unique
|
||||
|
||||
duplicates = await cli.find_duplicate_episodes()
|
||||
assert len(duplicates) == 1
|
||||
|
||||
deleted = await cli.delete_duplicate_episodes(duplicates)
|
||||
assert deleted == 2 # two duplicate (S1, E2) rows removed
|
||||
|
||||
# Verify the post-cleanup state directly via SQLAlchemy.
|
||||
async with AsyncSession(in_memory_engine) as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(Episode).order_by(Episode.id)
|
||||
)
|
||||
).scalars().all()
|
||||
remaining_ids = [int(r.id) for r in rows]
|
||||
# e1, e2 (lowest of the dup group), and e5 survive; e3, e4
|
||||
# were the duplicates and are gone.
|
||||
assert remaining_ids == [e1, e2, e5]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_keeps_lowest_even_with_title_metadata(in_memory_engine):
|
||||
"""When a higher-id row has populated metadata but the lowest-id
|
||||
row is empty, the lowest-id row is still kept — that's the
|
||||
documented behavior (oldest insert wins)."""
|
||||
async with AsyncSession(in_memory_engine) as session:
|
||||
sid = await _add_series(session, "metadata")
|
||||
e1 = await _add_episode(session, sid, 1, 1) # no title
|
||||
e2 = await _add_episode(
|
||||
session, sid, 1, 1, title="Better Episode 1"
|
||||
)
|
||||
|
||||
duplicates = await cli.find_duplicate_episodes()
|
||||
assert len(duplicates) == 1
|
||||
|
||||
await cli.delete_duplicate_episodes(duplicates)
|
||||
|
||||
async with AsyncSession(in_memory_engine) as session:
|
||||
rows = (
|
||||
await session.execute(select(Episode))
|
||||
).scalars().all()
|
||||
assert len(rows) == 1
|
||||
assert int(rows[0].id) == e1
|
||||
assert rows[0].title is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_full_workflow_is_idempotent(in_memory_engine):
|
||||
"""Running find -> delete -> find yields nothing the second
|
||||
time. Mirrors the production usage pattern."""
|
||||
async with AsyncSession(in_memory_engine) as session:
|
||||
sid = await _add_series(session, "idempotent")
|
||||
for ep_num in (1, 2, 3):
|
||||
for _ in range(3): # 3 copies of each
|
||||
await _add_episode(session, sid, 1, ep_num)
|
||||
|
||||
first = await cli.find_duplicate_episodes()
|
||||
assert len(first) == 3
|
||||
|
||||
await cli.delete_duplicate_episodes(first)
|
||||
second = await cli.find_duplicate_episodes()
|
||||
assert second == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_max_series_filter_limits_report(in_memory_engine):
|
||||
"""``--max-series`` caps which series appear in the report but
|
||||
the underlying find still returns every duplicate for those
|
||||
series."""
|
||||
async with AsyncSession(in_memory_engine) as session:
|
||||
s1 = await _add_series(session, "alpha")
|
||||
s2 = await _add_series(session, "beta")
|
||||
s3 = await _add_series(session, "gamma")
|
||||
for sid in (s1, s2, s3):
|
||||
for ep_num in (1, 2):
|
||||
for _ in range(2): # duplicates
|
||||
await _add_episode(session, sid, 1, ep_num)
|
||||
|
||||
# Without filter: all three series have duplicates.
|
||||
full = await cli.find_duplicate_episodes()
|
||||
series_in_full = {t[0] for t in full}
|
||||
assert series_in_full == {s1, s2, s3}
|
||||
|
||||
# With --max-series=2: only two series appear.
|
||||
limited = await cli.find_duplicate_episodes(max_series=2)
|
||||
series_in_limited = {t[0] for t in limited}
|
||||
assert len(series_in_limited) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_format_report_human_readable(in_memory_engine):
|
||||
"""The report includes the series name, season, episode, count
|
||||
and keep_id for each duplicate tuple."""
|
||||
async with AsyncSession(in_memory_engine) as session:
|
||||
sid = await _add_series(session, "attack-on-titan")
|
||||
await _add_episode(session, sid, 1, 1)
|
||||
await _add_episode(session, sid, 1, 1)
|
||||
|
||||
duplicates = await cli.find_duplicate_episodes()
|
||||
names = await cli._load_series_names([sid])
|
||||
|
||||
report = cli.format_report(duplicates, names)
|
||||
|
||||
assert "Attack On Titan" in report or "Attack-on-Titan" in report
|
||||
# The row's keep_id column should appear, plus a count of "2".
|
||||
assert "2" in report
|
||||
assert "found" in report.lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_format_report_handles_no_duplicates():
|
||||
"""Empty input yields a friendly 'nothing to do' message."""
|
||||
report = cli.format_report([], {})
|
||||
assert "no duplicate" in report.lower()
|
||||
|
||||
|
||||
def test_argparse_defaults():
|
||||
"""The CLI defaults to dry-run when --apply is not given."""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
parser.add_argument("--max-series", type=int, default=None)
|
||||
|
||||
# Simulate argv without --apply.
|
||||
args = parser.parse_args([])
|
||||
assert args.apply is False
|
||||
assert args.max_series is None
|
||||
Reference in New Issue
Block a user