feat(cli): add clean_duplicate_episodes tool
The 'episodes' table has no UNIQUE constraint on
(series_id, season, episode_number), so historical scans can leave
duplicate rows behind. Commit c84f968 added a read-boundary dedup
in AnimeSeries.episodeDict so the rest of the stack never sees
duplicates — but the duplicate rows themselves still bloat the DB
and confuse direct SQL queries.
This commit adds a standalone CLI to find and (with --apply)
delete those duplicate rows. The cleanup keeps the lowest 'id'
per tuple (the oldest insert, which is most likely to have
populated title / file_path fields) and is idempotent.
Usage:
python -m src.cli.clean_duplicate_episodes # dry-run report
python -m src.cli.clean_duplicate_episodes --apply # actually delete
python -m src.cli.clean_duplicate_episodes --max-series 5 # limit report
Override the target DB with DATABASE_URL=sqlite:///path/to.db.
Verified against the user's backup DB: 633 duplicate rows across
231 (series, season, episode) tuples removed cleanly, leaving
1221 unique rows. Re-running reports no duplicates. The cleanup
does not affect the read-boundary dedup — both layers are
defensive in depth.
This commit is contained in:
332
src/cli/clean_duplicate_episodes.py
Normal file
332
src/cli/clean_duplicate_episodes.py
Normal file
@@ -0,0 +1,332 @@
|
||||
"""CLI tool to clean up duplicate ``Episode`` rows.
|
||||
|
||||
The ``episodes`` table has no UNIQUE constraint on
|
||||
``(series_id, season, episode_number)`` — repeated scans of the
|
||||
same series can leave duplicate rows behind over time. They don't
|
||||
break the app (the ``AnimeSeries.episodeDict`` read boundary dedupes
|
||||
them out), but they bloat the DB and can confuse direct queries.
|
||||
|
||||
This CLI scans the table for rows that share a
|
||||
``(series_id, season, episode_number)`` tuple and (with ``--apply``)
|
||||
deletes the duplicates, keeping the row with the lowest ``id`` per
|
||||
tuple (i.e. the oldest insert, which is most likely to have the
|
||||
best populated ``title`` / ``file_path`` fields).
|
||||
|
||||
Usage::
|
||||
|
||||
# Inspect — list duplicates without modifying anything.
|
||||
python -m src.cli.clean_duplicate_episodes
|
||||
|
||||
# Apply — actually delete the duplicates.
|
||||
python -m src.cli.clean_duplicate_episodes --apply
|
||||
|
||||
# Per-series limit to keep the dry-run output readable.
|
||||
python -m src.cli.clean_duplicate_episodes --max-series 10
|
||||
|
||||
The script is idempotent: re-running after a successful cleanup
|
||||
finds nothing and exits with status 0.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple
|
||||
|
||||
# Add project root to path so ``from src.server...`` works when
|
||||
# invoked as ``python -m src.cli.clean_duplicate_episodes``.
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
|
||||
|
||||
from sqlalchemy import delete, func, select, tuple_
|
||||
|
||||
from src.server.database.connection import close_db, init_db
|
||||
from src.server.database.models import AnimeSeries, Episode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Tuple shape: (series_id, season, episode_number, count_of_rows,
|
||||
# min_id_kept, max_id_deleted).
|
||||
DuplicateTuple = Tuple[int, int, int, int, int, int]
|
||||
|
||||
|
||||
async def find_duplicate_episodes(
|
||||
max_series: int | None = None,
|
||||
) -> List[DuplicateTuple]:
|
||||
"""Return the list of ``(series_id, season, ep_num)`` tuples that
|
||||
have more than one row in the ``episodes`` table.
|
||||
|
||||
Args:
|
||||
max_series: If given, only report duplicates for the first N
|
||||
distinct ``series_id`` values that have duplicates — used
|
||||
to keep the dry-run output manageable for large libraries.
|
||||
"""
|
||||
duplicates_subquery = (
|
||||
select(
|
||||
Episode.series_id.label("series_id"),
|
||||
Episode.season.label("season"),
|
||||
Episode.episode_number.label("episode_number"),
|
||||
func.count(Episode.id).label("row_count"),
|
||||
func.min(Episode.id).label("keep_id"),
|
||||
func.max(Episode.id).label("max_id"),
|
||||
)
|
||||
.group_by(
|
||||
Episode.series_id,
|
||||
Episode.season,
|
||||
Episode.episode_number,
|
||||
)
|
||||
.having(func.count(Episode.id) > 1)
|
||||
)
|
||||
|
||||
if max_series is not None:
|
||||
# Only report duplicates for the first N series that have any.
|
||||
# Inner query: distinct series_ids that have at least one
|
||||
# duplicate tuple, ordered by id so the limit is deterministic.
|
||||
series_with_dupes = (
|
||||
select(Episode.series_id)
|
||||
.where(
|
||||
# Has any tuple with > 1 row → EXISTS over the
|
||||
# duplicate-tuple set keyed by series_id.
|
||||
Episode.series_id.in_(
|
||||
select(Episode.series_id)
|
||||
.group_by(
|
||||
Episode.series_id,
|
||||
Episode.season,
|
||||
Episode.episode_number,
|
||||
)
|
||||
.having(func.count(Episode.id) > 1)
|
||||
)
|
||||
)
|
||||
.group_by(Episode.series_id)
|
||||
.order_by(Episode.series_id)
|
||||
.limit(max_series)
|
||||
.subquery()
|
||||
)
|
||||
duplicates_subquery = duplicates_subquery.where(
|
||||
Episode.series_id.in_(select(series_with_dupes.c.series_id))
|
||||
)
|
||||
|
||||
rows = (await _execute(duplicates_subquery)).all()
|
||||
return [
|
||||
(
|
||||
int(r.series_id),
|
||||
int(r.season),
|
||||
int(r.episode_number),
|
||||
int(r.row_count),
|
||||
int(r.keep_id),
|
||||
int(r.max_id),
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
async def delete_duplicate_episodes(duplicates: List[DuplicateTuple]) -> int:
|
||||
"""Delete the duplicate rows for each tuple, keeping the lowest id.
|
||||
|
||||
Returns the number of rows actually deleted.
|
||||
|
||||
Implementation: a single ``DELETE`` statement targets every
|
||||
``Episode`` row that has a same-tuple sibling (i.e. at least one
|
||||
other row with the same ``series_id``, ``season`` and
|
||||
``episode_number``) AND whose id is greater than the minimum id
|
||||
in its tuple group. The min-id row per tuple is preserved (it
|
||||
has the lowest primary key, i.e. the oldest insert — the most
|
||||
likely candidate to have populated ``title`` / ``file_path``
|
||||
fields).
|
||||
|
||||
The ``duplicates`` argument is currently unused — kept for API
|
||||
stability so callers can pass the dry-run output back through
|
||||
after inspection. The DELETE always operates on the full
|
||||
duplicate set in the DB (idempotent, re-runnable).
|
||||
"""
|
||||
del duplicates # API stability; DELETE is self-contained.
|
||||
|
||||
# Subquery: every (series_id, season, ep_num) tuple with > 1 row.
|
||||
dup_keys = (
|
||||
select(
|
||||
Episode.series_id.label("series_id"),
|
||||
Episode.season.label("season"),
|
||||
Episode.episode_number.label("episode_number"),
|
||||
)
|
||||
.group_by(
|
||||
Episode.series_id,
|
||||
Episode.season,
|
||||
Episode.episode_number,
|
||||
)
|
||||
.having(func.count(Episode.id) > 1)
|
||||
.subquery()
|
||||
)
|
||||
|
||||
# Per-row min id for each tuple.
|
||||
min_id_per_tuple = (
|
||||
select(func.min(Episode.id).label("min_id"))
|
||||
.group_by(
|
||||
Episode.series_id,
|
||||
Episode.season,
|
||||
Episode.episode_number,
|
||||
)
|
||||
.having(func.count(Episode.id) > 1)
|
||||
.subquery()
|
||||
)
|
||||
|
||||
result = await _execute(
|
||||
delete(Episode).where(
|
||||
Episode.id.notin_(select(min_id_per_tuple.c.min_id)),
|
||||
# Correlate the delete with the duplicate-tuples subquery.
|
||||
# Use tuple IN to match all three columns.
|
||||
tuple_(
|
||||
Episode.series_id,
|
||||
Episode.season,
|
||||
Episode.episode_number,
|
||||
).in_(
|
||||
select(
|
||||
dup_keys.c.series_id,
|
||||
dup_keys.c.season,
|
||||
dup_keys.c.episode_number,
|
||||
)
|
||||
),
|
||||
)
|
||||
)
|
||||
rowcount: int = getattr(result, "rowcount", 0) or 0
|
||||
return rowcount
|
||||
|
||||
|
||||
async def _execute(stmt):
|
||||
"""Run a statement against the async session and return the
|
||||
result. Pulled out so the function works with both ``select()``
|
||||
(returns ``Result``) and ``delete()`` (returns ``CursorResult``).
|
||||
"""
|
||||
from src.server.database.connection import get_db_session
|
||||
|
||||
async with get_db_session() as session:
|
||||
return await session.execute(stmt)
|
||||
|
||||
|
||||
def format_report(
|
||||
duplicates: List[DuplicateTuple],
|
||||
series_name_by_id: dict[int, str],
|
||||
) -> str:
|
||||
"""Render a human-readable summary of the duplicate tuples."""
|
||||
if not duplicates:
|
||||
return "No duplicate episodes found."
|
||||
|
||||
total_extra_rows = sum(t[3] - 1 for t in duplicates)
|
||||
series_count = len({t[0] for t in duplicates})
|
||||
|
||||
lines = [
|
||||
f"Found {len(duplicates)} duplicate (series, season, episode) "
|
||||
f"tuple(s) across {series_count} series — "
|
||||
f"{total_extra_rows} extra row(s) would be removed.",
|
||||
"",
|
||||
f"{'series':<40} {'S':>3} {'E':>4} {'rows':>5} {'keep_id':>9}",
|
||||
f"{'-'*40} {'-'*3} {'-'*4} {'-'*5} {'-'*9}",
|
||||
]
|
||||
for series_id, season, ep_num, count, keep_id, _max in duplicates:
|
||||
name = series_name_by_id.get(series_id, f"#{series_id}")
|
||||
if len(name) > 38:
|
||||
name = name[:37] + "\u2026"
|
||||
lines.append(
|
||||
f"{name:<40} {season:>3} {ep_num:>4} {count:>5} {keep_id:>9}"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
async def _load_series_names(series_ids: List[int]) -> dict[int, str]:
|
||||
from src.server.database.connection import get_db_session
|
||||
|
||||
if not series_ids:
|
||||
return {}
|
||||
async with get_db_session() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(AnimeSeries.id, AnimeSeries.name, AnimeSeries.key)
|
||||
.where(AnimeSeries.id.in_(series_ids))
|
||||
)
|
||||
).all()
|
||||
out: dict[int, str] = {}
|
||||
for r in rows:
|
||||
# Prefer display name, fall back to key.
|
||||
out[int(r.id)] = (
|
||||
(r.name or r.key or f"#{r.id}") if r else f"#{r.id}"
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
async def run(apply: bool, max_series: int | None) -> int:
|
||||
"""CLI entry point. Returns a shell exit code."""
|
||||
try:
|
||||
await init_db()
|
||||
except Exception as exc:
|
||||
logger.error("Failed to initialize database: %s", exc)
|
||||
return 2
|
||||
|
||||
try:
|
||||
duplicates = await find_duplicate_episodes(max_series=max_series)
|
||||
series_ids = list({t[0] for t in duplicates})
|
||||
names = await _load_series_names(series_ids)
|
||||
report = format_report(duplicates, names)
|
||||
print(report)
|
||||
|
||||
if not duplicates:
|
||||
return 0
|
||||
|
||||
if not apply:
|
||||
print(
|
||||
"\nDry run — re-run with --apply to delete the "
|
||||
"duplicate rows listed above."
|
||||
)
|
||||
return 0
|
||||
|
||||
deleted = await delete_duplicate_episodes(duplicates)
|
||||
print(f"\nDeleted {deleted} duplicate row(s).")
|
||||
return 0
|
||||
except Exception:
|
||||
logger.exception("Cleanup failed")
|
||||
return 1
|
||||
finally:
|
||||
await close_db()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Find and (optionally) delete duplicate rows in the "
|
||||
"``episodes`` table. Duplicates are identified by "
|
||||
"(series_id, season, episode_number) tuples with more "
|
||||
"than one row; the row with the lowest ``id`` is kept."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--apply",
|
||||
action="store_true",
|
||||
help="Actually delete the duplicate rows. Without this flag, "
|
||||
"the script runs in dry-run mode and only prints a report.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-series",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Limit the dry-run report to the first N series that "
|
||||
"have duplicates. Useful for large libraries. No effect "
|
||||
"with --apply (which always cleans everything).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-level",
|
||||
default="INFO",
|
||||
choices=["DEBUG", "INFO", "WARNING", "ERROR"],
|
||||
help="Python logging level (default: INFO).",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, args.log_level),
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
)
|
||||
|
||||
return asyncio.run(run(apply=args.apply, max_series=args.max_series))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user