TASK-023: Make database migrations atomic
Replace non-atomic db.executescript() with explicit transaction control. Wrap each migration's DDL statements and schema_migrations insert in a single BEGIN IMMEDIATE ... COMMIT transaction to ensure atomicity. Changes: - Add _parse_migration_statements() to split migration scripts into individual statements while handling comments and string literals - Update _apply_migration() to wrap all statements in a single explicit transaction with rollback on error - Ensure _get_current_schema_version() uses execute() instead of executescript() - Add 9 new tests for migration atomicity and statement parsing - Update Backend-Development.md with migration authoring guidelines If a crash occurs between DDL execution and schema_migrations insert, the next startup will re-apply the entire migration atomically, preventing partial migrations and data corruption. Test coverage: 98% on db.py (up from 55%) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
@@ -1,9 +1,16 @@
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import aiosqlite
|
||||
import pytest
|
||||
|
||||
from app.db import open_db
|
||||
from app.db import (
|
||||
_apply_migration,
|
||||
_parse_migration_statements,
|
||||
init_db,
|
||||
open_db,
|
||||
)
|
||||
|
||||
|
||||
async def test_open_db_applies_hardening_pragmas(tmp_path: Path) -> None:
|
||||
@@ -56,3 +63,181 @@ async def test_open_db_respects_busy_timeout_for_concurrent_writes(tmp_path: Pat
|
||||
assert row is not None and row[0] == "locked"
|
||||
finally:
|
||||
await connection_a.close()
|
||||
|
||||
|
||||
async def test_parse_migration_statements_single_statement() -> None:
|
||||
"""Test parsing a single statement without comments."""
|
||||
script = "CREATE TABLE test (id INTEGER PRIMARY KEY);"
|
||||
statements = await _parse_migration_statements(script)
|
||||
assert len(statements) == 1
|
||||
assert statements[0] == "CREATE TABLE test (id INTEGER PRIMARY KEY)"
|
||||
|
||||
|
||||
async def test_parse_migration_statements_multiple_statements() -> None:
|
||||
"""Test parsing multiple statements separated by semicolons."""
|
||||
script = """
|
||||
CREATE TABLE test (id INTEGER PRIMARY KEY);
|
||||
CREATE INDEX idx_test ON test(id);
|
||||
"""
|
||||
statements = await _parse_migration_statements(script)
|
||||
assert len(statements) == 2
|
||||
assert "CREATE TABLE test" in statements[0]
|
||||
assert "CREATE INDEX idx_test" in statements[1]
|
||||
|
||||
|
||||
async def test_parse_migration_statements_with_line_comments() -> None:
|
||||
"""Test parsing statements with line comments."""
|
||||
script = """
|
||||
-- This is a comment
|
||||
CREATE TABLE test (id INTEGER PRIMARY KEY);
|
||||
-- Another comment
|
||||
"""
|
||||
statements = await _parse_migration_statements(script)
|
||||
assert len(statements) == 1
|
||||
assert "CREATE TABLE test" in statements[0]
|
||||
|
||||
|
||||
async def test_parse_migration_statements_with_block_comments() -> None:
|
||||
"""Test parsing statements with block comments."""
|
||||
script = """
|
||||
/* Block comment */
|
||||
CREATE TABLE test (id INTEGER PRIMARY KEY);
|
||||
/* Another block */
|
||||
"""
|
||||
statements = await _parse_migration_statements(script)
|
||||
assert len(statements) == 1
|
||||
assert "CREATE TABLE test" in statements[0]
|
||||
|
||||
|
||||
async def test_parse_migration_statements_with_string_literals() -> None:
|
||||
"""Test parsing statements with string literals containing semicolons."""
|
||||
script = """
|
||||
CREATE TABLE test (id INTEGER PRIMARY KEY, data TEXT);
|
||||
INSERT INTO test (data) VALUES ('value; with; semicolons');
|
||||
"""
|
||||
statements = await _parse_migration_statements(script)
|
||||
assert len(statements) == 2
|
||||
assert "CREATE TABLE test" in statements[0]
|
||||
assert "INSERT INTO test" in statements[1]
|
||||
assert "value; with; semicolons" in statements[1]
|
||||
|
||||
|
||||
async def test_parse_migration_statements_with_escaped_quotes() -> None:
|
||||
"""Test parsing statements with escaped quotes in string literals."""
|
||||
script = """
|
||||
INSERT INTO test (data) VALUES ('it''s a test');
|
||||
"""
|
||||
statements = await _parse_migration_statements(script)
|
||||
assert len(statements) == 1
|
||||
assert "it''s a test" in statements[0]
|
||||
|
||||
|
||||
async def test_apply_migration_is_atomic_success(tmp_path: Path) -> None:
|
||||
"""Test that migration is atomic when all statements succeed."""
|
||||
database_path = str(tmp_path / "bangui_atomic.db")
|
||||
db = await open_db(database_path)
|
||||
try:
|
||||
# Initialize schema_migrations table
|
||||
await db.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations (version INTEGER PRIMARY KEY, migrated_at TEXT);"
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
# Apply a test migration
|
||||
await _apply_migration(db, 1)
|
||||
|
||||
# Verify the migration was recorded
|
||||
async with db.execute(
|
||||
"SELECT version FROM schema_migrations WHERE version = 1;"
|
||||
) as cursor:
|
||||
row = await cursor.fetchone()
|
||||
assert row is not None and row[0] == 1
|
||||
|
||||
# Verify the schema tables exist
|
||||
async with db.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='settings';"
|
||||
) as cursor:
|
||||
row = await cursor.fetchone()
|
||||
assert row is not None
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
async def test_apply_migration_is_atomic_rollback(tmp_path: Path) -> None:
|
||||
"""Test that migration is rolled back when a statement fails.
|
||||
|
||||
This test verifies that when an error occurs mid-migration, the
|
||||
transaction is rolled back and the schema_migrations table is NOT updated.
|
||||
"""
|
||||
database_path = str(tmp_path / "bangui_rollback.db")
|
||||
db = await open_db(database_path)
|
||||
try:
|
||||
# Initialize schema_migrations table
|
||||
await db.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations (version INTEGER PRIMARY KEY, migrated_at TEXT);"
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
# Create a custom migration that will fail
|
||||
from app import db as db_module
|
||||
|
||||
original_migrations = db_module._MIGRATIONS.copy()
|
||||
|
||||
# Add a migration that will fail on the second statement
|
||||
db_module._MIGRATIONS[99] = """
|
||||
CREATE TABLE test_rollback (id INTEGER PRIMARY KEY);
|
||||
INSERT INTO nonexistent_table VALUES (1);
|
||||
"""
|
||||
|
||||
try:
|
||||
# Attempt migration; it should fail
|
||||
with pytest.raises(Exception): # sqlite3 will raise an error
|
||||
await _apply_migration(db, 99)
|
||||
|
||||
# Verify the migration was NOT recorded
|
||||
async with db.execute(
|
||||
"SELECT version FROM schema_migrations WHERE version = 99;"
|
||||
) as cursor:
|
||||
row = await cursor.fetchone()
|
||||
assert row is None
|
||||
|
||||
# Verify the test table was NOT created (rollback occurred)
|
||||
async with db.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='test_rollback';"
|
||||
) as cursor:
|
||||
row = await cursor.fetchone()
|
||||
assert row is None
|
||||
finally:
|
||||
# Restore original migrations
|
||||
db_module._MIGRATIONS = original_migrations
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
async def test_init_db_idempotent(tmp_path: Path) -> None:
|
||||
"""Test that init_db is idempotent."""
|
||||
database_path = str(tmp_path / "bangui_idempotent.db")
|
||||
db = await open_db(database_path)
|
||||
try:
|
||||
# Initialize once
|
||||
await init_db(db)
|
||||
|
||||
# Get schema version
|
||||
async with db.execute(
|
||||
"SELECT MAX(version) FROM schema_migrations;"
|
||||
) as cursor:
|
||||
row1 = await cursor.fetchone()
|
||||
|
||||
# Initialize again (should be no-op)
|
||||
await init_db(db)
|
||||
|
||||
# Verify schema version is unchanged
|
||||
async with db.execute(
|
||||
"SELECT MAX(version) FROM schema_migrations;"
|
||||
) as cursor:
|
||||
row2 = await cursor.fetchone()
|
||||
|
||||
assert row1 == row2
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user