nl-processing 0.3.0__tar.gz → 0.5.0__tar.gz

{nl_processing-0.3.0 → nl_processing-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nl_processing
-Version: 0.3.0
+Version: 0.5.0
 Summary: Natural language processing playground
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
@@ -9,6 +9,7 @@ Requires-Dist: langchain<1,>=0.3
 Requires-Dist: langchain-openai<1,>=0.3
 Requires-Dist: opencv-python<5,>=4.10
 Requires-Dist: asyncpg<1,>=0.30
+Requires-Dist: aiosqlite<1,>=0.20
 
 # nl_processing
 

{nl_processing-0.3.0 → nl_processing-0.5.0}/nl_processing/database/backend/_neon_exercise.py

@@ -90,3 +90,30 @@ async def mark_event(
         await conn.execute(mark_event_applied_query(table), event_id)
     except asyncpg.PostgresError as exc:
         raise DatabaseError(str(exc)) from exc
+
+
+async def atomic_apply_delta(
+    conn: asyncpg.Connection,  # type: ignore[type-arg]
+    score_table: str,
+    events_table: str,
+    user_id: str,
+    event_id: str,
+    source_word_id: int,
+    delta: int,
+) -> bool:
+    """Atomically check-apply-mark a score delta in one transaction."""
+    try:
+        async with conn.transaction():
+            already = await conn.fetchrow(check_event_applied_query(events_table), event_id)
+            if already is not None:
+                return False
+            await conn.fetchrow(
+                increment_score_query(score_table),
+                user_id,
+                source_word_id,
+                delta,
+            )
+            await conn.execute(mark_event_applied_query(events_table), event_id)
+            return True
+    except asyncpg.PostgresError as exc:
+        raise DatabaseError(str(exc)) from exc

{nl_processing-0.3.0 → nl_processing-0.5.0}/nl_processing/database/backend/_queries.py

@@ -133,6 +133,19 @@ def get_user_words_query(
     return query
 
 
+def count_user_words_query(language: str, word_type: str | None) -> str:
+    # Table name from Language enum value, not user input  # noqa: S608
+    query = f"""
+        SELECT COUNT(*) AS cnt
+        FROM user_words uw
+        JOIN words_{language} w ON uw.word_id = w.id
+        WHERE uw.user_id = $1 AND uw.language = $2
+    """  # noqa: S608
+    if word_type is not None:
+        query += " AND w.word_type = $3"
+    return query
+
+
 def increment_score_query(table: str) -> str:
     # Table name from Language enum values, not user input  # noqa: S608
     return f"""

{nl_processing-0.3.0 → nl_processing-0.5.0}/nl_processing/database/backend/abstract.py

@@ -53,6 +53,15 @@ class AbstractBackend(ABC):
         and random ordering.
         """
 
+    @abstractmethod
+    async def count_user_words(
+        self,
+        user_id: str,
+        language: str,
+        word_type: str | None = None,
+    ) -> int:
+        """Return total user-word associations for the given user and language."""
+
     @abstractmethod
     async def add_user_word(
         self,
@@ -103,6 +112,22 @@ class AbstractBackend(ABC):
     ) -> None:
         """Insert event_id into the applied_events table."""
 
+    @abstractmethod
+    async def apply_score_delta_atomic(
+        self,
+        score_table: str,
+        events_table: str,
+        user_id: str,
+        event_id: str,
+        source_word_id: int,
+        delta: int,
+    ) -> bool:
+        """Atomically check-apply-mark a score delta in one transaction.
+
+        Returns True if the delta was applied, False if event_id was already applied.
+        The entire operation (check + increment + mark) runs in a single transaction.
+        """
+
     @abstractmethod
     async def create_tables(
         self,

{nl_processing-0.3.0 → nl_processing-0.5.0}/nl_processing/database/backend/neon.py

@@ -1,8 +1,9 @@
-"""NeonBackend — asyncpg implementation of AbstractBackend for Neon PostgreSQL."""
+"""NeonBackend asyncpg implementation for Neon PostgreSQL."""
 
 import asyncpg
 
 from nl_processing.database.backend._neon_exercise import (
+    atomic_apply_delta,
     check_event,
     create_exercise_tables,
     get_scores,
@@ -14,6 +15,7 @@ from nl_processing.database.backend._queries import (
     CREATE_USER_WORDS,
     add_translation_link_query,
     add_word_query,
+    count_user_words_query,
     create_translations_table,
     create_words_table,
     get_user_words_query,
@@ -34,7 +36,6 @@ class NeonBackend(AbstractBackend):
         self._connection: asyncpg.Connection | None = None  # type: ignore[type-arg]
 
     async def _connect(self) -> asyncpg.Connection:  # type: ignore[type-arg]
-        """Return cached connection, creating it lazily on first call."""
         if self._connection is None:
             try:
                 self._connection = await asyncpg.connect(dsn=self._database_url)
@@ -43,13 +44,12 @@ class NeonBackend(AbstractBackend):
                 raise DatabaseError(str(exc)) from exc
             except OSError as exc:
                 raise DatabaseError(str(exc)) from exc
+        if self._connection is None:
+            raise DatabaseError("Database connection was not initialized")
         return self._connection
 
     async def create_tables(
-        self,
-        languages: list[str],
-        pairs: list[tuple[str, str]],
-        exercise_slugs: list[str],
+        self, languages: list[str], pairs: list[tuple[str, str]], exercise_slugs: list[str]
     ) -> None:
         conn = await self._connect()
         try:
@@ -78,11 +78,7 @@ class NeonBackend(AbstractBackend):
             return None
         return int(row["id"])
 
-    async def get_word(
-        self,
-        table: str,
-        normalized_form: str,
-    ) -> dict[str, str | int] | None:
+    async def get_word(self, table: str, normalized_form: str) -> dict[str, str | int] | None:
         conn = await self._connect()
         try:
             row = await conn.fetchrow(get_word_query(table), normalized_form)
@@ -96,24 +92,14 @@ class NeonBackend(AbstractBackend):
             "word_type": row["word_type"],
         }
 
-    async def add_translation_link(
-        self,
-        table: str,
-        source_id: int,
-        target_id: int,
-    ) -> None:
+    async def add_translation_link(self, table: str, source_id: int, target_id: int) -> None:
         conn = await self._connect()
         try:
             await conn.execute(add_translation_link_query(table), source_id, target_id)
         except asyncpg.PostgresError as exc:
             raise DatabaseError(str(exc)) from exc
 
-    async def add_user_word(
-        self,
-        user_id: str,
-        word_id: int,
-        language: str,
-    ) -> None:
+    async def add_user_word(self, user_id: str, word_id: int, language: str) -> None:
         conn = await self._connect()
         try:
             await conn.execute(ADD_USER_WORD, user_id, word_id, language)
@@ -144,6 +130,19 @@ class NeonBackend(AbstractBackend):
             raise DatabaseError(str(exc)) from exc
         return [dict(row) for row in rows]
 
+    async def count_user_words(self, user_id: str, language: str, word_type: str | None = None) -> int:
+        conn = await self._connect()
+        args: list[str] = [user_id, language]
+        if word_type is not None:
+            args.append(word_type)
+        try:
+            count = await conn.fetchval(count_user_words_query(language, word_type), *args)
+        except asyncpg.PostgresError as exc:
+            raise DatabaseError(str(exc)) from exc
+        if count is None:
+            return 0
+        return int(count)
+
     async def increment_user_exercise_score(
         self,
         table: str,
@@ -155,36 +154,29 @@ class NeonBackend(AbstractBackend):
         return await increment_score(conn, table, user_id, source_word_id, delta)
 
     async def get_user_exercise_scores(
-        self,
-        table: str,
-        user_id: str,
-        source_word_ids: list[int],
+        self, table: str, user_id: str, source_word_ids: list[int]
     ) -> list[dict[str, str | int]]:
         conn = await self._connect()
         return await get_scores(conn, table, user_id, source_word_ids)
 
-    async def check_event_applied(
-        self,
-        table: str,
-        event_id: str,
-    ) -> bool:
+    async def check_event_applied(self, table: str, event_id: str) -> bool:
         conn = await self._connect()
         return await check_event(conn, table, event_id)
 
-    async def mark_event_applied(
-        self,
-        table: str,
-        event_id: str,
-    ) -> None:
+    async def mark_event_applied(self, table: str, event_id: str) -> None:
         conn = await self._connect()
         await mark_event(conn, table, event_id)
 
+    async def apply_score_delta_atomic(
+        self, score_table: str, events_table: str,
+        user_id: str, event_id: str, source_word_id: int, delta: int,
+    ) -> bool:  # fmt: skip
+        conn = await self._connect()
+        return await atomic_apply_delta(conn, score_table, events_table, user_id, event_id, source_word_id, delta)
 
-def _infer_target_language(source_language: str) -> str:
-    """Infer the target language for translation lookups.
 
-    With only two languages (nl, ru), the target is always the other one.
-    """
+def _infer_target_language(source_language: str) -> str:
+    """Infer the other language in the nl/ru pair."""
     if source_language == "nl":
         return "ru"
     return "nl"

{nl_processing-0.3.0 → nl_processing-0.5.0}/nl_processing/database/exercise_progress.py

@@ -119,25 +119,21 @@ class ExerciseProgressStore:
     ) -> None:
         """Apply a score delta idempotently, guarded by event deduplication.
 
-        Validates exercise_type. Skips if event_id was already applied.
+        Validates exercise_type and delta. Skips if event_id was already applied.
+        The check-increment-mark operation is atomic (single transaction).
         """
         self._validate_exercise_type(exercise_type)
-        already_applied = await self._backend.check_event_applied(
-            self._applied_events_table,
-            event_id,
-        )
-        if already_applied:
-            return
+        if delta not in (1, -1):
+            msg = f"delta must be +1 or -1, got {delta}"
+            raise ValueError(msg)
         table = self._score_tables[exercise_type]
-        await self._backend.increment_user_exercise_score(
-            table,
-            self._user_id,
-            source_word_id,
-            delta,
-        )
-        await self._backend.mark_event_applied(
-            self._applied_events_table,
-            event_id,
+        await self._backend.apply_score_delta_atomic(
+            score_table=table,
+            events_table=self._applied_events_table,
+            user_id=self._user_id,
+            event_id=event_id,
+            source_word_id=source_word_id,
+            delta=delta,
         )
 
     def _validate_exercise_type(self, exercise_type: str) -> None:

{nl_processing-0.3.0 → nl_processing-0.5.0}/nl_processing/database/service.py

@@ -140,6 +140,19 @@ class DatabaseService:
                 language=self._target_language,
             )
             pairs.append(WordPair(source=source, target=target))
+        if limit is None and not random:
+            total_count = await self._backend.count_user_words(
+                self._user_id,
+                self._source_language.value,
+                word_type=word_type.value if word_type else None,
+            )
+            if total_count > len(pairs):
+                excluded_count = total_count - len(pairs)
+                _logger.warning(
+                    "%d of %d words excluded from get_words() due to missing translations",
+                    excluded_count,
+                    total_count,
+                )
         return pairs
 
     @classmethod

nl_processing-0.5.0/nl_processing/database_cache/_local_store_queries.py

@@ -0,0 +1,58 @@
+"""DDL and query constants for the local SQLite cache store."""
+
+DDL_CACHED_WORD_PAIRS = """
+CREATE TABLE IF NOT EXISTS cached_word_pairs (
+    source_word_id INTEGER PRIMARY KEY,
+    source_normalized_form TEXT NOT NULL,
+    source_word_type TEXT NOT NULL,
+    target_word_id INTEGER NOT NULL,
+    target_normalized_form TEXT NOT NULL,
+    target_word_type TEXT NOT NULL
+)"""
+
+DDL_CACHED_SCORES = """
+CREATE TABLE IF NOT EXISTS cached_scores (
+    source_word_id INTEGER NOT NULL,
+    exercise_type TEXT NOT NULL,
+    score INTEGER NOT NULL DEFAULT 0,
+    updated_at TEXT NOT NULL,
+    PRIMARY KEY (source_word_id, exercise_type)
+)"""
+
+DDL_PENDING_SCORE_EVENTS = """
+CREATE TABLE IF NOT EXISTS pending_score_events (
+    event_id TEXT PRIMARY KEY,
+    source_word_id INTEGER NOT NULL,
+    exercise_type TEXT NOT NULL,
+    delta INTEGER NOT NULL,
+    created_at TEXT NOT NULL,
+    flushed_at TEXT,
+    last_error TEXT
+)"""
+
+DDL_CACHE_METADATA = """
+CREATE TABLE IF NOT EXISTS cache_metadata (
+    id INTEGER PRIMARY KEY DEFAULT 1,
+    exercise_types TEXT NOT NULL,
+    schema_version INTEGER NOT NULL DEFAULT 1,
+    last_refresh_started_at TEXT,
+    last_refresh_completed_at TEXT,
+    last_flush_completed_at TEXT,
+    last_error TEXT
+)"""
+
+ALL_DDL = [DDL_CACHED_WORD_PAIRS, DDL_CACHED_SCORES, DDL_PENDING_SCORE_EVENTS, DDL_CACHE_METADATA]
+
+UPSERT_SCORE = (
+    "INSERT INTO cached_scores (source_word_id, exercise_type, score, updated_at) VALUES (?, ?, ?, ?)"
+    " ON CONFLICT(source_word_id, exercise_type) DO UPDATE SET score = score + ?, updated_at = ?"
+)
+
+INSERT_PENDING_EVENT = (
+    "INSERT INTO pending_score_events (event_id, source_word_id, exercise_type, delta, created_at)"
+    " VALUES (?, ?, ?, ?, ?)"
+)
+
+INSERT_WORD_PAIR = "INSERT INTO cached_word_pairs VALUES (?, ?, ?, ?, ?, ?)"
+
+INSERT_SCORE = "INSERT INTO cached_scores (source_word_id, exercise_type, score, updated_at) VALUES (?, ?, ?, ?)"

nl_processing-0.5.0/nl_processing/database_cache/exceptions.py

@@ -0,0 +1,10 @@
+class CacheNotReadyError(Exception):
+    """Raised when cached data is requested before the first usable snapshot exists."""
+
+
+class CacheStorageError(Exception):
+    """Raised when the local SQLite cache file cannot be opened, read, or updated."""
+
+
+class CacheSyncError(Exception):
+    """Raised when an explicit refresh or flush operation fails synchronously."""

nl_processing-0.5.0/nl_processing/database_cache/local_store.py

@@ -0,0 +1,200 @@
+"""SQLite data-access layer for the local word-pair / score cache."""
+
+from datetime import UTC, datetime
+import json
+import sqlite3
+
+import aiosqlite
+
+from nl_processing.database_cache._local_store_queries import (
+    ALL_DDL,
+    INSERT_PENDING_EVENT,
+    INSERT_SCORE,
+    INSERT_WORD_PAIR,
+    UPSERT_SCORE,
+)
+from nl_processing.database_cache.exceptions import CacheStorageError
+
+
+def _now() -> str:
+    return datetime.now(tz=UTC).isoformat()
+
+
+class LocalStore:
+    """Async SQLite store for cached word pairs, scores, and pending events."""
+
+    def __init__(self, db_path: str) -> None:
+        self._db_path = db_path
+        self._db: aiosqlite.Connection | None = None
+
+    @property
+    def _conn(self) -> aiosqlite.Connection:
+        if self._db is None:
+            raise CacheStorageError("LocalStore is not open")
+        return self._db
+
+    async def open(self) -> None:
+        """Open the SQLite connection and create tables."""
+        try:
+            self._db = await aiosqlite.connect(self._db_path)
+            self._db.row_factory = aiosqlite.Row
+            await self._db.execute("PRAGMA journal_mode=WAL")
+            for ddl in ALL_DDL:
+                await self._db.execute(ddl)
+            await self._db.commit()
+        except sqlite3.Error as exc:
+            raise CacheStorageError(str(exc)) from exc
+
+    async def close(self) -> None:
+        """Close the SQLite connection."""
+        if self._db:
+            await self._db.close()
+            self._db = None
+
+    async def get_cached_word_pairs(
+        self,
+        word_type: str | None = None,
+        limit: int | None = None,
+        *,
+        random: bool = False,
+    ) -> list[dict[str, str | int]]:
+        """Query cached word pairs with optional filter, limit, and random ordering."""
+        sql = "SELECT * FROM cached_word_pairs"
+        params: list[str | int] = []
+        if word_type is not None:
+            sql += " WHERE source_word_type = ?"
+            params.append(word_type)
+        if random:
+            sql += " ORDER BY RANDOM()"
+        if limit is not None:
+            sql += " LIMIT ?"
+            params.append(limit)
+        return await self._fetch_all(sql, params)
+
+    async def get_cached_word_pairs_with_scores(self, exercise_types: list[str]) -> list[dict[str, str | int]]:
+        """Query word pairs and attach scores per exercise type (missing = 0)."""
+        try:
+            rows = await self._fetch_all("SELECT * FROM cached_word_pairs")
+            for row in rows:
+                for et in exercise_types:
+                    sc = await self._conn.execute(
+                        "SELECT score FROM cached_scores WHERE source_word_id=? AND exercise_type=?",
+                        (row["source_word_id"], et),
+                    )
+                    score_row = await sc.fetchone()
+                    row[f"score_{et}"] = int(score_row["score"]) if score_row else 0
+            return rows
+        except sqlite3.Error as exc:
+            raise CacheStorageError(str(exc)) from exc
+
+    async def get_pending_events(self) -> list[dict[str, str | int]]:
+        return await self._fetch_all("SELECT * FROM pending_score_events WHERE flushed_at IS NULL ORDER BY created_at")
+
+    async def get_pending_event_count(self) -> int:
+        try:
+            cur = await self._conn.execute("SELECT COUNT(*) FROM pending_score_events WHERE flushed_at IS NULL")
+            row = await cur.fetchone()
+            return int(row[0]) if row else 0
+        except sqlite3.Error as exc:
+            raise CacheStorageError(str(exc)) from exc
+
+    async def get_metadata(self) -> dict[str, str | int] | None:
+        try:
+            cur = await self._conn.execute("SELECT * FROM cache_metadata WHERE id = 1")
+            row = await cur.fetchone()
+            return dict(row) if row else None
+        except sqlite3.Error as exc:
+            raise CacheStorageError(str(exc)) from exc
+
+    async def has_snapshot(self) -> bool:
+        try:
+            cur = await self._conn.execute("SELECT 1 FROM cached_word_pairs LIMIT 1")
+            return (await cur.fetchone()) is not None
+        except sqlite3.Error as exc:
+            raise CacheStorageError(str(exc)) from exc
+
+    async def record_score_and_event(
+        self,
+        source_word_id: int,
+        exercise_type: str,
+        delta: int,
+        event_id: str,
+    ) -> None:
+        """Atomically upsert a cached score and insert a pending event."""
+        now = _now()
+        try:
+            await self._conn.execute(UPSERT_SCORE, (source_word_id, exercise_type, delta, now, delta, now))
+            await self._conn.execute(INSERT_PENDING_EVENT, (event_id, source_word_id, exercise_type, delta, now))
+            await self._conn.commit()
+        except sqlite3.Error as exc:
+            raise CacheStorageError(str(exc)) from exc
+
+    async def rebuild_snapshot(
+        self,
+        word_pairs: list[tuple[int, str, str, int, str, str]],
+        scores: dict[tuple[int, str], int],
+    ) -> None:
+        """Atomically replace cached word pairs and scores, then reapply pending events."""
+        now = _now()
+        try:
+            await self._conn.execute("DELETE FROM cached_word_pairs")
+            await self._conn.execute("DELETE FROM cached_scores")
+            for wp in word_pairs:
+                await self._conn.execute(INSERT_WORD_PAIR, wp)
+            for (wid, et), score in scores.items():
+                await self._conn.execute(INSERT_SCORE, (wid, et, score, now))
+            for evt in await self.get_pending_events():
+                await self._conn.execute(
+                    UPSERT_SCORE,
+                    (evt["source_word_id"], evt["exercise_type"], evt["delta"], now, evt["delta"], now),
+                )
+            await self._conn.commit()
+        except sqlite3.Error as exc:
+            raise CacheStorageError(str(exc)) from exc
+
+    async def mark_event_flushed(self, event_id: str) -> None:
+        await self._exec_commit("UPDATE pending_score_events SET flushed_at=? WHERE event_id=?", (_now(), event_id))
+
+    async def mark_event_failed(self, event_id: str, error: str) -> None:
+        await self._exec_commit("UPDATE pending_score_events SET last_error=? WHERE event_id=?", (error, event_id))
+
+    async def update_metadata(self, **fields: str | int | None) -> None:
+        if not fields:
+            return
+        set_clause = ", ".join(f"{k} = ?" for k in fields)
+        await self._exec_commit(
+            f"UPDATE cache_metadata SET {set_clause} WHERE id = 1",  # noqa: S608
+            tuple(fields.values()),
+        )
+
+    async def ensure_metadata(self, exercise_types: list[str]) -> None:
+        await self._exec_commit(
+            "INSERT OR REPLACE INTO cache_metadata (id, exercise_types, schema_version) VALUES (1, ?, 1)",
+            (json.dumps(exercise_types),),
+        )
+
+    async def get_source_word_id(self, normalized_form: str, word_type: str) -> int | None:
+        """Look up a source_word_id from cached_word_pairs."""
+        try:
+            cur = await self._conn.execute(
+                "SELECT source_word_id FROM cached_word_pairs WHERE source_normalized_form=? AND source_word_type=?",
+                (normalized_form, word_type),
+            )
+            row = await cur.fetchone()
+            return int(row["source_word_id"]) if row else None
+        except sqlite3.Error as exc:
+            raise CacheStorageError(str(exc)) from exc
+
+    async def _fetch_all(self, sql: str, params: list[str | int] | None = None) -> list[dict[str, str | int]]:
+        try:
+            cur = await self._conn.execute(sql, params or [])
+            return [dict(row) for row in await cur.fetchall()]
+        except sqlite3.Error as exc:
+            raise CacheStorageError(str(exc)) from exc
+
+    async def _exec_commit(self, sql: str, params: tuple[str | int | None, ...]) -> None:
+        try:
+            await self._conn.execute(sql, params)
+            await self._conn.commit()
+        except sqlite3.Error as exc:
+            raise CacheStorageError(str(exc)) from exc

nl_processing-0.5.0/nl_processing/database_cache/logging.py

@@ -0,0 +1,5 @@
+import logging
+
+
+def get_logger(name: str) -> logging.Logger:
+    return logging.getLogger(f"nl_processing.database_cache.{name}")

nl_processing-0.5.0/nl_processing/database_cache/models.py

@@ -0,0 +1,12 @@
+from datetime import datetime
+
+from pydantic import BaseModel
+
+
+class CacheStatus(BaseModel):
+    is_ready: bool
+    is_stale: bool
+    has_snapshot: bool
+    pending_events: int
+    last_refresh_completed_at: datetime | None
+    last_flush_completed_at: datetime | None