datamask-store 1.0.0__py3-none-any.whl

datamask_store/__init__.py

@@ -0,0 +1,22 @@
+"""
+datamask-store — 存储层 (W11.3)
+
+W6.11 已实现: 多用户隔离
+- 每用户独立 SQLite (./data/users/<user_id>/)
+- 命名空间化 task_id (user:task)
+- 路径穿越防御
+
+W9.P2.2 已实现: SQLite WAL 模式
+- 4 workers 并发 database is locked 修复
+- WAL + busy_timeout 5000 + 5 retry
+"""
+from datamask_store.user_store import UserStore
+from datamask_store.audit_store import AuditStore
+from datamask_store.labeling_store import LabelingStore
+
+__version__ = "0.1.0"
+__all__ = [
+    "UserStore",
+    "AuditStore",
+    "LabelingStore",
+]

datamask_store/audit_store.py

@@ -0,0 +1,49 @@
+"""
+datamask-store — 审计日志存储 (W7.B.2)
+"""
+import sqlite3
+import json
+import time
+from typing import Dict, Any, Optional, List
+
+
+class AuditStore:
+    """审计追溯存储 (W7.B.2 已下沉到 backend/main.py middleware)"""
+
+    def __init__(self, conn: sqlite3.Connection):
+        self.conn = conn
+        self._init_table()
+
+    def _init_table(self):
+        self.conn.execute("""
+            CREATE TABLE IF NOT EXISTS audit_traces (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                task_id TEXT NOT NULL,
+                user_id TEXT,
+                action TEXT NOT NULL,
+                entity_type TEXT,
+                entity_text TEXT,
+                meta_json TEXT,
+                created_at REAL NOT NULL
+            )
+        """)
+        self.conn.execute("CREATE INDEX IF NOT EXISTS idx_audit_task ON audit_traces(task_id)")
+        self.conn.commit()
+
+    def log(self, task_id: str, action: str, user_id: Optional[str] = None,
+            entity_type: Optional[str] = None, entity_text: Optional[str] = None,
+            meta: Optional[Dict[str, Any]] = None):
+        self.conn.execute(
+            "INSERT INTO audit_traces (task_id, user_id, action, entity_type, entity_text, meta_json, created_at) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?)",
+            (task_id, user_id, action, entity_type, entity_text,
+             json.dumps(meta) if meta else None, time.time())
+        )
+        self.conn.commit()
+
+    def query_by_task(self, task_id: str, limit: int = 100) -> List[Dict]:
+        cursor = self.conn.execute(
+            "SELECT * FROM audit_traces WHERE task_id = ? ORDER BY id DESC LIMIT ?",
+            (task_id, limit)
+        )
+        return [dict(zip([d[0] for d in cursor.description], row)) for row in cursor.fetchall()]

datamask_store/labeling_store.py

@@ -0,0 +1,72 @@
+"""
+datamask-store — 标注数据存储 (W7.D.1)
+"""
+import sqlite3
+import json
+import time
+from typing import Dict, Any, Optional, List
+
+
+class LabelingStore:
+    """标注数据存储 (含 W9.P2.2 独立 samples_extended 表)"""
+
+    def __init__(self, conn: sqlite3.Connection):
+        self.conn = conn
+        self._init_tables()
+
+    def _init_tables(self):
+        # 基础标注
+        self.conn.execute("""
+            CREATE TABLE IF NOT EXISTS samples (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                text TEXT NOT NULL,
+                entities_json TEXT NOT NULL,
+                confidence REAL NOT NULL,
+                source TEXT,
+                created_at REAL NOT NULL,
+                UNIQUE(text, source)
+            )
+        """)
+        # 扩展实体标注 (W9.P2.2 解决 INTEGER PRIMARY KEY 联合唯一约束)
+        self.conn.execute("""
+            CREATE TABLE IF NOT EXISTS samples_extended (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                text TEXT NOT NULL,
+                entities_json TEXT NOT NULL,
+                confidence REAL NOT NULL,
+                source TEXT,
+                created_at REAL NOT NULL
+            )
+        """)
+        self.conn.commit()
+
+    def upsert_sample(self, text: str, entities: list, confidence: float,
+                      source: str = "deepseek", extended: bool = False) -> int:
+        """W7.D.3: P0 主键冲突修复 (upsert 防御)"""
+        table = "samples_extended" if extended else "samples"
+        entities_json = json.dumps(entities, ensure_ascii=False)
+        try:
+            cur = self.conn.execute(
+                f"INSERT INTO {table} (text, entities_json, confidence, source, created_at) VALUES (?, ?, ?, ?, ?)",
+                (text, entities_json, confidence, source, time.time())
+            )
+            self.conn.commit()
+            return cur.lastrowid
+        except sqlite3.IntegrityError:
+            # upsert
+            self.conn.execute(
+                f"UPDATE {table} SET entities_json=?, confidence=?, created_at=? WHERE text=? AND source=?",
+                (entities_json, confidence, time.time(), text, source)
+            )
+            self.conn.commit()
+            return -1
+
+    def get_pending(self, limit: int = 50, min_confidence: float = 0.0) -> List[Dict]:
+        cursor = self.conn.execute(
+            "SELECT id, text, entities_json, confidence FROM samples WHERE confidence >= ? ORDER BY confidence ASC LIMIT ?",
+            (min_confidence, limit)
+        )
+        return [
+            {"id": row[0], "text": row[1], "entities": json.loads(row[2]), "confidence": row[3]}
+            for row in cursor.fetchall()
+        ]

datamask_store/user_store.py

@@ -0,0 +1,35 @@
+"""
+datamask-store — 多用户隔离存储 (W6.11)
+"""
+import sqlite3
+import os
+import re
+from pathlib import Path
+from typing import Optional, List, Dict, Any
+
+
+USER_ID_PATTERN = re.compile(r'^[a-zA-Z0-9_-]{1,32}$')
+
+
+class UserStore:
+    """每用户独立 SQLite (W6.11)"""
+
+    def __init__(self, base_dir: str = "./data/users"):
+        self.base_dir = Path(base_dir)
+        self.base_dir.mkdir(parents=True, exist_ok=True)
+
+    def get_user_db_path(self, user_id: str) -> Path:
+        """获取用户数据库路径 (路径穿越防御)"""
+        if not USER_ID_PATTERN.match(user_id):
+            raise ValueError(f"Invalid user_id: {user_id}")
+        user_dir = self.base_dir / user_id
+        user_dir.mkdir(parents=True, exist_ok=True)
+        return user_dir / "data.db"
+
+    def connect(self, user_id: str) -> sqlite3.Connection:
+        """连接用户数据库 (WAL 模式)"""
+        db_path = self.get_user_db_path(user_id)
+        conn = sqlite3.connect(str(db_path), timeout=5.0)
+        conn.execute("PRAGMA journal_mode=WAL")
+        conn.execute("PRAGMA busy_timeout=5000")
+        return conn

datamask_store-1.0.0.dist-info/METADATA

@@ -0,0 +1,16 @@
+Metadata-Version: 2.4
+Name: datamask-store
+Version: 1.0.0
+Summary: DataMask 存储层 — SQLite WAL 模式、多用户隔离、审计追溯、标注持久化
+Author-email: TianluAudit <contact@datamask.cn>
+License: Proprietary
+Project-URL: Homepage, https://datamask.cn
+Project-URL: Documentation, https://datamask.cn/docs
+Keywords: data-masking,privacy,NER,entity-recognition,FPE
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.9

datamask_store-1.0.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+datamask_store/__init__.py,sha256=bCs24eL3ckzG_j9dhyNGRFR9f69n1a9KYV0hRApcJlw,555
+datamask_store/audit_store.py,sha256=c3m4whx1wEyknoat1VFgxKf3Qu0xREikDgwoF28shxA,1799
+datamask_store/labeling_store.py,sha256=L6pfLQmBvcv_cSkogY79frxg-IXZXoHEn7mXdLfGhMs,2710
+datamask_store/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamask_store/user_store.py,sha256=64Z3Ad0HCr1CN5TpYAUZPNtX-ZRnikk8qPfL7qq5z4U,1121
+datamask_store-1.0.0.dist-info/METADATA,sha256=MUwpI7veVXypdnIJYUUnhYYrPlL3i4tRhug8C-UCuak,695
+datamask_store-1.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+datamask_store-1.0.0.dist-info/top_level.txt,sha256=hZChoVV5m36ZfQW16r2Nh08H5Lq5mDgEYxi3muTlAlM,15
+datamask_store-1.0.0.dist-info/RECORD,,

datamask_store-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

datamask_store-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+datamask_store