edip 1.1.0__py3-none-any.whl

edip/__init__.py

@@ -0,0 +1,13 @@
+
+from .inspector import SchemaInspector
+from .engine import EDIPEngine
+from .watcher import AmbientWatcher
+
+
+__all__ = [
+    "SchemaInspector",
+    "EDIPEngine",
+    "AmbientWatcher"
+]
+
+__version__ = "1.0.0"

edip/engine.py

@@ -0,0 +1,548 @@
+
+import os
+import json
+import shutil
+import sqlite3
+import datetime
+import requests
+
+from typing import Optional, Dict, Any, Tuple, List, Union
+
+from .inspector import SchemaInspector
+
+
+class EDIPEngine:
+    """
+    Universal Edge Deterministic Ingestion Protocol (EDIP) Engine.
+
+    Handles arbitrary SQLite schemas, multi-table extraction,
+    foreign key resolution, atomic transaction rollback loops,
+    automated DLQ quarantine logging, and physical file routing
+    (Processed Archive vs. Human Review Quarantine).
+    """
+
+    def __init__(
+        self,
+        db_path: str,
+        target_tables: Optional[Union[str, List[str]]] = None,
+        model: str = "llama3.2:1b",
+        api_url: str = "http://localhost:11434/api/generate",
+        max_retries: int = 3,
+        timeout: int = 15,
+        enable_quarantine: bool = True,
+        quarantine_table: str = "quarantine_invoices",
+        quarantine_dir: Optional[str] = "./Quarantine_Documents",
+        archive_dir: Optional[str] = "./Processed_Archive"
+    ):
+        self.db_path = db_path
+        self.model = model
+        self.api_url = api_url
+        self.max_retries = max_retries
+        self.timeout = timeout
+        self.enable_quarantine = enable_quarantine
+        self.quarantine_table = quarantine_table
+        self.quarantine_dir = quarantine_dir
+        self.archive_dir = archive_dir
+
+        self.inspector = SchemaInspector(db_path)
+
+        # Standardize target_tables or inspect all non-system tables
+        if isinstance(target_tables, str):
+            self.target_tables = [target_tables]
+
+        elif isinstance(target_tables, list):
+            self.target_tables = target_tables
+
+        else:
+            full_meta = self.inspector.inspect_full_database()
+            self.target_tables = list(full_meta.keys())
+
+        # Compile database schema rules dynamically
+        self.schema_prompt = (
+            self.inspector.build_system_prompt_schema(
+                self.target_tables
+            )
+        )
+
+        # Ensure directories and DLQ table exist
+        if self.enable_quarantine:
+            self._ensure_quarantine_table()
+
+        if self.quarantine_dir:
+            os.makedirs(
+                self.quarantine_dir,
+                exist_ok=True
+            )
+
+        if self.archive_dir:
+            os.makedirs(
+                self.archive_dir,
+                exist_ok=True
+            )
+
+    def get_connection(self) -> sqlite3.Connection:
+        """Creates a configured SQLite database connection."""
+
+        conn = sqlite3.connect(
+            self.db_path,
+            timeout=self.timeout
+        )
+
+        conn.execute("PRAGMA journal_mode=WAL;")
+        conn.execute("PRAGMA foreign_keys = ON;")
+
+        return conn
+
+    def _ensure_quarantine_table(self) -> None:
+        """
+        Ensures the Dead-Letter Queue (DLQ)
+        quarantine table exists.
+        """
+
+        conn = self.get_connection()
+
+        conn.execute(f"""
+            CREATE TABLE IF NOT EXISTS {self.quarantine_table} (
+                quarantine_id INTEGER PRIMARY KEY AUTOINCREMENT,
+                timestamp TEXT NOT NULL,
+                file_reference TEXT,
+                target_table TEXT,
+                raw_document TEXT,
+                last_offending_payload TEXT,
+                rejection_reason TEXT,
+                retry_attempts INTEGER,
+                audit_status TEXT DEFAULT 'PENDING_HUMAN_AUDIT'
+            );
+        """)
+
+        conn.commit()
+        conn.close()
+
+    def _move_file_safely(
+        self,
+        source_path: str,
+        target_directory: str
+    ) -> Optional[str]:
+        """
+        Safely moves a physical file to a destination directory,
+        avoiding filename collisions.
+        """
+
+        if (
+            not source_path
+            or not os.path.exists(source_path)
+            or not target_directory
+        ):
+            return None
+
+        os.makedirs(
+            target_directory,
+            exist_ok=True
+        )
+
+        filename = os.path.basename(source_path)
+
+        dest_path = os.path.join(
+            target_directory,
+            filename
+        )
+
+        # Resolve collisions by appending a sequential counter
+        base, ext = os.path.splitext(filename)
+
+        counter = 1
+
+        while os.path.exists(dest_path):
+            dest_path = os.path.join(
+                target_directory,
+                f"{base}_{counter}{ext}"
+            )
+
+            counter += 1
+
+        try:
+            shutil.move(
+                source_path,
+                dest_path
+            )
+
+            return dest_path
+
+        except Exception as e:
+            print(
+                f"[EDIP WARNING] Failed to move file "
+                f"{source_path} to {target_directory}: {e}"
+            )
+
+            return None
+
+    def _log_to_quarantine(
+        self,
+        raw_document: str,
+        rejection_reason: str,
+        attempts: int,
+        last_payload: Optional[Dict[str, Any]] = None,
+        file_reference: Optional[str] = None
+    ) -> int:
+        """
+        Logs an unresolvable or rejected record
+        into the Dead-Letter Queue.
+        """
+
+        conn = self.get_connection()
+        cursor = conn.cursor()
+
+        now_ts = datetime.datetime.now(
+            datetime.timezone.utc
+        ).isoformat()
+
+        cursor.execute(
+            f"""
+            INSERT INTO {self.quarantine_table} (
+                timestamp,
+                file_reference,
+                target_table,
+                raw_document,
+                last_offending_payload,
+                rejection_reason,
+                retry_attempts,
+                audit_status
+            )
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+            (
+                now_ts,
+                file_reference or "inline_document",
+                ", ".join(self.target_tables),
+                raw_document,
+                (
+                    json.dumps(last_payload)
+                    if last_payload else "None"
+                ),
+                rejection_reason,
+                attempts,
+                "PENDING_HUMAN_AUDIT"
+            )
+        )
+
+        conn.commit()
+
+        qid = cursor.lastrowid
+
+        conn.close()
+
+        return qid
+
+    def _call_llm(self, prompt: str) -> str:
+        """Sends the extraction prompt to the local Ollama model."""
+
+        response = requests.post(
+            self.api_url,
+            json={
+                "model": self.model,
+                "prompt": prompt,
+                "stream": False,
+                "format": "json"
+            },
+            timeout=45
+        )
+
+        response.raise_for_status()
+
+        return response.json().get("response", "{}")
+
+    def _insert_payload_transaction(
+        self,
+        conn: sqlite3.Connection,
+        payload: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        """
+        Dynamically executes parameterized SQL inserts
+        across arbitrary tables.
+        """
+
+        cursor = conn.cursor()
+        inserted_summary = {}
+
+        # Normalize single-table payloads
+        normalized_payload = payload
+
+        if (
+            len(self.target_tables) == 1
+            and self.target_tables[0] not in payload
+        ):
+            normalized_payload = {
+                self.target_tables[0]: (
+                    [payload] if isinstance(payload, dict)
+                    else payload
+                )
+            }
+
+        for table_name, records in normalized_payload.items():
+
+            if table_name not in self.target_tables:
+                continue
+
+            if isinstance(records, dict):
+                records = [records]
+
+            inserted_rows = 0
+
+            for record in records:
+
+                if not isinstance(record, dict) or not record:
+                    continue
+
+                columns = list(record.keys())
+
+                placeholders = ", ".join(
+                    ["?"] * len(columns)
+                )
+
+                col_names = ", ".join(columns)
+
+                values = [
+                    record[col]
+                    for col in columns
+                ]
+
+                sql = (
+                    f"INSERT INTO {table_name} "
+                    f"({col_names}) "
+                    f"VALUES ({placeholders})"
+                )
+
+                cursor.execute(sql, values)
+
+                inserted_rows += 1
+
+            inserted_summary[table_name] = inserted_rows
+
+        return inserted_summary
+
+    def ingest_text(
+        self,
+        document_text: str,
+        file_path: Optional[str] = None
+    ) -> Tuple[bool, Dict[str, Any]]:
+        """
+        Executes the closed-loop ingestion protocol.
+
+        On success, moves the file to archive_dir (if provided).
+        On rejection/exhaustion, logs to DLQ and moves the file
+        to quarantine_dir.
+        """
+
+        prompt = (
+            "You are an automated deterministic data extraction engine.\n"
+            "Extract structured data from the document text into "
+            "valid JSON matching this schema:\n\n"
+            f"{self.schema_prompt}\n\n"
+            "Rules:\n"
+            "1. Output a JSON object mapping table names "
+            "to objects or lists of rows.\n"
+            "2. Ensure all column names and foreign keys "
+            "match the schema strictly.\n"
+            "3. Output ONLY the JSON object.\n\n"
+            f"Document Text:\n{document_text}"
+        )
+
+        last_payload = None
+        last_error = "Unknown error"
+
+        file_ref = (
+            os.path.basename(file_path)
+            if file_path else "inline_text"
+        )
+
+        for attempt in range(1, self.max_retries + 1):
+
+            try:
+                llm_output = self._call_llm(prompt)
+                last_payload = json.loads(llm_output)
+
+            except Exception as e:
+                last_error = f"JSON/API parsing error: {e}"
+
+                prompt = (
+                    "Your response could not be parsed as valid JSON. "
+                    f"Error: {e}.\n"
+                    "Output ONLY valid JSON matching the schema:\n"
+                    f"{self.schema_prompt}"
+                )
+
+                continue
+
+            conn = self.get_connection()
+
+            try:
+                # Execute inside an atomic transaction
+                with conn:
+                    summary = self._insert_payload_transaction(
+                        conn,
+                        last_payload
+                    )
+
+                conn.close()
+
+                # Physical routing:
+                # Move file to archive directory on successful ingestion
+                archived_path = None
+
+                if file_path and self.archive_dir:
+                    archived_path = self._move_file_safely(
+                        file_path,
+                        self.archive_dir
+                    )
+
+                return True, {
+                    "status": "inserted",
+                    "summary": summary,
+                    "attempt": attempt,
+                    "payload": last_payload,
+                    "archived_path": archived_path
+                }
+
+            except sqlite3.IntegrityError as db_err:
+                conn.close()
+
+                last_error = f"SQLite IntegrityError: {db_err}"
+
+                # Idempotent deduplication check
+                if "UNIQUE constraint failed" in str(db_err):
+
+                    archived_path = None
+
+                    if file_path and self.archive_dir:
+                        archived_path = self._move_file_safely(
+                            file_path,
+                            self.archive_dir
+                        )
+
+                    return True, {
+                        "status": "idempotent_duplicate",
+                        "error": str(db_err),
+                        "attempt": attempt,
+                        "payload": last_payload,
+                        "archived_path": archived_path
+                    }
+
+                prompt = (
+                    "Your extracted data was rejected by SQLite "
+                    f"constraint validation: {db_err}.\n"
+                    f"Offending payload: {json.dumps(last_payload)}\n"
+                    f"Database Schema Constraints:\n"
+                    f"{self.schema_prompt}\n\n"
+                    "Correct the foreign keys, required values, "
+                    "or types and output ONLY valid corrected JSON.\n"
+                    f"Original Document:\n{document_text}"
+                )
+
+            except Exception as err:
+                conn.close()
+
+                last_error = f"Execution error: {err}"
+
+                prompt = (
+                    f"Execution failed with error: {err}.\n"
+                    f"Schema:\n{self.schema_prompt}\n\n"
+                    f"Original Document:\n{document_text}"
+                )
+
+        # Retry budget exhausted:
+        # Isolate to Dead-Letter Queue and physical quarantine directory
+        qid = None
+
+        if self.enable_quarantine:
+            qid = self._log_to_quarantine(
+                raw_document=document_text,
+                rejection_reason=last_error,
+                attempts=self.max_retries,
+                last_payload=last_payload,
+                file_reference=file_ref
+            )
+
+        quarantined_path = None
+
+        if file_path and self.quarantine_dir:
+            quarantined_path = self._move_file_safely(
+                file_path,
+                self.quarantine_dir
+            )
+
+        return False, {
+            "status": "quarantined",
+            "error": (
+                "Exceeded maximum retry limit without "
+                "satisfying constraints"
+            ),
+            "last_error": last_error,
+            "attempts": self.max_retries,
+            "quarantine_id": qid,
+            "quarantined_path": quarantined_path,
+            "audit_status": (
+                "PENDING_HUMAN_AUDIT" if qid else None
+            )
+        }
+
+    def ingest_document(
+        self,
+        file_path: str
+    ) -> Tuple[bool, Dict[str, Any]]:
+        """
+        Extracts text from a local document (PDF or text)
+        and runs the ingestion protocol.
+        """
+
+        if not os.path.exists(file_path):
+            return False, {
+                "status": "file_not_found",
+                "error": f"File does not exist: {file_path}"
+            }
+
+        text = ""
+
+        try:
+            if file_path.lower().endswith(".pdf"):
+
+                try:
+                    import pymupdf
+
+                    doc = pymupdf.open(file_path)
+
+                    text = "".join(
+                        page.get_text()
+                        for page in doc
+                    )
+
+                    doc.close()
+
+                except ImportError:
+                    from pypdf import PdfReader
+
+                    reader = PdfReader(file_path)
+
+                    text = "".join(
+                        page.extract_text() or ""
+                        for page in reader.pages
+                    )
+
+            else:
+                with open(
+                    file_path,
+                    "r",
+                    encoding="utf-8",
+                    errors="ignore"
+                ) as f:
+                    text = f.read()
+
+            return self.ingest_text(
+                document_text=text,
+                file_path=file_path
+            )
+
+        except Exception as e:
+            return False, {
+                "status": "read_error",
+                "error": str(e)
+            }

edip/inspector.py

@@ -0,0 +1,207 @@
+
+import sqlite3
+from typing import Dict, Any, List
+
+
+class SchemaInspector:
+    """
+    Dynamically maps any SQLite database:
+    tables, columns, constraints, and relational dependencies.
+    """
+
+    def __init__(self, db_path: str):
+        self.db_path = db_path
+
+    def get_connection(self) -> sqlite3.Connection:
+        """Creates a database connection with foreign keys enabled."""
+        conn = sqlite3.connect(self.db_path)
+        conn.execute("PRAGMA foreign_keys = ON;")
+        return conn
+
+    def inspect_full_database(self) -> Dict[str, Any]:
+        """
+        Discovers all non-system tables, columns, constraints,
+        and valid foreign references.
+        """
+        conn = self.get_connection()
+        cursor = conn.cursor()
+
+        # 1. Discover all user tables
+        cursor.execute("""
+            SELECT name
+            FROM sqlite_master
+            WHERE type = 'table'
+            AND name NOT LIKE 'sqlite_%';
+        """)
+
+        tables = [row[0] for row in cursor.fetchall()]
+        full_schema: Dict[str, Any] = {}
+
+        for table in tables:
+
+            # 2. Inspect table columns
+            cursor.execute(f"PRAGMA table_info('{table}');")
+            cols_raw = cursor.fetchall()
+
+            columns = [
+                {
+                    "name": col[1],
+                    "type": col[2] or "TEXT",
+                    "required": bool(col[3]),
+                    "is_pk": bool(col[5])
+                }
+                for col in cols_raw
+            ]
+
+            # 3. Inspect foreign key constraints
+            cursor.execute(f"PRAGMA foreign_key_list('{table}');")
+            fks_raw = cursor.fetchall()
+
+            foreign_keys = [
+                {
+                    "from_col": fk[3],
+                    "to_table": fk[2],
+                    "to_col": fk[4]
+                }
+                for fk in fks_raw
+            ]
+
+            # 4. Fetch active reference values from parent tables
+            valid_references: Dict[str, List[str]] = {}
+
+            for fk in foreign_keys:
+                parent_table = fk["to_table"]
+                parent_col = fk["to_col"]
+
+                try:
+                    cursor.execute(
+                        f"PRAGMA table_info('{parent_table}');"
+                    )
+
+                    p_cols = [
+                        c[1] for c in cursor.fetchall()
+                    ]
+
+                    name_col = next(
+                        (
+                            c for c in p_cols
+                            if any(
+                                k in c.lower()
+                                for k in [
+                                    "name",
+                                    "title",
+                                    "vendor",
+                                    "label"
+                                ]
+                            )
+                        ),
+                        None
+                    )
+
+                    if name_col:
+                        cursor.execute(
+                            f"""
+                            SELECT "{parent_col}", "{name_col}"
+                            FROM "{parent_table}"
+                            LIMIT 50;
+                            """
+                        )
+
+                        valid_references[fk["from_col"]] = [
+                            f"{r[0]} ({r[1]})"
+                            for r in cursor.fetchall()
+                        ]
+
+                    else:
+                        cursor.execute(
+                            f"""
+                            SELECT "{parent_col}"
+                            FROM "{parent_table}"
+                            LIMIT 50;
+                            """
+                        )
+
+                        valid_references[fk["from_col"]] = [
+                            str(r[0])
+                            for r in cursor.fetchall()
+                        ]
+
+                except sqlite3.OperationalError:
+                    pass
+
+            # 5. Store complete table metadata
+            full_schema[table] = {
+                "columns": columns,
+                "foreign_keys": foreign_keys,
+                "valid_references": valid_references
+            }
+
+        conn.close()
+
+        return full_schema
+
+    def build_system_prompt_schema(
+        self,
+        target_tables: List[str] = None
+    ) -> str:
+        """
+        Translates database metadata into concise,
+        multi-table JSON extraction rules for LLMs.
+        """
+
+        schema = self.inspect_full_database()
+
+        tables_to_include = target_tables or list(schema.keys())
+
+        schema_lines = []
+
+        for table in tables_to_include:
+
+            if table not in schema:
+                continue
+
+            meta = schema[table]
+
+            # Identify insertable columns
+            insertable_cols = [
+                f"{c['name']} ({c['type']})"
+                for c in meta["columns"]
+                if not c["is_pk"]
+            ]
+
+            # Build foreign key constraint rules
+            fk_rules = ""
+
+            if meta["foreign_keys"]:
+                fk_clauses = [
+                    (
+                        f"{fk['from_col']} -> "
+                        f"{fk['to_table']}({fk['to_col']})"
+                    )
+                    for fk in meta["foreign_keys"]
+                ]
+
+                fk_rules = (
+                    " | Constraints: "
+                    + "; ".join(fk_clauses)
+                )
+
+            # Build valid reference rules
+            ref_rules = ""
+
+            if meta["valid_references"]:
+                ref_clauses = [
+                    f"Valid {col}: {vals}"
+                    for col, vals in meta["valid_references"].items()
+                ]
+
+                ref_rules = " | " + "; ".join(ref_clauses)
+
+            # Build final table schema description
+            schema_lines.append(
+                f"Table '{table}': "
+                f"Columns: [{', '.join(insertable_cols)}]"
+                f"{fk_rules}{ref_rules}"
+            )
+
+        return "\n".join(schema_lines)

edip/watcher.py

@@ -0,0 +1,136 @@
+
+import os
+import time
+
+from typing import Optional, Callable
+
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+
+from .engine import EDIPEngine
+
+
+class DocumentWatchHandler(FileSystemEventHandler):
+    """
+    Handles incoming document events and sends supported
+    files to the EDIP ingestion engine.
+    """
+
+    def __init__(
+        self,
+        engine: EDIPEngine,
+        callback: Optional[Callable] = None
+    ):
+        super().__init__()
+
+        self.engine = engine
+        self.callback = callback
+
+    def on_created(self, event):
+        """
+        Processes newly created PDF and TXT files.
+        """
+
+        # Ignore directories and unsupported file types
+        if (
+            event.is_directory
+            or not event.src_path.lower().endswith((".pdf", ".txt"))
+        ):
+            return
+
+        # Wait for OS file write operations to complete
+        time.sleep(1.2)
+
+        file_path = event.src_path
+
+        # If the file has already been moved or deleted, skip it
+        if not os.path.exists(file_path):
+            return
+
+        try:
+            # Process document through EDIP
+            success, info = self.engine.ingest_document(file_path)
+
+            # Notify callback with processing results
+            if self.callback:
+                self.callback(
+                    file_path,
+                    success,
+                    info
+                )
+
+        except Exception as e:
+            if self.callback:
+                self.callback(
+                    file_path,
+                    False,
+                    {"error": str(e)}
+                )
+
+
+class AmbientWatcher:
+    """
+    Monitors a local inbound directory, runs EDIP,
+    and routes files to Archive or Quarantine.
+    """
+
+    def __init__(
+        self,
+        engine: EDIPEngine,
+        watch_dir: str = "./Inbound_Invoices",
+        callback: Optional[Callable] = None
+    ):
+        self.engine = engine
+        self.watch_dir = watch_dir
+        self.callback = callback
+
+        self.observer: Optional[Observer] = None
+
+    def start(self, blocking: bool = True):
+        """
+        Starts monitoring the inbound directory.
+        """
+
+        # Ensure the monitored directory exists
+        os.makedirs(
+            self.watch_dir,
+            exist_ok=True
+        )
+
+        # Initialize document event handler
+        handler = DocumentWatchHandler(
+            self.engine,
+            self.callback
+        )
+
+        # Initialize filesystem observer
+        self.observer = Observer()
+
+        self.observer.schedule(
+            handler,
+            path=self.watch_dir,
+            recursive=False
+        )
+
+        self.observer.start()
+
+        # Keep the watcher running
+        if blocking:
+            try:
+                while True:
+                    time.sleep(1)
+
+            except KeyboardInterrupt:
+                self.stop()
+
+    def stop(self):
+        """
+        Stops filesystem monitoring safely.
+        """
+
+        if (
+            self.observer
+            and self.observer.is_alive()
+        ):
+            self.observer.stop()
+            self.observer.join()

edip-1.1.0.dist-info/METADATA

@@ -0,0 +1,8 @@
+Metadata-Version: 2.4
+Name: edip
+Version: 1.1.0
+Summary: Edge Deterministic Ingestion Protocol using dynamic SQLite constraint reflection
+Author: Uditha Manohara Prasad
+Requires-Python: >=3.9
+Requires-Dist: requests>=2.28.0
+Requires-Dist: watchdog>=3.0.0

edip-1.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+edip/__init__.py,sha256=EDFrUs6WSL69J7ZuSoNPsnoJ81nVTaOac1pNi8iSo8g,221
+edip/engine.py,sha256=HDV11zRK1v-bM03-e2vlrvNLjvFhZwBKekuhYU1JL0A,16136
+edip/inspector.py,sha256=9FZZwWNiH4JJkVDrLCeZmzj4eaxMAPnBswl0vJvp1rg,6125
+edip/watcher.py,sha256=Gn86d-QSd737ZF1FmEkDJGUHwlDc2EADqJz0CUOle18,3320
+edip-1.1.0.dist-info/METADATA,sha256=GnvD3lHOJcsEmjYOnCGKbDA1IxtunLoolPcNHYuhUtg,263
+edip-1.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+edip-1.1.0.dist-info/top_level.txt,sha256=g21vaLf3JPYAh4da0CvJ7iflfpUzYU5NHlguAfjPIJ4,5
+edip-1.1.0.dist-info/RECORD,,

edip-1.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

edip-1.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+edip