epi-recorder 2.1.3__py3-none-any.whl → 2.2.0__py3-none-any.whl

epi_core/container.py

@@ -11,6 +11,7 @@ Implements the EPI file format specification:
 import hashlib
 import json
 import tempfile
+import threading
 import zipfile
 from pathlib import Path
 from typing import Optional
@@ -21,6 +22,9 @@ from epi_core.schemas import ManifestModel
 # EPI mimetype constant (vendor-specific MIME type per RFC 6838)
 EPI_MIMETYPE = "application/vnd.epi+zip"
 
+# Thread-safe lock for ZIP packing operations (prevents concurrent corruption)
+_zip_pack_lock = threading.Lock()
+
 
 class EPIContainer:
     """
@@ -157,6 +161,8 @@ class EPIContainer:
         """
         Create a .epi file from a source directory.
         
+        Thread-safe: Uses a module-level lock to prevent concurrent ZIP corruption.
+        
         The packing process:
         1. Write mimetype first (uncompressed) per ZIP spec
         2. Hash all files in source_dir
@@ -173,64 +179,67 @@ class EPIContainer:
             FileNotFoundError: If source_dir doesn't exist
             ValueError: If source_dir is not a directory
         """
-        if not source_dir.exists():
-            raise FileNotFoundError(f"Source directory not found: {source_dir}")
-        
-        if not source_dir.is_dir():
-            raise ValueError(f"Source must be a directory: {source_dir}")
-        
-        # Ensure output directory exists
-        output_path.parent.mkdir(parents=True, exist_ok=True)
-        
-        # Collect all files and compute hashes
-        file_manifest = {}
-        files_to_pack = []
-        
-        for file_path in source_dir.rglob("*"):
-            if file_path.is_file():
-                # Get relative path for archive
-                rel_path = file_path.relative_to(source_dir)
-                arc_name = str(rel_path).replace("\\", "/")  # Use forward slashes in ZIP
-                
-                # Compute hash
-                file_hash = EPIContainer._compute_file_hash(file_path)
-                file_manifest[arc_name] = file_hash
-                
-                files_to_pack.append((file_path, arc_name))
-        
-        # Update manifest with file hashes
-        manifest.file_manifest = file_manifest
-        
-        # Create embedded viewer with data injection
-        viewer_html = EPIContainer._create_embedded_viewer(source_dir, manifest)
-        
-        # Create ZIP file
-        with zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as zf:
-            # 1. Write mimetype FIRST and UNCOMPRESSED (per EPI spec)
-            zf.writestr(
-                "mimetype",
-                EPI_MIMETYPE,
-                compress_type=zipfile.ZIP_STORED  # No compression
-            )
+        # CRITICAL: Acquire lock to prevent concurrent ZIP corruption
+        # Multiple threads writing to ZIP simultaneously causes file header mismatches
+        with _zip_pack_lock:
+            if not source_dir.exists():
+                raise FileNotFoundError(f"Source directory not found: {source_dir}")
+            
+            if not source_dir.is_dir():
+                raise ValueError(f"Source must be a directory: {source_dir}")
+            
+            # Ensure output directory exists
+            output_path.parent.mkdir(parents=True, exist_ok=True)
+            
+            # Collect all files and compute hashes
+            file_manifest = {}
+            files_to_pack = []
+            
+            for file_path in source_dir.rglob("*"):
+                if file_path.is_file():
+                    # Get relative path for archive
+                    rel_path = file_path.relative_to(source_dir)
+                    arc_name = str(rel_path).replace("\\", "/")  # Use forward slashes in ZIP
+                    
+                    # Compute hash
+                    file_hash = EPIContainer._compute_file_hash(file_path)
+                    file_manifest[arc_name] = file_hash
+                    
+                    files_to_pack.append((file_path, arc_name))
             
-            # 2. Write all other files
-            for file_path, arc_name in files_to_pack:
-                zf.write(file_path, arc_name, compress_type=zipfile.ZIP_DEFLATED)
+            # Update manifest with file hashes
+            manifest.file_manifest = file_manifest
             
-            # 3. Write embedded viewer
-            zf.writestr(
-                "viewer.html",
-                viewer_html,
-                compress_type=zipfile.ZIP_DEFLATED
-            )
+            # Create embedded viewer with data injection
+            viewer_html = EPIContainer._create_embedded_viewer(source_dir, manifest)
             
-            # 4. Write manifest.json LAST (after all files are hashed)
-            manifest_json = manifest.model_dump_json(indent=2)
-            zf.writestr(
-                "manifest.json",
-                manifest_json,
-                compress_type=zipfile.ZIP_DEFLATED
-            )
+            # Create ZIP file
+            with zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as zf:
+                # 1. Write mimetype FIRST and UNCOMPRESSED (per EPI spec)
+                zf.writestr(
+                    "mimetype",
+                    EPI_MIMETYPE,
+                    compress_type=zipfile.ZIP_STORED  # No compression
+                )
+                
+                # 2. Write all other files
+                for file_path, arc_name in files_to_pack:
+                    zf.write(file_path, arc_name, compress_type=zipfile.ZIP_DEFLATED)
+                
+                # 3. Write embedded viewer
+                zf.writestr(
+                    "viewer.html",
+                    viewer_html,
+                    compress_type=zipfile.ZIP_DEFLATED
+                )
+                
+                # 4. Write manifest.json LAST (after all files are hashed)
+                manifest_json = manifest.model_dump_json(indent=2)
+                zf.writestr(
+                    "manifest.json",
+                    manifest_json,
+                    compress_type=zipfile.ZIP_DEFLATED
+                )
     
     @staticmethod
     def unpack(epi_path: Path, dest_dir: Optional[Path] = None) -> Path:
@@ -350,3 +359,7 @@ class EPIContainer:
                     mismatches[filename] = f"Hash mismatch: expected {expected_hash}, got {actual_hash}"
         
         return (len(mismatches) == 0, mismatches)
+
+
+
+ 

epi_core/redactor.py

@@ -277,3 +277,7 @@ def get_default_redactor() -> Redactor:
             pass  # Fail silently, use defaults
     
     return Redactor(config_path=config_path if config_path.exists() else None)
+
+
+
+ 

epi_core/schemas.py

@@ -18,7 +18,7 @@ class ManifestModel(BaseModel):
     """
     
     spec_version: str = Field(
-        default="1.1-json",
+        default="2.2.0",
         description="EPI specification version"
     )
     
@@ -145,4 +145,8 @@ class StepModel(BaseModel):
                 }
             }
         }
-    )
+    )
+
+
+
+ 

epi_core/serialize.py

@@ -158,3 +158,7 @@ def verify_hash(model: BaseModel, expected_hash: str, exclude_fields: set[str] |
     """
     actual_hash = get_canonical_hash(model, exclude_fields)
     return actual_hash == expected_hash
+
+
+
+ 

epi_core/storage.py

@@ -0,0 +1,186 @@
+"""
+SQLite-based storage for EPI recordings.
+
+Provides atomic, crash-safe storage replacing JSONL files.
+SQLite transactions ensure no data corruption on crashes.
+"""
+
+import sqlite3
+import json
+import time
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+from datetime import datetime
+
+from .schemas import StepModel
+
+
+class EpiStorage:
+    """
+    SQLite-based atomic storage for agent execution.
+    Replaces JSONL (which corrupts on crashes).
+    """
+    
+    def __init__(self, session_id: str, output_dir: Path):
+        """
+        Initialize SQLite storage.
+        
+        Args:
+            session_id: Unique session identifier
+            output_dir: Directory for database file
+        """
+        self.session_id = session_id
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        
+        self.db_path = self.output_dir / f"{session_id}_temp.db"
+        self.conn = sqlite3.connect(str(self.db_path), check_same_thread=False)
+        self._init_tables()
+    
+    def _init_tables(self):
+        """Initialize database schema"""
+        self.conn.execute('''
+            CREATE TABLE IF NOT EXISTS steps (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                step_index INTEGER NOT NULL,
+                timestamp TEXT NOT NULL,
+                kind TEXT NOT NULL,
+                content TEXT NOT NULL,
+                created_at REAL NOT NULL
+            )
+        ''')
+        
+        self.conn.execute('''
+            CREATE TABLE IF NOT EXISTS metadata (
+                key TEXT PRIMARY KEY,
+                value TEXT NOT NULL
+            )
+        ''')
+        
+        self.conn.execute('''
+            CREATE INDEX IF NOT EXISTS idx_steps_index 
+            ON steps(step_index)
+        ''')
+        
+        self.conn.commit()
+    
+    def add_step(self, step: StepModel) -> None:
+        """
+        Atomic insert of execution step.
+        Survives process crashes.
+        
+        Args:
+            step: StepModel to persist
+        """
+        self.conn.execute(
+            '''INSERT INTO steps 
+               (step_index, timestamp, kind, content, created_at) 
+               VALUES (?, ?, ?, ?, ?)''',
+            (
+                step.index,
+                step.timestamp.isoformat(),
+                step.kind,
+                step.model_dump_json(),
+                time.time()
+            )
+        )
+        self.conn.commit()
+    
+    def get_steps(self) -> List[StepModel]:
+        """
+        Retrieve all steps in order.
+        
+        Returns:
+            List of StepModel instances
+        """
+        cursor = self.conn.execute(
+            'SELECT content FROM steps ORDER BY step_index'
+        )
+        rows = cursor.fetchall()
+        
+        steps = []
+        for row in rows:
+            step_data = json.loads(row[0])
+            steps.append(StepModel(**step_data))
+        
+        return steps
+    
+    def set_metadata(self, key: str, value: str) -> None:
+        """
+        Set metadata key-value pair.
+        
+        Args:
+            key: Metadata key
+            value: Metadata value
+        """
+        self.conn.execute(
+            'INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)',
+            (key, value)
+        )
+        self.conn.commit()
+    
+    def get_metadata(self, key: str) -> Optional[str]:
+        """
+        Get metadata value.
+        
+        Args:
+            key: Metadata key
+            
+        Returns:
+            Metadata value or None
+        """
+        cursor = self.conn.execute(
+            'SELECT value FROM metadata WHERE key = ?',
+            (key,)
+        )
+        row = cursor.fetchone()
+        return row[0] if row else None
+    
+    def close(self) -> None:
+        """Close database connection."""
+        if self.conn:
+            self.conn.close()
+    
+    def export_to_jsonl(self, output_path: Path) -> None:
+        """
+        Export steps to JSONL file for backwards compatibility.
+        
+        Args:
+            output_path: Path to JSONL file
+        """
+        steps = self.get_steps()
+        with open(output_path, 'w', encoding='utf-8') as f:
+            for step in steps:
+                f.write(step.model_dump_json() + '\n')
+    
+    def finalize(self) -> Path:
+        """
+        Finalize recording and rename to final path.
+        This ensures we never have half-written files.
+        
+        Returns:
+            Path to finalized database file
+        """
+        # Add finalization metadata
+        self.set_metadata('finalized_at', datetime.utcnow().isoformat())
+        self.set_metadata('session_id', self.session_id)
+        
+        # Close connection
+        self.close()
+        
+        # Atomic rename (SQLite transaction guarantees consistency)
+        final_path = self.output_dir / "steps.jsonl"
+        
+        # Export to JSONL for backwards compatibility
+        self.conn = sqlite3.connect(str(self.db_path), check_same_thread=False)
+        self.export_to_jsonl(final_path)
+        self.close()
+        
+        # Clean up temp DB
+        self.db_path.unlink(missing_ok=True)
+        
+        return final_path
+
+
+
+ 

epi_core/trust.py

@@ -244,3 +244,7 @@ def create_verification_report(
         report["trust_message"] = "Integrity compromised - do not trust"
     
     return report
+
+
+
+ 

epi_recorder/__init__.py

@@ -4,7 +4,7 @@ EPI Recorder - Runtime interception and workflow capture.
 Python API for recording AI workflows with cryptographic verification.
 """
 
-__version__ = "2.1.3"
+__version__ = "2.2.0"
 
 # Export Python API
 from epi_recorder.api import (
@@ -19,3 +19,7 @@ __all__ = [
     "get_current_session",
     "__version__"
 ]
+
+
+
+ 

epi_recorder/api.py

@@ -176,6 +176,11 @@ class EpiRecorderSession:
                 output_path=self.output_path
             )
             
+            # CRITICAL: Windows file system flush
+            # Allow OS to finalize file before signing
+            import time
+            time.sleep(0.1)
+            
             # Sign if requested
             if self.auto_sign:
                 self._sign_epi_file()
@@ -355,7 +360,24 @@ class EpiRecorderSession:
                     encoding="utf-8"
                 )
                 
-                # Repack the ZIP with signed manifest
+                # Regenerate viewer.html with signed manifest
+                steps = []
+                steps_file = tmp_path / "steps.jsonl"
+                if steps_file.exists():
+                    for line in steps_file.read_text(encoding="utf-8").strip().split("\n"):
+                        if line:
+                            try:
+                                steps.append(json.loads(line))
+                            except json.JSONDecodeError:
+                                pass
+                
+                # Regenerate viewer with signed manifest
+                from epi_core.container import EPIContainer
+                viewer_html = EPIContainer._create_embedded_viewer(tmp_path, signed_manifest)
+                viewer_path = tmp_path / "viewer.html"
+                viewer_path.write_text(viewer_html, encoding="utf-8")
+                
+                # Repack the ZIP with signed manifest and updated viewer
                 # CRITICAL: Write to temp file first to prevent data loss
                 temp_output = self.output_path.with_suffix('.epi.tmp')
                 
@@ -590,4 +612,8 @@ def get_current_session() -> Optional[EpiRecorderSession]:
     Returns:
         EpiRecorderSession or None
     """
-    return getattr(_thread_local, 'active_session', None)
+    return getattr(_thread_local, 'active_session', None)
+
+
+
+ 

epi_recorder/async_api.py

@@ -0,0 +1,151 @@
+import asyncio
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import asynccontextmanager
+from typing import Optional, Dict, Any
+from datetime import datetime
+
+from epi_core.storage import EpiStorage
+from epi_core.schemas import StepModel
+
+class AsyncRecorder:
+    """
+    Async-native recorder that doesn't block the event loop.
+    Uses background thread for SQLite writes.
+    """
+    
+    def __init__(self, session_name: str, output_dir: str = "."):
+        self.session_name = session_name
+        self.output_dir = output_dir
+        
+        # Thread-safe queue for steps
+        self._queue = asyncio.Queue()
+        
+        # Background thread executor (1 thread is enough for SQLite)
+        self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="epi_writer")
+        
+        # Storage instance (created in background thread)
+        self._storage: Optional[EpiStorage] = None
+        self._writer_task: Optional[asyncio.Task] = None
+        
+        # State tracking
+        self._step_count = 0
+        self._done = asyncio.Event()
+        self._error: Optional[Exception] = None
+    
+    async def start(self):
+        """Initialize storage in background thread and start writer"""
+        # Create storage in thread (SQLite init is also blocking)
+        loop = asyncio.get_event_loop()
+        self._storage = await loop.run_in_executor(
+            self._executor,
+            lambda: EpiStorage(self.session_name, self.output_dir)
+        )
+        
+        # Start background writer task
+        self._writer_task = asyncio.create_task(self._writer_loop())
+    
+    async def record_step(self, step_type: str, content: dict):
+        """Non-blocking step recording"""
+        if self._error:
+            raise self._error
+        
+        self._step_count += 1
+        
+        # Put in queue (never blocks, just buffers in memory)
+        await self._queue.put({
+            'index': self._step_count,
+            'type': step_type,
+            'content': content,
+            'timestamp': datetime.utcnow() # StepModel expects datetime
+        })
+    
+    async def _writer_loop(self):
+        """Background task: Drains queue to SQLite"""
+        try:
+            while True:
+                # Wait for item with timeout to check for shutdown
+                try:
+                    step_data = await asyncio.wait_for(self._queue.get(), timeout=0.5)
+                except asyncio.TimeoutError:
+                    continue
+                
+                if step_data is None: # Shutdown sentinel
+                   self._queue.task_done()
+                   break
+
+                # Write to SQLite in background thread (non-blocking for async)
+                loop = asyncio.get_event_loop()
+                await loop.run_in_executor(
+                    self._executor,
+                    self._write_to_storage,
+                    step_data
+                )
+                
+                self._queue.task_done()
+                
+        except asyncio.CancelledError:
+            # Graceful shutdown
+            pass
+        except Exception as e:
+            self._error = e
+    
+    def _write_to_storage(self, step_data: dict):
+        """Synchronous SQLite write (runs in background thread)"""
+        if self._storage:
+            # Construct StepModel
+            step = StepModel(
+                index=step_data['index'],
+                timestamp=step_data['timestamp'],
+                kind=step_data['type'],
+                content=step_data['content']
+            )
+            self._storage.add_step(step)
+    
+    async def stop(self):
+        """Finalize: Drain queue, close storage"""
+        if not self._writer_task:
+            return
+        
+        # Signal writer to finish
+        await self._queue.put(None)
+        await self._queue.join()
+        
+        # Wait for task
+        self._writer_task.cancel()
+        try:
+            await self._writer_task
+        except asyncio.CancelledError:
+            pass
+        
+        # Finalize storage in background thread
+        if self._storage:
+            loop = asyncio.get_event_loop()
+            await loop.run_in_executor(
+                self._executor,
+                self._storage.finalize
+            )
+        
+        # Shutdown executor
+        self._executor.shutdown(wait=True)
+
+@asynccontextmanager
+async def record_async(session_name: str, output_dir: str = "."):
+    """
+    Async context manager for recording.
+    
+    Usage:
+        async with record_async("my_agent") as rec:
+            await agent.arun("task")  # Non-blocking
+    """
+    recorder = AsyncRecorder(session_name, output_dir)
+    await recorder.start()
+    try:
+        yield recorder
+    finally:
+        await recorder.stop()
+
+
+
+ 

epi_recorder/bootstrap.py

@@ -56,3 +56,7 @@ def initialize_recording():
 # Auto-initialize if EPI_RECORD is set
 if os.environ.get("EPI_RECORD") == "1":
     initialize_recording()
+
+
+
+ 

epi_recorder/environment.py

@@ -235,3 +235,7 @@ def capture_environment(
         include_all_env_vars=include_all_env_vars,
         redact_env_vars=redact_env_vars
     )
+
+
+
+