upii 0.7.0__py3-none-any.whl

upii/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.7.0"

upii/ambient/calendar_connector.py

@@ -0,0 +1,106 @@
+import os
+import re
+import datetime
+import uuid
+from typing import List, Dict, Generator
+from upii.core.logger import logger
+from upii.storage.db import DB
+
+class CalendarConnector:
+    """
+    Parses .ics files and ingests metadata into LTM.
+    Focus: Title, Start, End, Participants.
+    """
+    
+    def __init__(self):
+        self.db = DB()
+        
+    def process_file(self, file_path: str) -> int:
+        """Parses an ICS file and saves events. Returns count."""
+        if not os.path.exists(file_path):
+            logger.error(f"Calendar file not found: {file_path}")
+            return 0
+            
+        count = 0
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                content = f.read()
+                
+            events = self._parse_ics(content)
+            self.db.init_db()
+            
+            for event in events:
+                event['source_file'] = os.path.basename(file_path)
+                self.db.add_calendar_event(event)
+                count += 1
+                
+            logger.info(f"Ingested {count} calendar events from {file_path}")
+        except Exception as e:
+            logger.error(f"Failed to process calendar {file_path}: {e}")
+            
+        return count
+
+    def _parse_ics(self, content: str) -> List[Dict]:
+        """
+        Robust-ish Regex parser for ICS. 
+        (Production would use `icalendar` lib, but we avoid deps here).
+        """
+        events = []
+        # Split by VEVENT
+        raw_events = re.findall(r'BEGIN:VEVENT(.*?)END:VEVENT', content, re.DOTALL)
+        
+        for raw in raw_events:
+            try:
+                # 1. Title (SUMMARY)
+                summary_match = re.search(r'SUMMARY:(.*)', raw)
+                title = summary_match.group(1).strip() if summary_match else "Untitled Event"
+                
+                # 2. Date (DTSTART/DTEND)
+                # Formats: 20230101T120000Z or ;TZID=...:2023...
+                dtstart_match = re.search(r'DTSTART.*?:(\d{8}T\d{6}Z|\d{8}T\d{6}|\d{8})', raw)
+                dtend_match = re.search(r'DTEND.*?:(\d{8}T\d{6}Z|\d{8}T\d{6}|\d{8})', raw)
+                
+                start_str = dtstart_match.group(1) if dtstart_match else None
+                end_str = dtend_match.group(1) if dtend_match else None
+                
+                if not start_str: continue # Skip if no time
+                
+                # Normalize Time
+                start_time = self._parse_ical_date(start_str)
+                end_time = self._parse_ical_date(end_str) if end_str else start_time
+                
+                # 3. Participants (ATTENDEE)
+                # ATTENDEE;CN="Alice Smith":mailto:alice@example.com
+                attendees = []
+                for line in raw.split('\n'):
+                    if line.startswith('ATTENDEE'):
+                        # Extract email or CN
+                        if 'mailto:' in line:
+                            email = line.split('mailto:')[-1].strip()
+                            attendees.append(email)
+                        elif 'CN=' in line:
+                            cn = line.split('CN=')[1].split(':')[0].replace('"','').strip()
+                            attendees.append(cn)
+                
+                events.append({
+                    "event_id": str(uuid.uuid4()),
+                    "title": title,
+                    "start_time": start_time,
+                    "end_time": end_time,
+                    "participants": attendees
+                })
+                
+            except Exception as e:
+                logger.warning(f"Failed to parse event: {e}")
+                
+        return events
+
+    def _parse_ical_date(self, date_str: str) -> str:
+        """Converts ICS date/datetime to ISO8601 string."""
+        # YYYYMMDD
+        if len(date_str) == 8:
+            return f"{date_str[:4]}-{date_str[4:6]}-{date_str[6:]}T00:00:00"
+        
+        # YYYYMMDDTHHMMSS(Z)
+        clean = date_str.replace('Z', '')
+        return f"{clean[:4]}-{clean[4:6]}-{clean[6:8]}T{clean[9:11]}:{clean[11:13]}:{clean[13:]}"

upii/ambient/dummy_sources.py

@@ -0,0 +1,54 @@
+import time
+import random
+from upii.ambient.sources import Source, registry
+from upii.core.concurrency import SafeRunner
+from upii.core.logger import logger
+
+class BrowserSource(Source):
+    """Simulates capturing high-dwell web pages."""
+    def __init__(self):
+        super().__init__("browser", "Captures metadata from high-dwell browser tabs.")
+        self.thread = None
+        
+    def start(self):
+        self.is_running = True
+        self.thread = SafeRunner.run_daemon(self._loop, name="BrowserWatcher")
+        
+    def stop(self):
+        self.is_running = False
+        
+    def _loop(self):
+        while self.is_running:
+            time.sleep(10) # Simulate infrequent capture
+            # In a real app, this would poll a browser extension API
+            # Here we just randomly decide to 'capture' something if enabled
+            if random.random() < 0.1: # 10% chance per loop
+                 logger.info("BrowserSource: simulating capture")
+                 fake_url = "https://example.com/interesting-article"
+                 self.storage.add_event("browser_visit", fake_url)
+                 self.storage.log_audit(self.name, "capture", {"url": fake_url, "dwell_time": 120})
+
+class CalendarSource(Source):
+    """Simulates capturing calendar metadata."""
+    def __init__(self):
+        super().__init__("calendar", "Ingests event titles and participants.")
+        
+    def start(self):
+        self.is_running = True
+        # Calendar might be poll-once or interval
+        self.thread = SafeRunner.run_daemon(self._loop, name="CalendarWatcher")
+        
+    def stop(self):
+        self.is_running = False
+        
+    def _loop(self):
+        while self.is_running:
+            time.sleep(20)
+            if random.random() < 0.1:
+                logger.info("CalendarSource: simulating capture")
+                self.storage.add_event("calendar_event", "Meeting with Omega Team")
+                self.storage.log_audit(self.name, "capture", {"summary": "Meeting", "participants": ["alice@example.com"]})
+
+# Register
+registry.register(BrowserSource())
+registry.register(CalendarSource())

upii/ambient/email_connector.py

@@ -0,0 +1,74 @@
+import mailbox
+import os
+import email
+from email.header import decode_header
+from typing import Generator, Dict, Any
+from upii.core.logger import logger
+
+class EmailConnector:
+    """
+    Parses email files (.mbox, .eml) and extracts content.
+    """
+    
+    @staticmethod
+    def parse_mbox(file_path: str) -> Generator[Dict[str, Any], None, None]:
+        """Parses an MBOX file and yields email data dictionaries."""
+        if not os.path.exists(file_path):
+            logger.error(f"Mbox file not found: {file_path}")
+            return
+
+        try:
+            mbox = mailbox.mbox(file_path)
+            for message in mbox:
+                try:
+                    yield EmailConnector._extract_message_data(message, file_path)
+                except Exception as e:
+                    logger.warning(f"Failed to parse email message: {e}")
+        except Exception as e:
+            logger.error(f"Failed to open mbox {file_path}: {e}")
+
+    @staticmethod
+    def _extract_message_data(msg, source_file: str) -> Dict[str, Any]:
+        """Extracts title, sender, date, and body from an email message."""
+        
+        def decode_str(s):
+            if not s: return ""
+            try:
+                decoded_list = decode_header(s)
+                return "".join([
+                    (t[0].decode(t[1] or 'utf-8') if isinstance(t[0], bytes) else t[0]) 
+                    for t in decoded_list
+                ])
+            except:
+                return str(s)
+
+        subject = decode_str(msg.get('subject', 'No Subject'))
+        sender = decode_str(msg.get('from', 'Unknown'))
+        date = msg.get('date', '')
+        
+        # Body extraction
+        body = ""
+        if msg.is_multipart():
+            for part in msg.walk():
+                content_type = part.get_content_type()
+                content_disposition = str(part.get("Content-Disposition"))
+                
+                if content_type == "text/plain" and "attachment" not in content_disposition:
+                    try:
+                        body += part.get_payload(decode=True).decode('utf-8', errors='ignore')
+                    except: pass
+        else:
+            try:
+                body = msg.get_payload(decode=True).decode('utf-8', errors='ignore')
+            except: pass
+
+        return {
+            "title": subject,
+            "sender": sender,
+            "date": date,
+            "content": f"From: {sender}\nDate: {date}\nSubject: {subject}\n\n{body}",
+            "source_type": "email",
+             # We might want a unique ID for the email if available, else random
+            "email_id": msg.get('Message-ID', ''),
+            "source_file": source_file
+        }

upii/ambient/sources.py

@@ -0,0 +1,111 @@
+import abc
+import os
+import yaml
+import logging
+from typing import Dict, Any, List, Optional
+from upii.core.config import config
+from upii.core.concurrency import SafeRunner
+from upii.ambient.storage import StagingDB
+
+logger = logging.getLogger("upii.ambient.sources")
+
+class Source(abc.ABC):
+    """Abstract Base Class for Ambient Sources."""
+    
+    def __init__(self, name: str, description: str):
+        self.name = name
+        self.description = description
+        self.is_running = False
+        self.config: Dict[str, Any] = {}
+        self.storage = StagingDB()
+        
+    @abc.abstractmethod
+    def start(self):
+        """Start the passive capture loop."""
+        pass
+        
+    @abc.abstractmethod
+    def stop(self):
+        """Stop capture."""
+        pass
+        
+    def configure(self, new_config: Dict[str, Any]):
+        """Update configuration."""
+        self.config.update(new_config)
+
+class SourceRegistry:
+    """Manages lifecycle and configuration of sources."""
+    
+    _instance = None
+    
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super(SourceRegistry, cls).__new__(cls)
+            cls._instance._init()
+        return cls._instance
+        
+    def _init(self):
+        self.sources: Dict[str, Source] = {}
+        self.enabled_state: Dict[str, bool] = {} # Persistent state
+        self.config_path = os.path.join(os.path.dirname(config.db_path), "sources.yaml")
+        self._load_state()
+        
+    def register(self, source: Source):
+        """Register a source plugin."""
+        self.sources[source.name] = source
+        # Load config if exists
+        # (For MVP we rely on defaults or manual config injection)
+        logger.debug(f"Registered source: {source.name}")
+        
+    def is_enabled(self, name: str) -> bool:
+        return self.enabled_state.get(name, False)
+        
+    def enable(self, name: str):
+        if name not in self.sources:
+            raise ValueError(f"Source {name} not found")
+            
+        self.enabled_state[name] = True
+        self.sources[name].start()
+        self._save_state()
+        self.sources[name].storage.log_audit(name, "enable", {"status": "started"})
+        logger.info(f"Source enabled: {name}")
+
+    def disable(self, name: str):
+        if name not in self.sources:
+             raise ValueError(f"Source {name} not found")
+             
+        self.enabled_state[name] = False
+        self.sources[name].stop()
+        self._save_state()
+        self.sources[name].storage.log_audit(name, "disable", {"status": "stopped"})
+        logger.info(f"Source disabled: {name}")
+        
+    def _load_state(self):
+        if os.path.exists(self.config_path):
+            try:
+                with open(self.config_path, 'r') as f:
+                    data = yaml.safe_load(f) or {}
+                    self.enabled_state = data.get("enabled", {})
+                    # Ideally we also load per-source config here
+            except Exception as e:
+                logger.error(f"Failed to load source state: {e}")
+                
+    def _save_state(self):
+        try:
+            with open(self.config_path, 'w') as f:
+                yaml.dump({"enabled": self.enabled_state}, f)
+        except Exception as e:
+            logger.error(f"Failed to save source state: {e}")
+            
+    def get_all(self) -> List[Dict]:
+        return [
+            {
+                "name": s.name,
+                "description": s.description,
+                "enabled": self.is_enabled(s.name),
+                "running": s.is_running
+            } 
+            for s in self.sources.values()
+        ]
+
+registry = SourceRegistry()

upii/ambient/storage.py

@@ -0,0 +1,159 @@
+import sqlite3
+import json
+import uuid
+import os
+from typing import Optional, Dict, List
+from upii.core.config import config
+from upii.core.logger import logger
+
+class StagingDB:
+    """Interface to the v1.0 Staging Database (Ambient)."""
+    
+    def __init__(self):
+        self.db_path = config.staging_db_path
+        
+    def init_db(self):
+        """Initialize staging schema."""
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+        
+        # Events Table
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS events (
+                event_id TEXT PRIMARY KEY,
+                event_type TEXT NOT NULL,
+                file_path TEXT NOT NULL,
+                detected_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                status TEXT DEFAULT 'pending'
+            )
+        """)
+        
+        # Staging Docs Table
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS staging_docs (
+                staging_id TEXT PRIMARY KEY,
+                source_event_id TEXT,
+                file_path TEXT NOT NULL,
+                content_hash TEXT,
+                parsed_content TEXT,
+                metadata JSON,
+                status TEXT DEFAULT 'review',
+                FOREIGN KEY(source_event_id) REFERENCES events(event_id)
+            )
+        """)
+        
+        # Audit Log Table
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS audit_logs (
+                audit_id TEXT PRIMARY KEY,
+                source_name TEXT NOT NULL,
+                action TEXT NOT NULL,         -- 'enable', 'disable', 'capture', 'error'
+                details JSON,
+                timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+        
+        conn.commit()
+        conn.close()
+
+    def log_audit(self, source: str, action: str, details: Dict = {}):
+        """Write an entry to the audit log."""
+        audit_id = str(uuid.uuid4())
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+        cursor.execute(
+            "INSERT INTO audit_logs (audit_id, source_name, action, details) VALUES (?, ?, ?, ?)",
+            (audit_id, source, action, json.dumps(details))
+        )
+        conn.commit()
+        conn.close()
+        return audit_id
+        
+    def get_audit_logs(self, limit: int = 50) -> List[Dict]:
+        conn = sqlite3.connect(self.db_path)
+        conn.row_factory = sqlite3.Row
+        cursor = conn.cursor()
+        # timestamp is CURRENT_TIMESTAMP (1s resolution); break ties on the implicit
+        # insertion-order rowid so the most recent entry is always first, even when
+        # several entries land in the same second.
+        cursor.execute("SELECT * FROM audit_logs ORDER BY timestamp DESC, rowid DESC LIMIT ?", (limit,))
+        rows = cursor.fetchall()
+        conn.close()
+        return [dict(row) for row in rows]
+
+    def add_event(self, event_type: str, file_path: str) -> str:
+        """Log a raw file system event."""
+        event_id = str(uuid.uuid4())
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+        cursor.execute(
+            "INSERT INTO events (event_id, event_type, file_path) VALUES (?, ?, ?)",
+            (event_id, event_type, file_path)
+        )
+        conn.commit()
+        conn.close()
+        return event_id
+
+    def add_staging_doc(self, source_event_id: str, file_path: str, content: str, content_hash: str, metadata: Dict) -> str:
+        """Add a parsed document to staging."""
+        staging_id = str(uuid.uuid4())
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+        cursor.execute(
+            """INSERT INTO staging_docs 
+               (staging_id, source_event_id, file_path, content_hash, parsed_content, metadata) 
+               VALUES (?, ?, ?, ?, ?, ?)""",
+            (staging_id, source_event_id, file_path, content_hash, content, json.dumps(metadata))
+        )
+        conn.commit()
+        conn.close()
+        return staging_id
+        
+    def get_pending_events(self) -> List[Dict]:
+        conn = sqlite3.connect(self.db_path)
+        conn.row_factory = sqlite3.Row
+        cursor = conn.cursor()
+        cursor.execute("SELECT * FROM events WHERE status = 'pending' ORDER BY detected_at DESC")
+        rows = cursor.fetchall()
+        conn.close()
+        return [dict(row) for row in rows]
+
+    def get_all_events(self) -> List[Dict]:
+        """Return every event regardless of status (pending/approved/rejected/acknowledged)."""
+        conn = sqlite3.connect(self.db_path)
+        conn.row_factory = sqlite3.Row
+        cursor = conn.cursor()
+        cursor.execute("SELECT * FROM events ORDER BY detected_at DESC")
+        rows = cursor.fetchall()
+        conn.close()
+        return [dict(row) for row in rows]
+
+    def get_staging_doc_by_event(self, event_id: str) -> Optional[Dict]:
+        """Fetch the staged (reviewed) document content for a given event.
+
+        This is the content the operator approved — promotion must use this,
+        NOT a fresh disk read, so what enters LTM is exactly what was reviewed.
+        """
+        conn = sqlite3.connect(self.db_path)
+        conn.row_factory = sqlite3.Row
+        cursor = conn.cursor()
+        cursor.execute(
+            "SELECT * FROM staging_docs WHERE source_event_id = ? LIMIT 1", (event_id,)
+        )
+        row = cursor.fetchone()
+        conn.close()
+        return dict(row) if row else None
+
+    def update_event_status(self, event_id: str, status: str):
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+        cursor.execute("UPDATE events SET status = ? WHERE event_id = ?", (status, event_id))
+        conn.commit()
+        conn.close()
+
+    def update_staging_status(self, staging_id: str, status: str):
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+        cursor.execute("UPDATE staging_docs SET status = ? WHERE staging_id = ?", (status, staging_id))
+        conn.commit()
+        conn.close()