synth-ai 0.2.9.dev11__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/environments/examples/enron/art_helpers/email_search_tools.py

@@ -1,156 +0,0 @@
-import logging
-import sqlite3
-import textwrap
-from dataclasses import dataclass
-from typing import List, Optional
-
-from synth_ai.environments.environment.db.sqlite import SQLiteManager
-from synth_ai.environments.examples.enron.art_helpers.types_enron import Email
-
-# Configure logger for this module
-logger = logging.getLogger(__name__)
-if not logger.handlers:  # avoid duplicate handlers in pytest -x
-    h = logging.StreamHandler()
-    h.setFormatter(logging.Formatter("%(asctime)s  %(levelname)s  %(message)s"))
-    logger.addHandler(h)
-logger.setLevel(logging.DEBUG)  # DEBUG so we see the raw SQL
-
-
-@dataclass
-class SearchResult:
-    message_id: str
-    snippet: str
-    score: float
-
-
-def search_emails(
-    sqlite_manager: SQLiteManager,
-    inbox: str,
-    keywords: List[str],
-    from_addr: Optional[str] = None,
-    to_addr: Optional[str] = None,
-    sent_after: Optional[str] = None,
-    sent_before: Optional[str] = None,
-    max_results: int = 10,
-) -> List[SearchResult]:
-    """
-    Searches the email database based on keywords, inbox, sender, recipient, and date range.
-
-    Args:
-        sqlite_manager: The SQLiteManager instance for database operations.
-        inbox: The email address of the user performing the search.
-               Results include emails sent from or to (inc. cc/bcc) this address.
-        keywords: A list of keywords that must all appear in the subject or body.
-        from_addr: Optional email address to filter emails sent *from*.
-        to_addr: Optional email address to filter emails sent *to* (inc. cc/bcc).
-        sent_after: Optional date string 'YYYY-MM-DD'. Filters for emails sent on or after this date.
-        sent_before: Optional date string 'YYYY-MM-DD'. Filters for emails sent before this date.
-        max_results: The maximum number of results to return. Cannot exceed 10.
-
-    Returns:
-        A list of SearchResult objects, each containing 'message_id' and 'snippet'.
-        Returns an empty list if no results are found or an error occurs.
-    """
-
-    if not keywords:
-        raise ValueError("No keywords provided for search.")
-    if max_results > 10:
-        # The user snippet implies max_results isn't part of the simplified SQL here.
-        # Keeping the check, but the new SQL query below does not use all filters.
-        # This might need reconciliation if all filters are intended to be used with the new SQL.
-        logger.warning(
-            "max_results > 10, but the provided SQL snippet for logging might not respect all filters."
-        )
-
-    safe_keywords = [k.replace("'", "''") for k in keywords]
-    fts_match_query = " ".join(f'"{k}"' for k in safe_keywords)
-
-    sql_query = textwrap.dedent("""
-        SELECT DISTINCT
-               e.message_id,
-               snippet(emails_fts, -1, '⟪', '⟫', ' … ', 15) AS snip
-          FROM emails e
-          JOIN emails_fts ON e.id = emails_fts.rowid
-         WHERE emails_fts MATCH ?
-         LIMIT ?
-    """).strip()
-
-    params = (fts_match_query, max_results)
-
-    try:
-        with sqlite_manager.connection() as db_conn:
-            rows = db_conn.execute(sql_query, params).fetchall()
-        return [SearchResult(message_id=row[0], snippet=row[1], score=0.0) for row in rows]
-    except sqlite3.Error as e:
-        logger.error(f"Database error during search: {e}\nSQL: {sql_query}\nParams: {params}")
-        return []
-
-
-def read_email(sqlite_manager: SQLiteManager, message_id: str) -> Optional[Email]:
-    """
-    Retrieves a single email by its message_id from the database.
-
-    Args:
-        sqlite_manager: The SQLiteManager instance for database operations.
-        message_id: The unique identifier of the email to retrieve.
-
-    Returns:
-        An Email object containing the details of the found email,
-        or None if the email is not found or an error occurs.
-    """
-
-    email_sql = """
-        SELECT id, message_id, date, subject, from_address, body, file_name
-        FROM emails
-        WHERE message_id = ?;
-    """
-
-    recipients_sql = """
-        SELECT recipient_address, recipient_type
-        FROM recipients
-        WHERE email_id = ?;
-    """
-
-    try:
-        with sqlite_manager.connection() as db_conn:
-            cursor = db_conn.cursor()
-            cursor.execute(email_sql, (message_id,))
-            email_row = cursor.fetchone()
-
-            if not email_row:
-                logging.warning(f"Email with message_id '{message_id}' not found.")
-                return None
-
-            email_id, msg_id, date, subject, from_addr, body, file_name = email_row
-            # Fetch recipients for this email primary key
-            cursor.execute(recipients_sql, (email_id,))
-            recipient_rows = cursor.fetchall()
-    except sqlite3.Error as e:
-        logger.error(f"Database error reading email {message_id}: {e}")
-        return None
-
-    to_addresses: List[str] = []
-    cc_addresses: List[str] = []
-    bcc_addresses: List[str] = []
-
-    for addr, type_val in recipient_rows:
-        type_lower = type_val.lower()
-        if type_lower == "to":
-            to_addresses.append(addr)
-        elif type_lower == "cc":
-            cc_addresses.append(addr)
-        elif type_lower == "bcc":
-            bcc_addresses.append(addr)
-
-    email_obj = Email(
-        message_id=msg_id,
-        date=date,
-        subject=subject,
-        from_address=from_addr,
-        to_addresses=to_addresses,
-        cc_addresses=cc_addresses,
-        bcc_addresses=bcc_addresses,
-        body=body,
-        file_name=file_name,
-    )
-    return email_obj

synth_ai/environments/examples/enron/art_helpers/local_email_db.py

@@ -1,281 +0,0 @@
-import logging
-import os
-import sqlite3
-from datetime import datetime
-
-from datasets import Dataset, Features, Sequence, Value, load_dataset
-from tqdm import tqdm
-
-# Resolve paths relative to this file so it works regardless of the current working directory
-BASE_DIR = os.path.dirname(os.path.abspath(__file__))
-# Database will live in "../data/enron_emails.db" relative to project root
-DEFAULT_DB_PATH = os.path.join(BASE_DIR, "..", "..", "data", "enron_emails.db")
-
-DEFAULT_REPO_ID = "corbt/enron-emails"
-
-logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
-
-
-# --- Database Schema ---
-SQL_CREATE_TABLES = """
-DROP TABLE IF EXISTS recipients;
-DROP TABLE IF EXISTS emails_fts;
-DROP TABLE IF EXISTS emails;
-
-CREATE TABLE emails (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    message_id TEXT UNIQUE,
-    subject TEXT,
-    from_address TEXT,
-    date TEXT, -- Store as ISO 8601 string 'YYYY-MM-DD HH:MM:SS'
-    body TEXT,
-    file_name TEXT
-);
-
-CREATE TABLE recipients (
-    email_id INTEGER,
-    recipient_address TEXT,
-    recipient_type TEXT, -- 'to', 'cc', 'bcc'
-    FOREIGN KEY(email_id) REFERENCES emails(id) ON DELETE CASCADE
-);
-"""
-
-SQL_CREATE_INDEXES_TRIGGERS = """
-CREATE INDEX idx_emails_from ON emails(from_address);
-CREATE INDEX idx_emails_date ON emails(date);
-CREATE INDEX idx_emails_message_id ON emails(message_id);
-CREATE INDEX idx_recipients_address ON recipients(recipient_address);
-CREATE INDEX idx_recipients_type ON recipients(recipient_type);
-CREATE INDEX idx_recipients_email_id ON recipients(email_id);
-CREATE INDEX idx_recipients_address_email ON recipients(recipient_address, email_id);
-
-CREATE VIRTUAL TABLE emails_fts USING fts5(
-    subject,
-    body,
-    content='emails',
-    content_rowid='id'
-);
-
-CREATE TRIGGER emails_ai AFTER INSERT ON emails BEGIN
-    INSERT INTO emails_fts (rowid, subject, body)
-    VALUES (new.id, new.subject, new.body);
-END;
-
-CREATE TRIGGER emails_ad AFTER DELETE ON emails BEGIN
-    DELETE FROM emails_fts WHERE rowid=old.id;
-END;
-
-CREATE TRIGGER emails_au AFTER UPDATE ON emails BEGIN
-    UPDATE emails_fts SET subject=new.subject, body=new.body WHERE rowid=old.id;
-END;
-
-INSERT INTO emails_fts (rowid, subject, body) SELECT id, subject, body FROM emails;
-"""
-
-
-# --- Functions ---
-
-
-def download_dataset(repo_id: str) -> Dataset:
-    """Downloads the dataset from Hugging Face Hub."""
-    logging.info(f"Attempting to download dataset from Hugging Face Hub: {repo_id}")
-    expected_features = Features(
-        {
-            "message_id": Value("string"),
-            "subject": Value("string"),
-            "from": Value("string"),
-            "to": Sequence(Value("string")),
-            "cc": Sequence(Value("string")),
-            "bcc": Sequence(Value("string")),
-            "date": Value("timestamp[us]"),
-            "body": Value("string"),
-            "file_name": Value("string"),
-        }
-    )
-    dataset_obj = load_dataset(repo_id, features=expected_features, split="train")
-    # Basic type check remains useful
-    if not isinstance(dataset_obj, Dataset):
-        raise TypeError(f"Expected Dataset, got {type(dataset_obj)}")
-    logging.info(f"Successfully loaded dataset '{repo_id}' with {len(dataset_obj)} records.")
-    return dataset_obj
-
-
-def create_database(db_path: str):
-    """Creates the SQLite database and tables."""
-    logging.info(f"Creating SQLite database and tables at: {db_path}")
-    conn = sqlite3.connect(db_path)
-    cursor = conn.cursor()
-    cursor.executescript(SQL_CREATE_TABLES)
-    conn.commit()
-    conn.close()
-    logging.info("Database tables created successfully.")
-
-
-def populate_database(db_path: str, dataset: Dataset):
-    """Populates the database with data from the Hugging Face dataset."""
-    logging.info(f"Populating database {db_path}...")
-    conn = sqlite3.connect(db_path)
-    cursor = conn.cursor()
-
-    # --- Performance Pragmas ---
-    conn.execute("PRAGMA synchronous = OFF;")
-    conn.execute("PRAGMA journal_mode = MEMORY;")
-
-    record_count = 0
-    skipped_count = 0  # Keep track of skipped emails due to filters
-    duplicate_count = 0  # Keep track of skipped duplicate emails
-    processed_emails = set()  # Track (subject, body, from) tuples to dedupe
-
-    conn.execute("BEGIN TRANSACTION;")  # Single transaction for bulk insert
-
-    for email_data in tqdm(dataset, desc="Inserting emails"):
-        assert isinstance(email_data, dict)
-        message_id = email_data["message_id"]
-        subject = email_data["subject"]
-        from_address = email_data["from"]
-        date_obj: datetime = email_data["date"]
-        body = email_data["body"]
-        file_name = email_data["file_name"]
-        to_list_raw = email_data["to"]
-        cc_list_raw = email_data["cc"]
-        bcc_list_raw = email_data["bcc"]
-
-        # --- Data Cleaning and Filtering ---
-        date_str = date_obj.strftime("%Y-%m-%d %H:%M:%S")
-        to_list = [str(addr) for addr in to_list_raw if addr]
-        cc_list = [str(addr) for addr in cc_list_raw if addr]
-        bcc_list = [str(addr) for addr in bcc_list_raw if addr]
-
-        # Check body length
-        if len(body) > 5000:
-            logging.debug(f"Skipping email {message_id}: Body length > 5000 characters.")
-            skipped_count += 1
-            continue
-
-        # Check total recipients
-        total_recipients = len(to_list) + len(cc_list) + len(bcc_list)
-        if total_recipients > 30:
-            logging.debug(
-                f"Skipping email {message_id}: Total recipients ({total_recipients}) > 30."
-            )
-            skipped_count += 1
-            continue
-        # --- End Filtering ---
-
-        # --- Deduplication Check ---
-        email_key = (subject, body, from_address)
-        if email_key in processed_emails:
-            logging.debug(
-                f"Skipping duplicate email (Subject: {subject[:50]}..., From: {from_address})"
-            )
-            duplicate_count += 1
-            continue
-        else:
-            processed_emails.add(email_key)
-        # --- End Deduplication ---
-
-        cursor.execute(
-            """
-            INSERT INTO emails (message_id, subject, from_address, date, body, file_name)
-            VALUES (?, ?, ?, ?, ?, ?)
-            """,
-            (message_id, subject, from_address, date_str, body, file_name),
-        )
-        email_pk_id = cursor.lastrowid
-
-        recipient_data = []
-        for addr in to_list:
-            recipient_data.append((email_pk_id, addr, "to"))
-        for addr in cc_list:
-            recipient_data.append((email_pk_id, addr, "cc"))
-        for addr in bcc_list:
-            recipient_data.append((email_pk_id, addr, "bcc"))
-
-        if recipient_data:
-            cursor.executemany(
-                """
-                INSERT INTO recipients (email_id, recipient_address, recipient_type)
-                VALUES (?, ?, ?)
-                """,
-                recipient_data,
-            )
-        record_count += 1
-
-    conn.commit()
-    conn.close()
-    logging.info(f"Successfully inserted {record_count} email records.")
-    if skipped_count > 0:
-        logging.info(f"Skipped {skipped_count} email records due to length or recipient limits.")
-    if duplicate_count > 0:
-        logging.info(
-            f"Skipped {duplicate_count} duplicate email records (based on subject, body, from)."
-        )
-
-
-def create_indexes_and_triggers(db_path: str):
-    """Creates indexes and triggers on the populated database."""
-    logging.info(f"Creating indexes and triggers for database: {db_path}...")
-    conn = sqlite3.connect(db_path)
-    cursor = conn.cursor()
-    cursor.executescript(SQL_CREATE_INDEXES_TRIGGERS)
-    conn.commit()
-    conn.close()
-    logging.info("Indexes and triggers created successfully.")
-
-
-def generate_database(overwrite: bool = False):
-    """
-    Generates the SQLite database from the specified Hugging Face dataset.
-    Simplified version without extensive error handling.
-
-    Args:
-        repo_id: The Hugging Face repository ID for the dataset.
-        db_path: The path where the SQLite database file should be created.
-        overwrite: If True, any existing database file at db_path will be removed.
-    """
-    logging.info(
-        f"Starting database generation for repo '{DEFAULT_REPO_ID}' at '{DEFAULT_DB_PATH}'"
-    )
-    logging.info(f"Overwrite existing database: {overwrite}")
-
-    db_dir = os.path.dirname(DEFAULT_DB_PATH)
-    if db_dir and not os.path.exists(db_dir):
-        logging.info(f"Creating data directory: {db_dir}")
-        os.makedirs(db_dir)
-
-    if overwrite and os.path.exists(DEFAULT_DB_PATH):
-        logging.info(f"Removing existing database file: {DEFAULT_DB_PATH}")
-        os.remove(DEFAULT_DB_PATH)
-
-    if not os.path.exists(DEFAULT_DB_PATH):
-        dataset = download_dataset(DEFAULT_REPO_ID)
-        create_database(DEFAULT_DB_PATH)
-        populate_database(DEFAULT_DB_PATH, dataset)
-        create_indexes_and_triggers(DEFAULT_DB_PATH)
-
-        # ---- new: add unique index post-creation if not already handled by SQL_CREATE_INDEXES_TRIGGERS ---
-        # This ensures the index exists even if SQL_CREATE_INDEXES_TRIGGERS was modified
-        # or if we want to be absolutely certain this specific index is applied.
-        conn = sqlite3.connect(DEFAULT_DB_PATH)
-        cur = conn.cursor()
-        logging.info("Ensuring UNIQUE index on emails.message_id exists...")
-        cur.executescript(
-            """
-            CREATE UNIQUE INDEX IF NOT EXISTS idx_emails_message_id
-                ON emails(message_id);
-            """
-        )
-        conn.commit()
-        conn.close()
-        logging.info("UNIQUE index on emails.message_id verified/created.")
-        # ---- end new section ----
-    else:
-        logging.info(
-            f"Database already exists at {DEFAULT_DB_PATH}. Set overwrite=True to regenerate."
-        )
-
-    logging.info("Database generation process complete.")
-
-
-if __name__ == "__main__":
-    generate_database(overwrite=True)

synth_ai/environments/examples/enron/art_helpers/types_enron.py

@@ -1,25 +0,0 @@
-from typing import List, Optional
-
-from pydantic import BaseModel
-
-
-class SyntheticQuery(BaseModel):
-    id: int
-    question: str
-    answer: str
-    message_ids: List[str]  # message_ids (strings) of referenced emails
-    how_realistic: float
-    inbox_address: str
-    query_date: str
-
-
-class Email(BaseModel):
-    message_id: str
-    date: str  # ISO 8601 string 'YYYY-MM-DD HH:MM:SS'
-    subject: Optional[str] = None
-    from_address: Optional[str] = None
-    to_addresses: List[str] = []  # Populated from recipients table
-    cc_addresses: List[str] = []  # Populated from recipients table
-    bcc_addresses: List[str] = []  # Populated from recipients table
-    body: Optional[str] = None
-    file_name: Optional[str] = None