edsl 0.1.14__py3-none-any.whl → 0.1.40__py3-none-any.whl

edsl/scenarios/handlers/sql.py

@@ -0,0 +1,313 @@
+from edsl.scenarios.file_methods import FileMethods
+import tempfile
+import re
+from typing import List, Optional
+import textwrap
+
+
+class SqlMethods(FileMethods):
+    suffix = "sql"
+
+    def view_system(self):
+        import os
+        import subprocess
+
+        if os.path.exists(self.path):
+            try:
+                if (os_name := os.name) == "posix":
+                    subprocess.run(["open", self.path], check=True)  # macOS
+                elif os_name == "nt":
+                    os.startfile(self.path)  # Windows
+                else:
+                    subprocess.run(["xdg-open", self.path], check=True)  # Linux
+            except Exception as e:
+                print(f"Error opening SQL file: {e}")
+        else:
+            print("SQL file was not found.")
+
+    def view_notebook(self):
+        from IPython.display import FileLink, display, HTML
+        import pygments
+        from pygments.lexers import SqlLexer
+        from pygments.formatters import HtmlFormatter
+
+        try:
+            with open(self.path, "r", encoding="utf-8") as f:
+                content = f.read()
+
+            formatter = HtmlFormatter(style="monokai")
+            highlighted_sql = pygments.highlight(content, SqlLexer(), formatter)
+            css = formatter.get_style_defs(".highlight")
+            display(HTML(f"<style>{css}</style>{highlighted_sql}"))
+            display(FileLink(self.path))
+        except Exception as e:
+            print(f"Error displaying SQL: {e}")
+
+    def _format_keywords(self, sql: str) -> str:
+        """Capitalize SQL keywords."""
+        keywords = {
+            "select",
+            "from",
+            "where",
+            "and",
+            "or",
+            "insert",
+            "update",
+            "delete",
+            "create",
+            "drop",
+            "alter",
+            "table",
+            "into",
+            "values",
+            "group",
+            "by",
+            "having",
+            "order",
+            "limit",
+            "join",
+            "left",
+            "right",
+            "inner",
+            "outer",
+            "on",
+            "as",
+            "distinct",
+            "count",
+            "sum",
+            "avg",
+            "max",
+            "min",
+            "between",
+            "like",
+            "in",
+            "is",
+            "null",
+            "not",
+            "case",
+            "when",
+            "then",
+            "else",
+            "end",
+        }
+
+        words = sql.split()
+        formatted_words = []
+        for word in words:
+            lower_word = word.lower()
+            if lower_word in keywords:
+                formatted_words.append(word.upper())
+            else:
+                formatted_words.append(word.lower())
+        return " ".join(formatted_words)
+
+    def _indent_sql(self, sql: str) -> str:
+        """Add basic indentation to SQL statement."""
+        lines = sql.split("\n")
+        indented_lines = []
+        indent_level = 0
+
+        for line in lines:
+            line = line.strip()
+
+            # Decrease indent for closing parentheses
+            if line.startswith(")"):
+                indent_level = max(0, indent_level - 1)
+
+            # Add indentation
+            if line:
+                indented_lines.append("    " * indent_level + line)
+            else:
+                indented_lines.append("")
+
+            # Increase indent after opening parentheses
+            if line.endswith("("):
+                indent_level += 1
+
+            # Special cases for common SQL clauses
+            lower_line = line.lower()
+            if any(
+                clause in lower_line
+                for clause in [
+                    "select",
+                    "from",
+                    "where",
+                    "group by",
+                    "having",
+                    "order by",
+                ]
+            ):
+                indent_level = 1
+
+        return "\n".join(indented_lines)
+
+    def format_sql(self) -> bool:
+        """Format the SQL file with proper indentation and keyword capitalization."""
+        try:
+            with open(self.path, "r", encoding="utf-8") as f:
+                content = f.read()
+
+            # Remove extra whitespace and format
+            content = " ".join(content.split())
+            content = self._format_keywords(content)
+            content = self._indent_sql(content)
+
+            # Wrap long lines
+            wrapped_content = []
+            for line in content.split("\n"):
+                if len(line) > 80:
+                    wrapped_line = textwrap.fill(
+                        line, width=80, subsequent_indent="    "
+                    )
+                    wrapped_content.append(wrapped_line)
+                else:
+                    wrapped_content.append(line)
+
+            formatted_sql = "\n".join(wrapped_content)
+
+            with open(self.path, "w", encoding="utf-8") as f:
+                f.write(formatted_sql)
+
+            return True
+        except Exception as e:
+            print(f"Error formatting SQL: {e}")
+            return False
+
+    def split_statements(self) -> List[str]:
+        """Split the SQL file into individual statements."""
+        try:
+            with open(self.path, "r", encoding="utf-8") as f:
+                content = f.read()
+
+            # Handle both semicolon and GO statement terminators
+            statements = []
+            current_stmt = []
+
+            for line in content.split("\n"):
+                line = line.strip()
+
+                # Skip empty lines and comments
+                if not line or line.startswith("--"):
+                    continue
+
+                if line.endswith(";"):
+                    current_stmt.append(line[:-1])  # Remove semicolon
+                    statements.append(" ".join(current_stmt))
+                    current_stmt = []
+                elif line.upper() == "GO":
+                    if current_stmt:
+                        statements.append(" ".join(current_stmt))
+                        current_stmt = []
+                else:
+                    current_stmt.append(line)
+
+            # Add any remaining statement
+            if current_stmt:
+                statements.append(" ".join(current_stmt))
+
+            return [stmt.strip() for stmt in statements if stmt.strip()]
+        except Exception as e:
+            print(f"Error splitting SQL statements: {e}")
+            return []
+
+    def validate_basic_syntax(self) -> bool:
+        """
+        Perform basic SQL syntax validation.
+        This is a simple check and doesn't replace proper SQL parsing.
+        """
+        try:
+            with open(self.path, "r", encoding="utf-8") as f:
+                content = f.read()
+
+            statements = self.split_statements()
+            for stmt in statements:
+                # Check for basic SQL keywords
+                stmt_upper = stmt.upper()
+                if not any(
+                    keyword in stmt_upper
+                    for keyword in [
+                        "SELECT",
+                        "INSERT",
+                        "UPDATE",
+                        "DELETE",
+                        "CREATE",
+                        "DROP",
+                        "ALTER",
+                    ]
+                ):
+                    print(f"Warning: Statement might be incomplete: {stmt}")
+
+                # Check for basic parentheses matching
+                if stmt.count("(") != stmt.count(")"):
+                    print(f"Error: Unmatched parentheses in statement: {stmt}")
+                    return False
+
+                # Check for basic quote matching
+                if stmt.count("'") % 2 != 0:
+                    print(f"Error: Unmatched quotes in statement: {stmt}")
+                    return False
+
+            return True
+        except Exception as e:
+            print(f"Error validating SQL: {e}")
+            return False
+
+    def extract_table_names(self) -> List[str]:
+        """Extract table names from the SQL file."""
+        tables = set()
+        try:
+            with open(self.path, "r", encoding="utf-8") as f:
+                content = f.read()
+
+            patterns = [
+                r"FROM\s+([a-zA-Z_][a-zA-Z0-9_]*)",
+                r"JOIN\s+([a-zA-Z_][a-zA-Z0-9_]*)",
+                r"UPDATE\s+([a-zA-Z_][a-zA-Z0-9_]*)",
+                r"INSERT\s+INTO\s+([a-zA-Z_][a-zA-Z0-9_]*)",
+                r"CREATE\s+TABLE\s+([a-zA-Z_][a-zA-Z0-9_]*)",
+            ]
+
+            for pattern in patterns:
+                tables.update(re.findall(pattern, content, re.IGNORECASE))
+
+            return sorted(list(tables))
+        except Exception as e:
+            print(f"Error extracting table names: {e}")
+            return []
+
+    def example(self):
+        sample_sql = """-- Sample SQL file with common operations
+CREATE TABLE employees (
+    id INTEGER PRIMARY KEY,
+    name VARCHAR(100) NOT NULL,
+    department VARCHAR(50),
+    salary DECIMAL(10,2),
+    hire_date DATE
+);
+
+INSERT INTO employees (name, department, salary, hire_date)
+VALUES 
+    ('John Doe', 'Engineering', 75000.00, '2023-01-15'),
+    ('Jane Smith', 'Marketing', 65000.00, '2023-02-01');
+
+-- Query to analyze employee data
+SELECT 
+    department,
+    COUNT(*) as employee_count,
+    AVG(salary) as avg_salary
+FROM employees
+GROUP BY department
+HAVING COUNT(*) > 0
+ORDER BY avg_salary DESC;
+
+-- Update salary with conditions
+UPDATE employees
+SET salary = salary * 1.1
+WHERE department = 'Engineering'
+    AND hire_date < '2024-01-01';
+"""
+        with tempfile.NamedTemporaryFile(
+            delete=False, suffix=".sql", mode="w", encoding="utf-8"
+        ) as f:
+            f.write(sample_sql)
+        return f.name

edsl/scenarios/handlers/sqlite.py

@@ -0,0 +1,149 @@
+from edsl.scenarios.file_methods import FileMethods
+import os
+import tempfile
+import sqlite3
+
+
+class SQLiteMethods(FileMethods):
+    suffix = "db"  # or "sqlite", depending on your preference
+
+    def extract_text(self):
+        """
+        Extracts a text representation of the database schema and table contents.
+        """
+        with sqlite3.connect(self.path) as conn:
+            cursor = conn.cursor()
+
+            # Get all table names
+            cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+            tables = cursor.fetchall()
+
+            full_text = []
+
+            # For each table, get schema and contents
+            for (table_name,) in tables:
+                # Get table schema
+                cursor.execute(
+                    f"SELECT sql FROM sqlite_master WHERE type='table' AND name='{table_name}';"
+                )
+                schema = cursor.fetchone()[0]
+                full_text.append(f"Table: {table_name}")
+                full_text.append(f"Schema: {schema}")
+
+                # Get table contents
+                cursor.execute(f"SELECT * FROM {table_name};")
+                rows = cursor.fetchall()
+
+                # Get column names
+                column_names = [description[0] for description in cursor.description]
+                full_text.append(f"Columns: {', '.join(column_names)}")
+
+                # Add row data
+                for row in rows:
+                    full_text.append(str(row))
+                full_text.append("\n")
+
+        return "\n".join(full_text)
+
+    def view_system(self):
+        """
+        Opens the database with the system's default SQLite viewer if available.
+        """
+        import os
+        import subprocess
+
+        if os.path.exists(self.path):
+            try:
+                if (os_name := os.name) == "posix":
+                    # Try DB Browser for SQLite on macOS
+                    subprocess.run(
+                        ["open", "-a", "DB Browser for SQLite", self.path], check=True
+                    )
+                elif os_name == "nt":
+                    # Try DB Browser for SQLite on Windows
+                    subprocess.run(["DB Browser for SQLite.exe", self.path], check=True)
+                else:
+                    # Try sqlitebrowser on Linux
+                    subprocess.run(["sqlitebrowser", self.path], check=True)
+            except Exception as e:
+                print(f"Error opening SQLite database: {e}")
+        else:
+            print("SQLite database file was not found.")
+
+    def view_notebook(self):
+        """
+        Displays database contents in a Jupyter notebook.
+        """
+        import pandas as pd
+        from IPython.display import HTML, display
+
+        with sqlite3.connect(self.path) as conn:
+            # Get all table names
+            cursor = conn.cursor()
+            cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+            tables = cursor.fetchall()
+
+            html_parts = []
+            for (table_name,) in tables:
+                # Read table into pandas DataFrame
+                df = pd.read_sql_query(f"SELECT * FROM {table_name}", conn)
+
+                # Convert to HTML with styling
+                table_html = f"""
+                <div style="margin-bottom: 20px;">
+                    <h3>{table_name}</h3>
+                    {df.to_html(index=False)}
+                </div>
+                """
+                html_parts.append(table_html)
+
+            # Combine all tables into one scrollable div
+            html = f"""
+            <div style="width: 800px; height: 800px; padding: 20px; 
+                       border: 1px solid #ccc; overflow-y: auto;">
+                {''.join(html_parts)}
+            </div>
+            """
+            display(HTML(html))
+
+    def example(self):
+        """
+        Creates an example SQLite database for testing.
+        """
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as tmp:
+            conn = sqlite3.connect(tmp.name)
+            cursor = conn.cursor()
+
+            # Create a sample table
+            cursor.execute(
+                """
+                CREATE TABLE survey_responses (
+                    id INTEGER PRIMARY KEY,
+                    question TEXT,
+                    response TEXT
+                )
+            """
+            )
+
+            # Insert some sample data
+            sample_data = [
+                (1, "First Survey Question", "Response 1"),
+                (2, "Second Survey Question", "Response 2"),
+            ]
+            cursor.executemany(
+                "INSERT INTO survey_responses (id, question, response) VALUES (?, ?, ?)",
+                sample_data,
+            )
+
+            conn.commit()
+            conn.close()
+            tmp.close()
+
+            return tmp.name
+
+
+if __name__ == "__main__":
+    sqlite_temp = SQLiteMethods.example()
+    from edsl.scenarios.FileStore import FileStore
+
+    fs = FileStore(sqlite_temp)

edsl/scenarios/handlers/txt.py

@@ -0,0 +1,33 @@
+from edsl.scenarios.file_methods import FileMethods
+import tempfile
+
+
+class TxtMethods(FileMethods):
+    suffix = "txt"
+
+    def view_system(self):
+        import os
+        import subprocess
+
+        if os.path.exists(self.path):
+            try:
+                if (os_name := os.name) == "posix":
+                    subprocess.run(["open", self.path], check=True)  # macOS
+                elif os_name == "nt":
+                    os.startfile(self.path)  # Windows
+                else:
+                    subprocess.run(["xdg-open", self.path], check=True)  # Linux
+            except Exception as e:
+                print(f"Error opening TXT: {e}")
+        else:
+            print("TXT file was not found.")
+
+    def view_notebook(self):
+        from IPython.display import FileLink, display
+
+        display(FileLink(self.path))
+
+    def example(self):
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f:
+            f.write(b"Hello, World!")
+        return f.name

edsl/scenarios/scenario_join.py

@@ -0,0 +1,131 @@
+from __future__ import annotations
+from typing import Union, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from edsl.scenarios.ScenarioList import ScenarioList
+    from edsl.scenarios.Scenario import Scenario
+
+
+class ScenarioJoin:
+    """Handles join operations between two ScenarioLists.
+
+    This class encapsulates all join-related logic, making it easier to maintain
+    and extend with other join types (inner, right, full) in the future.
+    """
+
+    def __init__(self, left: "ScenarioList", right: "ScenarioList"):
+        """Initialize join operation with two ScenarioLists.
+
+        Args:
+            left: The left ScenarioList
+            right: The right ScenarioList
+        """
+        self.left = left
+        self.right = right
+
+    def left_join(self, by: Union[str, list[str]]) -> "ScenarioList":
+        """Perform a left join between the two ScenarioLists.
+
+        Args:
+            by: String or list of strings representing the key(s) to join on. Cannot be empty.
+
+        Returns:
+            A new ScenarioList containing the joined scenarios
+
+        Raises:
+            ValueError: If by is empty or if any join keys don't exist in both ScenarioLists
+        """
+        from edsl.scenarios.ScenarioList import ScenarioList
+
+        self._validate_join_keys(by)
+        by_keys = [by] if isinstance(by, str) else by
+
+        other_dict = self._create_lookup_dict(self.right, by_keys)
+        all_keys = self._get_all_keys()
+
+        return ScenarioList(
+            self._create_joined_scenarios(by_keys, other_dict, all_keys)
+        )
+
+    def _validate_join_keys(self, by: Union[str, list[str]]) -> None:
+        """Validate join keys exist in both ScenarioLists."""
+        if not by:
+            raise ValueError(
+                "Join keys cannot be empty. Please specify at least one key to join on."
+            )
+
+        by_keys = [by] if isinstance(by, str) else by
+        left_keys = set(next(iter(self.left)).keys()) if self.left else set()
+        right_keys = set(next(iter(self.right)).keys()) if self.right else set()
+
+        missing_left = set(by_keys) - left_keys
+        missing_right = set(by_keys) - right_keys
+        if missing_left or missing_right:
+            missing = missing_left | missing_right
+            raise ValueError(f"Join key(s) {missing} not found in both ScenarioLists")
+
+    @staticmethod
+    def _get_key_tuple(scenario: Scenario, keys: list[str]) -> tuple:
+        """Create a tuple of values for the join keys."""
+        return tuple(scenario[k] for k in keys)
+
+    def _create_lookup_dict(self, scenarios: ScenarioList, by_keys: list[str]) -> dict:
+        """Create a lookup dictionary for the right scenarios."""
+        return {
+            self._get_key_tuple(scenario, by_keys): scenario for scenario in scenarios
+        }
+
+    def _get_all_keys(self) -> set:
+        """Get all unique keys from both ScenarioLists."""
+        all_keys = set()
+        for scenario in self.left:
+            all_keys.update(scenario.keys())
+        for scenario in self.right:
+            all_keys.update(scenario.keys())
+        return all_keys
+
+    def _create_joined_scenarios(
+        self, by_keys: list[str], other_dict: dict, all_keys: set
+    ) -> list[Scenario]:
+        """Create the joined scenarios."""
+        from edsl.scenarios.Scenario import Scenario
+
+        new_scenarios = []
+
+        for scenario in self.left:
+            new_scenario = {key: None for key in all_keys}
+            new_scenario.update(scenario)
+
+            key_tuple = self._get_key_tuple(scenario, by_keys)
+            if matching_scenario := other_dict.get(key_tuple):
+                self._handle_matching_scenario(
+                    new_scenario, scenario, matching_scenario, by_keys
+                )
+
+            new_scenarios.append(Scenario(new_scenario))
+
+        return new_scenarios
+
+    def _handle_matching_scenario(
+        self,
+        new_scenario: dict,
+        left_scenario: "Scenario",
+        right_scenario: "Scenario",
+        by_keys: list[str],
+    ) -> None:
+        """Handle merging of matching scenarios and conflict warnings."""
+        overlapping_keys = set(left_scenario.keys()) & set(right_scenario.keys())
+
+        for key in overlapping_keys:
+            if key not in by_keys and left_scenario[key] != right_scenario[key]:
+                join_conditions = [f"{k}='{left_scenario[k]}'" for k in by_keys]
+                print(
+                    f"Warning: Conflicting values for key '{key}' where "
+                    f"{' AND '.join(join_conditions)}. "
+                    f"Keeping left value: {left_scenario[key]} "
+                    f"(discarding: {right_scenario[key]})"
+                )
+
+        # Only update with non-overlapping keys from matching scenario
+        new_keys = set(right_scenario.keys()) - set(left_scenario.keys())
+        new_scenario.update({k: right_scenario[k] for k in new_keys})