lam-cli 0.1.4__tar.gz → 0.1.7__tar.gz

{lam_cli-0.1.4/lam_cli.egg-info → lam_cli-0.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lam-cli
-Version: 0.1.4
+Version: 0.1.7
 Summary: Secure data transformation tool supporting JQ and JavaScript (Bun)
 Home-page: https://github.com/laminar-run/lam
 Author: Laminar Run, Inc.

{lam_cli-0.1.4 → lam_cli-0.1.7}/lam/lam.py

@@ -3,6 +3,7 @@
 import json
 import logging
 import os
+import re
 import shutil
 import socket
 import subprocess
@@ -87,6 +88,7 @@ class Stats:
 class EngineType(Enum):
     JQ = "jq"
     JAVASCRIPT = "js"
+    PYTHON = "py"
 
 class ProcessingError(Exception):
     """Custom exception for processing errors"""
@@ -441,11 +443,402 @@ class BunEngine(Engine):
                 "type": e.__class__.__name__
             }, str(e)
 
+class PythonEngine(Engine):
+    """Python execution engine with improved sandboxing for security"""
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.modules_dir = Path(tempfile.gettempdir()) / "lam_python_modules"
+        self.modules_dir.mkdir(exist_ok=True)
+        # Define allowed modules that can be safely imported
+        self.allowed_modules = {
+            "json", "datetime", "math", "statistics", "collections", 
+            "itertools", "functools", "re", "copy", "decimal",
+            "csv", "io", "dataclasses", "typing", "enum"
+        }
+        
+    def validate_environment(self) -> bool:
+        logger.debug("Validating Python environment")
+        return sys.executable is not None
+        
+    def create_safe_globals(self) -> dict:
+        """Create a restricted globals dictionary for safer execution"""
+        safe_globals = {
+            "__builtins__": {
+                # Safe builtins only
+                "abs": abs, "all": all, "any": any, "bool": bool,
+                "chr": chr, "dict": dict, "dir": dir, "divmod": divmod,
+                "enumerate": enumerate, "filter": filter, "float": float,
+                "format": format, "frozenset": frozenset, "hash": hash,
+                "hex": hex, "int": int, "isinstance": isinstance,
+                "issubclass": issubclass, "iter": iter, "len": len,
+                "list": list, "map": map, "max": max, "min": min,
+                "next": next, "oct": oct, "ord": ord, "pow": pow,
+                "print": print, "range": range, "repr": repr,
+                "reversed": reversed, "round": round, "set": set,
+                "slice": slice, "sorted": sorted, "str": str, "sum": sum,
+                "tuple": tuple, "type": type, "zip": zip,
+                # Add Exception types for error handling
+                "Exception": Exception, "ValueError": ValueError,
+                "TypeError": TypeError, "KeyError": KeyError,
+                "IndexError": IndexError
+            },
+            # Pre-import safe modules
+            "json": json,
+            "datetime": datetime,
+            "math": __import__("math"),
+            "statistics": __import__("statistics"),
+            "collections": __import__("collections"),
+            "itertools": __import__("itertools"),
+            "functools": __import__("functools"),
+            "re": __import__("re")
+        }
+        return safe_globals
+
+    def check_for_dangerous_code(self, code: str) -> Optional[str]:
+        """Check for potentially dangerous patterns in the code"""
+        dangerous_patterns = [
+            (r"__import__\s*\(", "Use of __import__ is not allowed"),
+            (r"eval\s*\(", "Use of eval() is not allowed"),
+            (r"exec\s*\(", "Use of exec() is not allowed"),
+            (r"globals\s*\(", "Access to globals() is not allowed"),
+            (r"locals\s*\(", "Access to locals() is not allowed"),
+            (r"getattr\s*\(", "Use of getattr() is not allowed"),
+            (r"setattr\s*\(", "Use of setattr() is not allowed"),
+            (r"delattr\s*\(", "Use of delattr() is not allowed"),
+            (r"compile\s*\(", "Use of compile() is not allowed"),
+            (r"open\s*\(", "Use of open() is not allowed"),
+            (r"__subclasses__", "Access to __subclasses__ is not allowed"),
+            (r"subprocess", "Access to subprocess module is not allowed"),
+            (r"sys\.", "Access to sys module is not allowed"),
+            (r"os\.", "Access to os module is not allowed"),
+            (r"shutil", "Access to shutil module is not allowed"),
+            (r"pathlib", "Access to pathlib module is not allowed"),
+            (r"importlib", "Access to importlib module is not allowed"),
+            (r"builtins", "Access to builtins module is not allowed"),
+            (r"_thread", "Access to _thread module is not allowed"),
+            (r"ctypes", "Access to ctypes module is not allowed"),
+            (r"socket", "Access to socket module is not allowed"),
+            (r"pickle", "Access to pickle module is not allowed"),
+            (r"multiprocessing", "Access to multiprocessing module is not allowed"),
+            (r"__\w+__", "Access to dunder attributes may not be allowed")
+        ]
+        
+        for pattern, message in dangerous_patterns:
+            if re.search(pattern, code):
+                return message
+        
+        # Check for imports outside of allowed modules
+        import_pattern = r"import\s+(\w+)|from\s+(\w+)\s+import"
+        for match in re.finditer(import_pattern, code):
+            module = match.group(1) or match.group(2)
+            if module and module not in self.allowed_modules:
+                return f"Import of '{module}' is not allowed, only these modules are permitted: {', '.join(sorted(self.allowed_modules))}"
+        
+        return None
+
+    def create_wrapper(self, input_data: str, user_script: str) -> str:
+        """Create the wrapper script with proper escaping and sandboxing"""
+        # Perform safety checks before creating wrapper
+        safety_issue = self.check_for_dangerous_code(user_script)
+        if safety_issue:
+            # Return a wrapper that will immediately exit with the safety error
+            return f'''
+import json
+import sys
+
+sys.stdout.write(json.dumps({{
+    "error": "Security violation detected: {safety_issue}",
+    "stack": []
+}}))
+sys.exit(1)
+'''
+
+        return f'''
+import json
+import sys
+import traceback
+from datetime import datetime
+import re
+import math
+import statistics
+import collections
+import itertools
+import functools
+
+# Resource limiting
+import resource
+import signal
+
+# Set resource limits
+def set_resource_limits():
+    # 5 seconds CPU time
+    resource.setrlimit(resource.RLIMIT_CPU, (5, 5))
+    
+    # 100MB memory limit
+    memory_limit = 100 * 1024 * 1024  # 100MB in bytes
+    resource.setrlimit(resource.RLIMIT_AS, (memory_limit, memory_limit))
+    
+    # Set timeout handler
+    def timeout_handler(signum, frame):
+        sys.stderr.write(json.dumps({{
+            "error": "Execution timed out (5 seconds)",
+            "stack": []
+        }}))
+        sys.exit(1)
+    
+    signal.signal(signal.SIGALRM, timeout_handler)
+    signal.alarm(5)  # 5 second timeout
+
+try:
+    set_resource_limits()
+except Exception as e:
+    # Continue if resource limiting is not available (e.g., on Windows)
+    pass
+
+# Setup basic logging
+logs = []
+
+class LogCapture:
+    def __init__(self, log_type):
+        self.log_type = log_type
+        
+    def write(self, message):
+        if message.strip():
+            logs.append({{"type": self.log_type, "message": message.strip()}})
+        return len(message)
+        
+    def flush(self):
+        pass
+
+# Custom safer importer
+class RestrictedImporter:
+    def __init__(self, allowed_modules):
+        self.allowed_modules = allowed_modules
+        
+    def __call__(self, name, *args, **kwargs):
+        if name in self.allowed_modules:
+            return __import__(name, *args, **kwargs)
+        else:
+            raise ImportError(f"Import of '{{name}}' is not allowed for security reasons. " +
+                             f"Allowed modules: {{', '.join(sorted(self.allowed_modules))}}")
+
+# Capture stdout and stderr
+original_stdout = sys.stdout
+original_stderr = sys.stderr
+sys.stdout = LogCapture("log")
+sys.stderr = LogCapture("error")
+
+# Parse input data
+try:
+    input_data = json.loads(r"""{input_data}""")
+except json.JSONDecodeError as e:
+    original_stderr.write(json.dumps({{"error": f"Failed to parse input data: {{e}}"}}))
+    sys.exit(1)
+
+# Create safe environment
+safe_globals = {{
+    "__builtins__": {{
+        # Safe builtins only
+        "abs": abs, "all": all, "any": any, "bool": bool,
+        "chr": chr, "dict": dict, "divmod": divmod,
+        "enumerate": enumerate, "filter": filter, "float": float,
+        "format": format, "frozenset": frozenset, "hash": hash,
+        "hex": hex, "int": int, "isinstance": isinstance,
+        "issubclass": issubclass, "iter": iter, "len": len,
+        "list": list, "map": map, "max": max, "min": min,
+        "next": next, "oct": oct, "ord": ord, "pow": pow,
+        "print": print, "range": range, "repr": repr,
+        "reversed": reversed, "round": round, "set": set,
+        "slice": slice, "sorted": sorted, "str": str, "sum": sum,
+        "tuple": tuple, "type": type, "zip": zip,
+        # Exception types for error handling
+        "Exception": Exception, "ValueError": ValueError,
+        "TypeError": TypeError, "KeyError": KeyError,
+        "IndexError": IndexError,
+        # Add a safe import function
+        "__import__": RestrictedImporter({{
+            "json", "datetime", "math", "statistics", "collections", 
+            "itertools", "functools", "re", "copy", "decimal",
+            "csv", "io", "dataclasses", "typing", "enum"
+        }})
+    }},
+    # Pre-import safe modules
+    "json": json,
+    "datetime": datetime,
+    "math": math,
+    "statistics": statistics,
+    "collections": collections,
+    "itertools": itertools,
+    "functools": functools,
+    "re": re
+}}
+
+safe_locals = {{"input_data": input_data}}
+
+# Define transform function from user script in a safe context
+try:
+    compiled_code = compile(r"""{user_script}""", "<user_script>", "exec")
+    exec(compiled_code, safe_globals, safe_locals)
+    
+    # Validate transform function exists and has correct signature
+    if 'transform' not in safe_locals:
+        original_stderr.write(json.dumps({{"error": "No transform function defined"}}))
+        sys.exit(1)
+        
+    if not callable(safe_locals['transform']):
+        original_stderr.write(json.dumps({{"error": "transform must be a function"}}))
+        sys.exit(1)
+        
+    transform_fn = safe_locals['transform']
+    
+except Exception as e:
+    original_stderr.write(json.dumps({{
+        "error": str(e),
+        "stack": traceback.format_exc().split('\\n')
+    }}))
+    sys.exit(1)
+
+# Execute transform with input data
+try:
+    # Cancel the alarm if we reach here (we have our own timeout)
+    try:
+        signal.alarm(0)
+    except:
+        pass
+        
+    result = transform_fn(input_data)
+    
+    # Basic validation of output (to prevent non-serializable data)
+    try:
+        json.dumps(result)
+    except TypeError as e:
+        raise TypeError(f"Transform result is not JSON serializable: {{e}}")
+    
+    # Write result to original stdout
+    original_stdout.write(json.dumps({{"result": result, "logs": logs}}))
+    
+except Exception as e:
+    original_stderr.write(json.dumps({{
+        "error": str(e),
+        "stack": traceback.format_exc().split('\\n')
+    }}))
+    sys.exit(1)
+finally:
+    # Restore stdout and stderr
+    sys.stdout = original_stdout
+    sys.stderr = original_stderr
+'''
+
+    def execute(self, program_file: str, input_data: str) -> Tuple[Union[Dict, str], Optional[str]]:
+        logger.info(f"Executing Python script: {program_file}")
+        stats = Stats()
+
+        try:
+            check_resource_limits(self.modules_dir)
+
+            with tempfile.TemporaryDirectory() as temp_dir:
+                temp_dir = Path(temp_dir)
+                
+                # Read user script
+                with open(program_file, 'r') as f:
+                    user_script = f.read()
+                    logger.debug("Loaded user Python script: %d characters", len(user_script))
+                
+                # Check for dangerous code
+                safety_issue = self.check_for_dangerous_code(user_script)
+                if safety_issue:
+                    logger.warning(f"Security violation detected in script: {safety_issue}")
+                    return {
+                        "lam.error": f"Security violation: {safety_issue}",
+                        "type": "SecurityError"
+                    }, f"Security violation: {safety_issue}"
+
+                # Create wrapper script
+                wrapper = self.create_wrapper(input_data, user_script)
+                script_path = temp_dir / "script.py"
+                with open(script_path, 'w') as f:
+                    f.write(wrapper)
+                logger.debug("Generated Python wrapper script: %s", script_path)
+
+                # Execute with Python in isolated environment
+                process = subprocess.Popen(
+                    [
+                        sys.executable,
+                        "-I",  # Isolated mode, ignores environment variables/site packages
+                        str(script_path)
+                    ],
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                    text=True,
+                    cwd=temp_dir,
+                    # Prevent access to system environment variables
+                    env={"PATH": os.environ.get("PATH", "")}
+                )
+                logger.info("Started Python process PID %d", process.pid)
+
+                try:
+                    output, error = process.communicate(timeout=5)
+                    logger.debug("Process completed with code %d", process.returncode)
+                except subprocess.TimeoutExpired as e:
+                    logger.warning("Process timeout after 5 seconds")
+                    process.kill()
+                    return {"lam.error": "Script execution timed out"}, "Execution timed out after 5 seconds"
+
+                # Handle process errors
+                if process.returncode != 0:
+                    try:
+                        # Try to parse structured error from stderr
+                        error_data = json.loads(error.strip())
+                        error_msg = error_data.get('error', 'Unknown error')
+                        stack = error_data.get('stack', [])
+                        
+                        # Format error message
+                        error_details = {
+                            "lam.error": error_msg,
+                            "stack_trace": stack
+                        }
+                        return error_details, error_msg
+                        
+                    except json.JSONDecodeError:
+                        # Fallback to raw error output
+                        error_msg = error.strip() or "Unknown error"
+                        return {"lam.error": error_msg}, error_msg
+
+                # Handle successful output
+                try:
+                    output_data = json.loads(output)
+                    
+                    # Process Python logs (if any)
+                    if 'logs' in output_data:
+                        for log_entry in output_data.get('logs', []):
+                            if log_entry['type'] == 'error':
+                                logger.error("[Python] %s", log_entry['message'])
+                            else:
+                                logger.debug("[Python] %s", log_entry['message'])
+                    
+                    result = output_data.get('result', {})
+                    return result, None
+
+                except json.JSONDecodeError as e:
+                    logger.error("Failed to parse output: %s", str(e))
+                    return {
+                        "lam.error": "Invalid JSON output",
+                        "raw_output": output.strip()
+                    }, "Output format error"
+
+        except Exception as e:
+            logger.exception("Execution failed")
+            return {
+                "lam.error": str(e),
+                "type": e.__class__.__name__
+            }, str(e)
+
 def get_engine(engine_type: str, workspace_id: str, flow_id: str, execution_id: str) -> Engine:
     """Factory function to get the appropriate execution engine"""
     engines = {
         EngineType.JQ.value: JQEngine,
-        EngineType.JAVASCRIPT.value: BunEngine
+        EngineType.JAVASCRIPT.value: BunEngine,
+        EngineType.PYTHON.value: PythonEngine
     }
     
     engine_class = engines.get(engine_type)
@@ -478,10 +871,45 @@ def lam():
     """LAM - Laminar Data Transformation Tool"""
     pass
 
+@lam.command()
+def initialize():
+    """Initialize shared modules for supported engines."""
+    click.echo("Starting LAM initialization...")
+    
+    engine_classes = [BunEngine, PythonEngine, JQEngine] # add other engines so we dont miss them in the future
+
+    # Define placeholder IDs for engine instantiation during initialization
+    init_workspace_id = "lam_init_workspace"
+    init_flow_id = "lam_init_flow"
+    init_execution_id = "lam_init_execution"
+
+    for engine_class in engine_classes:
+        engine_name = engine_class.__name__
+        click.echo(f"Checking {engine_name} for shared module setup...")
+        try:
+            # Instantiate engine to access instance methods like _setup_shared_modules
+            engine_instance = engine_class(
+                workspace_id=init_workspace_id, 
+                flow_id=init_flow_id, 
+                execution_id=init_execution_id
+            )
+            
+            if hasattr(engine_instance, '_setup_shared_modules') and callable(getattr(engine_instance, '_setup_shared_modules')):
+                click.echo(f"Running _setup_shared_modules for {engine_name}...")
+                getattr(engine_instance, '_setup_shared_modules')()
+                click.echo(f"Successfully initialized shared modules for {engine_name}.")
+            else:
+                click.echo(f"{engine_name} does not have a _setup_shared_modules method or it's not callable.")
+        except Exception as e:
+            click.echo(f"Error during initialization of {engine_name}: {e}", err=True)
+            logger.error(f"Initialization error for {engine_name}", exc_info=True)
+
+    click.echo("LAM initialization complete.")
+
 @lam.command()
 @click.argument('program_file', type=click.Path(exists=True))
 @click.argument('input', type=str)
-@click.option('--language', type=click.Choice(['jq', 'js']), default='jq',
+@click.option('--language', type=click.Choice(['jq', 'js', 'py']), default='jq',
               help='Script language (default: jq)')
 @click.option('--workspace_id', default="local", help="Workspace ID")
 @click.option('--flow_id', default="local", help="Flow ID")

{lam_cli-0.1.4 → lam_cli-0.1.7/lam_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lam-cli
-Version: 0.1.4
+Version: 0.1.7
 Summary: Secure data transformation tool supporting JQ and JavaScript (Bun)
 Home-page: https://github.com/laminar-run/lam
 Author: Laminar Run, Inc.

{lam_cli-0.1.4 → lam_cli-0.1.7}/setup.py

@@ -8,7 +8,7 @@ long_description = docs_path.read_text() if docs_path.exists() else ""
 
 setup(
     name="lam-cli",
-    version="0.1.4",
+    version="0.1.7",
     packages=find_packages(),
     install_requires=[
         "backoff>=2.2.1",