setta 0.0.14.dev2__py3-none-any.whl → 0.0.14.dev3__py3-none-any.whl

setta/__init__.py

@@ -1 +1 @@
-__version__ = "0.0.14.dev2"
+__version__ = "0.0.14.dev3"

setta/tasks/tasks.py

@@ -151,6 +151,8 @@ class Tasks:
             sp = self.in_memory_subprocesses.get(subprocess_key, {}).get("subprocess")
             if sp:
                 sp.close()
+                del self.in_memory_subprocesses[subprocess_key]
+
             sp = SettaInMemoryFnSubprocess(
                 self.stop_event, self.websockets, c["subprocessStartMethod"]
             )

setta/tasks/utils.py

@@ -7,6 +7,7 @@ import sys
 import threading
 import traceback
 import uuid
+from collections import defaultdict
 
 from setta.tasks.fns.utils import TaskDefinition
 from setta.utils.constants import CWD
@@ -15,35 +16,6 @@ from setta.utils.utils import nested_access
 logger = logging.getLogger(__name__)
 
 
-def import_code_from_string(code_string, module_name=None, add_to_sys_modules=True):
-    # Generate a unique module name if one isn't provided
-    if module_name is None:
-        module_name = f"setta_dynamic_module_{uuid.uuid4().hex}"
-
-    # Add current directory to sys.path if it's not already there
-    current_dir = str(CWD)
-    if current_dir not in sys.path:
-        sys.path.insert(0, current_dir)
-
-    spec = importlib.util.spec_from_loader(module_name, loader=None)
-
-    # Create a new module based on the spec
-    module = importlib.util.module_from_spec(spec)
-
-    # Optionally add the module to sys.modules
-    if add_to_sys_modules:
-        print(f"adding {module_name} to sys.modules", flush=True)
-        sys.modules[module_name] = module
-
-    # Compile the code string
-    code_object = compile(code_string, module_name, "exec")
-
-    # Execute the compiled code object in the module's namespace
-    exec(code_object, module.__dict__)
-
-    return module
-
-
 class SettaInMemoryFnSubprocess:
     def __init__(self, stop_event, websockets, start_method):
         logger.debug(
@@ -68,20 +40,20 @@ class SettaInMemoryFnSubprocess:
             self.start_stdout_processor_task()
 
     def _subprocess_main(self):
-        """Main loop in subprocess that handles all requests"""
+        """Main loop in subprocess that handles all requests with parallel function execution"""
         # Initialize store for imported modules
         fns_dict = {}
         cache = {}
 
-        class OutputCapture:
-            def __init__(self, stdout_pipe):
-                self.stdout_pipe = stdout_pipe
+        # Message queues per function
+        fn_message_queues = defaultdict(queue.Queue)
 
-            def write(self, text):
-                self.stdout_pipe.send(text)
+        # Create a lock for thread-safe operations
+        lock = threading.RLock()
+        send_lock = threading.Lock()
 
-            def flush(self):
-                pass
+        # Function worker threads
+        fn_workers = {}
 
         # Redirect stdout as soon as subprocess starts
         output_capture = OutputCapture(self.stdout_child_conn)
@@ -91,9 +63,16 @@ class SettaInMemoryFnSubprocess:
         while True:
             msg = self.child_conn.recv()  # Wait for requests
             msg_type = msg["type"]
-            return_message_type = None
 
             if msg_type == "shutdown":
+                # Signal all worker threads to stop
+                for fn_name in fn_workers:
+                    fn_message_queues[fn_name].put(None)
+
+                # Wait for all worker threads to finish (with timeout)
+                for fn_name, worker in fn_workers.items():
+                    worker.join(timeout=1.0)
+
                 break
 
             try:
@@ -104,108 +83,217 @@ class SettaInMemoryFnSubprocess:
                         module_name = to_import["module_name"]
                         # Import and store module
                         module = import_code_from_string(code, module_name)
-                        added_fn_names = add_fns_from_module(
-                            fns_dict, module, module_name
+                        with lock:
+                            added_fn_names = add_fns_from_module(
+                                fns_dict, module, module_name
+                            )
+                            for k in added_fn_names:
+                                cache[k] = msg["exporter_obj"]
+                                dependencies[k] = get_task_metadata(
+                                    fns_dict[k], cache[k]
+                                )
+                                # Start a worker thread for each function
+                                self._start_worker_for_fn(
+                                    k,
+                                    fn_workers,
+                                    fn_message_queues,
+                                    fns_dict,
+                                    cache,
+                                    lock,
+                                    send_lock,
+                                    self.child_conn,
+                                )
+
+                    with send_lock:
+                        self.child_conn.send(
+                            {
+                                "status": "success",
+                                "content": dependencies,
+                            }
                         )
-                        for k in added_fn_names:
-                            cache[k] = msg["exporter_obj"]
-                            dependencies[k] = get_task_metadata(fns_dict[k], cache[k])
 
-                    self.child_conn.send(
-                        {
-                            "status": "success",
-                            "content": dependencies,
-                        }
+                elif msg_type == "call" or msg_type == "call_with_new_exporter_obj":
+                    fn_name = msg["fn_name"]
+
+                    # Start a worker for this function if needed
+                    self._start_worker_for_fn(
+                        fn_name,
+                        fn_workers,
+                        fn_message_queues,
+                        fns_dict,
+                        cache,
+                        lock,
+                        send_lock,
+                        self.child_conn,
                     )
 
-                elif msg_type == "call":
-                    result, return_message_type = self.call_imported_fn(
-                        msg, fns_dict, cache
-                    )
-                    self.child_conn.send(
-                        {
-                            "status": "success",
-                            "content": result,
-                            "messageType": return_message_type,
-                        }
-                    )
+                    # Add the message to the function's queue
+                    fn_message_queues[fn_name].put(msg)
 
-                elif msg_type == "call_with_new_exporter_obj":
-                    # replace old exporter_obj
-                    cache[msg["fn_name"]] = msg["other_data"]["exporter_obj"]
-                    result, return_message_type = self.call_imported_fn(
-                        msg, fns_dict, cache
-                    )
+            except Exception as e:
+                traceback.print_exc()
+                with send_lock:
                     self.child_conn.send(
                         {
-                            "status": "success",
-                            "content": result,
-                            "messageType": return_message_type,
+                            "status": "error",
+                            "error": str(e),
+                            "messageType": None,
                         }
                     )
 
+    def _worker_thread(
+        self, fn_name, fn_message_queues, fns_dict, cache, lock, send_lock, child_conn
+    ):
+        """Worker thread that processes messages for a specific function"""
+        while True:
+            try:
+                # Get a message from the queue
+                msg = fn_message_queues[fn_name].get()
+
+                if msg is None:  # Sentinel value to stop the thread
+                    break
+
+                msg_type = msg["type"]
+                return_message_type = None
+
+                if msg_type == "call" or msg_type == "call_with_new_exporter_obj":
+                    try:
+                        # Handle updating exporter_obj for call_with_new_exporter_obj
+                        if msg_type == "call_with_new_exporter_obj":
+                            with lock:
+                                cache[fn_name] = msg["other_data"]["exporter_obj"]
+
+                        # Get a thread-safe copy of what we need
+                        with lock:
+                            in_memory_fn_obj = fns_dict[fn_name]
+                            exporter_obj = cache.get(fn_name)
+
+                        # Process message
+                        message_content = process_message(msg["message"], exporter_obj)
+
+                        # Call function
+                        result = in_memory_fn_obj.fn(message_content)
+                        return_message_type = in_memory_fn_obj.return_message_type
+
+                        # Send result back
+                        with send_lock:
+                            child_conn.send(
+                                {
+                                    "status": "success",
+                                    "content": result,
+                                    "messageType": return_message_type,
+                                }
+                            )
+                    except Exception as e:
+                        traceback.print_exc()
+                        with send_lock:
+                            child_conn.send(
+                                {
+                                    "status": "error",
+                                    "error": str(e),
+                                    "messageType": return_message_type,
+                                }
+                            )
+
+                # Mark task as done
+                fn_message_queues[fn_name].task_done()
+
             except Exception as e:
                 traceback.print_exc()
-                self.child_conn.send(
-                    {
-                        "status": "error",
-                        "error": str(e),
-                        "messageType": return_message_type,
-                    }
-                )
-
-    def call_imported_fn(self, msg, fns_dict, cache):
-        fn_name = msg["fn_name"]
-        message = self.process_message(fn_name, msg["message"], cache)
-        fn = fns_dict[fn_name]
-        result = fn.fn(message)
-        return_message_type = fn.return_message_type
-        return result, return_message_type
+                print(f"Error in worker thread for {fn_name}: {e}", flush=True)
+
+    def _start_worker_for_fn(
+        self,
+        fn_name,
+        fn_workers,
+        fn_message_queues,
+        fns_dict,
+        cache,
+        lock,
+        send_lock,
+        child_conn,
+    ):
+        """Start a worker thread for a function if not already running"""
+        if fn_name not in fn_workers or not fn_workers[fn_name].is_alive():
+            worker = threading.Thread(
+                target=self._worker_thread,
+                args=(
+                    fn_name,
+                    fn_message_queues,
+                    fns_dict,
+                    cache,
+                    lock,
+                    send_lock,
+                    child_conn,
+                ),
+                daemon=True,
+                name=f"worker-{fn_name}",
+            )
+            fn_workers[fn_name] = worker
+            worker.start()
 
     def close(self):
         try:
             logger.debug("Initiating shutdown sequence")
-            self.parent_conn.send({"type": "shutdown"})
-            self.process.join(timeout=2)  # Add timeout to process join
 
+            # Set our stop event - this signals all tasks to stop
+            self.stop_event.set()
+
+            # Send shutdown message to the subprocess
+            try:
+                self.parent_conn.send({"type": "shutdown"})
+            except (BrokenPipeError, EOFError):
+                # Pipe might already be closed, that's okay
+                pass
+
+            # Join the process with timeout
+            self.process.join(timeout=2)
+
+            # If still alive, escalate to terminate
             if self.process.is_alive():
-                logger.debug("Process still alive after timeout, forcing termination")
+                logger.debug(
+                    "Process still alive after graceful shutdown, forcing termination"
+                )
                 self.process.terminate()
                 self.process.join(timeout=1)
-        except Exception as e:
-            logger.debug(f"Error during process shutdown: {e}")
 
-        # Set stop event before closing pipes
-        self.stop_event.set()
+                # Last resort: kill
+                if self.process.is_alive():
+                    logger.debug(
+                        "Process still alive after terminate, killing forcefully"
+                    )
+                    self.process.kill()
+                    self.process.join(timeout=1)
 
-        # Close all connections
-        for conn in [
-            self.parent_conn,
-            self.child_conn,
-            self.stdout_parent_conn,
-            self.stdout_child_conn,
-        ]:
-            conn.close()
+        except Exception as e:
+            logger.exception(f"Error during process shutdown: {e}")
 
-        self.stdout_thread.join(timeout=2)  # Add timeout to thread join
+        # Now handle the async tasks and threads
+        try:
+            # Cancel the stdout processor task if it exists
+            if self.stdout_processor_task:
+                self.stdout_processor_task.cancel()
+
+            # Close all connections - this will cause pending operations to fail fast
+            for conn in [
+                self.parent_conn,
+                self.child_conn,
+                self.stdout_parent_conn,
+                self.stdout_child_conn,
+            ]:
+                try:
+                    conn.close()
+                except:
+                    pass
 
-        if self.stdout_thread.is_alive():
-            logger.debug("Stdout thread failed to terminate within timeout")
+            # Join the stdout thread with timeout
+            if self.stdout_thread and self.stdout_thread.is_alive():
+                self.stdout_thread.join(timeout=2)
+                if self.stdout_thread.is_alive():
+                    logger.debug("Stdout thread failed to terminate within timeout")
 
-        if self.stdout_processor_task:
-            self.stdout_processor_task.cancel()
-
-    def process_message(self, fn_name, message, cache):
-        if fn_name in cache:
-            exporter_obj = cache[fn_name]
-            for k, v in message.content.items():
-                nice_str = exporter_obj.var_name_mapping.get(k)
-                if not nice_str:
-                    continue
-                p_dict, key = nested_access(exporter_obj.output, nice_str)
-                p_dict[key] = v
-            message.content = exporter_obj.output
-        return message.content
+        except Exception as e:
+            logger.exception(f"Error during resource cleanup: {e}")
 
     def start_stdout_processor_task(self):
         if self.stdout_processor_task is None or self.stdout_processor_task.done():
@@ -290,3 +378,59 @@ def get_task_metadata(in_memory_fn, exporter_obj):
             exporter_obj.var_name_reverse_mapping[d] for d in in_memory_fn.dependencies
         )
     return dependencies
+
+
+# Class for capturing and redirecting stdout/stderr
+class OutputCapture:
+    def __init__(self, stdout_pipe):
+        self.stdout_pipe = stdout_pipe
+        self.lock = threading.Lock()
+
+    def write(self, text):
+        with self.lock:
+            self.stdout_pipe.send(text)
+
+    def flush(self):
+        pass
+
+
+def process_message(message, exporter_obj):
+    """Process a message before passing it to a function"""
+    if exporter_obj:
+        for k, v in message.content.items():
+            nice_str = exporter_obj.var_name_mapping.get(k)
+            if not nice_str:
+                continue
+            p_dict, key = nested_access(exporter_obj.output, nice_str)
+            p_dict[key] = v
+        return exporter_obj.output
+    return message.content
+
+
+def import_code_from_string(code_string, module_name=None, add_to_sys_modules=True):
+    # Generate a unique module name if one isn't provided
+    if module_name is None:
+        module_name = f"setta_dynamic_module_{uuid.uuid4().hex}"
+
+    # Add current directory to sys.path if it's not already there
+    current_dir = str(CWD)
+    if current_dir not in sys.path:
+        sys.path.insert(0, current_dir)
+
+    spec = importlib.util.spec_from_loader(module_name, loader=None)
+
+    # Create a new module based on the spec
+    module = importlib.util.module_from_spec(spec)
+
+    # Optionally add the module to sys.modules
+    if add_to_sys_modules:
+        print(f"adding {module_name} to sys.modules", flush=True)
+        sys.modules[module_name] = module
+
+    # Compile the code string
+    code_object = compile(code_string, module_name, "exec")
+
+    # Execute the compiled code object in the module's namespace
+    exec(code_object, module.__dict__)
+
+    return module

{setta-0.0.14.dev2.dist-info → setta-0.0.14.dev3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: setta
-Version: 0.0.14.dev2
+Version: 0.0.14.dev3
 Summary: Python without the donkeywork.
 Home-page: https://setta.dev
 Author: Kevin Musgrave, Jeff Musgrave

{setta-0.0.14.dev2.dist-info → setta-0.0.14.dev3.dist-info}/RECORD

@@ -1,4 +1,4 @@
-setta/__init__.py,sha256=v8sPkSKHEFL488Wxp-yVEGsW9I0nkkHNUUO3b_4XqSQ,28
+setta/__init__.py,sha256=dRyAeVBboRTo_BMS4gP2ZewakrwbpbNaV3_dLZTmbgQ,28
 setta/server.py,sha256=q4w9WG7SuLxwYtgXUCQyLt7t_HLmQV4y5abqvm7-uEA,4861
 setta/start.py,sha256=5sMZ7WH3KV9Q0v186PsaYqsWOz7hebyrpXbBOp9wQww,3589
 setta/cli/__init__.py,sha256=UxZG_VOMuF6lEBT3teUgTS9ulsK3wt3Gu3BbAQiAmt8,47
@@ -230,8 +230,8 @@ setta/static/seed/.DS_Store,sha256=ENxJvDQd7Te_U8gExcXtHE-mAeBUYOHELRfDWgN1NmA,6
 setta/static/seed/examples/.DS_Store,sha256=1lFlJ5EFymdzGAUAaI30vcaaLHt3F1LwpG7xILf9jsM,6148
 setta/tasks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 setta/tasks/task_runner.py,sha256=gMXpfZWFMQbix2MfrHVCKB7BxQCjO8JH2P8cxUmt1ms,849
-setta/tasks/tasks.py,sha256=kgV1Z0xzM9dBS5nreO8QYGPLG-njR-zZ5-viT2rYpwg,10141
-setta/tasks/utils.py,sha256=iqbsLYBcu4Qd-MAHd0SWK9wPaJezgEh1Yg5YC9goOLU,10631
+setta/tasks/tasks.py,sha256=uG-S3jN9qZEolHtXkJZ4LMi-8LrNPctQR6TVWPlyeTs,10206
+setta/tasks/utils.py,sha256=cTmHJGPk6HHbl7nNTPV2KaTugkdSRJTow0z4A05b0lg,15901
 setta/tasks/fns/__init__.py,sha256=JhGzzQGaT9BWtF3pOmguh6pzIF9kdG3jdDNLyYZ2w7g,461
 setta/tasks/fns/codeAreaAutocomplete.py,sha256=gJ5JbjkWDyTothr-UF-YlOxrbVzj2iyOVK7XD3lfhSQ,6416
 setta/tasks/fns/codeAreaFindTemplateVars.py,sha256=vD9rY8VNPavv6VKa1bnxRPPRDNvFQy6mPIZRl-_3GnY,3708
@@ -252,9 +252,9 @@ setta/utils/generate_new_filename.py,sha256=KBLX6paDmTvXR-027TpqQkfijIXc7mCfhen-
 setta/utils/section_contents.py,sha256=V2HQPik6DfSXw4j7IalbP5AZ3OEGCbtL5ub3xL-Q_Qo,4141
 setta/utils/utils.py,sha256=KjzcvgM3Ab3IcE8vaWYtgBpwzPLKg0LmblnHLoYZJHM,9164
 setta/utils/websocket_manager.py,sha256=MBIMI8xxOFQF4lT3on4pupi1ttEWXdWPV4fI2YP_UJU,3925
-setta-0.0.14.dev2.dist-info/LICENSE,sha256=us9fuCq9wmiZVzayjKxNZ2iJYF6dROe0Qp57ToCO7XU,11361
-setta-0.0.14.dev2.dist-info/METADATA,sha256=ySjZVaCLgQKmWfS0edY3UcYcrPWaAqJeubY07dtRsNc,7517
-setta-0.0.14.dev2.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-setta-0.0.14.dev2.dist-info/entry_points.txt,sha256=P0qCESy9fWF2q1EQ9JufGldCSnPHplDPn8J6Bgk5hB0,42
-setta-0.0.14.dev2.dist-info/top_level.txt,sha256=8G4lmRzVOnJ11_DescPVHE6MQZH-o06A0nGsDDV2ngY,6
-setta-0.0.14.dev2.dist-info/RECORD,,
+setta-0.0.14.dev3.dist-info/LICENSE,sha256=us9fuCq9wmiZVzayjKxNZ2iJYF6dROe0Qp57ToCO7XU,11361
+setta-0.0.14.dev3.dist-info/METADATA,sha256=_B3NlxJtJjD-3RX-EifIfhK5bIgRfFbzFnCgNsOTL04,7517
+setta-0.0.14.dev3.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+setta-0.0.14.dev3.dist-info/entry_points.txt,sha256=P0qCESy9fWF2q1EQ9JufGldCSnPHplDPn8J6Bgk5hB0,42
+setta-0.0.14.dev3.dist-info/top_level.txt,sha256=8G4lmRzVOnJ11_DescPVHE6MQZH-o06A0nGsDDV2ngY,6
+setta-0.0.14.dev3.dist-info/RECORD,,