PyPI - setta - Versions diffs - 0.0.2__py3-none-any.whl → 0.0.3__py3-none-any.whl - Mend

setta 0.0.2py3-none-any.whl → 0.0.3py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (42) hide show

setta/__init__.py +1 -1
setta/cli/__init__.py +1 -1
setta/cli/logger.py +1 -3
setta/code_gen/create_runnable_scripts.py +87 -34
setta/code_gen/export_selected.py +1 -5
setta/code_gen/find_placeholders.py +13 -2
setta/database/backup.py +1 -2
setta/database/db/artifacts/save.py +20 -4
setta/database/db/projects/save.py +1 -1
setta/database/db/sections/load.py +5 -2
setta/database/db/sections/save.py +2 -2
setta/database/db_objs.py +6 -0
setta/database/export_db/export_raw.py +3 -3
setta/database/export_db/utils.py +2 -3
setta/database/settings_file.py +3 -3
setta/lsp/file_watcher.py +24 -23
setta/lsp/server.py +1 -1
setta/lsp/writer.py +2 -2
setta/routers/interactive.py +38 -20
setta/server.py +3 -4
setta/start.py +4 -3
setta/static/constants/constants.json +4 -0
setta/static/constants/db_init.sql +2 -1
setta/static/constants/defaultValues.json +2 -1
setta/static/constants/settingsProject.json +31 -31
setta/static/frontend/assets/index-03be034e.css +32 -0
setta/static/frontend/assets/{index-ee99dc72.js → index-59443547.js} +171 -171
setta/static/frontend/index.html +2 -2
setta/tasks/tasks.py +166 -98
setta/tasks/utils.py +108 -21
setta/terminals/terminals.py +7 -6
setta/utils/constants.py +5 -2
setta/utils/websocket_manager.py +8 -3
setta-0.0.3.dist-info/METADATA +146 -0
{setta-0.0.2.dist-info → setta-0.0.3.dist-info}/RECORD +39 -40
setta/database/db_path.py +0 -8
setta/static/frontend/assets/index-1d4b4ecf.css +0 -32
setta-0.0.2.dist-info/METADATA +0 -24
{setta-0.0.2.dist-info → setta-0.0.3.dist-info}/LICENSE +0 -0
{setta-0.0.2.dist-info → setta-0.0.3.dist-info}/WHEEL +0 -0
{setta-0.0.2.dist-info → setta-0.0.3.dist-info}/entry_points.txt +0 -0
{setta-0.0.2.dist-info → setta-0.0.3.dist-info}/top_level.txt +0 -0

setta/static/frontend/index.html CHANGED Viewed

@@ -15,8 +15,8 @@
     <title>setta.dev</title>
-    <script type="module" crossorigin src="/static/assets/index-ee99dc72.js"></script>
-    <link rel="stylesheet" href="/static/assets/index-1d4b4ecf.css">
+    <script type="module" crossorigin src="/static/assets/index-59443547.js"></script>
+    <link rel="stylesheet" href="/static/assets/index-03be034e.css">
   </head>
   <body>
     <noscript>You need to enable JavaScript to run this app.</noscript>

setta/tasks/tasks.py CHANGED Viewed

@@ -1,10 +1,11 @@
 import asyncio
+import copy
 import logging
-import queue
-import threading
+import time
 from typing import Dict
 from setta.database.utils import create_new_id
+from setta.utils.constants import C
 from . import fns
 from .fns.utils import TaskDefinition, TaskMessage
@@ -20,75 +21,31 @@ class Tasks:
         self.task_runner = TaskRunner()
         self.cache = {}
         self.fns: Dict[str, TaskDefinition] = {}
-        self.in_memory_subprocess = SettaInMemoryFnSubprocess()
-        self.websockets = []  # Store the websocket connections
+        self.in_memory_subprocesses = {}
+        self.websockets = []
+        self.stop_event = asyncio.Event()
         add_fns_from_module(self.fns, fns)
-        # Start stdout listener thread
-        self._stop_event = asyncio.Event()
-        self.stdout_queue = queue.Queue()  # regular Queue
-        self._stdout_processor_task = None
-        self.stdout_thread = threading.Thread(target=self._stdout_listener, daemon=True)
-        self.stdout_thread.start()
-    # Backend Changes (Tasks class)
     async def connect(self, websocket):
         # Accept the new connection
         await websocket.accept()
         self.websockets.append(websocket)
-        # Start the processor task if it's not running
-        if self._stdout_processor_task is None or self._stdout_processor_task.done():
-            self._stdout_processor_task = asyncio.create_task(
-                self._process_stdout_queue()
-            )
+        for k, v in self.in_memory_subprocesses.items():
+            v["subprocess"].start_stdout_processor_task()
+            logger.debug(f"listening to subprocess {k}")
     async def disconnect(self, websocket):
         self.websockets.remove(websocket)
         if len(self.websockets) == 0:
-            # Cancel the processor task
-            if self._stdout_processor_task and not self._stdout_processor_task.done():
-                self._stdout_processor_task.cancel()
-                try:
-                    await self._stdout_processor_task
-                except asyncio.CancelledError:
-                    pass
-                self._stdout_processor_task = None
-    def _stdout_listener(self):
-        while not self._stop_event.is_set():
-            try:
-                stdout_data = self.in_memory_subprocess.stdout_parent_conn.recv()
-                self.stdout_queue.put(stdout_data)  # simple put, no async needed
-            except Exception as e:
-                if self._stop_event.is_set():
-                    break
-                logger.debug(f"Error in stdout listener: {e}")
-    async def _process_stdout_queue(self):
-        while not self._stop_event.is_set():
-            try:
-                if self._stop_event.is_set():
-                    break
-                if len(self.websockets) > 0:
-                    stdout_data = self.stdout_queue.get_nowait()
-                    stdout_data = stdout_data.replace("\n", "\r\n")
-                    for w in self.websockets:
-                        await w.send_text(stdout_data)
-                    self.stdout_queue.task_done()
-            except queue.Empty:
-                await asyncio.sleep(0.1)  # Check for connection every 100ms
-            except asyncio.CancelledError:
-                break
-            except Exception as e:
-                if self._stop_event.is_set():
-                    break
-                logger.debug(f"Error processing stdout: {e}")
-    async def __call__(self, fn_name, message: TaskMessage):
-        if fn_name in self.fns:
-            return await self.call_regular_fn(fn_name, message)
-        return await self.call_in_memory_subprocess_fn(fn_name, message)
+            for v in self.in_memory_subprocesses.values():
+                await v["subprocess"].stop_stdout_processor_task()
+    async def __call__(
+        self, message_type, message: TaskMessage, websocket_manager=None
+    ):
+        if message_type == "inMemoryFn":
+            return await self.call_in_memory_subprocess_fn(message, websocket_manager)
+        return await self.call_regular_fn(message_type, message)
     async def call_regular_fn(self, fn_name, message: TaskMessage):
         fn = self.fns[fn_name]
@@ -101,52 +58,163 @@ class Tasks:
         result["messageType"] = fn.return_message_type
         return result
-    async def call_in_memory_subprocess_fn(self, fn_name, message: TaskMessage):
-        self.in_memory_subprocess.parent_conn.send(
-            {"type": "call", "fn_name": fn_name, "message": message}
-        )
-        result = await self.task_runner.run(
-            self.in_memory_subprocess.parent_conn.recv, [], RunType.THREAD
-        )
-        # if result["status"] == "success":
-        # for x in result["content"]:
-        # x.setdefault("sectionType", default_section_type(x["type"]))
-        if result["status"] != "success":
-            result["content"] = {}
-        return {"content": result["content"], "messageType": result["messageType"]}
-    async def add_custom_fns(self, code_list, to_cache):
-        error_msgs = {}
-        task_metadata = {}
-        initial_content = []
-        for c in code_list:
-            # Send import request to subprocess
-            self.in_memory_subprocess.parent_conn.send(
+    async def call_in_memory_subprocess_fn(
+        self,
+        message: TaskMessage,
+        websocket_manager=None,
+        call_all=False,
+        subprocess_key=None,
+    ):
+        # Create a list of tasks to run concurrently
+        tasks = []
+        results = []
+        for sp_key, sp_info in self.in_memory_subprocesses.items():
+            if subprocess_key and sp_key != subprocess_key:
+                continue
+            for fn_name, fnInfo in sp_info["fnInfo"].items():
+                if (
+                    call_all
+                    or None in fnInfo["dependencies"]
+                    or any(k in fnInfo["dependencies"] for k in message.content.keys())
+                ):
+                    # Send message to subprocess
+                    sp_info["subprocess"].parent_conn.send(
+                        {"type": "call", "fn_name": fn_name, "message": message}
+                    )
+                    # Create task for receiving response
+                    task = asyncio.create_task(
+                        self._handle_subprocess_response(
+                            sp_key,
+                            fn_name,
+                            message.id,
+                            sp_info["subprocess"].parent_conn.recv,
+                            websocket_manager,
+                            results,
+                        )
+                    )
+                    tasks.append(task)
+        # Wait for all tasks to complete concurrently
+        if tasks:
+            await asyncio.gather(*tasks)
+        if websocket_manager:
+            return {}
+        content = []
+        for r in results:
+            if r["content"]:
+                content.extend(r["content"])
+        return {"content": content, "messageType": C.WS_IN_MEMORY_FN_RETURN}
+    async def _handle_subprocess_response(
+        self, subprocess_key, fn_name, msg_id, recv_fn, websocket_manager, results
+    ):
+        # Run the receive function in a thread
+        start_time = time.perf_counter()
+        result = await self.task_runner.run(recv_fn, [], RunType.THREAD)
+        elapsed_time = time.perf_counter() - start_time
+        if result["status"] == "success":
+            self.update_average_subprocess_fn_time(
+                subprocess_key, fn_name, elapsed_time
+            )
+            if websocket_manager is not None:
+                if result["content"]:
+                    await websocket_manager.send_message_to_requester(
+                        msg_id, result["content"], result["messageType"]
+                    )
+                await self.maybe_send_latest_run_time_info(
+                    subprocess_key, fn_name, msg_id, websocket_manager
+                )
+            else:
+                results.append(result)
+    async def add_custom_fns(self, code_graph, to_cache):
+        for c in code_graph:
+            subprocess_key = c["subprocess_key"]
+            module_name = c["module_name"]
+            sp = self.in_memory_subprocesses.get(subprocess_key, {}).get("subprocess")
+            if sp:
+                sp.close()
+            logger.debug(f"Creating new subprocess for {module_name}")
+            sp = SettaInMemoryFnSubprocess(self.stop_event, self.websockets)
+            self.in_memory_subprocesses[subprocess_key] = {
+                "subprocess": sp,
+                "fnInfo": {},
+            }
+            sp.parent_conn.send(
                 {
                     "type": "import",
                     "code": c["code"],
-                    "module_name": c["module_name"],
+                    "module_name": module_name,
                     "to_cache": to_cache,
                 }
             )
-            result = await self.task_runner.run(
-                self.in_memory_subprocess.parent_conn.recv, [], RunType.THREAD
-            )
+            result = await self.task_runner.run(sp.parent_conn.recv, [], RunType.THREAD)
+            fnInfo = self.in_memory_subprocesses[subprocess_key]["fnInfo"]
             if result["status"] == "success":
-                task_metadata.update(result["content"])
+                for k, v in result["content"].items():
+                    if k not in fnInfo:
+                        fnInfo[k] = {
+                            "dependencies": set(),
+                            "averageRunTime": None,
+                            "callCount": 0,
+                            "lastStatsUpdate": time.time(),
+                        }
+                    fnInfo[k]["dependencies"].update(v)
             else:
-                error_msgs[c["module_name"]] = result["error"]
+                # TODO: store error message and display on frontend?
+                pass
+        initial_result = await self.call_in_memory_subprocess_fn(
+            TaskMessage(id=create_new_id(), content={}),
+            call_all=True,
+            subprocess_key=subprocess_key,
+        )
-        for k in task_metadata.keys():
-            task_output = await self(k, TaskMessage(id=create_new_id(), content={}))
-            initial_content.extend(task_output["content"])
+        logger.debug(
+            f"self.in_memory_subprocesses keys: {self.in_memory_subprocesses.keys()}"
+        )
-        return task_metadata, error_msgs, initial_content
+        return initial_result["content"]
     def close(self):
-        self._stop_event.set()
-        self.in_memory_subprocess.close()
-        self.stdout_thread.join()
-        if self._stdout_processor_task:
-            self._stdout_processor_task.cancel()
+        self.stop_event.set()
+        for v in self.in_memory_subprocesses.values():
+            v["subprocess"].close()
+    def update_average_subprocess_fn_time(self, subprocess_key, fn_name, new_time):
+        fnInfo = self.in_memory_subprocesses[subprocess_key]["fnInfo"][fn_name]
+        current_avg = fnInfo["averageRunTime"]
+        new_avg = (
+            new_time
+            if current_avg is None
+            else ((0.9) * current_avg) + (0.1 * new_time)
+        )
+        fnInfo["averageRunTime"] = new_avg
+        fnInfo["callCount"] += 1
+        fnInfo["lastStatsUpdate"] = time.time()
+    async def maybe_send_latest_run_time_info(
+        self, subprocess_key, fn_name, msg_id, websocket_manager
+    ):
+        fnInfo = self.in_memory_subprocesses[subprocess_key]["fnInfo"][fn_name]
+        if fnInfo["callCount"] % 10 == 0 or (
+            fnInfo["lastStatsUpdate"] and (time.time() - fnInfo["lastStatsUpdate"]) > 10
+        ):
+            newInfo = self.getInMemorySubprocessInfo()
+            await websocket_manager.send_message_to_requester(
+                msg_id, newInfo, C.WS_IN_MEMORY_FN_AVG_RUN_TIME
+            )
+    def getInMemorySubprocessInfo(self):
+        output = {}
+        for sp_key, sp_info in self.in_memory_subprocesses.items():
+            output[sp_key] = {"fnInfo": copy.deepcopy(sp_info["fnInfo"])}
+            for fnInfo in output[sp_key]["fnInfo"].values():
+                fnInfo["dependencies"] = list(fnInfo["dependencies"])
+        return output

setta/tasks/utils.py CHANGED Viewed

@@ -1,14 +1,20 @@
+import asyncio
 import importlib.util
 import json
+import logging
 import multiprocessing
-import os
+import queue
 import sys
+import threading
 import traceback
 import uuid
 from setta.tasks.fns.utils import TaskDefinition
+from setta.utils.constants import CWD
 from setta.utils.utils import nested_access
+logger = logging.getLogger(__name__)
 def import_code_from_string(code_string, module_name=None, add_to_sys_modules=True):
     # Generate a unique module name if one isn't provided
@@ -16,7 +22,7 @@ def import_code_from_string(code_string, module_name=None, add_to_sys_modules=Tr
         module_name = f"setta_dynamic_module_{uuid.uuid4().hex}"
     # Add current directory to sys.path if it's not already there
-    current_dir = os.getcwd()
+    current_dir = str(CWD)
     if current_dir not in sys.path:
         sys.path.insert(0, current_dir)
@@ -40,13 +46,24 @@ def import_code_from_string(code_string, module_name=None, add_to_sys_modules=Tr
 class SettaInMemoryFnSubprocess:
-    def __init__(self):
+    def __init__(self, stop_event, websockets):
         self.parent_conn, self.child_conn = multiprocessing.Pipe()
         self.process = multiprocessing.Process(target=self._subprocess_main)
         self.stdout_parent_conn, self.stdout_child_conn = multiprocessing.Pipe()
         self.process.daemon = True  # Ensure process dies with parent
         self.process.start()
+        self.stop_event = asyncio.Event()
+        self.tasks_stop_event = stop_event
+        self.websockets = websockets
+        self.stdout_queue = queue.Queue()
+        self.stdout_processor_task = None
+        self.stdout_thread = threading.Thread(target=self.stdout_listener, daemon=True)
+        self.stdout_thread.start()
+        if len(self.websockets) > 0:
+            self.start_stdout_processor_task()
     def _subprocess_main(self):
         """Main loop in subprocess that handles all requests"""
         # Initialize store for imported modules
@@ -83,15 +100,15 @@ class SettaInMemoryFnSubprocess:
                     # Import and store module
                     module = import_code_from_string(code, module_name)
                     added_fn_names = add_fns_from_module(fns_dict, module, module_name)
-                    task_metadata = {}
+                    dependencies = {}
                     for k in added_fn_names:
                         cache[k] = msg["to_cache"]
-                        task_metadata[k] = get_task_metadata(fns_dict[k], cache[k])
+                        dependencies[k] = get_task_metadata(fns_dict[k], cache[k])
                     self.child_conn.send(
                         {
                             "status": "success",
-                            "content": task_metadata,
+                            "content": dependencies,
                         }
                     )
@@ -122,20 +139,36 @@ class SettaInMemoryFnSubprocess:
     def close(self):
         try:
+            logger.debug("Initiating shutdown sequence")
             self.parent_conn.send({"type": "shutdown"})
-            self.process.join(timeout=1.0)
-        except:
-            pass
+            self.process.join(timeout=2)  # Add timeout to process join
+            if self.process.is_alive():
+                logger.debug("Process still alive after timeout, forcing termination")
+                self.process.terminate()
+                self.process.join(timeout=1)
+        except Exception as e:
+            logger.debug(f"Error during process shutdown: {e}")
-        if self.process.is_alive():
-            self.process.terminate()
-            self.process.join()
+        # Set stop event before closing pipes
+        self.stop_event.set()
-        # Close both sets of connections
-        self.parent_conn.close()
-        self.child_conn.close()
-        self.stdout_parent_conn.close()
-        self.stdout_child_conn.close()
+        # Close all connections
+        for conn in [
+            self.parent_conn,
+            self.child_conn,
+            self.stdout_parent_conn,
+            self.stdout_child_conn,
+        ]:
+            conn.close()
+        self.stdout_thread.join(timeout=2)  # Add timeout to thread join
+        if self.stdout_thread.is_alive():
+            logger.debug("Stdout thread failed to terminate within timeout")
+        if self.stdout_processor_task:
+            self.stdout_processor_task.cancel()
     def process_message(self, fn_name, message, cache):
         if fn_name in cache:
@@ -147,6 +180,60 @@ class SettaInMemoryFnSubprocess:
             message.content = exporter_obj.output
         return message.content
+    def start_stdout_processor_task(self):
+        if self.stdout_processor_task is None or self.stdout_processor_task.done():
+            self.stdout_processor_task = asyncio.create_task(
+                self.process_stdout_queue()
+            )
+    async def stop_stdout_processor_task(self):
+        if self.stdout_processor_task and not self.stdout_processor_task.done():
+            self.stdout_processor_task.cancel()
+            try:
+                await self.stdout_processor_task
+            except asyncio.CancelledError:
+                pass
+            self.stdout_processor_task = None
+    async def process_stdout_queue(self):
+        while not self.should_stop():
+            try:
+                if self.should_stop():
+                    break
+                if len(self.websockets) > 0:
+                    stdout_data = self.stdout_queue.get_nowait()
+                    stdout_data = stdout_data.replace("\n", "\r\n")
+                    for w in self.websockets:
+                        await w.send_text(stdout_data)
+                    self.stdout_queue.task_done()
+            except queue.Empty:
+                await asyncio.sleep(0.1)  # Check for connection every 100ms
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                if self.should_stop():
+                    break
+                logger.debug(f"Error processing stdout: {e}")
+    def stdout_listener(self):
+        while not self.should_stop():
+            if self.stdout_parent_conn.poll(0.1):  # Check for data with timeout
+                try:
+                    stdout_data = self.stdout_parent_conn.recv()
+                    self.stdout_queue.put(stdout_data)
+                except EOFError:  # Pipe was closed
+                    break
+                except Exception as e:
+                    logger.debug(f"Error in stdout listener: {e}")
+                    if self.should_stop():
+                        break
+            else:  # No data available within timeout
+                if self.should_stop():
+                    break
+    def should_stop(self):
+        return self.stop_event.is_set() or self.tasks_stop_event.is_set()
 def add_fns_from_module(fns_dict, module, module_name=None):
     count = 1
@@ -168,11 +255,11 @@ def add_fns_from_module(fns_dict, module, module_name=None):
 def get_task_metadata(in_memory_fn, exporter_obj):
     # None means run the task on every change
     if in_memory_fn.dependencies is None:
-        dependencies = None
+        dependencies = set([None])
     # Empty array means only run when the task imported.
     # Non-empty array means run when specified dependencies update.
     else:
-        dependencies = [
+        dependencies = set(
             exporter_obj.var_name_reverse_mapping[d] for d in in_memory_fn.dependencies
-        ]
-    return {"dependencies": dependencies}
+        )
+    return dependencies

setta/terminals/terminals.py CHANGED Viewed

@@ -4,6 +4,7 @@ import json
 import logging
 import platform
 import select
+import shlex
 import time
 import traceback
 from asyncio import CancelledError
@@ -44,12 +45,14 @@ def is_command_running_in_pty(pid):
 def get_terminal_shell():
     if USER_SETTINGS["backend"]["defaultTerminalShell"]:
-        return USER_SETTINGS["backend"]["defaultTerminalShell"]
+        return shlex.split(
+            USER_SETTINGS["backend"]["defaultTerminalShell"], posix=not IS_WINDOWS
+        )
     if IS_WINDOWS:
-        return "bash.exe"
+        return ["bash.exe"]
     if IS_MACOS:
-        return "zsh"
-    return "bash"
+        return ["zsh"]
+    return ["bash"]
 class TerminalWebsockets:
@@ -61,8 +64,6 @@ class TerminalWebsockets:
     def new_terminal(self, projectConfigId, sectionId, isTemporary):
         if sectionId not in self.PTY_PIDS:
             terminal_shell = get_terminal_shell()
-            if not IS_WINDOWS:
-                terminal_shell = [terminal_shell]
             pty_process = PtyProcess.spawn(terminal_shell)
             self.PTY_PIDS[sectionId] = {
                 "pid": pty_process.pid,

setta/utils/constants.py CHANGED Viewed

@@ -10,8 +10,10 @@ CONSTANTS_FOLDER = (
     Path("../../../constants") if is_dev_mode() else Path("../static/constants")
 )
 SEED_FOLDER = Path("../../../seed") if is_dev_mode() else Path("../static/seed")
-CODE_FOLDER = "setta_code"
-DB_BACKUP_FOLDER = ".setta_backups"
+CWD = Path.cwd()
+SETTA_FILES_FOLDER = CWD / "setta_files"
+CODE_FOLDER = SETTA_FILES_FOLDER / "code"
+DB_BACKUP_FOLDER = SETTA_FILES_FOLDER / "backups"
 CODE_FOLDER_ENV_VARIABLE = "SETTA_CODE_FOLDER"
 HOST_ENV_VARIABLE = "SETTA_HOST"
 PORT_ENV_VARIABLE = "SETTA_PORT"
@@ -99,6 +101,7 @@ CODE_INFO_TABLE_DATA_JSON_FIELDS = set(
         "positionalOnly",
         "isPinned",
         "isFrozen",
+        "ignoreTypeErrors",
     )
 )

setta/utils/websocket_manager.py CHANGED Viewed

@@ -43,7 +43,9 @@ class WebsocketManager:
             if wid == self.server_cli_id:
                 # process task and send any results back to requester
                 result = await tasks(
-                    message["messageType"], TaskMessage.parse_obj(message)
+                    message["messageType"],
+                    TaskMessage.parse_obj(message),
+                    websocket_manager=self,
                 )
                 if "content" in result:
                     websocket = self.sockets[fromWebsocketId]["websocket"]
@@ -64,10 +66,13 @@ class WebsocketManager:
         if "location" in message:
             self.sockets[fromWebsocketId]["location"] = message["location"]
-    async def send_message_to_requester(self, id, content):
+    async def send_message_to_requester(self, id, content, messageType=None):
         # just send data to target websocket
         websocket = self.sockets[self.message_id_to_sender_id[id]]["websocket"]
-        await websocket.send_text(json.dumps({"id": id, "content": content}))
+        return_val = {"id": id, "content": content}
+        if messageType:
+            return_val["messageType"] = messageType
+        await websocket.send_text(json.dumps(return_val))
     async def send_message_to_location(self, content, messageType, location):
         for w in self.sockets.values():

setta 0.0.2__py3-none-any.whl → 0.0.3__py3-none-any.whl

setta 0.0.2py3-none-any.whl → 0.0.3py3-none-any.whl