PyPI - setta - Versions diffs - 0.0.2.dev0__py3-none-any.whl → 0.0.3__py3-none-any.whl - Mend

setta 0.0.2.dev0py3-none-any.whl → 0.0.3py3-none-any.whl

Files changed (42) hide show

setta/__init__.py +1 -1
setta/cli/__init__.py +1 -1
setta/cli/logger.py +1 -3
setta/code_gen/create_runnable_scripts.py +87 -34
setta/code_gen/export_selected.py +1 -5
setta/code_gen/find_placeholders.py +13 -2
setta/database/backup.py +1 -2
setta/database/db/artifacts/save.py +20 -4
setta/database/db/projects/save.py +1 -1
setta/database/db/sections/load.py +5 -2
setta/database/db/sections/save.py +2 -2
setta/database/db_objs.py +6 -0
setta/database/export_db/export_raw.py +3 -3
setta/database/export_db/utils.py +2 -3
setta/database/settings_file.py +3 -3
setta/lsp/file_watcher.py +24 -23
setta/lsp/server.py +1 -1
setta/lsp/writer.py +2 -2
setta/routers/interactive.py +38 -20
setta/server.py +3 -4
setta/start.py +4 -3
setta/static/constants/constants.json +4 -0
setta/static/constants/db_init.sql +2 -1
setta/static/constants/defaultValues.json +2 -1
setta/static/constants/settingsProject.json +31 -31
setta/static/frontend/assets/index-03be034e.css +32 -0
setta/static/frontend/assets/{index-ee99dc72.js → index-59443547.js} +171 -171
setta/static/frontend/index.html +2 -2
setta/tasks/tasks.py +166 -98
setta/tasks/utils.py +108 -21
setta/terminals/terminals.py +7 -6
setta/utils/constants.py +5 -2
setta/utils/websocket_manager.py +8 -3
setta-0.0.3.dist-info/METADATA +146 -0
{setta-0.0.2.dev0.dist-info → setta-0.0.3.dist-info}/RECORD +39 -40
setta/database/db_path.py +0 -8
setta/static/frontend/assets/index-1d4b4ecf.css +0 -32
setta-0.0.2.dev0.dist-info/METADATA +0 -24
{setta-0.0.2.dev0.dist-info → setta-0.0.3.dist-info}/LICENSE +0 -0
{setta-0.0.2.dev0.dist-info → setta-0.0.3.dist-info}/WHEEL +0 -0
{setta-0.0.2.dev0.dist-info → setta-0.0.3.dist-info}/entry_points.txt +0 -0
{setta-0.0.2.dev0.dist-info → setta-0.0.3.dist-info}/top_level.txt +0 -0

setta/static/frontend/index.html CHANGED Viewed

@@ -15,8 +15,8 @@
     <title>setta.dev</title>
-    <script type="module" crossorigin src="/static/assets/index-ee99dc72.js"></script>
-    <link rel="stylesheet" href="/static/assets/index-1d4b4ecf.css">
+    <script type="module" crossorigin src="/static/assets/index-59443547.js"></script>
+    <link rel="stylesheet" href="/static/assets/index-03be034e.css">
   </head>
   <body>
     <noscript>You need to enable JavaScript to run this app.</noscript>

setta/tasks/tasks.py CHANGED Viewed

@@ -1,10 +1,11 @@
 import asyncio
+import copy
 import logging
-import queue
-import threading
+import time
 from typing import Dict
 from setta.database.utils import create_new_id
+from setta.utils.constants import C
 from . import fns
 from .fns.utils import TaskDefinition, TaskMessage
@@ -20,75 +21,31 @@ class Tasks:
         self.task_runner = TaskRunner()
         self.cache = {}
         self.fns: Dict[str, TaskDefinition] = {}
-        self.in_memory_subprocess = SettaInMemoryFnSubprocess()
-        self.websockets = []  # Store the websocket connections
+        self.in_memory_subprocesses = {}
+        self.websockets = []
+        self.stop_event = asyncio.Event()
         add_fns_from_module(self.fns, fns)
-        # Start stdout listener thread
-        self._stop_event = asyncio.Event()
-        self.stdout_queue = queue.Queue()  # regular Queue
-        self._stdout_processor_task = None
-        self.stdout_thread = threading.Thread(target=self._stdout_listener, daemon=True)
-        self.stdout_thread.start()
-    # Backend Changes (Tasks class)
     async def connect(self, websocket):
         # Accept the new connection
         await websocket.accept()
         self.websockets.append(websocket)
-        # Start the processor task if it's not running
-        if self._stdout_processor_task is None or self._stdout_processor_task.done():
-            self._stdout_processor_task = asyncio.create_task(
-                self._process_stdout_queue()
-            )
+        for k, v in self.in_memory_subprocesses.items():
+            v["subprocess"].start_stdout_processor_task()
+            logger.debug(f"listening to subprocess {k}")
     async def disconnect(self, websocket):
         self.websockets.remove(websocket)
         if len(self.websockets) == 0:
-            # Cancel the processor task
-            if self._stdout_processor_task and not self._stdout_processor_task.done():
-                self._stdout_processor_task.cancel()
-                try:
-                    await self._stdout_processor_task
-                except asyncio.CancelledError:
-                    pass
-                self._stdout_processor_task = None
-    def _stdout_listener(self):
-        while not self._stop_event.is_set():
-            try:
-                stdout_data = self.in_memory_subprocess.stdout_parent_conn.recv()
-                self.stdout_queue.put(stdout_data)  # simple put, no async needed
-            except Exception as e:
-                if self._stop_event.is_set():
-                    break
-                logger.debug(f"Error in stdout listener: {e}")
-    async def _process_stdout_queue(self):
-        while not self._stop_event.is_set():
-            try:
-                if self._stop_event.is_set():
-                    break
-                if len(self.websockets) > 0:
-                    stdout_data = self.stdout_queue.get_nowait()
-                    stdout_data = stdout_data.replace("\n", "\r\n")
-                    for w in self.websockets:
-                        await w.send_text(stdout_data)
-                    self.stdout_queue.task_done()
-            except queue.Empty:
-                await asyncio.sleep(0.1)  # Check for connection every 100ms
-            except asyncio.CancelledError:
-                break
-            except Exception as e:
-                if self._stop_event.is_set():
-                    break
-                logger.debug(f"Error processing stdout: {e}")
-    async def __call__(self, fn_name, message: TaskMessage):
-        if fn_name in self.fns:
-            return await self.call_regular_fn(fn_name, message)
-        return await self.call_in_memory_subprocess_fn(fn_name, message)
+            for v in self.in_memory_subprocesses.values():
+                await v["subprocess"].stop_stdout_processor_task()
+    async def __call__(
+        self, message_type, message: TaskMessage, websocket_manager=None
+    ):
+        if message_type == "inMemoryFn":
+            return await self.call_in_memory_subprocess_fn(message, websocket_manager)
+        return await self.call_regular_fn(message_type, message)
     async def call_regular_fn(self, fn_name, message: TaskMessage):
         fn = self.fns[fn_name]
@@ -101,52 +58,163 @@ class Tasks:
         result["messageType"] = fn.return_message_type
         return result
-    async def call_in_memory_subprocess_fn(self, fn_name, message: TaskMessage):
-        self.in_memory_subprocess.parent_conn.send(
-            {"type": "call", "fn_name": fn_name, "message": message}
-        )
-        result = await self.task_runner.run(
-            self.in_memory_subprocess.parent_conn.recv, [], RunType.THREAD
-        )
-        # if result["status"] == "success":
-        # for x in result["content"]:
-        # x.setdefault("sectionType", default_section_type(x["type"]))
-        if result["status"] != "success":
-            result["content"] = {}
-        return {"content": result["content"], "messageType": result["messageType"]}
-    async def add_custom_fns(self, code_list, to_cache):
-        error_msgs = {}
-        task_metadata = {}
-        initial_content = []
-        for c in code_list:
-            # Send import request to subprocess
-            self.in_memory_subprocess.parent_conn.send(
+    async def call_in_memory_subprocess_fn(
+        self,
+        message: TaskMessage,
+        websocket_manager=None,
+        call_all=False,
+        subprocess_key=None,
+    ):
+        # Create a list of tasks to run concurrently
+        tasks = []
+        results = []
+        for sp_key, sp_info in self.in_memory_subprocesses.items():
+            if subprocess_key and sp_key != subprocess_key:
+                continue
+            for fn_name, fnInfo in sp_info["fnInfo"].items():
+                if (
+                    call_all
+                    or None in fnInfo["dependencies"]
+                    or any(k in fnInfo["dependencies"] for k in message.content.keys())
+                ):
+                    # Send message to subprocess
+                    sp_info["subprocess"].parent_conn.send(
+                        {"type": "call", "fn_name": fn_name, "message": message}
+                    )
+                    # Create task for receiving response
+                    task = asyncio.create_task(
+                        self._handle_subprocess_response(
+                            sp_key,
+                            fn_name,
+                            message.id,
+                            sp_info["subprocess"].parent_conn.recv,
+                            websocket_manager,
+                            results,
+                        )
+                    )
+                    tasks.append(task)
+        # Wait for all tasks to complete concurrently
+        if tasks:
+            await asyncio.gather(*tasks)
+        if websocket_manager:
+            return {}
+        content = []
+        for r in results:
+            if r["content"]:
+                content.extend(r["content"])
+        return {"content": content, "messageType": C.WS_IN_MEMORY_FN_RETURN}
+    async def _handle_subprocess_response(
+        self, subprocess_key, fn_name, msg_id, recv_fn, websocket_manager, results
+    ):
+        # Run the receive function in a thread
+        start_time = time.perf_counter()
+        result = await self.task_runner.run(recv_fn, [], RunType.THREAD)
+        elapsed_time = time.perf_counter() - start_time
+        if result["status"] == "success":
+            self.update_average_subprocess_fn_time(
+                subprocess_key, fn_name, elapsed_time
+            )
+            if websocket_manager is not None:
+                if result["content"]:
+                    await websocket_manager.send_message_to_requester(
+                        msg_id, result["content"], result["messageType"]
+                    )
+                await self.maybe_send_latest_run_time_info(
+                    subprocess_key, fn_name, msg_id, websocket_manager
+                )
+            else:
+                results.append(result)
+    async def add_custom_fns(self, code_graph, to_cache):
+        for c in code_graph:
+            subprocess_key = c["subprocess_key"]
+            module_name = c["module_name"]
+            sp = self.in_memory_subprocesses.get(subprocess_key, {}).get("subprocess")
+            if sp:
+                sp.close()
+            logger.debug(f"Creating new subprocess for {module_name}")
+            sp = SettaInMemoryFnSubprocess(self.stop_event, self.websockets)
+            self.in_memory_subprocesses[subprocess_key] = {
+                "subprocess": sp,
+                "fnInfo": {},
+            }
+            sp.parent_conn.send(
                 {
                     "type": "import",
                     "code": c["code"],
-                    "module_name": c["module_name"],
+                    "module_name": module_name,
                     "to_cache": to_cache,
                 }
             )
-            result = await self.task_runner.run(
-                self.in_memory_subprocess.parent_conn.recv, [], RunType.THREAD
-            )
+            result = await self.task_runner.run(sp.parent_conn.recv, [], RunType.THREAD)
+            fnInfo = self.in_memory_subprocesses[subprocess_key]["fnInfo"]
             if result["status"] == "success":
-                task_metadata.update(result["content"])
+                for k, v in result["content"].items():
+                    if k not in fnInfo:
+                        fnInfo[k] = {
+                            "dependencies": set(),
+                            "averageRunTime": None,
+                            "callCount": 0,
+                            "lastStatsUpdate": time.time(),
+                        }
+                    fnInfo[k]["dependencies"].update(v)
             else:
-                error_msgs[c["module_name"]] = result["error"]
+                # TODO: store error message and display on frontend?
+                pass
+        initial_result = await self.call_in_memory_subprocess_fn(
+            TaskMessage(id=create_new_id(), content={}),
+            call_all=True,
+            subprocess_key=subprocess_key,
+        )
-        for k in task_metadata.keys():
-            task_output = await self(k, TaskMessage(id=create_new_id(), content={}))
-            initial_content.extend(task_output["content"])
+        logger.debug(
+            f"self.in_memory_subprocesses keys: {self.in_memory_subprocesses.keys()}"
+        )
-        return task_metadata, error_msgs, initial_content
+        return initial_result["content"]
     def close(self):
-        self._stop_event.set()
-        self.in_memory_subprocess.close()
-        self.stdout_thread.join()
-        if self._stdout_processor_task:
-            self._stdout_processor_task.cancel()
+        self.stop_event.set()
+        for v in self.in_memory_subprocesses.values():
+            v["subprocess"].close()
+    def update_average_subprocess_fn_time(self, subprocess_key, fn_name, new_time):
+        fnInfo = self.in_memory_subprocesses[subprocess_key]["fnInfo"][fn_name]
+        current_avg = fnInfo["averageRunTime"]
+        new_avg = (
+            new_time
+            if current_avg is None
+            else ((0.9) * current_avg) + (0.1 * new_time)
+        )
+        fnInfo["averageRunTime"] = new_avg
+        fnInfo["callCount"] += 1
+        fnInfo["lastStatsUpdate"] = time.time()
+    async def maybe_send_latest_run_time_info(
+        self, subprocess_key, fn_name, msg_id, websocket_manager
+    ):
+        fnInfo = self.in_memory_subprocesses[subprocess_key]["fnInfo"][fn_name]
+        if fnInfo["callCount"] % 10 == 0 or (
+            fnInfo["lastStatsUpdate"] and (time.time() - fnInfo["lastStatsUpdate"]) > 10
+        ):
+            newInfo = self.getInMemorySubprocessInfo()
+            await websocket_manager.send_message_to_requester(
+                msg_id, newInfo, C.WS_IN_MEMORY_FN_AVG_RUN_TIME
+            )
+    def getInMemorySubprocessInfo(self):
+        output = {}
+        for sp_key, sp_info in self.in_memory_subprocesses.items():
+            output[sp_key] = {"fnInfo": copy.deepcopy(sp_info["fnInfo"])}
+            for fnInfo in output[sp_key]["fnInfo"].values():
+                fnInfo["dependencies"] = list(fnInfo["dependencies"])
+        return output

setta/tasks/utils.py CHANGED Viewed

@@ -1,14 +1,20 @@
+import asyncio
 import importlib.util
 import json
+import logging
 import multiprocessing
-import os
+import queue
 import sys
+import threading
 import traceback
 import uuid
 from setta.tasks.fns.utils import TaskDefinition
+from setta.utils.constants import CWD
 from setta.utils.utils import nested_access
+logger = logging.getLogger(__name__)
 def import_code_from_string(code_string, module_name=None, add_to_sys_modules=True):
     # Generate a unique module name if one isn't provided
@@ -16,7 +22,7 @@ def import_code_from_string(code_string, module_name=None, add_to_sys_modules=Tr
         module_name = f"setta_dynamic_module_{uuid.uuid4().hex}"
     # Add current directory to sys.path if it's not already there
-    current_dir = os.getcwd()
+    current_dir = str(CWD)
     if current_dir not in sys.path:
         sys.path.insert(0, current_dir)
@@ -40,13 +46,24 @@ def import_code_from_string(code_string, module_name=None, add_to_sys_modules=Tr
 class SettaInMemoryFnSubprocess:
-    def __init__(self):
+    def __init__(self, stop_event, websockets):
         self.parent_conn, self.child_conn = multiprocessing.Pipe()
         self.process = multiprocessing.Process(target=self._subprocess_main)
         self.stdout_parent_conn, self.stdout_child_conn = multiprocessing.Pipe()
         self.process.daemon = True  # Ensure process dies with parent
         self.process.start()
+        self.stop_event = asyncio.Event()
+        self.tasks_stop_event = stop_event
+        self.websockets = websockets
+        self.stdout_queue = queue.Queue()
+        self.stdout_processor_task = None
+        self.stdout_thread = threading.Thread(target=self.stdout_listener, daemon=True)
+        self.stdout_thread.start()
+        if len(self.websockets) > 0:
+            self.start_stdout_processor_task()
     def _subprocess_main(self):
         """Main loop in subprocess that handles all requests"""
         # Initialize store for imported modules
@@ -83,15 +100,15 @@ class SettaInMemoryFnSubprocess:
                     # Import and store module
                     module = import_code_from_string(code, module_name)
                     added_fn_names = add_fns_from_module(fns_dict, module, module_name)
-                    task_metadata = {}
+                    dependencies = {}
                     for k in added_fn_names:
                         cache[k] = msg["to_cache"]
-                        task_metadata[k] = get_task_metadata(fns_dict[k], cache[k])
+                        dependencies[k] = get_task_metadata(fns_dict[k], cache[k])
                     self.child_conn.send(
                         {
                             "status": "success",
-                            "content": task_metadata,
+                            "content": dependencies,
                         }
                     )
@@ -122,20 +139,36 @@ class SettaInMemoryFnSubprocess:
     def close(self):
         try:
+            logger.debug("Initiating shutdown sequence")
             self.parent_conn.send({"type": "shutdown"})
-            self.process.join(timeout=1.0)
-        except:
-            pass
+            self.process.join(timeout=2)  # Add timeout to process join
+            if self.process.is_alive():
+                logger.debug("Process still alive after timeout, forcing termination")
+                self.process.terminate()
+                self.process.join(timeout=1)
+        except Exception as e:
+            logger.debug(f"Error during process shutdown: {e}")
-        if self.process.is_alive():
-            self.process.terminate()
-            self.process.join()
+        # Set stop event before closing pipes
+        self.stop_event.set()
-        # Close both sets of connections
-        self.parent_conn.close()
-        self.child_conn.close()
-        self.stdout_parent_conn.close()
-        self.stdout_child_conn.close()
+        # Close all connections
+        for conn in [
+            self.parent_conn,
+            self.child_conn,
+            self.stdout_parent_conn,
+            self.stdout_child_conn,
+        ]:
+            conn.close()
+        self.stdout_thread.join(timeout=2)  # Add timeout to thread join
+        if self.stdout_thread.is_alive():
+            logger.debug("Stdout thread failed to terminate within timeout")
+        if self.stdout_processor_task:
+            self.stdout_processor_task.cancel()
     def process_message(self, fn_name, message, cache):
         if fn_name in cache:
@@ -147,6 +180,60 @@ class SettaInMemoryFnSubprocess:
             message.content = exporter_obj.output
         return message.content
+    def start_stdout_processor_task(self):
+        if self.stdout_processor_task is None or self.stdout_processor_task.done():
+            self.stdout_processor_task = asyncio.create_task(
+                self.process_stdout_queue()
+            )
+    async def stop_stdout_processor_task(self):
+        if self.stdout_processor_task and not self.stdout_processor_task.done():
+            self.stdout_processor_task.cancel()
+            try:
+                await self.stdout_processor_task
+            except asyncio.CancelledError:
+                pass
+            self.stdout_processor_task = None
+    async def process_stdout_queue(self):
+        while not self.should_stop():
+            try:
+                if self.should_stop():
+                    break
+                if len(self.websockets) > 0:
+                    stdout_data = self.stdout_queue.get_nowait()
+                    stdout_data = stdout_data.replace("\n", "\r\n")
+                    for w in self.websockets:
+                        await w.send_text(stdout_data)
+                    self.stdout_queue.task_done()
+            except queue.Empty:
+                await asyncio.sleep(0.1)  # Check for connection every 100ms
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                if self.should_stop():
+                    break
+                logger.debug(f"Error processing stdout: {e}")
+    def stdout_listener(self):
+        while not self.should_stop():
+            if self.stdout_parent_conn.poll(0.1):  # Check for data with timeout
+                try:
+                    stdout_data = self.stdout_parent_conn.recv()
+                    self.stdout_queue.put(stdout_data)
+                except EOFError:  # Pipe was closed
+                    break
+                except Exception as e:
+                    logger.debug(f"Error in stdout listener: {e}")
+                    if self.should_stop():
+                        break
+            else:  # No data available within timeout
+                if self.should_stop():
+                    break
+    def should_stop(self):
+        return self.stop_event.is_set() or self.tasks_stop_event.is_set()
 def add_fns_from_module(fns_dict, module, module_name=None):
     count = 1
@@ -168,11 +255,11 @@ def add_fns_from_module(fns_dict, module, module_name=None):
 def get_task_metadata(in_memory_fn, exporter_obj):
     # None means run the task on every change
     if in_memory_fn.dependencies is None:
-        dependencies = None
+        dependencies = set([None])
     # Empty array means only run when the task imported.
     # Non-empty array means run when specified dependencies update.
     else:
-        dependencies = [
+        dependencies = set(
             exporter_obj.var_name_reverse_mapping[d] for d in in_memory_fn.dependencies
-        ]
-    return {"dependencies": dependencies}
+        )
+    return dependencies

setta/terminals/terminals.py CHANGED Viewed

@@ -4,6 +4,7 @@ import json
 import logging
 import platform
 import select
+import shlex
 import time
 import traceback
 from asyncio import CancelledError
@@ -44,12 +45,14 @@ def is_command_running_in_pty(pid):
 def get_terminal_shell():
     if USER_SETTINGS["backend"]["defaultTerminalShell"]:
-        return USER_SETTINGS["backend"]["defaultTerminalShell"]
+        return shlex.split(
+            USER_SETTINGS["backend"]["defaultTerminalShell"], posix=not IS_WINDOWS
+        )
     if IS_WINDOWS:
-        return "bash.exe"
+        return ["bash.exe"]
     if IS_MACOS:
-        return "zsh"
-    return "bash"
+        return ["zsh"]
+    return ["bash"]
 class TerminalWebsockets:
@@ -61,8 +64,6 @@ class TerminalWebsockets:
     def new_terminal(self, projectConfigId, sectionId, isTemporary):
         if sectionId not in self.PTY_PIDS:
             terminal_shell = get_terminal_shell()
-            if not IS_WINDOWS:
-                terminal_shell = [terminal_shell]
             pty_process = PtyProcess.spawn(terminal_shell)
             self.PTY_PIDS[sectionId] = {
                 "pid": pty_process.pid,

setta/utils/constants.py CHANGED Viewed

@@ -10,8 +10,10 @@ CONSTANTS_FOLDER = (
     Path("../../../constants") if is_dev_mode() else Path("../static/constants")
 )
 SEED_FOLDER = Path("../../../seed") if is_dev_mode() else Path("../static/seed")
-CODE_FOLDER = "setta_code"
-DB_BACKUP_FOLDER = ".setta_backups"
+CWD = Path.cwd()
+SETTA_FILES_FOLDER = CWD / "setta_files"
+CODE_FOLDER = SETTA_FILES_FOLDER / "code"
+DB_BACKUP_FOLDER = SETTA_FILES_FOLDER / "backups"
 CODE_FOLDER_ENV_VARIABLE = "SETTA_CODE_FOLDER"
 HOST_ENV_VARIABLE = "SETTA_HOST"
 PORT_ENV_VARIABLE = "SETTA_PORT"
@@ -99,6 +101,7 @@ CODE_INFO_TABLE_DATA_JSON_FIELDS = set(
         "positionalOnly",
         "isPinned",
         "isFrozen",
+        "ignoreTypeErrors",
     )
 )

setta/utils/websocket_manager.py CHANGED Viewed

@@ -43,7 +43,9 @@ class WebsocketManager:
             if wid == self.server_cli_id:
                 # process task and send any results back to requester
                 result = await tasks(
-                    message["messageType"], TaskMessage.parse_obj(message)
+                    message["messageType"],
+                    TaskMessage.parse_obj(message),
+                    websocket_manager=self,
                 )
                 if "content" in result:
                     websocket = self.sockets[fromWebsocketId]["websocket"]
@@ -64,10 +66,13 @@ class WebsocketManager:
         if "location" in message:
             self.sockets[fromWebsocketId]["location"] = message["location"]
-    async def send_message_to_requester(self, id, content):
+    async def send_message_to_requester(self, id, content, messageType=None):
         # just send data to target websocket
         websocket = self.sockets[self.message_id_to_sender_id[id]]["websocket"]
-        await websocket.send_text(json.dumps({"id": id, "content": content}))
+        return_val = {"id": id, "content": content}
+        if messageType:
+            return_val["messageType"] = messageType
+        await websocket.send_text(json.dumps(return_val))
     async def send_message_to_location(self, content, messageType, location):
         for w in self.sockets.values():

setta 0.0.2.dev0__py3-none-any.whl → 0.0.3__py3-none-any.whl

setta 0.0.2.dev0py3-none-any.whl → 0.0.3py3-none-any.whl