dbos 0.25.0a12__py3-none-any.whl → 0.25.0a14__py3-none-any.whl

dbos/_admin_server.py

@@ -7,8 +7,6 @@ from functools import partial
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from typing import TYPE_CHECKING, Any, List, TypedDict
 
-import jsonpickle  # type: ignore
-
 from ._logger import dbos_logger
 from ._recovery import recover_pending_workflows
 
@@ -162,7 +160,17 @@ class AdminRequestHandler(BaseHTTPRequestHandler):
 
     def _handle_steps(self, workflow_id: str) -> None:
         steps = self.dbos._sys_db.get_workflow_steps(workflow_id)
-        json_steps = jsonpickle.encode(steps, unpicklable=False).encode("utf-8")
+
+        updated_steps = [
+            {
+                **step,
+                "output": str(step["output"]) if step["output"] is not None else None,
+                "error": str(step["error"]) if step["error"] is not None else None,
+            }
+            for step in steps
+        ]
+
+        json_steps = json.dumps(updated_steps).encode("utf-8")
         self.send_response(200)
         self._end_headers()
         self.wfile.write(json_steps)

dbos/_core.py

@@ -279,25 +279,14 @@ def _init_workflow(
             raise DBOSNonExistentWorkflowError(wfid)
         wf_status = get_status_result["status"]
     else:
-        if temp_wf_type != "transaction" or queue is not None:
-            # Synchronously record the status and inputs for workflows and single-step workflows
-            # We also have to do this for single-step workflows because of the foreign key constraint on the operation outputs table
-            # TODO: Make this transactional (and with the queue step below)
-            wf_status = dbos._sys_db.insert_workflow_status(
-                status, max_recovery_attempts=max_recovery_attempts
-            )
-
-            # TODO: Modify the inputs if they were changed by `update_workflow_inputs`
-            dbos._sys_db.update_workflow_inputs(
-                wfid, _serialization.serialize_args(inputs)
-            )
+        # Synchronously record the status and inputs for workflows
+        # TODO: Make this transactional (and with the queue step below)
+        wf_status = dbos._sys_db.insert_workflow_status(
+            status, max_recovery_attempts=max_recovery_attempts
+        )
 
-        else:
-            # Buffer the inputs for single-transaction workflows, but don't buffer the status
-
-            dbos._sys_db.buffer_workflow_inputs(
-                wfid, _serialization.serialize_args(inputs)
-            )
+        # TODO: Modify the inputs if they were changed by `update_workflow_inputs`
+        dbos._sys_db.update_workflow_inputs(wfid, _serialization.serialize_args(inputs))
 
         if queue is not None and wf_status == WorkflowStatusString.ENQUEUED.value:
             dbos._sys_db.enqueue(wfid, queue)
@@ -311,6 +300,18 @@ def _get_wf_invoke_func(
     status: WorkflowStatusInternal,
 ) -> Callable[[Callable[[], R]], R]:
     def persist(func: Callable[[], R]) -> R:
+        if not dbos.debug_mode and (
+            status["status"] == WorkflowStatusString.ERROR.value
+            or status["status"] == WorkflowStatusString.SUCCESS.value
+        ):
+            dbos.logger.debug(
+                f"Workflow {status['workflow_uuid']} is already completed with status {status['status']}"
+            )
+            # Directly return the result if the workflow is already completed
+            recorded_result: R = dbos._sys_db.await_workflow_result(
+                status["workflow_uuid"]
+            )
+            return recorded_result
         try:
             output = func()
             status["status"] = "SUCCESS"
@@ -319,7 +320,7 @@ def _get_wf_invoke_func(
                 if status["queue_name"] is not None:
                     queue = dbos._registry.queue_info_map[status["queue_name"]]
                     dbos._sys_db.remove_from_queue(status["workflow_uuid"], queue)
-                dbos._sys_db.buffer_workflow_status(status)
+                dbos._sys_db.update_workflow_status(status)
             return output
         except DBOSWorkflowConflictIDError:
             # Await the workflow result

dbos/_dbos.py

@@ -485,14 +485,6 @@ class DBOS:
             notification_listener_thread.start()
             self._background_threads.append(notification_listener_thread)
 
-            # Start flush workflow buffers thread
-            flush_workflow_buffers_thread = threading.Thread(
-                target=self._sys_db.flush_workflow_buffers,
-                daemon=True,
-            )
-            flush_workflow_buffers_thread.start()
-            self._background_threads.append(flush_workflow_buffers_thread)
-
             # Start the queue thread
             evt = threading.Event()
             self.stop_events.append(evt)
@@ -549,7 +541,9 @@ class DBOS:
         if _dbos_global_instance is not None:
             _dbos_global_instance._reset_system_database()
         else:
-            dbos_logger.warning("reset_system_database has no effect because global DBOS object does not exist")
+            dbos_logger.warning(
+                "reset_system_database has no effect because global DBOS object does not exist"
+            )
 
     def _reset_system_database(self) -> None:
         assert (

dbos/_migrations/versions/f4b9b32ba814_functionname_childid_op_outputs.py

@@ -25,7 +25,7 @@ def upgrade() -> None:
             "function_name",
             sa.Text(),
             nullable=False,
-            default="",
+            server_default="",
         ),
         schema="dbos",
     )

dbos/_sys_db.py

@@ -166,8 +166,6 @@ class StepInfo(TypedDict):
 
 
 _dbos_null_topic = "__null__topic__"
-_buffer_flush_batch_size = 100
-_buffer_flush_interval_secs = 1.0
 
 
 class SystemDatabase:
@@ -264,32 +262,17 @@ class SystemDatabase:
         self.notifications_map: Dict[str, threading.Condition] = {}
         self.workflow_events_map: Dict[str, threading.Condition] = {}
 
-        # Initialize the workflow status and inputs buffers
-        self._workflow_status_buffer: Dict[str, WorkflowStatusInternal] = {}
-        self._workflow_inputs_buffer: Dict[str, str] = {}
-        # Two sets for tracking which single-transaction workflows have been exported to the status table
-        self._exported_temp_txn_wf_status: Set[str] = set()
-        self._temp_txn_wf_ids: Set[str] = set()
-        self._is_flushing_status_buffer = False
-
         # Now we can run background processes
         self._run_background_processes = True
         self._debug_mode = debug_mode
 
     # Destroy the pool when finished
     def destroy(self) -> None:
-        self.wait_for_buffer_flush()
         self._run_background_processes = False
         if self.notification_conn is not None:
             self.notification_conn.close()
         self.engine.dispose()
 
-    def wait_for_buffer_flush(self) -> None:
-        # Wait until the buffers are flushed.
-        while self._is_flushing_status_buffer or not self._is_buffers_empty:
-            dbos_logger.debug("Waiting for system buffers to be exported")
-            time.sleep(1)
-
     def insert_workflow_status(
         self,
         status: WorkflowStatusInternal,
@@ -440,10 +423,6 @@ class SystemDatabase:
             with self.engine.begin() as c:
                 c.execute(cmd)
 
-        # If this is a single-transaction workflow, record that its status has been exported
-        if status["workflow_uuid"] in self._temp_txn_wf_ids:
-            self._exported_temp_txn_wf_status.add(status["workflow_uuid"])
-
     def cancel_workflow(
         self,
         workflow_id: str,
@@ -621,10 +600,7 @@ class SystemDatabase:
                     f"Workflow {workflow_uuid} has been called multiple times with different inputs"
                 )
             # TODO: actually changing the input
-        if workflow_uuid in self._temp_txn_wf_ids:
-            # Clean up the single-transaction tracking sets
-            self._exported_temp_txn_wf_status.discard(workflow_uuid)
-            self._temp_txn_wf_ids.discard(workflow_uuid)
+
         return
 
     def get_workflow_inputs(
@@ -1275,106 +1251,6 @@ class SystemDatabase:
             )
         return value
 
-    def _flush_workflow_status_buffer(self) -> None:
-        if self._debug_mode:
-            raise Exception("called _flush_workflow_status_buffer in debug mode")
-
-        """Export the workflow status buffer to the database, up to the batch size."""
-        if len(self._workflow_status_buffer) == 0:
-            return
-
-        # Record the exported status so far, and add them back on errors.
-        exported_status: Dict[str, WorkflowStatusInternal] = {}
-        with self.engine.begin() as c:
-            exported = 0
-            status_iter = iter(list(self._workflow_status_buffer))
-            wf_id: Optional[str] = None
-            while (
-                exported < _buffer_flush_batch_size
-                and (wf_id := next(status_iter, None)) is not None
-            ):
-                # Pop the first key in the buffer (FIFO)
-                status = self._workflow_status_buffer.pop(wf_id, None)
-                if status is None:
-                    continue
-                exported_status[wf_id] = status
-                try:
-                    self.update_workflow_status(status, conn=c)
-                    exported += 1
-                except Exception as e:
-                    dbos_logger.error(f"Error while flushing status buffer: {e}")
-                    c.rollback()
-                    # Add the exported status back to the buffer, so they can be retried next time
-                    self._workflow_status_buffer.update(exported_status)
-                    break
-
-    def _flush_workflow_inputs_buffer(self) -> None:
-        if self._debug_mode:
-            raise Exception("called _flush_workflow_inputs_buffer in debug mode")
-
-        """Export the workflow inputs buffer to the database, up to the batch size."""
-        if len(self._workflow_inputs_buffer) == 0:
-            return
-
-        # Record exported inputs so far, and add them back on errors.
-        exported_inputs: Dict[str, str] = {}
-        with self.engine.begin() as c:
-            exported = 0
-            input_iter = iter(list(self._workflow_inputs_buffer))
-            wf_id: Optional[str] = None
-            while (
-                exported < _buffer_flush_batch_size
-                and (wf_id := next(input_iter, None)) is not None
-            ):
-                if wf_id not in self._exported_temp_txn_wf_status:
-                    # Skip exporting inputs if the status has not been exported yet
-                    continue
-                inputs = self._workflow_inputs_buffer.pop(wf_id, None)
-                if inputs is None:
-                    continue
-                exported_inputs[wf_id] = inputs
-                try:
-                    self.update_workflow_inputs(wf_id, inputs, conn=c)
-                    exported += 1
-                except Exception as e:
-                    dbos_logger.error(f"Error while flushing inputs buffer: {e}")
-                    c.rollback()
-                    # Add the exported inputs back to the buffer, so they can be retried next time
-                    self._workflow_inputs_buffer.update(exported_inputs)
-                    break
-
-    def flush_workflow_buffers(self) -> None:
-        """Flush the workflow status and inputs buffers periodically, via a background thread."""
-        while self._run_background_processes:
-            try:
-                self._is_flushing_status_buffer = True
-                # Must flush the status buffer first, as the inputs table has a foreign key constraint on the status table.
-                self._flush_workflow_status_buffer()
-                self._flush_workflow_inputs_buffer()
-                self._is_flushing_status_buffer = False
-                if self._is_buffers_empty:
-                    # Only sleep if both buffers are empty
-                    time.sleep(_buffer_flush_interval_secs)
-            except Exception as e:
-                dbos_logger.error(f"Error while flushing buffers: {e}")
-                time.sleep(_buffer_flush_interval_secs)
-                # Will retry next time
-
-    def buffer_workflow_status(self, status: WorkflowStatusInternal) -> None:
-        self._workflow_status_buffer[status["workflow_uuid"]] = status
-
-    def buffer_workflow_inputs(self, workflow_id: str, inputs: str) -> None:
-        # inputs is a serialized WorkflowInputs string
-        self._workflow_inputs_buffer[workflow_id] = inputs
-        self._temp_txn_wf_ids.add(workflow_id)
-
-    @property
-    def _is_buffers_empty(self) -> bool:
-        return (
-            len(self._workflow_status_buffer) == 0
-            and len(self._workflow_inputs_buffer) == 0
-        )
-
     def enqueue(self, workflow_id: str, queue_name: str) -> None:
         if self._debug_mode:
             raise Exception("called enqueue in debug mode")

{dbos-0.25.0a12.dist-info → dbos-0.25.0a14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbos
-Version: 0.25.0a12
+Version: 0.25.0a14
 Summary: Ultra-lightweight durable execution in Python
 Author-Email: "DBOS, Inc." <contact@dbos.dev>
 License: MIT

{dbos-0.25.0a12.dist-info → dbos-0.25.0a14.dist-info}/RECORD

@@ -1,10 +1,10 @@
-dbos-0.25.0a12.dist-info/METADATA,sha256=_7a_sxE2zkGk8RlELQfYhjJ9az9uFS4FUNHA2haL3No,5554
-dbos-0.25.0a12.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
-dbos-0.25.0a12.dist-info/entry_points.txt,sha256=_QOQ3tVfEjtjBlr1jS4sHqHya9lI2aIEIWkz8dqYp14,58
-dbos-0.25.0a12.dist-info/licenses/LICENSE,sha256=VGZit_a5-kdw9WT6fY5jxAWVwGQzgLFyPWrcVVUhVNU,1067
+dbos-0.25.0a14.dist-info/METADATA,sha256=47yo_N6VbWT0oM9z9vavQ9nIVGLzzerDaIJXqeAyI_Q,5554
+dbos-0.25.0a14.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
+dbos-0.25.0a14.dist-info/entry_points.txt,sha256=_QOQ3tVfEjtjBlr1jS4sHqHya9lI2aIEIWkz8dqYp14,58
+dbos-0.25.0a14.dist-info/licenses/LICENSE,sha256=VGZit_a5-kdw9WT6fY5jxAWVwGQzgLFyPWrcVVUhVNU,1067
 dbos/__init__.py,sha256=2Ur8QyNElSVn7CeL9Ovek2Zsye8A_ZCyjb9djF-N4A4,785
 dbos/__main__.py,sha256=G7Exn-MhGrVJVDbgNlpzhfh8WMX_72t3_oJaFT9Lmt8,653
-dbos/_admin_server.py,sha256=7kguOf9jEt4vg9LO-QJdh4jYddp6Uqtrt14gh7mKA2Y,6387
+dbos/_admin_server.py,sha256=FLUacm9WGIPjB5s3QhdpMCilc8JHJOF0KMNStF82qs0,6625
 dbos/_app_db.py,sha256=4EGrYL14rVx96TXn34hoibN9ltf4-2DKcj6nd-HvBxA,6262
 dbos/_classproperty.py,sha256=f0X-_BySzn3yFDRKB2JpCbLYQ9tLwt1XftfshvY7CBs,626
 dbos/_cloudutils/authentication.py,sha256=V0fCWQN9stCkhbuuxgPTGpvuQcDqfU3KAxPAh01vKW4,5007
@@ -13,10 +13,10 @@ dbos/_cloudutils/databases.py,sha256=_shqaqSvhY4n2ScgQ8IP5PDZvzvcx3YBKV8fj-cxhSY
 dbos/_conductor/conductor.py,sha256=7elKINsgl4s1Tg5DwrU-K7xQ5vQvmDAIfAvUgfwpGN0,16784
 dbos/_conductor/protocol.py,sha256=xN7pmooyF1pqbH1b6WhllU5718P7zSb_b0KCwA6bzcs,6716
 dbos/_context.py,sha256=3He4w46OTFbR7h8U1MLcdaU10wNyIPBSRqzLkdggv7U,19368
-dbos/_core.py,sha256=llBq1lW2spbRDa0ICgVdtgAxVwmGjYNtdXoOVut6xKQ,45732
+dbos/_core.py,sha256=kIj_4wlIff8ptlACJKXAPSNoyJIt2h44swjMKxfwv0k,45789
 dbos/_croniter.py,sha256=XHAyUyibs_59sJQfSNWkP7rqQY6_XrlfuuCxk4jYqek,47559
 dbos/_db_wizard.py,sha256=VnMa6OL87Lc-XPDD1RnXp8NjsJE8YgiQLj3wtWAXp-8,8252
-dbos/_dbos.py,sha256=YXQjNLR9SgOr2Y9R3tMBA_DkXBJPECzYRTaz-i9GNWA,45458
+dbos/_dbos.py,sha256=IZ3Qj8UBUqYYJiG6HVz35A0GQnQyI1Po_-E6DDvWlQg,45147
 dbos/_dbos_config.py,sha256=7Qm3FARP3lTKZS0gSxDHLbpaDCT30GzfyERxfCde4bc,21566
 dbos/_debug.py,sha256=mmgvLkqlrljMBBow9wk01PPur9kUf2rI_11dTJXY4gw,1822
 dbos/_error.py,sha256=B6Y9XLS1f6yrawxB2uAEYFMxFwk9BHhdxPNddKco-Fw,5399
@@ -34,7 +34,7 @@ dbos/_migrations/versions/a3b18ad34abe_added_triggers.py,sha256=Rv0ZsZYZ_WdgGEUL
 dbos/_migrations/versions/d76646551a6b_job_queue_limiter.py,sha256=8PyFi8rd6CN-mUro43wGhsg5wcQWKZPRHD6jw8R5pVc,986
 dbos/_migrations/versions/d76646551a6c_workflow_queue.py,sha256=G942nophZ2uC2vc4hGBC02Ptng1715roTjY3xiyzZU4,729
 dbos/_migrations/versions/eab0cc1d9a14_job_queue.py,sha256=uvhFOtqbBreCePhAxZfIT0qCAI7BiZTou9wt6QnbY7c,1412
-dbos/_migrations/versions/f4b9b32ba814_functionname_childid_op_outputs.py,sha256=rwfMdqL6LYI4RwtxmB-OJbRKP_wQi-OA8fsmxley-OM,1042
+dbos/_migrations/versions/f4b9b32ba814_functionname_childid_op_outputs.py,sha256=m90Lc5YH0ZISSq1MyxND6oq3RZrZKrIqEsZtwJ1jWxA,1049
 dbos/_outcome.py,sha256=EXxBg4jXCVJsByDQ1VOCIedmbeq_03S6d-p1vqQrLFU,6810
 dbos/_queue.py,sha256=yYwKCjxCSFjtD63vpnRQmb835BZAe9UATgWjMW6dcnY,3341
 dbos/_recovery.py,sha256=4KyZb0XJEUGH7ekYT1kpx38i6y5vygPeH75Ta7RZjYo,2596
@@ -46,7 +46,7 @@ dbos/_schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbos/_schemas/application_database.py,sha256=KeyoPrF7hy_ODXV7QNike_VFSD74QBRfQ76D7QyE9HI,966
 dbos/_schemas/system_database.py,sha256=W9eSpL7SZzQkxcEZ4W07BOcwkkDr35b9oCjUOgfHWek,5336
 dbos/_serialization.py,sha256=YCYv0qKAwAZ1djZisBC7khvKqG-5OcIv9t9EC5PFIog,1743
-dbos/_sys_db.py,sha256=k9fMfg6daAQB012uaPwgkiBhA0wPON8PIwY5Ki9_7Io,68208
+dbos/_sys_db.py,sha256=33euvXfpbpVaCpR0Sx5eQ4yBt1gRKGdvfGQUugqoJBY,62320
 dbos/_templates/dbos-db-starter/README.md,sha256=GhxhBj42wjTt1fWEtwNriHbJuKb66Vzu89G4pxNHw2g,930
 dbos/_templates/dbos-db-starter/__package/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbos/_templates/dbos-db-starter/__package/main.py,sha256=nJMN3ZD2lmwg4Dcgmiwqc-tQGuCJuJal2Xl85iA277U,2453
@@ -66,4 +66,4 @@ dbos/cli/cli.py,sha256=ut47q-R6A423je0zvBTEgwdxENagaKKoyIvyTeACFIU,15977
 dbos/dbos-config.schema.json,sha256=HtF_njVTGHLdzBGZ4OrGQz3qbPPT0Go-iwd1PgFVTNg,5847
 dbos/py.typed,sha256=QfzXT1Ktfk3Rj84akygc7_42z0lRpCq0Ilh8OXI6Zas,44
 version/__init__.py,sha256=L4sNxecRuqdtSFdpUGX3TtBi9KL3k7YsZVIvv-fv9-A,1678
-dbos-0.25.0a12.dist-info/RECORD,,
+dbos-0.25.0a14.dist-info/RECORD,,