kodexa 7.0.1a8003211616__py3-none-any.whl → 7.0.1a9196667375__py3-none-any.whl

kodexa/model/persistence.py

@@ -96,6 +96,14 @@ class SqliteDocumentPersistence(object):
         self.cursor.execute("pragma temp_store = memory")
         self.cursor.execute("pragma mmap_size = 30000000000")
 
+        try:
+            # We need to populate node_type_id_by_name
+            for n_type in self.cursor.execute("select id,name from n_type"):
+                self.node_types[n_type[0]] = n_type[1]
+                self.node_type_id_by_name[n_type[1]] = n_type[0]
+        except:
+            pass
+
     def create_in_memory_database(self, disk_db_path: str):
         # Connect to the in-memory database
         mem_conn = sqlite3.connect(':memory:')
@@ -106,15 +114,21 @@ class SqliteDocumentPersistence(object):
         disk_cursor = disk_conn.cursor()
 
         # Load the contents of the disk database into memory
-        disk_cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+        disk_cursor.execute("SELECT name, sql FROM sqlite_master WHERE type='table';")
         tables = disk_cursor.fetchall()
-        for table in tables:
-            table_name = table[0]
+        for table_name, create_table_sql in tables:
+            if "sqlite" in table_name:
+                continue
+
+            # Create the table structure in the in-memory database
+            mem_cursor.execute(create_table_sql)
+
+            # Populate the table with data from the disk database
             disk_cursor.execute(f"SELECT * FROM {table_name}")
             rows = disk_cursor.fetchall()
-            mem_cursor.execute(f"CREATE TABLE {table_name} AS SELECT * FROM {table_name}", [])
             for row in rows:
-                mem_cursor.execute(f"INSERT INTO {table_name} VALUES ({', '.join('?' * len(row))})", row)
+                placeholders = ', '.join('?' * len(row))
+                mem_cursor.execute(f"INSERT INTO {table_name} VALUES ({placeholders})", row)
 
         # Commit changes and close disk connection
         mem_conn.commit()
@@ -131,7 +145,7 @@ class SqliteDocumentPersistence(object):
         """
         features = []
         for feature in self.cursor.execute(
-            "select name from f_type where name like 'tag:%'"
+                "select name from f_type where name like 'tag:%'"
         ).fetchall():
             features.append(feature[0].split(":")[1])
 
@@ -730,8 +744,8 @@ class SqliteDocumentPersistence(object):
         # We need to get the child nodes
         children = []
         for child_node in self.cursor.execute(
-            "select id, pid, nt, idx from cn where pid = ? order by idx",
-            [content_node.uuid],
+                "select id, pid, nt, idx from cn where pid = ? order by idx",
+                [content_node.uuid],
         ).fetchall():
             children.append(self.__build_node(child_node))
         return children
@@ -750,8 +764,8 @@ class SqliteDocumentPersistence(object):
         # We need to get the child nodes
         children = []
         for child_node in self.cursor.execute(
-            "select id, pid, nt, idx from cn where pid = ? order by idx",
-            [content_node.uuid],
+                "select id, pid, nt, idx from cn where pid = ? order by idx",
+                [content_node.uuid],
         ).fetchall():
             children.append(child_node[0])
         return children
@@ -827,7 +841,7 @@ class SqliteDocumentPersistence(object):
     def dump_in_memory_db_to_file(self):
         # Connect to a new or existing database file
         disk_conn = sqlite3.connect(self.current_filename)
-    
+
         # Use the backup API to copy the in-memory database to the disk file
         with disk_conn:
             self.connection.backup(disk_conn)
@@ -843,10 +857,10 @@ class SqliteDocumentPersistence(object):
             bytes: The document as bytes.
         """
         self.sync()
-        
+
         if self.inmemory:
             self.dump_in_memory_db_to_file()
-        
+
         with open(self.current_filename, "rb") as f:
             return f.read()
 
@@ -864,8 +878,8 @@ class SqliteDocumentPersistence(object):
 
         features = []
         for feature in self.cursor.execute(
-            "select id, cn_id, f_type, binary_value, single from ft where cn_id = ?",
-            [node.uuid],
+                "select id, cn_id, f_type, binary_value, single from ft where cn_id = ?",
+                [node.uuid],
         ).fetchall():
             feature_type_name = self.feature_type_names[feature[2]]
             single = feature[4] == 1
@@ -914,26 +928,27 @@ class SqliteDocumentPersistence(object):
         def get_all_node_ids(node):
             """
             This function recursively traverses a node tree, collecting the ids of all non-virtual nodes.
-
-            Args:
-                node (Node): The root node to start the traversal from.
-
-            Returns:
-                list: A list of ids of all non-virtual nodes in the tree.
             """
             all_node_ids = []
             if not node.virtual:
-                all_node_ids.append([node.uuid])
+                all_node_ids.append(node.uuid)  # Append the uuid directly, not as a list
                 for child in node.get_children():
                     all_node_ids.extend(get_all_node_ids(child))
-
             return all_node_ids
 
         all_child_ids = get_all_node_ids(node)
-
-        self.cursor.executemany("delete from cnp where cn_id=?", all_child_ids)
-        self.cursor.executemany("delete from cn where id=?", all_child_ids)
-        self.cursor.executemany("delete from ft where cn_id=?", all_child_ids)
+        parameter_tuples = [(id,) for id in all_child_ids]  # Prepare the parameters as tuples
+
+        # Assuming `self.cursor` is part of a larger transaction management system
+        try:
+            self.cursor.executemany("delete from cnp where cn_id=?", parameter_tuples)
+            self.cursor.executemany("delete from cn where id=?", parameter_tuples)
+            self.cursor.executemany("delete from ft where cn_id=?", parameter_tuples)
+            self.connection.commit()  # Commit the transaction if part of one
+            return all_child_ids
+        except Exception as e:
+            self.connection.rollback()  # Rollback in case of error
+            logger.error(f"An error occurred: {e}")
 
     def remove_all_features(self, node):
         """
@@ -1087,6 +1102,26 @@ class SqliteDocumentPersistence(object):
 
         return content_nodes
 
+    def get_nodes_by_type(self, node_type):
+        """
+        Retrieves nodes of a given type from the document.
+
+        Args:
+            node_type (str): The type of the nodes to be retrieved.
+
+        Returns:
+            list: A list of nodes of the given type.
+        """
+        content_nodes = []
+
+        node_type_id = self.node_type_id_by_name.get(node_type)
+
+        query = "select id, pid, nt, idx from cn where nt = ? order by idx"
+        for content_node in self.cursor.execute(query, [node_type_id]).fetchall():
+            content_nodes.append(self.__build_node(content_node))
+
+        return content_nodes
+
 
 class SimpleObjectCache(object):
     """
@@ -1139,14 +1174,14 @@ class SimpleObjectCache(object):
         self.objs[obj.uuid] = obj
         self.dirty_objs.add(obj.uuid)
 
-    def remove_obj(self, obj):
+    def remove_obj(self, obj: ContentNode):
         """
         Remove an object from the cache.
 
         Args:
             obj (object): The object to remove.
         """
-        if obj.uuid in self.objs:
+        if obj and obj.uuid in self.objs:
             self.objs.pop(obj.uuid)
             if obj.uuid in self.dirty_objs:
                 self.dirty_objs.remove(obj.uuid)
@@ -1225,6 +1260,18 @@ class PersistenceManager(object):
             document, filename, delete_on_close, inmemory=inmemory
         )
 
+    def get_nodes_by_type(self, node_type: str) -> List[ContentNode]:
+        """
+        Retrieves all nodes of a given type from the underlying persistence layer.
+
+        Args:
+            node_type (str): The type of the nodes to be retrieved.
+
+        Returns:
+            List[ContentNode]: A list of all nodes of the given type.
+        """
+        return self._underlying_persistence.get_nodes_by_type(node_type)
+
     def get_node_by_uuid(self, uuid: int) -> ContentNode:
         """
         Retrieves a node by its uuid.
@@ -1473,6 +1520,14 @@ class PersistenceManager(object):
         if node.index is None:
             node.index = 0
 
+        # Check if the node exists in the DB
+        if node.uuid is None:
+            node.uuid = self.node_cache.next_id
+            self.node_cache.next_id += 1
+
+        if self._underlying_persistence.get_node(node.uuid) is None:
+            self._underlying_persistence.add_content_node(node, parent)
+
         if parent:
             node._parent_uuid = parent.uuid
             self.node_cache.add_obj(parent)
@@ -1486,8 +1541,8 @@ class PersistenceManager(object):
             update_child_cache = True
 
         if (
-            node.uuid in self.node_parent_cache
-            and node._parent_uuid != self.node_parent_cache[node.uuid]
+                node.uuid in self.node_parent_cache
+                and node._parent_uuid != self.node_parent_cache[node.uuid]
         ):
             # Remove from the old parent
             self.child_id_cache[self.node_parent_cache[node.uuid]].remove(node.uuid)
@@ -1505,8 +1560,8 @@ class PersistenceManager(object):
                     self.child_id_cache[node._parent_uuid].add(node.uuid)
                     current_children = self.child_cache[node._parent_uuid]
                     if (
-                        len(current_children) == 0
-                        or node.index >= current_children[-1].index
+                            len(current_children) == 0
+                            or node.index >= current_children[-1].index
                     ):
                         self.child_cache[node._parent_uuid].append(node)
                     else:
@@ -1568,7 +1623,14 @@ class PersistenceManager(object):
         self.content_parts_cache.pop(node.uuid, None)
         self.feature_cache.pop(node.uuid, None)
 
-        self._underlying_persistence.remove_content_node(node)
+        all_ids = self._underlying_persistence.remove_content_node(node)
+
+        # remove all the ids from the cache
+        for id in all_ids:
+            tmp_node = self.node_cache.get_obj(id)
+            if tmp_node is not None:
+                self.node_cache.remove_obj(tmp_node)
+            self.node_cache.dirty_objs.remove(id) if id in self.node_cache.dirty_objs else None
 
     def get_children(self, node):
         """

kodexa/pipeline/pipeline.py

@@ -132,16 +132,17 @@ class PipelineContext:
         self.cancellation_handler = cancellation_handler
 
     def update_status(
-        self, status_message: str, status_full_message: Optional[str] = None
+        self, status_message: str, progress: Optional[int] = None, progress_max: Optional[int] = None
     ):
         """Updates the status of the pipeline.
 
         Args:
             status_message (str): The status message.
-            status_full_message (str, optional): The full status message. Defaults to None.
+            progress (int, optional): The progress of the pipeline. Defaults to None.
+            progress_max (int, optional): The maximum progress of the pipeline. Defaults to None.
         """
         if self.status_handler is not None:
-            self.status_handler(status_message, status_full_message)
+            self.status_handler(status_message, progress, progress_max)
 
     def is_cancelled(self) -> bool:
         """Checks if the pipeline is cancelled.
@@ -233,14 +234,21 @@ class PipelineStep:
     """
 
     def __init__(
-        self, step, name=None, options=None, attach_source=False, step_type="ACTION"
+        self, step, name=None, options=None, attach_source=False, step_type="ACTION", conditional=None
     ):
         if options is None:
-            options = {}
+            try:
+                if 'options' in step:
+                    options = step['options']
+                else:
+                    options = {}
+            except:
+                options = {}
         self.step = step
         self.name = name
         self.options = options
         self.step_type = step_type
+        self.conditional = conditional
 
         if str(type(self.step)) == "<class 'type'>":
             logger.info(f"Adding new step class {step.__name__} to pipeline")
@@ -253,7 +261,10 @@ class PipelineStep:
             from kodexa import RemoteStep
 
             self.step = RemoteStep(
-                step, step_type=step_type, options=options, attach_source=attach_source
+                step, step_type=step_type,
+                options=options,
+                attach_source=attach_source,
+                conditional=conditional
             )
         else:
             logger.info(f"Adding new step {type(step)} to pipeline")
@@ -273,7 +284,7 @@ class PipelineStep:
                     "You can not yet deploy a pipeline with a class instance style step"
                 )
             if isinstance(self.step, str):
-                return {"ref": self.step, "options": self.options}
+                return {"ref": self.step, "options": self.options, "conditional": self.conditional}
             if callable(self.step):
                 metadata = {
                     "function": self.step.__name__,
@@ -284,6 +295,8 @@ class PipelineStep:
 
             metadata["name"] = self.name
             metadata["stepType"] = self.step_type
+            metadata["options"] = self.options
+            metadata["conditional"] = self.conditional
             return metadata
         except AttributeError as e:
             raise Exception("All steps must implement to_dict() for deployment", e)
@@ -478,7 +491,7 @@ class Pipeline:
         return self
 
     def add_step(
-        self, step, name=None, options=None, attach_source=False, step_type="ACTION"
+        self, step, name=None, options=None, attach_source=False, step_type="ACTION", conditional=None
     ):
         """Add the given step to the current pipeline.
 
@@ -488,12 +501,18 @@ class Pipeline:
             options (optional): Options to be passed to the step if it is a simplified remote action. Defaults to None.
             attach_source (bool, optional): If step is simplified remote action this determines if we need to add the source. Defaults to False.
             step_type (str, optional): The type of step to add, can either be an ACTION or MODEL. Defaults to 'ACTION'.
-
+            conditional (str, optional): The conditional to use for the step. Defaults to None.
         Returns:
             Pipeline: The instance of the pipeline.
         """
         if options is None:
-            options = {}
+            try:
+                if 'options' in step:
+                    options = step['options']
+                else:
+                    options = {}
+            except:
+                options = {}
         self.steps.append(
             PipelineStep(
                 step=step,
@@ -501,6 +520,7 @@ class Pipeline:
                 options=options,
                 attach_source=attach_source,
                 step_type=step_type,
+                conditional=conditional
             )
         )