kodexa 7.0.12396563598__tar.gz → 7.0.12399073274__tar.gz

{kodexa-7.0.12396563598 → kodexa-7.0.12399073274}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kodexa
-Version: 7.0.12396563598
+Version: 7.0.12399073274
 Summary: Python SDK for the Kodexa Platform
 Author: Austin Redenbaugh
 Author-email: austin@kodexa.com

{kodexa-7.0.12396563598 → kodexa-7.0.12399073274}/kodexa/model/model.py

@@ -293,8 +293,6 @@ class ContentNode(object):
         if content is not None and len(self.get_content_parts()) == 0:
             self.set_content_parts([content])
 
-        self.cached_all_content = None
-
     def get_content_parts(self):
         return self.document.get_persistence().get_content_parts(self)
 
@@ -793,12 +791,13 @@ class ContentNode(object):
         result = self.select(selector, variables)
         return result[0] if len(result) > 0 else None
 
-    def select(self, selector, variables=None):
+    def select(self, selector, variables=None, first_only=False):
         """Select and return the child nodes of this node that match the selector value.
 
         Args:
           selector (str): The selector (ie. //*)
           variables (dict, optional): A dictionary of variable name/value to use in substituion; defaults to None.  Dictionary keys should match a variable specified in the selector.
+          first_only (bool, optional): If True, only the first matching node will be returned; defaults to False.
 
         Returns:
           list[ContentNode]: A list of the matching content nodes.  If no matches are found, the list will be empty.
@@ -815,18 +814,17 @@ class ContentNode(object):
         from kodexa.selectors import parse
         from kodexa.selectors.ast import SelectorContext
 
-        context = SelectorContext(self.document)
-        parsed_selector = parse(selector)
+        context = SelectorContext(self.document, first_only=first_only)
         self.document.get_persistence().flush_cache()
+        parsed_selector = parse(selector)
         return parsed_selector.resolve(self, variables, context)
 
-    def get_all_content(self, separator=" ", strip=True, use_cache=False):
+    def get_all_content(self, separator=" ", strip=True):
         """Get this node's content, concatenated with all of its children's content.
 
         Args:
           separator(str, optional): The separator to use in joining content together; defaults to " ".
           strip(boolean, optional): Strip the result
-          use_cache(boolean, optional): Use the cache
 
         Returns:
           str: The complete content for this node concatenated with the content of all child nodes.
@@ -836,9 +834,6 @@ class ContentNode(object):
             "This string is made up of multiple nodes"
         """
         s = ""
-        if self.cached_all_content is not None and use_cache:
-            return self.cached_all_content
-
         children = self.get_content_parts()
         for part in children:
             if isinstance(part, str):
@@ -861,14 +856,13 @@ class ContentNode(object):
                     s += separator
                 s += child.get_all_content(separator, strip=strip)
 
-        self.cached_all_content = s.strip() if strip else s
-        return self.cached_all_content
+        return s.strip() if strip else s
 
     def adopt_children(self, nodes_to_adopt, replace=False):
         """This will take a list of content nodes and adopt them under this node, ensuring they are re-parented.
 
         Args:
-          children (List[ContentNode]): A list of ContentNodes that will be added to the end of this node's children collection
+          nodes_to_adopt (List[ContentNode]): A list of ContentNodes that will be added to the end of this node's children collection
           replace (bool): If True, will remove all current children and replace them with the new list; defaults to True
 
         >>> # select all nodes of type 'line', then the root node 'adopts' them
@@ -3109,17 +3103,18 @@ class Document(object):
         >>> document.get_root().select_first('//*[hasTag($tagName)]', {"tagName": "div"})
            ContentNode
         """
-        result = self.select(selector, variables)
+        result = self.select(selector, variables, first_only=True)
         return result[0] if len(result) > 0 else None
 
     def select(
-            self, selector: str, variables: Optional[dict] = None
+            self, selector: str, variables: Optional[dict] = None, first_only=False
     ) -> List[ContentNode]:
         """Execute a selector on the root node and then return a list of the matching nodes.
 
         Args:
           selector (str): The selector (ie. //*)
           variables (Optional[dict): A dictionary of variable name/value to use in substituion; defaults to an empty
+          first_only (bool): If True, only the first matching node is returned; defaults to False.
           dictionary.  Dictionary keys should match a variable specified in the selector.
 
         Returns:
@@ -3131,7 +3126,7 @@ class Document(object):
         if variables is None:
             variables = {}
         if self.content_node:
-            result = self.content_node.select(selector, variables)
+            result = self.content_node.select(selector, variables, first_only)
             if isinstance(result, list):
                 return result
 

{kodexa-7.0.12396563598 → kodexa-7.0.12399073274}/kodexa/model/persistence.py

@@ -3,6 +3,7 @@ import logging
 import pathlib
 import sqlite3
 import tempfile
+import time
 import uuid
 from typing import List, Optional
 
@@ -42,6 +43,22 @@ FEATURE_TYPE_LOOKUP = "select id from f_type where name = ?"
 METADATA_INSERT = "insert into metadata(id,metadata) values (1,?)"
 METADATA_DELETE = "delete from metadata where id=1"
 
+# Configuration constants
+CACHE_SIZE = 10000  # Number of nodes to cache
+BATCH_SIZE = 1000   # Size of batches for bulk operations
+SLOW_QUERY_THRESHOLD = 1.0  # Seconds
+MAX_CONNECTIONS = 5  # Maximum number of database connections
+
+def monitor_performance(func):
+    """Performance monitoring decorator"""
+    def wrapper(*args, **kwargs):
+        start_time = time.time()
+        result = func(*args, **kwargs)
+        duration = time.time() - start_time
+        if duration > SLOW_QUERY_THRESHOLD:
+            logger.warning(f"Slow operation detected: {func.__name__}, duration: {duration}s")
+        return result
+    return wrapper
 
 class SqliteDocumentPersistence(object):
     """
@@ -57,7 +74,7 @@ class SqliteDocumentPersistence(object):
     The Sqlite persistence engine to support large scale documents (part of the V4 Kodexa Document Architecture)
     """
 
-    def __init__(self, document: Document, filename: str = None, delete_on_close=False, inmemory=False):
+    def __init__(self, document: Document, filename: str = None, delete_on_close=False, inmemory=False, persistence_manager=None):
         self.document = document
 
         self.node_types = {}
@@ -138,6 +155,7 @@ class SqliteDocumentPersistence(object):
 
         return mem_conn
 
+    @monitor_performance
     def get_all_tags(self):
         """
         Retrieves all tags from the document.
@@ -153,6 +171,7 @@ class SqliteDocumentPersistence(object):
 
         return features
 
+    @monitor_performance
     def update_features(self, node):
         """
         Updates the features of a given node in the document.
@@ -188,6 +207,7 @@ class SqliteDocumentPersistence(object):
         self.cursor.execute("DELETE FROM ft where cn_id=?", [node.uuid])
         self.cursor.executemany(FEATURE_INSERT, all_features)
 
+    @monitor_performance
     def update_node(self, node):
         """
         Updates a given node in the document.
@@ -200,6 +220,7 @@ class SqliteDocumentPersistence(object):
             [node.index, node._parent_uuid, node.uuid],
         )
 
+    @monitor_performance
     def get_content_nodes(self, node_type, parent_node: ContentNode, include_children):
         """
         Retrieves content nodes from the document based on the given parameters.
@@ -213,8 +234,8 @@ class SqliteDocumentPersistence(object):
             list: A list of content nodes that match the given parameters.
         """
         nodes = []
-
-        results = []
+        if not self.connection.in_transaction:
+            self.cursor.execute("BEGIN TRANSACTION")
         if include_children:
             if node_type == "*":
                 query = """
@@ -284,6 +305,7 @@ class SqliteDocumentPersistence(object):
                         ],
                     ).fetchall()
                 except StopIteration:
+                    self.connection.commit()
                     return []
         else:
             query = "select id, pid, nt, idx from cn where pid=? and nt=? order by idx"
@@ -300,11 +322,14 @@ class SqliteDocumentPersistence(object):
                     ],
                 ).fetchall()
             except StopIteration:
+                self.connection.commit()
                 return []
 
         for raw_node in list(results):
             nodes.append(self.__build_node(raw_node))
 
+        self.connection.commit()
+
         return nodes
 
     def initialize(self):
@@ -326,6 +351,7 @@ class SqliteDocumentPersistence(object):
             self.cursor.close()
             self.connection.close()
 
+    @monitor_performance
     def get_max_feature_id(self):
         """
         Retrieves the maximum feature id from the document.
@@ -396,6 +422,7 @@ class SqliteDocumentPersistence(object):
 
         self.__update_metadata()
 
+    @monitor_performance
     def content_node_count(self):
         """
         Counts the number of content nodes in the document.
@@ -405,6 +432,7 @@ class SqliteDocumentPersistence(object):
         """
         self.cursor.execute("select * from cn").fetchall()
 
+    @monitor_performance
     def get_feature_type_id(self, feature):
         """
         Retrieves the id of a given feature.
@@ -466,6 +494,7 @@ class SqliteDocumentPersistence(object):
 
         return result[0]
 
+    @monitor_performance
     def __insert_node(self, node: ContentNode, parent, execute=True):
         """
         Inserts a node into the document.
@@ -1385,7 +1414,7 @@ class PersistenceManager(object):
         self.node_parent_cache = {}
 
         self._underlying_persistence = SqliteDocumentPersistence(
-            document, filename, delete_on_close, inmemory=inmemory
+            document, filename, delete_on_close, inmemory=inmemory, persistence_manager=self
         )
 
     def get_steps(self) -> list[ProcessingStep]:
@@ -1523,7 +1552,6 @@ class PersistenceManager(object):
         Returns:
             List[Node]: A list of nodes tagged with the specified tag.
         """
-        self.flush_cache()
         return self._underlying_persistence.get_tagged_nodes(tag, tag_uuid)
 
     def get_all_tagged_nodes(self):
@@ -1533,7 +1561,6 @@ class PersistenceManager(object):
         Returns:
             List[Node]: A list of all tagged nodes.
         """
-        self.flush_cache()
         return self._underlying_persistence.get_all_tagged_nodes()
 
     def initialize(self):
@@ -1565,6 +1592,7 @@ class PersistenceManager(object):
         """
         self._underlying_persistence.close()
 
+    @monitor_performance
     def flush_cache(self):
         """
         Flushes the cache by merging it with the underlying persistence layer.
@@ -1574,11 +1602,14 @@ class PersistenceManager(object):
         all_content_parts = []
         all_features = []
         node_id_with_features = []
-
-        logger.debug("Merging cache to persistence")
         dirty_nodes = self.node_cache.get_dirty_objs()
 
+        if len(dirty_nodes) == 0:
+            return
+
         logger.debug(f"Identified {len(dirty_nodes)} nodes to update")
+        if not self._underlying_persistence.connection.in_transaction:
+            self._underlying_persistence.connection.execute("BEGIN TRANSACTION")
 
         next_feature_id = self._underlying_persistence.get_max_feature_id()
         for node in dirty_nodes:
@@ -1618,7 +1649,6 @@ class PersistenceManager(object):
 
                 self.node_cache.undirty(node)
 
-        logger.debug(f"Writing {len(all_node_ids)} nodes")
         self._underlying_persistence.cursor.executemany(
             "DELETE FROM cn where id=?", all_node_ids
         )
@@ -1631,14 +1661,11 @@ class PersistenceManager(object):
         self._underlying_persistence.cursor.executemany(
             "DELETE FROM cnp where cn_id=?", all_node_ids
         )
-        logger.debug(f"Writing {len(all_content_parts)} content parts")
-
         self._underlying_persistence.cursor.executemany(
             CONTENT_NODE_PART_INSERT, all_content_parts
         )
-
-        logger.debug(f"Writing {len(all_features)} features")
         self._underlying_persistence.cursor.executemany(FEATURE_INSERT, all_features)
+        self._underlying_persistence.connection.commit()
 
     def get_content_nodes(self, node_type, parent_node, include_children):
         """

{kodexa-7.0.12396563598 → kodexa-7.0.12399073274}/kodexa/selectors/ast.py

@@ -36,11 +36,12 @@ __all__ = [
 
 
 class SelectorContext:
-    def __init__(self, document: Document):
+    def __init__(self, document: Document, first_only=False):
         self.pattern_cache = {}
         self.last_op = None
         self.document: Document = document
         self.stream = 0
+        self.first_only = first_only
 
     def cache_pattern(self, pattern):
         if pattern not in self.pattern_cache:
@@ -53,20 +54,26 @@ class PipelineExpression(object):
 
     def __init__(self, left, op, right):
         self.left = left
-        """the left side of the pipeline expression"""
         self.op = op
-        """the operator of the pipeline expression"""
         self.right = right
-        """the right side of the pipeline expression"""
 
     def resolve(self, content_node: ContentNode, variables, context: SelectorContext):
         left_nodes = self.left.resolve(content_node, variables, context)
         result_nodes: List[ContentNode] = []
         context.stream = context.stream + 1
-        for node in left_nodes:
-            result_nodes.extend(self.right.resolve(node, variables, context))
+
+        # If first_only is True and we already have left nodes, only process the first one
+        nodes_to_process = left_nodes[:1] if context.first_only and left_nodes else left_nodes
+
+        for node in nodes_to_process:
+            right_results = self.right.resolve(node, variables, context)
+            result_nodes.extend(right_results)
+            # If first_only is True and we found a match, return immediately
+            if context.first_only and result_nodes:
+                break
+
         context.stream = context.stream - 1
-        return result_nodes
+        return result_nodes[:1] if context.first_only else result_nodes
 
 
 class UnaryExpression(object):
@@ -181,17 +188,12 @@ class AbsolutePath(object):
 
 
 class Step(object):
-    """
-    A single step in a relative path. a; @b; text(); parent::foo:bar[5].
-    """
+    """A single step in a relative path."""
 
     def __init__(self, axis, node_test, predicates):
         self.axis = axis
-        """the step's axis, or @ or None if abbreviated or undefined"""
         self.node_test = node_test
-        """a NameTest or NodeType object describing the test represented"""
         self.predicates = predicates
-        """a list of predicates filtering the step"""
 
     def resolve(self, obj, variables, context: SelectorContext):
         match = True
@@ -217,8 +219,9 @@ class Step(object):
                 return []
 
             nodes = self.node_test.test(axis_node, variables, context)
-
             final_nodes = []
+
+            # If first_only is True, only process until we find the first match
             for node in nodes:
                 match = True
                 for predicate in self.predicates:
@@ -230,6 +233,8 @@ class Step(object):
 
                 if match:
                     final_nodes.append(node)
+                    if context.first_only:
+                        break
 
             return final_nodes
 
@@ -247,9 +252,7 @@ class NameTest(object):
 
     def __init__(self, prefix, name):
         self.prefix = prefix
-        """the namespace prefix used for the test, or None if unset"""
         self.name = name
-        """the node name used for the test, or *"""
 
     def test(self, obj, variables, context: SelectorContext):
         if isinstance(obj, ContentNode):
@@ -257,12 +260,15 @@ class NameTest(object):
                 if self.name == "*" or self.name == obj.node_type:
                     return [obj]
             else:
-                return context.document.get_persistence().get_content_nodes(
+                nodes = context.document.get_persistence().get_content_nodes(
                     self.name, obj, context.last_op != "/"
                 )
+                # If first_only is True, return only the first matching node
+                return nodes[:1] if context.first_only else nodes
+
         if isinstance(obj, ContentFeature):
             return self.name == "*" or (
-                obj.feature_type == self.prefix and obj.name == self.name
+                    obj.feature_type == self.prefix and obj.name == self.name
             )
         return False
 

{kodexa-7.0.12396563598 → kodexa-7.0.12399073274}/kodexa/testing/test_utils.py

@@ -333,27 +333,27 @@ class ExtensionPackUtil:
         if options is None:
             options = {}
 
-        for service in self.kodexa_metadata.services:
-            if service.type == "action" and service.slug == action_slug:
+        for service in self.kodexa_metadata["services"]:
+            if service["type"] == "action" and service["slug"] == action_slug:
                 # TODO We need to validate all the options
 
-                if len(service.metadata.options) > 0:
+                if len(service["metadata"]["options"]) > 0:
                     option_names = []
-                    for option in service.metadata.options:
-                        option_names.append(option.name)
-                        if option.name not in options and option.default is not None:
-                            options[option.name] = option.default
-                        if option.required and option.name not in options:
+                    for option in service["metadata"]["options"]:
+                        option_names.append(option["name"])
+                        if option["name"] not in options and "default" in option and option["default"] is not None:
+                            options[option["name"]] = option["default"]
+                        if option["required"] and option["name"] not in options:
                             raise OptionException(
-                                f"Missing required option {option.name}"
+                                f"Missing required option {option['name']}"
                             )
 
                     for option_name in options.keys():
                         if option_name not in option_names:
                             # We need to determine if this is actually a group
                             is_group = False
-                            for check_option in service.metadata.options:
-                                if check_option["group"] is not None:
+                            for check_option in service["metadata"]["options"]:
+                                if "group" in check_option and check_option["group"] is not None:
                                     if check_option["group"]["name"] == option_name:
                                         is_group = True
 
@@ -363,8 +363,8 @@ class ExtensionPackUtil:
                                 )
 
                 # We need to create and return our action
-                module = importlib.import_module(service.step.package)
-                klass = getattr(module, service.step["class"])
+                module = importlib.import_module(service["step"]["package"])
+                klass = getattr(module, service["step"]["class"])
                 new_instance = klass(**options)
 
                 # Since we will be using to access metadata we will need to
@@ -421,15 +421,15 @@ class ExtensionPackUtil:
         if options is None:
             options = {}
 
-        for service in self.kodexa_metadata.services:
-            if service.type == "assistant" and service.slug == assistant_slug:
+        for service in self.kodexa_metadata["services"]:
+            if service["type"] == "assistant" and service["slug"] == assistant_slug:
                 # TODO We need to validate all the options
 
                 # We need to create and return our action
 
-                logger.info(f"Creating new assistant {service.assistant}")
-                module = importlib.import_module(service.assistant.package)
-                klass = getattr(module, service.assistant["class"])
+                logger.info(f"Creating new assistant {service['assistant']}")
+                module = importlib.import_module(service["assistant"]["package"])
+                klass = getattr(module, service["assistant"]["class"])
                 return klass(**options)
 
         raise Exception("Unable to find the assistant " + assistant_slug)

{kodexa-7.0.12396563598 → kodexa-7.0.12399073274}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "kodexa"
-version = "7.0.012396563598"
+version = "7.0.012399073274"
 description = "Python SDK for the Kodexa Platform"
 authors = ["Austin Redenbaugh <austin@kodexa.com>", "Philip Dodds <philip@kodexa.com>", "Romar Cablao <rcablao@kodexa.com>", "Amadea Paula Dodds <amadeapaula@kodexa.com>"]
 readme = "README.md"