kodexa 7.0.12397101638__py3-none-any.whl → 7.0.12399073274__py3-none-any.whl

kodexa/model/model.py

@@ -815,8 +815,8 @@ class ContentNode(object):
         from kodexa.selectors.ast import SelectorContext
 
         context = SelectorContext(self.document, first_only=first_only)
-        parsed_selector = parse(selector)
         self.document.get_persistence().flush_cache()
+        parsed_selector = parse(selector)
         return parsed_selector.resolve(self, variables, context)
 
     def get_all_content(self, separator=" ", strip=True):

kodexa/model/persistence.py

@@ -3,6 +3,7 @@ import logging
 import pathlib
 import sqlite3
 import tempfile
+import time
 import uuid
 from typing import List, Optional
 
@@ -42,6 +43,22 @@ FEATURE_TYPE_LOOKUP = "select id from f_type where name = ?"
 METADATA_INSERT = "insert into metadata(id,metadata) values (1,?)"
 METADATA_DELETE = "delete from metadata where id=1"
 
+# Configuration constants
+CACHE_SIZE = 10000  # Number of nodes to cache
+BATCH_SIZE = 1000   # Size of batches for bulk operations
+SLOW_QUERY_THRESHOLD = 1.0  # Seconds
+MAX_CONNECTIONS = 5  # Maximum number of database connections
+
+def monitor_performance(func):
+    """Performance monitoring decorator"""
+    def wrapper(*args, **kwargs):
+        start_time = time.time()
+        result = func(*args, **kwargs)
+        duration = time.time() - start_time
+        if duration > SLOW_QUERY_THRESHOLD:
+            logger.warning(f"Slow operation detected: {func.__name__}, duration: {duration}s")
+        return result
+    return wrapper
 
 class SqliteDocumentPersistence(object):
     """
@@ -57,7 +74,7 @@ class SqliteDocumentPersistence(object):
     The Sqlite persistence engine to support large scale documents (part of the V4 Kodexa Document Architecture)
     """
 
-    def __init__(self, document: Document, filename: str = None, delete_on_close=False, inmemory=False):
+    def __init__(self, document: Document, filename: str = None, delete_on_close=False, inmemory=False, persistence_manager=None):
         self.document = document
 
         self.node_types = {}
@@ -138,6 +155,7 @@ class SqliteDocumentPersistence(object):
 
         return mem_conn
 
+    @monitor_performance
     def get_all_tags(self):
         """
         Retrieves all tags from the document.
@@ -153,6 +171,7 @@ class SqliteDocumentPersistence(object):
 
         return features
 
+    @monitor_performance
     def update_features(self, node):
         """
         Updates the features of a given node in the document.
@@ -188,6 +207,7 @@ class SqliteDocumentPersistence(object):
         self.cursor.execute("DELETE FROM ft where cn_id=?", [node.uuid])
         self.cursor.executemany(FEATURE_INSERT, all_features)
 
+    @monitor_performance
     def update_node(self, node):
         """
         Updates a given node in the document.
@@ -200,6 +220,7 @@ class SqliteDocumentPersistence(object):
             [node.index, node._parent_uuid, node.uuid],
         )
 
+    @monitor_performance
     def get_content_nodes(self, node_type, parent_node: ContentNode, include_children):
         """
         Retrieves content nodes from the document based on the given parameters.
@@ -213,8 +234,8 @@ class SqliteDocumentPersistence(object):
             list: A list of content nodes that match the given parameters.
         """
         nodes = []
-
-        results = []
+        if not self.connection.in_transaction:
+            self.cursor.execute("BEGIN TRANSACTION")
         if include_children:
             if node_type == "*":
                 query = """
@@ -284,6 +305,7 @@ class SqliteDocumentPersistence(object):
                         ],
                     ).fetchall()
                 except StopIteration:
+                    self.connection.commit()
                     return []
         else:
             query = "select id, pid, nt, idx from cn where pid=? and nt=? order by idx"
@@ -300,11 +322,14 @@ class SqliteDocumentPersistence(object):
                     ],
                 ).fetchall()
             except StopIteration:
+                self.connection.commit()
                 return []
 
         for raw_node in list(results):
             nodes.append(self.__build_node(raw_node))
 
+        self.connection.commit()
+
         return nodes
 
     def initialize(self):
@@ -326,6 +351,7 @@ class SqliteDocumentPersistence(object):
             self.cursor.close()
             self.connection.close()
 
+    @monitor_performance
     def get_max_feature_id(self):
         """
         Retrieves the maximum feature id from the document.
@@ -396,6 +422,7 @@ class SqliteDocumentPersistence(object):
 
         self.__update_metadata()
 
+    @monitor_performance
     def content_node_count(self):
         """
         Counts the number of content nodes in the document.
@@ -405,6 +432,7 @@ class SqliteDocumentPersistence(object):
         """
         self.cursor.execute("select * from cn").fetchall()
 
+    @monitor_performance
     def get_feature_type_id(self, feature):
         """
         Retrieves the id of a given feature.
@@ -466,6 +494,7 @@ class SqliteDocumentPersistence(object):
 
         return result[0]
 
+    @monitor_performance
     def __insert_node(self, node: ContentNode, parent, execute=True):
         """
         Inserts a node into the document.
@@ -1385,7 +1414,7 @@ class PersistenceManager(object):
         self.node_parent_cache = {}
 
         self._underlying_persistence = SqliteDocumentPersistence(
-            document, filename, delete_on_close, inmemory=inmemory
+            document, filename, delete_on_close, inmemory=inmemory, persistence_manager=self
         )
 
     def get_steps(self) -> list[ProcessingStep]:
@@ -1523,7 +1552,6 @@ class PersistenceManager(object):
         Returns:
             List[Node]: A list of nodes tagged with the specified tag.
         """
-        self.flush_cache()
         return self._underlying_persistence.get_tagged_nodes(tag, tag_uuid)
 
     def get_all_tagged_nodes(self):
@@ -1533,7 +1561,6 @@ class PersistenceManager(object):
         Returns:
             List[Node]: A list of all tagged nodes.
         """
-        self.flush_cache()
         return self._underlying_persistence.get_all_tagged_nodes()
 
     def initialize(self):
@@ -1565,6 +1592,7 @@ class PersistenceManager(object):
         """
         self._underlying_persistence.close()
 
+    @monitor_performance
     def flush_cache(self):
         """
         Flushes the cache by merging it with the underlying persistence layer.
@@ -1574,11 +1602,14 @@ class PersistenceManager(object):
         all_content_parts = []
         all_features = []
         node_id_with_features = []
-
-        logger.debug("Merging cache to persistence")
         dirty_nodes = self.node_cache.get_dirty_objs()
 
+        if len(dirty_nodes) == 0:
+            return
+
         logger.debug(f"Identified {len(dirty_nodes)} nodes to update")
+        if not self._underlying_persistence.connection.in_transaction:
+            self._underlying_persistence.connection.execute("BEGIN TRANSACTION")
 
         next_feature_id = self._underlying_persistence.get_max_feature_id()
         for node in dirty_nodes:
@@ -1618,7 +1649,6 @@ class PersistenceManager(object):
 
                 self.node_cache.undirty(node)
 
-        logger.debug(f"Writing {len(all_node_ids)} nodes")
         self._underlying_persistence.cursor.executemany(
             "DELETE FROM cn where id=?", all_node_ids
         )
@@ -1631,14 +1661,11 @@ class PersistenceManager(object):
         self._underlying_persistence.cursor.executemany(
             "DELETE FROM cnp where cn_id=?", all_node_ids
         )
-        logger.debug(f"Writing {len(all_content_parts)} content parts")
-
         self._underlying_persistence.cursor.executemany(
             CONTENT_NODE_PART_INSERT, all_content_parts
         )
-
-        logger.debug(f"Writing {len(all_features)} features")
         self._underlying_persistence.cursor.executemany(FEATURE_INSERT, all_features)
+        self._underlying_persistence.connection.commit()
 
     def get_content_nodes(self, node_type, parent_node, include_children):
         """

kodexa/testing/test_utils.py

@@ -333,27 +333,27 @@ class ExtensionPackUtil:
         if options is None:
             options = {}
 
-        for service in self.kodexa_metadata.services:
-            if service.type == "action" and service.slug == action_slug:
+        for service in self.kodexa_metadata["services"]:
+            if service["type"] == "action" and service["slug"] == action_slug:
                 # TODO We need to validate all the options
 
-                if len(service.metadata.options) > 0:
+                if len(service["metadata"]["options"]) > 0:
                     option_names = []
-                    for option in service.metadata.options:
-                        option_names.append(option.name)
-                        if option.name not in options and option.default is not None:
-                            options[option.name] = option.default
-                        if option.required and option.name not in options:
+                    for option in service["metadata"]["options"]:
+                        option_names.append(option["name"])
+                        if option["name"] not in options and "default" in option and option["default"] is not None:
+                            options[option["name"]] = option["default"]
+                        if option["required"] and option["name"] not in options:
                             raise OptionException(
-                                f"Missing required option {option.name}"
+                                f"Missing required option {option['name']}"
                             )
 
                     for option_name in options.keys():
                         if option_name not in option_names:
                             # We need to determine if this is actually a group
                             is_group = False
-                            for check_option in service.metadata.options:
-                                if check_option["group"] is not None:
+                            for check_option in service["metadata"]["options"]:
+                                if "group" in check_option and check_option["group"] is not None:
                                     if check_option["group"]["name"] == option_name:
                                         is_group = True
 
@@ -363,8 +363,8 @@ class ExtensionPackUtil:
                                 )
 
                 # We need to create and return our action
-                module = importlib.import_module(service.step.package)
-                klass = getattr(module, service.step["class"])
+                module = importlib.import_module(service["step"]["package"])
+                klass = getattr(module, service["step"]["class"])
                 new_instance = klass(**options)
 
                 # Since we will be using to access metadata we will need to
@@ -421,15 +421,15 @@ class ExtensionPackUtil:
         if options is None:
             options = {}
 
-        for service in self.kodexa_metadata.services:
-            if service.type == "assistant" and service.slug == assistant_slug:
+        for service in self.kodexa_metadata["services"]:
+            if service["type"] == "assistant" and service["slug"] == assistant_slug:
                 # TODO We need to validate all the options
 
                 # We need to create and return our action
 
-                logger.info(f"Creating new assistant {service.assistant}")
-                module = importlib.import_module(service.assistant.package)
-                klass = getattr(module, service.assistant["class"])
+                logger.info(f"Creating new assistant {service['assistant']}")
+                module = importlib.import_module(service["assistant"]["package"])
+                klass = getattr(module, service["assistant"]["class"])
                 return klass(**options)
 
         raise Exception("Unable to find the assistant " + assistant_slug)

{kodexa-7.0.12397101638.dist-info → kodexa-7.0.12399073274.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kodexa
-Version: 7.0.12397101638
+Version: 7.0.12399073274
 Summary: Python SDK for the Kodexa Platform
 Author: Austin Redenbaugh
 Author-email: austin@kodexa.com

{kodexa-7.0.12397101638.dist-info → kodexa-7.0.12399073274.dist-info}/RECORD

@@ -11,9 +11,9 @@ kodexa/model/entities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
 kodexa/model/entities/check_response.py,sha256=eqBHxO6G2OAziL3p9bHGI-oiPkAG82H6Choc8wyvtM4,3949
 kodexa/model/entities/product.py,sha256=ZDpHuBE_9FJ-klnkyBvTfPwYOqBkM1wraZMtHqNA8FQ,3526
 kodexa/model/entities/product_subscription.py,sha256=UcmWR-qgLfdV7VCtJNwzgkanoS8nBSL6ngVuxQUK1M8,3810
-kodexa/model/model.py,sha256=OfBUVYRRbW685NNu-lAgp0Z9z4ksiX0yuySSCis_NpM,118647
+kodexa/model/model.py,sha256=Bjy1n6_9kQLLch-wCUDpT31sTctE2kCGdi4cb1ptZN4,118647
 kodexa/model/objects.py,sha256=4Oyjs6omlHfwziAK1m2tFk4jSnzN7lFdXACog07ed1c,185124
-kodexa/model/persistence.py,sha256=AVITArEFBnl3UWL0-L7dz_Y35sxaLq0Dys-vFV_kH4k,68471
+kodexa/model/persistence.py,sha256=Kai1M51TM56G0_0ol7OIVf40cnJXzpZ3NHAa20uh5Co,69554
 kodexa/model/utils.py,sha256=6R-3rFiW9irBwj0Mq5yhp7EDXkNUFaeFhr3bWmnlW4g,2961
 kodexa/pipeline/__init__.py,sha256=sA7f5D6qkdMrpp2xTIeefnrUBI6xxEEWostvxfX_1Cs,236
 kodexa/pipeline/pipeline.py,sha256=ZYpJAWcwV4YRK589DUhU0vXGQlkNSj4J2TsGbYqTLjo,25221
@@ -39,11 +39,11 @@ kodexa/steps/__init__.py,sha256=qpmAtYePTv7G-HzUBx087bA3kq-PPGcFJf4_Z5P--0k,179
 kodexa/steps/common.py,sha256=fGEuKxcztcqrYFpXbu_OYkxh42yR9s5mkszmtkJhnQ8,10428
 kodexa/testing/__init__.py,sha256=P8W-SOnWsi48asfnQV06iyHrzZAzuX69j9oYwBvgp5s,323
 kodexa/testing/test_components.py,sha256=g5lP-GY0nTHuH5cIEw45vIejEeBaWkPKQGHL36jejBQ,1052
-kodexa/testing/test_utils.py,sha256=DrLCkHxdb6AbZ-X3WmTMbQmnVIm55VEBL8MjtUK9POs,14021
+kodexa/testing/test_utils.py,sha256=v44p__gE7ia67W7WeHN2HBFCWSCUrCZt7G4xBNCmwf8,14154
 kodexa/training/__init__.py,sha256=xs2L62YpRkIRfslQwtQZ5Yxjhm7sLzX2TrVX6EuBnZQ,52
 kodexa/training/train_utils.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kodexa/utils/__init__.py,sha256=Pnim1o9_db5YEnNvDTxpM7HG-qTlL6n8JwFwOafU9wo,5928
-kodexa-7.0.12397101638.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-kodexa-7.0.12397101638.dist-info/METADATA,sha256=-sl2RR48OFRUFzFyPGHh9ev2u9qy1i0FR6NjSbMAA_s,3527
-kodexa-7.0.12397101638.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
-kodexa-7.0.12397101638.dist-info/RECORD,,
+kodexa-7.0.12399073274.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+kodexa-7.0.12399073274.dist-info/METADATA,sha256=FHfSkW8bgm7lc-G9sT-rLxxLJfSaJP2-6ZaDGz6v1sE,3527
+kodexa-7.0.12399073274.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+kodexa-7.0.12399073274.dist-info/RECORD,,