kodexa 7.0.1a10177063353__py3-none-any.whl → 7.0.1a11918232720__py3-none-any.whl

kodexa/model/persistence.py

@@ -4,7 +4,7 @@ import pathlib
 import sqlite3
 import tempfile
 import uuid
-from typing import List
+from typing import List, Optional
 
 import msgpack
 
@@ -13,8 +13,9 @@ from kodexa.model.model import (
     DocumentMetadata,
     ContentFeature,
     ContentException,
-    ModelInsight,
+    ModelInsight, ProcessingStep,
 )
+from kodexa.model.objects import DocumentTaxonValidation
 
 logger = logging.getLogger()
 
@@ -1122,6 +1123,131 @@ class SqliteDocumentPersistence(object):
 
         return content_nodes
 
+    def __ensure_ed_table_exists(self):
+        """
+        Ensure the 'ed' table exists in the database.
+        Creates the table if it does not exist.
+        """
+        self.cursor.execute("""
+            CREATE TABLE IF NOT EXISTS ed (
+                obj BLOB
+            )
+        """)
+
+        # Check if the table has any rows, if not, insert an initial empty row
+        result = self.cursor.execute("SELECT COUNT(*) FROM ed").fetchone()
+        if result[0] == 0:
+            self.cursor.execute("INSERT INTO ed (obj) VALUES (?)", [sqlite3.Binary(msgpack.packb({}))])
+
+    def __ensure_validations_table_exists(self):
+        """
+        Ensure the 'validations' table exists in the database.
+        Creates the table if it does not exist and initializes it with an empty list.
+        """
+        self.cursor.execute("""
+            CREATE TABLE IF NOT EXISTS validations (
+                obj BLOB
+            )
+        """)
+
+        # Check if the table has any rows, if not, insert an initial empty row
+        result = self.cursor.execute("SELECT COUNT(*) FROM validations").fetchone()
+        if result[0] == 0:
+            self.cursor.execute("INSERT INTO validations (obj) VALUES (?)", [sqlite3.Binary(msgpack.packb([]))])
+
+    def set_validations(self, validations: List[DocumentTaxonValidation]):
+        """
+        Sets the validations for the document.
+
+        Args:
+            validations (List[DocumentTaxonValidation]): The validations to store.
+        """
+        self.__ensure_validations_table_exists()
+        serialized_data = sqlite3.Binary(msgpack.packb([v.model_dump(by_alias=True) for v in validations]))
+        self.cursor.execute("UPDATE validations SET obj = ? WHERE rowid = 1", [serialized_data])
+        self.connection.commit()
+
+    def get_validations(self) -> List[DocumentTaxonValidation]:
+        """
+        Gets the validations associated with this document.
+
+        Returns:
+            List[DocumentTaxonValidation]: The list of validations stored in the validations table.
+        """
+        self.__ensure_validations_table_exists()
+        result = self.cursor.execute("SELECT obj FROM validations WHERE rowid = 1").fetchone()
+        if result and result[0]:
+            return [DocumentTaxonValidation.model_validate(v) for v in msgpack.unpackb(result[0])]
+        return []
+
+    def set_external_data(self, external_data: dict):
+        """
+        Sets the external data for the document.
+
+        Args:
+            external_data (dict): The external data to store, must be JSON serializable.
+        """
+        self.__ensure_ed_table_exists()
+        serialized_data = sqlite3.Binary(msgpack.packb(external_data))
+        self.cursor.execute("UPDATE ed SET obj = ? WHERE rowid = 1", [serialized_data])
+        self.connection.commit()
+
+    def get_external_data(self) -> dict:
+        """
+        Gets the external data associated with this document.
+
+        Returns:
+            dict: The external data stored in the ed table.
+        """
+        self.__ensure_ed_table_exists()
+        result = self.cursor.execute("SELECT obj FROM ed WHERE rowid = 1").fetchone()
+        if result and result[0]:
+            return msgpack.unpackb(result[0])
+        return {}
+
+    def __ensure_steps_table_exists(self):
+        """
+        Ensure the 'steps' table exists in the database.
+        Creates the table if it does not exist.
+        """
+        self.cursor.execute("""
+                CREATE TABLE IF NOT EXISTS steps (
+                    obj BLOB
+                )
+            """)
+
+        # Check if the table has any rows, if not, insert an initial empty row
+        result = self.cursor.execute("SELECT COUNT(*) FROM steps").fetchone()
+        if result[0] == 0:
+            self.cursor.execute("INSERT INTO steps (obj) VALUES (?)", [sqlite3.Binary(msgpack.packb([]))])
+
+    def set_steps(self, steps: List[ProcessingStep]):
+        """
+        Sets the processing steps for the document.
+
+        Args:
+            steps (List[ProcessingStep]): A list of ProcessingStep objects to store.
+        """
+        self.__ensure_steps_table_exists()
+        serialized_steps = [step.to_dict() for step in steps]
+        packed_data = sqlite3.Binary(msgpack.packb(serialized_steps))
+        self.cursor.execute("UPDATE steps SET obj = ? WHERE rowid = 1", [packed_data])
+        self.connection.commit()
+
+    def get_steps(self) -> List[ProcessingStep]:
+        """
+        Gets the processing steps associated with this document.
+
+        Returns:
+            List[ProcessingStep]: A list of ProcessingStep objects.
+        """
+        self.__ensure_steps_table_exists()
+        result = self.cursor.execute("SELECT obj FROM steps WHERE rowid = 1").fetchone()
+        if result and result[0]:
+            unpacked_data = msgpack.unpackb(result[0])
+            return [ProcessingStep(**step) for step in unpacked_data]
+        return []
+
 
 class SimpleObjectCache(object):
     """
@@ -1146,7 +1272,7 @@ class SimpleObjectCache(object):
         self.next_id = 1
         self.dirty_objs = set()
 
-    def get_obj(self, obj_id):
+    def get_obj(self, obj_id) -> Optional[ContentNode]:
         """
         Get the object with the given ID.
 
@@ -1161,7 +1287,7 @@ class SimpleObjectCache(object):
 
         return None
 
-    def add_obj(self, obj):
+    def add_obj(self, obj: ContentNode):
         """
         Add an object to the cache.
 
@@ -1186,7 +1312,7 @@ class SimpleObjectCache(object):
             if obj.uuid in self.dirty_objs:
                 self.dirty_objs.remove(obj.uuid)
 
-    def get_dirty_objs(self):
+    def get_dirty_objs(self) -> list[ContentNode]:
         """
         Get all dirty objects in the cache.
 
@@ -1260,6 +1386,40 @@ class PersistenceManager(object):
             document, filename, delete_on_close, inmemory=inmemory
         )
 
+    def get_steps(self) -> list[ProcessingStep]:
+        """
+        Gets the processing steps for this document
+
+        :return:
+        """
+        return self._underlying_persistence.get_steps()
+
+    def set_steps(self, steps: list[ProcessingStep]):
+        self._underlying_persistence.set_steps(steps)
+
+    def set_validations(self, validations: list[DocumentTaxonValidation]):
+        self._underlying_persistence.set_validations(validations)
+
+    def get_validations(self) -> list[DocumentTaxonValidation]:
+        return self._underlying_persistence.get_validations()
+
+    def get_external_data(self) -> dict:
+        """
+        Gets the external data object associated with this document
+
+        :return: dict of the external data
+        """
+        return self._underlying_persistence.get_external_data()
+
+    def set_external_data(self, external_data:dict):
+        """
+        Sets the external data for this document
+
+        :param external_data: dict representing the external data, must be JSON serializable
+        :return:
+        """
+        self._underlying_persistence.set_external_data(external_data)
+
     def get_nodes_by_type(self, node_type: str) -> List[ContentNode]:
         """
         Retrieves all nodes of a given type from the underlying persistence layer.
@@ -1283,7 +1443,10 @@ class PersistenceManager(object):
             ContentNode: The node with the given uuid.
         """
         if self.node_cache.get_obj(uuid) is None:
-            self.node_cache.add_obj(self._underlying_persistence.get_node(uuid))
+            node = self._underlying_persistence.get_node(uuid)
+            if node:
+                self.node_cache.add_obj(node)
+                return node
 
         return self.node_cache.get_obj(uuid) # return the cached version
 

kodexa/platform/client.py

@@ -86,7 +86,7 @@ from kodexa.model.objects import (
     PageExtensionPack,
     PageOrganization,
     DocumentFamilyStatistics, MessageContext, PagePrompt, Prompt, GuidanceSet, PageGuidanceSet, DocumentEmbedding,
-    DocumentExternalData,
+    DocumentExternalData, Task, PageTask, RetainedGuidance, PageRetainedGuidance,
 )
 
 logger = logging.getLogger()
@@ -494,7 +494,7 @@ class ComponentEndpoint(ClientEndpoint, OrganizationOwned):
                     self.get_page_class(list_response.json())
                             .model_validate(list_response.json())
                             .set_client(self.client)
-                            .to_endpoints()
+                            .to_endpoints().content
             ):
                 yield endpoint
 
@@ -1173,6 +1173,35 @@ class PagePipelineEndpoint(PagePipeline, PageEndpoint):
         return "pipeline"
 
 
+class PageTaskEndpoint(PageTask, PageEndpoint):
+    def get_type(self) -> Optional[str]:
+        return "task"
+
+
+class PageRetainedGuidanceEndpoint(PageRetainedGuidance, PageEndpoint):
+    """Represents a page retained guidance endpoint.
+
+    This class is used to represent a page retained guidance endpoint which is a
+    combination of a page retained guidance and a page endpoint.
+
+    Attributes:
+        None
+    """
+
+    """Represents a page retained guidance endpoint"""
+
+    def get_type(self) -> Optional[str]:
+        """Get the type of the endpoint.
+
+        This method is used to get the type of the endpoint. In this case,
+        it will always return "retainedGuidance".
+
+        Returns:
+            str: The type of the endpoint, which is "retainedGuidance".
+        """
+        return "retainedGuidance"
+
+
 class PageProjectEndpoint(PageProject, PageEndpoint):
     """Represents a page project endpoint.
 
@@ -2517,14 +2546,42 @@ class WorkspaceEndpoint(EntityEndpoint, Workspace):
             raise ValueError("Workspace has no channel")
 
 
+class TaskEndpoint(EntityEndpoint, Task):
+    """Represents a task endpoint.
+
+    This class is used to interact with the task endpoint of the API.
+    """
+
+    def get_type(self) -> str:
+        """Get the type of the endpoint.
+
+        Returns:
+            str: The type of the endpoint, in this case "projects".
+        """
+        return "tasks"
+
+
+class RetainedGuidanceEndpoint(EntityEndpoint, RetainedGuidance):
+    """Represents a retained guidance endpoint.
+
+    This class is used to interact with the retained guidance endpoint of the API.
+    """
+
+    def get_type(self) -> str:
+        """Get the type of the endpoint.
+
+        Returns:
+            str: The type of the endpoint, in this case "retainedGuidance".
+        """
+        return "retainedGuidance"
+
+
 class ProjectEndpoint(EntityEndpoint, Project):
     """Represents a project endpoint.
 
     This class is used to interact with the project endpoint of the API.
     """
 
-    """Represents a project endpoint"""
-
     def get_type(self) -> str:
         """Get the type of the endpoint.
 
@@ -2849,6 +2906,72 @@ class AssistantsEndpoint(EntitiesEndpoint):
         return PageAssistantEndpoint
 
 
+class TasksEndpoint(EntitiesEndpoint):
+    """Represents a projects endpoint"""
+
+    """Represents a projects endpoint"""
+
+    def get_type(self) -> str:
+        """
+        Get the type of the endpoint.
+
+        Returns:
+            str: The type of the endpoint.
+        """
+        return "tasks"
+
+    def get_instance_class(self, object_dict=None):
+        """
+        Get the instance class of the endpoint.
+
+        Returns:
+            ProjectEndpoint: The instance class of the endpoint.
+        """
+        return TaskEndpoint
+
+    def get_page_class(self, object_dict=None):
+        """
+        Get the page class of the endpoint.
+
+        Returns:
+            PageProjectEndpoint: The page class of the endpoint.
+        """
+        return PageTaskEndpoint
+
+
+class RetainedGuidancesEndpoint(EntitiesEndpoint):
+    """Represents a projects endpoint"""
+
+    """Represents a projects endpoint"""
+
+    def get_type(self) -> str:
+        """
+        Get the type of the endpoint.
+
+        Returns:
+            str: The type of the endpoint.
+        """
+        return "retainedGuidance"
+
+    def get_instance_class(self, object_dict=None):
+        """
+        Get the instance class of the endpoint.
+
+        Returns:
+            ProjectEndpoint: The instance class of the endpoint.
+        """
+        return RetainedGuidanceEndpoint
+
+    def get_page_class(self, object_dict=None):
+        """
+        Get the page class of the endpoint.
+
+        Returns:
+            PageProjectEndpoint: The page class of the endpoint.
+        """
+        return PageRetainedGuidanceEndpoint
+
+
 class ProjectsEndpoint(EntitiesEndpoint):
     """Represents a projects endpoint"""
 
@@ -3946,6 +4069,18 @@ class TaxonomyEndpoint(ComponentInstanceEndpoint, Taxonomy):
             params={"format": "xsd"},
         ).text
 
+    def to_json_schema(self) -> dict:
+        """
+        Convert the taxonomy to an XSD.
+
+        Returns:
+            str: The XSD representation of the taxonomy.
+        """
+        return self.client.get(
+            f'/api/taxonomies/{self.ref.replace(":", "/")}/export',
+            params={"format": "json-schema"},
+        ).json()
+
     def get_taxon_by_path(self, path) -> Optional[Taxon]:
         """
         Get a taxon by its path.
@@ -4457,7 +4592,7 @@ class DocumentFamilyEndpoint(DocumentFamily, ClientEndpoint):
         response = self.client.get(url)
         process_response(response)
 
-    def get_external_data(self) -> DocumentExternalData:
+    def get_external_data(self) -> dict:
         """
         Get the external data of the document family.
 
@@ -4466,18 +4601,21 @@ class DocumentFamilyEndpoint(DocumentFamily, ClientEndpoint):
         """
         url = f"/api/documentFamilies/{self.id}/externalData"
         response = self.client.get(url)
-        return DocumentExternalData.model_validate(response.json())
+        return response.json()
 
-    def update_external_data(self, external_data: DocumentExternalData):
+    def set_external_data(self, external_data: dict) -> dict:
         """
-        Update the external data of the document family.
+        Set the external data of the document family.
 
         Args:
-            external_data (DocumentExternalData): The external data to update.
+            external_data (dict): The external data to set for the document family.
+
+        Returns:
+            dict: The updated external data of the document family.
         """
         url = f"/api/documentFamilies/{self.id}/externalData"
-        response = self.client.put(url, body=external_data.model_dump(mode="json", by_alias=True))
-        process_response(response)
+        response = self.client.put(url, data=external_data)
+        return response.json()
 
     def export(self) -> bytes:
         """
@@ -5289,6 +5427,7 @@ class DocumentStoreEndpoint(StoreEndpoint):
             object_path: Optional[str] = None,
             replace=False,
             additional_metadata: Optional[dict] = None,
+            external_data: Optional[dict] = None,
     ):
         """
         Upload a file to the store.
@@ -5298,6 +5437,7 @@ class DocumentStoreEndpoint(StoreEndpoint):
             object_path (Optional[str]): The path to the object (Default is the same the file path).
             replace (bool): Replace the file if it already exists (Default False).
             additional_metadata (Optional[dict]): Additional metadata to add to the file (Default None).
+            external_data (Optional[dict]): External data to add to the file (Default None).
         """
         if Path(file_path).is_file():
             logger.info(f"Uploading {file_path}")
@@ -5307,6 +5447,7 @@ class DocumentStoreEndpoint(StoreEndpoint):
                     content=path_content,
                     replace=replace,
                     additional_metadata=additional_metadata,
+                    external_data=external_data
                 )
         else:
             raise Exception(f"{file_path} is not a file")
@@ -5317,6 +5458,7 @@ class DocumentStoreEndpoint(StoreEndpoint):
             content,
             replace=False,
             additional_metadata: Optional[dict] = None,
+            external_data: Optional[dict] = None,
     ) -> DocumentFamilyEndpoint:
         """
         Put the content into the store at the given path.
@@ -5326,6 +5468,7 @@ class DocumentStoreEndpoint(StoreEndpoint):
             content: The content for that object.
             replace (bool): Replace the content if it exists.
             additional_metadata (Optional[dict]): Additional metadata to store with the document (not it can't include 'path').
+            external_data (Optional[dict]): External data to store with the document.
 
         Returns:
             DocumentFamilyEndpoint: The document family that was created.
@@ -5335,6 +5478,9 @@ class DocumentStoreEndpoint(StoreEndpoint):
         if additional_metadata is None:
             additional_metadata = {}
 
+        if external_data is not None:
+            additional_metadata["externalData"] = json.dumps(external_data)
+
         if replace and self.client.exists(
                 f"/api/stores/{self.ref.replace(':', '/')}/fs", params={"path": path}
         ):
@@ -6332,6 +6478,10 @@ class KodexaClient:
         executions (ExecutionsEndpoint): An endpoint for executions.
         channels (ChannelsEndpoint): An endpoint for channels.
         messages (MessagesEndpoint): An endpoint for messages.
+        assistants (AssistantsEndpoint): An endpoint for assistants.
+        products (ProductsEndpoint): An endpoint for products.
+        tasks (TasksEndpoint): An endpoint for tasks.
+        retained_guidances (RetainedGuidancesEndpoint): An endpoint for retained guidances.
     """
 
     def __init__(self, url=None, access_token=None, profile=None):
@@ -6354,6 +6504,8 @@ class KodexaClient:
         self.messages = MessagesEndpoint(self)
         from kodexa.model.entities.product import ProductsEndpoint
         self.products = ProductsEndpoint(self)
+        self.tasks = TasksEndpoint(self)
+        self.retained_guidances = RetainedGuidancesEndpoint(self)
 
     @staticmethod
     def login(url, token):
@@ -6370,7 +6522,6 @@ class KodexaClient:
         Raises:
             Exception: If the status code is not 200.
         """
-        from requests.auth import HTTPBasicAuth
 
         obj_response = requests.get(
             f"{url}/api/account/me",
@@ -6751,7 +6902,6 @@ class KodexaClient:
                     )
                 )
 
-
     def import_project(self, organization: OrganizationEndpoint, import_path: str):
         """
         A method to import a project.
@@ -6947,8 +7097,10 @@ class KodexaClient:
                 "message": MessageEndpoint,
                 "prompt": PromptEndpoint,
                 "guidance": GuidanceSetEndpoint,
+                "retainedGuidance": RetainedGuidanceEndpoint,
                 "channel": ChannelEndpoint,
                 "product": ProductEndpoint,
+                "task": TaskEndpoint,
                 "productSubscription": ProductSubscriptionEndpoint,
                 "checkResponse": CheckResponseEndpoint
             }

kodexa/platform/kodexa.py

@@ -29,7 +29,7 @@ from kodexa.model.objects import (
     DocumentFamilyEvent,
     ChannelEvent,
     DataObjectEvent,
-    WorkspaceEvent,
+    WorkspaceEvent, DocumentFamily,
 )
 from kodexa.pipeline import PipelineContext, PipelineStatistics
 from kodexa.platform.client import DocumentStoreEndpoint, KodexaClient, process_response
@@ -327,7 +327,7 @@ class KodexaPlatform:
             dict: The server information.
         """
         r = requests.get(
-            f"{KodexaPlatform.get_url()}/api",
+            f"{KodexaPlatform.get_url()}/api/overview",
             headers={
                 "x-access-token": KodexaPlatform.get_access_token(),
                 "cf-access-token": os.environ.get("CF_TOKEN", ""),
@@ -884,7 +884,7 @@ class EventHelper:
             ContentObject: The posted content object.
         """
         files = {"content": content}
-        data = {"contentObjectJson": json.dumps(content_object.dict(by_alias=True))}
+        data = {"contentObjectJson": json.dumps(content_object.model_dump(by_alias=True))}
         logger.info("Posting back content object to execution object")
         co_response = requests.post(
             f"{KodexaPlatform.get_url()}/api/sessions/{self.event.session_id}/executions/{self.event.execution.id}/objects",
@@ -901,7 +901,7 @@ class EventHelper:
 
         return ContentObject.model_validate(co_response.json())
 
-    def build_pipeline_context(self) -> PipelineContext:
+    def build_pipeline_context(self, event) -> PipelineContext:
         """Builds a pipeline context.
 
         Returns:
@@ -911,16 +911,26 @@ class EventHelper:
             context={}, content_provider=self, execution_id=self.event.execution.id
         )
 
-        if self.event.store_ref and self.event.document_family_id:
-            logger.info("We have storeRef and document family")
-            rds: DocumentStoreEndpoint = KodexaClient().get_object_by_ref(
-                "store", self.event.store_ref
-            )
-            document_family = rds.get_family(self.event.document_family_id)
-
-            context.document_family = document_family
-            context.document_store = rds
-
+        if isinstance(event, dict):
+            event = self.get_base_event(event)
+
+        if isinstance(event, DocumentFamilyEvent):
+            # Can we get the document family
+            dfe:DocumentFamilyEvent = self.event
+            if dfe.document_family:
+                logger.info(f"Setting document family for context: {dfe.document_family}")
+                context.document_family = dfe.document_family
+                logger.info(f"Getting document store for family: {context.document_family.store_ref}")
+                context.document_store = KodexaClient().get_object_by_ref("store", context.document_family.store_ref)
+        if isinstance(event, ContentEvent):
+            ce:ContentEvent = self.event
+            if ce.content_object:
+                logger.info(f"Setting content object for context: {ce.content_object}")
+                context.content_object = ce.content_object
+                context.document_family = ce.document_family
+                logger.info(f"Getting document store for content object: {context.content_object.store_ref}")
+                context.document_store = KodexaClient().get_object_by_ref("store", context.content_object.store_ref)
+        logger.info("Returning context")
         return context
 
     def get_input_document(self, context):