kodexa 7.0.12399109365__py3-none-any.whl → 7.4.5a13228665254__py3-none-any.whl

kodexa/model/objects.py

@@ -631,7 +631,7 @@ class ProjectGuidance(BaseModel):
     description: Optional[str] = None
     guidance: Optional[List[Guidance]] = Field(None)
     active_store: bool = Field(False, alias="activeStore")
-    storage: GuidanceSetStorage = Field(None, description="The storage for the guidance set")
+    storage: Optional[GuidanceSetStorage]= Field(None, description="The storage for the guidance set")
     template_ref: Optional[str] = Field(None, alias="templateRef")
     ref: Optional[str] = None
 
@@ -822,7 +822,9 @@ class SelectionOption(BaseModel):
     label: Optional[str] = None
     id: Optional[str] = None
     description: Optional[str] = None
-    lexical_relations: Optional[List[LexicalRelation]] = Field([], alias="lexicalRelations")
+    lexical_relations: Optional[List[LexicalRelation]] = Field(default_factory=list, alias="lexicalRelations")
+    is_conditional: Optional[bool] = Field(None, alias="isConditional")
+    conditional_formula: Optional[str] = Field(None, alias="conditionalFormula")
 
 
 class SlugBasedMetadata1(BaseModel):
@@ -2645,6 +2647,8 @@ class Taxon(BaseModel):
     expression: Optional[str] = None
     enable_fallback_expression: Optional[bool] = Field(None, alias="enableFallbackExpression")
     fallback_expression: Optional[str] = Field(None, alias="fallbackExpression")
+    enable_serialization_expression: Optional[bool] = Field(None, alias="enableSerializationExpression")
+    serialization_expression: Optional[str] = Field(None, alias="serializationExpression")
     nullable: Optional[bool] = None
     null_value: Optional[str] = Field(None, alias="nullValue")
     denormalize_to_children: Optional[bool] = Field(False, alias="denormalizeToChildren")
@@ -2729,6 +2733,16 @@ class Taxon(BaseModel):
             structure["taxonType"] = self.taxon_type
         return structure
 
+    def get_taxon_by_path(self, path):
+        if self.path == path:
+            return self
+
+        if self.children:
+            for child in self.children:
+                result = child.get_taxon_by_path(path)
+                if result:
+                    return result
+
 
 class ContentObject(BaseModel):
     """
@@ -2916,14 +2930,51 @@ class Project(BaseModel):
     owner: Optional[User] = None
     options: Optional[ProjectOptions] = Field(None, alias="options")
 
-
 class TaskStatus(str, Enum):
     TODO = "TODO"
     IN_PROGRESS = "IN_PROGRESS"
     DONE = "DONE"
 
 
-class TaskCheckItem(BaseModel):
+class TaskActivityType(str, Enum):
+    TASK_CREATED = "TASK_CREATED"
+    TITLE_CHANGED = "TITLE_CHANGED"
+    DESCRIPTION_UPDATED = "DESCRIPTION_UPDATED"
+    STATUS_CHANGED = "STATUS_CHANGED"
+    ASSIGNEE_CHANGED = "ASSIGNEE_CHANGED"
+    DUE_DATE_CHANGED = "DUE_DATE_CHANGED"
+    PROJECT_CHANGED = "PROJECT_CHANGED"
+    COMMENT = "COMMENT"
+
+
+class TaskActivityDetail(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+        use_enum_values=True,
+        arbitrary_types_allowed=True,
+        protected_namespaces=("model_config",),
+    )
+    type: TaskActivityType
+    interpolated_values: Dict[str, Any] = Field(default_factory=dict, alias="interpolatedValues")
+
+
+class TaskActivity(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+        use_enum_values=True,
+        arbitrary_types_allowed=True,
+        protected_namespaces=("model_config",),
+    )
+
+    task: Optional['Task'] = None
+    content: Optional[str] = None
+    detail: Optional[TaskActivityDetail] = None
+    user: Optional['User'] = None
+    search_text: Optional[str] = Field(None, alias="searchText")
+    transient_values: Dict[str, Any] = Field(default_factory=dict, alias="transientValues")
+
+
+class TaskTag(BaseModel):
     model_config = ConfigDict(
         populate_by_name=True,
         use_enum_values=True,
@@ -2931,10 +2982,61 @@ class TaskCheckItem(BaseModel):
         protected_namespaces=("model_config",),
     )
 
+    project: Optional['Project'] = None
     name: str
+    color: Optional[str] = None
+
+
+class DataFormAction(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+        use_enum_values=True,
+        arbitrary_types_allowed=True,
+        protected_namespaces=("model_config",),
+    )
+
+    type: Optional[str] = None
+    label: Optional[str] = None
+    properties: Dict[str, Any] = Field(default_factory=dict)
+
+
+class TemplateDataForm(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+        use_enum_values=True,
+        arbitrary_types_allowed=True,
+        protected_namespaces=("model_config",),
+    )
+
+    data_form_ref: Optional[str] = Field(None, alias="dataFormRef")
+    actions: List[DataFormAction] = Field(default_factory=list)
+
+
+class TaskTemplateMetadata(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+        use_enum_values=True,
+        arbitrary_types_allowed=True,
+        protected_namespaces=("model_config",),
+    )
+
+    options: List[Option] = Field(default_factory=list)
+    forms: List[TemplateDataForm] = Field(default_factory=list)
+    workspace_id: Optional[str] = Field(None, alias="workspaceId")
+
+
+class TaskTemplate(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+        use_enum_values=True,
+        arbitrary_types_allowed=True,
+        protected_namespaces=("model_config",),
+    )
+
+    project: Optional['Project'] = None
+    name: Optional[str] = None
     description: Optional[str] = None
-    taxon_path: Optional[str] = Field(None, alias="taxonPath")
-    taxonomy_ref: Optional[str] = Field(None, alias="taxonomyRef")
+    metadata: Optional[TaskTemplateMetadata] = None
 
 
 class TaskMetadata(BaseModel):
@@ -2945,10 +3047,19 @@ class TaskMetadata(BaseModel):
         protected_namespaces=("model_config",),
     )
 
-    field_values: Dict[str, Any] = Field(default_factory=dict)
-    fields: List[Option] = Field(default_factory=list)
-    check_items: List[TaskCheckItem] = Field(default_factory=list)
-    document_store_ref: Optional[str] = None
+    properties: Dict[str, Any] = Field(default_factory=dict)
+
+
+class TaskDocumentFamily(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+        use_enum_values=True,
+        arbitrary_types_allowed=True,
+        protected_namespaces=("model_config",),
+    )
+
+    task: Optional['Task'] = None
+    document_family: Optional['DocumentFamily'] = Field(None, alias="documentFamily")
 
 
 class Task(BaseModel):
@@ -2959,16 +3070,19 @@ class Task(BaseModel):
         protected_namespaces=("model_config",),
     )
 
-    project: Optional['Project'] = Field(None)
-    title: Optional[str] = Field(None)
-    template: Optional[bool] = Field(None)
-    description: Optional[str] = Field(None)
-    metadata: Optional['TaskMetadata'] = Field(None)
+    project: Optional['Project'] = None
+    title: Optional[str] = None
+    description: Optional[str] = None
+    metadata: Optional[TaskMetadata] = None
+    template: Optional[TaskTemplate] = None
     due_date: Optional[StandardDateTime] = Field(None, alias="dueDate")
     completed_date: Optional[StandardDateTime] = Field(None, alias="completedDate")
-    status: Optional['TaskStatus'] = Field(None)
-    assignee: Optional['User'] = Field(None)
-
+    status: Optional[TaskStatus] = None
+    assignee: Optional['User'] = None
+    task_activity: List[TaskActivity] = Field(default_factory=list, alias="taskActivity")
+    task_document_families: List[TaskDocumentFamily] = Field(default_factory=list, alias="taskDocumentFamilies")
+    search_text: Optional[str] = Field(None, alias="searchText")
+    tags: List[TaskTag] = Field(default_factory=list)
 
 class FeatureSet(BaseModel):
     """
@@ -4504,11 +4618,7 @@ class PageTaxonomy(BaseModel):
     last: Optional[bool] = None
     empty: Optional[bool] = None
 
-
 class GuidanceTagResult(BaseModel):
-    """
-
-    """
     value: Optional[str] = None
     line_uuid: Optional[str] = Field(None, alias="lineUuid")
 
@@ -4519,7 +4629,6 @@ class UserSelection(BaseModel):
 
 
 class GuidanceRelationEmbedding(BaseModel):
-
     model_config = ConfigDict(
         populate_by_name=True,
         use_enum_values=True,
@@ -4533,7 +4642,10 @@ class GuidanceRelationEmbedding(BaseModel):
 
 
 class Guidance(BaseModel):
-    id: Optional[str] = Field(None, description="The ID of the guidance")
+    """
+    A guidance is a set of instructions and examples to guide taxonomies and extraction
+    """
+    id: Optional[str] = Field(None)
     name: Optional[str] = None
     guidance_type: Optional[str] = Field(None, alias="guidanceType")
     guidance_key: Optional[str] = Field(None, alias="guidanceKey")
@@ -4543,13 +4655,13 @@ class Guidance(BaseModel):
     document_name: Optional[str] = Field(None, alias="documentName")
     document_page: Optional[int] = Field(None, alias="documentPage")
     guidance_text: Optional[str] = Field(None, alias="guidanceText")
-    relation_embeddings: Optional[List[GuidanceRelationEmbedding]] = Field([], alias="relationEmbeddings")
+    relation_embeddings: Optional[List[GuidanceRelationEmbedding]] = Field(None, alias="relationEmbeddings")
     summary: Optional[str] = None
     guidance_response: Optional[Dict[str, Any]] = Field(None, alias="guidanceResponse")
-    active: Optional[bool] = True
+    active: bool = True
     applicable_tags: Optional[List[str]] = Field(None, alias="applicableTags")
     required_tags: Optional[List[str]] = Field(None, alias="requiredTags")
-    priority: Optional[int] = 1
+    priority: int = 1
     user_instructions: Optional[str] = Field(None, alias="userInstructions")
     user_instructions_properties: Optional[Dict[str, Any]] = Field(None, alias="userInstructionsProperties")
     user_id: Optional[str] = Field(None, alias="userId")
@@ -4567,17 +4679,11 @@ class GuidanceEmbeddingType(Enum):
 
 
 class GuidanceSetStorage(BaseModel):
-    model_config = ConfigDict(
-        populate_by_name=True,
-        use_enum_values=True,
-        arbitrary_types_allowed=True,
-        protected_namespaces=("model_config",),
-    )
-
+    embedding_model_id: Optional[str] = Field(None, alias="embeddingModelId")
     summarize_model_id: Optional[str] = Field(None, alias="summarizeModelId")
+    use_custom_summarize_prompt: Optional[bool] = Field(None, alias="useCustomSummarizePrompt")
     summarize_prompt: Optional[str] = Field(None, alias="summarizePrompt")
-    embedding_model_id: Optional[str] = Field(None, alias="embeddingModelId")
-    embedding_types: List[GuidanceEmbeddingType] = Field(default_factory=list, alias="embeddingTypes")
+    embedding_types: Optional[List[GuidanceEmbeddingType]] = Field(None, alias="embeddingTypes")
 
 
 class GuidanceSet(ExtensionPackProvided):
@@ -4591,10 +4697,9 @@ class GuidanceSet(ExtensionPackProvided):
         protected_namespaces=("model_config",),
     )
 
-    active_store: bool = Field(False, alias="activeStore",
-                               description="If true, allows guidance to be stored through the API")
-    storage: GuidanceSetStorage = Field(default_factory=GuidanceSetStorage)
-    guidance: List[Guidance] = Field(default_factory=list, description="The guidance in the set")
+    active_store: bool = False
+    storage: Optional[GuidanceSetStorage] = None
+    guidance: Optional[List[Guidance]] = None
 
     def get_type(self) -> str:
         return "guidance"
@@ -5674,6 +5779,16 @@ class Taxonomy(ExtensionPackProvided):
         description="A list of references to an external data taxonomy",
     )
 
+    def get_taxon_by_path(self, path):
+        for taxon in self.taxons:
+            if taxon.path == path:
+                return taxon
+            if taxon.children is not None:
+                child_taxon = taxon.get_taxon_by_path(path)
+                if child_taxon is not None:
+                    return child_taxon
+        return None
+
     def update_paths(self):
         for taxon in self.taxons:
             taxon.update_path()

kodexa/model/persistence.py

@@ -111,9 +111,10 @@ class SqliteDocumentPersistence(object):
 
         self.cursor = self.connection.cursor()
         self.cursor.execute("PRAGMA journal_mode=OFF")
-        self.cursor.execute("pragma temp_store = memory")
-        self.cursor.execute("pragma mmap_size = 30000000000")
-        self.cursor.execute("pragma cache_size = 10000")  # Set the cache size to 10,000 pages
+        self.cursor.execute("PRAGMA temp_store=MEMORY")
+        self.cursor.execute("PRAGMA mmap_size=30000000000")
+        self.cursor.execute("PRAGMA cache_size=10000")
+        self.cursor.execute("PRAGMA page_size=4096")
 
         try:
             # We need to populate node_type_id_by_name
@@ -865,10 +866,11 @@ class SqliteDocumentPersistence(object):
         self.connection.commit()
         self.cursor.execute("VACUUM")
         self.cursor = self.connection.cursor()
-        self.cursor.execute("pragma journal_mode=OFF")
-        self.cursor.execute("pragma temp_store = memory")
-        self.cursor.execute("pragma mmap_size = 30000000000")
-        self.cursor.execute("pragma cache_size = 10000")  # Set the cache size to 10,000 pages
+        self.cursor.execute("PRAGMA journal_mode=OFF")
+        self.cursor.execute("PRAGMA temp_store=MEMORY")
+        self.cursor.execute("PRAGMA mmap_size=30000000000")
+        self.cursor.execute("PRAGMA cache_size=10000")
+        self.cursor.execute("PRAGMA page_size=4096")
 
     def dump_in_memory_db_to_file(self):
         # Connect to a new or existing database file
@@ -1154,22 +1156,6 @@ class SqliteDocumentPersistence(object):
 
         return content_nodes
 
-    def __ensure_ed_table_exists(self):
-        """
-        Ensure the 'ed' table exists in the database.
-        Creates the table if it does not exist.
-        """
-        self.cursor.execute("""
-            CREATE TABLE IF NOT EXISTS ed (
-                obj BLOB
-            )
-        """)
-
-        # Check if the table has any rows, if not, insert an initial empty row
-        result = self.cursor.execute("SELECT COUNT(*) FROM ed").fetchone()
-        if result[0] == 0:
-            self.cursor.execute("INSERT INTO ed (obj) VALUES (?)", [sqlite3.Binary(msgpack.packb({}))])
-
     def __ensure_validations_table_exists(self):
         """
         Ensure the 'validations' table exists in the database.
@@ -1211,31 +1197,98 @@ class SqliteDocumentPersistence(object):
             return [DocumentTaxonValidation.model_validate(v) for v in msgpack.unpackb(result[0])]
         return []
 
-    def set_external_data(self, external_data: dict):
+    def set_external_data(self, external_data: dict, key: str = "default"):
         """
-        Sets the external data for the document.
+        Sets the external data for the document for a specific key.
 
         Args:
             external_data (dict): The external data to store, must be JSON serializable.
+            key (str): The key to store the data under, defaults to "default"
         """
         self.__ensure_ed_table_exists()
         serialized_data = sqlite3.Binary(msgpack.packb(external_data))
-        self.cursor.execute("UPDATE ed SET obj = ? WHERE rowid = 1", [serialized_data])
+        self.cursor.execute("DELETE FROM ed WHERE key = ?", [key])
+        self.cursor.execute("INSERT INTO ed (key, obj) VALUES (?, ?)", [key, serialized_data])
         self.connection.commit()
 
-    def get_external_data(self) -> dict:
+    def get_external_data(self, key: str = "default") -> dict:
         """
-        Gets the external data associated with this document.
+        Gets the external data associated with this document for a specific key.
+
+        Args:
+            key (str): The key to retrieve data for, defaults to "default"
 
         Returns:
-            dict: The external data stored in the ed table.
+            dict: The external data stored in the ed table for the given key.
         """
         self.__ensure_ed_table_exists()
-        result = self.cursor.execute("SELECT obj FROM ed WHERE rowid = 1").fetchone()
+        result = self.cursor.execute("SELECT obj FROM ed WHERE key = ?", [key]).fetchone()
         if result and result[0]:
             return msgpack.unpackb(result[0])
         return {}
 
+    def get_external_data_keys(self) -> List[str]:
+        """
+        Gets all keys under which external data is stored.
+
+        Returns:
+            List[str]: A list of all keys that have external data stored.
+        """
+        self.__ensure_ed_table_exists()
+        results = self.cursor.execute("SELECT key FROM ed").fetchall()
+        return [row[0] for row in results]
+
+    def __ensure_ed_table_exists(self):
+        """
+        Ensure the 'ed' table exists in the database.
+        Creates the table if it does not exist.
+        """
+        # First check if the old table exists and has key column
+        old_table = self.cursor.execute("""
+            SELECT name FROM sqlite_master 
+            WHERE type='table' AND name='ed'
+        """).fetchone()
+
+        if old_table:
+            # Check if table has key column
+            table_info = self.cursor.execute("PRAGMA table_info(ed)").fetchall()
+            has_key_column = any(col[1] == 'key' for col in table_info)
+
+            if not has_key_column:
+                # Get the old data and drop the table
+                data = self.cursor.execute("SELECT obj FROM ed").fetchone()
+                self.cursor.execute("DROP TABLE ed")
+
+                # Create new table with key column
+                self.cursor.execute("""
+                    CREATE TABLE ed (
+                        key TEXT PRIMARY KEY,
+                        obj BLOB
+                    )
+                """)
+
+                # If there was data in the old table, insert it with default key
+                if data:
+                    self.cursor.execute("INSERT INTO ed (key, obj) VALUES (?, ?)",
+                                      ["default", data[0]])
+            else:
+                # Table exists and has key column - do nothing
+                return
+        else:
+            # Create new table if it doesn't exist
+            self.cursor.execute("""
+                CREATE TABLE IF NOT EXISTS ed (
+                    key TEXT PRIMARY KEY,
+                    obj BLOB
+                )
+            """)
+
+            # Check if default key exists, if not insert empty data
+            result = self.cursor.execute("SELECT COUNT(*) FROM ed WHERE key = 'default'").fetchone()
+            if result[0] == 0:
+                self.cursor.execute("INSERT INTO ed (key, obj) VALUES (?, ?)",
+                                  ["default", sqlite3.Binary(msgpack.packb({}))])
+
     def __ensure_steps_table_exists(self):
         """
         Ensure the 'steps' table exists in the database.
@@ -1434,22 +1487,31 @@ class PersistenceManager(object):
     def get_validations(self) -> list[DocumentTaxonValidation]:
         return self._underlying_persistence.get_validations()
 
-    def get_external_data(self) -> dict:
+    def get_external_data(self, key="default") -> dict:
         """
         Gets the external data object associated with this document
 
         :return: dict of the external data
         """
-        return self._underlying_persistence.get_external_data()
+        return self._underlying_persistence.get_external_data(key)
+
+    def get_external_data_keys(self) -> List[str]:
+        """
+        Gets all keys under which external data is stored.
+
+        Returns:
+            List[str]: A list of all keys that have external data stored.
+        """
+        return self._underlying_persistence.get_external_data_keys()
 
-    def set_external_data(self, external_data:dict):
+    def set_external_data(self, external_data:dict, key="default"):
         """
         Sets the external data for this document
 
         :param external_data: dict representing the external data, must be JSON serializable
         :return:
         """
-        self._underlying_persistence.set_external_data(external_data)
+        self._underlying_persistence.set_external_data(external_data, key)
 
     def get_nodes_by_type(self, node_type: str) -> List[ContentNode]:
         """

kodexa/pipeline/pipeline.py

@@ -124,10 +124,12 @@ class PipelineContext:
         self.content_provider = content_provider
         self.context: Dict = context
         self.stop_on_exception = True
-        self.current_document = None
-        self.document_family = None
-        self.content_object = None
-        self.document_store = None
+        self.current_document: Optional[Document] = None
+        from kodexa.platform.client import DocumentFamilyEndpoint
+        self.document_family:Optional[DocumentFamilyEndpoint] = None
+        self.content_object:Optional[ContentObject] = None
+        from kodexa.platform.client import DocumentStoreEndpoint
+        self.document_store:Optional[DocumentStoreEndpoint] = None
         self.status_handler = status_handler
         self.cancellation_handler = cancellation_handler