by-qa 0.4.3.dev1__tar.gz → 0.4.4.dev1__tar.gz

{by_qa-0.4.3.dev1 → by_qa-0.4.4.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: by-qa
-Version: 0.4.3.dev1
+Version: 0.4.4.dev1
 Summary: Open-source knowledge-base ingestion and retrieval service.
 Author: AI Assistant
 License: MIT

{by_qa-0.4.3.dev1 → by_qa-0.4.4.dev1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "by-qa"
-version = "0.4.3.dev1"
+version = "0.4.4.dev1"
 description = "Open-source knowledge-base ingestion and retrieval service."
 readme = "README.md"
 requires-python = ">=3.12"

{by_qa-0.4.3.dev1 → by_qa-0.4.4.dev1}/src/by_qa/knowledge_base/api/knowledge_entity_schemas.py

@@ -129,6 +129,11 @@ class EntityDiscoveryRequest(_ApiModel):
         default=None,
         validation_alias=AliasChoices("directoryPath", "directory_path"),
     )
+    target_directory_path: str | None = Field(
+        default=None,
+        validation_alias=AliasChoices("targetDirectoryPath", "target_directory_path"),
+        serialization_alias="targetDirectoryPath",
+    )
     max_entities: int = Field(
         default=12,
         ge=1,
@@ -142,6 +147,7 @@ class EntityDiscoveryRequest(_ApiModel):
         validation_alias=AliasChoices("maxTopics", "max_topics"),
     )
     force: bool = False
+    tags: list[str] | None = None
     extra_params: dict[str, Any] | None = Field(
         default=None,
         validation_alias=AliasChoices("extraParams", "extra_params"),
@@ -153,10 +159,13 @@ class EntityDiscoveryRequest(_ApiModel):
     _normalize_directory_path = field_validator("directory_path")(
         _validate_directory_path
     )
+    _normalize_target_directory_path = field_validator("target_directory_path")(
+        _validate_directory_path
+    )
 
 
 class EntityEnrichRequest(_ApiModel):
-    """Request for single-file or whole-knowledge-base entity enrichment."""
+    """Request for file, directory, or whole-knowledge-base entity enrichment."""
 
     kb_code: str = Field(
         min_length=1,
@@ -166,6 +175,10 @@ class EntityEnrichRequest(_ApiModel):
         default=None,
         validation_alias=AliasChoices("filePath", "file_path"),
     )
+    directory_path: str | None = Field(
+        default=None,
+        validation_alias=AliasChoices("directoryPath", "directory_path"),
+    )
     top_k: int = Field(
         default=20,
         ge=1,
@@ -181,6 +194,9 @@ class EntityEnrichRequest(_ApiModel):
         description="Deprecated compatibility field; accepted but ignored.",
     )
     _normalize_file_path = field_validator("file_path")(_validate_file_path)
+    _normalize_directory_path = field_validator("directory_path")(
+        _validate_directory_path
+    )
 
 
 class DeleteKnowledgeEntityRequest(_ApiModel):

{by_qa-0.4.3.dev1 → by_qa-0.4.4.dev1}/src/by_qa/knowledge_base/api/routes.py

@@ -1854,9 +1854,10 @@ def register_routes(
                 status_code=422,
             )
         logger.info(
-            "entity_enrich request received: kb_code=%s, file_path=%s, force=%s",
+            "entity_enrich request received: kb_code=%s, file_path=%s, directory_path=%s, force=%s",
             request.kb_code,
             request.file_path,
+            request.directory_path,
             request.force,
         )
         try:

{by_qa-0.4.3.dev1 → by_qa-0.4.4.dev1}/src/by_qa/knowledge_base/dsl/compiler.py

@@ -74,7 +74,7 @@ def _custom_field_filter(
 def _query_value_type(operator: str, value: Any) -> str:
     if operator in ("prefix", "wildcard"):
         return "string"
-    if operator == "contains":
+    if operator in {"contains", "containsAll", "containsAny"}:
         return "stringList"
     if operator in ORDER_OPS and isinstance(value, str):
         if _parse_datetime_literal(value) is not None:
@@ -195,10 +195,10 @@ def _compile_node(
             f"AND mv.{col} LIKE %({val_key})s ESCAPE '{LIKE_ESCAPE_CHAR}')"
         )
 
-    if operator == "contains":
-        # `contains` is validator-restricted to user-defined stringList
-        # fields; no system-field branch here.
-        body = node["contains"]
+    if operator in {"contains", "containsAll", "containsAny"}:
+        # These operators are validator-restricted to user-defined stringList
+        # fields; there is no system-field branch here.
+        body = node[operator]
         field_name = body["fieldName"]
         value = body["value"]
         prop = property_map[field_name]
@@ -206,11 +206,20 @@ def _compile_node(
         filters = _custom_field_filter(
             ctx, field_name=field_name, value_type=value_type
         )
-        val_key = ctx.next_param(json.dumps([value]))
+        values = [value] if operator == "contains" else value
+        if operator == "containsAny":
+            predicates = []
+            for item in values:
+                val_key = ctx.next_param(json.dumps([item]))
+                predicates.append(f"mv.value_string_list @> %({val_key})s::jsonb")
+            value_predicate = "(" + " OR ".join(predicates) + ")"
+        else:
+            val_key = ctx.next_param(json.dumps(values))
+            value_predicate = f"mv.value_string_list @> %({val_key})s::jsonb"
         return (
             f"EXISTS (SELECT 1 FROM knowledge_file_metadata_value mv "
             f"WHERE {filters} "
-            f"AND mv.value_string_list @> %({val_key})s::jsonb)"
+            f"AND {value_predicate})"
         )
 
     if operator == "in":

{by_qa-0.4.3.dev1 → by_qa-0.4.4.dev1}/src/by_qa/knowledge_base/dsl/validator.py

@@ -14,6 +14,8 @@ LEAF_OPERATORS = {
     "ne",
     "in",
     "contains",
+    "containsAll",
+    "containsAny",
     "exists",
     "gt",
     "gte",
@@ -267,20 +269,20 @@ def _validate_leaf(
         )
         return
 
-    if operator == "contains":
+    if operator in {"contains", "containsAll", "containsAny"}:
         if value_type != "stringList":
             errors.append(
                 DslValidationDetail(
-                    path=f"{path}.contains.fieldName",
+                    path=f"{path}.{operator}.fieldName",
                     code="INVALID_FIELD_VALUE_TYPE",
                     message=(
-                        f"'contains' is only valid for stringList fields; "
+                        f"'{operator}' is only valid for stringList fields; "
                         f"'{field_name}' is {value_type}"
                     ),
                 )
             )
             return
-        if not isinstance(value, str):
+        if operator == "contains" and not isinstance(value, str):
             errors.append(
                 DslValidationDetail(
                     path=f"{path}.contains.value",
@@ -288,6 +290,26 @@ def _validate_leaf(
                     message="'contains' value must be a string",
                 )
             )
+        elif operator != "contains":
+            if not isinstance(value, list) or not value:
+                errors.append(
+                    DslValidationDetail(
+                        path=f"{path}.{operator}.value",
+                        code="INVALID_FIELD_VALUE_TYPE",
+                        message=f"'{operator}' value must be a non-empty array",
+                    )
+                )
+                return
+            for i, item in enumerate(value):
+                if not isinstance(item, str):
+                    errors.append(
+                        DslValidationDetail(
+                            path=f"{path}.{operator}.value[{i}]",
+                            code="INVALID_FIELD_VALUE_TYPE",
+                            message=f"'{operator}' value at index {i} must be a string",
+                        )
+                    )
+                    return
         return
 
     if operator == "in":
@@ -297,7 +319,8 @@ def _validate_leaf(
                     path=f"{path}.in.fieldName",
                     code="INVALID_FIELD_VALUE_TYPE",
                     message=(
-                        "'in' is not supported for stringList fields; use 'contains'"
+                        "'in' is not supported for stringList fields; use "
+                        "'containsAll' or 'containsAny'"
                     ),
                 )
             )
@@ -381,7 +404,7 @@ def _value_matches_type(value: Any, value_type: str) -> bool:
 def _infer_query_value_type(operator: str, value: Any) -> str:
     if operator in ("prefix", "wildcard"):
         return "string"
-    if operator == "contains":
+    if operator in {"contains", "containsAll", "containsAny"}:
         return "stringList"
     if operator == "in" and isinstance(value, list) and value:
         return _infer_query_value_type("eq", value[0])

{by_qa-0.4.3.dev1 → by_qa-0.4.4.dev1}/src/by_qa/knowledge_base/infrastructure/runtime.py

@@ -642,6 +642,15 @@ async def build_knowledge_entity_processing_service(
             settings, "entity_synonym_adjudication_enabled", True
         ),
     )
+    move_service = KnowledgeBaseService(
+        connection_factory=connection_factory,
+        knowledge_base_repository=knowledge_base_repository,
+        knowledge_fs_entry_repository=KnowledgeFsEntryRepository(),
+        retrieval_projection_repository=RetrievalProjectionRepository(),
+        knowledge_fetch_cache_repository=KnowledgeFetchCacheRepository(),
+        storage_provider=storage_provider,
+        file_metadata_value_repository=FileMetadataValueRepository(),
+    )
     worker = KnowledgeEntityTaskWorker(
         connection_factory=connection_factory,
         knowledge_entity_repository=knowledge_entity_repository,
@@ -654,6 +663,8 @@ async def build_knowledge_entity_processing_service(
         knowledge_entity_discovery=KnowledgeEntityDiscovery(discovery_llm),
         knowledge_entity_enricher=KnowledgeEntityEnricher(enrichment_llm),
         knowledge_entity_asset_service=asset_service,
+        semantic_task_repository=semantic_processing_task_repository,
+        knowledge_item_move_service=move_service,
     )
     service = KnowledgeEntityProcessingOrchestrator(
         connection_factory=connection_factory,

{by_qa-0.4.3.dev1 → by_qa-0.4.4.dev1}/src/by_qa/knowledge_base/repositories/knowledge_entity_repository.py

@@ -24,8 +24,9 @@ class KnowledgeEntityRepository:
         "subjectFileId",
         "entityType",
         "entityEnriched",
+        "tags",
     )
-    _LIST_METADATA_FIELDS = frozenset({"processingCapabilities", "aliases"})
+    _LIST_METADATA_FIELDS = frozenset({"processingCapabilities", "aliases", "tags"})
     _OUTPUT_FIELD_BY_PROPERTY = {
         "documentKind": "document_kind",
         "processingCapabilities": "processing_capabilities",
@@ -34,6 +35,7 @@ class KnowledgeEntityRepository:
         "subjectFileId": "subject_file_id",
         "entityType": "entity_type",
         "entityEnriched": "entity_enriched",
+        "tags": "tags",
     }
 
     async def get_directory_by_path(
@@ -102,8 +104,13 @@ class KnowledgeEntityRepository:
         knowledge_base_id: int,
         path_prefix: str | None = None,
         exclude_knowledge_entities: bool = False,
+        include_knowledge_entities_only: bool = False,
     ) -> list[dict[str, Any]]:
         """List live files in stable path order with optional scope filters."""
+        if exclude_knowledge_entities and include_knowledge_entities_only:
+            raise ValueError(
+                "entity include and exclude filters are mutually exclusive"
+            )
         normalized_prefix = (
             self._normalize_path(path_prefix) if path_prefix is not None else None
         )
@@ -143,6 +150,34 @@ class KnowledgeEntityRepository:
                         )
                     )
                   )
+              AND (
+                    NOT %(include_knowledge_entities_only)s
+                    OR EXISTS (
+                        SELECT 1
+                        FROM knowledge_file_metadata_value entity_document_kind
+                        WHERE entity_document_kind.fs_entry_id = fe.kid
+                          AND entity_document_kind.is_deleted = FALSE
+                          AND entity_document_kind.property_name = 'documentKind'
+                          AND entity_document_kind.value_type = 'string'
+                          AND entity_document_kind.value_string = 'knowledgeEntity'
+                    )
+                    OR (
+                        (
+                            fe.virtual_path = '/KnowledgeEntity'
+                            OR LEFT(
+                                fe.virtual_path,
+                                LENGTH('/KnowledgeEntity/')
+                            ) = '/KnowledgeEntity/'
+                        )
+                        AND NOT EXISTS (
+                            SELECT 1
+                            FROM knowledge_file_metadata_value configured_document_kind
+                            WHERE configured_document_kind.fs_entry_id = fe.kid
+                              AND configured_document_kind.is_deleted = FALSE
+                              AND configured_document_kind.property_name = 'documentKind'
+                        )
+                    )
+                  )
             ORDER BY fe.virtual_path ASC, fe.kid ASC,
                      mv.property_name ASC, mv.kid ASC
             """,
@@ -150,6 +185,7 @@ class KnowledgeEntityRepository:
                 "knowledge_base_id": knowledge_base_id,
                 "path_prefix": normalized_prefix,
                 "exclude_knowledge_entities": exclude_knowledge_entities,
+                "include_knowledge_entities_only": include_knowledge_entities_only,
                 "property_names": list(self._METADATA_FIELDS),
             },
         )
@@ -335,6 +371,7 @@ class KnowledgeEntityRepository:
                     "subject_file_id": None,
                     "entity_type": None,
                     "entity_enriched": None,
+                    "tags": [],
                 }
                 records[file_id] = record
             cls._apply_metadata(record, row)

{by_qa-0.4.3.dev1 → by_qa-0.4.4.dev1}/src/by_qa/knowledge_base/repositories/knowledge_fs_entry_repository.py

@@ -30,7 +30,6 @@ class KnowledgeFsEntryRepository:
         current_path_ltree: str | None = None
         current_virtual_path: str = "/"
         path_segments = normalized_path.split("/")
-        await self._lock_entry_tree_for_write(cursor)
 
         for index, segment in enumerate(path_segments[:-1], start=1):
             existing = await self._get_child_entry(
@@ -38,6 +37,7 @@ class KnowledgeFsEntryRepository:
                 knowledge_base_id=knowledge_base_id,
                 parent_entry_id=current_parent_id,
                 name=segment,
+                for_share=True,
             )
             if existing is not None:
                 if existing.get("entry_type") != "DIRECTORY":
@@ -101,6 +101,7 @@ class KnowledgeFsEntryRepository:
         full_path: str,
         file_description: str | None = None,
         created_parent_entries: list[dict[str, Any]] | None = None,
+        create_missing_parents: bool = True,
     ) -> dict[str, Any] | None:
         """Create one file entry under an existing parent directory."""
         normalized_path = full_path.strip("/")
@@ -111,7 +112,6 @@ class KnowledgeFsEntryRepository:
         current_path_ltree: str | None = None
         current_virtual_path: str = "/"
         path_segments = normalized_path.split("/")
-        await self._lock_entry_tree_for_write(cursor)
 
         for index, segment in enumerate(path_segments[:-1], start=1):
             existing = await self._get_child_entry(
@@ -119,6 +119,7 @@ class KnowledgeFsEntryRepository:
                 knowledge_base_id=knowledge_base_id,
                 parent_entry_id=current_parent_id,
                 name=segment,
+                for_share=True,
             )
             if existing is not None:
                 if existing.get("entry_type") != "DIRECTORY":
@@ -131,6 +132,9 @@ class KnowledgeFsEntryRepository:
                 current_virtual_path = self._row_value(existing, "virtual_path")
                 continue
 
+            if not create_missing_parents:
+                raise ValueError(f"parent directory not found: {normalized_path}")
+
             parent_directory, created = await self._insert_directory_entry(
                 cursor,
                 knowledge_base_id=knowledge_base_id,
@@ -165,62 +169,69 @@ class KnowledgeFsEntryRepository:
             if current_virtual_path != "/"
             else f"/{path_segments[-1]}"
         )
-        await cursor.execute(
-            """
-            INSERT INTO knowledge_fs_entry (
-                knowledge_base_id,
-                parent_entry_id,
-                entry_type,
-                is_root,
-                name,
-                path_ltree,
-                depth,
-                description,
-                virtual_path,
-                file_bucket_name,
-                file_object_key,
-                markdown_bucket_name,
-                markdown_object_key,
-                file_size,
-                mime_type,
-                checksum,
-                line_count,
-                created_at,
-                updated_at
-            )
-            VALUES (
-                %(knowledge_base_id)s,
-                %(parent_entry_id)s,
-                'FILE',
-                FALSE,
-                %(name)s,
-                %(path_ltree)s::ltree,
-                %(depth)s,
-                %(description)s,
-                %(virtual_path)s,
-                NULL,
-                NULL,
-                NULL,
-                NULL,
-                NULL,
-                NULL,
-                NULL,
-                NULL,
-                NOW(),
-                NOW()
+        try:
+            await cursor.execute(
+                """
+                INSERT INTO knowledge_fs_entry (
+                    knowledge_base_id,
+                    parent_entry_id,
+                    entry_type,
+                    is_root,
+                    name,
+                    path_ltree,
+                    depth,
+                    description,
+                    virtual_path,
+                    file_bucket_name,
+                    file_object_key,
+                    markdown_bucket_name,
+                    markdown_object_key,
+                    file_size,
+                    mime_type,
+                    checksum,
+                    line_count,
+                    created_at,
+                    updated_at
+                )
+                VALUES (
+                    %(knowledge_base_id)s,
+                    %(parent_entry_id)s,
+                    'FILE',
+                    FALSE,
+                    %(name)s,
+                    %(path_ltree)s::ltree,
+                    %(depth)s,
+                    %(description)s,
+                    %(virtual_path)s,
+                    NULL,
+                    NULL,
+                    NULL,
+                    NULL,
+                    NULL,
+                    NULL,
+                    NULL,
+                    NULL,
+                    NOW(),
+                    NOW()
+                )
+                RETURNING kid, knowledge_base_id, parent_entry_id, path_ltree, name, entry_type, is_root, depth, virtual_path
+                """,
+                {
+                    "knowledge_base_id": knowledge_base_id,
+                    "parent_entry_id": current_parent_id,
+                    "name": path_segments[-1],
+                    "path_ltree": path_ltree,
+                    "depth": len(path_segments),
+                    "description": file_description,
+                    "virtual_path": virtual_path,
+                },
             )
-            RETURNING kid, knowledge_base_id, parent_entry_id, path_ltree, name, entry_type, is_root, depth, virtual_path
-            """,
-            {
-                "knowledge_base_id": knowledge_base_id,
-                "parent_entry_id": current_parent_id,
-                "name": path_segments[-1],
-                "path_ltree": path_ltree,
-                "depth": len(path_segments),
-                "description": file_description,
-                "virtual_path": virtual_path,
-            },
-        )
+        except Exception as exc:
+            if self._is_unique_violation(exc):
+                raise ValueError(
+                    f"file path already exists: /{normalized_path}"
+                ) from exc
+            raise
         return await cursor.fetchone()
 
     async def update_file_entry_storage(
@@ -439,11 +450,6 @@ class KnowledgeFsEntryRepository:
             or getattr(exc, "pgcode", None) == "23505"
         )
 
-    async def _lock_entry_tree_for_write(self, cursor: Any) -> None:
-        await cursor.execute(
-            "LOCK TABLE knowledge_fs_entry IN SHARE ROW EXCLUSIVE MODE"
-        )
-
     async def lock_checksum_scope(
         self,
         cursor: Any,
@@ -797,6 +803,7 @@ class KnowledgeFsEntryRepository:
         knowledge_base_id: int,
         parent_entry_id: int | None,
         name: str,
+        for_share: bool = False,
     ) -> dict[str, Any] | None:
         if parent_entry_id is None:
             await cursor.execute(
@@ -824,7 +831,8 @@ class KnowledgeFsEntryRepository:
                   AND parent_entry_id IS NULL
                   AND name = %(name)s
                   AND is_deleted = FALSE
-                """,
+                """
+                + (" FOR SHARE" if for_share else ""),
                 {
                     "knowledge_base_id": knowledge_base_id,
                     "name": name,
@@ -856,7 +864,8 @@ class KnowledgeFsEntryRepository:
                   AND parent_entry_id = %(parent_entry_id)s
                   AND name = %(name)s
                   AND is_deleted = FALSE
-                """,
+                """
+                + (" FOR SHARE" if for_share else ""),
                 {
                     "knowledge_base_id": knowledge_base_id,
                     "parent_entry_id": parent_entry_id,
@@ -944,6 +953,12 @@ class KnowledgeFsEntryRepository:
         """Fetch one filesystem entry by id."""
         return await self._get_entry_by_id(cursor, entry_id=entry_id)
 
+    async def get_entry_by_id_for_update(
+        self, cursor: Any, *, entry_id: int
+    ) -> dict[str, Any] | None:
+        """Fetch and lock one live filesystem entry by id."""
+        return await self._get_entry_by_id(cursor, entry_id=entry_id, for_update=True)
+
     async def get_child_entry(
         self,
         cursor: Any,

{by_qa-0.4.3.dev1 → by_qa-0.4.4.dev1}/src/by_qa/knowledge_base/services/knowledge_base_service.py

@@ -347,6 +347,15 @@ class KnowledgeBaseService:
         finally:
             await connection.close()
 
+    @staticmethod
+    async def _lock_entry_for_tree_mutation(cursor: Any, entry_id: int) -> None:
+        # Wait for uploads holding shared ancestor locks before taking the
+        # subtree snapshot, so newly committed files are included in the change.
+        await cursor.execute(
+            "SELECT kid FROM knowledge_fs_entry WHERE kid = %(entry_id)s FOR UPDATE",
+            {"entry_id": entry_id},
+        )
+
     async def delete_directory(self, request: DeleteDirectoryRequest) -> None:
         """Logically delete one directory subtree and its retrieval projection rows."""
         logger.info(
@@ -381,6 +390,7 @@ class KnowledgeBaseService:
                     f"directory not found: {request.directory_path}"
                 )
             root_fs_entry_id = int(directory_row["kid"])
+            await self._lock_entry_for_tree_mutation(cursor, root_fs_entry_id)
             fs_entry_ids = (
                 await self.knowledge_fs_entry_repository.list_subtree_entry_ids(
                     cursor,
@@ -523,6 +533,7 @@ class KnowledgeBaseService:
                     f"directory not found: {request.directory_path}"
                 )
             fs_entry_id = int(fs_entry_row["kid"])
+            await self._lock_entry_for_tree_mutation(cursor, fs_entry_id)
             moved: list = []
             locator_updates: list = []
 
@@ -753,6 +764,8 @@ class KnowledgeBaseService:
         if source is None:
             raise KnowledgeBaseValidationError(f"source path not found: {source_path}")
 
+        await self._lock_entry_for_tree_mutation(cursor, int(source["kid"]))
+
         target_parent_path, target_name, target_path = await self._resolve_move_target(
             request=request,
             source=source,

{by_qa-0.4.3.dev1 → by_qa-0.4.4.dev1}/src/by_qa/knowledge_base/services/knowledge_entity_enrichment.py

@@ -1173,8 +1173,13 @@ def _canonical_link_target(target: str) -> str:
     return quote(decoded, safe="/:@-._~!$&'*+,;=%")
 
 
-def _is_knowledge_entity_target(target: str) -> bool:
-    return unquote(_canonical_link_target(target)).startswith("/KnowledgeEntity/")
+def _is_knowledge_entity_target(
+    target: str, *, known_entity_targets: set[str] | None = None
+) -> bool:
+    canonical = _canonical_link_target(target)
+    if known_entity_targets and canonical in known_entity_targets:
+        return True
+    return unquote(canonical).startswith("/KnowledgeEntity/")
 
 
 def normalize_generated_references(
@@ -1234,11 +1239,21 @@ def normalize_generated_references(
         for item in evidence
         if item.document_kind != "knowledgeEntity"
     }
+    known_entity_targets = {
+        _canonical_link_target(
+            _MARKDOWN_LINK_RE.search(
+                format_source_reference(item.document_path, item.document_kind)
+            ).group(2)
+        )
+        for item in evidence
+        if item.document_kind == "knowledgeEntity"
+    }
     original_source_targets.update(
         target
         for match in _MARKDOWN_LINK_RE.finditer(existing_markdown or "")
         if not _is_knowledge_entity_target(
-            target := _canonical_link_target(match.group(2))
+            target := _canonical_link_target(match.group(2)),
+            known_entity_targets=known_entity_targets,
         )
     )
     normalized, moved_count = _move_trailing_original_links_to_references(