by-qa 0.4.4.dev2__tar.gz → 0.4.4.dev4__tar.gz

{by_qa-0.4.4.dev2 → by_qa-0.4.4.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: by-qa
-Version: 0.4.4.dev2
+Version: 0.4.4.dev4
 Summary: Open-source knowledge-base ingestion and retrieval service.
 Author: AI Assistant
 License: MIT

{by_qa-0.4.4.dev2 → by_qa-0.4.4.dev4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "by-qa"
-version = "0.4.4.dev2"
+version = "0.4.4.dev4"
 description = "Open-source knowledge-base ingestion and retrieval service."
 readme = "README.md"
 requires-python = ">=3.12"

{by_qa-0.4.4.dev2 → by_qa-0.4.4.dev4}/src/by_qa/knowledge_base/api/routes.py

@@ -1,11 +1,12 @@
 """Route registration for knowledge base APIs."""
 
-import io
 import json
 import mimetypes
 import zipfile
+from contextlib import AsyncExitStack
 from inspect import isawaitable
-from pathlib import PurePosixPath
+from pathlib import Path, PurePosixPath
+from tempfile import TemporaryDirectory
 from typing import Any, Callable, Optional
 from urllib.parse import quote
 
@@ -42,6 +43,7 @@ from by_qa.knowledge_base.api.schemas import (
     DocumentUpdateRequest,
     FileBuildStatusRequest,
     FileToMarkdownIndexRequest,
+    KnowledgeBaseFileCountRequest,
     KnowledgeItemDownloadRequest,
     KnowledgeItemGlobRequest,
     KnowledgeItemListDirRequest,
@@ -63,6 +65,11 @@ from by_qa.knowledge_base.services.errors import (
     KnowledgeBaseConfigurationError,
     KnowledgeBaseValidationError,
 )
+from by_qa.knowledge_base.services.file_upload_source import (
+    UPLOAD_CHUNK_SIZE,
+    FileUploadRequest,
+    file_io,
+)
 from by_qa.knowledge_base.services.knowledge_item_ingestion_service import (
     convert_uploaded_file_to_markdown,
 )
@@ -98,15 +105,17 @@ def _documented_error_response(
     result_msg: str,
     result_object: dict[str, Any] | None = None,
     status_code: int = 422,
+    preserve_http_status: bool = False,
 ) -> JSONResponse:
-    """Return the documented error envelope with HTTP status normalized to 200."""
-    logger.info(
-        "knowledge_base error response normalized to HTTP 200: business_status_code=%s, result_msg=%s",
-        status_code,
-        result_msg,
-    )
+    """Return the error envelope, normalizing legacy endpoints to HTTP 200."""
+    if not preserve_http_status:
+        logger.info(
+            "knowledge_base error response normalized to HTTP 200: business_status_code=%s, result_msg=%s",
+            status_code,
+            result_msg,
+        )
     return JSONResponse(
-        status_code=200,
+        status_code=status_code if preserve_http_status else 200,
         content=jsonable_encoder(
             {
                 "resultCode": "-1",
@@ -501,6 +510,61 @@ def register_routes(
             )
         return _documented_success_response(result_object={})
 
+    @app.post(
+        "/api/v1/knowledgeBases/fileCount",
+        openapi_extra={
+            "requestBody": {
+                "required": True,
+                "content": {
+                    "application/json": {
+                        "schema": KnowledgeBaseFileCountRequest.model_json_schema(
+                            by_alias=True
+                        )
+                    }
+                },
+            }
+        },
+    )
+    async def knowledge_base_file_count(http_request: Request):
+        # Parse locally so malformed bodies also honor this endpoint's HTTP 422
+        # contract rather than the application's legacy HTTP 200 error handler.
+        try:
+            request = KnowledgeBaseFileCountRequest.model_validate(
+                await http_request.json()
+            )
+        except ValueError:
+            return _documented_error_response(
+                result_msg="request validation failed",
+                status_code=422,
+                preserve_http_status=True,
+            )
+
+        logger.info(
+            "knowledge_base_file_count request received: kb_code=%s", request.kb_code
+        )
+        try:
+            service = await get_knowledge_base_service()
+            result = await service.file_count(request)
+        except KnowledgeBaseConfigurationError as exc:
+            return _documented_error_response(
+                result_msg=str(exc), status_code=503, preserve_http_status=True
+            )
+        except KnowledgeBaseValidationError as exc:
+            return _documented_error_response(
+                result_msg=str(exc), status_code=422, preserve_http_status=True
+            )
+        except Exception as exc:
+            logger.exception(
+                "knowledge_base_file_count unexpected error: kb_code=%s",
+                request.kb_code,
+            )
+            return _documented_error_response(
+                result_msg=str(exc) or "internal error",
+                status_code=500,
+                preserve_http_status=True,
+            )
+        return _documented_success_response(result_object=result)
+
     @app.post("/api/v1/directories/create")
     async def create_directory(
         background_tasks: BackgroundTasks,
@@ -698,148 +762,168 @@ def register_routes(
         skip_if_duplicate: bool = Form(False, alias="skipIfDuplicate"),
         metadata: str | None = Form(None, alias="metadata"),
     ):
-        try:
-            payload = await file_content.read() if file_content is not None else None
-            request = KnowledgeItemUploadRequest.model_validate(
-                {
-                    "knCode": kn_code,
-                    "filePath": file_path,
-                    "fileDescription": file_description,
-                    "fileContent": payload,
-                    "mimeType": (
-                        file_content.content_type if file_content is not None else None
+        async with AsyncExitStack() as stack:
+            try:
+                filename = (
+                    (file_content.filename or "") if file_content is not None else ""
+                )
+                is_zip = extract_zip and filename.lower().endswith(".zip")
+                if is_zip and file_content is not None:
+                    temp_dir = TemporaryDirectory(prefix="byqa-upload-")
+                    stack.push_async_callback(file_io, temp_dir.cleanup)
+                    payload = Path(temp_dir.name) / "upload.zip"
+                    with payload.open("wb") as stream:
+                        while chunk := await file_content.read(UPLOAD_CHUNK_SIZE):
+                            await file_io(stream.write, chunk)
+                    request_model = FileUploadRequest
+                else:
+                    payload = (
+                        await file_content.read() if file_content is not None else None
+                    )
+                    request_model = KnowledgeItemUploadRequest
+                request = request_model.model_validate(
+                    {
+                        "knCode": kn_code,
+                        "filePath": file_path,
+                        "fileDescription": file_description,
+                        "fileContent": payload,
+                        "mimeType": (
+                            file_content.content_type
+                            if file_content is not None
+                            else None
+                        ),
+                        "processFrontMatter": process_front_matter,
+                        "extractZip": extract_zip,
+                        "skipIfDuplicate": skip_if_duplicate,
+                        "metadata": _parse_optional_metadata_form(metadata),
+                    }
+                )
+            except (ValidationError, ValueError) as exc:
+                return _documented_error_response(
+                    result_msg="request validation failed",
+                    result_object=(
+                        {"errors": json.loads(exc.json())}
+                        if isinstance(exc, ValidationError)
+                        else {"errors": [{"msg": str(exc)}]}
                     ),
-                    "processFrontMatter": process_front_matter,
-                    "extractZip": extract_zip,
-                    "skipIfDuplicate": skip_if_duplicate,
-                    "metadata": _parse_optional_metadata_form(metadata),
-                }
-            )
-        except (ValidationError, ValueError) as exc:
-            return _documented_error_response(
-                result_msg="request validation failed",
-                result_object=(
-                    {"errors": json.loads(exc.json())}
-                    if isinstance(exc, ValidationError)
-                    else {"errors": [{"msg": str(exc)}]}
-                ),
-                status_code=422,
+                    status_code=422,
+                )
+            logger.info(
+                "upload_file request received: kb_code=%s, file_path=%s, has_description=%s, process_front_matter=%s, extract_zip=%s",
+                request.kb_code,
+                request.file_path,
+                request.file_description is not None,
+                request.process_front_matter,
+                request.extract_zip,
             )
-        logger.info(
-            "upload_file request received: kb_code=%s, file_path=%s, has_description=%s, process_front_matter=%s, extract_zip=%s",
-            request.kb_code,
-            request.file_path,
-            request.file_description is not None,
-            request.process_front_matter,
-            request.extract_zip,
-        )
-        filename = (file_content.filename or "") if file_content is not None else ""
-        try:
-            service = await get_knowledge_item_ingestion_service()
-            if request.extract_zip and filename.lower().endswith(".zip"):
-                if not zipfile.is_zipfile(io.BytesIO(payload or b"")):
+            try:
+                service = await get_knowledge_item_ingestion_service()
+                if request.extract_zip and filename.lower().endswith(".zip"):
+                    if not await file_io(zipfile.is_zipfile, payload):
+                        return _documented_error_response(
+                            result_msg="invalid zip file",
+                            result_object={},
+                            status_code=422,
+                        )
+                    directory_service = await get_knowledge_base_service()
+                    batch_service = ZipBatchImportService(
+                        ingestion_service=service,
+                        directory_service=directory_service,
+                    )
+                    result = await batch_service.import_zip(
+                        kb_code=request.kb_code,
+                        target_dir=request.file_path,
+                        zip_path=payload,
+                        process_front_matter=request.process_front_matter,
+                        file_description=request.file_description,
+                        skip_if_duplicate=request.skip_if_duplicate,
+                        metadata=request.metadata,
+                    )
+                    result_object = {
+                        "data": [
+                            item.model_dump(by_alias=True) for item in result.data
+                        ],
+                        "summary": result.summary.model_dump(by_alias=True),
+                    }
+                    if result.post_process_errors:
+                        result_object["postProcessErrors"] = result.post_process_errors
+                    if result.summary.succeeded > 0:
+                        await _try_schedule_event(
+                            background_tasks,
+                            lambda: build_resource_event(
+                                event_type=ResourceEventType.FILE_IMPORTED,
+                                kb_code=request.kb_code,
+                                source_path=None,
+                                target_path=_ensure_leading_slash(request.file_path),
+                                items=[
+                                    {
+                                        "sourcePath": item.file_path,
+                                        "targetPath": (
+                                            item.file_path if item.success else None
+                                        ),
+                                        "resourceType": item.resource_type,
+                                        "success": item.success,
+                                        "error": item.error,
+                                    }
+                                    for item in result.data
+                                ],
+                                result=result.summary.model_dump(),
+                            ),
+                        )
+                    return _documented_success_response(result_object=result_object)
+                # single file
+                file_path_norm = "/" + request.file_path.strip("/")
+                segments = [s for s in file_path_norm.split("/") if s]
+                if any(s == ".." for s in segments):
                     return _documented_error_response(
-                        result_msg="invalid zip file",
-                        result_object={},
-                        status_code=422,
+                        result_msg="unsafe path", result_object={}, status_code=422
                     )
-                directory_service = await get_knowledge_base_service()
-                batch_service = ZipBatchImportService(
-                    ingestion_service=service,
-                    directory_service=directory_service,
-                )
-                result = await batch_service.import_zip(
-                    kb_code=request.kb_code,
-                    target_dir=request.file_path,
-                    zip_bytes=payload,
-                    process_front_matter=request.process_front_matter,
-                    file_description=request.file_description,
-                    skip_if_duplicate=request.skip_if_duplicate,
-                    metadata=request.metadata,
-                )
+                await service.upload_file(request)
                 result_object = {
-                    "data": [item.model_dump(by_alias=True) for item in result.data],
-                    "summary": result.summary.model_dump(by_alias=True),
+                    "data": [
+                        {
+                            "filePath": file_path_norm,
+                            "success": True,
+                            "error": None,
+                        }
+                    ],
+                    "summary": {"total": 1, "succeeded": 1, "failed": 0},
                 }
-                if result.post_process_errors:
-                    result_object["postProcessErrors"] = result.post_process_errors
-                if result.summary.succeeded > 0:
-                    await _try_schedule_event(
-                        background_tasks,
-                        lambda: build_resource_event(
-                            event_type=ResourceEventType.FILE_IMPORTED,
-                            kb_code=request.kb_code,
-                            source_path=None,
-                            target_path=_ensure_leading_slash(request.file_path),
-                            items=[
-                                {
-                                    "sourcePath": item.file_path,
-                                    "targetPath": (
-                                        item.file_path if item.success else None
-                                    ),
-                                    "resourceType": item.resource_type,
-                                    "success": item.success,
-                                    "error": item.error,
-                                }
-                                for item in result.data
-                            ],
-                            result=result.summary.model_dump(),
-                        ),
-                    )
+                await _try_schedule_event(
+                    background_tasks,
+                    lambda: build_resource_event(
+                        event_type=ResourceEventType.FILE_IMPORTED,
+                        kb_code=request.kb_code,
+                        source_path=None,
+                        target_path=file_path_norm,
+                        result=result_object["summary"],
+                    ),
+                )
                 return _documented_success_response(result_object=result_object)
-            # single file
-            file_path_norm = "/" + request.file_path.strip("/")
-            segments = [s for s in file_path_norm.split("/") if s]
-            if any(s == ".." for s in segments):
+            except KnowledgeBaseConfigurationError as exc:
                 return _documented_error_response(
-                    result_msg="unsafe path", result_object={}, status_code=422
+                    result_msg=str(exc),
+                    result_object={},
+                    status_code=503,
+                )
+            except KnowledgeBaseValidationError as exc:
+                return _documented_error_response(
+                    result_msg=str(exc),
+                    result_object={},
+                    status_code=422,
+                )
+            except Exception as exc:
+                logger.exception(
+                    "upload_file unexpected error: kb_code=%s, file_path=%s, error=%s",
+                    request.kb_code,
+                    request.file_path,
+                    exc,
+                )
+                return _documented_error_response(
+                    result_msg=str(exc) or "internal error",
+                    result_object={},
+                    status_code=500,
                 )
-            await service.upload_file(request)
-            result_object = {
-                "data": [
-                    {
-                        "filePath": file_path_norm,
-                        "success": True,
-                        "error": None,
-                    }
-                ],
-                "summary": {"total": 1, "succeeded": 1, "failed": 0},
-            }
-            await _try_schedule_event(
-                background_tasks,
-                lambda: build_resource_event(
-                    event_type=ResourceEventType.FILE_IMPORTED,
-                    kb_code=request.kb_code,
-                    source_path=None,
-                    target_path=file_path_norm,
-                    result=result_object["summary"],
-                ),
-            )
-            return _documented_success_response(result_object=result_object)
-        except KnowledgeBaseConfigurationError as exc:
-            return _documented_error_response(
-                result_msg=str(exc),
-                result_object={},
-                status_code=503,
-            )
-        except KnowledgeBaseValidationError as exc:
-            return _documented_error_response(
-                result_msg=str(exc),
-                result_object={},
-                status_code=422,
-            )
-        except Exception as exc:
-            logger.exception(
-                "upload_file unexpected error: kb_code=%s, file_path=%s, error=%s",
-                request.kb_code,
-                request.file_path,
-                exc,
-            )
-            return _documented_error_response(
-                result_msg=str(exc) or "internal error",
-                result_object={},
-                status_code=500,
-            )
 
     @app.post("/api/v1/knowledgeItems/update")
     @app.post("/api/v1/knowledge-items/update")

{by_qa-0.4.4.dev2 → by_qa-0.4.4.dev4}/src/by_qa/knowledge_base/api/schemas.py

@@ -54,6 +54,17 @@ class DeleteKnowledgeBaseRequest(BaseModel):
     )
 
 
+class KnowledgeBaseFileCountRequest(BaseModel):
+    """Request body for counting all live files in a knowledge base."""
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    kb_code: str = Field(
+        min_length=1,
+        validation_alias=AliasChoices("knCode", "kb_code"),
+    )
+
+
 class UpdateKnowledgeBaseRequest(BaseModel):
     """Request body for updating knowledge base business fields."""
 

{by_qa-0.4.4.dev2 → by_qa-0.4.4.dev4}/src/by_qa/knowledge_base/infrastructure/storage.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
+from pathlib import Path
 from typing import Protocol, runtime_checkable
 
 
@@ -73,10 +74,11 @@ class KnowledgeStorageProvider(Protocol):
     async def write(
         self,
         location: StorageLocation,
-        content: bytes,
+        content: bytes | Path,
         *,
         content_type: str,
-    ) -> StoredObject: ...
+    ) -> StoredObject:
+        """Write bytes or stream a local Path; the caller owns the source file."""
 
     async def read(self, location: StorageLocation) -> bytes: ...
 

{by_qa-0.4.4.dev2 → by_qa-0.4.4.dev4}/src/by_qa/knowledge_base/infrastructure/storage_s3.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from pathlib import PurePosixPath
+from pathlib import Path, PurePosixPath
 from typing import Any
 
 from botocore.exceptions import ClientError
@@ -20,6 +20,7 @@ from by_qa.knowledge_base.infrastructure.storage import (
     StorageOperationError,
     StoredObject,
 )
+from by_qa.knowledge_base.services.file_upload_source import file_io
 
 _NOT_FOUND_CODES = {"NoSuchKey", "404", "NotFound"}
 _MISSING_BUCKET_CODES = {"NoSuchBucket"}
@@ -71,25 +72,62 @@ class S3KnowledgeStorageProvider:
     async def write(
         self,
         location: StorageLocation,
-        content: bytes,
+        content: bytes | Path,
         *,
         content_type: str,
     ) -> StoredObject:
+        size = content.stat().st_size if isinstance(content, Path) else len(content)
+        target = {"Bucket": location.namespace, "Key": location.key}
         try:
             async with self.storage._client() as s3:  # noqa: SLF001
-                await s3.put_object(
-                    Bucket=location.namespace,
-                    Key=location.key,
-                    Body=content,
-                    ContentType=content_type,
-                )
+                if isinstance(content, bytes):
+                    await s3.put_object(
+                        **target, Body=content, ContentType=content_type
+                    )
+                else:
+                    # One part in flight per file; ZIP workers provide concurrency.
+                    part_size = max(8 * 1024 * 1024, (size + 9999) // 10000)
+                    with content.open("rb") as stream:
+                        chunk = await file_io(stream.read, part_size)
+                        if size <= part_size:
+                            await s3.put_object(
+                                **target, Body=chunk, ContentType=content_type
+                            )
+                        else:
+                            response = await s3.create_multipart_upload(
+                                **target, ContentType=content_type
+                            )
+                            upload_id = response["UploadId"]
+                            parts = []
+                            try:
+                                while chunk:
+                                    number = len(parts) + 1
+                                    response = await s3.upload_part(
+                                        **target,
+                                        UploadId=upload_id,
+                                        PartNumber=number,
+                                        Body=chunk,
+                                    )
+                                    parts.append(
+                                        {"ETag": response["ETag"], "PartNumber": number}
+                                    )
+                                    chunk = await file_io(stream.read, part_size)
+                                await s3.complete_multipart_upload(
+                                    **target,
+                                    UploadId=upload_id,
+                                    MultipartUpload={"Parts": parts},
+                                )
+                            except BaseException:
+                                try:
+                                    await s3.abort_multipart_upload(
+                                        **target, UploadId=upload_id
+                                    )
+                                except Exception:
+                                    pass
+                                raise
         except ClientError as exc:
             raise self._translate(exc, "write") from exc
-        return StoredObject(
-            location=location,
-            size=len(content),
-            content_type=content_type,
-        )
+        return StoredObject(location=location, size=size, content_type=content_type)
 
     async def read(self, location: StorageLocation) -> bytes:
         try:

{by_qa-0.4.4.dev2 → by_qa-0.4.4.dev4}/src/by_qa/knowledge_base/repositories/knowledge_fs_entry_repository.py

@@ -1261,6 +1261,21 @@ class KnowledgeFsEntryRepository:
         )
         return [dict(row) for row in await self._fetchall(cursor)]
 
+    async def count_file_entries_by_knowledge_base_id(
+        self, cursor, *, knowledge_base_id: int
+    ) -> int:
+        """Count all live files, regardless of directory depth or build state."""
+        await cursor.execute(
+            """SELECT COUNT(*) AS file_count
+            FROM knowledge_fs_entry
+            WHERE knowledge_base_id = %(knowledge_base_id)s
+              AND is_deleted = FALSE
+              AND entry_type = 'FILE'""",
+            {"knowledge_base_id": knowledge_base_id},
+        )
+        row = await cursor.fetchone()
+        return int(row["file_count"])
+
     async def list_file_entries_by_knowledge_base_id(
         self, cursor, *, knowledge_base_id: int
     ) -> list[dict[str, Any]]:

by_qa-0.4.4.dev4/src/by_qa/knowledge_base/services/file_upload_source.py

@@ -0,0 +1,42 @@
+"""Internal disk-backed upload requests and bounded file operations."""
+
+import asyncio
+import hashlib
+from pathlib import Path
+
+from pydantic import Field, FilePath, field_validator
+
+from by_qa.knowledge_base.api.schemas import KnowledgeItemUploadRequest
+
+UPLOAD_CHUNK_SIZE = 1024 * 1024
+
+
+class FileUploadRequest(KnowledgeItemUploadRequest):
+    """Internal request; local paths are never accepted from the public API."""
+
+    file_content: FilePath = Field(validation_alias="fileContent")
+
+    @field_validator("file_content")
+    @classmethod
+    def nonempty_file(cls, value: Path) -> Path:
+        if value.stat().st_size == 0:
+            raise ValueError("fileContent must not be empty")
+        return value
+
+
+async def file_io(function, *args):
+    """Finish an outstanding disk operation before its temporary files are removed."""
+    task = asyncio.create_task(asyncio.to_thread(function, *args))
+    try:
+        return await asyncio.shield(task)
+    except asyncio.CancelledError:
+        await asyncio.gather(task, return_exceptions=True)
+        raise
+
+
+def file_checksum(source: Path) -> str:
+    digest = hashlib.sha256()
+    with source.open("rb") as stream:
+        while chunk := stream.read(UPLOAD_CHUNK_SIZE):
+            digest.update(chunk)
+    return digest.hexdigest()