by-qa 0.4.4.dev3__tar.gz → 0.4.4.dev4__tar.gz

{by_qa-0.4.4.dev3 → by_qa-0.4.4.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: by-qa
-Version: 0.4.4.dev3
+Version: 0.4.4.dev4
 Summary: Open-source knowledge-base ingestion and retrieval service.
 Author: AI Assistant
 License: MIT

{by_qa-0.4.4.dev3 → by_qa-0.4.4.dev4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "by-qa"
-version = "0.4.4.dev3"
+version = "0.4.4.dev4"
 description = "Open-source knowledge-base ingestion and retrieval service."
 readme = "README.md"
 requires-python = ">=3.12"

{by_qa-0.4.4.dev3 → by_qa-0.4.4.dev4}/src/by_qa/knowledge_base/api/routes.py

@@ -1,11 +1,12 @@
 """Route registration for knowledge base APIs."""
 
-import io
 import json
 import mimetypes
 import zipfile
+from contextlib import AsyncExitStack
 from inspect import isawaitable
-from pathlib import PurePosixPath
+from pathlib import Path, PurePosixPath
+from tempfile import TemporaryDirectory
 from typing import Any, Callable, Optional
 from urllib.parse import quote
 
@@ -64,6 +65,11 @@ from by_qa.knowledge_base.services.errors import (
     KnowledgeBaseConfigurationError,
     KnowledgeBaseValidationError,
 )
+from by_qa.knowledge_base.services.file_upload_source import (
+    UPLOAD_CHUNK_SIZE,
+    FileUploadRequest,
+    file_io,
+)
 from by_qa.knowledge_base.services.knowledge_item_ingestion_service import (
     convert_uploaded_file_to_markdown,
 )
@@ -756,148 +762,168 @@ def register_routes(
         skip_if_duplicate: bool = Form(False, alias="skipIfDuplicate"),
         metadata: str | None = Form(None, alias="metadata"),
     ):
-        try:
-            payload = await file_content.read() if file_content is not None else None
-            request = KnowledgeItemUploadRequest.model_validate(
-                {
-                    "knCode": kn_code,
-                    "filePath": file_path,
-                    "fileDescription": file_description,
-                    "fileContent": payload,
-                    "mimeType": (
-                        file_content.content_type if file_content is not None else None
+        async with AsyncExitStack() as stack:
+            try:
+                filename = (
+                    (file_content.filename or "") if file_content is not None else ""
+                )
+                is_zip = extract_zip and filename.lower().endswith(".zip")
+                if is_zip and file_content is not None:
+                    temp_dir = TemporaryDirectory(prefix="byqa-upload-")
+                    stack.push_async_callback(file_io, temp_dir.cleanup)
+                    payload = Path(temp_dir.name) / "upload.zip"
+                    with payload.open("wb") as stream:
+                        while chunk := await file_content.read(UPLOAD_CHUNK_SIZE):
+                            await file_io(stream.write, chunk)
+                    request_model = FileUploadRequest
+                else:
+                    payload = (
+                        await file_content.read() if file_content is not None else None
+                    )
+                    request_model = KnowledgeItemUploadRequest
+                request = request_model.model_validate(
+                    {
+                        "knCode": kn_code,
+                        "filePath": file_path,
+                        "fileDescription": file_description,
+                        "fileContent": payload,
+                        "mimeType": (
+                            file_content.content_type
+                            if file_content is not None
+                            else None
+                        ),
+                        "processFrontMatter": process_front_matter,
+                        "extractZip": extract_zip,
+                        "skipIfDuplicate": skip_if_duplicate,
+                        "metadata": _parse_optional_metadata_form(metadata),
+                    }
+                )
+            except (ValidationError, ValueError) as exc:
+                return _documented_error_response(
+                    result_msg="request validation failed",
+                    result_object=(
+                        {"errors": json.loads(exc.json())}
+                        if isinstance(exc, ValidationError)
+                        else {"errors": [{"msg": str(exc)}]}
                     ),
-                    "processFrontMatter": process_front_matter,
-                    "extractZip": extract_zip,
-                    "skipIfDuplicate": skip_if_duplicate,
-                    "metadata": _parse_optional_metadata_form(metadata),
-                }
-            )
-        except (ValidationError, ValueError) as exc:
-            return _documented_error_response(
-                result_msg="request validation failed",
-                result_object=(
-                    {"errors": json.loads(exc.json())}
-                    if isinstance(exc, ValidationError)
-                    else {"errors": [{"msg": str(exc)}]}
-                ),
-                status_code=422,
+                    status_code=422,
+                )
+            logger.info(
+                "upload_file request received: kb_code=%s, file_path=%s, has_description=%s, process_front_matter=%s, extract_zip=%s",
+                request.kb_code,
+                request.file_path,
+                request.file_description is not None,
+                request.process_front_matter,
+                request.extract_zip,
             )
-        logger.info(
-            "upload_file request received: kb_code=%s, file_path=%s, has_description=%s, process_front_matter=%s, extract_zip=%s",
-            request.kb_code,
-            request.file_path,
-            request.file_description is not None,
-            request.process_front_matter,
-            request.extract_zip,
-        )
-        filename = (file_content.filename or "") if file_content is not None else ""
-        try:
-            service = await get_knowledge_item_ingestion_service()
-            if request.extract_zip and filename.lower().endswith(".zip"):
-                if not zipfile.is_zipfile(io.BytesIO(payload or b"")):
+            try:
+                service = await get_knowledge_item_ingestion_service()
+                if request.extract_zip and filename.lower().endswith(".zip"):
+                    if not await file_io(zipfile.is_zipfile, payload):
+                        return _documented_error_response(
+                            result_msg="invalid zip file",
+                            result_object={},
+                            status_code=422,
+                        )
+                    directory_service = await get_knowledge_base_service()
+                    batch_service = ZipBatchImportService(
+                        ingestion_service=service,
+                        directory_service=directory_service,
+                    )
+                    result = await batch_service.import_zip(
+                        kb_code=request.kb_code,
+                        target_dir=request.file_path,
+                        zip_path=payload,
+                        process_front_matter=request.process_front_matter,
+                        file_description=request.file_description,
+                        skip_if_duplicate=request.skip_if_duplicate,
+                        metadata=request.metadata,
+                    )
+                    result_object = {
+                        "data": [
+                            item.model_dump(by_alias=True) for item in result.data
+                        ],
+                        "summary": result.summary.model_dump(by_alias=True),
+                    }
+                    if result.post_process_errors:
+                        result_object["postProcessErrors"] = result.post_process_errors
+                    if result.summary.succeeded > 0:
+                        await _try_schedule_event(
+                            background_tasks,
+                            lambda: build_resource_event(
+                                event_type=ResourceEventType.FILE_IMPORTED,
+                                kb_code=request.kb_code,
+                                source_path=None,
+                                target_path=_ensure_leading_slash(request.file_path),
+                                items=[
+                                    {
+                                        "sourcePath": item.file_path,
+                                        "targetPath": (
+                                            item.file_path if item.success else None
+                                        ),
+                                        "resourceType": item.resource_type,
+                                        "success": item.success,
+                                        "error": item.error,
+                                    }
+                                    for item in result.data
+                                ],
+                                result=result.summary.model_dump(),
+                            ),
+                        )
+                    return _documented_success_response(result_object=result_object)
+                # single file
+                file_path_norm = "/" + request.file_path.strip("/")
+                segments = [s for s in file_path_norm.split("/") if s]
+                if any(s == ".." for s in segments):
                     return _documented_error_response(
-                        result_msg="invalid zip file",
-                        result_object={},
-                        status_code=422,
+                        result_msg="unsafe path", result_object={}, status_code=422
                     )
-                directory_service = await get_knowledge_base_service()
-                batch_service = ZipBatchImportService(
-                    ingestion_service=service,
-                    directory_service=directory_service,
-                )
-                result = await batch_service.import_zip(
-                    kb_code=request.kb_code,
-                    target_dir=request.file_path,
-                    zip_bytes=payload,
-                    process_front_matter=request.process_front_matter,
-                    file_description=request.file_description,
-                    skip_if_duplicate=request.skip_if_duplicate,
-                    metadata=request.metadata,
-                )
+                await service.upload_file(request)
                 result_object = {
-                    "data": [item.model_dump(by_alias=True) for item in result.data],
-                    "summary": result.summary.model_dump(by_alias=True),
+                    "data": [
+                        {
+                            "filePath": file_path_norm,
+                            "success": True,
+                            "error": None,
+                        }
+                    ],
+                    "summary": {"total": 1, "succeeded": 1, "failed": 0},
                 }
-                if result.post_process_errors:
-                    result_object["postProcessErrors"] = result.post_process_errors
-                if result.summary.succeeded > 0:
-                    await _try_schedule_event(
-                        background_tasks,
-                        lambda: build_resource_event(
-                            event_type=ResourceEventType.FILE_IMPORTED,
-                            kb_code=request.kb_code,
-                            source_path=None,
-                            target_path=_ensure_leading_slash(request.file_path),
-                            items=[
-                                {
-                                    "sourcePath": item.file_path,
-                                    "targetPath": (
-                                        item.file_path if item.success else None
-                                    ),
-                                    "resourceType": item.resource_type,
-                                    "success": item.success,
-                                    "error": item.error,
-                                }
-                                for item in result.data
-                            ],
-                            result=result.summary.model_dump(),
-                        ),
-                    )
+                await _try_schedule_event(
+                    background_tasks,
+                    lambda: build_resource_event(
+                        event_type=ResourceEventType.FILE_IMPORTED,
+                        kb_code=request.kb_code,
+                        source_path=None,
+                        target_path=file_path_norm,
+                        result=result_object["summary"],
+                    ),
+                )
                 return _documented_success_response(result_object=result_object)
-            # single file
-            file_path_norm = "/" + request.file_path.strip("/")
-            segments = [s for s in file_path_norm.split("/") if s]
-            if any(s == ".." for s in segments):
+            except KnowledgeBaseConfigurationError as exc:
                 return _documented_error_response(
-                    result_msg="unsafe path", result_object={}, status_code=422
+                    result_msg=str(exc),
+                    result_object={},
+                    status_code=503,
+                )
+            except KnowledgeBaseValidationError as exc:
+                return _documented_error_response(
+                    result_msg=str(exc),
+                    result_object={},
+                    status_code=422,
+                )
+            except Exception as exc:
+                logger.exception(
+                    "upload_file unexpected error: kb_code=%s, file_path=%s, error=%s",
+                    request.kb_code,
+                    request.file_path,
+                    exc,
+                )
+                return _documented_error_response(
+                    result_msg=str(exc) or "internal error",
+                    result_object={},
+                    status_code=500,
                 )
-            await service.upload_file(request)
-            result_object = {
-                "data": [
-                    {
-                        "filePath": file_path_norm,
-                        "success": True,
-                        "error": None,
-                    }
-                ],
-                "summary": {"total": 1, "succeeded": 1, "failed": 0},
-            }
-            await _try_schedule_event(
-                background_tasks,
-                lambda: build_resource_event(
-                    event_type=ResourceEventType.FILE_IMPORTED,
-                    kb_code=request.kb_code,
-                    source_path=None,
-                    target_path=file_path_norm,
-                    result=result_object["summary"],
-                ),
-            )
-            return _documented_success_response(result_object=result_object)
-        except KnowledgeBaseConfigurationError as exc:
-            return _documented_error_response(
-                result_msg=str(exc),
-                result_object={},
-                status_code=503,
-            )
-        except KnowledgeBaseValidationError as exc:
-            return _documented_error_response(
-                result_msg=str(exc),
-                result_object={},
-                status_code=422,
-            )
-        except Exception as exc:
-            logger.exception(
-                "upload_file unexpected error: kb_code=%s, file_path=%s, error=%s",
-                request.kb_code,
-                request.file_path,
-                exc,
-            )
-            return _documented_error_response(
-                result_msg=str(exc) or "internal error",
-                result_object={},
-                status_code=500,
-            )
 
     @app.post("/api/v1/knowledgeItems/update")
     @app.post("/api/v1/knowledge-items/update")

{by_qa-0.4.4.dev3 → by_qa-0.4.4.dev4}/src/by_qa/knowledge_base/infrastructure/storage.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
+from pathlib import Path
 from typing import Protocol, runtime_checkable
 
 
@@ -73,10 +74,11 @@ class KnowledgeStorageProvider(Protocol):
     async def write(
         self,
         location: StorageLocation,
-        content: bytes,
+        content: bytes | Path,
         *,
         content_type: str,
-    ) -> StoredObject: ...
+    ) -> StoredObject:
+        """Write bytes or stream a local Path; the caller owns the source file."""
 
     async def read(self, location: StorageLocation) -> bytes: ...
 

{by_qa-0.4.4.dev3 → by_qa-0.4.4.dev4}/src/by_qa/knowledge_base/infrastructure/storage_s3.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from pathlib import PurePosixPath
+from pathlib import Path, PurePosixPath
 from typing import Any
 
 from botocore.exceptions import ClientError
@@ -20,6 +20,7 @@ from by_qa.knowledge_base.infrastructure.storage import (
     StorageOperationError,
     StoredObject,
 )
+from by_qa.knowledge_base.services.file_upload_source import file_io
 
 _NOT_FOUND_CODES = {"NoSuchKey", "404", "NotFound"}
 _MISSING_BUCKET_CODES = {"NoSuchBucket"}
@@ -71,25 +72,62 @@ class S3KnowledgeStorageProvider:
     async def write(
         self,
         location: StorageLocation,
-        content: bytes,
+        content: bytes | Path,
         *,
         content_type: str,
     ) -> StoredObject:
+        size = content.stat().st_size if isinstance(content, Path) else len(content)
+        target = {"Bucket": location.namespace, "Key": location.key}
         try:
             async with self.storage._client() as s3:  # noqa: SLF001
-                await s3.put_object(
-                    Bucket=location.namespace,
-                    Key=location.key,
-                    Body=content,
-                    ContentType=content_type,
-                )
+                if isinstance(content, bytes):
+                    await s3.put_object(
+                        **target, Body=content, ContentType=content_type
+                    )
+                else:
+                    # One part in flight per file; ZIP workers provide concurrency.
+                    part_size = max(8 * 1024 * 1024, (size + 9999) // 10000)
+                    with content.open("rb") as stream:
+                        chunk = await file_io(stream.read, part_size)
+                        if size <= part_size:
+                            await s3.put_object(
+                                **target, Body=chunk, ContentType=content_type
+                            )
+                        else:
+                            response = await s3.create_multipart_upload(
+                                **target, ContentType=content_type
+                            )
+                            upload_id = response["UploadId"]
+                            parts = []
+                            try:
+                                while chunk:
+                                    number = len(parts) + 1
+                                    response = await s3.upload_part(
+                                        **target,
+                                        UploadId=upload_id,
+                                        PartNumber=number,
+                                        Body=chunk,
+                                    )
+                                    parts.append(
+                                        {"ETag": response["ETag"], "PartNumber": number}
+                                    )
+                                    chunk = await file_io(stream.read, part_size)
+                                await s3.complete_multipart_upload(
+                                    **target,
+                                    UploadId=upload_id,
+                                    MultipartUpload={"Parts": parts},
+                                )
+                            except BaseException:
+                                try:
+                                    await s3.abort_multipart_upload(
+                                        **target, UploadId=upload_id
+                                    )
+                                except Exception:
+                                    pass
+                                raise
         except ClientError as exc:
             raise self._translate(exc, "write") from exc
-        return StoredObject(
-            location=location,
-            size=len(content),
-            content_type=content_type,
-        )
+        return StoredObject(location=location, size=size, content_type=content_type)
 
     async def read(self, location: StorageLocation) -> bytes:
         try:

by_qa-0.4.4.dev4/src/by_qa/knowledge_base/services/file_upload_source.py

@@ -0,0 +1,42 @@
+"""Internal disk-backed upload requests and bounded file operations."""
+
+import asyncio
+import hashlib
+from pathlib import Path
+
+from pydantic import Field, FilePath, field_validator
+
+from by_qa.knowledge_base.api.schemas import KnowledgeItemUploadRequest
+
+UPLOAD_CHUNK_SIZE = 1024 * 1024
+
+
+class FileUploadRequest(KnowledgeItemUploadRequest):
+    """Internal request; local paths are never accepted from the public API."""
+
+    file_content: FilePath = Field(validation_alias="fileContent")
+
+    @field_validator("file_content")
+    @classmethod
+    def nonempty_file(cls, value: Path) -> Path:
+        if value.stat().st_size == 0:
+            raise ValueError("fileContent must not be empty")
+        return value
+
+
+async def file_io(function, *args):
+    """Finish an outstanding disk operation before its temporary files are removed."""
+    task = asyncio.create_task(asyncio.to_thread(function, *args))
+    try:
+        return await asyncio.shield(task)
+    except asyncio.CancelledError:
+        await asyncio.gather(task, return_exceptions=True)
+        raise
+
+
+def file_checksum(source: Path) -> str:
+    digest = hashlib.sha256()
+    with source.open("rb") as stream:
+        while chunk := stream.read(UPLOAD_CHUNK_SIZE):
+            digest.update(chunk)
+    return digest.hexdigest()

{by_qa-0.4.4.dev3 → by_qa-0.4.4.dev4}/src/by_qa/knowledge_base/services/knowledge_item_ingestion_service.py

@@ -5,7 +5,7 @@ import hashlib
 import mimetypes
 from collections.abc import Iterable, Mapping
 from dataclasses import dataclass, field
-from pathlib import PurePosixPath
+from pathlib import Path, PurePosixPath
 from typing import Any, Callable
 
 from by_qa.core import logger
@@ -37,6 +37,11 @@ from by_qa.knowledge_base.services.errors import KnowledgeBaseValidationError
 from by_qa.knowledge_base.services.file_build_execution_service import (
     FileBuildTerminalError,
 )
+from by_qa.knowledge_base.services.file_upload_source import (
+    FileUploadRequest,
+    file_checksum,
+    file_io,
+)
 from by_qa.knowledge_base.services.knowledge_document_metadata import (
     ensure_document_kind_metadata,
 )
@@ -175,13 +180,22 @@ class KnowledgeItemIngestionService:
             document_chunking_service=document_chunking_service,
         )
 
-    async def upload_file(self, request: KnowledgeItemUploadRequest) -> dict[str, Any]:
+    async def upload_file(
+        self, request: KnowledgeItemUploadRequest | FileUploadRequest
+    ) -> dict[str, Any]:
         """Upload one original file and register its storage metadata on the file entry."""
+        source = request.file_content
+        source_size = source.stat().st_size if isinstance(source, Path) else len(source)
+        checksum = (
+            await file_io(file_checksum, source)
+            if isinstance(source, Path)
+            else hashlib.sha256(source).hexdigest()
+        )
         logger.info(
             "knowledge_item_ingestion_service.upload_file started: kb_code=%s, file_path=%s, file_size=%s",
             request.kb_code,
             request.file_path,
-            len(request.file_content),
+            source_size,
         )
         normalized_file_path = request.file_path.strip()
         if not normalized_file_path:
@@ -231,7 +245,6 @@ class KnowledgeItemIngestionService:
                     )
                 await connection.commit()
 
-            checksum = hashlib.sha256(request.file_content).hexdigest()
             await self.knowledge_fs_entry_repository.lock_checksum_scope(
                 cursor,
                 knowledge_base_id=knowledge_base_id,
@@ -265,8 +278,11 @@ class KnowledgeItemIngestionService:
                 raise KnowledgeBaseValidationError(str(exc)) from exc
 
             fs_entry_id = self._row_id(file_entry_row)
-            content = request.file_content
+            content = source
+            original_content = source
             if self._is_markdown_upload(normalized_object_path, mime_type):
+                if isinstance(source, Path):
+                    original_content = content = await file_io(source.read_bytes)
                 _, content = split_front_matter(content)
                 content = await self._rewrite_markdown_references(
                     content,
@@ -285,17 +301,16 @@ class KnowledgeItemIngestionService:
             )
 
             stored = await self.storage_provider.write(
-                original_location,
-                content,
-                content_type=mime_type,
+                original_location, content, content_type=mime_type
             )
+            stored_size = source_size if isinstance(content, Path) else len(content)
 
             await self.knowledge_fs_entry_repository.update_file_entry_storage(
                 cursor,
                 fs_entry_id=fs_entry_id,
                 file_description=request.file_description,
                 original_location=stored.location,
-                file_size=len(content),
+                file_size=stored_size,
                 mime_type=mime_type,
                 checksum=checksum,
             )
@@ -304,7 +319,7 @@ class KnowledgeItemIngestionService:
             if request.process_front_matter and self._is_markdown_upload(
                 normalized_object_path, mime_type
             ):
-                front_matter = parse_front_matter(request.file_content)
+                front_matter = parse_front_matter(original_content)
             await upsert_entry_metadata(
                 cursor,
                 metadata_repository=self.file_metadata_value_repository,
@@ -345,7 +360,7 @@ class KnowledgeItemIngestionService:
             await connection.close()
 
     async def find_duplicate_file(
-        self, *, kb_code: str, file_content: bytes
+        self, *, kb_code: str, file_content: bytes | Path
     ) -> str | None:
         """Return the path of a same-checksum file for guarded batch imports."""
         connection = await self.connection_factory()
@@ -357,7 +372,11 @@ class KnowledgeItemIngestionService:
                     f"knowledge base not found: {kb_code}"
                 )
             knowledge_base_id = self._row_id(kb_row)
-            checksum = hashlib.sha256(file_content).hexdigest()
+            checksum = (
+                await file_io(file_checksum, file_content)
+                if isinstance(file_content, Path)
+                else hashlib.sha256(file_content).hexdigest()
+            )
             await self.knowledge_fs_entry_repository.lock_checksum_scope(
                 cursor,
                 knowledge_base_id=knowledge_base_id,