knowhere-python-sdk 0.1.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

knowhere/__init__.py

@@ -22,6 +22,7 @@ from knowhere._exceptions import (
     ConflictError,
     GatewayTimeoutError,
     InternalServerError,
+    InvalidStateError,
     JobFailedError,
     KnowhereError,
     NotFoundError,
@@ -30,6 +31,7 @@ from knowhere._exceptions import (
     PollingTimeoutError,
     RateLimitError,
     ServiceUnavailableError,
+    ValidationError,
 )
 from knowhere._types import PollProgressCallback, UploadProgressCallback
 from knowhere._version import __version__
@@ -44,6 +46,10 @@ from knowhere.types.result import (
     ImageFileInfo,
     Manifest,
     ParseResult,
+    ProcessingCost,
+    ProcessingMetadata,
+    ProcessingTiming,
+    SlimChunk,
     Statistics,
     TableChunk,
     TableFileInfo,
@@ -58,6 +64,8 @@ __all__: list[str] = [
     "__version__",
     # Exceptions
     "KnowhereError",
+    "ValidationError",
+    "InvalidStateError",
     "APIConnectionError",
     "APITimeoutError",
     "APIStatusError",
@@ -87,6 +95,10 @@ __all__: list[str] = [
     "FileIndex",
     "ImageFileInfo",
     "TableFileInfo",
+    "ProcessingCost",
+    "ProcessingMetadata",
+    "ProcessingTiming",
+    "SlimChunk",
     "BaseChunk",
     "TextChunk",
     "ImageChunk",

knowhere/_base_client.py

@@ -25,6 +25,7 @@ from knowhere._constants import (
 from knowhere._exceptions import (
     APIConnectionError,
     APITimeoutError,
+    ValidationError,
     makeStatusError,
 )
 from knowhere._logging import getLogger, redactSensitiveHeaders
@@ -35,17 +36,23 @@ T = TypeVar("T")
 
 _logger = getLogger()
 
-# Error codes that are safe to retry
-_RETRYABLE_ERROR_CODES: frozenset[str] = frozenset({
-    "rate_limit_exceeded",
-    "service_unavailable",
-    "gateway_timeout",
-    "internal_server_error",
-    "timeout",
+# Error codes that are always safe to retry (matches server ALWAYS_RETRYABLE_ERROR_CODES)
+_ALWAYS_RETRYABLE_ERROR_CODES: frozenset[str] = frozenset({
+    "ABORTED",            # 409 - Concurrency conflict
+    "UNAVAILABLE",        # 503 - Service temporarily down
+    "DEADLINE_EXCEEDED",  # 504 - Timeout
 })
 
-# Status codes that are safe to retry
-_RETRYABLE_STATUS_CODES: frozenset[int] = frozenset({408, 429, 500, 502, 503, 504})
+# RESOURCE_EXHAUSTED (429) is conditionally retryable:
+#   - Rate limit: details.retry_after present → RETRY
+#   - Quota exceeded: no retry_after → DO NOT RETRY
+_CONDITIONALLY_RETRYABLE_ERROR_CODE: str = "RESOURCE_EXHAUSTED"
+
+# HTTP status codes that are always safe to retry
+_ALWAYS_RETRYABLE_STATUS_CODES: frozenset[int] = frozenset({409, 502, 503, 504})
+
+# HTTP status code that is conditionally retryable (only with retry_after)
+_CONDITIONALLY_RETRYABLE_STATUS_CODE: int = 429
 
 
 class BaseClient:
@@ -71,7 +78,7 @@ class BaseClient:
         # Resolve: arg > env > default
         resolved_key: Optional[str] = api_key or os.environ.get(ENV_API_KEY)
         if not resolved_key:
-            raise ValueError(
+            raise ValidationError(
                 "An API key must be provided via the 'api_key' argument "
                 f"or the {ENV_API_KEY} environment variable."
             )
@@ -122,12 +129,68 @@ class BaseClient:
         self,
         status_code: int,
         error_code: Optional[str] = None,
-        details: Optional[Any] = None,
+        details: Optional[Dict[str, Any]] = None,
     ) -> bool:
-        """Decide whether a request should be retried."""
-        if error_code and error_code in _RETRYABLE_ERROR_CODES:
+        """Decide whether a request should be retried.
+
+        Follows server-side retry semantics:
+        - ABORTED, UNAVAILABLE, DEADLINE_EXCEEDED → always retry
+        - RESOURCE_EXHAUSTED (429) → retry only if details.retry_after present
+        - All other errors → never retry
+        """
+        if error_code:
+            if error_code in _ALWAYS_RETRYABLE_ERROR_CODES:
+                return True
+            if error_code == _CONDITIONALLY_RETRYABLE_ERROR_CODE:
+                return self._hasRetryAfter(details)
+            return False
+
+        # Fallback to status code when error_code is unavailable
+        if status_code in _ALWAYS_RETRYABLE_STATUS_CODES:
             return True
-        return status_code in _RETRYABLE_STATUS_CODES
+        if status_code == _CONDITIONALLY_RETRYABLE_STATUS_CODE:
+            return self._hasRetryAfter(details)
+        return False
+
+    @staticmethod
+    def _hasRetryAfter(details: Optional[Dict[str, Any]]) -> bool:
+        """Check if details contains a retry_after hint."""
+        if not isinstance(details, dict):
+            return False
+        retry_after: Any = details.get("retry_after")
+        return retry_after is not None
+
+    @staticmethod
+    def _extractRetryAfter(
+        error_body: Optional[Dict[str, Any]],
+        response: httpx.Response,
+    ) -> Optional[float]:
+        """Extract retry_after from the response body or Retry-After header.
+
+        The server puts retry_after in ``error.details.retry_after``.
+        Falls back to the HTTP ``Retry-After`` header.
+        """
+        # Prefer body: error.details.retry_after
+        if isinstance(error_body, dict):
+            err_obj: Any = error_body.get("error", error_body)
+            if isinstance(err_obj, dict):
+                details: Any = err_obj.get("details")
+                if isinstance(details, dict):
+                    raw: Any = details.get("retry_after")
+                    if raw is not None:
+                        try:
+                            return float(raw)
+                        except (ValueError, TypeError):
+                            pass
+
+        # Fallback: HTTP Retry-After header
+        header_raw: Optional[str] = response.headers.get("retry-after")
+        if header_raw is not None:
+            try:
+                return float(header_raw)
+            except (ValueError, TypeError):
+                pass
+        return None
 
     def _calculateRetryDelay(
         self,
@@ -257,24 +320,24 @@ class SyncAPIClient(BaseClient):
                 response
             )
             error_code: Optional[str] = None
+            error_details: Optional[Dict[str, Any]] = None
             if isinstance(error_body, dict):
                 err_obj: Any = error_body.get("error", error_body)
                 if isinstance(err_obj, dict):
                     error_code = err_obj.get("code")
+                    raw_details: Any = err_obj.get("details")
+                    if isinstance(raw_details, dict):
+                        error_details = raw_details
 
             if (
                 attempt < self.max_retries
-                and self._shouldRetry(response.status_code, error_code)
+                and self._shouldRetry(
+                    response.status_code, error_code, error_details
+                )
             ):
-                retry_after_raw: Optional[str] = response.headers.get(
-                    "retry-after"
+                retry_after_val: Optional[float] = self._extractRetryAfter(
+                    error_body, response
                 )
-                retry_after_val: Optional[float] = None
-                if retry_after_raw:
-                    try:
-                        retry_after_val = float(retry_after_raw)
-                    except (ValueError, TypeError):
-                        pass
                 delay = self._calculateRetryDelay(attempt, retry_after_val)
                 _logger.warning(
                     "Retryable error %d on attempt %d/%d, retrying in %.1fs",
@@ -404,22 +467,24 @@ class AsyncAPIClient(BaseClient):
 
             error_body: Optional[Dict[str, Any]] = self._parseErrorResponse(response)
             error_code: Optional[str] = None
+            error_details: Optional[Dict[str, Any]] = None
             if isinstance(error_body, dict):
                 err_obj: Any = error_body.get("error", error_body)
                 if isinstance(err_obj, dict):
                     error_code = err_obj.get("code")
+                    raw_details: Any = err_obj.get("details")
+                    if isinstance(raw_details, dict):
+                        error_details = raw_details
 
             if (
                 attempt < self.max_retries
-                and self._shouldRetry(response.status_code, error_code)
+                and self._shouldRetry(
+                    response.status_code, error_code, error_details
+                )
             ):
-                retry_after_raw: Optional[str] = response.headers.get("retry-after")
-                retry_after_val: Optional[float] = None
-                if retry_after_raw:
-                    try:
-                        retry_after_val = float(retry_after_raw)
-                    except (ValueError, TypeError):
-                        pass
+                retry_after_val: Optional[float] = self._extractRetryAfter(
+                    error_body, response
+                )
                 delay = self._calculateRetryDelay(attempt, retry_after_val)
                 _logger.warning(
                     "Retryable error %d on attempt %d/%d, retrying in %.1fs",

knowhere/_client.py

@@ -13,6 +13,7 @@ from typing import BinaryIO, Optional, Union, overload
 
 from knowhere._base_client import AsyncAPIClient, SyncAPIClient
 from knowhere._constants import DEFAULT_POLL_INTERVAL, DEFAULT_POLL_TIMEOUT
+from knowhere._exceptions import ValidationError
 from knowhere._logging import getLogger
 from knowhere._types import (
     PollProgressCallback,
@@ -94,9 +95,9 @@ class Knowhere(SyncAPIClient):
         Provide exactly one of *url* or *file*.
         """
         if url and file:
-            raise ValueError("Provide either 'url' or 'file', not both.")
+            raise ValidationError("Provide either 'url' or 'file', not both.")
         if not url and file is None:
-            raise ValueError("Provide either 'url' or 'file'.")
+            raise ValidationError("Provide either 'url' or 'file'.")
 
         # Determine source type and create job
         if url:
@@ -196,9 +197,9 @@ class AsyncKnowhere(AsyncAPIClient):
     ) -> ParseResult:
         """Parse a document end-to-end (async version)."""
         if url and file:
-            raise ValueError("Provide either 'url' or 'file', not both.")
+            raise ValidationError("Provide either 'url' or 'file', not both.")
         if not url and file is None:
-            raise ValueError("Provide either 'url' or 'file'.")
+            raise ValidationError("Provide either 'url' or 'file'.")
 
         if url:
             job: Job = await self.jobs.create(

knowhere/_constants.py

@@ -18,6 +18,7 @@ DEFAULT_POLL_INTERVAL: float = 10.0
 
 # Retry configuration
 DEFAULT_MAX_RETRIES: int = 5
+DEFAULT_UPLOAD_MAX_RETRIES: int = 2
 
 # Polling configuration
 MAX_POLL_INTERVAL: float = 30.0

knowhere/_exceptions.py

@@ -41,6 +41,19 @@ class APITimeoutError(APIConnectionError):
         super().__init__(message)
 
 
+# ---------------------------------------------------------------------------
+# Validation / state
+# ---------------------------------------------------------------------------
+
+
+class ValidationError(KnowhereError):
+    """Raised when the caller provides invalid arguments."""
+
+
+class InvalidStateError(KnowhereError):
+    """Raised when an object is in an unexpected state for the operation."""
+
+
 # ---------------------------------------------------------------------------
 # Polling / job errors
 # ---------------------------------------------------------------------------
@@ -161,9 +174,17 @@ class ConflictError(APIStatusError):
 
 
 class RateLimitError(APIStatusError):
-    """HTTP 429 — includes optional ``retry_after`` hint."""
+    """HTTP 429 — includes optional rate limit hints from the server.
+
+    Attributes:
+        retry_after: Seconds to wait before retrying (``None`` for quota exceeded).
+        limit: Maximum allowed requests in the rate window.
+        period: Rate window unit (``"second"``, ``"minute"``, ``"hour"``, ``"day"``).
+    """
 
     retry_after: Optional[float]
+    limit: Optional[int]
+    period: Optional[str]
 
     def __init__(
         self,
@@ -176,6 +197,8 @@ class RateLimitError(APIStatusError):
         body: Optional[Any] = None,
         response: httpx.Response,
         retry_after: Optional[float] = None,
+        limit: Optional[int] = None,
+        period: Optional[str] = None,
     ) -> None:
         super().__init__(
             status_code,
@@ -187,6 +210,8 @@ class RateLimitError(APIStatusError):
             response=response,
         )
         self.retry_after = retry_after
+        self.limit = limit
+        self.period = period
 
 
 class InternalServerError(APIStatusError):
@@ -194,9 +219,17 @@ class InternalServerError(APIStatusError):
 
 
 class ServiceUnavailableError(APIStatusError):
-    """HTTP 502 / 503 — includes optional ``retry_after`` hint."""
+    """HTTP 502 / 503 — includes optional rate limit hints from the server.
+
+    Attributes:
+        retry_after: Seconds to wait before retrying.
+        limit: Maximum allowed requests in the rate window (optional).
+        period: Rate window unit (optional).
+    """
 
     retry_after: Optional[float]
+    limit: Optional[int]
+    period: Optional[str]
 
     def __init__(
         self,
@@ -209,6 +242,8 @@ class ServiceUnavailableError(APIStatusError):
         body: Optional[Any] = None,
         response: httpx.Response,
         retry_after: Optional[float] = None,
+        limit: Optional[int] = None,
+        period: Optional[str] = None,
     ) -> None:
         super().__init__(
             status_code,
@@ -220,12 +255,22 @@ class ServiceUnavailableError(APIStatusError):
             response=response,
         )
         self.retry_after = retry_after
+        self.limit = limit
+        self.period = period
 
 
 class GatewayTimeoutError(APIStatusError):
-    """HTTP 504 — includes optional ``retry_after`` hint."""
+    """HTTP 504 — includes optional rate limit hints from the server.
+
+    Attributes:
+        retry_after: Seconds to wait before retrying.
+        limit: Maximum allowed requests in the rate window (optional).
+        period: Rate window unit (optional).
+    """
 
     retry_after: Optional[float]
+    limit: Optional[int]
+    period: Optional[str]
 
     def __init__(
         self,
@@ -238,6 +283,8 @@ class GatewayTimeoutError(APIStatusError):
         body: Optional[Any] = None,
         response: httpx.Response,
         retry_after: Optional[float] = None,
+        limit: Optional[int] = None,
+        period: Optional[str] = None,
     ) -> None:
         super().__init__(
             status_code,
@@ -249,6 +296,8 @@ class GatewayTimeoutError(APIStatusError):
             response=response,
         )
         self.retry_after = retry_after
+        self.limit = limit
+        self.period = period
 
 
 # ---------------------------------------------------------------------------
@@ -298,14 +347,36 @@ def makeStatusError(
         status_code, APIStatusError
     )
 
-    # Extract retry_after for classes that support it
+    # Extract retry hints for classes that support them
+    # Prefer body: error.details.retry_after, fallback to HTTP header
     retry_after: Optional[float] = None
-    raw_retry: Optional[str] = response.headers.get("retry-after")
-    if raw_retry is not None:
-        try:
-            retry_after = float(raw_retry)
-        except (ValueError, TypeError):
-            retry_after = None
+    limit: Optional[int] = None
+    period: Optional[str] = None
+
+    if isinstance(details, dict):
+        raw_body_retry: Any = details.get("retry_after")
+        if raw_body_retry is not None:
+            try:
+                retry_after = float(raw_body_retry)
+            except (ValueError, TypeError):
+                pass
+        raw_limit: Any = details.get("limit")
+        if raw_limit is not None:
+            try:
+                limit = int(raw_limit)
+            except (ValueError, TypeError):
+                pass
+        raw_period: Any = details.get("period")
+        if isinstance(raw_period, str):
+            period = raw_period
+
+    if retry_after is None:
+        raw_header_retry: Optional[str] = response.headers.get("retry-after")
+        if raw_header_retry is not None:
+            try:
+                retry_after = float(raw_header_retry)
+            except (ValueError, TypeError):
+                pass
 
     common_kwargs: Dict[str, Any] = dict(
         code=code,
@@ -316,9 +387,31 @@ def makeStatusError(
         response=response,
     )
 
-    if exception_class in (RateLimitError, ServiceUnavailableError, GatewayTimeoutError):
-        return exception_class(
-            status_code, **common_kwargs, retry_after=retry_after  # type: ignore[call-arg]
+    if exception_class is RateLimitError:
+        return RateLimitError(
+            status_code,
+            **common_kwargs,
+            retry_after=retry_after,
+            limit=limit,
+            period=period,
+        )
+
+    if exception_class is ServiceUnavailableError:
+        return ServiceUnavailableError(
+            status_code,
+            **common_kwargs,
+            retry_after=retry_after,
+            limit=limit,
+            period=period,
+        )
+
+    if exception_class is GatewayTimeoutError:
+        return GatewayTimeoutError(
+            status_code,
+            **common_kwargs,
+            retry_after=retry_after,
+            limit=limit,
+            period=period,
         )
 
     return exception_class(status_code, **common_kwargs)

knowhere/_version.py

@@ -1 +1 @@
-__version__ = "0.1.0"  # x-release-please-version
+__version__ = "0.2.1"  # x-release-please-version

knowhere/lib/result_parser.py

@@ -16,8 +16,10 @@ from knowhere.types.result import (
     ImageChunk,
     Manifest,
     ParseResult,
+    SlimChunk,
     TableChunk,
     TextChunk,
+    TextChunkTokens,
 )
 
 _logger = getLogger()
@@ -79,6 +81,38 @@ def _extractFilePath(raw: Dict[str, Any]) -> Optional[str]:
     return fallback
 
 
+def _normalizeTokenList(raw_tokens: List[Any]) -> List[str]:
+    """Return a string-only token list with empty values removed."""
+    normalized_tokens: List[str] = []
+    for raw_token in raw_tokens:
+        token_text: str = str(raw_token).strip()
+        if token_text:
+            normalized_tokens.append(token_text)
+    return normalized_tokens
+
+
+def _parseTextChunkTokens(
+    raw_tokens: Any,
+    *,
+    chunk_id: str,
+) -> Optional[TextChunkTokens]:
+    """Normalize text chunk tokens from the current backend payload."""
+    if raw_tokens is None:
+        return None
+    if isinstance(raw_tokens, bool):
+        raise KnowhereError(
+            f"Invalid tokens payload for text chunk '{chunk_id}': expected list[str], got bool."
+        )
+    if isinstance(raw_tokens, list):
+        return _normalizeTokenList(raw_tokens)
+
+    raise KnowhereError(
+        "Invalid tokens payload for text chunk "
+        f"'{chunk_id}': expected list[str], "
+        f"got {type(raw_tokens).__name__}."
+    )
+
+
 def _buildChunks(
     raw_chunks: List[Dict[str, Any]],
     zf: zipfile.ZipFile,
@@ -101,6 +135,7 @@ def _buildChunks(
                 type="image",
                 content=raw.get("content", ""),
                 path=raw.get("path"),
+                page_nums=metadata.get("page_nums", raw.get("page_nums")),
                 length=metadata.get("length", raw.get("length", 0)),
                 file_path=file_path,
                 original_name=metadata.get("original_name", raw.get("original_name")),
@@ -118,6 +153,7 @@ def _buildChunks(
                 type="table",
                 content=raw.get("content", ""),
                 path=raw.get("path"),
+                page_nums=metadata.get("page_nums", raw.get("page_nums")),
                 length=metadata.get("length", raw.get("length", 0)),
                 file_path=file_path,
                 original_name=metadata.get("original_name", raw.get("original_name")),
@@ -127,15 +163,19 @@ def _buildChunks(
             )
         else:
             metadata = raw.get("metadata", {})
+            chunk_id: str = raw.get("chunk_id", "")
+            raw_tokens: Any = metadata.get("tokens", raw.get("tokens"))
             chunk = TextChunk(
-                chunk_id=raw.get("chunk_id", ""),
+                chunk_id=chunk_id,
                 type="text",
                 content=raw.get("content", ""),
                 path=raw.get("path"),
+                page_nums=metadata.get("page_nums", raw.get("page_nums")),
                 length=metadata.get("length", raw.get("length", 0)),
-                tokens=metadata.get("tokens", raw.get("tokens")),
+                tokens=_parseTextChunkTokens(raw_tokens, chunk_id=chunk_id),
                 keywords=metadata.get("keywords", raw.get("keywords")),
                 summary=metadata.get("summary", raw.get("summary")),
+                connect_to=metadata.get("connect_to", raw.get("connect_to")),
                 relationships=metadata.get("relationships", raw.get("relationships")),
             )
 
@@ -195,12 +235,39 @@ def parseResultZip(
         json.loads(hierarchy_text) if hierarchy_text else None
     )
 
+    # -- Optimized sidecar files --
+    chunks_slim_text: Optional[str] = _readZipText(zf, "chunks_slim.json")
+    parsed_chunks_slim: Any = json.loads(chunks_slim_text) if chunks_slim_text else None
+    if isinstance(parsed_chunks_slim, dict) and "chunks" in parsed_chunks_slim:
+        raw_chunks_slim: List[Dict[str, Any]] = parsed_chunks_slim["chunks"]
+    elif isinstance(parsed_chunks_slim, list):
+        raw_chunks_slim = parsed_chunks_slim
+    else:
+        raw_chunks_slim = []
+    chunks_slim: Optional[List[SlimChunk]] = (
+        [SlimChunk.model_validate(chunk) for chunk in raw_chunks_slim]
+        if chunks_slim_text is not None
+        else None
+    )
+
+    toc_hierarchies_text: Optional[str] = _readZipText(zf, "toc_hierarchies.json")
+    toc_hierarchies: Optional[Any] = (
+        json.loads(toc_hierarchies_text) if toc_hierarchies_text else None
+    )
+
+    kb_csv: Optional[str] = _readZipText(zf, "kb.csv")
+    hierarchy_view_html: Optional[str] = _readZipText(zf, "hierarchy_view.html")
+
     zf.close()
 
     return ParseResult(
         manifest=manifest,
         chunks=chunks,
+        chunks_slim=chunks_slim,
         full_markdown=full_markdown,
         hierarchy=hierarchy,
+        toc_hierarchies=toc_hierarchies,
+        kb_csv=kb_csv,
+        hierarchy_view_html=hierarchy_view_html,
         raw_zip=zip_bytes,
     )