embed-client-dev 4.0.78__py3-none-any.whl

embed_client/md_embed_file_prep.py

@@ -0,0 +1,197 @@
+"""
+Prepare Markdown (or plain text) files as UTF-8 ``json_array`` inputs for ``embed_file``.
+
+Reads a ``.md`` file, optionally strips YAML front matter between the first pair of
+``---`` lines, splits the body on blank lines (``\\n\\n``) into paragraph blocks,
+enforces a per-block UTF-8 byte cap via
+:func:`~embed_client.json_embed_file_preflight.resolve_max_json_block_bytes`,
+and writes a top-level JSON array of strings suitable for
+:func:`~embed_client.local_json_embed_file.embed_file_from_local_json_file`.
+
+Author: Vasiliy Zdanovskiy
+email: vasilyvz@gmail.com
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from typing import Any, List, Optional
+
+from embed_client.exceptions import EmbedClientError
+from embed_client.json_embed_file_preflight import (
+    resolve_max_json_block_bytes,
+    validate_local_json_embed_file,
+)
+
+
+def strip_optional_yaml_front_matter(text: str) -> str:
+    """
+    If *text* begins with a line ``---`` (after optional BOM, already decoded),
+    return the substring after the next line that is exactly ``---`` (stripped).
+    Otherwise return *text* unchanged.
+
+    If an opening ``---`` is present but no closing ``---`` line is found, returns
+    *text* unchanged (no strip).
+    """
+    if not text:
+        return text
+    lines = text.splitlines(keepends=True)
+    if not lines:
+        return text
+    if lines[0].strip() != "---":
+        return text
+    for i in range(1, len(lines)):
+        if lines[i].strip() == "---":
+            return "".join(lines[i + 1 :])
+    return text
+
+
+def split_body_into_text_blocks(body: str, max_block_bytes: int) -> List[str]:
+    """
+    Split *body* on double newlines (paragraph boundaries) only.
+
+    Line endings are normalized to ``\\n`` (``\\r\\n`` and lone ``\\r`` → ``\\n``)
+    before splitting on the two-character separator ``\\n\\n``. Each segment is
+    stripped of leading/trailing whitespace, and segments empty after strip are
+    dropped (runs of blank lines collapse to one boundary).
+
+    Each non-empty block must encode to at most *max_block_bytes* UTF-8 bytes.
+    If any block exceeds the cap, raises :class:`~embed_client.exceptions.EmbedClientError`
+    (no byte-level re-chunking).
+    """
+    if max_block_bytes < 1:
+        raise EmbedClientError("max_block_bytes must be >= 1")
+    normalized = body.replace("\r\n", "\n").replace("\r", "\n")
+    paragraphs = normalized.split("\n\n")
+    out: List[str] = []
+    for raw in paragraphs:
+        block = raw.strip()
+        if not block:
+            continue
+        encoded = block.encode("utf-8")
+        if len(encoded) > max_block_bytes:
+            raise EmbedClientError(
+                "paragraph exceeds max_block_bytes after split on blank lines "
+                f"({len(encoded)} bytes > {max_block_bytes}); shorten the paragraph "
+                "or increase --max-block-bytes / EMBED_CLIENT_MAX_JSON_BLOCK_BYTES"
+            )
+        out.append(block)
+    return out
+
+
+def build_embed_file_json_array_from_markdown_text(
+    text: str,
+    *,
+    max_block_bytes: Optional[int] = None,
+    strip_front_matter: bool = True,
+    max_chunks: Optional[int] = None,
+) -> List[str]:
+    """
+    Return a non-empty list of strings (``json_array`` items) from Markdown/plain text.
+
+    Raises:
+        EmbedClientError: Empty output, or size/schema issues after splitting.
+    """
+    max_b = resolve_max_json_block_bytes(max_block_bytes)
+    body = strip_optional_yaml_front_matter(text) if strip_front_matter else text
+    blocks = split_body_into_text_blocks(body, max_b)
+    if max_chunks is not None:
+        if max_chunks < 1:
+            raise EmbedClientError("max_chunks must be >= 1 when set")
+        blocks = blocks[:max_chunks]
+    if not blocks:
+        raise EmbedClientError(
+            "no non-empty text blocks after preparation (empty file or body?)"
+        )
+    return list(blocks)
+
+
+def prepare_markdown_path_to_json_file(
+    md_path: str,
+    output_path: str,
+    *,
+    max_block_bytes: Optional[int] = None,
+    strip_front_matter: bool = True,
+    max_chunks: Optional[int] = None,
+) -> str:
+    """
+    Read UTF-8 (with optional BOM) from *md_path*, build ``json_array``, write *output_path*,
+    then run :func:`~embed_client.json_embed_file_preflight.validate_local_json_embed_file`
+    on the result.
+
+    Returns:
+        Absolute path to *output_path*.
+    """
+    if not md_path or not md_path.strip():
+        raise EmbedClientError("markdown input path is empty")
+    ap_in = os.path.abspath(md_path)
+    if not os.path.isfile(ap_in):
+        raise EmbedClientError(f"not a file or not found: {ap_in}")
+    if not os.access(ap_in, os.R_OK):
+        raise EmbedClientError(f"file not readable: {ap_in}")
+    try:
+        with open(ap_in, "rb") as f:
+            raw = f.read()
+    except OSError as e:
+        raise EmbedClientError(f"cannot read file: {ap_in}: {e}") from e
+    try:
+        decoded = raw.decode("utf-8-sig")
+    except UnicodeDecodeError as e:
+        raise EmbedClientError(
+            f"file must be UTF-8 (with optional BOM): {ap_in}"
+        ) from e
+
+    data = build_embed_file_json_array_from_markdown_text(
+        decoded,
+        max_block_bytes=max_block_bytes,
+        strip_front_matter=strip_front_matter,
+        max_chunks=max_chunks,
+    )
+    ap_out = os.path.abspath(output_path)
+    _parent = os.path.dirname(ap_out)
+    if _parent:
+        os.makedirs(_parent, exist_ok=True)
+    try:
+        with open(ap_out, "w", encoding="utf-8") as f:
+            json.dump(data, f, ensure_ascii=False)
+    except OSError as e:
+        raise EmbedClientError(f"cannot write file: {ap_out}: {e}") from e
+
+    validate_local_json_embed_file(ap_out, max_block_bytes=max_block_bytes)
+    return ap_out
+
+
+def prepare_markdown_cli_main(args: Any) -> int:
+    """
+    CLI entry for ``prepare-embed-file-json`` (expects argparse namespace attributes).
+    """
+    import sys
+    from pathlib import Path
+
+    try:
+        out_path = getattr(args, "output", None)
+        if not out_path:
+            base = Path(args.md_path).resolve()
+            out_path = str(base.parent / f"{base.stem}_embed_file.json")
+        out = prepare_markdown_path_to_json_file(
+            args.md_path,
+            out_path,
+            max_block_bytes=getattr(args, "max_block_bytes", None),
+            strip_front_matter=not bool(getattr(args, "no_strip_front_matter", False)),
+            max_chunks=getattr(args, "max_chunks", None),
+        )
+        print(out, file=sys.stdout)
+        return 0
+    except EmbedClientError as e:
+        print(f"❌ prepare-embed-file-json: {e}", file=sys.stderr)
+        return 1
+
+
+__all__ = [
+    "build_embed_file_json_array_from_markdown_text",
+    "prepare_markdown_cli_main",
+    "prepare_markdown_path_to_json_file",
+    "split_body_into_text_blocks",
+    "strip_optional_yaml_front_matter",
+]

embed_client/response_normalizer.py

@@ -0,0 +1,160 @@
+"""
+Response Normalizer
+
+This module provides functions to normalize adapter responses to legacy format.
+
+Author: Vasiliy Zdanovskiy
+email: vasilyvz@gmail.com
+"""
+
+from typing import Any, Dict
+
+
+class ResponseNormalizer:
+    """
+    Normalizes adapter responses to legacy format expected by EmbeddingClient.
+
+    Adapter returns responses in format:
+    - Immediate: {"mode": "immediate", "command": "...", "result": {...}, "queued": False}
+    - Queued: {"mode": "queued", "job_id": "...", "status": "...", "result": {...}, "queued": True}
+
+    Legacy format expected:
+    - {"result": {"success": bool, "data": {...}}}
+    - {"error": {"code": int, "message": str, "details": ...}}
+    """
+
+    @staticmethod
+    def normalize_command_response(adapter_response: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Normalize adapter command response to legacy format.
+
+        Args:
+            adapter_response: Response from adapter execute_command_unified
+
+        Returns:
+            Normalized response in legacy format
+        """
+        mode = adapter_response.get("mode", "immediate")
+
+        if mode == "queued":
+            return ResponseNormalizer._normalize_queued_response(adapter_response)
+        else:
+            return ResponseNormalizer._normalize_immediate_response(adapter_response)
+
+    @staticmethod
+    def _normalize_immediate_response(
+        adapter_response: Dict[str, Any],
+    ) -> Dict[str, Any]:
+        """Normalize immediate response."""
+        result = adapter_response.get("result", {})
+
+        # If result is already in legacy format, return as-is
+        if isinstance(result, dict) and ("success" in result or "data" in result):
+            return {"result": result}
+
+        # Wrap in legacy format
+        return {"result": {"success": True, "data": result}}
+
+    @staticmethod
+    def _normalize_queued_response(adapter_response: Dict[str, Any]) -> Dict[str, Any]:
+        """Normalize queued response."""
+        result = adapter_response.get("result")
+        status = adapter_response.get("status", "unknown")
+        job_id = adapter_response.get("job_id")
+
+        # If result is already in legacy format, return as-is
+        if isinstance(result, dict) and ("success" in result or "data" in result):
+            # Add job metadata if available
+            if job_id and "data" in result:
+                result["data"]["job_id"] = job_id
+                result["data"]["status"] = status
+            return {"result": result}
+
+        # Wrap in legacy format with job metadata
+        data: Dict[str, Any]
+        if isinstance(result, dict):
+            data = result
+        else:
+            data = {}
+
+        normalized: Dict[str, Any] = {
+            "result": {
+                "success": status in ("completed", "success"),
+                "data": data,
+            }
+        }
+
+        # Add job metadata
+        if job_id:
+            data["job_id"] = job_id
+            data["status"] = status
+
+        return normalized
+
+    @staticmethod
+    def extract_error_from_adapter(adapter_error: Exception) -> Dict[str, Any]:
+        """
+        Extract error information from adapter exception.
+
+        Args:
+            adapter_error: Exception raised by adapter
+
+        Returns:
+            Error dictionary in legacy format
+        """
+        error_msg = str(adapter_error)
+
+        # Try to extract structured error if available
+        if hasattr(adapter_error, "error"):
+            error_data = adapter_error.error
+            if isinstance(error_data, dict):
+                return {"error": error_data}
+
+        # Default error format
+        return {"error": {"code": -32000, "message": error_msg, "details": None}}
+
+    @staticmethod
+    def normalize_queue_status(queue_status: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Normalize queue status response to legacy format.
+
+        Args:
+            queue_status: Queue status from adapter (can be from queue_get_job_status or embed_job_status)
+
+        Returns:
+            Normalized status in legacy format
+        """
+        # Handle embed_job_status format
+        if "result" in queue_status and isinstance(queue_status["result"], dict):
+            result_data = queue_status["result"]
+            if result_data.get("success") and "data" in result_data:
+                data = result_data["data"]
+                status = data.get("status", "unknown")
+                # Check if done flag is set (embed_job_status specific)
+                if data.get("done", False) and status == "completed":
+                    status = "completed"
+                result = data.get("result")
+                return {
+                    "status": status,
+                    "result": result,
+                    "exists": data.get("exists", True),
+                }
+
+        # Extract status from nested structures (standard queue format)
+        status = queue_status.get("status")
+        if not status:
+            status = (
+                queue_status.get("data", {}).get("status")
+                or queue_status.get("result", {}).get("status")
+                or queue_status.get("result", {}).get("data", {}).get("status")
+                or "unknown"
+            )
+
+        # Extract result
+        result = queue_status.get("result")
+        if not result:
+            result = queue_status.get("data", {}).get("result") or queue_status.get(
+                "data", {}
+            )
+
+        return {"status": status, "result": result, "exists": status != "unknown"}

embed_client/response_parsers.py

@@ -0,0 +1,219 @@
+"""
+Response Parsers
+
+This module provides functions to extract data from API responses.
+These functions were extracted from EmbeddingClient to reduce file size.
+
+Author: Vasiliy Zdanovskiy
+email: vasilyvz@gmail.com
+"""
+
+from typing import Any, Dict, List
+
+
+def extract_embeddings(result: Dict[str, Any]) -> List[List[float]]:
+    """
+    Extract embeddings from API response, supporting both old and new formats.
+
+    Args:
+        result: API response dictionary
+
+    Returns:
+        List of embedding vectors (list of lists of floats)
+
+    Raises:
+        ValueError: If embeddings cannot be extracted from the response
+    """
+    # Handle direct embeddings field (old format compatibility)
+    if "embeddings" in result:
+        return result["embeddings"]
+
+    # Handle result wrapper
+    if "result" in result:
+        res = result["result"]
+
+        # Handle direct list in result (old format)
+        if isinstance(res, list):
+            return res
+
+        if isinstance(res, dict):
+            # Handle old format: result.embeddings
+            if "embeddings" in res:
+                return res["embeddings"]
+
+            # Handle old format: result.data.embeddings
+            if (
+                "data" in res
+                and isinstance(res["data"], dict)
+                and "embeddings" in res["data"]
+            ):
+                return res["data"]["embeddings"]
+
+            # Handle adapter format: result.data.results[].embedding
+            if (
+                "data" in res
+                and isinstance(res["data"], dict)
+                and "results" in res["data"]
+            ):
+                results = res["data"]["results"]
+                if isinstance(results, list):
+                    embeddings = []
+                    for item in results:
+                        if isinstance(item, dict) and "embedding" in item:
+                            embeddings.append(item["embedding"])
+                        else:
+                            raise ValueError(
+                                f"Invalid item format in adapter response: {item}"
+                            )
+                    return embeddings
+
+            # Handle new format: result.data[].embedding
+            if "data" in res and isinstance(res["data"], list):
+                embeddings = []
+                for item in res["data"]:
+                    if isinstance(item, dict) and "embedding" in item:
+                        embeddings.append(item["embedding"])
+                    else:
+                        raise ValueError(
+                            f"Invalid item format in new API response: {item}"
+                        )
+                return embeddings
+
+    raise ValueError(f"Cannot extract embeddings from response: {result}")
+
+
+def extract_embedding_data(result: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """
+    Extract full embedding data from API response (new format only).
+
+    Args:
+        result: API response dictionary
+
+    Returns:
+        List of dictionaries with 'body', 'embedding', 'tokens', and 'bm25_tokens' fields
+
+    Raises:
+        ValueError: If data cannot be extracted or is in old format
+    """
+    if "result" in result and isinstance(result["result"], dict):
+        res = result["result"]
+        if "data" in res and isinstance(res["data"], dict) and "results" in res["data"]:
+            # New format: result.data.results[]
+            results = res["data"]["results"]
+            if isinstance(results, list):
+                # Validate that all items have required fields
+                for i, item in enumerate(results):
+                    if not isinstance(item, dict):
+                        raise ValueError(f"Item {i} is not a dictionary: {item}")
+                    if "body" not in item:
+                        raise ValueError(f"Item {i} missing 'body' field: {item}")
+                    if "embedding" not in item:
+                        raise ValueError(f"Item {i} missing 'embedding' field: {item}")
+                    if "tokens" not in item:
+                        raise ValueError(f"Item {i} missing 'tokens' field: {item}")
+                    if "bm25_tokens" not in item:
+                        raise ValueError(
+                            f"Item {i} missing 'bm25_tokens' field: {item}"
+                        )
+
+                return results
+
+        # Legacy support for old format: result.data[]
+        if "data" in res and isinstance(res["data"], list):
+            # Validate that all items have required fields
+            for i, item in enumerate(res["data"]):
+                if not isinstance(item, dict):
+                    raise ValueError(f"Item {i} is not a dictionary: {item}")
+                if "body" not in item:
+                    raise ValueError(f"Item {i} missing 'body' field: {item}")
+                if "embedding" not in item:
+                    raise ValueError(f"Item {i} missing 'embedding' field: {item}")
+                # Old format had 'chunks' instead of 'tokens'
+                if "chunks" not in item and "tokens" not in item:
+                    raise ValueError(
+                        f"Item {i} missing 'chunks' or 'tokens' field: {item}"
+                    )
+
+            return res["data"]
+
+    raise ValueError(
+        f"Cannot extract embedding data from response (new format required): {result}"
+    )
+
+
+def extract_texts(result: Dict[str, Any]) -> List[str]:
+    """
+    Extract original texts from API response (new format only).
+
+    Args:
+        result: API response dictionary
+
+    Returns:
+        List of original text strings
+
+    Raises:
+        ValueError: If texts cannot be extracted or is in old format
+    """
+    data = extract_embedding_data(result)
+    return [item["body"] for item in data]
+
+
+def extract_chunks(result: Dict[str, Any]) -> List[List[str]]:
+    """
+    Extract text chunks from API response (new format only).
+    Note: This method now extracts 'tokens' instead of 'chunks' for compatibility.
+
+    Args:
+        result: API response dictionary
+
+    Returns:
+        List of token lists for each text
+
+    Raises:
+        ValueError: If chunks cannot be extracted or is in old format
+    """
+    data = extract_embedding_data(result)
+    chunks = []
+    for item in data:
+        # New format uses 'tokens', old format used 'chunks'
+        if "tokens" in item:
+            chunks.append(item["tokens"])
+        elif "chunks" in item:
+            chunks.append(item["chunks"])
+        else:
+            raise ValueError(f"Item missing both 'tokens' and 'chunks' fields: {item}")
+    return chunks
+
+
+def extract_tokens(result: Dict[str, Any]) -> List[List[str]]:
+    """
+    Extract tokens from API response (new format only).
+
+    Args:
+        result: API response dictionary
+
+    Returns:
+        List of token lists for each text
+
+    Raises:
+        ValueError: If tokens cannot be extracted or is in old format
+    """
+    data = extract_embedding_data(result)
+    return [item["tokens"] for item in data]
+
+
+def extract_bm25_tokens(result: Dict[str, Any]) -> List[List[str]]:
+    """
+    Extract BM25 tokens from API response (new format only).
+
+    Args:
+        result: API response dictionary
+
+    Returns:
+        List of BM25 token lists for each text
+
+    Raises:
+        ValueError: If BM25 tokens cannot be extracted or is in old format
+    """
+    data = extract_embedding_data(result)
+    return [item["bm25_tokens"] for item in data]