deepdoc-lib 0.2.0__py3-none-any.whl

deepdoc/download_models.py

@@ -0,0 +1,169 @@
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+from pathlib import Path
+
+
+def _parse_args(argv: list[str]) -> argparse.Namespace:
+    from deepdoc.common import model_store
+
+    parser = argparse.ArgumentParser(
+        prog="deepdoc-download-models",
+        description="Download/cache all Deepdoc model bundles, NLTK data, and tiktoken assets for offline use.",
+    )
+
+    # Default behavior: no args downloads everything using the remote provider into the default cache dirs
+    # (~/.cache/deepdoc unless DEEPDOC_MODEL_HOME is set).
+    parser.add_argument(
+        "--provider",
+        default="modelscope",
+        choices=("auto", "local", "modelscope"),
+        help="Model provider to use (default: %(default)s).",
+    )
+    parser.add_argument(
+        "--model-home",
+        default=None,
+        help="Model cache root (default: $DEEPDOC_MODEL_HOME or ~/.cache/deepdoc).",
+    )
+    parser.add_argument(
+        "--offline",
+        action="store_true",
+        help="Disable remote downloads (also disables NLTK auto-download).",
+    )
+    parser.add_argument(
+        "--bundle",
+        action="append",
+        choices=tuple(model_store.BUNDLES.keys()),
+        help="Bundle(s) to download. Repeatable. Default: all bundles.",
+    )
+    parser.add_argument(
+        "--no-nltk",
+        action="store_true",
+        help="Skip downloading required NLTK resources used by the tokenizer.",
+    )
+    parser.add_argument(
+        "--nltk-data-dir",
+        default=None,
+        help="Where to store NLTK data (default: $DEEPDOC_NLTK_DATA_DIR, $NLTK_DATA, or ~/.cache/deepdoc/nltk_data).",
+    )
+    parser.add_argument(
+        "--no-tiktoken",
+        action="store_true",
+        help="Skip downloading the cached cl100k_base tiktoken file used by token_utils.",
+    )
+    parser.add_argument(
+        "--tiktoken-cache-dir",
+        default=None,
+        help="Where to store the cached tiktoken file (default: $DEEPDOC_TIKTOKEN_CACHE_DIR, $TIKTOKEN_CACHE_DIR, $DEEPDOC_MODEL_HOME/tiktoken_cache, or ~/.cache/deepdoc/tiktoken_cache).",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        default=0,
+        help="Increase logging verbosity (repeatable).",
+    )
+
+    args = parser.parse_args(argv)
+    if not args.bundle:
+        args.bundle = list(model_store.BUNDLES.keys())
+    return args
+
+
+def download_all(*, provider: str = "modelscope", model_home: str | None = None, offline: bool = False) -> dict[str, str]:
+    """Download/cache all bundles into the configured cache directories."""
+    from deepdoc.common import model_store
+
+    resolved: dict[str, str] = {}
+    for bundle in model_store.BUNDLES:
+        resolved[bundle] = model_store.resolve_bundle_dir(
+            bundle,
+            model_home=model_home,
+            provider=provider,
+            offline=offline,
+        )
+    return resolved
+
+
+def main(argv: list[str] | None = None) -> int:
+    argv = list(sys.argv[1:] if argv is None else argv)
+    args = _parse_args(argv)
+
+    log_level = logging.WARNING
+    if args.verbose == 1:
+        log_level = logging.INFO
+    elif args.verbose >= 2:
+        log_level = logging.DEBUG
+    logging.basicConfig(level=log_level, format="%(levelname)s %(message)s")
+
+    model_home: str | None
+    if args.model_home:
+        model_home = str(Path(args.model_home).expanduser().resolve())
+    else:
+        model_home = None
+
+    from deepdoc.common import model_store
+
+    failures: list[str] = []
+    resolved: dict[str, str] = {}
+    tiktoken_cache_path: str | None = None
+
+    for bundle in args.bundle:
+        try:
+            resolved_dir = model_store.resolve_bundle_dir(
+                bundle,
+                model_home=model_home,
+                provider=args.provider,
+                offline=args.offline,
+            )
+            resolved[bundle] = resolved_dir
+        except Exception as exc:
+            failures.append(f"{bundle}: {exc}")
+
+    if not args.no_nltk:
+        try:
+            from deepdoc.depend.nltk_manager import ensure_nltk_data
+
+            ensure_nltk_data(
+                data_dir=args.nltk_data_dir,
+                offline=args.offline,
+            )
+        except Exception as exc:
+            failures.append(f"nltk: {exc}")
+
+    if not args.no_tiktoken:
+        try:
+            from deepdoc.common.tiktoken_cache import configure_tiktoken_cache_env, download_cl100k_base
+
+            tiktoken_cache_path = str(
+                download_cl100k_base(
+                    cache_dir=args.tiktoken_cache_dir,
+                    model_home=model_home,
+                    offline=args.offline,
+                )
+            )
+            configure_tiktoken_cache_env(
+                cache_dir=args.tiktoken_cache_dir,
+                model_home=model_home,
+            )
+        except Exception as exc:
+            failures.append(f"tiktoken: {exc}")
+
+    for bundle_name in sorted(resolved):
+        print(f"{bundle_name}\t{resolved[bundle_name]}")
+
+    if tiktoken_cache_path:
+        print(f"tiktoken\t{tiktoken_cache_path}")
+
+    if failures:
+        for item in failures:
+            print(f"ERROR\t{item}", file=sys.stderr)
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

deepdoc/llm_adapter/__init__.py

@@ -0,0 +1,15 @@
+"""
+LLM Adapter Layer for DeepDoc
+
+This module provides a thin adapter layer that handles LLM-related dependencies,
+allowing DeepDoc to work in both FenixAOS environments and standalone configurations.
+"""
+
+from .adapter import LLMAdapter, LLMType
+from .vision import vision_llm_chunk
+
+__all__ = [
+    "LLMAdapter",
+    "LLMType",
+    "vision_llm_chunk",
+]

deepdoc/llm_adapter/adapter.py

@@ -0,0 +1,223 @@
+"""
+LLM Adapter - Thin wrapper for LLM services
+
+This module provides a unified interface for LLM services that works in both
+FenixAOS environments and standalone configurations.
+"""
+
+import logging
+from enum import Enum
+from typing import Any, Optional, Union
+
+logger = logging.getLogger(__name__)
+
+# Try to import from FenixAOS first, fallback to local implementations
+try:
+    from fenixaos.core.model import LLMType as FenixLLMType
+    from fenixaos.core.model.chat.basic.adapter import create_base_llm
+    from fenixaos.core.model.model import ChatModelConfig, ImageModelConfig
+    FENIXAOS_AVAILABLE = True
+    logger.info("Using FenixAOS LLM services")
+except ImportError:
+    FENIXAOS_AVAILABLE = False
+    logger.info("FenixAOS not available, using local LLM implementations")
+
+# Local LLM implementations
+try:
+    from ..depend.simple_cv_model import create_vision_model
+    LOCAL_VISION_AVAILABLE = True
+except ImportError:
+    LOCAL_VISION_AVAILABLE = False
+
+
+class LLMType(str, Enum):
+    """Unified LLM Type enumeration"""
+    CHAT = "chat"
+    EMBEDDING = "embedding"
+    IMAGE2TEXT = "image2text"
+    SPEECH2TEXT = "speech2text"
+    RERANK = "rerank"
+
+    @classmethod
+    def from_fenix(cls, fenix_type: Any) -> 'LLMType':
+        """Convert FenixAOS LLMType to unified type"""
+        if FENIXAOS_AVAILABLE:
+            # Map FenixAOS types to unified types
+            type_mapping = {
+                'chat': cls.CHAT,
+                'embedding': cls.EMBEDDING,
+                'image2text': cls.IMAGE2TEXT,
+                'speech2text': cls.SPEECH2TEXT,
+                'rerank': cls.RERANK,
+            }
+            return type_mapping.get(str(fenix_type).lower(), cls.CHAT)
+        return cls.CHAT
+
+
+class LLMServiceInterface:
+    """Unified interface for LLM services"""
+
+    def describe_with_prompt(self, image: Union[bytes, Any], prompt: Optional[str] = None) -> str:
+        """Describe an image with optional prompt"""
+        raise NotImplementedError
+
+    def encode(self, texts: list) -> tuple:
+        """Encode texts to embeddings"""
+        raise NotImplementedError
+
+    def chat(self, messages: list, **kwargs) -> str:
+        """Chat completion"""
+        raise NotImplementedError
+
+
+class FenixAOSLLMService(LLMServiceInterface):
+    """LLM Service adapter for FenixAOS environment"""
+
+    def __init__(self, tenant_id: Optional[str], llm_type: LLMType, llm_name: Optional[str] = None, **kwargs):
+        self.tenant_id = tenant_id
+        self.llm_type = llm_type
+        self.llm_name = llm_name
+        self._api_key = kwargs.get('api_key')
+        self._base_url = kwargs.get('base_url')
+
+        # Try to create vision service from FenixAOS
+        try:
+            self._service = self._create_fenix_service()
+        except Exception as e:
+            logger.warning(f"Failed to create FenixAOS LLM service: {e}")
+            raise
+
+    def _create_fenix_service(self):
+        """Create service using FenixAOS APIs"""
+        # Try to get vision model from FenixAOS
+        try:
+            # Import FenixAOS components
+            from fenixaos.core.model.image.adapter import ImageModelAdapter
+            from fenixaos.core.model.model import ImageModelConfig
+
+            # Create vision model config
+            config = ImageModelConfig(
+                id=f"deepdoc_vision_{self.llm_name or 'default'}",
+                model_name=self.llm_name or "gpt-4-vision-preview",
+                model_provider="openai",  # Default to OpenAI
+                api_key=getattr(self, '_api_key', None),
+                base_url=getattr(self, '_base_url', None),
+            )
+
+            # Create and return adapter
+            return ImageModelAdapter(config)
+
+        except Exception as e:
+            logger.error(f"Failed to create FenixAOS vision service: {e}")
+            raise
+
+    def describe_with_prompt(self, image: Union[bytes, Any], prompt: Optional[str] = None) -> str:
+        return self._service.describe_with_prompt(image, prompt)
+
+    def encode(self, texts: list) -> tuple:
+        return self._service.encode(texts)
+
+    def chat(self, messages: list, **kwargs) -> str:
+        return self._service.chat(messages, **kwargs)
+
+
+class LocalLLMService(LLMServiceInterface):
+    """LLM Service using local implementations"""
+
+    def __init__(self, config: Optional[dict] = None, **kwargs):
+        self.config = config or {}
+        self._vision_model = None
+
+        if LOCAL_VISION_AVAILABLE:
+            try:
+                self._vision_model = create_vision_model()
+                logger.info("Local vision model initialized")
+            except Exception as e:
+                logger.warning(f"Failed to initialize local vision model: {e}")
+
+    def describe_with_prompt(self, image: Union[bytes, Any], prompt: Optional[str] = None) -> str:
+        if self._vision_model:
+            return self._vision_model.describe_with_prompt(image, prompt)
+        else:
+            logger.warning("No vision model available")
+            return "Vision model not available"
+
+    def encode(self, texts: list) -> tuple:
+        # Placeholder for embedding functionality
+        logger.warning("Local embedding not implemented")
+        return [], 0
+
+    def chat(self, messages: list, **kwargs) -> str:
+        # Placeholder for chat functionality
+        logger.warning("Local chat not implemented")
+        return "Local chat not available"
+
+
+class LLMAdapter:
+    """Main adapter class that provides unified LLM service access"""
+
+    def __init__(self, tenant_id: Optional[str] = None, llm_type: LLMType = LLMType.IMAGE2TEXT,
+                 llm_name: Optional[str] = None, **kwargs):
+        self.tenant_id = tenant_id
+        self.llm_type = llm_type
+        self.llm_name = llm_name
+        self.kwargs = kwargs
+
+        # Try FenixAOS first, then fallback to local
+        self._service = self._create_service()
+
+    def _create_service(self) -> LLMServiceInterface:
+        """Create appropriate LLM service based on environment"""
+        # Always try FenixAOS first if available, regardless of tenant_id
+        if FENIXAOS_AVAILABLE:
+            try:
+                return FenixAOSLLMService(self.tenant_id, self.llm_type, self.llm_name, **self.kwargs)
+            except Exception as e:
+                logger.warning(f"FenixAOS LLM service creation failed: {e}, falling back to local")
+
+        # Fallback to local implementation
+        return LocalLLMService(**self.kwargs)
+
+    def describe_with_prompt(self, image: Union[bytes, Any], prompt: Optional[str] = None) -> str:
+        """Describe an image with optional prompt"""
+        return self._service.describe_with_prompt(image, prompt)
+
+    def encode(self, texts: list) -> tuple:
+        """Encode texts to embeddings"""
+        return self._service.encode(texts)
+
+    def chat(self, messages: list, **kwargs) -> str:
+        """Chat completion"""
+        return self._service.chat(messages, **kwargs)
+
+    # Compatibility methods for LLMBundle-like interface
+    def bind_tools(self, toolcall_session, tools):
+        """Bind tools (placeholder for compatibility)"""
+        logger.debug("Tool binding not implemented in adapter")
+
+    @property
+    def is_tools(self) -> bool:
+        """Check if tools are supported"""
+        return False
+
+    @property
+    def max_length(self) -> int:
+        """Maximum context length"""
+        return 4096  # Default value
+
+
+def create_llm_service(tenant_id: Optional[str] = None, llm_type: LLMType = LLMType.IMAGE2TEXT,
+                       llm_name: Optional[str] = None, **kwargs) -> LLMAdapter:
+    """
+    Factory function to create LLM service
+
+    Args:
+        tenant_id: Tenant ID (required for FenixAOS)
+        llm_type: Type of LLM service
+        llm_name: Specific model name
+        **kwargs: Additional configuration
+
+    Returns:
+        LLMAdapter instance
+    """
+    return LLMAdapter(tenant_id, llm_type, llm_name, **kwargs)

deepdoc/llm_adapter/utils.py

@@ -0,0 +1,104 @@
+"""
+Utility functions for LLM adapter
+"""
+
+import re
+
+
+def clean_markdown_block(text: str) -> str:
+    """
+    Clean markdown block formatting from text.
+
+    Args:
+        text: Text that may contain markdown code blocks
+
+    Returns:
+        str: Cleaned text without markdown formatting
+    """
+    if not text:
+        return ""
+
+    # Remove markdown code block markers
+    text = re.sub(r'^\s*```markdown\s*\n?', '', text)
+    text = re.sub(r'\n?\s*```\s*$', '', text)
+
+    return text.strip()
+
+
+def extract_image_description(text: str) -> str:
+    """
+    Extract the main description from vision model output.
+
+    Args:
+        text: Raw output from vision model
+
+    Returns:
+        str: Cleaned description
+    """
+    # Clean markdown formatting
+    text = clean_markdown_block(text)
+
+    # Remove common prefixes that vision models might add
+    prefixes_to_remove = [
+        "The image shows",
+        "This image depicts",
+        "The picture shows",
+        "This is an image of",
+        "The photo shows",
+    ]
+
+    for prefix in prefixes_to_remove:
+        if text.lower().startswith(prefix.lower()):
+            # Only remove if it's followed by actual content
+            remaining = text[len(prefix):].strip()
+            if remaining and not remaining.startswith(("a", "an", "the")):
+                continue
+            text = remaining
+            break
+
+    return text.strip()
+
+
+def validate_image_data(image_data: bytes, max_size: int = 10 * 1024 * 1024) -> bool:
+    """
+    Validate image data.
+
+    Args:
+        image_data: Raw image bytes
+        max_size: Maximum allowed size in bytes
+
+    Returns:
+        bool: True if valid
+    """
+    if not image_data:
+        return False
+
+    if len(image_data) > max_size:
+        return False
+
+    # Check for common image signatures
+    if len(image_data) < 8:
+        return False
+
+    # JPEG signature
+    if image_data.startswith(b'\xff\xd8\xff'):
+        return True
+
+    # PNG signature
+    if image_data.startswith(b'\x89PNG\r\n\x1a\n'):
+        return True
+
+    # GIF signature
+    if image_data.startswith(b'GIF87a') or image_data.startswith(b'GIF89a'):
+        return True
+
+    # BMP signature
+    if image_data.startswith(b'BM'):
+        return True
+
+    # WebP signature
+    if image_data.startswith(b'RIFF') and len(image_data) >= 12:
+        if image_data[8:12] == b'WEBP':
+            return True
+
+    return False

deepdoc/llm_adapter/vision.py

@@ -0,0 +1,163 @@
+"""
+Vision LLM utilities for DeepDoc
+
+Enhanced vision_llm_chunk with better error handling and format support.
+"""
+
+import io
+import logging
+from typing import Any, Callable, Optional, Union
+
+logger = logging.getLogger(__name__)
+
+# Try to import markdown cleaning function
+try:
+    from .utils import clean_markdown_block
+except ImportError:
+    def clean_markdown_block(text: str) -> str:
+        """Fallback markdown cleaning function"""
+        import re
+        text = re.sub(r'^\s*```markdown\s*\n?', '', text)
+        text = re.sub(r'\n?\s*```\s*$', '', text)
+        return text.strip()
+
+
+def vision_llm_chunk(
+    binary: Any,
+    vision_model: Any,
+    prompt: Optional[str] = None,
+    callback: Optional[Callable] = None
+) -> str:
+    """
+    Enhanced vision LLM chunk processing with better error handling and format support.
+
+    This is an improved version that supports multiple image formats and provides
+    better error handling compared to the basic depend/vision_llm_chunk.py.
+
+    Args:
+        binary: Image binary data (PIL Image, bytes, or BytesIO)
+        vision_model: Vision model instance with describe_with_prompt method
+        prompt: Optional prompt for image description
+        callback: Optional callback function for progress reporting
+
+    Returns:
+        str: Processed markdown text from vision model
+    """
+    callback = callback or (lambda prog, msg: None)
+
+    img = binary
+    txt = ""
+
+    try:
+        # Convert image to bytes with format fallback
+        img_binary = io.BytesIO()
+
+        # Try different formats in order of preference
+        formats_to_try = ['JPEG', 'PNG', 'WEBP', 'BMP']
+
+        saved_successfully = False
+        for fmt in formats_to_try:
+            try:
+                img.save(img_binary, format=fmt)
+                saved_successfully = True
+                break
+            except Exception:
+                img_binary.seek(0)
+                img_binary.truncate()  # Clear buffer for next attempt
+                continue
+
+        if not saved_successfully:
+            raise ValueError("Unable to save image in any supported format")
+
+        img_binary.seek(0)
+
+        # Call vision model
+        ans = clean_markdown_block(
+            vision_model.describe_with_prompt(img_binary.read(), prompt)
+        )
+
+        txt += "\n" + ans
+
+        return txt
+
+    except Exception as e:
+        error_msg = f"Vision model processing failed: {str(e)}"
+        logger.error(error_msg)
+        callback(-1, error_msg)
+        return ""
+
+
+def vision_llm_chunk_with_fallback(
+    binary: Any,
+    vision_model: Any,
+    prompt: Optional[str] = None,
+    callback: Optional[Callable] = None,
+    fallback_text: str = "Image processing failed"
+) -> str:
+    """
+    Vision LLM chunk processing with fallback text on failure.
+
+    Args:
+        binary: Image binary data
+        vision_model: Vision model instance
+        prompt: Optional prompt
+        callback: Optional callback
+        fallback_text: Text to return on failure
+
+    Returns:
+        str: Processed text or fallback text
+    """
+    result = vision_llm_chunk(binary, vision_model, prompt, callback)
+    return result if result.strip() else fallback_text
+
+
+def batch_vision_llm_chunk(
+    images: list,
+    vision_model: Any,
+    prompts: Optional[list] = None,
+    callback: Optional[Callable] = None,
+    max_workers: int = 3
+) -> list:
+    """
+    Process multiple images in parallel.
+
+    Args:
+        images: List of image binary data
+        vision_model: Vision model instance
+        prompts: Optional list of prompts (same length as images)
+        callback: Optional callback function
+        max_workers: Maximum number of parallel workers
+
+    Returns:
+        list: List of processed text results
+    """
+    from concurrent.futures import ThreadPoolExecutor, as_completed
+
+    if not images:
+        return []
+
+    if prompts and len(prompts) != len(images):
+        raise ValueError("prompts list must have same length as images list")
+
+    results = [None] * len(images)
+    prompts = prompts or [None] * len(images)
+
+    def process_single(idx: int, img: Any, prompt: Optional[str]) -> tuple:
+        try:
+            result = vision_llm_chunk(img, vision_model, prompt, callback)
+            return idx, result
+        except Exception as e:
+            logger.error(f"Failed to process image {idx}: {e}")
+            return idx, ""
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = [
+            executor.submit(process_single, i, img, prompt)
+            for i, (img, prompt) in enumerate(zip(images, prompts))
+        ]
+
+        for future in as_completed(futures):
+            idx, result = future.result()
+            results[idx] = result
+
+    return results