ashmatics-tools 0.7.2__py3-none-any.whl

ashmatics_tools/__init__.py

@@ -0,0 +1,356 @@
+"""Ashmatics Tools - Shared utilities for Ashmatics Knowledge Base applications.
+
+This package provides modular utilities for document processing, embeddings,
+vector stores, LLM clients, and external API integrations.
+
+Installation Options:
+    # Minimal install (no torch/CUDA dependencies)
+    pip install ashmatics-tools
+
+    # With Azure storage and MongoDB
+    pip install ashmatics-tools[api]
+
+    # With document parsing (includes torch)
+    pip install ashmatics-tools[parsers,chunkers]
+
+    # Full install with all features
+    pip install ashmatics-tools[full]
+
+Copyright (c) 2025 Asher Informatics PBC. All Rights Reserved.
+See LICENSE file for full license details.
+"""
+
+import importlib
+from typing import TYPE_CHECKING
+
+__version__ = "0.7.2"
+
+# =============================================================================
+# LAZY LOADING INFRASTRUCTURE
+# =============================================================================
+# We use __getattr__ to lazily import heavy modules only when accessed.
+# This prevents importing torch/CUDA dependencies when they're not needed.
+
+# Map of attribute names to their module paths for lazy loading
+_LAZY_IMPORTS: dict[str, tuple[str, str]] = {
+    # Parsers (heavy - require docling)
+    "DoclingParser": ("ashmatics_tools.parsers.docling_parser", "DoclingParser"),
+    "LlamaParser": ("ashmatics_tools.parsers.llama_parser", "LlamaParser"),
+    "SimpleParser": ("ashmatics_tools.parsers.simple_parser", "SimpleParser"),
+    # Chunkers (heavy - require transformers)
+    "DoclingChunker": ("ashmatics_tools.chunkers.docling_chunker", "DoclingChunker"),
+    "AzureChunker": ("ashmatics_tools.chunkers.azure_chunker", "AzureChunker"),
+    "SimpleChunker": ("ashmatics_tools.chunkers.simple_chunker", "SimpleChunker"),
+    # Embedders
+    "AzureEmbedder": ("ashmatics_tools.embedders.azure_embedder", "AzureEmbedder"),
+    "OpenAIEmbedder": ("ashmatics_tools.embedders.openai_embedder", "OpenAIEmbedder"),
+    # Vector stores
+    "CosmosDBVectorStore": ("ashmatics_tools.vector_stores.cosmosdb_store", "CosmosDBVectorStore"),
+    "PgVectorStore": ("ashmatics_tools.vector_stores.pgvector_store", "PgVectorStore"),
+    "QdrantVectorStore": ("ashmatics_tools.vector_stores.qdrant_store", "QdrantVectorStore"),
+    "VectorIndexManager": ("ashmatics_tools.vector_stores.index_manager", "VectorIndexManager"),
+    "VectorCollectionStats": ("ashmatics_tools.vector_stores.index_manager", "VectorCollectionStats"),
+}
+
+def __getattr__(name: str):
+    """Lazy import for heavy modules."""
+    if name in _LAZY_IMPORTS:
+        module_path, attr_name = _LAZY_IMPORTS[name]
+        module = importlib.import_module(module_path)
+        return getattr(module, attr_name)
+    raise AttributeError(f"module 'ashmatics_tools' has no attribute '{name}'")
+
+
+# =============================================================================
+# EAGERLY IMPORTED: Lightweight core modules
+# These are always available and don't pull in heavy dependencies
+# =============================================================================
+
+# Chunkers - Base classes and factory (lightweight)
+from .chunkers import (
+    ChunkingConfig,
+    ChunkingStrategy,
+    DocumentChunk,
+    create_chunker,
+)
+
+# Document Storage - Artifact storage managers
+from .document_storage import (
+    FigureStorageManager,
+    ProcessedFigure,
+    StoredTable,
+    TableStorageManager,
+)
+
+# Embedders - Base classes and factory (lightweight)
+from .embedders import EmbeddingConfig, EmbeddingProvider, create_embedder
+
+# Embedding pipelines - MongoDB workflows
+from .embedding import MongoDBEmbeddingPipeline
+from .embedding.specialized import (
+    create_ai_card_pipeline,
+    create_framework_pipeline,
+    create_manufacturer_card_pipeline,
+    create_usecase_pipeline,
+)
+
+# Enrichers - Document enrichment components
+from .enrichers import (
+    ConsolidatedTable,
+    DatasetCharacteristics,
+    DomainKnowledgeProvider,
+    ExtractedMetric,
+    ExtractedStudy,
+    MetricsExtractionResult,
+    MetricsExtractor,
+    PatientDemographics,
+    TableCandidate,
+    TableCategory,
+    TableClassifier,
+    TableConsolidator,
+    TrainingDataExtractor,
+    TrainingDataResult,
+)
+
+# External APIs - External data source clients
+from .external_apis import (
+    APIProvider,
+    BaseAPIClient,
+    BaseAPIConfig,
+    OpenFDAClient,
+    OpenFDAConfig,
+    OpenFDAEndpoint,
+    create_api_client,
+    list_api_providers,
+    register_api_provider,
+)
+
+# GraphQL client
+from .graphql.client import GraphQLClient
+
+# LLM - Language model clients (lightweight, uses httpx)
+from .llm import (
+    AzureOpenAIClient,
+    AzureOpenAIConfig,
+    BaseLLMClient,
+    LlamaCppClient,
+    LlamaCppConfig,
+    LLMAPIError,
+    LLMContextLengthError,
+    LLMError,
+    LLMMessage,
+    LLMRateLimitError,
+    LLMResponse,
+    OllamaClient,
+    OllamaConfig,
+    OpenAIClient,
+    OpenAIConfig,
+    TokenUsage,
+    create_llm_client,
+    get_config_class,
+    list_llm_providers,
+    register_llm_provider,
+    unregister_llm_provider,
+)
+
+# MCP Servers - Model Context Protocol servers
+from .mcp_servers import (
+    BaseMCPServer,
+    MCPServerConfig,
+    OpenFDAMCPConfig,
+    OpenFDAMCPServer,
+    create_mcp_server,
+    list_mcp_servers,
+    register_mcp_server,
+)
+
+# Ontology - Term and ontology management
+from .ontology import (
+    AshmaticsOntology,
+    BioPortalClient,
+    CategoryManager,
+    TermResolver,
+)
+
+# Parsers - Base classes and factory (lightweight)
+from .parsers import (
+    DocumentParser,
+    FigureData,
+    ParsedDocument,
+    ParsedSection,
+    ParseError,
+    ParserCapabilities,
+    ParserConfig,
+    ParserType,
+    TableData,
+    create_parser,
+    get_available_parsers,
+)
+
+# Processors - Document processing base
+from .processors.base import DocumentProcessor
+
+# Storage - Cloud storage abstraction (lightweight)
+from .storage import (
+    ADLSStorageClient,
+    AuthType,
+    MinIOStorageClient,
+    StorageClient,
+    StorageConfig,
+    StorageObject,
+    StorageProvider,
+    create_storage_client,
+)
+
+# Utils
+from .utils.export_utils import DataExporter
+from .utils.import_utils import KBImporter
+from .utils.schema_utils import SchemaInspector
+
+# Vector stores - Base classes and factory (lightweight)
+from .vector_stores import VectorStore, VectorStoreProvider, create_vector_store
+from .vector_stores.base import (
+    DistanceMetric,
+    IndexInfo,
+    IndexType,
+    SearchResult,
+    VectorStoreConfig,
+)
+
+__all__ = [
+    # Version
+    "__version__",
+    # Core utilities
+    "KBImporter",
+    "DataExporter",
+    "SchemaInspector",
+    "GraphQLClient",
+    "DocumentProcessor",
+    # Parsers - Base/Factory (always available)
+    "create_parser",
+    "get_available_parsers",
+    "DocumentParser",
+    "ParsedDocument",
+    "ParsedSection",
+    "TableData",
+    "FigureData",
+    "ParserConfig",
+    "ParserType",
+    "ParseError",
+    "ParserCapabilities",
+    # Parsers - Implementations (lazy loaded)
+    "SimpleParser",
+    "DoclingParser",
+    "LlamaParser",
+    # Chunkers - Base/Factory (always available)
+    "create_chunker",
+    "ChunkingConfig",
+    "ChunkingStrategy",
+    "DocumentChunk",
+    # Chunkers - Implementations (lazy loaded)
+    "SimpleChunker",
+    "AzureChunker",
+    "DoclingChunker",
+    # Embedders - Base/Factory (always available)
+    "create_embedder",
+    "EmbeddingConfig",
+    "EmbeddingProvider",
+    # Embedders - Implementations (lazy loaded)
+    "AzureEmbedder",
+    "OpenAIEmbedder",
+    # Embedding pipelines
+    "MongoDBEmbeddingPipeline",
+    "create_framework_pipeline",
+    "create_usecase_pipeline",
+    "create_ai_card_pipeline",
+    "create_manufacturer_card_pipeline",
+    # Vector stores - Base/Factory (always available)
+    "create_vector_store",
+    "VectorStore",
+    "VectorStoreProvider",
+    "VectorStoreConfig",
+    "SearchResult",
+    "IndexInfo",
+    "DistanceMetric",
+    "IndexType",
+    # Vector stores - Implementations (lazy loaded)
+    "VectorIndexManager",
+    "VectorCollectionStats",
+    "CosmosDBVectorStore",
+    "PgVectorStore",
+    "QdrantVectorStore",
+    # Storage
+    "create_storage_client",
+    "StorageClient",
+    "StorageConfig",
+    "StorageObject",
+    "StorageProvider",
+    "AuthType",
+    "ADLSStorageClient",
+    "MinIOStorageClient",
+    # LLM
+    "create_llm_client",
+    "register_llm_provider",
+    "unregister_llm_provider",
+    "list_llm_providers",
+    "get_config_class",
+    "BaseLLMClient",
+    "LLMMessage",
+    "LLMResponse",
+    "TokenUsage",
+    "LLMError",
+    "LLMAPIError",
+    "LLMRateLimitError",
+    "LLMContextLengthError",
+    "AzureOpenAIClient",
+    "AzureOpenAIConfig",
+    "OpenAIClient",
+    "OpenAIConfig",
+    "OllamaClient",
+    "OllamaConfig",
+    "LlamaCppClient",
+    "LlamaCppConfig",
+    # Ontology
+    "TermResolver",
+    "CategoryManager",
+    "BioPortalClient",
+    "AshmaticsOntology",
+    # External APIs
+    "create_api_client",
+    "list_api_providers",
+    "register_api_provider",
+    "BaseAPIClient",
+    "BaseAPIConfig",
+    "APIProvider",
+    "OpenFDAClient",
+    "OpenFDAConfig",
+    "OpenFDAEndpoint",
+    # MCP Servers
+    "create_mcp_server",
+    "list_mcp_servers",
+    "register_mcp_server",
+    "BaseMCPServer",
+    "MCPServerConfig",
+    "OpenFDAMCPServer",
+    "OpenFDAMCPConfig",
+    # Enrichers
+    "TableClassifier",
+    "TableCategory",
+    "TableConsolidator",
+    "TableCandidate",
+    "ConsolidatedTable",
+    "MetricsExtractor",
+    "MetricsExtractionResult",
+    "ExtractedMetric",
+    "ExtractedStudy",
+    "DomainKnowledgeProvider",
+    "TrainingDataExtractor",
+    "TrainingDataResult",
+    "DatasetCharacteristics",
+    "PatientDemographics",
+    # Document Storage
+    "FigureStorageManager",
+    "ProcessedFigure",
+    "TableStorageManager",
+    "StoredTable",
+]

ashmatics_tools/chunkers/__init__.py

@@ -0,0 +1,42 @@
+"""Document chunking utilities with pluggable chunker implementations.
+
+This module provides abstract base classes and concrete implementations for
+chunking documents into smaller pieces suitable for embedding and retrieval.
+
+Supported Chunkers:
+- DoclingChunker: Uses Docling's HybridChunker for token-aware semantic chunking
+- SimpleChunker: Basic paragraph-based chunking without Docling dependency
+- FixedSizeChunker: Simple fixed-size chunking with overlap
+- AzureChunker: Azure-compatible chunker with tiktoken and multiple strategies
+
+Usage:
+    from common.chunkers import create_chunker, ChunkingConfig
+
+    # Create a chunker
+    config = ChunkingConfig(max_tokens=512, use_semantic_splitting=True)
+    chunker = create_chunker(config)
+
+    # Chunk a document
+    chunks = await chunker.chunk_document(
+        content=markdown_text,
+        title="Document Title",
+        source="document.pdf"
+    )
+
+Copyright (c) 2025 Asher Informatics PBC. All Rights Reserved.
+See LICENSE file for full license details.
+"""
+
+from .base import (
+    ChunkingConfig,
+    ChunkingStrategy,
+    DocumentChunk,
+)
+from .factory import create_chunker
+
+__all__ = [
+    "ChunkingConfig",
+    "DocumentChunk",
+    "ChunkingStrategy",
+    "create_chunker",
+]