amsdal_ml 0.1.4__py3-none-any.whl → 0.2.0__py3-none-any.whl

amsdal_ml/mcp_client/stdio_client.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import asyncio
 import base64
+import logging
 import os
 from collections.abc import Iterable
 from contextlib import AsyncExitStack
@@ -16,6 +17,8 @@ from mcp.shared.exceptions import McpError
 from amsdal_ml.mcp_client.base import ToolClient
 from amsdal_ml.mcp_client.base import ToolInfo
 
+logger = logging.getLogger(__name__)
+
 
 class StdioClient(ToolClient):
     """
@@ -23,22 +26,22 @@ class StdioClient(ToolClient):
     """
 
     def __init__(
-    self,
-    alias: str,
-    module_or_cmd: str,
-    *args: str,
-    persist_session: bool = True,
-    send_amsdal_config: bool = True,
+        self,
+        alias: str,
+        module_or_cmd: str,
+        *args: str,
+        persist_session: bool = True,
+        send_amsdal_config: bool = True,
     ):
         self.alias = alias
-        if module_or_cmd in ("python", "python3"):
+        if module_or_cmd in ('python', 'python3'):
             self._command = module_or_cmd
             self._args = list(args)
         else:
-            self._command = "python"
-            self._args = ["-m", module_or_cmd]
+            self._command = 'python'
+            self._args = ['-m', module_or_cmd]
         if send_amsdal_config:
-            self._args.append("--amsdal-config")
+            self._args.append('--amsdal-config')
             self._args.append(self._build_amsdal_config_arg())
         self._persist = persist_session
         self._lock = asyncio.Lock()
@@ -82,8 +85,8 @@ class StdioClient(ToolClient):
             ToolInfo(
                 alias=alias,
                 name=t.name,
-                description=(getattr(t, "description", None) or ""),
-                input_schema=(getattr(t, "inputSchema", None) or {}),
+                description=(getattr(t, 'description', None) or ''),
+                input_schema=(getattr(t, 'inputSchema', None) or {}),
             )
             for t in resp_tools
         ]
@@ -127,20 +130,20 @@ class StdioClient(ToolClient):
                 rx, tx = await stack.enter_async_context(stdio_client(params))
                 s = await stack.enter_async_context(ClientSession(rx, tx))
                 await s.initialize()
-                print("Calling tool:", tool_name, "with args:", args)  # noqa: T201
+                logger.debug("Calling tool: %s with args: %s", tool_name, args)
                 res = await self._call_with_timeout(s.call_tool(tool_name, args), timeout=timeout)
-                return getattr(res, "content", res)
+                return getattr(res, 'content', res)
 
         # Persistent session path
         s = await self._ensure_session()
         try:
             res = await self._call_with_timeout(s.call_tool(tool_name, args), timeout=timeout)
-            return getattr(res, "content", res)
+            return getattr(res, 'content', res)
         except (TimeoutError, McpError):
             await self._reset_session()
             s = await self._ensure_session()
             res = await self._call_with_timeout(s.call_tool(tool_name, args), timeout=timeout)
-            return getattr(res, "content", res)
+            return getattr(res, 'content', res)
 
     def _build_amsdal_config_arg(self) -> str:
         """

amsdal_ml/mcp_server/server_retriever_stdio.py

@@ -18,32 +18,29 @@ from amsdal_ml.agents.retriever_tool import retriever_search
 logging.basicConfig(
     level=logging.INFO,
     format='%(asctime)s [%(levelname)s] %(message)s',
-    handlers=[
-        logging.FileHandler("server.log"),
-        logging.StreamHandler(sys.stdout)
-    ]
+    handlers=[logging.FileHandler('server.log'), logging.StreamHandler(sys.stdout)],
 )
 
 parser = argparse.ArgumentParser()
-parser.add_argument("--amsdal-config", required=False, help="Base64-encoded config string")
+parser.add_argument('--amsdal-config', required=False, help='Base64-encoded config string')
 args = parser.parse_args()
 
-logging.info(f"Starting server with args: {args}")
+logging.info(f'Starting server with args: {args}')
 
 if args.amsdal_config:
-    decoded = base64.b64decode(args.amsdal_config).decode("utf-8")
+    decoded = base64.b64decode(args.amsdal_config).decode('utf-8')
     amsdal_config = AmsdalConfig(**json.loads(decoded))
-    logging.info(f"Loaded Amsdal config: {amsdal_config}")
+    logging.info(f'Loaded Amsdal config: {amsdal_config}')
     AmsdalConfigManager().set_config(amsdal_config)
 
     manager: Any
     if amsdal_config.async_mode:
         manager = AsyncAmsdalManager()
-        logging.info("pre-setup")
+        logging.info('pre-setup')
         asyncio.run(cast(Any, manager).setup())
-        logging.info("post-setup")
+        logging.info('post-setup')
         asyncio.run(cast(Any, manager).post_setup())
-        logging.info("manager inited")
+        logging.info('manager inited')
     else:
         manager = AmsdalManager()
         cast(Any, manager).setup()

amsdal_ml/ml_ingesting/__init__.py

@@ -0,0 +1,29 @@
+from amsdal_ml.ml_ingesting.embedders.embedder import Embedder
+from amsdal_ml.ml_ingesting.loaders.loader import Loader
+from amsdal_ml.ml_ingesting.loaders.text_loader import TextLoader
+from amsdal_ml.ml_ingesting.model_ingester import ModelIngester
+from amsdal_ml.ml_ingesting.pipeline import DefaultIngestionPipeline
+from amsdal_ml.ml_ingesting.pipeline_interface import IngestionPipeline
+from amsdal_ml.ml_ingesting.processors.cleaner import Cleaner
+from amsdal_ml.ml_ingesting.splitters.splitter import Splitter
+from amsdal_ml.ml_ingesting.stores.store import EmbeddingStore
+from amsdal_ml.ml_ingesting.types import IngestionSource
+from amsdal_ml.ml_ingesting.types import LoadedDocument
+from amsdal_ml.ml_ingesting.types import LoadedPage
+from amsdal_ml.ml_ingesting.types import TextChunk
+
+__all__ = [
+    'Cleaner',
+    'DefaultIngestionPipeline',
+    'Embedder',
+    'EmbeddingStore',
+    'IngestionPipeline',
+    'IngestionSource',
+    'LoadedDocument',
+    'LoadedPage',
+    'Loader',
+    'ModelIngester',
+    'Splitter',
+    'TextChunk',
+    'TextLoader',
+]

amsdal_ml/ml_ingesting/default_ingesting.py

@@ -25,7 +25,7 @@ _MIN_WORDS_PER_SENT = 4
 
 class DepthLimitReached(str):
     def __str__(self) -> str:
-        return "Truncated due to reached depth limit"
+        return 'Truncated due to reached depth limit'
 
 
 @dataclass
@@ -33,17 +33,17 @@ class VisitedObject:
     obj: Any
 
     def __str__(self) -> str:
-        return f"Recursion reference to object {self.obj}"
+        return f'Recursion reference to object {self.obj}'
 
 
 class MissingRelation(str):
     def __str__(self) -> str:
-        return "Relation not present"
+        return 'Relation not present'
 
 
 class NoChildren(str):
     def __str__(self) -> str:
-        return "No nested data"
+        return 'No nested data'
 
 
 # UP007: use X | Y style
@@ -79,109 +79,109 @@ class DefaultIngesting(MLIngesting):
         self._afacts_transform = afacts_transform
 
     def _default_header(self, instance: Any, facts: list[str]) -> str:
-        doc = getattr(instance.__class__, "__doc__", "") or f"Instance of {instance.__class__.__name__}"
-        return (doc.strip() + "\n\nKey facts:\n" + "\n".join(facts)).strip()
+        doc = getattr(instance.__class__, '__doc__', '') or f'Instance of {instance.__class__.__name__}'
+        return (doc.strip() + '\n\nKey facts:\n' + '\n'.join(facts)).strip()
 
     def _walk_sync(self, obj: Any, depth: int, visited: set[tuple[str, str]]) -> list[str | Marker]:
         if depth > self.max_depth:
-            return [DepthLimitReached("")]
-        key = (obj.__class__.__name__, str(getattr(obj, "object_id", id(obj))))
+            return [DepthLimitReached('')]
+        key = (obj.__class__.__name__, str(getattr(obj, 'object_id', id(obj))))
         if key in visited:
             return [VisitedObject(obj)]
         visited.add(key)
 
         out: list[str | Marker] = []
-        fields = getattr(obj.__class__, "model_fields", {})
-        for name, field in getattr(fields, "items", lambda: [])():
+        fields = getattr(obj.__class__, 'model_fields', {})
+        for name, field in getattr(fields, 'items', lambda: [])():
             try:
                 v = getattr(obj, name)
-                title = getattr(field, "title", None) or name.replace("_", " ").capitalize()
+                title = getattr(field, 'title', None) or name.replace('_', ' ').capitalize()
                 if v is None:
                     continue
                 if isinstance(v, str | int | float | bool | date):
-                    out.append(f"{title}: {v}")
-                elif hasattr(v.__class__, "model_fields"):
+                    out.append(f'{title}: {v}')
+                elif hasattr(v.__class__, 'model_fields'):
                     sub = self._walk_sync(v, depth + 1, visited)
-                    out.append(f'{title} → {"; ".join(map(str, sub))}' if sub else str(NoChildren("")))
+                    out.append(f'{title} → {"; ".join(map(str, sub))}' if sub else str(NoChildren('')))
                 elif isinstance(v, list):
                     simple = [str(x) for x in v if isinstance(x, str | int | float)]
                     if simple:
                         out.append(f'{title}: {", ".join(simple)}')
             except Exception as e:  # noqa: BLE001
-                logger.warning(f"[walk_sync] field {name}: {e}")
+                logger.warning(f'[walk_sync] field {name}: {e}')
 
-        fks = getattr(obj.__class__, "FOREIGN_KEYS", [])
+        fks = getattr(obj.__class__, 'FOREIGN_KEYS', [])
         if not fks and not out:
-            out.append(NoChildren(""))
+            out.append(NoChildren(''))
         for fk in fks:
             try:
                 rel = getattr(obj, fk, None)
                 if rel is None:
-                    out.append(MissingRelation(""))
+                    out.append(MissingRelation(''))
                     continue
                 if isinstance(rel, list):
                     for i, item in enumerate(rel):
-                        if hasattr(item.__class__, "model_fields"):
+                        if hasattr(item.__class__, 'model_fields'):
                             sub = self._walk_sync(item, depth + 1, visited)
                             out.append(f'{fk}[{i}] → {"; ".join(map(str, sub))}')
-                elif hasattr(rel.__class__, "model_fields"):
+                elif hasattr(rel.__class__, 'model_fields'):
                     sub = self._walk_sync(rel, depth + 1, visited)
                     out.append(f'{fk} → {"; ".join(map(str, sub))}')
             except Exception as e:  # noqa: BLE001
-                logger.warning(f"[walk_sync] FK {fk}: {e}")
+                logger.warning(f'[walk_sync] FK {fk}: {e}')
         return out
 
     async def _walk_async(self, obj: Any, depth: int, visited: set[tuple[str, str]]) -> list[str | Marker]:
         if depth > self.max_depth:
-            return [DepthLimitReached("")]
-        key = (obj.__class__.__name__, str(getattr(obj, "object_id", id(obj))))
+            return [DepthLimitReached('')]
+        key = (obj.__class__.__name__, str(getattr(obj, 'object_id', id(obj))))
         if key in visited:
             return [VisitedObject(obj)]
         visited.add(key)
 
         out: list[str | Marker] = []
-        fields = getattr(obj.__class__, "model_fields", {})
-        for name, field in getattr(fields, "items", lambda: [])():
+        fields = getattr(obj.__class__, 'model_fields', {})
+        for name, field in getattr(fields, 'items', lambda: [])():
             try:
                 v = getattr(obj, name)
                 if asyncio.iscoroutine(v):
                     v = await v
-                title = getattr(field, "title", None) or name.replace("_", " ").capitalize()
+                title = getattr(field, 'title', None) or name.replace('_', ' ').capitalize()
                 if v is None:
                     continue
                 if isinstance(v, str | int | float | bool | date):
-                    out.append(f"{title}: {v}")
-                elif hasattr(v.__class__, "model_fields"):
+                    out.append(f'{title}: {v}')
+                elif hasattr(v.__class__, 'model_fields'):
                     sub = await self._walk_async(v, depth + 1, visited)
-                    out.append(f'{title} → {"; ".join(map(str, sub))}' if sub else str(NoChildren("")))
+                    out.append(f'{title} → {"; ".join(map(str, sub))}' if sub else str(NoChildren('')))
                 elif isinstance(v, list):
                     simple = [str(x) for x in v if isinstance(x, str | int | float)]
                     if simple:
                         out.append(f'{title}: {", ".join(simple)}')
             except Exception as e:  # noqa: BLE001
-                logger.warning(f"[walk_async] field {name}: {e}")
+                logger.warning(f'[walk_async] field {name}: {e}')
 
-        fks = getattr(obj.__class__, "FOREIGN_KEYS", [])
+        fks = getattr(obj.__class__, 'FOREIGN_KEYS', [])
         if not fks and not out:
-            out.append(NoChildren(""))
+            out.append(NoChildren(''))
         for fk in fks:
             try:
                 rel = getattr(obj, fk, None)
                 if asyncio.iscoroutine(rel):
                     rel = await rel
                 if rel is None:
-                    out.append(MissingRelation(""))
+                    out.append(MissingRelation(''))
                     continue
                 if isinstance(rel, list):
                     for i, item in enumerate(rel):
-                        if hasattr(item.__class__, "model_fields"):
+                        if hasattr(item.__class__, 'model_fields'):
                             sub = await self._walk_async(item, depth + 1, visited)
                             out.append(f'{fk}[{i}] → {"; ".join(map(str, sub))}')
-                elif hasattr(rel.__class__, "model_fields"):
+                elif hasattr(rel.__class__, 'model_fields'):
                     sub = await self._walk_async(rel, depth + 1, visited)
                     out.append(f'{fk} → {"; ".join(map(str, sub))}')
             except Exception as e:  # noqa: BLE001
-                logger.warning(f"[walk_async] FK {fk}: {e}")
+                logger.warning(f'[walk_async] FK {fk}: {e}')
         return out
 
     def collect_facts(self, instance: Any) -> list[str | Marker]:
@@ -221,38 +221,36 @@ class DefaultIngesting(MLIngesting):
         return list(self._tags)
 
     def _split(self, text: str, max_sentences: int = 7) -> list[str]:
-        sents = re.split(r"(?<=[.!?])\s+", text.strip())
+        sents = re.split(r'(?<=[.!?])\s+', text.strip())
         sents = [s.strip() for s in sents if len(s.split()) >= _MIN_WORDS_PER_SENT]
         chunks: list[str] = []
         cur: list[str] = []
         for s in sents:
-            proposal = (" ".join([*cur, s])).strip()
+            proposal = (' '.join([*cur, s])).strip()
             if self._token_len_fn(proposal) <= self.max_tokens_per_chunk and len(cur) < max_sentences:
                 cur.append(s)
             else:
                 if cur:
-                    ch = " ".join(cur).strip()
-                    if ch and not ch.endswith("."):
-                        ch += "."
+                    ch = ' '.join(cur).strip()
+                    if ch and not ch.endswith('.'):
+                        ch += '.'
                     chunks.append(ch)
                 cur = [s]
         if cur:
-            ch = " ".join(cur).strip()
-            if ch and not ch.endswith("."):
-                ch += "."
+            ch = ' '.join(cur).strip()
+            if ch and not ch.endswith('.'):
+                ch += '.'
             chunks.append(ch)
         return chunks
 
     def _resolve_link(self, instance: Any) -> tuple[str, str]:
         cls = instance.__class__.__name__
-        oid = getattr(instance, "object_id", None)
+        oid = getattr(instance, 'object_id', None)
         if oid is None:
-            oid = str(getattr(instance, "id", None) or id(instance))
+            oid = str(getattr(instance, 'id', None) or id(instance))
         return cls, str(oid)
 
-    def _make_records(
-        self, chunks: list[str], vectors: list[list[float]], tags: list[str]
-    ) -> list[EmbeddingData]:
+    def _make_records(self, chunks: list[str], vectors: list[list[float]], tags: list[str]) -> list[EmbeddingData]:
         out: list[EmbeddingData] = []
         for i, (t, v) in enumerate(zip(chunks[: self.max_chunks], vectors, strict=False)):
             out.append(EmbeddingData(chunk_index=i, raw_text=t, embedding=v, tags=tags))
@@ -262,7 +260,7 @@ class DefaultIngesting(MLIngesting):
         self, instance: Any, embed_func: Callable[[str], list[float]] | None = None
     ) -> list[EmbeddingData]:
         if embed_func is None:
-            msg = "embed_func is required for DefaultIngesting.generate_embeddings"
+            msg = 'embed_func is required for DefaultIngesting.generate_embeddings'
             raise RuntimeError(msg)
         text = self.generate_text(instance)
         chunks = self._split(text)
@@ -274,7 +272,7 @@ class DefaultIngesting(MLIngesting):
         self, instance: Any, embed_func: Callable[[str], Awaitable[list[float]]] | None = None
     ) -> list[EmbeddingData]:
         if embed_func is None:
-            msg = "embed_func is required for DefaultIngesting.agenerate_embeddings"
+            msg = 'embed_func is required for DefaultIngesting.agenerate_embeddings'
             raise RuntimeError(msg)
         text = await self.agenerate_text(instance)
         chunks = self._split(text)

amsdal_ml/ml_ingesting/embedders/__init__.py

@@ -0,0 +1,4 @@
+from amsdal_ml.ml_ingesting.embedders.embedder import Embedder
+from amsdal_ml.ml_ingesting.embedders.openai_embedder import OpenAIEmbedder
+
+__all__ = ['Embedder', 'OpenAIEmbedder']

amsdal_ml/ml_ingesting/embedders/embedder.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+from abc import ABC
+from abc import abstractmethod
+
+
+class Embedder(ABC):
+    @abstractmethod
+    def embed(self, text: str) -> list[float]: ...
+
+    @abstractmethod
+    async def aembed(self, text: str) -> list[float]: ...

amsdal_ml/ml_ingesting/embedders/openai_embedder.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import os
+
+from openai import AsyncOpenAI
+from openai import OpenAI
+
+from amsdal_ml.ml_config import ml_config
+from amsdal_ml.ml_ingesting.embedders.embedder import Embedder
+
+DEFAULT_EMBED_MODEL = ml_config.embed_model_name
+
+
+class OpenAIEmbedder(Embedder):
+    def __init__(self, *, api_key: str | None = None, embed_model: str | None = None) -> None:
+        self.api_key = api_key or ml_config.resolved_openai_key or os.getenv('OPENAI_API_KEY')
+        if not self.api_key:
+            msg = 'OPENAI_API_KEY is required for OpenAIEmbedder'
+            raise RuntimeError(msg)
+        self.embed_model = embed_model or DEFAULT_EMBED_MODEL
+        self.client = OpenAI(api_key=self.api_key)
+        self.aclient = AsyncOpenAI(api_key=self.api_key)
+
+    def embed(self, text: str) -> list[float]:
+        resp = self.client.embeddings.create(model=self.embed_model, input=text)
+        return resp.data[0].embedding
+
+    async def aembed(self, text: str) -> list[float]:
+        resp = await self.aclient.embeddings.create(model=self.embed_model, input=text)
+        return resp.data[0].embedding

amsdal_ml/ml_ingesting/embedding_data.py

@@ -1,3 +1,5 @@
+from typing import Any
+
 from pydantic import BaseModel
 from pydantic import Field
 
@@ -7,3 +9,4 @@ class EmbeddingData(BaseModel):
     raw_text: str = Field(..., title='Raw text used for embedding')
     embedding: list[float] = Field(..., title='Vector embedding')
     tags: list[str] = Field(default_factory=list, title='Embedding tags')
+    metadata: dict[str, Any] = Field(default_factory=dict, title='Embedding metadata')

amsdal_ml/ml_ingesting/loaders/__init__.py

@@ -0,0 +1,6 @@
+from amsdal_ml.ml_ingesting.loaders.folder_loader import FolderLoader
+from amsdal_ml.ml_ingesting.loaders.folder_loader import PdfFolderLoader
+from amsdal_ml.ml_ingesting.loaders.loader import Loader
+from amsdal_ml.ml_ingesting.loaders.pdf_loader import PdfLoader
+
+__all__ = ['FolderLoader', 'Loader', 'PdfFolderLoader', 'PdfLoader']

amsdal_ml/ml_ingesting/loaders/folder_loader.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Iterator
+from pathlib import Path
+
+from amsdal_ml.ml_ingesting.loaders.loader import Loader
+from amsdal_ml.ml_ingesting.loaders.pdf_loader import PdfLoader
+from amsdal_ml.ml_ingesting.types import IngestionSource
+from amsdal_ml.ml_ingesting.types import LoadedDocument
+
+
+class FolderLoader:
+    """Generic folder loader that delegates file parsing to a Loader."""
+
+    def __init__(self, *, loader: Loader) -> None:
+        self.loader = loader
+
+    def _iter_paths(self, folder: Path) -> Iterator[Path]:
+        for path in folder.rglob('*'):
+            if path.is_file() and self._accepts(path):
+                yield path
+
+    def _accepts(self, _path: Path) -> bool:
+        return True
+
+    def _load_path(self, path: Path, *, source: IngestionSource | None) -> LoadedDocument:
+        with path.open('rb') as f:
+            doc = self.loader.load(f, filename=path.name, metadata=(source.metadata if source else None))
+            doc.metadata.setdefault('filename', path.name)
+            doc.metadata.setdefault('path', str(path))
+            return doc
+
+    def load_all(self, folder: str | Path, *, source: IngestionSource | None = None) -> list[LoadedDocument]:
+        root = Path(folder)
+        docs: list[LoadedDocument] = []
+        for path in self._iter_paths(root):
+            docs.append(self._load_path(path, source=source))
+        return docs
+
+    async def aload_all(self, folder: str | Path, *, source: IngestionSource | None = None) -> list[LoadedDocument]:
+        root = Path(folder)
+        tasks = [asyncio.to_thread(self._load_path, path, source=source) for path in self._iter_paths(root)]
+        return await asyncio.gather(*tasks)
+
+
+class PdfFolderLoader(FolderLoader):
+    def __init__(self, *, pdf_loader: Loader | None = None) -> None:
+        super().__init__(loader=pdf_loader or PdfLoader())
+
+    def _accepts(self, path: Path) -> bool:
+        return path.suffix.lower() == '.pdf'

amsdal_ml/ml_ingesting/loaders/loader.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from abc import ABC
+from abc import abstractmethod
+from typing import IO
+from typing import Any
+
+from amsdal_ml.ml_ingesting.types import LoadedDocument
+
+
+class Loader(ABC):
+    @abstractmethod
+    def load(
+        self,
+        file: IO[Any],
+        *,
+        filename: str | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> LoadedDocument: ...
+
+    @abstractmethod
+    async def aload(
+        self,
+        file: IO[Any],
+        *,
+        filename: str | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> LoadedDocument: ...