ai-microcore 5.0.0a1__tar.gz → 5.0.0.dev2__tar.gz

{ai_microcore-5.0.0a1 → ai_microcore-5.0.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ai-microcore
-Version: 5.0.0a1
+Version: 5.0.0.dev2
 Summary: # Minimalistic Foundation for AI Applications
 Keywords: llm,large language models,ai,similarity search,ai search,gpt,openai,framework,adapter
 Author-email: Vitalii Stepanenko <mail@vitaliy.in>
@@ -300,6 +300,10 @@ Benchmark accuracy of 20+ state of the art models on solving olympiad math probl
 
 #### [Generate meme image](https://github.com/Nayjest/ai-microcore/blob/main/examples/generate_meme_image.py)
 Simple example demonstrating image generation using [OpenAI GPT Image](https://platform.openai.com/docs/guides/image-generation?image-generation-model=gpt-image-1) model.
+
+
+#### [Local inference with PyTorch / Transformers](https://github.com/Nayjest/ai-microcore/blob/main/examples/pytorch_transformers.py)
+Text generation using HF/Transformers model locally (example withQwen 3 0.6B).
  
 #### [Other examples](https://github.com/llm-microcore/microcore/tree/main/examples)
 

{ai_microcore-5.0.0a1 → ai_microcore-5.0.0.dev2}/README.md

@@ -269,6 +269,10 @@ Benchmark accuracy of 20+ state of the art models on solving olympiad math probl
 
 #### [Generate meme image](https://github.com/Nayjest/ai-microcore/blob/main/examples/generate_meme_image.py)
 Simple example demonstrating image generation using [OpenAI GPT Image](https://platform.openai.com/docs/guides/image-generation?image-generation-model=gpt-image-1) model.
+
+
+#### [Local inference with PyTorch / Transformers](https://github.com/Nayjest/ai-microcore/blob/main/examples/pytorch_transformers.py)
+Text generation using HF/Transformers model locally (example withQwen 3 0.6B).
  
 #### [Other examples](https://github.com/llm-microcore/microcore/tree/main/examples)
 

{ai_microcore-5.0.0a1 → ai_microcore-5.0.0.dev2}/microcore/__init__.py

@@ -12,6 +12,7 @@ from . import mcp
 from . import ui
 from . import tokenizing
 from . import presets
+from . import file_cache
 from .embedding_db import SearchResult, AbstractEmbeddingDB, SearchResults
 from .file_storage import storage
 from ._env import configure, env, config, min_setup
@@ -19,12 +20,16 @@ from .logging import use_logging
 from .message_types import UserMsg, AssistantMsg, SysMsg, Msg, PartialMsg
 from .configuration import (
     ApiType,
+    LLMApiBaseError,
+    LLMApiDeploymentIdError,
+    LLMApiKeyError,
+    LLMApiVersionError,
     LLMConfigError,
     Config,
     EmbeddingDbType,
     PRINT_STREAM,
 )
-from .types import BadAIJsonAnswer, BadAIAnswer
+from .types import BadAIJsonAnswer, BadAIAnswer, LLMContextLengthExceededError
 from .wrappers.prompt_wrapper import PromptWrapper
 from .wrappers.llm_response_wrapper import LLMResponse
 from ._llm_functions import llm, allm, llm_parallel
@@ -69,6 +74,15 @@ def use_model(name: str):
     config().LLM_DEFAULT_ARGS["model"] = name
 
 
+def model_names() -> list[str]:
+    """
+    Return a list of available model names from the default LLM client.
+    """
+    if env().default_client is None:
+        raise ValueError("No default LLM client supporting models list configured.")
+    return env().default_client.model_names()
+
+
 def validate_config():
     """
     Validates current MicroCore configuration
@@ -185,7 +199,12 @@ __all__ = [
     "PRINT_STREAM",
     "presets",
     "BadAIAnswer",
+    "LLMApiBaseError",
+    "LLMApiDeploymentIdError",
+    "LLMApiKeyError",
+    "LLMApiVersionError",
     "LLMConfigError",
+    "LLMContextLengthExceededError",
     "LLMResponse",
     "PromptWrapper",
     "parse",
@@ -194,6 +213,7 @@ __all__ = [
     "dedent",
     # submodules
     "embedding_db",
+    "file_cache",
     "file_storage",
     "message_types",
     "utils",
@@ -206,7 +226,8 @@ __all__ = [
     "tokenizing",
     "Metrics",
     "interactive_setup",
+    "model_names",
     # "wrappers",
 ]
 
-__version__ = "5.0.0a1"
+__version__ = "5.0.0dev2"

{ai_microcore-5.0.0a1 → ai_microcore-5.0.0.dev2}/microcore/_env.py

@@ -1,3 +1,6 @@
+"""
+MicroCore environment object / initialization.
+"""
 import os.path
 from dataclasses import dataclass, field, asdict, fields
 from importlib.util import find_spec
@@ -14,9 +17,9 @@ from .configuration import (
     PRINT_STREAM,
 )
 from .presets import MIN_SETUP
+from .lm_client import BaseAIClient
 from .types import TplFunctionType, LLMAsyncFunctionType, LLMFunctionType
 from .templating.jinja2 import make_jinja2_env, make_tpl_function
-from .llm.openai_llm import make_llm_functions as make_openai_llm_functions
 from .llm.local_llm import make_llm_functions as make_local_llm_functions
 
 if TYPE_CHECKING:
@@ -41,6 +44,7 @@ class Env:
     tokenizer: "PreTrainedTokenizer" = field(  # noqa
         default=None, init=False, repr=False
     )
+    default_client: BaseAIClient | None = None
     _mcp_registry: "MCPRegistry" = field(init=False, default=None)
 
     def __post_init__(self):
@@ -58,30 +62,38 @@ class Env:
         raise NotImplementedError
 
     def init_templating(self):
+        """Initialize Jinja2 environment and template function for templates rendering."""
         self.jinja_env = make_jinja2_env(self)
         self.tpl_function = make_tpl_function(self)
 
     @property
-    def mcp_registry(self):
+    def mcp_registry(self) -> "MCPRegistry":
+        """Lazily initialize and return the registry of preconfigured MCP servers."""
         if self._mcp_registry is None:
             from .mcp import MCPRegistry
             self._mcp_registry = MCPRegistry(self.config.MCP_SERVERS)
         return self._mcp_registry
 
     def init_llm(self):
-        if self.config.LLM_API_TYPE == ApiType.NONE:
+        """Initialize language model functions based on configuration."""
 
-            def not_configured(*args, **kwargs) -> "LLMResponse":
-                raise LLMConfigError("Language model is not configured")
+        def default_llm(*args, **kwargs) -> "LLMResponse":
+            if self.default_client:
+                return self.default_client.generate(*args, **kwargs)
+            raise LLMConfigError("Language model is not configured")
 
-            async def a_not_configured(*args, **kwargs) -> "LLMResponse":
-                raise LLMConfigError("Language model is not configured")
+        async def aio_default_llm(*args, **kwargs) -> "LLMResponse":
+            if self.default_client:
+                return await self.default_client.aio.generate(*args, **kwargs)
+            raise LLMConfigError("Language model is not configured")
 
-            self.llm_function, self.llm_async_function = (
-                not_configured,
-                a_not_configured,
-            )
+        self.llm_function, self.llm_async_function = (
+            default_llm,
+            aio_default_llm,
+        )
 
+        if self.config.LLM_API_TYPE == ApiType.NONE:
+            pass
         elif self.config.LLM_API_TYPE == ApiType.FUNCTION:
             self.llm_function, self.llm_async_function = make_local_llm_functions(
                 self.config
@@ -130,25 +142,19 @@ class Env:
                 self.llm_function,
                 self.llm_async_function,
             ) = make_google_vertex_llm_functions(self.config)
-        elif self.config.LLM_API_TYPE == ApiType.GOOGLE_AI_STUDIO:
+        elif self.config.LLM_API_TYPE in (ApiType.GOOGLE, ApiType.GOOGLE_AI_STUDIO):
             try:
-                from .llm.google_genai import (
-                    make_llm_functions as make_google_genai_llm_functions,
-                )
+                from .llm.google_genai import GoogleClient
             except ModuleNotFoundError as e:
                 raise ModuleNotFoundError(
-                    "To use the Google Gemini language models via AI Studio, "
-                    "you need to install the `google-generativeai` package. "
-                    "Run `pip install google-generativeai`."
+                    "To use the Google Gemini language models via Google GenAI SDK, "
+                    "you need to install the `google-genai` package. "
+                    "Run `pip install google-genai`."
                 ) from e
-            (
-                self.llm_function,
-                self.llm_async_function,
-            ) = make_google_genai_llm_functions(self.config)
+            self.default_client = GoogleClient(self.config)
         else:
-            self.llm_function, self.llm_async_function = make_openai_llm_functions(
-                self.config
-            )
+            from .llm.openai import OpenAIClient
+            self.default_client = OpenAIClient(self.config)
 
     def init_similarity_search(self):
         if (
@@ -225,10 +231,10 @@ _env: Env | None = None
 
 
 def env() -> Env:
-    """Returns the current MicroCore environment"""
+    """Return current MicroCore environment object."""
     return _env or Env()
 
 
 def config() -> Config:
-    """Resolve current configuration"""
+    """Resolve current configuration."""
     return env().config

ai_microcore-5.0.0.dev2/microcore/_llm_functions.py

@@ -0,0 +1,299 @@
+import re
+import logging
+from datetime import datetime
+from typing import Any
+
+
+from .utils import run_parallel, RETURN_EXCEPTION
+from .wrappers.llm_response_wrapper import LLMResponse, DictFromLLMResponse, ImageGenerationResponse
+from .types import TPrompt, LLMContextLengthExceededError
+from .file_cache import (
+    cache_hit,
+    load_cache,
+    save_cache,
+    build_cache_name,
+    delete_cache
+)
+from ._env import env
+
+
+def convert_exception(e: Exception, model: str = None) -> Exception | None:
+    """
+    Convert LLM exceptions microcore-specific exceptions if possible.
+    Args:
+        e (Exception): Original exception
+        model (str): LLM model name, used for better error messages
+    Returns:
+        Converted exception or None if no conversion is possible
+    """
+    if not isinstance(e, Exception):
+        return None
+    t, msg = f"{type(e).__module__}.{type(e).__name__}", str(e)
+    max_tokens, actual_tokens = None, None
+    if t == "openai.BadRequestError" and "context_length_exceeded" in msg:
+        match = re.search(
+            r"maximum context length is (\d+) tokens.*?resulted in (\d+) tokens",
+            msg
+        )
+        if match:
+            max_tokens = int(match.group(1))
+            actual_tokens = int(match.group(2))
+        return LLMContextLengthExceededError(
+            actual_tokens=actual_tokens,
+            max_tokens=max_tokens,
+            model=model
+        )
+    if t == "anthropic.BadRequestError" and "prompt is too long:" in msg:
+        if match := re.search(r"(\d+)\s+tokens\s+>\s+(\d+)\s+maximum", msg):
+            max_tokens = int(match.group(2))
+            actual_tokens = int(match.group(1))
+        return LLMContextLengthExceededError(
+            actual_tokens=actual_tokens,
+            max_tokens=max_tokens,
+            model=model
+        )
+    if (
+        t == "google.api_core.exceptions.InvalidArgument"
+        and "The input token count exceeds the maximum number of tokens allowed" in msg
+    ):
+        if match := re.search(
+            r"The input token count exceeds the maximum number of tokens allowed (\d+)",
+            msg
+        ):
+            max_tokens = int(match.group(1))
+        return LLMContextLengthExceededError(
+            actual_tokens=actual_tokens,
+            max_tokens=max_tokens,
+            model=model
+        )
+    return None
+
+
+def llm(
+    prompt: TPrompt,
+    retries: int = 0,
+    parse_json: bool | dict = False,
+    file_cache: bool | str = False,
+    **kwargs
+) -> str | LLMResponse | ImageGenerationResponse:
+    """
+    Request Large Language Model synchronously
+
+    Args:
+        prompt (str | Msg | dict | list[str | Msg | dict]): Text to send to LLM.
+        retries (int):
+            Number of retries in case of error.
+            Default is 0 (no retries).
+        parse_json (bool|dict):
+            If True, parses response as JSON,
+            alternatively non-empty dict can be used as parse_json arguments
+            Default is False (no parsing).
+        file_cache (bool | str):
+            If True or non-empty string, enables file caching of LLM responses.
+            If string, it will be used as cache prefix.
+            When enabled, identical requests with identical parameters
+            will return cached responses instead of making new API calls.
+            Default is False (no caching).
+        **kwargs: Parameters supported by the LLM API.
+
+            See parameters supported by the OpenAI:
+
+            - https://platform.openai.com/docs/api-reference/completions/create
+            - https://platform.openai.com/docs/api-reference/chat/create
+
+            **Additional parameters:**
+
+                - callback: callable - callback function
+                to be called on each chunk of text,
+                enables response streaming if supported by the LLM API
+                - callbacks: list[callable] - collection of callbacks
+                to be called on each chunk of text,
+                enables response streaming if supported by the LLM API
+
+    Returns:
+
+        Text generated by the LLM as string
+        with all fields returned by API accessible as an attributes.
+
+        See fields returned by the OpenAI:
+
+        - https://platform.openai.com/docs/api-reference/completions/object
+        - https://platform.openai.com/docs/api-reference/chat/object
+    """
+    [h(prompt, **kwargs) for h in env().llm_before_handlers]
+    start = datetime.now()
+
+    if (file_cache and cache_hit(
+        cache_name := build_cache_name(
+            prompt, kwargs,
+            prefix=file_cache if isinstance(file_cache, str) else "llm_requests"
+        )
+    )):
+        response: LLMResponse = load_cache(cache_name)
+        response.from_file_cache = True
+        tries = 0
+    else:
+        tries = retries + 1
+        while tries > 0:
+            try:
+                tries -= 1
+                response = env().llm_function(prompt, **kwargs)
+                break
+            except Exception as e:  # pylint: disable=W0718
+                converted_exception = convert_exception(e)
+                # If context length exceeded, or no tries left --> do not retry
+                if tries == 0 or isinstance(converted_exception, LLMContextLengthExceededError):
+                    if converted_exception:
+                        raise converted_exception from e
+                    raise e
+                logging.error(f"LLM error: {e}")
+                logging.info(f"Retrying... {tries} retries left")
+                continue
+        try:
+            response.gen_duration = (datetime.now() - start).total_seconds()
+            if not env().config.SAVE_MEMORY:
+                response.prompt = prompt
+        except AttributeError:
+            ...
+        if file_cache:
+            save_cache(cache_name, response)
+    [h(response) for h in env().llm_after_handlers]
+    if tries > 0:
+        retry_params = dict(**kwargs)
+        retry_params["retries"] = tries - 1
+        setattr(
+            response,
+            "_retry_callback",
+            lambda: llm(prompt, **retry_params)
+        )
+    if parse_json:
+        parsing_params = parse_json if isinstance(parse_json, dict) else {}
+        return response.parse_json(**parsing_params)
+    return response
+
+
+async def allm(
+    prompt: TPrompt,
+    retries: int = 0,
+    parse_json: bool | dict = False,
+    file_cache: bool | str = False,
+    **kwargs
+) -> str | LLMResponse | DictFromLLMResponse | ImageGenerationResponse:
+    """
+    Request Large Language Model asynchronously
+
+    Args:
+        prompt (str | Msg | dict | list[str | Msg | dict]): Text to send to LLM.
+        retries (int):
+            Number of retries in case of error.
+            Default is 0 (no retries).
+        parse_json (bool|dict):
+            If True, parses response as JSON,
+            alternatively non-empty dict can be used as parse_json arguments.
+        **kwargs: Parameters supported by the LLM API.
+
+            See parameters supported by the OpenAI:
+
+            - https://platform.openai.com/docs/api-reference/completions/create
+            - https://platform.openai.com/docs/api-reference/chat/create
+
+            **Additional parameters:**
+
+            - callback: callable - callback function
+            to be called on each chunk of text,
+            enables response streaming if supported by the LLM API
+            - callbacks: list[callable] - collection of callbacks
+            to be called on each chunk of text,
+            enables response streaming if supported by the LLM API
+
+            Note: async callbacks are supported only for async LLM API calls
+
+    Returns:
+
+        Text generated by the LLM as string
+        with all fields returned by API accessible as an attributes.
+
+        See fields returned by the OpenAI:
+
+        - https://platform.openai.com/docs/api-reference/completions/object
+        - https://platform.openai.com/docs/api-reference/chat/object
+    """
+    [h(prompt, **kwargs) for h in env().llm_before_handlers]
+    start = datetime.now()
+
+    if (file_cache and cache_hit(
+        cache_name := build_cache_name(
+            prompt, kwargs,
+            prefix=file_cache if isinstance(file_cache, str) else "llm_requests"
+        )
+    )):
+        response: LLMResponse = load_cache(cache_name)
+        response.from_file_cache = True
+        tries = 0
+    else:
+        tries = retries + 1
+        while tries > 0:
+            try:
+                tries -= 1
+                response = await env().llm_async_function(prompt, **kwargs)
+                break
+            except Exception as e:  # pylint: disable=W0718
+                converted_exception = convert_exception(e)
+                # If context length exceeded, or no tries left --> do not retry
+                if tries == 0 or isinstance(converted_exception, LLMContextLengthExceededError):
+                    if converted_exception:
+                        raise converted_exception from e
+                    raise e
+                logging.error(f"LLM error: {e}")
+                logging.info(f"Retrying... {tries} retries left")
+                continue
+        try:
+            response.gen_duration = (datetime.now() - start).total_seconds()
+            if not env().config.SAVE_MEMORY:
+                response.prompt = prompt
+        except AttributeError:
+            ...
+        if file_cache:
+            save_cache(cache_name, response)
+    [h(response) for h in env().llm_after_handlers]
+    if parse_json:
+        try:
+            parsing_params = parse_json if isinstance(parse_json, dict) else {}
+            return response.parse_json(**parsing_params)
+        except Exception as e:  # pylint: disable=W0718
+            if tries > 0:
+                logging.error(f"LLM error: {e}")
+                logging.info(f"Retrying... {tries} retries left")
+                if file_cache:
+                    delete_cache(cache_name)
+                return await allm(prompt, retries=tries - 1, parse_json=parse_json, **kwargs)
+    return response
+
+
+async def llm_parallel(
+    prompts: list[TPrompt],
+    max_concurrent_tasks: int = None,
+    allow_failures: bool = False,
+    return_on_failure: Any = RETURN_EXCEPTION,
+    log_errors: bool = True,
+    **kwargs
+) -> list[str | LLMResponse]:
+    """
+    Execute multiple LLM requests in parallel
+
+    Returns (list[LLMResponse | str]): a list of responses in the same order as the prompts
+    """
+    tasks = [allm(prompt, **kwargs) for prompt in prompts]
+
+    if max_concurrent_tasks is None:
+        max_concurrent_tasks = int(env().config.MAX_CONCURRENT_TASKS or 0)
+    if not max_concurrent_tasks:
+        max_concurrent_tasks = len(tasks)
+
+    return await run_parallel(
+        tasks,
+        max_concurrent_tasks=max_concurrent_tasks,
+        allow_failures=allow_failures,
+        return_on_failure=return_on_failure,
+        log_errors=log_errors,
+    )

ai_microcore-5.0.0.dev2/microcore/_prepare_llm_args.py

@@ -0,0 +1,47 @@
+from dataclasses import asdict
+from typing import Any
+
+from .message_types import DEFAULT_MESSAGE_ROLE, Msg, MsgContent
+from .types import TPrompt
+
+
+def prepare_prompt(prompt) -> str:
+    """Converts prompt to string for LLM completion API"""
+    return "\n".join(
+        [
+            str(p["content"]) if isinstance(p, dict) and "content" in p else str(p)
+            for p in (prompt if isinstance(prompt, list) else [prompt])
+        ]
+    )
+
+
+def prompt_item_to_message_dict(item: Any, strict=False) -> dict | Any:
+    """
+    Convert a single prompt item to message dict for LLM inference chat API (OpenAI-like).
+    Args:
+        item: The prompt item to convert. Can be a string, Msg instance, or dict.
+        strict: If True, raises TypeError for unsupported types. If False, returns the item as is.
+    Returns:
+        A dict representing the message,
+        or the original item if not convertible and strict is False.
+    """
+    if isinstance(item, Msg):
+        message_dict = asdict(item, dict_factory=item.DICT_FACTORY)
+    elif isinstance(item, dict):
+        message_dict = item
+    else:
+        if strict and not isinstance(item, str | MsgContent):
+            raise TypeError(f"Unsupported message type: {type(item)}")
+        message_dict = dict(role=DEFAULT_MESSAGE_ROLE, content=item)
+    return message_dict
+
+
+def prompt_to_message_dicts(prompt: TPrompt, strict=False) -> list[dict | Any]:
+    """
+    Convert prompt to messages for LLM inference chat API (OpenAI-like).
+    Args:
+        prompt: The prompt to convert. Can be a string, Msg instance, dict, or list of these.
+        strict: If True, raises TypeError for unsupported types. If False, returns the item as is.
+    """
+    message_like_items: list[Any] = prompt if isinstance(prompt, list) else [prompt]
+    return [prompt_item_to_message_dict(item, strict=strict) for item in message_like_items]

{ai_microcore-5.0.0a1 → ai_microcore-5.0.0.dev2}/microcore/ai_func/__init__.py

@@ -72,7 +72,7 @@ def func_metadata(func, name=None) -> Dict[str, Any]:
             metadata["args"][key]["type"] = param_type
 
     arg_comments = func_arg_comments(func)
-    for key, val in metadata["args"].items():
+    for key, val in metadata.get("args", {}).items():
         val["comment"] = arg_comments[key]
 
     # Parse docstring
@@ -80,7 +80,7 @@ def func_metadata(func, name=None) -> Dict[str, Any]:
 
     # Add descriptions from parsed docstring to parameters
     for param in parsed_docstring.params:
-        if param.arg_name in metadata["args"]:
+        if param.arg_name in metadata.get("args", []):
             metadata["args"][param.arg_name]["docstr"] = param.description
 
     return metadata

{ai_microcore-5.0.0a1 → ai_microcore-5.0.0.dev2}/microcore/ai_func/ai-func.json.j2

@@ -4,7 +4,7 @@
 {%- for k,v in args.items() %}
   "{{ k }}":
     {%- if v.type %} <{{ v.type }}>{% endif -%}
-    {%- if v.default != 'NOT_SET' %} (default = {{ v.default }}){%endif-%}
+    {%- if v.default != 'NOT_SET' %} (default = {{ v.default }}){% endif -%}
     {%- if not loop.last -%},{%- endif -%}
     {%- if v.comment %} {{ v.comment }}{% endif -%}
 {%- endfor -%}

{ai_microcore-5.0.0a1 → ai_microcore-5.0.0.dev2}/microcore/ai_func/ai-func.pythonic.j2

@@ -3,7 +3,7 @@
 {%- for name,v in args.items() -%}
 {{ "\n\t" }}{{ name }}
     {%- if v.type %}: {{ v.type }}{% endif -%}
-    {%- if v.default != 'NOT_SET' %} = {{ v.default }}{%endif-%}
+    {%- if v.default != 'NOT_SET' %} = {{ v.default }}{% endif -%}
     {%- if not loop.last -%},{%- endif -%}
     {%- if v.comment %} # {{ v.comment }}{% endif -%}
 {%- endfor -%}

{ai_microcore-5.0.0a1 → ai_microcore-5.0.0.dev2}/microcore/ai_func/ai-func.tag.j2

@@ -26,7 +26,7 @@ Tool:<{{ name }}> {{ description or name.replace('_', ' ').capitalize() }}
   {%- for k,v in args.items() %}
   "{{ k }}":
     {%- if v.type %} <{{ v.type }}>{% endif -%}
-    {%- if v.default != 'NOT_SET' %} (default = {{ v.default }}){%endif-%}
+    {%- if v.default != 'NOT_SET' %} (default = {{ v.default }}){% endif -%}
     {%- if not loop.last -%},{%- endif -%}
     {%- if v.comment %} {{ v.comment }}{% endif -%}
   {%- endfor -%}