scitex 2.0.0__py2.py3-none-any.whl

scitex/ai/genai/format_output_func.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-04 01:39:25 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/ai/_gen_ai/_format_output_func.py
+
+"""
+Functionality:
+    - Formats AI model output text
+    - Wraps URLs in HTML anchor tags
+    - Converts markdown to HTML
+    - Handles DOI links specially
+Input:
+    - Raw text output from AI models
+    - Optional API key for masking
+Output:
+    - Formatted HTML text with proper link handling
+Prerequisites:
+    - markdown2 package
+    - Regular expressions support
+"""
+
+"""Imports"""
+import re
+import sys
+from typing import List, Optional
+
+import markdown2
+import matplotlib.pyplot as plt
+import scitex
+
+"""Functions & Classes"""
+
+
+def format_output_func(out_text: str) -> str:
+    """Formats AI output text with proper link handling and markdown conversion.
+
+    Example
+    -------
+    >>> text = "Check https://example.com or doi:10.1234/abc"
+    >>> print(format_output_func(text))
+    Check <a href="https://example.com">https://example.com</a> or <a href="https://doi.org/10.1234/abc">https://doi.org/10.1234/abc</a>
+
+    Parameters
+    ----------
+    out_text : str
+        Raw text output from AI model
+
+    Returns
+    -------
+    str
+        HTML formatted text with proper link handling
+    """
+
+    def find_unwrapped_urls(text: str) -> List[str]:
+        url_pattern = r'(?<!<a href=")(https?://|doi:|http://doi.org/)[^\s,<>"]+'
+        return re.findall(url_pattern, text)
+
+    def add_a_href_tag(text: str) -> str:
+        def replace_url(match) -> str:
+            url = match.group(0)
+            if url.startswith("doi:"):
+                url = "https://doi.org/" + url[4:]
+            return f'<a href="{url}">{url}</a>'
+
+        url_pattern = r'(?<!<a href=")(https?://|doi:|http://doi.org/)[^\s,<>"]+'
+        return re.sub(url_pattern, replace_url, text)
+
+    def add_masked_api_key(text: str, api_key: str) -> str:
+        masked_key = f"{api_key[:4]}****{api_key[-4:]}"
+        return text + f"\n(API Key: {masked_key}"
+
+    formatted_text = markdown2.markdown(out_text)
+    formatted_text = add_a_href_tag(formatted_text)
+    formatted_text = re.sub(r"^<p>(.*)</p>$", r"\1", formatted_text, flags=re.DOTALL)
+    return formatted_text
+
+
+def main() -> None:
+    pass
+
+
+if __name__ == "__main__":
+    CONFIG, sys.stdout, sys.stderr, plt, CC = scitex.gen.start(sys, plt, verbose=False)
+    main()
+    scitex.gen.close(CONFIG, verbose=False, notify=False)
+
+# EOF
+# #!/usr/bin/env python3
+# # -*- coding: utf-8 -*-
+# # Time-stamp: "2024-11-04 01:32:10 (ywatanabe)"
+# # File: ./scitex_repo/src/scitex/ai/_gen_ai/_format_output_func.py
+
+# """This script does XYZ."""
+
+
+# """Imports"""
+# import re
+# import sys
+
+# import markdown2
+# import matplotlib.pyplot as plt
+# import scitex
+
+# """
+# Warnings
+# """
+# # warnings.simplefilter("ignore", UserWarning)
+
+
+# """
+# Config
+# """
+# # CONFIG = scitex.gen.load_configs()
+
+
+# """
+# Functions & Classes
+# """
+
+
+# def format_output_func(out_text):
+#     def find_unwrapped_urls(text):
+#         # Regex to find URLs that are not already within <a href> tags
+#         url_pattern = (
+#             r'(?<!<a href=")(https?://|doi:|http://doi.org/)[^\s,<>"]+'
+#         )
+
+#         # Find all matches that are not already wrapped
+#         unwrapped_urls = re.findall(url_pattern, text)
+
+#         return unwrapped_urls
+
+#     def add_a_href_tag(text):
+#         # Function to replace each URL with its wrapped version
+#         def replace_url(match):
+#             url = match.group(0)
+#             # Normalize DOI URLs
+#             if url.startswith("doi:"):
+#                 url = "https://doi.org/" + url[4:]
+#             return f'<a href="{url}">{url}</a>'
+
+#         # Regex pattern to match URLs not already wrapped in <a> tags
+#         url_pattern = (
+#             r'(?<!<a href=")(https?://|doi:|http://doi.org/)[^\s,<>"]+'
+#         )
+
+#         # Replace all occurrences of unwrapped URLs in the text
+#         updated_text = re.sub(url_pattern, replace_url, text)
+
+#         return updated_text
+
+#     def add_masked_api_key(text, api_key):
+#         masked_api_key = f"{api_key[:4]}****{api_key[-4:]}"
+#         return text + f"\n(API Key: {masked_api_key}"
+
+#     out_text = markdown2.markdown(out_text)
+#     out_text = add_a_href_tag(out_text)
+#     out_text = re.sub(r"^<p>(.*)</p>$", r"\1", out_text, flags=re.DOTALL)
+#     return out_text
+
+
+# def main():
+#     pass
+
+
+# if __name__ == "__main__":
+#     # # Argument Parser
+#     # import argparse
+#     # parser = argparse.ArgumentParser(description='')
+#     # parser.add_argument('--var', '-v', type=int, default=1, help='')
+#     # parser.add_argument('--flag', '-f', action='store_true', default=False, help='')
+#     # args = parser.parse_args()
+
+#     # Main
+#     CONFIG, sys.stdout, sys.stderr, plt, CC = scitex.gen.start(
+#         sys, plt, verbose=False
+#     )
+#     main()
+#     scitex.gen.close(CONFIG, verbose=False, notify=False)
+
+#
+
+# EOF

scitex/ai/genai/genai_factory.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-05-03 11:57:10 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex_repo/src/scitex/ai/_gen_ai/_genai_factory.py
+# ----------------------------------------
+import os
+
+__FILE__ = "./src/scitex/ai/_gen_ai/_genai_factory.py"
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+import random
+
+from .anthropic import Anthropic
+from .deepseek import DeepSeek
+from .google import Google
+from .groq import Groq
+from .llama import Llama
+from .openai import OpenAI
+from .params import MODELS
+from .perplexity import Perplexity
+
+
+def genai_factory(
+    model="gpt-3.5-turbo",
+    stream=False,
+    api_key=None,
+    seed=None,
+    temperature=1.0,
+    n_keep=1,
+    chat_history=None,
+    max_tokens=4096,
+):
+    """Factory function to create an instance of an AI model handler."""
+    AVAILABLE_MODELS = MODELS.name.tolist()
+
+    if model not in AVAILABLE_MODELS:
+        raise ValueError(
+            f'Model "{model}" is not available. Please choose from:{MODELS.name.tolist()}'
+        )
+
+    provider = MODELS[MODELS.name == model].provider.iloc[0]
+
+    # model_class = globals()[provider]
+    model_class = {
+        "OpenAI": OpenAI,
+        "Anthropic": Anthropic,
+        "Google": Google,
+        "Llama": Llama,
+        "Perplexity": Perplexity,
+        "DeepSeek": DeepSeek,
+        "Groq": Groq,
+    }[provider]
+
+    # Select a random API key from the list
+    if isinstance(api_key, (list, tuple)):
+        api_key = random.choice(api_key)
+
+    return model_class(
+        model=model,
+        stream=stream,
+        api_key=api_key,
+        seed=seed,
+        temperature=temperature,
+        n_keep=n_keep,
+        chat_history=chat_history,
+        max_tokens=max_tokens,
+    )
+
+
+# EOF

scitex/ai/genai/google.py

@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-02-06 13:47:23 (ywatanabe)"
+# File: _Google.py
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/ai/_gen_ai/_Google.py"
+
+
+"""
+Functionality:
+    - Implements Google's Generative AI (Gemini) interface
+    - Handles both streaming and static text generation
+Input:
+    - User prompts and chat history
+    - Model configurations and API credentials
+Output:
+    - Generated text responses from Gemini models
+    - Token usage statistics
+Prerequisites:
+    - Google API key (GOOGLE_API_KEY environment variable)
+    - google.generativeai package
+
+DEPRECATED: This module is deprecated. Please use google_provider.py instead.
+The new provider-based architecture offers better modularity and maintainability.
+"""
+
+"""Imports"""
+import os
+import sys
+from pprint import pprint
+from typing import Any, Dict, Generator, List, Optional
+import warnings
+
+import matplotlib.pyplot as plt
+import scitex
+
+from google import genai
+
+from .base_genai import BaseGenAI
+
+warnings.warn(
+    "google.py is deprecated. Please use google_provider.py instead. "
+    "See PROVIDER_MIGRATION_GUIDE.md for migration instructions.",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+"""Functions & Classes"""
+
+
+class Google(BaseGenAI):
+    def __init__(
+        self,
+        system_setting: str = "",
+        api_key: Optional[str] = os.getenv("GOOGLE_API_KEY"),
+        model: str = "gemini-1.5-pro-latest",
+        stream: bool = False,
+        seed: Optional[int] = None,
+        n_keep: int = 1,
+        temperature: float = 1.0,
+        chat_history: Optional[List[Dict[str, str]]] = None,
+        max_tokens: int = 32_768,
+    ) -> None:
+
+        api_key = api_key or os.getenv("GOOGLE_API_KEY")
+
+        if not api_key:
+            raise ValueError("GOOGLE_API_KEY environment variable not set")
+
+        super().__init__(
+            system_setting=system_setting,
+            model=model,
+            api_key=api_key,
+            stream=stream,
+            seed=seed,
+            n_keep=n_keep,
+            temperature=temperature,
+            provider="Google",
+            chat_history=chat_history,
+            max_tokens=max_tokens,
+        )
+
+    def _init_client(self) -> Any:
+        return genai.Client(api_key=self.api_key)
+
+    def _api_call_static(self) -> str:
+        response = self.client.models.generate_content(
+            model=self.model, contents=self.history
+        )
+
+        try:
+            self.input_tokens += response.usage_metadata.prompt_token_count
+            self.output_tokens += response.usage_metadata.candidates_token_count
+        except:
+            pass
+
+        return response.text
+
+    def _api_call_stream(self) -> Generator[str, None, None]:
+        # print("========================================")
+        # pprint(self.history)
+        # print("========================================")
+
+        # return self.client.models.generate_content_stream(
+        #     model=self.model, contents=self.history
+        # )
+
+        for chunk in self.client.models.generate_content_stream(
+            model=self.model, contents=self.history
+        ):
+            if chunk:
+                try:
+                    self.input_tokens += chunk.usage_metadata.prompt_token_count
+                    self.output_tokens += chunk.usage_metadata.candidates_token_count
+                except:
+                    pass
+
+                yield chunk.text
+
+    def _api_format_history(
+        self, history: List[Dict[str, str]]
+    ) -> List[Dict[str, str]]:
+        """Formats the chat history for the Google Generative AI API."""
+        formatted_history = []
+        for item in history:
+            if isinstance(item.get("parts"), list):
+                # Rename role from assistant to model
+                if item.get("role") == "assistant":
+                    item["role"] = "model"
+                formatted_history.append(item)
+            else:
+                formatted_history.append(
+                    {
+                        "role": item["role"],
+                        "parts": [{"text": item["content"]}],
+                    }
+                )
+        # print(formatted_history)
+        return formatted_history
+
+
+def main() -> None:
+    ai = scitex.ai.GenAI(
+        # "gemini-2.0-flash-exp",
+        # "gemini-2.0-flash",
+        # "gemini-2.0-flash-lite-preview-02-05",
+        # "gemini-2.0-pro-exp-02-05",
+        "gemini-2.0-flash-thinking-exp-01-21",
+        stream=True,
+        n_keep=10,
+    )
+    print(ai("hi"))
+    print(ai("My name is Yusuke"))
+    print(ai("do you remember my name?"))
+
+
+if __name__ == "__main__":
+    CONFIG, sys.stdout, sys.stderr, plt, CC = scitex.gen.start(sys, plt, verbose=False)
+    main()
+    scitex.gen.close(CONFIG, verbose=False, notify=False)
+
+
+"""
+python src/scitex/ai/_gen_ai/_Google.py
+python -m src.scitex.ai._gen_ai._Google
+"""
+
+
+# EOF

scitex/ai/genai/google_provider.py

@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-13 20:25:55 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/ai/genai/google_provider.py
+
+"""Google Generative AI (Gemini) provider implementation using the new component-based architecture.
+
+This module provides integration with Google's Gemini models through the google.generativeai API.
+"""
+
+import os
+from typing import Dict, List, Iterator, Optional, Any
+
+try:
+    from google import genai
+except ImportError:
+    raise ImportError(
+        "google-generativeai package is required for Google provider. "
+        "Install with: pip install google-generativeai"
+    )
+
+from .base_provider import BaseProvider, ProviderConfig
+
+
+class GoogleProvider(BaseProvider):
+    """Google Generative AI (Gemini) provider implementation."""
+
+    SUPPORTED_MODELS = [
+        "gemini-1.5-pro-latest",
+        "gemini-1.5-flash-latest",
+        "gemini-2.0-flash-exp",
+        "gemini-2.0-flash",
+        "gemini-2.0-flash-thinking-exp-01-21",
+        "gemini-2.0-pro-exp-02-05",
+        "gemini-2.0-flash-lite-preview-02-05",
+    ]
+
+    DEFAULT_MODEL = "gemini-1.5-pro-latest"
+
+    def __init__(self, config: ProviderConfig):
+        """Initialize Google provider.
+
+        Args:
+            config: Provider configuration
+        """
+        self.config = config
+        self.api_key = config.api_key or os.getenv("GOOGLE_API_KEY")
+
+        if not self.api_key:
+            raise ValueError(
+                "Google API key not provided. Set GOOGLE_API_KEY environment variable or pass api_key."
+            )
+
+        # Initialize Google Generative AI client
+        self.client = genai.Client(api_key=self.api_key)
+
+        # Set default max_tokens if not provided
+        if self.config.max_tokens is None:
+            self.config.max_tokens = 32_768
+
+    def validate_messages(self, messages: List[Dict[str, Any]]) -> None:
+        """Validate message format.
+
+        Args:
+            messages: List of message dictionaries
+
+        Raises:
+            ValueError: If messages are invalid
+        """
+        if not messages:
+            raise ValueError("Messages cannot be empty")
+
+        for msg in messages:
+            if "role" not in msg:
+                raise ValueError(f"Missing role in message: {msg}")
+            if "content" not in msg:
+                raise ValueError(f"Missing content in message: {msg}")
+            if msg["role"] not in ["system", "user", "assistant", "model"]:
+                raise ValueError(f"Invalid role: {msg['role']}")
+
+    def format_messages(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Format messages for Google Generative AI API.
+
+        Google expects a specific format with 'parts' and 'model' role instead of 'assistant'.
+
+        Args:
+            messages: List of message dictionaries
+
+        Returns:
+            Formatted messages for Google API
+        """
+        formatted = []
+
+        # Add system prompt if configured
+        if self.config.system_prompt:
+            formatted.append(
+                {"role": "user", "parts": [{"text": self.config.system_prompt}]}
+            )
+            formatted.append(
+                {
+                    "role": "model",
+                    "parts": [
+                        {"text": "I understand. I will follow these instructions."}
+                    ],
+                }
+            )
+
+        # Format user messages
+        for msg in messages:
+            role = msg["role"]
+            # Convert assistant to model for Google API
+            if role == "assistant":
+                role = "model"
+            elif role == "system":
+                # System messages are handled as user messages in Google API
+                role = "user"
+
+            # Check if already formatted with parts
+            if isinstance(msg.get("parts"), list):
+                formatted.append({"role": role, "parts": msg["parts"]})
+            else:
+                formatted.append({"role": role, "parts": [{"text": msg["content"]}]})
+
+        return formatted
+
+    def complete(self, messages: List[Dict[str, Any]], **kwargs) -> Dict[str, Any]:
+        """Generate a completion.
+
+        Args:
+            messages: List of message dictionaries
+            **kwargs: Additional parameters for the API
+
+        Returns:
+            Completion response dictionary
+        """
+        self.validate_messages(messages)
+        formatted_messages = self.format_messages(messages)
+
+        # Merge config parameters with kwargs
+        model = self.config.model or self.DEFAULT_MODEL
+
+        try:
+            response = self.client.models.generate_content(
+                model=model,
+                contents=formatted_messages,
+                # Google API doesn't support all OpenAI parameters directly
+                # Temperature and other settings would need to be set via generation_config
+            )
+
+            # Extract token usage if available
+            usage = {}
+            if hasattr(response, "usage_metadata"):
+                usage = {
+                    "prompt_tokens": response.usage_metadata.prompt_token_count,
+                    "completion_tokens": response.usage_metadata.candidates_token_count,
+                    "total_tokens": (
+                        response.usage_metadata.prompt_token_count
+                        + response.usage_metadata.candidates_token_count
+                    ),
+                }
+
+            return {
+                "content": response.text,
+                "model": model,
+                "usage": usage,
+                "finish_reason": "stop",
+            }
+        except Exception as e:
+            raise RuntimeError(f"Google Generative AI error: {str(e)}")
+
+    def stream(
+        self, messages: List[Dict[str, Any]], **kwargs
+    ) -> Iterator[Dict[str, Any]]:
+        """Stream a completion.
+
+        Args:
+            messages: List of message dictionaries
+            **kwargs: Additional parameters for the API
+
+        Yields:
+            Chunks of the completion
+        """
+        self.validate_messages(messages)
+        formatted_messages = self.format_messages(messages)
+
+        model = self.config.model or self.DEFAULT_MODEL
+
+        try:
+            stream = self.client.models.generate_content_stream(
+                model=model,
+                contents=formatted_messages,
+            )
+
+            total_prompt_tokens = 0
+            total_completion_tokens = 0
+
+            for chunk in stream:
+                if chunk and hasattr(chunk, "text") and chunk.text:
+                    # Track token usage if available
+                    if hasattr(chunk, "usage_metadata"):
+                        if hasattr(chunk.usage_metadata, "prompt_token_count"):
+                            total_prompt_tokens += (
+                                chunk.usage_metadata.prompt_token_count
+                            )
+                        if hasattr(chunk.usage_metadata, "candidates_token_count"):
+                            total_completion_tokens += (
+                                chunk.usage_metadata.candidates_token_count
+                            )
+
+                    yield {
+                        "content": chunk.text,
+                        "model": model,
+                    }
+
+            # Yield final chunk with usage info if we collected any
+            if total_prompt_tokens > 0 or total_completion_tokens > 0:
+                yield {
+                    "content": "",
+                    "usage": {
+                        "prompt_tokens": total_prompt_tokens,
+                        "completion_tokens": total_completion_tokens,
+                        "total_tokens": total_prompt_tokens + total_completion_tokens,
+                    },
+                    "finish_reason": "stop",
+                }
+
+        except Exception as e:
+            raise RuntimeError(f"Google Generative AI error: {str(e)}")