autonomous-app 0.3.5__py3-none-any.whl → 0.3.24__py3-none-any.whl

autonomous/ai/models/gemini.py

@@ -0,0 +1,309 @@
+import io
+import os
+import random
+import wave
+
+from google import genai
+from google.genai import types
+from pydub import AudioSegment
+
+from autonomous import log
+from autonomous.model.autoattr import ListAttr, StringAttr
+from autonomous.model.automodel import AutoModel
+
+
+class GeminiAIModel(AutoModel):
+    _client = None
+    _text_model = "gemini-3-pro-preview"
+    _summary_model = "gemini-2.5-flash"
+    _image_model = "gemini-3-pro-image-preview"
+    _json_model = "gemini-3-pro-preview"
+    _stt_model = "gemini-3-pro-preview"
+    _tts_model = "gemini-2.5-flash-preview-tts"
+    MAX_FILES = 14
+    VOICES = {
+        "Zephyr": ["female"],
+        "Puck": ["male"],
+        "Charon": ["male"],
+        "Kore": ["female"],
+        "Fenrir": ["non-binary"],
+        "Leda": ["female"],
+        "Orus": ["male"],
+        "Aoede": ["female"],
+        "Callirhoe": ["female"],
+        "Autonoe": ["female"],
+        "Enceladus": ["male"],
+        "Iapetus": ["male"],
+        "Umbriel": ["male"],
+        "Algieba": ["male"],
+        "Despina": ["female"],
+        "Erinome": ["female"],
+        "Algenib": ["male"],
+        "Rasalgethi": ["non-binary"],
+        "Laomedeia": ["female"],
+        "Achernar": ["female"],
+        "Alnilam": ["male"],
+        "Schedar": ["male"],
+        "Gacrux": ["female"],
+        "Pulcherrima": ["non-binary"],
+        "Achird": ["male"],
+        "Zubenelgenubi": ["male"],
+        "Vindemiatrix": ["female"],
+        "Sadachbia": ["male"],
+        "Sadaltager": ["male"],
+        "Sulafar": ["female"],
+    }
+
+    messages = ListAttr(StringAttr(default=[]))
+    name = StringAttr(default="agent")
+    instructions = StringAttr(
+        default="You are highly skilled AI trained to assist with various tasks."
+    )
+    description = StringAttr(
+        default="A helpful AI assistant trained to assist with various tasks."
+    )
+
+    @property
+    def client(self):
+        if not self._client:
+            # log("=== Initializing Gemini AI Client ===", _print=True)
+            self._client = genai.Client(api_key=os.environ.get("GOOGLEAI_KEY"))
+            # log("=== Gemini AI Client Initialized ===", _print=True)
+        return self._client
+
+    def _add_function(self, user_function):
+        # This function is now a bit more advanced to conform to the Tool Use schema
+        tool_schema = {
+            "name": user_function.get("name"),
+            "description": user_function.get("description"),
+            "parameters": user_function.get("parameters"),
+        }
+
+        # Validate that the schema has a name, description, and parameters
+        if not all(
+            [tool_schema["name"], tool_schema["description"], tool_schema["parameters"]]
+        ):
+            raise ValueError(
+                "Tool schema must have a 'name', 'description', and 'parameters' field."
+            )
+
+        return tool_schema
+
+    def _create_wav_header(
+        self, raw_audio_bytes, channels=1, rate=24000, sample_width=2
+    ):
+        """Creates an in-memory WAV file from raw PCM audio bytes."""
+        buffer = io.BytesIO()
+        with wave.open(buffer, "wb") as wav_file:
+            # Set audio parameters
+            wav_file.setnchannels(channels)
+            wav_file.setsampwidth(sample_width)
+            wav_file.setframerate(rate)  # 16,000 Hz sample rate
+
+            # Write the raw audio data
+            wav_file.writeframes(raw_audio_bytes)
+
+        buffer.seek(0)
+        return buffer
+
+    def generate_json(self, message, function, additional_instructions=""):
+        # The API call must use the 'tools' parameter instead of 'response_json_schema'
+        function_definition = self._add_function(function)
+
+        response = self.client.models.generate_content(
+            model=self._json_model,
+            contents=message,
+            config=types.GenerateContentConfig(
+                system_instruction=f"{self.instructions}.{additional_instructions}",
+                tools=[types.Tool(function_declarations=[function_definition])],
+                tool_config={
+                    "function_calling_config": {
+                        "mode": "ANY",  # Force a function call
+                    }
+                },
+            ),
+        )
+
+        # The response is now a ToolCall, not a JSON string
+        try:
+            # log(response.candidates[0].content.parts[0].function_call, _print=True)
+            tool_call = response.candidates[0].content.parts[0].function_call
+            if tool_call and tool_call.name == function["name"]:
+                return tool_call.args
+            else:
+                log(
+                    "==== Model did not return a tool call or returned the wrong one. ===="
+                )
+                log(f"Response: {response.text}", _print=True)
+                return {}
+        except Exception as e:
+            log(f"==== Failed to parse ToolCall response: {e} ====")
+            return {}
+
+    def generate_text(self, message, additional_instructions=""):
+        response = self.client.models.generate_content(
+            model=self._text_model,
+            config=types.GenerateContentConfig(
+                system_instruction=f"{self.instructions}.{additional_instructions}",
+            ),
+            contents=message,
+        )
+
+        # log(results, _print=True)
+        # log("=================== END REPORT ===================", _print=True)
+        return response.text
+
+    def summarize_text(self, text, primer=""):
+        primer = primer or self.instructions
+        response = self.client.models.generate_content(
+            model=self._summary_model,
+            config=types.GenerateContentConfig(
+                system_instruction=f"{primer}",
+            ),
+            contents=text,
+        )
+        log(response)
+        try:
+            result = response.candidates[0].content.parts[0].text
+        except Exception as e:
+            log(f"{type(e)}:{e}\n\n Unable to generate content ====")
+            return None
+
+        return result
+
+    def generate_audio_text(
+        self, audio_file, prompt="Transcribe this audio clip", **kwargs
+    ):
+        myfile = self.client.files.upload(
+            file=io.BytesIO(audio_file),
+            config={
+                "mime_type": "audio/mp3",
+                "display_name": kwargs.get("display_name", "audio.mp3"),
+            },
+        )
+
+        response = self.client.models.generate_content(
+            model=self._stt_model,
+            contents=[
+                prompt,
+                myfile,
+            ],
+        )
+        return response.text
+
+    def list_voices(self, filters=[]):
+        if not filters:
+            return list(self.VOICES.keys())
+        voices = []
+        for voice, attribs in self.VOICES.items():
+            if any(f.lower() in attribs for f in filters):
+                voices.append(voice)
+        return voices
+
+    def generate_audio(self, prompt, voice=None, **kwargs):
+        voice = voice or random.choice(self.list_voices())
+        try:
+            response = self.client.models.generate_content(
+                model=self._tts_model,
+                contents=prompt,
+                config=types.GenerateContentConfig(
+                    response_modalities=["AUDIO"],
+                    speech_config=types.SpeechConfig(
+                        voice_config=types.VoiceConfig(
+                            prebuilt_voice_config=types.PrebuiltVoiceConfig(
+                                voice_name=voice,
+                            )
+                        )
+                    ),
+                ),
+            )
+            blob = response.candidates[0].content.parts[0].inline_data
+
+            # Create a WAV file in memory from the raw audio bytes
+            wav_buffer = self._create_wav_header(blob.data)
+
+            # 2. Load the WAV audio using pydub, which will now correctly read the header
+            audio_segment = AudioSegment.from_file(wav_buffer, format="wav")
+
+            # 3. Create a new in-memory buffer for the MP3 output
+            mp3_buffer = io.BytesIO()
+
+            # 4. Export the audio segment directly to the in-memory buffer
+            audio_segment.export(mp3_buffer, format="mp3")
+
+            # 5. Return the bytes from the buffer, not the filename
+            return mp3_buffer.getvalue()
+
+        except Exception as e:
+            log(
+                f"==== Error: Unable to generate audio ====\n{type(e)}:{e}", _print=True
+            )
+            # You can return a default empty byte string or re-raise the exception
+            raise e
+
+    def generate_image(self, prompt, **kwargs):
+        image = None
+        contents = [prompt]
+
+        if kwargs.get("files"):
+            counter = 0
+            for fn, f in kwargs.get("files").items():
+                media = io.BytesIO(f)
+                myfile = self.client.files.upload(
+                    file=media,
+                    config={"mime_type": "image/webp", "display_name": fn},
+                )
+                contents += [myfile]
+                counter += 1
+                if counter >= self.MAX_FILES:
+                    break
+
+        try:
+            # log(self._image_model, contents, _print=True)
+            response = self.client.models.generate_content(
+                model=self._image_model,
+                contents=contents,
+                config=types.GenerateContentConfig(
+                    safety_settings=[
+                        types.SafetySetting(
+                            category=types.HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+                            threshold=types.HarmBlockThreshold.BLOCK_NONE,
+                        ),
+                        types.SafetySetting(
+                            category=types.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+                            threshold=types.HarmBlockThreshold.BLOCK_NONE,
+                        ),
+                        types.SafetySetting(
+                            category=types.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+                            threshold=types.HarmBlockThreshold.BLOCK_NONE,
+                        ),
+                        types.SafetySetting(
+                            category=types.HarmCategory.HARM_CATEGORY_HARASSMENT,
+                            threshold=types.HarmBlockThreshold.BLOCK_NONE,
+                        ),
+                        types.SafetySetting(
+                            category=types.HarmCategory.HARM_CATEGORY_CIVIC_INTEGRITY,
+                            threshold=types.HarmBlockThreshold.BLOCK_NONE,
+                        ),
+                    ],
+                    image_config=types.ImageConfig(
+                        aspect_ratio=kwargs.get("aspect_ratio", "3:4"),
+                        image_size=kwargs.get("image_size", "2K"),
+                    ),
+                ),
+            )
+            # log(response, _print=True)
+            # log(response.candidates[0], _print=True)
+            image_parts = [
+                part.inline_data.data
+                for part in response.candidates[0].content.parts
+                if part.inline_data
+            ]
+            image = image_parts[0]
+        except Exception as e:
+            log(
+                f"==== Error: Unable to create image ====\n\n{e}",
+                _print=True,
+            )
+            raise e
+        return image

autonomous/ai/models/local.py

@@ -0,0 +1,99 @@
+import io
+import json
+import os
+import random
+import time
+from base64 import b64decode
+
+import openai
+from ollama import ChatResponse, chat
+
+from autonomous import log
+from autonomous.model.autoattr import DictAttr, ListAttr, StringAttr
+from autonomous.model.automodel import AutoModel
+
+
+class LocalAIModel(AutoModel):
+    _client = None
+    instructions = StringAttr(
+        default="You are highly skilled AI trained to assist with various tasks."
+    )
+    description = StringAttr(
+        default="A helpful AI assistant trained to assist with various tasks."
+    )
+
+    @property
+    def client(self):
+        if not self._client:
+            self._client = "deepseek-r1"  # OpenAI(api_key=os.environ.get("OPENAI_KEY"))
+        return self._client
+
+    def clear_agent(self):
+        pass
+
+    def clear_agents(self):
+        pass
+
+        # def _get_agent_id(self):
+        #     pass
+
+        # def _add_function(self, user_function):
+        pass
+
+    def _format_messages(self, messages):
+        pass
+
+    def clear_files(self, file_id=None):
+        pass
+
+    def attach_file(self, file_contents, filename="dbdata.json"):
+        pass
+
+    def generate_json(self, messages, function, additional_instructions=""):
+        message = messages + additional_instructions
+        message += f"""
+IMPORTANT: Respond in JSON FORMAT using the SCHEMA below. DO NOT add any text to the response outside of the supplied JSON schema:
+{function}
+"""
+        response: ChatResponse = chat(
+            model=self.client,
+            messages=[
+                {
+                    "role": "user",
+                    "content": message,
+                },
+            ],
+        )
+        return response.message.content
+
+    def generate_text(self, messages, additional_instructions=""):
+        message = messages + additional_instructions
+        response: ChatResponse = chat(
+            model=self.client,
+            messages=[
+                {
+                    "role": "user",
+                    "content": message,
+                },
+            ],
+        )
+        return response.message.content
+
+    def generate_audio(self, prompt, **kwargs):
+        raise NotImplementedError
+
+    def generate_image(self, prompt, **kwargs):
+        raise NotImplementedError
+
+    def summarize_text(self, text, primer=""):
+        response: ChatResponse = chat(
+            model=self.client,
+            messages=[
+                {
+                    "role": "system",
+                    "content": f"You are a highly skilled AI trained in language comprehension and summarization.{primer}",
+                },
+                {"role": "user", "content": text},
+            ],
+        )
+        return response.message.content

autonomous/ai/models/openai.py

@@ -1,3 +1,4 @@
+import io
 import json
 import os
 import random
@@ -15,7 +16,7 @@ from autonomous.model.automodel import AutoModel
 
 class OpenAIModel(AutoModel):
     _client = None
-    _text_model = "gpt-4o-mini"
+    _text_model = "o3-mini"
     _image_model = "dall-e-3"
     _json_model = "gpt-4o"
     agent_id = StringAttr()
@@ -36,6 +37,15 @@ class OpenAIModel(AutoModel):
             self._client = OpenAI(api_key=os.environ.get("OPENAI_KEY"))
         return self._client
 
+    def delete(self):
+        self.clear_files()
+        if self.agent_id:
+            try:
+                self.client.beta.assistants.delete(self.agent_id)
+            except openai_NotFoundError:
+                log(f"==== Agent with ID: {self.agent_id} not found ====")
+        return super().delete()
+
     def clear_agent(self):
         if self.agent_id:
             self.client.beta.assistants.delete(self.agent_id)
@@ -44,18 +54,22 @@ class OpenAIModel(AutoModel):
 
     def clear_agents(self):
         assistants = self.client.beta.assistants.list().data
-        log(assistants)
-        for assistant in assistants:
-            log(f"==== Deleting Agent with ID: {assistant.id} ====")
-            try:
-                self.client.beta.assistants.delete(assistant.id)
-            except openai_NotFoundError:
-                log(f"==== Agent with ID: {assistant.id} not found ====")
-        self.agent_id = ""
-        self.save()
+        if assistants:
+            log(assistants)
+            for assistant in assistants:
+                log(f"==== Deleting Agent with ID: {assistant.id} ====")
+                try:
+                    self.client.beta.assistants.delete(assistant.id)
+                except openai_NotFoundError:
+                    log(f"==== Agent with ID: {assistant.id} not found ====")
+            self.agent_id = ""
+            self.save()
 
     def _get_agent_id(self):
-        if not self.agent_id or not self.client.beta.assistants.retrieve(self.agent_id):
+        try:
+            self.client.beta.assistants.retrieve(self.agent_id)
+        except (ValueError, openai.NotFoundError) as e:
+            log(f"{e} -- no agent found, creating a new one")
             agent = self.client.beta.assistants.create(
                 instructions=self.instructions,
                 description=self.description,
@@ -67,18 +81,15 @@ class OpenAIModel(AutoModel):
             self.save()
         return self.agent_id
 
-    def clear_files(self, file_id=None, all=False):
+    def clear_files(self, file_id=None):
         if not file_id:
-            store_files = self.client.files.list().data
-
-            for vs in self.client.beta.vector_stores.list().data:
+            for vs in self.client.vector_stores.list().data:
                 try:
-                    self.client.beta.vector_stores.delete(vs.id)
+                    self.client.vector_stores.delete(vs.id)
                 except openai_NotFoundError:
                     log(f"==== Vector Store {vs.id} not found ====")
-            if all:
-                for sf in store_files:
-                    self.client.files.delete(file_id=sf.id)
+            for sf in self.client.files.list().data:
+                self.client.files.delete(file_id=sf.id)
         else:
             self.client.files.delete(file_id=file_id)
         self.tools.pop("file_search", None)
@@ -89,19 +100,27 @@ class OpenAIModel(AutoModel):
         # Upload the user provided file to OpenAI
         self.tools["file_search"] = {"type": "file_search"}
         # Create a vector store
-        if vs := self.client.beta.vector_stores.list().data:
-            self.vector_store = vs[0].id
-        else:
-            self.vector_store = self.client.beta.vector_stores.create(
-                name="Data Reference",
+        try:
+            if vs := self.client.vector_stores.list().data:
+                self.vector_store = self.client.vector_stores.retrieve(
+                    vector_store_id=vs[0].id
+                ).id
+            else:
+                for sf in self.client.files.list().data:
+                    self.client.files.delete(file_id=sf.id)
+                raise FileNotFoundError("No vector store found")
+        except FileNotFoundError:
+            self.vector_store = self.client.vector_stores.create(
+                name="World Reference",
                 expires_after={"anchor": "last_active_at", "days": 14},
             ).id
-
+        log(f"==== Vector Store ID: {self.vector_store}====", _print=True)
+        # Attach File
         file_obj = self.client.files.create(
             file=(filename, file_contents), purpose="assistants"
         )
-
-        self.client.beta.vector_stores.files.create(
+        log(f"==== FileStore ID: {file_obj.id}====", _print=True)
+        self.client.vector_stores.files.create(
             vector_store_id=self.vector_store,
             file_id=file_obj.id,
         )
@@ -183,14 +202,14 @@ IMPORTANT: Always use the function 'response' tool to respond to the user with o
                 ]:
                     running_job = False
 
-            except openai.error.BadRequestError as e:
+            except openai.BadRequestError as err:
                 # Handle specific bad request errors
-                error_message = e.json_body.get("error", {}).get("message", "")
-                if "already has an active run" in error_message:
+                log(f"==== Error: {err} ====", _print=True)
+                if "already has an active run" in str(err):
                     log("Previous run is still active. Waiting...", _print=True)
                     time.sleep(2)  # wait before retrying or checking run status
                 else:
-                    raise e
+                    raise err
 
         # while run.status in ["queued", "in_progress"]:
         #     run = self.client.beta.threads.runs.retrieve(
@@ -202,7 +221,7 @@ IMPORTANT: Always use the function 'response' tool to respond to the user with o
             log(f"==== !!! ERROR !!!: {run.last_error} ====", _print=True)
             return None
         log("=================== RUN COMPLETED ===================", _print=True)
-        log(run.status, _print=True)
+        # log(run.status, _print=True)
         if run.status == "completed":
             response = self.client.beta.threads.messages.list(thread_id=thread.id)
             results = response.data[0].content[0].text.value
@@ -218,14 +237,15 @@ IMPORTANT: Always use the function 'response' tool to respond to the user with o
         try:
             results = json.loads(results, strict=False)
         except Exception:
-            print(f"==== Invalid JSON:\n{results}", _print=True)
+            log(f"==== Invalid JSON:\n{results}", _print=True)
             return {}
         else:
-            log(f"==== Results: {results}", _print=True)
-            log("=================== END REPORT ===================", _print=True)
+            # log(f"==== Results: {results}", _print=True)
+            # log("=================== END REPORT ===================", _print=True)
             return results
 
     def generate_text(self, messages, additional_instructions=""):
+        self._get_agent_id()
         formatted_messages = self._format_messages(messages)
         thread = self.client.beta.threads.create(messages=formatted_messages)
 
@@ -247,8 +267,8 @@ IMPORTANT: Always use the function 'response' tool to respond to the user with o
         if run.status in ["failed", "expired", "canceled"]:
             log(f"==== Error: {run.last_error} ====", _print=True)
             return None
-        log("=================== RUN COMPLETED ===================", _print=True)
-        log(run.status, _print=True)
+        # log("=================== RUN COMPLETED ===================", _print=True)
+        # log(run.status, _print=True)
         if run.status == "completed":
             response = self.client.beta.threads.messages.list(thread_id=thread.id)
             results = response.data[0].content[0].text.value
@@ -256,21 +276,39 @@ IMPORTANT: Always use the function 'response' tool to respond to the user with o
             log(f"====Status: {run.status} Error: {run.last_error} ====", _print=True)
             return None
 
-        log(results, _print=True)
-        log("=================== END REPORT ===================", _print=True)
+        # log(results, _print=True)
+        # log("=================== END REPORT ===================", _print=True)
         return results
 
-    def generate_audio(self, prompt, file_path, **kwargs):
+    def generate_audio(self, prompt, **kwargs):
         voice = kwargs.get("voice") or random.choice(
-            ["alloy", "echo", "fable", "onyx", "nova", "shimmer"]
+            [
+                "alloy",
+                "ash",
+                "ballad",
+                "coral",
+                "echo",
+                "fable",
+                "onyx",
+                "nova",
+                "sage",
+                "shimmer",
+            ]
         )
         response = self.client.audio.speech.create(
             model="tts-1",
             voice=voice,
             input=prompt,
         )
+        # log(response, _print=True)
+        return response.read()
 
-        return response.stream_to_file(file_path)
+    def generate_audio_text(self, audio_file, **kwargs):
+        response = self.client.audio.transcriptions.create(
+            model="gpt-4o-transcribe", file=audio_file, language="en", **kwargs
+        )
+        log(response, _print=True)
+        return response.text
 
     def generate_image(self, prompt, **kwargs):
         image = None
@@ -283,7 +321,8 @@ IMPORTANT: Always use the function 'response' tool to respond to the user with o
             )
             image_dict = response.data[0]
         except Exception as e:
-            print(f"==== Error: Unable to create image ====\n\n{e}")
+            log(f"==== Error: Unable to create image ====\n\n{e}", _print=True)
+            raise e
         else:
             image = b64decode(image_dict.b64_json)
         return image

autonomous/ai/textagent.py

@@ -1,12 +1,11 @@
 from autonomous import log
 from autonomous.model.autoattr import ReferenceAttr, StringAttr
 from autonomous.model.automodel import AutoModel
-
+from autonomous.ai.baseagent import BaseAgent
 from .models.openai import OpenAIModel
 
 
-class TextAgent(AutoModel):
-    client = ReferenceAttr(choices=[OpenAIModel])
+class TextAgent(BaseAgent):
     name = StringAttr(default="textagent")
     instructions = StringAttr(
         default="You are highly skilled AI trained to assist with generating text according to the given requirements."
@@ -15,19 +14,6 @@ class TextAgent(AutoModel):
         default="A helpful AI assistant trained to assist with generating text according to the given requirements."
     )
 
-    _ai_model = OpenAIModel
-
-    def get_client(self):
-        if self.client is None:
-            self.client = self._ai_model(
-                name=self.name,
-                instructions=self.instructions,
-                description=self.description,
-            )
-            self.client.save()
-            self.save()
-        return self.client
-
     def summarize_text(self, text, primer=""):
         return self.get_client().summarize_text(text, primer)