zrb 1.0.0b2__tar.gz → 1.0.0b4__tar.gz

{zrb-1.0.0b2 → zrb-1.0.0b4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: zrb
-Version: 1.0.0b2
+Version: 1.0.0b4
 Summary: Your Automation Powerhouse
 Home-page: https://github.com/state-alchemists/zrb
 License: AGPL-3.0-or-later
@@ -26,6 +26,7 @@ Requires-Dist: pdfplumber (>=0.11.4,<0.12.0) ; extra == "rag"
 Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
 Requires-Dist: python-jose[cryptography] (>=3.3.0,<4.0.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)
+Requires-Dist: ulid-py (>=1.1.0,<2.0.0)
 Project-URL: Documentation, https://github.com/state-alchemists/zrb
 Project-URL: Repository, https://github.com/state-alchemists/zrb
 Description-Content-Type: text/markdown

{zrb-1.0.0b2 → zrb-1.0.0b4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "zrb"
-version = "1.0.0b2"
+version = "1.0.0b4"
 description = "Your Automation Powerhouse"
 authors = ["Go Frendi Gunawan <gofrendiasgard@gmail.com>"]
 license = "AGPL-3.0-or-later"
@@ -50,6 +50,7 @@ pdfplumber = {version = "^0.11.4", optional = true}
 beautifulsoup4 = "^4.12.3"
 fastapi = {extras = ["standard"], version = "^0.115.6"}
 python-jose = {extras = ["cryptography"], version = "^3.3.0"}
+ulid-py = "^1.1.0"
 
 [tool.poetry.extras]
 # poetry install -E rag
@@ -63,7 +64,6 @@ pytest-cov = "~6.0.0"
 pytest-asyncio = "~0.24.0"
 sqlmodel = "^0.0.22" # FastApp dependencies
 tomlkit = "^0.13.2"
-ulid = "^1.1" # FastApp dependencies
 
 [tool.pytest.ini_options]
 asyncio_default_fixture_loop_scope = "function"

{zrb-1.0.0b2 → zrb-1.0.0b4}/src/zrb/__main__.py

@@ -16,5 +16,8 @@ def serve_cli():
         cli.run(sys.argv[1:])
     except KeyboardInterrupt:
         print(stylize_warning("\nStopped"), file=sys.stderr)
+    except RuntimeError as e:
+        if f"{e}".lower() != "event loop is closed":
+            raise e
     except NodeNotFoundError as e:
         print(stylize_error(f"{e}"), file=sys.stderr)

zrb-1.0.0b4/src/zrb/builtin/llm/llm_chat.py

@@ -0,0 +1,127 @@
+import json
+import os
+from typing import Any
+
+from zrb.builtin.group import llm_group
+from zrb.builtin.llm.tool.api import get_current_location, get_current_weather
+from zrb.builtin.llm.tool.cli import run_shell_command
+from zrb.builtin.llm.tool.web import open_web_route, query_internet
+from zrb.config import (
+    LLM_ALLOW_ACCESS_INTERNET,
+    LLM_ALLOW_ACCESS_SHELL,
+    LLM_HISTORY_DIR,
+    LLM_MODEL,
+    LLM_SYSTEM_PROMPT,
+)
+from zrb.context.any_shared_context import AnySharedContext
+from zrb.input.bool_input import BoolInput
+from zrb.input.str_input import StrInput
+from zrb.input.text_input import TextInput
+from zrb.task.llm_task import LLMTask
+from zrb.util.file import read_file, write_file
+from zrb.util.string.conversion import to_pascal_case
+
+
+class PreviousSessionInput(StrInput):
+
+    def to_html(self, ctx: AnySharedContext) -> str:
+        name = self.name
+        description = self.description
+        default = self.get_default_str(ctx)
+        script = read_file(
+            file_path=os.path.join(os.path.dirname(__file__), "previous-session.js"),
+            replace_map={
+                "CURRENT_INPUT_NAME": name,
+                "CurrentPascalInputName": to_pascal_case(name),
+            },
+        )
+        return "\n".join(
+            [
+                f'<input name="{name}" placeholder="{description}" value="{default}" />',
+                f"<script>{script}</script>",
+            ]
+        )
+
+
+def _read_chat_conversation(ctx: AnySharedContext) -> list[dict[str, Any]]:
+    if ctx.input.start_new:
+        return []
+    previous_session_name = ctx.input.previous_session
+    if previous_session_name == "" or previous_session_name is None:
+        last_session_file_path = os.path.join(LLM_HISTORY_DIR, "last-session")
+        if os.path.isfile(last_session_file_path):
+            previous_session_name = read_file(last_session_file_path).strip()
+    conversation_file_path = os.path.join(
+        LLM_HISTORY_DIR, f"{previous_session_name}.json"
+    )
+    if not os.path.isfile(conversation_file_path):
+        return []
+    return json.loads(read_file(conversation_file_path))
+
+
+def _write_chat_conversation(
+    ctx: AnySharedContext, conversations: list[dict[str, Any]]
+):
+    os.makedirs(LLM_HISTORY_DIR, exist_ok=True)
+    current_session_name = ctx.session.name
+    conversation_file_path = os.path.join(
+        LLM_HISTORY_DIR, f"{current_session_name}.json"
+    )
+    write_file(conversation_file_path, json.dumps(conversations, indent=2))
+    last_session_file_path = os.path.join(LLM_HISTORY_DIR, "last-session")
+    write_file(last_session_file_path, current_session_name)
+
+
+llm_chat: LLMTask = llm_group.add_task(
+    LLMTask(
+        name="llm-chat",
+        input=[
+            StrInput(
+                "model",
+                description="LLM Model",
+                prompt="LLM Model",
+                default_str=LLM_MODEL,
+                allow_positional_parsing=False,
+            ),
+            TextInput(
+                "system-prompt",
+                description="System prompt",
+                prompt="System prompt",
+                default_str=LLM_SYSTEM_PROMPT,
+                allow_positional_parsing=False,
+            ),
+            BoolInput(
+                "start-new",
+                description="Start new conversation session",
+                prompt="Forget everything and start new conversation session",
+                default_str="false",
+                allow_positional_parsing=False,
+            ),
+            TextInput("message", description="User message", prompt="Your message"),
+            PreviousSessionInput(
+                "previous-session",
+                description="Previous conversation session",
+                prompt="Previous conversation session (can be empty)",
+                allow_positional_parsing=False,
+                allow_empty=True,
+            ),
+        ],
+        conversation_history_reader=_read_chat_conversation,
+        conversation_history_writer=_write_chat_conversation,
+        description="Chat with LLM",
+        model="{ctx.input.model}",
+        system_prompt="{ctx.input['system-prompt']}",
+        message="{ctx.input.message}",
+        retries=0,
+    ),
+    alias="chat",
+)
+
+if LLM_ALLOW_ACCESS_SHELL:
+    llm_chat.add_tool(run_shell_command)
+
+if LLM_ALLOW_ACCESS_INTERNET:
+    llm_chat.add_tool(open_web_route)
+    llm_chat.add_tool(query_internet)
+    llm_chat.add_tool(get_current_location)
+    llm_chat.add_tool(get_current_weather)

zrb-1.0.0b4/src/zrb/builtin/llm/previous-session.js

@@ -0,0 +1,13 @@
+let hasUpdateCurrentPascalInputName = false;
+document.getElementById("submit-task-form").addEventListener("change", async function(event) {
+    const currentInput = event.target;
+    if (hasUpdateCurrentPascalInputName || currentInput.name === "CURRENT_INPUT_NAME") {
+        return
+    }
+    const previousSessionInput = submitTaskForm.querySelector('[name="CURRENT_INPUT_NAME"]');
+    if (previousSessionInput) {
+        const currentSessionName = cfg.SESSION_NAME
+        previousSessionInput.value = currentSessionName;
+    }
+    hasUpdateCurrentPascalInputName = true;
+});

zrb-1.0.0b4/src/zrb/builtin/llm/tool/api.py

@@ -0,0 +1,29 @@
+import json
+from typing import Annotated, Literal
+
+import requests
+
+
+def get_current_location() -> (
+    Annotated[str, "JSON string representing latitude and longitude"]
+):  # noqa
+    """Get the user's current location."""
+    return json.dumps(requests.get("http://ip-api.com/json?fields=lat,lon").json())
+
+
+def get_current_weather(
+    latitude: float,
+    longitude: float,
+    temperature_unit: Literal["celsius", "fahrenheit"],
+) -> str:
+    """Get the current weather in a given location."""
+    resp = requests.get(
+        "https://api.open-meteo.com/v1/forecast",
+        params={
+            "latitude": latitude,
+            "longitude": longitude,
+            "temperature_unit": temperature_unit,
+            "current_weather": True,
+        },
+    )
+    return json.dumps(resp.json())

{zrb-1.0.0b2 → zrb-1.0.0b4}/src/zrb/builtin/llm/tool/cli.py

@@ -2,7 +2,7 @@ import subprocess
 
 
 def run_shell_command(command: str) -> str:
-    """Running a shell command"""
+    """Running an actual shell command on user's computer."""
     output = subprocess.check_output(
         command, shell=True, stderr=subprocess.STDOUT, text=True
     )

zrb-1.0.0b4/src/zrb/builtin/llm/tool/rag.py

@@ -0,0 +1,152 @@
+import hashlib
+import json
+import os
+import sys
+
+import litellm
+import ulid
+
+from zrb.config import (
+    RAG_CHUNK_SIZE,
+    RAG_EMBEDDING_MODEL,
+    RAG_MAX_RESULT_COUNT,
+    RAG_OVERLAP,
+)
+from zrb.util.cli.style import stylize_error, stylize_faint
+from zrb.util.file import read_file
+
+
+def create_rag_from_directory(
+    tool_name: str,
+    tool_description: str,
+    document_dir_path: str = "./documents",
+    model: str = RAG_EMBEDDING_MODEL,
+    vector_db_path: str = "./chroma",
+    vector_db_collection: str = "documents",
+    chunk_size: int = RAG_CHUNK_SIZE,
+    overlap: int = RAG_OVERLAP,
+    max_result_count: int = RAG_MAX_RESULT_COUNT,
+):
+    async def retrieve(query: str) -> str:
+        from chromadb import PersistentClient
+        from chromadb.config import Settings
+
+        client = PersistentClient(
+            path=vector_db_path, settings=Settings(allow_reset=True)
+        )
+        collection = client.get_or_create_collection(vector_db_collection)
+
+        # Track file changes using a hash-based approach
+        hash_file_path = os.path.join(vector_db_path, "file_hashes.json")
+        previous_hashes = _load_hashes(hash_file_path)
+        current_hashes = {}
+
+        updated_files = []
+
+        for root, _, files in os.walk(document_dir_path):
+            for file in files:
+                file_path = os.path.join(root, file)
+                file_hash = _compute_file_hash(file_path)
+                relative_path = os.path.relpath(file_path, document_dir_path)
+                current_hashes[relative_path] = file_hash
+
+                if previous_hashes.get(relative_path) != file_hash:
+                    updated_files.append(file_path)
+
+        if updated_files:
+            print(
+                stylize_faint(f"Updating {len(updated_files)} changed files"),
+                file=sys.stderr,
+            )
+
+            for file_path in updated_files:
+                try:
+                    relative_path = os.path.relpath(file_path, document_dir_path)
+                    collection.delete(where={"file_path": relative_path})
+                    content = _read_file_content(file_path)
+                    file_id = ulid.new().str
+                    for i in range(0, len(content), chunk_size - overlap):
+                        chunk = content[i : i + chunk_size]
+                        if chunk:
+                            chunk_id = ulid.new().str
+                            print(
+                                stylize_faint(
+                                    f"Vectorizing {relative_path} chunk {chunk_id}"
+                                ),
+                                file=sys.stderr,
+                            )
+                            response = await litellm.aembedding(
+                                model=model, input=[chunk]
+                            )
+                            vector = response["data"][0]["embedding"]
+                            collection.upsert(
+                                ids=[chunk_id],
+                                embeddings=[vector],
+                                documents=[chunk],
+                                metadatas={
+                                    "file_path": relative_path,
+                                    "file_id": file_id,
+                                },
+                            )
+                except Exception as e:
+                    print(
+                        stylize_error(f"Error processing {file_path}: {e}"),
+                        file=sys.stderr,
+                    )
+
+            _save_hashes(hash_file_path, current_hashes)
+        else:
+            print(
+                stylize_faint("No changes detected. Skipping database update."),
+                file=sys.stderr,
+            )
+
+        print(stylize_faint("Vectorizing query"), file=sys.stderr)
+        query_response = await litellm.aembedding(model=model, input=[query])
+        query_vector = query_response["data"][0]["embedding"]
+
+        print(stylize_faint("Searching documents"), file=sys.stderr)
+        results = collection.query(
+            query_embeddings=query_vector,
+            n_results=max_result_count,
+        )
+        return json.dumps(results)
+
+    retrieve.__name__ = tool_name
+    retrieve.__doc__ = tool_description
+    return retrieve
+
+
+def _compute_file_hash(file_path: str) -> str:
+    hash_md5 = hashlib.md5()
+    with open(file_path, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_md5.update(chunk)
+    return hash_md5.hexdigest()
+
+
+def _read_file_content(file_path: str) -> str:
+    if file_path.lower().endswith(".pdf"):
+        return _read_pdf(file_path)
+    return read_file(file_path)
+
+
+def _read_pdf(file_path: str) -> str:
+    import pdfplumber
+
+    with pdfplumber.open(file_path) as pdf:
+        return "\n".join(
+            page.extract_text() for page in pdf.pages if page.extract_text()
+        )
+
+
+def _load_hashes(file_path: str) -> dict:
+    if os.path.exists(file_path):
+        with open(file_path, "r") as f:
+            return json.load(f)
+    return {}
+
+
+def _save_hashes(file_path: str, hashes: dict):
+    with open(file_path, "w") as f:
+        json.dump(hashes, f)

{zrb-1.0.0b2 → zrb-1.0.0b4}/src/zrb/builtin/project/add/fastapp/fastapp_template/my_app_name/_zrb/entity/template/client_method.py

@@ -18,17 +18,17 @@ async def get_my_entities(
 
 
 @abstractmethod
-async def create_my_entity(self, data: MyEntityCreateWithAudit) -> MyEntityResponse:
-    """Create a new my entities"""
-
-
-@abstractmethod
-async def create_my_entity(
+async def create_my_entity_bulk(
     self, data: list[MyEntityCreateWithAudit]
 ) -> list[MyEntityResponse]:
     """Create new my entities"""
 
 
+@abstractmethod
+async def create_my_entity(self, data: MyEntityCreateWithAudit) -> MyEntityResponse:
+    """Create a new my entities"""
+
+
 @abstractmethod
 async def update_my_entity_bulk(
     self, my_entity_ids: list[str], data: MyEntityUpdateWithAudit

{zrb-1.0.0b2 → zrb-1.0.0b4}/src/zrb/builtin/project/add/fastapp/fastapp_template/my_app_name/_zrb/entity/template/gateway_subroute.py

@@ -51,7 +51,9 @@ async def update_my_entity_bulk(my_entity_ids: list[str], data: MyEntityUpdate):
     response_model=MyEntityResponse,
 )
 async def update_my_entity(my_entity_id: str, data: MyEntityUpdate):
-    return await my_module_client.update_my_entity(data.with_audit(updated_by="system"))
+    return await my_module_client.update_my_entity(
+        my_entity_id, data.with_audit(updated_by="system")
+    )
 
 
 @app.delete(

{zrb-1.0.0b2 → zrb-1.0.0b4}/src/zrb/builtin/project/add/fastapp/fastapp_template/my_app_name/common/base_db_repository.py

@@ -4,10 +4,15 @@ from typing import Any, Callable, Generic, Type, TypeVar
 
 import ulid
 from my_app_name.common.error import InvalidValueError, NotFoundError
-from my_app_name.common.parser_factory import parse_filter_param, parse_sort_param
+from my_app_name.common.parser_factory import (
+    parse_filter_param as default_parse_filter_param,
+)
+from my_app_name.common.parser_factory import (
+    parse_sort_param as default_parse_sort_param,
+)
 from sqlalchemy import Engine, delete, func, insert, select, update
 from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession
-from sqlalchemy.sql import Select
+from sqlalchemy.sql import ClauseElement, ColumnElement, Select
 from sqlmodel import Session, SQLModel
 
 DBModel = TypeVar("DBModel", bound=SQLModel)
@@ -24,16 +29,48 @@ class BaseDBRepository(Generic[DBModel, ResponseModel, CreateModel, UpdateModel]
     entity_name: str = "entity"
     column_preprocessors: dict[str, Callable[[Any], Any]] = {}
 
-    def __init__(self, engine: Engine | AsyncEngine):
+    def __init__(
+        self,
+        engine: Engine | AsyncEngine,
+        filter_param_parser: (
+            Callable[[SQLModel, str], list[ClauseElement]] | None
+        ) = None,
+        sort_param_parser: Callable[[SQLModel, str], list[ColumnElement]] | None = None,
+    ):
         self.engine = engine
-        self.is_async = isinstance(engine, AsyncEngine)
+        self._is_async = isinstance(engine, AsyncEngine)
+        self._parse_filter_param = (
+            filter_param_parser if filter_param_parser else default_parse_filter_param
+        )
+        self._parse_sort_param = (
+            sort_param_parser if sort_param_parser else default_parse_sort_param
+        )
+
+    @property
+    def is_async(self) -> bool:
+        return self._is_async
 
     def _select(self) -> Select:
+        """
+        This method is used to contruct select statement for get, get_by_id and get_by_ids.
+        To parse the result of the statement, make sure you override _rows_to_response as well.
+        """
         return select(self.db_model)
 
-    def _rows_to_responses(self, rows: list[tuple[Any]]) -> list[ResponseModel]:
+    def _rows_to_responses(self, rows: list[tuple[Any, ...]]) -> list[ResponseModel]:
+        """
+        This method is used to parse the result of select statement generated by _select.
+        """
         return [self.response_model.model_validate(row[0]) for row in rows]
 
+    async def _select_to_response(
+        self, query_modifier: Callable[[Select], Any]
+    ) -> list[ResponseModel]:
+        statement = query_modifier(self._select())
+        async with self._session_scope() as session:
+            result = await self._execute_statement(session, statement)
+            return self._rows_to_responses(result.all())
+
     def _ensure_one(self, responses: list[ResponseModel]) -> ResponseModel:
         if not responses:
             raise NotFoundError(f"{self.entity_name} not found")
@@ -41,6 +78,22 @@ class BaseDBRepository(Generic[DBModel, ResponseModel, CreateModel, UpdateModel]
             raise InvalidValueError(f"Duplicate {self.entity_name}")
         return responses[0]
 
+    def _model_to_data_dict(
+        self, data: SQLModel, **additional_data: Any
+    ) -> dict[str, Any]:
+        """
+        This method transform SQLModel into dictionary for insert/update operation.
+        """
+        data_dict = data.model_dump(exclude_unset=True)
+        data_dict.update(additional_data)
+        for key, preprocessor in self.column_preprocessors.items():
+            if key not in data_dict:
+                continue
+            if not hasattr(self.db_model, key):
+                raise InvalidValueError(f"Invalid {self.entity_name} property: {key}")
+            data_dict[key] = preprocessor(data_dict[key])
+        return data_dict
+
     @asynccontextmanager
     async def _session_scope(self):
         if self.is_async:
@@ -65,25 +118,18 @@ class BaseDBRepository(Generic[DBModel, ResponseModel, CreateModel, UpdateModel]
             return session.execute(statement)
 
     async def get_by_id(self, id: str) -> ResponseModel:
-        statement = self._select().where(self.db_model.id == id)
-        async with self._session_scope() as session:
-            result = await self._execute_statement(session, statement)
-            responses = self._rows_to_responses(result.all())
-            return self._ensure_one(responses)
+        rows = await self._select_to_response(lambda q: q.where(self.db_model.id == id))
+        return self._ensure_one(rows)
 
     async def get_by_ids(self, id_list: list[str]) -> list[ResponseModel]:
-        statement = self._select().where(self.db_model.id.in_(id_list))
-        async with self._session_scope() as session:
-            result = await self._execute_statement(session, statement)
-            return [
-                self.db_model(**entity.model_dump())
-                for entity in result.scalars().all()
-            ]
+        return await self._select_to_response(
+            lambda q: q.where(self.db_model.id.in_(id_list))
+        )
 
     async def count(self, filter: str | None = None) -> int:
         count_statement = select(func.count(1)).select_from(self.db_model)
         if filter:
-            filter_param = parse_filter_param(self.db_model, filter)
+            filter_param = self._parse_filter_param(self.db_model, filter)
             count_statement = count_statement.where(*filter_param)
         async with self._session_scope() as session:
             result = await self._execute_statement(session, count_statement)
@@ -96,33 +142,31 @@ class BaseDBRepository(Generic[DBModel, ResponseModel, CreateModel, UpdateModel]
         filter: str | None = None,
         sort: str | None = None,
     ) -> list[ResponseModel]:
-        offset = (page - 1) * page_size
-        statement = self._select().offset(offset).limit(page_size)
-        if filter:
-            filter_param = parse_filter_param(self.db_model, filter)
-            statement = statement.where(*filter_param)
-        if sort:
-            sort_param = parse_sort_param(self.db_model, sort)
-            statement = statement.order_by(*sort_param)
-        async with self._session_scope() as session:
-            result = await self._execute_statement(session, statement)
-            return [
-                self.db_model(**entity.model_dump())
-                for entity in result.scalars().all()
-            ]
+        return await self._select_to_response(
+            self._get_pagination_query_modifier(
+                page=page, page_size=page_size, filter=filter, sort=sort
+            )
+        )
 
-    def _model_to_data_dict(
-        self, data: SQLModel, **additional_data: Any
-    ) -> dict[str, Any]:
-        data_dict = data.model_dump(exclude_unset=True)
-        data_dict.update(additional_data)
-        for key, preprocessor in self.column_preprocessors.items():
-            if key not in data_dict:
-                continue
-            if not hasattr(self.db_model, key):
-                raise InvalidValueError(f"Invalid {self.entity_name} property: {key}")
-            data_dict[key] = preprocessor(data_dict[key])
-        return data_dict
+    def _get_pagination_query_modifier(
+        self,
+        page: int = 1,
+        page_size: int = 10,
+        filter: str | None = None,
+        sort: str | None = None,
+    ) -> Callable[[Select], Any]:
+        def pagination_query_modifier(statement: Select) -> Any:
+            offset = (page - 1) * page_size
+            statement = statement.offset(offset).limit(page_size)
+            if filter:
+                filter_param = self._parse_filter_param(self.db_model, filter)
+                statement = statement.where(*filter_param)
+            if sort:
+                sort_param = self._parse_sort_param(self.db_model, sort)
+                statement = statement.order_by(*sort_param)
+            return statement
+
+        return pagination_query_modifier
 
     async def create(self, data: CreateModel) -> DBModel:
         now = datetime.datetime.now(datetime.timezone.utc)

{zrb-1.0.0b2 → zrb-1.0.0b4}/src/zrb/builtin/project/add/fastapp/fastapp_template/my_app_name/config.py

@@ -50,5 +50,17 @@ APP_AUTH_SUPER_USER = os.getenv("MY_APP_NAME_AUTH_SUPER_USER", "admin")
 APP_AUTH_SUPER_USER_PASSWORD = os.getenv(
     "MY_APP_NAME_AUTH_SUPER_USER_PASSWORD", "my-secure-password"
 )
+APP_AUTH_GUEST_USER = os.getenv("MY_APP_NAME_AUTH_GUEST_USER", "user")
+APP_AUTH_GUEST_USER_PERMISSIONS = (
+    permission_name.strip()
+    for permission_name in os.getenv(
+        "MY_APP_NAME_AUTH_GUEST_USER_PERMISSIONS", ""
+    ).split(",")
+    if permission_name.strip() != ""
+)
+APP_MAX_PARALLEL_SESSION = int(os.getenv("MY_APP_NAME_MAX_PARALLEL_SESSION", "1"))
+APP_SESSION_EXPIRE_MINUTES = int(
+    os.getenv("MY_APP_NAME_SESSION_EXPIRE_MINUTES", "1440")
+)
 
 APP_AUTH_BASE_URL = os.getenv("MY_APP_NAME_AUTH_BASE_URL", "http://localhost:3001")