zrb 1.0.0a20__tar.gz → 1.0.0b3__tar.gz

{zrb-1.0.0a20 → zrb-1.0.0b3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: zrb
-Version: 1.0.0a20
+Version: 1.0.0b3
 Summary: Your Automation Powerhouse
 Home-page: https://github.com/state-alchemists/zrb
 License: AGPL-3.0-or-later
@@ -26,15 +26,18 @@ Requires-Dist: pdfplumber (>=0.11.4,<0.12.0) ; extra == "rag"
 Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
 Requires-Dist: python-jose[cryptography] (>=3.3.0,<4.0.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)
+Requires-Dist: ulid-py (>=1.1.0,<2.0.0)
 Project-URL: Documentation, https://github.com/state-alchemists/zrb
 Project-URL: Repository, https://github.com/state-alchemists/zrb
 Description-Content-Type: text/markdown
 
 ![](https://raw.githubusercontent.com/state-alchemists/zrb/main/_images/zrb/android-chrome-192x192.png)
 
+[Documentation](https://github.com/state-alchemists/zrb/blob/main/docs/README.md)
+
 # 🤖 Zrb: Your Automation Powerhouse
 
-With Zrb, you can write your automation tasks like this:
+Zrb allows you to write your automation tasks in Python and declaratively:
 
 
 ```python
@@ -52,59 +55,13 @@ math.add_task(Task(
 ))
 ```
 
-You can then access the task in various ways.
-
-__Using CLI with arguments__
-
-```bash
-zrb math add 4 5
-```
-
-Result:
-
-```
-9
-To run again: zrb math add --a=4 --b=5
-```
-
-__Using CLI with keyword arguments__
-
-```bash
-zrb math add --a 4 --b 5
-```
-
-Result:
-
-```
-9
-To run again: zrb math add --a=4 --b=5
-```
-
-__Using CLI with incomplete arguments__
-
-```bash
-zrb math add 4
-```
-
-Result:
-
-```
-b [0]: 5
-9
-To run again: zrb math add 4
-```
-
-__Using Web Interface__
-
-```bash
-zrb server start
-```
+Once defined, you will be able to access your automation tasks from the CLI, Web Interface, or via HTTP API.
 
-Result (you need to access `http://localhost:21213`)
+For more complex scenario, you can also defined Task dependencies (upstreams) and retry mechanisms. You can also make a scheduled tasks, just like in Apache Airflow.
 
-![](https://raw.githubusercontent.com/state-alchemists/zrb/refs/heads/1.0.0/_images/web.png)
+Furthermore, Zrb has some builtin tasks to manage monorepo, generate FastAPI application, or play around with LLM.
 
-__More:__
+See the [getting started guide](https://github.com/state-alchemists/zrb/blob/main/docs/recipes/getting-started/README.md) for more information. Or just watch the demo:
 
 [![Video Title](https://img.youtube.com/vi/W7dgk96l__o/0.jpg)](https://www.youtube.com/watch?v=W7dgk96l__o)
 

{zrb-1.0.0a20 → zrb-1.0.0b3}/README.md

@@ -1,8 +1,10 @@
 ![](https://raw.githubusercontent.com/state-alchemists/zrb/main/_images/zrb/android-chrome-192x192.png)
 
+[Documentation](https://github.com/state-alchemists/zrb/blob/main/docs/README.md)
+
 # 🤖 Zrb: Your Automation Powerhouse
 
-With Zrb, you can write your automation tasks like this:
+Zrb allows you to write your automation tasks in Python and declaratively:
 
 
 ```python
@@ -20,59 +22,13 @@ math.add_task(Task(
 ))
 ```
 
-You can then access the task in various ways.
-
-__Using CLI with arguments__
-
-```bash
-zrb math add 4 5
-```
-
-Result:
-
-```
-9
-To run again: zrb math add --a=4 --b=5
-```
-
-__Using CLI with keyword arguments__
-
-```bash
-zrb math add --a 4 --b 5
-```
-
-Result:
-
-```
-9
-To run again: zrb math add --a=4 --b=5
-```
-
-__Using CLI with incomplete arguments__
-
-```bash
-zrb math add 4
-```
-
-Result:
-
-```
-b [0]: 5
-9
-To run again: zrb math add 4
-```
-
-__Using Web Interface__
-
-```bash
-zrb server start
-```
+Once defined, you will be able to access your automation tasks from the CLI, Web Interface, or via HTTP API.
 
-Result (you need to access `http://localhost:21213`)
+For more complex scenario, you can also defined Task dependencies (upstreams) and retry mechanisms. You can also make a scheduled tasks, just like in Apache Airflow.
 
-![](https://raw.githubusercontent.com/state-alchemists/zrb/refs/heads/1.0.0/_images/web.png)
+Furthermore, Zrb has some builtin tasks to manage monorepo, generate FastAPI application, or play around with LLM.
 
-__More:__
+See the [getting started guide](https://github.com/state-alchemists/zrb/blob/main/docs/recipes/getting-started/README.md) for more information. Or just watch the demo:
 
 [![Video Title](https://img.youtube.com/vi/W7dgk96l__o/0.jpg)](https://www.youtube.com/watch?v=W7dgk96l__o)
 

{zrb-1.0.0a20 → zrb-1.0.0b3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "zrb"
-version = "1.0.0a20"
+version = "1.0.0b3"
 description = "Your Automation Powerhouse"
 authors = ["Go Frendi Gunawan <gofrendiasgard@gmail.com>"]
 license = "AGPL-3.0-or-later"
@@ -50,16 +50,19 @@ pdfplumber = {version = "^0.11.4", optional = true}
 beautifulsoup4 = "^4.12.3"
 fastapi = {extras = ["standard"], version = "^0.115.6"}
 python-jose = {extras = ["cryptography"], version = "^3.3.0"}
+ulid-py = "^1.1.0"
 
 [tool.poetry.extras]
 # poetry install -E rag
 rag = ["chromadb", "pdfplumber"]
 
 [tool.poetry.dev-dependencies]
+alembic = "^1.14.0" # FastApp dependencies
 flake8 = "~7.1.1"
 pytest = "~8.3.3"
 pytest-cov = "~6.0.0"
 pytest-asyncio = "~0.24.0"
+sqlmodel = "^0.0.22" # FastApp dependencies
 tomlkit = "^0.13.2"
 
 [tool.pytest.ini_options]

{zrb-1.0.0a20 → zrb-1.0.0b3}/src/zrb/__init__.py

@@ -34,7 +34,8 @@ from zrb.input.password_input import PasswordInput
 from zrb.input.str_input import StrInput
 from zrb.input.text_input import TextInput
 from zrb.runner.cli import cli
-from zrb.runner.web_config import User, web_config
+from zrb.runner.web_config.config_factory import web_config
+from zrb.runner.web_schema.user import User
 from zrb.session.session import Session
 from zrb.task.any_task import AnyTask
 from zrb.task.base_task import BaseTask

{zrb-1.0.0a20 → zrb-1.0.0b3}/src/zrb/__main__.py

@@ -14,10 +14,10 @@ def serve_cli():
         for init_script in INIT_SCRIPTS:
             load_file(init_script, -1)
         cli.run(sys.argv[1:])
-    except RuntimeError as e:
-        if "Event loop is closed" not in f"{e}":
-            raise e
     except KeyboardInterrupt:
         print(stylize_warning("\nStopped"), file=sys.stderr)
+    except RuntimeError as e:
+        if f"{e}".lower() != "event loop is closed":
+            raise e
     except NodeNotFoundError as e:
         print(stylize_error(f"{e}"), file=sys.stderr)

{zrb-1.0.0a20 → zrb-1.0.0b3}/src/zrb/builtin/__init__.py

@@ -12,6 +12,7 @@ from zrb.builtin.md5 import hash_md5, sum_md5
 from zrb.builtin.project.add.fastapp.fastapp_task import add_fastapp_to_project
 from zrb.builtin.project.create.project_task import create_project
 from zrb.builtin.python import format_python_code
+from zrb.builtin.random import shuffle_values, throw_dice
 from zrb.builtin.setup.asdf.asdf import setup_asdf
 from zrb.builtin.setup.latex.ubuntu import setup_latex_on_ubuntu
 from zrb.builtin.setup.tmux.tmux import setup_tmux
@@ -55,6 +56,8 @@ assert edit_todo
 assert complete_todo
 assert log_todo
 assert show_todo
+assert throw_dice
+assert shuffle_values
 assert setup_ubuntu
 assert setup_latex_on_ubuntu
 assert setup_asdf

{zrb-1.0.0a20 → zrb-1.0.0b3}/src/zrb/builtin/group.py

@@ -2,6 +2,7 @@ from zrb.group.group import Group
 from zrb.runner.cli import cli
 
 base64_group = cli.add_group(Group(name="base64", description="📄 Base64 operations"))
+random_group = cli.add_group(Group(name="random", description="🔀 Random operation"))
 git_group = cli.add_group(Group(name="git", description="🌱 Git related commands"))
 git_branch_group = git_group.add_group(
     Group(name="branch", description="🌿 Git branch related commands")

{zrb-1.0.0a20 → zrb-1.0.0b3}/src/zrb/builtin/llm/llm_chat.py

@@ -1,6 +1,7 @@
 from zrb.builtin.group import llm_group
 from zrb.builtin.llm.tool.cli import run_shell_command
-from zrb.builtin.llm.tool.web import open_web_page, query_internet
+from zrb.builtin.llm.tool.rag import create_rag_from_directory
+from zrb.builtin.llm.tool.web import open_web_route, query_internet
 from zrb.config import (
     LLM_ALLOW_ACCESS_SHELL,
     LLM_ALLOW_ACCESS_WEB,
@@ -22,7 +23,7 @@ llm_chat: LLMTask = llm_group.add_task(
                 prompt="LLM Model",
                 default_str=LLM_MODEL,
             ),
-            StrInput(
+            TextInput(
                 "system-prompt",
                 description="System prompt",
                 prompt="System prompt",
@@ -35,6 +36,7 @@ llm_chat: LLMTask = llm_group.add_task(
         model="{ctx.input.model}",
         system_prompt="{ctx.input['system-prompt']}",
         message="{ctx.input.message}",
+        retries=0,
     ),
     alias="chat",
 )
@@ -43,5 +45,5 @@ if LLM_ALLOW_ACCESS_SHELL:
     llm_chat.add_tool(run_shell_command)
 
 if LLM_ALLOW_ACCESS_WEB:
-    llm_chat.add_tool(open_web_page)
+    llm_chat.add_tool(open_web_route)
     llm_chat.add_tool(query_internet)

{zrb-1.0.0a20 → zrb-1.0.0b3}/src/zrb/builtin/llm/tool/cli.py

@@ -2,7 +2,7 @@ import subprocess
 
 
 def run_shell_command(command: str) -> str:
-    """Running a shell command"""
+    """Running an actual shell command on user's computer."""
     output = subprocess.check_output(
         command, shell=True, stderr=subprocess.STDOUT, text=True
     )

zrb-1.0.0b3/src/zrb/builtin/llm/tool/rag.py

@@ -0,0 +1,152 @@
+import hashlib
+import json
+import os
+import sys
+
+import litellm
+import ulid
+
+from zrb.config import (
+    RAG_CHUNK_SIZE,
+    RAG_EMBEDDING_MODEL,
+    RAG_MAX_RESULT_COUNT,
+    RAG_OVERLAP,
+)
+from zrb.util.cli.style import stylize_error, stylize_faint
+from zrb.util.file import read_file
+
+
+def create_rag_from_directory(
+    tool_name: str,
+    tool_description: str,
+    document_dir_path: str = "./documents",
+    model: str = RAG_EMBEDDING_MODEL,
+    vector_db_path: str = "./chroma",
+    vector_db_collection: str = "documents",
+    chunk_size: int = RAG_CHUNK_SIZE,
+    overlap: int = RAG_OVERLAP,
+    max_result_count: int = RAG_MAX_RESULT_COUNT,
+):
+    async def retrieve(query: str) -> str:
+        from chromadb import PersistentClient
+        from chromadb.config import Settings
+
+        client = PersistentClient(
+            path=vector_db_path, settings=Settings(allow_reset=True)
+        )
+        collection = client.get_or_create_collection(vector_db_collection)
+
+        # Track file changes using a hash-based approach
+        hash_file_path = os.path.join(vector_db_path, "file_hashes.json")
+        previous_hashes = _load_hashes(hash_file_path)
+        current_hashes = {}
+
+        updated_files = []
+
+        for root, _, files in os.walk(document_dir_path):
+            for file in files:
+                file_path = os.path.join(root, file)
+                file_hash = _compute_file_hash(file_path)
+                relative_path = os.path.relpath(file_path, document_dir_path)
+                current_hashes[relative_path] = file_hash
+
+                if previous_hashes.get(relative_path) != file_hash:
+                    updated_files.append(file_path)
+
+        if updated_files:
+            print(
+                stylize_faint(f"Updating {len(updated_files)} changed files"),
+                file=sys.stderr,
+            )
+
+            for file_path in updated_files:
+                try:
+                    relative_path = os.path.relpath(file_path, document_dir_path)
+                    collection.delete(where={"file_path": relative_path})
+                    content = _read_file_content(file_path)
+                    file_id = ulid.new().str
+                    for i in range(0, len(content), chunk_size - overlap):
+                        chunk = content[i : i + chunk_size]
+                        if chunk:
+                            chunk_id = ulid.new().str
+                            print(
+                                stylize_faint(
+                                    f"Vectorizing {relative_path} chunk {chunk_id}"
+                                ),
+                                file=sys.stderr,
+                            )
+                            response = await litellm.aembedding(
+                                model=model, input=[chunk]
+                            )
+                            vector = response["data"][0]["embedding"]
+                            collection.upsert(
+                                ids=[chunk_id],
+                                embeddings=[vector],
+                                documents=[chunk],
+                                metadatas={
+                                    "file_path": relative_path,
+                                    "file_id": file_id,
+                                },
+                            )
+                except Exception as e:
+                    print(
+                        stylize_error(f"Error processing {file_path}: {e}"),
+                        file=sys.stderr,
+                    )
+
+            _save_hashes(hash_file_path, current_hashes)
+        else:
+            print(
+                stylize_faint("No changes detected. Skipping database update."),
+                file=sys.stderr,
+            )
+
+        print(stylize_faint("Vectorizing query"), file=sys.stderr)
+        query_response = await litellm.aembedding(model=model, input=[query])
+        query_vector = query_response["data"][0]["embedding"]
+
+        print(stylize_faint("Searching documents"), file=sys.stderr)
+        results = collection.query(
+            query_embeddings=query_vector,
+            n_results=max_result_count,
+        )
+        return json.dumps(results)
+
+    retrieve.__name__ = tool_name
+    retrieve.__doc__ = tool_description
+    return retrieve
+
+
+def _compute_file_hash(file_path: str) -> str:
+    hash_md5 = hashlib.md5()
+    with open(file_path, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_md5.update(chunk)
+    return hash_md5.hexdigest()
+
+
+def _read_file_content(file_path: str) -> str:
+    if file_path.lower().endswith(".pdf"):
+        return _read_pdf(file_path)
+    return read_file(file_path)
+
+
+def _read_pdf(file_path: str) -> str:
+    import pdfplumber
+
+    with pdfplumber.open(file_path) as pdf:
+        return "\n".join(
+            page.extract_text() for page in pdf.pages if page.extract_text()
+        )
+
+
+def _load_hashes(file_path: str) -> dict:
+    if os.path.exists(file_path):
+        with open(file_path, "r") as f:
+            return json.load(f)
+    return {}
+
+
+def _save_hashes(file_path: str, hashes: dict):
+    with open(file_path, "w") as f:
+        json.dump(hashes, f)

{zrb-1.0.0a20 → zrb-1.0.0b3}/src/zrb/builtin/llm/tool/web.py

@@ -2,7 +2,7 @@ import json
 from typing import Annotated
 
 
-def open_web_page(url: str) -> str:
+def open_web_route(url: str) -> str:
     """Get content from a web page."""
     import requests
 

{zrb-1.0.0a20 → zrb-1.0.0b3}/src/zrb/builtin/project/add/fastapp/fastapp_task.py

@@ -49,6 +49,7 @@ scaffold_fastapp = Scaffolder(
     transform_content=[
         # Common transformation (project_dir/app_dir/**/*)
         ContentTransformer(
+            name="transform-app-dir",
             match=is_in_project_app_dir,
             transform={
                 "My App Name": "{ctx.input.app.title()}",
@@ -60,6 +61,7 @@ scaffold_fastapp = Scaffolder(
         ),
         # Register fastapp's tasks to project's zrb_init (project_dir/zrb_init.py)
         ContentTransformer(
+            name="trasnform-zrb-init",
             match=is_project_zrb_init_file,
             transform=update_project_zrb_init_file,
         ),

{zrb-1.0.0a20 → zrb-1.0.0b3}/src/zrb/builtin/project/add/fastapp/fastapp_template/my_app_name/_zrb/config.py

@@ -9,9 +9,12 @@ MICROSERVICES_ENV_VARS = {
     "MY_APP_NAME_MODE": "microservices",
     "MY_APP_NAME_AUTH_BASE_URL": "http://localhost:3002",
 }
-MONOLITH_ENV_VARS = {"MY_APP_NAME_MODE": "monolith"}
+MONOLITH_ENV_VARS = {
+    "MY_APP_NAME_MODE": "monolith",
+    "MY_APP_NAME_MODULES": "",
+}
 
 if platform.system() == "Windows":
     ACTIVATE_VENV_SCRIPT = "Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser; . .venv\\Scripts\\Activate"  # noqa
 else:
-    ACTIVATE_VENV_SCRIPT = "source .venv/bin/activate"
+    ACTIVATE_VENV_SCRIPT = ". .venv/bin/activate"

{zrb-1.0.0a20 → zrb-1.0.0b3}/src/zrb/builtin/project/add/fastapp/fastapp_template/my_app_name/_zrb/entity/add_entity_task.py

@@ -1,19 +1,21 @@
 import os
 
-from my_app_name._zrb.config import APP_DIR
+from my_app_name._zrb.config import ACTIVATE_VENV_SCRIPT, APP_DIR
 from my_app_name._zrb.entity.add_entity_util import (
     is_in_app_schema_dir,
     is_in_module_entity_dir,
-    is_module_any_client_file,
     is_module_api_client_file,
+    is_module_client_file,
     is_module_direct_client_file,
+    is_module_gateway_subroute_file,
     is_module_migration_metadata_file,
     is_module_route_file,
-    update_any_client,
-    update_api_client,
-    update_direct_client,
-    update_migration_metadata,
-    update_route,
+    update_api_client_file,
+    update_client_file,
+    update_direct_client_file,
+    update_gateway_subroute_file,
+    update_migration_metadata_file,
+    update_route_file,
 )
 from my_app_name._zrb.format_task import format_my_app_name_code
 from my_app_name._zrb.group import app_create_group
@@ -23,9 +25,25 @@ from my_app_name._zrb.input import (
     new_entity_input,
     plural_entity_input,
 )
-from my_app_name._zrb.util import get_existing_module_names, get_existing_schema_names
+from my_app_name._zrb.util import (
+    cd_module_script,
+    get_existing_module_names,
+    get_existing_schema_names,
+    set_create_migration_db_url_env,
+    set_env,
+)
+from my_app_name._zrb.venv_task import prepare_venv
 
-from zrb import AnyContext, ContentTransformer, Scaffolder, Task, make_task
+from zrb import (
+    AnyContext,
+    Cmd,
+    CmdTask,
+    ContentTransformer,
+    EnvFile,
+    Scaffolder,
+    Task,
+    make_task,
+)
 from zrb.util.string.conversion import to_snake_case
 
 
@@ -66,6 +84,7 @@ scaffold_my_app_name_entity = Scaffolder(
     transform_content=[
         # Schema tranformation (my_app_name/schema/snake_entity_name)
         ContentTransformer(
+            name="transform-schema-file",
             match=is_in_app_schema_dir,
             transform={
                 "MyEntity": "{to_pascal_case(ctx.input.entity)}",
@@ -75,6 +94,7 @@ scaffold_my_app_name_entity = Scaffolder(
         # Common module's entity transformation
         # (my_app_name/module/snake_module_name/service/snake_entity_name)
         ContentTransformer(
+            name="transform-module-entity-dir",
             match=is_in_module_entity_dir,
             transform={
                 "my_module": "{to_snake_case(ctx.input.module)}",
@@ -87,40 +107,80 @@ scaffold_my_app_name_entity = Scaffolder(
         # Add entity to migration metadata
         # (my_app_name/module/snake_module_name/migration_metadata.py)
         ContentTransformer(
+            name="transform-module-migration-metadata",
             match=is_module_migration_metadata_file,
-            transform=update_migration_metadata,
+            transform=update_migration_metadata_file,
         ),
-        # Update API Client (my_app_name/module/snake_module_name/client/api_client.py)
+        # Update API Client
+        # (my_app_name/module/snake_module_name/client/snake_module_name_api_client.py)
         ContentTransformer(
+            name="transform-module-api-client",
             match=is_module_api_client_file,
-            transform=update_api_client,
+            transform=update_api_client_file,
         ),
-        # Update Direct Client (my_app_name/module/snake_module_name/client/direct_client.py)
+        # Update Direct Client
+        # (my_app_name/module/snake_module_name/client/snake_module_name_direct_client.py)
         ContentTransformer(
+            name="transform-module-direct-client",
             match=is_module_direct_client_file,
-            transform=update_direct_client,
+            transform=update_direct_client_file,
         ),
-        # Update Any Client (my_app_name/module/snake_module_name/client/any_client.py)
+        # Update Client
+        # (my_app_name/module/snake_module_name/client/snake_module_name_client.py)
         ContentTransformer(
-            match=is_module_any_client_file,
-            transform=update_any_client,
+            name="transform-module-any-client",
+            match=is_module_client_file,
+            transform=update_client_file,
         ),
         # Update module route (my_app_name/module/route.py)
         ContentTransformer(
+            name="transform-module-route",
             match=is_module_route_file,
-            transform=update_route,
+            transform=update_route_file,
+        ),
+        # Update module gateway subroute
+        # (my_app_name/module/gateway/subroute/snake_module_name.py)
+        ContentTransformer(
+            name="transform-module-gateway-subroute",
+            match=is_module_gateway_subroute_file,
+            transform=update_gateway_subroute_file,
         ),
-        # TODO: Register gateway route
     ],
     retries=0,
     upstream=validate_create_my_app_name_entity,
 )
 
+create_my_app_name_entity_migration = CmdTask(
+    name="create-my-app-name-entity-migration",
+    input=[
+        existing_module_input,
+        new_entity_input,
+    ],
+    env=EnvFile(path=os.path.join(APP_DIR, "template.env")),
+    cwd=APP_DIR,
+    cmd=[
+        ACTIVATE_VENV_SCRIPT,
+        Cmd(lambda ctx: set_create_migration_db_url_env(ctx.input.module)),
+        Cmd(lambda ctx: set_env("MY_APP_NAME_MODULES", ctx.input.module)),
+        Cmd(lambda ctx: cd_module_script(ctx.input.module)),
+        "alembic upgrade head",
+        Cmd(
+            'alembic revision --autogenerate -m "create_{to_snake_case(ctx.input.entity)}_table"',  # noqa
+        ),
+    ],
+    render_cmd=False,
+    retries=0,
+    upstream=[
+        prepare_venv,
+        scaffold_my_app_name_entity,
+    ],
+)
+
 add_my_app_name_entity = app_create_group.add_task(
     Task(
         name="add-my-app-name-entity",
         description="🏗️ Create new entity on a module",
-        upstream=scaffold_my_app_name_entity,
+        upstream=create_my_app_name_entity_migration,
         successor=format_my_app_name_code,
         retries=0,
     ),