zrb 1.0.0a20__py3-none-any.whl → 1.0.0b3__py3-none-any.whl

zrb/__init__.py

@@ -34,7 +34,8 @@ from zrb.input.password_input import PasswordInput
 from zrb.input.str_input import StrInput
 from zrb.input.text_input import TextInput
 from zrb.runner.cli import cli
-from zrb.runner.web_config import User, web_config
+from zrb.runner.web_config.config_factory import web_config
+from zrb.runner.web_schema.user import User
 from zrb.session.session import Session
 from zrb.task.any_task import AnyTask
 from zrb.task.base_task import BaseTask

zrb/__main__.py

@@ -14,10 +14,10 @@ def serve_cli():
         for init_script in INIT_SCRIPTS:
             load_file(init_script, -1)
         cli.run(sys.argv[1:])
-    except RuntimeError as e:
-        if "Event loop is closed" not in f"{e}":
-            raise e
     except KeyboardInterrupt:
         print(stylize_warning("\nStopped"), file=sys.stderr)
+    except RuntimeError as e:
+        if f"{e}".lower() != "event loop is closed":
+            raise e
     except NodeNotFoundError as e:
         print(stylize_error(f"{e}"), file=sys.stderr)

zrb/builtin/__init__.py

@@ -12,6 +12,7 @@ from zrb.builtin.md5 import hash_md5, sum_md5
 from zrb.builtin.project.add.fastapp.fastapp_task import add_fastapp_to_project
 from zrb.builtin.project.create.project_task import create_project
 from zrb.builtin.python import format_python_code
+from zrb.builtin.random import shuffle_values, throw_dice
 from zrb.builtin.setup.asdf.asdf import setup_asdf
 from zrb.builtin.setup.latex.ubuntu import setup_latex_on_ubuntu
 from zrb.builtin.setup.tmux.tmux import setup_tmux
@@ -55,6 +56,8 @@ assert edit_todo
 assert complete_todo
 assert log_todo
 assert show_todo
+assert throw_dice
+assert shuffle_values
 assert setup_ubuntu
 assert setup_latex_on_ubuntu
 assert setup_asdf

zrb/builtin/group.py

@@ -2,6 +2,7 @@ from zrb.group.group import Group
 from zrb.runner.cli import cli
 
 base64_group = cli.add_group(Group(name="base64", description="📄 Base64 operations"))
+random_group = cli.add_group(Group(name="random", description="🔀 Random operation"))
 git_group = cli.add_group(Group(name="git", description="🌱 Git related commands"))
 git_branch_group = git_group.add_group(
     Group(name="branch", description="🌿 Git branch related commands")

zrb/builtin/llm/llm_chat.py

@@ -1,6 +1,7 @@
 from zrb.builtin.group import llm_group
 from zrb.builtin.llm.tool.cli import run_shell_command
-from zrb.builtin.llm.tool.web import open_web_page, query_internet
+from zrb.builtin.llm.tool.rag import create_rag_from_directory
+from zrb.builtin.llm.tool.web import open_web_route, query_internet
 from zrb.config import (
     LLM_ALLOW_ACCESS_SHELL,
     LLM_ALLOW_ACCESS_WEB,
@@ -22,7 +23,7 @@ llm_chat: LLMTask = llm_group.add_task(
                 prompt="LLM Model",
                 default_str=LLM_MODEL,
             ),
-            StrInput(
+            TextInput(
                 "system-prompt",
                 description="System prompt",
                 prompt="System prompt",
@@ -35,6 +36,7 @@ llm_chat: LLMTask = llm_group.add_task(
         model="{ctx.input.model}",
         system_prompt="{ctx.input['system-prompt']}",
         message="{ctx.input.message}",
+        retries=0,
     ),
     alias="chat",
 )
@@ -43,5 +45,5 @@ if LLM_ALLOW_ACCESS_SHELL:
     llm_chat.add_tool(run_shell_command)
 
 if LLM_ALLOW_ACCESS_WEB:
-    llm_chat.add_tool(open_web_page)
+    llm_chat.add_tool(open_web_route)
     llm_chat.add_tool(query_internet)

zrb/builtin/llm/tool/cli.py

@@ -2,7 +2,7 @@ import subprocess
 
 
 def run_shell_command(command: str) -> str:
-    """Running a shell command"""
+    """Running an actual shell command on user's computer."""
     output = subprocess.check_output(
         command, shell=True, stderr=subprocess.STDOUT, text=True
     )

zrb/builtin/llm/tool/rag.py

@@ -1,9 +1,10 @@
+import hashlib
 import json
 import os
 import sys
-from collections.abc import Callable, Iterable
 
 import litellm
+import ulid
 
 from zrb.config import (
     RAG_CHUNK_SIZE,
@@ -13,10 +14,6 @@ from zrb.config import (
 )
 from zrb.util.cli.style import stylize_error, stylize_faint
 from zrb.util.file import read_file
-from zrb.util.run import run_async
-
-Document = str | Callable[[], str]
-Documents = Callable[[], Iterable[Document]] | Iterable[Document]
 
 
 def create_rag_from_directory(
@@ -30,86 +27,87 @@ def create_rag_from_directory(
     overlap: int = RAG_OVERLAP,
     max_result_count: int = RAG_MAX_RESULT_COUNT,
 ):
-    return create_rag(
-        tool_name=tool_name,
-        tool_description=tool_description,
-        documents=get_rag_documents(os.path.expanduser(document_dir_path)),
-        model=model,
-        vector_db_path=vector_db_path,
-        vector_db_collection=vector_db_collection,
-        reset_db=get_rag_reset_db(
-            document_dir_path=os.path.expanduser(document_dir_path),
-            vector_db_path=os.path.expanduser(vector_db_path),
-        ),
-        chunk_size=chunk_size,
-        overlap=overlap,
-        max_result_count=max_result_count,
-    )
-
-
-def create_rag(
-    tool_name: str,
-    tool_description: str,
-    documents: Documents = [],
-    model: str = RAG_EMBEDDING_MODEL,
-    vector_db_path: str = "./chroma",
-    vector_db_collection: str = "documents",
-    reset_db: Callable[[], bool] | bool = False,
-    chunk_size: int = RAG_CHUNK_SIZE,
-    overlap: int = RAG_OVERLAP,
-    max_result_count: int = RAG_MAX_RESULT_COUNT,
-) -> Callable[[str], str]:
     async def retrieve(query: str) -> str:
-        import chromadb
+        from chromadb import PersistentClient
         from chromadb.config import Settings
 
-        is_db_exist = os.path.isdir(vector_db_path)
-        client = chromadb.PersistentClient(
+        client = PersistentClient(
             path=vector_db_path, settings=Settings(allow_reset=True)
         )
-        should_reset_db = (
-            await run_async(reset_db()) if callable(reset_db) else reset_db
-        )
-        if (not is_db_exist) or should_reset_db:
-            client.reset()
-            collection = client.get_or_create_collection(vector_db_collection)
-            chunk_index = 0
-            print(stylize_faint("Scanning documents"), file=sys.stderr)
-            docs = await run_async(documents()) if callable(documents) else documents
-            for document in docs:
-                if callable(document):
-                    try:
-                        document = await run_async(document())
-                    except Exception as error:
-                        print(stylize_error(f"Error: {error}"), file=sys.stderr)
-                        continue
-                for i in range(0, len(document), chunk_size - overlap):
-                    chunk = document[i : i + chunk_size]
-                    if len(chunk) > 0:
-                        print(
-                            stylize_faint(f"Vectorize chunk {chunk_index}"),
-                            file=sys.stderr,
-                        )
-                        response = await litellm.aembedding(model=model, input=[chunk])
-                        vector = response["data"][0]["embedding"]
-                        print(
-                            stylize_faint(f"Adding chunk {chunk_index} to db"),
-                            file=sys.stderr,
-                        )
-                        collection.upsert(
-                            ids=[f"id{chunk_index}"],
-                            embeddings=[vector],
-                            documents=[chunk],
-                        )
-                        chunk_index += 1
         collection = client.get_or_create_collection(vector_db_collection)
-        # Generate embedding for the query
-        print(stylize_faint("Vectorize query"), file=sys.stderr)
+
+        # Track file changes using a hash-based approach
+        hash_file_path = os.path.join(vector_db_path, "file_hashes.json")
+        previous_hashes = _load_hashes(hash_file_path)
+        current_hashes = {}
+
+        updated_files = []
+
+        for root, _, files in os.walk(document_dir_path):
+            for file in files:
+                file_path = os.path.join(root, file)
+                file_hash = _compute_file_hash(file_path)
+                relative_path = os.path.relpath(file_path, document_dir_path)
+                current_hashes[relative_path] = file_hash
+
+                if previous_hashes.get(relative_path) != file_hash:
+                    updated_files.append(file_path)
+
+        if updated_files:
+            print(
+                stylize_faint(f"Updating {len(updated_files)} changed files"),
+                file=sys.stderr,
+            )
+
+            for file_path in updated_files:
+                try:
+                    relative_path = os.path.relpath(file_path, document_dir_path)
+                    collection.delete(where={"file_path": relative_path})
+                    content = _read_file_content(file_path)
+                    file_id = ulid.new().str
+                    for i in range(0, len(content), chunk_size - overlap):
+                        chunk = content[i : i + chunk_size]
+                        if chunk:
+                            chunk_id = ulid.new().str
+                            print(
+                                stylize_faint(
+                                    f"Vectorizing {relative_path} chunk {chunk_id}"
+                                ),
+                                file=sys.stderr,
+                            )
+                            response = await litellm.aembedding(
+                                model=model, input=[chunk]
+                            )
+                            vector = response["data"][0]["embedding"]
+                            collection.upsert(
+                                ids=[chunk_id],
+                                embeddings=[vector],
+                                documents=[chunk],
+                                metadatas={
+                                    "file_path": relative_path,
+                                    "file_id": file_id,
+                                },
+                            )
+                except Exception as e:
+                    print(
+                        stylize_error(f"Error processing {file_path}: {e}"),
+                        file=sys.stderr,
+                    )
+
+            _save_hashes(hash_file_path, current_hashes)
+        else:
+            print(
+                stylize_faint("No changes detected. Skipping database update."),
+                file=sys.stderr,
+            )
+
+        print(stylize_faint("Vectorizing query"), file=sys.stderr)
         query_response = await litellm.aembedding(model=model, input=[query])
-        print(stylize_faint("Search documents"), file=sys.stderr)
-        # Search for the top_k most similar documents
+        query_vector = query_response["data"][0]["embedding"]
+
+        print(stylize_faint("Searching documents"), file=sys.stderr)
         results = collection.query(
-            query_embeddings=query_response["data"][0]["embedding"],
+            query_embeddings=query_vector,
             n_results=max_result_count,
         )
         return json.dumps(results)
@@ -119,71 +117,36 @@ def create_rag(
     return retrieve
 
 
-def get_rag_documents(document_dir_path: str) -> Callable[[], list[Callable[[], str]]]:
-    def get_documents() -> list[Callable[[], str]]:
-        # Walk through the directory
-        readers = []
-        for root, _, files in os.walk(document_dir_path):
-            for file in files:
-                file_path = os.path.join(root, file)
-                if file_path.lower().endswith(".pdf"):
-                    readers.append(_get_pdf_reader(file_path))
-                    continue
-                readers.append(_get_text_reader(file_path))
-        return readers
-
-    return get_documents
-
-
-def _get_text_reader(file_path: str):
-    def read():
-        print(stylize_faint(f"Start reading {file_path}"), file=sys.stderr)
-        content = read_file(file_path)
-        print(stylize_faint(f"Complete reading {file_path}"), file=sys.stderr)
-        return content
-
-    return read
-
-
-def _get_pdf_reader(file_path):
-    def read():
-        import pdfplumber
-
-        print(stylize_faint(f"Start reading {file_path}"), file=sys.stderr)
-        contents = []
-        with pdfplumber.open(file_path) as pdf:
-            for page in pdf.pages:
-                contents.append(page.extract_text())
-        print(stylize_faint(f"Complete reading {file_path}"), file=sys.stderr)
-        return "\n".join(contents)
-
-    return read
-
-
-def get_rag_reset_db(
-    document_dir_path: str, vector_db_path: str = "./chroma"
-) -> Callable[[], bool]:
-    def should_reset_db() -> bool:
-        document_exist = os.path.isdir(document_dir_path)
-        if not document_exist:
-            raise ValueError(f"Document directory not exists: {document_dir_path}")
-        vector_db_exist = os.path.isdir(vector_db_path)
-        if not vector_db_exist:
-            return True
-        document_mtime = _get_most_recent_mtime(document_dir_path)
-        vector_db_mtime = _get_most_recent_mtime(vector_db_path)
-        return document_mtime > vector_db_mtime
-
-    return should_reset_db
-
-
-def _get_most_recent_mtime(directory):
-    most_recent_mtime = 0
-    for root, dirs, files in os.walk(directory):
-        # Check mtime for directories
-        for name in dirs + files:
-            file_path = os.path.join(root, name)
-            mtime = os.path.getmtime(file_path)
-            if mtime > most_recent_mtime:
-                most_recent_mtime = mtime
-    return most_recent_mtime
+def _compute_file_hash(file_path: str) -> str:
+    hash_md5 = hashlib.md5()
+    with open(file_path, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_md5.update(chunk)
+    return hash_md5.hexdigest()
+
+
+def _read_file_content(file_path: str) -> str:
+    if file_path.lower().endswith(".pdf"):
+        return _read_pdf(file_path)
+    return read_file(file_path)
+
+
+def _read_pdf(file_path: str) -> str:
+    import pdfplumber
+
+    with pdfplumber.open(file_path) as pdf:
+        return "\n".join(
+            page.extract_text() for page in pdf.pages if page.extract_text()
+        )
+
+
+def _load_hashes(file_path: str) -> dict:
+    if os.path.exists(file_path):
+        with open(file_path, "r") as f:
+            return json.load(f)
+    return {}
+
+
+def _save_hashes(file_path: str, hashes: dict):
+    with open(file_path, "w") as f:
+        json.dump(hashes, f)

zrb/builtin/llm/tool/web.py

@@ -2,7 +2,7 @@ import json
 from typing import Annotated
 
 
-def open_web_page(url: str) -> str:
+def open_web_route(url: str) -> str:
     """Get content from a web page."""
     import requests
 

zrb/builtin/project/add/fastapp/fastapp_task.py

@@ -49,6 +49,7 @@ scaffold_fastapp = Scaffolder(
     transform_content=[
         # Common transformation (project_dir/app_dir/**/*)
         ContentTransformer(
+            name="transform-app-dir",
             match=is_in_project_app_dir,
             transform={
                 "My App Name": "{ctx.input.app.title()}",
@@ -60,6 +61,7 @@ scaffold_fastapp = Scaffolder(
         ),
         # Register fastapp's tasks to project's zrb_init (project_dir/zrb_init.py)
         ContentTransformer(
+            name="trasnform-zrb-init",
             match=is_project_zrb_init_file,
             transform=update_project_zrb_init_file,
         ),

zrb/builtin/project/add/fastapp/fastapp_template/my_app_name/_zrb/config.py

@@ -9,9 +9,12 @@ MICROSERVICES_ENV_VARS = {
     "MY_APP_NAME_MODE": "microservices",
     "MY_APP_NAME_AUTH_BASE_URL": "http://localhost:3002",
 }
-MONOLITH_ENV_VARS = {"MY_APP_NAME_MODE": "monolith"}
+MONOLITH_ENV_VARS = {
+    "MY_APP_NAME_MODE": "monolith",
+    "MY_APP_NAME_MODULES": "",
+}
 
 if platform.system() == "Windows":
     ACTIVATE_VENV_SCRIPT = "Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser; . .venv\\Scripts\\Activate"  # noqa
 else:
-    ACTIVATE_VENV_SCRIPT = "source .venv/bin/activate"
+    ACTIVATE_VENV_SCRIPT = ". .venv/bin/activate"

zrb/builtin/project/add/fastapp/fastapp_template/my_app_name/_zrb/entity/add_entity_task.py

@@ -1,19 +1,21 @@
 import os
 
-from my_app_name._zrb.config import APP_DIR
+from my_app_name._zrb.config import ACTIVATE_VENV_SCRIPT, APP_DIR
 from my_app_name._zrb.entity.add_entity_util import (
     is_in_app_schema_dir,
     is_in_module_entity_dir,
-    is_module_any_client_file,
     is_module_api_client_file,
+    is_module_client_file,
     is_module_direct_client_file,
+    is_module_gateway_subroute_file,
     is_module_migration_metadata_file,
     is_module_route_file,
-    update_any_client,
-    update_api_client,
-    update_direct_client,
-    update_migration_metadata,
-    update_route,
+    update_api_client_file,
+    update_client_file,
+    update_direct_client_file,
+    update_gateway_subroute_file,
+    update_migration_metadata_file,
+    update_route_file,
 )
 from my_app_name._zrb.format_task import format_my_app_name_code
 from my_app_name._zrb.group import app_create_group
@@ -23,9 +25,25 @@ from my_app_name._zrb.input import (
     new_entity_input,
     plural_entity_input,
 )
-from my_app_name._zrb.util import get_existing_module_names, get_existing_schema_names
+from my_app_name._zrb.util import (
+    cd_module_script,
+    get_existing_module_names,
+    get_existing_schema_names,
+    set_create_migration_db_url_env,
+    set_env,
+)
+from my_app_name._zrb.venv_task import prepare_venv
 
-from zrb import AnyContext, ContentTransformer, Scaffolder, Task, make_task
+from zrb import (
+    AnyContext,
+    Cmd,
+    CmdTask,
+    ContentTransformer,
+    EnvFile,
+    Scaffolder,
+    Task,
+    make_task,
+)
 from zrb.util.string.conversion import to_snake_case
 
 
@@ -66,6 +84,7 @@ scaffold_my_app_name_entity = Scaffolder(
     transform_content=[
         # Schema tranformation (my_app_name/schema/snake_entity_name)
         ContentTransformer(
+            name="transform-schema-file",
             match=is_in_app_schema_dir,
             transform={
                 "MyEntity": "{to_pascal_case(ctx.input.entity)}",
@@ -75,6 +94,7 @@ scaffold_my_app_name_entity = Scaffolder(
         # Common module's entity transformation
         # (my_app_name/module/snake_module_name/service/snake_entity_name)
         ContentTransformer(
+            name="transform-module-entity-dir",
             match=is_in_module_entity_dir,
             transform={
                 "my_module": "{to_snake_case(ctx.input.module)}",
@@ -87,40 +107,80 @@ scaffold_my_app_name_entity = Scaffolder(
         # Add entity to migration metadata
         # (my_app_name/module/snake_module_name/migration_metadata.py)
         ContentTransformer(
+            name="transform-module-migration-metadata",
             match=is_module_migration_metadata_file,
-            transform=update_migration_metadata,
+            transform=update_migration_metadata_file,
         ),
-        # Update API Client (my_app_name/module/snake_module_name/client/api_client.py)
+        # Update API Client
+        # (my_app_name/module/snake_module_name/client/snake_module_name_api_client.py)
         ContentTransformer(
+            name="transform-module-api-client",
             match=is_module_api_client_file,
-            transform=update_api_client,
+            transform=update_api_client_file,
         ),
-        # Update Direct Client (my_app_name/module/snake_module_name/client/direct_client.py)
+        # Update Direct Client
+        # (my_app_name/module/snake_module_name/client/snake_module_name_direct_client.py)
         ContentTransformer(
+            name="transform-module-direct-client",
             match=is_module_direct_client_file,
-            transform=update_direct_client,
+            transform=update_direct_client_file,
         ),
-        # Update Any Client (my_app_name/module/snake_module_name/client/any_client.py)
+        # Update Client
+        # (my_app_name/module/snake_module_name/client/snake_module_name_client.py)
         ContentTransformer(
-            match=is_module_any_client_file,
-            transform=update_any_client,
+            name="transform-module-any-client",
+            match=is_module_client_file,
+            transform=update_client_file,
         ),
         # Update module route (my_app_name/module/route.py)
         ContentTransformer(
+            name="transform-module-route",
             match=is_module_route_file,
-            transform=update_route,
+            transform=update_route_file,
+        ),
+        # Update module gateway subroute
+        # (my_app_name/module/gateway/subroute/snake_module_name.py)
+        ContentTransformer(
+            name="transform-module-gateway-subroute",
+            match=is_module_gateway_subroute_file,
+            transform=update_gateway_subroute_file,
         ),
-        # TODO: Register gateway route
     ],
     retries=0,
     upstream=validate_create_my_app_name_entity,
 )
 
+create_my_app_name_entity_migration = CmdTask(
+    name="create-my-app-name-entity-migration",
+    input=[
+        existing_module_input,
+        new_entity_input,
+    ],
+    env=EnvFile(path=os.path.join(APP_DIR, "template.env")),
+    cwd=APP_DIR,
+    cmd=[
+        ACTIVATE_VENV_SCRIPT,
+        Cmd(lambda ctx: set_create_migration_db_url_env(ctx.input.module)),
+        Cmd(lambda ctx: set_env("MY_APP_NAME_MODULES", ctx.input.module)),
+        Cmd(lambda ctx: cd_module_script(ctx.input.module)),
+        "alembic upgrade head",
+        Cmd(
+            'alembic revision --autogenerate -m "create_{to_snake_case(ctx.input.entity)}_table"',  # noqa
+        ),
+    ],
+    render_cmd=False,
+    retries=0,
+    upstream=[
+        prepare_venv,
+        scaffold_my_app_name_entity,
+    ],
+)
+
 add_my_app_name_entity = app_create_group.add_task(
     Task(
         name="add-my-app-name-entity",
         description="🏗️ Create new entity on a module",
-        upstream=scaffold_my_app_name_entity,
+        upstream=create_my_app_name_entity_migration,
         successor=format_my_app_name_code,
         retries=0,
     ),