zrb 0.28.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

zrb/builtin/git_subtree.py

@@ -0,0 +1,111 @@
+from zrb.builtin.git import git_commit
+from zrb.builtin.group import git_subtree_group
+from zrb.context.any_context import AnyContext
+from zrb.input.str_input import StrInput
+from zrb.task.make_task import make_task
+from zrb.util.cli.style import stylize_faint
+from zrb.util.git import get_repo_dir
+from zrb.util.git_subtree import add_subtree, load_config, pull_subtree, push_subtree
+
+
+@make_task(
+    name="git-add-subtree",
+    input=[
+        StrInput(
+            name="name", description="Subtree config name", prompt="Subtree config name"
+        ),
+        StrInput(
+            name="repo-url", description="Subtree repo URL", prompt="Subtree repo URL"
+        ),
+        StrInput(
+            name="repo-branch",
+            description="Subtree repo branch",
+            prompt="Subtree repo branch",
+        ),
+        StrInput(
+            name="repo-prefix",
+            description="Subtree repo prefix",
+            prompt="Subtree repo prefix",
+        ),
+    ],
+    description="➕ Add subtree",
+    upstream=git_commit,
+    group=git_subtree_group,
+    alias="add",
+)
+async def git_add_subtree(ctx: AnyContext):
+    ctx.print(stylize_faint("Get directory"))
+    repo_dir = await get_repo_dir(print_method=ctx.print)
+    ctx.print(stylize_faint("Add subtree"))
+    await add_subtree(
+        repo_dir=repo_dir,
+        name=ctx.input.name,
+        repo_url=ctx.input["repo-url"],
+        branch=ctx.input["repo-branch"],
+        prefix=ctx.input["repo-prefix"],
+        print_method=ctx.print,
+    )
+
+
+@make_task(
+    name="git-pull-subtree",
+    description="📥 Pull subtree",
+    upstream=git_commit,
+    group=git_subtree_group,
+    alias="pull",
+)
+async def git_pull_subtree(ctx: AnyContext):
+    ctx.print(stylize_faint("Get directory"))
+    repo_dir = await get_repo_dir(print_method=ctx.print)
+    config = load_config(repo_dir)
+    if not config.data:
+        raise ValueError("No subtree config found")
+    first_err: Exception | None = None
+    for name, detail in config.data.items():
+        try:
+            ctx.print(stylize_faint(f"Pull from subtree {name}"))
+            await pull_subtree(
+                repo_dir=repo_dir,
+                prefix=detail.prefix,
+                repo_url=detail.repo_url,
+                branch=detail.branch,
+                print_method=ctx.print,
+            )
+        except Exception as e:
+            if first_err is None:
+                first_err = e
+            ctx.log_error(e)
+    if first_err is not None:
+        raise first_err
+
+
+@make_task(
+    name="git-push-subtree",
+    description="📤 Push subtree",
+    upstream=git_commit,
+    group=git_subtree_group,
+    alias="push",
+)
+async def git_push_subtree(ctx: AnyContext):
+    ctx.print(stylize_faint("Get directory"))
+    repo_dir = await get_repo_dir(print_method=ctx.print)
+    config = load_config(repo_dir)
+    if not config.data:
+        raise ValueError("No subtree config found")
+    first_err: Exception | None = None
+    for name, detail in config.data.items():
+        try:
+            ctx.print(stylize_faint(f"Push to subtree {name}"))
+            await push_subtree(
+                repo_dir=repo_dir,
+                prefix=detail.prefix,
+                repo_url=detail.repo_url,
+                branch=detail.branch,
+                print_method=ctx.print,
+            )
+        except Exception as e:
+            if first_err is None:
+                first_err = e
+            ctx.log_error(e)
+    if first_err is not None:
+        raise first_err

zrb/builtin/group.py

@@ -1,48 +1,40 @@
-import sys
+from zrb.group.group import Group
+from zrb.runner.cli import cli
 
-from zrb.builtin.base64 import base64_group
-from zrb.builtin.devtool import devtool_group, devtool_install_group
-from zrb.builtin.docker import docker_group
-from zrb.builtin.env import env_group
-from zrb.builtin.explain import explain_group
-from zrb.builtin.git import git_group
-from zrb.builtin.md5 import md5_group
-from zrb.builtin.process import process_group, process_pid_group
-from zrb.builtin.project import (
-    project_add_app_group,
-    project_add_fastapp_group,
-    project_add_group,
-    project_add_task_group,
-    project_group,
-)
-from zrb.builtin.ubuntu import ubuntu_group, ubuntu_install_group
-from zrb.helper.accessories.color import colored
-
-_DEPRECATION_WARNING = """
-DEPRECATED: zrb.builtin.group
-User zrb.builtin instead
+base64_group = cli.add_group(Group(name="base64", description="📄 Base64 operations"))
+random_group = cli.add_group(Group(name="random", description="🔀 Random operation"))
+git_group = cli.add_group(Group(name="git", description="🌱 Git related commands"))
+git_branch_group = git_group.add_group(
+    Group(name="branch", description="🌿 Git branch related commands")
+)
+git_subtree_group = git_group.add_group(
+    Group(name="subtree", description="🌳 Git subtree related commands")
+)
+llm_group = cli.add_group(Group(name="llm", description="🤖 LLM operations"))
+md5_group = cli.add_group(Group(name="md5", description="🔢 Md5 operations"))
+python_group = cli.add_group(
+    Group(name="python", description="🐍 Python related commands")
+)
+todo_group = cli.add_group(Group(name="todo", description="✅ Todo management"))
 
-```python
-from zrb.builtin import project_group
-```
-"""
+shell_group = cli.add_group(
+    Group(name="shell", description="💬 Shell related commands")
+)
+shell_autocomplete_group: Group = shell_group.add_group(
+    Group(name="autocomplete", description="⌨️ Shell autocomplete related commands")
+)
 
-print(colored(_DEPRECATION_WARNING, color="red", attrs=["bold"]), file=sys.stderr)
+project_group = cli.add_group(
+    Group(name="project", description="📁 Project related commands")
+)
+add_to_project_group = project_group.add_group(
+    Group(name="add", description="➕ Add things to project")
+)
+add_fastapp_to_project_group = add_to_project_group.add_group(
+    Group(name="fastapp", description="🚀 Add Fastapp resources")
+)
 
-assert base64_group
-assert devtool_group
-assert devtool_install_group
-assert docker_group
-assert env_group
-assert explain_group
-assert git_group
-assert md5_group
-assert process_group
-assert process_pid_group
-assert project_group
-assert project_add_group
-assert project_add_app_group
-assert project_add_fastapp_group
-assert project_add_task_group
-assert ubuntu_group
-assert ubuntu_install_group
+setup_group = cli.add_group(Group(name="setup", description="🔧 Setup"))
+setup_latex_group = setup_group.add_group(
+    Group(name="latex", description="✍️ Setup LaTeX")
+)

zrb/builtin/llm/llm_chat.py

@@ -0,0 +1,127 @@
+import json
+import os
+from typing import Any
+
+from zrb.builtin.group import llm_group
+from zrb.builtin.llm.tool.api import get_current_location, get_current_weather
+from zrb.builtin.llm.tool.cli import run_shell_command
+from zrb.builtin.llm.tool.web import open_web_route, query_internet
+from zrb.config import (
+    LLM_ALLOW_ACCESS_INTERNET,
+    LLM_ALLOW_ACCESS_SHELL,
+    LLM_HISTORY_DIR,
+    LLM_MODEL,
+    LLM_SYSTEM_PROMPT,
+)
+from zrb.context.any_shared_context import AnySharedContext
+from zrb.input.bool_input import BoolInput
+from zrb.input.str_input import StrInput
+from zrb.input.text_input import TextInput
+from zrb.task.llm_task import LLMTask
+from zrb.util.file import read_file, write_file
+from zrb.util.string.conversion import to_pascal_case
+
+
+class PreviousSessionInput(StrInput):
+
+    def to_html(self, ctx: AnySharedContext) -> str:
+        name = self.name
+        description = self.description
+        default = self.get_default_str(ctx)
+        script = read_file(
+            file_path=os.path.join(os.path.dirname(__file__), "previous-session.js"),
+            replace_map={
+                "CURRENT_INPUT_NAME": name,
+                "CurrentPascalInputName": to_pascal_case(name),
+            },
+        )
+        return "\n".join(
+            [
+                f'<input name="{name}" placeholder="{description}" value="{default}" />',
+                f"<script>{script}</script>",
+            ]
+        )
+
+
+def _read_chat_conversation(ctx: AnySharedContext) -> list[dict[str, Any]]:
+    if ctx.input.start_new:
+        return []
+    previous_session_name = ctx.input.previous_session
+    if previous_session_name == "" or previous_session_name is None:
+        last_session_file_path = os.path.join(LLM_HISTORY_DIR, "last-session")
+        if os.path.isfile(last_session_file_path):
+            previous_session_name = read_file(last_session_file_path).strip()
+    conversation_file_path = os.path.join(
+        LLM_HISTORY_DIR, f"{previous_session_name}.json"
+    )
+    if not os.path.isfile(conversation_file_path):
+        return []
+    return json.loads(read_file(conversation_file_path))
+
+
+def _write_chat_conversation(
+    ctx: AnySharedContext, conversations: list[dict[str, Any]]
+):
+    os.makedirs(LLM_HISTORY_DIR, exist_ok=True)
+    current_session_name = ctx.session.name
+    conversation_file_path = os.path.join(
+        LLM_HISTORY_DIR, f"{current_session_name}.json"
+    )
+    write_file(conversation_file_path, json.dumps(conversations, indent=2))
+    last_session_file_path = os.path.join(LLM_HISTORY_DIR, "last-session")
+    write_file(last_session_file_path, current_session_name)
+
+
+llm_chat: LLMTask = llm_group.add_task(
+    LLMTask(
+        name="llm-chat",
+        input=[
+            StrInput(
+                "model",
+                description="LLM Model",
+                prompt="LLM Model",
+                default_str=LLM_MODEL,
+                allow_positional_parsing=False,
+            ),
+            TextInput(
+                "system-prompt",
+                description="System prompt",
+                prompt="System prompt",
+                default_str=LLM_SYSTEM_PROMPT,
+                allow_positional_parsing=False,
+            ),
+            BoolInput(
+                "start-new",
+                description="Start new conversation (LLM will forget everything)",
+                prompt="Start new conversation (LLM will forget everything)",
+                default_str="false",
+                allow_positional_parsing=False,
+            ),
+            TextInput("message", description="User message", prompt="Your message"),
+            PreviousSessionInput(
+                "previous-session",
+                description="Previous conversation session",
+                prompt="Previous conversation session (can be empty)",
+                allow_positional_parsing=False,
+                allow_empty=True,
+            ),
+        ],
+        conversation_history_reader=_read_chat_conversation,
+        conversation_history_writer=_write_chat_conversation,
+        description="Chat with LLM",
+        model="{ctx.input.model}",
+        system_prompt="{ctx.input['system-prompt']}",
+        message="{ctx.input.message}",
+        retries=0,
+    ),
+    alias="chat",
+)
+
+if LLM_ALLOW_ACCESS_SHELL:
+    llm_chat.add_tool(run_shell_command)
+
+if LLM_ALLOW_ACCESS_INTERNET:
+    llm_chat.add_tool(open_web_route)
+    llm_chat.add_tool(query_internet)
+    llm_chat.add_tool(get_current_location)
+    llm_chat.add_tool(get_current_weather)

zrb/builtin/llm/previous-session.js

@@ -0,0 +1,21 @@
+async function updatePreviousSession(event) {
+    const currentInput = event.target;
+    if (currentInput.name === "CURRENT_INPUT_NAME") {
+        return
+    }
+    const previousSessionInput = submitTaskForm.querySelector('[name="CURRENT_INPUT_NAME"]');
+    if (previousSessionInput) {
+        const currentSessionName = cfg.SESSION_NAME
+        previousSessionInput.value = currentSessionName;
+    }
+}
+
+document.getElementById("submit-task-form").querySelectorAll("input[name], textarea[name]").forEach((element) => {
+    element.addEventListener("input", updatePreviousSession);
+    element.addEventListener("keyup", updatePreviousSession);
+});
+
+document.getElementById("submit-task-form").querySelectorAll("select[name]").forEach((element) => {
+    element.addEventListener("change", updatePreviousSession);
+});
+

zrb/builtin/llm/tool/api.py

@@ -0,0 +1,29 @@
+import json
+from typing import Annotated, Literal
+
+import requests
+
+
+def get_current_location() -> (
+    Annotated[str, "JSON string representing latitude and longitude"]
+):  # noqa
+    """Get the user's current location."""
+    return json.dumps(requests.get("http://ip-api.com/json?fields=lat,lon").json())
+
+
+def get_current_weather(
+    latitude: float,
+    longitude: float,
+    temperature_unit: Literal["celsius", "fahrenheit"],
+) -> str:
+    """Get the current weather in a given location."""
+    resp = requests.get(
+        "https://api.open-meteo.com/v1/forecast",
+        params={
+            "latitude": latitude,
+            "longitude": longitude,
+            "temperature_unit": temperature_unit,
+            "current_weather": True,
+        },
+    )
+    return json.dumps(resp.json())

zrb/builtin/llm/tool/cli.py

@@ -0,0 +1,9 @@
+import subprocess
+
+
+def run_shell_command(command: str) -> str:
+    """Running an actual shell command on user's computer."""
+    output = subprocess.check_output(
+        command, shell=True, stderr=subprocess.STDOUT, text=True
+    )
+    return output

zrb/builtin/llm/tool/rag.py

@@ -0,0 +1,150 @@
+import hashlib
+import json
+import os
+import sys
+
+import ulid
+
+from zrb.config import (
+    RAG_CHUNK_SIZE,
+    RAG_EMBEDDING_MODEL,
+    RAG_MAX_RESULT_COUNT,
+    RAG_OVERLAP,
+)
+from zrb.util.cli.style import stylize_error, stylize_faint
+from zrb.util.file import read_file
+
+
+def create_rag_from_directory(
+    tool_name: str,
+    tool_description: str,
+    document_dir_path: str = "./documents",
+    model: str = RAG_EMBEDDING_MODEL,
+    vector_db_path: str = "./chroma",
+    vector_db_collection: str = "documents",
+    chunk_size: int = RAG_CHUNK_SIZE,
+    overlap: int = RAG_OVERLAP,
+    max_result_count: int = RAG_MAX_RESULT_COUNT,
+):
+    async def retrieve(query: str) -> str:
+        from chromadb import PersistentClient
+        from chromadb.config import Settings
+        from fastembed import TextEmbedding
+
+        embedding_model = TextEmbedding(model_name=model)
+        client = PersistentClient(
+            path=vector_db_path, settings=Settings(allow_reset=True)
+        )
+        collection = client.get_or_create_collection(vector_db_collection)
+
+        # Track file changes using a hash-based approach
+        hash_file_path = os.path.join(vector_db_path, "file_hashes.json")
+        previous_hashes = _load_hashes(hash_file_path)
+        current_hashes = {}
+
+        updated_files = []
+
+        for root, _, files in os.walk(document_dir_path):
+            for file in files:
+                file_path = os.path.join(root, file)
+                file_hash = _compute_file_hash(file_path)
+                relative_path = os.path.relpath(file_path, document_dir_path)
+                current_hashes[relative_path] = file_hash
+
+                if previous_hashes.get(relative_path) != file_hash:
+                    updated_files.append(file_path)
+
+        if updated_files:
+            print(
+                stylize_faint(f"Updating {len(updated_files)} changed files"),
+                file=sys.stderr,
+            )
+
+            for file_path in updated_files:
+                try:
+                    relative_path = os.path.relpath(file_path, document_dir_path)
+                    collection.delete(where={"file_path": relative_path})
+                    content = _read_file_content(file_path)
+                    file_id = ulid.new().str
+                    for i in range(0, len(content), chunk_size - overlap):
+                        chunk = content[i : i + chunk_size]
+                        if chunk:
+                            chunk_id = ulid.new().str
+                            print(
+                                stylize_faint(
+                                    f"Vectorizing {relative_path} chunk {chunk_id}"
+                                ),
+                                file=sys.stderr,
+                            )
+                            embedding_result = list(embedding_model.embed([chunk]))
+                            vector = embedding_result[0]
+                            collection.upsert(
+                                ids=[chunk_id],
+                                embeddings=[vector],
+                                documents=[chunk],
+                                metadatas={
+                                    "file_path": relative_path,
+                                    "file_id": file_id,
+                                },
+                            )
+                except Exception as e:
+                    print(
+                        stylize_error(f"Error processing {file_path}: {e}"),
+                        file=sys.stderr,
+                    )
+
+            _save_hashes(hash_file_path, current_hashes)
+        else:
+            print(
+                stylize_faint("No changes detected. Skipping database update."),
+                file=sys.stderr,
+            )
+
+        print(stylize_faint("Vectorizing query"), file=sys.stderr)
+        embedding_result = list(embedding_model.embed([query]))
+        query_vector = embedding_result[0]
+        print(stylize_faint("Searching documents"), file=sys.stderr)
+        results = collection.query(
+            query_embeddings=query_vector,
+            n_results=max_result_count,
+        )
+        return json.dumps(results)
+
+    retrieve.__name__ = tool_name
+    retrieve.__doc__ = tool_description
+    return retrieve
+
+
+def _compute_file_hash(file_path: str) -> str:
+    hash_md5 = hashlib.md5()
+    with open(file_path, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_md5.update(chunk)
+    return hash_md5.hexdigest()
+
+
+def _read_file_content(file_path: str) -> str:
+    if file_path.lower().endswith(".pdf"):
+        return _read_pdf(file_path)
+    return read_file(file_path)
+
+
+def _read_pdf(file_path: str) -> str:
+    import pdfplumber
+
+    with pdfplumber.open(file_path) as pdf:
+        return "\n".join(
+            page.extract_text() for page in pdf.pages if page.extract_text()
+        )
+
+
+def _load_hashes(file_path: str) -> dict:
+    if os.path.exists(file_path):
+        with open(file_path, "r") as f:
+            return json.load(f)
+    return {}
+
+
+def _save_hashes(file_path: str, hashes: dict):
+    with open(file_path, "w") as f:
+        json.dump(hashes, f)

zrb/builtin/llm/tool/web.py

@@ -0,0 +1,74 @@
+import json
+from typing import Annotated
+
+
+def open_web_route(url: str) -> str:
+    """Get content from a web page."""
+    import requests
+
+    response = requests.get(
+        url,
+        headers={
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"  # noqa
+        },
+    )
+    if response.status_code != 200:
+        raise Exception(
+            f"Error: Unable to retrieve search results (status code: {response.status_code})"  # noqa
+        )
+    return json.dumps(parse_html_text(response.text))
+
+
+def query_internet(
+    query: Annotated[str, "Search query"],
+    num_results: Annotated[int, "Search result count, by default 10"] = 10,
+) -> str:
+    """Search factual information from the internet by using Google."""
+    import requests
+
+    response = requests.get(
+        "https://google.com/search",
+        headers={
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"  # noqa
+        },
+        params={
+            "q": query,
+            "num": num_results,
+            "hl": "en",
+            "safe": "off",
+        },
+    )
+    if response.status_code != 200:
+        raise Exception(
+            f"Error: Unable to retrieve search results (status code: {response.status_code})"  # noqa
+        )
+    return json.dumps(parse_html_text(response.text))
+
+
+def parse_html_text(html_text: str) -> dict[str, str]:
+    from bs4 import BeautifulSoup
+
+    ignored_tags = [
+        "script",
+        "link",
+        "meta",
+        "style",
+        "code",
+        "footer",
+        "nav",
+        "header",
+        "aside",
+    ]
+    soup = BeautifulSoup(html_text, "html.parser")
+    links = []
+    for anchor in soup.find_all("a"):
+        if not anchor or "href" not in anchor.attrs:
+            continue
+        link: str = anchor["href"]
+        if link.startswith("#") or link.startswith("/"):
+            continue
+        links.append(link)
+    for tag in soup(ignored_tags):
+        tag.decompose()
+    html_text = soup.get_text(separator=" ", strip=True)
+    return {"content": html_text, "links_on_page": links}

zrb/builtin/md5.py

@@ -0,0 +1,36 @@
+from zrb.builtin.group import md5_group
+from zrb.context.any_context import AnyContext
+from zrb.input.str_input import StrInput
+from zrb.task.make_task import make_task
+
+
+@make_task(
+    name="hash-md5",
+    input=StrInput(name="text", description="Text", prompt="Text to encode"),
+    description="🧩 Hash text to md5",
+    group=md5_group,
+    alias="hash",
+)
+def hash_md5(ctx: AnyContext) -> str:
+    import hashlib
+
+    result = hashlib.md5(ctx.input.text.encode()).hexdigest()
+    ctx.print(result)
+    return result
+
+
+@make_task(
+    name="sum-md5",
+    input=StrInput(name="file", description="File name", prompt="File name"),
+    description="➕ Get md5 checksum of a file",
+    group=md5_group,
+    alias="sum",
+)
+def sum_md5(ctx: AnyContext) -> str:
+    import hashlib
+
+    with open(ctx.input.file, mode="rb") as file:
+        content = file.read()
+    result = hashlib.md5(content).hexdigest()
+    ctx.print(result)
+    return result

zrb/builtin/project/__init__.py

@@ -1,35 +0,0 @@
-from zrb.builtin.project._group import project_group
-from zrb.builtin.project.add import (
-    add_cmd_task,
-    add_docker_compose_task,
-    add_fastapp_application,
-    add_fastapp_crud,
-    add_fastapp_field,
-    add_fastapp_module,
-    add_plugin,
-    add_project_tasks,
-    add_python_app,
-    add_python_task,
-    project_add_app_group,
-    project_add_fastapp_group,
-    project_add_group,
-    project_add_task_group,
-)
-from zrb.builtin.project.create import create_project
-
-assert project_group
-assert project_add_group
-assert project_add_app_group
-assert project_add_fastapp_group
-assert project_add_task_group
-assert create_project
-assert add_plugin
-assert add_project_tasks
-assert add_cmd_task
-assert add_docker_compose_task
-assert add_fastapp_application
-assert add_fastapp_crud
-assert add_fastapp_module
-assert add_fastapp_field
-assert add_python_task
-assert add_python_app