hyperpocket 0.3.7__py3-none-any.whl → 0.4.1__py3-none-any.whl

hyperpocket/repository/lock.py

@@ -1,240 +0,0 @@
-import abc
-import pathlib
-import shutil
-from typing import Optional, Tuple, ClassVar
-
-import git
-from pydantic import BaseModel, Field
-from pydantic.fields import ModelPrivateAttr
-
-from hyperpocket.config import pocket_logger, settings
-
-
-class Lock(BaseModel, abc.ABC):
-    tool_source: str = None
-
-    @abc.abstractmethod
-    def __str__(self):
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def key(self) -> tuple[str, ...]:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def sync(self, **kwargs):
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    def toolpkg_path(self) -> pathlib.Path:
-        raise NotImplementedError
-
-    def eject_to_path(self, dest_path: pathlib.Path, src_sub_path: str = None):
-        ## local locks are already tracked by git
-        raise NotImplementedError
-
-
-class LocalLock(Lock):
-    tool_source: str = Field(default="local")
-    tool_path: str
-
-    def __init__(self, tool_path: str):
-        super().__init__(
-            tool_source="local", tool_path=str(pathlib.Path(tool_path).resolve())
-        )
-
-    def __str__(self):
-        return f"local\t{self.tool_path}"
-
-    def key(self):
-        return self.tool_source, self.tool_path.rstrip("/")
-
-    def sync(self, **kwargs):
-        pocket_logger.info(f"Syncing path: {self.tool_path} ...")
-        pkg_path = self.toolpkg_path()
-        if pkg_path.exists():
-            shutil.rmtree(pkg_path)
-        shutil.copytree(self.tool_path, pkg_path)
-
-    def toolpkg_path(self) -> pathlib.Path:
-        pocket_pkgs = settings.toolpkg_path
-        return pocket_pkgs / "local" / self.tool_path[1:]
-
-
-class GitLock(Lock):
-    _remote_cache: ClassVar[dict[str, dict[str, str]]]
-    tool_source: str = "git"
-    repository_url: str
-    git_ref: str
-    ref_sha: Optional[str] = None
-
-    def __str__(self):
-        return f"git\t{self.repository_url}\t{self.git_ref}\t{self.ref_sha}"
-
-    def key(self):
-        return self.tool_source, self.repository_url.rstrip("/"), self.git_ref
-
-    def toolpkg_path(self) -> pathlib.Path:
-        if not self.ref_sha:
-            raise ValueError("ref_sha is not set")
-        cleansed_url = self.repository_url
-        if self.repository_url.startswith("http://"):
-            cleansed_url = self.repository_url[7:]
-        elif self.repository_url.startswith("https://"):
-            cleansed_url = self.repository_url[8:]
-        elif self.repository_url.startswith("git@"):
-            cleansed_url = self.repository_url[4:]
-        return settings.toolpkg_path / cleansed_url / self.ref_sha
-
-    def sync(self, force_update: bool = False, **kwargs):
-        """
-        Synchronize the local git repository with the target remote branch.
-
-        1. Check if the SHA of the target ref in the remote repository matches the current local SHA.
-        2. If they do not match, fetch the target ref from the remote repository and do a hard reset
-           to align the local repository with the remote version.
-        """
-        try:
-            pocket_logger.info(
-                f"Syncing git: {self.repository_url} @ ref: {self.git_ref} ..."
-            )
-
-            # get new sha from refs
-            new_sha = self._get_new_sha_if_exists_in_remote()
-            if new_sha is None:
-                raise ValueError(
-                    f"Could not find ref {self.git_ref} in {self.repository_url}"
-                )
-
-            # check self.ref_sha should be updated
-            if self.ref_sha != new_sha:
-                if force_update or self.ref_sha is None:
-                    self.ref_sha = new_sha
-
-            # make pkg_version_path dir if not exists
-            pkg_version_path = self.toolpkg_path()
-            if not pkg_version_path.exists():
-                pkg_version_path.mkdir(parents=True)
-
-            # init git repo in local and set origin url
-            repo = git.Repo.init(pkg_version_path)
-            try:
-                remote = repo.remote("origin")
-                remote.set_url(self.repository_url)
-            except ValueError:
-                remote = repo.create_remote("origin", self.repository_url)
-
-            # check current local commit include new_sha
-            # if not included, fetch and do hard reset
-            exist_sha = None
-            try:
-                exist_sha = repo.head.commit.hexsha
-            except ValueError:
-                pass
-            if exist_sha is None or exist_sha != self.ref_sha:
-                remote.fetch(depth=1, refspec=self.ref_sha)
-                repo.git.checkout(new_sha)
-                repo.git.reset("--hard", new_sha)
-                repo.git.clean("-fd")
-        except Exception as e:
-            pocket_logger.error(
-                f"failed to sync git: {self.repository_url} @ ref: {self.git_ref}. reason : {e}"
-            )
-            raise e
-
-    def _get_new_sha_if_exists_in_remote(self):
-        """
-        get new sha in refs
-        First, check remote sha is matched to saved ref_sha
-        Second, check remote ref name is matched to saved ref name
-        Third, check local ref name is matched to saved ref name
-        And last, check tag ref name is matched to saved ref name
-        """
-        refs = git.cmd.Git().ls_remote(self.repository_url)
-
-        new_sha = None
-        for r in refs.split("\n"):
-            sha, ref = r.split("\t")
-            if sha == self.ref_sha:
-                new_sha = sha
-                break
-            elif ref == self.git_ref:
-                new_sha = sha
-                break
-            elif ref == f"refs/heads/{self.git_ref}":
-                new_sha = sha
-                break
-            elif ref == f"refs/tags/{self.git_ref}":
-                new_sha = sha
-                break
-        return new_sha
-
-    @classmethod
-    def get_git_branches(cls, repo_url):
-        if not hasattr(cls, "_remote_cache"):
-            cls._remote_cache = {}
-        if cls._remote_cache.get(repo_url) is None:
-            ls_lists = git.cmd.Git().ls_remote(repo_url)
-
-            branches = {}
-            for line in ls_lists.split("\n"):
-                sha, ref = line.split("\t")
-                if ref.startswith("refs/heads/"):
-                    branch_name = ref.replace("refs/heads/", "")
-                    branches[branch_name] = sha
-            cls._remote_cache[repo_url] = branches
-        return cls._remote_cache[repo_url]
-
-    @classmethod
-    def parse_repo_url(cls, repo_url: str) -> Tuple[str, str, str]:
-        """
-        Parses a GitHub repository URL with optional branch and path information.
-
-        Returns:
-            Tuple[str, str, str]: base_repo, branch_name, directory_path
-        """
-        if not repo_url.startswith("https://github.com/"):
-            raise AttributeError("Only GitHub URLs are supported")
-
-        # Remove the base URL and split the path
-        repo_path = repo_url.removeprefix("https://github.com/")
-        repo_path_list = repo_path.split("/")
-
-        # Check if the URL contains 'tree' (indicating branch and sub-path information)
-        if "tree" not in repo_path_list:
-            # If no 'tree', return the full repository URL
-            return repo_url, "HEAD", ""
-
-        # Parse base repo URL and remaining path
-        tree_index = repo_path_list.index("tree")
-        base_repo = f"https://github.com/{'/'.join(repo_path_list[:tree_index])}"
-        sub_path = repo_path_list[tree_index + 1 :]
-
-        # Fetch branch information
-        branches = cls.get_git_branches(base_repo)
-
-        # Find branch and sub-directory path
-        for idx in range(1, len(sub_path) + 1):
-            branch_name = "/".join(sub_path[:idx])
-            if branch_name in branches:
-                directory_path = (
-                    "/".join(sub_path[idx:]) if idx < len(sub_path) else None
-                )
-                return base_repo, branch_name, directory_path
-
-        # If no valid branch is found, raise an error
-        raise ValueError("Branch not found in repository")
-
-    def eject_to_path(self, dest_path: pathlib.Path, src_sub_path: str = None):
-        # clone the git repository to the target path
-        pocket_logger.info(
-            f"Ejecting git: {self.repository_url} @ ref: {self.git_ref} source in path: {src_sub_path} to {dest_path} ..."
-        )
-        if dest_path.exists():
-            shutil.rmtree(dest_path)
-
-        if src_sub_path:
-            src_path = self.toolpkg_path() / src_sub_path
-        else:
-            src_path = self.toolpkg_path()
-        shutil.copytree(src_path, dest_path)

hyperpocket/repository/lockfile.py

@@ -1,62 +0,0 @@
-import pathlib
-from concurrent.futures.thread import ThreadPoolExecutor
-
-from hyperpocket.repository.lock import GitLock, LocalLock, Lock
-
-
-class Lockfile:
-    path: pathlib.Path = None
-    locks: dict[tuple, Lock] = None
-    referenced_locks: set[tuple] = None
-
-    def __init__(self, path: pathlib.Path):
-        self.path = path
-        self.locks = {}
-        self.referenced_locks = set()
-        if self.path.exists():
-            with open(self.path, "r") as f:
-                for line in f:
-                    split = line.strip().split("\t")
-                    source = split[0]
-                    if source == "local":
-                        lock = LocalLock(tool_path=split[1])
-                    elif source == "git":
-                        lock = GitLock(
-                            repository_url=split[1],
-                            git_ref=split[2],
-                            ref_sha=split[3],
-                        )
-                    else:
-                        raise ValueError(f"Unknown tool source: {source}")
-                    self.locks[lock.key()] = lock
-        else:
-            self.path.touch()
-
-    def add_lock(self, lock: Lock):
-        if lock.key() not in self.locks:
-            self.locks[lock.key()] = lock
-        self.referenced_locks.add(lock.key())
-
-    def remove_lock(self, key: tuple[str, ...]):
-        self.locks.pop(key)
-        if key in self.referenced_locks:
-            self.referenced_locks.remove(key)
-
-    def get_lock(self, key: tuple[str, ...]):
-        return self.locks[key]
-
-    def sync(self, force_update: bool, referenced_only: bool = False):
-        if referenced_only:
-            locks = [self.get_lock(key) for key in self.referenced_locks]
-        else:
-            locks = list(self.locks.values())
-        with ThreadPoolExecutor(
-            max_workers=min(len(locks) + 1, 100), thread_name_prefix="repository_loader"
-        ) as executor:
-            executor.map(lambda lock: lock.sync(force_update=force_update), locks)
-        self.write()
-
-    def write(self):
-        with open(self.path, "w") as f:
-            for lock in self.locks.values():
-                f.write(str(lock) + "\n")

hyperpocket/server/tool/__init__.py

@@ -1,10 +0,0 @@
-from fastapi import APIRouter
-
-from hyperpocket.server.tool.wasm import wasm_tool_router
-
-tool_router = APIRouter(
-    prefix="/tools",
-)
-tool_router.include_router(wasm_tool_router)
-
-__all__ = ["tool_router"]

hyperpocket/server/tool/dto/script.py

@@ -1,33 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field
-
-from hyperpocket.tool.wasm.script import ScriptFileNode
-
-
-class Script(BaseModel):
-    id: str = Field(alias="id")
-    tool_id: str = Field(alias="tool_id")
-
-
-class ScriptResult(BaseModel):
-    stdout: Optional[str] = Field(alias="stdout", default=None)
-    stderr: Optional[str] = Field(alias="stderr", default=None)
-    error: Optional[str] = Field(alias="error", default=None)
-
-
-class ScriptFileTree(BaseModel):
-    tree: dict[str, ScriptFileNode] = Field(alias="tree")
-
-
-class ScriptEntrypoint(BaseModel):
-    package_name: Optional[str] = Field(alias="package_name")
-    entrypoint: str = Field(alias="entrypoint")
-
-
-class ScriptEncodedFile(BaseModel):
-    encoded_file: str = Field(alias="encoded_file")
-
-
-class ScriptFileRequest(BaseModel):
-    path: str = Field(alias="path")

hyperpocket/server/tool/wasm.py

@@ -1,46 +0,0 @@
-from fastapi import APIRouter
-from fastapi.responses import FileResponse, HTMLResponse
-
-from hyperpocket.futures import FutureStore
-from hyperpocket.server.tool.dto import script as scriptdto
-from hyperpocket.tool.wasm.script import ScriptStore
-
-wasm_tool_router = APIRouter(prefix="/wasm")
-
-
-@wasm_tool_router.get("/scripts/{script_id}/browse", response_class=HTMLResponse)
-async def browse_script_page(script_id: str):
-    html = ScriptStore.get_script(script_id).rendered_html
-    return HTMLResponse(content=html)
-
-
-@wasm_tool_router.post("/scripts/{script_id}/done")
-async def done_script_page(
-    script_id: str, req: scriptdto.ScriptResult
-) -> scriptdto.ScriptResult:
-    FutureStore.resolve_future(
-        script_id, {"stdout": req.stdout, "stderr": req.stderr, "error": req.error}
-    )
-    return req
-
-
-@wasm_tool_router.get("/scripts/{script_id}/file_tree")
-async def get_file_tree(script_id: str) -> scriptdto.ScriptFileTree:
-    script = ScriptStore.get_script(script_id)
-    return scriptdto.ScriptFileTree(tree=script.load_file_tree())
-
-
-@wasm_tool_router.get("/scripts/{script_id}/entrypoint")
-async def get_entrypoint(script_id: str) -> scriptdto.ScriptEntrypoint:
-    script = ScriptStore.get_script(script_id)
-    package_name = script.package_name
-    entrypoint = f"/tools/wasm/scripts/{script_id}/file/{script.entrypoint}"
-    return scriptdto.ScriptEntrypoint(package_name=package_name, entrypoint=entrypoint)
-
-
-@wasm_tool_router.get(
-    "/scripts/{script_id}/file/{file_name}", response_class=FileResponse
-)
-async def get_dist_file(script_id: str, file_name: str):
-    script = ScriptStore.get_script(script_id)
-    return FileResponse(script.dist_file_path(file_name))

hyperpocket/tool/wasm/README.md

@@ -1,166 +0,0 @@
-# Wasm(WebAssembly)
-
----
-
-## How To Use
-
-### plain url
-
-load the tool by providing a GitHub URL
-
-```python
-from hyperpocket import Pocket
-
-pocket = Pocket(tools=[
-    'https://github.com/your-organization/your-repository/tree/main',
-])
-```
-
-- in this case, read tool code from `main` branch by default
-
-### from_git
-
-- using `from_git`, you can specify branch and tool code path
-
-```python
-from hyperpocket import Pocket
-from hyperpocket.tool import from_git
-
-pocket = Pocket(tools=[
-    from_git("https://github.com/your-organization/your-repository", "branch-name", "your/tool/code/path"),
-])
-```
-
-### from_local
-
-- using `from_local`, you can load a tool by specifying a local path
-
-```python
-from hyperpocket import Pocket
-from hyperpocket.tool import from_local
-
-pocket = Pocket(tools=[
-    from_local("your/local/tool/code/path")
-])
-```
-
-## WasmToolRequest
-
-The class to initiate WasmTool
-
-WasmToolRequest offer those build method:
-
-- `from_local`
-- `from_git`
-
-Each `WasmToolRequest` has it's own `Lock` object
-
-Those `Lock` object is managed by `Lockfile` in the `Pocket` class
-
-```mermaid
-classDiagram
-    class WasmToolRequest {
-        +Lock lock
-        + from_git()
-        + from_local()
-    }
-
-    class Lock{
-        +key()
-        +sync()
-    }
-
-    class GitLock{
-        +key()
-        +sync()
-    }
-
-    class LocalLock{
-        +key()
-        +sync()
-    }
-
-    class Lockfile {
-        +dict[tuple,Lock] locks
-
-        + add_lock()
-        + get_lock()
-        + sync()
-        + write()
-    }
-
-    ToolRequest <|.. WasmToolRequest : Implementation
-    Lock o-- WasmToolRequest : 1..1
-    Lock <|.. GitLock : Implementation
-    Lock <|.. LocalLock : Implementation
-    Lock o-- Lockfile : n..1
-    Lockfile o-- Pocket : 1..1
-```
-
-### Inject tool variables
-
-If the user specifies [tool_vars] in the `config.toml` of the tool's repository, which is allowed to be injected dynamically when the user develops an agent, it can be injected through the following steps.
-
-```toml
-# config.toml of a tool
-
-[tool_vars]
-config1 = "config1"
-config2 = "config2"
-```
-
-1. Injecting tool_vars when importing tool in code.
-
-```python
-from_git('https://github.com/your-organization/your-repository/tree/main',tool_vars = {
-    "config1": "modified_config1"
-})
-```
-
-2. Injecting tool_vars by settings.toml
-   Hyperpocket checks the `settings.toml` from the agent code directory.
-
-## WasmTool
-
-```python
-class WasmTool(Tool):
-    _invoker: WasmInvoker = None
-    pkg_lock: Lock = None
-    rel_path: str
-    runtime: ScriptRuntime = None
-    json_schema: Optional[dict] = None
-    readme: Optional[str] = None
-```
-
-- `_invoker`: A class for executing WASM.
-- `pkg_lock`: The lock class for the tool.
-- Used for determining the package path where the current WASM code is stored.
-- `rel_path`: The relative path to the location of the WASM code within the package.
-- `runtime`: The runtime language of the WASM code.
-- `json_schema`: The JSON schema for the WASM tool.
-- Information read from schema.json.
-- `readme`: The README information.
-
-## WasmInvoker
-
-A class for running actual WASM
-
-1. Combines runtime information, authentication details, and body content to render HTML.
-2. Stores the rendered HTML in memory.
-3. Launches a browser to execute the WASM.
-4. The browser sends a request to the server to retrieve the rendered HTML.
-5. Executes code specific to each runtime within the HTML and returns the result.
-
-```mermaid
-sequenceDiagram
-    participant WasmTool as WasmTool (Includes Server)
-    participant Browser as Browser (Executes WASM Runtime)
-
-
-    WasmTool->>WasmTool: Render HTML and Store in Memory
-    WasmTool->>Browser: Launch Browser
-    Browser->>WasmTool: Request Rendered HTML
-    WasmTool->>Browser: Provide Rendered HTML
-    Browser->>Browser: Execute Injected WASM Runtime Script
-    Browser->>WasmTool: Return Execution Result
-```

hyperpocket/tool/wasm/__init__.py

@@ -1,3 +0,0 @@
-from hyperpocket.tool.wasm.tool import WasmTool
-
-__all__ = ["WasmTool"]

hyperpocket/tool/wasm/browser.py

@@ -1,63 +0,0 @@
-import asyncio
-
-from playwright.async_api import (
-    BrowserContext,
-    Page,
-    Playwright,
-    Route,
-    async_playwright,
-)
-
-
-class InvokerBrowser(object):
-    _instance: "InvokerBrowser" = None
-    _lock = asyncio.Lock()
-    playwright: Playwright
-    browser_context: BrowserContext
-
-    def __init__(self):
-        raise RuntimeError("Use InvokerBrowser.get_instance() instead")
-
-    async def _async_init(self):
-        self.playwright = await async_playwright().start()
-        self.browser_context = await self.playwright.chromium.launch_persistent_context(
-            headless=True,
-            args=[
-                "--disable-web-security=True",
-            ],
-            user_data_dir="/tmp/chrome_dev_user",
-        )
-
-    @classmethod
-    async def get_instance(cls):
-        if not cls._instance:
-            async with cls._lock:
-                if cls._instance is None:
-                    instance = cls.__new__(cls)
-                    await instance._async_init()
-                    cls._instance = instance
-        return cls._instance
-
-    async def new_page(self) -> Page:
-        page = await self.browser_context.new_page()
-
-        async def _hijack_route(route: Route):
-            response = await route.fetch()
-            body = await response.body()
-            await route.fulfill(
-                response=response,
-                body=body,
-                headers={
-                    **response.headers,
-                    "Cross-Origin-Opener-Policy": "same-origin",
-                    "Cross-Origin-Embedder-Policy": "require-corp",
-                    "Cross-Origin-Resource-Policy": "cross-origin",
-                },
-            )
-
-        await page.route("**/*", _hijack_route)
-        return page
-
-    async def teardown(self):
-        await self.browser_context.close()
-        await self.playwright.stop()

hyperpocket/tool/wasm/invoker.py

@@ -1,41 +0,0 @@
-import asyncio
-import json
-import uuid
-from typing import Any
-from urllib.parse import urljoin
-
-from hyperpocket.config import config
-from hyperpocket.futures import FutureStore
-from hyperpocket.tool.wasm.browser import InvokerBrowser
-from hyperpocket.tool.wasm.script import Script, ScriptRuntime, ScriptStore
-from hyperpocket.tool.wasm.templates import render
-
-
-class WasmInvoker(object):
-    def invoke(
-        self, tool_path: str, runtime: ScriptRuntime, body: Any, envs: dict, **kwargs
-    ) -> str:
-        loop = asyncio.get_running_loop()
-        return loop.run_until_complete(
-            self.ainvoke(tool_path, runtime, body, envs, **kwargs)
-        )
-
-    async def ainvoke(
-        self, tool_path: str, runtime: ScriptRuntime, body: Any, envs: dict, **kwargs
-    ) -> str:
-        uid = str(uuid.uuid4())
-        html = render(runtime.value, uid, envs, json.dumps(body))
-        script = Script(
-            id=uid, tool_path=tool_path, rendered_html=html, runtime=runtime
-        )
-        ScriptStore.add_script(script=script)
-        future_data = FutureStore.create_future(uid=uid)
-        browser = await InvokerBrowser.get_instance()
-        page = await browser.new_page()
-        url = urljoin(
-            config().internal_base_url + "/", f"tools/wasm/scripts/{uid}/browse"
-        )
-        await page.goto(url)
-        stdout = await future_data.future
-        await page.close()
-        return stdout