hyperpocket 0.0.1__py3-none-any.whl

hyperpocket/tool/wasm/browser.py

@@ -0,0 +1,63 @@
+import asyncio
+
+from playwright.async_api import async_playwright, Page, Playwright, BrowserContext, Route
+
+
+class InvokerBrowser(object):
+    _instance: 'InvokerBrowser' = None
+    _lock = asyncio.Lock()
+    playwright: Playwright
+    browser_context: BrowserContext
+
+    def __init__(self):
+        raise RuntimeError("Use InvokerBrowser.get_instance() instead")
+
+    async def _async_init(self):
+        # false only in dev
+        # TODO(moon.dev) : load from config by environment
+        import os
+        pocket_env = os.getenv("POCKET_ENV", "DEVELOPMENT")
+        is_headless = False if pocket_env == "DEVELOPMENT" else True
+
+        self.playwright = await async_playwright().start()
+        self.browser_context = await self.playwright.chromium.launch_persistent_context(
+            headless=is_headless,
+            args=[
+                '--disable-web-security=True',
+            ],
+            user_data_dir='/tmp/chrome_dev_user',
+        )
+
+    @classmethod
+    async def get_instance(cls):
+        if not cls._instance:
+            async with cls._lock:
+                if cls._instance is None:
+                    instance = cls.__new__(cls)
+                    await instance._async_init()
+                    cls._instance = instance
+        return cls._instance
+
+    async def new_page(self) -> Page:
+        page = await self.browser_context.new_page()
+
+        async def _hijack_route(route: Route):
+            response = await route.fetch()
+            body = await response.body()
+            await route.fulfill(
+                response=response,
+                body=body,
+                headers={
+                    **response.headers,
+                    'Cross-Origin-Opener-Policy': 'same-origin',
+                    'Cross-Origin-Embedder-Policy': 'require-corp',
+                    'Cross-Origin-Resource-Policy': 'cross-origin',
+                }
+            )
+
+        await page.route('**/*', _hijack_route)
+        return page
+
+    async def teardown(self):
+        await self.browser_context.close()
+        await self.playwright.stop()

hyperpocket/tool/wasm/invoker.py

@@ -0,0 +1,41 @@
+import asyncio
+import json
+import uuid
+from typing import Any
+from urllib.parse import urljoin
+
+from hyperpocket.config import config
+from hyperpocket.futures import FutureStore
+from hyperpocket.tool.wasm.browser import InvokerBrowser
+from hyperpocket.tool.wasm.script import ScriptRuntime, ScriptStore, Script
+from hyperpocket.tool.wasm.templates import render
+
+
+class WasmInvoker(object):
+    def invoke(self,
+               tool_path: str,
+               runtime: ScriptRuntime,
+               body: Any,
+               envs: dict,
+               **kwargs) -> str:
+        loop = asyncio.get_running_loop()
+        return loop.run_until_complete(self.ainvoke(tool_path, runtime, body, envs, **kwargs))
+
+    async def ainvoke(self,
+                      tool_path: str,
+                      runtime: ScriptRuntime,
+                      body: Any,
+                      envs: dict,
+                      **kwargs) -> str:
+        uid = str(uuid.uuid4())
+        html = render(runtime.value, uid, envs, json.dumps(body))
+        script = Script(id=uid, tool_path=tool_path, rendered_html=html, runtime=runtime)
+        ScriptStore.add_script(script=script)
+        future_data = FutureStore.create_future(uid=uid)
+        browser = await InvokerBrowser.get_instance()
+        page = await browser.new_page()
+        url = urljoin(config.internal_base_url + '/', f'tools/wasm/scripts/{uid}/browse')
+        await page.goto(url)
+        stdout = await future_data.future
+        await page.close()
+        return stdout

hyperpocket/tool/wasm/script.py

@@ -0,0 +1,82 @@
+import base64
+import enum
+import pathlib
+from typing import Optional
+
+from pydantic import BaseModel
+
+
+class ScriptRuntime(enum.Enum):
+    Node = "node"
+    Python = "python"
+    Wasm = "wasm"
+
+_RuntimePackageFiles = {
+    ScriptRuntime.Node: ["dist/index.js"],
+    ScriptRuntime.Python: ["main.py", "requirements.txt"],
+    ScriptRuntime.Wasm: ["dist/index.wasm"],
+}
+
+class ScriptFileNodeContent(BaseModel):
+    contents: str
+
+class ScriptFileNode(BaseModel):
+    directory: Optional[dict[str, 'ScriptFileNode']] = None
+    file: Optional[ScriptFileNodeContent] = None
+    
+    @classmethod
+    def create_file_tree(cls, path: str, contents: str) -> dict[str, 'ScriptFileNode']:
+        path_split = path.split("/")
+        if len(path_split) == 1:
+            return {path_split[0]: ScriptFileNode(file=ScriptFileNodeContent(contents=contents))}
+        node = cls.create_file_tree('/'.join(path_split[1:]), contents)
+        return {path_split[0]: ScriptFileNode(directory=node)}
+    
+    @staticmethod
+    def merge(a: dict[str, 'ScriptFileNode'], b: [str, 'ScriptFileNode']) -> dict[str, 'ScriptFileNode']:
+        for k, v in b.items():
+            if k in a:
+                if a[k].directory and v.directory:
+                    a[k].directory = ScriptFileNode.merge(a[k].directory, v.directory)
+                elif a[k].file and v.file:
+                    a[k].file = v.file
+            else:
+                a[k] = v
+        return a
+
+
+class Script(BaseModel):
+    id: str
+    tool_path: str
+    rendered_html: str
+    runtime: ScriptRuntime
+    
+    def load_file_tree(self) -> dict[str, ScriptFileNode]:
+        relpaths = _RuntimePackageFiles[self.runtime]
+        file_tree = dict()
+        for p in relpaths:
+            filepath = pathlib.Path(self.tool_path) / p
+            with filepath.open("r") as f:
+                contents = f.read().encode('utf-8')
+                encoded_bytes = base64.b64encode(contents)
+                encoded_str = encoded_bytes.decode()
+                file_tree = ScriptFileNode.merge(file_tree, ScriptFileNode.create_file_tree(p, encoded_str))
+        return file_tree
+                
+
+class _ScriptStore(object):
+    scripts: dict[str, Script] = {}
+
+    def __init__(self):
+        self.rendered_html = {}
+
+    def add_script(self, script: Script):
+        if script.id in self.scripts:
+            raise ValueError("Script id already exists")
+        self.scripts[script.id] = script
+    
+    def get_script(self, script_id: str) -> Script:
+        # ValueError exception is intentional
+        return self.scripts[script_id]
+
+ScriptStore = _ScriptStore()

hyperpocket/tool/wasm/templates/__init__.py

@@ -0,0 +1,28 @@
+import base64
+import json
+
+from jinja2 import Environment, DictLoader
+
+from hyperpocket.tool.wasm.templates.node import node_template
+from hyperpocket.tool.wasm.templates.python import python_template
+
+TemplateEnvironments = Environment(
+    loader=DictLoader({
+        'python.html': python_template,
+        'node.html': node_template,
+    }),
+    autoescape=False
+)
+
+
+def render(language: str, script_id: str, env: dict[str, str], body: str, **kwargs) -> str:
+    env_json = json.dumps(env)
+    template = TemplateEnvironments.get_template(f'{language.lower()}.html')
+    body_bytes = body.encode('utf-8')
+    body_b64_bytes = base64.b64encode(body_bytes)
+    body_b64 = body_b64_bytes.decode('ascii')
+    return template.render(**{
+                                 'SCRIPT_ID': script_id,
+                                 'ENV_JSON': env_json,
+                                 'BODY_JSON_B64': body_b64,
+                             } | kwargs)

hyperpocket/tool/wasm/templates/node.py

@@ -0,0 +1,87 @@
+node_template = '''
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>PyScript Offline</title>
+</head>
+<body>
+<script type="module">
+    function loadConfig() {
+        globalThis.toolConfigs = {
+            envs: `{{ ENV_JSON }}`,
+            body: `{{ BODY_JSON_B64 }}`,
+            scriptID: `{{ SCRIPT_ID }}`
+        }
+    }
+    import { WebContainer } from 'https://esm.run/@webcontainer/api@1.5.1';
+    function decodeContent(content) {
+        return Uint8Array.from(atob(content), c => c.charCodeAt(0));
+    }
+    function decodeFileTree(filetree) {
+        const decoded = {};
+        for (const [key, value] of Object.entries(filetree)) {
+            if (value.file) {
+                decoded[key] = {
+                    file: {
+                        contents: decodeContent(value.file.contents)
+                    }
+                }
+            } else if (value.directory) {
+                decoded[key] = {
+                    directory: decodeFileTree(value.directory)
+                }
+            } else if (value.symlink) {
+                decoded[key] = {
+                    symlink: value.symlink
+                }
+            }
+        }
+        return decoded;
+    }
+    async function main() {
+        loadConfig();
+        const b64FilesResp = await fetch(`/tools/wasm/scripts/${globalThis.toolConfigs.scriptID}/file_tree`);
+        const b64Files = await b64FilesResp.json();
+        const files = decodeFileTree(b64Files.tree);
+        const webcontainer = await WebContainer.boot();
+        
+        await webcontainer.mount(files)
+        const envs = JSON.parse(globalThis.toolConfigs.envs)
+        envs['DEPLOYED'] = 'true'
+        const runProcess = await webcontainer.spawn('node', ['dist/index.js'], {
+            output: true,
+            env: envs,
+        });
+        const stdin = runProcess.input.getWriter();
+        const decodedBytes = atob(globalThis.toolConfigs.body);
+        await (async () => {
+            await stdin.ready
+            await stdin.write(decodedBytes);
+        })()
+        let stdout = '';
+        runProcess.output.pipeTo(
+            new WritableStream({
+                write(chunk) {
+                    stdout += chunk;
+                }
+            })
+        )
+        await runProcess.exit;
+        if (stdout.startsWith(decodedBytes)) {
+            stdout = stdout.slice(decodedBytes);
+        }
+        await fetch(`/tools/wasm/scripts/${globalThis.toolConfigs.scriptID}/done`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json'
+            },
+            body: JSON.stringify({ stdout })
+        });
+    }
+    main();
+</script>
+</body>
+</html>
+'''

hyperpocket/tool/wasm/templates/python.py

@@ -0,0 +1,75 @@
+python_template = '''
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>PyScript Offline</title>
+    <script src="https://cdn.jsdelivr.net/pyodide/v0.26.4/full/pyodide.js"></script>
+</head>
+<body>
+<script type="module">
+    function loadConfig() {
+        globalThis.toolConfigs = {
+            envs: `{{ ENV_JSON }}`,
+            body: `{{ BODY_JSON_B64 }}`,
+            scriptID: `{{ SCRIPT_ID }}`
+        }
+    }
+    async function main() {
+        loadConfig();
+        const b64FilesResp = await fetch(`/tools/wasm/scripts/${globalThis.toolConfigs.scriptID}/file_tree`);
+        const b64Files = await b64FilesResp.json();
+        const code = atob(b64Files.tree["main.py"].file.contents);
+        const requirements = atob(b64Files.tree["requirements.txt"].file.contents);
+        
+        const pyodide = await loadPyodide({
+            env: JSON.parse(globalThis.toolConfigs.envs),
+        });
+        await pyodide.loadPackage("micropip");
+        await pyodide.loadPackage("ssl");
+        const micropip = pyodide.pyimport("micropip");
+        await micropip.install("pyodide-http")
+        const installation = requirements.split("\\n").map(async (req) => {
+            if (req) {
+                const pkg = req.split("==")[0];
+                await micropip.install(pkg);
+            }
+        });
+        await Promise.all(installation);
+        let emitted = false;
+        const decodedBytes = atob(globalThis.toolConfigs.body);
+        pyodide.setStdin({
+            stdin: () => {
+                if (emitted) {
+                    return null;
+                }
+                emitted = true;
+                return decodedBytes;
+            },
+            autoEOF: true,
+        })
+        let stdout = "";
+        pyodide.setStdout({
+            batched: (x) => { stdout += x; },
+        })
+        await pyodide.runPythonAsync(`
+import pyodide_http
+pyodide_http.patch_all()
+
+${code}
+`);
+        console.log(stdout)
+        await fetch(`/tools/wasm/scripts/${globalThis.toolConfigs.scriptID}/done`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json'
+            },
+            body: JSON.stringify({ stdout })
+        });
+    }
+    main();
+</script>
+</body>
+</html>
+'''

hyperpocket/tool/wasm/tool.py

@@ -0,0 +1,147 @@
+import json
+import pathlib
+from typing import Any, Optional
+
+import toml
+
+from hyperpocket.auth import AuthProvider
+from hyperpocket.config import pocket_logger
+from hyperpocket.repository import Lock, Lockfile
+from hyperpocket.repository.lock import GitLock, LocalLock
+from hyperpocket.tool import Tool, ToolRequest
+from hyperpocket.tool.tool import ToolAuth
+from hyperpocket.tool.wasm.invoker import WasmInvoker
+from hyperpocket.tool.wasm.script import ScriptRuntime
+
+
+class WasmToolRequest(ToolRequest):
+    lock: Lock
+    rel_path: str
+
+    def __init__(self, lock: Lock, rel_path: str):
+        self.lock = lock
+        self.rel_path = rel_path
+
+    def __str__(self):
+        return f"ToolRequest(lock={self.lock}, rel_path={self.rel_path})"
+
+
+def from_local(path: str) -> WasmToolRequest:
+    return WasmToolRequest(LocalLock(path), "")
+
+
+def from_git(repository: str, ref: str, rel_path: str) -> WasmToolRequest:
+    return WasmToolRequest(GitLock(repository_url=repository, git_ref=ref), rel_path)
+
+
+def from_github(owner: str, repo: str, ref: str, rel_path: str) -> WasmToolRequest:
+    repository = f"https://github.com/{owner}/{repo}"
+    return from_git(repository, ref, rel_path)
+
+
+class WasmTool(Tool):
+    """
+    WasmTool is Tool executing local python method.
+    """
+
+    _invoker: WasmInvoker = None
+    pkg_lock: Lock = None
+    rel_path: str
+    runtime: ScriptRuntime = None
+    json_schema: Optional[dict] = None
+    readme: Optional[str] = None
+
+    @property
+    def invoker(self) -> WasmInvoker:
+        if not self._invoker:
+            self._invoker = WasmInvoker()
+        return self._invoker
+
+    @classmethod
+    def from_tool_request(cls, tool_req: WasmToolRequest, lockfile: Lockfile = None, **kwargs) -> 'WasmTool':
+        if not lockfile:
+            raise ValueError("lockfile is required")
+        tool_req.lock = lockfile.get_lock(tool_req.lock.key())
+        toolpkg_path = tool_req.lock.toolpkg_path()
+        rel_path = tool_req.rel_path
+        rootpath = pathlib.Path(toolpkg_path) / rel_path
+        schema_path = rootpath / "schema.json"
+        config_path = rootpath / "config.toml"
+        readme_path = rootpath / "README.md"
+
+        try:
+            with schema_path.open("r") as f:
+                json_schema = json.load(f)
+        except Exception as e:
+            pocket_logger.warning(f"{toolpkg_path} failed to load json schema. error : {e}")
+            json_schema = None
+
+        try:
+            with config_path.open("r") as f:
+                config = toml.load(f)
+                name = config.get('name')
+                description = config.get('description')
+                if language := config.get('language'):
+                    lang = language.lower()
+                    if lang == 'python':
+                        runtime = ScriptRuntime.Python
+                    elif lang == 'node':
+                        runtime = ScriptRuntime.Node
+                    else:
+                        raise ValueError(f"The language `{lang}` is not supported.")
+                else:
+                    raise ValueError("`language` field is required in config.toml")
+                auth = cls._get_auth(config)
+        except Exception as e:
+            raise ValueError(f"Failed to load config.toml: {e}")
+
+        if readme_path.exists():
+            with readme_path.open("r") as f:
+                readme = f.read()
+        else:
+            readme = None
+        return cls(
+            name=name,
+            description=description,
+            argument_json_schema=json_schema,
+            auth=auth,
+            runtime=runtime,
+            readme=readme,
+            pkg_lock=tool_req.lock,
+            rel_path=tool_req.rel_path,
+        )
+
+    @classmethod
+    def _get_auth(cls, config: dict) -> Optional[ToolAuth]:
+        auth = config.get("auth")
+        if not auth:
+            return
+        auth_provider = auth.get("auth_provider")
+        auth_handler = auth.get("auth_handler")
+        scopes = auth.get("scopes", [])
+        return ToolAuth(
+            auth_provider=AuthProvider.get_auth_provider(auth_provider),
+            auth_handler=auth_handler,
+            scopes=scopes,
+        )
+
+    def template_arguments(self) -> dict[str, str]:
+        return {}
+
+    def invoke(self, body: Any, envs: dict, **kwargs) -> str:
+        return self.invoker.invoke(
+            str(self.pkg_lock.toolpkg_path() / self.rel_path),
+            self.runtime,
+            body,
+            envs,
+            **kwargs,
+        )
+
+    async def ainvoke(self, body: Any, envs: dict, **kwargs) -> str:
+        return await self.invoker.ainvoke(
+            str(self.pkg_lock.toolpkg_path() / self.rel_path),
+            self.runtime,
+            body,
+            envs,
+            **kwargs,
+        )

hyperpocket/util/__init__.py

@@ -0,0 +1 @@
+__all__ = ['json_schema_to_model', "tool_to_open_ai_spec"]

hyperpocket/util/extract_func_param_desc_from_docstring.py

@@ -0,0 +1,97 @@
+import inspect
+import re
+
+from hyperpocket.config import pocket_logger
+
+
+def extract_param_docstring_mapping(func) -> dict[str, str]:
+    """
+    Extracts a mapping between function parameters and their descriptions
+    from the Google-style docstring.
+
+    Args:
+        func (function): The function whose docstring needs to be parsed.
+
+    Returns:
+        list: A list of tuples where each tuple contains a parameter name
+              and its description.
+    """
+    # Get the docstring of the function
+    docstring = inspect.getdoc(func)
+    func_params = inspect.signature(func).parameters.keys()
+
+    if not docstring:
+        return {}
+
+    pocket_logger.debug(f"try to extract docstring of {func.__name__} by google style..")
+    param_mapping = extract_param_desc_by_google_stype_docstring(docstring, func_params)
+    if param_mapping:
+        pocket_logger.debug(f"success extract docstring of {func.__name__} by google style!")
+        return param_mapping
+    pocket_logger.debug(f"not found param desc of {func.__name__} by google style..")
+
+    pocket_logger.debug(f"try to extract docstring of {func.__name__} by other style..")
+    param_mapping = extract_param_desc_by_other_styles(docstring, func_params)
+    if param_mapping:
+        pocket_logger.debug(f"success extract docstring of {func.__name__} by other style!")
+        return param_mapping
+    pocket_logger.debug(f"not found param desc of {func.__name__} by other styles..")
+
+    # Plain Text Style matching
+    pocket_logger.debug(f"try to extract docstring of {func.__name__} by plain text style..")
+    param_descriptions = []
+    for line in docstring.split("\n"):
+        l = line.strip().split(":")
+        if len(l) < 2:
+            continue
+
+        param_name = l[0]
+        cleaned_param_name = clean_bracket_content(param_name)
+        description = ":".join(l[1:]).strip()
+        if cleaned_param_name in func_params:
+            param_descriptions.append((cleaned_param_name, description))
+
+    # Ensure no duplicates and match with function parameters
+    param_mapping = {param: desc for param, desc in param_descriptions if param in func_params}
+    pocket_logger.debug(f"final param_mapping of {func.__name__} : {param_mapping}")
+
+    return param_mapping
+
+
+def clean_bracket_content(content):
+    return re.sub(r"[(\[{<].*?[)\]}>]", "", content)
+
+
+def extract_param_desc_by_other_styles(docstring, func_params) -> dict[str, str]:
+    param_descriptions = []
+    # Pattern for Sphinx-style or Javadoc-style `:param`, `@param`, `:arg`, `@arg`
+    param_pattern = r"(?:@param|:param|:arg|@arg)\s+(\w+)(?:\s*:\s*|\s+|:\s+)(.*)"
+    matches = re.findall(param_pattern, docstring)
+    for param, desc in matches:
+        cleaned_param = clean_bracket_content(param)
+        param_descriptions.append((cleaned_param, desc.strip()))
+    # Ensure no duplicates and match with function parameters
+    param_mapping = {param: desc for param, desc in param_descriptions if param in func_params}
+    return param_mapping
+
+
+def extract_param_desc_by_google_stype_docstring(docstring, func_params) -> dict[str, str]:
+    # Regex pattern to extract parameter descriptions in Google style
+    param_pattern = r"Args:\n(.*?)(?=\n\S|$)"  # Matches the Args: section
+    match = re.search(param_pattern, docstring, re.DOTALL)
+    if not match:
+        return {}
+    param_section = match.group(1)
+    # Parse the parameter names and descriptions
+    param_lines = param_section.split("\n")
+    param_descriptions = {}
+    for line in param_lines:
+        # Match parameter line with "name (type): description"
+        param_match = re.match(r"^\s*(\w+)\s*\(\s*(.*?)\s*\)\s*:\s*(.*)", line)
+        if param_match:
+            param, _, desc = param_match.groups()
+            cleaned_param = clean_bracket_content(param)
+            param_descriptions[cleaned_param] = desc.strip()
+    # Match parameters to descriptions
+    param_mapping = {param: desc for param, desc in param_descriptions.items() if param in func_params}
+    return param_mapping

hyperpocket/util/find_all_leaf_class_in_package.py

@@ -0,0 +1,17 @@
+from typing import TypeVar, Type, List
+
+
+from hyperpocket.util.find_all_subclass_in_package import find_all_subclass_in_package
+
+T = TypeVar("T")
+
+
+def find_all_leaf_class_in_package(package_name: str, interface_type: Type[T]) -> List[T]:
+    parent_class_set = set()
+    subclasses = find_all_subclass_in_package(package_name, interface_type)
+
+    for sub in subclasses:
+        parent_class_set.add(*sub.__bases__)
+
+    leaf_sub_classes = [sub for sub in subclasses if sub not in parent_class_set]
+    return leaf_sub_classes

hyperpocket/util/find_all_subclass_in_package.py

@@ -0,0 +1,29 @@
+import importlib
+import inspect
+import pkgutil
+from typing import TypeVar, Type, List
+
+from hyperpocket.config import pocket_logger
+
+T = TypeVar("T")
+
+
+def find_all_subclass_in_package(package_name: str, interface_type: Type[T]) -> List[T]:
+    subclasses = set()
+    package = importlib.import_module(package_name)
+
+    for _, module_name, is_pkg in pkgutil.walk_packages(package.__path__, package.__name__ + "."):
+        try:
+            if "tests" in module_name or is_pkg:
+                continue
+
+            module = importlib.import_module(module_name)
+            module_classes = inspect.getmembers(module, inspect.isclass)
+            for _, obj in module_classes:
+                if issubclass(obj, interface_type) and obj is not interface_type:
+                    subclasses.add(obj)
+        except ImportError as e:
+            pocket_logger.warning(f"failed to import {module_name}. error : {e}")
+            continue
+
+    return list(subclasses)