hyperpocket 0.3.7__py3-none-any.whl → 0.4.0__py3-none-any.whl

hyperpocket/auth/provider.py

@@ -72,6 +72,7 @@ class AuthProvider(Enum):
     APITOKEN = "apitoken"
     ZINC = "zinc"
     SEMANTIC_SCHOLAR = "semantic_scholar"
+    WEAVIATE = "weaviate"
 
     @classmethod
     def get_auth_provider(cls, auth_provider_name: str) -> "AuthProvider":

hyperpocket/auth/weaviate/context.py

@@ -0,0 +1,12 @@
+
+from hyperpocket.auth.context import AuthContext
+class WeaviateAuthContext(AuthContext):
+    _ACCESS_TOKEN_KEY: str = "WEAVIATE_TOKEN"
+    def to_dict(self) -> dict[str, str]:
+        return {
+            self._ACCESS_TOKEN_KEY: self.access_token,
+        }
+    def to_profiled_dict(self, profile: str) -> dict[str, str]:
+        return {
+            f"{profile.upper()}_{self._ACCESS_TOKEN_KEY}": self.access_token,
+        }

hyperpocket/auth/weaviate/token_context.py

@@ -0,0 +1,11 @@
+from hyperpocket.auth.weaviate.context import WeaviateAuthContext
+from hyperpocket.auth.weaviate.token_schema import WeaviateTokenResponse
+class WeaviateTokenAuthContext(WeaviateAuthContext):
+    @classmethod
+    def from_weaviate_token_response(cls, response: WeaviateTokenResponse):
+        description = f'Weaviate Token Context logged in'
+        return cls(
+            access_token=response.access_token,
+            description=description,
+            expires_at=None
+        )

hyperpocket/auth/weaviate/token_handler.py

@@ -0,0 +1,68 @@
+from typing import Optional
+from urllib.parse import urljoin, urlencode
+
+from hyperpocket.auth import AuthProvider
+from hyperpocket.auth.context import AuthContext
+from hyperpocket.auth.handler import AuthHandlerInterface, AuthenticateRequest
+from hyperpocket.auth.weaviate.token_context import WeaviateTokenAuthContext
+from hyperpocket.auth.weaviate.token_schema import WeaviateTokenResponse, WeaviateTokenRequest
+from hyperpocket.config import config
+from hyperpocket.futures import FutureStore
+
+
+class WeaviateTokenAuthHandler(AuthHandlerInterface):
+    name: str = "weaviate-token"
+    description: str = "This handler is used to authenticate users using the Weaviate token."
+    scoped: bool = False
+
+    _TOKEN_URL: str = urljoin(config().public_base_url + "/", f"{config().callback_url_rewrite_prefix}/auth/token")
+
+    @staticmethod
+    def provider() -> AuthProvider:
+        return AuthProvider.WEAVIATE
+    
+    @staticmethod
+    def provider_default() -> bool:
+        return True
+
+    @staticmethod
+    def recommended_scopes() -> set[str]:
+        return set()
+
+    def prepare(self, auth_req: WeaviateTokenRequest, thread_id: str, profile: str,
+                future_uid: str, *args, **kwargs) -> str:
+        redirect_uri = urljoin(
+            config().public_base_url + "/",
+            f"{config().callback_url_rewrite_prefix}/auth/weaviate/token/callback",
+        )
+        url = self._make_auth_url(auth_req=auth_req, redirect_uri=redirect_uri, state=future_uid)
+        FutureStore.create_future(future_uid, data={
+            "redirect_uri": redirect_uri,
+            "thread_id": thread_id,
+            "profile": profile,
+        })
+
+        return f'User needs to authenticate using the following URL: {url}'
+
+    async def authenticate(self, auth_req: WeaviateTokenRequest, future_uid: str, *args, **kwargs) -> AuthContext:
+        future_data = FutureStore.get_future(future_uid)
+        access_token = await future_data.future
+
+        response = WeaviateTokenResponse(access_token=access_token)
+        context = WeaviateTokenAuthContext.from_weaviate_token_response(response)
+
+        return context
+
+    async def refresh(self, auth_req: WeaviateTokenRequest, context: AuthContext, *args, **kwargs) -> AuthContext:
+        raise Exception("Weaviate token doesn't support refresh")
+
+    def _make_auth_url(self, auth_req: WeaviateTokenRequest, redirect_uri: str, state: str):
+        params = {
+            "redirect_uri": redirect_uri,
+            "state": state,
+        }
+        auth_url = f"{self._TOKEN_URL}?{urlencode(params)}"
+        return auth_url
+
+    def make_request(self, auth_scopes: Optional[list[str]] = None, **kwargs) -> WeaviateTokenRequest:
+        return WeaviateTokenRequest()

hyperpocket/auth/weaviate/token_schema.py

@@ -0,0 +1,7 @@
+from typing import List, Optional
+from pydantic import BaseModel
+from hyperpocket.auth.schema import AuthenticateRequest, AuthenticateResponse
+class WeaviateTokenRequest(AuthenticateRequest):
+    pass
+class WeaviateTokenResponse(AuthenticateResponse):
+    access_token: str

hyperpocket/cli/eject.py

@@ -10,10 +10,5 @@ import hyperpocket.repository as repository
 @click.argument("url", type=str)
 @click.argument("ref", type=str)
 @click.argument("remote_path", type=str)
-@click.option("--lockfile", envvar="PATHS", type=click.Path(exists=True))
-def eject(url: str, ref: str, remote_path: str, lockfile: Optional[pathlib.Path]):
-    if not lockfile:
-        lockfile = pathlib.Path.cwd() / "pocket.lock"
-    if not lockfile.exists():
-        raise ValueError("To eject a tool, you first need to pull it")
-    repository.eject(url, ref, remote_path, repository.Lockfile(lockfile))
+def eject(url: str, ref: str, remote_path: str):
+    repository.eject(url, ref, remote_path)

hyperpocket/cli/pull.py

@@ -8,11 +8,6 @@ import hyperpocket.repository as repository
 
 @click.command()
 @click.argument("url", type=str)
-@click.option("--lockfile", envvar="PATHS", type=click.Path(exists=True))
 @click.option("--git-ref", type=str, default="HEAD")
-def pull(url: str, lockfile: Optional[pathlib.Path], git_ref: str):
-    if not lockfile:
-        lockfile = pathlib.Path.cwd() / "pocket.lock"
-    if not lockfile.exists():
-        lockfile.touch()
-    repository.pull(repository.Lockfile(path=lockfile), url, git_ref)
+def pull(url: str, git_ref: str):
+    repository.pull(url, git_ref)

hyperpocket/config/settings.py

@@ -2,7 +2,7 @@ import os
 from pathlib import Path
 
 from dynaconf import Dynaconf
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, Extra
 
 from hyperpocket.config.auth import AuthConfig, DefaultAuthConfig
 from hyperpocket.config.session import DefaultSessionConfig, SessionConfig
@@ -47,6 +47,7 @@ class Config(BaseModel):
     auth: AuthConfig = DefaultAuthConfig
     session: SessionConfig = DefaultSessionConfig
     tool_vars: dict[str, str] = Field(default_factory=dict)
+    docks: dict[str, dict] = Field(default_factory=dict)
 
     @property
     def internal_base_url(self):

hyperpocket/pocket_core.py

@@ -1,53 +1,69 @@
 import asyncio
-import pathlib
 from typing import Any, Callable, List, Optional, Union
 
 from hyperpocket.builtin import get_builtin_tools
 from hyperpocket.config import pocket_logger
 from hyperpocket.pocket_auth import PocketAuth
-from hyperpocket.repository import Lockfile
-from hyperpocket.repository.lock import GitLock, LocalLock
-from hyperpocket.tool import Tool, ToolRequest
+from hyperpocket.tool import Tool
+from hyperpocket.tool.dock import Dock
 from hyperpocket.tool.function import from_func
-from hyperpocket.tool.wasm import WasmTool
-from hyperpocket.tool.wasm.tool import WasmToolRequest
 from hyperpocket.tool_like import ToolLike
 
 
 class PocketCore:
     auth: PocketAuth
     tools: dict[str, Tool]
-    lockfile: Lockfile
+    docks: list[Dock]
 
+    @staticmethod
+    def _default_dock() -> Dock:
+        try:
+            from hyperdock_container.dock import ContainerDock
+            pocket_logger.info("hyperdock-container is loaded.")
+            return ContainerDock()
+        except ImportError:
+            pocket_logger.warning("Failed to import hyperdock_container.")
+        
+        try:
+            from hyperdock_wasm.dock import WasmDock
+            pocket_logger.info("hyperdock-wasm is loaded.")
+            return WasmDock()
+        except ImportError:
+            raise ImportError("No default dock available. To register a remote tool, you need to install either hyperdock_wasm or hyperdock_container.")
+        
+    
     def __init__(
         self,
         tools: list[ToolLike],
         auth: PocketAuth = None,
-        lockfile_path: Optional[str] = None,
-        force_update: bool = False,
     ):
         if auth is None:
             auth = PocketAuth()
         self.auth = auth
 
-        # init lockfile
-        if lockfile_path is None:
-            lockfile_path = "./pocket.lock"
-        lockfile_pathlib_path = pathlib.Path(lockfile_path)
-        self.lockfile = Lockfile(lockfile_pathlib_path)
-
-        # parse tool_likes and add lock of the tool like to the lockfile
-        tool_likes = []
-        for tool_like in tools:
-            parsed_tool_like = self._parse_tool_like(tool_like)
-            tool_likes.append(parsed_tool_like)
-            self._add_tool_like_lock_to_lockfile(parsed_tool_like)
-        self.lockfile.sync(force_update=force_update, referenced_only=True)
-
-        # load tool from tool_like
+        # filter strs out first and register the tools to default dock
+        str_tool_likes = [tool for tool in tools if (isinstance(tool, str) or isinstance(tool, tuple))]
+        function_tool_likes = [tool for tool in tools if (not isinstance(tool, str) and not isinstance(tool, Dock) and not isinstance(tool, tuple))]
+        # especially, docks are maintained by core
+        self.docks = [dock for dock in tools if isinstance(dock, Dock)]
+        
+        if len(str_tool_likes) > 0:
+            default_dock = self._default_dock()
+            for str_tool_like in str_tool_likes:
+                default_dock.plug(req_like=str_tool_like)
+            # append default dock
+            self.docks.append(default_dock)
+
         self.tools = dict()
-        for tool_like in tool_likes:
+        
+        # for each tool like, load the tool 
+        for tool_like in function_tool_likes:
             self._load_tool(tool_like)
+        
+        for dock in self.docks:
+            tools = dock.tools()
+            for tool in tools:
+                self._load_tool(tool)
 
         pocket_logger.info(
             f"All Registered Tools Loaded successfully. total registered tools : {len(self.tools)}"
@@ -253,8 +269,6 @@ class PocketCore:
         pocket_logger.info(f"Loading Tool {tool_like}")
         if isinstance(tool_like, Tool):
             tool = tool_like
-        elif isinstance(tool_like, ToolRequest):
-            tool = Tool.from_tool_request(tool_like, lockfile=self.lockfile)
         elif isinstance(tool_like, Callable):
             tool = from_func(tool_like)
         else:
@@ -267,44 +281,3 @@ class PocketCore:
 
         pocket_logger.info(f"Complete Loading Tool {tool.name}")
         return tool
-
-    def _add_tool_like_lock_to_lockfile(self, tool_like: ToolLike):
-        if isinstance(tool_like, WasmToolRequest):  # lock is only in WasmToolRequest
-            self.lockfile.add_lock(tool_like.lock)
-        return
-
-    @classmethod
-    def _parse_tool_like(cls, tool_like: ToolLike) -> ToolLike:
-        if isinstance(tool_like, str):
-            return cls._parse_str_tool_like(tool_like)
-
-        elif isinstance(tool_like, WasmToolRequest):
-            return tool_like
-
-        elif isinstance(tool_like, ToolRequest):
-            raise ValueError(f"unreachable. tool_like:{tool_like}")
-        elif isinstance(tool_like, WasmTool):
-            raise ValueError("WasmTool should pass ToolRequest instance instead.")
-        else:  # Callable, Tool
-            return tool_like
-
-    @classmethod
-    def _parse_str_tool_like(cls, tool_like: str) -> ToolLike:
-        if pathlib.Path(tool_like).exists():
-            lock = LocalLock(tool_like)
-            parsed_tool_like = WasmToolRequest(lock=lock, rel_path="", tool_vars={})
-        elif tool_like.startswith("https://github.com"):
-            base_repo_url, git_ref, rel_path = GitLock.parse_repo_url(
-                repo_url=tool_like
-            )
-            lock = GitLock(repository_url=base_repo_url, git_ref=git_ref)
-            parsed_tool_like = WasmToolRequest(
-                lock=lock, rel_path=rel_path, tool_vars={}
-            )
-        else:
-            parsed_tool_like = None
-            RuntimeError(
-                f"Can't convert to ToolRequest. it might be wrong github url or local path. path: {tool_like}"
-            )
-
-        return parsed_tool_like

hyperpocket/pocket_main.py

@@ -1,5 +1,6 @@
 import asyncio
-from typing import Any, List, Optional, Union
+import uuid
+from typing import Any, List, Union
 
 from hyperpocket.config import pocket_logger
 from hyperpocket.pocket_auth import PocketAuth
@@ -11,29 +12,39 @@ from hyperpocket.tool_like import ToolLike
 class Pocket(object):
     server: PocketServer
     core: PocketCore
+    _uid: str
 
     def __init__(
         self,
         tools: list[ToolLike],
         auth: PocketAuth = None,
-        lockfile_path: Optional[str] = None,
-        force_update: bool = False,
         use_profile: bool = False,
     ):
-        self.use_profile = use_profile
-        self.core = PocketCore(
-            tools=tools,
-            auth=auth,
-            lockfile_path=lockfile_path,
-            force_update=force_update,
-        )
-        self.server = PocketServer()
-        self.server.run(self.core)
         try:
+            self._uid = str(uuid.uuid4())
+            self.use_profile = use_profile
+            self.server = PocketServer.get_instance_and_refcnt_up(self._uid)
             self.server.wait_initialized()
+            self.core = PocketCore(
+                tools=tools,
+                auth=auth,
+            )
         except Exception as e:
+            if hasattr(self, "server"):
+                self.server.refcnt_down(self._uid)
             pocket_logger.error(f"Failed to initialize pocket server. error : {e}")
             self._teardown_server()
+            raise e
+        
+        try:
+            asyncio.get_running_loop()
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+        else:
+            import nest_asyncio
+            loop = asyncio.new_event_loop()
+            nest_asyncio.apply(loop=loop)
+        loop.run_until_complete(self.server.plug_core(self._uid, self.core))
 
     def invoke(
         self,
@@ -158,8 +169,17 @@ class Pocket(object):
         Returns:
             tuple[str, bool]: tool result and state.
         """
+        _kwargs = {
+            "tool_name": tool_name,
+            "body": body,
+            "thread_id": thread_id,
+            "profile": profile,
+            **kwargs,
+        }
+
         result, paused = await self.server.call_in_subprocess(
             PocketServerOperations.CALL,
+            self._uid,
             args,
             {
                 "tool_name": tool_name,
@@ -257,6 +277,7 @@ class Pocket(object):
     ):
         prepare = await self.server.call_in_subprocess(
             PocketServerOperations.PREPARE_AUTH,
+            self._uid,
             args,
             {
                 "tool_name": tool_name,
@@ -278,6 +299,7 @@ class Pocket(object):
     ):
         credentials = await self.server.call_in_subprocess(
             PocketServerOperations.AUTHENTICATE,
+            self._uid,
             args,
             {
                 "tool_name": tool_name,
@@ -300,6 +322,7 @@ class Pocket(object):
     ):
         result = await self.server.call_in_subprocess(
             PocketServerOperations.TOOL_CALL,
+            self._uid,
             args,
             {
                 "tool_name": tool_name,
@@ -319,12 +342,10 @@ class Pocket(object):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        if self.__dict__.get("server"):
-            self.server.teardown()
+        self.server.refcnt_down(self._uid)
 
     def __del__(self):
-        if self.__dict__.get("server"):
-            self.server.teardown()
+        self.server.refcnt_down(self._uid)
 
     def __getstate__(self):
         state = self.__dict__.copy()

hyperpocket/repository/__init__.py

@@ -1,5 +1,4 @@
-from hyperpocket.repository.lock import Lock
-from hyperpocket.repository.lockfile import Lockfile
-from hyperpocket.repository.repository import eject, pull, sync
+from hyperpocket.repository.tool_reference import ToolReference
+from hyperpocket.repository.repository import eject, pull
 
-__all__ = ["Lock", "Lockfile", "pull", "sync", "eject"]
+__all__ = ["ToolReference", "pull", "eject"]

hyperpocket/repository/repository.py

@@ -1,43 +1,8 @@
-import pathlib
+def pull(urllike: str, git_ref: str):
+    # TODO
+    pass
 
-from hyperpocket.repository.lock import GitLock, LocalLock
-from hyperpocket.repository.lockfile import Lockfile
 
-
-def pull(lockfile: Lockfile, urllike: str, git_ref: str):
-    path = pathlib.Path(urllike)
-    if path.exists():
-        lockfile.add_lock(LocalLock(tool_path=str(path)))
-    else:
-        lockfile.add_lock(GitLock(repository_url=urllike, git_ref=git_ref))
-    lockfile.sync(force_update=False)
-    lockfile.write()
-
-
-def sync(lockfile: Lockfile, force_update: bool):
-    lockfile.sync(force_update=force_update)
-    lockfile.write()
-
-
-def eject(url: str, ref: str, remote_path: str, lockfile: Lockfile):
-    path = pathlib.Path(url)
-    if path.exists():
-        raise ValueError("Local tools are already ejected")
-    else:
-        lock = lockfile.get_lock(GitLock(repository_url=url, git_ref=ref).key())
-
-    # first copy source to ejected directory
-    working_dir = pathlib.Path.cwd()
-    ejected_dir = working_dir / "ejected_tools"
-    if not ejected_dir.exists():
-        ejected_dir.mkdir()
-
-    tool_name = pathlib.Path(remote_path).name
-    eject_path = ejected_dir / tool_name
-    lock.eject_to_path(eject_path, remote_path)
-
-    # then update lockfile
-    lockfile.remove_lock(lock.key())
-    lockfile.add_lock(LocalLock(tool_path=str(eject_path)))
-    lockfile.sync(force_update=True)
-    lockfile.write()
+def eject(url: str, ref: str, remote_path: str):
+    # TODO
+    pass

hyperpocket/repository/tool_reference.py

@@ -0,0 +1,28 @@
+import abc
+import pathlib
+
+from pydantic import BaseModel
+
+
+class ToolReference(BaseModel, abc.ABC):
+    tool_source: str = None
+
+    @abc.abstractmethod
+    def __str__(self):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def key(self) -> str:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def sync(self, **kwargs):
+        raise NotImplementedError
+
+    def eject_to_path(self, dest_path: pathlib.Path, src_sub_path: str = None):
+        ## local locks are already tracked by git
+        raise NotImplementedError
+
+
+
+

hyperpocket/server/auth/weaviate.py

@@ -0,0 +1,27 @@
+from fastapi import APIRouter
+from starlette.responses import HTMLResponse
+from hyperpocket.futures import FutureStore
+
+weaviate_auth_router = APIRouter(
+    prefix="/weaviate"
+)
+
+
+@weaviate_auth_router.get("/oauth2/callback")
+async def weaviate_oauth2_callback(state: str, code: str):
+    try:
+        FutureStore.resolve_future(state, code)
+    except ValueError:
+        return HTMLResponse(content="failed")
+
+    return HTMLResponse(content="success")
+
+
+@weaviate_auth_router.get("/token/callback")
+async def weaviate_token_callback(state: str, token: str):
+    try:
+        FutureStore.resolve_future(state, token)
+    except ValueError:
+        return HTMLResponse(content="failed")
+
+    return HTMLResponse(content="success")