ghoshell-common 0.4.0__py3-none-any.whl

ghoshell_common/contracts/storage.py

@@ -0,0 +1,209 @@
+from abc import abstractmethod
+from typing import Optional, Iterable, Protocol, Dict
+import os
+from ghoshell_container import Container, Provider
+import fnmatch
+
+__all__ = ['Storage', 'FileStorage', 'DefaultFileStorage', 'FileStorageProvider', 'MemoryStorage']
+
+
+class Storage(Protocol):
+
+    @abstractmethod
+    def sub_storage(self, relative_path: str) -> "Storage":
+        """
+        生成一个次级目录下的 storage.
+        :param relative_path:
+        :return:
+        """
+        pass
+
+    @abstractmethod
+    def get(self, file_path: str) -> bytes:
+        """
+        获取一个 Storage 路径下一个文件的内容.
+        :param file_path: storage 下的一个相对路径.
+        """
+        pass
+
+    @abstractmethod
+    def remove(self, file_path: str) -> None:
+        pass
+
+    @abstractmethod
+    def exists(self, file_path: str) -> bool:
+        """
+        if the object exists
+        :param file_path: file_path or directory path
+        """
+        pass
+
+    @abstractmethod
+    def put(self, file_path: str, content: bytes) -> None:
+        """
+        保存一个文件的内容到 file_path .
+        :param file_path: storage 下的一个相对路径.
+        :param content: 文件的内容.
+        """
+        pass
+
+    @abstractmethod
+    def dir(self, prefix_dir: str, recursive: bool, patten: Optional[str] = None) -> Iterable[str]:
+        """
+        遍历一个路径下的文件, 返回相对的文件名.
+        :param prefix_dir: 目录的相对路径位置.
+        :param recursive: 是否递归查找.
+        :param patten: 文件的正则规范.
+        :return: 多个文件路径名.
+        """
+        pass
+
+
+class FileStorage(Storage, Protocol):
+    """
+    Storage Based on FileSystem.
+    """
+
+    @abstractmethod
+    def abspath(self) -> str:
+        """
+        storage root directory's absolute path
+        """
+        pass
+
+    @abstractmethod
+    def sub_storage(self, relative_path: str) -> "FileStorage":
+        """
+        FileStorage's sub storage is still FileStorage
+        """
+        pass
+
+
+class DefaultFileStorage(FileStorage):
+    """
+    FileStorage implementation based on python filesystem.
+    Simplest implementation.
+    """
+
+    def __init__(self, dir_: str):
+        self._dir: str = os.path.abspath(dir_)
+
+    def abspath(self) -> str:
+        return self._dir
+
+    def get(self, file_path: str) -> bytes:
+        file_path = self._join_file_path(file_path)
+        with open(file_path, 'rb') as f:
+            return f.read()
+
+    def remove(self, file_path: str) -> None:
+        file_path = self._join_file_path(file_path)
+        os.remove(file_path)
+
+    def exists(self, file_path: str) -> bool:
+        file_path = self._join_file_path(file_path)
+        return os.path.exists(file_path)
+
+    def _join_file_path(self, path: str) -> str:
+        file_path = os.path.join(self._dir, path)
+        file_path = os.path.abspath(file_path)
+        if not file_path.startswith(self._dir):
+            raise FileNotFoundError(f"file path {path} is not allowed")
+        return file_path
+
+    def put(self, file_path: str, content: bytes) -> None:
+        file_path = self._join_file_path(file_path)
+        if not file_path.startswith(self._dir):
+            raise FileNotFoundError(f"file path {file_path} is not allowed")
+        file_dir = os.path.dirname(file_path)
+        if not os.path.exists(file_dir):
+            os.makedirs(file_dir)
+        with open(file_path, 'wb') as f:
+            f.write(content)
+
+    def sub_storage(self, relative_path: str) -> "FileStorage":
+        if not relative_path:
+            return self
+        dir_path = self._join_file_path(relative_path)
+        return DefaultFileStorage(dir_path)
+
+    def dir(self, prefix_dir: str, recursive: bool, patten: Optional[str] = None) -> Iterable[str]:
+        dir_path = self._join_file_path(prefix_dir)
+        for root, ds, fs in os.walk(dir_path):
+            # 遍历单层的文件名.
+            for filename in fs:
+                if self._match_file_pattern(filename, patten):
+                    yield filename
+
+            # 深入遍历目录.
+            if recursive and ds:
+                for _dir in ds:
+                    sub_dir_iterator = self.dir(_dir, recursive, patten)
+                    for filename in sub_dir_iterator:
+                        yield filename
+
+    @staticmethod
+    def _match_file_pattern(filename: str, pattern: Optional[str]) -> bool:
+        if pattern is None:
+            return True
+        matched = True
+        if pattern.startswith('!'):
+            matched = False
+            pattern = pattern[1:]
+        if pattern and fnmatch.fnmatch(filename, pattern):
+            return matched
+        return not matched
+
+
+class MemoryStorage(Storage):
+
+    def __init__(self, dir_: str):
+        self._dir: str = dir_
+        self._children: Dict[str, Storage] = {}
+        self._data: Dict[str, bytes] = {}
+
+    def sub_storage(self, relative_path: str) -> "Storage":
+        dir_ = os.path.join(self._dir, relative_path)
+        if dir_ in self._children:
+            return self._children[dir_]
+        sub = MemoryStorage(dir_)
+        self._children[dir_] = sub
+        return sub
+
+    def get(self, file_path: str) -> bytes:
+        _path = os.path.join(self._dir, file_path)
+        if _path not in self._data:
+            raise FileNotFoundError(f"file {file_path} is not found")
+        return self._data[_path]
+
+    def remove(self, file_path: str) -> None:
+        _path = os.path.join(self._dir, file_path)
+        if _path in self._data:
+            del self._data[_path]
+
+    def exists(self, file_path: str) -> bool:
+        _path = os.path.join(self._dir, file_path)
+        return _path in self._data
+
+    def put(self, file_path: str, content: bytes) -> None:
+        _path = os.path.join(self._dir, file_path)
+        self._data[_path] = content
+
+    def dir(self, prefix_dir: str, recursive: bool, patten: Optional[str] = None) -> Iterable[str]:
+        for dir_ in self._children.keys():
+            yield dir_[len(self._dir):]
+
+
+class FileStorageProvider(Provider[FileStorage]):
+
+    def __init__(self, dir_: str):
+        self._dir: str = dir_
+
+    def singleton(self) -> bool:
+        return True
+
+    def aliases(self) -> Iterable[Storage]:
+        yield Storage
+
+    def factory(self, con: Container) -> Optional[Storage]:
+        return DefaultFileStorage(self._dir)

ghoshell_common/contracts/workspace.py

@@ -0,0 +1,91 @@
+import os
+from abc import ABC, abstractmethod
+from typing import Optional
+
+from ghoshell_common.contracts.storage import Storage, DefaultFileStorage, MemoryStorage
+from ghoshell_container import Container, Provider
+from os.path import abspath
+import shutil
+
+__all__ = ['Workspace', 'LocalWorkspace', 'LocalWorkspaceProvider']
+
+
+class Workspace(ABC):
+    """
+    workspace 目录文件管理.
+    用于管理一个项目的本地文件存储.
+    """
+
+    @abstractmethod
+    def root(self) -> Storage:
+        """
+        workspace 根 storage.
+        """
+        pass
+
+    @abstractmethod
+    def configs(self) -> Storage:
+        """
+        配置文件存储路径.
+        """
+        pass
+
+    @abstractmethod
+    def runtime(self) -> Storage:
+        """
+        运行时数据存储路径.
+        """
+        pass
+
+    @abstractmethod
+    def assets(self) -> Storage:
+        """
+        数据资产存储路径.
+        """
+        pass
+
+
+class LocalWorkspace(Workspace):
+
+    def __init__(self, workspace_dir: str):
+        workspace_dir = abspath(workspace_dir)
+        self._ws_root_dir = workspace_dir
+        if not self._ws_root_dir:
+            self._root = MemoryStorage(self._ws_root_dir)
+        else:
+            self._root = DefaultFileStorage(workspace_dir)
+
+    def root(self) -> Storage:
+        return self._root
+
+    def configs(self) -> Storage:
+        return self._root.sub_storage("configs")
+
+    def runtime(self) -> Storage:
+        return self._root.sub_storage("runtime")
+
+    def assets(self) -> Storage:
+        return self._root.sub_storage("assets")
+
+
+class LocalWorkspaceProvider(Provider[Workspace]):
+
+    def __init__(
+            self,
+            workspace_dir: str = "",
+            stub_dir: Optional[str] = None,
+    ):
+        if workspace_dir == "":
+            # 使用脚本运行的路径作为 workspace.
+            workspace_dir = abspath(workspace_dir)
+        self._ws_dir = workspace_dir
+        self._stub_dir = abspath(stub_dir) if stub_dir else None
+
+    def singleton(self) -> bool:
+        return True
+
+    def factory(self, con: Container) -> Optional[Workspace]:
+        if self._ws_dir and self._stub_dir and not os.path.exists(self._stub_dir):
+            os.makedirs(self._stub_dir)
+            shutil.copytree(self._stub_dir, self._ws_dir)
+        return LocalWorkspace(self._ws_dir)

ghoshell_common/entity.py

@@ -0,0 +1,233 @@
+from __future__ import annotations
+
+import json
+from abc import ABC, abstractmethod
+from typing import Union, Any, TypeVar, Type, Optional, Protocol
+from typing_extensions import Required, Self, TypedDict
+from types import ModuleType
+from pydantic import BaseModel
+from ghoshell_common.helpers import generate_import_path, import_from_path, parse_import_path_module_and_attr_name
+import inspect
+import pickle
+import base64
+import yaml
+
+__all__ = [
+
+    'to_entity_meta', 'from_entity_meta', 'get_entity',
+    'is_entity_type',
+    'EntityMeta',
+    'Entity', 'EntityType',
+    'EntityClass', 'ModelEntity',
+
+    'ModelEntityMeta',
+    'to_entity_model_meta',
+    'from_entity_model_meta',
+
+]
+
+"""
+Experimental Feature
+"""
+
+
+class Entity(Protocol):
+    """
+    Experimental interface
+
+    Protocol to define a class that generate transportable data (EntityMeta)
+    Instead of pickle, I want Entity is a light-weighted language independent protocol,
+    suite for JSON RPC or JSON API e.t.c.
+    """
+
+    @abstractmethod
+    def __to_entity_meta__(self) -> EntityMeta:
+        """
+        self-defined method to generate EntityMeta
+        """
+        pass
+
+    @classmethod
+    @abstractmethod
+    def __from_entity_meta__(cls, meta: EntityMeta) -> Self:
+        """
+        self-defined method to factory instance from EntityMeta
+        """
+        pass
+
+
+class EntityClass(ABC):
+    """
+    Abstract Class for Entity
+
+    Experimental interface
+    """
+
+    @abstractmethod
+    def __to_entity_meta__(self) -> EntityMeta:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def __from_entity_meta__(cls, meta: EntityMeta) -> Self:
+        pass
+
+
+class ModelEntity(BaseModel, EntityClass, ABC):
+    """
+    pydantic Model that implements Entity interface
+    """
+
+    def __to_entity_meta__(self) -> EntityMeta:
+        return EntityMeta(
+            type=generate_import_path(self.__class__),
+            content=self.model_dump_json(exclude_defaults=True),
+        )
+
+    @classmethod
+    def __from_entity_meta__(cls, meta: EntityMeta) -> Self:
+        data = json.loads(meta['content'])
+        return cls(**data)
+
+
+class EntityMeta(TypedDict):
+    """
+    Experimental feature
+
+    I want python has a light-weight way to marshal and unmarshal any instance and make it readable if allowed.
+    I found so many package-level implements like various kinds of Serializable etc.
+
+    So, I develop EntityMeta as a wrapper for any kind.
+    The EntityType will grow bigger with more marshaller, but do not affect who (me) is using the EntityMeta.
+    One day I can replace it with any better way inside the functions (but in-compatible)
+    """
+    type: Required[str]
+    content: Required[str]
+
+
+class ModelEntityMeta(TypedDict):
+    """
+    Experimental feature
+
+    Data is dict, for pydantic.BaseModel
+    """
+    type: Required[str]
+    data: Required[dict]
+
+
+EntityType = Union[Entity, EntityMeta, BaseModel]
+
+
+def is_entity_type(value: Any) -> bool:
+    return hasattr(value, '__to_entity_meta__')
+
+
+def to_entity_model_meta(value: BaseModel) -> ModelEntityMeta:
+    type_ = generate_import_path(type(value))
+    data = value.model_dump(exclude_defaults=True)
+    return ModelEntityMeta(type=type_, data=data)
+
+
+def from_entity_model_meta(value: ModelEntityMeta) -> BaseModel:
+    cls = import_from_path(value['type'])
+    return cls(**value['data'])
+
+
+def to_entity_meta(value: Union[EntityType, Any]) -> EntityMeta:
+    if value is None:
+        return EntityMeta(
+            type="None",
+            content="",
+        )
+    elif value is True or value is False:
+        return EntityMeta(type="bool", content=str(value))
+    elif isinstance(value, int):
+        return EntityMeta(type="int", content=str(value))
+    elif isinstance(value, str):
+        return EntityMeta(type="str", content=str(value))
+    elif isinstance(value, float):
+        return EntityMeta(type="float", content=str(value))
+    elif isinstance(value, list):
+        content = yaml.safe_dump(value)
+        return EntityMeta(type="list", content=content)
+    elif isinstance(value, dict):
+        content = yaml.safe_dump(value)
+        return EntityMeta(type="dict", content=content)
+    elif hasattr(value, '__to_entity_meta__'):
+        return getattr(value, '__to_entity_meta__')()
+    elif isinstance(value, BaseModel):
+        return EntityMeta(
+            type=generate_import_path(value.__class__),
+            content=value.model_dump_json(exclude_defaults=True),
+        )
+    elif inspect.isfunction(value):
+        return EntityMeta(
+            type=generate_import_path(value),
+            content="",
+        )
+    elif isinstance(value, BaseModel):
+        type_ = generate_import_path(value.__class__)
+        content = value.model_dump_json(exclude_defaults=True)
+        return EntityMeta(type=type_, content=content)
+    else:
+        content_bytes = pickle.dumps(value)
+        content = base64.encodebytes(content_bytes)
+        return EntityMeta(
+            type="pickle",
+            content=content.decode(),
+        )
+
+
+T = TypeVar("T")
+
+
+def get_entity(meta: EntityMeta, expect: Type[T]) -> T:
+    if meta is None:
+        raise ValueError("EntityMeta cannot be None")
+    entity = from_entity_meta(meta)
+    if not isinstance(entity, expect):
+        raise TypeError(f"Expected entity type {expect} but got {type(entity)}")
+    return entity
+
+
+def from_entity_meta(meta: EntityMeta, module: Optional[ModuleType] = None) -> Any:
+    if meta is None:
+        return None
+    unmarshal_type = meta['type']
+    if unmarshal_type == "None":
+        return None
+    elif unmarshal_type == "int":
+        return int(meta['content'])
+    elif unmarshal_type == "str":
+        return str(meta['content'])
+    elif unmarshal_type == "bool":
+        return meta['content'] == "True"
+    elif unmarshal_type == "float":
+        return float(meta['content'])
+    elif unmarshal_type == "list" or unmarshal_type == "dict":
+        return yaml.safe_load(meta['content'])
+    elif unmarshal_type == 'pickle':
+        content = meta['content']
+        content_bytes = base64.decodebytes(content.encode())
+        return pickle.loads(content_bytes)
+
+    # raise if import error
+    cls = None
+    if module:
+        module_name, local_name = parse_import_path_module_and_attr_name(unmarshal_type)
+        if module_name == module.__name__:
+            cls = module.__dict__[local_name]
+    if cls is None:
+        cls = import_from_path(unmarshal_type)
+
+    if inspect.isfunction(cls):
+        return cls
+    # method is prior
+    elif hasattr(cls, "__from_entity_meta__"):
+        return getattr(cls, "__from_entity_meta__")(meta)
+
+    elif issubclass(cls, BaseModel):
+        data = json.loads(meta["content"])
+        return cls(**data)
+
+    raise TypeError(f"unsupported entity meta type: {unmarshal_type}")

ghoshell_common/helpers/__init__.py

@@ -0,0 +1,55 @@
+from typing import TYPE_CHECKING
+from ghoshell_common.helpers.dictionary import (dict_without_none, dict_without_zero)
+from ghoshell_common.helpers.string import camel_to_snake
+from ghoshell_common.helpers.yaml import yaml_pretty_dump, yaml_multiline_string_pipe
+from ghoshell_common.helpers.modules import (
+    import_from_path,
+    import_class_from_path,
+    import_instance_from_path,
+    parse_import_path_module_and_attr_name,
+    join_import_module_and_spec,
+    get_module_attr,
+    generate_module_and_attr_name,
+    generate_import_path,
+    get_module_fullname_from_path,
+    Importer,
+    is_method_belongs_to_class,
+    get_calling_modulename,
+    rewrite_module,
+    rewrite_module_by_path,
+    create_module,
+    create_and_bind_module,
+)
+from ghoshell_common.helpers.io import BufferPrint
+from ghoshell_common.helpers.timeutils import Timeleft, timestamp_datetime, timestamp, timestamp_ms
+from ghoshell_common.helpers.hashes import md5, sha1, sha256
+from ghoshell_common.helpers.trans import gettext, ngettext
+
+from ghoshell_common.helpers.coding import reflect_module_code, unwrap
+from ghoshell_common.helpers.openai import get_openai_key
+from ghoshell_common.helpers.tree_sitter import tree_sitter_parse, code_syntax_check
+from ghoshell_common.helpers.code_analyser import (
+    get_code_interface, get_code_interface_str,
+    get_attr_source_from_code, get_attr_interface_from_code,
+)
+from ghoshell_common.helpers.files import generate_directory_tree, list_dir, is_pathname_ignored
+
+if TYPE_CHECKING:
+    from typing import Callable
+
+
+# --- private methods --- #
+def __uuid() -> str:
+    from uuid import uuid4
+    # keep uuid in 32 chars
+    return str(uuid4())
+
+
+# --- facade --- #
+
+uuid: "Callable[[], str]" = __uuid
+""" patch this method to change global uuid generator"""
+
+
+def uuid_md5() -> str:
+    return md5(uuid())