ghoshell-common 0.4.0.dev0__py3-none-any.whl

ghoshell_common/entity.py

@@ -0,0 +1,233 @@
+from __future__ import annotations
+
+import json
+from abc import ABC, abstractmethod
+from typing import Union, Any, TypeVar, Type, Optional, Protocol
+from typing_extensions import Required, Self, TypedDict
+from types import ModuleType
+from pydantic import BaseModel
+from ghoshell_common.helpers import generate_import_path, import_from_path, parse_import_path_module_and_attr_name
+import inspect
+import pickle
+import base64
+import yaml
+
+__all__ = [
+
+    'to_entity_meta', 'from_entity_meta', 'get_entity',
+    'is_entity_type',
+    'EntityMeta',
+    'Entity', 'EntityType',
+    'EntityClass', 'ModelEntity',
+
+    'ModelEntityMeta',
+    'to_entity_model_meta',
+    'from_entity_model_meta',
+
+]
+
+"""
+Experimental Feature
+"""
+
+
+class Entity(Protocol):
+    """
+    Experimental interface
+
+    Protocol to define a class that generate transportable data (EntityMeta)
+    Instead of pickle, I want Entity is a light-weighted language independent protocol,
+    suite for JSON RPC or JSON API e.t.c.
+    """
+
+    @abstractmethod
+    def __to_entity_meta__(self) -> EntityMeta:
+        """
+        self-defined method to generate EntityMeta
+        """
+        pass
+
+    @classmethod
+    @abstractmethod
+    def __from_entity_meta__(cls, meta: EntityMeta) -> Self:
+        """
+        self-defined method to factory instance from EntityMeta
+        """
+        pass
+
+
+class EntityClass(ABC):
+    """
+    Abstract Class for Entity
+
+    Experimental interface
+    """
+
+    @abstractmethod
+    def __to_entity_meta__(self) -> EntityMeta:
+        pass
+
+    @classmethod
+    @abstractmethod
+    def __from_entity_meta__(cls, meta: EntityMeta) -> Self:
+        pass
+
+
+class ModelEntity(BaseModel, EntityClass, ABC):
+    """
+    pydantic Model that implements Entity interface
+    """
+
+    def __to_entity_meta__(self) -> EntityMeta:
+        return EntityMeta(
+            type=generate_import_path(self.__class__),
+            content=self.model_dump_json(exclude_defaults=True),
+        )
+
+    @classmethod
+    def __from_entity_meta__(cls, meta: EntityMeta) -> Self:
+        data = json.loads(meta['content'])
+        return cls(**data)
+
+
+class EntityMeta(TypedDict):
+    """
+    Experimental feature
+
+    I want python has a light-weight way to marshal and unmarshal any instance and make it readable if allowed.
+    I found so many package-level implements like various kinds of Serializable etc.
+
+    So, I develop EntityMeta as a wrapper for any kind.
+    The EntityType will grow bigger with more marshaller, but do not affect who (me) is using the EntityMeta.
+    One day I can replace it with any better way inside the functions (but in-compatible)
+    """
+    type: Required[str]
+    content: Required[str]
+
+
+class ModelEntityMeta(TypedDict):
+    """
+    Experimental feature
+
+    Data is dict, for pydantic.BaseModel
+    """
+    type: Required[str]
+    data: Required[dict]
+
+
+EntityType = Union[Entity, EntityMeta, BaseModel]
+
+
+def is_entity_type(value: Any) -> bool:
+    return hasattr(value, '__to_entity_meta__')
+
+
+def to_entity_model_meta(value: BaseModel) -> ModelEntityMeta:
+    type_ = generate_import_path(type(value))
+    data = value.model_dump(exclude_defaults=True)
+    return ModelEntityMeta(type=type_, data=data)
+
+
+def from_entity_model_meta(value: ModelEntityMeta) -> BaseModel:
+    cls = import_from_path(value['type'])
+    return cls(**value['data'])
+
+
+def to_entity_meta(value: Union[EntityType, Any]) -> EntityMeta:
+    if value is None:
+        return EntityMeta(
+            type="None",
+            content="",
+        )
+    elif value is True or value is False:
+        return EntityMeta(type="bool", content=str(value))
+    elif isinstance(value, int):
+        return EntityMeta(type="int", content=str(value))
+    elif isinstance(value, str):
+        return EntityMeta(type="str", content=str(value))
+    elif isinstance(value, float):
+        return EntityMeta(type="float", content=str(value))
+    elif isinstance(value, list):
+        content = yaml.safe_dump(value)
+        return EntityMeta(type="list", content=content)
+    elif isinstance(value, dict):
+        content = yaml.safe_dump(value)
+        return EntityMeta(type="dict", content=content)
+    elif hasattr(value, '__to_entity_meta__'):
+        return getattr(value, '__to_entity_meta__')()
+    elif isinstance(value, BaseModel):
+        return EntityMeta(
+            type=generate_import_path(value.__class__),
+            content=value.model_dump_json(exclude_defaults=True),
+        )
+    elif inspect.isfunction(value):
+        return EntityMeta(
+            type=generate_import_path(value),
+            content="",
+        )
+    elif isinstance(value, BaseModel):
+        type_ = generate_import_path(value.__class__)
+        content = value.model_dump_json(exclude_defaults=True)
+        return EntityMeta(type=type_, content=content)
+    else:
+        content_bytes = pickle.dumps(value)
+        content = base64.encodebytes(content_bytes)
+        return EntityMeta(
+            type="pickle",
+            content=content.decode(),
+        )
+
+
+T = TypeVar("T")
+
+
+def get_entity(meta: EntityMeta, expect: Type[T]) -> T:
+    if meta is None:
+        raise ValueError("EntityMeta cannot be None")
+    entity = from_entity_meta(meta)
+    if not isinstance(entity, expect):
+        raise TypeError(f"Expected entity type {expect} but got {type(entity)}")
+    return entity
+
+
+def from_entity_meta(meta: EntityMeta, module: Optional[ModuleType] = None) -> Any:
+    if meta is None:
+        return None
+    unmarshal_type = meta['type']
+    if unmarshal_type == "None":
+        return None
+    elif unmarshal_type == "int":
+        return int(meta['content'])
+    elif unmarshal_type == "str":
+        return str(meta['content'])
+    elif unmarshal_type == "bool":
+        return meta['content'] == "True"
+    elif unmarshal_type == "float":
+        return float(meta['content'])
+    elif unmarshal_type == "list" or unmarshal_type == "dict":
+        return yaml.safe_load(meta['content'])
+    elif unmarshal_type == 'pickle':
+        content = meta['content']
+        content_bytes = base64.decodebytes(content.encode())
+        return pickle.loads(content_bytes)
+
+    # raise if import error
+    cls = None
+    if module:
+        module_name, local_name = parse_import_path_module_and_attr_name(unmarshal_type)
+        if module_name == module.__name__:
+            cls = module.__dict__[local_name]
+    if cls is None:
+        cls = import_from_path(unmarshal_type)
+
+    if inspect.isfunction(cls):
+        return cls
+    # method is prior
+    elif hasattr(cls, "__from_entity_meta__"):
+        return getattr(cls, "__from_entity_meta__")(meta)
+
+    elif issubclass(cls, BaseModel):
+        data = json.loads(meta["content"])
+        return cls(**data)
+
+    raise TypeError(f"unsupported entity meta type: {unmarshal_type}")

ghoshell_common/helpers/__init__.py

@@ -0,0 +1,55 @@
+from typing import TYPE_CHECKING
+from ghoshell_common.helpers.dictionary import (dict_without_none, dict_without_zero)
+from ghoshell_common.helpers.string import camel_to_snake
+from ghoshell_common.helpers.yaml import yaml_pretty_dump, yaml_multiline_string_pipe
+from ghoshell_common.helpers.modules import (
+    import_from_path,
+    import_class_from_path,
+    import_instance_from_path,
+    parse_import_path_module_and_attr_name,
+    join_import_module_and_spec,
+    get_module_attr,
+    generate_module_and_attr_name,
+    generate_import_path,
+    get_module_fullname_from_path,
+    Importer,
+    is_method_belongs_to_class,
+    get_calling_modulename,
+    rewrite_module,
+    rewrite_module_by_path,
+    create_module,
+    create_and_bind_module,
+)
+from ghoshell_common.helpers.io import BufferPrint
+from ghoshell_common.helpers.timeutils import Timeleft, timestamp_datetime, timestamp, timestamp_ms
+from ghoshell_common.helpers.hashes import md5, sha1, sha256
+from ghoshell_common.helpers.trans import gettext, ngettext
+
+from ghoshell_common.helpers.coding import reflect_module_code, unwrap
+from ghoshell_common.helpers.openai import get_openai_key
+from ghoshell_common.helpers.tree_sitter import tree_sitter_parse, code_syntax_check
+from ghoshell_common.helpers.code_analyser import (
+    get_code_interface, get_code_interface_str,
+    get_attr_source_from_code, get_attr_interface_from_code,
+)
+from ghoshell_common.helpers.files import generate_directory_tree, list_dir, is_pathname_ignored
+
+if TYPE_CHECKING:
+    from typing import Callable
+
+
+# --- private methods --- #
+def __uuid() -> str:
+    from uuid import uuid4
+    # keep uuid in 32 chars
+    return str(uuid4())
+
+
+# --- facade --- #
+
+uuid: "Callable[[], str]" = __uuid
+""" patch this method to change global uuid generator"""
+
+
+def uuid_md5() -> str:
+    return md5(uuid())

ghoshell_common/helpers/code_analyser.py

@@ -0,0 +1,287 @@
+from typing import List, Union, Dict
+from tree_sitter import (
+    Node as TreeSitterNode,
+)
+from ghoshell_common.helpers.tree_sitter import tree_sitter_parse
+from functools import lru_cache
+
+__all__ = ['get_code_interface', 'get_code_interface_str', 'get_attr_source_from_code', 'get_attr_interface_from_code']
+
+
+@lru_cache(maxsize=256)
+def get_code_interface_str(code: str) -> str:
+    return "\n\n".join(get_code_interface(code))
+
+
+def get_code_interface(code: str) -> List[str]:
+    try:
+        # 解析代码，获取语法树
+        tree = tree_sitter_parse(code)
+        root_node = tree.root_node
+
+        # 用于存储解析后的抽象描述
+        interfaces = []
+        for child in root_node.children:
+            child_interface = _get_child_interface_in_depth(code, child, depth=0)
+            if child_interface:
+                interfaces.append(child_interface)
+
+        return interfaces
+
+    except Exception as e:
+        # 异常处理
+        raise
+
+
+@lru_cache(maxsize=256)
+def get_attr_source_from_code(code: str) -> Dict[str, str]:
+    """
+    get function or class attribute code.
+    """
+    try:
+        data = {}
+        # 解析代码，获取语法树
+        tree = tree_sitter_parse(code)
+        root_node = tree.root_node
+
+        for child in root_node.children:
+            name = _get_func_or_class_name(child)
+            if name is not None:
+                data[name] = child.text.decode()
+        return data
+    except Exception as e:
+        # 异常处理
+        raise
+
+
+def get_attr_interface_from_code(code: str) -> Dict[str, str]:
+    """
+    get function or class interface map from the code.
+    """
+    data = {}
+    # 解析代码，获取语法树
+    tree = tree_sitter_parse(code)
+    root_node = tree.root_node
+
+    for child in root_node.children:
+        name = _get_func_or_class_name(child)
+        if name is not None:
+            interface = _get_child_interface(code, child)
+            if interface:
+                data[name] = interface
+    return data
+
+
+def _get_func_or_class_name(child: TreeSitterNode) -> Union[str, None]:
+    if child.type == 'function_definition':
+        # 处理函数定义
+        func_name = child.child_by_field_name('name').text.decode()
+        return func_name
+    elif child.type == 'class_definition':
+        class_name = child.child_by_field_name('name').text.decode()
+        return class_name
+
+    elif child.type == 'decorated_definition':
+        # 处理装饰器
+        return _get_decorator_target_name(child)
+    return None
+
+
+def _get_decorator_target_name(child: TreeSitterNode) -> str:
+    definition = child.children[1]
+    return _get_func_or_class_name(definition)
+
+
+def _get_child_interface_in_depth(code: str, node: TreeSitterNode, depth: int) -> str:
+    interface = _get_child_interface(code, node)
+    if interface and depth > 0:
+        intend = ' ' * 4
+        splits = [intend + content for content in interface.split('\n')]
+        return '\n'.join(splits)
+    return interface
+
+
+def _get_child_interface(code: str, child: TreeSitterNode) -> str:
+    if child.type == 'import_statement' or child.type == 'import_from_statement':
+        # 如果是引用语句，保持原样
+        return child.text.decode()
+
+    elif child.type == 'expression_statement':
+        # 处理表达式语句
+        return _get_assignment_interface(code, child)
+
+    elif child.type == 'function_definition':
+        # 处理函数定义
+        func_name = child.child_by_field_name('name').text.decode()
+        if func_name and not func_name.startswith('_'):
+            definition = _get_function_interface(child)
+            return definition
+        return ""
+
+    elif child.type == 'class_definition':
+        return _get_class_interface(code, child)
+
+    elif child.type == 'decorated_definition':
+        # 处理装饰器
+        return _get_decorator_interface(code, child)
+    else:
+        return ""
+
+
+def _get_decorator_interface(code: str, child: TreeSitterNode) -> str:
+    decorator = child.children[0]
+    definition = child.children[1]
+    body = _get_child_interface_in_depth(code, definition, 0)
+    if body:
+        decorator_interfaces = [f"{decorator.text.decode()}", body]
+        return "\n".join(decorator_interfaces)
+    return ""
+
+
+def _get_full_definition(node: TreeSitterNode, docstring: bool = True) -> str:
+    definition = node.children[0].text.decode() + ' '
+    for child in node.children[1:]:
+        if child.type != "block":
+            text = child.text.decode()
+            if child.type == 'identifier':
+                definition = definition.rstrip() + ' '
+
+            definition += text
+        else:
+            if len(child.children) > 0 and docstring:
+                doc_node = child.children[0]
+                if doc_node.type == 'expression_statement' and doc_node.children[0].type == "string":
+                    doc_node = doc_node.children[0]
+                    doc = "\n".join([line.lstrip() for line in doc_node.text.decode().splitlines()])
+                    doc = insert_depth_to_code(doc, 1)
+                    definition += "\n" + doc
+            break
+    return definition.lstrip()
+
+
+def _get_assignment_interface(code: str, node: TreeSitterNode) -> str:
+    expression = node.children[0]
+    if expression.type == 'string':
+        return node.text.decode()
+
+    elif expression.type != 'assignment':
+        return ""
+
+    # 处理赋值语句
+    left = expression.children[0]
+    right = expression.children[2]
+    if left.type == 'identifier':
+        var_name = left.text.decode()
+        if var_name.startswith('__') and var_name.endswith('__'):
+            # 保持魔术变量和类型变量赋值原样
+            return node.text.decode()
+        elif var_name.startswith('_') or var_name.startswith('__'):
+            # 忽略私有变量赋值
+            return ""
+        else:
+            # 如果赋值语句长度小于50个字符，保留原样
+            assignment_code = node.text.decode()
+            if len(assignment_code) < 500:
+                return assignment_code
+            else:
+                # 折叠其它赋值
+                return f"{var_name} = ..."
+
+
+def _get_class_interface(code, class_node: TreeSitterNode) -> str:
+    # 处理类定义
+    class_name = class_node.child_by_field_name('name').text.decode()
+    if class_name.startswith('_'):
+        return ""
+    class_definition = _get_full_definition(class_node, docstring=False)
+
+    class_body_interface = []
+    # 处理类中的公共方法和__init__中的公共属性赋值
+    property_count = 0
+    class_body = class_node.children[-1]
+    for class_child in class_body.children:
+        if class_child.type == 'decorated_definition':
+            # 处理装饰器
+            decorator_interface = _get_decorator_interface(code, class_child)
+            class_body_interface.append(decorator_interface)
+
+        elif class_child.type == 'function_definition':
+            method_name = class_child.child_by_field_name('name').text.decode()
+            if method_name == '__init__':
+                init_method_interface = _get_class_init_body_interface(code, class_child)
+                class_body_interface.append(init_method_interface)
+            elif not method_name.startswith('_'):
+                method_interface = _get_function_interface(class_child)
+                class_body_interface.append(method_interface)
+            property_count += 1
+        else:
+            interface = _get_child_interface_in_depth(code, class_child, 0)
+            if interface:
+                class_body_interface.append(interface)
+                property_count += 1
+
+    if property_count == 0:
+        class_body_interface.append("    pass")
+    class_body = "\n\n".join([itf for itf in class_body_interface if itf])
+    class_body = insert_depth_to_code(class_body, 1)
+    return class_definition + "\n" + class_body
+
+
+def insert_depth_to_code(code: str, depth: int) -> str:
+    if depth < 1:
+        return code
+    indent = ' ' * 4 * depth
+    lines = []
+    for line in code.splitlines():
+        if line:
+            line = indent + line
+        lines.append(line)
+    return '\n'.join(lines)
+
+
+def _get_function_interface(method_node: TreeSitterNode) -> str:
+    definition = _get_full_definition(method_node)
+    method_interface = [definition, "    pass"]
+    return "\n".join(method_interface)
+
+
+def _get_class_init_body_interface(code: str, method_node: TreeSitterNode) -> str:
+    # 处理__init__方法中的公共属性赋值
+    definition = _get_full_definition(method_node)
+    init_method_interface = [definition]
+    body_interface = []
+    for init_child in method_node.children:
+        if init_child.type == 'block':
+            for block_child in init_child.children:
+                if block_child.type == 'expression_statement':
+                    expr = block_child.children[0]
+                    if expr.type == 'assignment':
+                        left = expr.children[0]
+                        if left.type == 'attribute':
+                            attr_name = left.text.decode()
+                            if not (attr_name.startswith('_') or attr_name.startswith('self._')):
+                                expr_line = expr.text.decode()
+                                if len(expr_line) > 50:
+                                    expr_line = expr_line[:50] + "..."
+                                body_interface.append(f"    {expr_line}")
+    body = "\n".join(body_interface)
+    if body:
+        init_method_interface.append(body)
+    else:
+        init_method_interface.append("        pass")
+
+    return "\n".join(init_method_interface)
+
+
+def _get_node_docstring(node: TreeSitterNode) -> str:
+    """
+    获取函数或类的 docstring.
+    """
+    for child in node.children:
+        if child.type == 'block':
+            for block_child in child.children:
+                if block_child.type == 'expression_statement':
+                    expr = block_child.children[0]
+                    if expr.type == 'string':
+                        return expr.text.decode().strip()
+    return ""

ghoshell_common/helpers/coding.py

@@ -0,0 +1,21 @@
+from typing import TypeVar, Union, Callable
+from types import ModuleType
+
+T = TypeVar("T")
+
+
+def unwrap(value: Union[T, Callable[[], T]]) -> T:
+    """
+    unwrap T from value or a callable
+    """
+    if isinstance(value, Callable):
+        return value()
+    return value
+
+
+def reflect_module_code(module: ModuleType) -> str:
+    """
+    get the module's code from file
+    """
+    with open(module.__file__) as f:
+        return f.read()

ghoshell_common/helpers/dictionary.py

@@ -0,0 +1,20 @@
+def dict_without_none(data: dict) -> dict:
+    """
+    Removes None values from a dictionary.
+    """
+    result = {}
+    for key, value in data.items():
+        if value is not None:
+            result[key] = value
+    return result
+
+
+def dict_without_zero(data: dict) -> dict:
+    """
+    Removes zero values from a dictionary.
+    """
+    result = {}
+    for key, value in data.items():
+        if not value:
+            result[key] = value
+    return result