cloudoll 2.4.0__py3-none-any.whl

cloudoll/robot/slack.py

@@ -0,0 +1,115 @@
+from cloudoll.web import http
+
+""" 
+web_hook: https://infloww.slack.com/apps/A0F7XDUAZ-incoming-webhook?tab=settings&next_id=0
+blocks_info: https://api.slack.com/messaging/composing/layouts
+"""
+
+
+class Header:
+    def __init__(self, text: str):
+        self._text = text
+
+    def get_json(self):
+        return {"type": "header", "text": {"type": "plain_text", "text": self._text}}
+
+
+class Text:
+    def __init__(self, text: str | int):
+        self._text = text
+
+    def get_json(self):
+        return {"type": "section", "text": {"type": "mrkdwn", "text": self._text}}
+
+
+class Field:
+    def __init__(self, title: str | int, text: str | int):
+        self._title = title
+        self._text = text
+
+    def get_json(self):
+        return {"type": "mrkdwn", "text": f"{self._title}\n{self._text}"}
+
+
+class Link:
+    def __init__(self, title: str | int, url: str | int):
+        self._title = title
+        self._url = url
+
+    def get_json(self):
+        return {
+            "type": "context",
+            "elements": [{"type": "mrkdwn", "text": f"<{self._url}|_{self._title}_>"}],
+        }
+
+
+class Section:
+    def __init__(self):
+        self._fileds: list[Field] = []
+
+    def add(self, field: Field):
+        self._fileds.append(field.get_json())
+        return self
+
+    def get_json(self):
+        return {"type": "section", "fileds": self._fileds}
+
+
+class Elements:
+    def __init__(self):
+        self._elements: list[Field] = []
+
+    def add(self, field: Field):
+        self._elements.append(field.get_json())
+        return self
+
+    def get_json(self):
+        return {"type": "context", "elements": self._elements}
+
+
+class Divider:
+    def __init__(self):
+        self._fileds: list[Field] = []
+
+    def get_json(self):
+        return {"type": "divider"}
+
+
+class Blocks:
+    def __init__(self):
+        self._blocks = []
+
+    def add(self, block: Section | Header | Text | Divider | Link):
+        self._blocks.append(block)
+        return self
+
+    def add_header(self, text):
+        self._blocks.append(Header(text))
+        return self
+
+    def add_link(self, title, url: str):
+        self._blocks.append(Link(title, url))
+        return self
+
+    def add_text(self, text):
+        self._blocks.append(Text(text))
+        return self
+
+    def add_divider(self):
+        self._blocks.append(Divider())
+        return self
+
+    def get_json(self):
+        blocks = []
+        for block in self._blocks:
+            blocks.append(block.get_json())
+        return blocks
+
+
+class Slack:
+    def __init__(self, url: str):
+        self._url = url
+
+    def send_notify(self, blocks: Blocks):
+        res = http.post(url=self._url, json={"blocks": blocks})
+        return res

cloudoll/utils/cli_main.py

@@ -0,0 +1,72 @@
+from typing import Any
+from ..web.settings import get_config
+from ..logging import print_info
+from .common import Object, chainMap
+from .watch import AppTask
+from aiohttp import web
+from ..web import app
+from pathlib import Path
+from .m2d import create_models, create_tables
+from ..orm import create_engine
+import os
+
+
+def run_app(**config_kwargs: Any):
+    config = Object(config_kwargs)
+    print_info("Current mode: ", config.mode)
+    app_config = get_config(config.environment)
+    # print(environment, host, port, mode, path, entry)
+    # return
+    if config.mode == 'production':
+        App = app.create(env=config.environment, config=app_config, entry_model=config.entry)
+        App.run()
+    else:
+        aux_app = web.Application(
+            logger=None,
+        )
+        defaults = {"host": "0.0.0.0", "port": 9001, "path": None}
+        conf_server = app_config.get("server", {})
+        env_server = {"host": config.host, "port": config.port, "path": config.path}
+        server = chainMap(defaults, conf_server, env_server)
+        app_config["server"] = server
+        aux_port = int(server.port) + 1
+        task = AppTask(Path(".").resolve(), app_config, entry=config.entry, env=config.environment)
+        aux_app.cleanup_ctx.append(task.cleanup_ctx)
+        web.run_app(
+            aux_app,
+            access_log=None,
+            host=config.host,
+            port=aux_port,
+            print=None,
+            shutdown_timeout=0.1,
+        )
+
+
+async def run_gen(**config_kwargs: Any):
+    config = Object(config_kwargs)
+    configs = get_config(config.environment)
+    db_configs = configs.get("database")
+    if db_configs is None:
+        raise KeyError(
+            f"Can't find the database config in conf.{config.environment}.yaml"
+        )
+    db_config = db_configs.get(config.database)
+    if db_config is None:
+        raise KeyError(
+            f"Can't find the database config key ->{config.database} in conf.{config.environment}.yaml"
+        )
+    sa = await create_engine(**db_config)
+    if sa.pool is None:
+        return
+    model_path = os.path.join(os.path.abspath("."), config.path)
+    tables = None
+    if config.table != "ALL":
+        tables = config.table.split(",")
+
+    if config.create == "model":
+        await create_models(sa, config.path, tables=tables)
+        print_info(f"Model save at:{model_path}")
+    elif config.create == "table":
+        if model_path is None:
+            raise ValueError("Need package name or model name.")
+        await create_tables(sa, model_path, tables=tables)

cloudoll/utils/cli_mian.py

@@ -0,0 +1,74 @@
+
+
+from typing import Any
+from ..web.settings import get_config
+from ..logging import print_info
+from .common import Object, chainMap
+from ..utils.watch import AppTask
+from aiohttp import web
+from ..web import app
+from pathlib import Path
+from ..utils.m2d import create_models, create_tables
+from ..orm import create_engine
+import os
+
+
+def run_app(**config_kwargs: Any):
+    config = Object(config_kwargs)
+    print_info("Current mode: ", config.mode)
+    app_config = get_config(config.environment)
+    # print(environment, host, port, mode, path, entry)
+    # return
+    if config.mode == 'production':
+        App = app.create(env=config.environment, config=app_config, entry_model=config.entry)
+        App.run()
+    else:
+        aux_app = web.Application(
+            logger=None,
+        )
+        defaults = {"host": "0.0.0.0", "port": 9001, "path": None}
+        conf_server = app_config.get("server", {})
+        env_server = {"host": config.host, "port": config.port, "path": config.path}
+        server = chainMap(defaults, conf_server, env_server)
+        app_config["server"] = server
+        aux_port = int(server.port) + 1
+        task = AppTask(Path(".").resolve(), app_config, entry=config.entry, env=config.environment)
+        aux_app.cleanup_ctx.append(task.cleanup_ctx)
+        web.run_app(
+            aux_app,
+            access_log=None,
+            host=config.host,
+            port=aux_port,
+            print=None,
+            shutdown_timeout=0.1,
+        )
+
+
+async def run_gen(**config_kwargs: Any):
+    config = Object(config_kwargs)
+    configs = get_config(config.environment)
+    db_configs = configs.get("database")
+    if db_configs is None:
+        raise KeyError(
+            f"Can't find the database config in conf.{config.environment}.yaml"
+        )
+    db_config = db_configs.get(config.database)
+    if db_config is None:
+        raise KeyError(
+            f"Can't find the database config key ->{config.database} in conf.{config.environment}.yaml"
+        )
+    sa = await create_engine(**db_config)
+    if sa.pool is None:
+        return
+    model_path = os.path.join(os.path.abspath("."), config.path)
+    tables = None
+    if config.table != "ALL":
+        tables = config.table.split(",")
+
+    if config.create == "model":
+        await create_models(sa, config.path, tables=tables)
+        print_info(f"Model save at:{model_path}")
+    elif config.create == "table":
+        if model_path is None:
+            raise ValueError("Need package name or model name.")
+        await create_tables(sa, model_path, tables=tables)

cloudoll/utils/common.py

@@ -0,0 +1,57 @@
+import asyncio
+from ..logging import warning
+
+EADDRINUSE: int
+
+
+class Object(dict):
+    # __setattr__ = dict.__setitem__
+    # __getattr__ = dict.__getitem__
+    def __setattr__(self, key, value):
+        self[key] = value
+
+    def __getattr__(self, key):
+        return self.get(key, None)
+
+    def __delattr__(self, key):
+        try:
+            del self[key]
+        except KeyError:
+            raise AttributeError(f"'DotDict' object has no attribute '{key}'")
+
+    def __getstate__(self):
+        """to fix pickle.dumps"""
+        return dict(self)
+
+    def __setstate__(self, state):
+        self.update(state)
+
+
+def chainMap(*dicts):
+    merged_dict = Object()
+    for d in dicts:
+        for key, value in d.items():
+            if key not in merged_dict or value is not None:
+                merged_dict[key] = value
+    return merged_dict
+
+
+async def check_port_open(port: int, delay: float = 1) -> None:
+    loop = asyncio.get_running_loop()
+    # the "s = socket.socket; s.bind" approach sometimes says a port is in use when it's not
+    # this approach replicates aiohttp so should always give the same answer
+    for i in range(5, 0, -1):
+        try:
+            server = await loop.create_server(
+                asyncio.Protocol, host="0.0.0.0", port=port
+            )
+        except OSError as e:
+            if e.errno != EADDRINUSE:
+                raise
+            warning("port %d is already in use, waiting %d...", port, i)
+            await asyncio.sleep(delay)
+        else:
+            server.close()
+            await server.wait_closed()
+            return
+    raise Exception("The port {} is already is use".format(port))

cloudoll/utils/m2d.py

@@ -0,0 +1,311 @@
+import enum
+import os
+import re
+from ..logging import print_info, print_warn
+import importlib
+
+
+def snake_to_camel(snake_str):
+    components = snake_str.split("_")
+    # camel_str = components[0] + ''.join(x.title() for x in components[1:])
+    # x.capitalize() //first
+    # x.title() //all
+    # return camel_str[0].upper() + camel_str[1:]
+    return "".join(x.title() for x in components[0:])
+
+
+async def create_model(pool, table_name) -> str:
+    """
+    Create table
+    :params table name
+    """
+    print_info(f"create model from {table_name}")
+    # rows = await pool.all(f"show full COLUMNS from `{table_name}`", None)
+    rows = await get_table_cols(pool, table_name)
+    # print(rows)
+    tb = f"\nclass {snake_to_camel(table_name)}(Model):\n\n"
+    tb += f"\t__table__ = '{table_name}'\n\n"
+    for f in rows:
+        fields = get_col(f, pool.driver)
+        # print("fields",fields)
+        name = fields["name"]
+        column_type = fields["column_type"]
+        values = []
+        if fields["primary_key"]:
+            values.append("primary_key=True")
+        if fields["charset"]:
+            values.append(f"charset='{fields['charset']}'")
+        if fields["max_length"] and column_type != "tinyint":
+            values.append(
+                f"max_length=({fields['max_length']})"
+                if isinstance(fields["max_length"], str) and "," in fields["max_length"]
+                else f"max_length={fields['max_length']}"
+            )
+        if fields["scale_length"]:
+            values.append(f"scale_length={fields['scale_length']}")
+        if fields["default"]:
+            values.append(f"default='{fields['default']}'")
+        if fields["auto_increment"]:
+            values.append("auto_increment=True")
+        if fields["NOT_NULL"]:
+            values.append("not_null=True")
+        if fields["created_generated"]:
+            values.append("created_generated=True")
+        if fields["update_generated"]:
+            values.append("update_generated=True")
+        if fields["comment"]:
+            values.append(f"comment='{fields['comment']}'")
+        if "unsigned" in column_type:
+            column_type = column_type.replace(" unsigned", "")
+            values.append("unsigned=True")
+        name = re.sub(r"\s", "", name)
+        tb += f"\t{name} = models.{ColTypes[column_type].value}Field({', '.join(values)})\n"
+    tb += "\n"
+    return tb
+
+
+async def get_table_cols(pool, table_name):
+    if pool.driver == "mysql":
+        return await pool.all(f"show full COLUMNS from `{table_name}`", None)
+    elif pool.driver == "postgres":
+        pri_row = await pool.one(
+            f"SELECT column_name from information_schema.key_column_usage WHERE table_name = '{table_name}'",
+            None,
+        )
+        sql = f"""SELECT column_name Field, 
+            column_default Default,
+            data_type column_type, 
+            is_nullable Null,
+            numeric_precision num_length,
+            numeric_scale scale_length,
+            character_maximum_length str_length,
+            datetime_precision date_length,
+            col_description('{table_name}'::regclass, ordinal_position) Comment
+            FROM information_schema.columns WHERE table_name ='{table_name}'"""
+
+        rows = await pool.all(sql, None)
+        for row in rows:
+            row["Key"] = "PRI" if pri_row and pri_row["column_name"] == row["field"] else None
+       
+        # print("rows", rows)
+        
+        return rows
+    else:
+        raise ValueError("Database not support.")
+
+
+async def get_all_tables(pool):
+    if pool.driver == "mysql":
+        return await pool.all("show tables", None)
+    elif pool.driver == "postgres":
+        return await pool.all(
+            "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'",
+            None,
+        )
+    else:
+        raise ValueError("Database not support.")
+
+
+async def create_models(pool, save_path: str = None, tables: list = None):
+    """
+    Create models
+    :params tables
+    :params save_path
+    """
+    if tables and len(tables) > 0:
+        tbs = tables
+    else:
+        # result = await pool.all("show tables", None)
+        all_tables = await get_all_tables(pool)
+        tbs = [list(c.values())[0] for c in all_tables]
+    content = ""
+    import_line = "from cloudoll.orm.model import models, Model\n\n"
+    for t in tbs:
+        content += await create_model(pool, t)
+    if save_path:
+        first_append = True
+        if os.path.exists(save_path):
+            with open(save_path, "r", encoding="utf-8") as f:
+                t = f.readlines(5)
+                first_append = import_line not in t
+        with open(save_path, "a", encoding="utf-8") as f:
+            if first_append:
+                content = import_line + content
+            f.write(content)
+    else:
+        return content
+
+
+async def create_table(pool, models: list, tables: list = None):
+    for model in models:
+        # print(table.__name__)
+        if model.__name__ == "Model":
+            continue
+        tb = model.__table__
+
+        if tables and tb not in tables:
+            continue
+
+        if tb.startswith("v_"):
+            print_warn(f"{tb} look like a view so skip.")
+            continue
+
+        # sql = f"DROP TABLE IF EXISTS `{tb}`;\n"
+        sql = ""
+        sql += f"CREATE TABLE `{tb}` (\n"
+
+        # labels = get_filed(model)
+        labels = model.__fields__
+        sqls = []
+        for f in labels:
+            lb = getattr(model, f)
+            row = get_col_sql(lb)
+            sqls.append(row)
+        sql += ",\n".join(sqls)
+        sql += ") ENGINE=InnoDB;"
+
+        print_info(f"create table {tb} ...\n\n")
+        # print(sql)
+        await pool.query(sql, None)
+
+
+async def create_tables(pool, model_name: str = None, tables: list = None):
+
+    # parts = model_name.split('.')
+    # package_name = '.'.join(parts[:-1]) or '.'
+    # module_name = parts[-1]
+    # print(module_name,package_name)
+    # models = importlib.import_module(module_name,package_name)
+    # module_classes = [cls for cls in vars(module_name).values() if isinstance(cls, type)]
+
+    name = os.path.basename(model_name)[:-3]
+    # 创建模块规范
+    module_spec = importlib.util.spec_from_file_location(name, model_name)
+    # 创建模块对象
+    module = importlib.util.module_from_spec(module_spec)
+    # 执行模块并加载其内容
+    module_spec.loader.exec_module(module)
+    # 获取模块中的所有实体
+    module_classes = [cls for cls in vars(module).values() if isinstance(cls, type)]
+
+    # print(module_classes)
+    await create_table(pool, models=module_classes, tables=tables)
+
+
+def get_col(field, driver="mysql"):
+    if driver == "mysql":
+        fields = {
+            "name": field["Field"],
+            "column_type": None,
+            "primary_key": field["Key"] == "PRI",
+            "default": field["Default"],
+            "charset": field["Collation"],
+            "max_length": None,
+            "scale_length":None,
+            "auto_increment": field["Extra"] == "auto_increment",
+            "NOT_NULL": field["Null"] == "NO",
+            "created_generated": "DEFAULT_GENERATED" == field["Extra"],
+            "update_generated": "on update" in field["Extra"],
+            "comment": field["Comment"],
+        }
+        field_type = field["Type"]
+        t = re.match(r"(\w+)[(](.*?)[)]", field_type)
+        if not t:
+            fields["column_type"] = field_type
+        else:
+            fields["column_type"] = t.groups()[0]
+            fields["max_length"] = t.groups()[1]
+        return fields
+    elif driver == "postgres" or driver == "postgresql":
+        fields = {
+            "name": field["field"],
+            "column_type": field["column_type"].replace(" ", "_"),
+            "primary_key": field["Key"] == "PRI",
+            "default": field["default"],
+            "charset": None,
+            "max_length": field["num_length"]
+            or field["str_length"]
+            or field["date_length"],
+            "scale_length": field["scale_length"],
+            "auto_increment": None,
+            "NOT_NULL": field["null"] == "NO",
+            "created_generated": None,
+            "update_generated": None,
+            "comment": field["comment"],
+        }
+        return fields
+
+
+def get_col_sql(field):
+    sql = f"`{field.name}` {field.column_type}"
+
+    if field.max_length:
+        if isinstance(field.max_length, tuple):
+            sql += f"{field.max_length}"
+        elif field.scale_length:
+            sql += f"({field.max_length},{field.scale_length})"
+        else:
+            sql += f"({field.max_length})"
+
+    if field.charset:
+        # _ci 不区分大小写 _cs Yes
+        cs = field.charset.split("_")[0]
+        sql += f" CHARACTER SET {cs} COLLATE {field.charset}"
+    if field.primary_key:
+        sql += " PRIMARY KEY"
+    if field.auto_increment:
+        # todo: postgres SERIAL
+        sql += " AUTO_INCREMENT" 
+    if field.NOT_NULL:
+        sql += " NOT NULL"
+    if field.default:
+        sql += " DEFAULT " + (
+            field.default if "(" in field.default else f"'{field.default}'"
+        )
+    # else:
+    # sql += " DEFAULT NULL"
+    # print(field.name, field.update_generated)
+    if field.update_generated:
+        sql += " ON UPDATE " + (
+            field.default if "(" in field.default else f"'{field.default}'"
+        )
+    if field.comment:
+        sql += f" COMMENT '{field.comment}'"
+
+    return sql
+
+
+def get_filed(model):
+    return [
+        attr
+        for attr in dir(model)
+        if not callable(getattr(model, attr)) and not attr.startswith("__")
+    ]
+
+
+class ColTypes(enum.Enum):
+    char = "Char"
+    varchar = "VarChar"
+    text = "Text"
+    longtext = "LongText"
+    mediumtext = "MediumText"
+    tinyint = "Boolean"
+    smallint = "Integer"
+    mediuint = "Integer"
+    int = "Integer"
+    bigint = "BigInteger"
+    float = "Float"
+    double = "Double"
+    decimal = "Decimal"
+    numeric = "Numeric"
+    date = "Date"
+    datetime = "Datetime"
+    timestamp = "Timestamp"
+    json = "Json"
+    # pg
+    timestamp_with_time_zone = "Timestamp"
+    timestamp_without_time_zone = "Timestamp"
+    character_varying = "VarChar"
+    character = "Char"
+    integer = "Integer"
+    boolean = "Boolean"