cpbox 1.6.859__py3-none-any.whl

cpbox/__init__.py

@@ -0,0 +1 @@
+

cpbox/_assets/docker/Dockerfile.j2

@@ -0,0 +1,41 @@
+FROM {% if XENV_DOCKER_BASE_IMAGE_PROXY_HOST %}{{ XENV_DOCKER_BASE_IMAGE_PROXY_HOST }}/{% endif %}continuumio/miniconda3:24.11.1-0
+
+ARG TZ=Asia/Shanghai
+ENV XENV_CONDA_ENV=xenv
+
+WORKDIR /xmnt/app-working-root
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=${TZ}
+
+COPY output/xenv-storage/runtime/build-assets/docker/debian.sources.tuna /etc/apt/sources.list.d/debian.sources
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime \
+    && ln -snf /usr/share/zoneinfo/${TZ} /etc/timezone
+
+COPY output/xenv-storage/runtime/build-assets/docker/pip.conf /root/.pip/pip.conf
+
+COPY environment.yml /tmp/environment.yml
+COPY requirements-prod.txt /tmp/requirements-prod.txt
+COPY requirements-dev.txt /tmp/requirements-dev.txt
+COPY pyproject.toml /tmp/pyproject.toml
+
+RUN conda env create -n ${XENV_CONDA_ENV} -f /tmp/environment.yml \
+    && conda clean -afy
+
+RUN conda run -n ${XENV_CONDA_ENV} pip install --no-cache-dir -r /tmp/requirements-prod.txt \
+    && conda run -n ${XENV_CONDA_ENV} pip install --no-cache-dir -r /tmp/requirements-dev.txt
+
+RUN printf '%s\n' '#!/bin/bash' 'exec conda run -n "$XENV_CONDA_ENV" --no-capture-output "$@"' > /usr/local/bin/xenv-entrypoint \
+    && chmod +x /usr/local/bin/xenv-entrypoint
+
+ENV PYTHONPATH=/xmnt/app-working-root/src
+ENV PATH=/opt/conda/envs/xenv/bin:/opt/conda/condabin:${PATH}
+ENV CONDA_DEFAULT_ENV=${XENV_CONDA_ENV}
+
+ENTRYPOINT ["/usr/local/bin/xenv-entrypoint"]
+CMD ["python", "--version"]

cpbox/_assets/docker/debian.sources.tuna.j2

@@ -0,0 +1,11 @@
+Types: deb
+URIs: https://mirrors.tuna.tsinghua.edu.cn/debian
+Suites: bookworm bookworm-updates
+Components: main
+Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg
+
+Types: deb
+URIs: https://security.debian.org/debian-security
+Suites: bookworm-security
+Components: main
+Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg

cpbox/_assets/docker/pip.conf.j2

@@ -0,0 +1,8 @@
+[global]
+{% if XENV_PIP_INDEX_HOST -%}
+index=https://{{ XENV_PIP_INDEX_HOST }}/repository/pypi-all/pypi
+index-url=https://{{ XENV_PIP_INDEX_HOST }}/repository/pypi-all/simple
+{% else -%}
+index = https://pypi.tuna.tsinghua.edu.cn/pypi
+index-url = https://pypi.tuna.tsinghua.edu.cn/simple
+{% endif %}

cpbox/app/appconfig.py

@@ -0,0 +1,27 @@
+from cpbox.tool.utils import Singleton
+
+
+class ConfigData():
+    __metaclass__ = Singleton
+
+    def __init__(self):
+        self._env = None
+        self._app_name = None
+        self._app_config = {}
+
+    def get_env(self):
+        return self._env
+
+    def get_app_name(self):
+        return self._app_name
+
+    def init(self, app_name, env = None, app_config = {}):
+        self._app_name = app_name
+        if env is not None:
+            self._env = env
+        self._app_config = app_config
+
+    def get_app_config(self):
+        return self._app_config
+
+appconfig = ConfigData()

cpbox/app/contextdata.py

@@ -0,0 +1,76 @@
+from cpbox.tool import datatypes
+
+
+class EnvDefaults(datatypes.TypedValueMap):
+
+    allowed_keys = (
+        'APP_ENV',
+        'LOG.LEVEL',
+        'LOG.DISABLE_STDOUT_LOGGER',
+        'LOG.NOISE_LOGGERS',
+        'LOG.NOISE_LEVEL',
+        'CONDA_DEFAULT_ENV_NAME',
+        'XENV_IMAGE_NAME',
+        'XENV_IMAGE_PUSH_REGISTRY_HOST',
+        'XENV_IMAGE_PULL_REGISTRY_HOST',
+        'XENV_DOCKER_BASE_IMAGE_PROXY_HOST',
+        'XENV_PIP_INDEX_HOST',
+    )
+
+    def __init__(self):
+        super().__init__({})
+        self._used = False
+
+    def set(self, key, value):
+        self[key] = value
+
+    def __setitem__(self, key, value):
+        self._check_writable(key)
+        dict.__setitem__(self, key, value)
+
+    def update(self, *args, **kwargs):
+        data = dict(*args, **kwargs)
+        for key in data:
+            self._check_writable(key)
+        for key, value in data.items():
+            dict.__setitem__(self, key, value)
+
+    def setdefault(self, key, default=None):
+        self._check_writable(key)
+        return dict.setdefault(self, key, default)
+
+    def pop(self, key, default=None):
+        self._check_mutable()
+        return dict.pop(self, key, default)
+
+    def popitem(self):
+        self._check_mutable()
+        return dict.popitem(self)
+
+    def clear(self):
+        self._check_mutable()
+        dict.clear(self)
+
+    def mark_used(self):
+        self._used = True
+
+    def _check_mutable(self):
+        if self._used:
+            raise RuntimeError('gcontext.env_defaults has already been used to initialize env_config.')
+
+    def _check_writable(self, key):
+        self._check_mutable()
+        if key not in self.allowed_keys:
+            raise KeyError(f'env key is not allowed in gcontext.env_defaults: {key}')
+
+
+class RuntimeStore:
+
+    def __init__(self):
+        self._data = {}
+
+    def set(self, key, value):
+        self._data[key] = value
+
+    def get(self, key, default=None):
+        return self._data.get(key, default)

cpbox/app/contracts.py

@@ -0,0 +1,152 @@
+import os
+from pathlib import Path
+import sys
+import importlib
+
+import dotenv
+
+from cpbox.tool import pkgres
+from cpbox.tool import datatypes
+from cpbox.tool import serde
+
+
+_XAPP_CONFIG_DIR = 'xapp-config'
+_XAPP_STORAGE_DIR = 'xapp-storage'
+_XAPP_ENV_FILE = 'xapp.env'
+
+
+def _detect_start_dir():
+    # 启动目录判定规则：
+    # 1) 正常脚本方式启动（python xxx.py）时，优先使用入口脚本所在目录，后续向上查找 .env 会更符合项目预期。
+    # 2) 交互式场景（如 REPL / notebook / python -c）通常没有 __main__.__file__，回退到当前工作目录 cwd。
+    if hasattr(sys.modules['__main__'], '__file__'):
+        return Path(sys.argv[0]).resolve().parent
+    return Path.cwd()
+
+
+def detect_working_root(start_dir=None):
+    if start_dir is None:
+        start_dir = _detect_start_dir()
+    else:
+        start_dir = Path(start_dir).resolve()
+    while True:
+        if (start_dir / '.env').exists():
+            return start_dir
+        if start_dir == start_dir.parent:
+            return None
+        start_dir = start_dir.parent
+
+
+def is_bundle_runtime():
+    return bool(getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'))
+
+
+def detect_bundle_root_dir():
+    if is_bundle_runtime():
+        return Path(sys._MEIPASS).resolve()
+    return None
+
+
+def _find_package_path(package):
+    module = importlib.import_module(package)
+    package_paths = list(getattr(module, '__path__', []))
+    if package_paths:
+        return Path(package_paths[0]).resolve()
+    if getattr(module, '__file__', None):
+        return Path(module.__file__).resolve().parent
+    return None
+
+
+def package_resource_root(package, source_rel_path, resource_package):
+    package_path = _find_package_path(package)
+    if package_path is not None:
+        source_tree_root = detect_working_root(package_path)
+        if source_tree_root is not None:
+            source_path = source_tree_root / source_rel_path
+            if source_path.exists():
+                return pkgres.ResourceRoot.from_dir(source_path)
+    return pkgres.ResourceRoot.from_package(resource_package)
+
+
+def _resolve_env_path(env_key, fallback):
+    value = os.environ.get(env_key)
+    return Path(value).expanduser() if value else fallback
+
+
+def system_config_root():
+    if sys.platform == 'win32':
+        return _resolve_env_path('PROGRAMDATA', Path('C:/ProgramData')) / _XAPP_CONFIG_DIR
+    return Path('/etc') / _XAPP_CONFIG_DIR
+
+
+def user_config_home():
+    if sys.platform == 'win32':
+        return _resolve_env_path('APPDATA', Path.home() / 'AppData' / 'Roaming')
+    return _resolve_env_path('XDG_CONFIG_HOME', Path.home() / '.config')
+
+
+def user_config_root():
+    return user_config_home() / _XAPP_CONFIG_DIR
+
+
+def system_storage_root():
+    if sys.platform == 'win32':
+        return _resolve_env_path('PROGRAMDATA', Path('C:/ProgramData')) / _XAPP_STORAGE_DIR
+    return Path('/var/lib') / _XAPP_STORAGE_DIR
+
+
+def user_data_home():
+    if sys.platform == 'win32':
+        return _resolve_env_path('LOCALAPPDATA', Path.home() / 'AppData' / 'Local')
+    return _resolve_env_path('XDG_DATA_HOME', Path.home() / '.local' / 'share')
+
+
+def user_storage_root():
+    return user_data_home() / _XAPP_STORAGE_DIR
+
+
+def xapp_default_system_config_file():
+    return system_config_root() / _XAPP_ENV_FILE
+
+
+def xapp_default_user_config_file():
+    return user_config_root() / _XAPP_ENV_FILE
+
+
+def load_system_user_config_file(config_file_path):
+    merged = {}
+    for config_root in (system_config_root(), user_config_root()):
+        path = config_root / config_file_path
+        if not path.exists():
+            continue
+        datatypes.merge(merged, serde.load_yaml_file(path, {}) or {})
+    return datatypes.TypedValueMap(merged)
+
+
+def _load_dotenv_values(env_file, fallback):
+    return dotenv.dotenv_values(env_file) if env_file.exists() else fallback
+
+
+def load_env_config_by_priority(working_root=None, env_defaults=None):
+    # Priority: process env > .env.local > .env > user config > system config > env_defaults > code fallbacks.
+    # env_defaults is a handy code-provided config source; it avoids forcing defaults into .env and stays overrideable by env files and process env.
+    # code fallbacks are caller-provided defaults used when reading env_config values.
+    # Env files are not exported to os.environ.
+    env_data = dict(env_defaults or {})
+    env_data.update(_load_dotenv_values(xapp_default_system_config_file(), {}))
+    env_data.update(_load_dotenv_values(xapp_default_user_config_file(), {}))
+    if working_root:
+        env_data.update(_load_dotenv_values(working_root / '.env', {}))
+        env_data.update(_load_dotenv_values(working_root / '.env.local', {}))
+    env_data = {key: value for key, value in env_data.items() if value is not None}
+    env_data.update(os.environ)
+    return datatypes.TypedValueMap(env_data)
+
+
+def normalize_app_env(app_env=None):
+    # APP_ENV default: missing, empty, or invalid values resolve to prod.
+    app_env = app_env or 'prod'
+    app_env = str(app_env).lower()
+    if app_env not in ('dev', 'test', 'pre', 'prod'):
+        return 'prod'
+    return app_env

cpbox/app/devops.py

@@ -0,0 +1,200 @@
+import os
+import socket
+import sys
+from pathlib import Path
+
+from cpbox.app.appconfig import appconfig
+from cpbox.tool import file
+from cpbox.tool import functocli
+from cpbox.tool import spec
+from cpbox.tool import system
+from cpbox.tool import template
+from cpbox.tool import utils
+from cpbox.app.tools import tibaselog
+
+is_windows = os.name == 'nt'
+
+class DevOpsAppConfig:
+
+    def __init__(self, app_name, provider):
+        appconfig.init(app_name, provider.get_env(), provider.get_app_config())
+
+
+class DevOpsAppConfigContext:
+
+    def __init__(self, app_name, kwargs):
+
+        self.app_base_name = app_name
+        self.kwargs = kwargs
+        self.sandbox_version = kwargs.get('sandbox_version', 0)
+
+        self.env = self._get_env()
+        self.app_name = self._determine_app_name()
+
+        self.construct_baisc_dir()
+
+    def _determine_app_name(self):
+        app_name = self.app_base_name
+        if self.sandbox_version != 0:
+            app_name = '%s-sandbox-%03d' % (app_name, self.sandbox_version)
+        return app_name
+
+    def construct_baisc_dir(self):
+        default_data_dir = '/opt/data' if not is_windows else 'D:\\opt\\data'
+        data_dir = Path(os.environ.get('CPBOX_DATA_DIR', default_data_dir))
+
+        self.app_storage_dir = data_dir / self.app_name
+
+        self.app_persistent_storage_dir = self.app_storage_dir / 'persistent'
+        self.app_runtime_storage_dir = self.app_storage_dir / 'runtime'
+        self.app_logs_dir = self.app_runtime_storage_dir / 'logs'
+
+    def is_dev(self):
+        return self.env == 'dev' or self.env == 'test'
+
+    def _get_env(self):
+        env = 'dev'
+        if 'PUPPY_ENV' in os.environ:
+            env = os.environ['PUPPY_ENV']
+        if 'CPBOX_ENV' in os.environ:
+            env = os.environ['CPBOX_ENV']
+        if 'env' in self.kwargs:
+            env = self.kwargs['env']
+        return env
+
+    def get_sandbox_code_dir(self):
+        sandbox_code_dir = None
+        base_dir = self.app_persistent_storage_dir
+        if self.sandbox_version != 0:
+            sandbox_code_dir = '%s/sandbox-code-%03d' % (base_dir, self.sandbox_version)
+            sandbox_code_dir = Path(sandbox_code_dir)
+        return sandbox_code_dir
+
+    def get_root_dir(self):
+        if hasattr(sys.modules['__main__'], '__file__'):
+            root_dir = Path(sys.argv[0]).resolve().parent
+            return root_dir
+        else:
+            root_dir = Path.cwd()
+            return root_dir
+
+    def ensure_dir_and_write_permission(self):
+        file.ensure_dir(self.app_persistent_storage_dir)
+        file.ensure_dir(self.app_runtime_storage_dir)
+        file.ensure_dir(self.app_logs_dir)
+        file.ensure_dir(self.app_logs_dir / 'syslog')
+
+
+class DevOpsApp(functocli.SimpleApp):
+
+    def __init__(self, app_name, **kwargs):
+        functocli.SimpleApp.__init__(self)
+        self.config_context = DevOpsAppConfigContext(app_name, kwargs)
+        self.env = self.config_context.env
+
+        self.construct_baisc_dir()
+
+        self.logger = self._make_logger(app_name, kwargs)
+
+        self.file_lock = None
+
+    def run_cmd_ret(self, cmd, log=True):
+        return self.run_cmd(cmd, log=log)[1]
+
+    def run_cmd(self, cmd, log=True):
+        if log:
+            self.logger.info('run_cmd: %s', cmd)
+        return system.run_cmd(cmd)
+
+    def shell_run(self, cmd, keep_pipeline=True, exit_on_error=True, dry_run=False, log=True):
+        if log:
+            self.logger.info('shell_run: %s', cmd)
+        if dry_run:
+            return 0
+        return system.shell_run(cmd, keep_pipeline=keep_pipeline, exit_on_error=exit_on_error)
+
+    def _make_logger(self, app_name, kwargs):
+        tibaselog.log_manager.add_file_logger(self.app_logs_dir, f'{app_name}.log')
+        import logging
+        return logging.getLogger(app_name)
+
+    def construct_baisc_dir(self):
+
+        context = self.config_context
+        context.construct_baisc_dir()
+        context.ensure_dir_and_write_permission()
+
+        hostname = socket.gethostname()
+        self.hostname_fqdn = hostname
+        self.hostname_short = hostname.split('.', 1)[0]
+
+        self.root_dir = context.get_sandbox_code_dir() if context.sandbox_version else context.get_root_dir()
+        self._roles_dir = self.root_dir / 'roles'
+
+        app_root_dir = self._roles_dir / context.app_base_name
+
+        self.app_root_dir = app_root_dir
+        self.app_config_dir = app_root_dir / 'config'
+        self.app_templates_dir = app_root_dir / 'templates'
+        self.app_scripts_dir = app_root_dir / 'scripts'
+
+        self.app_storage_dir = context.app_storage_dir
+        self.app_persistent_storage_dir = context.app_persistent_storage_dir
+        self.app_runtime_storage_dir = context.app_runtime_storage_dir
+        self.app_logs_dir = context.app_logs_dir
+
+    def remove_container(self, name, force=False, dry_run=False):
+        cmd = 'docker rm %s' % (name)
+        if force:
+            cmd = 'docker rm -f %s' % (name)
+        return self.shell_run(cmd, exit_on_error=False, dry_run=dry_run)
+
+    def container_is_running(self, name):
+        cmd = 'docker ps | grep %s' % (name)
+        return self.run_cmd(cmd, log=False)[0] == 0
+
+    def stop_container(self, name, timeout=300, dry_run=False):
+        cmd = 'docker stop --time %d %s' % (timeout, name)
+        return self.shell_run(cmd, exit_on_error=False, dry_run=dry_run)
+
+    def _check_lock(self):
+        filepath = self.app_runtime_storage_dir / 'locks' / file.compute_lock_filepath(sys.argv)
+        file_lock = file.obtain_lock(filepath)
+        if file_lock is None:
+            pid = 0
+            with open(filepath, 'r') as f:
+                pid = f.read()
+            self.logger.warning('lock file exists, pid: %s => %s', pid, filepath)
+            sys.exit(1)
+        else:
+            self.file_lock = file_lock
+
+    def template_to(self, template_filename, dst, payload, app_name=None):
+        template_payload = {'payload': payload}
+        src = self.app_templates_dir_for(app_name) / template_filename
+        template.render_to_file(src, template_payload, dst)
+        self.logger.info('template_to: %s => %s', src, dst)
+
+    def read_app_config(self, config_filebasename, app_name=None):
+        if '.yml' not in config_filebasename:
+            config_filebasename = config_filebasename + '.yml'
+        data = utils.load_yaml(self.app_config_dir_for(app_name) / config_filebasename, {})
+        return data
+
+    def app_templates_dir_for(self, app_name=None):
+        if app_name is None:
+            return self.app_templates_dir
+        return self._roles_dir / app_name / 'templates'
+
+    def app_config_dir_for(self, app_name=None):
+        if app_name is None:
+            return self.app_config_dir
+        return self._roles_dir / app_name / 'config'
+
+    @functocli.keep_method_for_inheritance
+    def check_health(self):
+        print('%s is good' % (sys.argv[0]))
+
+    @staticmethod
+    def run_app(app, log_level='info', default_method=None, common_args_option=None):
+        functocli.run_app(app, default_method=default_method)

cpbox/app/env.py

@@ -0,0 +1,34 @@
+from cpbox.tool import cache
+from cpbox.tool import utils
+
+
+class EnvTool(object):
+
+    def __init__(self, app):
+        self.app = app
+        fn = self.app.root_dir / '.box-config.yml'
+        self.box_config = utils.load_yaml(fn, {})
+
+    def docker_url_by_env(self, image_name):
+        group_names = self.app.get_group_names()
+        if 'nexus-genesis' in group_names:
+            return image_name
+        if 'firelighter-server' in group_names:
+            return self.docker_genesis_url(image_name)
+        else:
+            return self.docker_url(image_name)
+
+    def docker_url(self, image_name=''):
+        return self.box_config.get('docker_url_pre') + image_name
+
+    def docker_publish_url(self, image_name=''):
+        return self.box_config.get('docker_publish_url_pre') + image_name
+
+    def docker_genesis_url(self, image_name=''):
+        return self.box_config.get('docker_genesis_url_pre') + image_name
+
+    def setup_redis_for_cache(self):
+        config = self.box_config[self.app.env]
+        redis = config['redis']
+        redis_url = 'redis://:%s@%s:%s' % (redis['auth'], redis['host'], redis['port'])
+        cache.set_redis_url(redis_url)

cpbox/app/eventlog.py

@@ -0,0 +1,40 @@
+import json
+import logging
+import time
+from cpbox.tool import timeutil
+
+from cpbox.app.appconfig import appconfig
+
+event_logger = logging.getLogger('event-log')
+
+
+def add_event_log(event_key, payload):
+    payload['env'] = appconfig.get_env()
+    msg = 'cp %s %s' % (event_key, json.dumps(payload))
+    event_logger.info(msg)
+
+def add_app_monitor_log(key, value, time=None):
+    time = time if time else timeutil.local_now_iso8601_str()
+    data = {
+            'event_key': key,
+            'event_value': value,
+            'time': time,
+            }
+    payload = {}
+    payload['env'] = appconfig.get_env()
+    payload['log'] = json.dumps(data)
+    msg = 'app-monitor monitor-log %s' % (json.dumps(payload))
+    event_logger.info(msg)
+
+
+def log_func_call(func, *args, **kwargs):
+    def timed(*args, **kw):
+        start = time.time() * 1000
+        result = func(*args, **kw)
+        payload = {}
+        payload['name'] = func.__name__
+        payload['rt'] = time.time() * 1000 - start
+        add_event_log('func-call', payload)
+        return result
+
+    return timed