openkosmos-core 0.1.2__tar.gz

openkosmos_core-0.1.2/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.3
+Name: openkosmos-core
+Version: 0.1.2
+Summary: openkosmos-core
+Author: wangtao
+Author-email: wangtao <wangtao@openkosmos.com>
+Requires-Dist: jinja2>=3.1.6
+Requires-Dist: parsel>=1.10.0
+Requires-Dist: pyjwt>=2.10.1
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: tqdm>=4.67.1
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+openkosmos-core

openkosmos_core-0.1.2/README.md

@@ -0,0 +1 @@
+openkosmos-core

openkosmos_core-0.1.2/pyproject.toml

@@ -0,0 +1,24 @@
+[project]
+name = "openkosmos-core"
+version = "0.1.2"
+description = "openkosmos-core"
+readme = "README.md"
+authors = [
+    { name = "wangtao", email = "wangtao@openkosmos.com" }
+]
+requires-python = ">=3.12"
+dependencies = [
+    "jinja2>=3.1.6",
+    "parsel>=1.10.0",
+    "pyjwt>=2.10.1",
+    "pyyaml>=6.0.3",
+    "tqdm>=4.67.1",
+]
+
+[build-system]
+requires = ["uv_build>=0.9.15,<0.10.0"]
+build-backend = "uv_build"
+
+[[tool.uv.index]]
+url = "https://pypi.tuna.tsinghua.edu.cn/simple"
+default = true

openkosmos_core-0.1.2/src/openkosmos_core/__init__.py

@@ -0,0 +1,4 @@
+name = "openkosmos-core"
+
+def hello() -> str:
+    return "Hello from openkosmos-core!"

openkosmos_core-0.1.2/src/openkosmos_core/arguments.py

@@ -0,0 +1,64 @@
+import argparse
+import logging
+import logging.config
+import os
+import sys
+from multiprocessing import Lock
+
+CONTAINER_ENV = os.environ.get("CONTAINER_ENV", "DEFAULT")
+
+
+class Argument:
+    __log_init = False
+    if CONTAINER_ENV == "WSL_DOCKER":
+        __init_lock = None
+    else:
+        __init_lock = Lock()
+
+    def __init__(self, instance):
+        self.__parser = argparse.ArgumentParser()
+        Argument.__current = instance
+
+    def param(self, name, default=None, help=None, action=None):
+        self.__parser.add_argument("-" + name, default=default, help=help, action=action)
+        return self
+
+    def parse(self, args=sys.argv[1:]):
+        self.__args = self.__parser.parse_args(args)
+        self.__parameters = vars(self.__args)
+        return self
+
+    def get(self, name):
+        return self.__parameters.get(name)
+
+    def __str__(self):
+        return str(self.__parameters)
+
+    def usage(self):
+        self.__parser.print_help()
+
+    @classmethod
+    def current(cls):
+        return Argument.__current
+
+    @classmethod
+    def set_log_config(cls, config_file, create_logs_dir=True):
+        Argument.__config_file = config_file
+        Argument.__create_logs_dir = create_logs_dir
+
+    @classmethod
+    def logger(cls, tag):
+        if not Argument.__log_init:
+            if Argument.__init_lock is not None:
+                Argument.__init_lock.acquire()
+            if not Argument.__log_init:
+                config_file = os.getenv("KOSMOS_LOG_CONFIG", default="log.conf")
+                create_log_dir = os.environ.get("KOSMOS_LOG_DIR", default=".log")
+                os.makedirs(create_log_dir, exist_ok=True)
+                logging.config.fileConfig(config_file)
+                Argument.__log_init = True
+            else:
+                return logging.getLogger(tag)
+            if Argument.__init_lock is not None:
+                Argument.__init_lock.release()
+        return logging.getLogger(tag)

openkosmos_core-0.1.2/src/openkosmos_core/auth.py

@@ -0,0 +1,30 @@
+import time
+
+import jwt
+
+
+class AuthToken:
+    def __init__(self, secret_key: str):
+        self.secret_key = secret_key.strip()
+        self.algorithms = "HS256"
+
+    def generate(self, data: dict, expired_seconds: int = 300) -> str:
+        payload = data;
+        payload["exp"] = int(time.time()) + expired_seconds
+        return jwt.encode(payload, self.secret_key, algorithm=self.algorithms);
+
+    def retrieve(self, token: str):
+        payload = jwt.decode(token.strip(), self.secret_key, algorithms=[self.algorithms])
+        del payload["exp"]
+        return payload
+
+    def generate_auth_headers(self, data: dict, expired_seconds: int = 300) -> str:
+        payload = data;
+        payload["exp"] = int(time.time()) + expired_seconds
+
+        gateway_headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.generate(data, expired_seconds)}"
+        }
+
+        return gateway_headers;

openkosmos_core-0.1.2/src/openkosmos_core/config.py

@@ -0,0 +1,36 @@
+import json
+import os
+
+import yaml
+
+
+class Config:
+    config = {}
+
+    def __init__(self, config_file=None):
+        self.load(config_file)
+
+    def load(self, config_file=None):
+        c_file = config_file
+        if c_file is None:
+            c_file = os.getenv("KOSMOS_CONFIG", default="config.yml")
+        with open(c_file, "r", encoding="utf-8") as f:
+            self.config = yaml.load(f, Loader=yaml.FullLoader)
+            return self.config
+
+    def get(self, name: str | list[str], config_type=None):
+        last = None
+        if type(name) is str:
+            last = self.config.get(name)
+        else:
+            last = self.config
+            for n in name:
+                last = last.get(n)
+
+        if config_type is None:
+            return last
+        else:
+            return config_type(**last)
+
+    def __str__(self):
+        return json.dumps(self.config, indent=3, ensure_ascii=False)

openkosmos_core-0.1.2/src/openkosmos_core/exception.py

@@ -0,0 +1,11 @@
+class FatalException(Exception):
+    status_code = 505
+
+    def __init__(self, message, detail=None):
+        Exception.__init__(self)
+        self.message = message
+        self.detail = detail
+
+    def to_dict(self):
+        value = {"message": self.message, "detail": dict(self.detail or ())}
+        return value

openkosmos_core-0.1.2/src/openkosmos_core/executor.py

@@ -0,0 +1,75 @@
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from tqdm import tqdm
+
+from openkosmos_core.setting import Setting
+
+log = Setting.logger("executor")
+
+
+class TaskExecutor:
+
+    def __init__(self, fn, completed_callback=None, delay=0, meta={}, exception_callback=None):
+        self.fn = fn
+        self.completed_callback = completed_callback
+        if exception_callback is None:
+            self.exception_callback = lambda exception: True
+        else:
+            self.exception_callback = exception_callback
+        self.delay = delay
+        self.meta = meta
+
+    def start(self, feed, max_worker=1, wait=True, show_progress=True):
+        if wait:
+            start = time.time()
+            with ThreadPoolExecutor(max_workers=max_worker) as executor:
+                futures = {executor.submit(self.execute, param): param for param in feed}
+                results = []
+                if show_progress:
+                    with tqdm(total=len(feed), desc="progress") as progress_bar:
+                        for future in as_completed(futures):
+                            param = futures[future]
+                            try:
+                                result = future.result()
+                                results.append(result)
+                                progress_bar.update(1)
+                            except Exception as e:
+                                log.exception(e)
+                                log.error("an exception when execute : {}, {}".format(param, e))
+                                if self.exception_callback(e):
+                                    progress_bar.close()
+                                    executor.shutdown()
+                                    return results
+                else:
+                    for future in as_completed(futures):
+                        param = futures[future]
+                        try:
+                            result = future.result()
+                            results.append(result)
+                        except Exception as e:
+                            log.exception(e)
+                            log.error("an exception when execute : {}, {}".format(param, e))
+                            if self.exception_callback(e):
+                                executor.shutdown()
+                                return results
+
+                log.info("all completed...")
+                if self.completed_callback is not None:
+                    self.completed_callback(results, self.meta)
+                else:
+                    log.info("all done[{:.3f}] with results : {}, {}".format(time.time() - start, results,
+                                                                             self.meta))
+                return results
+        else:
+            executor = ThreadPoolExecutor(max_workers=max_worker)
+            futures = {executor.submit(self.execute, param): param for param in feed}
+            return futures
+
+    def execute(self, param):
+        # log.debug("start : {}".format(param))
+        if self.delay > 0:
+            time.sleep(self.delay)
+        result = self.fn(param, self.meta)
+        # log.debug("stop : {}".format(param))
+        return result

openkosmos_core-0.1.2/src/openkosmos_core/html_extractor.py

@@ -0,0 +1,57 @@
+from types import SimpleNamespace as T
+
+import yaml
+from parsel import Selector
+
+
+class HtmlExtractor:
+
+    def __init__(self, template):
+        self.__template = template
+
+    def __extract(self, current, parent_bean, beans, kid):
+        node = current.xpath(kid["xpath"])
+        name = kid.get("name")
+        bean_name = kid.get("bean")
+        kids = kid.get("kids")
+        p_bean = parent_bean
+
+        if name is not None:
+            if parent_bean is not None:
+                parent_bean[name] = node.get()
+            elif name is not None:
+                beans.get(bean_name)[name] = node.get()
+
+        if kids is not None:
+            if bean_name is not None and name is None:
+                for n in node:
+                    p_bean = dict()
+                    for k in kid["kids"]:
+                        self.__extract(n, p_bean, beans, k)
+                    beans.get(bean_name).append(p_bean)
+            else:
+                for k in kid["kids"]:
+                    self.__extract(node, p_bean, beans, k)
+
+    def __create_extractor(self, content):
+        selector = Selector(text=content)
+        beans = dict()
+        for b in self.__template["beans"]:
+            if b.get("type") == "list":
+                beans[b["name"]] = []
+            else:
+                beans[b["name"]] = dict()
+        return T(selector=selector, beans=beans)
+
+    def extract(self, content):
+        extractor = self.__create_extractor(content)
+        for kid in self.__template["template"]:
+            self.__extract(extractor.selector, None, extractor.beans, kid)
+        return extractor.beans
+
+    @classmethod
+    def test(cls, templatefile, contentfile):
+        with open(templatefile) as tf:
+            template = yaml.load(tf, Loader=yaml.FullLoader)
+            with open(contentfile, "r", encoding="utf8") as ff:
+                return HtmlExtractor(template).extract(str(ff.read()))

openkosmos_core-0.1.2/src/openkosmos_core/setting.py

@@ -0,0 +1,43 @@
+from openkosmos_core import arguments
+
+
+class Setting(arguments.Argument):
+    def __init__(self, params=None):
+        super().__init__(instance=self)
+        if params is not None:
+            for name, value in params.items():
+                self.param(name, default=value.get("default"), help=value.get("help"),
+                           action=value.get("action"))
+        self.parse()
+        Setting.logger("setting").info(str(self))
+
+    def get_param(self, name):
+        return self.get(name)
+
+    def get_str_param(self, name, default_value=None):
+        value = self.get(name)
+        if value is not None:
+            return str(value)
+        else:
+            return default_value
+
+    def get_int_param(self, name, default_value=None):
+        value = self.get(name)
+        if value is not None:
+            return int(value)
+        else:
+            return default_value
+
+    def get_float_param(self, name, default_value=None):
+        value = self.get(name)
+        if value is not None:
+            return float(value)
+        else:
+            return default_value
+
+    def get_bool_param(self, name, default_value=None):
+        value = self.get(name)
+        if value is not None:
+            return bool(value)
+        else:
+            return default_value

openkosmos_core-0.1.2/src/openkosmos_core/template_loader.py

@@ -0,0 +1,2 @@
+class TemplateLoader:
+    pass