jaseci 1.4.0.9__py3-none-any.whl → 1.4.0.11__py3-none-any.whl

jaseci/svc/config.py

@@ -1,9 +0,0 @@
-KUBERNETES_CONFIG = {"in_cluster": True, "config": None}
-
-META_CONFIG = {
-    "automation": False,
-    "backoff_interval": 10,
-    "namespace": "default",
-    "keep_alive": ["promon", "redis", "task", "mail", "elastic"],
-    "kubernetes": KUBERNETES_CONFIG,
-}

jaseci/svc/elastic/__init__.py

@@ -1,3 +0,0 @@
-from .elastic import ElasticService
-
-__all__ = ["ElasticService"]

jaseci/svc/elastic/config.py

@@ -1,8 +0,0 @@
-ELASTIC_CONFIG = {
-    "enabled": False,
-    "quiet": True,
-    "url": "",
-    "auth": "",
-    "common_index": "",
-    "activity_index": "",
-}

jaseci/svc/elastic/manifest.py

@@ -1 +0,0 @@
-ELASTIC_MANIFEST = {}

jaseci/svc/jsorc-backup/jsorc.py

@@ -1,182 +0,0 @@
-from .promon.promon import Promon
-import time
-from kubernetes import client, config
-from kubernetes.client.rest import ApiException
-import os
-import yaml
-import multiprocessing
-from jaseci.utils.utils import logger
-
-
-class KubeController:
-    # A set of all functions that are helpful for kubernetes operations
-    # Configs can be set in Configuration class directly or using helper utility
-
-    def __init__(self, configuration):
-        self.config = configuration
-        self.api_instance = client.CoreV1Api(self.config)
-        self.app_client = client.ApiClient(self.config)
-        self.app_api = client.AppsV1Api(self.app_client)
-
-    def get_pod_list(self):
-        ret = self.api_instance.list_pod_for_all_namespaces(watch=False)
-        res = []
-        for i in ret.items:
-            res.append({"namespace": i.metadata.namespace, "name": i.metadata.name})
-        return res
-
-    def get_deployment_list(self):
-        ret = self.app_api.list_deployment_for_all_namespaces(watch=False)
-        res = []
-        for i in ret.items:
-            res.append({"namespace": i.metadata.namespace, "name": i.metadata.name})
-        return res
-
-    def create_deployment(self, config: dict, namespace: str = "default"):
-        return self.app_api.create_namespaced_deployment(
-            namespace=namespace, body=config
-        )
-
-    def get_deployment_conf(self, name: str, namespace: str = "default"):
-        api_response = self.app_api.read_namespaced_deployment(
-            name=name, namespace=namespace
-        )
-        return api_response
-
-    def patch_deployment_conf(self, config, name: str, namespace: str = "default"):
-        api_response = self.app_api.patch_namespaced_deployment(
-            name=name, namespace=namespace, body=config
-        )
-        return api_response
-
-    def deployment_set_scale(
-        self, name: str, namespace: str = "default", scale: int = 1
-    ):
-        # This is just a small shortcut to set the scale of a deployment
-        conf = self.get_deployment_conf(name, namespace)
-        conf.spec.replicas = scale
-        self.patch_deployment_conf(conf, name, namespace)
-
-    def kill_deployment(self, name: str, namespace: str = "default"):
-        try:
-            api_response = self.app_api.delete_namespaced_deployment(
-                name=name, namespace=namespace
-            )
-            return api_response
-        except ApiException as e:
-            print("Exception when calling CoreV1Api->delete_namespaced_pod: %s\n" % e)
-
-
-class Monitor:
-    def __init__(self, promon_url: str, k8sconfig):
-        self.promon = Promon(promon_url)
-        self.controller = KubeController(k8sconfig)
-
-    def strategy_start_redis(self):
-        deployments = self.controller.get_deployment_list()
-        exsits = False
-        for deployment in deployments:
-            if deployment["name"] == "jaseci-redis":
-                exsits = True
-        if not exsits:
-            logger.info("Creating new jaseci redis")
-            dirpath = os.path.dirname(os.path.realpath(__file__))
-            filepath = os.path.join(dirpath, "jaseci-redis.yaml")
-            self.controller.create_deployment(
-                config=yaml.safe_load(open(filepath, "r"))
-            )
-
-    def strategy_redis_cpu(
-        self, node_name: str, deployment_namespace: str, deployment_name: str
-    ):
-        cpu = self.promon.cpu_utilization_percentage()
-        cpu_usage = cpu[node_name]
-        print(f"Detect CPU Usage: {cpu_usage}")
-        if cpu_usage > 10:
-            pods = self.controller.get_deployment_list()
-            for pod in pods:
-                namespace = pod["namespace"]
-                name = pod["name"]
-                if name == deployment_name:
-                    print("Kill deployment")
-                    self.controller.kill_deployment(name=name, namespace=namespace)
-        if cpu_usage < 5:
-            pods = self.controller.get_deployment_list()
-            redis_running = False
-            for pod in pods:
-                if pod["name"] == deployment_name:
-                    redis_running = True
-            if not redis_running:
-                print("Creating new deployment")
-                self.controller.create_deployment(
-                    config=yaml.safe_load(open("jaseci.yaml", "r"))
-                )
-
-    def strategy_service_cpu(self):
-        cpu = self.promon.cpu_utilization_per_pod_cores()
-        count = 0
-        total = 0
-        for pod_name in cpu.keys():
-            if pod_name.startswith("jaseci-redis"):
-                count = count + 1
-                total = total + cpu[pod_name]
-
-        avg = total / count
-        print(avg)
-        if avg > 0.001:
-            conf = self.controller.get_deployment_conf("jaseci-redis", "default")
-            replicas = conf.spec.replicas + 1
-            print(f"Num of Replicas: {replicas}")
-            self.controller.deployment_set_scale("jaseci-redis", "default", replicas)
-        if avg < 0.001:
-            conf = self.controller.get_deployment_conf("jaseci-redis", "default")
-            replicas = conf.spec.replicas - 1
-            print(f"Num of Replicas: {replicas}")
-            self.controller.deployment_set_scale("jaseci-redis", "default", replicas)
-
-
-def daemon(k8s_conf, prometheus_url: str):
-    m = Monitor(prometheus_url, k8s_conf)
-    while True:
-        # m.strategy_redis_cpu("minikube", "default", "jaseci-redis")
-        m.strategy_start_redis()
-        time.sleep(10)
-
-
-def start_monitoring(k8s_conf, prometheus_url: str):
-    monitor = multiprocessing.Process(target=daemon, args=(k8s_conf, prometheus_url))
-    monitor.start()
-    return monitor
-
-
-def wait_monitoring(monitor_thread):
-    monitor_thread.join()
-
-
-def stop_monitoring(monitor_thread):
-    monitor_thread.terminate()
-
-
-def remote_k8s_conf(k8s_url: str):
-    k8sconf = client.Configuration()
-    k8sconf.host = k8s_url
-    k8sconf.verify_ssl = False
-    return k8sconf
-
-
-def incluster():
-    try:
-        config.load_incluster_config()
-    except config.config_exception.ConfigException:
-        return False
-    return True
-
-
-def jsorc_start():
-    logger.info("JSORC Running")
-    if incluster():
-        k8s_config = None
-        start_monitoring(
-            k8s_conf=k8s_config, prometheus_url="http://js-prometheus:9090"
-        )
-        logger.info("Monitoring started")

jaseci/svc/jsorc-backup/promon/promon.py

@@ -1,202 +0,0 @@
-from prometheus_api_client import PrometheusConnect
-
-
-class Promon:
-    def __init__(self, url: str):
-        self.prom = PrometheusConnect(url=url, disable_ssl=True)
-
-    def all_metrics(self) -> list:
-        return self.prom.all_metrics()
-
-    def cpu_utilization_core(self) -> dict:
-        util = self.prom.get_current_metric_value(
-            'sum(irate(node_cpu_seconds_total{mode!="idle"}[10m])) by (node)'
-        )
-        res = {}
-        for node in util:
-            node_name = node["metric"]["node"]
-            node_util = float(node["value"][1])
-            res[node_name] = node_util
-        return res
-
-    def cpu_utilization_percentage(self) -> dict:
-        util = self.prom.get_current_metric_value(
-            '(sum(irate(node_cpu_seconds_total{mode!="idle"}[10m])) by '
-            '(node)) / (sum(irate(node_cpu_seconds_total{mode!=""}[10m])) by '
-            "(node)) * 100"
-        )
-        res = {}
-        for node in util:
-            node_name = node["metric"]["node"]
-            node_util = float(node["value"][1])
-            res[node_name] = node_util
-        return res
-
-    def cpu_utilization_per_pod_cores(self) -> dict:
-        util = self.prom.get_current_metric_value(
-            'sum(irate(container_cpu_usage_seconds_total{pod!=""}[10m])) by (pod)'
-        )
-        res = {}
-        for pod in util:
-            pod_name = pod["metric"]["pod"]
-            value = float(pod["value"][1])
-            res[pod_name] = float(value)
-        return res
-
-    def mem_total_bytes(self) -> dict:
-        util = self.prom.get_current_metric_value(
-            "sum(node_memory_MemTotal_bytes) by (node)"
-        )
-        res = {}
-        for node in util:
-            node_name = node["metric"]["node"]
-            node_util = float(node["value"][1])
-            res[node_name] = node_util
-        return res
-
-    def mem_utilization_bytes(self) -> dict:
-        util = self.prom.get_current_metric_value(
-            "sum(node_memory_Active_bytes) by (node)"
-        )
-        res = {}
-        for node in util:
-            node_name = node["metric"]["node"]
-            node_util = float(node["value"][1])
-            res[node_name] = node_util
-        return res
-
-    def mem_utilization_percentage(self) -> dict:
-        util = self.prom.get_current_metric_value(
-            "sum(node_memory_Active_bytes / node_memory_MemTotal_bytes * 100 ) by "
-            "(node)"
-        )
-        res = {}
-        for node in util:
-            node_name = node["metric"]["node"]
-            node_util = float(node["value"][1])
-            res[node_name] = node_util
-        return res
-
-    def mem_utilization_per_pod_bytes(self) -> dict:
-        util = self.prom.get_current_metric_value(
-            'sum(container_memory_working_set_bytes{pod!=""}) by (pod)'
-        )
-        res = {}
-        for pod in util:
-            pod_name = pod["metric"]["pod"]
-            value = float(pod["value"][1])
-            res[pod_name] = float(value)
-        return res
-
-    def node_pods(self) -> dict:
-        util = self.prom.get_current_metric_value("kube_pod_info")
-        res = {}
-        for pod in util:
-            info = pod["metric"]
-            node = info["node"]
-            pod = info["pod"]
-            if res.get(node) is None:
-                res[node] = set()
-            res[node].add(pod)
-        return res
-
-    def network_receive_bytes(self) -> dict:
-        util = self.prom.get_current_metric_value(
-            "sum (rate (node_network_receive_bytes_total{}[10m])) by (node)"
-        )
-        res = {}
-        for node in util:
-            node_name = node["metric"]["node"]
-            node_util = float(node["value"][1])
-            res[node_name] = node_util
-        return res
-
-    def network_receive_per_pod_bytes(self):
-        util = self.prom.get_current_metric_value(
-            'sum (rate (container_network_receive_bytes_total{pod!=""}[10m])) by (pod)'
-        )
-        res = {}
-        for pod in util:
-            pod_name = pod["metric"]["pod"]
-            value = pod["value"][1]
-            res[pod_name] = float(value)
-        return res
-
-    def network_transmit_bytes(self) -> dict:
-        util = self.prom.get_current_metric_value(
-            "sum (rate (node_network_transmit_bytes_total{}[10m])) by (node)"
-        )
-        res = {}
-        for node in util:
-            node_name = node["metric"]["node"]
-            node_util = float(node["value"][1])
-            res[node_name] = node_util
-        return res
-
-    def network_transmit_per_pod_bytes(self):
-        util = self.prom.get_current_metric_value(
-            'sum (rate (container_network_transmit_bytes_total{pod!=""}[10m])) by (pod)'
-        )
-        res = {}
-        for pod in util:
-            pod_name = pod["metric"]["pod"]
-            value = pod["value"][1]
-            res[pod_name] = float(value)
-        return res
-
-    def disk_total_bytes(self) -> dict:
-        util = self.prom.get_current_metric_value(
-            'sum(avg (node_filesystem_size_bytes{mountpoint!="/boot", '
-            'fstype!="tmpfs"}) without (mountpoint)) by (node)'
-        )
-        res = {}
-        for node in util:
-            node_name = node["metric"]["node"]
-            node_util = float(node["value"][1])
-            res[node_name] = node_util
-        return res
-
-    def disk_free_bytes(self) -> dict:
-        util = self.prom.get_current_metric_value(
-            'sum(avg (node_filesystem_free_bytes{mountpoint!="/boot", '
-            'fstype!="tmpfs"}) without (mountpoint)) by (node)'
-        )
-        res = {}
-        for node in util:
-            node_name = node["metric"]["node"]
-            node_util = float(node["value"][1])
-            res[node_name] = node_util
-        return res
-
-    def pod_info(self) -> dict:
-        util = self.prom.get_current_metric_value("kube_pod_info")
-        res = {}
-        for pod in util:
-            pod_name = pod["metric"]["pod"]
-            res[pod_name] = pod["metric"]
-
-        cpu = self.cpu_utilization_per_pod_cores()
-        for pod in util:
-            pod_name = pod["metric"]["pod"]
-            pod_cpu = cpu.get(pod_name, 0)
-            res[pod_name]["cpu_utilization_cores"] = pod_cpu
-
-        mem = self.mem_utilization_per_pod_bytes()
-        for pod in util:
-            pod_name = pod["metric"]["pod"]
-            pod_mem = mem.get(pod_name, 0)
-            res[pod_name]["mem_utilization_bytes"] = pod_mem
-
-        recv = self.network_receive_per_pod_bytes()
-        for pod in util:
-            pod_name = pod["metric"]["pod"]
-            pod_recv = recv.get(pod_name, 0)
-            res[pod_name]["network_recv_bytes"] = pod_recv
-
-        tran = self.network_transmit_per_pod_bytes()
-        for pod in util:
-            pod_name = pod["metric"]["pod"]
-            pod_tran = tran.get(pod_name, 0)
-            res[pod_name]["network_tran_bytes"] = pod_tran
-
-        return res

jaseci/svc/mail/__init__.py

@@ -1,4 +0,0 @@
-from .config import MAIL_CONFIG
-from .mail import MailService, Mailer
-
-__all__ = ["MAIL_CONFIG", "MailService", "Mailer"]

jaseci/svc/mail/config.py

@@ -1,25 +0,0 @@
-MAIL_CONFIG = {
-    "enabled": False,
-    "quiet": True,
-    "version": 2,
-    "tls": True,
-    "host": "",
-    "port": 587,
-    "sender": "",
-    "user": "",
-    "pass": "",
-    "backend": "smtp",
-    "templates": {
-        "activation_subj": "Please activate your account!",
-        "activation_body": "Thank you for creating an account!\n\n"
-        "Activation Code: {{code}}\n"
-        "Please click below to activate:\n{{link}}",
-        "activation_html_body": "Thank you for creating an account!<br><br>"
-        "Activation Code: {{code}}<br>"
-        "Please click below to activate:<br>"
-        "{{link}}",
-        "resetpass_subj": "Password Reset for Jaseci Account",
-        "resetpass_body": "Your Jaseci password reset token is: {{token}}",
-        "resetpass_html_body": "Your Jaseci password reset" "token is: {{token}}",
-    },
-}

jaseci/svc/meta.py

@@ -1,164 +0,0 @@
-import signal
-from jaseci.utils.utils import logger
-from jaseci.svc import (
-    CommonService,
-    JsOrc,
-    MetaProperties,
-    MailService,
-    RedisService,
-    TaskService,
-    StripeService,
-    PrometheusService,
-    ElasticService,
-    ServiceState as Ss,
-)
-
-
-class MetaService(CommonService, MetaProperties):
-    def __init__(self, run_svcs=True):
-        self.run_svcs = run_svcs
-        MetaProperties.__init__(self, __class__)
-
-        self.start()
-
-    ###################################################
-    #                     BUILDER                     #
-    ###################################################
-
-    def run(self, hook=None):
-        self.app = JsOrc(self)
-        self.populate_context()
-        self.populate_services()
-
-    def post_run(self, hook=None):
-        if self.run_svcs:
-            self.app.build()
-            if self.is_automated():
-                logger.info("JsOrc is automated. Pushing interval check alarm...")
-                self.push_interval(1)
-            else:
-                logger.info("JsOrc is not automated.")
-
-    def push_interval(self, interval):
-        if self.running_interval == 0:
-            self.running_interval += 1
-            signal.alarm(interval)
-        else:
-            logger.info("Reusing current running interval...")
-
-    ###################################################
-    #                    SERVICES                     #
-    ###################################################
-
-    def add_service_builder(self, name, svc):
-        self.app.add_service_builder(name, svc)
-
-    def build_service(self, name, background, *args, **kwargs):
-        return self.app.build_service(name, background, *args, **kwargs)
-
-    def get_service(self, name, *args, **kwargs):
-        return self.app.get_service(name, *args, **kwargs)
-
-    ###################################################
-    #                    CONTEXT                      #
-    ###################################################
-
-    def add_context(self, ctx, cls, *args, **kwargs):
-        self.app.add_context(ctx, cls, *args, **kwargs)
-
-    def build_context(self, ctx, *args, **kwargs):
-        return self.app.build_context(ctx, *args, **kwargs)
-
-    def get_context(self, ctx):
-        return self.app.context.get(ctx, {}).get("class")
-
-    ###################################################
-    #                     COMMON                      #
-    ###################################################
-
-    def is_automated(self):
-        return self.is_running() and self.app and self.app.automated
-
-    def in_cluster(self):
-        return self.app.in_cluster()
-
-    ###################################################
-    #                     BUILDER                     #
-    ###################################################
-
-    def build_hook(self):
-        h = self.build_context("hook")
-        h.meta = self
-        if self.run_svcs:
-            h.promon = self.get_service("promon", h)
-            h.redis = self.get_service("redis", h)
-            h.task = self.get_service("task", h)
-            h.mail = self.get_service("mail", h)
-            h.elastic = self.get_service("elastic", h)
-            h.stripe = self.get_service("stripe", h)
-
-            if not self.is_automated():
-                h.mail.start(h)
-                h.redis.start(h)
-                h.task.start(h)
-                h.elastic.start(h)
-                h.stripe.start(h)
-
-        return h
-
-    def build_master(self, *args, **kwargs):
-        return self.__common("master", *args, **kwargs)
-
-    def build_super_master(self, *args, **kwargs):
-        return self.__common("super_master", *args, **kwargs)
-
-    def __common(self, ctx, *args, **kwargs):
-        if not kwargs.get("h", None):
-            kwargs["h"] = self.build_hook()
-
-        return self.build_context(ctx, *args, **kwargs)
-
-    ###################################################
-    #                   OVERRIDEN                     #
-    ###################################################
-
-    def reset(self, hook=None, start=True):
-        self.terminate_daemon("jsorc")
-        self.app = None
-        self.state = Ss.NOT_STARTED
-        self.__init__()
-
-    def populate_context(self):
-        from jaseci.hook import RedisHook
-        from jaseci.element.master import Master
-        from jaseci.element.super_master import SuperMaster
-
-        self.add_context("hook", RedisHook)
-        self.add_context("master", Master)
-        self.add_context("super_master", SuperMaster)
-
-    def populate_services(self):
-        self.add_service_builder("redis", RedisService)
-        self.add_service_builder("task", TaskService)
-        self.add_service_builder("mail", MailService)
-        self.add_service_builder("promon", PrometheusService)
-        self.add_service_builder("elastic", ElasticService)
-        self.add_service_builder("stripe", StripeService)
-
-
-def interval_check(signum, frame):
-    meta = MetaService()
-    if meta.is_automated():
-        meta.app.interval_check()
-        logger.info(
-            f"Backing off for {meta.app.backoff_interval} seconds before the next interval check..."
-        )
-
-        # wait interval_check to be finished before decrement
-        meta.running_interval -= 1
-        meta.push_interval(meta.app.backoff_interval)
-    else:
-        meta.running_interval -= 1
-
-
-signal.signal(signal.SIGALRM, interval_check)