jaseci 1.4.0.9__py3-none-any.whl → 1.4.0.11__py3-none-any.whl

jaseci/{svc/jsorc-backup/jaseci-redis.yaml → manifests/redis.yaml}

@@ -1,3 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: jaseci-redis
+spec:
+  selector:
+    pod: jaseci-redis
+  ports:
+    - protocol: TCP
+      port: 6379
+      targetPort: 6379
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -39,3 +50,12 @@ spec:
             items:
               - key: redis-config
                 path: redis.conf
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: jaseci-redis-config
+data:
+  redis-config: |
+    maxmemory 1000mb
+    maxmemory-policy allkeys-lru

jaseci/svc/__init__.py

@@ -1,25 +0,0 @@
-from .state import ServiceState
-from .common import CommonService, ProxyService, JsOrc, MetaProperties, Kube
-from .redis import RedisService
-from .task import TaskService
-from .stripe import StripeService
-from .mail import MailService
-from .prometheus import PrometheusService
-from .elastic import ElasticService
-from .meta import MetaService
-
-__all__ = [
-    "ServiceState",
-    "JsOrc",
-    "MetaProperties",
-    "CommonService",
-    "ProxyService",
-    "RedisService",
-    "TaskService",
-    "StripeService",
-    "MailService",
-    "PrometheusService",
-    "ElasticService",
-    "MetaService",
-    "Kube",
-]

jaseci/svc/{elastic/elastic.py → elastic_svc.py}

@@ -1,6 +1,4 @@
-from jaseci.svc import CommonService
-from .config import ELASTIC_CONFIG
-from .manifest import ELASTIC_MANIFEST
+from jaseci import JsOrc
 from requests import get, post
 from datetime import datetime
 from copy import copy
@@ -11,28 +9,19 @@ from copy import copy
 #################################################
 
 
-class ElasticService(CommonService):
+@JsOrc.service(name="elastic", config="ELASTIC_CONFIG", manifest="ELASTIC_MANIFEST")
+class ElasticService(JsOrc.CommonService):
     ###################################################
     #                     BUILDER                     #
     ###################################################
 
-    def run(self, hook=None):
+    def run(self):
         self.app = Elastic(self.config)
         self.app.health("timeout=1s")
 
-    ####################################################
-    #                    OVERRIDDEN                    #
-    ####################################################
-
-    def build_config(self, hook) -> dict:
-        return hook.service_glob("ELASTIC_CONFIG", ELASTIC_CONFIG)
-
-    def build_manifest(self, hook) -> dict:
-        return hook.service_glob("ELASTIC_MANIFEST", ELASTIC_MANIFEST)
-
 
 class Elastic:
-    def __init__(self, config):
+    def __init__(self, config: dict):
         if not config.get("url"):
             raise Exception("URL is required!")
         self.url = config["url"]

jaseci/svc/kube_svc.py

@@ -0,0 +1,240 @@
+from jaseci import JsOrc
+
+from kubernetes import config as kubernetes_config
+from kubernetes.client import (
+    ApiClient,
+    CoreV1Api,
+    AppsV1Api,
+    RbacAuthorizationV1Api,
+    ApiextensionsV1Api,
+    AdmissionregistrationV1Api,
+)
+from kubernetes.client.rest import ApiException
+
+from jaseci.utils.utils import logger
+
+
+@JsOrc.service(name="kube", config="KUBE_CONFIG")
+class KubeService(JsOrc.CommonService):
+    ###################################################
+    #                     BUILDER                     #
+    ###################################################
+
+    _no_namespace = [
+        "Namespace",
+        "ClusterRole",
+        "ClusterRoleBinding",
+        "CustomResourceDefinition",
+        "ValidatingWebhookConfiguration",
+    ]
+
+    def run(self):
+        self._in_cluster = self.config.get("in_cluster", True)
+        if self._in_cluster:
+            kubernetes_config.load_incluster_config()
+        else:
+            kubernetes_config.load_kube_config()
+
+        self.namespace = self.config.get("namespace", "default")
+        config = self.config.get("config")
+
+        self.app = ApiClient(config)
+        self.core = CoreV1Api(config)
+        self.api = AppsV1Api(self.app)
+        self.api_ext = ApiextensionsV1Api(self.app)
+        self.auth = RbacAuthorizationV1Api(self.app)
+        self.reg_api = AdmissionregistrationV1Api(self.app)
+
+        self.ping()
+        self.defaults()
+
+    def defaults(self):
+        self.create_apis = {
+            "Namespace": self.core.create_namespace,
+            "Service": self.core.create_namespaced_service,
+            "Deployment": self.api.create_namespaced_deployment,
+            "ConfigMap": self.core.create_namespaced_config_map,
+            "ServiceAccount": self.core.create_namespaced_service_account,
+            "ClusterRole": self.auth.create_cluster_role,
+            "ClusterRoleBinding": self.auth.create_cluster_role_binding,
+            "Secret": self.core.create_namespaced_secret,
+            "PersistentVolumeClaim": (
+                self.core.create_namespaced_persistent_volume_claim
+            ),
+            "DaemonSet": self.api.create_namespaced_daemon_set,
+            "StatefulSet": self.api.create_namespaced_stateful_set,
+            "CustomResourceDefinition": self.api_ext.create_custom_resource_definition,
+            "ValidatingWebhookConfiguration": self.reg_api.create_validating_webhook_configuration,
+        }
+        self.patch_apis = {
+            "Namespace": self.core.patch_namespace,
+            "Service": self.core.patch_namespaced_service,
+            "Deployment": self.api.patch_namespaced_deployment,
+            "ConfigMap": self.core.patch_namespaced_config_map,
+            "ServiceAccount": self.core.patch_namespaced_service_account,
+            "ClusterRole": self.auth.patch_cluster_role,
+            "ClusterRoleBinding": self.auth.patch_cluster_role_binding,
+            "Secret": self.core.patch_namespaced_secret,
+            "PersistentVolumeClaim": (
+                self.core.patch_namespaced_persistent_volume_claim
+            ),
+            "DaemonSet": self.api.patch_namespaced_daemon_set,
+            "StatefulSet": self.api.patch_namespaced_stateful_set,
+            "CustomResourceDefinition": self.api_ext.patch_custom_resource_definition,
+            "ValidatingWebhookConfiguration": self.reg_api.patch_validating_webhook_configuration,
+        }
+        self.delete_apis = {
+            "Namespace": self.core.delete_namespace,
+            "Service": self.core.delete_namespaced_service,
+            "Deployment": self.api.delete_namespaced_deployment,
+            "ConfigMap": self.core.delete_namespaced_config_map,
+            "ServiceAccount": self.core.delete_namespaced_service_account,
+            "ClusterRole": self.auth.delete_cluster_role,
+            "ClusterRoleBinding": self.auth.delete_cluster_role_binding,
+            "Secret": self.core.delete_namespaced_secret,
+            "PersistentVolumeClaim": (
+                self.core.delete_namespaced_persistent_volume_claim
+            ),
+            "DaemonSet": self.api.delete_namespaced_daemon_set,
+            "StatefulSet": self.api.delete_namespaced_stateful_set,
+            "CustomResourceDefinition": self.api_ext.delete_custom_resource_definition,
+            "ValidatingWebhookConfiguration": self.reg_api.delete_validating_webhook_configuration,
+        }
+        self.read_apis = {
+            "Namespace": self.core.read_namespace,
+            "Service": self.core.read_namespaced_service,
+            "Endpoints": self.core.read_namespaced_endpoints,
+            "Deployment": self.api.read_namespaced_deployment,
+            "ConfigMap": self.core.read_namespaced_config_map,
+            "ServiceAccount": self.core.read_namespaced_service_account,
+            "ClusterRole": self.auth.read_cluster_role,
+            "ClusterRoleBinding": self.auth.read_cluster_role_binding,
+            "Secret": self.core.read_namespaced_secret,
+            "PersistentVolumeClaim": self.core.read_namespaced_persistent_volume_claim,
+            "DaemonSet": self.api.read_namespaced_daemon_set,
+            "StatefulSet": self.api.read_namespaced_stateful_set,
+            "CustomResourceDefinition": self.api_ext.read_custom_resource_definition,
+            "ValidatingWebhookConfiguration": self.reg_api.read_validating_webhook_configuration,
+        }
+
+    ###################################################
+    #                     COMMONS                     #
+    ###################################################
+
+    def ping(self):
+        res = self.app.call_api("/readyz", "GET")
+        return res[1] == 200
+
+    def in_cluster(self):
+        """
+        Check if JSORC/Jaseci is running in a kubernetes cluster
+        """
+        try:
+            return self._in_cluster and self.ping()
+        except ApiException as e:
+            logger.info(f"Kubernetes cluster environment check failed: {e}")
+            return False
+
+    def resolve_namespace(self, conf: dict, namespace: str = None):
+        return conf.get("metadata", {}).get("namespace") or namespace or self.namespace
+
+    def create(
+        self,
+        kind: str,
+        name: str,
+        conf: dict,
+        namespace: str = None,
+        log_pref: str = "",
+    ):
+        namespace = self.resolve_namespace(conf, namespace)
+        try:
+            logger.info(
+                f"{log_pref} Creating {kind} for `{name}` with namespace `{namespace}`"
+            )
+            if kind in self._no_namespace:
+                self.create_apis[kind](body=conf)
+            else:
+                self.create_apis[kind](namespace=namespace, body=conf)
+        except ApiException as e:
+            logger.error(
+                f"{log_pref} Error creating {kind} for `{name}` with namespace `{namespace}` -- {e}"
+            )
+
+    def patch(
+        self,
+        kind: str,
+        name: str,
+        conf: dict,
+        namespace: str = None,
+        log_pref: str = "",
+    ):
+        namespace = self.resolve_namespace(conf, namespace)
+        try:
+            logger.info(
+                f"{log_pref} Patching {kind} for `{name}` with namespace `{namespace}`"
+            )
+            if kind in self._no_namespace:
+                self.patch_apis[kind](name=name, body=conf)
+            else:
+                self.patch_apis[kind](name=name, namespace=namespace, body=conf)
+        except ApiException as e:
+            logger.error(
+                f"{log_pref} Error patching {kind} for `{name}` with namespace `{namespace}` -- {e}"
+            )
+
+    def read(self, kind: str, name: str, namespace: str = None, log_pref: str = ""):
+        namespace = namespace or self.namespace
+        try:
+            logger.info(
+                f"{log_pref} Retrieving {kind} for `{name}` with namespace `{namespace}`"
+            )
+            if kind in self._no_namespace:
+                return self.read_apis[kind](name=name)
+            else:
+                return self.read_apis[kind](name=name, namespace=namespace)
+        except ApiException as e:
+            logger.error(
+                f"{log_pref} Error retrieving {kind} for `{name}` with namespace `{namespace}` -- {e}"
+            )
+            return e
+
+    def delete(self, kind: str, name: str, namespace: str = None, log_pref: str = ""):
+        namespace = namespace or self.namespace
+        try:
+            logger.info(
+                f"{log_pref} Deleting {kind} for `{name}` with namespace `{namespace}`"
+            )
+            if kind in self._no_namespace:
+                return self.delete_apis[kind](name=name)
+            else:
+                return self.delete_apis[kind](name=name, namespace=namespace)
+        except ApiException as e:
+            logger.error(
+                f"{log_pref} Error deleting {kind} for `{name}` with namespace `{namespace}` -- {e}"
+            )
+            return e
+
+    def is_pod_running(self, name: str, namespace: str = None):
+        namespace = namespace or self.namespace
+        try:
+            return (
+                self.core.list_namespaced_pod(
+                    namespace=namespace, label_selector=f"pod={name}"
+                )
+                .items[0]
+                .status.phase
+                == "Running"
+            )
+        except Exception:
+            return False
+
+    def terminate_jaseci(self, name: str, namespace: str = None) -> bool:
+        namespace = namespace or self.namespace
+        try:
+            for item in self.core.list_namespaced_pod(
+                namespace=namespace, label_selector=f"pod={name}"
+            ).items:
+                self.core.delete_namespaced_pod(item.metadata.name, namespace)
+                raise SystemExit("Force termination to restart the pod!")
+        except Exception:
+            return False

jaseci/svc/{mail/mail.py → mail_svc.py}

@@ -1,25 +1,25 @@
 import ssl
+from json import dumps
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
-from json import dumps
 from smtplib import SMTP, SMTP_SSL
 
-from jaseci.svc import CommonService
-from .config import MAIL_CONFIG
+from jaseci import JsOrc
 
 
 #################################################
-#                  EMAIL APP                   #
+#                   EMAIL APP                   #
 #################################################
 
 
-class MailService(CommonService):
+@JsOrc.service(name="mail", config="MAIL_CONFIG")
+class MailService(JsOrc.CommonService):
     ###################################################
     #                     BUILDER                     #
     ###################################################
 
-    def run(self, hook=None):
-        self.__convert_config(hook)
+    def run(self):
+        self.__convert_config()
         self.app = self.connect()
 
     # ----------- BACKWARD COMPATIBILITY ------------ #
@@ -36,7 +36,8 @@ class MailService(CommonService):
                         holder[v] = conf
         return holder
 
-    def __convert_config(self, hook):
+    def __convert_config(self):
+        hook = JsOrc.hook()
         version = self.config.get("version", 2)
         migrate = self.config.get("migrate", False)
         if version == 1 or migrate:
@@ -78,12 +79,12 @@ class MailService(CommonService):
     #                     CLEANER                     #
     ###################################################
 
-    def reset(self, hook, start=True):
+    # ---------------- PROXY EVENTS ----------------- #
+
+    def on_delete(self):
         if self.is_running():
             self.app.terminate()
 
-        super().reset(hook, start)
-
     ####################################################
     #                    OVERRIDDEN                    #
     ####################################################
@@ -109,20 +110,16 @@ class MailService(CommonService):
 
         return Mailer(server, sender)
 
-    def build_config(self, hook) -> dict:
-        return hook.service_glob("MAIL_CONFIG", MAIL_CONFIG)
-
 
 # ----------------------------------------------- #
 
-
 ####################################################
-#                   EMAIL CONFIG                   #
+#                      MAILER                      #
 ####################################################
 
 
 class Mailer:
-    def __init__(self, server, sender):
+    def __init__(self, server: SMTP, sender):
         self.server = server
         self.sender = sender
 

jaseci/svc/{prometheus/prometheus.py → prome_svc.py}

@@ -1,16 +1,15 @@
-from prometheus_api_client import PrometheusConnect
-from jaseci.svc import CommonService
-from .config import PROMON_CONFIG
-from .manifest import PROMON_MANIFEST
 import time
+from prometheus_api_client import PrometheusConnect
+from jaseci import JsOrc
 
 
-class PrometheusService(CommonService):
+@JsOrc.service(name="prome", config="PROME_CONFIG", manifest="PROME_MANIFEST")
+class PrometheusService(JsOrc.CommonService):
     ###################################################
     #                     BUILDER                     #
     ###################################################
 
-    def run(self, hook=None):
+    def run(self):
         self.app = PrometheusConnect(url=self.config.get("url"), disable_ssl=True)
         self.ping()
         self.cpu = Cpu(self.app)
@@ -117,16 +116,6 @@ class PrometheusService(CommonService):
 
         return res
 
-    ####################################################
-    #                    OVERRIDDEN                    #
-    ####################################################
-
-    def build_config(self, hook) -> dict:
-        return hook.service_glob("PROMON_CONFIG", PROMON_CONFIG)
-
-    def build_manifest(self, hook) -> dict:
-        return hook.service_glob("PROMON_MANIFEST", PROMON_MANIFEST)
-
 
 class Info:
     def __init__(self, app):

jaseci/svc/{redis/redis.py → redis_svc.py}

@@ -1,21 +1,16 @@
+from jaseci import JsOrc
 from redis import Redis
 
-from jaseci.svc import CommonService
-from .config import REDIS_CONFIG
-from .manifest import REDIS_MANIFEST
 
-
-#################################################
-#                  REDIS HOOK                   #
-#################################################
-
-
-class RedisService(CommonService):
-    ###################################################
-    #                     BUILDER                     #
-    ###################################################
-
-    def run(self, hook=None):
+@JsOrc.service(
+    name="redis",
+    config="REDIS_CONFIG",
+    manifest="REDIS_MANIFEST",
+    priority=0,
+    proxy=True,
+)
+class RedisService(JsOrc.CommonService):
+    def run(self):
         self.app = Redis(**self.config, decode_responses=True)
         self.app.ping()
 
@@ -58,15 +53,8 @@ class RedisService(CommonService):
         if self.is_running():
             self.app.flushdb()
 
-    ###################################################
-    #                     CONFIG                      #
-    ###################################################
-
-    def build_config(self, hook) -> dict:
-        return hook.service_glob("REDIS_CONFIG", REDIS_CONFIG)
+    # ---------------- PROXY EVENTS ----------------- #
 
-    def build_manifest(self, hook) -> dict:
-        return hook.service_glob("REDIS_MANIFEST", REDIS_MANIFEST)
-
-
-# ----------------------------------------------- #
+    def on_delete(self):
+        if self.is_running():
+            self.app.close()

jaseci/svc/{stripe/stripe.py → stripe_svc.py}

@@ -1,18 +1,18 @@
 import stripe
-from jaseci.svc import CommonService
-from .config import STRIPE_CONFIG
+from jaseci import JsOrc
 
 #################################################
 #                  STRIPE APP                   #
 #################################################
 
 
-class StripeService(CommonService):
+@JsOrc.service("stripe", config="STRIPE_CONFIG")
+class StripeService(JsOrc.CommonService):
     ###################################################
     #                     BUILDER                     #
     ###################################################
 
-    def run(self, hook=None):
+    def run(self):
         self.app = stripe
         if not self.config.get("api_key"):
             raise Exception("api_key is required!")
@@ -49,6 +49,3 @@ class StripeService(CommonService):
     def reset(self, hook, start=True):
         stripe.api_key = None
         super().reset(hook, start)
-
-    def build_config(self, hook) -> dict:
-        return hook.service_glob("STRIPE_CONFIG", STRIPE_CONFIG)

jaseci/svc/{task/task.py → task_svc.py}

@@ -1,54 +1,64 @@
 from celery import Celery
+from celery.app.trace import build_tracer
 from celery.app.control import Inspect
 from celery.backends.base import DisabledBackend
 
-from jaseci.svc import CommonService
-from .common import Queue, ScheduledWalker, ScheduledSequence
-from .config import TASK_CONFIG
+from jaseci import JsOrc
+from .tasks import Queue, ScheduledWalker, ScheduledSequence
 
 #################################################
 #                   TASK APP                   #
 #################################################
 
 
-class TaskService(CommonService):
+@JsOrc.service(name="task", config="TASK_CONFIG")
+class TaskService(JsOrc.CommonService):
     ###################################################
     #                   INITIALIZER                   #
     ###################################################
 
-    def __init__(self, hook=None):
+    def __init__(self, config: dict, manifest: dict):
         self.inspect: Inspect = None
         self.queue: Queue = None
         self.scheduled_walker: ScheduledWalker = None
         self.scheduled_sequence: ScheduledSequence = None
 
-        super().__init__(hook)
+        super().__init__(config, manifest)
 
     ###################################################
     #                     BUILDER                     #
     ###################################################
 
-    def run(self, hook=None):
+    def run(self):
         self.app = Celery("celery")
         self.app.conf.update(**self.config)
 
         # -------------------- TASKS -------------------- #
-
-        self.queue = self.app.register_task(Queue())
-        self.scheduled_walker = self.app.register_task(ScheduledWalker())
-        self.scheduled_sequence = self.app.register_task(ScheduledSequence())
+        (
+            self.queue,
+            self.scheduled_walker,
+            self.scheduled_sequence,
+        ) = self.register_tasks(Queue, ScheduledWalker, ScheduledSequence)
 
         # ------------------ INSPECTOR ------------------ #
 
         self.inspect = self.app.control.inspect()
         self.ping()
 
-    def post_run(self, hook=None):
+    def post_run(self):
         self.spawn_daemon(
             worker=self.app.Worker(quiet=self.quiet).start,
             scheduler=self.app.Beat(socket_timeout=None, quiet=self.quiet).run,
         )
 
+    def register_tasks(self, *tasks) -> tuple:
+        registered = []
+        for task in tasks:
+            task = self.app.register_task(task())
+            task.__trace__ = build_tracer(task.name, task, app=self.app)
+            registered.append(task)
+        return tuple(registered)
+
     ###################################################
     #              COMMON GETTER/SETTER               #
     ###################################################
@@ -93,18 +103,11 @@ class TaskService(CommonService):
     #                     CLEANER                     #
     ###################################################
 
-    def reset(self, hook, start=True):
+    def failed(self, error):
+        super().failed(error)
         self.terminate_daemon("worker", "scheduler")
-        self.inspect = None
-        super().reset(hook, start)
 
-    def failed(self):
-        super().failed()
-        self.terminate_daemon("worker", "scheduler")
+    # ---------------- PROXY EVENTS ----------------- #
 
-    ####################################################
-    #                    OVERRIDDEN                    #
-    ####################################################
-
-    def build_config(self, hook) -> dict:
-        return hook.service_glob("TASK_CONFIG", TASK_CONFIG)
+    def on_delete(self):
+        self.terminate_daemon("worker", "scheduler")