engforge 0.3.0__py3-none-any.whl

engforge/env_var.py

@@ -0,0 +1,214 @@
+"""Defines a class called `EnvVariable` that defines an interface for env variables with an option to obscure and convert variables, as well as provide a default option.
+
+A global record of variables is kept for informational purposes in keeping track of progam variables
+
+To prevent storage of env vars in program memory, access to the os env variables is provided on access of the `secret` variable. It is advisable to use the result of this as directly as possible when dealing with actual secrets.
+
+For example add: `db_driver(DB_HOST.secret,DB_PASSWORD.secret,...)
+"""
+
+import os
+from engforge.logging import LoggingMixin
+from typing import Any
+import socket
+import inspect
+
+global warned
+warned = set()  # a nice global variable to hold any warnings
+
+FALSE_VALUES = (False, "false", "no", "n")
+TRUE_VALUES = (True, "checked", "true", "yes", "y")
+
+
+def parse_bool(input: str):
+    if isinstance(input, str):
+        input = input.lower()
+
+    if not input:
+        return False
+    elif input in TRUE_VALUES:
+        return True
+    elif input in FALSE_VALUES:
+        return False
+    return False
+
+
+DEFAULT_CONVERTERS = {bool: parse_bool}
+
+
+class EnvVariable(LoggingMixin):
+    """A method to wrap SECRETS and in application with a way to get the value using self.secret
+    Do not store values from self.secret to ensure security
+
+    You can override the secret with _override"""
+
+    var_name: str = None
+    type_conv: Any = None
+    default: Any = None
+    obscure: bool = True
+    _override: str
+    _secrets = {}  # its class based so like a singleton
+    _replaced = set()
+    fail_on_missing: bool
+    desc: str = None
+    _upgrd_warn: bool = False
+    _dontovrride: bool = False
+
+    def __init__(
+        self,
+        secret_var_name,
+        type_conv=None,
+        default=None,
+        obscure=False,
+        fail_on_missing=False,
+        desc: str = None,
+        dontovrride=False,
+    ):
+        """pass arguments to SecretVariable to have it look up information at runtime from envargs, but not store it in memory.
+        :param secret_var_name: the enviornmental variable
+        :param type_conv: the data from env vars will be converted with this function
+        :param default: the value to use if the secret_var_name doesn't exist in enviornmental variables
+        :param obscure: default True, will prevent the result being printed by str(self)
+        :param fail_on_missing: if the secret env variable is not found, and default is None
+        :param desc: a description of the purpose of the variable
+        """
+        self.var_name = secret_var_name
+        self.type_conv = (
+            type_conv
+            if type_conv not in DEFAULT_CONVERTERS
+            else DEFAULT_CONVERTERS[type_conv]
+        )
+        self._dontovrride = dontovrride
+
+        if default is not None:
+            self.default = default
+        self.obscure = obscure
+        # UserString.__init__(self,f'[SECRET:{secret_var_name}]')
+
+        self.fail_on_missing = fail_on_missing
+
+        # record env vars
+        if secret_var_name in self.__class__._secrets:
+            cur = self.__class__._secrets[secret_var_name]
+            if dontovrride:
+                self.debug(f"not replacing: {cur}->{self}")
+                self.__dict__ = cur.__dict__
+            else:
+                self.info(f"replacing {cur}->{self}")
+                self._replaced.add(cur)
+                self.__class__._secrets[secret_var_name] = self
+        else:
+            self.__class__._secrets[secret_var_name] = self
+
+        # FIXME: prevent engforge var from replacing other module instnace
+        # not possible to locate where other instances
+        # if secret_var_name in self.__class__._secrets:
+        #     cur = self.__class__._secrets[secret_var_name]
+        #     if cur != self and self not in self._replaced:
+        #         self._replaced.add(cur)
+        #         self.info(f'replacing {cur}->{self}')
+        #         self.__class__._secrets[secret_var_name] = self
+        #     elif self in self._replaced:
+        #         self.info(f'skipping replaced readd {self}')
+        #         #self.__class__._secrets[secret_var_name] = self
+        # else:
+        #     self.__class__._secrets[secret_var_name] = self
+
+    def __str__(self):
+        if self.obscure:
+            return f"{self.obscured_name:<40} = XXXXXX"
+        return f"{self.obscured_name:<40} = {self.secret}"
+
+    def __add__(self, other) -> str:
+        return str(str.__add__(str(self), other))
+
+    def __radd__(self, other) -> str:
+        return str(str.__add__(other, str(self)))
+
+    @property
+    def obscured_name(self) -> str:
+        if hasattr(self, "_override"):
+            return f"SECRETS[OVERRIDE]"
+        return f"SECRETS[{self.var_name}]"
+
+    @property
+    def secret(self):
+        # Check if this secret is the one in the secrets registry
+        sec = self.__class__._secrets[self.var_name]
+        if sec is not self and not sec._dontovrride:
+            # Provide warning that the secret is being replaced
+            if not self._upgrd_warn:
+                self._upgrd_warn = True
+                self.info(f"upgrading: {self.var_name} from {id(self)}->{id(sec)}")
+
+            # Monkeypatch dictionary
+            self.__dict__ = sec.__dict__
+
+        if hasattr(self, "_override"):
+            return self._override
+
+        if self.var_name in os.environ:
+            secval = os.environ[self.var_name]
+        elif self.default is not None:
+            if self.var_name not in warned:
+                if self.obscure:
+                    dflt = "XXXXXXX"
+                else:
+                    dflt = self.default
+
+                self.debug(f"Env Var: {self.var_name} Not Found! Using: {dflt}")
+                warned.add(self.var_name)
+
+            secval = self.default
+        else:
+            if self.fail_on_missing:
+                raise FileNotFoundError(f"Could Not Find Env Variable {self.var_name}")
+            else:
+                if self.var_name not in warned:
+                    self.debug(f"Env Var: {self.var_name} Not Found!")
+                    warned.add(self.var_name)
+                return None
+
+        if self.type_conv is None:
+            return secval
+        else:
+            return self.type_conv(secval)
+
+    @property
+    def in_env(self):
+        return self.var_name in os.environ
+
+    def remove(self):
+        """removes this secret from the record"""
+        if self in self.__class__._secrets:
+            self.__class__._secrets.remove(self)
+
+    @classmethod
+    def load_env_vars(self):
+        for s in EnvVariable._secrets.values():
+            str(s)
+
+    @classmethod
+    def print_env_vars(cls):
+        """prints env vars in memory"""
+        # preload
+        cls.load_env_vars()
+        for var, s in sorted(
+            EnvVariable._secrets.items(), key=lambda kv: kv[1].var_name
+        ):
+            print(f"{s.var_name:<40}|{s}")
+
+
+# DEFAULT ENV VARIABLES
+try:
+    # This should always work unless we don't have privideges (rare assumed)
+    host = socket.gethostname().upper()
+except:
+    host = "MASTER"
+
+global HOSTNAME, SLACK_WEBHOOK
+
+HOSTNAME = EnvVariable("FORGE_HOSTNAME", default=host, obscure=False, dontovrride=True)
+SLACK_WEBHOOK = EnvVariable(
+    "FORGE_SLACK_LOG_WEBHOOK", default=None, obscure=False, dontovrride=True
+)

engforge/locations.py

@@ -0,0 +1,18 @@
+from engforge.env_var import EnvVariable
+import os
+
+FORGE_ROOT_VAR = EnvVariable(
+    "FORGE_ROOT", default=None, dontovrride=True, type_conv=os.path.expanduser
+)
+
+
+def client_path(alternate_path=None, **kw):
+    path = FORGE_ROOT_VAR.secret
+    if path is None:
+        if alternate_path is None:
+            raise KeyError(
+                f"no `FORGE_ROOT` set and no alternate path in client_path call"
+            )
+        return alternate_path
+    else:
+        return path

engforge/logging.py

@@ -0,0 +1,290 @@
+import logging
+import logging
+import traceback
+import sys, os
+from pyee import EventEmitter
+from termcolor import colored
+import requests
+import json
+import uuid
+
+global log_change_emitter
+log_change_emitter = EventEmitter()
+
+
+BASIC_LOG_FMT = "[%(name)-24s]%(message)s"
+
+global LOG_LEVEL
+LOG_LEVEL = logging.INFO
+
+
+def change_all_log_levels(new_log_level: int = 20, inst=None, check_function=None):
+    """Changes All Log Levels With pyee broadcast before reactor is running
+    :param new_log_level: int - changes unit level log level (10-msg,20-debug,30-info,40-warning,50-error,60-crit)
+    :param check_function: callable -> bool - (optional) if provided if check_function(unit) is true then the new_log_level is applied
+    """
+    if isinstance(new_log_level, float):
+        new_log_level = int(new_log_level)  # Float Case Is Handled
+
+    assert (
+        isinstance(new_log_level, int) and new_log_level >= 1 and new_log_level <= 100
+    )
+
+    global LOG_LEVEL
+    LOG_LEVEL = new_log_level
+
+    if LoggingMixin.log_level != new_log_level:
+        print(f"changing log levels to {new_log_level}...")
+        log.info(f"Changing All Logging Units To Level {new_log_level}")
+    log_change_emitter.emit("change_level", new_log_level, check_function)
+    LoggingMixin.log_level = new_log_level
+    if inst:
+        inst.log_level = new_log_level
+
+
+class LoggingMixin(logging.Filter):
+    """Class to include easy formatting in subclasses"""
+
+    log_level = LOG_LEVEL
+    _log = None
+
+    log_on = True
+
+    log_fmt = "[%(name)-24s]%(message)s"
+
+    slack_webhook_url = None
+    # log_silo = False
+
+    change_all_log_lvl = lambda s, *a, **kw: change_all_log_levels(*a, inst=s, **kw)
+
+    @property
+    def logger(self):
+        global LOG_LEVEL
+        if self._log is None:
+            inst_log_name = "engforgelog_" + self.identity + "_" + str(uuid.uuid4())
+            self._log = logging.getLogger(inst_log_name)
+            self._log.setLevel(level=LOG_LEVEL)
+
+            # Apply Filter Info
+            self._log.addFilter(self)
+            self.installSTDLogger()
+            from engforge.env_var import EnvVariable, SLACK_WEBHOOK
+
+            # Hot Patch Class (EnvVar is logging mixin... soo... here we are)
+            if LoggingMixin.slack_webhook_url is None:
+                # Do this on the fly since we SecretVariable is a log component
+                LoggingMixin.slack_webhook_url = SLACK_WEBHOOK
+
+        if not hasattr(self, "_f_change_log"):
+            # change log level listener
+            def _change_log(new_level, check_function=None):
+                if new_level != self.log_level:
+                    if check_function is None or check_function(self):
+                        msg = f"changing {self.identity} log level: {self.log_level} -> {new_level}"
+                        self.__class__.log_level = new_level
+                        self.info(msg)
+                        self._log.setLevel(new_level)
+                        self.log_level = new_level
+                        self.resetLog()
+
+            # our emitter gets set with a callback
+            log_change_emitter.add_listener("change_level", _change_log)
+
+            self._f_change_log = _change_log
+
+        return self._log
+
+    def resetLog(self):
+        """reset log"""
+        self._log = None
+        self.debug(f"reset!")
+
+    def resetSystemLogs(self, reseted=None):
+        """resets log on all internal instance LoggingMixins"""
+        self.resetLog()
+        self.debug(f"reset!")
+        if reseted is None:
+            reseted = set()
+        for k, v in self.__dict__.items():
+            if isinstance(v, LoggingMixin) and id(v) not in reseted:
+                reseted.add(id(v))
+                v.resetSystemLogs(reseted)
+
+    def installSTDLogger(self):
+        """We only want std logging to start"""
+        sh = logging.StreamHandler(sys.stdout)
+        peerlog = logging.Formatter(self.log_fmt)
+        sh.setFormatter(peerlog)
+        self._log.addHandler(sh)
+
+    def add_fields(self, record):
+        """Overwrite this to modify logging fields"""
+        pass
+
+    def filter(self, record):
+        """This acts as the interface for `logging.Filter`
+        Don't overwrite this, use `add_fields` instead."""
+        record.name = self.identity.lower()[:24]
+        self.add_fields(record)
+        return True
+
+    def msg(self, *args, lvl=5):
+        """Writes to log... this should be for raw data or something... least priorty"""
+        if self.log_on:
+            self.logger.log(
+                lvl,
+                self.message_with_identiy(self.extract_message(args), "blue"),
+            )
+
+    def debug(self, *args):
+        """Writes at a low level to the log file... usually this should
+        be detailed messages about what exactly is going on"""
+        if self.log_on:
+            self.logger.debug(
+                self.message_with_identiy(self.extract_message(args), "cyan")
+            )
+
+    def info(self, *args):
+        """Writes to log but with info category, these are important typically
+        and inform about progress of process in general"""
+        if self.log_on:
+            self.logger.info(
+                self.message_with_identiy(self.extract_message(args), "white")
+            )
+
+    def warning(self, *args):
+        """Writes to log as a warning"""
+        self.logger.warning(
+            self.message_with_identiy("WARN: " + self.extract_message(args), "yellow")
+        )
+
+    def error(self, error, msg=""):
+        """Writes to log as a error"""
+
+        # fmt = 'ERROR: {msg!r}|{err!r}'
+
+        tb = error.__traceback__
+        fmt = "ERROR:{msg}->{err}"
+
+        tb = "\n".join(traceback.format_exception(error, value=error, tb=tb))
+        msgfmt = ("\n" + " " * 51 + "|").join(str(msg).split("\n"))
+
+        # tbcl = colored(tb, "red")
+        # self.logger.exception( fmt.format(msg=msgfmt,err=tbcl))
+        # self.logger.exception( msgfmt)
+
+        m = colored(fmt.format(msg=msgfmt, err=tb), "red")
+        self.logger.error(m)
+
+    def critical(self, *args):
+        """A routine to communicate to the root of the server network that there is an issue"""
+        msg = self.extract_message(args)
+        msg = self.message_with_identiy(msg, "magenta")
+        self.logger.critical(msg)
+
+        # FIXME: setup slack notificatinos with env var
+        self.slack_notification(self.identity.title(), msg)
+
+    def slack_notification(self, category, message):
+        from engforge.env_var import SLACK_WEBHOOK, HOSTNAME
+
+        if SLACK_WEBHOOK.var_name in os.environ:
+            self.info("getting slack webhook")
+            url = SLACK_WEBHOOK.secret
+        else:
+            return
+        stage = HOSTNAME.secret
+        headers = {"Content-type": "application/json"}
+        data = {
+            "text": "{category} on {stage}:\n```{message}```".format(
+                category=category.upper(), stage=stage, message=message
+            )
+        }
+        self.info(f"Slack Notification : {url}:{category},{message}")
+        slack_note = requests.post(
+            url, data=json.dumps(data).encode("ascii"), headers=headers
+        )
+
+    def message_with_identiy(self, message: str, color=None):
+        """converts to color and string via the termcolor library
+        :param message: a string convertable entity
+        :param color: a color in [grey,red,green,yellow,blue,magenta,cyan,white]
+        """
+        if color != None:
+            return colored(str(message), color)
+        return str(message)
+
+    def extract_message(self, args):
+        for arg in args:
+            if type(arg) is str:
+                return arg
+        if self.log_level < 0:
+            print(f"no string found for {args}")
+        return ""
+
+    @property
+    def identity(self):
+        return type(self).__name__
+
+    def __getstate__(self):
+        d = dict(self.__dict__)
+        d["_f_change_log"] = None
+        return d
+
+
+class Log(LoggingMixin):
+    pass
+
+
+log = Log()
+
+
+# try:
+#     logging.getLogger('parso.cache').disabled=True
+#     logging.getLogger('parso.cache.pickle').disabled=True
+#     logging.getLogger('parso.python.diff').disabled=True
+#
+# except Exception as e:
+#     log.warning(f'could not diable parso {e}')
+# def installGELFLogger():
+#     '''Installs GELF Logger'''
+#     # self.gelf = graypy.GELFTLSHandler(GELF_HOST,GELF_PORT, validate=True,\
+#     #                         ca_certs=credfile('graylog-clients-ca.crt'),\
+#     #                         certfile = credfile('test-client.crt'),
+#     #                         keyfile = credfile('test-client.key')
+#     #                         )
+#     log = logging.getLogger('')
+#     gelf = graypy.GELFUDPHandler(host=GELF_HOST,port=12203, extra_fields=True)
+#     log.addHandler(gelf)
+
+
+# def installSTDLogger(fmt = BASIC_LOG_FMT):
+#     '''We only want std logging to start'''
+#     log = logging.getLogger('')
+#     sh = logging.StreamHandler(sys.stdout)
+#     peerlog = logging.Formatter()
+#     sh.setFormatter(peerlog)
+#     log.addHandler( sh )
+#
+#
+# def set_all_loggers_to(level,set_stdout=False,all_loggers=False):
+#     global LOG_LEVEL
+#     LOG_LEVEL = level
+#
+#     if set_stdout: installSTDLogger()
+#
+#     logging.basicConfig(level = LOG_LEVEL) #basic config
+#
+#     log = logging.getLogger()
+#     log.setLevel(LOG_LEVEL)# Set Root Logger
+#
+#     log.setLevel(level) #root
+#
+#     loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict]
+#     for logger in loggers:
+#         if logger.__class__.__name__.lower().startswith('engforge'):
+#             logger.log(LOG_LEVEL,'setting log level: {}'.format(LOG_LEVEL))
+#             logger.setLevel(LOG_LEVEL)
+#         elif all_loggers:
+#             logger.log(LOG_LEVEL,'setting log level: {}'.format(LOG_LEVEL))
+#             logger.setLevel(LOG_LEVEL)