hcs-core 0.1.250__py3-none-any.whl → 0.1.316__py3-none-any.whl

hcs_core/ctxp/task_schd.py

@@ -0,0 +1,168 @@
+import logging
+import threading
+from copy import deepcopy
+from dataclasses import dataclass
+from time import sleep, time
+
+import schedule
+
+from . import duration
+
+log = logging.getLogger(__name__)
+
+
+class TaskRef:
+    def __init__(self, ref1, ref2=None):
+        self.ref1 = ref1
+        self.ref2 = ref2
+
+    def cancel(self):
+        if self.ref1:
+            fn = list(self.ref1.job_func.args)[0]
+            log.debug("Cancel job %s: %s" % (fn.__name__, self.ref1))
+            schedule.cancel_job(self.ref1)
+            self.ref1 = None
+        if self.ref2:
+            fn2 = list(self.ref2.job_func.args)[0]
+            log.debug("Cancel job %s: %s" % (fn2.__name__, self.ref2))
+            schedule.cancel_job(self.ref2)
+            self.ref2 = None
+
+
+@dataclass
+class JobStatistics:
+    last_start: int = 0
+    last_end: int = 0
+    history_total: int = 0
+    last_cycle_total: int = 0
+
+    def start(self):
+        self.last_start = int(time())
+        self.last_cycle_total += 1
+        self.history_total += 1
+
+    def end(self):
+        self.last_end = int(time())
+
+    def reset_cycle(self):
+        self.last_cycle_total = 0
+
+
+@dataclass
+class Statistics:
+    run_once_job: JobStatistics = JobStatistics()
+    recurring_job: JobStatistics = JobStatistics()
+    size: int = 0
+
+    def reset_cycle(self):
+        self.run_once_job.reset_cycle()
+        self.recurring_job.reset_cycle()
+
+
+_g_worker_thread: threading.Event = None
+_g_flag_stop_daemon: threading.Thread = None
+_g_flag_running: bool = True
+_g_statistics = Statistics()
+
+
+def _task_wrapper_repeat(fn_impl, kwargs):
+    _g_statistics.recurring_job.start()
+    fn_impl(**kwargs)
+    _g_statistics.recurring_job.end()
+
+
+def _task_wrapper_once(fn_impl, kwargs):
+    _g_statistics.run_once_job.start()
+    fn_impl(**kwargs)
+    _g_statistics.run_once_job.end()
+    return schedule.CancelJob
+
+
+def submit(fn_task: callable, initial_delay: str = None, repeat_interval: str = None, **kwargs):
+    if initial_delay:
+        initial_delay_seconds = duration.to_seconds(initial_delay)
+    else:
+        initial_delay_seconds = 1
+
+    if repeat_interval:
+        # Have an initial run
+        # TODO: this is not correct, but better than no lib-provided initial delay.
+        job1 = schedule.every(initial_delay_seconds).seconds.do(_task_wrapper_once, fn_task, kwargs)
+        log.debug(f"Register initial run {fn_task.__name__} {job1}")
+
+        # TODO: identify a way to use the scheduler with initial delay, with cancellation in mind.
+        seconds = duration.to_seconds(repeat_interval)
+        job2 = schedule.every(seconds).seconds.do(_task_wrapper_repeat, fn_task, kwargs)
+        log.debug(f"Register scheduled job {fn_task.__name__} at interval {repeat_interval} {job2}")
+    else:
+        job1 = schedule.every(initial_delay_seconds).seconds.do(_task_wrapper_once, fn_task, kwargs)
+        log.debug(f"Register one-shot job {fn_task.__name__} {job1}")
+        job2 = None
+    return TaskRef(job1, job2)
+
+
+def statistics(reset_cycle: bool = False):
+    _g_statistics.size = len(schedule.get_jobs())
+    ret = deepcopy(_g_statistics)
+    if reset_cycle:
+        _g_statistics.reset_cycle()
+    return ret
+
+
+def _daemon_worker():
+    log.info("task scheduler daemon thread start")
+    while not _g_flag_stop_daemon.is_set():
+        if _g_flag_running:
+            schedule.run_pending()
+        sleep(1)
+    log.info("task scheduler daemon thread exit")
+
+
+def start_daemon(paused: bool = False):
+    global _g_flag_stop_daemon
+    global _g_worker_thread
+    if _g_worker_thread:
+        raise Exception("Already started")
+
+    if paused:
+        global _g_flag_running
+        _g_flag_running = False
+
+    _g_flag_stop_daemon = threading.Event()
+    _g_worker_thread = threading.Thread(target=_daemon_worker, daemon=True, name="task-schd")
+    _g_worker_thread.start()
+
+
+def pause():
+    global _g_flag_running
+    if _g_flag_running:
+        _g_flag_running = False
+        log.info("task scheduler daemon paused")
+
+
+def resume():
+    global _g_flag_running
+    if not _g_flag_running:
+        _g_flag_running = True
+        log.info("task scheduler daemon resumed")
+
+
+def stop_daemon():
+    global _g_worker_thread
+    if _g_worker_thread:
+        _g_flag_stop_daemon.set()
+        _g_worker_thread.join()
+        _g_worker_thread = None
+
+
+if __name__ == "__main__":
+
+    def job1(a, b):
+        log.info(f"job1 {a} {b}")
+
+    # logutil.setup()
+    start_daemon()
+    j1 = submit(fn_task=job1, initial_delay="PT3S", repeat_interval="PT5S", a="aa", b="bb")
+    sleep(10)
+    stop_daemon()
+    log.info("exit")

hcs_core/ctxp/telemetry.py

@@ -0,0 +1,145 @@
+import json
+import logging
+import sys
+import time
+from datetime import datetime, timezone
+
+import click
+import httpx
+from yumako import env
+
+log = logging.getLogger(__name__)
+
+_record = None
+_enabled = None
+_app_name = ""
+
+
+def disable():
+    global _enabled
+    _enabled = False
+
+
+def _is_disabled():
+    global _enabled
+    if _enabled is None:
+        _enabled = env.bool("HCS_CLI_TELEMETRY", True)
+    return not _enabled
+
+
+def _get_version():
+    try:
+        from importlib.metadata import version
+
+        return version("hcs-cli")
+    except Exception as e:
+        log.debug(f"Failed to get hcs-cli version: {e}")
+        return "unknown"
+
+
+def _get_record():
+    global _record
+    if _record is None:
+        _record = {
+            "@timestamp": datetime.now(timezone.utc).isoformat(timespec="milliseconds"),
+            "app": _app_name,
+            "command": None,
+            "options": [],
+            "return": -1,
+            "error": None,
+            "time_ms": -1,
+            "version": _get_version(),
+            "env": {
+                "python_version": sys.version,
+                "platform": sys.platform,
+                "executable": sys.executable,
+            },
+        }
+    return _record
+
+
+def start(app_name: str = None):
+    if _is_disabled():
+        return
+
+    global _app_name
+    _app_name = app_name
+    _get_record()
+
+
+def update(cmd_path: str, params: dict):
+    if _is_disabled():
+        return
+
+    record = _get_record()
+    record["command"] = cmd_path
+    record["options"] = [k.replace("_", "-") for k, v in params.items() if v]
+
+
+def end(return_code: int = 0, error: Exception = None):
+    if _is_disabled():
+        return
+
+    record = _get_record()
+    if error:
+        if isinstance(error, click.exceptions.Exit):
+            return_code = error.exit_code
+        elif isinstance(error, SystemExit):
+            return_code = error.code
+        else:
+            record["error"] = str(error)
+            if return_code == 0:
+                return_code = 1
+    record["return"] = return_code
+    record["time_ms"] = int((time.time() - datetime.fromisoformat(record["@timestamp"]).timestamp()) * 1000)
+
+    _fix_missing_commands(record)
+    _injest(record)
+    return record
+
+
+def _fix_missing_commands(record):
+    if record["command"]:
+        return
+
+    args = sys.argv[1:]
+
+    # this does not work for all cases, but only as best effort.
+    options_started = False
+    options = record["options"]
+    command = [_app_name]
+    for arg in args:
+        if arg.startswith("-"):
+            options_started = True
+
+        if options_started:
+            if arg.startswith("--"):
+                options.append(arg[2:])
+            elif arg.startswith("-"):
+                options.append(arg[1:])
+            else:
+                # value. For privacy no logging.
+                continue
+        else:
+            command.append(arg)
+
+    record["command"] = " ".join(command)
+
+
+def _injest(doc):
+
+    # print('TELEMETRY end', json.dumps(doc, indent=4), flush=True)
+
+    try:
+        response = httpx.post(
+            "https://collie.omnissa.com/es/hcs-cli/_doc",
+            auth=("append_user", "public"),
+            headers={"Content-Type": "application/json"},
+            content=json.dumps(doc),
+            timeout=4,
+            verify=False,
+        )
+        response.raise_for_status()
+    except Exception as e:
+        log.debug(f"Telemetry ingestion failed: {e}", exc_info=True)
+        return

hcs_core/ctxp/template_util.py

@@ -0,0 +1,21 @@
+import re
+
+
+def replace(text: str, mapping: dict, strict_on_unresolved_vars: bool = True, strict_on_unused_var: bool = False):
+    unused_vars = set(mapping.keys())
+    for k, v in mapping.items():
+        new_text = text.replace("{{" + k + "}}", str(v))
+        if new_text != text:
+            text = new_text
+            unused_vars.remove(k)
+
+    if strict_on_unresolved_vars:
+        unresolved_vars = re.findall(r"\{\{([^}]+)\}\}", text)
+        if unresolved_vars:
+            raise Exception(f"Strict mode: template variables unresolved: {unresolved_vars}")
+
+    if strict_on_unused_var:
+        if unused_vars:
+            raise Exception(f"Strict mode: var specified but not in template: {unused_vars}")
+
+    return text

hcs_core/ctxp/timeutil.py

@@ -0,0 +1,11 @@
+from datetime import datetime, timezone
+
+
+def iso_date_to_timestamp(datetime_string: str) -> int:
+    dt_object = datetime.strptime(datetime_string, "%Y-%m-%dT%H:%M:%S.%fZ")
+    return int(dt_object.replace(tzinfo=timezone.utc).timestamp() * 1000)
+
+
+def timestamp_to_iso_date(timestamp_ms: int) -> str:
+    dt = datetime.fromtimestamp(timestamp_ms / 1000, tz=timezone.utc)
+    return dt.isoformat(timespec="milliseconds").replace("+00:00", "Z")

hcs_core/ctxp/util.py

@@ -13,17 +13,21 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
+import datetime
+import json
 import os
+import re
 import subprocess
-import json
-import yaml
-import types
 import sys
-import httpx
 import traceback
-import click
+import types
 from typing import Any, Callable
+
+import click
+import httpx
 import questionary
+import yaml
+import yumako
 
 
 class CtxpException(Exception):
@@ -55,7 +59,8 @@ def validate_error_return(reason, return_code):
 def print_output(data: Any, args: dict, file=sys.stdout):
     output = args.get("output", "json")
     fields = args.get("field")
-    id_only = args.get("id_only", False)
+    exclude_field = args.get("exclude_field")
+    ids = args.get("ids", False)
     first = args.get("first", False)
 
     if type(data) is str:
@@ -66,20 +71,25 @@ def print_output(data: Any, args: dict, file=sys.stdout):
         try:
             data = _convert_generator(data)
             if first and isinstance(data, list):
+                if len(data) == 0:
+                    return
                 data = data[0]
 
-            if id_only:
+            if ids:
                 if fields:
-                    raise CtxpException("--id-only and --fields should not be used together.")
+                    raise CtxpException("--ids and --fields should not be used together.")
                 data = _convert_to_id_only(data)
-            elif fields:
-                data = _filter_fields(data, fields)
+            else:
+                if fields:
+                    data = _filter_fields(data, fields)
+                if exclude_field:
+                    data = _exclude_fields(data, exclude_field)
 
             if output is None or output == "json":
                 text = json.dumps(data, default=vars, indent=4)
             elif output == "json-compact":
                 text = json.dumps(data, default=vars)
-            elif output == "yaml":
+            elif output == "yaml" or output == "yml":
                 from . import jsondot
 
                 text = yaml.dump(jsondot.plain(data), sort_keys=False)
@@ -87,7 +97,18 @@ def print_output(data: Any, args: dict, file=sys.stdout):
                 if isinstance(data, list):
                     text = ""
                     for i in data:
-                        line = i if type(i) is str else json.dumps(i)
+                        t = type(i)
+                        if t is str:
+                            line = i
+                        elif isinstance(i, dict):
+                            if len(i) == 0:
+                                continue
+                            if len(i) == 1:
+                                line = str(next(iter(i.values())))
+                            else:
+                                line = json.dumps(i)
+                        else:
+                            line = json.dumps(i)
                         text += line + "\n"
                 elif isinstance(data, dict):
                     text = json.dumps(data, indent=4)
@@ -95,7 +116,7 @@ def print_output(data: Any, args: dict, file=sys.stdout):
                     text = data
                 else:
                     text = json.dumps(data, indent=4)
-            elif output == "table":
+            elif output == "table" or output == "t":
                 formatter = args["format"]
                 text = formatter(data)
             else:
@@ -108,7 +129,7 @@ def print_output(data: Any, args: dict, file=sys.stdout):
 
 
 def print_error(error):
-    critical_errors = [KeyError, TypeError]
+    critical_errors = [KeyError, TypeError, AttributeError, ValueError, IndentationError, ImportError]
     for ex in critical_errors:
         if isinstance(error, ex):
             traceback.print_exception(type(error), error, error.__traceback__, file=sys.stderr)
@@ -155,7 +176,27 @@ def _filter_fields(obj: Any, fields: str):
     return _filter_obj(obj)
 
 
+def _exclude_fields(obj: Any, fields_exclude: str):
+    parts = fields_exclude.split(",")
+
+    def _filter_obj(o):
+        if not isinstance(o, dict):
+            return o
+        for k in list(o.keys()):
+            if k in parts:
+                del o[k]
+        return o
+
+    if isinstance(obj, list):
+        return list(map(_filter_obj, obj))
+    return _filter_obj(obj)
+
+
 def panic(reason: Any = None, code: int = 1):
+    if isinstance(reason, SystemExit):
+        os._exit(reason.code)
+    if isinstance(reason, click.exceptions.Exit):
+        os._exit(reason.exit_code)
     if isinstance(reason, Exception):
         text = error_details(reason)
     else:
@@ -182,7 +223,11 @@ def choose(prompt: str, items: list, fn_get_text: Callable = None, selected=None
         panic(prompt + " ERROR: no item available.")
 
     if fn_get_text is None:
-        fn_get_text = lambda t: str(t)
+
+        def _default_fn_get_text(t):
+            return str(t)
+
+        fn_get_text = _default_fn_get_text
 
     if select_by_default and len(items) == 1:
         ret = items[0]
@@ -221,31 +266,44 @@ def input_array(prompt: str, default: list[str] = None):
     return ret
 
 
-def error_details(e):
-    if isinstance(e, Exception):
-        if isinstance(e, CtxpException):
-            details = str(e)
-        else:
-            details = e.__class__.__name__
-            msg = str(e)
-            if msg:
-                details += ": " + msg
-        cause = e.__cause__
-        if cause and cause != e:
-            details += " | Caused by: " + error_details(cause)
+def error_details(ex):
+    if not isinstance(ex, Exception):
+        return str(ex)
+
+    collector = []
 
+    def _collect_details(e):
+        if isinstance(e, click.ClickException):
+            collector.append(str(e))
+            return
+
+        details = e.__class__.__name__
+        msg = str(e)
+        if msg:
+            details += ": " + msg
         if isinstance(e, httpx.HTTPStatusError):
             details += "\n" + e.response.text
-        return details
-    else:
-        return str(e)
+        collector.append(details)
+
+        cause = e.__cause__
+        if cause and cause != e:
+            _collect_details(cause)
+
+    _collect_details(ex)
+
+    # remove_consecutive_duplicates
+    result = [collector[0]]
+    for item in collector[1:]:
+        if item != result[-1]:
+            result.append(item)
+    return " | Caused by: ".join(result)
 
 
 def avoid_trace_for_ctrl_c():
     import sys
 
     def my_except_hook(exctype, value, traceback):
-        if exctype == KeyboardInterrupt:
+        if exctype is KeyboardInterrupt:
             print("Aborted (KeyboardInterrupt).", flush=True)
             sys.exit(1)
         else:
@@ -292,14 +350,117 @@ def flatten_dict(data, fields_mapping):
     return flattened_data
 
 
-def format_table(data: list, fields_mapping: dict):
+def strip_ansi(text):
+    # Regular expression to match ANSI escape sequences
+    ansi_escape = re.compile(r"\x1b\[[0-9;]*m")
+    return ansi_escape.sub("", text)
+
+
+def format_table(data: list, fields_mapping: dict, columns_to_sum: list = None):
     from tabulate import tabulate
 
     flattened_data = flatten_dict(data, fields_mapping)
     try:
         headers = list(fields_mapping.values())
         table = [[item.get(field) for field in headers] for item in flattened_data]
-    except Exception as e:
+
+        if columns_to_sum:
+            columns_to_sum_indices = {col: headers.index(col) for col in columns_to_sum if col in headers}
+            footer = [""] * len(headers)
+            footer[0] = "Total"
+            for col_name, col_index in columns_to_sum_indices.items():
+                total = 0
+                for row in table:
+                    v = row[col_index]
+                    if isinstance(v, str):
+                        v = strip_ansi(v)
+                        v = int(v)
+                    elif isinstance(v, int) or isinstance(v, float):
+                        pass
+                    elif v is None:
+                        continue
+                    else:
+                        raise Exception(f"Unexpected cell value type. Type={type(v)}, value={v}, col={col_name}")
+                    total += v
+                footer[col_index] = total
+            separator = ["-" * len(header) for header in headers]
+            table += [separator, footer]
+    except Exception:
         traceback.print_exc()
 
     return tabulate(table, headers=headers) + "\n"
+
+
+def colorize(data: dict, name: str, mapping: dict):
+    if os.environ.get("TERM_COLOR") == "0":
+        return
+
+    s = data.get(name)
+    if not s:
+        return
+
+    if isinstance(mapping, dict):
+        c = mapping.get(s)
+        if c:
+            if isinstance(c, str):
+                data[name] = click.style(s, fg=c)
+            elif callable(c):
+                color = c(data)
+                data[name] = click.style(s, fg=color)
+            else:
+                raise Exception(f"Unexpected color type: {type(c)} {c}")
+    elif callable(mapping):
+        c = mapping(s)
+        if c:
+            data[name] = click.style(s, fg=c)
+    else:
+        raise Exception(f"Unexpected mapping type: {type(mapping)} {mapping}")
+
+
+def default_table_formatter(data: Any, mapping: dict = None):
+    if not isinstance(data, list):
+        return data
+
+    def _restrict_readable_length(data: dict, name: str, length: int):
+        text = data.get(name)
+        if not text:
+            return
+        if len(text) > length:
+            data[name] = text[: length - 3] + "..."
+
+    field_mapping = {}
+    for d in data:
+        if "id" in d:
+            field_mapping["id"] = "Id"
+        if "name" in d:
+            field_mapping["name"] = "Name"
+        if "location" in d:
+            field_mapping["location"] = "Location"
+        if "type" in d:
+            field_mapping["type"] = "Type"
+        if "status" in d:
+            field_mapping["status"] = "Status"
+        if "createdAt" in d:
+            d["_createdStale"] = yumako.time.stale(d["createdAt"], datetime.timezone.utc)
+            field_mapping["_createdStale"] = "Created At"
+        if "updatedAt" in d:
+            d["_updatedStale"] = yumako.time.stale(d["updatedAt"], datetime.timezone.utc)
+            field_mapping["_updatedStale"] = "Updated At"
+
+        colorize(
+            d,
+            "status",
+            {
+                "READY": "green",
+                "SUCCESS": "green",
+                "ERROR": "red",
+            },
+        )
+        _restrict_readable_length(d, "name", 60)
+    if mapping:
+        for k, v in mapping.items():
+            if v is None:
+                field_mapping.pop(k, None)
+            else:
+                field_mapping[k] = v
+    return format_table(data, fields_mapping=field_mapping)

hcs_core/ctxp/var_template.py

@@ -13,13 +13,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-import re
 import logging
+import re
 from typing import Any
-from . import jsondot
+
+from . import context, jsondot, profile
 from .jsondot import dotdict
-from . import profile
-from . import context
 
 log = logging.getLogger(__name__)
 

hcs_core/plan/__init__.py

@@ -1,5 +1,11 @@
-from .core import apply, destroy, graph, get_deployment_data, clear
-from .helper import PlanException, PluginException
-from .actions import actions
-from . import context
-from .kop import attach_job_view
+from . import context as context
+from .actions import actions as actions
+from .core import apply as apply
+from .core import clear as clear
+from .core import destroy as destroy
+from .core import get_deployment_data as get_deployment_data
+from .core import graph as graph
+from .core import resolve as resolve
+from .helper import PlanException as PlanException
+from .helper import PluginException as PluginException
+from .kop import attach_job_view as attach_job_view