hcs-cli 0.1.330__tar.gz → 0.1.332__tar.gz

{hcs_cli-0.1.330 → hcs_cli-0.1.332}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hcs-cli
-Version: 0.1.330
+Version: 0.1.332
 Summary: Horizon Cloud Service CLI.
 Project-URL: Homepage, https://github.com/euc-eng/hcs-cli
 Project-URL: Bug Tracker, https://github.com/euc-eng/hcs-cli/issues
@@ -14,7 +14,7 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.9
-Requires-Dist: hcs-core>=0.1.330
+Requires-Dist: hcs-core>=0.1.332
 Requires-Dist: inquirerpy>=0.3.4
 Requires-Dist: matplotlib>=3.8.0
 Requires-Dist: paho-mqtt>=2.1.0

{hcs_cli-0.1.330 → hcs_cli-0.1.332}/hcs_cli/__init__.py

@@ -1,3 +1,3 @@
-__version__ = "0.1.330"
+__version__ = "0.1.332"
 
 from . import service as service  # noqa: F401

hcs_cli-0.1.332/hcs_cli/cmds/hoc/log.py

@@ -0,0 +1,327 @@
+"""
+Copyright © 2025 Omnissa, LLC.
+"""
+
+import json
+import re
+import sys
+from datetime import timezone
+from typing import Optional
+
+import click
+import yumako
+from hcs_core.ctxp import util
+from hcs_core.sglib import cli_options as cli
+
+from hcs_cli.service import hoc
+from hcs_cli.support import predefined_payload
+
+_hdcs = ["US", "EU", "JP"]
+
+_regions = [
+    "eastus2",
+    "westus2",
+    "northcentralus",
+    "northeurope",
+    "germanywestcentral",
+    "uksouth",
+    "japaneast",
+    "australiaeast",
+    "centralindia",
+]
+_stacks = ["prod", "staging", "stress"]
+
+
+@click.command()
+@cli.org_id
+@click.option(
+    "--from",
+    "from_param",
+    type=str,
+    required=False,
+    default="-24h",
+    help="Sepcify the from date. E.g. '-1d', or '-1h35m', or '-1w', or '2023-12-04T00:19:22.854Z'.",
+)
+@click.option(
+    "--to",
+    type=str,
+    required=False,
+    default="now",
+    help="Sepcify the to date. E.g. 'now', or '-1d', or '-1h35m', or '-1w', or '2023-12-04T00:19:22.854Z'.",
+)
+@click.option("--hdc", required=False, type=click.Choice(_hdcs, case_sensitive=False), help="The HDC location")
+@click.option("--region", required=False, type=click.Choice(_regions, case_sensitive=False), help="The region")
+@click.option("--stack", required=False, type=click.Choice(_stacks, case_sensitive=False), help="The stack")
+@click.option(
+    "--service",
+    required=False,
+    help="Filter by kubernetes.pod_labels.app. If multiple apps, use comma to separate, e.g. --service inventory,lcm",
+)
+@click.option("--raw", is_flag=True, default=False, help="If raw, output the raw result.")
+@click.argument("query", type=str, required=False)
+def log(org: str, from_param: str, to: str, hdc: str, region: str, stack: str, service: str, raw: bool, query: str):
+    """Search Kibana Log."""
+
+    # org_id = cli.get_org_id(org)
+
+    if not hdc:
+        hdc = "US"
+    else:
+        hdc = hdc.upper()
+    index_map = {
+        "US": "prod110-hdc-westus2-cp102-*",
+        # "US": "prod110-hdc-centralus-cp102-*",
+        "EU": "hoc-logs-eu-prod-*",
+        "JP": "hoc-logs-jp-prod-*",
+        "AUS": "",
+        "IND": "",
+        "GER": "",
+        "UK": "",
+        "US_APM": "prod110-hdc-westus2-cp102-*",
+        "EU_APM": "",
+        "JP_APM": "",
+    }
+    es_logs_search_index = index_map[hdc]
+
+    text = predefined_payload.load("hoc/logs.jsont")
+    replacements = {
+        "from": int(yumako.time.of(from_param, tz=timezone.utc).timestamp() * 1000),
+        "to": int(yumako.time.of(to, tz=timezone.utc).timestamp() * 1000),
+        "query": query or "",
+        "es_logs_search_index": es_logs_search_index,
+    }
+    try:
+        text = yumako.template.replace(
+            text=text,
+            mapping=replacements,
+            raise_on_unresolved_vars=True,
+            raise_on_unused_vars=True,
+        )
+    except ValueError as e:
+        return util.error_details(e), 1
+
+    parts = text.split("///", 2)
+    try:
+        first_line = json.loads(parts[0])
+    except Exception as e:
+        print(text, file=sys.stderr)
+        print(e, file=sys.stderr)
+        return
+    try:
+        second_line = json.loads(parts[1])
+    except Exception as e:
+        print(text, file=sys.stderr)
+        print(e, file=sys.stderr)
+        return
+
+    _update_query(second_line, any_apps=service)
+
+    payload_jsonp = json.dumps(first_line) + "\n" + json.dumps(second_line) + "\n"
+
+    hdc_location_mapping = {
+        # "US", "EU", "JP", "AUS", "IND", "GER", "UK", "US_APM", "EU_APM", "JP_APM"
+        "US": "US_APM",
+        "EU": "EU_APM",
+        "JP": "JP_APM",
+    }
+
+    # print(payload_jsonp)
+    result = hoc.es.raw_query(payload_jsonp, hdc_location=hdc_location_mapping[hdc])
+    if raw:
+        return result
+
+    # format the result to pure message array
+    ret = []
+    for item in result:
+        message = item["message"]
+        parsed = _parse_message(message)
+        ret.append(parsed)
+    return ret
+
+
+# Pattern 1: LCM / service-gateway style
+# "...INFO  [163053][mplateMonitor-119158] c.v.h.servicegateway.lcm.utils.KopLog..."
+_PATTERN_LCM = re.compile(
+    r"(?P<timestamp>\S+)"
+    r"\s+\[trace_id=(?P<trace_id>[^\]]*)\]"
+    r"\s+\[transaction_id=(?P<transaction_id>[^\]]*)\]"
+    r"\s+(?P<level>\S+)"
+    r"\s+\[(?P<thread_id>[^\]]*)\]\[(?P<thread_name>[^\]]*)\]"
+    r"\s+(?P<logger>\S+)"
+    r"\s+-\s+"
+    r"(?P<body>.*)",
+)
+
+# Pattern 2: clouddriver style (single thread bracket after level)
+# "...INFO          [http-nio-8080-exec-6] c.v.horizon.sg.clouddriver..."
+_PATTERN_CLOUDDRIVER = re.compile(
+    r"(?P<timestamp>\S+)"
+    r"\s+\[trace_id=(?P<trace_id>[^\]]*)\]"
+    r"\s+\[transaction_id=(?P<transaction_id>[^\]]*)\]"
+    r"\s+(?P<level>\S+)"
+    r"\s+\[(?P<thread_name>[^\]]*)\]"
+    r"\s+(?P<logger>\S+)"
+    r"\s+-\s+"
+    r"(?P<body>.*)",
+)
+
+# Pattern 3: task-service style (thread bracket before level)
+# "...[transaction_id=...] [tsch:iss-32521] INFO  c.v.h.s.t.service.TaskService..."
+_PATTERN_TASK = re.compile(
+    r"(?P<timestamp>\S+)"
+    r"\s+\[trace_id=(?P<trace_id>[^\]]*)\]"
+    r"\s+\[transaction_id=(?P<transaction_id>[^\]]*)\]"
+    r"\s+\[(?P<thread_name>[^\]]*)\]"
+    r"\s+(?P<level>\S+)"
+    r"\s+(?P<logger>\S+)"
+    r"\s+-\s+"
+    r"(?P<body>.*)",
+)
+
+# Pattern 4
+# "2026-04-10 00:24:26,559 [http-nio-8080-exec-10] WARN  co.elastic.apm.agent.impl.transaction.TraceState - sample rate already set to 1.0, trying to set it to 1.0 through header will be ignored"
+_PATTERN_ELASTIC = re.compile(
+    r"(?P<timestamp>\S+\s\S+)" r"\s+\[(?P<thread_name>[^\]]*)\]" r"\s+(?P<level>\S+)" r"\s+(?P<logger>\S+)" r"\s+-\s+" r"(?P<body>.*)",
+)
+
+# Pattern 5: JSON structured log (apm-server style)
+# "{\"log.level\":\"info\",\"@timestamp\":\"2026-04-10T00:24:26.596Z\",\"log.logger\":\"request\",...}"
+_PATTERN_JSON_LOG = re.compile(r"\s*\{.*\"@timestamp\".*\}")
+
+# Pattern 6: Envoy/Istio access log
+# "[2026-04-10T00:24:25.684Z] \"POST /ad-twin/v1/... HTTP/1.1\" 200 ..."
+_PATTERN_ENVOY = re.compile(
+    r"\[(?P<timestamp>[^\]]+)\]"
+    r"\s+\"(?P<method>\S+)\s+(?P<path>\S+)\s+(?P<protocol>[^\"]+)\""
+    r"\s+(?P<status_code>\d+)"
+    r"\s+(?P<body>.*)",
+)
+
+# Pattern 7 & 8: portal/pool style (empty transaction_id bracket)
+# "2026-04-10T00:27:57.372Z [trace_id=...] [] [kafka-consumer-listener-...] INFO  c.v.h.p.handler..."
+# "2026-04-10T00:27:57.376Z [trace_id=...] [] [http-nio-8080-exec-26] INFO  c.v.h.portal.service..."
+_PATTERN_PORTAL = re.compile(
+    r"(?P<timestamp>\S+)"
+    r"\s+\[trace_id=(?P<trace_id>[^\]]*)\]"
+    r"\s+\[(?P<transaction_id>[^\]]*)\]"
+    r"\s+\[(?P<thread_name>[^\]]*)\]"
+    r"\s+(?P<level>\S+)"
+    r"\s+(?P<logger>\S+)"
+    r"\s+-\s+"
+    r"(?P<body>.*)",
+)
+
+# Pattern 9: Spring Boot style (--- separator)
+# "2026-04-10 00:29:34.037  INFO 8 --- [istenerId-0-C-1] c.v.h.a.f.kafka.KafkaConsumerService     : Created ..."
+_PATTERN_SPRING = re.compile(
+    r"(?P<timestamp>\S+\s\S+)"
+    r"\s+(?P<level>\S+)"
+    r"\s+\S+"
+    r"\s+---"
+    r"\s+\[(?P<thread_name>[^\]]*)\]"
+    r"\s+(?P<logger>\S+)"
+    r"\s+:\s+"
+    r"(?P<body>.*)",
+)
+
+_PATTERNS = [_PATTERN_LCM, _PATTERN_CLOUDDRIVER, _PATTERN_TASK, _PATTERN_ELASTIC, _PATTERN_ENVOY, _PATTERN_PORTAL, _PATTERN_SPRING]
+
+
+def _parse_message(message: str) -> dict:
+    if _PATTERN_JSON_LOG.match(message):
+        try:
+            obj = json.loads(message)
+            return {
+                "timestamp": obj.get("@timestamp", ""),
+                "level": obj.get("log.level", ""),
+                "logger": obj.get("log.logger", ""),
+                "body": obj.get("message", ""),
+            }
+        except json.JSONDecodeError:
+            pass
+    for pattern in _PATTERNS:
+        m = pattern.match(message)
+        if m:
+            return m.groupdict()
+    return {"body": message}
+
+
+def _update_query(query_object: dict, any_apps: Optional[str]) -> dict:
+    # Example query object: query string "KOP" and "V1-CUSFP0101I", time range, and kubernetes.pod_labels.app=lcm
+    # {
+    # "query": {
+    #     "bool": {
+    #         "must": [],
+    #         "filter": [
+    #             {
+    #                 "bool": {
+    #                     "filter": [
+    #                         {
+    #                             "multi_match": {
+    #                                 "type": "phrase",
+    #                                 "query": "KOP",
+    #                                 "lenient": true
+    #                             }
+    #                         },
+    #                         {
+    #                             "multi_match": {
+    #                                 "type": "phrase",
+    #                                 "query": "V1-CUSFP0101I",
+    #                                 "lenient": true
+    #                             }
+    #                         }
+    #                     ]
+    #                 }
+    #             },
+    #             {
+    #                 "range": {
+    #                     "timestamp": {
+    #                         "format": "strict_date_optional_time",
+    #                         "gte": "2026-04-09T22:42:54.486Z",
+    #                         "lte": "2026-04-09T22:47:54.486Z"
+    #                     }
+    #                 }
+    #             },
+    #             {
+    #                 "match_phrase": {
+    #                     "kubernetes.pod_labels.app": "lcm"
+    #                 }
+    #             }
+    #         ],
+    #         "should": [],
+    #         "must_not": []
+    #     }
+    # }
+    # }
+
+    # example of the last match condition, if there are multiple app specified (is one of):
+    # {
+    #     "bool": {
+    #         "minimum_should_match": 1,
+    #         "should": [
+    #             {
+    #                 "match_phrase": {
+    #                     "kubernetes.pod_labels.app": "clouddriver"
+    #                 }
+    #             },
+    #             {
+    #                 "match_phrase": {
+    #                     "kubernetes.pod_labels.app": "lcm"
+    #                 }
+    #             }
+    #         ]
+    #     }
+    # }
+
+    filter_array = query_object["query"]["bool"]["filter"]
+
+    if any_apps:
+        apps = [a.strip() for a in any_apps.split(",") if a.strip()]
+        if len(apps) == 1:
+            filter_array.append({"match_phrase": {"kubernetes.pod_labels.app": apps[0]}})
+        elif len(apps) > 1:
+            filter_array.append(
+                {"bool": {"minimum_should_match": 1, "should": [{"match_phrase": {"kubernetes.pod_labels.app": app}} for app in apps]}}
+            )
+
+    return query_object

{hcs_cli-0.1.330 → hcs_cli-0.1.332}/hcs_cli/cmds/hoc/search.py

@@ -36,6 +36,17 @@ def _formalize_query_string(input_str):
     return " ".join(parts).strip()
 
 
+def _parse_kv_args(arg_list: list[str]) -> dict[str, str]:
+    result = {}
+    for arg in arg_list:
+        if "=" not in arg:
+            raise ValueError(f"Invalid argument: {arg}. Expected format: key=value")
+        key, value = arg.split("=", 1)
+        key = key.strip()
+        result[key] = value.strip()
+    return result
+
+
 @click.command()
 @cli.org_id
 @click.option(
@@ -167,100 +178,3 @@ def es(org: str, from_param: str, to: str, file, raw: bool, predefined: str, arg
 
     print(payload_jsonp)
     return hoc.es.raw_query(payload_jsonp)
-
-
-_hdcs = ["US", "EU", "JP"]
-_regions = [
-    "eastus2",
-    "westus2",
-    "northcentralus",
-    "northeurope",
-    "germanywestcentral",
-    "uksouth",
-    "japaneast",
-    "australiaeast",
-    "centralindia",
-]
-
-
-@click.command()
-@cli.org_id
-@click.option(
-    "--from",
-    "from_param",
-    type=str,
-    required=False,
-    default="-12h",
-    help="Sepcify the from date. E.g. '-1d', or '-1h35m', or '-1w', or '2023-12-04T00:19:22.854Z'.",
-)
-@click.option(
-    "--to",
-    type=str,
-    required=False,
-    default="now",
-    help="Sepcify the to date. E.g. 'now', or '-1d', or '-1h35m', or '-1w', or '2023-12-04T00:19:22.854Z'.",
-)
-@click.option("--hdc", required=False, type=click.Choice(_hdcs, case_sensitive=False), help="The HDC location")
-@click.option("--region", required=False, type=click.Choice(_regions, case_sensitive=False), help="The HDC location")
-@click.argument("query", type=str, required=False)
-def logs(org: str, from_param: str, to: str, hdc: str, region: str, query: str):
-    """Perform a raw ES query for HCS logs."""
-
-    # org_id = cli.get_org_id(org)
-
-    if not hdc:
-        hdc = "US"
-    else:
-        hdc = hdc.upper()
-    index_map = {
-        "US": "prod110-hdc-centralus-cp102-*",
-        "EU": "hoc-logs-eu-prod-*",
-        "JP": "hoc-logs-jp-prod-*",
-    }
-    es_logs_search_index = index_map[hdc]
-
-    text = predefined_payload.load("hoc/logs.jsont")
-    replacements = {
-        "from": int(yumako.time.of(from_param, tz=timezone.utc).timestamp() * 1000),
-        "to": int(yumako.time.of(to, tz=timezone.utc).timestamp() * 1000),
-        "query": query if query else "",
-        "es_logs_search_index": es_logs_search_index,
-    }
-    try:
-        text = yumako.template.replace(
-            text=text,
-            mapping=replacements,
-            raise_on_unresolved_vars=True,
-            raise_on_unused_vars=True,
-        )
-    except ValueError as e:
-        return util.error_details(e), 1
-
-    parts = text.split("///", 2)
-    try:
-        first_line = json.loads(parts[0])
-    except Exception as e:
-        print(text, file=sys.stderr)
-        print(e, file=sys.stderr)
-        return
-    try:
-        second_line = json.loads(parts[1])
-    except Exception as e:
-        print(text, file=sys.stderr)
-        print(e, file=sys.stderr)
-        return
-    payload_jsonp = json.dumps(first_line) + "\n" + json.dumps(second_line) + "\n"
-
-    print(payload_jsonp)
-    return hoc.es.raw_query(payload_jsonp)
-
-
-def _parse_kv_args(arg_list: list[str]) -> dict[str, str]:
-    result = {}
-    for arg in arg_list:
-        if "=" not in arg:
-            raise ValueError(f"Invalid argument: {arg}. Expected format: key=value")
-        key, value = arg.split("=", 1)
-        key = key.strip()
-        result[key] = value.strip()
-    return result

{hcs_cli-0.1.330 → hcs_cli-0.1.332}/hcs_cli/cmds/task.py

@@ -495,12 +495,15 @@ def list_tasks(
 @cli.org_id
 @click.option("--namespace", "-n", type=str, required=False)
 @click.option("--group", "-g", type=str, required=False)
+@click.option(
+    "--location", hidden=True, type=str, required=False, help="Task location, only for global scheduler. E.g. US, JP, IN, AU, DE, IE."
+)  # to be removed later
 @click.argument("smart_path", type=str, required=False)
-def get(org: str, namespace: str, group: str, smart_path: str, **kwargs):
+def get(org: str, namespace: str, group: str, location: str, smart_path: str, **kwargs):
     """Get a task. E.g. 'task get [[<namespace>/]<group>/]<key>'."""
     namespace, group, key = _parse_task_param(namespace, group, smart_path)
     org_id = cli.get_org_id(org)
-    return task.get(org_id, namespace, group, key, **kwargs)
+    return task.get(org_id, namespace, group, key, location=location, **kwargs)
 
 
 @task_cmd_group.command()
@@ -551,7 +554,9 @@ def get_logs(org: str, namespace: str, group: str, smart_path: str, last: bool,
     if state and last:
         return "--state cannot be used with --last.", 1
 
-    if org == "all" or org is None:
+    if last and org is None:
+        org_id = cli.get_org_id(org)
+    elif org == "all" or org is None:
         org_id = None
     else:
         org_id = cli.get_org_id(org)
@@ -568,7 +573,7 @@ def get_logs(org: str, namespace: str, group: str, smart_path: str, last: bool,
             search = state_search
 
     if last:
-        t = task.last(org_id, namespace, group, key, **kwargs)
+        t = task.last(org_id, namespace, group, key)
         if t:
             return t.log
         else:

{hcs_cli-0.1.330 → hcs_cli-0.1.332}/hcs_cli/cmds/vm/log.py

@@ -28,7 +28,7 @@ _DCT_FAILED = frozenset({"ERROR", "EXPIRED"})
     help="Whether to recreate the log if it already exists. By default, it will not recreate the log and just return the existing log.",
 )
 @click.option(
-    "--silent", is_flag=True, default=False, help="Whether to print progress messages. By default, it will print progress messages."
+    "--verbose", is_flag=True, default=False, help="Whether to print progress messages. By default, it will print progress messages."
 )
 @click.option(
     "--save-as",
@@ -39,15 +39,13 @@ _DCT_FAILED = frozenset({"ERROR", "EXPIRED"})
 )
 @click.argument("vm_path", type=str, required=False)
 @cli.org_id
-def log(recreate: bool, silent: bool, save_path, vm_path: str, org: str, **kwargs):
+def log(recreate: bool, verbose: bool, save_path, vm_path: str, org: str, **kwargs):
     """Download VM DCT log"""
     org_id = cli.get_org_id(org)
     template_id, vm_id = parse_vm_path(vm_path)
 
     is_ops = has_any_permission(["sgadm:supportw", "sgadm:supportlog", "sgadm:service", "sgadm:support"])
 
-    verbose = not silent
-
     dct_info = dct_service.latest(org_id, template_id, vm_id, is_ops)
     if not dct_info:
         if verbose:
@@ -102,6 +100,9 @@ def log(recreate: bool, silent: bool, save_path, vm_path: str, org: str, **kwarg
         return f"Failed to download archive: {e}", 1
 
     dct_info["file_path"] = file_path
+
+    if verbose:
+        click.secho("----- DCT log downloaded -----", fg="bright_black")
     return dct_info
 
 

{hcs_cli-0.1.330 → hcs_cli-0.1.332}/hcs_cli/service/hoc/es.py

@@ -46,6 +46,7 @@ def query(
     to_date: Union[str, datetime] = "now",
     template_file: str = None,
     template: str = None,
+    hdc_location: str = None,
     **kwargs,
 ):
     if template_file:
@@ -87,8 +88,11 @@ def query(
         raise e
 
     payload_jsonp = json.dumps(payload_type) + "\n" + json.dumps(payload_query) + "\n"
-    return raw_query(payload_jsonp)
+    return raw_query(payload_jsonp, hdc_location=hdc_location)
 
 
-def raw_query(payload_jsonp: str):
-    return _client.post("/v1/data/query/search", text=payload_jsonp)
+def raw_query(payload_jsonp: str, hdc_location: str = None):
+    url = "/v1/data/query/search"
+    if hdc_location:
+        url = f"/v1/data/query/search?hdcLocation={hdc_location}"
+    return _client.post(url, text=payload_jsonp)

{hcs_cli-0.1.330 → hcs_cli-0.1.332}/hcs_cli/service/task.py

@@ -312,7 +312,7 @@ def _get_v1(namespace: str, group: str, key: str) -> Optional[TaskModel]:
     return t
 
 
-def _get_v2(org_id: str, namespace: str, group: str, key: str) -> Optional[TaskModel]:
+def _get_v2(org_id: str, namespace: str, group: str, key: str, location: str = None) -> Optional[TaskModel]:
     url = f"/v1/tasks/{key}?"
     if org_id:
         url += f"&orgId={org_id}"
@@ -320,6 +320,8 @@ def _get_v2(org_id: str, namespace: str, group: str, key: str) -> Optional[TaskM
         url += f"&namespace={namespace}"
     if group:
         url += f"&group={group}"
+    if location:
+        url += f"&location={location}"
     t = _scm_client.get(url, type=TaskModel)
     if not t:
         return
@@ -333,14 +335,14 @@ def get(org_id: str, namespace: str, group: str, key: str, **kwargs) -> Optional
         t = _get_v1(namespace, group, key)
         t.log = _lastlog(org_id, namespace, group, key)
     elif version == "v2":
-        t = _get_v2(org_id, namespace, group, key)
+        t = _get_v2(org_id, namespace, group, key, location=kwargs.get("location"))
     else:
         t = _get_v1(namespace, group, key)
         if t:
             t.log = _lastlog(org_id, namespace, group, key)
             _task_temp_lru[_k] = "v1"
         else:
-            t = _get_v2(org_id, namespace, group, key)
+            t = _get_v2(org_id, namespace, group, key, location=kwargs.get("location"))
             if t:
                 _task_temp_lru[_k] = "v2"