malevich-coretools 0.2.17__tar.gz → 0.2.19__tar.gz

{malevich-coretools-0.2.17/malevich_coretools.egg-info → malevich-coretools-0.2.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: malevich-coretools
-Version: 0.2.17
+Version: 0.2.19
 Author: Andrew Pogrebnoj
 Author-email: andrew@onjulius.co
 License-File: LICENSE

malevich-coretools-0.2.19/VERSION

@@ -0,0 +1 @@
+0.2.19

{malevich-coretools-0.2.17 → malevich-coretools-0.2.19}/malevich_coretools/__init__.py

@@ -1,2 +1,3 @@
+from .secondary import logs_streaming  # noqa: F401
 from .tools import vast_settings  # noqa: F401
 from .utils import *  # noqa: F403

{malevich-coretools-0.2.17 → malevich-coretools-0.2.19}/malevich_coretools/abstract/abstract.py

@@ -346,8 +346,8 @@ class AppLog(BaseModel):
 
 class AppLogs(BaseModel):
     operationId: Alias.Id
-    dagLogs: str
-    data: Dict[str, AppLog]
+    dagLogs: str = ""
+    data: Dict[str, AppLog] = {}
     error: Optional[str] = None
 
 

{malevich-coretools-0.2.17 → malevich-coretools-0.2.19}/malevich_coretools/secondary/config.py

@@ -8,6 +8,7 @@ class Config:
     TOKEN = os.environ.get("GITLAB_ACCESS_TOKEN")   # FIXME
 
     HOST_PORT = None
+    KAFKA_HOST_PORT = None
     CORE_USERNAME = None
     CORE_PASSWORD = None
     VERBOSE = False

{malevich-coretools-0.2.17 → malevich-coretools-0.2.19}/malevich_coretools/secondary/helpers.py

@@ -1,15 +1,18 @@
 import json
 import random as rand
 import string
-from typing import Any, Callable, Dict, Tuple, Union
+from typing import Any, Callable, Dict, Optional, Tuple, Union
 
 from pydantic import BaseModel
 
 from malevich_coretools.abstract.abstract import Alias, AppLogs, LogsResult
 from malevich_coretools.secondary import Config
+from malevich_coretools.secondary.kafka import handle_logs
 
 __mini__delimiter = "-" * 25
 __delimiter = "-" * 50
+__colors = [f"\x1b[{i};20m" for i in range(31, 38)]
+__color_reset = "\x1b[0m"
 
 
 def to_json(data: Union[Dict[str, Any], Alias.Json], condition_and_msg: Tuple[Callable[[Dict[str, Any]], bool], str] = (lambda _: True, "")) -> str:
@@ -53,7 +56,7 @@ def __show_logs_result(res: LogsResult):  # noqa: ANN202
         print(logs)
 
 
-def __show_logs(app_logs: AppLogs, err: bool = False):  # noqa: ANN202
+def show_logs(app_logs: AppLogs, err: bool = False) -> None:  # noqa: ANN202
     show = Config.logger.error if err else Config.logger.info
     show(f"operation_id = {app_logs.operationId}")
     if app_logs.error is not None:
@@ -73,10 +76,51 @@ def __show_logs(app_logs: AppLogs, err: bool = False):  # noqa: ANN202
         print(__delimiter)
 
 
+def show_logs_colored(app_logs: AppLogs, colors_dict: Optional[Dict[str, str]] = None) -> None:
+    """colors_dict - should be unique for all app_logs by operation_id"""
+    def format(log, color: Optional[str]) -> None:
+        if color is None:
+            Config.logger.warning(log)
+        else:
+            Config.logger.warning(color + log + __color_reset)
+
+    def get_color(name: str) -> str:
+        if colors_dict is None:
+            return None
+        color = colors_dict.get(name, None)
+        if color is None:
+            color = __colors[len(colors_dict) % len(__colors)]
+            colors_dict[name] = color
+        return color
+
+    if app_logs.error is not None:
+        format(f"error: {app_logs.error}", get_color("error"))
+    if len(app_logs.dagLogs) > 0:
+        color = get_color("dagLogs")
+        for line in app_logs.dagLogs.splitlines():
+            format(f"dag: {line}", color)
+    for app_name, app_log in app_logs.data.items():
+        color = get_color(f"${app_name}")
+        for i, logs_result in enumerate(app_log.data):
+            app_name_prefix = f"{app_name}${i}" if i != 0 else app_name
+            if len(logs_result.data) > 0:
+                for line in logs_result.data.splitlines():
+                    format(f"{app_name_prefix}$main: {line}", color)
+            if len(logs_result.logs) > 0:
+                for run_id, logs in logs_result.logs.items():
+                    user_logs = logs_result.userLogs.get(run_id, "")
+                    if len(user_logs) > 0:
+                        for line in user_logs.splitlines():
+                            format(f"{app_name_prefix}${run_id}: {line}", color)
+                    if len(logs) > 0:
+                        for line in logs.splitlines():
+                            format(f"{app_name_prefix}${run_id}: {line}", color)
+
+
 def show_logs_func(data: str, err: bool = False):  # noqa: ANN201
     try:
         app_logs = AppLogs.parse_raw(data)
-        __show_logs(app_logs, err=err)
+        show_logs(app_logs, err=err)
     except BaseException:
         Config.logger.error("decode logs failed")
         show = Config.logger.error if err else Config.logger.info
@@ -92,3 +136,9 @@ def show_fail_app_info(data: str, err: bool):  # noqa: ANN201
     except BaseException:
         Config.logger.error("decode unsuccessful app_info fail")
         print(data)
+
+
+def logs_streaming(operation_id: str, kafka_host_port: Optional[str] = None, app_logs_show: Callable[[AppLogs], None] = show_logs_colored) -> None:
+    colors_dict = {}
+    for appLogs in handle_logs(operation_id, kafka_host_port=kafka_host_port):
+        app_logs_show(appLogs, colors_dict=colors_dict)

malevich-coretools-0.2.19/malevich_coretools/secondary/kafka.py

@@ -0,0 +1,32 @@
+from typing import Generator, Optional
+
+from kafka import KafkaConsumer
+
+from malevich_coretools.abstract import AppLogs
+from malevich_coretools.secondary import Config
+
+
+def __logs_topic(operation_id: str) -> str:
+    return f"{operation_id}-logs"
+
+
+def handle_logs(operation_id: str, kafka_host_port: Optional[str] = None) -> Generator[AppLogs, None, None]:
+    if kafka_host_port is None:
+        kafka_host_port = Config.KAFKA_HOST_PORT
+    assert kafka_host_port is not None, "kafka_host_port not set"
+
+    topic = __logs_topic(operation_id)
+    consumer = KafkaConsumer(topic, bootstrap_servers=kafka_host_port)
+
+    try:
+        for message in consumer:
+            if message.value == b'end':
+                return
+            try:
+                logs = AppLogs.model_validate_json(message.value)
+            except BaseException as ex:
+                Config.logger.error(message.value)
+                raise ex
+            yield logs
+    except KeyboardInterrupt:
+        pass

{malevich-coretools-0.2.17 → malevich-coretools-0.2.19}/malevich_coretools/utils.py

@@ -29,6 +29,12 @@ def set_host_port(host_port: str) -> None:
     Config.HOST_PORT = host_port
 
 
+def set_kafka_host_port(host_port: str) -> None:
+    """update kafka host and port for malevich-kafka, example: `localhost:9092` """
+    assert len(host_port) > 0, "empty host port"
+    Config.KAFKA_HOST_PORT = host_port
+
+
 def set_conn_url(conn_url: str) -> None:
     """analogue set_host_port; update `conn_url` for malevich-core, example: `http://localhost:8080/` """
     set_host_port(conn_url)

{malevich-coretools-0.2.17 → malevich-coretools-0.2.19/malevich_coretools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: malevich-coretools
-Version: 0.2.17
+Version: 0.2.19
 Author: Andrew Pogrebnoj
 Author-email: andrew@onjulius.co
 License-File: LICENSE

{malevich-coretools-0.2.17 → malevich-coretools-0.2.19}/malevich_coretools.egg-info/SOURCES.txt

@@ -23,5 +23,6 @@ malevich_coretools/secondary/__init__.py
 malevich_coretools/secondary/config.py
 malevich_coretools/secondary/const.py
 malevich_coretools/secondary/helpers.py
+malevich_coretools/secondary/kafka.py
 malevich_coretools/tools/__init__.py
 malevich_coretools/tools/vast.py

malevich-coretools-0.2.17/VERSION

@@ -1 +0,0 @@
-0.2.17