malevich-coretools 0.2.17__py3-none-any.whl → 0.2.19__py3-none-any.whl

malevich_coretools/__init__.py

@@ -1,2 +1,3 @@
+from .secondary import logs_streaming  # noqa: F401
 from .tools import vast_settings  # noqa: F401
 from .utils import *  # noqa: F403

malevich_coretools/abstract/abstract.py

@@ -346,8 +346,8 @@ class AppLog(BaseModel):
 
 class AppLogs(BaseModel):
     operationId: Alias.Id
-    dagLogs: str
-    data: Dict[str, AppLog]
+    dagLogs: str = ""
+    data: Dict[str, AppLog] = {}
     error: Optional[str] = None
 
 

malevich_coretools/secondary/config.py

@@ -8,6 +8,7 @@ class Config:
     TOKEN = os.environ.get("GITLAB_ACCESS_TOKEN")   # FIXME
 
     HOST_PORT = None
+    KAFKA_HOST_PORT = None
     CORE_USERNAME = None
     CORE_PASSWORD = None
     VERBOSE = False

malevich_coretools/secondary/helpers.py

@@ -1,15 +1,18 @@
 import json
 import random as rand
 import string
-from typing import Any, Callable, Dict, Tuple, Union
+from typing import Any, Callable, Dict, Optional, Tuple, Union
 
 from pydantic import BaseModel
 
 from malevich_coretools.abstract.abstract import Alias, AppLogs, LogsResult
 from malevich_coretools.secondary import Config
+from malevich_coretools.secondary.kafka import handle_logs
 
 __mini__delimiter = "-" * 25
 __delimiter = "-" * 50
+__colors = [f"\x1b[{i};20m" for i in range(31, 38)]
+__color_reset = "\x1b[0m"
 
 
 def to_json(data: Union[Dict[str, Any], Alias.Json], condition_and_msg: Tuple[Callable[[Dict[str, Any]], bool], str] = (lambda _: True, "")) -> str:
@@ -53,7 +56,7 @@ def __show_logs_result(res: LogsResult):  # noqa: ANN202
         print(logs)
 
 
-def __show_logs(app_logs: AppLogs, err: bool = False):  # noqa: ANN202
+def show_logs(app_logs: AppLogs, err: bool = False) -> None:  # noqa: ANN202
     show = Config.logger.error if err else Config.logger.info
     show(f"operation_id = {app_logs.operationId}")
     if app_logs.error is not None:
@@ -73,10 +76,51 @@ def __show_logs(app_logs: AppLogs, err: bool = False):  # noqa: ANN202
         print(__delimiter)
 
 
+def show_logs_colored(app_logs: AppLogs, colors_dict: Optional[Dict[str, str]] = None) -> None:
+    """colors_dict - should be unique for all app_logs by operation_id"""
+    def format(log, color: Optional[str]) -> None:
+        if color is None:
+            Config.logger.warning(log)
+        else:
+            Config.logger.warning(color + log + __color_reset)
+
+    def get_color(name: str) -> str:
+        if colors_dict is None:
+            return None
+        color = colors_dict.get(name, None)
+        if color is None:
+            color = __colors[len(colors_dict) % len(__colors)]
+            colors_dict[name] = color
+        return color
+
+    if app_logs.error is not None:
+        format(f"error: {app_logs.error}", get_color("error"))
+    if len(app_logs.dagLogs) > 0:
+        color = get_color("dagLogs")
+        for line in app_logs.dagLogs.splitlines():
+            format(f"dag: {line}", color)
+    for app_name, app_log in app_logs.data.items():
+        color = get_color(f"${app_name}")
+        for i, logs_result in enumerate(app_log.data):
+            app_name_prefix = f"{app_name}${i}" if i != 0 else app_name
+            if len(logs_result.data) > 0:
+                for line in logs_result.data.splitlines():
+                    format(f"{app_name_prefix}$main: {line}", color)
+            if len(logs_result.logs) > 0:
+                for run_id, logs in logs_result.logs.items():
+                    user_logs = logs_result.userLogs.get(run_id, "")
+                    if len(user_logs) > 0:
+                        for line in user_logs.splitlines():
+                            format(f"{app_name_prefix}${run_id}: {line}", color)
+                    if len(logs) > 0:
+                        for line in logs.splitlines():
+                            format(f"{app_name_prefix}${run_id}: {line}", color)
+
+
 def show_logs_func(data: str, err: bool = False):  # noqa: ANN201
     try:
         app_logs = AppLogs.parse_raw(data)
-        __show_logs(app_logs, err=err)
+        show_logs(app_logs, err=err)
     except BaseException:
         Config.logger.error("decode logs failed")
         show = Config.logger.error if err else Config.logger.info
@@ -92,3 +136,9 @@ def show_fail_app_info(data: str, err: bool):  # noqa: ANN201
     except BaseException:
         Config.logger.error("decode unsuccessful app_info fail")
         print(data)
+
+
+def logs_streaming(operation_id: str, kafka_host_port: Optional[str] = None, app_logs_show: Callable[[AppLogs], None] = show_logs_colored) -> None:
+    colors_dict = {}
+    for appLogs in handle_logs(operation_id, kafka_host_port=kafka_host_port):
+        app_logs_show(appLogs, colors_dict=colors_dict)

malevich_coretools/secondary/kafka.py

@@ -0,0 +1,32 @@
+from typing import Generator, Optional
+
+from kafka import KafkaConsumer
+
+from malevich_coretools.abstract import AppLogs
+from malevich_coretools.secondary import Config
+
+
+def __logs_topic(operation_id: str) -> str:
+    return f"{operation_id}-logs"
+
+
+def handle_logs(operation_id: str, kafka_host_port: Optional[str] = None) -> Generator[AppLogs, None, None]:
+    if kafka_host_port is None:
+        kafka_host_port = Config.KAFKA_HOST_PORT
+    assert kafka_host_port is not None, "kafka_host_port not set"
+
+    topic = __logs_topic(operation_id)
+    consumer = KafkaConsumer(topic, bootstrap_servers=kafka_host_port)
+
+    try:
+        for message in consumer:
+            if message.value == b'end':
+                return
+            try:
+                logs = AppLogs.model_validate_json(message.value)
+            except BaseException as ex:
+                Config.logger.error(message.value)
+                raise ex
+            yield logs
+    except KeyboardInterrupt:
+        pass

malevich_coretools/utils.py

@@ -29,6 +29,12 @@ def set_host_port(host_port: str) -> None:
     Config.HOST_PORT = host_port
 
 
+def set_kafka_host_port(host_port: str) -> None:
+    """update kafka host and port for malevich-kafka, example: `localhost:9092` """
+    assert len(host_port) > 0, "empty host port"
+    Config.KAFKA_HOST_PORT = host_port
+
+
 def set_conn_url(conn_url: str) -> None:
     """analogue set_host_port; update `conn_url` for malevich-core, example: `http://localhost:8080/` """
     set_host_port(conn_url)

{malevich_coretools-0.2.17.dist-info → malevich_coretools-0.2.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: malevich-coretools
-Version: 0.2.17
+Version: 0.2.19
 Author: Andrew Pogrebnoj
 Author-email: andrew@onjulius.co
 License-File: LICENSE

{malevich_coretools-0.2.17.dist-info → malevich_coretools-0.2.19.dist-info}/RECORD

@@ -1,20 +1,21 @@
-malevich_coretools/__init__.py,sha256=88IRvwNti4YOMJc6RaVRutwurJzP23g30D-rgo04Fgc,82
-malevich_coretools/utils.py,sha256=bk2xYQaGrcdQbyNiEuR2TkPFbdh-oxQD4qvfnEUXj_I,65142
+malevich_coretools/__init__.py,sha256=DJtPESxkCZD2SbTZTrR_x0TKDQ4MJpmBqGw5YpKYidM,134
+malevich_coretools/utils.py,sha256=BlzS_hfWzvqFWaE-dbdcH8oDdgwzMwEcsW9jnFWQNSw,65365
 malevich_coretools/abstract/__init__.py,sha256=ZWtP4gFLpNVFXzlMKXEK4iMnUqqMg07pL8yKGSJd6QI,38
-malevich_coretools/abstract/abstract.py,sha256=zRyUW_PHJvL8NNABZZL3PV_05qkYKp6ugAkfyls95V4,9232
+malevich_coretools/abstract/abstract.py,sha256=wQ_1e8KfWWqHs6xSIQk1j2Ph5OO8saxMcmMdnUB4bo0,9242
 malevich_coretools/admin/__init__.py,sha256=zdIcHs3T_NZ8HYWts-O7OpBEWHIu779QDZMGF5HRCLg,35
 malevich_coretools/admin/utils.py,sha256=NE0Mw_Tytsi0PLD62_sI8tUVZNQaRrtUaa4oFwfUt84,1497
 malevich_coretools/funcs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 malevich_coretools/funcs/funcs.py,sha256=LGOarZ-nyBVo8V96EMU4zRTyz42aTEa5p4ECam6lzSw,32368
 malevich_coretools/funcs/helpers.py,sha256=Y8v6uyawYl2_G0XmhmiVLqGIUzvxnkpKF0ODA9vrB3Y,1276
 malevich_coretools/secondary/__init__.py,sha256=048HqvG36_1WdDVZK_RuECmaf14Iq2fviUysG1inlaE,78
-malevich_coretools/secondary/config.py,sha256=b7RysQTkPKib8FEhfm3uU7TPvz8vgWLlqakvZN5NFs4,391
+malevich_coretools/secondary/config.py,sha256=Lrnh3xl2WPqWF3TvbcPoUnj4LRtceY4pUSMl-4_pQcI,418
 malevich_coretools/secondary/const.py,sha256=2_Uer7-sCRdHRoAyl-LsEBPnQd5BHloWAOzyvs_ZQyI,9592
-malevich_coretools/secondary/helpers.py,sha256=lcWN1ctfVQa5KFqh5mXdp_16hA3TXhJGe4mtNdQHnUA,3072
+malevich_coretools/secondary/helpers.py,sha256=DnQ1J6Kw1AdN2SicRUz5naFZOC3THgCUjUgMfPsMxDg,5365
+malevich_coretools/secondary/kafka.py,sha256=SIUnBFyfwsquN6MAUrEkKCw-1l7979Znl7OTQSX2UKo,989
 malevich_coretools/tools/__init__.py,sha256=jDxlCa5Dr6Y43qlI7JwsRAlBkKmFeTHTEnjNUvu-0iw,46
 malevich_coretools/tools/vast.py,sha256=u1cnfylSgq3nYfonXbTy1zwor3OPkhD3vRf2OnqlMZY,12504
-malevich_coretools-0.2.17.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-malevich_coretools-0.2.17.dist-info/METADATA,sha256=li4ALUaC86aHorIDMCkH7vvyqqhkptY2GvM0DnW9HfM,237
-malevich_coretools-0.2.17.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-malevich_coretools-0.2.17.dist-info/top_level.txt,sha256=wDX3s1Tso0otBPNrFRfXqyNpm48W4Bp5v6JfbITO2Z8,19
-malevich_coretools-0.2.17.dist-info/RECORD,,
+malevich_coretools-0.2.19.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+malevich_coretools-0.2.19.dist-info/METADATA,sha256=SDxX7twPtlbc-j41EHwdeqrjHYIJSnWKP3YzRtEvtMw,237
+malevich_coretools-0.2.19.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+malevich_coretools-0.2.19.dist-info/top_level.txt,sha256=wDX3s1Tso0otBPNrFRfXqyNpm48W4Bp5v6JfbITO2Z8,19
+malevich_coretools-0.2.19.dist-info/RECORD,,