malevich-coretools 0.3.54__py3-none-any.whl → 0.3.61__py3-none-any.whl

malevich_coretools/abstract/abstract.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
 from pydantic import BaseModel
 
@@ -197,6 +197,7 @@ class MainTask(BaseModel):
     profileMode: Optional[str] = None
     withLogs: bool = False  # use only in prepare
     saveFails: bool = True
+    clearDagLogs: bool = True
     scaleCount: int = 1
     scaleInfo: List[ScaleInfo]
     component: TaskComponent
@@ -228,6 +229,7 @@ class MainPipeline(BaseModel):
     run: bool = True
     synthetic: bool = False
     saveFails: bool = True
+    clearDagLogs: bool = True
     scaleCount: int = 1
     tags: Optional[Dict[str, str]] = None
 
@@ -241,6 +243,7 @@ class RunTask(Operation):
     profileMode: Optional[str] = None
     withLogs: bool = False
     schedule: Optional[Schedule] = None
+    broadcast: bool = False
 
 
 class AppManage(Operation):
@@ -385,8 +388,8 @@ class CollectionMetadata(BaseModel):
 
 class LogsResult(BaseModel):
     data: str
-    logs: Dict[str, str]
-    userLogs: Dict[str, str] = {}
+    logs: Optional[Dict[str, str]] = {}
+    userLogs: Optional[Dict[str, str]] = {}
 
 
 class AppLog(BaseModel):
@@ -603,7 +606,7 @@ class UserLimits(BaseModel):
     defaultGpuDisk: int
 
 
-class BasePlatformSettings(BaseModel):
+class BasePlatformSettingsMain(BaseModel):
     memoryRequest: Optional[int] = None
     memoryLimit: Optional[int] = None
     cpuRequest: Optional[int] = None
@@ -613,7 +616,11 @@ class BasePlatformSettings(BaseModel):
     kubeconfig: Optional[str] = None
 
 
-class Limits(BasePlatformSettings):
+class BasePlatformSettings(BasePlatformSettingsMain):
+    allowKafka: bool = False
+
+
+class Limits(BasePlatformSettingsMain):
     gpuDisk: Optional[int] = None
 
 
@@ -675,3 +682,37 @@ class MCPToolCall(BaseModel):
 class RunsFilter(BaseModel):
     data: Optional[Dict[str, str]] = None
     withTags: bool = False
+
+
+class AppLocalScheme(BaseModel):
+    keys: List[str]
+    optionalKeys: Set[str]
+
+
+class AppErrorInfo(BaseModel):
+    operationId: str
+    runId: str
+    bindId: str
+    funId: str
+    iteration: int
+    isProcessor: bool = True
+    trace: str
+    errType: str
+    errArgs: List[str]
+    isMalevichErr: bool
+    cfg: Optional[Dict[str, Any]]
+    schemes: Optional[Dict[str, AppLocalScheme]] = None
+    args: List[List[Union[Union[str, List[str]], List[Union[str, List[str]]]]]] = None  # mb empty for send info structure
+    argsNames: List[str]
+
+
+class AppErrorInfos(BaseModel):
+    data: List[AppErrorInfo]
+
+
+class AppErrorInfoFilter(BaseModel):
+    operationId: str
+    runId: Optional[str] = None
+    bindId: Optional[str] = None
+    errType: Optional[str] = None
+    isMalevichErr: bool = False

malevich_coretools/funcs/funcs.py

@@ -1222,14 +1222,22 @@ async def get_image_info_async(data: JsonImage, parse: bool, *args, **kwargs) ->
     return res
 
 
-def get_task_schedules(data: Operation, with_show: bool, *args, **kwargs) -> Schedules:
+def get_task_schedules(*args, **kwargs) -> ResultIds:
+    return model_from_json(send_to_core_get(MANAGER_TASK_SCHEDULES, *args, **kwargs), ResultIds)
+
+
+async def get_task_schedules_async(*args, **kwargs) -> ResultIds:
+    return model_from_json(await send_to_core_get_async(MANAGER_TASK_SCHEDULES, *args, **kwargs), ResultIds)
+
+
+def post_task_schedules(data: Operation, with_show: bool, *args, **kwargs) -> Schedules:
     res = model_from_json(send_to_core_modify(MANAGER_TASK_SCHEDULES, data, with_show=False, *args, **kwargs), Schedules)
     if with_show:
         Config.logger.info(res)
     return res
 
 
-async def get_task_schedules_async(data: Operation, with_show: bool, *args, **kwargs) -> Schedules:
+async def post_task_schedules_async(data: Operation, with_show: bool, *args, **kwargs) -> Schedules:
     res = model_from_json(await send_to_core_modify_async(MANAGER_TASK_SCHEDULES, data, with_show=False, *args, **kwargs), Schedules)
     if with_show:
         Config.logger.info(res)
@@ -1578,6 +1586,30 @@ async def delete_mcp_tools_async(wait: bool, *args, **kwargs) -> Alias.Info:
     return await send_to_core_modify_async(MCP_TOOLS_ALL(wait), *args, **kwargs, is_post=False)
 
 
+def get_app_error_info(data: AppErrorInfoFilter, *args, **kwargs) -> AppErrorInfos:
+    return model_from_json(send_to_core_modify(APP_ERROR_INFO(None), data, *args, **kwargs), AppErrorInfos)
+
+
+async def get_app_error_info_async(data: AppErrorInfoFilter, *args, **kwargs) -> AppErrorInfos:
+    return model_from_json(await send_to_core_modify_async(APP_ERROR_INFO(None), data, *args, **kwargs), AppErrorInfos)
+
+
+def delete_app_error_info(id: str, wait: bool, *args, **kwargs) -> Alias.Info:
+    return send_to_core_modify(APP_ERROR_INFO_ID(id, wait), *args, **kwargs, is_post=False)
+
+
+async def delete_app_error_info_async(id: str, wait: bool, *args, **kwargs) -> Alias.Info:
+    return await send_to_core_modify_async(APP_ERROR_INFO_ID(id, wait), *args, **kwargs, is_post=False)
+
+
+def delete_app_error_infos(wait: bool, *args, **kwargs) -> Alias.Info:
+    return send_to_core_modify(APP_ERROR_INFO(wait), *args, **kwargs, is_post=False)
+
+
+async def delete_app_error_infos_async(wait: bool, *args, **kwargs) -> Alias.Info:
+    return await send_to_core_modify_async(APP_ERROR_INFO(wait), *args, **kwargs, is_post=False)
+
+
 async def kafka_send(data: KafkaMsg, *args, **kwargs) -> Union[Alias.Info, KafkaMsg]:
     result = await send_to_core_post_async(KAFKA_SEND, data, *args, **kwargs)
     try:
@@ -1687,7 +1719,7 @@ async def send_to_core_get_async(path: str, with_auth=True, show_func: Optional[
         auth = (Config.CORE_USERNAME, Config.CORE_PASSWORD) if with_auth else None
     if auth is not None:
         auth = aiohttp.BasicAuth(login=auth[0], password=auth[1], encoding='utf-8')
-    async with async_session or aiohttp.ClientSession(auth=auth, connector=aiohttp.TCPConnector(verify_ssl=False)) as session:
+    async with async_session or aiohttp.ClientSession(auth=auth, connector=aiohttp.TCPConnector(verify_ssl=False), timeout=aiohttp.ClientTimeout(total=None)) as session:
         async with session.get(f"{host}{path}", headers=HEADERS) as response:
             await __async_check_response(response, show_func, f"{host}{path}")
             if response.status == HTTPStatus.NO_CONTENT:
@@ -1738,7 +1770,7 @@ async def send_to_core_modify_async(path: str, operation: Optional[Any] = None,
     if operation is not None:
         operation = json.dumps(operation.model_dump())
 
-    async with async_session or aiohttp.ClientSession(auth=auth if with_auth else None, connector=aiohttp.TCPConnector(verify_ssl=False)) as session:
+    async with async_session or aiohttp.ClientSession(auth=auth if with_auth else None, connector=aiohttp.TCPConnector(verify_ssl=False), timeout=aiohttp.ClientTimeout(total=None)) as session:
         if is_post:
             response_cm = session.post(f"{host}{path}", data=operation, headers=HEADERS)
         else:
@@ -1792,7 +1824,7 @@ async def send_to_core_modify_raw_async(path: str, data: bytes, with_auth: bool=
         auth = (Config.CORE_USERNAME, Config.CORE_PASSWORD)
     auth = aiohttp.BasicAuth(login=auth[0], password=auth[1], encoding='utf-8')
 
-    async with async_session or aiohttp.ClientSession(auth=auth if with_auth else None, connector=aiohttp.TCPConnector(verify_ssl=False)) as session:
+    async with async_session or aiohttp.ClientSession(auth=auth if with_auth else None, connector=aiohttp.TCPConnector(verify_ssl=False), timeout=aiohttp.ClientTimeout(total=None)) as session:
         if is_post:
             response_cm = session.post(f"{host}{path}", data=data, headers=HEADERS_RAW)
         else:

malevich_coretools/funcs/helpers.py

@@ -439,6 +439,7 @@ def base_settings(
     storage_request: Optional[int] = None,
     storage_limit: Optional[int] = None,
     kubeconfig: Optional[str] = None,
+    allow_kafka: bool = False,
 ) -> str:
     return BasePlatformSettings(
         memoryRequest=memory_request,
@@ -448,4 +449,5 @@ def base_settings(
         storageRequest=storage_request,
         storageLimit=storage_limit,
         kubeconfig=kubeconfig,
+        allowKafka=allow_kafka,
     ).model_dump_json()

malevich_coretools/secondary/const.py

@@ -39,13 +39,13 @@ def with_key_values(url: str, key_values: Dict[str, Optional[str]]) -> str:
 
 ## DocsController
 DOCS_MAIN = f"{API_VERSION}/docs"
-DOCS = lambda wait: with_wait(f"{DOCS_MAIN}/", wait)
+DOCS = lambda wait: with_wait(DOCS_MAIN, wait)
 DOCS_ID = lambda id, wait: with_wait(f"{DOCS_MAIN}/{id}", wait)
 DOCS_NAME = lambda name, wait: with_wait(f"{DOCS_MAIN}/name/{name}", wait)
 
 ## CollectionsController
 COLLECTIONS_MAIN = f"{API_VERSION}/collections"
-COLLECTIONS = lambda wait: with_wait(f"{COLLECTIONS_MAIN}/", wait)
+COLLECTIONS = lambda wait: with_wait(COLLECTIONS_MAIN, wait)
 COLLECTIONS_IDS_NAME = lambda name, operation_id, run_id: with_key_values(f"{COLLECTIONS_MAIN}/ids/name/{urllib.parse.quote(str(name), safe='')}", {"operationId": operation_id, "runId": run_id})
 COLLECTIONS_NAME = lambda name, operation_id, run_id, offset, limit: with_key_values(f"{COLLECTIONS_MAIN}/name/{urllib.parse.quote(str(name), safe='')}", {"operationId": operation_id, "runId": run_id, "offset": offset, "limit": limit})
 COLLECTIONS_IDS_GROUP_NAME = lambda name, operation_id, run_id: with_key_values(f"{COLLECTIONS_MAIN}/ids/groupName/{urllib.parse.quote(str(name), safe='')}", {"operationId": operation_id, "runId": run_id})
@@ -67,7 +67,7 @@ COLLECTIONS_METADATA = lambda id, wait: with_wait(f"{COLLECTIONS_MAIN}/{urllib.p
 COLLECTION_OBJECTS_MAIN = f"{API_VERSION}/collectionObjects"
 COLLECTION_OBJECTS_ALL_GET = lambda path, recursive: with_key_values(f"{COLLECTION_OBJECTS_MAIN}/all", {"path": path, "recursive": recursive})
 COLLECTION_OBJECTS_ALL = lambda wait: with_wait(f"{COLLECTION_OBJECTS_MAIN}/all", wait)
-COLLECTION_OBJECTS_PATH = lambda path, wait, zip: with_key_values(f"{COLLECTION_OBJECTS_MAIN}/", {"path": path, "wait": None if wait is None else bool_to_str(wait), "zip": None if zip is None else bool_to_str(zip)})
+COLLECTION_OBJECTS_PATH = lambda path, wait, zip: with_key_values(COLLECTION_OBJECTS_MAIN, {"path": path, "wait": None if wait is None else bool_to_str(wait), "zip": None if zip is None else bool_to_str(zip)})
 COLLECTION_OBJECTS_PRESIGN_PUT = lambda path, callback_url, expires_in, wait: with_key_values(f"{COLLECTION_OBJECTS_MAIN}/presign/put", {"path": path, "callback_url": callback_url, "expiresIn": expires_in, "wait": bool_to_str(wait)})
 COLLECTION_OBJECTS_PRESIGN_GET = lambda path, callback_url, expires_in, wait: with_key_values(f"{COLLECTION_OBJECTS_MAIN}/presign/get", {"path": path, "callback_url": callback_url, "expiresIn": expires_in, "wait": bool_to_str(wait)})
 COLLECTION_OBJECTS_PRESIGN = lambda signature, zip: with_key_values(f"{COLLECTION_OBJECTS_MAIN}/presign", {"signature": signature, "zip": None if zip is None else bool_to_str(zip)})
@@ -84,7 +84,7 @@ ENDPOINTS_RESUME = lambda hash, wait: with_wait(f"{ENDPOINTS_MAIN}/resume/{urlli
 
 ## SchemeController
 SCHEMES_MAIN = f"{API_VERSION}/schemes"
-SCHEMES = lambda wait: with_wait(f"{SCHEMES_MAIN}/", wait)
+SCHEMES = lambda wait: with_wait(SCHEMES_MAIN, wait)
 SCHEMES_ID = lambda id, wait: with_wait(f"{SCHEMES_MAIN}/{urllib.parse.quote(str(id), safe='')}", wait)
 SCHEMES_ID_RAW = lambda id: f"{SCHEMES_MAIN}/{urllib.parse.quote(str(id), safe='')}/raw"
 SCHEMES_MAPPING = lambda wait: with_wait(f"{SCHEMES_MAIN}/mapping", wait)
@@ -100,7 +100,7 @@ PING = "ping"
 
 ## UserShareController
 SHARE_MAIN = f"{API_VERSION}/share"
-SHARE = lambda wait: with_wait(f"{SHARE_MAIN}/", wait)
+SHARE = lambda wait: with_wait(SHARE_MAIN, wait)
 SHARE_COLLECTION_ID = lambda id, wait: with_wait(f"{SHARE_MAIN}/collection/{urllib.parse.quote(str(id), safe='')}", wait)
 SHARE_SCHEME_ID = lambda id, wait: with_wait(f"{SHARE_MAIN}/scheme/{urllib.parse.quote(str(id), safe='')}", wait)
 SHARE_USER_APP_ID = lambda id, wait: with_wait(f"{SHARE_MAIN}/userApp/{urllib.parse.quote(str(id), safe='')}", wait)
@@ -109,13 +109,13 @@ SHARE_ALL = lambda wait: with_wait(f"{SHARE_MAIN}/all", wait)
 
 ## RegistrationController
 REGISTER_MAIN = f"{API_VERSION}/register"
-REGISTER = f"{REGISTER_MAIN}/"
+REGISTER = REGISTER_MAIN
 REGISTER_LOGIN = lambda login, wait: with_wait(f"{REGISTER_MAIN}/login/{urllib.parse.quote(str(login), safe='')}", wait)
 REGISTER_ALL = f"{REGISTER_MAIN}/all"
 
 ## UserAppsController
 USER_APPS_MAIN = f"{API_VERSION}/userApps"
-USER_APPS = lambda wait: with_wait(f"{USER_APPS_MAIN}/", wait)
+USER_APPS = lambda wait: with_wait(USER_APPS_MAIN, wait)
 USER_APPS_REAL_IDS = f"{USER_APPS_MAIN}/realIds"
 USER_APPS_MAP_IDS = f"{USER_APPS_MAIN}/mapIds"
 USER_APPS_MAP_ID = lambda id: f"{USER_APPS_MAIN}/mapIds/{urllib.parse.quote(str(id), safe='')}"
@@ -124,7 +124,7 @@ USER_APPS_REAL_ID = lambda id: f"{USER_APPS_MAIN}/realIds/{urllib.parse.quote(st
 
 ## UserTasksController
 USER_TASKS_MAIN = f"{API_VERSION}/userTasks"
-USER_TASKS = lambda wait: with_wait(f"{USER_TASKS_MAIN}/", wait)
+USER_TASKS = lambda wait: with_wait(USER_TASKS_MAIN, wait)
 USER_TASKS_REAL_IDS = f"{USER_TASKS_MAIN}/realIds"
 USER_TASKS_MAP_IDS = f"{USER_TASKS_MAIN}/mapIds"
 USER_TASKS_MAP_ID = lambda id: f"{USER_TASKS_MAIN}/mapIds/{urllib.parse.quote(str(id), safe='')}"
@@ -133,7 +133,7 @@ USER_TASKS_REAL_ID = lambda id: f"{USER_TASKS_MAIN}/realIds/{urllib.parse.quote(
 
 ## UserPipelinesController
 USER_PIPELINES_MAIN = f"{API_VERSION}/userPipelines"
-USER_PIPELINES = lambda wait: with_wait(f"{USER_PIPELINES_MAIN}/", wait)
+USER_PIPELINES = lambda wait: with_wait(USER_PIPELINES_MAIN, wait)
 USER_PIPELINES_REAL_IDS = f"{USER_PIPELINES_MAIN}/realIds"
 USER_PIPELINES_MAP_IDS = f"{USER_PIPELINES_MAIN}/mapIds"
 USER_PIPELINES_MAP_ID = lambda id: f"{USER_PIPELINES_MAIN}/mapIds/{urllib.parse.quote(str(id), safe='')}"
@@ -144,7 +144,7 @@ USER_PIPELINES_TAG_REAL_IDS = lambda tag: f"{USER_PIPELINES_MAIN}/realIds/tagSea
 
 ## UserCfgsController
 USER_CFGS_MAIN = f"{API_VERSION}/userCfgs"
-USER_CFGS = lambda wait: with_wait(f"{USER_CFGS_MAIN}/", wait)
+USER_CFGS = lambda wait: with_wait(USER_CFGS_MAIN, wait)
 USER_CFGS_REAL_IDS = f"{USER_CFGS_MAIN}/realIds"
 USER_CFGS_MAP_IDS = f"{USER_CFGS_MAIN}/mapIds"
 USER_CFGS_MAP_ID = lambda id: f"{USER_CFGS_MAIN}/mapIds/{urllib.parse.quote(str(id), safe='')}"
@@ -153,7 +153,7 @@ USER_CFGS_REAL_ID = lambda id: f"{USER_CFGS_MAIN}/realIds/{urllib.parse.quote(st
 
 ## OperationResultsController
 OPERATION_RESULTS_MAIN = f"{API_VERSION}/operationResults"
-OPERATION_RESULTS = lambda wait: with_wait(f"{OPERATION_RESULTS_MAIN}/", wait)
+OPERATION_RESULTS = lambda wait: with_wait(OPERATION_RESULTS_MAIN, wait)
 OPERATION_RESULTS_ID = lambda id, wait: with_wait(f"{OPERATION_RESULTS_MAIN}/{urllib.parse.quote(str(id), safe='')}", wait)
 
 ## TempRunController
@@ -195,16 +195,16 @@ MANAGER_APP_PAUSE = lambda wait: with_wait(f"{MANAGER_MAIN}/app/pause", wait)
 
 ## LimitsController
 LIMITS_MAIN = f"{API_VERSION}/limits"
-LIMITS = lambda wait: with_wait(f"{LIMITS_MAIN}/", wait)
+LIMITS = lambda wait: with_wait(LIMITS_MAIN, wait)
 LIMITS_USER = lambda wait: with_wait(f"{LIMITS_MAIN}/user", wait)
 
 ## HandlerUrlController
 HANDLER_URL_MAIN = f"{API_VERSION}/handlerUrls"
-HANDLER_URL = lambda url, wait: with_key_values(f"{HANDLER_URL_MAIN}/", {"url": url, "wait": bool_to_str(wait)})
+HANDLER_URL = lambda url, wait: with_key_values(HANDLER_URL_MAIN, {"url": url, "wait": bool_to_str(wait)})
 
 ## AnalyticsController
 ANALYTICS_MAIN = f"{API_VERSION}/analytics"
-ANALYTICS = lambda wait: with_wait(f"{ANALYTICS_MAIN}/", wait)
+ANALYTICS = lambda wait: with_wait(ANALYTICS_MAIN, wait)
 ANALYTICS_MANY = lambda wait: with_wait(f"{ANALYTICS_MAIN}/many", wait)
 ANALYTICS_ID = lambda id, wait: with_wait(f"{ANALYTICS_MAIN}/{id}", wait)
 ANALYTICS_NAME = lambda name, wait: with_wait(f"{ANALYTICS_MAIN}/name/{name}", wait)
@@ -216,20 +216,25 @@ RUNS_INFO_LAST_FAILED = lambda count: with_key_values(f"{RUNS_INFO_MAIN}/lastFai
 
 ## WSAppsController
 WS_APPS_MAIN = f"{API_VERSION}/ws/apps"
-WS_APPS = lambda only_active, full: with_key_values(f"{WS_APPS_MAIN}/", {"onlyActive": only_active, "full": full})
-WS_APPS_ = lambda only_not_active, wait: with_key_values(f"{WS_APPS_MAIN}/", {"onlyNotActive": only_not_active, "wait": wait})
+WS_APPS = lambda only_active, full: with_key_values(WS_APPS_MAIN, {"onlyActive": only_active, "full": full})
+WS_APPS_ = lambda only_not_active, wait: with_key_values(WS_APPS_MAIN, {"onlyNotActive": only_not_active, "wait": wait})
 WS_APPS_ID = lambda id, wait: with_wait(f"{WS_APPS_MAIN}/{id}", wait)
 
 ## McpToolController
 MCP_TOOLS_MAIN = f"{API_VERSION}/tools"
-MCP_TOOLS = lambda id, name, wait: with_key_values(f"{MCP_TOOLS_MAIN}/", {"id": id, "name": name, "wait": wait})
+MCP_TOOLS = lambda id, name, wait: with_key_values(MCP_TOOLS_MAIN, {"id": id, "name": name, "wait": wait})
 MCP_TOOLS_ALL = lambda wait: with_wait(f"{MCP_TOOLS_MAIN}/all", wait)
 MCP_TOOLS_LIST = f"{MCP_TOOLS_MAIN}/list"
 MCP_TOOLS_CALL = f"{MCP_TOOLS_MAIN}/call"
 
+## AppErrorInfoController
+APP_ERROR_INFO_MAIN = f"{API_VERSION}/errors"
+APP_ERROR_INFO = lambda wait: with_wait(APP_ERROR_INFO_MAIN, wait)
+APP_ERROR_INFO_ID = lambda operationId, wait: with_wait(f"{APP_ERROR_INFO_MAIN}/{operationId}", wait)
+
 ### Kafka
 KAFKA_SEND = f"{MANAGER_MAIN}/kafkaMsg"
 
 ## BatchController
 BATCH_MAIN = f"{API_VERSION}/batch"
-BATCH = f"{BATCH_MAIN}/"
+BATCH = BATCH_MAIN

malevich_coretools/secondary/helpers.py

@@ -7,6 +7,7 @@ from pydantic import BaseModel
 
 from malevich_coretools.abstract.abstract import (
     Alias,
+    AppLog,
     AppLogs,
     FlattenAppLogsWithResults,
     LogsResult,
@@ -56,17 +57,36 @@ def bool_to_str(b: bool) -> str:
     return "true" if b else "false"
 
 
-def __show_logs_result(res: LogsResult):  # noqa: ANN202
+def __show_logs_result(res: LogsResult, i: Optional[int]):  # noqa: ANN202
     if len(res.data) > 0:
-        print("------- main:")
+        print(f"------- main ({i}):" if i is not None else "------- main:")
         print(res.data)
+    if res.logs is None:
+        res.logs = {}
+    if res.userLogs is None:
+        res.userLogs = {}
     for run_id, logs in res.logs.items():
-        print(f"------- {run_id}:")
         userLogs = res.userLogs.get(run_id, "")
-        if len(userLogs) > 0:
+        print(f"------- {run_id} ({i}):" if i is not None else f"------- {run_id}:")
+        if len(logs) == 0:
             print(userLogs)
-            print("-------")
-        print(logs)
+        else:
+            if len(userLogs) > 0:
+                print(userLogs)
+                print("-------")
+            print(logs)
+    for run_id, logs in res.userLogs.items():
+        if run_id not in res.logs:
+            print(f"------- {run_id} ({i}):" if i is not None else f"------- {run_id}:")
+            print(logs)
+
+
+def __logs_parts(app_log: AppLog) -> Dict[int, LogsResult]:
+    parts = {}
+    for i, log_res in enumerate(app_log.data):
+        if len(log_res.data) > 0 or (log_res.logs is not None and len(log_res.logs) > 0) or (log_res.userLogs is not None and len(log_res.userLogs) > 0):
+            parts[i] = log_res
+    return parts
 
 
 def show_logs(app_logs: AppLogs, err: bool = False) -> None:  # noqa: ANN202
@@ -75,59 +95,81 @@ def show_logs(app_logs: AppLogs, err: bool = False) -> None:  # noqa: ANN202
     if app_logs.error is not None:
         show(f"error: {app_logs.error}")
         print(__delimiter)
-    print("------- dag logs -------")
-    print(app_logs.dagLogs)
+    if len(app_logs.dagLogs) > 0:
+        print("------- dm logs -------")
+        print(app_logs.dagLogs)
     for app_name, app_log in app_logs.data.items():
+        parts = __logs_parts(app_log)
+        if len(parts) == 0:
+            continue
         print(f"------- {app_name} -------")
         if len(app_log.data) == 1:
             __show_logs_result(app_log.data[0])
         else:
-            for i, log_res in enumerate(app_log.data):
-                print(f"------- {i}:")
-                __show_logs_result(log_res)
-                print(__mini__delimiter)
+            c = len(parts)
+            for i, log_res in parts.items():
+                __show_logs_result(log_res, i)
+                c -= 1
+                if c != 0:
+                    print(__mini__delimiter)
         print(__delimiter)
 
 
 def show_logs_colored(app_logs: AppLogs, colors_dict: Optional[Dict[str, str]] = None) -> None:
     """colors_dict - should be unique for all app_logs by operation_id"""
-    def format(log, color: Optional[str]) -> None:
+    def format(log, color: Optional[str] = None) -> None:
         if color is None:
             Config.logger.warning(log)
         else:
             Config.logger.warning(color + log + __color_reset)
 
     def get_color(name: str) -> str:
-        if colors_dict is None:
-            return None
         color = colors_dict.get(name, None)
         if color is None:
             color = __colors[len(colors_dict) % len(__colors)]
             colors_dict[name] = color
         return color
 
+    if colors_dict is None:
+        colors_dict = {}
+
+    format(f"operation_id = {app_logs.operationId}")
     if app_logs.error is not None:
-        format(f"error: {app_logs.error}", get_color("error"))
+        color = get_color("error")
+        format(f"error: {app_logs.error}", color)
+        format(__delimiter, color)
     if len(app_logs.dagLogs) > 0:
-        color = get_color("dagLogs")
+        color = get_color("dm")
+        format("------- dm logs -------", color)
         for line in app_logs.dagLogs.splitlines():
-            format(f"dag: {line}", color)
+            format(f"dm: {line}", color)
     for app_name, app_log in app_logs.data.items():
-        color = get_color(f"${app_name}")
-        for i, logs_result in enumerate(app_log.data):
-            app_name_prefix = f"{app_name}${i}" if i != 0 else app_name
+        parts = __logs_parts(app_log)
+        if len(parts) == 0:
+            continue
+
+        color = get_color(app_name)
+        for i, logs_result in parts.items():
+            app_name_prefix = f"{app_name}${i}" if i != 0 or len(parts) > 1 else app_name
             if len(logs_result.data) > 0:
                 for line in logs_result.data.splitlines():
                     format(f"{app_name_prefix}$main: {line}", color)
-            if len(logs_result.logs) > 0:
-                for run_id, logs in logs_result.logs.items():
-                    user_logs = logs_result.userLogs.get(run_id, "")
-                    if len(user_logs) > 0:
-                        for line in user_logs.splitlines():
-                            format(f"{app_name_prefix}${run_id}: {line}", color)
-                    if len(logs) > 0:
-                        for line in logs.splitlines():
-                            format(f"{app_name_prefix}${run_id}: {line}", color)
+            if logs_result.logs is None:
+                logs_result.logs = {}
+            if logs_result.userLogs is None:
+                logs_result.userLogs = {}
+            for run_id, logs in logs_result.logs.items():
+                user_logs = logs_result.userLogs.get(run_id, "")
+                if len(user_logs) > 0:
+                    for line in user_logs.splitlines():
+                        format(f"{app_name_prefix}${run_id}$user: {line}", color)
+                if len(logs) > 0:
+                    for line in logs.splitlines():
+                        format(f"{app_name_prefix}${run_id}: {line}", color)
+            for run_id, user_logs in logs_result.userLogs.items():
+                if run_id not in logs_result.logs:
+                    for line in user_logs.splitlines():
+                        format(f"{app_name_prefix}${run_id}$user: {line}", color)
 
 
 def show_logs_func(data: str, err: bool = False):  # noqa: ANN201

malevich_coretools/utils.py

@@ -6735,7 +6735,7 @@ def get_task_schedules(
     conn_url: Optional[str] = None,
     batcher: Optional[Batcher] = None,
     is_async: Literal[False] = False,
-) -> Schedules:
+) -> Union[Schedules, ResultIds]:
     pass
 
 
@@ -6748,32 +6748,39 @@ def get_task_schedules(
     conn_url: Optional[str] = None,
     batcher: Optional[Batcher] = None,
     is_async: Literal[True],
-) -> Coroutine[Any, Any, Schedules]:
+) -> Coroutine[Any, Any, Union[Schedules, ResultIds]]:
     pass
 
 
 def get_task_schedules(
-    operation_id: str,
+    operation_id: Optional[str] = None,
     with_show: bool = True,
     *,
     auth: Optional[AUTH] = None,
     conn_url: Optional[str] = None,
     batcher: Optional[Batcher] = None,
     is_async: bool = False,
-) -> Union[Schedules, Coroutine[Any, Any, Schedules]]:
+) -> Union[ResultIds, Schedules, Coroutine[Any, Any, Schedules], Coroutine[Any, Any, ResultIds]]:
     """return schedule ids by `operation_id` """
     if batcher is None:
         batcher = Config.BATCHER
-    data = Operation(operationId=operation_id)
-    if batcher is not None:
-        return batcher.add("sendTaskSchedules", data=data, result_model=Schedules)
-    if is_async:
-        return f.get_task_schedules_async(
+    if operation_id is None:
+        if batcher is not None:
+            return batcher.add("getTaskSchedules", result_model=ResultIds)
+        if is_async:
+            return f.get_task_schedules_async(auth=auth, conn_url=conn_url)
+        return f.get_task_schedules(auth=auth, conn_url=conn_url)
+    else:
+        data = Operation(operationId=operation_id)
+        if batcher is not None:
+            return batcher.add("sendTaskSchedules", data=data, result_model=Schedules)
+        if is_async:
+            return f.post_task_schedules_async(
+                data, with_show=with_show, auth=auth, conn_url=conn_url
+            )
+        return f.post_task_schedules(
             data, with_show=with_show, auth=auth, conn_url=conn_url
         )
-    return f.get_task_schedules(
-        data, with_show=with_show, auth=auth, conn_url=conn_url
-    )
 
 
 @overload
@@ -6834,7 +6841,6 @@ def task_full(
     pass
 
 
-# FIXME check component
 def task_full(
     task_id: str,
     cfg_id: str,
@@ -6844,6 +6850,7 @@ def task_full(
     single_request: bool = False,
     profile_mode: Optional[str] = None,
     save_fails: bool = True,
+    clear_dag_logs: bool = True,
     with_show: bool = True,
     long: bool = False,
     long_timeout: Optional[int] = WAIT_RESULT_TIMEOUT,
@@ -6906,6 +6913,7 @@ def task_full(
         profileMode=profile_mode,
         withLogs=True,
         saveFails=save_fails,
+        clearDagLogs=clear_dag_logs,
         scaleInfo=scaleInfo,
         component=component,
         policy=policy,
@@ -7019,6 +7027,7 @@ def task_prepare(
     with_logs: bool = False,
     profile_mode: Optional[str] = None,
     save_fails: bool = True,
+    clear_dag_logs: bool = True,
     with_show: bool = None,
     long: bool = False,
     long_timeout: int = WAIT_RESULT_TIMEOUT,
@@ -7091,6 +7100,7 @@ def task_prepare(
         waitRuns=wait_runs,
         withLogs=with_logs,
         saveFails=save_fails,
+        clearDagLogs=clear_dag_logs,
         profileMode=profile_mode,
         scaleInfo=scaleInfo,
         component=component,
@@ -7135,6 +7145,7 @@ def task_run(
     long_timeout: int = WAIT_RESULT_TIMEOUT,
     with_logs: bool = False,
     schedule: Optional[Schedule] = None,
+    broadcast: bool = False,
     wait: bool = True,
     *,
     auth: Optional[AUTH] = None,
@@ -7159,6 +7170,7 @@ def task_run(
     long_timeout: int = WAIT_RESULT_TIMEOUT,
     with_logs: bool = False,
     schedule: Optional[Schedule] = None,
+    broadcast: bool = False,
     wait: bool = True,
     *,
     auth: Optional[AUTH] = None,
@@ -7182,6 +7194,7 @@ def task_run(
     long_timeout: int = WAIT_RESULT_TIMEOUT,
     with_logs: bool = False,
     schedule: Optional[Schedule] = None,
+    broadcast: bool = False,
     wait: bool = True,
     *,
     auth: Optional[AUTH] = None,
@@ -7204,6 +7217,7 @@ def task_run(
         long_timeout (Optional[int]): default timeout for long run (hour by default). If 'long=False' ignored. If None, then there is no limit. Doesn't stop the task, just stops trying to get the run result
         with_logs (bool): return run logs if True after end
         schedule: (Optional[Schedule]): schedule task runs settings - return scheduleId instead of operationId
+        broadcast: run scale pipeline (for new scale)
         wait (bool): is it worth waiting for the result or immediately return `operation_id`
         auth (Optional[AUTH]): redefined auth if not None"""
     if schedule is not None:
@@ -7224,6 +7238,7 @@ def task_run(
         profileMode=profile_mode,
         withLogs=with_logs,
         schedule=schedule,
+        broadcast=broadcast,
     )
     if batcher is not None:
         return batcher.add("sendTaskRun", data=data, result_model=AppLogs if with_logs or schedule is not None else None)
@@ -7322,6 +7337,7 @@ def pipeline_full(
     restrictions: Optional[Restrictions] = None,
     scaleInfo: List[ScaleInfo] = None,
     save_fails: bool = True,
+    clear_dag_logs: bool = True,
     tags: Optional[Dict[str, str]] = None,
     with_show: bool = True,
     long: bool = False,
@@ -7367,6 +7383,7 @@ def pipeline_full(
         kafkaModeUrl=None,
         run=True,
         saveFails=save_fails,
+        clearDagLogs=clear_dag_logs,
         scaleCount=1,
         tags=tags,
     )
@@ -7489,6 +7506,7 @@ def pipeline_prepare(
     kafka_mode_url_response: Optional[str] = None,
     synthetic: bool = False,
     save_fails: bool = True,
+    clear_dag_logs: bool = True,
     scale_count: int = 1,
     tags: Optional[Dict[str, str]] = None,
     with_show: bool = True,
@@ -7533,6 +7551,7 @@ def pipeline_prepare(
         run=False,
         synthetic=synthetic,
         saveFails=save_fails,
+        clearDagLogs=clear_dag_logs,
         scaleCount=scale_count,
         tags=tags,
     )
@@ -9263,6 +9282,156 @@ def delete_mcp_tools(
     return f.delete_mcp_tools(wait=wait, auth=auth, conn_url=conn_url)
 
 
+# app error info
+
+
+@overload
+def get_app_error_info(
+    operation_id: str,
+    run_id: Optional[str] = None,
+    bind_id: Optional[str] = None,
+    err_type: Optional[str] = None,
+    is_malevich_err: bool = False,
+    *,
+    with_auth: bool = True,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+    is_async: Literal[False] = False,
+) -> AppErrorInfos:
+    pass
+
+
+@overload
+def get_app_error_info(
+    operation_id: str,
+    run_id: Optional[str] = None,
+    bind_id: Optional[str] = None,
+    err_type: Optional[str] = None,
+    is_malevich_err: bool = False,
+    *,
+    with_auth: bool = True,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, AppErrorInfos]:
+    pass
+
+
+def get_app_error_info(
+    operation_id: str,
+    run_id: Optional[str] = None,
+    bind_id: Optional[str] = None,
+    err_type: Optional[str] = None,
+    is_malevich_err: bool = False,
+    *,
+    with_auth: bool = True,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+    is_async: bool = False,
+) -> Union[AppErrorInfos, Coroutine[Any, Any, AppErrorInfos]]:
+    """return mcp tool struct by `id` or `name` """
+    assert id is not None or name is not None, "id or name should be set"
+    data = AppErrorInfoFilter(operationId=operation_id, runId=run_id, bindId=bind_id, errType=err_type, isMalevichErr=is_malevich_err)
+    if batcher is None:
+        batcher = Config.BATCHER
+    if batcher is not None:
+        return batcher.add("getAppErrorInfo", data=data, result_model=AppErrorInfos)
+    if is_async:
+        return f.get_app_error_info_async(data, with_auth=with_auth, auth=auth, conn_url=conn_url)
+    return f.get_app_error_info(data, with_auth=with_auth, auth=auth, conn_url=conn_url)
+
+
+@overload
+def delete_app_error_info(
+    operation_id: str,
+    wait: bool = True,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+    is_async: Literal[False] = False,
+) -> Alias.Info:
+    pass
+
+
+@overload
+def delete_app_error_info(
+    operation_id: str,
+    wait: bool = True,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, Alias.Info]:
+    pass
+
+
+def delete_app_error_info(
+    operation_id: str,
+    wait: bool = True,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+    is_async: bool = False,
+) -> Union[Alias.Info, Coroutine[Any, Any, Alias.Info]]:
+    """delete app error info by `operationId` """
+    assert id is not None or name is not None, "id or name should be set"
+    if batcher is None:
+        batcher = Config.BATCHER
+    if batcher is not None:
+        return batcher.add("deleteAppErrorInfoByOperationId", vars={"operationId": operation_id})
+    if is_async:
+        return f.delete_app_error_info_async(operation_id, wait=wait, auth=auth, conn_url=conn_url)
+    return f.delete_app_error_info(operation_id, wait=wait, auth=auth, conn_url=conn_url)
+
+
+@overload
+def delete_app_error_infos(
+    wait: bool = True,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+    is_async: Literal[False] = False,
+) -> Alias.Info:
+    pass
+
+
+@overload
+def delete_app_error_infos(
+    wait: bool = True,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, Alias.Info]:
+    pass
+
+
+def delete_app_error_infos(
+    wait: bool = True,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+    is_async: bool = False,
+) -> Union[Alias.Info, Coroutine[Any, Any, Alias.Info]]:
+    """delete all app error infos"""
+    if batcher is None:
+        batcher = Config.BATCHER
+    if batcher is not None:
+        return batcher.add("deleteAppErrorInfos")
+    if is_async:
+        return f.delete_app_error_infos_async(wait=wait, auth=auth, conn_url=conn_url)
+    return f.delete_app_error_infos(wait=wait, auth=auth, conn_url=conn_url)
+
+
 # kafka
 
 

{malevich_coretools-0.3.54.dist-info → malevich_coretools-0.3.61.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: malevich-coretools
-Version: 0.3.54
+Version: 0.3.61
 Author: Andrew Pogrebnoj
 Author-email: andrew@onjulius.co
 License-File: LICENSE

{malevich_coretools-0.3.54.dist-info → malevich_coretools-0.3.61.dist-info}/RECORD

@@ -1,7 +1,7 @@
 malevich_coretools/__init__.py,sha256=DJtPESxkCZD2SbTZTrR_x0TKDQ4MJpmBqGw5YpKYidM,134
-malevich_coretools/utils.py,sha256=NlYAn3qCiCfhCpvkOyJBsrY-2duTw-0PJlxnX_JZQwg,270535
+malevich_coretools/utils.py,sha256=9k2I7YMrLtSlfHWkhVuYWdtljZgWKnfJvFDdqexhx-I,275747
 malevich_coretools/abstract/__init__.py,sha256=6vQ08c8HPYyT_pPkKlc-EwQKE8xG3HTEo2p_GiI5rik,142
-malevich_coretools/abstract/abstract.py,sha256=CZRNn-pqg_eORgJh3XXhpf6hSb2ubR718DdHRz4p05c,16463
+malevich_coretools/abstract/abstract.py,sha256=S5yYRhzpqsiIEN083LqUJlvu0MZ9Z__DdOLhS5ZsPdc,17475
 malevich_coretools/abstract/operations.py,sha256=cWlo2xzW-rzkTInzpDjBYeL68KfLYqSpZJRzCQ4OzjA,3070
 malevich_coretools/abstract/pipeline.py,sha256=HwhYp5G9yaZYaeDypChfpNd2W-kmJQfM9I54uek0B9k,7914
 malevich_coretools/abstract/statuses.py,sha256=9ISSw_evsylBshLXoU44TCoFOrZm4bXIxyAFFDqdUWc,333
@@ -11,18 +11,18 @@ malevich_coretools/batch/__init__.py,sha256=taxyZl8YOZd2EBd3leN6slzMkejUtjQ64Na3
 malevich_coretools/batch/utils.py,sha256=FRmCYU-zr-RjgT1Mo3CUNcB2mW1t_gKCJazcMx6aIW4,7719
 malevich_coretools/funcs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 malevich_coretools/funcs/checks.py,sha256=Q5pRtRevQrGv_-SMbn2GgYnulhclDLBXdRtbw2QOYKU,223
-malevich_coretools/funcs/funcs.py,sha256=ZbVsmd-4ypBK5yaKS5WQIgw-iu2bL3GyD2C7YYw-D50,83669
-malevich_coretools/funcs/helpers.py,sha256=0zkHyuuBGWHmoLFJiLQHXkcmtoFrK3gDvNWiU9oKvHw,13767
+malevich_coretools/funcs/funcs.py,sha256=FgBrMRbGrZylVGzew48fUfy2VxRMK9gH-RXmJdyG7F4,85252
+malevich_coretools/funcs/helpers.py,sha256=3-2s5-8hiBeslir8Qa1IgKkHwgYnjPG5w7Kog1e7dHI,13830
 malevich_coretools/secondary/__init__.py,sha256=048HqvG36_1WdDVZK_RuECmaf14Iq2fviUysG1inlaE,78
 malevich_coretools/secondary/config.py,sha256=hRlSJuPQnhKyt1wmOAJX_XmcliaO0fPGbW94AE_Mazs,463
-malevich_coretools/secondary/const.py,sha256=raoPX6aFuk_Rj9t4aYyLoqzEv6N8aHzIfdaF9QIeijE,15187
-malevich_coretools/secondary/helpers.py,sha256=lbLgHJeDQvn38DnKuXMaaNUD6PCxjzCusZO8DcLXqDk,6384
+malevich_coretools/secondary/const.py,sha256=r_P9x0F8YFDFmPHF4oCUQJ4Va3v39CYG_Isg16CDpmo,15321
+malevich_coretools/secondary/helpers.py,sha256=9zRU_EjESks_DbDxs6PjaubtKeJWTqOI_fCoTsEsP8w,7979
 malevich_coretools/secondary/kafka_utils.py,sha256=SIUnBFyfwsquN6MAUrEkKCw-1l7979Znl7OTQSX2UKo,989
 malevich_coretools/tools/__init__.py,sha256=jDxlCa5Dr6Y43qlI7JwsRAlBkKmFeTHTEnjNUvu-0iw,46
 malevich_coretools/tools/abstract.py,sha256=B1RW1FeNHrQ6r1k-cQZ4k4noCRXkIGt-JUwVoXEDkAg,4466
 malevich_coretools/tools/vast.py,sha256=63tvy70qQV9vnK0eWytlgjBGSnfA7l3kSIDgACBbMMs,12893
-malevich_coretools-0.3.54.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-malevich_coretools-0.3.54.dist-info/METADATA,sha256=CFIWe_2rqGcnDQJOmB3GDRjoPToxQbmhvYlmXJnkA7c,347
-malevich_coretools-0.3.54.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-malevich_coretools-0.3.54.dist-info/top_level.txt,sha256=wDX3s1Tso0otBPNrFRfXqyNpm48W4Bp5v6JfbITO2Z8,19
-malevich_coretools-0.3.54.dist-info/RECORD,,
+malevich_coretools-0.3.61.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+malevich_coretools-0.3.61.dist-info/METADATA,sha256=Itxpu3ReVN0qcFy1YQuozsf4zOZAT6NIY4wmaYe6NNQ,347
+malevich_coretools-0.3.61.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+malevich_coretools-0.3.61.dist-info/top_level.txt,sha256=wDX3s1Tso0otBPNrFRfXqyNpm48W4Bp5v6JfbITO2Z8,19
+malevich_coretools-0.3.61.dist-info/RECORD,,