malevich-coretools 0.3.51__tar.gz → 0.3.74__tar.gz

{malevich_coretools-0.3.51/malevich_coretools.egg-info → malevich_coretools-0.3.74}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: malevich-coretools
-Version: 0.3.51
+Version: 0.3.74
 Author: Andrew Pogrebnoj
 Author-email: andrew@onjulius.co
 License-File: LICENSE

malevich_coretools-0.3.74/VERSION

@@ -0,0 +1 @@
+0.3.74

{malevich_coretools-0.3.51 → malevich_coretools-0.3.74}/malevich_coretools/__init__.py

@@ -1,3 +1,4 @@
+from .dm_utils import *  # noqa: F403
 from .secondary import logs_streaming  # noqa: F401
 from .tools import vast_settings  # noqa: F401
 from .utils import *  # noqa: F403

{malevich_coretools-0.3.51 → malevich_coretools-0.3.74}/malevich_coretools/abstract/abstract.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
 from pydantic import BaseModel
 
@@ -28,6 +28,10 @@ class Operation(BaseModel):
     operationId: Alias.Id
 
 
+class Tags(BaseModel):
+    data: List[str]
+
+
 class StopOperation(Operation):
     withLogs: bool = False
     infoUrl: Optional[str] = None
@@ -101,6 +105,7 @@ class JsonImage(BaseModel):
     tag: str = ""
     user: Optional[str] = None
     token: Optional[str] = None
+    syncRef: bool = True
 
 
 class UserApp(BaseModel):
@@ -132,6 +137,14 @@ class KeysValues(Operation):
     data: Dict[str, str]
 
 
+class Keys(BaseModel):
+    data: Dict[str, str]
+
+
+class Statuses(BaseModel):
+    data: Dict[str, str]    # key - runId, value - one of: IN_PROGRESS, SUCCESS, FAILED
+
+
 class ScaleInfo(BaseModel):
     taskId: Optional[Alias.Id] = None
     appId: Alias.Id
@@ -139,12 +152,8 @@ class ScaleInfo(BaseModel):
 
 
 class TaskComponent(BaseModel):
-    appControl: Optional[str] = None
-    control: Optional[str] = None
-    extra: Optional[str] = None
-    internal: Optional[str] = None
+    main: Optional[str] = None
     keyValue: Optional[str] = None
-    minimal: Optional[str] = None
     objectStorage: Optional[str] = None
 
 
@@ -197,6 +206,7 @@ class MainTask(BaseModel):
     profileMode: Optional[str] = None
     withLogs: bool = False  # use only in prepare
     saveFails: bool = True
+    clearDagLogs: bool = True
     scaleCount: int = 1
     scaleInfo: List[ScaleInfo]
     component: TaskComponent
@@ -228,6 +238,7 @@ class MainPipeline(BaseModel):
     run: bool = True
     synthetic: bool = False
     saveFails: bool = True
+    clearDagLogs: bool = True
     scaleCount: int = 1
     tags: Optional[Dict[str, str]] = None
 
@@ -241,6 +252,7 @@ class RunTask(Operation):
     profileMode: Optional[str] = None
     withLogs: bool = False
     schedule: Optional[Schedule] = None
+    broadcast: bool = False
 
 
 class AppManage(Operation):
@@ -254,6 +266,10 @@ class ResultIds(BaseModel):
     ids: List[Alias.Id]
 
 
+class ResultNames(BaseModel):
+    names: List[str]
+
+
 class FilesDirs(BaseModel):
     files: Dict[str, int]
     directories: List[str]
@@ -385,8 +401,8 @@ class CollectionMetadata(BaseModel):
 
 class LogsResult(BaseModel):
     data: str
-    logs: Dict[str, str]
-    userLogs: Dict[str, str] = {}
+    logs: Optional[Dict[str, str]] = {}
+    userLogs: Optional[Dict[str, str]] = {}
 
 
 class AppLog(BaseModel):
@@ -400,6 +416,7 @@ class PipelineRunInfo(BaseModel):
 
 class AppLogs(BaseModel):
     operationId: Alias.Id
+    runId: Optional[Alias.Id] = None
     dagLogs: str = ""
     data: Dict[str, AppLog] = {}
     error: Optional[str] = None
@@ -449,6 +466,7 @@ class InputFunctionInfo(FunctionInfo):
 
 class ProcessorFunctionInfo(FunctionInfo):
     isStream: bool
+    objectDfConvert: bool = False
     contextClass: Optional[Dict[str, Any]] = None   # model_json_schema
 
 
@@ -501,6 +519,24 @@ class OperationOrNone(BaseModel):
     operationId: Optional[Alias.Id] = None
 
 
+class AdminRunInfoReq(OperationOrNone):
+    dmId: Optional[str] = None
+
+
+class AdminDMRegister(BaseModel):
+    id: int
+    url: Optional[str]
+    secret: Optional[str]
+    appSecret: Optional[str]
+    login: Optional[str]
+    actuaryDMId: Optional[int]
+
+
+class AdminDMUnregister(BaseModel):
+    id: str
+    actuaryDMId: Optional[int]
+
+
 class AdminStopOperation(BaseModel):
     operationId: Optional[Alias.Id] = None
     withLogs: bool
@@ -603,7 +639,7 @@ class UserLimits(BaseModel):
     defaultGpuDisk: int
 
 
-class BasePlatformSettings(BaseModel):
+class BasePlatformSettingsMain(BaseModel):
     memoryRequest: Optional[int] = None
     memoryLimit: Optional[int] = None
     cpuRequest: Optional[int] = None
@@ -613,7 +649,11 @@ class BasePlatformSettings(BaseModel):
     kubeconfig: Optional[str] = None
 
 
-class Limits(BasePlatformSettings):
+class BasePlatformSettings(BasePlatformSettingsMain):
+    allowKafka: bool = False
+
+
+class Limits(BasePlatformSettingsMain):
     gpuDisk: Optional[int] = None
 
 
@@ -675,3 +715,37 @@ class MCPToolCall(BaseModel):
 class RunsFilter(BaseModel):
     data: Optional[Dict[str, str]] = None
     withTags: bool = False
+
+
+class AppLocalScheme(BaseModel):
+    keys: List[str]
+    optionalKeys: Set[str]
+
+
+class AppErrorInfo(BaseModel):
+    operationId: str
+    runId: str
+    bindId: str
+    funId: str
+    iteration: int
+    isProcessor: bool = True
+    trace: str
+    errType: str
+    errArgs: List[str]
+    isMalevichErr: bool
+    cfg: Optional[Dict[str, Any]]
+    schemes: Optional[Dict[str, AppLocalScheme]] = None
+    args: List[List[Union[Union[str, List[str]], List[Union[str, List[str]]]]]] = None  # mb empty for send info structure
+    argsNames: List[str]
+
+
+class AppErrorInfos(BaseModel):
+    data: List[AppErrorInfo]
+
+
+class AppErrorInfoFilter(BaseModel):
+    operationId: str
+    runId: Optional[str] = None
+    bindId: Optional[str] = None
+    errType: Optional[str] = None
+    isMalevichErr: bool = False

{malevich_coretools-0.3.51 → malevich_coretools-0.3.74}/malevich_coretools/abstract/pipeline.py

@@ -1,5 +1,6 @@
+import json
 from enum import IntEnum
-from typing import Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 from pydantic import BaseModel
 
@@ -7,6 +8,7 @@ from malevich_coretools.abstract.abstract import (  # noqa: F401
     AdminRunInfo,
     Alias,
     JsonImage,
+    MainTaskCfg,
     Restrictions,
     ScaleInfo,
     Schedule,
@@ -26,6 +28,14 @@ from malevich_coretools.abstract.operations import (
 )
 
 
+def _validation_common(arg: 'BaseArgument', other_k = 0) -> None:
+    k = (arg.id is not None) + (arg.collectionName is not None) + (arg.collectionId is not None) + other_k
+    if k == 0:
+        raise ValueError("argument construction requires exactly one parameter, got none")
+    if k > 1:
+        raise ValueError("argument construction requires exactly one parameter, multiple provided")
+
+
 class PullCollectionPolicy(IntEnum):
     INIT = 0
     IF_NOT_EXIST = 1
@@ -42,7 +52,7 @@ class BaseArgument(BaseModel):
     collectionId: Optional[str] = None                  # hardcode collection with id (or obj path)
 
     def validation(self) -> None:
-        assert (self.id is not None) + (self.collectionName is not None) + (self.collectionId is not None) == 1, "one way of constructing the argument must be chosen"
+        _validation_common(self)
 
 
 class Argument(BaseArgument):
@@ -51,11 +61,11 @@ class Argument(BaseArgument):
 
     def validation(self) -> None:
         if self.group is not None:
-            assert self.id is None and self.collectionName is None and self.collectionId is None, "one way of constructing the argument must be chosen"
+            _validation_common(self, 1)
             for subarg in self.group:
                 subarg.validation()
         else:
-            assert (self.id is not None) + (self.collectionName is not None) + (self.collectionId is not None) == 1, "one way of constructing the argument must be chosen"
+            _validation_common(self)
 
 
 class AlternativeArgument(BaseArgument):
@@ -64,23 +74,23 @@ class AlternativeArgument(BaseArgument):
 
     def validation(self) -> None:
         if self.group is not None:
-            assert self.id is None and self.collectionName is None and self.collectionId is None, "one way of constructing the argument must be chosen"
+            _validation_common(self, 1)
             for subarg in self.group:
                 subarg.validation()
         elif self.alternative is not None:
             for alt_arg in self.alternative:
                 if alt_arg.group is not None:
-                    assert alt_arg.id is None and alt_arg.collectionName is None and alt_arg.collectionId is None, "one way of constructing the argument must be chosen"
+                    _validation_common(self, 1)
                     for subarg in alt_arg.group:
                         subarg.validation()
                 else:
-                    assert (self.id is not None) + (self.collectionName is not None) + (self.collectionId is not None) == 1, "one way of constructing the argument must be chosen"
+                    _validation_common(self)
         else:
-            assert (self.id is not None) + (self.collectionName is not None) + (self.collectionId is not None) == 1, "one way of constructing the argument must be chosen"
+            _validation_common(self)
 
 
 class AppEntity(BaseModel):
-    cfg: Optional[str] = None                           # local cfg for processor/condition
+    cfg: Optional[Union[Alias.Json, Dict]] = None       # local cfg for processor/condition
     arguments: Dict[str, AlternativeArgument] = {}      # TODO or List[Argument]?
     conditions: Optional[Union[Dict[str, bool], List[Dict[str, bool]]]] = None          # condition bindId to it result (list - any variant of them)
     conditionsStructure: Optional[Union[Not, And, Or, InternalNot, InternalAnd, InternalOr, Value]] = None  # set BoolOp, it transform to internal before send
@@ -92,6 +102,8 @@ class AppEntity(BaseModel):
     image: JsonImage
     platform: str = "base"
     platformSettings: Optional[str] = None
+    requestedKeys: Optional[list[str]] = None   # user secret keys
+    optionalKeys: Optional[list[str]] = None    # user secret keys
 
     def internal(self) -> None:
         assert self.conditions is None or self.conditionsStructure is None, "should be set not more, than one of (conditions, conditionsStructure)"
@@ -105,6 +117,8 @@ class AppEntity(BaseModel):
             self.loopConditions = [self.loopConditions]
         if isinstance(self.loopConditionsStructure, BoolOp):
             self.loopConditionsStructure = self.loopConditionsStructure.internal()
+        if isinstance(self.cfg, Dict):
+            self.cfg = json.dumps(self.cfg)
 
     def simplify(self) -> None:
         if isinstance(self.conditionsStructure, InternalBoolOp):
@@ -133,6 +147,7 @@ class Pipeline(BaseModel):
     conditions: Dict[str, Condition] = {}               # bindConditionId to Condition
     results: Dict[str, List[Result]] = {}               # bindProcessorId to results
     pullCollectionPolicy: PullCollectionPolicy = PullCollectionPolicy.IF_NOT_EXIST
+    tags: List[str] = []
 
     def internal(self) -> 'Pipeline':
         for proc in self.processors.values():
@@ -156,7 +171,7 @@ class MainPipelineCfg(BaseModel):
     conditions: Dict[str, Condition]
     results: Dict[str, List[Result]]
     pullCollectionPolicy: PullCollectionPolicy
-    cfg: str
+    cfg: Alias.Json
     infoUrl: Optional[str] = None
     debugMode: bool
     coreManage: bool
@@ -197,3 +212,19 @@ class AdminRunPipelineInfo(BaseModel):
 class AdminRunsInfo(BaseModel):
     tasks: List[AdminRunInfo]
     pipelines: List[AdminRunPipelineInfo]
+
+
+class TaskRunInfo(BaseModel):
+    logs: str
+    inputs: Dict[str, Union[str, Dict[str, Any]]]   # collections from cfg
+    results: Dict[str, List[str]]                   # name -> collections list (1 collection common)
+
+
+class RunInfo(BaseModel):
+    jsonMainTask: Optional[MainTaskCfg] = None
+    jsonMainPipeline: Optional[MainPipelineCfg] = None
+    tags: Optional[Dict[str, str]] = None
+    cfgId: str
+    runs: Dict[str, str] = {}
+    info: Dict[str, TaskRunInfo] = {}
+    stopped: bool