malevich-coretools 0.3.31__tar.gz → 0.3.33__tar.gz

{malevich_coretools-0.3.31/malevich_coretools.egg-info → malevich_coretools-0.3.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: malevich-coretools
-Version: 0.3.31
+Version: 0.3.33
 Author: Andrew Pogrebnoj
 Author-email: andrew@onjulius.co
 License-File: LICENSE

malevich_coretools-0.3.33/VERSION

@@ -0,0 +1 @@
+0.3.33

{malevich_coretools-0.3.31 → malevich_coretools-0.3.33}/malevich_coretools/abstract/abstract.py

@@ -209,11 +209,13 @@ class MainPipeline(BaseModel):
     withLogs: bool = False
     component: TaskComponent = TaskComponent()
     policy: TaskPolicy = TaskPolicy()
+    schedule: Optional[Schedule] = None
     restrictions: Optional[Restrictions] = Restrictions()
     scaleInfo: List[ScaleInfo] = []
     withListener: bool = False
     kafkaModeUrl: Optional[str] = None
     run: bool = True
+    synthetic: bool = False
 
 
 class RunTask(Operation):
@@ -373,7 +375,7 @@ class AppLog(BaseModel):
     data: List[LogsResult]
 
 
-class PipelineInfo(BaseModel):
+class PipelineRunInfo(BaseModel):
     conditions: Dict[str, Dict[int, bool]]  # condition bindId -> iteration -> value
     fails: Dict[str, List[int]]             # bindId -> fail iterations (1 in common situation)
 
@@ -383,7 +385,7 @@ class AppLogs(BaseModel):
     dagLogs: str = ""
     data: Dict[str, AppLog] = {}
     error: Optional[str] = None
-    pipeline: Optional[PipelineInfo] = None # only for pipeline
+    pipeline: Optional[PipelineRunInfo] = None # only for pipeline
 
 
 class AppLogsWithResults(AppLogs):
@@ -471,10 +473,6 @@ class AdminRunInfo(BaseModel):
     cfgId: Alias.Id
 
 
-class AdminRunsInfo(BaseModel):
-    data: List[AdminRunInfo]
-
-
 class OperationOrNone(BaseModel):
     operationId: Optional[Alias.Id] = None
 
@@ -505,6 +503,7 @@ class RunSettings(BaseModel):
 class Endpoint(BaseModel):
     hash: Optional[Alias.Id] = None
     taskId: Optional[Alias.Id] = None
+    pipelineId: Optional[Alias.Id] = None   # should set taskId or pipelineId
     cfgId: Optional[Alias.Id] = None
     sla: Optional[str] = None
     active: Optional[bool] = None

{malevich_coretools-0.3.31 → malevich_coretools-0.3.33}/malevich_coretools/abstract/pipeline.py

@@ -3,10 +3,13 @@ from typing import Dict, List, Optional
 
 from pydantic import BaseModel
 
-from malevich_coretools.abstract import JsonImage
 from malevich_coretools.abstract.abstract import (  # noqa: F401
+    AdminRunInfo,
+    Alias,
+    JsonImage,
     Restrictions,
     ScaleInfo,
+    Schedule,
     TaskComponent,
     TaskPolicy,
 )
@@ -98,3 +101,53 @@ class Pipeline(BaseModel):
     conditions: Dict[str, Condition] = {}               # bindConditionId to Condition
     results: Dict[str, List[Result]] = {}               # bindProcessorId to results
     pullCollectionPolicy: PullCollectionPolicy = PullCollectionPolicy.IF_NOT_EXIST
+
+
+class MainPipelineCfg(BaseModel):
+    operationId: Alias.Id
+    pipelineId: Alias.Id
+    processors: Dict[str, Processor]
+    conditions: Dict[str, Condition]
+    results: Dict[str, List[Result]]
+    pullCollectionPolicy: PullCollectionPolicy
+    cfg: str
+    infoUrl: Optional[str] = None
+    debugMode: bool
+    coreManage: bool
+    kafkaMode: bool
+    singleRequest: bool
+    tlWithoutData: Optional[int] = None
+    waitRuns: bool
+    profileMode: Optional[str] = None
+    withLogs: bool
+    component: TaskComponent
+    policy: TaskPolicy
+    schedule: Optional[Schedule] = None
+    restrictions: Optional[Restrictions] = None
+    scaleInfo: List[ScaleInfo]
+    schemesNames: List[str]
+    login: Optional[Alias.Login] = None
+    withListener: bool
+    kafkaModeUrl: Optional[str] = None
+    run: bool
+    synthetic: bool = False
+
+
+class PipelineInfo(BaseModel):
+    pipelineId: Alias.Id
+    processors: Dict[str, Processor]
+    conditions: Dict[str, Condition]
+    results: Dict[str, List[Result]]
+    cfg: Alias.Json
+    login: Alias.Login
+
+
+class AdminRunPipelineInfo(BaseModel):
+    operationId: Alias.Id
+    pipelineInfo: PipelineInfo
+    cfgId: Alias.Id
+
+
+class AdminRunsInfo(BaseModel):
+    tasks: List[AdminRunInfo]
+    pipelines: List[AdminRunPipelineInfo]

{malevich_coretools-0.3.31 → malevich_coretools-0.3.33}/malevich_coretools/admin/utils.py

@@ -24,7 +24,7 @@ def admin_get_run_info(
     auth: Optional[AUTH] = None,
     conn_url: Optional[str] = None,
     batcher: Optional[Batcher] = None,
-) -> Alias.Info:
+) -> Alias.Json:
     """return run info by operation `id`"""
     if batcher is None:
         batcher = Config.BATCHER

{malevich_coretools-0.3.31 → malevich_coretools-0.3.33}/malevich_coretools/funcs/funcs.py

@@ -10,7 +10,11 @@ import requests
 from requests.models import Response
 
 from malevich_coretools.abstract.abstract import *  # noqa: F403
-from malevich_coretools.abstract.pipeline import Pipeline
+from malevich_coretools.abstract.pipeline import (
+    AdminRunsInfo,
+    MainPipelineCfg,
+    Pipeline,
+)
 from malevich_coretools.funcs.checks import check_profile_mode
 from malevich_coretools.secondary import Config, model_from_json, show_logs_func
 from malevich_coretools.secondary.const import *  # noqa: F403
@@ -576,6 +580,10 @@ def get_run_mainTaskCfg(id: str, *args, **kwargs) -> MainTaskCfg:
     return model_from_json(send_to_core_get(TEMP_RUN_MAIN_TASK_CFG(id), *args, **kwargs), MainTaskCfg)
 
 
+def get_run_mainPipelineCfg(id: str, *args, **kwargs) -> MainPipelineCfg:
+    return model_from_json(send_to_core_get(TEMP_RUN_MAIN_PIPELINE_CFG(id), *args, **kwargs), MainPipelineCfg)
+
+
 def get_run_operationsIds(task_id: str, cfg_id: Optional[str]=None, *args, **kwargs) -> ResultIds:
     return model_from_json(send_to_core_get(TEMP_RUN_OPERATIONS_IDS(task_id, cfg_id), *args, **kwargs), ResultIds)
 
@@ -586,7 +594,7 @@ def get_admin_runs(*args, **kwargs) -> AdminRunsInfo:
     return model_from_json(send_to_core_get(ADMIN_RUNS, *args, **kwargs), AdminRunsInfo)
 
 
-def get_admin_runs_info(data: OperationOrNone, *args, **kwargs) -> Union[Alias.Json, Alias.Info]:
+def get_admin_runs_info(data: OperationOrNone, *args, **kwargs) -> Alias.Json:
     return send_to_core_modify(ADMIN_RUNS_INFO, data, *args, **kwargs)
 
 

{malevich_coretools-0.3.31 → malevich_coretools-0.3.33}/malevich_coretools/secondary/const.py

@@ -157,6 +157,7 @@ TEMP_RUN_MAIN = f"{API_VERSION}/run"
 TEMP_RUN_CONDITION = lambda operationId: f"{TEMP_RUN_MAIN}/condition/{urllib.parse.quote(str(operationId), safe='')}"
 TEMP_RUN_ACTIVE_RUNS = f"{TEMP_RUN_MAIN}/activeRuns"
 TEMP_RUN_MAIN_TASK_CFG = lambda operationId: f"{TEMP_RUN_MAIN}/mainTaskCfg/{urllib.parse.quote(str(operationId), safe='')}"
+TEMP_RUN_MAIN_PIPELINE_CFG = lambda operationId: f"{TEMP_RUN_MAIN}/mainPipelineCfg/{urllib.parse.quote(str(operationId), safe='')}"
 TEMP_RUN_OPERATIONS_IDS = lambda taskId, cfgId: f"{TEMP_RUN_MAIN}/operationsIds/{urllib.parse.quote(str(taskId), safe='')}" if cfgId is None else f"{TEMP_RUN_MAIN}/operationsIds/{urllib.parse.quote(str(taskId), safe='')}/{urllib.parse.quote(str(cfgId), safe='')}"
 
 ## AdminController

{malevich_coretools-0.3.31 → malevich_coretools-0.3.33}/malevich_coretools/utils.py

@@ -896,6 +896,7 @@ def run_endpoint(
 
 def create_endpoint(
     task_id: Optional[str] = None,
+    pipeline_id: Optional[str] = None,
     cfg_id: Optional[str] = None,
     sla: Optional[str] = None,
     active: Optional[bool] = None,
@@ -911,9 +912,10 @@ def create_endpoint(
     batcher: Optional[Batcher] = None,
 ) -> Alias.Id:
     """create endpoint, return hash, url - api/v1/endpoints/run/{hash}. fields may not be initialized - this will only cause an error at startup"""
+    assert task_id is None or pipeline_id is None, "endpoint should be defined in one way"
     if batcher is None:
         batcher = Config.BATCHER
-    data = Endpoint(taskId=task_id, cfgId=cfg_id, sla=sla, active=active, prepare=prepare, runSettings=run_settings,
+    data = Endpoint(taskId=task_id, pipelineId=pipeline_id, cfgId=cfg_id, sla=sla, active=active, prepare=prepare, runSettings=run_settings,
                     enableNotAuthorized=enable_not_auth, expectedCollectionsWithSchemes=expected_colls_with_schemes, description=description)
     if batcher is not None:
         return batcher.add("postEndpointCreate", data=data)
@@ -923,6 +925,7 @@ def create_endpoint(
 def update_endpoint(
     hash: str,
     task_id: Optional[str] = None,
+    pipeline_id: Optional[str] = None,
     cfg_id: Optional[str] = None,
     sla: Optional[str] = None,
     active: Optional[bool] = None,
@@ -938,9 +941,10 @@ def update_endpoint(
     batcher: Optional[Batcher] = None,
 ) -> Alias.Id:
     """update endpoint, return hash, url - api/v1/endpoints/run/{hash}. update field if it is not None"""
+    assert task_id is None or pipeline_id is None, "endpoint should be defined in one way"
     if batcher is None:
         batcher = Config.BATCHER
-    data = Endpoint(hash=hash, taskId=task_id, cfgId=cfg_id, sla=sla, active=active, prepare=prepare, runSettings=run_settings,
+    data = Endpoint(hash=hash, taskId=task_id, pipelineId=pipeline_id, cfgId=cfg_id, sla=sla, active=active, prepare=prepare, runSettings=run_settings,
                     enableNotAuthorized=enable_not_auth, expectedCollectionsWithSchemes=expected_colls_with_schemes, description=description)
     if batcher is not None:
         return batcher.add("postEndpointUpdate", data=data)
@@ -2413,6 +2417,21 @@ def get_run_main_task_cfg(
     return f.get_run_mainTaskCfg(id, auth=auth, conn_url=conn_url)
 
 
+def get_run_main_pipeline_cfg(
+    id: str,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+) -> MainPipelineCfg:
+    """return mainPipelineCfg by operation `id` for running pipeline"""
+    if batcher is None:
+        batcher = Config.BATCHER
+    if batcher is not None:
+        return batcher.add("getMainPipelineCfg", vars={"operationId": id}, result_model=MainPipelineCfg)
+    return f.get_run_mainPipelineCfg(id, auth=auth, conn_url=conn_url)
+
+
 def get_task_runs(
     task_id: str,
     cfg_id: Optional[str] = None,
@@ -2881,6 +2900,7 @@ def pipeline_full(
     profile_mode: Optional[str] = None,
     component: TaskComponent = None,
     policy: TaskPolicy = None,
+    schedule: Optional[Schedule] = None,
     restrictions: Optional[Restrictions] = None,
     scaleInfo: List[ScaleInfo] = None,
     with_show: bool = True,
@@ -2916,6 +2936,7 @@ def pipeline_full(
         withLogs=True,
         component=component,
         policy=policy,
+        schedule=schedule,
         restrictions=restrictions,
         scaleInfo=scaleInfo,
         withListener=False,
@@ -2953,6 +2974,7 @@ def pipeline_prepare(
     scaleInfo: List[ScaleInfo] = None,
     with_listener: bool = False,
     kafka_mode_url_response: Optional[str] = None,
+    synthetic: bool = False,
     with_show: bool = True,
     long: bool = False,
     long_timeout: Optional[int] = WAIT_RESULT_TIMEOUT,
@@ -2991,6 +3013,7 @@ def pipeline_prepare(
         withListener=with_listener,
         kafkaModeUrl=kafka_mode_url_response,
         run=False,
+        synthetic=synthetic,
     )
     if batcher is not None:
         return batcher.add("sendPipeline", data=data, result_model=AppLogs)

{malevich_coretools-0.3.31 → malevich_coretools-0.3.33/malevich_coretools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: malevich-coretools
-Version: 0.3.31
+Version: 0.3.33
 Author: Andrew Pogrebnoj
 Author-email: andrew@onjulius.co
 License-File: LICENSE

malevich_coretools-0.3.31/VERSION

@@ -1 +0,0 @@
-0.3.31