malevich-coretools 0.3.12__tar.gz → 0.3.14__tar.gz

{malevich_coretools-0.3.12/malevich_coretools.egg-info → malevich_coretools-0.3.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: malevich-coretools
-Version: 0.3.12
+Version: 0.3.14
 Author: Andrew Pogrebnoj
 Author-email: andrew@onjulius.co
 License-File: LICENSE

malevich_coretools-0.3.14/VERSION

@@ -0,0 +1 @@
+0.3.14

{malevich_coretools-0.3.12 → malevich_coretools-0.3.14}/malevich_coretools/abstract/__init__.py

@@ -1 +1,2 @@
 from .abstract import *  # noqa: F403
+from .pipeline import *  # noqa: F403

{malevich_coretools-0.3.12 → malevich_coretools-0.3.14}/malevich_coretools/abstract/abstract.py

@@ -195,6 +195,27 @@ class MainTask(BaseModel):
     restrictions: Optional[Restrictions] = Restrictions()
 
 
+class MainPipeline(BaseModel):
+    pipelineId: str
+    cfgId: str
+    infoUrl: Optional[str] = None
+    debugMode: bool = False
+    coreManage: bool = False
+    kafkaMode: bool = False
+    singleRequest: bool = True
+    tlWithoutData: Optional[int] = None
+    waitRuns: bool = True
+    profileMode: Optional[str] = None
+    withLogs: bool = False
+    component: TaskComponent = TaskComponent()
+    policy: TaskPolicy = TaskPolicy()
+    restrictions: Optional[Restrictions] = Restrictions()
+    scaleInfo: List[ScaleInfo] = []
+    withListener: bool = False
+    kafkaModeUrl: Optional[str] = None
+    run: bool = True
+
+
 class RunTask(Operation):
     cfgId: Optional[Alias.Id] = None
     infoUrl: Optional[str] = None

malevich_coretools-0.3.14/malevich_coretools/abstract/pipeline.py

@@ -0,0 +1,78 @@
+from enum import IntEnum
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel
+
+from malevich_coretools.abstract import JsonImage
+from malevich_coretools.abstract.abstract import (  # noqa: F401
+    Restrictions,
+    ScaleInfo,
+    TaskComponent,
+    TaskPolicy,
+)
+
+
+class PullCollectionPolicy(IntEnum):
+    INIT = 0
+    IF_NOT_EXIST = 1
+    FORCE_RELOAD = 2
+    FORCE_RELOAD_ALL = 3
+
+
+class BaseArgument(BaseModel):
+    id: Optional[str] = None                            # from - bindProcessorId
+    indices: Optional[List[int]] = None                 # full if None
+    # or
+    collectionName: Optional[str] = None                # get id (or obj path) from cfg by it
+    # or
+    collectionId: Optional[str] = None                  # hardcode collection with id (or obj path)
+
+    def validation(self) -> None:
+        assert (self.id is not None) + (self.collectionName is not None) + (self.collectionId is not None) == 1, "one way of constructing the argument must be chosen"
+
+
+class Argument(BaseArgument):
+    group: Optional[List[BaseArgument]] = None                    # for constructed dfs, sink
+
+    def validation(self) -> None:
+        if self.group is not None:
+            assert self.id is None and self.collectionName is None and self.collectionId is None, "one way of constructing the argument must be chosen"
+            for subarg in self.group:
+                subarg.validation()
+        else:
+            assert (self.id is not None) + (self.collectionName is not None) + (self.collectionId is not None) == 1, "one way of constructing the argument must be chosen"
+
+
+class AppEntity(BaseModel):
+    cfg: Optional[str] = None                           # local cfg for processor/condition
+    arguments: Dict[str, Argument] = {}                 # TODO or List[Argument]?
+    conditions: Optional[Dict[str, bool]] = None        # condition bindId to it result
+
+    loopArguments: Optional[Dict[str, Argument]] = None # other calls, TODO or List[Argument]?, problems
+    loopConditions: Optional[Dict[str, bool]] = None    # condition bindId to it result for loop
+
+    image: JsonImage
+    platform: str = "base"
+    platformSettings: str = ""
+
+
+class Processor(AppEntity):
+    processorId: str
+    outputId: Optional[str] = None
+
+
+class Condition(AppEntity):                             # at least one of (true, false) set
+    conditionId: str
+
+
+class Result(BaseModel):                                # save collection (only processor)
+    name: str
+    index: Optional[int] = None
+
+
+class Pipeline(BaseModel):
+    pipelineId: str
+    processors: Dict[str, Processor] = {}               # bindProcessorId to Processor
+    conditions: Dict[str, Condition] = {}               # bindConditionId to Condition
+    results: Dict[str, List[Result]] = {}               # bindProcessorId to results
+    pullCollectionPolicy: PullCollectionPolicy = PullCollectionPolicy.IF_NOT_EXIST

{malevich_coretools-0.3.12 → malevich_coretools-0.3.14}/malevich_coretools/funcs/funcs.py

@@ -442,6 +442,44 @@ def delete_userTasks(wait: bool, *args, **kwargs) -> Alias.Info:
 def delete_userTasks_id(id: str, wait: bool, *args, **kwargs) -> Alias.Info:
     return send_to_core_modify(USER_TASKS_ID(id, wait), *args, **kwargs, is_post=False)
 
+# UserPipelinesController
+
+
+def get_userPipelines(*args, **kwargs) -> ResultIds:
+    return model_from_json(send_to_core_get(USER_PIPELINES(None), *args, **kwargs), ResultIds)
+
+
+def get_userPipelines_realIds(*args, **kwargs) -> ResultIds:
+    return model_from_json(send_to_core_get(USER_PIPELINES_REAL_IDS, *args, **kwargs), ResultIds)
+
+
+def get_userPipelines_mapIds(*args, **kwargs) -> ResultIdsMap:
+    return model_from_json(send_to_core_get(USER_PIPELINES_MAP_IDS, *args, **kwargs), ResultIdsMap)
+
+
+def get_userPipelines_id(id: str, *args, **kwargs) -> UserTask:
+    return model_from_json(send_to_core_get(USER_PIPELINES_ID(id, None), *args, **kwargs), UserTask)
+
+
+def get_userPipelines_realId(id: str, *args, **kwargs) -> UserTask:
+    return model_from_json(send_to_core_get(USER_PIPELINES_REAL_ID(id), *args, **kwargs), UserTask)
+
+
+def post_userPipelines(data: UserTask, wait: bool, *args, **kwargs) -> Alias.Id:
+    return send_to_core_modify(USER_PIPELINES(wait), data, *args, **kwargs)
+
+
+def post_userPipelines_id(id: str, data: UserTask, wait: bool, *args, **kwargs) -> Alias.Info:
+    return send_to_core_modify(USER_PIPELINES_ID(id, wait), data, *args, **kwargs)
+
+
+def delete_userPipelines(wait: bool, *args, **kwargs) -> Alias.Info:
+    return send_to_core_modify(USER_PIPELINES(wait), *args, **kwargs, is_post=False)
+
+
+def delete_userPipelines_id(id: str, wait: bool, *args, **kwargs) -> Alias.Info:
+    return send_to_core_modify(USER_PIPELINES_ID(id, wait), *args, **kwargs, is_post=False)
+
 # UserCfgsController
 
 
@@ -606,6 +644,16 @@ def post_manager_task_run(data: RunTask, with_show: bool, long: bool, long_timeo
         return AppLogs.model_validate_json(res)
 
 
+def post_manager_pipeline(data: MainPipeline, with_show: bool, long: bool, long_timeout: int, wait: bool, auth: Optional[AUTH], conn_url: Optional[str]=None, *args, **kwargs) -> Union[Alias.Id, AppLogs]:
+    check_profile_mode(data.profileMode)
+    res = send_to_core_modify(MANAGER_PIPELINE(wait and not long), data, with_show=with_show, show_func=show_logs_func, auth=auth, conn_url=conn_url, *args, **kwargs)
+    if wait and long:
+        res = asyncio.run(__get_result(res, timeout=long_timeout, auth=auth))
+    if not wait:
+        return res
+    return AppLogs.model_validate_json(res)
+
+
 def post_manager_task_unschedule(data: UnscheduleOperation, with_show: bool, wait: bool, *args, **kwargs) -> Alias.Info:
     return send_to_core_modify(MANAGER_TASK_UNSCHEDULE(wait), data, with_show=with_show, *args, **kwargs)
 

{malevich_coretools-0.3.12 → malevich_coretools-0.3.14}/malevich_coretools/secondary/const.py

@@ -122,6 +122,14 @@ USER_TASKS_MAP_IDS = f"{USER_TASKS_MAIN}/mapIds"
 USER_TASKS_ID = lambda id, wait: with_wait(f"{USER_TASKS_MAIN}/{urllib.parse.quote(id, safe='')}", wait)
 USER_TASKS_REAL_ID = lambda id: f"{USER_TASKS_MAIN}/realIds/{urllib.parse.quote(id, safe='')}"
 
+## UserPipelinesController
+USER_PIPELINES_MAIN = f"{API_VERSION}/userPipelines"
+USER_PIPELINES = lambda wait: with_wait(f"{USER_PIPELINES_MAIN}/", wait)
+USER_PIPELINES_REAL_IDS = f"{USER_PIPELINES_MAIN}/realIds"
+USER_PIPELINES_MAP_IDS = f"{USER_PIPELINES_MAIN}/mapIds"
+USER_PIPELINES_ID = lambda id, wait: with_wait(f"{USER_PIPELINES_MAIN}/{urllib.parse.quote(id, safe='')}", wait)
+USER_PIPELINES_REAL_ID = lambda id: f"{USER_PIPELINES_MAIN}/realIds/{urllib.parse.quote(id, safe='')}"
+
 ## UserCfgsController
 USER_CFGS_MAIN = f"{API_VERSION}/userCfgs"
 USER_CFGS = lambda wait: with_wait(f"{USER_CFGS_MAIN}/", wait)
@@ -166,6 +174,7 @@ MANAGER_TASK_STOP = lambda wait: with_wait(f"{MANAGER_MAIN}/task/stop", wait)
 MANAGER_TASK_STOP_ALL = lambda wait: with_wait(f"{MANAGER_MAIN}/task/stopAll", wait)
 MANAGER_TASK_RESUME = lambda wait: with_wait(f"{MANAGER_MAIN}/task/resume", wait)
 MANAGER_TASK_PAUSE = lambda wait: with_wait(f"{MANAGER_MAIN}/task/pause", wait)
+MANAGER_PIPELINE = lambda wait: with_wait(f"{MANAGER_MAIN}/pipeline", wait)
 MANAGER_APP_STOP = lambda wait: with_wait(f"{MANAGER_MAIN}/app/stop", wait)
 MANAGER_APP_RESUME = lambda wait: with_wait(f"{MANAGER_MAIN}/app/resume", wait)
 MANAGER_APP_PAUSE = lambda wait: with_wait(f"{MANAGER_MAIN}/app/pause", wait)

{malevich_coretools-0.3.12 → malevich_coretools-0.3.14}/malevich_coretools/utils.py

@@ -1861,6 +1861,179 @@ def delete_task(
     return f.delete_userTasks_id(id, wait=wait, auth=auth, conn_url=conn_url)
 
 
+# UserPipelines
+
+
+def get_pipelines(
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+) -> ResultIds:
+    """return pipelines ids for current user"""
+    if batcher is None:
+        batcher = Config.BATCHER
+    if batcher is not None:
+        return batcher.add("getPipelines", result_model=ResultIds)
+    return f.get_userPipelines(auth=auth, conn_url=conn_url)
+
+
+def get_pipelines_real(
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+) -> ResultIds:
+    """return pipelines real ids for current user"""
+    if batcher is None:
+        batcher = Config.BATCHER
+    if batcher is not None:
+        return batcher.add("getPipelinesReal", result_model=ResultIds)
+    return f.get_userPipelines_realIds(auth=auth, conn_url=conn_url)
+
+
+def get_pipelines_map(
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+) -> ResultIdsMap:
+    """return pipelines ids with real ids for current user"""
+    if batcher is None:
+        batcher = Config.BATCHER
+    if batcher is not None:
+        return batcher.add("getPipelinesMapIds", result_model=ResultIdsMap)
+    return f.get_userPipelines_mapIds(auth=auth, conn_url=conn_url)
+
+
+def get_pipeline(
+    id: str,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+) -> Pipeline:
+    """return pipeline by `id` """
+    if batcher is None:
+        batcher = Config.BATCHER
+    if batcher is not None:
+        return batcher.add("getPipelineById", vars={"id": id}, result_model=Pipeline)
+    return f.get_userTasks_id(id, auth=auth, conn_url=conn_url)
+
+
+def get_pipeline_real(
+    id: str,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+) -> Pipeline:
+    """return pipeline by real `id` """
+    if batcher is None:
+        batcher = Config.BATCHER
+    if batcher is not None:
+        return batcher.add("getPipelineByRealId", vars={"id": id}, result_model=Pipeline)
+    return f.get_userPipelines_realId(id, auth=auth, conn_url=conn_url)
+
+
+def create_pipeline(
+    pipeline_id: str,
+    processors: Dict[str, Processor] = None,
+    conditions: Dict[str, Condition] = None,
+    results: Dict[str, List[Result]] = None,
+    pull_collection_policy: PullCollectionPolicy = PullCollectionPolicy.IF_NOT_EXIST,
+    wait: bool = True,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+) -> Alias.Id:
+    """create pipeline"""
+    if batcher is None:
+        batcher = Config.BATCHER
+    if processors is None:
+        processors = {}
+    if conditions is None:
+        conditions = {}
+    if results is None:
+        results = {}
+    data = Pipeline(
+        pipelineId=pipeline_id,
+        processors=processors,
+        conditions=conditions,
+        results=results,
+        pullCollectionPolicy=pull_collection_policy,
+    )
+    if batcher is not None:
+        return batcher.add("postPipeline", data=data)
+    return f.post_userPipelines(data, wait=wait, auth=auth, conn_url=conn_url)
+
+
+def update_pipeline(
+    id: str,
+    pipeline_id: str,
+    processors: Dict[str, Processor] = None,
+    conditions: Dict[str, Condition] = None,
+    results: Dict[str, List[Result]] = None,
+    pull_collection_policy: PullCollectionPolicy = PullCollectionPolicy.IF_NOT_EXIST,
+    wait: bool = True,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+) -> Alias.Info:
+    """update pipeline by `id` """
+    if batcher is None:
+        batcher = Config.BATCHER
+    if processors is None:
+        processors = {}
+    if conditions is None:
+        conditions = {}
+    if results is None:
+        results = {}
+    data = Pipeline(
+        pipelineId=pipeline_id,
+        processors=processors,
+        conditions=conditions,
+        results=results,
+        pullCollectionPolicy=pull_collection_policy,
+    )
+    if batcher is not None:
+        return batcher.add("postPipelineById", data=data, vars={"id": id})
+    return f.post_userPipelines_id(id, data, wait=wait, auth=auth, conn_url=conn_url)
+
+
+def delete_pipelines(
+    wait: bool = True,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+) -> Alias.Info:
+    """delete all user pipelines"""
+    if batcher is None:
+        batcher = Config.BATCHER
+    if batcher is not None:
+        return batcher.add("deletePipelines")
+    return f.delete_userPipelines(wait=wait, auth=auth, conn_url=conn_url)
+
+
+def delete_pipeline(
+    id: str,
+    wait: bool = True,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+) -> Alias.Info:
+    """delete user pipeline by `id` """
+    if batcher is None:
+        batcher = Config.BATCHER
+    if batcher is not None:
+        return batcher.add("deletePipelineById", vars={"id": id})
+    return f.delete_userPipelines_id(id, wait=wait, auth=auth, conn_url=conn_url)
+
+
 # UserCfgs
 
 
@@ -2592,6 +2765,140 @@ def task_run(
     )
 
 
+def pipeline_full(
+    pipeline_id: str,
+    cfg_id: str,
+    info_url: Optional[str] = None,
+    debug_mode: bool = False,
+    core_manage: bool = False,
+    single_request: bool = False,
+    profile_mode: Optional[str] = None,
+    component: TaskComponent = None,
+    policy: TaskPolicy = None,
+    restrictions: Optional[Restrictions] = None,
+    scaleInfo: List[ScaleInfo] = None,
+    with_show: bool = True,
+    long: bool = False,
+    long_timeout: Optional[int] = WAIT_RESULT_TIMEOUT,
+    wait: bool = True,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+) -> Union[Alias.Id, AppLogs]:
+    if batcher is None:
+        batcher = Config.BATCHER
+    if scaleInfo is None:
+        scaleInfo = []
+    if component is None:
+        component = TaskComponent()
+    if policy is None:
+        policy = TaskPolicy()
+    if restrictions is None:
+        restrictions = Restrictions()
+    data = MainPipeline(
+        pipelineId=pipeline_id,
+        cfgId=cfg_id,
+        infoUrl=info_url,
+        debugMode=debug_mode,
+        coreManage=core_manage,
+        kafkaMode=False,
+        singleRequest=single_request,
+        tlWithoutData=None,
+        waitRuns=True,
+        profileMode=profile_mode,
+        withLogs=True,
+        component=component,
+        policy=policy,
+        restrictions=restrictions,
+        scaleInfo=scaleInfo,
+        withListener=False,
+        kafkaModeUrl=None,
+        run=True,
+    )
+    if batcher is not None:
+        return batcher.add("sendPipeline", data=data, result_model=AppLogs)
+    return f.post_manager_pipeline(
+        data,
+        with_show=with_show,
+        long=long,
+        long_timeout=long_timeout,
+        wait=wait,
+        auth=auth,
+        conn_url=conn_url,
+    )
+
+
+def pipeline_prepare(
+    pipeline_id: str,
+    cfg_id: str,
+    info_url: Optional[str] = None,
+    debug_mode: bool = False,
+    core_manage: bool = False,
+    kafka_mode: bool = False,
+    single_request: bool = False,
+    tl_without_data: Optional[int] = None,
+    wait_runs: bool = True,
+    profile_mode: Optional[str] = None,
+    with_logs: bool = False,
+    component: TaskComponent = None,
+    policy: TaskPolicy = None,
+    restrictions: Optional[Restrictions] = None,
+    scaleInfo: List[ScaleInfo] = None,
+    with_listener: bool = False,
+    kafka_mode_url_response: Optional[str] = None,
+    with_show: bool = True,
+    long: bool = False,
+    long_timeout: Optional[int] = WAIT_RESULT_TIMEOUT,
+    wait: bool = True,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+) -> Union[Alias.Id, AppLogs]:
+    if batcher is None:
+        batcher = Config.BATCHER
+    if scaleInfo is None:
+        scaleInfo = []
+    if component is None:
+        component = TaskComponent()
+    if policy is None:
+        policy = TaskPolicy()
+    if restrictions is None:
+        restrictions = Restrictions()
+    data = MainPipeline(
+        pipelineId=pipeline_id,
+        cfgId=cfg_id,
+        infoUrl=info_url,
+        debugMode=debug_mode,
+        coreManage=core_manage,
+        kafkaMode=kafka_mode,
+        singleRequest=single_request,
+        tlWithoutData=tl_without_data,
+        waitRuns=wait_runs,
+        profileMode=profile_mode,
+        withLogs=with_logs,
+        component=component,
+        policy=policy,
+        restrictions=restrictions,
+        scaleInfo=scaleInfo,
+        withListener=with_listener,
+        kafkaModeUrl=kafka_mode_url_response,
+        run=False,
+    )
+    if batcher is not None:
+        return batcher.add("sendPipeline", data=data, result_model=AppLogs)
+    return f.post_manager_pipeline(
+        data,
+        with_show=with_show,
+        long=long,
+        long_timeout=long_timeout,
+        wait=wait,
+        auth=auth,
+        conn_url=conn_url,
+    )
+
+
 def task_unschedule(
     schedule_id: str,
     with_show: bool = True,

{malevich_coretools-0.3.12 → malevich_coretools-0.3.14/malevich_coretools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: malevich-coretools
-Version: 0.3.12
+Version: 0.3.14
 Author: Andrew Pogrebnoj
 Author-email: andrew@onjulius.co
 License-File: LICENSE

{malevich_coretools-0.3.12 → malevich_coretools-0.3.14}/malevich_coretools.egg-info/SOURCES.txt

@@ -14,6 +14,7 @@ malevich_coretools.egg-info/requires.txt
 malevich_coretools.egg-info/top_level.txt
 malevich_coretools/abstract/__init__.py
 malevich_coretools/abstract/abstract.py
+malevich_coretools/abstract/pipeline.py
 malevich_coretools/abstract/statuses.py
 malevich_coretools/admin/__init__.py
 malevich_coretools/admin/utils.py

malevich_coretools-0.3.12/VERSION

@@ -1 +0,0 @@
-0.3.12