malevich-coretools 0.3.11__tar.gz → 0.3.13__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of malevich-coretools might be problematic. Click here for more details.

Files changed (37) hide show
  1. {malevich-coretools-0.3.11/malevich_coretools.egg-info → malevich_coretools-0.3.13}/PKG-INFO +1 -1
  2. malevich_coretools-0.3.13/VERSION +1 -0
  3. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/abstract/__init__.py +1 -0
  4. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/abstract/abstract.py +57 -36
  5. malevich_coretools-0.3.13/malevich_coretools/abstract/pipeline.py +73 -0
  6. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/funcs/funcs.py +48 -0
  7. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/secondary/const.py +9 -0
  8. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/utils.py +307 -0
  9. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13/malevich_coretools.egg-info}/PKG-INFO +1 -1
  10. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools.egg-info/SOURCES.txt +1 -0
  11. malevich-coretools-0.3.11/VERSION +0 -1
  12. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/LICENSE +0 -0
  13. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/MANIFEST.in +0 -0
  14. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/README.md +0 -0
  15. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/__init__.py +0 -0
  16. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/abstract/statuses.py +0 -0
  17. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/admin/__init__.py +0 -0
  18. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/admin/utils.py +0 -0
  19. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/batch/__init__.py +0 -0
  20. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/batch/utils.py +0 -0
  21. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/funcs/__init__.py +0 -0
  22. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/funcs/checks.py +0 -0
  23. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/funcs/helpers.py +0 -0
  24. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/secondary/__init__.py +0 -0
  25. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/secondary/config.py +0 -0
  26. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/secondary/helpers.py +0 -0
  27. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/secondary/kafka_utils.py +0 -0
  28. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/tools/__init__.py +0 -0
  29. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/tools/abstract.py +0 -0
  30. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools/tools/vast.py +0 -0
  31. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools.egg-info/dependency_links.txt +0 -0
  32. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools.egg-info/requires.txt +0 -0
  33. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/malevich_coretools.egg-info/top_level.txt +0 -0
  34. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/pyproject.toml +0 -0
  35. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/requirements.txt +0 -0
  36. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/setup.cfg +0 -0
  37. {malevich-coretools-0.3.11 → malevich_coretools-0.3.13}/setup.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: malevich-coretools
3
- Version: 0.3.11
3
+ Version: 0.3.13
4
4
  Author: Andrew Pogrebnoj
5
5
  Author-email: andrew@onjulius.co
6
6
  License-File: LICENSE
@@ -0,0 +1 @@
1
+ 0.3.13
@@ -1 +1,2 @@
1
1
  from .abstract import * # noqa: F403
2
+ from .pipeline import *
@@ -40,7 +40,7 @@ class StopOperationMany(BaseModel):
40
40
 
41
41
  class DocWithName(BaseModel):
42
42
  data: Alias.Json
43
- name: Optional[str]
43
+ name: Optional[str] = None
44
44
 
45
45
 
46
46
  class DocsCollection(BaseModel):
@@ -82,9 +82,9 @@ class SchemesIds(BaseModel):
82
82
 
83
83
  class Shared(BaseModel):
84
84
  userLogins: List[Alias.Login]
85
- collectionsIds: Optional[List[Alias.Id]]
86
- schemesIds: Optional[List[Alias.Id]]
87
- userAppsIds: Optional[List[Alias.Id]]
85
+ collectionsIds: Optional[List[Alias.Id]] = None
86
+ schemesIds: Optional[List[Alias.Id]] = None
87
+ userAppsIds: Optional[List[Alias.Id]] = None
88
88
 
89
89
 
90
90
  class SharedWithUsers(BaseModel):
@@ -132,7 +132,7 @@ class KeysValues(Operation):
132
132
 
133
133
 
134
134
  class ScaleInfo(BaseModel):
135
- taskId: Optional[Alias.Id]
135
+ taskId: Optional[Alias.Id] = None
136
136
  appId: Alias.Id
137
137
  scale: int
138
138
 
@@ -175,7 +175,7 @@ class Restrictions(BaseModel):
175
175
 
176
176
  class MainTask(BaseModel):
177
177
  taskId: Alias.Id
178
- cfgId: Optional[Alias.Id]
178
+ cfgId: Optional[Alias.Id] = None
179
179
  infoUrl: Optional[str] = None
180
180
  debugMode: bool = False
181
181
  run: bool = True
@@ -195,6 +195,27 @@ class MainTask(BaseModel):
195
195
  restrictions: Optional[Restrictions] = Restrictions()
196
196
 
197
197
 
198
+ class MainPipeline(BaseModel):
199
+ pipelineId: str
200
+ cfgId: str
201
+ infoUrl: Optional[str] = None
202
+ debugMode: bool = False
203
+ coreManage: bool = False
204
+ kafkaMode: bool = False
205
+ singleRequest: bool = True
206
+ tlWithoutData: Optional[int] = None
207
+ waitRuns: bool = True
208
+ profileMode: Optional[str] = None
209
+ withLogs: bool = False
210
+ component: TaskComponent = TaskComponent()
211
+ policy: TaskPolicy = TaskPolicy()
212
+ restrictions: Optional[Restrictions] = Restrictions()
213
+ scaleInfo: List[ScaleInfo] = []
214
+ withListener: bool = False
215
+ kafkaModeUrl: Optional[str] = None
216
+ run: bool = True
217
+
218
+
198
219
  class RunTask(Operation):
199
220
  cfgId: Optional[Alias.Id] = None
200
221
  infoUrl: Optional[str] = None
@@ -207,7 +228,7 @@ class RunTask(Operation):
207
228
 
208
229
 
209
230
  class AppManage(Operation):
210
- taskId: Optional[Alias.Id]
231
+ taskId: Optional[Alias.Id] = None
211
232
  appId: Alias.Id
212
233
  runId: Alias.Id
213
234
 
@@ -322,12 +343,12 @@ class MainTaskCfg(BaseModel):
322
343
  tasks: Dict[Alias.Id, UserTask]
323
344
  cfg: str
324
345
  schemesNames: List[str]
325
- infoUrl: Optional[str]
346
+ infoUrl: Optional[str] = None
326
347
  debugMode: bool
327
- login: Optional[Alias.Login]
348
+ login: Optional[Alias.Login] = None
328
349
  coreManage: bool
329
350
  kafkaMode: bool
330
- tlWithoutData: Optional[int]
351
+ tlWithoutData: Optional[int] = None
331
352
 
332
353
 
333
354
  class KafkaMsg(BaseModel):
@@ -386,14 +407,14 @@ class FunctionInfo(BaseModel):
386
407
  id: Alias.Id
387
408
  name: str
388
409
  arguments: List[Tuple[str, Optional[str]]]
389
- finishMsg: Optional[str]
390
- doc: Optional[str]
410
+ finishMsg: Optional[str] = None
411
+ doc: Optional[str] = None
391
412
 
392
413
 
393
414
  class InputFunctionInfo(FunctionInfo):
394
- collectionsNames: Optional[List[str]]
395
- extraCollectionsNames: Optional[List[str]]
396
- query: Optional[str]
415
+ collectionsNames: Optional[List[str]] = None
416
+ extraCollectionsNames: Optional[List[str]] = None
417
+ query: Optional[str] = None
397
418
  mode: str
398
419
 
399
420
 
@@ -402,16 +423,16 @@ class ProcessorFunctionInfo(FunctionInfo):
402
423
 
403
424
 
404
425
  class OutputFunctionInfo(FunctionInfo):
405
- collectionOutNames: Optional[List[str]]
426
+ collectionOutNames: Optional[List[str]] = None
406
427
 
407
428
 
408
429
  class InitInfo(BaseModel):
409
430
  id: Alias.Id
410
431
  enable: bool
411
- tl: Optional[int]
432
+ tl: Optional[int] = None
412
433
  prepare: bool
413
- argname: Optional[str]
414
- doc: Optional[str]
434
+ argname: Optional[str] = None
435
+ doc: Optional[str] = None
415
436
 
416
437
 
417
438
  class AppFunctionsInfo(BaseModel):
@@ -484,7 +505,7 @@ class Endpoint(BaseModel):
484
505
 
485
506
  class EndpointRunInfo(BaseModel):
486
507
  active: bool
487
- description: Optional[str]
508
+ description: Optional[str] = None
488
509
  author: str
489
510
  collections: Optional[Dict[str, str]] = None
490
511
 
@@ -548,26 +569,26 @@ class UserLimits(BaseModel):
548
569
 
549
570
 
550
571
  class BasePlatformSettings(BaseModel):
551
- memoryRequest: Optional[int]
552
- memoryLimit: Optional[int]
553
- cpuRequest: Optional[int]
554
- cpuLimit: Optional[int]
555
- storageRequest: Optional[int]
556
- storageLimit: Optional[int]
572
+ memoryRequest: Optional[int] = None
573
+ memoryLimit: Optional[int] = None
574
+ cpuRequest: Optional[int] = None
575
+ cpuLimit: Optional[int] = None
576
+ storageRequest: Optional[int] = None
577
+ storageLimit: Optional[int] = None
557
578
 
558
579
 
559
580
  class Limits(BasePlatformSettings):
560
- gpuDisk: Optional[int]
581
+ gpuDisk: Optional[int] = None
561
582
 
562
583
 
563
584
  class LimitsScope(BaseModel): # for superuser/admin
564
585
  login: str
565
- appMemoryRequest: Optional[int]
566
- appMemoryLimit: Optional[int]
567
- appCpuRequest: Optional[int]
568
- appCpuLimit: Optional[int]
569
- appStorageRequest: Optional[int]
570
- appStorageLimit: Optional[int]
571
- assetsLimit: Optional[int]
572
- allowCommonGpu: Optional[bool]
573
- gpuDiskMax: Optional[int]
586
+ appMemoryRequest: Optional[int] = None
587
+ appMemoryLimit: Optional[int] = None
588
+ appCpuRequest: Optional[int] = None
589
+ appCpuLimit: Optional[int] = None
590
+ appStorageRequest: Optional[int] = None
591
+ appStorageLimit: Optional[int] = None
592
+ assetsLimit: Optional[int] = None
593
+ allowCommonGpu: Optional[bool] = None
594
+ gpuDiskMax: Optional[int] = None
@@ -0,0 +1,73 @@
1
+ from enum import IntEnum
2
+ from typing import Optional, List, Dict
3
+ from malevich_coretools.abstract import JsonImage
4
+
5
+ from pydantic import BaseModel
6
+
7
+ from malevich_coretools.abstract.abstract import Restrictions, ScaleInfo, TaskComponent, TaskPolicy
8
+
9
+
10
+ class PullCollectionPolicy(IntEnum):
11
+ INIT = 0
12
+ IF_NOT_EXIST = 1
13
+ FORCE_RELOAD = 2
14
+ FORCE_RELOAD_ALL = 3
15
+
16
+
17
+ class BaseArgument(BaseModel):
18
+ id: Optional[str] = None # from - bindProcessorId
19
+ indices: Optional[List[int]] = None # full if None
20
+ # or
21
+ collectionName: Optional[str] = None # get id (or obj path) from cfg by it
22
+ # or
23
+ collectionId: Optional[str] = None # hardcode collection with id (or obj path)
24
+
25
+ def validation(self):
26
+ assert (self.id is not None) + (self.collectionName is not None) + (self.collectionId is not None) == 1, "one way of constructing the argument must be chosen"
27
+
28
+
29
+ class Argument(BaseArgument):
30
+ group: Optional[List[BaseArgument]] = None # for constructed dfs, sink
31
+
32
+ def validation(self):
33
+ if self.group is not None:
34
+ assert self.id is None and self.collectionName is None and self.collectionId is None, "one way of constructing the argument must be chosen"
35
+ for subarg in self.group:
36
+ subarg.validation()
37
+ else:
38
+ assert (self.id is not None) + (self.collectionName is not None) + (self.collectionId is not None) == 1, "one way of constructing the argument must be chosen"
39
+
40
+
41
+ class AppEntity(BaseModel):
42
+ cfg: Optional[str] = None # local cfg for processor/condition
43
+ arguments: Dict[str, Argument] = {} # TODO or List[Argument]?
44
+ conditions: Optional[Dict[str, bool]] = None # condition bindId to it result
45
+
46
+ loopArguments: Optional[Dict[str, Argument]] = None # other calls, TODO or List[Argument]?, problems
47
+ loopConditions: Optional[Dict[str, bool]] = None # condition bindId to it result for loop
48
+
49
+ image: JsonImage
50
+ platform: str = "base"
51
+ platformSettings: str = ""
52
+
53
+
54
+ class Processor(AppEntity):
55
+ processorId: str
56
+ outputId: Optional[str] = None
57
+
58
+
59
+ class Condition(AppEntity): # at least one of (true, false) set
60
+ conditionId: str
61
+
62
+
63
+ class Result(BaseModel): # save collection (only processor)
64
+ name: str
65
+ index: Optional[int] = None
66
+
67
+
68
+ class Pipeline(BaseModel):
69
+ pipelineId: str
70
+ processors: Dict[str, Processor] = {} # bindProcessorId to Processor
71
+ conditions: Dict[str, Condition] = {} # bindConditionId to Condition
72
+ results: Dict[str, List[Result]] = {} # bindProcessorId to results
73
+ pullCollectionPolicy: PullCollectionPolicy = PullCollectionPolicy.IF_NOT_EXIST
@@ -442,6 +442,44 @@ def delete_userTasks(wait: bool, *args, **kwargs) -> Alias.Info:
442
442
  def delete_userTasks_id(id: str, wait: bool, *args, **kwargs) -> Alias.Info:
443
443
  return send_to_core_modify(USER_TASKS_ID(id, wait), *args, **kwargs, is_post=False)
444
444
 
445
+ # UserPipelinesController
446
+
447
+
448
+ def get_userPipelines(*args, **kwargs) -> ResultIds:
449
+ return model_from_json(send_to_core_get(USER_PIPELINES(None), *args, **kwargs), ResultIds)
450
+
451
+
452
+ def get_userPipelines_realIds(*args, **kwargs) -> ResultIds:
453
+ return model_from_json(send_to_core_get(USER_PIPELINES_REAL_IDS, *args, **kwargs), ResultIds)
454
+
455
+
456
+ def get_userPipelines_mapIds(*args, **kwargs) -> ResultIdsMap:
457
+ return model_from_json(send_to_core_get(USER_PIPELINES_MAP_IDS, *args, **kwargs), ResultIdsMap)
458
+
459
+
460
+ def get_userPipelines_id(id: str, *args, **kwargs) -> UserTask:
461
+ return model_from_json(send_to_core_get(USER_PIPELINES_ID(id, None), *args, **kwargs), UserTask)
462
+
463
+
464
+ def get_userPipelines_realId(id: str, *args, **kwargs) -> UserTask:
465
+ return model_from_json(send_to_core_get(USER_PIPELINES_REAL_ID(id), *args, **kwargs), UserTask)
466
+
467
+
468
+ def post_userPipelines(data: UserTask, wait: bool, *args, **kwargs) -> Alias.Id:
469
+ return send_to_core_modify(USER_PIPELINES(wait), data, *args, **kwargs)
470
+
471
+
472
+ def post_userPipelines_id(id: str, data: UserTask, wait: bool, *args, **kwargs) -> Alias.Info:
473
+ return send_to_core_modify(USER_PIPELINES_ID(id, wait), data, *args, **kwargs)
474
+
475
+
476
+ def delete_userPipelines(wait: bool, *args, **kwargs) -> Alias.Info:
477
+ return send_to_core_modify(USER_PIPELINES(wait), *args, **kwargs, is_post=False)
478
+
479
+
480
+ def delete_userPipelines_id(id: str, wait: bool, *args, **kwargs) -> Alias.Info:
481
+ return send_to_core_modify(USER_PIPELINES_ID(id, wait), *args, **kwargs, is_post=False)
482
+
445
483
  # UserCfgsController
446
484
 
447
485
 
@@ -606,6 +644,16 @@ def post_manager_task_run(data: RunTask, with_show: bool, long: bool, long_timeo
606
644
  return AppLogs.model_validate_json(res)
607
645
 
608
646
 
647
+ def post_manager_pipeline(data: MainPipeline, with_show: bool, long: bool, long_timeout: int, wait: bool, auth: Optional[AUTH], conn_url: Optional[str]=None, *args, **kwargs) -> Union[Alias.Id, AppLogs]:
648
+ check_profile_mode(data.profileMode)
649
+ res = send_to_core_modify(MANAGER_PIPELINE(wait and not long), data, with_show=with_show, show_func=show_logs_func, auth=auth, conn_url=conn_url, *args, **kwargs)
650
+ if wait and long:
651
+ res = asyncio.run(__get_result(res, timeout=long_timeout, auth=auth))
652
+ if not wait:
653
+ return res
654
+ return AppLogs.model_validate_json(res)
655
+
656
+
609
657
  def post_manager_task_unschedule(data: UnscheduleOperation, with_show: bool, wait: bool, *args, **kwargs) -> Alias.Info:
610
658
  return send_to_core_modify(MANAGER_TASK_UNSCHEDULE(wait), data, with_show=with_show, *args, **kwargs)
611
659
 
@@ -122,6 +122,14 @@ USER_TASKS_MAP_IDS = f"{USER_TASKS_MAIN}/mapIds"
122
122
  USER_TASKS_ID = lambda id, wait: with_wait(f"{USER_TASKS_MAIN}/{urllib.parse.quote(id, safe='')}", wait)
123
123
  USER_TASKS_REAL_ID = lambda id: f"{USER_TASKS_MAIN}/realIds/{urllib.parse.quote(id, safe='')}"
124
124
 
125
+ ## UserPipelinesController
126
+ USER_PIPELINES_MAIN = f"{API_VERSION}/userPipelines"
127
+ USER_PIPELINES = lambda wait: with_wait(f"{USER_PIPELINES_MAIN}/", wait)
128
+ USER_PIPELINES_REAL_IDS = f"{USER_PIPELINES_MAIN}/realIds"
129
+ USER_PIPELINES_MAP_IDS = f"{USER_PIPELINES_MAIN}/mapIds"
130
+ USER_PIPELINES_ID = lambda id, wait: with_wait(f"{USER_PIPELINES_MAIN}/{urllib.parse.quote(id, safe='')}", wait)
131
+ USER_PIPELINES_REAL_ID = lambda id: f"{USER_PIPELINES_MAIN}/realIds/{urllib.parse.quote(id, safe='')}"
132
+
125
133
  ## UserCfgsController
126
134
  USER_CFGS_MAIN = f"{API_VERSION}/userCfgs"
127
135
  USER_CFGS = lambda wait: with_wait(f"{USER_CFGS_MAIN}/", wait)
@@ -166,6 +174,7 @@ MANAGER_TASK_STOP = lambda wait: with_wait(f"{MANAGER_MAIN}/task/stop", wait)
166
174
  MANAGER_TASK_STOP_ALL = lambda wait: with_wait(f"{MANAGER_MAIN}/task/stopAll", wait)
167
175
  MANAGER_TASK_RESUME = lambda wait: with_wait(f"{MANAGER_MAIN}/task/resume", wait)
168
176
  MANAGER_TASK_PAUSE = lambda wait: with_wait(f"{MANAGER_MAIN}/task/pause", wait)
177
+ MANAGER_PIPELINE = lambda wait: with_wait(f"{MANAGER_MAIN}/pipeline", wait)
169
178
  MANAGER_APP_STOP = lambda wait: with_wait(f"{MANAGER_MAIN}/app/stop", wait)
170
179
  MANAGER_APP_RESUME = lambda wait: with_wait(f"{MANAGER_MAIN}/app/resume", wait)
171
180
  MANAGER_APP_PAUSE = lambda wait: with_wait(f"{MANAGER_MAIN}/app/pause", wait)
@@ -1861,6 +1861,179 @@ def delete_task(
1861
1861
  return f.delete_userTasks_id(id, wait=wait, auth=auth, conn_url=conn_url)
1862
1862
 
1863
1863
 
1864
+ # UserPipelines
1865
+
1866
+
1867
+ def get_pipelines(
1868
+ *,
1869
+ auth: Optional[AUTH] = None,
1870
+ conn_url: Optional[str] = None,
1871
+ batcher: Optional[Batcher] = None,
1872
+ ) -> ResultIds:
1873
+ """return pipelines ids for current user"""
1874
+ if batcher is None:
1875
+ batcher = Config.BATCHER
1876
+ if batcher is not None:
1877
+ return batcher.add("getPipelines", result_model=ResultIds)
1878
+ return f.get_userPipelines(auth=auth, conn_url=conn_url)
1879
+
1880
+
1881
+ def get_pipelines_real(
1882
+ *,
1883
+ auth: Optional[AUTH] = None,
1884
+ conn_url: Optional[str] = None,
1885
+ batcher: Optional[Batcher] = None,
1886
+ ) -> ResultIds:
1887
+ """return pipelines real ids for current user"""
1888
+ if batcher is None:
1889
+ batcher = Config.BATCHER
1890
+ if batcher is not None:
1891
+ return batcher.add("getPipelinesReal", result_model=ResultIds)
1892
+ return f.get_userPipelines_realIds(auth=auth, conn_url=conn_url)
1893
+
1894
+
1895
+ def get_pipelines_map(
1896
+ *,
1897
+ auth: Optional[AUTH] = None,
1898
+ conn_url: Optional[str] = None,
1899
+ batcher: Optional[Batcher] = None,
1900
+ ) -> ResultIdsMap:
1901
+ """return pipelines ids with real ids for current user"""
1902
+ if batcher is None:
1903
+ batcher = Config.BATCHER
1904
+ if batcher is not None:
1905
+ return batcher.add("getPipelinesMapIds", result_model=ResultIdsMap)
1906
+ return f.get_userPipelines_mapIds(auth=auth, conn_url=conn_url)
1907
+
1908
+
1909
+ def get_pipeline(
1910
+ id: str,
1911
+ *,
1912
+ auth: Optional[AUTH] = None,
1913
+ conn_url: Optional[str] = None,
1914
+ batcher: Optional[Batcher] = None,
1915
+ ) -> Pipeline:
1916
+ """return pipeline by `id` """
1917
+ if batcher is None:
1918
+ batcher = Config.BATCHER
1919
+ if batcher is not None:
1920
+ return batcher.add("getPipelineById", vars={"id": id}, result_model=Pipeline)
1921
+ return f.get_userTasks_id(id, auth=auth, conn_url=conn_url)
1922
+
1923
+
1924
+ def get_pipeline_real(
1925
+ id: str,
1926
+ *,
1927
+ auth: Optional[AUTH] = None,
1928
+ conn_url: Optional[str] = None,
1929
+ batcher: Optional[Batcher] = None,
1930
+ ) -> Pipeline:
1931
+ """return pipeline by real `id` """
1932
+ if batcher is None:
1933
+ batcher = Config.BATCHER
1934
+ if batcher is not None:
1935
+ return batcher.add("getPipelineByRealId", vars={"id": id}, result_model=Pipeline)
1936
+ return f.get_userPipelines_realId(id, auth=auth, conn_url=conn_url)
1937
+
1938
+
1939
+ def create_pipeline(
1940
+ pipeline_id: str,
1941
+ processors: Dict[str, Processor] = None,
1942
+ conditions: Dict[str, Condition] = None,
1943
+ results: Dict[str, List[Result]] = None,
1944
+ pull_collection_policy: PullCollectionPolicy = PullCollectionPolicy.IF_NOT_EXIST,
1945
+ wait: bool = True,
1946
+ *,
1947
+ auth: Optional[AUTH] = None,
1948
+ conn_url: Optional[str] = None,
1949
+ batcher: Optional[Batcher] = None,
1950
+ ) -> Alias.Id:
1951
+ """create pipeline"""
1952
+ if batcher is None:
1953
+ batcher = Config.BATCHER
1954
+ if processors is None:
1955
+ processors = {}
1956
+ if conditions is None:
1957
+ conditions = {}
1958
+ if results is None:
1959
+ results = {}
1960
+ data = Pipeline(
1961
+ pipelineId=pipeline_id,
1962
+ processors=processors,
1963
+ conditions=conditions,
1964
+ results=results,
1965
+ pullCollectionPolicy=pull_collection_policy,
1966
+ )
1967
+ if batcher is not None:
1968
+ return batcher.add("postPipeline", data=data)
1969
+ return f.post_userPipelines(data, wait=wait, auth=auth, conn_url=conn_url)
1970
+
1971
+
1972
+ def update_pipeline(
1973
+ id: str,
1974
+ pipeline_id: str,
1975
+ processors: Dict[str, Processor] = None,
1976
+ conditions: Dict[str, Condition] = None,
1977
+ results: Dict[str, List[Result]] = None,
1978
+ pull_collection_policy: PullCollectionPolicy = PullCollectionPolicy.IF_NOT_EXIST,
1979
+ wait: bool = True,
1980
+ *,
1981
+ auth: Optional[AUTH] = None,
1982
+ conn_url: Optional[str] = None,
1983
+ batcher: Optional[Batcher] = None,
1984
+ ) -> Alias.Info:
1985
+ """update pipeline by `id` """
1986
+ if batcher is None:
1987
+ batcher = Config.BATCHER
1988
+ if processors is None:
1989
+ processors = {}
1990
+ if conditions is None:
1991
+ conditions = {}
1992
+ if results is None:
1993
+ results = {}
1994
+ data = Pipeline(
1995
+ pipelineId=pipeline_id,
1996
+ processors=processors,
1997
+ conditions=conditions,
1998
+ results=results,
1999
+ pullCollectionPolicy=pull_collection_policy,
2000
+ )
2001
+ if batcher is not None:
2002
+ return batcher.add("postPipelineById", data=data, vars={"id": id})
2003
+ return f.post_userPipelines_id(id, data, wait=wait, auth=auth, conn_url=conn_url)
2004
+
2005
+
2006
+ def delete_pipelines(
2007
+ wait: bool = True,
2008
+ *,
2009
+ auth: Optional[AUTH] = None,
2010
+ conn_url: Optional[str] = None,
2011
+ batcher: Optional[Batcher] = None,
2012
+ ) -> Alias.Info:
2013
+ """delete all user pipelines"""
2014
+ if batcher is None:
2015
+ batcher = Config.BATCHER
2016
+ if batcher is not None:
2017
+ return batcher.add("deletePipelines")
2018
+ return f.delete_userPipelines(wait=wait, auth=auth, conn_url=conn_url)
2019
+
2020
+
2021
+ def delete_pipeline(
2022
+ id: str,
2023
+ wait: bool = True,
2024
+ *,
2025
+ auth: Optional[AUTH] = None,
2026
+ conn_url: Optional[str] = None,
2027
+ batcher: Optional[Batcher] = None,
2028
+ ) -> Alias.Info:
2029
+ """delete user pipeline by `id` """
2030
+ if batcher is None:
2031
+ batcher = Config.BATCHER
2032
+ if batcher is not None:
2033
+ return batcher.add("deletePipelineById", vars={"id": id})
2034
+ return f.delete_userPipelines_id(id, wait=wait, auth=auth, conn_url=conn_url)
2035
+
2036
+
1864
2037
  # UserCfgs
1865
2038
 
1866
2039
 
@@ -2592,6 +2765,140 @@ def task_run(
2592
2765
  )
2593
2766
 
2594
2767
 
2768
+ def pipeline_full(
2769
+ pipeline_id: str,
2770
+ cfg_id: str,
2771
+ info_url: Optional[str] = None,
2772
+ debug_mode: bool = False,
2773
+ core_manage: bool = False,
2774
+ single_request: bool = False,
2775
+ profile_mode: Optional[str] = None,
2776
+ component: TaskComponent = None,
2777
+ policy: TaskPolicy = None,
2778
+ restrictions: Optional[Restrictions] = None,
2779
+ scaleInfo: List[ScaleInfo] = None,
2780
+ with_show: bool = True,
2781
+ long: bool = False,
2782
+ long_timeout: Optional[int] = WAIT_RESULT_TIMEOUT,
2783
+ wait: bool = True,
2784
+ *,
2785
+ auth: Optional[AUTH] = None,
2786
+ conn_url: Optional[str] = None,
2787
+ batcher: Optional[Batcher] = None,
2788
+ ) -> Union[Alias.Id, AppLogs]:
2789
+ if batcher is None:
2790
+ batcher = Config.BATCHER
2791
+ if scaleInfo is None:
2792
+ scaleInfo = []
2793
+ if component is None:
2794
+ component = TaskComponent()
2795
+ if policy is None:
2796
+ policy = TaskPolicy()
2797
+ if restrictions is None:
2798
+ restrictions = Restrictions()
2799
+ data = MainPipeline(
2800
+ pipelineId=pipeline_id,
2801
+ cfgId=cfg_id,
2802
+ infoUrl=info_url,
2803
+ debugMode=debug_mode,
2804
+ coreManage=core_manage,
2805
+ kafkaMode=False,
2806
+ singleRequest=single_request,
2807
+ tlWithoutData=None,
2808
+ waitRuns=True,
2809
+ profileMode=profile_mode,
2810
+ withLogs=True,
2811
+ component=component,
2812
+ policy=policy,
2813
+ restrictions=restrictions,
2814
+ scaleInfo=scaleInfo,
2815
+ withListener=False,
2816
+ kafkaModeUrl=None,
2817
+ run=True,
2818
+ )
2819
+ if batcher is not None:
2820
+ return batcher.add("sendPipeline", data=data, result_model=AppLogs)
2821
+ return f.post_manager_pipeline(
2822
+ data,
2823
+ with_show=with_show,
2824
+ long=long,
2825
+ long_timeout=long_timeout,
2826
+ wait=wait,
2827
+ auth=auth,
2828
+ conn_url=conn_url,
2829
+ )
2830
+
2831
+
2832
+ def pipeline_prepare(
2833
+ pipeline_id: str,
2834
+ cfg_id: str,
2835
+ info_url: Optional[str] = None,
2836
+ debug_mode: bool = False,
2837
+ core_manage: bool = False,
2838
+ kafka_mode: bool = False,
2839
+ single_request: bool = False,
2840
+ tl_without_data: Optional[int] = None,
2841
+ wait_runs: bool = True,
2842
+ profile_mode: Optional[str] = None,
2843
+ with_logs: bool = False,
2844
+ component: TaskComponent = None,
2845
+ policy: TaskPolicy = None,
2846
+ restrictions: Optional[Restrictions] = None,
2847
+ scaleInfo: List[ScaleInfo] = None,
2848
+ with_listener: bool = False,
2849
+ kafka_mode_url_response: Optional[str] = None,
2850
+ with_show: bool = True,
2851
+ long: bool = False,
2852
+ long_timeout: Optional[int] = WAIT_RESULT_TIMEOUT,
2853
+ wait: bool = True,
2854
+ *,
2855
+ auth: Optional[AUTH] = None,
2856
+ conn_url: Optional[str] = None,
2857
+ batcher: Optional[Batcher] = None,
2858
+ ) -> Union[Alias.Id, AppLogs]:
2859
+ if batcher is None:
2860
+ batcher = Config.BATCHER
2861
+ if scaleInfo is None:
2862
+ scaleInfo = []
2863
+ if component is None:
2864
+ component = TaskComponent()
2865
+ if policy is None:
2866
+ policy = TaskPolicy()
2867
+ if restrictions is None:
2868
+ restrictions = Restrictions()
2869
+ data = MainPipeline(
2870
+ pipelineId=pipeline_id,
2871
+ cfgId=cfg_id,
2872
+ infoUrl=info_url,
2873
+ debugMode=debug_mode,
2874
+ coreManage=core_manage,
2875
+ kafkaMode=kafka_mode,
2876
+ singleRequest=single_request,
2877
+ tlWithoutData=tl_without_data,
2878
+ waitRuns=wait_runs,
2879
+ profileMode=profile_mode,
2880
+ withLogs=with_logs,
2881
+ component=component,
2882
+ policy=policy,
2883
+ restrictions=restrictions,
2884
+ scaleInfo=scaleInfo,
2885
+ withListener=with_listener,
2886
+ kafkaModeUrl=kafka_mode_url_response,
2887
+ run=False,
2888
+ )
2889
+ if batcher is not None:
2890
+ return batcher.add("sendPipeline", data=data, result_model=AppLogs)
2891
+ return f.post_manager_pipeline(
2892
+ data,
2893
+ with_show=with_show,
2894
+ long=long,
2895
+ long_timeout=long_timeout,
2896
+ wait=wait,
2897
+ auth=auth,
2898
+ conn_url=conn_url,
2899
+ )
2900
+
2901
+
2595
2902
  def task_unschedule(
2596
2903
  schedule_id: str,
2597
2904
  with_show: bool = True,
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: malevich-coretools
3
- Version: 0.3.11
3
+ Version: 0.3.13
4
4
  Author: Andrew Pogrebnoj
5
5
  Author-email: andrew@onjulius.co
6
6
  License-File: LICENSE
@@ -14,6 +14,7 @@ malevich_coretools.egg-info/requires.txt
14
14
  malevich_coretools.egg-info/top_level.txt
15
15
  malevich_coretools/abstract/__init__.py
16
16
  malevich_coretools/abstract/abstract.py
17
+ malevich_coretools/abstract/pipeline.py
17
18
  malevich_coretools/abstract/statuses.py
18
19
  malevich_coretools/admin/__init__.py
19
20
  malevich_coretools/admin/utils.py
@@ -1 +0,0 @@
1
- 0.3.11