malevich-coretools 0.3.37__tar.gz → 0.3.39__tar.gz

{malevich_coretools-0.3.37/malevich_coretools.egg-info → malevich_coretools-0.3.39}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: malevich-coretools
-Version: 0.3.37
+Version: 0.3.39
 Author: Andrew Pogrebnoj
 Author-email: andrew@onjulius.co
 License-File: LICENSE
@@ -9,3 +9,6 @@ Requires-Dist: pydantic
 Requires-Dist: pandas
 Requires-Dist: aiohttp
 Requires-Dist: kafka-python
+Dynamic: author
+Dynamic: author-email
+Dynamic: requires-dist

malevich_coretools-0.3.39/VERSION

@@ -0,0 +1 @@
+0.3.39

{malevich_coretools-0.3.37 → malevich_coretools-0.3.39}/malevich_coretools/abstract/abstract.py

@@ -188,6 +188,7 @@ class MainTask(BaseModel):
     waitRuns: bool = True
     profileMode: Optional[str] = None
     withLogs: bool = False  # use only in prepare
+    saveFails: bool = True
     scaleInfo: List[ScaleInfo]
     component: TaskComponent
     policy: TaskPolicy
@@ -216,6 +217,7 @@ class MainPipeline(BaseModel):
     kafkaModeUrl: Optional[str] = None
     run: bool = True
     synthetic: bool = False
+    saveFails: bool = True
 
 
 class RunTask(Operation):

{malevich_coretools-0.3.37 → malevich_coretools-0.3.39}/malevich_coretools/funcs/funcs.py

@@ -142,7 +142,7 @@ def delete_collections_id_del(id: str, data: DocsCollectionChange, wait: bool, *
 # CollectionObjectsController
 
 
-def get_collection_objects(path: Optional[str], recursive: Optional[str], *args, **kwargs) -> FilesDirs:
+def get_collection_objects(path: Optional[str], recursive: Optional[bool], *args, **kwargs) -> FilesDirs:
     return model_from_json(send_to_core_get(COLLECTION_OBJECTS_ALL_GET(path, recursive), *args, **kwargs), FilesDirs)
 
 
@@ -816,6 +816,14 @@ def post_analytics_many(data: UserAnalyticsBatch, wait: bool, *args, **kwargs) -
     return model_from_json(send_to_core_modify(ANALYTICS_MANY(wait), data, *args, **kwargs), ResultIds)
 
 
+def get_runsInfo_last_operation_ids(count: int, *args, **kwargs) -> ResultIds:
+    return model_from_json(send_to_core_get(RUNS_INFO_LAST(count), *args, **kwargs), ResultIds)
+
+
+def get_runsInfo_last_failed_operation_ids(count: int, *args, **kwargs) -> ResultIds:
+    return model_from_json(send_to_core_get(RUNS_INFO_LAST_FAILED(count), *args, **kwargs), ResultIds)
+
+
 async def kafka_send(data: KafkaMsg, *args, **kwargs) -> Union[Alias.Info, KafkaMsg]:
     result = await send_to_core_post_async(KAFKA_SEND, data, *args, **kwargs)
     try:
@@ -837,7 +845,8 @@ async def __get_result(id: str, is_text: bool = True, check_time: float = LONG_S
     host = Config.HOST_PORT if conn_url is None else conn_url
     assert host is not None, "host port not set"
     if auth is None:
-        auth = aiohttp.BasicAuth(login=Config.CORE_USERNAME, password=Config.CORE_PASSWORD, encoding='utf-8')
+        auth = (Config.CORE_USERNAME, Config.CORE_PASSWORD)
+    auth = aiohttp.BasicAuth(login=auth[0], password=auth[1], encoding='utf-8')
 
     timeout_deadline = None if timeout is None else datetime.datetime.now() + datetime.timedelta(0, timeout)
     while True:
@@ -912,7 +921,8 @@ async def send_to_core_get_async(path: str, with_auth=True, show_func: Optional[
     assert host is not None, "host port not set"
     if auth is None or not with_auth:
         auth = (Config.CORE_USERNAME, Config.CORE_PASSWORD) if with_auth else None
-
+    if auth is not None:
+        auth = aiohttp.BasicAuth(login=auth[0], password=auth[1], encoding='utf-8')
     async with async_session or aiohttp.ClientSession(auth=auth, connector=aiohttp.TCPConnector(verify_ssl=False)) as session:
         async with session.get(f"{host}{path}", headers=HEADERS) as response:
             await __async_check_response(response, show_func, f"{host}{path}")
@@ -960,6 +970,7 @@ async def send_to_core_modify_async(path: str, operation: Optional[Any] = None,
     assert host is not None, "host port not set"
     if auth is None:
         auth = (Config.CORE_USERNAME, Config.CORE_PASSWORD)
+    auth = aiohttp.BasicAuth(login=auth[0], password=auth[1], encoding='utf-8')
     if operation is not None:
         operation = json.dumps(operation.model_dump())
 
@@ -1015,6 +1026,7 @@ async def send_to_core_modify_raw_async(path: str, data: bytes, with_auth: bool=
     assert host is not None, "host port not set"
     if auth is None:
         auth = (Config.CORE_USERNAME, Config.CORE_PASSWORD)
+    auth = aiohttp.BasicAuth(login=auth[0], password=auth[1], encoding='utf-8')
 
     async with async_session or aiohttp.ClientSession(auth=auth if with_auth else None, connector=aiohttp.TCPConnector(verify_ssl=False)) as session:
         if is_post:
@@ -1040,7 +1052,9 @@ async def send_to_core_post_async(path: str, operation: Optional[str] = None, wi
     host = Config.HOST_PORT if conn_url is None else conn_url
     assert host is not None, "host port not set"
     if auth is None or not with_auth:
-        auth = aiohttp.BasicAuth(login=Config.CORE_USERNAME, password=Config.CORE_PASSWORD, encoding='utf-8') if with_auth else None
+        auth = (Config.CORE_USERNAME, Config.CORE_PASSWORD) if with_auth else None
+    if auth is not None:
+        auth = aiohttp.BasicAuth(login=auth[0], password=auth[1], encoding='utf-8')
     if operation is not None:
         operation = json.dumps(operation.model_dump())
 

{malevich_coretools-0.3.37 → malevich_coretools-0.3.39}/malevich_coretools/secondary/const.py

@@ -205,6 +205,11 @@ ANALYTICS_MANY = lambda wait: with_wait(f"{ANALYTICS_MAIN}/many", wait)
 ANALYTICS_ID = lambda id, wait: with_wait(f"{ANALYTICS_MAIN}/{id}", wait)
 ANALYTICS_NAME = lambda name, wait: with_wait(f"{ANALYTICS_MAIN}/name/{name}", wait)
 
+## RunsInfoController
+RUNS_INFO_MAIN = f"{API_VERSION}/runsInfo"
+RUNS_INFO_LAST = lambda count: with_key_values(f"{RUNS_INFO_MAIN}/lastOperationsIds", {"count": count})
+RUNS_INFO_LAST_FAILED = lambda count: with_key_values(f"{RUNS_INFO_MAIN}/lastFailedOperationsIds", {"count": count})
+
 ### Kafka
 KAFKA_SEND = f"{MANAGER_MAIN}/kafkaMsg"
 

{malevich_coretools-0.3.37 → malevich_coretools-0.3.39}/malevich_coretools/utils.py

@@ -165,7 +165,7 @@ def get_doc_by_name(
 
 def create_doc(
     data: Union[Alias.Json, Dict, Type[BaseModel]],
-    name: Optional[str],
+    name: Optional[str] = None,
     wait: bool = True,
     *,
     auth: Optional[AUTH] = None,
@@ -677,7 +677,7 @@ def delete_from_collection(
 
 def get_collection_objects(
     path: Optional[str] = None,
-    recursive: Optional[str] = None,
+    recursive: Optional[bool] = None,
     *,
     auth: Optional[AUTH] = None,
     conn_url: Optional[str] = None,
@@ -2650,6 +2650,7 @@ def task_full(
     core_manage: bool = False,
     single_request: bool = False,
     profile_mode: Optional[str] = None,
+    save_fails: bool = True,
     with_show: bool = True,
     long: bool = False,
     long_timeout: Optional[int] = WAIT_RESULT_TIMEOUT,
@@ -2708,6 +2709,7 @@ def task_full(
         waitRuns=True,
         profileMode=profile_mode,
         withLogs=True,
+        saveFails=save_fails,
         scaleInfo=scaleInfo,
         component=component,
         policy=policy,
@@ -2752,6 +2754,7 @@ def task_prepare(
     wait_runs: bool = True,
     with_logs: bool = False,
     profile_mode: Optional[str] = None,
+    save_fails: bool = True,
     with_show: bool = None,
     long: bool = False,
     long_timeout: int = WAIT_RESULT_TIMEOUT,
@@ -2822,6 +2825,7 @@ def task_prepare(
         run=False,
         waitRuns=wait_runs,
         withLogs=with_logs,
+        saveFails=save_fails,
         profileMode=profile_mode,
         scaleInfo=scaleInfo,
         component=component,
@@ -2943,6 +2947,7 @@ def pipeline_full(
     schedule: Optional[Schedule] = None,
     restrictions: Optional[Restrictions] = None,
     scaleInfo: List[ScaleInfo] = None,
+    save_fails: bool = True,
     with_show: bool = True,
     long: bool = False,
     long_timeout: Optional[int] = WAIT_RESULT_TIMEOUT,
@@ -2984,6 +2989,7 @@ def pipeline_full(
         withListener=False,
         kafkaModeUrl=None,
         run=True,
+        saveFails=save_fails,
     )
     if batcher is not None:
         return batcher.add("sendPipeline", data=data, result_model=AppLogs)
@@ -3030,9 +3036,11 @@ def pipeline_prepare(
     with_listener: bool = False,
     kafka_mode_url_response: Optional[str] = None,
     synthetic: bool = False,
+    save_fails: bool = True,
     with_show: bool = True,
     long: bool = False,
     long_timeout: Optional[int] = WAIT_RESULT_TIMEOUT,
+    return_response: bool = False,
     wait: bool = True,
     *,
     auth: Optional[AUTH] = None,
@@ -3070,6 +3078,7 @@ def pipeline_prepare(
         kafkaModeUrl=kafka_mode_url_response,
         run=False,
         synthetic=synthetic,
+        saveFails=save_fails,
     )
     if batcher is not None:
         return batcher.add("sendPipeline", data=data, result_model=AppLogs)
@@ -3079,6 +3088,7 @@ def pipeline_prepare(
             with_show=with_show,
             long=long,
             long_timeout=long_timeout,
+            return_response=return_response,
             wait=wait,
             auth=auth,
             conn_url=conn_url,
@@ -3089,6 +3099,7 @@ def pipeline_prepare(
             with_show=with_show,
             long=long,
             long_timeout=long_timeout,
+            return_response=return_response,
             wait=wait,
             auth=auth,
             conn_url=conn_url,
@@ -3509,6 +3520,39 @@ def update_analytics_many(
     )
 
 
+# RunsInfo
+
+
+def get_last_runs(
+    count: int = 10,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None
+) -> ResultIds:
+    """return last operationIds (prepared task/pipeline)"""
+    if batcher is None:
+        batcher = Config.BATCHER
+    if batcher is not None:
+        return batcher.add("getLastOperationsIds", vars={"count": count}, result_model=ResultIds)
+    return f.get_runsInfo_last_operation_ids(count, auth=auth, conn_url=conn_url)
+
+
+def get_last_failed_runs(
+    count: int = 5,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None
+) -> ResultIds:
+    """return last operationIds (failed prepare/run of task/pipeline)"""
+    if batcher is None:
+        batcher = Config.BATCHER
+    if batcher is not None:
+        return batcher.add("getLastFailedOperationsIds", vars={"count": count}, result_model=ResultIds)
+    return f.get_runsInfo_last_failed_operation_ids(count, auth=auth, conn_url=conn_url)
+
+
 # kafka
 
 

{malevich_coretools-0.3.37 → malevich_coretools-0.3.39/malevich_coretools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: malevich-coretools
-Version: 0.3.37
+Version: 0.3.39
 Author: Andrew Pogrebnoj
 Author-email: andrew@onjulius.co
 License-File: LICENSE
@@ -9,3 +9,6 @@ Requires-Dist: pydantic
 Requires-Dist: pandas
 Requires-Dist: aiohttp
 Requires-Dist: kafka-python
+Dynamic: author
+Dynamic: author-email
+Dynamic: requires-dist

malevich_coretools-0.3.37/VERSION

@@ -1 +0,0 @@
-0.3.37