malevich-coretools 0.3.66__py3-none-any.whl → 0.3.68__py3-none-any.whl

malevich_coretools/__init__.py

@@ -1,3 +1,4 @@
+from .dm_utils import *  # noqa: F403
 from .secondary import logs_streaming  # noqa: F401
 from .tools import vast_settings  # noqa: F401
 from .utils import *  # noqa: F403

malevich_coretools/abstract/abstract.py

@@ -400,6 +400,7 @@ class PipelineRunInfo(BaseModel):
 
 class AppLogs(BaseModel):
     operationId: Alias.Id
+    runId: Optional[Alias.Id] = None
     dagLogs: str = ""
     data: Dict[str, AppLog] = {}
     error: Optional[str] = None

malevich_coretools/dm_utils.py

@@ -0,0 +1,233 @@
+from typing import (
+    Any,
+    AsyncIterable,
+    Coroutine,
+    Iterable,
+    List,
+    Literal,
+    Optional,
+    Union,
+    overload,
+)
+
+import malevich_coretools.funcs.dm_funcs as f
+
+
+@overload
+def dm_stream(
+    operation_id: str,
+    run_id: str,
+    bind_id: str,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[False] = False,
+) -> Iterable:
+    pass
+
+
+@overload
+def dm_stream(
+    operation_id: str,
+    run_id: str,
+    bind_id: str,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, AsyncIterable]:
+    pass
+
+
+def dm_stream(
+    operation_id: str,
+    run_id: str,
+    bind_id: str,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: bool = False,
+) -> Union[Iterable, Coroutine[Any, Any, AsyncIterable]]:
+    if is_async:
+        return f.dm_stream_async(operation_id, run_id, bind_id, conn_url=conn_url)
+    return f.dm_stream(operation_id, run_id, bind_id, conn_url=conn_url)
+
+
+@overload
+def dm_continue(
+    operation_id: str,
+    run_id: str,
+    id: str,
+    data: Any,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[False] = False,
+) -> None:
+    pass
+
+
+@overload
+def dm_continue(
+    operation_id: str,
+    run_id: str,
+    id: str,
+    data: Any,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, None]:
+    pass
+
+
+def dm_continue(
+    operation_id: str,
+    run_id: str,
+    id: str,
+    data: Any,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: bool = False,
+) -> Union[None, Coroutine[Any, Any, None]]:
+    if is_async:
+        return f.dm_continue_async(operation_id, run_id, id, data, conn_url=conn_url)
+    return f.dm_continue(operation_id, run_id, id, data, conn_url=conn_url)
+
+
+@overload
+def dm_state(
+    operation_id: str,
+    run_id: str,
+    bind_id: str,
+    key: Optional[str] = None,
+    index: Optional[int] = None,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[False] = False,
+) -> Any:
+    pass
+
+
+@overload
+def dm_state(
+    operation_id: str,
+    run_id: str,
+    bind_id: str,
+    key: Optional[str] = None,
+    index: Optional[int] = None,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, Any]:
+    pass
+
+
+def dm_state(
+    operation_id: str,
+    run_id: str,
+    bind_id: str,
+    key: Optional[str] = None,
+    index: Optional[int] = None,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: bool = False,
+) -> Union[Any, Coroutine[Any, Any, Any]]:
+    if is_async:
+        return f.dm_state_async(operation_id, run_id, bind_id, key, index, conn_url=conn_url)
+    return f.dm_state(operation_id, run_id, bind_id, key, index, conn_url=conn_url)
+
+
+@overload
+def dm_journal_list(
+    operation_id: str,
+    run_id: str,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[False] = False,
+) -> List[str]:
+    pass
+
+
+@overload
+def dm_journal_list(
+    operation_id: str,
+    run_id: str,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, List[str]]:
+    pass
+
+
+def dm_journal_list(
+    operation_id: str,
+    run_id: str,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: bool = False,
+) -> Union[List[str], Coroutine[Any, Any, List[str]]]:
+    if is_async:
+        return f.dm_journal_list_async(operation_id, run_id, conn_url=conn_url)
+    return f.dm_journal_list(operation_id, run_id, conn_url=conn_url)
+
+
+@overload
+def dm_journal(
+    operation_id: str,
+    run_id: str,
+    key: str,
+    is_stream: Literal[False] = False,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[False] = False,
+) -> Any:
+    pass
+
+
+@overload
+def dm_journal(
+    operation_id: str,
+    run_id: str,
+    key: str,
+    is_stream: Literal[False] = False,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, Any]:
+    pass
+
+
+@overload
+def dm_journal(
+    operation_id: str,
+    run_id: str,
+    key: str,
+    is_stream: Literal[True],
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[False] = False,
+) -> Iterable:
+    pass
+
+
+@overload
+def dm_journal(
+    operation_id: str,
+    run_id: str,
+    key: str,
+    is_stream: Literal[True],
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, AsyncIterable]:
+    pass
+
+
+def dm_journal(
+    operation_id: str,
+    run_id: str,
+    key: str,
+    is_stream: bool = False,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: bool = False,
+) -> Union[Any, Iterable, Coroutine[Any, Any, Union[Any, AsyncIterable]]]:
+    if is_async:
+        return f.dm_journal_async(operation_id, run_id, key, is_stream, conn_url=conn_url)
+    return f.dm_journal(operation_id, run_id, key, is_stream, conn_url=conn_url)

malevich_coretools/funcs/dm_funcs.py

@@ -0,0 +1,162 @@
+from http import HTTPStatus
+from typing import Any, AsyncIterable, Iterable, List, Optional, Union
+
+import aiohttp
+import requests
+from requests.models import Response
+
+from malevich_coretools.secondary import Config
+from malevich_coretools.secondary.const import *  # noqa: F403
+
+
+def dm_stream(operation_id: str, run_id: str, bind_id: str, conn_url: Optional[str]=None) -> Iterable:
+    return send_to_dm_stream(DM_STREAM(operation_id, run_id, bind_id), conn_url=conn_url)
+
+
+async def dm_stream_async(operation_id: str, run_id: str, bind_id: str, conn_url: Optional[str]=None) -> AsyncIterable:
+    return await send_to_dm_stream_async(DM_STREAM(operation_id, run_id, bind_id), conn_url=conn_url)
+
+
+def dm_continue(operation_id: str, run_id: str, id: str, data: Any, conn_url: Optional[str]=None) -> None:
+    return send_to_dm_post(DM_CONTINUE(operation_id, run_id, id), data, conn_url=conn_url)
+
+
+async def dm_continue_async(operation_id: str, run_id: str, id: str, data: Any, conn_url: Optional[str]=None) -> None:
+    return await send_to_dm_post_async(DM_CONTINUE(operation_id, run_id, id), data, conn_url=conn_url)
+
+
+def dm_state(operation_id: str, run_id: str, bind_id: str, key: Optional[str] = None, index: Optional[int] = None, conn_url: Optional[str]=None) -> Any:
+    return send_to_dm_get(DM_STATE(operation_id, run_id, bind_id, key, index), conn_url=conn_url)
+
+
+async def dm_state_async(operation_id: str, run_id: str, bind_id: str, key: Optional[str] = None, index: Optional[int] = None, conn_url: Optional[str]=None) -> Any:
+    return await send_to_dm_get_async(DM_STATE(operation_id, run_id, bind_id, key, index), conn_url=conn_url)
+
+
+def dm_journal_list(operation_id: str, run_id: str, conn_url: Optional[str]=None) -> List[str]:
+    return send_to_dm_get(DM_JOURNAL_LIST(operation_id, run_id), is_text=False, conn_url=conn_url)
+
+
+async def dm_journal_list_async(operation_id: str, run_id: str, conn_url: Optional[str]=None) -> List[str]:
+    return await send_to_dm_get_async(DM_JOURNAL_LIST(operation_id, run_id), is_text=False, conn_url=conn_url)
+
+
+def dm_journal(operation_id: str, run_id: str, key: str, is_stream: bool, conn_url: Optional[str]=None) -> Union[Any, Iterable]:
+    if is_stream:
+        return send_to_dm_stream(DM_JOURNAL(operation_id, run_id, key, is_stream), conn_url=conn_url)
+    return send_to_dm_get(DM_JOURNAL(operation_id, run_id, key, is_stream), conn_url=conn_url)
+
+
+async def dm_journal_async(operation_id: str, run_id: str, key: str, is_stream: bool, conn_url: Optional[str]=None) -> Union[Any, AsyncIterable]:
+    if is_stream:
+        return await send_to_dm_stream_async(DM_JOURNAL(operation_id, run_id, key, is_stream), conn_url=conn_url)
+    return await send_to_dm_get_async(DM_JOURNAL(operation_id, run_id, key, is_stream), conn_url=conn_url)
+
+
+#
+
+def __check_response(path: str, response: Response):  # noqa: ANN202
+    if response.status_code >= 400:
+        text = response.text
+        msg = f"failed: {text}" if len(text) > 0 else "failed"
+        Config.logger.error(f"{path} {msg}")
+
+        if response.reason is not None and len(response.reason) == 0:
+            response.reason = text
+    response.raise_for_status()
+
+
+async def __async_check_response(response: aiohttp.ClientResponse, path: Optional[str] = None):  # noqa: ANN202
+    if not response.ok:
+        text = await response.text()
+        if path is not None:
+            msg = f"failed: {text}" if len(text) > 0 else "failed"
+            Config.logger.error(f"{path} {msg}")
+        else:
+            Config.logger.error(text)
+
+        if response.reason is not None and len(response.reason) == 0:
+            response.reason = text
+    response.raise_for_status()
+
+
+def send_to_dm_get(path: str, is_text: bool=True, conn_url: Optional[str]=None) -> Optional[Union[str, bytes]]:
+    host = Config.DM_HOST_PORT if conn_url is None else conn_url
+    assert host is not None, "dm host port not set"
+    response = requests.get(f"{host}{path}", headers=HEADERS)
+    __check_response(f"{host}{path}", response)
+    if response.status_code == HTTPStatus.NO_CONTENT:
+        return None
+    if is_text is True:
+        return response.text
+    elif is_text is False:
+        return response.json()
+    else:
+        return response.content
+
+
+async def send_to_dm_get_async(path: str, is_text: bool=True, conn_url: Optional[str]=None, async_session = None) -> Optional[Union[str, bytes]]:
+    host = Config.DM_HOST_PORT if conn_url is None else conn_url
+    assert host is not None, "dm host port not set"
+    async with async_session or aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=False), timeout=aiohttp.ClientTimeout(total=None)) as session:
+        async with session.get(f"{host}{path}", headers=HEADERS) as response:
+            await __async_check_response(response, f"{host}{path}")
+            if response.status == HTTPStatus.NO_CONTENT:
+                return None
+            if is_text is True:
+                return await response.text()
+            elif is_text is False:
+                return await response.json()
+            else:
+                return await response.read()
+
+
+def send_to_dm_post(path: str, operation: Optional[Any] = None, conn_url: Optional[str]=None) -> Optional[str]:  # noqa: ANN401
+    host = Config.DM_HOST_PORT if conn_url is None else conn_url
+    assert host is not None, "dm host port not set"
+    response = requests.post(f"{host}{path}", data=operation, headers=HEADERS)
+    __check_response(f"{host}{path}", response)
+    if response.status_code == HTTPStatus.NO_CONTENT:
+        return None
+    return response.text
+
+
+async def send_to_dm_post_async(path: str, operation: Optional[Any] = None, conn_url: Optional[str]=None, async_session=None) -> str:  # noqa: ANN401
+    host = Config.DM_HOST_PORT if conn_url is None else conn_url
+    assert host is not None, "dm host port not set"
+    async with async_session or aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=False), timeout=aiohttp.ClientTimeout(total=None)) as session:
+        async with session.post(f"{host}{path}", data=operation, headers=HEADERS) as response:
+            await __async_check_response(response, f"{host}{path}")
+            if response.status == HTTPStatus.NO_CONTENT:
+                return None
+            return response.text()
+
+
+def send_to_dm_stream(path: str, conn_url: Optional[str]=None) -> Iterable:
+    host = Config.DM_HOST_PORT if conn_url is None else conn_url
+    assert host is not None, "dm host port not set"
+
+    def stream_generator() -> None:
+        with requests.get(f"{host}{path}", headers=HEADERS, stream=True) as response:
+            __check_response(f"{host}{path}", response)
+            if response.status_code == HTTPStatus.NO_CONTENT:
+                return
+            for chunk in response.iter_content(chunk_size=4096):
+                if chunk:
+                    yield chunk.decode("utf-8", errors="ignore")
+    return stream_generator()
+
+
+async def send_to_dm_stream_async(path: str, conn_url: Optional[str]=None, async_session = None) -> AsyncIterable:
+    host = Config.DM_HOST_PORT if conn_url is None else conn_url
+    assert host is not None, "dm host port not set"
+
+    async def stream_generator() -> None:
+        async with async_session or aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=False), timeout=aiohttp.ClientTimeout(total=None)) as session:
+            async with session.get(f"{host}{path}", headers=HEADERS) as response:
+                await __async_check_response(response, f"{host}{path}")
+                if response.status == HTTPStatus.NO_CONTENT:
+                    return
+                async for chunk in response.content.iter_any():
+                    yield chunk.decode("utf-8", errors="ignore")
+    return stream_generator()

malevich_coretools/funcs/funcs.py

@@ -124,12 +124,18 @@ async def get_collections_groupName_async(name: str, operation_id: str, run_id:
     return model_from_json(await send_to_core_get_async(COLLECTIONS_GROUP_NAME(name, operation_id, run_id), *args, **kwargs), ResultCollections)
 
 
-def get_collections_id(id: str, offset: int, limit: int, *args, **kwargs) -> ResultCollection:
-    return model_from_json(send_to_core_get(COLLECTIONS_ID(id, offset, limit), *args, **kwargs), ResultCollection)
+def get_collections_id(id: str, offset: int, limit: int, raw: bool, *args, **kwargs) -> Union[ResultCollection, Alias.Json]:
+    res = send_to_core_get(COLLECTIONS_ID(id, offset, limit, raw), *args, is_text=raw, **kwargs)
+    if not raw:
+        res = model_from_json(res, ResultCollection)
+    return res
 
 
-async def get_collections_id_async(id: str, offset: int, limit: int, *args, **kwargs) -> ResultCollection:
-    return model_from_json(await send_to_core_get_async(COLLECTIONS_ID(id, offset, limit), *args, **kwargs), ResultCollection)
+async def get_collections_id_async(id: str, offset: int, limit: int, raw: bool, *args, **kwargs) -> Union[ResultCollection, Alias.Json]:
+    res = await send_to_core_get_async(COLLECTIONS_ID(id, offset, limit, raw), *args, is_text=raw, **kwargs)
+    if not raw:
+        res = model_from_json(res, ResultCollection)
+    return res
 
 
 def post_collections(data: DocsCollection, wait: bool, *args, **kwargs) -> Alias.Id:
@@ -1280,26 +1286,24 @@ async def post_manager_task_async(data: MainTask, with_show: bool, long: bool, l
     return AppLogs.model_validate_json(res)
 
 
-def post_manager_task_run(data: RunTask, with_show: bool, long: bool, long_timeout: int, wait: bool, auth: Optional[AUTH], conn_url: Optional[str]=None, *args, **kwargs) -> Optional[Union[Alias.Id, AppLogs]]:
+def post_manager_task_run(data: RunTask, with_show: bool, long: bool, long_timeout: int, wait: bool, auth: Optional[AUTH], conn_url: Optional[str]=None, *args, **kwargs) -> Union[Alias.Id, AppLogs]:
     check_profile_mode(data.profileMode)
     res = send_to_core_modify(MANAGER_TASK_RUN(wait and not long), data, with_show=with_show, show_func=show_logs_func, auth=auth, conn_url=conn_url, *args, **kwargs)
     if wait and long:
         res = asyncio.run(__get_result(res, timeout=long_timeout, auth=auth))
-    if data.withLogs or data.schedule is not None:
-        if not wait:
-            return res
-        return AppLogs.model_validate_json(res)
+    if not wait:
+        return res
+    return AppLogs.model_validate_json(res)
 
 
-async def post_manager_task_run_async(data: RunTask, with_show: bool, long: bool, long_timeout: int, wait: bool, auth: Optional[AUTH], conn_url: Optional[str]=None, *args, **kwargs) -> Optional[Union[Alias.Id, AppLogs]]:
+async def post_manager_task_run_async(data: RunTask, with_show: bool, long: bool, long_timeout: int, wait: bool, auth: Optional[AUTH], conn_url: Optional[str]=None, *args, **kwargs) -> Union[Alias.Id, AppLogs]:
     check_profile_mode(data.profileMode)
     res = await send_to_core_modify_async(MANAGER_TASK_RUN(wait and not long), data, with_show=with_show, show_func=show_logs_func, auth=auth, conn_url=conn_url, *args, **kwargs)
     if wait and long:
         res = await __get_result(res, timeout=long_timeout, auth=auth)
-    if data.withLogs or data.schedule is not None:
-        if not wait:
-            return res
-        return AppLogs.model_validate_json(res)
+    if not wait:
+        return res
+    return AppLogs.model_validate_json(res)
 
 
 def post_manager_pipeline(data: MainPipeline, with_show: bool, long: bool, long_timeout: int, return_response: bool, wait: bool, auth: Optional[AUTH], conn_url: Optional[str]=None, *args, **kwargs) -> Union[Alias.Id, AppLogs]:

malevich_coretools/secondary/config.py

@@ -8,6 +8,7 @@ class Config:
     TOKEN = os.environ.get("GITLAB_ACCESS_TOKEN")   # FIXME
 
     HOST_PORT = None
+    DM_HOST_PORT = None
     KAFKA_HOST_PORT = None
     CORE_USERNAME = None
     CORE_PASSWORD = None

malevich_coretools/secondary/const.py

@@ -50,7 +50,7 @@ COLLECTIONS_IDS_NAME = lambda name, operation_id, run_id: with_key_values(f"{COL
 COLLECTIONS_NAME = lambda name, operation_id, run_id, offset, limit: with_key_values(f"{COLLECTIONS_MAIN}/name/{urllib.parse.quote(str(name), safe='')}", {"operationId": operation_id, "runId": run_id, "offset": offset, "limit": limit})
 COLLECTIONS_IDS_GROUP_NAME = lambda name, operation_id, run_id: with_key_values(f"{COLLECTIONS_MAIN}/ids/groupName/{urllib.parse.quote(str(name), safe='')}", {"operationId": operation_id, "runId": run_id})
 COLLECTIONS_GROUP_NAME = lambda name, operation_id, run_id: with_key_values(f"{COLLECTIONS_MAIN}/groupName/{urllib.parse.quote(str(name), safe='')}", {"operationId": operation_id, "runId": run_id})
-COLLECTIONS_ID = lambda id, offset, limit: with_key_values(f"{COLLECTIONS_MAIN}/{urllib.parse.quote(str(id), safe='')}", {"offset": offset, "limit": limit})
+COLLECTIONS_ID = lambda id, offset, limit, raw: with_key_values(f"{COLLECTIONS_MAIN}/{urllib.parse.quote(str(id), safe='')}", {"offset": offset, "limit": limit, "raw": raw})
 COLLECTIONS_ID_MODIFY = lambda id, wait: with_wait(f"{COLLECTIONS_MAIN}/{urllib.parse.quote(str(id), safe='')}", wait)
 COLLECTIONS_ID_S3 = lambda id, wait: with_wait(f"{COLLECTIONS_MAIN}/s3/{urllib.parse.quote(str(id), safe='')}", wait)
 COLLECTIONS_ID_ADD = lambda id, wait: with_wait(f"{COLLECTIONS_MAIN}/{urllib.parse.quote(str(id), safe='')}/add", wait)
@@ -239,3 +239,10 @@ KAFKA_SEND = f"{MANAGER_MAIN}/kafkaMsg"
 ## BatchController
 BATCH_MAIN = f"{API_VERSION}/batch"
 BATCH = BATCH_MAIN
+
+##### DM
+DM_STREAM = lambda operationId, runId, bindId : f"stream/{operationId}/{runId}/{bindId}"
+DM_CONTINUE = lambda operationId, runId, id: f"continue/{operationId}/{runId}/{id}"
+DM_STATE = lambda operationId, runId, bindId, key, index : with_key_values(f"state/{operationId}/{runId}/{bindId}", {"key": key, "index": index})
+DM_JOURNAL_LIST = lambda operationId, runId : f"journal/{operationId}/{runId}"
+DM_JOURNAL = lambda operationId, runId, key, stream : with_key_values(f"journal/{operationId}/{runId}/{key}", {"stream": stream})

malevich_coretools/utils.py

@@ -47,15 +47,24 @@ def set_host_port(host_port: str) -> None:
     Config.HOST_PORT = host_port
 
 
+def set_dm_host_port(host_port: str) -> None:
+    """update host and port for malevich-dm, example: `http://localhost:8000/` """
+    assert len(host_port) > 0, "empty host port"
+    host_port = host_port if host_port[-1] == "/" else f"{host_port}/"
+    Config.DM_HOST_PORT = host_port
+
+
 def set_kafka_host_port(host_port: str) -> None:
     """update kafka host and port for malevich-kafka, example: `localhost:9092` """
     assert len(host_port) > 0, "empty host port"
     Config.KAFKA_HOST_PORT = host_port
 
 
-def set_conn_url(conn_url: str) -> None:
+def set_conn_url(conn_url: str, dm_url: Optional[str] = None) -> None:
     """analogue set_host_port; update `conn_url` for malevich-core, example: `http://localhost:8080/` """
     set_host_port(conn_url)
+    if dm_url is not None:
+        set_dm_host_port(dm_url)
 
 
 def set_verbose(verbose: bool) -> None:
@@ -609,6 +618,7 @@ def get_collection(
     auth: Optional[AUTH] = None,
     conn_url: Optional[str] = None,
     batcher: Optional[Batcher] = None,
+    raw: Literal[False] = False,
     is_async: Literal[False] = False,
 ) -> ResultCollection:
     pass
@@ -623,11 +633,28 @@ def get_collection(
     auth: Optional[AUTH] = None,
     conn_url: Optional[str] = None,
     batcher: Optional[Batcher] = None,
+    raw: Literal[False] = False,
     is_async: Literal[True],
 ) -> Coroutine[Any, Any, ResultCollection]:
     pass
 
 
+@overload
+def get_collection(
+    id: str,
+    offset: int = 0,
+    limit: int = -1,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+    raw: Literal[True],
+    is_async: Literal[False] = False,
+) -> Alias.Json:
+    pass
+
+
+@overload
 def get_collection(
     id: str,
     offset: int = 0,
@@ -636,16 +663,32 @@ def get_collection(
     auth: Optional[AUTH] = None,
     conn_url: Optional[str] = None,
     batcher: Optional[Batcher] = None,
+    raw: Literal[True],
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, Alias.Json]:
+    pass
+
+
+def get_collection(
+    id: str,
+    offset: int = 0,
+    limit: int = -1,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+    batcher: Optional[Batcher] = None,
+    raw: bool = False,
     is_async: bool = False,
-) -> Union[ResultCollection, Coroutine[Any, Any, ResultCollection]]:
+) -> Union[ResultCollection, Alias.Json, Coroutine[Any, Any, ResultCollection], Coroutine[Any, Any, Alias.Json]]:
     """return collection by `id`, pagination: unlimited - `limit` < 0"""
     if batcher is None:
         batcher = Config.BATCHER
     if batcher is not None:
-        return batcher.add("getCollectionById", vars={"id": id, "offset": offset, "limit": limit}, result_model=ResultCollection)
+        result_model = None if raw else ResultCollection
+        return batcher.add("getCollectionById", vars={"id": id, "offset": offset, "limit": limit, "raw": raw}, result_model=result_model)
     if is_async:
-        return f.get_collections_id_async(id, offset, limit, auth=auth, conn_url=conn_url)
-    return f.get_collections_id(id, offset, limit, auth=auth, conn_url=conn_url)
+        return f.get_collections_id_async(id, offset, limit, raw, auth=auth, conn_url=conn_url)
+    return f.get_collections_id(id, offset, limit, raw, auth=auth, conn_url=conn_url)
 
 
 @overload
@@ -7161,7 +7204,7 @@ def task_run(
     conn_url: Optional[str] = None,
     batcher: Optional[Batcher] = None,
     is_async: Literal[False] = False,
-) -> Optional[Union[Alias.Id, AppLogs]]:
+) -> Union[Alias.Id, AppLogs]:
     pass
 
 
@@ -7186,7 +7229,7 @@ def task_run(
     conn_url: Optional[str] = None,
     batcher: Optional[Batcher] = None,
     is_async: Literal[True],
-) -> Coroutine[Any, Any, Optional[Union[Alias.Id, AppLogs]]]:
+) -> Coroutine[Any, Any, Union[Alias.Id, AppLogs]]:
     pass
 
 
@@ -7210,7 +7253,7 @@ def task_run(
     conn_url: Optional[str] = None,
     batcher: Optional[Batcher] = None,
     is_async: bool = False,
-) -> Union[Optional[Union[Alias.Id, AppLogs]], Coroutine[Any, Any, Optional[Union[Alias.Id, AppLogs]]]]:
+) -> Union[Union[Alias.Id, AppLogs], Coroutine[Any, Any, Union[Alias.Id, AppLogs]]]:
     """run prepared task by `operation_id` with `cfg_id` and other overridden parameters
 
     Args:

{malevich_coretools-0.3.66.dist-info → malevich_coretools-0.3.68.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: malevich-coretools
-Version: 0.3.66
+Version: 0.3.68
 Author: Andrew Pogrebnoj
 Author-email: andrew@onjulius.co
 License-File: LICENSE

{malevich_coretools-0.3.66.dist-info → malevich_coretools-0.3.68.dist-info}/RECORD

@@ -1,7 +1,8 @@
-malevich_coretools/__init__.py,sha256=DJtPESxkCZD2SbTZTrR_x0TKDQ4MJpmBqGw5YpKYidM,134
-malevich_coretools/utils.py,sha256=nc88BxU6gbq5ANiYt3XgRp6YzMFJpFrOWnCtdWnYUBY,276102
+malevich_coretools/__init__.py,sha256=0WUBUhATgnU3tctwv0AxzvRr0zRlW6V5INQljBCLRzo,172
+malevich_coretools/dm_utils.py,sha256=WvjtqVaAiwahej-oMnB5JJKFiAcTRiyNStDu0WpifHs,4775
+malevich_coretools/utils.py,sha256=imtU54FZ1poleLROQ-b20rynXlCEudYK0B_tnjKWoS0,277240
 malevich_coretools/abstract/__init__.py,sha256=6vQ08c8HPYyT_pPkKlc-EwQKE8xG3HTEo2p_GiI5rik,142
-malevich_coretools/abstract/abstract.py,sha256=QJMMzW98agH8pYWNjLPvshXC5jK3zAEyqJ0oK4ZfZ34,17695
+malevich_coretools/abstract/abstract.py,sha256=SSmGmn2U4hBV3-e1Fxi1SsfgddB4TTE4_VFRR6VMP7g,17732
 malevich_coretools/abstract/operations.py,sha256=cWlo2xzW-rzkTInzpDjBYeL68KfLYqSpZJRzCQ4OzjA,3070
 malevich_coretools/abstract/pipeline.py,sha256=yjeDaU-i2yNqnQTSwe9Kdv44dzf-p5AWk_qXxvoYvhM,7530
 malevich_coretools/abstract/statuses.py,sha256=9ISSw_evsylBshLXoU44TCoFOrZm4bXIxyAFFDqdUWc,333
@@ -11,18 +12,19 @@ malevich_coretools/batch/__init__.py,sha256=taxyZl8YOZd2EBd3leN6slzMkejUtjQ64Na3
 malevich_coretools/batch/utils.py,sha256=FRmCYU-zr-RjgT1Mo3CUNcB2mW1t_gKCJazcMx6aIW4,7719
 malevich_coretools/funcs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 malevich_coretools/funcs/checks.py,sha256=Q5pRtRevQrGv_-SMbn2GgYnulhclDLBXdRtbw2QOYKU,223
-malevich_coretools/funcs/funcs.py,sha256=gG-vAXigztV15451X9Glx4XIn2wo19JHBWllkKyocVk,85942
+malevich_coretools/funcs/dm_funcs.py,sha256=Z0oSzMb9x6N2yANbIICz6erF9fJ5drg3Zheh-_VTPx4,7812
+malevich_coretools/funcs/funcs.py,sha256=JcRnt-wiOBOz-xt463Jv5ma8DmXXnR3R_5ALXWZAoZY,85988
 malevich_coretools/funcs/helpers.py,sha256=nYbUdtAuSSa9VMr7Oy2y0yvEMLv9EI1jzGq6eynuNLU,13573
 malevich_coretools/secondary/__init__.py,sha256=048HqvG36_1WdDVZK_RuECmaf14Iq2fviUysG1inlaE,78
-malevich_coretools/secondary/config.py,sha256=hRlSJuPQnhKyt1wmOAJX_XmcliaO0fPGbW94AE_Mazs,463
-malevich_coretools/secondary/const.py,sha256=1x-49VxzVdtk064xn92gQnqlEfMuYAvWcz-vp3D6fO4,15369
+malevich_coretools/secondary/config.py,sha256=cjqKiWLm6m1kArOq4DWOHaNxKT_kHP9WUyHVkYe3UeI,487
+malevich_coretools/secondary/const.py,sha256=yS--jwK01b98f2heACJfbPzCE2rItyPSX2PFIoQIy6M,15924
 malevich_coretools/secondary/helpers.py,sha256=V5xNv-Rt4SNkthTcNnMtYPjiYfoHmwUR8ApU8qFmzT0,7986
 malevich_coretools/secondary/kafka_utils.py,sha256=SIUnBFyfwsquN6MAUrEkKCw-1l7979Znl7OTQSX2UKo,989
 malevich_coretools/tools/__init__.py,sha256=jDxlCa5Dr6Y43qlI7JwsRAlBkKmFeTHTEnjNUvu-0iw,46
 malevich_coretools/tools/abstract.py,sha256=B1RW1FeNHrQ6r1k-cQZ4k4noCRXkIGt-JUwVoXEDkAg,4466
 malevich_coretools/tools/vast.py,sha256=63tvy70qQV9vnK0eWytlgjBGSnfA7l3kSIDgACBbMMs,12893
-malevich_coretools-0.3.66.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-malevich_coretools-0.3.66.dist-info/METADATA,sha256=4_no2l1oNwKjuRKfjp2nPxCIJxJq32_7NNmGV3HNYoU,347
-malevich_coretools-0.3.66.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-malevich_coretools-0.3.66.dist-info/top_level.txt,sha256=wDX3s1Tso0otBPNrFRfXqyNpm48W4Bp5v6JfbITO2Z8,19
-malevich_coretools-0.3.66.dist-info/RECORD,,
+malevich_coretools-0.3.68.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+malevich_coretools-0.3.68.dist-info/METADATA,sha256=gaBdEDfWy1DOC6l_7GlMOiIMOP_ntOBjn7yMhYmbccg,347
+malevich_coretools-0.3.68.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+malevich_coretools-0.3.68.dist-info/top_level.txt,sha256=wDX3s1Tso0otBPNrFRfXqyNpm48W4Bp5v6JfbITO2Z8,19
+malevich_coretools-0.3.68.dist-info/RECORD,,