malevich-coretools 0.3.67__py3-none-any.whl → 0.3.68__py3-none-any.whl

malevich_coretools/__init__.py

@@ -1,3 +1,4 @@
+from .dm_utils import *  # noqa: F403
 from .secondary import logs_streaming  # noqa: F401
 from .tools import vast_settings  # noqa: F401
 from .utils import *  # noqa: F403

malevich_coretools/dm_utils.py

@@ -0,0 +1,233 @@
+from typing import (
+    Any,
+    AsyncIterable,
+    Coroutine,
+    Iterable,
+    List,
+    Literal,
+    Optional,
+    Union,
+    overload,
+)
+
+import malevich_coretools.funcs.dm_funcs as f
+
+
+@overload
+def dm_stream(
+    operation_id: str,
+    run_id: str,
+    bind_id: str,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[False] = False,
+) -> Iterable:
+    pass
+
+
+@overload
+def dm_stream(
+    operation_id: str,
+    run_id: str,
+    bind_id: str,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, AsyncIterable]:
+    pass
+
+
+def dm_stream(
+    operation_id: str,
+    run_id: str,
+    bind_id: str,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: bool = False,
+) -> Union[Iterable, Coroutine[Any, Any, AsyncIterable]]:
+    if is_async:
+        return f.dm_stream_async(operation_id, run_id, bind_id, conn_url=conn_url)
+    return f.dm_stream(operation_id, run_id, bind_id, conn_url=conn_url)
+
+
+@overload
+def dm_continue(
+    operation_id: str,
+    run_id: str,
+    id: str,
+    data: Any,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[False] = False,
+) -> None:
+    pass
+
+
+@overload
+def dm_continue(
+    operation_id: str,
+    run_id: str,
+    id: str,
+    data: Any,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, None]:
+    pass
+
+
+def dm_continue(
+    operation_id: str,
+    run_id: str,
+    id: str,
+    data: Any,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: bool = False,
+) -> Union[None, Coroutine[Any, Any, None]]:
+    if is_async:
+        return f.dm_continue_async(operation_id, run_id, id, data, conn_url=conn_url)
+    return f.dm_continue(operation_id, run_id, id, data, conn_url=conn_url)
+
+
+@overload
+def dm_state(
+    operation_id: str,
+    run_id: str,
+    bind_id: str,
+    key: Optional[str] = None,
+    index: Optional[int] = None,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[False] = False,
+) -> Any:
+    pass
+
+
+@overload
+def dm_state(
+    operation_id: str,
+    run_id: str,
+    bind_id: str,
+    key: Optional[str] = None,
+    index: Optional[int] = None,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, Any]:
+    pass
+
+
+def dm_state(
+    operation_id: str,
+    run_id: str,
+    bind_id: str,
+    key: Optional[str] = None,
+    index: Optional[int] = None,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: bool = False,
+) -> Union[Any, Coroutine[Any, Any, Any]]:
+    if is_async:
+        return f.dm_state_async(operation_id, run_id, bind_id, key, index, conn_url=conn_url)
+    return f.dm_state(operation_id, run_id, bind_id, key, index, conn_url=conn_url)
+
+
+@overload
+def dm_journal_list(
+    operation_id: str,
+    run_id: str,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[False] = False,
+) -> List[str]:
+    pass
+
+
+@overload
+def dm_journal_list(
+    operation_id: str,
+    run_id: str,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, List[str]]:
+    pass
+
+
+def dm_journal_list(
+    operation_id: str,
+    run_id: str,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: bool = False,
+) -> Union[List[str], Coroutine[Any, Any, List[str]]]:
+    if is_async:
+        return f.dm_journal_list_async(operation_id, run_id, conn_url=conn_url)
+    return f.dm_journal_list(operation_id, run_id, conn_url=conn_url)
+
+
+@overload
+def dm_journal(
+    operation_id: str,
+    run_id: str,
+    key: str,
+    is_stream: Literal[False] = False,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[False] = False,
+) -> Any:
+    pass
+
+
+@overload
+def dm_journal(
+    operation_id: str,
+    run_id: str,
+    key: str,
+    is_stream: Literal[False] = False,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, Any]:
+    pass
+
+
+@overload
+def dm_journal(
+    operation_id: str,
+    run_id: str,
+    key: str,
+    is_stream: Literal[True],
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[False] = False,
+) -> Iterable:
+    pass
+
+
+@overload
+def dm_journal(
+    operation_id: str,
+    run_id: str,
+    key: str,
+    is_stream: Literal[True],
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, AsyncIterable]:
+    pass
+
+
+def dm_journal(
+    operation_id: str,
+    run_id: str,
+    key: str,
+    is_stream: bool = False,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: bool = False,
+) -> Union[Any, Iterable, Coroutine[Any, Any, Union[Any, AsyncIterable]]]:
+    if is_async:
+        return f.dm_journal_async(operation_id, run_id, key, is_stream, conn_url=conn_url)
+    return f.dm_journal(operation_id, run_id, key, is_stream, conn_url=conn_url)

malevich_coretools/funcs/dm_funcs.py

@@ -0,0 +1,162 @@
+from http import HTTPStatus
+from typing import Any, AsyncIterable, Iterable, List, Optional, Union
+
+import aiohttp
+import requests
+from requests.models import Response
+
+from malevich_coretools.secondary import Config
+from malevich_coretools.secondary.const import *  # noqa: F403
+
+
+def dm_stream(operation_id: str, run_id: str, bind_id: str, conn_url: Optional[str]=None) -> Iterable:
+    return send_to_dm_stream(DM_STREAM(operation_id, run_id, bind_id), conn_url=conn_url)
+
+
+async def dm_stream_async(operation_id: str, run_id: str, bind_id: str, conn_url: Optional[str]=None) -> AsyncIterable:
+    return await send_to_dm_stream_async(DM_STREAM(operation_id, run_id, bind_id), conn_url=conn_url)
+
+
+def dm_continue(operation_id: str, run_id: str, id: str, data: Any, conn_url: Optional[str]=None) -> None:
+    return send_to_dm_post(DM_CONTINUE(operation_id, run_id, id), data, conn_url=conn_url)
+
+
+async def dm_continue_async(operation_id: str, run_id: str, id: str, data: Any, conn_url: Optional[str]=None) -> None:
+    return await send_to_dm_post_async(DM_CONTINUE(operation_id, run_id, id), data, conn_url=conn_url)
+
+
+def dm_state(operation_id: str, run_id: str, bind_id: str, key: Optional[str] = None, index: Optional[int] = None, conn_url: Optional[str]=None) -> Any:
+    return send_to_dm_get(DM_STATE(operation_id, run_id, bind_id, key, index), conn_url=conn_url)
+
+
+async def dm_state_async(operation_id: str, run_id: str, bind_id: str, key: Optional[str] = None, index: Optional[int] = None, conn_url: Optional[str]=None) -> Any:
+    return await send_to_dm_get_async(DM_STATE(operation_id, run_id, bind_id, key, index), conn_url=conn_url)
+
+
+def dm_journal_list(operation_id: str, run_id: str, conn_url: Optional[str]=None) -> List[str]:
+    return send_to_dm_get(DM_JOURNAL_LIST(operation_id, run_id), is_text=False, conn_url=conn_url)
+
+
+async def dm_journal_list_async(operation_id: str, run_id: str, conn_url: Optional[str]=None) -> List[str]:
+    return await send_to_dm_get_async(DM_JOURNAL_LIST(operation_id, run_id), is_text=False, conn_url=conn_url)
+
+
+def dm_journal(operation_id: str, run_id: str, key: str, is_stream: bool, conn_url: Optional[str]=None) -> Union[Any, Iterable]:
+    if is_stream:
+        return send_to_dm_stream(DM_JOURNAL(operation_id, run_id, key, is_stream), conn_url=conn_url)
+    return send_to_dm_get(DM_JOURNAL(operation_id, run_id, key, is_stream), conn_url=conn_url)
+
+
+async def dm_journal_async(operation_id: str, run_id: str, key: str, is_stream: bool, conn_url: Optional[str]=None) -> Union[Any, AsyncIterable]:
+    if is_stream:
+        return await send_to_dm_stream_async(DM_JOURNAL(operation_id, run_id, key, is_stream), conn_url=conn_url)
+    return await send_to_dm_get_async(DM_JOURNAL(operation_id, run_id, key, is_stream), conn_url=conn_url)
+
+
+#
+
+def __check_response(path: str, response: Response):  # noqa: ANN202
+    if response.status_code >= 400:
+        text = response.text
+        msg = f"failed: {text}" if len(text) > 0 else "failed"
+        Config.logger.error(f"{path} {msg}")
+
+        if response.reason is not None and len(response.reason) == 0:
+            response.reason = text
+    response.raise_for_status()
+
+
+async def __async_check_response(response: aiohttp.ClientResponse, path: Optional[str] = None):  # noqa: ANN202
+    if not response.ok:
+        text = await response.text()
+        if path is not None:
+            msg = f"failed: {text}" if len(text) > 0 else "failed"
+            Config.logger.error(f"{path} {msg}")
+        else:
+            Config.logger.error(text)
+
+        if response.reason is not None and len(response.reason) == 0:
+            response.reason = text
+    response.raise_for_status()
+
+
+def send_to_dm_get(path: str, is_text: bool=True, conn_url: Optional[str]=None) -> Optional[Union[str, bytes]]:
+    host = Config.DM_HOST_PORT if conn_url is None else conn_url
+    assert host is not None, "dm host port not set"
+    response = requests.get(f"{host}{path}", headers=HEADERS)
+    __check_response(f"{host}{path}", response)
+    if response.status_code == HTTPStatus.NO_CONTENT:
+        return None
+    if is_text is True:
+        return response.text
+    elif is_text is False:
+        return response.json()
+    else:
+        return response.content
+
+
+async def send_to_dm_get_async(path: str, is_text: bool=True, conn_url: Optional[str]=None, async_session = None) -> Optional[Union[str, bytes]]:
+    host = Config.DM_HOST_PORT if conn_url is None else conn_url
+    assert host is not None, "dm host port not set"
+    async with async_session or aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=False), timeout=aiohttp.ClientTimeout(total=None)) as session:
+        async with session.get(f"{host}{path}", headers=HEADERS) as response:
+            await __async_check_response(response, f"{host}{path}")
+            if response.status == HTTPStatus.NO_CONTENT:
+                return None
+            if is_text is True:
+                return await response.text()
+            elif is_text is False:
+                return await response.json()
+            else:
+                return await response.read()
+
+
+def send_to_dm_post(path: str, operation: Optional[Any] = None, conn_url: Optional[str]=None) -> Optional[str]:  # noqa: ANN401
+    host = Config.DM_HOST_PORT if conn_url is None else conn_url
+    assert host is not None, "dm host port not set"
+    response = requests.post(f"{host}{path}", data=operation, headers=HEADERS)
+    __check_response(f"{host}{path}", response)
+    if response.status_code == HTTPStatus.NO_CONTENT:
+        return None
+    return response.text
+
+
+async def send_to_dm_post_async(path: str, operation: Optional[Any] = None, conn_url: Optional[str]=None, async_session=None) -> str:  # noqa: ANN401
+    host = Config.DM_HOST_PORT if conn_url is None else conn_url
+    assert host is not None, "dm host port not set"
+    async with async_session or aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=False), timeout=aiohttp.ClientTimeout(total=None)) as session:
+        async with session.post(f"{host}{path}", data=operation, headers=HEADERS) as response:
+            await __async_check_response(response, f"{host}{path}")
+            if response.status == HTTPStatus.NO_CONTENT:
+                return None
+            return response.text()
+
+
+def send_to_dm_stream(path: str, conn_url: Optional[str]=None) -> Iterable:
+    host = Config.DM_HOST_PORT if conn_url is None else conn_url
+    assert host is not None, "dm host port not set"
+
+    def stream_generator() -> None:
+        with requests.get(f"{host}{path}", headers=HEADERS, stream=True) as response:
+            __check_response(f"{host}{path}", response)
+            if response.status_code == HTTPStatus.NO_CONTENT:
+                return
+            for chunk in response.iter_content(chunk_size=4096):
+                if chunk:
+                    yield chunk.decode("utf-8", errors="ignore")
+    return stream_generator()
+
+
+async def send_to_dm_stream_async(path: str, conn_url: Optional[str]=None, async_session = None) -> AsyncIterable:
+    host = Config.DM_HOST_PORT if conn_url is None else conn_url
+    assert host is not None, "dm host port not set"
+
+    async def stream_generator() -> None:
+        async with async_session or aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=False), timeout=aiohttp.ClientTimeout(total=None)) as session:
+            async with session.get(f"{host}{path}", headers=HEADERS) as response:
+                await __async_check_response(response, f"{host}{path}")
+                if response.status == HTTPStatus.NO_CONTENT:
+                    return
+                async for chunk in response.content.iter_any():
+                    yield chunk.decode("utf-8", errors="ignore")
+    return stream_generator()

malevich_coretools/secondary/config.py

@@ -8,6 +8,7 @@ class Config:
     TOKEN = os.environ.get("GITLAB_ACCESS_TOKEN")   # FIXME
 
     HOST_PORT = None
+    DM_HOST_PORT = None
     KAFKA_HOST_PORT = None
     CORE_USERNAME = None
     CORE_PASSWORD = None

malevich_coretools/secondary/const.py

@@ -239,3 +239,10 @@ KAFKA_SEND = f"{MANAGER_MAIN}/kafkaMsg"
 ## BatchController
 BATCH_MAIN = f"{API_VERSION}/batch"
 BATCH = BATCH_MAIN
+
+##### DM
+DM_STREAM = lambda operationId, runId, bindId : f"stream/{operationId}/{runId}/{bindId}"
+DM_CONTINUE = lambda operationId, runId, id: f"continue/{operationId}/{runId}/{id}"
+DM_STATE = lambda operationId, runId, bindId, key, index : with_key_values(f"state/{operationId}/{runId}/{bindId}", {"key": key, "index": index})
+DM_JOURNAL_LIST = lambda operationId, runId : f"journal/{operationId}/{runId}"
+DM_JOURNAL = lambda operationId, runId, key, stream : with_key_values(f"journal/{operationId}/{runId}/{key}", {"stream": stream})

malevich_coretools/utils.py

@@ -47,15 +47,24 @@ def set_host_port(host_port: str) -> None:
     Config.HOST_PORT = host_port
 
 
+def set_dm_host_port(host_port: str) -> None:
+    """update host and port for malevich-dm, example: `http://localhost:8000/` """
+    assert len(host_port) > 0, "empty host port"
+    host_port = host_port if host_port[-1] == "/" else f"{host_port}/"
+    Config.DM_HOST_PORT = host_port
+
+
 def set_kafka_host_port(host_port: str) -> None:
     """update kafka host and port for malevich-kafka, example: `localhost:9092` """
     assert len(host_port) > 0, "empty host port"
     Config.KAFKA_HOST_PORT = host_port
 
 
-def set_conn_url(conn_url: str) -> None:
+def set_conn_url(conn_url: str, dm_url: Optional[str] = None) -> None:
     """analogue set_host_port; update `conn_url` for malevich-core, example: `http://localhost:8080/` """
     set_host_port(conn_url)
+    if dm_url is not None:
+        set_dm_host_port(dm_url)
 
 
 def set_verbose(verbose: bool) -> None:

{malevich_coretools-0.3.67.dist-info → malevich_coretools-0.3.68.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: malevich-coretools
-Version: 0.3.67
+Version: 0.3.68
 Author: Andrew Pogrebnoj
 Author-email: andrew@onjulius.co
 License-File: LICENSE

{malevich_coretools-0.3.67.dist-info → malevich_coretools-0.3.68.dist-info}/RECORD

@@ -1,5 +1,6 @@
-malevich_coretools/__init__.py,sha256=DJtPESxkCZD2SbTZTrR_x0TKDQ4MJpmBqGw5YpKYidM,134
-malevich_coretools/utils.py,sha256=o_uH_HJKs4od0ExFmG4LMwZ1pH4V6_90QYxn_sgVyC8,276863
+malevich_coretools/__init__.py,sha256=0WUBUhATgnU3tctwv0AxzvRr0zRlW6V5INQljBCLRzo,172
+malevich_coretools/dm_utils.py,sha256=WvjtqVaAiwahej-oMnB5JJKFiAcTRiyNStDu0WpifHs,4775
+malevich_coretools/utils.py,sha256=imtU54FZ1poleLROQ-b20rynXlCEudYK0B_tnjKWoS0,277240
 malevich_coretools/abstract/__init__.py,sha256=6vQ08c8HPYyT_pPkKlc-EwQKE8xG3HTEo2p_GiI5rik,142
 malevich_coretools/abstract/abstract.py,sha256=SSmGmn2U4hBV3-e1Fxi1SsfgddB4TTE4_VFRR6VMP7g,17732
 malevich_coretools/abstract/operations.py,sha256=cWlo2xzW-rzkTInzpDjBYeL68KfLYqSpZJRzCQ4OzjA,3070
@@ -11,18 +12,19 @@ malevich_coretools/batch/__init__.py,sha256=taxyZl8YOZd2EBd3leN6slzMkejUtjQ64Na3
 malevich_coretools/batch/utils.py,sha256=FRmCYU-zr-RjgT1Mo3CUNcB2mW1t_gKCJazcMx6aIW4,7719
 malevich_coretools/funcs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 malevich_coretools/funcs/checks.py,sha256=Q5pRtRevQrGv_-SMbn2GgYnulhclDLBXdRtbw2QOYKU,223
+malevich_coretools/funcs/dm_funcs.py,sha256=Z0oSzMb9x6N2yANbIICz6erF9fJ5drg3Zheh-_VTPx4,7812
 malevich_coretools/funcs/funcs.py,sha256=JcRnt-wiOBOz-xt463Jv5ma8DmXXnR3R_5ALXWZAoZY,85988
 malevich_coretools/funcs/helpers.py,sha256=nYbUdtAuSSa9VMr7Oy2y0yvEMLv9EI1jzGq6eynuNLU,13573
 malevich_coretools/secondary/__init__.py,sha256=048HqvG36_1WdDVZK_RuECmaf14Iq2fviUysG1inlaE,78
-malevich_coretools/secondary/config.py,sha256=hRlSJuPQnhKyt1wmOAJX_XmcliaO0fPGbW94AE_Mazs,463
-malevich_coretools/secondary/const.py,sha256=rZ8d2tmBmiDEk5zhLKMe6mZRnwMvFrquJeK0RlTNzWw,15386
+malevich_coretools/secondary/config.py,sha256=cjqKiWLm6m1kArOq4DWOHaNxKT_kHP9WUyHVkYe3UeI,487
+malevich_coretools/secondary/const.py,sha256=yS--jwK01b98f2heACJfbPzCE2rItyPSX2PFIoQIy6M,15924
 malevich_coretools/secondary/helpers.py,sha256=V5xNv-Rt4SNkthTcNnMtYPjiYfoHmwUR8ApU8qFmzT0,7986
 malevich_coretools/secondary/kafka_utils.py,sha256=SIUnBFyfwsquN6MAUrEkKCw-1l7979Znl7OTQSX2UKo,989
 malevich_coretools/tools/__init__.py,sha256=jDxlCa5Dr6Y43qlI7JwsRAlBkKmFeTHTEnjNUvu-0iw,46
 malevich_coretools/tools/abstract.py,sha256=B1RW1FeNHrQ6r1k-cQZ4k4noCRXkIGt-JUwVoXEDkAg,4466
 malevich_coretools/tools/vast.py,sha256=63tvy70qQV9vnK0eWytlgjBGSnfA7l3kSIDgACBbMMs,12893
-malevich_coretools-0.3.67.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-malevich_coretools-0.3.67.dist-info/METADATA,sha256=NyxNf6rMc1Q-bscMz5eHBbtm1hhdtbfdRAjY4FHg47k,347
-malevich_coretools-0.3.67.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-malevich_coretools-0.3.67.dist-info/top_level.txt,sha256=wDX3s1Tso0otBPNrFRfXqyNpm48W4Bp5v6JfbITO2Z8,19
-malevich_coretools-0.3.67.dist-info/RECORD,,
+malevich_coretools-0.3.68.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+malevich_coretools-0.3.68.dist-info/METADATA,sha256=gaBdEDfWy1DOC6l_7GlMOiIMOP_ntOBjn7yMhYmbccg,347
+malevich_coretools-0.3.68.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+malevich_coretools-0.3.68.dist-info/top_level.txt,sha256=wDX3s1Tso0otBPNrFRfXqyNpm48W4Bp5v6JfbITO2Z8,19
+malevich_coretools-0.3.68.dist-info/RECORD,,