malevich-coretools 0.3.67__tar.gz → 0.3.68__tar.gz

{malevich_coretools-0.3.67/malevich_coretools.egg-info → malevich_coretools-0.3.68}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: malevich-coretools
-Version: 0.3.67
+Version: 0.3.68
 Author: Andrew Pogrebnoj
 Author-email: andrew@onjulius.co
 License-File: LICENSE

malevich_coretools-0.3.68/VERSION

@@ -0,0 +1 @@
+0.3.68

{malevich_coretools-0.3.67 → malevich_coretools-0.3.68}/malevich_coretools/__init__.py

@@ -1,3 +1,4 @@
+from .dm_utils import *  # noqa: F403
 from .secondary import logs_streaming  # noqa: F401
 from .tools import vast_settings  # noqa: F401
 from .utils import *  # noqa: F403

malevich_coretools-0.3.68/malevich_coretools/dm_utils.py

@@ -0,0 +1,233 @@
+from typing import (
+    Any,
+    AsyncIterable,
+    Coroutine,
+    Iterable,
+    List,
+    Literal,
+    Optional,
+    Union,
+    overload,
+)
+
+import malevich_coretools.funcs.dm_funcs as f
+
+
+@overload
+def dm_stream(
+    operation_id: str,
+    run_id: str,
+    bind_id: str,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[False] = False,
+) -> Iterable:
+    pass
+
+
+@overload
+def dm_stream(
+    operation_id: str,
+    run_id: str,
+    bind_id: str,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, AsyncIterable]:
+    pass
+
+
+def dm_stream(
+    operation_id: str,
+    run_id: str,
+    bind_id: str,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: bool = False,
+) -> Union[Iterable, Coroutine[Any, Any, AsyncIterable]]:
+    if is_async:
+        return f.dm_stream_async(operation_id, run_id, bind_id, conn_url=conn_url)
+    return f.dm_stream(operation_id, run_id, bind_id, conn_url=conn_url)
+
+
+@overload
+def dm_continue(
+    operation_id: str,
+    run_id: str,
+    id: str,
+    data: Any,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[False] = False,
+) -> None:
+    pass
+
+
+@overload
+def dm_continue(
+    operation_id: str,
+    run_id: str,
+    id: str,
+    data: Any,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, None]:
+    pass
+
+
+def dm_continue(
+    operation_id: str,
+    run_id: str,
+    id: str,
+    data: Any,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: bool = False,
+) -> Union[None, Coroutine[Any, Any, None]]:
+    if is_async:
+        return f.dm_continue_async(operation_id, run_id, id, data, conn_url=conn_url)
+    return f.dm_continue(operation_id, run_id, id, data, conn_url=conn_url)
+
+
+@overload
+def dm_state(
+    operation_id: str,
+    run_id: str,
+    bind_id: str,
+    key: Optional[str] = None,
+    index: Optional[int] = None,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[False] = False,
+) -> Any:
+    pass
+
+
+@overload
+def dm_state(
+    operation_id: str,
+    run_id: str,
+    bind_id: str,
+    key: Optional[str] = None,
+    index: Optional[int] = None,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, Any]:
+    pass
+
+
+def dm_state(
+    operation_id: str,
+    run_id: str,
+    bind_id: str,
+    key: Optional[str] = None,
+    index: Optional[int] = None,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: bool = False,
+) -> Union[Any, Coroutine[Any, Any, Any]]:
+    if is_async:
+        return f.dm_state_async(operation_id, run_id, bind_id, key, index, conn_url=conn_url)
+    return f.dm_state(operation_id, run_id, bind_id, key, index, conn_url=conn_url)
+
+
+@overload
+def dm_journal_list(
+    operation_id: str,
+    run_id: str,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[False] = False,
+) -> List[str]:
+    pass
+
+
+@overload
+def dm_journal_list(
+    operation_id: str,
+    run_id: str,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, List[str]]:
+    pass
+
+
+def dm_journal_list(
+    operation_id: str,
+    run_id: str,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: bool = False,
+) -> Union[List[str], Coroutine[Any, Any, List[str]]]:
+    if is_async:
+        return f.dm_journal_list_async(operation_id, run_id, conn_url=conn_url)
+    return f.dm_journal_list(operation_id, run_id, conn_url=conn_url)
+
+
+@overload
+def dm_journal(
+    operation_id: str,
+    run_id: str,
+    key: str,
+    is_stream: Literal[False] = False,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[False] = False,
+) -> Any:
+    pass
+
+
+@overload
+def dm_journal(
+    operation_id: str,
+    run_id: str,
+    key: str,
+    is_stream: Literal[False] = False,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, Any]:
+    pass
+
+
+@overload
+def dm_journal(
+    operation_id: str,
+    run_id: str,
+    key: str,
+    is_stream: Literal[True],
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[False] = False,
+) -> Iterable:
+    pass
+
+
+@overload
+def dm_journal(
+    operation_id: str,
+    run_id: str,
+    key: str,
+    is_stream: Literal[True],
+    *,
+    conn_url: Optional[str] = None,
+    is_async: Literal[True],
+) -> Coroutine[Any, Any, AsyncIterable]:
+    pass
+
+
+def dm_journal(
+    operation_id: str,
+    run_id: str,
+    key: str,
+    is_stream: bool = False,
+    *,
+    conn_url: Optional[str] = None,
+    is_async: bool = False,
+) -> Union[Any, Iterable, Coroutine[Any, Any, Union[Any, AsyncIterable]]]:
+    if is_async:
+        return f.dm_journal_async(operation_id, run_id, key, is_stream, conn_url=conn_url)
+    return f.dm_journal(operation_id, run_id, key, is_stream, conn_url=conn_url)

malevich_coretools-0.3.68/malevich_coretools/funcs/dm_funcs.py

@@ -0,0 +1,162 @@
+from http import HTTPStatus
+from typing import Any, AsyncIterable, Iterable, List, Optional, Union
+
+import aiohttp
+import requests
+from requests.models import Response
+
+from malevich_coretools.secondary import Config
+from malevich_coretools.secondary.const import *  # noqa: F403
+
+
+def dm_stream(operation_id: str, run_id: str, bind_id: str, conn_url: Optional[str]=None) -> Iterable:
+    return send_to_dm_stream(DM_STREAM(operation_id, run_id, bind_id), conn_url=conn_url)
+
+
+async def dm_stream_async(operation_id: str, run_id: str, bind_id: str, conn_url: Optional[str]=None) -> AsyncIterable:
+    return await send_to_dm_stream_async(DM_STREAM(operation_id, run_id, bind_id), conn_url=conn_url)
+
+
+def dm_continue(operation_id: str, run_id: str, id: str, data: Any, conn_url: Optional[str]=None) -> None:
+    return send_to_dm_post(DM_CONTINUE(operation_id, run_id, id), data, conn_url=conn_url)
+
+
+async def dm_continue_async(operation_id: str, run_id: str, id: str, data: Any, conn_url: Optional[str]=None) -> None:
+    return await send_to_dm_post_async(DM_CONTINUE(operation_id, run_id, id), data, conn_url=conn_url)
+
+
+def dm_state(operation_id: str, run_id: str, bind_id: str, key: Optional[str] = None, index: Optional[int] = None, conn_url: Optional[str]=None) -> Any:
+    return send_to_dm_get(DM_STATE(operation_id, run_id, bind_id, key, index), conn_url=conn_url)
+
+
+async def dm_state_async(operation_id: str, run_id: str, bind_id: str, key: Optional[str] = None, index: Optional[int] = None, conn_url: Optional[str]=None) -> Any:
+    return await send_to_dm_get_async(DM_STATE(operation_id, run_id, bind_id, key, index), conn_url=conn_url)
+
+
+def dm_journal_list(operation_id: str, run_id: str, conn_url: Optional[str]=None) -> List[str]:
+    return send_to_dm_get(DM_JOURNAL_LIST(operation_id, run_id), is_text=False, conn_url=conn_url)
+
+
+async def dm_journal_list_async(operation_id: str, run_id: str, conn_url: Optional[str]=None) -> List[str]:
+    return await send_to_dm_get_async(DM_JOURNAL_LIST(operation_id, run_id), is_text=False, conn_url=conn_url)
+
+
+def dm_journal(operation_id: str, run_id: str, key: str, is_stream: bool, conn_url: Optional[str]=None) -> Union[Any, Iterable]:
+    if is_stream:
+        return send_to_dm_stream(DM_JOURNAL(operation_id, run_id, key, is_stream), conn_url=conn_url)
+    return send_to_dm_get(DM_JOURNAL(operation_id, run_id, key, is_stream), conn_url=conn_url)
+
+
+async def dm_journal_async(operation_id: str, run_id: str, key: str, is_stream: bool, conn_url: Optional[str]=None) -> Union[Any, AsyncIterable]:
+    if is_stream:
+        return await send_to_dm_stream_async(DM_JOURNAL(operation_id, run_id, key, is_stream), conn_url=conn_url)
+    return await send_to_dm_get_async(DM_JOURNAL(operation_id, run_id, key, is_stream), conn_url=conn_url)
+
+
+#
+
+def __check_response(path: str, response: Response):  # noqa: ANN202
+    if response.status_code >= 400:
+        text = response.text
+        msg = f"failed: {text}" if len(text) > 0 else "failed"
+        Config.logger.error(f"{path} {msg}")
+
+        if response.reason is not None and len(response.reason) == 0:
+            response.reason = text
+    response.raise_for_status()
+
+
+async def __async_check_response(response: aiohttp.ClientResponse, path: Optional[str] = None):  # noqa: ANN202
+    if not response.ok:
+        text = await response.text()
+        if path is not None:
+            msg = f"failed: {text}" if len(text) > 0 else "failed"
+            Config.logger.error(f"{path} {msg}")
+        else:
+            Config.logger.error(text)
+
+        if response.reason is not None and len(response.reason) == 0:
+            response.reason = text
+    response.raise_for_status()
+
+
+def send_to_dm_get(path: str, is_text: bool=True, conn_url: Optional[str]=None) -> Optional[Union[str, bytes]]:
+    host = Config.DM_HOST_PORT if conn_url is None else conn_url
+    assert host is not None, "dm host port not set"
+    response = requests.get(f"{host}{path}", headers=HEADERS)
+    __check_response(f"{host}{path}", response)
+    if response.status_code == HTTPStatus.NO_CONTENT:
+        return None
+    if is_text is True:
+        return response.text
+    elif is_text is False:
+        return response.json()
+    else:
+        return response.content
+
+
+async def send_to_dm_get_async(path: str, is_text: bool=True, conn_url: Optional[str]=None, async_session = None) -> Optional[Union[str, bytes]]:
+    host = Config.DM_HOST_PORT if conn_url is None else conn_url
+    assert host is not None, "dm host port not set"
+    async with async_session or aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=False), timeout=aiohttp.ClientTimeout(total=None)) as session:
+        async with session.get(f"{host}{path}", headers=HEADERS) as response:
+            await __async_check_response(response, f"{host}{path}")
+            if response.status == HTTPStatus.NO_CONTENT:
+                return None
+            if is_text is True:
+                return await response.text()
+            elif is_text is False:
+                return await response.json()
+            else:
+                return await response.read()
+
+
+def send_to_dm_post(path: str, operation: Optional[Any] = None, conn_url: Optional[str]=None) -> Optional[str]:  # noqa: ANN401
+    host = Config.DM_HOST_PORT if conn_url is None else conn_url
+    assert host is not None, "dm host port not set"
+    response = requests.post(f"{host}{path}", data=operation, headers=HEADERS)
+    __check_response(f"{host}{path}", response)
+    if response.status_code == HTTPStatus.NO_CONTENT:
+        return None
+    return response.text
+
+
+async def send_to_dm_post_async(path: str, operation: Optional[Any] = None, conn_url: Optional[str]=None, async_session=None) -> str:  # noqa: ANN401
+    host = Config.DM_HOST_PORT if conn_url is None else conn_url
+    assert host is not None, "dm host port not set"
+    async with async_session or aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=False), timeout=aiohttp.ClientTimeout(total=None)) as session:
+        async with session.post(f"{host}{path}", data=operation, headers=HEADERS) as response:
+            await __async_check_response(response, f"{host}{path}")
+            if response.status == HTTPStatus.NO_CONTENT:
+                return None
+            return response.text()
+
+
+def send_to_dm_stream(path: str, conn_url: Optional[str]=None) -> Iterable:
+    host = Config.DM_HOST_PORT if conn_url is None else conn_url
+    assert host is not None, "dm host port not set"
+
+    def stream_generator() -> None:
+        with requests.get(f"{host}{path}", headers=HEADERS, stream=True) as response:
+            __check_response(f"{host}{path}", response)
+            if response.status_code == HTTPStatus.NO_CONTENT:
+                return
+            for chunk in response.iter_content(chunk_size=4096):
+                if chunk:
+                    yield chunk.decode("utf-8", errors="ignore")
+    return stream_generator()
+
+
+async def send_to_dm_stream_async(path: str, conn_url: Optional[str]=None, async_session = None) -> AsyncIterable:
+    host = Config.DM_HOST_PORT if conn_url is None else conn_url
+    assert host is not None, "dm host port not set"
+
+    async def stream_generator() -> None:
+        async with async_session or aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=False), timeout=aiohttp.ClientTimeout(total=None)) as session:
+            async with session.get(f"{host}{path}", headers=HEADERS) as response:
+                await __async_check_response(response, f"{host}{path}")
+                if response.status == HTTPStatus.NO_CONTENT:
+                    return
+                async for chunk in response.content.iter_any():
+                    yield chunk.decode("utf-8", errors="ignore")
+    return stream_generator()

{malevich_coretools-0.3.67 → malevich_coretools-0.3.68}/malevich_coretools/secondary/config.py

@@ -8,6 +8,7 @@ class Config:
     TOKEN = os.environ.get("GITLAB_ACCESS_TOKEN")   # FIXME
 
     HOST_PORT = None
+    DM_HOST_PORT = None
     KAFKA_HOST_PORT = None
     CORE_USERNAME = None
     CORE_PASSWORD = None

{malevich_coretools-0.3.67 → malevich_coretools-0.3.68}/malevich_coretools/secondary/const.py

@@ -239,3 +239,10 @@ KAFKA_SEND = f"{MANAGER_MAIN}/kafkaMsg"
 ## BatchController
 BATCH_MAIN = f"{API_VERSION}/batch"
 BATCH = BATCH_MAIN
+
+##### DM
+DM_STREAM = lambda operationId, runId, bindId : f"stream/{operationId}/{runId}/{bindId}"
+DM_CONTINUE = lambda operationId, runId, id: f"continue/{operationId}/{runId}/{id}"
+DM_STATE = lambda operationId, runId, bindId, key, index : with_key_values(f"state/{operationId}/{runId}/{bindId}", {"key": key, "index": index})
+DM_JOURNAL_LIST = lambda operationId, runId : f"journal/{operationId}/{runId}"
+DM_JOURNAL = lambda operationId, runId, key, stream : with_key_values(f"journal/{operationId}/{runId}/{key}", {"stream": stream})

{malevich_coretools-0.3.67 → malevich_coretools-0.3.68}/malevich_coretools/utils.py

@@ -47,15 +47,24 @@ def set_host_port(host_port: str) -> None:
     Config.HOST_PORT = host_port
 
 
+def set_dm_host_port(host_port: str) -> None:
+    """update host and port for malevich-dm, example: `http://localhost:8000/` """
+    assert len(host_port) > 0, "empty host port"
+    host_port = host_port if host_port[-1] == "/" else f"{host_port}/"
+    Config.DM_HOST_PORT = host_port
+
+
 def set_kafka_host_port(host_port: str) -> None:
     """update kafka host and port for malevich-kafka, example: `localhost:9092` """
     assert len(host_port) > 0, "empty host port"
     Config.KAFKA_HOST_PORT = host_port
 
 
-def set_conn_url(conn_url: str) -> None:
+def set_conn_url(conn_url: str, dm_url: Optional[str] = None) -> None:
     """analogue set_host_port; update `conn_url` for malevich-core, example: `http://localhost:8080/` """
     set_host_port(conn_url)
+    if dm_url is not None:
+        set_dm_host_port(dm_url)
 
 
 def set_verbose(verbose: bool) -> None:

{malevich_coretools-0.3.67 → malevich_coretools-0.3.68/malevich_coretools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: malevich-coretools
-Version: 0.3.67
+Version: 0.3.68
 Author: Andrew Pogrebnoj
 Author-email: andrew@onjulius.co
 License-File: LICENSE

{malevich_coretools-0.3.67 → malevich_coretools-0.3.68}/malevich_coretools.egg-info/SOURCES.txt

@@ -6,6 +6,7 @@ pyproject.toml
 requirements.txt
 setup.py
 malevich_coretools/__init__.py
+malevich_coretools/dm_utils.py
 malevich_coretools/utils.py
 malevich_coretools.egg-info/PKG-INFO
 malevich_coretools.egg-info/SOURCES.txt
@@ -23,6 +24,7 @@ malevich_coretools/batch/__init__.py
 malevich_coretools/batch/utils.py
 malevich_coretools/funcs/__init__.py
 malevich_coretools/funcs/checks.py
+malevich_coretools/funcs/dm_funcs.py
 malevich_coretools/funcs/funcs.py
 malevich_coretools/funcs/helpers.py
 malevich_coretools/secondary/__init__.py

malevich_coretools-0.3.67/VERSION

@@ -1 +0,0 @@
-0.3.67