malevich-coretools 0.2.9__py3-none-any.whl → 0.2.11__py3-none-any.whl

malevich_coretools/abstract/abstract.py

@@ -211,6 +211,11 @@ class ResultIds(BaseModel):
     ids: List[Alias.Id]
 
 
+class FilesDirs(BaseModel):
+    files: List[str]
+    directories: List[str]
+
+
 class ResultDoc(BaseModel):
     data: Alias.Json
     name: str
@@ -424,5 +429,5 @@ class OperationOrNone(BaseModel):
 
 
 class AdminStopOperation(BaseModel):
-    operationId: Optional[Alias.Id]
+    operationId: Optional[Alias.Id] = None
     withLogs: bool

malevich_coretools/admin/utils.py

@@ -17,8 +17,8 @@ def admin_get_run_info(
     auth: Optional[AUTH] = None,
     conn_url: Optional[str] = None,
 ) -> Alias.Info:
-    """return run info by operation \"id\""""
-    data = OperationOrNone(id)
+    """return run info by operation `id`"""
+    data = OperationOrNone(operationId=id)
     return f.get_admin_runs_info(data, auth=auth, conn_url=conn_url)
 
 
@@ -39,7 +39,7 @@ def admin_delete_run(
     auth: Optional[AUTH] = None,
     conn_url: Optional[str] = None,
 ) -> Alias.Json:
-    """delete run by operation \"id\""""
+    """delete run by operation `id`"""
     data = AdminStopOperation(operationId=id, withLogs=withLogs)
     return f.delete_admin_runs(data, auth=auth, conn_url=conn_url)
 
@@ -51,5 +51,5 @@ def admin_delete_runs(
     conn_url: Optional[str] = None,
 ) -> Alias.Json:
     """delete all runs"""
-    data = AdminStopOperation(operationId=None, withLogs=withLogs)
+    data = AdminStopOperation(withLogs=withLogs)
     return f.delete_admin_runs(data, auth=auth, conn_url=conn_url)

malevich_coretools/funcs/funcs.py

@@ -121,6 +121,29 @@ def delete_collections_id_s3(key: str, wait: bool, *args, **kwargs) -> Alias.Inf
 def delete_collections_id_del(id: str, data: DocsCollectionChange, wait: bool, *args, **kwargs) -> Alias.Info:
     return send_to_core_modify(COLLECTIONS_ID_DEL(id, wait), data, *args, **kwargs, is_post=False)
 
+# CollectionObjectsController
+
+
+def get_collection_objects(path: Optional[str], recursive: Optional[str], *args, **kwargs) -> FilesDirs:
+    return model_from_json(send_to_core_get(COLLECTION_OBJECTS_ALL_GET(path, recursive), *args, **kwargs), FilesDirs)
+
+
+def get_collection_object(path: str, *args, **kwargs) -> bytes:
+    return send_to_core_get(COLLECTION_OBJECTS_PATH(path, None), is_text=None, *args, **kwargs)
+
+
+def post_collections_object(path: str, data: bytes, wait: bool, *args, **kwargs) -> Alias.Info:
+    return send_to_core_modify_raw(COLLECTION_OBJECTS_PATH(path, wait), data, *args, **kwargs)
+
+
+def delete_collection_objects(wait: bool, *args, **kwargs) -> Alias.Info:
+    return send_to_core_modify(COLLECTION_OBJECTS_ALL(wait), *args, **kwargs, is_post=False)
+
+
+def delete_collection_object(path: str, wait: bool, *args, **kwargs) -> Alias.Info:
+    return send_to_core_modify(COLLECTION_OBJECTS_PATH(path, wait), *args, **kwargs, is_post=False)
+
+
 # SchemeController
 
 
@@ -599,7 +622,7 @@ def __check_response(path: str, response: Response, show_func: Optional[Callable
     response.raise_for_status()
 
 
-def send_to_core_get(path: str, with_auth=True, show_func: Optional[Callable]=None, is_text=False, auth: Optional[AUTH]=None, conn_url: Optional[str]=None) -> Optional[str]:
+def send_to_core_get(path: str, with_auth=True, show_func: Optional[Callable]=None, is_text=False, auth: Optional[AUTH]=None, conn_url: Optional[str]=None) -> Optional[Union[str, bytes]]:
     host = Config.HOST_PORT if conn_url is None else conn_url
     assert host is not None, "host port not set"
     if auth is None or not with_auth:
@@ -608,10 +631,12 @@ def send_to_core_get(path: str, with_auth=True, show_func: Optional[Callable]=No
     __check_response(f"{host}{path}", response, show_func)
     if response.status_code == HTTPStatus.NO_CONTENT:
         return None
-    if is_text:
+    if is_text is True:
         return response.text
-    else:
+    elif is_text is False:
         return response.json()
+    else:
+        return response.content
 
 
 def send_to_core_modify(path: str, operation: Optional[Any] = None, with_auth: bool=True, with_show: Optional[bool]=None, show_func: Optional[Callable]=None, is_post: bool=True, auth: Optional[AUTH]=None, conn_url: Optional[str]=None) -> str:  # noqa: ANN401
@@ -640,6 +665,30 @@ def send_to_core_modify(path: str, operation: Optional[Any] = None, with_auth: b
     return result
 
 
+def send_to_core_modify_raw(path: str, data: bytes, with_auth: bool=True, with_show: Optional[bool]=None, show_func: Optional[Callable]=None, is_post: bool=True, auth: Optional[AUTH]=None, conn_url: Optional[str]=None) -> str:  # noqa: ANN401
+    """modify: post by default, else - delete"""
+    host = Config.HOST_PORT if conn_url is None else conn_url
+    assert host is not None, "host port not set"
+    if auth is None:
+        auth = (Config.CORE_USERNAME, Config.CORE_PASSWORD)
+    if is_post:
+        response = requests.post(f"{host}{path}", data=data, headers=HEADERS_RAW, auth=auth if with_auth else None)
+    else:   # delete
+        response = requests.delete(f"{host}{path}", data=data, headers=HEADERS_RAW, auth=auth if with_auth else None)
+    __check_response(f"{host}{path}", response, show_func=show_func)
+    if response.status_code == HTTPStatus.NO_CONTENT:
+        return ""
+    result = response.text
+    if with_show is None:
+        with_show = Config.VERBOSE
+    if with_show:
+        if show_func is None:
+            Config.logger.info(result)
+        else:
+            show_func(result)
+    return result
+
+
 async def send_to_core_post_async(path: str, operation: Optional[str] = None, with_auth=True, with_show=True, show_func: Optional[Callable]=None, auth: Optional[AUTH]=None, conn_url: Optional[str]=None) -> str:
     host = Config.HOST_PORT if conn_url is None else conn_url
     assert host is not None, "host port not set"

malevich_coretools/secondary/const.py

@@ -7,6 +7,7 @@ from malevich_coretools.secondary.helpers import bool_to_str
 # const
 API_VERSION = "api/v1"
 HEADERS = {'Content-type': 'application/json', 'Accept': 'application/json', 'User-Agent': 'malevich user agent'}
+HEADERS_RAW = {'Content-type': 'text/plain; base64', 'Accept': 'application/json', 'User-Agent': 'malevich user agent'}
 SLEEP_TIME = 0.1
 LONG_SLEEP_TIME = 1             # second
 WAIT_RESULT_TIMEOUT = 60 * 60   # hour
@@ -52,6 +53,12 @@ COLLECTIONS_FIX_SCHEME = lambda id, wait: with_wait(f"{COLLECTIONS_MAIN}/{id}/fi
 COLLECTIONS_UNFIX_SCHEME = lambda id, wait: with_wait(f"{COLLECTIONS_MAIN}/{id}/unfixScheme", wait)
 COLLECTIONS_METADATA = lambda id, wait: with_wait(f"{COLLECTIONS_MAIN}/{id}/metadata", wait)
 
+## CollectionObjectsController
+COLLECTION_OBJECTS_MAIN = f"{API_VERSION}/collectionObjects"
+COLLECTION_OBJECTS_ALL_GET = lambda path, recursive: with_key_values(f"{COLLECTION_OBJECTS_MAIN}/all", {"path": path, "recursive": recursive})
+COLLECTION_OBJECTS_ALL = lambda wait: with_wait(f"{COLLECTION_OBJECTS_MAIN}/all", wait)
+COLLECTION_OBJECTS_PATH = lambda path, wait: with_key_values(f"{COLLECTION_OBJECTS_MAIN}/", {"path": path, "wait": None if wait is None else bool_to_str(wait)})
+
 ## SchemeController
 SCHEMES_MAIN = f"{API_VERSION}/schemes"
 SCHEMES = lambda wait: with_wait(f"{SCHEMES_MAIN}/", wait)

malevich_coretools/secondary/helpers.py

@@ -46,7 +46,7 @@ def __show_logs_result(res: LogsResult):  # noqa: ANN202
         print(res.data)
     for run_id, logs in res.logs.items():
         print(f"------- {run_id}:")
-        userLogs = res.userLogs[run_id]
+        userLogs = res.userLogs.get(run_id, "")
         if len(userLogs) > 0:
             print(userLogs)
             print("-------")

malevich_coretools/utils.py

@@ -447,6 +447,60 @@ def delete_from_collection(
     )
 
 
+# CollectionObjects
+
+
+def get_collection_objects(
+    path: Optional[str] = None,
+    recursive: Optional[str] = None,
+    *, auth: Optional[AUTH] = None, conn_url: Optional[str] = None
+) -> FilesDirs:
+    """return struct with list of paths: directories and files: walk if `recursive` else ls"""
+    return f.get_collection_objects(path, recursive, auth=auth, conn_url=conn_url)
+
+
+def get_collection_object(
+    path: str,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+) -> bytes:
+    """return collection object bytes by `path`"""
+    return f.get_collection_object(
+        path, auth=auth, conn_url=conn_url
+    )
+
+
+def update_collection_object(
+    path: str,
+    data: bytes,
+    wait: bool = True,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+) -> Alias.Info:
+    """update collection object with `path` by `data` """
+    return f.post_collections_object(path, data, wait=wait, auth=auth, conn_url=conn_url)
+
+
+def delete_collection_objects(
+    wait: bool = True, *, auth: Optional[AUTH] = None, conn_url: Optional[str] = None
+) -> Alias.Info:
+    """delete all collection objects"""
+    return f.delete_collection_objects(wait=wait, auth=auth, conn_url=conn_url)
+
+
+def delete_collection_object(
+    path: str,
+    wait: bool = True,
+    *,
+    auth: Optional[AUTH] = None,
+    conn_url: Optional[str] = None,
+) -> Alias.Info:
+    """delete collection object with `path` (or directory with them) """
+    return f.delete_collection_object(path, wait=wait, auth=auth, conn_url=conn_url)
+
+
 # Scheme
 
 

{malevich_coretools-0.2.9.dist-info → malevich_coretools-0.2.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: malevich-coretools
-Version: 0.2.9
+Version: 0.2.11
 Author: Andrew Pogrebnoj
 Author-email: andrew@onjulius.co
 License-File: LICENSE

malevich_coretools-0.2.11.dist-info/RECORD

@@ -0,0 +1,20 @@
+malevich_coretools/__init__.py,sha256=88IRvwNti4YOMJc6RaVRutwurJzP23g30D-rgo04Fgc,82
+malevich_coretools/utils.py,sha256=TGUMBqUy2TghICobolUWrE4JiFPDRhSH3np9YTBsAt4,62026
+malevich_coretools/abstract/__init__.py,sha256=ZWtP4gFLpNVFXzlMKXEK4iMnUqqMg07pL8yKGSJd6QI,38
+malevich_coretools/abstract/abstract.py,sha256=uV1FiH4X8F-36CiYsHNloFFA_u29iuwa_6U_G8CCCck,8950
+malevich_coretools/admin/__init__.py,sha256=zdIcHs3T_NZ8HYWts-O7OpBEWHIu779QDZMGF5HRCLg,35
+malevich_coretools/admin/utils.py,sha256=NE0Mw_Tytsi0PLD62_sI8tUVZNQaRrtUaa4oFwfUt84,1497
+malevich_coretools/funcs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+malevich_coretools/funcs/funcs.py,sha256=coGZ-OsOnKoxHeP-AjK-Ms9paWYvNlwP1XU_Fih7TI4,30458
+malevich_coretools/funcs/helpers.py,sha256=Y8v6uyawYl2_G0XmhmiVLqGIUzvxnkpKF0ODA9vrB3Y,1276
+malevich_coretools/secondary/__init__.py,sha256=048HqvG36_1WdDVZK_RuECmaf14Iq2fviUysG1inlaE,78
+malevich_coretools/secondary/config.py,sha256=b7RysQTkPKib8FEhfm3uU7TPvz8vgWLlqakvZN5NFs4,391
+malevich_coretools/secondary/const.py,sha256=VFt8nZWnts9rMCJ8R65wE6l0VAZHprq8IR7zOZcyDxQ,8631
+malevich_coretools/secondary/helpers.py,sha256=lcWN1ctfVQa5KFqh5mXdp_16hA3TXhJGe4mtNdQHnUA,3072
+malevich_coretools/tools/__init__.py,sha256=jDxlCa5Dr6Y43qlI7JwsRAlBkKmFeTHTEnjNUvu-0iw,46
+malevich_coretools/tools/vast.py,sha256=u1cnfylSgq3nYfonXbTy1zwor3OPkhD3vRf2OnqlMZY,12504
+malevich_coretools-0.2.11.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+malevich_coretools-0.2.11.dist-info/METADATA,sha256=2A3WfOJ8bHcdiuJxIEEFyWEcsphqn8hPZIjdrxOLDf8,237
+malevich_coretools-0.2.11.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+malevich_coretools-0.2.11.dist-info/top_level.txt,sha256=wDX3s1Tso0otBPNrFRfXqyNpm48W4Bp5v6JfbITO2Z8,19
+malevich_coretools-0.2.11.dist-info/RECORD,,

malevich_coretools-0.2.9.dist-info/RECORD

@@ -1,20 +0,0 @@
-malevich_coretools/__init__.py,sha256=88IRvwNti4YOMJc6RaVRutwurJzP23g30D-rgo04Fgc,82
-malevich_coretools/utils.py,sha256=X_4SK2a8SOPAfWKLj1or391WtqMlEXhGtBrY2EV82tg,60471
-malevich_coretools/abstract/__init__.py,sha256=ZWtP4gFLpNVFXzlMKXEK4iMnUqqMg07pL8yKGSJd6QI,38
-malevich_coretools/abstract/abstract.py,sha256=gqqsd7sbLYnkTZsjcHHMl6RPJUpZ6QMJ29M6iD_91EQ,8865
-malevich_coretools/admin/__init__.py,sha256=zdIcHs3T_NZ8HYWts-O7OpBEWHIu779QDZMGF5HRCLg,35
-malevich_coretools/admin/utils.py,sha256=P1RmYfc_n6plBayfOhVuwVV33vMxw1yuS11NmKD4Hto,1507
-malevich_coretools/funcs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-malevich_coretools/funcs/funcs.py,sha256=gD3Ybb5puXQQOmGnhhQS2IgdArbUEMbRgnOw0tgnMTw,28275
-malevich_coretools/funcs/helpers.py,sha256=Y8v6uyawYl2_G0XmhmiVLqGIUzvxnkpKF0ODA9vrB3Y,1276
-malevich_coretools/secondary/__init__.py,sha256=048HqvG36_1WdDVZK_RuECmaf14Iq2fviUysG1inlaE,78
-malevich_coretools/secondary/config.py,sha256=b7RysQTkPKib8FEhfm3uU7TPvz8vgWLlqakvZN5NFs4,391
-malevich_coretools/secondary/const.py,sha256=cRFOMVZXofIce9OECKmgcg5AFTcoWT-wfDFsta6_b3A,8026
-malevich_coretools/secondary/helpers.py,sha256=si5WM8OXc8j35cwHBVH0HPTMmZ1QyQePq81Ct-KIoSI,3064
-malevich_coretools/tools/__init__.py,sha256=jDxlCa5Dr6Y43qlI7JwsRAlBkKmFeTHTEnjNUvu-0iw,46
-malevich_coretools/tools/vast.py,sha256=u1cnfylSgq3nYfonXbTy1zwor3OPkhD3vRf2OnqlMZY,12504
-malevich_coretools-0.2.9.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-malevich_coretools-0.2.9.dist-info/METADATA,sha256=KLD4XcCgysBtfb_FcUKO_svCCE4ggoI_Uq1-eEMS3pU,236
-malevich_coretools-0.2.9.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-malevich_coretools-0.2.9.dist-info/top_level.txt,sha256=wDX3s1Tso0otBPNrFRfXqyNpm48W4Bp5v6JfbITO2Z8,19
-malevich_coretools-0.2.9.dist-info/RECORD,,