pyfirecrest 2.2.0__tar.gz → 2.3.0__tar.gz

{pyfirecrest-2.2.0 → pyfirecrest-2.3.0}/.github/workflows/testing.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v2

{pyfirecrest-2.2.0 → pyfirecrest-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyfirecrest
-Version: 2.2.0
+Version: 2.3.0
 Summary: pyFirecrest is a python wrapper for FirecREST
 Author: CSCS Swiss National Supercomputing Center
 Maintainer-email: Eirini Koutsaniti <eirini.koutsaniti@cscs.ch>, Juan Pablo Dorsch <juanpablo.dorsch@cscs.ch>

{pyfirecrest-2.2.0 → pyfirecrest-2.3.0}/firecrest/AsyncClient.py

@@ -26,6 +26,7 @@ from packaging.version import Version, parse
 import firecrest.FirecrestException as fe
 import firecrest.types as t
 from firecrest.AsyncExternalStorage import AsyncExternalUpload, AsyncExternalDownload
+from firecrest.utilities import time_block
 
 
 if sys.version_info >= (3, 8):
@@ -91,6 +92,12 @@ class AsyncFirecrest:
         async def wrapper(*args, **kwargs):
             client = args[0]
             num_retries = 0
+            try:
+                f = kwargs["files"]["file"]
+                file_original_position = f[1].tell() if isinstance(f, tuple) else f.tell()
+            except KeyError:
+                file_original_position = None
+
             resp = await func(*args, **kwargs)
             while True:
                 if resp.status_code != client.TOO_MANY_REQUESTS_CODE:
@@ -110,6 +117,20 @@ class AsyncFirecrest:
                         default=resp.headers.get("RateLimit-Reset", default=10),
                     )
                     reset = int(reset)
+                    try:
+                        f = kwargs["files"]["file"]
+                        logger.debug(
+                            f"Resetting the file pointer of the uploaded file "
+                            f"to {file_original_position}"
+                        )
+                        if isinstance(f, tuple):
+                            f[1].seek(file_original_position)
+                        else:
+                            f.seek(file_original_position)
+
+                    except KeyError:
+                        pass
+
                     microservice = kwargs["endpoint"].split("/")[1]
                     client = args[0]
                     logger.info(
@@ -569,6 +590,26 @@ class AsyncFirecrest:
         resp = await self._get_request(endpoint="/status/parameters")
         return self._json_response([resp], 200)["out"]
 
+    async def filesystems(self, system_name: Optional[str] = None) -> dict[str, List[t.Filesystem]]:
+        """Returns the status of the filesystems per system.
+
+        :param system_name: the system name
+        :calls: GET `/status/filesystems`
+        :calls: GET `/status/filesystems/{system_name}`
+
+        .. warning:: This is available only for FirecREST>=1.15.0
+        """
+        if system_name:
+            resp = await self._get_request(endpoint=f"/status/filesystems/{system_name}")
+            # Return the result in the same structure
+            result = {
+                system_name: self._json_response([resp], 200)["out"]
+            }
+            return result
+        else:
+            resp = await self._get_request(endpoint="/status/filesystems")
+            return self._json_response([resp], 200)["out"]
+
     # Utilities
     async def list_files(
         self, machine: str, target_path: str, show_hidden: bool = False
@@ -953,6 +994,22 @@ class AsyncFirecrest:
             # Invalid token, cannot retrieve username
             return None
 
+    async def groups(self, machine) -> t.UserId:
+        """Returns the output of the `id` command, user and group ids.
+
+        :calls: GET `/utilities/whoami`
+
+        .. warning:: This is available only for FirecREST>=1.15.0
+        """
+        resp = await self._get_request(
+            endpoint="/utilities/whoami",
+            additional_headers={"X-Machine-Name": machine},
+            params={
+                "groups": True
+            }
+        )
+        return self._json_response([resp], 200)["output"]
+
     # Compute
     async def submit(
         self,

{pyfirecrest-2.2.0 → pyfirecrest-2.3.0}/firecrest/AsyncExternalStorage.py

@@ -229,7 +229,7 @@ class AsyncExternalDownload(AsyncExternalStorage):
         :calls: GET `/tasks/{taskid}`
         """
         if self._client._api_version > Version("1.13.0"):
-            return await self.object_storage_data["url"]
+            return (await self.object_storage_data)["url"]
         else:
             return await self.object_storage_data
 
@@ -240,7 +240,7 @@ class AsyncExternalDownload(AsyncExternalStorage):
 
         :param target_path: the local path to save the file
         """
-        url = await self.object_storage_data
+        url = await self.object_storage_link
         logger.info(f"Downloading the file from {url} and saving to {target_path}")
         # LOCAL FIX FOR MAC
         # url = url.replace("192.168.220.19", "localhost")

{pyfirecrest-2.2.0 → pyfirecrest-2.3.0}/firecrest/BasicClient.py

@@ -25,6 +25,7 @@ from packaging.version import Version, parse
 import firecrest.FirecrestException as fe
 import firecrest.types as t
 from firecrest.ExternalStorage import ExternalUpload, ExternalDownload
+from firecrest.utilities import time_block
 
 if sys.version_info >= (3, 8):
     from typing import Literal
@@ -64,6 +65,12 @@ class Firecrest:
         def wrapper(*args, **kwargs):
             client = args[0]
             num_retries = 0
+            try:
+                f = kwargs["files"]["file"]
+                file_original_position = f[1].tell() if isinstance(f, tuple) else f.tell()
+            except KeyError:
+                file_original_position = None
+
             resp = func(*args, **kwargs)
             while True:
                 if resp.status_code != client.TOO_MANY_REQUESTS_CODE:
@@ -87,6 +94,20 @@ class Firecrest:
                         f"{reset} seconds and try again"
                     )
                     reset = int(reset)
+                    try:
+                        f = kwargs["files"]["file"]
+                        logger.debug(
+                            f"Resetting the file pointer of the uploaded file "
+                            f"to {file_original_position}"
+                        )
+                        if isinstance(f, tuple):
+                            f[1].seek(file_original_position)
+                        else:
+                            f.seek(file_original_position)
+
+                    except KeyError:
+                        pass
+
                     time.sleep(reset)
                     resp = func(*args, **kwargs)
                     num_retries += 1
@@ -137,13 +158,15 @@ class Firecrest:
             headers.update(additional_headers)
 
         logger.info(f"Making GET request to {endpoint}")
-        resp = self._session.get(
-            url=url,
-            headers=headers,
-            params=params,
-            verify=self._verify,
-            timeout=self.timeout,
-        )
+        with time_block(f"GET request to {endpoint}", logger):
+            resp = self._session.get(
+                url=url,
+                headers=headers,
+                params=params,
+                verify=self._verify,
+                timeout=self.timeout,
+            )
+
         return resp
 
     @_retry_requests  # type: ignore
@@ -156,14 +179,16 @@ class Firecrest:
             headers.update(additional_headers)
 
         logger.info(f"Making POST request to {endpoint}")
-        resp = self._session.post(
-            url=url,
-            headers=headers,
-            data=data,
-            files=files,
-            verify=self._verify,
-            timeout=self.timeout,
-        )
+        with time_block(f"POST request to {endpoint}", logger):
+            resp = self._session.post(
+                url=url,
+                headers=headers,
+                data=data,
+                files=files,
+                verify=self._verify,
+                timeout=self.timeout,
+            )
+
         return resp
 
     @_retry_requests  # type: ignore
@@ -176,13 +201,15 @@ class Firecrest:
             headers.update(additional_headers)
 
         logger.info(f"Making PUT request to {endpoint}")
-        resp = self._session.put(
-            url=url,
-            headers=headers,
-            data=data,
-            verify=self._verify,
-            timeout=self.timeout,
-        )
+        with time_block(f"PUT request to {endpoint}", logger):
+            resp = self._session.put(
+                url=url,
+                headers=headers,
+                data=data,
+                verify=self._verify,
+                timeout=self.timeout,
+            )
+
         return resp
 
     @_retry_requests  # type: ignore
@@ -195,13 +222,15 @@ class Firecrest:
             headers.update(additional_headers)
 
         logger.info(f"Making DELETE request to {endpoint}")
-        resp = self._session.delete(
-            url=url,
-            headers=headers,
-            data=data,
-            verify=self._verify,
-            timeout=self.timeout,
-        )
+        with time_block(f"DELETE request to {endpoint}", logger):
+            resp = self._session.delete(
+                url=url,
+                headers=headers,
+                data=data,
+                verify=self._verify,
+                timeout=self.timeout,
+            )
+
         return resp
 
     @overload
@@ -397,6 +426,26 @@ class Firecrest:
         resp = self._get_request(endpoint="/status/parameters")
         return self._json_response([resp], 200)["out"]
 
+    def filesystems(self, system_name: Optional[str] = None) -> dict[str, List[t.Filesystem]]:
+        """Returns the status of the filesystems per system.
+
+        :param system_name: the system name
+        :calls: GET `/status/filesystems`
+        :calls: GET `/status/filesystems/{system_name}`
+
+        .. warning:: This is available only for FirecREST>=1.15.0
+        """
+        if system_name:
+            resp = self._get_request(endpoint=f"/status/filesystems/{system_name}")
+            # Return the result in the same structure
+            result = {
+                system_name: self._json_response([resp], 200)["out"]
+            }
+            return result
+        else:
+            resp = self._get_request(endpoint="/status/filesystems")
+            return self._json_response([resp], 200)["out"]
+
     # Utilities
     def list_files(
         self, machine: str, target_path: str, show_hidden: bool = False
@@ -779,6 +828,22 @@ class Firecrest:
             # Invalid token, cannot retrieve username
             return None
 
+    def groups(self, machine) -> t.UserId:
+        """Returns the output of the `id` command, user and group ids.
+
+        :calls: GET `/utilities/whoami`
+
+        .. warning:: This is available only for FirecREST>=1.15.0
+        """
+        resp = self._get_request(
+            endpoint="/utilities/whoami",
+            additional_headers={"X-Machine-Name": machine},
+            params={
+                "groups": True
+            }
+        )
+        return self._json_response([resp], 200)["output"]
+
     # Compute
     def _submit_request(self, machine: str, job_script, local_file, account=None, env_vars=None):
         data = {}

{pyfirecrest-2.2.0 → pyfirecrest-2.3.0}/firecrest/__init__.py

@@ -7,7 +7,7 @@
 import sys
 
 
-__version__ = "2.2.0"
+__version__ = "2.3.0"
 __app_name__ = "firecrest"
 MIN_PYTHON_VERSION = (3, 7, 0)
 

{pyfirecrest-2.2.0 → pyfirecrest-2.3.0}/firecrest/cli/__init__.py

@@ -207,6 +207,35 @@ def parameters():
         raise typer.Exit(code=1)
 
 
+@app.command(rich_help_panel="Status commands")
+def filesystems(
+    system: Optional[str] = typer.Option(
+        None,
+        "-s",
+        "--system",
+        help="The name of the system where the filesystems belongs to.",
+        envvar="FIRECREST_SYSTEM",
+    ),
+):
+    """Information about the filesystems that are available through FirecREST"""
+    try:
+        result = client.filesystems(system)
+        for system in result.keys():
+            table = create_table(
+                f"Status of filesystems for `{system}`",
+                result[system],
+                ("Name", "name"),
+                ("Path", "path"),
+                ("Status code", "status_code"),
+                ("Status", "status"),
+                ("Description", "description"),
+            )
+            console.print(table, overflow="fold")
+    except Exception as e:
+        examine_exeption(e)
+        raise typer.Exit(code=1)
+
+
 @app.command(rich_help_panel="Status commands")
 def tasks(
     taskids: Optional[List[str]] = typer.Argument(
@@ -769,6 +798,34 @@ def whoami(
         raise typer.Exit(code=1)
 
 
+@app.command(rich_help_panel="Utilities commands")
+def id(
+    config_from_parent: str = typer.Option(None,
+        callback=config_parent_load_callback,
+        is_eager=True,
+        hidden=True
+    ),
+    system: str = typer.Option(
+        None,
+        "-s",
+        "--system",
+        help="The name of the system where the `id` command will run.",
+        envvar="FIRECREST_SYSTEM",
+    ),
+):
+    """Return the identity of the user in the remote system.
+    """
+    try:
+        result = client.groups(system)
+        user = f"{result['user']['id']}({result['user']['name']})"
+        group = f"{result['group']['id']}({result['group']['name']})"
+        all_groups = ",".join([f"{g['id']}({g['name']})" for g in result["groups"]])
+        console.print(f"uid={user} gid={group} groups={all_groups}")
+    except Exception as e:
+        examine_exeption(e)
+        raise typer.Exit(code=1)
+
+
 class TransferType(str, Enum):
     direct = "direct"
     external = "external"

{pyfirecrest-2.2.0 → pyfirecrest-2.3.0}/firecrest/types.py

@@ -46,6 +46,16 @@ class System(TypedDict):
     system: str
 
 
+class Filesystem(TypedDict):
+    """A filesystem record, from `status/filesystems`"""
+
+    name: str
+    path: str
+    description: str
+    status: int
+    status_code: str
+
+
 class Task(TypedDict):
     """A task record, from `/tasks`"""
 
@@ -140,3 +150,16 @@ class JobSubmit(TypedDict):
     job_file_out: str
     jobid: int
     result: str
+
+
+class Id(TypedDict):
+    name: str
+    id: str
+
+
+class UserId(TypedDict):
+    """A record from the `id` command"""
+
+    user: Id
+    group: Id
+    groups: list[Id]

pyfirecrest-2.3.0/firecrest/utilities.py

@@ -0,0 +1,11 @@
+import time
+from contextlib import contextmanager
+
+@contextmanager
+def time_block(label, logger):
+    start_time = time.time()
+    try:
+        yield
+    finally:
+        end_time = time.time()
+        logger.debug(f"{label} took {end_time - start_time:.6f} seconds")

{pyfirecrest-2.2.0 → pyfirecrest-2.3.0}/tests/test_status.py

@@ -2,6 +2,7 @@ import common
 import json
 import pytest
 import re
+import os
 import test_authorisation as auth
 
 from context import firecrest
@@ -60,6 +61,10 @@ def fc_server(httpserver):
         parameters_handler
     )
 
+    httpserver.expect_request(
+        re.compile("^/status/filesystems.*"), method="GET"
+    ).respond_with_handler(filesystems_handler)
+
     return httpserver
 
 
@@ -195,6 +200,49 @@ def parameters_handler(request: Request):
     return Response(json.dumps(ret), status=200, content_type="application/json")
 
 
+def filesystems_handler(request: Request):
+    if request.headers["Authorization"] != "Bearer VALID_TOKEN":
+        return Response(
+            json.dumps({"message": "Bad token; invalid JSON"}),
+            status=401,
+            content_type="application/json",
+        )
+
+    ret = {
+        "description": "Filesystem information",
+        "out": {
+            "cluster": [
+                {
+                    "description": "Users home filesystem",
+                    "name": "HOME",
+                    "path": "/home",
+                    "status": "available",
+                    "status_code": 200
+                },
+                {
+                    "description": "Scratch filesystem",
+                    "name": "SCRATCH",
+                    "path": "/scratch",
+                    "status": "not available",
+                    "status_code": 400
+                }
+            ]
+        }
+    }
+    ret_status = 200
+    uri = request.url
+    if not uri.endswith("/status/filesystems"):
+        system = uri.split("/")[-1]
+        if system == "cluster":
+            ret["description"] = f"Filesystem information for system {system}"
+            ret["out"] = ret["out"][system]
+        else:
+            ret = {"description": f"System '{system}' does not exists."}
+            ret_status = 400
+
+    return Response(json.dumps(ret), status=ret_status, content_type="application/json")
+
+
 def test_all_services(valid_client):
     assert valid_client.all_services() == [
         {
@@ -334,3 +382,69 @@ def test_cli_parameters(valid_credentials):
 def test_parameters_invalid(invalid_client):
     with pytest.raises(firecrest.UnauthorizedException):
         invalid_client.parameters()
+
+
+def test_filesystems(valid_client):
+    assert valid_client.filesystems() == {
+        "cluster": [
+            {
+                "description": "Users home filesystem",
+                "name": "HOME",
+                "path": "/home",
+                "status": "available",
+                "status_code": 200
+            },
+            {
+                "description": "Scratch filesystem",
+                "name": "SCRATCH",
+                "path": "/scratch",
+                "status": "not available",
+                "status_code": 400
+            }
+        ]
+    }
+
+    assert valid_client.filesystems(system_name="cluster") == {
+        "cluster": [
+            {
+                "description": "Users home filesystem",
+                "name": "HOME",
+                "path": "/home",
+                "status": "available",
+                "status_code": 200
+            },
+            {
+                "description": "Scratch filesystem",
+                "name": "SCRATCH",
+                "path": "/scratch",
+                "status": "not available",
+                "status_code": 400
+            }
+        ]
+    }
+
+
+def test_cli_filesystems(valid_credentials):
+    # Clean up the env var that may be set in the environment
+    os.environ.pop("FIRECREST_SYSTEM", None)
+    args = valid_credentials + ["filesystems"]
+    result = runner.invoke(cli.app, args=args)
+    stdout = common.clean_stdout(result.stdout)
+    assert result.exit_code == 0
+    assert "Status of filesystems for `cluster`" in stdout
+
+    args = valid_credentials + ["filesystems", "--system", "cluster"]
+    result = runner.invoke(cli.app, args=args)
+    stdout = common.clean_stdout(result.stdout)
+    assert result.exit_code == 0
+    assert "Status of filesystems for `cluster`" in stdout
+
+    args = valid_credentials + ["filesystems", "--system", "invalid_cluster"]
+    result = runner.invoke(cli.app, args=args)
+    stdout = common.clean_stdout(result.stdout)
+    assert result.exit_code == 1
+
+
+def test_filesystems_invalid(invalid_client):
+    with pytest.raises(firecrest.UnauthorizedException):
+        invalid_client.filesystems()

{pyfirecrest-2.2.0 → pyfirecrest-2.3.0}/tests/test_status_async.py

@@ -62,6 +62,10 @@ def fc_server(httpserver):
         basic_status.parameters_handler
     )
 
+    httpserver.expect_request(
+        re.compile("^/status/filesystems.*"), method="GET"
+    ).respond_with_handler(basic_status.filesystems_handler)
+
     return httpserver
 
 
@@ -167,3 +171,50 @@ async def test_parameters(valid_client):
 async def test_parameters_invalid(invalid_client):
     with pytest.raises(firecrest.UnauthorizedException):
         await invalid_client.parameters()
+
+
+@pytest.mark.asyncio
+async def test_filesystems(valid_client):
+    assert await valid_client.filesystems() == {
+        "cluster": [
+            {
+                "description": "Users home filesystem",
+                "name": "HOME",
+                "path": "/home",
+                "status": "available",
+                "status_code": 200
+            },
+            {
+                "description": "Scratch filesystem",
+                "name": "SCRATCH",
+                "path": "/scratch",
+                "status": "not available",
+                "status_code": 400
+            }
+        ]
+    }
+
+    assert await valid_client.filesystems(system_name="cluster") == {
+        "cluster": [
+            {
+                "description": "Users home filesystem",
+                "name": "HOME",
+                "path": "/home",
+                "status": "available",
+                "status_code": 200
+            },
+            {
+                "description": "Scratch filesystem",
+                "name": "SCRATCH",
+                "path": "/scratch",
+                "status": "not available",
+                "status_code": 400
+            }
+        ]
+    }
+
+
+@pytest.mark.asyncio
+async def test_filesystems_invalid(invalid_client):
+    with pytest.raises(firecrest.UnauthorizedException):
+        await invalid_client.filesystems()

{pyfirecrest-2.2.0 → pyfirecrest-2.3.0}/tests/test_utilities.py

@@ -730,8 +730,21 @@ def whoami_handler(request: Request):
             content_type="application/json",
         )
 
+    groups = request.args.get("groups", False)
+    if groups:
+        ret = {
+            "description": "User information",
+            "output": {
+                "group": {"id": "1000", "name": "group1"},
+                "groups": [{"id": "1000", "name": "group1"}, {"id": "1001", "name": "group2"}],
+                "user": {"id": "10000", "name": "test_user"},
+            }
+        }
+    else:
+        ret = {"description": "Success on whoami operation.", "output": "username"}
+
     return Response(
-        json.dumps({"description": "Success on whoami operation.", "output": "username"}),
+        json.dumps(ret),
         status=200,
         content_type="application/json",
     )
@@ -1510,3 +1523,37 @@ def test_whoami_invalid_machine(valid_client):
 def test_whoami_invalid_client(invalid_client):
     with pytest.raises(firecrest.UnauthorizedException):
         invalid_client.whoami("cluster1")
+
+
+def test_cli_whoami(valid_credentials):
+    args = valid_credentials + ["whoami", "--system", "cluster1"]
+    result = runner.invoke(cli.app, args=args)
+    stdout = common.clean_stdout(result.stdout)
+    assert result.exit_code == 0
+    assert "username" in stdout
+
+
+def test_groups(valid_client):
+    assert valid_client.groups("cluster1") == {
+        "group": {"id": "1000", "name": "group1"},
+        "groups": [{"id": "1000", "name": "group1"}, {"id": "1001", "name": "group2"}],
+        "user": {"id": "10000", "name": "test_user"},
+    }
+
+
+def test_groups_invalid_machine(valid_client):
+    with pytest.raises(firecrest.HeaderException):
+        valid_client.groups("cluster2")
+
+
+def test_groups_invalid_client(invalid_client):
+    with pytest.raises(firecrest.UnauthorizedException):
+        invalid_client.groups("cluster1")
+
+
+def test_cli_id(valid_credentials):
+    args = valid_credentials + ["id", "--system", "cluster1"]
+    result = runner.invoke(cli.app, args=args)
+    stdout = common.clean_stdout(result.stdout)
+    assert result.exit_code == 0
+    assert "uid=10000(test_user) gid=1000(group1) groups=1000(group1),1001(group2)" in stdout

{pyfirecrest-2.2.0 → pyfirecrest-2.3.0}/tests/test_utilities_async.py

@@ -651,3 +651,25 @@ async def test_whoami_invalid_machine(valid_client):
 async def test_whoami_invalid_client(invalid_client):
     with pytest.raises(firecrest.UnauthorizedException):
         await invalid_client.whoami("cluster1")
+
+
+@pytest.mark.asyncio
+async def test_groups(valid_client):
+    assert await valid_client.groups("cluster1") == {
+        "group": {"id": "1000", "name": "group1"},
+        "groups": [{"id": "1000", "name": "group1"}, {"id": "1001", "name": "group2"}],
+        "user": {"id": "10000", "name": "test_user"},
+    }
+
+
+@pytest.mark.asyncio
+async def test_groups_invalid_machine(valid_client):
+    with pytest.raises(firecrest.HeaderException):
+        await valid_client.groups("cluster2")
+
+
+@pytest.mark.asyncio
+async def test_groups_invalid_client(invalid_client):
+    with pytest.raises(firecrest.UnauthorizedException):
+        await invalid_client.groups("cluster1")
+