pyfirecrest 3.2.2__tar.gz → 3.3.1__tar.gz

{pyfirecrest-3.2.2 → pyfirecrest-3.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyfirecrest
-Version: 3.2.2
+Version: 3.3.1
 Summary: pyFirecrest is a python wrapper for FirecREST
 Author: CSCS Swiss National Supercomputing Center
 Maintainer-email: Eirini Koutsaniti <eirini.koutsaniti@cscs.ch>, Juan Pablo Dorsch <juanpablo.dorsch@cscs.ch>
@@ -12,10 +12,10 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
 License-File: LICENSE
-Requires-Dist: aiofiles~=23.2.1
+Requires-Dist: aiofiles>=23.2.1
 Requires-Dist: requests>=2.14.0
 Requires-Dist: PyJWT>=2.4.0
-Requires-Dist: typer[all]~=0.7.0
+Requires-Dist: typer[all]>=0.7.0,<1.0.0
 Requires-Dist: packaging>=21.0
 Requires-Dist: httpx>=0.24.0
 Requires-Dist: PyYAML>=5.1

pyfirecrest-3.3.1/docs/source/authorization.rst

@@ -0,0 +1,121 @@
+Authorization
+=============
+
+For every request to FirecREST you need to have a valid access token.
+This will enable your application to have access to the requested resources.
+
+The pyFirecREST Authorization Object
+------------------------------------
+
+You can take care of the access token by yourself any way you want, or even better use a python library to take care of this for you, depending on the grant type.
+What pyFirecREST will need in the end is only a python object with the method ``get_access_token()``, that when called will provide a valid access token.
+
+Let's say for example you have somehow obtained a long-lasting access token.
+The Authorization class you would need to make and give to Firecrest would look like this:
+
+.. code-block:: Python
+
+    class MyAuthorizationClass:
+        def __init__(self):
+            pass
+
+        def get_access_token(self):
+            return <TOKEN>
+
+If this is your case you can move on to the next session, where you can see how to use this Authorization object, in order to make your requests.
+
+If you want to use the Client Credentials authorization grant, you can use the ``ClientCredentialsAuth`` class from pyFirecREST and setup the authorization object like this:
+
+.. code-block:: Python
+
+    import firecrest as f7t
+
+    keycloak = f7t.ClientCredentialsAuth(
+        <client_id>, <client_secret>, <token_uri>
+    )
+
+Device Authorization Grant Workflow
+-----------------------------------
+
+When your application cannot open a browser or embed a user-agent, you can use the OAuth 2.0 Device Authorization Grant.
+In this flow, your app is responsible for:
+ 1. Requesting a device code & user code from the token endpoint
+ 2. Instructing the user to visit a verification URL and enter the code
+ 3. Polling the token endpoint until the user has authorized
+ 4. Caching and refreshing the access token for future calls
+
+This workflow is not handled by pyFirecREST directly, but you can implement it using the `requests` library or any other HTTP client of your choice.
+The `Authorization` object you hand to pyFirecREST needs the `get_access_token()` method, but it would also have orchestrate to workflow behind the scenes.
+
+Here we provide an example implementation using `requests`.
+We chose to inform the user about the verification URL and user code by simply printing it, but you should adapt this to the needs of your application.
+
+.. code-block:: python
+
+    import time
+    import requests
+
+    class DeviceAuthorizationAuth:
+        def __init__(self, client_id, device_uri, token_uri, scope=None):
+            self.client_id   = client_id
+            self.device_uri  = device_uri
+            self.token_uri   = token_uri
+            self.scope       = scope or "openid"
+            self._token_data = None
+
+        def _start_device_flow(self):
+            resp = requests.post(self.device_uri, data={
+                "client_id": self.client_id,
+                "scope":     self.scope,
+            })
+            resp.raise_for_status()
+            return resp.json()
+
+        def _poll_for_token(self, device_code, interval, expires_in):
+            deadline = time.time() + expires_in
+            while time.time() < deadline:
+                resp = requests.post(self.token_uri, data={
+                    "grant_type":  "urn:ietf:params:oauth:grant-type:device_code",
+                    "device_code": device_code,
+                    "client_id":   self.client_id,
+                })
+                if resp.status_code == 200:
+                    return resp.json()
+
+                time.sleep(interval)
+            raise RuntimeError("User did not authorize in time")
+
+        def get_access_token(self):
+            # If we already have a (non-expired) token, reuse it
+            if self._token_data and time.time() < self._token_data["expires_at"]:
+                return self._token_data["access_token"]
+
+            # 1) Start device flow
+            flow = self._start_device_flow()
+            print(f"Go to {flow['verification_uri']} and enter code {flow['user_code']}")
+
+            # 2) Poll until the user completes authorization
+            token = self._poll_for_token(
+                device_code=flow["device_code"],
+                interval=flow.get("interval", 5),
+                expires_in=flow.get("expires_in", 600),
+            )
+
+            # 3) Compute absolute expiry & cache
+            token["expires_at"] = time.time() + token["expires_in"]
+            self._token_data = token
+            return token["access_token"]
+
+    # Usage with pyFirecREST
+    import firecrest as f7t
+
+    auth = DeviceAuthorizationAuth(
+        client_id="YOUR_CLIENT_ID",
+        device_uri="https://auth.example.com/device",
+        token_uri="https://auth.example.com/token",
+        scope="openid profile",
+    )
+    firecrest = f7t.FirecrestClient(url="https://api.firecrest.example.com", auth=auth)
+
+
+In a similar way you can reuse other packages to support different grant types, like `Flask-OIDC <https://flask-oidc.readthedocs.io/>`__.

{pyfirecrest-3.2.2 → pyfirecrest-3.3.1}/firecrest/FirecrestException.py

@@ -135,6 +135,18 @@ class TransferJobFailedException(Exception):
         )
 
 
+class JobTimeoutException(Exception):
+    """Exception when the job exceeds the user-defined timeout"""
+
+    def __init__(self, jobid):
+        self._jobid = jobid
+
+    def __str__(self):
+        return (
+            f"Job {self._jobid} has exceeded the user-defined timeout."
+        )
+
+
 class TransferJobTimeoutException(TransferJobFailedException):
     """Exception when the transfer job exceeds the user-defined timeout"""
 

{pyfirecrest-3.2.2 → pyfirecrest-3.3.1}/firecrest/__init__.py

@@ -5,20 +5,6 @@
 #  SPDX-License-Identifier: BSD-3-Clause
 #
 import sys
-
-
-__version__ = "3.2.2"
-__app_name__ = "firecrest"
-MIN_PYTHON_VERSION = (3, 7, 0)
-
-# Check python version
-if sys.version_info[:3] < MIN_PYTHON_VERSION:
-    sys.stderr.write(
-        "Unsupported Python version: "
-        "Python >= %d.%d.%d is required\n" % MIN_PYTHON_VERSION
-    )
-    sys.exit(1)
-
 from . import v1, v2
 from firecrest.Authorization import ClientCredentialsAuth
 from firecrest.FirecrestException import (
@@ -31,3 +17,31 @@ from firecrest.FirecrestException import (
     UnauthorizedException,
     UnexpectedStatusException,
 )
+
+__version__ = "3.3.1"
+__app_name__ = "firecrest"
+MIN_PYTHON_VERSION = (3, 7, 0)
+
+__all__ = [
+    "v1",
+    "v2",
+    "ClientCredentialsAuth",
+    "ClientsCredentialsException",
+    "FirecrestException",
+    "HeaderException",
+    "NotImplementedOnAPIversion",
+    "StorageDownloadException",
+    "StorageUploadException",
+    "UnauthorizedException",
+    "UnexpectedStatusException",
+    "__version__",
+    "__app_name__",
+]
+
+# Check python version
+if sys.version_info[:3] < MIN_PYTHON_VERSION:
+    sys.stderr.write(
+        "Unsupported Python version: "
+        "Python >= %d.%d.%d is required\n" % MIN_PYTHON_VERSION
+    )
+    sys.exit(1)

{pyfirecrest-3.2.2 → pyfirecrest-3.3.1}/firecrest/cli2/__init__.py

@@ -850,6 +850,25 @@ def job_info(
         raise typer.Exit(code=1)
 
 
+@app.command(rich_help_panel="Compute commands")
+def wait_for_job(
+    system: str = typer.Option(
+        ..., "-s", "--system", help="The name of the system.", envvar="FIRECREST_SYSTEM"
+    ),
+    jobid: str = typer.Argument(
+        ..., help="Job id of the job to wait for."
+    ),
+):
+    """Wait for a job to complete. It will return the job information when the job is completed.
+    """
+    try:
+        result = client.wait_for_job(system, jobid)
+        console.print(json.dumps(result, indent=4))
+    except Exception as e:
+        examine_exeption(e)
+        raise typer.Exit(code=1)
+
+
 @app.command(rich_help_panel="Compute commands")
 def job_metadata(
     system: str = typer.Option(

{pyfirecrest-3.2.2 → pyfirecrest-3.3.1}/firecrest/v1/__init__.py

@@ -15,5 +15,16 @@ from firecrest.v1.BasicClient import Firecrest
 from firecrest.v1.ExternalStorage import (
     ExternalDownload,
     ExternalUpload,
-    ExternalStorage)
+    ExternalStorage,
+)
 
+__all__ = [
+    "AsyncFirecrest",
+    "AsyncExternalDownload",
+    "AsyncExternalUpload",
+    "AsyncExternalStorage",
+    "Firecrest",
+    "ExternalDownload",
+    "ExternalUpload",
+    "ExternalStorage",
+]

{pyfirecrest-3.2.2 → pyfirecrest-3.3.1}/firecrest/v2/__init__.py

@@ -15,3 +15,12 @@ from firecrest.v2._sync.Client import (  # noqa
     ExternalUpload,
     Firecrest,
 )
+
+__all__ = [
+    "AsyncExternalDownload",
+    "AsyncExternalUpload",
+    "AsyncFirecrest",
+    "ExternalDownload",
+    "ExternalUpload",
+    "Firecrest",
+]

{pyfirecrest-3.2.2 → pyfirecrest-3.3.1}/firecrest/v2/_async/Client.py

@@ -26,6 +26,7 @@ from firecrest.utilities import (
 )
 from firecrest.FirecrestException import (
     FirecrestException,
+    JobTimeoutException,
     MultipartUploadException,
     TransferJobFailedException,
     TransferJobTimeoutException,
@@ -931,15 +932,83 @@ class AsyncFirecrest:
         )
         self._check_response(resp, 204)
 
-    async def _wait_for_transfer_job(self, job_info, timeout=None):
+    async def _wait_for_transfer_job(
+        self,
+        job_info: dict,
+        timeout: Optional[float] = None
+    ) -> None:
+        job_id = job_info["transferJob"]["jobId"]
+        system_name = job_info["transferJob"]["system"]
+        try:
+            await self.wait_for_job(
+                system_name,
+                job_id,
+                timeout=timeout
+            )
+        except JobTimeoutException:
+            self.log(
+                logging.DEBUG,
+                f"Transfer job {job_id} timed out"
+            )
+            raise TransferJobTimeoutException(job_info)
+
+        try:
+            # If job is cancelled before it starts running, the log file will
+            # not be available
+            stdout_file = await self.view(
+                system_name,
+                job_info["transferJob"]["logs"]["outputLog"]
+            )
+        except FirecrestException as e:
+            if (
+                e.responses[-1].status_code == 404 and
+                "No such file or directory" in e.responses[-1].json()['message']
+            ):
+                raise TransferJobFailedException(
+                    job_info, file_not_found=True
+                )
+            else:
+                raise e
+
+        if (
+            "Files were successfully" not in stdout_file and
+            "File was successfully" not in stdout_file and
+            "Multipart file upload successfully completed" not in stdout_file
+        ):
+            raise TransferJobFailedException(job_info)
+
+    async def wait_for_job(
+        self,
+        system_name: str,
+        job_id: str,
+        timeout: Optional[float] = None,
+        not_found_timeout: float = 20.0
+    ) -> List[Any]:
+        """Wait for a job to complete. When the job is completed, it will
+        return the job information.
+        :param system_name: the system name where the filesystem belongs to
+        :param job_id: the ID of the job to wait for
+        :param timeout: the maximum time to wait for the job to complete
+                        in seconds. If the timeout is set and the job is not
+                        completed within this time, it will be cancelled.
+        :param not_found_timeout: the maximum time to wait for the job to be
+                                  available in the scheduler, in seconds.
+                                  Slurm or other scheduler will need a few
+                                  seconds before updating the database. If
+                                  the job is not found within this time, the
+                                  function will raise an exception.
+        :calls: GET `/jobs/{system_name}/{job_id}`
+        """
         if timeout:
             timeout_time = asyncio.get_event_loop().time() + timeout
 
-        job_id = job_info["transferJob"]["jobId"]
-        system_name = job_info["transferJob"]["system"]
+        not_found_timeout_time = (
+            asyncio.get_event_loop().time() + not_found_timeout
+        )
+
         self.log(
             logging.DEBUG,
-            f"Waiting for transfer job {job_id}."
+            f"Waiting for job {job_id}."
         )
 
         async def check_timeout(sleep_time):
@@ -950,8 +1019,7 @@ class AsyncFirecrest:
                 self.log(
                     logging.DEBUG,
                     f"Timeout is about to be reached, while waiting for "
-                    f"transfer job {job_id}. Will cancel the job to stop "
-                    f"the transfer."
+                    f"job {job_id}. Will cancel the job to stop the transfer."
                 )
 
                 try:
@@ -962,7 +1030,7 @@ class AsyncFirecrest:
                         f"Failed to cancel job {job_id}: {e}"
                     )
 
-                raise TransferJobTimeoutException(job_info)
+                raise JobTimeoutException(job_id)
 
         for i in sleep_generator():
             try:
@@ -972,6 +1040,9 @@ class AsyncFirecrest:
                     e.responses[-1].status_code == 404 and
                     "Job not found" in e.responses[-1].json()['message']
                 ):
+                    if asyncio.get_event_loop().time() > not_found_timeout_time:
+                        raise e
+
                     self.log(
                         logging.DEBUG,
                         f"Job {job_id} information is not yet available, will "
@@ -999,30 +1070,7 @@ class AsyncFirecrest:
             await check_timeout(i)
             await asyncio.sleep(i)
 
-        try:
-            # If job is cancelled before it starts running, the log file will
-            # not be available
-            stdout_file = await self.view(
-                system_name,
-                job_info["transferJob"]["logs"]["outputLog"]
-            )
-        except FirecrestException as e:
-            if (
-                e.responses[-1].status_code == 404 and
-                "No such file or directory" in e.responses[-1].json()['message']
-            ):
-                raise TransferJobFailedException(
-                    job_info, file_not_found=True
-                )
-            else:
-                raise e
-
-        if (
-            "Files were successfully" not in stdout_file and
-            "File was successfully" not in stdout_file and
-            "Multipart file upload successfully completed" not in stdout_file
-        ):
-            raise TransferJobFailedException(job_info)
+        return job
 
     async def cp(
         self,
@@ -1083,9 +1131,9 @@ class AsyncFirecrest:
         :param system_name: the system name where the filesystem belongs to
         :param path: the absolute target path
         :param account: the account to be used for the transfer job  (only
-                            relevant when the file is not deleted directly)
+                        relevant when the file is not deleted directly)
         :param blocking: whether to wait for the job to complete (only
-                            relevant when the file is not deleted directly)
+                         relevant when the file is not deleted directly)
         :param timeout: the maximum time to wait for the job to complete
         :calls: DELETE `/filesystem/{system_name}/ops/rm`
 

{pyfirecrest-3.2.2 → pyfirecrest-3.3.1}/firecrest/v2/_sync/Client.py

@@ -25,6 +25,7 @@ from firecrest.utilities import (
 )
 from firecrest.FirecrestException import (
     FirecrestException,
+    JobTimeoutException,
     MultipartUploadException,
     TransferJobFailedException,
     TransferJobTimeoutException,
@@ -115,16 +116,16 @@ class ExternalUpload:
         else:
             content_length = chunk_size
 
-        with open(self._local_file, "rb") as f:
-            f.seek(start)
-            resp = self._client._session.put(
-                url=url,
-                content=chunk_reader(f, self.chunk_size),
-                timeout=None,
-                headers={
-                    "Content-Length": str(content_length)
-                }
-            )
+            with open(self._local_file, "rb") as f:
+                f.seek(start)
+                resp = self._client._session.put(
+                    url=url,
+                    content=chunk_reader(f, self.chunk_size),
+                    timeout=None,
+                    headers={
+                        "Content-Length": str(content_length)
+                    }
+                )
 
         if resp.status_code >= 400:
             raise MultipartUploadException(
@@ -916,15 +917,83 @@ class Firecrest:
         )
         self._check_response(resp, 204)
 
-    def _wait_for_transfer_job(self, job_info, timeout=None):
+    def _wait_for_transfer_job(
+        self,
+        job_info: dict,
+        timeout: Optional[float] = None
+    ) -> None:
+        job_id = job_info["transferJob"]["jobId"]
+        system_name = job_info["transferJob"]["system"]
+        try:
+            self.wait_for_job(
+                system_name,
+                job_id,
+                timeout=timeout
+            )
+        except JobTimeoutException:
+            self.log(
+                logging.DEBUG,
+                f"Transfer job {job_id} timed out"
+            )
+            raise TransferJobTimeoutException(job_info)
+
+        try:
+            # If job is cancelled before it starts running, the log file will
+            # not be available
+            stdout_file = self.view(
+                system_name,
+                job_info["transferJob"]["logs"]["outputLog"]
+            )
+        except FirecrestException as e:
+            if (
+                e.responses[-1].status_code == 404 and
+                "No such file or directory" in e.responses[-1].json()['message']
+            ):
+                raise TransferJobFailedException(
+                    job_info, file_not_found=True
+                )
+            else:
+                raise e
+
+        if (
+            "Files were successfully" not in stdout_file and
+            "File was successfully" not in stdout_file and
+            "Multipart file upload successfully completed" not in stdout_file
+        ):
+            raise TransferJobFailedException(job_info)
+
+    def wait_for_job(
+        self,
+        system_name: str,
+        job_id: str,
+        timeout: Optional[float] = None,
+        not_found_timeout: float = 20.0
+    ) -> List[Any]:
+        """Wait for a job to complete. When the job is completed, it will
+        return the job information.
+        :param system_name: the system name where the filesystem belongs to
+        :param job_id: the ID of the job to wait for
+        :param timeout: the maximum time to wait for the job to complete
+                        in seconds. If the timeout is set and the job is not
+                        completed within this time, it will be cancelled.
+        :param not_found_timeout: the maximum time to wait for the job to be
+                                  available in the scheduler, in seconds.
+                                  Slurm or other scheduler will need a few
+                                  seconds before updating the database. If
+                                  the job is not found within this time, the
+                                  function will raise an exception.
+        :calls: GET `/jobs/{system_name}/{job_id}`
+        """
         if timeout:
             timeout_time = time.time() + timeout
 
-        job_id = job_info["transferJob"]["jobId"]
-        system_name = job_info["transferJob"]["system"]
+        not_found_timeout_time = (
+            time.time() + not_found_timeout
+        )
+
         self.log(
             logging.DEBUG,
-            f"Waiting for transfer job {job_id}."
+            f"Waiting for job {job_id}."
         )
 
         def check_timeout(sleep_time):
@@ -935,8 +1004,7 @@ class Firecrest:
                 self.log(
                     logging.DEBUG,
                     f"Timeout is about to be reached, while waiting for "
-                    f"transfer job {job_id}. Will cancel the job to stop "
-                    f"the transfer."
+                    f"job {job_id}. Will cancel the job to stop the transfer."
                 )
 
                 try:
@@ -947,7 +1015,7 @@ class Firecrest:
                         f"Failed to cancel job {job_id}: {e}"
                     )
 
-                raise TransferJobTimeoutException(job_info)
+                raise JobTimeoutException(job_id)
 
         for i in sleep_generator():
             try:
@@ -957,6 +1025,9 @@ class Firecrest:
                     e.responses[-1].status_code == 404 and
                     "Job not found" in e.responses[-1].json()['message']
                 ):
+                    if time.time() > not_found_timeout_time:
+                        raise e
+
                     self.log(
                         logging.DEBUG,
                         f"Job {job_id} information is not yet available, will "
@@ -984,30 +1055,7 @@ class Firecrest:
             check_timeout(i)
             time.sleep(i)
 
-        try:
-            # If job is cancelled before it starts running, the log file will
-            # not be available
-            stdout_file = self.view(
-                system_name,
-                job_info["transferJob"]["logs"]["outputLog"]
-            )
-        except FirecrestException as e:
-            if (
-                e.responses[-1].status_code == 404 and
-                "No such file or directory" in e.responses[-1].json()['message']
-            ):
-                raise TransferJobFailedException(
-                    job_info, file_not_found=True
-                )
-            else:
-                raise e
-
-        if (
-            "Files were successfully" not in stdout_file and
-            "File was successfully" not in stdout_file and
-            "Multipart file upload successfully completed" not in stdout_file
-        ):
-            raise TransferJobFailedException(job_info)
+        return job
 
     def cp(
         self,
@@ -1350,7 +1398,7 @@ class Firecrest:
                 "be set."
             )
 
-        data: Any = {
+        data: dict = {
             "job": {
                 "working_directory": working_dir
             }

{pyfirecrest-3.2.2 → pyfirecrest-3.3.1}/pyproject.toml

@@ -25,10 +25,10 @@ classifiers = [
 ]
 requires-python = ">=3.7"
 dependencies = [
-    "aiofiles~=23.2.1",
+    "aiofiles>=23.2.1",
     "requests>=2.14.0",
     "PyJWT>=2.4.0",
-    "typer[all]~=0.7.0",
+    "typer[all]>=0.7.0,<1.0.0",
     "packaging>=21.0",
     "httpx>=0.24.0",
     "PyYAML>=5.1"

pyfirecrest-3.2.2/docs/source/authorization.rst

@@ -1,47 +0,0 @@
-Authorization
-=============
-
-For every request to FirecREST you need to have a valid access token.
-This will enable your application to have access to the requested resources.
-
-
-.. Supported grant types
-.. ---------------------
-
-.. Implicit
-.. ^^^^^^^^
-
-.. Client Credentials
-.. ^^^^^^^^^^^^^^^^^^
-
-The pyFirecREST Authorization Object
-------------------------------------
-
-You can take care of the access token by yourself any way you want, or even better use a python library to take care of this for you, depending on the grant type.
-What pyFirecREST will need in the end is only a python object with the method ``get_access_token()``, that when called will provide a valid access token.
-
-Let's say for example you have somehow obtained a long-lasting access token.
-The Authorization class you would need to make and give to Firecrest would look like this:
-
-.. code-block:: Python
-
-    class MyAuthorizationClass:
-        def __init__(self):
-            pass
-
-        def get_access_token(self):
-            return <TOKEN>
-
-If this is your case you can move on to the next session, where you can see how to use this Authorization object, in order to make your requests.
-
-If you want to use the Client Credentials authorization grant, you can use the ``ClientCredentialsAuth`` class from pyFirecREST and setup the authorization object like this:
-
-.. code-block:: Python
-
-    import firecrest as f7t
-
-    keycloak = f7t.ClientCredentialsAuth(
-        <client_id>, <client_secret>, <token_uri>
-    )
-
-In a similar way you can reuse other packages to support different grant types, like `Flask-OIDC <https://flask-oidc.readthedocs.io/>`__.