sfapi-client 0.4.0__tar.gz → 0.5.0__tar.gz

{sfapi_client-0.4.0 → sfapi_client-0.5.0}/.github/workflows/black.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: psf/black@23.9.1
+      - uses: psf/black@26.3.0
         with:
           options: "--check --verbose"
-          jupyter: false
+          jupyter: false

{sfapi_client-0.4.0 → sfapi_client-0.5.0}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: sfapi_client
-Version: 0.4.0
+Version: 0.5.0
 Summary: Python client for NERSC SF API
 Author-email: Chris Harris <cjh@lbl.gov>, Nicholas Tyler <tylern@lbl.gov>
 License-File: LICENSE
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.7
+Requires-Python: >=3.8
 Requires-Dist: authlib
 Requires-Dist: httpx
 Requires-Dist: pydantic-settings

{sfapi_client-0.4.0 → sfapi_client-0.5.0}/docs/quickstart/index.md

@@ -95,13 +95,12 @@ export SFAPI_SECRET='{"kty": "RSA", "n": ...}'
     ```pycon
     >>> from sfapi_client import AsyncClient
     >>> from sfapi_client.compute import Machine
-    >>> from authlib.jose import JsonWebKey
     >>> import json
     >>> import os
     >>>
     >>> client_id = os.getenv("SFAPI_CLIENT_ID")
     >>> sfapi_secret = os.getenv("SFAPI_SECRET")
-    >>> client_secret = JsonWebKey.import_key(json.loads(sfapi_secret))
+    >>> client_secret = json.loads(sfapi_secret)
     >>>
     >>> async with AsyncClient(client_id, client_secret) as client:
     ...     perlmutter = await client.compute(Machine.perlmutter)
@@ -110,13 +109,12 @@ export SFAPI_SECRET='{"kty": "RSA", "n": ...}'
     ```pycon
     >>> from sfapi_client import Client
     >>> from sfapi_client.compute import Machine
-    >>> from authlib.jose import JsonWebKey
     >>> import json
     >>> import os
     >>>
     >>> client_id = os.getenv("SFAPI_CLIENT_ID")
     >>> sfapi_secret = os.getenv("SFAPI_SECRET")
-    >>> client_secret = JsonWebKey.import_key(json.loads(sfapi_secret))
+    >>> client_secret = json.loads(sfapi_secret)
     >>>
     >>> with Client(client_id, client_secret) as client:
     ...     perlmutter = client.compute(Machine.perlmutter)

{sfapi_client-0.4.0 → sfapi_client-0.5.0}/pyproject.toml

@@ -14,7 +14,7 @@ authors = [
 ]
 description = "Python client for NERSC SF API"
 readme = "README.md"
-requires-python = ">=3.7"
+requires-python = ">=3.8"
 dependencies = [
   "authlib",
   "httpx",

{sfapi_client-0.4.0 → sfapi_client-0.5.0}/scripts/run.py

@@ -10,7 +10,6 @@ from typing import List, Optional
 
 from datamodel_code_generator import InputFileType, generate
 
-
 cli = typer.Typer()
 
 
@@ -24,6 +23,8 @@ def run_unasync():
         "AsyncApi": "Api",
         "AsyncResources": "Resources",
         "AsyncCompute": "Compute",
+        "AsyncCommandTask": "CommandTask",
+        "AsyncCommandTaskResult": "CommandTaskResult",
         "AsyncGroup": "Group",
         "AsyncGroupMember": "GroupMember",
         "AsyncRemotePath": "RemotePath",
@@ -34,6 +35,9 @@ def run_unasync():
         "AsyncJobSacct": "JobSacct",
         "AsyncJobSqueue": "JobSqueue",
         "AsyncOAuth2Client": "OAuth2Client",
+        "AsyncStorage": "Storage",
+        "AsyncGlobusStorage": "GlobusStorage",
+        "AsyncGlobusTransfer": "GlobusTransfer",
         "aclose": "close",
         "_ASYNC_SLEEP": "_SLEEP",
     }
@@ -100,7 +104,7 @@ def _to_str_enum(code: str) -> str:
 def _fix_date_import(code: str) -> str:
     replacements = {
         "import date": "import date as date_",
-        "date: Optional\[date\]": "date: Optional[date_]",
+        "date: Optional\\[date\\]": "date: Optional[date_]",
     }
 
     for target, replacement in replacements.items():
@@ -193,7 +197,7 @@ def resources_codegen(
         name = name if name != "int" else f"_{name}"
         name = name.replace("-", "_")
         names.append(f'    {name} = "{value}"')
-        names.append(f"    \"\"\" {s['system_type']}: {s['full_name']}\"\"\"")
+        names.append(f'    """ {s["system_type"]}: {s["full_name"]}"""')
 
     resources = "\n".join(names)
     now = datetime.datetime.now()

{sfapi_client-0.4.0 → sfapi_client-0.5.0}/src/sfapi_client/__init__.py

@@ -1,5 +1,6 @@
 from .client import AsyncClient  # noqa: F401
 from .client import Client  # noqa: F401
 from .exceptions import SfApiError  # noqa: F401
+from ._models import OutageStatusValue  # noqa: F401
 from ._models import StatusValue  # noqa: F401
 from ._models.resources import Resource  # noqa: F401

{sfapi_client-0.4.0 → sfapi_client-0.5.0}/src/sfapi_client/_async/client.py

@@ -7,7 +7,6 @@ from authlib.integrations.httpx_client.oauth2_client import AsyncOAuth2Client
 from authlib.oauth2.rfc7523 import PrivateKeyJWT
 import httpx
 import tenacity
-from authlib.jose import JsonWebKey
 
 from .compute import Machine, AsyncCompute
 from ..exceptions import ClientKeyError
@@ -21,6 +20,7 @@ from .._models import (
 from .._models.resources import Resource
 from .groups import AsyncGroup, AsyncGroupMember
 from .users import AsyncUser
+from .storage import AsyncGlobusStorage, AsyncStorage
 from .projects import AsyncProject, AsyncRole
 from .paths import AsyncRemotePath
 
@@ -252,6 +252,7 @@ class AsyncClient:
         self.__http_client = None
         self._api = None
         self._resources = None
+        self._storage = None
         self._wait_interval = wait_interval
         self._access_token = access_token
 
@@ -350,7 +351,7 @@ class AsyncClient:
             if key_path.suffix == ".json":
                 # Json file in the format {"client_id": "", "secret": ""}
                 json_web_key = json.loads(secret.read())
-                self._secret = JsonWebKey.import_key(json_web_key["secret"])
+                self._secret = json_web_key["secret"]
                 self._client_id = json_web_key["client_id"]
             else:
                 self._secret = secret.read()
@@ -504,8 +505,19 @@ class AsyncClient:
 
         return self._resources
 
+    @property
+    def storage(self) -> AsyncStorage:
+        """Storage related objects and methods
+
+        :return AsyncStorage: Object with storage related objects and methods
+        """
+        if self._storage is None:
+            self._storage = AsyncStorage(self)
+        return self._storage
+
 
 AsyncCompute.model_rebuild()
+AsyncGlobusStorage.model_rebuild()
 AsyncGroup.model_rebuild()
 AsyncUser.model_rebuild()
 AsyncProject.model_rebuild()
@@ -513,6 +525,7 @@ AsyncRemotePath.model_rebuild()
 AsyncRole.model_rebuild()
 AsyncGroupMember.model_rebuild()
 
+
 # Ensure that the job models are built, we need to import here to
 # avoid circular imports
 from .jobs import AsyncJobSacct, AsyncJobSqueue  # noqa: E402

{sfapi_client-0.4.0 → sfapi_client-0.5.0}/src/sfapi_client/_async/compute.py

@@ -1,13 +1,11 @@
-from typing import Dict, List, Optional, Union, Callable
-import json
-from pydantic import PrivateAttr, ConfigDict
+from typing import Dict, List, Optional, Union
+from pydantic import PrivateAttr, ConfigDict, BaseModel
 from ..exceptions import SfApiError
 from .._utils import _ASYNC_SLEEP
 from .jobs import AsyncJobSacct, AsyncJobSqueue, JobCommand
 from .._models import (
     AppRoutersStatusModelsStatus as ComputeBase,
-    Task,
-    StatusValue,
+    Task as TaskResponse,
     PublicHost as Machine,
     BodyRunCommandUtilitiesCommandMachinePost as RunCommandBody,
     AppRoutersComputeModelsCommandOutput as RunCommandResponse,
@@ -15,25 +13,85 @@ from .._models import (
 )
 from .paths import AsyncRemotePath
 from .._monitor import AsyncJobMonitor
-from .._compute import CommandResult, SubmitJobResponse, SubmitJobResponseStatus
+from .._compute import SubmitJobResponse, SubmitJobResponseStatus, TaskStatus
+from .._utils import check_auth
 
 # Patch to return str names from Enum of py3.11
 Machine.__str__ = lambda self: self.value
 
 
-def check_auth(method: Callable):
-    def wrapper(self, *args, **kwargs):
-        if self.client._client_id is None and self.client._access_token is None:
-            raise SfApiError(
-                f"Cannot call {self.__class__.__name__}.{method.__name__}() with an unauthenticated client."  # noqa: E501
-            )
-        elif self.status in [StatusValue.unavailable]:
-            raise SfApiError(
-                f"Compute resource {self.name} is {self.status.name}, {self.notes}"
-            )
-        return method(self, *args, **kwargs)
+class AsyncCommandTaskResult(BaseModel):
+    """
+    Result data returned by a command execution task.
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+    status: Optional[str] = None
+    output: Optional[str] = None
+    error: Optional[str] = None
+    exit_code: Optional[int] = None
+
 
-    return wrapper
+class AsyncCommandTask(BaseModel):
+    """
+    Models an asynchronous command execution task.
+    """
+
+    compute: "AsyncCompute"
+    id: str
+    status: Optional[TaskStatus] = None
+    result: Optional[AsyncCommandTaskResult] = None
+
+    async def _fetch(self) -> TaskResponse:
+        """
+        Fetch the latest task state from the API.
+
+        :return: The raw task response returned by the API.
+        """
+        r = await self.compute.client.get(f"tasks/{self.id}")
+        json_response = r.json()
+
+        return TaskResponse.model_validate(json_response)
+
+    async def update(self):
+        """
+        Refresh the task state.
+
+        :return: The updated task instance.
+        """
+        task = await self._fetch()
+        if task.status is not None:
+            # Map the status return by the API (lowercase string) to our TaskStatus enum ( uppercase )
+            # We do this for consistency as all our enums are uppercase
+            self.status = TaskStatus[task.status.upper()]
+
+        if task.result is not None:
+            self.result = AsyncCommandTaskResult.model_validate_json(task.result)
+
+        return self
+
+    async def _wait(self):
+        while True:
+            await self.update()
+
+            if self.status in [
+                TaskStatus.COMPLETED,
+                TaskStatus.CANCELLED,
+                TaskStatus.FAILED,
+            ]:
+                return self
+
+            await _ASYNC_SLEEP(1)
+
+    def __await__(self):
+        return self._wait().__await__()
+
+    async def cancel(self):
+        """
+        Cancel the running task.
+        """
+        await self.compute.client.delete(f"tasks/{self.id}")
 
 
 class AsyncCompute(ComputeBase):
@@ -51,18 +109,17 @@ class AsyncCompute(ComputeBase):
             kwargs["exclude"] = {"client"}
         return super().dict(*args, **kwargs)
 
-    async def _wait_for_task(self, task_id) -> str:
-        while True:
-            r = await self.client.get(f"tasks/{task_id}")
+    async def _wait_for_task(self, task_id) -> AsyncCommandTaskResult:
+
+        task = AsyncCommandTask(compute=self, id=task_id)
 
-            json_response = r.json()
-            task = Task.model_validate(json_response)
+        while True:
+            await task.update()
 
-            status = task.status.lower()
-            if status in ["error", "failed"]:
+            if task.status is TaskStatus.FAILED:
                 raise SfApiError(task.result)
 
-            if status not in ["completed", "cancelled"]:
+            if task.status not in [TaskStatus.COMPLETED, TaskStatus.CANCELLED]:
                 await _ASYNC_SLEEP(1)
                 continue
 
@@ -107,11 +164,11 @@ class AsyncCompute(ComputeBase):
 
         # We now need waiting for the task to complete!
         task_result = await self._wait_for_task(task_id)
-        result = json.loads(task_result)
-        if result.get("status") == "error":
-            raise SfApiError(result["error"])
+        if task_result.status == "error":
+            raise SfApiError(task_result.error)
 
-        jobid = result.get("jobid")
+        # Get the jobid from the extra fields of the result
+        jobid = task_result.__pydantic_extra__.get("jobid")
         if jobid is None:
             raise SfApiError(f"Unable to extract jobid if for task: {task_id}")
 
@@ -155,6 +212,17 @@ class AsyncCompute(ComputeBase):
 
     @check_auth
     async def run(self, args: Union[str, AsyncRemotePath, List[str]]):
+        task = await self._run_task(args)
+        await task._wait()
+
+        if task.result.error:
+            raise SfApiError(task.result.error)
+
+        return task.result.output
+
+    async def _run_task(
+        self, args: Union[str, AsyncRemotePath, List[str]]
+    ) -> AsyncCommandTask:
         body: RunCommandBody = {
             "executable": args if not isinstance(args, list) else " ".join(args)
         }
@@ -165,13 +233,19 @@ class AsyncCompute(ComputeBase):
         if run_response.status == RunCommandResponseStatus.ERROR:
             raise SfApiError(run_response.error)
 
-        task_id = run_response.task_id
-        task_result = await self._wait_for_task(task_id)
-        command_result = CommandResult.model_validate_json(task_result)
-        if command_result.status == "error":
-            raise SfApiError(command_result.error)
+        return AsyncCommandTask(compute=self, id=run_response.task_id)
 
-        return command_result.output
+    @check_auth
+    async def run_task(
+        self, args: Union[str, AsyncRemotePath, List[str]]
+    ) -> AsyncCommandTask:
+        """
+        Submit a command to the compute resource and return a task.
+
+        :param args: Command to execute.
+        :return: Task object that can be awaited ( async version ), updated, or cancelled.
+        """
+        return await self._run_task(args)
 
     async def outages(self):
         return await self.client.resources.outages(self.name)

{sfapi_client-0.4.0 → sfapi_client-0.5.0}/src/sfapi_client/_async/groups.py

@@ -1,21 +1,9 @@
-from typing import Optional, Union, List, Any, Callable
-from functools import wraps
+from typing import Optional, Union, List, Any
 from pydantic import ValidationError, Field, BaseModel, ConfigDict
 from .._models import BatchGroupAction as GroupAction, UserStats as GroupMemberBase
 from ..exceptions import SfApiError
 from .users import AsyncUser
-
-
-def check_auth(method: Callable):
-    @wraps(method)
-    def wrapper(self, *args, **kwargs):
-        if self._client_id is None:
-            raise SfApiError(
-                f"Cannot call {self.__class__.__name__}.{method.__name__}() with an unauthenticated client."  # noqa: E501
-            )
-        return method(self, *args, **kwargs)
-
-    return wrapper
+from .._utils import check_auth
 
 
 class AsyncGroupMember(GroupMemberBase):

{sfapi_client-0.4.0 → sfapi_client-0.5.0}/src/sfapi_client/_async/paths.py

@@ -5,6 +5,7 @@ from io import StringIO, BytesIO
 from base64 import b64decode
 from contextlib import asynccontextmanager
 import tempfile
+import re
 
 from .._models import (
     DirectoryEntry as PathBase,
@@ -22,6 +23,9 @@ def _is_no_such(error: SfApiError):
     return "No such file or directory" in error.message
 
 
+SYMLINK_REGEX = r"^(\/.*) ->"
+
+
 class AsyncRemotePath(PathBase):
     """
     RemotePath is used to model a remote path, it takes inspiration from
@@ -133,7 +137,16 @@ class AsyncRemotePath(PathBase):
         """
         :return: Returns True if path is a file, False otherwise.
         """
-        return not await self.is_dir()
+        if self.perms is None:
+            await self.update()
+
+        return self.perms[0] == "-"
+
+    async def is_symlink(self) -> bool:
+        if self.perms is None:
+            await self.update()
+
+        return self.perms[0] == "l"
 
     async def download(self, binary=False) -> IO[AnyStr]:
         """
@@ -148,7 +161,8 @@ class AsyncRemotePath(PathBase):
             raise IsADirectoryError(self._path)
 
         r = await self.compute.client.get(
-            f"utilities/download/{self.compute.name}/{self._path}?binary={binary}"
+            f"utilities/download/{self.compute.name}/{self._path}",
+            params={"binary": binary},
         )
         json_response = r.json()
         download_response = FileDownloadResponse.model_validate(json_response)
@@ -188,15 +202,22 @@ class AsyncRemotePath(PathBase):
 
             return p
 
-        # Special case for listing file
+        # Special case for listing file or symlink
         if len(directory_listing_response.entries) == 1:
             entry = directory_listing_response.entries[0]
-            # The API can add an extra /
-            path = entry.name
-            if entry.name.startswith("//"):
-                path = path[1:]
-            filename = PurePosixPath(path).name
-            entry.name = filename
+            # symlink
+            if entry.perms[0] == "l":
+                if match := re.search(SYMLINK_REGEX, entry.name):
+                    entry.name = match.group(1)
+            # file
+            else:
+                # The API can add an extra /
+                path = entry.name
+                if entry.name.startswith("//"):
+                    path = path[1:]
+                filename = PurePosixPath(path).name
+                entry.name = filename
+
             paths.append(_to_remote_path(path, entry))
         else:
             for entry in directory_listing_response.entries:
@@ -209,6 +230,10 @@ class AsyncRemotePath(PathBase):
                 elif directory and entry.name == ".":
                     entry.name = PurePosixPath(path).name
                     return [_to_remote_path(path, entry)]
+                # Special case for symlink
+                elif entry.perms[0] == "l":
+                    if match := re.search(SYMLINK_REGEX, entry.name):
+                        entry.name = match.group(1)
 
                 paths.append(_to_remote_path(f"{path}/{entry.name}", entry))
 
@@ -260,9 +285,9 @@ class AsyncRemotePath(PathBase):
             if not _is_no_such(ex):
                 raise
 
-            # Check if the parent is a directory ( as in we are creating a new file ),
+            # Check if the parent is a directory or symlink ( as in we are creating a new file ),
             # if not re raise the original exception
-            if not await self.parent.is_dir():
+            if not (await self.parent.is_dir() or await self.parent.is_symlink()):
                 raise
             else:
                 upload_path = str(self._path)
@@ -301,9 +326,9 @@ class AsyncRemotePath(PathBase):
             if not _is_no_such(ex):
                 raise
 
-            # Check if the parent is a directory ( as in we are creating a new file ),
+            # Check if the parent is a directory or symlink ( as in we are creating a new file ),
             # if not re raise the original exception
-            if not await self.parent.is_dir():
+            if not (await self.parent.is_dir() or await self.parent.is_symlink()):
                 raise
 
         valid_modes_chars = set("rwb")