dapi 0.5.0__tar.gz → 0.5.2__tar.gz

{dapi-0.5.0 → dapi-0.5.2}/.github/workflows/docs.yml

@@ -6,6 +6,9 @@ on:
       - main
   workflow_dispatch:
 
+env:
+  BASE_URL: /${{ github.event.repository.name }}
+
 permissions:
   contents: read
   pages: write

{dapi-0.5.0 → dapi-0.5.2}/.gitignore

@@ -180,3 +180,4 @@ cython_debug/
 *.sqlite
 
 .aider*
+uv.lock

dapi-0.5.2/AUTHORS.md

@@ -0,0 +1,5 @@
+# Authors
+
+- Prof. Krishna Kumar, University of Texas at Austin
+- Prof. Pedro Arduino, University of Washington
+- Prof. Scott Brandenberg, University of California Los Angeles

dapi-0.5.2/CHANGELOG.md

@@ -0,0 +1,74 @@
+# Changelog
+
+## v0.5.2
+
+### New features
+
+- **Publications module** (`ds.publications`): search and access published datasets on DesignSafe
+  - `ds.publications.list()`: DataFrame of all published datasets (1,500+)
+  - `ds.publications.search()`: filter by `query`, `pi`, `keyword`, `publication_type` (AND logic)
+  - `ds.publications.get("PRJ-XXXX")`: full metadata with DOIs, description, keywords
+  - `ds.publications.files("PRJ-XXXX")`: list files in a published dataset
+- **Systems listing** (`ds.systems.list()`): DataFrame of available HPC and storage systems
+  - Filter by category: `"hpc"`, `"storage"`, `"all"`
+  - Shows TMS credential status for HPC systems
+  - Filters out internal, duplicate, and project-specific systems
+- **`ds.systems.queues()`** now returns a clean DataFrame instead of printing verbose output
+
+### Documentation
+
+- New `docs/publications.md` with search filter reference
+- Updated `docs/systems.md` with `list()` and DataFrame queues
+- Added `examples/publications.ipynb` and `examples/systems.ipynb`
+- Updated examples sidebar with all new notebooks
+
+## v0.5.1
+
+### New features
+
+- **Projects module** (`ds.projects`): list, inspect, and access files in DesignSafe projects
+  - `ds.projects.list()`: returns a DataFrame of all projects you have access to
+  - `ds.projects.get("PRJ-XXXX")`: get full project metadata (title, PI, DOIs, keywords, team, systemId)
+  - `ds.projects.files("PRJ-XXXX")`: list files in a project as a DataFrame
+  - PRJ number to Tapis UUID resolution via DesignSafe portal API (`/api/projects/v2/`)
+- **NHERI-Published and NEES storage support**: `ds.files.to_uri()` and `ds.files.to_path()` now handle `/NHERI-Published/` (`designsafe.storage.published`) and `/NEES/` (`nees.public`) paths
+- **`ds.jobs.job(uuid)`**: get a `SubmittedJob` object for an existing job by UUID
+
+### Fixes
+
+- Fix `ds.files.list()` failing on root paths (e.g., `tapis://designsafe.storage.community/`) where the parsed path was empty
+- Fix project PRJ resolution: replaced broken Tapis system description search with DesignSafe portal API lookup
+
+### Documentation
+
+- New `docs/projects.md` with full API reference and "How it works" section
+- Updated `docs/files.md` with NHERI-Published and NEES path formats
+- Added development section to `docs/installation.md` (dev branch install, editable install, pre-commit hook, running tests)
+
+### Developer experience
+
+- Added `scripts/pre-commit` hook: auto-formats with `ruff format` and blocks commits failing `ruff check`
+- Added `examples/files.ipynb` and `examples/projects.ipynb`
+
+## v0.5.0
+
+### New features
+
+- **PyLauncher parameter sweeps** (`ds.jobs.parametric_sweep`): generate and submit parameter sweeps
+  - `ds.jobs.parametric_sweep.generate()`: generate `runsList.txt` and `call_pylauncher.py`, or preview as DataFrame
+  - `ds.jobs.parametric_sweep.submit()`: submit sweep jobs to TACC
+- **`ds.jobs.list()`**: list jobs with optional filtering by app_id and status, returns DataFrame by default
+- **Auto-TMS credentials**: `DSClient()` automatically sets up TMS credentials on TACC execution systems at init
+- **Ruff**: switched from black to ruff for formatting and linting
+
+### API changes
+
+- Renamed methods for brevity: `ds.jobs.generate()` (was `generate_job_info`), `ds.jobs.submit()` (was `submit_job`)
+- `ds.jobs.list()` supports `output="df"` (default), `"list"`, or `"raw"`
+- Added `ds.files.to_uri()` and `ds.files.to_path()` for path translation
+
+### Infrastructure
+
+- Migrated from Poetry to uv + hatchling
+- Migrated docs from mkdocs to Jupyter Book v2 (MyST)
+- Added TMS credential management (`ds.systems.establish_credentials()`, `check_credentials()`, `revoke_credentials()`)

{dapi-0.5.0 → dapi-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dapi
-Version: 0.5.0
+Version: 0.5.2
 Summary: DesignSafe API
 Author-email: Krishna Kumar <krishnak@utexas.edu>, Pedro Arduino <parduino@uw.edu>, Scott Brandenberg <sjbrandenberg@ucla.edu>
 License-Expression: MIT
@@ -188,6 +188,6 @@ jupyter-book start
 
 ## Authors
 
-- Krishna Kumar, University of Texas at Austin
+- Prof. Krishna Kumar, University of Texas at Austin
 - Prof. Pedro Arduino, University of Washington
 - Prof. Scott Brandenberg, University of California Los Angeles

{dapi-0.5.0 → dapi-0.5.2}/README.md

@@ -157,6 +157,6 @@ jupyter-book start
 
 ## Authors
 
-- Krishna Kumar, University of Texas at Austin
+- Prof. Krishna Kumar, University of Texas at Austin
 - Prof. Pedro Arduino, University of Washington
 - Prof. Scott Brandenberg, University of California Los Angeles

{dapi-0.5.0 → dapi-0.5.2}/dapi/__init__.py

@@ -49,7 +49,7 @@ from .jobs import (
     TAPIS_TERMINAL_STATES,
 )
 
-__version__ = "0.5.0"
+__version__ = "0.5.2"
 
 __all__ = [
     "DSClient",

{dapi-0.5.0 → dapi-0.5.2}/dapi/client.py

@@ -6,6 +6,8 @@ from . import files as files_module
 from . import jobs as jobs_module
 from . import systems as systems_module
 from . import launcher as launcher_module
+from . import projects as projects_module
+from . import publications as publications_module
 from .db.accessor import DatabaseAccessor
 
 # Import only the necessary classes/functions from jobs
@@ -31,6 +33,8 @@ class DSClient:
         apps (AppMethods): Interface for application discovery and details.
         files (FileMethods): Interface for file operations (upload, download, list).
         jobs (JobMethods): Interface for job submission and monitoring.
+        projects (ProjectMethods): Interface for DesignSafe project management.
+        publications (PublicationMethods): Interface for published datasets.
         systems (SystemMethods): Interface for system information and queues.
         db (DatabaseAccessor): Interface for database connections and queries.
 
@@ -90,6 +94,8 @@ class DSClient:
         self.apps = AppMethods(self.tapis)
         self.files = FileMethods(self.tapis)
         self.jobs = JobMethods(self.tapis)
+        self.projects = ProjectMethods(self.tapis)
+        self.publications = PublicationMethods(self.tapis)
         self.systems = SystemMethods(self.tapis)
         self.db = DatabaseAccessor()
 
@@ -250,6 +256,191 @@ class FileMethods:
         return files_module.list_files(self._tapis, *args, **kwargs)
 
 
+class ProjectMethods:
+    """Interface for DesignSafe project management.
+
+    Provides methods for listing projects, getting project metadata,
+    and listing files within projects.
+
+    Args:
+        tapis_client (Tapis): Authenticated Tapis client instance.
+    """
+
+    def __init__(self, tapis_client: Tapis):
+        self._tapis = tapis_client
+
+    def list(self, limit: int = 100, offset: int = 0, output: str = "df"):
+        """List DesignSafe projects you have access to.
+
+        Args:
+            limit (int, optional): Maximum projects to return. Defaults to 100.
+            offset (int, optional): Number of projects to skip. Defaults to 0.
+            output (str, optional): "df" for DataFrame (default), "list" for
+                list of dicts.
+
+        Returns:
+            DataFrame or List[Dict]: Projects with projectId, title, pi, type, etc.
+
+        Example:
+            >>> ds.projects.list()  # returns a DataFrame
+            >>> ds.projects.list(output="list")  # returns list of dicts
+        """
+        return projects_module.list_projects(
+            self._tapis, limit=limit, offset=offset, output=output
+        )
+
+    def get(self, project_id: str) -> Dict:
+        """Get detailed metadata for a project.
+
+        Args:
+            project_id (str): Project ID (e.g., "PRJ-1305").
+
+        Returns:
+            Dict: Project metadata including title, description, PI, team,
+                DOIs, award numbers, keywords, systemId.
+
+        Example:
+            >>> info = ds.projects.get("PRJ-6270")
+            >>> print(info["title"])
+            >>> print(info["systemId"])
+        """
+        return projects_module.get_project(self._tapis, project_id)
+
+    def files(
+        self, project_id: str, path: str = "/", limit: int = 100, output: str = "df"
+    ):
+        """List files in a project.
+
+        Args:
+            project_id (str): Project ID (e.g., "PRJ-1305").
+            path (str, optional): Path within the project. Defaults to "/".
+            limit (int, optional): Max items to return. Defaults to 100.
+            output (str, optional): "df" for DataFrame (default), "raw" for
+                Tapis file objects.
+
+        Returns:
+            DataFrame or List: Files with name, type, size, lastModified, path.
+
+        Example:
+            >>> ds.projects.files("PRJ-1305", "/Training/")
+            >>> ds.projects.files("PRJ-1305", output="raw")
+        """
+        return projects_module.list_project_files(
+            self._tapis, project_id, path=path, limit=limit, output=output
+        )
+
+
+class PublicationMethods:
+    """Interface for DesignSafe published datasets.
+
+    Provides methods for listing, searching, and inspecting published datasets
+    and their files on DesignSafe.
+
+    Args:
+        tapis_client (Tapis): Authenticated Tapis client instance.
+    """
+
+    def __init__(self, tapis_client: Tapis):
+        self._tapis = tapis_client
+
+    def list(self, limit: int = 100, offset: int = 0, output: str = "df"):
+        """List published datasets on DesignSafe.
+
+        Args:
+            limit (int, optional): Maximum publications to return. Defaults to 100.
+            offset (int, optional): Number to skip. Defaults to 0.
+            output (str, optional): "df" for DataFrame (default), "list" for dicts.
+
+        Returns:
+            DataFrame or List[Dict]: Publications with projectId, title, pi, type, keywords, created.
+
+        Example:
+            >>> ds.publications.list()
+        """
+        return publications_module.list_publications(
+            self._tapis, limit=limit, offset=offset, output=output
+        )
+
+    def search(
+        self,
+        query: Optional[str] = None,
+        *,
+        pi: Optional[str] = None,
+        keyword: Optional[str] = None,
+        publication_type: Optional[str] = None,
+        limit: int = 100,
+        output: str = "df",
+    ):
+        """Search published datasets with optional filters.
+
+        All filters are AND-combined and case-insensitive.
+
+        Args:
+            query (str, optional): General search across title, description, keywords, PI.
+            pi (str, optional): Filter by PI name.
+            keyword (str, optional): Filter by keyword.
+            publication_type (str, optional): Filter by type: "simulation",
+                "experimental", "field_recon", "other", "hybrid_simulation".
+            limit (int, optional): Max publications to fetch. Defaults to 100.
+            output (str, optional): "df" for DataFrame (default), "list" for dicts.
+
+        Returns:
+            DataFrame or List[Dict]: Matching publications.
+
+        Example:
+            >>> ds.publications.search("liquefaction")
+            >>> ds.publications.search(pi="Rathje")
+            >>> ds.publications.search(
+            ...     keyword="storm surge", publication_type="simulation"
+            ... )
+        """
+        return publications_module.search_publications(
+            self._tapis,
+            query,
+            pi=pi,
+            keyword=keyword,
+            publication_type=publication_type,
+            limit=limit,
+            output=output,
+        )
+
+    def get(self, project_id: str) -> Dict:
+        """Get detailed metadata for a published dataset.
+
+        Args:
+            project_id (str): Project ID (e.g., "PRJ-1271").
+
+        Returns:
+            Dict: Publication metadata including title, description, DOIs, keywords, PI.
+
+        Example:
+            >>> info = ds.publications.get("PRJ-6270")
+            >>> print(info["dois"])
+        """
+        return publications_module.get_publication(self._tapis, project_id)
+
+    def files(
+        self, project_id: str, path: str = "/", limit: int = 100, output: str = "df"
+    ):
+        """List files in a published dataset.
+
+        Args:
+            project_id (str): Project ID (e.g., "PRJ-1271").
+            path (str, optional): Path within the publication. Defaults to "/".
+            limit (int, optional): Max items to return. Defaults to 100.
+            output (str, optional): "df" for DataFrame (default), "raw" for Tapis objects.
+
+        Returns:
+            DataFrame or List: Files with name, type, size, lastModified, path.
+
+        Example:
+            >>> ds.publications.files("PRJ-1271")
+        """
+        return publications_module.list_publication_files(
+            self._tapis, project_id, path=path, limit=limit, output=output
+        )
+
+
 class SystemMethods:
     """Interface for Tapis system information and queue management.
 
@@ -268,27 +459,45 @@ class SystemMethods:
         """
         self._tapis = tapis_client
 
-    def queues(self, system_id: str, verbose: bool = True) -> List[Any]:
-        """List logical queues available on a Tapis execution system.
+    def list(self, category: Optional[str] = None, output: str = "df"):
+        """List Tapis systems you have access to.
 
-        This is a convenience wrapper around systems_module.list_system_queues().
+        Filters out internal and project-specific systems by default.
 
         Args:
-            system_id (str): The ID of the execution system (e.g., 'frontera').
-            verbose (bool, optional): If True, prints detailed queue information.
-                Defaults to True.
+            category (str, optional): "hpc" for execution systems,
+                "storage" for storage systems, "all" for everything,
+                None for HPC + storage (default).
+            output (str, optional): "df" for DataFrame (default), "list" for dicts.
 
         Returns:
-            List[Any]: List of queue objects with queue configuration details.
+            DataFrame or List[Dict]: Systems with id, host, category, authn, credentials.
 
-        Raises:
-            SystemInfoError: If the system is not found or queue retrieval fails.
-            ValueError: If system_id is empty.
+        Example:
+            >>> ds.systems.list()  # HPC + storage
+            >>> ds.systems.list("hpc")  # HPC only with credential status
+            >>> ds.systems.list("storage")  # Storage only
+            >>> ds.systems.list("all")  # Everything including internal
         """
-        return systems_module.list_system_queues(
-            self._tapis, system_id, verbose=verbose
+        return systems_module.list_systems(
+            self._tapis, category=category, output=output
         )
 
+    def queues(self, system_id: str, output: str = "df"):
+        """List batch queues available on a Tapis execution system.
+
+        Args:
+            system_id (str): The ID of the execution system (e.g., "stampede3").
+            output (str, optional): "df" for DataFrame (default), "raw" for Tapis objects.
+
+        Returns:
+            DataFrame or List: Queues with name, maxNodes, maxMinutes, etc.
+
+        Example:
+            >>> ds.systems.queues("stampede3")
+        """
+        return systems_module.list_system_queues(self._tapis, system_id, output=output)
+
     def check_credentials(self, system_id: str, username: str = None) -> bool:
         """Check whether TMS credentials exist for a user on a system.
 

{dapi-0.5.0 → dapi-0.5.2}/dapi/files.py

@@ -2,8 +2,6 @@
 import os
 import urllib.parse
 
-# No JWT needed if we rely on t.username
-# import jwt
 from tapipy.tapis import Tapis
 from tapipy.errors import BaseTapyException
 from .exceptions import FileOperationError, AuthenticationError
@@ -131,6 +129,16 @@ def tapis_uri_to_local_path(tapis_uri: str) -> str:
                 else "/home/jupyter/CommunityData/"
             )
 
+        elif system_id == "designsafe.storage.published":
+            return (
+                f"/home/jupyter/NHERI-Published/{path}"
+                if path
+                else "/home/jupyter/NHERI-Published/"
+            )
+
+        elif system_id == "nees.public":
+            return f"/home/jupyter/NEES/{path}" if path else "/home/jupyter/NEES/"
+
         elif system_id.startswith("project-"):
             # For Projects: tapis://project-*/path -> /home/jupyter/MyProjects/path
             return (
@@ -242,16 +250,44 @@ def get_ds_path_uri(t: Tapis, path: str, verify_exists: bool = False) -> str:
                 print(f"Translated '{path}' to '{input_uri}'")
                 break  # Found match, exit loop
 
-    # 3. Handle Project variations (if not already matched)
+    # 3. Handle NHERI-Published variations (if not already matched)
+    if input_uri is None:
+        published_patterns = [
+            ("jupyter/NHERI-Published", "designsafe.storage.published", False),
+            ("/NHERI-Published", "designsafe.storage.published", False),
+            ("NHERI-Published", "designsafe.storage.published", False),
+        ]
+        for pattern, storage_system_id, use_username in published_patterns:
+            if pattern in path:
+                path_remainder = path.split(pattern, 1)[1].lstrip("/")
+                input_uri = f"tapis://{storage_system_id}/{path_remainder}"
+                print(f"Translated '{path}' to '{input_uri}'")
+                break
+
+    # 4. Handle NEES variations (if not already matched)
+    if input_uri is None:
+        nees_patterns = [
+            ("jupyter/NEES", "nees.public", False),
+            ("/NEES", "nees.public", False),
+            ("NEES", "nees.public", False),
+        ]
+        for pattern, storage_system_id, use_username in nees_patterns:
+            if pattern in path:
+                path_remainder = path.split(pattern, 1)[1].lstrip("/")
+                input_uri = f"tapis://{storage_system_id}/{path_remainder}"
+                print(f"Translated '{path}' to '{input_uri}'")
+                break
+
+    # 5. Handle Project variations (if not already matched)
     if input_uri is None:
         project_patterns = [
-            ("jupyter/MyProjects", "project-"),
-            ("jupyter/projects", "project-"),
-            ("/projects", "project-"),
-            ("projects", "project-"),
-            ("/MyProjects", "project-"),
+            "jupyter/MyProjects",
+            "jupyter/projects",
+            "/projects",
+            "projects",
+            "/MyProjects",
         ]
-        for pattern, system_prefix in project_patterns:
+        for pattern in project_patterns:
             if pattern in path:
                 path_remainder_full = path.split(pattern, 1)[1].lstrip("/")
                 if not path_remainder_full:
@@ -262,73 +298,26 @@ def get_ds_path_uri(t: Tapis, path: str, verify_exists: bool = False) -> str:
                 project_id_part = parts[0]
                 path_within_project = parts[1] if len(parts) > 1 else ""
 
-                print(f"Searching Tapis systems for project ID '{project_id_part}'...")
-                found_system_id = None
-                try:
-                    search_query = (
-                        f"description.like.%{project_id_part}%&id.like.{system_prefix}*"
-                    )
-                    systems = t.systems.getSystems(
-                        search=search_query,
-                        listType="ALL",
-                        select="id,owner,description",
-                        limit=10,
-                    )
-                    matches = []
-                    if systems:
-                        for sys in systems:
-                            if (
-                                project_id_part.lower()
-                                in getattr(sys, "description", "").lower()
-                            ):
-                                matches.append(sys.id)
-                    if len(matches) == 1:
-                        found_system_id = matches[0]
-                        print(f"Found unique matching system: {found_system_id}")
-                    elif len(matches) == 0:
-                        if "-" in project_id_part and len(project_id_part) > 30:
-                            potential_sys_id = f"{system_prefix}{project_id_part}"
-                            print(
-                                f"Search failed, attempting direct lookup for system ID: {potential_sys_id}"
-                            )
-                            try:
-                                t.systems.getSystem(
-                                    systemId=potential_sys_id, select="id"
-                                )  # Select minimal field
-                                found_system_id = potential_sys_id
-                                print(f"Direct lookup successful: {found_system_id}")
-                            except BaseTapyException:
-                                print(
-                                    f"Direct lookup for {potential_sys_id} also failed."
-                                )
-                                raise FileOperationError(
-                                    f"No project system found matching ID '{project_id_part}' via Tapis v3 search or direct UUID lookup."
-                                )
-                        else:
-                            raise FileOperationError(
-                                f"No project system found matching ID '{project_id_part}' via Tapis v3 search."
-                            )
-                    else:
+                # If it looks like a UUID already, use it directly
+                if "-" in project_id_part and len(project_id_part) > 30:
+                    found_system_id = f"project-{project_id_part}"
+                    try:
+                        t.systems.getSystem(systemId=found_system_id, select="id")
+                    except BaseTapyException:
                         raise FileOperationError(
-                            f"Multiple project systems found potentially matching ID '{project_id_part}': {matches}. Cannot determine unique system."
+                            f"Project system '{found_system_id}' not found via direct lookup."
                         )
-                except BaseTapyException as e:
-                    raise FileOperationError(
-                        f"Tapis API error searching for project system '{project_id_part}': {e}"
-                    ) from e
-                except Exception as e:
-                    raise FileOperationError(
-                        f"Unexpected error searching for project system '{project_id_part}': {e}"
-                    ) from e
-
-                if not found_system_id:
-                    raise FileOperationError(
-                        f"Could not resolve project ID '{project_id_part}' to a Tapis system ID."
+                else:
+                    # Resolve PRJ number via DesignSafe projects API
+                    from . import projects as projects_module
+
+                    found_system_id = projects_module.resolve_project_uuid(
+                        t, project_id_part
                     )
 
                 input_uri = f"tapis://{found_system_id}/{path_within_project}"
-                print(f"Translated '{path}' to '{input_uri}' using Tapis v3 lookup")
-                break  # Found match, exit loop
+                print(f"Translated '{path}' to '{input_uri}'")
+                break
 
     # 4. Handle direct tapis:// URI input (if not already matched)
     if input_uri is None and path.startswith("tapis://"):
@@ -514,6 +503,8 @@ def list_files(
     """
     try:
         system_id, path = _parse_tapis_uri(remote_uri)
+        if not path:
+            path = "/"
         print(f"Listing files in system '{system_id}' at path '{path}'...")
         # URL-encode the path for API call
         encoded_path = _safe_quote(path)