dapi 0.2.0__py3-none-any.whl

dapi/__init__.py

@@ -0,0 +1,18 @@
+"""
+`dapi` is a library that simplifies the process of submitting, running, and monitoring [TAPIS v2 / AgavePy](https://agavepy.readthedocs.io/en/latest/index.html) jobs on [DesignSafe](https://designsafe-ci.org) via [Jupyter Notebooks](https://jupyter.designsafe-ci.org).
+
+
+## Features
+
+* Simplified TAPIS v2 Calls: No need to fiddle with complex API requests. `dapi` abstracts away the complexities.
+
+* Seamless Integration with DesignSafe Jupyter Notebooks: Launch DesignSafe applications directly from the Jupyter environment.
+
+## Installation
+
+```shell
+pip3 install dapi
+```
+
+"""
+from . import jobs

dapi/db/__init__.py

@@ -0,0 +1,2 @@
+name = "designsafe_db"
+from .db import DSDatabase

dapi/db/config.py

@@ -0,0 +1,6 @@
+# Mapping of shorthand names to actual database names and environment prefixes
+db_config = {
+    "ngl": {"dbname": "sjbrande_ngl_db", "env_prefix": "NGL_"},
+    "vp": {"dbname": "sjbrande_vpdb", "env_prefix": "VP_"},
+    "eq": {"dbname": "post_earthquake_recovery", "env_prefix": "EQ_"},
+}

dapi/db/db.py

@@ -0,0 +1,94 @@
+import os
+import pandas as pd
+from sqlalchemy import create_engine, exc
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy import text
+
+from .config import db_config
+
+
+class DSDatabase:
+    """A database utility class for connecting to a DesignSafe SQL database.
+
+    This class provides functionality to connect to a MySQL database using
+    SQLAlchemy and PyMySQL. It supports executing SQL queries and returning
+    results in different formats.
+
+    Attributes:
+        user (str): Database username, defaults to 'dspublic'.
+        password (str): Database password, defaults to 'R3ad0nlY'.
+        host (str): Database host address, defaults to '129.114.52.174'.
+        port (int): Database port, defaults to 3306.
+        db (str): Database name, can be 'sjbrande_ngl_db', 'sjbrande_vpdb', or 'post_earthquake_recovery'.
+        recycle_time (int): Time in seconds to recycle database connections.
+        engine (Engine): SQLAlchemy engine for database connection.
+        Session (sessionmaker): SQLAlchemy session maker bound to the engine.
+    """
+
+    def __init__(self, dbname="ngl"):
+        """Initializes the DSDatabase instance with environment variables and creates the database engine.
+
+        Args:
+            dbname (str): Shorthand for the database name. Must be one of 'ngl', 'vp', or 'eq'.
+        """
+
+        if dbname not in db_config:
+            raise ValueError(
+                f"Invalid database shorthand '{dbname}'. Allowed shorthands are: {', '.join(db_config.keys())}"
+            )
+
+        config = db_config[dbname]
+        env_prefix = config["env_prefix"]
+
+        self.user = os.getenv(f"{env_prefix}DB_USER", "dspublic")
+        self.password = os.getenv(f"{env_prefix}DB_PASSWORD", "R3ad0nlY")
+        self.host = os.getenv(f"{env_prefix}DB_HOST", "129.114.52.174")
+        self.port = os.getenv(f"{env_prefix}DB_PORT", 3306)
+        self.db = config["dbname"]
+
+        # Setup the database connection
+        self.engine = create_engine(
+            f"mysql+pymysql://{self.user}:{self.password}@{self.host}:{self.port}/{self.db}",
+            pool_recycle=3600,  # 1 hour in seconds
+        )
+        self.Session = sessionmaker(bind=self.engine)
+
+    def read_sql(self, sql, output_type="DataFrame"):
+        """Executes a SQL query and returns the results.
+
+        Args:
+            sql (str): The SQL query string to be executed.
+            output_type (str, optional): The format for the query results. Defaults to 'DataFrame'.
+                Possible values are 'DataFrame' for a pandas DataFrame, or 'dict' for a list of dictionaries.
+
+        Returns:
+            pandas.DataFrame or list of dict: The result of the SQL query.
+
+        Raises:
+            ValueError: If the SQL query string is empty or if the output type is not valid.
+            SQLAlchemyError: If an error occurs during query execution.
+        """
+        if not sql:
+            raise ValueError("SQL query string is required")
+
+        if output_type not in ["DataFrame", "dict"]:
+            raise ValueError('Output type must be either "DataFrame" or "dict"')
+
+        session = self.Session()
+
+        try:
+            if output_type == "DataFrame":
+                return pd.read_sql_query(sql, session.bind)
+            else:
+                # Convert SQL string to a text object
+                sql_text = text(sql)
+                result = session.execute(sql_text)
+                return [dict(row) for row in result]
+        except exc.SQLAlchemyError as e:
+            raise Exception(f"SQLAlchemyError: {e}")
+        finally:
+            session.close()
+
+    def close(self):
+        """Close the database connection."""
+        self.engine.dispose()

dapi/jobs/__init__.py

@@ -0,0 +1,19 @@
+"""
+`dapi` is a library that simplifies the process of submitting, running, and monitoring [TAPIS v2 / AgavePy](https://agavepy.readthedocs.io/en/latest/index.html) jobs on [DesignSafe](https://designsafe-ci.org) via [Jupyter Notebooks](https://jupyter.designsafe-ci.org).
+
+
+## Features
+
+* Simplified TAPIS v2 Calls: No need to fiddle with complex API requests. `dapi` abstracts away the complexities.
+
+* Seamless Integration with DesignSafe Jupyter Notebooks: Launch DesignSafe applications directly from the Jupyter environment.
+
+## Installation
+
+```shell
+pip3 install dapi
+```
+
+"""
+from .dir import get_ds_path_uri
+from .jobs import get_status, runtime_summary, generate_job_info, get_archive_path

dapi/jobs/dir.py

@@ -0,0 +1,51 @@
+import os
+
+
+def get_ds_path_uri(ag, path):
+    """
+    Given a path on DesignSafe, determine the correct input URI.
+
+    Args:
+        ag (object): Agave object to fetch profiles or metadata.
+        path (str): The directory path.
+
+    Returns:
+        str: The corresponding input URI.
+
+    Raises:
+        ValueError: If no matching directory pattern is found.
+    """
+
+    # If any of the following directory patterns are found in the path,
+    # process them accordingly.
+    directory_patterns = [
+        ("jupyter/MyData", "designsafe.storage.default", True),
+        ("jupyter/mydata", "designsafe.storage.default", True),
+        ("jupyter/CommunityData", "designsafe.storage.community", False),
+        ("/MyData", "designsafe.storage.default", True),
+        ("/mydata", "designsafe.storage.default", True),
+    ]
+
+    for pattern, storage, use_username in directory_patterns:
+        if pattern in path:
+            path = path.split(pattern).pop()
+            input_dir = ag.profiles.get()["username"] + path if use_username else path
+            input_uri = f"agave://{storage}/{input_dir}"
+            return input_uri.replace(" ", "%20")
+
+    project_patterns = [
+        ("jupyter/MyProjects", "project-"),
+        ("jupyter/projects", "project-"),
+    ]
+
+    for pattern, prefix in project_patterns:
+        if pattern in path:
+            path = path.split(pattern + "/").pop()
+            project_id = path.split("/")[0]
+            query = {"value.projectId": str(project_id)}
+            path = path.split(project_id).pop()
+            project_uuid = ag.meta.listMetadata(q=str(query))[0]["uuid"]
+            input_uri = f"agave://{prefix}{project_uuid}{path}"
+            return input_uri.replace(" ", "%20")
+
+    raise ValueError(f"No matching directory pattern found for: {path}")

dapi/jobs/jobs.py

@@ -0,0 +1,212 @@
+import time
+from datetime import datetime, timedelta, timezone
+from tqdm import tqdm
+import logging
+
+# Configuring the logging system
+# logging.basicConfig(
+#     level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+# )
+
+
+def get_status(ag, job_id, time_lapse=15):
+    """
+    Retrieves and monitors the status of a job from Agave.
+
+    This function initially waits for the job to start, displaying its progress using
+    a tqdm progress bar. Once the job starts, it monitors the job's status up to
+    a maximum duration specified by the job's "maxHours". If the job completes or fails
+    before reaching this maximum duration, it returns the job's final status.
+
+    Args:
+      ag (object): The Agave job object used to interact with the job.
+      job_id (str): The unique identifier of the job to monitor.
+      time_lapse (int, optional): Time interval, in seconds, to wait between status
+        checks. Defaults to 15 seconds.
+
+    Returns:
+      str: The final status of the job. Typical values include "FINISHED", "FAILED",
+           and "STOPPED".
+
+    Raises:
+      No exceptions are explicitly raised, but potential exceptions raised by the Agave
+      job object or other called functions/methods will propagate.
+    """
+
+    previous_status = None
+    # Initially check if the job is already running
+    status = ag.jobs.getStatus(jobId=job_id)["status"]
+
+    job_details = ag.jobs.get(jobId=job_id)
+    max_hours = job_details["maxHours"]
+
+    # Using tqdm to provide visual feedback while waiting for job to start
+    with tqdm(desc="Waiting for job to start", dynamic_ncols=True) as pbar:
+        while status not in ["RUNNING", "FINISHED", "FAILED", "STOPPED"]:
+            time.sleep(time_lapse)
+            status = ag.jobs.getStatus(jobId=job_id)["status"]
+            pbar.update(1)
+            pbar.set_postfix_str(f"Status: {status}")
+
+    # Once the job is running, monitor it for up to maxHours
+    max_iterations = int(max_hours * 3600 // time_lapse)
+
+    # Using tqdm for progress bar
+    for _ in tqdm(range(max_iterations), desc="Monitoring job", ncols=100):
+        status = ag.jobs.getStatus(jobId=job_id)["status"]
+
+        # Print status if it has changed
+        if status != previous_status:
+            tqdm.write(f"\tStatus: {status}")
+            previous_status = status
+
+        # Break the loop if job reaches one of these statuses
+        if status in ["FINISHED", "FAILED", "STOPPED"]:
+            break
+
+        time.sleep(time_lapse)
+    else:
+        # This block will execute if the for loop completes without a 'break'
+        logging.warn("Warning: Maximum monitoring time reached!")
+
+    return status
+
+
+def runtime_summary(ag, job_id, verbose=False):
+    """Get the runtime of a job.
+
+    Args:
+        ag (object): The Agave object that has the job details.
+        job_id (str): The ID of the job for which the runtime needs to be determined.
+        verbose (bool): If True, prints all statuses. Otherwise, prints only specific statuses.
+
+    Returns:
+        None: This function doesn't return a value, but it prints the runtime details.
+
+    """
+
+    print("Runtime Summary")
+    print("---------------")
+
+    job_history = ag.jobs.getHistory(jobId=job_id)
+    total_time = job_history[-1]["created"] - job_history[0]["created"]
+
+    status_times = {}
+
+    for i in range(len(job_history) - 1):
+        current_status = job_history[i]["status"]
+        elapsed_time = job_history[i + 1]["created"] - job_history[i]["created"]
+
+        # Aggregate times for each status
+        if current_status in status_times:
+            status_times[current_status] += elapsed_time
+        else:
+            status_times[current_status] = elapsed_time
+
+    # Filter the statuses if verbose is False
+    if not verbose:
+        filtered_statuses = {
+            "PENDING",
+            "QUEUED",
+            "RUNNING",
+            "FINISHED",
+            "FAILED",
+        }
+        status_times = {
+            status: time
+            for status, time in status_times.items()
+            if status in filtered_statuses
+        }
+
+    # Determine the max width of status names for alignment
+    max_status_width = max(len(status) for status in status_times.keys())
+
+    # Print the aggregated times for each unique status in a table format
+    for status, time in status_times.items():
+        print(f"{status.upper():<{max_status_width + 2}} time: {time}")
+
+    print(f"{'TOTAL':<{max_status_width + 2}} time: {total_time}")
+    print("---------------")
+
+
+def generate_job_info(
+    ag,
+    appid: str,
+    jobname: str = "dsjob",
+    queue: str = "development",
+    nnodes: int = 1,
+    nprocessors: int = 1,
+    runtime: str = "00:10:00",
+    inputs=None,
+    parameters=None,
+) -> dict:
+    """Generate a job information dictionary based on provided arguments.
+
+    Args:
+        ag (object): The Agave object to interact with the platform.
+        appid (str): The application ID for the job.
+        jobname (str, optional): The name of the job. Defaults to 'dsjob'.
+        queue (str, optional): The batch queue name. Defaults to 'skx-dev'.
+        nnodes (int, optional): The number of nodes required. Defaults to 1.
+        nprocessors (int, optional): The number of processors per node. Defaults to 1.
+        runtime (str, optional): The maximum runtime in the format 'HH:MM:SS'. Defaults to '00:10:00'.
+        inputs (dict, optional): The inputs for the job. Defaults to None.
+        parameters (dict, optional): The parameters for the job. Defaults to None.
+
+    Returns:
+        dict: A dictionary containing the job information.
+
+    Raises:
+        ValueError: If the provided appid is not valid.
+    """
+
+    try:
+        app = ag.apps.get(appId=appid)
+    except Exception:
+        raise ValueError(f"Invalid app ID: {appid}")
+
+    job_info = {
+        "appId": appid,
+        "name": jobname,
+        "batchQueue": queue,
+        "nodeCount": nnodes,
+        "processorsPerNode": nprocessors,
+        "memoryPerNode": "1",
+        "maxRunTime": runtime,
+        "archive": True,
+        "inputs": inputs,
+        "parameters": parameters,
+    }
+
+    return job_info
+
+
+def get_archive_path(ag, job_id):
+    """
+    Get the archive path for a given job ID and modifies the user directory
+    to '/home/jupyter/MyData'.
+
+    Args:
+        ag (object): The Agave object to interact with the platform.
+        job_id (str): The job ID to retrieve the archive path for.
+
+    Returns:
+        str: The modified archive path.
+
+    Raises:
+        ValueError: If the archivePath format is unexpected.
+    """
+
+    # Fetch the job info.
+    job_info = ag.jobs.get(jobId=job_id)
+
+    # Try to split the archive path to extract the user.
+    try:
+        user, _ = job_info.archivePath.split("/", 1)
+    except ValueError:
+        raise ValueError(f"Unexpected archivePath format for jobId={job_id}")
+
+    # Construct the new path.
+    new_path = job_info.archivePath.replace(user, "/home/jupyter/MyData")
+
+    return new_path

dapi-0.2.0.dist-info/LICENSE.md

@@ -0,0 +1,20 @@
+# MIT License
+> Copyright (c) [2023] [Authors]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dapi-0.2.0.dist-info/METADATA

@@ -0,0 +1,155 @@
+Metadata-Version: 2.1
+Name: dapi
+Version: 0.2.0
+Summary: dapi simplifies accessing TAPIS on DesignSafe
+Author: Krishna Kumar
+Author-email: krishnak@utexas.edu
+Requires-Python: >=3.9,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: agavepy (>0.9.5)
+Requires-Dist: exceptiongroup (>=1.1.3,<2.0.0)
+Requires-Dist: numpy (>=1.24.0,<2.0.0)
+Requires-Dist: pandas (>=2.1.3,<3.0.0)
+Requires-Dist: pymysql (>=1.1.0,<2.0.0)
+Requires-Dist: sqlalchemy (>=2.0.23,<3.0.0)
+Requires-Dist: tqdm (>=4.66.1,<5.0.0)
+Description-Content-Type: text/markdown
+
+# DesignSafe API (dapi)
+
+![dapi](dapi.png)
+
+[![build and test](https://github.com/DesignSafe-CI/dapi/actions/workflows/build-test.yml/badge.svg)](https://github.com/DesignSafe-CI/dapi/actions/workflows/build-test.yml)
+[![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE.md)
+[![Docs](https://img.shields.io/badge/view-docs-8A2BE2?color=8A2BE2)](https://designsafe-ci.github.io/dapi/dapi/index.html)
+
+`dapi` is a library that simplifies the process of submitting, running, and monitoring [TAPIS v2 / AgavePy](https://agavepy.readthedocs.io/en/latest/index.html) jobs on [DesignSafe](https://designsafe-ci.org) via [Jupyter Notebooks](https://jupyter.designsafe-ci.org).
+
+## Features
+
+### Jobs
+
+* Simplified TAPIS v2 Calls: No need to fiddle with complex API requests. `dapi` abstracts away the complexities.
+
+* Seamless Integration with DesignSafe Jupyter Notebooks: Launch DesignSafe applications directly from the Jupyter environment.
+
+### Database
+
+Connects to SQL databases on DesignSafe:
+
+| Database | dbname | env_prefix |
+|----------|--------|------------|
+| NGL | `ngl`| `NGL_` |
+| Earthake Recovery | `eq` | `EQ_` |
+| Vp | `vp` | `VP_` |
+
+Define the following environment variables:
+```
+{env_prefix}DB_USER
+{env_prefix}DB_PASSWORD
+{env_prefix}DB_HOST
+{env_prefix}DB_PORT
+```
+
+For e.g., to add the environment variable `NGL_DB_USER` edit `~/.bashrc`, `~/.zshrc`, or a similar shell-specific configuration file for the current user and add `export NGL_DB_USER="dspublic"`.
+
+
+## Installation
+
+Install `dapi` via pip
+
+```shell
+pip3 install dapi
+```
+
+To install the current development version of the library use:
+
+```shell
+pip install git+https://github.com/DesignSafe-CI/dapi.git --quiet
+```
+
+## Example usage:
+
+### Jobs
+
+* [Jupyter Notebook Templates](example-notebooks/template-mpm-run.ipynb) using dapi.
+
+* View [dapi API doc](https://designsafe-ci.github.io/dapi/dapi/index.html)
+
+On [DesignSafe Jupyter](https://jupyter.designsafe-ci.org/):
+
+Install the latest version of `dapi` and restart the kernel (Kernel >> Restart Kernel):
+
+```python
+# Remove any previous installations
+!pip uninstall dapi -y
+# Install 
+!pip install dapi --quiet
+```
+
+* Import `dapi` library
+```python
+import dapi
+```
+
+* To list all functions in `dapi`
+```python
+dir(dapi)
+```
+
+### Database
+```python
+import dapi
+
+db = dapi.DSDatabase("ngl")
+sql = 'SELECT * FROM SITE'
+df = db.read_sql(sql)
+print(df)
+
+# Optionally, close the database connection when done
+db.close()
+```
+
+## Documentation
+
+View [dapi API doc](https://designsafe-ci.github.io/dapi/dapi/index.html)
+
+To generate API docs:
+
+```
+pdoc --html --output-dir docs dapi --force
+```
+
+## Support
+
+For any questions, issues, or feedback submit an [issue](https://github.com/DesignSafe-CI/dapi/issues/new)
+
+## Development
+
+To develop or test the library locally. Install [Poetry](https://python-poetry.org/docs/#installation). In the current repository run the following commands
+
+```shell
+poetry shell
+poetry install
+poetry build
+```
+
+To run the unit test
+```shell
+poetry run pytest -v
+```
+
+
+## License
+
+`dapi` is licensed under the [MIT License](LICENSE.md).
+
+## Authors
+
+* Krishna Kumar, University of Texas at Austin
+* Prof. Pedro Arduino, University of Washington
+* Prof. Scott Brandenberg, University of California Los Angeles

dapi-0.2.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+dapi/__init__.py,sha256=SKEAGaKxpRYafsqxol5RYep4VLujzYLDofzYtZ_AFBI,604
+dapi/db/__init__.py,sha256=68vCJOTiQSIyGnlfF8__ufayGP52ItELF_TbPcZaPUk,50
+dapi/db/config.py,sha256=wkaDhkV5CS1qcaKGo5GHXU34X8gUtwLPdd-PKlvNHFA,290
+dapi/db/db.py,sha256=3brwLgealA00dgabv8O08Vog0gDIkkHPmTbGcxpj10Q,3701
+dapi/jobs/__init__.py,sha256=e22e_EEaaWSSOMzA6ur1VyKqfMghn6NSwHmCNGwSsiw,701
+dapi/jobs/dir.py,sha256=u2EtsHHotB7c-h-3QbMvLuGkZIxtrmTRajb0VWmqEH8,1807
+dapi/jobs/jobs.py,sha256=_RQSetuLqdvCOLOVD5IkvUDna1iEbDdE_TKw9zj5TwM,7002
+dapi-0.2.0.dist-info/LICENSE.md,sha256=BAQUrW-janfTWmXxSfvvnUvnTfS5qSEw_vVJn-SW3nE,1071
+dapi-0.2.0.dist-info/METADATA,sha256=V79-jDXsF_TrBIkQIMIyySZknY5YeyzF1zJT2NA2g7A,4047
+dapi-0.2.0.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+dapi-0.2.0.dist-info/RECORD,,

dapi-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 1.8.1
+Root-Is-Purelib: true
+Tag: py3-none-any