gdexws 0.1.0__tar.gz

gdexws-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 University Corporation for Atmospheric Research (UCAR)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

gdexws-0.1.0/PKG-INFO

@@ -0,0 +1,61 @@
+Metadata-Version: 2.4
+Name: gdexws
+Version: 0.1.0
+Summary: GDEX web service tools for data processing and management
+Author: GDEX Team
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/NCAR/gdex-web-services
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: netCDF4>=1.5.0
+Requires-Dist: httpx>=0.24.0
+Dynamic: license-file
+
+# gdexws
+
+Service tools for data processing and management.
+
+## Installation
+
+Install the package in development mode:
+
+```bash
+pip install -e .
+```
+
+Or install normally:
+
+```bash
+pip install .
+```
+
+## CLI Commands
+
+### add-global-meta
+
+Add a global attribute to a netCDF file.
+
+```bash
+add-global-meta -file /path/to/file.nc -global_attr_name "attribute_name" -global_attr_value "attribute_value"
+```
+
+### create-exchange-dir
+
+Create a directory directly under `/glade/campaign/collections/gdex/data/exchange` and grant an HPC user read/write access (`rwx`, plus a default ACL so new contents inherit it) via POSIX ACLs. Requires `setfacl`/`getfacl`.
+
+```bash
+create-exchange-dir <username> <dirname>
+```
+
+The directory name must be a single path component (`[A-Za-z0-9._-]`, starting with a letter or digit), the user must exist on the machine (and not be uid 0), and the command fails if the directory already exists. If the ACL cannot be applied and verified, the new directory is removed.
+
+## Development
+
+To install with development dependencies:
+
+```bash
+pip install -e ".[dev]"
+```

gdexws-0.1.0/README.md

@@ -0,0 +1,45 @@
+# gdexws
+
+Service tools for data processing and management.
+
+## Installation
+
+Install the package in development mode:
+
+```bash
+pip install -e .
+```
+
+Or install normally:
+
+```bash
+pip install .
+```
+
+## CLI Commands
+
+### add-global-meta
+
+Add a global attribute to a netCDF file.
+
+```bash
+add-global-meta -file /path/to/file.nc -global_attr_name "attribute_name" -global_attr_value "attribute_value"
+```
+
+### create-exchange-dir
+
+Create a directory directly under `/glade/campaign/collections/gdex/data/exchange` and grant an HPC user read/write access (`rwx`, plus a default ACL so new contents inherit it) via POSIX ACLs. Requires `setfacl`/`getfacl`.
+
+```bash
+create-exchange-dir <username> <dirname>
+```
+
+The directory name must be a single path component (`[A-Za-z0-9._-]`, starting with a letter or digit), the user must exist on the machine (and not be uid 0), and the command fails if the directory already exists. If the ACL cannot be applied and verified, the new directory is removed.
+
+## Development
+
+To install with development dependencies:
+
+```bash
+pip install -e ".[dev]"
+```

gdexws-0.1.0/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

gdexws-0.1.0/composers/transform.py

@@ -0,0 +1,128 @@
+"""Transform script that executes commands on files based on a payload JSON."""
+import sys
+import argparse
+from gdexws.utils import load_payload, build_command, execute_command, service_log
+
+
+def transform(payload_path: str) -> None:
+    """
+    Main transform function that processes files 
+    based on the commands listed in the payload configuration.
+
+    payload format:
+    {
+        "Files": ["Web-services/test.nc"],
+        "Commands": [
+            {
+            "command": "add_global_meta",
+            "global-attr-name": "gdex_dsid",
+            "global-attr-value": "dPayLoadTest",
+            "debug": true
+            }
+        ]
+    }
+
+
+    Parameters
+    ----------
+    payload_path : str
+        Path to the payload JSON file
+    """
+    # Load the payload JSON
+    try:
+        payload = load_payload(payload_path)
+    except FileNotFoundError:
+        service_log(
+            command_name="transform",
+            level="ERROR",
+            process_message="Payload file not found",
+            payload_path=payload_path
+        )
+        sys.exit(1)
+    except Exception as e:
+        service_log(
+            command_name="transform",
+            level="ERROR",
+            process_message="Failed to load payload",
+            error=str(e)
+        )
+        sys.exit(1)
+
+    # Parse files and commands from the payload
+    files = payload.get("Files", [])
+    commands = payload.get("Commands", [])
+
+    if not files:
+        service_log(
+            command_name="transform",
+            level="ERROR",
+            process_message="No files specified in payload"
+        )
+        sys.exit(1)
+
+    if not commands:
+        service_log(
+            command_name="transform",
+            level="ERROR",
+            process_message="No commands specified in payload"
+        )
+        sys.exit(1)
+
+    # Start processing each file and loop through the commands
+    for file_path in files:
+        service_log(
+            command_name="transform",
+            level="INFO",
+            process_message="Processing file",
+            file_path=file_path
+        )
+
+        for command_spec in commands:
+            # Extract command name and parameters
+            if "command" not in command_spec:
+                service_log(
+                    command_name="transform",
+                    level="ERROR",
+                    process_message=f"Command spec missing 'command' key: {command_spec}"
+                )
+                sys.exit(1)
+
+            command_name = command_spec["command"]
+
+            # Convert underscore to hyphen for CLI command name
+            cli_command = command_name.replace("_", "-")
+
+            # Build the full command
+            cmd = build_command(cli_command, command_spec, file_path)
+
+            # Execute the command
+            return_code = execute_command(cmd)
+
+            if return_code != 0:
+                service_log(
+                    command_name="transform",
+                    level="ERROR",
+                    process_message=f"Command failed with return code {return_code}",
+                    command=" ".join(cmd)
+                )
+                sys.exit(1)
+
+def main():
+    """CLI entry point."""
+    parser = argparse.ArgumentParser(
+        description="Execute commands on files based on a payload JSON"
+    )
+    parser.add_argument("-p", "--payload", help="Path to the payload JSON file", required=True)
+    args = parser.parse_args()
+
+    service_log(
+        command_name="transform",
+        level="INFO",
+        process_message="composer.transform started",
+        payload_path=args.payload
+    )
+    transform(args.payload)
+
+
+if __name__ == "__main__":
+    main()

gdexws-0.1.0/gdexws.egg-info/PKG-INFO

@@ -0,0 +1,61 @@
+Metadata-Version: 2.4
+Name: gdexws
+Version: 0.1.0
+Summary: GDEX web service tools for data processing and management
+Author: GDEX Team
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/NCAR/gdex-web-services
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: netCDF4>=1.5.0
+Requires-Dist: httpx>=0.24.0
+Dynamic: license-file
+
+# gdexws
+
+Service tools for data processing and management.
+
+## Installation
+
+Install the package in development mode:
+
+```bash
+pip install -e .
+```
+
+Or install normally:
+
+```bash
+pip install .
+```
+
+## CLI Commands
+
+### add-global-meta
+
+Add a global attribute to a netCDF file.
+
+```bash
+add-global-meta -file /path/to/file.nc -global_attr_name "attribute_name" -global_attr_value "attribute_value"
+```
+
+### create-exchange-dir
+
+Create a directory directly under `/glade/campaign/collections/gdex/data/exchange` and grant an HPC user read/write access (`rwx`, plus a default ACL so new contents inherit it) via POSIX ACLs. Requires `setfacl`/`getfacl`.
+
+```bash
+create-exchange-dir <username> <dirname>
+```
+
+The directory name must be a single path component (`[A-Za-z0-9._-]`, starting with a letter or digit), the user must exist on the machine (and not be uid 0), and the command fails if the directory already exists. If the ACL cannot be applied and verified, the new directory is removed.
+
+## Development
+
+To install with development dependencies:
+
+```bash
+pip install -e ".[dev]"
+```

gdexws-0.1.0/gdexws.egg-info/SOURCES.txt

@@ -0,0 +1,31 @@
+LICENSE
+README.md
+__init__.py
+pyproject.toml
+./__init__.py
+./composers/__init__.py
+./composers/transform.py
+./tools/__init__.py
+./tools/add_global_meta.py
+./tools/create_exchange_dir.py
+./utils/__init__.py
+./utils/file_validation.py
+./utils/logging.py
+./utils/parse_payload.py
+composers/__init__.py
+composers/transform.py
+gdexws.egg-info/PKG-INFO
+gdexws.egg-info/SOURCES.txt
+gdexws.egg-info/dependency_links.txt
+gdexws.egg-info/entry_points.txt
+gdexws.egg-info/requires.txt
+gdexws.egg-info/scm_file_list.json
+gdexws.egg-info/top_level.txt
+pbs/transform.pbs
+tools/__init__.py
+tools/add_global_meta.py
+tools/create_exchange_dir.py
+utils/__init__.py
+utils/file_validation.py
+utils/logging.py
+utils/parse_payload.py

gdexws-0.1.0/gdexws.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

gdexws-0.1.0/gdexws.egg-info/entry_points.txt

@@ -0,0 +1,4 @@
+[console_scripts]
+add-global-meta = gdexws.tools.add_global_meta:main
+create-exchange-dir = gdexws.tools.create_exchange_dir:main
+transform = gdexws.composers.transform:main

gdexws-0.1.0/gdexws.egg-info/requires.txt

@@ -0,0 +1,2 @@
+netCDF4>=1.5.0
+httpx>=0.24.0

gdexws-0.1.0/gdexws.egg-info/scm_file_list.json

@@ -0,0 +1,18 @@
+{
+  "files": [
+    "LICENSE",
+    "README.md",
+    "__init__.py",
+    "composers/__init__.py",
+    "composers/transform.py",
+    "pbs/transform.pbs",
+    "pyproject.toml",
+    "tools/__init__.py",
+    "tools/add_global_meta.py",
+    "tools/create_exchange_dir.py",
+    "utils/__init__.py",
+    "utils/file_validation.py",
+    "utils/logging.py",
+    "utils/parse_payload.py"
+  ]
+}

gdexws-0.1.0/gdexws.egg-info/top_level.txt

@@ -0,0 +1 @@
+gdexws

gdexws-0.1.0/pbs/transform.pbs

@@ -0,0 +1,49 @@
+#!/bin/bash -l
+
+### Job Name
+#PBS -N gdex-web-service-transform
+### Charging account
+#PBS -A P43713000
+### Request one chunk of resources with 1 CPU and 10 GB of memory
+#PBS -l select=1:ncpus=1:mem=1GB
+### Allow job to run up to 1 hour 30 minutes
+#PBS -l walltime=00:05:00
+### Route the job to the casper queue
+#PBS -q gdex
+### Join output and error streams into single file
+#PBS -j oe
+
+# Environment Management
+module --force purge
+# module load ncarenv/24.12
+# module load conda
+# conda activate /glade/work/gdexdata/conda-envs/pg-gdex
+source /glade/u/home/chiaweih/gdex-web-services/test-gdexws-env/bin/activate
+
+### Parse positional arguments
+PAYLOAD="$1"
+# PAYLOAD="/glade/u/home/chiaweih/gdex-web-services/tests/test.payload.json"
+# CINDEX="1"
+
+### SETUP .jsonl
+jsonl_dir="/glade/campaign/collections/gdex/data/exchange/Web-services/"
+output_jsonl="$jsonl_dir/$CINDEX.gdexws.jsonl"
+
+# Clear or create file
+> "$output_jsonl"
+
+# Shell start message
+time_iso=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+echo '{"command": "pbs", "time_of_process": "'"$time_iso"'", "level": "INFO", "process_message": "PBS job started"}' >> "$output_jsonl"
+
+# Execute transform (the payload can be Boreas link)
+transform -p "$PAYLOAD" >> "$output_jsonl"
+EXIT_CODE=$?
+
+# Shell end message
+time_iso=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+if [ $EXIT_CODE -eq 0 ]; then
+    echo '{"command": "pbs", "time_of_process": "'"$time_iso"'", "level": "INFO", "process_message": "PBS job completed"}' >> "$output_jsonl"
+else
+    echo '{"command": "pbs", "time_of_process": "'"$time_iso"'", "level": "ERROR", "process_message": "PBS job failed", "exit_code": '"$EXIT_CODE"'}' >> "$output_jsonl"
+fi

gdexws-0.1.0/pyproject.toml

@@ -0,0 +1,41 @@
+[build-system]
+requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "gdexws"
+dynamic = ["version"]
+description = "GDEX web service tools for data processing and management"
+readme = "README.md"
+requires-python = ">=3.8"
+license = "MIT"
+authors = [
+    {name = "GDEX Team"},
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+
+dependencies = [
+    "netCDF4>=1.5.0",
+    "httpx>=0.24.0",
+]
+
+[project.scripts]
+add-global-meta = "gdexws.tools.add_global_meta:main"
+create-exchange-dir = "gdexws.tools.create_exchange_dir:main"
+transform = "gdexws.composers.transform:main"
+
+[project.urls]
+Homepage = "https://github.com/NCAR/gdex-web-services"
+
+[tool.setuptools]
+packages = ["gdexws", "gdexws.tools", "gdexws.utils", "gdexws.composers"]
+package-dir = {"gdexws" = "."}
+
+[tool.setuptools_scm]
+root = ".."
+local_scheme = "no-local-version"
+fallback_version = "0.1.0"
+

gdexws-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

gdexws-0.1.0/tools/add_global_meta.py

@@ -0,0 +1,45 @@
+"""Add global attribute to a netCDF file."""
+import os
+import time
+import argparse
+import netCDF4
+from gdexws.utils import service_log
+from gdexws.utils.file_validation import relpath_validate
+
+def add_global_meta(file_path, key, value, debug=False):
+    """Add a global attribute to a netCDF file."""
+    full_path = relpath_validate([file_path])[0]  # Validate the file path
+    if debug:
+        service_log("add-global-meta", "DEBUG", f"Adding global attribute {key}={value} to {file_path}")
+
+    with netCDF4.Dataset(full_path, mode="r+") as ds:
+        ds.setncattr(key, value)
+
+    for i in range(6):
+        time.sleep(10)
+        if debug:
+            service_log("add-global-meta", "DEBUG" , "Loop iteration", iteration=i)
+
+    if debug:
+        service_log("add-global-meta", "DEBUG", "Process completed")
+
+def main():
+    """CLI entry point."""
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    script_name = os.path.basename(__file__)
+
+    parser = argparse.ArgumentParser(description="Add a global attribute to a netCDF file")
+    parser.add_argument("-f", "--file", help="Path to the netCDF file", required=True)
+    parser.add_argument("-n", "--global-attr-name", help="Name of the global attribute to add", required=True)
+    parser.add_argument("-v", "--global-attr-value", help="Value of the global attribute to add", required=True)
+    parser.add_argument("-d", "--debug", action="store_true", help="Enable debug mode")
+    args = parser.parse_args()
+
+    if args.debug:
+        service_log("add-global-meta", "DEBUG", "Beginning process")
+        service_log("add-global-meta", "DEBUG", "Script info", script_dir=script_dir, script_name=script_name)
+
+    add_global_meta(args.file, args.global_attr_name, args.global_attr_value, args.debug)
+
+if __name__ == "__main__":
+    main()

gdexws-0.1.0/tools/create_exchange_dir.py

@@ -0,0 +1,267 @@
+"""Create a directory under the GDEX exchange area and grant a user read/write access via POSIX ACLs."""
+import os
+import re
+import pwd
+import shutil
+import argparse
+import subprocess
+import sys
+
+from gdexws.utils import service_log
+
+COMMAND_NAME = "create-exchange-dir"
+
+# The only location under which this tool will ever create a directory.
+EXCHANGE_DIR = "/glade/campaign/collections/gdex/data/exchange"
+
+# A single, plain path component: must start with an alphanumeric (no hidden
+# dirs, no leading '-'), then only alphanumerics, '.', '_' or '-'.
+_DIRNAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
+_DIRNAME_MAX_LEN = 255
+
+# POSIX portable username: lowercase letters, digits, '_' and '-', max 32 chars.
+_USERNAME_RE = re.compile(r"^[a-z_][a-z0-9_-]{0,31}$")
+
+# rwx: 'x' is required on a directory for the user to traverse into it.
+_ACL_PERMS = "rwx"
+_DIR_MODE = 0o770
+
+
+class ExchangeDirError(Exception):
+    """Raised when the exchange directory cannot be safely created or shared."""
+
+
+def validate_dirname(dirname):
+    """Validate that ``dirname`` is a single, safe path component.
+
+    Parameters
+    ----------
+    dirname : str
+        Requested directory name.
+
+    Returns
+    -------
+    str
+        The validated directory name, unchanged.
+
+    Raises
+    ------
+    ExchangeDirError
+        If the name is empty, too long, contains a path separator, NUL, or
+        characters outside ``[A-Za-z0-9._-]``, starts with a non-alphanumeric
+        character, or is ``.``/``..``.
+    """
+    if not isinstance(dirname, str) or not dirname:
+        raise ExchangeDirError("Directory name must be a non-empty string")
+    if dirname in (".", ".."):
+        raise ExchangeDirError(f"Invalid directory name: '{dirname}'")
+    if len(dirname) > _DIRNAME_MAX_LEN:
+        raise ExchangeDirError(f"Directory name exceeds {_DIRNAME_MAX_LEN} characters")
+    if "/" in dirname or "\x00" in dirname or os.sep in dirname:
+        raise ExchangeDirError("Directory name must be a single path component (no '/' allowed)")
+    if not _DIRNAME_RE.fullmatch(dirname):
+        raise ExchangeDirError(
+            "Directory name must start with a letter or digit and contain only "
+            "letters, digits, '.', '_' and '-'"
+        )
+    return dirname
+
+
+def validate_username(username):
+    """Validate the username format and verify the account exists on this machine.
+
+    Parameters
+    ----------
+    username : str
+        HPC username.
+
+    Returns
+    -------
+    str
+        The validated username, unchanged.
+
+    Raises
+    ------
+    ExchangeDirError
+        If the username is malformed, does not exist in the system user
+        database, or resolves to root (uid 0).
+    """
+    if not isinstance(username, str) or not _USERNAME_RE.fullmatch(username):
+        raise ExchangeDirError(f"Invalid username format: '{username}'")
+    try:
+        entry = pwd.getpwnam(username)
+    except KeyError:
+        raise ExchangeDirError(f"User '{username}' does not exist on this machine")
+    if entry.pw_uid == 0:
+        raise ExchangeDirError("Refusing to grant an ACL to uid 0")
+    return username
+
+
+def _resolve_exchange_dir(exchange_dir):
+    """Return the real path of the exchange directory, verifying it is a directory.
+
+    Parameters
+    ----------
+    exchange_dir : str
+        Configured exchange directory.
+
+    Returns
+    -------
+    str
+        Fully resolved (symlink-free) absolute path.
+
+    Raises
+    ------
+    ExchangeDirError
+        If the path does not exist or is not a directory.
+    """
+    base = os.path.realpath(exchange_dir)
+    if not os.path.isdir(base):
+        raise ExchangeDirError(f"Exchange directory does not exist: {exchange_dir}")
+    return base
+
+
+def _run_acl_tool(args):
+    """Run an ACL tool with a fixed argument list (no shell).
+
+    Parameters
+    ----------
+    args : list of str
+        Command and arguments; ``args[0]`` is looked up on ``PATH``.
+
+    Returns
+    -------
+    str
+        The command's stdout.
+
+    Raises
+    ------
+    ExchangeDirError
+        If the tool is missing or exits non-zero.
+    """
+    exe = shutil.which(args[0])
+    if exe is None:
+        raise ExchangeDirError(f"Required tool '{args[0]}' was not found on PATH")
+    result = subprocess.run([exe] + args[1:], capture_output=True, text=True, check=False)
+    if result.returncode != 0:
+        raise ExchangeDirError(f"{args[0]} failed: {result.stderr.strip()}")
+    return result.stdout
+
+
+def _grant_acl(path, username):
+    """Grant ``username`` rwx on ``path`` (access and default ACL) and verify it took effect.
+
+    Parameters
+    ----------
+    path : str
+        Directory to modify.
+    username : str
+        User to add to the ACL.
+
+    Raises
+    ------
+    ExchangeDirError
+        If setfacl fails or the entries cannot be confirmed with getfacl.
+    """
+    # -P: never follow symlinks; '--' ends option parsing so the path cannot be read as a flag.
+    _run_acl_tool([
+        "setfacl", "-P",
+        "-m", f"u:{username}:{_ACL_PERMS},d:u:{username}:{_ACL_PERMS}",
+        "--", path,
+    ])
+    listing = _run_acl_tool(["getfacl", "--absolute-names", "--", path])
+    # Drop trailing "#effective:..." comments before comparing entries.
+    entries = {line.split("#")[0].strip() for line in listing.splitlines()}
+    expected = {f"user:{username}:{_ACL_PERMS}", f"default:user:{username}:{_ACL_PERMS}"}
+    if not expected <= entries:
+        raise ExchangeDirError(f"ACL verification failed for '{username}' on {path}")
+
+
+def create_exchange_dir(username, dirname, exchange_dir=EXCHANGE_DIR, debug=False):
+    """Create ``dirname`` under the exchange directory and give ``username`` rwx via ACL.
+
+    The directory must not already exist. If the ACL step fails, the newly
+    created (still empty) directory is removed.
+
+    Parameters
+    ----------
+    username : str
+        HPC username that must exist on this machine.
+    dirname : str
+        Name of the single directory to create directly under ``exchange_dir``.
+    exchange_dir : str
+        Root under which directories may be created. Default: ``EXCHANGE_DIR``.
+    debug : bool
+        Emit DEBUG log lines.
+
+    Returns
+    -------
+    str
+        Absolute path of the created directory.
+
+    Raises
+    ------
+    ExchangeDirError
+        On any validation, creation, or ACL failure.
+    """
+    validate_dirname(dirname)
+    validate_username(username)
+    base = _resolve_exchange_dir(exchange_dir)
+    target = os.path.join(base, dirname)
+
+    # Defense in depth: the lexical join must still sit directly under base.
+    if os.path.dirname(os.path.normpath(target)) != base:
+        raise ExchangeDirError("Target path is not directly inside the exchange directory")
+    if os.path.lexists(target):
+        raise ExchangeDirError(f"'{dirname}' already exists in the exchange directory")
+
+    if debug:
+        service_log(COMMAND_NAME, "DEBUG", "Creating directory", path=target, username=username)
+
+    # mkdir relative to an open fd on the base dir: atomic, fails with EEXIST if
+    # anything (including a symlink) already occupies the name, and is not
+    # affected by the base path being swapped after the checks above.
+    base_fd = os.open(base, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        try:
+            os.mkdir(dirname, mode=_DIR_MODE, dir_fd=base_fd)
+        except FileExistsError:
+            raise ExchangeDirError(f"'{dirname}' already exists in the exchange directory")
+        except OSError as exc:
+            raise ExchangeDirError(f"Could not create directory: {exc}")
+
+        try:
+            _grant_acl(target, username)
+        except ExchangeDirError:
+            try:
+                os.rmdir(dirname, dir_fd=base_fd)
+            except OSError as exc:
+                service_log(COMMAND_NAME, "WARNING", "Could not remove directory after ACL failure",
+                            path=target, error=str(exc))
+            raise
+    finally:
+        os.close(base_fd)
+
+    service_log(COMMAND_NAME, "INFO", "Created directory and granted access",
+                path=target, username=username, permissions=_ACL_PERMS)
+    return target
+
+
+def main():
+    """CLI entry point."""
+    parser = argparse.ArgumentParser(
+        description=f"Create a directory under {EXCHANGE_DIR} and grant a user read/write access via ACL"
+    )
+    parser.add_argument("username", help="HPC username to grant read/write access")
+    parser.add_argument("dirname", help="Name of the directory to create (a single path component)")
+    parser.add_argument("-d", "--debug", action="store_true", help="Enable debug mode")
+    args = parser.parse_args()
+
+    try:
+        create_exchange_dir(args.username, args.dirname, debug=args.debug)
+    except ExchangeDirError as exc:
+        service_log(COMMAND_NAME, "ERROR", str(exc), username=args.username, dirname=args.dirname)
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

gdexws-0.1.0/utils/__init__.py

@@ -0,0 +1,4 @@
+from .logging import service_log
+from .parse_payload import load_payload, build_command, execute_command
+
+__all__ = ["service_log", "load_payload", "build_command", "execute_command"]

gdexws-0.1.0/utils/file_validation.py

@@ -0,0 +1,47 @@
+"""Utility functions for file path validation."""
+
+from pathlib import Path
+from typing import List, Union, Dict, Any
+
+from gdexws.utils import service_log
+
+def relpath_validate(
+    file_paths: List[str],
+    base_dir: str = '/glade/campaign/collections/gdex/data/exchange/'
+) -> Union[bool, Dict[str, Any]]:
+    """Validate that all relative file paths are within the base directory.
+
+    Parameters
+    ----------
+    file_paths : List[str]
+        List of relative file paths to validate.
+    base_dir : str
+        Base directory that all paths must be within.
+        Default: '/glade/campaign/collections/gdex/data/exchange/'
+
+    Returns
+    -------
+    bool or dict
+        Returns True if all paths are valid.
+        Returns standardized dscheck_json error response if any path is invalid.
+
+    Examples
+    --------
+    >>> result = relpath_validate(["file1.nc", "file2.nc"])
+    >>> if result is True:
+    ...     # Process files
+    ... else:
+    ...     # result is an error dict, return to user
+    ...     return result
+    """
+    base_path = Path(base_dir).resolve()
+    full_paths = []
+    for file_path in file_paths:
+        try:
+            full_path = (base_path / file_path).resolve()
+            # This will raise ValueError if path escapes base directory
+            full_path.relative_to(base_path)
+            full_paths.append(str(full_path))
+        except ValueError:
+            return service_log("relpath-validate", "ERROR", f"Access denied: '{file_path}' escapes base directory")
+    return full_paths

gdexws-0.1.0/utils/logging.py

@@ -0,0 +1,82 @@
+import json
+import sys
+from datetime import datetime
+from typing import Dict, Any
+
+
+def log_format(
+    command_name: str,
+    level: str,
+    process_message: str,
+    **kwargs
+) -> Dict[str, Any]:
+    """
+    Create a structured log dictionary in the required format.
+
+    Parameters
+    ----------
+    command_name : str
+        Name of the command being executed
+    level : str
+        Log level (ERROR, WARNING, INFO, DEBUG)
+    process_message : str
+        Free format message for status tracking
+    **kwargs
+        Additional key=value pairs to include in the dict
+
+    Returns
+    -------
+    dict
+        Dictionary with the exact format:
+        {
+            "command": "...",
+            "time_of_process": "...",
+            "level": "...",
+            "process_message": "...",
+            ... (additional kwargs)
+        }
+
+    Raises
+    ------
+    ValueError
+        If level is not ERROR, WARNING, INFO, or DEBUG
+    """
+    valid_levels = {"ERROR", "WARNING", "INFO", "DEBUG"}
+    if level not in valid_levels:
+        raise ValueError(f"level must be one of {valid_levels}, got '{level}'")
+
+    log_dict = {
+        "command": command_name,
+        "time_of_process": datetime.now().isoformat(),
+        "level": level,
+        "process_message": process_message,
+    }
+
+    log_dict.update(kwargs)
+
+    return log_dict
+
+
+
+def service_log(command_name: str, level: str, process_message: str, **kwargs) -> None:
+    """
+    Log a message at the specified level and print it.
+
+    Parameters
+    ----------
+    command_name : str
+        Name of the command being executed
+    level : str
+        Log level (ERROR, WARNING, INFO, DEBUG)
+    process_message : str
+        Process message to log
+    **kwargs
+        Additional key=value pairs to include in the log
+    """
+    log_dict = log_format(
+        command_name=command_name,
+        level=level,
+        process_message=process_message,
+        **kwargs
+    )
+    print(json.dumps(log_dict), file=sys.stdout, flush=True)

gdexws-0.1.0/utils/parse_payload.py

@@ -0,0 +1,175 @@
+"""Utilities for parsing and executing payload configurations."""
+import json
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any, Dict, List
+
+try:
+    import httpx
+except ImportError:
+    httpx = None
+
+from .logging import service_log
+
+
+def load_payload(payload_path: str, timeout: float = 10.0) -> Dict[str, Any]:
+    """
+    Load payload JSON from a file path or URL.
+
+    Accepts both local file paths (POSIX) and URLs (http/https).
+    Automatically detects the input type and loads accordingly.
+
+    Parameters
+    ----------
+    payload_path : str
+        Either a local file path (POSIX) or a URL (http/https)
+    timeout : float
+        Timeout in seconds for HTTP requests. Default: 10.0
+
+    Returns
+    -------
+    dict
+        Dictionary containing payload configuration
+
+    Raises
+    ------
+    FileNotFoundError
+        If the local file does not exist
+    json.JSONDecodeError
+        If the content is not valid JSON
+    httpx.RequestError
+        If the HTTP request fails
+    ImportError
+        If loading from URL but httpx is not installed
+    """
+    # Check if it's a URL
+    if payload_path.startswith('http://') or payload_path.startswith('https://'):
+        return _load_payload_from_url(payload_path, timeout)
+    else:
+        return _load_payload_from_file(payload_path)
+
+
+def _load_payload_from_file(file_path: str) -> Dict[str, Any]:
+    """Load payload JSON from a local file.
+
+    Parameters
+    ----------
+    file_path : str
+        Path to the payload JSON file
+
+    Returns
+    -------
+    dict
+        Dictionary containing payload configuration
+
+    Raises
+    ------
+    FileNotFoundError
+        If the payload file does not exist
+    json.JSONDecodeError
+        If the file is not valid JSON
+    """
+    with open(file_path, "r") as f:
+        return json.load(f)
+
+
+def _load_payload_from_url(url: str, timeout: float = 10.0) -> Dict[str, Any]:
+    """Load payload JSON from a URL.
+
+    Parameters
+    ----------
+    url : str
+        HTTP or HTTPS URL to fetch payload JSON from
+    timeout : float
+        Timeout in seconds for the request
+
+    Returns
+    -------
+    dict
+        Dictionary containing payload configuration
+
+    Raises
+    ------
+    ImportError
+        If httpx is not installed
+    httpx.RequestError
+        If the HTTP request fails
+    json.JSONDecodeError
+        If the response is not valid JSON
+    """
+    if httpx is None:
+        raise ImportError(
+            "httpx is required for loading payload from URLs. "
+            "Install with: pip install httpx"
+        )
+
+    response = httpx.get(url, timeout=timeout, follow_redirects=True)
+    response.raise_for_status()
+    return response.json()
+
+
+def build_command(command_name: str, params: Dict[str, Any], file_path: str) -> List[str]:
+    """
+    Build a CLI command with arguments.
+
+    Parameters
+    ----------
+    command_name : str
+        Name of the CLI command
+    params : dict
+        Dictionary of parameters (excluding "command" key)
+    file_path : str
+        Path to the file to process
+
+    Returns
+    -------
+    list
+        List of command and arguments ready for subprocess
+    """
+    cmd = [command_name]
+
+    # Add file argument
+    cmd.extend(["-f", file_path])
+
+    # Add other parameters
+    for key, value in params.items():
+        # Skip the command key itself
+        if key == "command":
+            continue
+
+        # Convert underscore to hyphen for CLI arguments
+        cli_key = f"--{key.replace('_', '-')}"
+
+        # Handle boolean flags
+        if isinstance(value, bool):
+            if value:
+                cmd.append(cli_key)
+        else:
+            cmd.extend([cli_key, str(value)])
+
+    return cmd
+
+
+def execute_command(cmd: List[str]) -> int:
+    """
+    Execute a CLI command.
+
+    Parameters
+    ----------
+    cmd : list
+        List of command and arguments
+
+    Returns
+    -------
+    int
+        Return code from subprocess
+    """
+    service_log(
+        command_name="execute-command",
+        level="INFO",
+        process_message="Executing command",
+        command=" ".join(cmd)
+    )
+    result = subprocess.run(cmd, capture_output=False, check=True)
+    return result.returncode