sliderule-cli 5.6.1__tar.gz → 5.6.3__tar.gz

{sliderule_cli-5.6.1/src/sliderule_cli.egg-info → sliderule_cli-5.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sliderule-cli
-Version: 5.6.1
+Version: 5.6.3
 Summary: SlideRule Command Line Tool
 License-Expression: BSD-3-Clause
 Requires-Python: >=3.9

{sliderule_cli-5.6.1 → sliderule_cli-5.6.3/src/sliderule_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sliderule-cli
-Version: 5.6.1
+Version: 5.6.3
 Summary: SlideRule Command Line Tool
 License-Expression: BSD-3-Clause
 Requires-Python: >=3.9

sliderule_cli-5.6.3/src/sliderule_cli.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "5.6.3",
+  "distance": 0,
+  "node": "ge620b5509bb5a993af03131885c3a6f23922cb24",
+  "dirty": false,
+  "branch": "main",
+  "node_date": "2026-09-30"
+}

{sliderule_cli-5.6.1 → sliderule_cli-5.6.3}/src/sliderule_cluster/main.py

@@ -1,3 +1,4 @@
+import sys
 import json
 import uuid
 import tempfile
@@ -42,10 +43,9 @@ class Tool:
 
     # whoami
     def whoami(self):
-        rsps = self.session.source("whoami", parm={}, path="/discovery", retries=0, rethrow=True)
-        result = json.dumps(rsps)
-        self.__display_result(result)
-        return result
+        result = self.session.source("whoami", parm={}, path="/discovery", retries=0, rethrow=True)
+        self.__display_result(result, reformat=True)
+        return json.dumps(result)
 
     # status
     def status(self):
@@ -53,22 +53,21 @@ class Tool:
             parm = {"service":self.session.service}
         else:
             parm = {}
-        rsps = self.session.source("status", parm=parm, path="/discovery", retries=0, rethrow=True)
-        result = json.dumps(rsps)
-        self.__display_result(result)
-        return result
+        result = self.session.source("status", parm=parm, path="/discovery", retries=0, rethrow=True)
+        self.__display_result(result, reformat=True)
+        return json.dumps(result)
 
     # version
     def version(self):
         result = sliderule.get_version(session=self.session)
         self.__display_result(result, reformat=True)
-        return result
+        return json.dumps(result)
 
     # defaults
     def defaults(self):
         result = sliderule.source("defaults", session=self.session)
         self.__display_result(result, reformat=True)
-        return result
+        return json.dumps(result)
 
     # earthdata
     def earthdata(self):
@@ -84,13 +83,13 @@ class Tool:
         }.items() if v is not None })
         result = sliderule.source("earthdata", parms, rethrow=True, session=self.session)
         self.__display_result(result, reformat=True)
-        return result
+        return json.dumps(result)
 
     # run
     def run(self):
         # get request parameters
-        if self.args.parms.startswith('"') or self.args.parms.startswith('{'):
-            parms = self.args.parms
+        if self.args.parms.startswith('{'):
+            parms = json.loads(self.args.parms)
         else:
             parms = self.__get_parms()
         # build output parameters (if not supplied in request)
@@ -117,7 +116,7 @@ class Tool:
                     break
         # set and return results
         result = output_path
-        self.__display_result(result, reformat=True)
+        self.__display_result(result, reformat=False)
         return result
 
 #########################################
@@ -157,7 +156,7 @@ def main():
     # earthdata
     earthdata = subparsers.add_parser("earthdata", parents=[common], help="earthdata search")
     earthdata.add_argument('--asset',           type=str,               default="icesat2", help="SlideRule asset name")
-    earthdata.add_argument('--poly',            type=float, nargs='+',  default=None, help="closed counter-clockwise list of latitude longitude pairs defining the area of interest; e.g. lon1 lat1 lon2 lat2 lon3 lat3 ... lon1 lat1")
+    earthdata.add_argument('--poly',            type=float, nargs='+',  default=None, help="closed counter-clockwise list of longitude/latitude pairs defining the area of interest; e.g. lon1 lat1 lon2 lat2 lon3 lat3 ... lon1 lat1")
     earthdata.add_argument('--geojson',         type=str,               default=None, help="geojson filename defining the area of interest")
     earthdata.add_argument('--bbox',            type=float, nargs='+',  default=None, help="bounding box defining the area of interest; e.g. lon_ll lat_ll lon_ur lat_ur")
     earthdata.add_argument('--t0',              type=str,               default=None, help="start time as an ISO datetime string YYYY-MM-DDTHH:MM:SSZ") # datetime.fromisoformat(args.t0)
@@ -166,6 +165,7 @@ def main():
     earthdata.add_argument('--short_name',      type=str,               default=None, help="CMR dataset name")
     earthdata.add_argument('--with_meta',       action='store_true',    default=False, help="return metadata along with query results")
     earthdata.add_argument('--name_filter',     type=str,               default=None, help="regular expression to evaluate against resource names returned by query")
+    earthdata.add_argument('--parms',           type=str,               default=None, help="string or filename containing json parameters to use in request")
     earthdata.set_defaults(func=Tool.earthdata)
 
     # run
@@ -189,6 +189,7 @@ def main():
     except Exception as e:
         if args.verbose: raise
         print(f"Unhandled error: {e}")
+        sys.exit(1) # returns error back to system
 
 # running via direct invocation
 if __name__ == "__main__": main()

{sliderule_cli-5.6.1 → sliderule_cli-5.6.3}/src/sliderule_provisioner/main.py

@@ -69,6 +69,14 @@ class Tool:
     def info(self):
         return self.session.provisioner.info()
 
+    # S3 Access
+    def s3access(self):
+        credentials = self.session.provisioner.s3access()
+        print(f'export AWS_ACCESS_KEY_ID={credentials["access_key_id"]}')
+        print(f'export AWS_SECRET_ACCESS_KEY={credentials["secret_access_key"]}')
+        print(f'export AWS_SESSION_TOKEN={credentials["session_token"]}')
+        print(f'export AWS_CREDENTIAL_EXPIRATION={credentials["expiration"]}')
+
     # Authenticate
     def authenticate(self):
         return self.session.authenticate(force_login=True)
@@ -81,7 +89,7 @@ class Tool:
 def main():
 
     # command line arguments
-    parser = argparse.ArgumentParser(prog="sliderule-runner", description="""SlideRule Runner""")
+    parser = argparse.ArgumentParser(prog="sliderule-provisioner", description="""SlideRule Provisioner""")
     subparsers = parser.add_subparsers(dest="command", required=True)
 
     # options shared by every subcommand; parents= lets them appear after the command name
@@ -89,7 +97,6 @@ def main():
     common.add_argument('--domain',         type=str,               default="slideruleearth.io")
     common.add_argument('--cluster',        type=str,               default="developers")
     common.add_argument('--user_service',   action='store_true',    default=False)
-    common.add_argument('--concise',        action='store_true',    default=False)
     common.add_argument('--verbose',        action='store_true',    default=False)
     common.add_argument('--timezone',       type=str,               default="America/New_York")
 
@@ -120,7 +127,7 @@ def main():
 
     # report
     report = subparsers.add_parser("report", parents=[common], help="report metadata of active clusters")
-    report.add_argument('--kind',   type=str,   default="clusters", help="'cluster' for cluster report, 'test' for test report")
+    report.add_argument('--kind',   type=str,   default="cluster", help="'cluster' for cluster report, 'test' for test report")
     report.set_defaults(func=Tool.report)
 
     # test
@@ -132,6 +139,10 @@ def main():
     info = subparsers.add_parser("info", parents=[common], help="display information about provisioner")
     info.set_defaults(func=Tool.info)
 
+    # s3access
+    s3access = subparsers.add_parser("s3access", parents=[common], help="sets AWS credentials by outputting shell commands to 'eval' into current environment")
+    s3access.set_defaults(func=Tool.s3access)
+
     # authenticate
     authenticate = subparsers.add_parser("authenticate", parents=[common], help="force re-authentication of user account")
     authenticate.set_defaults(func=Tool.authenticate)
@@ -145,7 +156,8 @@ def main():
     # route command
     try:
         result = args.func(tool)
-        print(f'{json.dumps(result, indent=2)}')
+        if result:
+            print(f'{json.dumps(result, indent=2)}')
     except Exception as e:
         if args.verbose: raise
         print(f"Unhandled error: {e}")

{sliderule_cli-5.6.1 → sliderule_cli-5.6.3}/src/sliderule_runner/database.py

@@ -14,6 +14,7 @@
 #             },
 #             "complete": <true|false>,
 #             "results": [ {result 1}, {result 2}, ... {result N} ]
+#             "jobs": { <child_job_id>: {child job status} }
 #         },
 #         ...
 #     }

{sliderule_cli-5.6.1 → sliderule_cli-5.6.3}/src/sliderule_runner/main.py

@@ -4,6 +4,7 @@ import boto3
 import random
 import string
 import argparse
+import traceback
 from sliderule import sliderule
 from .database import Database, JobState, QueuePriority, JobStatus
 from pathlib import Path
@@ -27,7 +28,7 @@ class Tool:
         # open database
         self.database = Database(args.database)
         # create sliderule session
-        self.session = sliderule.create_session(verbose=args.verbose)
+        self.session = sliderule.create_session(domain=args.domain, verbose=args.verbose)
         self.session.authenticate() # gives privileges to access SlideRule Runner
         # aws clients
         self.s3 = boto3.client("s3", region_name="us-west-2")
@@ -36,7 +37,10 @@ class Tool:
     def __load_remote_file(self, bucket, key):
         obj = self.s3.get_object(Bucket=bucket, Key=key)
         contents = obj["Body"].read().decode("utf-8")
-        return json.loads(contents)
+        try:
+            return json.loads(contents)
+        except Exception:
+            return contents
 
     # Get Results for a Job Run
     #   output of job must include the following fields for this to work:
@@ -51,15 +55,22 @@ class Tool:
         bucket = run_url.split("s3://")[-1].split("/")[0]
         prefix = "/".join(run_url.split("s3://")[-1].split("/")[1:])
         receipt = self.__load_remote_file(bucket, f"{prefix}/receipt.json") # {"name": ..., "username": ... "args": <path to arg file>, "environment": ...}
-        args_list = self.__load_remote_file(bucket, receipt["args"])
+        args_bucket = receipt["args"].split("s3://")[-1].split("/")[0]
+        args_file = "/".join(receipt["args"].split("s3://")[-1].split("/")[1:])
+        args_content = self.__load_remote_file(args_bucket, args_file)
+        if isinstance(args_content, list):
+            args_list = args_content
+        elif isinstance(args_content, dict) or isinstance(args_content, str):
+            args_list = [args_content]
         for i in tqdm(range(len(args_list)), total=len(args_list), desc=f"{run_url}", unit="granule"):
+            result_file = f"{prefix}/result{len(args_list) > 1 and str(i) or ''}.json"
             try:
                 result = {
-                    "file": f"{prefix}/result{i}.json",
+                    "file": f"s3://{bucket}/{result_file}",
                     "environment": receipt["environment"],
                     "arg": args_list[i]
                 }
-                rsps = self.__load_remote_file(bucket, f"{prefix}/result{i}.json")
+                rsps = self.__load_remote_file(bucket, result_file)
                 try:
                     result |= {
                         "status": rsps["status"] and JobStatus.SUCCESS or JobStatus.FAILURE,
@@ -89,7 +100,8 @@ class Tool:
         # pull out arguments
         name = self.args.name
         script_file = self.args.script
-        arguments_file = self.args.arguments
+        arguments_input = self.args.arguments
+        arg_as_str = self.args.arg_as_str
         batch_size = self.args.batch_size
         vcpus = self.args.vcpus
         memory = self.args.memory
@@ -99,9 +111,12 @@ class Tool:
         with open(script_file, "r") as file:
             script = file.read()
         # read arguments
-        with open(arguments_file, "r") as file:
-            arguments = [line.strip() for line in file.readlines()]
-        # process job in batches
+        if arg_as_str:
+            arguments = [str(arguments_input)]
+        else:
+            with open(arguments_input, "r") as file:
+                arguments = [line.strip() for line in file.readlines()]
+            # process job in batches
         for i in range(0, len(arguments), batch_size):
             # build and check name
             job_name = f"{name}_{i}"
@@ -111,29 +126,34 @@ class Tool:
             # submit & save job
             args_list = arguments[i:i+batch_size]
             rsps = self.session.runner.submit(name=job_name, script=script, args=args_list, optional_args={"vcpus":vcpus, "memory":memory, "image":image, "queue":queue})
-            self.database.submissions[job_name] = rsps | {"complete": False}
+            self.database.submissions[job_name] = rsps | {"complete": False, "job_size": len(args_list)}
             print(f"Submitted job {job_name} using script {script_file} with {len(args_list)} entries: {rsps}")
 
     # Scrape Submissions
     def scrape_submissions(self):
-        arg_list = []
+        output_list = []
         for name,submission in self.database.submissions.items():
             if self.args.name and self.args.name != name: continue
             try:
                 if submission["complete"]:
                     print(f"Scraping {name} ...")
-                    for result in submission["results"]:
+                    for i in range(len(submission["results"])):
+                        result = submission["results"][i]
                         if result["status"] in self.args.status:
-                            print(f"{result["arg"]}")
-                            arg_list.append(result["arg"])
+                            if self.args.verbose:
+                                print(f'{i:-5d}: {result}\n')
+                                output_list.append(json.dumps(result))
+                            else:
+                                print(f'{i:-5d}: {result["arg"]}')
+                                output_list.append(result["arg"])
                 else:
                     print(f"Skipping {name} because it is still pending")
             except Exception as e:
                 print(f"Failed to scrape {name}: {e}")
         if self.args.output:
             with open(self.args.output, "w") as file:
-                for arg in arg_list:
-                    file.write(f"{arg}\n")
+                for output in output_list:
+                    file.write(f"{output}\n")
 
     # Get Status
     def get_status(self):
@@ -142,44 +162,52 @@ class Tool:
             complete = job["complete"]
             print(f"Statusing {name} ...")
             if not complete:
-                report = self.session.runner.queue(job_id=job["job_id"], queue=queue)["report"]
+                if job["job_size"] > 1: # array processing, use parent job id
+                    queue_status = self.session.runner.queue(job_id=job["job_id"], queue=queue, verbose=True)
+                    child_jobs = queue_status["jobs"]
+                    self.database.submissions[name]["jobs"] = {}
+                    for child_job in child_jobs:
+                        self.database.submissions[name]["jobs"][int(child_job["index"])] = child_job
+                else: # single job, use name
+                    queue_status = self.session.runner.queue(name=name, queue=queue, verbose=False)
+                # report
+                report = queue_status["report"]
                 self.database.submissions[name]["status"] = report
-                if sum([report[s] for s in [JobState.SUBMITTED, JobState.PENDING, JobState.RUNNABLE, JobState.STARTING, JobState.RUNNING]]) == 0:
+                jobs_in_progress = sum([report[s] for s in [JobState.SUBMITTED, JobState.PENDING, JobState.RUNNABLE, JobState.STARTING, JobState.RUNNING]])
+                jobs_complete = sum([report[s] for s in [JobState.SUCCEEDED, JobState.FAILED]])
+                if jobs_in_progress == 0 and jobs_complete > 0:
                     print(f"Job {name} complete, reading results ...")
                     self.database.submissions[name]["complete"] = True
                     self.database.submissions[name]["results"] = self.__get_results(job["run_url"])
                 else:
                     print(f"Job {name} still pending")
-        print(",".join([f"{c:>30}" for c in ["NAME"]] + [f"{c:>10}" for c in list(JobState)]))
-        for name,job in self.database.submissions.items():
-            print(",".join([f"{c:>30}" for c in [name]] + [f"{c:>10}" for c in [job["status"][state] for state in list(JobState)]]))
+        if self.args.verbose:
+            print(",".join([f"{c:>30}" for c in ["NAME"]] + [f"{c:>10}" for c in ["INDEX", "STATUS"]]))
+            for name,job in self.database.submissions.items():
+                for index, child_job in job["jobs"].items():
+                    print(",".join([f"{c:>30}" for c in [name]] + [f"{c:>10}" for c in [index, child_job["status"]]]))
+        else:
+            print(",".join([f"{c:>30}" for c in ["NAME"]] + [f"{c:>10}" for c in list(JobState)]))
+            for name,job in self.database.submissions.items():
+                print(",".join([f"{c:>30}" for c in [name]] + [f"{c:>10}" for c in [job["status"][state] for state in list(JobState)]]))
 
     # Generate Report
     def generate_report(self):
-        stats = {status.value: 0 for status in JobStatus}
-        duration = {"avg": 0.0, "total": 0.0}
-        processed = 0
-        pending = 0
-        errors = 0
+        print(",".join([f"{c:>30}" for c in ["name"]] + [f"{c:>12}" for c in ["processed"]] + [f"{c:>12}" for c in list(JobStatus)] + [f"{c:>12}" for c in ["duration"]]))
         for name,submission in self.database.submissions.items():
             if self.args.name and self.args.name != name: continue
+            stats = {status.value: 0 for status in JobStatus}
+            duration = {"avg": 0.0, "total": 0.0}
+            processed = 0
             if submission["complete"]:
                 for result in submission["results"]:
-                    try:
-                        stats[result["status"]] += 1
+                    stats[result["status"]] += 1
+                    if "duration" in result:
                         duration["total"] += result["duration"]
                         processed += 1
-                    except Exception as e:
-                        errors += 1
-            else:
-                pending += 1
-        if processed > 0:
-            duration["avg"] = duration["total"] / processed
-        print("Processed:", processed)
-        print("Pending:", pending)
-        print("Errors:", errors)
-        print("Status:", json.dumps(stats, indent=2))
-        print("Duration:", json.dumps(duration, indent=2))
+                if processed > 0:
+                    duration["avg"] = duration["total"] / processed
+                print(",".join([f"{c:>30}" for c in [name]] + [f"{c:>12}" for c in [processed]] + [f"{c:>12}" for c in [stats[status] for status in list(JobStatus)]] + [f"{c:>12.3}" for c in [duration["avg"]]]))
 
     # Cancel Job
     def cancel_job(self):
@@ -189,9 +217,21 @@ class Tool:
         rsps = self.session.runner.cancel(job_list=[job_id], queue=queue)
         print(f"Cancelled submission {name}: {rsps}")
 
+    # Get Logs
+    def get_logs(self):
+        name = self.args.name
+        index = self.args.index
+        if index is not None:
+            job_id = self.database.submissions[name]["jobs"][str(index)]["job_id"]
+        else:
+            job_id = self.database.submissions[name]["job_id"]
+        events = self.session.runner.logs(job_id=job_id)
+        for event in events:
+            print("->", event)
+
     # Finish
     def finish(self):
-        if not self.args.dryrun:
+        if not self.args.dryrun and "archive" not in self.args:
             # save database
             self.database.write()
 
@@ -207,6 +247,7 @@ def main():
 
     # options shared by every subcommand; parents= lets them appear after the command name
     common = argparse.ArgumentParser(add_help=False)
+    common.add_argument('--domain',     type=str,                   default="slideruleearth.io")
     common.add_argument('--database',   type=Path,                  default=Path.home() / ".cache" / "sliderule" / "runner_database.json")
     common.add_argument('--queue',      type=QueuePriority,         default=QueuePriority.DEFAULT, choices=list(QueuePriority))
     common.add_argument('--verbose',    action='store_true',        default=False)
@@ -222,6 +263,7 @@ def main():
     submit.add_argument('name',         metavar="<name>")
     submit.add_argument('script',       metavar="<script.lua>",     type=Path)
     submit.add_argument('arguments',    metavar="<arguments.txt>",  type=Path)
+    submit.add_argument('--arg_as_str', action='store_true',        default=False) # changes arguments to be just a string instead of a file
     submit.add_argument('--vcpus',      type=int,                   default=4)
     submit.add_argument('--memory',     type=int,                   default=16000)
     submit.add_argument('--batch_size', type=int,                   default=10000)
@@ -240,7 +282,7 @@ def main():
     status.set_defaults(func=Tool.get_status)
 
     # report
-    report = subparsers.add_parser("report", parents=[common], help="generate report of submitted jobs")
+    report = subparsers.add_parser("report", parents=[common], help="generate report of completed jobs")
     report.add_argument('--name',       type=str,                   default=None) # name of submission
     report.set_defaults(func=Tool.generate_report)
 
@@ -249,6 +291,12 @@ def main():
     cancel.add_argument('--name',       type=str,                   required=True) # name of submission
     cancel.set_defaults(func=Tool.cancel_job)
 
+    # logs
+    logs = subparsers.add_parser("logs", parents=[common], help="get log messages for a child job")
+    logs.add_argument('--name',       type=str,                     required=True) # name of submission
+    logs.add_argument('--index',      type=int,                     default=None) # job index
+    logs.set_defaults(func=Tool.get_logs)
+
     # parse command line
     args = parser.parse_args()
 
@@ -262,6 +310,7 @@ def main():
     except Exception as e:
         if args.verbose: raise
         print(f"Unhandled error: {e}")
+        traceback.print_exc()
 
 # running via direct invocation
 if __name__ == "__main__": main()

sliderule_cli-5.6.1/src/sliderule_cli.egg-info/scm_version.json

@@ -1,8 +0,0 @@
-{
-  "tag": "5.6.1",
-  "distance": 0,
-  "node": "g5bc2a6eb3c8fc3927b3ffb14b511be2698a8261c",
-  "dirty": false,
-  "branch": "main",
-  "node_date": "2026-09-18"
-}