sliderule-cli 5.6.2__tar.gz → 5.7.0__tar.gz

{sliderule_cli-5.6.2/src/sliderule_cli.egg-info → sliderule_cli-5.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sliderule-cli
-Version: 5.6.2
+Version: 5.7.0
 Summary: SlideRule Command Line Tool
 License-Expression: BSD-3-Clause
 Requires-Python: >=3.9

{sliderule_cli-5.6.2 → sliderule_cli-5.7.0/src/sliderule_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sliderule-cli
-Version: 5.6.2
+Version: 5.7.0
 Summary: SlideRule Command Line Tool
 License-Expression: BSD-3-Clause
 Requires-Python: >=3.9

sliderule_cli-5.7.0/src/sliderule_cli.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "5.7.0",
+  "distance": 0,
+  "node": "g084d9e1b17c9db584fa57119be4c8c89da45b645",
+  "dirty": false,
+  "branch": "main",
+  "node_date": "2026-10-22"
+}

{sliderule_cli-5.6.2 → sliderule_cli-5.7.0}/src/sliderule_cluster/main.py

@@ -1,3 +1,4 @@
+import sys
 import json
 import uuid
 import tempfile
@@ -42,10 +43,9 @@ class Tool:
 
     # whoami
     def whoami(self):
-        rsps = self.session.source("whoami", parm={}, path="/discovery", retries=0, rethrow=True)
-        result = json.dumps(rsps)
-        self.__display_result(result)
-        return result
+        result = self.session.source("whoami", parm={}, path="/discovery", retries=0, rethrow=True)
+        self.__display_result(result, reformat=True)
+        return json.dumps(result)
 
     # status
     def status(self):
@@ -53,22 +53,21 @@ class Tool:
             parm = {"service":self.session.service}
         else:
             parm = {}
-        rsps = self.session.source("status", parm=parm, path="/discovery", retries=0, rethrow=True)
-        result = json.dumps(rsps)
-        self.__display_result(result)
-        return result
+        result = self.session.source("status", parm=parm, path="/discovery", retries=0, rethrow=True)
+        self.__display_result(result, reformat=True)
+        return json.dumps(result)
 
     # version
     def version(self):
         result = sliderule.get_version(session=self.session)
         self.__display_result(result, reformat=True)
-        return result
+        return json.dumps(result)
 
     # defaults
     def defaults(self):
         result = sliderule.source("defaults", session=self.session)
         self.__display_result(result, reformat=True)
-        return result
+        return json.dumps(result)
 
     # earthdata
     def earthdata(self):
@@ -84,13 +83,13 @@ class Tool:
         }.items() if v is not None })
         result = sliderule.source("earthdata", parms, rethrow=True, session=self.session)
         self.__display_result(result, reformat=True)
-        return result
+        return json.dumps(result)
 
     # run
     def run(self):
         # get request parameters
-        if self.args.parms.startswith('"') or self.args.parms.startswith('{'):
-            parms = self.args.parms
+        if self.args.parms.startswith('{'):
+            parms = json.loads(self.args.parms)
         else:
             parms = self.__get_parms()
         # build output parameters (if not supplied in request)
@@ -117,7 +116,7 @@ class Tool:
                     break
         # set and return results
         result = output_path
-        self.__display_result(result, reformat=True)
+        self.__display_result(result, reformat=False)
         return result
 
 #########################################
@@ -157,7 +156,7 @@ def main():
     # earthdata
     earthdata = subparsers.add_parser("earthdata", parents=[common], help="earthdata search")
     earthdata.add_argument('--asset',           type=str,               default="icesat2", help="SlideRule asset name")
-    earthdata.add_argument('--poly',            type=float, nargs='+',  default=None, help="closed counter-clockwise list of latitude longitude pairs defining the area of interest; e.g. lon1 lat1 lon2 lat2 lon3 lat3 ... lon1 lat1")
+    earthdata.add_argument('--poly',            type=float, nargs='+',  default=None, help="closed counter-clockwise list of longitude/latitude pairs defining the area of interest; e.g. lon1 lat1 lon2 lat2 lon3 lat3 ... lon1 lat1")
     earthdata.add_argument('--geojson',         type=str,               default=None, help="geojson filename defining the area of interest")
     earthdata.add_argument('--bbox',            type=float, nargs='+',  default=None, help="bounding box defining the area of interest; e.g. lon_ll lat_ll lon_ur lat_ur")
     earthdata.add_argument('--t0',              type=str,               default=None, help="start time as an ISO datetime string YYYY-MM-DDTHH:MM:SSZ") # datetime.fromisoformat(args.t0)
@@ -166,6 +165,7 @@ def main():
     earthdata.add_argument('--short_name',      type=str,               default=None, help="CMR dataset name")
     earthdata.add_argument('--with_meta',       action='store_true',    default=False, help="return metadata along with query results")
     earthdata.add_argument('--name_filter',     type=str,               default=None, help="regular expression to evaluate against resource names returned by query")
+    earthdata.add_argument('--parms',           type=str,               default=None, help="string or filename containing json parameters to use in request")
     earthdata.set_defaults(func=Tool.earthdata)
 
     # run
@@ -189,6 +189,7 @@ def main():
     except Exception as e:
         if args.verbose: raise
         print(f"Unhandled error: {e}")
+        sys.exit(1) # returns error back to system
 
 # running via direct invocation
 if __name__ == "__main__": main()

{sliderule_cli-5.6.2 → sliderule_cli-5.7.0}/src/sliderule_provisioner/main.py

@@ -1,3 +1,4 @@
+import sys
 import json
 import argparse
 from datetime import datetime
@@ -69,6 +70,20 @@ class Tool:
     def info(self):
         return self.session.provisioner.info()
 
+    # S3 Access
+    def s3access(self):
+        credentials = self.session.provisioner.s3access()
+        fmt = {
+            "bash":       "export {}='{}'",
+            "powershell": "$env:{}='{}'",
+            "cmd":        'set "{}={}"',
+        }[self.args.shell]
+        for name, key in [("AWS_ACCESS_KEY_ID", "access_key_id"),
+                          ("AWS_SECRET_ACCESS_KEY", "secret_access_key"),
+                          ("AWS_SESSION_TOKEN", "session_token"),
+                          ("AWS_CREDENTIAL_EXPIRATION", "expiration")]:
+            print(fmt.format(name, credentials[key]))
+
     # Authenticate
     def authenticate(self):
         return self.session.authenticate(force_login=True)
@@ -81,7 +96,7 @@ class Tool:
 def main():
 
     # command line arguments
-    parser = argparse.ArgumentParser(prog="sliderule-runner", description="""SlideRule Runner""")
+    parser = argparse.ArgumentParser(prog="sliderule-provisioner", description="""SlideRule Provisioner""")
     subparsers = parser.add_subparsers(dest="command", required=True)
 
     # options shared by every subcommand; parents= lets them appear after the command name
@@ -89,7 +104,6 @@ def main():
     common.add_argument('--domain',         type=str,               default="slideruleearth.io")
     common.add_argument('--cluster',        type=str,               default="developers")
     common.add_argument('--user_service',   action='store_true',    default=False)
-    common.add_argument('--concise',        action='store_true',    default=False)
     common.add_argument('--verbose',        action='store_true',    default=False)
     common.add_argument('--timezone',       type=str,               default="America/New_York")
 
@@ -120,7 +134,7 @@ def main():
 
     # report
     report = subparsers.add_parser("report", parents=[common], help="report metadata of active clusters")
-    report.add_argument('--kind',   type=str,   default="clusters", help="'cluster' for cluster report, 'test' for test report")
+    report.add_argument('--kind',   type=str,   default="clusters", help="'clusters' for cluster report, 'test' for test report")
     report.set_defaults(func=Tool.report)
 
     # test
@@ -132,6 +146,11 @@ def main():
     info = subparsers.add_parser("info", parents=[common], help="display information about provisioner")
     info.set_defaults(func=Tool.info)
 
+    # s3access
+    s3access = subparsers.add_parser("s3access", parents=[common], help="sets AWS credentials by outputting shell commands to 'eval' into current environment")
+    s3access.add_argument('--shell',  type=str,   default="bash", choices=["bash", "powershell", "cmd"])
+    s3access.set_defaults(func=Tool.s3access)
+
     # authenticate
     authenticate = subparsers.add_parser("authenticate", parents=[common], help="force re-authentication of user account")
     authenticate.set_defaults(func=Tool.authenticate)
@@ -145,10 +164,11 @@ def main():
     # route command
     try:
         result = args.func(tool)
-        print(f'{json.dumps(result, indent=2)}')
+        if result:
+            print(f'{json.dumps(result, indent=2)}')
     except Exception as e:
         if args.verbose: raise
-        print(f"Unhandled error: {e}")
+        print(f"Unhandled error: {e}", file=sys.stderr)
 
 # running via direct invocation
 if __name__ == "__main__": main()

sliderule_cli-5.7.0/src/sliderule_runner/main.py

@@ -0,0 +1,379 @@
+import importlib
+import sys
+import json
+import boto3
+import random
+import string
+import argparse
+import traceback
+from sliderule import sliderule
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from .database import Database, JobState, QueuePriority, JobStatus
+from pathlib import Path
+
+try:
+    tqdm = importlib.import_module("tqdm").tqdm
+except Exception:
+    print(f"tqdm unavailable, progress of operations will not be reported")
+    def tqdm(iterable, **kwargs):
+        return iterable
+
+#########################################
+# Tool
+#########################################
+
+class Tool:
+
+    # Constructor
+    def __init__(self, args):
+        self.args = args
+        # open database
+        self.database = Database(args.database)
+        # create sliderule session
+        if args.net:
+            self.session = sliderule.create_session(domain=args.domain, verbose=args.verbose, rethrow=True)
+            self.session.authenticate() # gives privileges to access SlideRule Runner
+        # aws clients
+        self.s3 = boto3.client("s3", region_name="us-west-2")
+
+    # Split out bucket and key from url
+    def __parse_url(self, url):
+        path = url.split("s3://")[-1]
+        bucket = path.split("/")[0]
+        key = '/'.join(path.split("/")[1:])
+        return bucket, key
+
+    # Load Remote File from S3
+    def __load_remote_file(self, bucket, key, parse_json=True):
+        obj = self.s3.get_object(Bucket=bucket, Key=key)
+        contents = obj["Body"].read().decode("utf-8")
+        if parse_json:
+            try:
+                return json.loads(contents)
+            except Exception:
+                pass
+        return contents
+
+    # Get Result for a Job Run
+    #   output of job must include the following fields for this to work:
+    #   {
+    #       "status": <boolean status of run>,
+    #       "start": <start time in seconds of run>,
+    #       "stop": <stop time in seconds of run>,
+    #       "outputs": [<output1>, <output2>, ... <outputN>]
+    #   }
+    def __get_result(self, results, i):
+        result = results[i]
+        bucket, result_file = self.__parse_url(result["file"])
+        try:
+            rsps = self.__load_remote_file(bucket, result_file)
+            try:
+                result |= {
+                    "status": rsps["status"] and JobStatus.SUCCESS or JobStatus.FAILURE,
+                    "duration": rsps["status"] and (rsps["stop"] - rsps["start"]) or 0.0,
+                    "outputs": rsps["outputs"]
+                }
+            except Exception as e:
+                result |= {
+                    "status": JobStatus.UNSUPPORTED,
+                    "result": rsps,
+                    "error": f"{e}"
+                }
+        except Exception as e:
+            result |= {
+                "status": JobStatus.ERROR,
+                "error": f"{e}"
+            }
+
+    # Get all results for a submission
+    def __get_results(self, run_url, job_size):
+        bucket, prefix = self.__parse_url(run_url)
+        receipt = self.__load_remote_file(bucket, f"{prefix}/receipt.json") # {"name": ..., "username": ... "args": <path to arg file>, "environment": ...}
+        args_bucket, args_file = self.__parse_url(receipt["args"])
+        args_content = self.__load_remote_file(args_bucket, args_file, job_size > 1)
+        if isinstance(args_content, list):
+            args_list = args_content
+        else:
+            args_list = [args_content]
+        if len(args_list) != job_size:
+            print(f"Incorrect number of arguments {len(args_list)}, expected {job_size}")
+            return None
+        results = [{
+            "file": f"s3://{bucket}/{prefix}/result{len(args_list) > 1 and str(i) or ''}.json",
+            "environment": receipt["environment"],
+            "arg": args_list[i]
+        } for i in range(len(args_list))]
+        with ThreadPoolExecutor(max_workers=self.args.concurrency) as executor:
+            futures = [executor.submit(self.__get_result, results, i) for i in range(len(results))]
+            for future in tqdm(as_completed(futures), total=len(results), desc=f"{run_url}", unit="job"):
+                future.result()
+        return results
+
+    # Archive Database
+    def archive_database(self):
+        self.database.write(self.args.archive)
+        self.database.remove()
+
+    # Prune Database
+    def prune_database(self):
+        name = self.args.name
+        if name in self.database.submissions:
+            print(f"Pruning submission {name} from database")
+            del self.database.submissions[name]
+        else:
+            print(f"Unable to find submission {name} in database")
+
+    # Submit Job
+    def submit_job(self):
+        # pull out arguments
+        name = self.args.name
+        script_file = self.args.script
+        arguments_input = self.args.arguments
+        arg_as_str = self.args.arg_as_str
+        batch_size = self.args.batch_size
+        vcpus = self.args.vcpus
+        memory = self.args.memory
+        image = self.args.image
+        queue = self.args.queue
+        secrets_file = self.args.secrets
+        # read script
+        with open(script_file, "r") as file:
+            script = file.read()
+        # read arguments
+        if arg_as_str:
+            arguments = [str(arguments_input)]
+        else:
+            with open(arguments_input, "r") as file:
+                arguments = [line.strip() for line in file.readlines() if line.strip()]
+        # read secrets
+        if secrets_file:
+            with open(secrets_file, "r") as file:
+                secret_values = json.load(file)
+        else:
+            secret_values = None
+        # process job in batches
+        for i in range(0, len(arguments), batch_size):
+            # build and check name
+            job_name = f"{name}_{i}"
+            if job_name in self.database.submissions:
+                if self.args.dups_ok:
+                    unique = ''.join(random.choices(string.ascii_lowercase, k=3))
+                    job_name = f"{name}_{unique}_{i}"
+                else:
+                    print(f"Skipping duplicate submission: {job_name}")
+                    continue
+            # submit & save job
+            args_list = arguments[i:i+batch_size]
+            rsps = self.session.runner.submit(name=job_name, script=script, args=args_list, optional_args={"vcpus":vcpus, "memory":memory, "image":image, "queue":queue}, secret_values=secret_values)
+            self.database.submissions[job_name] = rsps | {"complete": False, "job_size": len(args_list)}
+            print(f"Submitted job {job_name} using script {script_file} with {len(args_list)} entries: {rsps}")
+
+    # Scrape Submissions
+    def scrape_submissions(self):
+        output_list = []
+        for name,submission in self.database.submissions.items():
+            if self.args.name and self.args.name != name: continue
+            try:
+                if submission["complete"]:
+                    print(f"Scraping {name} ...")
+                    for i in range(len(submission["results"])):
+                        result = submission["results"][i]
+                        if result["status"] in self.args.status:
+                            if self.args.verbose:
+                                print(f'{i:-5d}: {result}\n')
+                                output_list.append(json.dumps(result))
+                            else:
+                                print(f'{i:-5d}: {result["arg"]}')
+                                output_list.append(result["arg"])
+                else:
+                    print(f"Skipping {name} because it is still pending")
+            except Exception as e:
+                print(f"Failed to scrape {name}: {e}")
+        if self.args.output:
+            with open(self.args.output, "w") as file:
+                for output in output_list:
+                    file.write(f"{output}\n")
+
+    # Get Status
+    def get_status(self):
+        queue = self.args.queue
+        for name,job in self.database.submissions.items():
+            complete = job["complete"]
+            sys.stdout.write(f"Statusing {name} ... ")
+            sys.stdout.flush()
+            if not complete:
+                if job["job_size"] > 1: # array processing
+                    queue_status = self.session.runner.queue(job_id=job["job_id"], queue=queue, verbose=True)
+                    child_jobs = queue_status["jobs"]
+                    self.database.submissions[name]["jobs"] = {}
+                    for child_job in child_jobs:
+                        self.database.submissions[name]["jobs"][int(child_job["index"])] = child_job
+                else: # single job
+                    queue_status = self.session.runner.queue(job_id=job["job_id"], queue=queue, verbose=False)
+                # report
+                report = queue_status["report"]
+                self.database.submissions[name]["status"] = report
+                jobs_in_progress = sum([report[s] for s in [JobState.SUBMITTED, JobState.PENDING, JobState.RUNNABLE, JobState.STARTING, JobState.RUNNING]])
+                jobs_complete = sum([report[s] for s in [JobState.SUCCEEDED, JobState.FAILED]])
+                if jobs_in_progress == 0 and jobs_complete > 0:
+                    print("complete")
+                    results = self.__get_results(job["run_url"], self.database.submissions[name]["job_size"])
+                    if results:
+                        self.database.submissions[name]["results"] = results
+                        self.database.submissions[name]["complete"] = True
+                else:
+                    print(f"still pending")
+            else:
+                print("") # flush with new line
+        if self.args.verbose:
+            print(",".join([f"{c:>30}" for c in ["NAME"]] + [f"{c:>10}" for c in ["INDEX", "STATUS"]]))
+            for name,job in self.database.submissions.items():
+                for index, child_job in job["jobs"].items():
+                    print(",".join([f"{c:>30}" for c in [name]] + [f"{c:>10}" for c in [index, child_job["status"]]]))
+        else:
+            print(",".join([f"{c:>30}" for c in ["NAME"]] + [f"{c:>10}" for c in list(JobState)]))
+            for name,job in self.database.submissions.items():
+                print(",".join([f"{c:>30}" for c in [name]] + [f"{c:>10}" for c in [job["status"][state] for state in list(JobState)]]))
+
+    # Generate Report
+    def generate_report(self):
+        print(",".join([f"{c:>30}" for c in ["name"]] + [f"{c:>12}" for c in list(JobStatus)] + [f"{c:>12}" for c in ["duration"]]))
+        for name,submission in self.database.submissions.items():
+            if self.args.name and self.args.name != name: continue
+            stats = {status.value: 0 for status in JobStatus}
+            duration = {"avg": 0.0, "total": 0.0, "cnt": 0}
+            if submission["complete"]:
+                for result in submission["results"]:
+                    stats[result["status"]] += 1
+                    if ("duration" in result) and (result["status"] == JobStatus.SUCCESS):
+                        duration["total"] += result["duration"]
+                        duration["cnt"] += 1
+                if duration["cnt"] > 0:
+                    duration["avg"] = duration["total"] / duration["cnt"]
+                print(",".join([f"{c:>30}" for c in [name]] + [f"{c:>12}" for c in [stats[status] for status in list(JobStatus)]] + [f"{c:>12.3}" for c in [duration["avg"]]]))
+
+    # Cancel Job
+    def cancel_job(self):
+        name = self.args.name
+        queue = self.args.queue
+        job_id = self.database.submissions[name]["job_id"]
+        rsps = self.session.runner.cancel(job_list=[job_id], queue=queue)
+        print(f"Cancelled submission {name}: {rsps}")
+
+    # Get Logs
+    def get_logs(self):
+        name = self.args.name
+        index = self.args.index
+        if index is not None:
+            job_id = self.database.submissions[name]["jobs"][str(index)]["job_id"]
+        else:
+            job_id = self.database.submissions[name]["job_id"]
+        events = self.session.runner.logs(job_id=job_id)
+        for event in events:
+            print("->", event)
+
+    # Finish
+    def finish(self):
+        if not self.args.ro:
+            # save database
+            self.database.write()
+
+#########################################
+# Main
+#########################################
+
+def main():
+
+    # ----------------------------------------------------------------------------
+    # The sliderule-runner has a set of commands along with common options.  Each
+    # command has three attributes:
+    #   func - the function to call when that subcommand is issued
+    #   ro - "read-only", i.e. is the database written to or only read from
+    #   net - "network", i.e. does a sliderule session need to be created
+    # ----------------------------------------------------------------------------
+
+    # command line arguments
+    parser = argparse.ArgumentParser(prog="sliderule-runner", description="""SlideRule Runner""")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    # options shared by every subcommand; parents= lets them appear after the command name
+    common = argparse.ArgumentParser(add_help=False)
+    common.add_argument('--domain',     type=str,                   default="slideruleearth.io")
+    common.add_argument('--database',   type=Path,                  default=Path.home() / ".cache" / "sliderule" / "runner_database.json")
+    common.add_argument('--queue',      type=QueuePriority,         default=QueuePriority.DEFAULT, choices=list(QueuePriority))
+    common.add_argument('--verbose',    action='store_true',        default=False)
+
+    # archive
+    archive = subparsers.add_parser("archive", parents=[common], help="save and clear the database")
+    archive.add_argument('archive',     metavar="<full path to archive file>")
+    archive.set_defaults(func=Tool.archive_database, ro=True, net=False)
+
+    # prune
+    prune = subparsers.add_parser("prune", parents=[common], help="remove a submission from the database")
+    prune.add_argument('name',          type=str,                   metavar="<name>")
+    prune.set_defaults(func=Tool.prune_database, ro=False, net=False)
+
+    # submit
+    submit = subparsers.add_parser("submit", parents=[common], help="submit a job")
+    submit.add_argument('name',         type=str,                   metavar="<name>")
+    submit.add_argument('script',       type=str,                   metavar="<script file>")
+    submit.add_argument('arguments',    type=str,                   metavar="<arguments file>")
+    submit.add_argument('--arg_as_str', action='store_true',        default=False) # changes arguments to be just a string instead of a file
+    submit.add_argument('--dups_ok',    action='store_true',        default=False) # allow a duplicate submission (will provide random unique qualitfier to name)
+    submit.add_argument('--vcpus',      type=int,                   default=4)
+    submit.add_argument('--memory',     type=int,                   default=16000)
+    submit.add_argument('--batch_size', type=int,                   default=10000)
+    submit.add_argument('--image',      type=str,                   default="pysr:latest")
+    submit.add_argument('--secrets',    type=Path,                  default=None) # contents of file must be json
+    submit.set_defaults(func=Tool.submit_job, ro=False, net=True)
+
+    # scrape
+    scrape = subparsers.add_parser("scrape", parents=[common], help="generate list of arguments from jobs with provided job status")
+    scrape.add_argument('--status',     type=JobStatus, nargs='+',  default=[JobStatus.FAILURE, JobStatus.UNSUPPORTED, JobStatus.ERROR])
+    scrape.add_argument('--name',       type=str,                   default=None) # name of submission
+    scrape.add_argument('--output',     type=str,                   default=None)
+    scrape.set_defaults(func=Tool.scrape_submissions, ro=True, net=False)
+
+    # status
+    status = subparsers.add_parser("status", parents=[common], help="display status of submitted jobs")
+    status.add_argument('--concurrency',type=int,                   default=20)
+    status.set_defaults(func=Tool.get_status, ro=False, net=True)
+
+    # report
+    report = subparsers.add_parser("report", parents=[common], help="generate report of completed jobs")
+    report.add_argument('--name',       type=str,                   default=None) # name of submission
+    report.set_defaults(func=Tool.generate_report, ro=True, net=False)
+
+    # cancel
+    cancel = subparsers.add_parser("cancel", parents=[common], help="cancel a submitted job")
+    cancel.add_argument('--name',       type=str,                   required=True) # name of submission
+    cancel.set_defaults(func=Tool.cancel_job, ro=True, net=True)
+
+    # logs
+    logs = subparsers.add_parser("logs", parents=[common], help="get log messages for a child job")
+    logs.add_argument('--name',         type=str,                   required=True) # name of submission
+    logs.add_argument('--index',        type=int,                   default=None) # job index
+    logs.set_defaults(func=Tool.get_logs, ro=True, net=True)
+
+    # parse command line
+    args = parser.parse_args()
+
+    # create tool
+    tool = Tool(args)
+
+    # route command
+    exit_status = 0
+    try:
+        args.func(tool)
+    except Exception as e:
+        print(f"Unhandled error: {e}")
+        traceback.print_exc()
+        exit_status = 1
+    finally:
+        tool.finish()
+
+    # return back to caller
+    sys.exit(exit_status)
+
+# running via direct invocation
+if __name__ == "__main__": main()

sliderule_cli-5.6.2/src/sliderule_cli.egg-info/scm_version.json

@@ -1,8 +0,0 @@
-{
-  "tag": "5.6.2",
-  "distance": 0,
-  "node": "g76fb3116a5b77c6fbeb8f8df0a2cdffe60a1418b",
-  "dirty": false,
-  "branch": "main",
-  "node_date": "2026-09-23"
-}

sliderule_cli-5.6.2/src/sliderule_runner/main.py

@@ -1,298 +0,0 @@
-import importlib
-import json
-import boto3
-import random
-import string
-import argparse
-from sliderule import sliderule
-from .database import Database, JobState, QueuePriority, JobStatus
-from pathlib import Path
-
-try:
-    tqdm = importlib.import_module("tqdm").tqdm
-except Exception:
-    print(f"tqdm unavailable, progress of operations will not be reported")
-    def tqdm(iterable, **kwargs):
-        return iterable
-
-#########################################
-# Tool
-#########################################
-
-class Tool:
-
-    # Constructor
-    def __init__(self, args):
-        self.args = args
-        # open database
-        self.database = Database(args.database)
-        # create sliderule session
-        self.session = sliderule.create_session(verbose=args.verbose)
-        self.session.authenticate() # gives privileges to access SlideRule Runner
-        # aws clients
-        self.s3 = boto3.client("s3", region_name="us-west-2")
-
-    # Load Remote File from S3
-    def __load_remote_file(self, bucket, key):
-        obj = self.s3.get_object(Bucket=bucket, Key=key)
-        contents = obj["Body"].read().decode("utf-8")
-        return json.loads(contents)
-
-    # Get Results for a Job Run
-    #   output of job must include the following fields for this to work:
-    #   {
-    #       "status": <boolean status of run>,
-    #       "start": <start time in seconds of run>,
-    #       "stop": <stop time in seconds of run>,
-    #       "outputs": [<output1>, <output2>, ... <outputN>]
-    #   }
-    def __get_results(self, run_url):
-        results = []
-        bucket = run_url.split("s3://")[-1].split("/")[0]
-        prefix = "/".join(run_url.split("s3://")[-1].split("/")[1:])
-        receipt = self.__load_remote_file(bucket, f"{prefix}/receipt.json") # {"name": ..., "username": ... "args": <path to arg file>, "environment": ...}
-        args_list = self.__load_remote_file(bucket, receipt["args"])
-        for i in tqdm(range(len(args_list)), total=len(args_list), desc=f"{run_url}", unit="granule"):
-            try:
-                result = {
-                    "file": f"{prefix}/result{i}.json",
-                    "environment": receipt["environment"],
-                    "arg": args_list[i]
-                }
-                rsps = self.__load_remote_file(bucket, f"{prefix}/result{i}.json")
-                try:
-                    result |= {
-                        "status": rsps["status"] and JobStatus.SUCCESS or JobStatus.FAILURE,
-                        "duration": rsps["status"] and (rsps["stop"] - rsps["start"]) or 0.0,
-                        "outputs": rsps["outputs"]
-                    }
-                except Exception as e:
-                    result |= {
-                        "status": JobStatus.UNSUPPORTED,
-                        "result": rsps
-                    }
-            except Exception as e:
-                result |= {
-                    "status": JobStatus.ERROR,
-                    "error": f"{e}"
-                }
-            results.append(result)
-        return results
-
-    # Archive Database
-    def archive_database(self):
-        self.database.write(self.args.archive)
-        self.database.remove()
-
-    # Submit Job
-    def submit_job(self):
-        # pull out arguments
-        name = self.args.name
-        script_file = self.args.script
-        arguments_file = self.args.arguments
-        batch_size = self.args.batch_size
-        vcpus = self.args.vcpus
-        memory = self.args.memory
-        image = self.args.image
-        queue = self.args.queue
-        # read script
-        with open(script_file, "r") as file:
-            script = file.read()
-        # read arguments
-        with open(arguments_file, "r") as file:
-            arguments = [line.strip() for line in file.readlines()]
-        # process job in batches
-        for i in range(0, len(arguments), batch_size):
-            # build and check name
-            job_name = f"{name}_{i}"
-            if job_name in self.database.submissions:
-                unique = ''.join(random.choices(string.ascii_lowercase, k=3))
-                job_name = f"{name}_{unique}_{i}"
-            # submit & save job
-            args_list = arguments[i:i+batch_size]
-            rsps = self.session.runner.submit(name=job_name, script=script, args=args_list, optional_args={"vcpus":vcpus, "memory":memory, "image":image, "queue":queue})
-            self.database.submissions[job_name] = rsps | {"complete": False}
-            print(f"Submitted job {job_name} using script {script_file} with {len(args_list)} entries: {rsps}")
-
-    # Scrape Submissions
-    def scrape_submissions(self):
-        arg_list = []
-        for name,submission in self.database.submissions.items():
-            if self.args.name and self.args.name != name: continue
-            try:
-                if submission["complete"]:
-                    print(f"Scraping {name} ...")
-                    for i in range(len(submission["results"])):
-                        result = submission["results"][i]
-                        if result["status"] in self.args.status:
-                            print(f'{i:-5d}: {result["arg"]}')
-                            arg_list.append(result["arg"])
-                else:
-                    print(f"Skipping {name} because it is still pending")
-            except Exception as e:
-                print(f"Failed to scrape {name}: {e}")
-        if self.args.output:
-            with open(self.args.output, "w") as file:
-                for arg in arg_list:
-                    file.write(f"{arg}\n")
-
-    # Get Status
-    def get_status(self):
-        queue = self.args.queue
-        for name,job in self.database.submissions.items():
-            complete = job["complete"]
-            print(f"Statusing {name} ...")
-            if not complete:
-                queue_status = self.session.runner.queue(job_id=job["job_id"], queue=queue, verbose=True)
-                # jobs
-                child_jobs = queue_status["jobs"]
-                self.database.submissions[name]["jobs"] = {}
-                for child_job in child_jobs:
-                    self.database.submissions[name]["jobs"][int(child_job["index"])] = child_job
-                # report
-                report = queue_status["report"]
-                self.database.submissions[name]["status"] = report
-                jobs_in_progress = sum([report[s] for s in [JobState.SUBMITTED, JobState.PENDING, JobState.RUNNABLE, JobState.STARTING, JobState.RUNNING]])
-                jobs_complete = sum([report[s] for s in [JobState.SUCCEEDED, JobState.FAILED]])
-                if jobs_in_progress == 0 and jobs_complete > 0:
-                    print(f"Job {name} complete, reading results ...")
-                    self.database.submissions[name]["complete"] = True
-                    self.database.submissions[name]["results"] = self.__get_results(job["run_url"])
-                else:
-                    print(f"Job {name} still pending")
-        if self.args.verbose:
-            print(",".join([f"{c:>30}" for c in ["NAME"]] + [f"{c:>10}" for c in ["INDEX", "STATUS"]]))
-            for name,job in self.database.submissions.items():
-                for index, child_job in job["jobs"].items():
-                    print(",".join([f"{c:>30}" for c in [name]] + [f"{c:>10}" for c in [index, child_job["status"]]]))
-        else:
-            print(",".join([f"{c:>30}" for c in ["NAME"]] + [f"{c:>10}" for c in list(JobState)]))
-            for name,job in self.database.submissions.items():
-                print(",".join([f"{c:>30}" for c in [name]] + [f"{c:>10}" for c in [job["status"][state] for state in list(JobState)]]))
-
-    # Generate Report
-    def generate_report(self):
-        stats = {status.value: 0 for status in JobStatus}
-        duration = {"avg": 0.0, "total": 0.0}
-        processed = 0
-        pending = 0
-        errors = 0
-        for name,submission in self.database.submissions.items():
-            if self.args.name and self.args.name != name: continue
-            if submission["complete"]:
-                for result in submission["results"]:
-                    try:
-                        stats[result["status"]] += 1
-                        duration["total"] += result["duration"]
-                        processed += 1
-                    except Exception as e:
-                        errors += 1
-            else:
-                pending += 1
-        if processed > 0:
-            duration["avg"] = duration["total"] / processed
-        print("Processed:", processed)
-        print("Pending:", pending)
-        print("Errors:", errors)
-        print("Status:", json.dumps(stats, indent=2))
-        print("Duration:", json.dumps(duration, indent=2))
-
-    # Cancel Job
-    def cancel_job(self):
-        name = self.args.name
-        queue = self.args.queue
-        job_id = self.database.submissions[name]["job_id"]
-        rsps = self.session.runner.cancel(job_list=[job_id], queue=queue)
-        print(f"Cancelled submission {name}: {rsps}")
-
-    # Get Logs
-    def get_logs(self):
-        name = self.args.name
-        index = self.args.index
-        job_id = self.database.submissions[name]["jobs"][str(index)]["job_id"]
-        events = self.session.runner.logs(job_id=job_id)
-        for event in events:
-            print(event)
-
-    # Finish
-    def finish(self):
-        if not self.args.dryrun and "archive" not in self.args:
-            # save database
-            self.database.write()
-
-#########################################
-# Main
-#########################################
-
-def main():
-
-    # command line arguments
-    parser = argparse.ArgumentParser(prog="sliderule-runner", description="""SlideRule Runner""")
-    subparsers = parser.add_subparsers(dest="command", required=True)
-
-    # options shared by every subcommand; parents= lets them appear after the command name
-    common = argparse.ArgumentParser(add_help=False)
-    common.add_argument('--database',   type=Path,                  default=Path.home() / ".cache" / "sliderule" / "runner_database.json")
-    common.add_argument('--queue',      type=QueuePriority,         default=QueuePriority.DEFAULT, choices=list(QueuePriority))
-    common.add_argument('--verbose',    action='store_true',        default=False)
-    common.add_argument('--dryrun',     action='store_true',        default=False)
-
-    # archive
-    archive = subparsers.add_parser("archive", parents=[common], help="save and clear the database")
-    archive.add_argument('archive',     metavar="<full path to archive file>")
-    archive.set_defaults(func=Tool.archive_database)
-
-    # submit
-    submit = subparsers.add_parser("submit", parents=[common], help="submit a job")
-    submit.add_argument('name',         metavar="<name>")
-    submit.add_argument('script',       metavar="<script.lua>",     type=Path)
-    submit.add_argument('arguments',    metavar="<arguments.txt>",  type=Path)
-    submit.add_argument('--vcpus',      type=int,                   default=4)
-    submit.add_argument('--memory',     type=int,                   default=16000)
-    submit.add_argument('--batch_size', type=int,                   default=10000)
-    submit.add_argument('--image',      type=str,                   default="sliderule:latest")
-    submit.set_defaults(func=Tool.submit_job)
-
-    # scrape
-    scrape = subparsers.add_parser("scrape", parents=[common], help="generate list of arguments from jobs with provided job status")
-    scrape.add_argument('--status',     type=JobStatus, nargs='+',  default=[JobStatus.FAILURE, JobStatus.UNSUPPORTED, JobStatus.ERROR])
-    scrape.add_argument('--name',       type=str,                   default=None) # name of submission
-    scrape.add_argument('--output',     type=str,                   default=None)
-    scrape.set_defaults(func=Tool.scrape_submissions)
-
-    # status
-    status = subparsers.add_parser("status", parents=[common], help="display status of submitted jobs")
-    status.set_defaults(func=Tool.get_status)
-
-    # report
-    report = subparsers.add_parser("report", parents=[common], help="generate report of submitted jobs")
-    report.add_argument('--name',       type=str,                   default=None) # name of submission
-    report.set_defaults(func=Tool.generate_report)
-
-    # cancel
-    cancel = subparsers.add_parser("cancel", parents=[common], help="cancel a submitted job")
-    cancel.add_argument('--name',       type=str,                   required=True) # name of submission
-    cancel.set_defaults(func=Tool.cancel_job)
-
-    # logs
-    logs = subparsers.add_parser("logs", parents=[common], help="get log messages for a child job")
-    logs.add_argument('--name',       type=str,                     required=True) # name of submission
-    logs.add_argument('--index',      type=int,                     required=True) # job index
-    logs.set_defaults(func=Tool.get_logs)
-
-    # parse command line
-    args = parser.parse_args()
-
-    # create tool
-    tool = Tool(args)
-
-    # route command
-    try:
-        args.func(tool)
-        tool.finish() # only execute if tool function completed successfully
-    except Exception as e:
-        if args.verbose: raise
-        print(f"Unhandled error: {e}")
-
-# running via direct invocation
-if __name__ == "__main__": main()