parsl 2024.3.18__py3-none-any.whl → 2025.1.13__py3-none-any.whl

parsl/providers/cobalt/cobalt.py

@@ -1,236 +0,0 @@
-import logging
-import os
-import time
-import warnings
-
-from parsl.providers.errors import ScaleOutFailed
-from parsl.channels import LocalChannel
-from parsl.launchers import AprunLauncher
-from parsl.providers.cobalt.template import template_string
-from parsl.providers.cluster_provider import ClusterProvider
-from parsl.jobs.states import JobState, JobStatus
-from parsl.utils import RepresentationMixin, wtime_to_minutes
-
-logger = logging.getLogger(__name__)
-
-translate_table = {
-    'QUEUED': JobState.PENDING,
-    'STARTING': JobState.PENDING,
-    'RUNNING': JobState.RUNNING,
-    'EXITING': JobState.COMPLETED,
-    'KILLING': JobState.COMPLETED
-}
-
-
-class CobaltProvider(ClusterProvider, RepresentationMixin):
-    """ Cobalt Execution Provider
-
-    WARNING: CobaltProvider is deprecated and will be removed by 2024.04
-
-    This provider uses cobalt to submit (qsub), obtain the status of (qstat), and cancel (qdel)
-    jobs. Theo script to be used is created from a template file in this
-    same module.
-
-    Parameters
-    ----------
-    channel : Channel
-        Channel for accessing this provider. Possible channels include
-        :class:`~parsl.channels.LocalChannel` (the default),
-        :class:`~parsl.channels.SSHChannel`, or
-        :class:`~parsl.channels.SSHInteractiveLoginChannel`.
-    nodes_per_block : int
-        Nodes to provision per block.
-    min_blocks : int
-        Minimum number of blocks to maintain.
-    max_blocks : int
-        Maximum number of blocks to maintain.
-    walltime : str
-        Walltime requested per block in HH:MM:SS.
-    account : str
-        Account that the job will be charged against.
-    queue : str
-        Torque queue to request blocks from.
-    scheduler_options : str
-        String to prepend to the submit script to the scheduler.
-    worker_init : str
-        Command to be run before starting a worker, such as 'module load Anaconda; source activate env'.
-    launcher : Launcher
-        Launcher for this provider. Possible launchers include
-        :class:`~parsl.launchers.AprunLauncher` (the default) or,
-        :class:`~parsl.launchers.SingleNodeLauncher`
-    """
-    def __init__(self,
-                 channel=LocalChannel(),
-                 nodes_per_block=1,
-                 init_blocks=0,
-                 min_blocks=0,
-                 max_blocks=1,
-                 parallelism=1,
-                 walltime="00:10:00",
-                 account=None,
-                 queue=None,
-                 scheduler_options='',
-                 worker_init='',
-                 launcher=AprunLauncher(),
-                 cmd_timeout=10):
-        label = 'cobalt'
-        super().__init__(label,
-                         channel=channel,
-                         nodes_per_block=nodes_per_block,
-                         init_blocks=init_blocks,
-                         min_blocks=min_blocks,
-                         max_blocks=max_blocks,
-                         parallelism=parallelism,
-                         walltime=walltime,
-                         launcher=launcher,
-                         cmd_timeout=cmd_timeout)
-
-        self.account = account
-        self.queue = queue
-        self.scheduler_options = scheduler_options
-        self.worker_init = worker_init
-        warnings.warn("CobaltProvider is deprecated; This will be removed after 2024-04",
-                      DeprecationWarning,
-                      stacklevel=2)
-
-    def _status(self):
-        """Returns the status list for a list of job_ids
-
-        Args:
-              self
-
-        Returns:
-              [status...] : Status list of all jobs
-        """
-
-        jobs_missing = list(self.resources.keys())
-
-        retcode, stdout, stderr = self.execute_wait("qstat -u $USER")
-
-        # Execute_wait failed. Do no update.
-        if retcode != 0:
-            return
-
-        for line in stdout.split('\n'):
-            if line.startswith('='):
-                continue
-
-            parts = line.upper().split()
-            if parts and parts[0] != 'JOBID':
-                job_id = parts[0]
-
-                if job_id not in self.resources:
-                    continue
-
-                status = translate_table.get(parts[4], JobState.UNKNOWN)
-
-                self.resources[job_id]['status'] = JobStatus(status)
-                jobs_missing.remove(job_id)
-
-        # squeue does not report on jobs that are not running. So we are filling in the
-        # blanks for missing jobs, we might lose some information about why the jobs failed.
-        for missing_job in jobs_missing:
-            self.resources[missing_job]['status'] = JobStatus(JobState.COMPLETED)
-
-    def submit(self, command, tasks_per_node, job_name="parsl.cobalt"):
-        """ Submits the command onto an Local Resource Manager job of parallel elements.
-        Submit returns an ID that corresponds to the task that was just submitted.
-
-        If tasks_per_node <  1 : ! This is illegal. tasks_per_node should be integer
-
-        If tasks_per_node == 1:
-             A single node is provisioned
-
-        If tasks_per_node >  1 :
-             tasks_per_node number of nodes are provisioned.
-
-        Args:
-             - command  :(String) Commandline invocation to be made on the remote side.
-             - tasks_per_node (int) : command invocations to be launched per node
-
-        Kwargs:
-             - job_name (String): Name for job, must be unique
-
-        Returns:
-             - None: At capacity, cannot provision more
-             - job_id: (string) Identifier for the job
-
-        """
-
-        account_opt = '-A {}'.format(self.account) if self.account is not None else ''
-
-        job_name = "parsl.{0}.{1}".format(job_name, time.time())
-
-        script_path = "{0}/{1}.submit".format(self.script_dir, job_name)
-        script_path = os.path.abspath(script_path)
-
-        job_config = {}
-        job_config["scheduler_options"] = self.scheduler_options
-        job_config["worker_init"] = self.worker_init
-
-        logger.debug("Requesting nodes_per_block:%s tasks_per_node:%s",
-                     self.nodes_per_block, tasks_per_node)
-
-        # Wrap the command
-        job_config["user_script"] = self.launcher(command, tasks_per_node, self.nodes_per_block)
-
-        queue_opt = '-q {}'.format(self.queue) if self.queue is not None else ''
-
-        logger.debug("Writing submit script")
-        self._write_submit_script(template_string, script_path, job_name, job_config)
-
-        channel_script_path = self.channel.push_file(script_path, self.channel.script_dir)
-
-        command = 'qsub -n {0} {1} -t {2} {3} {4}'.format(
-            self.nodes_per_block, queue_opt, wtime_to_minutes(self.walltime), account_opt, channel_script_path)
-        logger.debug("Executing {}".format(command))
-
-        retcode, stdout, stderr = self.execute_wait(command)
-
-        # TODO : FIX this block
-        if retcode != 0:
-            logger.error("Failed command: {0}".format(command))
-            logger.error("Launch failed stdout:\n{0} \nstderr:{1}\n".format(stdout, stderr))
-
-        logger.debug("Retcode:%s STDOUT:%s STDERR:%s", retcode, stdout.strip(), stderr.strip())
-
-        job_id = None
-
-        if retcode == 0:
-            # We should be getting only one line back
-            job_id = stdout.strip()
-            self.resources[job_id] = {'job_id': job_id, 'status': JobStatus(JobState.PENDING)}
-        else:
-            logger.error("Submit command failed: {0}".format(stderr))
-            raise ScaleOutFailed(self.__class__, "Request to submit job to local scheduler failed")
-
-        logger.debug("Returning job id : {0}".format(job_id))
-        return job_id
-
-    def cancel(self, job_ids):
-        """ Cancels the jobs specified by a list of job ids
-
-        Args:
-        job_ids : [<job_id> ...]
-
-        Returns :
-        [True/False...] : If the cancel operation fails the entire list will be False.
-        """
-
-        job_id_list = ' '.join(job_ids)
-        retcode, stdout, stderr = self.execute_wait("qdel {0}".format(job_id_list))
-        rets = None
-        if retcode == 0:
-            for jid in job_ids:
-                # ???
-                # self.resources[jid]['status'] = translate_table['KILLING']  # Setting state to cancelled
-                self.resources[jid]['status'] = JobStatus(JobState.COMPLETED)
-            rets = [True for i in job_ids]
-        else:
-            rets = [False for i in job_ids]
-
-        return rets
-
-    @property
-    def status_polling_interval(self):
-        return 60

parsl/providers/cobalt/template.py

@@ -1,17 +0,0 @@
-template_string = '''#!/bin/bash -el
-${scheduler_options}
-
-${worker_init}
-
-echo "Starting Cobalt job script"
-
-echo "----Cobalt Nodefile: -----"
-cat $$COBALT_NODEFILE
-echo "--------------------------"
-
-export JOBNAME="${jobname}"
-
-$user_script
-
-echo "End of Cobalt job"
-'''

parsl/tests/configs/ad_hoc_cluster_htex.py

@@ -1,35 +0,0 @@
-from parsl.providers import AdHocProvider
-from parsl.channels import SSHChannel
-from parsl.executors import HighThroughputExecutor
-from parsl.config import Config
-
-from typing import Any, Dict
-
-user_opts = {'adhoc':
-             {'username': 'YOUR_USERNAME',
-              'script_dir': 'YOUR_SCRIPT_DIR',
-              'remote_hostnames': ['REMOTE_HOST_URL_1', 'REMOTE_HOST_URL_2']
-              }
-             }  # type: Dict[str, Dict[str, Any]]
-
-config = Config(
-    executors=[
-        HighThroughputExecutor(
-            label='remote_htex',
-            max_workers_per_node=2,
-            worker_logdir_root=user_opts['adhoc']['script_dir'],
-            encrypted=True,
-            provider=AdHocProvider(
-                # Command to be run before starting a worker, such as:
-                # 'module load Anaconda; source activate parsl_env'.
-                worker_init='',
-                channels=[SSHChannel(hostname=m,
-                                     username=user_opts['adhoc']['username'],
-                                     script_dir=user_opts['adhoc']['script_dir'],
-                                     ) for m in user_opts['adhoc']['remote_hostnames']]
-            )
-        )
-    ],
-    #  AdHoc Clusters should not be setup with scaling strategy.
-    strategy='none',
-)

parsl/tests/configs/cooley_htex.py

@@ -1,37 +0,0 @@
-# UNTESTED
-
-from parsl.config import Config
-from parsl.executors import HighThroughputExecutor
-from parsl.launchers import MpiRunLauncher
-from parsl.providers import CobaltProvider
-
-# If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py
-# If you are a user copying-and-pasting this as an example, make sure to either
-#       1) create a local `user_opts.py`, or
-#       2) delete the user_opts import below and replace all appearances of `user_opts` with the literal value
-#          (i.e., user_opts['swan']['username'] -> 'your_username')
-from .user_opts import user_opts
-
-config = Config(
-    executors=[
-        HighThroughputExecutor(
-            label="cooley_htex",
-            worker_debug=False,
-            cores_per_worker=1,
-            encrypted=True,
-            provider=CobaltProvider(
-                queue='debug',
-                account=user_opts['cooley']['account'],
-                launcher=MpiRunLauncher(),  # UNTESTED COMPONENT
-                scheduler_options=user_opts['cooley']['scheduler_options'],
-                worker_init=user_opts['cooley']['worker_init'],
-                init_blocks=1,
-                max_blocks=1,
-                min_blocks=1,
-                nodes_per_block=4,
-                cmd_timeout=60,
-                walltime='00:10:00',
-            ),
-        )
-    ]
-)

parsl/tests/configs/htex_ad_hoc_cluster.py

@@ -1,28 +0,0 @@
-from parsl.providers import AdHocProvider
-from parsl.channels import SSHChannel
-from parsl.executors import HighThroughputExecutor
-
-from parsl.config import Config
-
-from parsl.tests.configs.user_opts import user_opts
-
-config = Config(
-    executors=[
-        HighThroughputExecutor(
-            label='remote_htex',
-            cores_per_worker=1,
-            worker_debug=False,
-            address=user_opts['public_ip'],
-            encrypted=True,
-            provider=AdHocProvider(
-                move_files=False,
-                parallelism=1,
-                worker_init=user_opts['adhoc']['worker_init'],
-                channels=[SSHChannel(hostname=m,
-                                     username=user_opts['adhoc']['username'],
-                                     script_dir=user_opts['adhoc']['script_dir'],
-                                     ) for m in user_opts['adhoc']['remote_hostnames']]
-            )
-        )
-    ],
-)

parsl/tests/configs/local_adhoc.py

@@ -1,18 +0,0 @@
-from parsl.config import Config
-from parsl.channels import LocalChannel
-from parsl.executors import HighThroughputExecutor
-from parsl.providers import AdHocProvider
-
-
-def fresh_config():
-    return Config(
-        executors=[
-            HighThroughputExecutor(
-                label='AdHoc',
-                encrypted=True,
-                provider=AdHocProvider(
-                    channels=[LocalChannel(), LocalChannel()]
-                )
-            )
-        ]
-    )

parsl/tests/configs/swan_htex.py

@@ -1,43 +0,0 @@
-"""
-================== Block
-| ++++++++++++++ | Node
-| |            | |
-| |    Task    | |             . . .
-| |            | |
-| ++++++++++++++ |
-==================
-"""
-from parsl.channels import SSHChannel
-from parsl.launchers import AprunLauncher
-from parsl.providers import TorqueProvider
-from parsl.config import Config
-from parsl.executors import HighThroughputExecutor
-
-# If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py
-# If you are a user copying-and-pasting this as an example, make sure to either
-#       1) create a local `user_opts.py`, or
-#       2) delete the user_opts import below and replace all appearances of `user_opts` with the literal value
-#          (i.e., user_opts['swan']['username'] -> 'your_username')
-from .user_opts import user_opts
-
-config = Config(
-    executors=[
-        HighThroughputExecutor(
-            label='swan_htex',
-            encrypted=True,
-            provider=TorqueProvider(
-                channel=SSHChannel(
-                    hostname='swan.cray.com',
-                    username=user_opts['swan']['username'],
-                    script_dir=user_opts['swan']['script_dir'],
-                ),
-                nodes_per_block=1,
-                init_blocks=1,
-                max_blocks=1,
-                launcher=AprunLauncher(),
-                scheduler_options=user_opts['swan']['scheduler_options'],
-                worker_init=user_opts['swan']['worker_init'],
-            ),
-        )
-    ]
-)

parsl/tests/configs/theta.py

@@ -1,37 +0,0 @@
-from parsl.config import Config
-from parsl.providers import CobaltProvider
-from parsl.launchers import AprunLauncher
-from parsl.executors import HighThroughputExecutor
-
-from .user_opts import user_opts
-
-
-def fresh_config():
-    return Config(
-        executors=[
-            HighThroughputExecutor(
-                label='theta_local_htex_multinode',
-                max_workers_per_node=1,
-                encrypted=True,
-                provider=CobaltProvider(
-                    queue=user_opts['theta']['queue'],
-                    account=user_opts['theta']['account'],
-                    launcher=AprunLauncher(overrides="-d 64"),
-                    walltime='00:10:00',
-                    nodes_per_block=2,
-                    init_blocks=1,
-                    max_blocks=1,
-                    # string to prepend to #COBALT blocks in the submit
-                    # script to the scheduler eg: '#COBALT -t 50'
-                    scheduler_options='',
-                    # Command to be run before starting a worker, such as:
-                    # 'module load Anaconda; source activate parsl_env'.
-                    worker_init=user_opts['theta']['worker_init'],
-                    cmd_timeout=120,
-                ),
-            )
-        ],
-    )
-
-
-config = fresh_config()

parsl/tests/integration/test_channels/test_channels.py

@@ -1,17 +0,0 @@
-from parsl.channels.local.local import LocalChannel
-
-
-def test_local():
-
-    channel = LocalChannel(None, None)
-
-    ec, out, err = channel.execute_wait('echo "pwd: $PWD"', 2)
-
-    assert ec == 0, "Channel execute failed"
-    print("Stdout: ", out)
-    print("Stderr: ", err)
-
-
-if __name__ == "__main__":
-
-    test_local()

parsl/tests/integration/test_channels/test_local_channel.py

@@ -1,42 +0,0 @@
-from parsl.channels.local.local import LocalChannel
-
-
-def test_env():
-    ''' Regression testing for issue #27
-    '''
-
-    lc = LocalChannel()
-    rc, stdout, stderr = lc.execute_wait("env", 1)
-
-    stdout = stdout.split('\n')
-    x = [s for s in stdout if s.startswith("PATH=")]
-    assert x, "PATH not found"
-
-    x = [s for s in stdout if s.startswith("HOME=")]
-    assert x, "HOME not found"
-
-    print("RC:{} \nSTDOUT:{} \nSTDERR:{}".format(rc, stdout, stderr))
-
-
-def test_env_mod():
-    ''' Testing for env update at execute time.
-    '''
-
-    lc = LocalChannel()
-    rc, stdout, stderr = lc.execute_wait("env", 1, {'TEST_ENV': 'fooo'})
-
-    stdout = stdout.split('\n')
-    x = [s for s in stdout if s.startswith("PATH=")]
-    assert x, "PATH not found"
-
-    x = [s for s in stdout if s.startswith("HOME=")]
-    assert x, "HOME not found"
-
-    x = [s for s in stdout if s.startswith("TEST_ENV=fooo")]
-    assert x, "User set env missing"
-
-
-if __name__ == "__main__":
-
-    test_env()
-    test_env_mod()

parsl/tests/integration/test_channels/test_scp_1.py

@@ -1,45 +0,0 @@
-import os
-
-from parsl.channels.ssh.ssh import SSHChannel as SSH
-
-
-def connect_and_list(hostname, username):
-    out = ''
-    conn = SSH(hostname, username=username)
-    conn.push_file(os.path.abspath('remote_run.sh'), '/home/davidk/')
-    # ec, out, err = conn.execute_wait("ls /tmp/remote_run.sh; bash /tmp/remote_run.sh")
-    conn.close()
-    return out
-
-
-script = '''#!/bin/bash
-echo "Hostname: $HOSTNAME"
-echo "Cpu info -----"
-cat /proc/cpuinfo
-echo "Done----------"
-'''
-
-
-def test_connect_1():
-    with open('remote_run.sh', 'w') as f:
-        f.write(script)
-
-    sites = {
-        'midway': {
-            'url': 'midway.rcc.uchicago.edu',
-            'uname': 'yadunand'
-        },
-        'swift': {
-            'url': 'swift.rcc.uchicago.edu',
-            'uname': 'yadunand'
-        }
-    }
-
-    for site in sites.values():
-        out = connect_and_list(site['url'], site['uname'])
-        print("Sitename :{0}  hostname:{1}".format(site['url'], out))
-
-
-if __name__ == "__main__":
-
-    test_connect_1()

parsl/tests/integration/test_channels/test_ssh_1.py

@@ -1,40 +0,0 @@
-from parsl.channels.ssh.ssh import SSHChannel as SSH
-
-
-def connect_and_list(hostname, username):
-    conn = SSH(hostname, username=username)
-    ec, out, err = conn.execute_wait("echo $HOSTNAME")
-    conn.close()
-    return out
-
-
-def test_midway():
-    ''' Test ssh channels to midway
-    '''
-    url = 'midway.rcc.uchicago.edu'
-    uname = 'yadunand'
-    out = connect_and_list(url, uname)
-    print("Sitename :{0}  hostname:{1}".format(url, out))
-
-
-def test_beagle():
-    ''' Test ssh channels to beagle
-    '''
-    url = 'login04.beagle.ci.uchicago.edu'
-    uname = 'yadunandb'
-    out = connect_and_list(url, uname)
-    print("Sitename :{0}  hostname:{1}".format(url, out))
-
-
-def test_osg():
-    ''' Test ssh connectivity to osg
-    '''
-    url = 'login.osgconnect.net'
-    uname = 'yadunand'
-    out = connect_and_list(url, uname)
-    print("Sitename :{0}  hostname:{1}".format(url, out))
-
-
-if __name__ == "__main__":
-
-    pass

parsl/tests/integration/test_channels/test_ssh_errors.py

@@ -1,46 +0,0 @@
-from parsl.channels.errors import SSHException, BadHostKeyException
-from parsl.channels.ssh.ssh import SSHChannel as SSH
-
-
-def connect_and_list(hostname, username):
-    conn = SSH(hostname, username=username)
-    ec, out, err = conn.execute_wait("echo $HOSTNAME")
-    conn.close()
-    return out
-
-
-def test_error_1():
-    try:
-        connect_and_list("bad.url.gov", "ubuntu")
-    except Exception as e:
-        assert type(e) is SSHException, "Expected SSException, got: {0}".format(e)
-
-
-def test_error_2():
-    try:
-        connect_and_list("swift.rcc.uchicago.edu", "mango")
-    except SSHException:
-        print("Caught the right exception")
-    else:
-        raise Exception("Expected SSException, got: {0}".format(e))
-
-
-def test_error_3():
-    ''' This should work
-    '''
-    try:
-        connect_and_list("edison.nersc.gov", "yadunand")
-    except BadHostKeyException as e:
-        print("Caught exception BadHostKeyException: ", e)
-    else:
-        assert False, "Expected SSException, got: {0}".format(e)
-
-
-if __name__ == "__main__":
-
-    tests = [test_error_1, test_error_2, test_error_3]
-
-    for test in tests:
-        print("---------Running : {0}---------------".format(test))
-        test()
-        print("----------------------DONE--------------------------")