parsl 2024.10.28__py3-none-any.whl → 2024.11.11__py3-none-any.whl

parsl/providers/cobalt/cobalt.py

@@ -1,236 +0,0 @@
-import logging
-import os
-import time
-import warnings
-
-from parsl.channels import LocalChannel
-from parsl.jobs.states import JobState, JobStatus
-from parsl.launchers import AprunLauncher
-from parsl.providers.cluster_provider import ClusterProvider
-from parsl.providers.cobalt.template import template_string
-from parsl.providers.errors import ScaleOutFailed
-from parsl.utils import RepresentationMixin, wtime_to_minutes
-
-logger = logging.getLogger(__name__)
-
-translate_table = {
-    'QUEUED': JobState.PENDING,
-    'STARTING': JobState.PENDING,
-    'RUNNING': JobState.RUNNING,
-    'EXITING': JobState.COMPLETED,
-    'KILLING': JobState.COMPLETED
-}
-
-
-class CobaltProvider(ClusterProvider, RepresentationMixin):
-    """ Cobalt Execution Provider
-
-    WARNING: CobaltProvider is deprecated and will be removed by 2024.04
-
-    This provider uses cobalt to submit (qsub), obtain the status of (qstat), and cancel (qdel)
-    jobs. Theo script to be used is created from a template file in this
-    same module.
-
-    Parameters
-    ----------
-    channel : Channel
-        Channel for accessing this provider. Possible channels include
-        :class:`~parsl.channels.LocalChannel` (the default),
-        :class:`~parsl.channels.SSHChannel`, or
-        :class:`~parsl.channels.SSHInteractiveLoginChannel`.
-    nodes_per_block : int
-        Nodes to provision per block.
-    min_blocks : int
-        Minimum number of blocks to maintain.
-    max_blocks : int
-        Maximum number of blocks to maintain.
-    walltime : str
-        Walltime requested per block in HH:MM:SS.
-    account : str
-        Account that the job will be charged against.
-    queue : str
-        Torque queue to request blocks from.
-    scheduler_options : str
-        String to prepend to the submit script to the scheduler.
-    worker_init : str
-        Command to be run before starting a worker, such as 'module load Anaconda; source activate env'.
-    launcher : Launcher
-        Launcher for this provider. Possible launchers include
-        :class:`~parsl.launchers.AprunLauncher` (the default) or,
-        :class:`~parsl.launchers.SingleNodeLauncher`
-    """
-    def __init__(self,
-                 channel=LocalChannel(),
-                 nodes_per_block=1,
-                 init_blocks=0,
-                 min_blocks=0,
-                 max_blocks=1,
-                 parallelism=1,
-                 walltime="00:10:00",
-                 account=None,
-                 queue=None,
-                 scheduler_options='',
-                 worker_init='',
-                 launcher=AprunLauncher(),
-                 cmd_timeout=10):
-        label = 'cobalt'
-        super().__init__(label,
-                         channel=channel,
-                         nodes_per_block=nodes_per_block,
-                         init_blocks=init_blocks,
-                         min_blocks=min_blocks,
-                         max_blocks=max_blocks,
-                         parallelism=parallelism,
-                         walltime=walltime,
-                         launcher=launcher,
-                         cmd_timeout=cmd_timeout)
-
-        self.account = account
-        self.queue = queue
-        self.scheduler_options = scheduler_options
-        self.worker_init = worker_init
-        warnings.warn("CobaltProvider is deprecated; This will be removed after 2024-04",
-                      DeprecationWarning,
-                      stacklevel=2)
-
-    def _status(self):
-        """Returns the status list for a list of job_ids
-
-        Args:
-              self
-
-        Returns:
-              [status...] : Status list of all jobs
-        """
-
-        jobs_missing = list(self.resources.keys())
-
-        retcode, stdout, stderr = self.execute_wait("qstat -u $USER")
-
-        # Execute_wait failed. Do no update.
-        if retcode != 0:
-            return
-
-        for line in stdout.split('\n'):
-            if line.startswith('='):
-                continue
-
-            parts = line.upper().split()
-            if parts and parts[0] != 'JOBID':
-                job_id = parts[0]
-
-                if job_id not in self.resources:
-                    continue
-
-                status = translate_table.get(parts[4], JobState.UNKNOWN)
-
-                self.resources[job_id]['status'] = JobStatus(status)
-                jobs_missing.remove(job_id)
-
-        # squeue does not report on jobs that are not running. So we are filling in the
-        # blanks for missing jobs, we might lose some information about why the jobs failed.
-        for missing_job in jobs_missing:
-            self.resources[missing_job]['status'] = JobStatus(JobState.COMPLETED)
-
-    def submit(self, command, tasks_per_node, job_name="parsl.cobalt"):
-        """ Submits the command onto an Local Resource Manager job of parallel elements.
-        Submit returns an ID that corresponds to the task that was just submitted.
-
-        If tasks_per_node <  1 : ! This is illegal. tasks_per_node should be integer
-
-        If tasks_per_node == 1:
-             A single node is provisioned
-
-        If tasks_per_node >  1 :
-             tasks_per_node number of nodes are provisioned.
-
-        Args:
-             - command  :(String) Commandline invocation to be made on the remote side.
-             - tasks_per_node (int) : command invocations to be launched per node
-
-        Kwargs:
-             - job_name (String): Name for job, must be unique
-
-        Returns:
-             - None: At capacity, cannot provision more
-             - job_id: (string) Identifier for the job
-
-        """
-
-        account_opt = '-A {}'.format(self.account) if self.account is not None else ''
-
-        job_name = "parsl.{0}.{1}".format(job_name, time.time())
-
-        script_path = "{0}/{1}.submit".format(self.script_dir, job_name)
-        script_path = os.path.abspath(script_path)
-
-        job_config = {}
-        job_config["scheduler_options"] = self.scheduler_options
-        job_config["worker_init"] = self.worker_init
-
-        logger.debug("Requesting nodes_per_block:%s tasks_per_node:%s",
-                     self.nodes_per_block, tasks_per_node)
-
-        # Wrap the command
-        job_config["user_script"] = self.launcher(command, tasks_per_node, self.nodes_per_block)
-
-        queue_opt = '-q {}'.format(self.queue) if self.queue is not None else ''
-
-        logger.debug("Writing submit script")
-        self._write_submit_script(template_string, script_path, job_name, job_config)
-
-        channel_script_path = self.channel.push_file(script_path, self.channel.script_dir)
-
-        command = 'qsub -n {0} {1} -t {2} {3} {4}'.format(
-            self.nodes_per_block, queue_opt, wtime_to_minutes(self.walltime), account_opt, channel_script_path)
-        logger.debug("Executing {}".format(command))
-
-        retcode, stdout, stderr = self.execute_wait(command)
-
-        # TODO : FIX this block
-        if retcode != 0:
-            logger.error("Failed command: {0}".format(command))
-            logger.error("Launch failed stdout:\n{0} \nstderr:{1}\n".format(stdout, stderr))
-
-        logger.debug("Retcode:%s STDOUT:%s STDERR:%s", retcode, stdout.strip(), stderr.strip())
-
-        job_id = None
-
-        if retcode == 0:
-            # We should be getting only one line back
-            job_id = stdout.strip()
-            self.resources[job_id] = {'job_id': job_id, 'status': JobStatus(JobState.PENDING)}
-        else:
-            logger.error("Submit command failed: {0}".format(stderr))
-            raise ScaleOutFailed(self.__class__, "Request to submit job to local scheduler failed")
-
-        logger.debug("Returning job id : {0}".format(job_id))
-        return job_id
-
-    def cancel(self, job_ids):
-        """ Cancels the jobs specified by a list of job ids
-
-        Args:
-        job_ids : [<job_id> ...]
-
-        Returns :
-        [True/False...] : If the cancel operation fails the entire list will be False.
-        """
-
-        job_id_list = ' '.join(job_ids)
-        retcode, stdout, stderr = self.execute_wait("qdel {0}".format(job_id_list))
-        rets = None
-        if retcode == 0:
-            for jid in job_ids:
-                # ???
-                # self.resources[jid]['status'] = translate_table['KILLING']  # Setting state to cancelled
-                self.resources[jid]['status'] = JobStatus(JobState.COMPLETED)
-            rets = [True for i in job_ids]
-        else:
-            rets = [False for i in job_ids]
-
-        return rets
-
-    @property
-    def status_polling_interval(self):
-        return 60

parsl/providers/cobalt/template.py

@@ -1,17 +0,0 @@
-template_string = '''#!/bin/bash -el
-${scheduler_options}
-
-${worker_init}
-
-echo "Starting Cobalt job script"
-
-echo "----Cobalt Nodefile: -----"
-cat $$COBALT_NODEFILE
-echo "--------------------------"
-
-export JOBNAME="${jobname}"
-
-$user_script
-
-echo "End of Cobalt job"
-'''

parsl/tests/configs/cooley_htex.py

@@ -1,37 +0,0 @@
-# UNTESTED
-
-from parsl.config import Config
-from parsl.executors import HighThroughputExecutor
-from parsl.launchers import MpiRunLauncher
-from parsl.providers import CobaltProvider
-
-# If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py
-# If you are a user copying-and-pasting this as an example, make sure to either
-#       1) create a local `user_opts.py`, or
-#       2) delete the user_opts import below and replace all appearances of `user_opts` with the literal value
-#          (i.e., user_opts['swan']['username'] -> 'your_username')
-from .user_opts import user_opts
-
-config = Config(
-    executors=[
-        HighThroughputExecutor(
-            label="cooley_htex",
-            worker_debug=False,
-            cores_per_worker=1,
-            encrypted=True,
-            provider=CobaltProvider(
-                queue='debug',
-                account=user_opts['cooley']['account'],
-                launcher=MpiRunLauncher(),  # UNTESTED COMPONENT
-                scheduler_options=user_opts['cooley']['scheduler_options'],
-                worker_init=user_opts['cooley']['worker_init'],
-                init_blocks=1,
-                max_blocks=1,
-                min_blocks=1,
-                nodes_per_block=4,
-                cmd_timeout=60,
-                walltime='00:10:00',
-            ),
-        )
-    ]
-)

parsl/tests/configs/local_adhoc.py

@@ -1,18 +0,0 @@
-from parsl.channels import LocalChannel
-from parsl.config import Config
-from parsl.executors import HighThroughputExecutor
-from parsl.providers.ad_hoc.ad_hoc import DeprecatedAdHocProvider
-
-
-def fresh_config():
-    return Config(
-        executors=[
-            HighThroughputExecutor(
-                label='AdHoc',
-                encrypted=True,
-                provider=DeprecatedAdHocProvider(
-                    channels=[LocalChannel(), LocalChannel()]
-                )
-            )
-        ]
-    )

parsl/tests/configs/theta.py

@@ -1,37 +0,0 @@
-from parsl.config import Config
-from parsl.executors import HighThroughputExecutor
-from parsl.launchers import AprunLauncher
-from parsl.providers import CobaltProvider
-
-from .user_opts import user_opts
-
-
-def fresh_config():
-    return Config(
-        executors=[
-            HighThroughputExecutor(
-                label='theta_local_htex_multinode',
-                max_workers_per_node=1,
-                encrypted=True,
-                provider=CobaltProvider(
-                    queue=user_opts['theta']['queue'],
-                    account=user_opts['theta']['account'],
-                    launcher=AprunLauncher(overrides="-d 64"),
-                    walltime='00:10:00',
-                    nodes_per_block=2,
-                    init_blocks=1,
-                    max_blocks=1,
-                    # string to prepend to #COBALT blocks in the submit
-                    # script to the scheduler eg: '#COBALT -t 50'
-                    scheduler_options='',
-                    # Command to be run before starting a worker, such as:
-                    # 'module load Anaconda; source activate parsl_env'.
-                    worker_init=user_opts['theta']['worker_init'],
-                    cmd_timeout=120,
-                ),
-            )
-        ],
-    )
-
-
-config = fresh_config()

parsl/tests/manual_tests/test_fan_in_out_htex_remote.py

@@ -1,88 +0,0 @@
-import logging
-
-import parsl
-from parsl.app.app import python_app
-from parsl.config import Config
-from parsl.executors import HighThroughputExecutor
-from parsl.launchers import AprunLauncher
-from parsl.monitoring.monitoring import MonitoringHub
-from parsl.providers import CobaltProvider
-
-
-def local_setup():
-    threads_config = Config(
-        executors=[
-            HighThroughputExecutor(
-                label="theta_htex",
-                # worker_debug=True,
-                cores_per_worker=4,
-                encrypted=True,
-                provider=CobaltProvider(
-                    queue='debug-flat-quad',
-                    account='CSC249ADCD01',
-                    launcher=AprunLauncher(overrides="-d 64"),
-                    worker_init='source activate parsl-issues',
-                    init_blocks=1,
-                    max_blocks=1,
-                    min_blocks=1,
-                    nodes_per_block=4,
-                    cmd_timeout=60,
-                    walltime='00:10:00',
-                ),
-            )
-        ],
-        monitoring=MonitoringHub(
-            hub_port=55055,
-            logging_level=logging.DEBUG,
-            resource_monitoring_interval=10),
-        strategy='none')
-    parsl.load(threads_config)
-
-
-def local_teardown():
-    parsl.clear()
-
-
-@python_app
-def inc(x):
-    import time
-    start = time.time()
-    sleep_duration = 30.0
-    while True:
-        x += 1
-        end = time.time()
-        if end - start >= sleep_duration:
-            break
-    return x
-
-
-@python_app
-def add_inc(inputs=[]):
-    import time
-    start = time.time()
-    sleep_duration = 30.0
-    res = sum(inputs)
-    while True:
-        res += 1
-        end = time.time()
-        if end - start >= sleep_duration:
-            break
-    return res
-
-
-if __name__ == "__main__":
-
-    total = 200
-    half = int(total / 2)
-    one_third = int(total / 3)
-    two_third = int(total / 3 * 2)
-    futures_1 = [inc(i) for i in range(total)]
-    futures_2 = [add_inc(inputs=futures_1[0:half]),
-                 add_inc(inputs=futures_1[half:total])]
-    futures_3 = [inc(futures_2[0]) for _ in range(half)] + [inc(futures_2[1]) for _ in range(half)]
-    futures_4 = [add_inc(inputs=futures_3[0:one_third]),
-                 add_inc(inputs=futures_3[one_third:two_third]),
-                 add_inc(inputs=futures_3[two_third:total])]
-
-    print([f.result() for f in futures_4])
-    print("Done")

parsl/tests/sites/test_local_adhoc.py

@@ -1,62 +0,0 @@
-import logging
-
-import pytest
-
-from parsl import python_app
-from parsl.tests.configs.local_adhoc import fresh_config as local_config
-
-logger = logging.getLogger(__name__)
-
-
-@python_app
-def python_app_2():
-    import os
-    import threading
-    import time
-    time.sleep(1)
-    return "Hello from PID[{}] TID[{}]".format(os.getpid(), threading.current_thread())
-
-
-@python_app
-def python_app_1():
-    import os
-    import threading
-    import time
-    time.sleep(1)
-    return "Hello from PID[{}] TID[{}]".format(os.getpid(), threading.current_thread())
-
-
-@python_app
-def bash_app(stdout=None, stderr=None):
-    return 'echo "Hello from $(uname -a)" ; sleep 2'
-
-
-@pytest.mark.local
-def test_python(N=2):
-    """Testing basic python functionality."""
-
-    r1 = {}
-    r2 = {}
-    for i in range(0, N):
-        r1[i] = python_app_1()
-        r2[i] = python_app_2()
-    print("Waiting ....")
-
-    for x in r1:
-        print("python_app_1 : ", r1[x].result())
-    for x in r2:
-        print("python_app_2 : ", r2[x].result())
-
-    return
-
-
-@pytest.mark.local
-def test_bash():
-    """Testing basic bash functionality."""
-
-    import os
-    fname = os.path.basename(__file__)
-
-    x = bash_app(stdout="{0}.out".format(fname))
-    print("Waiting ....")
-    print(x.result())

parsl/tests/test_channels/test_dfk_close.py

@@ -1,26 +0,0 @@
-from unittest.mock import Mock
-
-import pytest
-
-import parsl
-from parsl.channels.base import Channel
-from parsl.executors import HighThroughputExecutor
-from parsl.providers import LocalProvider
-
-
-@pytest.mark.local
-def test_dfk_close():
-
-    mock_channel = Mock(spec=Channel)
-
-    # block settings all 0 because the mock channel won't be able to
-    # do anything to make a block exist
-    p = LocalProvider(channel=mock_channel, init_blocks=0, min_blocks=0, max_blocks=0)
-
-    e = HighThroughputExecutor(provider=p)
-
-    c = parsl.Config(executors=[e])
-    with parsl.load(c):
-        pass
-
-    assert mock_channel.close.called

parsl/tests/test_providers/test_cobalt_deprecation_warning.py

@@ -1,18 +0,0 @@
-import warnings
-
-import pytest
-
-from parsl.providers import CobaltProvider
-
-
-@pytest.mark.local
-def test_deprecation_warning():
-
-    with warnings.catch_warnings(record=True) as w:
-        warnings.simplefilter("always")
-
-        CobaltProvider()
-
-        assert len(w) == 1
-        assert issubclass(w[-1].category, DeprecationWarning)
-        assert "CobaltProvider" in str(w[-1].message)