parsl 2024.11.4__py3-none-any.whl → 2024.11.18__py3-none-any.whl

parsl/channels/ssh_il/ssh_il.py

@@ -1,85 +0,0 @@
-import getpass
-import logging
-
-from parsl.channels.ssh.ssh import DeprecatedSSHChannel
-from parsl.errors import OptionalModuleMissing
-
-try:
-    import paramiko
-    _ssh_enabled = True
-except (ImportError, NameError, FileNotFoundError):
-    _ssh_enabled = False
-
-
-logger = logging.getLogger(__name__)
-
-
-class DeprecatedSSHInteractiveLoginChannel(DeprecatedSSHChannel):
-    """SSH persistent channel. This enables remote execution on sites
-    accessible via ssh. This channel supports interactive login and is appropriate when
-    keys are not set up.
-    """
-
-    def __init__(self, hostname, username=None, password=None, script_dir=None, envs=None):
-        ''' Initialize a persistent connection to the remote system.
-        We should know at this point whether ssh connectivity is possible
-
-        Args:
-            - hostname (String) : Hostname
-
-        KWargs:
-            - username (string) : Username on remote system
-            - password (string) : Password for remote system
-            - script_dir (string) : Full path to a script dir where
-              generated scripts could be sent to.
-            - envs (dict) : A dictionary of env variables to be set when executing commands
-
-        Raises:
-        '''
-        if not _ssh_enabled:
-            raise OptionalModuleMissing(['ssh'],
-                                        "SSHInteractiveLoginChannel requires the ssh module and config.")
-
-        self.hostname = hostname
-        self.username = username
-        self.password = password
-
-        self.ssh_client = paramiko.SSHClient()
-        self.ssh_client.load_system_host_keys()
-        self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
-        self.script_dir = script_dir
-
-        self.envs = {}
-        if envs is not None:
-            self.envs = envs
-
-        try:
-            self.ssh_client.connect(
-                hostname, username=username, password=password, allow_agent=True
-            )
-
-        except Exception:
-            logger.debug("Caught the SSHException in SSHInteractive")
-            pass
-        '''
-        except paramiko.BadHostKeyException as e:
-            raise BadHostKeyException(e, self.hostname)
-
-        except paramiko.AuthenticationException as e:
-            raise AuthException(e, self.hostname)
-
-        except paramiko.SSHException as e:
-            logger.debug("Caught the SSHException in SSHInteractive")
-            pass
-
-        except Exception as e:
-            raise SSHException(e, self.hostname)
-        '''
-
-        transport = self.ssh_client.get_transport()
-
-        il_password = getpass.getpass('Enter {0} Logon password :'.format(hostname))
-        transport.auth_password(username, il_password)
-
-        self.sftp_client = paramiko.SFTPClient.from_transport(transport)

parsl/providers/ad_hoc/ad_hoc.py

@@ -1,252 +0,0 @@
-import logging
-import os
-import time
-
-from parsl.channels import LocalChannel
-from parsl.jobs.states import JobState, JobStatus
-from parsl.launchers import SimpleLauncher
-from parsl.providers.base import ExecutionProvider
-from parsl.providers.errors import ScriptPathError
-from parsl.utils import RepresentationMixin
-
-logger = logging.getLogger(__name__)
-
-
-class DeprecatedAdHocProvider(ExecutionProvider, RepresentationMixin):
-    """ Deprecated ad-hoc execution provider
-
-    The (former) AdHocProvider is deprecated. See
-    `issue #3515 <https://github.com/Parsl/parsl/issues/3515>`_
-    for further discussion.
-
-    This provider is used to provision execution resources over one or more ad hoc nodes
-    that are each accessible over a Channel (say, ssh) but otherwise lack a cluster scheduler.
-
-    Parameters
-    ----------
-
-    channels : list of Channel ojects
-      Each channel represents a connection to a remote node
-
-    worker_init : str
-      Command to be run before starting a worker, such as 'module load Anaconda; source activate env'.
-      Since this provider calls the same worker_init across all nodes in the ad-hoc cluster, it is
-      recommended that a single script is made available across nodes such as ~/setup.sh that can
-      be invoked.
-
-    cmd_timeout : int
-      Duration for which the provider will wait for a command to be invoked on a remote system.
-      Defaults to 30s
-
-    parallelism : float
-      Determines the ratio of workers to tasks as managed by the strategy component
-
-    """
-
-    def __init__(self,
-                 channels=[],
-                 worker_init='',
-                 cmd_timeout=30,
-                 parallelism=1,
-                 move_files=None):
-
-        self.channels = channels
-        self._label = 'ad-hoc'
-        self.worker_init = worker_init
-        self.cmd_timeout = cmd_timeout
-        self.parallelism = 1
-        self.move_files = move_files
-        self.launcher = SimpleLauncher()
-        self.init_blocks = self.min_blocks = self.max_blocks = len(channels)
-
-        # This will be overridden by the DFK to the rundirs.
-        self.script_dir = "."
-
-        # In ad-hoc mode, nodes_per_block should be 1
-        self.nodes_per_block = 1
-
-        # Dictionary that keeps track of jobs, keyed on job_id
-        self.resources = {}
-
-        self.least_loaded = self._least_loaded()
-        logger.debug("AdHoc provider initialized")
-
-    def _write_submit_script(self, script_string, script_filename):
-        '''
-        Load the template string with config values and write the generated submit script to
-        a submit script file.
-
-        Parameters
-        ----------
-        script_string: (string)
-          The template string to be used for the writing submit script
-
-        script_filename: (string)
-          Name of the submit script
-
-        Returns
-        -------
-        None: on success
-
-        Raises
-        ------
-        ScriptPathError
-          Unable to write submit script out
-        '''
-
-        try:
-            with open(script_filename, 'w') as f:
-                f.write(script_string)
-
-        except IOError as e:
-            logger.error("Failed writing to submit script: %s", script_filename)
-            raise ScriptPathError(script_filename, e)
-
-        return None
-
-    def _least_loaded(self):
-        """ Find channels that are not in use
-
-        Returns
-        -------
-        channel : Channel object
-        None : When there are no more available channels
-        """
-        while True:
-            channel_counts = {channel: 0 for channel in self.channels}
-            for job_id in self.resources:
-                channel = self.resources[job_id]['channel']
-                if self.resources[job_id]['status'].state == JobState.RUNNING:
-                    channel_counts[channel] = channel_counts.get(channel, 0) + 1
-                else:
-                    channel_counts[channel] = channel_counts.get(channel, 0)
-
-            logger.debug("Channel_counts : {}".format(channel_counts))
-            if 0 not in channel_counts.values():
-                yield None
-
-            for channel in channel_counts:
-                if channel_counts[channel] == 0:
-                    yield channel
-
-    def submit(self, command, tasks_per_node, job_name="parsl.adhoc"):
-        ''' Submits the command onto a channel from the list of channels
-
-        Submit returns an ID that corresponds to the task that was just submitted.
-
-        Parameters
-        ----------
-        command: (String)
-          Commandline invocation to be made on the remote side.
-
-        tasks_per_node: (int)
-          command invocations to be launched per node
-
-        job_name: (String)
-          Name of the job. Default : parsl.adhoc
-
-
-        Returns
-        -------
-        None
-          At capacity, cannot provision more
-
-        job_id: (string)
-          Identifier for the job
-
-        '''
-        channel = next(self.least_loaded)
-        if channel is None:
-            logger.warning("All Channels in Ad-Hoc provider are in use")
-            return None
-
-        job_name = "{0}.{1}".format(job_name, time.time())
-
-        # Set script path
-        script_path = "{0}/{1}.sh".format(self.script_dir, job_name)
-        script_path = os.path.abspath(script_path)
-
-        wrap_command = self.worker_init + '\n' + self.launcher(command, tasks_per_node, self.nodes_per_block)
-
-        self._write_submit_script(wrap_command, script_path)
-
-        job_id = None
-        remote_pid = None
-        final_cmd = None
-
-        if (self.move_files is None and not isinstance(channel, LocalChannel)) or (self.move_files):
-            logger.debug("Pushing start script")
-            script_path = channel.push_file(script_path, channel.script_dir)
-
-        # Bash would return until the streams are closed. So we redirect to a outs file
-        final_cmd = 'bash {0} > {0}.out 2>&1 & \n echo "PID:$!" '.format(script_path)
-        retcode, stdout, stderr = channel.execute_wait(final_cmd, self.cmd_timeout)
-        for line in stdout.split('\n'):
-            if line.startswith("PID:"):
-                remote_pid = line.split("PID:")[1].strip()
-                job_id = remote_pid
-        if job_id is None:
-            logger.warning("Channel failed to start remote command/retrieve PID")
-
-        self.resources[job_id] = {'job_id': job_id,
-                                  'status': JobStatus(JobState.RUNNING),
-                                  'cmd': final_cmd,
-                                  'channel': channel,
-                                  'remote_pid': remote_pid}
-
-        return job_id
-
-    def status(self, job_ids):
-        """ Get status of the list of jobs with job_ids
-
-        Parameters
-        ----------
-        job_ids : list of strings
-          List of job id strings
-
-        Returns
-        -------
-        list of JobStatus objects
-        """
-        for job_id in job_ids:
-            channel = self.resources[job_id]['channel']
-            status_command = "ps --pid {} | grep {}".format(self.resources[job_id]['job_id'],
-                                                            self.resources[job_id]['cmd'].split()[0])
-            retcode, stdout, stderr = channel.execute_wait(status_command)
-            if retcode != 0 and self.resources[job_id]['status'].state == JobState.RUNNING:
-                self.resources[job_id]['status'] = JobStatus(JobState.FAILED)
-
-        return [self.resources[job_id]['status'] for job_id in job_ids]
-
-    def cancel(self, job_ids):
-        """ Cancel a list of jobs with job_ids
-
-        Parameters
-        ----------
-        job_ids : list of strings
-          List of job id strings
-
-        Returns
-        -------
-        list of confirmation bools: [True, False...]
-        """
-        logger.debug("Cancelling jobs: {}".format(job_ids))
-        rets = []
-        for job_id in job_ids:
-            channel = self.resources[job_id]['channel']
-            cmd = "kill -TERM -$(ps -o pgid= {} | grep -o '[0-9]*')".format(self.resources[job_id]['job_id'])
-            retcode, stdout, stderr = channel.execute_wait(cmd)
-            if retcode == 0:
-                rets.append(True)
-            else:
-                rets.append(False)
-            self.resources[job_id]['status'] = JobStatus(JobState.COMPLETED)
-        return rets
-
-    @property
-    def label(self):
-        return self._label
-
-    @property
-    def status_polling_interval(self):
-        return 10

parsl/tests/configs/local_adhoc.py

@@ -1,18 +0,0 @@
-from parsl.channels import LocalChannel
-from parsl.config import Config
-from parsl.executors import HighThroughputExecutor
-from parsl.providers.ad_hoc.ad_hoc import DeprecatedAdHocProvider
-
-
-def fresh_config():
-    return Config(
-        executors=[
-            HighThroughputExecutor(
-                label='AdHoc',
-                encrypted=True,
-                provider=DeprecatedAdHocProvider(
-                    channels=[LocalChannel(), LocalChannel()]
-                )
-            )
-        ]
-    )

parsl/tests/sites/test_local_adhoc.py

@@ -1,62 +0,0 @@
-import logging
-
-import pytest
-
-from parsl import python_app
-from parsl.tests.configs.local_adhoc import fresh_config as local_config
-
-logger = logging.getLogger(__name__)
-
-
-@python_app
-def python_app_2():
-    import os
-    import threading
-    import time
-    time.sleep(1)
-    return "Hello from PID[{}] TID[{}]".format(os.getpid(), threading.current_thread())
-
-
-@python_app
-def python_app_1():
-    import os
-    import threading
-    import time
-    time.sleep(1)
-    return "Hello from PID[{}] TID[{}]".format(os.getpid(), threading.current_thread())
-
-
-@python_app
-def bash_app(stdout=None, stderr=None):
-    return 'echo "Hello from $(uname -a)" ; sleep 2'
-
-
-@pytest.mark.local
-def test_python(N=2):
-    """Testing basic python functionality."""
-
-    r1 = {}
-    r2 = {}
-    for i in range(0, N):
-        r1[i] = python_app_1()
-        r2[i] = python_app_2()
-    print("Waiting ....")
-
-    for x in r1:
-        print("python_app_1 : ", r1[x].result())
-    for x in r2:
-        print("python_app_2 : ", r2[x].result())
-
-    return
-
-
-@pytest.mark.local
-def test_bash():
-    """Testing basic bash functionality."""
-
-    import os
-    fname = os.path.basename(__file__)
-
-    x = bash_app(stdout="{0}.out".format(fname))
-    print("Waiting ....")
-    print(x.result())

parsl/tests/test_channels/test_dfk_close.py

@@ -1,26 +0,0 @@
-from unittest.mock import Mock
-
-import pytest
-
-import parsl
-from parsl.channels.base import Channel
-from parsl.executors import HighThroughputExecutor
-from parsl.providers import LocalProvider
-
-
-@pytest.mark.local
-def test_dfk_close():
-
-    mock_channel = Mock(spec=Channel)
-
-    # block settings all 0 because the mock channel won't be able to
-    # do anything to make a block exist
-    p = LocalProvider(channel=mock_channel, init_blocks=0, min_blocks=0, max_blocks=0)
-
-    e = HighThroughputExecutor(provider=p)
-
-    c = parsl.Config(executors=[e])
-    with parsl.load(c):
-        pass
-
-    assert mock_channel.close.called