dpdispatcher 0.6.6__py3-none-any.whl → 1.0.0__py3-none-any.whl

dpdispatcher/_version.py

@@ -1,16 +1,34 @@
-# file generated by setuptools_scm
+# file generated by setuptools-scm
 # don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
 TYPE_CHECKING = False
 if TYPE_CHECKING:
-    from typing import Tuple, Union
+    from typing import Tuple
+    from typing import Union
+
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = '1.0.0'
+__version_tuple__ = version_tuple = (1, 0, 0)
 
-__version__ = version = '0.6.6'
-__version_tuple__ = version_tuple = (0, 6, 6)
+__commit_id__ = commit_id = None

dpdispatcher/base_context.py

@@ -1,5 +1,5 @@
 from abc import ABCMeta, abstractmethod
-from typing import List, Tuple
+from typing import Any, List, Tuple
 
 from dargs import Argument
 
@@ -73,6 +73,65 @@ class BaseContext(metaclass=ABCMeta):
     def check_finish(self, proc):
         raise NotImplementedError("abstract method")
 
+    def block_checkcall(self, cmd, asynchronously=False) -> Tuple[Any, Any, Any]:
+        """Run command with arguments. Wait for command to complete.
+
+        Parameters
+        ----------
+        cmd : str
+            The command to run.
+        asynchronously : bool, optional, default=False
+            Run command asynchronously. If True, `nohup` will be used to run the command.
+
+        Returns
+        -------
+        stdin
+            standard inout
+        stdout
+            standard output
+        stderr
+            standard error
+
+        Raises
+        ------
+        RuntimeError
+            when the return code is not zero
+        """
+        if asynchronously:
+            cmd = f"nohup {cmd} >/dev/null &"
+        exit_status, stdin, stdout, stderr = self.block_call(cmd)
+        if exit_status != 0:
+            raise RuntimeError(
+                "Get error code {} in calling {} with job: {} . message: {}".format(
+                    exit_status,
+                    cmd,
+                    self.submission.submission_hash,
+                    stderr.read().decode("utf-8"),
+                )
+            )
+        return stdin, stdout, stderr
+
+    @abstractmethod
+    def block_call(self, cmd) -> Tuple[int, Any, Any, Any]:
+        """Run command with arguments. Wait for command to complete.
+
+        Parameters
+        ----------
+        cmd : str
+            The command to run.
+
+        Returns
+        -------
+        exit_status
+            exit code
+        stdin
+            standard inout
+        stdout
+            standard output
+        stderr
+            standard error
+        """
+
     @classmethod
     def machine_arginfo(cls) -> Argument:
         """Generate the machine arginfo.

dpdispatcher/contexts/dp_cloud_server_context.py

@@ -161,7 +161,9 @@ class BohriumContext(BaseContext):
         # return oss_task_zip
         # api.upload(self.oss_task_dir, zip_task_file)
 
-    def download(self, submission):
+    def download(
+        self, submission, check_exists=False, mark_failure=True, back_error=False
+    ):
         jobs = submission.belonging_jobs
         job_hashs = {}
         job_infos = {}
@@ -335,6 +337,11 @@ class BohriumContext(BaseContext):
             )
         ]
 
+    def block_call(self, cmd):
+        raise RuntimeError(
+            "Unsupported method. You may use an unsupported combination of the machine and the context."
+        )
+
 
 DpCloudServerContext = BohriumContext
 LebesgueContext = BohriumContext

dpdispatcher/contexts/hdfs_context.py

@@ -244,3 +244,8 @@ class HDFSContext(BaseContext):
 
     def read_file(self, fname):
         return HDFS.read_hdfs_file(os.path.join(self.remote_root, fname))
+
+    def block_call(self, cmd):
+        raise RuntimeError(
+            "Unsupported method. You may use an unsupported combination of the machine and the context."
+        )

dpdispatcher/contexts/lazy_local_context.py

@@ -83,7 +83,7 @@ class LazyLocalContext(BaseContext):
 
     def upload(
         self,
-        jobs,
+        submission,
         # local_up_files,
         dereference=True,
     ):
@@ -91,7 +91,7 @@ class LazyLocalContext(BaseContext):
 
     def download(
         self,
-        jobs,
+        submission,
         # remote_down_files,
         check_exists=False,
         mark_failure=True,
@@ -112,23 +112,6 @@ class LazyLocalContext(BaseContext):
     #                else:
     #                    raise RuntimeError('do not find download file ' + fname)
 
-    def block_checkcall(self, cmd):
-        # script_dir = os.path.join(self.local_root, self.submission.work_base)
-        # os.chdir(script_dir)
-        proc = sp.Popen(
-            cmd, cwd=self.local_root, shell=True, stdout=sp.PIPE, stderr=sp.PIPE
-        )
-        o, e = proc.communicate()
-        stdout = SPRetObj(o)
-        stderr = SPRetObj(e)
-        code = proc.returncode
-        if code != 0:
-            raise RuntimeError(
-                "Get error code %d in locally calling %s with job: %s ",
-                (code, cmd, self.submission.submission_hash),
-            )
-        return None, stdout, stderr
-
     def block_call(self, cmd):
         proc = sp.Popen(
             cmd, cwd=self.local_root, shell=True, stdout=sp.PIPE, stderr=sp.PIPE

dpdispatcher/contexts/local_context.py

@@ -3,6 +3,9 @@ import shutil
 import subprocess as sp
 from glob import glob
 from subprocess import TimeoutExpired
+from typing import List
+
+from dargs import Argument
 
 from dpdispatcher.base_context import BaseContext
 from dpdispatcher.dlog import dlog
@@ -60,6 +63,7 @@ class LocalContext(BaseContext):
         self.temp_local_root = os.path.abspath(local_root)
         self.temp_remote_root = os.path.abspath(remote_root)
         self.remote_profile = remote_profile
+        self.symlink = remote_profile.get("symlink", True)
 
     @classmethod
     def load_from_dict(cls, context_dict):
@@ -83,6 +87,25 @@ class LocalContext(BaseContext):
             self.temp_remote_root, submission.submission_hash
         )
 
+    def _copy_from_local_to_remote(self, local_path, remote_path):
+        if not os.path.exists(local_path):
+            raise FileNotFoundError(
+                f"cannot find uploaded file {os.path.join(local_path)}"
+            )
+        if os.path.exists(remote_path):
+            os.remove(remote_path)
+        _check_file_path(remote_path)
+
+        if self.symlink:
+            # ensure the file exist
+            os.symlink(local_path, remote_path)
+        elif os.path.isfile(local_path):
+            shutil.copyfile(local_path, remote_path)
+        elif os.path.isdir(local_path):
+            shutil.copytree(local_path, remote_path)
+        else:
+            raise ValueError(f"Unknown file type: {local_path}")
+
     def upload(self, submission):
         os.makedirs(self.remote_root, exist_ok=True)
         for ii in submission.belonging_tasks:
@@ -103,14 +126,9 @@ class LocalContext(BaseContext):
                 file_list.extend(rel_file_list)
 
             for jj in file_list:
-                if not os.path.exists(os.path.join(local_job, jj)):
-                    raise FileNotFoundError(
-                        "cannot find upload file " + os.path.join(local_job, jj)
-                    )
-                if os.path.exists(os.path.join(remote_job, jj)):
-                    os.remove(os.path.join(remote_job, jj))
-                _check_file_path(os.path.join(remote_job, jj))
-                os.symlink(os.path.join(local_job, jj), os.path.join(remote_job, jj))
+                self._copy_from_local_to_remote(
+                    os.path.join(local_job, jj), os.path.join(remote_job, jj)
+                )
 
         local_job = self.local_root
         remote_job = self.remote_root
@@ -128,14 +146,9 @@ class LocalContext(BaseContext):
             file_list.extend(rel_file_list)
 
         for jj in file_list:
-            if not os.path.exists(os.path.join(local_job, jj)):
-                raise FileNotFoundError(
-                    "cannot find upload file " + os.path.join(local_job, jj)
-                )
-            if os.path.exists(os.path.join(remote_job, jj)):
-                os.remove(os.path.join(remote_job, jj))
-            _check_file_path(os.path.join(remote_job, jj))
-            os.symlink(os.path.join(local_job, jj), os.path.join(remote_job, jj))
+            self._copy_from_local_to_remote(
+                os.path.join(local_job, jj), os.path.join(remote_job, jj)
+            )
 
     def download(
         self, submission, check_exists=False, mark_failure=True, back_error=False
@@ -288,21 +301,6 @@ class LocalContext(BaseContext):
                 # no nothing in the case of linked files
                 pass
 
-    def block_checkcall(self, cmd):
-        proc = sp.Popen(
-            cmd, cwd=self.remote_root, shell=True, stdout=sp.PIPE, stderr=sp.PIPE
-        )
-        o, e = proc.communicate()
-        stdout = SPRetObj(o)
-        stderr = SPRetObj(e)
-        code = proc.returncode
-        if code != 0:
-            raise RuntimeError(
-                f"Get error code {code} in locally calling {cmd} with job: {self.submission.submission_hash}"
-                f"\nStandard error: {stderr}"
-            )
-        return None, stdout, stderr
-
     def block_call(self, cmd):
         proc = sp.Popen(
             cmd, cwd=self.remote_root, shell=True, stdout=sp.PIPE, stderr=sp.PIPE
@@ -351,3 +349,31 @@ class LocalContext(BaseContext):
                 stdout = None
                 stderr = None
         return ret, stdout, stderr
+
+    @classmethod
+    def machine_subfields(cls) -> List[Argument]:
+        """Generate the machine subfields.
+
+        Returns
+        -------
+        list[Argument]
+            machine subfields
+        """
+        doc_remote_profile = "The information used to maintain the local machine."
+        return [
+            Argument(
+                "remote_profile",
+                dict,
+                optional=True,
+                doc=doc_remote_profile,
+                sub_fields=[
+                    Argument(
+                        "symlink",
+                        bool,
+                        optional=True,
+                        default=True,
+                        doc="Whether to use symbolic links to replace copy. This option should be turned off if the local directory is not accessible on the Batch system.",
+                    ),
+                ],
+            )
+        ]

dpdispatcher/contexts/openapi_context.py

@@ -1,18 +1,20 @@
+import glob
 import os
 import shutil
 import uuid
+from zipfile import ZipFile
 
 import tqdm
 
 try:
-    from bohriumsdk.client import Client
-    from bohriumsdk.job import Job
-    from bohriumsdk.storage import Storage
-    from bohriumsdk.util import Util
-except ModuleNotFoundError:
+    from bohrium import Bohrium
+    from bohrium.resources import Job, Tiefblue
+except ModuleNotFoundError as e:
     found_bohriumsdk = False
+    import_bohrium_error = e
 else:
     found_bohriumsdk = True
+    import_bohrium_error = None
 
 from dpdispatcher.base_context import BaseContext
 from dpdispatcher.dlog import dlog
@@ -23,6 +25,36 @@ DP_CLOUD_SERVER_HOME_DIR = os.path.join(
 )
 
 
+def unzip_file(zip_file, out_dir="./"):
+    obj = ZipFile(zip_file, "r")
+    for item in obj.namelist():
+        obj.extract(item, out_dir)
+
+
+def zip_file_list(root_path, zip_filename, file_list=[]):
+    out_zip_file = os.path.join(root_path, zip_filename)
+    # print('debug: file_list', file_list)
+    zip_obj = ZipFile(out_zip_file, "w")
+    for f in file_list:
+        matched_files = os.path.join(root_path, f)
+        for ii in glob.glob(matched_files):
+            # print('debug: matched_files:ii', ii)
+            if os.path.isdir(ii):
+                arcname = os.path.relpath(ii, start=root_path)
+                zip_obj.write(ii, arcname)
+                for root, dirs, files in os.walk(ii):
+                    for file in files:
+                        filename = os.path.join(root, file)
+                        arcname = os.path.relpath(filename, start=root_path)
+                        # print('debug: filename:arcname:root_path', filename, arcname, root_path)
+                        zip_obj.write(filename, arcname)
+            else:
+                arcname = os.path.relpath(ii, start=root_path)
+                zip_obj.write(ii, arcname)
+    zip_obj.close()
+    return out_zip_file
+
+
 class OpenAPIContext(BaseContext):
     def __init__(
         self,
@@ -35,16 +67,41 @@ class OpenAPIContext(BaseContext):
         if not found_bohriumsdk:
             raise ModuleNotFoundError(
                 "bohriumsdk not installed. Install dpdispatcher with `pip install dpdispatcher[bohrium]`"
-            )
+            ) from import_bohrium_error
         self.init_local_root = local_root
         self.init_remote_root = remote_root
         self.temp_local_root = os.path.abspath(local_root)
         self.remote_profile = remote_profile
-        self.client = Client()
-        self.storage = Storage(client=self.client)
-        self.job = Job(client=self.client)
-        self.util = Util()
+        access_key = (
+            remote_profile.get("access_key", None)
+            or os.getenv("BOHRIUM_ACCESS_KEY", None)
+            or os.getenv("ACCESS_KEY", None)
+        )
+        project_id = (
+            remote_profile.get("project_id", None)
+            or os.getenv("BOHRIUM_PROJECT_ID", None)
+            or os.getenv("PROJECT_ID", None)
+        )
+        app_key = (
+            remote_profile.get("app_key", None)
+            or os.getenv("BOHRIUM_APP_KEY", None)
+            or os.getenv("APP_KEY", None)
+        )
+        if access_key is None:
+            raise ValueError(
+                "remote_profile must contain 'access_key' or set environment variable 'BOHRIUM_ACCESS_KEY'"
+            )
+        if project_id is None:
+            raise ValueError(
+                "remote_profile must contain 'project_id' or set environment variable 'BOHRIUM_PROJECT_ID'"
+            )
+        self.client = Bohrium(  # type: ignore[reportPossiblyUnboundVariable]
+            access_key=access_key, project_id=project_id, app_key=app_key
+        )
+        self.storage = Tiefblue()  # type: ignore[reportPossiblyUnboundVariable]
+        self.job = Job(client=self.client)  # type: ignore[reportPossiblyUnboundVariable]
         self.jgid = None
+        os.makedirs(DP_CLOUD_SERVER_HOME_DIR, exist_ok=True)
 
     @classmethod
     def load_from_dict(cls, context_dict):
@@ -97,7 +154,7 @@ class OpenAPIContext(BaseContext):
             for file in task.forward_files:
                 upload_file_list.append(os.path.join(task.task_work_path, file))
 
-        upload_zip = Util.zip_file_list(
+        upload_zip = zip_file_list(
             self.local_root, zip_task_file, file_list=upload_file_list
         )
         project_id = self.remote_profile.get("project_id", 0)
@@ -113,7 +170,7 @@ class OpenAPIContext(BaseContext):
         object_key = os.path.join(data["storePath"], zip_filename)  # type: ignore
         job.upload_path = object_key
         job.job_id = data["jobId"]  # type: ignore
-        job.jgid = data["jobGroupId"]  # type: ignore
+        job.jgid = data.get("jobGroupId", "")  # type: ignore
         self.storage.upload_From_file_multi_part(
             object_key=object_key, file_path=upload_zip, token=token
         )
@@ -149,7 +206,9 @@ class OpenAPIContext(BaseContext):
         # return oss_task_zip
         # api.upload(self.oss_task_dir, zip_task_file)
 
-    def download(self, submission):
+    def download(
+        self, submission, check_exists=False, mark_failure=True, back_error=False
+    ):
         jobs = submission.belonging_jobs
         job_hashs = {}
         job_infos = {}
@@ -189,7 +248,7 @@ class OpenAPIContext(BaseContext):
             ):
                 continue
             self.storage.download_from_url(info["resultUrl"], target_result_zip)
-            Util.unzip_file(target_result_zip, out_dir=self.local_root)
+            unzip_file(target_result_zip, out_dir=self.local_root)
             self._backup(self.local_root, target_result_zip)
         self._clean_backup(
             self.local_root, keep_backup=self.remote_profile.get("keep_backup", True)
@@ -258,3 +317,8 @@ class OpenAPIContext(BaseContext):
             dir_to_be_removed = os.path.join(local_root, "backup")
             if os.path.exists(dir_to_be_removed):
                 shutil.rmtree(dir_to_be_removed)
+
+    def block_call(self, cmd):
+        raise RuntimeError(
+            "Unsupported method. You may use an unsupported combination of the machine and the context."
+        )

dpdispatcher/contexts/ssh_context.py

@@ -44,6 +44,8 @@ class SSHSession:
         totp_secret=None,
         tar_compress=True,
         look_for_keys=True,
+        execute_command=None,
+        proxy_command=None,
     ):
         self.hostname = hostname
         self.username = username
@@ -56,6 +58,8 @@ class SSHSession:
         self.ssh = None
         self.tar_compress = tar_compress
         self.look_for_keys = look_for_keys
+        self.execute_command = execute_command
+        self.proxy_command = proxy_command
         self._keyboard_interactive_auth = False
         self._setup_ssh()
 
@@ -88,8 +92,7 @@ class SSHSession:
         while not self._check_alive():
             if count == max_check:
                 raise RuntimeError(
-                    "cannot connect ssh after %d failures at interval %d s"
-                    % (max_check, sleep_time)
+                    f"cannot connect ssh after {max_check} failures at interval {sleep_time} s"
                 )
             dlog.info("connection check failed, try to reconnect to " + self.hostname)
             self._setup_ssh()
@@ -141,7 +144,12 @@ class SSHSession:
         # transport = self.ssh.get_transport()
         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         sock.settimeout(self.timeout)
-        sock.connect((self.hostname, self.port))
+
+        # Use ProxyCommand if configured (either directly or via jump host parameters)
+        if self.proxy_command is not None:
+            sock = paramiko.ProxyCommand(self.proxy_command)
+        else:
+            sock.connect((self.hostname, self.port))
 
         # Make a Paramiko Transport object using the socket
         ts = paramiko.Transport(sock)
@@ -162,7 +170,6 @@ class SSHSession:
             if os.path.exists(key_path):
                 for pkey_class in (
                     paramiko.RSAKey,
-                    paramiko.DSSKey,
                     paramiko.ECDSAKey,
                     paramiko.Ed25519Key,
                 ):
@@ -180,7 +187,6 @@ class SSHSession:
         elif self.look_for_keys:
             for keytype, name in [
                 (paramiko.RSAKey, "rsa"),
-                (paramiko.DSSKey, "dsa"),
                 (paramiko.ECDSAKey, "ecdsa"),
                 (paramiko.Ed25519Key, "ed25519"),
             ]:
@@ -237,6 +243,8 @@ class SSHSession:
         self.ssh._transport = ts  # type: ignore
         # reset sftp
         self._sftp = None
+        if self.execute_command is not None:
+            self.exec_command(self.execute_command)
 
     def inter_handler(self, title, instructions, prompt_list):
         """inter_handler: the callback for paramiko.transport.auth_interactive.
@@ -338,6 +346,10 @@ class SSHSession:
         doc_look_for_keys = (
             "enable searching for discoverable private key files in ~/.ssh/"
         )
+        doc_execute_command = "execute command after ssh connection is established."
+        doc_proxy_command = (
+            "ProxyCommand to use for SSH connection through intermediate servers."
+        )
         ssh_remote_profile_args = [
             Argument("hostname", str, optional=False, doc=doc_hostname),
             Argument("username", str, optional=False, doc=doc_username),
@@ -379,6 +391,20 @@ class SSHSession:
                 default=True,
                 doc=doc_look_for_keys,
             ),
+            Argument(
+                "execute_command",
+                str,
+                optional=True,
+                default=None,
+                doc=doc_execute_command,
+            ),
+            Argument(
+                "proxy_command",
+                [str, type(None)],
+                optional=True,
+                default=None,
+                doc=doc_proxy_command,
+            ),
         ]
         ssh_remote_profile_format = Argument(
             "ssh_session", dict, ssh_remote_profile_args
@@ -387,23 +413,37 @@ class SSHSession:
 
     def put(self, from_f, to_f):
         if self.rsync_available:
+            # For rsync, we need to use %h:%p placeholders for target host/port
+            proxy_cmd_rsync = None
+            if self.proxy_command is not None:
+                proxy_cmd_rsync = self.proxy_command.replace(
+                    f"{self.hostname}:{self.port}", "%h:%p"
+                )
             return rsync(
                 from_f,
                 self.remote + ":" + to_f,
                 port=self.port,
                 key_filename=self.key_filename,
                 timeout=self.timeout,
+                proxy_command=proxy_cmd_rsync,
             )
         return self.sftp.put(from_f, to_f)
 
     def get(self, from_f, to_f):
         if self.rsync_available:
+            # For rsync, we need to use %h:%p placeholders for target host/port
+            proxy_cmd_rsync = None
+            if self.proxy_command is not None:
+                proxy_cmd_rsync = self.proxy_command.replace(
+                    f"{self.hostname}:{self.port}", "%h:%p"
+                )
             return rsync(
                 self.remote + ":" + from_f,
                 to_f,
                 port=self.port,
                 key_filename=self.key_filename,
                 timeout=self.timeout,
+                proxy_command=proxy_cmd_rsync,
             )
         return self.sftp.get(from_f, to_f)
 
@@ -438,7 +478,9 @@ class SSHContext(BaseContext):
         self.init_local_root = local_root
         self.init_remote_root = remote_root
         self.temp_local_root = os.path.abspath(local_root)
-        assert os.path.isabs(remote_root), "remote_root must be a abspath"
+        assert os.path.isabs(os.path.realpath(remote_root)), (
+            "remote_root must be a abspath"
+        )
         self.temp_remote_root = remote_root
         self.remote_profile = remote_profile
         self.remote_root = None
@@ -755,41 +797,6 @@ class SSHContext(BaseContext):
                 tar_compress=self.remote_profile.get("tar_compress", None),
             )
 
-    def block_checkcall(self, cmd, asynchronously=False, stderr_whitelist=None):
-        """Run command with arguments. Wait for command to complete. If the return code
-        was zero then return, otherwise raise RuntimeError.
-
-        Parameters
-        ----------
-        cmd : str
-            The command to run.
-        asynchronously : bool, optional, default=False
-            Run command asynchronously. If True, `nohup` will be used to run the command.
-        stderr_whitelist : list of str, optional, default=None
-            If not None, the stderr will be checked against the whitelist. If the stderr
-            contains any of the strings in the whitelist, the command will be considered
-            successful.
-        """
-        assert self.remote_root is not None
-        self.ssh_session.ensure_alive()
-        if asynchronously:
-            cmd = f"nohup {cmd} >/dev/null &"
-        stdin, stdout, stderr = self.ssh_session.exec_command(
-            (f"cd {shlex.quote(self.remote_root)} ;") + cmd
-        )
-        exit_status = stdout.channel.recv_exit_status()
-        if exit_status != 0:
-            raise RuntimeError(
-                "Get error code %d in calling %s through ssh with job: %s . message: %s"
-                % (
-                    exit_status,
-                    cmd,
-                    self.submission.submission_hash,
-                    stderr.read().decode("utf-8"),
-                )
-            )
-        return stdin, stdout, stderr
-
     def block_call(self, cmd):
         assert self.remote_root is not None
         self.ssh_session.ensure_alive()
@@ -849,8 +856,8 @@ class SSHContext(BaseContext):
         # print(pid)
         return {"stdin": stdin, "stdout": stdout, "stderr": stderr}
 
-    def check_finish(self, cmd_pipes):
-        return cmd_pipes["stdout"].channel.exit_status_ready()
+    def check_finish(self, proc):
+        return proc["stdout"].channel.exit_status_ready()
 
     def get_return(self, cmd_pipes):
         if not self.check_finish(cmd_pipes):
@@ -912,11 +919,11 @@ class SSHContext(BaseContext):
         # local tar
         if os.path.isfile(os.path.join(self.local_root, of)):
             os.remove(os.path.join(self.local_root, of))
-        with tarfile.open(
+        with tarfile.open(  # type: ignore[reportCallIssue, reportArgumentType]
             os.path.join(self.local_root, of),
-            tarfile_mode,
-            dereference=dereference,
-            **kwargs,
+            mode=tarfile_mode,  # type: ignore[reportArgumentType]
+            dereference=dereference,  # type: ignore[reportArgumentType]
+            **kwargs,  # type: ignore[reportArgumentType]
         ) as tar:
             # avoid compressing duplicated files or directories
             for ii in set(files):