brynq-sdk-ftp 2.0.1__tar.gz → 2.0.4__tar.gz

{brynq_sdk_ftp-2.0.1 → brynq_sdk_ftp-2.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: brynq_sdk_ftp
-Version: 2.0.1
+Version: 2.0.4
 Summary: FTP wrapper from BrynQ
 Home-page: UNKNOWN
 Author: BrynQ

brynq_sdk_ftp-2.0.4/brynq_sdk_ftp/__init__.py

@@ -0,0 +1,2 @@
+from .ftps import FTPS
+from .sftp import SFTP

brynq_sdk_ftp-2.0.4/brynq_sdk_ftp/ftps.py

@@ -0,0 +1,227 @@
+from brynq_sdk_brynq import BrynQ
+import os
+import ftplib
+import time
+from ftplib import FTP
+from typing import Union, List
+from tenacity import retry, stop_after_attempt, wait_exponential_jitter, retry_if_exception
+
+
+def is_ftp_exception(e: BaseException) -> bool:
+    if isinstance(e, ftplib.all_errors):
+        error = str(e)[:400].replace('\'', '').replace('\"', '')
+        print(f"{error}, retrying")
+        return True
+    else:
+        return False
+
+
+class FTPS(BrynQ):
+    def __init__(self, label: Union[str, List] = None, debug=False):
+        super().__init__()
+        if label is not None:
+            credentials = self.get_system_credential(system='ftps', label=label)
+            self.host = credentials['host']
+            self.username = credentials['username']
+            self.password = credentials['password']
+        elif os.getenv("QLIK_HOST") is not None and os.getenv("QLIK_USER") is not None and os.getenv("QLIK_PASSWORD") is not None:
+            self.host = os.getenv("QLIK_HOST")
+            self.username = os.getenv("QLIK_USER")
+            self.password = os.getenv("QLIK_PASSWORD")
+        else:
+            raise ValueError("Set the environment variables QLIK_HOST, QLIK_USER and QLIK_PASSWORD or provide the label parameter for a credential from BrynQ")
+        self.debug = debug
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=60, max=900), retry=retry_if_exception(is_ftp_exception), reraise=True)
+    def upload_file(self, local_path, filename, remote_path, remove_after_upload=False):
+        """
+        Upload a file from the client to another client or server
+        :param local_path: the path where the upload file is located
+        :param filename: The file which should be uploaded
+        :param remote_path: The path on the destination client where the file should be saved
+        :param remove_after_upload: If true, remove the file after a succesfull upload
+        :return: a status if the upload is succesfull or not
+        """
+        with FTP(host=self.host, user=self.username, passwd=self.password) as ftp:
+            ftp.cwd(remote_path)
+            with open(local_path + filename, 'rb') as fp:
+                # This runs until upload is successful, then breaks
+                while True:
+                    try:
+                        ftp.storbinary("STOR " + filename, fp)
+                        # If the file should be removed after upload, remove it from the local path
+                        if remove_after_upload:
+                            os.remove(f'{local_path}{filename}')
+                    except ftplib.error_temp as e:
+                        # this catches 421 errors (socket timeout), sleeps 10 seconds and tries again. If any other exception is encountered, breaks.
+                        if str(e).split()[0] == '421':
+                            time.sleep(10)
+                            continue
+                        else:
+                            raise
+                    break
+            ftp.close()
+            return 'File is transferred'
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=60, max=900), retry=retry_if_exception(is_ftp_exception), reraise=True)
+    def upload_multiple_files(self, local_path, remote_path, remove_after_upload=False):
+        """
+        Upload all files in a directory from the client to another client or server
+        :param local_path: the path from where all the files should be uploaded
+        :param remote_path: The path on the destination client where the file should be saved
+        :param remove_after_upload: If true, remove the file after a succesfull upload
+        :return: a status if the upload is succesfull or not
+        """
+        ftp = FTP(host=self.host, user=self.username, passwd=self.password)
+        ftp.cwd(remote_path)
+        for filename in os.listdir(local_path):
+            file = local_path + filename
+            if self.debug:
+                print(f"Remote path: {remote_path}, local file: {file}")
+            if os.path.isfile(file):
+                with open(file, 'rb') as fp:
+                    # This runs until upload is successful, then breaks
+                    while True:
+                        try:
+                            ftp.storbinary("STOR " + filename, fp)
+                            # If the file should be removed after upload, remove it from the local path
+                            if remove_after_upload:
+                                os.remove(file)
+                        except ftplib.error_temp as e:
+                            # this catches 421 errors (socket timeout), sleeps 10 seconds and tries again. If any other exception is encountered, breaks.
+                            if str(e).split()[0] == '421':
+                                time.sleep(10)
+                                continue
+                            else:
+                                raise
+                        break
+        ftp.close()
+        return 'All files are transferred'
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=60, max=900), retry=retry_if_exception(is_ftp_exception), reraise=True)
+    def download_file(self, local_path, remote_path, filename, remove_after_download=False):
+        """
+        Returns a single file from a given remote path
+        :param local_path: the folder where the downloaded file should be stored
+        :param remote_path: the folder on the server where the file should be downloaded from
+        :param filename: the filename itself
+        :param remove_after_download: Should the file be removed on the server after the download or not
+        :return: a status
+        """
+        if self.debug:
+            print(f"Remote path: {remote_path}, local file path: {filename}")
+        with FTP(host=self.host, user=self.username, passwd=self.password) as ftp:
+            with open('{}/{}'.format(local_path, filename), 'wb') as fp:
+                res = ftp.retrbinary('RETR {}/{}'.format(remote_path, filename), fp.write)
+                if not res.startswith('226 Successfully transferred'):
+                    # Remove the created file on the local client if the download is failed
+                    if os.path.isfile(f'{local_path}/{filename}'):
+                        os.remove(f'{local_path}/{filename}')
+                else:
+                    if remove_after_download:
+                        ftp.delete(filename)
+
+                return res
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=60, max=900), retry=retry_if_exception(is_ftp_exception), reraise=True)
+    def make_dir(self, dir_name):
+        """
+        Create a directory on a remote machine
+        :param dir_name: give the path name which should be created
+        :return: the status if the creation is successfull or not
+        """
+        with FTP(host=self.host, user=self.username, passwd=self.password) as ftp:
+            status = ftp.mkd(dir_name)
+            return status
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=60, max=900), retry=retry_if_exception(is_ftp_exception), reraise=True)
+    def list_directories(self, remote_path=''):
+        """
+        Give a NoneType of directories and files in a given directory. This one is only for information. The Nonetype
+        can't be iterated or something like that
+        :param remote_path: give the folder where to start in
+        :return: a NoneType with folders and files
+        """
+        with FTP(host=self.host, user=self.username, passwd=self.password) as ftp:
+            ftp.cwd(remote_path)
+            if self.debug:
+                print(ftp.dir())
+            return ftp.dir()
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=60, max=900), retry=retry_if_exception(is_ftp_exception), reraise=True)
+    def make_dirs(self, filepath: str):
+        """
+        shadows os.makedirs but for ftp
+        :param filepath: filepath that you want to create
+        :return: nothing
+        """
+        filepath = filepath.split('/')
+        with FTP(host=self.host, user=self.username, passwd=self.password) as ftp:
+            ftp.cwd('/')
+            for subpath in filepath:
+                if subpath != '':
+                    file_list = []
+                    ftp.retrlines('NLST', file_list.append)
+                    if subpath in file_list:
+                        ftp.cwd(subpath)
+                    else:
+                        ftp.mkd(subpath)
+                        ftp.cwd(subpath)
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=60, max=900), retry=retry_if_exception(is_ftp_exception), reraise=True)
+    def list_files(self, remote_path=''):
+        """
+        Give a list with files in a certain folder
+        :param remote_path: give the folder where to look in
+        :return: a list with files
+        """
+        with FTP(host=self.host, user=self.username, passwd=self.password) as ftp:
+            ftp.cwd(remote_path)
+            if self.debug:
+                print(ftp.nlst())
+            return ftp.nlst()
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=60, max=900), retry=retry_if_exception(is_ftp_exception), reraise=True)
+    def remove_file(self, remote_filepath):
+        """
+        Remove a file on a remote location
+        :param remote_file: the full path of the file that needs to be removed
+        """
+        with FTP(host=self.host, user=self.username, passwd=self.password) as ftp:
+            ftp.delete(remote_filepath)
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=60, max=900), retry=retry_if_exception(is_ftp_exception), reraise=True)
+    def remove_directory(self, remote_directory, recursive=False):
+        """
+        Remove a file on a remote location
+        :param remote_directory: the directory you want to remove
+        :param recursive: if you want to remove the directory recursively (including all files and subdirectories)
+        """
+        ftp = FTP(host=self.host, user=self.username, passwd=self.password)
+        if recursive:
+            ftp.cwd(remote_directory)
+            files = ftp.nlst()  # List directory contents
+            for file_name in files:
+                if file_name in ('.', '..'):
+                    continue
+                try:
+                    ftp.delete(file_name)  # Delete file
+                except:
+                    self.remove_directory(f"{remote_directory}/{file_name}", recursive=True)  # Recursive call for subdirectories
+            ftp.cwd('..')  # Move back to the parent directory
+            ftp.rmd(remote_directory.split('/')[-1])  # Remove the empty directory
+        else:
+            ftp.rmd(remote_directory)
+
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=60, max=900), retry=retry_if_exception(is_ftp_exception), reraise=True)
+    def send_command(self, command):
+        """
+        Send a command to the ftp server
+        :param command: the command that needs to be send
+        :return: the response of the server
+        """
+        with FTP(host=self.host, user=self.username, passwd=self.password) as ftp:
+            response = ftp.sendcmd(command)
+            return response
+

brynq_sdk_ftp-2.0.4/brynq_sdk_ftp/sftp.py

@@ -0,0 +1,120 @@
+from brynq_sdk_brynq import BrynQ
+from io import StringIO
+from paramiko.client import SSHClient, AutoAddPolicy
+from paramiko import RSAKey
+from paramiko.sftp_attr import SFTPAttributes
+import pysftp
+from typing import Union, List
+from stat import S_ISREG
+import os
+
+
+class SFTP(BrynQ):
+    def __init__(self, label: Union[str, List], debug=False):
+        """
+        Init the SFTP class
+        :param label: The label of the connector
+        :param debug: If you want to see debug messages
+        """
+        super().__init__()
+        credentials = self.get_system_credential(system='sftp', label=label)
+        self.debug = debug
+        if self.debug:
+            print(credentials)
+        self.host = credentials['host']
+        self.port = 22 if credentials['port'] is None else credentials['port']
+        self.username = credentials['username']
+        self.password = credentials['password']
+        self.cnopts = pysftp.CnOpts()
+        self.cnopts.hostkeys = None
+        self.private_key_path = credentials.get('private_key_password', None)
+        self.private_key_passphrase = credentials.get('private_key_password', None)
+        self.private_key = RSAKey(file_obj=StringIO(credentials.get('private_key')), password=self.private_key_passphrase)
+        self.client = SSHClient()
+        self.client.set_missing_host_key_policy(AutoAddPolicy())
+
+
+
+
+
+    def upload_file(self, local_filepath, remote_filepath, confirm=True) -> SFTPAttributes:
+        """
+        Upload a single file to a remote location. If there is no Private key
+        :param local_filepath: The file and the full path on your local machine
+        :param remote_filepath: The path and filename on the remote location
+        :param confirm: If you want to confirm the upload
+        :return: status
+        """
+        self.client.connect(hostname=self.host, port=self.port, username=self.username, password=self.password, pkey=self.private_key, passphrase=self.private_key_passphrase)
+        sftp = self.client.open_sftp()
+        response = sftp.put(local_filepath, remote_filepath, confirm=confirm)
+        self.client.close()
+
+        return response
+
+    def list_dir(self, remote_filepath, get_folders: bool = False) -> List[str]:
+        """
+        Read the files and folders an a certain location
+        :param remote_filepath: The full path where you want to get the content from
+        :return: a list with files and folders in the given location
+        """
+        self.client.connect(hostname=self.host, port=self.port, username=self.username, pkey=self.private_key, password=self.password)
+        sftp = self.client.open_sftp()
+        sftp.chdir(remote_filepath)
+        list_files = sftp.listdir_attr()
+        list_files = [file.filename for file in list_files if S_ISREG(file.st_mode) or get_folders]
+        self.client.close()
+
+        return list_files
+
+    def download_file(self, remote_path, remote_file, local_path):
+        """
+        Download a single file
+        :param remote_path: the path where the remote file exists
+        :param remote_file: the remote file itself
+        :param local_path: the path where the file needs to be downloaded to
+        :return: a file object
+        """
+        self.client.connect(hostname=self.host, port=self.port, username=self.username, pkey=self.private_key, password=self.password)
+        sftp = self.client.open_sftp()
+        sftp.get(remotepath=f'{remote_path}{remote_file}', localpath=f'{local_path}/{remote_file}')
+        self.client.close()
+
+    def make_dir(self, remote_path, new_dir_name):
+        """
+        Create a new folder on a remote location
+        :param remote_path: The location where you want to create the new folder
+        :param new_dir_name: The name of the new folder
+        :return: a status if creating succeeded or not
+        """
+        self.client.connect(hostname=self.host, port=self.port, username=self.username, pkey=self.private_key, password=self.password)
+        sftp = self.client.open_sftp()
+        sftp.chdir(remote_path)
+        sftp.mkdir(new_dir_name)
+        self.client.close()
+
+    def remove_file(self, remote_file):
+        """
+        Remove a file on a remote location
+        :param remote_file: the full path of the file that needs to be removed
+        :return: a status if deleting succeeded or not
+        """
+        self.client.connect(hostname=self.host, port=self.port, username=self.username, pkey=self.private_key, password=self.password)
+        sftp = self.client.open_sftp()
+        sftp.remove(remote_file)
+        self.client.close()
+
+    def move_file(self, old_file_path: str, new_file_path: str):
+        """
+        Move or rename a file on a remote location
+        :param old_file_path: the full path of the file that needs to be moved or renamed
+        :param new_file_path: the full path of the new location of the file
+        :return:
+        """
+        self.client.connect(hostname=self.host, port=self.port, username=self.username, pkey=self.private_key, password=self.password)
+        sftp = self.client.open_sftp()
+        sftp.rename(oldpath=old_file_path, newpath=new_file_path)
+        self.client.close()
+
+    def rename_file(self, old_file_path: str, new_file_path: str):
+        self.move_file(old_file_path=old_file_path, new_file_path=new_file_path)

{brynq_sdk_ftp-2.0.1 → brynq_sdk_ftp-2.0.4}/brynq_sdk_ftp.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: brynq-sdk-ftp
-Version: 2.0.1
+Version: 2.0.4
 Summary: FTP wrapper from BrynQ
 Home-page: UNKNOWN
 Author: BrynQ

{brynq_sdk_ftp-2.0.1 → brynq_sdk_ftp-2.0.4}/brynq_sdk_ftp.egg-info/SOURCES.txt

@@ -1,4 +1,7 @@
 setup.py
+brynq_sdk_ftp/__init__.py
+brynq_sdk_ftp/ftps.py
+brynq_sdk_ftp/sftp.py
 brynq_sdk_ftp.egg-info/PKG-INFO
 brynq_sdk_ftp.egg-info/SOURCES.txt
 brynq_sdk_ftp.egg-info/dependency_links.txt

brynq_sdk_ftp-2.0.4/brynq_sdk_ftp.egg-info/top_level.txt

@@ -0,0 +1 @@
+brynq_sdk_ftp

{brynq_sdk_ftp-2.0.1 → brynq_sdk_ftp-2.0.4}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_namespace_packages
 
 setup(
     name='brynq_sdk_ftp',
-    version='2.0.1',
+    version='2.0.4',
     description='FTP wrapper from BrynQ',
     long_description='FTP wrapper from Brynq',
     author='BrynQ',

brynq_sdk_ftp-2.0.1/brynq_sdk_ftp.egg-info/top_level.txt

@@ -1 +0,0 @@
-