brynq-sdk-ftp 3.0.4__py3-none-any.whl

brynq_sdk_ftp/__init__.py

@@ -0,0 +1,2 @@
+from .ftps import FTPS
+from .sftp import SFTP

brynq_sdk_ftp/ftps.py

@@ -0,0 +1,236 @@
+from brynq_sdk_brynq import BrynQ
+import os
+import ftplib
+import time
+from ftplib import FTP
+from typing import Union, List, Optional, Literal
+from tenacity import retry, stop_after_attempt, wait_exponential_jitter, retry_if_exception
+
+
+def is_ftp_exception(e: BaseException) -> bool:
+    if isinstance(e, ftplib.all_errors):
+        error = str(e)[:400].replace('\'', '').replace('\"', '')
+        print(f"{error}, retrying")
+        return True
+    else:
+        return False
+
+
+class FTPS(BrynQ):
+    def __init__(self, system_type: Optional[Literal['source', 'target']] = None, debug=False):
+        super().__init__()
+        try:
+            credentials = self.interfaces.credentials.get(system="ftps", system_type=system_type)
+            credentials = credentials.get('data')
+            self.host = credentials['host']
+            self.username = credentials['username']
+            self.password = credentials['password']
+        except ValueError:
+            if debug:
+                print("No FTPS credentials found in the platform, falling back to environment variables")
+            self.host = None
+        # if no credentials are retrieved from the platform, fallback to Qlik FTP (can only be target, never source)
+        if self.host is None:
+            if system_type == 'target' and os.getenv("QLIK_HOST") is not None and os.getenv("QLIK_USER") is not None and os.getenv("QLIK_PASSWORD") is not None:
+                if debug:
+                    print("Connecting to Qlik FTPs server")
+                self.host = os.getenv("QLIK_HOST")
+                self.username = os.getenv("QLIK_USER")
+                self.password = os.getenv("QLIK_PASSWORD")
+            else:
+                raise ValueError("Set the environment variables QLIK_HOST, QLIK_USER and QLIK_PASSWORD and pass system_type=target to use the Qlik FTPS server, otherwise connect an FTPS authorization to your interface in BrynQ")
+        self.debug = debug
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=60, max=900), retry=retry_if_exception(is_ftp_exception), reraise=True)
+    def upload_file(self, local_path, filename, remote_path, remove_after_upload=False):
+        """
+        Upload a file from the client to another client or server
+        :param local_path: the path where the upload file is located
+        :param filename: The file which should be uploaded
+        :param remote_path: The path on the destination client where the file should be saved
+        :param remove_after_upload: If true, remove the file after a succesfull upload
+        :return: a status if the upload is succesfull or not
+        """
+        with FTP(host=self.host, user=self.username, passwd=self.password) as ftp:
+            ftp.cwd(remote_path)
+            with open(local_path + filename, 'rb') as fp:
+                # This runs until upload is successful, then breaks
+                while True:
+                    try:
+                        ftp.storbinary("STOR " + filename, fp)
+                        # If the file should be removed after upload, remove it from the local path
+                        if remove_after_upload:
+                            os.remove(f'{local_path}{filename}')
+                    except ftplib.error_temp as e:
+                        # this catches 421 errors (socket timeout), sleeps 10 seconds and tries again. If any other exception is encountered, breaks.
+                        if str(e).split()[0] == '421':
+                            time.sleep(10)
+                            continue
+                        else:
+                            raise
+                    break
+            ftp.close()
+            return 'File is transferred'
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=60, max=900), retry=retry_if_exception(is_ftp_exception), reraise=True)
+    def upload_multiple_files(self, local_path, remote_path, remove_after_upload=False):
+        """
+        Upload all files in a directory from the client to another client or server
+        :param local_path: the path from where all the files should be uploaded
+        :param remote_path: The path on the destination client where the file should be saved
+        :param remove_after_upload: If true, remove the file after a succesfull upload
+        :return: a status if the upload is succesfull or not
+        """
+        ftp = FTP(host=self.host, user=self.username, passwd=self.password)
+        ftp.cwd(remote_path)
+        for filename in os.listdir(local_path):
+            file = local_path + filename
+            if self.debug:
+                print(f"Remote path: {remote_path}, local file: {file}")
+            if os.path.isfile(file):
+                with open(file, 'rb') as fp:
+                    # This runs until upload is successful, then breaks
+                    while True:
+                        try:
+                            ftp.storbinary("STOR " + filename, fp)
+                            # If the file should be removed after upload, remove it from the local path
+                            if remove_after_upload:
+                                os.remove(file)
+                        except ftplib.error_temp as e:
+                            # this catches 421 errors (socket timeout), sleeps 10 seconds and tries again. If any other exception is encountered, breaks.
+                            if str(e).split()[0] == '421':
+                                time.sleep(10)
+                                continue
+                            else:
+                                raise
+                        break
+        ftp.close()
+        return 'All files are transferred'
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=60, max=900), retry=retry_if_exception(is_ftp_exception), reraise=True)
+    def download_file(self, local_path, remote_path, filename, remove_after_download=False):
+        """
+        Returns a single file from a given remote path
+        :param local_path: the folder where the downloaded file should be stored
+        :param remote_path: the folder on the server where the file should be downloaded from
+        :param filename: the filename itself
+        :param remove_after_download: Should the file be removed on the server after the download or not
+        :return: a status
+        """
+        if self.debug:
+            print(f"Remote path: {remote_path}, local file path: {filename}")
+        with FTP(host=self.host, user=self.username, passwd=self.password) as ftp:
+            with open('{}/{}'.format(local_path, filename), 'wb') as fp:
+                res = ftp.retrbinary('RETR {}/{}'.format(remote_path, filename), fp.write)
+                if not res.startswith('226 Successfully transferred'):
+                    # Remove the created file on the local client if the download is failed
+                    if os.path.isfile(f'{local_path}/{filename}'):
+                        os.remove(f'{local_path}/{filename}')
+                else:
+                    if remove_after_download:
+                        ftp.delete(filename)
+
+                return res
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=60, max=900), retry=retry_if_exception(is_ftp_exception), reraise=True)
+    def make_dir(self, dir_name):
+        """
+        Create a directory on a remote machine
+        :param dir_name: give the path name which should be created
+        :return: the status if the creation is successfull or not
+        """
+        with FTP(host=self.host, user=self.username, passwd=self.password) as ftp:
+            status = ftp.mkd(dir_name)
+            return status
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=60, max=900), retry=retry_if_exception(is_ftp_exception), reraise=True)
+    def list_directories(self, remote_path=''):
+        """
+        Give a NoneType of directories and files in a given directory. This one is only for information. The Nonetype
+        can't be iterated or something like that
+        :param remote_path: give the folder where to start in
+        :return: a NoneType with folders and files
+        """
+        with FTP(host=self.host, user=self.username, passwd=self.password) as ftp:
+            ftp.cwd(remote_path)
+            if self.debug:
+                print(ftp.dir())
+            return ftp.dir()
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=60, max=900), retry=retry_if_exception(is_ftp_exception), reraise=True)
+    def make_dirs(self, filepath: str):
+        """
+        shadows os.makedirs but for ftp
+        :param filepath: filepath that you want to create
+        :return: nothing
+        """
+        filepath = filepath.split('/')
+        with FTP(host=self.host, user=self.username, passwd=self.password) as ftp:
+            ftp.cwd('/')
+            for subpath in filepath:
+                if subpath != '':
+                    file_list = []
+                    ftp.retrlines('NLST', file_list.append)
+                    if subpath in file_list:
+                        ftp.cwd(subpath)
+                    else:
+                        ftp.mkd(subpath)
+                        ftp.cwd(subpath)
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=60, max=900), retry=retry_if_exception(is_ftp_exception), reraise=True)
+    def list_files(self, remote_path=''):
+        """
+        Give a list with files in a certain folder
+        :param remote_path: give the folder where to look in
+        :return: a list with files
+        """
+        with FTP(host=self.host, user=self.username, passwd=self.password) as ftp:
+            ftp.cwd(remote_path)
+            if self.debug:
+                print(ftp.nlst())
+            return ftp.nlst()
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=60, max=900), retry=retry_if_exception(is_ftp_exception), reraise=True)
+    def remove_file(self, remote_filepath):
+        """
+        Remove a file on a remote location
+        :param remote_file: the full path of the file that needs to be removed
+        """
+        with FTP(host=self.host, user=self.username, passwd=self.password) as ftp:
+            ftp.delete(remote_filepath)
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=60, max=900), retry=retry_if_exception(is_ftp_exception), reraise=True)
+    def remove_directory(self, remote_directory, recursive=False):
+        """
+        Remove a file on a remote location
+        :param remote_directory: the directory you want to remove
+        :param recursive: if you want to remove the directory recursively (including all files and subdirectories)
+        """
+        ftp = FTP(host=self.host, user=self.username, passwd=self.password)
+        if recursive:
+            ftp.cwd(remote_directory)
+            files = ftp.nlst()  # List directory contents
+            for file_name in files:
+                if file_name in ('.', '..'):
+                    continue
+                try:
+                    ftp.delete(file_name)  # Delete file
+                except:
+                    self.remove_directory(f"{remote_directory}/{file_name}", recursive=True)  # Recursive call for subdirectories
+            ftp.cwd('..')  # Move back to the parent directory
+            ftp.rmd(remote_directory.split('/')[-1])  # Remove the empty directory
+        else:
+            ftp.rmd(remote_directory)
+
+
+    @retry(stop=stop_after_attempt(5), wait=wait_exponential_jitter(initial=60, max=900), retry=retry_if_exception(is_ftp_exception), reraise=True)
+    def send_command(self, command):
+        """
+        Send a command to the ftp server
+        :param command: the command that needs to be send
+        :return: the response of the server
+        """
+        with FTP(host=self.host, user=self.username, passwd=self.password) as ftp:
+            response = ftp.sendcmd(command)
+            return response
+

brynq_sdk_ftp/sftp.py

@@ -0,0 +1,149 @@
+from brynq_sdk_brynq import BrynQ
+from io import StringIO
+from paramiko.client import SSHClient, AutoAddPolicy
+from paramiko import RSAKey
+from paramiko.sftp_attr import SFTPAttributes
+from typing import Union, List, Literal, Optional
+from stat import S_ISREG
+import os
+
+
+class SFTP(BrynQ):
+    def __init__(self, system_type: Optional[Literal['source', 'target']] = None, debug=False):
+        """
+        Init the SFTP class
+        :param label: The label of the connector
+        :param debug: If you want to see debug messages
+        """
+        super().__init__()
+        self.debug = debug
+
+        self.client = SSHClient()
+        self.client.set_missing_host_key_policy(AutoAddPolicy())
+
+        # Try to fetch credentials from BrynQ; if not present, attributes remain unset
+        try:
+            credentials = self.interfaces.credentials.get(system="sftp", system_type=system_type)
+            credentials = credentials.get('data')
+            if credentials:
+                if self.debug:
+                    print(credentials)
+                self.host = credentials['host']
+                self.port = 22 if credentials.get('port') is None else credentials.get('port')
+                self.username = credentials.get('username')
+                self.password = credentials.get('password')
+                self.private_key_path = credentials.get('private_key_password', None)
+                self.private_key_passphrase = credentials.get('private_key_password', None)
+                self.private_key = None
+                if credentials.get('private_key'):
+                    self.private_key = RSAKey(file_obj=StringIO(credentials.get('private_key')), password=self.private_key_passphrase)
+        except ValueError:
+            print("No credentials found for SFTP. If this was intended, use _set_credentials() to set the credentials to pass variables ['host', 'port', 'username', 'password', 'private_key', 'private_key_password']")
+
+    def _set_credentials(self, credentials: dict):
+        """
+        When a child class(ex:Meta4) needs to set the credentials, use this method.
+        Set SFTP connection credentials programmatically.
+
+        Expected keys in credentials dict:
+        - host (str)
+        - port (int, optional; defaults to 22)
+        - username (str)
+        - password (str, optional)
+        - private_key (str, optional; PEM string)
+        - private_key_password (str, optional; passphrase for private_key)
+        """
+        # Core connection parameters
+        self.host = credentials['host']
+        self.port = 22 if credentials.get('port') is None else credentials.get('port')
+        self.username = credentials.get('username')
+        self.password = credentials.get('password')
+
+        # Key options
+        self.private_key_path = credentials.get('private_key_path')
+        self.private_key_passphrase = credentials.get('private_key_password')
+        self.private_key = None
+        if credentials.get('private_key'):
+            self.private_key = RSAKey(file_obj=StringIO(credentials.get('private_key')), password=self.private_key_passphrase)
+
+    def upload_file(self, local_filepath, remote_filepath, confirm=True) -> SFTPAttributes:
+        """
+        Upload a single file to a remote location. If there is no Private key
+        :param local_filepath: The file and the full path on your local machine
+        :param remote_filepath: The path and filename on the remote location
+        :param confirm: If you want to confirm the upload
+        :return: status
+        """
+        self.client.connect(hostname=self.host, port=self.port, username=self.username, password=self.password, pkey=self.private_key, passphrase=self.private_key_passphrase)
+        sftp = self.client.open_sftp()
+        response = sftp.put(local_filepath, remote_filepath, confirm=confirm)
+        self.client.close()
+
+        return response
+
+    def list_dir(self, remote_filepath, get_folders: bool = False) -> List[str]:
+        """
+        Read the files and folders an a certain location
+        :param remote_filepath: The full path where you want to get the content from
+        :return: a list with files and folders in the given location
+        """
+        self.client.connect(hostname=self.host, port=self.port, username=self.username, pkey=self.private_key, password=self.password)
+        sftp = self.client.open_sftp()
+        sftp.chdir(remote_filepath)
+        list_files = sftp.listdir_attr()
+        list_files = [file.filename for file in list_files if S_ISREG(file.st_mode) or get_folders]
+        self.client.close()
+
+        return list_files
+
+    def download_file(self, remote_path, remote_file, local_path):
+        """
+        Download a single file
+        :param remote_path: the path where the remote file exists
+        :param remote_file: the remote file itself
+        :param local_path: the path where the file needs to be downloaded to
+        :return: a file object
+        """
+        self.client.connect(hostname=self.host, port=self.port, username=self.username, pkey=self.private_key, password=self.password)
+        sftp = self.client.open_sftp()
+        sftp.get(remotepath=f'{remote_path}{remote_file}', localpath=f'{local_path}/{remote_file}')
+        self.client.close()
+
+    def make_dir(self, remote_path, new_dir_name):
+        """
+        Create a new folder on a remote location
+        :param remote_path: The location where you want to create the new folder
+        :param new_dir_name: The name of the new folder
+        :return: a status if creating succeeded or not
+        """
+        self.client.connect(hostname=self.host, port=self.port, username=self.username, pkey=self.private_key, password=self.password)
+        sftp = self.client.open_sftp()
+        sftp.chdir(remote_path)
+        sftp.mkdir(new_dir_name)
+        self.client.close()
+
+    def remove_file(self, remote_file):
+        """
+        Remove a file on a remote location
+        :param remote_file: the full path of the file that needs to be removed
+        :return: a status if deleting succeeded or not
+        """
+        self.client.connect(hostname=self.host, port=self.port, username=self.username, pkey=self.private_key, password=self.password)
+        sftp = self.client.open_sftp()
+        sftp.remove(remote_file)
+        self.client.close()
+
+    def move_file(self, old_file_path: str, new_file_path: str):
+        """
+        Move or rename a file on a remote location
+        :param old_file_path: the full path of the file that needs to be moved or renamed
+        :param new_file_path: the full path of the new location of the file
+        :return:
+        """
+        self.client.connect(hostname=self.host, port=self.port, username=self.username, pkey=self.private_key, password=self.password)
+        sftp = self.client.open_sftp()
+        sftp.rename(oldpath=old_file_path, newpath=new_file_path)
+        self.client.close()
+
+    def rename_file(self, old_file_path: str, new_file_path: str):
+        self.move_file(old_file_path=old_file_path, new_file_path=new_file_path)

brynq_sdk_ftp-3.0.4.dist-info/METADATA

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: brynq_sdk_ftp
+Version: 3.0.4
+Summary: FTP wrapper from BrynQ
+Author: BrynQ
+Author-email: support@brynq.com
+License: BrynQ License
+Requires-Dist: brynq-sdk-brynq<5,>=4
+Requires-Dist: requests<=3,>=2
+Requires-Dist: paramiko<=4,>=2
+Requires-Dist: pysftp<1,>0.2
+Requires-Dist: tenacity<9,>=8
+Dynamic: author
+Dynamic: author-email
+Dynamic: description
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: summary
+
+FTP wrapper from Brynq

brynq_sdk_ftp-3.0.4.dist-info/RECORD

@@ -0,0 +1,7 @@
+brynq_sdk_ftp/__init__.py,sha256=iAKN9HUpIb5qfnatYo6uFinfv97Nf_Q6vNpIMOrhgSY,45
+brynq_sdk_ftp/ftps.py,sha256=uXYc2ny6H5ihWbqOgajys98YBSeFYy28btRyVD6RTE0,11954
+brynq_sdk_ftp/sftp.py,sha256=WIYdpJo4Vcsve-DDFIouMye_xo0ZimK9L3-ay5Aoi-o,7065
+brynq_sdk_ftp-3.0.4.dist-info/METADATA,sha256=VOTm__doP_SZ0Vw8eN8fW_OtmgeLFXz4UxasKPfVabM,456
+brynq_sdk_ftp-3.0.4.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+brynq_sdk_ftp-3.0.4.dist-info/top_level.txt,sha256=GuAKZGWkVuRr4MtdiQfEfxZzWE4hwHIFmXa_hwUqxJg,14
+brynq_sdk_ftp-3.0.4.dist-info/RECORD,,

brynq_sdk_ftp-3.0.4.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

brynq_sdk_ftp-3.0.4.dist-info/top_level.txt

@@ -0,0 +1 @@
+brynq_sdk_ftp