recurvedata-lib 0.1.487__py2.py3-none-any.whl

recurvedata/operators/transfer_operator/load_task_mysql.py

@@ -0,0 +1,157 @@
+try:
+    from recurvedata.pigeon.loader.csv_to_mysql import CSVToMySQLLoader
+except ImportError:
+    pass
+
+from recurvedata.core.translation import _l
+from recurvedata.operators.transfer_operator import const
+from recurvedata.operators.transfer_operator.task import LoadTask
+from recurvedata.operators.transfer_operator.utils import allowed_modes
+
+
+class MySQLLoadTask(LoadTask):
+    ds_name_fields = ("mysql_data_source_name",)
+    ds_types = ("mysql", "tidb")
+    worker_install_require = [
+        "pigeon[mysql]",
+    ]
+    default_dumper_handler_options = {
+        "null": r"\N",
+    }
+
+    def execute_impl(self, *args, **kwargs):
+        mysql_ds = self.must_get_connection_by_name(self.config["mysql_data_source_name"])
+        load_options = self.rendered_config.copy()
+        for k in ["mysql_data_source_name"]:
+            load_options.pop(k, None)
+        load_options.update(
+            {
+                "filename": self.filename,
+                "connector": mysql_ds.connector,
+                "delete_file": True,
+                "database": mysql_ds.database,
+            }
+        )
+        loader = CSVToMySQLLoader(**load_options)
+        return loader.execute()
+
+    @classmethod
+    def config_schema(cls):
+        # get_choices_by_type = cls.get_connection_names_by_type
+        # dss = get_choices_by_type(cls.ds_types)
+
+        schema = {
+            "type": "object",
+            "properties": {
+                "mysql_data_source_name": {
+                    "type": "string",
+                    "title": _l("MySQL Connection"),
+                    "ui:field": "ProjectConnectionSelectorField",
+                    "ui:options": {
+                        "supportTypes": cls.ds_types,
+                    },
+                    # 'default': cls.first_or_default(dss),
+                },
+                # "database": {
+                #     "type": "string",
+                #     "title": "Database",
+                #     "ui:field": "CodeEditorWithReferencesField",
+                #     "ui:options": {
+                #         "type": "plain",
+                #     },
+                # },
+                "table": {
+                    "type": "string",
+                    "title": _l("Target Table"),
+                    "description": _l("Name of the table to load data into"),
+                    "ui:field": "CodeEditorWithReferencesField",
+                    "ui:options": {
+                        "type": "plain",
+                    },
+                },
+                "create_table_ddl": {
+                    "type": "string",
+                    "title": _l("Table Creation SQL"),
+                    "description": _l("SQL statement to create the target table if it doesn't exist"),
+                    "ui:field": "CodeEditorWithReferencesField",
+                    "ui:options": {
+                        "type": "code",
+                        "lang": "sql",
+                        "sqlLang": "mysql",
+                    },
+                },
+                "mode": {
+                    "type": "string",
+                    "title": _l("Load Mode"),
+                    "description": _l("How to handle existing data in the target table"),
+                    "enum": list(allowed_modes),
+                    "default": const.LOAD_OVERWRITE,
+                },
+                "primary_keys": {
+                    "ui:hidden": '{{parentFormData.mode !== "MERGE"}}',
+                    "type": "string",
+                    "title": _l("Primary Keys"),
+                    "description": _l(
+                        "Comma-separated list of columns used for deduplication in MERGE mode. "
+                        "Should be primary or unique key columns."
+                    ),
+                    "ui:field": "CodeEditorWithReferencesField",
+                    "ui:options": {
+                        "type": "plain",
+                    },
+                },
+                "using_insert": {
+                    "type": "boolean",
+                    "title": _l("Use INSERT Statements"),
+                    "default": False,
+                    "description": _l(
+                        "If disabled (default), uses fast bulk loading via `LOAD DATA LOCAL INFILE`. "
+                        "If enabled, uses standard `INSERT` statements instead."
+                    ),
+                },
+                "insert_batch_size": {
+                    "ui:hidden": "{{!parentFormData.using_insert}}",
+                    "type": "number",
+                    "ui:options": {"controls": False},
+                    "title": _l("Batch Size"),
+                    "default": 1000,
+                    "minimum": 1,
+                    "maximum": 2000,
+                    "description": _l("Number of rows to insert in each batch"),
+                },
+                "insert_concurrency": {
+                    "ui:hidden": "{{!parentFormData.using_insert}}",
+                    "type": "number",
+                    "ui:options": {"controls": False},
+                    "title": _l("Concurrent Inserts"),
+                    "default": 1,
+                    "minimum": 1,
+                    "maximum": 10,
+                    "description": _l("Number of parallel insert operations"),
+                },
+                # "pre_queries": {
+                #     "type": "string",
+                #     "title": "Queries Ran Before Loading",
+                #     "description": '新数据导入前运行的 SQL，多条 SQL 用 `;` 分隔；支持传入变量，详见 <a target="_blank" href="http://bit.ly/2JMutjn">文档</a>',
+                #     "ui:field": "CodeEditorWithReferencesField",
+                #     "ui:options": {
+                #         "type": "code",
+                #         "lang": "sql",
+                #         "sqlLang": "mysql",
+                #     },
+                # },
+                # "post_queries": {
+                #     "type": "string",
+                #     "title": "Queries Ran After Loading",
+                #     "description": '新数据导入后运行的 SQL，多条 SQL 用 `;` 分隔；支持传入变量，详见 <a target="_blank" href="http://bit.ly/2JMutjn">文档</a>',
+                #     "ui:field": "CodeEditorWithReferencesField",
+                #     "ui:options": {
+                #         "type": "code",
+                #         "lang": "sql",
+                #         "sqlLang": "mysql",
+                #     },
+                # },
+            },
+            "required": ["mysql_data_source_name", "table"],
+        }
+        return schema

recurvedata/operators/transfer_operator/load_task_owncloud.py

@@ -0,0 +1,135 @@
+import os
+
+try:
+    import owncloud
+except ImportError:
+    pass
+import logging
+
+from recurvedata.core.translation import _l
+from recurvedata.operators.transfer_operator.task import LoadTask
+from recurvedata.pigeon.utils import fs, trim_suffix
+
+logger = logging.getLogger(__name__)
+
+
+class OwnCloudLoadTask(LoadTask):
+    ds_name_fields = ("data_source_name",)
+    ds_types = ("owncloud",)
+    should_write_header = True
+    worker_install_require = [
+        "pyocclient>=0.6",
+    ]
+
+    def execute_impl(self, *args, **kwargs):
+        conf = self.rendered_config
+        ds = self.must_get_connection_by_name(conf["data_source_name"])
+        file_to_upload = self.filename
+        remote_filename = conf["filename"]
+
+        cm = conf["compress_method"]
+        if cm != "None":
+            logger.info("compressing file using %s...", cm)
+            if cm == "Gzip":
+                ext = ".gz"
+                file_to_upload = fs.gzip_compress(self.filename, using_cmd=True)
+            elif cm == "Zip":
+                ext = ".zip"
+                arcname = trim_suffix(os.path.basename(remote_filename), ext)
+                file_to_upload = fs.zip_compress(self.filename, using_cmd=False, arcname=arcname)
+            else:
+                # won't reach here
+                raise ValueError(f"compress method {cm} is not supported")
+
+            if not remote_filename.endswith(ext):
+                remote_filename = f"{remote_filename}{ext}"
+
+        client = OwnCloudClient(ds.host, ds.user, ds.password)
+
+        remote_fold = conf["directory"]
+        remote_filename = os.path.join(conf["directory"], remote_filename)
+        logger.info("uploading %s to %s", file_to_upload, remote_filename)
+        client.upload(file_to_upload, remote_fold, remote_filename)
+        fs.remove_files_safely([self.filename, file_to_upload])
+        logger.info(f"remove {self.filename} local_file: {file_to_upload} success!")
+
+    @classmethod
+    def config_schema(cls):
+        schema = {
+            "type": "object",
+            "properties": {
+                "data_source_name": {
+                    "type": "string",
+                    "title": _l("OwnCloud Connection"),
+                    "ui:field": "ProjectConnectionSelectorField",
+                    "ui:options": {
+                        "supportTypes": cls.ds_types,
+                    },
+                    # 'default': cls.first_or_default(dss, ''),
+                },
+                "directory": {
+                    "type": "string",
+                    "title": _l("Target Directory"),
+                    "description": _l("Target directory to upload files to"),
+                    "ui:field": "CodeEditorWithReferencesField",
+                    "ui:options": {
+                        "type": "plain",
+                    },
+                },
+                "filename": {
+                    "type": "string",
+                    "title": _l("Filename"),
+                    "description": _l("Uploaded file name, supports template variables"),
+                    "ui:field": "CodeEditorWithReferencesField",
+                    "ui:options": {
+                        "type": "plain",
+                    },
+                },
+                "compress_method": {
+                    "type": "string",
+                    "title": _l("Compression Method"),
+                    "description": _l("Compress file before uploading using specified method"),
+                    "enum": ["None", "Gzip", "Zip"],
+                    "enumNames": ["None", "Gzip", "Zip"],
+                    "default": "None",
+                },
+            },
+            "required": ["data_source_name", "directory", "filename", "compress_method"],
+        }
+        return schema
+
+
+class OwnCloudClient(object):
+    # ownCloud没有提供API关闭客户端，只需要停止与ownCloud服务器交互，Python脚本就会自动关闭连接。
+    def __init__(self, host, username, password):
+        self.host = host
+        self.username = username
+        self.password = password
+        self.oc = self.login()
+
+    def login(self):
+        try:
+            owncloud_client = owncloud.Client(self.host)
+            owncloud_client.login(self.username, self.password)
+            logger.info("login owncloud success.")
+        except Exception as e:
+            logger.info("login owncloud failed. http error is: {}".format(e))
+        return owncloud_client
+
+    def create_owncloud_fold(self, fold):
+        try:
+            self.oc.file_info(fold)
+            logger.info("fold {} is exists.".format(fold))
+        except Exception as e:
+            logger.info("{} does not exists, start creating the folder. error is: {}".format(fold, e))
+            self.oc.mkdir(fold)
+            logger.info("create {} success.".format(fold))
+
+    def upload(self, local_filename, remote_fold, remote_filename):
+        self.create_owncloud_fold(remote_fold)
+        logger.info("start put {} to {}.".format(local_filename, remote_filename))
+        try:
+            self.oc.put_file(remote_path=remote_filename, local_source_file=local_filename)
+            logger.info("upload {} success.".format(remote_filename))
+        except Exception as e:
+            logger.info("upload {} failed. http error is: {}".format(remote_filename, e))

recurvedata/operators/transfer_operator/load_task_postgresql.py

@@ -0,0 +1,109 @@
+import copy
+
+try:
+    from recurvedata.pigeon.loader.csv_to_postgresql import CSVToPostgresqlLoader
+except ImportError:
+    pass
+
+from recurvedata.core.translation import _l
+from recurvedata.operators.transfer_operator import utils
+from recurvedata.operators.transfer_operator.task import LoadTask
+
+
+class PostgresqlLoadTask(LoadTask):
+    ds_name_fields = ("data_source_name",)
+    ds_types = ("postgres",)
+    worker_install_require = ["pigeon[postgres]"]
+
+    def execute_impl(self, *args, **kwargs):
+        ds = self.must_get_connection_by_name(self.config["data_source_name"])
+        load_options = self.rendered_config.copy()
+        for k in ["data_source_name"]:
+            load_options.pop(k, None)
+        load_options.setdefault("database", ds.database)
+        load_options.update(
+            {
+                "filename": self.filename,
+                "connector": ds.connector,
+                "delete_file": True,
+                "database": ds.database,
+                "schema": ds.data.get("schema"),
+            }
+        )
+        loader = CSVToPostgresqlLoader(**load_options)
+        return loader.execute()
+
+    @classmethod
+    def config_schema(cls):
+        # get_choices_by_type = cls.get_connection_names_by_type
+        # dss = get_choices_by_type(cls.ds_types)
+
+        schema = {
+            "type": "object",
+            "properties": {
+                "data_source_name": {
+                    "type": "string",
+                    "title": _l("PostgreSQL Connection"),
+                    "ui:field": "ProjectConnectionSelectorField",
+                    "ui:options": {
+                        "supportTypes": cls.ds_types,
+                    },
+                    # 'default': cls.first_or_default(dss),
+                },
+                # "database": {
+                #     "type": "string",
+                #     "title": "Database",
+                #     "ui:field": "CodeEditorWithReferencesField",
+                #     "ui:options": {
+                #         "type": "plain",
+                #     },
+                # },
+                "table": {
+                    "type": "string",
+                    "title": _l("Target Table"),
+                    "description": _l("Name of the table to load data into"),
+                    "ui:field": "CodeEditorWithReferencesField",
+                    "ui:options": {
+                        "type": "plain",
+                    },
+                },
+                "create_table_ddl": {
+                    "type": "string",
+                    "title": _l("Create Table DDL"),
+                    "description": _l("SQL statement to create the target table if it doesn't exist"),
+                    "ui:field": "CodeEditorWithReferencesField",
+                    "ui:options": {
+                        "type": "code",
+                        "lang": "sql",
+                        "sqlLang": "postgresql",
+                    },
+                },
+                # 'using_insert': {
+                #     'type': 'boolean',
+                #     'title': 'Using INSERT',
+                #     'default': False,
+                #     'description': '默认使用 `LOAD DATA LOCAL INFILE` 导入数据，也可以选择使用 INSERT 语句'
+                # },
+                "insert_batch_size": {
+                    "type": "number",
+                    "ui:options": {"controls": False},
+                    "title": _l("Batch Size"),
+                    "default": 1000,
+                    "minimum": 1,
+                    "maximum": 20000,
+                    "description": _l("Number of rows inserted in each batch"),
+                },
+                "insert_concurrency": {
+                    "type": "number",
+                    "ui:options": {"controls": False},
+                    "title": _l("Concurrent Inserts"),
+                    "default": 1,
+                    "minimum": 1,
+                    "maximum": 20,
+                    "description": _l("Number of parallel insert operations"),
+                },
+            },
+            "required": ["data_source_name", "table"],
+        }
+        schema["properties"].update(copy.deepcopy(utils.LOAD_COMMON))
+        return schema

recurvedata/operators/transfer_operator/load_task_qcloud_cos.py

@@ -0,0 +1,119 @@
+import logging
+import os
+
+try:
+    from recurvedata.pigeon.connector.qcloud_cos import COSConnector
+except ImportError:
+    pass
+
+from recurvedata.core.translation import _l
+from recurvedata.operators.transfer_operator.task import LoadTask
+from recurvedata.utils import extract_dict
+from recurvedata.utils.files import is_file_empty, remove_files_safely
+
+logger = logging.getLogger(__name__)
+
+
+class TencentCOSLoadTask(LoadTask):
+    ds_name_fields = ("data_source_name",)
+    ds_types = ("cos",)
+    should_write_header = True
+    worker_install_require = ["pigeon[qcloud]"]
+
+    def execute_impl(self, *args, **kwargs):
+        if is_file_empty(self.filename):
+            logger.warning("file %s not exists or has no content, skip.", self.filename)
+            return
+
+        ds = self.must_get_connection_by_name(self.config["data_source_name"])
+
+        load_options = self.rendered_config.copy()
+        ds_extra_bucket = ds.extra.get("bucket")
+        config_bucket = load_options.get("bucket_name")
+        bucket_upload = config_bucket if config_bucket else ds_extra_bucket
+
+        opt_keys = ["secret_id", "secret_key", "region", "endpoint", "proxies"]
+        cos = COSConnector(**extract_dict(ds.extra, opt_keys))
+
+        compress_mode = load_options["compress_mode"]
+        if compress_mode != "None" and not load_options["key"].endswith(("/",)):
+            target_filename = os.path.join(os.path.dirname(self.filename), os.path.basename(load_options["key"]))
+        else:
+            target_filename = None
+        file_upload, ext = self.compress_file(
+            filename=self.filename, target_filename=target_filename, compress_mode=compress_mode
+        )
+        if compress_mode != "None" and not load_options["key"].endswith(("/", ext)):
+            load_options["key"] = f"{load_options['key']}{ext}"
+
+        upload_conf = {
+            "bucket_name": bucket_upload,
+            "filename": file_upload,
+            "overwrite": load_options["overwrite"],
+        }
+        if load_options["key"].endswith("/"):
+            upload_conf.update({"folder": load_options["key"]})
+        elif load_options["key"]:
+            upload_conf.update({"key": load_options["key"]})
+        else:
+            upload_conf.update({"key": os.path.basename(file_upload)})
+
+        logger.info("uploading...")
+        cos.upload(**upload_conf)
+        return remove_files_safely([self.filename, file_upload])
+
+    @classmethod
+    def config_schema(cls):
+        # get_choices_by_type = cls.get_connection_names_by_type
+        # dss = get_choices_by_type(cls.ds_types)
+        schema = {
+            "type": "object",
+            "properties": {
+                "data_source_name": {
+                    "type": "string",
+                    "title": _l("Tencent COS Connection"),
+                    "ui:field": "ProjectConnectionSelectorField",
+                    "ui:options": {
+                        "supportTypes": cls.ds_types,
+                    },
+                    # 'default': cls.first_or_default(dss, ''),
+                },
+                "bucket_name": {
+                    "type": "string",
+                    "title": _l("Bucket"),
+                    "description": _l("Bucket name, required if not set in data source"),
+                    "ui:field": "CodeEditorWithReferencesField",
+                    "ui:options": {
+                        "type": "plain",
+                    },
+                },
+                "key": {
+                    "type": "string",
+                    "title": _l("Upload Path"),
+                    "description": _l(
+                        "Target path in the bucket. Can be an object key or folder path (ending with /). "
+                        "Supports Jinja templating."
+                    ),
+                    "ui:field": "CodeEditorWithReferencesField",
+                    "ui:options": {
+                        "type": "plain",
+                    },
+                },
+                "compress_mode": {
+                    "type": "string",
+                    "title": _l("Compression Method"),
+                    "description": _l("Compress file before uploading using specified method"),
+                    "enum": ["None", "Gzip", "Zip"],
+                    "enumNames": ["None", "Gzip", "Zip"],
+                    "default": "None",
+                },
+                "overwrite": {
+                    "type": "boolean",
+                    "title": _l("Overwrite Existing"),
+                    "description": _l("Whether to overwrite if target object already exists"),
+                    "default": True,
+                },
+            },
+            "required": ["compress_mode", "data_source_name"],
+        }
+        return schema

recurvedata/operators/transfer_operator/load_task_recurve_data_prep.py

@@ -0,0 +1,75 @@
+import logging
+import os
+
+from recurvedata.operators.context import context
+
+try:
+    from recurvedata.pigeon.utils import fs
+except ImportError:
+    pass
+
+from recurvedata.operators.transfer_operator.task import LoadTask
+
+logger = logging.getLogger(__name__)
+
+
+class DataPrepLoadTask(LoadTask):
+    should_write_header = True
+    enabled = False
+
+    def _rename_filename_to_csv(self):
+        filename2 = self.filename + ".csv"
+        if not os.path.exists(filename2):
+            if os.path.exists(self.filename):
+                os.rename(self.filename, filename2)
+        self.filename = filename2
+
+    def execute_impl(self, *args, **kwargs):
+        self._rename_filename_to_csv()
+        if fs.is_file_empty(self.filename):
+            logger.warning("file %s not exists or has no content, skip.", self.filename)
+            return
+
+        conf = self.rendered_config
+        context.client.upload_file_to_data_preparation(
+            project_id=conf.get("project_id") and int(conf["project_id"]),
+            file_name=self.filename,
+            table_id=conf.get("table_id") and int(conf["table_id"]),
+            import_strategy=conf.get("import_strategy"),
+            keep_user_modified_data=conf.get("keep_user_modified_data"),
+            publish=conf.get("publish"),
+        )
+
+    @classmethod
+    def config_schema(cls):
+        schema = {
+            "type": "object",
+            "properties": {
+                "project_id": {"type": "string", "title": "Project ID"},
+                "table_id": {
+                    "type": "string",
+                    "title": "Table ID",
+                },
+                "keep_user_modified_data": {
+                    "type": "boolean",
+                    "title": "Keep User Modified Data",
+                    "default": False,
+                    "description": "是否保留用户修改的数据",
+                },
+                "import_strategy": {
+                    "type": "string",
+                    "title": "Import Strategy",
+                    "description": "导入策略",
+                    "enum": ["overwrite", "replace", "merge"],
+                    "enumNames": ["overwrite", "replace", "merge"],
+                },
+                "publish": {
+                    "type": "boolean",
+                    "title": "Publish",
+                    "default": False,
+                    "description": "是否发布",
+                },
+            },
+            "required": ["project_id", "table_id"],
+        }
+        return schema