mkpipe-loader-sqlserver 0.1.1__tar.gz → 0.4.0__tar.gz

mkpipe_loader_sqlserver-0.4.0/MANIFEST.in

@@ -0,0 +1 @@
+include mkpipe_extractor_postgres/jars/*

mkpipe_loader_sqlserver-0.4.0/PKG-INFO

@@ -0,0 +1,17 @@
+Metadata-Version: 2.4
+Name: mkpipe-loader-sqlserver
+Version: 0.4.0
+Summary: SQL Server loader for mkpipe.
+Author: Metin Karakus
+Author-email: metin_karakus@yahoo.com
+License: Apache License 2.0
+Requires-Python: >=3.9
+License-File: LICENSE
+Requires-Dist: mkpipe
+Dynamic: author
+Dynamic: author-email
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

mkpipe_loader_sqlserver-0.4.0/README.md

@@ -0,0 +1,3 @@
+# mkpipe-loader-sqlserver
+
+SQL Server loader plugin for mkpipe.

mkpipe_loader_sqlserver-0.4.0/mkpipe_loader_sqlserver/__init__.py

@@ -0,0 +1,18 @@
+from urllib.parse import unquote
+
+from mkpipe.spark import JdbcLoader
+
+
+class SqlserverLoader(JdbcLoader, variant='sqlserver'):
+    driver_name = 'sqlserver'
+    driver_jdbc = 'com.microsoft.sqlserver.jdbc.SQLServerDriver'
+
+    def build_jdbc_url(self):
+        password = unquote(self.password)
+        return (
+            f'jdbc:{self.driver_name}://{self.host}:{self.port}'
+            f';databaseName={self.database}'
+            f';user={self.username}'
+            f';password={password}'
+            f';encrypt=false;trustServerCertificate=false'
+        )

mkpipe_loader_sqlserver-0.4.0/mkpipe_loader_sqlserver.egg-info/PKG-INFO

@@ -0,0 +1,17 @@
+Metadata-Version: 2.4
+Name: mkpipe-loader-sqlserver
+Version: 0.4.0
+Summary: SQL Server loader for mkpipe.
+Author: Metin Karakus
+Author-email: metin_karakus@yahoo.com
+License: Apache License 2.0
+Requires-Python: >=3.9
+License-File: LICENSE
+Requires-Dist: mkpipe
+Dynamic: author
+Dynamic: author-email
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

{mkpipe_loader_sqlserver-0.1.1 → mkpipe_loader_sqlserver-0.4.0}/mkpipe_loader_sqlserver.egg-info/SOURCES.txt

@@ -8,5 +8,4 @@ mkpipe_loader_sqlserver.egg-info/SOURCES.txt
 mkpipe_loader_sqlserver.egg-info/dependency_links.txt
 mkpipe_loader_sqlserver.egg-info/entry_points.txt
 mkpipe_loader_sqlserver.egg-info/requires.txt
-mkpipe_loader_sqlserver.egg-info/top_level.txt
-mkpipe_loader_sqlserver/jars/com.microsoft.sqlserver_mssql-jdbc-12.8.1.jre11.jar
+mkpipe_loader_sqlserver.egg-info/top_level.txt

mkpipe_loader_sqlserver-0.4.0/setup.py

@@ -0,0 +1,19 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='mkpipe-loader-sqlserver',
+    version='0.4.0',
+    license='Apache License 2.0',
+    packages=find_packages(),
+    install_requires=['mkpipe'],
+    include_package_data=True,
+    entry_points={
+        'mkpipe.loaders': [
+            'sqlserver = mkpipe_loader_sqlserver:SqlserverLoader',
+        ],
+    },
+    description='SQL Server loader for mkpipe.',
+    author='Metin Karakus',
+    author_email='metin_karakus@yahoo.com',
+    python_requires='>=3.9',
+)

mkpipe_loader_sqlserver-0.1.1/MANIFEST.in

@@ -1 +0,0 @@
-include mkpipe_loader_sqlserver/jars/*

mkpipe_loader_sqlserver-0.1.1/PKG-INFO

@@ -1,39 +0,0 @@
-Metadata-Version: 2.1
-Name: mkpipe-loader-sqlserver
-Version: 0.1.1
-Summary: SQLServer loader for mkpipe.
-Author: Metin Karakus
-Author-email: metin_karakus@yahoo.com
-License: Apache License 2.0
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: Apache Software License
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: mkpipe
-
-# MkPipe
-
-**MkPipe** is a modular, open-source ETL (Extract, Transform, Load) tool that allows you to integrate various data sources and sinks easily. It is designed to be extensible with a plugin-based architecture that supports extractors, transformers, and loaders.
-
-## Documentation
-
-For more detailed documentation, please visit the [GitHub repository](https://github.com/mkpipe-etl/mkpipe).
-
-## License
-
-This project is licensed under the Apache 2.0 License - see the [LICENSE](LICENSE) file for details.
-
-
-## mkpipe_project.yaml Variables
-```yaml
-...
-  connections:
-    source:
-      host: 'XXX'
-      port: 'XXX'
-      database: 'XXX'
-      user: 'XXX'
-      password: 'XXX'
-...
-```

mkpipe_loader_sqlserver-0.1.1/README.md

@@ -1,25 +0,0 @@
-# MkPipe
-
-**MkPipe** is a modular, open-source ETL (Extract, Transform, Load) tool that allows you to integrate various data sources and sinks easily. It is designed to be extensible with a plugin-based architecture that supports extractors, transformers, and loaders.
-
-## Documentation
-
-For more detailed documentation, please visit the [GitHub repository](https://github.com/mkpipe-etl/mkpipe).
-
-## License
-
-This project is licensed under the Apache 2.0 License - see the [LICENSE](LICENSE) file for details.
-
-
-## mkpipe_project.yaml Variables
-```yaml
-...
-  connections:
-    source:
-      host: 'XXX'
-      port: 'XXX'
-      database: 'XXX'
-      user: 'XXX'
-      password: 'XXX'
-...
-```

mkpipe_loader_sqlserver-0.1.1/mkpipe_loader_sqlserver/__init__.py

@@ -1,146 +0,0 @@
-import time
-from pathlib import Path
-from urllib.parse import quote_plus
-from pyspark.sql import functions as F
-from pyspark.sql.types import TimestampType
-from mkpipe.config import load_config
-from mkpipe.functions_db import get_db_connector
-from mkpipe.functions_spark import remove_partitioned_parquet, get_parser
-from mkpipe.utils import log_container, Logger
-from mkpipe.utils.base_class import PipeSettings
-
-
-class SqlserverLoader:
-    def __init__(self, config, settings):
-        if isinstance(settings, dict):
-            self.settings = PipeSettings(**settings)
-        else:
-            self.settings = settings
-        self.connection_params = config['connection_params']
-        self.host = self.connection_params['host']
-        self.port = self.connection_params['port']
-        self.username = self.connection_params['user']
-        # self.password = quote_plus(str(self.connection_params['password']))
-        self.password = str(self.connection_params['password'])
-        self.database = self.connection_params['database']
-
-        self.driver_name = 'sqlserver'
-        self.driver_jdbc = 'com.microsoft.sqlserver.jdbc.SQLServerDriver'
-        self.settings.driver_name = self.driver_name
-        self.jdbc_url = f'jdbc:{self.driver_name}://{self.host}:{self.port};databaseName={self.database};user={self.username};password={self.password};encrypt=false;trustServerCertificate=false'
-
-        config = load_config()
-        connection_params = config['settings']['backend']
-        db_type = connection_params['database_type']
-        self.backend = get_db_connector(db_type)(connection_params)
-
-    def add_custom_columns(self, df, elt_start_time):
-        if 'etl_time' in df.columns:
-            df = df.drop('etl_time')
-
-        df = df.withColumn('etl_time', F.lit(elt_start_time).cast(TimestampType()))
-        return df
-
-    @log_container(__file__)
-    def load(self, data, elt_start_time):
-        try:
-            logger = Logger(__file__)
-            start_time = time.time()
-            name = data['table_name']
-
-            write_mode = data.get('write_mode', None)
-            file_type = data.get('file_type', None)
-            last_point_value = data.get('last_point_value', None)
-            iterate_column_type = data.get('iterate_column_type', None)
-            replication_method = data.get('replication_method', 'full')
-            batchsize = data.get('fetchsize', 100_000)
-            pass_on_error = data.get('pass_on_error', None)
-
-            if not file_type:
-                'means that the data fetched before no new data'
-                self.backend.manifest_table_update(
-                    name=name,
-                    value=None,  # Last point remains unchanged
-                    value_type=None,  # Type remains unchanged
-                    status='completed',  # ('completed', 'failed', 'extracting', 'loading')
-                    replication_method=replication_method,  # ('incremental', 'full')
-                    error_message='',
-                )
-                return
-
-            self.backend.manifest_table_update(
-                name=name,
-                value=None,  # Last point remains unchanged
-                value_type=None,  # Type remains unchanged
-                status='loading',  # ('completed', 'failed', 'extracting', 'loading')
-                replication_method=replication_method,  # ('incremental', 'full')
-                error_message='',
-            )
-
-            df = get_parser(file_type)(data, self.settings)
-            df = self.add_custom_columns(df, elt_start_time)
-            message = dict(
-                table_name=name,
-                status='loading',
-                total_partition_count=df.rdd.getNumPartitions(),
-            )
-            logger.info(message)
-
-            (
-                df.write.format('jdbc')
-                .mode(
-                    write_mode
-                )  # Use write_mode for the first iteration, 'append' for others
-                .option('url', self.jdbc_url)
-                .option('dbtable', name)
-                .option('driver', self.driver_jdbc)
-                .option('batchsize', batchsize)
-                .save()
-            )
-
-            # Update last point in the mkpipe_manifest table if applicable
-            self.backend.manifest_table_update(
-                name=name,
-                value=last_point_value,
-                value_type=iterate_column_type,
-                status='completed',
-                replication_method=replication_method,
-                error_message='',
-            )
-
-            message = dict(table_name=name, status=write_mode)
-            logger.info(message)
-
-            # remove the parquet to reduce the storage
-            remove_partitioned_parquet(data['path'])
-
-            run_time = time.time() - start_time
-            message = dict(table_name=name, status='success', run_time=run_time)
-            logger.info(message)
-
-        except Exception as e:
-            # Log the error message and update the mkpipe_manifest with the error details
-            message = dict(
-                table_name=name,
-                status='failed',
-                type='loading',
-                error_message=str(e),
-                etl_start_time=str(elt_start_time),
-            )
-
-            self.backend.manifest_table_update(
-                name=name,
-                value=None,  # Last point remains unchanged
-                value_type=None,  # Type remains unchanged
-                status='failed',
-                replication_method=replication_method,
-                error_message=str(e),
-            )
-
-            if pass_on_error:
-                logger.warning(message)
-                return
-            else:
-                logger.error(message)
-                raise Exception(message) from e
-        return

mkpipe_loader_sqlserver-0.1.1/mkpipe_loader_sqlserver.egg-info/PKG-INFO

@@ -1,39 +0,0 @@
-Metadata-Version: 2.1
-Name: mkpipe-loader-sqlserver
-Version: 0.1.1
-Summary: SQLServer loader for mkpipe.
-Author: Metin Karakus
-Author-email: metin_karakus@yahoo.com
-License: Apache License 2.0
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: Apache Software License
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: mkpipe
-
-# MkPipe
-
-**MkPipe** is a modular, open-source ETL (Extract, Transform, Load) tool that allows you to integrate various data sources and sinks easily. It is designed to be extensible with a plugin-based architecture that supports extractors, transformers, and loaders.
-
-## Documentation
-
-For more detailed documentation, please visit the [GitHub repository](https://github.com/mkpipe-etl/mkpipe).
-
-## License
-
-This project is licensed under the Apache 2.0 License - see the [LICENSE](LICENSE) file for details.
-
-
-## mkpipe_project.yaml Variables
-```yaml
-...
-  connections:
-    source:
-      host: 'XXX'
-      port: 'XXX'
-      database: 'XXX'
-      user: 'XXX'
-      password: 'XXX'
-...
-```

mkpipe_loader_sqlserver-0.1.1/setup.py

@@ -1,25 +0,0 @@
-from setuptools import setup, find_packages
-
-setup(
-    name='mkpipe-loader-sqlserver',
-    version='0.1.1',
-    license='Apache License 2.0',
-    packages=find_packages(exclude=['tests', 'scripts', 'deploy', 'install_jars.py']),
-    install_requires=['mkpipe'],
-    include_package_data=True,
-    entry_points={
-        'mkpipe.loaders': [
-            'sqlserver = mkpipe_loader_sqlserver:SqlserverLoader',
-        ],
-    },
-    description='SQLServer loader for mkpipe.',
-    long_description=open('README.md').read(),
-    long_description_content_type='text/markdown',
-    author='Metin Karakus',
-    author_email='metin_karakus@yahoo.com',
-    classifiers=[
-        'Programming Language :: Python :: 3',
-        'License :: OSI Approved :: Apache Software License',
-    ],
-    python_requires='>=3.8',
-)