mkpipe-extractor-sqlserver 0.1.1__tar.gz → 0.1.3__tar.gz

{mkpipe_extractor_sqlserver-0.1.1/mkpipe_extractor_sqlserver.egg-info → mkpipe_extractor_sqlserver-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: mkpipe-extractor-sqlserver
-Version: 0.1.1
+Version: 0.1.3
 Summary: SQLserver extractor for mkpipe.
 Author: Metin Karakus
 Author-email: metin_karakus@yahoo.com
@@ -11,6 +11,16 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: mkpipe
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # MkPipe
 

{mkpipe_extractor_sqlserver-0.1.1 → mkpipe_extractor_sqlserver-0.1.3}/mkpipe_extractor_sqlserver/__init__.py

@@ -1,6 +1,5 @@
 import os
 import datetime
-from pathlib import Path
 from urllib.parse import quote_plus
 from pyspark.sql import SparkSession
 from pyspark import SparkConf
@@ -20,6 +19,8 @@ class SqlserverExtractor:
         else:
             self.settings = settings
         self.connection_params = config['connection_params']
+        self.table = config['table']
+        self.pass_on_error = config.get('pass_on_error', None)
         self.host = self.connection_params['host']
         self.port = self.connection_params['port']
         self.username = self.connection_params['user']
@@ -32,9 +33,6 @@ class SqlserverExtractor:
         self.settings.driver_name = self.driver_name
         self.jdbc_url = f'jdbc:{self.driver_name}://{self.host}:{self.port};databaseName={self.database};user={self.username};password={self.password};encrypt=false;trustServerCertificate=false'
 
-        self.table = config['table']
-        self.pass_on_error = config.get('pass_on_error', None)
-        
         config = load_config()
         connection_params = config['settings']['backend']
         db_type = connection_params['database_type']
@@ -78,8 +76,9 @@ class SqlserverExtractor:
             name = t['name']
             target_name = t['target_name']
             iterate_column_type = t['iterate_column_type']
-            iterate_batch_size = t.get(
-                'iterate_batch_size', self.settings.default_iterate_batch_size
+            chunk_count_for_partition = t.get(
+                'chunk_count_for_partition',
+                self.settings.default_chunk_count_for_partition,
             )
             iterate_max_loop = t.get(
                 'iterate_max_loop', self.settings.default_iterate_max_loop
@@ -101,7 +100,6 @@ class SqlserverExtractor:
 
             partitions_column = partitions_column_.split(' as ')[0].strip()
             p_col_name = partitions_column_.split(' as ')[-1].strip()
-            p_col_select = f'{partitions_column} as {p_col_name}'
 
             message = dict(table_name=target_name, status='extracting')
             logger.info(message)
@@ -112,57 +110,65 @@ class SqlserverExtractor:
             last_point = self.backend.get_last_point(target_name)
             if last_point:
                 write_mode = 'append'
-
-                iterate_query = f"""(SELECT {p_col_select} from {name} where {partitions_column} > '{last_point}' ) q"""
-
-                df_itarate_list = (
-                    spark.read.format('jdbc')
-                    .option('url', self.jdbc_url)
-                    .option('dbtable', iterate_query)
-                    .option('driver', self.driver_jdbc)
-                    .option('fetchsize', fetchsize)
-                    .load()
-                )
-
-                min_val = last_point
-                max_val = df_itarate_list.agg(F.max(p_col_name).alias('max')).collect()[
-                    0
-                ][0]
-                df_itarate_list = df_itarate_list.where(F.col(p_col_name) > min_val)
+                iterate_query = f"""(SELECT min({partitions_column}) as min_val, max({partitions_column}) as max_val from {name} where {partitions_column} > '{last_point}' ) q"""
             else:
                 write_mode = 'overwrite'
-
-                iterate_query = f'(SELECT {p_col_select} from {name}) q'
-                df_itarate_list = (
-                    spark.read.format('jdbc')
-                    .option('url', self.jdbc_url)
-                    .option('dbtable', iterate_query)
-                    .option('driver', self.driver_jdbc)
-                    .option('fetchsize', fetchsize)
-                    .load()
-                )
-
-                min_max_vals = df_itarate_list.agg(
-                    F.min(p_col_name).alias('min'), F.max(p_col_name).alias('max')
-                ).collect()[0]
-                min_val = min_max_vals[0]
-                max_val = min_max_vals[1]
-                df_itarate_list = df_itarate_list.where(F.col(p_col_name) >= min_val)
-
-            key_list = (
-                df_itarate_list.select(p_col_name)
-                .distinct()
-                .rdd.flatMap(lambda x: x)
-                .collect()
+                iterate_query = f"""(SELECT min({partitions_column}) as min_val, max({partitions_column}) as max_val from {name}) q"""
+
+            df_itarate_list = (
+                spark.read.format('jdbc')
+                .option('url', self.jdbc_url)
+                .option('dbtable', iterate_query)
+                .option('driver', self.driver_jdbc)
+                .option('fetchsize', fetchsize)
+                .load()
             )
-            key_list.sort()
-
-            chunks = [
-                key_list[x : x + iterate_batch_size]
-                for x in range(0, len(key_list), iterate_batch_size)
-            ]
-
-            min_max_tuple = [(min(x), max(x)) for x in chunks]
+            min_val = df_itarate_list.first()['min_val']
+            max_val = df_itarate_list.first()['max_val']
+
+            if min_val is None or max_val is None:
+                min_max_tuple = None
+            elif iterate_column_type == 'int':
+                min_val = int(min_val)
+                max_val = int(max_val)
+
+                total_range = max_val - min_val + 1  # inclusive
+                step = total_range // chunk_count_for_partition
+                remainder = total_range % chunk_count_for_partition
+
+                min_max_tuple = []
+                start = min_val
+                for _ in range(chunk_count_for_partition):
+                    end = start + step - 1
+                    if remainder > 0:
+                        end += 1
+                        remainder -= 1
+                    if start <= max_val:
+                        min_max_tuple.append((start, min(end, max_val)))
+                        start = end + 1
+
+            elif iterate_column_type == 'datetime':
+                total_seconds = (
+                    int((max_val - min_val).total_seconds()) + 1
+                )  # include max_val
+                step = total_seconds // chunk_count_for_partition
+                remainder = total_seconds % chunk_count_for_partition
+
+                min_max_tuple = []
+                start = min_val
+                for _ in range(chunk_count_for_partition):
+                    step_with_remainder = step
+                    if remainder > 0:
+                        step_with_remainder += 1
+                        remainder -= 1
+                    end = start + datetime.timedelta(seconds=step_with_remainder - 1)
+                    if start <= max_val:
+                        min_max_tuple.append((start, min(end, max_val)))
+                        start = end + datetime.timedelta(seconds=1)
+            else:
+                raise ValueError(
+                    f'Unsupported iterate_column_type: {iterate_column_type}'
+                )
 
             if not min_max_tuple:
                 if not last_point:
@@ -209,20 +215,24 @@ class SqlserverExtractor:
                     if custom_query:
                         updated_query = custom_query.replace(
                             '{query_filter}',
-                            f""" where {partitions_column} between {min_filter} and {max_filter} """,
+                            f""" where {partitions_column} >= {min_filter} and {partitions_column} < {max_filter} """,
                         )
                     else:
-                        updated_query = f'(SELECT * from {name} where  {partitions_column} between {min_filter} and {max_filter}) q'
-                else:
-                    min_filter = str(chunk[0])
-                    max_filter = str(chunk[-1])
+                        updated_query = f'(SELECT * from {name} where {partitions_column} >= {min_filter} and {partitions_column} < {max_filter}) q'
+                elif iterate_column_type == 'datetime':
+                    min_filter = chunk[0].strftime('%Y-%m-%d %H:%M:%S')
+                    max_filter = chunk[-1].strftime('%Y-%m-%d %H:%M:%S')
                     if custom_query:
                         updated_query = custom_query.replace(
                             '{query_filter}',
-                            f""" where {partitions_column} between '{min_filter}' and '{max_filter}' """,
+                            f""" where {partitions_column} >= '{min_filter}' and {partitions_column} < '{max_filter}' """,
                         )
                     else:
-                        updated_query = f"""(SELECT * from {name} where  {partitions_column} between '{min_filter}' and '{max_filter}') q"""
+                        updated_query = f"""(SELECT * from {name} where  {partitions_column} >= '{min_filter}' and {partitions_column} < '{max_filter}') q"""
+                else:
+                    raise ValueError(
+                        f'Unsupported iterate_column_type: {iterate_column_type}'
+                    )
 
                 df = (
                     spark.read.format('jdbc')
@@ -244,6 +254,7 @@ class SqlserverExtractor:
                     .mode(p_write_mode)
                     .parquet(parquet_path)
                 )
+
                 count_col = len(df.columns)
                 count_row = df.count()
                 last_point_value = max_filter

{mkpipe_extractor_sqlserver-0.1.1 → mkpipe_extractor_sqlserver-0.1.3/mkpipe_extractor_sqlserver.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: mkpipe-extractor-sqlserver
-Version: 0.1.1
+Version: 0.1.3
 Summary: SQLserver extractor for mkpipe.
 Author: Metin Karakus
 Author-email: metin_karakus@yahoo.com
@@ -11,6 +11,16 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: mkpipe
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # MkPipe
 

{mkpipe_extractor_sqlserver-0.1.1 → mkpipe_extractor_sqlserver-0.1.3}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='mkpipe-extractor-sqlserver',
-    version='0.1.1',
+    version='0.1.3',
     license='Apache License 2.0',
     packages=find_packages(exclude=['tests', 'scripts', 'deploy', 'install_jars.py']),
     install_requires=['mkpipe'],