mgcomtools 0.1.2__tar.gz → 0.1.4__tar.gz

{mgcomtools-0.1.2 → mgcomtools-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mgcomtools
-Version: 0.1.2
+Version: 0.1.4
 Summary: usefull tools for data-stories team.
 Home-page: UNKNOWN
 Author: Mkozyrev (Maksim Kozyrev)

{mgcomtools-0.1.2 → mgcomtools-0.1.4}/mgcomtools/postgres.py

@@ -27,7 +27,8 @@ def connect_with_connector() -> sqlalchemy.engine.base.Engine:
     db_pass = 's7<f?exPFix65SUD'
     db_name = 'test_db'
 
-    ip_type = IPTypes.PUBLIC
+    os.environ["PRIVATE_IP"] = "10.77.125.3" 
+    ip_type = IPTypes.PRIVATE if os.environ.get("PRIVATE_IP") else IPTypes.PUBLIC
 
     # initialize Cloud SQL Python Connector object
     connector = Connector()
@@ -62,13 +63,13 @@ def fetch(executed):
     return [dict(row) for row in executed.fetchall()]
 
 # данные для трансформации из таблицы BQ
-def get_logs_to_transform(previous, step):
+def get_logs_to_transform(previous, step, condition=''):
 
     #Получаем данные для обработки
     query = f"""
         SELECT ftp, log_type, log_date, path, MAX(ftp_upload_time)
         FROM "adriver-logs"
-        WHERE {previous} = TRUE AND {step} is NULL
+        WHERE {previous} = TRUE AND {step} is NULL {condition}
         GROUP BY ftp, log_type, log_date, path
     """
     
@@ -80,7 +81,7 @@ def update_meta_data(path, step, status, bool='TRUE', option=''):
     query = f"""
         UPDATE "adriver-logs"
         SET status = '{status}', {step} = {bool}
-        WHERE path = '{path}' AND {step} is NULL{option};
+        WHERE path = '{path}' AND {step} is NULL {option};
         """
     
     conn.execute(query)

{mgcomtools-0.1.2 → mgcomtools-0.1.4}/mgcomtools/schema.py

@@ -11,7 +11,6 @@ class UpdateSchema:
 
     def __init__(self, df, path):
         self.path = path
-        print(path)
         self.df = df
 
 
@@ -20,8 +19,8 @@ class UpdateSchema:
             UpdateSchema._query_already_got[self.path] = self.get_schema_from_bq()
         self.schema = UpdateSchema._query_already_got[self.path]
         self.transform()
-        print(UpdateSchema._query_already_got)
-        print('Schema has been updated succesfully')
+        # print(UpdateSchema._query_already_got)
+        # print('Schema has been updated succesfully')
         
         return self.df
 
@@ -34,7 +33,7 @@ class UpdateSchema:
         """      
         query_job = bq_client.query(query).result()
         schema_dict = dict()
-        print(query)
+        # print(query)
 
         for row in query_job:
             column_name = row['field_name']
@@ -84,7 +83,7 @@ class PySparkDataFrame(UpdateSchema):
 def transform(data, path):
 
     if isinstance(data, pd.DataFrame):
-        print("Это pandas DataFrame")
+        # print("Это pandas DataFrame")
         df_obj = PandasDataFrame(data, path)
         
     # Проверка типа для PySpark DataFrame

{mgcomtools-0.1.2 → mgcomtools-0.1.4}/mgcomtools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mgcomtools
-Version: 0.1.2
+Version: 0.1.4
 Summary: usefull tools for data-stories team.
 Home-page: UNKNOWN
 Author: Mkozyrev (Maksim Kozyrev)

{mgcomtools-0.1.2 → mgcomtools-0.1.4}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 import codecs
 import os
 
-VERSION = '0.1.002'
+VERSION = '0.1.004'
 DESCRIPTION = 'usefull tools for data-stories team.'
 
 # Setting up