rdxz2-utill 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

{rdxz2_utill-0.1.1.dist-info → rdxz2_utill-0.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rdxz2-utill
-Version: 0.1.1
+Version: 0.1.3
 Summary: Your daily Python utility
 Author-email: Richard Dharmawan <richard.dharmawan@gmail.com>
 License: MIT License

{rdxz2_utill-0.1.1.dist-info → rdxz2_utill-0.1.3.dist-info}/RECORD

@@ -1,6 +1,6 @@
-rdxz2_utill-0.1.1.dist-info/licenses/LICENSE,sha256=PF9CUvzP8XFYopEAzrMzSCovF7RdBdscPqJCDC6KjPc,1073
+rdxz2_utill-0.1.3.dist-info/licenses/LICENSE,sha256=PF9CUvzP8XFYopEAzrMzSCovF7RdBdscPqJCDC6KjPc,1073
 utill/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-utill/my_bq.py,sha256=ANyKcw4VDn3L868ezYUFSgbmbaUjD_PW6XZ0fqxUx6s,25093
+utill/my_bq.py,sha256=U3YlLTEQfDSsIbDi2Cjw8fv51uwjavNscLo3dZ-R4fU,25452
 utill/my_compare.py,sha256=619QbVk3GihWxen95yVnivKHkah8GgPTLGiSkgHxykw,886
 utill/my_const.py,sha256=88dOqn6NPQ5-hfRqdkew5POoAIyO91XXOGvN76oNsdo,251
 utill/my_csv.py,sha256=AT5sAbAlYqnAmNgQMTSqEueRXM4D42yNPb5C3Hedy6c,2921
@@ -9,7 +9,7 @@ utill/my_dict.py,sha256=jPaPfdn4WYpm0uIBPiYFinpHhx1jXpFVDJ9npmvxGZQ,391
 utill/my_encryption.py,sha256=SCF7PPur39cW4RHidsRhw-9BZP-ymUH-6LZ9nAHJDsY,2105
 utill/my_env.py,sha256=E7XW3fuhxbDlFqmLPHrziJJZVRogzGh6rfQdyNV49f8,2130
 utill/my_file.py,sha256=-b6_dGDDBdS228kgwTYpmIa3vxW1c1TtWrLdzdlHjKY,1873
-utill/my_gcs.py,sha256=VBJ8lsJ-fHr_BzMoSuT5JUrvxidGyMc2VNtE6Um1T_M,3060
+utill/my_gcs.py,sha256=qFH47gQmk6_v0DajihymX7Xx1ZVp4xWuxDQh90PrY0g,3088
 utill/my_input.py,sha256=OyKLoutXpwISReltuL_Gw2oojv16tYWJqQpqabBOQx4,350
 utill/my_json.py,sha256=WgW6mavGhfs4h1N5XbhsDnRk2dbh_ttJWdJUj4iWDN4,1473
 utill/my_mb.py,sha256=IyrySs92TqtjBUvPMeUN3P2kRK8EttTFRPZsv5Cr-xw,15090
@@ -24,12 +24,13 @@ utill/cmd/_bq.py,sha256=MQGLIv_WBUBl2tf18bfYrAszx0Koa5kdTW1c8A5HDDg,520
 utill/cmd/_conf.py,sha256=DKl3IVVLp6-5P43tvh6bYmHR5rOL9XnKVuQ7kQJtzrc,1863
 utill/cmd/_enc.py,sha256=DBy3Iwa5DTtww7lgHPRLEilrYPrWDG1vRv5PO-YzNO8,997
 utill/cmd/_main.py,sha256=UJ_XTIGDO9XPIypgHhS81SJQ_8qy8JOyw98Or0Nb2x8,273
+utill/cmd/_mb.py,sha256=dtYk9HriSaYsn-6ZFIJhs8OPHUqI5fLSZosEHNuwykU,1823
 utill/cmd/_pg.py,sha256=RVxEiSifyIwMDYDM69vt6WSLdVDr1cMzY6r4T2PzNRA,492
-utill/cmd/utill.py,sha256=TlHfiwOUcK1m58PrRCjX9sARiPYZUsoTk-KOTCOz1vM,3558
+utill/cmd/utill.py,sha256=ZopLhpPo3Qn8B_Un64qVwuB1oGaAacgN6DVPCTK5V_o,4849
 utill/templates/mb.json,sha256=M46ZHSaSh4rbD_KGUViGr2B2ZV8_PC-O5Evqi35JK5g,59
 utill/templates/pg.json,sha256=LkJt0VV3zcyt7Tpn6gulsoVQgUc-9uImXOStvzu8cdU,271
-rdxz2_utill-0.1.1.dist-info/METADATA,sha256=qroXZ2Hm-L-YQHR8xuIcpTqrbEuy4_CDceCZeNqTi1Q,4401
-rdxz2_utill-0.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-rdxz2_utill-0.1.1.dist-info/entry_points.txt,sha256=9n5NWz5Wi9jDvYhB_81_4icgT5xABZ-QivHD8ibcafg,47
-rdxz2_utill-0.1.1.dist-info/top_level.txt,sha256=tuAYZoCsr02JYbpZj7I6fl1IIo53v3GG0uoj-_fINVk,6
-rdxz2_utill-0.1.1.dist-info/RECORD,,
+rdxz2_utill-0.1.3.dist-info/METADATA,sha256=mJqMSx_wABv4itLJfc6USR1q1Oyd-u91qfyXpFTmtMM,4401
+rdxz2_utill-0.1.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+rdxz2_utill-0.1.3.dist-info/entry_points.txt,sha256=9n5NWz5Wi9jDvYhB_81_4icgT5xABZ-QivHD8ibcafg,47
+rdxz2_utill-0.1.3.dist-info/top_level.txt,sha256=tuAYZoCsr02JYbpZj7I6fl1IIo53v3GG0uoj-_fINVk,6
+rdxz2_utill-0.1.3.dist-info/RECORD,,

utill/cmd/_mb.py

@@ -0,0 +1,60 @@
+def _grant(emails: list[str], urls: list[str]):
+    from ..my_mb import MB
+
+    mb = MB()
+    mb.make_sure_all_email_exists(emails)
+
+    question_urls = []
+    collection_urls = []
+    dashboard_urls = []
+    for url in urls:
+        if not url.startswith(mb.base_url):
+            raise ValueError(f'URL must be a Metabase URL: {url}')
+
+        url_stripped = url.removeprefix(mb.base_url)
+        if url_stripped.startswith('/collection/'):
+            collection_urls.append(url)
+        elif url_stripped.startswith('/question/'):
+            question_urls.append(url)
+        elif url_stripped.startswith('/dashboard/'):
+            dashboard_urls.append(url)
+        else:
+            raise ValueError(f'URL is neither a collection/question/dashboard: {url} --> {url_stripped}')
+
+    for email in emails:
+        for question_url in question_urls:
+            mb.grant_user_email_to_question_by_url(email, question_url)
+        for collection_url in collection_urls:
+            mb.grant_user_email_to_collection_by_url(email, collection_url)
+        for dashboard_url in dashboard_urls:
+            mb.grant_user_email_to_dashboard_by_url(email, dashboard_url)
+
+
+def _copy_permissions(src_email: str, dst_emails: list[str]):
+    from ..my_mb import MB
+
+    mb = MB()
+    mb.make_sure_all_email_exists([src_email] + dst_emails)
+
+    for dst_email in dst_emails:
+        mb.mirror_user_permission_by_email(src_email, dst_email)
+
+
+def _reset_password(emails: list[str]):
+    from ..my_mb import MB
+
+    mb = MB()
+    mb.make_sure_all_email_exists(emails)
+
+    for email in emails:
+        mb.reset_password_by_email(email)
+
+
+def _deactivate_user(emails: list[str]):
+    from ..my_mb import MB
+
+    mb = MB()
+    mb.make_sure_all_email_exists(emails)
+
+    for email in emails:
+        mb.deactivate_user_by_email(email)

utill/cmd/utill.py

@@ -5,7 +5,7 @@ import click
 def main(): pass
 
 
-# Conf
+# MARK: Conf
 
 
 @main.group('conf', help='Configure this library')
@@ -21,7 +21,28 @@ def main__conf__list(**kwargs): from ._conf import _list; _list(**kwargs)
 def main__conf__set(**kwargs): from ._conf import _set; _set(**kwargs)
 
 
-# PG
+# MARK: Metabase
+
+
+@main.group('mb', help='Metabase utility commands')
+def main__mb(): pass
+@main__mb.command('julo-grant', help='Grant access to Metabase questions/collections/dashboards')
+@click.option('-u', '--email', 'emails', type=str, multiple=True, help='User emails')
+@click.option('-l', '--url', 'urls', type=str, multiple=True, help='URLs')
+def main__mb__grant(**kwargs): from ._mb import _grant; _grant(**kwargs)
+@main__mb.command('copy-permissions', help='Copy all permissions from one user to another')
+@click.argument('src_email', type=str)
+@click.argument('dst_emails', type=str, nargs=-1)
+def main__mb__copy_permissions(**kwargs): from ._mb import _copy_permissions; _copy_permissions(**kwargs)
+@main__mb.command('reset-password', help='Reset Metabase user password')
+@click.option('-u', '--email', 'emails', type=str, required=True, multiple=True, help='User emails')
+def main__mb__reset_password(**kwargs): from ._mb import _reset_password; _reset_password(**kwargs)
+@main__mb.command('deactivate-user', help='Deactivate Metabase user')
+@click.option('-u', '--email', 'emails', type=str, required=True, multiple=True, help='User emails')
+def main__mb__deactivate_user(**kwargs): from ._mb import _deactivate_user; _deactivate_user(**kwargs)
+
+
+# MARK: PG
 
 
 @main.group('pg', help='PostgreSQL utility')
@@ -40,7 +61,7 @@ def main__pg__pg_to_pg(**kwargs): from ._pg import _pg_to_pg; _pg_to_pg(**kwargs
 def main__pg__upload_csv(**kwargs): from ._pg import _upload_csv; _upload_csv(**kwargs)
 
 
-# BQ
+# MARK: BQ
 
 
 @main.group('bq', help='BigQuery utility')
@@ -60,7 +81,7 @@ def main__bq__upload_csv(**kwargs): from ._bq import _upload_csv; _upload_csv(**
 def main__bq__download_table(**kwargs): from ._bq import _download_table; _download_table(**kwargs)
 
 
-# Encyrption
+# MARK: Encyrption
 
 
 @main.group('enc', help='Encryption utility')
@@ -71,7 +92,7 @@ def main__enc(): pass
 def main__enc__encrypt(**kwargs): from ._enc import _encrypt; _encrypt(**kwargs)
 
 
-# Other utilities
+# MARK: Other utilities
 
 
 @main.command('random', help='Generate random string')

utill/my_bq.py

@@ -312,21 +312,24 @@ class BQ():
 
     def upload_csv(
         self,
-        src_filename: str,
+        src_filepath: str,
         dst_table_fqn: str,
         schema: list[dict] | None = None,
         gcs_bucket: str | None = None,
         partition_by: str = None,
-        cluster_cols: list[str] = None,
+        clustering_fields: list[str] = None,
         compression: DataFileCompression | None = None,
         load_strategy: LoadStrategy = LoadStrategy.APPEND,
     ):
         self.raise_for_invalid_table_fqn(dst_table_fqn)
 
-        if compression == DataFileCompression.GZIP and not src_filename.endswith('.gz'):
+        if compression == DataFileCompression.GZIP and not src_filepath.endswith('.gz'):
             raise ValueError('Please provide file path with .gz extension if using compression = GZIP')
-        elif not src_filename.endswith('.csv'):
+        elif not src_filepath.endswith('.csv'):
             raise ValueError('Please provide file path with .csv extension')
+        
+        src_filename, src_fileextension = os.path.splitext(src_filepath)
+        src_filename = os.path.basename(src_filename)  # Only get filename
 
         # # <<----- START: Upload to GCS
 
@@ -354,12 +357,21 @@ class BQ():
         # Upload to GCS
         # TODO: Re-implement the producer-consumer model to upload multiple files
         gcs = my_gcs.GCS(bucket=gcs_bucket, project_id=self.client.project)
-        dst_blobpath = f'tmp/my_bq/{my_datetime.get_current_datetime_str()}/{my_string.replace_nonnumeric(os.path.basename(src_filename), "_").lower()}'
-        gcs.upload(src_filename, dst_blobpath)
+        dst_blobpath = f'tmp/my_bq/{my_datetime.get_current_datetime_str()}/{my_string.replace_nonnumeric(src_filename, "_").lower()}{src_fileextension}'
+        gcs.upload(src_filepath, dst_blobpath)
 
         # Load to BQ
         try:
-            self.load_data(dst_blobpath, dst_table_fqn, schema=schema, partition_by=partition_by, cluster_cols=cluster_cols, format=DataFileFormat.CSV, compression=compression, load_strategy=load_strategy)
+            self.load_data(
+                f'gs://{gcs.bucket.name}/{dst_blobpath}',
+                dst_table_fqn,
+                schema=schema,
+                partition_by=partition_by,
+                clustering_fields=clustering_fields,
+                format=DataFileFormat.CSV,
+                compression=compression,
+                load_strategy=load_strategy,
+            )
         except:
             raise
         finally:
@@ -399,7 +411,7 @@ class BQ():
                 for tmp_blobs in gcs.list_blobs(dst_gcs_prefix):
                     local_tmp_filepath = os.path.join(tmp_dirname, tmp_blobs.name.split('/')[-1])
                     gcs.download(tmp_blobs, local_tmp_filepath, move=True)
-                    logger.debug(f'Downloaded {tmp_blobs.name} to {local_tmp_filepath}')
+                    # logger.debug(f'Downloaded {tmp_blobs.name} to {local_tmp_filepath}')
                     local_tmp_filepaths.append(local_tmp_filepath)
 
                 # Combine downloaded files
@@ -435,7 +447,7 @@ class BQ():
                 for part in range(parts):
                     dst_filepath_part = f'{dst_filepath.removesuffix(".csv")}_{part + 1:06}.csv'
                     _export_download_combine(
-                        f'SELECT * EXCEPT(_rn) FROM `{tmp_table_fqn_rn}` WHERE _rn BETWEEN {(part * csv_row_limit) + 1} AND {(part + 1) * csv_row_limit}',
+                        f'SELECT * EXCEPT(_rn) FROM `{tmp_table_fqn_rn}` WHERE _rn BETWEEN {(part * csv_row_limit) + 1} AND {(part + 1) * csv_row_limit} ORDER BY _rn',
                         dst_gcs_prefix=gcs.build_tmp_dirpath(),
                         dst_filepath=dst_filepath_part,
                     )

utill/my_gcs.py

@@ -16,7 +16,7 @@ class GCS:
 
         self.client = storage.Client(project=project_id or envs.GCP_PROJECT_ID)
         self.bucket = self.client.bucket(bucket or envs.GCS_BUCKET)
-        logger.debug(f'GCS client open, project: {self.client.project}')
+        logger.debug(f'GCS client open, project: {self.client.project}, bucket: {self.bucket.name}')
 
     def get_blob(self, blobpath: str) -> storage.Blob:
         return self.bucket.blob(blobpath)