rdxz2-utill 0.1.3__py3-none-any.whl → 0.1.4__py3-none-any.whl

{rdxz2_utill-0.1.3.dist-info → rdxz2_utill-0.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rdxz2-utill
-Version: 0.1.3
+Version: 0.1.4
 Summary: Your daily Python utility
 Author-email: Richard Dharmawan <richard.dharmawan@gmail.com>
 License: MIT License
@@ -42,6 +42,7 @@ Requires-Dist: pydantic
 Requires-Dist: requests
 Requires-Dist: sshtunnel==0.4.0
 Provides-Extra: google-cloud
+Requires-Dist: google-api-python-client; extra == "google-cloud"
 Requires-Dist: google-cloud-bigquery; extra == "google-cloud"
 Requires-Dist: google-cloud-storage; extra == "google-cloud"
 Provides-Extra: postgresql

rdxz2_utill-0.1.4.dist-info/RECORD

@@ -0,0 +1,37 @@
+rdxz2_utill-0.1.4.dist-info/licenses/LICENSE,sha256=PF9CUvzP8XFYopEAzrMzSCovF7RdBdscPqJCDC6KjPc,1073
+utill/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+utill/my_bq.py,sha256=gVxtkKXzwKVJ8lcz7z-c6qW10_8gnks41Z5Aur7MJtM,27205
+utill/my_compare.py,sha256=232I5cJFBYDkd-whcl371YkYbQfJTscBhlZo5LFknvY,885
+utill/my_const.py,sha256=J2DhpPJ__hwyfciU2fmAsxvuRDNfBut7x-IqHAEmDOw,317
+utill/my_csv.py,sha256=Y0esABZTDKJdRLzkQdbqP5TjD_pGU0ntpofbxhCnv7A,3009
+utill/my_datetime.py,sha256=zd-YM-Lx5ZaS-35Opp1krvK4pzImld9xZK0OBNRVgEM,2204
+utill/my_dict.py,sha256=jPaPfdn4WYpm0uIBPiYFinpHhx1jXpFVDJ9npmvxGZQ,391
+utill/my_encryption.py,sha256=N8ltNH05gwYawaFfh77SzpsMKkRHdUcSkA7X-1l6VKs,2200
+utill/my_env.py,sha256=WZfwEBT5YIxWHsw42uJhHljJjU5ckyEi5RwKgNp-kG0,2226
+utill/my_file.py,sha256=zhyxpmIC3zmdtlvlcDR53PMjov68ZSSscOhQ2dOdbfM,1891
+utill/my_gcs.py,sha256=sKnG2YcAA0RPVmtaKmZ3lj0LCFvBhMCBmN9kilhL1Cg,3369
+utill/my_gdrive.py,sha256=5BTfHPCIYRi6lC1VsQUbLwkInN-WcPahnJqSdrItkWk,6386
+utill/my_input.py,sha256=hOpoFmDi2_AyTv6LWv7pOaMWMXrDjkRUCs8pLtzoWCY,364
+utill/my_json.py,sha256=6FYldd4sNpnUbEHdCbUZ4sU1JDiSSnxGIOYDsa0cBnc,1472
+utill/my_mb.py,sha256=656Mh6XYJKadLM8eSL9zpwyk_5XlGMDzzGQxRNCx2rk,13965
+utill/my_pg.py,sha256=MYiMMQVwbVBJ8CDNUSCIhTJiZc2pekutvrH893JdKUM,7176
+utill/my_queue.py,sha256=vtcQvO65bIiJ9_slT2pkshaHWfULE-ADxJHwIh20pEg,4746
+utill/my_string.py,sha256=upy5wdW71fzZqkhGDQRGTbc-XFCotW2tawriTCMexBg,965
+utill/my_style.py,sha256=gY6ZPB0PoXVBEMKeBlhCJYYVgNEQiB-zz-FcPO6s23M,906
+utill/my_tunnel.py,sha256=GifDuLT1J7Zi4X6l2xPjnirszLnUre3UhlqJN592lMM,1454
+utill/my_xlsx.py,sha256=aMUhGDjWnOCMxuAR3BGn-x2GKPUIY-0TJ43OdeWJlAw,726
+utill/cmd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+utill/cmd/_bq.py,sha256=JhY3HcUv8ByXXV4l_8HB_uXTSljU3gv8u2kOZ-XjyCE,622
+utill/cmd/_conf.py,sha256=V4YSvubJ-HTuMKY3Muf7Zl0LALOE4fG4AUlCxzENzwk,1861
+utill/cmd/_enc.py,sha256=aCD_8wMTPvMbjqByg0Yiac2AABrLr2Gz2KRRHXi6-ZM,1029
+utill/cmd/_main.py,sha256=UJ_XTIGDO9XPIypgHhS81SJQ_8qy8JOyw98Or0Nb2x8,273
+utill/cmd/_mb.py,sha256=lRl1Pqp8xIO9pyvnsh9WxugKGi7rdf-HUXdsLHvPqz4,5274
+utill/cmd/_pg.py,sha256=Vn9BY0aiuLmk-HbBjBHEghG6KzR7NATQgvaDGDE2wgo,498
+utill/cmd/utill.py,sha256=HsCfPtWSOn8dOfS31JQi4Lq83CmSd5yQmIkva3bhvWc,5344
+utill/templates/mb.json,sha256=M46ZHSaSh4rbD_KGUViGr2B2ZV8_PC-O5Evqi35JK5g,59
+utill/templates/pg.json,sha256=LkJt0VV3zcyt7Tpn6gulsoVQgUc-9uImXOStvzu8cdU,271
+rdxz2_utill-0.1.4.dist-info/METADATA,sha256=9K2krYizIKTQp3s-uNnfDLOdH2gA3n0Es8aOaie-w5o,4466
+rdxz2_utill-0.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+rdxz2_utill-0.1.4.dist-info/entry_points.txt,sha256=9n5NWz5Wi9jDvYhB_81_4icgT5xABZ-QivHD8ibcafg,47
+rdxz2_utill-0.1.4.dist-info/top_level.txt,sha256=tuAYZoCsr02JYbpZj7I6fl1IIo53v3GG0uoj-_fINVk,6
+rdxz2_utill-0.1.4.dist-info/RECORD,,

utill/cmd/_bq.py

@@ -1,12 +1,25 @@
-def _upload_csv(src_filename: str, dst_table_fqn: str, cols: list[tuple[str, str]], partition_col: str = None, cluster_cols: list[str] = None, project: str = None):
+def _upload_csv(
+    src_filename: str,
+    dst_table_fqn: str,
+    columns: list[dict[str, str]],
+    partition_col: str = None,
+    cluster_cols: list[str] = None,
+    project: str = None,
+):
     from ..my_bq import BQ
 
     bq = BQ(project)
-    bq.upload_csv(src_filename, dst_table_fqn, {col: dtype for col, dtype in cols}, partition_col, cluster_cols)
+    bq.upload_csv(
+        src_filename,
+        dst_table_fqn,
+        [{"name": col, "data_type": dtype} for col, dtype in columns],
+        partition_col,
+        cluster_cols,
+    )
 
 
 def _download_table(src_table_fqn: str, dst_filename: str, project: str):
     from ..my_bq import BQ
 
     bq = BQ(project)
-    bq.download_csv(f'SELECT * FROM {src_table_fqn}', dst_filename)
+    bq.download_csv(f"SELECT * FROM {src_table_fqn}", dst_filename)

utill/cmd/_conf.py

@@ -4,18 +4,18 @@ def _init(mode: str):
     from ..my_env import envs, init_pg_file, init_mb_file
 
     match mode:
-        case 'google-cloud':
-            setattr(envs, 'GCP_PROJECT_ID', input('GCP_PROJECT_ID: '))
-            setattr(envs, 'GCP_REGION', input('GCP_REGION: '))
-            setattr(envs, 'GCS_BUCKET', input('GCS_BUCKET: '))
+        case "google-cloud":
+            setattr(envs, "GCP_PROJECT_ID", input("GCP_PROJECT_ID: "))
+            setattr(envs, "GCP_REGION", input("GCP_REGION: "))
+            setattr(envs, "GCS_BUCKET", input("GCS_BUCKET: "))
             envs.write()
-            logger.info('Google cloud configuration initialized')
-        case 'postgresql':
+            logger.info("Google cloud configuration initialized")
+        case "postgresql":
             init_pg_file()
-        case 'metabase':
+        case "metabase":
             init_mb_file()
         case _:
-            logger.warning(f'Mode \'{mode}\' not recognized')
+            logger.warning(f"Mode '{mode}' not recognized")
 
 
 def _list(module: str = None):
@@ -28,28 +28,28 @@ def _list(module: str = None):
     from ..my_string import mask
 
     match module:
-        case 'postgresql':
+        case "postgresql":
             if not os.path.exists(PG_FILENAME):
-                logger.error('PostgreSQL configuraiton not exists')
+                logger.error("PostgreSQL configuraiton not exists")
                 return
 
-            config: dict = json.loads(open(PG_FILENAME, 'r').read())
+            config: dict = json.loads(open(PG_FILENAME, "r").read())
             for k, v in config.items():
                 print(k)
                 for k2, v2 in v.items():
                     print(f'\t{k2} = {mask(str(v2)) if k2 in ("password", ) else v2}')
 
-        case 'metabase':
+        case "metabase":
             if not os.path.exists(MB_FILENAME):
-                logger.error('Metabase configuration not exists')
+                logger.error("Metabase configuration not exists")
                 return
 
-            config: dict = json.loads(open(MB_FILENAME, 'r').read())
+            config: dict = json.loads(open(MB_FILENAME, "r").read())
             for k, v in config.items():
                 print(f'{k} = {mask(str(v)) if k in ("api_key", ) else v}')
         case _:
             for env in envs.model_fields:
-                print(f'{env} = {getattr(envs, env)}')
+                print(f"{env} = {getattr(envs, env)}")
 
 
 def _set(vars: list[tuple[str, str]]):

utill/cmd/_enc.py

@@ -5,22 +5,26 @@ def _encrypt(src: str, password: str, output: str = None, force: bool = False):
     path_password = Path(password).expanduser()
     if path_password.exists():
         if not path_password.is_file():
-            raise ValueError(f'Password path is not a file: {password}')
+            raise ValueError(f"Password path is not a file: {password}")
         else:
-            password = open(path_password.as_posix(), 'r').read().strip()
+            password = open(path_password.as_posix(), "r").read().strip()
 
     path_src = Path(src).expanduser()
     if path_src.exists():
         if path_src.is_dir():
-            raise ValueError(f'Source file is a directory: {src}')
+            raise ValueError(f"Source file is a directory: {src}")
 
         # Do encryption
         from ..my_encryption import encrypt_file
+
         if output:
-            encrypt_file(path_src.as_posix(), password, dst_filename=output, overwrite=force)
+            encrypt_file(
+                path_src.as_posix(), password, dst_filename=output, overwrite=force
+            )
         else:
             print(encrypt_file(path_src.as_posix(), password))
     else:
         # Do encryption
         from ..my_encryption import encrypt_string
+
         print(encrypt_string(src, password))

utill/cmd/_mb.py

@@ -1,60 +1,140 @@
-def _grant(emails: list[str], urls: list[str]):
+
+def _jl_grant(emails: list[str], url: str, create_user_if_not_exists: bool = False):
     from ..my_mb import MB
+    from loguru import logger
 
     mb = MB()
-    mb.make_sure_all_email_exists(emails)
-
-    question_urls = []
-    collection_urls = []
-    dashboard_urls = []
-    for url in urls:
-        if not url.startswith(mb.base_url):
-            raise ValueError(f'URL must be a Metabase URL: {url}')
-
-        url_stripped = url.removeprefix(mb.base_url)
-        if url_stripped.startswith('/collection/'):
-            collection_urls.append(url)
-        elif url_stripped.startswith('/question/'):
-            question_urls.append(url)
-        elif url_stripped.startswith('/dashboard/'):
-            dashboard_urls.append(url)
-        else:
-            raise ValueError(f'URL is neither a collection/question/dashboard: {url} --> {url_stripped}')
-
+    all_users_by_email = {
+        user["email"]: user for user in mb.get_all_users(all=True)
+    }
+    all_groups_by_name = {x["name"]: x for x in mb.get_all_groups()}
+
+    # Get information for this object
+    logger.info("Getting Metabase object information")
+    object_type, object_id = mb.get_object_info_from_url(url)
+    collection_id: int | None = None
+    collection_location: str | None = None
+    match (object_type):
+        case "question":
+            question = mb.get_question(object_id)
+            collection_id = int(question["collection"]["id"])
+            collection_location = question["collection"]["location"] + str(
+                question["collection"]["id"]
+            )
+        case "dashboard":
+            dashboard = mb.get_dashboard(object_id)
+            collection_id = int(dashboard["collection"]["id"])
+            collection_location = dashboard["collection"]["location"] + str(
+                dashboard["collection"]["id"]
+            )
+        case "collection":
+            collection = mb.get_collection(object_id)
+            collection_id = object_id
+            collection_location = collection["location"] + str(
+                collection["collection"]["id"]
+            )
+        case _:
+            raise ValueError(
+                f"Unknown object type {object_type} from {url}"
+            )
+    logger.info(
+        f'Object found: type "{object_type}", ID {object_id}, collection ID {collection_id}'
+    )
+
+    # Get group info that this collection should be granted to
+    logger.info(f"Getting group information for the object: {collection_location}")
+    group_name = mb.decode_collection_location_to_group(collection_location)
+    if group_name not in all_groups_by_name:
+        # If group not exists, create it and immediately grant readonly access to the collectiond
+        mb.create_group(group_name)
+        all_groups_by_name = {x["name"]: x for x in mb.get_all_groups()}
+        group_id = int(all_groups_by_name[group_name]["id"])
+        mb.grant_group_to_collection(group_id, collection_id)
+    else:
+        group_id = int(all_groups_by_name[group_name]["id"])
+    logger.info(f"Group found: [{group_id}] {group_name}")
+
+    # Get user informations, create if not exists
+    logger.info(f"Getting information from {len(emails)} users")
+    users = set()
+    created_users = 0
+    not_found_emails = []
+    for email in emails:
+        if email not in all_users_by_email:
+            if create_user_if_not_exists:
+                logger.info(f"Creating user {email}")
+                email_name, email_domain = email.split("@", 1)
+                mb.create_user(
+                    first_name=email_name,
+                    last_name=email_domain,
+                    email=email,
+                    group_ids=[1],  # Add to 'All Users' group
+                )
+                # all_users_by_email = {
+                #     user["email"]: user for user in mb.get_all_users(all=True)
+                # }
+                created_users += 1
+            else:
+                not_found_emails.append(email)
+    if not_found_emails:
+        raise ValueError(f"Users not found: {', '.join(not_found_emails)}")
+
+    # Re-fetch all users if there are new users created
+    if created_users:
+        logger.info("Users created, re-fetching all users")
+        all_users_by_email = {
+            user["email"]: user for user in mb.get_all_users(all=True)
+        }
+
+    # Grant access
+    logger.info(
+        f"Granting access to group [{group_id}] {group_name} for {len(emails)} users"
+    )
     for email in emails:
-        for question_url in question_urls:
-            mb.grant_user_email_to_question_by_url(email, question_url)
-        for collection_url in collection_urls:
-            mb.grant_user_email_to_collection_by_url(email, collection_url)
-        for dashboard_url in dashboard_urls:
-            mb.grant_user_email_to_dashboard_by_url(email, dashboard_url)
+        user = all_users_by_email[email]
+        if (
+            not user["is_active"]
+        ) and create_user_if_not_exists:  # Reactivate user if disabled
+            logger.info(f"Reactivating user {user['id']}")
+            mb.enable_user(user["id"])
+
+        user_id = int(user["id"])
+        user_email = user["email"]
+        if group_id in user["group_ids"]:
+            # Skip if user already in the group because it will cause 500 error on Metabase later (it tries to insert the permissions to its DB and got duplicate key error)
+            logger.info(f"User {user_id} already in group {group_id}, skipping")
+            continue
+        users.add((user_id, user_email))
+    logger.info(
+        f"Users to be granted: {', '.join([f'[{user_id}] {user_email}' for user_id, user_email in users])}"
+    )
+
+    # Assign all user to the group
+    for user_id, user_email in users:
+        logger.info(f"Assigning user {user_id} to group {group_id}")
+        mb.grant_user_to_group(user_id, group_id)
+    logger.info("All users assigned to the group")
 
 
 def _copy_permissions(src_email: str, dst_emails: list[str]):
     from ..my_mb import MB
 
     mb = MB()
-    mb.make_sure_all_email_exists([src_email] + dst_emails)
-
     for dst_email in dst_emails:
-        mb.mirror_user_permission_by_email(src_email, dst_email)
+        mb.mirror_permission(src_email, dst_email)
 
 
 def _reset_password(emails: list[str]):
     from ..my_mb import MB
 
     mb = MB()
-    mb.make_sure_all_email_exists(emails)
-
     for email in emails:
-        mb.reset_password_by_email(email)
+        mb.reset_password(email)
 
 
-def _deactivate_user(emails: list[str]):
+def _disable_user(emails: list[str]):
     from ..my_mb import MB
 
     mb = MB()
-    mb.make_sure_all_email_exists(emails)
-
     for email in emails:
-        mb.deactivate_user_by_email(email)
+        mb.disable_user(email)

utill/cmd/_pg.py

@@ -1,7 +1,9 @@
-def _pg_to_pg(src_profile: str, src_table: str, dst_profile: str, dst_table: str, columns: str):
+def _pg_to_pg(
+    src_profile: str, src_table: str, dst_profile: str, dst_table: str, columns: str
+):
     from ..my_pg import PG
 
-    columns = ','.join([f"{x}" for x in columns.split(',')]) if columns != '*' else None
+    columns = ",".join([f"{x}" for x in columns.split(",")]) if columns != "*" else None
     pg_src = PG(src_profile)
     pg_dst = PG(dst_profile)