rdxz2-utill 0.1.3__tar.gz → 0.1.4__tar.gz

{rdxz2_utill-0.1.3/src/rdxz2_utill.egg-info → rdxz2_utill-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rdxz2-utill
-Version: 0.1.3
+Version: 0.1.4
 Summary: Your daily Python utility
 Author-email: Richard Dharmawan <richard.dharmawan@gmail.com>
 License: MIT License
@@ -42,6 +42,7 @@ Requires-Dist: pydantic
 Requires-Dist: requests
 Requires-Dist: sshtunnel==0.4.0
 Provides-Extra: google-cloud
+Requires-Dist: google-api-python-client; extra == "google-cloud"
 Requires-Dist: google-cloud-bigquery; extra == "google-cloud"
 Requires-Dist: google-cloud-storage; extra == "google-cloud"
 Provides-Extra: postgresql

{rdxz2_utill-0.1.3 → rdxz2_utill-0.1.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rdxz2-utill"
-version = "0.1.3"
+version = "0.1.4"
 authors = [
   { name="Richard Dharmawan", email="richard.dharmawan@gmail.com" },
 ]
@@ -36,6 +36,7 @@ utill = "utill.cmd.utill:main"
 
 [project.optional-dependencies]
 google-cloud = [
+  "google-api-python-client",
   "google-cloud-bigquery",
   "google-cloud-storage",
 ]

{rdxz2_utill-0.1.3 → rdxz2_utill-0.1.4/src/rdxz2_utill.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rdxz2-utill
-Version: 0.1.3
+Version: 0.1.4
 Summary: Your daily Python utility
 Author-email: Richard Dharmawan <richard.dharmawan@gmail.com>
 License: MIT License
@@ -42,6 +42,7 @@ Requires-Dist: pydantic
 Requires-Dist: requests
 Requires-Dist: sshtunnel==0.4.0
 Provides-Extra: google-cloud
+Requires-Dist: google-api-python-client; extra == "google-cloud"
 Requires-Dist: google-cloud-bigquery; extra == "google-cloud"
 Requires-Dist: google-cloud-storage; extra == "google-cloud"
 Provides-Extra: postgresql

{rdxz2_utill-0.1.3 → rdxz2_utill-0.1.4}/src/rdxz2_utill.egg-info/SOURCES.txt

@@ -18,6 +18,7 @@ src/utill/my_encryption.py
 src/utill/my_env.py
 src/utill/my_file.py
 src/utill/my_gcs.py
+src/utill/my_gdrive.py
 src/utill/my_input.py
 src/utill/my_json.py
 src/utill/my_mb.py

{rdxz2_utill-0.1.3 → rdxz2_utill-0.1.4}/src/rdxz2_utill.egg-info/requires.txt

@@ -9,6 +9,7 @@ requests
 sshtunnel==0.4.0
 
 [google-cloud]
+google-api-python-client
 google-cloud-bigquery
 google-cloud-storage
 

rdxz2_utill-0.1.4/src/utill/cmd/_bq.py

@@ -0,0 +1,25 @@
+def _upload_csv(
+    src_filename: str,
+    dst_table_fqn: str,
+    columns: list[dict[str, str]],
+    partition_col: str = None,
+    cluster_cols: list[str] = None,
+    project: str = None,
+):
+    from ..my_bq import BQ
+
+    bq = BQ(project)
+    bq.upload_csv(
+        src_filename,
+        dst_table_fqn,
+        [{"name": col, "data_type": dtype} for col, dtype in columns],
+        partition_col,
+        cluster_cols,
+    )
+
+
+def _download_table(src_table_fqn: str, dst_filename: str, project: str):
+    from ..my_bq import BQ
+
+    bq = BQ(project)
+    bq.download_csv(f"SELECT * FROM {src_table_fqn}", dst_filename)

{rdxz2_utill-0.1.3 → rdxz2_utill-0.1.4}/src/utill/cmd/_conf.py

@@ -4,18 +4,18 @@ def _init(mode: str):
     from ..my_env import envs, init_pg_file, init_mb_file
 
     match mode:
-        case 'google-cloud':
-            setattr(envs, 'GCP_PROJECT_ID', input('GCP_PROJECT_ID: '))
-            setattr(envs, 'GCP_REGION', input('GCP_REGION: '))
-            setattr(envs, 'GCS_BUCKET', input('GCS_BUCKET: '))
+        case "google-cloud":
+            setattr(envs, "GCP_PROJECT_ID", input("GCP_PROJECT_ID: "))
+            setattr(envs, "GCP_REGION", input("GCP_REGION: "))
+            setattr(envs, "GCS_BUCKET", input("GCS_BUCKET: "))
             envs.write()
-            logger.info('Google cloud configuration initialized')
-        case 'postgresql':
+            logger.info("Google cloud configuration initialized")
+        case "postgresql":
             init_pg_file()
-        case 'metabase':
+        case "metabase":
             init_mb_file()
         case _:
-            logger.warning(f'Mode \'{mode}\' not recognized')
+            logger.warning(f"Mode '{mode}' not recognized")
 
 
 def _list(module: str = None):
@@ -28,28 +28,28 @@ def _list(module: str = None):
     from ..my_string import mask
 
     match module:
-        case 'postgresql':
+        case "postgresql":
             if not os.path.exists(PG_FILENAME):
-                logger.error('PostgreSQL configuraiton not exists')
+                logger.error("PostgreSQL configuraiton not exists")
                 return
 
-            config: dict = json.loads(open(PG_FILENAME, 'r').read())
+            config: dict = json.loads(open(PG_FILENAME, "r").read())
             for k, v in config.items():
                 print(k)
                 for k2, v2 in v.items():
                     print(f'\t{k2} = {mask(str(v2)) if k2 in ("password", ) else v2}')
 
-        case 'metabase':
+        case "metabase":
             if not os.path.exists(MB_FILENAME):
-                logger.error('Metabase configuration not exists')
+                logger.error("Metabase configuration not exists")
                 return
 
-            config: dict = json.loads(open(MB_FILENAME, 'r').read())
+            config: dict = json.loads(open(MB_FILENAME, "r").read())
             for k, v in config.items():
                 print(f'{k} = {mask(str(v)) if k in ("api_key", ) else v}')
         case _:
             for env in envs.model_fields:
-                print(f'{env} = {getattr(envs, env)}')
+                print(f"{env} = {getattr(envs, env)}")
 
 
 def _set(vars: list[tuple[str, str]]):

{rdxz2_utill-0.1.3 → rdxz2_utill-0.1.4}/src/utill/cmd/_enc.py

@@ -5,22 +5,26 @@ def _encrypt(src: str, password: str, output: str = None, force: bool = False):
     path_password = Path(password).expanduser()
     if path_password.exists():
         if not path_password.is_file():
-            raise ValueError(f'Password path is not a file: {password}')
+            raise ValueError(f"Password path is not a file: {password}")
         else:
-            password = open(path_password.as_posix(), 'r').read().strip()
+            password = open(path_password.as_posix(), "r").read().strip()
 
     path_src = Path(src).expanduser()
     if path_src.exists():
         if path_src.is_dir():
-            raise ValueError(f'Source file is a directory: {src}')
+            raise ValueError(f"Source file is a directory: {src}")
 
         # Do encryption
         from ..my_encryption import encrypt_file
+
         if output:
-            encrypt_file(path_src.as_posix(), password, dst_filename=output, overwrite=force)
+            encrypt_file(
+                path_src.as_posix(), password, dst_filename=output, overwrite=force
+            )
         else:
             print(encrypt_file(path_src.as_posix(), password))
     else:
         # Do encryption
         from ..my_encryption import encrypt_string
+
         print(encrypt_string(src, password))

rdxz2_utill-0.1.4/src/utill/cmd/_mb.py

@@ -0,0 +1,140 @@
+
+def _jl_grant(emails: list[str], url: str, create_user_if_not_exists: bool = False):
+    from ..my_mb import MB
+    from loguru import logger
+
+    mb = MB()
+    all_users_by_email = {
+        user["email"]: user for user in mb.get_all_users(all=True)
+    }
+    all_groups_by_name = {x["name"]: x for x in mb.get_all_groups()}
+
+    # Get information for this object
+    logger.info("Getting Metabase object information")
+    object_type, object_id = mb.get_object_info_from_url(url)
+    collection_id: int | None = None
+    collection_location: str | None = None
+    match (object_type):
+        case "question":
+            question = mb.get_question(object_id)
+            collection_id = int(question["collection"]["id"])
+            collection_location = question["collection"]["location"] + str(
+                question["collection"]["id"]
+            )
+        case "dashboard":
+            dashboard = mb.get_dashboard(object_id)
+            collection_id = int(dashboard["collection"]["id"])
+            collection_location = dashboard["collection"]["location"] + str(
+                dashboard["collection"]["id"]
+            )
+        case "collection":
+            collection = mb.get_collection(object_id)
+            collection_id = object_id
+            collection_location = collection["location"] + str(
+                collection["collection"]["id"]
+            )
+        case _:
+            raise ValueError(
+                f"Unknown object type {object_type} from {url}"
+            )
+    logger.info(
+        f'Object found: type "{object_type}", ID {object_id}, collection ID {collection_id}'
+    )
+
+    # Get group info that this collection should be granted to
+    logger.info(f"Getting group information for the object: {collection_location}")
+    group_name = mb.decode_collection_location_to_group(collection_location)
+    if group_name not in all_groups_by_name:
+        # If group not exists, create it and immediately grant readonly access to the collectiond
+        mb.create_group(group_name)
+        all_groups_by_name = {x["name"]: x for x in mb.get_all_groups()}
+        group_id = int(all_groups_by_name[group_name]["id"])
+        mb.grant_group_to_collection(group_id, collection_id)
+    else:
+        group_id = int(all_groups_by_name[group_name]["id"])
+    logger.info(f"Group found: [{group_id}] {group_name}")
+
+    # Get user informations, create if not exists
+    logger.info(f"Getting information from {len(emails)} users")
+    users = set()
+    created_users = 0
+    not_found_emails = []
+    for email in emails:
+        if email not in all_users_by_email:
+            if create_user_if_not_exists:
+                logger.info(f"Creating user {email}")
+                email_name, email_domain = email.split("@", 1)
+                mb.create_user(
+                    first_name=email_name,
+                    last_name=email_domain,
+                    email=email,
+                    group_ids=[1],  # Add to 'All Users' group
+                )
+                # all_users_by_email = {
+                #     user["email"]: user for user in mb.get_all_users(all=True)
+                # }
+                created_users += 1
+            else:
+                not_found_emails.append(email)
+    if not_found_emails:
+        raise ValueError(f"Users not found: {', '.join(not_found_emails)}")
+
+    # Re-fetch all users if there are new users created
+    if created_users:
+        logger.info("Users created, re-fetching all users")
+        all_users_by_email = {
+            user["email"]: user for user in mb.get_all_users(all=True)
+        }
+
+    # Grant access
+    logger.info(
+        f"Granting access to group [{group_id}] {group_name} for {len(emails)} users"
+    )
+    for email in emails:
+        user = all_users_by_email[email]
+        if (
+            not user["is_active"]
+        ) and create_user_if_not_exists:  # Reactivate user if disabled
+            logger.info(f"Reactivating user {user['id']}")
+            mb.enable_user(user["id"])
+
+        user_id = int(user["id"])
+        user_email = user["email"]
+        if group_id in user["group_ids"]:
+            # Skip if user already in the group because it will cause 500 error on Metabase later (it tries to insert the permissions to its DB and got duplicate key error)
+            logger.info(f"User {user_id} already in group {group_id}, skipping")
+            continue
+        users.add((user_id, user_email))
+    logger.info(
+        f"Users to be granted: {', '.join([f'[{user_id}] {user_email}' for user_id, user_email in users])}"
+    )
+
+    # Assign all user to the group
+    for user_id, user_email in users:
+        logger.info(f"Assigning user {user_id} to group {group_id}")
+        mb.grant_user_to_group(user_id, group_id)
+    logger.info("All users assigned to the group")
+
+
+def _copy_permissions(src_email: str, dst_emails: list[str]):
+    from ..my_mb import MB
+
+    mb = MB()
+    for dst_email in dst_emails:
+        mb.mirror_permission(src_email, dst_email)
+
+
+def _reset_password(emails: list[str]):
+    from ..my_mb import MB
+
+    mb = MB()
+    for email in emails:
+        mb.reset_password(email)
+
+
+def _disable_user(emails: list[str]):
+    from ..my_mb import MB
+
+    mb = MB()
+    for email in emails:
+        mb.disable_user(email)

{rdxz2_utill-0.1.3 → rdxz2_utill-0.1.4}/src/utill/cmd/_pg.py

@@ -1,7 +1,9 @@
-def _pg_to_pg(src_profile: str, src_table: str, dst_profile: str, dst_table: str, columns: str):
+def _pg_to_pg(
+    src_profile: str, src_table: str, dst_profile: str, dst_table: str, columns: str
+):
     from ..my_pg import PG
 
-    columns = ','.join([f"{x}" for x in columns.split(',')]) if columns != '*' else None
+    columns = ",".join([f"{x}" for x in columns.split(",")]) if columns != "*" else None
     pg_src = PG(src_profile)
     pg_dst = PG(dst_profile)
 

rdxz2_utill-0.1.4/src/utill/cmd/utill.py

@@ -0,0 +1,232 @@
+import click
+
+
+@click.group()
+def main():
+    pass
+
+
+# MARK: Conf
+
+
+@main.group("conf", help="Configure this library")
+def main__conf():
+    pass
+
+
+@main__conf.command("init", help="Initialize env files")
+@click.argument("mode", type=click.Choice(["google-cloud", "postgresql", "metabase"]))
+def main__conf__init(**kwargs):
+    from ._conf import _init
+
+    _init(**kwargs)
+
+
+@main__conf.command("list", help="List all configs")
+@click.option(
+    "-m",
+    "module",
+    type=click.Choice(["postgresql", "metabase"]),
+    help="List config for a specific modules",
+)
+def main__conf__list(**kwargs):
+    from ._conf import _list
+
+    _list(**kwargs)
+
+
+@main__conf.command("set", help="Set configuration variables")
+@click.option(
+    "-e", "vars", type=(str, str), multiple=True, required=True, help="Variables -> K V"
+)
+def main__conf__set(**kwargs):
+    from ._conf import _set
+
+    _set(**kwargs)
+
+
+# MARK: Metabase
+
+
+@main.group("mb", help="Metabase utility commands")
+def main__mb():
+    pass
+
+
+@main__mb.command(
+    "jl-grant", help="Grant access to Metabase questions/collections/dashboards"
+)
+@click.option("-u", "--email", "emails", type=str, multiple=True, help="User emails")
+@click.option("-l", "--url", "urls", type=str, multiple=True, help="URLs")
+@click.option(
+    "-c",
+    "create_user_if_not_exists",
+    type=bool,
+    is_flag=True,
+    help="Create user if not exists, also reactivate user if it's already exists default: False",
+)
+def main__mb__grant(**kwargs):
+    from ._mb import _jl_grant
+
+    _jl_grant(**kwargs)
+
+
+@main__mb.command(
+    "copy-permissions", help="Copy all permissions from one user to another"
+)
+@click.argument("src_email", type=str)
+@click.argument("dst_emails", type=str, nargs=-1)
+def main__mb__copy_permissions(**kwargs):
+    from ._mb import _copy_permissions
+
+    _copy_permissions(**kwargs)
+
+
+@main__mb.command("reset-password", help="Reset Metabase user password")
+@click.option(
+    "-u",
+    "--email",
+    "emails",
+    type=str,
+    required=True,
+    multiple=True,
+    help="User emails",
+)
+def main__mb__reset_password(**kwargs):
+    from ._mb import _reset_password
+
+    _reset_password(**kwargs)
+
+
+@main__mb.command("disable-user", help="Disable Metabase user")
+@click.option(
+    "-u",
+    "--email",
+    "emails",
+    type=str,
+    required=True,
+    multiple=True,
+    help="User emails",
+)
+def main__mb__disable_user(**kwargs):
+    from ._mb import _disable_user
+
+    _disable_user(**kwargs)
+
+
+# MARK: PG
+
+
+@main.group("pg", help="PostgreSQL utility")
+def main__pg():
+    pass
+
+
+@main__pg.command("pg-to-pg", help="Copy table from one PG instance to another")
+@click.argument("src_profile", type=str)
+@click.argument("src_table", type=str)
+@click.argument("dst_profile", type=str)
+@click.argument("dst_table", type=str)
+@click.option("-c", "--columns", type=str, default="*", help="Columns to copy")
+def main__pg__pg_to_pg(**kwargs):
+    from ._pg import _pg_to_pg
+
+    _pg_to_pg(**kwargs)
+
+
+@main__pg.command("upload-csv", help="Upload CSV file into PG table")
+@click.argument("profile", type=str)
+@click.argument("src_filename", type=click.Path())
+@click.argument("dst_table", type=str)
+def main__pg__upload_csv(**kwargs):
+    from ._pg import _upload_csv
+
+    _upload_csv(**kwargs)
+
+
+# MARK: BQ
+
+
+@main.group("bq", help="BigQuery utility")
+def main__bq():
+    pass
+
+
+@main__bq.command("upload-csv", help="Upload CSV file into BQ table")
+@click.argument("src_filename", type=click.Path())
+@click.argument("dst_table_fqn", type=str)
+@click.option(
+    "-c",
+    "columns",
+    type=(str, str),
+    required=True,
+    multiple=True,
+    help="Columns -> Name DataType",
+)
+@click.option("--partition-col", "partition_col", type=str, help="Partition column")
+@click.option(
+    "--cluster-col", "cluster_cols", type=str, multiple=True, help="Cluster column(s)"
+)
+@click.option("--project", type=str, help="Billing project")
+def main__bq__upload_csv(**kwargs):
+    from ._bq import _upload_csv
+
+    _upload_csv(**kwargs)
+
+
+@main__bq.command("download-table", help="Download a BQ table into CSV file")
+@click.argument("src_table_fqn", type=str)
+@click.argument("dst_filename", type=str)
+@click.option("--project", type=str, help="Billing project")
+def main__bq__download_table(**kwargs):
+    from ._bq import _download_table
+
+    _download_table(**kwargs)
+
+
+# MARK: Encyrption
+
+
+@main.group("enc", help="Encryption utility")
+def main__enc():
+    pass
+
+
+@main__enc.command("encrypt", help="Encrypt a string / file")
+@click.argument("src", type=str)
+@click.option("-p", "password", type=str, required=True, help="The password")
+def main__enc__encrypt(**kwargs):
+    from ._enc import _encrypt
+
+    _encrypt(**kwargs)
+
+
+# MARK: Other utilities
+
+
+@main.command("random", help="Generate random string")
+@click.option("-l", "length", type=int, default=32, help="Length of the random string")
+@click.option(
+    "-a",
+    "alphanum",
+    is_flag=True,
+    default=False,
+    help="Use alphanumeric only (a-Z, 0-9)",
+)
+def main__random(**kwargs):
+    from ._main import _random
+
+    _random(**kwargs)
+
+
+@main.command("unique", help="Get unique values")
+@click.argument("strings", nargs=-1)
+@click.option("-s", "sort", type=bool, is_flag=True, help="Sort the output")
+def main__unique(**kwargs):
+    from ._main import _unique
+
+    _unique(**kwargs)
+
+
+if __name__ == "__main__":
+    main()