rdxz2-utill 1.0.0__py3-none-any.whl

rdxz2_utill-1.0.0.dist-info/METADATA

@@ -0,0 +1,168 @@
+Metadata-Version: 2.4
+Name: rdxz2-utill
+Version: 1.0.0
+Summary: A small example package
+Author-email: Richard Dharmawan <richard.dharmawan@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2024 Richard Dharmawan
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: click==8.1.8
+Requires-Dist: duckdb==1.1.3
+Requires-Dist: humanize==4.11.0
+Requires-Dist: loguru==0.7.3
+Requires-Dist: paramiko==3.5.0
+Requires-Dist: pydantic-settings==2.7.1
+Requires-Dist: pydantic==2.10.6
+Requires-Dist: requests==2.32.3
+Requires-Dist: sshtunnel==0.4.0
+Provides-Extra: google-cloud
+Requires-Dist: google-cloud-bigquery==3.29.0; extra == "google-cloud"
+Requires-Dist: google-cloud-storage==3.0.0; extra == "google-cloud"
+Provides-Extra: postgresql
+Requires-Dist: psycopg==3.2.2; extra == "postgresql"
+Provides-Extra: pdf
+Requires-Dist: PyPDF2==3.0.1; extra == "pdf"
+Dynamic: license-file
+
+# Using this library
+
+Installation
+
+```sh
+pip install utill
+```
+
+Usage syntax
+
+```py
+from utill.__MODULE__ import __OBJECT__
+```
+
+Example
+
+```py
+# Using the string module
+from utill.my_string import generate_random_string
+
+print(generate_random_string(16))
+```
+
+## Initial set up
+
+This package contains CLI command
+
+```sh
+utill conf init
+```
+
+# Additional extensions
+
+Syntax
+
+```sh
+pip install utill[__EXTENSION_NAME__]
+```
+
+Extension list:
+
+- google-cloud
+- postgresql
+- pdf
+
+# Per module usages
+
+## my_bq
+
+Executing a query
+
+```py
+from utill.my_bq import BQ
+
+# Initialize BigQuery client
+bq = BQ()
+
+# Execute a query, returns iterable QueryJob
+job = bq.execute_query('...')
+
+# Convert into list for quick data conversion
+results = list(job)
+
+# Iterate the results
+for row in job:
+    # Do anything with the row
+```
+
+Uploading CSV file into BigQuery table
+
+```py
+from utill.my_bq import BQ, Dtype, LoadStrategy
+
+# Initialize BigQuery client
+bq = BQ()
+
+# Load the data
+filename = '/path/to/file.csv'  # Your local CSV file location
+bq_table_fqn = 'project.dataset.table'  # An FQN (fully qualified name) of a BigQuery table to export
+columns = {
+    'col1': Dtype.INT64,
+    'col2': Dtype.STRING,
+    'col3': Dtype.DATE,
+    ...
+}
+partition_col = 'col3'  # Optional, for performance and cost optimization
+cluster_cols = ['col1']  # Optional, for performance and cost optimization
+load_strategy = LoadStrategy.APPEND  # Optional, default to APPEND
+bq.upload_csv(filename, bq_table_fqn, columns, partition_col, cluster_cols, load_strategy)
+```
+
+Exporting query into CSV
+
+```py
+from utill.my_bq import BQ
+
+# Initialize BigQuery client
+bq = BQ()
+
+query = 'SELECT * FROM `project.dataset.table`'  # The query to export
+filename = '/path/to/file.csv'  # Destination CSV file location
+bq.download_csv(query, filename)
+```
+
+Exporting table into XLSX
+
+```py
+from utill.my_bq import BQ
+
+# Initialize BigQuery client
+bq = BQ()
+
+bq_table_fqn = 'project.dataset.table'  # An FQN (fully qualified name) of a BigQuery table to export
+filename = '/path/to/file.xlsx'  # Destination XLSX file location
+bq.download_xlsx(src_table_fqn, dst_filename)
+```

rdxz2_utill-1.0.0.dist-info/RECORD

@@ -0,0 +1,32 @@
+rdxz2_utill-1.0.0.dist-info/licenses/LICENSE,sha256=PF9CUvzP8XFYopEAzrMzSCovF7RdBdscPqJCDC6KjPc,1073
+utill/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+utill/my_bq.py,sha256=ZsjRbrbczGQ70CavXfyeAPaWT1OW5LyRFBxyusNZ0cc,14701
+utill/my_const.py,sha256=88dOqn6NPQ5-hfRqdkew5POoAIyO91XXOGvN76oNsdo,251
+utill/my_csv.py,sha256=76Q7IM7T_WYF5SybObN_MbkPqycirX4FnLd0DX3Kdyg,2742
+utill/my_datetime.py,sha256=KEZTplLk3tgVqqC3wClXFcsF_zo40fma_rtPg4kSJHc,2125
+utill/my_dict.py,sha256=jPaPfdn4WYpm0uIBPiYFinpHhx1jXpFVDJ9npmvxGZQ,391
+utill/my_encryption.py,sha256=SCF7PPur39cW4RHidsRhw-9BZP-ymUH-6LZ9nAHJDsY,2105
+utill/my_env.py,sha256=mREys72Ybg2p9p2s7ApOt0s_6F5-qxR8FyYEcSJ8pmU,2093
+utill/my_file.py,sha256=H3QmIOwubQCUMoOuk7jwf6AnqsljWZIuM7OjelyZby4,1865
+utill/my_gcs.py,sha256=VY2CXQbzBUhX-HunvAZ_y4E19eiuZ1b3TF33bUkJfp4,3953
+utill/my_input.py,sha256=OyKLoutXpwISReltuL_Gw2oojv16tYWJqQpqabBOQx4,350
+utill/my_json.py,sha256=WgW6mavGhfs4h1N5XbhsDnRk2dbh_ttJWdJUj4iWDN4,1473
+utill/my_mb.py,sha256=3_A5kXHgnkxGbd38vK5t5MfFcj84lohjS7C2OtlSo30,14841
+utill/my_pg.py,sha256=a2s5osWDXo4-u03Pgfydb1Lxg2XmhTNSihmTAyxrnzw,5832
+utill/my_queue.py,sha256=hINP4_yjmboSjHgo1J3CtPm2X9SE3HfczyED3ip7nfk,1930
+utill/my_string.py,sha256=pINYFR1ligTyVZYzV8P_FolCsZQwYE1jaFNTuQ3XS_8,833
+utill/my_style.py,sha256=Wy6j4WL9RgGeX6cS9hhlOrufc9UC4UPTQ5UJa0ZJ3Yo,900
+utill/my_tunnel.py,sha256=uCpGtiG8AcRYiaN7rLnTulsZI4iFTRM8EHxwyAAfDrE,1292
+utill/my_xlsx.py,sha256=YcQRp6DC9girSS1fkUPVKsHspyQpr8JC8GymSSnRV-w,729
+utill/cmd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+utill/cmd/_bq.py,sha256=MQGLIv_WBUBl2tf18bfYrAszx0Koa5kdTW1c8A5HDDg,520
+utill/cmd/_conf.py,sha256=74smxrq9Nbq6XIaTWsjfPkD-fKERRwWhk0YAhcPPahU,1930
+utill/cmd/_enc.py,sha256=DBy3Iwa5DTtww7lgHPRLEilrYPrWDG1vRv5PO-YzNO8,997
+utill/cmd/_main.py,sha256=UJ_XTIGDO9XPIypgHhS81SJQ_8qy8JOyw98Or0Nb2x8,273
+utill/cmd/_pg.py,sha256=RVxEiSifyIwMDYDM69vt6WSLdVDr1cMzY6r4T2PzNRA,492
+utill/cmd/utill.py,sha256=TlHfiwOUcK1m58PrRCjX9sARiPYZUsoTk-KOTCOz1vM,3558
+rdxz2_utill-1.0.0.dist-info/METADATA,sha256=EvXLTOlqccEEr_RLCoPmae95KibF1ySxpUy0ogtJby4,4487
+rdxz2_utill-1.0.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+rdxz2_utill-1.0.0.dist-info/entry_points.txt,sha256=9n5NWz5Wi9jDvYhB_81_4icgT5xABZ-QivHD8ibcafg,47
+rdxz2_utill-1.0.0.dist-info/top_level.txt,sha256=tuAYZoCsr02JYbpZj7I6fl1IIo53v3GG0uoj-_fINVk,6
+rdxz2_utill-1.0.0.dist-info/RECORD,,

rdxz2_utill-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (78.1.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

rdxz2_utill-1.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+utill = utill.cmd.utill:main

rdxz2_utill-1.0.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Richard Dharmawan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

rdxz2_utill-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+utill

utill/cmd/_bq.py

@@ -0,0 +1,12 @@
+def _upload_csv(src_filename: str, dst_table_fqn: str, cols: list[tuple[str, str]], partition_col: str = None, cluster_cols: list[str] = None, project: str = None):
+    from ..my_bq import BQ
+
+    bq = BQ(project)
+    bq.upload_csv(src_filename, dst_table_fqn, {col: dtype for col, dtype in cols}, partition_col, cluster_cols)
+
+
+def _download_table(src_table_fqn: str, dst_filename: str, project: str):
+    from ..my_bq import BQ
+
+    bq = BQ(project)
+    bq.download_csv(f'SELECT * FROM {src_table_fqn}', dst_filename)

utill/cmd/_conf.py

@@ -0,0 +1,62 @@
+def _init(mode: str):
+    from loguru import logger
+
+    from ..my_env import envs, init_pg_file, init_mb_file
+
+    match mode:
+        case 'google-cloud':
+            setattr(envs, 'GCP_PROJECT_ID', input('GCP_PROJECT_ID: '))
+            setattr(envs, 'GCS_BUCKET', input('GCS_BUCKET: '))
+            envs.write()
+            logger.info('Google cloud configuration initialized')
+        case 'postgresql':
+            init_pg_file()
+            logger.info('PostgreSQL connection file initialized')
+        case 'metabase':
+            init_mb_file()
+            logger.info('Metabase connection file initialized')
+        case _:
+            logger.warning(f'Mode \'{mode}\' not recognized')
+
+
+def _list(module: str = None):
+    import json
+    import os
+
+    from loguru import logger
+
+    from ..my_env import envs, PG_FILENAME, MB_FILENAME
+    from ..my_string import mask
+
+    match module:
+        case 'postgresql':
+            if not os.path.exists(PG_FILENAME):
+                logger.error('PostgreSQL configuraiton not exists')
+                return
+
+            config: dict = json.loads(open(PG_FILENAME, 'r').read())
+            for k, v in config.items():
+                print(k)
+                for k2, v2 in v.items():
+                    print(f'\t{k2} = {mask(str(v2)) if k2 in ("password", ) else v2}')
+
+        case 'metabase':
+            if not os.path.exists(MB_FILENAME):
+                logger.error('Metabase configuration not exists')
+                return
+
+            config: dict = json.loads(open(MB_FILENAME, 'r').read())
+            for k, v in config.items():
+                print(f'{k} = {mask(str(v)) if k in ("api_key", ) else v}')
+        case _:
+            for env in envs.model_fields:
+                print(f'{env} = {getattr(envs, env)}')
+
+
+def _set(vars: list[tuple[str, str]]):
+    from ..my_env import envs
+
+    for k, v in vars:
+        setattr(envs, k, v)
+
+    envs.write()

utill/cmd/_enc.py

@@ -0,0 +1,26 @@
+def _encrypt(src: str, password: str, output: str = None, force: bool = False):
+    from pathlib import Path
+
+    # Get the password string from file, if exists
+    path_password = Path(password).expanduser()
+    if path_password.exists():
+        if not path_password.is_file():
+            raise ValueError(f'Password path is not a file: {password}')
+        else:
+            password = open(path_password.as_posix(), 'r').read().strip()
+
+    path_src = Path(src).expanduser()
+    if path_src.exists():
+        if path_src.is_dir():
+            raise ValueError(f'Source file is a directory: {src}')
+
+        # Do encryption
+        from ..my_encryption import encrypt_file
+        if output:
+            encrypt_file(path_src.as_posix(), password, dst_filename=output, overwrite=force)
+        else:
+            print(encrypt_file(path_src.as_posix(), password))
+    else:
+        # Do encryption
+        from ..my_encryption import encrypt_string
+        print(encrypt_string(src, password))

utill/cmd/_main.py

@@ -0,0 +1,8 @@
+def _random(length: int, alphanum: bool):
+    from ..my_string import generate_random_string
+
+    print(generate_random_string(length, alphanum))
+
+
+def _unique(strings: list[str], sort: bool = True):
+    [print(x) for x in (sorted(set(strings)) if sort else set(strings))]

utill/cmd/_pg.py

@@ -0,0 +1,15 @@
+def _pg_to_pg(src_profile: str, src_table: str, dst_profile: str, dst_table: str, columns: str):
+    from ..my_pg import PG
+
+    columns = ','.join([f"{x}" for x in columns.split(',')]) if columns != '*' else None
+    pg_src = PG(src_profile)
+    pg_dst = PG(dst_profile)
+
+    pg_src.pg_to_pg(pg_dst, src_table, dst_table, columns)
+
+
+def _upload_csv(profile: str, src_filename: str, dst_table: str):
+    from ..my_pg import PG
+
+    pg = PG(profile)
+    pg.upload_csv(src_filename, dst_table)

utill/cmd/utill.py

@@ -0,0 +1,90 @@
+import click
+
+
+@click.group()
+def main(): pass
+
+
+# Conf
+
+
+@main.group('conf', help='Configure this library')
+def main__conf(): pass
+@main__conf.command('init', help='Initialize env files')
+@click.argument('mode', type=click.Choice(['google-cloud', 'postgresql', 'metabase']))
+def main__conf__init(**kwargs): from ._conf import _init; _init(**kwargs)
+@main__conf.command('list', help='List all configs')
+@click.option('-m', 'module', type=click.Choice(['postgresql', 'metabase']), help='List config for a specific modules')
+def main__conf__list(**kwargs): from ._conf import _list; _list(**kwargs)
+@main__conf.command('set', help='Set configuration variables')
+@click.option('-e', 'vars', type=(str, str), multiple=True, required=True, help='Variables -> K V')
+def main__conf__set(**kwargs): from ._conf import _set; _set(**kwargs)
+
+
+# PG
+
+
+@main.group('pg', help='PostgreSQL utility')
+def main__pg(): pass
+@main__pg.command('pg-to-pg', help='Copy table from one PG instance to another')
+@click.argument('src_profile', type=str)
+@click.argument('src_table', type=str)
+@click.argument('dst_profile', type=str)
+@click.argument('dst_table', type=str)
+@click.option('-c', '--columns', type=str, default='*', help='Columns to copy')
+def main__pg__pg_to_pg(**kwargs): from ._pg import _pg_to_pg; _pg_to_pg(**kwargs)
+@main__pg.command('upload-csv', help='Upload CSV file into PG table')
+@click.argument('profile', type=str)
+@click.argument('src_filename', type=click.Path())
+@click.argument('dst_table', type=str)
+def main__pg__upload_csv(**kwargs): from ._pg import _upload_csv; _upload_csv(**kwargs)
+
+
+# BQ
+
+
+@main.group('bq', help='BigQuery utility')
+def main__bq(): pass
+@main__bq.command('upload-csv', help='Upload CSV file into BQ table')
+@click.argument('src_filename', type=click.Path())
+@click.argument('dst_table_fqn', type=str)
+@click.option('-c', 'columns', type=(str, str), required=True, multiple=True, help='Columns -> Name DataType')
+@click.option('--partition-col', 'partition_col', type=str, help='Partition column')
+@click.option('--cluster-col', 'cluster_cols', type=str, multiple=True, help='Cluster column(s)')
+@click.option('--project', type=str, help='Billing project')
+def main__bq__upload_csv(**kwargs): from ._bq import _upload_csv; _upload_csv(**kwargs)
+@main__bq.command('download-table', help='Download a BQ table into CSV file')
+@click.argument('src_table_fqn', type=str)
+@click.argument('dst_filename', type=str)
+@click.option('--project', type=str, help='Billing project')
+def main__bq__download_table(**kwargs): from ._bq import _download_table; _download_table(**kwargs)
+
+
+# Encyrption
+
+
+@main.group('enc', help='Encryption utility')
+def main__enc(): pass
+@main__enc.command('encrypt', help='Encrypt a string / file')
+@click.argument('src', type=str)
+@click.option('-p', 'password', type=str, required=True, help='The password')
+def main__enc__encrypt(**kwargs): from ._enc import _encrypt; _encrypt(**kwargs)
+
+
+# Other utilities
+
+
+@main.command('random', help='Generate random string')
+@click.option('-l', 'length', type=int, default=32, help='Length of the random string')
+@click.option('-a', 'alphanum', is_flag=True, default=False, help='Use alphanumeric only (a-Z, 0-9)')
+def main__random(**kwargs): from ._main import _random; _random(**kwargs)
+
+
+@main.command('unique', help='Get unique values')
+@click.argument('strings', nargs=-1)
+@click.option('-s', 'sort', type=bool, is_flag=True, help='Sort the output')
+def main__unique(**kwargs): from ._main import _unique; _unique(**kwargs)
+
+
+if __name__ == '__main__':
+    main()