rdxz2-utill 0.0.11__py3-none-any.whl → 0.1.0__py3-none-any.whl

{rdxz2_utill-0.0.11.dist-info → rdxz2_utill-0.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rdxz2-utill
-Version: 0.0.11
+Version: 0.1.0
 Summary: Your daily Python utility
 Author-email: Richard Dharmawan <richard.dharmawan@gmail.com>
 License: MIT License

{rdxz2_utill-0.0.11.dist-info → rdxz2_utill-0.1.0.dist-info}/RECORD

@@ -1,35 +1,35 @@
-rdxz2_utill-0.0.11.dist-info/licenses/LICENSE,sha256=PF9CUvzP8XFYopEAzrMzSCovF7RdBdscPqJCDC6KjPc,1073
+rdxz2_utill-0.1.0.dist-info/licenses/LICENSE,sha256=PF9CUvzP8XFYopEAzrMzSCovF7RdBdscPqJCDC6KjPc,1073
 utill/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-utill/my_bq.py,sha256=eWHnCz-tPHLtK4Ac9uNRLw_iyoEvOEy-mk9OMxyWZGc,14771
+utill/my_bq.py,sha256=1BKjYa05yfOBInm245373lfi4sbOZKybYXdYmON2npM,24985
 utill/my_compare.py,sha256=619QbVk3GihWxen95yVnivKHkah8GgPTLGiSkgHxykw,886
 utill/my_const.py,sha256=88dOqn6NPQ5-hfRqdkew5POoAIyO91XXOGvN76oNsdo,251
-utill/my_csv.py,sha256=svgu93R0pP7UW0B58eJMi0vuJnYhqMtafzCsTIk4yUU,2781
-utill/my_datetime.py,sha256=KEZTplLk3tgVqqC3wClXFcsF_zo40fma_rtPg4kSJHc,2125
+utill/my_csv.py,sha256=AT5sAbAlYqnAmNgQMTSqEueRXM4D42yNPb5C3Hedy6c,2921
+utill/my_datetime.py,sha256=8AUO9l_MSzdthRsgASuyGZpvjgpoQb9Lowt4goHjyqw,2129
 utill/my_dict.py,sha256=jPaPfdn4WYpm0uIBPiYFinpHhx1jXpFVDJ9npmvxGZQ,391
 utill/my_encryption.py,sha256=SCF7PPur39cW4RHidsRhw-9BZP-ymUH-6LZ9nAHJDsY,2105
-utill/my_env.py,sha256=mREys72Ybg2p9p2s7ApOt0s_6F5-qxR8FyYEcSJ8pmU,2093
-utill/my_file.py,sha256=H2V8qGSCwnztBKiLYA38-4KUaGFQhznJz86cdilLtAE,1879
-utill/my_gcs.py,sha256=KUx89rZx2-dq-GV1LbbvbZ79Qr9NznjG1Zipop4hMZE,4216
+utill/my_env.py,sha256=E7XW3fuhxbDlFqmLPHrziJJZVRogzGh6rfQdyNV49f8,2130
+utill/my_file.py,sha256=-b6_dGDDBdS228kgwTYpmIa3vxW1c1TtWrLdzdlHjKY,1873
+utill/my_gcs.py,sha256=VBJ8lsJ-fHr_BzMoSuT5JUrvxidGyMc2VNtE6Um1T_M,3060
 utill/my_input.py,sha256=OyKLoutXpwISReltuL_Gw2oojv16tYWJqQpqabBOQx4,350
 utill/my_json.py,sha256=WgW6mavGhfs4h1N5XbhsDnRk2dbh_ttJWdJUj4iWDN4,1473
 utill/my_mb.py,sha256=IyrySs92TqtjBUvPMeUN3P2kRK8EttTFRPZsv5Cr-xw,15090
 utill/my_pg.py,sha256=J9USygc-oug4w7AkBacA9x043jHZrDfQPGFEqXavZAY,6799
-utill/my_queue.py,sha256=hINP4_yjmboSjHgo1J3CtPm2X9SE3HfczyED3ip7nfk,1930
+utill/my_queue.py,sha256=Qf3Nm_ZRoVD34oAoym8A9hoH9Y27kUHeWLhylAUj5Q4,4749
 utill/my_string.py,sha256=pINYFR1ligTyVZYzV8P_FolCsZQwYE1jaFNTuQ3XS_8,833
 utill/my_style.py,sha256=Wy6j4WL9RgGeX6cS9hhlOrufc9UC4UPTQ5UJa0ZJ3Yo,900
 utill/my_tunnel.py,sha256=uCpGtiG8AcRYiaN7rLnTulsZI4iFTRM8EHxwyAAfDrE,1292
 utill/my_xlsx.py,sha256=YcQRp6DC9girSS1fkUPVKsHspyQpr8JC8GymSSnRV-w,729
 utill/cmd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 utill/cmd/_bq.py,sha256=MQGLIv_WBUBl2tf18bfYrAszx0Koa5kdTW1c8A5HDDg,520
-utill/cmd/_conf.py,sha256=hNdkApzRhPloRSe4RyxbWuLoyeqkK7Yx9g44kcvKOEM,1800
+utill/cmd/_conf.py,sha256=DKl3IVVLp6-5P43tvh6bYmHR5rOL9XnKVuQ7kQJtzrc,1863
 utill/cmd/_enc.py,sha256=DBy3Iwa5DTtww7lgHPRLEilrYPrWDG1vRv5PO-YzNO8,997
 utill/cmd/_main.py,sha256=UJ_XTIGDO9XPIypgHhS81SJQ_8qy8JOyw98Or0Nb2x8,273
 utill/cmd/_pg.py,sha256=RVxEiSifyIwMDYDM69vt6WSLdVDr1cMzY6r4T2PzNRA,492
 utill/cmd/utill.py,sha256=TlHfiwOUcK1m58PrRCjX9sARiPYZUsoTk-KOTCOz1vM,3558
 utill/templates/mb.json,sha256=M46ZHSaSh4rbD_KGUViGr2B2ZV8_PC-O5Evqi35JK5g,59
 utill/templates/pg.json,sha256=LkJt0VV3zcyt7Tpn6gulsoVQgUc-9uImXOStvzu8cdU,271
-rdxz2_utill-0.0.11.dist-info/METADATA,sha256=ZGCUekaj1Zr5C8zjKwLBL6Q7tf1oPFEzuwS29Dsr7h0,4402
-rdxz2_utill-0.0.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-rdxz2_utill-0.0.11.dist-info/entry_points.txt,sha256=9n5NWz5Wi9jDvYhB_81_4icgT5xABZ-QivHD8ibcafg,47
-rdxz2_utill-0.0.11.dist-info/top_level.txt,sha256=tuAYZoCsr02JYbpZj7I6fl1IIo53v3GG0uoj-_fINVk,6
-rdxz2_utill-0.0.11.dist-info/RECORD,,
+rdxz2_utill-0.1.0.dist-info/METADATA,sha256=iB4cHw4zIQnP_2DHvIkBpwKxR5s32RVXr9xyJiY-GX4,4401
+rdxz2_utill-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+rdxz2_utill-0.1.0.dist-info/entry_points.txt,sha256=9n5NWz5Wi9jDvYhB_81_4icgT5xABZ-QivHD8ibcafg,47
+rdxz2_utill-0.1.0.dist-info/top_level.txt,sha256=tuAYZoCsr02JYbpZj7I6fl1IIo53v3GG0uoj-_fINVk,6
+rdxz2_utill-0.1.0.dist-info/RECORD,,

utill/cmd/_conf.py

@@ -6,6 +6,7 @@ def _init(mode: str):
     match mode:
         case 'google-cloud':
             setattr(envs, 'GCP_PROJECT_ID', input('GCP_PROJECT_ID: '))
+            setattr(envs, 'GCP_REGION', input('GCP_REGION: '))
             setattr(envs, 'GCS_BUCKET', input('GCS_BUCKET: '))
             envs.write()
             logger.info('Google cloud configuration initialized')

utill/my_bq.py

@@ -1,30 +1,46 @@
-from .my_const import ByteSize
-from .my_csv import read_header, combine as compress
-from .my_datetime import current_datetime_str
-from .my_env import envs
-from .my_gcs import GCS
-from .my_queue import ThreadingQ
-from .my_string import replace_nonnumeric
-from .my_xlsx import csv_to_xlsx
-from enum import Enum
-from google.cloud import bigquery, storage
+from . import my_csv
+from . import my_datetime
+from . import my_env
+from . import my_gcs
+from . import my_queue
+from . import my_string
+from . import my_xlsx
+from enum import StrEnum, Enum, auto
+from google.cloud import bigquery
+from google.cloud.exceptions import NotFound
+from humanize import precisedelta, naturalsize
 from loguru import logger
-from textwrap import dedent
 import csv
-import humanize
+import datetime
 import math
 import os
+import shutil
+import textwrap
+import time
 
-MAP__PYTHON_DTYPE__BQ_DTYPE = {
+PY_DATA_TYPE__BQ_DATA_TYPE = {
     int: 'INTEGER',
     str: 'STRING',
     float: 'STRING',
 }
 
 
+class DataFileFormat(StrEnum):
+    CSV = 'CSV'
+    JSON = 'JSON'
+    AVRO = 'AVRO'
+    PARQUET = 'PARQUET'
+    ORC = 'ORC'
+
+
+class DataFileCompression(StrEnum):
+    GZIP = 'GZIP'
+    SNAPPY = 'SNAPPY'
+
+
 class LoadStrategy(Enum):
-    OVERWRITE = 1
-    APPEND = 2
+    OVERWRITE = auto()
+    APPEND = auto()
 
 
 class Dtype:
@@ -56,231 +72,415 @@ class Dtype:
 
 
 class BQ():
-    def __init__(self, project: str = None, service_account_filename: str = None):
-        self.project = project or envs.GCP_PROJECT_ID
+    def __init__(self, location: str | None = None, project_id: str = None):
+        if project_id is None and my_env.envs.GCP_PROJECT_ID is None:
+            logger.warning('Using ADC for BigQuery authentication')
 
-        if service_account_filename is not None:
-            self.client = bigquery.Client.from_service_account_json(service_account_filename)
-        else:
-            self.client = bigquery.Client(project=self.project)
+        # if location is None and my_env.envs.GCP_REGION is None:
+        #     raise ValueError('GCP region must be set in environment variables.')
 
+        self.client = bigquery.Client(project=project_id or my_env.envs.GCP_PROJECT_ID, location=location or my_env.envs.GCP_REGION)
         logger.debug(f'BQ client open, project: {self.client.project}')
 
-    def __enter__(self):
-        return self
+    # MARK: Query execution
+
+    def execute_query(
+        self,
+        query: str | list[str],
+        parameters: dict = {},
+        dry_run: bool = False,
+        temporary_table: bool = False,
+    ) -> bigquery.QueryJob:
+        # Reconstruct query, handle multiple queries in a single job
+        is_multi = isinstance(query, list)
+        queries = query if is_multi else [query]
+        queries = [textwrap.dedent(q).strip() for q in queries]
+        queries = [q if q.endswith(';') else q + ';' for q in queries]  # Append ';' character for each query
+        query = '\n'.join(queries)
+
+        # Evaluate parameter
+        query_parameters = []
+        for parameter, value in parameters.items():
+            is_array = isinstance(value, list)
+            value_type_py = type(value[0]) if is_array else type(value)
+            if value_type_py not in PY_DATA_TYPE__BQ_DATA_TYPE:
+                raise ValueError(f'Unsupported type for parameter {parameter}: {value_type_py}. Supported types are: {list(PY_DATA_TYPE__BQ_DATA_TYPE.keys())}')
 
-    def __exit__(self, exc_type, exc_value, exc_tb):
-        self.close_client()
+            value_type_bq = PY_DATA_TYPE__BQ_DATA_TYPE[value_type_py]
 
-    def execute_query(self, query: str | list[str], dry_run: bool = False, parameters: dict = {}) -> bigquery.QueryJob:
-        multi = type(query) == list
-        if multi:
-            query = '\n'.join([x if str(x).strip().endswith(';') else x + ';' for x in query if x])
-        else:
-            query = query.strip()
+            # Handle data type conversions
+            if value_type_py == datetime.date:
+                value = [v.strftime('%Y-%m-%d') for v in value] if is_array else value.strftime('%Y-%m-%d')
 
-        # Build paramters
-        query_parameters = []
-        for parameter, value in parameters.items():
-            if type(value) == list:
-                query_parameters.append(bigquery.ArrayQueryParameter(parameter, MAP__PYTHON_DTYPE__BQ_DTYPE[type(value[0])], value))
+            if is_array:
+                query_parameters.append(bigquery.ArrayQueryParameter(parameter, value_type_bq, value))
             else:
-                query_parameters.append(bigquery.ScalarQueryParameter(parameter, MAP__PYTHON_DTYPE__BQ_DTYPE[type(value)], value))
+                query_parameters.append(bigquery.ScalarQueryParameter(parameter, value_type_bq, value))
 
         logger.debug(f'🔎 Query:\n{query}')
         query_job_config = bigquery.QueryJobConfig(dry_run=dry_run, query_parameters=query_parameters)
+        if temporary_table:
+            query_job_config.destination = None
+        t = time.time()
         query_job = self.client.query(query, job_config=query_job_config)
+        logger.info(f'Job tracking: https://console.cloud.google.com/bigquery?project={self.client.project}&j=bq:{self.client.location}:{query_job.job_id}&page=queryresults') if not dry_run else None
+        query_job.result()  # Wait for the job to complete
+        elapsed = precisedelta(datetime.timedelta(seconds=time.time() - t))
 
-        if not multi:
-            logger.debug(f'[Job ID] {query_job.job_id}, [Processed] {humanize.naturalsize(query_job.total_bytes_processed)}, [Billed] {humanize.naturalsize(query_job.total_bytes_billed)}, [Affected] {query_job.num_dml_affected_rows or 0} row(s)',)
+        if not is_multi:
+            logger.info(f'[Job ID] {query_job.job_id}, [Processed] {naturalsize(query_job.total_bytes_processed)}, [Billed] {naturalsize(query_job.total_bytes_billed)}, [Affected] {query_job.num_dml_affected_rows or 0} row(s), [Elapsed] {elapsed}',)
         else:
-            logger.debug(f'[Job ID] {query_job.job_id}')
+            logger.info(f'[Job ID] {query_job.job_id} [Elapsed] {elapsed}')
 
-            jobs: list[bigquery.QueryJob] = self.client.list_jobs(parent_job=query_job.job_id)
-            [logger.debug(f'[Script ID] {job.job_id}, [Processed] {humanize.naturalsize(job.total_bytes_processed)}, [Billed] {humanize.naturalsize(job.total_bytes_billed)}, [Affected] {job.num_dml_affected_rows or 0} row(s)',) for job in jobs]
+            jobs: list[bigquery.QueryJob] = list(self.client.list_jobs(parent_job=query_job.job_id))
+            [logger.info(f'[Script ID] {job.job_id}, [Processed] {naturalsize(job.total_bytes_processed)}, [Billed] {naturalsize(job.total_bytes_billed)}, [Affected] {job.num_dml_affected_rows or 0} row(s)',) for job in jobs]
 
         return query_job
 
-    def create_table(self, bq_table_fqn: str, schema: list[bigquery.SchemaField], partition_col: str, cluster_cols: list[str]):
-        table = bigquery.Table(bq_table_fqn, schema=schema)
-
-        if partition_col:
-            table.time_partitioning = bigquery.TimePartitioning(field=partition_col)
-            table.partitioning_type = 'DAY'
-
-        if cluster_cols:
-            table.clustering_fields = cluster_cols
-
-        bq_table = self.client.create_table(table)
-        logger.info(f'✅ Table created: {bq_table_fqn}')
-        return bq_table
+    # MARK: Table operations
+
+    def create_table(
+        self,
+        dst_table_fqn: str,
+        query: str,
+        query_parameters: dict = {},
+        *,
+        description: str | None = None,
+        schema: list[dict] | None = None,
+        partition_by: str | None = None,
+        clustering_fields: list[str] | None = None,
+        expiration_timestamp_utc: datetime.datetime | None = None,
+        require_partition_filter: bool = False,
+        replace: bool = False,
+    ):
+        self.raise_for_invalid_table_fqn(dst_table_fqn)
+
+        # Construct table options
+        logger.debug('Constructing table options ...')
+        table_options = []
+        if expiration_timestamp_utc:
+            table_options.append(f'  expiration_timestamp=\'{expiration_timestamp_utc.isoformat()}\'')
+        if partition_by and require_partition_filter:
+            table_options.append(f'  require_partition_filter=TRUE')
+        if description:
+            table_options.append(f'  description=\'{description}\'')
+
+        # Check if table exists
+        logger.debug('Checking if destination table exists ...')
+        dst_table_project_id, dst_table_dataset_id, dst_table_id = self.get_table_fqn_parts(dst_table_fqn)
+        table_exist = self.is_table_exists(project_id=dst_table_project_id, dataset_id=dst_table_dataset_id, table_id=dst_table_id)
+
+        # Construct beautiful query string
+        if table_exist and not replace:
+            logger.debug('Table exists, constructing INSERT query ...')
+            query_parts = [f'INSERT INTO `{dst_table_fqn}`']
+            if schema:
+                schema_str = ',\n'.join([column['name'] for column in schema])
+                query_parts.append(f'(\n{schema_str}\n)')
+            if table_options:
+                query_parts.append(f'OPTIONS (\n{",\n".join(table_options)}\n)')
+        else:
+            logger.debug('Table not exist, constructing CREATE TABLE query ...')
+            query_parts = [
+                f'CREATE OR REPLACE TABLE `{dst_table_fqn}`',
+            ]
+            if schema:
+                schema_str = ',\n'.join([f'  {column["name"]} {column["data_type"]}' for column in schema])
+                query_parts.append(f'(\n{schema_str}\n)')
+            if partition_by:
+                query_parts.append(f'PARTITION BY {partition_by}')
+            if clustering_fields:
+                clustering_fields_str = ', '.join([f'`{field}`' for field in clustering_fields])
+                query_parts.append(f'CLUSTER BY {clustering_fields_str}')
+            if table_options:
+                query_parts.append(f'OPTIONS (\n{",\n".join(table_options)}\n)')
+            query_parts.append('AS')
+        query_parts.append(textwrap.dedent(query).strip())
+
+        # Execute
+        logger.debug('Executing query ...')
+        query = '\n'.join(query_parts)
+        self.execute_query(query, parameters=query_parameters)
 
     def drop_table(self, bq_table_fqn: str):
-        self.client.delete_table(bq_table_fqn)
-        logger.info(f'✅ Table dropped: {bq_table_fqn}')
-
-    def load_data_into(self, bq_table_fqn: str, gcs_path: list[str] | str, cols: dict[str, Dtype], partition_col: str = None, cluster_cols: list[str] = None, overwrite: bool = False):
-        if type(gcs_path) == str:
-            gcs_path = [gcs_path]
-        gcs_path_str = ',\n'.join([f'        \'{x}\'' for x in gcs_path])
-
-        load_data_keyword = 'OVERWRITE' if overwrite else 'INTO'
-        cols_str = ',\n'.join([f'    `{x}` {y}' for x, y in cols.items()])
-        cluster_cols_str = ','.join([f'`{x}`' for x in cluster_cols]) if cluster_cols else None
-        query = dedent(
-            f'''
-            LOAD DATA {load_data_keyword} `{bq_table_fqn}` (
-            {cols_str}
-            )
-            {f"PARTITION BY `{partition_col}`" if partition_col is not None else "-- No partition column provided"}
-            {f"CLUSTER BY {cluster_cols_str}" if cluster_cols_str is not None else "-- No cluster column provided"}
-            FROM FILES(
-                skip_leading_rows=1,
-                allow_quoted_newlines=true,
-                format='csv',
-                compression='gzip',
-                uris = [
-            {gcs_path_str}
-                ]
-            );
-            '''
-        )
-
-        logger.debug(f'⌛ Load data into: {bq_table_fqn}')
-        query_job = self.execute_query(query)
-        logger.info(f'✅ Load data into: {bq_table_fqn}')
-        return query_job
-
-    def export_data(self, query: str, gcs_path: str, pre_query: str = None):
-        if '*' not in gcs_path:
-            raise ValueError('GCS path need to have a single \'*\' wildcard character')
-
-        query = dedent(
-            f'''
-            EXPORT DATA OPTIONS (
-                uri='{gcs_path}',
-                format='csv',
-                compression='gzip',
-                overwrite=true,
-                header=true,
-                field_delimiter=',')
-            AS (
-            {query}
-            ORDER BY 1
-            );
-            '''
+        logger.info(f'Dropping table: {bq_table_fqn} ...')
+        self.raise_for_invalid_table_fqn(bq_table_fqn)
+        self.client.delete_table(bq_table_fqn, not_found_ok=True)
+
+    # MARK: Table data
+
+    def load_data(
+        self,
+        src_gcs_uri: str,
+        dst_table_fqn: str,
+        *,
+        schema: list[dict] | None = None,
+        partition_by: str | None = None,
+        clustering_fields: list[str] | None = None,
+        field_delimiter: str = ',',
+        load_strategy: LoadStrategy = LoadStrategy.APPEND,
+        format: DataFileFormat = DataFileFormat.CSV,
+        compression=None,
+    ):
+
+        self.raise_for_invalid_table_fqn(dst_table_fqn)
+
+        logger.debug(f'Loading CSV from {src_gcs_uri} into {dst_table_fqn} ...')
+
+        # Construct LOAD options
+        logger.debug('Constructing LOAD options ...')
+        load_options = [  # https://cloud.google.com/bigquery/docs/reference/standard-sql/load-statements#load_option_list
+            f'  format=\'{format}\'',
+            f'  uris=[\'{src_gcs_uri}\']',
+        ]
+        if format == DataFileFormat.CSV:
+            load_options.append(f'  skip_leading_rows=1')
+            load_options.append(f'  field_delimiter=\'{field_delimiter}\'')
+            load_options.append(f'  allow_quoted_newlines=true')
+        if compression:
+            load_options.append(f'  compression=\'{compression}\'')
+        load_options_str = ',\n'.join(load_options)
+
+        # Construct beautiful query string
+        logger.debug('Constructing LOAD query ...')
+        schema_str = ',\n'.join([f'  {column["name"]} {column["data_type"]}' for column in schema])
+        query_parts = [f'LOAD DATA {"OVERWRITE" if load_strategy == LoadStrategy.OVERWRITE else "INTO"} `{dst_table_fqn}` (\n{schema_str}\n)']
+        if partition_by:
+            query_parts.append(f'PARTITION BY {partition_by}')
+        if clustering_fields:
+            clustering_fields_str = ', '.join([f'`{field}`' for field in clustering_fields])
+            query_parts.append(f'CLUSTER BY {clustering_fields_str}')
+        query_parts.append(f'FROM FILES (\n{load_options_str}\n)')
+        query = '\n'.join(query_parts)
+
+        # Execute
+        logger.debug('Executing query ...')
+        self.execute_query(query)
+
+    def export_data(
+        self,
+        query: str,
+        dst_gcs_uri: str,
+        *,
+        parameters: dict = {},
+        format: DataFileFormat = DataFileFormat.CSV,
+        compression: DataFileCompression | None = None,
+        header: bool = True,
+        delimiter: str = ',',
+    ):
+        logger.debug(f'Exporting query into {dst_gcs_uri} ...')
+
+        # GCS uri validation
+        if format == DataFileFormat.CSV and compression == DataFileCompression.GZIP and not dst_gcs_uri.endswith('.gz'):
+            raise ValueError('GCS path need to ends with .gz if using compression = GCSCompression.GZIP')
+        elif format == DataFileFormat.CSV and compression != DataFileCompression.GZIP and not dst_gcs_uri.endswith('.csv'):
+            raise ValueError('GCS path need to ends with .csv if using format = GCSExportFormat.CSV')
+        elif format == DataFileFormat.PARQUET and not dst_gcs_uri.endswith('.parquet'):
+            raise ValueError('GCS path need to ends with .parquet if using format = GCSExportFormat.PARQUET')
+
+        # Construct options
+        logger.debug('Constructing EXPORT options ...')
+        options = [
+            f'  uri=\'{dst_gcs_uri}\'',
+            f'  format=\'{format}\'',
+            f'  overwrite=TRUE',
+        ]
+        if format == DataFileFormat.CSV:
+            options.append(f'  field_delimiter=\'{delimiter}\'',)
+            if header:
+                options.append(f'  header={"true" if header else "false"}',)
+        if compression:
+            options.append(f'  compression=\'{compression}\'')
+        options_str = ',\n'.join(options)
+
+        # Construct beautiful query string
+        logger.debug('Constructing EXPORT query ...')
+        query = (
+            f'EXPORT DATA OPTIONS (\n'
+            f'{options_str}\n'
+            f')\n'
+            f'AS (\n'
+            f'{textwrap.dedent(query).strip()}\n'
+            f');'
         )
 
-        if pre_query:
-            query = [pre_query, query]
-
-        logger.debug(f'⌛ Export data into: {gcs_path}')
-        query_job = self.execute_query(query)
-        logger.info(f'✅ Exported data into: {gcs_path}')
-        return query_job
-
-    def upload_csv(self, src_filename: str, bq_table_fqn: str, cols: dict[str, Dtype], partition_col: str = None, cluster_cols: list[str] = None, load_strategy: LoadStrategy = LoadStrategy.APPEND):
-        # <<----- START: Validation
-
-        if load_strategy not in LoadStrategy:
-            raise ValueError('Invalid load strategy')
-
-        if not src_filename.endswith('.csv'):
-            raise ValueError('Please provide file path with .csv extension!')
-
-        if partition_col is not None:
-            if partition_col not in cols.keys():
-                raise ValueError(f'Partition \'{partition_col}\' not exists in columns!')
-        if cluster_cols is not None:
-            if cluster_cols not in cols.keys():
-                raise ValueError(f'Cluster \'{cluster_cols}\' not exists in columns!')
-
-        # Build list of columns with its datatypes
-        csv_cols = set(read_header(src_filename))
-        excessive_cols = set(cols.keys()) - set(csv_cols)
-        if excessive_cols:
-            raise ValueError(f'{len(excessive_cols)} columns not exists in CSV file: {", ".join(excessive_cols)}')
-        nonexistent_cols = set(csv_cols) - set(cols.keys())
-        if nonexistent_cols:
-            raise ValueError(f'{len(nonexistent_cols)} columns from CSV are missing: {", ".join(nonexistent_cols)}')
-
-        # END: Validation ----->>
-
-        # <<----- START: Upload to GCS
-
-        gcs = GCS(self.project)
-        tmp_dir = f'tmp/upload__{current_datetime_str()}'
-
-        # This will compress while splitting the compressed file to a certain bytes size because of GCS 4GB file limitation
-        # A single file can produce more than one compressed file in GCS
-        def producer(src_file: str):
-            for dst_file in compress(src_file, keep=True, max_size_bytes=ByteSize.GB * 3):
-                yield (dst_file, )
-
-        def consumer(dst_file: str):
-            remote_file_name = f'{tmp_dir}/{replace_nonnumeric(os.path.basename(dst_file), "_").lower()}.csv.gz'
-            logger.debug(f'Uploading {dst_file} to {remote_file_name}...')
-            blob = gcs.upload(dst_file, remote_file_name, mv=True)
-            return blob
-
-        blobs: list[storage.Blob]
-        _, blobs = ThreadingQ().add_producer(producer, src_filename).add_consumer(consumer).execute()
-
-        # END: Upload to GCS ----->>
-
-        # <<----- START: Load to BQ
-
+        # Execute
+        logger.debug('Executing query ...')
+        self.execute_query(query=query, parameters=parameters)
+
+    def upload_csv(
+        self,
+        src_filename: str,
+        dst_table_fqn: str,
+        schema: list[dict] | None = None,
+        gcs_bucket: str | None = None,
+        partition_by: str = None,
+        cluster_cols: list[str] = None,
+        compression: DataFileCompression | None = None,
+        load_strategy: LoadStrategy = LoadStrategy.APPEND,
+    ):
+        self.raise_for_invalid_table_fqn(dst_table_fqn)
+
+        if compression == DataFileCompression.GZIP and not src_filename.endswith('.gz'):
+            raise ValueError('Please provide file path with .gz extension if using compression = GZIP')
+        elif not src_filename.endswith('.csv'):
+            raise ValueError('Please provide file path with .csv extension')
+
+        # # <<----- START: Upload to GCS
+
+        # gcs = GCS(self.project_id)
+        # tmp_dir = f'tmp/upload__{current_datetime_str()}'
+
+        # # This will compress while splitting the compressed file to a certain bytes size because of GCS 4GB file limitation
+        # # A single file can produce more than one compressed file in GCS
+        # def producer(src_file: str):
+        #     for dst_file in compress(src_file,
+        # keep=True, max_size_bytes=ByteSize.GB * 3):
+        #         yield (dst_file, )
+
+        # def consumer(dst_file: str):
+        #     remote_file_name = f'{tmp_dir}/{replace_nonnumeric(os.path.basename(dst_file), "_").lower()}.csv.gz'
+        #     logger.debug(f'Uploading {dst_file} to {remote_file_name}...')
+        #     blob = gcs.upload(dst_file, remote_file_name, move=True)
+        #     return blob
+
+        # blobs: list[storage.Blob]
+        # _, blobs = ThreadingQ().add_producer(producer, src_filename).add_consumer(consumer).execute()
+
+        # # END: Upload to GCS ----->>
+
+        # Upload to GCS
+        # TODO: Re-implement the producer-consumer model to upload multiple files
+        gcs = my_gcs.GCS(bucket=gcs_bucket, project_id=self.client.project)
+        dst_blobpath = f'tmp/my_bq/{my_datetime.get_current_datetime_str()}/{my_string.replace_nonnumeric(os.path.basename(src_filename), "_").lower()}'
+        gcs.upload(src_filename, dst_blobpath)
+
+        # Load to BQ
         try:
-            gcs_filename_fqns = [f'gs://{blob.bucket.name}/{blob.name}' for blob in blobs]
-            match load_strategy:
-                case LoadStrategy.OVERWRITE:
-                    self.load_data_into(bq_table_fqn, gcs_filename_fqns, cols, partition_col=partition_col, cluster_cols=cluster_cols, overwrite=True)
-                case LoadStrategy.APPEND:
-                    self.load_data_into(bq_table_fqn, gcs_filename_fqns, cols, partition_col=partition_col, cluster_cols=cluster_cols)
-                case _:
-                    return ValueError(f'Load strategy not recognized: {load_strategy}')
-        except Exception as e:
-            raise e
+            self.load_data(dst_blobpath, dst_table_fqn, schema=schema, partition_by=partition_by, cluster_cols=cluster_cols, format=DataFileFormat.CSV, compression=compression, load_strategy=load_strategy)
+        except:
+            raise
         finally:
-            [GCS.remove_blob(blob) for blob in blobs]
-
-        # END: Load to BQ ----->>
-
-    def download_csv(self, query: str, dst_filepath: str, row_limit: int | None = None):
+            gcs.delete_blob(dst_blobpath)
+
+    def download_csv(
+        self,
+        query: str,
+        dst_filepath: str,
+        *,
+        gcs_bucket: str | None = None,
+        query_parameters: dict = {},
+        csv_row_limit: int | None = None,
+    ) -> str | list[str]:
         if not dst_filepath.endswith('.csv'):
             raise ValueError('Destination filename must ends with .csv')
 
-        dst_filepath = os.path.expanduser(dst_filepath)  # /path/to/file.csv
-
-        query_job = self.execute_query(query)
-        query_job_result = query_job.result()
-        row_count = 0
-        file_index = 1
-
-        # Stream-download-split result
-        def open_file(f):
-            if f:
-                f.close()
-            dst_filepath_part = f'{dst_filepath.removesuffix(".csv")}_{file_index:06}.csv' if row_limit else dst_filepath
-            logger.info(f'Writing into file: {dst_filepath_part} ...')
-            f = open(dst_filepath_part, 'w', newline='', encoding='utf-8')
-            writer = csv.writer(f)
-            writer.writerow([field.name for field in query_job_result.schema])  # Write header
-
-            return f, writer
-
-        f, writer = open_file(None)
-        for row in query_job_result:
-            writer.writerow(row)
-
-            if row_limit:
-                row_count += 1
-                if row_count >= row_limit:
-                    row_count = 0
-                    file_index += 1
-                    f, writer = open_file(f)
-        if f:
-            f.close()
+        # Init
+        gcs = my_gcs.GCS(bucket=gcs_bucket, project_id=self.client.project)
+
+        # Generic function to export-download-combine csv file from BQ->GCS->local
+        def _export_download_combine(query: str, dst_gcs_prefix: str, dst_filepath: str, query_parameters: dict = {}):
+            # Init tmp directory
+            tmp_dirname = f'/tmp/my_bq_{my_datetime.get_current_datetime_str()}'
+            if os.path.exists(tmp_dirname):
+                shutil.rmtree(tmp_dirname, ignore_errors=True)
+            os.makedirs(tmp_dirname, exist_ok=True)
+            logger.debug(f'Temporary directory created: {tmp_dirname}')
+
+            try:
+                # Export to GCS
+                dst_gcs_uri = f'gs://{gcs.bucket.name}/{dst_gcs_prefix}/*.csv.gz'
+                self.export_data(query, dst_gcs_uri, parameters=query_parameters, format=DataFileFormat.CSV, compression=DataFileCompression.GZIP)
+
+                # Download from GCS
+                local_tmp_filepaths = []
+                for tmp_blobs in gcs.list_blobs(dst_gcs_prefix):
+                    local_tmp_filepath = os.path.join(tmp_dirname, tmp_blobs.name.split('/')[-1])
+                    gcs.download(tmp_blobs, local_tmp_filepath, move=True)
+                    logger.debug(f'Downloaded {tmp_blobs.name} to {local_tmp_filepath}')
+                    local_tmp_filepaths.append(local_tmp_filepath)
+
+                # Combine downloaded files
+                my_csv.combine(local_tmp_filepaths, dst_filepath, gzip=True, delete=True)
+            except:
+                raise
+            finally:
+                shutil.rmtree(tmp_dirname, ignore_errors=True)  # Remove local folder
+                [gcs.delete_blob(blob_filepath) for blob_filepath in gcs.list_blobs(dst_gcs_prefix)]  # Remove temporary GCS files
+
+            logger.info(f'Export-download-combine done: {dst_filepath}')
+
+        # Limited csv rows
+        if csv_row_limit:
+            tmp_table_fqn: str | None = None
+            tmp_table_fqn_rn: str | None = None
+            try:
+                # Create temporary table
+                query_job = self.execute_query(query, temporary_table=True)
+                tmp_table_fqn = str(query_job.destination)
+                logger.debug(f'Create temp table: {tmp_table_fqn}')
+
+                # Create another temporary table for row numbering
+                query_job = self.execute_query(f'SELECT *, ROW_NUMBER() OVER() AS _rn FROM `{tmp_table_fqn}`', temporary_table=True)
+                tmp_table_fqn_rn = str(query_job.destination)
+                logger.debug(f'Create temp table (rn): {tmp_table_fqn_rn}')
+
+                # Process parts
+                count = list(self.execute_query(f'SELECT COUNT(1) FROM `{tmp_table_fqn_rn}`').result())[0][0]
+                parts = math.ceil(count / csv_row_limit)
+                logger.info(f'Total part: {count} / {csv_row_limit} = {parts}')
+                dst_filepaths = []
+                for part in range(parts):
+                    dst_filepath_part = f'{dst_filepath.removesuffix(".csv")}_{part + 1:06}.csv'
+                    _export_download_combine(
+                        f'SELECT * EXCEPT(_rn) FROM `{tmp_table_fqn_rn}` WHERE _rn BETWEEN {(part * csv_row_limit) + 1} AND {(part + 1) * csv_row_limit}',
+                        dst_gcs_prefix=gcs.build_tmp_dirpath(),
+                        dst_filepath=dst_filepath_part,
+                    )
+                    dst_filepaths.append(dst_filepath_part)
+                return dst_filepaths
+            except:
+                raise
+            finally:
+                # Drop temporary tables
+                if tmp_table_fqn_rn:
+                    self.drop_table(tmp_table_fqn_rn)
+                if tmp_table_fqn:
+                    self.drop_table(tmp_table_fqn)
+
+        # Unlimited csv rows
+        else:
+            _export_download_combine(query, gcs.build_tmp_dirpath(), dst_filepath, query_parameters=query_parameters)
+            return dst_filepath
+
+        # query_job_result = query_job.result()
+        # row_count = 0
+        # file_index = 1
+
+        # # Stream-download-split result
+        # def open_file(f):
+        #     if f:
+        #         f.close()
+        #     dst_filepath_part = f'{dst_filepath.removesuffix(".csv")}_{file_index:06}.csv' if row_limit else dst_filepath
+        #     logger.info(f'Writing into file: {dst_filepath_part} ...')
+        #     f = open(dst_filepath_part, 'w', newline='', encoding='utf-8')
+        #     writer = csv.writer(f)
+        #     writer.writerow([field.name for field in query_job_result.schema])  # Write header
+
+        #     return f, writer
+
+        # f, writer = open_file(None)
+        # for row in query_job_result:
+        #     writer.writerow(row)
+
+        #     if row_limit:
+        #         row_count += 1
+        #         if row_count >= row_limit:
+        #             row_count = 0
+        #             file_index += 1
+        #             f, writer = open_file(f)
+        # if f:
+        #     f.close()
 
     def download_xlsx(self, src_table_fqn: str, dst_filename: str, xlsx_row_limit: int = 950000):
         if not dst_filename.endswith('.xlsx'):
@@ -302,60 +502,94 @@ class BQ():
                 file_path_tmp = f'{dst_filename}_part{part + 1}'
                 file_path_tmp_csv = f'{file_path_tmp}.csv'
                 self.download_csv(f'SELECT * EXCEPT(_rn) FROM `{table_name_tmp}` WHERE _rn BETWEEN {(part * xlsx_row_limit) + 1} AND {(part + 1) * xlsx_row_limit}', f'{file_path_tmp}{os.sep}')
-                csv_to_xlsx(file_path_tmp_csv, f'{file_path_tmp}.xlsx')
+                my_xlsx.csv_to_xlsx(file_path_tmp_csv, f'{file_path_tmp}.xlsx')
                 os.remove(file_path_tmp_csv)
         except Exception as e:
             raise e
         finally:
             self.execute_query(f'DROP TABLE IF EXISTS `{table_name_tmp}`')
 
-    def copy_table(self, src_table_id: str, dst_table_id: str, drop: bool = False):
-        # Create or replace
-        self.client.delete_table(dst_table_id, not_found_ok=True)
-        self.client.copy_table(src_table_id, dst_table_id).result()
-        logger.debug(f'Table {src_table_id} copied to {dst_table_id}')
-
-        if drop:
-            self.client.delete_table(src_table_id)
-            logger.debug(f'Table {src_table_id} dropped')
-
-    def copy_view(self, src_view_id: str, dst_view_id: str, drop: bool = False):
-        src_project_id, src_dataset_id, _ = src_view_id.split('.')
-        dst_project_id, dst_dataset_id, _ = dst_view_id.split('.')
-
-        # Create or replace
-        src_view = self.client.get_table(src_view_id)
-        dst_view = bigquery.Table(dst_view_id)
-        dst_view.view_query = src_view.view_query.replace(f'{src_project_id}.{src_dataset_id}', f'{dst_project_id}.{dst_dataset_id}')
-        self.client.delete_table(dst_view, not_found_ok=True)
-        self.client.create_table(dst_view)
-        logger.debug(f'View {src_view_id} copied to {dst_view}')
-
-        if drop:
-            self.client.delete_table(src_view_id)
-            logger.debug(f'View {src_view_id} dropped')
-
-    def copy_routine(self, src_routine_id: str, dst_routine_id: str, drop: bool = False):
-        src_project_id, src_dataset_id, _ = src_routine_id.split('.')
-        dst_project_id, dst_dataset_id, _ = dst_routine_id.split('.')
-
-        # Create or replace
-        src_routine = self.client.get_routine(src_routine_id)
-        dst_routine = bigquery.Routine(dst_routine_id)
-        dst_routine.body = src_routine.body.replace(f'{src_project_id}.{src_dataset_id}', f'{dst_project_id}.{dst_dataset_id}')
-        dst_routine.type_ = src_routine.type_
-        dst_routine.description = src_routine.description
-        dst_routine.language = src_routine.language
-        dst_routine.arguments = src_routine.arguments
-        dst_routine.return_type = src_routine.return_type
-        self.client.delete_routine(dst_routine, not_found_ok=True)
-        self.client.create_routine(dst_routine)
-        logger.debug(f'Routine {src_routine_id} copied to {dst_routine_id}')
-
-        if drop:
-            self.client.delete_routine(src_routine_id)
-            logger.debug(f'Routine {src_routine_id} dropped')
-
-    def close_client(self):
+    # def copy_view(self, src_view_id: str, dst_view_id: str, drop: bool = False):
+    #     src_project_id, src_dataset_id, _ = src_view_id.split('.')
+    #     dst_project_id, dst_dataset_id, _ = dst_view_id.split('.')
+
+    #     # Create or replace
+    #     src_view = self.client.get_table(src_view_id)
+    #     dst_view = bigquery.Table(dst_view_id)
+    #     dst_view.view_query = src_view.view_query.replace(f'{src_project_id}.{src_dataset_id}', f'{dst_project_id}.{dst_dataset_id}')
+    #     self.client.delete_table(dst_view, not_found_ok=True)
+    #     self.client.create_table(dst_view)
+    #     logger.debug(f'View {src_view_id} copied to {dst_view}')
+
+    #     if drop:
+    #         self.client.delete_table(src_view_id)
+    #         logger.debug(f'View {src_view_id} dropped')
+
+    # def copy_routine(self, src_routine_id: str, dst_routine_id: str, drop: bool = False):
+    #     src_project_id, src_dataset_id, _ = src_routine_id.split('.')
+    #     dst_project_id, dst_dataset_id, _ = dst_routine_id.split('.')
+
+    #     # Create or replace
+    #     src_routine = self.client.get_routine(src_routine_id)
+    #     dst_routine = bigquery.Routine(dst_routine_id)
+    #     dst_routine.body = src_routine.body.replace(f'{src_project_id}.{src_dataset_id}', f'{dst_project_id}.{dst_dataset_id}')
+    #     dst_routine.type_ = src_routine.type_
+    #     dst_routine.description = src_routine.description
+    #     dst_routine.language = src_routine.language
+    #     dst_routine.arguments = src_routine.arguments
+    #     dst_routine.return_type = src_routine.return_type
+    #     self.client.delete_routine(dst_routine, not_found_ok=True)
+    #     self.client.create_routine(dst_routine)
+    #     logger.debug(f'Routine {src_routine_id} copied to {dst_routine_id}')
+
+    #     if drop:
+    #         self.client.delete_routine(src_routine_id)
+    #         logger.debug(f'Routine {src_routine_id} dropped')
+
+    # MARK: Utilities
+
+    @staticmethod
+    def get_table_fqn_parts(name: str | list[str]) -> list[str] | list[list[str]]:
+        """Get  fully qualified table name, following this format `<projectid>.<datasetid>.<tableid>`
+
+        Args:
+            name (str | list[str]): Input name (can be multiple)
+
+        Returns:
+            list[str] | list[list[str]]: The FQN parts. If the input is list then returns list of FQN parts instead.
+        """
+
+        if isinstance(name, list):
+            return [BQ.get_table_fqn_parts(x) for x in name]
+
+        split = name.split('.')
+        if len(split) == 3:
+            return split
+        else:
+            raise ValueError(f'{name} is not a valid table FQN')
+
+    @staticmethod
+    def raise_for_invalid_table_fqn(name: str | list[str]):
+        """Raise an error if the provied name is a fully qualified table name
+
+        Args:
+            name (str | list[str]): Input name (can be multiple)
+
+        Raises:
+            ValueError: If name is not a fully qualified table name
+        """
+
+        if not BQ.get_table_fqn_parts(name):
+            raise ValueError(f'{name} is not a valid table FQN')
+
+    def is_table_exists(self, table_fqn: str) -> bool:
+        self.raise_for_invalid_table_fqn(table_fqn)
+        try:
+            self.client.get_table(table_fqn)
+            return True
+        except NotFound:
+            return False
+
+    def close(self):
         self.client.close()
         logger.debug('BQ client close')

utill/my_csv.py

@@ -57,13 +57,13 @@ def compress(src_filename: str, keep: bool = False, max_size_bytes=ByteSize.GB,
     yield dst_filename
 
 
-def combine(src_filenames: list[str], dst_filename: str) -> None:
+def combine(src_filenames: list[str], dst_filename: str, gzip: bool = False, delete: bool = False) -> None:
     csv.field_size_limit(min(sys.maxsize, 2147483646))  # FIX: _csv.Error: field larger than field limit (131072)
 
     if not dst_filename.endswith('.csv'):
         raise ValueError('Output filename must ends with \'.csv\'!')
 
-    first_file = True
+    first_src_file = True
     with open(dst_filename, 'w') as fout:
         csvwriter = csv.writer(fout)
 
@@ -71,21 +71,25 @@ def combine(src_filenames: list[str], dst_filename: str) -> None:
             src_filename = os.path.expanduser(src_filename)
 
             # Decompress gzipped csv
-            if src_filename.endswith('.csv.gz'):
+            if gzip:
                 src_filename = decompress(src_filename)
 
-            # Copy
+            # Write content into file
             with open(src_filename, 'r') as fin:
                 csvreader = csv.reader(fin)
 
-                # Copy the header if this is the first file
-                if first_file:
+                # Write header only at first file
+                if first_src_file:
                     csvwriter.writerow(next(csvreader))
-                    first_file = False
-                # Else, skip the header
+                    first_src_file = False
                 else:
                     next(csvreader)
 
+                # Write body
                 [csvwriter.writerow(row) for row in csvreader]
 
-            logger.info(f'✅ Combine {src_filename}')
+            logger.debug(f'Combine {src_filename}')
+
+            if delete:
+                os.remove(src_filename)
+                logger.debug(f'Delete {src_filename}')

utill/my_datetime.py

@@ -11,7 +11,7 @@ def get_current_date_str(use_separator: bool = False) -> str:
     return datetime.now().strftime('%Y-%m-%d' if use_separator else '%Y%m%d')
 
 
-def current_datetime_str(use_separator: bool = False) -> str:
+def get_current_datetime_str(use_separator: bool = False) -> str:
     return datetime.now().strftime('%Y-%m-%d %H:%M:%S' if use_separator else '%Y%m%d%H%M%S')
 
 

utill/my_env.py

@@ -49,6 +49,7 @@ def init_mb_file():
 class Envs(BaseSettings):
 
     GCP_PROJECT_ID: Optional[str] = None
+    GCP_REGION: Optional[str] = None
     GCS_BUCKET: Optional[str] = None
 
     def set_var(self, k: str, v: str):

utill/my_file.py

@@ -28,7 +28,7 @@ def decompress(src_file: str, keep: bool = False):
     dst_file = src_file.removesuffix('.gz')
 
     os.remove(dst_file) if os.path.exists(dst_file) else None
-    logger.debug(f'📄 Decompress {src_file} --> {dst_file}')
+    logger.debug(f'Decompress {src_file} to {dst_file}')
     with gzip.open(src_file, 'rb') as f_in:
         with open(dst_file, 'wb') as f_out:
             shutil.copyfileobj(f_in, f_out)

utill/my_gcs.py

@@ -1,123 +1,79 @@
-import os
-import re
-
+from .my_datetime import get_current_datetime_str
+from .my_env import envs
 from google.cloud import storage
 from loguru import logger
-
-from .my_env import envs
+import os
 
 
 class GCS:
 
-    def __init__(self, project: str = None, service_account_filename: str = None, bucket_name: str = None):
-        self.project = project if project is not None else envs.GCP_PROJECT_ID
-
-        if service_account_filename is not None:
-            self.client = storage.Client.from_service_account_json(service_account_filename)
-        else:
-            self.client = storage.Client(project=self.project)
+    def __init__(self, bucket: str | None = None, project_id: str | None = None):
+        if project_id is None and envs.GCP_PROJECT_ID is None:
+            logger.warning('Using ADC for GCS authentication')
 
-        bucket_name_parts = (bucket_name or envs.GCS_BUCKET).split('/')
-        self.change_bucket(bucket_name_parts[0])
-        self.base_path = '/'.join(bucket_name_parts[1:]) if len(bucket_name_parts) > 1 else None
-        not self.base_path or logger.debug(f'Base path: {self.base_path}')
+        if bucket is None and envs.GCS_BUCKET is None:
+            raise ValueError('Bucket name must be provided either as an argument or set in environment variables.')
 
+        self.client = storage.Client(project=project_id or envs.GCP_PROJECT_ID)
+        self.bucket = self.client.bucket(bucket or envs.GCS_BUCKET)
         logger.debug(f'GCS client open, project: {self.client.project}')
 
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_value, exc_tb):
-        self.close_client()
-
-    def _construct_path(self, path: str) -> str:
-        return f'{self.base_path}/{path}' if self.base_path else path
-
-    def change_bucket(self, bucket_name: str):
-        if not bucket_name:
-            raise ValueError('Bucket name needed')
-        self.bucket = self.client.bucket(bucket_name)
-        logger.debug(f'Change bucket to {self.bucket.name}')
-
-    def get(self, path: str) -> storage.Blob:
-        path = self._construct_path(path)
-        return self.bucket.blob(path)
-
-    def list(self, path: str) -> list[storage.Blob]:
-        path = self._construct_path(path)
-        if '*' in path:
-            path_prefix = path.split('*')[0]
-            regex_pattern = '^' + re.escape(path).replace('\\*', '.*') + '$'
-            regex = re.compile(regex_pattern)
-            return [x for x in self.bucket.list_blobs(prefix=path_prefix) if regex.match(x.name)]
-
-        return list(self.bucket.list_blobs(prefix=path))
-
-    def copy(self, src_path: str, dst_path: str, mv: bool = False):
-        src_blob = self.get(src_path)
-        dst_blob = self.get(dst_path)
-
-        dst_blob.rewrite(src_blob)
+    def get_blob(self, blobpath: str) -> storage.Blob:
+        return self.bucket.blob(blobpath)
 
-        logger.debug(f'✅ Copy gs://{src_blob.bucket.name}/{src_blob.name} to gs://{dst_blob.bucket.name}/{dst_blob.name}')
+    def list_blobs(self, prefix: str) -> list[storage.Blob]:
+        return self.bucket.list_blobs(prefix=prefix)
 
-        not mv or GCS.remove_blob(src_blob)
+    def delete_blob(self, blobpath: str | storage.Blob) -> storage.Blob:
+        blob = self.get_blob(blobpath) if isinstance(blobpath, str) else blobpath
+        return blob.delete()
 
-        return dst_blob
+    def copy(self, src_blobpath: str, dst_blobpath: str, dst_bucket: str = None, move: bool = False):
+        src_bucket = self.bucket
+        src_blob = self.get_blob(src_blobpath)
+        dst_bucket = dst_bucket or src_bucket.name
 
-    def copy_to_other_gcs(self, src_blob: storage.Blob, dst_gcs: "GCS", dst_path: str, mv: bool = False):
-        self.bucket.copy_blob(src_blob, dst_gcs.bucket, dst_path)
-        dst_blob = dst_gcs.get(dst_path)
+        self.bucket.copy_blob(src_blob, dst_bucket, dst_blobpath)
 
-        not mv or GCS.remove_blob(src_blob)
-
-        return dst_blob
-
-    def upload(self, local_path: str, remote_path: str, mv: bool = False):
-        local_path = os.path.expanduser(local_path)
-
-        if not os.path.exists(local_path):
-            raise FileNotFoundError(f'File not found: {local_path}')
-
-        blob = self.get(remote_path)
-        blob.upload_from_filename(local_path)
-
-        logger.debug(f'✅ Upload {local_path} to gs://{self.bucket.name}/{blob.name}')
-
-        not mv or os.remove(local_path)
-
-        return blob
-
-    def download(self, obj: str | storage.Blob, local_path: str, mv: bool = False):
-        local_path = os.path.expanduser(local_path)
-        is_blob = type(obj) == storage.Blob
-
-        if os.path.isdir(local_path):
-            local_path = os.path.join(local_path, obj.name.split('/')[-1] if is_blob else os.path.basename(obj))
-            if not os.path.dirname(local_path):
-                raise FileNotFoundError(f'Destination directory not found: {os.path.dirname(local_path)}')
-
-        blob = obj if is_blob else self.get(obj)
-        blob.download_to_filename(local_path)
+        # Move mode
+        if move:
+            self.delete_blob(src_blobpath)
+            logger.debug(f'Moved gs://{src_bucket}/{src_blobpath} to gs://{dst_bucket}/{dst_blobpath}')
+        # Copy mode
+        else:
+            logger.debug(f'Copied gs://{src_bucket}/{src_blobpath} to gs://{dst_bucket}/{dst_blobpath}')
 
-        logger.debug(f'✅ Download gs://{self.bucket.name}/{blob.name} to {local_path}')
+    def upload(self, src_filepath: str, dst_blobpath: str, move: bool = False):
+        blob = self.get_blob(dst_blobpath)
+        blob.upload_from_filename(src_filepath)
 
-        not mv or GCS.remove_blob(blob)
+        # Move mode
+        if move:
+            os.remove(src_filepath)
+            logger.debug(f'Moved {src_filepath} to gs://{self.bucket.name}/{blob.name}')
+        # Copy mode
+        else:
+            logger.debug(f'Uploaded {src_filepath} to gs://{self.bucket.name}/{blob.name}')
 
-        return blob
+    def download(self, src_blobpath: str | storage.Blob, dst_filepath: str, move: bool = False):
+        blob = self.get_blob(src_blobpath) if isinstance(src_blobpath, str) else src_blobpath
+        blob.download_to_filename(dst_filepath)
 
-    def remove(self, remote_path: str):
-        blob = self.get(remote_path)
+        if move:
+            self.delete_blob(blob)
+            logger.debug(f'Moved gs://{self.bucket.name}/{blob.name} to {dst_filepath}')
+        else:
+            logger.debug(f'Copied gs://{self.bucket.name}/{blob.name} to {dst_filepath}')
 
-        GCS.remove_blob(blob)
+    # MARK: Utilities
 
-        return blob
+    @staticmethod
+    def build_tmp_dirpath(prefix: str = 'tmp') -> str:
+        """
+        Builds a temporary directory path in the GCS bucket.
+        """
+        return f'{prefix}/{get_current_datetime_str()}'
 
-    def close_client(self):
+    def close(self):
         self.client.close()
-        logger.debug('GCS client close')
-
-    @staticmethod
-    def remove_blob(blob: storage.Blob):
-        blob.delete()
-        logger.debug(f'🗑️ Remove gs://{blob.bucket.name}/{blob.name}')
+        logger.debug('GCS client closed')

utill/my_queue.py

@@ -1,7 +1,82 @@
-import queue
+from loguru import logger
+from typing import Callable
 import concurrent.futures
+import queue
+
+class StreamingQ:
+    def __init__(self, producer_func: Callable, producer_args: tuple, consumer_func: Callable, max_queue_size: int = 0):
+        self.producer_func = producer_func
+        self.producer_args = producer_args
+        self.consumer_func = consumer_func
+        
+        # Use maxsize for backpressure control (0 = unlimited)
+        self.q = queue.Queue(maxsize=max_queue_size)
+
+    def execute(self):
+        """
+        Execute producer and consumer with true streaming using generators.
+        Yields consumer results as they become available.
+        """
+        def producer():
+            try:
+                for item in self.producer_func(*self.producer_args):
+                    self.q.put(item)
+                    logger.debug(f'🌾 Produced {item}')
+            except Exception as e:
+                logger.error(f'Producer error: {e}')
+                self.q.put(('ERROR', e))
+            finally:
+                # Signal end of production
+                self.q.put(None)
+                logger.debug('🌾 Producer finished')
+
+        def consumer():
+            while True:
+                item = self.q.get()
+                
+                if item is None:
+                    # End of stream signal
+                    self.q.task_done()
+                    break
+                
+                if isinstance(item, tuple) and item[0] == 'ERROR':
+                    # Propagate producer error
+                    self.q.task_done()
+                    raise item[1]
+                
+                try:
+                    # Unpack item if it's a tuple, otherwise pass as single arg
+                    if isinstance(item, tuple):
+                        result = self.consumer_func(*item)
+                    else:
+                        result = self.consumer_func(item)
+                    
+                    self.q.task_done()
+                    logger.debug(f'🔥 Consumed {item} -> {result}')
+                    yield result
+                
+                except Exception as e:
+                    self.q.task_done()
+                    logger.error(f'Consumer error processing {item}: {e}')
+                    raise
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+            # Start producer in background
+            future_producer = executor.submit(producer)
+            
+            try:
+                # Yield results as they become available
+                for result in consumer():
+                    yield result
+                
+                # Wait for producer to complete
+                future_producer.result()
+                
+            except Exception as e:
+                # Cancel producer if consumer fails
+                future_producer.cancel()
+                raise
 
-from loguru import logger
 
 
 class ThreadingQ: