rdxz2-utill 0.0.2__py3-none-any.whl → 0.1.5__py3-none-any.whl

utill/my_cli.py

@@ -0,0 +1,48 @@
+import subprocess
+
+from loguru import logger
+
+
+def _cli(command: list[str], cwd: str | None = None, shell: bool = False):
+    if shell:  # bash
+        command_str = " ".join(command)
+        process = subprocess.Popen(
+            command_str,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            shell=True,
+            executable="/bin/bash",
+            cwd=cwd,
+        )
+    else:
+        process = subprocess.Popen(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            cwd=cwd,
+        )
+
+    for stdout_line in iter(process.stdout.readline, ""):
+        yield stdout_line
+
+    process.stdout.close()
+    return_code = process.wait()
+    if return_code != 0:
+        logger.error(f"Command failed with return code {process.returncode}")
+        raise subprocess.CalledProcessError(return_code, command)
+
+
+def shell(command: list[str], cwd: str | None = None, print_stdout: bool = True):
+    logger.info(f"Executing command: {' '.join(command)}")
+    for res in _cli(command, cwd, shell=True):
+        if print_stdout:
+            logger.info(res.replace("\n", ""))
+
+
+def bash(command: list[str], cwd: str | None = None, print_stdout: bool = True):
+    logger.info(f"Executing command: {' '.join(command)}")
+    for res in _cli(command, cwd, shell=False):
+        if print_stdout:
+            logger.info(res.replace("\n", ""))

utill/my_compare.py

@@ -0,0 +1,34 @@
+import math
+
+
+def try_float(value) -> float:
+    try:
+        return float(value)
+    except:
+        return value
+
+
+def same(a, b, float_precision=None) -> tuple[bool, float]:
+    if a is None and b is None:
+        return True, None
+
+    if a is None or b is None:
+        return False, None
+
+    # Compare float
+    a_float = try_float(a)
+    b_float = try_float(b)
+    if isinstance(a_float, float) or isinstance(b_float, float):
+        try:
+            if math.isnan(a_float) and math.isnan(b_float):
+                return True, None
+
+            if float_precision:
+                a_float = round(a_float, float_precision)
+                b_float = round(b_float, float_precision)
+
+            return a_float == b_float, abs(a_float - b_float)
+        except (ValueError, TypeError):
+            raise Exception(f"Can't compare {a} to {b}")
+
+    return str(a) == str(b), None

utill/my_const.py

@@ -1,4 +1,4 @@
-from enum import Enum
+from enum import StrEnum
 
 
 class ByteSize:
@@ -8,11 +8,11 @@ class ByteSize:
     TB = 1_099_511_627_776
 
 
-class HttpMethod(Enum):
-    GET = 1
-    POST = 2
-    PUT = 3
-    DELETE = 4
-
-    def __str__(self):
-        return self.name
+class HttpMethod(StrEnum):
+    GET = "GET"
+    POST = "POST"
+    PUT = "PUT"
+    DELETE = "DELETE"
+    PATCH = "PATCH"
+    HEAD = "HEAD"
+    OPTIONS = "OPTIONS"

utill/my_csv.py

@@ -11,25 +11,32 @@ from .my_file import decompress
 
 def read_header(filename: str):
     filename = os.path.expanduser(filename)
-    with open(filename, 'r') as f:
+    with open(filename, "r") as f:
         csvreader = csv.reader(f)
         return next(csvreader)
 
 
 def write(filename: str, rows: list[tuple], append: bool = False):
     filename = os.path.expanduser(filename)
-    with open(filename, 'a' if append else 'w') as f:
+    with open(filename, "a" if append else "w") as f:
         csvwriter = csv.writer(f)
         csvwriter.writerows(rows)
 
 
-def compress(src_filename: str, keep: bool = False, max_size_bytes=ByteSize.GB, src_fopen=None, header=None, file_count=1):
+def compress(
+    src_filename: str,
+    keep: bool = False,
+    max_size_bytes=ByteSize.GB,
+    src_fopen=None,
+    header=None,
+    file_count=1,
+):
     src_filename = os.path.expanduser(src_filename)
     current_size = 0
     dst_filename = f'{src_filename}_part{str(file_count).rjust(6, "0")}.gz'
     os.remove(dst_filename) if os.path.exists(dst_filename) else None
-    logger.debug(f'📄 Compress csv {src_filename} --> {dst_filename}')
-    gz = gzip.open(dst_filename, 'wt')
+    logger.debug(f"📄 Compress csv {src_filename} --> {dst_filename}")
+    gz = gzip.open(dst_filename, "wt")
 
     src_fopen = src_fopen or open(src_filename)
     header = header or src_fopen.readline()
@@ -42,14 +49,16 @@ def compress(src_filename: str, keep: bool = False, max_size_bytes=ByteSize.GB,
             break
 
         gz.write(line)
-        current_size += len(line.encode('utf-8'))
+        current_size += len(line.encode("utf-8"))
 
         if current_size >= max_size_bytes:
             gz.close()
             yield dst_filename
 
             file_count += 1
-            yield from compress(src_filename, keep, max_size_bytes, src_fopen, header, file_count)
+            yield from compress(
+                src_filename, keep, max_size_bytes, src_fopen, header, file_count
+            )
             return
 
     gz.close()
@@ -57,34 +66,46 @@ def compress(src_filename: str, keep: bool = False, max_size_bytes=ByteSize.GB,
     yield dst_filename
 
 
-def combine(src_filenames: list[str], dst_filename: str) -> None:
-    csv.field_size_limit(min(sys.maxsize, 2147483646))  # FIX: _csv.Error: field larger than field limit (131072)
+def combine(
+    src_filenames: list[str],
+    dst_filename: str,
+    gzip: bool = False,
+    delete: bool = False,
+) -> None:
+    csv.field_size_limit(
+        min(sys.maxsize, 2147483646)
+    )  # FIX: _csv.Error: field larger than field limit (131072)
 
-    if not dst_filename.endswith('.csv'):
-        raise ValueError('Output filename must ends with \'.csv\'!')
+    if not dst_filename.endswith(".csv"):
+        raise ValueError("Output filename must ends with '.csv'!")
 
-    first_file = True
-    with open(dst_filename, 'w') as fout:
+    first_src_file = True
+    with open(dst_filename, "w") as fout:
         csvwriter = csv.writer(fout)
 
         for src_filename in src_filenames:
             src_filename = os.path.expanduser(src_filename)
 
             # Decompress gzipped csv
-            if src_filename.endswith('.csv.gz'):
+            if gzip:
                 src_filename = decompress(src_filename)
 
-            # Copy
-            with open(src_filename, 'r') as fin:
+            # Write content into file
+            with open(src_filename, "r") as fin:
                 csvreader = csv.reader(fin)
 
-                # Copy the header if this is the first file
-                if first_file:
+                # Write header only at first file
+                if first_src_file:
                     csvwriter.writerow(next(csvreader))
-                # Else, skip the header
+                    first_src_file = False
                 else:
                     next(csvreader)
 
+                # Write body
                 [csvwriter.writerow(row) for row in csvreader]
 
-            logger.info(f'✅ Combine {src_filename}')
+            logger.debug(f"Combine {src_filename}")
+
+            if delete:
+                os.remove(src_filename)
+                logger.debug(f"Delete {src_filename}")

utill/my_datetime.py

@@ -1,4 +1,6 @@
-from datetime import date, datetime, timedelta
+from datetime import date
+from datetime import datetime
+from datetime import timedelta
 from enum import Enum
 
 
@@ -8,28 +10,35 @@ class Level(Enum):
 
 
 def get_current_date_str(use_separator: bool = False) -> str:
-    return datetime.now().strftime('%Y-%m-%d' if use_separator else '%Y%m%d')
+    return datetime.now().strftime("%Y-%m-%d" if use_separator else "%Y%m%d")
 
 
-def current_datetime_str(use_separator: bool = False) -> str:
-    return datetime.now().strftime('%Y-%m-%d %H:%M:%S' if use_separator else '%Y%m%d%H%M%S')
+def get_current_datetime_str(use_separator: bool = False) -> str:
+    return datetime.now().strftime(
+        "%Y-%m-%d %H:%M:%S" if use_separator else "%Y%m%d%H%M%S"
+    )
 
 
 def get_month_first_and_last_day(string: str) -> tuple:
     try:
-        dt = datetime.strptime(string, '%Y-%m')
+        dt = datetime.strptime(string, "%Y-%m")
     except ValueError:
-        dt = datetime.strptime(string, '%Y-%m-%d').replace(day=1)
+        dt = datetime.strptime(string, "%Y-%m-%d").replace(day=1)
 
     return (dt, (dt + timedelta(days=32)).replace(day=1) - timedelta(days=1))
 
 
-def generate_dates(start_date: date | str, end_date: date | str, level: Level, is_output_strings: bool = False):
+def generate_dates(
+    start_date: date | str,
+    end_date: date | str,
+    level: Level,
+    is_output_strings: bool = False,
+):
     # Auto convert strings
     if type(start_date) == str:
-        start_date = datetime.strptime(start_date, '%Y-%m-%d').date()
+        start_date = datetime.strptime(start_date, "%Y-%m-%d").date()
     if type(end_date) == str:
-        end_date = datetime.strptime(end_date, '%Y-%m-%d').date()
+        end_date = datetime.strptime(end_date, "%Y-%m-%d").date()
 
     # Auto convert datetime
     if type(start_date) == datetime:
@@ -38,7 +47,9 @@ def generate_dates(start_date: date | str, end_date: date | str, level: Level, i
         end_date = end_date.date()
 
     if start_date > end_date:
-        raise ValueError(f'start_date \'{start_date}\' cannot be larger than end_date \'{end_date}\'')
+        raise ValueError(
+            f"start_date '{start_date}' cannot be larger than end_date '{end_date}'"
+        )
 
     dates: list[date] = []
 
@@ -55,9 +66,11 @@ def generate_dates(start_date: date | str, end_date: date | str, level: Level, i
                 dates.append(end_date)
                 end_date = end_date - timedelta(days=1)
         case _:
-            raise ValueError(f'level \'{level}\' not recognized. available levels are: \'day\', \'month\'')
+            raise ValueError(
+                f"level '{level}' not recognized. available levels are: 'day', 'month'"
+            )
 
     if is_output_strings:
-        return sorted([date.strftime('%Y-%m-%d') for date in dates])
+        return sorted([date.strftime("%Y-%m-%d") for date in dates])
     else:
         return sorted(dates)

utill/my_encryption.py

@@ -5,16 +5,26 @@ from loguru import logger
 
 
 def __fernet_encrypt_or_decrypt(encrypt: bool, string: str, password: str):
-    return Fernet(password).encrypt(string.encode()) if encrypt else Fernet(password).encrypt(string.encode())
-
-
-def __file_encrypt_or_decrypt(encrypt: bool, src_filename: str, password: str, dst_filename: str = None, overwrite: bool = False):
+    return (
+        Fernet(password).encrypt(string.encode())
+        if encrypt
+        else Fernet(password).encrypt(string.encode())
+    )
+
+
+def __file_encrypt_or_decrypt(
+    encrypt: bool,
+    src_filename: str,
+    password: str,
+    dst_filename: str = None,
+    overwrite: bool = False,
+):
     src_filename = os.path.expanduser(src_filename)
 
     if not os.path.exists(src_filename):
-        return ValueError(f'Source file not exists: {src_filename}')
+        return ValueError(f"Source file not exists: {src_filename}")
 
-    with open(src_filename, 'r') as fr:
+    with open(src_filename, "r") as fr:
         # If destination file is not specified, return the encrypted string
         if not dst_filename:
             return __fernet_encrypt_or_decrypt(encrypt, fr.read(), password)
@@ -25,23 +35,31 @@ def __file_encrypt_or_decrypt(encrypt: bool, src_filename: str, password: str, d
             # Destination file exists checker
             if os.path.exists(dst_filename):
                 if overwrite:
-                    return ValueError(f'Destination file exists: {dst_filename}')
+                    return ValueError(f"Destination file exists: {dst_filename}")
                 else:
                     os.remove(dst_filename)
 
-            with open(dst_filename, 'w') as fw:
+            with open(dst_filename, "w") as fw:
                 fw.write(__fernet_encrypt_or_decrypt(encrypt, fr.read()), password)
 
-            logger.info(f'Encrypted into {dst_filename}')
+            logger.info(f"Encrypted into {dst_filename}")
             return dst_filename
 
 
-def encrypt_file(src_filename: str, password: str, dst_filename: str = None, overwrite: bool = False) -> str:
-    return __file_encrypt_or_decrypt(True, src_filename, password, dst_filename, overwrite)
+def encrypt_file(
+    src_filename: str, password: str, dst_filename: str = None, overwrite: bool = False
+) -> str:
+    return __file_encrypt_or_decrypt(
+        True, src_filename, password, dst_filename, overwrite
+    )
 
 
-def decrypt_file(src_filename: str, password: str, dst_filename: str = None, overwrite: bool = False) -> str:
-    return __file_encrypt_or_decrypt(False, src_filename, password, dst_filename, overwrite)
+def decrypt_file(
+    src_filename: str, password: str, dst_filename: str = None, overwrite: bool = False
+) -> str:
+    return __file_encrypt_or_decrypt(
+        False, src_filename, password, dst_filename, overwrite
+    )
 
 
 def encrypt_string(string: str, password: str) -> str:

utill/my_env.py

@@ -1,18 +1,23 @@
 import os
 import shutil
+from typing import Optional
 
 from loguru import logger
 from pydantic_settings import BaseSettings
-from typing import Optional
 
 from .my_input import ask_yes_no
 
-ENV_DIR = os.path.expanduser(os.path.join('~', '.utill'))
-ENV_FILE = os.path.join(ENV_DIR, 'env')
 
-TEMPLATE_DIR = 'templates'
-TEMPLATE_PG_FILENAME = os.path.join(os.path.dirname(__file__), TEMPLATE_DIR, 'pg.json')  # PostgreSQL connections
-TEMPLATE_MB_FILENAME = os.path.join(os.path.dirname(__file__), TEMPLATE_DIR, 'mb.json')  # Metabase connections
+ENV_DIR = os.path.expanduser(os.path.join("~", ".utill"))
+ENV_FILE = os.path.join(ENV_DIR, "env")
+
+TEMPLATE_DIR = "templates"
+TEMPLATE_PG_FILENAME = os.path.join(
+    os.path.dirname(__file__), TEMPLATE_DIR, "pg.json"
+)  # PostgreSQL connections
+TEMPLATE_MB_FILENAME = os.path.join(
+    os.path.dirname(__file__), TEMPLATE_DIR, "mb.json"
+)  # Metabase connections
 
 PG_FILENAME = os.path.join(ENV_DIR, os.path.basename(TEMPLATE_PG_FILENAME))
 MB_FILENAME = os.path.join(ENV_DIR, os.path.basename(TEMPLATE_MB_FILENAME))
@@ -24,39 +29,45 @@ if not os.path.exists(ENV_DIR):
 
 def init_pg_file():
     if os.path.exists(PG_FILENAME):
-        if ask_yes_no(f'PostgreSQL connection file exists: {PG_FILENAME}, overwrite?'):
+        if ask_yes_no(f"PostgreSQL connection file exists: {PG_FILENAME}, overwrite?"):
             shutil.copy(TEMPLATE_PG_FILENAME, PG_FILENAME)
-            logger.warning(f'PostgreSQL connection file overwritten! {PG_FILENAME}')
+            logger.warning(f"PostgreSQL connection file overwritten! {PG_FILENAME}")
         else:
             return
 
     shutil.copy(TEMPLATE_PG_FILENAME, PG_FILENAME)
-    logger.info(f'PostgreSQL connection file created: {PG_FILENAME}')
+    logger.info(f"PostgreSQL connection file created: {PG_FILENAME}")
 
 
 def init_mb_file():
     if os.path.exists(MB_FILENAME):
-        if ask_yes_no(f'Metabase connection file exists: {MB_FILENAME}, overwrite?'):
+        if ask_yes_no(f"Metabase connection file exists: {MB_FILENAME}, overwrite?"):
             shutil.copy(TEMPLATE_MB_FILENAME, MB_FILENAME)
-            logger.warning(f'Metabase connection file overwritten! {MB_FILENAME}')
+            logger.warning(f"Metabase connection file overwritten! {MB_FILENAME}")
         else:
             return
 
     shutil.copy(TEMPLATE_MB_FILENAME, MB_FILENAME)
-    logger.info(f'Metabase connection file created: {MB_FILENAME}')
+    logger.info(f"Metabase connection file created: {MB_FILENAME}")
 
 
 class Envs(BaseSettings):
 
     GCP_PROJECT_ID: Optional[str] = None
+    GCP_REGION: Optional[str] = None
     GCS_BUCKET: Optional[str] = None
 
     def set_var(self, k: str, v: str):
         setattr(self, k, v)
 
     def write(self):
-        with open(ENV_FILE, 'w') as f:
-            data = '\n'.join(['{}=\"{}\"'.format(k, str(getattr(self, k)).replace('\"', '\\\"')) for k in self.model_fields.keys()])
+        with open(ENV_FILE, "w") as f:
+            data = "\n".join(
+                [
+                    '{}="{}"'.format(k, str(getattr(self, k)).replace('"', '\\"'))
+                    for k in self.model_fields.keys()
+                ]
+            )
             f.write(data)
 
     class Config:

utill/my_file.py

@@ -7,12 +7,12 @@ from loguru import logger
 
 def compress(src_file: str, keep: bool = False):
     src_file = os.path.expanduser(src_file)
-    dst_file = src_file + '.gz'
+    dst_file = src_file + ".gz"
 
     os.remove(dst_file) if os.path.exists(dst_file) else None
-    logger.debug(f'📄 Compress {dst_file} --> {dst_file}')
-    with open(src_file, 'rb') as f_in:
-        with gzip.open(dst_file, 'wb') as f_out:
+    logger.debug(f"📄 Compress {dst_file} --> {dst_file}")
+    with open(src_file, "rb") as f_in:
+        with gzip.open(dst_file, "wb") as f_out:
             shutil.copyfileobj(f_in, f_out)
 
     os.remove(src_file) if not keep else None
@@ -21,19 +21,19 @@ def compress(src_file: str, keep: bool = False):
 
 
 def decompress(src_file: str, keep: bool = False):
-    if not src_file.endswith('.gz'):
-        raise ValueError('File name not ends with .gz!')
+    if not src_file.endswith(".gz"):
+        raise ValueError("File name not ends with .gz!")
 
     src_file = os.path.expanduser(src_file)
-    dst_file = src_file.removesuffix('.gz')
+    dst_file = src_file.removesuffix(".gz")
 
     os.remove(dst_file) if os.path.exists(dst_file) else None
-    logger.debug(f'📄 Decompress {src_file} --> {dst_file}')
-    with gzip.open(src_file, 'rb') as f_in:
-        with open(dst_file, 'wb') as f_out:
+    logger.debug(f"Decompress {src_file} to {dst_file}")
+    with gzip.open(src_file, "rb") as f_in:
+        with open(dst_file, "wb") as f_out:
             shutil.copyfileobj(f_in, f_out)
 
-    keep or os.remove(src_file)
+    os.remove(src_file) if not keep else None
 
     return dst_file
 
@@ -45,15 +45,17 @@ def make_sure_directory_exists(dirname: str):
 
 def make_sure_path_is_directory(path: str):
     if not path.endswith(os.sep):
-        raise ValueError(f'Please specify directory name ending with \'{os.sep}\' character, example for Linux: \'/home/my_username/Downloads/my_folder/\'!')
+        raise ValueError(
+            f"Please specify directory name ending with '{os.sep}' character, example for Linux: '/home/my_username/Downloads/my_folder/'!"
+        )
 
 
 def read_last_line(filename: str) -> str:
     filename = os.path.expanduser(filename)
-    with open(filename, 'rb') as f:
+    with open(filename, "rb") as f:
         try:  # Catch OSError in case of a one line file
             f.seek(-2, os.SEEK_END)
-            while f.read(1) != b'\n':
+            while f.read(1) != b"\n":
                 f.seek(-2, os.SEEK_CUR)
         except OSError:
             f.seek(0)