machineconfig 3.99__py3-none-any.whl → 7.66__py3-none-any.whl

machineconfig/utils/files/dbms.py

@@ -0,0 +1,257 @@
+import time
+from typing import Optional, Any, Callable
+
+import polars as pl
+
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy import create_engine, text, inspect as inspect__
+from sqlalchemy.engine import Engine
+from sqlalchemy.sql.schema import MetaData
+from pathlib import Path as P
+
+OPLike = Optional[P] | str | None
+
+
+class DBMS:
+    def __init__(self, engine: Engine):
+        self.eng: Engine = engine
+
+    @staticmethod
+    def from_local_db(path: OPLike = None, echo: bool = False, share_across_threads: bool = False, pool_size: int = 5, **kwargs: Any):
+        return DBMS(engine=DBMS.make_sql_engine(path=path, echo=echo, share_across_threads=share_across_threads, pool_size=pool_size, **kwargs))
+
+    def __repr__(self): return f"DataBase @ {self.eng}"
+    def close(self, sleep: int = 2):
+        self.eng.pool.dispose()
+        self.eng.dispose()
+        time.sleep(sleep)
+    @staticmethod
+    def _get_table_identifier(engine: Engine, table: str, sch: Optional[str]):
+        if sch is not None:
+            # Handle DuckDB schema names that contain dots (e.g., "klines.main")
+            if engine.url.drivername == 'duckdb' and '.' in sch and sch.endswith('.main'):
+                # For DuckDB schemas like "klines.main", just use the table name without schema
+                return f'"{table}"'
+            else:
+                return f'"{sch}"."{table}"'
+        else:
+            return f'"{table}"'
+
+    # ==================== QUERIES =====================================
+    def execute_as_you_go(self, *commands: str, res_func: Callable[[Any], Any] = lambda x: x.all(), df: bool = False):
+        with self.eng.begin() as conn:
+            result = None
+            for command in commands:
+                result = conn.execute(text(command))
+            # conn.commit()  # if driver is sqlite3, the connection is autocommitting. # this commit is only needed in case of DBAPI driver.
+            return res_func(result) if not df else pl.DataFrame(res_func(result))
+
+    def execute_begin_once(self, command: str, res_func: Callable[[Any], Any] = lambda x: x.all(), df: bool = False):
+        with self.eng.begin() as conn:
+            result = conn.execute(text(command))  # no need for commit regardless of driver
+            result = res_func(result)
+        return result if not df else pl.DataFrame(result)
+
+    def execute(self, command: str):
+        with self.eng.begin() as conn:
+            result = conn.execute(text(command))
+            # conn.commit()
+        return result
+
+    # def execute_script(self, command: str, df: bool = False):
+    #     with self.eng.begin() as conn: result = conn.executescript(text(command))
+    #     return result if not df else pl.DataFrame(result)
+
+    # ========================== TABLES =====================================
+    def insert_dicts(self, table: str, *mydicts: dict[str, Any]) -> None:
+        cmd = f"""INSERT INTO {table} VALUES """
+        for mydict in mydicts: cmd += f"""({tuple(mydict)}), """
+        self.execute_begin_once(cmd)
+
+    def refresh(self, sch: Optional[str] = None) -> dict[str, Any]:
+        con = self.eng.connect()
+        ses = sessionmaker()(bind=self.eng)
+        meta = MetaData()
+        meta.reflect(bind=self.eng, schema=sch)
+        insp = inspect__(subject=self.eng)
+        schema = insp.get_schema_names()
+        sch_tab = {k: v for k, v in zip(schema, [insp.get_table_names(schema=x) for x in schema])}
+        sch_vws = {k: v for k, v in zip(schema, [insp.get_view_names(schema=x) for x in schema])}
+        return {'con': con, 'ses': ses, 'meta': meta, 'insp': insp, 'schema': schema, 'sch_tab': sch_tab, 'sch_vws': sch_vws}
+
+    def get_columns(self, table: str, sch: Optional[str] = None) -> list[str]:
+        meta = MetaData()
+        meta.reflect(bind=self.eng, schema=sch)
+        return list(meta.tables[self._get_table_identifier(self.eng, table, sch)].exported_columns.keys())
+
+    def read_table(self, table: Optional[str] = None, sch: Optional[str] = None, size: int = 5) -> pl.DataFrame:
+        insp = inspect__(self.eng)
+        schema = insp.get_schema_names()
+        sch_tab = {k: v for k, v in zip(schema, [insp.get_table_names(schema=x) for x in schema])}
+        if sch is None:
+            # First try to find schemas that have tables (excluding system schemas)
+            schemas_with_tables = []
+            for schema_name in schema:
+                if schema_name not in ["information_schema", "pg_catalog", "system"]:
+                    if schema_name in sch_tab and len(sch_tab[schema_name]) > 0:
+                        schemas_with_tables.append(schema_name)
+
+            if len(schemas_with_tables) == 0:
+                raise ValueError(f"No schemas with tables found. Available schemas: {schema}")
+
+            # Prefer non-"main" schemas if available, otherwise use main
+            if len(schemas_with_tables) > 1 and "main" in schemas_with_tables:
+                sch = [s for s in schemas_with_tables if s != "main"][0]
+            else:
+                sch = schemas_with_tables[0]
+            print(f"Auto-selected schema: `{sch}` from available schemas: {schemas_with_tables}")
+
+        if table is None:
+            if sch not in sch_tab:
+                raise ValueError(f"Schema `{sch}` not found. Available schemas: {list(sch_tab.keys())}")
+            tables = sch_tab[sch]
+            assert len(tables) > 0, f"No tables found in schema `{sch}`"
+            import random
+            table = random.choice(tables)
+            print(f"Reading table `{table}` from schema `{sch}`")
+        with self.eng.connect() as conn:
+            try:
+                res = conn.execute(text(f'''SELECT * FROM {self._get_table_identifier(self.eng, table, sch)} '''))
+                return pl.DataFrame(res.fetchmany(size))
+            except Exception:
+                print(f"Error executing query for table `{table}` in schema `{sch}`")
+                print(f"Available schemas and tables: {sch_tab}")
+                raise
+
+    def describe_db(self, sch: Optional[str] = None) -> pl.DataFrame:
+        meta = MetaData()
+        meta.reflect(bind=self.eng, schema=sch)
+        ses = sessionmaker()(bind=self.eng)
+        res_all = []
+        from rich.progress import Progress
+        with Progress() as progress:
+            task = progress.add_task("Inspecting tables", total=len(meta.sorted_tables))
+            for tbl in meta.sorted_tables:
+                table = tbl.name
+                if sch is not None:
+                    table = f"{sch}.{table}"
+                count = ses.query(tbl).count()
+                res = dict(table=table, count=count, size_mb=count * len(tbl.exported_columns) * 10 / 1e6,
+                           columns=len(tbl.exported_columns), schema=sch)
+                res_all.append(res)
+                progress.update(task, advance=1)
+        return pl.DataFrame(res_all)
+
+    def describe_table(self, table: str, sch: Optional[str] = None, dtype: bool = True) -> None:
+        print(table.center(100, "="))
+        meta = MetaData()
+        meta.reflect(bind=self.eng, schema=sch)
+        tbl = meta.tables[self._get_table_identifier(self.eng, table, sch)]
+        ses = sessionmaker()(bind=self.eng)
+        count = ses.query(tbl).count()
+        res = dict(name=table, count=count, size_mb=count * len(tbl.exported_columns) * 10 / 1e6)
+        from machineconfig.utils.accessories import pprint
+        pprint(res, title="TABLE DETAILS")
+        dat = self.read_table(table=table, sch=sch, size=2)
+        df = dat
+        print("SAMPLE:\n", df)
+        insp = inspect__(self.eng)
+        if dtype: print("\nDETAILED COLUMNS:\n", pl.DataFrame(insp.get_columns(self._get_table_identifier(self.eng, table, sch))))
+        print("\n" * 3)
+
+    @staticmethod
+    def make_sql_engine(path: OPLike = None, echo: bool = False, share_across_threads: bool = False, pool_size: int = 5, **kwargs: Any) -> Engine:
+        if path is None:
+            url = 'sqlite:///:memory:'
+        elif isinstance(path, str) and path.startswith(('sqlite://', 'postgresql://', 'mysql://', 'duckdb://')):
+            url = path
+        else:
+            path_str = str(P(path))
+            if path_str.endswith('.duckdb'):
+                url = f'duckdb:///{path_str}'
+            else:
+                url = f'sqlite:///{path_str}'
+        connect_args = {}
+        if share_across_threads and 'sqlite' in url:
+            connect_args['check_same_thread'] = False
+        return create_engine(url, echo=echo, pool_size=pool_size, connect_args=connect_args, **kwargs)
+
+DB_TMP_PATH = P.home().joinpath(".tmp").joinpath("tmp_dbs").joinpath("results").joinpath("data.sqlite")
+
+
+def to_db(table: str, idx: int, idx_max: int, data: Any):
+    import pickle
+    DB_TMP_PATH.parent.mkdir(parents=True, exist_ok=True)
+    db = DBMS.from_local_db(DB_TMP_PATH)
+    time_now = time.time_ns()
+    data_blob = pickle.dumps(data)
+    create_table = f"""CREATE TABLE IF NOT EXISTS "{table}" (time INT PRIMARY KEY, idx INT, idx_max INT, data BLOB)"""
+    insert_row = f"""INSERT INTO "{table}" (time, idx, idx_max, data) VALUES (:time, :idx, :idx_max, :data)"""
+    with db.eng.begin() as conn:
+        conn.execute(text(create_table))
+        conn.execute(
+            text(insert_row),
+            {'time': time_now, 'idx': idx, 'idx_max': idx_max, 'data': data_blob}
+        )
+    db.close()
+
+
+def from_db(table: str):
+    import pickle
+    DB_TMP_PATH.parent.mkdir(parents=True, exist_ok=True)
+    db = DBMS.from_local_db(DB_TMP_PATH)
+    with db.eng.connect() as conn:
+        res = conn.execute(text(f"""SELECT * FROM "{table}" """))
+        records = res.fetchall()
+        df = pl.DataFrame(records, schema=['time', 'idx', 'idx_max', 'data'])
+        df = df.with_columns(pl.col('data').map_elements(pickle.loads))
+        return df
+
+
+def get_table_specs(engine: Engine, table_name: str) -> pl.DataFrame:
+    inspector = inspect__(engine)
+    # Collect table information
+    columns_info = [{
+        'name': col['name'],
+        'type': str(col['type']),
+        'nullable': col['nullable'],
+        'default': col['default'],
+        'autoincrement': col.get('autoincrement'),
+        'category': 'column'
+    } for col in inspector.get_columns(table_name)]
+    # Primary keys
+    pk_info = [{
+        'name': pk,
+        'type': None,
+        'nullable': False,
+        'default': None,
+        'autoincrement': None,
+        'category': 'primary_key'
+    } for pk in inspector.get_pk_constraint(table_name)['constrained_columns']]
+    # Foreign keys
+    fk_info = [{
+        'name': fk['constrained_columns'][0],
+        'type': f"FK -> {fk['referred_table']}.{fk['referred_columns'][0]}",
+        'nullable': None,
+        'default': None,
+        'autoincrement': None,
+        'category': 'foreign_key'
+    } for fk in inspector.get_foreign_keys(table_name)]
+    # Indexe
+    index_info = [{
+        'name': idx['name'],
+        'type': f"Index on {', '.join(col for col in idx['column_names'] if col)}",
+        'nullable': None,
+        'default': None,
+        'autoincrement': None,
+        'category': 'index',
+        'unique': idx['unique']
+    } for idx in inspector.get_indexes(table_name)]
+    # Combine all information
+    all_info = columns_info + pk_info + fk_info + index_info
+    # Convert to DataFrame
+    df = pl.DataFrame(all_info)
+    return df
+
+if __name__ == '__main__':
+    pass

machineconfig/utils/files/headers.py

@@ -0,0 +1,68 @@
+
+import glob
+import os
+import platform
+import random
+from pathlib import Path
+from rich import pretty
+from rich.console import Console
+
+
+def print_header():
+    console = Console()
+    pretty.install()
+
+    # Environment Information Panel
+    from rich.panel import Panel
+    from rich.table import Table
+
+    table = Table(show_header=False, show_edge=False, pad_edge=False)
+    table.add_column("Label", style="cyan", no_wrap=True)
+    table.add_column("Value", style="white")
+
+    table.add_row("Python Version", platform.python_version())
+    table.add_row("Operating System", platform.system())
+    table.add_row("Virtual Environment", os.getenv('VIRTUAL_ENV', 'None'))
+    table.add_row("Running @", str(Path.cwd()))
+
+    from machineconfig.utils.installer_utils.installer_runner import get_machineconfig_version
+
+    console.print(Panel(table, title=f"[bold blue]✨ 🐊 Machineconfig Shell {get_machineconfig_version()} ✨ Made with 🐍 | Built with ❤️[/bold blue]", border_style="blue"))
+def print_logo(logo: str):
+    from machineconfig.utils.files.ascii_art import font_box_color, character_color, character_or_box_color
+    if platform.system() == "Windows":
+        _1x = Path.home().joinpath(r"AppData/Roaming/npm/figlet").exists()
+        _2x = Path.home().joinpath(r"AppData/Roaming/npm/lolcatjs").exists()
+        _3x = Path.home().joinpath(r"AppData/Local/Microsoft/WindowsApps/boxes.exe").exists()
+        if _1x and _2x and _3x:
+            if random.choice([True, True, False]): font_box_color(logo)
+            else: character_color(logo)
+        else:
+            # print("\n" + "🚫 " + "-" * 70 + " 🚫")
+            # print("🔍 Missing ASCII art dependencies. Install with: iwr bit.ly/cfgasciiartwindows | iex")
+            # print("🚫 " + "-" * 70 + " 🚫\n")
+            _default_art = Path(random.choice(glob.glob(str(Path(__file__).parent.joinpath("art", "*")))))
+            print(_default_art.read_text())
+    elif platform.system() in ["Linux", "Darwin"]:  # Explicitly handle both Linux and macOS
+        from machineconfig.utils.installer_utils.installer_locator_utils import is_executable_in_path
+        avail_cowsay = is_executable_in_path("cowsay")
+        avail_lolcat = is_executable_in_path("lolcat")
+        avail_boxes = is_executable_in_path("boxes")
+        avail_figlet = is_executable_in_path("figlet")
+        if avail_cowsay and avail_lolcat and avail_boxes and avail_figlet:
+            # _dynamic_art = random.choice([True, True, True, True, False])
+            # if _dynamic_art: character_or_box_color(logo=logo)
+            # else:
+            #     print(Path(random.choice(glob.glob(str(Path(__file__).parent.joinpath("art", "*"))))).read_text())
+            character_or_box_color(logo=logo)
+        else:
+            print("\n" + "🚫 " + "-" * 70 + " 🚫")
+            install_cmd = "devops install --group TerminalEyeCandy" if platform.system() == "Linux" else "brew install cowsay lolcat boxes figlet"
+            print(f"🔍 Missing ASCII art dependencies. Install with: {install_cmd}")
+            print("🚫 " + "-" * 70 + " 🚫\n")
+            _default_art = Path(random.choice(glob.glob(str(Path(__file__).parent.joinpath("art", "*")))))
+            print(_default_art.read_text())
+    else:
+        print(f"⚠️ Platform {platform.system()} not supported for ASCII art. Using default art.")
+        _default_art = Path(random.choice(glob.glob(str(Path(__file__).parent.joinpath("art", "*")))))
+        print(_default_art.read_text())

machineconfig/utils/files/ouch/decompress.py

@@ -0,0 +1,45 @@
+
+
+from pathlib import Path
+from zipfile import ZipFile, BadZipFile
+from typing import Union
+
+
+def decompress_and_remove_zip(zip_path: Union[str, Path]) -> None:
+    """
+    Decompress a ZIP file and remove it after extraction.
+
+    Args:
+        zip_path (Union[str, Path]): Path to the ZIP file.
+
+    Raises:
+        FileNotFoundError: If the zip file does not exist.
+        BadZipFile: If the file is not a valid ZIP archive.
+        PermissionError: If the file cannot be deleted due to permission issues.
+        Exception: For other unexpected errors.
+    """
+    zip_path = Path(zip_path)
+
+    if not zip_path.exists():
+        raise FileNotFoundError(f"The file '{zip_path}' does not exist.")
+    
+    if not zip_path.is_file():
+        raise FileNotFoundError(f"The path '{zip_path}' is not a file.")
+
+    extract_dir = zip_path.parent
+
+    try:
+        with ZipFile(zip_path, 'r') as zip_ref:
+            zip_ref.extractall(extract_dir)
+    except BadZipFile as e:
+        raise BadZipFile(f"The file '{zip_path}' is not a valid zip archive.") from e
+    except Exception as e:
+        raise Exception(f"Failed to extract '{zip_path}': {e}") from e
+
+    try:
+        zip_path.unlink()
+    except PermissionError as e:
+        raise PermissionError(f"Permission denied when deleting '{zip_path}'.") from e
+    except Exception as e:
+        raise Exception(f"Failed to delete '{zip_path}': {e}") from e
+

machineconfig/utils/files/read.py

@@ -0,0 +1,95 @@
+
+
+
+from pathlib import Path
+from typing import Any, Optional
+
+
+class Read:
+    @staticmethod
+    def read(path: 'Path', **kwargs: Any) -> Any:
+        if Path(path).is_dir(): raise IsADirectoryError(f"Path is a directory, not a file: {path}")
+        suffix = Path(path).suffix[1:]
+        if suffix == "": raise ValueError(f"File type could not be inferred from suffix. Suffix is empty. Path: {path}")
+        if suffix in ("sqlite", "sqlite3", "db", "duckdb"):
+            from machineconfig.utils.files.dbms import DBMS
+            res = DBMS.from_local_db(path=path)
+            print(res.describe_db())
+            return res
+        try: return getattr(Read, suffix)(str(path), **kwargs)
+        except AttributeError as err:
+            if "type object 'Read' has no attribute" not in str(err): raise AttributeError(err) from err
+            if suffix in ('eps', 'jpg', 'jpeg', 'pdf', 'pgf', 'png', 'ps', 'raw', 'rgba', 'svg', 'svgz', 'tif', 'tiff'):
+                import matplotlib.pyplot as pyplot
+                return pyplot.imread(str(path), **kwargs)  # from: plt.gcf().canvas.get_supported_filetypes().keys():
+            if suffix == "parquet":
+                import polars as pl
+                return pl.read_parquet(path, **kwargs)
+            elif suffix == "csv":
+                import polars as pl
+                return pl.read_csv(path, **kwargs)
+            try:
+                # guess = install_n_import('magic', 'python-magic').from_file(path)
+                guess = "IDKm"
+                raise AttributeError(f"Unknown file type. failed to recognize the suffix `{suffix}`. According to libmagic1, the file seems to be: {guess}") from err
+            except ImportError as err2:
+                print(f"💥 Unknown file type. failed to recognize the suffix `{suffix}` of file {path} ")
+                raise ImportError(err) from err2
+    @staticmethod
+    def json(path: 'Path', r: bool = False, **kwargs: Any) -> Any:  # return could be list or dict etc
+        from machineconfig.utils.io import read_json
+        return read_json(path, r=r, **kwargs)
+    @staticmethod
+    def yaml(path: 'Path', r: bool = False) -> Any:  # return could be list or dict etc
+        import yaml
+        with open(str(path), "r", encoding="utf-8") as file:
+            mydict = yaml.load(file, Loader=yaml.FullLoader)
+        _ = r
+        return mydict
+    @staticmethod
+    def ini(path: 'Path', encoding: Optional[str] = None):
+        if not Path(path).exists() or Path(path).is_dir(): raise FileNotFoundError(f"File not found or is a directory: {path}")
+        import configparser
+        res = configparser.ConfigParser()
+        res.read(filenames=[str(path)], encoding=encoding)
+        return res
+    @staticmethod
+    def toml(path: 'Path'):
+        import tomllib
+        return tomllib.loads(Path(path).read_text(encoding='utf-8'))
+    @staticmethod
+    def npy(path: 'Path', **kwargs: Any):
+        import numpy as np
+        data = np.load(str(path), allow_pickle=True, **kwargs)
+        # data = data.item() if data.dtype == np.object else data
+        return data
+    @staticmethod
+    def pickle(path: 'Path', **kwargs: Any):
+        import pickle
+        try: return pickle.loads(Path(path).read_bytes(), **kwargs)
+        except BaseException as ex:
+            print(f"💥 Failed to load pickle file `{path}` with error:\n{ex}")
+            raise ex
+    @staticmethod
+    def pkl(path: 'Path', **kwargs: Any): return Read.pickle(path, **kwargs)
+    # @staticmethod
+    # def dill(path: 'Path', **kwargs: Any) -> Any:
+    #     """handles imports automatically provided that saved object was from an imported class (not in defined in __main__)"""
+    #     import dill
+    #     obj = dill.loads(str=Path(path).read_bytes(), **kwargs)
+    #     return obj
+    @staticmethod
+    def py(path: 'Path', init_globals: Optional[dict[str, Any]] = None, run_name: Optional[str] = None):
+        import runpy
+        return runpy.run_path(str(path), init_globals=init_globals, run_name=run_name)
+    @staticmethod
+    def txt(path: 'Path', encoding: str = 'utf-8') -> str: return Path(path).read_text(encoding=encoding)
+    @staticmethod
+    def parquet(path: 'Path', **kwargs: Any):
+        import polars as pl
+        return pl.read_parquet(path, **kwargs)
+
+
+
+if __name__ == '__main__':
+    pass

machineconfig/utils/installer_utils/github_release_bulk.py

@@ -109,8 +109,7 @@ def main() -> None:
     current_dir = Path(__file__).parent
     installer_dir = current_dir.parent.parent / "jobs" / "installer"
     
-    standard_json = installer_dir / "packages_standard.json"
-    dev_json = installer_dir / "packages_dev.json"
+    standard_json = installer_dir / "installer_data.json"
     output_json = current_dir / "github_releases.json"
     
     print("🔍 Starting GitHub release data extraction...")
@@ -125,16 +124,7 @@ def main() -> None:
         all_github_repos.update(repos)
         print(f"   Found {len(repos)} GitHub repos")
     else:
-        print(f"⚠️  File not found: {standard_json}")
-        
-    if dev_json.exists():
-        print(f"📄 Reading {dev_json.name}...")
-        repos = extract_github_repos_from_json(dev_json)
-        all_github_repos.update(repos)
-        print(f"   Found {len(repos)} GitHub repos")
-    else:
-        print(f"⚠️  File not found: {dev_json}")
-    
+        print(f"⚠️  File not found: {standard_json}")    
     print(f"🎯 Total unique GitHub repositories found: {len(all_github_repos)}")
     
     if not all_github_repos: