halib 0.1.99__tar.gz → 0.2.2__tar.gz

{halib-0.1.99 → halib-0.2.2}/.gitignore

@@ -50,7 +50,6 @@ Thumbs.db
 
 build
 dist
-data
 
 venv*/
 

{halib-0.1.99 → halib-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: halib
-Version: 0.1.99
+Version: 0.2.2
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -53,6 +53,12 @@ Dynamic: summary
 
 # Helper package for coding and automation
 
+**Version 0.2.2**
++ reorganize packages with most changes in `research` package
+
+**Version 0.2.01**
++ `research/base_exp`: add `eval_exp` method to evaluate experiment (e.g., model evaluation on test set) after experiment running is done.
+
 **Version 0.1.99**
 + `filetype/ipynb`: add `gen_ipynb_name` generator to create file name based on current notebook name as prefix (with optional timestamp)
 

{halib-0.1.99 → halib-0.2.2}/README.md

@@ -1,5 +1,11 @@
 # Helper package for coding and automation
 
+**Version 0.2.2**
++ reorganize packages with most changes in `research` package
+
+**Version 0.2.01**
++ `research/base_exp`: add `eval_exp` method to evaluate experiment (e.g., model evaluation on test set) after experiment running is done.
+
 **Version 0.1.99**
 + `filetype/ipynb`: add `gen_ipynb_name` generator to create file name based on current notebook name as prefix (with optional timestamp)
 

{halib-0.1.99 → halib-0.2.2}/halib/__init__.py

@@ -56,8 +56,7 @@ from .filetype.yamlfile import load_yaml
 from .system import cmd
 from .system import filesys as fs
 from .filetype import csvfile
-from .cuda import tcuda
-from .common import (
+from .common.common import (
     console,
     console_log,
     ConsoleLog,
@@ -65,6 +64,7 @@ from .common import (
     norm_str,
     pprint_box,
     pprint_local_path,
+    tcuda
 )
 
 # for log
@@ -76,7 +76,7 @@ from timebudget import timebudget
 import omegaconf
 from omegaconf import OmegaConf
 from omegaconf.dictconfig import DictConfig
-from .rich_color import rcolor_str, rcolor_palette, rcolor_palette_all, rcolor_all_str
+from .common.rich_color import rcolor_str, rcolor_palette, rcolor_palette_all, rcolor_all_str
 
 # for visualization
 import seaborn as sns

{halib-0.1.99/halib → halib-0.2.2/halib/common}/common.py

@@ -1,20 +1,20 @@
 import os
 import re
-import rich
 import arrow
-import pathlib
-from pathlib import Path
-import urllib.parse
+import importlib
 
+import rich
 from rich import print
 from rich.panel import Panel
 from rich.console import Console
 from rich.pretty import pprint, Pretty
-from pathlib import PureWindowsPath
+
+from pathlib import Path, PureWindowsPath
 
 
 console = Console()
 
+
 def seed_everything(seed=42):
     import random
     import numpy as np
@@ -61,6 +61,7 @@ def pprint_box(obj, title="", border_style="green"):
         Panel(Pretty(obj, expand_all=True), title=title, border_style=border_style)
     )
 
+
 def console_rule(msg, do_norm_msg=True, is_end_tag=False):
     msg = norm_str(msg) if do_norm_msg else msg
     if is_end_tag:
@@ -149,3 +150,29 @@ def pprint_local_path(
         console.print(content_str)
 
     return file_uri
+
+
+def tcuda():
+    NOT_INSTALLED = "Not Installed"
+    GPU_AVAILABLE = "GPU(s) Available"
+    ls_lib = ["torch", "tensorflow"]
+    lib_stats = {lib: NOT_INSTALLED for lib in ls_lib}
+    for lib in ls_lib:
+        spec = importlib.util.find_spec(lib)
+        if spec:
+            if lib == "torch":
+                import torch
+
+                lib_stats[lib] = str(torch.cuda.device_count()) + " " + GPU_AVAILABLE
+            elif lib == "tensorflow":
+                import tensorflow as tf
+
+                lib_stats[lib] = (
+                    str(len(tf.config.list_physical_devices("GPU")))
+                    + " "
+                    + GPU_AVAILABLE
+                )
+    console.rule("<CUDA Library Stats>")
+    pprint(lib_stats)
+    console.rule("</CUDA Library Stats>")
+    return lib_stats

{halib-0.1.99 → halib-0.2.2}/halib/filetype/csvfile.py

@@ -1,19 +1,13 @@
+import csv
+import textwrap
 import pandas as pd
+import pygwalker as pyg
 from tabulate import tabulate
 from rich.console import Console
-from rich import print as rprint
-from rich import inspect
-from rich.pretty import pprint
-from tqdm import tqdm
-from loguru import logger
 from itables import init_notebook_mode, show
-import pygwalker as pyg
-import textwrap
-import csv
 
 console = Console()
 
-
 def read(file, separator=","):
     df = pd.read_csv(file, separator)
     return df

{halib-0.1.99 → halib-0.2.2}/halib/filetype/ipynb.py

@@ -1,10 +1,8 @@
-from contextlib import contextmanager
-from pathlib import Path
-
 import ipynbname
+from pathlib import Path
+from contextlib import contextmanager
 
-from ..common import console, now_str
-
+from ..common.common import now_str
 
 @contextmanager
 def gen_ipynb_name(

{halib-0.1.99 → halib-0.2.2}/halib/filetype/jsonfile.py

@@ -1,17 +1,14 @@
 import json
 
-
 def read(file):
     with open(file) as f:
         data = json.load(f)
         return data
 
-
 def write(data_dict, outfile):
     with open(outfile, "w") as json_file:
         json.dump(data_dict, json_file)
 
-
 def beautify(json_str):
     formatted_json = json_str
     try:

{halib-0.1.99 → halib-0.2.2}/halib/filetype/textfile.py

@@ -4,7 +4,6 @@ def read_line_by_line(file_path):
         lines = [line.rstrip() for line in lines]
         return lines
 
-
 def write(lines, outfile, append=False):
     mode = "a" if append else "w"
     with open(outfile, mode, encoding="utf-8") as f:

{halib-0.1.99 → halib-0.2.2}/halib/filetype/videofile.py

@@ -1,11 +1,100 @@
+import os
 import cv2
-import textfile
 import enlighten
+
 from enum import Enum
-from ..system import filesys
 from tube_dl import Youtube, Playlist
 from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip
 
+from . import textfile
+from . import csvfile
+from ..system import filesys
+
+class VideoUtils:
+    @staticmethod
+    def _default_meta_extractor(video_path):
+        """Default video metadata extractor function."""
+        # Open the video file
+        cap = cv2.VideoCapture(video_path)
+
+        # Check if the video was opened successfully
+        if not cap.isOpened():
+            print(f"Error: Could not open video file {video_path}")
+            return None
+
+        # Get the frame count
+        frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+
+        # Get the FPS
+        fps = cap.get(cv2.CAP_PROP_FPS)
+
+        # get frame size
+        width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+        height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+
+        # Release the video capture object
+        cap.release()
+
+        meta_dict = {
+            "video_path": video_path,
+            "width": width,
+            "height": height,
+            "frame_count": frame_count,
+            "fps": fps,
+        }
+        return meta_dict
+
+    @staticmethod
+    def get_video_meta_dict(video_path, meta_dict_extractor_func=None):
+        assert os.path.exists(video_path), f"Video file {video_path} does not exist"
+        if meta_dict_extractor_func and callable(meta_dict_extractor_func):
+            assert (
+                meta_dict_extractor_func.__code__.co_argcount == 1
+            ), "meta_dict_extractor_func must take exactly one argument (video_path)"
+            meta_dict = meta_dict_extractor_func(video_path)
+            assert isinstance(
+                meta_dict, dict
+            ), "meta_dict_extractor_func must return a dictionary"
+            assert "video_path" in meta_dict, "meta_dict must contain 'video_path'"
+        else:
+            meta_dict = VideoUtils._default_meta_extractor(video_path=video_path)
+        return meta_dict
+
+    @staticmethod
+    def get_video_dir_meta_df(
+        video_dir,
+        video_exts=[".mp4", ".avi", ".mov", ".mkv"],
+        search_recursive=False,
+        csv_outfile=None,
+    ):
+        assert os.path.exists(video_dir), f"Video directory {video_dir} does not exist"
+        video_files = filesys.filter_files_by_extension(
+            video_dir, video_exts, recursive=search_recursive
+        )
+        assert (
+            len(video_files) > 0
+        ), f"No video files found in {video_dir} with extensions {video_exts}"
+        video_meta_list = []
+        for vfile in video_files:
+            meta_dict = VideoUtils.get_video_meta_dict(vfile)
+            if meta_dict:
+                video_meta_list.append(meta_dict)
+        dfmk = csvfile.DFCreator()
+        columns = list(video_meta_list[0].keys())
+        assert len(columns) > 0, "No video metadata found"
+        assert "video_path" in columns, "video_path column not found in video metadata"
+        # move video_path to the first column
+        columns.remove("video_path")
+        columns.insert(0, "video_path")
+        dfmk.create_table("video_meta", columns)
+        rows = [[meta[col] for col in columns] for meta in video_meta_list]
+        dfmk.insert_rows("video_meta", rows)
+        dfmk.fill_table_from_row_pool("video_meta")
+
+        if csv_outfile:
+            dfmk["video_meta"].to_csv(csv_outfile, index=False, sep=";")
+        return dfmk["video_meta"].copy()
+
 
 class VideoResolution(Enum):
     VR480p = "720x480"

{halib-0.1.99 → halib-0.2.2}/halib/filetype/yamlfile.py

@@ -2,15 +2,15 @@ import time
 import networkx as nx
 from rich import inspect
 from rich.pretty import pprint
-from omegaconf import OmegaConf
 from rich.console import Console
+
+from omegaconf import OmegaConf
 from argparse import ArgumentParser
 
-from ..research.mics import *
+from ..system.path import *
 
 console = Console()
 
-
 def _load_yaml_recursively(
     yaml_file, yaml_files=[], share_nx_graph=nx.DiGraph(), log_info=False
 ):

{halib-0.1.99 → halib-0.2.2}/halib/online/projectmake.py

@@ -1,17 +1,18 @@
 # coding=utf-8
-import json
+
 import os
+import json
+import pycurl
 import shutil
-from argparse import ArgumentParser
-from io import BytesIO
+import certifi
 import subprocess
+from io import BytesIO
+
+from argparse import ArgumentParser
 
-import certifi
-import pycurl
 from ..filetype import jsonfile
 from ..system import filesys
 
-
 def get_curl(url, user_and_pass, verbose=True):
     c = pycurl.Curl()
     c.setopt(pycurl.VERBOSE, verbose)

{halib-0.1.99/halib/utils → halib-0.2.2/halib/online}/tele_noti.py

@@ -25,12 +25,11 @@ def parse_args():
         "--cfg",
         type=str,
         help="yaml file for tele",
-        default=r"E:\Dev\halib\cfg_tele_noti.yaml",
+        default=r"E:\Dev\__halib\halib\online\tele_noti_cfg.yaml",
     )
 
     return parser.parse_args()
 
-
 def get_watcher_message_df(target_file, num_last_lines):
     file_ext = fs.get_file_name(target_file, split_file_ext=True)[1]
     supported_ext = [".txt", ".log", ".csv"]

{halib-0.1.99/halib/research → halib-0.2.2/halib/research/core}/base_config.py

@@ -1,6 +1,10 @@
 import os
 from rich.pretty import pprint
 from abc import ABC, abstractmethod
+from typing import List, Optional, TypeVar, Generic
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
 from dataclass_wizard import YAMLWizard
 
 
@@ -19,6 +23,46 @@ class NamedConfig(ABC):
         pass
 
 
+@dataclass
+class AutoNamedConfig(YAMLWizard, NamedConfig):
+    """
+    Mixin that automatically implements get_name() by returning self.name.
+    Classes using this MUST have a 'name' field.
+    """
+
+    name: Optional[str] = None
+
+    def get_name(self):
+        return self.name
+
+    def __post_init__(self):
+        # Enforce the "MUST" rule here
+        if self.name is None:
+            # We allow None during initial load, but it must be set before usage
+            # or handled by the loader.
+            pass
+
+T = TypeVar("T", bound=AutoNamedConfig)
+
+class BaseSelectorConfig(Generic[T]):
+    """
+    Base class to handle the logic of selecting an item from a list by name.
+    """
+
+    def _resolve_selection(self, items: List[T], selected_name: str, context: str) -> T:
+        if selected_name is None:
+            raise ValueError(f"No {context} selected in the configuration.")
+
+        # Create a lookup dict for O(1) access, or just iterate if list is short
+        for item in items:
+            if item.name == selected_name:
+                return item
+
+        raise ValueError(
+            f"{context.capitalize()} '{selected_name}' not found in the configuration list."
+        )
+
+
 class ExpBaseConfig(ABC, YAMLWizard):
     """
     Base class for configuration objects.

halib-0.2.2/halib/research/core/base_exp.py

@@ -0,0 +1,157 @@
+from abc import ABC, abstractmethod
+from typing import Tuple, Any, Optional
+from base_config import ExpBaseConfig
+from ..perf.perfcalc import PerfCalc
+from ..perf.perfmetrics import MetricsBackend
+
+# ! SEE https://github.com/hahv/base_exp for sample usage
+class BaseExperiment(PerfCalc, ABC):
+    """
+    Base class for experiments.
+    Orchestrates the experiment pipeline using a pluggable metrics backend.
+    """
+
+    def __init__(self, config: ExpBaseConfig):
+        self.config = config
+        self.metric_backend = None
+        # Flag to track if init_general/prepare_dataset has run
+        self._is_env_ready = False
+
+    # -----------------------
+    # PerfCalc Required Methods
+    # -----------------------
+    def get_dataset_name(self):
+        return self.config.get_dataset_cfg().get_name()
+
+    def get_experiment_name(self):
+        return self.config.get_cfg_name()
+
+    def get_metric_backend(self):
+        if not self.metric_backend:
+            self.metric_backend = self.prepare_metrics(self.config.get_metric_cfg())
+        return self.metric_backend
+
+    # -----------------------
+    # Abstract Experiment Steps
+    # -----------------------
+    @abstractmethod
+    def init_general(self, general_cfg):
+        """Setup general settings like SEED, logging, env variables."""
+        pass
+
+    @abstractmethod
+    def prepare_dataset(self, dataset_cfg):
+        """Load/prepare dataset."""
+        pass
+
+    @abstractmethod
+    def prepare_metrics(self, metric_cfg) -> MetricsBackend:
+        """
+        Prepare the metrics for the experiment.
+        This method should be implemented in subclasses.
+        """
+        pass
+
+    @abstractmethod
+    def before_exec_exp_once(self, *args, **kwargs):
+        """Optional: any setup before exec_exp. Note this is called once per run_exp."""
+        pass
+
+    @abstractmethod
+    def exec_exp(self, *args, **kwargs) -> Optional[Tuple[Any, Any]]:
+        """Run experiment process, e.g.: training/evaluation loop.
+        Return: either `None` or a tuple of (raw_metrics_data, extra_data) for calc_and_save_exp_perfs
+        """
+        pass
+
+    @abstractmethod
+    def exec_eval(self, *args, **kwargs) -> Optional[Tuple[Any, Any]]:
+        """Run evaluation process.
+        Return: either `None` or a tuple of (raw_metrics_data, extra_data) for calc_and_save_exp_perfs
+        """
+        pass
+
+    # -----------------------
+    # Internal Helpers
+    # -----------------------
+    def _validate_and_unpack(self, results):
+        if results is None:
+            return None
+        if not isinstance(results, (tuple, list)) or len(results) != 2:
+            raise ValueError("exec must return (metrics_data, extra_data)")
+        return results[0], results[1]
+
+    def _prepare_environment(self, force_reload: bool = False):
+        """
+        Common setup. Skips if already initialized, unless force_reload is True.
+        """
+        if self._is_env_ready and not force_reload:
+            # Environment is already prepared, skipping setup.
+            return
+
+        # 1. Run Setup
+        self.init_general(self.config.get_general_cfg())
+        self.prepare_dataset(self.config.get_dataset_cfg())
+
+        # 2. Update metric backend (refresh if needed)
+        self.metric_backend = self.prepare_metrics(self.config.get_metric_cfg())
+
+        # 3. Mark as ready
+        self._is_env_ready = True
+
+    # -----------------------
+    # Main Experiment Runner
+    # -----------------------
+    def run_exp(self, should_calc_metrics=True, reload_env=False, *args, **kwargs):
+        """
+        Run the whole experiment pipeline.
+        :param reload_env: If True, forces dataset/general init to run again.
+        :param should_calc_metrics: Whether to calculate and save metrics after execution.
+        :kwargs Params:
+            + 'outfile' to save csv file results,
+            + 'outdir' to set output directory for experiment results.
+            + 'return_df' to return a DataFrame of results instead of a dictionary.
+
+        Full pipeline:
+            1. Init
+            2. Prepare Environment (General + Dataset + Metrics)
+            3. Save Config
+            4. Execute
+            5. Calculate & Save Metrics
+        """
+        self._prepare_environment(force_reload=reload_env)
+
+        # Any pre-exec setup (loading models, etc)
+        self.before_exec_exp_once(*args, **kwargs)
+        # Save config before running
+        self.config.save_to_outdir()
+
+        # Execute experiment
+        results = self.exec_exp(*args, **kwargs)
+
+        if should_calc_metrics and results is not None:
+            metrics_data, extra_data = self._validate_and_unpack(results)
+            # Calculate & Save metrics
+            perf_results = self.calc_perfs(
+                raw_metrics_data=metrics_data, extra_data=extra_data, *args, **kwargs
+            )
+            return perf_results
+        else:
+            return results
+
+    # -----------------------
+    # Main Experiment Evaluator
+    # -----------------------
+    def eval_exp(self, reload_env=False, *args, **kwargs):
+        """
+        Run evaluation only.
+        :param reload_env: If True, forces dataset/general init to run again.
+        """
+        self._prepare_environment(force_reload=reload_env)
+        results = self.exec_eval(*args, **kwargs)
+        if results is not None:
+            metrics_data, extra_data = self._validate_and_unpack(results)
+            return self.calc_perfs(
+                raw_metrics_data=metrics_data, extra_data=extra_data, *args, **kwargs
+            )
+        return None

halib-0.1.99/halib/research/params_gen.py → halib-0.2.2/halib/research/core/param_gen.py

@@ -1,12 +1,12 @@
-from typing import Dict, Any, List
-import numpy as np
-from ..common import *
-from ..filetype import yamlfile
-import yaml
 import os
+import yaml
+import numpy as np
+from typing import Dict, Any, List
 
-class ParamGen:
+from ...common.common import *
+from ...filetype import yamlfile
 
+class ParamGen:
     @staticmethod
     def build_from_file(params_file):
         builder = ParamGen(params_file)

{halib-0.1.99/halib/research → halib-0.2.2/halib/research/core}/wandb_op.py

@@ -1,11 +1,12 @@
-import glob
-from rich.pretty import pprint
 import os
-import subprocess
-import argparse
+import glob
 import wandb
+import argparse
+import subprocess
+
 from tqdm import tqdm
 from rich.console import Console
+
 console = Console()
 
 def sync_runs(outdir):

{halib-0.1.99/halib/utils → halib-0.2.2/halib/research/data}/dataclass_util.py

@@ -1,10 +1,11 @@
 import yaml
 from typing import Any
+
 from rich.pretty import pprint
-from ..filetype import yamlfile
-# from halib.filetype import yamlfile
 from dataclasses import make_dataclass
 
+from ...filetype import yamlfile
+
 def dict_to_dataclass(name: str, data: dict):
     fields = []
     values = {}

{halib-0.1.99/halib/research → halib-0.2.2/halib/research/data}/dataset.py

@@ -13,8 +13,8 @@ from rich.pretty import pprint
 from torchvision.datasets import ImageFolder
 from sklearn.model_selection import StratifiedShuffleSplit, ShuffleSplit
 
-from ..system import filesys as fs
-from ..common import console, seed_everything, ConsoleLog
+from ...common.common import console, seed_everything, ConsoleLog
+from ...system import filesys as fs
 
 def parse_args():
     parser = ArgumentParser(description="desc text")

{halib-0.1.99/halib/research → halib-0.2.2/halib/research/data}/torchloader.py

@@ -6,19 +6,22 @@
  * @desc this module works as a utility tools for finding the best configuration for dataloader (num_workers, batch_size, pin_menory, etc.) that fits your hardware.
 """
 from argparse import ArgumentParser
-from ..common import *
-from ..filetype import csvfile
-from ..filetype.yamlfile import load_yaml
-from rich import inspect
-from torch.utils.data import DataLoader
-from torchvision import datasets, transforms
-from tqdm import tqdm
-from typing import Union
-import itertools as it  # for cartesian product
+
 import os
 import time
 import traceback
 
+from tqdm import tqdm
+from rich import inspect
+from typing import Union
+import itertools as it  # for cartesian product
+
+from torch.utils.data import DataLoader
+from torchvision import datasets, transforms
+
+from ...common.common import *
+from ...filetype import csvfile
+from ...filetype.yamlfile import load_yaml
 
 def parse_args():
     parser = ArgumentParser(description="desc text")