al-data-utils 0.3.1__py3-none-any.whl

al_data_utils/backup_utils/mssql/scripter/__init__.py

@@ -0,0 +1 @@
+# placeholder for future MSSQL scripting utilities

al_data_utils/config_utils/__init__.py

@@ -0,0 +1,6 @@
+from .config_utils import (
+    load_config,
+    save_config
+)
+
+__all__ = ['load_config', 'save_config']

al_data_utils/config_utils/config_utils.py

@@ -0,0 +1,31 @@
+import yaml
+from al_data_utils import logging_utils
+import os
+
+
+def load_config(config_file: str) -> dict:
+    """Loads a YAML configuration file safely.
+
+    Args:
+        config_file (str): Path to the YAML file.
+
+    Returns:
+        dict: Parsed YAML content.
+    """
+    with open(config_file, 'r', encoding='utf-8') as file:
+        config = yaml.safe_load(file)
+
+    return config
+
+
+def save_config(config_json: dict, destination_config_file: str) -> None:
+    """Stores a dictionary as a YAML configuration file.
+
+    Args:
+        config_json (dict): Configuration data to save.
+        destination_config_file (str): Path to the YAML file.
+    """
+    os.makedirs(os.path.dirname(destination_config_file), exist_ok=True)
+    with open(destination_config_file, 'w', encoding='utf-8') as file:
+        yaml.safe_dump(config_json, file, default_flow_style=False, allow_unicode=True)
+    logging_utils.info(f'successfully stored config in: {destination_config_file}')

al_data_utils/files_sync_utils/__init__.py

@@ -0,0 +1,15 @@
+from .files_sync_utils import (
+    build_plan_df,
+    log_activity,
+    log_folder_retention,
+    SyncConfig,
+    execute_plan,
+)
+
+__all__ = [
+    'build_plan_df',
+    'log_activity',
+    'log_folder_retention',
+    'SyncConfig',
+    'execute_plan',
+]

al_data_utils/files_sync_utils/files_sync_utils.py

@@ -0,0 +1,567 @@
+from __future__ import annotations
+
+import os
+import shutil
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Any, cast
+
+import pandas as pd
+from al_data_utils import logging_utils
+
+# =========================
+# Config
+# =========================
+
+
+@dataclass(frozen=True)
+class SyncConfig:
+    source_paths: list[str] | dict[str, str]
+    target_path: str
+    log_folder: str
+
+    # COPY: if path matches these => DO NOT COPY. If exists in destination => DELETE it there.
+    copy_folders_names_to_skip: list[str]
+    copy_files_names_to_skip: list[str]
+
+    # DELETE: if extra path exists in destination (missing in source) and matches these => DO NOT DELETE
+    delete_folders_names_to_skip: list[str]
+    delete_files_names_to_skip: list[str]
+
+    # DELETE TRY: if deletion fails => mark as try_delete_failed (not fatal)
+    delete_folders_names_to_try: list[str]
+    delete_files_names_to_try: list[str]
+
+
+# =========================
+# Helpers
+# =========================
+
+
+def _lower_set(items: list[str]) -> set[str]:
+    return {x.lower() for x in (items or [])}
+
+
+def _is_under_folder_name(rel_path: Path, names_lower: set[str]) -> bool:
+    return any(part.lower() in names_lower for part in rel_path.parts)
+
+
+def _files_equal_by_stat(src: Path, dst: Path) -> bool:
+    try:
+        s1 = src.stat()
+        s2 = dst.stat()
+        return s1.st_size == s2.st_size and int(s1.st_mtime) == int(s2.st_mtime)
+    except FileNotFoundError:
+        return False
+
+
+def _safe_name_lower(p: object) -> str:
+    try:
+        if p is None:
+            return ''
+        if cast(bool, pd.isna(cast(Any, p))):
+            return ''
+        return Path(str(p)).name.lower()
+    except Exception:
+        return ''
+
+
+# =========================
+# Inventory: "full list" but doesn't explode
+# - we LIST excluded dirs, but DON'T descend into them
+# =========================
+
+
+def build_inventory(
+    root: Path,
+    folders_to_not_descend_lower: set[str] | None = None,
+    files_to_mark_excluded_lower: set[str] | None = None,
+) -> pd.DataFrame:
+    cols = ['kind', 'rel_path', 'abs_path', 'excluded', 'excluded_reason']
+
+    folders_to_not_descend_lower = folders_to_not_descend_lower or set()
+    files_to_mark_excluded_lower = files_to_mark_excluded_lower or set()
+
+    rows: list[dict] = []
+    if not root.exists():
+        return pd.DataFrame(columns=cols)
+
+    for cur_root, dirnames, filenames in os.walk(root, topdown=True):
+        cur = Path(cur_root)
+        rel_dir = cur.relative_to(root)
+
+        # record current directory (except ".")
+        if rel_dir != Path('.'):
+            rows.append(
+                dict(
+                    kind='dir',
+                    rel_path=str(rel_dir),
+                    abs_path=str(cur),
+                    excluded=False,
+                    excluded_reason='',
+                )
+            )
+
+        # record excluded dirs but don't descend
+        kept_dirs: list[str] = []
+        for d in dirnames:
+            if d.lower() in folders_to_not_descend_lower:
+                rel_ex_dir = (cur / d).relative_to(root)
+                rows.append(
+                    dict(
+                        kind='dir',
+                        rel_path=str(rel_ex_dir),
+                        abs_path=str(cur / d),
+                        excluded=True,
+                        excluded_reason='excluded_dir',
+                    )
+                )
+            else:
+                kept_dirs.append(d)
+        dirnames[:] = kept_dirs
+
+        # record files
+        for fn in filenames:
+            rel_file = (cur / fn).relative_to(root)
+            excluded = fn.lower() in files_to_mark_excluded_lower
+            rows.append(
+                dict(
+                    kind='file',
+                    rel_path=str(rel_file),
+                    abs_path=str(cur / fn),
+                    excluded=excluded,
+                    excluded_reason='excluded_file' if excluded else '',
+                )
+            )
+
+    # Keep a stable schema even when the root exists but contains no files/dirs.
+    return pd.DataFrame(rows, columns=cols)
+
+
+# =========================
+# Plan
+# =========================
+
+
+def build_plan_df(cfg: SyncConfig) -> pd.DataFrame:
+    sources = cfg.source_paths
+    target_root = Path(cfg.target_path)
+
+    copy_skip_dirs = _lower_set(cfg.copy_folders_names_to_skip)
+    copy_skip_files = _lower_set(cfg.copy_files_names_to_skip)
+
+    del_skip_dirs = _lower_set(cfg.delete_folders_names_to_skip)
+    del_skip_files = _lower_set(cfg.delete_files_names_to_skip)
+
+    del_try_dirs = _lower_set(cfg.delete_folders_names_to_try)
+    del_try_files = _lower_set(cfg.delete_files_names_to_try)
+
+    src_inventories = []
+    dest_root = target_root
+
+    for src_root in sources:
+        src_inv = build_inventory(
+            Path(src_root),
+            folders_to_not_descend_lower=copy_skip_dirs,  # don't enumerate inside copy-skip dirs
+            files_to_mark_excluded_lower=copy_skip_files,  # list but mark excluded (no copy)
+        ).rename(
+            columns={
+                'abs_path': 'src_path',
+                'excluded': 'src_excluded',
+                'excluded_reason': 'src_excluded_reason',
+            }
+        )
+        src_inv['source_root'] = str(src_root)
+        src_inventories.append(src_inv)
+
+    # concat into one DF
+    src_inv = pd.concat(src_inventories, ignore_index=True)
+
+    dst_inv = build_inventory(
+        dest_root,
+        folders_to_not_descend_lower=None,  # don't enumerate inside copy-skip dirs
+        files_to_mark_excluded_lower=None,
+    ).rename(
+        columns={
+            'abs_path': 'dst_path',
+            'excluded': 'dst_excluded',
+            'excluded_reason': 'dst_excluded_reason',
+        }
+    )
+
+    plan = src_inv.merge(dst_inv, on=['kind', 'rel_path'], how='outer')
+    plan['source_root'] = str(src_root)
+    plan['dest_root'] = str(dest_root)
+
+    plan['src_exists'] = plan['src_path'].notna()
+    plan['dst_exists'] = plan['dst_path'].notna()
+
+    rel_p = plan['rel_path'].fillna('').map(Path)
+
+    # copy-skip by folder subtree
+    plan['_under_copy_skip_dir'] = rel_p.map(
+        lambda p: _is_under_folder_name(p, copy_skip_dirs)
+    )
+
+    # copy-skip by file name (for files only)
+    src_name_lower = plan['src_path'].map(_safe_name_lower)
+    dst_name_lower = plan['dst_path'].map(_safe_name_lower)
+    plan['_is_copy_skip_file'] = (plan['kind'] == 'file') & (
+        src_name_lower.isin(copy_skip_files) | dst_name_lower.isin(copy_skip_files)
+    )
+
+    copy_skip_any = plan['_under_copy_skip_dir'] | plan['_is_copy_skip_file']
+
+    # delete-skip by folder subtree / file name (for "extra in dst" deletes)
+    plan['_under_delete_skip_dir'] = rel_p.map(
+        lambda p: _is_under_folder_name(p, del_skip_dirs)
+    )
+    is_delete_skip_file = dst_name_lower.isin(del_skip_files)
+
+    # for try-delete names
+    is_try_file = dst_name_lower.isin(del_try_files)
+    rel_last_part_lower = rel_p.map(lambda p: p.parts[-1].lower() if p.parts else '')
+    is_try_dir = rel_last_part_lower.isin(del_try_dirs)
+
+    # initialize
+    plan['action'] = ''
+    plan['reason'] = ''
+    plan['action_order'] = 0
+    plan['_created_at'] = datetime.now().isoformat(timespec='seconds')
+
+    # -----------------------------
+    # 0) NEW RULE (your requirement):
+    # If in copy skip list and exists in destination -> DELETE in destination
+    # If in copy skip list and destination missing -> SKIP
+    # -----------------------------
+
+    delete_skip_any = plan['_under_delete_skip_dir'] | is_delete_skip_file
+
+    # file cleanup (copy-skip) BUT respect delete-skip
+    plan.loc[
+        copy_skip_any
+        & plan['dst_exists']
+        & (plan['kind'] == 'file')
+        & ~delete_skip_any
+        & (plan['action'] == ''),
+        ['action', 'reason', 'action_order'],
+    ] = ['delete', 'copy_skip_cleanup', 70]
+
+    # dir cleanup (copy-skip) BUT respect delete-skip
+    plan.loc[
+        copy_skip_any
+        & plan['dst_exists']
+        & (plan['kind'] == 'dir')
+        & ~delete_skip_any
+        & (plan['action'] == ''),
+        ['action', 'reason', 'action_order'],
+    ] = ['delete_dir', 'copy_skip_cleanup', 80]
+
+    # copy-skip but protected from delete => skip
+    plan.loc[
+        copy_skip_any & plan['dst_exists'] & delete_skip_any & (plan['action'] == ''),
+        ['action', 'reason', 'action_order'],
+    ] = ['skip', 'copy_skip_but_delete_skipped', 90]
+
+    # dir level skip when exists in src but not in dst (nothing to delete)
+    plan.loc[
+        ~copy_skip_any
+        & plan['src_exists']
+        & ~plan['dst_exists']
+        & (plan['kind'] == 'dir')
+        & (plan['action'] == ''),
+        ['action', 'reason', 'action_order'],
+    ] = ['skip', 'dir_tree_skip', 95]
+
+    # -----------------------------
+    # 1) Normal copy/update (only undecided, only NOT copy-skip)
+    # -----------------------------
+    files_mask = (
+        (plan['kind'] == 'file')
+        & plan['src_exists']
+        & (plan['action'] == '')
+        & ~copy_skip_any
+        & ~(plan['src_excluded'] == True)  # noqa: E712
+    )
+
+    copy_mask = files_mask & ~plan['dst_exists']
+    plan.loc[copy_mask, ['action', 'reason', 'action_order']] = [
+        'copy',
+        'missing_in_dst',
+        10,
+    ]
+
+    update_candidates = files_mask & plan['dst_exists']
+
+    # FIX dtype warning
+    plan['_files_equal'] = pd.Series(pd.NA, index=plan.index, dtype='boolean')
+
+    eq_flags = [
+        _files_equal_by_stat(Path(sp), Path(dp))
+        for sp, dp in plan.loc[update_candidates, ['src_path', 'dst_path']].itertuples(
+            index=False
+        )
+    ]
+    plan.loc[update_candidates, '_files_equal'] = pd.Series(
+        eq_flags, index=plan.index[update_candidates], dtype='boolean'
+    )
+
+    update_mask = update_candidates & (plan['_files_equal'] == False)  # noqa: E712
+    plan.loc[update_mask, ['action', 'reason', 'action_order']] = [
+        'update',
+        'different',
+        20,
+    ]
+
+    same_mask = update_candidates & (plan['_files_equal'] == True)  # noqa: E712
+    plan.loc[same_mask, ['action', 'reason', 'action_order']] = ['skip', 'same', 90]
+
+    # -----------------------------
+    # 2) Delete extras in destination (dst exists, src missing), only undecided
+    # (copy-skip cleanup already handled above)
+    # -----------------------------
+    extra_in_dst = plan['dst_exists'] & ~plan['src_exists'] & (plan['action'] == '')
+
+    # delete-skip
+    delete_skip = extra_in_dst & (plan['_under_delete_skip_dir'] | is_delete_skip_file)
+    plan.loc[delete_skip, ['action', 'reason', 'action_order']] = [
+        'skip',
+        'delete_skip_rule',
+        90,
+    ]
+
+    # delete files
+    del_file_candidates = (
+        extra_in_dst & (plan['kind'] == 'file') & (plan['action'] == '')
+    )
+    plan.loc[
+        del_file_candidates & is_try_file, ['action', 'reason', 'action_order']
+    ] = ['delete_try', 'missing_in_src_try', 71]
+    plan.loc[
+        del_file_candidates & ~is_try_file, ['action', 'reason', 'action_order']
+    ] = ['delete', 'missing_in_src', 70]
+
+    # delete dirs (bottom-up sorting later)
+    del_dir_candidates = extra_in_dst & (plan['kind'] == 'dir') & (plan['action'] == '')
+    plan.loc[del_dir_candidates & is_try_dir, ['action', 'reason', 'action_order']] = [
+        'delete_dir_try',
+        'missing_in_src_try',
+        81,
+    ]
+    plan.loc[del_dir_candidates & ~is_try_dir, ['action', 'reason', 'action_order']] = [
+        'delete_dir',
+        'missing_in_src',
+        80,
+    ]
+
+    # -----------------------------
+    # 3) Anything still undecided -> skip
+    # -----------------------------
+    plan.loc[plan['action'] == '', ['action', 'reason', 'action_order']] = [
+        'skip',
+        'no_rule_matched',
+        90,
+    ]
+
+    # depth for bottom-up dir deletion
+    plan['_depth'] = rel_p.map(lambda p: len(p.parts))
+
+    # Execution order:
+    # copy (10), update (20), delete file (70/71), delete dir (80/81 deepest first), skip (90)
+    plan = plan.sort_values(
+        ['dest_root', 'action_order', '_depth'], ascending=[True, True, False]
+    ).reset_index(drop=True)
+
+    # cleanup helper columns (keep if you want to debug)
+    plan = plan.drop(
+        columns=[
+            '_under_copy_skip_dir',
+            '_is_copy_skip_file',
+            '_under_delete_skip_dir',
+            '_files_equal',
+            '_depth',
+        ],
+        errors='ignore',
+    )
+
+    return plan
+
+
+# =========================
+# Execute plan with fact columns
+# =========================
+
+
+def execute_plan(df: pd.DataFrame) -> tuple[pd.DataFrame, dict]:
+    df = df.copy()
+
+    # fact columns
+    df['result'] = (
+        ''  # copied / updated / deleted_file / deleted_dir / skipped / already_missing / try_delete_failed / error
+    )
+    df['error'] = ''  # exception text
+
+    summary = {
+        'copied_files': 0,
+        'updated_files': 0,
+        'deleted_files': 0,
+        'skipped_files': 0,
+        'deleted_dirs': 0,
+        'skipped_dirs': 0,
+        'try_delete_failed': 0,
+        'errors': 0,
+        'invalid_paths': 0,
+    }
+
+    def _get_str(val) -> str | None:
+        if val is None or pd.isna(val):
+            return None
+        return str(val)
+
+    for i, row in df.iterrows():
+        action = row.get('action', '')
+
+        try:
+            if action == 'skip':
+                if row.get('kind') == 'file':
+                    summary['skipped_files'] += 1
+                elif row.get('kind') == 'dir':
+                    summary['skipped_dirs'] += 1
+                df.at[i, 'result'] = 'skipped'
+                continue
+
+            if action in ('copy', 'update'):
+                src_s = _get_str(row.get('src_path'))
+                dst_s = _get_str(row.get('dst_path'))
+                if not src_s:
+                    summary['invalid_paths'] += 1
+                    summary['errors'] += 1
+                    df.at[i, 'result'] = 'error'
+                    df.at[i, 'error'] = 'missing src_path'
+                    continue
+                if not dst_s:
+                    dest_root = _get_str(row.get('dest_root', '')) or ''
+                    rel_path = _get_str(row.get('rel_path', '')) or ''
+                    dst_s = os.path.join(dest_root, rel_path)
+                if not dst_s:
+                    summary['invalid_paths'] += 1
+                    summary['errors'] += 1
+                    df.at[i, 'result'] = 'error'
+                    df.at[i, 'error'] = 'missing dst_path'
+                    continue
+
+                src = Path(src_s)
+                dst = Path(dst_s)
+                dst.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copy2(src, dst)
+
+                if action == 'copy':
+                    summary['copied_files'] += 1
+                    df.at[i, 'result'] = 'copied'
+                    # print(f'[COPY] {src} -> {dst}')
+                else:
+                    summary['updated_files'] += 1
+                    df.at[i, 'result'] = 'updated'
+                    # print(f'[UPDATE] {src} -> {dst}')
+                continue
+
+            if action in ('delete', 'delete_try'):
+                dst_s = _get_str(row.get('dst_path'))
+                if not dst_s:
+                    summary['invalid_paths'] += 1
+                    summary['errors'] += 1
+                    df.at[i, 'result'] = 'error'
+                    df.at[i, 'error'] = 'missing dst_path'
+                    continue
+
+                dst = Path(dst_s)
+                if not dst.exists():
+                    df.at[i, 'result'] = 'already_missing'
+                    continue
+
+                try:
+                    dst.unlink()
+                    summary['deleted_files'] += 1
+                    df.at[i, 'result'] = 'deleted_file'
+                    # print(f'[DELETE FILE] {dst}')
+                except Exception as e:
+                    if action == 'delete_try':
+                        summary['try_delete_failed'] += 1
+                        df.at[i, 'result'] = 'try_delete_failed'
+                        df.at[i, 'error'] = str(e)
+                        # print(f'[TRY DELETE FILE - SKIPPED] {dst} ({e})')
+                    else:
+                        raise
+                continue
+
+            if action in ('delete_dir', 'delete_dir_try'):
+                dst_s = _get_str(row.get('dst_path'))
+                if not dst_s:
+                    summary['invalid_paths'] += 1
+                    summary['errors'] += 1
+                    df.at[i, 'result'] = 'error'
+                    df.at[i, 'error'] = 'missing dst_path'
+                    continue
+
+                dst = Path(dst_s)
+                if not dst.exists():
+                    df.at[i, 'result'] = 'already_missing'
+                    continue
+
+                try:
+                    shutil.rmtree(dst)
+                    summary['deleted_dirs'] += 1
+                    df.at[i, 'result'] = 'deleted_dir'
+                    # print(f'[DELETE DIR] {dst}')
+                except Exception as e:
+                    if action == 'delete_dir_try':
+                        summary['try_delete_failed'] += 1
+                        df.at[i, 'result'] = 'try_delete_failed'
+                        df.at[i, 'error'] = str(e)
+                        # print(f'[TRY DELETE DIR - SKIPPED] {dst} ({e})')
+                    else:
+                        raise
+                continue
+
+            # unknown action
+            summary['errors'] += 1
+            df.at[i, 'result'] = 'error'
+            df.at[i, 'error'] = f'Unknown action: {action}'
+
+        except Exception as e:
+            summary['errors'] += 1
+            df.at[i, 'result'] = 'error'
+            df.at[i, 'error'] = str(e)
+            # print(f'[ERROR] action={action} rel={row.get("rel_path")} err={e}')
+
+    return df, summary
+
+
+def log_folder_retention(log_folder: str, retain_days: int = 0) -> None:
+    """Delete log files older than retain_days in the log_folder."""
+    cutoff_datetime = datetime.now() - pd.Timedelta(days=retain_days)
+
+    folder = Path(log_folder)
+    for pattern in ('sync_executed_*.csv', 'sync_plan_debug*.csv'):
+        for log_file in folder.glob(pattern):
+            try:
+                file_mtime = datetime.fromtimestamp(log_file.stat().st_mtime)
+                if file_mtime < cutoff_datetime:
+                    log_file.unlink()
+                    logging_utils.info(f'Deleted old log file: {log_file}')
+            except Exception as e:
+                logging_utils.error(f'Error checking/deleting log file {log_file}: {e}')
+
+
+def log_activity(df: pd.DataFrame, summary: dict, log_folder: str) -> None:
+    """Simple logger function to log activity messages with timestamps."""
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+    folder = Path(log_folder)
+    folder.mkdir(parents=True, exist_ok=True)
+    df.to_csv(folder / f'sync_executed_{timestamp}.csv', index=False)
+    summary_msg = '\n===== Summary ====='
+    for k, v in summary.items():
+        summary_msg += f'\n{k:18}: {v}'
+    summary_msg += '\n==================='
+    logging_utils.info(summary_msg)

al_data_utils/git_utils/__init__.py

@@ -0,0 +1,27 @@
+from .git_utils import (
+    commit_all_changes,
+    push_changes,
+    is_git_repo,
+    repo_root,
+    current_branch,
+    current_commit,
+    is_clean,
+    list_tags,
+    list_branches,
+    file_last_commit,
+    GitError,
+)
+
+__all__ = [
+    'commit_all_changes',
+    'push_changes',
+    'is_git_repo',
+    'repo_root',
+    'current_branch',
+    'current_commit',
+    'is_clean',
+    'list_tags',
+    'list_branches',
+    'file_last_commit',
+    'GitError',
+]