abstract-utilities 0.2.2.493__py3-none-any.whl → 0.2.2.496__py3-none-any.whl

abstract_utilities/file_utils/src/file_utils.py

@@ -0,0 +1,279 @@
+
+from .filter_params import *
+from ..imports import *
+##from abstract_utilities import make_list,get_media_exts, is_media_type
+def get_allowed_predicate(allowed=None):
+    if allowed != False:
+        if allowed == True:
+            allowed = None
+        allowed = allowed or make_allowed_predicate()
+    else:
+        def allowed(*args):
+            return True
+        allowed = allowed
+    return allowed
+def get_globs(items,recursive: bool = True,allowed=None):
+    glob_paths = []
+    items = [item for item in make_list(items) if item]
+    for item in items:
+        pattern = os.path.join(item, "**/*")  # include all files recursively\n
+        nuItems = glob.glob(pattern, recursive=recursive)
+        if allowed:
+            nuItems = [nuItem for nuItem in nuItems if nuItem and allowed(nuItem)]
+        glob_paths += nuItems
+    return glob_paths
+def get_allowed_files(items,allowed=True):
+    allowed = get_allowed_predicate(allowed=allowed)
+    return [item for item in items if item and os.path.isfile(item) and allowed(item)]
+def get_allowed_dirs(items,allowed=False):
+    allowed = get_allowed_predicate(allowed=allowed)
+    return [item for item in items if item and os.path.isdir(item) and allowed(item)]
+
+def get_filtered_files(items,allowed=None,files = []):
+    allowed = get_allowed_predicate(allowed=allowed)
+    glob_paths = get_globs(items)
+    return [glob_path for glob_path in glob_paths if glob_path and os.path.isfile(glob_path) and glob_path not in files and allowed(glob_path)]
+def get_filtered_dirs(items,allowed=None,dirs = []):
+    allowed = get_allowed_predicate(allowed=allowed)
+    glob_paths = get_globs(items)
+    return [glob_path for glob_path in glob_paths if glob_path and os.path.isdir(glob_path) and glob_path not in dirs and allowed(glob_path)]
+
+def get_all_allowed_files(items,allowed=None):
+    dirs = get_all_allowed_dirs(items)
+    files = get_allowed_files(items)
+    nu_files = []
+    for directory in dirs:
+        files += get_filtered_files(directory,allowed=allowed,files=files)
+    return files
+def get_all_allowed_dirs(items,allowed=None):
+    allowed = get_allowed_predicate(allowed=allowed)
+    dirs = get_allowed_dirs(items)
+    nu_dirs=[]
+    for directory in dirs:
+        nu_dirs += get_filtered_dirs(directory,allowed=allowed,dirs=nu_dirs)
+    return nu_dirs
+def get_files_and_dirs(
+    directory: str,
+    cfg: Optional["ScanConfig"] = None,
+    allowed_exts: Optional[Set[str]] = False,
+    unallowed_exts: Optional[Set[str]] = False,
+    allowed_types: Optional[Set[str]] = False,
+    exclude_types: Optional[Set[str]] = False,
+    allowed_dirs: Optional[List[str]] = False,
+    exclude_dirs: Optional[List[str]] = False,
+    allowed_patterns: Optional[List[str]] = False,
+    exclude_patterns: Optional[List[str]] = False,
+    add = False,
+    recursive: bool = True,
+    include_files: bool = True,
+    **kwargs
+    ):
+    cfg = cfg or define_defaults(
+            allowed_exts = allowed_exts,
+            unallowed_exts = unallowed_exts,
+            allowed_types = allowed_types,
+            exclude_types = exclude_types,
+            allowed_dirs = allowed_dirs,
+            exclude_dirs = exclude_dirs,
+            allowed_patterns = allowed_patterns,
+            exclude_patterns = exclude_patterns,
+        add=add
+        )
+    allowed = make_allowed_predicate(cfg)
+    items=[]
+    files =[]
+    if recursive:
+        items = get_globs(directory,recursive=recursive,allowed=allowed)
+    else:
+        directories = make_list(directory)
+        for directory in directories:
+            items +=[os.path.join(directory,item) for item in os.listdir(directory)]
+    dirs = get_allowed_dirs(items,allowed=allowed)
+    if include_files:
+        files = get_allowed_files(items,allowed=allowed)
+    return dirs,files
+def make_allowed_predicate(cfg: ScanConfig) -> Callable[[str], bool]:
+    """
+    Build a predicate that returns True if a given path is considered allowed
+    under the given ScanConfig. Applies allowed_* and exclude_* logic symmetrically.
+    """
+    def allowed(path: str) -> bool:
+        p = Path(path)
+        name = p.name.lower()
+        path_str = str(p).lower()
+
+        # --------------------
+        # A) directory filters
+        # --------------------
+        if cfg.exclude_dirs:
+            for dpat in cfg.exclude_dirs:
+                dpat_l = dpat.lower()
+                if dpat_l in path_str or fnmatch.fnmatch(name, dpat_l):
+                    if p.is_dir() or dpat_l in path_str:
+                        return False
+
+        if cfg.allowed_dirs and cfg.allowed_dirs != ["*"]:
+            # must be in at least one allowed dir
+            if not any(
+                fnmatch.fnmatch(path_str, f"*{dpat.lower()}*") for dpat in cfg.allowed_dirs
+            ):
+                return False
+
+        # --------------------
+        # B) pattern filters
+        # --------------------
+        if cfg.allowed_patterns and cfg.allowed_patterns != ["*"]:
+            if not any(fnmatch.fnmatch(name, pat.lower()) for pat in cfg.allowed_patterns):
+                return False
+
+        if cfg.exclude_patterns:
+            for pat in cfg.exclude_patterns:
+                if fnmatch.fnmatch(name, pat.lower()):
+                    return False
+
+        # --------------------
+        # C) extension filters
+        # --------------------
+        if p.is_file():
+            ext = p.suffix.lower()
+            if cfg.allowed_exts and ext not in cfg.allowed_exts:
+                return False
+            if cfg.unallowed_exts and ext in cfg.unallowed_exts:
+                return False
+
+        # --------------------
+        # D) type filters (optional)
+        # --------------------
+        if cfg.allowed_types and cfg.allowed_types != {"*"}:
+            if not any(t in path_str for t in cfg.allowed_types):
+                return False
+        if cfg.exclude_types and cfg.exclude_types != {"*"}:
+            if any(t in path_str for t in cfg.exclude_types):
+                return False
+
+        return True
+
+    return allowed
+def correct_kwargs(**kwargs):
+    for key,values in kwargs.items():
+        if key.startswith('excluded'):
+            post_fix = key.split('_')[-1]
+            correct_key = f'exclude_{post_fix}'
+            correct_vals = kwargs.get(correct_key)
+            kwargs[correct_key]=combine_params(correct_vals,values)
+            del kwargs[key]
+    
+def collect_filepaths(
+    directory: List[str],
+    cfg: ScanConfig=None,
+    allowed_exts: Optional[Set[str]] = False,
+    unallowed_exts: Optional[Set[str]] = False,
+    allowed_types: Optional[Set[str]] = False,
+    exclude_types: Optional[Set[str]] = False,
+    allowed_dirs: Optional[List[str]] = False,
+    exclude_dirs: Optional[List[str]] = False,
+    allowed_patterns: Optional[List[str]] = False,
+    exclude_patterns: Optional[List[str]] = False,
+    add=False,
+    allowed: Optional[Callable[[str], bool]] = None,
+    **kwargs
+    ) -> List[str]:
+    kwargs = correct_kwargs(allowed_exts = allowed_exts,
+        unallowed_exts = unallowed_exts,
+        allowed_types = allowed_types,
+        exclude_types = exclude_types,
+        allowed_dirs = allowed_dirs,
+        exclude_dirs = exclude_dirs,
+        allowed_patterns = allowed_patterns,
+        exclude_patterns = exclude_patterns,
+        **kwargs)
+    cfg = cfg or define_defaults(
+            allowed_exts = allowed_exts,
+            unallowed_exts = unallowed_exts,
+            allowed_types = allowed_types,
+            exclude_types = exclude_types,
+            allowed_dirs = allowed_dirs,
+            exclude_dirs = exclude_dirs,
+            allowed_patterns = allowed_patterns,
+            exclude_patterns = exclude_patterns,
+                                add = add
+                                )
+    allowed = allowed or make_allowed_predicate(cfg)
+    directories = make_list(directory)
+    roots = [r for r in directories if r]
+
+    # your existing helpers (get_dirs, get_globs, etc.) stay the same
+    original_dirs = get_allowed_dirs(roots, allowed=allowed)
+    original_globs = get_globs(original_dirs)
+    files = get_allowed_files(original_globs, allowed=allowed)
+
+    for d in get_filtered_dirs(original_dirs, allowed=allowed):
+        files += get_filtered_files(d, allowed=allowed, files=files)
+
+    # de-dupe while preserving order
+    seen, out = set(), []
+    for f in files:
+        if f not in seen:
+            seen.add(f)
+            out.append(f)
+    return out
+
+
+def _fast_walk(
+    root: Path,
+    exts: Iterable[str],
+    skip_dirs: Iterable[str] = (),
+    skip_patterns: Iterable[str] = (),
+) -> List[Path]:
+    exts = tuple(exts)
+    skip_dirs = set(sd.lower() for sd in skip_dirs or ())
+    skip_patterns = tuple(sp.lower() for sp in (skip_patterns or ()))
+
+    out = []
+    for p in root.rglob("*"):
+        # skip directories by name hit
+        if p.is_dir():
+            name = p.name.lower()
+            if name in skip_dirs:
+                # rglob doesn't let us prune mid-iteration cleanly; we just won't collect under it
+                continue
+            # nothing to collect for dirs
+            continue
+
+        # file filters
+        name = p.name.lower()
+        if any(fnmatch.fnmatch(name, pat) for pat in skip_patterns):
+            continue
+        if p.suffix.lower() in exts:
+            out.append(p)
+
+    # de-dup and normalize
+    return sorted({pp.resolve() for pp in out})
+
+
+def enumerate_source_files(
+    src_root: Path,
+    cfg: Optional["ScanConfig"] = None,
+    *,
+    exts: Optional[Iterable[str]] = None,
+    fast_skip_dirs: Optional[Iterable[str]] = None,
+    fast_skip_patterns: Optional[Iterable[str]] = None,
+) -> List[Path]:
+    """
+    Unified enumerator:
+      - If `cfg` is provided: use collect_filepaths(...) with full rules.
+      - Else: fast walk using rglob over `exts` (defaults to EXTS) with optional light excludes.
+    """
+    src_root = Path(src_root)
+
+    if cfg is not None:
+        files = collect_filepaths([str(src_root)], cfg=cfg)
+        return sorted({Path(f).resolve() for f in files})
+
+    # Fast mode
+    return _fast_walk(
+        src_root,
+        exts or EXTS,
+        skip_dirs=fast_skip_dirs or (),
+        skip_patterns=fast_skip_patterns or (),
+    )

abstract_utilities/file_utils/src/filter_params.py

@@ -0,0 +1,155 @@
+
+from ..imports import *
+import re
+def combine_params(*values,typ=None):
+    nu_values = None
+    for value in values:
+        if value is not None:
+            if nu_values is None:
+                typ = typ or type(value)
+                nu_values = typ()
+            else:
+                if val_type is set:
+                    nu_values = nu_values | typ(value)
+                if val_type is list:
+                    nu_values += typ(value)
+    return nu_values
+# -------------------------
+# Default sets
+# -------------------------
+
+DEFAULT_ALLOWED_EXTS: Set[str] = {
+    ".py", ".pyw", ".js", ".jsx", ".ts", ".tsx", ".mjs",
+    ".html", ".htm", ".xml", ".css", ".scss", ".sass", ".less",
+    ".json", ".yaml", ".yml", ".toml", ".ini", ".cfg",
+    ".md", ".markdown", ".rst", ".sh", ".bash", ".env", ".txt"
+}
+
+DEFAULT_EXCLUDE_TYPES: Set[str] = {
+    "image", "video", "audio", "presentation",
+    "spreadsheet", "archive", "executable"
+}
+
+_unallowed = set(get_media_exts(DEFAULT_EXCLUDE_TYPES)) | {
+    ".bak", ".shp", ".cpg", ".dbf", ".shx", ".geojson",
+    ".pyc", ".prj", ".sbn", ".sbx"
+}
+DEFAULT_UNALLOWED_EXTS = {e for e in _unallowed if e not in DEFAULT_ALLOWED_EXTS}
+
+DEFAULT_EXCLUDE_DIRS: List[str] = [
+    "node_modules", "__pycache__", "backups", "backup",
+    "trash", "deprecated", "old", "__init__"
+]
+
+DEFAULT_EXCLUDE_PATTERNS: List[str] = [
+    "__init__*", "*.tmp", "*.log", "*.lock", "*.zip", "*~"
+]
+
+DEFAULT_ALLOWED_PATTERNS: List[str] = ["*"]
+DEFAULT_ALLOWED_DIRS: List[str] = ["*"]
+DEFAULT_ALLOWED_TYPES: List[str] = ["*"]
+
+REMOTE_RE = re.compile(r"^(?P<host>[^:\s]+@[^:\s]+):(?P<path>/.*)$")
+AllowedPredicate = Optional[Callable[[str], bool]]
+
+# -------------------------
+# Config dataclass
+# -------------------------
+
+@dataclass
+class ScanConfig:
+    allowed_exts: Set[str]
+    unallowed_exts: Set[str]
+    allowed_types: Set[str]
+    exclude_types: Set[str]
+    allowed_dirs: List[str] = field(default_factory=list)
+    exclude_dirs: List[str] = field(default_factory=list)
+    allowed_patterns: List[str] = field(default_factory=list)
+    exclude_patterns: List[str] = field(default_factory=list)
+
+# -------------------------
+# Utility functions
+# -------------------------
+
+def _normalize_listlike(value, typ=list, sep=','):
+    """Normalize comma-separated or iterable values into the desired type."""
+    if value in [True, None, False]:
+        return value
+    if isinstance(value, str):
+        value = [v.strip() for v in value.split(sep) if v.strip()]
+    return typ(value)
+
+def _ensure_exts(exts):
+    if exts in [True, None, False]:
+        return exts
+    out = []
+    for ext in _normalize_listlike(exts, list):
+        if not ext.startswith('.'):
+            ext = f".{ext}"
+        out.append(ext)
+    return set(out)
+
+def _ensure_patterns(patterns):
+    """Normalize pattern list and ensure they are valid globs."""
+    if patterns in [True, None, False]:
+        return patterns
+    patterns = _normalize_listlike(patterns, list)
+    out = []
+    for pattern in patterns:
+        if not pattern:
+            continue
+        if '*' not in pattern and '?' not in pattern:
+            # Implicitly make it a prefix match
+            if pattern.startswith('.') or pattern.startswith('~'):
+                pattern = f"*{pattern}"
+            else:
+                pattern = f"{pattern}*"
+        out.append(pattern)
+    return out
+
+
+def _get_default_modular(value, default, add=False, typ=set):
+    """Merge user and default values intelligently."""
+    if value in [True, None]:
+        return default
+    if value is False:
+        return typ()
+    if add:
+        return combine_params(value,default,typ=None)
+
+    return typ(value)
+
+# -------------------------
+# Default derivation logic
+# -------------------------
+
+def derive_file_defaults(
+    allowed_exts=False, unallowed_exts=False,
+    allowed_types=False, exclude_types=False,
+    allowed_dirs=False, exclude_dirs=False,
+    allowed_patterns=False, exclude_patterns=False,
+    add=False
+):
+    allowed_exts = _get_default_modular(_ensure_exts(allowed_exts), DEFAULT_ALLOWED_EXTS, add, set)
+    unallowed_exts = _get_default_modular(_ensure_exts(unallowed_exts), DEFAULT_UNALLOWED_EXTS, add, set)
+    allowed_types = _get_default_modular(_normalize_listlike(allowed_types, set), DEFAULT_ALLOWED_TYPES, add, set)
+    exclude_types = _get_default_modular(_normalize_listlike(exclude_types, set), DEFAULT_EXCLUDE_TYPES, add, set)
+    allowed_dirs = _get_default_modular(_normalize_listlike(allowed_dirs, list), DEFAULT_ALLOWED_DIRS, add, list)
+    exclude_dirs = _get_default_modular(_normalize_listlike(exclude_dirs, list), DEFAULT_EXCLUDE_DIRS, add, list)
+    allowed_patterns = _get_default_modular(_ensure_patterns(allowed_patterns), DEFAULT_ALLOWED_PATTERNS, add, list)
+    exclude_patterns = _get_default_modular(_ensure_patterns(exclude_patterns), DEFAULT_EXCLUDE_PATTERNS, add, list)
+
+    return {
+        "allowed_exts": allowed_exts,
+        "unallowed_exts": unallowed_exts,
+        "allowed_types": allowed_types,
+        "exclude_types": exclude_types,
+        "allowed_dirs": allowed_dirs,
+        "exclude_dirs": exclude_dirs,
+        "allowed_patterns": allowed_patterns,
+        "exclude_patterns": exclude_patterns,
+    }
+
+def define_defaults(**kwargs):
+    defaults = derive_file_defaults(**kwargs)
+    return ScanConfig(**defaults)

abstract_utilities/file_utils/src/find_collect.py

@@ -0,0 +1,154 @@
+from ..imports import *
+from .filter_params import *
+from .file_filters import enumerate_source_files
+
+
+def check_path_type(
+    path: str,
+    user: Optional[str] = None,
+    host: Optional[str] = None,
+    user_as_host: Optional[str] = None,
+    use_shell: bool = False
+) -> Literal["file", "directory", "missing", "unknown"]:
+    """
+    Determine whether a given path is a file, directory, or missing.
+    Works locally or remotely (via SSH).
+
+    Args:
+        path: The path to check.
+        user, host, user_as_host: SSH parameters if remote.
+        use_shell: Force shell test instead of Python os.path.
+    Returns:
+        One of: 'file', 'directory', 'missing', or 'unknown'
+    """
+
+    # --- remote check if user/host is given ---
+    if user_as_host or (user and host):
+        remote_target = user_as_host or f"{user}@{host}"
+        cmd = f"if [ -f '{path}' ]; then echo file; elif [ -d '{path}' ]; then echo directory; else echo missing; fi"
+        try:
+            result = subprocess.check_output(
+                ["ssh", remote_target, cmd],
+                stderr=subprocess.DEVNULL,
+                text=True,
+                timeout=5
+            ).strip()
+            return result if result in ("file", "directory", "missing") else "unknown"
+        except Exception:
+            return "unknown"
+
+    # --- local check ---
+    if not use_shell:
+        if os.path.isfile(path):
+            return "file"
+        elif os.path.isdir(path):
+            return "directory"
+        elif not os.path.exists(path):
+            return "missing"
+        return "unknown"
+    else:
+        # fallback using shell tests (useful for sandboxed contexts)
+        cmd = f"if [ -f '{path}' ]; then echo file; elif [ -d '{path}' ]; then echo directory; else echo missing; fi"
+        try:
+            output = subprocess.check_output(
+                cmd, shell=True, stderr=subprocess.DEVNULL, text=True
+            ).strip()
+            return output if output in ("file", "directory", "missing") else "unknown"
+        except Exception:
+            return "unknown"
+
+
+
+
+def get_find_cmd(
+    directory: str,
+    *,
+    mindepth: Optional[int] = None,
+    maxdepth: Optional[int] = None,
+    depth: Optional[int] = None,
+    file_type: Optional[str] = None,  # 'f' or 'd'
+    name: Optional[str] = None,
+    size: Optional[str] = None,
+    mtime: Optional[str] = None,
+    perm: Optional[str] = None,
+    user: Optional[str] = None,
+    **kwargs
+) -> str:
+    """Constructs a Unix `find` command string from keyword args."""
+    cmd = [f"find {directory}"]
+
+    if depth is not None:
+        cmd += [f"-mindepth {depth}", f"-maxdepth {depth}"]
+    else:
+        if mindepth is not None:
+            cmd.append(f"-mindepth {mindepth}")
+        if maxdepth is not None:
+            cmd.append(f"-maxdepth {maxdepth}")
+
+    if file_type in ("f", "d"):
+        cmd.append(f"-type {file_type}")
+    if name:
+        cmd.append(f"-name '{name}'")
+    if size:
+        cmd.append(f"-size {size}")
+    if mtime:
+        cmd.append(f"-mtime {mtime}")
+    if perm:
+        cmd.append(f"-perm {perm}")
+    if user:
+        cmd.append(f"-user {user}")
+
+    return " ".join(cmd)
+
+
+def collect_globs(
+    directory: str,
+    cfg: Optional["ScanConfig"] = None,
+    *,
+    exts: Optional[Set[str]] = None,
+    patterns: Optional[List[str]] = None,
+    mindepth: Optional[int] = None,
+    maxdepth: Optional[int] = None,
+    depth: Optional[int] = None,
+    file_type: Optional[str] = None,
+    user_at_host: Optional[str] = None,
+    add: bool = False,
+    **kwargs
+) -> List[str]:
+    """
+    Collect file or directory paths using either:
+      - local recursive logic (rglob)
+      - or remote shell call (find via run_cmd)
+    """
+    cfg = cfg or define_defaults(add=add)
+    directory = str(directory)
+    exts = ensure_exts(exts)
+    patterns = ensure_patterns(patterns)
+
+    # Remote path via SSH
+    if user_at_host:
+        find_cmd = get_find_cmd(
+            directory,
+            mindepth=mindepth,
+            maxdepth=maxdepth,
+            depth=depth,
+            file_type=file_type,
+            **{k: v for k, v in kwargs.items() if v},
+        )
+        return run_cmd(find_cmd, user_at_host=user_at_host)
+
+    # Local path (Python-native walk)
+    root = Path(directory)
+    results = []
+    for p in root.rglob("*"):
+        if file_type == "f" and not p.is_file():
+            continue
+        if file_type == "d" and not p.is_dir():
+            continue
+        if exts and p.suffix.lower() not in exts:
+            continue
+        if patterns and not any(p.match(pat) for pat in patterns):
+            continue
+        results.append(str(p.resolve()))
+
+    return sorted(results)