abstract-utilities 0.2.2.513__py3-none-any.whl → 0.2.2.583__py3-none-any.whl

abstract_utilities/file_utils/src/filter_params.py

@@ -5,68 +5,78 @@ def combine_params(*values,typ=None):
     nu_values = None
     for value in values:
         if value is not None:
+            typ = typ or type(value)
             if nu_values is None:
-                typ = typ or type(value)
                 nu_values = typ()
-            else:
-                if val_type is set:
-                    nu_values = nu_values | typ(value)
-                if val_type is list:
-                    nu_values += typ(value)
+            
+            if typ is set:
+                nu_values = nu_values | typ(value)
+            if typ is list:
+                nu_values += typ(value)
     return nu_values
-# -------------------------
-# Default sets
-# -------------------------
-
-DEFAULT_ALLOWED_EXTS: Set[str] = {
-    ".py", ".pyw", ".js", ".jsx", ".ts", ".tsx", ".mjs",
-    ".html", ".htm", ".xml", ".css", ".scss", ".sass", ".less",
-    ".json", ".yaml", ".yml", ".toml", ".ini", ".cfg",
-    ".md", ".markdown", ".rst", ".sh", ".bash", ".env", ".txt"
-}
-
-DEFAULT_EXCLUDE_TYPES: Set[str] = {
-    "image", "video", "audio", "presentation",
-    "spreadsheet", "archive", "executable"
-}
-
-_unallowed = set(get_media_exts(DEFAULT_EXCLUDE_TYPES)) | {
-    ".bak", ".shp", ".cpg", ".dbf", ".shx", ".geojson",
-    ".pyc", ".prj", ".sbn", ".sbx"
-}
-DEFAULT_UNALLOWED_EXTS = {e for e in _unallowed if e not in DEFAULT_ALLOWED_EXTS}
-
-DEFAULT_EXCLUDE_DIRS: List[str] = [
-    "node_modules", "__pycache__", "backups", "backup",
-    "trash", "deprecated", "old", "__init__"
-]
-
-DEFAULT_EXCLUDE_PATTERNS: List[str] = [
-    "__init__*", "*.tmp", "*.log", "*.lock", "*.zip", "*~"
-]
-
-DEFAULT_ALLOWED_PATTERNS: List[str] = ["*"]
-DEFAULT_ALLOWED_DIRS: List[str] = ["*"]
-DEFAULT_ALLOWED_TYPES: List[str] = ["*"]
-
-REMOTE_RE = re.compile(r"^(?P<host>[^:\s]+@[^:\s]+):(?P<path>/.*)$")
-AllowedPredicate = Optional[Callable[[str], bool]]
-
-# -------------------------
-# Config dataclass
-# -------------------------
+def get_safe_kwargs(canonical_map, **kwargs):
+    # Lowercase all keys for safety
+    canonical_map = canonical_map or CANONICAL_MAP
+    norm_kwargs = {k.lower(): v for k, v in kwargs.items() if v is not None}
+
+    # Inverse lookup: alias → canonical key
+    alias_lookup = {
+        alias: canon
+        for canon, aliases in canonical_map.items()
+        if aliases
+        for alias in aliases
+    }
 
-@dataclass
-class ScanConfig:
-    allowed_exts: Set[str]
-    unallowed_exts: Set[str]
-    allowed_types: Set[str]
-    exclude_types: Set[str]
-    allowed_dirs: List[str] = field(default_factory=list)
-    exclude_dirs: List[str] = field(default_factory=list)
-    allowed_patterns: List[str] = field(default_factory=list)
-    exclude_patterns: List[str] = field(default_factory=list)
+    # Preserve correctly named keys
+    safe_kwargs = {k: v for k, v in norm_kwargs.items() if k in canonical_map}
 
+    for k, v in norm_kwargs.items():
+        if k in alias_lookup:
+            canonical_key = alias_lookup[k]
+            prev = safe_kwargs.get(canonical_key)
+            if prev is None:
+                safe_kwargs[canonical_key] = v
+            else:
+                # merge intelligently if both exist
+                if isinstance(prev, (set, list)) and isinstance(v, (set, list)):
+                    safe_kwargs[canonical_key] = list(set(prev) | set(v))
+                else:
+                    safe_kwargs[canonical_key] = v  # overwrite for non-iterables
+
+    # fill defaults if missing
+    for canon in canonical_map:
+        safe_kwargs.setdefault(canon, None)
+
+    return safe_kwargs
+
+def create_canonical_map(*args,canonical_map=None):
+    keys = [arg for arg in args if arg]
+    if not keys:
+        return CANONICAL_MAP
+    canonical_map = canonical_map or CANONICAL_MAP
+
+    return {key:canonical_map.get(key) for key in keys}
+def get_safe_canonical_kwargs(*args,canonical_map=None,**kwargs):
+    canonical_map = canonical_map or create_canonical_map(*args)
+    
+    return get_safe_kwargs(canonical_map=canonical_map,**kwargs)    
+def get_dir_filter_kwargs(**kwargs):
+    canonical_map = create_canonical_map("directories")
+    return get_safe_kwargs(canonical_map=canonical_map,**kwargs)
+def get_file_filter_kwargs(**kwargs):
+    """
+    Normalize arbitrary keyword arguments for file scanning configuration.
+    
+    Examples:
+      - 'excluded_ext' or 'unallowed_exts' → 'exclude_exts'
+      - 'include_dirs' or 'allow_dir' → 'allowed_dirs'
+      - 'excludePattern' or 'excluded_patterns' → 'exclude_patterns'
+      - 'allowed_type' or 'include_types' → 'allowed_types'
+    """
+    # Canonical keys and aliases
+    canonical_keys =["allowed_exts","exclude_exts","allowed_types","exclude_types","allowed_dirs","exclude_dirs","allowed_patterns","exclude_patterns"]
+   
+    return get_safe_canonical_kwargs(*canonical_keys,**kwargs)
 # -------------------------
 # Utility functions
 # -------------------------
@@ -110,10 +120,12 @@ def ensure_patterns(patterns):
 
 def _get_default_modular(value, default, add=False, typ=set):
     """Merge user and default values intelligently."""
-    if value in [True, None]:
+    if value == None:
+        value = add
+    if value in [True]:
         return default
     if value is False:
-        return typ()
+        return value
     if add:
         return combine_params(value,default,typ=None)
 
@@ -122,33 +134,63 @@ def _get_default_modular(value, default, add=False, typ=set):
 # -------------------------
 # Default derivation logic
 # -------------------------
-
-def derive_file_defaults(
-    allowed_exts=False, unallowed_exts=False,
-    allowed_types=False, exclude_types=False,
-    allowed_dirs=False, exclude_dirs=False,
-    allowed_patterns=False, exclude_patterns=False,
-    add=False
-):
-    allowed_exts = _get_default_modular(ensure_exts(allowed_exts), DEFAULT_ALLOWED_EXTS, add, set)
-    unallowed_exts = _get_default_modular(ensure_exts(unallowed_exts), DEFAULT_UNALLOWED_EXTS, add, set)
-    allowed_types = _get_default_modular(_normalize_listlike(allowed_types, set), DEFAULT_ALLOWED_TYPES, add, set)
-    exclude_types = _get_default_modular(_normalize_listlike(exclude_types, set), DEFAULT_EXCLUDE_TYPES, add, set)
-    allowed_dirs = _get_default_modular(_normalize_listlike(allowed_dirs, list), DEFAULT_ALLOWED_DIRS, add, list)
-    exclude_dirs = _get_default_modular(_normalize_listlike(exclude_dirs, list), DEFAULT_EXCLUDE_DIRS, add, list)
-    allowed_patterns = _get_default_modular(ensure_patterns(allowed_patterns), DEFAULT_ALLOWED_PATTERNS, add, list)
-    exclude_patterns = _get_default_modular(ensure_patterns(exclude_patterns), DEFAULT_EXCLUDE_PATTERNS, add, list)
-
-    return {
-        "allowed_exts": allowed_exts,
-        "unallowed_exts": unallowed_exts,
-        "allowed_types": allowed_types,
-        "exclude_types": exclude_types,
-        "allowed_dirs": allowed_dirs,
-        "exclude_dirs": exclude_dirs,
-        "allowed_patterns": allowed_patterns,
-        "exclude_patterns": exclude_patterns,
-    }
+def _get_default_modular(value, default, add=None, typ=set):
+    """Merge user and default values intelligently."""
+    add = add or False
+    if value == None:
+        value = add
+    if value in [True]:
+        return default
+    if value is False:
+        return value
+    if add:
+        return combine_params(value,default,typ=None)
+    return typ(value)
+def derive_all_defaults(**kwargs):
+    kwargs = get_safe_canonical_kwargs(**kwargs)
+    add = kwargs.get("add",False)
+    nu_defaults = {}
+    for key,values in DEFAULT_CANONICAL_MAP.items():
+        default = values.get("default")
+        typ = values.get("type")
+        key_value = kwargs.get(key)
+        if key in DEFAULT_ALLOWED_EXCLUDE_MAP:
+
+            if key.endswith('exts'):
+                input_value = ensure_exts(key_value)
+            if key.endswith('patterns'):
+                input_value = ensure_patterns(key_value)
+            else:
+                input_value = _normalize_listlike(key_value, typ)
+            nu_defaults[key] = _get_default_modular(input_value, default, add, typ)
+        else:
+            value = default if key_value is None else key_value
+            if typ == list:
+                value = make_list(value)
+            elif typ == bool:
+                value = bool(value)
+            nu_defaults[key] = value
+   
+    return nu_defaults
+# -------------------------
+# Default derivation logic
+# -------------------------
+def derive_file_defaults(**kwargs):
+    kwargs = derive_all_defaults(**kwargs)
+    add = kwargs.get("add",False)
+    nu_defaults = {}
+    for key,values in DEFAULT_ALLOWED_EXCLUDE_MAP.items():
+        default = values.get("default")
+        typ = values.get("type")
+        key_value = kwargs.get(key)
+        if key.endswith('exts'):
+            input_value = ensure_exts(key_value)
+        if key.endswith('patterns'):
+            input_value = ensure_patterns(key_value)
+        else:
+            input_value = _normalize_listlike(key_value, typ)
+        nu_defaults[key] = _get_default_modular(input_value, default, add, typ)
+    return nu_defaults
 
 def define_defaults(**kwargs):
     defaults = derive_file_defaults(**kwargs)

abstract_utilities/file_utils/src/find_collect.py

@@ -1,79 +1,15 @@
 from ..imports import *
-from .filter_params import *
-from .file_filters import enumerate_source_files
+from .file_filters import *
 from pathlib import Path
 from typing import Optional, List, Set
 
-def check_path_type(
-    path: str,
-    user: Optional[str] = None,
-    host: Optional[str] = None,
-    user_as_host: Optional[str] = None,
-    use_shell: bool = False
-) -> Literal["file", "directory", "missing", "unknown"]:
-    """
-    Determine whether a given path is a file, directory, or missing.
-    Works locally or remotely (via SSH).
 
-    Args:
-        path: The path to check.
-        user, host, user_as_host: SSH parameters if remote.
-        use_shell: Force shell test instead of Python os.path.
-    Returns:
-        One of: 'file', 'directory', 'missing', or 'unknown'
-    """
-
-    # --- remote check if user/host is given ---
-    if user_as_host or (user and host):
-        remote_target = user_as_host or f"{user}@{host}"
-        cmd = f"if [ -f '{path}' ]; then echo file; elif [ -d '{path}' ]; then echo directory; else echo missing; fi"
-        try:
-            result = subprocess.check_output(
-                ["ssh", remote_target, cmd],
-                stderr=subprocess.DEVNULL,
-                text=True,
-                timeout=5
-            ).strip()
-            return result if result in ("file", "directory", "missing") else "unknown"
-        except Exception:
-            return "unknown"
-
-    # --- local check ---
-    if not use_shell:
-        if os.path.isfile(path):
-            return "file"
-        elif os.path.isdir(path):
-            return "directory"
-        elif not os.path.exists(path):
-            return "missing"
-        return "unknown"
-    else:
-        # fallback using shell tests (useful for sandboxed contexts)
-        cmd = f"if [ -f '{path}' ]; then echo file; elif [ -d '{path}' ]; then echo directory; else echo missing; fi"
-        try:
-            output = subprocess.check_output(
-                cmd, shell=True, stderr=subprocess.DEVNULL, text=True
-            ).strip()
-            return output if output in ("file", "directory", "missing") else "unknown"
-        except Exception:
-            return "unknown"
 
 
 
 
 def get_find_cmd(
-    directory: str,
-    *,
-    cfg: Optional["ScanConfig"] = None,
-    *,
-    allowed_exts: Optional[Set[str]] = False,
-    unallowed_exts: Optional[Set[str]] = False,
-    allowed_types: Optional[Set[str]] = False,
-    exclude_types: Optional[Set[str]] = False,
-    allowed_dirs: Optional[List[str]] = False,
-    exclude_dirs: Optional[List[str]] = False,
-    allowed_patterns: Optional[List[str]] = False,
-    exclude_patterns: Optional[List[str]] = False,
+    *args,
     mindepth: Optional[int] = None,
     maxdepth: Optional[int] = None,
     depth: Optional[int] = None,
@@ -86,14 +22,23 @@ def get_find_cmd(
     **kwargs
 ) -> str:
     """
-    Construct a Unix `find` command string from cfg and explicit keyword args.
-
-    - Honors allowed/excluded patterns, dirs, and extensions from ScanConfig.
-    - Automatically applies mindepth/maxdepth/depth/file_type filters.
+    Construct a Unix `find` command string that supports multiple directories.
+    Accepts filtering via ScanConfig-compatible kwargs.
     """
-    cmd = [f"find {shlex.quote(directory)}"]
+    # Normalize inputs into canonical form
+    kwargs = get_safe_canonical_kwargs(*args, **kwargs)
+    cfg = kwargs.get('cfg') or define_defaults(**kwargs)
 
-    # --- depth ---
+    # Get directory list (may come from args or kwargs)
+    kwargs["directories"] = ensure_directories(*args, **kwargs)
+    if not kwargs["directories"]:
+        return []
+
+    # Build base command for all directories
+    dir_expr = " ".join(shlex.quote(d) for d in kwargs["directories"])
+    cmd = [f"find {dir_expr}"]
+
+    # --- depth filters ---
     if depth is not None:
         cmd += [f"-mindepth {depth}", f"-maxdepth {depth}"]
     else:
@@ -106,7 +51,7 @@ def get_find_cmd(
     if file_type in ("f", "d"):
         cmd.append(f"-type {file_type}")
 
-    # --- base attributes ---
+    # --- basic attributes ---
     if name:
         cmd.append(f"-name {shlex.quote(name)}")
     if size:
@@ -117,63 +62,53 @@ def get_find_cmd(
         cmd.append(f"-perm {shlex.quote(perm)}")
     if user:
         cmd.append(f"-user {shlex.quote(user)}")
-    cfg = cfg or define_defaults(
-            allowed_exts = allowed_exts,
-            unallowed_exts = unallowed_exts,
-            allowed_types = allowed_types,
-            exclude_types = exclude_types,
-            allowed_dirs = allowed_dirs,
-            exclude_dirs = exclude_dirs,
-            allowed_patterns = allowed_patterns,
-            exclude_patterns = exclude_patterns,
-            add = add
-        )
+
     # --- cfg-based filters ---
     if cfg:
-        # allowed extensions
+        # Allowed extensions
         if cfg.allowed_exts and cfg.allowed_exts != {"*"}:
             ext_expr = " -o ".join(
                 [f"-name '*{e}'" for e in cfg.allowed_exts if e]
             )
             cmd.append(f"\\( {ext_expr} \\)")
 
-        # disallowed extensions
-        if cfg.unallowed_exts:
-            for e in cfg.unallowed_exts:
+        # Excluded extensions
+        if cfg.exclude_exts:
+            for e in cfg.exclude_exts:
                 cmd.append(f"! -name '*{e}'")
 
-        # allowed directories (match any path)
+        # Allowed directories
         if cfg.allowed_dirs and cfg.allowed_dirs != ["*"]:
             dir_expr = " -o ".join(
                 [f"-path '*{d}*'" for d in cfg.allowed_dirs if d]
             )
             cmd.append(f"\\( {dir_expr} \\)")
 
-        # exclude directories
+        # Excluded directories
         if cfg.exclude_dirs:
             for d in cfg.exclude_dirs:
                 cmd.append(f"! -path '*{d}*'")
 
-        # allowed patterns
+        # Allowed patterns
         if cfg.allowed_patterns and cfg.allowed_patterns != ["*"]:
             pat_expr = " -o ".join(
                 [f"-name '{p}'" for p in cfg.allowed_patterns if p]
             )
             cmd.append(f"\\( {pat_expr} \\)")
 
-        # exclude patterns
+        # Excluded patterns
         if cfg.exclude_patterns:
             for p in cfg.exclude_patterns:
                 cmd.append(f"! -name '{p}'")
 
-        # allowed types
+        # Allowed types (semantic, not `-type`)
         if cfg.allowed_types and cfg.allowed_types != {"*"}:
             type_expr = " -o ".join(
                 [f"-path '*{t}*'" for t in cfg.allowed_types if t]
             )
             cmd.append(f"\\( {type_expr} \\)")
 
-        # excluded types
+        # Excluded types
         if cfg.exclude_types:
             for t in cfg.exclude_types:
                 cmd.append(f"! -path '*{t}*'")
@@ -183,24 +118,12 @@ def get_find_cmd(
 
 
 def collect_globs(
-    directory: str,
-    cfg: Optional["ScanConfig"] = None,
-    *,
-    allowed_exts: Optional[Set[str]] = False,
-    unallowed_exts: Optional[Set[str]] = False,
-    allowed_types: Optional[Set[str]] = False,
-    exclude_types: Optional[Set[str]] = False,
-    allowed_dirs: Optional[List[str]] = False,
-    exclude_dirs: Optional[List[str]] = False,
-    allowed_patterns: Optional[List[str]] = False,
-    exclude_patterns: Optional[List[str]] = False,
-    patterns: Optional[List[str]] = None,
+    *args,
     mindepth: Optional[int] = None,
     maxdepth: Optional[int] = None,
     depth: Optional[int] = None,
     file_type: Optional[str] = None,   # "f", "d", or None
-    user_at_host: Optional[str] = None,
-    add: bool = False,
+    allowed: Optional[Callable[[str], bool]] = None,
     **kwargs
 ) -> List[str] | dict:
     """
@@ -210,61 +133,58 @@ def collect_globs(
     - If file_type is "f" or "d" → returns a list of that type
     - Supports SSH mode via `user_at_host`
     """
-    cfg = cfg or define_defaults(
-            allowed_exts = allowed_exts,
-            unallowed_exts = unallowed_exts,
-            allowed_types = allowed_types,
-            exclude_types = exclude_types,
-            allowed_dirs = allowed_dirs,
-            exclude_dirs = exclude_dirs,
-            allowed_patterns = allowed_patterns,
-            exclude_patterns = exclude_patterns,
-            add = add
-        )
-    allowed = allowed or make_allowed_predicate(cfg)
-    directory = str(directory)
-
-    # Remote path (SSH)
-    if user_at_host:
+    user_pass_host_key = get_user_pass_host_key(**kwargs)
+    kwargs["directories"] = ensure_directories(*args, **kwargs)
+    kwargs= get_safe_canonical_kwargs(**kwargs)
+    kwargs["cfg"] = kwargs.get('cfg') or define_defaults(**kwargs)
+    
+    type_strs = {"f":"files","d":"dirs"}
+    file_type = get_proper_type_str(file_type)
+    file_types = make_list(file_type)
+    if file_type == None:
+        file_types = ["f","d"]
+    return_results = {}
+    return_result=[]
+    for file_type in file_types:
+        type_str = type_strs.get(file_type)
+        # Remote path (SSH)
         find_cmd = get_find_cmd(
-            directory,
-            cfg=cfg,
-            mindepth=mindepth,
-            maxdepth=maxdepth,
-            depth=depth,
-            file_type=file_type,
-            **{k: v for k, v in kwargs.items() if v},
-        )
-        return run_cmd(find_cmd, user_at_host=user_at_host)
-
-    # Local path (Python-native walk)
-    root = Path(directory)
-    results_js = {"f": [], "d": []}
-
-    for p in root.rglob("*"):
-        if p.is_file():
-            kind = "f"
-        elif p.is_dir():
-            kind = "d"
-        else:
-            continue
-
-        # If file_type is specified, skip the other kind
-        if file_type and kind != file_type:
-            continue
-
-        if exts and kind == "f" and p.suffix.lower() not in exts:
-            continue
-
-        if patterns and not any(p.match(pat) for pat in patterns):
-            continue
-
-        results_js[kind].append(str(p))
-
-    # Return based on selection
-    if file_type is None:
-        return results_js
-    else:
-        return results_js[file_type]
-
-
+            directories=kwargs.get("directories"),
+            cfg=kwargs.get('cfg'),
+                mindepth=mindepth,
+                maxdepth=maxdepth,
+                depth=depth,
+                file_type=file_type,
+                **user_pass_host_key,
+            )
+        result = run_pruned_func(run_cmd,find_cmd,
+            **kwargs
+            
+            )
+        return_result = [res for res in result.split('\n') if res]
+        return_results[type_str]=return_result
+    if len(file_types) == 1:
+        return return_result
+    return return_results
+def get_files_and_dirs(
+    *args,
+    recursive: bool = True,
+    include_files: bool = True,
+    **kwargs
+    ):
+    if recursive == False:
+        kwargs['maxdepth']=1
+    if include_files == False:
+        kwargs['file_type']='d'
+    result = collect_globs(*args,**kwargs)
+    if include_files == False:
+        return result,[]
+    dirs = result.get("dirs")
+    files = result.get("files")
+    return dirs,files
+def collect_filepaths(
+    *args,
+    **kwargs
+    ) -> List[str]:
+    kwargs['file_type']='f'
+    return collect_globs(*args,**kwargs)