abstract-utilities 0.2.2.467__py3-none-any.whl → 0.2.2.513__py3-none-any.whl

abstract_utilities/read_write_utils/read_write_utils.py

@@ -0,0 +1,338 @@
+"""
+read_write_utils.py
+-------------------
+Unified read/write utility for safe file operations.
+Supports:
+- Writing content to a file
+- Reading content from a file
+- Creating and reading if missing
+- Detecting file/content params via positional args or kwargs
+
+Usage:
+    from abstract_utilities.read_write_utils import *
+"""
+
+from .imports import *
+_FILE_PATH_KEYS = ['file', 'filepath', 'file_path', 'path', 'directory', 'f', 'dst', 'dest']
+_CONTENTS_KEYS = ['cont', 'content', 'contents', 'data', 'datas', 'dat', 'src', 'source']
+
+            
+# --- Helper utilities --------------------------------------------------------
+def string_in_keys(strings, kwargs):
+    """Find a matching keyword in kwargs that contains any of the given substrings."""
+    for key in kwargs:
+        for s in strings:
+            if s.lower() in key.lower():
+                return key
+    return None
+def make_dirs(path, exist_ok=True, **kwargs):
+    remote = get_user_pass_host_key(**kwargs)
+ 
+    if remote:
+        kwargs['cmd'] = f"mkdir -p {path}"
+       
+        resp = run_pruned_func(run_cmd, **kwargs)
+       
+    else:
+        os.makedirs(path, exist_ok=exist_ok)
+    return path
+def make_path(path, home_dir=None, file=None, **kwargs):
+    if not path:
+        return None
+
+    basename = os.path.basename(path)
+    parts = [p for p in path.split('/') if p]
+
+    # Detect whether this is a file or a folder
+    is_file = file if file is not None else ('.' in basename)
+    pieces = parts[:-1] if is_file else parts
+  
+    full_dir = home_dir or '/'
+    for piece in pieces:
+        full_dir = os.path.join(full_dir, piece)
+        make_dirs(full_dir, exist_ok=True, **kwargs)
+       
+    if is_file:
+        full_dir = os.path.join(full_dir, basename)
+
+    return full_dir
+def get_rel_path(src,src_rel,dst,**kwargs):
+    if src.startswith(src_rel):
+        nu_src = src[len(src_rel):]
+        nu_src= eatAll(nu_src,'/')
+        directory= eatOuter(dst,'/')
+        rel_path = os.path.join(dst,nu_src)
+        return rel_path
+def make_relative_path(src,src_rel,dst,**kwargs):
+
+    if src.startswith(src_rel):
+        rel_path = get_rel_path(src,src_rel,dst)
+      
+        path = make_path(rel_path,**kwargs)
+      
+        return path
+
+def path_join(*args):
+    path = None
+    for i,arg in enumerate(args):
+        if arg:
+            if i == 0:
+                path = arg
+            else:
+                path = os.path.join(path,arg)
+    return path
+
+def get_path(paths,**kwargs):
+    """Return the first valid path among given paths."""
+    for path in paths:
+        if isinstance(path, str):
+            if is_file(path,**kwargs):
+                return path
+            dirname = os.path.dirname(path)
+            if is_exists(dirname,**kwargs):
+                return path
+    return None
+
+
+def break_down_find_existing(path,**kwargs):
+    """Return the first non-existent subpath within a path chain."""
+    test_path = ''
+    for part in path.split(os.sep):
+        test_path = os.path.join(test_path, part)
+        if not is_exists(test_path,**kwargs):
+            return test_path if test_path else None
+    return test_path
+
+
+# --- Parameter parsing --------------------------------------------------------
+def check_read_write_params(*args, **kwargs):
+    """
+    Determine file_path and contents from arguments.
+    Returns a tuple: (file_path, contents)
+    """
+    file_key = string_in_keys(_FILE_PATH_KEYS, kwargs)
+    content_key = string_in_keys(_CONTENTS_KEYS, kwargs)
+
+    file_path = kwargs.get(file_key) if file_key else None
+    contents = kwargs.get(content_key) if content_key else None
+
+    # Handle positional args (fallback)
+    if file_path is None and len(args) > 0:
+        file_path = args[0]
+    if contents is None and len(args) > 1:
+        contents = args[1]
+
+    if file_path is None:
+        raise ValueError("Missing file_path argument.")
+    return file_path, contents
+
+def write_to_path(
+        file_path: str,
+        contents: str,
+        *,
+        user_at_host: str = None,
+        cwd: str | None = None,
+        password=None,
+        key=None,
+        env_path=None,
+        **kwargs
+    ) -> str:
+    """
+    Completely overwrite a file (locally or remotely).
+    Supports sudo and password-based remote execution.
+    """
+
+    # sanitize for shell safety
+    quoted_path = shlex.quote(file_path)
+    quoted_data = shlex.quote(str(contents))
+
+    # shell command that fully overwrites
+    # (no append, replaces contents entirely)
+    base_cmd = f'sudo sh -c "echo {quoted_data} > {quoted_path}"'
+
+    # optional sudo password injection
+    full_cmd = get_print_sudo_cmd(
+        cmd=base_cmd,
+        password=password,
+        key=key,
+        env_path=env_path
+    )
+
+    # local or remote dispatch
+    if user_at_host:
+        return run_remote_cmd(
+            user_at_host=user_at_host,
+            cmd=full_cmd,
+            cwd=cwd,
+            password=password,
+            key=key,
+            env_path=env_path,
+            **kwargs
+        )
+    else:
+        return run_local_cmd(
+            cmd=full_cmd,
+            cwd=cwd,
+            password=password,
+            key=key,
+            env_path=env_path,
+            **kwargs
+        )
+### --- Core functionality -------------------------------------------------------
+##def write_to_file(*args, **kwargs):
+##    """
+##    Write contents to a file (create if missing).
+##
+##    Returns the file_path written.
+##    """
+##    file_path, contents = check_read_write_params(*args, **kwargs)
+##    if contents is None:
+##        raise ValueError("Missing contents to write.")
+##
+##    os.makedirs(os.path.dirname(file_path) or ".", exist_ok=True)
+##    with open(file_path, "w", encoding="utf-8") as f:
+##        f.write(str(contents))
+##    return file_path
+# --- Core functionality -------------------------------------------------------
+def write_to_file(*args, **kwargs):
+    """
+    Write contents to a file (create if missing).
+
+    Returns the file_path written.
+    """
+    file_path, contents = check_read_write_params(*args, **kwargs)
+    values,kwargs = get_from_kwargs(['file_path','contents'],del_kwarg=True,**kwargs)
+    dirname = os.path.dirname(file_path)
+    
+    if contents is None:
+        raise ValueError("Missing contents to write.")
+    user_at_host = kwargs.get("user_at_host")
+    if get_user_pass_host_key(**kwargs):
+        make_dirs(dirname, exist_ok=True,**kwargs)
+        kwargs["cwd"] = kwargs.get('cwd') or os.path.dirname(file_path)
+        # sanitize for shell safety
+        quoted_path = shlex.quote(file_path)
+        quoted_data = shlex.quote(str(contents))
+        # shell command that fully overwrites
+        # (no append, replaces contents entirely)
+        kwargs["cmd"] = f'sh -c "echo {quoted_data} > {quoted_path}"'
+        if not kwargs.get('password') and not kwargs.get('key'):
+            kwargs["cmd"]=f'sudo {kwargs["cmd"]}'
+        result = run_pruned_func(run_cmd,**kwargs)
+        if 'file_path' in kwargs:
+            del kwargs['file_path']
+        if not is_file(file_path,**kwargs) or str(contents) != read_from_file(file_path,**kwargs):
+            kwargs["cmd"]=f'sudo {kwargs["cmd"]}'
+            result = run_pruned_func(run_cmd,**kwargs)
+        return result
+
+    make_dirs(dirname or ".", exist_ok=True)
+    with open(file_path, "w", encoding="utf-8") as f:
+        f.write(str(contents))
+    return file_path
+
+
+def read_from_file(file_path,**kwargs):
+    if get_user_pass_host_key(**kwargs):
+        kwargs["cwd"] = kwargs.get('cwd') or os.path.dirname(file_path)
+        basename = os.path.basename(file_path)
+        kwargs["cmd"] = f'cat {basename}'
+        return run_pruned_func(run_cmd,**kwargs)
+    """Read text content from a file."""
+    with open(file_path, "r", encoding="utf-8") as f:
+        return f.read()
+
+
+def copy_dirs(dirs, dst_root, src_rel=None, **kwargs):
+    """
+    Recursively copy directory structures (without files) from dirs → dst_root.
+    """
+    for src in dirs:
+        # build destination path preserving relative structure
+        dst_path = make_relative_path(src, src_rel, dst_root, **kwargs) if src_rel else dst_root
+        make_path(dst_path, **kwargs)  # ensures directory exists
+     
+
+
+def copy_file(src, dst_root, src_rel=None, **kwargs):
+    """
+    Copy a single file to dst_root, preserving relative structure if src_rel provided.
+    Supports remote copy via read/write.
+    """
+    # derive destination file path
+    dst_path = make_relative_path(src, src_rel, dst_root, **kwargs) if src_rel else os.path.join(dst_root, os.path.basename(src))
+    make_path(dst_path, **kwargs)
+
+    if get_user_pass_host_key(**kwargs):  # remote mode
+        contents = read_from_file(src, **kwargs)
+        write_to_file(contents=contents, file_path=dst_path, **kwargs)
+    else:  # local
+        os.makedirs(os.path.dirname(dst_path), exist_ok=True)
+        shutil.copy2(src, dst_path)
+
+
+    return dst_path
+
+
+def copy_files(files, dst_root, src_rel=None, **kwargs):
+    """
+    Copy a list of files to dst_root.
+    """
+    for src in files:
+        copy_file(src=src, dst_root=dst_root, src_rel=src_rel, **kwargs)
+
+def create_and_read_file(*args, **kwargs):
+    """
+    Create the file (if missing) and read contents from it.
+    """
+    file_path, contents = check_read_write_params(*args, **kwargs)
+    if not os.path.isfile(file_path):
+        write_to_file(file_path, contents or "")
+    return read_from_file(file_path)
+
+
+def is_file_extension(obj: str) -> bool:
+    """Return True if obj looks like a filename with extension."""
+    if not isinstance(obj, str):
+        return False
+    root, ext = os.path.splitext(obj)
+    return bool(root and ext)
+
+
+def delete_file(file_path: str):
+    """Safely delete a file if it exists."""
+    if os.path.isfile(file_path):
+        os.remove(file_path)
+        return True
+    return False
+
+
+def get_content_lines(*args, **kwargs):
+    """Return a list of lines from string or file path."""
+    file_path, contents = check_read_write_params(*args, **kwargs)
+    if os.path.isfile(file_path):
+        contents = read_from_file(filepath)
+
+    if isinstance(contents, str):
+        return contents.splitlines()
+    elif isinstance(contents, list):
+        return contents
+    return []
+def collate_text_docs(directory=None):
+    return [read_from_file(item) for item in get_all_files(directory=directory)]
+def get_content(*paths):
+    item_path = os.path.join(*paths)
+    if os.path.isfile(item_path):
+        try:
+            content = read_from_file(item_path)
+            return content
+        except:
+            pass
+    return None
+def get_text_or_read(text=None,file_path=None):
+    text = text or ''
+    imports_js = {}
+    if not text and file_path and os.path.isfile(file_path):
+        text=read_from_file(file_path)
+    return text
+##

abstract_utilities/read_write_utils.py

@@ -12,15 +12,16 @@ Usage:
     from abstract_utilities.read_write_utils import *
 """
 
-import os
-import shlex
+import os,shlex
+from .string_clean import *
 from .ssh_utils.utils import run_cmd,get_print_sudo_cmd,run_local_cmd,run_remote_cmd
-from .file_utils.file_utils.type_checks import is_file,is_dir
 from .abstract_classes import run_pruned_func
+from .string_utils import get_from_kwargs
+from .path_utils import get_all_files,is_file,is_dir,get_user_pass_host_key,is_exists
 _FILE_PATH_KEYS = ['file', 'filepath', 'file_path', 'path', 'directory', 'f', 'dst', 'dest']
 _CONTENTS_KEYS = ['cont', 'content', 'contents', 'data', 'datas', 'dat', 'src', 'source']
 
-
+            
 # --- Helper utilities --------------------------------------------------------
 def string_in_keys(strings, kwargs):
     """Find a matching keyword in kwargs that contains any of the given substrings."""
@@ -29,26 +30,81 @@ def string_in_keys(strings, kwargs):
             if s.lower() in key.lower():
                 return key
     return None
-
-
-def get_path(paths):
+def make_dirs(path, exist_ok=True, **kwargs):
+    remote = get_user_pass_host_key(**kwargs)
+ 
+    if remote:
+        kwargs['cmd'] = f"mkdir -p {path}"
+       
+        resp = run_pruned_func(run_cmd, **kwargs)
+       
+    else:
+        os.makedirs(path, exist_ok=exist_ok)
+    return path
+def make_path(path, home_dir=None, file=None, **kwargs):
+    if not path:
+        return None
+
+    basename = os.path.basename(path)
+    parts = [p for p in path.split('/') if p]
+
+    # Detect whether this is a file or a folder
+    is_file = file if file is not None else ('.' in basename)
+    pieces = parts[:-1] if is_file else parts
+  
+    full_dir = home_dir or '/'
+    for piece in pieces:
+        full_dir = os.path.join(full_dir, piece)
+        make_dirs(full_dir, exist_ok=True, **kwargs)
+       
+    if is_file:
+        full_dir = os.path.join(full_dir, basename)
+
+    return full_dir
+def get_rel_path(src,src_rel,dst,**kwargs):
+    if src.startswith(src_rel):
+        nu_src = src[len(src_rel):]
+        nu_src= eatAll(nu_src,'/')
+        directory= eatOuter(dst,'/')
+        rel_path = os.path.join(dst,nu_src)
+        return rel_path
+def make_relative_path(src,src_rel,dst,**kwargs):
+
+    if src.startswith(src_rel):
+        rel_path = get_rel_path(src,src_rel,dst)
+      
+        path = make_path(rel_path,**kwargs)
+      
+        return path
+
+def path_join(*args):
+    path = None
+    for i,arg in enumerate(args):
+        if arg:
+            if i == 0:
+                path = arg
+            else:
+                path = os.path.join(path,arg)
+    return path
+
+def get_path(paths,**kwargs):
     """Return the first valid path among given paths."""
     for path in paths:
         if isinstance(path, str):
-            if os.path.isfile(path):
+            if is_file(path,**kwargs):
                 return path
             dirname = os.path.dirname(path)
-            if os.path.exists(dirname):
+            if is_exists(dirname,**kwargs):
                 return path
     return None
 
 
-def break_down_find_existing(path):
+def break_down_find_existing(path,**kwargs):
     """Return the first non-existent subpath within a path chain."""
     test_path = ''
     for part in path.split(os.sep):
         test_path = os.path.join(test_path, part)
-        if not os.path.exists(test_path):
+        if not is_exists(test_path,**kwargs):
             return test_path if test_path else None
     return test_path
 
@@ -97,7 +153,7 @@ def write_to_path(
 
     # shell command that fully overwrites
     # (no append, replaces contents entirely)
-    base_cmd = f"echo {quoted_data} > {quoted_path}"
+    base_cmd = f'sudo sh -c "echo {quoted_data} > {quoted_path}"'
 
     # optional sudo password injection
     full_cmd = get_print_sudo_cmd(
@@ -150,29 +206,39 @@ def write_to_file(*args, **kwargs):
     Returns the file_path written.
     """
     file_path, contents = check_read_write_params(*args, **kwargs)
+    values,kwargs = get_from_kwargs(['file_path','contents'],del_kwarg=True,**kwargs)
+    dirname = os.path.dirname(file_path)
+    
     if contents is None:
         raise ValueError("Missing contents to write.")
     user_at_host = kwargs.get("user_at_host")
-    if user_at_host:
+    if get_user_pass_host_key(**kwargs):
+        make_dirs(dirname, exist_ok=True,**kwargs)
         kwargs["cwd"] = kwargs.get('cwd') or os.path.dirname(file_path)
         # sanitize for shell safety
         quoted_path = shlex.quote(file_path)
         quoted_data = shlex.quote(str(contents))
         # shell command that fully overwrites
         # (no append, replaces contents entirely)
-        kwargs["cmd"] = f"echo {quoted_data} > {quoted_path}"
-        return run_pruned_func(run_cmd,**kwargs)
-
-
-    os.makedirs(os.path.dirname(file_path) or ".", exist_ok=True)
+        kwargs["cmd"] = f'sh -c "echo {quoted_data} > {quoted_path}"'
+        if not kwargs.get('password') and not kwargs.get('key'):
+            kwargs["cmd"]=f'sudo {kwargs["cmd"]}'
+        result = run_pruned_func(run_cmd,**kwargs)
+        if 'file_path' in kwargs:
+            del kwargs['file_path']
+        if not is_file(file_path,**kwargs) or str(contents) != read_from_file(file_path,**kwargs):
+            kwargs["cmd"]=f'sudo {kwargs["cmd"]}'
+            result = run_pruned_func(run_cmd,**kwargs)
+        return result
+
+    make_dirs(dirname or ".", exist_ok=True)
     with open(file_path, "w", encoding="utf-8") as f:
         f.write(str(contents))
     return file_path
 
 
 def read_from_file(file_path,**kwargs):
-    user_at_host = kwargs.get("user_at_host")
-    if user_at_host:
+    if get_user_pass_host_key(**kwargs):
         kwargs["cwd"] = kwargs.get('cwd') or os.path.dirname(file_path)
         basename = os.path.basename(file_path)
         kwargs["cmd"] = f'cat {basename}'
@@ -182,6 +248,44 @@ def read_from_file(file_path,**kwargs):
         return f.read()
 
 
+def copy_dirs(dirs, dst_root, src_rel=None, **kwargs):
+    """
+    Recursively copy directory structures (without files) from dirs → dst_root.
+    """
+    for src in dirs:
+        # build destination path preserving relative structure
+        dst_path = make_relative_path(src, src_rel, dst_root, **kwargs) if src_rel else dst_root
+        make_path(dst_path, **kwargs)  # ensures directory exists
+     
+
+
+def copy_file(src, dst_root, src_rel=None, **kwargs):
+    """
+    Copy a single file to dst_root, preserving relative structure if src_rel provided.
+    Supports remote copy via read/write.
+    """
+    # derive destination file path
+    dst_path = make_relative_path(src, src_rel, dst_root, **kwargs) if src_rel else os.path.join(dst_root, os.path.basename(src))
+    make_path(dst_path, **kwargs)
+
+    if get_user_pass_host_key(**kwargs):  # remote mode
+        contents = read_from_file(src, **kwargs)
+        write_to_file(contents=contents, file_path=dst_path, **kwargs)
+    else:  # local
+        os.makedirs(os.path.dirname(dst_path), exist_ok=True)
+        shutil.copy2(src, dst_path)
+
+
+    return dst_path
+
+
+def copy_files(files, dst_root, src_rel=None, **kwargs):
+    """
+    Copy a list of files to dst_root.
+    """
+    for src in files:
+        copy_file(src=src, dst_root=dst_root, src_rel=src_rel, **kwargs)
+
 def create_and_read_file(*args, **kwargs):
     """
     Create the file (if missing) and read contents from it.
@@ -219,3 +323,21 @@ def get_content_lines(*args, **kwargs):
     elif isinstance(contents, list):
         return contents
     return []
+def collate_text_docs(directory=None):
+    return [read_from_file(item) for item in get_all_files(directory=directory)]
+def get_content(*paths):
+    item_path = os.path.join(*paths)
+    if os.path.isfile(item_path):
+        try:
+            content = read_from_file(item_path)
+            return content
+        except:
+            pass
+    return None
+def get_text_or_read(text=None,file_path=None):
+    text = text or ''
+    imports_js = {}
+    if not text and file_path and os.path.isfile(file_path):
+        text=read_from_file(file_path)
+    return text
+##

abstract_utilities/robust_reader/imports/imports.py

@@ -1,12 +1,3 @@
-import os,tempfile,shutil,logging,ezodf,fnmatch,pytesseract,pdfplumber
-import pandas as pd
-import geopandas as gpd
-from datetime import datetime
-from pathlib import Path
-from typing import *
-from werkzeug.utils import secure_filename
-from werkzeug.datastructures import FileStorage
-from pdf2image import convert_from_path   # only used for OCR fallback
 from ...abstract_classes import SingletonMeta
 from ..pdf_utils import *
 from ...read_write_utils import *

abstract_utilities/robust_readers/import_utils/__init__.py

@@ -4,3 +4,4 @@ from .import_utils import *
 from .sysroot_utils import *
 from .utils import *
 from .safe_import_utils import *
+from .clean_imports import *