graph-knowledge-doc-parser 0.1.0__py3-none-any.whl

kg_doc_parser/text_processing_utils.py

@@ -0,0 +1,30 @@
+
+from typing import Dict, Literal, Any
+from .semantic_document_splitting_layerwise_edits import parse_doc
+
+def text_to_ocr_format(text: str, filename: str = "input_text") -> Dict:
+    """
+    Wraps a raw string into the expected OCR dictionary format with a single dummy cluster.
+    """
+    return {
+        filename: [
+            {
+                "pdf_page_num": 1,
+                "OCR_text_clusters": [
+                    {
+                        "text": text,
+                        "bb_x_min": 0, "bb_y_min": 0, "bb_x_max": 1000, "bb_y_max": 1000,
+                        "cluster_number": 0
+                    }
+                ],
+                "non_text_objects": []
+            }
+        ]
+    }
+
+def parse_doc_text(text: str, doc_id: str = "text_doc", parsing_mode: Literal["snippet", "delimiter"] = "snippet", max_depth: int = 10):
+    """
+    Convenience function to parse a raw text string.
+    """
+    raw_doc = text_to_ocr_format(text)
+    return parse_doc(doc_id, raw_doc, parsing_mode=parsing_mode, max_depth=max_depth)

kg_doc_parser/utils/bounded_threadpool_executor.py

@@ -0,0 +1,37 @@
+from concurrent.futures import ThreadPoolExecutor
+from threading import Semaphore, Lock, Condition
+
+
+class BoundedExecutor:
+    def __init__(self, max_workers=5, max_pending=10):
+        self._executor = ThreadPoolExecutor(max_workers=max_workers)
+        self._semaphore = Semaphore(max_pending)
+        self._lock = Lock()
+        self._condition = Condition(self._lock)
+        self._active_tasks = 0
+
+    def submit(self, fn, *args, **kwargs):
+        self._semaphore.acquire()
+
+        def wrapped_fn(*args, **kwargs):
+            with self._condition:
+                self._active_tasks += 1
+            try:
+                return fn(*args, **kwargs)
+            finally:
+                with self._condition:
+                    self._active_tasks -= 1
+                    self._condition.notify_all()
+                self._semaphore.release()
+
+        return self._executor.submit(wrapped_fn, *args, **kwargs)
+
+    def wait_for_all(self):
+        """Block until all submitted tasks have completed."""
+        with self._condition:
+            while self._active_tasks > 0:
+                self._condition.wait()
+
+    def shutdown(self, wait=True):
+        """Shut down the underlying ThreadPoolExecutor."""
+        self._executor.shutdown(wait=wait)

kg_doc_parser/utils/file_loaders.py

@@ -0,0 +1,405 @@
+
+import logging
+logger = logging.getLogger(__name__)
+logger.addHandler(logging.NullHandler())
+logger.debug("library loading")
+from json import JSONDecodeError
+import os
+import pathlib
+import pathspec
+from typing import Iterable, Tuple, Optional, Generator, Any
+from typing import Callable
+from functools import partial
+
+def bool2yn(maybe_bool: bool):
+    if type(maybe_bool) is bool:
+        return "Yes" if maybe_bool else "No"
+    else:
+        return maybe_bool
+
+def recur_apply_json_inplace(json : dict | list | bool |str | int | float, fn: Callable):
+    if isinstance(json, dict):
+        for k, v in json.items():
+            json[k] = recur_apply_json_inplace(v, fn)
+            
+    elif isinstance(json, list):
+        for i,v in enumerate(json):
+            json[i] = recur_apply_json_inplace(json[i], fn)
+    else:
+        return fn(json)
+    return json
+
+# convert any boolean to yes no
+json_bool_to_yn: Callable = partial(recur_apply_json_inplace, fn = bool2yn)
+
+def nullable_concat(a: list| None, b: list| None) -> list | None:
+    if a is None and b is None:
+        return None
+    return (a or []) + (b or [])
+
+def fast_walk(path, prune_condition=None):
+    stack = [path]
+    while stack:
+        current = stack.pop()  # DFS, or use `.pop(0)` for BFS
+        try:
+            with os.scandir(current) as it:
+                dirs = []
+                files = []
+                for entry in it:
+                    if entry.is_dir(follow_symlinks=False):
+                        if prune_condition and prune_condition(entry.path):
+                            continue
+                        dirs.append(entry.path)
+                    elif entry.is_file():
+                        files.append(entry.path)
+                yield current, dirs, files
+                stack.extend(reversed(dirs))  # DFS
+        except PermissionError:
+            continue
+def find_folders_two_levels_from_leaves_mem_optimized(root_path: str, required_level: int = 2) -> Generator[Tuple[str , list[str],list[str]], Any, None]:
+    """
+    Yields folders that are exactly two levels away from leaf nodes using a
+    highly memory-efficient iterator.
+
+    The memoization cache is actively pruned to keep memory usage proportional
+    to the filesystem's width, not its total size.
+
+    Args:
+        root_path: The absolute or relative path to the directory to start searching from.
+
+    Yields:
+        A path to a qualifying folder as soon as it is identified.
+    """
+    if not os.path.isdir(root_path):
+        print(f"Error: Provided path '{root_path}' is not a valid directory.")
+        return
+
+    memo = {}
+    
+    # Using realpath normalizes paths (e.g., C:/Users vs C:\Users) and handles
+    # symlinks, making the cache keys consistent.
+    # real_root = os.path.realpath(root_path)
+
+    for dirpath, dirnames, filenames in os.walk(root_path, topdown=False):
+        no_solution = False
+        # 1. Get the depths of all children of the current `dirpath`.
+        child_depths = [0] * len(filenames) # Files are leaves, depth 0.
+
+        for dirname in dirnames:
+            child_path = os.path.join(dirpath, dirname)
+            child_depth = memo.get(child_path, -1)
+            if child_depth is None:
+                no_solution = True
+                child_depths = [None]
+                break
+            child_depths.append(memo.get(child_path, -1))
+        
+        # 2. THE CORE LOGIC: Check if this directory qualifies.
+        # if child_depths and all(depth == required_level for depth in child_depths):
+        #     # Yield the original path if it's the root, otherwise the current dirpath
+        #     if dirpath == real_root:
+        #         return # yield root_path
+        #     else:
+        #         yield dirpath, dirnames, filenames
+        
+        # 3. Calculate and cache the depth for the current path.
+        if no_solution:
+            current_depth = None
+        else:
+            if len(set(child_depths)) > 1:
+                current_depth = None
+            else:
+                
+                if not child_depths:
+                    current_depth = 0
+                elif -1 in child_depths:
+                    current_depth = -1
+                else:
+                    # child_depths must have no None here
+                    current_depth = 1 + child_depths[0] # type: ignore
+        if current_depth == required_level:
+            yield dirpath, dirnames, filenames
+        memo[dirpath] = current_depth
+
+        # 4. PRUNING STEP: Release memory by removing child depths.
+        # This is safe because we are done with this level and the parent
+        # only needs the depth of `dirpath`, which we just cached.
+        for dirname in dirnames:
+            child_path = os.path.join(dirpath, dirname)
+            memo.pop(child_path, None) # Safely remove the key
+class RawFileLoader():
+    def __init__(self, env_flist_path: Optional[str] = None, 
+                 allow_file_list: None | list[str]  = None,
+                 allow_file_list_ref: Optional[str] = None,
+                 max_num_file = float('inf'), oldest_datetime = None, newest_datetime = None, root_folder_name : str | None= None, 
+                #  in_folder_name :Optional[str] = None,
+                 walk_root:Optional[str] = None, compare_root:Optional[str] = None,
+                 include = None,
+                 bucket_blob_connection_str = None,
+                 file_walker_callback: Callable[[str, int], Generator[Tuple[str , list[str],list[str]], Any, None]] | None = None,
+                 pattern = None,
+                 allow_startwith_relative_paths = False,
+                 filtering_callbacks : Optional[list[Callable]] = None):
+        """file loader to either backward support for local folder loading behaviour, or cloud bucket/ blob stoages
+
+        Args:
+            env_var_name (_type_): environment variable that contains the path to the file list
+            allow_file_list (None | list[str] | str): file path that contains the file information or a list of filenames directly passed into
+            pattern re.compile returned pattern to apply on the path
+            filtering_callbacks: that apply to each path, when True, it allows continue, when False, it continue to next file
+        """
+        
+        assert not ((bucket_blob_connection_str is not None) and (env_flist_path is not None))
+        self.allow_startwith_relative_paths = allow_startwith_relative_paths
+        self.pattern = pattern
+        self.include = include or []
+        self.oldest_datetime = oldest_datetime
+        self.newest_datetime = newest_datetime
+        self.bucket_blob_connection_str = bucket_blob_connection_str
+        self.file_walker_callback: Optional[Callable] = file_walker_callback
+        if root_folder_name is None:
+            root_folder_name = os.getcwd()
+        if walk_root is None:
+            walk_root = os.getcwd()
+        self.walk_root = walk_root
+        if compare_root is None:
+            compare_root = walk_root
+        self.compare_root = compare_root
+        self.max_num_file = max_num_file
+        allowed_relative_paths: Optional[list[str]] = None
+        
+        if allow_file_list_ref is not None:
+            if allow_file_list is None:
+                allow_file_list = []
+            if isinstance(allow_file_list_ref, str):
+                need_attempt_readlines = False
+                try:
+                    if allow_file_list_ref.endswith('.json'):
+                        import json
+                        with open(allow_file_list_ref, 'r') as f:
+                            flist = json.load(f)
+                            if isinstance(flist, list):
+                                pass
+                            else:
+                                raise Exception('only list-like json is accepted')
+                        allow_file_list = flist # type: ignore
+                    else:
+                        need_attempt_readlines = True
+                except JSONDecodeError as e:
+                    need_attempt_readlines = True
+                
+                except Exception as e:
+                    raise
+                if need_attempt_readlines:
+                    with open(allow_file_list_ref, 'r') as f:
+                        flist = f.readlines()
+                    
+                    allow_file_list = flist # type: ignore
+            else:
+                raise(ValueError("allow_file_list does not either provide a path to file containing file list "
+                                "or directly passing file list"))
+        
+        # allow combine flist from argument with env specifed paths list concatenated
+        if env_flist_path is not None:
+            if allowed_relative_paths is None:
+                allowed_relative_paths = []
+            if flist:=os.environ.get(env_flist_path, None):
+                if flist is not None:
+                    if os.path.exists(flist):
+                        if flist.endswith('.txt'):
+                            with open(flist, 'r', encoding='utf-8') as f:
+                                allowed_relative_paths = [i.strip() for i in f.readlines()]
+                                    
+                        if flist.endswith('.csv'):
+                            with open(flist, 'r') as f:
+                                for ln in f.readlines():
+                                    allowed_relative_paths.append(os.path.join(*(i.strip() for i in ln.split(','))))
+                        elif flist.endswith('.xls') or flist.endswith('.xlsx'):
+                            from pandas import read_excel
+                            df = read_excel(flist)
+                            allowed_relative_paths = []
+                            for i, row in df.iterrows():
+                                new_path = os.path.join(*(row[:3]))
+                                allowed_relative_paths.append(new_path)
+        if allowed_relative_paths is not None:
+            if allow_file_list is None:
+                allow_file_list = []
+                
+            allow_file_list = allow_file_list + allowed_relative_paths
+        self.allow_file_list = allow_file_list
+        self.filtering_callbacks = filtering_callbacks or []
+        self.resolve_paths = False
+        if self.resolve_paths:
+            self.check_allowed_relative_path()
+        pass
+    def check_allowed_relative_path(self, paths= None, ):
+        # ensure allowed 
+        allow_file_list = nullable_concat(self.allow_file_list, paths)
+        if allow_file_list is None:
+            return
+        out_file_list = []
+        try:
+            for p in allow_file_list:
+                out_path = pathlib.Path(p).relative_to(self.compare_root)
+                out_file_list.append(str(out_path))
+        except ValueError as e:
+            if "is not in the subpath of" in str(e):
+                print("allow_file_list contains path outside of compare root")
+                raise
+        self.allow_file_list = out_file_list
+        
+    def __iter__(self, leaf_only = False, file_non_exist_ok = False, include = None,
+                 allowed_files: Optional[list[str]] = None,
+                 # allowed_prefixes : Optional[list[str | int]] = None,
+                 allowed_relative_paths: Optional[list[str]]= None,
+                 ):
+        """Iterate through availble files 
+
+        Args:
+            leaf_only (bool, optional): _description_. Defaults to True.
+            file_non_exist_ok (bool, optional): _description_. Defaults to False.
+            include (_type_, optional): _description_. Defaults to None.
+            allowed_files (Optional[list[str]], optional): _description_. Defaults to None.
+            allowed_relative_paths (Optional[list[str]], optional): _description_. Defaults to None.
+            allow_startwith_relative_paths also check if the file start with any of the allowed relative paths
+        Yields:
+            _type_: _description_
+        """
+        from collections import Counter
+        filter_stat = Counter()
+        if include is None:
+            include = include or self.include or set(['files'])#, ['files', "dirs"]
+        else:
+            include = set(include)
+            include.update(self.include)
+        count = 0
+        from datetime import datetime
+        if self.oldest_datetime is not None:
+            if type(self.oldest_datetime) is str:
+                time_threshold_dt = datetime.strptime(self.oldest_datetime, '%Y-%m-%d %H:%M')
+            else:
+                time_threshold_dt = self.oldest_datetime
+        else:
+            time_threshold_dt = None
+        if self.newest_datetime is not None:
+            if type(self.newest_datetime) is str:
+                time_threshold_upper_dt = datetime.strptime(self.newest_datetime, '%Y-%m-%d %H:%M')
+            else:
+                time_threshold_upper_dt = self.newest_datetime
+        else:
+            time_threshold_upper_dt = None
+        nullable_allowed_relative_paths_set = []
+        nullable_allowed_relative_paths_set_t = nullable_concat(allowed_relative_paths, self.allow_file_list)
+        if nullable_allowed_relative_paths_set_t is not None:
+            nullable_allowed_relative_paths_set_t = set(nullable_allowed_relative_paths_set_t)
+            if len(nullable_allowed_relative_paths_set_t) > 0:
+                nullable_allowed_relative_paths_set = list(nullable_allowed_relative_paths_set_t)
+        else:
+            nullable_allowed_relative_paths_set = None
+        # debug_list = []
+        if self.file_walker_callback is None:
+            if self.bucket_blob_connection_str is None:
+                file_walker = os.walk(self.walk_root)
+            else:
+                raise NotImplementedError('blob or bucket stage file walker not implemented')
+        else:
+            file_walker = self.file_walker_callback(self.walk_root)
+        spec = None
+        if nullable_allowed_relative_paths_set is not None:
+            if self.allow_startwith_relative_paths:
+                
+                prefixes = [pathlib.Path(i).as_posix() for i in nullable_allowed_relative_paths_set]
+                # normalized_prefixes = [p.rstrip("/\\") + "/" for p in raw_prefixes]
+                spec = pathspec.PathSpec.from_lines("gitwildmatch", prefixes)
+        for root, dirs, files in file_walker:
+            # print(root)
+            if leaf_only:
+                if dirs != []:
+                    filter_stat.update(['dir != []'])
+                    continue
+            if allowed_files:
+                if pathlib.Path(root).parts[-1] in allowed_files:
+                    pass
+                else:
+                    filter_stat.update(['pathlib.Path(root).parts[-1] in allowed_files'])
+                    continue
+            to_iter = []
+            if "files" in include:
+                to_iter += files
+            if "dirs"  in include:
+                to_iter += dirs
+            if "root" in include:
+                to_iter += ['.']
+            for f in to_iter:
+                if self.pattern:
+                    if self.pattern.match(os.path.join(root, f)):
+                        pass
+                    else:
+                        filter_stat.update(['self.pattern.match(os.path.join(root, f))'])
+                        continue
+                f_rel = str(pathlib.Path(os.path.join(root, f)).relative_to(self.compare_root))
+                if spec is not None:
+                    matched = True
+                    matched = spec.match_file(pathlib.Path(f_rel).as_posix())
+                    
+                    if not matched:
+                        filter_stat.update(['not spec.match_file(pathlib.Path(f_rel).as_posix())'])
+                        continue
+                else:
+                    if nullable_allowed_relative_paths_set is None:
+                        pass
+                    elif f_rel in nullable_allowed_relative_paths_set:
+                        pass
+                    else:
+                        filter_stat.update(['f_rel in allowed_relative_paths'])
+                        continue
+                filtered = False
+                for cb in self.filtering_callbacks:
+                    
+                    if cb(os.path.join(self.compare_root, f_rel)):
+                        pass
+                    else:
+                        filter_stat.update([f'callback filtered by {cb}'])
+                        filtered = True
+                        break
+                if filtered:
+                    continue
+                creation_timestamp = os.path.getctime(os.path.join(self.compare_root, f_rel))
+                creation_time = datetime.fromtimestamp(creation_timestamp)
+                if time_threshold_dt is not None:
+                    if not (creation_time > time_threshold_dt):
+                        filter_stat.update(['not (creation_time > time_threshold_dt)'])
+                        continue
+                if time_threshold_upper_dt is not None:
+                    if not (creation_time < time_threshold_upper_dt):
+                        filter_stat.update(['not (creation_time < time_threshold_upper_dt)'])
+                        continue
+                count+= 1
+                # debug_list.append(f_rel)
+                yield f_rel
+                if count >= self.max_num_file:
+                    print(filter_stat.most_common())
+                    return
+        pass
+        print(filter_stat.most_common())
+    pass
+
+def filter_folder(folder_root = os.path.join(
+        "..", "odc_data", "split_pages"
+        
+        ), min_page = 45, max_page: int | float = 55, first = 10, verbose = True):
+    folders = []
+    for root, dirs, files in os.walk(folder_root):
+        if dirs == []:
+            n_page = len([i for i in files if i.lower().endswith('.pdf')])
+
+            if n_page < max_page and min_page < n_page:
+                rel_path = pathlib.Path(root).relative_to(folder_root)
+                folders.append(str(rel_path))
+
+    if verbose:
+        print(folders)
+    if first is not None:
+        return (sorted(folders)[:first])
+    else:
+        return (sorted(folders))