text2rel 0.1__tar.gz → 0.2__tar.gz

{text2rel-0.1 → text2rel-0.2}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.3
 Name: text2rel
-Version: 0.1
-Summary: An HTML cleaner customized for kinship ties extraction from large text corpra.
+Version: 0.2
+Summary: An HTML cleaner, LLM processor and Reliability Assessment customized for kinship ties extraction from large text corpra.
 Requires-Python: >=3.10,<3.13
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

{text2rel-0.1 → text2rel-0.2}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "text2rel"
-version = "0.1"
-description = "An HTML cleaner customized for kinship ties extraction from large text corpra."
+version = "0.2"
+description = "An HTML cleaner, LLM processor and Reliability Assessment customized for kinship ties extraction from large text corpra."
 
 readme = "readme.md"
 

text2rel-0.2/text2rel/__init__.py

@@ -0,0 +1,12 @@
+from .cleaner import HTMLCleaner
+from .llm_processor import LLMProcessor
+from .reliability_assessor import ReliabilityAssessment
+from .state import set_default_inventory_path, get_default_inventory_path
+
+__all__ = [
+	"HTMLCleaner",
+	"LLMProcessor",
+	"ReliabilityAssessment",
+	"set_default_inventory_path",
+	"get_default_inventory_path",
+]

{text2rel-0.1 → text2rel-0.2}/text2rel/cleaner.py

@@ -2,6 +2,7 @@
 import os
 import sys
 import json
+import webbrowser
 import numpy as np
 import pandas as pd
 import re
@@ -11,26 +12,32 @@ import string
 from pathlib import Path
 from text2rel.utils import insert_jumpjump
 from text2rel.io import _load_inventory, _save_inventory
-from typing import Dict
+from text2rel.state import set_default_inventory_path
+from typing import Dict, TYPE_CHECKING, Optional, List, Any
+
+if TYPE_CHECKING:
+    from text2rel.llm_processor import LLMProcessor
 
 
 
 class HTMLCleaner:
-    def __init__(self, json_path: str, html_directory: str = None, extension: str = ".htm"):
+    def __init__(self, json_name: str, html_directory: str = None, extension: str = ".htm"):
         """
         Initializes the cleaner. If the JSON doesn't exist but a directory is given, it builds the inventory.
         """
-        if not json_path:
-            raise ValueError("json_path parameter is required. Please provide a path to your JSON inventory file. It can be simply and empty JSON file")
+        if not json_name:
+            raise ValueError("json_name parameter is required. Please provide a name for your JSON inventory file. It can be simply an empty JSON file")
         if not html_directory:
             raise ValueError("html_directory parameter is required. Please provide a path to your HTML files directory.")
-        self.json_path = Path(json_path)
+
+        self.json_path = Path(json_name)
+        set_default_inventory_path(self.json_path)
 
         if not self.json_path.exists():
             if html_directory is None:
                 raise FileNotFoundError(f"JSON not found at {json_path} and no directory given to create one.")
             self._create_inventory(html_directory, extension)
-            print(f"Created new inventory at {json_path}")
+            print(f"Created new inventory at {json_name}")
 
         self.inventory = _load_inventory(self.json_path)
 
@@ -64,6 +71,29 @@ class HTMLCleaner:
         for idx, file_info in enumerate(self.inventory['files']):
             print(f"{idx}: {Path(file_info['file_path']).name}")
 
+    def show_file(self, file_index: int, open_in_browser: bool = True) -> str:
+        """
+        Open an indexed HTML file from the inventory.
+
+        Parameters:
+        - file_index (int): index in self.inventory['files']
+        - open_in_browser (bool): if True, opens the file in the default browser
+
+        Returns:
+        - str: absolute path of the selected file
+        """
+        if file_index < 0 or file_index >= len(self.inventory['files']):
+            raise IndexError(f"file_index {file_index} out of range")
+
+        file_path = Path(self.inventory['files'][file_index]['file_path']).resolve()
+        if not file_path.exists():
+            raise FileNotFoundError(f"File not found: {file_path}")
+
+        if open_in_browser:
+            webbrowser.open(file_path.as_uri())
+
+        return str(file_path)
+
     #Allowing the user to set the relevant beginning and end for each file, the files can be accessed by filename
     def set_relevant_bounds(self, filename: str, beginning, end):
         for file in self.inventory['files']:
@@ -73,6 +103,179 @@ class HTMLCleaner:
                 _save_inventory(self.json_path,self.inventory)
                 return
         raise ValueError(f"File {filename} not found in inventory.")
+
+    def find_str(
+        self,
+        what: str,
+        where: Optional[str] = None,
+        window_left: int = 120,
+        window_right: int = 120,
+        *,
+        file_index: Optional[int] = None,
+        use_regex: bool = False,
+        html_flexible: bool = False,
+        robust_html: bool = False,
+        ignore_case: bool = True,
+        max_matches: Optional[int] = None,
+        return_matches: bool = False,
+    ) -> Optional[List[Dict[str, Any]]]:
+        """
+        Search text (or an inventory HTML file) and print contextual matches.
+
+        Parameters:
+        - what: query string or regex pattern.
+        - where: raw text to search. If None, file_index is used to load a file.
+        - window_left/window_right: chars shown before/after each match.
+        - file_index: inventory index used when where is None.
+        - use_regex: interpret `what` as regex pattern.
+                - html_flexible: for literal search, allow whitespace/newline drift.
+                    This uses a fast token-based pattern by default.
+                - robust_html: if True with html_flexible, use a heavier HTML-aware
+                    matcher (slower on very large files, but more tolerant).
+        - ignore_case: case-insensitive search.
+        - max_matches: optional cap on matches returned/printed.
+        - return_matches: return structured match metadata.
+        """
+        if where is None:
+            if file_index is None:
+                raise ValueError("Provide either `where` or `file_index`.")
+            if file_index < 0 or file_index >= len(self.inventory['files']):
+                raise IndexError(f"file_index {file_index} out of range")
+
+            file_path = Path(self.inventory['files'][file_index]['file_path'])
+            if not file_path.exists():
+                raise FileNotFoundError(f"File not found: {file_path}")
+            with open(file_path, 'r', encoding='utf-8') as f:
+                where = f.read()
+
+        if what is None or str(what).strip() == "":
+            raise ValueError("Search query cannot be empty.")
+
+        # If query was copied from repr(...) output, decode common escape sequences
+        # so patterns like "\\n" and escaped quotes do not block matching.
+        raw_query = str(what)
+        if any(token in raw_query for token in [r"\n", r"\t", r"\r", r"\"", r"\'"]):
+            try:
+                raw_query = bytes(raw_query, "utf-8").decode("unicode_escape")
+            except UnicodeDecodeError:
+                pass
+
+        flags = re.IGNORECASE if ignore_case else 0
+
+        if use_regex:
+            pattern = raw_query
+        else:
+            if html_flexible:
+                # Fast path: token-based whitespace-flexible matching.
+                # This is much faster on large HTML files.
+                tokens = re.findall(r"\S+", raw_query)
+                pattern = r"\s*".join(re.escape(token) for token in tokens)
+
+                # Optional robust path for difficult HTML snippets.
+                if robust_html and "<" in raw_query and ">" in raw_query:
+                    parts = re.findall(r"<[^>]+>|[^<]+", raw_query)
+                    part_patterns: List[str] = []
+
+                    for part in parts:
+                        if not part:
+                            continue
+
+                        if part.startswith("<") and part.endswith(">"):
+                            inner = part[1:-1].strip()
+                            if not inner:
+                                continue
+
+                            is_closing = inner.startswith("/")
+                            if is_closing:
+                                tag_name = inner[1:].strip().split()[0].strip("/").lower()
+                                if tag_name:
+                                    part_patterns.append(rf"<\s*/\s*{re.escape(tag_name)}\s*>")
+                                continue
+
+                            tag_name = inner.split()[0].strip("/").lower()
+                            if not tag_name:
+                                continue
+
+                            attrs = re.findall(
+                                r"([a-zA-Z_:][-a-zA-Z0-9_:.]*)\s*=\s*([\"\'])(.*?)\2",
+                                inner,
+                            )
+                            lookaheads = "".join(
+                                rf"(?=[^>]*{re.escape(attr)}\s*=\s*[\"\']{re.escape(val)}[\"\'])"
+                                for attr, _quote, val in attrs
+                            )
+                            part_patterns.append(
+                                rf"<\s*{re.escape(tag_name)}\b{lookaheads}[^>]*>"
+                            )
+                        else:
+                            text_tokens = re.findall(r"\S+", part)
+                            if text_tokens:
+                                part_patterns.append(
+                                    r"\s+".join(re.escape(token) for token in text_tokens)
+                                )
+
+                    if part_patterns:
+                        pattern = r".*?".join(part_patterns)
+            else:
+                pattern = re.escape(raw_query)
+
+        search_flags = flags | (re.DOTALL if html_flexible else 0)
+        all_matches = list(re.finditer(pattern, where, flags=search_flags))
+
+        # Fallback for copied HTML fragments: if tag-level matching fails,
+        # search by the visible text contained in the fragment.
+        if not all_matches and html_flexible and not use_regex:
+            text_query = BeautifulSoup(raw_query, 'html.parser').get_text(" ", strip=True)
+            if text_query:
+                text_tokens = re.findall(r"\S+", text_query)
+                if text_tokens:
+                    text_pattern = r"\s+".join(re.escape(token) for token in text_tokens)
+                    all_matches = list(re.finditer(text_pattern, where, flags=flags | re.DOTALL))
+
+            if all_matches:
+                print(
+                    "No exact HTML-fragment match was found; "
+                    "falling back to visible-text matching."
+                )
+
+        total_found = len(all_matches)
+
+        if total_found == 0:
+            print("No matches found.")
+            return [] if return_matches else None
+
+        if max_matches is not None:
+            all_matches = all_matches[:max_matches]
+
+        matches_out: List[Dict[str, Any]] = []
+        for idx, match in enumerate(all_matches, start=1):
+            start = match.start()
+            end = match.end()
+            context_start = max(0, start - window_left)
+            context_end = min(len(where), end + window_right)
+            context = where[context_start:context_end]
+
+            print(
+                f"\n\n-------\n\nMatch number {idx} of {total_found} "
+                f"for\n\n|-> {what} <-| in {start}:{end}"
+            )
+            print(
+                "Context: \n\n",
+                repr(context),
+                "\n\n",
+                end='\n_________________\n   EndOfPrinting\n_________________'
+            )
+
+            matches_out.append({
+                "match_number": idx,
+                "total_matches": total_found,
+                "start": start,
+                "end": end,
+                "matched_text": match.group(0),
+                "context": context,
+            })
+
+        return matches_out if return_matches else None
     
     def analyze_html_fonts(self, file_index: int):
         """