aait 2.2.2__tar.gz → 2.2.2.1__tar.gz

aait-2.2.2.1/PKG-INFO

@@ -0,0 +1,28 @@
+Metadata-Version: 2.1
+Name: aait
+Version: 2.2.2.1
+Summary: Advanced Artificial Intelligence Tools is a package meant to develop and enable advanced AI functionalities in Orange
+Home-page: 
+Author: Orange community
+Author-email: 
+Keywords: orange3 add-on
+License-File: License.txt
+Requires-Dist: torch
+Requires-Dist: sentence-transformers==5.0.0
+Requires-Dist: gpt4all[all]==2.8.2
+Requires-Dist: sacremoses==0.1.1
+Requires-Dist: transformers==4.51.3
+Requires-Dist: sentencepiece==0.2.0
+Requires-Dist: optuna
+Requires-Dist: spacy==3.7.6
+Requires-Dist: markdown
+Requires-Dist: python-multipart
+Requires-Dist: PyMuPDF==1.24.14
+Requires-Dist: chonkie==0.4.1
+Requires-Dist: GPUtil==1.4.0
+Requires-Dist: unidecode==1.3.8
+Requires-Dist: python-docx==1.1.2
+Requires-Dist: psutil
+Requires-Dist: thefuzz==0.22.1
+Requires-Dist: beautifulsoup4==4.12.3
+Requires-Dist: CATEGORIT

aait-2.2.2.1/aait.egg-info/PKG-INFO

@@ -0,0 +1,28 @@
+Metadata-Version: 2.1
+Name: aait
+Version: 2.2.2.1
+Summary: Advanced Artificial Intelligence Tools is a package meant to develop and enable advanced AI functionalities in Orange
+Home-page: 
+Author: Orange community
+Author-email: 
+Keywords: orange3 add-on
+License-File: License.txt
+Requires-Dist: torch
+Requires-Dist: sentence-transformers==5.0.0
+Requires-Dist: gpt4all[all]==2.8.2
+Requires-Dist: sacremoses==0.1.1
+Requires-Dist: transformers==4.51.3
+Requires-Dist: sentencepiece==0.2.0
+Requires-Dist: optuna
+Requires-Dist: spacy==3.7.6
+Requires-Dist: markdown
+Requires-Dist: python-multipart
+Requires-Dist: PyMuPDF==1.24.14
+Requires-Dist: chonkie==0.4.1
+Requires-Dist: GPUtil==1.4.0
+Requires-Dist: unidecode==1.3.8
+Requires-Dist: python-docx==1.1.2
+Requires-Dist: psutil
+Requires-Dist: thefuzz==0.22.1
+Requires-Dist: beautifulsoup4==4.12.3
+Requires-Dist: CATEGORIT

{aait-2.2.2 → aait-2.2.2.1}/aait.egg-info/SOURCES.txt

@@ -1,4 +1,5 @@
 License.txt
+setup.cfg
 setup.py
 aait.egg-info/PKG-INFO
 aait.egg-info/SOURCES.txt

{aait-2.2.2 → aait-2.2.2.1}/orangecontrib/AAIT/llm/process_documents.py

@@ -1,7 +1,6 @@
 import os
 import json
 import ntpath
-from pathlib import Path
 
 from Orange.data import Table, Domain, StringVariable
 
@@ -9,37 +8,31 @@ import fitz
 import docx
 
 
+
 def process_documents(dirpath):
-    if dirpath is None or not Path(dirpath).exists():
+    if dirpath is None or not os.path.exists(dirpath):
         return None, None
 
-    # Normalize dirpath
-    dirpath = Path(dirpath).resolve()
-
     # get path from user selection
     embeddings = check_for_embeddings(dirpath)
+    dirpath = dirpath.replace("\\","/")
 
     # Set selected path in the saved embeddings
     if embeddings is not None:
-        common_path = Path(find_common_root(embeddings)).resolve()
+        common_path = find_common_root(embeddings).replace("\\","/")
         for row in embeddings:
-            row_path = Path(str(row["path"].value)).resolve()
-            # Replace common root with the current dirpath
-            try:
-                row["path"] = str(dirpath / row_path.relative_to(common_path))
-            except ValueError:
-                # If relative_to fails (paths not matching), just normalize
-                row["path"] = str(row_path)
+            row["path"] = row["path"].value.replace("\\","/").replace(common_path, dirpath)
 
     # Verify which files are already processed
     files_to_process = get_files_to_process(dirpath, embeddings)
 
     rows = []
     for file in files_to_process:
-        file = Path(file).resolve()
-        content = extract_text(str(file))   # extractor may expect string
-        filename = file.name
-        row = [str(file), filename, content]  # store strings in Orange table
+        # Get the text content from the file
+        content = extract_text(file)
+        filename = ntpath.basename(file)
+        # Build a row containing dirpath | filename | content
+        row = [file, filename, content]
         rows.append(row)
 
     # Build a table with the constructed rows
@@ -51,115 +44,93 @@ def process_documents(dirpath):
     return out_data, embeddings
 
 
-def check_for_embeddings(folder_path):
-    """
-    Check for an embeddings.pkl file in a given folder. Return its content if it exists.
-
-    Parameters:
-        folder_path (str | Path): The path to the folder where embeddings.pkl may exist.
-
-    Returns:
-        Table or None: The content of embeddings.pkl, or None if not found.
-    """
-    folder_path = Path(folder_path).resolve()
-
-    filepaths = [
-        folder_path / "embeddings_question.pkl",
-        folder_path / "embeddings.pkl"
-    ]
-
-    for filepath in filepaths:
-        if filepath.exists():
-            return Table.from_file(str(filepath))  # Table.from_file expects a str
-    return None
-
-
 def find_common_root(data_table, column_name="path"):
     """Finds the common root path from a column of file paths in an Orange Data Table."""
-    paths = [Path(str(row[column_name].value)).resolve()
-             for row in data_table if row[column_name] is not None]
+    paths = [str(row[column_name]) for row in data_table if row[column_name] is not None]
     if not paths:
         return ""
-    return str(Path(os.path.commonpath(paths)).resolve())
+    return os.path.commonpath(paths)
 
 
 def get_files_to_process(folder_path, table=None):
     """
-    Finds all PDF/DOCX files in a folder (including subfolders) that are not already in the table
-    or that have changed since last check (based on file size).
+    Finds all PDF files in a folder (including subfolders) that are not already in the table.
+    The comparison is based on "name" (relative path from the main folder) instead of full paths.
 
     :param folder_path: Path to the folder to scan for documents.
-    :param table: Orange Data Table with column "path".
-    :return: List of new/changed file paths.
+    :param table: Orange Data Table with columns "path", "name", and "content".
+    :return: List of paths to files not present in the table (by name, including subfolder structure).
     """
-    supported_extensions = {".pdf", ".docx"}
+    #TODO
+    # Supported file extensions
+    supported_extensions = [".pdf", ".docx"]
 
-    folder_path = Path(folder_path).resolve()
-    filepath_sizes = folder_path / "sizes.json"
-
-    print(filepath_sizes)
-    print(filepath_sizes.exists())
-
-    # Load previous file sizes
-    if filepath_sizes.exists():
+    # Read the json containing file sizes
+    filepath_sizes = os.path.join(folder_path, "sizes.json")
+    if os.path.exists(filepath_sizes):
         with open(filepath_sizes, "r") as json_file:
-            sizes = {Path(k): v for k, v in json.load(json_file).items()}
+            sizes = json.load(json_file)
     else:
-        sizes = {}
+        sizes = dict()
 
-    print(sizes)
-
-    # Extract the existing paths from the Orange Data Table
+    # Extract the existing file names from the Orange Data Table
     if table:
-        # Orange stores metas as strings → turn them into Paths
-        existing_paths = {Path(str(p)).resolve() for p in table[:, "path"].metas.flatten()}
+        existing_paths = set(table[:, "path"].metas.flatten())  # Extract names from the table
     else:
         existing_paths = set()
 
-    new_files = []
-
-    # Walk through the folder and subfolders
-    for file in folder_path.rglob("*"):
-        if file.suffix.lower() in supported_extensions:
-            file = file.resolve()
-            size = file.stat().st_size
-
-            if file not in existing_paths:
-                # New file
-                new_files.append(str(file))
-                sizes[Path(ntpath.basename(file))] = size
-            else:
-                # File already in table: check if size changed
-                print("File: ", Path(ntpath.basename(file)))
-                old_size = sizes.get(Path(ntpath.basename(file)))
-                print(old_size)
-                if old_size is None or old_size != size:
-                    new_files.append(str(file))
-                    table = remove_from_table(file, table)
-                    sizes[Path(ntpath.basename(file))] = size
 
-
-    # Save updated sizes.json (keys must be strings for JSON)
+    # Walk through the folder and its subfolders
+    new_files = []
+    for root, _, files in os.walk(folder_path):
+        for file in files:
+            # Check if the file has a supported extension
+            if os.path.splitext(file)[1].lower() in supported_extensions:
+                # Add the file if it is not already in the table
+                filepath = os.path.join(root, file).replace("\\","/")
+                if filepath not in existing_paths:
+                    new_files.append(filepath)
+                    sizes[filepath] = os.path.getsize(filepath)
+                # If the file is in the table, verify if the file has been modified (comparing the size)
+                else:
+                    new_size = os.path.getsize(filepath)
+                    if filepath not in sizes.keys():
+                        sizes[filepath] = new_size
+                    else:
+                        old_size = sizes[filepath]
+                        if old_size != new_size:
+                            new_files.append(filepath)
+                            table = remove_from_table(filepath, table)
+                            sizes[filepath] = new_size
     with open(filepath_sizes, "w") as json_file:
-        json.dump({str(k): v for k, v in sizes.items()}, json_file, indent=4)
-
+        json.dump(sizes, json_file, indent=4)
     return new_files
 
 
 def remove_from_table(filepath, table):
+    filtered_table = Table.from_list(domain=table.domain,
+                                      rows=[row for row in table if row["path"].value != filepath])
+    return filtered_table
+
+
+def check_for_embeddings(folder_path):
     """
-    Remove rows from the Orange table where 'path' matches the given filepath.
+    Check for an embeddings.pkl file in a given folder. Return its content if it exists.
+
+    Parameters:
+        folder_path (str): The path to the folder where embeddings.pkl may exist.
+
+    Returns:
+        Table or None: The content of embeddings.pkl.
     """
-    filepath = Path(filepath).resolve()
-
-    filtered_table = Table.from_list(
-        domain=table.domain,
-        rows=[
-            row for row in table
-            if Path(str(row["path"].value)).resolve() != filepath
-        ]
-    )
-    return filtered_table
+    filepaths = [os.path.join(folder_path, "embeddings_question.pkl"),
+                 os.path.join(folder_path, "embeddings.pkl")]
+    for filepath in filepaths:
+        if os.path.exists(filepath):
+            data = Table.from_file(filepath)
+            return data
+    else:
+        return None
 
 
 def extract_text(filepath):

{aait-2.2.2 → aait-2.2.2.1}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 NAME = "aait"
 
 # Version du package PyPI
-VERSION = "2.2.2"  # la version doit être supérieure à la précédente sinon la publication sera refusée
+VERSION = "2.2.2.1"  # la version doit être supérieure à la précédente sinon la publication sera refusée
 
 # Facultatif / Adaptable à souhait
 AUTHOR = "Orange community"

aait-2.2.2/PKG-INFO

@@ -1,9 +0,0 @@
-Metadata-Version: 2.1
-Name: aait
-Version: 2.2.2
-Summary: Advanced Artificial Intelligence Tools is a package meant to develop and enable advanced AI functionalities in Orange
-Home-page: 
-Author: Orange community
-Author-email: 
-Keywords: orange3 add-on
-License-File: License.txt

aait-2.2.2/aait.egg-info/PKG-INFO

@@ -1,9 +0,0 @@
-Metadata-Version: 2.1
-Name: aait
-Version: 2.2.2
-Summary: Advanced Artificial Intelligence Tools is a package meant to develop and enable advanced AI functionalities in Orange
-Home-page: 
-Author: Orange community
-Author-email: 
-Keywords: orange3 add-on
-License-File: License.txt