aait 2.2.2.3__tar.gz → 2.2.3__tar.gz

aait-2.2.3/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.1
+Name: aait
+Version: 2.2.3
+Summary: Advanced Artificial Intelligence Tools is a package meant to develop and enable advanced AI functionalities in Orange
+Home-page: 
+Author: Orange community
+Author-email: 
+Keywords: orange3 add-on
+License-File: License.txt

aait-2.2.3/aait.egg-info/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.1
+Name: aait
+Version: 2.2.3
+Summary: Advanced Artificial Intelligence Tools is a package meant to develop and enable advanced AI functionalities in Orange
+Home-page: 
+Author: Orange community
+Author-email: 
+Keywords: orange3 add-on
+License-File: License.txt

{aait-2.2.2.3 → aait-2.2.3}/aait.egg-info/SOURCES.txt

@@ -105,12 +105,10 @@ orangecontrib/AAIT/widgets/OWGenerateQuestions.py
 orangecontrib/AAIT/widgets/OWGenerateSynthesis.py
 orangecontrib/AAIT/widgets/OWGenerateWord.py
 orangecontrib/AAIT/widgets/OWInputSelector.py
-orangecontrib/AAIT/widgets/OWKeywords.py
 orangecontrib/AAIT/widgets/OWKeywordsDetection.py
 orangecontrib/AAIT/widgets/OWLMStudio.py
 orangecontrib/AAIT/widgets/OWLanguageDetection.py
 orangecontrib/AAIT/widgets/OWLemmatizer.py
-orangecontrib/AAIT/widgets/OWLoadDocuments.py
 orangecontrib/AAIT/widgets/OWModel_CE_MiniLML6.py
 orangecontrib/AAIT/widgets/OWModel_Falcon.py
 orangecontrib/AAIT/widgets/OWModel_HelsinkiEnFr.py
@@ -171,7 +169,6 @@ orangecontrib/AAIT/widgets/designer/owkeyword.ui
 orangecontrib/AAIT/widgets/designer/owkeywordsdetection.ui
 orangecontrib/AAIT/widgets/designer/owlangdetect.ui
 orangecontrib/AAIT/widgets/designer/owlemmatizer.ui
-orangecontrib/AAIT/widgets/designer/owloaddocuments.ui
 orangecontrib/AAIT/widgets/designer/owloadworkflow.ui
 orangecontrib/AAIT/widgets/designer/owmodel_ce_minilml6.ui
 orangecontrib/AAIT/widgets/designer/owmodel_falcon.ui
@@ -230,7 +227,7 @@ orangecontrib/AAIT/widgets/icons/optimisation.png
 orangecontrib/AAIT/widgets/icons/optimizer.png
 orangecontrib/AAIT/widgets/icons/output.png
 orangecontrib/AAIT/widgets/icons/owCN2_intersect_rules.svg
-orangecontrib/AAIT/widgets/icons/owaccumulatator.png
+orangecontrib/AAIT/widgets/icons/owaccumulator.png
 orangecontrib/AAIT/widgets/icons/owchunking.png
 orangecontrib/AAIT/widgets/icons/owconversellm.svg
 orangecontrib/AAIT/widgets/icons/owedittable.svg
@@ -246,10 +243,8 @@ orangecontrib/AAIT/widgets/icons/owfilewithpath.svg
 orangecontrib/AAIT/widgets/icons/owfusion_nm.png
 orangecontrib/AAIT/widgets/icons/owgeneratequestions.svg
 orangecontrib/AAIT/widgets/icons/owgeneratesynthesis.png
-orangecontrib/AAIT/widgets/icons/owkeywords.png
 orangecontrib/AAIT/widgets/icons/owkeywordsdetection.png
 orangecontrib/AAIT/widgets/icons/owlemmatizer.svg
-orangecontrib/AAIT/widgets/icons/owloaddocuments.svg
 orangecontrib/AAIT/widgets/icons/owmodel_ce_minilml6.svg
 orangecontrib/AAIT/widgets/icons/owmodel_falcon.png
 orangecontrib/AAIT/widgets/icons/owmodel_helsinki_en_fr.svg

{aait-2.2.2.3 → aait-2.2.3}/orangecontrib/AAIT/llm/answers.py

@@ -224,7 +224,7 @@ def write_tokens_to_file(token: str, workflow_id=""):
             f.flush()
 
 
-def run_query(prompt, model, max_tokens=4096, temperature=0, top_p=0, top_k=40, repeat_penalty=1.15,
+def run_query(prompt, model, max_tokens=4096, temperature=0.4, top_p=0.4, top_k=40, repeat_penalty=1.15,
               workflow_id="", argself=None, progress_callback=None):
     """
     Generates a response from a local LLM model using the given prompt, with support for streaming output

{aait-2.2.2.3 → aait-2.2.3}/orangecontrib/AAIT/llm/process_documents.py

@@ -9,31 +9,37 @@ import fitz
 import docx
 
 
-
 def process_documents(dirpath):
-    if dirpath is None or not os.path.exists(dirpath):
+    if dirpath is None or not Path(dirpath).exists():
         return None, None
 
+    # Normalize dirpath
+    dirpath = Path(dirpath).resolve()
+
     # get path from user selection
     embeddings = check_for_embeddings(dirpath)
-    dirpath = dirpath.replace("\\","/")
 
     # Set selected path in the saved embeddings
     if embeddings is not None:
-        common_path = find_common_root(embeddings).replace("\\","/")
+        common_path = Path(find_common_root(embeddings)).resolve()
         for row in embeddings:
-            row["path"] = row["path"].value.replace("\\","/").replace(common_path, dirpath)
+            row_path = Path(str(row["path"].value)).resolve()
+            # Replace common root with the current dirpath
+            try:
+                row["path"] = str(dirpath / row_path.relative_to(common_path))
+            except ValueError:
+                # If relative_to fails (paths not matching), just normalize
+                row["path"] = str(row_path)
 
     # Verify which files are already processed
     files_to_process = get_files_to_process(dirpath, embeddings)
 
     rows = []
     for file in files_to_process:
-        # Get the text content from the file
-        content = extract_text(file)
-        filename = ntpath.basename(file)
-        # Build a row containing dirpath | filename | content
-        row = [file, filename, content]
+        file = Path(file).resolve()
+        content = extract_text(str(file))   # extractor may expect string
+        filename = file.name
+        row = [str(file), filename, content]  # store strings in Orange table
         rows.append(row)
 
     # Build a table with the constructed rows
@@ -45,136 +51,115 @@ def process_documents(dirpath):
     return out_data, embeddings
 
 
+def check_for_embeddings(folder_path):
+    """
+    Check for an embeddings.pkl file in a given folder. Return its content if it exists.
+
+    Parameters:
+        folder_path (str | Path): The path to the folder where embeddings.pkl may exist.
+
+    Returns:
+        Table or None: The content of embeddings.pkl, or None if not found.
+    """
+    folder_path = Path(folder_path).resolve()
+
+    filepaths = [
+        folder_path / "embeddings_question.pkl",
+        folder_path / "embeddings.pkl"
+    ]
+
+    for filepath in filepaths:
+        if filepath.exists():
+            return Table.from_file(str(filepath))  # Table.from_file expects a str
+    return None
+
+
 def find_common_root(data_table, column_name="path"):
     """Finds the common root path from a column of file paths in an Orange Data Table."""
-    paths = [str(row[column_name]) for row in data_table if row[column_name] is not None]
+    paths = [Path(str(row[column_name].value)).resolve()
+             for row in data_table if row[column_name] is not None]
     if not paths:
         return ""
-    return os.path.commonpath(paths)
+    return str(Path(os.path.commonpath(paths)).resolve())
 
 
 def get_files_to_process(folder_path, table=None):
     """
-    Finds all PDF files in a folder (including subfolders) that are not already in the table.
-    The comparison is based on "name" (relative path from the main folder) instead of full paths.
+    Finds all PDF/DOCX files in a folder (including subfolders) that are not already in the table
+    or that have changed since last check (based on file size).
 
     :param folder_path: Path to the folder to scan for documents.
-    :param table: Orange Data Table with columns "path", "name", and "content".
-    :return: List of paths to files not present in the table (by name, including subfolder structure).
+    :param table: Orange Data Table with column "path".
+    :return: List of new/changed file paths.
     """
-    #TODO
-    # Supported file extensions
-    supported_extensions = [".pdf", ".docx"]
+    supported_extensions = {".pdf", ".docx"}
 
-    # Read the json containing file sizes
-    filepath_sizes = os.path.join(folder_path, "sizes.json")
-    if os.path.exists(filepath_sizes):
+    folder_path = Path(folder_path).resolve()
+    filepath_sizes = folder_path / "sizes.json"
+
+    print(filepath_sizes)
+    print(filepath_sizes.exists())
+
+    # Load previous file sizes
+    if filepath_sizes.exists():
         with open(filepath_sizes, "r") as json_file:
-            sizes = json.load(json_file)
+            sizes = {Path(k): v for k, v in json.load(json_file).items()}
     else:
-        sizes = dict()
+        sizes = {}
+
+    print(sizes)
 
-    # Extract the existing file names from the Orange Data Table
+    # Extract the existing paths from the Orange Data Table
     if table:
-        existing_paths = set(table[:, "path"].metas.flatten())  # Extract names from the table
+        # Orange stores metas as strings → turn them into Paths
+        existing_paths = {Path(str(p)).resolve() for p in table[:, "path"].metas.flatten()}
     else:
         existing_paths = set()
 
-
-    # Walk through the folder and its subfolders
     new_files = []
-    for root, _, files in os.walk(folder_path):
-        for file in files:
-            # Check if the file has a supported extension
-            if os.path.splitext(file)[1].lower() in supported_extensions:
-                # Add the file if it is not already in the table
-                filepath = os.path.join(root, file).replace("\\","/")
-                if filepath not in existing_paths:
-                    new_files.append(filepath)
-                    sizes[filepath] = os.path.getsize(filepath)
-                # If the file is in the table, verify if the file has been modified (comparing the size)
-                else:
-                    new_size = os.path.getsize(filepath)
-                    if filepath not in sizes.keys():
-                        sizes[filepath] = new_size
-                    else:
-                        old_size = sizes[filepath]
-                        if old_size != new_size:
-                            new_files.append(filepath)
-                            table = remove_from_table(filepath, table)
-                            sizes[filepath] = new_size
-    with open(filepath_sizes, "w") as json_file:
-        json.dump(sizes, json_file, indent=4)
-    return new_files
 
+    # Walk through the folder and subfolders
+    for file in folder_path.rglob("*"):
+        if file.suffix.lower() in supported_extensions:
+            file = file.resolve()
+            size = file.stat().st_size
 
-def remove_from_table(filepath, table):
-    filtered_table = Table.from_list(domain=table.domain,
-                                      rows=[row for row in table if row["path"].value != filepath])
-    return filtered_table
-
+            if file not in existing_paths:
+                # New file
+                new_files.append(str(file))
+                sizes[Path(ntpath.basename(file))] = size
+            else:
+                # File already in table: check if size changed
+                print("File: ", Path(ntpath.basename(file)))
+                old_size = sizes.get(Path(ntpath.basename(file)))
+                print(old_size)
+                if old_size is None or old_size != size:
+                    new_files.append(str(file))
+                    table = remove_from_table(file, table)
+                    sizes[Path(ntpath.basename(file))] = size
 
-def check_for_embeddings(folder_path):
-    """
-    Check for an embeddings.pkl file in a given folder. Return its content if it exists.
 
-    Parameters:
-        folder_path (str): The path to the folder where embeddings.pkl may exist.
+    # Save updated sizes.json (keys must be strings for JSON)
+    with open(filepath_sizes, "w") as json_file:
+        json.dump({str(k): v for k, v in sizes.items()}, json_file, indent=4)
 
-    Returns:
-        Table or None: The content of embeddings.pkl.
-    """
-    filepaths = [os.path.join(folder_path, "embeddings_question.pkl"),
-                 os.path.join(folder_path, "embeddings.pkl")]
-    for filepath in filepaths:
-        if os.path.exists(filepath):
-            data = Table.from_file(filepath)
-            return data
-    else:
-        return None
+    return new_files
 
 
-def load_documents_in_table(table, progress_callback=None, argself=None):
+def remove_from_table(filepath, table):
     """
-    Load the text content of each document listed in a table and add it
-    as a new column "content".
-
-    :param table: Orange.data.Table containing file paths in a column named "path".
-    :return: Orange.data.Table with an added meta column "content" containing the extracted text.
+    Remove rows from the Orange table where 'path' matches the given filepath.
     """
-    # Make a copy of the table to avoid modifying the original
-    data = table.copy()
-
-    # List to store text from each document
-    texts = []
-    names = []
-    # Iterate over all rows in the table
-    for i, row in enumerate(data):
-        # Get file path from the "path" column
-        filepath = row["path"].value
-        # Get text and name
-        name = Path(filepath).name
-        text = extract_text(filepath)
-        # Store results
-        names.append(name)
-        texts.append(text)
-        # Update progress if a callback is provided
-        if progress_callback is not None:
-            progress_value = float(100 * (i + 1) / len(data))
-            progress_callback(progress_value)
-        # Check if processing should be stopped
-        if argself is not None and getattr(argself, "stop", False):
-            break
-
-    # Create a StringVariable for the new column
-    var_content = StringVariable("content")
-    var_name = StringVariable("name")
-
-    # Add the column as a meta-column in the table
-    data = data.add_column(variable=var_name, data=names, to_metas=True)
-    data = data.add_column(variable=var_content, data=texts, to_metas=True)
-    return data
-
+    filepath = Path(filepath).resolve()
+
+    filtered_table = Table.from_list(
+        domain=table.domain,
+        rows=[
+            row for row in table
+            if Path(str(row["path"].value)).resolve() != filepath
+        ]
+    )
+    return filtered_table
 
 
 def extract_text(filepath):
@@ -192,13 +177,11 @@ def extract_text(filepath):
             return extract_text_from_pdf(filepath)
         elif file_extension == ".docx":
             return extract_text_from_docx(filepath)
-        elif file_extension in [".txt", ".md"]:
-            return extract_text_from_txt(filepath)
         else:
-            return "ERROR: Unsupported file format. Please use a .pdf, .docx, .txt, or .md file."
+            raise ValueError("Format de fichier non supporté. Utilisez un fichier PDF ou DOCX.")
     except Exception as e:
         print(f"Erreur lors de l'extraction de texte depuis {filepath}: {e}")
-        return f"ERROR: Extraction Error ({e})"
+        return "Extraction Error"
 
 
 def extract_text_from_pdf(pdf_path):
@@ -222,7 +205,7 @@ def extract_text_from_pdf(pdf_path):
         return extracted_text
     except Exception as e:
         print(f"Erreur lors de l'extraction de texte depuis {pdf_path}: {e}")
-        return f"ERROR: Extraction Error ({e})"
+        return "Extraction Error"
 
 
 def extract_text_from_docx(docx_path):
@@ -267,20 +250,13 @@ def extract_text_from_docx(docx_path):
                 row_text = [cell.text.strip() for cell in row.cells]  # Extrait le texte de chaque cellule
                 table_text.append("\t".join(row_text))  # Sépare les colonnes par des tabulations
             extracted_text.append("\n".join(table_text))  # Ajoute le tableau sous forme de texte
+
         return "\n".join(filter(None, extracted_text))  # Retourne le texte en filtrant les vides
 
     except Exception as e:
         print(f"Erreur lors de l'extraction de texte depuis {docx_path}: {e}")
-        return f"ERROR: Extraction Error ({e})"
-
+        return "Extraction Error"
 
-def extract_text_from_txt(filepath):
-    try:
-        with open(filepath, 'r', encoding='utf-8') as f:
-            return f.read()
-    except Exception as e:
-        print(f"Erreur lors de l'extraction de texte depuis {filepath}: {e}")
-        return f"ERROR: Extraction Error ({e})"
 
 
 def get_pages_of_extract(pdf_path, extract):

{aait-2.2.2.3 → aait-2.2.3}/orangecontrib/AAIT/utils/MetManagement.py

@@ -1368,19 +1368,6 @@ def write_file_time(path):
     except Exception as e:
         print(f"Error creating .ok file: {e}")
     return
-def write_file_arbitrary_time(path,time):
-    time= int(time)
-    time_ok=path[:-4]+".ok"
-    reset_files([time_ok])
-    with open(path, "w") as f:
-        f.write(str(time))
-        f.flush()
-    try:
-        with open(time_ok, "w"):
-            pass
-    except Exception as e:
-        print(f"Error creating .ok file: {e}")
-    return
 
 def read_file_time(path):
     time_ok = path[:-4] + ".ok"