aait 2.2.2.2__tar.gz → 2.2.3__tar.gz

aait-2.2.3/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.1
+Name: aait
+Version: 2.2.3
+Summary: Advanced Artificial Intelligence Tools is a package meant to develop and enable advanced AI functionalities in Orange
+Home-page: 
+Author: Orange community
+Author-email: 
+Keywords: orange3 add-on
+License-File: License.txt

aait-2.2.3/aait.egg-info/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.1
+Name: aait
+Version: 2.2.3
+Summary: Advanced Artificial Intelligence Tools is a package meant to develop and enable advanced AI functionalities in Orange
+Home-page: 
+Author: Orange community
+Author-email: 
+Keywords: orange3 add-on
+License-File: License.txt

{aait-2.2.2.2 → aait-2.2.3}/aait.egg-info/SOURCES.txt

@@ -1,5 +1,4 @@
 License.txt
-setup.cfg
 setup.py
 aait.egg-info/PKG-INFO
 aait.egg-info/SOURCES.txt
@@ -82,6 +81,7 @@ orangecontrib/AAIT/utils/tools/concat_splitted_pypi.py
 orangecontrib/AAIT/utils/tools/first_time_check.py
 orangecontrib/AAIT/utils/tools/owcorpus_ok.txt
 orangecontrib/AAIT/widgets/OWAAITResourcesManager.py
+orangecontrib/AAIT/widgets/OWAccumulator.py
 orangecontrib/AAIT/widgets/OWAddColumns.py
 orangecontrib/AAIT/widgets/OWApplyRules.py
 orangecontrib/AAIT/widgets/OWCN2rule_view.py
@@ -105,7 +105,6 @@ orangecontrib/AAIT/widgets/OWGenerateQuestions.py
 orangecontrib/AAIT/widgets/OWGenerateSynthesis.py
 orangecontrib/AAIT/widgets/OWGenerateWord.py
 orangecontrib/AAIT/widgets/OWInputSelector.py
-orangecontrib/AAIT/widgets/OWKeywords.py
 orangecontrib/AAIT/widgets/OWKeywordsDetection.py
 orangecontrib/AAIT/widgets/OWLMStudio.py
 orangecontrib/AAIT/widgets/OWLanguageDetection.py
@@ -146,6 +145,7 @@ orangecontrib/AAIT/widgets/__init__.py
 orangecontrib/AAIT/widgets/designer/owMarkdown.ui
 orangecontrib/AAIT/widgets/designer/ow_in_or_out_path.ui
 orangecontrib/AAIT/widgets/designer/ow_widget_random_data.ui
+orangecontrib/AAIT/widgets/designer/owaccumulator.ui
 orangecontrib/AAIT/widgets/designer/owaddcolumns.ui
 orangecontrib/AAIT/widgets/designer/owapplyrules.ui
 orangecontrib/AAIT/widgets/designer/owchunking.ui
@@ -227,6 +227,7 @@ orangecontrib/AAIT/widgets/icons/optimisation.png
 orangecontrib/AAIT/widgets/icons/optimizer.png
 orangecontrib/AAIT/widgets/icons/output.png
 orangecontrib/AAIT/widgets/icons/owCN2_intersect_rules.svg
+orangecontrib/AAIT/widgets/icons/owaccumulator.png
 orangecontrib/AAIT/widgets/icons/owchunking.png
 orangecontrib/AAIT/widgets/icons/owconversellm.svg
 orangecontrib/AAIT/widgets/icons/owedittable.svg
@@ -242,7 +243,6 @@ orangecontrib/AAIT/widgets/icons/owfilewithpath.svg
 orangecontrib/AAIT/widgets/icons/owfusion_nm.png
 orangecontrib/AAIT/widgets/icons/owgeneratequestions.svg
 orangecontrib/AAIT/widgets/icons/owgeneratesynthesis.png
-orangecontrib/AAIT/widgets/icons/owkeywords.png
 orangecontrib/AAIT/widgets/icons/owkeywordsdetection.png
 orangecontrib/AAIT/widgets/icons/owlemmatizer.svg
 orangecontrib/AAIT/widgets/icons/owmodel_ce_minilml6.svg

{aait-2.2.2.2 → aait-2.2.3}/orangecontrib/AAIT/llm/process_documents.py

@@ -1,6 +1,7 @@
 import os
 import json
 import ntpath
+from pathlib import Path
 
 from Orange.data import Table, Domain, StringVariable
 
@@ -8,31 +9,37 @@ import fitz
 import docx
 
 
-
 def process_documents(dirpath):
-    if dirpath is None or not os.path.exists(dirpath):
+    if dirpath is None or not Path(dirpath).exists():
         return None, None
 
+    # Normalize dirpath
+    dirpath = Path(dirpath).resolve()
+
     # get path from user selection
     embeddings = check_for_embeddings(dirpath)
-    dirpath = dirpath.replace("\\","/")
 
     # Set selected path in the saved embeddings
     if embeddings is not None:
-        common_path = find_common_root(embeddings).replace("\\","/")
+        common_path = Path(find_common_root(embeddings)).resolve()
         for row in embeddings:
-            row["path"] = row["path"].value.replace("\\","/").replace(common_path, dirpath)
+            row_path = Path(str(row["path"].value)).resolve()
+            # Replace common root with the current dirpath
+            try:
+                row["path"] = str(dirpath / row_path.relative_to(common_path))
+            except ValueError:
+                # If relative_to fails (paths not matching), just normalize
+                row["path"] = str(row_path)
 
     # Verify which files are already processed
     files_to_process = get_files_to_process(dirpath, embeddings)
 
     rows = []
     for file in files_to_process:
-        # Get the text content from the file
-        content = extract_text(file)
-        filename = ntpath.basename(file)
-        # Build a row containing dirpath | filename | content
-        row = [file, filename, content]
+        file = Path(file).resolve()
+        content = extract_text(str(file))   # extractor may expect string
+        filename = file.name
+        row = [str(file), filename, content]  # store strings in Orange table
         rows.append(row)
 
     # Build a table with the constructed rows
@@ -44,93 +51,115 @@ def process_documents(dirpath):
     return out_data, embeddings
 
 
+def check_for_embeddings(folder_path):
+    """
+    Check for an embeddings.pkl file in a given folder. Return its content if it exists.
+
+    Parameters:
+        folder_path (str | Path): The path to the folder where embeddings.pkl may exist.
+
+    Returns:
+        Table or None: The content of embeddings.pkl, or None if not found.
+    """
+    folder_path = Path(folder_path).resolve()
+
+    filepaths = [
+        folder_path / "embeddings_question.pkl",
+        folder_path / "embeddings.pkl"
+    ]
+
+    for filepath in filepaths:
+        if filepath.exists():
+            return Table.from_file(str(filepath))  # Table.from_file expects a str
+    return None
+
+
 def find_common_root(data_table, column_name="path"):
     """Finds the common root path from a column of file paths in an Orange Data Table."""
-    paths = [str(row[column_name]) for row in data_table if row[column_name] is not None]
+    paths = [Path(str(row[column_name].value)).resolve()
+             for row in data_table if row[column_name] is not None]
     if not paths:
         return ""
-    return os.path.commonpath(paths)
+    return str(Path(os.path.commonpath(paths)).resolve())
 
 
 def get_files_to_process(folder_path, table=None):
     """
-    Finds all PDF files in a folder (including subfolders) that are not already in the table.
-    The comparison is based on "name" (relative path from the main folder) instead of full paths.
+    Finds all PDF/DOCX files in a folder (including subfolders) that are not already in the table
+    or that have changed since last check (based on file size).
 
     :param folder_path: Path to the folder to scan for documents.
-    :param table: Orange Data Table with columns "path", "name", and "content".
-    :return: List of paths to files not present in the table (by name, including subfolder structure).
+    :param table: Orange Data Table with column "path".
+    :return: List of new/changed file paths.
     """
-    #TODO
-    # Supported file extensions
-    supported_extensions = [".pdf", ".docx"]
+    supported_extensions = {".pdf", ".docx"}
 
-    # Read the json containing file sizes
-    filepath_sizes = os.path.join(folder_path, "sizes.json")
-    if os.path.exists(filepath_sizes):
+    folder_path = Path(folder_path).resolve()
+    filepath_sizes = folder_path / "sizes.json"
+
+    print(filepath_sizes)
+    print(filepath_sizes.exists())
+
+    # Load previous file sizes
+    if filepath_sizes.exists():
         with open(filepath_sizes, "r") as json_file:
-            sizes = json.load(json_file)
+            sizes = {Path(k): v for k, v in json.load(json_file).items()}
     else:
-        sizes = dict()
+        sizes = {}
 
-    # Extract the existing file names from the Orange Data Table
+    print(sizes)
+
+    # Extract the existing paths from the Orange Data Table
     if table:
-        existing_paths = set(table[:, "path"].metas.flatten())  # Extract names from the table
+        # Orange stores metas as strings → turn them into Paths
+        existing_paths = {Path(str(p)).resolve() for p in table[:, "path"].metas.flatten()}
     else:
         existing_paths = set()
 
-
-    # Walk through the folder and its subfolders
     new_files = []
-    for root, _, files in os.walk(folder_path):
-        for file in files:
-            # Check if the file has a supported extension
-            if os.path.splitext(file)[1].lower() in supported_extensions:
-                # Add the file if it is not already in the table
-                filepath = os.path.join(root, file).replace("\\","/")
-                if filepath not in existing_paths:
-                    new_files.append(filepath)
-                    sizes[filepath] = os.path.getsize(filepath)
-                # If the file is in the table, verify if the file has been modified (comparing the size)
-                else:
-                    new_size = os.path.getsize(filepath)
-                    if filepath not in sizes.keys():
-                        sizes[filepath] = new_size
-                    else:
-                        old_size = sizes[filepath]
-                        if old_size != new_size:
-                            new_files.append(filepath)
-                            table = remove_from_table(filepath, table)
-                            sizes[filepath] = new_size
-    with open(filepath_sizes, "w") as json_file:
-        json.dump(sizes, json_file, indent=4)
-    return new_files
 
+    # Walk through the folder and subfolders
+    for file in folder_path.rglob("*"):
+        if file.suffix.lower() in supported_extensions:
+            file = file.resolve()
+            size = file.stat().st_size
 
-def remove_from_table(filepath, table):
-    filtered_table = Table.from_list(domain=table.domain,
-                                      rows=[row for row in table if row["path"].value != filepath])
-    return filtered_table
+            if file not in existing_paths:
+                # New file
+                new_files.append(str(file))
+                sizes[Path(ntpath.basename(file))] = size
+            else:
+                # File already in table: check if size changed
+                print("File: ", Path(ntpath.basename(file)))
+                old_size = sizes.get(Path(ntpath.basename(file)))
+                print(old_size)
+                if old_size is None or old_size != size:
+                    new_files.append(str(file))
+                    table = remove_from_table(file, table)
+                    sizes[Path(ntpath.basename(file))] = size
 
 
-def check_for_embeddings(folder_path):
-    """
-    Check for an embeddings.pkl file in a given folder. Return its content if it exists.
+    # Save updated sizes.json (keys must be strings for JSON)
+    with open(filepath_sizes, "w") as json_file:
+        json.dump({str(k): v for k, v in sizes.items()}, json_file, indent=4)
 
-    Parameters:
-        folder_path (str): The path to the folder where embeddings.pkl may exist.
+    return new_files
 
-    Returns:
-        Table or None: The content of embeddings.pkl.
+
+def remove_from_table(filepath, table):
     """
-    filepaths = [os.path.join(folder_path, "embeddings_question.pkl"),
-                 os.path.join(folder_path, "embeddings.pkl")]
-    for filepath in filepaths:
-        if os.path.exists(filepath):
-            data = Table.from_file(filepath)
-            return data
-    else:
-        return None
+    Remove rows from the Orange table where 'path' matches the given filepath.
+    """
+    filepath = Path(filepath).resolve()
+
+    filtered_table = Table.from_list(
+        domain=table.domain,
+        rows=[
+            row for row in table
+            if Path(str(row["path"].value)).resolve() != filepath
+        ]
+    )
+    return filtered_table
 
 
 def extract_text(filepath):

aait-2.2.3/orangecontrib/AAIT/widgets/OWAccumulator.py

@@ -0,0 +1,147 @@
+from orangewidget.widget import OWBaseWidget
+import sys
+import os
+
+from AnyQt.QtWidgets import QApplication
+from Orange.widgets.utils.signals import Input, Output
+from Orange.data import Domain, Table
+
+if "site-packages/Orange/widgets" in os.path.dirname(os.path.abspath(__file__)).replace("\\", "/"):
+    from Orange.widgets.orangecontrib.AAIT.utils.import_uic import uic
+    from Orange.widgets.orangecontrib.AAIT.utils.initialize_from_ini import apply_modification_from_python_file
+else:
+    from orangecontrib.AAIT.utils.import_uic import uic
+    from orangecontrib.AAIT.utils.initialize_from_ini import apply_modification_from_python_file
+
+@apply_modification_from_python_file(filepath_original_widget=__file__)
+class OWAccumulator(OWBaseWidget):
+    name = "Data Accumulator (Flexible Columns)"
+    description = "Allows for data accumulation by concatenation, automatically merging non-matching columns."
+    priority = 10
+    category = "AAIT - TOOLBOX"
+    icon = "icons/owaccumulator.png"
+    if "site-packages/Orange/widgets" in os.path.dirname(os.path.abspath(__file__)).replace("\\", "/"):
+        icon = "icons_dev/owaccumulator.png"
+
+    # Le chemin du fichier .ui
+    gui = os.path.join(os.path.dirname(os.path.abspath(__file__)), "designer/owaccumulator.ui")
+    priority = 1094
+
+    want_main_area = True
+    # want_control_area est mis à False pour utiliser l'UI chargée dans mainArea
+    want_control_area = False
+
+    class Inputs:
+        data = Input("Input Data", Table, auto_summary=False)
+
+    class Outputs:
+        sample = Output("Output", Table, auto_summary=False)
+        preview = Output("Preview", Table, auto_summary=False)
+
+    def __init__(self):
+        super().__init__()
+
+        # --- NOUVEAU : Chargement de l'UI et connexion du bouton ---
+        # Charge le contenu du .ui dans l'instance du widget (self)
+        uic.loadUi(self.gui, self)
+
+        # Connexion du bouton 'pushButton_send' défini dans le .ui
+        self.pushButton_send.clicked.connect(self.push)
+
+        # --- Variables d'état ---
+        self.data = None
+        self.out_data = None
+
+        # --- Appel à post_initialized pour l'alignement ---
+        self.post_initialized()
+
+    def push(self):
+        """Sends the accumulated data and resets the accumulator."""
+        if self.data is not None:
+            self.out_data = self.data.copy()
+
+            # Send the final output
+            self.Outputs.sample.send(self.out_data)
+
+            # Purge the accumulator
+            self.data = None
+            self.Outputs.preview.send(None)  # Clear preview after purge
+            self.information("Data sent and accumulator purged.")
+        else:
+            self.Outputs.sample.send(None)
+            self.warning("Accumulator is empty, nothing to send.")
+
+    @Inputs.data
+    def set_data(self, dataset):
+        """Accumulates incoming data, merging columns if necessary, with robust variable handling."""
+        self.error("")  # Clear previous errors
+        if dataset is None:
+            # If input is cleared, clear the preview output
+            if self.data is not None:
+                self.data = None
+                self.Outputs.preview.send(None)
+            return
+
+        # --- LOGIQUE DE FUSION DES COLONNES (CONSERVÉE) ---
+
+        if self.data is None:
+            # First data block: initialize the accumulator
+            self.data = dataset.copy()
+
+        else:
+            try:
+                # 1. Collecter TOUTES les variables (attributs réguliers + méta) des deux tables
+                current_all_vars = self.data.domain.variables + self.data.domain.metas
+                new_all_vars = dataset.domain.variables + dataset.domain.metas
+
+                # Utiliser un dictionnaire pour ne conserver qu'une seule variable par nom
+                unique_vars = {}
+                for var in current_all_vars + new_all_vars:
+                    if var.name not in unique_vars:
+                        unique_vars[var.name] = var
+
+                # 2. Identifier les noms d'attributs réguliers et de méta-attributs uniques
+                current_regular_names = set(v.name for v in self.data.domain.variables)
+                new_regular_names = set(v.name for v in dataset.domain.variables)
+                current_metas_names = set(v.name for v in self.data.domain.metas)
+                new_metas_names = set(v.name for v in dataset.domain.metas)
+
+                all_vars_names = current_regular_names | new_regular_names
+                all_metas_names = current_metas_names | new_metas_names
+
+                # 3. Filtrer les variables uniques pour créer le nouveau domaine
+                all_vars = sorted([unique_vars[name] for name in all_vars_names if name not in all_metas_names],
+                                  key=lambda x: x.name)
+                all_metas = sorted([unique_vars[name] for name in all_metas_names], key=lambda x: x.name)
+
+                # 4. Créer le nouveau domaine
+                new_domain = Domain(all_vars, metas=all_metas)
+
+                # 5. Transform (expand) both tables to the new, common domain
+                data_expanded = self.data.transform(new_domain)
+                dataset_expanded = dataset.transform(new_domain)
+
+                # 6. Concatenate the rows of the two uniformly expanded tables
+                self.data = data_expanded.__class__.concatenate((data_expanded, dataset_expanded))
+
+            except Exception as e:
+                self.error(f"Data tables could not be aggregated/concatenated. Error: {e}")
+                return
+        # Send the current accumulated data for preview
+        self.Outputs.preview.send(self.data)
+
+    def post_initialized(self):
+        """
+        Method for post-initialization tasks, aligned with OWChunker structure.
+        """
+        pass
+
+
+if __name__ == "__main__":
+    app = QApplication(sys.argv)
+    my_widget = OWAccumulator()
+    my_widget.show()
+    if hasattr(app, "exec"):
+        app.exec()
+    else:
+        app.exec_()

aait-2.2.3/orangecontrib/AAIT/widgets/designer/owaccumulator.ui

@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Form</class>
+ <widget class="QWidget" name="Form">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>470</width>
+    <height>293</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Form</string>
+  </property>
+  <widget class="QGroupBox" name="groupBox">
+   <property name="geometry">
+    <rect>
+     <x>19</x>
+     <y>30</y>
+     <width>431</width>
+     <height>191</height>
+    </rect>
+   </property>
+   <property name="title">
+    <string>Parameters</string>
+   </property>
+   <widget class="QLabel" name="label_2">
+    <property name="geometry">
+     <rect>
+      <x>70</x>
+      <y>150</y>
+      <width>151</width>
+      <height>21</height>
+     </rect>
+    </property>
+    <property name="text">
+     <string>Send data</string>
+    </property>
+   </widget>
+   <widget class="QLabel" name="Description">
+    <property name="geometry">
+     <rect>
+      <x>20</x>
+      <y>10</y>
+      <width>401</width>
+      <height>61</height>
+     </rect>
+    </property>
+    <property name="text">
+     <string>This widget accumulate data in order to propagate it to a final table.</string>
+    </property>
+    <property name="textFormat">
+     <enum>Qt::AutoText</enum>
+    </property>
+    <property name="alignment">
+     <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter</set>
+    </property>
+    <property name="wordWrap">
+     <bool>true</bool>
+    </property>
+   </widget>
+   <widget class="QPushButton" name="pushButton_send">
+    <property name="geometry">
+     <rect>
+      <x>130</x>
+      <y>150</y>
+      <width>291</width>
+      <height>31</height>
+     </rect>
+    </property>
+    <property name="text">
+     <string>Send Accumulated Data</string>
+    </property>
+   </widget>
+  </widget>
+ </widget>
+ <resources/>
+ <connections/>
+</ui>

{aait-2.2.2.2 → aait-2.2.3}/orangecontrib/AAIT/widgets/designer/owkeyword.ui

@@ -35,7 +35,7 @@
      </rect>
     </property>
     <property name="text">
-     <string>The input Data must contain a column &quot;Content&quot;. The input Keywords must contain a column &quot;Keywords&quot;. This widget will count the number of keywords that occur in the content with a fuzzy matching (percentage based on small variations).</string>
+     <string>The input Data must contain a column &quot;content&quot;. The input Keywords must contain a column &quot;keywords&quot;. This widget will count the number of keywords that occur in the content with a fuzzy matching (percentage based on small variations).</string>
     </property>
     <property name="textFormat">
      <enum>Qt::AutoText</enum>

{aait-2.2.2.2 → aait-2.2.3}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 NAME = "aait"
 
 # Version du package PyPI
-VERSION = "2.2.2.2"  # la version doit être supérieure à la précédente sinon la publication sera refusée
+VERSION = "2.2.3"  # la version doit être supérieure à la précédente sinon la publication sera refusée
 
 # Facultatif / Adaptable à souhait
 AUTHOR = "Orange community"

aait-2.2.2.2/PKG-INFO

@@ -1,28 +0,0 @@
-Metadata-Version: 2.1
-Name: aait
-Version: 2.2.2.2
-Summary: Advanced Artificial Intelligence Tools is a package meant to develop and enable advanced AI functionalities in Orange
-Home-page: 
-Author: Orange community
-Author-email: 
-Keywords: orange3 add-on
-License-File: License.txt
-Requires-Dist: torch
-Requires-Dist: sentence-transformers==5.0.0
-Requires-Dist: gpt4all[all]==2.8.2
-Requires-Dist: sacremoses==0.1.1
-Requires-Dist: transformers==4.51.3
-Requires-Dist: sentencepiece==0.2.0
-Requires-Dist: optuna
-Requires-Dist: spacy==3.7.6
-Requires-Dist: markdown
-Requires-Dist: python-multipart
-Requires-Dist: PyMuPDF==1.24.14
-Requires-Dist: chonkie==0.4.1
-Requires-Dist: GPUtil==1.4.0
-Requires-Dist: unidecode==1.3.8
-Requires-Dist: python-docx==1.1.2
-Requires-Dist: psutil
-Requires-Dist: thefuzz==0.22.1
-Requires-Dist: beautifulsoup4==4.12.3
-Requires-Dist: CATEGORIT

aait-2.2.2.2/aait.egg-info/PKG-INFO

@@ -1,28 +0,0 @@
-Metadata-Version: 2.1
-Name: aait
-Version: 2.2.2.2
-Summary: Advanced Artificial Intelligence Tools is a package meant to develop and enable advanced AI functionalities in Orange
-Home-page: 
-Author: Orange community
-Author-email: 
-Keywords: orange3 add-on
-License-File: License.txt
-Requires-Dist: torch
-Requires-Dist: sentence-transformers==5.0.0
-Requires-Dist: gpt4all[all]==2.8.2
-Requires-Dist: sacremoses==0.1.1
-Requires-Dist: transformers==4.51.3
-Requires-Dist: sentencepiece==0.2.0
-Requires-Dist: optuna
-Requires-Dist: spacy==3.7.6
-Requires-Dist: markdown
-Requires-Dist: python-multipart
-Requires-Dist: PyMuPDF==1.24.14
-Requires-Dist: chonkie==0.4.1
-Requires-Dist: GPUtil==1.4.0
-Requires-Dist: unidecode==1.3.8
-Requires-Dist: python-docx==1.1.2
-Requires-Dist: psutil
-Requires-Dist: thefuzz==0.22.1
-Requires-Dist: beautifulsoup4==4.12.3
-Requires-Dist: CATEGORIT