AAIT 2.2.2.2__tar.gz → 2.2.6__tar.gz

{aait-2.2.2.2 → aait-2.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aait
-Version: 2.2.2.2
+Version: 2.2.6
 Summary: Advanced Artificial Intelligence Tools is a package meant to develop and enable advanced AI functionalities in Orange
 Home-page: 
 Author: Orange community

{aait-2.2.2.2 → aait-2.2.6}/aait.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: aait
-Version: 2.2.2.2
+Version: 2.2.6
 Summary: Advanced Artificial Intelligence Tools is a package meant to develop and enable advanced AI functionalities in Orange
 Home-page: 
 Author: Orange community

{aait-2.2.2.2 → aait-2.2.6}/aait.egg-info/SOURCES.txt

@@ -66,6 +66,7 @@ orangecontrib/AAIT/utils/MetManagement.py
 orangecontrib/AAIT/utils/SimpleDialogQt.py
 orangecontrib/AAIT/utils/__init__.py
 orangecontrib/AAIT/utils/aait_repo_file.py
+orangecontrib/AAIT/utils/base_widget.py
 orangecontrib/AAIT/utils/check_data_in.py
 orangecontrib/AAIT/utils/delta_local_shared_fodler.py
 orangecontrib/AAIT/utils/import_uic.py
@@ -82,6 +83,7 @@ orangecontrib/AAIT/utils/tools/concat_splitted_pypi.py
 orangecontrib/AAIT/utils/tools/first_time_check.py
 orangecontrib/AAIT/utils/tools/owcorpus_ok.txt
 orangecontrib/AAIT/widgets/OWAAITResourcesManager.py
+orangecontrib/AAIT/widgets/OWAccumulator.py
 orangecontrib/AAIT/widgets/OWAddColumns.py
 orangecontrib/AAIT/widgets/OWApplyRules.py
 orangecontrib/AAIT/widgets/OWCN2rule_view.py
@@ -110,6 +112,7 @@ orangecontrib/AAIT/widgets/OWKeywordsDetection.py
 orangecontrib/AAIT/widgets/OWLMStudio.py
 orangecontrib/AAIT/widgets/OWLanguageDetection.py
 orangecontrib/AAIT/widgets/OWLemmatizer.py
+orangecontrib/AAIT/widgets/OWLoadDocuments.py
 orangecontrib/AAIT/widgets/OWModel_CE_MiniLML6.py
 orangecontrib/AAIT/widgets/OWModel_Falcon.py
 orangecontrib/AAIT/widgets/OWModel_HelsinkiEnFr.py
@@ -136,6 +139,7 @@ orangecontrib/AAIT/widgets/OWReranking.py
 orangecontrib/AAIT/widgets/OWSaveFilepathEntry.py
 orangecontrib/AAIT/widgets/OWSelectColumnDynamique.py
 orangecontrib/AAIT/widgets/OWSelectRowsDynamic.py
+orangecontrib/AAIT/widgets/OWSortAndSelect.py
 orangecontrib/AAIT/widgets/OWStartLoop.py
 orangecontrib/AAIT/widgets/OWTable2Corpus.py
 orangecontrib/AAIT/widgets/OWTranslation.py
@@ -146,8 +150,10 @@ orangecontrib/AAIT/widgets/__init__.py
 orangecontrib/AAIT/widgets/designer/owMarkdown.ui
 orangecontrib/AAIT/widgets/designer/ow_in_or_out_path.ui
 orangecontrib/AAIT/widgets/designer/ow_widget_random_data.ui
+orangecontrib/AAIT/widgets/designer/owaccumulator.ui
 orangecontrib/AAIT/widgets/designer/owaddcolumns.ui
 orangecontrib/AAIT/widgets/designer/owapplyrules.ui
+orangecontrib/AAIT/widgets/designer/owchunkanalysis.ui
 orangecontrib/AAIT/widgets/designer/owchunking.ui
 orangecontrib/AAIT/widgets/designer/owconcatrules.ui
 orangecontrib/AAIT/widgets/designer/owconversellm.ui
@@ -155,6 +161,7 @@ orangecontrib/AAIT/widgets/designer/owembeddings.ui
 orangecontrib/AAIT/widgets/designer/owemptyswitch.ui
 orangecontrib/AAIT/widgets/designer/owendloop.ui
 orangecontrib/AAIT/widgets/designer/owexecutescript.ui
+orangecontrib/AAIT/widgets/designer/owexecutescript_TEST.ui
 orangecontrib/AAIT/widgets/designer/owextrachunks.ui
 orangecontrib/AAIT/widgets/designer/owextracttokens.ui
 orangecontrib/AAIT/widgets/designer/owfilemetadata.ui
@@ -169,6 +176,8 @@ orangecontrib/AAIT/widgets/designer/owkeyword.ui
 orangecontrib/AAIT/widgets/designer/owkeywordsdetection.ui
 orangecontrib/AAIT/widgets/designer/owlangdetect.ui
 orangecontrib/AAIT/widgets/designer/owlemmatizer.ui
+orangecontrib/AAIT/widgets/designer/owllm4all.ui
+orangecontrib/AAIT/widgets/designer/owloaddocuments.ui
 orangecontrib/AAIT/widgets/designer/owloadworkflow.ui
 orangecontrib/AAIT/widgets/designer/owmodel_ce_minilml6.ui
 orangecontrib/AAIT/widgets/designer/owmodel_falcon.ui
@@ -194,6 +203,7 @@ orangecontrib/AAIT/widgets/designer/owreranking.ui
 orangecontrib/AAIT/widgets/designer/owsavewithpath.ui
 orangecontrib/AAIT/widgets/designer/owselect_column_dynamic.ui
 orangecontrib/AAIT/widgets/designer/owselect_row_dynamic.ui
+orangecontrib/AAIT/widgets/designer/owsortandselect.ui
 orangecontrib/AAIT/widgets/designer/owstartloop.ui
 orangecontrib/AAIT/widgets/designer/owtable2corpus.ui
 orangecontrib/AAIT/widgets/designer/owtranslation.ui
@@ -204,10 +214,12 @@ orangecontrib/AAIT/widgets/icons/MDViewer.png
 orangecontrib/AAIT/widgets/icons/Mistral.png
 orangecontrib/AAIT/widgets/icons/apply_rules.svg
 orangecontrib/AAIT/widgets/icons/category.svg
+orangecontrib/AAIT/widgets/icons/chunking_analysis.png
 orangecontrib/AAIT/widgets/icons/dark_green.txt
 orangecontrib/AAIT/widgets/icons/de.png
 orangecontrib/AAIT/widgets/icons/document_generator.png
 orangecontrib/AAIT/widgets/icons/documents.png
+orangecontrib/AAIT/widgets/icons/edge_llm.svg
 orangecontrib/AAIT/widgets/icons/endloop.png
 orangecontrib/AAIT/widgets/icons/extra_chunks.png
 orangecontrib/AAIT/widgets/icons/in_or_out.png
@@ -227,6 +239,7 @@ orangecontrib/AAIT/widgets/icons/optimisation.png
 orangecontrib/AAIT/widgets/icons/optimizer.png
 orangecontrib/AAIT/widgets/icons/output.png
 orangecontrib/AAIT/widgets/icons/owCN2_intersect_rules.svg
+orangecontrib/AAIT/widgets/icons/owaccumulator.png
 orangecontrib/AAIT/widgets/icons/owchunking.png
 orangecontrib/AAIT/widgets/icons/owconversellm.svg
 orangecontrib/AAIT/widgets/icons/owedittable.svg
@@ -245,6 +258,7 @@ orangecontrib/AAIT/widgets/icons/owgeneratesynthesis.png
 orangecontrib/AAIT/widgets/icons/owkeywords.png
 orangecontrib/AAIT/widgets/icons/owkeywordsdetection.png
 orangecontrib/AAIT/widgets/icons/owlemmatizer.svg
+orangecontrib/AAIT/widgets/icons/owloaddocuments.svg
 orangecontrib/AAIT/widgets/icons/owmodel_ce_minilml6.svg
 orangecontrib/AAIT/widgets/icons/owmodel_falcon.png
 orangecontrib/AAIT/widgets/icons/owmodel_helsinki_en_fr.svg
@@ -258,6 +272,7 @@ orangecontrib/AAIT/widgets/icons/owqueryllm.svg
 orangecontrib/AAIT/widgets/icons/owreranking.svg
 orangecontrib/AAIT/widgets/icons/owsavefilepathentry.svg
 orangecontrib/AAIT/widgets/icons/owselectcolumndynamique.png
+orangecontrib/AAIT/widgets/icons/owsortandselect.svg
 orangecontrib/AAIT/widgets/icons/owtable2corpus.svg
 orangecontrib/AAIT/widgets/icons/owtranslation.svg
 orangecontrib/AAIT/widgets/icons/owtrigger.svg

{aait-2.2.2.2 → aait-2.2.6}/orangecontrib/AAIT/llm/answers.py

@@ -1,6 +1,9 @@
 import copy
 import os
-import GPUtil
+try:
+    import GPUtil #sometimes errors occurs on gpu testing
+except:
+    pass
 import psutil
 import ntpath
 import platform
@@ -44,50 +47,56 @@ def check_gpu(model_path, argself):
         argself.can_run = False
         argself.error("Model is not compatible. It must be a .gguf format.")
         return
-    # Calculate the model size in MB with a 1500 MB buffer
-    model_size = os.path.getsize(model_path) / (1024 ** 3) * 1000
-    model_size += token_weight * int(argself.n_ctx)
-    print(f"Required memory: {model_size/1000:.2f}GB")
-    # If there is no GPU, set use_gpu to False
-    if len(GPUtil.getGPUs()) == 0:
-        argself.use_gpu = False
-        argself.information("Running on CPU. No GPU detected.")
-        return
-    # Else
-    else:
-        # Get the available VRAM on the first GPU
-        gpu = GPUtil.getGPUs()[0]
-        free_vram = gpu.memoryFree
-    # If there is not enough VRAM on GPU
-    if free_vram < model_size:
-        # Set use_gpu to False
-        argself.use_gpu = False
-        # Check for available RAM
-        available_ram = psutil.virtual_memory().available / 1024 / 1024
-        if available_ram < model_size:
-            argself.can_run = False
-            argself.error(f"Cannot run. Both GPU and CPU are too small for this model (required: {model_size/1000:.2f}GB).")
+    #  sometimes errors occurs on gpu testing
+    try:
+        # Calculate the model size in MB with a 1500 MB buffer
+        model_size = os.path.getsize(model_path) / (1024 ** 3) * 1000
+        model_size += token_weight * int(argself.n_ctx)
+        print(f"Required memory: {model_size/1000:.2f}GB")
+        # If there is no GPU, set use_gpu to False
+        if len(GPUtil.getGPUs()) == 0:
+            argself.use_gpu = False
+            argself.information("Running on CPU. No GPU detected.")
             return
+        # Else
         else:
-            argself.warning(f"Running on CPU. GPU seems to be too small for this model (available: {free_vram/1000:.2f}GB || required: {model_size/1000:.2f}GB).")
-            return
-    # If there is enough space on GPU
-    else:
-        try:
-            # Load the model and test it
-            # model = GPT4All(model_name=model_path, model_path=model_path, n_ctx=int(argself.n_ctx),
-            #                 allow_download=False, device="cuda")
-            # answer = model.generate("What if ?", max_tokens=3)
-            # # If it works, set use_gpu to True
-            argself.use_gpu = True
-            argself.information("Running on GPU.")
-            return
-        # If importing Llama and reading the model doesn't work
-        except Exception as e:
+            # Get the available VRAM on the first GPU
+            gpu = GPUtil.getGPUs()[0]
+            free_vram = gpu.memoryFree
+        # If there is not enough VRAM on GPU
+        if free_vram < model_size:
             # Set use_gpu to False
             argself.use_gpu = False
-            argself.warning(f"GPU cannot be used. (detail: {e})")
-            return
+            # Check for available RAM
+            available_ram = psutil.virtual_memory().available / 1024 / 1024
+            if available_ram < model_size:
+                argself.can_run = False
+                argself.error(f"Cannot run. Both GPU and CPU are too small for this model (required: {model_size/1000:.2f}GB).")
+                return
+            else:
+                argself.warning(f"Running on CPU. GPU seems to be too small for this model (available: {free_vram/1000:.2f}GB || required: {model_size/1000:.2f}GB).")
+                return
+        # If there is enough space on GPU
+        else:
+            try:
+                # Load the model and test it
+                # model = GPT4All(model_name=model_path, model_path=model_path, n_ctx=int(argself.n_ctx),
+                #                 allow_download=False, device="cuda")
+                # answer = model.generate("What if ?", max_tokens=3)
+                # # If it works, set use_gpu to True
+                argself.use_gpu = True
+                argself.information("Running on GPU.")
+                return
+            # If importing Llama and reading the model doesn't work
+            except Exception as e:
+                # Set use_gpu to False
+                argself.use_gpu = False
+                argself.warning(f"GPU cannot be used. (detail: {e})")
+                return
+    except:
+        argself.use_gpu = False
+        argself.warning(f"GPU cannot be used.")
+        return
 
 
 def load_model(model_path, use_gpu, n_ctx=10000):

{aait-2.2.2.2 → aait-2.2.6}/orangecontrib/AAIT/llm/process_documents.py

@@ -1,6 +1,7 @@
 import os
 import json
 import ntpath
+from pathlib import Path
 
 from Orange.data import Table, Domain, StringVariable
 
@@ -133,6 +134,49 @@ def check_for_embeddings(folder_path):
         return None
 
 
+def load_documents_in_table(table, progress_callback=None, argself=None):
+    """
+    Load the text content of each document listed in a table and add it
+    as a new column "content".
+
+    :param table: Orange.data.Table containing file paths in a column named "path".
+    :return: Orange.data.Table with an added meta column "content" containing the extracted text.
+    """
+    # Make a copy of the table to avoid modifying the original
+    data = table.copy()
+
+    # List to store text from each document
+    texts = []
+    names = []
+    # Iterate over all rows in the table
+    for i, row in enumerate(data):
+        # Get file path from the "path" column
+        filepath = row["path"].value
+        # Get text and name
+        name = Path(filepath).name
+        text = extract_text(filepath)
+        # Store results
+        names.append(name)
+        texts.append(text)
+        # Update progress if a callback is provided
+        if progress_callback is not None:
+            progress_value = float(100 * (i + 1) / len(data))
+            progress_callback(progress_value)
+        # Check if processing should be stopped
+        if argself is not None and getattr(argself, "stop", False):
+            break
+
+    # Create a StringVariable for the new column
+    var_content = StringVariable("content")
+    var_name = StringVariable("name")
+
+    # Add the column as a meta-column in the table
+    data = data.add_column(variable=var_name, data=names, to_metas=True)
+    data = data.add_column(variable=var_content, data=texts, to_metas=True)
+    return data
+
+
+
 def extract_text(filepath):
     """
     Extrait le texte d'un fichier en fonction de son type (PDF ou DOCX).
@@ -148,11 +192,13 @@ def extract_text(filepath):
             return extract_text_from_pdf(filepath)
         elif file_extension == ".docx":
             return extract_text_from_docx(filepath)
+        elif file_extension in [".txt", ".md"]:
+            return extract_text_from_txt(filepath)
         else:
-            raise ValueError("Format de fichier non supporté. Utilisez un fichier PDF ou DOCX.")
+            return "ERROR: Unsupported file format. Please use a .pdf, .docx, .txt, or .md file."
     except Exception as e:
         print(f"Erreur lors de l'extraction de texte depuis {filepath}: {e}")
-        return "Extraction Error"
+        return f"ERROR: Extraction Error ({e})"
 
 
 def extract_text_from_pdf(pdf_path):
@@ -176,7 +222,7 @@ def extract_text_from_pdf(pdf_path):
         return extracted_text
     except Exception as e:
         print(f"Erreur lors de l'extraction de texte depuis {pdf_path}: {e}")
-        return "Extraction Error"
+        return f"ERROR: Extraction Error ({e})"
 
 
 def extract_text_from_docx(docx_path):
@@ -221,13 +267,20 @@ def extract_text_from_docx(docx_path):
                 row_text = [cell.text.strip() for cell in row.cells]  # Extrait le texte de chaque cellule
                 table_text.append("\t".join(row_text))  # Sépare les colonnes par des tabulations
             extracted_text.append("\n".join(table_text))  # Ajoute le tableau sous forme de texte
-
         return "\n".join(filter(None, extracted_text))  # Retourne le texte en filtrant les vides
 
     except Exception as e:
         print(f"Erreur lors de l'extraction de texte depuis {docx_path}: {e}")
-        return "Extraction Error"
+        return f"ERROR: Extraction Error ({e})"
+
 
+def extract_text_from_txt(filepath):
+    try:
+        with open(filepath, 'r', encoding='utf-8') as f:
+            return f.read()
+    except Exception as e:
+        print(f"Erreur lors de l'extraction de texte depuis {filepath}: {e}")
+        return f"ERROR: Extraction Error ({e})"
 
 
 def get_pages_of_extract(pdf_path, extract):

{aait-2.2.2.2 → aait-2.2.6}/orangecontrib/AAIT/utils/MetManagement.py

@@ -1368,6 +1368,19 @@ def write_file_time(path):
     except Exception as e:
         print(f"Error creating .ok file: {e}")
     return
+def write_file_arbitrary_time(path,time):
+    time= int(time)
+    time_ok=path[:-4]+".ok"
+    reset_files([time_ok])
+    with open(path, "w") as f:
+        f.write(str(time))
+        f.flush()
+    try:
+        with open(time_ok, "w"):
+            pass
+    except Exception as e:
+        print(f"Error creating .ok file: {e}")
+    return
 
 def read_file_time(path):
     time_ok = path[:-4] + ".ok"

aait-2.2.6/orangecontrib/AAIT/utils/base_widget.py

@@ -0,0 +1,173 @@
+import os
+from AnyQt.QtWidgets import QWidget, QVBoxLayout, QLineEdit, QListWidget, QListWidgetItem
+from AnyQt.QtCore import pyqtSignal
+import Orange
+from Orange.widgets import gui, widget
+from Orange.widgets.settings import Setting
+from Orange.widgets.utils.signals import Input, Output
+
+if "site-packages/Orange/widgets" in os.path.dirname(os.path.abspath(__file__)).replace("\\", "/"):
+    from Orange.widgets.orangecontrib.AAIT.utils.import_uic import uic
+else:
+    from orangecontrib.AAIT.utils.import_uic import uic
+
+
+class BaseListWidget(widget.OWWidget):
+    """
+    Base Orange widget providing a filterable list of variables from the input data domain.
+
+    This class automatically:
+      - Adds a `FilterableVariableList` UI component.
+      - Handles input data and populates the list with variables.
+      - Restores the last selected variable (`selected_column_name`).
+
+    ______________________________________________________________________________________
+    How to use:
+    ⚠️ The linked .ui file **must include a `QWidget` named "placeholder"`**, where the
+    variable list will be inserted (see owexecutescript_TEST.ui).
+
+    1. Inherit from `BaseListWidget` instead of `Orange.widgets.OWWidget`.
+       Your widget will automatically have the attribute `selected_column_name`.
+
+    2. Use `selected_column_name` in your `run()` function to determine
+       which column should be processed. For example, check if it's in
+       `data.domain` or verify its type (e.g., `StringVariable`).
+
+    3. Pass `selected_column_name` to any threaded function you use:
+       ```python
+       thread_management.Thread(self.my_function, self.data, self.selected_column_name)
+       ```
+
+    4. Update your processing functions (e.g., `generate_answers`, `create_embeddings`,
+       `execute_scripts_in_table`) to accept the column name as an argument.
+
+    5. Remove any `uic.loadUi(self.gui, self)` calls from your widget, as the UI is already loaded automatically by `BaseListWidget`.
+
+    6. If you only need "Data" as input and "Data" as output, remove the Inputs, Outputs and set() methods from your widget.
+
+    6. By default, your widget already handles "Data" as input and output.
+       You **must NOT** define the `Inputs`, `Outputs`, or `set_data()` methods
+       in your subclass, because they are already implemented in `BaseListWidget`.
+
+       ⚠️ Optional: Only if you truly need to customize input handling, you can
+       override the `set_data()` method. In that case, make sure to update
+       the `var_selector` (a `FilterableVariableList`) like this:
+
+       ```python
+       self.var_selector.add_variables(self.data.domain)
+       self.var_selector.select_variable_by_name(self.selected_column_name)
+    """
+    # Settings
+    selected_column_name = Setting("Default")  # set the targeted column by default as "Default"
+
+    class Inputs:
+        data = Input("Data", Orange.data.Table)
+
+    class Outputs:
+        data = Output("Data", Orange.data.Table)
+
+    @Inputs.data
+    def set_data(self, in_data):
+        self.data = in_data
+        if self.data:
+            self.var_selector.add_variables(self.data.domain)
+            self.var_selector.select_variable_by_name(self.selected_column_name)
+        if self.autorun:
+            self.run()
+
+
+    def __init__(self):
+        super().__init__()
+        # Load the .ui file if defined
+        if self.gui and os.path.exists(self.gui):
+            uic.loadUi(self.gui, self)
+
+        # --- Insert filterable variable list ---
+        self.var_selector = FilterableVariableList(self)
+        placeholder = self.findChild(QWidget, "placeholder")  # add a QWidget in your .ui
+        if placeholder:
+            self.var_selector.setParent(placeholder)
+            self.var_selector.setGeometry(0, 0, placeholder.width(), placeholder.height())
+            self.var_selector.show()
+        self.var_selector.selected_variable.connect(self.on_variable_selected)
+
+        # Data management
+        self.data = None
+        self.autorun = True
+
+
+    def on_variable_selected(self, var_name):
+        """Update the selected column when the user clicks an item."""
+        self.selected_column_name = var_name
+
+
+
+
+# Widget Qt pour sélectionner des variables dans un widget Orange
+class FilterableVariableList(QWidget):
+    """
+    A scrollable list of Orange variables with:
+      - Single selection
+      - Colored icons based on type
+      - Filter field at the top
+    """
+    selected_variable = pyqtSignal(str)
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self.all_vars = []
+
+        # Layout
+        self.layout = QVBoxLayout(self)
+        self.filter_edit = QLineEdit()
+        self.filter_edit.setPlaceholderText("Filter variables...")
+        self.layout.addWidget(self.filter_edit)
+
+        self.list_widget = QListWidget()
+        self.list_widget.setSelectionMode(QListWidget.SingleSelection)
+        self.layout.addWidget(self.list_widget)
+
+        # Connect filter
+        self.filter_edit.textChanged.connect(self.apply_filter)
+        # Connect variable selection
+        self.list_widget.itemSelectionChanged.connect(self.on_selection_changed)
+
+
+    def add_variables(self, variables):
+        """Store all variables and populate the list"""
+        self.all_vars = variables
+        self.populate_list(self.all_vars)
+
+    def populate_list(self, variables):
+        """Populate the QListWidget with items and icons using Orange standard icons"""
+        self.list_widget.clear()
+        if variables is not None:
+            for var in variables:
+                item = QListWidgetItem(var.name)
+                item.setIcon(gui.attributeIconDict[var])  # <-- standard Orange icon
+                self.list_widget.addItem(item)
+
+    def apply_filter(self, text):
+        """Filter the list based on input"""
+        text = text.lower()
+        filtered = [v for v in self.all_vars if text in v.name.lower()]
+        self.populate_list(filtered)
+
+    def on_selection_changed(self):
+        selected = self.get_selected_name()
+        if selected:
+            self.selected_variable.emit(selected)
+
+    def get_selected_name(self):
+        """Get the selected variable's name"""
+        selected_items = self.list_widget.selectedItems()
+        return selected_items[0].text() if selected_items else None
+
+    def select_variable_by_name(self, name):
+        """Select an item in the list by its variable name."""
+        for i in range(self.list_widget.count()):
+            item = self.list_widget.item(i)
+            if item.text() == name:
+                self.list_widget.setCurrentItem(item)
+                self.list_widget.scrollToItem(item)  # optional: ensure it's visible
+                return