aait 2.2.2__tar.gz → 2.2.2.2__tar.gz

aait-2.2.2.2/PKG-INFO

@@ -0,0 +1,28 @@
+Metadata-Version: 2.1
+Name: aait
+Version: 2.2.2.2
+Summary: Advanced Artificial Intelligence Tools is a package meant to develop and enable advanced AI functionalities in Orange
+Home-page: 
+Author: Orange community
+Author-email: 
+Keywords: orange3 add-on
+License-File: License.txt
+Requires-Dist: torch
+Requires-Dist: sentence-transformers==5.0.0
+Requires-Dist: gpt4all[all]==2.8.2
+Requires-Dist: sacremoses==0.1.1
+Requires-Dist: transformers==4.51.3
+Requires-Dist: sentencepiece==0.2.0
+Requires-Dist: optuna
+Requires-Dist: spacy==3.7.6
+Requires-Dist: markdown
+Requires-Dist: python-multipart
+Requires-Dist: PyMuPDF==1.24.14
+Requires-Dist: chonkie==0.4.1
+Requires-Dist: GPUtil==1.4.0
+Requires-Dist: unidecode==1.3.8
+Requires-Dist: python-docx==1.1.2
+Requires-Dist: psutil
+Requires-Dist: thefuzz==0.22.1
+Requires-Dist: beautifulsoup4==4.12.3
+Requires-Dist: CATEGORIT

aait-2.2.2.2/aait.egg-info/PKG-INFO

@@ -0,0 +1,28 @@
+Metadata-Version: 2.1
+Name: aait
+Version: 2.2.2.2
+Summary: Advanced Artificial Intelligence Tools is a package meant to develop and enable advanced AI functionalities in Orange
+Home-page: 
+Author: Orange community
+Author-email: 
+Keywords: orange3 add-on
+License-File: License.txt
+Requires-Dist: torch
+Requires-Dist: sentence-transformers==5.0.0
+Requires-Dist: gpt4all[all]==2.8.2
+Requires-Dist: sacremoses==0.1.1
+Requires-Dist: transformers==4.51.3
+Requires-Dist: sentencepiece==0.2.0
+Requires-Dist: optuna
+Requires-Dist: spacy==3.7.6
+Requires-Dist: markdown
+Requires-Dist: python-multipart
+Requires-Dist: PyMuPDF==1.24.14
+Requires-Dist: chonkie==0.4.1
+Requires-Dist: GPUtil==1.4.0
+Requires-Dist: unidecode==1.3.8
+Requires-Dist: python-docx==1.1.2
+Requires-Dist: psutil
+Requires-Dist: thefuzz==0.22.1
+Requires-Dist: beautifulsoup4==4.12.3
+Requires-Dist: CATEGORIT

{aait-2.2.2 → aait-2.2.2.2}/aait.egg-info/SOURCES.txt

@@ -1,4 +1,5 @@
 License.txt
+setup.cfg
 setup.py
 aait.egg-info/PKG-INFO
 aait.egg-info/SOURCES.txt
@@ -104,6 +105,7 @@ orangecontrib/AAIT/widgets/OWGenerateQuestions.py
 orangecontrib/AAIT/widgets/OWGenerateSynthesis.py
 orangecontrib/AAIT/widgets/OWGenerateWord.py
 orangecontrib/AAIT/widgets/OWInputSelector.py
+orangecontrib/AAIT/widgets/OWKeywords.py
 orangecontrib/AAIT/widgets/OWKeywordsDetection.py
 orangecontrib/AAIT/widgets/OWLMStudio.py
 orangecontrib/AAIT/widgets/OWLanguageDetection.py
@@ -240,6 +242,7 @@ orangecontrib/AAIT/widgets/icons/owfilewithpath.svg
 orangecontrib/AAIT/widgets/icons/owfusion_nm.png
 orangecontrib/AAIT/widgets/icons/owgeneratequestions.svg
 orangecontrib/AAIT/widgets/icons/owgeneratesynthesis.png
+orangecontrib/AAIT/widgets/icons/owkeywords.png
 orangecontrib/AAIT/widgets/icons/owkeywordsdetection.png
 orangecontrib/AAIT/widgets/icons/owlemmatizer.svg
 orangecontrib/AAIT/widgets/icons/owmodel_ce_minilml6.svg

{aait-2.2.2 → aait-2.2.2.2}/orangecontrib/AAIT/llm/process_documents.py

@@ -1,7 +1,6 @@
 import os
 import json
 import ntpath
-from pathlib import Path
 
 from Orange.data import Table, Domain, StringVariable
 
@@ -9,37 +8,31 @@ import fitz
 import docx
 
 
+
 def process_documents(dirpath):
-    if dirpath is None or not Path(dirpath).exists():
+    if dirpath is None or not os.path.exists(dirpath):
         return None, None
 
-    # Normalize dirpath
-    dirpath = Path(dirpath).resolve()
-
     # get path from user selection
     embeddings = check_for_embeddings(dirpath)
+    dirpath = dirpath.replace("\\","/")
 
     # Set selected path in the saved embeddings
     if embeddings is not None:
-        common_path = Path(find_common_root(embeddings)).resolve()
+        common_path = find_common_root(embeddings).replace("\\","/")
         for row in embeddings:
-            row_path = Path(str(row["path"].value)).resolve()
-            # Replace common root with the current dirpath
-            try:
-                row["path"] = str(dirpath / row_path.relative_to(common_path))
-            except ValueError:
-                # If relative_to fails (paths not matching), just normalize
-                row["path"] = str(row_path)
+            row["path"] = row["path"].value.replace("\\","/").replace(common_path, dirpath)
 
     # Verify which files are already processed
     files_to_process = get_files_to_process(dirpath, embeddings)
 
     rows = []
     for file in files_to_process:
-        file = Path(file).resolve()
-        content = extract_text(str(file))   # extractor may expect string
-        filename = file.name
-        row = [str(file), filename, content]  # store strings in Orange table
+        # Get the text content from the file
+        content = extract_text(file)
+        filename = ntpath.basename(file)
+        # Build a row containing dirpath | filename | content
+        row = [file, filename, content]
         rows.append(row)
 
     # Build a table with the constructed rows
@@ -51,115 +44,93 @@ def process_documents(dirpath):
     return out_data, embeddings
 
 
-def check_for_embeddings(folder_path):
-    """
-    Check for an embeddings.pkl file in a given folder. Return its content if it exists.
-
-    Parameters:
-        folder_path (str | Path): The path to the folder where embeddings.pkl may exist.
-
-    Returns:
-        Table or None: The content of embeddings.pkl, or None if not found.
-    """
-    folder_path = Path(folder_path).resolve()
-
-    filepaths = [
-        folder_path / "embeddings_question.pkl",
-        folder_path / "embeddings.pkl"
-    ]
-
-    for filepath in filepaths:
-        if filepath.exists():
-            return Table.from_file(str(filepath))  # Table.from_file expects a str
-    return None
-
-
 def find_common_root(data_table, column_name="path"):
     """Finds the common root path from a column of file paths in an Orange Data Table."""
-    paths = [Path(str(row[column_name].value)).resolve()
-             for row in data_table if row[column_name] is not None]
+    paths = [str(row[column_name]) for row in data_table if row[column_name] is not None]
     if not paths:
         return ""
-    return str(Path(os.path.commonpath(paths)).resolve())
+    return os.path.commonpath(paths)
 
 
 def get_files_to_process(folder_path, table=None):
     """
-    Finds all PDF/DOCX files in a folder (including subfolders) that are not already in the table
-    or that have changed since last check (based on file size).
+    Finds all PDF files in a folder (including subfolders) that are not already in the table.
+    The comparison is based on "name" (relative path from the main folder) instead of full paths.
 
     :param folder_path: Path to the folder to scan for documents.
-    :param table: Orange Data Table with column "path".
-    :return: List of new/changed file paths.
+    :param table: Orange Data Table with columns "path", "name", and "content".
+    :return: List of paths to files not present in the table (by name, including subfolder structure).
     """
-    supported_extensions = {".pdf", ".docx"}
+    #TODO
+    # Supported file extensions
+    supported_extensions = [".pdf", ".docx"]
 
-    folder_path = Path(folder_path).resolve()
-    filepath_sizes = folder_path / "sizes.json"
-
-    print(filepath_sizes)
-    print(filepath_sizes.exists())
-
-    # Load previous file sizes
-    if filepath_sizes.exists():
+    # Read the json containing file sizes
+    filepath_sizes = os.path.join(folder_path, "sizes.json")
+    if os.path.exists(filepath_sizes):
         with open(filepath_sizes, "r") as json_file:
-            sizes = {Path(k): v for k, v in json.load(json_file).items()}
+            sizes = json.load(json_file)
     else:
-        sizes = {}
+        sizes = dict()
 
-    print(sizes)
-
-    # Extract the existing paths from the Orange Data Table
+    # Extract the existing file names from the Orange Data Table
     if table:
-        # Orange stores metas as strings → turn them into Paths
-        existing_paths = {Path(str(p)).resolve() for p in table[:, "path"].metas.flatten()}
+        existing_paths = set(table[:, "path"].metas.flatten())  # Extract names from the table
     else:
         existing_paths = set()
 
-    new_files = []
-
-    # Walk through the folder and subfolders
-    for file in folder_path.rglob("*"):
-        if file.suffix.lower() in supported_extensions:
-            file = file.resolve()
-            size = file.stat().st_size
-
-            if file not in existing_paths:
-                # New file
-                new_files.append(str(file))
-                sizes[Path(ntpath.basename(file))] = size
-            else:
-                # File already in table: check if size changed
-                print("File: ", Path(ntpath.basename(file)))
-                old_size = sizes.get(Path(ntpath.basename(file)))
-                print(old_size)
-                if old_size is None or old_size != size:
-                    new_files.append(str(file))
-                    table = remove_from_table(file, table)
-                    sizes[Path(ntpath.basename(file))] = size
 
-
-    # Save updated sizes.json (keys must be strings for JSON)
+    # Walk through the folder and its subfolders
+    new_files = []
+    for root, _, files in os.walk(folder_path):
+        for file in files:
+            # Check if the file has a supported extension
+            if os.path.splitext(file)[1].lower() in supported_extensions:
+                # Add the file if it is not already in the table
+                filepath = os.path.join(root, file).replace("\\","/")
+                if filepath not in existing_paths:
+                    new_files.append(filepath)
+                    sizes[filepath] = os.path.getsize(filepath)
+                # If the file is in the table, verify if the file has been modified (comparing the size)
+                else:
+                    new_size = os.path.getsize(filepath)
+                    if filepath not in sizes.keys():
+                        sizes[filepath] = new_size
+                    else:
+                        old_size = sizes[filepath]
+                        if old_size != new_size:
+                            new_files.append(filepath)
+                            table = remove_from_table(filepath, table)
+                            sizes[filepath] = new_size
     with open(filepath_sizes, "w") as json_file:
-        json.dump({str(k): v for k, v in sizes.items()}, json_file, indent=4)
-
+        json.dump(sizes, json_file, indent=4)
     return new_files
 
 
 def remove_from_table(filepath, table):
+    filtered_table = Table.from_list(domain=table.domain,
+                                      rows=[row for row in table if row["path"].value != filepath])
+    return filtered_table
+
+
+def check_for_embeddings(folder_path):
     """
-    Remove rows from the Orange table where 'path' matches the given filepath.
+    Check for an embeddings.pkl file in a given folder. Return its content if it exists.
+
+    Parameters:
+        folder_path (str): The path to the folder where embeddings.pkl may exist.
+
+    Returns:
+        Table or None: The content of embeddings.pkl.
     """
-    filepath = Path(filepath).resolve()
-
-    filtered_table = Table.from_list(
-        domain=table.domain,
-        rows=[
-            row for row in table
-            if Path(str(row["path"].value)).resolve() != filepath
-        ]
-    )
-    return filtered_table
+    filepaths = [os.path.join(folder_path, "embeddings_question.pkl"),
+                 os.path.join(folder_path, "embeddings.pkl")]
+    for filepath in filepaths:
+        if os.path.exists(filepath):
+            data = Table.from_file(filepath)
+            return data
+    else:
+        return None
 
 
 def extract_text(filepath):

aait-2.2.2.2/orangecontrib/AAIT/widgets/OWKeywords.py

@@ -0,0 +1,153 @@
+import os
+import sys
+
+from Orange.widgets import widget
+from Orange.widgets.utils.signals import Input, Output
+from Orange.data import Table, Domain, ContinuousVariable
+from thefuzz import fuzz
+from AnyQt.QtWidgets import QApplication
+from AnyQt.QtCore import QObject, QThread, pyqtSignal
+
+# Import intelligent selon contexte
+if "site-packages/Orange/widgets" in os.path.dirname(os.path.abspath(__file__)).replace("\\", "/"):
+    from Orange.widgets.orangecontrib.AAIT.utils.import_uic import uic
+    from Orange.widgets.orangecontrib.AAIT.utils.initialize_from_ini import apply_modification_from_python_file
+else:
+    from orangecontrib.AAIT.utils.import_uic import uic
+    from orangecontrib.AAIT.utils.initialize_from_ini import apply_modification_from_python_file
+
+# Worker pour exécution asynchrone
+class KeywordWorker(QObject):
+    finished = pyqtSignal(Table)
+    error = pyqtSignal(str)
+
+    def __init__(self, data, keywords, threshold=80):
+        super().__init__()
+        self.data = data
+        self.keywords = keywords
+        self.threshold = threshold
+
+    def extract_matched_keywords(self, text, keywords):
+        words = text.split(" ")
+        matched_keywords = []
+        for keyword in keywords:
+            best_score = max(fuzz.ratio(word.lower(), keyword.lower()) for word in words)
+            if best_score >= self.threshold:
+                matched_keywords.append(keyword)
+        return matched_keywords
+
+    def run(self):
+        try:
+            if "Content" not in self.data.domain or "Keywords" not in self.keywords.domain:
+                self.error.emit("Missing 'Content' or 'Keywords' column")
+                return
+
+            keyword_list = [
+                str(row["Keywords"])
+                for row in self.keywords
+                if str(row["Keywords"]).strip() != ""
+            ]
+
+            new_metas_vars = list(self.data.domain.metas)
+            if not any(var.name == "Keywords" for var in new_metas_vars):
+                new_metas_vars.append(ContinuousVariable("Keywords"))
+
+            new_domain = Domain(
+                self.data.domain.attributes,
+                self.data.domain.class_vars,
+                new_metas_vars
+            )
+
+            new_metas = []
+            for row in self.data:
+                text = str(row["Content"])
+                matched_keywords = self.extract_matched_keywords(text, keyword_list)
+                score = (
+                    sum(fuzz.ratio(word.lower(), kw.lower()) for kw in matched_keywords for word in text.split())
+                    / len(keyword_list)
+                    if matched_keywords else 0.0
+                )
+                new_metas.append(list(row.metas) + [score])
+
+            out_data = Table(new_domain, self.data.X, self.data.Y, new_metas)
+            self.finished.emit(out_data)
+
+        except Exception as e:
+            self.error.emit(str(e))
+
+# Définition du widget principal
+@apply_modification_from_python_file(filepath_original_widget=__file__)
+class OWKeywords(widget.OWWidget):
+    name = "Keywords Detection"
+    description = "Give the amount of keywords from in_object in in_data"
+    icon = "icons/keywords.png"
+    category = "AAIT - LLM INTEGRATION"
+    if "site-packages/Orange/widgets" in os.path.dirname(os.path.abspath(__file__)).replace("\\", "/"):
+        icon = "icons_dev/keywords.png"
+    gui = os.path.join(os.path.dirname(os.path.abspath(__file__)), "designer/owkeyword.ui")
+    want_control_area = False
+    priority = 1050
+
+    class Inputs:
+        data = Input("Content", Table)
+        keywords = Input("Keywords", Table)
+
+    class Outputs:
+        data = Output("Keywords per Content", Table)
+
+    def __init__(self):
+        super().__init__()
+        self.data = None
+        self.keywords = None
+        self.setFixedWidth(470)
+        self.setFixedHeight(300)
+        uic.loadUi(self.gui, self)
+        self.autorun = True
+        self.thread = None
+        self.worker = None
+
+    @Inputs.data
+    def set_data(self, in_data):
+        self.data = in_data
+        if self.autorun and self.keywords:
+            self.process()
+
+    @Inputs.keywords
+    def set_keywords(self, in_keywords):
+        self.keywords = in_keywords
+        if self.autorun and self.data:
+            self.process()
+
+    def process(self):
+        if self.data is None or self.keywords is None:
+            self.Outputs.data.send(None)
+            return
+
+        self.error("")  # Clear errors
+        self.thread = QThread()
+        self.worker = KeywordWorker(self.data, self.keywords)
+        self.worker.moveToThread(self.thread)
+
+        self.thread.started.connect(self.worker.run)
+        self.worker.finished.connect(self.on_worker_finished)
+        self.worker.error.connect(self.on_worker_error)
+        self.worker.finished.connect(self.thread.quit)
+        self.worker.finished.connect(self.worker.deleteLater)
+        self.thread.finished.connect(self.thread.deleteLater)
+
+        self.thread.start()
+
+    def on_worker_finished(self, out_data):
+        print("[INFO] Thread finished. Output ready.")
+        self.Outputs.data.send(out_data)
+
+    def on_worker_error(self, message):
+        print(f"[ERROR] Worker error: {message}")
+        self.error(message)
+
+
+if __name__ == "__main__":
+    app = QApplication(sys.argv)
+    ow = OWKeywords()
+    ow.show()
+    app.exec_()

{aait-2.2.2 → aait-2.2.2.2}/orangecontrib/AAIT/widgets/designer/owkeyword.ui

@@ -35,7 +35,7 @@
      </rect>
     </property>
     <property name="text">
-     <string>The input Data must contain a column &quot;content&quot;. The input Keywords must contain a column &quot;keywords&quot;. This widget will count the number of keywords that occur in the content with a fuzzy matching (percentage based on small variations).</string>
+     <string>The input Data must contain a column &quot;Content&quot;. The input Keywords must contain a column &quot;Keywords&quot;. This widget will count the number of keywords that occur in the content with a fuzzy matching (percentage based on small variations).</string>
     </property>
     <property name="textFormat">
      <enum>Qt::AutoText</enum>

{aait-2.2.2 → aait-2.2.2.2}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 NAME = "aait"
 
 # Version du package PyPI
-VERSION = "2.2.2"  # la version doit être supérieure à la précédente sinon la publication sera refusée
+VERSION = "2.2.2.2"  # la version doit être supérieure à la précédente sinon la publication sera refusée
 
 # Facultatif / Adaptable à souhait
 AUTHOR = "Orange community"

aait-2.2.2/PKG-INFO

@@ -1,9 +0,0 @@
-Metadata-Version: 2.1
-Name: aait
-Version: 2.2.2
-Summary: Advanced Artificial Intelligence Tools is a package meant to develop and enable advanced AI functionalities in Orange
-Home-page: 
-Author: Orange community
-Author-email: 
-Keywords: orange3 add-on
-License-File: License.txt

aait-2.2.2/aait.egg-info/PKG-INFO

@@ -1,9 +0,0 @@
-Metadata-Version: 2.1
-Name: aait
-Version: 2.2.2
-Summary: Advanced Artificial Intelligence Tools is a package meant to develop and enable advanced AI functionalities in Orange
-Home-page: 
-Author: Orange community
-Author-email: 
-Keywords: orange3 add-on
-License-File: License.txt