io4it 2.1.0.4__tar.gz → 2.1.0.5__tar.gz

io4it-2.1.0.5/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.1
+Name: io4it
+Version: 2.1.0.5
+Home-page: 
+Author: 
+Author-email: 
+Keywords: orange3 add-on

io4it-2.1.0.5/io4it.egg-info/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.1
+Name: io4it
+Version: 2.1.0.5
+Home-page: 
+Author: 
+Author-email: 
+Keywords: orange3 add-on

{io4it-2.1.0.4 → io4it-2.1.0.5}/io4it.egg-info/SOURCES.txt

@@ -17,8 +17,13 @@ orangecontrib/IO4IT/utils/pool_exec_utils.py
 orangecontrib/IO4IT/utils/utils_md.py
 orangecontrib/IO4IT/widgets/OWChatGpt.py
 orangecontrib/IO4IT/widgets/OWDeep_Search.py
+orangecontrib/IO4IT/widgets/OWDoclingMarkdownizerSimple.py
 orangecontrib/IO4IT/widgets/OWExportMarkdown.py
+orangecontrib/IO4IT/widgets/OWMarkdownLoader.py
 orangecontrib/IO4IT/widgets/OWMarkdownizer.py
+orangecontrib/IO4IT/widgets/OWOfficeNormalizer.py
+orangecontrib/IO4IT/widgets/OWPdfType.py
+orangecontrib/IO4IT/widgets/OWProcessPoolExecutor.py
 orangecontrib/IO4IT/widgets/OWS3Uploader.py
 orangecontrib/IO4IT/widgets/OWS3downloader.py
 orangecontrib/IO4IT/widgets/OWS3list.py
@@ -28,28 +33,36 @@ orangecontrib/IO4IT/widgets/OWmailSender.py
 orangecontrib/IO4IT/widgets/OWwordpdf2docx.py
 orangecontrib/IO4IT/widgets/__init__.py
 orangecontrib/IO4IT/widgets/designer/__init__.py
-orangecontrib/IO4IT/widgets/designer/chart.html
 orangecontrib/IO4IT/widgets/designer/nogui.ui
 orangecontrib/IO4IT/widgets/designer/ow_file_ext_selector.ui
 orangecontrib/IO4IT/widgets/designer/owchatgpt.ui
 orangecontrib/IO4IT/widgets/designer/owdeepsearch.ui
+orangecontrib/IO4IT/widgets/designer/owdoclingmarkdownizersimple.ui
 orangecontrib/IO4IT/widgets/designer/owexportmarkdown.ui
 orangecontrib/IO4IT/widgets/designer/owmailloader.ui
 orangecontrib/IO4IT/widgets/designer/owmailsender.ui
 orangecontrib/IO4IT/widgets/designer/owmarkdownizer.ui
+orangecontrib/IO4IT/widgets/designer/owmarkdownloader.ui
+orangecontrib/IO4IT/widgets/designer/owofficenormalizer.ui
+orangecontrib/IO4IT/widgets/designer/owpdftype.ui
+orangecontrib/IO4IT/widgets/designer/owprocesspoolexecutor.ui
 orangecontrib/IO4IT/widgets/designer/owspeechtotext.ui
-orangecontrib/IO4IT/widgets/designer/owvisualizationer.ui
 orangecontrib/IO4IT/widgets/designer/wordpdf2docx.ui
 orangecontrib/IO4IT/widgets/icons/__init__.py
 orangecontrib/IO4IT/widgets/icons/chatgpt.png
+orangecontrib/IO4IT/widgets/icons/check_pdf.png
 orangecontrib/IO4IT/widgets/icons/deepsearch.svg
 orangecontrib/IO4IT/widgets/icons/download.png
 orangecontrib/IO4IT/widgets/icons/export_md.png
 orangecontrib/IO4IT/widgets/icons/file_extensor.png
 orangecontrib/IO4IT/widgets/icons/list_aws.png
+orangecontrib/IO4IT/widgets/icons/load_md.png
 orangecontrib/IO4IT/widgets/icons/mail_loader.png
 orangecontrib/IO4IT/widgets/icons/mail_writer.png
 orangecontrib/IO4IT/widgets/icons/md.png
+orangecontrib/IO4IT/widgets/icons/md_old.png
+orangecontrib/IO4IT/widgets/icons/office_normalizer.png
+orangecontrib/IO4IT/widgets/icons/process_pool_executor.png
 orangecontrib/IO4IT/widgets/icons/speech_to_text.png
 orangecontrib/IO4IT/widgets/icons/upload.png
 orangecontrib/IO4IT/widgets/icons/visualizationer.png

{io4it-2.1.0.4 → io4it-2.1.0.5}/orangecontrib/IO4IT/ocr_function/word_converter.py

@@ -16,7 +16,7 @@ def enable_long_path(path):
     """Simplifie la gestion des chemins longs sous Windows."""
     return pathlib.Path(r"\\?\\" + str(path))
 
-def convert_pdf_structure(input_dir: str, output_dir: str,ignore_exsting_out_put=False,progress_callback=None):
+def convert_pdf_structure(input_dir: str, output_dir: str,ignore_exsting_out_put=False,forceBasicConvertion=False,progress_callback=None):
     """
     return a string with log in  case of error
     Recursively lists all .pdf and .PDF files in the input directory,
@@ -60,7 +60,7 @@ def convert_pdf_structure(input_dir: str, output_dir: str,ignore_exsting_out_put
                     continue
 
 
-            if 0!= convert_pdf_with_temp(str(pdf_file),str(new_file_path)):#convert_pdf_with_temp #convert_pdf_to_docx
+            if 0!= convert_pdf_with_temp(str(pdf_file),str(new_file_path),forceBasicConvertion):#convert_pdf_with_temp #convert_pdf_to_docx
                 if error_log!="":
                     error_log+="\n"
                 error_log+="error -> "+str(pdf_file)
@@ -259,7 +259,7 @@ def write_two_strings_to_file(file_path: str,string1: str, string2: str):
     return 0
 
 
-def convert_pdf_with_temp(temp_pdf, output_path):
+def convert_pdf_with_temp(temp_pdf, output_path,forceBasicConvertion=False):
     """
     Copie le PDF source dans un dossier temporaire, le convertit en DOCX,
     puis copie le fichier résultant vers le chemin de sortie spécifié,
@@ -294,15 +294,15 @@ def convert_pdf_with_temp(temp_pdf, output_path):
         # Copie du fichier source vers le dossier temporaire
         shutil.copy2(pdf_path, temp_pdf)
         wait_for_file_access(temp_pdf)
-
-        if is_pdf_a4(temp_pdf)==False:
-            temp_pdf2 = os.path.join(dest_dir, "input_totoA4.pdf")
-            if 0!=convert_pdf_to_a4(temp_pdf,temp_pdf2):
-                print("erreur au resize du pdf")
-                return 1
-            temp_pdf=temp_pdf2
-            wait_for_file_access(temp_pdf)
-            time.sleep(1)
+        if forceBasicConvertion==False:
+            if is_pdf_a4(temp_pdf)==False:
+                temp_pdf2 = os.path.join(dest_dir, "input_totoA4.pdf")
+                if 0!=convert_pdf_to_a4(temp_pdf,temp_pdf2):
+                    print("erreur au resize du pdf")
+                    return 1
+                temp_pdf=temp_pdf2
+                wait_for_file_access(temp_pdf)
+                time.sleep(1)
         result=0
         # Conversion du PDF en DOCX
         for _ in range(4):

{io4it-2.1.0.4 → io4it-2.1.0.5}/orangecontrib/IO4IT/widgets/OWDeep_Search.py

@@ -30,7 +30,7 @@ class OWDeep_Search(widget.OWWidget):
     category = "AAIT - LLM INTEGRATION"
     icon = "icons/deepsearch.svg"
     if "site-packages/Orange/widgets" in os.path.dirname(os.path.abspath(__file__)).replace("\\", "/"):
-        icon = "icons_dev/owqueryllm.svg"
+        icon = "icons_dev/deepsearch.svg"
     gui = os.path.join(os.path.dirname(os.path.abspath(__file__)), "designer/owdeepsearch.ui")
     want_control_area = True
     priority = 1089

io4it-2.1.0.5/orangecontrib/IO4IT/widgets/OWDoclingMarkdownizerSimple.py

@@ -0,0 +1,281 @@
+import os, time, datetime
+from pathlib import Path
+from concurrent.futures import as_completed
+
+from AnyQt.QtWidgets import QLabel, QTextEdit
+from AnyQt.QtCore import pyqtSignal
+from Orange.widgets import widget
+from Orange.widgets.utils.signals import Input, Output
+from Orange.data import Domain, StringVariable, Table, DiscreteVariable
+
+# --- Docling (unique lib utilisée pour la conversion) ---
+from docling.backend.pypdfium2_backend import PyPdfiumDocumentBackend
+from docling.datamodel.base_models import InputFormat
+from docling.document_converter import (
+    DocumentConverter,
+    PdfFormatOption,
+    WordFormatOption,
+)
+from docling.pipeline.simple_pipeline import SimplePipeline
+from docling.pipeline.standard_pdf_pipeline import StandardPdfPipeline
+
+# --- Orange contrib Imports ---
+if "site-packages/Orange/widgets" in os.path.dirname(os.path.abspath(__file__)).replace("\\", "/"):
+    from Orange.widgets.orangecontrib.AAIT.utils.thread_management import Thread
+    from Orange.widgets.orangecontrib.AAIT.utils.import_uic import uic
+else:
+    from orangecontrib.AAIT.utils.thread_management import Thread
+    from orangecontrib.AAIT.utils.import_uic import uic
+
+
+# --------- worker stateless : convertit 1 fichier avec Docling ----------
+def _convert_one_file(file_path_str: str):
+    """Convertit un fichier (PDF/DOCX/PPTX) en Markdown via Docling.
+    Écrit <parent>/a_md/<stem>.md et renvoie [input_path, output_md, status, duration_sec, message].
+    Pensé pour être appelé soit directement, soit via ProcessPoolExecutor.
+    """
+    t0 = time.time()
+    src = Path(file_path_str)
+    out_dir = src.parent / "conversion_markdown"
+    out_dir.mkdir(parents=True, exist_ok=True)
+    out_md = out_dir / f"{src.stem}.md"
+
+    # Si déjà converti : on ne refait pas
+    if out_md.exists():
+        status = "ok"
+        message = "existant: deja converti"
+        duration = time.time() - t0
+        return [str(src), str(out_md), status, f"{duration:.2f}", message]
+
+    try:
+        # Docling minimal config (inspiré du snippet)
+        doc_converter = DocumentConverter(
+            allowed_formats=[InputFormat.PDF, InputFormat.DOCX, InputFormat.PPTX],
+            format_options={
+                InputFormat.PDF: PdfFormatOption(
+                    pipeline_cls=StandardPdfPipeline,
+                    backend=PyPdfiumDocumentBackend
+                ),
+                InputFormat.DOCX: WordFormatOption(
+                    pipeline_cls=SimplePipeline
+                ),
+                # PPTX: pas d'option spécifique; géré par défaut
+            },
+        )
+        doc = doc_converter.convert(str(src)).document
+        md = doc.export_to_markdown()
+        out_md.write_text(md, encoding="utf-8")
+        status, message = "ok", ""
+    except Exception as e:
+        status = "nok"
+        message = f"{type(e).__name__}: {e}"
+        # on écrit quand même un trace .md
+        try:
+            out_md.write_text(f"[Erreur conversion] {message}", encoding="utf-8")
+        except Exception:
+            pass
+
+    duration = time.time() - t0
+    return [str(src), str(out_md), status, f"{duration:.2f}", message]
+
+
+class OWDoclingMarkdownizerSimple(widget.OWWidget):
+    name = "Docling To Markdown"
+    description = "Convert DOCX/PPTX/PDF to Markdown via Docling"
+    icon = "icons/md.png"
+    if "site-packages/Orange/widgets" in os.path.dirname(os.path.abspath(__file__)).replace("\\", "/"):
+        icon = "icons_dev/md.png"
+    gui = os.path.join(os.path.dirname(os.path.abspath(__file__)), "designer/owdoclingmarkdownizersimple.ui")
+    want_main_area = False
+    want_control_area = True
+    priority = 1004
+
+    # --- New signal for incremental status updates ---
+    status_update_signal = pyqtSignal(list)
+
+    class Inputs:
+        data = Input("Files Table", Table)
+        executor = Input("ProcessPoolExecutor", object)
+
+    class Outputs:
+        data = Output("Markdown Table", Table)
+        status_data = Output("Status Table", Table)
+
+    def __init__(self):
+        super().__init__()
+        self.data = None
+        self.external_executor = None
+        self.thread = None
+        self.result = None
+        self.exec_info = QLabel("Exécution: séquentielle (aucun executor connecté).", self)
+        self.processed_statuses = {}  # Dictionary to accumulate statuses for each file
+
+        uic.loadUi(self.gui, self)
+
+        self.error("")
+        self.warning("")
+
+    @Inputs.data
+    def set_data(self, in_data: Table | None):
+        self.data = in_data
+        self.error("")
+        self.warning("")
+
+        if not in_data:
+            self.Outputs.data.send(None)
+            self.Outputs.status_data.send(None)
+            return
+
+        try:
+            _ = in_data.domain["file_path"]
+        except Exception:
+            self.error('Colonne "file_path" (Text) requise.')
+            self.Outputs.data.send(None)
+            self.Outputs.status_data.send(None)
+            return
+
+        self._convert_now()
+
+    @Inputs.executor
+    def set_executor(self, executor_obj):
+        self.external_executor = executor_obj
+        if executor_obj is not None:
+            self.exec_info.setText("Exécution: via ProcessPoolExecutor externe (parallèle).")
+        else:
+            self.exec_info.setText("Exécution: séquentielle (aucun executor connecté).")
+
+    def _convert_now(self):
+        if self.thread is not None and self.thread.isRunning():
+            self.thread.safe_quit()
+
+        if not self.data:
+            self.Outputs.data.send(None)
+            self.Outputs.status_data.send(None)
+            return
+
+        # Start progress bar
+        self.progressBarInit()
+
+        # Récupère les chemins et filtre par extensions supportées
+        try:
+            paths = [Path(str(x)) for x in self.data.get_column("file_path")]
+        except Exception as e:
+            self.error(f"Lecture de 'file_path' impossible: {e}")
+            self.Outputs.data.send(None)
+            self.Outputs.status_data.send(None)
+            return
+
+        files = [p for p in paths if p.exists() and p.suffix.lower() in (".pdf", ".docx", ".pptx")]
+        if not files:
+            self.Outputs.data.send(None)
+            self.Outputs.status_data.send(None)
+            self.progressBarFinished()
+            return
+
+        # Initialize status dictionary for incremental updates
+        self.processed_statuses = {str(p): ["pending", ""] for p in files}
+        self.update_status_output()
+
+        # Connect internal signal
+        self.status_update_signal.connect(self.handle_status_update)
+
+        # Connect and start thread for the main function
+        self.thread = Thread(self._run_conversion, files)
+        self.thread.progress.connect(self.handle_progress)
+        self.thread.result.connect(self.handle_result)
+        self.thread.finish.connect(self.handle_finish)
+        self.thread.start()
+
+    def update_status_output(self):
+        """Helper function to create and send the status table."""
+        status_domain = Domain(
+            [],  # This list must be empty because the table has no attributes.
+            metas=[
+                StringVariable("input_path"),
+                DiscreteVariable("status", values=["pending", "in_progress", "ok", "nok"]),
+                StringVariable("message"),
+            ],
+        )
+
+        status_rows = []
+        for path_str, status_info in self.processed_statuses.items():
+            status, message = status_info
+            # Orange's Table.from_list expects a flat list of values matching the domain's order.
+            # The row should contain the values for input_path, status, and message.
+            # The status_info is a list [status, message]. We need to prepend the path_str.
+            status_rows.append([path_str, status, message])
+
+        status_table = Table.from_list(status_domain, status_rows)
+        self.Outputs.status_data.send(status_table)
+
+    def _run_conversion(self, files, progress_callback):
+        """Main function to run the conversion, supports sequential and parallel modes."""
+        results = []
+
+        if self.external_executor is None:
+            # --- Mode simple séquentiel ---
+            for i, p in enumerate(files):
+                path_str = str(p)
+                # on met à jour le statut en "in_progress" et on envoie
+                self.status_update_signal.emit([path_str, "in_progress", ""])
+
+                row = _convert_one_file(path_str)
+                results.append(row)
+
+                # Mise à jour du tableau avec le résultat et envoi immédiat
+                self.status_update_signal.emit([row[0], row[2], row[4]])
+
+                progress_callback((i + 1) / len(files) * 100)
+        else:
+            # --- Mode parallèle via executor externe ---
+            fut_map = {self.external_executor.submit(_convert_one_file, str(p)): str(p) for p in files}
+
+            for i, fut in enumerate(as_completed(fut_map), 1):
+                file_path_str = fut_map[fut]
+                # on met à jour le statut en "in_progress" et on envoie
+                self.status_update_signal.emit([file_path_str, "in_progress", ""])
+
+                try:
+                    row = fut.result()
+                    results.append(row)
+                    # Mise à jour du résultat de la future et envoi
+                    self.status_update_signal.emit([row[0], row[2], row[4]])
+                except Exception as e:
+                    # Gestion des erreurs de la future et envoi
+                    row = [file_path_str, str((Path(file_path_str).parent / 'a_md' / f"{Path(file_path_str).stem}.md")),
+                           "nok", "0.00", f"FutureError: {e}"]
+                    results.append(row)
+                    self.status_update_signal.emit([row[0], "nok", f"FutureError: {e}"])
+
+                progress_callback(i / len(files) * 100)
+
+        # The final result is the table built from all results
+        domain = Domain([], metas=[
+            StringVariable("input_path"),
+            StringVariable("output_md"),
+            StringVariable("status"),
+            StringVariable("duration_sec"),
+            StringVariable("message"),
+        ])
+        final_table = Table.from_list(domain, results)
+        return final_table
+
+    def handle_progress(self, value: float) -> None:
+        self.progressBarSet(value)
+
+    def handle_status_update(self, status_info: list):
+        """Receives a single status update and updates the internal dictionary and the output."""
+        path_str, status, message = status_info
+        self.processed_statuses[path_str] = [status, message]
+        self.update_status_output()
+
+    def handle_result(self, result: Table):
+        try:
+            self.result = result
+            self.Outputs.data.send(result)
+        except Exception as e:
+            print("An error occurred when sending out_data:", e)
+            self.Outputs.data.send(None)
+
+    def handle_finish(self):
+        self.progressBarFinished()

{io4it-2.1.0.4 → io4it-2.1.0.5}/orangecontrib/IO4IT/widgets/OWExportMarkdown.py

@@ -29,7 +29,7 @@ class OWExportMarkdown(widget.OWWidget):
     icon = "icons/export_md.png"
     if "site-packages/Orange/widgets" in os.path.dirname(os.path.abspath(__file__)).replace("\\", "/"):
         icon = "icons_dev/export_md.png"
-    want_control_area = True
+    want_control_area = False
     priority = 9999
     category = "AAIT - TOOLBOX"
 

io4it-2.1.0.5/orangecontrib/IO4IT/widgets/OWMarkdownLoader.py

@@ -0,0 +1,112 @@
+# ow_markdown_loader.py
+import os
+from pathlib import Path
+import numpy as np
+
+from AnyQt.QtWidgets import QLabel, QCheckBox
+from Orange.widgets import widget
+from Orange.widgets.utils.signals import Input, Output
+from Orange.data import Domain, StringVariable, Table
+
+if "site-packages/Orange/widgets" in os.path.dirname(os.path.abspath(__file__)).replace("\\", "/"):
+    from Orange.widgets.orangecontrib.IO4IT.utils import utils_md
+else:
+    from orangecontrib.IO4IT.utils import utils_md
+
+
+class OWMarkdownLoader(widget.OWWidget):
+    name = "Markdown Loader"
+    description = "Charge tous les fichiers Markdown d’un dossier (récursif)"
+    icon = "icons/load_md.png"
+    if "site-packages/Orange/widgets" in os.path.dirname(os.path.abspath(__file__)).replace("\\", "/"):
+        icon = "icons_dev/load_md.png"
+    gui = os.path.join(os.path.dirname(os.path.abspath(__file__)), "designer/owmarkdownloader.ui")
+    want_control_area = False
+    priority = 1001
+
+    class Inputs:
+        data = Input("Data", Table)
+
+    class Outputs:
+        md_files = Output("Markdown Files", Table)   # -> (file_path, content)
+        data = Output("Data", Table)                 # passthrough de l'entrée
+
+    def __init__(self):
+        super().__init__()
+        self.in_data = None
+        self.input_dir = None
+        self.recursive = True
+
+        # UI minimal
+        self.label = QLabel(self)
+        self.checkbox = QCheckBox("Recherche récursive", self)
+        self.checkbox.setChecked(True)
+        self.checkbox.stateChanged.connect(self._on_recursive_toggled)
+
+        self.layout().addWidget(self.label)
+        self.layout().addWidget(self.checkbox)
+        self.warning("")
+
+    def _on_recursive_toggled(self, _state):
+        self.recursive = self.checkbox.isChecked()
+        # Si on a déjà un dossier, on relance la production
+        if self.input_dir:
+            self._produce()
+
+    @Inputs.data
+    def set_data(self, in_data: Table | None):
+        self.in_data = in_data
+        self.warning("")
+
+        # Toujours émettre le passthrough (même si None)
+        self.Outputs.data.send(in_data)
+
+        if not in_data:
+            # Rien à charger côté MD : on émet une table vide sur Markdown Files
+            self.Outputs.md_files.send(self._empty_md_table())
+            return
+
+        # Cherche la colonne 'input_dir' et récupère le premier dossier
+        try:
+            _ = in_data.domain["input_dir"]
+            self.input_dir = str(in_data.get_column("input_dir")[0])
+        except Exception:
+            self.warning('"input_dir" (Text) est requis en entrée')
+            self.Outputs.md_files.send(self._empty_md_table())
+            return
+
+        self.label.setText(f"Dossier : {self.input_dir}")
+        self._produce()
+
+    def _empty_md_table(self) -> Table:
+        domain = Domain([], metas=[StringVariable("file_path"), StringVariable("content")])
+        X = np.empty((0, 0))
+        metas = np.empty((0, 2), dtype=object)
+        return Table.from_numpy(domain, X, metas=metas)
+
+    def _produce(self):
+        base = Path(self.input_dir)
+        patterns = ["*.md"]
+        paths = []
+
+        for patt in patterns:
+            if self.recursive:
+                paths.extend(base.rglob(patt))
+            else:
+                paths.extend(base.glob(patt))
+
+        md_rows = []
+        for p in sorted(set(paths)):
+            try:
+                md_rows.append([str(p), utils_md.try_read_text(p)])
+            except Exception:
+                md_rows.append([str(p), ""])
+
+        # Construit la table pour "Markdown Files"
+        domain = Domain([], metas=[StringVariable("file_path"), StringVariable("content")])
+        X = np.empty((len(md_rows), 0))
+        metas = np.array(md_rows, dtype=object) if md_rows else np.empty((0, 2), dtype=object)
+        md_table = Table.from_numpy(domain, X, metas=metas)
+
+        self.Outputs.md_files.send(md_table)
+        # Le passthrough est déjà envoyé dans set_data ; on n'y retouche pas ici.

{io4it-2.1.0.4 → io4it-2.1.0.5}/orangecontrib/IO4IT/widgets/OWMarkdownizer.py

@@ -522,9 +522,9 @@ class MarkdownConversionThread(QThread):
 class FileProcessorApp(widget.OWWidget):
     name = "Markdownizer"
     description = "Convert PDFs, DOCX, PPTX to Markdown (texte seul & word only)"
-    icon = "icons/md.png"
+    icon = "icons/md_old.png"
     if "site-packages/Orange/widgets" in os.path.dirname(os.path.abspath(__file__)).replace("\\", "/"):
-       icon = "icons_dev/md.png"
+       icon = "icons_dev/md_old.png"
     want_control_area = False
     priority = 1001
     gui = os.path.join(os.path.dirname(os.path.abspath(__file__)), "designer/owmarkdownizer.ui")