io4it 3.0.5.99__tar.gz → 3.0.5.991__tar.gz

io4it-3.0.5.991/PKG-INFO

@@ -0,0 +1,37 @@
+Metadata-Version: 2.1
+Name: io4it
+Version: 3.0.5.991
+Summary: UNKNOWN
+Home-page: 
+Author: 
+Author-email: 
+License: UNKNOWN
+Keywords: orange3 add-on
+Platform: UNKNOWN
+Requires-Dist: torchvision
+Requires-Dist: torchaudio
+Requires-Dist: torch
+Requires-Dist: pylatexenc
+Requires-Dist: docopt
+Requires-Dist: boto3
+Requires-Dist: opencv-python-headless
+Requires-Dist: docling
+Requires-Dist: docling-core
+Requires-Dist: speechbrain
+Requires-Dist: whisper
+Requires-Dist: whisper-openai
+Requires-Dist: pyannote.audio
+Requires-Dist: pyannote-core
+Requires-Dist: pypandoc
+Requires-Dist: pypandoc-binary
+Requires-Dist: scikit-learn
+Requires-Dist: openai
+Requires-Dist: pip-system-certs
+Requires-Dist: docx2pdf
+Requires-Dist: msal
+Requires-Dist: exchangelib
+Requires-Dist: html2text
+Requires-Dist: ddgs
+Requires-Dist: CATEGORIT
+
+UNKNOWN

io4it-3.0.5.991/io4it.egg-info/PKG-INFO

@@ -0,0 +1,37 @@
+Metadata-Version: 2.1
+Name: io4it
+Version: 3.0.5.991
+Summary: UNKNOWN
+Home-page: 
+Author: 
+Author-email: 
+License: UNKNOWN
+Keywords: orange3 add-on
+Platform: UNKNOWN
+Requires-Dist: torchvision
+Requires-Dist: torchaudio
+Requires-Dist: torch
+Requires-Dist: pylatexenc
+Requires-Dist: docopt
+Requires-Dist: boto3
+Requires-Dist: opencv-python-headless
+Requires-Dist: docling
+Requires-Dist: docling-core
+Requires-Dist: speechbrain
+Requires-Dist: whisper
+Requires-Dist: whisper-openai
+Requires-Dist: pyannote.audio
+Requires-Dist: pyannote-core
+Requires-Dist: pypandoc
+Requires-Dist: pypandoc-binary
+Requires-Dist: scikit-learn
+Requires-Dist: openai
+Requires-Dist: pip-system-certs
+Requires-Dist: docx2pdf
+Requires-Dist: msal
+Requires-Dist: exchangelib
+Requires-Dist: html2text
+Requires-Dist: ddgs
+Requires-Dist: CATEGORIT
+
+UNKNOWN

{io4it-3.0.5.99 → io4it-3.0.5.991}/orangecontrib/IO4IT/widgets/OWParserHTML.py

@@ -4,10 +4,15 @@ import Orange
 from Orange.widgets.widget import OWWidget, Input, Output
 from AnyQt.QtWidgets import QApplication
 import asyncio
-import aiohttp
 import html2text
 from bs4 import BeautifulSoup
 import urllib.request
+import urllib3
+import requests
+from requests_ntlm import HttpNtlmAuth
+
+# Désactive les avertissements SSL pour certificats d'entreprise auto-signés
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 if "site-packages/Orange/widgets" in os.path.dirname(os.path.abspath(__file__)).replace("\\", "/"):
     from Orange.widgets.orangecontrib.AAIT.utils import thread_management
@@ -25,7 +30,6 @@ class ParseHMTL(OWWidget):
     if "site-packages/Orange/widgets" in os.path.dirname(os.path.abspath(__file__)).replace("\\", "/"):
         icon = "icons_dev/html.png"
     priority = 3000
-    gui = ""
     want_control_area = False
     category = "AAIT - TOOLBOX"
     gui = os.path.join(os.path.dirname(os.path.abspath(__file__)), "designer/owparserhtml.ui")
@@ -33,17 +37,6 @@ class ParseHMTL(OWWidget):
     class Inputs:
         data = Input("Data", Orange.data.Table)
 
-    @Inputs.data
-    def set_data(self, in_data):
-        if in_data is None:
-            return
-        if "url" not in in_data.domain:
-            self.error("input table need a url column")
-            return
-        self.data = in_data
-        self.url_data = in_data.get_column("url")
-        self.run()
-
     class Outputs:
         data = Output("Data", Orange.data.Table)
 
@@ -55,93 +48,146 @@ class ParseHMTL(OWWidget):
         uic.loadUi(self.gui, self)
 
         self.data = None
+        self.url_data = []
         self.thread = None
         self.markdown = True
+        self.proxy_url = self._get_enterprise_proxy_url()
+        self.ntlm_auth = HttpNtlmAuth('', '')
         self.run()
 
     def update_parameters(self):
         return
 
 
+
+    @Inputs.data
+    def set_data(self, in_data):
+        if in_data is None:
+            return
+        if "url" not in in_data.domain:
+            self.error("input table need a url column")
+            return
+        self.data = in_data
+        self.url_data = list(in_data.get_column("url"))
+        self.run()
+
+    def _get_enterprise_proxy_url(self):
+        proxies_dict = urllib.request.getproxies()
+        raw_proxy = proxies_dict.get("http") or proxies_dict.get("https")
+        if raw_proxy and not raw_proxy.startswith("http"):
+            return f"http://{raw_proxy}"
+        return raw_proxy
+
+    def _sync_fetch(self, url: str) -> str:
+        """
+        Appel réseau réel (synchrone) compatible NTLM, exécuté dans un thread via run_in_executor.
+        """
+        session = requests.Session()
+        session.verify = False  # ignore certifs d'entreprise
+
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+                          '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Accept-Language': 'fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7',
+            'Connection': 'keep-alive',
+            'Upgrade-Insecure-Requests': '1'
+        }
+
+        proxies = None
+        if self.proxy_url:
+            proxies = {"http": self.proxy_url, "https": self.proxy_url}
+
+        resp = session.get(
+            url,
+            proxies=proxies,
+            auth=self.ntlm_auth,
+            headers=headers,
+            timeout=30
+        )
+        resp.raise_for_status()
+        return resp.text
+
     def parse_html(self):
         """Execute le parsing"""
         try:
             loop = asyncio.new_event_loop()
             asyncio.set_event_loop(loop)
-            results = loop.run_until_complete(self.parse_all_urls())
+            results = loop.run_until_complete(self.parse_all_urls(progress_callback=self._progress_cb))
             loop.close()
             return results
         except Exception as e:
             self.error(str(e))
             return
 
+    def _progress_cb(self, value: int, text: str = None):
+        """
+        Callback interne utilisé par parse_all_urls.
+        On renvoie un tuple (value, text) au thread Qt, comme ton handle_progress s'y attend.
+        """
+        if self.thread is not None:
+            self.thread.progress.emit((value, text))
 
     async def parse_all_urls(self, progress_callback=None):
-        """Parse toutes les URLs de manière asynchrone"""
+        """
+        Parse toutes les URLs en concurrence (comme dans le 2e code) en gardant une progression fluide.
+        """
         results = []
         total = len(self.url_data)
-        timeout = aiohttp.ClientTimeout(total=30)
-        connector = aiohttp.TCPConnector(limit=5, limit_per_host=2)
-        async with aiohttp.ClientSession(timeout=timeout, connector=connector,cookie_jar=aiohttp.CookieJar()) as session:
-            for idx, url_data in enumerate(self.url_data):
-                if progress_callback is not None:
-                    progress_value = int((idx / total) * 100)
-                    progress_callback(progress_value)
-                try:
-                    parsed = await self.parse_single_url(session, url_data)
-                    results.append(parsed)
-
-                except Exception as e:
-                    results.append({
-                        "url": url_data,
-                        'content': '',
-                        'meta_description': '',
-                        'word_count': 0,
-                        'status': f'error: {str(e)}'
-                    })
+        if total == 0:
+            return results
+
+        tasks = [self.parse_single_url(url) for url in self.url_data]
+
+        for i, task in enumerate(asyncio.as_completed(tasks)):
+            result = await task
+            results.append(result)
+
+            if progress_callback:
+                progress_value = int(((i + 1) / total) * 100)
+                progress_callback(progress_value, None)
 
         return results
 
-    async def parse_single_url(self, session, url):
-        """Parse une seule URL"""
-        # Fetch HTML
-        headers = {
-                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-                'Accept-Language': 'fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7',
-                'Accept-Encoding': 'gzip, deflate, br',
-                'Connection': 'keep-alive',
-                'Upgrade-Insecure-Requests': '1'
-        }
-        proxies = urllib.request.getproxies()
-        if "http" in proxies:
-            proxies = proxies["http"]
-        else:
-            proxies = None
-        async with session.get(url, headers=headers, proxy=proxies) as response:
-            if response.status != 200:
-                raise Exception(f"HTTP {response.status}")
-            html = await response.text()
-        soup = BeautifulSoup(html, 'html.parser')
-        meta_desc = ''
-        meta_tag = soup.find('meta', attrs={'name': 'description'})
-        if not meta_tag:
-            meta_tag = soup.find('meta', property='og:description')
-        if meta_tag:
-            meta_desc = meta_tag.get('content', '')
-        content = ''
+    async def parse_single_url(self, url: str):
+        """
+        Appel sync (requests+ntlm) dans executor, puis parsing BeautifulSoup + extraction contenu.
+        """
         try:
-            content = self._extract_main_content(soup)
+            loop = asyncio.get_event_loop()
+            html = await loop.run_in_executor(None, self._sync_fetch, url)
+
+            soup = BeautifulSoup(html, 'html.parser')
+            meta_desc = ''
+            meta_tag = soup.find('meta', attrs={'name': 'description'})
+            if not meta_tag:
+                meta_tag = soup.find('meta', property='og:description')
+            if meta_tag:
+                meta_desc = meta_tag.get('content', '')
+
+            try:
+                content = self._extract_main_content(soup)
+            except Exception:
+                content = ''
+
+            word_count = len(content.split()) if content else 0
+
+            return {
+                "url": url,
+                "content": content,
+                "meta_description": meta_desc,
+                "word_count": word_count,
+                "status": "success"
+            }
+
         except Exception as e:
-            print(e)
-        word_count = len(content.split())
-        return {
-            "url": url,
-            'content': content,
-            'meta_description': meta_desc,
-            'word_count': word_count,
-            'status': 'success'
-        }
+            return {
+                "url": url,
+                "content": "",
+                "meta_description": "",
+                "word_count": 0,
+                "status": f"error: {str(e)}"
+            }
 
     def _extract_main_content(self, soup):
         """Extrait le contenu principal et le convertit en Markdown"""
@@ -172,7 +218,7 @@ class ParseHMTL(OWWidget):
                     if paragraphs:
                         text = ' '.join([p.get_text(strip=True) for p in paragraphs])
                         if len(text) > 100:
-                                return text
+                            return text
         paragraphs = soup.find_all('p')
         if paragraphs:
             return ' '.join([p.get_text(strip=True) for p in paragraphs])
@@ -223,4 +269,3 @@ if __name__ == "__main__":
         sys.exit(app.exec())
     else:
         sys.exit(app.exec_())
-

{io4it-3.0.5.99 → io4it-3.0.5.991}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 # Configuration
 NAME = "io4it"
-VERSION = "3.0.5.99"
+VERSION = "3.0.5.991"
 
 INSTALL_REQUIRES = [
     "torchvision",

io4it-3.0.5.99/PKG-INFO

@@ -1,7 +0,0 @@
-Metadata-Version: 2.1
-Name: io4it
-Version: 3.0.5.99
-Home-page: 
-Author: 
-Author-email: 
-Keywords: orange3 add-on

io4it-3.0.5.99/io4it.egg-info/PKG-INFO

@@ -1,7 +0,0 @@
-Metadata-Version: 2.1
-Name: io4it
-Version: 3.0.5.99
-Home-page: 
-Author: 
-Author-email: 
-Keywords: orange3 add-on