scitex 2.5.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

scitex/scholar/url_finder/translators/individual/bocc.py

@@ -18,13 +18,13 @@ TRANSLATOR_INFO = {
     "in_repository": True,
     "translator_type": 4,
     "browser_support": "gcsbv",
-    "last_updated": "2014-04-04 10:08:43"
+    "last_updated": "2014-04-04 10:08:43",
 }
 
 
 def detect_web(doc: Any, url: str) -> Optional[str]:
     """Detect if the page is multiple items"""
-    agenda_td = doc.select_one('table.ag tbody tr td.agenda')
+    agenda_td = doc.select_one("table.ag tbody tr td.agenda")
     if agenda_td:
         return "multiple"
     return None
@@ -32,12 +32,12 @@ def detect_web(doc: Any, url: str) -> Optional[str]:
 
 def get_site_base(url: str) -> str:
     """Extract site base URL"""
-    match = re.match(r'^https?://[^/]*bocc[^/]*/', url)
+    match = re.match(r"^https?://[^/]*bocc[^/]*/", url)
     if match:
         site = match.group(0)
-        site = site.replace('/_esp', '')
-        site = site.replace('/_listas', '')
-        return site.rstrip('/')
+        site = site.replace("/_esp", "")
+        site = site.replace("/_listas", "")
+        return site.rstrip("/")
     return ""
 
 
@@ -45,40 +45,40 @@ def get_tags(doc: Any) -> List[str]:
     """Extract tags from tematica"""
     tags = []
 
-    title_elem = doc.select_one('title')
+    title_elem = doc.select_one("title")
     if not title_elem:
         return tags
 
     title_text = title_elem.get_text(strip=True)
 
-    if 'Temática' not in title_text:
+    if "Temática" not in title_text:
         return tags
 
     # Get tematicas list
-    tematica_links = doc.select('a.tematica')
+    tematica_links = doc.select("a.tematica")
     tematicas = {}
 
     for link in tematica_links:
-        href = link.get('href', '')
+        href = link.get("href", "")
         name = link.get_text(strip=True)
 
         # Extract number from href
-        match = re.search(r'=(\d+)$', href)
+        match = re.search(r"=(\d+)$", href)
         if match:
             num = match.group(1)
             tematicas[num] = name
 
     # Get current tematica number
-    match = re.search(r':\s(\d+)\s-', title_text)
+    match = re.search(r":\s(\d+)\s-", title_text)
     if match and match.group(1) in tematicas:
         tematica_name = tematicas[match.group(1)]
 
         # Split tematica name into tags
-        if ' e ' in tematica_name:
-            parts = tematica_name.split(' e ')
+        if " e " in tematica_name:
+            parts = tematica_name.split(" e ")
             for part in parts:
-                if ',' in part:
-                    sub_parts = part.split(',')
+                if "," in part:
+                    sub_parts = part.split(",")
                     tags.extend([p.strip() for p in sub_parts if p.strip()])
                 else:
                     if part.strip():
@@ -93,20 +93,20 @@ def get_tags(doc: Any) -> List[str]:
 def parse_articles(html_content: str, site_base: str) -> List[Dict[str, Any]]:
     """Parse articles from HTML content"""
     articles = []
-    lines = html_content.split('<br><br>')
+    lines = html_content.split("<br><br>")
 
     re_url = re.compile(r'href="([^"]+)')
     re_autor = re.compile(r'autor.php[^>]+"agenda">([^<]+)', re.IGNORECASE)
-    re_date = re.compile(r'(\d{4})$')
+    re_date = re.compile(r"(\d{4})$")
 
     for line in lines:
         # Get first br-separated part as title
-        parts = line.split('<br>')
+        parts = line.split("<br>")
         if not parts:
             continue
 
         # Clean title
-        title = re.sub(r'<[^>]+>', '', parts[0])
+        title = re.sub(r"<[^>]+>", "", parts[0])
         title = title.strip()
 
         # Get URL
@@ -115,7 +115,7 @@ def parse_articles(html_content: str, site_base: str) -> List[Dict[str, Any]]:
             continue
 
         url = url_match.group(1)
-        if 'autor' in url:
+        if "autor" in url:
             continue
 
         # Get authors
@@ -131,37 +131,39 @@ def parse_articles(html_content: str, site_base: str) -> List[Dict[str, Any]]:
             date = date_match.group(1)
 
         # Determine file type
-        file_url = site_base + url.replace('..', '')
-        if re.search(r'\.(html?|HTML?)$', file_url):
-            file_title = 'Anexo HTML'
-            file_mime = 'text/html'
-        elif re.search(r'\.(pdf|PDF)$', file_url):
-            file_title = 'Anexo PDF'
-            file_mime = 'application/pdf'
+        file_url = site_base + url.replace("..", "")
+        if re.search(r"\.(html?|HTML?)$", file_url):
+            file_title = "Anexo HTML"
+            file_mime = "text/html"
+        elif re.search(r"\.(pdf|PDF)$", file_url):
+            file_title = "Anexo PDF"
+            file_mime = "application/pdf"
         else:
-            file_title = 'Anexo'
-            file_mime = 'application/octet-stream'
-
-        articles.append({
-            'url': url,
-            'title': title,
-            'authors': authors,
-            'date': date,
-            'file_url': file_url,
-            'file_title': file_title,
-            'file_mime': file_mime
-        })
+            file_title = "Anexo"
+            file_mime = "application/octet-stream"
+
+        articles.append(
+            {
+                "url": url,
+                "title": title,
+                "authors": authors,
+                "date": date,
+                "file_url": file_url,
+                "file_title": file_title,
+                "file_mime": file_mime,
+            }
+        )
 
     return articles
 
 
 def get_search_results(doc: Any, check_only: bool = False) -> Optional[Dict[str, str]]:
     """Get search results from a multiple item page"""
-    agenda_td = doc.select_one('table.ag tbody tr td.agenda')
+    agenda_td = doc.select_one("table.ag tbody tr td.agenda")
     if not agenda_td:
         return None
 
-    site_base = get_site_base(doc.find('base').get('href') if doc.find('base') else '')
+    site_base = get_site_base(doc.find("base").get("href") if doc.find("base") else "")
 
     html_content = str(agenda_td)
     articles = parse_articles(html_content, site_base)
@@ -174,7 +176,7 @@ def get_search_results(doc: Any, check_only: bool = False) -> Optional[Dict[str,
 
     items = {}
     for article in articles:
-        items[article['url']] = article['title']
+        items[article["url"]] = article["title"]
 
     return items if items else None
 
@@ -182,7 +184,7 @@ def get_search_results(doc: Any, check_only: bool = False) -> Optional[Dict[str,
 def scrape(doc: Any, url: str, selected_items: Dict[str, str]) -> List[Dict[str, Any]]:
     """Scrape selected items"""
     site_base = get_site_base(url)
-    agenda_td = doc.select_one('table.ag tbody tr td.agenda')
+    agenda_td = doc.select_one("table.ag tbody tr td.agenda")
     if not agenda_td:
         return []
 
@@ -194,37 +196,41 @@ def scrape(doc: Any, url: str, selected_items: Dict[str, str]) -> List[Dict[str,
 
     results = []
     for article in articles:
-        if article['url'] not in selected_items:
+        if article["url"] not in selected_items:
             continue
 
         item = {
             "itemType": "journalArticle",
-            "title": article['title'],
+            "title": article["title"],
             "creators": [],
-            "date": article['date'],
+            "date": article["date"],
             "publicationTitle": "Biblioteca Online de Ciências da Comunicação",
             "ISSN": "1646-3137",
             "journalAbbreviation": "BOCC",
-            "url": article['file_url'],
-            "attachments": [{
-                "url": article['file_url'],
-                "title": article['file_title'],
-                "mimeType": article['file_mime']
-            }],
+            "url": article["file_url"],
+            "attachments": [
+                {
+                    "url": article["file_url"],
+                    "title": article["file_title"],
+                    "mimeType": article["file_mime"],
+                }
+            ],
             "tags": tags[:],
             "notes": [],
-            "seeAlso": []
+            "seeAlso": [],
         }
 
         # Add authors
-        for author_name in article['authors']:
+        for author_name in article["authors"]:
             parts = author_name.split()
             if len(parts) > 1:
-                item["creators"].append({
-                    "firstName": " ".join(parts[:-1]),
-                    "lastName": parts[-1],
-                    "creatorType": "author"
-                })
+                item["creators"].append(
+                    {
+                        "firstName": " ".join(parts[:-1]),
+                        "lastName": parts[-1],
+                        "creatorType": "author",
+                    }
+                )
 
         results.append(item)
 

scitex/scholar/url_finder/translators/individual/boe.py

@@ -19,31 +19,33 @@ TRANSLATOR_INFO = {
     "in_repository": True,
     "translator_type": 4,
     "browser_support": "gcsibv",
-    "last_updated": "2021-07-26 17:07:40"
+    "last_updated": "2021-07-26 17:07:40",
 }
 
 
 def detect_web(doc: Any, url: str) -> Optional[str]:
     """Detect if the page is a statute"""
-    if 'diario_boe' in url or 'www.boe.es/eli' in url or '/doc.php' in url:
+    if "diario_boe" in url or "www.boe.es/eli" in url or "/doc.php" in url:
         return "statute"
     return None
 
 
 def get_metadata_uri(doc: Any, url: str) -> Optional[str]:
     """Get the XML metadata URI"""
-    if '/xml' not in url:
+    if "/xml" not in url:
         # Find the XML embodiment
         index = 1
         while True:
             xpath = f'(//meta[@property="http://data.europa.eu/eli/ontology#is_embodied_by"])[{index}]'
-            meta_elem = doc.select_one(f'meta[property="http://data.europa.eu/eli/ontology#is_embodied_by"]')
+            meta_elem = doc.select_one(
+                f'meta[property="http://data.europa.eu/eli/ontology#is_embodied_by"]'
+            )
 
             if not meta_elem:
                 break
 
-            resource = meta_elem.get('resource', '')
-            if '/xml' in resource:
+            resource = meta_elem.get("resource", "")
+            if "/xml" in resource:
                 return resource
 
             index += 1
@@ -56,8 +58,8 @@ def get_metadata_uri(doc: Any, url: str) -> Optional[str]:
 def parse_xml_metadata(xml_text: str) -> Dict[str, str]:
     """Parse XML metadata"""
     # Clean XML
-    xml_text = re.sub(r'<!DOCTYPE[^>]*>', '', xml_text)
-    xml_text = re.sub(r'<\?xml[^>]*\?>', '', xml_text)
+    xml_text = re.sub(r"<!DOCTYPE[^>]*>", "", xml_text)
+    xml_text = re.sub(r"<\?xml[^>]*\?>", "", xml_text)
     xml_text = xml_text.strip()
 
     metadata = {}
@@ -66,51 +68,51 @@ def parse_xml_metadata(xml_text: str) -> Dict[str, str]:
         root = ET.fromstring(xml_text)
 
         # Extract fields
-        departamento = root.find('.//departamento')
+        departamento = root.find(".//departamento")
         if departamento is not None and departamento.text:
-            metadata['author'] = departamento.text
+            metadata["author"] = departamento.text
 
-        fecha_publicacion = root.find('.//fecha_publicacion')
+        fecha_publicacion = root.find(".//fecha_publicacion")
         if fecha_publicacion is not None and fecha_publicacion.text:
             fecha = fecha_publicacion.text
             # Convert from YYYYMMDD to YYYY-MM-DD
             if len(fecha) == 8:
-                metadata['dateEnacted'] = f"{fecha[:4]}-{fecha[4:6]}-{fecha[6:8]}"
+                metadata["dateEnacted"] = f"{fecha[:4]}-{fecha[4:6]}-{fecha[6:8]}"
 
-        titulo = root.find('.//titulo')
+        titulo = root.find(".//titulo")
         if titulo is not None and titulo.text:
             # Remove trailing dot
-            metadata['nameOfAct'] = titulo.text.rstrip('.')
+            metadata["nameOfAct"] = titulo.text.rstrip(".")
 
-        seccion = root.find('.//seccion')
+        seccion = root.find(".//seccion")
         if seccion is not None and seccion.text:
-            metadata['section'] = seccion.text
+            metadata["section"] = seccion.text
 
-        pagina_inicial = root.find('.//pagina_inicial')
-        pagina_final = root.find('.//pagina_final')
+        pagina_inicial = root.find(".//pagina_inicial")
+        pagina_final = root.find(".//pagina_final")
         if pagina_inicial is not None and pagina_final is not None:
             if pagina_inicial.text and pagina_final.text:
-                metadata['pages'] = f"{pagina_inicial.text}-{pagina_final.text}"
+                metadata["pages"] = f"{pagina_inicial.text}-{pagina_final.text}"
 
-        diario = root.find('.//diario')
-        diario_numero = root.find('.//diario_numero')
+        diario = root.find(".//diario")
+        diario_numero = root.find(".//diario_numero")
         if diario is not None and diario_numero is not None:
             if diario.text and diario_numero.text:
-                metadata['session'] = f"{diario.text} núm. {diario_numero.text}"
+                metadata["session"] = f"{diario.text} núm. {diario_numero.text}"
 
-        identificador = root.find('.//identificador')
+        identificador = root.find(".//identificador")
         if identificador is not None and identificador.text:
-            metadata['codeNumber'] = identificador.text
+            metadata["codeNumber"] = identificador.text
 
-        rango = root.find('.//rango')
-        numero_oficial = root.find('.//numero_oficial')
+        rango = root.find(".//rango")
+        numero_oficial = root.find(".//numero_oficial")
         if rango is not None and numero_oficial is not None:
             if rango.text and numero_oficial.text:
-                metadata['publicLawNumber'] = f"{rango.text} {numero_oficial.text}"
+                metadata["publicLawNumber"] = f"{rango.text} {numero_oficial.text}"
 
-        url_eli = root.find('.//url_eli')
+        url_eli = root.find(".//url_eli")
         if url_eli is not None and url_eli.text:
-            metadata['url'] = url_eli.text
+            metadata["url"] = url_eli.text
 
     except ET.ParseError:
         pass
@@ -137,7 +139,7 @@ def scrape(doc: Any, url: str, xml_text: Optional[str] = None) -> Dict[str, Any]
         "attachments": [{"title": "Snapshot", "mimeType": "text/html", "url": url}],
         "tags": [],
         "notes": [],
-        "seeAlso": []
+        "seeAlso": [],
     }
 
     if not xml_text:
@@ -149,36 +151,34 @@ def scrape(doc: Any, url: str, xml_text: Optional[str] = None) -> Dict[str, Any]
     metadata = parse_xml_metadata(xml_text)
 
     # Populate item
-    if 'nameOfAct' in metadata:
-        item["nameOfAct"] = metadata['nameOfAct']
+    if "nameOfAct" in metadata:
+        item["nameOfAct"] = metadata["nameOfAct"]
 
-    if 'author' in metadata:
-        item["creators"].append({
-            "lastName": metadata['author'],
-            "creatorType": "author",
-            "fieldMode": 1
-        })
+    if "author" in metadata:
+        item["creators"].append(
+            {"lastName": metadata["author"], "creatorType": "author", "fieldMode": 1}
+        )
 
-    if 'dateEnacted' in metadata:
-        item["dateEnacted"] = metadata['dateEnacted']
+    if "dateEnacted" in metadata:
+        item["dateEnacted"] = metadata["dateEnacted"]
 
-    if 'section' in metadata:
-        item["section"] = metadata['section']
+    if "section" in metadata:
+        item["section"] = metadata["section"]
 
-    if 'pages' in metadata:
-        item["pages"] = metadata['pages']
+    if "pages" in metadata:
+        item["pages"] = metadata["pages"]
 
-    if 'session' in metadata:
-        item["session"] = metadata['session']
+    if "session" in metadata:
+        item["session"] = metadata["session"]
 
-    if 'codeNumber' in metadata:
-        item["codeNumber"] = metadata['codeNumber']
+    if "codeNumber" in metadata:
+        item["codeNumber"] = metadata["codeNumber"]
 
-    if 'publicLawNumber' in metadata:
-        item["publicLawNumber"] = metadata['publicLawNumber']
+    if "publicLawNumber" in metadata:
+        item["publicLawNumber"] = metadata["publicLawNumber"]
 
-    if 'url' in metadata:
-        item["url"] = metadata['url']
+    if "url" in metadata:
+        item["url"] = metadata["url"]
 
     return item
 

scitex/scholar/url_finder/translators/individual/brill.py

@@ -12,7 +12,9 @@ class BrillTranslator(BaseTranslator):
     """Brill."""
 
     LABEL = "Brill"
-    URL_TARGET_PATTERN = r"^https?://(www\.|referenceworks\.|bibliographies\.)?brill(online)?\.com/"
+    URL_TARGET_PATTERN = (
+        r"^https?://(www\.|referenceworks\.|bibliographies\.)?brill(online)?\.com/"
+    )
 
     @classmethod
     def matches_url(cls, url: str) -> bool:

scitex/scholar/url_finder/translators/individual/business_standard.py

@@ -34,7 +34,7 @@ class BusinessStandardTranslator:
         "inRepository": True,
         "translatorType": 4,
         "browserSupport": "gcsibv",
-        "lastUpdated": "2016-11-01 18:25:24"
+        "lastUpdated": "2016-11-01 18:25:24",
     }
 
     def detect_web(self, doc: BeautifulSoup, url: str) -> str:
@@ -48,12 +48,12 @@ class BusinessStandardTranslator:
         Returns:
             'multiple' for search results, 'newspaperArticle' for articles, empty string otherwise
         """
-        if '/search?' in url:
+        if "/search?" in url:
             if doc.select('div[class*="main-cont-left"] h2 a'):
-                return 'multiple'
-        if doc.select_one('div.content-main h1'):
-            return 'newspaperArticle'
-        return ''
+                return "multiple"
+        if doc.select_one("div.content-main h1"):
+            return "newspaperArticle"
+        return ""
 
     def do_web(self, doc: BeautifulSoup, url: str) -> Dict[str, Any]:
         """
@@ -66,7 +66,7 @@ class BusinessStandardTranslator:
         Returns:
             Dictionary containing article metadata or list of items for multiple
         """
-        if self.detect_web(doc, url) == 'multiple':
+        if self.detect_web(doc, url) == "multiple":
             return self.get_search_results(doc)
         return self.scrape(doc, url)
 
@@ -83,7 +83,7 @@ class BusinessStandardTranslator:
         items = {}
         rows = doc.select('div[class*="main-cont-left"] h2 a')
         for row in rows:
-            href = row.get('href')
+            href = row.get("href")
             title = row.get_text(strip=True)
             if href and title:
                 items[href] = title
@@ -101,55 +101,57 @@ class BusinessStandardTranslator:
             Dictionary containing article metadata
         """
         item = {
-            'itemType': 'newspaperArticle',
-            'publicationTitle': 'Business Standard India',
-            'url': url,
-            'creators': [],
-            'tags': [],
-            'attachments': []
+            "itemType": "newspaperArticle",
+            "publicationTitle": "Business Standard India",
+            "url": url,
+            "creators": [],
+            "tags": [],
+            "attachments": [],
         }
 
         # Extract title
-        title_elem = doc.select_one('div.content-main h1')
+        title_elem = doc.select_one("div.content-main h1")
         if title_elem:
-            item['title'] = title_elem.get_text(strip=True)
+            item["title"] = title_elem.get_text(strip=True)
 
         # Extract creators
         creator_elem = doc.select_one('p[class*="fL"] span:first-child')
         if creator_elem:
             creator_text = creator_elem.get_text(strip=True)
             # Remove pipe and everything after it
-            creator_text = re.sub(r'\|\n?.+', '', creator_text)
+            creator_text = re.sub(r"\|\n?.+", "", creator_text)
             if creator_text:
-                item['creators'].append(self._clean_author(creator_text))
+                item["creators"].append(self._clean_author(creator_text))
 
         # Extract date
         date_meta = doc.select_one('meta[itemprop="datePublished"]')
-        if date_meta and date_meta.get('content'):
-            item['date'] = date_meta['content']
+        if date_meta and date_meta.get("content"):
+            item["date"] = date_meta["content"]
 
         # Extract abstract
         abstract_meta = doc.select_one('meta[name="description"]')
-        if abstract_meta and abstract_meta.get('content'):
-            item['abstractNote'] = abstract_meta['content']
+        if abstract_meta and abstract_meta.get("content"):
+            item["abstractNote"] = abstract_meta["content"]
 
         # Extract tags
         tags_div = doc.select('div.related-keyword div[class*="readmore_tagBG"]')
         for tag_elem in tags_div:
             tag_text = tag_elem.get_text(strip=True)
             if tag_text:
-                for tag in tag_text.split(','):
+                for tag in tag_text.split(","):
                     tag = tag.strip()
                     if tag:
-                        item['tags'].append({'tag': tag})
+                        item["tags"].append({"tag": tag})
 
         # Add snapshot attachment
-        print_url = url.replace('node', 'print')
-        item['attachments'].append({
-            'title': 'Business Standard India Snapshot',
-            'mimeType': 'text/html',
-            'url': print_url
-        })
+        print_url = url.replace("node", "print")
+        item["attachments"].append(
+            {
+                "title": "Business Standard India Snapshot",
+                "mimeType": "text/html",
+                "url": print_url,
+            }
+        )
 
         return item
 
@@ -168,13 +170,9 @@ class BusinessStandardTranslator:
 
         if len(parts) >= 2:
             return {
-                'firstName': ' '.join(parts[:-1]),
-                'lastName': parts[-1],
-                'creatorType': 'author'
+                "firstName": " ".join(parts[:-1]),
+                "lastName": parts[-1],
+                "creatorType": "author",
             }
         else:
-            return {
-                'lastName': name,
-                'creatorType': 'author',
-                'fieldMode': True
-            }
+            return {"lastName": name, "creatorType": "author", "fieldMode": True}