scitex 2.5.0__py3-none-any.whl → 2.7.3__py3-none-any.whl

scitex/scholar/url_finder/translators/individual/daum_news.py

@@ -34,7 +34,7 @@ class DaumNewsTranslator:
         "inRepository": True,
         "translatorType": 4,
         "browserSupport": "gcsibv",
-        "lastUpdated": "2021-06-07 16:41:08"
+        "lastUpdated": "2021-06-07 16:41:08",
     }
 
     def detect_web(self, doc: BeautifulSoup, url: str) -> str:
@@ -48,10 +48,10 @@ class DaumNewsTranslator:
         Returns:
             'newspaperArticle' if detected, empty string otherwise
         """
-        og_type = doc.find('meta', {'property': 'og:type'})
-        if og_type and og_type.get('content') == 'article':
-            return 'newspaperArticle'
-        return ''
+        og_type = doc.find("meta", {"property": "og:type"})
+        if og_type and og_type.get("content") == "article":
+            return "newspaperArticle"
+        return ""
 
     def do_web(self, doc: BeautifulSoup, url: str) -> Dict[str, Any]:
         """
@@ -78,56 +78,56 @@ class DaumNewsTranslator:
             Dictionary containing article metadata
         """
         item = {
-            'itemType': 'newspaperArticle',
-            'url': url,
-            'creators': [],
-            'tags': [],
-            'attachments': []
+            "itemType": "newspaperArticle",
+            "url": url,
+            "creators": [],
+            "tags": [],
+            "attachments": [],
         }
 
         # Extract title
-        title_tag = doc.find('meta', {'property': 'og:title'})
-        if title_tag and title_tag.get('content'):
-            item['title'] = title_tag['content']
+        title_tag = doc.find("meta", {"property": "og:title"})
+        if title_tag and title_tag.get("content"):
+            item["title"] = title_tag["content"]
 
         # Extract author
-        author_elem = doc.select_one('.info_view .txt_info')
+        author_elem = doc.select_one(".info_view .txt_info")
         if author_elem and author_elem.get_text(strip=True):
-            item['creators'].append({
-                'lastName': author_elem.get_text(strip=True),
-                'creatorType': 'author',
-                'fieldMode': True
-            })
+            item["creators"].append(
+                {
+                    "lastName": author_elem.get_text(strip=True),
+                    "creatorType": "author",
+                    "fieldMode": True,
+                }
+            )
 
         # Extract publication
-        pub_elem = doc.select_one('.link_cp .thumb_g')
-        if pub_elem and pub_elem.get('alt'):
-            item['publicationTitle'] = pub_elem['alt']
+        pub_elem = doc.select_one(".link_cp .thumb_g")
+        if pub_elem and pub_elem.get("alt"):
+            item["publicationTitle"] = pub_elem["alt"]
 
         # Extract abstract and clean it
-        abstract_tag = doc.find('meta', {'property': 'og:description'})
-        if abstract_tag and abstract_tag.get('content'):
-            abstract = abstract_tag['content']
+        abstract_tag = doc.find("meta", {"property": "og:description"})
+        if abstract_tag and abstract_tag.get("content"):
+            abstract = abstract_tag["content"]
             # Remove Korean journalist prefix patterns
-            abstract = re.sub(r'^\[[^\]]+\]', '', abstract)
-            abstract = re.sub(r'^.+ 기자 =', '', abstract)
-            item['abstractNote'] = abstract.strip()
+            abstract = re.sub(r"^\[[^\]]+\]", "", abstract)
+            abstract = re.sub(r"^.+ 기자 =", "", abstract)
+            item["abstractNote"] = abstract.strip()
 
         # Extract language
-        lang_tag = doc.find('meta', {'property': 'og:locale'})
-        if lang_tag and lang_tag.get('content'):
-            item['language'] = lang_tag['content']
+        lang_tag = doc.find("meta", {"property": "og:locale"})
+        if lang_tag and lang_tag.get("content"):
+            item["language"] = lang_tag["content"]
         else:
-            item['language'] = 'ko'  # Default to Korean
+            item["language"] = "ko"  # Default to Korean
 
         # Set library catalog
-        item['libraryCatalog'] = 'news.v.daum.net'
+        item["libraryCatalog"] = "news.v.daum.net"
 
         # Add snapshot attachment
-        item['attachments'].append({
-            'title': 'Snapshot',
-            'mimeType': 'text/html',
-            'url': url
-        })
+        item["attachments"].append(
+            {"title": "Snapshot", "mimeType": "text/html", "url": url}
+        )
 
         return item

scitex/scholar/url_finder/translators/individual/dblp.py

@@ -31,7 +31,7 @@ class DBLPTranslator:
         "priority": 100,
         "inRepository": True,
         "translatorType": 4,  # Web
-        "lastUpdated": "2023-04-20 13:37:43"
+        "lastUpdated": "2023-04-20 13:37:43",
     }
 
     def detect_web(self, doc, url: str) -> Optional[str]:
@@ -130,15 +130,11 @@ class DBLPTranslator:
         Returns:
             Item dictionary
         """
-        item = {
-            "creators": [],
-            "tags": [],
-            "attachments": []
-        }
+        item = {"creators": [], "tags": [], "attachments": []}
 
         # Extract entry type and fields
         # This is a simplified implementation
-        entry_match = re.search(r'@(\w+)\{([^,]+),', bibtex)
+        entry_match = re.search(r"@(\w+)\{([^,]+),", bibtex)
         if entry_match:
             entry_type = entry_match.group(1).lower()
 
@@ -148,7 +144,7 @@ class DBLPTranslator:
                 "inproceedings": "conferencePaper",
                 "incollection": "bookSection",
                 "book": "book",
-                "phdthesis": "thesis"
+                "phdthesis": "thesis",
             }
             item["itemType"] = type_map.get(entry_type, "journalArticle")
 

scitex/scholar/url_finder/translators/individual/dblp_computer_science_bibliography.py

@@ -16,11 +16,14 @@ import re
 from typing import List, Optional
 from playwright.async_api import Page
 
+
 class DBLPTranslator:
     """DBLP Computer Science Bibliography translator."""
 
     LABEL = "DBLP Computer Science Bibliography"
-    URL_TARGET_PATTERN = r"^https?://(www\.)?(dblp\d?(\.org|\.uni-trier\.de/|\.dagstuhl\.de/))"
+    URL_TARGET_PATTERN = (
+        r"^https?://(www\.)?(dblp\d?(\.org|\.uni-trier\.de/|\.dagstuhl\.de/))"
+    )
 
     @classmethod
     def matches_url(cls, url: str) -> bool:
@@ -53,11 +56,13 @@ class DBLPTranslator:
         if bibtex_data:
             # Look for ee (electronic edition) or url fields
             # ee = {http://...} or url = {http://...}
-            url_match = re.search(r'(?:ee|url)\s*=\s*\{([^}]+)\}', bibtex_data, re.IGNORECASE)
+            url_match = re.search(
+                r"(?:ee|url)\s*=\s*\{([^}]+)\}", bibtex_data, re.IGNORECASE
+            )
             if url_match:
                 url = url_match.group(1).strip()
                 # Check if it's a PDF
-                if '.pdf' in url.lower():
+                if ".pdf" in url.lower():
                     pdf_urls.append(url)
 
         return pdf_urls

scitex/scholar/url_finder/translators/individual/dbpia.py

@@ -30,7 +30,7 @@ class DBpiaTranslator:
         "priority": 100,
         "inRepository": True,
         "translatorType": 4,  # Web
-        "lastUpdated": "2021-09-14 00:23:40"
+        "lastUpdated": "2021-09-14 00:23:40",
     }
 
     def detect_web(self, doc, url: str) -> Optional[str]:
@@ -46,7 +46,9 @@ class DBpiaTranslator:
         """
         if "/journal/articleDetail" in url:
             return "journalArticle"
-        elif ("/search/" in url or "/journal/articleList/" in url) and self._has_search_results(doc):
+        elif (
+            "/search/" in url or "/journal/articleList/" in url
+        ) and self._has_search_results(doc):
             return "multiple"
         return None
 
@@ -86,7 +88,7 @@ class DBpiaTranslator:
             "tags": [],
             "attachments": [],
             "libraryCatalog": "www.dbpia.co.kr",
-            "url": url
+            "url": url,
         }
 
         # Uses embedded metadata translator in JavaScript version

scitex/scholar/url_finder/translators/individual/defense_technical_information_center.py

@@ -33,24 +33,26 @@ class DefenseTechnicalInformationCenterTranslator:
         "inRepository": True,
         "translatorType": 4,
         "browserSupport": "gcsibv",
-        "lastUpdated": "2013-01-09 15:36:32"
+        "lastUpdated": "2013-01-09 15:36:32",
     }
 
     def detect_web(self, doc: BeautifulSoup, url: str) -> str:
         """Detect if page is DTIC report or index."""
         if "DTIC OAI Index for" in doc.title.string if doc.title else "":
-            return 'multiple'
-        elif 'verb=getRecord' in url:
-            return 'report'
-        return ''
-
-    def get_search_results(self, doc: BeautifulSoup, check_only: bool = False) -> Optional[Dict[str, str]]:
+            return "multiple"
+        elif "verb=getRecord" in url:
+            return "report"
+        return ""
+
+    def get_search_results(
+        self, doc: BeautifulSoup, check_only: bool = False
+    ) -> Optional[Dict[str, str]]:
         """Extract search results."""
         items = {}
         links = doc.select('a[href*="verb=getRecord"]')
 
         for link in links:
-            href = link.get('href', '')
+            href = link.get("href", "")
             # Get preceding text as title
             title_text = link.find_previous(text=True)
             if title_text:
@@ -62,10 +64,10 @@ class DefenseTechnicalInformationCenterTranslator:
                 continue
 
             if check_only:
-                return {'found': 'true'}
+                return {"found": "true"}
 
             # Replace HTML metadata with Dublin Core
-            href = href.replace('&metadataPrefix=html', '&metadataPrefix=oai_dc')
+            href = href.replace("&metadataPrefix=html", "&metadataPrefix=oai_dc")
             items[href] = title
 
         return items if items else None
@@ -74,7 +76,7 @@ class DefenseTechnicalInformationCenterTranslator:
         """Main extraction method."""
         page_type = self.detect_web(doc, url)
 
-        if page_type == 'multiple':
+        if page_type == "multiple":
             return self.get_search_results(doc, check_only=False)
         else:
             return self.scrape(doc, url)
@@ -82,29 +84,29 @@ class DefenseTechnicalInformationCenterTranslator:
     def scrape(self, doc: BeautifulSoup, url: str) -> Dict[str, Any]:
         """Scrape report metadata."""
         item = {
-            'itemType': 'report',
-            'libraryCatalog': 'Defense Technical Information Center',
-            'creators': [],
-            'tags': [],
-            'attachments': []
+            "itemType": "report",
+            "libraryCatalog": "Defense Technical Information Center",
+            "creators": [],
+            "tags": [],
+            "attachments": [],
         }
 
         # Find PDF URL
-        pdf_link = doc.find('a', href=lambda x: x and 'doc=GetTRDoc.pdf' in x)
+        pdf_link = doc.find("a", href=lambda x: x and "doc=GetTRDoc.pdf" in x)
         if pdf_link:
-            pdf_url = pdf_link.get('href')
-            item['attachments'].append({
-                'url': pdf_url,
-                'title': 'DTIC Full Text PDF',
-                'mimeType': 'application/pdf'
-            })
+            pdf_url = pdf_link.get("href")
+            item["attachments"].append(
+                {
+                    "url": pdf_url,
+                    "title": "DTIC Full Text PDF",
+                    "mimeType": "application/pdf",
+                }
+            )
 
         # Add snapshot
-        item['attachments'].append({
-            'title': 'DTIC Snapshot',
-            'mimeType': 'text/html',
-            'url': url
-        })
+        item["attachments"].append(
+            {"title": "DTIC Snapshot", "mimeType": "text/html", "url": url}
+        )
 
         # Note: Full implementation would fetch OAI-DC XML and parse it
         # For now, returning basic structure

scitex/scholar/url_finder/translators/individual/delpher.py

@@ -34,38 +34,40 @@ class DelpherTranslator:
         "inRepository": True,
         "translatorType": 4,
         "browserSupport": "gcsibv",
-        "lastUpdated": "2022-01-20 14:35:30"
+        "lastUpdated": "2022-01-20 14:35:30",
     }
 
     def detect_web(self, doc: BeautifulSoup, url: str) -> str:
         """Detect item type based on URL."""
-        if '/view' in url:
-            if '/boeken/' in url:
-                return 'book'
-            elif '/tijdschriften/' in url:
-                return 'journalArticle'
-            elif '/kranten/' in url:
-                return 'newspaperArticle'
-            elif '/radiobulletins/' in url:
-                return 'radioBroadcast'
+        if "/view" in url:
+            if "/boeken/" in url:
+                return "book"
+            elif "/tijdschriften/" in url:
+                return "journalArticle"
+            elif "/kranten/" in url:
+                return "newspaperArticle"
+            elif "/radiobulletins/" in url:
+                return "radioBroadcast"
         elif self.get_search_results(doc, check_only=True):
-            return 'multiple'
-        return ''
+            return "multiple"
+        return ""
 
-    def get_search_results(self, doc: BeautifulSoup, check_only: bool = False) -> Optional[Dict[str, str]]:
+    def get_search_results(
+        self, doc: BeautifulSoup, check_only: bool = False
+    ) -> Optional[Dict[str, str]]:
         """Extract search results."""
         items = {}
         rows = doc.select('article a.search-result__link[href^="/"]')
 
         for row in rows:
-            href = row.get('href')
+            href = row.get("href")
             title = row.get_text(strip=True)
 
             if not href or not title:
                 continue
 
             if check_only:
-                return {'found': 'true'}
+                return {"found": "true"}
 
             items[href] = title
 
@@ -75,18 +77,24 @@ class DelpherTranslator:
         """Main extraction method."""
         page_type = self.detect_web(doc, url)
 
-        if page_type == 'multiple':
+        if page_type == "multiple":
             return self.get_search_results(doc, check_only=False)
         else:
             return self.scrape(doc, url)
 
-    def _get_metadata_field(self, details: BeautifulSoup, field_name: str) -> Optional[str]:
+    def _get_metadata_field(
+        self, details: BeautifulSoup, field_name: str
+    ) -> Optional[str]:
         """Extract metadata field value from details section."""
-        dt = details.find('dt', class_='metadata__details-text', string=lambda x: x and field_name in x)
+        dt = details.find(
+            "dt",
+            class_="metadata__details-text",
+            string=lambda x: x and field_name in x,
+        )
         if dt:
-            dd = dt.find_next_sibling('dd')
+            dd = dt.find_next_sibling("dd")
             if dd:
-                a_tag = dd.find('a')
+                a_tag = dd.find("a")
                 if a_tag:
                     return a_tag.get_text(strip=True)
                 return dd.get_text(strip=True)
@@ -96,106 +104,121 @@ class DelpherTranslator:
         """Scrape item metadata."""
         item_type = self.detect_web(doc, url)
         item = {
-            'itemType': item_type,
-            'url': url,
-            'creators': [],
-            'tags': [],
-            'attachments': []
+            "itemType": item_type,
+            "url": url,
+            "creators": [],
+            "tags": [],
+            "attachments": [],
         }
 
         # Find metadata section
-        details = doc.select_one('dl.metadata__details-description-list')
+        details = doc.select_one("dl.metadata__details-description-list")
         if not details:
             return item
 
         # Extract title
-        title = self._get_metadata_field(details, 'Titel') or self._get_metadata_field(details, 'Kop')
+        title = self._get_metadata_field(details, "Titel") or self._get_metadata_field(
+            details, "Kop"
+        )
         if not title:
-            title = self._get_metadata_field(details, 'Krantentitel')
+            title = self._get_metadata_field(details, "Krantentitel")
         if title:
-            item['title'] = title
+            item["title"] = title
 
         # Extract date
-        date = self._get_metadata_field(details, 'Publicatiedatum') or self._get_metadata_field(details, 'Datum')
+        date = self._get_metadata_field(
+            details, "Publicatiedatum"
+        ) or self._get_metadata_field(details, "Datum")
         if not date:
-            date = self._get_metadata_field(details, 'Jaar van uitgave')
+            date = self._get_metadata_field(details, "Jaar van uitgave")
         if date and len(date) > 4:
             # Convert DD-MM-YYYY to YYYY-MM-DD
-            date = re.sub(r'(\d{2})-(\d{2})-(\d{4})', r'\3-\2-\1', date)
+            date = re.sub(r"(\d{2})-(\d{2})-(\d{4})", r"\3-\2-\1", date)
         if date:
-            item['date'] = date
+            item["date"] = date
 
         # Extract other fields
-        item['libraryCatalog'] = self._get_metadata_field(details, 'Herkomst') or 'Delpher'
-        item['publisher'] = self._get_metadata_field(details, 'Drukker/Uitgever') or self._get_metadata_field(details, 'Uitgever')
-        item['callNumber'] = self._get_metadata_field(details, 'PPN')
-        item['language'] = self._get_metadata_field(details, 'Taal')
-        item['place'] = self._get_metadata_field(details, 'Plaats van uitgave')
-        item['numPages'] = self._get_metadata_field(details, 'Omvang')
-
-        if item_type in ['newspaperArticle', 'journalArticle']:
-            item['publicationTitle'] = self._get_metadata_field(details, 'Krantentitel')
-            item['issue'] = self._get_metadata_field(details, 'Aflevering')
-            item['edition'] = self._get_metadata_field(details, 'Editie')
+        item["libraryCatalog"] = (
+            self._get_metadata_field(details, "Herkomst") or "Delpher"
+        )
+        item["publisher"] = self._get_metadata_field(
+            details, "Drukker/Uitgever"
+        ) or self._get_metadata_field(details, "Uitgever")
+        item["callNumber"] = self._get_metadata_field(details, "PPN")
+        item["language"] = self._get_metadata_field(details, "Taal")
+        item["place"] = self._get_metadata_field(details, "Plaats van uitgave")
+        item["numPages"] = self._get_metadata_field(details, "Omvang")
+
+        if item_type in ["newspaperArticle", "journalArticle"]:
+            item["publicationTitle"] = self._get_metadata_field(details, "Krantentitel")
+            item["issue"] = self._get_metadata_field(details, "Aflevering")
+            item["edition"] = self._get_metadata_field(details, "Editie")
 
         # Extract authors
-        author_dts = details.find_all('dt', class_='metadata__details-text',
-                                      string=lambda x: x and ('Auteur' in x or 'Coauteur' in x))
+        author_dts = details.find_all(
+            "dt",
+            class_="metadata__details-text",
+            string=lambda x: x and ("Auteur" in x or "Coauteur" in x),
+        )
         for dt in author_dts:
-            dd = dt.find_next_sibling('dd')
+            dd = dt.find_next_sibling("dd")
             if dd:
                 author_text = dd.get_text(strip=True)
                 names = author_text.split()
                 if len(names) >= 2:
-                    item['creators'].append({
-                        'firstName': ' '.join(names[:-1]),
-                        'lastName': names[-1],
-                        'creatorType': 'author'
-                    })
+                    item["creators"].append(
+                        {
+                            "firstName": " ".join(names[:-1]),
+                            "lastName": names[-1],
+                            "creatorType": "author",
+                        }
+                    )
                 else:
-                    item['creators'].append({
-                        'lastName': author_text,
-                        'creatorType': 'author',
-                        'fieldMode': True
-                    })
+                    item["creators"].append(
+                        {
+                            "lastName": author_text,
+                            "creatorType": "author",
+                            "fieldMode": True,
+                        }
+                    )
 
         # Extract tags
         tag_dds = details.select('dt:contains("Onderwerp") ~ dd')
         for dd in tag_dds:
-            tag_links = dd.find_all('a')
+            tag_links = dd.find_all("a")
             for link in tag_links:
                 tag_text = link.get_text(strip=True)
                 if tag_text:
-                    item['tags'].append({'tag': tag_text})
+                    item["tags"].append({"tag": tag_text})
 
         # Extract URL from share input
-        url_input = doc.select_one('input.object-view-menu__share-links-details-input:last-of-type')
-        if url_input and url_input.get('value'):
-            item['url'] = url_input['value']
+        url_input = doc.select_one(
+            "input.object-view-menu__share-links-details-input:last-of-type"
+        )
+        if url_input and url_input.get("value"):
+            item["url"] = url_input["value"]
 
         # Add attachments
-        item['attachments'].append({
-            'title': 'Snapshot',
-            'mimeType': 'text/html',
-            'url': url
-        })
+        item["attachments"].append(
+            {"title": "Snapshot", "mimeType": "text/html", "url": url}
+        )
 
         # Add PDF if available
         pdf_link = doc.select_one('a.object-view-menu__downloads-link:contains("pdf")')
-        if pdf_link and pdf_link.get('href'):
-            item['attachments'].append({
-                'title': 'Full Text PDF',
-                'mimeType': 'application/pdf',
-                'url': pdf_link['href']
-            })
+        if pdf_link and pdf_link.get("href"):
+            item["attachments"].append(
+                {
+                    "title": "Full Text PDF",
+                    "mimeType": "application/pdf",
+                    "url": pdf_link["href"],
+                }
+            )
 
         # Add JPG if available
         jpg_link = doc.select_one('a.object-view-menu__downloads-link:contains("jpg")')
-        if jpg_link and jpg_link.get('href'):
-            item['attachments'].append({
-                'title': 'Image',
-                'mimeType': 'image/jpeg',
-                'url': jpg_link['href']
-            })
+        if jpg_link and jpg_link.get("href"):
+            item["attachments"].append(
+                {"title": "Image", "mimeType": "image/jpeg", "url": jpg_link["href"]}
+            )
 
         return item