scitex 2.5.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

scitex/scholar/url_finder/translators/individual/cabi_cab_abstracts.py

@@ -34,7 +34,7 @@ class CABICABAbstractsTranslator:
         "inRepository": True,
         "translatorType": 4,
         "browserSupport": "gcsibv",
-        "lastUpdated": "2017-06-14 03:41:30"
+        "lastUpdated": "2017-06-14 03:41:30",
     }
 
     def detect_web(self, doc: BeautifulSoup, url: str) -> str:
@@ -48,13 +48,15 @@ class CABICABAbstractsTranslator:
         Returns:
             'journalArticle' for articles, 'multiple' for search, empty string otherwise
         """
-        if 'cabdirect/abstract/' in url or 'cabdirect/FullTextPDF/' in url:
-            return 'journalArticle'
-        if 'cabdirect/search' in url and self.get_search_results(doc, True):
-            return 'multiple'
-        return ''
-
-    def get_search_results(self, doc: BeautifulSoup, check_only: bool = False) -> Dict[str, str]:
+        if "cabdirect/abstract/" in url or "cabdirect/FullTextPDF/" in url:
+            return "journalArticle"
+        if "cabdirect/search" in url and self.get_search_results(doc, True):
+            return "multiple"
+        return ""
+
+    def get_search_results(
+        self, doc: BeautifulSoup, check_only: bool = False
+    ) -> Dict[str, str]:
         """
         Get search results from the page.
 
@@ -69,12 +71,12 @@ class CABICABAbstractsTranslator:
         rows = doc.select('div.list-content h2 a[href*="/abstract/"]')
 
         for row in rows:
-            href = row.get('href')
+            href = row.get("href")
             title = row.get_text(strip=True)
             if not href or not title:
                 continue
             if check_only:
-                return {'found': 'true'}
+                return {"found": "true"}
             items[href] = title
 
         return items
@@ -90,7 +92,7 @@ class CABICABAbstractsTranslator:
         Returns:
             Dictionary containing article metadata or search results
         """
-        if self.detect_web(doc, url) == 'multiple':
+        if self.detect_web(doc, url) == "multiple":
             return self.get_search_results(doc)
         return self.scrape(doc, url)
 
@@ -106,51 +108,51 @@ class CABICABAbstractsTranslator:
             Dictionary containing article metadata
         """
         item = {
-            'itemType': 'journalArticle',
-            'url': url,
-            'creators': [],
-            'tags': [],
-            'attachments': [],
-            'libraryCatalog': 'CABI - CAB Abstracts',
-            'archive': ''  # We don't want CAB in archive field
+            "itemType": "journalArticle",
+            "url": url,
+            "creators": [],
+            "tags": [],
+            "attachments": [],
+            "libraryCatalog": "CABI - CAB Abstracts",
+            "archive": "",  # We don't want CAB in archive field
         }
 
         # Extract PDF URL
         pdf_elem = doc.select_one('p.btnabstract a[href$=".pdf"]')
         if pdf_elem:
-            pdf_url = pdf_elem.get('href')
+            pdf_url = pdf_elem.get("href")
             if pdf_url:
-                item['attachments'].append({
-                    'title': 'Full Text PDF',
-                    'mimeType': 'application/pdf',
-                    'url': pdf_url
-                })
+                item["attachments"].append(
+                    {
+                        "title": "Full Text PDF",
+                        "mimeType": "application/pdf",
+                        "url": pdf_url,
+                    }
+                )
 
         # Extract abstract
-        abstract_elem = doc.select_one('div.abstract')
+        abstract_elem = doc.select_one("div.abstract")
         if abstract_elem:
-            item['abstractNote'] = abstract_elem.get_text(strip=True)
+            item["abstractNote"] = abstract_elem.get_text(strip=True)
 
         # Extract editors
         editor_elems = doc.select('p[id*="ulEditors"] a')
         for editor in editor_elems:
             editor_text = editor.get_text(strip=True)
             if editor_text:
-                item['creators'].append(self._clean_author(editor_text, 'editor'))
+                item["creators"].append(self._clean_author(editor_text, "editor"))
 
         # Add snapshot attachment
-        item['attachments'].append({
-            'title': 'Snapshot',
-            'mimeType': 'text/html',
-            'url': url
-        })
+        item["attachments"].append(
+            {"title": "Snapshot", "mimeType": "text/html", "url": url}
+        )
 
         # Note: The actual translator uses RIS import which we can't fully replicate here
         # This is a simplified version that extracts what we can from the HTML
 
         return item
 
-    def _clean_author(self, name: str, creator_type: str = 'author') -> Dict[str, Any]:
+    def _clean_author(self, name: str, creator_type: str = "author") -> Dict[str, Any]:
         """
         Parse author name into first and last name.
 
@@ -166,13 +168,9 @@ class CABICABAbstractsTranslator:
 
         if len(parts) >= 2:
             return {
-                'firstName': ' '.join(parts[:-1]),
-                'lastName': parts[-1],
-                'creatorType': creator_type
+                "firstName": " ".join(parts[:-1]),
+                "lastName": parts[-1],
+                "creatorType": creator_type,
             }
         else:
-            return {
-                'lastName': name,
-                'creatorType': creator_type,
-                'fieldMode': True
-            }
+            return {"lastName": name, "creatorType": creator_type, "fieldMode": True}

scitex/scholar/url_finder/translators/individual/cambridge.py

@@ -12,7 +12,9 @@ class CambridgeTranslator(BaseTranslator):
     """Cambridge Core."""
 
     LABEL = "Cambridge Core"
-    URL_TARGET_PATTERN = r"^https?://www\.cambridge\.org/core/(search\?|journals/|books/|.+/listing?)"
+    URL_TARGET_PATTERN = (
+        r"^https?://www\.cambridge\.org/core/(search\?|journals/|books/|.+/listing?)"
+    )
 
     @classmethod
     def matches_url(cls, url: str) -> bool:

scitex/scholar/url_finder/translators/individual/cambridge_core.py

@@ -26,7 +26,9 @@ class CambridgeCoreTranslator(BaseTranslator):
     """
 
     LABEL = "Cambridge Core"
-    URL_TARGET_PATTERN = r"^https?://www\.cambridge\.org/core/(search\?|journals/|books/|.+/listing?)"
+    URL_TARGET_PATTERN = (
+        r"^https?://www\.cambridge\.org/core/(search\?|journals/|books/|.+/listing?)"
+    )
 
     @classmethod
     def matches_url(cls, url: str) -> bool:
@@ -82,18 +84,18 @@ class CambridgeCoreTranslator(BaseTranslator):
             # This is the most reliable method for both books and articles
             citation_pdf = await page.locator(
                 'meta[name="citation_pdf_url"], meta[name="citation_ pdf_url"]'
-            ).first.get_attribute('content', timeout=2000)
+            ).first.get_attribute("content", timeout=2000)
 
             if citation_pdf:
                 # Make absolute URL if needed
-                if not citation_pdf.startswith('http'):
-                    if citation_pdf.startswith('/'):
-                        base_url = await page.evaluate('window.location.origin')
+                if not citation_pdf.startswith("http"):
+                    if citation_pdf.startswith("/"):
+                        base_url = await page.evaluate("window.location.origin")
                         citation_pdf = f"{base_url}{citation_pdf}"
                     else:
-                        base_url = await page.evaluate('window.location.href')
+                        base_url = await page.evaluate("window.location.href")
                         # Remove trailing path
-                        base_url = re.sub(r'/[^/]*$', '', base_url)
+                        base_url = re.sub(r"/[^/]*$", "", base_url)
                         citation_pdf = f"{base_url}/{citation_pdf}"
 
                 pdf_urls.append(citation_pdf)
@@ -107,17 +109,17 @@ class CambridgeCoreTranslator(BaseTranslator):
             # Pattern: .actions a[target="_blank"][href*=".pdf"]
             pdf_link = await page.locator(
                 '.actions a[target="_blank"][href*=".pdf"]'
-            ).first.get_attribute('href', timeout=2000)
+            ).first.get_attribute("href", timeout=2000)
 
             if pdf_link:
                 # Make absolute URL if needed
-                if not pdf_link.startswith('http'):
-                    if pdf_link.startswith('/'):
-                        base_url = await page.evaluate('window.location.origin')
+                if not pdf_link.startswith("http"):
+                    if pdf_link.startswith("/"):
+                        base_url = await page.evaluate("window.location.origin")
                         pdf_link = f"{base_url}{pdf_link}"
                     else:
-                        base_url = await page.evaluate('window.location.href')
-                        base_url = re.sub(r'/[^/]*$', '', base_url)
+                        base_url = await page.evaluate("window.location.href")
+                        base_url = re.sub(r"/[^/]*$", "", base_url)
                         pdf_link = f"{base_url}/{pdf_link}"
 
                 pdf_urls.append(pdf_link)
@@ -131,17 +133,17 @@ class CambridgeCoreTranslator(BaseTranslator):
             # Cambridge Core often has a "PDF" button or link
             pdf_button = await page.locator(
                 'a[href*=".pdf"], a[data-track-label="pdf"], a:has-text("PDF")'
-            ).first.get_attribute('href', timeout=2000)
+            ).first.get_attribute("href", timeout=2000)
 
-            if pdf_button and '.pdf' in pdf_button.lower():
+            if pdf_button and ".pdf" in pdf_button.lower():
                 # Make absolute URL if needed
-                if not pdf_button.startswith('http'):
-                    if pdf_button.startswith('/'):
-                        base_url = await page.evaluate('window.location.origin')
+                if not pdf_button.startswith("http"):
+                    if pdf_button.startswith("/"):
+                        base_url = await page.evaluate("window.location.origin")
                         pdf_button = f"{base_url}{pdf_button}"
                     else:
-                        base_url = await page.evaluate('window.location.href')
-                        base_url = re.sub(r'/[^/]*$', '', base_url)
+                        base_url = await page.evaluate("window.location.href")
+                        base_url = re.sub(r"/[^/]*$", "", base_url)
                         pdf_button = f"{base_url}/{pdf_button}"
 
                 pdf_urls.append(pdf_button)
@@ -153,15 +155,19 @@ class CambridgeCoreTranslator(BaseTranslator):
         try:
             # Method 4: Extract DOI and construct PDF URL
             # Cambridge Core PDF URLs often follow pattern: /core/services/aop-cambridge-core/content/view/{DOI}
-            doi_meta = await page.locator('meta[name="citation_doi"]').first.get_attribute('content', timeout=2000)
+            doi_meta = await page.locator(
+                'meta[name="citation_doi"]'
+            ).first.get_attribute("content", timeout=2000)
 
             if doi_meta:
                 # Remove any prefix like "doi:" or "https://doi.org/"
-                doi = re.sub(r'^(doi:|https?://doi\.org/)', '', doi_meta)
+                doi = re.sub(r"^(doi:|https?://doi\.org/)", "", doi_meta)
 
                 # Try constructing PDF URL from DOI
-                base_url = await page.evaluate('window.location.origin')
-                pdf_url = f"{base_url}/core/services/aop-cambridge-core/content/view/{doi}"
+                base_url = await page.evaluate("window.location.origin")
+                pdf_url = (
+                    f"{base_url}/core/services/aop-cambridge-core/content/view/{doi}"
+                )
 
                 pdf_urls.append(pdf_url)
                 return pdf_urls

scitex/scholar/url_finder/translators/individual/caod.py

@@ -33,7 +33,7 @@ class CAODTranslator:
         "inRepository": True,
         "translatorType": 4,
         "browserSupport": "gcsibv",
-        "lastUpdated": "2018-09-08 13:38:50"
+        "lastUpdated": "2018-09-08 13:38:50",
     }
 
     def detect_web(self, doc: BeautifulSoup, url: str) -> str:
@@ -47,14 +47,16 @@ class CAODTranslator:
         Returns:
             'journalArticle' for articles, 'multiple' for search, empty string otherwise
         """
-        if 'articles/found.htm?' in url:
+        if "articles/found.htm?" in url:
             if self.get_search_results(doc, True):
-                return 'multiple'
+                return "multiple"
         else:
-            return 'journalArticle'
-        return ''
+            return "journalArticle"
+        return ""
 
-    def get_search_results(self, doc: BeautifulSoup, check_only: bool = False) -> Dict[str, str]:
+    def get_search_results(
+        self, doc: BeautifulSoup, check_only: bool = False
+    ) -> Dict[str, str]:
         """
         Get search results from the page.
 
@@ -66,15 +68,15 @@ class CAODTranslator:
             Dictionary mapping URLs to titles, or empty dict
         """
         items = {}
-        rows = doc.select('div.searchlist a:has(b)')
+        rows = doc.select("div.searchlist a:has(b)")
 
         for row in rows:
-            href = row.get('href')
+            href = row.get("href")
             title = row.get_text(strip=True)
             if not href or not title:
                 continue
             if check_only:
-                return {'found': 'true'}
+                return {"found": "true"}
             items[href] = title
 
         return items
@@ -90,7 +92,7 @@ class CAODTranslator:
         Returns:
             Dictionary containing article metadata or search results
         """
-        if self.detect_web(doc, url) == 'multiple':
+        if self.detect_web(doc, url) == "multiple":
             return self.get_search_results(doc)
         return self.scrape(doc, url)
 
@@ -106,12 +108,12 @@ class CAODTranslator:
             Dictionary containing article metadata
         """
         item = {
-            'itemType': 'journalArticle',
-            'url': url,
-            'creators': [],
-            'tags': [],
-            'attachments': [],
-            'libraryCatalog': 'caod.oriprobe.com'
+            "itemType": "journalArticle",
+            "url": url,
+            "creators": [],
+            "tags": [],
+            "attachments": [],
+            "libraryCatalog": "caod.oriprobe.com",
         }
 
         # Extract title
@@ -119,9 +121,13 @@ class CAODTranslator:
         if not title_elem:
             title_elem = doc.select_one('meta[itemprop="headline"]')
         if title_elem:
-            content = title_elem.get('content') if title_elem.name == 'meta' else title_elem.get_text(strip=True)
+            content = (
+                title_elem.get("content")
+                if title_elem.name == "meta"
+                else title_elem.get_text(strip=True)
+            )
             if content:
-                item['title'] = content
+                item["title"] = content
 
         # Extract authors - surname comes first and is often capitalized
         author_elems = doc.select('span[itemprop="author"] a')
@@ -132,64 +138,66 @@ class CAODTranslator:
                 parts = author_text.split()
                 if parts:
                     # Capitalize first char and lowercase rest of surname
-                    parts[0] = parts[0][0] + parts[0][1:].lower() if len(parts[0]) > 1 else parts[0]
+                    parts[0] = (
+                        parts[0][0] + parts[0][1:].lower()
+                        if len(parts[0]) > 1
+                        else parts[0]
+                    )
                     # Reverse to put given name first
-                    reversed_name = ' '.join(reversed(parts))
-                    item['creators'].append(self._clean_author(reversed_name))
+                    reversed_name = " ".join(reversed(parts))
+                    item["creators"].append(self._clean_author(reversed_name))
 
         # Extract publication details
         pub_elem = doc.select_one('span[itemprop="name"]')
         if pub_elem:
-            item['publicationTitle'] = pub_elem.get_text(strip=True)
+            item["publicationTitle"] = pub_elem.get_text(strip=True)
 
         # Extract date
         date_elem = doc.select_one('meta[itemprop="datePublished"]')
         if date_elem:
-            date_content = date_elem.get('content')
+            date_content = date_elem.get("content")
             if date_content:
-                item['date'] = date_content
+                item["date"] = date_content
 
         # Extract volume
         volume_elem = doc.select_one('meta[itemprop="volumeNumber"]')
         if volume_elem:
-            volume_content = volume_elem.get('content')
+            volume_content = volume_elem.get("content")
             if volume_content:
-                item['volume'] = volume_content
+                item["volume"] = volume_content
 
         # Extract issue
         issue_elem = doc.select_one('meta[itemprop="issueNumber"]')
         if issue_elem:
-            issue_content = issue_elem.get('content')
+            issue_content = issue_elem.get("content")
             if issue_content:
-                item['issue'] = issue_content
+                item["issue"] = issue_content
 
         # Extract pages
         pages_elem = doc.select_one('meta[itemprop="pagination"]')
         if pages_elem:
-            pages_content = pages_elem.get('content')
+            pages_content = pages_elem.get("content")
             if pages_content:
-                item['pages'] = pages_content
+                item["pages"] = pages_content
 
         # Extract abstract
         abstract_elem = doc.select_one('meta[itemprop="description"]')
         if abstract_elem:
-            abstract_content = abstract_elem.get('content')
+            abstract_content = abstract_elem.get("content")
             if abstract_content:
-                item['abstractNote'] = abstract_content
+                item["abstractNote"] = abstract_content
 
         # Extract keywords
         keyword_elems = doc.select('span[itemprop="headline"] a')
         for keyword in keyword_elems:
             keyword_text = keyword.get_text(strip=True)
             if keyword_text:
-                item['tags'].append({'tag': keyword_text})
+                item["tags"].append({"tag": keyword_text})
 
         # Add snapshot attachment
-        item['attachments'].append({
-            'title': 'Snapshot',
-            'mimeType': 'text/html',
-            'url': url
-        })
+        item["attachments"].append(
+            {"title": "Snapshot", "mimeType": "text/html", "url": url}
+        )
 
         return item
 
@@ -208,13 +216,9 @@ class CAODTranslator:
 
         if len(parts) >= 2:
             return {
-                'firstName': ' '.join(parts[:-1]),
-                'lastName': parts[-1],
-                'creatorType': 'author'
+                "firstName": " ".join(parts[:-1]),
+                "lastName": parts[-1],
+                "creatorType": "author",
             }
         else:
-            return {
-                'lastName': name,
-                'creatorType': 'author',
-                'fieldMode': True
-            }
+            return {"lastName": name, "creatorType": "author", "fieldMode": True}