scitex 2.5.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

scitex/scholar/url_finder/translators/individual/australian_dictionary_of_biography.py

@@ -18,13 +18,13 @@ TRANSLATOR_INFO = {
     "in_repository": True,
     "translator_type": 4,
     "browser_support": "gcsibv",
-    "last_updated": "2021-07-14 04:18:08"
+    "last_updated": "2021-07-14 04:18:08",
 }
 
 
 def detect_web(doc: Any, url: str) -> Optional[str]:
     """Detect if the page is a single item or multiple items"""
-    if '/biography/' in url:
+    if "/biography/" in url:
         return "bookSection"
     elif get_search_results(doc, check_only=True):
         return "multiple"
@@ -34,10 +34,10 @@ def detect_web(doc: Any, url: str) -> Optional[str]:
 def get_search_results(doc: Any, check_only: bool = False) -> Optional[Dict[str, str]]:
     """Get search results from a multiple item page"""
     items = {}
-    rows = doc.select('a.name')
+    rows = doc.select("a.name")
 
     for row in rows:
-        href = row.get('href')
+        href = row.get("href")
         title = row.get_text(strip=True)
         if href and title:
             if check_only:
@@ -63,48 +63,52 @@ def scrape(doc: Any, url: str) -> Dict[str, Any]:
         "attachments": [{"title": "Snapshot", "mimeType": "text/html"}],
         "tags": [],
         "notes": [],
-        "seeAlso": []
+        "seeAlso": [],
     }
 
-    main = doc.select_one('#pageColumnMain')
+    main = doc.select_one("#pageColumnMain")
     if not main:
         return item
 
     # Get title
-    title_elem = main.select_one('h2')
+    title_elem = main.select_one("h2")
     if title_elem:
         item["title"] = title_elem.get_text(strip=True)
 
     # Get abstract
-    abstract_elem = main.select_one('.biographyContent p')
+    abstract_elem = main.select_one(".biographyContent p")
     if abstract_elem:
         item["abstractNote"] = abstract_elem.get_text(strip=True)
 
     # Get volume and date from notice
-    notice_elem = main.select_one('.textNotice')
+    notice_elem = main.select_one(".textNotice")
     if notice_elem:
         notice_text = notice_elem.get_text(strip=True)
         # Try to match: "volume X, (MUP), YYYY"
-        match = re.search(r'([^,]+), \(MUP\), ([^,]+)', notice_text)
+        match = re.search(r"([^,]+), \(MUP\), ([^,]+)", notice_text)
         if match:
             volume_text = match.group(1)
-            item["volume"] = re.sub(r'^\s*volume\s*', '', volume_text, flags=re.IGNORECASE)
+            item["volume"] = re.sub(
+                r"^\s*volume\s*", "", volume_text, flags=re.IGNORECASE
+            )
             item["date"] = match.group(2).strip()
         else:
             # Just use the text as date
             item["date"] = notice_text.strip()
 
     # Get authors
-    author_links = main.select('.authorName a')
+    author_links = main.select(".authorName a")
     for author_link in author_links:
         author_name = author_link.get_text(strip=True)
         parts = author_name.split()
         if len(parts) > 1:
-            item["creators"].append({
-                "firstName": " ".join(parts[:-1]),
-                "lastName": parts[-1],
-                "creatorType": "author"
-            })
+            item["creators"].append(
+                {
+                    "firstName": " ".join(parts[:-1]),
+                    "lastName": parts[-1],
+                    "creatorType": "author",
+                }
+            )
 
     return item
 

scitex/scholar/url_finder/translators/individual/bailii.py

@@ -18,11 +18,13 @@ TRANSLATOR_INFO = {
     "in_repository": True,
     "translator_type": 4,
     "browser_support": "gcsibv",
-    "last_updated": "2021-09-16 20:57:17"
+    "last_updated": "2021-09-16 20:57:17",
 }
 
 
-LII_REGEXP = re.compile(r'^https?://www\.bailii\.org(?:/cgi-bin/markup\.cgi\?doc=)?/\w+/cases/.+\.html')
+LII_REGEXP = re.compile(
+    r"^https?://www\.bailii\.org(?:/cgi-bin/markup\.cgi\?doc=)?/\w+/cases/.+\.html"
+)
 
 
 def detect_web(doc: Any, url: str) -> Optional[str]:
@@ -31,9 +33,9 @@ def detect_web(doc: Any, url: str) -> Optional[str]:
         return "case"
     else:
         # Check if there are any links matching the pattern
-        links = doc.select('a')
+        links = doc.select("a")
         for link in links:
-            href = link.get('href', '')
+            href = link.get("href", "")
             if LII_REGEXP.match(href):
                 return "multiple"
     return None
@@ -52,52 +54,53 @@ def scrape(doc: Any, url: str) -> Dict[str, Any]:
         "attachments": [{"url": url, "title": "Snapshot", "mimeType": "text/html"}],
         "tags": [],
         "notes": [],
-        "seeAlso": []
+        "seeAlso": [],
     }
 
     # Get title
-    title_tag = doc.select_one('title')
+    title_tag = doc.select_one("title")
     if title_tag:
         title_text = title_tag.get_text(strip=True)
         item["title"] = title_text
 
         # Parse title: CaseName [YYYY] Citation (DD Month YYYY)
         title_match = re.match(
-            r'^(.+)\s+\[(\d+)\]\s+(.+)\s+\((\d+)\s+(\w+)\s+(\d+)\)',
-            title_text
+            r"^(.+)\s+\[(\d+)\]\s+(.+)\s+\((\d+)\s+(\w+)\s+(\d+)\)", title_text
         )
         if title_match:
-            item["caseName"] = f"{title_match.group(1)} [{title_match.group(2)}] {title_match.group(3)}"
-            item["dateDecided"] = f"{title_match.group(4)} {title_match.group(5)} {title_match.group(6)}"
+            item["caseName"] = (
+                f"{title_match.group(1)} [{title_match.group(2)}] {title_match.group(3)}"
+            )
+            item["dateDecided"] = (
+                f"{title_match.group(4)} {title_match.group(5)} {title_match.group(6)}"
+            )
         else:
             item["caseName"] = title_text
 
     # Get court from URL
-    court_match = re.search(r'/cases/([^/]+)/([^/]+)/', url)
+    court_match = re.search(r"/cases/([^/]+)/([^/]+)/", url)
     if court_match:
         jurisdiction = court_match.group(1)
         court_code = court_match.group(2)
 
         # Check if court_code looks like a division
-        if re.match(r'\w+', court_code):
+        if re.match(r"\w+", court_code):
             item["court"] = f"{jurisdiction} ({court_code})"
         else:
             item["court"] = jurisdiction
 
     # Get judge/panel
-    panel_tags = doc.select('PANEL')
+    panel_tags = doc.select("PANEL")
     if panel_tags:
         for panel in panel_tags:
             name = panel.get_text(strip=True)
             if name:
-                item["creators"].append({
-                    "lastName": name,
-                    "creatorType": "author",
-                    "fieldMode": 1
-                })
+                item["creators"].append(
+                    {"lastName": name, "creatorType": "author", "fieldMode": 1}
+                )
 
     # Get citation note
-    cite_tags = doc.select('CITATION')
+    cite_tags = doc.select("CITATION")
     if cite_tags:
         for cite in cite_tags:
             note_text = cite.get_text(strip=True)

scitex/scholar/url_finder/translators/individual/bbc.py

@@ -20,25 +20,25 @@ TRANSLATOR_INFO = {
     "in_repository": True,
     "translator_type": 4,
     "browser_support": "gcsibv",
-    "last_updated": "2019-06-10 21:51:43"
+    "last_updated": "2019-06-10 21:51:43",
 }
 
 
 def detect_web(doc: Any, url: str) -> Optional[str]:
     """Detect if the page is a newspaper article, blog post, video, or multiple items"""
-    url = re.sub(r'[?#].+', '', url)
+    url = re.sub(r"[?#].+", "", url)
 
-    if re.search(r'\d{8}$', url) or re.search(r'\d{7}\.(stm)$', url):
-        page_node = doc.select_one('#page')
+    if re.search(r"\d{8}$", url) or re.search(r"\d{7}\.(stm)$", url):
+        page_node = doc.select_one("#page")
         if page_node:
-            classes = page_node.get('class', [])
+            classes = page_node.get("class", [])
             if isinstance(classes, str):
                 classes = classes.split()
-            if any(cls in classes for cls in ['media-asset-page', 'vxp-headlines']):
+            if any(cls in classes for cls in ["media-asset-page", "vxp-headlines"]):
                 return "videoRecording"
         return "newspaperArticle"
 
-    if '/newsbeat/article' in url:
+    if "/newsbeat/article" in url:
         return "blogPost"
 
     if get_search_results(doc, check_only=True):
@@ -50,14 +50,14 @@ def detect_web(doc: Any, url: str) -> Optional[str]:
 def get_search_results(doc: Any, check_only: bool = False) -> Optional[Dict[str, str]]:
     """Get search results from a multiple item page"""
     items = {}
-    rows = doc.select('a:has(h3)')
+    rows = doc.select("a:has(h3)")
 
     # For NewsBeat
     if not rows:
         rows = doc.select('article div h1[itemprop="headline"] a')
 
     for row in rows:
-        href = row.get('href')
+        href = row.get("href")
         title = row.get_text(strip=True)
         if href and title:
             if check_only:
@@ -69,7 +69,7 @@ def get_search_results(doc: Any, check_only: bool = False) -> Optional[Dict[str,
 
 def scrape(doc: Any, url: str) -> Dict[str, Any]:
     """Scrape a single item page"""
-    url = re.sub(r'[?#].+', '', url)
+    url = re.sub(r"[?#].+", "", url)
     item_type = detect_web(doc, url)
     if not item_type or item_type == "multiple":
         return {}
@@ -85,7 +85,7 @@ def scrape(doc: Any, url: str) -> Dict[str, Any]:
         "attachments": [{"title": "Snapshot", "mimeType": "text/html"}],
         "tags": [],
         "notes": [],
-        "seeAlso": []
+        "seeAlso": [],
     }
 
     # Try to get data from JSON-LD
@@ -102,7 +102,7 @@ def scrape(doc: Any, url: str) -> Dict[str, Any]:
     if not item["date"]:
         seconds_elem = doc.select_one('div:has(h1, h2) *[class*="date"][data-seconds]')
         if seconds_elem:
-            seconds = seconds_elem.get('data-seconds')
+            seconds = seconds_elem.get("data-seconds")
             if seconds:
                 try:
                     timestamp = int(seconds)
@@ -115,88 +115,92 @@ def scrape(doc: Any, url: str) -> Dict[str, Any]:
     if not item["date"]:
         date_meta = doc.select_one('meta[property="rnews:datePublished"]')
         if date_meta:
-            item["date"] = date_meta.get('content', '')
+            item["date"] = date_meta.get("content", "")
         else:
-            timestamp_elem = doc.select_one('p.timestamp')
+            timestamp_elem = doc.select_one("p.timestamp")
             if timestamp_elem:
                 item["date"] = timestamp_elem.get_text(strip=True)
             else:
                 date_meta2 = doc.select_one('meta[name="OriginalPublicationDate"]')
                 if date_meta2:
-                    item["date"] = date_meta2.get('content', '')
+                    item["date"] = date_meta2.get("content", "")
 
     # Get title from embedded metadata
     title_meta = doc.select_one('meta[property="og:title"]')
     if title_meta:
-        item["title"] = title_meta.get('content', '')
+        item["title"] = title_meta.get("content", "")
     else:
-        h1 = doc.select_one('h1')
+        h1 = doc.select_one("h1")
         if h1:
             item["title"] = h1.get_text(strip=True)
 
     # For old .stm pages
-    if url.endswith('.stm'):
+    if url.endswith(".stm"):
         headline_meta = doc.select_one('meta[name="Headline"]')
         if headline_meta:
-            item["title"] = headline_meta.get('content', '')
+            item["title"] = headline_meta.get("content", "")
 
     # Get authors from byline
-    byline_elem = doc.select_one('span.byline__name')
+    byline_elem = doc.select_one("span.byline__name")
     if byline_elem:
         author_string = byline_elem.get_text(strip=True)
-        author_string = author_string.replace('By', '').replace('...', '').strip()
+        author_string = author_string.replace("By", "").replace("...", "").strip()
 
         # Check if author is real or just webpage title
-        h1_elem = doc.select_one('h1')
+        h1_elem = doc.select_one("h1")
         webpage_title = h1_elem.get_text(strip=True).lower() if h1_elem else ""
 
-        authors = author_string.split('&')
+        authors = author_string.split("&")
         for author in authors:
             author = author.strip()
             if webpage_title and author.lower() in webpage_title:
                 continue
             parts = author.split()
             if len(parts) > 1:
-                item["creators"].append({
-                    "firstName": " ".join(parts[:-1]),
-                    "lastName": parts[-1],
-                    "creatorType": "author"
-                })
+                item["creators"].append(
+                    {
+                        "firstName": " ".join(parts[:-1]),
+                        "lastName": parts[-1],
+                        "creatorType": "author",
+                    }
+                )
     else:
         # Try old format
-        byline_p = doc.select_one('p.byline')
+        byline_p = doc.select_one("p.byline")
         if byline_p:
             author_string = byline_p.get_text(strip=True)
-            title_em = doc.select_one('em.title')
+            title_em = doc.select_one("em.title")
             if title_em:
-                author_string = author_string.replace(title_em.get_text(), '')
-            author_string = author_string.replace('By', '').strip()
-            authors = author_string.split('&')
+                author_string = author_string.replace(title_em.get_text(), "")
+            author_string = author_string.replace("By", "").strip()
+            authors = author_string.split("&")
             for author in authors:
                 author = author.strip()
                 parts = author.split()
                 if len(parts) > 1:
-                    item["creators"].append({
-                        "firstName": " ".join(parts[:-1]),
-                        "lastName": parts[-1],
-                        "creatorType": "author"
-                    })
+                    item["creators"].append(
+                        {
+                            "firstName": " ".join(parts[:-1]),
+                            "lastName": parts[-1],
+                            "creatorType": "author",
+                        }
+                    )
 
     # Get abstract
     desc_meta = doc.select_one('meta[property="og:description"]')
     if desc_meta:
-        item["abstractNote"] = desc_meta.get('content', '')
+        item["abstractNote"] = desc_meta.get("content", "")
     else:
         desc_meta2 = doc.select_one('meta[name="Description"]')
         if desc_meta2:
-            item["abstractNote"] = desc_meta2.get('content', '')
+            item["abstractNote"] = desc_meta2.get("content", "")
 
     # Get publication title and section
     if item_type == "newspaperArticle":
         item["publicationTitle"] = "BBC News"
         section_meta = doc.select_one('meta[property="article:section"]')
         if section_meta:
-            item["section"] = section_meta.get('content', '')
+            item["section"] = section_meta.get("content", "")
 
     elif item_type == "blogPost":
         item["blogTitle"] = "BBC Newsbeat"
@@ -207,7 +211,7 @@ def scrape(doc: Any, url: str) -> Dict[str, Any]:
     # Get tags
     tags_meta = doc.select('meta[property="article:tag"]')
     for tag_meta in tags_meta:
-        tag_content = tag_meta.get('content', '')
+        tag_content = tag_meta.get("content", "")
         if tag_content:
             # Capitalize first letter
             item["tags"].append(tag_content[0].upper() + tag_content[1:])

scitex/scholar/url_finder/translators/individual/bbc_genome.py

@@ -18,19 +18,31 @@ TRANSLATOR_INFO = {
     "in_repository": True,
     "translator_type": 4,
     "browser_support": "gcsibv",
-    "last_updated": "2017-09-04 22:38:30"
+    "last_updated": "2017-09-04 22:38:30",
 }
 
 
-TV_PROGRAMS = ["bbcone", "bbctwo", "bbcthree", "bbcfour", "cbbc", "cbeebies",
-               "bbcnews", "bbcparliament", "bbchd", "bbctv", "bbcchoice", "bbcknowledge"]
+TV_PROGRAMS = [
+    "bbcone",
+    "bbctwo",
+    "bbcthree",
+    "bbcfour",
+    "cbbc",
+    "cbeebies",
+    "bbcnews",
+    "bbcparliament",
+    "bbchd",
+    "bbctv",
+    "bbcchoice",
+    "bbcknowledge",
+]
 
 
 def detect_web(doc: Any, url: str) -> Optional[str]:
     """Detect if the page is a magazine article or multiple items"""
-    if '/search/' in url and get_search_results(doc, check_only=True):
+    if "/search/" in url and get_search_results(doc, check_only=True):
         return "multiple"
-    elif doc.select_one('div.programme-details'):
+    elif doc.select_one("div.programme-details"):
         return "magazineArticle"
     return None
 
@@ -38,10 +50,10 @@ def detect_web(doc: Any, url: str) -> Optional[str]:
 def get_search_results(doc: Any, check_only: bool = False) -> Optional[Dict[str, str]]:
     """Get search results from a multiple item page"""
     items = {}
-    rows = doc.select('h2 > a.title')
+    rows = doc.select("h2 > a.title")
 
     for row in rows:
-        href = row.get('href')
+        href = row.get("href")
         title = row.get_text(strip=True)
         if href and title:
             if check_only:
@@ -69,11 +81,11 @@ def scrape(doc: Any, url: str) -> List[Dict[str, Any]]:
         "attachments": [{"title": "Snapshot", "mimeType": "text/html"}],
         "tags": [],
         "notes": [],
-        "seeAlso": []
+        "seeAlso": [],
     }
 
     # Get title
-    title_elem = doc.select_one('h1')
+    title_elem = doc.select_one("h1")
     if title_elem:
         title_text = title_elem.get_text(strip=True)
         # Convert from uppercase if needed
@@ -83,24 +95,24 @@ def scrape(doc: Any, url: str) -> List[Dict[str, Any]]:
             mag_item["title"] = title_text
 
     # Get issue info from aside
-    aside = doc.select_one('aside.issue p')
+    aside = doc.select_one("aside.issue p")
     if aside:
         aside_text = aside.get_text(strip=True)
-        parts = aside_text.split('\n')
+        parts = aside_text.split("\n")
         if len(parts) > 0:
-            mag_item["issue"] = parts[0].replace('Issue', '').strip()
+            mag_item["issue"] = parts[0].replace("Issue", "").strip()
         if len(parts) > 1:
             mag_item["date"] = parts[1].strip()
         if len(parts) > 2:
-            mag_item["pages"] = parts[2].replace('Page', '').strip()
+            mag_item["pages"] = parts[2].replace("Page", "").strip()
 
     # Get aired info
-    aired_elem = doc.select_one('.primary-content a')
-    url_program_elem = doc.select_one('.primary-content a')
-    synopsis_elem = doc.select_one('.synopsis')
+    aired_elem = doc.select_one(".primary-content a")
+    url_program_elem = doc.select_one(".primary-content a")
+    synopsis_elem = doc.select_one(".synopsis")
 
     aired = aired_elem.get_text(strip=True) if aired_elem else ""
-    url_program = url_program_elem.get('href') if url_program_elem else ""
+    url_program = url_program_elem.get("href") if url_program_elem else ""
     synopsis = synopsis_elem.get_text(strip=True) if synopsis_elem else ""
 
     if aired:
@@ -112,7 +124,7 @@ def scrape(doc: Any, url: str) -> List[Dict[str, Any]]:
 
     # Broadcast item (radio or TV)
     if url_program:
-        program = url_program.split('/')[2] if len(url_program.split('/')) > 2 else ""
+        program = url_program.split("/")[2] if len(url_program.split("/")) > 2 else ""
         broadcast_type = "tvBroadcast" if program in TV_PROGRAMS else "radioBroadcast"
 
         broadcast_item = {
@@ -124,29 +136,29 @@ def scrape(doc: Any, url: str) -> List[Dict[str, Any]]:
             "seeAlso": [mag_item["itemID"]],
             "attachments": [],
             "tags": [],
-            "notes": []
+            "notes": [],
         }
 
         # Parse aired text: "BBC Radio 4 FM, 30 September 1967 6.35"
         if aired:
-            pieces = aired.split(',')
+            pieces = aired.split(",")
             if len(pieces) > 0:
                 broadcast_item["programTitle"] = pieces[0].strip()
             if len(pieces) > 1:
-                time_elem = doc.select_one('.primary-content a span.time')
+                time_elem = doc.select_one(".primary-content a span.time")
                 time_text = time_elem.get_text(strip=True) if time_elem else ""
 
                 # Add leading zero if needed
-                if time_text and time_text.find('.') == 1:
-                    time_text = '0' + time_text
+                if time_text and time_text.find(".") == 1:
+                    time_text = "0" + time_text
 
                 # Convert time format
-                time_text = time_text.replace('.', ':')
+                time_text = time_text.replace(".", ":")
 
                 # Extract date
                 date_part = pieces[1].strip()
                 # Remove time from date_part if present
-                date_part = re.sub(r'\s+\d+[\.:]\d+.*$', '', date_part)
+                date_part = re.sub(r"\s+\d+[\.:]\d+.*$", "", date_part)
 
                 if time_text:
                     broadcast_item["date"] = f"{date_part}T{time_text}"

scitex/scholar/url_finder/translators/individual/biblioteca_nacional_de_maestros.py

@@ -34,7 +34,7 @@ class BibliotecaNacionalDeMaestrosTranslator:
         "inRepository": True,
         "translatorType": 4,
         "browserSupport": "gcsibv",
-        "lastUpdated": "2020-06-22 00:23:44"
+        "lastUpdated": "2020-06-22 00:23:44",
     }
 
     def detect_web(self, doc: BeautifulSoup, url: str) -> str:
@@ -48,13 +48,15 @@ class BibliotecaNacionalDeMaestrosTranslator:
         Returns:
             'book' for single record, 'multiple' for search results, empty string otherwise
         """
-        if re.search(r'/Record/\d+', url):
-            return 'book'
+        if re.search(r"/Record/\d+", url):
+            return "book"
         elif self._get_search_results(doc, check_only=True):
-            return 'multiple'
-        return ''
+            return "multiple"
+        return ""
 
-    def _get_search_results(self, doc: BeautifulSoup, check_only: bool = False) -> Dict[str, str]:
+    def _get_search_results(
+        self, doc: BeautifulSoup, check_only: bool = False
+    ) -> Dict[str, str]:
         """
         Get search results from the page.
 
@@ -66,15 +68,17 @@ class BibliotecaNacionalDeMaestrosTranslator:
             Dictionary mapping URLs to titles
         """
         items = {}
-        rows = doc.select('.result div[class*="resultItemLine"]>a.title[href*="/Record/"]')
+        rows = doc.select(
+            '.result div[class*="resultItemLine"]>a.title[href*="/Record/"]'
+        )
 
         for row in rows:
-            href = row.get('href')
+            href = row.get("href")
             title = row.get_text(strip=True)
             if not href or not title:
                 continue
             if check_only:
-                return {'found': True}
+                return {"found": True}
             items[href] = title
 
         return items if items else {}
@@ -92,14 +96,14 @@ class BibliotecaNacionalDeMaestrosTranslator:
         """
         page_type = self.detect_web(doc, url)
 
-        if page_type == 'multiple':
+        if page_type == "multiple":
             # For multiple items, return list of URLs for processing
             items = self._get_search_results(doc, check_only=False)
-            return [{'url': url} for url in items.keys()]
+            return [{"url": url} for url in items.keys()]
         else:
             # For single item, construct MARC URL and return
             marc_url = self._construct_marc_url(url)
-            return [{'_marc_url': marc_url, 'url': url}]
+            return [{"_marc_url": marc_url, "url": url}]
 
     def _construct_marc_url(self, url: str) -> str:
         """
@@ -112,8 +116,8 @@ class BibliotecaNacionalDeMaestrosTranslator:
             MARC XML export URL
         """
         # Remove panels like /Details or /Holdings
-        url = re.sub(r'/(Details|Holdings)([#?].*)?', '', url)
-        return url + '/Export?style=MARCXML'
+        url = re.sub(r"/(Details|Holdings)([#?].*)?", "", url)
+        return url + "/Export?style=MARCXML"
 
     def scrape(self, marc_text: str, url: str) -> Dict[str, Any]:
         """
@@ -129,12 +133,12 @@ class BibliotecaNacionalDeMaestrosTranslator:
         # This would typically call the MARC XML translator
         # For now, return a minimal structure
         item = {
-            'itemType': 'book',
-            'libraryCatalog': 'Biblioteca Nacional de Maestros',
-            'url': url,
-            'creators': [],
-            'tags': [],
-            'attachments': []
+            "itemType": "book",
+            "libraryCatalog": "Biblioteca Nacional de Maestros",
+            "url": url,
+            "creators": [],
+            "tags": [],
+            "attachments": [],
         }
 
         return item