scitex 2.4.3__py3-none-any.whl → 2.7.0__py3-none-any.whl

scitex/scholar/url_finder/translators/individual/bibliotheque_archives_nationale_quebec_pistard.py

@@ -32,24 +32,24 @@ class BibliothequeArchivesNationaleQuebecPistardTranslator:
         "priority": 100,
         "inRepository": True,
         "translatorType": 4,
-        "lastUpdated": "2008-08-06 17:00:00"
+        "lastUpdated": "2008-08-06 17:00:00",
     }
 
     def detect_web(self, doc: BeautifulSoup, url: str) -> str:
         """Detect page type."""
-        title = doc.title.string if doc.title else ''
+        title = doc.title.string if doc.title else ""
 
-        if 'Liste détaillée des fonds' in title:
-            return 'multiple'
-        elif 'Description fonds' in title:
-            return 'book'
-        return ''
+        if "Liste détaillée des fonds" in title:
+            return "multiple"
+        elif "Description fonds" in title:
+            return "book"
+        return ""
 
     def do_web(self, doc: BeautifulSoup, url: str) -> List[Dict[str, Any]]:
         """Extract data from the page."""
         page_type = self.detect_web(doc, url)
 
-        if page_type == 'multiple':
+        if page_type == "multiple":
             return self._get_search_results(doc)
         else:
             return [self.scrape(doc, url)]
@@ -60,76 +60,75 @@ class BibliothequeArchivesNationaleQuebecPistardTranslator:
         links = doc.select('td:nth-of-type(2) a[href*="description_fonds"]')
 
         for link in links:
-            href = link.get('href')
+            href = link.get("href")
             title = link.get_text(strip=True)
             if href and title:
-                items.append({'url': href, 'title': title})
+                items.append({"url": href, "title": title})
 
         return items
 
     def scrape(self, doc: BeautifulSoup, url: str) -> Dict[str, Any]:
         """Scrape item data from the document."""
         item = {
-            'itemType': 'book',
-            'url': url,
-            'creators': [],
-            'tags': [],
-            'attachments': []
+            "itemType": "book",
+            "url": url,
+            "creators": [],
+            "tags": [],
+            "attachments": [],
         }
 
         # Extract data fields
         data_tags = {}
-        headers = doc.select('strong')
-        content_elem = doc.select_one('div#Content div table')
+        headers = doc.select("strong")
+        content_elem = doc.select_one("div#Content div table")
 
         if content_elem:
             contents = content_elem.get_text()
 
             # Process headers and associate with content
             for i, header in enumerate(headers):
-                field_title = header.get_text().strip().replace(' ', '')
+                field_title = header.get_text().strip().replace(" ", "")
                 # Extract corresponding content
                 # This is a simplified version
-                data_tags[field_title] = ''
+                data_tags[field_title] = ""
 
         # Extract title
-        if 'Titre,Dates,Quantité' in data_tags:
-            title_data = data_tags['Titre,Dates,Quantité']
-            lines = title_data.split('\n')
+        if "Titre,Dates,Quantité" in data_tags:
+            title_data = data_tags["Titre,Dates,Quantité"]
+            lines = title_data.split("\n")
             if lines:
-                item['title'] = lines[0].strip()
+                item["title"] = lines[0].strip()
 
                 # Extract creators
                 for line in lines:
-                    if line.strip().startswith('/ '):
+                    if line.strip().startswith("/ "):
                         author = line.strip()[2:]
-                        item['creators'].append({
-                            'lastName': author,
-                            'creatorType': 'creator'
-                        })
+                        item["creators"].append(
+                            {"lastName": author, "creatorType": "creator"}
+                        )
         else:
-            item['title'] = doc.title.string if doc.title else ''
+            item["title"] = doc.title.string if doc.title else ""
 
         # Extract other fields
-        if 'Languedesdocuments' in data_tags:
-            item['language'] = data_tags['Languedesdocuments']
+        if "Languedesdocuments" in data_tags:
+            item["language"] = data_tags["Languedesdocuments"]
 
-        if 'Cote:' in data_tags:
-            item['callNumber'] = data_tags['Cote:']
+        if "Cote:" in data_tags:
+            item["callNumber"] = data_tags["Cote:"]
 
-        if 'Collation' in data_tags:
-            item['pages'] = data_tags['Collation']
+        if "Collation" in data_tags:
+            item["pages"] = data_tags["Collation"]
 
-        if 'Centre:' in data_tags:
-            item['place'] = data_tags['Centre:']
+        if "Centre:" in data_tags:
+            item["place"] = data_tags["Centre:"]
 
-        if 'Portéeetcontenu' in data_tags:
-            item['abstractNote'] = data_tags['Portéeetcontenu']
+        if "Portéeetcontenu" in data_tags:
+            item["abstractNote"] = data_tags["Portéeetcontenu"]
 
         # Extract tags
-        if 'Termesrattachés' in data_tags:
-            tags_text = data_tags['Termesrattachés']
-            tag_list = [t.strip() for t in tags_text.split('\n') if t.strip()]
-            item['tags'] = [{'tag': t} for t in tag_list]
+        if "Termesrattachés" in data_tags:
+            tags_text = data_tags["Termesrattachés"]
+            tag_list = [t.strip() for t in tags_text.split("\n") if t.strip()]
+            item["tags"] = [{"tag": t} for t in tag_list]
 
         return item

scitex/scholar/url_finder/translators/individual/bibliotheque_archives_nationales_quebec.py

@@ -34,40 +34,40 @@ class BibliothequeArchivesNationalesQuebecTranslator:
         "inRepository": True,
         "translatorType": 4,
         "browserSupport": "gcsibv",
-        "lastUpdated": "2015-06-29 17:02:02"
+        "lastUpdated": "2015-06-29 17:02:02",
     }
 
     def detect_web(self, doc: BeautifulSoup, url: str) -> str:
         """Detect page type based on title and images."""
-        title = doc.title.string if doc.title else ''
+        title = doc.title.string if doc.title else ""
 
-        if 'Search' in title or 'Recherche' in title:
-            return 'multiple'
+        if "Search" in title or "Recherche" in title:
+            return "multiple"
 
         # Check for media type indicators
-        img = doc.select_one('td:nth-of-type(2) a img')
-        if img and img.get('src'):
-            src = img['src']
-            if 'book' in src or 'mmusic' in src or 'manalytic' in src:
-                return 'book'
-            elif 'msdisc' in src or 'msound' in src or 'mscas' in src:
-                return 'audioRecording'
-            elif 'mvdisc' in src:
-                return 'videoRecording'
-            elif 'mpaint' in src:
-                return 'artwork'
-            elif 'mserial' in src:
-                return 'report'
-            elif 'mcomponent' in src:
-                return 'newspaperArticle'
-
-        return ''
+        img = doc.select_one("td:nth-of-type(2) a img")
+        if img and img.get("src"):
+            src = img["src"]
+            if "book" in src or "mmusic" in src or "manalytic" in src:
+                return "book"
+            elif "msdisc" in src or "msound" in src or "mscas" in src:
+                return "audioRecording"
+            elif "mvdisc" in src:
+                return "videoRecording"
+            elif "mpaint" in src:
+                return "artwork"
+            elif "mserial" in src:
+                return "report"
+            elif "mcomponent" in src:
+                return "newspaperArticle"
+
+        return ""
 
     def do_web(self, doc: BeautifulSoup, url: str) -> List[Dict[str, Any]]:
         """Extract data from the page."""
         page_type = self.detect_web(doc, url)
 
-        if page_type == 'multiple':
+        if page_type == "multiple":
             return self._get_search_results(doc)
         else:
             return [self.scrape(doc, url, page_type)]
@@ -75,107 +75,113 @@ class BibliothequeArchivesNationalesQuebecTranslator:
     def _get_search_results(self, doc: BeautifulSoup) -> List[Dict[str, Any]]:
         """Get search results."""
         items = []
-        rows = doc.select('p table tbody tr td b')
-        links = doc.select('p table tbody tr td a[href] img')
+        rows = doc.select("p table tbody tr td b")
+        links = doc.select("p table tbody tr td a[href] img")
 
         for i, row in enumerate(rows[:10]):  # Limit to 10 results
             title = row.get_text(strip=True)
             if links and i < len(links):
-                href = links[i].find_parent('a').get('href')
+                href = links[i].find_parent("a").get("href")
                 if href and title:
-                    items.append({'url': href, 'title': title})
+                    items.append({"url": href, "title": title})
 
         return items
 
     def scrape(self, doc: BeautifulSoup, url: str, item_type: str) -> Dict[str, Any]:
         """Scrape item data from the document."""
         # Map item types
-        description_field = 'pages'
-        if item_type in ['audioRecording', 'videoRecording']:
-            description_field = 'runningTime'
-        elif item_type == 'artwork':
-            description_field = 'artworkSize'
+        description_field = "pages"
+        if item_type in ["audioRecording", "videoRecording"]:
+            description_field = "runningTime"
+        elif item_type == "artwork":
+            description_field = "artworkSize"
 
         item = {
-            'itemType': item_type if item_type else 'book',
-            'url': url,
-            'creators': [],
-            'tags': [],
-            'attachments': []
+            "itemType": item_type if item_type else "book",
+            "url": url,
+            "creators": [],
+            "tags": [],
+            "attachments": [],
         }
 
         # Determine language from images
-        lang_imgs = doc.select('td:nth-of-type(2) a img')
+        lang_imgs = doc.select("td:nth-of-type(2) a img")
         if len(lang_imgs) > 1:
-            lang_src = lang_imgs[1].get('src', '')
-            if 'lfre' in lang_src:
-                item['language'] = 'French'
-            elif 'leng' in lang_src:
-                item['language'] = 'English'
+            lang_src = lang_imgs[1].get("src", "")
+            if "lfre" in lang_src:
+                item["language"] = "French"
+            elif "leng" in lang_src:
+                item["language"] = "English"
 
         # Extract data fields
-        headings = doc.select('td table tbody tr td:nth-of-type(2) b')
-        contents = doc.select('td:nth-of-type(2) table tbody tr td table tbody tr td:nth-of-type(4)')
+        headings = doc.select("td table tbody tr td:nth-of-type(2) b")
+        contents = doc.select(
+            "td:nth-of-type(2) table tbody tr td table tbody tr td:nth-of-type(4)"
+        )
 
         for i, heading in enumerate(headings):
             if i >= len(contents):
                 break
 
-            field_title = heading.get_text().strip().replace(' ', '')
+            field_title = heading.get_text().strip().replace(" ", "")
             content = contents[i].get_text(strip=True)
 
             # Remove tags marked with [*]
-            if '[*]' in content and field_title not in ['Publisher', 'Éditeur']:
-                content = content[:content.find('[')].strip()
+            if "[*]" in content and field_title not in ["Publisher", "Éditeur"]:
+                content = content[: content.find("[")].strip()
 
             # Process authors
-            if field_title in ['Author', 'Auteur']:
-                if ',' in content:
-                    parts = content.split(',', 1)
-                    first_name = parts[1].strip() if len(parts) > 1 else ''
+            if field_title in ["Author", "Auteur"]:
+                if "," in content:
+                    parts = content.split(",", 1)
+                    first_name = parts[1].strip() if len(parts) > 1 else ""
                     last_name = parts[0].strip()
-                    item['creators'].append({
-                        'firstName': first_name,
-                        'lastName': last_name,
-                        'creatorType': 'author'
-                    })
+                    item["creators"].append(
+                        {
+                            "firstName": first_name,
+                            "lastName": last_name,
+                            "creatorType": "author",
+                        }
+                    )
 
             # Process publisher info
-            elif field_title in ['Publisher', 'Éditeur']:
-                if ':' in content:
-                    parts = content.split(':', 1)
-                    item['place'] = parts[0].strip().replace('[', '').replace(']', '')
+            elif field_title in ["Publisher", "Éditeur"]:
+                if ":" in content:
+                    parts = content.split(":", 1)
+                    item["place"] = parts[0].strip().replace("[", "").replace("]", "")
 
                     if len(parts) > 1:
-                        pub_parts = parts[1].split(',')
-                        item['publisher'] = pub_parts[0].strip().replace('[', '').replace(']', '')
+                        pub_parts = parts[1].split(",")
+                        item["publisher"] = (
+                            pub_parts[0].strip().replace("[", "").replace("]", "")
+                        )
 
                         # Extract date (last 4 characters)
-                        date_text = parts[1].strip().replace('[', '').replace(']', '')
+                        date_text = parts[1].strip().replace("[", "").replace("]", "")
                         if len(date_text) >= 4:
-                            item['date'] = date_text[-4:]
+                            item["date"] = date_text[-4:]
                 else:
-                    item['date'] = content
+                    item["date"] = content
 
             # Process subjects/tags
-            elif field_title in ['Subjects', 'Sujets']:
-                tags = content.split('\n')
-                item['tags'] = [{'tag': t.strip()} for t in tags if t.strip()]
+            elif field_title in ["Subjects", "Sujets"]:
+                tags = content.split("\n")
+                item["tags"] = [{"tag": t.strip()} for t in tags if t.strip()]
 
             # Process other fields
-            elif field_title == 'Description':
+            elif field_title == "Description":
                 item[description_field] = content
-            elif field_title == 'Title' or field_title == 'Titre':
-                item['title'] = content
-            elif field_title == 'ISBN':
-                item['ISBN'] = content
-            elif field_title in ['Series', 'Collection']:
-                item['series'] = content
-            elif field_title == 'Notes':
-                item['abstractNote'] = content
-            elif field_title in ['Numbering', 'Numérotation']:
-                item['reportNumber'] = content
-            elif field_title == 'Source':
-                item['extra'] = 'Source: ' + content
+            elif field_title == "Title" or field_title == "Titre":
+                item["title"] = content
+            elif field_title == "ISBN":
+                item["ISBN"] = content
+            elif field_title in ["Series", "Collection"]:
+                item["series"] = content
+            elif field_title == "Notes":
+                item["abstractNote"] = content
+            elif field_title in ["Numbering", "Numérotation"]:
+                item["reportNumber"] = content
+            elif field_title == "Source":
+                item["extra"] = "Source: " + content
 
         return item

scitex/scholar/url_finder/translators/individual/bibliotheque_nationale_france.py

@@ -34,7 +34,7 @@ class BibliothequeNationaleFranceTranslator:
         "inRepository": True,
         "translatorType": 4,
         "browserSupport": "gcsibv",
-        "lastUpdated": "2024-01-09 03:40:58"
+        "lastUpdated": "2024-01-09 03:40:58",
     }
 
     TYPE_MAPPING = {
@@ -51,49 +51,49 @@ class BibliothequeNationaleFranceTranslator:
         "modern manuscript or archive": "manuscript",
         "coin or medal": "document",
         "physical object": "document",
-        "three dimensional object": "document"
+        "three dimensional object": "document",
     }
 
     def detect_web(self, doc: BeautifulSoup, url: str) -> str:
         """Detect page type."""
-        result_regexp = re.compile(r'ark:/12148/cb[0-9]+', re.IGNORECASE)
+        result_regexp = re.compile(r"ark:/12148/cb[0-9]+", re.IGNORECASE)
 
         if result_regexp.search(url):
             # Single result - check item type
-            dc_type_tag = doc.find('meta', {'name': 'DC.type', 'lang': 'eng'})
+            dc_type_tag = doc.find("meta", {"name": "DC.type", "lang": "eng"})
             if dc_type_tag:
-                item_type = dc_type_tag.get('content', '')
-                return self.TYPE_MAPPING.get(item_type, 'document')
-            return 'document'
+                item_type = dc_type_tag.get("content", "")
+                return self.TYPE_MAPPING.get(item_type, "document")
+            return "document"
         elif self._get_results_table(doc):
-            return 'multiple'
+            return "multiple"
 
-        return ''
+        return ""
 
     def _get_results_table(self, doc: BeautifulSoup) -> bool:
         """Check if results table exists."""
-        results = doc.select('div.liste-notices')
+        results = doc.select("div.liste-notices")
         return len(results) > 0
 
     def _get_selected_items(self, doc: BeautifulSoup) -> Dict[str, str]:
         """Get selected items from search results."""
         items = {}
-        rows = doc.select('div.liste-notices div.notice-item div.notice-contenu')
+        rows = doc.select("div.liste-notices div.notice-item div.notice-contenu")
 
         for row in rows:
-            title_link = row.select_one('div.notice-synthese a')
+            title_link = row.select_one("div.notice-synthese a")
             if not title_link:
                 continue
 
-            href = title_link.get('href')
+            href = title_link.get("href")
             title = title_link.get_text(strip=True)
 
             # Add document year if available
-            year_elem = row.select_one('span.notice-ordre')
+            year_elem = row.select_one("span.notice-ordre")
             if year_elem:
                 year = year_elem.get_text(strip=True)
                 if len(year) == 6:
-                    title += ' / ' + year
+                    title += " / " + year
 
             if href and title:
                 items[href] = title
@@ -102,18 +102,18 @@ class BibliothequeNationaleFranceTranslator:
 
     def _reform_url(self, url: str) -> str:
         """Convert URL to UNIMARC format."""
-        url = re.sub(r'(^.*\/ark:\/12148\/cb[0-9]+[a-z]*)(.*$)', r'\1.unimarc', url)
+        url = re.sub(r"(^.*\/ark:\/12148\/cb[0-9]+[a-z]*)(.*$)", r"\1.unimarc", url)
         return url
 
     def do_web(self, doc: BeautifulSoup, url: str) -> List[Dict[str, Any]]:
         """Extract data from the page."""
         page_type = self.detect_web(doc, url)
 
-        if page_type == 'multiple':
+        if page_type == "multiple":
             items = self._get_selected_items(doc)
-            return [{'url': self._reform_url(u)} for u in items.keys()]
+            return [{"url": self._reform_url(u)} for u in items.keys()]
         else:
-            return [{'url': self._reform_url(url), '_unimarc': True}]
+            return [{"url": self._reform_url(url), "_unimarc": True}]
 
     def scrape(self, doc: BeautifulSoup, url: str) -> Dict[str, Any]:
         """
@@ -129,33 +129,35 @@ class BibliothequeNationaleFranceTranslator:
             Dictionary containing item metadata
         """
         item = {
-            'itemType': self.detect_web(doc, url) or 'document',
-            'libraryCatalog': 'BnF Catalogue général (http://catalogue.bnf.fr)',
-            'url': url,
-            'creators': [],
-            'tags': [],
-            'attachments': []
+            "itemType": self.detect_web(doc, url) or "document",
+            "libraryCatalog": "BnF Catalogue général (http://catalogue.bnf.fr)",
+            "url": url,
+            "creators": [],
+            "tags": [],
+            "attachments": [],
         }
 
         # Extract title from meta tags
-        title_tag = doc.find('meta', {'property': 'og:title'})
+        title_tag = doc.find("meta", {"property": "og:title"})
         if title_tag:
-            item['title'] = title_tag.get('content', '')
+            item["title"] = title_tag.get("content", "")
 
         # Add attachment for catalog record
-        ark_match = re.search(r'ark:/12148/cb[0-9]+[a-z]*', url)
+        ark_match = re.search(r"ark:/12148/cb[0-9]+[a-z]*", url)
         if ark_match:
-            permalink = 'http://catalogue.bnf.fr/' + ark_match.group()
-            item['attachments'].append({
-                'title': 'Lien vers la notice du catalogue',
-                'url': permalink,
-                'mimeType': 'text/html',
-                'snapshot': False
-            })
+            permalink = "http://catalogue.bnf.fr/" + ark_match.group()
+            item["attachments"].append(
+                {
+                    "title": "Lien vers la notice du catalogue",
+                    "url": permalink,
+                    "mimeType": "text/html",
+                    "snapshot": False,
+                }
+            )
 
         # Check for Gallica URL (digital version)
-        gallica_link = doc.find('a', href=re.compile(r'gallica\.bnf\.fr'))
+        gallica_link = doc.find("a", href=re.compile(r"gallica\.bnf\.fr"))
         if gallica_link:
-            item['url'] = gallica_link.get('href')
+            item["url"] = gallica_link.get("href")
 
         return item