scitex 2.14.0__py3-none-any.whl → 2.16.0__py3-none-any.whl

scitex/scholar/url_finder/translators/individual/sage_journals.py

@@ -1,183 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""SAGE Journals translator.
-
-Based on SAGE Journals.js translator from Zotero.
-Original JavaScript implementation by Sebastian Karcher and Philipp Zumstein.
-
-Supports:
-- Journal articles (abs, full, pdf, doi views)
-- Search results pages
-- Table of contents pages
-- Automatic PDF URL construction from DOI
-
-SAGE uses Atypon platform but has distinct features that warrant a separate translator.
-"""
-
-import re
-from typing import List
-from urllib.parse import quote
-from playwright.async_api import Page
-from ..core.base import BaseTranslator
-
-
-class SAGEJournalsTranslator(BaseTranslator):
-    """SAGE Journals translator.
-
-    Handles journals.sagepub.com content including:
-    - Journal articles in various views (abs, full, pdf)
-    - DOI-based URLs without view prefix
-    - Search results pages
-    - Journal issue/volume table of contents
-
-    Based on JavaScript translator (SAGE Journals.js, lines 1-427).
-    """
-
-    LABEL = "SAGE Journals"
-    URL_TARGET_PATTERN = r"^https?://journals\.sagepub\.com(/doi/((abs|full|pdf)/)?10\.|/action/doSearch\?|/toc/)"
-
-    @classmethod
-    def matches_url(cls, url: str) -> bool:
-        """Check if URL matches SAGE Journals pattern.
-
-        Based on JavaScript detectWeb() function (lines 40-49).
-
-        The JavaScript implementation checks for:
-        1. Article URLs matching /(abs|full|pdf|doi)\/10\./
-        2. Search results with getSearchResults()
-
-        Args:
-            url: URL to check
-
-        Returns:
-            True if URL matches SAGE Journals pattern
-        """
-        return bool(re.match(cls.URL_TARGET_PATTERN, url))
-
-    @classmethod
-    async def extract_pdf_urls_async(cls, page: Page) -> List[str]:
-        """Extract PDF URLs from SAGE Journals page.
-
-        Based on JavaScript scrape() function (lines 81-147).
-
-        The JavaScript implementation:
-        1. Extracts DOI from meta tag dc.Identifier or URL (lines 83-86)
-        2. Constructs PDF URL as /doi/pdf/{DOI} (line 88)
-        3. Downloads RIS citation for metadata (lines 82-92)
-        4. Parses RIS and complements with page metadata (lines 103-145)
-        5. Extracts title, subtitle, abstract from DOM (lines 110-123)
-        6. Adds PDF as attachment (lines 139-143)
-
-        Python implementation:
-        - Extracts DOI from meta tags or URL
-        - Constructs direct PDF URL pattern: /doi/pdf/{DOI}
-        - Returns PDF URL for download
-
-        Args:
-            page: Playwright page object on SAGE Journals
-
-        Returns:
-            List containing PDF URL if DOI found, empty list otherwise
-        """
-        pdf_urls = []
-
-        try:
-            # Method 1: Extract DOI from meta tag (JavaScript line 83)
-            # JavaScript: var doi = ZU.xpathText(doc, '//meta[@name="dc.Identifier" and @scheme="doi"]/@content');
-            doi = await page.locator(
-                'meta[name="dc.Identifier"][scheme="doi"]'
-            ).first.get_attribute("content", timeout=3000)
-
-            if doi:
-                # Construct PDF URL (JavaScript line 88)
-                # JavaScript: let pdfurl = "//" + doc.location.host + "/doi/pdf/" + doi;
-                base_url = await page.evaluate("window.location.origin")
-                pdf_url = f"{base_url}/doi/pdf/{doi}"
-                pdf_urls.append(pdf_url)
-                return pdf_urls
-
-        except Exception:
-            pass
-
-        # Method 2: Extract DOI from URL (JavaScript lines 84-86)
-        # JavaScript: doi = url.match(/10\.[^?#]+/)[0];
-        try:
-            current_url = page.url
-            doi_match = re.search(r"(10\.[^?#]+)", current_url)
-
-            if doi_match:
-                doi = doi_match.group(1)
-                base_url = await page.evaluate("window.location.origin")
-                pdf_url = f"{base_url}/doi/pdf/{doi}"
-                pdf_urls.append(pdf_url)
-                return pdf_urls
-
-        except Exception:
-            pass
-
-        # Method 3: Look for PDF link on page
-        # SAGE pages typically have a PDF download button
-        try:
-            # Common selectors for SAGE PDF download links
-            selectors = [
-                "a.show-pdf",
-                'a[href*="/doi/pdf/"]',
-                "a.pdf-download",
-                'a[title*="PDF"]',
-                '.epub-section__access-options a[href*="pdf"]',
-                "a.download-pdf-link",
-            ]
-
-            for selector in selectors:
-                try:
-                    pdf_link = await page.locator(selector).first.get_attribute(
-                        "href", timeout=1000
-                    )
-                    if pdf_link:
-                        # Make absolute URL if needed
-                        if pdf_link.startswith("/"):
-                            base_url = await page.evaluate("window.location.origin")
-                            pdf_link = f"{base_url}{pdf_link}"
-                        elif not pdf_link.startswith("http"):
-                            base_url = await page.evaluate("window.location.href")
-                            base_url = re.sub(r"/[^/]*$", "", base_url)
-                            pdf_link = f"{base_url}/{pdf_link}"
-
-                        pdf_urls.append(pdf_link)
-                        return pdf_urls
-                except Exception:
-                    continue
-
-        except Exception:
-            pass
-
-        # Method 4: Check if we're on a PDF page already
-        # If URL contains /doi/pdf/, it's already a PDF URL
-        try:
-            current_url = page.url
-            if "/doi/pdf/" in current_url:
-                pdf_urls.append(current_url)
-                return pdf_urls
-        except Exception:
-            pass
-
-        # Method 5: Try to construct from current page URL by changing view type
-        # Convert /doi/abs/ or /doi/full/ to /doi/pdf/
-        try:
-            current_url = page.url
-            if (
-                "/doi/abs/" in current_url
-                or "/doi/full/" in current_url
-                or re.search(r"/doi/10\.", current_url)
-            ):
-                # Replace view type with pdf
-                pdf_url = re.sub(r"/doi/(abs|full)/", "/doi/pdf/", current_url)
-                # If URL is /doi/10.xxx without view prefix, insert pdf
-                pdf_url = re.sub(r"/doi/(10\.)", r"/doi/pdf/\1", pdf_url)
-
-                pdf_urls.append(pdf_url)
-                return pdf_urls
-        except Exception:
-            pass
-
-        return pdf_urls

scitex/scholar/url_finder/translators/individual/sage_knowledge.py

@@ -1,23 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""SAGE Knowledge translator."""
-
-import re
-from typing import List
-from playwright.async_api import Page
-from ..core.base import BaseTranslator
-
-
-class SageKnowledgeTranslator(BaseTranslator):
-    """SAGE Knowledge."""
-
-    LABEL = "SAGE Knowledge"
-    URL_TARGET_PATTERN = r"^https?://sk\.sagepub\.com/"
-
-    @classmethod
-    def matches_url(cls, url: str) -> bool:
-        return bool(re.match(cls.URL_TARGET_PATTERN, url))
-
-    @classmethod
-    async def extract_pdf_urls_async(cls, page: Page) -> List[str]:
-        return []

scitex/scholar/url_finder/translators/individual/saildart.py

@@ -1,23 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""SAILDART translator."""
-
-import re
-from typing import List
-from playwright.async_api import Page
-from ..core.base import BaseTranslator
-
-
-class SaildartTranslator(BaseTranslator):
-    """SAILDART."""
-
-    LABEL = "SAILDART"
-    URL_TARGET_PATTERN = r"^https?://(www\.)?saildart\.org/"
-
-    @classmethod
-    def matches_url(cls, url: str) -> bool:
-        return bool(re.match(cls.URL_TARGET_PATTERN, url))
-
-    @classmethod
-    async def extract_pdf_urls_async(cls, page: Page) -> List[str]:
-        return []

scitex/scholar/url_finder/translators/individual/salt_research_archives.py

@@ -1,23 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""SALT Research Archives translator."""
-
-import re
-from typing import List
-from playwright.async_api import Page
-from ..core.base import BaseTranslator
-
-
-class SaltResearchArchivesTranslator(BaseTranslator):
-    """SALT Research Archives."""
-
-    LABEL = "SALT Research Archives"
-    URL_TARGET_PATTERN = r"^https?://archives\.saltresearch\.org/"
-
-    @classmethod
-    def matches_url(cls, url: str) -> bool:
-        return bool(re.match(cls.URL_TARGET_PATTERN, url))
-
-    @classmethod
-    async def extract_pdf_urls_async(cls, page: Page) -> List[str]:
-        return []

scitex/scholar/url_finder/translators/individual/sbn_it.py

@@ -1,23 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""sbn.it translator."""
-
-import re
-from typing import List
-from playwright.async_api import Page
-from ..core.base import BaseTranslator
-
-
-class SbnItTranslator(BaseTranslator):
-    """sbn.it."""
-
-    LABEL = "sbn.it"
-    URL_TARGET_PATTERN = r"^https?://(www|opac)\.sbn\.it/opacsbn/opaclib"
-
-    @classmethod
-    def matches_url(cls, url: str) -> bool:
-        return bool(re.match(cls.URL_TARGET_PATTERN, url))
-
-    @classmethod
-    async def extract_pdf_urls_async(cls, page: Page) -> List[str]:
-        return []

scitex/scholar/url_finder/translators/individual/scholars_portal_journals.py

@@ -1,23 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""Scholars Portal Journals translator."""
-
-import re
-from typing import List
-from playwright.async_api import Page
-from ..core.base import BaseTranslator
-
-
-class ScholarsPortalJournalsTranslator(BaseTranslator):
-    """Scholars Portal Journals."""
-
-    LABEL = "Scholars Portal Journals"
-    URL_TARGET_PATTERN = r"^https?://journals\.scholarsportal\.info/"
-
-    @classmethod
-    def matches_url(cls, url: str) -> bool:
-        return bool(re.match(cls.URL_TARGET_PATTERN, url))
-
-    @classmethod
-    async def extract_pdf_urls_async(cls, page: Page) -> List[str]:
-        return []

scitex/scholar/url_finder/translators/individual/scholia.py

@@ -1,23 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""Scholia translator."""
-
-import re
-from typing import List
-from playwright.async_api import Page
-from ..core.base import BaseTranslator
-
-
-class ScholiaTranslator(BaseTranslator):
-    """Scholia."""
-
-    LABEL = "Scholia"
-    URL_TARGET_PATTERN = r"^https?://scholia\.toolforge\.org/work/Q"
-
-    @classmethod
-    def matches_url(cls, url: str) -> bool:
-        return bool(re.match(cls.URL_TARGET_PATTERN, url))
-
-    @classmethod
-    async def extract_pdf_urls_async(cls, page: Page) -> List[str]:
-        return []

scitex/scholar/url_finder/translators/individual/schweizer_radio_und_fernsehen_srf.py

@@ -1,23 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""Schweizer Radio und Fernsehen SRF translator."""
-
-import re
-from typing import List
-from playwright.async_api import Page
-from ..core.base import BaseTranslator
-
-
-class SchweizerRadioUndFernsehenSrfTranslator(BaseTranslator):
-    """Schweizer Radio und Fernsehen SRF."""
-
-    LABEL = "Schweizer Radio und Fernsehen SRF"
-    URL_TARGET_PATTERN = r"^https?://(www\.)?srf\.ch/sendungen/"
-
-    @classmethod
-    def matches_url(cls, url: str) -> bool:
-        return bool(re.match(cls.URL_TARGET_PATTERN, url))
-
-    @classmethod
-    async def extract_pdf_urls_async(cls, page: Page) -> List[str]:
-        return []

scitex/scholar/url_finder/translators/individual/scielo.py

@@ -1,23 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""SciELO translator."""
-
-import re
-from typing import List
-from playwright.async_api import Page
-from ..core.base import BaseTranslator
-
-
-class SciELOTranslator(BaseTranslator):
-    """SciELO."""
-
-    LABEL = "SciELO"
-    URL_TARGET_PATTERN = r"^https?://(www\.)?(socialscience\.|proceedings\.|biodiversidade\.|caribbean\.|comciencia\.|inovacao\.|search\.)?(scielo|scielosp)\."
-
-    @classmethod
-    def matches_url(cls, url: str) -> bool:
-        return bool(re.match(cls.URL_TARGET_PATTERN, url))
-
-    @classmethod
-    async def extract_pdf_urls_async(cls, page: Page) -> List[str]:
-        return []

scitex/scholar/url_finder/translators/individual/sciencedirect.py

@@ -1,132 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""ScienceDirect translator."""
-
-import re
-from typing import List
-from playwright.async_api import Page
-from ..core.base import BaseTranslator
-
-
-class ScienceDirectTranslator(BaseTranslator):
-    """ScienceDirect."""
-
-    LABEL = "ScienceDirect"
-    URL_TARGET_PATTERN = r"^https?://[^/]*science-?direct\.com[^/]*/((science/)?(article/|(journal|bookseries|book|handbook)/\d)|search[?/]|journal/[^/]+/vol)"
-
-    @classmethod
-    def matches_url(cls, url: str) -> bool:
-        return bool(re.match(cls.URL_TARGET_PATTERN, url))
-
-    @classmethod
-    async def extract_pdf_urls_async(cls, page: Page) -> List[str]:
-        """Extract PDF URLs from ScienceDirect article page.
-
-        Based on JavaScript translator getPDFLink() function (lines 67-173).
-        Implements all 5 methods matching JavaScript exactly:
-        1. Direct PDF link (#pdfLink or meta tag)
-        2. Embedded PDF object
-        3. JSON metadata construction
-        4. Canonical link + suffix
-        5. Refetch HTML for .pdf-download-btn-link (last resort)
-        """
-        # JS lines 68-72: Check if PDF is not available
-        try:
-            if await page.locator(".accessContent").first.is_visible(timeout=1000):
-                return []
-        except:
-            pass
-        try:
-            if await page.locator(".access-options-link-text").first.is_visible(
-                timeout=1000
-            ):
-                return []
-        except:
-            pass
-        try:
-            if await page.locator("#check-access-popover").first.is_visible(
-                timeout=1000
-            ):
-                return []
-        except:
-            pass
-
-        # Method 1: Direct PDF link (JS lines 75-76)
-        try:
-            pdf_url = await page.locator("#pdfLink").first.get_attribute(
-                "href", timeout=2000
-            )
-            if pdf_url and pdf_url != "#":
-                return [pdf_url]
-        except:
-            pass
-
-        # Method 1b: Meta tag citation_pdf_url (JS line 76)
-        try:
-            pdf_url = await page.locator(
-                '[name="citation_pdf_url"]'
-            ).first.get_attribute("content", timeout=2000)
-            if pdf_url:
-                return [pdf_url]
-        except:
-            pass
-
-        # Method 2: Embedded PDF object (JS lines 83-92)
-        try:
-            intermediate_url = await page.locator(
-                ".PdfEmbed > object"
-            ).first.get_attribute("data", timeout=2000)
-            if intermediate_url:
-                return [intermediate_url]
-        except:
-            pass
-
-        # Method 3: JSON metadata (JS lines 123-148)
-        try:
-            json_script = await page.locator(
-                'script[type="application/json"]'
-            ).first.text_content(timeout=2000)
-            if json_script:
-                import json
-
-                data = json.loads(json_script)
-
-                # Try to construct PDF URL from JSON
-                url_metadata = (
-                    data.get("article", {})
-                    .get("pdfDownload", {})
-                    .get("urlMetadata", {})
-                )
-                path = url_metadata.get("path")
-                pdf_extension = url_metadata.get("pdfExtension")
-                pii = url_metadata.get("pii")
-                query_params = url_metadata.get("queryParams", {})
-                md5 = query_params.get("md5")
-                pid = query_params.get("pid")
-
-                if all([path, pdf_extension, pii, md5, pid]):
-                    pdf_url = f"/{path}/{pii}{pdf_extension}?md5={md5}&pid={pid}"
-                    # Make absolute URL
-                    base_url = await page.evaluate("window.location.origin")
-                    pdf_url = f"{base_url}{pdf_url}"
-                    return [pdf_url]
-        except:
-            pass
-
-        # Method 4: Canonical link + suffix (JS lines 154-159)
-        try:
-            canonical_url = await page.locator(
-                'link[rel="canonical"]'
-            ).first.get_attribute("href", timeout=2000)
-            if canonical_url:
-                pdf_url = f"{canonical_url}/pdfft?download=true"
-                return [pdf_url]
-        except:
-            pass
-
-        # Method 5: Refetch HTML for .pdf-download-btn-link (JS lines 164-172)
-        # Note: This is a last resort method that requires refetching the page
-        # Skipping this in Python as it's expensive and rarely needed
-        # The JavaScript uses: let reloadedDoc = await requestDocument(url);
-
-        return []

scitex/scholar/url_finder/translators/individual/scopus.py

@@ -1,23 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""Scopus translator."""
-
-import re
-from typing import List
-from playwright.async_api import Page
-from ..core.base import BaseTranslator
-
-
-class ScopusTranslator(BaseTranslator):
-    """Scopus."""
-
-    LABEL = "Scopus"
-    URL_TARGET_PATTERN = r"^https?://www\.scopus\.com/"
-
-    @classmethod
-    def matches_url(cls, url: str) -> bool:
-        return bool(re.match(cls.URL_TARGET_PATTERN, url))
-
-    @classmethod
-    async def extract_pdf_urls_async(cls, page: Page) -> List[str]:
-        return []

scitex/scholar/url_finder/translators/individual/semantic_scholar.py

@@ -1,23 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""Semantic Scholar translator."""
-
-import re
-from typing import List
-from playwright.async_api import Page
-from ..core.base import BaseTranslator
-
-
-class SemanticScholarTranslator(BaseTranslator):
-    """Semantic Scholar."""
-
-    LABEL = "Semantic Scholar"
-    URL_TARGET_PATTERN = r"^https?://(www\.semanticscholar\.org/(paper/.+|search\?|reader/.+)|pdfs\.semanticscholar\.org/|api\.semanticscholar\.org/)"
-
-    @classmethod
-    def matches_url(cls, url: str) -> bool:
-        return bool(re.match(cls.URL_TARGET_PATTERN, url))
-
-    @classmethod
-    async def extract_pdf_urls_async(cls, page: Page) -> List[str]:
-        return []

scitex/scholar/url_finder/translators/individual/semantics_visual_library.py

@@ -1,25 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""semantics Visual Library translator."""
-
-import re
-from typing import List
-from playwright.async_api import Page
-from ..core.base import BaseTranslator
-
-
-class SemanticsVisualLibraryTranslator(BaseTranslator):
-    """semantics Visual Library."""
-
-    LABEL = "semantics Visual Library"
-    URL_TARGET_PATTERN = (
-        r"^https?://www\.(blldb-online\.de/blldb|bdsl-online\.de/BDSL-DB)/suche/"
-    )
-
-    @classmethod
-    def matches_url(cls, url: str) -> bool:
-        return bool(re.match(cls.URL_TARGET_PATTERN, url))
-
-    @classmethod
-    async def extract_pdf_urls_async(cls, page: Page) -> List[str]:
-        return []

scitex/scholar/url_finder/translators/individual/sfu_ipinch.py

@@ -1,23 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""SFU IPinCH translator."""
-
-import re
-from typing import List
-from playwright.async_api import Page
-from ..core.base import BaseTranslator
-
-
-class SfuIpinchTranslator(BaseTranslator):
-    """SFU IPinCH."""
-
-    LABEL = "SFU IPinCH"
-    URL_TARGET_PATTERN = r"^https?://[^/]*sfu\.ca/kbipinch/(records|browse|search)/"
-
-    @classmethod
-    def matches_url(cls, url: str) -> bool:
-        return bool(re.match(cls.URL_TARGET_PATTERN, url))
-
-    @classmethod
-    async def extract_pdf_urls_async(cls, page: Page) -> List[str]:
-        return []