scitex 2.17.0__py3-none-any.whl → 2.17.3__py3-none-any.whl

scitex/scholar/url_finder/.tmp/open_url/_ResolverLinkFinder.py

@@ -0,0 +1,344 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-07-29 03:10:08 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/open_url/_ResolverLinkFinder.py
+# ----------------------------------------
+from __future__ import annotations
+
+import os
+
+__FILE__ = (
+    "./src/scitex/scholar/open_url/_ResolverLinkFinder.py"
+)
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+"""Robust resolver link finder using a prioritized, multi-layered approach.
+
+Priority order:
+1. Link Target (domain matching) - Most reliable
+2. Page Structure (CSS selectors) - Very reliable
+3. Text Patterns - Good fallback
+"""
+
+import re
+from typing import List, Optional
+from urllib.parse import urlparse
+
+from playwright.async_api import ElementHandle, Page
+
+from scitex import logging
+
+logger = logging.getLogger(__name__)
+
+
+class ResolverLinkFinder:
+    """Finds full-text links on resolver pages using multiple strategies."""
+
+    # DOI prefix to publisher domain mapping
+    DOI_TO_DOMAIN = {
+        "10.1038": [
+            "nature.com",
+            "springernature.com",
+        ],  # Nature Publishing Group
+        "10.1016": ["sciencedirect.com", "elsevier.com"],  # Elsevier
+        "10.1002": ["wiley.com", "onlinelibrary.wiley.com"],  # Wiley
+        "10.1007": ["springer.com", "link.springer.com"],  # Springer
+        "10.1126": ["science.org", "sciencemag.org"],  # Science/AAAS
+        "10.1021": ["acs.org", "pubs.acs.org"],  # ACS Publications
+        "10.1111": [
+            "wiley.com",
+            "onlinelibrary.wiley.com",
+        ],  # Wiley (alternative)
+        "10.1080": ["tandfonline.com"],  # Taylor & Francis
+        "10.1177": ["sagepub.com", "journals.sagepub.com"],  # SAGE
+        "10.1093": ["oup.com", "academic.oup.com"],  # Oxford
+        "10.1109": ["ieee.org", "ieeexplore.ieee.org"],  # IEEE
+        "10.1371": ["plos.org", "journals.plos.org"],  # PLOS
+        "10.1073": ["pnas.org"],  # PNAS
+        "10.1136": ["bmj.com"],  # BMJ
+        "10.3389": ["frontiersin.org"],  # Frontiers
+        "10.3390": ["mdpi.com"],  # MDPI
+    }
+
+    # Common resolver page structures
+    STRUCTURE_SELECTORS = [
+        # SFX (ExLibris) - used by many universities
+        "div#fulltext a",
+        "div.sfx-fulltext a",
+        "div.results-title > a",
+        "td.object-cell a",
+        ".getFullTxt a",
+        'div[id*="fulltext"] a',
+        'div[class*="fulltext"] a',
+        # SFX specific selectors for University of Melbourne
+        "a[title*='Wiley Online Library']",
+        "a[href*='wiley.com']",
+        "a[href*='onlinelibrary.wiley.com']",
+        ".sfx-target a",
+        ".target a",
+        "td a[href*='wiley']",
+        # Primo (ExLibris)
+        "prm-full-view-service-container a",
+        "span.availability-status-available a",
+        # Summon (ProQuest)
+        ".summon-fulltext-link",
+        "a.summon-link",
+        # EDS (EBSCO)
+        "a.fulltext-link",
+        ".ft-link a",
+        # Generic patterns
+        "a.full-text-link",
+        "a.fulltext",
+        "a#full-text-link",
+        ".access-link a",
+        ".available-link a",
+    ]
+
+    # Text patterns in priority order
+    TEXT_PATTERNS = [
+        # Most specific
+        "View full text at",
+        "Available from Nature",
+        "Available from ScienceDirect",
+        "Available from Wiley",
+        "Available from Wiley Online Library",
+        "Full text available from",
+        # Common patterns
+        "View full text",
+        "Full Text from Publisher",
+        "Get full text",
+        "Access full text",
+        "Go to article",
+        "Access article",
+        # Generic but reliable
+        "Full Text",
+        "Full text",
+        "Article",
+        "View",
+        "PDF",
+        "Download",
+    ]
+
+    def __init__(self):
+        self._doi_pattern = re.compile(r"10\.\d{4,}/[-._;()/:\w]+")
+
+    def get_expected_domains(self, doi: str) -> List[str]:
+        """Get expected publisher domains for a DOI."""
+        # Extract DOI prefix
+        match = re.match(r"(10\.\d{4,})", doi)
+        if not match:
+            return []
+
+        prefix = match.group(1)
+        return self.DOI_TO_DOMAIN.get(prefix, [])
+
+    async def find_link_async(self, page, doi: str) -> dict:
+        """Find the best full-text link using prioritized strategies."""
+        logger.info(f"Finding resolver link for DOI: {doi}")
+
+        # Strategy 1: Link Target (Most Reliable)
+        link_url = await self._find_by_domain_async(page, doi)
+        if link_url:
+            logger.info("✓ Found link using domain matching (Strategy 1)")
+            return {"success": True, "url": link_url, "method": "domain"}
+
+        # Strategy 2: Page Structure with scoring
+        link_url = await self._find_by_structure_async(page, doi)
+        if link_url:
+            logger.info("✓ Found link using page structure (Strategy 2)")
+            return {"success": True, "url": link_url, "method": "structure"}
+
+        logger.warning("✗ No suitable links found")
+        return {"success": False, "url": None, "method": None}
+
+    async def _find_by_domain_async(self, page: Page, doi: str) -> Optional[str]:
+        """Strategy 1: Find link by expected publisher domain."""
+        expected_domains = self.get_expected_domains(doi)
+        if not expected_domains:
+            logger.debug(f"No known publisher domains for DOI prefix: {doi}")
+            return None
+
+        logger.debug(f"Looking for links to domains: {expected_domains}")
+        all_links = await page.query_selector_all("a[href]")
+
+        for link in all_links:
+            href = await link.get_attribute("href")
+            if not href:
+                continue
+
+            try:
+                parsed = urlparse(href)
+                domain = parsed.netloc.lower()
+
+                for expected in expected_domains:
+                    if expected in domain:
+                        text = await link.inner_text() or ""
+                        logger.info(
+                            f"Found domain match: {domain} (text: '{text[:50]}')"
+                        )
+
+                        if not any(
+                            bad in text.lower()
+                            for bad in ["abstract", "preview", "summary"]
+                        ):
+                            return href
+                        else:
+                            logger.debug(
+                                f"Skipping abstract/preview link: {text}"
+                            )
+            except Exception as e:
+                logger.debug(f"Error parsing URL {href}: {e}")
+
+        return None
+
+    async def _find_by_structure_async(self, page, doi: str):
+        """Find link by page structure with publisher prioritization."""
+        potential_links = []
+        expected_domains = self.get_expected_domains(doi)
+        publisher_keywords = [
+            domain.split(".")[0] for domain in expected_domains
+        ]
+        aggregator_keywords = ["gale", "proquest", "ebsco", "jstor", "onefile"]
+
+        # Gather all possible links
+        for selector in self.STRUCTURE_SELECTORS:
+            try:
+                elements = await page.query_selector_all(selector)
+                logger.debug(
+                    f"Found {len(elements)} elements with selector: {selector}"
+                )
+
+                for element in elements:
+                    if await element.is_visible():
+                        href = await element.get_attribute("href")
+                        text = (await element.inner_text() or "").lower()
+
+                        if href and href.strip():
+                            potential_links.append(
+                                {"href": href, "text": text, "score": 0}
+                            )
+            except Exception as element_error:
+                logger.debug(
+                    f"Error with selector '{selector}': {element_error}"
+                )
+
+        if not potential_links:
+            return None
+
+        # Score the links
+        for link in potential_links:
+            # Highest score for direct publisher match
+            if any(keyword in link["text"] for keyword in publisher_keywords):
+                link["score"] = 3
+            # High score for generic publisher
+            elif "publisher" in link["text"]:
+                link["score"] = 2
+            # Negative score for aggregators
+            elif any(
+                keyword in link["text"] for keyword in aggregator_keywords
+            ):
+                link["score"] = -1
+            # Default neutral score
+            else:
+                link["score"] = 0
+
+        # Sort by score, highest first
+        sorted_links = sorted(
+            potential_links, key=lambda x: x["score"], reverse=True
+        )
+        best_link = sorted_links[0]
+
+        logger.debug(
+            f"Found structural match: '{best_link['text'][:50]}' -> {best_link['href']}"
+        )
+        return best_link["href"]
+
+    async def _find_by_text_async(self, page: Page) -> Optional[str]:
+        """Strategy 3: Find link by text patterns."""
+        for pattern in self.TEXT_PATTERNS:
+            try:
+                selector = f'a:has-text("{pattern}")'
+                link = await page.query_selector(selector)
+                if link and await link.is_visible():
+                    href = await link.get_attribute("href")
+                    if href and href.strip():
+                        logger.debug(
+                            f"Found text match: '{pattern}' -> {href[:100]}"
+                        )
+                        return href
+            except Exception as e:
+                logger.debug(f"Error with text pattern '{pattern}': {e}")
+
+        return None
+
+    async def click_and_wait_async(self, page: Page, link: ElementHandle) -> bool:
+        """Click link and wait for navigation.
+
+        Returns True if navigation succeeded.
+        """
+        initial_url = page.url
+
+        try:
+            # Get link info for logging
+            href = await link.get_attribute("href") or ""
+            text = await link.inner_text() or ""
+            logger.info(f"Clicking link: '{text[:50]}' -> {href[:100]}")
+
+            # Click and wait for navigation
+            await link.click()
+
+            # Wait for either navigation or network idle
+            try:
+                await page.wait_for_load_state("networkidle", timeout=30000)
+            except:
+                # Fallback to domcontentloaded if network doesn't settle
+                await page.wait_for_load_state(
+                    "domcontentloaded", timeout=30000
+                )
+
+            # Additional wait for JavaScript redirects
+            await page.wait_for_timeout(3000)
+
+            # Check if we navigated
+            final_url = page.url
+            if final_url != initial_url:
+                logger.info(
+                    f"Successfully navigated: {initial_url} -> {final_url}"
+                )
+                return True
+            else:
+                logger.warning("No navigation occurred after click")
+                return False
+
+        except Exception as e:
+            logger.error(f"Error during click and navigation: {e}")
+            return False
+
+
+# Convenience function for integration
+async def find_and_click_resolver_link_async(page: Page, doi: str) -> Optional[str]:
+    """Find and click the best resolver link.
+
+    Args:
+        page: Playwright page object
+        doi: Target DOI
+
+    Returns:
+        Final URL after navigation, or None if failed
+    """
+    finder = ResolverLinkFinder()
+
+    # Find link
+    link = await finder.find_link_async(page, doi)
+    if not link:
+        return None
+
+    # Click and navigate
+    success = await finder.click_and_wait_async(page, link)
+    if success:
+        return page.url
+    else:
+        return None
+
+# EOF

scitex/scholar/url_finder/.tmp/open_url/__init__.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-07-31 00:53:24 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex_repo/src/scitex/scholar/open_url/__init__.py
+# ----------------------------------------
+from __future__ import annotations
+
+import os
+
+__FILE__ = (
+    "./src/scitex/scholar/open_url/__init__.py"
+)
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+from ._DOIToURLResolver import DOIToURLResolver
+from ._OpenURLResolver import OpenURLResolver
+
+__all__ = [
+    "OpenURLResolver",
+    "DOIToURLResolver",
+]
+
+# EOF

scitex/template/__init__.py

@@ -22,6 +22,7 @@ from .clone_pip_project import TEMPLATE_REPO_URL as PIP_PROJECT_URL
 from .clone_pip_project import clone_pip_project
 from .clone_research import TEMPLATE_REPO_URL as RESEARCH_URL
 from .clone_research import clone_research
+from .clone_research_minimal import clone_research_minimal
 from .clone_singularity import TEMPLATE_REPO_URL as SINGULARITY_URL
 from .clone_singularity import clone_singularity
 from .clone_writer_directory import (
@@ -54,9 +55,24 @@ def get_available_templates_info():
     ...     print(f"{template['name']}: {template['description']}")
     """
     return [
+        {
+            "id": "research_minimal",
+            "name": "Research Minimal",
+            "description": "Minimal SciTeX structure with writer, scholar, visualizer, and console",
+            "github_url": RESEARCH_URL,
+            "branch": "minimal",
+            "use_case": "Focused research workflow with essential SciTeX modules only",
+            "features": [
+                "scitex/writer/ - LaTeX manuscript writing",
+                "scitex/scholar/ - Bibliography management",
+                "scitex/visualizer/ - Figure creation",
+                "scitex/console/ - Code execution",
+                "scitex/management/ - Project management",
+            ],
+        },
         {
             "id": "research",
-            "name": "Research Project",
+            "name": "Research Project (Full)",
             "description": "Full scientific workflow structure for research projects",
             "github_url": RESEARCH_URL,
             "use_case": "Scientific research with data analysis, experiments, and paper writing",
@@ -113,6 +129,7 @@ def get_available_templates_info():
 
 __all__ = [
     "clone_research",
+    "clone_research_minimal",
     "clone_pip_project",
     "clone_singularity",
     "clone_writer_directory",

scitex/template/clone_research_minimal.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+# File: /home/ywatanabe/proj/scitex-code/src/scitex/template/clone_research_minimal.py
+# ----------------------------------------
+from __future__ import annotations
+
+import os
+
+__FILE__ = "./src/scitex/template/clone_research_minimal.py"
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+"""
+Create a new minimal research project from the scitex_template_research template.
+
+Uses the 'minimal' branch which contains only the scitex/ directory with:
+- writer/ - LaTeX manuscript writing
+- scholar/ - Bibliography management
+- visualizer/ - Figure creation
+- console/ - Code execution
+- management/ - Project management
+"""
+
+import sys
+from typing import Optional
+
+from ._clone_project import clone_project
+
+TEMPLATE_REPO_URL = "https://github.com/ywatanabe1989/scitex-minimal-template.git"
+
+
+def clone_research_minimal(
+    project_dir: str,
+    git_strategy: Optional[str] = "child",
+    branch: Optional[str] = None,
+    tag: Optional[str] = None,
+) -> bool:
+    """
+    Create a new minimal research project from the scitex-minimal-template.
+
+    This template contains only the essential scitex/ directory structure:
+    - writer/ - Full LaTeX manuscript writing with compilation scripts
+    - scholar/ - Bibliography management
+    - visualizer/ - Figure creation
+    - console/ - Code execution
+
+    Parameters
+    ----------
+    project_dir : str
+        Path to project directory (will be created). Can be a simple name like "my_project"
+        or a full path like "./projects/my_project"
+    git_strategy : str, optional
+        Git initialization strategy ('child', 'parent', None). Default is 'child'.
+    branch : str, optional
+        Specific branch of the template repository to clone.
+    tag : str, optional
+        Specific tag/release of the template repository to clone.
+
+    Returns
+    -------
+    bool
+        True if successful, False otherwise
+
+    Example
+    -------
+    >>> from scitex.template import clone_research_minimal
+    >>> clone_research_minimal("my_research_project")
+    >>> clone_research_minimal("./projects/my_project")
+    """
+    return clone_project(
+        project_dir,
+        TEMPLATE_REPO_URL,
+        "scitex-minimal-template",
+        git_strategy,
+        branch=branch,
+        tag=tag,
+    )
+
+
+def main(args: list = None) -> None:
+    """
+    Command-line interface for clone_research_minimal.
+
+    Parameters
+    ----------
+    args : list, optional
+        Command-line arguments. If None, uses sys.argv[1:]
+    """
+    if args is None:
+        args = sys.argv[1:]
+
+    if len(args) < 1:
+        print("Usage: python -m scitex clone_research_minimal <project-dir>")
+        print("")
+        print("Arguments:")
+        print("  project-dir   Path to project directory (will be created)")
+        print("                Can be a simple name like 'my_project' or a full path")
+        print("")
+        print("Example:")
+        print("  python -m scitex clone_research_minimal my_research_project")
+        sys.exit(1)
+
+    project_dir = args[0]
+
+    success = clone_research_minimal(project_dir)
+    sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+    main()
+
+# EOF

scitex/verify/README.md

@@ -34,7 +34,6 @@ scitex/verify/
 ├── _visualize.py        # Re-exports from _viz/
 └── _viz/
     ├── _mermaid.py      # Mermaid DAG generation
-    ├── _plotly.py       # Interactive Plotly DAGs
     ├── _json.py         # JSON DAG export
     ├── _format.py       # Terminal output formatting
     ├── _colors.py       # Color constants
@@ -199,17 +198,6 @@ render_dag("dag.mmd", target_file="output.csv")  # Mermaid code
 render_dag("dag.json", target_file="output.csv")  # Graph structure
 ```
 
-### Interactive Plotly
-
-```python
-from scitex.verify import generate_plotly_dag, render_plotly_dag
-
-fig = generate_plotly_dag(target_file="output.csv")
-fig.show()  # Opens browser
-
-render_plotly_dag("dag_plotly.html", target_file="output.csv")
-```
-
 ## Integration Hooks (`_integration.py`)
 
 Automatically called by `@stx.session` and `stx.io`:

scitex/verify/__init__.py

@@ -109,10 +109,8 @@ from ._visualize import (
     format_status,
     generate_html_dag,
     generate_mermaid_dag,
-    generate_plotly_dag,
     print_verification_summary,
     render_dag,
-    render_plotly_dag,
 )
 
 
@@ -191,9 +189,7 @@ __all__ = [
     "format_list",
     "generate_mermaid_dag",
     "generate_html_dag",
-    "generate_plotly_dag",
     "render_dag",
-    "render_plotly_dag",
     "print_verification_summary",
     # Convenience functions
     "list_runs",

scitex/verify/_visualize.py

@@ -22,10 +22,8 @@ from ._viz import (
     format_status,
     generate_html_dag,
     generate_mermaid_dag,
-    generate_plotly_dag,
     print_verification_summary,
     render_dag,
-    render_plotly_dag,
 )
 
 __all__ = [
@@ -38,9 +36,7 @@ __all__ = [
     "format_list",
     "generate_mermaid_dag",
     "generate_html_dag",
-    "generate_plotly_dag",
     "render_dag",
-    "render_plotly_dag",
     "print_verification_summary",
 ]
 

scitex/verify/_viz/__init__.py

@@ -7,7 +7,6 @@ Provides multiple visualization backends:
 - Terminal: Colored text output with status icons
 - Mermaid: Text-based diagrams for docs/GitHub
 - HTML: Interactive web visualization
-- Plotly: Interactive Python-based visualization (optional)
 """
 
 from ._colors import Colors, VerificationLevel
@@ -21,21 +20,6 @@ from ._format import (
 from ._mermaid import generate_html_dag, generate_mermaid_dag, render_dag
 from ._utils import print_verification_summary
 
-# Optional Plotly support
-try:
-    from ._plotly import generate_plotly_dag, render_plotly_dag
-
-    _HAS_PLOTLY = True
-except ImportError:
-    _HAS_PLOTLY = False
-
-    def generate_plotly_dag(*args, **kwargs):
-        raise ImportError("plotly required: pip install plotly")
-
-    def render_plotly_dag(*args, **kwargs):
-        raise ImportError("plotly required: pip install plotly")
-
-
 __all__ = [
     "Colors",
     "VerificationLevel",
@@ -48,8 +32,6 @@ __all__ = [
     "generate_html_dag",
     "render_dag",
     "print_verification_summary",
-    "generate_plotly_dag",
-    "render_plotly_dag",
 ]
 
 

{scitex-2.17.0.dist-info → scitex-2.17.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scitex
-Version: 2.17.0
+Version: 2.17.3
 Summary: A comprehensive Python library for scientific computing and data analysis
 Project-URL: Homepage, https://github.com/ywatanabe1989/scitex-python
 Project-URL: Documentation, https://scitex.readthedocs.io
@@ -442,6 +442,7 @@ Requires-Dist: matplotlib; extra == 'utils'
 Requires-Dist: natsort; extra == 'utils'
 Requires-Dist: tqdm; extra == 'utils'
 Requires-Dist: xarray; extra == 'utils'
+Provides-Extra: verify
 Provides-Extra: web
 Requires-Dist: aiohttp; extra == 'web'
 Requires-Dist: anthropic; extra == 'web'