scitex 2.4.2__py3-none-any.whl → 2.4.3__py3-none-any.whl

scitex/browser/debugging/_visual_cursor.py

@@ -0,0 +1,432 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: 2025-12-08
+# File: /home/ywatanabe/proj/scitex-code/src/scitex/browser/debugging/_visual_cursor.py
+
+"""
+Visual cursor and click effects for E2E test feedback.
+
+Provides visual feedback during browser automation:
+- Visual cursor indicator that follows mouse movements
+- Click ripple effects
+- Drag state visualization
+- Step progress messages
+
+Works with both async and sync Playwright APIs.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Union
+
+if TYPE_CHECKING:
+    from playwright.async_api import Page as AsyncPage
+    from playwright.sync_api import Page as SyncPage
+
+# CSS styles for visual effects
+VISUAL_EFFECTS_CSS = """
+/* Visual cursor indicator */
+#_scitex_cursor {
+    position: fixed;
+    width: 24px;
+    height: 24px;
+    border: 3px solid #FF4444;
+    border-radius: 50%;
+    pointer-events: none;
+    z-index: 2147483647;
+    transform: translate(-50%, -50%);
+    transition: all 0.15s ease-out;
+    box-shadow: 0 0 15px rgba(255, 68, 68, 0.6);
+    display: none;
+}
+#_scitex_cursor.clicking {
+    transform: translate(-50%, -50%) scale(0.6);
+    background: rgba(255, 68, 68, 0.4);
+    box-shadow: 0 0 25px rgba(255, 68, 68, 0.8);
+}
+#_scitex_cursor.dragging {
+    border-color: #28A745;
+    box-shadow: 0 0 15px rgba(40, 167, 69, 0.6);
+    width: 28px;
+    height: 28px;
+}
+
+/* Click ripple effect */
+.scitex-click-ripple {
+    position: fixed;
+    border-radius: 50%;
+    border: 3px solid #FF4444;
+    pointer-events: none;
+    z-index: 2147483646;
+    animation: clickRipple 0.5s ease-out forwards;
+}
+@keyframes clickRipple {
+    0% { width: 0; height: 0; opacity: 1; transform: translate(-50%, -50%); }
+    100% { width: 80px; height: 80px; opacity: 0; transform: translate(-50%, -50%); }
+}
+
+/* Step message container */
+#_scitex_step_messages {
+    position: fixed;
+    top: 10px;
+    left: 10px;
+    z-index: 2147483647;
+    display: flex;
+    flex-direction: column;
+    gap: 8px;
+    max-width: 600px;
+    pointer-events: none;
+}
+.scitex-step-msg {
+    background: rgba(0, 0, 0, 0.9);
+    color: white;
+    padding: 14px 24px;
+    border-radius: 8px;
+    font-size: 16px;
+    font-family: 'Courier New', monospace;
+    box-shadow: 0 4px 12px rgba(0, 0, 0, 0.5);
+    word-wrap: break-word;
+    animation: stepSlideIn 0.3s ease-out;
+}
+@keyframes stepSlideIn {
+    0% { opacity: 0; transform: translateX(-20px); }
+    100% { opacity: 1; transform: translateX(0); }
+}
+
+/* Test result banner */
+#_scitex_result_banner {
+    position: fixed;
+    top: 50%;
+    left: 50%;
+    transform: translate(-50%, -50%);
+    padding: 40px 80px;
+    border-radius: 16px;
+    font-size: 48px;
+    font-weight: bold;
+    font-family: 'Arial', sans-serif;
+    z-index: 2147483647;
+    pointer-events: none;
+    animation: resultPulse 0.5s ease-out;
+}
+#_scitex_result_banner.success {
+    background: rgba(40, 167, 69, 0.95);
+    color: white;
+    box-shadow: 0 0 50px rgba(40, 167, 69, 0.8);
+}
+#_scitex_result_banner.failure {
+    background: rgba(220, 53, 69, 0.95);
+    color: white;
+    box-shadow: 0 0 50px rgba(220, 53, 69, 0.8);
+}
+@keyframes resultPulse {
+    0% { transform: translate(-50%, -50%) scale(0.5); opacity: 0; }
+    50% { transform: translate(-50%, -50%) scale(1.1); }
+    100% { transform: translate(-50%, -50%) scale(1); opacity: 1; }
+}
+"""
+
+# JavaScript to inject visual effects
+INJECT_EFFECTS_JS = f"""
+() => {{
+    if (document.getElementById('_scitex_visual_effects')) return;
+
+    const style = document.createElement('style');
+    style.id = '_scitex_visual_effects';
+    style.textContent = `{VISUAL_EFFECTS_CSS}`;
+    document.head.appendChild(style);
+
+    // Create cursor element
+    const cursor = document.createElement('div');
+    cursor.id = '_scitex_cursor';
+    document.body.appendChild(cursor);
+
+    // Create step messages container
+    const msgContainer = document.createElement('div');
+    msgContainer.id = '_scitex_step_messages';
+    document.body.appendChild(msgContainer);
+}}
+"""
+
+
+def inject_visual_effects(page: Union["AsyncPage", "SyncPage"]) -> None:
+    """Inject CSS and elements for visual effects (sync version)."""
+    page.evaluate(INJECT_EFFECTS_JS)
+
+
+async def inject_visual_effects_async(page: "AsyncPage") -> None:
+    """Inject CSS and elements for visual effects (async version)."""
+    await page.evaluate(INJECT_EFFECTS_JS)
+
+
+def show_cursor_at(
+    page: Union["AsyncPage", "SyncPage"],
+    x: float,
+    y: float,
+    state: str = "normal"
+) -> None:
+    """Move visual cursor to position (sync version).
+
+    Args:
+        page: Playwright page object
+        x: X coordinate
+        y: Y coordinate
+        state: Cursor state - "normal", "clicking", or "dragging"
+    """
+    page.evaluate("""
+    ([x, y, state]) => {
+        let cursor = document.getElementById('_scitex_cursor');
+        if (!cursor) {
+            cursor = document.createElement('div');
+            cursor.id = '_scitex_cursor';
+            document.body.appendChild(cursor);
+        }
+        cursor.style.display = 'block';
+        cursor.style.left = x + 'px';
+        cursor.style.top = y + 'px';
+        cursor.className = state === 'clicking' ? 'clicking' :
+                          state === 'dragging' ? 'dragging' : '';
+        if (state === 'dragging') {
+            cursor.style.borderColor = '#28A745';
+            cursor.style.boxShadow = '0 0 15px rgba(40, 167, 69, 0.6)';
+        } else {
+            cursor.style.borderColor = '#FF4444';
+            cursor.style.boxShadow = '0 0 15px rgba(255, 68, 68, 0.6)';
+        }
+    }
+    """, [x, y, state])
+
+
+async def show_cursor_at_async(
+    page: "AsyncPage",
+    x: float,
+    y: float,
+    state: str = "normal"
+) -> None:
+    """Move visual cursor to position (async version)."""
+    await page.evaluate("""
+    ([x, y, state]) => {
+        let cursor = document.getElementById('_scitex_cursor');
+        if (!cursor) {
+            cursor = document.createElement('div');
+            cursor.id = '_scitex_cursor';
+            document.body.appendChild(cursor);
+        }
+        cursor.style.display = 'block';
+        cursor.style.left = x + 'px';
+        cursor.style.top = y + 'px';
+        cursor.className = state === 'clicking' ? 'clicking' :
+                          state === 'dragging' ? 'dragging' : '';
+        if (state === 'dragging') {
+            cursor.style.borderColor = '#28A745';
+            cursor.style.boxShadow = '0 0 15px rgba(40, 167, 69, 0.6)';
+        } else {
+            cursor.style.borderColor = '#FF4444';
+            cursor.style.boxShadow = '0 0 15px rgba(255, 68, 68, 0.6)';
+        }
+    }
+    """, [x, y, state])
+
+
+def show_click_effect(page: Union["AsyncPage", "SyncPage"], x: float, y: float) -> None:
+    """Show click ripple effect at position (sync version)."""
+    page.evaluate("""
+    ([x, y]) => {
+        const ripple = document.createElement('div');
+        ripple.className = 'scitex-click-ripple';
+        ripple.style.left = x + 'px';
+        ripple.style.top = y + 'px';
+        document.body.appendChild(ripple);
+        setTimeout(() => ripple.remove(), 600);
+
+        const cursor = document.getElementById('_scitex_cursor');
+        if (cursor) {
+            cursor.classList.add('clicking');
+            setTimeout(() => cursor.classList.remove('clicking'), 150);
+        }
+    }
+    """, [x, y])
+
+
+async def show_click_effect_async(page: "AsyncPage", x: float, y: float) -> None:
+    """Show click ripple effect at position (async version)."""
+    await page.evaluate("""
+    ([x, y]) => {
+        const ripple = document.createElement('div');
+        ripple.className = 'scitex-click-ripple';
+        ripple.style.left = x + 'px';
+        ripple.style.top = y + 'px';
+        document.body.appendChild(ripple);
+        setTimeout(() => ripple.remove(), 600);
+
+        const cursor = document.getElementById('_scitex_cursor');
+        if (cursor) {
+            cursor.classList.add('clicking');
+            setTimeout(() => cursor.classList.remove('clicking'), 150);
+        }
+    }
+    """, [x, y])
+
+
+def show_step(
+    page: Union["AsyncPage", "SyncPage"],
+    step: int,
+    total: int,
+    message: str,
+    level: str = "info"
+) -> None:
+    """Show numbered step message in browser (sync version).
+
+    Args:
+        page: Playwright page object
+        step: Current step number
+        total: Total number of steps
+        message: Message to display
+        level: Message level - "info", "success", "warning", or "error"
+    """
+    color_map = {
+        "info": "#17A2B8",
+        "success": "#28A745",
+        "warning": "#FFC107",
+        "error": "#DC3545",
+    }
+    color = color_map.get(level, color_map["info"])
+
+    page.evaluate("""
+    ([step, total, message, color]) => {
+        let container = document.getElementById('_scitex_step_messages');
+        if (!container) {
+            container = document.createElement('div');
+            container.id = '_scitex_step_messages';
+            container.style.cssText = `
+                position: fixed; top: 10px; left: 10px; z-index: 2147483647;
+                display: flex; flex-direction: column; gap: 8px;
+                max-width: 600px; pointer-events: none;
+            `;
+            document.body.appendChild(container);
+        }
+        const popup = document.createElement('div');
+        popup.className = 'scitex-step-msg';
+        popup.innerHTML = `<strong>[${step}/${total}] ${message}</strong>`;
+        popup.style.cssText = `
+            background: rgba(0, 0, 0, 0.9); color: white;
+            padding: 14px 24px; border-radius: 8px; font-size: 16px;
+            font-family: 'Courier New', monospace;
+            box-shadow: 0 4px 12px rgba(0, 0, 0, 0.5);
+            border-left: 6px solid ${color}; word-wrap: break-word;
+        `;
+        container.appendChild(popup);
+        while (container.children.length > 5) container.removeChild(container.firstChild);
+        setTimeout(() => { if (popup.parentNode) popup.parentNode.removeChild(popup); }, 8000);
+    }
+    """, [step, total, message, color])
+    page.wait_for_timeout(200)
+
+
+async def show_step_async(
+    page: "AsyncPage",
+    step: int,
+    total: int,
+    message: str,
+    level: str = "info"
+) -> None:
+    """Show numbered step message in browser (async version)."""
+    color_map = {
+        "info": "#17A2B8",
+        "success": "#28A745",
+        "warning": "#FFC107",
+        "error": "#DC3545",
+    }
+    color = color_map.get(level, color_map["info"])
+
+    await page.evaluate("""
+    ([step, total, message, color]) => {
+        let container = document.getElementById('_scitex_step_messages');
+        if (!container) {
+            container = document.createElement('div');
+            container.id = '_scitex_step_messages';
+            container.style.cssText = `
+                position: fixed; top: 10px; left: 10px; z-index: 2147483647;
+                display: flex; flex-direction: column; gap: 8px;
+                max-width: 600px; pointer-events: none;
+            `;
+            document.body.appendChild(container);
+        }
+        const popup = document.createElement('div');
+        popup.className = 'scitex-step-msg';
+        popup.innerHTML = `<strong>[${step}/${total}] ${message}</strong>`;
+        popup.style.cssText = `
+            background: rgba(0, 0, 0, 0.9); color: white;
+            padding: 14px 24px; border-radius: 8px; font-size: 16px;
+            font-family: 'Courier New', monospace;
+            box-shadow: 0 4px 12px rgba(0, 0, 0, 0.5);
+            border-left: 6px solid ${color}; word-wrap: break-word;
+        `;
+        container.appendChild(popup);
+        while (container.children.length > 5) container.removeChild(container.firstChild);
+        setTimeout(() => { if (popup.parentNode) popup.parentNode.removeChild(popup); }, 8000);
+    }
+    """, [step, total, message, color])
+    await page.wait_for_timeout(200)
+
+
+def show_test_result(
+    page: Union["AsyncPage", "SyncPage"],
+    success: bool,
+    message: str = "",
+    delay_ms: int = 3000
+) -> None:
+    """Show test result banner (PASS/FAIL) and wait (sync version).
+
+    Args:
+        page: Playwright page object
+        success: True for PASS, False for FAIL
+        message: Optional message to display
+        delay_ms: How long to display before continuing
+    """
+    status = "PASS" if success else "FAIL"
+    css_class = "success" if success else "failure"
+    display_text = f"{status}" + (f": {message}" if message else "")
+
+    page.evaluate("""
+    ([displayText, cssClass]) => {
+        // Remove existing banner
+        const existing = document.getElementById('_scitex_result_banner');
+        if (existing) existing.remove();
+
+        const banner = document.createElement('div');
+        banner.id = '_scitex_result_banner';
+        banner.className = cssClass;
+        banner.textContent = displayText;
+        document.body.appendChild(banner);
+    }
+    """, [display_text, css_class])
+    page.wait_for_timeout(delay_ms)
+
+
+async def show_test_result_async(
+    page: "AsyncPage",
+    success: bool,
+    message: str = "",
+    delay_ms: int = 3000
+) -> None:
+    """Show test result banner (PASS/FAIL) and wait (async version)."""
+    status = "PASS" if success else "FAIL"
+    css_class = "success" if success else "failure"
+    display_text = f"{status}" + (f": {message}" if message else "")
+
+    await page.evaluate("""
+    ([displayText, cssClass]) => {
+        const existing = document.getElementById('_scitex_result_banner');
+        if (existing) existing.remove();
+
+        const banner = document.createElement('div');
+        banner.id = '_scitex_result_banner';
+        banner.className = cssClass;
+        banner.textContent = displayText;
+        document.body.appendChild(banner);
+    }
+    """, [display_text, css_class])
+    await page.wait_for_timeout(delay_ms)
+
+
+# EOF

scitex/scholar/citation_graph/database.py

@@ -44,9 +44,16 @@ class CitationDatabase:
             read_only: If True, open in read-only mode (default)
         """
         if read_only:
-            self.conn = sqlite3.connect(f"file:{self.db_path}?mode=ro", uri=True)
+            self.conn = sqlite3.connect(
+                f"file:{self.db_path}?mode=ro",
+                uri=True,
+                check_same_thread=False  # Allow multi-threaded access (e.g., Django)
+            )
         else:
-            self.conn = sqlite3.connect(self.db_path)
+            self.conn = sqlite3.connect(
+                self.db_path,
+                check_same_thread=False
+            )
 
         self.conn.row_factory = sqlite3.Row
 

scitex/scholar/config/ScholarConfig.py

@@ -29,8 +29,22 @@ logger = getLogger(__name__)
 
 
 class ScholarConfig:
-    def __init__(self, config_path: Optional[Union[str, Path]] = None):
+    def __init__(
+        self,
+        config_path: Optional[Union[str, Path]] = None,
+        scholar_dir: Optional[Union[str, Path]] = None,
+    ):
+        """Initialize ScholarConfig.
+
+        Args:
+            config_path: Path to custom config YAML file
+            scholar_dir: Direct path to scholar directory (e.g., /data/users/alice/.scitex)
+                        This bypasses SCITEX_DIR env var for thread-safe multi-user usage.
+                        Use this in Django/multi-user environments to avoid race conditions.
+        """
         self.name = self.__class__.__name__
+        self._explicit_scholar_dir = scholar_dir  # Store for thread-safe access
+
         if config_path and Path(config_path).exists():
             config_data = self.load_yaml(config_path)
         else:
@@ -114,8 +128,14 @@ class ScholarConfig:
 
     # Path Management ----------------------------------------
     def _setup_path_manager(self, scholar_dir=None):
-        scholar_dir = self.cascade.resolve("scholar_dir", default="~/.scitex")
-        base_path = Path(scholar_dir).expanduser() / "scholar"
+        # Priority: explicit parameter > env var > config > default
+        if self._explicit_scholar_dir:
+            # Use explicitly provided path (thread-safe for multi-user)
+            base_path = Path(self._explicit_scholar_dir).expanduser() / "scholar"
+        else:
+            # Fall back to cascade resolution (uses SCITEX_DIR env var)
+            scholar_dir = self.cascade.resolve("scholar_dir", default="~/.scitex")
+            base_path = Path(scholar_dir).expanduser() / "scholar"
         self.path_manager = PathManager(scholar_dir=base_path)
 
     @property

scitex/scholar/config/default.yaml

@@ -38,6 +38,61 @@ timeout: 60
 # Note: Download directory is fixed at {scholar_dir}/library/downloads/
 # Use get_library_downloads_dir() to access it
 
+# Open Access & Paywall Options
+# ----------------------------------------
+# Prefer Open Access versions when available
+prefer_open_access: ${SCITEX_SCHOLAR_PREFER_OPEN_ACCESS:-true}
+# Allow attempting paywalled journal downloads (requires authentication)
+# Only enable for local/personal use with valid institutional access
+enable_paywall_access: ${SCITEX_SCHOLAR_ENABLE_PAYWALL_ACCESS:-false}
+# Track paywall bypass attempts in metadata (for transparency)
+track_paywall_attempts: ${SCITEX_SCHOLAR_TRACK_PAYWALL_ATTEMPTS:-true}
+
+# Open Access Sources (repositories that are always free to download)
+OPENACCESS_SOURCES:
+  - arxiv
+  - pmc
+  - pubmed_central
+  - biorxiv
+  - medrxiv
+  - chemrxiv
+  - doaj
+  - plos
+  - peerj
+  - frontiers
+  - mdpi
+  - hindawi
+  - nature_communications  # Most articles are OA
+
+# Open Access Journals (partial list, DOAJ has full list)
+# These are journal name patterns (case-insensitive substring match)
+OPENACCESS_JOURNALS:
+  - "plos one"
+  - "plos biology"
+  - "plos medicine"
+  - "plos computational biology"
+  - "plos genetics"
+  - "plos pathogens"
+  - "plos neglected tropical diseases"
+  - "scientific reports"
+  - "nature communications"
+  - "elife"
+  - "peerj"
+  - "frontiers in"  # All Frontiers journals
+  - "bmc "          # All BMC journals
+  - "journal of open source software"
+  - "f1000research"
+  - "gigascience"
+  - "cell reports"
+  - "science advances"
+  - "iscience"
+  - "heliyon"
+  - "cureus"
+  - "jmir"          # Journal of Medical Internet Research
+
+# Unpaywall API email (required for polite access)
+unpaywall_email: ${SCITEX_SCHOLAR_UNPAYWALL_EMAIL:-"research@scitex.io"}
+
 # ----------------------------------------
 # cache
 # ----------------------------------------

scitex/scholar/core/Paper.py

@@ -284,6 +284,37 @@ class PathMetadata(BaseModel):
         validate_assignment = True  # Validate on attribute assignment too
 
 
+class AccessMetadata(BaseModel):
+    """Open access and licensing metadata with source tracking.
+
+    Tracks whether a paper is open access and provides URLs for OA versions.
+    Also includes license information when available.
+    """
+
+    is_open_access: Optional[bool] = None
+    is_open_access_engines: List[str] = Field(default_factory=list)
+
+    oa_status: Optional[str] = None  # gold, green, bronze, hybrid, closed
+    oa_status_engines: List[str] = Field(default_factory=list)
+
+    oa_url: Optional[str] = None  # URL to open access version
+    oa_url_engines: List[str] = Field(default_factory=list)
+
+    license: Optional[str] = None  # CC-BY, CC-BY-NC, etc.
+    license_engines: List[str] = Field(default_factory=list)
+
+    license_url: Optional[str] = None
+    license_url_engines: List[str] = Field(default_factory=list)
+
+    # For paywalled journals - opt-in for local/personal users
+    paywall_bypass_attempted: Optional[bool] = None
+    paywall_bypass_success: Optional[bool] = None
+
+    class Config:
+        populate_by_name = True
+        validate_assignment = True
+
+
 class SystemMetadata(BaseModel):
     """System tracking metadata (which engines were used to search)."""
 
@@ -313,6 +344,7 @@ class PaperMetadataStructure(BaseModel):
     )
     url: URLMetadata = Field(default_factory=URLMetadata)
     path: PathMetadata = Field(default_factory=PathMetadata)
+    access: AccessMetadata = Field(default_factory=AccessMetadata)
     system: SystemMetadata = Field(default_factory=SystemMetadata)
 
     class Config:
@@ -418,6 +450,7 @@ class PaperMetadataStructure(BaseModel):
             ),
             "url": self.url.model_dump(by_alias=True, **kwargs),
             "path": self.path.model_dump(by_alias=True, **kwargs),
+            "access": self.access.model_dump(by_alias=True, **kwargs),
             "system": self.system.model_dump(by_alias=True, **kwargs),
         }
 
@@ -489,6 +522,75 @@ class Paper(BaseModel):
         """
         return self.model_dump()
 
+    def detect_open_access(
+        self,
+        use_unpaywall: bool = False,
+        update_metadata: bool = True,
+    ) -> "OAResult":
+        """
+        Detect open access status for this paper.
+
+        Uses identifiers (DOI, arXiv ID, PMCID) and known OA sources
+        to determine if the paper is freely available.
+
+        Args:
+            use_unpaywall: If True, query Unpaywall API for uncertain cases
+            update_metadata: If True, update self.metadata.access with results
+
+        Returns:
+            OAResult with detection results
+        """
+        from .open_access import check_oa_status, OAResult
+
+        result = check_oa_status(
+            doi=self.metadata.id.doi,
+            arxiv_id=self.metadata.id.arxiv_id,
+            pmcid=None,  # Not currently in IDMetadata
+            source=None,  # Source tracking not in Paper
+            journal=self.metadata.publication.journal,
+            is_open_access_flag=self.metadata.access.is_open_access,
+            use_unpaywall=use_unpaywall,
+        )
+
+        if update_metadata:
+            self.metadata.access.is_open_access = result.is_open_access
+            self.metadata.access.is_open_access_engines.append(
+                f"detect_oa:{result.source}"
+            )
+            if result.status:
+                self.metadata.access.oa_status = result.status.value
+                self.metadata.access.oa_status_engines.append(
+                    f"detect_oa:{result.source}"
+                )
+            if result.oa_url:
+                self.metadata.access.oa_url = result.oa_url
+                self.metadata.access.oa_url_engines.append(
+                    f"detect_oa:{result.source}"
+                )
+            if result.license:
+                self.metadata.access.license = result.license
+                self.metadata.access.license_engines.append(
+                    f"detect_oa:{result.source}"
+                )
+
+        return result
+
+    @property
+    def is_open_access(self) -> bool:
+        """Check if paper is open access (quick check without API calls)."""
+        if self.metadata.access.is_open_access is not None:
+            return self.metadata.access.is_open_access
+
+        # Quick detection from identifiers
+        from .open_access import detect_oa_from_identifiers
+
+        result = detect_oa_from_identifiers(
+            doi=self.metadata.id.doi,
+            arxiv_id=self.metadata.id.arxiv_id,
+            journal=self.metadata.publication.journal,
+        )
+        return result.is_open_access
+
 
 if __name__ == "__main__":
     import json