scitex 2.1.2__py3-none-any.whl → 2.4.2__py3-none-any.whl

scitex/web/_scraping.py

@@ -0,0 +1,373 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# File: ./src/scitex/web/_scraping.py
+
+"""Web scraping utilities for extracting URLs and downloading images."""
+
+import os
+import re
+import urllib.parse
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional, Set, Tuple
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import requests
+from bs4 import BeautifulSoup
+from tqdm import tqdm
+
+try:
+    from PIL import Image
+    from io import BytesIO
+    PILLOW_AVAILABLE = True
+except ImportError:
+    PILLOW_AVAILABLE = False
+
+from scitex.logging import getLogger
+
+logger = getLogger(__name__)
+
+
+def _get_default_download_dir() -> str:
+    """Get default download directory using SCITEX_DIR if available."""
+    scitex_root = os.environ.get('SCITEX_DIR')
+    if scitex_root is None:
+        scitex_root = os.path.expanduser("~/.scitex")
+    return os.path.join(scitex_root, "web", "downloads")
+
+
+def get_urls(
+    url: str,
+    pattern: Optional[str] = None,
+    absolute: bool = True,
+    same_domain: bool = False,
+    include_external: bool = True,
+) -> List[str]:
+    """
+    Extract all URLs from a webpage.
+
+    Args:
+        url: The URL of the webpage to scrape
+        pattern: Optional regex pattern to filter URLs (e.g., r'\.pdf$' for PDF files)
+        absolute: If True, convert relative URLs to absolute URLs
+        same_domain: If True, only return URLs from the same domain
+        include_external: If True, include external links (only applies if same_domain=False)
+
+    Returns:
+        List of URLs found on the page
+
+    Example:
+        >>> urls = get_urls('https://example.com', pattern=r'\.pdf$')
+        >>> urls = get_urls('https://example.com', same_domain=True)
+    """
+    try:
+        logger.info(f"Fetching URLs from: {url}")
+        response = requests.get(url, timeout=30)
+        response.raise_for_status()
+    except requests.RequestException as e:
+        logger.error(f"Failed to fetch URL {url}: {e}")
+        return []
+
+    soup = BeautifulSoup(response.text, 'html.parser')
+    urls_found: Set[str] = set()
+
+    # Parse the base domain
+    parsed_base = urllib.parse.urlparse(url)
+    base_domain = f"{parsed_base.scheme}://{parsed_base.netloc}"
+
+    # Find all links
+    for link in soup.find_all('a', href=True):
+        href = link['href']
+
+        # Convert to absolute URL if requested
+        if absolute:
+            href = urllib.parse.urljoin(url, href)
+
+        # Filter by domain if requested
+        if same_domain:
+            parsed_href = urllib.parse.urlparse(href)
+            if parsed_href.netloc != parsed_base.netloc:
+                continue
+        elif not include_external:
+            parsed_href = urllib.parse.urlparse(href)
+            if parsed_href.netloc and parsed_href.netloc != parsed_base.netloc:
+                continue
+
+        # Filter by pattern if provided
+        if pattern:
+            if not re.search(pattern, href):
+                continue
+
+        urls_found.add(href)
+
+    result = sorted(list(urls_found))
+    logger.info(f"Found {len(result)} URLs")
+    return result
+
+
+def download_images(
+    url: str,
+    output_dir: Optional[str] = None,
+    pattern: Optional[str] = None,
+    min_size: Optional[Tuple[int, int]] = None,
+    max_workers: int = 5,
+    same_domain: bool = False,
+) -> List[str]:
+    """
+    Download all images from a webpage.
+
+    Args:
+        url: The URL of the webpage to scrape
+        output_dir: Directory to save images. Priority:
+                   1. This parameter if specified
+                   2. $SCITEX_WEB_DOWNLOADS_DIR environment variable
+                   3. $SCITEX_DIR/web/downloads (default)
+        pattern: Optional regex pattern to filter image URLs
+        min_size: Optional minimum size as (width, height) tuple to filter images
+        max_workers: Number of concurrent download threads
+        same_domain: If True, only download images from the same domain
+
+    Returns:
+        List of paths to downloaded images
+
+    Note:
+        - SVG files are automatically skipped (vector graphics)
+        - Images are saved in timestamped subdirectories: images-YYYYMMDD_HHMMSS/
+
+    Example:
+        >>> paths = download_images('https://example.com', output_dir='./downloads')
+        >>> paths = download_images('https://example.com', min_size=(100, 100))
+        >>> # Uses $SCITEX_WEB_DOWNLOADS_DIR or $SCITEX_DIR/web/downloads
+        >>> paths = download_images('https://example.com')
+    """
+    if not PILLOW_AVAILABLE:
+        logger.warning("Pillow is not available. min_size filtering will be disabled.")
+
+    # Set default output directory
+    if output_dir is None:
+        # Check SCITEX_WEB_DOWNLOADS_DIR first
+        output_dir = os.environ.get('SCITEX_WEB_DOWNLOADS_DIR')
+        if output_dir is None:
+            # Fall back to SCITEX_DIR/web/downloads
+            output_dir = _get_default_download_dir()
+
+    # Create timestamped subdirectory
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    output_path = Path(output_dir).expanduser() / f"images-{timestamp}"
+    output_path.mkdir(parents=True, exist_ok=True)
+
+    logger.info(f"Saving images to: {output_path}")
+
+    try:
+        logger.info(f"Fetching page: {url}")
+        response = requests.get(url, timeout=30)
+        response.raise_for_status()
+    except requests.RequestException as e:
+        logger.error(f"Failed to fetch URL {url}: {e}")
+        return []
+
+    soup = BeautifulSoup(response.text, 'html.parser')
+    image_urls: Set[str] = set()
+
+    # Parse the base domain
+    parsed_base = urllib.parse.urlparse(url)
+
+    # Find all image tags
+    for img in soup.find_all('img', src=True):
+        img_url = img['src']
+
+        # Convert to absolute URL
+        img_url = urllib.parse.urljoin(url, img_url)
+
+        # Skip SVG files (vector graphics, not raster images)
+        if img_url.lower().endswith(('.svg', '.svgz')):
+            continue
+
+        # Filter by domain if requested
+        if same_domain:
+            parsed_img = urllib.parse.urlparse(img_url)
+            if parsed_img.netloc != parsed_base.netloc:
+                continue
+
+        # Filter by pattern if provided
+        if pattern:
+            if not re.search(pattern, img_url):
+                continue
+
+        image_urls.add(img_url)
+
+    logger.info(f"Found {len(image_urls)} images")
+
+    # Download images
+    downloaded_paths = []
+
+    def download_image(img_url: str) -> Optional[str]:
+        try:
+            img_response = requests.get(img_url, timeout=30)
+            img_response.raise_for_status()
+
+            # Check image size if requested and Pillow is available
+            if min_size and PILLOW_AVAILABLE:
+                try:
+                    img = Image.open(BytesIO(img_response.content))
+                    if img.size[0] < min_size[0] or img.size[1] < min_size[1]:
+                        return None
+                except Exception:
+                    pass
+
+            # Generate filename from URL
+            parsed_url = urllib.parse.urlparse(img_url)
+            filename = Path(parsed_url.path).name
+
+            # If filename is empty or doesn't have extension, generate one
+            if not filename or '.' not in filename:
+                ext = '.jpg'  # default extension
+                if 'content-type' in img_response.headers:
+                    content_type = img_response.headers['content-type']
+                    if 'png' in content_type:
+                        ext = '.png'
+                    elif 'gif' in content_type:
+                        ext = '.gif'
+                    elif 'webp' in content_type:
+                        ext = '.webp'
+                filename = f"image_{hash(img_url)}{ext}"
+
+            # Save image
+            file_path = output_path / filename
+
+            # Handle duplicate filenames
+            counter = 1
+            original_stem = file_path.stem
+            while file_path.exists():
+                file_path = output_path / f"{original_stem}_{counter}{file_path.suffix}"
+                counter += 1
+
+            with open(file_path, 'wb') as f:
+                f.write(img_response.content)
+
+            return str(file_path)
+
+        except Exception as e:
+            logger.warning(f"Failed to download image {img_url}: {e}")
+            return None
+
+    # Download images concurrently
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        future_to_url = {executor.submit(download_image, img_url): img_url
+                         for img_url in image_urls}
+
+        for future in tqdm(as_completed(future_to_url),
+                          total=len(image_urls),
+                          desc="Downloading images"):
+            result = future.result()
+            if result:
+                downloaded_paths.append(result)
+
+    logger.info(f"Downloaded {len(downloaded_paths)} images to {output_dir}")
+    return downloaded_paths
+
+
+def get_image_urls(
+    url: str,
+    pattern: Optional[str] = None,
+    same_domain: bool = False,
+) -> List[str]:
+    """
+    Extract all image URLs from a webpage without downloading them.
+
+    Args:
+        url: The URL of the webpage to scrape
+        pattern: Optional regex pattern to filter image URLs
+        same_domain: If True, only return images from the same domain
+
+    Returns:
+        List of image URLs found on the page
+
+    Note:
+        - SVG files are automatically skipped (vector graphics)
+
+    Example:
+        >>> img_urls = get_image_urls('https://example.com')
+        >>> img_urls = get_image_urls('https://example.com', pattern=r'\.png$')
+    """
+    try:
+        logger.info(f"Fetching image URLs from: {url}")
+        response = requests.get(url, timeout=30)
+        response.raise_for_status()
+    except requests.RequestException as e:
+        logger.error(f"Failed to fetch URL {url}: {e}")
+        return []
+
+    soup = BeautifulSoup(response.text, 'html.parser')
+    image_urls: Set[str] = set()
+
+    # Parse the base domain
+    parsed_base = urllib.parse.urlparse(url)
+
+    # Find all image tags
+    for img in soup.find_all('img', src=True):
+        img_url = img['src']
+
+        # Convert to absolute URL
+        img_url = urllib.parse.urljoin(url, img_url)
+
+        # Skip SVG files (vector graphics, not raster images)
+        if img_url.lower().endswith(('.svg', '.svgz')):
+            continue
+
+        # Filter by domain if requested
+        if same_domain:
+            parsed_img = urllib.parse.urlparse(img_url)
+            if parsed_img.netloc != parsed_base.netloc:
+                continue
+
+        # Filter by pattern if provided
+        if pattern:
+            if not re.search(pattern, img_url):
+                continue
+
+        image_urls.add(img_url)
+
+    result = sorted(list(image_urls))
+    logger.info(f"Found {len(result)} image URLs")
+    return result
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Web scraping utilities")
+    parser.add_argument("url", type=str, help="URL to scrape")
+    parser.add_argument("--mode", "-m", choices=["urls", "images", "image_urls"],
+                       default="urls", help="Scraping mode")
+    parser.add_argument("--output", "-o", type=str, help="Output directory for images")
+    parser.add_argument("--pattern", "-p", type=str, help="Regex pattern to filter URLs")
+    parser.add_argument("--same-domain", action="store_true",
+                       help="Only include URLs from same domain")
+    parser.add_argument("--min-size", type=str, help="Minimum image size as WIDTHxHEIGHT")
+
+    args = parser.parse_args()
+
+    if args.mode == "urls":
+        urls = get_urls(args.url, pattern=args.pattern, same_domain=args.same_domain)
+        for url in urls:
+            print(url)
+    elif args.mode == "images":
+        min_size = None
+        if args.min_size:
+            width, height = map(int, args.min_size.split('x'))
+            min_size = (width, height)
+
+        paths = download_images(
+            args.url,
+            output_dir=args.output,
+            pattern=args.pattern,
+            min_size=min_size,
+            same_domain=args.same_domain,
+        )
+        for path in paths:
+            print(path)
+    elif args.mode == "image_urls":
+        img_urls = get_image_urls(args.url, pattern=args.pattern, same_domain=args.same_domain)
+        for img_url in img_urls:
+            print(img_url)

scitex/writer/README.md

@@ -19,7 +19,10 @@ writer = Writer(Path("my_paper"))
 
 # # Project-integrated manuscript (use parent's git repository)
 # writer = Writer(Path("my_project/scitex/writer"), git_strategy='parent')
-#  
+#
+# # Clone specific branch of template
+# writer = Writer(Path("my_paper"), branch="develop")
+#
 # # No git (temporary work)
 # writer = Writer(Path("temp_work"), git_strategy=None)
 
@@ -64,7 +67,13 @@ writer.delete()
 ### Writer Class
 
 ```python
-Writer(project_dir, name=None, git_strategy='child')
+Writer(project_dir, name=None, git_strategy='child', branch=None)
+
+# Parameters:
+# - project_dir: Path to project directory
+# - name: Project name (optional, defaults to directory name)
+# - git_strategy: 'child' (default), 'parent', 'origin', or None
+# - branch: Specific branch of template to clone (optional)
 
 # Attributes:
 writer.project_dir       # Path to project

scitex/writer/Writer.py

@@ -35,6 +35,7 @@ from .dataclasses import ManuscriptTree
 from .dataclasses import SupplementaryTree
 from .dataclasses import RevisionTree
 from .dataclasses.tree import ScriptsTree
+from .dataclasses.tree import SharedTree
 from scitex import logging
 from scitex.git import init_git_repo
 
@@ -49,6 +50,8 @@ class Writer:
         project_dir: Path,
         name: Optional[str] = None,
         git_strategy: Optional[str] = "child",
+        branch: Optional[str] = None,
+        tag: Optional[str] = None,
     ):
         """
         Initialize for project directory.
@@ -63,16 +66,27 @@ class Writer:
                 - 'parent': Use parent git repository
                 - 'origin': Preserve template's original git history
                 - None: Disable git initialization
+            branch: Specific branch of template repository to clone (optional)
+                If None, clones the default branch. Mutually exclusive with tag.
+            tag: Specific tag/release of template repository to clone (optional)
+                If None, clones the default branch. Mutually exclusive with branch.
         """
         self.project_name = name or Path(project_dir).name
         self.project_dir = Path(project_dir)
         self.git_strategy = git_strategy
-
+        self.branch = branch
+        self.tag = tag
+
+        ref_info = ""
+        if branch:
+            ref_info = f" (branch: {branch})"
+        elif tag:
+            ref_info = f" (tag: {tag})"
         logger.info(
             f"Writer: Initializing with:\n"
             f"    Project Name: {self.project_name}\n"
             f"    Project Directory: {self.project_dir.absolute()}\n"
-            f"    Git Strategy: {self.git_strategy}..."
+            f"    Git Strategy: {self.git_strategy}{ref_info}..."
         )
 
         # Create or attach to project
@@ -82,6 +96,9 @@ class Writer:
         self.git_root = init_git_repo(self.project_dir, self.git_strategy)
 
         # Document accessors (pass git_root for efficiency)
+        self.shared = SharedTree(
+            self.project_dir / "00_shared", git_root=self.git_root
+        )
         self.manuscript = ManuscriptTree(
             self.project_dir / "01_manuscript", git_root=self.git_root
         )
@@ -122,15 +139,14 @@ class Writer:
             return self.project_dir
 
         project_name = name or self.project_dir.name
-        target_dir = self.project_dir.parent
 
         logger.info(
-            f"Writer: Creating new project '{project_name}' in {target_dir}"
+            f"Writer: Creating new project '{project_name}' at {self.project_dir.absolute()}"
         )
 
         # Initialize project directory structure
         success = _clone_writer_project(
-            project_name, str(target_dir), self.git_strategy
+            str(self.project_dir), self.git_strategy, self.branch, self.tag
         )
 
         if not success:
@@ -192,14 +208,21 @@ class Writer:
             f"Writer: Project structure verified at {self.project_dir.absolute()}"
         )
 
-    def compile_manuscript(self, timeout: int = 300) -> CompilationResult:
+    def compile_manuscript(
+        self,
+        timeout: int = 300,
+        log_callback: Optional[Callable[[str], None]] = None,
+        progress_callback: Optional[Callable[[int, str], None]] = None,
+    ) -> CompilationResult:
         """
-        Compile manuscript to PDF.
+        Compile manuscript to PDF with optional live callbacks.
 
         Runs scripts/shell/compile_manuscript.sh with configured settings.
 
         Args:
             timeout: Maximum compilation time in seconds (default: 300)
+            log_callback: Called with each log line: log_callback("Running pdflatex...")
+            progress_callback: Called with progress: progress_callback(50, "Pass 2/3")
 
         Returns:
             CompilationResult with success status, PDF path, and errors/warnings
@@ -217,17 +240,39 @@ class Writer:
             >>> result = writer.compile_manuscript()
             >>> if result.success:
             ...     print(f"PDF created: {result.output_pdf}")
+
+            >>> # With callbacks for live updates
+            >>> def on_log(msg):
+            ...     append_to_job_log(job_id, msg)
+            >>> def on_progress(percent, step):
+            ...     update_job_progress(job_id, percent, step)
+            >>> result = writer.compile_manuscript(
+            ...     log_callback=on_log,
+            ...     progress_callback=on_progress
+            ... )
         """
-        return compile_manuscript(self.project_dir, timeout=timeout)
+        return compile_manuscript(
+            self.project_dir,
+            timeout=timeout,
+            log_callback=log_callback,
+            progress_callback=progress_callback,
+        )
 
-    def compile_supplementary(self, timeout: int = 300) -> CompilationResult:
+    def compile_supplementary(
+        self,
+        timeout: int = 300,
+        log_callback: Optional[Callable[[str], None]] = None,
+        progress_callback: Optional[Callable[[int, str], None]] = None,
+    ) -> CompilationResult:
         """
-        Compile supplementary materials to PDF.
+        Compile supplementary materials to PDF with optional live callbacks.
 
         Runs scripts/shell/compile_supplementary.sh with configured settings.
 
         Args:
             timeout: Maximum compilation time in seconds (default: 300)
+            log_callback: Called with each log line
+            progress_callback: Called with progress updates
 
         Returns:
             CompilationResult with success status, PDF path, and errors/warnings
@@ -244,19 +289,30 @@ class Writer:
             >>> if result.success:
             ...     print(f"PDF created: {result.output_pdf}")
         """
-        return compile_supplementary(self.project_dir, timeout=timeout)
+        return compile_supplementary(
+            self.project_dir,
+            timeout=timeout,
+            log_callback=log_callback,
+            progress_callback=progress_callback,
+        )
 
     def compile_revision(
-        self, track_changes: bool = False, timeout: int = 300
+        self,
+        track_changes: bool = False,
+        timeout: int = 300,
+        log_callback: Optional[Callable[[str], None]] = None,
+        progress_callback: Optional[Callable[[int, str], None]] = None,
     ) -> CompilationResult:
         """
-        Compile revision document with optional change tracking.
+        Compile revision document with optional change tracking and live callbacks.
 
         Runs scripts/shell/compile_revision.sh with configured settings.
 
         Args:
             track_changes: Enable change tracking in compiled PDF (default: False)
             timeout: Maximum compilation time in seconds (default: 300)
+            log_callback: Called with each log line
+            progress_callback: Called with progress updates
 
         Returns:
             CompilationResult with success status, PDF path, and errors/warnings
@@ -275,7 +331,11 @@ class Writer:
             ...     print(f"Revision PDF: {result.output_pdf}")
         """
         return compile_revision(
-            self.project_dir, track_changes=track_changes, timeout=timeout
+            self.project_dir,
+            track_changes=track_changes,
+            timeout=timeout,
+            log_callback=log_callback,
+            progress_callback=progress_callback,
         )
 
     def watch(self, on_compile: Optional[Callable] = None) -> None:
@@ -316,7 +376,7 @@ def run_session() -> None:
 
     args = parse_args()
 
-    CONFIG, sys.stdout, sys.stderr, plt, CC, rng = stx.session.start(
+    CONFIG, sys.stdout, sys.stderr, plt, CC, rng_manager = stx.session.start(
         sys,
         plt,
         args=args,