tt-news-scraper 0.4.3__tar.gz

tt_news_scraper-0.4.3/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.3
+Name: tt-news-scraper
+Version: 0.4.3
+Summary: Trinidad and Tobago news scraper using Guardian & Express as sources
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: pytest>=9.2.0
+Requires-Dist: requests>=2.31.0
+Requires-Dist: responses>=0.26.3
+Requires-Dist: ruff>=0.8.0 ; extra == 'dev'
+Requires-Python: >=3.12
+Provides-Extra: dev

tt_news_scraper-0.4.3/pyproject.toml

@@ -0,0 +1,32 @@
+[project]
+name = "tt-news-scraper"
+version = "0.4.3"
+description = "Trinidad and Tobago news scraper using Guardian & Express as sources"
+requires-python = ">=3.12"
+dependencies = [
+    "beautifulsoup4>=4.12.0",
+    "pandas>=2.0.0",
+    "pytest>=9.2.0",
+    "requests>=2.31.0",
+    "responses>=0.26.3",
+]
+
+[project.optional-dependencies]
+dev = ["ruff>=0.8.0"]
+
+[project.scripts]
+ttns = "tt_news_scraper.main:main"
+
+[tool.ruff.lint]
+ignore = [
+    "BLE001",
+    "FA100",
+    "UP045",
+]
+
+[build-system]
+requires = ["uv_build>=0.12.0,<0.13.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = ["ruff>=0.17.3"]

tt_news_scraper-0.4.3/pyproject.toml.orig

@@ -0,0 +1,36 @@
+[project]
+name = "tt-news-scraper"
+version = "0.4.3"
+description = "Trinidad and Tobago news scraper using Guardian & Express as sources"
+requires-python = ">=3.12"
+
+dependencies = [
+    "beautifulsoup4>=4.12.0",
+    "pandas>=2.0.0",
+    "pytest>=9.2.0",
+    "requests>=2.31.0",
+    "responses>=0.26.3",
+]
+
+[project.optional-dependencies]
+dev = [
+    "ruff>=0.8.0",
+]
+[tool.ruff.lint]
+ignore = [
+    "BLE001",  # Blind exception catches are intentional in scraper for graceful degradation
+    "FA100", # Optional type hints are okay
+    "UP045" # Optional type hints are okay
+]
+
+[build-system]
+requires = ["uv_build>=0.12.0,<0.13.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "ruff>=0.17.3",
+]
+
+[project.scripts]
+ttns = "tt_news_scraper.main:main"

tt_news_scraper-0.4.3/src/tt_news_scraper/__init__.py

@@ -0,0 +1,2 @@
+def main() -> None:
+    print("Hello from tt-news-scraper!")

tt_news_scraper-0.4.3/src/tt_news_scraper/const.py

@@ -0,0 +1,10 @@
+"""Shared configuration and constants for the news scraper."""
+
+# Canonical column schema for unified output across all sources
+CANONICAL_COLUMNS = ["title", "url", "author", "date", "summary"]
+
+# Base URLs for each source
+BASE_URLS = {
+    "express": "https://trinidadexpress.com/search/?t=article&sd=desc",
+    "guardian": "https://www.guardian.co.tt/news-6.12.0.4f1749cb44",
+}

tt_news_scraper-0.4.3/src/tt_news_scraper/dataset_handling/data_saving_utils.py

@@ -0,0 +1,75 @@
+"""Dataset export and saving utilities."""
+
+import logging
+from typing import Optional
+
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+
+def save_dataset(df: pd.DataFrame, output_path: str) -> None:
+    """Export DataFrame to CSV with unified schema.
+
+    Args:
+        df: The DataFrame containing news articles
+        output_path: Path to the output CSV file
+    """
+    df.to_csv(output_path, index=False)
+    logger.info(f"Dataset saved to {output_path}")
+
+
+def save_dataset_with_metadata(
+    df: pd.DataFrame,
+    output_path: str,
+    metadata: Optional[dict] = None,
+) -> None:
+    """Export DataFrame to CSV with embedded metadata.
+
+    Args:
+        df: The DataFrame containing news articles
+        output_path: Path to the output CSV file
+        metadata: Optional dictionary of metadata (source, count, date_range, etc.)
+    """
+    if metadata is not None and len(metadata) > 0:
+        meta_df = pd.DataFrame([metadata])
+        df_with_meta = pd.concat(
+            [meta_df.rename(columns={c: f"_meta_{c}" for c in metadata}), df],
+            axis=1,
+        )
+    else:
+        df_with_meta = df
+
+    df_with_meta.to_csv(output_path, index=False)
+    logger.info(f"Dataset with metadata saved to {output_path}")
+
+
+def validate_dataframe_schema(df: pd.DataFrame, expected_columns: list[str]) -> bool:
+    """Validate that DataFrame has the expected canonical columns.
+
+    Args:
+        df: The DataFrame to validate
+        expected_columns: List of expected column names in order
+
+    Returns:
+        True if schema matches, False otherwise
+    """
+    actual_columns = list(df.columns)
+    return actual_columns == expected_columns
+
+
+def get_dataframe_summary(df: pd.DataFrame) -> dict:
+    """Get a summary statistics dictionary for the DataFrame.
+
+    Args:
+        df: The DataFrame to summarize
+
+    Returns:
+        Dictionary with row count, column names, and basic stats per column
+    """
+    return {
+        "row_count": len(df),
+        "column_names": list(df.columns),
+        "column_types": {col: str(type(df[col].iloc[0])) for col in df.columns},
+        "non_null_counts": {col: df[col].notna().sum() for col in df.columns},
+    }

tt_news_scraper-0.4.3/src/tt_news_scraper/dataset_handling/merging_utils.py

@@ -0,0 +1,57 @@
+import logging
+
+import pandas as pd
+
+from tt_news_scraper.fetching_utils.articles import (
+    fetch_express_article_content,
+    fetch_guardian_article_content,
+)
+from tt_news_scraper.fetching_utils.headlines import (
+    fetch_express_headlines,
+    fetch_guardian_headlines,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def fetch_unified_dataset(
+    source: str,
+    limit: int | None = None,
+    search_term: str | None = None,
+    fetch_content: bool = False,
+) -> pd.DataFrame:
+    """Unified entry point for fetching headlines from any supported source."""
+    if source == "express":
+        logger.info(f"Fetching headlines from Express with limit={limit}, search_term={search_term}")
+        df = fetch_express_headlines(
+            url=None,  # Let the function construct URL from params
+            limit=limit or 10,
+            search_term=search_term,
+        )
+    else:  # guardian
+        logger.info(f"Fetching headlines from Guardian with search term: {search_term}")
+        df = fetch_guardian_headlines(
+            url=None, search_term=search_term, max_pages=limit or 10
+        )
+
+    # Normalize date format to ISO string for consistency
+    if not df["date"].empty:
+        df["date"] = pd.to_datetime(df["date"], errors="coerce").dt.strftime(
+            "%Y-%m-%dT%H:%M:%S"
+        )
+
+    # Fetch full article content if requested
+    if fetch_content and not df.empty:
+        logger.info(f"Fetching full content for {len(df)} articles...")
+        urls = df["url"].tolist()
+
+        if source == "express":
+            content_df = fetch_express_article_content(urls)
+        else:  # guardian
+            content_df = fetch_guardian_article_content(urls)
+
+        # Merge content back to headlines on URL column
+        if not content_df.empty:
+            df = pd.merge(df, content_df, on="url", how="left")
+
+    return df

tt_news_scraper-0.4.3/src/tt_news_scraper/fetching_utils/articles.py

@@ -0,0 +1,152 @@
+import logging
+from collections.abc import Iterable
+
+import pandas as pd
+import requests
+from bs4 import BeautifulSoup
+
+logger = logging.getLogger(__name__)
+
+# Base domain for URL normalization
+BASE_DOMAINS = {
+    "express": "https://trinidadexpress.com",
+    "guardian": "https://www.guardian.co.tt",
+}
+
+
+def fetch_express_article_content(urls: Iterable[str]) -> pd.DataFrame:
+    """Fetch full article content from Trinidad Express URLs.
+
+    Args:
+        urls: Iterable of article URLs to fetch.
+
+    Returns:
+        DataFrame with columns: url, content
+    """
+    articles = []
+
+    # Convert input to list to handle pandas Series/DataFrame correctly
+    if hasattr(urls, "__iter__") and not isinstance(urls, str):
+        url_list = [str(u).strip() for u in urls]
+    else:
+        url_list = [urls] if urls else []
+
+    for url in url_list:
+        if not url:
+            continue
+
+        logger.debug(f"Fetching Express article: {url}")
+
+        try:
+            response = requests.get(
+                url,
+                headers={
+                    "User-Agent": (
+                        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+                        "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+                    ),
+                },
+                timeout=30,
+            )
+
+            if not response.ok:
+                logger.warning(f"Failed to fetch Express article {url}: HTTP {response.status_code}")
+                continue
+
+            soup = BeautifulSoup(response.text, "html.parser")
+
+            # Extract article content from main article body
+            # TT Express uses .tnt-article-content or similar selectors
+            content_divs = soup.select(".tnt-article-content, .article-body, article p")
+            if not content_divs:
+                logger.debug(f"No content found in Express article {url}")
+                continue
+
+            # Join all paragraphs into a single content string
+            content_parts = [div.get_text(strip=True) for div in content_divs]
+            content = "\n".join(content_parts).strip()
+
+            if not content:
+                logger.debug(f"Empty content extracted from Express article {url}")
+                continue
+
+            articles.append({"url": url, "content": content})
+
+        except requests.RequestException as e:
+            logger.error(f"Request failed for Express article {url}: {e}")
+        except Exception as e:
+            logger.error(f"Error processing Express article {url}: {e}")
+
+    if not articles:
+        logger.warning("No articles were successfully fetched from Express.")
+
+    return pd.DataFrame(articles, columns=["url", "content"])
+
+
+def fetch_guardian_article_content(urls: Iterable[str]) -> pd.DataFrame:
+    """Fetch full article content from Trinidad Guardian URLs.
+
+    Args:
+        urls: Iterable of article URLs to fetch.
+
+    Returns:
+        DataFrame with columns: url, content
+    """
+    articles = []
+
+    # Convert input to list to handle pandas Series/DataFrame correctly
+    if hasattr(urls, "__iter__") and not isinstance(urls, str):
+        url_list = [str(u).strip() for u in urls]
+    else:
+        url_list = [urls] if urls else []
+
+    for url in url_list:
+        if not url:
+            continue
+
+        logger.debug(f"Fetching Guardian article: {url}")
+
+        try:
+            response = requests.get(
+                url,
+                headers={
+                    "User-Agent": (
+                        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+                        "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+                    ),
+                },
+                timeout=30,
+            )
+
+            if not response.ok:
+                logger.warning(f"Failed to fetch Guardian article {url}: HTTP {response.status_code}")
+                continue
+
+            soup = BeautifulSoup(response.text, "html.parser")
+
+            # Extract article content from main body
+            # Guardian uses .article-body or similar selectors
+            content_divs = soup.select(".article-body p, .story-content p, article p")
+            if not content_divs:
+                logger.debug(f"No content found in Guardian article {url}")
+                continue
+
+            # Join all paragraphs into a single content string
+            content_parts = [div.get_text(strip=True) for div in content_divs]
+            content = "\n".join(content_parts).strip()
+
+            if not content:
+                logger.debug(f"Empty content extracted from Guardian article {url}")
+                continue
+
+            articles.append({"url": url, "content": content})
+
+        except requests.RequestException as e:
+            logger.error(f"Request failed for Guardian article {url}: {e}")
+        except Exception as e:
+            logger.error(f"Error processing Guardian article {url}: {e}")
+
+    if not articles:
+        logger.warning("No articles were successfully fetched from Guardian.")
+
+    return pd.DataFrame(articles, columns=["url", "content"])

tt_news_scraper-0.4.3/src/tt_news_scraper/fetching_utils/headlines.py

@@ -0,0 +1,505 @@
+import logging
+import time
+import urllib.parse
+from datetime import datetime
+
+import pandas as pd
+import requests
+from bs4 import BeautifulSoup, Tag
+
+from tt_news_scraper.const import BASE_URLS, CANONICAL_COLUMNS
+
+logger = logging.getLogger(__name__)
+
+
+def fetch_express_headlines(
+    url: str | None = None, limit: int | None = None, search_term: str | None = None
+) -> pd.DataFrame:
+    """Fetch article headlines and metadata from TT Express page.
+
+    Args:
+        url: Optional Trinidad Express search URL to scrape. Defaults to homepage.
+              Supports both homepage lists and search result URLs (e.g., ?q=keyword).
+        limit: Maximum number of articles to fetch per page. Only applies when scraping
+               search result URLs with pagination support.
+        search_term: Search keyword for Express mode (used in URL construction if url not provided).
+
+    Returns:
+        DataFrame with columns: title, url, author, date, summary
+    """
+    # Construct URL if not provided (supports search_term for Express q-param)
+    if url is None:
+        base = BASE_URLS["express"]
+        params = ["&nsa=eedition"]  # Required param for TT Express
+        if limit:
+            params.append(f"&l={limit}")
+        if search_term:
+            from urllib.parse import quote_plus
+
+            params.append(f"&q={quote_plus(search_term)}")
+        url = base + "".join(params)
+
+    # Handle local file paths for testing (requests doesn't support file:// scheme)
+    if url.startswith("file://"):
+        with open(url, "r", encoding="utf-8") as f:
+            html_content = f.read()
+        soup = BeautifulSoup(html_content, "html.parser")
+    else:
+        headers = {
+            "User-Agent": (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36"
+            )
+        }
+
+        try:
+            response = requests.get(url, headers=headers, timeout=10)
+            response.raise_for_status()
+        except requests.RequestException as e:
+            logger.error(f"Failed to fetch {url}: {e}")
+            raise
+
+        soup = BeautifulSoup(response.text, "html.parser")
+    articles = []
+
+    # Find all article cards first (each is a <article> element)
+    article_cards = soup.select("article.tnt-asset-type-article")
+
+    # Apply limit if specified (respect Guardian's pagination-style fetching)
+    effective_limit = min(len(article_cards), limit or len(article_cards))
+    for card in article_cards[:effective_limit]:
+        headline_link = card.select_one(".card-headline h3 a")
+        if not headline_link:
+            continue
+
+        title_tag: Tag | None = headline_link.find_parent("h3") or headline_link
+
+        title = (title_tag.get_text(strip=True) or "").strip()
+        if not title:
+            continue  # Skip entries without a title
+
+        # Extract the partial article link from href attribute
+        partial_link = str(headline_link.get("href", "")).strip()
+
+        # Only append if we have both title and a valid link
+        if not partial_link:
+            continue  # Skip entries without a link
+
+        # Extract author (blank if missing)
+        author_elem = card.select_one(".card-byline span.tnt-byline.asset-byline")
+        author = author_elem.get_text(strip=True) if author_elem else ""
+
+        # Extract section type ("General" if missing, preserve exact text when present)
+        section_elem = card.select_one(".card-label-section a.tnt-section-tag")
+        section_type = section_elem.text.strip() if section_elem else "General"
+
+        # Extract published_date (ISO format from datetime attribute)
+        time_tag = card.select_one("time[datetime]")
+        published_date = ""
+        if time_tag and time_tag.has_attr("datetime"):
+            try:
+                dt_str = time_tag["datetime"]
+                parsed_dt = datetime.fromisoformat(dt_str)
+                published_date = parsed_dt.isoformat()
+            except ValueError:
+                pass  # Keep empty on parse failure
+
+        # Extract article summary (full paragraph, not truncated)
+        lead_p = card.select_one(".card-lead p.tnt-summary")
+        article_summary = lead_p.get_text(strip=True) if lead_p else ""
+
+        base_domain = "https://trinidadexpress.com"
+        articles.append(
+            {
+                "title": title,
+                "url": base_domain + partial_link,
+                "author": author,
+                "section_type": section_type,
+                "date": published_date,
+                "summary": article_summary,
+            }
+        )
+
+    # Log skipped entries
+    total_cards = len(article_cards)
+    extracted_count = len(articles)
+    skipped = max(0, total_cards - extracted_count)
+
+    if skipped > 0:
+        logger.info(f"Filtered out {skipped} non-article card(s)")
+
+    # Create DataFrame and rename columns to canonical schema before returning
+    df = pd.DataFrame(articles, columns=["url", "title", "summary", "author", "date"])
+    return _rename_express_columns(df)
+
+
+def _rename_express_columns(df: pd.DataFrame) -> pd.DataFrame:
+    """Rename Express-specific column names to canonical schema."""
+    col_map = {
+        "section_type": None,  # Drop unused column
+    }
+    df = df.drop(
+        columns=[col for col in col_map if col_map[col] is not None], errors="ignore"
+    )
+    return _rename_guardian_columns(df)
+
+
+def fetch_guardian_headlines(
+    url: str | None = None,
+    search_term: str | None = None,
+    max_pages: int = 10,
+) -> pd.DataFrame:
+    """Scrape Trinidad Guardian news headlines and descriptions.
+
+    Args:
+        url: Optional URL to scrape. Defaults to the homepage list page.
+              Supports both homepage lists and search result pages (e.g.,
+              ?search_content=keyword).
+        max_pages: Maximum number of pagination pages to fetch for search results.
+                   Only applies when scraping search result URLs.
+
+    Returns:
+        pandas.DataFrame with columns: url, headline, description, author, date
+    """
+    # Construct URL if search_term is provided without a base URL
+    if search_term and url is None:
+        url = (
+            f"{BASE_URLS['guardian']}?search_content={urllib.parse.quote(search_term)}"
+        )
+    elif url is None:
+        url = BASE_URLS["guardian"]
+
+    # Fetch page with proper headers to avoid bot detection
+
+    response = requests.get(
+        url,
+        headers={
+            "User-Agent": (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+                "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+            ),
+        },
+        timeout=30,
+    )
+
+    if not response.ok:
+        logger.warning(f"Got HTTP {response.status_code} for {url}, retrying in 2s...")
+
+        time.sleep(2)
+        response = requests.get(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=30)
+
+    if not response.ok:
+        logger.error(
+            f"Failed to fetch Guardian page {url}: HTTP {response.status_code}"
+        )
+        raise requests.HTTPError(
+            f"Guardian returned {response.status_code}", response=response
+        )
+
+    try:
+        # Use response.text directly (requests handles decoding for us)
+        html_content = (response.text or "").strip()
+        if not html_content:
+            logger.warning(f"Empty response from Guardian page {url}")
+            return pd.DataFrame(columns=CANONICAL_COLUMNS)
+
+        # Debug output
+        print(f"[DEBUG] HTML length: {len(html_content)}")
+        print(f"[DEBUG] First 200 chars: {html_content[:200]}")
+
+        soup = BeautifulSoup(html_content, "html.parser")
+    except Exception as e:
+        logger.error(f"Failed to parse HTML for {url}: {e}")
+        return pd.DataFrame(columns=CANONICAL_COLUMNS)
+
+    articles = []
+
+    # Find all list containers (contain article links)
+    for container in soup.find_all(class_=lambda x: x and "_LIST" in str(x)):
+        # Skip sponsored content
+        if "sponsored-list" in str(container.get("class", [])):
+            continue
+
+        for link in container.find_all("a", href=True):
+            # Extract headline from h1 inside the anchor
+            headline_elem = link.find("h1", class_="headline")
+
+            if not headline_elem:
+                continue
+
+            headline = headline_elem.get_text(strip=True)
+
+            # Skip pagination links (short text, numbers/arrows only)
+            link_text = link.get_text(strip=True)
+            if (
+                len(headline) < 20
+                or headline.isdigit()
+                or "→" in link_text
+                or "»" in link_text
+            ):
+                continue
+
+            # Get description from text before the headline (author/caption info)
+            parent_div = link.find_parent("div")
+            all_text = parent_div.get_text(strip=True) if parent_div else ""
+
+            # Description is everything between container start and headline
+            headline_pos = all_text.find(headline)
+            description = all_text[:headline_pos].strip() if headline_pos > 0 else ""
+
+            # Clean up description (remove extra whitespace, author names in brackets)
+            description = " ".join(description.split())
+
+            # Extract author (pattern: "byAuthorName" after headline)
+            import re
+
+            author_match = re.search(r"by(\S+)", all_text, re.IGNORECASE)
+            author = author_match.group(1).strip() if author_match else ""
+
+            # Extract timestamp (format: YYYYMMDDHHMMSS at end of metadata section)
+            timestamp_match = re.search(r"(\d{14})\b", all_text)
+            article_date = pd.NaT
+            if timestamp_match and len(timestamp_match.group(1)) == 14:
+                try:
+                    dt = datetime.strptime(timestamp_match.group(1), "%Y%m%d%H%M%S")
+                    article_date = dt
+                except ValueError:
+                    pass
+
+            articles.append(
+                {
+                    "url": link["href"],
+                    "headline": headline,
+                    "description": description[:200]
+                    if len(description) > 200
+                    else description,
+                    "author": author,
+                    "date": article_date,
+                }
+            )
+
+    # Only use pagination for search result URLs (those with query params), not homepage
+    if url is not None and isinstance(url, str) and "?" in url:
+        if soup.find(class_=lambda x: x and "ag_page_navigation" in str(x)):
+            articles = scrape_all_guardian_pages(url, max_pages)
+
+    # Convert URLs to absolute format for consistency
+    base_domain = "https://www.guardian.co.tt"
+    df = pd.DataFrame(articles).copy()
+    if not df.empty:
+        df["url"] = [
+            base_domain + url if not url.startswith("http") else url
+            for url in df["url"]
+        ]
+
+    # Rename columns to canonical schema before returning
+    return _rename_guardian_columns(df)
+
+
+def _rename_guardian_columns(df: pd.DataFrame) -> pd.DataFrame:
+    """Rename Guardian-specific column names to canonical schema."""
+    col_map = {
+        "headline": "title",
+        "description": "summary",
+    }
+    df.rename(columns=col_map, inplace=True)
+    return df
+
+
+def scrape_all_guardian_pages(base_url: str, max_pages: int = 10) -> list[dict]:
+    """Fetch all pagination pages for a search result URL.
+
+    Uses iterative link-following instead of URL construction to handle
+    Guardian's dynamic pagination structure correctly.
+
+    Args:
+        base_url: The original search result URL (e.g., with ?search_content=...)
+        max_pages: Maximum number of pagination pages to fetch
+
+    Returns:
+        list of article dicts from all pages
+    """
+    all_articles = []
+    visited_urls = set()
+    page_queue = [base_url]  # Queue for iterative BFS-style traversal
+
+    while page_queue and len(all_articles) < max_pages * 20:  # ~20 articles per page
+        current_url = page_queue.pop(0)
+
+        if current_url in visited_urls:
+            continue
+        visited_urls.add(current_url)
+
+        try:
+            import time
+
+            time.sleep(1)  # Polite delay between pagination requests
+
+            response = requests.get(
+                current_url,
+                headers={
+                    "User-Agent": (
+                        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+                        "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+                    ),
+                },
+                timeout=30,
+            )
+
+            if not response.ok:
+                logger.error(
+                    f"Failed to fetch Guardian page {current_url}: HTTP {response.status_code}"
+                )
+                continue
+
+            html = (response.text or "").strip()
+
+            soup = BeautifulSoup(html, "html.parser")
+
+            # Scrape articles from current page and extend the list
+            scraped_df = _scrape_single_guardian_page(current_url, visited_urls)
+            all_articles.extend(scraped_df.to_dict(orient="records"))
+
+            # Extract pagination links and add new pages to queue
+            for nav_elem in soup.find_all(
+                class_=lambda x: x and "ag_page_navigation" in str(x)
+            ):
+                for a in nav_elem.find_all("a", href=True):
+                    link_text = a.get_text(strip=True)
+                    if any(c.isdigit() for c in link_text) and len(link_text) > 1:
+                        new_url = urllib.parse.urljoin(current_url, a["href"])
+                        if new_url not in visited_urls:
+                            page_queue.append(new_url)
+
+        except Exception as e:
+            logger.error(f"Error fetching {current_url}: {e}")
+
+    # Convert URLs to absolute format for consistency
+    base_domain = "https://www.guardian.co.tt"
+    df = pd.DataFrame(all_articles).copy()
+    if not df.empty:
+        df["url"] = [
+            base_domain + url if not url.startswith("http") else url
+            for url in df["url"]
+        ]
+
+    return df
+
+
+def _scrape_single_guardian_page(url: str, visited_urls: set[str]) -> pd.DataFrame:
+    """Scrape articles from a single page.
+
+    Args:
+        url: The page URL to scrape
+        visited_urls: Set of already-visited URLs (for deduplication)
+
+    Returns:
+        list of article dicts
+    """
+    articles = []
+
+    try:
+        import time
+
+        time.sleep(1)  # Polite delay between requests
+
+        response = requests.get(
+            url,
+            headers={
+                "User-Agent": (
+                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+                    "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+                ),
+            },
+            timeout=30,
+        )
+
+        if not response.ok:
+            logger.error(
+                f"Failed to fetch Guardian page {url}: HTTP {response.status_code}"
+            )
+            return pd.DataFrame(
+                columns=["url", "headline", "description", "author", "date"]
+            )
+        html = (response.text or "").strip()
+
+        soup = BeautifulSoup(html, "html.parser")
+
+        # Scrape articles from list containers
+        for container in soup.find_all(class_=lambda x: x and "_LIST" in str(x)):
+            if "sponsored-list" in str(container.get("class", [])):
+                continue
+
+            for link in container.find_all("a", href=True):
+                headline_elem = link.find("h1", class_="headline")
+
+                if not headline_elem:
+                    continue
+
+                headline = headline_elem.get_text(strip=True)
+                link_text = link.get_text(strip=True)
+
+                # Skip pagination links and short/numeric text
+                if (
+                    len(headline) < 20
+                    or headline.isdigit()
+                    or "→" in link_text
+                    or "»" in link_text
+                ):
+                    continue
+
+                parent_div = link.find_parent("div")
+                all_text = parent_div.get_text(strip=True) if parent_div else ""
+                headline_pos = all_text.find(headline)
+                description = (
+                    all_text[:headline_pos].strip() if headline_pos > 0 else ""
+                )
+                description = " ".join(description.split())
+
+                # Extract author and date from metadata section
+                import re
+
+                parent_div = link.find_parent("div")
+                all_text = parent_div.get_text(strip=True) if parent_div else ""
+
+                author_match = re.search(r"\bby\s+(\S+)", all_text, re.IGNORECASE)
+                author = author_match.group(1).strip() if author_match else ""
+
+                timestamp_match = re.search(r"(\d{14})\b", all_text)
+                article_date = pd.NaT
+                if timestamp_match and len(timestamp_match.group(1)) == 14:
+                    try:
+                        dt = datetime.strptime(timestamp_match.group(1), "%Y%m%d%H%M%S")
+                        article_date = dt
+                    except ValueError:
+                        pass
+
+                article = {
+                    "url": link["href"],
+                    "headline": headline,
+                    "description": description[:200]
+                    if len(description) > 200
+                    else description,
+                    "author": author,
+                    "date": article_date,
+                }
+
+                # Only add if URL not already visited (prevents duplicates across pages)
+                if article["url"] not in visited_urls:
+                    articles.append(article)
+                    visited_urls.add(article["url"])
+
+    except Exception as e:
+        logger.error(f"Error scraping {url}: {e}")
+
+    # Convert URLs to absolute format for consistency
+    base_domain = "https://www.guardian.co.tt"
+    df = pd.DataFrame(articles).copy()
+    if not df.empty:
+        df["url"] = [
+            base_domain + url if not url.startswith("http") else url
+            for url in df["url"]
+        ]
+
+    # Rename columns to canonical schema before returning
+    return _rename_guardian_columns(df)

tt_news_scraper-0.4.3/src/tt_news_scraper/main.py

@@ -0,0 +1,103 @@
+"""Trinidad & Tobago News Scraper - Fetches headlines from Trinidad Express and Guardian by source."""
+
+import logging
+from importlib import metadata
+
+import pandas as pd
+
+from tt_news_scraper.dataset_handling.data_saving_utils import save_dataset
+from tt_news_scraper.dataset_handling.merging_utils import fetch_unified_dataset
+
+logger = logging.getLogger(__name__)
+
+
+def main() -> None:
+    """CLI entry point."""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Scrape Trinidad news headlines from Express or Guardian by source",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "-v",
+        "--version",
+        action="version",
+        version=f"tt-news-scraper v{metadata.version('tt-news-scraper')}",
+        help="Show the program's version number and exit.",
+    )
+    parser.add_argument(
+        "--source",
+        choices=["express", "guardian"],
+        default="express",
+        help="Data source to fetch from (default: express)",
+    )
+    parser.add_argument(
+        "--limit",
+        type=int,
+        nargs="?",
+        const=10,
+        metavar="N",
+        help="Number of articles/pages to fetch (default: 10)",
+    )
+    parser.add_argument(
+        "--search-term",
+        type=str,
+        default=None,
+        metavar="KEYWORD",
+        help='Search keyword for Express (q param) or Guardian mode (e.g., "politics")',
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        default="tt_news_headlines.csv",
+        metavar="FILE",
+        help="Output CSV filename (default: tt_news_headlines.csv)",
+    )
+    parser.add_argument(
+        "--content",
+        action="store_true",
+        default=False,
+        help="Fetch full article content in addition to headlines (slower, ~3x fetch time per article)",
+    )
+
+    args = parser.parse_args()
+
+    # Fetch unified dataset using the wrapper function
+    df = fetch_unified_dataset(
+        source=args.source,
+        limit=args.limit,
+        search_term=args.search_term,
+        fetch_content=args.content,
+    )
+
+    # Export with unified schema
+    save_dataset(df, args.output)
+
+    if df.empty:
+        logger.warning("No articles found or all entries were malformed.")
+        return
+
+    print("\n" + "=" * 80)
+    print(f"Found {len(df)} article(s)")
+    print("=" * 80)
+
+    # Display as a nicely formatted table using canonical column names
+    if len(df) > 0:
+        print("\n")
+
+        for idx, row in df.iterrows():
+            title = str(row["title"])[:60] if pd.notna(row["title"]) else ""
+            link = (
+                str(row["url"])[:70]
+                if pd.notna(row["url"]) and len(str(row["url"])) > 0
+                else ""
+            )
+
+            print(f"[{idx}] {title} | {link}")
+
+        print("\n" + "=" * 80)
+
+
+if __name__ == "__main__":
+    main()