tickertape-client 0.1.0__py3-none-any.whl

tickertape/__init__.py

@@ -0,0 +1,69 @@
+"""Unofficial TickerTape Client & SDK Package."""
+
+from tickertape.client import TickerTapeClient
+from tickertape.constants import (
+    BASE_URL,
+    DEFAULT_CACHE_DIR,
+    DEFAULT_HEADERS,
+    SITEMAP_URLS,
+    URLS,
+)
+from tickertape.errors import (
+    TickerTapeError,
+    TickerTapeHTTPError,
+    TickerTapeNotFoundError,
+    TickerTapeParseError,
+)
+from tickertape.lookup import (
+    ISINIndexer,
+    ISINLookupTable,
+    ISINMapping,
+    ISINResolver,
+)
+from tickertape.models import (
+    MFMeta,
+    MFScorecardItem,
+    MFSecurityInfo,
+    MutualFundDetail,
+    SitemapCacheData,
+    SitemapReference,
+    SitemapURL,
+)
+from tickertape.parsers import (
+    BaseParser,
+    MFParser,
+    SitemapParser,
+    get_parser,
+    register_parser,
+)
+from tickertape.storage import SitemapCacheManager
+
+__all__ = [
+    "TickerTapeClient",
+    "TickerTapeError",
+    "TickerTapeHTTPError",
+    "TickerTapeNotFoundError",
+    "TickerTapeParseError",
+    "SitemapURL",
+    "SitemapReference",
+    "SitemapCacheData",
+    "MutualFundDetail",
+    "MFSecurityInfo",
+    "MFMeta",
+    "MFScorecardItem",
+    "BaseParser",
+    "SitemapParser",
+    "MFParser",
+    "register_parser",
+    "get_parser",
+    "SitemapCacheManager",
+    "ISINMapping",
+    "ISINLookupTable",
+    "ISINResolver",
+    "ISINIndexer",
+    "BASE_URL",
+    "DEFAULT_CACHE_DIR",
+    "DEFAULT_HEADERS",
+    "SITEMAP_URLS",
+    "URLS",
+]

tickertape/client.py

@@ -0,0 +1,162 @@
+"""Unofficial TickerTape API and Scraper Client.
+
+An unofficial, asynchronous, typed Python client for TickerTape.
+Provides access to XML sitemaps (with persistent local caching) and mutual fund data
+with an extensible architecture for future asset types (stocks, ETFs, screens).
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any, Optional
+
+import httpx
+
+from tickertape.constants import (
+    BASE_URL,
+    DEFAULT_CACHE_DIR,
+    DEFAULT_HEADERS,
+    SITEMAP_URLS,
+)
+from tickertape.errors import (
+    TickerTapeHTTPError,
+    TickerTapeNotFoundError,
+)
+from tickertape.resources.mf import MutualFundsResource
+from tickertape.resources.sitemap import SitemapResource
+from tickertape.storage import SitemapCacheManager
+
+logger = logging.getLogger(__name__)
+
+
+class TickerTapeClient:
+    """Asynchronous client for interacting with TickerTape.
+
+    ```
+    Example:
+        async with TickerTapeClient() as client:
+            # Reads from persistent cache (.cache/tickertape) or fetches live XML if not cached:
+            mf_sitemap = await client.sitemap.get("mf")
+
+            # Force live refresh and update disk cache:
+            refreshed = await client.sitemap.refresh("mf")
+
+            # Fetch fund details and ISIN:
+            fund = await client.mf.get("quant-infrastructure-fund-M_QUNG")
+            print(fund.isin, fund.nav)
+    ```
+    """
+
+    def __init__(
+        self,
+        base_url: str = BASE_URL,
+        timeout: float = 30.0,
+        cache_dir: Path | str = DEFAULT_CACHE_DIR,
+        http_client: Optional[httpx.AsyncClient] = None,
+        headers: Optional[dict[str, str]] = None,
+        sitemap_urls: Optional[dict[str, str]] = None,
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.timeout = timeout
+        self.cache_dir = Path(cache_dir)
+        self.sitemap_urls = dict(sitemap_urls or SITEMAP_URLS)
+
+        req_headers = dict(DEFAULT_HEADERS)
+        if headers:
+            req_headers.update(headers)
+        self._default_headers = req_headers
+
+        self._owns_http_client = http_client is None
+        self._http_client = http_client or httpx.AsyncClient(
+            headers=self._default_headers,
+            timeout=self.timeout,
+            follow_redirects=True,
+        )
+
+        # Storage & persistent cache manager
+        self.cache_manager = SitemapCacheManager(cache_dir=self.cache_dir)
+
+        # Resource sub-clients
+        self.sitemap = SitemapResource(self)
+        self.mf = MutualFundsResource(self)
+
+    async def __aenter__(self) -> TickerTapeClient:
+        return self
+
+    async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        await self.close()
+
+    async def close(self) -> None:
+        """Close the underlying HTTP transport."""
+        if self._owns_http_client and not self._http_client.is_closed:
+            await self._http_client.aclose()
+
+    def _build_url(self, url_or_path: str) -> str:
+        """Build full URL if given relative path."""
+        url = url_or_path.strip()
+        if url.startswith(("http://", "https://")):
+            return url
+        if not url.startswith("/"):
+            url = f"/{url}"
+        return f"{self.base_url}{url}"
+
+    async def request(
+        self,
+        method: str,
+        url_or_path: str,
+        *,
+        params: Optional[dict[str, Any]] = None,
+        headers: Optional[dict[str, str]] = None,
+        **kwargs: Any,
+    ) -> str:
+        """Execute an HTTP request and return text response.
+
+        Args:
+            method: HTTP method ('GET', 'POST', etc.)
+            url_or_path: Full URL or path relative to base_url.
+            params: Optional query parameters.
+            headers: Optional extra headers.
+
+        Returns:
+            Response body string.
+
+        Raises:
+            TickerTapeNotFoundError: On HTTP 404.
+            TickerTapeHTTPError: On other HTTP 4xx/5xx status codes.
+        """
+        full_url = self._build_url(url_or_path)
+
+        req_headers = dict(self._default_headers)
+        if headers:
+            req_headers.update(headers)
+
+        try:
+            response = await self._http_client.request(
+                method=method,
+                url=full_url,
+                params=params,
+                headers=req_headers,
+                **kwargs,
+            )
+        except httpx.RequestError as exc:
+            raise TickerTapeHTTPError(
+                f"Request to {full_url} failed: {exc}",
+                status_code=0,
+            ) from exc
+
+        if response.status_code == 404:
+            raise TickerTapeNotFoundError(
+                f"Resource not found at {full_url}",
+                status_code=404,
+                response_data=response.text[:200],
+            )
+
+        if response.is_error:
+            raise TickerTapeHTTPError(
+                f"HTTP {response.status_code} error from {full_url}",
+                status_code=response.status_code,
+                response_data=response.text[:500],
+            )
+
+        return response.text

tickertape/constants.py

@@ -0,0 +1,28 @@
+"""Constants required for proper execution of TickerTapeClient."""
+
+from pathlib import Path
+
+BASE_URL = "https://www.tickertape.in"
+
+SITEMAP_URLS = {
+    "stocks": "https://www.tickertape.in/sitemaps/stocks/sitemap.xml",
+    "etf": "https://www.tickertape.in/sitemaps/etfs/sitemap.xml",
+    "mf": "https://www.tickertape.in/sitemaps/mutualfunds/sitemap.xml",
+    "us-stocks": "https://www.tickertape.in/sitemaps/us-stocks/sitemap.xml",
+    "us-etf": "https://www.tickertape.in/sitemaps/us-etfs/sitemap.xml",
+}
+
+# Kept for backward compatibility
+URLS = SITEMAP_URLS
+
+DEFAULT_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/120.0.0.0 Safari/537.36"
+    ),
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,application/json,*/*;q=0.8",
+}
+
+DEFAULT_CACHE_DIR = Path(".cache/tickertape")
+SITEMAP_NAMESPACE = "http://www.sitemaps.org/schemas/sitemap/0.9"

tickertape/errors.py

@@ -0,0 +1,40 @@
+"""Exception hierarchy for the TickerTape client.
+
+Keep exceptions focused, practical, and small.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class TickerTapeError(Exception):
+    """Base exception for all TickerTape client errors."""
+
+
+class TickerTapeHTTPError(TickerTapeError):
+    """Raised when an HTTP request to TickerTape fails."""
+
+    def __init__(
+        self,
+        message: str,
+        status_code: int,
+        response_data: Any | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.status_code = status_code
+        self.response_data = response_data
+
+    def __str__(self) -> str:
+        base = super().__str__()
+        if self.response_data:
+            return f"[{self.status_code}] {base} - {self.response_data}"
+        return f"[{self.status_code}] {base}"
+
+
+class TickerTapeNotFoundError(TickerTapeHTTPError):
+    """Raised when the requested TickerTape resource is not found (HTTP 404)."""
+
+
+class TickerTapeParseError(TickerTapeError):
+    """Raised when parsing response data (XML sitemap, HTML, __NEXT_DATA__) fails."""

tickertape/lookup/__init__.py

@@ -0,0 +1,13 @@
+"""ISIN to TickerTape lookup and mapping package."""
+
+from tickertape.lookup.indexer import ISINIndexer
+from tickertape.lookup.models import ISINMapping
+from tickertape.lookup.resolver import ISINResolver
+from tickertape.lookup.table import ISINLookupTable
+
+__all__ = [
+    "ISINMapping",
+    "ISINLookupTable",
+    "ISINResolver",
+    "ISINIndexer",
+]

tickertape/lookup/indexer.py

@@ -0,0 +1,181 @@
+"""Bulk indexer for crawling sitemap URLs and populating the SQLite ISIN lookup table."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import TYPE_CHECKING, Callable, Optional
+
+from tickertape.lookup.models import ISINMapping
+from tickertape.lookup.table import ISINLookupTable
+
+if TYPE_CHECKING:
+    from tickertape.client import TickerTapeClient
+
+logger = logging.getLogger(__name__)
+
+ProgressCallback = Callable[
+    [int, int, int, int], None
+]  # (completed, total, successful, failed)
+
+
+class ISINIndexer:
+    """Crawls mutual fund pages from sitemap and populates SQLite ISIN lookup table."""
+
+    def __init__(
+        self, client: TickerTapeClient, table: Optional[ISINLookupTable] = None
+    ) -> None:
+        self.client = client
+        self.table = table or ISINLookupTable(cache_dir=client.cache_dir)
+
+    async def build_index(
+        self,
+        *,
+        force_refresh: bool = False,
+        concurrency: int = 5,
+        delay: float = 0.5,
+        limit: Optional[int] = None,
+        progress_callback: Optional[ProgressCallback] = None,
+    ) -> int:
+        """Build or update the SQLite ISIN lookup table from the mutual funds sitemap.
+
+        Force Refresh Logic:
+            When force_refresh=True:
+            1. Fetches live sitemap XML from TickerTape.
+            2. Parses sitemap and updates the local sitemap cache (.cache/tickertape/sitemaps/mf.json).
+            3. Crawls fund pages and updates the SQLite database (.cache/tickertape/isin_lookup.db).
+
+        When force_refresh=False:
+            1. Loads sitemap from local disk cache.
+            2. Skips record_ids already present in SQLite database (resumable).
+            3. Crawls only missing funds and inserts into SQLite database.
+
+        Args:
+            force_refresh: If True, forces live sitemap fetch, updates sitemap cache, and re-indexes.
+            concurrency: Number of concurrent HTTP workers.
+            delay: Delay in seconds between requests per worker to respect rate limits.
+            limit: Optional maximum number of funds to process (useful for testing or batch runs).
+            progress_callback: Optional callback(completed, total, successful, failed).
+
+        Returns:
+            Number of mappings successfully indexed/updated in SQLite database.
+        """
+        # Step 1: Get sitemap items (force_refresh updates live XML and disk cache)
+        if force_refresh:
+            logger.info(
+                "Force refresh triggered: fetching live sitemap and updating sitemap cache..."
+            )
+            sitemap_items = await self.client.sitemap.refresh("mf")
+        else:
+            sitemap_items = await self.client.sitemap.get("mf", force_refresh=False)
+
+        if not sitemap_items:
+            logger.warning("No sitemap items found to index")
+            return 0
+
+        # Step 2: Determine items to process
+        if force_refresh:
+            items_to_process = sitemap_items
+        else:
+            existing_ids = self.table.get_all_record_ids()
+            items_to_process = [
+                item for item in sitemap_items if item.record_id not in existing_ids
+            ]
+
+        if limit is not None:
+            items_to_process = items_to_process[:limit]
+
+        total = len(items_to_process)
+        if total == 0:
+            logger.info("All sitemap items are already indexed in SQLite lookup table.")
+            return 0
+
+        logger.info(
+            "Starting ISIN indexing for %d funds (concurrency=%d)...",
+            total,
+            concurrency,
+        )
+
+        semaphore = asyncio.Semaphore(concurrency)
+        completed = 0
+        successful = 0
+        failed = 0
+        batch: list[ISINMapping] = []
+        lock = asyncio.Lock()
+
+        async def worker(item):
+            nonlocal completed, successful, failed
+            async with semaphore:
+                try:
+                    fund = await self.client.mf.get(item.url)
+                    if fund.isin:
+                        raw_slug = fund.slug or item.url
+                        clean_slug = raw_slug.split("/mutualfunds/")[-1].lstrip("/")
+
+                        sec_info = fund.security_info
+                        meta = fund.meta
+
+                        mapping = ISINMapping(
+                            isin=fund.isin.strip().upper(),
+                            record_id=fund.mf_id or item.record_id,
+                            slug=clean_slug,
+                            name=fund.name,
+                            amc=(meta.amc if meta else None)
+                            or (sec_info.amc if sec_info else None),
+                            amc_code=sec_info.amc_code if sec_info else None,
+                            sector=(sec_info.sector if sec_info else None)
+                            or (meta.sector if meta else None),
+                            subsector=(sec_info.subsector if sec_info else None)
+                            or (meta.subsector if meta else None),
+                            fund_type=meta.fund_type if meta else None,
+                            fund_class=meta.type if meta else None,
+                            plan=meta.plan if meta else None,
+                            option=(sec_info.option if sec_info else None)
+                            or (meta.option if meta else None),
+                            risk_level=meta.risk_classification if meta else None,
+                            benchmark=meta.benchmark_index if meta else None,
+                            url=item.url,
+                            nav=fund.nav,
+                        )
+
+                        async with lock:
+                            batch.append(mapping)
+                            successful += 1
+                            if len(batch) >= 20:
+                                self.table.upsert_batch(batch)
+                                batch.clear()
+                    else:
+                        async with lock:
+                            failed += 1
+                except Exception as exc:
+                    logger.debug("Failed to index %s: %s", item.url, exc)
+                    async with lock:
+                        failed += 1
+                finally:
+                    async with lock:
+                        completed += 1
+                        if progress_callback:
+                            try:
+                                progress_callback(completed, total, successful, failed)
+                            except Exception:
+                                pass
+
+                if delay > 0:
+                    await asyncio.sleep(delay)
+
+        # Run workers concurrently
+        tasks = [asyncio.create_task(worker(item)) for item in items_to_process]
+        await asyncio.gather(*tasks)
+
+        # Flush remaining batch
+        if batch:
+            self.table.upsert_batch(batch)
+            batch.clear()
+
+        logger.info(
+            "Indexing completed: %d/%d successfully saved to SQLite (%d failed)",
+            successful,
+            total,
+            failed,
+        )
+        return successful

tickertape/lookup/models.py

@@ -0,0 +1,45 @@
+"""Data models for ISIN to TickerTape lookup table."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Optional
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ISINMapping(BaseModel):
+    """Mapping entry linking a mutual fund ISIN to TickerTape record and categorization metadata.
+
+    Contains stable classification and grouping attributes used by AI agents
+    to cluster and identify similar/peer mutual funds.
+    """
+
+    model_config = ConfigDict(extra="ignore")
+
+    isin: str
+    record_id: str
+    slug: str
+    name: str
+    amc: Optional[str] = None
+    amc_code: Optional[str] = None
+    sector: Optional[str] = (
+        None  # Primary asset class (e.g. 'Equity', 'Debt', 'Hybrid')
+    )
+    subsector: Optional[str] = (
+        None  # Granular category (e.g. 'Sectoral Fund - Infrastructure', 'Large Cap Fund')
+    )
+    fund_type: Optional[str] = None  # Fund type (e.g. 'Equity')
+    fund_class: Optional[str] = None  # Structure type (e.g. 'Open', 'Close')
+    plan: Optional[str] = None  # 'Direct' vs 'Regular'
+    option: Optional[str] = None  # 'Growth' vs 'IDCW'
+    risk_level: Optional[str] = (
+        None  # Risk classification (e.g. 'Very High', 'Moderate', 'Low')
+    )
+    benchmark: Optional[str] = (
+        None  # Benchmark Index (e.g. 'Nifty Infrastructure - TRI')
+    )
+    url: str
+    nav: Optional[float] = None
+    updated_at: str = Field(
+        default_factory=lambda: datetime.now(timezone.utc).isoformat()
+    )

tickertape/lookup/resolver.py

@@ -0,0 +1,159 @@
+"""Targeted on-demand resolver for matching ISINs to TickerTape records."""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import TYPE_CHECKING, Optional
+
+from tickertape.lookup.models import ISINMapping
+from tickertape.lookup.table import ISINLookupTable
+
+if TYPE_CHECKING:
+    from tickertape.client import TickerTapeClient
+
+logger = logging.getLogger(__name__)
+
+STOP_WORDS = {
+    "fund",
+    "mutual",
+    "plan",
+    "growth",
+    "direct",
+    "regular",
+    "idcw",
+    "dividend",
+    "reinvestment",
+    "payout",
+    "scheme",
+    "equity",
+    "debt",
+    "the",
+    "and",
+    "of",
+    "in",
+    "to",
+    "for",
+    "a",
+    "an",
+    "is",
+}
+
+
+class ISINResolver:
+    """Resolves specific ISINs on demand using local sitemap slug matching and verification."""
+
+    def __init__(
+        self, client: TickerTapeClient, table: Optional[ISINLookupTable] = None
+    ) -> None:
+        self.client = client
+        self.table = table or ISINLookupTable(cache_dir=client.cache_dir)
+
+    async def resolve(
+        self,
+        isin: str,
+        hint_name: Optional[str] = None,
+        max_candidates: int = 3,
+    ) -> Optional[ISINMapping]:
+        """Resolve an ISIN to a TickerTape mapping.
+
+        Checks the persistent SQLite table first. If not found, uses hint_name
+        to search candidate slugs in the cached sitemap, verifies the page ISIN,
+        and saves it to the SQLite lookup table upon match.
+
+        Args:
+            isin: Mutual fund ISIN string (e.g. 'INF966L01721').
+            hint_name: Optional scheme name to narrow candidates.
+            max_candidates: Maximum candidate pages to inspect.
+
+        Returns:
+            ISINMapping if successfully resolved, otherwise None.
+        """
+        clean_isin = isin.strip().upper()
+
+        # 1. Check persistent SQLite table
+        existing = self.table.get(clean_isin)
+        if existing:
+            return existing
+
+        # 2. If no hint_name, we cannot narrow down 5,900+ sitemap URLs safely
+        if not hint_name:
+            logger.debug(
+                "Cannot resolve ISIN %s without a hint name or pre-built index",
+                clean_isin,
+            )
+            return None
+
+        # 3. Load cached sitemap
+        sitemap_items = await self.client.sitemap.get("mf")
+        if not sitemap_items:
+            return None
+
+        # 4. Extract search tokens from hint_name
+        tokens = [
+            t.lower()
+            for t in re.findall(r"[a-zA-Z0-9]+", hint_name)
+            if t.lower() not in STOP_WORDS and len(t) > 2
+        ]
+
+        if not tokens:
+            return None
+
+        # 5. Score sitemap slugs based on token overlap
+        scored_candidates = []
+        for item in sitemap_items:
+            slug_lower = item.url.lower()
+            matched_count = sum(1 for token in tokens if token in slug_lower)
+            if matched_count > 0:
+                scored_candidates.append((matched_count, item))
+
+        # Sort highest match count first
+        scored_candidates.sort(key=lambda x: x[0], reverse=True)
+        top_candidates = [item for _, item in scored_candidates[:max_candidates]]
+
+        # 6. Fetch candidate pages and verify ISIN
+        for candidate in top_candidates:
+            try:
+                fund = await self.client.mf.get(candidate.url)
+                if fund.isin and fund.isin.strip().upper() == clean_isin:
+                    raw_slug = fund.slug or candidate.url
+                    clean_slug = raw_slug.split("/mutualfunds/")[-1].lstrip("/")
+
+                    sec_info = fund.security_info
+                    meta = fund.meta
+
+                    mapping = ISINMapping(
+                        isin=clean_isin,
+                        record_id=fund.mf_id or candidate.record_id,
+                        slug=clean_slug,
+                        name=fund.name,
+                        amc=(meta.amc if meta else None)
+                        or (sec_info.amc if sec_info else None),
+                        amc_code=sec_info.amc_code if sec_info else None,
+                        sector=(sec_info.sector if sec_info else None)
+                        or (meta.sector if meta else None),
+                        subsector=(sec_info.subsector if sec_info else None)
+                        or (meta.subsector if meta else None),
+                        fund_type=meta.fund_type if meta else None,
+                        fund_class=meta.type if meta else None,
+                        plan=meta.plan if meta else None,
+                        option=(sec_info.option if sec_info else None)
+                        or (meta.option if meta else None),
+                        risk_level=meta.risk_classification if meta else None,
+                        benchmark=meta.benchmark_index if meta else None,
+                        url=candidate.url,
+                        nav=fund.nav,
+                    )
+
+                    self.table.upsert(mapping)
+                    logger.info(
+                        "Successfully resolved %s -> %s (%s)",
+                        clean_isin,
+                        mapping.record_id,
+                        mapping.name,
+                    )
+                    return mapping
+            except Exception as exc:
+                logger.debug("Candidate check failed for %s: %s", candidate.url, exc)
+
+        return None