marketmyna 0.2.0__py3-none-any.whl

marketmyna/data/mutual_funds.py

@@ -0,0 +1,34 @@
+"""Mutual Fund data fetcher for Indian mutual funds."""
+
+import pandas as pd
+import requests
+
+
+class MutualFundDataFetcher:
+    """Fetches mutual fund data from MFAPI.in (free API for Indian MFs)."""
+
+    BASE_URL = "https://api.mfapi.in"
+
+    def get_historical_nav(self, scheme_code: int, days: int = 365) -> pd.DataFrame:
+        """Get historical NAV data for a scheme."""
+        try:
+            response = requests.get(f"{self.BASE_URL}/mf/{scheme_code}", timeout=10)
+            response.raise_for_status()
+            data = response.json()
+
+            nav_data = data.get("data", [])[:days]
+
+            df = pd.DataFrame(nav_data)
+            if not df.empty:
+                df["date"] = pd.to_datetime(df["date"], format="%d-%m-%Y")
+                df["nav"] = df["nav"].astype(float)
+                df = df.sort_values("date")
+                df.set_index("date", inplace=True)
+
+            return df
+
+        except Exception as e:
+            print(f"Error fetching historical NAV for {scheme_code}: {e}")
+            return pd.DataFrame()
+
+

marketmyna/data/news.py

@@ -0,0 +1,98 @@
+"""News data fetcher for stocks."""
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Optional
+
+
+@dataclass
+class NewsArticle:
+    """Represents a news article."""
+    title: str
+    summary: str
+    source: str
+    url: str
+    published: datetime
+    sentiment: Optional[str] = None  # positive, negative, neutral
+    symbols: list[str] = None
+
+    def to_dict(self) -> dict:
+        return {
+            "title": self.title,
+            "summary": self.summary,
+            "source": self.source,
+            "url": self.url,
+            "published": self.published.isoformat() if self.published else None,
+            "sentiment": self.sentiment,
+            "symbols": self.symbols or [],
+        }
+
+
+class NewsFetcher:
+    """Fetches a stock's news from Yahoo Finance."""
+
+    def get_stock_news(self, symbol: str, limit: int = 10) -> list[NewsArticle]:
+        """Get news for a specific stock using yfinance."""
+        try:
+            import yfinance as yf
+
+            ticker = yf.Ticker(symbol)
+            news = ticker.news
+
+            articles = []
+            for item in news[:limit]:
+                # Handle both old and new yfinance API formats
+                content = item.get("content", item)  # New API nests data in 'content'
+
+                # Extract title
+                title = content.get("title", item.get("title", ""))
+
+                # Extract summary
+                summary = content.get("summary", item.get("summary", title))
+
+                # Extract source/publisher - try multiple locations
+                source = "Unknown"
+                if "provider" in content and isinstance(content["provider"], dict):
+                    source = content["provider"].get("displayName", "Unknown")
+                elif "publisher" in item:
+                    source = item["publisher"]
+                elif "publisher" in content:
+                    source = content["publisher"]
+
+                # Extract URL - try multiple locations
+                url = ""
+                if "canonicalUrl" in content and isinstance(content["canonicalUrl"], dict):
+                    url = content["canonicalUrl"].get("url", "")
+                elif "clickThroughUrl" in content and isinstance(content["clickThroughUrl"], dict):
+                    url = content["clickThroughUrl"].get("url", "")
+                elif "link" in item:
+                    url = item["link"]
+
+                # Extract published date
+                try:
+                    if "pubDate" in content:
+                        # ISO format: "2026-01-05T12:55:00Z"
+                        pub_str = content["pubDate"]
+                        published = datetime.fromisoformat(pub_str.replace("Z", "+00:00"))
+                    elif "providerPublishTime" in item:
+                        published = datetime.fromtimestamp(item["providerPublishTime"])
+                    else:
+                        published = datetime.now()
+                except:
+                    published = datetime.now()
+
+                article = NewsArticle(
+                    title=title,
+                    summary=summary,
+                    source=source,
+                    url=url,
+                    published=published,
+                    symbols=[symbol],
+                )
+                articles.append(article)
+
+            return articles
+        except Exception as e:
+            print(f"Error fetching news for {symbol}: {e}")
+            return []
+

marketmyna/data/news_search.py

@@ -0,0 +1,346 @@
+"""Stock news search across several providers, for the AI and the report.
+
+Same environment variables as daily_stock_analysis, so one set of
+secrets works for both projects:
+
+    TAVILY_API_KEYS    comma-separated, rotated        (tavily.com)
+    BRAVE_API_KEYS     comma-separated, rotated        (brave.com/search/api)
+    SERPAPI_API_KEYS   comma-separated, rotated        (serpapi.com, Google News)
+    BOCHA_API_KEYS     comma-separated, rotated        (bochaai.com, Chinese web)
+    SEARXNG_BASE_URLS  your own SearXNG instance(s), no key
+    NEWS_MAX_AGE_DAYS  how far back to look (default 3)
+
+With none of those set it still works: Google News RSS (free, no key)
+and then Yahoo Finance headlines are always available as fallbacks.
+Providers are tried in the order above; the first one that returns
+results wins, and a failing key is skipped on the next call.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import re
+import threading
+import time
+import xml.etree.ElementTree as ET
+from dataclasses import asdict, dataclass
+from datetime import datetime, timedelta, timezone
+from email.utils import parsedate_to_datetime
+from functools import lru_cache
+from typing import Callable, Mapping, Optional
+from urllib.parse import quote_plus
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+_UA = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36"}
+_TIMEOUT = 15
+_CACHE_SECONDS = 30 * 60
+
+# Titles that are quote pages, not news.
+_JUNK = re.compile(r"stock price, news, quote|share price today|live share price|stock quote|historical prices", re.I)
+
+# Google News locale per market: (hl, gl, ceid)
+_LOCALES = {
+    "india": ("en-IN", "IN", "IN:en"),
+    "us": ("en-US", "US", "US:en"),
+    "hong_kong": ("en-HK", "HK", "HK:en"),
+    "china_a": ("en-US", "US", "US:en"),  # English coverage of A-shares is far better via Google News
+    "crypto": ("en-US", "US", "US:en"),
+}
+
+
+@dataclass
+class NewsItem:
+    title: str
+    source: str = ""
+    url: str = ""
+    snippet: str = ""
+    published: Optional[str] = None  # ISO 8601
+    provider: str = ""
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+def _split(raw: Optional[str]) -> list[str]:
+    return [x.strip() for x in (raw or "").split(",") if x.strip()]
+
+
+def _iso(value) -> Optional[str]:
+    if not value:
+        return None
+    try:
+        if isinstance(value, (int, float)):
+            return datetime.fromtimestamp(value, tz=timezone.utc).isoformat()
+        text = str(value).strip()
+        try:
+            return parsedate_to_datetime(text).isoformat()
+        except (TypeError, ValueError):
+            return datetime.fromisoformat(text.replace("Z", "+00:00")).isoformat()
+    except (TypeError, ValueError, OverflowError):
+        return None
+
+
+def _fresh(item: NewsItem, days: int) -> bool:
+    if not item.published:
+        return True  # unknown age: keep, providers already filter by time
+    try:
+        published = datetime.fromisoformat(item.published)
+    except ValueError:
+        return True
+    if published.tzinfo is None:
+        published = published.replace(tzinfo=timezone.utc)
+    return published >= datetime.now(timezone.utc) - timedelta(days=days + 1)
+
+
+def _freshness(days: int, day: str, week: str, month: str, year: str) -> str:
+    return day if days <= 1 else week if days <= 7 else month if days <= 31 else year
+
+
+# ------------------------------------------------------------- providers
+
+
+def _tavily(query: str, key: str, limit: int, days: int, _market: str) -> list[NewsItem]:
+    resp = requests.post(
+        "https://api.tavily.com/search",
+        json={"query": query, "topic": "news", "days": days, "max_results": limit, "api_key": key},
+        headers={"Authorization": f"Bearer {key}"},
+        timeout=_TIMEOUT,
+    )
+    resp.raise_for_status()
+    return [
+        NewsItem(r.get("title", ""), source=re.sub(r"^www\.", "", (r.get("url") or "").split("/")[2] if r.get("url") else ""),
+                 url=r.get("url", ""), snippet=r.get("content", ""), published=_iso(r.get("published_date")))
+        for r in resp.json().get("results", [])
+    ]
+
+
+def _brave(query: str, key: str, limit: int, days: int, _market: str) -> list[NewsItem]:
+    resp = requests.get(
+        "https://api.search.brave.com/res/v1/news/search",
+        params={"q": query, "count": limit, "freshness": _freshness(days, "pd", "pw", "pm", "py")},
+        headers={"X-Subscription-Token": key, "Accept": "application/json"},
+        timeout=_TIMEOUT,
+    )
+    resp.raise_for_status()
+    return [
+        NewsItem(r.get("title", ""), source=(r.get("meta_url") or {}).get("hostname", ""), url=r.get("url", ""),
+                 snippet=r.get("description", ""), published=_iso(r.get("page_age")))
+        for r in resp.json().get("results", [])
+    ]
+
+
+def _serpapi(query: str, key: str, limit: int, days: int, market: str) -> list[NewsItem]:
+    hl, gl, _ = _LOCALES.get(market, _LOCALES["us"])
+    resp = requests.get(
+        "https://serpapi.com/search.json",
+        params={"engine": "google_news", "q": f"{query} when:{max(1, days)}d", "api_key": key,
+                "hl": hl.split("-")[0], "gl": gl.lower()},
+        timeout=_TIMEOUT,
+    )
+    resp.raise_for_status()
+    items = []
+    for r in resp.json().get("news_results", [])[:limit]:
+        source = r.get("source") or {}
+        items.append(NewsItem(r.get("title", ""), source=source.get("name", "") if isinstance(source, dict) else str(source),
+                              url=r.get("link", ""), snippet=r.get("snippet", ""), published=_iso(r.get("iso_date") or r.get("date"))))
+    return items
+
+
+def _bocha(query: str, key: str, limit: int, days: int, _market: str) -> list[NewsItem]:
+    resp = requests.post(
+        "https://api.bochaai.com/v1/web-search",
+        json={"query": query, "count": limit, "summary": True,
+              "freshness": _freshness(days, "oneDay", "oneWeek", "oneMonth", "oneYear")},
+        headers={"Authorization": f"Bearer {key}"},
+        timeout=_TIMEOUT,
+    )
+    resp.raise_for_status()
+    pages = ((resp.json().get("data") or {}).get("webPages") or {}).get("value", [])
+    return [
+        NewsItem(p.get("name", ""), source=p.get("siteName", ""), url=p.get("url", ""),
+                 snippet=p.get("summary") or p.get("snippet", ""), published=_iso(p.get("datePublished")))
+        for p in pages
+    ]
+
+
+def _searxng(query: str, base_url: str, limit: int, days: int, _market: str) -> list[NewsItem]:
+    resp = requests.get(
+        f"{base_url.rstrip('/')}/search",
+        params={"q": query, "format": "json", "categories": "news",
+                "time_range": _freshness(days, "day", "week", "month", "year")},
+        headers=_UA,
+        timeout=_TIMEOUT,
+    )
+    resp.raise_for_status()
+    return [
+        NewsItem(r.get("title", ""), source=r.get("engine", ""), url=r.get("url", ""),
+                 snippet=r.get("content", ""), published=_iso(r.get("publishedDate")))
+        for r in resp.json().get("results", [])[:limit]
+    ]
+
+
+def _google_news(query: str, _key, limit: int, days: int, market: str) -> list[NewsItem]:
+    hl, gl, ceid = _LOCALES.get(market, _LOCALES["us"])
+    url = (f"https://news.google.com/rss/search?q={quote_plus(query)}+when:{max(1, days)}d"
+           f"&hl={hl}&gl={gl}&ceid={ceid}")
+    resp = requests.get(url, headers=_UA, timeout=_TIMEOUT)
+    resp.raise_for_status()
+    items = []
+    for it in ET.fromstring(resp.content).iter("item"):
+        title = it.findtext("title") or ""
+        source = it.findtext("source") or ""
+        if source and title.endswith(f" - {source}"):
+            title = title[: -len(source) - 3]
+        items.append(NewsItem(title, source=source, url=it.findtext("link") or "", published=_iso(it.findtext("pubDate"))))
+        if len(items) >= limit * 3:  # headroom for junk filtering
+            break
+    return items
+
+
+def _yahoo(symbol_query: str, _key, limit: int, _days: int, _market: str) -> list[NewsItem]:
+    from .news import NewsFetcher
+
+    return [
+        NewsItem(a.title, source=a.source, url=a.url, snippet=a.summary if a.summary != a.title else "",
+                 published=a.published.isoformat() if a.published else None)
+        for a in NewsFetcher().get_stock_news(symbol_query, limit=limit)
+    ]
+
+
+@dataclass
+class _Provider:
+    name: str
+    fn: Callable[..., list[NewsItem]]
+    credentials: list  # keys or base URLs; [None] for keyless providers
+    uses_ticker: bool = False  # Yahoo searches by ticker, the rest by text
+
+
+class NewsSearchService:
+    def __init__(self, environ: Optional[Mapping[str, str]] = None):
+        env = os.environ if environ is None else environ
+        self.max_age_days = int(env.get("NEWS_MAX_AGE_DAYS") or 3)
+        self.providers = [
+            p for p in (
+                _Provider("tavily", _tavily, _split(env.get("TAVILY_API_KEYS") or env.get("TAVILY_API_KEY"))),
+                _Provider("brave", _brave, _split(env.get("BRAVE_API_KEYS") or env.get("BRAVE_API_KEY"))),
+                _Provider("serpapi", _serpapi, _split(env.get("SERPAPI_API_KEYS") or env.get("SERPAPI_API_KEY"))),
+                _Provider("bocha", _bocha, _split(env.get("BOCHA_API_KEYS") or env.get("BOCHA_API_KEY"))),
+                _Provider("searxng", _searxng, _split(env.get("SEARXNG_BASE_URLS"))),
+                _Provider("google_news", _google_news, [None]),
+                _Provider("yahoo", _yahoo, [None], uses_ticker=True),
+            ) if p.credentials
+        ]
+        self._cache: dict[tuple, tuple[float, list[NewsItem]]] = {}
+        self._bad_until: dict[tuple[str, int], float] = {}
+        self._rotation: dict[str, int] = {}
+        self._lock = threading.Lock()
+
+    def provider_names(self) -> list[str]:
+        return [p.name for p in self.providers]
+
+    def search(self, query: str, *, ticker: str = "", market: str = "us", limit: int = 5,
+               days: Optional[int] = None) -> list[NewsItem]:
+        days = days or self.max_age_days
+        cache_key = (query, ticker, market, limit, days)
+        with self._lock:
+            hit = self._cache.get(cache_key)
+            if hit and time.monotonic() - hit[0] < _CACHE_SECONDS:
+                return hit[1]
+
+        items: list[NewsItem] = []
+        for provider in self.providers:
+            q = ticker if provider.uses_ticker else query
+            if not q:
+                continue
+            start = self._rotation.get(provider.name, 0)
+            self._rotation[provider.name] = start + 1
+            creds = list(enumerate(provider.credentials))
+            creds = creds[start % len(creds):] + creds[: start % len(creds)]
+            for index, cred in creds:
+                if self._bad_until.get((provider.name, index), 0) > time.monotonic():
+                    continue
+                try:
+                    found = provider.fn(q, cred, limit, days, market)
+                except Exception as exc:
+                    logger.warning("News provider %s #%d failed: %s", provider.name, index, str(exc)[:160])
+                    self._bad_until[(provider.name, index)] = time.monotonic() + 10 * 60
+                    continue
+                items = self._clean(found, provider.name, limit, days)
+                if items:
+                    break
+            if items:
+                break
+
+        with self._lock:
+            self._cache[cache_key] = (time.monotonic(), items)
+        return items
+
+    @staticmethod
+    def _clean(found: list[NewsItem], provider: str, limit: int, days: int) -> list[NewsItem]:
+        seen, out = set(), []
+        for item in found:
+            title = " ".join((item.title or "").split())
+            key = re.sub(r"\W+", "", title.lower())[:80]
+            if not title or key in seen or _JUNK.search(title) or not _fresh(item, days):
+                continue
+            seen.add(key)
+            item.title, item.provider = title, provider
+            out.append(item)
+        out.sort(key=lambda i: i.published or "", reverse=True)
+        return out[:limit]
+
+
+@lru_cache(maxsize=512)
+def company_name(symbol: str, market: str) -> str:
+    """Best human name for a news query ("Reliance Industries" beats
+    "RELIANCE"). One cached Yahoo lookup per symbol."""
+    if market != "crypto":
+        try:
+            import yfinance as yf
+
+            from ..services.analysis_run import yahoo_symbol
+
+            info = yf.Ticker(yahoo_symbol(symbol, market)).info or {}
+            name = info.get("shortName") or info.get("longName")
+            if name:
+                return name
+        except Exception:
+            pass
+    return symbol
+
+
+_SUFFIXES = re.compile(r"\b(limited|ltd\.?|inc\.?|corp(oration)?\.?|co\.?|plc|holdings?|common stock|class [a-z])\b", re.I)
+
+
+def news_query(symbol: str, market: str, name: Optional[str] = None) -> str:
+    name = _SUFFIXES.sub("", name or company_name(symbol, market))
+    name = " ".join(re.sub(r"[.,()&]+", " ", name).split()).title() or symbol
+    if market == "crypto":
+        return f"{symbol.split('-')[0]} crypto"
+    return f"{name} share" if market == "india" else f"{name} stock"
+
+
+_service: Optional[NewsSearchService] = None
+_service_lock = threading.Lock()
+
+
+def get_news_service() -> NewsSearchService:
+    global _service
+    with _service_lock:
+        if _service is None:
+            _service = NewsSearchService()
+        return _service
+
+
+def stock_news(symbol: str, market: str, limit: int = 5, name: Optional[str] = None) -> list[dict]:
+    """Recent headlines for one stock, as plain dicts (picklable/JSON-safe)."""
+    from ..services.analysis_run import yahoo_symbol
+
+    service = get_news_service()
+    items = service.search(news_query(symbol, market, name), ticker=yahoo_symbol(symbol, market),
+                           market=market, limit=limit)
+    return [i.to_dict() for i in items]

marketmyna/data/nse_deals.py

@@ -0,0 +1,138 @@
+"""NSE bulk and block deals: large trades by named clients (funds, promoters,
+brokers) that the exchange publishes daily.
+
+NSE's archive CSVs (the same host the listings use, which works from cloud
+servers) only hold the latest trading day, so every fetch is also saved to
+~/.marketmyna/deals/<date>.json (the app data volume) and history is built
+from those files. The alert monitor refreshes hourly in the evening so days
+are collected even when nobody opens the page.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+import logging
+import threading
+import time
+from datetime import date, datetime, timedelta
+from pathlib import Path
+from typing import Callable, Optional
+
+from marketmyna.core.home import app_home
+
+logger = logging.getLogger(__name__)
+
+URLS = {"bulk": "https://nsearchives.nseindia.com/content/equities/bulk.csv",
+        "block": "https://nsearchives.nseindia.com/content/equities/block.csv"}
+CACHE_SECONDS = 1800
+
+Fetch = Callable[[str], str]
+
+_last: dict[str, float] = {}
+_lock = threading.Lock()
+
+
+def _dir() -> Path:
+    path = app_home() / "deals"
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def _http(url: str) -> str:
+    import requests
+
+    resp = requests.get(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=20)
+    resp.raise_for_status()
+    return resp.text
+
+
+def _num(v) -> Optional[float]:
+    try:
+        return float(str(v).replace(",", ""))
+    except (TypeError, ValueError):
+        return None
+
+
+def parse(text: str, kind: str) -> list[dict]:
+    rows = []
+    for r in csv.DictReader(io.StringIO(text.strip())):
+        r = {k.strip(): (v or "").strip() for k, v in r.items() if k}
+        try:
+            day = datetime.strptime(r["Date"], "%d-%b-%Y").date().isoformat()
+        except (KeyError, ValueError):
+            continue
+        qty, price = _num(r.get("Quantity Traded")), _num(r.get("Trade Price / Wght. Avg. Price"))
+        rows.append({"date": day, "kind": kind, "symbol": r.get("Symbol", "").upper(), "name": r.get("Security Name"),
+                     "client": r.get("Client Name"), "side": (r.get("Buy/Sell") or "").lower(), "qty": qty, "price": price,
+                     "value": qty * price if qty and price else None,
+                     "remarks": None if r.get("Remarks") in (None, "", "-") else r.get("Remarks")})
+    return rows
+
+
+def refresh(fetch: Fetch = _http, force: bool = False) -> int:
+    """Fetch today's files (at most every 30 min) and merge them into the per-day store."""
+    with _lock:
+        if not force and time.time() - _last.get("fetch", 0) < CACHE_SECONDS:
+            return 0
+        _last["fetch"] = time.time()
+    fresh: list[dict] = []
+    for kind, url in URLS.items():
+        try:
+            fresh += parse(fetch(url), kind)
+        except Exception as exc:
+            logger.warning("NSE %s deals unavailable: %s", kind, exc)
+    by_day: dict[str, list[dict]] = {}
+    for r in fresh:
+        by_day.setdefault(r["date"], []).append(r)
+    for day, rows in by_day.items():
+        path = _dir() / f"{day}.json"
+        existing = json.loads(path.read_text(encoding="utf-8")) if path.exists() else []
+        seen = {json.dumps(r, sort_keys=True) for r in existing}
+        merged = existing + [r for r in rows if json.dumps(r, sort_keys=True) not in seen]
+        path.write_text(json.dumps(merged), encoding="utf-8")
+    return len(fresh)
+
+
+def recent(days: int = 30, symbol: Optional[str] = None, today: Optional[date] = None, fetch: Fetch = _http) -> dict:
+    refresh(fetch)
+    today = today or date.today()
+    since = (today - timedelta(days=days)).isoformat()
+    rows: list[dict] = []
+    stored = sorted(p.stem for p in _dir().glob("*.json"))
+    for day in stored:
+        if day >= since:
+            rows += json.loads((_dir() / f"{day}.json").read_text(encoding="utf-8"))
+    if symbol:
+        rows = [r for r in rows if r["symbol"] == symbol.upper()]
+    rows = _merge_kinds(rows)
+    rows.sort(key=lambda r: (r["date"], r.get("value") or 0), reverse=True)
+    return {"days": days, "history_from": stored[0] if stored else None, "latest": stored[-1] if stored else None,
+            "count": len(rows), "deals": rows[:300]}
+
+
+def _merge_kinds(rows: list[dict]) -> list[dict]:
+    """A block trade large enough to be a bulk deal is listed in both files:
+    keep one row, kind "block & bulk"."""
+    merged: dict[tuple, dict] = {}
+    for r in rows:
+        key = (r["date"], r["symbol"], r.get("client"), r["side"], r.get("qty"), r.get("price"))
+        if key in merged and merged[key]["kind"] != r["kind"]:
+            merged[key] = {**merged[key], "kind": "block & bulk", "remarks": merged[key].get("remarks") or r.get("remarks")}
+        else:
+            merged.setdefault(key, r)
+    return list(merged.values())
+
+
+def maybe_refresh(now: Optional[datetime] = None) -> None:
+    """Alert-loop hook: NSE posts the files after the close; refresh hourly 16:00-21:59 IST."""
+    from zoneinfo import ZoneInfo
+
+    local = (now or datetime.now(ZoneInfo("Asia/Kolkata"))).astimezone(ZoneInfo("Asia/Kolkata"))
+    with _lock:
+        due = time.time() - _last.get("loop", 0) >= 3600
+    if 16 <= local.hour < 22 and local.weekday() < 5 and due:
+        with _lock:
+            _last["loop"] = time.time()
+        refresh(force=True)