agmarknet 0.1.0__py3-none-any.whl

agmarknet/__init__.py

@@ -0,0 +1,23 @@
+"""Python SDK for the official AGMARKNET REST API."""
+
+from agmarknet.client import Agmarknet
+from agmarknet.exceptions import (
+    AgmarknetAPIError,
+    AgmarknetError,
+    AgmarknetLookupError,
+    AgmarknetResponseError,
+    AgmarknetValidationError,
+)
+from agmarknet.historical import HistoricalDownloader
+
+__all__ = [
+    "Agmarknet",
+    "AgmarknetAPIError",
+    "AgmarknetError",
+    "AgmarknetLookupError",
+    "AgmarknetResponseError",
+    "AgmarknetValidationError",
+    "HistoricalDownloader",
+]
+
+__version__ = "0.1.0"

agmarknet/client.py

@@ -0,0 +1,250 @@
+"""Public client for the agmarknet package."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from datetime import date
+
+import pandas as pd
+import requests
+
+from agmarknet.constants import (
+    BASE_URL,
+    DEFAULT_GRADE_ID,
+    DEFAULT_TIMEOUT,
+    DEFAULT_VARIETY_ID,
+    FILTERS_ENDPOINT,
+    REPORT_ENDPOINT,
+)
+from agmarknet.dates import resolve_date_range
+from agmarknet.exceptions import AgmarknetLookupError
+from agmarknet.filters import FilterTables
+from agmarknet.historical import HistoricalDownloader, ProgressCallback
+from agmarknet.lookup import Lookup, LookupValue, LookupValues
+from agmarknet.models import ReportRequest
+from agmarknet.pagination import PaginatedReportDownloader
+from agmarknet.session import AgmarknetSession
+
+
+class Agmarknet:
+    """Client for the official AGMARKNET REST API.
+
+    Args:
+        base_url: API base URL. Override mainly for tests.
+        timeout: Request timeout in seconds.
+        session: Optional existing :class:`requests.Session`.
+    """
+
+    def __init__(
+        self,
+        *,
+        base_url: str = BASE_URL,
+        timeout: float = DEFAULT_TIMEOUT,
+        session: requests.Session | None = None,
+    ) -> None:
+        self._session = AgmarknetSession(
+            base_url=base_url,
+            timeout=timeout,
+            session=session,
+        )
+        self._filters: FilterTables | None = None
+        self._page_downloader = PaginatedReportDownloader(
+            lambda payload: self._session.post_json(REPORT_ENDPOINT, payload)
+        )
+
+    def filters(self) -> FilterTables:
+        """Return all AGMARKNET lookup tables as pandas DataFrames."""
+
+        payload = self._session.get_json(FILTERS_ENDPOINT)
+        self._filters = FilterTables.from_response(payload)
+        return self._filters
+
+    def lookup(self) -> Lookup:
+        """Return a lookup engine backed by the official filters endpoint."""
+
+        return Lookup(self._get_filters())
+
+    def report(
+        self,
+        *,
+        from_date: str | date | None = None,
+        to_date: str | date | None = None,
+        start: str | date | None = None,
+        end: str | date | None = None,
+        data_type: str | int = "price",
+        commodity: LookupValue,
+        state: LookupValues,
+        district: LookupValues | None = None,
+        market: LookupValues | None = None,
+        grade: LookupValues | None = None,
+        variety: LookupValues | None = None,
+        group: LookupValue | None = None,
+        page: int | str = 1,
+        limit: int | str = 1000,
+        progress: bool | ProgressCallback = False,
+        timeout: float | None = None,
+    ) -> pd.DataFrame:
+        """Return a daily price and arrival report as a pandas DataFrame.
+
+        Filter arguments accept AGMARKNET IDs or names. Names are resolved from
+        the official filters endpoint using exact, case-insensitive, then fuzzy
+        matching.
+        """
+
+        start_date, end_date = resolve_date_range(
+            from_date=from_date,
+            to_date=to_date,
+            start=start,
+            end=end,
+        )
+        resolved = self._resolve_report_filters(
+            group=group,
+            commodity=commodity,
+            state=state,
+            district=district,
+            market=market,
+            grade=grade or DEFAULT_GRADE_ID,
+            variety=variety or DEFAULT_VARIETY_ID,
+        )
+        request = ReportRequest.from_user_input(
+            from_date=start_date,
+            to_date=end_date,
+            data_type=data_type,
+            group=resolved["group"],
+            commodity=resolved["commodity"],
+            state=resolved["state"],
+            district=resolved["district"],
+            market=resolved["market"],
+            grade=resolved["grade"],
+            variety=resolved["variety"],
+            page=1,
+            limit=limit,
+        )
+        original_timeout = self._session.timeout
+        if timeout is not None:
+            self._session.timeout = timeout
+        try:
+            downloader = HistoricalDownloader(self._fetch_paginated_report)
+            return downloader.download(request, progress=progress)
+        finally:
+            self._session.timeout = original_timeout
+
+    def _fetch_paginated_report(
+        self,
+        request: ReportRequest,
+        progress_callback: Callable[[int, int], None] | None = None,
+    ) -> pd.DataFrame:
+        return self._page_downloader.download(
+            request, progress_callback=progress_callback
+        )
+
+    def _get_filters(self) -> FilterTables:
+        if self._filters is None:
+            return self.filters()
+        return self._filters
+
+    def _resolve_report_filters(
+        self,
+        *,
+        group: LookupValue | None,
+        commodity: LookupValue,
+        state: LookupValues,
+        district: LookupValues | None,
+        market: LookupValues | None,
+        grade: LookupValues,
+        variety: LookupValues,
+    ) -> dict[str, str | list[str]]:
+        if not _needs_lookup(group, commodity, state, district, market, grade, variety):
+            return {
+                "group": str(group),
+                "commodity": str(commodity),
+                "state": _to_text_list(state),
+                "district": _to_text_list(district),
+                "market": _to_text_list(market),
+                "grade": _to_text_list(grade),
+                "variety": _to_text_list(variety),
+            }
+
+        lookup = self.lookup()
+        commodity_result = lookup.commodity(commodity)
+        resolved_states = lookup.states(state)
+        resolved_districts = (
+            lookup.districts_for_state_ids(resolved_states)
+            if _is_all(district)
+            else lookup.districts(district, state_ids=resolved_states)
+        )
+        resolved_markets = (
+            lookup.markets_for_district_ids(resolved_districts)
+            if _is_all(market)
+            else lookup.markets(market, district_ids=resolved_districts)
+        )
+        resolved_group = (
+            lookup.commodity_group(group).value
+            if group is not None
+            else _infer_group_id(commodity_result.row)
+        )
+        return {
+            "group": resolved_group,
+            "commodity": commodity_result.value,
+            "state": resolved_states,
+            "district": _require_values(resolved_districts, "district"),
+            "market": _require_values(resolved_markets, "market"),
+            "grade": lookup.grades(grade),
+            "variety": lookup.varieties(variety),
+        }
+
+
+def _needs_lookup(*values: object) -> bool:
+    return any(_value_needs_lookup(value) for value in values)
+
+
+def _value_needs_lookup(value: object) -> bool:
+    if value is None:
+        return True
+    if isinstance(value, int):
+        return False
+    if isinstance(value, str):
+        return not value.strip().isdigit()
+    try:
+        return any(
+            _value_needs_lookup(item)
+            for item in value  # type: ignore[arg-type]
+        )
+    except TypeError:
+        return True
+
+
+def _to_text_list(values: LookupValues) -> list[str]:
+    if isinstance(values, str | int):
+        return [str(values)]
+    return [str(value) for value in values]
+
+
+def _is_all(value: object) -> bool:
+    if value is None:
+        return True
+    return isinstance(value, str) and value.strip().casefold() in {"all", "*"}
+
+
+def _require_values(values: list[str], field_name: str) -> list[str]:
+    if not values:
+        raise AgmarknetLookupError(f"No {field_name} values matched the filters.")
+    return values
+
+
+def _infer_group_id(row: dict[str, object]) -> str:
+    candidates = (
+        "group",
+        "group_id",
+        "cmdt_grp_id",
+        "cmdt_group_id",
+        "commodity_group_id",
+    )
+    for candidate in candidates:
+        value = row.get(candidate)
+        if value is not None and str(value).strip():
+            return str(value).strip()
+    raise AgmarknetLookupError(
+        "group is required because the commodity lookup row does not include "
+        "a commodity group ID."
+    )

agmarknet/constants.py

@@ -0,0 +1,35 @@
+"""Constants for AGMARKNET API access."""
+
+from __future__ import annotations
+
+BASE_URL = "https://api.agmarknet.gov.in/v1"
+PORTAL_URL = "https://agmarknet.gov.in"
+
+FILTERS_ENDPOINT = "/daily-price-arrival/filters"
+REPORT_ENDPOINT = "/daily-price-arrival/report"
+
+DEFAULT_TIMEOUT = 60.0
+DEFAULT_PAGE = 1
+DEFAULT_LIMIT = 1000
+DEFAULT_GRADE_ID = "100003"
+DEFAULT_VARIETY_ID = "100007"
+
+DATA_TYPE_PRICE = "100004"
+DATA_TYPE_ARRIVAL = "100005"
+DATA_TYPE_BOTH = "100006"
+DEFAULT_DATA_TYPE = DATA_TYPE_PRICE
+
+DATA_TYPES = {
+    "price": DATA_TYPE_PRICE,
+    "arrival": DATA_TYPE_ARRIVAL,
+    "both": DATA_TYPE_BOTH,
+    DATA_TYPE_PRICE: DATA_TYPE_PRICE,
+    DATA_TYPE_ARRIVAL: DATA_TYPE_ARRIVAL,
+    DATA_TYPE_BOTH: DATA_TYPE_BOTH,
+}
+
+USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/149.0.0.0 Safari/537.36"
+)

agmarknet/dates.py

@@ -0,0 +1,45 @@
+"""Date range helpers for public report arguments."""
+
+from __future__ import annotations
+
+from datetime import date
+
+from agmarknet.exceptions import AgmarknetValidationError
+from agmarknet.validators import normalize_date
+
+
+def resolve_date_range(
+    *,
+    from_date: str | date | None,
+    to_date: str | date | None,
+    start: str | date | None,
+    end: str | date | None,
+) -> tuple[str, str]:
+    """Resolve legacy and friendly date aliases into ISO date strings."""
+
+    resolved_start = _resolve_date_alias(from_date, start, "from_date", "start")
+    resolved_end = _resolve_date_alias(to_date, end, "to_date", "end")
+    start_text = normalize_date(resolved_start, "start")
+    end_text = normalize_date(resolved_end, "end")
+    if date.fromisoformat(start_text) > date.fromisoformat(end_text):
+        raise AgmarknetValidationError("start must be on or before end.")
+    return start_text, end_text
+
+
+def _resolve_date_alias(
+    primary: str | date | None,
+    alias: str | date | None,
+    primary_name: str,
+    alias_name: str,
+) -> str | date:
+    if primary is None and alias is None:
+        raise AgmarknetValidationError(f"{primary_name} or {alias_name} is required.")
+    if primary is not None and alias is not None:
+        primary_text = normalize_date(primary, primary_name)
+        alias_text = normalize_date(alias, alias_name)
+        if primary_text != alias_text:
+            raise AgmarknetValidationError(
+                f"{primary_name} and {alias_name} cannot disagree."
+            )
+        return primary_text
+    return primary if primary is not None else alias

agmarknet/exceptions.py

@@ -0,0 +1,36 @@
+"""Exception hierarchy for agmarknet."""
+
+from __future__ import annotations
+
+
+class AgmarknetError(Exception):
+    """Base exception for all package-specific errors."""
+
+
+class AgmarknetValidationError(AgmarknetError, ValueError):
+    """Raised when user input cannot be converted to a valid API request."""
+
+
+class AgmarknetLookupError(AgmarknetValidationError):
+    """Raised when a lookup value cannot be resolved unambiguously."""
+
+
+class AgmarknetAPIError(AgmarknetError):
+    """Raised when the AGMARKNET API returns an HTTP or transport error."""
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        status_code: int | None = None,
+        url: str | None = None,
+        response_text: str | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.status_code = status_code
+        self.url = url
+        self.response_text = response_text
+
+
+class AgmarknetResponseError(AgmarknetError):
+    """Raised when the AGMARKNET API response has an unexpected shape."""

agmarknet/filters.py

@@ -0,0 +1,107 @@
+"""Filter lookup table parsing."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator, Mapping
+from dataclasses import dataclass
+from typing import Any
+
+import pandas as pd
+
+from agmarknet.exceptions import AgmarknetResponseError
+
+FILTER_TABLES = {
+    "commodity_groups": "cmdt_group_data",
+    "commodities": "cmdt_data",
+    "states": "state_data",
+    "districts": "district_data",
+    "markets": "market_data",
+    "varieties": "variety_data",
+    "grades": "grade_data",
+    "types": "type_data",
+    "ranges": "range_data",
+    "notes": "note_data",
+}
+
+RAW_FILTER_TABLES = {
+    api_name: public_name for public_name, api_name in FILTER_TABLES.items()
+}
+
+
+@dataclass(frozen=True)
+class FilterTables:
+    """Container for AGMARKNET lookup tables as pandas DataFrames."""
+
+    commodity_groups: pd.DataFrame
+    commodities: pd.DataFrame
+    states: pd.DataFrame
+    districts: pd.DataFrame
+    markets: pd.DataFrame
+    varieties: pd.DataFrame
+    grades: pd.DataFrame
+    types: pd.DataFrame
+    ranges: pd.DataFrame
+    notes: pd.DataFrame
+
+    def __getitem__(self, key: str) -> pd.DataFrame:
+        """Return a lookup table by friendly name or raw API key."""
+
+        return getattr(self, _normalize_key(key))
+
+    def __iter__(self) -> Iterator[str]:
+        """Iterate over raw AGMARKNET filter table keys."""
+
+        return iter(RAW_FILTER_TABLES)
+
+    def __len__(self) -> int:
+        """Return the number of lookup tables."""
+
+        return len(FILTER_TABLES)
+
+    def keys(self) -> list[str]:
+        """Return raw AGMARKNET filter table keys."""
+
+        return list(RAW_FILTER_TABLES)
+
+    @classmethod
+    def from_response(cls, payload: Mapping[str, Any]) -> FilterTables:
+        """Parse a filters endpoint response."""
+
+        data = payload.get("data")
+        if not isinstance(data, Mapping):
+            raise AgmarknetResponseError("Filters response missing data object.")
+
+        tables = {
+            public_name: _records_to_frame(data.get(api_name), api_name)
+            for public_name, api_name in FILTER_TABLES.items()
+        }
+        return cls(**tables)
+
+    def as_dict(self) -> dict[str, pd.DataFrame]:
+        """Return lookup tables keyed by friendly table names."""
+
+        return {name: getattr(self, name) for name in FILTER_TABLES}
+
+    def as_raw_dict(self) -> dict[str, pd.DataFrame]:
+        """Return lookup tables keyed by raw AGMARKNET API names."""
+
+        return {
+            api_name: getattr(self, public_name)
+            for api_name, public_name in RAW_FILTER_TABLES.items()
+        }
+
+
+def _records_to_frame(records: Any, table_name: str) -> pd.DataFrame:
+    if records is None:
+        return pd.DataFrame()
+    if not isinstance(records, list):
+        raise AgmarknetResponseError(f"{table_name} must be a list.")
+    return pd.DataFrame.from_records(records)
+
+
+def _normalize_key(key: str) -> str:
+    if key in FILTER_TABLES:
+        return key
+    if key in RAW_FILTER_TABLES:
+        return RAW_FILTER_TABLES[key]
+    raise KeyError(key)

agmarknet/historical.py

@@ -0,0 +1,162 @@
+"""Historical report downloading by yearly chunks."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import date
+from sys import stderr
+
+import pandas as pd
+
+from agmarknet.models import ReportRequest
+from agmarknet.report import merge_report_frames
+
+ProgressCallback = Callable[[int, int, "DateChunk"], None]
+
+
+@dataclass(frozen=True)
+class DateChunk:
+    """Inclusive date range used for one API request window."""
+
+    from_date: date
+    to_date: date
+
+
+class HistoricalDownloader:
+    """Download reports sequentially across yearly API request chunks.
+
+    Args:
+        fetch_report: Callable that downloads one request, including pagination.
+    """
+
+    def __init__(
+        self,
+        fetch_report: Callable[
+            [ReportRequest, Callable[[int, int], None] | None], pd.DataFrame
+        ],
+    ) -> None:
+        self._fetch_report = fetch_report
+
+    def download(
+        self,
+        request: ReportRequest,
+        *,
+        progress: bool | ProgressCallback = False,
+    ) -> pd.DataFrame:
+        """Download all yearly chunks and return one de-duplicated DataFrame."""
+
+        chunks = split_yearly_chunks(
+            date.fromisoformat(request.from_date),
+            date.fromisoformat(request.to_date),
+        )
+        frames: list[pd.DataFrame] = []
+        total = len(chunks)
+        for index, chunk in enumerate(chunks, start=1):
+            _emit_progress(progress, index, total, chunk)
+
+            def make_page_progress(idx=index, tot=total, chk=chunk):
+                def page_progress(current_page: int, total_pages: int):
+                    if progress and not callable(progress):
+                        is_last = current_page == total_pages
+                        print(
+                            "Downloading "
+                            f"{chk.from_date.isoformat()} to {chk.to_date.isoformat()} "
+                            f"({idx}/{tot}) - Page {current_page}/{total_pages}",
+                            file=stderr,
+                            end="\n" if is_last else "\r",
+                        )
+
+                return page_progress
+
+            page_cb = make_page_progress() if progress else None
+
+            frames.append(
+                self._fetch_report(
+                    request.with_dates(
+                        from_date=chunk.from_date.isoformat(),
+                        to_date=chunk.to_date.isoformat(),
+                    ),
+                    page_cb,
+                )
+            )
+        return finalize_historical_frame(frames)
+
+
+def split_yearly_chunks(start: date, end: date) -> list[DateChunk]:
+    """Split an inclusive date range into calendar-year chunks."""
+
+    chunks: list[DateChunk] = []
+    current = start
+    while current <= end:
+        year_end = date(current.year, 12, 31)
+        chunk_end = min(year_end, end)
+        chunks.append(DateChunk(from_date=current, to_date=chunk_end))
+        current = date(current.year + 1, 1, 1)
+    return chunks
+
+
+def finalize_historical_frame(frames: list[pd.DataFrame]) -> pd.DataFrame:
+    """Merge, sort chronologically when possible, and remove duplicates."""
+
+    merged = merge_report_frames(frames)
+    if merged.empty:
+        return merged
+
+    attrs = dict(merged.attrs)
+    merged = _sort_chronologically(merged)
+    merged = merged.drop_duplicates(ignore_index=True)
+    merged.attrs.update(attrs)
+    return merged
+
+
+def _sort_chronologically(frame: pd.DataFrame) -> pd.DataFrame:
+    date_column = _date_column(frame)
+    if date_column is None:
+        return frame.reset_index(drop=True)
+
+    sorted_frame = frame.copy()
+    helper_column = "__agmarknet_sort_date"
+    sorted_frame[helper_column] = pd.to_datetime(
+        sorted_frame[date_column],
+        errors="coerce",
+        dayfirst=True,
+    )
+    sorted_frame = sorted_frame.sort_values(
+        by=[helper_column],
+        kind="stable",
+        na_position="last",
+    )
+    return sorted_frame.drop(columns=[helper_column]).reset_index(drop=True)
+
+
+def _date_column(frame: pd.DataFrame) -> str | None:
+    normalized = {_normalize_column(column): column for column in frame.columns}
+    candidates = (
+        "arrival_date",
+        "date",
+        "report_date",
+        "price_date",
+        "modal_date",
+    )
+    for candidate in candidates:
+        if candidate in normalized:
+            return str(normalized[candidate])
+    for column in frame.columns:
+        if "date" in _normalize_column(column):
+            return str(column)
+    return None
+
+
+def _normalize_column(value: object) -> str:
+    return str(value).strip().casefold().replace(" ", "_")
+
+
+def _emit_progress(
+    progress: bool | ProgressCallback,
+    index: int,
+    total: int,
+    chunk: DateChunk,
+) -> None:
+    if callable(progress):
+        progress(index, total, chunk)