desidata 0.1.0__tar.gz

desidata-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 DesiData (https://www.desidata.in)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

desidata-0.1.0/PKG-INFO

@@ -0,0 +1,165 @@
+Metadata-Version: 2.4
+Name: desidata
+Version: 0.1.0
+Summary: Load Indian datasets from desidata.in in one line — search the catalogue, download CSVs, get pandas DataFrames.
+Author-email: DesiData <hello@desidata.in>
+License: MIT
+Project-URL: Homepage, https://www.desidata.in
+Project-URL: Documentation, https://www.desidata.in
+Project-URL: Repository, https://github.com/krishnakaushik195/desidata-py
+Project-URL: Issues, https://github.com/krishnakaushik195/desidata-py/issues
+Keywords: india,indian,datasets,data,pandas,open-data,csv
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Education
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: pandas
+Requires-Dist: pandas>=1.3; extra == "pandas"
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Dynamic: license-file
+
+# desidata
+
+**Load Indian datasets from [desidata.in](https://www.desidata.in) in one line.**
+
+```python
+import desidata
+
+df = desidata.load("gender-policy-of-nabard-question-and-answer-dataset")
+```
+
+That's it — no URL copying, no sign-in, no API key. You get a pandas
+DataFrame of cleaned, India-focused public data: economy, agriculture,
+health, education, transport, demographics and more.
+
+## Install
+
+```bash
+pip install desidata
+```
+
+The core package has **zero dependencies**, so it installs instantly.
+`load()` needs pandas (almost always already installed) and will tell you
+exactly what to run if it is missing:
+
+```bash
+pip install desidata pandas
+```
+
+## Usage
+
+```python
+import desidata
+
+# Search the catalogue from Python
+results = desidata.search("census")
+for row in results:
+    print(row["slug"], row["category"], row["rows"])
+
+# Browse everything, or one category
+all_datasets = desidata.catalog()
+economy = desidata.catalog(category="Economy")
+
+# Metadata for one dataset
+info = desidata.info("gender-policy-of-nabard-question-and-answer-dataset")
+print(info["name"], info["size"], info["downloads"])
+
+# Load as a DataFrame (kwargs pass through to pandas.read_csv)
+df = desidata.load("gender-policy-of-nabard-question-and-answer-dataset")
+df = desidata.load("some-dataset", dtype={"year": int}, parse_dates=["date"])
+
+# Raw bytes, or save straight to disk
+data = desidata.download("some-dataset")            # bytes
+path = desidata.download("some-dataset", "a.csv")   # writes the file
+```
+
+Full dataset URLs work anywhere a slug does — paste straight from the browser:
+
+```python
+desidata.load("https://www.desidata.in/datasets/gender-policy-of-nabard-question-and-answer-dataset")
+```
+
+### Caching
+
+Loads are cached in `~/.desidata/cache` for 24 hours, so re-loading in a
+notebook (or in a classroom on weak wifi) is instant and offline-safe.
+
+```python
+desidata.load("some-dataset", refresh=True)  # bypass the cache
+desidata.clear_cache()                       # wipe all cached files
+```
+
+Set `DESIDATA_CACHE_DIR` to move the cache somewhere else.
+
+## Command line
+
+```bash
+desidata search nabard
+desidata info gender-policy-of-nabard-question-and-answer-dataset
+desidata catalog --category Agriculture
+desidata download gender-policy-of-nabard-question-and-answer-dataset -o nabard.csv
+desidata clear-cache
+```
+
+## Error handling
+
+Every error the package raises subclasses `desidata.DesiDataError`:
+
+| Exception | Meaning |
+| --- | --- |
+| `DesiDataNotFound` | That dataset doesn't exist (HTTP 404) |
+| `DesiDataConnectionError` | Network/DNS/timeout failure |
+| `DesiDataServerError` | desidata.in returned an unexpected error |
+
+```python
+try:
+    df = desidata.load("some-slug")
+except desidata.DesiDataNotFound:
+    print("Check the slug with desidata.search(...)")
+```
+
+## Development
+
+```bash
+git clone https://github.com/krishnakaushik195/desidata-py
+cd desidata-py
+pip install -e .[dev]
+pytest                    # offline unit tests
+RUN_LIVE=1 pytest tests/test_live.py -v   # integration tests against the real site
+```
+
+Environment overrides used by tests and local development:
+
+| Variable | Purpose |
+| --- | --- |
+| `DESIDATA_BASE_URL` | Point the client at a local/dev server |
+| `DESIDATA_CACHE_DIR` | Relocate the cache |
+| `DESIDATA_TIMEOUT` | Default request timeout in seconds |
+
+## Releasing
+
+Maintainers only:
+
+```bash
+pip install --upgrade build twine
+python -m build
+twine upload dist/*
+```
+
+## Licence
+
+MIT — see [LICENSE](LICENSE). The data itself belongs to its sources; each
+dataset page on [desidata.in](https://www.desidata.in) lists its source and licence.

desidata-0.1.0/README.md

@@ -0,0 +1,132 @@
+# desidata
+
+**Load Indian datasets from [desidata.in](https://www.desidata.in) in one line.**
+
+```python
+import desidata
+
+df = desidata.load("gender-policy-of-nabard-question-and-answer-dataset")
+```
+
+That's it — no URL copying, no sign-in, no API key. You get a pandas
+DataFrame of cleaned, India-focused public data: economy, agriculture,
+health, education, transport, demographics and more.
+
+## Install
+
+```bash
+pip install desidata
+```
+
+The core package has **zero dependencies**, so it installs instantly.
+`load()` needs pandas (almost always already installed) and will tell you
+exactly what to run if it is missing:
+
+```bash
+pip install desidata pandas
+```
+
+## Usage
+
+```python
+import desidata
+
+# Search the catalogue from Python
+results = desidata.search("census")
+for row in results:
+    print(row["slug"], row["category"], row["rows"])
+
+# Browse everything, or one category
+all_datasets = desidata.catalog()
+economy = desidata.catalog(category="Economy")
+
+# Metadata for one dataset
+info = desidata.info("gender-policy-of-nabard-question-and-answer-dataset")
+print(info["name"], info["size"], info["downloads"])
+
+# Load as a DataFrame (kwargs pass through to pandas.read_csv)
+df = desidata.load("gender-policy-of-nabard-question-and-answer-dataset")
+df = desidata.load("some-dataset", dtype={"year": int}, parse_dates=["date"])
+
+# Raw bytes, or save straight to disk
+data = desidata.download("some-dataset")            # bytes
+path = desidata.download("some-dataset", "a.csv")   # writes the file
+```
+
+Full dataset URLs work anywhere a slug does — paste straight from the browser:
+
+```python
+desidata.load("https://www.desidata.in/datasets/gender-policy-of-nabard-question-and-answer-dataset")
+```
+
+### Caching
+
+Loads are cached in `~/.desidata/cache` for 24 hours, so re-loading in a
+notebook (or in a classroom on weak wifi) is instant and offline-safe.
+
+```python
+desidata.load("some-dataset", refresh=True)  # bypass the cache
+desidata.clear_cache()                       # wipe all cached files
+```
+
+Set `DESIDATA_CACHE_DIR` to move the cache somewhere else.
+
+## Command line
+
+```bash
+desidata search nabard
+desidata info gender-policy-of-nabard-question-and-answer-dataset
+desidata catalog --category Agriculture
+desidata download gender-policy-of-nabard-question-and-answer-dataset -o nabard.csv
+desidata clear-cache
+```
+
+## Error handling
+
+Every error the package raises subclasses `desidata.DesiDataError`:
+
+| Exception | Meaning |
+| --- | --- |
+| `DesiDataNotFound` | That dataset doesn't exist (HTTP 404) |
+| `DesiDataConnectionError` | Network/DNS/timeout failure |
+| `DesiDataServerError` | desidata.in returned an unexpected error |
+
+```python
+try:
+    df = desidata.load("some-slug")
+except desidata.DesiDataNotFound:
+    print("Check the slug with desidata.search(...)")
+```
+
+## Development
+
+```bash
+git clone https://github.com/krishnakaushik195/desidata-py
+cd desidata-py
+pip install -e .[dev]
+pytest                    # offline unit tests
+RUN_LIVE=1 pytest tests/test_live.py -v   # integration tests against the real site
+```
+
+Environment overrides used by tests and local development:
+
+| Variable | Purpose |
+| --- | --- |
+| `DESIDATA_BASE_URL` | Point the client at a local/dev server |
+| `DESIDATA_CACHE_DIR` | Relocate the cache |
+| `DESIDATA_TIMEOUT` | Default request timeout in seconds |
+
+## Releasing
+
+Maintainers only:
+
+```bash
+pip install --upgrade build twine
+python -m build
+twine upload dist/*
+```
+
+## Licence
+
+MIT — see [LICENSE](LICENSE). The data itself belongs to its sources; each
+dataset page on [desidata.in](https://www.desidata.in) lists its source and licence.

desidata-0.1.0/pyproject.toml

@@ -0,0 +1,45 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "desidata"
+version = "0.1.0"
+description = "Load Indian datasets from desidata.in in one line — search the catalogue, download CSVs, get pandas DataFrames."
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+authors = [{ name = "DesiData", email = "hello@desidata.in" }]
+keywords = ["india", "indian", "datasets", "data", "pandas", "open-data", "csv"]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Developers",
+  "Intended Audience :: Science/Research",
+  "Intended Audience :: Education",
+  "License :: OSI Approved :: MIT License",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: Scientific/Engineering",
+]
+dependencies = []
+
+[project.urls]
+Homepage = "https://www.desidata.in"
+Documentation = "https://www.desidata.in"
+Repository = "https://github.com/krishnakaushik195/desidata-py"
+Issues = "https://github.com/krishnakaushik195/desidata-py/issues"
+
+[project.optional-dependencies]
+pandas = ["pandas>=1.3"]
+dev = ["pytest>=7"]
+
+[project.scripts]
+desidata = "desidata.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]

desidata-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

desidata-0.1.0/src/desidata/__init__.py

@@ -0,0 +1,113 @@
+"""desidata — load Indian datasets from desidata.in in one line.
+
+    import desidata
+
+    df = desidata.load("gender-policy-of-nabard-question-and-answer-dataset")
+    results = desidata.search("census")
+
+The core install is dependency-free; `load()` needs pandas and raises a
+helpful error telling you to install it if it is missing.
+"""
+from __future__ import annotations
+
+import io
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
+from .client import (  # noqa: F401  (re-exports are the public API)
+    DEFAULT_BASE_URL,
+    __version__,
+    base_url,
+    cache_directory,
+    clear_cache,
+    fetch_catalog,
+    fetch_csv_bytes,
+    fetch_info,
+    search_datasets,
+)
+from .client import (  # noqa: F401
+    DesiDataConnectionError,
+    DesiDataError,
+    DesiDataNotFound,
+    DesiDataServerError,
+)
+
+__all__ = [
+    "load",
+    "download",
+    "info",
+    "search",
+    "catalog",
+    "clear_cache",
+    "base_url",
+    "cache_directory",
+    "DEFAULT_BASE_URL",
+    "__version__",
+    "DesiDataError",
+    "DesiDataConnectionError",
+    "DesiDataNotFound",
+    "DesiDataServerError",
+]
+
+
+def load(
+    slug: str,
+    refresh: bool = False,
+    timeout: Optional[float] = None,
+    **pandas_kwargs: Any,
+):
+    """Load a dataset as a pandas DataFrame.
+
+    `slug` may be a bare slug or a full desidata.in dataset URL. Extra keyword
+    arguments pass straight through to `pandas.read_csv` (e.g. `dtype=`,
+    `parse_dates=`). Loads are cached for 24 hours; pass `refresh=True` to
+    re-download.
+    """
+    try:
+        import pandas
+    except ImportError as error:  # pragma: no cover - depends on environment
+        raise ImportError(
+            "desidata.load() needs pandas. Install it with: pip install pandas"
+        ) from error
+
+    data = fetch_csv_bytes(slug, refresh=refresh, timeout=timeout)
+    return pandas.read_csv(io.BytesIO(data), **pandas_kwargs)
+
+
+def download(
+    slug: str,
+    path: Optional[Union[str, Path]] = None,
+    refresh: bool = False,
+    timeout: Optional[float] = None,
+) -> Union[bytes, Path]:
+    """Download the raw CSV.
+
+    With `path`, writes the file there and returns the Path; without it,
+    returns the CSV bytes. Uses the same 24-hour cache as `load()`.
+    """
+    data = fetch_csv_bytes(slug, refresh=refresh, timeout=timeout)
+    if path is None:
+        return data
+    destination = Path(path)
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    destination.write_bytes(data)
+    return destination
+
+
+def info(slug: str, timeout: Optional[float] = None) -> Dict[str, Any]:
+    """Metadata for one dataset: name, description, format, size, downloads, tags."""
+    return fetch_info(slug, timeout=timeout)
+
+
+def search(query: str, timeout: Optional[float] = None) -> List[Dict[str, Any]]:
+    """Search dataset titles and descriptions."""
+    return search_datasets(query, timeout=timeout)
+
+
+def catalog(
+    category: Optional[str] = None,
+    language: Optional[str] = None,
+    timeout: Optional[float] = None,
+) -> List[Dict[str, Any]]:
+    """List every published dataset, optionally filtered."""
+    return fetch_catalog(category=category, language=language, timeout=timeout)

desidata-0.1.0/src/desidata/__main__.py

@@ -0,0 +1,5 @@
+"""python -m desidata support."""
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

desidata-0.1.0/src/desidata/cli.py

@@ -0,0 +1,115 @@
+"""Command line interface: desidata <command> [args].
+
+Zero dependencies (argparse), so the CLI ships with the base install.
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+from typing import List, Optional
+
+from . import __version__
+from .client import (
+    DesiDataError,
+    clear_cache,
+    fetch_catalog,
+    fetch_csv_bytes,
+    fetch_info,
+    search_datasets,
+)
+
+# Windows consoles default to legacy code pages; dataset titles contain ₹ etc.
+for _stream in (sys.stdout, sys.stderr):
+    if hasattr(_stream, "reconfigure"):
+        _stream.reconfigure(encoding="utf-8", errors="replace")
+
+
+def _print_table(rows: List[dict]) -> None:
+    if not rows:
+        print("No datasets found.")
+        return
+    slug_width = min(max(len(row["slug"]) for row in rows) + 2, 44)
+    category_width = min(max(len(str(row.get("category", ""))) for row in rows) + 2, 14)
+    print(f"{'slug':<{slug_width}}{'category':<{category_width}}{'rows':>7}  title")
+    for row in rows:
+        slug = row["slug"]
+        if len(slug) > slug_width - 1:
+            slug = slug[: slug_width - 4] + "..."
+        category = str(row.get("category", ""))
+        if len(category) > category_width - 1:
+            category = category[: category_width - 1]
+        title = str(row.get("title", "")).replace("\n", " ")
+        print(f"{slug:<{slug_width}}{category:<{category_width}}{row.get('rows', 0):>7}  {title}")
+
+
+def _print_info(metadata: dict) -> None:
+    order = ["slug", "name", "description", "category", "language", "format", "size", "downloads", "tags", "download_url"]
+    for key in order:
+        if key in metadata:
+            value = metadata[key]
+            if isinstance(value, list):
+                value = ", ".join(str(item) for item in value)
+            print(f"{key:>14}: {value}")
+    for key, value in metadata.items():
+        if key not in order:
+            print(f"{key:>14}: {value}")
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="desidata",
+        description="Load Indian datasets from desidata.in (https://www.desidata.in).",
+    )
+    parser.add_argument("--version", action="version", version=f"desidata {__version__}")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    search_parser = subparsers.add_parser("search", help="Search dataset titles and descriptions")
+    search_parser.add_argument("query")
+
+    info_parser = subparsers.add_parser("info", help="Show metadata for one dataset")
+    info_parser.add_argument("slug", help="Dataset slug or full desidata.in URL")
+
+    catalog_parser = subparsers.add_parser("catalog", help="List every published dataset")
+    catalog_parser.add_argument("--category", default=None)
+
+    download_parser = subparsers.add_parser("download", help="Download a dataset CSV")
+    download_parser.add_argument("slug", help="Dataset slug or full desidata.in URL")
+    download_parser.add_argument("-o", "--output", default=None, help="Output file (default: <slug>.csv)")
+    download_parser.add_argument("--refresh", action="store_true", help="Bypass the local cache")
+
+    subparsers.add_parser("clear-cache", help="Delete cached CSVs (~/.desidata/cache)")
+
+    return parser
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    args = _build_parser().parse_args(argv)
+
+    try:
+        if args.command == "search":
+            _print_table(search_datasets(args.query))
+        elif args.command == "info":
+            _print_info(fetch_info(args.slug))
+        elif args.command == "catalog":
+            _print_table(fetch_catalog(category=args.category))
+        elif args.command == "download":
+            data = fetch_csv_bytes(args.slug, refresh=args.refresh)
+            output = args.output or f"{args.slug.rsplit('/', 1)[-1].lower()}.csv"
+            with open(output, "wb") as handle:
+                handle.write(data)
+            print(f"Saved {len(data):,} bytes to {output}")
+        elif args.command == "clear-cache":
+            removed = clear_cache()
+            print(f"Removed {removed} cached file(s).")
+    except DesiDataError as error:
+        print(f"error: {error}", file=sys.stderr)
+        return 1
+    except ValueError as error:
+        print(f"error: {error}", file=sys.stderr)
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

desidata-0.1.0/src/desidata/client.py

@@ -0,0 +1,175 @@
+"""HTTP and cache layer. Everything network-facing lives here.
+
+The package is deliberately dependency-free: only the standard library is
+used, so `pip install desidata` works instantly everywhere. pandas stays an
+optional import in the public API layer.
+"""
+from __future__ import annotations
+
+import os
+import re
+import tempfile
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
+__version__ = "0.1.0"
+
+DEFAULT_BASE_URL = "https://www.desidata.in"
+DEFAULT_TIMEOUT = 60.0
+CACHE_MAX_AGE_SECONDS = 24 * 60 * 60
+
+_USER_AGENT = f"desidata-py/{__version__} (+https://www.desidata.in)"
+_SLUG_PATTERN = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$")
+
+
+class DesiDataError(Exception):
+    """Base class for every error raised by this package."""
+
+
+class DesiDataConnectionError(DesiDataError):
+    """desidata.in could not be reached (network, DNS or timeout failure)."""
+
+
+class DesiDataNotFound(DesiDataError):
+    """The requested dataset does not exist (HTTP 404)."""
+
+
+class DesiDataServerError(DesiDataError):
+    """The server responded with an unexpected error."""
+
+
+def base_url() -> str:
+    """API root, overridable with DESIDATA_BASE_URL (used by tests and dev)."""
+    return (os.environ.get("DESIDATA_BASE_URL") or DEFAULT_BASE_URL).rstrip("/")
+
+
+def cache_directory() -> Path:
+    """Cache location, overridable with DESIDATA_CACHE_DIR."""
+    directory = Path(os.environ.get("DESIDATA_CACHE_DIR") or Path.home() / ".desidata" / "cache")
+    directory.mkdir(parents=True, exist_ok=True)
+    return directory
+
+
+def clear_cache() -> int:
+    """Deletes every cached CSV. Returns the number of files removed."""
+    removed = 0
+    for path in cache_directory().glob("*.csv"):
+        try:
+            path.unlink()
+            removed += 1
+        except OSError:
+            pass
+    return removed
+
+
+def _timeout(timeout: Optional[float]) -> float:
+    if timeout is not None:
+        return float(timeout)
+    env = os.environ.get("DESIDATA_TIMEOUT")
+    return float(env) if env else DEFAULT_TIMEOUT
+
+
+def _normalise_slug(slug: str) -> str:
+    """Accepts a slug, a dataset URL or a path and returns the bare slug.
+
+    Students paste full URLs constantly, so 'https://www.desidata.in/datasets/
+    crop-prices' and 'crop-prices' both work.
+    """
+    value = slug.strip()
+    if ".." in value:
+        raise ValueError(f"Invalid dataset slug: {slug!r}.")
+    if "://" in value:
+        value = value.split("://", 1)[1]
+    if "/" in value:
+        value = value.rstrip("/").rsplit("/", 1)[-1]
+    value = value.strip().lower()
+    if not _SLUG_PATTERN.match(value):
+        raise ValueError(
+            f"Invalid dataset slug: {slug!r}. Slugs look like 'crop-prices-2021' "
+            "(lowercase letters, digits and dashes)."
+        )
+    return value
+
+
+def _open(url: str, timeout_value: float, accept: str) -> bytes:
+    request = urllib.request.Request(url, headers={"User-Agent": _USER_AGENT, "Accept": accept})
+    try:
+        with urllib.request.urlopen(request, timeout=timeout_value) as response:
+            return response.read()
+    except urllib.error.HTTPError as error:
+        if error.code == 404:
+            raise DesiDataNotFound(f"Not found: {url}") from error
+        raise DesiDataServerError(f"desidata.in returned HTTP {error.code} for {url}") from error
+    except urllib.error.URLError as error:
+        raise DesiDataConnectionError(f"Could not reach {url}: {error.reason}") from error
+
+
+def _get_json(path: str, timeout: Optional[float]) -> Dict[str, Any]:
+    payload = _open(f"{base_url()}{path}", _timeout(timeout), "application/json")
+    import json
+
+    return json.loads(payload.decode("utf-8"))
+
+
+def fetch_info(slug: str, timeout: Optional[float] = None) -> Dict[str, Any]:
+    """Metadata for one dataset: name, description, format, size, downloads, tags."""
+    return _get_json(f"/api/datasets/{_normalise_slug(slug)}", timeout)
+
+
+def fetch_catalog(
+    category: Optional[str] = None,
+    language: Optional[str] = None,
+    timeout: Optional[float] = None,
+) -> List[Dict[str, Any]]:
+    """Every published dataset, optionally filtered by category and language."""
+    params = []
+    if category:
+        params.append(f"category={urllib.parse.quote(category)}")
+    if language:
+        params.append(f"language={urllib.parse.quote(language)}")
+    path = "/api/datasets" + ("?" + "&".join(params) if params else "")
+    response = _get_json(path, timeout)
+    return response.get("datasets", [])
+
+
+def search_datasets(query: str, timeout: Optional[float] = None) -> List[Dict[str, Any]]:
+    """Catalogue search over titles and descriptions."""
+    params = f"?query={urllib.parse.quote(query)}"
+    response = _get_json(f"/api/datasets{params}", timeout)
+    return response.get("datasets", [])
+
+
+def fetch_csv_bytes(
+    slug: str,
+    refresh: bool = False,
+    timeout: Optional[float] = None,
+) -> bytes:
+    """Raw CSV bytes for a dataset, served from the local cache when fresh.
+
+    The cache lives in ~/.desidata/cache and entries expire after 24 hours.
+    `refresh=True` bypasses and replaces the cached copy.
+    """
+    normalised = _normalise_slug(slug)
+    path = cache_directory() / f"{normalised}.csv"
+
+    if not refresh and path.exists():
+        age = time.time() - path.stat().st_mtime
+        if age < CACHE_MAX_AGE_SECONDS:
+            return path.read_bytes()
+
+    data = _open(f"{base_url()}/api/datasets/{normalised}/download", _timeout(timeout), "text/csv")
+
+    # Caching is best-effort: a read-only home directory must not break loads.
+    try:
+        fd, temporary = tempfile.mkstemp(dir=str(path.parent), suffix=".tmp")
+        with os.fdopen(fd, "wb") as handle:
+            handle.write(data)
+        os.replace(temporary, path)
+    except OSError:
+        pass
+
+    return data

desidata-0.1.0/src/desidata.egg-info/PKG-INFO

@@ -0,0 +1,165 @@
+Metadata-Version: 2.4
+Name: desidata
+Version: 0.1.0
+Summary: Load Indian datasets from desidata.in in one line — search the catalogue, download CSVs, get pandas DataFrames.
+Author-email: DesiData <hello@desidata.in>
+License: MIT
+Project-URL: Homepage, https://www.desidata.in
+Project-URL: Documentation, https://www.desidata.in
+Project-URL: Repository, https://github.com/krishnakaushik195/desidata-py
+Project-URL: Issues, https://github.com/krishnakaushik195/desidata-py/issues
+Keywords: india,indian,datasets,data,pandas,open-data,csv
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Education
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: pandas
+Requires-Dist: pandas>=1.3; extra == "pandas"
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Dynamic: license-file
+
+# desidata
+
+**Load Indian datasets from [desidata.in](https://www.desidata.in) in one line.**
+
+```python
+import desidata
+
+df = desidata.load("gender-policy-of-nabard-question-and-answer-dataset")
+```
+
+That's it — no URL copying, no sign-in, no API key. You get a pandas
+DataFrame of cleaned, India-focused public data: economy, agriculture,
+health, education, transport, demographics and more.
+
+## Install
+
+```bash
+pip install desidata
+```
+
+The core package has **zero dependencies**, so it installs instantly.
+`load()` needs pandas (almost always already installed) and will tell you
+exactly what to run if it is missing:
+
+```bash
+pip install desidata pandas
+```
+
+## Usage
+
+```python
+import desidata
+
+# Search the catalogue from Python
+results = desidata.search("census")
+for row in results:
+    print(row["slug"], row["category"], row["rows"])
+
+# Browse everything, or one category
+all_datasets = desidata.catalog()
+economy = desidata.catalog(category="Economy")
+
+# Metadata for one dataset
+info = desidata.info("gender-policy-of-nabard-question-and-answer-dataset")
+print(info["name"], info["size"], info["downloads"])
+
+# Load as a DataFrame (kwargs pass through to pandas.read_csv)
+df = desidata.load("gender-policy-of-nabard-question-and-answer-dataset")
+df = desidata.load("some-dataset", dtype={"year": int}, parse_dates=["date"])
+
+# Raw bytes, or save straight to disk
+data = desidata.download("some-dataset")            # bytes
+path = desidata.download("some-dataset", "a.csv")   # writes the file
+```
+
+Full dataset URLs work anywhere a slug does — paste straight from the browser:
+
+```python
+desidata.load("https://www.desidata.in/datasets/gender-policy-of-nabard-question-and-answer-dataset")
+```
+
+### Caching
+
+Loads are cached in `~/.desidata/cache` for 24 hours, so re-loading in a
+notebook (or in a classroom on weak wifi) is instant and offline-safe.
+
+```python
+desidata.load("some-dataset", refresh=True)  # bypass the cache
+desidata.clear_cache()                       # wipe all cached files
+```
+
+Set `DESIDATA_CACHE_DIR` to move the cache somewhere else.
+
+## Command line
+
+```bash
+desidata search nabard
+desidata info gender-policy-of-nabard-question-and-answer-dataset
+desidata catalog --category Agriculture
+desidata download gender-policy-of-nabard-question-and-answer-dataset -o nabard.csv
+desidata clear-cache
+```
+
+## Error handling
+
+Every error the package raises subclasses `desidata.DesiDataError`:
+
+| Exception | Meaning |
+| --- | --- |
+| `DesiDataNotFound` | That dataset doesn't exist (HTTP 404) |
+| `DesiDataConnectionError` | Network/DNS/timeout failure |
+| `DesiDataServerError` | desidata.in returned an unexpected error |
+
+```python
+try:
+    df = desidata.load("some-slug")
+except desidata.DesiDataNotFound:
+    print("Check the slug with desidata.search(...)")
+```
+
+## Development
+
+```bash
+git clone https://github.com/krishnakaushik195/desidata-py
+cd desidata-py
+pip install -e .[dev]
+pytest                    # offline unit tests
+RUN_LIVE=1 pytest tests/test_live.py -v   # integration tests against the real site
+```
+
+Environment overrides used by tests and local development:
+
+| Variable | Purpose |
+| --- | --- |
+| `DESIDATA_BASE_URL` | Point the client at a local/dev server |
+| `DESIDATA_CACHE_DIR` | Relocate the cache |
+| `DESIDATA_TIMEOUT` | Default request timeout in seconds |
+
+## Releasing
+
+Maintainers only:
+
+```bash
+pip install --upgrade build twine
+python -m build
+twine upload dist/*
+```
+
+## Licence
+
+MIT — see [LICENSE](LICENSE). The data itself belongs to its sources; each
+dataset page on [desidata.in](https://www.desidata.in) lists its source and licence.

desidata-0.1.0/src/desidata.egg-info/SOURCES.txt

@@ -0,0 +1,16 @@
+LICENSE
+README.md
+pyproject.toml
+src/desidata/__init__.py
+src/desidata/__main__.py
+src/desidata/cli.py
+src/desidata/client.py
+src/desidata.egg-info/PKG-INFO
+src/desidata.egg-info/SOURCES.txt
+src/desidata.egg-info/dependency_links.txt
+src/desidata.egg-info/entry_points.txt
+src/desidata.egg-info/requires.txt
+src/desidata.egg-info/top_level.txt
+tests/test_cli.py
+tests/test_client.py
+tests/test_live.py

desidata-0.1.0/src/desidata.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

desidata-0.1.0/src/desidata.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+desidata = desidata.cli:main

desidata-0.1.0/src/desidata.egg-info/requires.txt

@@ -0,0 +1,6 @@
+
+[dev]
+pytest>=7
+
+[pandas]
+pandas>=1.3

desidata-0.1.0/src/desidata.egg-info/top_level.txt

@@ -0,0 +1 @@
+desidata

desidata-0.1.0/tests/test_cli.py

@@ -0,0 +1,82 @@
+"""CLI tests with API calls stubbed at the client boundary."""
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from desidata import cli, client
+
+
+@pytest.fixture()
+def runner(capsys):
+    def run(*argv):
+        code = cli.main(list(argv))
+        return code, capsys.readouterr().out
+
+    return run
+
+
+def test_cli_version_flag(runner):
+    with pytest.raises(SystemExit) as excinfo:
+        cli.main(["--version"])
+    assert excinfo.value.code == 0
+
+
+def test_cli_info_prints_fields(runner, monkeypatch):
+    monkeypatch.setattr(
+        cli, "fetch_info", lambda slug: {"slug": slug, "name": "Prices", "downloads": 7}
+    )
+    code, out = runner("info", "prices")
+    assert code == 0
+    assert "prices" in out and "Prices" in out and "7" in out
+
+
+def test_cli_search_prints_table(runner, monkeypatch):
+    monkeypatch.setattr(
+        cli,
+        "search_datasets",
+        lambda query: [{"slug": "crop-prices", "category": "Agriculture", "rows": 100, "title": "Crop"}],
+    )
+    code, out = runner("search", "crop")
+    assert code == 0
+    assert "crop-prices" in out and "Agriculture" in out
+
+
+def test_cli_catalog_category_passthrough(runner, monkeypatch):
+    seen = {}
+
+    def fake_catalog(category=None):
+        seen["category"] = category
+        return []
+
+    monkeypatch.setattr(cli, "fetch_catalog", fake_catalog)
+    code, out = runner("catalog", "--category", "Economy")
+    assert code == 0
+    assert seen["category"] == "Economy"
+    assert "No datasets found." in out
+
+
+def test_cli_download_writes_file(runner, monkeypatch, tmp_path):
+    monkeypatch.setattr(cli, "fetch_csv_bytes", lambda slug, refresh=False: b"a,b\n1,2\n")
+    output = tmp_path / "out.csv"
+    code, out = runner("download", "some-slug", "-o", str(output))
+    assert code == 0
+    assert output.read_bytes() == b"a,b\n1,2\n"
+    assert "8 bytes" in out
+
+
+def test_cli_error_returns_1(runner, monkeypatch):
+    def boom(slug):
+        raise client.DesiDataNotFound("Not found: x")
+
+    monkeypatch.setattr(cli, "fetch_info", boom)
+    code, _ = runner("info", "x")
+    assert code == 1
+
+
+def test_cli_clear_cache(runner, monkeypatch):
+    monkeypatch.setattr(cli, "clear_cache", lambda: 3)
+    code, out = runner("clear-cache")
+    assert code == 0
+    assert "3" in out

desidata-0.1.0/tests/test_client.py

@@ -0,0 +1,136 @@
+"""Unit tests with the network fully mocked out."""
+from __future__ import annotations
+
+import json
+import urllib.error
+
+import pytest
+
+from desidata import client
+
+
+class FakeResponse:
+    def __init__(self, payload: bytes):
+        self._payload = payload
+
+    def read(self) -> bytes:
+        return self._payload
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        return False
+
+
+@pytest.fixture()
+def cache_dir(tmp_path, monkeypatch):
+    directory = tmp_path / "cache"
+    directory.mkdir()
+    monkeypatch.setenv("DESIDATA_CACHE_DIR", str(directory))
+    return directory
+
+
+def test_version_is_exported():
+    assert client.__version__.count(".") == 2
+
+
+def test_normalise_slug_accepts_bare_slug():
+    assert client._normalise_slug("crop-prices-2021") == "crop-prices-2021"
+
+
+def test_normalise_slug_accepts_full_url():
+    url = "https://www.desidata.in/datasets/gender-policy-of-nabard-question-and-answer-dataset"
+    assert client._normalise_slug(url) == "gender-policy-of-nabard-question-and-answer-dataset"
+
+
+def test_normalise_slug_rejects_traversal():
+    with pytest.raises(ValueError):
+        client._normalise_slug("../../etc/passwd")
+
+
+def test_fetch_info_parses_json(monkeypatch):
+    payload = {"slug": "x", "name": "X", "downloads": 5}
+    calls = []
+
+    def fake_urlopen(request, timeout=None):
+        calls.append(request.full_url)
+        return FakeResponse(json.dumps(payload).encode("utf-8"))
+
+    monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
+    result = client.fetch_info("x")
+
+    assert result == payload
+    assert calls == ["https://www.desidata.in/api/datasets/x"]
+
+
+def test_fetch_info_404_raises_not_found(monkeypatch):
+    def fake_urlopen(request, timeout=None):
+        raise urllib.error.HTTPError(request.full_url, 404, "Not Found", None, None)
+
+    monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
+    with pytest.raises(client.DesiDataNotFound):
+        client.fetch_info("missing-slug")
+
+
+def test_urlerror_maps_to_connection_error(monkeypatch):
+    def fake_urlopen(request, timeout=None):
+        raise urllib.error.URLError("no route to host")
+
+    monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
+    with pytest.raises(client.DesiDataConnectionError):
+        client.fetch_info("any-slug")
+
+
+def test_fetch_catalog_builds_filter_url(monkeypatch):
+    calls = []
+
+    def fake_urlopen(request, timeout=None):
+        calls.append(request.full_url)
+        return FakeResponse(json.dumps({"count": 1, "datasets": [{"slug": "a"}]}).encode("utf-8"))
+
+    monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
+    result = client.fetch_catalog(category="Economy", language="English")
+
+    assert result == [{"slug": "a"}]
+    assert "category=Economy" in calls[0]
+    assert "language=English" in calls[0]
+
+
+def test_fetch_csv_bytes_caches(monkeypatch, cache_dir):
+    calls = []
+
+    def fake_urlopen(request, timeout=None):
+        calls.append(request.full_url)
+        return FakeResponse(b"col1,col2\n1,2\n")
+
+    monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
+
+    first = client.fetch_csv_bytes("some-dataset")
+    second = client.fetch_csv_bytes("some-dataset")
+
+    assert first == second == b"col1,col2\n1,2\n"
+    assert len(calls) == 1  # second call came from cache
+    assert (cache_dir / "some-dataset.csv").exists()
+
+
+def test_fetch_csv_bytes_refresh_bypasses_cache(monkeypatch, cache_dir):
+    calls = []
+
+    def fake_urlopen(request, timeout=None):
+        calls.append(request.full_url)
+        return FakeResponse(b"a\n1\n")
+
+    monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
+
+    client.fetch_csv_bytes("some-dataset")
+    client.fetch_csv_bytes("some-dataset", refresh=True)
+
+    assert len(calls) == 2
+
+
+def test_clear_cache_counts_removals(monkeypatch, cache_dir):
+    for slug in ("one", "two"):
+        (cache_dir / f"{slug}.csv").write_bytes(b"a\n")
+    assert client.clear_cache() == 2
+    assert not list(cache_dir.glob("*.csv"))

desidata-0.1.0/tests/test_live.py

@@ -0,0 +1,57 @@
+"""Live integration tests against the real desidata.in.
+
+Skipped unless RUN_LIVE=1, so `pytest` stays offline-friendly:
+    RUN_LIVE=1 pytest tests/test_live.py -v
+"""
+from __future__ import annotations
+
+import os
+
+import pytest
+
+import desidata
+from desidata import client
+
+pytestmark = pytest.mark.skipif(
+    os.environ.get("RUN_LIVE") != "1",
+    reason="live tests are opt-in (RUN_LIVE=1)",
+)
+
+NABARD = "gender-policy-of-nabard-question-and-answer-dataset"
+
+
+def test_live_info():
+    metadata = desidata.info(NABARD)
+    assert metadata["slug"] == NABARD
+    assert metadata["format"].lower() == "csv"
+    assert "download_url" in metadata
+
+
+def test_live_catalog_is_large():
+    datasets = desidata.catalog()
+    assert len(datasets) > 100
+    sample = datasets[0]
+    for key in ("slug", "title", "category", "url"):
+        assert key in sample
+
+
+def test_live_search():
+    results = desidata.search("nabard")
+    assert any("nabard" in row["slug"] for row in results)
+
+
+def test_live_load_dataframe():
+    df = desidata.load(NABARD)
+    assert df.shape[0] > 0
+    assert "question" in df.columns
+
+
+def test_live_download_bytes():
+    data = desidata.download(NABARD)
+    assert data[:4] in (b"ques", b"\"que")  # header starts with the question column
+    assert b"," in data
+
+
+def test_live_not_found():
+    with pytest.raises(client.DesiDataNotFound):
+        desidata.info("this-dataset-does-not-exist-12345")