open-data-mexico 1.0.0__tar.gz

open_data_mexico-1.0.0/.github/workflows/ci.yml

@@ -0,0 +1,59 @@
+name: CI
+
+on:
+  push:
+    branches: [main, feature/**]
+  pull_request:
+    branches: [main]
+
+jobs:
+  lint:
+    name: Lint & type check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install dev dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Ruff lint
+        run: ruff check .
+
+      - name: Ruff format check
+        run: ruff format --check .
+
+      - name: Mypy
+        run: mypy open_data_mexico/ --ignore-missing-imports
+
+  test:
+    name: Test (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    needs: lint
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dev dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Run tests with coverage
+        run: pytest
+
+      - name: Upload coverage report
+        if: matrix.python-version == '3.13'
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report
+          path: .coverage

open_data_mexico-1.0.0/.github/workflows/docs.yml

@@ -0,0 +1,24 @@
+name: Deploy docs
+
+on:
+  push:
+    branches: [main]
+
+permissions:
+  contents: write
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install docs dependencies
+        run: pip install mkdocs-material
+
+      - name: Deploy to GitHub Pages
+        run: mkdocs gh-deploy --force

open_data_mexico-1.0.0/.github/workflows/publish.yml

@@ -0,0 +1,47 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  id-token: write  # Required for Trusted Publisher (OIDC)
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install build tools
+        run: pip install build
+
+      - name: Build wheel and sdist
+        run: python -m build
+
+      - name: Upload dist artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    needs: build
+    environment: pypi  # Requires a GitHub environment named "pypi"
+    steps:
+      - name: Download dist artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        # No token needed — uses OIDC Trusted Publisher configured on pypi.org

open_data_mexico-1.0.0/.gitignore

@@ -0,0 +1,13 @@
+.claude
+.venv
+.env
+__pycache__
+.pytest_cache
+.ruff_cache
+.mypy_cache
+.pytest_cache
+.pytest_cache
+.pytest_cache
+PROJECT.md
+dist/
+.coverage

open_data_mexico-1.0.0/.pre-commit-config.yaml

@@ -0,0 +1,14 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.8
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.19.1
+    hooks:
+      - id: mypy
+        args: [--ignore-missing-imports]
+        additional_dependencies: [pydantic>=2.0.0]

open_data_mexico-1.0.0/CHANGELOG.md

@@ -0,0 +1,63 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+---
+
+## [Unreleased]
+
+## [1.0.0] — 2026-03-27
+
+### Added
+- `request_delay` parameter on `DatosGobMX` for configurable rate limiting between requests.
+- `max_retries` parameter on `DatosGobMX` for automatic retry with exponential backoff on 5xx/429 and network errors.
+- `cache_ttl` parameter on `DatosGobMX` for in-memory TTL caching of responses (default 300 s).
+- `open_data_mexico/_http.py` — `robust_get()` helper used by all scrapers.
+- FastAPI server uses a single shared `DatosGobMX` client via lifespan (connection pool + shared cache).
+- `py.typed` marker for PEP 561 compliance (mypy/pyright support).
+- GitHub Actions CI workflow — lint, type-check and test on Python 3.11, 3.12, 3.13.
+- GitHub Actions publish workflow — build and publish to PyPI on GitHub Release via OIDC Trusted Publisher.
+- `.pre-commit-config.yaml` with ruff and mypy hooks.
+- `pytest-cov` with 80 % minimum coverage enforced.
+- MkDocs site configuration.
+- `client.search(query)` — full-text dataset search via CKAN `package_search` JSON API; supports `category`, `limit`, `offset` parameters; returns `SearchResponse`.
+- `client.get_organizations()` — lists all 184+ publishing institutions via CKAN `organization_list`; returns `list[Organization]`.
+- `client.get_organization(slug)` — fetches a single organization by slug via CKAN `organization_show`; returns `None` on 404.
+- `Organization` and `OrganizationsResponse` Pydantic models.
+- `SearchResponse` Pydantic model.
+- `GET /organizations` and `GET /organizations/{slug}` FastAPI endpoints.
+- `GET /search?q=...` FastAPI endpoint with optional `category`, `limit`, `offset` query params.
+- `open_data_mexico/_utils.py` — `parse_spanish_date()` and `parse_iso_dt()` helpers for robust datetime parsing.
+
+### Changed
+- `_get_total_pages()` now scans all `<li>` elements in `ul.pagination` instead of only `<a>` tags, making pagination detection more robust against disabled/active page items rendered as `<span>`.
+- `Dataset.last_updated`, `DatasetDetail.created`, and `DatasetDetail.last_updated` changed from `str | None` to `datetime | None`; values are now timezone-aware UTC datetimes.
+- `pyproject.toml`: corrected project URLs to the real GitHub repository.
+- `pyproject.toml`: `license` field now points to the `LICENSE` file.
+- `pyproject.toml`: added Python 3.13 classifier.
+- README updated to document new client parameters and corrected test count.
+
+### Fixed
+- Unused imports removed across `client.py`, `server/app.py`, and test files (ruff F401).
+- `Optional[str]` annotations modernised to `str | None` (ruff UP045).
+
+---
+
+## [0.1.0] — 2026-03-23
+
+### Added
+- Initial implementation of `DatosGobMX` async client.
+- Scraping of categories (`/group/`), dataset listings (`/group/{slug}`), and dataset detail pages (`/dataset/{slug}`).
+- Pydantic models: `Category`, `Dataset`, `DatasetDetail`, `Resource`, `CategoriesResponse`, `DatasetsResponse`.
+- Auto-pagination for categories and dataset listings.
+- `get_resource_data()` for in-memory CSV download with UTF-8 / latin-1 fallback.
+- Optional FastAPI REST server (`pip install open-data-mexico[server]`).
+- 44 tests with mock HTML fixtures (pytest-asyncio + pytest-httpx).
+- Documentation for all 28 available categories.
+
+[Unreleased]: https://github.com/lehcimhdz/open-data-mexico-api/compare/v1.0.0...HEAD
+[1.0.0]: https://github.com/lehcimhdz/open-data-mexico-api/releases/tag/v1.0.0
+[0.1.0]: https://github.com/lehcimhdz/open-data-mexico-api/releases/tag/v0.1.0

open_data_mexico-1.0.0/CONTRIBUTING.md

@@ -0,0 +1,115 @@
+# Contributing to open-data-mexico
+
+Thank you for your interest in contributing! This document covers everything you need to get started.
+
+---
+
+## Setup
+
+**Requirements:** Python 3.11+ and git.
+
+```bash
+git clone https://github.com/lehcimhdz/open-data-mexico-api.git
+cd open-data-mexico-api
+pip install -e ".[dev]"
+```
+
+Install pre-commit hooks so linting runs automatically before every commit:
+
+```bash
+pip install pre-commit
+pre-commit install
+```
+
+---
+
+## Running tests
+
+```bash
+# Full suite with coverage (must stay above 80 %)
+pytest
+
+# Skip coverage for a faster feedback loop
+pytest --no-cov -v
+
+# Single file
+pytest tests/test_datasets.py -v
+```
+
+---
+
+## Code quality
+
+All three checks must pass before opening a PR — they also run in CI:
+
+```bash
+ruff check .          # lint
+ruff format --check . # formatting
+mypy open_data_mexico/ --ignore-missing-imports  # type checking
+```
+
+Auto-fix lint and formatting issues:
+
+```bash
+ruff check . --fix
+ruff format .
+```
+
+---
+
+## Project structure
+
+```
+open_data_mexico/        # installable library (published to PyPI)
+├── client.py            # DatosGobMX — public entry point
+├── models.py            # Pydantic models
+├── _config.py           # BASE_URL, headers, defaults (private)
+├── _http.py             # robust_get() with retry/backoff (private)
+└── _scrapers/           # HTML scraping internals (private)
+    ├── categories.py
+    ├── datasets.py
+    └── dataset_detail.py
+server/                  # optional FastAPI REST server
+tests/                   # pytest suite with mock HTML fixtures
+docs/                    # MkDocs source
+```
+
+---
+
+## Making changes
+
+### Scraper changes
+
+The scrapers parse HTML from `datos.gob.mx`. If the site changes its structure, update the CSS selectors and add/update the corresponding HTML fixture in `tests/conftest.py` to reflect the new markup.
+
+### Adding a new public method
+
+1. Implement it in `client.py`.
+2. Export it from `open_data_mexico/__init__.py` and add it to `__all__`.
+3. Add a Pydantic model in `models.py` if the method returns a new type.
+4. Write tests in `tests/`.
+5. Document it in `README.md` (Client Reference section).
+
+### Updating the CHANGELOG
+
+Add a bullet under `## [Unreleased]` in `CHANGELOG.md` following the [Keep a Changelog](https://keepachangelog.com) convention (`Added`, `Changed`, `Fixed`, `Removed`).
+
+---
+
+## Pull request checklist
+
+- [ ] `pytest` passes with ≥ 80 % coverage
+- [ ] `ruff check .` reports no errors
+- [ ] `ruff format --check .` reports no changes needed
+- [ ] `mypy open_data_mexico/` reports no errors
+- [ ] `CHANGELOG.md` updated under `[Unreleased]`
+- [ ] New public API documented in `README.md`
+
+---
+
+## Reporting bugs
+
+Open an issue at [github.com/lehcimhdz/open-data-mexico-api/issues](https://github.com/lehcimhdz/open-data-mexico-api/issues) with:
+- Python version and OS
+- Minimal code that reproduces the problem
+- Full traceback if applicable

open_data_mexico-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 lehcimhdz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.