datannurpy 0.1.0__tar.gz

datannurpy-0.1.0/.github/copilot-instructions.md

@@ -0,0 +1,53 @@
+# datannurpy
+
+Python library for datannur catalog metadata management.
+
+## Architecture
+
+3 layers:
+
+1. **Readers** (`src/datannurpy/readers/`): Polars for CSV/Excel scanning
+2. **Entities** (`src/datannurpy/entities/`): dataclasses (Folder, Dataset, Variable)
+3. **Writers** (`src/datannurpy/writers/`): JSON stdlib for output
+
+## Public API
+
+- `Catalog.add_folder(path, folder=None)` → scans and adds to catalog
+- `Catalog.write(output_dir)` → exports JSON + JSON.js files only
+- `Catalog.export_app(output_dir, open_browser=False)` → exports full datannur app with data
+- `Folder(id, name)` → optional, for custom folder metadata
+
+Internal entities in `src/datannurpy/entities/`.
+
+## App Bundling
+
+The datannur visualization app is bundled in `src/datannurpy/app/` (gitignored).
+To download/update: `make download-app`
+
+## ID Conventions
+
+- Valid chars: `a-zA-Z0-9_, -` (space allowed)
+- Separator: `---`
+
+Example with `Folder(id="source")` scanning `data/sales.csv`:
+
+- Dataset: `source---sales_csv`
+- Variable: `source---sales_csv---amount`
+
+With subdirectory `data/2024/sales.csv`:
+
+- Folder: `source---2024`
+- Dataset: `source---2024---sales_csv`
+- Variable: `source---2024---sales_csv---amount`
+
+## Dev Commands
+
+```bash
+make check  # ruff + pyright + pytest
+```
+
+## Constraints
+
+- Python 3.9+ (use `from __future__ import annotations`)
+- pyright mode: standard
+- Polars typing issues: use `# pyright: ignore[reportCallIssue]` when needed

datannurpy-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,22 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.14"]
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v3
+      - run: uv python install ${{ matrix.python-version }}
+      - run: uv sync
+      - run: make download-app
+      - run: make check

datannurpy-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,117 @@
+name: Release
+
+on:
+  workflow_run:
+    workflows: ["CI"]
+    types: [completed]
+    branches: [main]
+
+permissions:
+  contents: write
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    # Only run if CI succeeded and it was a push (not a PR)
+    if: >
+      github.event.workflow_run.conclusion == 'success' &&
+      github.event.workflow_run.event == 'push'
+    permissions:
+      contents: write
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.14"
+
+      - name: Install build tools
+        run: pip install build tomli
+
+      - name: Build package
+        run: python -m build
+
+      - name: Get current version
+        id: current_version
+        run: |
+          VERSION=$(python -c "import tomli; print(tomli.load(open('pyproject.toml', 'rb'))['project']['version'])")
+          echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
+          echo "tag=v${VERSION}" >> "$GITHUB_OUTPUT"
+
+      - name: Check if version changed
+        id: version_check
+        run: |
+          CURRENT_VERSION="${{ steps.current_version.outputs.version }}"
+
+          # Get the latest release tag using GitHub API
+          LATEST_TAG=$(gh release list --limit 1 --exclude-pre-releases --json tagName --jq '.[0].tagName' 2>/dev/null || echo "")
+
+          if [[ -z "$LATEST_TAG" ]]; then
+            echo "No previous releases found, creating first release"
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+            echo "is_first_release=true" >> "$GITHUB_OUTPUT"
+          else
+            # Extract version from latest tag (remove 'v' prefix)
+            LATEST_VERSION="${LATEST_TAG#v}"
+            
+            if [[ "$CURRENT_VERSION" != "$LATEST_VERSION" ]]; then
+              echo "Version changed from ${LATEST_VERSION} to ${CURRENT_VERSION}"
+              echo "changed=true" >> "$GITHUB_OUTPUT"
+              echo "is_first_release=false" >> "$GITHUB_OUTPUT"
+            else
+              echo "Version unchanged (${CURRENT_VERSION})"
+              echo "changed=false" >> "$GITHUB_OUTPUT"
+              echo "is_first_release=false" >> "$GITHUB_OUTPUT"
+            fi
+          fi
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract changelog
+        id: changelog
+        if: steps.version_check.outputs.changed == 'true'
+        run: |
+          VERSION="${{ steps.current_version.outputs.version }}"
+
+          # Extract changelog section for this version
+          awk "
+          /^## ${VERSION} / { found=1; next }
+          /^## / && found { exit }
+          found { print }
+          " CHANGELOG.md > /tmp/changelog_section.md
+
+          # Check if changelog was found
+          if [ ! -s /tmp/changelog_section.md ]; then
+            echo "No changelog found for version ${VERSION}"
+            echo "found=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "Changelog found for version ${VERSION}"
+            echo "found=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Publish to PyPI
+        if: steps.version_check.outputs.changed == 'true'
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+      - name: Create GitHub release
+        if: steps.version_check.outputs.changed == 'true'
+        run: |
+          TAG="${{ steps.current_version.outputs.tag }}"
+          VERSION="${{ steps.current_version.outputs.version }}"
+
+          if [[ "${{ steps.changelog.outputs.found }}" == "true" ]]; then
+            # Use changelog file created in extract step
+            gh release create "${TAG}" \
+              --title "datannurpy ${TAG}" \
+              --notes-file /tmp/changelog_section.md \
+              dist/*
+          else
+            # Generate notes automatically
+            gh release create "${TAG}" \
+              --title "datannurpy ${TAG}" \
+              --generate-notes \
+              dist/*
+          fi
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

datannurpy-0.1.0/.gitignore

@@ -0,0 +1,7 @@
+.DS_Store
+.pytest_cache/
+.venv/
+.ruff_cache/
+__pycache__/
+src/datannurpy/app/
+examples/output/

datannurpy-0.1.0/.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+  "python.defaultInterpreterPath": "./.venv/bin/python",
+  "python.testing.pytestEnabled": true,
+  "editor.formatOnSave": true
+}

datannurpy-0.1.0/CHANGELOG.md

@@ -0,0 +1,7 @@
+# datannurpy
+
+## unreleased
+
+## 0.1.0 (2026-01-19)
+
+- initial release

datannurpy-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,33 @@
+# Contributing
+
+## Setup
+
+```bash
+git clone https://github.com/datannur/datannurpy.git
+cd datannurpy
+uv sync
+make download-app  # fetch visualization app for bundling
+```
+
+## Development
+
+```bash
+make check        # lint + typecheck + tests (run before committing)
+make test         # tests only
+make lint         # ruff check + format
+make typecheck    # pyright
+```
+
+## Pull Requests
+
+1. Create a branch from `main`
+2. Make changes with tests
+3. Run `make check`
+4. Submit PR
+
+## Code Style
+
+- Python 3.9+ (`from __future__ import annotations`)
+- Formatting: ruff (automatic)
+- Types: pyright standard mode
+- Tests: pytest with `tmp_path` for file operations

datannurpy-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright © 2025-present Bassim Matar - datannur
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

datannurpy-0.1.0/Makefile

@@ -0,0 +1,15 @@
+.PHONY: test lint typecheck check download-app
+
+test:
+	uv run pytest
+
+lint:
+	uv run ruff check . && uv run ruff format --check .
+
+typecheck:
+	uv run pyright src/datannurpy tests
+
+check: lint typecheck test
+
+download-app:
+	uv run python scripts/download_app.py

datannurpy-0.1.0/PKG-INFO

@@ -0,0 +1,89 @@
+Metadata-Version: 2.4
+Name: datannurpy
+Version: 0.1.0
+Summary: Python library for datannur catalog metadata management
+Project-URL: Homepage, https://github.com/datannur/datannurpy
+Project-URL: Repository, https://github.com/datannur/datannurpy
+Project-URL: Documentation, https://github.com/datannur/datannurpy#readme
+Project-URL: Issues, https://github.com/datannur/datannurpy/issues
+Author: datannur
+License: MIT
+License-File: LICENSE
+Requires-Python: >=3.9
+Requires-Dist: polars[calamine]>=1.0
+Requires-Dist: typing-extensions>=4.0; python_version < '3.11'
+Description-Content-Type: text/markdown
+
+# datannurpy
+
+[![PyPI version](https://img.shields.io/pypi/v/datannurpy.svg)](https://pypi.org/project/datannurpy/)
+[![Python](https://img.shields.io/pypi/pyversions/datannurpy.svg)](https://pypi.org/project/datannurpy/)
+[![CI](https://github.com/datannur/datannurpy/actions/workflows/ci.yml/badge.svg)](https://github.com/datannur/datannurpy/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+Python library for [datannur](https://github.com/datannur/datannur) catalog metadata management.
+
+## Installation
+
+```bash
+pip install datannurpy
+```
+
+## Usage
+
+```python
+from datannurpy import Catalog
+
+catalog = Catalog()
+catalog.add_folder("./data")
+catalog.write("./output")
+```
+
+### Multiple sources and custom folder metadata
+
+```python
+from datannurpy import Catalog, Folder
+
+catalog = Catalog()
+catalog.add_folder("./data", Folder(id="prod", name="Production"))
+catalog.add_folder("/mnt/archive", Folder(id="archive", name="Archives"))
+catalog.write("./output")
+```
+
+### Options
+
+```python
+catalog.add_folder(
+    "./data",
+    Folder(id="source", name="Source"),
+    include=["*.csv", "*.xlsx"],
+    exclude=["**/tmp/**"],
+    recursive=True,
+    infer_stats=True,
+)
+```
+
+### Adding individual datasets
+
+```python
+catalog.add_dataset("./data/sales.csv")
+catalog.add_dataset(
+    "./archive/old.csv",
+    folder=Folder(id="archive", name="Archives"),
+    name="Ventes 2023",
+    description="Données historiques",
+)
+```
+
+### Export with visualization app
+
+```python
+# Export a complete datannur app instance
+catalog.export_app("./my-catalog", open_browser=True)
+```
+
+This creates a standalone visualization app that can be opened directly in a browser or deployed to a web server.
+
+## License
+
+MIT License - see the [LICENSE](LICENSE) file for details.

datannurpy-0.1.0/README.md

@@ -0,0 +1,73 @@
+# datannurpy
+
+[![PyPI version](https://img.shields.io/pypi/v/datannurpy.svg)](https://pypi.org/project/datannurpy/)
+[![Python](https://img.shields.io/pypi/pyversions/datannurpy.svg)](https://pypi.org/project/datannurpy/)
+[![CI](https://github.com/datannur/datannurpy/actions/workflows/ci.yml/badge.svg)](https://github.com/datannur/datannurpy/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+Python library for [datannur](https://github.com/datannur/datannur) catalog metadata management.
+
+## Installation
+
+```bash
+pip install datannurpy
+```
+
+## Usage
+
+```python
+from datannurpy import Catalog
+
+catalog = Catalog()
+catalog.add_folder("./data")
+catalog.write("./output")
+```
+
+### Multiple sources and custom folder metadata
+
+```python
+from datannurpy import Catalog, Folder
+
+catalog = Catalog()
+catalog.add_folder("./data", Folder(id="prod", name="Production"))
+catalog.add_folder("/mnt/archive", Folder(id="archive", name="Archives"))
+catalog.write("./output")
+```
+
+### Options
+
+```python
+catalog.add_folder(
+    "./data",
+    Folder(id="source", name="Source"),
+    include=["*.csv", "*.xlsx"],
+    exclude=["**/tmp/**"],
+    recursive=True,
+    infer_stats=True,
+)
+```
+
+### Adding individual datasets
+
+```python
+catalog.add_dataset("./data/sales.csv")
+catalog.add_dataset(
+    "./archive/old.csv",
+    folder=Folder(id="archive", name="Archives"),
+    name="Ventes 2023",
+    description="Données historiques",
+)
+```
+
+### Export with visualization app
+
+```python
+# Export a complete datannur app instance
+catalog.export_app("./my-catalog", open_browser=True)
+```
+
+This creates a standalone visualization app that can be opened directly in a browser or deployed to a web server.
+
+## License
+
+MIT License - see the [LICENSE](LICENSE) file for details.

datannurpy-0.1.0/data/employees.csv

@@ -0,0 +1,21 @@
+id,nom,prenom,age,salaire,departement,ville,date_embauche,est_manager
+1,Dupont,Marie,28,45000,IT,Paris,2022-01-15,0
+2,Martin,Pierre,35,52000,RH,Lyon,2020-03-22,1
+3,Durand,Sophie,31,48000,IT,Paris,2021-06-10,0
+4,Bernard,Jean,42,65000,Finance,Marseille,2019-09-05,1
+5,Petit,Anne,26,42000,Marketing,Toulouse,2023-02-28,0
+6,Moreau,Paul,39,58000,IT,Paris,2020-11-12,1
+7,Simon,Claire,33,51000,RH,Lyon,2021-04-18,0
+8,Michel,Luc,45,72000,Finance,Marseille,2018-07-30,1
+9,Leroy,Emma,29,46000,Marketing,Toulouse,2022-08-14,0
+10,Roux,Thomas,37,54000,IT,Paris,2020-12-03,0
+11,David,Julie,32,49000,RH,Lyon,2021-10-25,0
+12,Bertrand,Marc,40,61000,,Marseille,2019-05-17,1
+13,Robert,Céline,27,43000,Marketing,Toulouse,2023-01-09,0
+14,Richard,Nicolas,36,55000,IT,Paris,2020-08-21,0
+15,Girard,Laura,30,47000,RH,Lyon,2022-03-14,0
+16,Blanc,François,44,68000,Finance,Marseille,2018-12-11,1
+17,Garnier,Sylvie,34,50000,Marketing,Toulouse,2021-09-06,0
+18,Faure,,,56000,IT,Paris,2020-04-29,0
+19,Andre,Isabelle,41,62000,RH,Lyon,2019-11-08,1
+20,Mercier,Julien,25,41000,Marketing,Toulouse,2023-05-22,0

datannurpy-0.1.0/data/regions_france.csv

@@ -0,0 +1,14 @@
+region,population,superficie_km2,pib_milliards_euros,taux_chomage,nb_entreprises,densite_pop
+Île-de-France,12140526,12011,765.8,7.1,1456789,1010.8
+Auvergne-Rhône-Alpes,8042936,69711,289.4,6.8,845123,115.4
+Hauts-de-France,5962662,31806,179.2,9.2,456789,187.5
+Nouvelle-Aquitaine,6010289,84061,189.3,7.4,567234,71.5
+Occitanie,5924858,72724,172.1,8.1,523456,81.4
+Grand Est,5518851,57433,159.7,8.3,445678,96.1
+Provence-Alpes-Côte d'Azur,5094283,31400,168.9,8.7,456123,162.2
+Pays de la Loire,3832120,32082,125.4,6.2,378901,119.5
+Bretagne,3373835,27208,104.2,6.9,334567,124.0
+Normandie,3322757,29906,91.8,7.8,289234,111.1
+Bourgogne-Franche-Comté,2795301,47784,71.9,7.5,234567,58.5
+Centre-Val de Loire,2573180,39151,69.8,7.2,223456,65.7
+Corse,344679,8722,8.9,8.9,34567,39.5

datannurpy-0.1.0/examples/demo.py

@@ -0,0 +1,12 @@
+from pathlib import Path
+
+from datannurpy import Catalog
+
+HERE = Path(__file__).parent
+DATA = HERE.parent / "data"
+
+catalog = Catalog()
+catalog.add_folder(DATA)
+catalog.export_app(HERE / "output", open_browser=True)
+
+print(f"✅ {catalog}")

datannurpy-0.1.0/pyproject.toml

@@ -0,0 +1,40 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "datannurpy"
+version = "0.1.0"
+description = "Python library for datannur catalog metadata management"
+authors = [{ name = "datannur" }]
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+dependencies = [
+  "polars[calamine]>=1.0",
+  "typing_extensions>=4.0;python_version<'3.11'",
+]
+
+[project.urls]
+Homepage = "https://github.com/datannur/datannurpy"
+Repository = "https://github.com/datannur/datannurpy"
+Documentation = "https://github.com/datannur/datannurpy#readme"
+Issues = "https://github.com/datannur/datannurpy/issues"
+
+[tool.uv]
+dev-dependencies = ["pytest", "ruff", "pyright"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.ruff]
+target-version = "py39"
+exclude = ["examples/output"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/datannurpy"]
+
+[tool.pyright]
+pythonVersion = "3.9"
+typeCheckingMode = "standard"
+exclude = ["src/datannurpy/app"]

datannurpy-0.1.0/scripts/download_app.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+"""Download the datannur app for bundling in the package.
+
+Usage: make download-app
+"""
+
+from __future__ import annotations
+
+import io
+import shutil
+import urllib.request
+import zipfile
+from pathlib import Path
+
+_URL = "https://github.com/datannur/datannur/releases/download/pre-release/datannur-app-pre-release.zip"
+_APP_DIR = Path(__file__).resolve().parent.parent / "src" / "datannurpy" / "app"
+
+
+def download_app() -> None:
+    """Download and extract the datannur app."""
+    print(f"Downloading from {_URL}...")
+
+    with urllib.request.urlopen(_URL, timeout=60) as r:
+        zip_data = r.read()
+
+    print(f"Downloaded {len(zip_data) / 1024:.0f} KB")
+
+    if _APP_DIR.exists():
+        shutil.rmtree(_APP_DIR)
+
+    _APP_DIR.mkdir(parents=True)
+
+    with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
+        zf.extractall(_APP_DIR)
+
+    # Clear demo data content (keep the data/ folder structure)
+    data_dir = _APP_DIR / "data"
+    if data_dir.exists():
+        for item in data_dir.iterdir():
+            if item.is_dir():
+                shutil.rmtree(item)
+            else:
+                item.unlink()
+        print("✓ Cleared demo data/")
+
+    print("✓ Done")
+
+
+if __name__ == "__main__":
+    download_app()

datannurpy-0.1.0/src/datannurpy/__init__.py

@@ -0,0 +1,8 @@
+"""datannurpy - Python library for datannur catalog metadata management."""
+
+__version__ = "0.1.0"
+
+from .catalog import Catalog
+from .entities import Folder
+
+__all__ = ["Catalog", "Folder"]

datannurpy-0.1.0/src/datannurpy/_ids.py

@@ -0,0 +1,21 @@
+"""ID generation and validation utilities."""
+
+from __future__ import annotations
+
+import re
+
+# Separator for path components (folder---dataset---variable)
+ID_SEPARATOR = "---"
+
+# Valid ID pattern: a-zA-Z0-9_, - (and space)
+_INVALID_ID_CHARS = re.compile(r"[^a-zA-Z0-9_,\- ]")
+
+
+def sanitize_id(value: str) -> str:
+    """Replace invalid characters with underscore."""
+    return _INVALID_ID_CHARS.sub("_", value)
+
+
+def make_id(*parts: str) -> str:
+    """Join parts with ID_SEPARATOR."""
+    return ID_SEPARATOR.join(parts)