hscida 0.1.0__tar.gz

hscida-0.1.0/.github/workflows/tests.yml

@@ -0,0 +1,29 @@
+name: Tests
+
+on:
+  push:
+    branches:
+      - "**"
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Sync dependencies
+        run: uv sync --dev
+
+      - name: Run tests
+        run: uv run pytest -q

hscida-0.1.0/.gitignore

@@ -0,0 +1,4 @@
+.Rproj.user
+.Rhistory
+.RData
+.Ruserdata

hscida-0.1.0/.history/.github/workflows/publish_20260605151839.yml

@@ -0,0 +1,56 @@
+name: Publish
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Sync dependencies
+        run: uv sync --dev
+
+      - name: Run tests
+        run: uv run pytest -q
+
+      - name: Build package
+        run: uv build
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: build
+    permissions:
+      id-token: write
+
+    environment:
+      name: pypi
+
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

hscida-0.1.0/.history/.github/workflows/tests_20260605151436.yml

@@ -0,0 +1,29 @@
+name: Tests
+
+on:
+  push:
+    branches:
+      - '**'
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Sync dependencies
+        run: uv sync --dev
+
+      - name: Run tests
+        run: uv run pytest -q

hscida-0.1.0/.history/.github/workflows/tests_20260605151454.yml

@@ -0,0 +1,29 @@
+name: Tests
+
+on:
+  push:
+    branches:
+      - "**"
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Sync dependencies
+        run: uv sync --dev
+
+      - name: Run tests
+        run: uv run pytest -q

hscida-0.1.0/.history/README_20260605151243.md

@@ -0,0 +1,15 @@
+# hsci-da
+
+Installable Python package for HSCI data access utilities built on DuckDB and Narwhals.
+
+## Install
+
+```bash
+pip install .
+```
+
+## Run tests
+
+```bash
+pytest
+```

hscida-0.1.0/.history/README_20260605151302.md

@@ -0,0 +1,15 @@
+# hsci-da
+
+Installable Python package for HSCI data access utilities built on DuckDB and Narwhals.
+
+## Install
+
+```bash
+pip install .
+```
+
+## Run tests
+
+```bash
+pytest
+```

hscida-0.1.0/.history/README_20260605151839.md

@@ -0,0 +1,37 @@
+# hsci-da
+
+Installable Python package for HSCI data access utilities built on DuckDB and Narwhals.
+
+## Install
+
+```bash
+pip install .
+```
+
+## Run tests
+
+```bash
+pytest
+```
+
+## Publish To PyPI
+
+This repository includes GitHub Actions publishing in [.github/workflows/publish.yml](.github/workflows/publish.yml) using PyPI Trusted Publishing.
+
+One-time setup on PyPI:
+
+1. Create a PyPI project named `hsci-da`.
+2. In PyPI project settings, add a Trusted Publisher for this GitHub repository and workflow file `.github/workflows/publish.yml`.
+
+Release flow:
+
+```bash
+uv run pytest -q
+# bump version in pyproject.toml
+git add pyproject.toml
+git commit -m "Release v0.1.1"
+git tag v0.1.1
+git push && git push --tags
+```
+
+Pushing a tag like `v0.1.1` will run tests, build artifacts, and publish to PyPI.

hscida-0.1.0/.history/README_20260605151903.md

@@ -0,0 +1,39 @@
+# hsci-da
+
+Installable Python package for HSCI data access utilities built on DuckDB and Narwhals.
+
+## Install
+
+```bash
+pip install .
+```
+
+## Run tests
+
+```bash
+pytest
+```
+
+## Publish To PyPI (Local)
+
+Set your token once in your shell profile:
+
+```bash
+export TWINE_USERNAME=__token__
+export TWINE_PASSWORD=pypi-xxxx
+```
+
+Release flow from your machine:
+
+```bash
+# bump version in pyproject.toml first
+uv sync --group publish --dev
+uv run pytest -q
+./scripts/publish.sh pypi
+```
+
+Dry-run against TestPyPI:
+
+```bash
+./scripts/publish.sh testpypi
+```

hscida-0.1.0/.history/README_20260605151949.md

@@ -0,0 +1,41 @@
+# hsci-da
+
+Installable Python package for HSCI data access utilities built on DuckDB and Narwhals.
+
+## Install
+
+```bash
+pip install .
+```
+
+## Run tests
+
+```bash
+pytest
+```
+
+## Publish To PyPI (Local)
+
+Set your token once in your shell profile:
+
+```bash
+export TWINE_USERNAME=__token__
+export TWINE_PASSWORD=pypi-xxxx
+```
+
+Release flow from your machine:
+
+```bash
+# bump version in pyproject.toml first
+uv sync --group publish --dev
+uv run pytest -q
+uv run --group publish python -m build
+uv run --group publish twine check dist/*
+uv run --group publish twine upload dist/*
+```
+
+Dry-run against TestPyPI:
+
+```bash
+uv run --group publish twine upload --repository testpypi dist/*
+```

hscida-0.1.0/.history/README_20260605152141.md

@@ -0,0 +1,39 @@
+# hsci-da
+
+Installable Python package for HSCI data access utilities built on DuckDB and Narwhals.
+
+## Install
+
+```bash
+pip install .
+```
+
+## Run tests
+
+```bash
+pytest
+```
+
+## Publish To PyPI (Local)
+
+Set your token once in your shell profile:
+
+```bash
+export UV_PUBLISH_TOKEN=pypi-xxxx
+```
+
+Release flow from your machine:
+
+```bash
+# bump version in pyproject.toml first
+uv sync --dev
+uv run pytest -q
+uv build
+uv publish
+```
+
+Dry-run against TestPyPI:
+
+```bash
+uv publish --publish-url https://test.pypi.org/legacy/
+```

hscida-0.1.0/.history/README_20260605152217.md

@@ -0,0 +1,39 @@
+# hsci-da
+
+Installable Python package for HSCI data access utilities built on DuckDB and Narwhals.
+
+## Install
+
+```bash
+pip install .
+```
+
+## Run tests
+
+```bash
+pytest
+```
+
+## Publish To PyPI (Local)
+
+Set your token once in your shell profile:
+
+```bash
+export UV_PUBLISH_TOKEN=pypi-xxxx
+```
+
+Release flow from your machine:
+
+```bash
+# bump version in pyproject.toml first
+uv sync --dev
+uv run pytest -q
+uv build
+uv publish
+```
+
+Dry-run against TestPyPI:
+
+```bash
+uv publish --publish-url https://test.pypi.org/legacy/
+```

hscida-0.1.0/.history/hscida_20260605143229.py

@@ -0,0 +1,82 @@
+from dataclasses import dataclass, field
+from typing import Any, cast
+from hereutil import here
+import narwhals as nw
+import duckdb
+import sqlglot
+import os
+import polars as pl
+
+from dotenv import dotenv_values
+
+_DEFAULT_DUCKDB_CONFIG = dict(parquet_metadata_cache="true", preserve_insertion_order="false", enable_fsst_vectors="true")
+_PROJROOT = str(here())
+
+@dataclass
+class DataAccessConfig:
+    glob_pattern: str
+    init_sql: str
+    duckdb_config: dict[str, Any] = field(default_factory=lambda: _DEFAULT_DUCKDB_CONFIG)
+    projroot: str = _PROJROOT
+
+def config_from_env() -> DataAccessConfig:
+    c = {
+        **dotenv_values(),
+        **os.environ,
+    }
+    return DataAccessConfig(
+        glob_pattern=c['GLOB_PATTERN'],
+        init_sql=c['INIT_SQL'],
+        duckdb_config={k: v for k, v in [pair.split('=') for pair in c['DUCKDB_CONFIG'].split(',')] } if 'DUCKDB_CONFIG' in c else _DEFAULT_DUCKDB_CONFIG,
+        projroot=c.get('PROJROOT', _PROJROOT)
+    )
+
+class DataAccess:
+    def __init__(self, config: DataAccessConfig = config_from_env()) -> None:
+        self.con = duckdb.connect(config=config.duckdb_config)
+        self.con.sql(config.init_sql)
+        self.datasets = dict[str, nw.LazyFrame[duckdb.DuckDBPyRelation]]()
+        self.config = config
+
+    def register_files_as_view(self, table_name: str, *paths: str, replace: bool = False) -> None:
+        self.con.sql(f"CREATE {('OR REPLACE' if replace else '')} VIEW {'IF NOT EXISTS' if not replace else ''} {table_name} AS FROM read_{'parquet' if paths[0].endswith('.parquet') else 'csv'}(['{"', '".join(paths)}'], hive_partitioning=true);")
+
+    def f(self, dataset: str, *paths: str,replace: bool = False, debug: bool = False) -> nw.LazyFrame[duckdb.DuckDBPyRelation]:
+        if dataset not in self.datasets or replace:
+            if not paths:
+                paths = tuple(path[0] for path in self.con.sql("FROM "+self.config.glob_pattern.format(dataset=dataset,projroot=self.config.projroot)).fetchall())
+            if debug:
+                print(f"DEBUG: Found paths for dataset {dataset}: {paths}")
+            if not paths:
+                print(f"No files found for dataset {dataset} in {self.config.glob_pattern.format(dataset=dataset,projroot=self.config.projroot)}")
+                return cast(nw.LazyFrame[duckdb.DuckDBPyRelation], None)
+            self.register_files_as_view(dataset, *paths, replace=replace)
+            self.datasets[dataset] = nw.from_native(self.con.sql(f'FROM {dataset}'))
+        return self.datasets[dataset]
+
+c = nw.col
+l = nw.lit
+
+def to_narwhals(duckdb_relation: duckdb.DuckDBPyRelation) -> nw.LazyFrame[duckdb.DuckDBPyRelation]:
+    return nw.from_native(duckdb_relation)
+
+n = to_narwhals
+
+def to_duckdb(lnf: nw.LazyFrame[duckdb.DuckDBPyRelation]) -> duckdb.DuckDBPyRelation:
+    return lnf.to_native()
+
+d = to_duckdb
+
+def to_polars(lnf: nw.LazyFrame[duckdb.DuckDBPyRelation]) -> pl.DataFrame:
+    return lnf.collect(backend='polars').to_native()
+
+p = to_polars
+
+def to_pandas(lnf: nw.LazyFrame[duckdb.DuckDBPyRelation]):
+    return d(lnf).df()
+
+def format_sql(query: str, read:str = 'duckdb', write:str = 'duckb') -> str:
+    return sqlglot.transpile(query, read=read, write=write, pretty=True)[0]
+
+
+__all__ = [ "DataAccess", "c", "l", "to_narwhals", "n", "to_duckdb", "d", "to_polars", "p","to_pandas", "format_sql" ]

hscida-0.1.0/.history/hscida_20260605151106.py

@@ -0,0 +1,81 @@
+from dataclasses import dataclass, field
+from typing import Any, cast
+from hereutil import here
+import narwhals as nw
+import duckdb
+import os
+import polars as pl
+
+from dotenv import dotenv_values
+
+_DEFAULT_DUCKDB_CONFIG = dict(parquet_metadata_cache="true", preserve_insertion_order="false", enable_fsst_vectors="true")
+_PROJROOT = str(here())
+
+@dataclass
+class DataAccessConfig:
+    glob_pattern: str
+    init_sql: str
+    duckdb_config: dict[str, Any] = field(default_factory=lambda: _DEFAULT_DUCKDB_CONFIG)
+    projroot: str = _PROJROOT
+
+def config_from_env() -> DataAccessConfig:
+    c = {
+        **dotenv_values(),
+        **os.environ,
+    }
+    return DataAccessConfig(
+        glob_pattern=c['GLOB_PATTERN'],
+        init_sql=c['INIT_SQL'],
+        duckdb_config={k: v for k, v in [pair.split('=') for pair in c['DUCKDB_CONFIG'].split(',')] } if 'DUCKDB_CONFIG' in c else _DEFAULT_DUCKDB_CONFIG,
+        projroot=c.get('PROJROOT', _PROJROOT)
+    )
+
+class DataAccess:
+    def __init__(self, config: DataAccessConfig = config_from_env()) -> None:
+        self.con = duckdb.connect(config=config.duckdb_config)
+        self.con.sql(config.init_sql)
+        self.datasets = dict[str, nw.LazyFrame[duckdb.DuckDBPyRelation]]()
+        self.config = config
+
+    def register_files_as_view(self, table_name: str, *paths: str, replace: bool = False) -> None:
+        self.con.sql(f"CREATE {('OR REPLACE' if replace else '')} VIEW {'IF NOT EXISTS' if not replace else ''} {table_name} AS FROM read_{'parquet' if paths[0].endswith('.parquet') else 'csv'}(['{"', '".join(paths)}'], hive_partitioning=true);")
+
+    def f(self, dataset: str, *paths: str,replace: bool = False, debug: bool = False) -> nw.LazyFrame[duckdb.DuckDBPyRelation]:
+        if dataset not in self.datasets or replace:
+            if not paths:
+                paths = tuple(path[0] for path in self.con.sql("FROM "+self.config.glob_pattern.format(dataset=dataset,projroot=self.config.projroot)).fetchall())
+            if debug:
+                print(f"DEBUG: Found paths for dataset {dataset}: {paths}")
+            if not paths:
+                print(f"No files found for dataset {dataset} in {self.config.glob_pattern.format(dataset=dataset,projroot=self.config.projroot)}")
+                return cast(nw.LazyFrame[duckdb.DuckDBPyRelation], None)
+            self.register_files_as_view(dataset, *paths, replace=replace)
+            self.datasets[dataset] = nw.from_native(self.con.sql(f'FROM {dataset}'))
+        return self.datasets[dataset]
+
+c = nw.col
+l = nw.lit
+
+def to_narwhals(duckdb_relation: duckdb.DuckDBPyRelation) -> nw.LazyFrame[duckdb.DuckDBPyRelation]:
+    return nw.from_native(duckdb_relation)
+
+n = to_narwhals
+
+def to_duckdb(lnf: nw.LazyFrame[duckdb.DuckDBPyRelation]) -> duckdb.DuckDBPyRelation:
+    return lnf.to_native()
+
+d = to_duckdb
+
+def to_polars(lnf: nw.LazyFrame[duckdb.DuckDBPyRelation]) -> pl.DataFrame:
+    return lnf.collect(backend='polars').to_native()
+
+p = to_polars
+
+def to_pandas(lnf: nw.LazyFrame[duckdb.DuckDBPyRelation]):
+    return d(lnf).df()
+
+def format_sql(query: str, read:str = 'duckdb', write:str = 'duckb') -> str:
+    return sqlglot.transpile(query, read=read, write=write, pretty=True)[0]
+
+
+__all__ = [ "DataAccess", "c", "l", "to_narwhals", "n", "to_duckdb", "d", "to_polars", "p","to_pandas", "format_sql" ]

hscida-0.1.0/.history/hscida_20260605151110.py

@@ -0,0 +1,78 @@
+from dataclasses import dataclass, field
+from typing import Any, cast
+from hereutil import here
+import narwhals as nw
+import duckdb
+import os
+import polars as pl
+
+from dotenv import dotenv_values
+
+_DEFAULT_DUCKDB_CONFIG = dict(parquet_metadata_cache="true", preserve_insertion_order="false", enable_fsst_vectors="true")
+_PROJROOT = str(here())
+
+@dataclass
+class DataAccessConfig:
+    glob_pattern: str
+    init_sql: str
+    duckdb_config: dict[str, Any] = field(default_factory=lambda: _DEFAULT_DUCKDB_CONFIG)
+    projroot: str = _PROJROOT
+
+def config_from_env() -> DataAccessConfig:
+    c = {
+        **dotenv_values(),
+        **os.environ,
+    }
+    return DataAccessConfig(
+        glob_pattern=c['GLOB_PATTERN'],
+        init_sql=c['INIT_SQL'],
+        duckdb_config={k: v for k, v in [pair.split('=') for pair in c['DUCKDB_CONFIG'].split(',')] } if 'DUCKDB_CONFIG' in c else _DEFAULT_DUCKDB_CONFIG,
+        projroot=c.get('PROJROOT', _PROJROOT)
+    )
+
+class DataAccess:
+    def __init__(self, config: DataAccessConfig = config_from_env()) -> None:
+        self.con = duckdb.connect(config=config.duckdb_config)
+        self.con.sql(config.init_sql)
+        self.datasets = dict[str, nw.LazyFrame[duckdb.DuckDBPyRelation]]()
+        self.config = config
+
+    def register_files_as_view(self, table_name: str, *paths: str, replace: bool = False) -> None:
+        self.con.sql(f"CREATE {('OR REPLACE' if replace else '')} VIEW {'IF NOT EXISTS' if not replace else ''} {table_name} AS FROM read_{'parquet' if paths[0].endswith('.parquet') else 'csv'}(['{"', '".join(paths)}'], hive_partitioning=true);")
+
+    def f(self, dataset: str, *paths: str,replace: bool = False, debug: bool = False) -> nw.LazyFrame[duckdb.DuckDBPyRelation]:
+        if dataset not in self.datasets or replace:
+            if not paths:
+                paths = tuple(path[0] for path in self.con.sql("FROM "+self.config.glob_pattern.format(dataset=dataset,projroot=self.config.projroot)).fetchall())
+            if debug:
+                print(f"DEBUG: Found paths for dataset {dataset}: {paths}")
+            if not paths:
+                print(f"No files found for dataset {dataset} in {self.config.glob_pattern.format(dataset=dataset,projroot=self.config.projroot)}")
+                return cast(nw.LazyFrame[duckdb.DuckDBPyRelation], None)
+            self.register_files_as_view(dataset, *paths, replace=replace)
+            self.datasets[dataset] = nw.from_native(self.con.sql(f'FROM {dataset}'))
+        return self.datasets[dataset]
+
+c = nw.col
+l = nw.lit
+
+def to_narwhals(duckdb_relation: duckdb.DuckDBPyRelation) -> nw.LazyFrame[duckdb.DuckDBPyRelation]:
+    return nw.from_native(duckdb_relation)
+
+n = to_narwhals
+
+def to_duckdb(lnf: nw.LazyFrame[duckdb.DuckDBPyRelation]) -> duckdb.DuckDBPyRelation:
+    return lnf.to_native()
+
+d = to_duckdb
+
+def to_polars(lnf: nw.LazyFrame[duckdb.DuckDBPyRelation]) -> pl.DataFrame:
+    return lnf.collect(backend='polars').to_native()
+
+p = to_polars
+
+def to_pandas(lnf: nw.LazyFrame[duckdb.DuckDBPyRelation]):
+    return d(lnf).df()
+
+
+__all__ = [ "DataAccess", "c", "l", "to_narwhals", "n", "to_duckdb", "d", "to_polars", "p","to_pandas", "format_sql" ]