hscida 0.1.0__py3-none-any.whl

hscida/__init__.py

@@ -0,0 +1,77 @@
+from dataclasses import dataclass, field
+from typing import Any, cast
+from hereutil import here
+import narwhals as nw
+import duckdb
+import os
+import polars as pl
+
+from dotenv import dotenv_values
+
+_DEFAULT_DUCKDB_CONFIG = dict(parquet_metadata_cache="true", preserve_insertion_order="false", enable_fsst_vectors="true")
+_PROJROOT = str(here())
+
+@dataclass
+class DataAccessConfig:
+    glob_pattern: str
+    init_sql: str
+    duckdb_config: dict[str, Any] = field(default_factory=lambda: _DEFAULT_DUCKDB_CONFIG)
+    projroot: str = _PROJROOT
+
+def config_from_env() -> DataAccessConfig:
+    c = {
+        **dotenv_values(),
+        **os.environ,
+    }
+    return DataAccessConfig(
+        glob_pattern=c.get('GLOB_PATTERN', ''),
+        init_sql=c.get('INIT_SQL', ''),
+        duckdb_config={k: v for k, v in [pair.split('=') for pair in c['DUCKDB_CONFIG'].split(',')] } if 'DUCKDB_CONFIG' in c else _DEFAULT_DUCKDB_CONFIG,
+        projroot=c.get('PROJROOT', _PROJROOT)
+    )
+
+class DataAccess:
+    def __init__(self, config: DataAccessConfig = config_from_env()) -> None:
+        self.con = duckdb.connect(config=config.duckdb_config)
+        self.con.sql(config.init_sql)
+        self.datasets = dict[str, nw.LazyFrame[duckdb.DuckDBPyRelation]]()
+        self.config = config
+
+    def register_files_as_view(self, table_name: str, *paths: str, replace: bool = False) -> None:
+        self.con.sql(f"CREATE {('OR REPLACE' if replace else '')} VIEW {'IF NOT EXISTS' if not replace else ''} {table_name} AS FROM read_{'parquet' if paths[0].endswith('.parquet') else 'csv'}(['{"', '".join(paths)}'], hive_partitioning=true);")
+
+    def f(self, dataset: str, *paths: str,replace: bool = False, debug: bool = False) -> nw.LazyFrame[duckdb.DuckDBPyRelation]:
+        if dataset not in self.datasets or replace:
+            if not paths:
+                paths = tuple(path[0] for path in self.con.sql("FROM "+self.config.glob_pattern.format(dataset=dataset,projroot=self.config.projroot)).fetchall())
+            if debug:
+                print(f"DEBUG: Found paths for dataset {dataset}: {paths}")
+            if not paths:
+                print(f"No files found for dataset {dataset} in {self.config.glob_pattern.format(dataset=dataset,projroot=self.config.projroot)}")
+                return cast(nw.LazyFrame[duckdb.DuckDBPyRelation], None)
+            self.register_files_as_view(dataset, *paths, replace=replace)
+            self.datasets[dataset] = nw.from_native(self.con.sql(f'FROM {dataset}'))
+        return self.datasets[dataset]
+
+c = nw.col
+l = nw.lit
+
+def to_narwhals(duckdb_relation: duckdb.DuckDBPyRelation) -> nw.LazyFrame[duckdb.DuckDBPyRelation]:
+    return nw.from_native(duckdb_relation)
+
+n = to_narwhals
+
+def to_duckdb(lnf: nw.LazyFrame[duckdb.DuckDBPyRelation]) -> duckdb.DuckDBPyRelation:
+    return lnf.to_native()
+
+d = to_duckdb
+
+def to_polars(lnf: nw.LazyFrame[duckdb.DuckDBPyRelation]) -> pl.DataFrame:
+    return lnf.collect(backend='polars').to_native()
+
+p = to_polars
+
+def to_pandas(lnf: nw.LazyFrame[duckdb.DuckDBPyRelation]):
+    return d(lnf).df()
+
+__all__ = [ "DataAccess", "c", "l", "to_narwhals", "n", "to_duckdb", "d", "to_polars", "p","to_pandas" ]

hscida-0.1.0.dist-info/METADATA

@@ -0,0 +1,53 @@
+Metadata-Version: 2.4
+Name: hscida
+Version: 0.1.0
+Summary: HSCI data access provider
+Requires-Python: >=3.12
+Requires-Dist: dotenv>=0.9.9
+Requires-Dist: duckdb>=1.5.2
+Requires-Dist: hereutil>=0.1.5
+Requires-Dist: narwhals>=2.21.0
+Requires-Dist: pandas>=3.0.3
+Requires-Dist: polars>=1.40.1
+Requires-Dist: pyarrow>=22.0.0
+Description-Content-Type: text/markdown
+
+# hsci-da
+
+Installable Python package for HSCI data access utilities built on DuckDB and Narwhals.
+
+## Install
+
+```bash
+pip install .
+```
+
+## Run tests
+
+```bash
+pytest
+```
+
+## Publish To PyPI (Local)
+
+Set your token once in your shell profile:
+
+```bash
+export UV_PUBLISH_TOKEN=pypi-xxxx
+```
+
+Release flow from your machine:
+
+```bash
+# bump version in pyproject.toml first
+uv sync --dev
+uv run pytest -q
+uv build
+uv publish
+```
+
+Dry-run against TestPyPI:
+
+```bash
+uv publish --publish-url https://test.pypi.org/legacy/
+```

hscida-0.1.0.dist-info/RECORD

@@ -0,0 +1,4 @@
+hscida/__init__.py,sha256=EQwvSz5e12kfWw72dFQdqy8ETLmls3-D82deLz1-824,3178
+hscida-0.1.0.dist-info/METADATA,sha256=lOWaogoQIHHc_4_uw6AfHlceWOesSnwWET3-OvgKCYk,883
+hscida-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+hscida-0.1.0.dist-info/RECORD,,

hscida-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any