piscal-processor 0.1.0__tar.gz

piscal_processor-0.1.0/PKG-INFO

@@ -0,0 +1,98 @@
+Metadata-Version: 2.4
+Name: piscal-processor
+Version: 0.1.0
+Summary: PISCAL CSV/Parquet processing: convert, export, and schema utilities for curves and measurements.
+Project-URL: Repository, https://github.com/kolpacksoftware/piscal-processor
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: pyarrow>=10.0.0
+Provides-Extra: s3
+Requires-Dist: s3fs>=2023.0.0; extra == "s3"
+Requires-Dist: fsspec>=2023.0.0; extra == "s3"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+
+# piscal-processor
+
+PISCAL CSV/Parquet processing: convert curves to Parquet, export to CSV/TSV, and use standard schemas. PISCAL is used at [LeafWeb.org](https://leafweb.org).
+
+Install from PyPI, a local clone, or a Git URL (see below).
+
+## Install
+
+**From PyPI** (when published):
+
+```bash
+pip install piscal-processor
+```
+
+**Optional S3 support** (e.g. for `s3a://` paths):
+
+```bash
+pip install piscal-processor[s3]
+```
+
+**From a local clone** (development):
+
+```bash
+pip install -e /path/to/piscal-processor
+```
+
+**From a Git URL** (CI or private install): use a personal access token or SSH:
+
+```bash
+pip install "piscal-processor @ git+https://github.com/kolpacksoftware/piscal-processor.git@main"
+# or
+pip install "piscal-processor @ git+ssh://git@github.com/kolpacksoftware/piscal-processor.git@main"
+```
+
+You can pin a branch (`@main`), tag (`@v0.1.0`), or commit (`@abc1234`).
+
+## CLI
+
+**Convert** PISCAL CSV files to Parquet (metadata + measurements):
+
+```bash
+piscal-processor /path/to/csv_dir --output-dir parquet_output
+```
+
+Options: `--no-discover-pathway-subdirs`, `--source-pathway`, `--metadata-name`, `--measurements-name`. Input can be a local path or `s3a://` URI.
+
+**Export** measurement Parquet to CSV or TSV:
+
+```bash
+piscal-processor-export curve_measurements.parquet -o out.tsv --format tsv --columns curve_id,AnetCO2,PARi
+```
+
+## Library
+
+```python
+from piscal_processor import convert_curves, export_curves, get_backend
+
+# Convert CSVs to DataFrames (or write Parquet via converter.normalize_and_write_parquet)
+backend = get_backend("/path/to/csv_dir")
+meta_df, meas_df = convert_curves("/path/to/csv_dir", backend, source_pathway="C3")
+
+# Export measurements to CSV/TSV
+export_curves(meas_df, "out.tsv", columns=["curve_id", "AnetCO2", "PARi"], format="tsv")
+```
+
+Schema constants and parser helpers are also available:
+
+```python
+from piscal_processor import STANDARD_MEASUREMENT_COLUMNS, STANDARD_METADATA_COLUMNS
+from piscal_processor.parser import parse_csv_line, parse_key_value_section
+```
+
+## Tests
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+## Documentation
+
+- `docs/csv_structure.md`: CSV file structure (header, site/parameter blocks, measurement table).
+- `docs/inputformat.txt`: PISCAL input file specification (official format description).

piscal_processor-0.1.0/README.md

@@ -0,0 +1,83 @@
+# piscal-processor
+
+PISCAL CSV/Parquet processing: convert curves to Parquet, export to CSV/TSV, and use standard schemas. PISCAL is used at [LeafWeb.org](https://leafweb.org).
+
+Install from PyPI, a local clone, or a Git URL (see below).
+
+## Install
+
+**From PyPI** (when published):
+
+```bash
+pip install piscal-processor
+```
+
+**Optional S3 support** (e.g. for `s3a://` paths):
+
+```bash
+pip install piscal-processor[s3]
+```
+
+**From a local clone** (development):
+
+```bash
+pip install -e /path/to/piscal-processor
+```
+
+**From a Git URL** (CI or private install): use a personal access token or SSH:
+
+```bash
+pip install "piscal-processor @ git+https://github.com/kolpacksoftware/piscal-processor.git@main"
+# or
+pip install "piscal-processor @ git+ssh://git@github.com/kolpacksoftware/piscal-processor.git@main"
+```
+
+You can pin a branch (`@main`), tag (`@v0.1.0`), or commit (`@abc1234`).
+
+## CLI
+
+**Convert** PISCAL CSV files to Parquet (metadata + measurements):
+
+```bash
+piscal-processor /path/to/csv_dir --output-dir parquet_output
+```
+
+Options: `--no-discover-pathway-subdirs`, `--source-pathway`, `--metadata-name`, `--measurements-name`. Input can be a local path or `s3a://` URI.
+
+**Export** measurement Parquet to CSV or TSV:
+
+```bash
+piscal-processor-export curve_measurements.parquet -o out.tsv --format tsv --columns curve_id,AnetCO2,PARi
+```
+
+## Library
+
+```python
+from piscal_processor import convert_curves, export_curves, get_backend
+
+# Convert CSVs to DataFrames (or write Parquet via converter.normalize_and_write_parquet)
+backend = get_backend("/path/to/csv_dir")
+meta_df, meas_df = convert_curves("/path/to/csv_dir", backend, source_pathway="C3")
+
+# Export measurements to CSV/TSV
+export_curves(meas_df, "out.tsv", columns=["curve_id", "AnetCO2", "PARi"], format="tsv")
+```
+
+Schema constants and parser helpers are also available:
+
+```python
+from piscal_processor import STANDARD_MEASUREMENT_COLUMNS, STANDARD_METADATA_COLUMNS
+from piscal_processor.parser import parse_csv_line, parse_key_value_section
+```
+
+## Tests
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+## Documentation
+
+- `docs/csv_structure.md`: CSV file structure (header, site/parameter blocks, measurement table).
+- `docs/inputformat.txt`: PISCAL input file specification (official format description).

piscal_processor-0.1.0/pyproject.toml

@@ -0,0 +1,32 @@
+[build-system]
+requires = ["setuptools>=61", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "piscal-processor"
+version = "0.1.0"
+description = "PISCAL CSV/Parquet processing: convert, export, and schema utilities for curves and measurements."
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "pandas>=2.0.0",
+    "pyarrow>=10.0.0",
+]
+
+[project.optional-dependencies]
+s3 = ["s3fs>=2023.0.0", "fsspec>=2023.0.0"]
+dev = ["pytest>=7.0.0"]
+
+[project.scripts]
+piscal-processor = "piscal_processor.cli:main"
+piscal-processor-export = "piscal_processor.cli:export_main"
+
+[project.urls]
+Repository = "https://github.com/kolpacksoftware/piscal-processor"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]

piscal_processor-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

piscal_processor-0.1.0/src/piscal_processor/__init__.py

@@ -0,0 +1,51 @@
+"""
+piscal-processor: PISCAL CSV/Parquet processing: convert curves, export to CSV/TSV, and use standard schemas.
+
+Use as a library:
+  from piscal_processor import convert_curves, export_curves, get_backend
+  meta, meas = convert_curves(input_dir, get_backend(input_dir))
+  export_curves(meas, "out.tsv", columns=["curve_id", "AnetCO2"], format="tsv")
+
+CLI: piscal-processor (convert), piscal-processor-export (export to CSV/TSV).
+"""
+
+from piscal_processor.converter import convert_curves, parse_curve_file
+from piscal_processor.export import export_curves, extract_columns
+from piscal_processor.parser import (
+    next_nonempty,
+    normalize_scalar,
+    parse_csv_line,
+    parse_key_value_section,
+    parse_triplet,
+)
+from piscal_processor.schema import (
+    MEASUREMENT_COLUMN_ALIASES,
+    MEASUREMENT_STRING_COLUMNS,
+    METADATA_COLUMN_ALIASES,
+    NUMERIC_METADATA_COLUMNS,
+    STANDARD_MEASUREMENT_COLUMNS,
+    STANDARD_METADATA_COLUMNS,
+)
+from piscal_processor.storage import FilesystemBackend, S3Backend, StorageBackend, get_backend
+
+__all__ = [
+    "convert_curves",
+    "parse_curve_file",
+    "export_curves",
+    "extract_columns",
+    "get_backend",
+    "StorageBackend",
+    "FilesystemBackend",
+    "S3Backend",
+    "next_nonempty",
+    "normalize_scalar",
+    "parse_csv_line",
+    "parse_key_value_section",
+    "parse_triplet",
+    "STANDARD_MEASUREMENT_COLUMNS",
+    "STANDARD_METADATA_COLUMNS",
+    "MEASUREMENT_COLUMN_ALIASES",
+    "METADATA_COLUMN_ALIASES",
+    "MEASUREMENT_STRING_COLUMNS",
+    "NUMERIC_METADATA_COLUMNS",
+]

piscal_processor-0.1.0/src/piscal_processor/cli.py

@@ -0,0 +1,156 @@
+"""Command-line interface for piscal-processor."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from piscal_processor.converter import (
+    _pathway_subdirs_from_csv_paths,
+    convert_curves,
+    normalize_and_write_parquet,
+)
+from piscal_processor.export import export_curves
+from piscal_processor.storage import get_backend
+
+
+def _parse_convert_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Convert PISCAL CSV curve files into metadata and measurement Parquet tables."
+    )
+    parser.add_argument(
+        "input_dir",
+        type=str,
+        help="Directory containing the CSV files (one curve per file). Local path or s3a:// URI.",
+    )
+    parser.add_argument(
+        "--no-discover-pathway-subdirs",
+        action="store_true",
+        help="Do not discover subdirs; treat input_dir as the folder of CSVs.",
+    )
+    parser.add_argument(
+        "--source-pathway",
+        type=str,
+        default=None,
+        help="Explicit pathway_subtype label (e.g. C4_NAD-ME). Only with --no-discover-pathway-subdirs.",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default="parquet_output",
+        help="Destination directory for Parquet output.",
+    )
+    parser.add_argument(
+        "--metadata-name",
+        default="curve_metadata.parquet",
+        help="Filename for the metadata parquet output.",
+    )
+    parser.add_argument(
+        "--measurements-name",
+        default="curve_measurements.parquet",
+        help="Filename for the measurement parquet output.",
+    )
+    return parser.parse_args()
+
+
+def _parse_export_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Export measurement Parquet to CSV or TSV (optionally selected columns)."
+    )
+    parser.add_argument(
+        "measurements_parquet",
+        type=str,
+        help="Path to curve_measurements.parquet (local or s3a://).",
+    )
+    parser.add_argument(
+        "-o", "--output",
+        type=str,
+        required=True,
+        help="Output CSV or TSV path.",
+    )
+    parser.add_argument(
+        "--columns",
+        type=str,
+        default=None,
+        help="Comma-separated column names to export (default: all).",
+    )
+    parser.add_argument(
+        "--format",
+        type=str,
+        choices=("csv", "tsv"),
+        default="csv",
+        help="Output format: csv or tsv.",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    """Main entry: convert CSV files to Parquet (default subcommand)."""
+    args = _parse_convert_args()
+    output_dir = Path(args.output_dir)
+
+    if args.no_discover_pathway_subdirs:
+        read_backend = get_backend(args.input_dir)
+        metadata_df, measurement_df = convert_curves(
+            args.input_dir, read_backend, source_pathway=args.source_pathway
+        )
+        normalize_and_write_parquet(
+            metadata_df,
+            measurement_df,
+            output_dir,
+            metadata_name=args.metadata_name,
+            measurements_name=args.measurements_name,
+        )
+        print(f"Wrote metadata to {output_dir / args.metadata_name}")
+        print(f"Wrote measurements to {output_dir / args.measurements_name}")
+        return
+
+    read_backend = get_backend(args.input_dir)
+    csv_paths = read_backend.list_csv_paths(args.input_dir)
+    subdirs = _pathway_subdirs_from_csv_paths(args.input_dir, csv_paths)
+    parent = args.input_dir.rstrip("/").split("/")[-1]
+
+    if not subdirs:
+        metadata_df, measurement_df = convert_curves(
+            args.input_dir, read_backend, source_pathway=None
+        )
+        normalize_and_write_parquet(
+            metadata_df,
+            measurement_df,
+            output_dir,
+            metadata_name=args.metadata_name,
+            measurements_name=args.measurements_name,
+        )
+        print(f"Wrote metadata to {output_dir / args.metadata_name}")
+        print(f"Wrote measurements to {output_dir / args.measurements_name}")
+    else:
+        for sub in subdirs:
+            input_path = f"{args.input_dir.rstrip('/')}/{sub}"
+            source_pathway = f"{parent}_{sub}"
+            sub_output = output_dir / source_pathway
+            metadata_df, measurement_df = convert_curves(
+                input_path, read_backend, source_pathway=source_pathway
+            )
+            normalize_and_write_parquet(
+                metadata_df,
+                measurement_df,
+                sub_output,
+                metadata_name=args.metadata_name,
+                measurements_name=args.measurements_name,
+            )
+            print(f"Wrote {source_pathway} to {sub_output}")
+
+
+def export_main() -> None:
+    """CLI entry for export: Parquet -> CSV/TSV."""
+    args = _parse_export_args()
+    backend = get_backend(args.measurements_parquet)
+    measurement_df = backend.read_parquet(args.measurements_parquet)
+    columns = [c.strip() for c in args.columns.split(",")] if args.columns else None
+    export_curves(
+        measurement_df,
+        args.output,
+        columns=columns,
+        format=args.format,
+    )
+    print(f"Exported to {args.output}")

piscal_processor-0.1.0/src/piscal_processor/converter.py

@@ -0,0 +1,226 @@
+"""Convert PISCAL CSV curves into metadata and measurement DataFrames and Parquet."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Dict, List, Set, Tuple
+
+import pandas as pd
+
+from piscal_processor.parser import (
+    next_nonempty,
+    normalize_scalar,
+    parse_csv_line,
+    parse_key_value_section,
+    parse_triplet,
+)
+from piscal_processor.schema import (
+    MEASUREMENT_COLUMN_ALIASES,
+    MEASUREMENT_STRING_COLUMNS,
+    METADATA_COLUMN_ALIASES,
+    NUMERIC_METADATA_COLUMNS,
+    STANDARD_MEASUREMENT_COLUMNS,
+    STANDARD_METADATA_COLUMNS,
+)
+from piscal_processor.storage import StorageBackend, get_backend
+
+OPTIONAL_MEASUREMENT_COLUMNS = ["BLCond"]
+
+
+def _pathway_subdirs_from_csv_paths(input_dir: str, csv_paths: List[str]) -> List[str]:
+    """Return sorted list of immediate subdir names under input_dir that contain CSVs."""
+    base = input_dir.rstrip("/")
+    if not base:
+        return []
+    subdirs: Set[str] = set()
+    for p in csv_paths:
+        p_str = str(p)
+        if p_str.startswith(base):
+            rest = p_str[len(base) :].lstrip("/")
+            if rest:
+                subdirs.add(rest.split("/")[0])
+    return sorted(subdirs)
+
+
+def pad_row(row: List[str], target_len: int) -> List[str]:
+    """Normalize a CSV row to a fixed length by padding or truncating."""
+    if len(row) < target_len:
+        return row + ["" for _ in range(target_len - len(row))]
+    if len(row) > target_len:
+        return row[:target_len]
+    return row
+
+
+def coerce_numeric_columns(df: pd.DataFrame, skip: set[str] | None = None) -> pd.DataFrame:
+    """Convert string columns to numeric when they contain numeric data."""
+    skip = skip or set()
+    for column in df.columns:
+        if column in skip:
+            continue
+        numeric = pd.to_numeric(df[column], errors="coerce")
+        if numeric.notna().any():
+            df[column] = numeric
+    return df
+
+
+def parse_curve_file(uri: str, backend: StorageBackend) -> Tuple[Dict[str, object], pd.DataFrame]:
+    """Parse a single PISCAL CSV file into metadata dict and measurements DataFrame."""
+    lines = backend.read_text(uri).splitlines()
+
+    idx, general_info = parse_key_value_section(lines)
+    idx, site_headers, _site_units, site_values = parse_triplet(lines, idx)
+    site_data = {
+        header.strip(): normalize_scalar(value)
+        for header, value in zip(site_headers, site_values)
+    }
+
+    idx, param_headers, param_units, param_values = parse_triplet(lines, idx)
+    param_data = {
+        f"param_{header.strip()}": normalize_scalar(value)
+        for header, value in zip(param_headers, param_values)
+    }
+
+    idx = next_nonempty(lines, idx)
+    measurement_headers = parse_csv_line(lines[idx])
+    idx += 1
+    idx = next_nonempty(lines, idx)
+    measurement_units = parse_csv_line(lines[idx])
+    idx += 1
+
+    data_rows: List[List[str]] = []
+    while idx < len(lines):
+        line = lines[idx]
+        idx += 1
+        if not line.strip():
+            continue
+        row = parse_csv_line(line)
+        if not row:
+            continue
+        data_rows.append(pad_row(row, len(measurement_headers)))
+
+    measurements_df = pd.DataFrame(data_rows, columns=measurement_headers)
+    measurements_df = measurements_df.rename(
+        columns={k: v for k, v in MEASUREMENT_COLUMN_ALIASES.items() if k in measurements_df.columns}
+    )
+    measurements_df.insert(0, "curve_id", backend.stem(uri))
+
+    insert_idx = 1
+    if "SpeciesSampled" in site_data:
+        measurements_df.insert(insert_idx, "SpeciesSampled", site_data.get("SpeciesSampled"))
+        insert_idx += 1
+    if "Major species" in general_info:
+        measurements_df.insert(insert_idx, "Major_species", general_info.get("Major species"))
+        insert_idx += 1
+    if "Photosynthetic pathway" in general_info:
+        measurements_df.insert(insert_idx, "Photosynthetic_pathway", general_info.get("Photosynthetic pathway"))
+        insert_idx += 1
+
+    measurements_df.replace(r"^-9999(\.0+)?$", pd.NA, regex=True, inplace=True)
+    measurements_df.replace("NA", pd.NA, inplace=True)
+
+    coerce_numeric_columns(
+        measurements_df,
+        skip={"HHMMSS", "curve_id", "SpeciesSampled", "Major_species", "Photosynthetic_pathway", "DataType", "ObsDate"},
+    )
+
+    for col in OPTIONAL_MEASUREMENT_COLUMNS:
+        if col not in measurements_df.columns:
+            measurements_df[col] = pd.NA
+
+    metadata_row: Dict[str, object] = {
+        "curve_id": backend.stem(uri),
+        "source_file": backend.name(uri),
+        **general_info,
+        **site_data,
+        **param_data,
+        "measurement_units": json.dumps(
+            {key.strip(): unit.strip() for key, unit in zip(measurement_headers, measurement_units)}
+        ),
+        "parameter_units": json.dumps(
+            {key.replace("param_", ""): unit.strip() for key, unit in zip(param_headers, param_units)}
+        ),
+    }
+
+    return metadata_row, measurements_df
+
+
+def convert_curves(
+    input_dir: str | Path,
+    backend: StorageBackend,
+    *,
+    source_pathway: str | None = None,
+) -> Tuple[pd.DataFrame, pd.DataFrame]:
+    """Convert all CSV files in a directory into combined metadata and measurement DataFrames."""
+    curve_files = backend.list_csv_paths(input_dir)
+    if not curve_files:
+        raise FileNotFoundError(f"No CSV files found in {input_dir}")
+
+    metadata_rows: List[Dict[str, object]] = []
+    measurement_frames: List[pd.DataFrame] = []
+
+    for uri in curve_files:
+        metadata_row, measurement_df = parse_curve_file(uri, backend)
+        metadata_rows.append(metadata_row)
+        measurement_frames.append(measurement_df)
+
+    all_columns = STANDARD_MEASUREMENT_COLUMNS
+
+    def _align_frame(df: pd.DataFrame) -> pd.DataFrame:
+        out = df.reindex(columns=all_columns, fill_value=pd.NA)
+        for col in OPTIONAL_MEASUREMENT_COLUMNS:
+            if col in out.columns:
+                out[col] = pd.to_numeric(out[col], errors="coerce")
+        all_na = out.columns[out.isna().all()].tolist()
+        if all_na:
+            out = out.drop(columns=all_na)
+        return out
+
+    measurement_frames = [_align_frame(df) for df in measurement_frames]
+
+    metadata_df = pd.DataFrame(metadata_rows)
+    measurement_df = pd.concat(measurement_frames, ignore_index=True, join="outer")
+    measurement_df = measurement_df.reindex(columns=STANDARD_MEASUREMENT_COLUMNS)
+
+    if source_pathway:
+        metadata_df.insert(1, "pathway_subtype", source_pathway)
+        measurement_df["pathway_subtype"] = source_pathway
+
+    return metadata_df, measurement_df
+
+
+def normalize_and_write_parquet(
+    metadata_df: pd.DataFrame,
+    measurement_df: pd.DataFrame,
+    output_dir: Path,
+    *,
+    metadata_name: str = "curve_metadata.parquet",
+    measurements_name: str = "curve_measurements.parquet",
+) -> None:
+    """Normalize schema and write both Parquet tables under output_dir."""
+    metadata_path = output_dir / metadata_name
+    measurement_path = output_dir / measurements_name
+    write_backend = get_backend(str(output_dir))
+    write_backend.ensure_output_parent(str(metadata_path))
+    write_backend.ensure_output_parent(str(measurement_path))
+
+    metadata_df = metadata_df.rename(
+        columns={k: v for k, v in METADATA_COLUMN_ALIASES.items() if k in metadata_df.columns}
+    )
+    if metadata_df.columns.duplicated().any():
+        metadata_df = metadata_df.loc[:, ~metadata_df.columns.duplicated(keep="first")]
+    metadata_df = metadata_df.reindex(columns=STANDARD_METADATA_COLUMNS)
+
+    for col in metadata_df.select_dtypes(include=["object", "string"]).columns:
+        metadata_df[col] = metadata_df[col].apply(lambda x: pd.NA if pd.isna(x) else str(x))
+    for col in NUMERIC_METADATA_COLUMNS:
+        if col in metadata_df.columns:
+            metadata_df[col] = pd.to_numeric(metadata_df[col], errors="coerce").astype("float64")
+
+    for col in MEASUREMENT_STRING_COLUMNS:
+        if col in measurement_df.columns:
+            s = measurement_df[col]
+            measurement_df[col] = s.apply(lambda x: pd.NA if pd.isna(x) else str(x)).astype("string")
+
+    write_backend.write_parquet(metadata_df, metadata_path)
+    write_backend.write_parquet(measurement_df, measurement_path)