py-devo 0.2.0__py3-none-any.whl

devo/__init__.py

@@ -0,0 +1,11 @@
+"""DEVO — CSV to iCSV enrichment and validation.
+
+Public API:
+    from devo.enrich import ICSVEnricher
+    from devo.validate import validate_icsv
+
+Intentionally imports nothing on package load to avoid side effects
+(frictionless, flask) in environments where only one function is needed.
+"""
+
+__version__ = "0.2.0"

devo/_infer.py

@@ -0,0 +1,112 @@
+"""Pure type-inference functions — no I/O, no side effects.
+
+All functions in this module are deterministic and dependency-free.
+They are shared by the enricher (CSV → type) and the validator (data re-inference).
+"""
+from __future__ import annotations
+
+import re
+from datetime import datetime
+
+# --- Constants ---
+
+INT_RE = re.compile(r"^-?\d+$")
+# Optional decimal: matches "5" and "5.0" — needed so mixed int/float columns
+# resolve to 'number' rather than falling through to 'string'.
+FLOAT_RE = re.compile(r"^-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?$")
+
+# Tried after fromisoformat fails; restored from DEVO_enricher.py (was dropped in refactor).
+STRPTIME_FORMATS: tuple[str, ...] = (
+    "%Y-%m-%d %H:%M:%S",
+    "%Y-%m-%d %H:%M",
+    "%Y-%m-%d",
+    "%d.%m.%Y",
+    "%d/%m/%Y",
+    "%m/%d/%Y",
+    "%Y/%m/%d",
+    "%d-%m-%Y",
+    "%Y%m%dT%H%M%S",
+    "%Y-%m-%dT%H:%M:%S%z",
+    "%Y-%m-%dT%H:%M:%S",
+)
+
+# iCSV spec EBNF: field_delimiter ::= [,|\/:;] — tab is not in the allowed set.
+VALID_ICSV_DELIMITERS: frozenset[str] = frozenset({",", "|", "\\", "/", ":", ";"})
+
+# Common missing-value sentinels. Single source of truth shared by enricher and validator.
+# EnviDat has no standardised sentinel, so we cast a wide net.
+COMMON_MISSING: frozenset[str] = frozenset({
+    "", "NA", "N/A", "na", "n/a", "NULL", "null", "nan", "NaN",
+    "-999", "-999.0", "-999.000000",
+})
+
+# Type subtype lattice: inferred → set of declared types it is valid under.
+# integer ⊂ number ⊂ string; datetime ⊂ string.
+# Used by the validator for Option-A cross-check (declared type is authoritative).
+_SUBTYPES: dict[str, frozenset[str]] = {
+    "integer":  frozenset({"integer", "number", "string"}),
+    "number":   frozenset({"number", "string"}),
+    "datetime": frozenset({"datetime", "string"}),
+    "string":   frozenset({"string"}),
+}
+
+
+# --- Type checkers ---
+
+def _is_integer(s: str) -> bool:
+    return bool(INT_RE.match(s))
+
+
+def _is_number(s: str) -> bool:
+    return bool(INT_RE.match(s) or FLOAT_RE.match(s))
+
+
+def _is_datetime(s: str) -> bool:
+    """Try fromisoformat first, then a fixed list of strptime formats."""
+    s = s.strip()
+    if not s:
+        return False
+    try:
+        datetime.fromisoformat(s)
+        return True
+    except (ValueError, TypeError):
+        pass
+    for fmt in STRPTIME_FORMATS:
+        try:
+            datetime.strptime(s, fmt)
+            return True
+        except (ValueError, TypeError):
+            continue
+    return False
+
+
+# --- Public API ---
+
+def infer_type(values: list[str], missing: frozenset[str] = COMMON_MISSING) -> str:
+    """
+    Infer a Frictionless field type from a list of string values.
+    Cascade: integer → number → datetime → string.
+    Missing-value sentinels are excluded before testing.
+    An all-missing or empty column returns 'string'.
+    """
+    pruned = [v.strip() for v in values if v.strip() not in missing]
+    if not pruned:
+        return "string"
+    if all(_is_integer(v) for v in pruned):
+        return "integer"
+    if all(_is_number(v) for v in pruned):
+        return "number"
+    if all(_is_datetime(v) for v in pruned):
+        return "datetime"
+    return "string"
+
+
+def is_subtype_or_equal(inferred: str, declared: str) -> bool:
+    """
+    True when the inferred type is at least as specific as (or equal to) the declared type.
+    This means existing data satisfies the declared schema:
+      - inferred=integer, declared=number  → True  (integers pass number validation)
+      - inferred=number,  declared=integer → False (floats fail integer validation)
+    Used by the validator to produce [WARN] when inferred is wider than declared.
+    """
+    return declared in _SUBTYPES.get(inferred, frozenset())

devo/_parser.py

@@ -0,0 +1,91 @@
+"""Canonical iCSV header parser — single implementation shared by enricher and validator.
+
+Parses [METADATA] and [FIELDS] sections from iCSV files per the iCSV 1.0 spec.
+Stops at # [DATA] and does not read data rows.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from .exceptions import ParseError
+
+
+@dataclass
+class ICSVHeader:
+    metadata: dict[str, str]
+    fields_meta: dict[str, list[str]]
+    field_delimiter: str
+
+
+def is_icsv(path: Path) -> bool:
+    """Return True if the file's first line marks it as an iCSV file."""
+    try:
+        # utf-8-sig strips the BOM if present
+        with open(path, "r", encoding="utf-8-sig") as fh:
+            return fh.readline().strip().startswith("# iCSV")
+    except (OSError, UnicodeDecodeError):
+        return False
+
+
+def parse_header(path: Path) -> ICSVHeader:
+    """
+    Parse [METADATA] and [FIELDS] sections of an iCSV file.
+
+    field_delimiter is read from metadata before the FIELDS section is split,
+    so key order in the file does not matter — the correct delimiter is always used.
+    Raises ParseError if the file is unreadable or has no [METADATA] section.
+    """
+    metadata: dict[str, str] = {}
+    raw_fields: dict[str, str] = {}  # key → unsplit value string; split after delimiter known
+    section: str | None = None
+
+    try:
+        with open(path, "r", encoding="utf-8-sig") as fh:
+            for line in fh:
+                stripped = line.rstrip("\r\n")
+
+                if not stripped.startswith("#"):
+                    continue
+
+                content = stripped.lstrip("#").strip()
+
+                if content == "[METADATA]":
+                    section = "metadata"
+                    continue
+                if content == "[FIELDS]":
+                    section = "fields"
+                    continue
+                if content == "[DATA]":
+                    break
+
+                # Skip blank comment lines and section headers
+                if not content or "=" not in content or section is None:
+                    continue
+
+                key, _, val = content.partition("=")
+                key = key.strip()
+                val = val.strip()
+
+                if section == "metadata":
+                    metadata[key] = val
+                else:
+                    raw_fields[key] = val
+
+    except OSError as e:
+        raise ParseError(f"Cannot read {path}: {e}") from e
+
+    if not metadata:
+        raise ParseError(f"{path.name}: no [METADATA] section found or file is empty")
+
+    field_delimiter = metadata.get("field_delimiter", ",")
+    fields_meta = {
+        k: [v.strip() for v in raw.split(field_delimiter)]
+        for k, raw in raw_fields.items()
+    }
+
+    return ICSVHeader(
+        metadata=metadata,
+        fields_meta=fields_meta,
+        field_delimiter=field_delimiter,
+    )

devo/_report.py

@@ -0,0 +1,88 @@
+"""Plain-text validation report writer.
+
+Produces a human-readable .txt file covering three checks:
+  1. Metadata completeness
+  2. Type consistency (declared vs re-inferred)
+  3. Frictionless data validation
+"""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+def write_report(
+    path: Path,
+    icsv_name: str,
+    metadata_issues: list[str],
+    type_issues: list[tuple[str, str, str, bool]],
+    frictionless_report: Any,
+    is_valid: bool,
+) -> None:
+    """
+    Write a plain-text DEVO validation report to `path`.
+
+    type_issues: list of (column_name, declared_type, inferred_type, is_ok).
+      is_ok=True means inferred is a subtype of (or equal to) declared.
+    frictionless_report: the object returned by frictionless Resource.validate().
+    """
+    now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    _SEP = "-" * 40
+
+    with open(path, "w", encoding="utf-8") as fh:
+
+        fh.write("DEVO Validation Report\n")
+        fh.write("=" * 22 + "\n")
+        fh.write(f"File:  {icsv_name}\n")
+        fh.write(f"Date:  {now}\n")
+        fh.write(f"Valid: {'YES' if is_valid else 'NO'}\n\n")
+
+        # --- Metadata ---
+        fh.write("METADATA\n")
+        fh.write(_SEP + "\n")
+        if metadata_issues:
+            for issue in metadata_issues:
+                fh.write(f"{issue}\n")
+        else:
+            fh.write("[OK] All required metadata present.\n")
+        fh.write("\n")
+
+        # --- Type consistency (Option A cross-check) ---
+        fh.write("TYPE CONSISTENCY\n")
+        fh.write(_SEP + "\n")
+        if not type_issues:
+            fh.write("[OK] No declared types to cross-check.\n")
+        else:
+            for col, declared, inferred, ok in type_issues:
+                if ok:
+                    fh.write(f"[OK]   {col}: declared={declared}, inferred={inferred}\n")
+                else:
+                    fh.write(f"[WARN] {col}: declared={declared}, inferred={inferred}\n")
+                    fh.write(
+                        f"       Inferred type is wider than declared. "
+                        f"Data may not satisfy '{declared}' constraints.\n"
+                    )
+        fh.write("\n")
+
+        # --- Frictionless data validation ---
+        fh.write("DATA VALIDATION\n")
+        fh.write(_SEP + "\n")
+        try:
+            errors = frictionless_report.flatten(
+                ["rowNumber", "fieldNumber", "fieldName", "code", "message"]
+            )
+        except (AttributeError, TypeError):
+            errors = []
+            fh.write("[WARN] Could not extract error details from frictionless report.\n")
+
+        if not errors:
+            fh.write("[PASS] No data errors found.\n")
+        else:
+            shown = errors[:50]
+            suffix = f" (showing first 50 of {len(errors)})" if len(errors) > 50 else ""
+            fh.write(f"[FAIL] {len(errors)} error(s) found{suffix}:\n")
+            for row, col_num, col_name, code, message in shown:
+                row_str = str(row) if row is not None else "?"
+                col_str = col_name or (str(col_num) if col_num is not None else "?")
+                fh.write(f"  Row {row_str}, Col {col_str} [{code}]: {message}\n")

devo/_schema.py

@@ -0,0 +1,111 @@
+"""Frictionless schema builder and per-column statistics.
+
+Separates DEVO-specific stats (min, max, missing_count — written to the iCSV FIELDS section)
+from the Frictionless schema JSON (which must only contain standard Frictionless keys).
+"""
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from ._infer import STRPTIME_FORMATS, COMMON_MISSING
+
+
+def _numeric_minmax(
+    pruned: list[str], as_type: str
+) -> tuple[Optional[float | int], Optional[float | int]]:
+    """Compute min/max for integer or number columns. Returns (None, None) on failure."""
+    if not pruned:
+        return None, None
+    try:
+        nums = [int(x) if as_type == "integer" else float(x) for x in pruned]
+        return min(nums), max(nums)
+    except (ValueError, TypeError):
+        return None, None
+
+
+def _datetime_minmax(pruned: list[str]) -> tuple[Optional[str], Optional[str]]:
+    """
+    Compute min/max for datetime columns.
+    Returns ISO-format strings or (None, None) if nothing can be parsed.
+    Uses the same format list as _infer.py to stay consistent.
+    """
+    from datetime import datetime
+
+    parsed = []
+    for v in pruned:
+        try:
+            parsed.append(datetime.fromisoformat(v))
+            continue
+        except (ValueError, TypeError):
+            pass
+        for fmt in STRPTIME_FORMATS:
+            try:
+                parsed.append(datetime.strptime(v, fmt))
+                break
+            except (ValueError, TypeError):
+                continue
+    if not parsed:
+        return None, None
+    return min(parsed).isoformat(), max(parsed).isoformat()
+
+
+def compute_col_stats(
+    vals: list[str],
+    inferred_type: str,
+    missing: frozenset[str] = COMMON_MISSING,
+) -> dict[str, Any]:
+    """
+    Compute per-column statistics for the iCSV [FIELDS] section.
+    These values go into # min =, # max =, # missing_count =.
+    They do NOT appear in the Frictionless schema JSON.
+    """
+    pruned = [v for v in vals if v not in missing and v.strip() != ""]
+    missing_count = len(vals) - len(pruned)
+    stats: dict[str, Any] = {
+        "type": inferred_type,
+        "min": None,
+        "max": None,
+        "missing_count": missing_count,
+        # required only if no missing values were observed in the current data
+        "required": missing_count == 0 and len(vals) > 0,
+    }
+    if inferred_type in ("integer", "number") and pruned:
+        stats["min"], stats["max"] = _numeric_minmax(pruned, inferred_type)
+    elif inferred_type == "datetime" and pruned:
+        stats["min"], stats["max"] = _datetime_minmax(pruned)
+    return stats
+
+
+def build_frictionless_schema(
+    header: list[str],
+    col_stats: list[dict[str, Any]],
+    missing: frozenset[str] = COMMON_MISSING,
+) -> dict[str, Any]:
+    """
+    Build a clean Frictionless Table Schema dict.
+    Only standard Frictionless keys are written here.
+    DEVO-specific stats (min, max, missing_count) live in the iCSV FIELDS section only.
+    """
+    fields = []
+    for name, stats in zip(header, col_stats):
+        field: dict[str, Any] = {"name": name, "type": stats["type"]}
+        # frictionless datetime/default rejects partial datetime strings (e.g. date-only).
+        # format=any tells frictionless to accept any parseable datetime representation,
+        # consistent with DEVO's own broad datetime detection.
+        if stats["type"] == "datetime":
+            field["format"] = "any"
+        constraints: dict[str, Any] = {}
+        if stats["min"] is not None:
+            constraints["minimum"] = stats["min"]
+        if stats["max"] is not None:
+            constraints["maximum"] = stats["max"]
+        if stats.get("required"):
+            constraints["required"] = True
+        if constraints:
+            field["constraints"] = constraints
+        fields.append(field)
+
+    return {
+        "fields": fields,
+        "missingValues": sorted(missing),
+    }

devo/cli.py

@@ -0,0 +1,104 @@
+"""Command-line front-end for DEVO.
+
+Three subcommands:
+  enrich   — CSV → iCSV + schema
+  validate — iCSV + schema → report
+  run      — enrich then validate (or just validate if input is already .icsv)
+
+Exit codes: 0 = success, 1 = validation failed (data errors), 2 = usage/runtime error.
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+
+from .enrich import ICSVEnricher
+from .exceptions import DEVOError
+from .validate import validate_icsv
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="devo", description="DEVO — CSV to iCSV enrichment and validation")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    p_enrich = sub.add_parser("enrich", help="Convert a CSV to iCSV + Frictionless schema")
+    p_enrich.add_argument("infile", help="Input CSV file")
+    p_enrich.add_argument("--out", default="DEVO_output", metavar="DIR", help="Output directory")
+    p_enrich.add_argument("--delimiter", metavar="CHAR", help="Force input delimiter")
+    p_enrich.add_argument("--nodata", metavar="VALUE", help="Force nodata sentinel")
+    p_enrich.add_argument("--app", metavar="PROFILE", help="iCSV application profile")
+
+    p_val = sub.add_parser("validate", help="Validate an iCSV against its schema")
+    p_val.add_argument("infile", help="Input .icsv file")
+    p_val.add_argument("--schema", metavar="PATH", help="Schema JSON path (default: auto-discover)")
+    p_val.add_argument("--out", default="DEVO_output", metavar="DIR", help="Output directory")
+
+    p_run = sub.add_parser(
+        "run",
+        help="Enrich then validate. If input is already .icsv, skips enrichment.",
+    )
+    p_run.add_argument("infile", help="Input CSV or iCSV file")
+    p_run.add_argument("--out", default="DEVO_output", metavar="DIR", help="Output directory")
+    p_run.add_argument("--delimiter", metavar="CHAR", help="Force input delimiter (CSV only)")
+    p_run.add_argument("--nodata", metavar="VALUE", help="Force nodata sentinel (CSV only)")
+    p_run.add_argument("--app", metavar="PROFILE", help="iCSV application profile (CSV only)")
+
+    return p
+
+
+def main(argv=None) -> None:
+    p = build_parser()
+    args = p.parse_args(argv)
+
+    try:
+        if args.cmd == "enrich":
+            enr = ICSVEnricher()
+            icsv, schema = enr.make_icsv(
+                args.infile, args.out,
+                user_delimiter=args.delimiter,
+                nodata_override=args.nodata,
+                application_profile=args.app,
+            )
+            print(f"[OK] {icsv}")
+            print(f"[OK] {schema}")
+
+        elif args.cmd == "validate":
+            report, valid = validate_icsv(
+                args.infile, schema_path=args.schema, outdir=args.out
+            )
+            print(f"[{'OK' if valid else 'FAIL'}] Report: {report}")
+            if not valid:
+                sys.exit(1)
+
+        elif args.cmd == "run":
+            from pathlib import Path
+            from ._parser import is_icsv
+
+            inpath = Path(args.infile)
+            if is_icsv(inpath):
+                # Already enriched — skip enrichment, use sibling schema if it exists
+                icsv = str(inpath)
+                schema = str(inpath.with_name(inpath.stem + "_schema.json"))
+                print(f"[OK] Input is already an iCSV — skipping enrichment.")
+            else:
+                enr = ICSVEnricher()
+                icsv, schema = enr.make_icsv(
+                    args.infile, args.out,
+                    user_delimiter=args.delimiter,
+                    nodata_override=args.nodata,
+                    application_profile=args.app,
+                )
+                print(f"[OK] Enriched: {icsv}")
+
+            report, valid = validate_icsv(icsv, schema_path=schema, outdir=args.out)
+            print(f"[{'OK' if valid else 'FAIL'}] Report: {report}")
+            if not valid:
+                sys.exit(1)
+
+    except (DEVOError, FileNotFoundError) as e:
+        print(f"[ERROR] {e}", file=sys.stderr)
+        sys.exit(2)
+
+
+if __name__ == "__main__":
+    main()

devo/enrich.py

@@ -0,0 +1,234 @@
+"""CSV → iCSV enrichment.
+
+Public API: ICSVEnricher().make_icsv(infile, outdir, ...)
+"""
+from __future__ import annotations
+
+import csv
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from ._infer import COMMON_MISSING, VALID_ICSV_DELIMITERS, infer_type
+from ._parser import is_icsv
+from ._schema import build_frictionless_schema, compute_col_stats
+from .exceptions import EnrichError
+
+
+def _detect_delimiter(sample: str) -> str:
+    """Sniff a delimiter from a text sample; fall back to comma on failure."""
+    try:
+        dialect = csv.Sniffer().sniff(sample, delimiters=",|;:\t/")
+        return dialect.delimiter
+    except csv.Error:
+        return ","
+
+
+def _to_icsv_delimiter(detected: str) -> str:
+    """
+    Map the detected input delimiter to a valid iCSV output delimiter.
+    - Comma → pipe: avoids ambiguity with the ',' separator inside metadata lines.
+    - Tab → pipe: tab is not in the iCSV spec's allowed set [,|/:;].
+    - Anything else not in the spec → pipe as a safe default.
+    """
+    if detected not in VALID_ICSV_DELIMITERS:
+        return "|"
+    return "|" if detected == "," else detected
+
+
+def _detect_geometry(header: list[str]) -> tuple[Optional[str], Optional[str]]:
+    """
+    Heuristic spatial-column detection.
+    Returns (geometry_value, srid_value) or (None, None) when no spatial columns are found.
+    Per Q1: geometry and srid are written only when spatial columns are detected.
+    """
+    lower = [h.lower() for h in header]
+    if "geometry" in lower:
+        idx = lower.index("geometry")
+        return f"column:{header[idx]}", None
+    lat_idx = lon_idx = None
+    for i, h in enumerate(lower):
+        if h in ("lat", "latitude"):
+            lat_idx = i
+        if h in ("lon", "lng", "longitude"):
+            lon_idx = i
+    if lat_idx is not None and lon_idx is not None:
+        return f"column:{header[lat_idx]},{header[lon_idx]}", "EPSG:4326"
+    return None, None
+
+
+class ICSVEnricher:
+    def __init__(self, nodata_candidates: Optional[set[str]] = None):
+        self.missing: frozenset[str] = (
+            frozenset(nodata_candidates) if nodata_candidates else COMMON_MISSING
+        )
+
+    def _load_rows(
+        self,
+        path: Path,
+        user_delimiter: Optional[str],
+    ) -> tuple[list[str], list[list[str]], str]:
+        """
+        Read a CSV in a single pass: collect a 10-line sample, sniff the delimiter,
+        then parse all rows. Returns (header, rows, detected_delimiter).
+        Using utf-8-sig to transparently strip a BOM if present.
+        """
+        lines: list[str] = []
+        try:
+            with open(path, "r", encoding="utf-8-sig", errors="replace") as fh:
+                lines = fh.readlines()
+        except OSError as e:
+            raise EnrichError(f"Cannot read {path}: {e}") from e
+
+        if not lines:
+            raise EnrichError(f"{path.name}: file is empty")
+
+        sample = "".join(lines[:10])
+        delimiter = user_delimiter or _detect_delimiter(sample)
+
+        header: list[str] = []
+        rows: list[list[str]] = []
+        for i, line in enumerate(lines):
+            row = list(csv.reader([line], delimiter=delimiter))[0]
+            if i == 0:
+                header = [c.strip() for c in row]
+            else:
+                rows.append(row)
+
+        if not header or all(c == "" for c in header):
+            raise EnrichError(f"{path.name}: no usable header row found")
+
+        return header, rows, delimiter
+
+    def _detect_nodata(self, rows: list[list[str]]) -> str:
+        """Return the most common missing-value sentinel seen in the data, or ''."""
+        counts: dict[str, int] = {}
+        for row in rows:
+            for cell in row:
+                if cell in self.missing:
+                    counts[cell] = counts.get(cell, 0) + 1
+        return max(counts, key=lambda k: counts[k]) if counts else ""
+
+    def make_icsv(
+        self,
+        infile: str,
+        outdir: str,
+        user_delimiter: Optional[str] = None,
+        nodata_override: Optional[str] = None,
+        application_profile: Optional[str] = None,
+    ) -> tuple[str, str]:
+        """
+        Convert a CSV to an iCSV + Frictionless schema JSON.
+        Returns (icsv_path, schema_path) as strings.
+        Raises EnrichError if the input is already an iCSV or cannot be read.
+        """
+        path = Path(infile)
+        if is_icsv(path):
+            raise EnrichError(
+                f"{path.name} is already an iCSV file. "
+                "Use 'devo validate' to validate it, or 'devo run' which handles both."
+            )
+
+        header, rows, detected_delim = self._load_rows(path, user_delimiter)
+        icsv_delim = _to_icsv_delimiter(detected_delim)
+        bad_names = [c for c in header if icsv_delim in c]
+        if bad_names:
+            raise EnrichError(
+                f"Column name(s) contain the iCSV delimiter '{icsv_delim}': {bad_names}. "
+                "Rename the columns or force a different delimiter with --delimiter."
+            )
+        nodata = nodata_override if nodata_override is not None else self._detect_nodata(rows)
+
+        # Normalise all rows to header length once, then transpose to column lists.
+        padded = []
+        for row in rows:
+            if len(row) < len(header):
+                row = row + [""] * (len(header) - len(row))
+            else:
+                row = row[: len(header)]
+            padded.append([c.strip() for c in row])
+
+        col_values: list[list[str]] = (
+            [[row[i] for row in padded] for i in range(len(header))]
+            if padded else [[] for _ in header]
+        )
+
+        types = [infer_type(col, self.missing) for col in col_values]
+        col_stats = [
+            compute_col_stats(col_values[i], types[i], self.missing)
+            for i in range(len(header))
+        ]
+        schema = build_frictionless_schema(header, col_stats, self.missing)
+
+        geometry, srid = _detect_geometry(header)
+
+        metadata: dict[str, str] = {
+            "iCSV_version": "1.0",
+        }
+        if application_profile:
+            metadata["application_profile"] = application_profile
+        metadata["field_delimiter"] = icsv_delim
+        metadata["rows"] = str(len(rows))
+        metadata["columns"] = str(len(header))
+        metadata["creation_date"] = (
+            datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+        )
+        if nodata:
+            metadata["nodata"] = nodata
+        if geometry:
+            metadata["geometry"] = geometry
+        if srid:
+            metadata["srid"] = srid
+        metadata["generator"] = "DEVO"
+
+        def _join(vals: list) -> str:
+            return icsv_delim.join("" if v is None else str(v) for v in vals)
+
+        fields_lines = [
+            f"fields = {_join(header)}",
+            f"types = {_join(types)}",
+            f"min = {_join(s['min'] for s in col_stats)}",
+            f"max = {_join(s['max'] for s in col_stats)}",
+            f"missing_count = {_join(s['missing_count'] for s in col_stats)}",
+            f"description = {_join('' for _ in header)}",
+        ]
+
+        out = Path(outdir)
+        out.mkdir(parents=True, exist_ok=True)
+        base = path.stem
+        icsv_path = out / f"{base}.icsv"
+        schema_path = out / f"{base}_schema.json"
+
+        self._write_icsv(icsv_path, metadata, fields_lines, header, padded, icsv_delim)
+
+        with open(schema_path, "w", encoding="utf-8") as fh:
+            json.dump(schema, fh, indent=2, ensure_ascii=False)
+
+        return str(icsv_path), str(schema_path)
+
+    def _write_icsv(
+        self,
+        path: Path,
+        metadata: dict[str, str],
+        fields_lines: list[str],
+        header: list[str],
+        rows: list[list[str]],
+        field_delimiter: str,
+    ) -> None:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with open(path, "w", encoding="utf-8", newline="") as fh:
+            fh.write("# iCSV 1.0 UTF-8\n")
+            fh.write("# [METADATA]\n")
+            for k, v in metadata.items():
+                fh.write(f"# {k} = {v}\n")
+            fh.write("\n")
+            fh.write("# [FIELDS]\n")
+            for line in fields_lines:
+                fh.write(f"# {line}\n")
+            fh.write("\n")
+            fh.write("# [DATA]\n")
+            writer = csv.writer(fh, delimiter=field_delimiter)
+            writer.writerow(header)
+            for row in rows:
+                writer.writerow(row)

devo/exceptions.py

@@ -0,0 +1,14 @@
+class DEVOError(Exception):
+    """Base for all DEVO errors — catch this to handle any DEVO failure."""
+
+
+class EnrichError(DEVOError):
+    """Raised during CSV → iCSV conversion (bad input, unreadable file, etc.)."""
+
+
+class ParseError(DEVOError):
+    """Raised when an iCSV file cannot be parsed (missing sections, malformed lines)."""
+
+
+class ValidationError(DEVOError):
+    """Raised when validation infrastructure fails — not for data errors themselves."""

devo/validate.py

@@ -0,0 +1,219 @@
+"""iCSV validation.
+
+Public API: validate_icsv(icsv_path, schema_path=None, outdir="DEVO_output")
+
+Three-stage check:
+  1. Metadata completeness (field_delimiter required; geometry/srid conditional on Q1).
+  2. Type consistency: re-infer column types from data and compare to declared types
+     (Option A: declared type is authoritative; inferred wider than declared → [WARN]).
+  3. Frictionless data validation against the schema JSON.
+"""
+from __future__ import annotations
+
+import csv
+import json
+import os
+import tempfile
+from pathlib import Path
+from typing import Optional
+
+from ._infer import COMMON_MISSING, infer_type, is_subtype_or_equal
+from ._parser import ICSVHeader, parse_header
+from ._report import write_report
+from .exceptions import ParseError, ValidationError
+
+# How many data rows (excluding header) to sample for type re-inference.
+_INFER_SAMPLE = 500
+
+
+def _check_metadata(header: ICSVHeader) -> list[str]:
+    """
+    Return a list of issue strings (empty = clean).
+    geometry/srid are only flagged when spatial column names are present (Q1).
+    srid is only required for lat/lon columns — WKT geometry embeds its own CRS.
+    """
+    issues = []
+
+    if "field_delimiter" not in header.metadata:
+        issues.append("[FAIL] Missing required metadata key: field_delimiter")
+
+    fields = header.fields_meta.get("fields", [])
+    lat_lon_names = {"lat", "latitude", "lon", "lng", "longitude"}
+    wkt_names = {"geometry"}
+    has_lat_lon = any(f.lower() in lat_lon_names for f in fields)
+    has_wkt = any(f.lower() in wkt_names for f in fields)
+
+    if has_lat_lon or has_wkt:
+        if "geometry" not in header.metadata:
+            issues.append(
+                "[WARN] Spatial columns detected but 'geometry' metadata key is missing"
+            )
+    if has_lat_lon:
+        if "srid" not in header.metadata:
+            issues.append(
+                "[WARN] Spatial columns detected but 'srid' metadata key is missing"
+            )
+
+    return issues
+
+
+def _extract_data(
+    icsv_path: Path,
+    tmp_csv: Path,
+    field_delimiter: str,
+) -> list[list[str]]:
+    """
+    Write the [DATA] section of the iCSV to a comma-delimited temp CSV.
+    Returns the first _INFER_SAMPLE data rows (excluding the header row) for type inference.
+
+    Writing with comma delimiter avoids fighting the Frictionless dialect API across v4/v5;
+    csv.writer quotes any values that contain a comma, so round-tripping is lossless.
+    """
+    sampled: list[list[str]] = []
+    in_data = False
+    header_done = False  # first DATA row is the column header, not a data row
+
+    with open(icsv_path, "r", encoding="utf-8-sig") as src, \
+         open(tmp_csv, "w", encoding="utf-8", newline="") as tgt:
+        writer = csv.writer(tgt)
+        for line in src:
+            if line.strip() == "# [DATA]":
+                in_data = True
+                continue
+            if not in_data:
+                continue
+            stripped = line.strip()
+            if not stripped or stripped.startswith("#"):
+                continue
+            row = list(csv.reader([line.rstrip("\r\n")], delimiter=field_delimiter))[0]
+            writer.writerow(row)
+            if not header_done:
+                header_done = True
+            elif len(sampled) < _INFER_SAMPLE:
+                sampled.append(row)
+
+    return sampled
+
+
+def _cross_check_types(
+    declared_types: list[str],
+    data_rows: list[list[str]],
+    field_names: list[str],
+    missing: frozenset[str] = COMMON_MISSING,
+) -> list[tuple[str, str, str, bool]]:
+    """
+    Re-infer column types from data_rows and compare to declared types.
+    Returns list of (col_name, declared, inferred, is_ok).
+    is_ok=True means inferred is a subtype of or equal to declared (Option A).
+    Pass the iCSV's own nodata sentinel merged into missing so custom sentinels
+    are not treated as real data values during re-inference.
+    """
+    if not declared_types or not data_rows:
+        return []
+
+    n = len(declared_types)
+    col_values: list[list[str]] = [[] for _ in range(n)]
+    for row in data_rows:
+        for i in range(min(len(row), n)):
+            col_values[i].append(row[i])
+
+    results = []
+    for i, declared in enumerate(declared_types):
+        name = field_names[i] if i < len(field_names) else str(i)
+        inferred = infer_type(col_values[i], missing)
+        results.append((name, declared, inferred, is_subtype_or_equal(inferred, declared)))
+    return results
+
+
+def _import_frictionless_schema():
+    """Lazy import for frictionless.Schema — avoids module-level import of an optional dep."""
+    try:
+        from frictionless import Schema
+        return Schema
+    except ImportError as exc:
+        raise ValidationError(
+            "The 'frictionless' package is required. Install it: pip install frictionless"
+        ) from exc
+
+
+def validate_icsv(
+    icsv_path: str,
+    schema_path: Optional[str] = None,
+    outdir: str = "DEVO_output",
+) -> tuple[str, bool]:
+    """
+    Validate an iCSV file. Returns (report_path, valid).
+    valid=True only when metadata is clean AND Frictionless reports no data errors.
+    Type-consistency [WARN] entries do not affect the valid flag.
+    Raises ValidationError if frictionless is not installed.
+    Raises FileNotFoundError if no schema can be found.
+    """
+    try:
+        from frictionless import Resource
+    except ImportError as exc:
+        raise ValidationError(
+            "The 'frictionless' package is required. Install it: pip install frictionless"
+        ) from exc
+
+    path = Path(icsv_path)
+    out = Path(outdir)
+    out.mkdir(parents=True, exist_ok=True)
+
+    header = parse_header(path)
+    metadata_issues = _check_metadata(header)
+    metadata_ok = not any(line.startswith("[FAIL]") for line in metadata_issues)
+
+    declared_types = header.fields_meta.get("types", [])
+    field_names = header.fields_meta.get("fields", [])
+
+    if not schema_path:
+        candidate = path.with_name(path.stem + "_schema.json")
+        if candidate.exists():
+            schema_path = str(candidate)
+        else:
+            raise FileNotFoundError(
+                f"No schema provided and none found alongside {path.name}. "
+                "Run 'devo enrich' first or pass --schema."
+            )
+
+    # Create a unique temp file so concurrent calls on different inputs do not collide.
+    fd, tmp_str = tempfile.mkstemp(suffix=".csv", dir=out)
+    os.close(fd)
+    tmp_csv = Path(tmp_str)
+
+    try:
+        data_rows = _extract_data(path, tmp_csv, header.field_delimiter)
+        nodata_val = header.metadata.get("nodata", "")
+        effective_missing = COMMON_MISSING | {nodata_val} if nodata_val else COMMON_MISSING
+        type_issues = _cross_check_types(declared_types, data_rows, field_names, effective_missing)
+
+        # frictionless v5 rejects absolute paths outside the working directory.
+        # Fix: pass the filename relative to its parent via basepath.
+        # Schema is loaded as a dict first so schema-path resolution is ours, not theirs.
+        schema_dict = json.loads(Path(schema_path).read_text(encoding="utf-8"))
+        Schema = _import_frictionless_schema()
+        schema_obj = Schema.from_descriptor(schema_dict)
+        resource = Resource(
+            path=tmp_csv.name,
+            basepath=str(tmp_csv.parent),
+            schema=schema_obj,
+        )
+        report = resource.validate()
+        data_valid = report.valid
+
+    finally:
+        if tmp_csv.exists():
+            tmp_csv.unlink()
+
+    is_valid = metadata_ok and data_valid
+    report_path = out / f"{path.stem}_DEVO_report.txt"
+    write_report(
+        path=report_path,
+        icsv_name=path.name,
+        metadata_issues=metadata_issues,
+        type_issues=type_issues,
+        frictionless_report=report,
+        is_valid=is_valid,
+    )
+
+    return str(report_path), is_valid

devo/webui.py

@@ -0,0 +1,46 @@
+"""A tiny Flask web UI which allows uploading a CSV, creating an iCSV, and running validation.
+
+This is intentionally minimal — suitable for local testing and demonstration.
+"""
+from flask import Flask, request, render_template_string, send_file
+from pathlib import Path
+from .enrich import ICSVEnricher
+from .validate import validate_icsv
+
+app = Flask(__name__)
+
+TEMPLATE = """
+<!doctype html>
+<title>DEVO demo</title>
+<h1>DEVO — upload CSV</h1>
+<form method=post enctype=multipart/form-data>
+  <input type=file name=file>
+  <input type=submit value=Upload>
+</form>
+{% if message %}
+<hr>
+<h2>Result</h2>
+<pre>{{ message }}</pre>
+{% endif %}
+"""
+
+@app.route("/", methods=["GET", "POST"])
+def index():
+    message = None
+    if request.method == "POST":
+        f = request.files.get("file")
+        if not f:
+            message = "No file uploaded"
+        else:
+            outdir = Path("DEVO_output")
+            outdir.mkdir(exist_ok=True)
+            infile = outdir / f.filename
+            f.save(infile)
+            enr = ICSVEnricher()
+            try:
+                icsv, schema = enr.make_icsv(str(infile), str(outdir))
+                report, valid = validate_icsv(icsv, schema_path=schema, outdir=str(outdir))
+                message = f"iCSV: {icsv}\nSchema: {schema}\nReport: {report}\nValid: {valid}"
+            except Exception as e:  # top-level demo catch-all; render any error to the UI rather than 500
+                message = f"Error: {e}"
+    return render_template_string(TEMPLATE, message=message)

py_devo-0.2.0.dist-info/METADATA

@@ -0,0 +1,167 @@
+Metadata-Version: 2.4
+Name: py-devo
+Version: 0.2.0
+Summary: DEVO — CSV to iCSV enrichment and Frictionless validation
+License-Expression: MIT
+Project-URL: Source, https://github.com/envidat/devo
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: frictionless>=4.0.0
+Provides-Extra: webui
+Requires-Dist: flask>=2.0.0; extra == "webui"
+Dynamic: license-file
+
+# DEVO
+<img title="whip it" alt="you know you should" height="50" src="/images/DEVO_Pixels_1.webp">
+
+**Data Enrichment and Validation Operator.** Takes a plain CSV, infers types and constraints, writes a self-documenting [iCSV](https://envidat.github.io/iCSV/) file plus a Frictionless schema, and validates the data against it.
+
+If you give it a `.csv`, it enriches → schema → validates. If you give it an `.icsv`, it skips enrichment.
+
+## Install
+
+```bash
+pip install -e .
+```
+
+For the Flask web demo:
+
+```bash
+pip install -e ".[webui]"
+```
+
+Requires Python 3.9+ and `frictionless` (v4 or v5).
+
+## Try it out
+
+A small sample dataset lives at `examples/sample.csv` — three columns (`timestamp`, `PSUM`, `TA`) representing hourly weather observations. Use it to take DEVO for a spin without needing your own data.
+
+### CLI
+
+```bash
+# Enrich, build schema, and validate in one command
+devo run examples/sample.csv
+
+# Results land in DEVO_output/ by default:
+#   sample.icsv               — annotated iCSV
+#   sample_schema.json        — Frictionless Table Schema
+#   sample_DEVO_report.txt    — human-readable validation report
+```
+
+Run `devo run examples/sample.csv --out my_output` to write to a different directory.
+
+### Python
+
+```python
+from devo.enrich import ICSVEnricher
+from devo.validate import validate_icsv
+
+icsv, schema = ICSVEnricher().make_icsv("examples/sample.csv", "DEVO_output")
+report_path, valid = validate_icsv(icsv, schema_path=schema)
+
+print(f"Valid: {valid}")
+print(f"Report written to: {report_path}")
+```
+
+### Web demo
+
+Install the optional Flask dependency first (if you haven't already):
+
+```bash
+pip install -e ".[webui]"
+```
+
+Start the local server:
+
+```bash
+flask --app devo.webui run
+```
+
+Then open `http://127.0.0.1:5000` in your browser. Click **Choose File**, select `examples/sample.csv`, and click **Upload**. The page will display the paths to the generated iCSV, schema, and report, along with the overall `Valid` result.
+
+> The web UI is a local demo only — do not expose it to a network.
+
+---
+
+## CLI
+
+```bash
+devo enrich   data.csv                    # write data.icsv + data_schema.json
+devo validate data.icsv                   # validate against neighbouring schema
+devo run      data.csv                    # do both in one go
+```
+
+Common flags: `--out DIR` (default `DEVO_output/`), `--delimiter CHAR`, `--nodata VALUE`, `--app PROFILE`, `--schema PATH`.
+
+Exit codes: `0` = success, `1` = validation failed, `2` = usage or runtime error.
+
+## What lands on disk
+
+For input `data.csv`, after `devo run`:
+
+| File | What |
+|---|---|
+| `DEVO_output/data.icsv` | iCSV with `# [METADATA]`, `# [FIELDS]`, `# [DATA]` |
+| `DEVO_output/data_schema.json` | Frictionless Table Schema JSON |
+| `DEVO_output/data_DEVO_report.txt` | Validation report (read this) |
+
+## Python API
+
+```python
+from devo.enrich import ICSVEnricher
+from devo.validate import validate_icsv
+
+icsv, schema = ICSVEnricher().make_icsv("data.csv", "DEVO_output")
+report_path, valid = validate_icsv(icsv, schema_path=schema)
+```
+
+## Files
+
+```
+devo/
+├── cli.py          # argparse front-end (enrich / validate / run)
+├── enrich.py       # CSV → iCSV + schema (ICSVEnricher class)
+├── validate.py     # iCSV + schema → Frictionless validation + report
+├── _infer.py       # pure type-inference functions (shared by enrich + validate)
+├── _parser.py      # iCSV header parser (shared by enrich + validate)
+├── _schema.py      # per-column statistics + Frictionless schema builder
+├── _report.py      # plain-text report writer
+├── exceptions.py   # DEVOError hierarchy
+└── webui.py        # Flask demo (optional; requires pip install -e ".[webui]")
+tests/
+├── conftest.py
+├── fixtures/       # sample CSV and iCSV files
+└── test_*.py
+```
+
+## How it works
+
+### Enrichment (`devo enrich`)
+
+1. **Read** — the CSV is read in one pass. If no `--delimiter` is given, `csv.Sniffer` detects it from the first 10 lines.
+2. **Delimiter mapping** — comma is remapped to pipe in the iCSV output (pipe is also the default fallback for non-spec delimiters). Column names that contain the output delimiter are rejected with a clear error.
+3. **Normalisation** — every row is padded or clipped to header length and stripped of leading/trailing whitespace.
+4. **Type inference** — each column is classified: `integer → number → datetime → string`. Scientific notation (`1.5e-3`, `2E10`) is recognised as `number`. Missing-value sentinels (and any custom `--nodata` value) are excluded before inference.
+5. **Statistics** — per-column `min`, `max`, and `missing_count` are computed from the normalised data and written to the iCSV `# [FIELDS]` section. They do not appear in the Frictionless schema JSON.
+6. **Geometry detection** — if the header contains `lat`/`latitude` + `lon`/`lng`/`longitude`, DEVO writes `geometry = column:lat,lon` and `srid = EPSG:4326` to metadata. A single column named `geometry` (WKT) gets `geometry = column:geometry` only — no `srid`, because WKT embeds its own CRS.
+7. **Write** — the normalised rows are written to the iCSV `# [DATA]` section, and the Frictionless schema is written to `_schema.json`.
+
+### Validation (`devo validate`)
+
+1. **Parse header** — `_parser.py` reads the `# [METADATA]` and `# [FIELDS]` sections, using `field_delimiter` from metadata to split field values.
+2. **Metadata check** — required keys are verified. `geometry` and `srid` are only checked when spatial column names are present; `srid` is only required for lat/lon columns (not WKT).
+3. **Type cross-check (Option A)** — column types are re-inferred from up to 500 data rows and compared to the declared types. The iCSV's own `nodata` sentinel is merged with the standard missing-value set before re-inference so custom sentinels are not mistaken for real data. Inferred type narrower than or equal to declared → `[OK]`. Inferred wider → `[WARN]`.
+4. **Frictionless validation** — data is written to a temporary comma-delimited CSV and validated against the schema using `frictionless.Resource`. The temp file is always deleted in a `finally` block.
+5. **Report** — a plain-text `.txt` report is written with three sections: `METADATA`, `TYPE CONSISTENCY`, and `DATA VALIDATION`. `Valid: YES` only when metadata has no `[FAIL]` entries and Frictionless reports no data errors. Type warnings do not affect the valid flag.
+
+## Limitations
+
+- Type inference is conservative: `integer → number → datetime → string`. Mixed-format columns fall back to `string`.
+- Datetime detection uses `datetime.fromisoformat()` and a fixed list of common strptime formats. Unusual formats need a custom schema.
+- Column descriptions are left blank in the iCSV `# [FIELDS]` section; fill them in by hand.
+- The web UI (`webui.py`) is a local demo only — do not expose it to a network.
+
+## License
+
+MIT. See `LICENSE`.

py_devo-0.2.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+devo/__init__.py,sha256=bB7C_WzwA9iA3r0u17gOINgpbNFQanfii1opOtnvNkY,318
+devo/_infer.py,sha256=_hUPO_4VVMmvuLYksHqbRxMb05iaW1YKNOrqSgghIVY,3719
+devo/_parser.py,sha256=TzQOIeIscQIh-gGzqfyiaE_cq-9cT2dxbLqNYKNyzNY,2867
+devo/_report.py,sha256=DHd58YYoM5l0SmalbVx3gKTk7ClNALP5G_34OXvkdlo,3251
+devo/_schema.py,sha256=c1HDTHqbk8QHukVruAHsyHQhCoe27wrIcbqnQ45H0IQ,3904
+devo/cli.py,sha256=Tb715NlmBMewejt9rDdohZKKxYYBtaw8Yiau7US1iZY,4059
+devo/enrich.py,sha256=3zNa7LIeusw6jfqK1WozZJYlbawIUuSHdSVKRY4jkZc,8491
+devo/exceptions.py,sha256=AtE1OEeK3CIgwWugiaWDu7jO9YpegBpWvuLu8TBH7tA,468
+devo/validate.py,sha256=oN4AN8xIIys9nS1tpG7dgRPQCPHV6jgWehL93X1KgwM,7861
+devo/webui.py,sha256=vemf9W-aMa0Nq69TXIfsUx349s60X5CU3dhJ7B5JqOs,1532
+py_devo-0.2.0.dist-info/licenses/LICENSE,sha256=EG2ApufGa51t5fPfMM9V79YYJ3QVUDqTOR251pNEQ3s,157
+py_devo-0.2.0.dist-info/METADATA,sha256=MOyDwbMwvKortJTeaBMRR8RzvCoaYfb7V4Wf4slJVLA,7343
+py_devo-0.2.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+py_devo-0.2.0.dist-info/entry_points.txt,sha256=HRSFN7b01VzDUIpM9sqW-kked7vUOBkXSSAwBSFVfqQ,39
+py_devo-0.2.0.dist-info/top_level.txt,sha256=_jGERvww3l96x1xDBpeKD7weHkxKWVR3UVMs7ty3Qqw,5
+py_devo-0.2.0.dist-info/RECORD,,

py_devo-0.2.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

py_devo-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+devo = devo.cli:main

py_devo-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,6 @@
+MIT License
+
+Copyright (c) 2026
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+... (standard MIT text elided for brevity) ...

py_devo-0.2.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+devo