tablassert 19.4.0__tar.gz → 19.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-19.4.0 → tablassert-19.5.0}/PKG-INFO +1 -1
- {tablassert-19.4.0 → tablassert-19.5.0}/pyproject.toml +1 -1
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/agent.py +15 -7
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/enums.py +1 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/lib.py +22 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/models.py +8 -2
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/runcache.py +1 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/LICENSE +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/README.md +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/rust/Cargo.lock +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/rust/Cargo.toml +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/rust/examples/count_tables.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/rust/src/fullmap.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/rust/src/json.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/rust/src/lib.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/rust/src/ndjson.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/rust/src/nlp.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/rust/src/uuid.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/rust/tests/build_golden.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/rust/tests/common/mod.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/rust/tests/fixtures/nlp_golden.tsv +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/rust/tests/nlp_golden.rs +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/__init__.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/_lazy.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/biolink.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/cli.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/cli_entry.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/coerce.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/data/experimental_taxa.yaml +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/distill.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/distill_reward.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/errors.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/extras.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/fullmap.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/graph_target.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/ingests.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/log.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/net.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/nlp.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/progress.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/qc.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/rig.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/rs.pyi +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/study.py +0 -0
- {tablassert-19.4.0 → tablassert-19.5.0}/src/tablassert/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "19.
|
|
3
|
+
version = "19.5.0"
|
|
4
4
|
description = "Tablassert turns biomedical spreadsheets (Excel, CSV, TSV) into knowledge graphs ready for NCATS Translator. Declare how your columns map to subject–predicate–object statements in YAML; Tablassert resolves free text to standard CURIEs, attaches provenance and statistical annotations, and emits KGX-compliant nodes and edges."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -115,7 +115,7 @@ def is_lazy() -> bool:
|
|
|
115
115
|
PMC_BUCKET: str = "pmc-oa-opendata"
|
|
116
116
|
PMC_HTTPS_BASE: str = "https://pmc-oa-opendata.s3.amazonaws.com"
|
|
117
117
|
PMC_S3API_BASE: str = "https://pmc-oa-opendata.s3.us-east-1.amazonaws.com"
|
|
118
|
-
TABLE_EXTENSIONS: frozenset[str] = frozenset({".xlsx", ".xls", ".csv", ".tsv"})
|
|
118
|
+
TABLE_EXTENSIONS: frozenset[str] = frozenset({".xlsx", ".xls", ".csv", ".tsv", ".parquet"})
|
|
119
119
|
#: Default minimum number of non-empty data rows for agent table candidates.
|
|
120
120
|
#:
|
|
121
121
|
#: Counts use the same header inference as the production readers, so the header is not included.
|
|
@@ -662,8 +662,9 @@ def _load_table(path: Path, sheet: str | None = None) -> pl.DataFrame:
|
|
|
662
662
|
"""Dispatch a local table file to the right polars reader by suffix.
|
|
663
663
|
|
|
664
664
|
``.csv`` -> ``read_csv``; ``.tsv``/``.txt`` -> ``read_csv(separator="\\t")``;
|
|
665
|
-
``.
|
|
666
|
-
|
|
665
|
+
``.parquet`` -> ``read_parquet`` (typed columns read as-is); ``.xlsx``/``.xls`` ->
|
|
666
|
+
:func:`_read_excel` (``sheet`` selects a worksheet by name; ignored for
|
|
667
|
+
csv/tsv/parquet). Any polars parse failure becomes a clear ``ValueError``; an
|
|
667
668
|
unknown suffix is a ``ValueError`` too (never a silent mis-read).
|
|
668
669
|
"""
|
|
669
670
|
suffix: str = path.suffix.lower()
|
|
@@ -674,6 +675,8 @@ def _load_table(path: Path, sheet: str | None = None) -> pl.DataFrame:
|
|
|
674
675
|
return pl.read_csv(path)
|
|
675
676
|
if suffix in {".tsv", ".txt"}:
|
|
676
677
|
return pl.read_csv(path, separator="\t")
|
|
678
|
+
if suffix == ".parquet":
|
|
679
|
+
return pl.read_parquet(path)
|
|
677
680
|
except Exception as e:
|
|
678
681
|
raise ValueError(f"Could not read table {path}: {e}") from e
|
|
679
682
|
raise ValueError(f"Could not read table {path}: unsupported extension {suffix!r}")
|
|
@@ -685,8 +688,9 @@ def _effective_rows_cached(path: str, mtime_ns: int, size: int, sheet: str | Non
|
|
|
685
688
|
|
|
686
689
|
``path``, ``mtime_ns``, and ``size`` form the cache key so a re-fetched or rewritten file
|
|
687
690
|
cannot reuse a stale count. Delimited files are counted by the parser rather than physical
|
|
688
|
-
lines, which handles quoted embedded newlines correctly. Excel
|
|
689
|
-
:func:`_load_table` and
|
|
691
|
+
lines, which handles quoted embedded newlines correctly. Excel and parquet use the same
|
|
692
|
+
reader as :func:`_load_table` and remove rows that are null in every column (formatted
|
|
693
|
+
blank rows).
|
|
690
694
|
"""
|
|
691
695
|
table_path: Path = Path(path)
|
|
692
696
|
suffix: str = table_path.suffix.lower()
|
|
@@ -694,6 +698,10 @@ def _effective_rows_cached(path: str, mtime_ns: int, size: int, sheet: str | Non
|
|
|
694
698
|
frame: pl.DataFrame = _read_excel(table_path, sheet)
|
|
695
699
|
return frame.filter(~pl.all_horizontal(pl.all().is_null())).height
|
|
696
700
|
|
|
701
|
+
if suffix == ".parquet":
|
|
702
|
+
pq: pl.DataFrame = pl.read_parquet(table_path)
|
|
703
|
+
return pq.filter(~pl.all_horizontal(pl.all().is_null())).height
|
|
704
|
+
|
|
697
705
|
if suffix == ".csv":
|
|
698
706
|
separator: str = ","
|
|
699
707
|
elif suffix in {".tsv", ".txt"}:
|
|
@@ -1233,7 +1241,7 @@ def normalize_agent_table_config(config_yaml: str, *, base_dirs: Sequence[Path]
|
|
|
1233
1241
|
# hand-maintained guess table), so a model default change automatically changes
|
|
1234
1242
|
# what counts as removable. Semantic guards: the ``provenance`` subtree is never
|
|
1235
1243
|
# touched (legal attribution), ``kind`` is never dropped (it discriminates the
|
|
1236
|
-
# ``Excel | Text`` source union), a non-default null such as ``taxon: null``
|
|
1244
|
+
# ``Excel | Text | Parquet`` source union), a non-default null such as ``taxon: null``
|
|
1237
1245
|
# (default 9606) is preserved, and in ``{template, sections}`` configs a section
|
|
1238
1246
|
# entry equal to a model default is only removed when the template cannot change
|
|
1239
1247
|
# the merged result (fastmerge lets section scalars override template values, so
|
|
@@ -1245,7 +1253,7 @@ _COMPACT_ABSENT: object = object()
|
|
|
1245
1253
|
|
|
1246
1254
|
#: Keys compaction never removes and never recurses into: ``provenance`` values are
|
|
1247
1255
|
#: the edge's legal attribution (untouched even when they equal a model default),
|
|
1248
|
-
#: and ``kind`` discriminates the ``Excel | Text`` source union — dropping it could
|
|
1256
|
+
#: and ``kind`` discriminates the ``Excel | Text | Parquet`` source union — dropping it could
|
|
1249
1257
|
#: flip which model a re-parsed source validates as.
|
|
1250
1258
|
_COMPACT_UNTOUCHED_KEYS: frozenset[str] = frozenset({"provenance", "kind"})
|
|
1251
1259
|
|
|
@@ -953,6 +953,26 @@ def excel(p: Path, sheet: str, engine: str = "calamine") -> pl.LazyFrame:
|
|
|
953
953
|
return df.lazy()
|
|
954
954
|
|
|
955
955
|
|
|
956
|
+
def parquet(p: Path) -> pl.LazyFrame:
|
|
957
|
+
"""Read a parquet source as a LazyFrame with positional ``column_<n>`` names.
|
|
958
|
+
|
|
959
|
+
Args:
|
|
960
|
+
p: Path to the parquet file.
|
|
961
|
+
|
|
962
|
+
Returns:
|
|
963
|
+
LazyFrame whose columns are renamed to ``column_1..`` in schema order.
|
|
964
|
+
|
|
965
|
+
Notes:
|
|
966
|
+
Parquet carries its own column names, but the config contract addresses columns
|
|
967
|
+
positionally (Excel-style ``A-ZZ`` encoding letters resolve to ``column_<n>``), so
|
|
968
|
+
the reader renames whatever names the file carries to ``column_1..`` -- the same
|
|
969
|
+
convention the headerless ``csv``/``excel`` readers produce.
|
|
970
|
+
"""
|
|
971
|
+
lf: pl.LazyFrame = pl.scan_parquet(source=p)
|
|
972
|
+
names: list[str] = list(lf.collect_schema().names())
|
|
973
|
+
return lf.rename(dict(zip(names, (f"column_{i + 1}" for i in range(len(names))), strict=True)))
|
|
974
|
+
|
|
975
|
+
|
|
956
976
|
def crop(lf: pl.LazyFrame, row_slice: list[NonNegativeInt | Literal[Tokens.AUTO]]) -> pl.LazyFrame:
|
|
957
977
|
"""Take a contiguous slice from a LazyFrame.
|
|
958
978
|
|
|
@@ -1296,6 +1316,7 @@ class Tcode(Section):
|
|
|
1296
1316
|
return [
|
|
1297
1317
|
(csv, (self.source.local, self.source.delimiter)) if self.source.kind == Files.TEXT else None, # pyright: ignore
|
|
1298
1318
|
(excel, (self.source.local, self.source.sheet)) if self.source.kind == Files.EXCEL else None, # pyright: ignore
|
|
1319
|
+
(parquet, (self.source.local,)) if self.source.kind == Files.PARQUET else None, # pyright: ignore
|
|
1299
1320
|
(idx, ()),
|
|
1300
1321
|
(crop, (self.source.row_slice,)) if self.source.row_slice else None,
|
|
1301
1322
|
(pick, (self.source.rows,)) if self.source.rows else None,
|
|
@@ -1507,6 +1528,7 @@ PHASE_OF: dict[Callable, str] = {
|
|
|
1507
1528
|
# Maps each Tcode op callable to a short lowercase phase label for progress UX.
|
|
1508
1529
|
csv: "load",
|
|
1509
1530
|
excel: "load",
|
|
1531
|
+
parquet: "load",
|
|
1510
1532
|
idx: "load",
|
|
1511
1533
|
crop: "filter",
|
|
1512
1534
|
pick: "filter",
|
|
@@ -67,7 +67,7 @@ def _shown(name: str, target: str) -> str:
|
|
|
67
67
|
return f"`{name}` (coerced to `{target}`)" if target != name else f"`{name}`"
|
|
68
68
|
|
|
69
69
|
|
|
70
|
-
def _section_source_label(source: Excel | Text) -> str:
|
|
70
|
+
def _section_source_label(source: Excel | Text | Parquet) -> str:
|
|
71
71
|
"""Render a section's source for a diagnostic message: the local path, plus the sheet for Excel.
|
|
72
72
|
|
|
73
73
|
Args:
|
|
@@ -174,6 +174,12 @@ class Text(BaseSource):
|
|
|
174
174
|
delimiter: str | None = Field(",", description="Field delimiter for headerless text/CSV scanning.", examples=[",", "\t", "|"])
|
|
175
175
|
|
|
176
176
|
|
|
177
|
+
# Parquet is self-describing (typed columns, real headers, no sheets or delimiters), so unlike
|
|
178
|
+
# `Text`/`Excel` it adds no format-specific fields beyond `BaseSource`.
|
|
179
|
+
class Parquet(BaseSource):
|
|
180
|
+
kind: Literal[Files.PARQUET] = Field(Files.PARQUET, description="Source kind; must be 'parquet'.")
|
|
181
|
+
|
|
182
|
+
|
|
177
183
|
class Regex(TablaBase):
|
|
178
184
|
pattern: int | float | str = Field(..., description="Regex pattern passed to string replacement.", examples=["\\s+", "\\.$"])
|
|
179
185
|
|
|
@@ -791,7 +797,7 @@ class Annotation(Encoding):
|
|
|
791
797
|
class Section(TablaBase):
|
|
792
798
|
"""Pydantic section model and coercion target for a single table configuration."""
|
|
793
799
|
|
|
794
|
-
source: Excel | Text = Field(..., description="Input source definition for reading tabular rows.")
|
|
800
|
+
source: Excel | Text | Parquet = Field(..., description="Input source definition for reading tabular rows.")
|
|
795
801
|
statement: Statement = Field(..., description="Subject-object statement mapping for this section.")
|
|
796
802
|
provenance: Provenance = Field(..., description="Provenance metadata applied to all produced edges.")
|
|
797
803
|
annotations: list[Annotation] | None = Field(None, description="Optional extra encoded columns added to each row.")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|