tablassert 17.0.0__tar.gz → 18.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tablassert-17.0.0 → tablassert-18.0.0}/PKG-INFO +7 -3
- {tablassert-17.0.0 → tablassert-18.0.0}/README.md +6 -2
- {tablassert-17.0.0 → tablassert-18.0.0}/pyproject.toml +1 -1
- {tablassert-17.0.0 → tablassert-18.0.0}/rust/src/lib.rs +57 -2
- {tablassert-17.0.0 → tablassert-18.0.0}/rust/src/ndjson.rs +159 -2
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/biolink.py +21 -17
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/cli.py +46 -19
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/errors.py +22 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/fullmap.py +34 -18
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/lib.py +5 -15
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/models.py +3 -4
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/rs.pyi +1 -0
- tablassert-18.0.0/src/tablassert/utils.py +53 -0
- tablassert-17.0.0/src/tablassert/utils.py +0 -18
- {tablassert-17.0.0 → tablassert-18.0.0}/LICENSE +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/rust/Cargo.lock +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/rust/Cargo.toml +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/rust/examples/count_tables.rs +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/rust/src/fullmap.rs +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/rust/src/json.rs +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/rust/src/uuid.rs +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/rust/tests/build_golden.rs +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/rust/tests/common/mod.rs +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/rust/tests/extract_prebuilt.rs +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/__init__.py +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/_lazy.py +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/agent.py +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/coerce.py +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/distill.py +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/enums.py +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/extras.py +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/graph_target.py +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/ingests.py +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/log.py +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/nlp.py +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/progress.py +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/qc.py +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/rig.py +0 -0
- {tablassert-17.0.0 → tablassert-18.0.0}/src/tablassert/study.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tablassert
|
|
3
|
-
Version:
|
|
3
|
+
Version: 18.0.0
|
|
4
4
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
5
5
|
Classifier: Development Status :: 5 - Production/Stable
|
|
6
6
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -92,7 +92,7 @@ template:
|
|
|
92
92
|
source:
|
|
93
93
|
kind: text
|
|
94
94
|
local: ./gene-disease.csv
|
|
95
|
-
url: https://example.com/data.csv
|
|
95
|
+
url: [https://example.com/data.csv]
|
|
96
96
|
row_slice: [1, auto]
|
|
97
97
|
delimiter: ","
|
|
98
98
|
statement:
|
|
@@ -178,8 +178,9 @@ CSV/TSV/Excel sources; optional extras add runtime and pipeline capabilities:
|
|
|
178
178
|
| `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
|
|
179
179
|
| `aria2` | bundled aria2c downloader for `build-fullmap --aria2c` (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
|
|
180
180
|
| `qc` | four-stage QC audit (exact → fuzzy → abbreviation → SapBERT embeddings) | `pip install "tablassert[qc]"` |
|
|
181
|
-
| `agent` | autonomous agent (smolagents, litellm,
|
|
181
|
+
| `agent` | autonomous agent (smolagents, litellm, article/table context) | `pip install "tablassert[agent]"` |
|
|
182
182
|
| `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) | `pip install "tablassert[optimize]"` |
|
|
183
|
+
| `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) | `pip install "tablassert[distill]"` |
|
|
183
184
|
| `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` |
|
|
184
185
|
|
|
185
186
|
QC is opt-in at build time (`build-kg --qc`). Reaching a feature whose extra is not installed never
|
|
@@ -215,9 +216,12 @@ no LazyFrame setup or NLP preprocessing required. See the
|
|
|
215
216
|
- **[Tutorial](https://skyeav.github.io/Tablassert/tutorial/)**: step-by-step example with synthetic data
|
|
216
217
|
- **[CLI Reference](https://skyeav.github.io/Tablassert/cli/)**: complete command-line flag reference
|
|
217
218
|
- **[Use Case Gallery](https://skyeav.github.io/Tablassert/examples/)**: real-world configuration patterns
|
|
219
|
+
- **[Fullmap](https://skyeav.github.io/Tablassert/fullmap/)**: building and querying the entity-resolution database
|
|
218
220
|
- **[Configuration](https://skyeav.github.io/Tablassert/configuration/graph/)**: graph and table configuration reference
|
|
219
221
|
- **[Agent](https://skyeav.github.io/Tablassert/agent/)**: the autonomous agent pipeline
|
|
220
222
|
- **[API Reference](https://skyeav.github.io/Tablassert/api/fullmap/)**: core functions documentation
|
|
223
|
+
- **[Development](https://skyeav.github.io/Tablassert/development/)**: dev environment setup and contributor workflow
|
|
224
|
+
- **[Changelog](https://skyeav.github.io/Tablassert/changelog/)**: release history
|
|
221
225
|
|
|
222
226
|
## Developing
|
|
223
227
|
|
|
@@ -33,7 +33,7 @@ template:
|
|
|
33
33
|
source:
|
|
34
34
|
kind: text
|
|
35
35
|
local: ./gene-disease.csv
|
|
36
|
-
url: https://example.com/data.csv
|
|
36
|
+
url: [https://example.com/data.csv]
|
|
37
37
|
row_slice: [1, auto]
|
|
38
38
|
delimiter: ","
|
|
39
39
|
statement:
|
|
@@ -119,8 +119,9 @@ CSV/TSV/Excel sources; optional extras add runtime and pipeline capabilities:
|
|
|
119
119
|
| `rt` | CPU-compatible Polars runtime | `pip install "tablassert[rt]"` |
|
|
120
120
|
| `aria2` | bundled aria2c downloader for `build-fullmap --aria2c` (Linux/Windows wheels only) | `pip install "tablassert[aria2]"` |
|
|
121
121
|
| `qc` | four-stage QC audit (exact → fuzzy → abbreviation → SapBERT embeddings) | `pip install "tablassert[qc]"` |
|
|
122
|
-
| `agent` | autonomous agent (smolagents, litellm,
|
|
122
|
+
| `agent` | autonomous agent (smolagents, litellm, article/table context) | `pip install "tablassert[agent]"` |
|
|
123
123
|
| `optimize` | GEPA prompt optimization for `agent --optimize` (dspy) | `pip install "tablassert[optimize]"` |
|
|
124
|
+
| `distill` | distillation dataset export (`tablassert distill-export`, HF `datasets`) | `pip install "tablassert[distill]"` |
|
|
124
125
|
| `log` | loguru-backed file/progress logging (rotation, enqueue) | `pip install "tablassert[log]"` |
|
|
125
126
|
|
|
126
127
|
QC is opt-in at build time (`build-kg --qc`). Reaching a feature whose extra is not installed never
|
|
@@ -156,9 +157,12 @@ no LazyFrame setup or NLP preprocessing required. See the
|
|
|
156
157
|
- **[Tutorial](https://skyeav.github.io/Tablassert/tutorial/)**: step-by-step example with synthetic data
|
|
157
158
|
- **[CLI Reference](https://skyeav.github.io/Tablassert/cli/)**: complete command-line flag reference
|
|
158
159
|
- **[Use Case Gallery](https://skyeav.github.io/Tablassert/examples/)**: real-world configuration patterns
|
|
160
|
+
- **[Fullmap](https://skyeav.github.io/Tablassert/fullmap/)**: building and querying the entity-resolution database
|
|
159
161
|
- **[Configuration](https://skyeav.github.io/Tablassert/configuration/graph/)**: graph and table configuration reference
|
|
160
162
|
- **[Agent](https://skyeav.github.io/Tablassert/agent/)**: the autonomous agent pipeline
|
|
161
163
|
- **[API Reference](https://skyeav.github.io/Tablassert/api/fullmap/)**: core functions documentation
|
|
164
|
+
- **[Development](https://skyeav.github.io/Tablassert/development/)**: dev environment setup and contributor workflow
|
|
165
|
+
- **[Changelog](https://skyeav.github.io/Tablassert/changelog/)**: release history
|
|
162
166
|
|
|
163
167
|
## Developing
|
|
164
168
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tablassert"
|
|
3
|
-
version = "
|
|
3
|
+
version = "18.0.0"
|
|
4
4
|
description = "Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON — declaratively, with entity resolution and quality control built in."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" }
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
use pyo3::exceptions::PyOSError;
|
|
1
2
|
use pyo3::prelude::*;
|
|
2
3
|
|
|
3
4
|
// mimalloc returns memory to the OS far better than glibc malloc under heavy
|
|
@@ -5,7 +6,10 @@ use pyo3::prelude::*;
|
|
|
5
6
|
// making millions of small String/Vec allocations). Without it, per-thread
|
|
6
7
|
// malloc arenas retain freed memory and inflate peak RSS several-fold.
|
|
7
8
|
use mimalloc::MiMalloc;
|
|
8
|
-
use
|
|
9
|
+
use std::fs::File;
|
|
10
|
+
use std::io::Read;
|
|
11
|
+
use std::path::PathBuf;
|
|
12
|
+
use xxhash_rust::xxh64::{xxh64 as xxh64_digest, Xxh64};
|
|
9
13
|
|
|
10
14
|
#[global_allocator]
|
|
11
15
|
static GLOBAL: MiMalloc = MiMalloc;
|
|
@@ -26,6 +30,27 @@ fn xxh64(data: &str) -> String {
|
|
|
26
30
|
format!("{:016x}", xxh64_digest(data.as_bytes(), 0))
|
|
27
31
|
}
|
|
28
32
|
|
|
33
|
+
/// XXH64 hex digest (seed 0) of a file's raw bytes, streamed in fixed-size chunks.
|
|
34
|
+
#[pyfunction]
|
|
35
|
+
fn xxh64_file(path: PathBuf) -> PyResult<String> {
|
|
36
|
+
const CHUNK_SIZE: usize = 8 * 1024 * 1024;
|
|
37
|
+
|
|
38
|
+
let mut file = File::open(&path)
|
|
39
|
+
.map_err(|error| PyOSError::new_err(format!("{}: {}", path.display(), error)))?;
|
|
40
|
+
let mut hasher = Xxh64::new(0);
|
|
41
|
+
let mut chunk = vec![0; CHUNK_SIZE];
|
|
42
|
+
loop {
|
|
43
|
+
let bytes_read = file
|
|
44
|
+
.read(&mut chunk)
|
|
45
|
+
.map_err(|error| PyOSError::new_err(format!("{}: {}", path.display(), error)))?;
|
|
46
|
+
if bytes_read == 0 {
|
|
47
|
+
break;
|
|
48
|
+
}
|
|
49
|
+
hasher.update(&chunk[..bytes_read]);
|
|
50
|
+
}
|
|
51
|
+
Ok(format!("{:016x}", hasher.digest()))
|
|
52
|
+
}
|
|
53
|
+
|
|
29
54
|
// Public re-exports of the fullmap `#[pyfunction]`s so Rust integration tests
|
|
30
55
|
// (`rust/tests/`) and other non-Python embedders can drive the exact production
|
|
31
56
|
// build/read path. The `fullmap` module itself stays private; only these
|
|
@@ -48,12 +73,13 @@ fn rs(module: &Bound<'_, PyModule>) -> PyResult<()> {
|
|
|
48
73
|
module.add_function(wrap_pyfunction!(ndjson::dedup_ndjson, module)?)?;
|
|
49
74
|
module.add_function(wrap_pyfunction!(uuid::namespace_uuid, module)?)?;
|
|
50
75
|
module.add_function(wrap_pyfunction!(xxh64, module)?)?;
|
|
76
|
+
module.add_function(wrap_pyfunction!(xxh64_file, module)?)?;
|
|
51
77
|
Ok(())
|
|
52
78
|
}
|
|
53
79
|
|
|
54
80
|
#[cfg(test)]
|
|
55
81
|
mod tests {
|
|
56
|
-
use super::xxh64;
|
|
82
|
+
use super::{xxh64, xxh64_file};
|
|
57
83
|
|
|
58
84
|
#[test]
|
|
59
85
|
fn xxh64_matches_known_digests() {
|
|
@@ -63,4 +89,33 @@ mod tests {
|
|
|
63
89
|
assert_eq!(xxh64("hello"), "26c7827d889f6da3");
|
|
64
90
|
assert_eq!(xxh64(""), "ef46db3751d8e999");
|
|
65
91
|
}
|
|
92
|
+
|
|
93
|
+
#[test]
|
|
94
|
+
fn xxh64_file_matches_known_digests() {
|
|
95
|
+
// WHY: pin file hashing to the same seed-0 digest as the string primitive,
|
|
96
|
+
// including the empty-file identity value and UTF-8 bytes.
|
|
97
|
+
let dir = tempfile::tempdir().unwrap();
|
|
98
|
+
let path = dir.path().join("hello");
|
|
99
|
+
std::fs::write(&path, b"hello").unwrap();
|
|
100
|
+
assert_eq!(xxh64_file(path).unwrap(), "26c7827d889f6da3");
|
|
101
|
+
|
|
102
|
+
let empty = dir.path().join("empty");
|
|
103
|
+
std::fs::write(&empty, b"").unwrap();
|
|
104
|
+
assert_eq!(xxh64_file(empty).unwrap(), "ef46db3751d8e999");
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
#[test]
|
|
108
|
+
fn xxh64_file_reports_io_failures_as_os_error() {
|
|
109
|
+
// WHY: callers must distinguish I/O failures from valid digests and see
|
|
110
|
+
// the offending path, including for missing files and directories.
|
|
111
|
+
let dir = tempfile::tempdir().unwrap();
|
|
112
|
+
let missing = dir.path().join("missing");
|
|
113
|
+
let error = xxh64_file(missing.clone()).unwrap_err();
|
|
114
|
+
assert!(error.to_string().contains(&missing.display().to_string()));
|
|
115
|
+
|
|
116
|
+
let error = xxh64_file(dir.path().to_path_buf()).unwrap_err();
|
|
117
|
+
assert!(error
|
|
118
|
+
.to_string()
|
|
119
|
+
.contains(&dir.path().display().to_string()));
|
|
120
|
+
}
|
|
66
121
|
}
|
|
@@ -208,6 +208,34 @@ struct MergeIndex {
|
|
|
208
208
|
scalar_conflicts: u64,
|
|
209
209
|
}
|
|
210
210
|
|
|
211
|
+
type OriginalValues = FxHashMap<String, FxHashSet<String>>;
|
|
212
|
+
|
|
213
|
+
fn collect_original_values(value: &Value) -> OriginalValues {
|
|
214
|
+
let mut values: OriginalValues = FxHashMap::default();
|
|
215
|
+
let Some(map) = value.as_object() else {
|
|
216
|
+
return values;
|
|
217
|
+
};
|
|
218
|
+
for (key, value) in map {
|
|
219
|
+
if key.starts_with("original_") {
|
|
220
|
+
if let Value::String(text) = value {
|
|
221
|
+
for part in text.split('|').filter(|part| !part.is_empty()) {
|
|
222
|
+
values
|
|
223
|
+
.entry(key.clone())
|
|
224
|
+
.or_default()
|
|
225
|
+
.insert(part.to_string());
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
values
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
fn format_original_values(values: &FxHashSet<String>) -> String {
|
|
234
|
+
let mut sorted: Vec<&str> = values.iter().map(String::as_str).collect();
|
|
235
|
+
sorted.sort_unstable_by(|left, right| left.as_bytes().cmp(right.as_bytes()));
|
|
236
|
+
sorted.join("|")
|
|
237
|
+
}
|
|
238
|
+
|
|
211
239
|
/// The canonical bytes of one list item, shared by every structure that needs them.
|
|
212
240
|
///
|
|
213
241
|
/// ONE heap allocation per distinct item instead of one per consumer: `Rc<T>`'s `Hash`,
|
|
@@ -281,6 +309,8 @@ struct MergedRecord {
|
|
|
281
309
|
hashes: FxHashSet<u64>,
|
|
282
310
|
/// Union state for every field whose current value is an array.
|
|
283
311
|
lists: FxHashMap<String, ListState>,
|
|
312
|
+
/// Distinct non-empty scalar source values for each `original_*` field.
|
|
313
|
+
original_values: OriginalValues,
|
|
284
314
|
}
|
|
285
315
|
|
|
286
316
|
impl MergedRecord {
|
|
@@ -298,6 +328,7 @@ impl MergedRecord {
|
|
|
298
328
|
let mut hashes: FxHashSet<u64> = FxHashSet::default();
|
|
299
329
|
hashes.insert(content);
|
|
300
330
|
Ok(Self {
|
|
331
|
+
original_values: collect_original_values(&value),
|
|
301
332
|
value,
|
|
302
333
|
hashes,
|
|
303
334
|
lists,
|
|
@@ -312,6 +343,11 @@ impl MergedRecord {
|
|
|
312
343
|
let Some(map) = self.value.as_object_mut() else {
|
|
313
344
|
return Err(runtime_error("expected JSON object"));
|
|
314
345
|
};
|
|
346
|
+
for (key, values) in &self.original_values {
|
|
347
|
+
if !values.is_empty() {
|
|
348
|
+
map.insert(key.clone(), Value::String(format_original_values(values)));
|
|
349
|
+
}
|
|
350
|
+
}
|
|
315
351
|
for (key, state) in &mut self.lists {
|
|
316
352
|
if !state.unioned {
|
|
317
353
|
continue;
|
|
@@ -362,7 +398,9 @@ impl MergedRecord {
|
|
|
362
398
|
/// checked against the field's `ListState` set in O(1); stored items are never
|
|
363
399
|
/// re-canonicalized, and the sort by canonical bytes is deferred to write-out
|
|
364
400
|
/// (`MergedRecord::finish`), where it runs only for fields that saw a real union;
|
|
365
|
-
/// - scalar fields: first-wins on conflict, counted;
|
|
401
|
+
/// - scalar fields: first-wins on conflict, counted, except `original_*` fields;
|
|
402
|
+
/// - `original_*` fields collect distinct non-empty strings and render sorted values as
|
|
403
|
+
/// `A`, `A|B`, or `A|B|C`;
|
|
366
404
|
/// - fields only on `incoming`: copied over (only a conflict when both sides disagree);
|
|
367
405
|
/// - `id` is never touched: both sides carry the same one by construction.
|
|
368
406
|
///
|
|
@@ -399,6 +437,15 @@ fn merge_records(stored: &mut MergedRecord, incoming: &Value) -> PyResult<u64> {
|
|
|
399
437
|
stored
|
|
400
438
|
.lists
|
|
401
439
|
.insert(key.clone(), ListState::from_items(items)?);
|
|
440
|
+
} else if key.starts_with("original_") {
|
|
441
|
+
if let Value::String(text) = incoming_value {
|
|
442
|
+
let values = stored.original_values.entry(key.clone()).or_default();
|
|
443
|
+
values.extend(
|
|
444
|
+
text.split('|')
|
|
445
|
+
.filter(|part| !part.is_empty())
|
|
446
|
+
.map(str::to_owned),
|
|
447
|
+
);
|
|
448
|
+
}
|
|
402
449
|
}
|
|
403
450
|
stored_map.insert(key.clone(), incoming_value.clone());
|
|
404
451
|
}
|
|
@@ -422,6 +469,26 @@ fn merge_records(stored: &mut MergedRecord, incoming: &Value) -> PyResult<u64> {
|
|
|
422
469
|
stored_items.push(item.clone());
|
|
423
470
|
}
|
|
424
471
|
}
|
|
472
|
+
} else if key.starts_with("original_") {
|
|
473
|
+
if let (Value::String(stored_text), Value::String(incoming_text)) =
|
|
474
|
+
(&*stored_value, incoming_value)
|
|
475
|
+
{
|
|
476
|
+
let values = stored.original_values.entry(key.clone()).or_default();
|
|
477
|
+
values.extend(
|
|
478
|
+
stored_text
|
|
479
|
+
.split('|')
|
|
480
|
+
.filter(|part| !part.is_empty())
|
|
481
|
+
.map(str::to_owned),
|
|
482
|
+
);
|
|
483
|
+
values.extend(
|
|
484
|
+
incoming_text
|
|
485
|
+
.split('|')
|
|
486
|
+
.filter(|part| !part.is_empty())
|
|
487
|
+
.map(str::to_owned),
|
|
488
|
+
);
|
|
489
|
+
} else if stored_value != incoming_value {
|
|
490
|
+
conflicts += 1;
|
|
491
|
+
}
|
|
425
492
|
} else if stored_value != incoming_value {
|
|
426
493
|
conflicts += 1;
|
|
427
494
|
}
|
|
@@ -529,6 +596,10 @@ fn merge_records_reference(stored: &mut Value, incoming: &Value) -> PyResult<u64
|
|
|
529
596
|
return Err(runtime_error("expected JSON object"));
|
|
530
597
|
};
|
|
531
598
|
let mut conflicts: u64 = 0;
|
|
599
|
+
let mut original_values: OriginalValues = collect_original_values(stored);
|
|
600
|
+
for (key, values) in collect_original_values(incoming) {
|
|
601
|
+
original_values.entry(key).or_default().extend(values);
|
|
602
|
+
}
|
|
532
603
|
// Read both counts BEFORE the fold: the fold may copy incoming's over a stored side
|
|
533
604
|
// that lacks it, and the recompute rule below needs to know each side contributed one.
|
|
534
605
|
let stored_cases: Option<Value> = stored.get("number_of_cases").cloned();
|
|
@@ -565,12 +636,21 @@ fn merge_records_reference(stored: &mut Value, incoming: &Value) -> PyResult<u64
|
|
|
565
636
|
}
|
|
566
637
|
keyed.sort_by(|left, right| left.0.cmp(&right.0));
|
|
567
638
|
stored_items.extend(keyed.into_iter().map(|(_, item)| item));
|
|
639
|
+
} else if key.starts_with("original_") {
|
|
640
|
+
// Aggregated and rendered after this fold, once all values are known.
|
|
568
641
|
} else if stored_value != incoming_value {
|
|
569
642
|
conflicts += 1;
|
|
570
643
|
}
|
|
571
644
|
}
|
|
572
645
|
}
|
|
573
646
|
}
|
|
647
|
+
if let Some(map) = stored.as_object_mut() {
|
|
648
|
+
for (key, values) in &original_values {
|
|
649
|
+
if !values.is_empty() {
|
|
650
|
+
map.insert(key.clone(), Value::String(format_original_values(values)));
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
}
|
|
574
654
|
// WHY: exact-unique `number_of_cases` semantics (see the docstring). Guarded on the
|
|
575
655
|
// merged record actually carrying the ID list: a one-sided carrier is fine (the union
|
|
576
656
|
// is just that side's list), while a carrier-less merge never recomputes.
|
|
@@ -759,8 +839,9 @@ pub fn dedup_ndjson(
|
|
|
759
839
|
|
|
760
840
|
#[cfg(test)]
|
|
761
841
|
mod tests {
|
|
762
|
-
use super::{dedup_ndjson, differing_keys, record_if_new};
|
|
842
|
+
use super::{dedup_ndjson, differing_keys, format_original_values, record_if_new};
|
|
763
843
|
use rustc_hash::FxHashMap;
|
|
844
|
+
use rustc_hash::FxHashSet;
|
|
764
845
|
use serde_json::Value;
|
|
765
846
|
use std::fs;
|
|
766
847
|
use tempfile::tempdir;
|
|
@@ -1465,6 +1546,82 @@ mod tests {
|
|
|
1465
1546
|
assert_eq!(sources[1]["resource_id"], serde_json::json!("infores:b"));
|
|
1466
1547
|
}
|
|
1467
1548
|
|
|
1549
|
+
#[test]
|
|
1550
|
+
fn format_original_values_uses_sorted_pipe_joining() {
|
|
1551
|
+
let empty: FxHashSet<String> = FxHashSet::default();
|
|
1552
|
+
assert_eq!(format_original_values(&empty), "");
|
|
1553
|
+
|
|
1554
|
+
let one = FxHashSet::from_iter([String::from("only")]);
|
|
1555
|
+
assert_eq!(format_original_values(&one), "only");
|
|
1556
|
+
|
|
1557
|
+
let two = FxHashSet::from_iter([String::from("z"), String::from("a")]);
|
|
1558
|
+
assert_eq!(format_original_values(&two), "a|z");
|
|
1559
|
+
|
|
1560
|
+
let three = FxHashSet::from_iter([String::from("z"), String::from("a"), String::from("m")]);
|
|
1561
|
+
assert_eq!(format_original_values(&three), "a|m|z");
|
|
1562
|
+
}
|
|
1563
|
+
|
|
1564
|
+
#[test]
|
|
1565
|
+
fn merge_mode_aggregates_original_scalars_across_all_records() {
|
|
1566
|
+
let dir = tempdir().expect("tempdir");
|
|
1567
|
+
let (input, output) = write_edges(
|
|
1568
|
+
dir.path(),
|
|
1569
|
+
concat!(
|
|
1570
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"original_subject\":\"z\",\"original_object\":\"\"}\n",
|
|
1571
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"original_subject\":\"a\",\"original_object\":\"x\"}\n",
|
|
1572
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"original_subject\":\"m\",\"original_object\":\"y\"}\n",
|
|
1573
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"original_subject\":\"a\",\"original_object\":\"x\"}\n"
|
|
1574
|
+
),
|
|
1575
|
+
);
|
|
1576
|
+
let (merged, conflicts) = dedup_ndjson(
|
|
1577
|
+
input,
|
|
1578
|
+
output.clone(),
|
|
1579
|
+
true,
|
|
1580
|
+
None,
|
|
1581
|
+
spo_fields(),
|
|
1582
|
+
Some("merge".to_string()),
|
|
1583
|
+
)
|
|
1584
|
+
.expect("merge dedup");
|
|
1585
|
+
|
|
1586
|
+
assert_eq!((merged, conflicts), (2, 0));
|
|
1587
|
+
let edge = merged_edge(&output);
|
|
1588
|
+
assert_eq!(edge["original_subject"], serde_json::json!("a|m|z"));
|
|
1589
|
+
assert_eq!(edge["original_object"], serde_json::json!("x|y"));
|
|
1590
|
+
}
|
|
1591
|
+
|
|
1592
|
+
#[test]
|
|
1593
|
+
fn merge_mode_ignores_empty_originals_and_is_order_independent() {
|
|
1594
|
+
let write = |dir: &std::path::Path, rows: &str| {
|
|
1595
|
+
let (input, output) = write_edges(dir, rows);
|
|
1596
|
+
dedup_ndjson(
|
|
1597
|
+
input,
|
|
1598
|
+
output.clone(),
|
|
1599
|
+
true,
|
|
1600
|
+
None,
|
|
1601
|
+
spo_fields(),
|
|
1602
|
+
Some("merge".to_string()),
|
|
1603
|
+
)
|
|
1604
|
+
.expect("merge dedup");
|
|
1605
|
+
merged_edge(&output)["original_subject"].clone()
|
|
1606
|
+
};
|
|
1607
|
+
let first = write(
|
|
1608
|
+
tempdir().expect("tempdir").path(),
|
|
1609
|
+
concat!(
|
|
1610
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"original_subject\":\"\"}\n",
|
|
1611
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"original_subject\":\"only\"}\n"
|
|
1612
|
+
),
|
|
1613
|
+
);
|
|
1614
|
+
let second = write(
|
|
1615
|
+
tempdir().expect("tempdir").path(),
|
|
1616
|
+
concat!(
|
|
1617
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"original_subject\":\"only\"}\n",
|
|
1618
|
+
"{\"subject\":\"A\",\"predicate\":\"r\",\"object\":\"B\",\"original_subject\":\"\"}\n"
|
|
1619
|
+
),
|
|
1620
|
+
);
|
|
1621
|
+
assert_eq!(first, serde_json::json!("only"));
|
|
1622
|
+
assert_eq!(first, second);
|
|
1623
|
+
}
|
|
1624
|
+
|
|
1468
1625
|
#[test]
|
|
1469
1626
|
fn merge_mode_keeps_fields_only_the_second_record_carries() {
|
|
1470
1627
|
// WHY: first-wins arbitrates CONFLICTS; a field absent from the first record is
|
|
@@ -257,9 +257,9 @@ def _association_model_fields() -> set[str]:
|
|
|
257
257
|
|
|
258
258
|
Unions ``model_fields`` across the entire association family rather than only
|
|
259
259
|
walking ``Association.__mro__``. Walking the base MRO alone silently excludes
|
|
260
|
-
subclass-only evidence slots -- ``clinical_approval_status
|
|
261
|
-
``number_of_cases
|
|
262
|
-
|
|
260
|
+
subclass-only evidence slots -- ``clinical_approval_status`` and
|
|
261
|
+
``number_of_cases`` -- which then get demoted into ``supporting_text`` by
|
|
262
|
+
:func:`lib.fold_unknown_to_supporting_text`.
|
|
263
263
|
"""
|
|
264
264
|
fields: set[str] = set()
|
|
265
265
|
for klass in _association_classes():
|
|
@@ -529,8 +529,8 @@ release from making ``species_context_qualifier`` silently emittable again.
|
|
|
529
529
|
|
|
530
530
|
|
|
531
531
|
CLASS_FIELD_OVERRIDES: dict[str, frozenset[str]] = {
|
|
532
|
-
"EntityToDiseaseAssociation": frozenset({"disease_context_qualifier"}),
|
|
533
|
-
"EntityToPhenotypicFeatureAssociation": frozenset({"disease_context_qualifier"}),
|
|
532
|
+
"EntityToDiseaseAssociation": frozenset({"disease_context_qualifier", "regulatory_approvals"}),
|
|
533
|
+
"EntityToPhenotypicFeatureAssociation": frozenset({"disease_context_qualifier", "regulatory_approvals"}),
|
|
534
534
|
}
|
|
535
535
|
"""Per-class grants of edge fields the resolved association class does not declare.
|
|
536
536
|
|
|
@@ -538,15 +538,14 @@ Keys are bare association class names (``association_class(cat).__name__``), val
|
|
|
538
538
|
the slots ``lib.prune_to_class`` keeps on rows resolved to that class even though the
|
|
539
539
|
installed model attaches them elsewhere.
|
|
540
540
|
|
|
541
|
-
The motivating case is a DAKP contraindication edge: ``
|
|
542
|
-
|
|
543
|
-
|
|
541
|
+
The motivating case is a DAKP contraindication edge: ``regulatory_approvals`` is a
|
|
542
|
+
canonical slot not yet attached by the installed model, while
|
|
543
|
+
``disease_context_qualifier`` is declared only on the
|
|
544
544
|
``ChemicalEntityToDiseaseOrPhenotypicFeatureAssociation`` lineage -- so one edge can
|
|
545
|
-
natively carry
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
deliberate gap is not reported as ``extra_forbidden``.
|
|
545
|
+
natively carry both only through these explicit class-scoped grants. Tablassert
|
|
546
|
+
deliberately emits the granted fields on the pinned classes ahead of the pinned model
|
|
547
|
+
(pending an upstream Biolink widening). ``_validation_record`` strips granted fields
|
|
548
|
+
before record validation so the deliberate gap is not reported as ``extra_forbidden``.
|
|
550
549
|
|
|
551
550
|
A tripwire test asserts every granted field is still absent from its class: the moment
|
|
552
551
|
a biolink-model release attaches the slot, the suite fails and the stale grant is
|
|
@@ -582,7 +581,12 @@ unresolved.
|
|
|
582
581
|
|
|
583
582
|
|
|
584
583
|
ALLOWED_EDGE_FIELDS: frozenset[str] = (
|
|
585
|
-
(
|
|
584
|
+
(
|
|
585
|
+
frozenset(_association_model_fields())
|
|
586
|
+
| {q.value for q in Qualifiers}
|
|
587
|
+
| TABLASERT_EDGE_EXTRAS
|
|
588
|
+
| frozenset().union(*CLASS_FIELD_OVERRIDES.values())
|
|
589
|
+
)
|
|
586
590
|
- UNSATISFIABLE_EDGE_FIELDS
|
|
587
591
|
- DISABLED_EDGE_FIELDS
|
|
588
592
|
)
|
|
@@ -591,9 +595,9 @@ ALLOWED_EDGE_FIELDS: frozenset[str] = (
|
|
|
591
595
|
Any column on an edge frame that is not in this set is folded into the
|
|
592
596
|
``supporting_text`` ``list[str]`` field by ``lib.fold_unknown_to_supporting_text()``
|
|
593
597
|
as a ``"column: value"`` string. Composed of the fields declared by *any* Biolink
|
|
594
|
-
association class, the derived qualifier slot names,
|
|
595
|
-
``TABLASERT_EDGE_EXTRAS
|
|
596
|
-
fields disabled by Tablassert policy.
|
|
598
|
+
association class, the derived qualifier slot names, the curated
|
|
599
|
+
``TABLASERT_EDGE_EXTRAS``, and explicit :data:`CLASS_FIELD_OVERRIDES` grants -- less the
|
|
600
|
+
slots that no Pydantic class can hold and the fields disabled by Tablassert policy.
|
|
597
601
|
|
|
598
602
|
Note this is a per-*family* allow-list: a field being permitted here does not mean the
|
|
599
603
|
specific association class chosen for a given edge accepts it. Per-record pruning
|
|
@@ -75,6 +75,44 @@ def _section_store_path(h: str, head: bool = False, release: bool = False, qc: b
|
|
|
75
75
|
return STORE / f"{h}{suffix}.parquet"
|
|
76
76
|
|
|
77
77
|
|
|
78
|
+
def _section_store_key_for_build(
|
|
79
|
+
section: dict[str, Any], configuration_file: Path, content_hashes: dict[tuple[Path, int, int], str]
|
|
80
|
+
) -> tuple[str, str]:
|
|
81
|
+
"""Return a section's content-aware store key and config-only fallback label.
|
|
82
|
+
|
|
83
|
+
The memo belongs to one ``build_graph_pipeline`` call and is invalidated by any
|
|
84
|
+
``(resolved path, mtime_ns, size)`` change. Progress and validation errors use the
|
|
85
|
+
content-aware key; the returned config-only hash labels ``SourceFileError`` when
|
|
86
|
+
hashing fails before a content-aware key can exist.
|
|
87
|
+
"""
|
|
88
|
+
from tablassert.utils import file_content_hash, mkhash, section_store_key
|
|
89
|
+
|
|
90
|
+
config_hash: str = mkhash(section)
|
|
91
|
+
source: object = section.get("source")
|
|
92
|
+
local: object = source.get("local") if isinstance(source, dict) else None
|
|
93
|
+
if local is None:
|
|
94
|
+
return section_store_key(section), config_hash
|
|
95
|
+
|
|
96
|
+
resolved: Path = Path(str(local)).resolve()
|
|
97
|
+
config_value: object = section.get("config", "section")
|
|
98
|
+
section_label: str = f"{Path(str(config_value)).stem} · {config_hash[:8]}"
|
|
99
|
+
digest: str | None = None
|
|
100
|
+
try:
|
|
101
|
+
stat = resolved.stat()
|
|
102
|
+
except OSError:
|
|
103
|
+
# Reuse the contextual helper so missing/unreadable paths never leak a raw OS error.
|
|
104
|
+
digest = file_content_hash(resolved, config=configuration_file, section_label=section_label)
|
|
105
|
+
return section_store_key(section, content_digest=digest), config_hash
|
|
106
|
+
|
|
107
|
+
signature: tuple[Path, int, int] = (resolved, stat.st_mtime_ns, stat.st_size)
|
|
108
|
+
if digest is None:
|
|
109
|
+
digest = content_hashes.get(signature)
|
|
110
|
+
if digest is None:
|
|
111
|
+
digest = file_content_hash(resolved, config=configuration_file, section_label=section_label)
|
|
112
|
+
content_hashes[signature] = digest
|
|
113
|
+
return section_store_key(section, content_digest=digest), config_hash
|
|
114
|
+
|
|
115
|
+
|
|
78
116
|
def _load_table_indexed(args: tuple[int, Path]) -> tuple[int, object]:
|
|
79
117
|
"""Load one table, tagged with its input index (multiprocessing worker).
|
|
80
118
|
|
|
@@ -139,17 +177,11 @@ def _load_graph(configuration_file: Path) -> Graph:
|
|
|
139
177
|
|
|
140
178
|
|
|
141
179
|
def build_pipeline(
|
|
142
|
-
configuration_file: Path,
|
|
143
|
-
progress: PipelineProgress,
|
|
144
|
-
release: bool = False,
|
|
145
|
-
qc: bool = False,
|
|
146
|
-
log: bool = False,
|
|
147
|
-
head: bool = False,
|
|
148
|
-
no_original: bool = False,
|
|
180
|
+
configuration_file: Path, progress: PipelineProgress, release: bool = False, qc: bool = False, log: bool = False, head: bool = False
|
|
149
181
|
) -> None:
|
|
150
182
|
"""Load a graph YAML and build it through the shared in-process core."""
|
|
151
183
|
graph: Graph = _load_graph(configuration_file)
|
|
152
|
-
build_graph_pipeline(graph, configuration_file, progress, release=release, qc=qc, log=log, head=head
|
|
184
|
+
build_graph_pipeline(graph, configuration_file, progress, release=release, qc=qc, log=log, head=head)
|
|
153
185
|
|
|
154
186
|
|
|
155
187
|
def build_graph_pipeline(
|
|
@@ -160,7 +192,6 @@ def build_graph_pipeline(
|
|
|
160
192
|
qc: bool = False,
|
|
161
193
|
log: bool = False,
|
|
162
194
|
head: bool = False,
|
|
163
|
-
no_original: bool = False,
|
|
164
195
|
audit_sources: bool = True,
|
|
165
196
|
) -> None:
|
|
166
197
|
"""Build a validated :class:`Graph` without loading another graph YAML.
|
|
@@ -178,13 +209,10 @@ def build_graph_pipeline(
|
|
|
178
209
|
qc: When ``True``, run quality-control audits and final study assertions.
|
|
179
210
|
log: When ``True``, enable per-section verbose logging.
|
|
180
211
|
head: When ``True``, build a random sample of up to five rows per section.
|
|
181
|
-
no_original: When ``True``, omit the verbatim ``original_*`` source-cell
|
|
182
|
-
copies from the final edge NDJSON.
|
|
183
212
|
"""
|
|
184
213
|
from tablassert.fullmap import fullmap_db_path
|
|
185
214
|
from tablassert.lib import Tcode, compile_graph, compile_subgraph
|
|
186
215
|
from tablassert.progress import flatten_pydantic_error, format_section_compact
|
|
187
|
-
from tablassert.utils import mkhash
|
|
188
216
|
|
|
189
217
|
# Stage 1/6: load tables.
|
|
190
218
|
progress.stage("Loading Tables")
|
|
@@ -223,8 +251,11 @@ def build_graph_pipeline(
|
|
|
223
251
|
progress.stage("Building TCode")
|
|
224
252
|
start, advance, _ = progress.section_loop(n, "TCode")
|
|
225
253
|
tcode: list[Tcode] = []
|
|
254
|
+
# This memo is intentionally scoped to one build: a changed stat signature re-hashes,
|
|
255
|
+
# while repeated sections pointing to the same unchanged file hash only once.
|
|
256
|
+
content_hashes: dict[tuple[Path, int, int], str] = {}
|
|
226
257
|
for s in sections:
|
|
227
|
-
h
|
|
258
|
+
h, _ = _section_store_key_for_build(s, configuration_file, content_hashes)
|
|
228
259
|
start(f"{Path(str(s['config'])).stem} · {h[:8]}")
|
|
229
260
|
# Mode flags change the cached parquet's content, so each combination caches
|
|
230
261
|
# to a distinct file and can never quick-exit another mode's build.
|
|
@@ -284,7 +315,6 @@ def build_graph_pipeline(
|
|
|
284
315
|
section_sources if audit_sources else None,
|
|
285
316
|
on_phase=sub_step,
|
|
286
317
|
on_subgraph=advance,
|
|
287
|
-
no_original=no_original,
|
|
288
318
|
uuid_fields=g.uuid_fields,
|
|
289
319
|
uuid_domain=g.uuid_namespace,
|
|
290
320
|
uuid_on_collision=g.uuid_on_collision,
|
|
@@ -691,16 +721,13 @@ def build_kg(
|
|
|
691
721
|
qc: Annotated[bool, cyclopts.Parameter(name=["--qc", "-q"], negative="")] = False,
|
|
692
722
|
log: Annotated[bool, cyclopts.Parameter(name=["--log", "-l"], negative="")] = False,
|
|
693
723
|
head: Annotated[bool, cyclopts.Parameter(name=["--head", "-hd"], negative="")] = False,
|
|
694
|
-
no_original: Annotated[bool, cyclopts.Parameter(name=["--no-original", "-no"], negative="")] = False,
|
|
695
724
|
) -> None:
|
|
696
725
|
"""Build a knowledge graph from a YAML configuration file.
|
|
697
726
|
|
|
698
727
|
The positional config is a Graph YAML that orchestrates one or more table
|
|
699
728
|
configs into a single knowledge-graph build.
|
|
700
729
|
|
|
701
|
-
``--
|
|
702
|
-
``original_object``, and any other ``original_*`` fields) from the final edge
|
|
703
|
-
NDJSON. ``--qc`` requires the ``[qc]`` extra (``pip install
|
|
730
|
+
``--qc`` requires the ``[qc]`` extra (``pip install
|
|
704
731
|
"tablassert[qc]"``); it is checked before the build starts, because the audit stage
|
|
705
732
|
runs LAST and a missing extra would otherwise surface only after entity resolution
|
|
706
733
|
has finished. It also runs a final study stage that asserts over the emitted NDJSON
|
|
@@ -711,7 +738,7 @@ def build_kg(
|
|
|
711
738
|
"""
|
|
712
739
|
if qc:
|
|
713
740
|
extras.require("qc", required_by="--qc")
|
|
714
|
-
run(7 if qc else 6, build_pipeline, graph_configuration_file, release=release, qc=qc, log=log, head=head
|
|
741
|
+
run(7 if qc else 6, build_pipeline, graph_configuration_file, release=release, qc=qc, log=log, head=head)
|
|
715
742
|
|
|
716
743
|
|
|
717
744
|
@APP.command(name="validate")
|
|
@@ -12,6 +12,7 @@ TablassertErrorCodes = Literal[
|
|
|
12
12
|
"graph-validation-failed",
|
|
13
13
|
"section-validation-failed",
|
|
14
14
|
"babel-download-failed",
|
|
15
|
+
"source-file-unreadable",
|
|
15
16
|
"resolve-bad-specs",
|
|
16
17
|
"config-rows-and-row-slice-conflict",
|
|
17
18
|
"comparison-bad-comparator-type",
|
|
@@ -185,3 +186,24 @@ class BabelDownloadError(TablassertError):
|
|
|
185
186
|
f"BABEL download failed after {retries} attempts: {url} (last error: {last_error}). Check network connectivity or pin a different BABEL version.",
|
|
186
187
|
code="babel-download-failed",
|
|
187
188
|
)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
class SourceFileError(TablassertError):
|
|
192
|
+
"""A table section's ``source.local`` file could not be read for content-hashing.
|
|
193
|
+
|
|
194
|
+
Notes:
|
|
195
|
+
``config`` and ``section_label`` are optional keyword context: the utils layer
|
|
196
|
+
that first notices the failure often lacks build context, but callers that have
|
|
197
|
+
it (the CLI's section loop) should pass them so the message names the exact
|
|
198
|
+
table config and section label (config stem + 8-char section digest).
|
|
199
|
+
"""
|
|
200
|
+
|
|
201
|
+
def __init__(self, path: Path, detail: str, *, config: Path | None = None, section_label: str | None = None) -> None:
|
|
202
|
+
context: str = ""
|
|
203
|
+
if config is not None and section_label is not None:
|
|
204
|
+
context = f" (config {config}, section {section_label})"
|
|
205
|
+
elif config is not None:
|
|
206
|
+
context = f" (config {config})"
|
|
207
|
+
elif section_label is not None:
|
|
208
|
+
context = f" (section {section_label})"
|
|
209
|
+
super().__init__(f"Source file unreadable{context}: {path} — {detail}", code="source-file-unreadable")
|
|
@@ -289,31 +289,37 @@ def distinct(lf: pl.LazyFrame, l1: str, l2: str, col: str = "term") -> pl.LazyFr
|
|
|
289
289
|
|
|
290
290
|
|
|
291
291
|
def deduplicate_result(result: pl.DataFrame, column_context: bool) -> pl.DataFrame:
|
|
292
|
-
"""Sort and de-duplicate fullmap matches
|
|
292
|
+
"""Sort and de-duplicate fullmap matches while retaining tied best candidates.
|
|
293
293
|
|
|
294
294
|
When ``column_context`` is set, a per-category ``FREQUENCY`` column is
|
|
295
295
|
attached and used as a high-priority tiebreaker (more common categories
|
|
296
|
-
first).
|
|
296
|
+
first). Lower-ranked rows are removed, but every distinct CURIE sharing a
|
|
297
|
+
term's complete best ranking tuple is retained. Repeated rows for the same
|
|
298
|
+
``(term, CURIE)`` pair collapse to one row.
|
|
297
299
|
|
|
298
300
|
Args:
|
|
299
301
|
result: Joined matches with a ``CATEGORY_NAME`` column.
|
|
300
302
|
column_context: Whether to compute/use the frequency tiebreaker.
|
|
301
303
|
|
|
302
304
|
Returns:
|
|
303
|
-
DataFrame with one
|
|
305
|
+
DataFrame with one or more best-ranked rows per ``term``, unique by
|
|
306
|
+
``(term, CURIE)``.
|
|
304
307
|
"""
|
|
305
|
-
|
|
308
|
+
ranking_columns: list[str] = ["term", "PR", "NLP_LEVEL"]
|
|
306
309
|
descending: list[bool] = [False, False, False]
|
|
307
310
|
|
|
308
311
|
if column_context:
|
|
309
312
|
frequency: pl.DataFrame = result.group_by("CATEGORY_NAME").agg(pl.len().alias("FREQUENCY"))
|
|
310
313
|
result = result.join(frequency, on="CATEGORY_NAME", how="left")
|
|
311
314
|
|
|
312
|
-
|
|
315
|
+
ranking_columns += ["FREQUENCY"]
|
|
313
316
|
descending += [True]
|
|
314
317
|
|
|
315
|
-
result = result.sort(
|
|
316
|
-
|
|
318
|
+
result = result.sort(ranking_columns, descending=descending)
|
|
319
|
+
best_tiers: pl.DataFrame = result.select(ranking_columns).unique(subset=["term"], keep="first")
|
|
320
|
+
result = result.join(best_tiers, on=ranking_columns, how="inner")
|
|
321
|
+
result = result.unique(subset=["term", "CURIE"], keep="first")
|
|
322
|
+
return result.sort([*ranking_columns, "CURIE"], descending=[*descending, False])
|
|
317
323
|
|
|
318
324
|
|
|
319
325
|
def _category_values(categories: list[Any]) -> list[str]:
|
|
@@ -357,7 +363,9 @@ def filter_and_rank(
|
|
|
357
363
|
exclude_regex: Regex patterns; any CURIE matching one is dropped.
|
|
358
364
|
|
|
359
365
|
Returns:
|
|
360
|
-
Ranked matches DataFrame with one
|
|
366
|
+
Ranked matches DataFrame with one or more best-ranked rows per term,
|
|
367
|
+
retaining distinct tied CURIEs and collapsing duplicate ``(term, CURIE)``
|
|
368
|
+
rows.
|
|
361
369
|
"""
|
|
362
370
|
# Resolve_batch reuses one shared redb fetch per column.
|
|
363
371
|
if raw.height == 0:
|
|
@@ -476,8 +484,10 @@ def _coalesce_expr(col: str, suffix: str, base: str, prefix: str, cast_str: bool
|
|
|
476
484
|
def join_matches(lf: pl.LazyFrame, col: str, matches: pl.DataFrame, tag: str = "_two", drop_unresolved: bool = True) -> pl.LazyFrame:
|
|
477
485
|
"""Join ranked fullmap matches back into ``lf`` for one column.
|
|
478
486
|
|
|
479
|
-
|
|
480
|
-
|
|
487
|
+
A source row may expand to one output row per distinct best-ranked CURIE
|
|
488
|
+
when the fullmap ranking heuristics tie. Coalesces level-one and level-two
|
|
489
|
+
hits per row (level one wins when present) and emits derived
|
|
490
|
+
``<col>_name``, ``<col>_category``,
|
|
481
491
|
``<col>_taxon``, ``<col>_source``, ``<col>_source_version`` and
|
|
482
492
|
``<col>_nlp_level`` columns.
|
|
483
493
|
|
|
@@ -493,8 +503,9 @@ def join_matches(lf: pl.LazyFrame, col: str, matches: pl.DataFrame, tag: str = "
|
|
|
493
503
|
edge and the null-stripper omits the qualifier key.
|
|
494
504
|
|
|
495
505
|
Returns:
|
|
496
|
-
New LazyFrame with resolved columns
|
|
497
|
-
|
|
506
|
+
New LazyFrame with resolved columns. Tied best-ranked CURIEs are
|
|
507
|
+
returned as separate rows; rows whose ``col`` did not match are dropped
|
|
508
|
+
unless ``drop_unresolved`` is False.
|
|
498
509
|
|
|
499
510
|
Notes:
|
|
500
511
|
Split out of ``resolve`` so ``resolve_batch`` can apply per-column
|
|
@@ -517,8 +528,9 @@ def _join_matches_eager(df: pl.DataFrame, col: str, matches: pl.DataFrame, tag:
|
|
|
517
528
|
when False the row is kept and the resolved columns stay null.
|
|
518
529
|
|
|
519
530
|
Returns:
|
|
520
|
-
DataFrame with resolved columns
|
|
521
|
-
|
|
531
|
+
DataFrame with resolved columns. Tied best-ranked CURIEs are returned as
|
|
532
|
+
separate rows; rows whose ``col`` did not match are dropped unless
|
|
533
|
+
``drop_unresolved`` is False.
|
|
522
534
|
"""
|
|
523
535
|
l1: str = col
|
|
524
536
|
l2: str = l1 + tag
|
|
@@ -573,8 +585,10 @@ def resolve_batch(
|
|
|
573
585
|
|
|
574
586
|
Each column still gets its own taxon/prioritize/avoid filtering and its own
|
|
575
587
|
join back into ``lf``. Taxon constraints apply to every taxon-bearing
|
|
576
|
-
match while retaining rows with no taxon metadata (TAXON_ID 0);
|
|
577
|
-
|
|
588
|
+
match while retaining rows with no taxon metadata (TAXON_ID 0); distinct
|
|
589
|
+
best-ranked CURIEs tied by all heuristics are retained as separate output
|
|
590
|
+
rows. Only the redb round trip itself (``rs.lookup_fullmap_terms``) is pooled
|
|
591
|
+
across columns.
|
|
578
592
|
|
|
579
593
|
Args:
|
|
580
594
|
lf: Source LazyFrame.
|
|
@@ -589,7 +603,8 @@ def resolve_batch(
|
|
|
589
603
|
column is processed, used to drive fine-grained progress UX.
|
|
590
604
|
|
|
591
605
|
Returns:
|
|
592
|
-
LazyFrame with resolved columns added.
|
|
606
|
+
LazyFrame with resolved columns added. A source row may produce one row
|
|
607
|
+
per distinct CURIE when the best ranking tier is tied.
|
|
593
608
|
|
|
594
609
|
Raises:
|
|
595
610
|
TablassertError: ``resolve-bad-specs`` when two specs share a column, or a
|
|
@@ -694,7 +709,8 @@ def resolve(
|
|
|
694
709
|
tag: Suffix used to derive the level-two column name.
|
|
695
710
|
|
|
696
711
|
Returns:
|
|
697
|
-
LazyFrame with resolved columns added.
|
|
712
|
+
LazyFrame with resolved columns added. A source row may produce one row
|
|
713
|
+
per distinct CURIE when the best ranking tier is tied.
|
|
698
714
|
"""
|
|
699
715
|
return resolve_batch(
|
|
700
716
|
lf,
|
|
@@ -1592,7 +1592,9 @@ def dedup_stream(p_in: Path, is_edges: bool, domain: str = "TABLASSERT", uuid_fi
|
|
|
1592
1592
|
uuid_fields: Optional edge fields that constitute edge identity
|
|
1593
1593
|
(``Graph.uuid_fields``). ``None`` hashes the whole record.
|
|
1594
1594
|
on_collision: ``error`` aborts on two different edges deriving one id;
|
|
1595
|
-
``merge`` folds them into one edge (``Graph.uuid_on_collision``).
|
|
1595
|
+
``merge`` folds them into one edge (``Graph.uuid_on_collision``). During a merge,
|
|
1596
|
+
distinct non-empty scalar ``original_*`` fields are sorted and rendered as
|
|
1597
|
+
``A``, ``A|B``, or ``A|B|C``.
|
|
1596
1598
|
|
|
1597
1599
|
Notes:
|
|
1598
1600
|
Also adds UUIDs to edges. Edges deduplicate on their derived id, so the
|
|
@@ -1714,11 +1716,7 @@ def fold_unknown_to_supporting_text(lf: pl.LazyFrame) -> pl.LazyFrame:
|
|
|
1714
1716
|
|
|
1715
1717
|
|
|
1716
1718
|
def _collect_subframes(
|
|
1717
|
-
subgraphs: list[Path],
|
|
1718
|
-
on_phase: Callable[[str], None] | None = None,
|
|
1719
|
-
on_subgraph: Callable[[], None] | None = None,
|
|
1720
|
-
infores_id: str | None = None,
|
|
1721
|
-
no_original: bool = False,
|
|
1719
|
+
subgraphs: list[Path], on_phase: Callable[[str], None] | None = None, on_subgraph: Callable[[], None] | None = None, infores_id: str | None = None
|
|
1722
1720
|
) -> tuple[list[pl.LazyFrame], list[pl.LazyFrame]]:
|
|
1723
1721
|
"""Scan and normalize subgraph parquets into node/edge subframes.
|
|
1724
1722
|
|
|
@@ -1733,8 +1731,6 @@ def _collect_subframes(
|
|
|
1733
1731
|
on_subgraph: Optional callback fired once after each subgraph is
|
|
1734
1732
|
processed, used to tick the progress bar.
|
|
1735
1733
|
infores_id: Graph-level infores CURIE recorded as node ``provided_by``.
|
|
1736
|
-
no_original: When True, also drop the verbatim ``original_*``
|
|
1737
|
-
source-cell copies from the final edge frames.
|
|
1738
1734
|
|
|
1739
1735
|
Returns:
|
|
1740
1736
|
Tuple of ``(subnodes, subedges)``: per-section node and edge
|
|
@@ -1759,9 +1755,6 @@ def _collect_subframes(
|
|
|
1759
1755
|
subnodes.append(partial)
|
|
1760
1756
|
# Drop internal pre-resolution snapshot columns from final edges.
|
|
1761
1757
|
lf = lf.drop([c for c in lf.collect_schema().names() if c.endswith("_pre_resolution")])
|
|
1762
|
-
# --no-original: drop the verbatim source-cell copies from final edges too.
|
|
1763
|
-
if no_original:
|
|
1764
|
-
lf = lf.drop([c for c in lf.collect_schema().names() if c.startswith("original_")])
|
|
1765
1758
|
lf = fold_unknown_to_supporting_text(lf)
|
|
1766
1759
|
subedges.append(lf)
|
|
1767
1760
|
if on_subgraph is not None:
|
|
@@ -1853,7 +1846,6 @@ def compile_graph(
|
|
|
1853
1846
|
section_sources: list[dict[str, object]] | None = None,
|
|
1854
1847
|
on_phase: Callable[[str], None] | None = None,
|
|
1855
1848
|
on_subgraph: Callable[[], None] | None = None,
|
|
1856
|
-
no_original: bool = False,
|
|
1857
1849
|
uuid_fields: list[str] | None = None,
|
|
1858
1850
|
uuid_domain: str | None = None,
|
|
1859
1851
|
uuid_on_collision: str = "error",
|
|
@@ -1875,8 +1867,6 @@ def compile_graph(
|
|
|
1875
1867
|
``write-edges`` / ``dedup`` / ``rig``), used to drive progress UX.
|
|
1876
1868
|
on_subgraph: Optional callback fired once per processed subgraph,
|
|
1877
1869
|
used to tick the progress bar.
|
|
1878
|
-
no_original: When ``True``, omit the verbatim ``original_*``
|
|
1879
|
-
source-cell copies from the final edge NDJSON.
|
|
1880
1870
|
uuid_fields: Optional edge fields that constitute edge identity
|
|
1881
1871
|
(``Graph.uuid_fields``). ``None`` hashes the whole edge record, so any
|
|
1882
1872
|
change to any field re-mints the id.
|
|
@@ -1916,7 +1906,7 @@ def compile_graph(
|
|
|
1916
1906
|
|
|
1917
1907
|
subnodes: list[pl.LazyFrame]
|
|
1918
1908
|
subedges: list[pl.LazyFrame]
|
|
1919
|
-
subnodes, subedges = _collect_subframes(subgraphs, on_phase, on_subgraph, rig_cfg.source_info.infores_id
|
|
1909
|
+
subnodes, subedges = _collect_subframes(subgraphs, on_phase, on_subgraph, rig_cfg.source_info.infores_id)
|
|
1920
1910
|
_write_ndjson(subnodes, subedges, n, e, name, version, rig_cfg, section_sources, on_phase, domain, uuid_fields, uuid_on_collision)
|
|
1921
1911
|
|
|
1922
1912
|
|
|
@@ -710,10 +710,9 @@ class Annotation(Encoding):
|
|
|
710
710
|
def clean_annotation(cls, annotation: str) -> str:
|
|
711
711
|
cleaned: str = annotation.strip()
|
|
712
712
|
lowered: str = cleaned.lower()
|
|
713
|
-
# Allow-listed slots may carry uppercase
|
|
714
|
-
#
|
|
715
|
-
#
|
|
716
|
-
# lowercased into an unknown name.
|
|
713
|
+
# Allow-listed slots may carry uppercase: any casing the author declares
|
|
714
|
+
# canonicalizes onto the allow-listed spelling verbatim, so the emitted edge
|
|
715
|
+
# field preserves its case instead of being lowercased into an unknown name.
|
|
717
716
|
return next((field for field in ALLOWED_EDGE_FIELDS if field.lower() == lowered), lowered)
|
|
718
717
|
|
|
719
718
|
@model_validator(mode="after")
|
|
@@ -19,3 +19,4 @@ def hydrate_sources(db: Path) -> list[str]: ...
|
|
|
19
19
|
def lookup_fullmap_terms(db: Path, terms: list[str], return_format: str = "rows") -> list[dict[str, Any]]: ...
|
|
20
20
|
def namespace_uuid(domain: str, values: list[str]) -> str: ...
|
|
21
21
|
def xxh64(data: str) -> str: ...
|
|
22
|
+
def xxh64_file(path: str) -> str: ...
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from tablassert import rs
|
|
7
|
+
from tablassert.errors import SourceFileError
|
|
8
|
+
|
|
9
|
+
BASE: Path = Path("./.tablassert")
|
|
10
|
+
STORE: Path = BASE / "store"
|
|
11
|
+
STORE.mkdir(parents=True, exist_ok=True)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def mkhash(x: Any) -> str:
|
|
15
|
+
# xxh64 (not xxh32): the digest is the content-addressed identity for section
|
|
16
|
+
# parquet stores and the section label in validation errors. 32 bits invites
|
|
17
|
+
# birthday collisions (~50% at ~77k sections) that would silently reuse another
|
|
18
|
+
# section's cached subgraph; the full 16-hex 64-bit digest avoids that.
|
|
19
|
+
return rs.xxh64(str(x))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def file_content_hash(path: Path, *, config: Path | None = None, section_label: str | None = None) -> str:
|
|
23
|
+
"""XXH64 hex digest of a source file's raw bytes, or a loud :class:`SourceFileError`.
|
|
24
|
+
|
|
25
|
+
Pure: no caching, so the digest always reflects the file's bytes at call time and an
|
|
26
|
+
edited source always yields a new section store key. ``config`` / ``section_label``
|
|
27
|
+
are optional build context forwarded to the error message when available.
|
|
28
|
+
"""
|
|
29
|
+
resolved: Path = path.resolve()
|
|
30
|
+
if not resolved.exists():
|
|
31
|
+
raise SourceFileError(resolved, "no such file", config=config, section_label=section_label)
|
|
32
|
+
if not resolved.is_file():
|
|
33
|
+
raise SourceFileError(resolved, "not a regular file", config=config, section_label=section_label)
|
|
34
|
+
try:
|
|
35
|
+
return rs.xxh64_file(str(resolved))
|
|
36
|
+
except OSError as e:
|
|
37
|
+
raise SourceFileError(resolved, str(e), config=config, section_label=section_label) from e
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def section_store_key(section: Any, local: Path | None = None, *, content_digest: str | None = None) -> str:
|
|
41
|
+
"""Content-addressed identity for a section's cached parquet store.
|
|
42
|
+
|
|
43
|
+
With a ``local`` source file or a precomputed ``content_digest`` the key mixes the
|
|
44
|
+
section config hash with the file's content digest, so editing the source file
|
|
45
|
+
invalidates the cache; without one the key is the plain section hash. Always a
|
|
46
|
+
single 16-hex xxh64 digest. ``content_digest`` lets a caller memoize file reads
|
|
47
|
+
without changing the key formula.
|
|
48
|
+
"""
|
|
49
|
+
if content_digest is None:
|
|
50
|
+
if local is None:
|
|
51
|
+
return mkhash(section)
|
|
52
|
+
content_digest = file_content_hash(local)
|
|
53
|
+
return mkhash(f"{mkhash(section)}:{content_digest}")
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from pathlib import Path
|
|
4
|
-
from typing import Any
|
|
5
|
-
|
|
6
|
-
from tablassert import rs
|
|
7
|
-
|
|
8
|
-
BASE: Path = Path("./.tablassert")
|
|
9
|
-
STORE: Path = BASE / "store"
|
|
10
|
-
STORE.mkdir(parents=True, exist_ok=True)
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
def mkhash(x: Any) -> str:
|
|
14
|
-
# xxh64 (not xxh32): the digest is the content-addressed identity for section
|
|
15
|
-
# parquet stores and the section label in validation errors. 32 bits invites
|
|
16
|
-
# birthday collisions (~50% at ~77k sections) that would silently reuse another
|
|
17
|
-
# section's cached subgraph; the full 16-hex 64-bit digest avoids that.
|
|
18
|
-
return rs.xxh64(str(x))
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|