openodke 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openodke-0.1.0/.github/ISSUE_TEMPLATE/bug.yml +29 -0
- openodke-0.1.0/.github/ISSUE_TEMPLATE/feature.yml +19 -0
- openodke-0.1.0/.github/pull_request_template.md +15 -0
- openodke-0.1.0/.github/workflows/ci.yml +129 -0
- openodke-0.1.0/.github/workflows/docs.yml +78 -0
- openodke-0.1.0/.github/workflows/release.yml +123 -0
- openodke-0.1.0/.gitignore +17 -0
- openodke-0.1.0/.python-version +1 -0
- openodke-0.1.0/CHANGELOG.md +201 -0
- openodke-0.1.0/CONTRIBUTING.md +65 -0
- openodke-0.1.0/DECISIONS.md +326 -0
- openodke-0.1.0/LICENSE +202 -0
- openodke-0.1.0/NOTICE +26 -0
- openodke-0.1.0/PKG-INFO +368 -0
- openodke-0.1.0/README.md +313 -0
- openodke-0.1.0/ROADMAP.md +203 -0
- openodke-0.1.0/docs/concepts.md +405 -0
- openodke-0.1.0/docs/decisions.md +1 -0
- openodke-0.1.0/docs/evaluation.md +382 -0
- openodke-0.1.0/docs/grounding.md +214 -0
- openodke-0.1.0/docs/index.md +151 -0
- openodke-0.1.0/docs/inference.md +339 -0
- openodke-0.1.0/docs/installation.md +114 -0
- openodke-0.1.0/docs/loaders-and-extraction.md +583 -0
- openodke-0.1.0/docs/neo4j.md +284 -0
- openodke-0.1.0/docs/ontology.md +473 -0
- openodke-0.1.0/docs/resolution-and-corroboration.md +417 -0
- openodke-0.1.0/docs/run.md +376 -0
- openodke-0.1.0/docs/sinks.md +376 -0
- openodke-0.1.0/examples/README.md +32 -0
- openodke-0.1.0/examples/e2e/README.md +256 -0
- openodke-0.1.0/examples/e2e/corpus/factsheet.md +6 -0
- openodke-0.1.0/examples/e2e/corpus/notes/corvid-analytics.md +3 -0
- openodke-0.1.0/examples/e2e/corpus/notes/halden-robotics.md +5 -0
- openodke-0.1.0/examples/e2e/corpus/register.csv +4 -0
- openodke-0.1.0/examples/e2e/corpus/staff.csv +3 -0
- openodke-0.1.0/examples/e2e/docker-compose.yml +21 -0
- openodke-0.1.0/examples/e2e/e2e_stages.py +56 -0
- openodke-0.1.0/examples/e2e/gold.jsonl +36 -0
- openodke-0.1.0/examples/e2e/odke.neo4j.yaml +49 -0
- openodke-0.1.0/examples/e2e/odke.yaml +41 -0
- openodke-0.1.0/examples/e2e/ontology.json +60 -0
- openodke-0.1.0/examples/e2e/queries.cypher +35 -0
- openodke-0.1.0/examples/e2e/recorded/extract.json +26 -0
- openodke-0.1.0/examples/e2e/recorded/ground.json +176 -0
- openodke-0.1.0/examples/people.ontology.json +74 -0
- openodke-0.1.0/examples/run.yaml +85 -0
- openodke-0.1.0/mkdocs.yml +79 -0
- openodke-0.1.0/pyproject.toml +118 -0
- openodke-0.1.0/scripts/assert_no_credentials.py +29 -0
- openodke-0.1.0/scripts/verify.sh +72 -0
- openodke-0.1.0/src/openodke/__init__.py +118 -0
- openodke-0.1.0/src/openodke/_text.py +37 -0
- openodke-0.1.0/src/openodke/chunking.py +164 -0
- openodke-0.1.0/src/openodke/cli/__init__.py +0 -0
- openodke-0.1.0/src/openodke/cli/main.py +399 -0
- openodke-0.1.0/src/openodke/corroborate/__init__.py +54 -0
- openodke-0.1.0/src/openodke/corroborate/merge.py +400 -0
- openodke-0.1.0/src/openodke/corroborate/normalize.py +346 -0
- openodke-0.1.0/src/openodke/corroborate/provenance.py +63 -0
- openodke-0.1.0/src/openodke/corroborate/resolve.py +410 -0
- openodke-0.1.0/src/openodke/corroborate/score.py +129 -0
- openodke-0.1.0/src/openodke/eval/__init__.py +102 -0
- openodke-0.1.0/src/openodke/eval/ablation.py +270 -0
- openodke-0.1.0/src/openodke/eval/calibration.py +142 -0
- openodke-0.1.0/src/openodke/eval/cost.py +258 -0
- openodke-0.1.0/src/openodke/eval/extraction.py +258 -0
- openodke-0.1.0/src/openodke/eval/formats.py +329 -0
- openodke-0.1.0/src/openodke/eval/grounding.py +159 -0
- openodke-0.1.0/src/openodke/eval/report.py +185 -0
- openodke-0.1.0/src/openodke/eval/resolution.py +202 -0
- openodke-0.1.0/src/openodke/eval/routing.py +99 -0
- openodke-0.1.0/src/openodke/eval/runner.py +145 -0
- openodke-0.1.0/src/openodke/eval/sinks.py +94 -0
- openodke-0.1.0/src/openodke/eval/validation.py +80 -0
- openodke-0.1.0/src/openodke/extract/__init__.py +24 -0
- openodke-0.1.0/src/openodke/extract/_common.py +202 -0
- openodke-0.1.0/src/openodke/extract/hybrid.py +145 -0
- openodke-0.1.0/src/openodke/extract/llm.py +412 -0
- openodke-0.1.0/src/openodke/extract/pattern.py +366 -0
- openodke-0.1.0/src/openodke/ground/__init__.py +32 -0
- openodke-0.1.0/src/openodke/ground/llm.py +304 -0
- openodke-0.1.0/src/openodke/ground/retry.py +151 -0
- openodke-0.1.0/src/openodke/ground/span.py +161 -0
- openodke-0.1.0/src/openodke/infer/__init__.py +82 -0
- openodke-0.1.0/src/openodke/infer/build.py +417 -0
- openodke-0.1.0/src/openodke/infer/candidates.py +107 -0
- openodke-0.1.0/src/openodke/infer/llm.py +483 -0
- openodke-0.1.0/src/openodke/infer/merge.py +438 -0
- openodke-0.1.0/src/openodke/infer/names.py +125 -0
- openodke-0.1.0/src/openodke/infer/propose.py +730 -0
- openodke-0.1.0/src/openodke/infer/review.py +220 -0
- openodke-0.1.0/src/openodke/infer/sample.py +257 -0
- openodke-0.1.0/src/openodke/llm/__init__.py +60 -0
- openodke-0.1.0/src/openodke/llm/base.py +111 -0
- openodke-0.1.0/src/openodke/llm/litellm_client.py +86 -0
- openodke-0.1.0/src/openodke/llm/openai_compat.py +143 -0
- openodke-0.1.0/src/openodke/llm/registry.py +66 -0
- openodke-0.1.0/src/openodke/llm/roles.py +59 -0
- openodke-0.1.0/src/openodke/llm/testing.py +351 -0
- openodke-0.1.0/src/openodke/loaders/__init__.py +61 -0
- openodke-0.1.0/src/openodke/loaders/base.py +75 -0
- openodke-0.1.0/src/openodke/loaders/directory.py +109 -0
- openodke-0.1.0/src/openodke/loaders/docx.py +182 -0
- openodke-0.1.0/src/openodke/loaders/html.py +495 -0
- openodke-0.1.0/src/openodke/loaders/pdf.py +137 -0
- openodke-0.1.0/src/openodke/loaders/records.py +196 -0
- openodke-0.1.0/src/openodke/loaders/sourcemap.py +247 -0
- openodke-0.1.0/src/openodke/loaders/structured.py +209 -0
- openodke-0.1.0/src/openodke/loaders/text.py +130 -0
- openodke-0.1.0/src/openodke/ontology/__init__.py +582 -0
- openodke-0.1.0/src/openodke/ontology/diff.py +224 -0
- openodke-0.1.0/src/openodke/ontology/from_models.py +204 -0
- openodke-0.1.0/src/openodke/ontology/from_neo4j.py +403 -0
- openodke-0.1.0/src/openodke/ontology/from_owl.py +599 -0
- openodke-0.1.0/src/openodke/ontology/load.py +216 -0
- openodke-0.1.0/src/openodke/ontology/validate.py +231 -0
- openodke-0.1.0/src/openodke/pipeline.py +253 -0
- openodke-0.1.0/src/openodke/py.typed +0 -0
- openodke-0.1.0/src/openodke/run/__init__.py +46 -0
- openodke-0.1.0/src/openodke/run/build.py +945 -0
- openodke-0.1.0/src/openodke/run/config.py +293 -0
- openodke-0.1.0/src/openodke/run/execute.py +330 -0
- openodke-0.1.0/src/openodke/sinks/__init__.py +12 -0
- openodke-0.1.0/src/openodke/sinks/bulk.py +526 -0
- openodke-0.1.0/src/openodke/sinks/jsonl.py +48 -0
- openodke-0.1.0/src/openodke/sinks/neo4j.py +727 -0
- openodke-0.1.0/src/openodke/sinks/networkx.py +177 -0
- openodke-0.1.0/src/openodke/sinks/rdf.py +385 -0
- openodke-0.1.0/src/openodke/stages.py +445 -0
- openodke-0.1.0/src/openodke/types.py +357 -0
- openodke-0.1.0/src/openodke/validators.py +65 -0
- openodke-0.1.0/tests/conftest.py +80 -0
- openodke-0.1.0/tests/fixtures/eval/extract.labels.jsonl +5 -0
- openodke-0.1.0/tests/fixtures/eval/extract.predictions.jsonl +5 -0
- openodke-0.1.0/tests/fixtures/eval/ground.labels.jsonl +6 -0
- openodke-0.1.0/tests/fixtures/eval/ground.predictions.jsonl +6 -0
- openodke-0.1.0/tests/fixtures/eval/resolve.labels.jsonl +6 -0
- openodke-0.1.0/tests/fixtures/eval/resolve.predictions.jsonl +6 -0
- openodke-0.1.0/tests/fixtures/eval/route.labels.jsonl +6 -0
- openodke-0.1.0/tests/fixtures/eval/route.predictions.jsonl +6 -0
- openodke-0.1.0/tests/fixtures/eval/score.labels.jsonl +8 -0
- openodke-0.1.0/tests/fixtures/eval/validate.labels.jsonl +6 -0
- openodke-0.1.0/tests/fixtures/eval/validate.predictions.jsonl +6 -0
- openodke-0.1.0/tests/fixtures/llm/grounding.json +26 -0
- openodke-0.1.0/tests/fixtures/llm/infer_people.json +20 -0
- openodke-0.1.0/tests/fixtures/llm/llm_ada_lovelace.json +19 -0
- openodke-0.1.0/tests/fixtures/llm/llm_hybrid_corpus.json +33 -0
- openodke-0.1.0/tests/fixtures/llm/llm_repair.json +32 -0
- openodke-0.1.0/tests/fixtures/llm/llm_uptime_percentiles.json +19 -0
- openodke-0.1.0/tests/fixtures/llm/replay_basics.json +23 -0
- openodke-0.1.0/tests/fixtures/neo4j/schema.json +69 -0
- openodke-0.1.0/tests/fixtures/ontologies/deep.json +27 -0
- openodke-0.1.0/tests/fixtures/ontologies/library.rdfs.ttl +29 -0
- openodke-0.1.0/tests/fixtures/ontologies/minimal.json +9 -0
- openodke-0.1.0/tests/fixtures/ontologies/pathological.json +23 -0
- openodke-0.1.0/tests/fixtures/ontologies/people.owl.ttl +81 -0
- openodke-0.1.0/tests/fixtures/ontologies/scoped.yaml +44 -0
- openodke-0.1.0/tests/fixtures/ontologies/taxonomy.skos.ttl +32 -0
- openodke-0.1.0/tests/fixtures/ontologies/unsupported.owl.ttl +52 -0
- openodke-0.1.0/tests/fixtures/ontologies/wide.json +2407 -0
- openodke-0.1.0/tests/test_bulk_sinks.py +505 -0
- openodke-0.1.0/tests/test_chunking.py +212 -0
- openodke-0.1.0/tests/test_cli.py +162 -0
- openodke-0.1.0/tests/test_corroborate.py +333 -0
- openodke-0.1.0/tests/test_docs_examples.py +54 -0
- openodke-0.1.0/tests/test_document_loaders.py +401 -0
- openodke-0.1.0/tests/test_e2e.py +242 -0
- openodke-0.1.0/tests/test_eval_ablation.py +176 -0
- openodke-0.1.0/tests/test_eval_calibration.py +114 -0
- openodke-0.1.0/tests/test_eval_cli.py +244 -0
- openodke-0.1.0/tests/test_eval_cost.py +172 -0
- openodke-0.1.0/tests/test_eval_extraction.py +186 -0
- openodke-0.1.0/tests/test_eval_formats.py +173 -0
- openodke-0.1.0/tests/test_eval_grounding.py +134 -0
- openodke-0.1.0/tests/test_eval_resolution.py +144 -0
- openodke-0.1.0/tests/test_eval_routing.py +110 -0
- openodke-0.1.0/tests/test_eval_validation.py +202 -0
- openodke-0.1.0/tests/test_goldens.py +236 -0
- openodke-0.1.0/tests/test_ground.py +453 -0
- openodke-0.1.0/tests/test_ground_batch.py +438 -0
- openodke-0.1.0/tests/test_html_loader.py +332 -0
- openodke-0.1.0/tests/test_hybrid.py +183 -0
- openodke-0.1.0/tests/test_infer_build.py +284 -0
- openodke-0.1.0/tests/test_infer_cli.py +214 -0
- openodke-0.1.0/tests/test_infer_llm.py +145 -0
- openodke-0.1.0/tests/test_infer_merge.py +291 -0
- openodke-0.1.0/tests/test_infer_propose.py +251 -0
- openodke-0.1.0/tests/test_infer_sample.py +111 -0
- openodke-0.1.0/tests/test_llm.py +324 -0
- openodke-0.1.0/tests/test_llm_extractor.py +226 -0
- openodke-0.1.0/tests/test_llm_replay.py +115 -0
- openodke-0.1.0/tests/test_loaders.py +112 -0
- openodke-0.1.0/tests/test_neo4j_constraints.py +333 -0
- openodke-0.1.0/tests/test_neo4j_sink.py +599 -0
- openodke-0.1.0/tests/test_networkx_sink.py +296 -0
- openodke-0.1.0/tests/test_normalize.py +201 -0
- openodke-0.1.0/tests/test_ontology.py +162 -0
- openodke-0.1.0/tests/test_ontology_diff.py +172 -0
- openodke-0.1.0/tests/test_ontology_io.py +171 -0
- openodke-0.1.0/tests/test_ontology_neo4j.py +356 -0
- openodke-0.1.0/tests/test_ontology_owl.py +314 -0
- openodke-0.1.0/tests/test_ontology_pydantic.py +167 -0
- openodke-0.1.0/tests/test_ontology_validate.py +216 -0
- openodke-0.1.0/tests/test_pattern.py +221 -0
- openodke-0.1.0/tests/test_pipeline.py +289 -0
- openodke-0.1.0/tests/test_rdf_sink.py +364 -0
- openodke-0.1.0/tests/test_readme.py +43 -0
- openodke-0.1.0/tests/test_resolve.py +290 -0
- openodke-0.1.0/tests/test_run.py +688 -0
- openodke-0.1.0/tests/test_score.py +240 -0
- openodke-0.1.0/tests/test_stages.py +171 -0
- openodke-0.1.0/tests/test_structured_loaders.py +234 -0
- openodke-0.1.0/tests/test_types.py +187 -0
- openodke-0.1.0/tests/test_validators.py +96 -0
- openodke-0.1.0/uv.lock +2327 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
name: Bug report
|
|
2
|
+
description: Something behaves differently from what it says it does.
|
|
3
|
+
labels: ["bug"]
|
|
4
|
+
body:
|
|
5
|
+
- type: textarea
|
|
6
|
+
id: what
|
|
7
|
+
attributes:
|
|
8
|
+
label: What happened
|
|
9
|
+
description: What you expected, and what you got instead.
|
|
10
|
+
validations: {required: true}
|
|
11
|
+
- type: textarea
|
|
12
|
+
id: repro
|
|
13
|
+
attributes:
|
|
14
|
+
label: Reproduction
|
|
15
|
+
description: The smallest snippet that shows it. An ontology fragment helps more than a description of one.
|
|
16
|
+
render: python
|
|
17
|
+
validations: {required: true}
|
|
18
|
+
- type: input
|
|
19
|
+
id: version
|
|
20
|
+
attributes:
|
|
21
|
+
label: openodke version
|
|
22
|
+
placeholder: "0.0.1"
|
|
23
|
+
validations: {required: true}
|
|
24
|
+
- type: input
|
|
25
|
+
id: model
|
|
26
|
+
attributes:
|
|
27
|
+
label: Model configuration
|
|
28
|
+
description: The ModelRoles or model string, if the failure involves a model call.
|
|
29
|
+
placeholder: "ollama/llama3.1"
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
name: Feature request
|
|
2
|
+
description: Something the library should be able to do and cannot.
|
|
3
|
+
labels: ["enhancement"]
|
|
4
|
+
body:
|
|
5
|
+
- type: textarea
|
|
6
|
+
id: problem
|
|
7
|
+
attributes:
|
|
8
|
+
label: The problem
|
|
9
|
+
description: What are you trying to do? Describe the situation, not the solution you have in mind.
|
|
10
|
+
validations: {required: true}
|
|
11
|
+
- type: textarea
|
|
12
|
+
id: shape
|
|
13
|
+
attributes:
|
|
14
|
+
label: What the API might look like
|
|
15
|
+
render: python
|
|
16
|
+
- type: textarea
|
|
17
|
+
id: alternatives
|
|
18
|
+
attributes:
|
|
19
|
+
label: What you are doing instead today
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
## What this changes
|
|
2
|
+
|
|
3
|
+
<!-- One paragraph. What is different after this merges? -->
|
|
4
|
+
|
|
5
|
+
## Why
|
|
6
|
+
|
|
7
|
+
<!-- The reason, not the restatement. If it changes a decision in DECISIONS.md,
|
|
8
|
+
update that file in this PR. -->
|
|
9
|
+
|
|
10
|
+
## Checklist
|
|
11
|
+
|
|
12
|
+
- [ ] `./scripts/verify.sh` passes locally
|
|
13
|
+
- [ ] New behaviour has a test that would fail without the change
|
|
14
|
+
- [ ] Public API changes are reflected in the README
|
|
15
|
+
- [ ] A design call that future-me would question is recorded in `DECISIONS.md`
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
concurrency:
|
|
10
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
11
|
+
cancel-in-progress: true
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
verify:
|
|
15
|
+
name: verify (python ${{ matrix.python }})
|
|
16
|
+
runs-on: ubuntu-latest
|
|
17
|
+
strategy:
|
|
18
|
+
# Both legs always report. A break on one version should not hide the
|
|
19
|
+
# result on another, which is the whole reason for testing three.
|
|
20
|
+
fail-fast: false
|
|
21
|
+
matrix:
|
|
22
|
+
# The range pyproject declares: requires-python = ">=3.11,<3.14".
|
|
23
|
+
# Advertising support for a version nothing runs is a claim, not a fact.
|
|
24
|
+
python: ["3.11", "3.12", "3.13"]
|
|
25
|
+
steps:
|
|
26
|
+
- uses: actions/checkout@v4
|
|
27
|
+
|
|
28
|
+
- name: Install uv
|
|
29
|
+
uses: astral-sh/setup-uv@v5
|
|
30
|
+
with:
|
|
31
|
+
enable-cache: true
|
|
32
|
+
|
|
33
|
+
# Every check lives in scripts/verify.sh so that what runs here and what a
|
|
34
|
+
# contributor runs locally cannot drift apart. See that file for the list.
|
|
35
|
+
- name: Verify
|
|
36
|
+
run: ./scripts/verify.sh
|
|
37
|
+
env:
|
|
38
|
+
UV_PYTHON: ${{ matrix.python }}
|
|
39
|
+
|
|
40
|
+
# The tests that talk to a real Neo4j skip everywhere else: verify.sh refuses
|
|
41
|
+
# to run with NEO4J_URI or NEO4J_PASSWORD set, because the suite must never
|
|
42
|
+
# write to somebody's real graph. So they get their own job, a database that
|
|
43
|
+
# lives exactly as long as the job does, and no verify.sh. What a recording
|
|
44
|
+
# driver cannot check — that the MERGEs really are idempotent, that the DDL
|
|
45
|
+
# is accepted, that a check query finds a real violation — is checked here.
|
|
46
|
+
neo4j-live:
|
|
47
|
+
name: neo4j live tests (python 3.12, neo4j 5.26)
|
|
48
|
+
runs-on: ubuntu-latest
|
|
49
|
+
services:
|
|
50
|
+
neo4j:
|
|
51
|
+
# Relationship uniqueness constraints need 5.7 or later; Community,
|
|
52
|
+
# because the constrainer emits nothing Enterprise-only.
|
|
53
|
+
image: neo4j:5.26-community
|
|
54
|
+
env:
|
|
55
|
+
# A throwaway password for a container that dies with the job,
|
|
56
|
+
# derived from the run id so no fixed credential sits in the repo.
|
|
57
|
+
NEO4J_AUTH: neo4j/ci-${{ github.run_id }}
|
|
58
|
+
ports:
|
|
59
|
+
- 7687:7687
|
|
60
|
+
# Healthy means bolt answers a query with these credentials.
|
|
61
|
+
options: >-
|
|
62
|
+
--health-cmd "cypher-shell -u neo4j -p ci-${{ github.run_id }} 'RETURN 1'"
|
|
63
|
+
--health-interval 5s
|
|
64
|
+
--health-timeout 20s
|
|
65
|
+
--health-retries 30
|
|
66
|
+
--health-start-period 20s
|
|
67
|
+
env:
|
|
68
|
+
UV_PYTHON: "3.12"
|
|
69
|
+
steps:
|
|
70
|
+
- uses: actions/checkout@v4
|
|
71
|
+
|
|
72
|
+
- name: Install uv
|
|
73
|
+
uses: astral-sh/setup-uv@v5
|
|
74
|
+
with:
|
|
75
|
+
enable-cache: true
|
|
76
|
+
|
|
77
|
+
- name: Install
|
|
78
|
+
run: uv sync --locked
|
|
79
|
+
|
|
80
|
+
# Only these two steps see the connection settings. Set any wider and the
|
|
81
|
+
# credential check in verify.sh would be right to refuse.
|
|
82
|
+
- name: Wait for bolt on 7687
|
|
83
|
+
env:
|
|
84
|
+
NEO4J_URI: bolt://localhost:7687
|
|
85
|
+
NEO4J_USER: neo4j
|
|
86
|
+
NEO4J_PASSWORD: ci-${{ github.run_id }}
|
|
87
|
+
run: |
|
|
88
|
+
uv run python - <<'EOF'
|
|
89
|
+
import os, time
|
|
90
|
+
from neo4j import GraphDatabase
|
|
91
|
+
|
|
92
|
+
auth = (os.environ["NEO4J_USER"], os.environ["NEO4J_PASSWORD"])
|
|
93
|
+
deadline = time.monotonic() + 120
|
|
94
|
+
while True:
|
|
95
|
+
try:
|
|
96
|
+
with GraphDatabase.driver(os.environ["NEO4J_URI"], auth=auth) as driver:
|
|
97
|
+
driver.verify_connectivity()
|
|
98
|
+
print("bolt is up:", driver.get_server_info().agent)
|
|
99
|
+
break
|
|
100
|
+
except Exception as exc:
|
|
101
|
+
if time.monotonic() > deadline:
|
|
102
|
+
raise
|
|
103
|
+
print(f"waiting for bolt: {exc}")
|
|
104
|
+
time.sleep(2)
|
|
105
|
+
EOF
|
|
106
|
+
|
|
107
|
+
# `-k live_neo4j` selects exactly the tests that need a server. A skipped
|
|
108
|
+
# test is a green tick that proved nothing, so the step then reads the
|
|
109
|
+
# JUnit report and fails unless at least one ran and none skipped.
|
|
110
|
+
- name: Live Neo4j tests
|
|
111
|
+
env:
|
|
112
|
+
NEO4J_URI: bolt://localhost:7687
|
|
113
|
+
NEO4J_USER: neo4j
|
|
114
|
+
NEO4J_PASSWORD: ci-${{ github.run_id }}
|
|
115
|
+
run: |
|
|
116
|
+
uv run pytest tests/test_neo4j_sink.py tests/test_neo4j_constraints.py tests/test_bulk_sinks.py tests/test_ontology_neo4j.py tests/test_e2e.py \
|
|
117
|
+
-k live_neo4j -rA --junitxml=live-neo4j.xml
|
|
118
|
+
python3 - <<'EOF'
|
|
119
|
+
import xml.etree.ElementTree as ET
|
|
120
|
+
|
|
121
|
+
root = ET.parse("live-neo4j.xml").getroot()
|
|
122
|
+
suite = root if root.tag == "testsuite" else root.find("testsuite")
|
|
123
|
+
tests, skipped = int(suite.get("tests")), int(suite.get("skipped"))
|
|
124
|
+
failed = int(suite.get("failures")) + int(suite.get("errors"))
|
|
125
|
+
ran = tests - skipped
|
|
126
|
+
print(f"live neo4j tests: {tests} selected, {ran} ran, {skipped} skipped, {failed} failed")
|
|
127
|
+
if ran < 1 or skipped or failed:
|
|
128
|
+
raise SystemExit("the live Neo4j tests must run, not skip")
|
|
129
|
+
EOF
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# The documentation site, built from docs/ and mkdocs.yml.
|
|
2
|
+
#
|
|
3
|
+
# A pull request that touches what the site is built from builds it with
|
|
4
|
+
# --strict, so a broken link, a missing page or a heading that moved fails the
|
|
5
|
+
# PR. Only main deploys, to https://deepskandpal.github.io/odke/, through the
|
|
6
|
+
# github-pages environment. The examples on the pages are run by
|
|
7
|
+
# tests/test_docs_examples.py in the ci workflow, not here.
|
|
8
|
+
|
|
9
|
+
name: docs
|
|
10
|
+
|
|
11
|
+
on:
|
|
12
|
+
push:
|
|
13
|
+
branches: [main]
|
|
14
|
+
paths:
|
|
15
|
+
- "docs/**"
|
|
16
|
+
- "mkdocs.yml"
|
|
17
|
+
- "src/**"
|
|
18
|
+
- "DECISIONS.md" # rendered into the site
|
|
19
|
+
- "pyproject.toml"
|
|
20
|
+
- "uv.lock"
|
|
21
|
+
- ".github/workflows/docs.yml"
|
|
22
|
+
pull_request:
|
|
23
|
+
paths:
|
|
24
|
+
- "docs/**"
|
|
25
|
+
- "mkdocs.yml"
|
|
26
|
+
- "src/**"
|
|
27
|
+
- "DECISIONS.md"
|
|
28
|
+
- "pyproject.toml"
|
|
29
|
+
- "uv.lock"
|
|
30
|
+
- ".github/workflows/docs.yml"
|
|
31
|
+
workflow_dispatch:
|
|
32
|
+
|
|
33
|
+
permissions:
|
|
34
|
+
contents: read
|
|
35
|
+
|
|
36
|
+
concurrency:
|
|
37
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
38
|
+
# A newer push supersedes a PR build; a deploy already running is let finish.
|
|
39
|
+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
40
|
+
|
|
41
|
+
jobs:
|
|
42
|
+
build:
|
|
43
|
+
runs-on: ubuntu-latest
|
|
44
|
+
steps:
|
|
45
|
+
- uses: actions/checkout@v4
|
|
46
|
+
|
|
47
|
+
- name: Install uv
|
|
48
|
+
uses: astral-sh/setup-uv@v5
|
|
49
|
+
with:
|
|
50
|
+
enable-cache: true
|
|
51
|
+
|
|
52
|
+
# Only the docs group: the site renders markdown and does not import the
|
|
53
|
+
# package, so it needs neither the package nor its extras to build.
|
|
54
|
+
- name: Build
|
|
55
|
+
run: uv run --locked --only-group docs mkdocs build --strict
|
|
56
|
+
env:
|
|
57
|
+
UV_PYTHON: "3.12"
|
|
58
|
+
|
|
59
|
+
- name: Upload the site
|
|
60
|
+
if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
|
|
61
|
+
uses: actions/upload-pages-artifact@v3
|
|
62
|
+
with:
|
|
63
|
+
path: site/
|
|
64
|
+
|
|
65
|
+
deploy:
|
|
66
|
+
needs: build
|
|
67
|
+
if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
|
|
68
|
+
runs-on: ubuntu-latest
|
|
69
|
+
permissions:
|
|
70
|
+
pages: write
|
|
71
|
+
id-token: write
|
|
72
|
+
environment:
|
|
73
|
+
name: github-pages
|
|
74
|
+
url: ${{ steps.deployment.outputs.page_url }}
|
|
75
|
+
steps:
|
|
76
|
+
- name: Deploy to GitHub Pages
|
|
77
|
+
id: deployment
|
|
78
|
+
uses: actions/deploy-pages@v4
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# Publishing to PyPI, with no credential anywhere in this repository.
|
|
2
|
+
#
|
|
3
|
+
# Trusted Publishing (OIDC): PyPI is configured to trust this exact repository,
|
|
4
|
+
# workflow filename and environment, and GitHub mints a short-lived token per
|
|
5
|
+
# run. There is no API token to create, share, paste into a secret, rotate, or
|
|
6
|
+
# leak. If this file is renamed, or the environment below changes, PyPI stops
|
|
7
|
+
# trusting it and the upload fails closed.
|
|
8
|
+
#
|
|
9
|
+
# The verify job runs the same checks that guard every push, on every supported
|
|
10
|
+
# interpreter, before anything is uploaded. A release that skipped the checks
|
|
11
|
+
# would defeat the point of having them.
|
|
12
|
+
|
|
13
|
+
name: release
|
|
14
|
+
|
|
15
|
+
on:
|
|
16
|
+
push:
|
|
17
|
+
tags: ["v*"]
|
|
18
|
+
workflow_dispatch:
|
|
19
|
+
inputs:
|
|
20
|
+
target:
|
|
21
|
+
description: "Where to publish"
|
|
22
|
+
required: true
|
|
23
|
+
default: testpypi
|
|
24
|
+
type: choice
|
|
25
|
+
options: [testpypi, pypi]
|
|
26
|
+
|
|
27
|
+
jobs:
|
|
28
|
+
verify:
|
|
29
|
+
name: verify (python ${{ matrix.python }})
|
|
30
|
+
runs-on: ubuntu-latest
|
|
31
|
+
strategy:
|
|
32
|
+
fail-fast: false
|
|
33
|
+
matrix:
|
|
34
|
+
python: ["3.11", "3.12", "3.13"]
|
|
35
|
+
steps:
|
|
36
|
+
- uses: actions/checkout@v4
|
|
37
|
+
- uses: astral-sh/setup-uv@v5
|
|
38
|
+
with:
|
|
39
|
+
enable-cache: true
|
|
40
|
+
- run: ./scripts/verify.sh
|
|
41
|
+
env:
|
|
42
|
+
UV_PYTHON: ${{ matrix.python }}
|
|
43
|
+
|
|
44
|
+
build:
|
|
45
|
+
needs: verify
|
|
46
|
+
runs-on: ubuntu-latest
|
|
47
|
+
steps:
|
|
48
|
+
- uses: actions/checkout@v4
|
|
49
|
+
- uses: astral-sh/setup-uv@v5
|
|
50
|
+
|
|
51
|
+
# The tag is the source of truth for the version. A tag that disagrees
|
|
52
|
+
# with pyproject.toml means one of the two is a typo, and guessing which
|
|
53
|
+
# is how a wrong version reaches PyPI permanently: uploads cannot be
|
|
54
|
+
# replaced, only yanked.
|
|
55
|
+
- name: Tag matches pyproject version
|
|
56
|
+
if: startsWith(github.ref, 'refs/tags/v')
|
|
57
|
+
run: |
|
|
58
|
+
tag="${GITHUB_REF_NAME#v}"
|
|
59
|
+
pkg=$(grep -m1 '^version' pyproject.toml | cut -d'"' -f2)
|
|
60
|
+
if [ "$tag" != "$pkg" ]; then
|
|
61
|
+
echo "tag v$tag does not match pyproject version $pkg" >&2
|
|
62
|
+
exit 1
|
|
63
|
+
fi
|
|
64
|
+
echo "v$tag matches pyproject"
|
|
65
|
+
|
|
66
|
+
- run: uv build
|
|
67
|
+
|
|
68
|
+
# Catches the metadata problems that make a page look broken on PyPI,
|
|
69
|
+
# before the upload rather than after it.
|
|
70
|
+
- name: Check the distribution metadata
|
|
71
|
+
run: uv run --isolated --no-project --with twine twine check --strict dist/*
|
|
72
|
+
|
|
73
|
+
- uses: actions/upload-artifact@v4
|
|
74
|
+
with:
|
|
75
|
+
name: dist
|
|
76
|
+
path: dist/
|
|
77
|
+
|
|
78
|
+
testpypi:
|
|
79
|
+
needs: build
|
|
80
|
+
if: github.event_name == 'workflow_dispatch' && inputs.target == 'testpypi'
|
|
81
|
+
runs-on: ubuntu-latest
|
|
82
|
+
environment: testpypi
|
|
83
|
+
permissions:
|
|
84
|
+
id-token: write # the whole credential story: a short-lived OIDC token
|
|
85
|
+
steps:
|
|
86
|
+
- uses: actions/download-artifact@v4
|
|
87
|
+
with:
|
|
88
|
+
name: dist
|
|
89
|
+
path: dist/
|
|
90
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
91
|
+
with:
|
|
92
|
+
repository-url: https://test.pypi.org/legacy/
|
|
93
|
+
|
|
94
|
+
pypi:
|
|
95
|
+
needs: build
|
|
96
|
+
if: startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'workflow_dispatch' && inputs.target == 'pypi')
|
|
97
|
+
runs-on: ubuntu-latest
|
|
98
|
+
environment: pypi
|
|
99
|
+
permissions:
|
|
100
|
+
id-token: write
|
|
101
|
+
steps:
|
|
102
|
+
- uses: actions/download-artifact@v4
|
|
103
|
+
with:
|
|
104
|
+
name: dist
|
|
105
|
+
path: dist/
|
|
106
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
107
|
+
|
|
108
|
+
github-release:
|
|
109
|
+
needs: pypi
|
|
110
|
+
if: startsWith(github.ref, 'refs/tags/v')
|
|
111
|
+
runs-on: ubuntu-latest
|
|
112
|
+
permissions:
|
|
113
|
+
contents: write
|
|
114
|
+
steps:
|
|
115
|
+
- uses: actions/checkout@v4
|
|
116
|
+
- uses: actions/download-artifact@v4
|
|
117
|
+
with:
|
|
118
|
+
name: dist
|
|
119
|
+
path: dist/
|
|
120
|
+
- name: Publish the GitHub release
|
|
121
|
+
env:
|
|
122
|
+
GH_TOKEN: ${{ github.token }}
|
|
123
|
+
run: gh release create "$GITHUB_REF_NAME" dist/* --generate-notes
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.py[cod]
|
|
3
|
+
.venv/
|
|
4
|
+
dist/
|
|
5
|
+
build/
|
|
6
|
+
*.egg-info/
|
|
7
|
+
.pytest_cache/
|
|
8
|
+
.ruff_cache/
|
|
9
|
+
.mypy_cache/
|
|
10
|
+
.hypothesis/
|
|
11
|
+
.coverage
|
|
12
|
+
htmlcov/
|
|
13
|
+
site/
|
|
14
|
+
.env
|
|
15
|
+
.DS_Store
|
|
16
|
+
# What `odke run` writes when the examples are run in place.
|
|
17
|
+
examples/e2e/out/
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.12
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project uses
|
|
5
|
+
[semantic versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
## [0.1.0] — 2026-09-14
|
|
10
|
+
|
|
11
|
+
The v0.1 milestones now on `main`: the data model (M0), ontology I/O (M1),
|
|
12
|
+
loaders and extraction (M2), grounding and corroboration (M3), the Neo4j sink and
|
|
13
|
+
`odke run` (M4), and evaluation against your own labels (M6). Not yet on PyPI:
|
|
14
|
+
the release (M7) waits on Trusted Publishing.
|
|
15
|
+
|
|
16
|
+
### Added
|
|
17
|
+
|
|
18
|
+
#### M0 — Data model (#60)
|
|
19
|
+
|
|
20
|
+
Every change here is to a frozen type, which is why it landed before anything
|
|
21
|
+
was serialised. Two of them fix live correctness bugs.
|
|
22
|
+
|
|
23
|
+
- `Fact.polarity` (`Polarity`: asserted / denied / partial), and it is part of
|
|
24
|
+
`Fact.signature`. A denial no longer merges with its own contradiction. (#47)
|
|
25
|
+
- Qualifier identity semantics. `Predicate.qualifiers` maps each key to a
|
|
26
|
+
`Qualifier(identity=...)`; `Ontology.identity_keys()` and `Fact.identity_keys`
|
|
27
|
+
carry the identity-bearing keys onto the fact, and `signature` includes them,
|
|
28
|
+
sorted. Reconcilable qualifiers stay out, as before. (#48)
|
|
29
|
+
- `EntityLink` (`LinkKind`: SAME_AS / SIMILAR / DIFFERENT, with `score`,
|
|
30
|
+
`evidence` and a `reason` naming the identifier that disagreed) and
|
|
31
|
+
`KnowledgeGraph.links`, so resolution is never destructive. (#49)
|
|
32
|
+
- `Fact.valid_from` / `valid_to` — the valid clock, alongside `retrieved_at`'s
|
|
33
|
+
transaction clock. Neither is in the signature. (#50)
|
|
34
|
+
- `Resolution` and `Entity.resolution` — how the key was decided: by the caller,
|
|
35
|
+
an external id, or a named linker with a score. (#51)
|
|
36
|
+
- `Chunk`, `RouteVerdict` and the `Router` protocol; the default passes every
|
|
37
|
+
chunk. (#52)
|
|
38
|
+
- `openodke.stages`: thirteen stage protocols — `Loader`, `Chunker`, `Router`,
|
|
39
|
+
`Extractor`, `Grounder`, `Normalizer`, `Resolver`, `Corroborator`, `Scorer`,
|
|
40
|
+
`Validator`, `Sink`, `Constrainer`, `Inferrer` — each with a pass-through
|
|
41
|
+
default, plus `ValidationVerdict`. (#53)
|
|
42
|
+
- `PlatformProfile` (declared on a sink), `Delegated(to=...)` (a pass-through
|
|
43
|
+
that satisfies every stage protocol and stamps provenance), and
|
|
44
|
+
`DoubleStageWarning`, raised once when a stage is configured in openodke and the
|
|
45
|
+
sink's platform does it too. Warned, never refused. (#59)
|
|
46
|
+
|
|
47
|
+
#### M1 — Ontology I/O & validation (#63)
|
|
48
|
+
|
|
49
|
+
- `Ontology.from_dict`, `from_json` and `from_yaml`. A failed load raises
|
|
50
|
+
`OntologyLoadError` with one line per problem: the dotted path to the key,
|
|
51
|
+
what was found there, a did-you-mean for a misspelt key, and a line and column
|
|
52
|
+
for a syntax error. `strict=True` also validates. PyYAML is imported lazily,
|
|
53
|
+
behind the new `[yaml]` extra.
|
|
54
|
+
- `Ontology.from_pydantic(*models)`: models become entity types, fields become
|
|
55
|
+
predicates, a field typed as another model is an edge. Adds
|
|
56
|
+
`Predicate.required`.
|
|
57
|
+
- `Ontology.validate()`, returning `Diagnostic`s and never raising: unknown
|
|
58
|
+
ranges, parents and keys, unreachable predicates, duplicate aliases, name
|
|
59
|
+
mismatches and inheritance cycles, each with the exact path.
|
|
60
|
+
- `Ontology.diff()`, marking every change breaking or compatible, and the
|
|
61
|
+
`odke ontology validate` and `odke ontology diff [--fail-on-breaking]` commands.
|
|
62
|
+
- `Predicate.cardinality_scope` and `Predicate.scope_keys`: what a single-valued
|
|
63
|
+
predicate is single *within*, shared by the corroborator and the Neo4j check.
|
|
64
|
+
|
|
65
|
+
#### M2 — Loaders & extraction (#66)
|
|
66
|
+
|
|
67
|
+
- `SentenceChunker(max_words, overlap)`: whole sentences, paragraph breaks
|
|
68
|
+
preferred, `doc.text[start:end] == chunk.text` always. (#6)
|
|
69
|
+
- `TextLoader`, `MarkdownLoader` (the raw file as text, the heading outline with
|
|
70
|
+
offsets in metadata) and `DirectoryLoader`, reading bytes so CRLF offsets are
|
|
71
|
+
file offsets. (#7)
|
|
72
|
+
- `CsvLoader`, `TsvLoader`, `JsonLoader`, `JsonlLoader`, `RecordsLoader` and
|
|
73
|
+
`record_document`: one structured document per record, rendered so a span can
|
|
74
|
+
point into it. `ParquetLoader` behind the new `[parquet]` extra. (#10)
|
|
75
|
+
- `PatternExtractor` and `RecordMapping`: records, pipe tables and `Key: value`
|
|
76
|
+
blocks to facts, with no model call. (#11)
|
|
77
|
+
- `LLMExtractor`: ontology snippets in, facts with checked evidence spans out;
|
|
78
|
+
every drop in `rejections`, every call's usage in `calls`. (#12)
|
|
79
|
+
- `HybridExtractor` and `PathReport`: routed by modality, merged by signature. (#13)
|
|
80
|
+
- `ReplayClient`, `Cassette` and `RecordingClient`: model paths in CI with no key
|
|
81
|
+
and no network. (#14)
|
|
82
|
+
|
|
83
|
+
#### M3 — Grounding (#61)
|
|
84
|
+
|
|
85
|
+
- `check_span`, `SpanStatus` and `SpanGrounder`: an offset that does not resolve
|
|
86
|
+
to its quote is rejected before any model is asked. (#15)
|
|
87
|
+
- `LLMGrounder`, `render_claim` and `parse_verdict`: one fact, one span, one
|
|
88
|
+
verdict from the `ground` role; `RecordedClient` for thread-safe replay. (#16)
|
|
89
|
+
- `LLMGrounder.ground_many`, `RetryPolicy` and `is_transient`: a document's facts
|
|
90
|
+
grounded concurrently, transient errors retried, a failed call left
|
|
91
|
+
`unchecked` rather than failing the run. (#17)
|
|
92
|
+
|
|
93
|
+
#### M3 — Normalise, resolve, corroborate, score (#65)
|
|
94
|
+
|
|
95
|
+
- `ValueNormalizer`: dates, numbers, quantities and name keys to one form each,
|
|
96
|
+
refusing when unsure and keeping the source spelling. (#18)
|
|
97
|
+
- `NativeResolver`: blocking, strong identifiers, a scored name match, and the
|
|
98
|
+
disagreement rule — a `DIFFERENT` link naming both identifiers. (#19)
|
|
99
|
+
- `SignatureCorroborator`: merge by signature, `support` as independent sources,
|
|
100
|
+
contested values ranked on trust × freshness × volume-discounted agreement;
|
|
101
|
+
losers kept with the reason. (#20)
|
|
102
|
+
- `EvidenceScorer`: confidence from the extractor, the verdict, support and any
|
|
103
|
+
lost contest, with its inputs kept on the fact. (#21)
|
|
104
|
+
|
|
105
|
+
#### M4 — Neo4j (#62)
|
|
106
|
+
|
|
107
|
+
- `Neo4jSink`: batched, idempotent `UNWIND … MERGE` on entity keys and fact
|
|
108
|
+
signatures, provenance on every fact relationship, literal facts as `:Claim`
|
|
109
|
+
nodes, `EntityLink`s as relationships. The driver is imported lazily behind
|
|
110
|
+
`[neo4j]`. (#22)
|
|
111
|
+
- `Neo4jConstrainer`, `Neo4jSink.bootstrap()` and `Neo4jSink.check()`: the
|
|
112
|
+
ontology compiled into uniqueness constraints and indexes, and a check query
|
|
113
|
+
per single-valued predicate for what Neo4j cannot enforce. (#23)
|
|
114
|
+
|
|
115
|
+
#### M6 — Evaluation, bring your own labelled dataset (#64)
|
|
116
|
+
|
|
117
|
+
- `StageReport`, a JSONL format per stage, fixtures that are not a benchmark, and
|
|
118
|
+
`odke eval <stage> --labels … [--predictions … | --run …] [--describe]`. (#36)
|
|
119
|
+
- Evaluators for routing (#54), extraction with the four-way error split (#37),
|
|
120
|
+
grounding and `grounding_ablation` (#55), resolution with B-cubed (#56),
|
|
121
|
+
calibration with Brier, reliability and ECE (#57), and validation with sink
|
|
122
|
+
idempotency (`check_idempotency`, `assert_idempotent`) (#58).
|
|
123
|
+
- `CostMeter`, `CostReport` and `compare_costs`: tokens, USD and latency per
|
|
124
|
+
stage, with unknown cost kept unknown. (#39)
|
|
125
|
+
|
|
126
|
+
#### M4 / M6 — The run command, the example, the ablation
|
|
127
|
+
|
|
128
|
+
- `odke run config.yaml [--dry-run]` and `openodke.run`: the whole pipeline from one
|
|
129
|
+
YAML or JSON file — inputs and loaders, ontology, model roles with recorded
|
|
130
|
+
responses and a cost meter, the implementation of each of the thirteen stages
|
|
131
|
+
by short name or `package.module:Name`, sinks, and `bootstrap`. Every stage's
|
|
132
|
+
counts are copied into `KnowledgeGraph.stats`, and a document's id is its
|
|
133
|
+
source path. `examples/run.yaml` comments every key. (#29)
|
|
134
|
+
- `VerdictValidator`: the default gate, refusing `contradicted` and, on request,
|
|
135
|
+
`not_found`. (#29)
|
|
136
|
+
- `examples/e2e/`: an invented corpus, ontology, recorded responses and configs,
|
|
137
|
+
run into JSON Lines or Neo4j, with the queries that show provenance, a
|
|
138
|
+
`DIFFERENT` link and a cardinality check. (#30)
|
|
139
|
+
- `odke eval ablation --config … --labels …` and `openodke.eval.run_ablation`:
|
|
140
|
+
extraction alone, + grounding, + corroboration over your own labels. (#38)
|
|
141
|
+
- `odke run` short names for the loaders and sinks that landed after its
|
|
142
|
+
builder: loaders `html`, `pdf` and `docx`; sinks `cypher_file`,
|
|
143
|
+
`neo4j_admin_csv`, `rdf` and `networkx` (which also writes node-link JSON
|
|
144
|
+
when given `path`). Options pass through as extra keys. A missing extra is a
|
|
145
|
+
config error naming it, raised while the config is built and before any sink
|
|
146
|
+
is opened, and each input's loader is now built then too.
|
|
147
|
+
|
|
148
|
+
### Changed
|
|
149
|
+
- `odke ontology infer` with no readable documents lists every suffix
|
|
150
|
+
`DirectoryLoader` reads, taken from its own table, so `.html`, `.pdf` and
|
|
151
|
+
`.docx` are no longer missing from the hint. The `docs` extra is pypdf and
|
|
152
|
+
python-docx: beautifulsoup4 and lxml were in it and nothing imported them.
|
|
153
|
+
- The distribution and the import package are now `openodke` (`pip install
|
|
154
|
+
"openodke[neo4j]"`, `import openodke`); the command is still `odke`, and an
|
|
155
|
+
`openodke` command runs the same app. The reserved `odke.*` keys, the RDF
|
|
156
|
+
`odke:` vocabulary and the Neo4j `odke_*` schema names are unchanged. Nothing
|
|
157
|
+
had been published under the old name, so there is no compatibility shim.
|
|
158
|
+
(DECISIONS #22)
|
|
159
|
+
- The stage protocols live in `openodke.stages`; `openodke.pipeline` re-exports them.
|
|
160
|
+
`Extractor` takes one `Chunk` and the ontology; `Grounder` takes one fact and
|
|
161
|
+
its document and sets the verdict rather than dropping; `Corroborator` returns
|
|
162
|
+
facts and the pipeline assembles the graph. (#60)
|
|
163
|
+
- `Pipeline.run` chunks and routes before extracting, resolves before
|
|
164
|
+
corroborating, validates before writing, and reports counts in
|
|
165
|
+
`KnowledgeGraph.stats`. `Pipeline.constraints()` exposes the constrainer's
|
|
166
|
+
output for a sink to apply. (#60)
|
|
167
|
+
- `Pipeline.run` grounds a document's candidates together, through
|
|
168
|
+
`ground_many` when the grounder has it. (#61)
|
|
169
|
+
- No `DoubleStageWarning` for a constrainer whose `platform` matches the sink's
|
|
170
|
+
profile: it is the store's other half, not a second pass. (#62)
|
|
171
|
+
- `Predicate.qualifiers` is a mapping; a bare list of names is still accepted
|
|
172
|
+
and every name in it is reconcilable. (#60)
|
|
173
|
+
- `JsonlSink` writes `links.jsonl` and counts links in the manifest. (#60)
|
|
174
|
+
- README rewritten around what runs today, with install from GitHub until the
|
|
175
|
+
PyPI release. ROADMAP marks M0–M4 and M6 done. (#43)
|
|
176
|
+
|
|
177
|
+
## [0.0.1] — 2026-09-01
|
|
178
|
+
|
|
179
|
+
The scaffold. Everything here is the contract later milestones are written
|
|
180
|
+
against, not a preview of the finished library.
|
|
181
|
+
|
|
182
|
+
### Added
|
|
183
|
+
- Core data model: `Document`, `Span`, `Evidence`, `Entity`, `Fact`,
|
|
184
|
+
`KnowledgeGraph`, with frozen semantics and character-offset provenance.
|
|
185
|
+
- Ontology compiler: `Ontology`, `EntityType`, `Predicate`, inheritance-aware
|
|
186
|
+
`predicates_for()`, and cycle-safe `lineage()`.
|
|
187
|
+
- `OntologySnippet` — the ranked, per-type schema fragment from the ODKE+ paper,
|
|
188
|
+
rendered either as prose or as JSON Schema from one object.
|
|
189
|
+
- Provider-neutral model layer: `LLMClient` protocol, `ModelSpec`, `ModelRoles`,
|
|
190
|
+
a standard-library OpenAI-compatible client (Ollama, vLLM, LM Studio,
|
|
191
|
+
llama.cpp, OpenRouter, Groq, Together, DeepSeek, gateways), a litellm adapter
|
|
192
|
+
for everything else, a `register()` escape hatch, and `ScriptedClient` for
|
|
193
|
+
offline tests.
|
|
194
|
+
- `Pipeline` and the five stage protocols: `Initiator`, `Retriever`, `Extractor`,
|
|
195
|
+
`Grounder`, `Corroborator`, plus `Sink`.
|
|
196
|
+
- `JsonlSink`, and the `odke` CLI with `ontology snippet` and `ontology types`.
|
|
197
|
+
- `scripts/verify.sh` — ten checks, run identically in CI and locally.
|
|
198
|
+
- CI on Python 3.11/3.12/3.13; PyPI release via Trusted Publishing.
|
|
199
|
+
|
|
200
|
+
[Unreleased]: https://github.com/deepskandpal/odke/compare/v0.0.1...HEAD
|
|
201
|
+
[0.0.1]: https://github.com/deepskandpal/odke/releases/tag/v0.0.1
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
## The whole check is one script
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
git clone https://github.com/deepskandpal/odke && cd odke
|
|
7
|
+
curl -LsSf https://astral.sh/uv/install.sh | sh # if you don't have uv
|
|
8
|
+
./scripts/verify.sh
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Ten steps: credentials, interpreter, lock, lint, format, types, tests, the
|
|
12
|
+
base-install guarantee, the build, and a smoke test of the built wheel in a clean
|
|
13
|
+
environment. CI runs this same script on three interpreters — there is no second
|
|
14
|
+
list of steps to drift out of sync with this one.
|
|
15
|
+
|
|
16
|
+
## What a good change looks like
|
|
17
|
+
|
|
18
|
+
- **A test that would fail without it.** Not coverage for its own sake: a test
|
|
19
|
+
that names the behaviour and would catch its loss.
|
|
20
|
+
- **The reason, in the code.** Comments here explain *why*, not *what*. If you
|
|
21
|
+
found the reasoning non-obvious, so will the next person.
|
|
22
|
+
- **A `DECISIONS.md` entry** for anything a future contributor would reasonably
|
|
23
|
+
try to reverse.
|
|
24
|
+
|
|
25
|
+
## Things worth knowing before you start
|
|
26
|
+
|
|
27
|
+
- **The base install talks to nothing.** `pip install openodke` must keep working
|
|
28
|
+
with no provider, no driver and no network. `verify.sh` step 8 enforces it, so
|
|
29
|
+
a new top-level import of `litellm`, `neo4j` or `rdflib` will fail the build.
|
|
30
|
+
Import those inside the module that needs them.
|
|
31
|
+
- **No vendor names outside `openodke/llm/`.** Everything else goes through
|
|
32
|
+
`LLMClient`. A `import anthropic` in the extractor is a bug, not a shortcut.
|
|
33
|
+
- **Tests never touch the network.** Use `ScriptedClient` for model calls and the
|
|
34
|
+
injected opener for HTTP. Step 1 refuses to run if a live key is in the
|
|
35
|
+
environment.
|
|
36
|
+
- **Facts are frozen.** Stages return new objects. A stage that mutated one in
|
|
37
|
+
place would make its own provenance wrong.
|
|
38
|
+
|
|
39
|
+
## Adding a sink
|
|
40
|
+
|
|
41
|
+
`Sink` is a `Protocol` — one method, no base class, no registration:
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
class MySink:
|
|
45
|
+
def write(self, kg: KnowledgeGraph) -> None: ...
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
If it needs a driver, put it behind an extra in `pyproject.toml` and import the
|
|
49
|
+
driver inside the module, not at package level.
|
|
50
|
+
|
|
51
|
+
## Adding a provider
|
|
52
|
+
|
|
53
|
+
Most providers need no code: they are either OpenAI-shaped (already covered) or
|
|
54
|
+
supported by litellm (already covered). If yours is genuinely neither, implement
|
|
55
|
+
`LLMClient` and register it:
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from openodke.llm import register
|
|
59
|
+
|
|
60
|
+
register("myprovider", lambda spec: MyClient(spec))
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
A new adapter in this repository needs a test proving it normalises to the same
|
|
64
|
+
`Completion` as the others — including `cost_usd is None` when the provider does
|
|
65
|
+
not report cost, rather than `0.0`.
|