tarel 0.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. tarel-0.0.1/.gitignore +20 -0
  2. tarel-0.0.1/CONTRIBUTING.md +39 -0
  3. tarel-0.0.1/LICENSE +18 -0
  4. tarel-0.0.1/PKG-INFO +597 -0
  5. tarel-0.0.1/README.md +559 -0
  6. tarel-0.0.1/SECURITY.md +30 -0
  7. tarel-0.0.1/THIRD_PARTY_NOTICES.md +26 -0
  8. tarel-0.0.1/docs/change-radar.md +49 -0
  9. tarel-0.0.1/docs/context-contract.md +79 -0
  10. tarel-0.0.1/docs/local-retrieval.md +59 -0
  11. tarel-0.0.1/docs/retail-demo.md +88 -0
  12. tarel-0.0.1/docs/workspaces.md +74 -0
  13. tarel-0.0.1/pyproject.toml +93 -0
  14. tarel-0.0.1/src/tarel/__init__.py +5 -0
  15. tarel-0.0.1/src/tarel/__main__.py +3 -0
  16. tarel-0.0.1/src/tarel/annotations/__init__.py +1 -0
  17. tarel-0.0.1/src/tarel/annotations/apply.py +102 -0
  18. tarel-0.0.1/src/tarel/annotations/contracts.py +179 -0
  19. tarel-0.0.1/src/tarel/annotations/review.py +377 -0
  20. tarel-0.0.1/src/tarel/annotations/runner.py +234 -0
  21. tarel-0.0.1/src/tarel/annotations/states.py +40 -0
  22. tarel-0.0.1/src/tarel/annotations/tasks.py +270 -0
  23. tarel-0.0.1/src/tarel/application.py +1015 -0
  24. tarel-0.0.1/src/tarel/cli.py +1689 -0
  25. tarel-0.0.1/src/tarel/connectors/__init__.py +1 -0
  26. tarel-0.0.1/src/tarel/connectors/authoring.py +179 -0
  27. tarel-0.0.1/src/tarel/connectors/contracts.py +389 -0
  28. tarel-0.0.1/src/tarel/connectors/host.py +90 -0
  29. tarel-0.0.1/src/tarel/connectors/sqlite/__init__.py +1 -0
  30. tarel-0.0.1/src/tarel/connectors/sqlite/connector.py +361 -0
  31. tarel-0.0.1/src/tarel/connectors/sqlite/manifest.toml +12 -0
  32. tarel-0.0.1/src/tarel/connectors/sqlite/references/authoring.md +13 -0
  33. tarel-0.0.1/src/tarel/connectors/sqlite/references/dialect.md +7 -0
  34. tarel-0.0.1/src/tarel/connectors/sqlserver/__init__.py +1 -0
  35. tarel-0.0.1/src/tarel/connectors/sqlserver/connector.py +744 -0
  36. tarel-0.0.1/src/tarel/connectors/sqlserver/manifest.toml +12 -0
  37. tarel-0.0.1/src/tarel/connectors/sqlserver/references/authoring.md +30 -0
  38. tarel-0.0.1/src/tarel/connectors/sqlserver/references/dialect.md +20 -0
  39. tarel-0.0.1/src/tarel/context.py +654 -0
  40. tarel-0.0.1/src/tarel/context_output.py +284 -0
  41. tarel-0.0.1/src/tarel/context_packets.py +352 -0
  42. tarel-0.0.1/src/tarel/demo.py +367 -0
  43. tarel-0.0.1/src/tarel/graph/__init__.py +1 -0
  44. tarel-0.0.1/src/tarel/graph/build.py +155 -0
  45. tarel-0.0.1/src/tarel/graph/change_store.py +88 -0
  46. tarel-0.0.1/src/tarel/graph/changes.py +339 -0
  47. tarel-0.0.1/src/tarel/graph/contracts.py +270 -0
  48. tarel-0.0.1/src/tarel/graph/refresh.py +422 -0
  49. tarel-0.0.1/src/tarel/graph/revision.py +18 -0
  50. tarel-0.0.1/src/tarel/graph/store.py +81 -0
  51. tarel-0.0.1/src/tarel/providers/__init__.py +1 -0
  52. tarel-0.0.1/src/tarel/providers/config.py +160 -0
  53. tarel-0.0.1/src/tarel/providers/contracts.py +55 -0
  54. tarel-0.0.1/src/tarel/providers/host.py +13 -0
  55. tarel-0.0.1/src/tarel/providers/openrouter.py +101 -0
  56. tarel-0.0.1/src/tarel/py.typed +1 -0
  57. tarel-0.0.1/src/tarel/relationships/__init__.py +1 -0
  58. tarel-0.0.1/src/tarel/relationships/core.py +439 -0
  59. tarel-0.0.1/src/tarel/retrieval/__init__.py +5 -0
  60. tarel-0.0.1/src/tarel/retrieval/bm25.py +111 -0
  61. tarel-0.0.1/src/tarel/retrieval/contracts.py +78 -0
  62. tarel-0.0.1/src/tarel/retrieval/documents.py +173 -0
  63. tarel-0.0.1/src/tarel/retrieval/index.py +409 -0
  64. tarel-0.0.1/src/tarel/retrieval/local.py +252 -0
  65. tarel-0.0.1/src/tarel/search.py +321 -0
  66. tarel-0.0.1/src/tarel/workspaces/__init__.py +21 -0
  67. tarel-0.0.1/src/tarel/workspaces/contracts.py +346 -0
  68. tarel-0.0.1/src/tarel/workspaces/core.py +318 -0
  69. tarel-0.0.1/src/tarel/workspaces/impact.py +68 -0
  70. tarel-0.0.1/src/tarel/workspaces/store.py +98 -0
  71. tarel-0.0.1/tests/fixtures/adventureworks_search.toml +175 -0
  72. tarel-0.0.1/tests/test_annotation_flow.py +318 -0
  73. tarel-0.0.1/tests/test_annotation_review.py +283 -0
  74. tarel-0.0.1/tests/test_change_radar.py +314 -0
  75. tarel-0.0.1/tests/test_cli.py +24 -0
  76. tarel-0.0.1/tests/test_connectors.py +55 -0
  77. tarel-0.0.1/tests/test_context.py +296 -0
  78. tarel-0.0.1/tests/test_context_packets.py +117 -0
  79. tarel-0.0.1/tests/test_relationships.py +214 -0
  80. tarel-0.0.1/tests/test_retrieval.py +251 -0
  81. tarel-0.0.1/tests/test_search.py +124 -0
  82. tarel-0.0.1/tests/test_sqlite_connector.py +107 -0
  83. tarel-0.0.1/tests/test_workspaces.py +180 -0
  84. tarel-0.0.1/tools/check_distribution.py +132 -0
  85. tarel-0.0.1/uv.lock +388 -0
tarel-0.0.1/.gitignore ADDED
@@ -0,0 +1,20 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ *.egg-info/
4
+ .coverage
5
+ .pytest_cache/
6
+ .ruff_cache/
7
+ .mypy_cache/
8
+ .venv/
9
+ .tarel/
10
+ *.gguf
11
+ build/
12
+ dist/
13
+ htmlcov/
14
+ *.log
15
+ .env
16
+ .env.*
17
+ !.env.example
18
+
19
+ # Local integration-test credentials and database targets.
20
+ tests/local_databases.toml
@@ -0,0 +1,39 @@
1
+ # Contributing to TAREL
2
+
3
+ TAREL welcomes focused bug fixes, documentation improvements, and small features that strengthen
4
+ its analytics context compiler.
5
+
6
+ ## Before changing code
7
+
8
+ - Open an issue before changing a core contract, graph identity rule, persisted format, connector
9
+ boundary, or context packet.
10
+ - Keep generated connectors inactive until a human has reviewed their permissions and queries.
11
+ - Never commit credentials, connection URLs, local database targets, samples, `.tarel/` state, or
12
+ model files.
13
+ - Keep new runtime dependencies optional unless the core cannot reasonably provide the required
14
+ behavior with the Python standard library.
15
+
16
+ ## Development setup
17
+
18
+ ```bash
19
+ python3 -m venv .venv
20
+ . .venv/bin/activate
21
+ python -m pip install -e '.[dev]'
22
+ ```
23
+
24
+ Before opening a pull request, run:
25
+
26
+ ```bash
27
+ ruff check src tests tools
28
+ python -m unittest discover -s tests -q
29
+ python -m compileall -q src tests tools
30
+ python -m build
31
+ python tools/check_distribution.py dist
32
+ ```
33
+
34
+ Pull requests should explain the concrete use case, the evidence behind semantic or connector
35
+ behavior, and any intentionally changed CLI or serialized output. Add focused tests for behavior
36
+ that could regress. Live-system tests must use ignored local configuration and must never expose
37
+ their targets in logs or fixtures.
38
+
39
+ By submitting a contribution, you agree that it is licensed under the repository's MIT License.
tarel-0.0.1/LICENSE ADDED
@@ -0,0 +1,18 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 MPS
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
6
+ associated documentation files (the "Software"), to deal in the Software without restriction,
7
+ including without limitation the rights to use, copy, modify, merge, publish, distribute,
8
+ sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
9
+ furnished to do so, subject to the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be included in all copies or
12
+ substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
15
+ NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
16
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
17
+ DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
18
+ OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
tarel-0.0.1/PKG-INFO ADDED
@@ -0,0 +1,597 @@
1
+ Metadata-Version: 2.4
2
+ Name: tarel
3
+ Version: 0.0.1
4
+ Summary: A local-first analytics context compiler for coding agents.
5
+ Project-URL: Homepage, https://github.com/Mattiascherzgit/tarel
6
+ Project-URL: Repository, https://github.com/Mattiascherzgit/tarel
7
+ Project-URL: Issues, https://github.com/Mattiascherzgit/tarel/issues
8
+ Project-URL: Documentation, https://github.com/Mattiascherzgit/tarel/tree/master/docs
9
+ Author: MPS
10
+ License-Expression: MIT
11
+ License-File: LICENSE
12
+ License-File: THIRD_PARTY_NOTICES.md
13
+ Keywords: agents,analytics,context,lineage,metadata,semantic-layer
14
+ Classifier: Development Status :: 2 - Pre-Alpha
15
+ Classifier: Environment :: Console
16
+ Classifier: Intended Audience :: Developers
17
+ Classifier: Intended Audience :: Information Technology
18
+ Classifier: Operating System :: OS Independent
19
+ Classifier: Programming Language :: Python
20
+ Classifier: Programming Language :: Python :: 3
21
+ Classifier: Programming Language :: Python :: 3 :: Only
22
+ Classifier: Programming Language :: Python :: 3.11
23
+ Classifier: Programming Language :: Python :: 3.12
24
+ Classifier: Topic :: Database
25
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
26
+ Classifier: Topic :: Utilities
27
+ Classifier: Typing :: Typed
28
+ Requires-Python: >=3.11
29
+ Provides-Extra: dev
30
+ Requires-Dist: build<2,>=1.2; extra == 'dev'
31
+ Requires-Dist: ruff>=0.11; extra == 'dev'
32
+ Provides-Extra: local-rag
33
+ Requires-Dist: llama-cpp-python<0.4,>=0.3.16; extra == 'local-rag'
34
+ Provides-Extra: sqlite
35
+ Provides-Extra: sqlserver
36
+ Requires-Dist: pymssql<3,>=2.3; extra == 'sqlserver'
37
+ Description-Content-Type: text/markdown
38
+
39
+ # TAREL
40
+
41
+ [![CI](https://github.com/Mattiascherzgit/tarel/actions/workflows/ci.yml/badge.svg)](https://github.com/Mattiascherzgit/tarel/actions/workflows/ci.yml)
42
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-3776AB.svg)](https://www.python.org/downloads/)
43
+ [![License: MIT](https://img.shields.io/badge/license-MIT-2ea44f.svg)](https://github.com/Mattiascherzgit/tarel/blob/master/LICENSE)
44
+
45
+ **Give coding agents the right analytics context -- and let them extend the tool when the source is
46
+ new.**
47
+
48
+ TAREL is a local-first context compiler for data warehouses, BI systems, and ERP schemas. It turns
49
+ technical metadata, bounded evidence, semantic annotations, and relationships into compact context
50
+ that an agent can actually use.
51
+
52
+ ```text
53
+ known source -> technical graph -> reviewed meaning -> retrieval -> agent context
54
+ new source -> agent-built connector candidate -> human gate -> same pipeline
55
+ ```
56
+
57
+ TAREL stands for **Topology, Annotation, Retrieval, Evidence & Lineage**. It is a dependency-light
58
+ Python CLI with no mandatory runtime dependency. Use it directly from Codex, Claude Code, Pi, or
59
+ another coding agent; later, embed the same application core into your own software.
60
+
61
+ TAREL follows a self-modifying-software idea with a deliberate safety boundary. When it encounters
62
+ an unsupported source, the coding agent operating TAREL can create an isolated connector candidate,
63
+ research the official metadata APIs and dialect, implement the adapter, and test it locally. A
64
+ human reviews the code and observed results before activation. The kernel stays small while the
65
+ installation can grow toward the systems it actually encounters.
66
+
67
+ TAREL is deliberately not another catalog UI, chat application, SQL execution engine, or mandatory
68
+ graph server. It is the small layer between a complex analytical estate and an agent's limited
69
+ context window.
70
+
71
+ > TAREL is pre-alpha software. The CLI and serialized contracts may still change during `0.x`.
72
+ > A stable public SDK, operational lineage, and semantic-standard interoperability are not part of
73
+ > the current release.
74
+
75
+ ## The problem it solves
76
+
77
+ An agent can inspect a small, well-named schema directly. Real analytical estates are different:
78
+ hundreds of tables, abbreviated names, missing foreign keys, undocumented measures, several source
79
+ systems, and knowledge that exists only in people's heads.
80
+
81
+ Dumping an entire schema into the prompt wastes context and still leaves the agent guessing. TAREL
82
+ instead builds a reusable map of the estate and compiles only the relevant part for each question:
83
+
84
+ 1. A connector observes schemas, tables, views, fields, keys, descriptions, and bounded samples.
85
+ 2. TAREL builds a deterministic technical graph with stable identities.
86
+ 3. A coding agent or optional LLM provider proposes semantic annotations.
87
+ 4. A human validates, edits, rejects, or defers those proposals.
88
+ 5. Search selects useful graph objects and expands through trusted relationships.
89
+ 6. The context compiler emits only the relevant objects, fields, joins, warnings, and provenance.
90
+
91
+ The result is small enough for an agent, explicit enough for a human to review, and deterministic
92
+ enough for prompt caching. The source database remains authoritative: TAREL stores metadata and
93
+ claims, not a copy of the warehouse.
94
+
95
+ ## Design principles
96
+
97
+ - **Self-extending, human-gated.** A coding agent can build missing source adapters locally from
98
+ explicit contracts; generated code remains isolated and inactive until reviewed.
99
+ - **Local first.** Graphs, indexes, reviews, and context packets work without a hosted service.
100
+ - **Evidence before confidence.** Technical observations, generated proposals, and human-validated
101
+ knowledge remain distinguishable.
102
+ - **Human-reviewed semantics.** Generated descriptions and inferred joins begin as proposals, never
103
+ as silent truth.
104
+ - **Small, stable context.** Deterministic ordering, explicit budgets, hashes, and visible omissions
105
+ make output inspectable and cache-friendly.
106
+ - **Optional complexity.** The core uses only the Python standard library. Database drivers, local
107
+ embeddings, and remote LLM calls are opt-in.
108
+ - **Agent-native, provider-neutral.** The agent already operating the CLI can research and extend
109
+ TAREL. An optional provider handles repeatable parallel annotation jobs.
110
+
111
+ ## Start here
112
+
113
+ If you are evaluating TAREL, follow the [five-minute local demo](#five-minute-local-demo). It needs
114
+ no credentials and exercises discovery, graph construction, annotation exchange, relationship
115
+ detection, retrieval, context compilation, and schema-drift reporting.
116
+
117
+ If you already have a source:
118
+
119
+ - read [Teach TAREL a new source](#teach-tarel-a-new-source) when no connector exists;
120
+ - read [Connector runtime](#connector-runtime) for the current built-in adapters;
121
+ - read [Semantic annotation and review](#semantic-annotation-and-review) for meaning and approval;
122
+ - read [Retrieval modes](#retrieval-modes) and
123
+ [Deterministic context packets](#deterministic-context-packets) for agent integration.
124
+
125
+ ## Implemented features
126
+
127
+ | Area | Current capability |
128
+ |---|---|
129
+ | Self-extension | Agent-readable connector tasks, isolated candidates, versioned manifests, dialect references, and a human activation gate |
130
+ | Connector runtime | Stable read-only contracts for probing, metadata discovery, sampling, and bounded relationship evidence |
131
+ | Discovery | Catalog, namespace, table, view, field, type, nullability, primary key, foreign key, and technical description observations |
132
+ | Sampling | Explicit, deterministic samples of 1–10 rows with field, value-size, and total-size limits |
133
+ | Graph | Stable technical node and edge identities; atomic local JSON persistence; deterministic revisions |
134
+ | Annotation | Provider-free coding-agent tasks and optional OpenRouter-backed parallel batches |
135
+ | Human review | Draft, validated, rejected, deferred, and `review_required` states with preserved originals and review reasons |
136
+ | Relationships | Declared foreign keys, human-defined joins, and bounded aggregate discovery of missing relationship candidates |
137
+ | Retrieval | Deterministic lexical search, dependency-free BM25, optional local vector search, and hybrid reciprocal-rank fusion |
138
+ | Context | Bounded object, field, join, and hop selection with visible paths, reasons, warnings, and omissions |
139
+ | Cache-friendly output | Stable and dynamic packet sections, graph revision, canonical hashes, packet diffing, and refresh impact checks |
140
+ | Workspaces | `system → area → schema` hierarchy plus explicit overlapping zones across schemas |
141
+ | Change Radar | Field, key, object, and relationship drift; possible renames; stale claims; affected areas, zones, and context packets |
142
+ | Demo | Deterministic local Retail DWH with a deliberate missing relationship and reproducible V1→V2 schema drift |
143
+
144
+ The runtime core has no mandatory third-party dependency. SQLite uses Python's standard library.
145
+ SQL Server and local embeddings are optional extras.
146
+
147
+ ## Installation
148
+
149
+ ```bash
150
+ python -m pip install tarel
151
+ tarel --version
152
+ ```
153
+
154
+ This installs the dependency-free core. TAREL supports Python 3.11 and 3.12.
155
+
156
+ Optional capabilities:
157
+
158
+ ```bash
159
+ # SQL Server through pymssql
160
+ python -m pip install 'tarel[sqlserver]'
161
+
162
+ # Local CPU embeddings through llama.cpp
163
+ python -m pip install 'tarel[local-rag]'
164
+ ```
165
+
166
+ ## Five-minute local demo
167
+
168
+ The bundled Retail DWH is the safest way to try TAREL. It contains synthetic analytical data,
169
+ requires no credentials, and is created only when requested.
170
+
171
+ ### 1. Create and probe the source
172
+
173
+ ```bash
174
+ tarel demo create retail-dwh
175
+ tarel connector check sqlite
176
+ tarel connector probe sqlite --config .tarel/demos/retail-dwh.toml
177
+ ```
178
+
179
+ The generated SQLite database and configuration live below the ignored `.tarel/` directory.
180
+
181
+ Inspect the catalog or a bounded sample:
182
+
183
+ ```bash
184
+ tarel connector discover sqlite \
185
+ --config .tarel/demos/retail-dwh.toml \
186
+ --schema main
187
+
188
+ tarel connector sample sqlite \
189
+ --config .tarel/demos/retail-dwh.toml \
190
+ --schema main \
191
+ --object F_SLS_01 \
192
+ --limit 3
193
+ ```
194
+
195
+ ### 2. Build the technical graph
196
+
197
+ ```bash
198
+ tarel graph build retail-demo \
199
+ --connector sqlite \
200
+ --config .tarel/demos/retail-dwh.toml \
201
+ --schema main
202
+
203
+ tarel graph show retail-demo
204
+ ```
205
+
206
+ The demo graph contains date, product, customer, geography, reseller, currency, and channel
207
+ dimensions; two sales facts; a return fact; a mapping table; and a union view. Its fact names are
208
+ deliberately abbreviated so that semantic annotation provides measurable value.
209
+
210
+ ### 3. Annotate with the current coding agent
211
+
212
+ TAREL can work without an API provider. Ask for one complete, structured task:
213
+
214
+ ```bash
215
+ tarel annotation next retail-demo \
216
+ --samples 5 \
217
+ --config .tarel/demos/retail-dwh.toml > annotation-task.json
218
+ ```
219
+
220
+ Give `annotation-task.json` to the coding agent already running TAREL. Save its structured response
221
+ as `proposal.json`, then apply it as a draft:
222
+
223
+ ```bash
224
+ tarel annotation apply retail-demo --input proposal.json
225
+ tarel annotation show retail-demo main.D_CHNL
226
+ tarel annotation validate retail-demo main.D_CHNL \
227
+ --include-fields \
228
+ --reason "Reviewed against the demo schema and bounded samples."
229
+ ```
230
+
231
+ Every proposal must cover the selected object and all supplied fields. Samples are input-only:
232
+ they are not persisted in the graph and must not be repeated in generated descriptions.
233
+
234
+ ### 4. Discover the deliberately missing relationship
235
+
236
+ `F_SLS_02.RSLR_KEY` plausibly joins `D_RSLR.RSLR_KEY`, but the demo intentionally declares no
237
+ foreign key. TAREL can test a small set of type-compatible candidates with bounded SQL aggregates:
238
+
239
+ ```bash
240
+ tarel relationship discover retail-demo \
241
+ --object main.F_SLS_02 \
242
+ --field RSLR_KEY \
243
+ --config .tarel/demos/retail-dwh.toml
244
+
245
+ tarel relationship list retail-demo
246
+ ```
247
+
248
+ Discovery stores counts and ratios, not the probed values. The result remains a draft and is not
249
+ used for graph expansion until a human validates its edge ID:
250
+
251
+ ```bash
252
+ tarel relationship validate retail-demo RELATIONSHIP_ID \
253
+ --reason "The domain overlap and target uniqueness were reviewed."
254
+ ```
255
+
256
+ ### 5. Search and compile agent context
257
+
258
+ Dependency-free BM25 is useful immediately after annotation:
259
+
260
+ ```bash
261
+ tarel search retail-demo \
262
+ "internet and reseller sales by year" \
263
+ --mode bm25 \
264
+ --limit 10
265
+
266
+ tarel context retail-demo \
267
+ "internet and reseller sales by year" \
268
+ --mode bm25 \
269
+ --max-objects 10
270
+ ```
271
+
272
+ The context result contains graph metadata and proposed joins needed to answer the question. It
273
+ does not execute the analytical aggregation itself.
274
+
275
+ Conceptually, the agent receives a bounded packet like this:
276
+
277
+ ```text
278
+ STABLE CONTEXT
279
+ graph: retail-demo
280
+ objects:
281
+ main.F_SLS_01 "Internet sales fact" [draft]
282
+ main.F_SLS_02 "Reseller sales fact" [draft]
283
+ main.D_DATE "Calendar dimension" [validated]
284
+ joins:
285
+ F_SLS_01.DOC_DT_KEY -> D_DATE.DATE_KEY [foreign_key]
286
+ F_SLS_02.RSLR_KEY -> D_RSLR.RSLR_KEY [candidate, draft]
287
+
288
+ DYNAMIC REQUEST
289
+ query: internet and reseller sales by year
290
+ selection: ...
291
+ omissions: ...
292
+ ```
293
+
294
+ This is illustrative rather than copied output; the real text and JSON renderers include stable
295
+ identities, review states, paths, retrieval reasons, budgets, warnings, hashes, and visible
296
+ omissions. Unreviewed proposals may be included, but their state is never hidden.
297
+
298
+ ### 6. Reproduce schema drift
299
+
300
+ Save the first packet, replace the local demo source with V2, refresh the graph, and check impact:
301
+
302
+ ```bash
303
+ tarel context retail-demo \
304
+ "internet and reseller sales by year" \
305
+ --mode bm25 \
306
+ --format json > retail-context-v1.json
307
+
308
+ tarel demo create retail-dwh --version 2 --force
309
+ tarel graph refresh retail-demo \
310
+ --config .tarel/demos/retail-dwh.toml
311
+
312
+ tarel context impact retail-context-v1.json --graph retail-demo
313
+ ```
314
+
315
+ V2 changes selected types and keys, adds and removes fields, renames a reseller field, and removes
316
+ one declared relationship. Change Radar preserves affected claims, moves validated annotations to
317
+ `review_required`, retains removed knowledge as stale claims, and identifies affected workspace
318
+ areas, zones, and context packets.
319
+
320
+ See the complete [Retail DWH walkthrough](https://github.com/Mattiascherzgit/tarel/blob/master/docs/retail-demo.md).
321
+
322
+ ## Teach TAREL a new source
323
+
324
+ The long-term value of TAREL is not a large built-in connector inventory. It is a small connector
325
+ contract that a capable coding agent can implement when it meets a new database or warehouse. The
326
+ same authoring pattern is intended for lakes, document systems, and orchestration platforms as
327
+ their contracts are added.
328
+
329
+ Start by creating an isolated candidate:
330
+
331
+ ```bash
332
+ tarel connector scaffold postgres \
333
+ --output .tarel/connectors/postgres-candidate
334
+ ```
335
+
336
+ Then give the coding agent already operating TAREL a concrete instruction:
337
+
338
+ > Implement the connector described in
339
+ > `.tarel/connectors/postgres-candidate/CONNECTOR_TASK.md`. Use official vendor documentation,
340
+ > keep the driver optional, implement the read-only probe first, record only the required dialect
341
+ > and metadata notes under `references/`, and stop after local tests for human review.
342
+
343
+ The generated workspace gives the agent:
344
+
345
+ - an explicit versioned connector contract and capability boundary;
346
+ - an inactive Python adapter with the required entry point;
347
+ - a read-only manifest for dependencies, permissions, dialect, and references;
348
+ - focused authoring and SQL-dialect reference files;
349
+ - rules for secret handling, bounded probes, stable ordering, and visible failures;
350
+ - a completion gate that requires testing against a private source and human review.
351
+
352
+ The agent may consult current official documentation when a proprietary metadata API or SQL dialect
353
+ is unknown. It can then edit the candidate, install only the source-specific driver, and iterate on
354
+ `probe` and `discover_catalog` using real error output. This is the self-modifying loop: the tool
355
+ provides the contract and evidence boundary; the coding agent supplies the source-specific code.
356
+
357
+ Today, `scaffold` creates the complete authoring workspace but does not invoke an LLM by itself,
358
+ register the candidate, or execute generated code. Integration and activation are explicit human
359
+ decisions. That gate is intentional: self-extension should remove repetitive integration work,
360
+ not turn unreviewed generated code into trusted infrastructure.
361
+
362
+ ## Connector runtime
363
+
364
+ Once approved and integrated, every connector uses the same small CLI surface:
365
+
366
+ ```bash
367
+ tarel connector check NAME
368
+ tarel connector probe NAME --config private.toml
369
+ tarel connector discover NAME --config private.toml --schema SCHEMA
370
+ tarel connector sample NAME --config private.toml --schema SCHEMA --object OBJECT
371
+ ```
372
+
373
+ Connectors are read-only adapters. They produce normalized observations and never mutate a TAREL
374
+ graph directly. The current distribution includes SQLite and SQL Server as working references for
375
+ the contract, not as the intended limit of the system.
376
+
377
+ Private connector files use TOML sections matching the connector name. They must remain outside
378
+ Git. Environment variables such as `TAREL_SQLSERVER_URL` can override local URLs.
379
+
380
+ ## Semantic annotation and review
381
+
382
+ Two execution modes use the same annotation contract:
383
+
384
+ - **Coding-agent mode:** `annotation plan`, `next`, and `apply` exchange JSON with the agent already
385
+ operating the CLI.
386
+ - **Provider mode:** `graph annotate` calls an optional provider once per object and can run several
387
+ independent calls in parallel.
388
+
389
+ Configure OpenRouter without writing a key into the repository:
390
+
391
+ ```bash
392
+ export OPENROUTER_API_KEY='...'
393
+ tarel provider configure openrouter --from-env --model MODEL_NAME
394
+ tarel provider test openrouter
395
+ ```
396
+
397
+ Run a bounded batch:
398
+
399
+ ```bash
400
+ tarel graph annotate retail-demo \
401
+ --provider openrouter \
402
+ --workers 4 \
403
+ --retry 1 \
404
+ --samples 5 \
405
+ --config .tarel/demos/retail-dwh.toml
406
+ ```
407
+
408
+ Using `--samples` with a remote provider is an explicit data-boundary decision and is announced on
409
+ stderr. Without it, only technical metadata and graph relationships are sent.
410
+
411
+ Review commands never silently turn generated meaning into trusted meaning:
412
+
413
+ ```bash
414
+ tarel annotation list retail-demo --state draft
415
+ tarel annotation show retail-demo main.F_SLS_01.NET_AMT
416
+ tarel annotation edit retail-demo main.F_SLS_01.NET_AMT \
417
+ --input annotation-patch.json \
418
+ --reason "Definition supplied by the data owner"
419
+ tarel annotation validate retail-demo main.F_SLS_01.NET_AMT \
420
+ --reason "Definition and currency behavior confirmed"
421
+ tarel annotation defer retail-demo main.F_SLS_01 \
422
+ --reason "Waiting for the data owner"
423
+ tarel annotation reject retail-demo main.F_SLS_01 \
424
+ --reason "Proposed meaning is incorrect"
425
+ ```
426
+
427
+ Human edits preserve the original proposal. Rejected claims are hidden from retrieval while the
428
+ technical schema stays available.
429
+
430
+ ## Retrieval modes
431
+
432
+ `search` and `context` support four retrieval modes:
433
+
434
+ - `lexical`: deterministic name and annotation matching;
435
+ - `bm25`: dependency-free ranked retrieval over safe graph documents;
436
+ - `vector`: local embeddings from a current persisted index;
437
+ - `hybrid`: BM25 and vector results combined with reciprocal-rank fusion.
438
+
439
+ For local vector and hybrid retrieval:
440
+
441
+ ```bash
442
+ python -m pip install 'tarel[local-rag]'
443
+ tarel model download
444
+ tarel model status
445
+ tarel index build retail-demo
446
+ tarel search retail-demo "annual online revenue" --mode hybrid
447
+ ```
448
+
449
+ The embedding model is downloaded only by the explicit command, checksum-verified, and stored in
450
+ the user cache. Models and vector indexes are not included in the repository or package. TAREL
451
+ embeds an allowlist of graph metadata; connection information, sample values, and arbitrary
452
+ provenance are excluded. See [Local retrieval](https://github.com/Mattiascherzgit/tarel/blob/master/docs/local-retrieval.md).
453
+
454
+ ## Deterministic context packets
455
+
456
+ `tarel.context.v0.2` separates a reusable stable prefix from question-specific dynamic content.
457
+ The packet includes:
458
+
459
+ - graph name and deterministic revision;
460
+ - selected annotation states and scope;
461
+ - ordered objects, fields, joins, and validated relationship candidates;
462
+ - selection paths, retrieval reasons, warnings, and visible omissions;
463
+ - stable, dynamic, and complete packet hashes;
464
+ - an explicit character budget without timestamps, runtimes, or local paths.
465
+
466
+ Compare two packets or test whether a source refresh invalidated an older packet:
467
+
468
+ ```bash
469
+ tarel context diff packet-a.json packet-b.json
470
+ tarel context impact packet-a.json --graph retail-demo
471
+ ```
472
+
473
+ TAREL intentionally emits neutral packet structure rather than provider-specific cache headers,
474
+ TTLs, or session controls. A consuming agent or application can map the stable and dynamic sections
475
+ to its own caching mechanism. See the
476
+ [context packet contract](https://github.com/Mattiascherzgit/tarel/blob/master/docs/context-contract.md).
477
+
478
+ ## Workspaces, areas, and zones
479
+
480
+ Graphs remain independent source projections. A workspace organizes them without copying or
481
+ rewriting their nodes:
482
+
483
+ ```text
484
+ workspace
485
+ └── system
486
+ └── area
487
+ └── schema
488
+ ```
489
+
490
+ Zones are explicit overlapping sets of tables and views. They may cross schemas and areas within
491
+ one system.
492
+
493
+ ```bash
494
+ tarel workspace create enterprise
495
+ tarel workspace system define enterprise commercial --graph retail-demo
496
+ tarel workspace area define enterprise commercial sales \
497
+ --schema retail-demo:main
498
+ tarel workspace zone define enterprise commercial revenue \
499
+ --object retail-demo:main.F_SLS_01 \
500
+ --object retail-demo:main.F_SLS_02 \
501
+ --object retail-demo:main.D_DATE
502
+ tarel workspace zone show enterprise commercial revenue
503
+ ```
504
+
505
+ Change Radar reports which areas and zones are affected by a graph refresh without rewriting their
506
+ definitions automatically. See
507
+ [Workspaces, systems, areas, and zones](https://github.com/Mattiascherzgit/tarel/blob/master/docs/workspaces.md).
508
+
509
+ ## Local persistence and data boundaries
510
+
511
+ TAREL is file-first. Runtime state is stored below the working directory:
512
+
513
+ ```text
514
+ .tarel/
515
+ ├── demos/
516
+ ├── graphs/
517
+ │ └── GRAPH/
518
+ │ ├── graph.json
519
+ │ └── changes/
520
+ ├── indexes/
521
+ └── workspaces/
522
+ ```
523
+
524
+ `.tarel/` is ignored by Git. Graph and workspace stores have explicit whole-document boundaries so
525
+ that a later shared database implementation can use the same application use cases.
526
+
527
+ Security defaults:
528
+
529
+ - connector manifests currently accept read permission only;
530
+ - normal CLI errors do not print connection URLs or passwords;
531
+ - provider keys are stored outside the repository with user-only file permissions;
532
+ - samples require an explicit command or positive sample limit;
533
+ - sample blocks are never persisted in graph annotations;
534
+ - inferred relationships are not traversed until human validation;
535
+ - model downloads and remote provider calls never happen during import.
536
+
537
+ ## Command overview
538
+
539
+ ```text
540
+ tarel demo create deterministic local demo sources
541
+ tarel connector check, probe, discover, sample, and scaffold connectors
542
+ tarel graph build, refresh, list, show, and batch-annotate graphs
543
+ tarel annotation plan, exchange, apply, inspect, edit, and review proposals
544
+ tarel relationship add, probe, discover, list, validate, and reject possible joins
545
+ tarel search retrieve relevant graph objects and fields
546
+ tarel context build, diff, and impact-check deterministic context packets
547
+ tarel model explicitly download and verify the optional embedding model
548
+ tarel index build and inspect local vector indexes
549
+ tarel workspace organize graphs into systems, areas, schemas, and zones
550
+ tarel provider configure and test optional annotation providers
551
+ ```
552
+
553
+ Every substantive command supports deterministic JSON output where it is useful. Run
554
+ `tarel COMMAND --help` for the exact current interface.
555
+
556
+ ## What TAREL does not do yet
557
+
558
+ - no stable public SDK;
559
+ - the distribution does not yet ship PostgreSQL, cloud warehouse, lake, document, or orchestration
560
+ connectors; agents can already author database and warehouse candidates;
561
+ - no operational lineage or ETL run history yet;
562
+ - no tested Apache Ossie import/export compatibility yet;
563
+ - no multi-user shared graph store yet;
564
+ - no automatic execution of analytical SQL;
565
+ - no autonomous activation of generated connector code -- activation remains a human trust gate;
566
+ - no web UI, catalog server, or mandatory cloud service.
567
+
568
+ These are scope boundaries, not hidden fallbacks. Unsupported capabilities fail visibly.
569
+
570
+ ## Development
571
+
572
+ ```bash
573
+ python -m pip install -e '.[dev]'
574
+ ruff check src tests tools
575
+ python -m unittest discover -s tests -q
576
+ python -m compileall -q src tests tools
577
+ python -m build
578
+ python tools/check_distribution.py dist
579
+ ```
580
+
581
+ Live database targets and credentials belong in ignored local configuration. The deterministic
582
+ SQLite demo and the unit suite require no external service.
583
+
584
+ ## Documentation
585
+
586
+ - [Retail DWH demo](https://github.com/Mattiascherzgit/tarel/blob/master/docs/retail-demo.md)
587
+ - [Change Radar and stale claims](https://github.com/Mattiascherzgit/tarel/blob/master/docs/change-radar.md)
588
+ - [Local retrieval](https://github.com/Mattiascherzgit/tarel/blob/master/docs/local-retrieval.md)
589
+ - [Context packet contract](https://github.com/Mattiascherzgit/tarel/blob/master/docs/context-contract.md)
590
+ - [Workspaces, systems, areas, and zones](https://github.com/Mattiascherzgit/tarel/blob/master/docs/workspaces.md)
591
+
592
+ ## License
593
+
594
+ TAREL is available under the
595
+ [MIT License](https://github.com/Mattiascherzgit/tarel/blob/master/LICENSE). Optional components and
596
+ the separately downloaded embedding model are listed in the
597
+ [third-party notices](https://github.com/Mattiascherzgit/tarel/blob/master/THIRD_PARTY_NOTICES.md).