tarel 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tarel-0.0.1/.gitignore +20 -0
- tarel-0.0.1/CONTRIBUTING.md +39 -0
- tarel-0.0.1/LICENSE +18 -0
- tarel-0.0.1/PKG-INFO +597 -0
- tarel-0.0.1/README.md +559 -0
- tarel-0.0.1/SECURITY.md +30 -0
- tarel-0.0.1/THIRD_PARTY_NOTICES.md +26 -0
- tarel-0.0.1/docs/change-radar.md +49 -0
- tarel-0.0.1/docs/context-contract.md +79 -0
- tarel-0.0.1/docs/local-retrieval.md +59 -0
- tarel-0.0.1/docs/retail-demo.md +88 -0
- tarel-0.0.1/docs/workspaces.md +74 -0
- tarel-0.0.1/pyproject.toml +93 -0
- tarel-0.0.1/src/tarel/__init__.py +5 -0
- tarel-0.0.1/src/tarel/__main__.py +3 -0
- tarel-0.0.1/src/tarel/annotations/__init__.py +1 -0
- tarel-0.0.1/src/tarel/annotations/apply.py +102 -0
- tarel-0.0.1/src/tarel/annotations/contracts.py +179 -0
- tarel-0.0.1/src/tarel/annotations/review.py +377 -0
- tarel-0.0.1/src/tarel/annotations/runner.py +234 -0
- tarel-0.0.1/src/tarel/annotations/states.py +40 -0
- tarel-0.0.1/src/tarel/annotations/tasks.py +270 -0
- tarel-0.0.1/src/tarel/application.py +1015 -0
- tarel-0.0.1/src/tarel/cli.py +1689 -0
- tarel-0.0.1/src/tarel/connectors/__init__.py +1 -0
- tarel-0.0.1/src/tarel/connectors/authoring.py +179 -0
- tarel-0.0.1/src/tarel/connectors/contracts.py +389 -0
- tarel-0.0.1/src/tarel/connectors/host.py +90 -0
- tarel-0.0.1/src/tarel/connectors/sqlite/__init__.py +1 -0
- tarel-0.0.1/src/tarel/connectors/sqlite/connector.py +361 -0
- tarel-0.0.1/src/tarel/connectors/sqlite/manifest.toml +12 -0
- tarel-0.0.1/src/tarel/connectors/sqlite/references/authoring.md +13 -0
- tarel-0.0.1/src/tarel/connectors/sqlite/references/dialect.md +7 -0
- tarel-0.0.1/src/tarel/connectors/sqlserver/__init__.py +1 -0
- tarel-0.0.1/src/tarel/connectors/sqlserver/connector.py +744 -0
- tarel-0.0.1/src/tarel/connectors/sqlserver/manifest.toml +12 -0
- tarel-0.0.1/src/tarel/connectors/sqlserver/references/authoring.md +30 -0
- tarel-0.0.1/src/tarel/connectors/sqlserver/references/dialect.md +20 -0
- tarel-0.0.1/src/tarel/context.py +654 -0
- tarel-0.0.1/src/tarel/context_output.py +284 -0
- tarel-0.0.1/src/tarel/context_packets.py +352 -0
- tarel-0.0.1/src/tarel/demo.py +367 -0
- tarel-0.0.1/src/tarel/graph/__init__.py +1 -0
- tarel-0.0.1/src/tarel/graph/build.py +155 -0
- tarel-0.0.1/src/tarel/graph/change_store.py +88 -0
- tarel-0.0.1/src/tarel/graph/changes.py +339 -0
- tarel-0.0.1/src/tarel/graph/contracts.py +270 -0
- tarel-0.0.1/src/tarel/graph/refresh.py +422 -0
- tarel-0.0.1/src/tarel/graph/revision.py +18 -0
- tarel-0.0.1/src/tarel/graph/store.py +81 -0
- tarel-0.0.1/src/tarel/providers/__init__.py +1 -0
- tarel-0.0.1/src/tarel/providers/config.py +160 -0
- tarel-0.0.1/src/tarel/providers/contracts.py +55 -0
- tarel-0.0.1/src/tarel/providers/host.py +13 -0
- tarel-0.0.1/src/tarel/providers/openrouter.py +101 -0
- tarel-0.0.1/src/tarel/py.typed +1 -0
- tarel-0.0.1/src/tarel/relationships/__init__.py +1 -0
- tarel-0.0.1/src/tarel/relationships/core.py +439 -0
- tarel-0.0.1/src/tarel/retrieval/__init__.py +5 -0
- tarel-0.0.1/src/tarel/retrieval/bm25.py +111 -0
- tarel-0.0.1/src/tarel/retrieval/contracts.py +78 -0
- tarel-0.0.1/src/tarel/retrieval/documents.py +173 -0
- tarel-0.0.1/src/tarel/retrieval/index.py +409 -0
- tarel-0.0.1/src/tarel/retrieval/local.py +252 -0
- tarel-0.0.1/src/tarel/search.py +321 -0
- tarel-0.0.1/src/tarel/workspaces/__init__.py +21 -0
- tarel-0.0.1/src/tarel/workspaces/contracts.py +346 -0
- tarel-0.0.1/src/tarel/workspaces/core.py +318 -0
- tarel-0.0.1/src/tarel/workspaces/impact.py +68 -0
- tarel-0.0.1/src/tarel/workspaces/store.py +98 -0
- tarel-0.0.1/tests/fixtures/adventureworks_search.toml +175 -0
- tarel-0.0.1/tests/test_annotation_flow.py +318 -0
- tarel-0.0.1/tests/test_annotation_review.py +283 -0
- tarel-0.0.1/tests/test_change_radar.py +314 -0
- tarel-0.0.1/tests/test_cli.py +24 -0
- tarel-0.0.1/tests/test_connectors.py +55 -0
- tarel-0.0.1/tests/test_context.py +296 -0
- tarel-0.0.1/tests/test_context_packets.py +117 -0
- tarel-0.0.1/tests/test_relationships.py +214 -0
- tarel-0.0.1/tests/test_retrieval.py +251 -0
- tarel-0.0.1/tests/test_search.py +124 -0
- tarel-0.0.1/tests/test_sqlite_connector.py +107 -0
- tarel-0.0.1/tests/test_workspaces.py +180 -0
- tarel-0.0.1/tools/check_distribution.py +132 -0
- tarel-0.0.1/uv.lock +388 -0
tarel-0.0.1/.gitignore
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.py[cod]
|
|
3
|
+
*.egg-info/
|
|
4
|
+
.coverage
|
|
5
|
+
.pytest_cache/
|
|
6
|
+
.ruff_cache/
|
|
7
|
+
.mypy_cache/
|
|
8
|
+
.venv/
|
|
9
|
+
.tarel/
|
|
10
|
+
*.gguf
|
|
11
|
+
build/
|
|
12
|
+
dist/
|
|
13
|
+
htmlcov/
|
|
14
|
+
*.log
|
|
15
|
+
.env
|
|
16
|
+
.env.*
|
|
17
|
+
!.env.example
|
|
18
|
+
|
|
19
|
+
# Local integration-test credentials and database targets.
|
|
20
|
+
tests/local_databases.toml
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Contributing to TAREL
|
|
2
|
+
|
|
3
|
+
TAREL welcomes focused bug fixes, documentation improvements, and small features that strengthen
|
|
4
|
+
its analytics context compiler.
|
|
5
|
+
|
|
6
|
+
## Before changing code
|
|
7
|
+
|
|
8
|
+
- Open an issue before changing a core contract, graph identity rule, persisted format, connector
|
|
9
|
+
boundary, or context packet.
|
|
10
|
+
- Keep generated connectors inactive until a human has reviewed their permissions and queries.
|
|
11
|
+
- Never commit credentials, connection URLs, local database targets, samples, `.tarel/` state, or
|
|
12
|
+
model files.
|
|
13
|
+
- Keep new runtime dependencies optional unless the core cannot reasonably provide the required
|
|
14
|
+
behavior with the Python standard library.
|
|
15
|
+
|
|
16
|
+
## Development setup
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
python3 -m venv .venv
|
|
20
|
+
. .venv/bin/activate
|
|
21
|
+
python -m pip install -e '.[dev]'
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
Before opening a pull request, run:
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
ruff check src tests tools
|
|
28
|
+
python -m unittest discover -s tests -q
|
|
29
|
+
python -m compileall -q src tests tools
|
|
30
|
+
python -m build
|
|
31
|
+
python tools/check_distribution.py dist
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Pull requests should explain the concrete use case, the evidence behind semantic or connector
|
|
35
|
+
behavior, and any intentionally changed CLI or serialized output. Add focused tests for behavior
|
|
36
|
+
that could regress. Live-system tests must use ignored local configuration and must never expose
|
|
37
|
+
their targets in logs or fixtures.
|
|
38
|
+
|
|
39
|
+
By submitting a contribution, you agree that it is licensed under the repository's MIT License.
|
tarel-0.0.1/LICENSE
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 MPS
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
|
6
|
+
associated documentation files (the "Software"), to deal in the Software without restriction,
|
|
7
|
+
including without limitation the rights to use, copy, modify, merge, publish, distribute,
|
|
8
|
+
sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
|
|
9
|
+
furnished to do so, subject to the following conditions:
|
|
10
|
+
|
|
11
|
+
The above copyright notice and this permission notice shall be included in all copies or
|
|
12
|
+
substantial portions of the Software.
|
|
13
|
+
|
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
|
|
15
|
+
NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
16
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
|
17
|
+
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
|
|
18
|
+
OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
tarel-0.0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,597 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tarel
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: A local-first analytics context compiler for coding agents.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Mattiascherzgit/tarel
|
|
6
|
+
Project-URL: Repository, https://github.com/Mattiascherzgit/tarel
|
|
7
|
+
Project-URL: Issues, https://github.com/Mattiascherzgit/tarel/issues
|
|
8
|
+
Project-URL: Documentation, https://github.com/Mattiascherzgit/tarel/tree/master/docs
|
|
9
|
+
Author: MPS
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
License-File: THIRD_PARTY_NOTICES.md
|
|
13
|
+
Keywords: agents,analytics,context,lineage,metadata,semantic-layer
|
|
14
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
15
|
+
Classifier: Environment :: Console
|
|
16
|
+
Classifier: Intended Audience :: Developers
|
|
17
|
+
Classifier: Intended Audience :: Information Technology
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python
|
|
20
|
+
Classifier: Programming Language :: Python :: 3
|
|
21
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
24
|
+
Classifier: Topic :: Database
|
|
25
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
26
|
+
Classifier: Topic :: Utilities
|
|
27
|
+
Classifier: Typing :: Typed
|
|
28
|
+
Requires-Python: >=3.11
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: build<2,>=1.2; extra == 'dev'
|
|
31
|
+
Requires-Dist: ruff>=0.11; extra == 'dev'
|
|
32
|
+
Provides-Extra: local-rag
|
|
33
|
+
Requires-Dist: llama-cpp-python<0.4,>=0.3.16; extra == 'local-rag'
|
|
34
|
+
Provides-Extra: sqlite
|
|
35
|
+
Provides-Extra: sqlserver
|
|
36
|
+
Requires-Dist: pymssql<3,>=2.3; extra == 'sqlserver'
|
|
37
|
+
Description-Content-Type: text/markdown
|
|
38
|
+
|
|
39
|
+
# TAREL
|
|
40
|
+
|
|
41
|
+
[](https://github.com/Mattiascherzgit/tarel/actions/workflows/ci.yml)
|
|
42
|
+
[](https://www.python.org/downloads/)
|
|
43
|
+
[](https://github.com/Mattiascherzgit/tarel/blob/master/LICENSE)
|
|
44
|
+
|
|
45
|
+
**Give coding agents the right analytics context -- and let them extend the tool when the source is
|
|
46
|
+
new.**
|
|
47
|
+
|
|
48
|
+
TAREL is a local-first context compiler for data warehouses, BI systems, and ERP schemas. It turns
|
|
49
|
+
technical metadata, bounded evidence, semantic annotations, and relationships into compact context
|
|
50
|
+
that an agent can actually use.
|
|
51
|
+
|
|
52
|
+
```text
|
|
53
|
+
known source -> technical graph -> reviewed meaning -> retrieval -> agent context
|
|
54
|
+
new source -> agent-built connector candidate -> human gate -> same pipeline
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
TAREL stands for **Topology, Annotation, Retrieval, Evidence & Lineage**. It is a dependency-light
|
|
58
|
+
Python CLI with no mandatory runtime dependency. Use it directly from Codex, Claude Code, Pi, or
|
|
59
|
+
another coding agent; later, embed the same application core into your own software.
|
|
60
|
+
|
|
61
|
+
TAREL follows a self-modifying-software idea with a deliberate safety boundary. When it encounters
|
|
62
|
+
an unsupported source, the coding agent operating TAREL can create an isolated connector candidate,
|
|
63
|
+
research the official metadata APIs and dialect, implement the adapter, and test it locally. A
|
|
64
|
+
human reviews the code and observed results before activation. The kernel stays small while the
|
|
65
|
+
installation can grow toward the systems it actually encounters.
|
|
66
|
+
|
|
67
|
+
TAREL is deliberately not another catalog UI, chat application, SQL execution engine, or mandatory
|
|
68
|
+
graph server. It is the small layer between a complex analytical estate and an agent's limited
|
|
69
|
+
context window.
|
|
70
|
+
|
|
71
|
+
> TAREL is pre-alpha software. The CLI and serialized contracts may still change during `0.x`.
|
|
72
|
+
> A stable public SDK, operational lineage, and semantic-standard interoperability are not part of
|
|
73
|
+
> the current release.
|
|
74
|
+
|
|
75
|
+
## The problem it solves
|
|
76
|
+
|
|
77
|
+
An agent can inspect a small, well-named schema directly. Real analytical estates are different:
|
|
78
|
+
hundreds of tables, abbreviated names, missing foreign keys, undocumented measures, several source
|
|
79
|
+
systems, and knowledge that exists only in people's heads.
|
|
80
|
+
|
|
81
|
+
Dumping an entire schema into the prompt wastes context and still leaves the agent guessing. TAREL
|
|
82
|
+
instead builds a reusable map of the estate and compiles only the relevant part for each question:
|
|
83
|
+
|
|
84
|
+
1. A connector observes schemas, tables, views, fields, keys, descriptions, and bounded samples.
|
|
85
|
+
2. TAREL builds a deterministic technical graph with stable identities.
|
|
86
|
+
3. A coding agent or optional LLM provider proposes semantic annotations.
|
|
87
|
+
4. A human validates, edits, rejects, or defers those proposals.
|
|
88
|
+
5. Search selects useful graph objects and expands through trusted relationships.
|
|
89
|
+
6. The context compiler emits only the relevant objects, fields, joins, warnings, and provenance.
|
|
90
|
+
|
|
91
|
+
The result is small enough for an agent, explicit enough for a human to review, and deterministic
|
|
92
|
+
enough for prompt caching. The source database remains authoritative: TAREL stores metadata and
|
|
93
|
+
claims, not a copy of the warehouse.
|
|
94
|
+
|
|
95
|
+
## Design principles
|
|
96
|
+
|
|
97
|
+
- **Self-extending, human-gated.** A coding agent can build missing source adapters locally from
|
|
98
|
+
explicit contracts; generated code remains isolated and inactive until reviewed.
|
|
99
|
+
- **Local first.** Graphs, indexes, reviews, and context packets work without a hosted service.
|
|
100
|
+
- **Evidence before confidence.** Technical observations, generated proposals, and human-validated
|
|
101
|
+
knowledge remain distinguishable.
|
|
102
|
+
- **Human-reviewed semantics.** Generated descriptions and inferred joins begin as proposals, never
|
|
103
|
+
as silent truth.
|
|
104
|
+
- **Small, stable context.** Deterministic ordering, explicit budgets, hashes, and visible omissions
|
|
105
|
+
make output inspectable and cache-friendly.
|
|
106
|
+
- **Optional complexity.** The core uses only the Python standard library. Database drivers, local
|
|
107
|
+
embeddings, and remote LLM calls are opt-in.
|
|
108
|
+
- **Agent-native, provider-neutral.** The agent already operating the CLI can research and extend
|
|
109
|
+
TAREL. An optional provider handles repeatable parallel annotation jobs.
|
|
110
|
+
|
|
111
|
+
## Start here
|
|
112
|
+
|
|
113
|
+
If you are evaluating TAREL, follow the [five-minute local demo](#five-minute-local-demo). It needs
|
|
114
|
+
no credentials and exercises discovery, graph construction, annotation exchange, relationship
|
|
115
|
+
detection, retrieval, context compilation, and schema-drift reporting.
|
|
116
|
+
|
|
117
|
+
If you already have a source:
|
|
118
|
+
|
|
119
|
+
- read [Teach TAREL a new source](#teach-tarel-a-new-source) when no connector exists;
|
|
120
|
+
- read [Connector runtime](#connector-runtime) for the current built-in adapters;
|
|
121
|
+
- read [Semantic annotation and review](#semantic-annotation-and-review) for meaning and approval;
|
|
122
|
+
- read [Retrieval modes](#retrieval-modes) and
|
|
123
|
+
[Deterministic context packets](#deterministic-context-packets) for agent integration.
|
|
124
|
+
|
|
125
|
+
## Implemented features
|
|
126
|
+
|
|
127
|
+
| Area | Current capability |
|
|
128
|
+
|---|---|
|
|
129
|
+
| Self-extension | Agent-readable connector tasks, isolated candidates, versioned manifests, dialect references, and a human activation gate |
|
|
130
|
+
| Connector runtime | Stable read-only contracts for probing, metadata discovery, sampling, and bounded relationship evidence |
|
|
131
|
+
| Discovery | Catalog, namespace, table, view, field, type, nullability, primary key, foreign key, and technical description observations |
|
|
132
|
+
| Sampling | Explicit, deterministic samples of 1–10 rows with field, value-size, and total-size limits |
|
|
133
|
+
| Graph | Stable technical node and edge identities; atomic local JSON persistence; deterministic revisions |
|
|
134
|
+
| Annotation | Provider-free coding-agent tasks and optional OpenRouter-backed parallel batches |
|
|
135
|
+
| Human review | Draft, validated, rejected, deferred, and `review_required` states with preserved originals and review reasons |
|
|
136
|
+
| Relationships | Declared foreign keys, human-defined joins, and bounded aggregate discovery of missing relationship candidates |
|
|
137
|
+
| Retrieval | Deterministic lexical search, dependency-free BM25, optional local vector search, and hybrid reciprocal-rank fusion |
|
|
138
|
+
| Context | Bounded object, field, join, and hop selection with visible paths, reasons, warnings, and omissions |
|
|
139
|
+
| Cache-friendly output | Stable and dynamic packet sections, graph revision, canonical hashes, packet diffing, and refresh impact checks |
|
|
140
|
+
| Workspaces | `system → area → schema` hierarchy plus explicit overlapping zones across schemas |
|
|
141
|
+
| Change Radar | Field, key, object, and relationship drift; possible renames; stale claims; affected areas, zones, and context packets |
|
|
142
|
+
| Demo | Deterministic local Retail DWH with a deliberate missing relationship and reproducible V1→V2 schema drift |
|
|
143
|
+
|
|
144
|
+
The runtime core has no mandatory third-party dependency. SQLite uses Python's standard library.
|
|
145
|
+
SQL Server and local embeddings are optional extras.
|
|
146
|
+
|
|
147
|
+
## Installation
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
python -m pip install tarel
|
|
151
|
+
tarel --version
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
This installs the dependency-free core. TAREL supports Python 3.11 and 3.12.
|
|
155
|
+
|
|
156
|
+
Optional capabilities:
|
|
157
|
+
|
|
158
|
+
```bash
|
|
159
|
+
# SQL Server through pymssql
|
|
160
|
+
python -m pip install 'tarel[sqlserver]'
|
|
161
|
+
|
|
162
|
+
# Local CPU embeddings through llama.cpp
|
|
163
|
+
python -m pip install 'tarel[local-rag]'
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
## Five-minute local demo
|
|
167
|
+
|
|
168
|
+
The bundled Retail DWH is the safest way to try TAREL. It contains synthetic analytical data,
|
|
169
|
+
requires no credentials, and is created only when requested.
|
|
170
|
+
|
|
171
|
+
### 1. Create and probe the source
|
|
172
|
+
|
|
173
|
+
```bash
|
|
174
|
+
tarel demo create retail-dwh
|
|
175
|
+
tarel connector check sqlite
|
|
176
|
+
tarel connector probe sqlite --config .tarel/demos/retail-dwh.toml
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
The generated SQLite database and configuration live below the ignored `.tarel/` directory.
|
|
180
|
+
|
|
181
|
+
Inspect the catalog or a bounded sample:
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
tarel connector discover sqlite \
|
|
185
|
+
--config .tarel/demos/retail-dwh.toml \
|
|
186
|
+
--schema main
|
|
187
|
+
|
|
188
|
+
tarel connector sample sqlite \
|
|
189
|
+
--config .tarel/demos/retail-dwh.toml \
|
|
190
|
+
--schema main \
|
|
191
|
+
--object F_SLS_01 \
|
|
192
|
+
--limit 3
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
### 2. Build the technical graph
|
|
196
|
+
|
|
197
|
+
```bash
|
|
198
|
+
tarel graph build retail-demo \
|
|
199
|
+
--connector sqlite \
|
|
200
|
+
--config .tarel/demos/retail-dwh.toml \
|
|
201
|
+
--schema main
|
|
202
|
+
|
|
203
|
+
tarel graph show retail-demo
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
The demo graph contains date, product, customer, geography, reseller, currency, and channel
|
|
207
|
+
dimensions; two sales facts; a return fact; a mapping table; and a union view. Its fact names are
|
|
208
|
+
deliberately abbreviated so that semantic annotation provides measurable value.
|
|
209
|
+
|
|
210
|
+
### 3. Annotate with the current coding agent
|
|
211
|
+
|
|
212
|
+
TAREL can work without an API provider. Ask for one complete, structured task:
|
|
213
|
+
|
|
214
|
+
```bash
|
|
215
|
+
tarel annotation next retail-demo \
|
|
216
|
+
--samples 5 \
|
|
217
|
+
--config .tarel/demos/retail-dwh.toml > annotation-task.json
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
Give `annotation-task.json` to the coding agent already running TAREL. Save its structured response
|
|
221
|
+
as `proposal.json`, then apply it as a draft:
|
|
222
|
+
|
|
223
|
+
```bash
|
|
224
|
+
tarel annotation apply retail-demo --input proposal.json
|
|
225
|
+
tarel annotation show retail-demo main.D_CHNL
|
|
226
|
+
tarel annotation validate retail-demo main.D_CHNL \
|
|
227
|
+
--include-fields \
|
|
228
|
+
--reason "Reviewed against the demo schema and bounded samples."
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
Every proposal must cover the selected object and all supplied fields. Samples are input-only:
|
|
232
|
+
they are not persisted in the graph and must not be repeated in generated descriptions.
|
|
233
|
+
|
|
234
|
+
### 4. Discover the deliberately missing relationship
|
|
235
|
+
|
|
236
|
+
`F_SLS_02.RSLR_KEY` plausibly joins `D_RSLR.RSLR_KEY`, but the demo intentionally declares no
|
|
237
|
+
foreign key. TAREL can test a small set of type-compatible candidates with bounded SQL aggregates:
|
|
238
|
+
|
|
239
|
+
```bash
|
|
240
|
+
tarel relationship discover retail-demo \
|
|
241
|
+
--object main.F_SLS_02 \
|
|
242
|
+
--field RSLR_KEY \
|
|
243
|
+
--config .tarel/demos/retail-dwh.toml
|
|
244
|
+
|
|
245
|
+
tarel relationship list retail-demo
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
Discovery stores counts and ratios, not the probed values. The result remains a draft and is not
|
|
249
|
+
used for graph expansion until a human validates its edge ID:
|
|
250
|
+
|
|
251
|
+
```bash
|
|
252
|
+
tarel relationship validate retail-demo RELATIONSHIP_ID \
|
|
253
|
+
--reason "The domain overlap and target uniqueness were reviewed."
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
### 5. Search and compile agent context
|
|
257
|
+
|
|
258
|
+
Dependency-free BM25 is useful immediately after annotation:
|
|
259
|
+
|
|
260
|
+
```bash
|
|
261
|
+
tarel search retail-demo \
|
|
262
|
+
"internet and reseller sales by year" \
|
|
263
|
+
--mode bm25 \
|
|
264
|
+
--limit 10
|
|
265
|
+
|
|
266
|
+
tarel context retail-demo \
|
|
267
|
+
"internet and reseller sales by year" \
|
|
268
|
+
--mode bm25 \
|
|
269
|
+
--max-objects 10
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
The context result contains graph metadata and proposed joins needed to answer the question. It
|
|
273
|
+
does not execute the analytical aggregation itself.
|
|
274
|
+
|
|
275
|
+
Conceptually, the agent receives a bounded packet like this:
|
|
276
|
+
|
|
277
|
+
```text
|
|
278
|
+
STABLE CONTEXT
|
|
279
|
+
graph: retail-demo
|
|
280
|
+
objects:
|
|
281
|
+
main.F_SLS_01 "Internet sales fact" [draft]
|
|
282
|
+
main.F_SLS_02 "Reseller sales fact" [draft]
|
|
283
|
+
main.D_DATE "Calendar dimension" [validated]
|
|
284
|
+
joins:
|
|
285
|
+
F_SLS_01.DOC_DT_KEY -> D_DATE.DATE_KEY [foreign_key]
|
|
286
|
+
F_SLS_02.RSLR_KEY -> D_RSLR.RSLR_KEY [candidate, draft]
|
|
287
|
+
|
|
288
|
+
DYNAMIC REQUEST
|
|
289
|
+
query: internet and reseller sales by year
|
|
290
|
+
selection: ...
|
|
291
|
+
omissions: ...
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
This is illustrative rather than copied output; the real text and JSON renderers include stable
|
|
295
|
+
identities, review states, paths, retrieval reasons, budgets, warnings, hashes, and visible
|
|
296
|
+
omissions. Unreviewed proposals may be included, but their state is never hidden.
|
|
297
|
+
|
|
298
|
+
### 6. Reproduce schema drift
|
|
299
|
+
|
|
300
|
+
Save the first packet, replace the local demo source with V2, refresh the graph, and check impact:
|
|
301
|
+
|
|
302
|
+
```bash
|
|
303
|
+
tarel context retail-demo \
|
|
304
|
+
"internet and reseller sales by year" \
|
|
305
|
+
--mode bm25 \
|
|
306
|
+
--format json > retail-context-v1.json
|
|
307
|
+
|
|
308
|
+
tarel demo create retail-dwh --version 2 --force
|
|
309
|
+
tarel graph refresh retail-demo \
|
|
310
|
+
--config .tarel/demos/retail-dwh.toml
|
|
311
|
+
|
|
312
|
+
tarel context impact retail-context-v1.json --graph retail-demo
|
|
313
|
+
```
|
|
314
|
+
|
|
315
|
+
V2 changes selected types and keys, adds and removes fields, renames a reseller field, and removes
|
|
316
|
+
one declared relationship. Change Radar preserves affected claims, moves validated annotations to
|
|
317
|
+
`review_required`, retains removed knowledge as stale claims, and identifies affected workspace
|
|
318
|
+
areas, zones, and context packets.
|
|
319
|
+
|
|
320
|
+
See the complete [Retail DWH walkthrough](https://github.com/Mattiascherzgit/tarel/blob/master/docs/retail-demo.md).
|
|
321
|
+
|
|
322
|
+
## Teach TAREL a new source
|
|
323
|
+
|
|
324
|
+
The long-term value of TAREL is not a large built-in connector inventory. It is a small connector
|
|
325
|
+
contract that a capable coding agent can implement when it meets a new database or warehouse. The
|
|
326
|
+
same authoring pattern is intended for lakes, document systems, and orchestration platforms as
|
|
327
|
+
their contracts are added.
|
|
328
|
+
|
|
329
|
+
Start by creating an isolated candidate:
|
|
330
|
+
|
|
331
|
+
```bash
|
|
332
|
+
tarel connector scaffold postgres \
|
|
333
|
+
--output .tarel/connectors/postgres-candidate
|
|
334
|
+
```
|
|
335
|
+
|
|
336
|
+
Then give the coding agent already operating TAREL a concrete instruction:
|
|
337
|
+
|
|
338
|
+
> Implement the connector described in
|
|
339
|
+
> `.tarel/connectors/postgres-candidate/CONNECTOR_TASK.md`. Use official vendor documentation,
|
|
340
|
+
> keep the driver optional, implement the read-only probe first, record only the required dialect
|
|
341
|
+
> and metadata notes under `references/`, and stop after local tests for human review.
|
|
342
|
+
|
|
343
|
+
The generated workspace gives the agent:
|
|
344
|
+
|
|
345
|
+
- an explicit versioned connector contract and capability boundary;
|
|
346
|
+
- an inactive Python adapter with the required entry point;
|
|
347
|
+
- a read-only manifest for dependencies, permissions, dialect, and references;
|
|
348
|
+
- focused authoring and SQL-dialect reference files;
|
|
349
|
+
- rules for secret handling, bounded probes, stable ordering, and visible failures;
|
|
350
|
+
- a completion gate that requires testing against a private source and human review.
|
|
351
|
+
|
|
352
|
+
The agent may consult current official documentation when a proprietary metadata API or SQL dialect
|
|
353
|
+
is unknown. It can then edit the candidate, install only the source-specific driver, and iterate on
|
|
354
|
+
`probe` and `discover_catalog` using real error output. This is the self-modifying loop: the tool
|
|
355
|
+
provides the contract and evidence boundary; the coding agent supplies the source-specific code.
|
|
356
|
+
|
|
357
|
+
Today, `scaffold` creates the complete authoring workspace but does not invoke an LLM by itself,
|
|
358
|
+
register the candidate, or execute generated code. Integration and activation are explicit human
|
|
359
|
+
decisions. That gate is intentional: self-extension should remove repetitive integration work,
|
|
360
|
+
not turn unreviewed generated code into trusted infrastructure.
|
|
361
|
+
|
|
362
|
+
## Connector runtime
|
|
363
|
+
|
|
364
|
+
Once approved and integrated, every connector uses the same small CLI surface:
|
|
365
|
+
|
|
366
|
+
```bash
|
|
367
|
+
tarel connector check NAME
|
|
368
|
+
tarel connector probe NAME --config private.toml
|
|
369
|
+
tarel connector discover NAME --config private.toml --schema SCHEMA
|
|
370
|
+
tarel connector sample NAME --config private.toml --schema SCHEMA --object OBJECT
|
|
371
|
+
```
|
|
372
|
+
|
|
373
|
+
Connectors are read-only adapters. They produce normalized observations and never mutate a TAREL
|
|
374
|
+
graph directly. The current distribution includes SQLite and SQL Server as working references for
|
|
375
|
+
the contract, not as the intended limit of the system.
|
|
376
|
+
|
|
377
|
+
Private connector files use TOML sections matching the connector name. They must remain outside
|
|
378
|
+
Git. Environment variables such as `TAREL_SQLSERVER_URL` can override local URLs.
|
|
379
|
+
|
|
380
|
+
## Semantic annotation and review
|
|
381
|
+
|
|
382
|
+
Two execution modes use the same annotation contract:
|
|
383
|
+
|
|
384
|
+
- **Coding-agent mode:** `annotation plan`, `next`, and `apply` exchange JSON with the agent already
|
|
385
|
+
operating the CLI.
|
|
386
|
+
- **Provider mode:** `graph annotate` calls an optional provider once per object and can run several
|
|
387
|
+
independent calls in parallel.
|
|
388
|
+
|
|
389
|
+
Configure OpenRouter without writing a key into the repository:
|
|
390
|
+
|
|
391
|
+
```bash
|
|
392
|
+
export OPENROUTER_API_KEY='...'
|
|
393
|
+
tarel provider configure openrouter --from-env --model MODEL_NAME
|
|
394
|
+
tarel provider test openrouter
|
|
395
|
+
```
|
|
396
|
+
|
|
397
|
+
Run a bounded batch:
|
|
398
|
+
|
|
399
|
+
```bash
|
|
400
|
+
tarel graph annotate retail-demo \
|
|
401
|
+
--provider openrouter \
|
|
402
|
+
--workers 4 \
|
|
403
|
+
--retry 1 \
|
|
404
|
+
--samples 5 \
|
|
405
|
+
--config .tarel/demos/retail-dwh.toml
|
|
406
|
+
```
|
|
407
|
+
|
|
408
|
+
Using `--samples` with a remote provider is an explicit data-boundary decision and is announced on
|
|
409
|
+
stderr. Without it, only technical metadata and graph relationships are sent.
|
|
410
|
+
|
|
411
|
+
Review commands never silently turn generated meaning into trusted meaning:
|
|
412
|
+
|
|
413
|
+
```bash
|
|
414
|
+
tarel annotation list retail-demo --state draft
|
|
415
|
+
tarel annotation show retail-demo main.F_SLS_01.NET_AMT
|
|
416
|
+
tarel annotation edit retail-demo main.F_SLS_01.NET_AMT \
|
|
417
|
+
--input annotation-patch.json \
|
|
418
|
+
--reason "Definition supplied by the data owner"
|
|
419
|
+
tarel annotation validate retail-demo main.F_SLS_01.NET_AMT \
|
|
420
|
+
--reason "Definition and currency behavior confirmed"
|
|
421
|
+
tarel annotation defer retail-demo main.F_SLS_01 \
|
|
422
|
+
--reason "Waiting for the data owner"
|
|
423
|
+
tarel annotation reject retail-demo main.F_SLS_01 \
|
|
424
|
+
--reason "Proposed meaning is incorrect"
|
|
425
|
+
```
|
|
426
|
+
|
|
427
|
+
Human edits preserve the original proposal. Rejected claims are hidden from retrieval while the
|
|
428
|
+
technical schema stays available.
|
|
429
|
+
|
|
430
|
+
## Retrieval modes
|
|
431
|
+
|
|
432
|
+
`search` and `context` support four retrieval modes:
|
|
433
|
+
|
|
434
|
+
- `lexical`: deterministic name and annotation matching;
|
|
435
|
+
- `bm25`: dependency-free ranked retrieval over safe graph documents;
|
|
436
|
+
- `vector`: local embeddings from a current persisted index;
|
|
437
|
+
- `hybrid`: BM25 and vector results combined with reciprocal-rank fusion.
|
|
438
|
+
|
|
439
|
+
For local vector and hybrid retrieval:
|
|
440
|
+
|
|
441
|
+
```bash
|
|
442
|
+
python -m pip install 'tarel[local-rag]'
|
|
443
|
+
tarel model download
|
|
444
|
+
tarel model status
|
|
445
|
+
tarel index build retail-demo
|
|
446
|
+
tarel search retail-demo "annual online revenue" --mode hybrid
|
|
447
|
+
```
|
|
448
|
+
|
|
449
|
+
The embedding model is downloaded only by the explicit command, checksum-verified, and stored in
|
|
450
|
+
the user cache. Models and vector indexes are not included in the repository or package. TAREL
|
|
451
|
+
embeds an allowlist of graph metadata; connection information, sample values, and arbitrary
|
|
452
|
+
provenance are excluded. See [Local retrieval](https://github.com/Mattiascherzgit/tarel/blob/master/docs/local-retrieval.md).
|
|
453
|
+
|
|
454
|
+
## Deterministic context packets
|
|
455
|
+
|
|
456
|
+
`tarel.context.v0.2` separates a reusable stable prefix from question-specific dynamic content.
|
|
457
|
+
The packet includes:
|
|
458
|
+
|
|
459
|
+
- graph name and deterministic revision;
|
|
460
|
+
- selected annotation states and scope;
|
|
461
|
+
- ordered objects, fields, joins, and validated relationship candidates;
|
|
462
|
+
- selection paths, retrieval reasons, warnings, and visible omissions;
|
|
463
|
+
- stable, dynamic, and complete packet hashes;
|
|
464
|
+
- an explicit character budget without timestamps, runtimes, or local paths.
|
|
465
|
+
|
|
466
|
+
Compare two packets or test whether a source refresh invalidated an older packet:
|
|
467
|
+
|
|
468
|
+
```bash
|
|
469
|
+
tarel context diff packet-a.json packet-b.json
|
|
470
|
+
tarel context impact packet-a.json --graph retail-demo
|
|
471
|
+
```
|
|
472
|
+
|
|
473
|
+
TAREL intentionally emits neutral packet structure rather than provider-specific cache headers,
|
|
474
|
+
TTLs, or session controls. A consuming agent or application can map the stable and dynamic sections
|
|
475
|
+
to its own caching mechanism. See the
|
|
476
|
+
[context packet contract](https://github.com/Mattiascherzgit/tarel/blob/master/docs/context-contract.md).
|
|
477
|
+
|
|
478
|
+
## Workspaces, areas, and zones
|
|
479
|
+
|
|
480
|
+
Graphs remain independent source projections. A workspace organizes them without copying or
|
|
481
|
+
rewriting their nodes:
|
|
482
|
+
|
|
483
|
+
```text
|
|
484
|
+
workspace
|
|
485
|
+
└── system
|
|
486
|
+
└── area
|
|
487
|
+
└── schema
|
|
488
|
+
```
|
|
489
|
+
|
|
490
|
+
Zones are explicit overlapping sets of tables and views. They may cross schemas and areas within
|
|
491
|
+
one system.
|
|
492
|
+
|
|
493
|
+
```bash
|
|
494
|
+
tarel workspace create enterprise
|
|
495
|
+
tarel workspace system define enterprise commercial --graph retail-demo
|
|
496
|
+
tarel workspace area define enterprise commercial sales \
|
|
497
|
+
--schema retail-demo:main
|
|
498
|
+
tarel workspace zone define enterprise commercial revenue \
|
|
499
|
+
--object retail-demo:main.F_SLS_01 \
|
|
500
|
+
--object retail-demo:main.F_SLS_02 \
|
|
501
|
+
--object retail-demo:main.D_DATE
|
|
502
|
+
tarel workspace zone show enterprise commercial revenue
|
|
503
|
+
```
|
|
504
|
+
|
|
505
|
+
Change Radar reports which areas and zones are affected by a graph refresh without rewriting their
|
|
506
|
+
definitions automatically. See
|
|
507
|
+
[Workspaces, systems, areas, and zones](https://github.com/Mattiascherzgit/tarel/blob/master/docs/workspaces.md).
|
|
508
|
+
|
|
509
|
+
## Local persistence and data boundaries
|
|
510
|
+
|
|
511
|
+
TAREL is file-first. Runtime state is stored below the working directory:
|
|
512
|
+
|
|
513
|
+
```text
|
|
514
|
+
.tarel/
|
|
515
|
+
├── demos/
|
|
516
|
+
├── graphs/
|
|
517
|
+
│ └── GRAPH/
|
|
518
|
+
│ ├── graph.json
|
|
519
|
+
│ └── changes/
|
|
520
|
+
├── indexes/
|
|
521
|
+
└── workspaces/
|
|
522
|
+
```
|
|
523
|
+
|
|
524
|
+
`.tarel/` is ignored by Git. Graph and workspace stores have explicit whole-document boundaries so
|
|
525
|
+
that a later shared database implementation can use the same application use cases.
|
|
526
|
+
|
|
527
|
+
Security defaults:
|
|
528
|
+
|
|
529
|
+
- connector manifests currently accept read permission only;
|
|
530
|
+
- normal CLI errors do not print connection URLs or passwords;
|
|
531
|
+
- provider keys are stored outside the repository with user-only file permissions;
|
|
532
|
+
- samples require an explicit command or positive sample limit;
|
|
533
|
+
- sample blocks are never persisted in graph annotations;
|
|
534
|
+
- inferred relationships are not traversed until human validation;
|
|
535
|
+
- model downloads and remote provider calls never happen during import.
|
|
536
|
+
|
|
537
|
+
## Command overview
|
|
538
|
+
|
|
539
|
+
```text
|
|
540
|
+
tarel demo create deterministic local demo sources
|
|
541
|
+
tarel connector check, probe, discover, sample, and scaffold connectors
|
|
542
|
+
tarel graph build, refresh, list, show, and batch-annotate graphs
|
|
543
|
+
tarel annotation plan, exchange, apply, inspect, edit, and review proposals
|
|
544
|
+
tarel relationship add, probe, discover, list, validate, and reject possible joins
|
|
545
|
+
tarel search retrieve relevant graph objects and fields
|
|
546
|
+
tarel context build, diff, and impact-check deterministic context packets
|
|
547
|
+
tarel model explicitly download and verify the optional embedding model
|
|
548
|
+
tarel index build and inspect local vector indexes
|
|
549
|
+
tarel workspace organize graphs into systems, areas, schemas, and zones
|
|
550
|
+
tarel provider configure and test optional annotation providers
|
|
551
|
+
```
|
|
552
|
+
|
|
553
|
+
Every substantive command supports deterministic JSON output where it is useful. Run
|
|
554
|
+
`tarel COMMAND --help` for the exact current interface.
|
|
555
|
+
|
|
556
|
+
## What TAREL does not do yet
|
|
557
|
+
|
|
558
|
+
- no stable public SDK;
|
|
559
|
+
- the distribution does not yet ship PostgreSQL, cloud warehouse, lake, document, or orchestration
|
|
560
|
+
connectors; agents can already author database and warehouse candidates;
|
|
561
|
+
- no operational lineage or ETL run history yet;
|
|
562
|
+
- no tested Apache Ossie import/export compatibility yet;
|
|
563
|
+
- no multi-user shared graph store yet;
|
|
564
|
+
- no automatic execution of analytical SQL;
|
|
565
|
+
- no autonomous activation of generated connector code -- activation remains a human trust gate;
|
|
566
|
+
- no web UI, catalog server, or mandatory cloud service.
|
|
567
|
+
|
|
568
|
+
These are scope boundaries, not hidden fallbacks. Unsupported capabilities fail visibly.
|
|
569
|
+
|
|
570
|
+
## Development
|
|
571
|
+
|
|
572
|
+
```bash
|
|
573
|
+
python -m pip install -e '.[dev]'
|
|
574
|
+
ruff check src tests tools
|
|
575
|
+
python -m unittest discover -s tests -q
|
|
576
|
+
python -m compileall -q src tests tools
|
|
577
|
+
python -m build
|
|
578
|
+
python tools/check_distribution.py dist
|
|
579
|
+
```
|
|
580
|
+
|
|
581
|
+
Live database targets and credentials belong in ignored local configuration. The deterministic
|
|
582
|
+
SQLite demo and the unit suite require no external service.
|
|
583
|
+
|
|
584
|
+
## Documentation
|
|
585
|
+
|
|
586
|
+
- [Retail DWH demo](https://github.com/Mattiascherzgit/tarel/blob/master/docs/retail-demo.md)
|
|
587
|
+
- [Change Radar and stale claims](https://github.com/Mattiascherzgit/tarel/blob/master/docs/change-radar.md)
|
|
588
|
+
- [Local retrieval](https://github.com/Mattiascherzgit/tarel/blob/master/docs/local-retrieval.md)
|
|
589
|
+
- [Context packet contract](https://github.com/Mattiascherzgit/tarel/blob/master/docs/context-contract.md)
|
|
590
|
+
- [Workspaces, systems, areas, and zones](https://github.com/Mattiascherzgit/tarel/blob/master/docs/workspaces.md)
|
|
591
|
+
|
|
592
|
+
## License
|
|
593
|
+
|
|
594
|
+
TAREL is available under the
|
|
595
|
+
[MIT License](https://github.com/Mattiascherzgit/tarel/blob/master/LICENSE). Optional components and
|
|
596
|
+
the separately downloaded embedding model are listed in the
|
|
597
|
+
[third-party notices](https://github.com/Mattiascherzgit/tarel/blob/master/THIRD_PARTY_NOTICES.md).
|