data-graf 0.1.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,398 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "https://github.com/apache/ossie/core-spec/ossie-schema.json",
4
+ "title": "Apache Ossie Core Metadata Specification",
5
+ "description": "JSON Schema for validating a single Apache Ossie semantic model document",
6
+ "type": "object",
7
+ "properties": {
8
+ "version": {
9
+ "type": "string",
10
+ "const": "0.3.0.dev0",
11
+ "description": "Apache Ossie specification version"
12
+ },
13
+ "name": {
14
+ "$ref": "#/$defs/SemanticModel/properties/name"
15
+ },
16
+ "description": {
17
+ "$ref": "#/$defs/SemanticModel/properties/description"
18
+ },
19
+ "ai_context": {
20
+ "$ref": "#/$defs/SemanticModel/properties/ai_context"
21
+ },
22
+ "datasets": {
23
+ "$ref": "#/$defs/SemanticModel/properties/datasets"
24
+ },
25
+ "relationships": {
26
+ "$ref": "#/$defs/SemanticModel/properties/relationships"
27
+ },
28
+ "metrics": {
29
+ "$ref": "#/$defs/SemanticModel/properties/metrics"
30
+ },
31
+ "nonportable_extensions": {
32
+ "$ref": "#/$defs/SemanticModel/properties/nonportable_extensions"
33
+ }
34
+ },
35
+ "required": ["version", "name", "datasets"],
36
+ "additionalProperties": false,
37
+ "$defs": {
38
+ "Dialect": {
39
+ "type": "string",
40
+ "minLength": 1,
41
+ "examples": ["OSSIE_SQL_2026", "ANSI_SQL", "DB_DIALECT", "BI_EXPRESSION", "SEMANTIC_MODEL_EXPRESSION", "PROGRAMMING_LANGUAGE"],
42
+ "description": "Expression language a metric or field expression is declared in. Any non-empty string is accepted. OSSIE_SQL_2026 is the only dialect this specification defines; others are opaque to Ossie. See extensions/dialects.md."
43
+ },
44
+ "Vendor": {
45
+ "type": "string",
46
+ "minLength": 1,
47
+ "examples": ["BI_TOOL", "DB_VENDOR", "SEMANTIC_LAYER"],
48
+ "description": "Vendor name for non-portable extensions. Any non-empty string is accepted. See extensions/vendors.md."
49
+ },
50
+ "AIContext": {
51
+ "description": "Additional context for AI tools",
52
+ "oneOf": [
53
+ {
54
+ "type": "string"
55
+ },
56
+ {
57
+ "type": "object",
58
+ "properties": {
59
+ "instructions": {
60
+ "type": "string",
61
+ "description": "Instructions for AI on how to use this entity"
62
+ },
63
+ "synonyms": {
64
+ "type": "array",
65
+ "items": {
66
+ "type": "string"
67
+ },
68
+ "description": "Alternative names and terms"
69
+ },
70
+ "examples": {
71
+ "type": "array",
72
+ "items": {
73
+ "type": "string"
74
+ },
75
+ "description": "Sample questions or use cases"
76
+ }
77
+ },
78
+ "additionalProperties": true
79
+ }
80
+ ]
81
+ },
82
+ "NonportableExtension": {
83
+ "type": "object",
84
+ "description": "Vendor-specific state that Ossie carries and does not interpret. Portable behavior does not depend on it.",
85
+ "properties": {
86
+ "vendor_name": {
87
+ "$ref": "#/$defs/Vendor"
88
+ },
89
+ "data": {
90
+ "type": "object",
91
+ "description": "Vendor-specific data as a structured object. The core schema does not reference the structure under data: no reserved key and no common sub-key. Keys are entirely vendor-defined. Values must be JSON-representable. Ossie carries the object but does not interpret it. See core-spec/spec.md#non-portable-extensions."
92
+ }
93
+ },
94
+ "required": ["vendor_name", "data"],
95
+ "additionalProperties": false
96
+ },
97
+ "DialectExpression": {
98
+ "type": "object",
99
+ "description": "Expression in a specific dialect",
100
+ "properties": {
101
+ "dialect": {
102
+ "$ref": "#/$defs/Dialect"
103
+ },
104
+ "expression": {
105
+ "type": "string",
106
+ "minLength": 1,
107
+ "description": "SQL or dialect-specific expression"
108
+ }
109
+ },
110
+ "required": ["dialect", "expression"],
111
+ "additionalProperties": false
112
+ },
113
+ "Expression": {
114
+ "type": "object",
115
+ "description": "Expression definition with multi-dialect support",
116
+ "properties": {
117
+ "sql": {
118
+ "type": "string",
119
+ "minLength": 1,
120
+ "description": "Primary, portable expression written in OSSIE_SQL_2026. When both this field and an OSSIE_SQL_2026 entry in `dialects` are present, this field is authoritative."
121
+ },
122
+ "dialects": {
123
+ "type": "array",
124
+ "items": {
125
+ "$ref": "#/$defs/DialectExpression"
126
+ },
127
+ "minItems": 1
128
+ }
129
+ },
130
+ "anyOf": [
131
+ { "required": ["sql"] },
132
+ { "required": ["dialects"] }
133
+ ],
134
+ "additionalProperties": false
135
+ },
136
+ "DataType": {
137
+ "type": "string",
138
+ "enum": [
139
+ "String",
140
+ "Integer",
141
+ "Decimal",
142
+ "Float",
143
+ "Boolean",
144
+ "Date",
145
+ "Time",
146
+ "DateTime",
147
+ "DateTimeTz",
148
+ "Opaque"
149
+ ],
150
+ "description": "Logical data type for fields and metrics, independent of role (e.g. dimension vs fact) and physical representation. `Decimal` is exact base-10 with unspecified precision and scale; `Float` is approximate. `DateTime` has no timezone or offset, while `DateTimeTz` identifies an instant using offset or timezone context but does not guarantee preservation of a named timezone. Omit `datatype` when unknown. `Opaque` means the type is outside the portable vocabulary. A vendor may record its own type name under `nonportable_extensions`; that record is not part of the portable type."
151
+ },
152
+ "Dimension": {
153
+ "type": "object",
154
+ "description": "Dimension metadata",
155
+ "properties": {
156
+ "is_time": {
157
+ "type": "boolean",
158
+ "description": "Temporal-role marker. When true, consumers that distinguish time dimensions (e.g. for time-series analysis or temporal filtering) should treat this field as a time dimension. This is a *role* flag, independent of the field's data type: a field with `is_time: true` may carry any `datatype` (e.g. `Integer` for a year grain, `String` for a month name, as well as temporal data types). When `is_time` is unset, it defaults to `true` if `datatype` is one of `Date`, `Time`, `DateTime`, or `DateTimeTz`, and `false` otherwise. Set `is_time: false` explicitly to opt a temporal-typed column (such as an audit timestamp) out of time-dimension treatment."
159
+ }
160
+ },
161
+ "additionalProperties": false
162
+ },
163
+ "Field": {
164
+ "type": "object",
165
+ "description": "Row-level attribute for grouping, filtering, and metric expressions",
166
+ "properties": {
167
+ "name": {
168
+ "type": "string",
169
+ "minLength": 1,
170
+ "description": "Unique identifier for the field within the dataset"
171
+ },
172
+ "expression": {
173
+ "$ref": "#/$defs/Expression"
174
+ },
175
+ "dimension": {
176
+ "$ref": "#/$defs/Dimension"
177
+ },
178
+ "label": {
179
+ "type": "string",
180
+ "description": "Label for categorization"
181
+ },
182
+ "description": {
183
+ "type": "string",
184
+ "description": "Human-readable description"
185
+ },
186
+ "datatype": {
187
+ "$ref": "#/$defs/DataType"
188
+ },
189
+ "ai_context": {
190
+ "$ref": "#/$defs/AIContext"
191
+ },
192
+ "nonportable_extensions": {
193
+ "type": "array",
194
+ "items": {
195
+ "$ref": "#/$defs/NonportableExtension"
196
+ }
197
+ }
198
+ },
199
+ "required": ["name", "expression"],
200
+ "additionalProperties": false
201
+ },
202
+ "Dataset": {
203
+ "type": "object",
204
+ "description": "Logical dataset representing a business entity (fact or dimension table)",
205
+ "properties": {
206
+ "name": {
207
+ "type": "string",
208
+ "minLength": 1,
209
+ "description": "Unique identifier for the dataset"
210
+ },
211
+ "source": {
212
+ "type": "string",
213
+ "minLength": 1,
214
+ "description": "Reference to underlying physical table/view (database.schema.table) or query"
215
+ },
216
+ "primary_key": {
217
+ "type": "array",
218
+ "items": {
219
+ "type": "string"
220
+ },
221
+ "minItems": 1,
222
+ "uniqueItems": true,
223
+ "description": "Dataset fields that uniquely identify rows, each named once"
224
+ },
225
+ "unique_keys": {
226
+ "type": "array",
227
+ "items": {
228
+ "type": "array",
229
+ "items": {
230
+ "type": "string"
231
+ },
232
+ "minItems": 1,
233
+ "uniqueItems": true
234
+ },
235
+ "minItems": 1,
236
+ "description": "Array of unique key definitions, each a list of dataset fields naming each field once"
237
+ },
238
+ "description": {
239
+ "type": "string",
240
+ "description": "Human-readable description"
241
+ },
242
+ "ai_context": {
243
+ "$ref": "#/$defs/AIContext"
244
+ },
245
+ "fields": {
246
+ "type": "array",
247
+ "items": {
248
+ "$ref": "#/$defs/Field"
249
+ },
250
+ "description": "The dataset's declared fields. Within the dataset, field names and metric names share one namespace and must all be unique across the two."
251
+ },
252
+ "metrics": {
253
+ "type": "array",
254
+ "items": {
255
+ "$ref": "#/$defs/Metric"
256
+ },
257
+ "description": "Dataset-scoped metrics: measures defined on this dataset's relation that semantically belong to the dataset. A metric aggregates the dataset's rows, or derives from measures of the dataset's source. The expression references this dataset's declared fields by single-part (unqualified) name, and columns of the dataset's source as source.column. When the source is a measure relation, it may evaluate a source measure with AGG(source.measure). It must resolve entirely within this dataset; an expression that must reference another dataset belongs in the model-scoped metrics instead. Within the dataset, field names and metric names share one namespace and must all be unique across the two."
258
+ },
259
+ "nonportable_extensions": {
260
+ "type": "array",
261
+ "items": {
262
+ "$ref": "#/$defs/NonportableExtension"
263
+ }
264
+ }
265
+ },
266
+ "required": ["name", "source"],
267
+ "additionalProperties": false
268
+ },
269
+ "Relationship": {
270
+ "type": "object",
271
+ "description": "Foreign key relationship between datasets",
272
+ "properties": {
273
+ "name": {
274
+ "type": "string",
275
+ "minLength": 1,
276
+ "description": "Unique identifier for the relationship"
277
+ },
278
+ "from": {
279
+ "type": "string",
280
+ "minLength": 1,
281
+ "description": "Dataset on the many side of the relationship"
282
+ },
283
+ "to": {
284
+ "type": "string",
285
+ "minLength": 1,
286
+ "description": "Dataset on the one side of the relationship"
287
+ },
288
+ "from_columns": {
289
+ "type": "array",
290
+ "items": {
291
+ "type": "string"
292
+ },
293
+ "minItems": 1,
294
+ "description": "Array of field names in the 'from' dataset (foreign key fields)"
295
+ },
296
+ "to_columns": {
297
+ "type": "array",
298
+ "items": {
299
+ "type": "string"
300
+ },
301
+ "minItems": 1,
302
+ "description": "Array of field names in the 'to' dataset (primary or unique key fields)"
303
+ },
304
+ "ai_context": {
305
+ "$ref": "#/$defs/AIContext"
306
+ },
307
+ "nonportable_extensions": {
308
+ "type": "array",
309
+ "items": {
310
+ "$ref": "#/$defs/NonportableExtension"
311
+ }
312
+ }
313
+ },
314
+ "required": ["name", "from", "to", "from_columns", "to_columns"],
315
+ "additionalProperties": false
316
+ },
317
+ "Metric": {
318
+ "type": "object",
319
+ "description": "Quantitative measure defined on business data",
320
+ "properties": {
321
+ "name": {
322
+ "type": "string",
323
+ "minLength": 1,
324
+ "description": "Unique identifier for the metric"
325
+ },
326
+ "expression": {
327
+ "$ref": "#/$defs/Expression"
328
+ },
329
+ "description": {
330
+ "type": "string",
331
+ "description": "Human-readable description of what the metric measures"
332
+ },
333
+ "datatype": {
334
+ "$ref": "#/$defs/DataType"
335
+ },
336
+ "ai_context": {
337
+ "$ref": "#/$defs/AIContext"
338
+ },
339
+ "nonportable_extensions": {
340
+ "type": "array",
341
+ "items": {
342
+ "$ref": "#/$defs/NonportableExtension"
343
+ }
344
+ }
345
+ },
346
+ "required": ["name", "expression"],
347
+ "additionalProperties": false
348
+ },
349
+ "SemanticModel": {
350
+ "type": "object",
351
+ "description": "Top-level container representing a complete semantic model",
352
+ "properties": {
353
+ "name": {
354
+ "type": "string",
355
+ "minLength": 1,
356
+ "description": "Unique identifier for the semantic model"
357
+ },
358
+ "description": {
359
+ "type": "string",
360
+ "description": "Human-readable description"
361
+ },
362
+ "ai_context": {
363
+ "$ref": "#/$defs/AIContext"
364
+ },
365
+ "datasets": {
366
+ "type": "array",
367
+ "items": {
368
+ "$ref": "#/$defs/Dataset"
369
+ },
370
+ "minItems": 1,
371
+ "description": "Collection of logical datasets"
372
+ },
373
+ "relationships": {
374
+ "type": "array",
375
+ "items": {
376
+ "$ref": "#/$defs/Relationship"
377
+ },
378
+ "description": "Defines how datasets are connected"
379
+ },
380
+ "metrics": {
381
+ "type": "array",
382
+ "items": {
383
+ "$ref": "#/$defs/Metric"
384
+ },
385
+ "description": "Model-scoped metrics: aggregations that span datasets or need to be aware of cross-dataset concepts. Expressions reference fields by multi-part name (dataset.field) and may combine fields from more than one dataset. They may evaluate a dataset-scoped metric with AGG(dataset.metric). Names must be unique across the model. See core-spec/spec.md#metric-scoping."
386
+ },
387
+ "nonportable_extensions": {
388
+ "type": "array",
389
+ "items": {
390
+ "$ref": "#/$defs/NonportableExtension"
391
+ }
392
+ }
393
+ },
394
+ "required": ["name", "datasets"],
395
+ "additionalProperties": false
396
+ }
397
+ }
398
+ }
datagraf/sqlite.py ADDED
@@ -0,0 +1,134 @@
1
+ """Databases, through dd-db: draft an Ossie skeleton from a catalogue, and profile the data behind a model. Read-only.
2
+
3
+ The connection is dd-db's `SQLiteDB(path, read_only=True)` (`mode=ro` plus `PRAGMA query_only`), so nothing here can
4
+ write. Identifiers come from the model or the catalogue and are always quoted; values are never interpolated.
5
+ dd-db has adapters for DuckDB, Postgres, Snowflake and more; `draft` and `profile` use only `BaseDB` methods, so
6
+ another adapter is a matter of `connect()` (see docs/DEV/plan.md, "Next").
7
+ """
8
+ from __future__ import annotations
9
+
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ from dd_db import BaseDB, SQLiteDB
14
+
15
+ from .model import SemanticModel
16
+
17
+ _TYPES = (("INT", "Integer"), ("CHAR", "String"), ("CLOB", "String"), ("TEXT", "String"), ("REAL", "Float"),
18
+ ("FLOA", "Float"), ("DOUB", "Float"), ("NUMERIC", "Decimal"), ("DECIMAL", "Decimal"), ("BOOL", "Boolean"),
19
+ ("DATETIME", "DateTime"), ("TIMESTAMP", "DateTime"), ("DATE", "Date"), ("TIME", "Time"))
20
+
21
+
22
+ def connect(db: str | Path) -> BaseDB:
23
+ """A read-only dd-db connection to a SQLite file."""
24
+ p = Path(db).resolve()
25
+ if not p.is_file():
26
+ raise FileNotFoundError(f"no database at {p}")
27
+ return SQLiteDB(str(p), read_only=True)
28
+
29
+
30
+ def q(name: str) -> str:
31
+ return '"' + name.replace('"', '""') + '"'
32
+
33
+
34
+ def ossie_type(decl: str) -> str | None:
35
+ """A declared SQL type as an Ossie datatype (SQLite's affinity rules, roughly); None when it says nothing."""
36
+ d = (decl or "").upper()
37
+ for key, t in _TYPES:
38
+ if key in d:
39
+ return t
40
+ return None
41
+
42
+
43
+ def _scalar(db: BaseDB, sql: str) -> Any:
44
+ df = db.run_query(sql)
45
+ v = df.iat[0, 0]
46
+ return v.item() if hasattr(v, "item") else v # numpy scalar -> Python
47
+
48
+
49
+ def _row(db: BaseDB, sql: str) -> list:
50
+ return [v.item() if hasattr(v, "item") else v for v in db.run_query(sql).iloc[0].tolist()]
51
+
52
+
53
+ def draft(db: str | Path | BaseDB, name: str | None = None, version: str = "0.3.0.dev0") -> dict:
54
+ """An Ossie document from the catalogue: one dataset per table, one field per column, the primary key, and one
55
+ relationship per foreign key. No descriptions and no metrics: those need a person or a model."""
56
+ conn = db if isinstance(db, BaseDB) else connect(db)
57
+ schemas = {t: conn.get_schema(t) for t in conn.list_tables() if not t.startswith("sqlite_")}
58
+ datasets, rels = [], []
59
+ for t, s in sorted(schemas.items()):
60
+ fields = []
61
+ for c in s.columns:
62
+ f: dict = {"name": c.name, "expression": {"sql": c.name}}
63
+ if ossie_type(c.data_type):
64
+ f["datatype"] = ossie_type(c.data_type)
65
+ if f.get("datatype") in ("Date", "Time", "DateTime"):
66
+ f["dimension"] = {"is_time": True}
67
+ fields.append(f)
68
+ ds: dict = {"name": t, "source": t, "fields": fields}
69
+ pk = [c.name for c in s.columns if c.primary_key] # dd-db lists columns in order; composite keys keep it
70
+ if pk:
71
+ ds["primary_key"] = pk
72
+ datasets.append(ds)
73
+ for fk in s.foreign_keys:
74
+ to_cols = fk.ref_columns or [c.name for c in schemas[fk.ref_table].columns if c.primary_key] \
75
+ if fk.ref_table in schemas else fk.ref_columns
76
+ rels.append({"name": f"{t}_to_{fk.ref_table}" + (f"_{fk.columns[0]}" if len(s.foreign_keys) > 1 else ""),
77
+ "from": t, "to": fk.ref_table, "from_columns": fk.columns, "to_columns": to_cols})
78
+ stem = Path(db).stem if not isinstance(db, BaseDB) else "model"
79
+ doc: dict = {"version": version, "name": name or stem, "datasets": datasets}
80
+ if rels:
81
+ doc["relationships"] = rels
82
+ return doc
83
+
84
+
85
+ def _table(tables: set[str], source: str) -> str | None:
86
+ """The table a dataset's `source` names: the whole name, else its last dotted part (db.schema.table)."""
87
+ for cand in (source, source.split(".")[-1]):
88
+ if cand in tables:
89
+ return cand
90
+ return None
91
+
92
+
93
+ def profile(model: SemanticModel, db: str | Path | BaseDB, *, distinct_limit: int = 1_000_000) -> dict:
94
+ """Row counts per dataset; per field the non-null and distinct counts, min and max; each dataset-scoped metric's value.
95
+ A field or metric whose SQL does not run reports its error instead of a value."""
96
+ conn = db if isinstance(db, BaseDB) else connect(db)
97
+ where = str(Path(db).resolve()) if not isinstance(db, BaseDB) else str(conn.connection_info())
98
+ out: dict = {"model": model.name, "database": where, "datasets": []}
99
+ tables = set(conn.list_tables())
100
+ for d in model.datasets:
101
+ entry: dict = {"name": d.name, "source": d.source, "fields": [], "metrics": []}
102
+ out["datasets"].append(entry)
103
+ table = _table(tables, d.source)
104
+ if table is None:
105
+ entry["error"] = f"no table for source {d.source!r}"
106
+ continue
107
+ entry["table"] = table
108
+ rows = _scalar(conn, f"SELECT COUNT(*) FROM {q(table)}")
109
+ entry["rows"] = rows
110
+ # each field's expression, named by the field, so dataset-scoped metrics can use field names
111
+ exprs = [(f, f.sql or q(f.name)) for f in d.fields]
112
+ inner = ", ".join(f"({sql}) AS {q(f.name)}" for f, sql in exprs) or "1 AS one"
113
+ view = f"(SELECT {inner} FROM {q(table)}) AS t"
114
+ for f, sql in exprs:
115
+ fe: dict = {"name": f.name}
116
+ try:
117
+ nn, dist, lo, hi = _row(conn,
118
+ f"SELECT COUNT({q(f.name)}), COUNT(DISTINCT {q(f.name)}), MIN({q(f.name)}), MAX({q(f.name)}) "
119
+ f"FROM (SELECT ({sql}) AS {q(f.name)} FROM {q(table)} LIMIT {int(distinct_limit)})")
120
+ fe.update(non_null=nn, nulls=rows - nn if rows <= distinct_limit else None, distinct=dist, min=lo, max=hi)
121
+ except Exception as e: # the database's own error, whichever adapter raised it
122
+ fe["error"] = str(e)
123
+ entry["fields"].append(fe)
124
+ for m in d.metrics:
125
+ me: dict = {"name": m.name, "sql": m.sql}
126
+ if m.sql is None:
127
+ me["error"] = "no SQL expression"
128
+ else:
129
+ try:
130
+ me["value"] = _scalar(conn, f"SELECT {m.sql} FROM {view}")
131
+ except Exception as e:
132
+ me["error"] = str(e)
133
+ entry["metrics"].append(me)
134
+ return out