dataowl 0.1.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. dataowl-0.3.0/CHANGELOG.md +88 -0
  2. dataowl-0.3.0/PKG-INFO +477 -0
  3. dataowl-0.3.0/README.md +447 -0
  4. {dataowl-0.1.0 → dataowl-0.3.0}/pyproject.toml +1 -1
  5. dataowl-0.3.0/src/dataowl/__init__.py +167 -0
  6. dataowl-0.3.0/src/dataowl/collect/__init__.py +80 -0
  7. {dataowl-0.1.0 → dataowl-0.3.0}/src/dataowl/collect/columns.py +22 -9
  8. dataowl-0.3.0/src/dataowl/collect/compare.py +110 -0
  9. dataowl-0.3.0/src/dataowl/collect/constraints.py +78 -0
  10. dataowl-0.3.0/src/dataowl/collect/history.py +424 -0
  11. dataowl-0.3.0/src/dataowl/collect/keys.py +205 -0
  12. {dataowl-0.1.0 → dataowl-0.3.0}/src/dataowl/collect/tables.py +8 -3
  13. dataowl-0.3.0/src/dataowl/collect/timestamps.py +395 -0
  14. dataowl-0.3.0/src/dataowl/model/analysis.py +132 -0
  15. dataowl-0.3.0/src/dataowl/model/history.py +126 -0
  16. {dataowl-0.1.0 → dataowl-0.3.0}/src/dataowl/model/overview.py +18 -0
  17. {dataowl-0.1.0 → dataowl-0.3.0}/src/dataowl/render/format.py +20 -1
  18. dataowl-0.3.0/src/dataowl/render/terminal.py +495 -0
  19. dataowl-0.3.0/tests/fixtures/analysis_days.txt +17 -0
  20. dataowl-0.3.0/tests/fixtures/analysis_full.txt +60 -0
  21. dataowl-0.3.0/tests/fixtures/analysis_minimal.txt +7 -0
  22. dataowl-0.3.0/tests/fixtures/analysis_unavailable.txt +35 -0
  23. dataowl-0.3.0/tests/fixtures/describe_history_metrics.json +83 -0
  24. dataowl-0.3.0/tests/fixtures/describe_history_table.json +263 -0
  25. dataowl-0.3.0/tests/fixtures/history_full.txt +31 -0
  26. dataowl-0.3.0/tests/fixtures/history_limit.txt +23 -0
  27. dataowl-0.3.0/tests/fixtures/history_partial.txt +23 -0
  28. dataowl-0.3.0/tests/fixtures/history_view.txt +8 -0
  29. {dataowl-0.1.0 → dataowl-0.3.0}/tests/fixtures/overview_table.txt +5 -0
  30. {dataowl-0.1.0 → dataowl-0.3.0}/tests/fixtures/overview_unavailable.txt +3 -0
  31. {dataowl-0.1.0 → dataowl-0.3.0}/tests/fixtures/overview_view.txt +3 -0
  32. {dataowl-0.1.0 → dataowl-0.3.0}/tests/integration/setup_test_tables.sql +225 -11
  33. dataowl-0.3.0/tests/integration/test_integration.py +607 -0
  34. dataowl-0.3.0/tests/unit/test_analyze.py +389 -0
  35. dataowl-0.3.0/tests/unit/test_collect.py +147 -0
  36. dataowl-0.3.0/tests/unit/test_collect_compare.py +211 -0
  37. dataowl-0.3.0/tests/unit/test_collect_constraints.py +113 -0
  38. dataowl-0.3.0/tests/unit/test_collect_history.py +999 -0
  39. dataowl-0.3.0/tests/unit/test_collect_keys.py +416 -0
  40. {dataowl-0.1.0 → dataowl-0.3.0}/tests/unit/test_collect_tables.py +35 -0
  41. dataowl-0.3.0/tests/unit/test_collect_timestamps.py +704 -0
  42. {dataowl-0.1.0 → dataowl-0.3.0}/tests/unit/test_format.py +48 -2
  43. dataowl-0.3.0/tests/unit/test_history.py +232 -0
  44. {dataowl-0.1.0 → dataowl-0.3.0}/tests/unit/test_inspect.py +80 -2
  45. {dataowl-0.1.0 → dataowl-0.3.0}/tests/unit/test_terminal.py +87 -1
  46. dataowl-0.3.0/tests/unit/test_terminal_analysis.py +349 -0
  47. dataowl-0.3.0/tests/unit/test_terminal_history.py +147 -0
  48. dataowl-0.1.0/CHANGELOG.md +0 -32
  49. dataowl-0.1.0/PKG-INFO +0 -158
  50. dataowl-0.1.0/README.md +0 -128
  51. dataowl-0.1.0/src/dataowl/__init__.py +0 -62
  52. dataowl-0.1.0/src/dataowl/collect/__init__.py +0 -40
  53. dataowl-0.1.0/src/dataowl/collect/history.py +0 -1
  54. dataowl-0.1.0/src/dataowl/collect/keys.py +0 -1
  55. dataowl-0.1.0/src/dataowl/collect/timestamps.py +0 -1
  56. dataowl-0.1.0/src/dataowl/model/analysis.py +0 -1
  57. dataowl-0.1.0/src/dataowl/model/history.py +0 -1
  58. dataowl-0.1.0/src/dataowl/render/terminal.py +0 -138
  59. dataowl-0.1.0/tests/integration/test_integration.py +0 -208
  60. dataowl-0.1.0/tests/unit/test_collect.py +0 -44
  61. {dataowl-0.1.0 → dataowl-0.3.0}/.gitignore +0 -0
  62. {dataowl-0.1.0 → dataowl-0.3.0}/LICENSE +0 -0
  63. {dataowl-0.1.0 → dataowl-0.3.0}/src/dataowl/collect/counts.py +0 -0
  64. {dataowl-0.1.0 → dataowl-0.3.0}/src/dataowl/collect/detail.py +0 -0
  65. {dataowl-0.1.0 → dataowl-0.3.0}/src/dataowl/collect/properties.py +0 -0
  66. {dataowl-0.1.0 → dataowl-0.3.0}/src/dataowl/errors.py +0 -0
  67. {dataowl-0.1.0 → dataowl-0.3.0}/src/dataowl/identifiers.py +0 -0
  68. {dataowl-0.1.0 → dataowl-0.3.0}/src/dataowl/model/__init__.py +0 -0
  69. {dataowl-0.1.0 → dataowl-0.3.0}/src/dataowl/model/facts.py +0 -0
  70. {dataowl-0.1.0 → dataowl-0.3.0}/src/dataowl/py.typed +0 -0
  71. {dataowl-0.1.0 → dataowl-0.3.0}/src/dataowl/render/__init__.py +0 -0
  72. {dataowl-0.1.0 → dataowl-0.3.0}/src/dataowl/runner.py +0 -0
  73. {dataowl-0.1.0 → dataowl-0.3.0}/tests/conftest.py +0 -0
  74. {dataowl-0.1.0 → dataowl-0.3.0}/tests/fixtures/describe_detail_table.json +0 -0
  75. {dataowl-0.1.0 → dataowl-0.3.0}/tests/unit/test_collect_columns.py +0 -0
  76. {dataowl-0.1.0 → dataowl-0.3.0}/tests/unit/test_collect_counts.py +0 -0
  77. {dataowl-0.1.0 → dataowl-0.3.0}/tests/unit/test_collect_detail.py +0 -0
  78. {dataowl-0.1.0 → dataowl-0.3.0}/tests/unit/test_collect_properties.py +0 -0
  79. {dataowl-0.1.0 → dataowl-0.3.0}/tests/unit/test_facts.py +0 -0
  80. {dataowl-0.1.0 → dataowl-0.3.0}/tests/unit/test_fake_runner.py +0 -0
  81. {dataowl-0.1.0 → dataowl-0.3.0}/tests/unit/test_identifiers.py +0 -0
  82. {dataowl-0.1.0 → dataowl-0.3.0}/tests/unit/test_import.py +0 -0
  83. {dataowl-0.1.0 → dataowl-0.3.0}/tests/unit/test_import_boundaries.py +0 -0
  84. {dataowl-0.1.0 → dataowl-0.3.0}/tests/unit/test_no_judgement.py +0 -0
  85. {dataowl-0.1.0 → dataowl-0.3.0}/tests/unit/test_read_only.py +0 -0
  86. {dataowl-0.1.0 → dataowl-0.3.0}/tests/unit/test_runner.py +0 -0
@@ -0,0 +1,88 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented in this file.
4
+
5
+ ## [Unreleased]
6
+
7
+ ## [0.3.0] - 2026-10-08
8
+
9
+ Write behaviour from the Delta history (`history`).
10
+
11
+ ### Added
12
+ - `dataowl.history(table, *, spark=None, limit=None)` returning a `HistoryAnalysis` with
13
+ `to_dict()` and `show()`. `limit` reads only the newest `limit` commits.
14
+ - Observation window (oldest and newest commit, calendar days, commits), commits per day
15
+ (median, min, max, with 0 for days without commits), commits per hour of day, and
16
+ commits per operation with `WRITE (overwrite)` as its own category.
17
+ - Inserted, updated and deleted rows per operation from `operationMetrics`
18
+ (`numTargetRowsInserted`, `numTargetRowsUpdated` and `numTargetRowsDeleted` for `MERGE`,
19
+ `numOutputRows` for `WRITE`, `WRITE (overwrite)` and `... AS SELECT`, `numUpdatedRows`
20
+ for `UPDATE`, `numDeletedRows` for `DELETE`): commits with the metric out of all, sum,
21
+ median and max.
22
+ - The session time zone (`current_timezone()`) as a fact. Days and hours are counted in
23
+ that time zone.
24
+ - Notes in the output: the history is limited by `delta.logRetentionDuration`, rows
25
+ replaced by `WRITE (overwrite)` are not counted as deleted, and the oldest day in the
26
+ window may be incomplete.
27
+
28
+ ### Changed
29
+ - `inspect` and `analyze` raise `TableNotFoundError` when the catalog does not exist.
30
+ Before, every fact in `inspect` was `n/a` and `analyze` raised `RuntimeError` about the
31
+ schema.
32
+
33
+ ## [0.2.0] - 2026-10-08
34
+
35
+ Column analysis (`analyze`) and an excerpt of the Delta history in `inspect`.
36
+
37
+ ### Added
38
+ - `dataowl.analyze(table, *, key=None, timestamp=None, compare=None, days=30, spark=None)`
39
+ returning a `ColumnAnalysis` with `to_dict()` and `show(show_days=False)`. Only the
40
+ columns given by the user are analyzed, and at least one of `key`, `timestamp` and
41
+ `compare` is required.
42
+ - Key analysis (`key`): rows, rows with null in the key, distinct keys, keys occurring more
43
+ than once and their rows, median and maximum rows per key, and the number of keys with 1,
44
+ 2–10, 11–100 and more than 100 rows. Single columns, composite keys and lists of both.
45
+ - Declared primary key from `information_schema.table_constraints` and
46
+ `information_schema.key_column_usage`, collected when `key` is given. Shown as
47
+ informational and not enforced.
48
+ - Time column analysis (`timestamp`) for `TIMESTAMP`, `TIMESTAMP_NTZ` and `DATE`: min, max,
49
+ nulls and their share, values after the current time, distinct dates, gaps in whole days
50
+ between distinct dates, and rows per day for the last `days` whole days without today.
51
+ - Comparison of two time columns (`compare`): rows where the second column is after, equal
52
+ to or before the first, and rows where either is null.
53
+ - `show(show_days=True)` prints one line per day in the rows-per-day window.
54
+ - History excerpt in `inspect` from `DESCRIBE HISTORY`: new `Overview` fields
55
+ `history_first_commit`, `history_last_commit`, `history_num_commits` and
56
+ `history_operations`. `WRITE` with mode `Overwrite` is counted as `WRITE (overwrite)`.
57
+ Not run for views.
58
+
59
+ ### Changed
60
+ - The `inspect` output has a HISTORY block between the summary and the schema, with the
61
+ observation window of the history.
62
+
63
+ ## [0.1.0] - 2026-10-08
64
+
65
+ First release: overview facts for a table (`inspect`).
66
+
67
+ ### Added
68
+ - `dataowl.inspect(table, *, spark=None, count_views=False)` returning an `Overview` with
69
+ `to_dict()` and `show()`.
70
+ - Facts from `information_schema.tables`: object type (with the raw `table_type`), format,
71
+ owner, comment and created time.
72
+ - Facts from `DESCRIBE DETAIL`: size, number of files, average file size, last data
73
+ modification, partition and clustering columns. Not run for views.
74
+ - Schema from `information_schema.columns`, and the field count including nested fields
75
+ from the Spark schema.
76
+ - Exact row count with `COUNT(*)`. Skipped for views, materialized views, foreign tables and
77
+ objects whose type cannot be read, unless `count_views=True`.
78
+ - Change Data Feed, log retention and deleted file retention from `SHOW TBLPROPERTIES`. Not
79
+ run for views.
80
+ - Every fact carries its source (`metadata`, `exact`, `derived`). Facts that cannot be
81
+ collected are marked unavailable with a reason; only a missing table raises
82
+ (`TableNotFoundError`).
83
+ - Terminal rendering of the overview.
84
+
85
+ ### Known deviations
86
+ - `information_schema.columns.ordinal_position` is observed 0-based in Databricks, although
87
+ the documentation says it is numbered from 1. `ColumnInfo.position` keeps the raw value;
88
+ the `#` column in the output is a running number from 1.
dataowl-0.3.0/PKG-INFO ADDED
@@ -0,0 +1,477 @@
1
+ Metadata-Version: 2.5
2
+ Name: dataowl
3
+ Version: 0.3.0
4
+ Summary: Facts about Databricks data products for building dbt staging models.
5
+ Project-URL: Homepage, https://github.com/alexD1990/dataowl
6
+ Project-URL: Repository, https://github.com/alexD1990/dataowl
7
+ Project-URL: Changelog, https://github.com/alexD1990/dataowl/blob/master/CHANGELOG.md
8
+ Project-URL: Issues, https://github.com/alexD1990/dataowl/issues
9
+ Author: Alexandro Dronnen
10
+ License-Expression: MIT
11
+ License-File: LICENSE
12
+ Keywords: data-engineering,databricks,dbt,delta-lake,unity-catalog
13
+ Classifier: Development Status :: 3 - Alpha
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3 :: Only
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Topic :: Database
22
+ Classifier: Typing :: Typed
23
+ Requires-Python: >=3.10
24
+ Provides-Extra: dev
25
+ Requires-Dist: mypy==2.3.1; extra == 'dev'
26
+ Requires-Dist: pyspark>=3.4; extra == 'dev'
27
+ Requires-Dist: pytest; extra == 'dev'
28
+ Requires-Dist: ruff==0.16.9; extra == 'dev'
29
+ Description-Content-Type: text/markdown
30
+
31
+ # dataowl
32
+
33
+ dataowl reports facts about a table in Databricks (Unity Catalog / Delta Lake): size, files,
34
+ rows, schema, table properties and an excerpt of the Delta history, and facts about the
35
+ columns you choose: grain and uniqueness of keys, time span and cadence of time columns, and
36
+ how two time columns compare. From the Delta history it also reports how the table is
37
+ written: commits per day and per hour of day, operations, and rows inserted, updated and
38
+ deleted per operation. It is meant for data engineers who build a dbt staging model and want
39
+ the facts without writing exploratory SQL. It reports what exists and never gives
40
+ recommendations.
41
+
42
+ ## Requirements
43
+
44
+ - Unity Catalog.
45
+ - Databricks Runtime 14.3 LTS or above, or serverless compute.
46
+ Verified on serverless (Spark Connect).
47
+ - PySpark is provided by Databricks. dataowl has no other runtime dependencies.
48
+
49
+ ## Installation
50
+
51
+ In a Databricks notebook:
52
+
53
+ ```python
54
+ %pip install dataowl==0.3.0
55
+ ```
56
+
57
+ ## Usage
58
+
59
+ dataowl works in two steps: `inspect` gives an overview of the table, and `analyze` gives
60
+ facts about columns you choose. In addition, `history` gives detailed facts from the Delta
61
+ history about how the table is written.
62
+
63
+ `inspect`, `analyze` and `history` raise `TableNotFoundError` when the table does not exist
64
+ or is not accessible, including when the catalog does not exist.
65
+
66
+ ### inspect
67
+
68
+ ```python
69
+ import dataowl
70
+
71
+ dataowl.inspect("catalog.schema.table").show()
72
+ ```
73
+
74
+ As a JSON-serializable dict, where every fact has a value, a source and a reason when it
75
+ is unavailable:
76
+
77
+ ```python
78
+ overview = dataowl.inspect("catalog.schema.table")
79
+ overview.to_dict()
80
+ ```
81
+
82
+ Row counts are skipped for views, materialized views and foreign tables unless you ask for
83
+ them:
84
+
85
+ ```python
86
+ dataowl.inspect("catalog.schema.some_view", count_views=True).show()
87
+ ```
88
+
89
+ Names with characters other than letters, digits and underscore are quoted with backticks:
90
+ `` dataowl.inspect("`my-catalog`.schema.table") ``.
91
+
92
+ The overview includes an excerpt of the Delta history from `DESCRIBE HISTORY`: the oldest and
93
+ newest commit, the number of commits and the number of commits per operation. The header of
94
+ the HISTORY block always shows the observation window (oldest – newest commit), because an
95
+ operation that is missing is only missing for the period the history covers:
96
+
97
+ - The history is limited by `delta.logRetentionDuration` (see **Log retention**). Older
98
+ commits are not included.
99
+ - `WRITE (overwrite)` is a `WRITE` commit with mode `Overwrite`. It also includes partial
100
+ overwrites, for example with `replaceWhere`.
101
+ - `CREATE OR REPLACE TABLE AS SELECT`, which dbt uses for the `table` materialization, is
102
+ shown under its own operation name, not as `WRITE (overwrite)`.
103
+
104
+ ### analyze
105
+
106
+ ```python
107
+ analysis = dataowl.analyze(
108
+ "catalog.schema.table",
109
+ key=["customer_id", ("customer_id", "order_id")],
110
+ timestamp="updated_at",
111
+ compare=("created_at", "updated_at"),
112
+ )
113
+ analysis.show()
114
+ ```
115
+
116
+ Full signature:
117
+
118
+ ```python
119
+ analyze(
120
+ table: str,
121
+ *,
122
+ key: str | tuple[str, ...] | list[str | tuple[str, ...]] | None = None,
123
+ timestamp: str | list[str] | None = None,
124
+ compare: tuple[str, str] | None = None,
125
+ days: int = 30,
126
+ spark: SparkSession | None = None,
127
+ ) -> ColumnAnalysis
128
+ ```
129
+
130
+ At least one of `key`, `timestamp` and `compare` must be given. Only the columns you give are
131
+ analyzed; dataowl never chooses or suggests columns. Each key, each timestamp column and the
132
+ comparison run their own queries against the table's data, so every column or combination
133
+ you add costs compute. Column names are matched case insensitively.
134
+
135
+ - **key** analyzes grain and uniqueness. A string is one column, a tuple is one composite
136
+ key, and a list holds several keys of either kind, each analyzed separately:
137
+ `key=["customer_id", ("customer_id", "order_id")]` analyzes `customer_id` on its own and
138
+ the combination of `customer_id` and `order_id`. MAP columns cannot be used.
139
+ - **Primary key.** When `key` is given, dataowl also reads the declared primary key from
140
+ `information_schema`. A primary key in Databricks is informational and not enforced. In
141
+ `ColumnAnalysis.primary_key`, `None` means it was not collected (no `key`), a fact with
142
+ value `None` means no primary key is declared, and an unavailable fact has a reason.
143
+ - **timestamp** analyzes one column or a list of columns of type `TIMESTAMP`,
144
+ `TIMESTAMP_NTZ` or `DATE`, each separately: min, max, nulls, values after the current
145
+ time, distinct dates, the gaps in whole days between distinct dates, and rows per day.
146
+ Other types, including dates stored as `STRING`, raise `ValueError`.
147
+ - **days** (default 30) sets the window for rows per day: the last `days` whole days, not
148
+ including today, in the session time zone. Days without rows count as 0.
149
+ - **compare** counts the rows where the second column is after, equal to or before the
150
+ first, and the rows where at least one of them is null. Columns of different types are
151
+ compared with Spark's implicit type conversion: a `DATE` is compared as that date at
152
+ 00:00:00, so a `DATE` on the same day as a `TIMESTAMP` later than midnight counts as
153
+ before it.
154
+
155
+ ```python
156
+ analysis.show(show_days=True) # also prints one line per day in the window
157
+ analysis.to_dict() # JSON-serializable, like Overview.to_dict()
158
+ ```
159
+
160
+ `analyze` raises:
161
+
162
+ - `ValueError` for an invalid table name, when `key`, `timestamp` and `compare` are all
163
+ `None`, for invalid `days`, and for an unknown or unsupported column. All input is
164
+ validated before any query against the table's data.
165
+ - `RuntimeError` when no SparkSession is available, and when the schema cannot be read from
166
+ `information_schema.columns`, since the columns cannot then be validated.
167
+ - `TableNotFoundError` when the table does not exist or is not accessible, including when
168
+ the catalog does not exist.
169
+
170
+ Every other failure makes the affected facts `n/a (<reason>)`.
171
+
172
+ ### history
173
+
174
+ ```python
175
+ dataowl.history("catalog.schema.table").show()
176
+ dataowl.history("catalog.schema.table", limit=100).show() # only the newest 100 commits
177
+ ```
178
+
179
+ Full signature:
180
+
181
+ ```python
182
+ history(
183
+ table: str,
184
+ *,
185
+ spark: SparkSession | None = None,
186
+ limit: int | None = None,
187
+ ) -> HistoryAnalysis
188
+ ```
189
+
190
+ `history` reads `DESCRIBE HISTORY` and reports:
191
+
192
+ - **The observation window:** the oldest and newest commit, the number of days and the
193
+ number of commits. The window is shown at the top, because every other number only
194
+ covers this period.
195
+ - **Commits per day:** median, min and max over every calendar day in the window. Days
196
+ without commits count as 0.
197
+ - **Commits per hour of day:** the number of commits in each hour, 0–23.
198
+ - **Operations:** the number of commits per operation, with `WRITE (overwrite)` as its own
199
+ category, as in `inspect`.
200
+ - **Rows per operation:** inserted, updated and deleted rows, from `operationMetrics`.
201
+
202
+ `limit` reads only the newest `limit` commits (`DESCRIBE HISTORY ... LIMIT <limit>`). Without
203
+ `limit`, every commit in the history is read.
204
+
205
+ Commit timestamps are used as Databricks returns them, in the session time zone, without
206
+ conversion. Days and hours of day are counted in that time zone, and its name is shown as
207
+ **Time zone** (from `current_timezone()`). The number of days counts calendar days from the
208
+ date of the oldest commit to the date of the newest, both included.
209
+
210
+ Rows per operation are based on these `operationMetrics` keys:
211
+
212
+ | Operation | inserted | updated | deleted |
213
+ |---|---|---|---|
214
+ | `MERGE` | `numTargetRowsInserted` | `numTargetRowsUpdated` | `numTargetRowsDeleted` |
215
+ | `WRITE`, `WRITE (overwrite)`, operations ending with `AS SELECT` | `numOutputRows` | | |
216
+ | `UPDATE` | | `numUpdatedRows` | |
217
+ | `DELETE` | | | `numDeletedRows` |
218
+
219
+ Other combinations of operation and row type are not shown. **commits** is shown as the
220
+ number of commits that have the metric out of all commits of the operation, for example
221
+ `27/29`. Sum, median and max cover only the commits that have the metric, and are shown as
222
+ `–` when none has it. If a metric has an unexpected format, only the rows per operation are
223
+ `n/a (<reason>)`.
224
+
225
+ The output ends with three notes:
226
+
227
+ - The history is limited by `delta.logRetentionDuration`; the counts cover the window shown.
228
+ - Rows replaced by `WRITE (overwrite)` are not counted as deleted.
229
+ - The oldest day in the window may be incomplete, because `limit` or the log retention can
230
+ cut the history in the middle of a day.
231
+
232
+ ```python
233
+ analysis = dataowl.history("catalog.schema.table")
234
+ analysis.to_dict() # JSON-serializable, like Overview.to_dict()
235
+ ```
236
+
237
+ For views, every fact except the time zone is `n/a (Not available for views)`, and
238
+ `DESCRIBE HISTORY` is not run.
239
+
240
+ `history` raises:
241
+
242
+ - `ValueError` for an invalid table name, and when `limit` is not `None` or an integer of
243
+ at least 1.
244
+ - `RuntimeError` when no SparkSession is available.
245
+ - `TableNotFoundError` when the table does not exist or is not accessible, including when
246
+ the catalog does not exist.
247
+
248
+ Every other failure makes the affected facts `n/a (<reason>)`.
249
+
250
+ ## Example output
251
+
252
+ Output for Databricks' public sample data (`samples.nyctaxi.trips`).
253
+
254
+ ```python
255
+ dataowl.inspect("samples.nyctaxi.trips").show()
256
+ ```
257
+
258
+ ```text
259
+ samples.nyctaxi.trips (MANAGED, DELTA)
260
+
261
+ Size: 354.2 KB
262
+ Files: 1 (avg 354.2 KB)
263
+ Rows: 21 932
264
+ Columns: 6 (6 incl. nested)
265
+ Partitioned by: –
266
+ Clustered by: –
267
+ Change Data Feed: true
268
+ Log retention: not set (default)
269
+ Deleted file retention: not set (default)
270
+ Created: 2025-09-30 11:28
271
+ Last modified: 2026-09-14 15:07
272
+ Owner: System user
273
+ Comment: –
274
+
275
+ HISTORY (2025-09-09 15:05 – 2026-09-14 15:07, 223 commits)
276
+ CREATE OR REPLACE TABLE AS SELECT: 212
277
+ SET TBLPROPERTIES: 11
278
+
279
+ SCHEMA
280
+ # name type nullable comment
281
+ 1 tpep_pickup_datetime timestamp yes
282
+ 2 tpep_dropoff_datetime timestamp yes
283
+ 3 trip_distance double yes
284
+ 4 fare_amount double yes
285
+ 5 pickup_zip int yes
286
+ 6 dropoff_zip int yes
287
+ ```
288
+
289
+ ```python
290
+ dataowl.analyze(
291
+ "samples.nyctaxi.trips",
292
+ key=["pickup_zip", ("pickup_zip", "dropoff_zip")],
293
+ timestamp="tpep_pickup_datetime",
294
+ compare=("tpep_pickup_datetime", "tpep_dropoff_datetime"),
295
+ ).show()
296
+ ```
297
+
298
+ ```text
299
+ samples.nyctaxi.trips
300
+
301
+ PRIMARY KEY none declared
302
+
303
+ KEY pickup_zip
304
+ Rows: 21 932
305
+ Rows with null in key: 0
306
+ Distinct keys: 128
307
+ Keys occurring >1 time: 100
308
+ Rows in those keys: 21 904
309
+ Rows per key: median 19 · max 1 227
310
+ 1 row: 28
311
+ 2–10 rows: 27
312
+ 11–100 rows: 31
313
+ >100 rows: 42
314
+
315
+ KEY pickup_zip + dropoff_zip
316
+ Rows: 21 932
317
+ Rows with null in key: 0
318
+ Distinct keys: 3 369
319
+ Keys occurring >1 time: 2 060
320
+ Rows in those keys: 20 623
321
+ Rows per key: median 2 · max 143
322
+ 1 row: 1 309
323
+ 2–10 rows: 1 533
324
+ 11–100 rows: 518
325
+ >100 rows: 9
326
+
327
+ TIMESTAMP tpep_pickup_datetime
328
+ Min: 2016-01-01 00:04
329
+ Max: 2016-02-29 23:51
330
+ Nulls: 0 (0.00 %)
331
+ Future values: 0
332
+ Distinct dates: 60
333
+ Days between distinct dates:
334
+ min 1 · median 1 · mean 1 · max 1
335
+ Rows per day, last 30 complete days (2026-09-08 – 2026-10-07):
336
+ median 0 · mean 0 · min 0 · max 0
337
+ Note: gaps are measured in whole days; cadence below one day is not visible.
338
+ Note: counts reflect current values of the column, not historical changes.
339
+
340
+ COMPARE tpep_pickup_datetime → tpep_dropoff_datetime
341
+ tpep_dropoff_datetime > tpep_pickup_datetime: 21 931
342
+ tpep_dropoff_datetime = tpep_pickup_datetime: 1
343
+ tpep_dropoff_datetime < tpep_pickup_datetime: 0
344
+ Either is null: 0
345
+ ```
346
+
347
+ The data in `samples.nyctaxi.trips` is from January and February 2016, so every day in the
348
+ rows-per-day window has 0 rows.
349
+
350
+ ```python
351
+ dataowl.history("samples.nyctaxi.trips").show()
352
+ ```
353
+
354
+ ```text
355
+ samples.nyctaxi.trips
356
+
357
+ HISTORY (2025-09-09 15:05 – 2026-09-14 15:07, 371 days, 223 commits)
358
+ Time zone: Etc/UTC (session)
359
+ Commits per day: median 1 · min 0 · max 2
360
+ Commits per hour of day:
361
+ 06:00 1
362
+ 15:00 222
363
+
364
+ OPERATIONS
365
+ CREATE OR REPLACE TABLE AS SELECT: 212
366
+ SET TBLPROPERTIES: 11
367
+
368
+ ROWS
369
+ operation rows commits sum median max
370
+ CREATE OR REPLACE TABLE AS SELECT inserted 212/212 4 649 584 21 932 21 932
371
+
372
+ Note: history is limited by delta.logRetentionDuration; counts cover the window above.
373
+ Note: rows replaced by WRITE (overwrite) are not counted as deleted.
374
+ Note: the oldest day in the window may be incomplete.
375
+ ```
376
+
377
+ ## Principles
378
+
379
+ - **Read-only.** Only `SELECT`, `DESCRIBE` and `SHOW`. Nothing is written, optimized or
380
+ analyzed.
381
+ - **Facts only.** No recommendations or assessments.
382
+ - **No row values leave Spark.** Only aggregates and metadata are collected. Min and max of
383
+ time columns are the only exception.
384
+ - **You choose the columns.** `analyze` only analyzes the columns you give.
385
+ - **Unavailable facts are reported with a reason.** Missing permissions or an unsupported
386
+ object type make the affected facts `n/a (<reason>)`; the rest of the report is still
387
+ produced. Only a table that does not exist or is not accessible, including a table in a
388
+ catalog that does not exist, raises `TableNotFoundError`.
389
+
390
+ ## Facts per object type
391
+
392
+ | Facts | Source | MANAGED, EXTERNAL | STREAMING_TABLE, MATERIALIZED_VIEW | FOREIGN | VIEW | UNKNOWN or n/a type |
393
+ |---|---|---|---|---|---|---|
394
+ | Object type, format, owner, comment, created | `information_schema.tables` | yes | yes | yes | yes | yes (n/a if the query fails) |
395
+ | Size, files, avg file size, last modified, partitioning, clustering | `DESCRIBE DETAIL` | yes | attempted¹ | attempted¹ | no | attempted¹ |
396
+ | Columns and schema | `information_schema.columns` | yes | yes | yes | yes | yes |
397
+ | Field count incl. nested | Spark schema | yes | yes | yes | yes | yes |
398
+ | Change Data Feed, log and deleted file retention | `SHOW TBLPROPERTIES` | yes | attempted¹ | attempted¹ | no | attempted¹ |
399
+ | History excerpt | `DESCRIBE HISTORY` | yes | attempted¹ | attempted¹ | no | attempted¹ |
400
+ | `history()`: window, commits, operations, rows per operation | `DESCRIBE HISTORY` | yes | attempted¹ | attempted¹ | no | attempted¹ |
401
+ | Rows | `COUNT(*)` | yes | MATERIALIZED_VIEW: with `count_views=True`; STREAMING_TABLE: yes | with `count_views=True` | with `count_views=True` | UNKNOWN: yes; n/a type: with `count_views=True` |
402
+
403
+ ¹ The query runs. If Databricks does not support it for the object, the facts are shown as
404
+ `n/a (<reason>)`.
405
+
406
+ "UNKNOWN" is a `table_type` dataowl does not recognize, such as `MANAGED_SHALLOW_CLONE`. The
407
+ original value is shown in the header. "n/a type" means the object type could not be read.
408
+
409
+ ## Fields
410
+
411
+ ### inspect
412
+
413
+ - **Last modified** is the last data change, from `DESCRIBE DETAIL`. **Created** is from
414
+ `information_schema.tables`.
415
+ - **Rows** is an exact `COUNT(*)`. On large tables this costs compute.
416
+ - **Files (avg ...)** is the size divided by the number of files.
417
+ - **Columns (... incl. nested)** counts every field, including fields inside structs, array
418
+ elements and map keys and values.
419
+ - **not set (default)** means the table property is not set, so Databricks uses its default.
420
+ dataowl does not report what the default is.
421
+ - **#** in the schema table is a running number from 1 in column order.
422
+ - **HISTORY** shows the oldest and newest commit in the available history, the number of
423
+ commits, and the number of commits per operation.
424
+
425
+ ### analyze
426
+
427
+ - **PRIMARY KEY** is the declared primary key, in declared order. It is informational and
428
+ not enforced by Databricks. The line is left out when `key` is not given.
429
+ - **KEY ... Rows** is every row in the table, including rows with null in a key column.
430
+ - **Rows with null in key** counts rows where at least one key column is null. These rows
431
+ are left out of every other line in the KEY block.
432
+ - **Distinct keys** is the number of distinct key values without null.
433
+ - **Keys occurring >1 time** is the number of key values that occur in more than one row.
434
+ **Rows in those keys** is the number of rows that have one of those key values.
435
+ - **Rows per key** is the median and the maximum number of rows per key value, followed by
436
+ the number of key values with 1, 2–10, 11–100 and more than 100 rows.
437
+ - **TIMESTAMP ... Nulls** is the number of rows with null, and their share of all rows.
438
+ - **Future values** counts values after the current time of the session. A `DATE` is
439
+ compared as that date at 00:00:00.
440
+ - **Days between distinct dates** is measured in whole days between consecutive distinct
441
+ dates. Cadence below one day is not visible.
442
+ - **Rows per day** covers the last `days` whole days without today, in the session time
443
+ zone, and counts current values of the column, not historical changes.
444
+ - **COMPARE first → second** counts rows where the second column is after (`>`), equal to
445
+ (`=`) or before (`<`) the first. Rows with null in either column are counted only in
446
+ **Either is null**, so the four numbers add up to all rows.
447
+ - Medians and means are rounded to one decimal and shown without decimals when the rounded
448
+ value is whole.
449
+
450
+ ### history
451
+
452
+ - **HISTORY (... days, ... commits)** is the oldest and newest commit, the number of
453
+ calendar days from the date of the oldest to the date of the newest commit, both
454
+ included, and the number of commits.
455
+ - **Limit** is shown only when `limit` is given: only the newest `limit` commits are read.
456
+ - **Time zone** is the session time zone, from `current_timezone()`. Commit timestamps,
457
+ days and hours of day are in this time zone.
458
+ - **Commits per day** is the median, min and max over every day in the window, with 0 for
459
+ days without commits.
460
+ - **Commits per hour of day** lists the hours of day that have commits, with the number of
461
+ commits.
462
+ - **OPERATIONS** is the number of commits per operation, as in the HISTORY block of
463
+ `inspect`.
464
+ - **ROWS** has one line per operation and row type (inserted, updated, deleted) with a
465
+ metric, see the table under **history** above. **commits** is the number of commits with
466
+ the metric out of all commits of the operation. **sum**, **median** and **max** cover the
467
+ commits with the metric, and are `–` when none has it.
468
+
469
+ ## Known deviations
470
+
471
+ - `information_schema.columns.ordinal_position` is documented as numbered from 1, but has
472
+ been observed 0-based in Databricks (serverless, October 2026). dataowl keeps the raw
473
+ value in `ColumnInfo.position` and shows a running number in the `#` column.
474
+
475
+ ## License
476
+
477
+ MIT. Copyright (c) 2026 Alexandro Dronnen. See [LICENSE](LICENSE).