coltrace 0.1.0.dev7__tar.gz → 0.1.0.dev8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {coltrace-0.1.0.dev7/src/coltrace.egg-info → coltrace-0.1.0.dev8}/PKG-INFO +12 -7
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/README.md +313 -308
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/backlog.md +8 -1
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/compatibility-checkpoint.md +69 -4
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/decisions.md +6 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/lineage-json.md +25 -4
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/supported-patterns.md +78 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/troubleshooting.md +21 -1
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/databricks_project/README.md +38 -33
- coltrace-0.1.0.dev8/examples/databricks_project/pipeline_a/extract/quality_extract.py +43 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/databricks_project/pipeline_a/main_orchestrator.py +9 -0
- coltrace-0.1.0.dev8/examples/databricks_project/shared/topic_changes_extract.py +30 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/notebook_cell.py +24 -6
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/pyproject.toml +1 -1
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/__init__.py +1 -1
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/analysis.py +362 -46
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/cli.py +7 -1
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/compatibility.py +77 -18
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/exporters.py +717 -711
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/graph.py +93 -18
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/journey.py +18 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/lineage_models.py +9 -2
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/models.py +1 -1
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/scanner.py +20 -2
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/sql_sources.py +561 -355
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8/src/coltrace.egg-info}/PKG-INFO +12 -7
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace.egg-info/SOURCES.txt +3 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_database_sources.py +296 -287
- coltrace-0.1.0.dev8/tests/integration/test_databricks_example.py +280 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_notebook_reporting.py +31 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_project_compatibility.py +458 -458
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_sql_reader_boundaries.py +321 -300
- coltrace-0.1.0.dev8/tests/integration/test_sql_templates.py +310 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/unit/test_exporters.py +286 -286
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/unit/test_source_models.py +283 -283
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/unit/test_source_presentation.py +118 -116
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/unit/test_sql_sources.py +69 -0
- coltrace-0.1.0.dev7/tests/integration/test_databricks_example.py +0 -131
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/LICENSE +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/MANIFEST.in +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/accuracy.md +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/architecture.md +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/column-history.md +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/column-journey-checkpoint.md +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/database-patterns.md +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/database-sources.md +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/discovery-json.md +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/graph-workspace-checkpoint.md +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/history-checkpoint.md +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/mvp-checkpoint.md +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/nlp-checkpoint.md +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/other-column-checkpoint.md +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/pandas-sources-proposal.md +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/phase-1.md +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/read-capture.md +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/docs/source-checkpoint.md +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/database_pipeline/README.md +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/database_pipeline/build_query.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/database_pipeline/pipeline.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/database_pipeline/pyproject.toml +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/databricks_project/main_orchestrator.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/databricks_project/pipeline_a/__init__.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/databricks_project/pipeline_a/extract/__init__.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/databricks_project/pipeline_a/extract/primary_extract.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/databricks_project/pipeline_a/publish/__init__.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/databricks_project/pipeline_a/publish/serialize_webapp.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/databricks_project/pipeline_a/transform/__init__.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/databricks_project/pipeline_a/transform/priority_rank.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/databricks_project/shared/__init__.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/databricks_project/shared/config.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/databricks_project/shared/token_tracker.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/databricks_project/shared/warehouse.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/pricing_pipeline/extract/prices.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/pricing_pipeline/features/pricing.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/pricing_pipeline/pipeline.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/pricing_pipeline/transform/pricing.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/read_capture/README.md +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/read_capture/project/input.csv +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/read_capture/project/pipeline.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/read_capture/run_capture.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/robustness_pipeline/README.md +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/robustness_pipeline/extract.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/robustness_pipeline/pipeline.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/robustness_pipeline/transform.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/sample_pipeline/prices.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/text_pipeline/README.md +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/text_pipeline/pipeline.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/text_pipeline/processing.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/text_pipeline/readers.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/examples/text_pipeline/text.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/setup.cfg +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/__main__.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/callbacks.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/capture.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/collector.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/history.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/nosql_sources.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/notebooks.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/pandas_api.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/py.typed +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/source_models.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/source_readers.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/templates/graph.html +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace/visual_layout.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace.egg-info/dependency_links.txt +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace.egg-info/entry_points.txt +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace.egg-info/requires.txt +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/src/coltrace.egg-info/top_level.txt +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/fixtures/sample_pipeline/extract/prices.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/fixtures/sample_pipeline/features/pricing.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/fixtures/sample_pipeline/pipeline.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/fixtures/sample_pipeline/transform/pricing.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_annotation_effects.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_callback_roles.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_capture_workflow.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_cli.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_column_story.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_compatibility_cli.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_conditional_flow.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_journey_boundaries.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_lineage_acceptance.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_nosql_extraction.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_other_column_stories.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_pandas_boundaries.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_pandas_observation.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_production_patterns.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_public_project_patterns.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_reader_inventory.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_resolution_boundaries.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_scan_project.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_source_config.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_source_precision.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/integration/test_text_pipeline.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/unit/test_analysis.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/unit/test_callbacks.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/unit/test_collector.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/unit/test_compatibility.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/unit/test_graph.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/unit/test_graph_workspace.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/unit/test_history.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/unit/test_journey.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/unit/test_nosql_sources.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/unit/test_notebooks.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/unit/test_read_capture.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/unit/test_scanner.py +0 -0
- {coltrace-0.1.0.dev7 → coltrace-0.1.0.dev8}/tests/unit/test_visual_layout.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: coltrace
|
|
3
|
-
Version: 0.1.0.
|
|
3
|
+
Version: 0.1.0.dev8
|
|
4
4
|
Summary: Static Pandas column lineage, impact analysis, and local dependency graphs
|
|
5
5
|
Author: Cristiano Diniz
|
|
6
6
|
License-Expression: MIT
|
|
@@ -32,7 +32,7 @@ column versions across direct function calls. Static scanning never imports or
|
|
|
32
32
|
executes the project or reads its input data. Optional read capture observes
|
|
33
33
|
column labels while you explicitly execute your own pipeline.
|
|
34
34
|
|
|
35
|
-
**Development preview: `0.1.0.
|
|
35
|
+
**Development preview: `0.1.0.dev8`.** The supported subset is deliberately small.
|
|
36
36
|
Unsupported operations produce diagnostics and may interrupt lineage. A graph
|
|
37
37
|
with no edge is not proof that a dependency does not exist.
|
|
38
38
|
|
|
@@ -58,10 +58,13 @@ separate release step; these commands install the local project.
|
|
|
58
58
|
|
|
59
59
|
The source checkpoint recognizes supported SQL-to-Pandas and NoSQL-to-Pandas
|
|
60
60
|
extractions. Literal SQL projections, aliases and expressions describe source
|
|
61
|
-
fields before their first Python use.
|
|
62
|
-
|
|
63
|
-
`
|
|
64
|
-
|
|
61
|
+
fields before their first Python use. Every database column the SQL reads is a
|
|
62
|
+
selectable lineage node (`origin="table"`), so a column used only inside
|
|
63
|
+
`SUM(...)` or a `WHERE` clause can be traced to the DataFrame columns it feeds.
|
|
64
|
+
Query filters are control metadata rather than value dependencies of every
|
|
65
|
+
output. `spark.sql(build_query(table)).toPandas()` keeps its SQL text even with
|
|
66
|
+
an injected, unresolved session; SQL assembled from f-strings, `.format`, `%`
|
|
67
|
+
and joins keeps its known text, with unknown parts shown as `{expression}`.
|
|
65
68
|
|
|
66
69
|
`report.readers` inventories resolved `pd.read_*` calls with file, line, context,
|
|
67
70
|
return shape and resulting DataFrame identity. File schemas are unknown unless
|
|
@@ -221,7 +224,7 @@ the shortest eligible distance, so a node reached both ways appears as direct.
|
|
|
221
224
|
| --- | --- |
|
|
222
225
|
| Explicit Pandas imports and aliases | Recognized origins; names such as `df` alone are insufficient |
|
|
223
226
|
| Resolved `pd.read_*` calls | Located inventory; supported DataFrame readers plus explicit/observed source schemas; iterators/collections remain distinct |
|
|
224
|
-
| SQL-to-Pandas extraction | Bounded SQL projections, aliases, expressions and helpers; documented readers/conversions or explicit contracts |
|
|
227
|
+
| SQL-to-Pandas extraction | Bounded SQL projections, aliases, expressions and helpers; documented readers/conversions or explicit contracts; database columns as nodes; dialect fallback |
|
|
225
228
|
| MongoDB / DynamoDB extraction | Literal projections and bounded Mongo expressions; candidate fields stay partial without observed schema |
|
|
226
229
|
| Literal dictionary `DataFrame` construction | String column keys and literal values |
|
|
227
230
|
| `frame["column"]`, `frame.loc[:, "column"]` | Literal or statically known labels (`for c in COLS`, f-strings) |
|
|
@@ -308,6 +311,8 @@ tox/nox and pytest/mypy/Ruff caches, `build`, and `dist`. Linked files/directori
|
|
|
308
311
|
and Windows reparse points are skipped. A `.coltraceignore` file at the analyzed
|
|
309
312
|
root uses gitignore syntax (root-level subset, including `!` negation). The root
|
|
310
313
|
`.gitignore` is also applied unless `respect_gitignore = false` is set.
|
|
314
|
+
`sql_dialect = "bigquery"` names the dialect of SQL text whose reader does not;
|
|
315
|
+
text that does not parse is still retried in common dialects and reported.
|
|
311
316
|
`observers = ["notify_team", "client.*"]` declares project helpers that only read
|
|
312
317
|
the DataFrames they receive, so they do not interrupt lineage.
|
|
313
318
|
`ScanConfig(exclude=("generated",))` overrides TOML settings and extends built-ins.
|