lineage-data-format 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. lineage_data_format-0.1.0/.gitignore +57 -0
  2. lineage_data_format-0.1.0/CHANGELOG.md +50 -0
  3. lineage_data_format-0.1.0/LICENSE +28 -0
  4. lineage_data_format-0.1.0/PKG-INFO +229 -0
  5. lineage_data_format-0.1.0/README.md +201 -0
  6. lineage_data_format-0.1.0/pyproject.toml +76 -0
  7. lineage_data_format-0.1.0/src/ldf/__init__.py +24 -0
  8. lineage_data_format-0.1.0/src/ldf/cli.py +95 -0
  9. lineage_data_format-0.1.0/src/ldf/converter.py +718 -0
  10. lineage_data_format-0.1.0/tests/__init__.py +3 -0
  11. lineage_data_format-0.1.0/tests/test_converter.py +1036 -0
lineage_data_format-0.1.0/.gitignore ADDED
@@ -0,0 +1,57 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+ *.so
6
+ .Python
7
+ build/
8
+ develop-eggs/
9
+ dist/
10
+ downloads/
11
+ eggs/
12
+ .eggs/
13
+ lib/
14
+ lib64/
15
+ parts/
16
+ sdist/
17
+ var/
18
+ wheels/
19
+ *.egg-info/
20
+ .installed.cfg
21
+ *.egg
22
+
23
+ # Virtual environments
24
+ .venv/
25
+ venv/
26
+ env/
27
+ ENV/
28
+
29
+ # IDE
30
+ .vscode/
31
+ .idea/
32
+ *.swp
33
+ *.swo
34
+ *~
35
+
36
+ # Testing
37
+ .pytest_cache/
38
+ .coverage
39
+ htmlcov/
40
+ .tox/
41
+
42
+ # OS
43
+ .DS_Store
44
+ Thumbs.db
45
+
46
+ # LDF and JSON outputs (generated artefacts — not source files)
47
+ # Allow: examples/mock_backend_response.json (committed example input)
48
+ # Allow: examples/output_backend.ldf (committed example output)
49
+ # Exclude: any other generated outputs in examples/
50
+ examples/output_*.ldf
51
+ examples/restored_*.json
52
+
53
+ # Exclude loose JSON/LDF files in root and other dirs (not examples/)
54
+ *.json
55
+ !examples/mock_backend_response.json
56
+
57
+ # Made with Bob
lineage_data_format-0.1.0/CHANGELOG.md ADDED
@@ -0,0 +1,50 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project will be documented in this file.
4
+
5
+ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
6
+ Versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
+
8
+ ---
9
+
10
+ ## [0.1.0] — 2024-07-22
11
+
12
+ ### Added
13
+
14
+ - Initial release of the Lineage Definition Format (LDF) Python library
15
+ - Full support for the backend `LineageGraph` API schema:
16
+ - `assets_in_view` with all `LineageAsset` fields
17
+ - `edges_in_view` with `FlowEdge.type` (`direct` / `summary`)
18
+ - `graph_calculation_datetime` and `graph_calculation_timestamp`
19
+ - **New LDF sections** for complete backend schema coverage:
20
+ - `EP` — edge properties (type per edge)
21
+ - `BC` — business classifications lookup
22
+ - `DC` — data classes lookup
23
+ - `HP` — hierarchical path per node
24
+ - `ATTR` — asset attributes per node
25
+ - `SCS` — source code snippets per node (JSON blob)
26
+ - `CA` / `PA` / `SA` — catalog / project / space assignments per node (JSON blob)
27
+ - `DSD` — data source definition asset per node
28
+ - `identity_key` field serialized as `ik=` token in `N:` lines
29
+ - Boolean flags (`is_deduced`, `is_transforming`, `is_operational`, `is_temporary`, `is_favorite`) packed as bitmask `flags=N` in `N:` lines
30
+ - `children` summary (`count`, `has_any`, `href`) as `ch=count,has_any,href` in `N:` lines
31
+ - Technology deduplication by `id` (not name) to prevent UUID loss
32
+ - **Lossless percent-encoding** (`pct_encode` / `pct_decode`) covering all structural characters:
33
+ - `%`, `\n`, `\r`, space, `,`, `[`, `]`, `|`, `=`, `:`
34
+ - Applied to: tags, `node_type`, asset `name`, `resource_key`, `origin`, `identity_key`, HP name/type, ATTR values
35
+ - `pip install git+https://github.com/IBM/lineage-data-format.git` support
36
+ - GitHub Actions CI on Python 3.9, 3.11, 3.13
37
+ - 64 tests covering round-trip losslessness for all schema fields and edge cases
38
+
39
+ ### Fixed
40
+
41
+ - Tags containing commas, spaces, brackets or `%` were silently corrupted
42
+ - `node_type` containing spaces was truncated at the first space
43
+ - Asset `name` containing newlines was split across `NAME:` section lines
44
+ - `ATTR` values containing `:` were truncated at the first colon
45
+ - `HP` name/type containing `:` or `|` were parsed incorrectly
46
+ - `resource_key` and `origin` containing spaces broke the N: line token parser
47
+ - Technology deduplication used `name` instead of `id` — two technologies with the same name but different IDs were merged into one
48
+ - `identity_key` field was not serialized or deserialized
49
+
50
+ [0.1.0]: https://github.com/IBM/lineage-data-format/releases/tag/v0.1.0
lineage_data_format-0.1.0/LICENSE ADDED
@@ -0,0 +1,28 @@
1
+ BSD 3-Clause License
2
+
3
+ Copyright (c) 2026, International Business Machines
4
+
5
+ Redistribution and use in source and binary forms, with or without
6
+ modification, are permitted provided that the following conditions are met:
7
+
8
+ 1. Redistributions of source code must retain the above copyright notice, this
9
+ list of conditions and the following disclaimer.
10
+
11
+ 2. Redistributions in binary form must reproduce the above copyright notice,
12
+ this list of conditions and the following disclaimer in the documentation
13
+ and/or other materials provided with the distribution.
14
+
15
+ 3. Neither the name of the copyright holder nor the names of its
16
+ contributors may be used to endorse or promote products derived from
17
+ this software without specific prior written permission.
18
+
19
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
23
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
lineage_data_format-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,229 @@
1
+ Metadata-Version: 2.5
2
+ Name: lineage-data-format
3
+ Version: 0.1.0
4
+ Summary: Lineage Definition Format — token-optimized serialization for data lineage graphs
5
+ Project-URL: Homepage, https://github.com/IBM/lineage-data-format
6
+ Project-URL: Repository, https://github.com/IBM/lineage-data-format
7
+ Project-URL: Issues, https://github.com/IBM/lineage-data-format/issues
8
+ Project-URL: Changelog, https://github.com/IBM/lineage-data-format/blob/main/CHANGELOG.md
9
+ Author-email: Karol Trzaska <ktrzaska@ibm.com>, Gregoire Cattan <Gregoire.Cattan@ibm.com>, Maciej Stokfisz <maciej.stokfisz@ibm.com>, Patryk Pierzchala <Patryk.Pierzchala1@ibm.com>
10
+ License-Expression: Apache-2.0
11
+ License-File: LICENSE
12
+ Keywords: data-lineage,graph,lineage,llm,serialization,token-optimization
13
+ Classifier: Development Status :: 4 - Beta
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.9
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Programming Language :: Python :: 3.13
21
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
22
+ Classifier: Typing :: Typed
23
+ Requires-Python: >=3.9
24
+ Provides-Extra: dev
25
+ Requires-Dist: pytest-cov; extra == 'dev'
26
+ Requires-Dist: pytest>=7; extra == 'dev'
27
+ Description-Content-Type: text/markdown
28
+
29
+ # Lineage Definition Format (LDF)
30
+
31
+ [![CI](https://github.com/IBM/lineage-data-format/actions/workflows/ci.yml/badge.svg)](https://github.com/IBM/lineage-data-format/actions/workflows/ci.yml)
32
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue.svg)](https://www.python.org/downloads/)
33
+
34
+ A Python library for converting between the backend `LineageGraph` JSON schema and a compact, token-optimized lineage definition format.
35
+
36
+ ## Installation
37
+
38
+ Install directly from GitHub:
39
+
40
+ ```bash
41
+ pip install git+https://github.com/IBM/lineage-data-format.git
42
+ ```
43
+
44
+ Pin to a specific release:
45
+
46
+ ```bash
47
+ pip install git+https://github.com/IBM/lineage-data-format.git@v0.1.0
48
+ ```
49
+
50
+ Install from source for development:
51
+
52
+ ```bash
53
+ git clone https://github.com/IBM/lineage-data-format.git
54
+ cd lineage-data-format
55
+ pip install -e ".[dev]"
56
+ ```
57
+
58
+ ## Input Schema
59
+
60
+ Both `json_to_lineage_format` and `convert_json_file_to_lineage` expect the backend `LineageGraph` response format:
61
+
62
+ ```json
63
+ {
64
+ "assets_in_view": [ /* list of LineageAsset objects */ ],
65
+ "edges_in_view": [ /* list of FlowEdge objects */ ],
66
+ "graph_calculation_datetime": "2024-07-22T08:16:22.694Z",
67
+ "graph_calculation_timestamp": 1721636182694
68
+ }
69
+ ```
70
+
71
+ ### LineageAsset fields
72
+
73
+ | Field | Type | Notes |
74
+ |---|---|---|
75
+ | `id` | string | Asset UUID |
76
+ | `name` | string | Display name |
77
+ | `type` | string | e.g. `"Column"`, `"Table"` |
78
+ | `resource_key` | string | e.g. `"PostgreSQL/db/schema/table/col"` |
79
+ | `origin` | string | e.g. `"runtime"` |
80
+ | `technology` | `{id, name}` | Technology lookup |
81
+ | `hierarchical_path` | `[{id, name, type}]` | Ancestry path |
82
+ | `attributes` | `[{name, value}]` | Asset attributes |
83
+ | `tags` | `string[]` | Tag list |
84
+ | `business_terms` | `[{id, name}]` | |
85
+ | `business_classifications` | `[{id, name}]` | |
86
+ | `data_classes` | `[{id, name}]` | |
87
+ | `source_code_snippets` | array | Full snippet objects |
88
+ | `catalog_assignments` | array | Full assignment objects |
89
+ | `project_assignments` | array | Full assignment objects |
90
+ | `space_assignments` | array | Full assignment objects |
91
+ | `data_source_definition_asset` | `{id, name}` | |
92
+ | `children` | `{count, has_any, href}` | Child summary |
93
+ | `is_deduced` | bool | |
94
+ | `is_transforming` | bool | |
95
+ | `is_operational` | bool | |
96
+ | `is_temporary` | bool | |
97
+ | `is_favorite` | bool | |
98
+
99
+ ### FlowEdge fields
100
+
101
+ | Field | Type | Notes |
102
+ |---|---|---|
103
+ | `source` | string | Source asset UUID |
104
+ | `target` | string | Target asset UUID |
105
+ | `type` | `"direct"` \| `"summary"` | Edge type |
106
+
107
+ ## Usage
108
+
109
+ ### As a Library
110
+
111
+ ```python
112
+ from ldf import (
113
+ json_to_lineage_format,
114
+ lineage_format_to_json,
115
+ convert_json_file_to_lineage,
116
+ convert_lineage_file_to_json,
117
+ )
118
+
119
+ # Convert a backend LineageGraph dict to LDF string
120
+ ldf_text = json_to_lineage_format(backend_response)
121
+
122
+ # Convert LDF string back to backend LineageGraph dict
123
+ restored = lineage_format_to_json(ldf_text)
124
+
125
+ # Convert files directly
126
+ convert_json_file_to_lineage('input.json', 'output.ldf')
127
+ convert_lineage_file_to_json('input.ldf', 'output.json')
128
+
129
+ # Compact mode (single blank line between sections instead of two)
130
+ ldf_compact = json_to_lineage_format(backend_response, {'compact': True})
131
+ ```
132
+
133
+ ### Command Line Interface
134
+
135
+ ```bash
136
+ # Convert JSON to lineage format
137
+ ldf json-to-lineage input.json output.ldf
138
+
139
+ # Convert with compact format (minimal blank lines)
140
+ ldf json-to-lineage input.json output.ldf --compact
141
+
142
+ # Convert lineage format back to JSON
143
+ ldf lineage-to-json input.ldf output.json
144
+
145
+ # Specify JSON indentation
146
+ ldf lineage-to-json input.ldf output.json --indent 4
147
+ ```
148
+
149
+ ## Format Overview
150
+
151
+ The Lineage Definition Format is a compact, plain-text format that defines repeated values once in named lookup sections and references them via short aliases throughout. All sections are optional except `GRAPH`, `START`, `E`, and `N`.
152
+
153
+ ### Section Reference
154
+
155
+ | Section | Type | Content |
156
+ |---|---|---|
157
+ | `GRAPH` | Structural | Marker line + optional `dt=` / `ts=` meta |
158
+ | `START` | Structural | Alias of the first asset |
159
+ | `E` | Structural | Directed edges: `N1>N2` |
160
+ | `EP` | Structural | Edge properties: `N1>N2 etype=direct` |
161
+ | `N` | Structural | Node definitions (see format below) |
162
+ | `T` | Lookup | Technology names: `T1=PostgreSQL` |
163
+ | `TID` | Lookup | Technology UUIDs: `T1=<uuid>` |
164
+ | `ID` | Lookup | Node UUIDs: `N1=<uuid>` |
165
+ | `NAME` | Lookup | Node display names: `N1=first_name` |
166
+ | `BT` | Lookup | Business terms: `BT1=<uuid>\|Term Name` |
167
+ | `BC` | Lookup | Business classifications: `BC1=<uuid>\|Name` |
168
+ | `DC` | Lookup | Data classes: `DC1=<uuid>\|Name` |
169
+ | `HP` | Per-node | Hierarchical path: `N1=id:name:type\|…` |
170
+ | `ATTR` | Per-node | Asset attributes: `N1=name:value\|…` |
171
+ | `SCS` | Per-node | Source code snippets (JSON): `N1=[…]` |
172
+ | `CA` | Per-node | Catalog assignments (JSON): `N1=[…]` |
173
+ | `PA` | Per-node | Project assignments (JSON): `N1=[…]` |
174
+ | `SA` | Per-node | Space assignments (JSON): `N1=[…]` |
175
+ | `DSD` | Per-node | Data source definition asset: `N1=id:name` |
176
+
177
+ ### Node Line Format
178
+
179
+ ```
180
+ N1:Column rk=PostgreSQL/db/s1/t/col origin=runtime path=T1 tags=[PII] bt=[BT1] bc=[BC1] dc=[DC1] flags=0 ch=0,0,https://…/children
181
+ ```
182
+
183
+ | Token | Meaning |
184
+ |---|---|
185
+ | `N1:Column` | alias:type |
186
+ | `rk=…` | `resource_key` |
187
+ | `origin=…` | `origin` |
188
+ | `path=T1` | technology alias (→ `T` / `TID` lookup) |
189
+ | `tags=[…]` | comma-separated tag list |
190
+ | `bt=[…]` | business term aliases |
191
+ | `bc=[…]` | business classification aliases |
192
+ | `dc=[…]` | data class aliases |
193
+ | `flags=N` | bitmask: bit0=`is_deduced`, bit1=`is_transforming`, bit2=`is_operational`, bit3=`is_temporary`, bit4=`is_favorite`; omitted when 0 |
194
+ | `ch=count,has_any,href` | children summary; comma-separated to avoid conflicts with URLs |
195
+
196
+ ### Example Output
197
+
198
+ See [`examples/output_backend.ldf`](examples/output_backend.ldf) for the full LDF representation of [`examples/mock_backend_response.json`](examples/mock_backend_response.json).
199
+
200
+ ## Features
201
+
202
+ - **Token-optimized**: Reduces token count by 4–6× compared to JSON
203
+ - **Bidirectional / lossless**: Full round-trip fidelity for all backend schema fields
204
+ - **Fully self-describing**: All lookup sections are embedded in the document
205
+ - **Human-readable**: Plain text, no binary encoding
206
+ - **CLI included**: Easy command-line conversion tools
207
+
208
+ ## Development
209
+
210
+ ```bash
211
+ # Create virtual environment and install dev dependencies
212
+ python3 -m venv .venv
213
+ source .venv/bin/activate
214
+ pip install -e ".[dev]"
215
+
216
+ # Run tests
217
+ pytest
218
+
219
+ # Run tests with coverage
220
+ pytest --cov=ldf
221
+ ```
222
+
223
+ ## License
224
+
225
+ See [LICENSE](LICENSE) file for details.
226
+
227
+ ## Contributing
228
+
229
+ Contributions are welcome! Please feel free to submit a Pull Request.
lineage_data_format-0.1.0/README.md ADDED
@@ -0,0 +1,201 @@
1
+ # Lineage Definition Format (LDF)
2
+
3
+ [![CI](https://github.com/IBM/lineage-data-format/actions/workflows/ci.yml/badge.svg)](https://github.com/IBM/lineage-data-format/actions/workflows/ci.yml)
4
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue.svg)](https://www.python.org/downloads/)
5
+
6
+ A Python library for converting between the backend `LineageGraph` JSON schema and a compact, token-optimized lineage definition format.
7
+
8
+ ## Installation
9
+
10
+ Install directly from GitHub:
11
+
12
+ ```bash
13
+ pip install git+https://github.com/IBM/lineage-data-format.git
14
+ ```
15
+
16
+ Pin to a specific release:
17
+
18
+ ```bash
19
+ pip install git+https://github.com/IBM/lineage-data-format.git@v0.1.0
20
+ ```
21
+
22
+ Install from source for development:
23
+
24
+ ```bash
25
+ git clone https://github.com/IBM/lineage-data-format.git
26
+ cd lineage-data-format
27
+ pip install -e ".[dev]"
28
+ ```
29
+
30
+ ## Input Schema
31
+
32
+ Both `json_to_lineage_format` and `convert_json_file_to_lineage` expect the backend `LineageGraph` response format:
33
+
34
+ ```json
35
+ {
36
+ "assets_in_view": [ /* list of LineageAsset objects */ ],
37
+ "edges_in_view": [ /* list of FlowEdge objects */ ],
38
+ "graph_calculation_datetime": "2024-07-22T08:16:22.694Z",
39
+ "graph_calculation_timestamp": 1721636182694
40
+ }
41
+ ```
42
+
43
+ ### LineageAsset fields
44
+
45
+ | Field | Type | Notes |
46
+ |---|---|---|
47
+ | `id` | string | Asset UUID |
48
+ | `name` | string | Display name |
49
+ | `type` | string | e.g. `"Column"`, `"Table"` |
50
+ | `resource_key` | string | e.g. `"PostgreSQL/db/schema/table/col"` |
51
+ | `origin` | string | e.g. `"runtime"` |
52
+ | `technology` | `{id, name}` | Technology lookup |
53
+ | `hierarchical_path` | `[{id, name, type}]` | Ancestry path |
54
+ | `attributes` | `[{name, value}]` | Asset attributes |
55
+ | `tags` | `string[]` | Tag list |
56
+ | `business_terms` | `[{id, name}]` | |
57
+ | `business_classifications` | `[{id, name}]` | |
58
+ | `data_classes` | `[{id, name}]` | |
59
+ | `source_code_snippets` | array | Full snippet objects |
60
+ | `catalog_assignments` | array | Full assignment objects |
61
+ | `project_assignments` | array | Full assignment objects |
62
+ | `space_assignments` | array | Full assignment objects |
63
+ | `data_source_definition_asset` | `{id, name}` | |
64
+ | `children` | `{count, has_any, href}` | Child summary |
65
+ | `is_deduced` | bool | |
66
+ | `is_transforming` | bool | |
67
+ | `is_operational` | bool | |
68
+ | `is_temporary` | bool | |
69
+ | `is_favorite` | bool | |
70
+
71
+ ### FlowEdge fields
72
+
73
+ | Field | Type | Notes |
74
+ |---|---|---|
75
+ | `source` | string | Source asset UUID |
76
+ | `target` | string | Target asset UUID |
77
+ | `type` | `"direct"` \| `"summary"` | Edge type |
78
+
79
+ ## Usage
80
+
81
+ ### As a Library
82
+
83
+ ```python
84
+ from ldf import (
85
+ json_to_lineage_format,
86
+ lineage_format_to_json,
87
+ convert_json_file_to_lineage,
88
+ convert_lineage_file_to_json,
89
+ )
90
+
91
+ # Convert a backend LineageGraph dict to LDF string
92
+ ldf_text = json_to_lineage_format(backend_response)
93
+
94
+ # Convert LDF string back to backend LineageGraph dict
95
+ restored = lineage_format_to_json(ldf_text)
96
+
97
+ # Convert files directly
98
+ convert_json_file_to_lineage('input.json', 'output.ldf')
99
+ convert_lineage_file_to_json('input.ldf', 'output.json')
100
+
101
+ # Compact mode (single blank line between sections instead of two)
102
+ ldf_compact = json_to_lineage_format(backend_response, {'compact': True})
103
+ ```
104
+
105
+ ### Command Line Interface
106
+
107
+ ```bash
108
+ # Convert JSON to lineage format
109
+ ldf json-to-lineage input.json output.ldf
110
+
111
+ # Convert with compact format (minimal blank lines)
112
+ ldf json-to-lineage input.json output.ldf --compact
113
+
114
+ # Convert lineage format back to JSON
115
+ ldf lineage-to-json input.ldf output.json
116
+
117
+ # Specify JSON indentation
118
+ ldf lineage-to-json input.ldf output.json --indent 4
119
+ ```
120
+
121
+ ## Format Overview
122
+
123
+ The Lineage Definition Format is a compact, plain-text format that defines repeated values once in named lookup sections and references them via short aliases throughout. All sections are optional except `GRAPH`, `START`, `E`, and `N`.
124
+
125
+ ### Section Reference
126
+
127
+ | Section | Type | Content |
128
+ |---|---|---|
129
+ | `GRAPH` | Structural | Marker line + optional `dt=` / `ts=` meta |
130
+ | `START` | Structural | Alias of the first asset |
131
+ | `E` | Structural | Directed edges: `N1>N2` |
132
+ | `EP` | Structural | Edge properties: `N1>N2 etype=direct` |
133
+ | `N` | Structural | Node definitions (see format below) |
134
+ | `T` | Lookup | Technology names: `T1=PostgreSQL` |
135
+ | `TID` | Lookup | Technology UUIDs: `T1=<uuid>` |
136
+ | `ID` | Lookup | Node UUIDs: `N1=<uuid>` |
137
+ | `NAME` | Lookup | Node display names: `N1=first_name` |
138
+ | `BT` | Lookup | Business terms: `BT1=<uuid>\|Term Name` |
139
+ | `BC` | Lookup | Business classifications: `BC1=<uuid>\|Name` |
140
+ | `DC` | Lookup | Data classes: `DC1=<uuid>\|Name` |
141
+ | `HP` | Per-node | Hierarchical path: `N1=id:name:type\|…` |
142
+ | `ATTR` | Per-node | Asset attributes: `N1=name:value\|…` |
143
+ | `SCS` | Per-node | Source code snippets (JSON): `N1=[…]` |
144
+ | `CA` | Per-node | Catalog assignments (JSON): `N1=[…]` |
145
+ | `PA` | Per-node | Project assignments (JSON): `N1=[…]` |
146
+ | `SA` | Per-node | Space assignments (JSON): `N1=[…]` |
147
+ | `DSD` | Per-node | Data source definition asset: `N1=id:name` |
148
+
149
+ ### Node Line Format
150
+
151
+ ```
152
+ N1:Column rk=PostgreSQL/db/s1/t/col origin=runtime path=T1 tags=[PII] bt=[BT1] bc=[BC1] dc=[DC1] flags=0 ch=0,0,https://…/children
153
+ ```
154
+
155
+ | Token | Meaning |
156
+ |---|---|
157
+ | `N1:Column` | alias:type |
158
+ | `rk=…` | `resource_key` |
159
+ | `origin=…` | `origin` |
160
+ | `path=T1` | technology alias (→ `T` / `TID` lookup) |
161
+ | `tags=[…]` | comma-separated tag list |
162
+ | `bt=[…]` | business term aliases |
163
+ | `bc=[…]` | business classification aliases |
164
+ | `dc=[…]` | data class aliases |
165
+ | `flags=N` | bitmask: bit0=`is_deduced`, bit1=`is_transforming`, bit2=`is_operational`, bit3=`is_temporary`, bit4=`is_favorite`; omitted when 0 |
166
+ | `ch=count,has_any,href` | children summary; comma-separated to avoid conflicts with URLs |
167
+
168
+ ### Example Output
169
+
170
+ See [`examples/output_backend.ldf`](examples/output_backend.ldf) for the full LDF representation of [`examples/mock_backend_response.json`](examples/mock_backend_response.json).
171
+
172
+ ## Features
173
+
174
+ - **Token-optimized**: Reduces token count by 4–6× compared to JSON
175
+ - **Bidirectional / lossless**: Full round-trip fidelity for all backend schema fields
176
+ - **Fully self-describing**: All lookup sections are embedded in the document
177
+ - **Human-readable**: Plain text, no binary encoding
178
+ - **CLI included**: Easy command-line conversion tools
179
+
180
+ ## Development
181
+
182
+ ```bash
183
+ # Create virtual environment and install dev dependencies
184
+ python3 -m venv .venv
185
+ source .venv/bin/activate
186
+ pip install -e ".[dev]"
187
+
188
+ # Run tests
189
+ pytest
190
+
191
+ # Run tests with coverage
192
+ pytest --cov=ldf
193
+ ```
194
+
195
+ ## License
196
+
197
+ See [LICENSE](LICENSE) file for details.
198
+
199
+ ## Contributing
200
+
201
+ Contributions are welcome! Please feel free to submit a Pull Request.
lineage_data_format-0.1.0/pyproject.toml ADDED
@@ -0,0 +1,76 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "lineage-data-format"
7
+ version = "0.1.0"
8
+ description = "Lineage Definition Format — token-optimized serialization for data lineage graphs"
9
+ readme = "README.md"
10
+ license = "Apache-2.0"
11
+ license-files = ["LICENSE"]
12
+ requires-python = ">=3.9"
13
+
14
+ authors = [
15
+ { name = "Karol Trzaska", email = "ktrzaska@ibm.com" },
16
+ { name = "Gregoire Cattan", email = "Gregoire.Cattan@ibm.com" },
17
+ { name = "Maciej Stokfisz", email = "maciej.stokfisz@ibm.com" },
18
+ { name = "Patryk Pierzchala", email = "Patryk.Pierzchala1@ibm.com" },
19
+ ]
20
+
21
+ dependencies = []
22
+
23
+ keywords = [
24
+ "lineage",
25
+ "data-lineage",
26
+ "graph",
27
+ "serialization",
28
+ "llm",
29
+ "token-optimization",
30
+ ]
31
+
32
+ classifiers = [
33
+ "Development Status :: 4 - Beta",
34
+ "Intended Audience :: Developers",
35
+ "Programming Language :: Python :: 3",
36
+ "Programming Language :: Python :: 3.9",
37
+ "Programming Language :: Python :: 3.10",
38
+ "Programming Language :: Python :: 3.11",
39
+ "Programming Language :: Python :: 3.12",
40
+ "Programming Language :: Python :: 3.13",
41
+ "Topic :: Software Development :: Libraries :: Python Modules",
42
+ "Typing :: Typed",
43
+ ]
44
+
45
+ [project.urls]
46
+ Homepage = "https://github.com/IBM/lineage-data-format"
47
+ Repository = "https://github.com/IBM/lineage-data-format"
48
+ Issues = "https://github.com/IBM/lineage-data-format/issues"
49
+ Changelog = "https://github.com/IBM/lineage-data-format/blob/main/CHANGELOG.md"
50
+
51
+ [project.optional-dependencies]
52
+ dev = [
53
+ "pytest>=7",
54
+ "pytest-cov",
55
+ ]
56
+
57
+ [project.scripts]
58
+ ldf = "ldf.cli:main"
59
+
60
+ [tool.pytest.ini_options]
61
+ testpaths = ["tests"]
62
+
63
+ [tool.hatch.build.targets.wheel]
64
+ packages = ["src/ldf"]
65
+
66
+ [tool.hatch.build.targets.sdist]
67
+ include = [
68
+ "src/",
69
+ "tests/",
70
+ "examples/mock_backend_response.json",
71
+ "examples/output_backend.ldf",
72
+ "README.md",
73
+ "CHANGELOG.md",
74
+ "LICENSE",
75
+ "pyproject.toml",
76
+ ]
lineage_data_format-0.1.0/src/ldf/__init__.py ADDED
@@ -0,0 +1,24 @@
1
+ """
2
+ Lineage Definition Format (LDF) - Token-optimized graph serialization.
3
+
4
+ This library provides functions to convert between JSON lineage data
5
+ and a compact lineage definition format.
6
+ """
7
+
8
+ from .converter import (
9
+ json_to_lineage_format,
10
+ lineage_format_to_json,
11
+ convert_json_file_to_lineage,
12
+ convert_lineage_file_to_json,
13
+ )
14
+
15
+ __version__ = "0.1.0"
16
+
17
+ __all__ = [
18
+ "json_to_lineage_format",
19
+ "lineage_format_to_json",
20
+ "convert_json_file_to_lineage",
21
+ "convert_lineage_file_to_json",
22
+ ]
23
+
24
+ # Made with Bob