pramiti-export-copilot 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pramiti_export_copilot-0.1.0/LICENSE +21 -0
- pramiti_export_copilot-0.1.0/PKG-INFO +112 -0
- pramiti_export_copilot-0.1.0/README.md +89 -0
- pramiti_export_copilot-0.1.0/pyproject.toml +33 -0
- pramiti_export_copilot-0.1.0/setup.cfg +4 -0
- pramiti_export_copilot-0.1.0/src/pramiti_export_copilot/__init__.py +24 -0
- pramiti_export_copilot-0.1.0/src/pramiti_export_copilot/backends.py +67 -0
- pramiti_export_copilot-0.1.0/src/pramiti_export_copilot/copilot.py +136 -0
- pramiti_export_copilot-0.1.0/src/pramiti_export_copilot/delivery_targets.py +182 -0
- pramiti_export_copilot-0.1.0/src/pramiti_export_copilot/exceptions.py +18 -0
- pramiti_export_copilot-0.1.0/src/pramiti_export_copilot/models.py +21 -0
- pramiti_export_copilot-0.1.0/src/pramiti_export_copilot.egg-info/PKG-INFO +112 -0
- pramiti_export_copilot-0.1.0/src/pramiti_export_copilot.egg-info/SOURCES.txt +15 -0
- pramiti_export_copilot-0.1.0/src/pramiti_export_copilot.egg-info/dependency_links.txt +1 -0
- pramiti_export_copilot-0.1.0/src/pramiti_export_copilot.egg-info/requires.txt +3 -0
- pramiti_export_copilot-0.1.0/src/pramiti_export_copilot.egg-info/top_level.txt +1 -0
- pramiti_export_copilot-0.1.0/tests/test_export_copilot_package.py +296 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Pramiti Labs, Inc.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pramiti-export-copilot
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Standalone semantic model export and delivery — push your data model to dbt, Snowflake, Looker, Tableau, GitHub, GitLab, or dbt Cloud whenever it changes.
|
|
5
|
+
License: MIT
|
|
6
|
+
Project-URL: Homepage, https://getpramiti.com
|
|
7
|
+
Keywords: ai,semantic-model,export,dbt,snowflake,looker,tableau,pramiti
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Topic :: Database
|
|
16
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# pramiti-export-copilot
|
|
25
|
+
|
|
26
|
+
Push your semantic data model to dbt, Snowflake, Databricks, Looker, Tableau, GitHub, GitLab, or dbt Cloud whenever it changes. Zero dependencies -- inject any data backend and delivery target via Protocol-based interfaces.
|
|
27
|
+
|
|
28
|
+
## Install
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pip install pramiti-export-copilot
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Quick Start
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
from pramiti_export_copilot import ExportCopilot, DeliveryConfig
|
|
38
|
+
from pramiti_export_copilot.backends import InMemoryBackend
|
|
39
|
+
from pramiti_export_copilot.delivery_targets import WebhookTarget
|
|
40
|
+
|
|
41
|
+
backend = InMemoryBackend(b"dbt schema content")
|
|
42
|
+
target = WebhookTarget("https://hooks.mycompany.com/epistom")
|
|
43
|
+
copilot = ExportCopilot(backend)
|
|
44
|
+
result = copilot.deliver("workspace-1", DeliveryConfig(format="dbt", target=target))
|
|
45
|
+
|
|
46
|
+
if result.success:
|
|
47
|
+
print(f"Delivered {result.bytes_delivered} bytes, hash={result.export_hash[:8]}...")
|
|
48
|
+
elif result.error:
|
|
49
|
+
print(f"Failed: {result.error}")
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Supported Formats
|
|
53
|
+
|
|
54
|
+
| Format | Backend method |
|
|
55
|
+
|--------|----------------|
|
|
56
|
+
| dbt | `get_dbt_yaml(workspace_id)` |
|
|
57
|
+
| Snowflake | `get_snowflake_sql(workspace_id)` |
|
|
58
|
+
| Databricks | `get_databricks_json(workspace_id)` |
|
|
59
|
+
| Looker | `get_lookml(workspace_id)` |
|
|
60
|
+
| Tableau | `get_tableau_tds(workspace_id, connection_info)` |
|
|
61
|
+
|
|
62
|
+
## Delivery Targets
|
|
63
|
+
|
|
64
|
+
All targets use `urllib.request` (stdlib) -- no external HTTP dependencies. All support self-hosted/enterprise instances via `endpoint_url`.
|
|
65
|
+
|
|
66
|
+
| Target | Description |
|
|
67
|
+
|--------|-------------|
|
|
68
|
+
| `WebhookTarget` | POST to any HTTP endpoint with optional Bearer auth. |
|
|
69
|
+
| `GitHubTarget` | Commit via GitHub Contents API. Supports GHE via custom `endpoint_url`. |
|
|
70
|
+
| `GitLabTarget` | Commit via GitLab Repository Files API. Supports self-hosted instances. |
|
|
71
|
+
| `DbtCloudTarget` | Push via dbt Cloud API. Supports enterprise instances. |
|
|
72
|
+
|
|
73
|
+
## Hash-Based Skip
|
|
74
|
+
|
|
75
|
+
Pass `last_hash` to skip delivery when content hasn't changed:
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
result = copilot.deliver("ws-1", config, last_hash="abc123...")
|
|
79
|
+
if result.skipped:
|
|
80
|
+
print(f"Skipped: {result.reason}") # "no_change"
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## API
|
|
84
|
+
|
|
85
|
+
| Export | Description |
|
|
86
|
+
|--------|-------------|
|
|
87
|
+
| `ExportCopilot` | Core delivery engine. Inject any `IModelDataBackend`. |
|
|
88
|
+
| `DeliveryConfig` | Configuration dataclass: `format`, `target`, optional `connection_info` (for Tableau). |
|
|
89
|
+
| `DeliveryResult` | Result dataclass: `success`, `skipped`, `reason`, `export_hash`, `bytes_delivered`, `error`. |
|
|
90
|
+
| `IModelDataBackend` | Protocol for model data retrieval backends. |
|
|
91
|
+
| `InMemoryBackend` | In-memory backend for testing (returns fixed bytes for all formats). |
|
|
92
|
+
| `IDeliveryTarget` | Protocol for delivery targets. Implement `deliver(content, metadata) -> int`. |
|
|
93
|
+
| `CopilotError` | Base exception for all export copilot errors. |
|
|
94
|
+
| `DeliveryError` | Raised when a delivery target rejects the payload. |
|
|
95
|
+
| `BackendError` | Raised when the backend cannot produce export bytes. |
|
|
96
|
+
|
|
97
|
+
## Custom Backend
|
|
98
|
+
|
|
99
|
+
Implement `IModelDataBackend` (structural subtyping -- no inheritance required):
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
class MyBackend:
|
|
103
|
+
def get_dbt_yaml(self, workspace_id: str) -> bytes: ...
|
|
104
|
+
def get_snowflake_sql(self, workspace_id: str) -> bytes: ...
|
|
105
|
+
def get_databricks_json(self, workspace_id: str) -> bytes: ...
|
|
106
|
+
def get_lookml(self, workspace_id: str) -> bytes: ...
|
|
107
|
+
def get_tableau_tds(self, workspace_id: str, connection_info: dict) -> bytes: ...
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
## License
|
|
111
|
+
|
|
112
|
+
MIT -- Pramiti Labs
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# pramiti-export-copilot
|
|
2
|
+
|
|
3
|
+
Push your semantic data model to dbt, Snowflake, Databricks, Looker, Tableau, GitHub, GitLab, or dbt Cloud whenever it changes. Zero dependencies -- inject any data backend and delivery target via Protocol-based interfaces.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install pramiti-export-copilot
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Quick Start
|
|
12
|
+
|
|
13
|
+
```python
|
|
14
|
+
from pramiti_export_copilot import ExportCopilot, DeliveryConfig
|
|
15
|
+
from pramiti_export_copilot.backends import InMemoryBackend
|
|
16
|
+
from pramiti_export_copilot.delivery_targets import WebhookTarget
|
|
17
|
+
|
|
18
|
+
backend = InMemoryBackend(b"dbt schema content")
|
|
19
|
+
target = WebhookTarget("https://hooks.mycompany.com/epistom")
|
|
20
|
+
copilot = ExportCopilot(backend)
|
|
21
|
+
result = copilot.deliver("workspace-1", DeliveryConfig(format="dbt", target=target))
|
|
22
|
+
|
|
23
|
+
if result.success:
|
|
24
|
+
print(f"Delivered {result.bytes_delivered} bytes, hash={result.export_hash[:8]}...")
|
|
25
|
+
elif result.error:
|
|
26
|
+
print(f"Failed: {result.error}")
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
## Supported Formats
|
|
30
|
+
|
|
31
|
+
| Format | Backend method |
|
|
32
|
+
|--------|----------------|
|
|
33
|
+
| dbt | `get_dbt_yaml(workspace_id)` |
|
|
34
|
+
| Snowflake | `get_snowflake_sql(workspace_id)` |
|
|
35
|
+
| Databricks | `get_databricks_json(workspace_id)` |
|
|
36
|
+
| Looker | `get_lookml(workspace_id)` |
|
|
37
|
+
| Tableau | `get_tableau_tds(workspace_id, connection_info)` |
|
|
38
|
+
|
|
39
|
+
## Delivery Targets
|
|
40
|
+
|
|
41
|
+
All targets use `urllib.request` (stdlib) -- no external HTTP dependencies. All support self-hosted/enterprise instances via `endpoint_url`.
|
|
42
|
+
|
|
43
|
+
| Target | Description |
|
|
44
|
+
|--------|-------------|
|
|
45
|
+
| `WebhookTarget` | POST to any HTTP endpoint with optional Bearer auth. |
|
|
46
|
+
| `GitHubTarget` | Commit via GitHub Contents API. Supports GHE via custom `endpoint_url`. |
|
|
47
|
+
| `GitLabTarget` | Commit via GitLab Repository Files API. Supports self-hosted instances. |
|
|
48
|
+
| `DbtCloudTarget` | Push via dbt Cloud API. Supports enterprise instances. |
|
|
49
|
+
|
|
50
|
+
## Hash-Based Skip
|
|
51
|
+
|
|
52
|
+
Pass `last_hash` to skip delivery when content hasn't changed:
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
result = copilot.deliver("ws-1", config, last_hash="abc123...")
|
|
56
|
+
if result.skipped:
|
|
57
|
+
print(f"Skipped: {result.reason}") # "no_change"
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## API
|
|
61
|
+
|
|
62
|
+
| Export | Description |
|
|
63
|
+
|--------|-------------|
|
|
64
|
+
| `ExportCopilot` | Core delivery engine. Inject any `IModelDataBackend`. |
|
|
65
|
+
| `DeliveryConfig` | Configuration dataclass: `format`, `target`, optional `connection_info` (for Tableau). |
|
|
66
|
+
| `DeliveryResult` | Result dataclass: `success`, `skipped`, `reason`, `export_hash`, `bytes_delivered`, `error`. |
|
|
67
|
+
| `IModelDataBackend` | Protocol for model data retrieval backends. |
|
|
68
|
+
| `InMemoryBackend` | In-memory backend for testing (returns fixed bytes for all formats). |
|
|
69
|
+
| `IDeliveryTarget` | Protocol for delivery targets. Implement `deliver(content, metadata) -> int`. |
|
|
70
|
+
| `CopilotError` | Base exception for all export copilot errors. |
|
|
71
|
+
| `DeliveryError` | Raised when a delivery target rejects the payload. |
|
|
72
|
+
| `BackendError` | Raised when the backend cannot produce export bytes. |
|
|
73
|
+
|
|
74
|
+
## Custom Backend
|
|
75
|
+
|
|
76
|
+
Implement `IModelDataBackend` (structural subtyping -- no inheritance required):
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
class MyBackend:
|
|
80
|
+
def get_dbt_yaml(self, workspace_id: str) -> bytes: ...
|
|
81
|
+
def get_snowflake_sql(self, workspace_id: str) -> bytes: ...
|
|
82
|
+
def get_databricks_json(self, workspace_id: str) -> bytes: ...
|
|
83
|
+
def get_lookml(self, workspace_id: str) -> bytes: ...
|
|
84
|
+
def get_tableau_tds(self, workspace_id: str, connection_info: dict) -> bytes: ...
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## License
|
|
88
|
+
|
|
89
|
+
MIT -- Pramiti Labs
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "pramiti-export-copilot"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Standalone semantic model export and delivery — push your data model to dbt, Snowflake, Looker, Tableau, GitHub, GitLab, or dbt Cloud whenever it changes."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.9"
|
|
7
|
+
dependencies = []
|
|
8
|
+
classifiers = [
|
|
9
|
+
"Development Status :: 4 - Beta",
|
|
10
|
+
"Intended Audience :: Developers",
|
|
11
|
+
"License :: OSI Approved :: MIT License",
|
|
12
|
+
"Programming Language :: Python :: 3.9",
|
|
13
|
+
"Programming Language :: Python :: 3.10",
|
|
14
|
+
"Programming Language :: Python :: 3.11",
|
|
15
|
+
"Programming Language :: Python :: 3.12",
|
|
16
|
+
"Topic :: Database",
|
|
17
|
+
"Topic :: Software Development :: Libraries",
|
|
18
|
+
]
|
|
19
|
+
keywords = ["ai", "semantic-model", "export", "dbt", "snowflake", "looker", "tableau", "pramiti"]
|
|
20
|
+
license = {text = "MIT"}
|
|
21
|
+
|
|
22
|
+
[project.optional-dependencies]
|
|
23
|
+
dev = ["pytest>=8.0"]
|
|
24
|
+
|
|
25
|
+
[project.urls]
|
|
26
|
+
Homepage = "https://getpramiti.com"
|
|
27
|
+
|
|
28
|
+
[build-system]
|
|
29
|
+
requires = ["setuptools>=68", "wheel"]
|
|
30
|
+
build-backend = "setuptools.build_meta"
|
|
31
|
+
|
|
32
|
+
[tool.setuptools.packages.find]
|
|
33
|
+
where = ["src"]
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""Pramiti Export Copilot — standalone semantic model export and delivery.
|
|
2
|
+
|
|
3
|
+
Install: pip install pramiti-export-copilot
|
|
4
|
+
Docs: https://getpramiti.com/docs/export-copilot
|
|
5
|
+
"""
|
|
6
|
+
from pramiti_export_copilot.backends import IModelDataBackend, InMemoryBackend
|
|
7
|
+
from pramiti_export_copilot.copilot import ExportCopilot
|
|
8
|
+
from pramiti_export_copilot.delivery_targets import (
|
|
9
|
+
DbtCloudTarget,
|
|
10
|
+
GitHubTarget,
|
|
11
|
+
GitLabTarget,
|
|
12
|
+
IDeliveryTarget,
|
|
13
|
+
WebhookTarget,
|
|
14
|
+
)
|
|
15
|
+
from pramiti_export_copilot.exceptions import BackendError, CopilotError, DeliveryError
|
|
16
|
+
from pramiti_export_copilot.models import DeliveryConfig, DeliveryResult
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"ExportCopilot", "DeliveryConfig", "DeliveryResult",
|
|
20
|
+
"IModelDataBackend", "InMemoryBackend",
|
|
21
|
+
"IDeliveryTarget", "WebhookTarget", "GitHubTarget", "GitLabTarget", "DbtCloudTarget",
|
|
22
|
+
"CopilotError", "DeliveryError", "BackendError",
|
|
23
|
+
]
|
|
24
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""IModelDataBackend protocol — implement this to connect the copilot to any data store.
|
|
2
|
+
|
|
3
|
+
Structural subtyping via Protocol: no ABC inheritance required.
|
|
4
|
+
Implement the methods and pass your instance to ExportCopilot.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from typing import Protocol, runtime_checkable
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@runtime_checkable
|
|
12
|
+
class IModelDataBackend(Protocol):
|
|
13
|
+
"""Read the semantic model and return export bytes in the requested format."""
|
|
14
|
+
|
|
15
|
+
def get_dbt_yaml(self, workspace_id: str) -> bytes:
|
|
16
|
+
"""Return a ZIP archive of dbt YAML schema files."""
|
|
17
|
+
...
|
|
18
|
+
|
|
19
|
+
def get_snowflake_sql(self, workspace_id: str) -> bytes:
|
|
20
|
+
"""Return Snowflake tag-assignment SQL as UTF-8 bytes."""
|
|
21
|
+
...
|
|
22
|
+
|
|
23
|
+
def get_databricks_json(self, workspace_id: str) -> bytes:
|
|
24
|
+
"""Return Unity Catalog JSON as UTF-8 bytes."""
|
|
25
|
+
...
|
|
26
|
+
|
|
27
|
+
def get_lookml(self, workspace_id: str) -> bytes:
|
|
28
|
+
"""Return LookML view file as UTF-8 bytes."""
|
|
29
|
+
...
|
|
30
|
+
|
|
31
|
+
def get_tableau_tds(self, workspace_id: str, connection_info: dict) -> bytes:
|
|
32
|
+
"""Return Tableau TDS XML as UTF-8 bytes.
|
|
33
|
+
|
|
34
|
+
connection_info must contain: server, port, dbname, username.
|
|
35
|
+
Raises BackendError if connection_info is incomplete.
|
|
36
|
+
"""
|
|
37
|
+
...
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class InMemoryBackend:
|
|
41
|
+
"""Test/demo backend — returns fixed bytes for every format.
|
|
42
|
+
|
|
43
|
+
Usage:
|
|
44
|
+
backend = InMemoryBackend(content=b"dbt: schema")
|
|
45
|
+
copilot = ExportCopilot(backend)
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
def __init__(self, content: bytes = b"test export content") -> None:
|
|
49
|
+
self._content = content
|
|
50
|
+
|
|
51
|
+
def get_dbt_yaml(self, workspace_id: str) -> bytes:
|
|
52
|
+
return self._content
|
|
53
|
+
|
|
54
|
+
def get_snowflake_sql(self, workspace_id: str) -> bytes:
|
|
55
|
+
return self._content
|
|
56
|
+
|
|
57
|
+
def get_databricks_json(self, workspace_id: str) -> bytes:
|
|
58
|
+
return self._content
|
|
59
|
+
|
|
60
|
+
def get_lookml(self, workspace_id: str) -> bytes:
|
|
61
|
+
return self._content
|
|
62
|
+
|
|
63
|
+
def get_tableau_tds(self, workspace_id: str, connection_info: dict) -> bytes:
|
|
64
|
+
if not connection_info.get("server"):
|
|
65
|
+
from pramiti_export_copilot.exceptions import BackendError
|
|
66
|
+
raise BackendError("Tableau TDS requires 'server' in connection_info.")
|
|
67
|
+
return self._content
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""ExportCopilot — the core delivery engine.
|
|
2
|
+
|
|
3
|
+
Zero dependencies. Inject any IModelDataBackend and IDeliveryTarget.
|
|
4
|
+
"""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import hashlib
|
|
8
|
+
import re
|
|
9
|
+
from datetime import datetime, timezone
|
|
10
|
+
|
|
11
|
+
from pramiti_export_copilot.backends import IModelDataBackend
|
|
12
|
+
from pramiti_export_copilot.exceptions import BackendError, DeliveryError
|
|
13
|
+
from pramiti_export_copilot.models import DeliveryConfig, DeliveryResult
|
|
14
|
+
|
|
15
|
+
# M-3: prefer structlog for structured observability inside Epistom; fall back
|
|
16
|
+
# to stdlib logging so the package stays zero-hard-dependency when sold standalone.
|
|
17
|
+
try:
|
|
18
|
+
import structlog
|
|
19
|
+
logger = structlog.get_logger(__name__)
|
|
20
|
+
except ImportError: # pragma: no cover - standalone install without structlog
|
|
21
|
+
import logging
|
|
22
|
+
logger = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
_FORMAT_TO_METHOD = {
|
|
25
|
+
"dbt": "get_dbt_yaml",
|
|
26
|
+
"snowflake": "get_snowflake_sql",
|
|
27
|
+
"databricks": "get_databricks_json",
|
|
28
|
+
"looker": "get_lookml",
|
|
29
|
+
"tableau": "get_tableau_tds",
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
# PK-2: delivery errors carry the target URL, which may embed a token in the
|
|
33
|
+
# query string ("?token=...") or in credentials before an "@" — either as URL
|
|
34
|
+
# userinfo ("//user:pass@host") or in a parser-error fragment that has already
|
|
35
|
+
# dropped the scheme (e.g. urllib's "nonnumeric port: 'SECRET@host'"). Redact
|
|
36
|
+
# both before any error string reaches a caller OR a log.
|
|
37
|
+
_QUERY_SECRET_RE = re.compile(r"([?&][^=&\s]+=)([^&\s]+)")
|
|
38
|
+
# Any run of non-space/quote/slash chars up to (greedily) the LAST "@" — so it
|
|
39
|
+
# is scheme-independent AND a multi-"@" password can't leak a tail. Aggressive
|
|
40
|
+
# by design: over-redacting an "x@host" fragment in an error string is safe.
|
|
41
|
+
_CREDS_AT_RE = re.compile(r"[^\s'\"/]+@")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _redact_secrets(msg: str) -> str:
|
|
45
|
+
msg = _QUERY_SECRET_RE.sub(r"\1***", msg)
|
|
46
|
+
msg = _CREDS_AT_RE.sub("***@", msg)
|
|
47
|
+
return msg
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class ExportCopilot:
|
|
51
|
+
"""Generate and deliver semantic model exports.
|
|
52
|
+
|
|
53
|
+
Usage (standalone):
|
|
54
|
+
from pramiti_export_copilot import ExportCopilot, DeliveryConfig
|
|
55
|
+
from pramiti_export_copilot.backends import InMemoryBackend
|
|
56
|
+
from pramiti_export_copilot.delivery_targets import WebhookTarget
|
|
57
|
+
|
|
58
|
+
backend = InMemoryBackend(b"dbt schema content")
|
|
59
|
+
target = WebhookTarget("https://hooks.mycompany.com/epistom")
|
|
60
|
+
copilot = ExportCopilot(backend)
|
|
61
|
+
result = copilot.deliver("workspace-1", DeliveryConfig(format="dbt", target=target))
|
|
62
|
+
|
|
63
|
+
Usage (Epistom integration):
|
|
64
|
+
backend = KSDataBackend(session)
|
|
65
|
+
copilot = ExportCopilot(backend)
|
|
66
|
+
result = copilot.deliver(str(workspace_id), config)
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
def __init__(self, backend: IModelDataBackend) -> None:
|
|
70
|
+
self._backend = backend
|
|
71
|
+
|
|
72
|
+
def deliver(
|
|
73
|
+
self,
|
|
74
|
+
workspace_id: str,
|
|
75
|
+
config: DeliveryConfig,
|
|
76
|
+
last_hash: str | None = None,
|
|
77
|
+
) -> DeliveryResult:
|
|
78
|
+
"""Generate export bytes, check hash, deliver to target.
|
|
79
|
+
|
|
80
|
+
Returns DeliveryResult — never raises. All errors are in result.error.
|
|
81
|
+
"""
|
|
82
|
+
try:
|
|
83
|
+
method_name = _FORMAT_TO_METHOD.get(config.format)
|
|
84
|
+
if method_name is None:
|
|
85
|
+
return DeliveryResult(error=f"Unsupported format: {config.format!r}")
|
|
86
|
+
|
|
87
|
+
if config.format == "tableau":
|
|
88
|
+
content = self._backend.get_tableau_tds(workspace_id, config.connection_info)
|
|
89
|
+
else:
|
|
90
|
+
content = getattr(self._backend, method_name)(workspace_id)
|
|
91
|
+
|
|
92
|
+
except BackendError as exc:
|
|
93
|
+
safe_err = _redact_secrets(str(exc))
|
|
94
|
+
logger.error("export_copilot_backend_error format=%s error=%s", config.format, safe_err)
|
|
95
|
+
return DeliveryResult(error=safe_err)
|
|
96
|
+
except Exception as exc:
|
|
97
|
+
safe_err = _redact_secrets(str(exc))
|
|
98
|
+
logger.error("export_copilot_backend_unexpected format=%s error=%s", config.format, safe_err)
|
|
99
|
+
return DeliveryResult(error=f"Backend error: {safe_err}")
|
|
100
|
+
|
|
101
|
+
export_hash = hashlib.sha256(content).hexdigest()
|
|
102
|
+
|
|
103
|
+
if last_hash and last_hash == export_hash:
|
|
104
|
+
return DeliveryResult(skipped=True, reason="no_change", export_hash=export_hash)
|
|
105
|
+
|
|
106
|
+
metadata = {
|
|
107
|
+
"format": config.format,
|
|
108
|
+
"workspace_id": workspace_id,
|
|
109
|
+
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
try:
|
|
113
|
+
bytes_delivered = config.target.deliver(content, metadata)
|
|
114
|
+
except DeliveryError as exc:
|
|
115
|
+
# PK-2: redact URL secrets before the error reaches EITHER the log
|
|
116
|
+
# OR the caller (DeliveryResult.error). Previously the caller-facing
|
|
117
|
+
# value was the raw str(exc), leaking tokens despite the redacted log.
|
|
118
|
+
safe_err = _redact_secrets(str(exc))
|
|
119
|
+
logger.error("export_copilot_delivery_error format=%s error=%s", config.format, safe_err)
|
|
120
|
+
return DeliveryResult(error=safe_err)
|
|
121
|
+
except Exception as exc:
|
|
122
|
+
safe_err = _redact_secrets(str(exc))
|
|
123
|
+
logger.error("export_copilot_delivery_unexpected format=%s error=%s", config.format, safe_err)
|
|
124
|
+
return DeliveryResult(error=f"Delivery error: {safe_err}")
|
|
125
|
+
|
|
126
|
+
logger.info(
|
|
127
|
+
"export_copilot_delivered format=%s bytes=%d hash=%.8s",
|
|
128
|
+
config.format,
|
|
129
|
+
bytes_delivered,
|
|
130
|
+
export_hash,
|
|
131
|
+
)
|
|
132
|
+
return DeliveryResult(
|
|
133
|
+
success=True,
|
|
134
|
+
export_hash=export_hash,
|
|
135
|
+
bytes_delivered=bytes_delivered,
|
|
136
|
+
)
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
"""IDeliveryTarget protocol and four production adapters.
|
|
2
|
+
|
|
3
|
+
All adapters use urllib.request (stdlib) — no external HTTP dependencies.
|
|
4
|
+
All adapters respect endpoint_url for self-hosted/enterprise instances.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import base64
|
|
9
|
+
import json
|
|
10
|
+
import urllib.parse
|
|
11
|
+
import urllib.request
|
|
12
|
+
from typing import Protocol, runtime_checkable
|
|
13
|
+
|
|
14
|
+
from pramiti_export_copilot.exceptions import DeliveryError
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@runtime_checkable
|
|
18
|
+
class IDeliveryTarget(Protocol):
|
|
19
|
+
def deliver(self, content: bytes, metadata: dict) -> int:
|
|
20
|
+
"""Send content to the target. Returns bytes delivered. Raises DeliveryError on failure."""
|
|
21
|
+
...
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class WebhookTarget:
|
|
25
|
+
"""POST to an HTTP endpoint.
|
|
26
|
+
|
|
27
|
+
auth_config keys: authorization (optional Bearer token string)
|
|
28
|
+
endpoint_url: full URL, e.g. "https://hooks.company.com/epistom"
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
def __init__(self, endpoint_url: str, authorization: str = "") -> None:
|
|
32
|
+
self._url = endpoint_url
|
|
33
|
+
self._auth = authorization
|
|
34
|
+
|
|
35
|
+
def deliver(self, content: bytes, metadata: dict) -> int:
|
|
36
|
+
payload = json.dumps({
|
|
37
|
+
"format": metadata.get("format"),
|
|
38
|
+
"content_b64": base64.b64encode(content).decode(),
|
|
39
|
+
"workspace_id": metadata.get("workspace_id"),
|
|
40
|
+
"timestamp": metadata.get("timestamp"),
|
|
41
|
+
}).encode()
|
|
42
|
+
headers = {"Content-Type": "application/json"}
|
|
43
|
+
if self._auth:
|
|
44
|
+
headers["Authorization"] = self._auth
|
|
45
|
+
try:
|
|
46
|
+
req = urllib.request.Request(self._url, data=payload, method="POST", headers=headers)
|
|
47
|
+
with urllib.request.urlopen(req, timeout=30) as r:
|
|
48
|
+
r.read()
|
|
49
|
+
return len(payload)
|
|
50
|
+
except Exception as exc:
|
|
51
|
+
raise DeliveryError("webhook", str(exc)) from exc
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class GitHubTarget:
|
|
55
|
+
"""PUT to GitHub Contents API. Supports GitHub Enterprise via endpoint_url.
|
|
56
|
+
|
|
57
|
+
auth_config keys: token, owner, repo, path, branch (default: main)
|
|
58
|
+
endpoint_url: API base, e.g. "https://api.github.com" or
|
|
59
|
+
"https://github.mycompany.com/api/v3" for GHE
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
def __init__(self, endpoint_url: str, token: str, owner: str, repo: str,
|
|
63
|
+
path: str, branch: str = "main") -> None:
|
|
64
|
+
base = endpoint_url.rstrip("/") if endpoint_url else "https://api.github.com"
|
|
65
|
+
self._api_url = f"{base}/repos/{owner}/{repo}/contents/{path}"
|
|
66
|
+
self._token = token
|
|
67
|
+
self._branch = branch
|
|
68
|
+
|
|
69
|
+
def deliver(self, content: bytes, metadata: dict) -> int:
|
|
70
|
+
headers = {
|
|
71
|
+
"Authorization": f"Bearer {self._token}",
|
|
72
|
+
"Accept": "application/vnd.github+json",
|
|
73
|
+
"Content-Type": "application/json",
|
|
74
|
+
}
|
|
75
|
+
# Try to get current file SHA for updates
|
|
76
|
+
sha = None
|
|
77
|
+
try:
|
|
78
|
+
# PK-3: scope the SHA lookup to the target branch — without ?ref the
|
|
79
|
+
# Contents API returns the DEFAULT branch's SHA, so update-vs-create
|
|
80
|
+
# misbehaved on any non-default branch.
|
|
81
|
+
get_url = f"{self._api_url}?ref={urllib.parse.quote(self._branch)}"
|
|
82
|
+
get_req = urllib.request.Request(get_url, method="GET", headers=headers)
|
|
83
|
+
with urllib.request.urlopen(get_req, timeout=15) as r:
|
|
84
|
+
sha = json.loads(r.read()).get("sha")
|
|
85
|
+
except Exception:
|
|
86
|
+
pass
|
|
87
|
+
|
|
88
|
+
body: dict = {
|
|
89
|
+
"message": f"[Epistom] Auto-export {metadata.get('format')} model update",
|
|
90
|
+
"content": base64.b64encode(content).decode(),
|
|
91
|
+
"branch": self._branch,
|
|
92
|
+
}
|
|
93
|
+
if sha:
|
|
94
|
+
body["sha"] = sha
|
|
95
|
+
try:
|
|
96
|
+
put_req = urllib.request.Request(
|
|
97
|
+
self._api_url,
|
|
98
|
+
data=json.dumps(body).encode(),
|
|
99
|
+
method="PUT",
|
|
100
|
+
headers=headers,
|
|
101
|
+
)
|
|
102
|
+
with urllib.request.urlopen(put_req, timeout=30) as r:
|
|
103
|
+
r.read()
|
|
104
|
+
return len(content)
|
|
105
|
+
except Exception as exc:
|
|
106
|
+
raise DeliveryError("github", str(exc)) from exc
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class GitLabTarget:
|
|
110
|
+
"""GitLab Repository Files API. Supports self-hosted GitLab via endpoint_url.
|
|
111
|
+
|
|
112
|
+
auth_config keys: token, project_id, file_path, branch (default: main)
|
|
113
|
+
endpoint_url: API base, e.g. "https://gitlab.com" or "https://gl.mycompany.com"
|
|
114
|
+
"""
|
|
115
|
+
|
|
116
|
+
def __init__(self, endpoint_url: str, token: str, project_id: str,
|
|
117
|
+
file_path: str, branch: str = "main") -> None:
|
|
118
|
+
base = endpoint_url.rstrip("/") if endpoint_url else "https://gitlab.com"
|
|
119
|
+
encoded = urllib.parse.quote(file_path, safe="")
|
|
120
|
+
self._api_url = f"{base}/api/v4/projects/{project_id}/repository/files/{encoded}"
|
|
121
|
+
self._token = token
|
|
122
|
+
self._branch = branch
|
|
123
|
+
|
|
124
|
+
def deliver(self, content: bytes, metadata: dict) -> int:
|
|
125
|
+
# Determine create vs update
|
|
126
|
+
method = "PUT"
|
|
127
|
+
try:
|
|
128
|
+
get_req = urllib.request.Request(
|
|
129
|
+
f"{self._api_url}?ref={self._branch}",
|
|
130
|
+
method="GET",
|
|
131
|
+
headers={"PRIVATE-TOKEN": self._token},
|
|
132
|
+
)
|
|
133
|
+
with urllib.request.urlopen(get_req, timeout=15):
|
|
134
|
+
pass
|
|
135
|
+
except Exception:
|
|
136
|
+
method = "POST"
|
|
137
|
+
|
|
138
|
+
body = json.dumps({
|
|
139
|
+
"branch": self._branch,
|
|
140
|
+
"content": base64.b64encode(content).decode(),
|
|
141
|
+
"encoding": "base64",
|
|
142
|
+
"commit_message": f"[Epistom] Auto-export {metadata.get('format')} model update",
|
|
143
|
+
}).encode()
|
|
144
|
+
try:
|
|
145
|
+
req = urllib.request.Request(
|
|
146
|
+
self._api_url, data=body, method=method,
|
|
147
|
+
headers={"PRIVATE-TOKEN": self._token, "Content-Type": "application/json"},
|
|
148
|
+
)
|
|
149
|
+
with urllib.request.urlopen(req, timeout=30) as r:
|
|
150
|
+
r.read()
|
|
151
|
+
return len(content)
|
|
152
|
+
except Exception as exc:
|
|
153
|
+
raise DeliveryError("gitlab", str(exc)) from exc
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
class DbtCloudTarget:
|
|
157
|
+
"""dbt Cloud API. Supports Enterprise self-hosted instances via endpoint_url.
|
|
158
|
+
|
|
159
|
+
auth_config keys: service_token, account_id
|
|
160
|
+
endpoint_url: API base, e.g. "https://cloud.getdbt.com" or enterprise URL
|
|
161
|
+
"""
|
|
162
|
+
|
|
163
|
+
def __init__(self, endpoint_url: str, service_token: str, account_id: str) -> None:
|
|
164
|
+
base = endpoint_url.rstrip("/") if endpoint_url else "https://cloud.getdbt.com"
|
|
165
|
+
self._api_url = f"{base}/api/v2/accounts/{account_id}/export/"
|
|
166
|
+
self._token = service_token
|
|
167
|
+
|
|
168
|
+
def deliver(self, content: bytes, metadata: dict) -> int:
|
|
169
|
+
body = json.dumps({
|
|
170
|
+
"format": metadata.get("format"),
|
|
171
|
+
"content_b64": base64.b64encode(content).decode(),
|
|
172
|
+
}).encode()
|
|
173
|
+
try:
|
|
174
|
+
req = urllib.request.Request(
|
|
175
|
+
self._api_url, data=body, method="POST",
|
|
176
|
+
headers={"Authorization": f"Token {self._token}", "Content-Type": "application/json"},
|
|
177
|
+
)
|
|
178
|
+
with urllib.request.urlopen(req, timeout=30) as r:
|
|
179
|
+
r.read()
|
|
180
|
+
return len(body)
|
|
181
|
+
except Exception as exc:
|
|
182
|
+
raise DeliveryError("dbt_cloud", str(exc)) from exc
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""Generic exceptions — no web framework dependency."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class CopilotError(Exception):
|
|
5
|
+
"""Base exception for all export copilot errors."""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class DeliveryError(CopilotError):
|
|
9
|
+
"""Raised when a downstream delivery target rejects the payload."""
|
|
10
|
+
|
|
11
|
+
def __init__(self, target_type: str, reason: str):
|
|
12
|
+
self.target_type = target_type
|
|
13
|
+
self.reason = reason
|
|
14
|
+
super().__init__(f"Delivery to {target_type} failed: {reason}")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class BackendError(CopilotError):
|
|
18
|
+
"""Raised when the model data backend cannot produce export bytes."""
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Pure dataclass models — no ORM, no web framework."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass
|
|
8
|
+
class DeliveryResult:
|
|
9
|
+
success: bool = False
|
|
10
|
+
skipped: bool = False
|
|
11
|
+
reason: str = ""
|
|
12
|
+
export_hash: str = ""
|
|
13
|
+
bytes_delivered: int = 0
|
|
14
|
+
error: str = ""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class DeliveryConfig:
|
|
19
|
+
format: str # "dbt" | "snowflake" | "databricks" | "looker" | "tableau"
|
|
20
|
+
target: object # Any IDeliveryTarget-compatible object
|
|
21
|
+
connection_info: dict = field(default_factory=dict) # Required for tableau
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pramiti-export-copilot
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Standalone semantic model export and delivery — push your data model to dbt, Snowflake, Looker, Tableau, GitHub, GitLab, or dbt Cloud whenever it changes.
|
|
5
|
+
License: MIT
|
|
6
|
+
Project-URL: Homepage, https://getpramiti.com
|
|
7
|
+
Keywords: ai,semantic-model,export,dbt,snowflake,looker,tableau,pramiti
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Topic :: Database
|
|
16
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# pramiti-export-copilot
|
|
25
|
+
|
|
26
|
+
Push your semantic data model to dbt, Snowflake, Databricks, Looker, Tableau, GitHub, GitLab, or dbt Cloud whenever it changes. Zero dependencies -- inject any data backend and delivery target via Protocol-based interfaces.
|
|
27
|
+
|
|
28
|
+
## Install
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pip install pramiti-export-copilot
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Quick Start
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
from pramiti_export_copilot import ExportCopilot, DeliveryConfig
|
|
38
|
+
from pramiti_export_copilot.backends import InMemoryBackend
|
|
39
|
+
from pramiti_export_copilot.delivery_targets import WebhookTarget
|
|
40
|
+
|
|
41
|
+
backend = InMemoryBackend(b"dbt schema content")
|
|
42
|
+
target = WebhookTarget("https://hooks.mycompany.com/epistom")
|
|
43
|
+
copilot = ExportCopilot(backend)
|
|
44
|
+
result = copilot.deliver("workspace-1", DeliveryConfig(format="dbt", target=target))
|
|
45
|
+
|
|
46
|
+
if result.success:
|
|
47
|
+
print(f"Delivered {result.bytes_delivered} bytes, hash={result.export_hash[:8]}...")
|
|
48
|
+
elif result.error:
|
|
49
|
+
print(f"Failed: {result.error}")
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Supported Formats
|
|
53
|
+
|
|
54
|
+
| Format | Backend method |
|
|
55
|
+
|--------|----------------|
|
|
56
|
+
| dbt | `get_dbt_yaml(workspace_id)` |
|
|
57
|
+
| Snowflake | `get_snowflake_sql(workspace_id)` |
|
|
58
|
+
| Databricks | `get_databricks_json(workspace_id)` |
|
|
59
|
+
| Looker | `get_lookml(workspace_id)` |
|
|
60
|
+
| Tableau | `get_tableau_tds(workspace_id, connection_info)` |
|
|
61
|
+
|
|
62
|
+
## Delivery Targets
|
|
63
|
+
|
|
64
|
+
All targets use `urllib.request` (stdlib) -- no external HTTP dependencies. All support self-hosted/enterprise instances via `endpoint_url`.
|
|
65
|
+
|
|
66
|
+
| Target | Description |
|
|
67
|
+
|--------|-------------|
|
|
68
|
+
| `WebhookTarget` | POST to any HTTP endpoint with optional Bearer auth. |
|
|
69
|
+
| `GitHubTarget` | Commit via GitHub Contents API. Supports GHE via custom `endpoint_url`. |
|
|
70
|
+
| `GitLabTarget` | Commit via GitLab Repository Files API. Supports self-hosted instances. |
|
|
71
|
+
| `DbtCloudTarget` | Push via dbt Cloud API. Supports enterprise instances. |
|
|
72
|
+
|
|
73
|
+
## Hash-Based Skip
|
|
74
|
+
|
|
75
|
+
Pass `last_hash` to skip delivery when content hasn't changed:
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
result = copilot.deliver("ws-1", config, last_hash="abc123...")
|
|
79
|
+
if result.skipped:
|
|
80
|
+
print(f"Skipped: {result.reason}") # "no_change"
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## API
|
|
84
|
+
|
|
85
|
+
| Export | Description |
|
|
86
|
+
|--------|-------------|
|
|
87
|
+
| `ExportCopilot` | Core delivery engine. Inject any `IModelDataBackend`. |
|
|
88
|
+
| `DeliveryConfig` | Configuration dataclass: `format`, `target`, optional `connection_info` (for Tableau). |
|
|
89
|
+
| `DeliveryResult` | Result dataclass: `success`, `skipped`, `reason`, `export_hash`, `bytes_delivered`, `error`. |
|
|
90
|
+
| `IModelDataBackend` | Protocol for model data retrieval backends. |
|
|
91
|
+
| `InMemoryBackend` | In-memory backend for testing (returns fixed bytes for all formats). |
|
|
92
|
+
| `IDeliveryTarget` | Protocol for delivery targets. Implement `deliver(content, metadata) -> int`. |
|
|
93
|
+
| `CopilotError` | Base exception for all export copilot errors. |
|
|
94
|
+
| `DeliveryError` | Raised when a delivery target rejects the payload. |
|
|
95
|
+
| `BackendError` | Raised when the backend cannot produce export bytes. |
|
|
96
|
+
|
|
97
|
+
## Custom Backend
|
|
98
|
+
|
|
99
|
+
Implement `IModelDataBackend` (structural subtyping -- no inheritance required):
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
class MyBackend:
|
|
103
|
+
def get_dbt_yaml(self, workspace_id: str) -> bytes: ...
|
|
104
|
+
def get_snowflake_sql(self, workspace_id: str) -> bytes: ...
|
|
105
|
+
def get_databricks_json(self, workspace_id: str) -> bytes: ...
|
|
106
|
+
def get_lookml(self, workspace_id: str) -> bytes: ...
|
|
107
|
+
def get_tableau_tds(self, workspace_id: str, connection_info: dict) -> bytes: ...
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
## License
|
|
111
|
+
|
|
112
|
+
MIT -- Pramiti Labs
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/pramiti_export_copilot/__init__.py
|
|
5
|
+
src/pramiti_export_copilot/backends.py
|
|
6
|
+
src/pramiti_export_copilot/copilot.py
|
|
7
|
+
src/pramiti_export_copilot/delivery_targets.py
|
|
8
|
+
src/pramiti_export_copilot/exceptions.py
|
|
9
|
+
src/pramiti_export_copilot/models.py
|
|
10
|
+
src/pramiti_export_copilot.egg-info/PKG-INFO
|
|
11
|
+
src/pramiti_export_copilot.egg-info/SOURCES.txt
|
|
12
|
+
src/pramiti_export_copilot.egg-info/dependency_links.txt
|
|
13
|
+
src/pramiti_export_copilot.egg-info/requires.txt
|
|
14
|
+
src/pramiti_export_copilot.egg-info/top_level.txt
|
|
15
|
+
tests/test_export_copilot_package.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
pramiti_export_copilot
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
"""Tests for pramiti-export-copilot standalone package."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import sys
|
|
5
|
+
from unittest.mock import MagicMock, patch
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def test_no_epistom_import():
|
|
11
|
+
"""Package must not import epistom.* in a clean process."""
|
|
12
|
+
import subprocess
|
|
13
|
+
script = (
|
|
14
|
+
"import pramiti_export_copilot;"
|
|
15
|
+
"bad=[k for k in __import__('sys').modules if k.startswith('epistom')];"
|
|
16
|
+
"assert bad==[], f'epistom modules: {bad}'"
|
|
17
|
+
)
|
|
18
|
+
result = subprocess.run(
|
|
19
|
+
[sys.executable, "-c", script],
|
|
20
|
+
capture_output=True,
|
|
21
|
+
text=True,
|
|
22
|
+
env={**__import__("os").environ, "PYTHONPATH": "packages/export-copilot/src"},
|
|
23
|
+
)
|
|
24
|
+
assert result.returncode == 0, f"stdout: {result.stdout}\nstderr: {result.stderr}"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_no_fastapi_import():
|
|
28
|
+
"""Package must not import fastapi.* in a clean process."""
|
|
29
|
+
import subprocess
|
|
30
|
+
script = (
|
|
31
|
+
"import pramiti_export_copilot;"
|
|
32
|
+
"bad=[k for k in __import__('sys').modules if k.startswith('fastapi')];"
|
|
33
|
+
"assert bad==[], f'fastapi modules: {bad}'"
|
|
34
|
+
)
|
|
35
|
+
result = subprocess.run(
|
|
36
|
+
[sys.executable, "-c", script],
|
|
37
|
+
capture_output=True,
|
|
38
|
+
text=True,
|
|
39
|
+
env={**__import__("os").environ, "PYTHONPATH": "packages/export-copilot/src"},
|
|
40
|
+
)
|
|
41
|
+
assert result.returncode == 0, f"stdout: {result.stdout}\nstderr: {result.stderr}"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_inmemory_backend_all_formats():
|
|
45
|
+
"""InMemoryBackend returns fixed bytes for all supported formats."""
|
|
46
|
+
from pramiti_export_copilot.backends import InMemoryBackend
|
|
47
|
+
|
|
48
|
+
content = b"test model content"
|
|
49
|
+
backend = InMemoryBackend(content=content)
|
|
50
|
+
|
|
51
|
+
assert backend.get_dbt_yaml("ws-1") == content
|
|
52
|
+
assert backend.get_snowflake_sql("ws-1") == content
|
|
53
|
+
assert backend.get_databricks_json("ws-1") == content
|
|
54
|
+
assert backend.get_lookml("ws-1") == content
|
|
55
|
+
assert backend.get_tableau_tds("ws-1", {"server": "db.example.com"}) == content
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def test_webhook_target_delivers_bytes():
|
|
59
|
+
"""WebhookTarget.deliver() POSTs content and returns bytes delivered."""
|
|
60
|
+
from pramiti_export_copilot.delivery_targets import WebhookTarget
|
|
61
|
+
|
|
62
|
+
target = WebhookTarget("https://hooks.example.com/epistom")
|
|
63
|
+
fake_response = MagicMock()
|
|
64
|
+
fake_response.read.return_value = b""
|
|
65
|
+
fake_response.__enter__ = MagicMock(return_value=fake_response)
|
|
66
|
+
fake_response.__exit__ = MagicMock(return_value=False)
|
|
67
|
+
|
|
68
|
+
with patch("urllib.request.urlopen", return_value=fake_response):
|
|
69
|
+
n = target.deliver(b"payload", {"format": "dbt", "workspace_id": "ws-1", "timestamp": "2026-01-01T00:00:00Z"})
|
|
70
|
+
|
|
71
|
+
assert n > 0
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def test_github_target_uses_endpoint_url():
|
|
75
|
+
"""GitHubTarget builds API URL from the provided endpoint_url (GHE support)."""
|
|
76
|
+
from pramiti_export_copilot.delivery_targets import GitHubTarget
|
|
77
|
+
|
|
78
|
+
target = GitHubTarget(
|
|
79
|
+
endpoint_url="https://github.mycompany.com/api/v3",
|
|
80
|
+
token="tok",
|
|
81
|
+
owner="org",
|
|
82
|
+
repo="repo",
|
|
83
|
+
path="models/schema.yml",
|
|
84
|
+
)
|
|
85
|
+
assert "github.mycompany.com/api/v3" in target._api_url
|
|
86
|
+
assert "org/repo/contents/models/schema.yml" in target._api_url
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def test_gitlab_target_uses_endpoint_url():
|
|
90
|
+
"""GitLabTarget builds API URL from the provided endpoint_url (self-hosted support)."""
|
|
91
|
+
from pramiti_export_copilot.delivery_targets import GitLabTarget
|
|
92
|
+
|
|
93
|
+
target = GitLabTarget(
|
|
94
|
+
endpoint_url="https://gl.mycompany.com",
|
|
95
|
+
token="tok",
|
|
96
|
+
project_id="123",
|
|
97
|
+
file_path="schema.yml",
|
|
98
|
+
)
|
|
99
|
+
assert "gl.mycompany.com" in target._api_url
|
|
100
|
+
assert "123" in target._api_url
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def test_dbt_cloud_target_uses_endpoint_url():
|
|
104
|
+
"""DbtCloudTarget builds API URL from the provided endpoint_url (enterprise support)."""
|
|
105
|
+
from pramiti_export_copilot.delivery_targets import DbtCloudTarget
|
|
106
|
+
|
|
107
|
+
target = DbtCloudTarget(
|
|
108
|
+
endpoint_url="https://dbt.enterprise.com",
|
|
109
|
+
service_token="tok",
|
|
110
|
+
account_id="999",
|
|
111
|
+
)
|
|
112
|
+
assert "dbt.enterprise.com" in target._api_url
|
|
113
|
+
assert "999" in target._api_url
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def test_tableau_requires_server_in_connection_info():
|
|
117
|
+
"""InMemoryBackend.get_tableau_tds raises BackendError when server missing."""
|
|
118
|
+
from pramiti_export_copilot.backends import InMemoryBackend
|
|
119
|
+
from pramiti_export_copilot.exceptions import BackendError
|
|
120
|
+
|
|
121
|
+
backend = InMemoryBackend()
|
|
122
|
+
with pytest.raises(BackendError):
|
|
123
|
+
backend.get_tableau_tds("ws-1", {}) # no server key
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def test_hash_unchanged_returns_skipped():
|
|
127
|
+
"""ExportCopilot.deliver() skips delivery when content hash matches last_hash."""
|
|
128
|
+
import hashlib
|
|
129
|
+
|
|
130
|
+
from pramiti_export_copilot import DeliveryConfig, ExportCopilot
|
|
131
|
+
from pramiti_export_copilot.backends import InMemoryBackend
|
|
132
|
+
|
|
133
|
+
content = b"static model content"
|
|
134
|
+
last_hash = hashlib.sha256(content).hexdigest()
|
|
135
|
+
|
|
136
|
+
backend = InMemoryBackend(content=content)
|
|
137
|
+
mock_target = MagicMock()
|
|
138
|
+
config = DeliveryConfig(format="dbt", target=mock_target)
|
|
139
|
+
|
|
140
|
+
copilot = ExportCopilot(backend)
|
|
141
|
+
result = copilot.deliver("ws-1", config, last_hash=last_hash)
|
|
142
|
+
|
|
143
|
+
assert result.skipped is True
|
|
144
|
+
assert result.reason == "no_change"
|
|
145
|
+
assert result.export_hash == last_hash
|
|
146
|
+
mock_target.deliver.assert_not_called()
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def test_hash_changed_triggers_delivery():
|
|
150
|
+
"""ExportCopilot.deliver() calls target.deliver() when content changes."""
|
|
151
|
+
from pramiti_export_copilot import DeliveryConfig, ExportCopilot
|
|
152
|
+
from pramiti_export_copilot.backends import InMemoryBackend
|
|
153
|
+
|
|
154
|
+
backend = InMemoryBackend(b"new content")
|
|
155
|
+
mock_target = MagicMock()
|
|
156
|
+
mock_target.deliver.return_value = 11
|
|
157
|
+
config = DeliveryConfig(format="dbt", target=mock_target)
|
|
158
|
+
|
|
159
|
+
copilot = ExportCopilot(backend)
|
|
160
|
+
result = copilot.deliver("ws-1", config, last_hash="old_hash_that_differs")
|
|
161
|
+
|
|
162
|
+
assert result.success is True
|
|
163
|
+
assert result.bytes_delivered == 11
|
|
164
|
+
mock_target.deliver.assert_called_once()
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def test_delivery_error_returns_result_not_raises():
|
|
168
|
+
"""ExportCopilot.deliver() catches DeliveryError and returns DeliveryResult with error."""
|
|
169
|
+
from pramiti_export_copilot import DeliveryConfig, ExportCopilot
|
|
170
|
+
from pramiti_export_copilot.backends import InMemoryBackend
|
|
171
|
+
from pramiti_export_copilot.exceptions import DeliveryError
|
|
172
|
+
|
|
173
|
+
backend = InMemoryBackend()
|
|
174
|
+
mock_target = MagicMock()
|
|
175
|
+
mock_target.deliver.side_effect = DeliveryError("webhook", "connection refused")
|
|
176
|
+
config = DeliveryConfig(format="dbt", target=mock_target)
|
|
177
|
+
|
|
178
|
+
copilot = ExportCopilot(backend)
|
|
179
|
+
result = copilot.deliver("ws-1", config)
|
|
180
|
+
|
|
181
|
+
assert result.success is False
|
|
182
|
+
assert result.error != ""
|
|
183
|
+
assert "webhook" in result.error
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def test_backend_error_returns_result_not_raises():
|
|
187
|
+
"""ExportCopilot.deliver() catches BackendError and returns DeliveryResult with error."""
|
|
188
|
+
from pramiti_export_copilot import DeliveryConfig, ExportCopilot
|
|
189
|
+
from pramiti_export_copilot.exceptions import BackendError
|
|
190
|
+
|
|
191
|
+
class FailingBackend:
|
|
192
|
+
def get_dbt_yaml(self, workspace_id):
|
|
193
|
+
raise BackendError("cannot read KS data")
|
|
194
|
+
|
|
195
|
+
def get_snowflake_sql(self, workspace_id): ...
|
|
196
|
+
def get_databricks_json(self, workspace_id): ...
|
|
197
|
+
def get_lookml(self, workspace_id): ...
|
|
198
|
+
def get_tableau_tds(self, workspace_id, connection_info): ...
|
|
199
|
+
|
|
200
|
+
mock_target = MagicMock()
|
|
201
|
+
config = DeliveryConfig(format="dbt", target=mock_target)
|
|
202
|
+
|
|
203
|
+
copilot = ExportCopilot(FailingBackend())
|
|
204
|
+
result = copilot.deliver("ws-1", config)
|
|
205
|
+
|
|
206
|
+
assert result.success is False
|
|
207
|
+
assert "cannot read KS data" in result.error
|
|
208
|
+
mock_target.deliver.assert_not_called()
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def test_unsupported_format_returns_error():
|
|
212
|
+
"""ExportCopilot.deliver() returns error for unknown format, never raises."""
|
|
213
|
+
from pramiti_export_copilot import DeliveryConfig, ExportCopilot
|
|
214
|
+
from pramiti_export_copilot.backends import InMemoryBackend
|
|
215
|
+
|
|
216
|
+
backend = InMemoryBackend()
|
|
217
|
+
mock_target = MagicMock()
|
|
218
|
+
config = DeliveryConfig(format="unknown_format", target=mock_target)
|
|
219
|
+
|
|
220
|
+
copilot = ExportCopilot(backend)
|
|
221
|
+
result = copilot.deliver("ws-1", config)
|
|
222
|
+
|
|
223
|
+
assert result.success is False
|
|
224
|
+
assert "unknown_format" in result.error
|
|
225
|
+
mock_target.deliver.assert_not_called()
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
# ── PK-2: delivery/backend errors must not leak URL secrets to callers ───────
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def test_redact_secrets_helper():
|
|
232
|
+
from pramiti_export_copilot.copilot import _redact_secrets
|
|
233
|
+
|
|
234
|
+
assert _redact_secrets("POST https://h/x?token=SECRET failed") == "POST https://h/x?token=*** failed"
|
|
235
|
+
assert _redact_secrets("https://user:pw@git/repo") == "https://***@git/repo"
|
|
236
|
+
out = _redact_secrets("https://api/x?access_token=ghp_ABC&ref=main")
|
|
237
|
+
assert "ghp_ABC" not in out
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def test_redact_secrets_userinfo_edge_cases():
|
|
241
|
+
"""PK-2 pass-2: userinfo must be redacted independent of scheme, including
|
|
242
|
+
the urllib port-parse-error fragment and multi-'@' passwords."""
|
|
243
|
+
from pramiti_export_copilot.copilot import _redact_secrets
|
|
244
|
+
|
|
245
|
+
# urllib "nonnumeric port: 'SECRET@host'" — no scheme, no query
|
|
246
|
+
assert "PWSECRET" not in _redact_secrets("nonnumeric port: 'PWSECRET@h'")
|
|
247
|
+
# multi-'@' password must not leak a tail
|
|
248
|
+
assert "ss" not in _redact_secrets("https://user:p@ss@host").split("@")[0]
|
|
249
|
+
assert _redact_secrets("https://user:p@ss@host") == "https://***@host"
|
|
250
|
+
# protocol-relative (no scheme)
|
|
251
|
+
assert "pass" not in _redact_secrets("//user:pass@host")
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def test_redact_secrets_redos_safe():
|
|
255
|
+
import time
|
|
256
|
+
|
|
257
|
+
from pramiti_export_copilot.copilot import _redact_secrets
|
|
258
|
+
|
|
259
|
+
t = time.time()
|
|
260
|
+
_redact_secrets("a" * 500000 + "@h")
|
|
261
|
+
assert time.time() - t < 1.0
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _copilot_with_failing_target(exc):
|
|
265
|
+
from pramiti_export_copilot.backends import InMemoryBackend
|
|
266
|
+
from pramiti_export_copilot.copilot import ExportCopilot
|
|
267
|
+
from pramiti_export_copilot.models import DeliveryConfig
|
|
268
|
+
|
|
269
|
+
class _Target:
|
|
270
|
+
def deliver(self, content, metadata):
|
|
271
|
+
raise exc
|
|
272
|
+
|
|
273
|
+
cop = ExportCopilot(InMemoryBackend())
|
|
274
|
+
config = DeliveryConfig(format="dbt", target=_Target())
|
|
275
|
+
return cop.deliver("ws-1", config)
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def test_delivery_error_result_is_redacted():
|
|
279
|
+
"""DeliveryResult.error (caller-facing) must be redacted, not raw str(exc)."""
|
|
280
|
+
from pramiti_export_copilot.exceptions import DeliveryError
|
|
281
|
+
|
|
282
|
+
result = _copilot_with_failing_target(
|
|
283
|
+
DeliveryError("webhook", "POST https://hooks.example.com/x?token=SECRET123 -> 500")
|
|
284
|
+
)
|
|
285
|
+
assert result.error is not None
|
|
286
|
+
assert "SECRET123" not in result.error
|
|
287
|
+
assert "token=***" in result.error
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def test_unexpected_delivery_error_result_is_redacted():
|
|
291
|
+
result = _copilot_with_failing_target(
|
|
292
|
+
RuntimeError("connect https://user:pw@internal.example/x failed")
|
|
293
|
+
)
|
|
294
|
+
assert result.error is not None
|
|
295
|
+
assert "user:pw@" not in result.error
|
|
296
|
+
assert "pw" not in result.error.split("@")[0].split("//")[-1]
|