definite-sdk 0.1.4__py3-none-any.whl → 0.1.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- definite_sdk/__init__.py +20 -0
- definite_sdk/client.py +39 -1
- definite_sdk/dlt.py +237 -0
- definite_sdk/integration.py +44 -5
- definite_sdk/py.typed +0 -0
- definite_sdk/secret.py +3 -3
- definite_sdk/sql.py +126 -0
- definite_sdk/store.py +18 -2
- definite_sdk-0.1.8.dist-info/METADATA +270 -0
- definite_sdk-0.1.8.dist-info/RECORD +12 -0
- {definite_sdk-0.1.4.dist-info → definite_sdk-0.1.8.dist-info}/WHEEL +1 -1
- definite_sdk-0.1.4.dist-info/METADATA +0 -21
- definite_sdk-0.1.4.dist-info/RECORD +0 -8
- {definite_sdk-0.1.4.dist-info → definite_sdk-0.1.8.dist-info}/LICENSE +0 -0
definite_sdk/__init__.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Definite SDK for Python
|
|
3
|
+
|
|
4
|
+
A Python library for interacting with the Definite API and tools.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from definite_sdk.client import DefiniteClient
|
|
8
|
+
from definite_sdk.integration import DefiniteIntegrationStore
|
|
9
|
+
from definite_sdk.secret import DefiniteSecretStore
|
|
10
|
+
from definite_sdk.sql import DefiniteSqlClient
|
|
11
|
+
from definite_sdk.store import DefiniteKVStore
|
|
12
|
+
|
|
13
|
+
__version__ = "0.1.8"
|
|
14
|
+
__all__ = [
|
|
15
|
+
"DefiniteClient",
|
|
16
|
+
"DefiniteIntegrationStore",
|
|
17
|
+
"DefiniteSecretStore",
|
|
18
|
+
"DefiniteSqlClient",
|
|
19
|
+
"DefiniteKVStore",
|
|
20
|
+
]
|
definite_sdk/client.py
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
1
4
|
from definite_sdk.integration import DefiniteIntegrationStore
|
|
2
5
|
from definite_sdk.secret import DefiniteSecretStore
|
|
6
|
+
from definite_sdk.sql import DefiniteSqlClient
|
|
3
7
|
from definite_sdk.store import DefiniteKVStore
|
|
4
8
|
|
|
5
9
|
API_URL = "https://api.definite.app"
|
|
@@ -8,11 +12,24 @@ API_URL = "https://api.definite.app"
|
|
|
8
12
|
class DefiniteClient:
|
|
9
13
|
"""Client for interacting with the Definite API."""
|
|
10
14
|
|
|
11
|
-
def __init__(self, api_key: str, api_url: str = API_URL):
|
|
15
|
+
def __init__(self, api_key: Optional[str] = None, api_url: str = API_URL):
|
|
12
16
|
"""Creates a definite client with the provided API key.
|
|
13
17
|
|
|
18
|
+
Args:
|
|
19
|
+
api_key: API key for authentication. If not provided, will look for
|
|
20
|
+
DEFINITE_API_KEY or DEF_API_KEY environment variables.
|
|
21
|
+
api_url: Base URL for the Definite API.
|
|
22
|
+
|
|
14
23
|
See: https://docs.definite.app/definite-api for how to obtain an API key.
|
|
15
24
|
"""
|
|
25
|
+
if api_key is None:
|
|
26
|
+
api_key = os.getenv("DEFINITE_API_KEY") or os.getenv("DEF_API_KEY")
|
|
27
|
+
if not api_key:
|
|
28
|
+
raise ValueError(
|
|
29
|
+
"API key must be provided or set in DEFINITE_API_KEY "
|
|
30
|
+
"or DEF_API_KEY environment variable"
|
|
31
|
+
)
|
|
32
|
+
|
|
16
33
|
self.api_key = api_key
|
|
17
34
|
self.api_url = api_url
|
|
18
35
|
|
|
@@ -39,3 +56,24 @@ class DefiniteClient:
|
|
|
39
56
|
"""
|
|
40
57
|
|
|
41
58
|
return DefiniteIntegrationStore(self.api_key, self.api_url)
|
|
59
|
+
|
|
60
|
+
def get_sql_client(self) -> DefiniteSqlClient:
|
|
61
|
+
"""Initializes the SQL client for executing SQL queries.
|
|
62
|
+
|
|
63
|
+
See DefiniteSqlClient for more how to execute SQL queries.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
return DefiniteSqlClient(self.api_key, self.api_url)
|
|
67
|
+
|
|
68
|
+
# Alias methods for consistency
|
|
69
|
+
def kv_store(self, name: str) -> DefiniteKVStore:
|
|
70
|
+
"""Alias for get_kv_store."""
|
|
71
|
+
return self.get_kv_store(name)
|
|
72
|
+
|
|
73
|
+
def secret_store(self) -> DefiniteSecretStore:
|
|
74
|
+
"""Alias for get_secret_store."""
|
|
75
|
+
return self.get_secret_store()
|
|
76
|
+
|
|
77
|
+
def integration_store(self) -> DefiniteIntegrationStore:
|
|
78
|
+
"""Alias for get_integration_store."""
|
|
79
|
+
return self.get_integration_store()
|
definite_sdk/dlt.py
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
"""DLT (Data Load Tool) integration for Definite SDK."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import json
|
|
5
|
+
from typing import Any, Dict, Optional, Tuple, TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
import dlt
|
|
9
|
+
from dlt.common import Destination
|
|
10
|
+
from dlt.pipeline import Pipeline
|
|
11
|
+
else:
|
|
12
|
+
try:
|
|
13
|
+
import dlt
|
|
14
|
+
from dlt.common import Destination
|
|
15
|
+
from dlt.pipeline import Pipeline
|
|
16
|
+
except ImportError:
|
|
17
|
+
dlt = None # type: ignore
|
|
18
|
+
Destination = None # type: ignore
|
|
19
|
+
Pipeline = None # type: ignore
|
|
20
|
+
|
|
21
|
+
from .client import DefiniteClient
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class DefiniteDLTPipeline:
|
|
25
|
+
"""Wrapper for DLT pipelines with Definite state management."""
|
|
26
|
+
|
|
27
|
+
def __init__(
|
|
28
|
+
self,
|
|
29
|
+
name: str,
|
|
30
|
+
dataset_name: Optional[str] = None,
|
|
31
|
+
destination: Optional["Destination"] = None,
|
|
32
|
+
**kwargs: Any,
|
|
33
|
+
):
|
|
34
|
+
"""Initialize a DLT pipeline with Definite integration.
|
|
35
|
+
|
|
36
|
+
Args:
|
|
37
|
+
name: Pipeline name
|
|
38
|
+
dataset_name: Dataset name for the destination
|
|
39
|
+
destination: DLT destination (defaults to duckdb)
|
|
40
|
+
**kwargs: Additional arguments passed to dlt.pipeline()
|
|
41
|
+
"""
|
|
42
|
+
if dlt is None:
|
|
43
|
+
raise ImportError(
|
|
44
|
+
"dlt package not installed. Install with: pip install definite-sdk[dlt]"
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
self.name = name
|
|
48
|
+
self._client = DefiniteClient()
|
|
49
|
+
self._state_store = self._client.kv_store(f"dlt_state_{name}")
|
|
50
|
+
|
|
51
|
+
# Default to DuckDB if no destination specified
|
|
52
|
+
if destination is None:
|
|
53
|
+
destination = "duckdb"
|
|
54
|
+
|
|
55
|
+
# Use dataset_name or default to pipeline name
|
|
56
|
+
if dataset_name is None:
|
|
57
|
+
dataset_name = name
|
|
58
|
+
|
|
59
|
+
# Create the underlying DLT pipeline
|
|
60
|
+
self._pipeline = dlt.pipeline(
|
|
61
|
+
pipeline_name=name,
|
|
62
|
+
destination=destination,
|
|
63
|
+
dataset_name=dataset_name,
|
|
64
|
+
**kwargs,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
@property
|
|
68
|
+
def pipeline(self) -> "Pipeline":
|
|
69
|
+
"""Get the underlying DLT pipeline."""
|
|
70
|
+
return self._pipeline
|
|
71
|
+
|
|
72
|
+
@property
|
|
73
|
+
def state(self) -> Dict[str, Any]:
|
|
74
|
+
"""Get the current pipeline state."""
|
|
75
|
+
return dict(self._pipeline.state)
|
|
76
|
+
|
|
77
|
+
def run(
|
|
78
|
+
self,
|
|
79
|
+
data: Any,
|
|
80
|
+
*,
|
|
81
|
+
table_name: Optional[str] = None,
|
|
82
|
+
write_disposition: Optional[str] = None,
|
|
83
|
+
**kwargs: Any,
|
|
84
|
+
) -> Any:
|
|
85
|
+
"""Run the pipeline with the given data.
|
|
86
|
+
|
|
87
|
+
Args:
|
|
88
|
+
data: Data to load (resource, source, or iterator)
|
|
89
|
+
table_name: Name of the table to load data into
|
|
90
|
+
write_disposition: How to write data (append, replace, merge)
|
|
91
|
+
**kwargs: Additional arguments passed to pipeline.run()
|
|
92
|
+
|
|
93
|
+
Returns:
|
|
94
|
+
Load info from the pipeline run
|
|
95
|
+
"""
|
|
96
|
+
# Run the pipeline
|
|
97
|
+
load_info = self._pipeline.run(
|
|
98
|
+
data, table_name=table_name, write_disposition=write_disposition, **kwargs
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
# Persist state to Definite after successful run
|
|
102
|
+
self._persist_state()
|
|
103
|
+
|
|
104
|
+
return load_info
|
|
105
|
+
|
|
106
|
+
def _persist_state(self) -> None:
|
|
107
|
+
"""Persist current pipeline state to Definite."""
|
|
108
|
+
state = self._pipeline.state
|
|
109
|
+
if state:
|
|
110
|
+
# Store each state key separately for easier access
|
|
111
|
+
for key, value in state.items():
|
|
112
|
+
# Serialize value to JSON string
|
|
113
|
+
self._state_store[key] = json.dumps(value)
|
|
114
|
+
self._state_store.commit()
|
|
115
|
+
|
|
116
|
+
def get_state(self, key: Optional[str] = None) -> Any:
|
|
117
|
+
"""Retrieve state from Definite store.
|
|
118
|
+
|
|
119
|
+
Args:
|
|
120
|
+
key: Specific state key to retrieve. If None, returns all state.
|
|
121
|
+
|
|
122
|
+
Returns:
|
|
123
|
+
State value or entire state dict
|
|
124
|
+
"""
|
|
125
|
+
if key is not None:
|
|
126
|
+
value = self._state_store.get(key, None)
|
|
127
|
+
return json.loads(value) if value else None
|
|
128
|
+
|
|
129
|
+
# Return all state as dict
|
|
130
|
+
return {k: json.loads(self._state_store[k]) for k in self._state_store}
|
|
131
|
+
|
|
132
|
+
def set_state(self, key: str, value: Any) -> None:
|
|
133
|
+
"""Set a state value.
|
|
134
|
+
|
|
135
|
+
Args:
|
|
136
|
+
key: State key
|
|
137
|
+
value: State value
|
|
138
|
+
"""
|
|
139
|
+
self._pipeline.state[key] = value
|
|
140
|
+
# Also persist to Definite immediately
|
|
141
|
+
self._state_store[key] = json.dumps(value)
|
|
142
|
+
self._state_store.commit()
|
|
143
|
+
|
|
144
|
+
def resume_from_state(self) -> None:
|
|
145
|
+
"""Resume pipeline from previously stored state."""
|
|
146
|
+
# Load state from Definite
|
|
147
|
+
stored_state: Dict[str, Any] = {
|
|
148
|
+
k: json.loads(self._state_store[k]) for k in self._state_store
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
# Apply state to pipeline
|
|
152
|
+
for key, value in stored_state.items():
|
|
153
|
+
self._pipeline.state[key] = value
|
|
154
|
+
|
|
155
|
+
def reset_state(self) -> None:
|
|
156
|
+
"""Reset pipeline state."""
|
|
157
|
+
# Clear pipeline state
|
|
158
|
+
self._pipeline.state.clear()
|
|
159
|
+
|
|
160
|
+
# Clear Definite state
|
|
161
|
+
keys_to_delete = list(self._state_store)
|
|
162
|
+
for key in keys_to_delete:
|
|
163
|
+
del self._state_store[key]
|
|
164
|
+
self._state_store.commit()
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
class DLTStateAdapter:
|
|
168
|
+
"""Adapter for DLT state management conforming to Definite patterns."""
|
|
169
|
+
|
|
170
|
+
def __init__(self, pipeline_name: str):
|
|
171
|
+
"""Initialize state adapter.
|
|
172
|
+
|
|
173
|
+
Args:
|
|
174
|
+
pipeline_name: Name of the DLT pipeline
|
|
175
|
+
"""
|
|
176
|
+
self.pipeline_name = pipeline_name
|
|
177
|
+
self._client = DefiniteClient()
|
|
178
|
+
self._store = self._client.kv_store(f"dlt_state_{pipeline_name}")
|
|
179
|
+
|
|
180
|
+
def save_state(self, state: Dict[str, Any]) -> None:
|
|
181
|
+
"""Save pipeline state.
|
|
182
|
+
|
|
183
|
+
Args:
|
|
184
|
+
state: State dictionary to save
|
|
185
|
+
"""
|
|
186
|
+
for key, value in state.items():
|
|
187
|
+
self._store[key] = json.dumps(value)
|
|
188
|
+
self._store.commit()
|
|
189
|
+
|
|
190
|
+
def load_state(self) -> Dict[str, Any]:
|
|
191
|
+
"""Load pipeline state.
|
|
192
|
+
|
|
193
|
+
Returns:
|
|
194
|
+
State dictionary
|
|
195
|
+
"""
|
|
196
|
+
return {k: json.loads(self._store[k]) for k in self._store}
|
|
197
|
+
|
|
198
|
+
def clear_state(self) -> None:
|
|
199
|
+
"""Clear all state."""
|
|
200
|
+
keys_to_delete = list(self._store)
|
|
201
|
+
for key in keys_to_delete:
|
|
202
|
+
del self._store[key]
|
|
203
|
+
self._store.commit()
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def get_duckdb_connection() -> Optional[Tuple[str, Any]]:
|
|
207
|
+
"""Get DuckDB connection from Definite integration.
|
|
208
|
+
|
|
209
|
+
Uses the DEFINITE_API_KEY environment variable to authenticate and
|
|
210
|
+
lookup the team's DuckDB integration.
|
|
211
|
+
|
|
212
|
+
Returns:
|
|
213
|
+
Optional[Tuple[str, Any]]: Tuple of (integration_id, connection) if found,
|
|
214
|
+
None if no DuckDB integration or API key.
|
|
215
|
+
"""
|
|
216
|
+
api_key = os.getenv("DEFINITE_API_KEY")
|
|
217
|
+
if not api_key:
|
|
218
|
+
return None
|
|
219
|
+
|
|
220
|
+
try:
|
|
221
|
+
import duckdb
|
|
222
|
+
except ImportError:
|
|
223
|
+
raise ImportError(
|
|
224
|
+
"duckdb package not installed. Install with: pip install duckdb"
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
client = DefiniteClient(api_key=api_key)
|
|
228
|
+
integration_store = client.integration_store()
|
|
229
|
+
|
|
230
|
+
result = integration_store.lookup_duckdb_integration()
|
|
231
|
+
if result:
|
|
232
|
+
integration_id, connection_uri = result
|
|
233
|
+
# Create DuckDB connection
|
|
234
|
+
connection = duckdb.connect(connection_uri)
|
|
235
|
+
return (integration_id, connection)
|
|
236
|
+
|
|
237
|
+
return None
|
definite_sdk/integration.py
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
from typing import Iterator
|
|
1
|
+
from typing import Iterator, Optional, Tuple
|
|
2
2
|
|
|
3
3
|
import requests
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
INTEGRATION_ENDPOINT = "/v1/api/integration"
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
class DefiniteIntegrationStore:
|
|
@@ -26,8 +26,8 @@ class DefiniteIntegrationStore:
|
|
|
26
26
|
api_key (str): The API key for authorization.
|
|
27
27
|
"""
|
|
28
28
|
self._api_key = api_key
|
|
29
|
-
self._integration_store_url = api_url +
|
|
30
|
-
|
|
29
|
+
self._integration_store_url = api_url + INTEGRATION_ENDPOINT
|
|
30
|
+
|
|
31
31
|
def list_integrations(self) -> Iterator[dict]:
|
|
32
32
|
"""
|
|
33
33
|
Lists all integrations in the store.
|
|
@@ -57,4 +57,43 @@ class DefiniteIntegrationStore:
|
|
|
57
57
|
headers={"Authorization": "Bearer " + self._api_key},
|
|
58
58
|
)
|
|
59
59
|
response.raise_for_status()
|
|
60
|
-
return response.json()
|
|
60
|
+
return dict(response.json())
|
|
61
|
+
|
|
62
|
+
def lookup_duckdb_integration(self) -> Optional[Tuple[str, str]]:
|
|
63
|
+
"""
|
|
64
|
+
Look up the team's DuckDB integration.
|
|
65
|
+
|
|
66
|
+
Note: Currently, the API only returns extractor (source) integrations.
|
|
67
|
+
Destination integrations like DuckDB are not yet exposed through this endpoint.
|
|
68
|
+
This method is provided for future compatibility when the API is updated.
|
|
69
|
+
|
|
70
|
+
Returns:
|
|
71
|
+
Optional[Tuple[str, str]]: Tuple of (integration_id, connection_uri)
|
|
72
|
+
if found, None if no DuckDB integration exists.
|
|
73
|
+
"""
|
|
74
|
+
try:
|
|
75
|
+
response = requests.get(
|
|
76
|
+
self._integration_store_url,
|
|
77
|
+
headers={"Authorization": "Bearer " + self._api_key},
|
|
78
|
+
)
|
|
79
|
+
response.raise_for_status()
|
|
80
|
+
integrations = response.json()
|
|
81
|
+
except:
|
|
82
|
+
return None
|
|
83
|
+
|
|
84
|
+
# Check for DuckDB in the integrations
|
|
85
|
+
# Note: Currently this will not find DuckDB as it's a destination integration
|
|
86
|
+
for integration in integrations:
|
|
87
|
+
integration_type = integration.get("integration_type", "").lower()
|
|
88
|
+
if integration_type == "duckdb" and integration.get("active", True):
|
|
89
|
+
integration_id = integration.get("id")
|
|
90
|
+
# Connection URI might be in config or connection_string field
|
|
91
|
+
connection_uri = (
|
|
92
|
+
integration.get("connection_string")
|
|
93
|
+
or integration.get("config", {}).get("database_path")
|
|
94
|
+
or integration.get("config", {}).get("connection_string")
|
|
95
|
+
)
|
|
96
|
+
if integration_id and connection_uri:
|
|
97
|
+
return (integration_id, connection_uri)
|
|
98
|
+
|
|
99
|
+
return None
|
definite_sdk/py.typed
ADDED
|
File without changes
|
definite_sdk/secret.py
CHANGED
|
@@ -47,7 +47,7 @@ class DefiniteSecretStore:
|
|
|
47
47
|
)
|
|
48
48
|
response.raise_for_status()
|
|
49
49
|
return iter(response.json()["secrets"])
|
|
50
|
-
|
|
50
|
+
|
|
51
51
|
def get_secret(self, key: str) -> str:
|
|
52
52
|
"""
|
|
53
53
|
Retrieves the value of a secret.
|
|
@@ -63,8 +63,8 @@ class DefiniteSecretStore:
|
|
|
63
63
|
headers={"Authorization": "Bearer " + self._api_key},
|
|
64
64
|
)
|
|
65
65
|
response.raise_for_status()
|
|
66
|
-
return response.json()["value"]
|
|
67
|
-
|
|
66
|
+
return str(response.json()["value"])
|
|
67
|
+
|
|
68
68
|
def set_secret(self, key: str, value: str):
|
|
69
69
|
"""
|
|
70
70
|
Sets the value of a secret.
|
definite_sdk/sql.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
from typing import Any, Dict, List, Optional, Union
|
|
2
|
+
|
|
3
|
+
import requests
|
|
4
|
+
|
|
5
|
+
SQL_ENDPOINT = "/v1/query"
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class DefiniteSqlClient:
|
|
9
|
+
"""
|
|
10
|
+
A SQL client for executing SQL queries via the Definite API.
|
|
11
|
+
|
|
12
|
+
Initialization:
|
|
13
|
+
>>> client = DefiniteClient("MY_API_KEY")
|
|
14
|
+
>>> sql_client = client.get_sql_client()
|
|
15
|
+
|
|
16
|
+
Executing SQL queries:
|
|
17
|
+
>>> result = sql_client.execute("SELECT * FROM my_table LIMIT 10")
|
|
18
|
+
>>> print(result)
|
|
19
|
+
|
|
20
|
+
Executing SQL queries with integration ID:
|
|
21
|
+
>>> result = sql_client.execute(
|
|
22
|
+
... "SELECT * FROM my_table LIMIT 10",
|
|
23
|
+
... integration_id="my_integration_id"
|
|
24
|
+
... )
|
|
25
|
+
>>> print(result)
|
|
26
|
+
|
|
27
|
+
Executing Cube queries:
|
|
28
|
+
>>> cube_query = {
|
|
29
|
+
... "dimensions": [],
|
|
30
|
+
... "measures": ["sales.total_amount"],
|
|
31
|
+
... "timeDimensions": [{"dimension": "sales.date", "granularity": "month"}],
|
|
32
|
+
... "limit": 1000
|
|
33
|
+
... }
|
|
34
|
+
>>> result = sql_client.execute_cube_query(cube_query, integration_id="my_cube_integration")
|
|
35
|
+
>>> print(result)
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
def __init__(self, api_key: str, api_url: str):
|
|
39
|
+
"""
|
|
40
|
+
Initializes the DefiniteSqlClient.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
api_key (str): The API key for authorization.
|
|
44
|
+
api_url (str): The base URL for the Definite API.
|
|
45
|
+
"""
|
|
46
|
+
self._api_key = api_key
|
|
47
|
+
self._sql_url = api_url + SQL_ENDPOINT
|
|
48
|
+
|
|
49
|
+
def execute(
|
|
50
|
+
self,
|
|
51
|
+
sql: str,
|
|
52
|
+
integration_id: Optional[str] = None
|
|
53
|
+
) -> Dict[str, Any]:
|
|
54
|
+
"""
|
|
55
|
+
Executes a SQL query against a database integration.
|
|
56
|
+
|
|
57
|
+
Args:
|
|
58
|
+
sql (str): The SQL query to execute.
|
|
59
|
+
integration_id (Optional[str]): The integration ID to query against.
|
|
60
|
+
If not provided, the default integration will be used.
|
|
61
|
+
|
|
62
|
+
Returns:
|
|
63
|
+
Dict[str, Any]: The query result as returned by the API.
|
|
64
|
+
|
|
65
|
+
Raises:
|
|
66
|
+
requests.HTTPError: If the API request fails.
|
|
67
|
+
|
|
68
|
+
Example:
|
|
69
|
+
>>> result = sql_client.execute("SELECT COUNT(*) FROM users")
|
|
70
|
+
>>> print(result)
|
|
71
|
+
"""
|
|
72
|
+
payload = {"sql": sql}
|
|
73
|
+
if integration_id:
|
|
74
|
+
payload["integration_id"] = integration_id
|
|
75
|
+
|
|
76
|
+
response = requests.post(
|
|
77
|
+
self._sql_url,
|
|
78
|
+
json=payload,
|
|
79
|
+
headers={"Authorization": "Bearer " + self._api_key},
|
|
80
|
+
)
|
|
81
|
+
response.raise_for_status()
|
|
82
|
+
return response.json()
|
|
83
|
+
|
|
84
|
+
def execute_cube_query(
|
|
85
|
+
self,
|
|
86
|
+
cube_query: Dict[str, Any],
|
|
87
|
+
integration_id: Optional[str] = None
|
|
88
|
+
) -> Dict[str, Any]:
|
|
89
|
+
"""
|
|
90
|
+
Executes a Cube query against a Cube integration.
|
|
91
|
+
|
|
92
|
+
Args:
|
|
93
|
+
cube_query (Dict[str, Any]): The Cube query in JSON format.
|
|
94
|
+
integration_id (Optional[str]): The Cube integration ID to query against.
|
|
95
|
+
If not provided, the default integration will be used.
|
|
96
|
+
|
|
97
|
+
Returns:
|
|
98
|
+
Dict[str, Any]: The query result as returned by the API.
|
|
99
|
+
|
|
100
|
+
Raises:
|
|
101
|
+
requests.HTTPError: If the API request fails.
|
|
102
|
+
|
|
103
|
+
Example:
|
|
104
|
+
>>> cube_query = {
|
|
105
|
+
... "dimensions": [],
|
|
106
|
+
... "measures": ["sales.total_amount"],
|
|
107
|
+
... "timeDimensions": [{
|
|
108
|
+
... "dimension": "sales.date",
|
|
109
|
+
... "granularity": "month"
|
|
110
|
+
... }],
|
|
111
|
+
... "limit": 1000
|
|
112
|
+
... }
|
|
113
|
+
>>> result = sql_client.execute_cube_query(cube_query, "my_cube_integration")
|
|
114
|
+
>>> print(result)
|
|
115
|
+
"""
|
|
116
|
+
payload = {"cube_query": cube_query}
|
|
117
|
+
if integration_id:
|
|
118
|
+
payload["integration_id"] = integration_id
|
|
119
|
+
|
|
120
|
+
response = requests.post(
|
|
121
|
+
self._sql_url,
|
|
122
|
+
json=payload,
|
|
123
|
+
headers={"Authorization": "Bearer " + self._api_key},
|
|
124
|
+
)
|
|
125
|
+
response.raise_for_status()
|
|
126
|
+
return response.json()
|
definite_sdk/store.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Iterator
|
|
1
|
+
from typing import Iterator, Union, Optional
|
|
2
2
|
|
|
3
3
|
import requests
|
|
4
4
|
|
|
@@ -109,7 +109,7 @@ class DefiniteKVStore:
|
|
|
109
109
|
self._data = {}
|
|
110
110
|
self._version_id = None
|
|
111
111
|
|
|
112
|
-
def __getitem__(self, key: str) -> str
|
|
112
|
+
def __getitem__(self, key: str) -> Optional[str]:
|
|
113
113
|
"""
|
|
114
114
|
Gets the value for a given key.
|
|
115
115
|
|
|
@@ -190,3 +190,19 @@ class DefiniteKVStore:
|
|
|
190
190
|
print(store)
|
|
191
191
|
"""
|
|
192
192
|
return repr(self._data)
|
|
193
|
+
|
|
194
|
+
def get(self, key: str, default: Optional[str] = None) -> Optional[str]:
|
|
195
|
+
"""
|
|
196
|
+
Gets the value for a given key with a default.
|
|
197
|
+
|
|
198
|
+
Args:
|
|
199
|
+
key (str): The key to retrieve the value for.
|
|
200
|
+
default: The default value to return if key doesn't exist.
|
|
201
|
+
|
|
202
|
+
Returns:
|
|
203
|
+
The value associated with the key, or default if the key does not exist.
|
|
204
|
+
|
|
205
|
+
Example:
|
|
206
|
+
value = store.get("key1", "default_value")
|
|
207
|
+
"""
|
|
208
|
+
return self._data.get(key, default)
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: definite-sdk
|
|
3
|
+
Version: 0.1.8
|
|
4
|
+
Summary: Definite SDK for Python
|
|
5
|
+
License: MIT
|
|
6
|
+
Author: Definite
|
|
7
|
+
Author-email: hello@definite.app
|
|
8
|
+
Requires-Python: >=3.9,<4.0
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Provides-Extra: dlt
|
|
17
|
+
Requires-Dist: dlt (>=1.0,<2.0) ; extra == "dlt"
|
|
18
|
+
Requires-Dist: duckdb (>=1.0,<2.0) ; extra == "dlt"
|
|
19
|
+
Requires-Dist: requests (>=2.31.0,<3.0.0)
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
|
|
22
|
+
# Definite SDK
|
|
23
|
+
|
|
24
|
+
A Python client for interacting with the Definite API, providing a convenient interface for key-value store operations, SQL query execution, secrets management, and DLT (Data Load Tool) integration with state persistence.
|
|
25
|
+
|
|
26
|
+
## Installation
|
|
27
|
+
|
|
28
|
+
**pip:**
|
|
29
|
+
```bash
|
|
30
|
+
pip install definite-sdk
|
|
31
|
+
|
|
32
|
+
# For dlt support
|
|
33
|
+
pip install "definite-sdk[dlt]"
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
**poetry:**
|
|
37
|
+
```bash
|
|
38
|
+
poetry add definite-sdk
|
|
39
|
+
|
|
40
|
+
# For dlt support
|
|
41
|
+
poetry add "definite-sdk[dlt]"
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## Quick Start
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
from definite_sdk import DefiniteClient
|
|
48
|
+
|
|
49
|
+
# Initialize the client
|
|
50
|
+
client = DefiniteClient("YOUR_API_KEY")
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Features
|
|
54
|
+
|
|
55
|
+
- **Key-Value Store**: Persistent storage with version control and transactional commits
|
|
56
|
+
- **SQL Query Execution**: Execute SQL queries against your connected database integrations
|
|
57
|
+
- **Cube Query Execution**: Execute Cube queries for advanced analytics and data modeling
|
|
58
|
+
- **Secret Management**: Secure storage and retrieval of application secrets
|
|
59
|
+
- **Integration Store**: Read-only access to integration configurations
|
|
60
|
+
- **dlt Integration**: Run dlt pipelines with automatic state persistence to Definite
|
|
61
|
+
- **DuckDB Support**: Automatic discovery and connection to team's DuckDB integrations
|
|
62
|
+
|
|
63
|
+
## Basic Usage
|
|
64
|
+
|
|
65
|
+
### 🗄️ Key-Value Store
|
|
66
|
+
|
|
67
|
+
Store and retrieve key-value pairs that can be accessed by custom Python scripts hosted on Definite.
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
# Initialize or retrieve an existing key-value store
|
|
71
|
+
store = client.get_kv_store('test_store')
|
|
72
|
+
# Or use the alias method
|
|
73
|
+
store = client.kv_store('test_store')
|
|
74
|
+
|
|
75
|
+
# Add or update key-value pairs
|
|
76
|
+
store['replication_key'] = 'created_at'
|
|
77
|
+
store['replication_state'] = '2024-05-20'
|
|
78
|
+
store["key1"] = "value1"
|
|
79
|
+
store["key2"] = {"nested": "data"}
|
|
80
|
+
|
|
81
|
+
# Commit changes
|
|
82
|
+
store.commit()
|
|
83
|
+
|
|
84
|
+
# Retrieve values
|
|
85
|
+
print(store['replication_key']) # 'created_at'
|
|
86
|
+
value = store["key1"]
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
### 🗃️ SQL Query Execution
|
|
90
|
+
|
|
91
|
+
Execute SQL queries against your connected database integrations.
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
# Initialize the SQL client
|
|
95
|
+
sql_client = client.get_sql_client()
|
|
96
|
+
|
|
97
|
+
# Execute a SQL query
|
|
98
|
+
result = sql_client.execute("SELECT * FROM users LIMIT 10")
|
|
99
|
+
print(result)
|
|
100
|
+
|
|
101
|
+
# Execute a SQL query with a specific integration
|
|
102
|
+
result = sql_client.execute(
|
|
103
|
+
"SELECT COUNT(*) FROM orders WHERE status = 'completed'",
|
|
104
|
+
integration_id="my_database_integration"
|
|
105
|
+
)
|
|
106
|
+
print(result)
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
### 📊 Cube Query Execution
|
|
110
|
+
|
|
111
|
+
Execute Cube queries for advanced analytics and data modeling.
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
# Prepare a Cube query
|
|
115
|
+
cube_query = {
|
|
116
|
+
"dimensions": [],
|
|
117
|
+
"measures": ["sales.total_amount"],
|
|
118
|
+
"timeDimensions": [{
|
|
119
|
+
"dimension": "sales.date",
|
|
120
|
+
"granularity": "month"
|
|
121
|
+
}],
|
|
122
|
+
"limit": 1000
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
# Execute the Cube query
|
|
126
|
+
result = sql_client.execute_cube_query(
|
|
127
|
+
cube_query,
|
|
128
|
+
integration_id="my_cube_integration"
|
|
129
|
+
)
|
|
130
|
+
print(result)
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
### 🔒 Secret Store
|
|
134
|
+
|
|
135
|
+
Securely store and retrieve secrets for your integrations.
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
# Initialize the secret store
|
|
139
|
+
secret_store = client.get_secret_store()
|
|
140
|
+
# Or use the alias method
|
|
141
|
+
secret_store = client.secret_store()
|
|
142
|
+
|
|
143
|
+
# Set a secret
|
|
144
|
+
secret_store.set_secret("database_password", "my_secure_password")
|
|
145
|
+
|
|
146
|
+
# Get a secret
|
|
147
|
+
password = secret_store.get_secret("database_password")
|
|
148
|
+
|
|
149
|
+
# List all secrets
|
|
150
|
+
secrets = list(secret_store.list_secrets())
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
### 🔗 Integration Management
|
|
154
|
+
|
|
155
|
+
Manage your data integrations and connections.
|
|
156
|
+
|
|
157
|
+
```python
|
|
158
|
+
# Initialize the integration store
|
|
159
|
+
integration_store = client.get_integration_store()
|
|
160
|
+
# Or use the alias method
|
|
161
|
+
integration_store = client.integration_store()
|
|
162
|
+
|
|
163
|
+
# List all integrations
|
|
164
|
+
integrations = list(integration_store.list_integrations())
|
|
165
|
+
|
|
166
|
+
# Get a specific integration
|
|
167
|
+
integration = integration_store.get_integration("my_integration")
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
### dlt Integration
|
|
171
|
+
|
|
172
|
+
```python
|
|
173
|
+
from definite_sdk.dlt import DefiniteDLTPipeline
|
|
174
|
+
import dlt
|
|
175
|
+
|
|
176
|
+
# Create an incremental resource
|
|
177
|
+
@dlt.resource(primary_key="id", write_disposition="merge")
|
|
178
|
+
def orders(cursor=dlt.sources.incremental("created_at")):
|
|
179
|
+
# Your data loading logic here
|
|
180
|
+
pass
|
|
181
|
+
|
|
182
|
+
# Create and run pipeline
|
|
183
|
+
pipeline = DefiniteDLTPipeline("orders_sync")
|
|
184
|
+
pipeline.run(orders())
|
|
185
|
+
|
|
186
|
+
# State is automatically persisted to Definite
|
|
187
|
+
last_cursor = pipeline.get_state("orders")
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
### DuckDB Integration Discovery
|
|
191
|
+
|
|
192
|
+
```python
|
|
193
|
+
from definite_sdk.dlt import get_duckdb_connection
|
|
194
|
+
|
|
195
|
+
# Automatically discovers DuckDB integration using DEFINITE_API_KEY env var
|
|
196
|
+
result = get_duckdb_connection()
|
|
197
|
+
if result:
|
|
198
|
+
integration_id, connection = result
|
|
199
|
+
# Use the DuckDB connection
|
|
200
|
+
connection.execute("SELECT * FROM my_table")
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
**Note**: DuckDB integration discovery is currently limited as the API only exposes source integrations, not destination integrations. This functionality is provided for future compatibility.
|
|
204
|
+
|
|
205
|
+
### State Management
|
|
206
|
+
|
|
207
|
+
```python
|
|
208
|
+
# Set custom state
|
|
209
|
+
pipeline.set_state("custom_key", "custom_value")
|
|
210
|
+
|
|
211
|
+
# Get all state
|
|
212
|
+
all_state = pipeline.get_state()
|
|
213
|
+
|
|
214
|
+
# Resume from previous state
|
|
215
|
+
pipeline.resume_from_state()
|
|
216
|
+
|
|
217
|
+
# Reset state
|
|
218
|
+
pipeline.reset_state()
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
## Authentication
|
|
222
|
+
|
|
223
|
+
To use the Definite SDK, you'll need an API key. You can find and copy your API key from the bottom left user menu in your Definite workspace.
|
|
224
|
+
|
|
225
|
+
For SQL queries, you'll also need your integration ID, which can be found in your integration's page URL.
|
|
226
|
+
|
|
227
|
+
## Environment Variables
|
|
228
|
+
|
|
229
|
+
- `DEFINITE_API_KEY`: Your Definite API key (auto-injected in Definite runtime)
|
|
230
|
+
- `DEF_API_KEY`: Alternative environment variable for API key
|
|
231
|
+
|
|
232
|
+
## Error Handling
|
|
233
|
+
|
|
234
|
+
The SDK uses standard HTTP status codes and raises `requests.HTTPError` for API errors:
|
|
235
|
+
|
|
236
|
+
```python
|
|
237
|
+
import requests
|
|
238
|
+
|
|
239
|
+
try:
|
|
240
|
+
result = sql_client.execute("SELECT * FROM invalid_table")
|
|
241
|
+
except requests.HTTPError as e:
|
|
242
|
+
print(f"API Error: {e}")
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
## Testing
|
|
246
|
+
|
|
247
|
+
```bash
|
|
248
|
+
# Run all tests
|
|
249
|
+
DEF_API_KEY=your_api_key poetry run pytest
|
|
250
|
+
|
|
251
|
+
# Run specific test file
|
|
252
|
+
DEF_API_KEY=your_api_key poetry run pytest tests/test_dlt.py
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
## Contributing
|
|
256
|
+
|
|
257
|
+
Contributions are welcome! Please feel free to submit a Pull Request.
|
|
258
|
+
|
|
259
|
+
## License
|
|
260
|
+
|
|
261
|
+
This project is licensed under the MIT License.
|
|
262
|
+
|
|
263
|
+
## Documentation
|
|
264
|
+
|
|
265
|
+
For more detailed documentation, visit: https://docs.definite.app/
|
|
266
|
+
|
|
267
|
+
## Support
|
|
268
|
+
|
|
269
|
+
If you encounter any issues or have questions, please reach out to hello@definite.app
|
|
270
|
+
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
definite_sdk/__init__.py,sha256=0XR4YMDG50jLcGpaZvU834L4wg9swm9hxAzV858ccDE,520
|
|
2
|
+
definite_sdk/client.py,sha256=Uc3hYMeS8icHBW7m6BQrWXUiW4PbOOu_bJXXca6y3l0,2714
|
|
3
|
+
definite_sdk/dlt.py,sha256=JC-S1jd6zqnpGSnvk9kYb8GQ1IPZRd1zVKTVYAbeA4w,7188
|
|
4
|
+
definite_sdk/integration.py,sha256=v15QIpWm3eAZr-cmavfA39LXTjpskEEQ2SM_TBV4E1U,3395
|
|
5
|
+
definite_sdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
|
+
definite_sdk/secret.py,sha256=3wvEnTZ946hOxJBdB29dxvppwJ0x-TJV_LxRcXGqtis,2503
|
|
7
|
+
definite_sdk/sql.py,sha256=K5cPXM7B60Fzd8CyI-B4rXwK7hLpLT1Z8XZSZJjSogQ,3878
|
|
8
|
+
definite_sdk/store.py,sha256=7VYvROIrlWyM_ZX3LpqWWAfVLNDm0XMccMLv6qi9id8,5575
|
|
9
|
+
definite_sdk-0.1.8.dist-info/LICENSE,sha256=jMd7PtwNWiMoGIDgFutC1v4taO69W9SRZLKe9o470Vk,1064
|
|
10
|
+
definite_sdk-0.1.8.dist-info/METADATA,sha256=P1BJsi8QwDzRWjXQGT-esrwWyzmunbQLoFNIip5aw4Y,6865
|
|
11
|
+
definite_sdk-0.1.8.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
|
|
12
|
+
definite_sdk-0.1.8.dist-info/RECORD,,
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: definite-sdk
|
|
3
|
-
Version: 0.1.4
|
|
4
|
-
Summary: Definite SDK for Python
|
|
5
|
-
License: MIT
|
|
6
|
-
Author: Definite
|
|
7
|
-
Author-email: hello@definite.app
|
|
8
|
-
Requires-Python: >=3.9,<4.0
|
|
9
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
-
Classifier: Programming Language :: Python :: 3
|
|
11
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
-
Requires-Dist: requests (>=2.31.0,<3.0.0)
|
|
16
|
-
Description-Content-Type: text/markdown
|
|
17
|
-
|
|
18
|
-
# Definite SDK
|
|
19
|
-
|
|
20
|
-
A Python client for interacting with the Definite API, providing a convenient interface for key-value store operations.
|
|
21
|
-
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
definite_sdk/client.py,sha256=DjckW-kZOGSQ5RKwRfpPub_ucxAcUaTLknZz5YKe4nE,1354
|
|
2
|
-
definite_sdk/integration.py,sha256=z7i_q9m5dMNJPG3_J29aH1jov0b5u6rhO3IZj1V9KVg,1647
|
|
3
|
-
definite_sdk/secret.py,sha256=pgRzY40MwOaZBVEw-yQPWX3ag1B7AMoCJKJjzcLQnTs,2506
|
|
4
|
-
definite_sdk/store.py,sha256=FimC3RqLtiJc3PlOSORCIiMFaUcNtc-r45c54j3l92g,5034
|
|
5
|
-
definite_sdk-0.1.4.dist-info/LICENSE,sha256=jMd7PtwNWiMoGIDgFutC1v4taO69W9SRZLKe9o470Vk,1064
|
|
6
|
-
definite_sdk-0.1.4.dist-info/METADATA,sha256=PWkSNNRsjSKb1oU0a32FPUtqnP010jriQfRlI67p0i4,702
|
|
7
|
-
definite_sdk-0.1.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
8
|
-
definite_sdk-0.1.4.dist-info/RECORD,,
|
|
File without changes
|