definite-sdk 0.1.3__tar.gz → 0.1.8__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,270 @@
1
+ Metadata-Version: 2.3
2
+ Name: definite-sdk
3
+ Version: 0.1.8
4
+ Summary: Definite SDK for Python
5
+ License: MIT
6
+ Author: Definite
7
+ Author-email: hello@definite.app
8
+ Requires-Python: >=3.9,<4.0
9
+ Classifier: License :: OSI Approved :: MIT License
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Programming Language :: Python :: 3.9
12
+ Classifier: Programming Language :: Python :: 3.10
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Provides-Extra: dlt
17
+ Requires-Dist: dlt (>=1.0,<2.0) ; extra == "dlt"
18
+ Requires-Dist: duckdb (>=1.0,<2.0) ; extra == "dlt"
19
+ Requires-Dist: requests (>=2.31.0,<3.0.0)
20
+ Description-Content-Type: text/markdown
21
+
22
+ # Definite SDK
23
+
24
+ A Python client for interacting with the Definite API, providing a convenient interface for key-value store operations, SQL query execution, secrets management, and DLT (Data Load Tool) integration with state persistence.
25
+
26
+ ## Installation
27
+
28
+ **pip:**
29
+ ```bash
30
+ pip install definite-sdk
31
+
32
+ # For dlt support
33
+ pip install "definite-sdk[dlt]"
34
+ ```
35
+
36
+ **poetry:**
37
+ ```bash
38
+ poetry add definite-sdk
39
+
40
+ # For dlt support
41
+ poetry add "definite-sdk[dlt]"
42
+ ```
43
+
44
+ ## Quick Start
45
+
46
+ ```python
47
+ from definite_sdk import DefiniteClient
48
+
49
+ # Initialize the client
50
+ client = DefiniteClient("YOUR_API_KEY")
51
+ ```
52
+
53
+ ## Features
54
+
55
+ - **Key-Value Store**: Persistent storage with version control and transactional commits
56
+ - **SQL Query Execution**: Execute SQL queries against your connected database integrations
57
+ - **Cube Query Execution**: Execute Cube queries for advanced analytics and data modeling
58
+ - **Secret Management**: Secure storage and retrieval of application secrets
59
+ - **Integration Store**: Read-only access to integration configurations
60
+ - **dlt Integration**: Run dlt pipelines with automatic state persistence to Definite
61
+ - **DuckDB Support**: Automatic discovery and connection to team's DuckDB integrations
62
+
63
+ ## Basic Usage
64
+
65
+ ### 🗄️ Key-Value Store
66
+
67
+ Store and retrieve key-value pairs that can be accessed by custom Python scripts hosted on Definite.
68
+
69
+ ```python
70
+ # Initialize or retrieve an existing key-value store
71
+ store = client.get_kv_store('test_store')
72
+ # Or use the alias method
73
+ store = client.kv_store('test_store')
74
+
75
+ # Add or update key-value pairs
76
+ store['replication_key'] = 'created_at'
77
+ store['replication_state'] = '2024-05-20'
78
+ store["key1"] = "value1"
79
+ store["key2"] = {"nested": "data"}
80
+
81
+ # Commit changes
82
+ store.commit()
83
+
84
+ # Retrieve values
85
+ print(store['replication_key']) # 'created_at'
86
+ value = store["key1"]
87
+ ```
88
+
89
+ ### 🗃️ SQL Query Execution
90
+
91
+ Execute SQL queries against your connected database integrations.
92
+
93
+ ```python
94
+ # Initialize the SQL client
95
+ sql_client = client.get_sql_client()
96
+
97
+ # Execute a SQL query
98
+ result = sql_client.execute("SELECT * FROM users LIMIT 10")
99
+ print(result)
100
+
101
+ # Execute a SQL query with a specific integration
102
+ result = sql_client.execute(
103
+ "SELECT COUNT(*) FROM orders WHERE status = 'completed'",
104
+ integration_id="my_database_integration"
105
+ )
106
+ print(result)
107
+ ```
108
+
109
+ ### 📊 Cube Query Execution
110
+
111
+ Execute Cube queries for advanced analytics and data modeling.
112
+
113
+ ```python
114
+ # Prepare a Cube query
115
+ cube_query = {
116
+ "dimensions": [],
117
+ "measures": ["sales.total_amount"],
118
+ "timeDimensions": [{
119
+ "dimension": "sales.date",
120
+ "granularity": "month"
121
+ }],
122
+ "limit": 1000
123
+ }
124
+
125
+ # Execute the Cube query
126
+ result = sql_client.execute_cube_query(
127
+ cube_query,
128
+ integration_id="my_cube_integration"
129
+ )
130
+ print(result)
131
+ ```
132
+
133
+ ### 🔒 Secret Store
134
+
135
+ Securely store and retrieve secrets for your integrations.
136
+
137
+ ```python
138
+ # Initialize the secret store
139
+ secret_store = client.get_secret_store()
140
+ # Or use the alias method
141
+ secret_store = client.secret_store()
142
+
143
+ # Set a secret
144
+ secret_store.set_secret("database_password", "my_secure_password")
145
+
146
+ # Get a secret
147
+ password = secret_store.get_secret("database_password")
148
+
149
+ # List all secrets
150
+ secrets = list(secret_store.list_secrets())
151
+ ```
152
+
153
+ ### 🔗 Integration Management
154
+
155
+ Manage your data integrations and connections.
156
+
157
+ ```python
158
+ # Initialize the integration store
159
+ integration_store = client.get_integration_store()
160
+ # Or use the alias method
161
+ integration_store = client.integration_store()
162
+
163
+ # List all integrations
164
+ integrations = list(integration_store.list_integrations())
165
+
166
+ # Get a specific integration
167
+ integration = integration_store.get_integration("my_integration")
168
+ ```
169
+
170
+ ### dlt Integration
171
+
172
+ ```python
173
+ from definite_sdk.dlt import DefiniteDLTPipeline
174
+ import dlt
175
+
176
+ # Create an incremental resource
177
+ @dlt.resource(primary_key="id", write_disposition="merge")
178
+ def orders(cursor=dlt.sources.incremental("created_at")):
179
+ # Your data loading logic here
180
+ pass
181
+
182
+ # Create and run pipeline
183
+ pipeline = DefiniteDLTPipeline("orders_sync")
184
+ pipeline.run(orders())
185
+
186
+ # State is automatically persisted to Definite
187
+ last_cursor = pipeline.get_state("orders")
188
+ ```
189
+
190
+ ### DuckDB Integration Discovery
191
+
192
+ ```python
193
+ from definite_sdk.dlt import get_duckdb_connection
194
+
195
+ # Automatically discovers DuckDB integration using DEFINITE_API_KEY env var
196
+ result = get_duckdb_connection()
197
+ if result:
198
+ integration_id, connection = result
199
+ # Use the DuckDB connection
200
+ connection.execute("SELECT * FROM my_table")
201
+ ```
202
+
203
+ **Note**: DuckDB integration discovery is currently limited as the API only exposes source integrations, not destination integrations. This functionality is provided for future compatibility.
204
+
205
+ ### State Management
206
+
207
+ ```python
208
+ # Set custom state
209
+ pipeline.set_state("custom_key", "custom_value")
210
+
211
+ # Get all state
212
+ all_state = pipeline.get_state()
213
+
214
+ # Resume from previous state
215
+ pipeline.resume_from_state()
216
+
217
+ # Reset state
218
+ pipeline.reset_state()
219
+ ```
220
+
221
+ ## Authentication
222
+
223
+ To use the Definite SDK, you'll need an API key. You can find and copy your API key from the bottom left user menu in your Definite workspace.
224
+
225
+ For SQL queries, you'll also need your integration ID, which can be found in your integration's page URL.
226
+
227
+ ## Environment Variables
228
+
229
+ - `DEFINITE_API_KEY`: Your Definite API key (auto-injected in Definite runtime)
230
+ - `DEF_API_KEY`: Alternative environment variable for API key
231
+
232
+ ## Error Handling
233
+
234
+ The SDK uses standard HTTP status codes and raises `requests.HTTPError` for API errors:
235
+
236
+ ```python
237
+ import requests
238
+
239
+ try:
240
+ result = sql_client.execute("SELECT * FROM invalid_table")
241
+ except requests.HTTPError as e:
242
+ print(f"API Error: {e}")
243
+ ```
244
+
245
+ ## Testing
246
+
247
+ ```bash
248
+ # Run all tests
249
+ DEF_API_KEY=your_api_key poetry run pytest
250
+
251
+ # Run specific test file
252
+ DEF_API_KEY=your_api_key poetry run pytest tests/test_dlt.py
253
+ ```
254
+
255
+ ## Contributing
256
+
257
+ Contributions are welcome! Please feel free to submit a Pull Request.
258
+
259
+ ## License
260
+
261
+ This project is licensed under the MIT License.
262
+
263
+ ## Documentation
264
+
265
+ For more detailed documentation, visit: https://docs.definite.app/
266
+
267
+ ## Support
268
+
269
+ If you encounter any issues or have questions, please reach out to hello@definite.app
270
+
@@ -0,0 +1,248 @@
1
+ # Definite SDK
2
+
3
+ A Python client for interacting with the Definite API, providing a convenient interface for key-value store operations, SQL query execution, secrets management, and DLT (Data Load Tool) integration with state persistence.
4
+
5
+ ## Installation
6
+
7
+ **pip:**
8
+ ```bash
9
+ pip install definite-sdk
10
+
11
+ # For dlt support
12
+ pip install "definite-sdk[dlt]"
13
+ ```
14
+
15
+ **poetry:**
16
+ ```bash
17
+ poetry add definite-sdk
18
+
19
+ # For dlt support
20
+ poetry add "definite-sdk[dlt]"
21
+ ```
22
+
23
+ ## Quick Start
24
+
25
+ ```python
26
+ from definite_sdk import DefiniteClient
27
+
28
+ # Initialize the client
29
+ client = DefiniteClient("YOUR_API_KEY")
30
+ ```
31
+
32
+ ## Features
33
+
34
+ - **Key-Value Store**: Persistent storage with version control and transactional commits
35
+ - **SQL Query Execution**: Execute SQL queries against your connected database integrations
36
+ - **Cube Query Execution**: Execute Cube queries for advanced analytics and data modeling
37
+ - **Secret Management**: Secure storage and retrieval of application secrets
38
+ - **Integration Store**: Read-only access to integration configurations
39
+ - **dlt Integration**: Run dlt pipelines with automatic state persistence to Definite
40
+ - **DuckDB Support**: Automatic discovery and connection to team's DuckDB integrations
41
+
42
+ ## Basic Usage
43
+
44
+ ### 🗄️ Key-Value Store
45
+
46
+ Store and retrieve key-value pairs that can be accessed by custom Python scripts hosted on Definite.
47
+
48
+ ```python
49
+ # Initialize or retrieve an existing key-value store
50
+ store = client.get_kv_store('test_store')
51
+ # Or use the alias method
52
+ store = client.kv_store('test_store')
53
+
54
+ # Add or update key-value pairs
55
+ store['replication_key'] = 'created_at'
56
+ store['replication_state'] = '2024-05-20'
57
+ store["key1"] = "value1"
58
+ store["key2"] = {"nested": "data"}
59
+
60
+ # Commit changes
61
+ store.commit()
62
+
63
+ # Retrieve values
64
+ print(store['replication_key']) # 'created_at'
65
+ value = store["key1"]
66
+ ```
67
+
68
+ ### 🗃️ SQL Query Execution
69
+
70
+ Execute SQL queries against your connected database integrations.
71
+
72
+ ```python
73
+ # Initialize the SQL client
74
+ sql_client = client.get_sql_client()
75
+
76
+ # Execute a SQL query
77
+ result = sql_client.execute("SELECT * FROM users LIMIT 10")
78
+ print(result)
79
+
80
+ # Execute a SQL query with a specific integration
81
+ result = sql_client.execute(
82
+ "SELECT COUNT(*) FROM orders WHERE status = 'completed'",
83
+ integration_id="my_database_integration"
84
+ )
85
+ print(result)
86
+ ```
87
+
88
+ ### 📊 Cube Query Execution
89
+
90
+ Execute Cube queries for advanced analytics and data modeling.
91
+
92
+ ```python
93
+ # Prepare a Cube query
94
+ cube_query = {
95
+ "dimensions": [],
96
+ "measures": ["sales.total_amount"],
97
+ "timeDimensions": [{
98
+ "dimension": "sales.date",
99
+ "granularity": "month"
100
+ }],
101
+ "limit": 1000
102
+ }
103
+
104
+ # Execute the Cube query
105
+ result = sql_client.execute_cube_query(
106
+ cube_query,
107
+ integration_id="my_cube_integration"
108
+ )
109
+ print(result)
110
+ ```
111
+
112
+ ### 🔒 Secret Store
113
+
114
+ Securely store and retrieve secrets for your integrations.
115
+
116
+ ```python
117
+ # Initialize the secret store
118
+ secret_store = client.get_secret_store()
119
+ # Or use the alias method
120
+ secret_store = client.secret_store()
121
+
122
+ # Set a secret
123
+ secret_store.set_secret("database_password", "my_secure_password")
124
+
125
+ # Get a secret
126
+ password = secret_store.get_secret("database_password")
127
+
128
+ # List all secrets
129
+ secrets = list(secret_store.list_secrets())
130
+ ```
131
+
132
+ ### 🔗 Integration Management
133
+
134
+ Manage your data integrations and connections.
135
+
136
+ ```python
137
+ # Initialize the integration store
138
+ integration_store = client.get_integration_store()
139
+ # Or use the alias method
140
+ integration_store = client.integration_store()
141
+
142
+ # List all integrations
143
+ integrations = list(integration_store.list_integrations())
144
+
145
+ # Get a specific integration
146
+ integration = integration_store.get_integration("my_integration")
147
+ ```
148
+
149
+ ### dlt Integration
150
+
151
+ ```python
152
+ from definite_sdk.dlt import DefiniteDLTPipeline
153
+ import dlt
154
+
155
+ # Create an incremental resource
156
+ @dlt.resource(primary_key="id", write_disposition="merge")
157
+ def orders(cursor=dlt.sources.incremental("created_at")):
158
+ # Your data loading logic here
159
+ pass
160
+
161
+ # Create and run pipeline
162
+ pipeline = DefiniteDLTPipeline("orders_sync")
163
+ pipeline.run(orders())
164
+
165
+ # State is automatically persisted to Definite
166
+ last_cursor = pipeline.get_state("orders")
167
+ ```
168
+
169
+ ### DuckDB Integration Discovery
170
+
171
+ ```python
172
+ from definite_sdk.dlt import get_duckdb_connection
173
+
174
+ # Automatically discovers DuckDB integration using DEFINITE_API_KEY env var
175
+ result = get_duckdb_connection()
176
+ if result:
177
+ integration_id, connection = result
178
+ # Use the DuckDB connection
179
+ connection.execute("SELECT * FROM my_table")
180
+ ```
181
+
182
+ **Note**: DuckDB integration discovery is currently limited as the API only exposes source integrations, not destination integrations. This functionality is provided for future compatibility.
183
+
184
+ ### State Management
185
+
186
+ ```python
187
+ # Set custom state
188
+ pipeline.set_state("custom_key", "custom_value")
189
+
190
+ # Get all state
191
+ all_state = pipeline.get_state()
192
+
193
+ # Resume from previous state
194
+ pipeline.resume_from_state()
195
+
196
+ # Reset state
197
+ pipeline.reset_state()
198
+ ```
199
+
200
+ ## Authentication
201
+
202
+ To use the Definite SDK, you'll need an API key. You can find and copy your API key from the bottom left user menu in your Definite workspace.
203
+
204
+ For SQL queries, you'll also need your integration ID, which can be found in your integration's page URL.
205
+
206
+ ## Environment Variables
207
+
208
+ - `DEFINITE_API_KEY`: Your Definite API key (auto-injected in Definite runtime)
209
+ - `DEF_API_KEY`: Alternative environment variable for API key
210
+
211
+ ## Error Handling
212
+
213
+ The SDK uses standard HTTP status codes and raises `requests.HTTPError` for API errors:
214
+
215
+ ```python
216
+ import requests
217
+
218
+ try:
219
+ result = sql_client.execute("SELECT * FROM invalid_table")
220
+ except requests.HTTPError as e:
221
+ print(f"API Error: {e}")
222
+ ```
223
+
224
+ ## Testing
225
+
226
+ ```bash
227
+ # Run all tests
228
+ DEF_API_KEY=your_api_key poetry run pytest
229
+
230
+ # Run specific test file
231
+ DEF_API_KEY=your_api_key poetry run pytest tests/test_dlt.py
232
+ ```
233
+
234
+ ## Contributing
235
+
236
+ Contributions are welcome! Please feel free to submit a Pull Request.
237
+
238
+ ## License
239
+
240
+ This project is licensed under the MIT License.
241
+
242
+ ## Documentation
243
+
244
+ For more detailed documentation, visit: https://docs.definite.app/
245
+
246
+ ## Support
247
+
248
+ If you encounter any issues or have questions, please reach out to hello@definite.app
@@ -0,0 +1,20 @@
1
+ """
2
+ Definite SDK for Python
3
+
4
+ A Python library for interacting with the Definite API and tools.
5
+ """
6
+
7
+ from definite_sdk.client import DefiniteClient
8
+ from definite_sdk.integration import DefiniteIntegrationStore
9
+ from definite_sdk.secret import DefiniteSecretStore
10
+ from definite_sdk.sql import DefiniteSqlClient
11
+ from definite_sdk.store import DefiniteKVStore
12
+
13
+ __version__ = "0.1.8"
14
+ __all__ = [
15
+ "DefiniteClient",
16
+ "DefiniteIntegrationStore",
17
+ "DefiniteSecretStore",
18
+ "DefiniteSqlClient",
19
+ "DefiniteKVStore",
20
+ ]
@@ -0,0 +1,79 @@
1
+ import os
2
+ from typing import Optional
3
+
4
+ from definite_sdk.integration import DefiniteIntegrationStore
5
+ from definite_sdk.secret import DefiniteSecretStore
6
+ from definite_sdk.sql import DefiniteSqlClient
7
+ from definite_sdk.store import DefiniteKVStore
8
+
9
+ API_URL = "https://api.definite.app"
10
+
11
+
12
+ class DefiniteClient:
13
+ """Client for interacting with the Definite API."""
14
+
15
+ def __init__(self, api_key: Optional[str] = None, api_url: str = API_URL):
16
+ """Creates a definite client with the provided API key.
17
+
18
+ Args:
19
+ api_key: API key for authentication. If not provided, will look for
20
+ DEFINITE_API_KEY or DEF_API_KEY environment variables.
21
+ api_url: Base URL for the Definite API.
22
+
23
+ See: https://docs.definite.app/definite-api for how to obtain an API key.
24
+ """
25
+ if api_key is None:
26
+ api_key = os.getenv("DEFINITE_API_KEY") or os.getenv("DEF_API_KEY")
27
+ if not api_key:
28
+ raise ValueError(
29
+ "API key must be provided or set in DEFINITE_API_KEY "
30
+ "or DEF_API_KEY environment variable"
31
+ )
32
+
33
+ self.api_key = api_key
34
+ self.api_url = api_url
35
+
36
+ def get_kv_store(self, name: str) -> DefiniteKVStore:
37
+ """Initializes a key-value store with the provided name.
38
+
39
+ See DefiniteKVStore for more how to interact with the store.
40
+ """
41
+
42
+ return DefiniteKVStore(name, self.api_key, self.api_url)
43
+
44
+ def get_secret_store(self) -> DefiniteSecretStore:
45
+ """Initializes the secret store.
46
+
47
+ See DefiniteSecretStore for more how to interact with the store.
48
+ """
49
+
50
+ return DefiniteSecretStore(self.api_key, self.api_url)
51
+
52
+ def get_integration_store(self) -> DefiniteIntegrationStore:
53
+ """Initializes the integration store.
54
+
55
+ See DefiniteIntegrationStore for more how to interact with the store.
56
+ """
57
+
58
+ return DefiniteIntegrationStore(self.api_key, self.api_url)
59
+
60
+ def get_sql_client(self) -> DefiniteSqlClient:
61
+ """Initializes the SQL client for executing SQL queries.
62
+
63
+ See DefiniteSqlClient for more how to execute SQL queries.
64
+ """
65
+
66
+ return DefiniteSqlClient(self.api_key, self.api_url)
67
+
68
+ # Alias methods for consistency
69
+ def kv_store(self, name: str) -> DefiniteKVStore:
70
+ """Alias for get_kv_store."""
71
+ return self.get_kv_store(name)
72
+
73
+ def secret_store(self) -> DefiniteSecretStore:
74
+ """Alias for get_secret_store."""
75
+ return self.get_secret_store()
76
+
77
+ def integration_store(self) -> DefiniteIntegrationStore:
78
+ """Alias for get_integration_store."""
79
+ return self.get_integration_store()
@@ -0,0 +1,237 @@
1
+ """DLT (Data Load Tool) integration for Definite SDK."""
2
+
3
+ import os
4
+ import json
5
+ from typing import Any, Dict, Optional, Tuple, TYPE_CHECKING
6
+
7
+ if TYPE_CHECKING:
8
+ import dlt
9
+ from dlt.common import Destination
10
+ from dlt.pipeline import Pipeline
11
+ else:
12
+ try:
13
+ import dlt
14
+ from dlt.common import Destination
15
+ from dlt.pipeline import Pipeline
16
+ except ImportError:
17
+ dlt = None # type: ignore
18
+ Destination = None # type: ignore
19
+ Pipeline = None # type: ignore
20
+
21
+ from .client import DefiniteClient
22
+
23
+
24
+ class DefiniteDLTPipeline:
25
+ """Wrapper for DLT pipelines with Definite state management."""
26
+
27
+ def __init__(
28
+ self,
29
+ name: str,
30
+ dataset_name: Optional[str] = None,
31
+ destination: Optional["Destination"] = None,
32
+ **kwargs: Any,
33
+ ):
34
+ """Initialize a DLT pipeline with Definite integration.
35
+
36
+ Args:
37
+ name: Pipeline name
38
+ dataset_name: Dataset name for the destination
39
+ destination: DLT destination (defaults to duckdb)
40
+ **kwargs: Additional arguments passed to dlt.pipeline()
41
+ """
42
+ if dlt is None:
43
+ raise ImportError(
44
+ "dlt package not installed. Install with: pip install definite-sdk[dlt]"
45
+ )
46
+
47
+ self.name = name
48
+ self._client = DefiniteClient()
49
+ self._state_store = self._client.kv_store(f"dlt_state_{name}")
50
+
51
+ # Default to DuckDB if no destination specified
52
+ if destination is None:
53
+ destination = "duckdb"
54
+
55
+ # Use dataset_name or default to pipeline name
56
+ if dataset_name is None:
57
+ dataset_name = name
58
+
59
+ # Create the underlying DLT pipeline
60
+ self._pipeline = dlt.pipeline(
61
+ pipeline_name=name,
62
+ destination=destination,
63
+ dataset_name=dataset_name,
64
+ **kwargs,
65
+ )
66
+
67
+ @property
68
+ def pipeline(self) -> "Pipeline":
69
+ """Get the underlying DLT pipeline."""
70
+ return self._pipeline
71
+
72
+ @property
73
+ def state(self) -> Dict[str, Any]:
74
+ """Get the current pipeline state."""
75
+ return dict(self._pipeline.state)
76
+
77
+ def run(
78
+ self,
79
+ data: Any,
80
+ *,
81
+ table_name: Optional[str] = None,
82
+ write_disposition: Optional[str] = None,
83
+ **kwargs: Any,
84
+ ) -> Any:
85
+ """Run the pipeline with the given data.
86
+
87
+ Args:
88
+ data: Data to load (resource, source, or iterator)
89
+ table_name: Name of the table to load data into
90
+ write_disposition: How to write data (append, replace, merge)
91
+ **kwargs: Additional arguments passed to pipeline.run()
92
+
93
+ Returns:
94
+ Load info from the pipeline run
95
+ """
96
+ # Run the pipeline
97
+ load_info = self._pipeline.run(
98
+ data, table_name=table_name, write_disposition=write_disposition, **kwargs
99
+ )
100
+
101
+ # Persist state to Definite after successful run
102
+ self._persist_state()
103
+
104
+ return load_info
105
+
106
+ def _persist_state(self) -> None:
107
+ """Persist current pipeline state to Definite."""
108
+ state = self._pipeline.state
109
+ if state:
110
+ # Store each state key separately for easier access
111
+ for key, value in state.items():
112
+ # Serialize value to JSON string
113
+ self._state_store[key] = json.dumps(value)
114
+ self._state_store.commit()
115
+
116
+ def get_state(self, key: Optional[str] = None) -> Any:
117
+ """Retrieve state from Definite store.
118
+
119
+ Args:
120
+ key: Specific state key to retrieve. If None, returns all state.
121
+
122
+ Returns:
123
+ State value or entire state dict
124
+ """
125
+ if key is not None:
126
+ value = self._state_store.get(key, None)
127
+ return json.loads(value) if value else None
128
+
129
+ # Return all state as dict
130
+ return {k: json.loads(self._state_store[k]) for k in self._state_store}
131
+
132
+ def set_state(self, key: str, value: Any) -> None:
133
+ """Set a state value.
134
+
135
+ Args:
136
+ key: State key
137
+ value: State value
138
+ """
139
+ self._pipeline.state[key] = value
140
+ # Also persist to Definite immediately
141
+ self._state_store[key] = json.dumps(value)
142
+ self._state_store.commit()
143
+
144
+ def resume_from_state(self) -> None:
145
+ """Resume pipeline from previously stored state."""
146
+ # Load state from Definite
147
+ stored_state: Dict[str, Any] = {
148
+ k: json.loads(self._state_store[k]) for k in self._state_store
149
+ }
150
+
151
+ # Apply state to pipeline
152
+ for key, value in stored_state.items():
153
+ self._pipeline.state[key] = value
154
+
155
+ def reset_state(self) -> None:
156
+ """Reset pipeline state."""
157
+ # Clear pipeline state
158
+ self._pipeline.state.clear()
159
+
160
+ # Clear Definite state
161
+ keys_to_delete = list(self._state_store)
162
+ for key in keys_to_delete:
163
+ del self._state_store[key]
164
+ self._state_store.commit()
165
+
166
+
167
+ class DLTStateAdapter:
168
+ """Adapter for DLT state management conforming to Definite patterns."""
169
+
170
+ def __init__(self, pipeline_name: str):
171
+ """Initialize state adapter.
172
+
173
+ Args:
174
+ pipeline_name: Name of the DLT pipeline
175
+ """
176
+ self.pipeline_name = pipeline_name
177
+ self._client = DefiniteClient()
178
+ self._store = self._client.kv_store(f"dlt_state_{pipeline_name}")
179
+
180
+ def save_state(self, state: Dict[str, Any]) -> None:
181
+ """Save pipeline state.
182
+
183
+ Args:
184
+ state: State dictionary to save
185
+ """
186
+ for key, value in state.items():
187
+ self._store[key] = json.dumps(value)
188
+ self._store.commit()
189
+
190
+ def load_state(self) -> Dict[str, Any]:
191
+ """Load pipeline state.
192
+
193
+ Returns:
194
+ State dictionary
195
+ """
196
+ return {k: json.loads(self._store[k]) for k in self._store}
197
+
198
+ def clear_state(self) -> None:
199
+ """Clear all state."""
200
+ keys_to_delete = list(self._store)
201
+ for key in keys_to_delete:
202
+ del self._store[key]
203
+ self._store.commit()
204
+
205
+
206
+ def get_duckdb_connection() -> Optional[Tuple[str, Any]]:
207
+ """Get DuckDB connection from Definite integration.
208
+
209
+ Uses the DEFINITE_API_KEY environment variable to authenticate and
210
+ lookup the team's DuckDB integration.
211
+
212
+ Returns:
213
+ Optional[Tuple[str, Any]]: Tuple of (integration_id, connection) if found,
214
+ None if no DuckDB integration or API key.
215
+ """
216
+ api_key = os.getenv("DEFINITE_API_KEY")
217
+ if not api_key:
218
+ return None
219
+
220
+ try:
221
+ import duckdb
222
+ except ImportError:
223
+ raise ImportError(
224
+ "duckdb package not installed. Install with: pip install duckdb"
225
+ )
226
+
227
+ client = DefiniteClient(api_key=api_key)
228
+ integration_store = client.integration_store()
229
+
230
+ result = integration_store.lookup_duckdb_integration()
231
+ if result:
232
+ integration_id, connection_uri = result
233
+ # Create DuckDB connection
234
+ connection = duckdb.connect(connection_uri)
235
+ return (integration_id, connection)
236
+
237
+ return None
@@ -0,0 +1,99 @@
1
+ from typing import Iterator, Optional, Tuple
2
+
3
+ import requests
4
+
5
+ INTEGRATION_ENDPOINT = "/v1/api/integration"
6
+
7
+
8
+ class DefiniteIntegrationStore:
9
+ """
10
+ Read only access to the integration store on Definite.
11
+
12
+ Initialization:
13
+ >>> client = DefiniteSdkClient("MY_API_KEY")
14
+ >>> integration_store = client.get_integration_store()
15
+
16
+ Accessing values:
17
+ >>> integration_store.list_integrations()
18
+ >>> integration_store.get_integration("name")
19
+ """
20
+
21
+ def __init__(self, api_key: str, api_url: str):
22
+ """
23
+ Initializes the DefiniteSecretStore
24
+
25
+ Args:
26
+ api_key (str): The API key for authorization.
27
+ """
28
+ self._api_key = api_key
29
+ self._integration_store_url = api_url + INTEGRATION_ENDPOINT
30
+
31
+ def list_integrations(self) -> Iterator[dict]:
32
+ """
33
+ Lists all integrations in the store.
34
+
35
+ Returns:
36
+ Iterator[str]: An iterator of integrations.
37
+ """
38
+ response = requests.get(
39
+ self._integration_store_url,
40
+ headers={"Authorization": "Bearer " + self._api_key},
41
+ )
42
+ response.raise_for_status()
43
+ return iter(response.json())
44
+
45
+ def get_integration(self, name: str) -> dict:
46
+ """
47
+ Retrieves an integration by name.
48
+
49
+ Args:
50
+ name (str): The name of the integration.
51
+
52
+ Returns:
53
+ str: The value of the integration.
54
+ """
55
+ response = requests.get(
56
+ self._integration_store_url + f"/{name}",
57
+ headers={"Authorization": "Bearer " + self._api_key},
58
+ )
59
+ response.raise_for_status()
60
+ return dict(response.json())
61
+
62
+ def lookup_duckdb_integration(self) -> Optional[Tuple[str, str]]:
63
+ """
64
+ Look up the team's DuckDB integration.
65
+
66
+ Note: Currently, the API only returns extractor (source) integrations.
67
+ Destination integrations like DuckDB are not yet exposed through this endpoint.
68
+ This method is provided for future compatibility when the API is updated.
69
+
70
+ Returns:
71
+ Optional[Tuple[str, str]]: Tuple of (integration_id, connection_uri)
72
+ if found, None if no DuckDB integration exists.
73
+ """
74
+ try:
75
+ response = requests.get(
76
+ self._integration_store_url,
77
+ headers={"Authorization": "Bearer " + self._api_key},
78
+ )
79
+ response.raise_for_status()
80
+ integrations = response.json()
81
+ except:
82
+ return None
83
+
84
+ # Check for DuckDB in the integrations
85
+ # Note: Currently this will not find DuckDB as it's a destination integration
86
+ for integration in integrations:
87
+ integration_type = integration.get("integration_type", "").lower()
88
+ if integration_type == "duckdb" and integration.get("active", True):
89
+ integration_id = integration.get("id")
90
+ # Connection URI might be in config or connection_string field
91
+ connection_uri = (
92
+ integration.get("connection_string")
93
+ or integration.get("config", {}).get("database_path")
94
+ or integration.get("config", {}).get("connection_string")
95
+ )
96
+ if integration_id and connection_uri:
97
+ return (integration_id, connection_uri)
98
+
99
+ return None
File without changes
@@ -0,0 +1,94 @@
1
+ from typing import Iterator
2
+
3
+ import requests
4
+
5
+ SECRET_STORE_ENDPOINT = "/v1/api/secret"
6
+
7
+
8
+ class DefiniteSecretStore:
9
+ """
10
+ A secret store hosted by Definite.
11
+
12
+ Initialization:
13
+ >>> client = DefiniteSdkClient("MY_API_KEY")
14
+ >>> secret_store = client.get_secret_store()
15
+
16
+ Accessing values:
17
+ >>> secret_store.list_secrets()
18
+ >>> secret_store.get_secret("key")
19
+
20
+ Setting values:
21
+ >>> secret_store.set_secret("key", "value")
22
+
23
+ To permanently delete a secret:
24
+ >>> secret_store.delete_secret("key")
25
+ """
26
+
27
+ def __init__(self, api_key: str, api_url: str):
28
+ """
29
+ Initializes the DefiniteSecretStore
30
+
31
+ Args:
32
+ api_key (str): The API key for authorization.
33
+ """
34
+ self._api_key = api_key
35
+ self._secret_store_url = api_url + SECRET_STORE_ENDPOINT
36
+
37
+ def list_secrets(self) -> Iterator[str]:
38
+ """
39
+ Lists all secrets in the store.
40
+
41
+ Returns:
42
+ Iterator[str]: An iterator of secret keys.
43
+ """
44
+ response = requests.get(
45
+ self._secret_store_url,
46
+ headers={"Authorization": "Bearer " + self._api_key},
47
+ )
48
+ response.raise_for_status()
49
+ return iter(response.json()["secrets"])
50
+
51
+ def get_secret(self, key: str) -> str:
52
+ """
53
+ Retrieves the value of a secret.
54
+
55
+ Args:
56
+ key (str): The key of the secret.
57
+
58
+ Returns:
59
+ str: The value of the secret.
60
+ """
61
+ response = requests.get(
62
+ self._secret_store_url + f"/{key}",
63
+ headers={"Authorization": "Bearer " + self._api_key},
64
+ )
65
+ response.raise_for_status()
66
+ return str(response.json()["value"])
67
+
68
+ def set_secret(self, key: str, value: str):
69
+ """
70
+ Sets the value of a secret.
71
+
72
+ Args:
73
+ key (str): The key of the secret.
74
+ value (str): The value of the secret.
75
+ """
76
+ response = requests.post(
77
+ self._secret_store_url + f"/{key}",
78
+ json={"value": value},
79
+ headers={"Authorization": "Bearer " + self._api_key},
80
+ )
81
+ response.raise_for_status()
82
+
83
+ def delete_secret(self, key: str):
84
+ """
85
+ Deletes a secret.
86
+
87
+ Args:
88
+ key (str): The key of the secret.
89
+ """
90
+ response = requests.delete(
91
+ self._secret_store_url + f"/{key}",
92
+ headers={"Authorization": "Bearer " + self._api_key},
93
+ )
94
+ response.raise_for_status()
@@ -0,0 +1,126 @@
1
+ from typing import Any, Dict, List, Optional, Union
2
+
3
+ import requests
4
+
5
+ SQL_ENDPOINT = "/v1/query"
6
+
7
+
8
+ class DefiniteSqlClient:
9
+ """
10
+ A SQL client for executing SQL queries via the Definite API.
11
+
12
+ Initialization:
13
+ >>> client = DefiniteClient("MY_API_KEY")
14
+ >>> sql_client = client.get_sql_client()
15
+
16
+ Executing SQL queries:
17
+ >>> result = sql_client.execute("SELECT * FROM my_table LIMIT 10")
18
+ >>> print(result)
19
+
20
+ Executing SQL queries with integration ID:
21
+ >>> result = sql_client.execute(
22
+ ... "SELECT * FROM my_table LIMIT 10",
23
+ ... integration_id="my_integration_id"
24
+ ... )
25
+ >>> print(result)
26
+
27
+ Executing Cube queries:
28
+ >>> cube_query = {
29
+ ... "dimensions": [],
30
+ ... "measures": ["sales.total_amount"],
31
+ ... "timeDimensions": [{"dimension": "sales.date", "granularity": "month"}],
32
+ ... "limit": 1000
33
+ ... }
34
+ >>> result = sql_client.execute_cube_query(cube_query, integration_id="my_cube_integration")
35
+ >>> print(result)
36
+ """
37
+
38
+ def __init__(self, api_key: str, api_url: str):
39
+ """
40
+ Initializes the DefiniteSqlClient.
41
+
42
+ Args:
43
+ api_key (str): The API key for authorization.
44
+ api_url (str): The base URL for the Definite API.
45
+ """
46
+ self._api_key = api_key
47
+ self._sql_url = api_url + SQL_ENDPOINT
48
+
49
+ def execute(
50
+ self,
51
+ sql: str,
52
+ integration_id: Optional[str] = None
53
+ ) -> Dict[str, Any]:
54
+ """
55
+ Executes a SQL query against a database integration.
56
+
57
+ Args:
58
+ sql (str): The SQL query to execute.
59
+ integration_id (Optional[str]): The integration ID to query against.
60
+ If not provided, the default integration will be used.
61
+
62
+ Returns:
63
+ Dict[str, Any]: The query result as returned by the API.
64
+
65
+ Raises:
66
+ requests.HTTPError: If the API request fails.
67
+
68
+ Example:
69
+ >>> result = sql_client.execute("SELECT COUNT(*) FROM users")
70
+ >>> print(result)
71
+ """
72
+ payload = {"sql": sql}
73
+ if integration_id:
74
+ payload["integration_id"] = integration_id
75
+
76
+ response = requests.post(
77
+ self._sql_url,
78
+ json=payload,
79
+ headers={"Authorization": "Bearer " + self._api_key},
80
+ )
81
+ response.raise_for_status()
82
+ return response.json()
83
+
84
+ def execute_cube_query(
85
+ self,
86
+ cube_query: Dict[str, Any],
87
+ integration_id: Optional[str] = None
88
+ ) -> Dict[str, Any]:
89
+ """
90
+ Executes a Cube query against a Cube integration.
91
+
92
+ Args:
93
+ cube_query (Dict[str, Any]): The Cube query in JSON format.
94
+ integration_id (Optional[str]): The Cube integration ID to query against.
95
+ If not provided, the default integration will be used.
96
+
97
+ Returns:
98
+ Dict[str, Any]: The query result as returned by the API.
99
+
100
+ Raises:
101
+ requests.HTTPError: If the API request fails.
102
+
103
+ Example:
104
+ >>> cube_query = {
105
+ ... "dimensions": [],
106
+ ... "measures": ["sales.total_amount"],
107
+ ... "timeDimensions": [{
108
+ ... "dimension": "sales.date",
109
+ ... "granularity": "month"
110
+ ... }],
111
+ ... "limit": 1000
112
+ ... }
113
+ >>> result = sql_client.execute_cube_query(cube_query, "my_cube_integration")
114
+ >>> print(result)
115
+ """
116
+ payload = {"cube_query": cube_query}
117
+ if integration_id:
118
+ payload["integration_id"] = integration_id
119
+
120
+ response = requests.post(
121
+ self._sql_url,
122
+ json=payload,
123
+ headers={"Authorization": "Bearer " + self._api_key},
124
+ )
125
+ response.raise_for_status()
126
+ return response.json()
@@ -1,4 +1,4 @@
1
- from typing import Iterator
1
+ from typing import Iterator, Union, Optional
2
2
 
3
3
  import requests
4
4
 
@@ -109,7 +109,7 @@ class DefiniteKVStore:
109
109
  self._data = {}
110
110
  self._version_id = None
111
111
 
112
- def __getitem__(self, key: str) -> str | None:
112
+ def __getitem__(self, key: str) -> Optional[str]:
113
113
  """
114
114
  Gets the value for a given key.
115
115
 
@@ -190,3 +190,19 @@ class DefiniteKVStore:
190
190
  print(store)
191
191
  """
192
192
  return repr(self._data)
193
+
194
+ def get(self, key: str, default: Optional[str] = None) -> Optional[str]:
195
+ """
196
+ Gets the value for a given key with a default.
197
+
198
+ Args:
199
+ key (str): The key to retrieve the value for.
200
+ default: The default value to return if key doesn't exist.
201
+
202
+ Returns:
203
+ The value associated with the key, or default if the key does not exist.
204
+
205
+ Example:
206
+ value = store.get("key1", "default_value")
207
+ """
208
+ return self._data.get(key, default)
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "definite-sdk"
3
- version = "0.1.3"
3
+ version = "0.1.8"
4
4
  description = "Definite SDK for Python"
5
5
  authors = ["Definite <hello@definite.app>"]
6
6
  license = "MIT"
@@ -9,6 +9,11 @@ readme = "README.md"
9
9
  [tool.poetry.dependencies]
10
10
  python = "^3.9"
11
11
  requests = "^2.31.0"
12
+ dlt = {version = "^1.0", optional = true}
13
+ duckdb = {version = "^1.0", optional = true}
14
+
15
+ [tool.poetry.extras]
16
+ dlt = ["dlt", "duckdb"]
12
17
 
13
18
 
14
19
  [tool.poetry.group.dev.dependencies]
@@ -1,21 +0,0 @@
1
- Metadata-Version: 2.1
2
- Name: definite-sdk
3
- Version: 0.1.3
4
- Summary: Definite SDK for Python
5
- License: MIT
6
- Author: Definite
7
- Author-email: hello@definite.app
8
- Requires-Python: >=3.9,<4.0
9
- Classifier: License :: OSI Approved :: MIT License
10
- Classifier: Programming Language :: Python :: 3
11
- Classifier: Programming Language :: Python :: 3.9
12
- Classifier: Programming Language :: Python :: 3.10
13
- Classifier: Programming Language :: Python :: 3.11
14
- Classifier: Programming Language :: Python :: 3.12
15
- Requires-Dist: requests (>=2.31.0,<3.0.0)
16
- Description-Content-Type: text/markdown
17
-
18
- # Definite SDK
19
-
20
- A Python client for interacting with the Definite API, providing a convenient interface for key-value store operations.
21
-
@@ -1,3 +0,0 @@
1
- # Definite SDK
2
-
3
- A Python client for interacting with the Definite API, providing a convenient interface for key-value store operations.
@@ -1,23 +0,0 @@
1
- from definite_sdk.store import DefiniteKVStore
2
-
3
- API_URL = "https://api.definite.app"
4
-
5
-
6
- class DefiniteClient:
7
- """Client for interacting with the Definite API."""
8
-
9
- def __init__(self, api_key: str, api_url: str = API_URL):
10
- """Creates a definite client with the provided API key.
11
-
12
- See: https://docs.definite.app/definite-api for how to obtain an API key.
13
- """
14
- self.api_key = api_key
15
- self.api_url = api_url
16
-
17
- def get_kv_store(self, name: str) -> DefiniteKVStore:
18
- """Initializes a key-value store with the provided name.
19
-
20
- See DefiniteKVStore for more how to interact with the store.
21
- """
22
-
23
- return DefiniteKVStore(name, self.api_key, self.api_url)
File without changes