tonic-fabricate 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tonic_fabricate-1.0.0/MANIFEST.in +5 -0
- tonic_fabricate-1.0.0/PKG-INFO +162 -0
- tonic_fabricate-1.0.0/README.md +130 -0
- tonic_fabricate-1.0.0/examples/download.py +51 -0
- tonic_fabricate-1.0.0/examples/push.py +54 -0
- tonic_fabricate-1.0.0/fabricate_client/__init__.py +18 -0
- tonic_fabricate-1.0.0/fabricate_client/client.py +249 -0
- tonic_fabricate-1.0.0/pyproject.toml +52 -0
- tonic_fabricate-1.0.0/requirements-dev.txt +5 -0
- tonic_fabricate-1.0.0/requirements.txt +1 -0
- tonic_fabricate-1.0.0/setup.cfg +4 -0
- tonic_fabricate-1.0.0/setup.py +44 -0
- tonic_fabricate-1.0.0/tonic_fabricate.egg-info/PKG-INFO +162 -0
- tonic_fabricate-1.0.0/tonic_fabricate.egg-info/SOURCES.txt +15 -0
- tonic_fabricate-1.0.0/tonic_fabricate.egg-info/dependency_links.txt +1 -0
- tonic_fabricate-1.0.0/tonic_fabricate.egg-info/requires.txt +11 -0
- tonic_fabricate-1.0.0/tonic_fabricate.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tonic-fabricate
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: The official Fabricate client for Python
|
|
5
|
+
Home-page: https://github.com/fabricate-tools/client-python
|
|
6
|
+
Author: Fabricate Tools
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Project-URL: Homepage, https://github.com/fabricate-tools/client-python
|
|
9
|
+
Project-URL: Repository, https://github.com/fabricate-tools/client-python
|
|
10
|
+
Project-URL: Documentation, https://github.com/fabricate-tools/client-python#readme
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Requires-Python: >=3.8
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
Requires-Dist: requests>=2.25.0
|
|
23
|
+
Requires-Dist: tomli>=1.2.0; python_version < "3.11"
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: python-dotenv>=0.19.0; extra == "dev"
|
|
26
|
+
Requires-Dist: pytest>=6.0.0; extra == "dev"
|
|
27
|
+
Requires-Dist: black>=22.0.0; extra == "dev"
|
|
28
|
+
Requires-Dist: flake8>=4.0.0; extra == "dev"
|
|
29
|
+
Requires-Dist: mypy>=0.900; extra == "dev"
|
|
30
|
+
Dynamic: home-page
|
|
31
|
+
Dynamic: requires-python
|
|
32
|
+
|
|
33
|
+
# Fabricate Client
|
|
34
|
+
|
|
35
|
+
The official Fabricate client package for Python.
|
|
36
|
+
|
|
37
|
+
## Installation
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
pip install tonic-fabricate
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Usage
|
|
44
|
+
|
|
45
|
+
To generate and download data from Fabricate:
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
from fabricate_client import generate
|
|
49
|
+
|
|
50
|
+
generate(
|
|
51
|
+
# The workspace to use
|
|
52
|
+
workspace='Default',
|
|
53
|
+
|
|
54
|
+
# The name of the database to generate
|
|
55
|
+
database='ecommerce',
|
|
56
|
+
|
|
57
|
+
# The format to generate. Should be one of:
|
|
58
|
+
# - 'sql'
|
|
59
|
+
# - 'sqlite'
|
|
60
|
+
# - 'csv'
|
|
61
|
+
# - 'jsonl'
|
|
62
|
+
# - 'xml'
|
|
63
|
+
format='sql',
|
|
64
|
+
|
|
65
|
+
# The destination to save the data
|
|
66
|
+
dest='./data',
|
|
67
|
+
|
|
68
|
+
# Optional: Overwrite the destination if it exists
|
|
69
|
+
overwrite=True,
|
|
70
|
+
|
|
71
|
+
# Optional: Generate a single table
|
|
72
|
+
# entity='Customers',
|
|
73
|
+
)
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
To push data to an existing database:
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
from fabricate_client import generate
|
|
80
|
+
import os
|
|
81
|
+
|
|
82
|
+
generate(
|
|
83
|
+
# The workspace to use
|
|
84
|
+
workspace='Default',
|
|
85
|
+
|
|
86
|
+
# The name of the database in Fabricate
|
|
87
|
+
database='ecommerce',
|
|
88
|
+
|
|
89
|
+
# The connection details for the target database
|
|
90
|
+
connection={
|
|
91
|
+
# The host of the target database
|
|
92
|
+
'host': 'host.example.com',
|
|
93
|
+
|
|
94
|
+
# The port of the target database
|
|
95
|
+
'port': 5432,
|
|
96
|
+
|
|
97
|
+
# The name of the target database
|
|
98
|
+
'database_name': 'ecommerce',
|
|
99
|
+
|
|
100
|
+
# The username for the target database
|
|
101
|
+
'username': os.environ.get('FABRICATE_DATABASE_USERNAME'),
|
|
102
|
+
|
|
103
|
+
# The password for the target database
|
|
104
|
+
'password': os.environ.get('FABRICATE_DATABASE_PASSWORD'),
|
|
105
|
+
|
|
106
|
+
# Whether to use TLS for the connection
|
|
107
|
+
'tls': True,
|
|
108
|
+
},
|
|
109
|
+
)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
## Progress Tracking
|
|
113
|
+
|
|
114
|
+
You can track the progress of data generation using a callback function:
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
from fabricate_client import generate
|
|
118
|
+
|
|
119
|
+
def on_progress(data):
|
|
120
|
+
phase = data.get('phase', '')
|
|
121
|
+
percent = data.get('percentComplete', 0)
|
|
122
|
+
status = data.get('status', '')
|
|
123
|
+
|
|
124
|
+
phase_text = f"[{phase}] " if phase else ""
|
|
125
|
+
status_text = f", {status}" if status else ""
|
|
126
|
+
print(f"{phase_text}{percent}% complete{status_text}...")
|
|
127
|
+
|
|
128
|
+
generate(
|
|
129
|
+
workspace='Default',
|
|
130
|
+
database='ecommerce',
|
|
131
|
+
format='sql',
|
|
132
|
+
dest='./data',
|
|
133
|
+
on_progress=on_progress
|
|
134
|
+
)
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
## Environment Variables
|
|
138
|
+
|
|
139
|
+
The client will automatically use the following environment variables if they are set:
|
|
140
|
+
|
|
141
|
+
- `FABRICATE_API_KEY`: Your Fabricate API key
|
|
142
|
+
- `FABRICATE_API_URL`: The Fabricate API URL (defaults to https://fabricate.tonic.ai/api/v1)
|
|
143
|
+
|
|
144
|
+
## Error Handling
|
|
145
|
+
|
|
146
|
+
The client raises appropriate exceptions for various error conditions:
|
|
147
|
+
|
|
148
|
+
```python
|
|
149
|
+
from fabricate_client import generate
|
|
150
|
+
|
|
151
|
+
try:
|
|
152
|
+
generate(
|
|
153
|
+
workspace='Default',
|
|
154
|
+
database='ecommerce',
|
|
155
|
+
format='sql',
|
|
156
|
+
dest='./data'
|
|
157
|
+
)
|
|
158
|
+
except ValueError as e:
|
|
159
|
+
print(f"Invalid parameters: {e}")
|
|
160
|
+
except Exception as e:
|
|
161
|
+
print(f"Generation failed: {e}")
|
|
162
|
+
```
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
# Fabricate Client
|
|
2
|
+
|
|
3
|
+
The official Fabricate client package for Python.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install tonic-fabricate
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Usage
|
|
12
|
+
|
|
13
|
+
To generate and download data from Fabricate:
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
from fabricate_client import generate
|
|
17
|
+
|
|
18
|
+
generate(
|
|
19
|
+
# The workspace to use
|
|
20
|
+
workspace='Default',
|
|
21
|
+
|
|
22
|
+
# The name of the database to generate
|
|
23
|
+
database='ecommerce',
|
|
24
|
+
|
|
25
|
+
# The format to generate. Should be one of:
|
|
26
|
+
# - 'sql'
|
|
27
|
+
# - 'sqlite'
|
|
28
|
+
# - 'csv'
|
|
29
|
+
# - 'jsonl'
|
|
30
|
+
# - 'xml'
|
|
31
|
+
format='sql',
|
|
32
|
+
|
|
33
|
+
# The destination to save the data
|
|
34
|
+
dest='./data',
|
|
35
|
+
|
|
36
|
+
# Optional: Overwrite the destination if it exists
|
|
37
|
+
overwrite=True,
|
|
38
|
+
|
|
39
|
+
# Optional: Generate a single table
|
|
40
|
+
# entity='Customers',
|
|
41
|
+
)
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
To push data to an existing database:
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
from fabricate_client import generate
|
|
48
|
+
import os
|
|
49
|
+
|
|
50
|
+
generate(
|
|
51
|
+
# The workspace to use
|
|
52
|
+
workspace='Default',
|
|
53
|
+
|
|
54
|
+
# The name of the database in Fabricate
|
|
55
|
+
database='ecommerce',
|
|
56
|
+
|
|
57
|
+
# The connection details for the target database
|
|
58
|
+
connection={
|
|
59
|
+
# The host of the target database
|
|
60
|
+
'host': 'host.example.com',
|
|
61
|
+
|
|
62
|
+
# The port of the target database
|
|
63
|
+
'port': 5432,
|
|
64
|
+
|
|
65
|
+
# The name of the target database
|
|
66
|
+
'database_name': 'ecommerce',
|
|
67
|
+
|
|
68
|
+
# The username for the target database
|
|
69
|
+
'username': os.environ.get('FABRICATE_DATABASE_USERNAME'),
|
|
70
|
+
|
|
71
|
+
# The password for the target database
|
|
72
|
+
'password': os.environ.get('FABRICATE_DATABASE_PASSWORD'),
|
|
73
|
+
|
|
74
|
+
# Whether to use TLS for the connection
|
|
75
|
+
'tls': True,
|
|
76
|
+
},
|
|
77
|
+
)
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
## Progress Tracking
|
|
81
|
+
|
|
82
|
+
You can track the progress of data generation using a callback function:
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
from fabricate_client import generate
|
|
86
|
+
|
|
87
|
+
def on_progress(data):
|
|
88
|
+
phase = data.get('phase', '')
|
|
89
|
+
percent = data.get('percentComplete', 0)
|
|
90
|
+
status = data.get('status', '')
|
|
91
|
+
|
|
92
|
+
phase_text = f"[{phase}] " if phase else ""
|
|
93
|
+
status_text = f", {status}" if status else ""
|
|
94
|
+
print(f"{phase_text}{percent}% complete{status_text}...")
|
|
95
|
+
|
|
96
|
+
generate(
|
|
97
|
+
workspace='Default',
|
|
98
|
+
database='ecommerce',
|
|
99
|
+
format='sql',
|
|
100
|
+
dest='./data',
|
|
101
|
+
on_progress=on_progress
|
|
102
|
+
)
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
## Environment Variables
|
|
106
|
+
|
|
107
|
+
The client will automatically use the following environment variables if they are set:
|
|
108
|
+
|
|
109
|
+
- `FABRICATE_API_KEY`: Your Fabricate API key
|
|
110
|
+
- `FABRICATE_API_URL`: The Fabricate API URL (defaults to https://fabricate.tonic.ai/api/v1)
|
|
111
|
+
|
|
112
|
+
## Error Handling
|
|
113
|
+
|
|
114
|
+
The client raises appropriate exceptions for various error conditions:
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
from fabricate_client import generate
|
|
118
|
+
|
|
119
|
+
try:
|
|
120
|
+
generate(
|
|
121
|
+
workspace='Default',
|
|
122
|
+
database='ecommerce',
|
|
123
|
+
format='sql',
|
|
124
|
+
dest='./data'
|
|
125
|
+
)
|
|
126
|
+
except ValueError as e:
|
|
127
|
+
print(f"Invalid parameters: {e}")
|
|
128
|
+
except Exception as e:
|
|
129
|
+
print(f"Generation failed: {e}")
|
|
130
|
+
```
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
Example script for downloading data from Fabricate.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
|
|
9
|
+
|
|
10
|
+
from fabricate_client import generate
|
|
11
|
+
|
|
12
|
+
# Load environment variables if python-dotenv is available
|
|
13
|
+
try:
|
|
14
|
+
from dotenv import load_dotenv
|
|
15
|
+
load_dotenv()
|
|
16
|
+
except ImportError:
|
|
17
|
+
pass
|
|
18
|
+
|
|
19
|
+
def on_progress(data):
|
|
20
|
+
"""Progress callback function."""
|
|
21
|
+
phase = data.get('phase', '')
|
|
22
|
+
percent = data.get('percentComplete', 0)
|
|
23
|
+
status = data.get('status', '')
|
|
24
|
+
|
|
25
|
+
phase_text = f"[{phase}] " if phase else ""
|
|
26
|
+
status_text = f", {status}" if status else ""
|
|
27
|
+
print(f"{phase_text}{percent}% complete{status_text}...")
|
|
28
|
+
|
|
29
|
+
if __name__ == "__main__":
|
|
30
|
+
print("starting...")
|
|
31
|
+
|
|
32
|
+
# Prepare arguments
|
|
33
|
+
kwargs = {
|
|
34
|
+
'database': 'XML',
|
|
35
|
+
'workspace': 'Default',
|
|
36
|
+
'format': 'xml',
|
|
37
|
+
'dest': './tmp/xml',
|
|
38
|
+
'overwrite': True,
|
|
39
|
+
'on_progress': on_progress,
|
|
40
|
+
# Uncomment to generate a single table
|
|
41
|
+
# 'entity': 'Customers',
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
# Add api_url if available
|
|
45
|
+
api_url = os.environ.get('FABRICATE_API_URL')
|
|
46
|
+
if api_url:
|
|
47
|
+
kwargs['api_url'] = api_url
|
|
48
|
+
|
|
49
|
+
generate(**kwargs)
|
|
50
|
+
|
|
51
|
+
print("done.")
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
Example script for pushing data to a database using Fabricate.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
|
|
9
|
+
|
|
10
|
+
from fabricate_client import generate
|
|
11
|
+
|
|
12
|
+
# Load environment variables if python-dotenv is available
|
|
13
|
+
try:
|
|
14
|
+
from dotenv import load_dotenv
|
|
15
|
+
load_dotenv()
|
|
16
|
+
except ImportError:
|
|
17
|
+
pass
|
|
18
|
+
|
|
19
|
+
def on_progress(data):
|
|
20
|
+
"""Progress callback function."""
|
|
21
|
+
phase = data.get('phase', '')
|
|
22
|
+
percent = data.get('percentComplete', 0)
|
|
23
|
+
status = data.get('status', '')
|
|
24
|
+
|
|
25
|
+
phase_text = f"[{phase}] " if phase else ""
|
|
26
|
+
status_text = f", {status}" if status else ""
|
|
27
|
+
print(f"{phase_text}{percent}% complete{status_text}...")
|
|
28
|
+
|
|
29
|
+
if __name__ == "__main__":
|
|
30
|
+
print("starting...")
|
|
31
|
+
|
|
32
|
+
# Prepare arguments
|
|
33
|
+
kwargs = {
|
|
34
|
+
'database': 'ecommerce',
|
|
35
|
+
'workspace': 'Default',
|
|
36
|
+
'connection': {
|
|
37
|
+
'host': 'localhost',
|
|
38
|
+
'port': 5432,
|
|
39
|
+
'database_name': 'ecommerce',
|
|
40
|
+
'username': os.environ.get('FABRICATE_DATABASE_USERNAME'),
|
|
41
|
+
'password': os.environ.get('FABRICATE_DATABASE_PASSWORD'),
|
|
42
|
+
'tls': False,
|
|
43
|
+
},
|
|
44
|
+
'on_progress': on_progress,
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
# Add api_url if available
|
|
48
|
+
api_url = os.environ.get('FABRICATE_API_URL')
|
|
49
|
+
if api_url:
|
|
50
|
+
kwargs['api_url'] = api_url
|
|
51
|
+
|
|
52
|
+
generate(**kwargs)
|
|
53
|
+
|
|
54
|
+
print("done.")
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Fabricate Client - The official Fabricate client for Python.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from .client import generate
|
|
6
|
+
try:
|
|
7
|
+
import tomllib
|
|
8
|
+
except ImportError:
|
|
9
|
+
import tomli as tomllib
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
# Read version from pyproject.toml
|
|
13
|
+
_pyproject_path = Path(__file__).parent.parent / "pyproject.toml"
|
|
14
|
+
with open(_pyproject_path, "rb") as f:
|
|
15
|
+
_pyproject = tomllib.load(f)
|
|
16
|
+
__version__ = _pyproject["project"]["version"]
|
|
17
|
+
|
|
18
|
+
__all__ = ["generate"]
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import time
|
|
3
|
+
import zipfile
|
|
4
|
+
import shutil
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Optional, Callable, Dict, Any, Union
|
|
7
|
+
import requests
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def generate(
|
|
11
|
+
api_key: Optional[str] = None,
|
|
12
|
+
api_url: str = "https://fabricate.tonic.ai/api/v1",
|
|
13
|
+
database: Optional[str] = None,
|
|
14
|
+
workspace: Optional[str] = None,
|
|
15
|
+
entity: Optional[str] = None,
|
|
16
|
+
format: Optional[str] = None,
|
|
17
|
+
overrides: Optional[Dict[str, Any]] = None,
|
|
18
|
+
on_progress: Optional[Callable[[Dict[str, Any]], None]] = None,
|
|
19
|
+
dest: Optional[str] = None,
|
|
20
|
+
unzip: bool = True,
|
|
21
|
+
overwrite: bool = False,
|
|
22
|
+
connection: Optional[Dict[str, Any]] = None,
|
|
23
|
+
) -> None:
|
|
24
|
+
"""
|
|
25
|
+
Generates data for a given database.
|
|
26
|
+
|
|
27
|
+
Args:
|
|
28
|
+
api_key: The API key for authentication. Defaults to FABRICATE_API_KEY env var.
|
|
29
|
+
api_url: The API URL. Defaults to https://fabricate.tonic.ai/api/v1
|
|
30
|
+
database: The name of the database to generate (required)
|
|
31
|
+
workspace: The workspace to use (required)
|
|
32
|
+
entity: Optional single entity/table to generate
|
|
33
|
+
format: The output format (sql, sqlite, csv, jsonl, xml)
|
|
34
|
+
overrides: Optional override parameters
|
|
35
|
+
on_progress: Optional progress callback function
|
|
36
|
+
dest: Destination path for downloaded files (required unless connection provided)
|
|
37
|
+
unzip: Whether to unzip downloaded files (default: True)
|
|
38
|
+
overwrite: Whether to overwrite existing destination (default: False)
|
|
39
|
+
connection: Database connection details for direct push
|
|
40
|
+
|
|
41
|
+
Raises:
|
|
42
|
+
ValueError: If required parameters are missing or invalid
|
|
43
|
+
requests.HTTPError: If API requests fail
|
|
44
|
+
Exception: If task execution fails
|
|
45
|
+
"""
|
|
46
|
+
# Get API key from parameter or environment
|
|
47
|
+
if api_key is None:
|
|
48
|
+
api_key = os.environ.get('FABRICATE_API_KEY')
|
|
49
|
+
|
|
50
|
+
# Validate required parameters
|
|
51
|
+
if not api_key:
|
|
52
|
+
raise ValueError('api_key is required')
|
|
53
|
+
|
|
54
|
+
if not database:
|
|
55
|
+
raise ValueError('database is required')
|
|
56
|
+
|
|
57
|
+
if not workspace:
|
|
58
|
+
raise ValueError('workspace is required')
|
|
59
|
+
|
|
60
|
+
if not format and not connection:
|
|
61
|
+
raise ValueError('format or connection is required')
|
|
62
|
+
|
|
63
|
+
if format and connection:
|
|
64
|
+
raise ValueError('format and connection cannot both be provided')
|
|
65
|
+
|
|
66
|
+
if not dest and not connection:
|
|
67
|
+
raise ValueError('dest is required unless connection is provided')
|
|
68
|
+
|
|
69
|
+
if dest and Path(dest).exists() and not overwrite:
|
|
70
|
+
raise ValueError('dest already exists')
|
|
71
|
+
|
|
72
|
+
# Create the generation task
|
|
73
|
+
headers = {'Authorization': f'Bearer {api_key}'}
|
|
74
|
+
payload = {
|
|
75
|
+
'format': format,
|
|
76
|
+
'connection': connection,
|
|
77
|
+
'database': database,
|
|
78
|
+
'entity': entity,
|
|
79
|
+
'overrides': overrides,
|
|
80
|
+
'workspace': workspace
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
try:
|
|
84
|
+
response = requests.post(
|
|
85
|
+
f'{api_url}/generate_tasks',
|
|
86
|
+
json=payload,
|
|
87
|
+
headers=headers
|
|
88
|
+
)
|
|
89
|
+
response.raise_for_status()
|
|
90
|
+
task = response.json()
|
|
91
|
+
except requests.exceptions.HTTPError as e:
|
|
92
|
+
if e.response.headers.get('content-type') == 'application/json':
|
|
93
|
+
error_data = e.response.json()
|
|
94
|
+
raise Exception(error_data.get('error', str(e)))
|
|
95
|
+
else:
|
|
96
|
+
raise e
|
|
97
|
+
|
|
98
|
+
if task.get('error'):
|
|
99
|
+
raise Exception(task['error'])
|
|
100
|
+
|
|
101
|
+
# Poll until completion
|
|
102
|
+
task = _poll(task['id'], api_url, api_key, on_progress)
|
|
103
|
+
|
|
104
|
+
data_url = task.get('data_url')
|
|
105
|
+
error = task.get('error')
|
|
106
|
+
|
|
107
|
+
if error:
|
|
108
|
+
print(f'Fabricate API returned an error: {error}')
|
|
109
|
+
raise Exception(error)
|
|
110
|
+
|
|
111
|
+
# Download data if not pushing to connection
|
|
112
|
+
if connection is None and dest is not None and data_url is not None:
|
|
113
|
+
temp_path = f'{dest}.tmp'
|
|
114
|
+
|
|
115
|
+
try:
|
|
116
|
+
# Download to temp location first
|
|
117
|
+
_download(data_url, temp_path)
|
|
118
|
+
|
|
119
|
+
if unzip and task.get('content_type') == 'application/zip':
|
|
120
|
+
_unzip_file(temp_path, dest)
|
|
121
|
+
else:
|
|
122
|
+
# Move temp file to final destination
|
|
123
|
+
Path(temp_path).rename(dest)
|
|
124
|
+
finally:
|
|
125
|
+
# Clean up temp file if it exists
|
|
126
|
+
temp_file = Path(temp_path)
|
|
127
|
+
if temp_file.exists():
|
|
128
|
+
temp_file.unlink()
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _poll(
|
|
132
|
+
task_id: str,
|
|
133
|
+
api_url: str,
|
|
134
|
+
api_key: str,
|
|
135
|
+
on_progress: Optional[Callable[[Dict[str, Any]], None]] = None
|
|
136
|
+
) -> Dict[str, Any]:
|
|
137
|
+
"""
|
|
138
|
+
Polls the generate task API until the task is completed.
|
|
139
|
+
|
|
140
|
+
Args:
|
|
141
|
+
task_id: The ID of the task to poll
|
|
142
|
+
api_url: The API URL
|
|
143
|
+
api_key: The API key
|
|
144
|
+
on_progress: Optional progress callback
|
|
145
|
+
|
|
146
|
+
Returns:
|
|
147
|
+
The completed task object
|
|
148
|
+
|
|
149
|
+
Raises:
|
|
150
|
+
Exception: If task fails or API returns an error
|
|
151
|
+
"""
|
|
152
|
+
headers = {'Authorization': f'Bearer {api_key}'}
|
|
153
|
+
last_progress = None
|
|
154
|
+
last_context = None
|
|
155
|
+
|
|
156
|
+
while True:
|
|
157
|
+
response = requests.get(
|
|
158
|
+
f'{api_url}/generate_tasks/{task_id}',
|
|
159
|
+
headers=headers
|
|
160
|
+
)
|
|
161
|
+
response.raise_for_status()
|
|
162
|
+
task = response.json()
|
|
163
|
+
|
|
164
|
+
if task.get('completed'):
|
|
165
|
+
_update_progress({'progress': 100}, on_progress, last_progress, last_context)
|
|
166
|
+
return task
|
|
167
|
+
elif task.get('error'):
|
|
168
|
+
raise Exception(task['error'])
|
|
169
|
+
else:
|
|
170
|
+
last_progress, last_context = _update_progress(
|
|
171
|
+
task, on_progress, last_progress, last_context
|
|
172
|
+
)
|
|
173
|
+
time.sleep(1)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _update_progress(
|
|
177
|
+
task: Dict[str, Any],
|
|
178
|
+
on_progress: Optional[Callable[[Dict[str, Any]], None]],
|
|
179
|
+
last_progress: Optional[Any],
|
|
180
|
+
last_context: Optional[str]
|
|
181
|
+
) -> tuple:
|
|
182
|
+
"""
|
|
183
|
+
Updates progress if callback is provided and progress has changed.
|
|
184
|
+
|
|
185
|
+
Args:
|
|
186
|
+
task: The task object containing progress information
|
|
187
|
+
on_progress: Optional progress callback
|
|
188
|
+
last_progress: The last reported progress value
|
|
189
|
+
last_context: The last reported context
|
|
190
|
+
|
|
191
|
+
Returns:
|
|
192
|
+
Tuple of (current_progress, current_context)
|
|
193
|
+
"""
|
|
194
|
+
if not on_progress:
|
|
195
|
+
return last_progress, last_context
|
|
196
|
+
|
|
197
|
+
current_progress = task.get('progress')
|
|
198
|
+
current_context = task.get('context')
|
|
199
|
+
|
|
200
|
+
if current_progress != last_progress or current_context != last_context:
|
|
201
|
+
progress_data = {
|
|
202
|
+
'percentComplete': current_progress,
|
|
203
|
+
'status': current_context,
|
|
204
|
+
'phase': task.get('phase')
|
|
205
|
+
}
|
|
206
|
+
on_progress(progress_data)
|
|
207
|
+
return current_progress, current_context
|
|
208
|
+
|
|
209
|
+
return last_progress, last_context
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _unzip_file(zip_path: str, target_dir: str) -> None:
|
|
213
|
+
"""
|
|
214
|
+
Unzips a file to a target directory.
|
|
215
|
+
|
|
216
|
+
Args:
|
|
217
|
+
zip_path: Path to the zip file
|
|
218
|
+
target_dir: Target directory to extract to
|
|
219
|
+
"""
|
|
220
|
+
with zipfile.ZipFile(zip_path, 'r') as zip_ref:
|
|
221
|
+
zip_ref.extractall(target_dir)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _download(url: str, dest_path: str) -> None:
|
|
225
|
+
"""
|
|
226
|
+
Downloads a file from a URL and saves it to a path.
|
|
227
|
+
|
|
228
|
+
Args:
|
|
229
|
+
url: The URL to download from
|
|
230
|
+
dest_path: The path to save the file to
|
|
231
|
+
"""
|
|
232
|
+
# Ensure the directory exists
|
|
233
|
+
dest_file = Path(dest_path)
|
|
234
|
+
dest_dir = dest_file.parent
|
|
235
|
+
|
|
236
|
+
if dest_dir != Path('.'):
|
|
237
|
+
# Remove directory if it exists
|
|
238
|
+
if dest_dir.exists():
|
|
239
|
+
shutil.rmtree(dest_dir)
|
|
240
|
+
|
|
241
|
+
# Create directory
|
|
242
|
+
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
243
|
+
|
|
244
|
+
# Download the file
|
|
245
|
+
with requests.get(url, stream=True) as response:
|
|
246
|
+
response.raise_for_status()
|
|
247
|
+
with open(dest_path, 'wb') as f:
|
|
248
|
+
for chunk in response.iter_content(chunk_size=8192):
|
|
249
|
+
f.write(chunk)
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=45", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "tonic-fabricate"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "The official Fabricate client for Python"
|
|
9
|
+
authors = [{name = "Fabricate Tools"}]
|
|
10
|
+
readme = "README.md"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
requires-python = ">=3.8"
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 4 - Beta",
|
|
15
|
+
"Intended Audience :: Developers",
|
|
16
|
+
|
|
17
|
+
"Operating System :: OS Independent",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.8",
|
|
20
|
+
"Programming Language :: Python :: 3.9",
|
|
21
|
+
"Programming Language :: Python :: 3.10",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
]
|
|
25
|
+
dependencies = [
|
|
26
|
+
"requests>=2.25.0",
|
|
27
|
+
"tomli>=1.2.0; python_version<'3.11'",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
[project.optional-dependencies]
|
|
31
|
+
dev = [
|
|
32
|
+
"python-dotenv>=0.19.0",
|
|
33
|
+
"pytest>=6.0.0",
|
|
34
|
+
"black>=22.0.0",
|
|
35
|
+
"flake8>=4.0.0",
|
|
36
|
+
"mypy>=0.900",
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
[project.urls]
|
|
40
|
+
Homepage = "https://github.com/fabricate-tools/client-python"
|
|
41
|
+
Repository = "https://github.com/fabricate-tools/client-python"
|
|
42
|
+
Documentation = "https://github.com/fabricate-tools/client-python#readme"
|
|
43
|
+
|
|
44
|
+
[tool.black]
|
|
45
|
+
line-length = 88
|
|
46
|
+
target-version = ['py38']
|
|
47
|
+
|
|
48
|
+
[tool.mypy]
|
|
49
|
+
python_version = "3.8"
|
|
50
|
+
warn_return_any = true
|
|
51
|
+
warn_unused_configs = true
|
|
52
|
+
disallow_untyped_defs = true
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
requests>=2.25.0
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from setuptools import setup, find_packages
|
|
2
|
+
try:
|
|
3
|
+
import tomllib
|
|
4
|
+
except ImportError:
|
|
5
|
+
import tomli as tomllib
|
|
6
|
+
|
|
7
|
+
with open("README.md", "r", encoding="utf-8") as fh:
|
|
8
|
+
long_description = fh.read()
|
|
9
|
+
|
|
10
|
+
with open("pyproject.toml", "rb") as f:
|
|
11
|
+
pyproject = tomllib.load(f)
|
|
12
|
+
version = pyproject["project"]["version"]
|
|
13
|
+
|
|
14
|
+
setup(
|
|
15
|
+
name="tonic-fabricate",
|
|
16
|
+
version=version,
|
|
17
|
+
author="Fabricate Tools",
|
|
18
|
+
description="The official Fabricate client for Python",
|
|
19
|
+
long_description=long_description,
|
|
20
|
+
long_description_content_type="text/markdown",
|
|
21
|
+
url="https://github.com/fabricate-tools/client-python",
|
|
22
|
+
packages=find_packages(),
|
|
23
|
+
classifiers=[
|
|
24
|
+
"Development Status :: 4 - Beta",
|
|
25
|
+
"Intended Audience :: Developers",
|
|
26
|
+
"License :: OSI Approved :: MIT License",
|
|
27
|
+
"Operating System :: OS Independent",
|
|
28
|
+
"Programming Language :: Python :: 3",
|
|
29
|
+
"Programming Language :: Python :: 3.8",
|
|
30
|
+
"Programming Language :: Python :: 3.9",
|
|
31
|
+
"Programming Language :: Python :: 3.10",
|
|
32
|
+
"Programming Language :: Python :: 3.11",
|
|
33
|
+
"Programming Language :: Python :: 3.12",
|
|
34
|
+
],
|
|
35
|
+
python_requires=">=3.8",
|
|
36
|
+
install_requires=[
|
|
37
|
+
"requests>=2.25.0",
|
|
38
|
+
],
|
|
39
|
+
extras_require={
|
|
40
|
+
"dev": [
|
|
41
|
+
"python-dotenv>=0.19.0",
|
|
42
|
+
],
|
|
43
|
+
},
|
|
44
|
+
)
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tonic-fabricate
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: The official Fabricate client for Python
|
|
5
|
+
Home-page: https://github.com/fabricate-tools/client-python
|
|
6
|
+
Author: Fabricate Tools
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Project-URL: Homepage, https://github.com/fabricate-tools/client-python
|
|
9
|
+
Project-URL: Repository, https://github.com/fabricate-tools/client-python
|
|
10
|
+
Project-URL: Documentation, https://github.com/fabricate-tools/client-python#readme
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Requires-Python: >=3.8
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
Requires-Dist: requests>=2.25.0
|
|
23
|
+
Requires-Dist: tomli>=1.2.0; python_version < "3.11"
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: python-dotenv>=0.19.0; extra == "dev"
|
|
26
|
+
Requires-Dist: pytest>=6.0.0; extra == "dev"
|
|
27
|
+
Requires-Dist: black>=22.0.0; extra == "dev"
|
|
28
|
+
Requires-Dist: flake8>=4.0.0; extra == "dev"
|
|
29
|
+
Requires-Dist: mypy>=0.900; extra == "dev"
|
|
30
|
+
Dynamic: home-page
|
|
31
|
+
Dynamic: requires-python
|
|
32
|
+
|
|
33
|
+
# Fabricate Client
|
|
34
|
+
|
|
35
|
+
The official Fabricate client package for Python.
|
|
36
|
+
|
|
37
|
+
## Installation
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
pip install tonic-fabricate
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Usage
|
|
44
|
+
|
|
45
|
+
To generate and download data from Fabricate:
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
from fabricate_client import generate
|
|
49
|
+
|
|
50
|
+
generate(
|
|
51
|
+
# The workspace to use
|
|
52
|
+
workspace='Default',
|
|
53
|
+
|
|
54
|
+
# The name of the database to generate
|
|
55
|
+
database='ecommerce',
|
|
56
|
+
|
|
57
|
+
# The format to generate. Should be one of:
|
|
58
|
+
# - 'sql'
|
|
59
|
+
# - 'sqlite'
|
|
60
|
+
# - 'csv'
|
|
61
|
+
# - 'jsonl'
|
|
62
|
+
# - 'xml'
|
|
63
|
+
format='sql',
|
|
64
|
+
|
|
65
|
+
# The destination to save the data
|
|
66
|
+
dest='./data',
|
|
67
|
+
|
|
68
|
+
# Optional: Overwrite the destination if it exists
|
|
69
|
+
overwrite=True,
|
|
70
|
+
|
|
71
|
+
# Optional: Generate a single table
|
|
72
|
+
# entity='Customers',
|
|
73
|
+
)
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
To push data to an existing database:
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
from fabricate_client import generate
|
|
80
|
+
import os
|
|
81
|
+
|
|
82
|
+
generate(
|
|
83
|
+
# The workspace to use
|
|
84
|
+
workspace='Default',
|
|
85
|
+
|
|
86
|
+
# The name of the database in Fabricate
|
|
87
|
+
database='ecommerce',
|
|
88
|
+
|
|
89
|
+
# The connection details for the target database
|
|
90
|
+
connection={
|
|
91
|
+
# The host of the target database
|
|
92
|
+
'host': 'host.example.com',
|
|
93
|
+
|
|
94
|
+
# The port of the target database
|
|
95
|
+
'port': 5432,
|
|
96
|
+
|
|
97
|
+
# The name of the target database
|
|
98
|
+
'database_name': 'ecommerce',
|
|
99
|
+
|
|
100
|
+
# The username for the target database
|
|
101
|
+
'username': os.environ.get('FABRICATE_DATABASE_USERNAME'),
|
|
102
|
+
|
|
103
|
+
# The password for the target database
|
|
104
|
+
'password': os.environ.get('FABRICATE_DATABASE_PASSWORD'),
|
|
105
|
+
|
|
106
|
+
# Whether to use TLS for the connection
|
|
107
|
+
'tls': True,
|
|
108
|
+
},
|
|
109
|
+
)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
## Progress Tracking
|
|
113
|
+
|
|
114
|
+
You can track the progress of data generation using a callback function:
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
from fabricate_client import generate
|
|
118
|
+
|
|
119
|
+
def on_progress(data):
|
|
120
|
+
phase = data.get('phase', '')
|
|
121
|
+
percent = data.get('percentComplete', 0)
|
|
122
|
+
status = data.get('status', '')
|
|
123
|
+
|
|
124
|
+
phase_text = f"[{phase}] " if phase else ""
|
|
125
|
+
status_text = f", {status}" if status else ""
|
|
126
|
+
print(f"{phase_text}{percent}% complete{status_text}...")
|
|
127
|
+
|
|
128
|
+
generate(
|
|
129
|
+
workspace='Default',
|
|
130
|
+
database='ecommerce',
|
|
131
|
+
format='sql',
|
|
132
|
+
dest='./data',
|
|
133
|
+
on_progress=on_progress
|
|
134
|
+
)
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
## Environment Variables
|
|
138
|
+
|
|
139
|
+
The client will automatically use the following environment variables if they are set:
|
|
140
|
+
|
|
141
|
+
- `FABRICATE_API_KEY`: Your Fabricate API key
|
|
142
|
+
- `FABRICATE_API_URL`: The Fabricate API URL (defaults to https://fabricate.tonic.ai/api/v1)
|
|
143
|
+
|
|
144
|
+
## Error Handling
|
|
145
|
+
|
|
146
|
+
The client raises appropriate exceptions for various error conditions:
|
|
147
|
+
|
|
148
|
+
```python
|
|
149
|
+
from fabricate_client import generate
|
|
150
|
+
|
|
151
|
+
try:
|
|
152
|
+
generate(
|
|
153
|
+
workspace='Default',
|
|
154
|
+
database='ecommerce',
|
|
155
|
+
format='sql',
|
|
156
|
+
dest='./data'
|
|
157
|
+
)
|
|
158
|
+
except ValueError as e:
|
|
159
|
+
print(f"Invalid parameters: {e}")
|
|
160
|
+
except Exception as e:
|
|
161
|
+
print(f"Generation failed: {e}")
|
|
162
|
+
```
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
MANIFEST.in
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
requirements-dev.txt
|
|
5
|
+
requirements.txt
|
|
6
|
+
setup.py
|
|
7
|
+
examples/download.py
|
|
8
|
+
examples/push.py
|
|
9
|
+
fabricate_client/__init__.py
|
|
10
|
+
fabricate_client/client.py
|
|
11
|
+
tonic_fabricate.egg-info/PKG-INFO
|
|
12
|
+
tonic_fabricate.egg-info/SOURCES.txt
|
|
13
|
+
tonic_fabricate.egg-info/dependency_links.txt
|
|
14
|
+
tonic_fabricate.egg-info/requires.txt
|
|
15
|
+
tonic_fabricate.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
fabricate_client
|