flexorch-sdk 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flexorch_sdk/__init__.py +63 -0
- flexorch_sdk/_transport.py +123 -0
- flexorch_sdk/client.py +184 -0
- flexorch_sdk/errors.py +75 -0
- flexorch_sdk/models/__init__.py +6 -0
- flexorch_sdk/models/connector.py +45 -0
- flexorch_sdk/models/dataset.py +106 -0
- flexorch_sdk/models/job.py +85 -0
- flexorch_sdk/models/search.py +29 -0
- flexorch_sdk/resources/__init__.py +13 -0
- flexorch_sdk/resources/connectors.py +54 -0
- flexorch_sdk/resources/datasets.py +24 -0
- flexorch_sdk/resources/jobs.py +24 -0
- flexorch_sdk/resources/usage.py +47 -0
- flexorch_sdk/resources/webhooks.py +59 -0
- flexorch_sdk-0.1.0.dist-info/METADATA +344 -0
- flexorch_sdk-0.1.0.dist-info/RECORD +19 -0
- flexorch_sdk-0.1.0.dist-info/WHEEL +4 -0
- flexorch_sdk-0.1.0.dist-info/licenses/LICENSE +21 -0
flexorch_sdk/__init__.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""
|
|
2
|
+
flexorch-sdk — Python SDK for the FlexOrch API.
|
|
3
|
+
|
|
4
|
+
from flexorch_sdk import FlexOrchClient
|
|
5
|
+
|
|
6
|
+
client = FlexOrchClient("fx_your_key_here")
|
|
7
|
+
|
|
8
|
+
# Process a document and wait for the result
|
|
9
|
+
job = client.process("contract.pdf", locale="tr").wait()
|
|
10
|
+
print(job.quality_grade) # "A"
|
|
11
|
+
|
|
12
|
+
# Download the dataset as JSONL
|
|
13
|
+
dataset = job.dataset()
|
|
14
|
+
dataset.export("jsonl", path="output.jsonl")
|
|
15
|
+
|
|
16
|
+
# Or use the context manager
|
|
17
|
+
with FlexOrchClient("fx_...") as client:
|
|
18
|
+
jobs = client.process_many(["a.pdf", "b.pdf"])
|
|
19
|
+
for job in jobs:
|
|
20
|
+
job.wait()
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from .client import FlexOrchClient
|
|
24
|
+
from .models.job import Job
|
|
25
|
+
from .models.dataset import Dataset
|
|
26
|
+
from .models.connector import Connector, ConnectorTestResult
|
|
27
|
+
from .models.search import SearchResult
|
|
28
|
+
from .resources.usage import UsageSnapshot
|
|
29
|
+
from .resources.webhooks import Webhook
|
|
30
|
+
from .errors import (
|
|
31
|
+
FlexOrchError,
|
|
32
|
+
AuthError,
|
|
33
|
+
QuotaError,
|
|
34
|
+
RateLimitError,
|
|
35
|
+
NotFoundError,
|
|
36
|
+
ValidationError,
|
|
37
|
+
ServerError,
|
|
38
|
+
JobFailedError,
|
|
39
|
+
TimeoutError,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
__version__ = "0.1.0"
|
|
43
|
+
|
|
44
|
+
__all__ = [
|
|
45
|
+
"FlexOrchClient",
|
|
46
|
+
"Job",
|
|
47
|
+
"Dataset",
|
|
48
|
+
"Connector",
|
|
49
|
+
"ConnectorTestResult",
|
|
50
|
+
"SearchResult",
|
|
51
|
+
"UsageSnapshot",
|
|
52
|
+
"Webhook",
|
|
53
|
+
"FlexOrchError",
|
|
54
|
+
"AuthError",
|
|
55
|
+
"QuotaError",
|
|
56
|
+
"RateLimitError",
|
|
57
|
+
"NotFoundError",
|
|
58
|
+
"ValidationError",
|
|
59
|
+
"ServerError",
|
|
60
|
+
"JobFailedError",
|
|
61
|
+
"TimeoutError",
|
|
62
|
+
"__version__",
|
|
63
|
+
]
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
import httpx
|
|
7
|
+
|
|
8
|
+
from .errors import (
|
|
9
|
+
AuthError,
|
|
10
|
+
FlexOrchError,
|
|
11
|
+
NotFoundError,
|
|
12
|
+
QuotaError,
|
|
13
|
+
RateLimitError,
|
|
14
|
+
ServerError,
|
|
15
|
+
ValidationError,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
_DEFAULT_BASE_URL = "https://api.flexorch.com/v1"
|
|
19
|
+
_DEFAULT_TIMEOUT = 30.0
|
|
20
|
+
_DEFAULT_MAX_RETRIES = 3
|
|
21
|
+
_RETRY_STATUSES = {429, 500, 502, 503, 504}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _parse_error(response: httpx.Response) -> FlexOrchError:
|
|
25
|
+
status = response.status_code
|
|
26
|
+
try:
|
|
27
|
+
body = response.json()
|
|
28
|
+
error_block = body.get("error", {})
|
|
29
|
+
code = error_block.get("code", "")
|
|
30
|
+
message = error_block.get("message", response.text)
|
|
31
|
+
except Exception:
|
|
32
|
+
code = ""
|
|
33
|
+
message = response.text or f"HTTP {status}"
|
|
34
|
+
|
|
35
|
+
if status == 401:
|
|
36
|
+
return AuthError(message, status_code=status, error_code=code)
|
|
37
|
+
if status in (402,):
|
|
38
|
+
return QuotaError(message, status_code=status, error_code=code)
|
|
39
|
+
if status == 404:
|
|
40
|
+
return NotFoundError(message, status_code=status, error_code=code)
|
|
41
|
+
if status == 422:
|
|
42
|
+
return ValidationError(message, status_code=status, error_code=code)
|
|
43
|
+
if status == 429:
|
|
44
|
+
retry_after = int(response.headers.get("Retry-After", 60))
|
|
45
|
+
return RateLimitError(message, retry_after=retry_after)
|
|
46
|
+
if status >= 500:
|
|
47
|
+
return ServerError(message, status_code=status, error_code=code)
|
|
48
|
+
return FlexOrchError(message, status_code=status, error_code=code)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class Transport:
|
|
52
|
+
def __init__(
|
|
53
|
+
self,
|
|
54
|
+
api_key: str,
|
|
55
|
+
base_url: str = _DEFAULT_BASE_URL,
|
|
56
|
+
timeout: float = _DEFAULT_TIMEOUT,
|
|
57
|
+
max_retries: int = _DEFAULT_MAX_RETRIES,
|
|
58
|
+
) -> None:
|
|
59
|
+
self._base_url = base_url.rstrip("/")
|
|
60
|
+
self._max_retries = max_retries
|
|
61
|
+
self._client = httpx.Client(
|
|
62
|
+
headers={
|
|
63
|
+
"X-API-KEY": api_key,
|
|
64
|
+
"User-Agent": "flexorch-sdk-python/0.1.0",
|
|
65
|
+
},
|
|
66
|
+
timeout=timeout,
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
def _url(self, path: str) -> str:
|
|
70
|
+
return f"{self._base_url}/{path.lstrip('/')}"
|
|
71
|
+
|
|
72
|
+
def _request(self, method: str, path: str, **kwargs: Any) -> Any:
|
|
73
|
+
url = self._url(path)
|
|
74
|
+
last_exc: Exception | None = None
|
|
75
|
+
|
|
76
|
+
for attempt in range(self._max_retries):
|
|
77
|
+
try:
|
|
78
|
+
response = self._client.request(method, url, **kwargs)
|
|
79
|
+
except httpx.TimeoutException as exc:
|
|
80
|
+
last_exc = exc
|
|
81
|
+
if attempt < self._max_retries - 1:
|
|
82
|
+
time.sleep(2 ** attempt)
|
|
83
|
+
continue
|
|
84
|
+
|
|
85
|
+
if response.status_code in _RETRY_STATUSES and attempt < self._max_retries - 1:
|
|
86
|
+
wait = int(response.headers.get("Retry-After", 2 ** attempt))
|
|
87
|
+
time.sleep(wait)
|
|
88
|
+
continue
|
|
89
|
+
|
|
90
|
+
if response.is_error:
|
|
91
|
+
raise _parse_error(response)
|
|
92
|
+
|
|
93
|
+
if response.status_code == 204 or not response.content:
|
|
94
|
+
return None
|
|
95
|
+
return response.json()
|
|
96
|
+
|
|
97
|
+
raise FlexOrchError(f"Request failed after {self._max_retries} attempts: {last_exc}")
|
|
98
|
+
|
|
99
|
+
def get(self, path: str, **kwargs: Any) -> Any:
|
|
100
|
+
return self._request("GET", path, **kwargs)
|
|
101
|
+
|
|
102
|
+
def get_bytes(self, path: str, **kwargs: Any) -> bytes:
|
|
103
|
+
"""GET that returns raw response bytes (for file download endpoints)."""
|
|
104
|
+
url = self._url(path)
|
|
105
|
+
response = self._client.request("GET", url, **kwargs)
|
|
106
|
+
if response.is_error:
|
|
107
|
+
raise _parse_error(response)
|
|
108
|
+
return response.content
|
|
109
|
+
|
|
110
|
+
def post(self, path: str, **kwargs: Any) -> Any:
|
|
111
|
+
return self._request("POST", path, **kwargs)
|
|
112
|
+
|
|
113
|
+
def delete(self, path: str, **kwargs: Any) -> Any:
|
|
114
|
+
return self._request("DELETE", path, **kwargs)
|
|
115
|
+
|
|
116
|
+
def close(self) -> None:
|
|
117
|
+
self._client.close()
|
|
118
|
+
|
|
119
|
+
def __enter__(self) -> Transport:
|
|
120
|
+
return self
|
|
121
|
+
|
|
122
|
+
def __exit__(self, *_: Any) -> None:
|
|
123
|
+
self.close()
|
flexorch_sdk/client.py
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from ._transport import Transport
|
|
8
|
+
from .models.job import Job
|
|
9
|
+
from .models.search import SearchResult
|
|
10
|
+
from .resources.jobs import JobsResource
|
|
11
|
+
from .resources.datasets import DatasetsResource
|
|
12
|
+
from .resources.usage import UsageResource
|
|
13
|
+
from .resources.webhooks import WebhooksResource
|
|
14
|
+
from .resources.connectors import ConnectorsResource
|
|
15
|
+
|
|
16
|
+
_DEFAULT_BASE_URL = "https://api.flexorch.com/v1"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class FlexOrchClient:
|
|
20
|
+
"""Main entry point for the FlexOrch API.
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
api_key: Your FlexOrch API key (fx_...).
|
|
24
|
+
Defaults to FLEXORCH_API_KEY environment variable.
|
|
25
|
+
base_url: Override the API base URL (useful for testing).
|
|
26
|
+
timeout: HTTP timeout in seconds. Default: 30.
|
|
27
|
+
max_retries: Maximum retry attempts for transient errors. Default: 3.
|
|
28
|
+
|
|
29
|
+
Example::
|
|
30
|
+
|
|
31
|
+
from flexorch_sdk import FlexOrchClient
|
|
32
|
+
|
|
33
|
+
client = FlexOrchClient("fx_your_key_here")
|
|
34
|
+
dataset = client.process("contract.pdf", locale="tr").wait().dataset()
|
|
35
|
+
dataset.export("jsonl", path="output.jsonl")
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
def __init__(
|
|
39
|
+
self,
|
|
40
|
+
api_key: str | None = None,
|
|
41
|
+
*,
|
|
42
|
+
base_url: str = _DEFAULT_BASE_URL,
|
|
43
|
+
timeout: float = 30.0,
|
|
44
|
+
max_retries: int = 3,
|
|
45
|
+
) -> None:
|
|
46
|
+
resolved_key = api_key or os.environ.get("FLEXORCH_API_KEY", "")
|
|
47
|
+
if not resolved_key:
|
|
48
|
+
raise ValueError(
|
|
49
|
+
"No API key provided. Pass api_key= or set the FLEXORCH_API_KEY environment variable."
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
self._transport = Transport(
|
|
53
|
+
api_key=resolved_key,
|
|
54
|
+
base_url=base_url,
|
|
55
|
+
timeout=timeout,
|
|
56
|
+
max_retries=max_retries,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
self.jobs = JobsResource(self._transport)
|
|
60
|
+
self.datasets = DatasetsResource(self._transport)
|
|
61
|
+
self.usage = UsageResource(self._transport)
|
|
62
|
+
self.webhooks = WebhooksResource(self._transport)
|
|
63
|
+
self.connectors = ConnectorsResource(self._transport)
|
|
64
|
+
|
|
65
|
+
def process(
|
|
66
|
+
self,
|
|
67
|
+
file_path: str | Path,
|
|
68
|
+
*,
|
|
69
|
+
locale: str = "und",
|
|
70
|
+
pipeline_config: dict[str, Any] | None = None,
|
|
71
|
+
) -> Job:
|
|
72
|
+
"""Upload a document and start the processing pipeline.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
file_path: Path to the file (PDF, DOCX, TXT, XLSX, …).
|
|
76
|
+
locale: Language hint for PII detection.
|
|
77
|
+
"und" = all detectors (default), "tr", "de", "en", etc.
|
|
78
|
+
pipeline_config: Optional pipeline overrides passed to the API.
|
|
79
|
+
|
|
80
|
+
Returns:
|
|
81
|
+
A :class:`Job` object. Call ``.wait()`` to block until processing completes.
|
|
82
|
+
"""
|
|
83
|
+
path = Path(file_path)
|
|
84
|
+
if not path.exists():
|
|
85
|
+
raise FileNotFoundError(f"File not found: {path}")
|
|
86
|
+
|
|
87
|
+
form: dict[str, Any] = {"locale": locale}
|
|
88
|
+
if pipeline_config:
|
|
89
|
+
import json
|
|
90
|
+
form["pipeline_config"] = json.dumps(pipeline_config)
|
|
91
|
+
|
|
92
|
+
with path.open("rb") as fh:
|
|
93
|
+
data = self._transport.post(
|
|
94
|
+
"/data-process/async",
|
|
95
|
+
files={"file": (path.name, fh, "application/octet-stream")},
|
|
96
|
+
data=form,
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
return Job._from_dict(data, self._transport)
|
|
100
|
+
|
|
101
|
+
def process_many(
|
|
102
|
+
self,
|
|
103
|
+
file_paths: list[str | Path],
|
|
104
|
+
*,
|
|
105
|
+
locale: str = "und",
|
|
106
|
+
) -> list[Job]:
|
|
107
|
+
"""Upload and start processing for multiple files sequentially.
|
|
108
|
+
|
|
109
|
+
Returns:
|
|
110
|
+
List of :class:`Job` objects in the same order as *file_paths*.
|
|
111
|
+
"""
|
|
112
|
+
return [self.process(p, locale=locale) for p in file_paths]
|
|
113
|
+
|
|
114
|
+
def process_from_s3(
|
|
115
|
+
self,
|
|
116
|
+
connector_id: str,
|
|
117
|
+
keys: list[str],
|
|
118
|
+
*,
|
|
119
|
+
locale: str = "und",
|
|
120
|
+
pipeline_config: dict[str, Any] | None = None,
|
|
121
|
+
) -> list[Job]:
|
|
122
|
+
"""Start processing for files already stored in an S3 connector.
|
|
123
|
+
|
|
124
|
+
Args:
|
|
125
|
+
connector_id: ID of an active S3 connector.
|
|
126
|
+
keys: List of S3 object keys to process.
|
|
127
|
+
locale: Language hint for PII detection.
|
|
128
|
+
pipeline_config: Optional pipeline overrides.
|
|
129
|
+
|
|
130
|
+
Returns:
|
|
131
|
+
List of :class:`Job` objects, one per key.
|
|
132
|
+
"""
|
|
133
|
+
import json as _json
|
|
134
|
+
|
|
135
|
+
jobs = []
|
|
136
|
+
for key in keys:
|
|
137
|
+
source = {"connector_id": connector_id, "keys": [key]}
|
|
138
|
+
form: dict[str, Any] = {
|
|
139
|
+
"locale": locale,
|
|
140
|
+
"source": _json.dumps(source),
|
|
141
|
+
}
|
|
142
|
+
if pipeline_config:
|
|
143
|
+
form["pipeline_config"] = _json.dumps(pipeline_config)
|
|
144
|
+
data = self._transport.post("/data-process/async", data=form)
|
|
145
|
+
jobs.append(Job._from_dict(data, self._transport))
|
|
146
|
+
return jobs
|
|
147
|
+
|
|
148
|
+
def search(
|
|
149
|
+
self,
|
|
150
|
+
query: str,
|
|
151
|
+
*,
|
|
152
|
+
top_k: int = 10,
|
|
153
|
+
filters: dict[str, Any] | None = None,
|
|
154
|
+
) -> list[SearchResult]:
|
|
155
|
+
"""Semantic search across all indexed datasets (Pro+ plan required).
|
|
156
|
+
|
|
157
|
+
Args:
|
|
158
|
+
query: Natural language search query.
|
|
159
|
+
top_k: Number of results to return (1–50). Default: 10.
|
|
160
|
+
filters: Optional filter dict. Supported keys:
|
|
161
|
+
``document_type``, ``language``, ``pii_masked``, ``quality_grade``.
|
|
162
|
+
|
|
163
|
+
Returns:
|
|
164
|
+
List of :class:`SearchResult` sorted by descending cosine score.
|
|
165
|
+
"""
|
|
166
|
+
body: dict[str, Any] = {"query": query, "top_k": top_k}
|
|
167
|
+
if filters:
|
|
168
|
+
body["filters"] = filters
|
|
169
|
+
data = self._transport.post("/search", json=body) or {}
|
|
170
|
+
items = data.get("results", [])
|
|
171
|
+
return [SearchResult._from_dict(item) for item in items]
|
|
172
|
+
|
|
173
|
+
def close(self) -> None:
|
|
174
|
+
"""Release the underlying HTTP connection pool."""
|
|
175
|
+
self._transport.close()
|
|
176
|
+
|
|
177
|
+
def __enter__(self) -> FlexOrchClient:
|
|
178
|
+
return self
|
|
179
|
+
|
|
180
|
+
def __exit__(self, *_: Any) -> None:
|
|
181
|
+
self.close()
|
|
182
|
+
|
|
183
|
+
def __repr__(self) -> str:
|
|
184
|
+
return f"FlexOrchClient(base_url={self._transport._base_url!r})"
|
flexorch_sdk/errors.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class FlexOrchError(Exception):
|
|
5
|
+
"""Base exception for all FlexOrch SDK errors."""
|
|
6
|
+
|
|
7
|
+
def __init__(self, message: str, status_code: int = 0, error_code: str = "") -> None:
|
|
8
|
+
super().__init__(message)
|
|
9
|
+
self.status_code = status_code
|
|
10
|
+
self.error_code = error_code
|
|
11
|
+
|
|
12
|
+
def __repr__(self) -> str:
|
|
13
|
+
return f"{self.__class__.__name__}({self.args[0]!r}, status={self.status_code}, code={self.error_code!r})"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class AuthError(FlexOrchError):
|
|
17
|
+
"""Invalid or missing API key (401)."""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class QuotaError(FlexOrchError):
|
|
21
|
+
"""Credit quota exceeded or trial expired (402 / 429)."""
|
|
22
|
+
|
|
23
|
+
def __init__(
|
|
24
|
+
self,
|
|
25
|
+
message: str,
|
|
26
|
+
status_code: int = 402,
|
|
27
|
+
error_code: str = "",
|
|
28
|
+
remaining_credits: int = 0,
|
|
29
|
+
reset_at: str = "",
|
|
30
|
+
) -> None:
|
|
31
|
+
super().__init__(message, status_code, error_code)
|
|
32
|
+
self.remaining_credits = remaining_credits
|
|
33
|
+
self.reset_at = reset_at
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class RateLimitError(FlexOrchError):
|
|
37
|
+
"""Too many requests — slow down (429 with rate-limit context)."""
|
|
38
|
+
|
|
39
|
+
def __init__(
|
|
40
|
+
self,
|
|
41
|
+
message: str,
|
|
42
|
+
retry_after: int = 60,
|
|
43
|
+
) -> None:
|
|
44
|
+
super().__init__(message, status_code=429, error_code="RATE_LIMIT_EXCEEDED")
|
|
45
|
+
self.retry_after = retry_after
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class NotFoundError(FlexOrchError):
|
|
49
|
+
"""Requested resource does not exist (404)."""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class ValidationError(FlexOrchError):
|
|
53
|
+
"""Request payload failed validation (422)."""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class ServerError(FlexOrchError):
|
|
57
|
+
"""Unexpected server-side error (5xx)."""
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class JobFailedError(FlexOrchError):
|
|
61
|
+
"""Job completed with status=failed — pipeline could not process the document."""
|
|
62
|
+
|
|
63
|
+
def __init__(self, job_id: str, reason: str = "") -> None:
|
|
64
|
+
super().__init__(f"Job {job_id!r} failed: {reason or 'unknown reason'}")
|
|
65
|
+
self.job_id = job_id
|
|
66
|
+
self.failure_reason = reason
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class TimeoutError(FlexOrchError): # noqa: A001
|
|
70
|
+
"""job.wait() exceeded the requested timeout without completing."""
|
|
71
|
+
|
|
72
|
+
def __init__(self, job_id: str, timeout: int) -> None:
|
|
73
|
+
super().__init__(f"Job {job_id!r} did not complete within {timeout}s")
|
|
74
|
+
self.job_id = job_id
|
|
75
|
+
self.timeout = timeout
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass
|
|
8
|
+
class Connector:
|
|
9
|
+
id: str
|
|
10
|
+
name: str
|
|
11
|
+
type: str
|
|
12
|
+
active: bool
|
|
13
|
+
last_tested_at: str | None = None
|
|
14
|
+
last_used_at: str | None = None
|
|
15
|
+
created_at: str = ""
|
|
16
|
+
|
|
17
|
+
@classmethod
|
|
18
|
+
def _from_dict(cls, data: dict) -> Connector:
|
|
19
|
+
return cls(
|
|
20
|
+
id=data.get("id", ""),
|
|
21
|
+
name=data.get("name", ""),
|
|
22
|
+
type=data.get("type", ""),
|
|
23
|
+
active=data.get("active", True),
|
|
24
|
+
last_tested_at=data.get("last_tested_at"),
|
|
25
|
+
last_used_at=data.get("last_used_at"),
|
|
26
|
+
created_at=data.get("created_at", ""),
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
def __repr__(self) -> str:
|
|
30
|
+
return f"Connector(id={self.id!r}, name={self.name!r}, type={self.type!r}, active={self.active})"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class ConnectorTestResult:
|
|
35
|
+
success: bool
|
|
36
|
+
latency_ms: int | None = None
|
|
37
|
+
message: str = ""
|
|
38
|
+
|
|
39
|
+
@classmethod
|
|
40
|
+
def _from_dict(cls, data: dict) -> ConnectorTestResult:
|
|
41
|
+
return cls(
|
|
42
|
+
success=data.get("success", False),
|
|
43
|
+
latency_ms=data.get("latency_ms"),
|
|
44
|
+
message=data.get("message", ""),
|
|
45
|
+
)
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any, TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
from .._transport import Transport
|
|
9
|
+
|
|
10
|
+
_SUPPORTED_FORMATS = {"json", "jsonl", "csv", "parquet", "md", "xml", "xlsx", "rag"}
|
|
11
|
+
|
|
12
|
+
_INDEX_STATUSES = {"not_indexed", "indexing", "ready", "failed"}
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class Dataset:
|
|
17
|
+
id: str
|
|
18
|
+
name: str
|
|
19
|
+
slug: str
|
|
20
|
+
status: str
|
|
21
|
+
row_count: int = 0
|
|
22
|
+
created_at: str = ""
|
|
23
|
+
available_formats: list[str] = field(default_factory=list)
|
|
24
|
+
_transport: Any = field(default=None, repr=False)
|
|
25
|
+
|
|
26
|
+
@classmethod
|
|
27
|
+
def _from_dict(cls, data: dict, transport: Transport) -> Dataset:
|
|
28
|
+
fmt_summary = data.get("format_summary", {})
|
|
29
|
+
formats = list(fmt_summary.get("files", {}).keys()) if isinstance(fmt_summary, dict) else []
|
|
30
|
+
return cls(
|
|
31
|
+
id=data.get("id", ""),
|
|
32
|
+
name=data.get("name", ""),
|
|
33
|
+
slug=data.get("slug", ""),
|
|
34
|
+
status=data.get("status", ""),
|
|
35
|
+
row_count=data.get("row_count", 0),
|
|
36
|
+
created_at=data.get("created_at", ""),
|
|
37
|
+
available_formats=formats,
|
|
38
|
+
_transport=transport,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
def export(self, format: str, path: str | Path | None = None) -> bytes:
|
|
42
|
+
"""Download dataset in the requested format.
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
format: One of json, jsonl, csv, parquet, md, xml, xlsx, rag.
|
|
46
|
+
path: If given, write bytes to this file and return them.
|
|
47
|
+
If None, return raw bytes without writing.
|
|
48
|
+
|
|
49
|
+
Returns:
|
|
50
|
+
Raw file content as bytes.
|
|
51
|
+
"""
|
|
52
|
+
if format not in _SUPPORTED_FORMATS:
|
|
53
|
+
raise ValueError(f"Unsupported format {format!r}. Choose from: {sorted(_SUPPORTED_FORMATS)}")
|
|
54
|
+
|
|
55
|
+
raw = self._transport.get_bytes(
|
|
56
|
+
f"/datasets/{self.id}/export",
|
|
57
|
+
params={"format": format},
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
if path is not None:
|
|
61
|
+
Path(path).write_bytes(raw)
|
|
62
|
+
|
|
63
|
+
return raw
|
|
64
|
+
|
|
65
|
+
def export_to_s3(
|
|
66
|
+
self,
|
|
67
|
+
connector_id: str,
|
|
68
|
+
format: str,
|
|
69
|
+
prefix: str = "",
|
|
70
|
+
) -> dict[str, Any]:
|
|
71
|
+
"""Push an exported file directly to an S3 connector.
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
connector_id: ID of an active S3 connector.
|
|
75
|
+
format: Export format (same set as :meth:`export`).
|
|
76
|
+
prefix: Optional S3 key prefix (e.g. "exports/datasets/").
|
|
77
|
+
|
|
78
|
+
Returns:
|
|
79
|
+
Dict with ``s3_key`` and ``size_bytes``.
|
|
80
|
+
"""
|
|
81
|
+
if format not in _SUPPORTED_FORMATS:
|
|
82
|
+
raise ValueError(f"Unsupported format {format!r}. Choose from: {sorted(_SUPPORTED_FORMATS)}")
|
|
83
|
+
return self._transport.post(
|
|
84
|
+
f"/datasets/{self.id}/export-s3",
|
|
85
|
+
json={"format": format, "connector_id": connector_id, "prefix": prefix},
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
def index(self) -> dict[str, Any]:
|
|
89
|
+
"""Trigger semantic indexing for this dataset (Pro+ plan required).
|
|
90
|
+
|
|
91
|
+
Returns:
|
|
92
|
+
Dict with ``status`` and ``message``.
|
|
93
|
+
"""
|
|
94
|
+
return self._transport.post(f"/datasets/{self.id}/index") or {}
|
|
95
|
+
|
|
96
|
+
def index_status(self) -> dict[str, Any]:
|
|
97
|
+
"""Return the current semantic index status.
|
|
98
|
+
|
|
99
|
+
Returns:
|
|
100
|
+
Dict with ``status`` (not_indexed | indexing | ready | failed),
|
|
101
|
+
``chunks_indexed``, and ``total_chunks``.
|
|
102
|
+
"""
|
|
103
|
+
return self._transport.get(f"/datasets/{self.id}/index/status") or {}
|
|
104
|
+
|
|
105
|
+
def __repr__(self) -> str:
|
|
106
|
+
return f"Dataset(id={self.id!r}, name={self.name!r}, rows={self.row_count}, status={self.status!r})"
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from typing import TYPE_CHECKING, Any
|
|
6
|
+
|
|
7
|
+
from ..errors import JobFailedError, TimeoutError
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from .dataset import Dataset
|
|
11
|
+
from .._transport import Transport
|
|
12
|
+
|
|
13
|
+
_TERMINAL_STATUSES = {"completed", "failed"}
|
|
14
|
+
_DEFAULT_POLL_INTERVAL = 2
|
|
15
|
+
_DEFAULT_TIMEOUT = 300
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class Job:
|
|
20
|
+
id: str
|
|
21
|
+
status: str
|
|
22
|
+
quality_grade: str | None = None
|
|
23
|
+
quality_score: float | None = None
|
|
24
|
+
document_id: str | None = None
|
|
25
|
+
has_dataset: bool = False
|
|
26
|
+
failure_reason: str | None = None
|
|
27
|
+
created_at: str = ""
|
|
28
|
+
completed_at: str | None = None
|
|
29
|
+
_transport: Any = field(default=None, repr=False)
|
|
30
|
+
|
|
31
|
+
@classmethod
|
|
32
|
+
def _from_dict(cls, data: dict, transport: Transport) -> Job:
|
|
33
|
+
return cls(
|
|
34
|
+
id=data.get("job_id") or data.get("id", ""),
|
|
35
|
+
status=data.get("status", ""),
|
|
36
|
+
quality_grade=data.get("quality", {}).get("grade") if isinstance(data.get("quality"), dict) else data.get("quality_grade"),
|
|
37
|
+
quality_score=data.get("quality", {}).get("score") if isinstance(data.get("quality"), dict) else data.get("quality_score"),
|
|
38
|
+
document_id=data.get("document_id"),
|
|
39
|
+
has_dataset=bool(data.get("has_dataset", False)),
|
|
40
|
+
failure_reason=data.get("failure_reason"),
|
|
41
|
+
created_at=data.get("created_at", ""),
|
|
42
|
+
completed_at=data.get("completed_at"),
|
|
43
|
+
_transport=transport,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
def wait(
|
|
47
|
+
self,
|
|
48
|
+
timeout: int = _DEFAULT_TIMEOUT,
|
|
49
|
+
poll_interval: int = _DEFAULT_POLL_INTERVAL,
|
|
50
|
+
) -> Job:
|
|
51
|
+
"""Poll until the job reaches a terminal status or timeout is exceeded."""
|
|
52
|
+
if self.status in _TERMINAL_STATUSES:
|
|
53
|
+
return self
|
|
54
|
+
|
|
55
|
+
deadline = time.monotonic() + timeout
|
|
56
|
+
while time.monotonic() < deadline:
|
|
57
|
+
data = self._transport.get(f"/jobs/{self.id}")
|
|
58
|
+
updated = Job._from_dict(data, self._transport)
|
|
59
|
+
self.__dict__.update(updated.__dict__)
|
|
60
|
+
|
|
61
|
+
if self.status in _TERMINAL_STATUSES:
|
|
62
|
+
break
|
|
63
|
+
time.sleep(poll_interval)
|
|
64
|
+
else:
|
|
65
|
+
raise TimeoutError(self.id, timeout)
|
|
66
|
+
|
|
67
|
+
if self.status == "failed":
|
|
68
|
+
raise JobFailedError(self.id, self.failure_reason or "")
|
|
69
|
+
|
|
70
|
+
return self
|
|
71
|
+
|
|
72
|
+
def dataset(self) -> Dataset | None:
|
|
73
|
+
"""Return the dataset built from this job, if one exists."""
|
|
74
|
+
from .dataset import Dataset
|
|
75
|
+
|
|
76
|
+
if not self.has_dataset:
|
|
77
|
+
return None
|
|
78
|
+
data = self._transport.get("/datasets", params={"job_id": self.id})
|
|
79
|
+
items = data.get("items", data) if isinstance(data, dict) else data
|
|
80
|
+
if not items:
|
|
81
|
+
return None
|
|
82
|
+
return Dataset._from_dict(items[0], self._transport)
|
|
83
|
+
|
|
84
|
+
def __repr__(self) -> str:
|
|
85
|
+
return f"Job(id={self.id!r}, status={self.status!r}, grade={self.quality_grade!r})"
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass
|
|
7
|
+
class SearchResult:
|
|
8
|
+
chunk_id: str
|
|
9
|
+
text: str
|
|
10
|
+
score: float
|
|
11
|
+
dataset_id: str
|
|
12
|
+
chunk_index: int
|
|
13
|
+
token_count: int
|
|
14
|
+
metadata: dict = field(default_factory=dict)
|
|
15
|
+
|
|
16
|
+
@classmethod
|
|
17
|
+
def _from_dict(cls, data: dict) -> SearchResult:
|
|
18
|
+
return cls(
|
|
19
|
+
chunk_id=data.get("chunk_id", ""),
|
|
20
|
+
text=data.get("text", ""),
|
|
21
|
+
score=data.get("score", 0.0),
|
|
22
|
+
dataset_id=data.get("dataset_id", ""),
|
|
23
|
+
chunk_index=data.get("chunk_index", 0),
|
|
24
|
+
token_count=data.get("token_count", 0),
|
|
25
|
+
metadata=data.get("metadata", {}),
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
def __repr__(self) -> str:
|
|
29
|
+
return f"SearchResult(score={self.score:.3f}, dataset_id={self.dataset_id!r}, chunk_index={self.chunk_index})"
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from .jobs import JobsResource
|
|
2
|
+
from .datasets import DatasetsResource
|
|
3
|
+
from .usage import UsageResource, UsageSnapshot
|
|
4
|
+
from .webhooks import WebhooksResource, Webhook
|
|
5
|
+
|
|
6
|
+
__all__ = [
|
|
7
|
+
"JobsResource",
|
|
8
|
+
"DatasetsResource",
|
|
9
|
+
"UsageResource",
|
|
10
|
+
"UsageSnapshot",
|
|
11
|
+
"WebhooksResource",
|
|
12
|
+
"Webhook",
|
|
13
|
+
]
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any, TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
from ..models.connector import Connector, ConnectorTestResult
|
|
6
|
+
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
from .._transport import Transport
|
|
9
|
+
|
|
10
|
+
_VALID_TYPES = {"s3", "gcs", "azure_blob"}
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ConnectorsResource:
|
|
14
|
+
def __init__(self, transport: Transport) -> None:
|
|
15
|
+
self._t = transport
|
|
16
|
+
|
|
17
|
+
def create(self, name: str, type: str, config: dict[str, Any]) -> Connector:
|
|
18
|
+
"""Register a new storage connector.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
name: Display name (e.g. "Production S3").
|
|
22
|
+
type: Connector type — "s3", "gcs", or "azure_blob".
|
|
23
|
+
config: Provider-specific credentials dict.
|
|
24
|
+
S3 example: {"bucket": "...", "region": "...",
|
|
25
|
+
"access_key_id": "...", "secret_access_key": "..."}
|
|
26
|
+
"""
|
|
27
|
+
if type not in _VALID_TYPES:
|
|
28
|
+
raise ValueError(f"Unknown connector type {type!r}. Valid: {sorted(_VALID_TYPES)}")
|
|
29
|
+
data = self._t.post("/connectors", json={"name": name, "type": type, "config": config})
|
|
30
|
+
return Connector._from_dict(data)
|
|
31
|
+
|
|
32
|
+
def list(self) -> list[Connector]:
|
|
33
|
+
"""Return all connectors for the current tenant."""
|
|
34
|
+
data = self._t.get("/connectors")
|
|
35
|
+
items = data.get("items", data) if isinstance(data, dict) else data
|
|
36
|
+
return [Connector._from_dict(item) for item in items]
|
|
37
|
+
|
|
38
|
+
def get(self, connector_id: str) -> Connector:
|
|
39
|
+
"""Fetch a single connector by ID."""
|
|
40
|
+
data = self._t.get(f"/connectors/{connector_id}")
|
|
41
|
+
return Connector._from_dict(data)
|
|
42
|
+
|
|
43
|
+
def delete(self, connector_id: str) -> None:
|
|
44
|
+
"""Delete a connector (sets active=False on the backend)."""
|
|
45
|
+
self._t.delete(f"/connectors/{connector_id}")
|
|
46
|
+
|
|
47
|
+
def test(self, connector_id: str) -> ConnectorTestResult:
|
|
48
|
+
"""Run a connectivity test for a connector.
|
|
49
|
+
|
|
50
|
+
Returns:
|
|
51
|
+
:class:`ConnectorTestResult` with ``success``, ``latency_ms``, and ``message``.
|
|
52
|
+
"""
|
|
53
|
+
data = self._t.post(f"/connectors/{connector_id}/test")
|
|
54
|
+
return ConnectorTestResult._from_dict(data or {})
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
from ..models.dataset import Dataset
|
|
6
|
+
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
from .._transport import Transport
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class DatasetsResource:
|
|
12
|
+
def __init__(self, transport: Transport) -> None:
|
|
13
|
+
self._t = transport
|
|
14
|
+
|
|
15
|
+
def get(self, dataset_id: str) -> Dataset:
|
|
16
|
+
"""Fetch a single dataset by ID."""
|
|
17
|
+
data = self._t.get(f"/datasets/{dataset_id}")
|
|
18
|
+
return Dataset._from_dict(data, self._t)
|
|
19
|
+
|
|
20
|
+
def list(self, page: int = 1, page_size: int = 20) -> list[Dataset]:
|
|
21
|
+
"""List datasets for the current tenant, newest first."""
|
|
22
|
+
data = self._t.get("/datasets", params={"page": page, "page_size": page_size})
|
|
23
|
+
items = data.get("items", data) if isinstance(data, dict) else data
|
|
24
|
+
return [Dataset._from_dict(item, self._t) for item in items]
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
from ..models.job import Job
|
|
6
|
+
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
from .._transport import Transport
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class JobsResource:
|
|
12
|
+
def __init__(self, transport: Transport) -> None:
|
|
13
|
+
self._t = transport
|
|
14
|
+
|
|
15
|
+
def get(self, job_id: str) -> Job:
|
|
16
|
+
"""Fetch a single job by ID."""
|
|
17
|
+
data = self._t.get(f"/jobs/{job_id}")
|
|
18
|
+
return Job._from_dict(data, self._t)
|
|
19
|
+
|
|
20
|
+
def list(self, page: int = 1, page_size: int = 20) -> list[Job]:
|
|
21
|
+
"""List jobs for the current tenant, newest first."""
|
|
22
|
+
data = self._t.get("/jobs", params={"page": page, "page_size": page_size})
|
|
23
|
+
items = data.get("items", data) if isinstance(data, dict) else data
|
|
24
|
+
return [Job._from_dict(item, self._t) for item in items]
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import TYPE_CHECKING
|
|
5
|
+
|
|
6
|
+
if TYPE_CHECKING:
|
|
7
|
+
from .._transport import Transport
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class UsageSnapshot:
|
|
12
|
+
plan: str
|
|
13
|
+
credits_used: int
|
|
14
|
+
credits_limit: int
|
|
15
|
+
credits_remaining: int
|
|
16
|
+
reset_at: str
|
|
17
|
+
period_start: str
|
|
18
|
+
period_end: str
|
|
19
|
+
|
|
20
|
+
@classmethod
|
|
21
|
+
def _from_dict(cls, data: dict) -> UsageSnapshot:
|
|
22
|
+
return cls(
|
|
23
|
+
plan=data.get("plan", ""),
|
|
24
|
+
credits_used=data.get("credits_used", 0),
|
|
25
|
+
credits_limit=data.get("credits_limit", 0),
|
|
26
|
+
credits_remaining=data.get("credits_remaining", 0),
|
|
27
|
+
reset_at=data.get("reset_at", ""),
|
|
28
|
+
period_start=data.get("period_start", ""),
|
|
29
|
+
period_end=data.get("period_end", ""),
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
def __repr__(self) -> str:
|
|
33
|
+
return (
|
|
34
|
+
f"UsageSnapshot(plan={self.plan!r}, "
|
|
35
|
+
f"used={self.credits_used}/{self.credits_limit}, "
|
|
36
|
+
f"remaining={self.credits_remaining})"
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class UsageResource:
|
|
41
|
+
def __init__(self, transport: Transport) -> None:
|
|
42
|
+
self._t = transport
|
|
43
|
+
|
|
44
|
+
def current(self) -> UsageSnapshot:
|
|
45
|
+
"""Return the current billing period usage and credit balance."""
|
|
46
|
+
data = self._t.get("/usage/current")
|
|
47
|
+
return UsageSnapshot._from_dict(data)
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import TYPE_CHECKING
|
|
5
|
+
|
|
6
|
+
if TYPE_CHECKING:
|
|
7
|
+
from .._transport import Transport
|
|
8
|
+
|
|
9
|
+
_VALID_EVENTS = {"dataset.ready", "job.completed", "job.failed"}
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class Webhook:
|
|
14
|
+
id: str
|
|
15
|
+
url: str
|
|
16
|
+
events: list[str]
|
|
17
|
+
active: bool
|
|
18
|
+
created_at: str
|
|
19
|
+
|
|
20
|
+
@classmethod
|
|
21
|
+
def _from_dict(cls, data: dict) -> Webhook:
|
|
22
|
+
return cls(
|
|
23
|
+
id=data.get("id", ""),
|
|
24
|
+
url=data.get("url", ""),
|
|
25
|
+
events=data.get("events", []),
|
|
26
|
+
active=data.get("active", True),
|
|
27
|
+
created_at=data.get("created_at", ""),
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
def __repr__(self) -> str:
|
|
31
|
+
return f"Webhook(id={self.id!r}, url={self.url!r}, events={self.events})"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class WebhooksResource:
|
|
35
|
+
def __init__(self, transport: Transport) -> None:
|
|
36
|
+
self._t = transport
|
|
37
|
+
|
|
38
|
+
def register(self, url: str, events: list[str]) -> Webhook:
|
|
39
|
+
"""Register a new webhook endpoint.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
url: HTTPS URL that will receive POST requests.
|
|
43
|
+
events: List of event types, e.g. ["dataset.ready"].
|
|
44
|
+
"""
|
|
45
|
+
invalid = set(events) - _VALID_EVENTS
|
|
46
|
+
if invalid:
|
|
47
|
+
raise ValueError(f"Unknown event types: {invalid}. Valid: {_VALID_EVENTS}")
|
|
48
|
+
data = self._t.post("/webhooks", json={"url": url, "events": events})
|
|
49
|
+
return Webhook._from_dict(data)
|
|
50
|
+
|
|
51
|
+
def list(self) -> list[Webhook]:
|
|
52
|
+
"""Return all registered webhooks for the current tenant."""
|
|
53
|
+
data = self._t.get("/webhooks")
|
|
54
|
+
items = data.get("items", data) if isinstance(data, dict) else data
|
|
55
|
+
return [Webhook._from_dict(item) for item in items]
|
|
56
|
+
|
|
57
|
+
def delete(self, webhook_id: str) -> None:
|
|
58
|
+
"""Delete a webhook by ID."""
|
|
59
|
+
self._t.delete(f"/webhooks/{webhook_id}")
|
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: flexorch-sdk
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Python SDK for the FlexOrch API — process documents, build LLM-ready datasets
|
|
5
|
+
Project-URL: Homepage, https://flexorch.com
|
|
6
|
+
Project-URL: Repository, https://github.com/flexorch/flexorch-sdk
|
|
7
|
+
Project-URL: Issues, https://github.com/flexorch/flexorch-sdk/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/flexorch/flexorch-sdk/blob/main/CHANGELOG.md
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: dataset,document,flexorch,llm,pipeline
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Requires-Dist: httpx>=0.27
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
25
|
+
Requires-Dist: pytest>=7; extra == 'dev'
|
|
26
|
+
Requires-Dist: respx>=0.21; extra == 'dev'
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
|
|
29
|
+
# flexorch-sdk
|
|
30
|
+
|
|
31
|
+
[](https://pypi.org/project/flexorch-sdk/)
|
|
32
|
+
[](https://pypi.org/project/flexorch-sdk/)
|
|
33
|
+
[](https://github.com/flexorch/flexorch-sdk/actions)
|
|
34
|
+
[](LICENSE)
|
|
35
|
+
|
|
36
|
+
Python SDK for the [FlexOrch](https://flexorch.com) API.
|
|
37
|
+
|
|
38
|
+
FlexOrch turns unstructured documents (PDF, DOCX, invoices, emails…) into clean, structured, LLM-ready datasets — with automatic PII detection and masking, quality scoring, and multiple export formats.
|
|
39
|
+
|
|
40
|
+
---
|
|
41
|
+
|
|
42
|
+
## Install
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install flexorch-sdk
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Requires Python 3.10+. The only dependency is [`httpx`](https://www.python-httpx.org/).
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
52
|
+
## Quick start
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from flexorch_sdk import FlexOrchClient
|
|
56
|
+
|
|
57
|
+
client = FlexOrchClient("fx_your_key_here")
|
|
58
|
+
|
|
59
|
+
# Upload a document and wait for the pipeline to finish
|
|
60
|
+
job = client.process("contract.pdf", locale="tr").wait()
|
|
61
|
+
|
|
62
|
+
print(job.quality_grade) # "A"
|
|
63
|
+
print(job.quality_score) # 0.91
|
|
64
|
+
|
|
65
|
+
# Download the resulting dataset
|
|
66
|
+
dataset = job.dataset()
|
|
67
|
+
dataset.export("jsonl", path="output.jsonl")
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
---
|
|
71
|
+
|
|
72
|
+
## Auth
|
|
73
|
+
|
|
74
|
+
Pass your API key directly or set the `FLEXORCH_API_KEY` environment variable:
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
export FLEXORCH_API_KEY=fx_...
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
from flexorch_sdk import FlexOrchClient
|
|
82
|
+
|
|
83
|
+
client = FlexOrchClient() # reads FLEXORCH_API_KEY automatically
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Get your API key from [app.flexorch.com](https://app.flexorch.com) → Settings.
|
|
87
|
+
|
|
88
|
+
---
|
|
89
|
+
|
|
90
|
+
## Supported input formats
|
|
91
|
+
|
|
92
|
+
| Category | Formats |
|
|
93
|
+
|---|---|
|
|
94
|
+
| Documents | PDF (text + scanned), DOCX, TXT |
|
|
95
|
+
| Spreadsheets | XLSX |
|
|
96
|
+
| Email | EML, MSG |
|
|
97
|
+
| E-invoices | XML/UBL (Peppol, GİB TR), FatturaPA (IT), XRechnung (DE), ZUGFeRD/Factur-X |
|
|
98
|
+
| Images | JPG, PNG, TIFF (OCR) |
|
|
99
|
+
| Web | HTML, HTM |
|
|
100
|
+
|
|
101
|
+
---
|
|
102
|
+
|
|
103
|
+
## Export formats
|
|
104
|
+
|
|
105
|
+
`json` · `jsonl` · `csv` · `parquet` · `md` · `xml` · `xlsx` · `rag`
|
|
106
|
+
|
|
107
|
+
```python
|
|
108
|
+
dataset.export("jsonl", path="output.jsonl") # write to file
|
|
109
|
+
raw = dataset.export("parquet") # return bytes
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
The `rag` format produces LlamaIndex/LangChain-compatible chunks with metadata.
|
|
113
|
+
|
|
114
|
+
---
|
|
115
|
+
|
|
116
|
+
## Processing
|
|
117
|
+
|
|
118
|
+
### Single file
|
|
119
|
+
|
|
120
|
+
```python
|
|
121
|
+
job = client.process("invoice.pdf", locale="de").wait()
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
`locale` is an [IETF language tag](https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry)
|
|
125
|
+
used to activate the right PII detectors (`tr`, `de`, `en`, `fr`, `it`, `nl`, `es`, `pl`, `und` = all).
|
|
126
|
+
|
|
127
|
+
### Batch
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
jobs = client.process_many(["a.pdf", "b.pdf", "c.pdf"], locale="und")
|
|
131
|
+
for job in jobs:
|
|
132
|
+
job.wait()
|
|
133
|
+
print(job.quality_grade, job.quality_score)
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
### From S3
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
# Register a connector once; store conn.id for reuse
|
|
140
|
+
conn = client.connectors.create(
|
|
141
|
+
"Production S3", "s3",
|
|
142
|
+
{
|
|
143
|
+
"bucket": "my-bucket",
|
|
144
|
+
"region": "eu-central-1",
|
|
145
|
+
"access_key_id": "AKIA...",
|
|
146
|
+
"secret_access_key": "...",
|
|
147
|
+
},
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
# Verify connectivity
|
|
151
|
+
result = client.connectors.test(conn.id)
|
|
152
|
+
print(result.success, result.latency_ms) # True, 38
|
|
153
|
+
|
|
154
|
+
# Process files from S3
|
|
155
|
+
jobs = client.process_from_s3(conn.id, ["invoices/inv-001.pdf", "invoices/inv-002.pdf"])
|
|
156
|
+
for job in jobs:
|
|
157
|
+
job.wait()
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
---
|
|
161
|
+
|
|
162
|
+
## Job polling
|
|
163
|
+
|
|
164
|
+
`Job.wait()` blocks until the pipeline completes or times out.
|
|
165
|
+
|
|
166
|
+
```python
|
|
167
|
+
job = client.process("large-report.pdf").wait(
|
|
168
|
+
timeout=600, # seconds before TimeoutError (default: 300)
|
|
169
|
+
poll_interval=5, # polling interval in seconds (default: 2)
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
print(job.status) # "completed"
|
|
173
|
+
print(job.quality_grade) # "A" | "B" | "C" | "D"
|
|
174
|
+
print(job.quality_score) # 0.0 – 1.0
|
|
175
|
+
print(job.has_dataset) # True
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
---
|
|
179
|
+
|
|
180
|
+
## Dataset operations
|
|
181
|
+
|
|
182
|
+
```python
|
|
183
|
+
ds = job.dataset() # fetch dataset linked to this job
|
|
184
|
+
ds = client.datasets.get("dataset-id")
|
|
185
|
+
|
|
186
|
+
print(ds.name) # "contract-2024-q1"
|
|
187
|
+
print(ds.row_count) # 142
|
|
188
|
+
print(ds.available_formats) # ["json", "jsonl", "csv", "parquet"]
|
|
189
|
+
|
|
190
|
+
# Download locally
|
|
191
|
+
ds.export("jsonl", path="output.jsonl")
|
|
192
|
+
|
|
193
|
+
# Push directly to S3
|
|
194
|
+
push = ds.export_to_s3(conn.id, "jsonl", prefix="processed/datasets/")
|
|
195
|
+
print(push["s3_key"]) # "processed/datasets/contract-2024-q1.jsonl"
|
|
196
|
+
print(push["size_bytes"]) # 84320
|
|
197
|
+
|
|
198
|
+
# Semantic indexing (Pro+)
|
|
199
|
+
ds.index()
|
|
200
|
+
status = ds.index_status() # {"status": "ready", "chunks_indexed": 48}
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
---
|
|
204
|
+
|
|
205
|
+
## Semantic search (Pro+)
|
|
206
|
+
|
|
207
|
+
```python
|
|
208
|
+
results = client.search(
|
|
209
|
+
"payment terms net 30",
|
|
210
|
+
top_k=10,
|
|
211
|
+
filters={
|
|
212
|
+
"document_type": "invoice",
|
|
213
|
+
"language": "de",
|
|
214
|
+
"quality_grade": "A",
|
|
215
|
+
"pii_masked": True,
|
|
216
|
+
},
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
for r in results:
|
|
220
|
+
print(f"{r.score:.3f} [{r.dataset_id}] {r.text[:120]}")
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
---
|
|
224
|
+
|
|
225
|
+
## Resources
|
|
226
|
+
|
|
227
|
+
```python
|
|
228
|
+
# Jobs
|
|
229
|
+
jobs = client.jobs.list(page=1, page_size=20)
|
|
230
|
+
job = client.jobs.get("job-id")
|
|
231
|
+
|
|
232
|
+
# Datasets
|
|
233
|
+
datasets = client.datasets.list()
|
|
234
|
+
ds = client.datasets.get("dataset-id")
|
|
235
|
+
|
|
236
|
+
# Usage
|
|
237
|
+
usage = client.usage.current()
|
|
238
|
+
print(f"{usage.credits_used} / {usage.credits_limit} credits used")
|
|
239
|
+
print(f"Plan: {usage.plan} — resets {usage.reset_at}")
|
|
240
|
+
|
|
241
|
+
# Webhooks
|
|
242
|
+
client.webhooks.register("https://your-server.com/hook", events=["dataset.ready"])
|
|
243
|
+
client.webhooks.list()
|
|
244
|
+
client.webhooks.delete("webhook-id")
|
|
245
|
+
|
|
246
|
+
# Connectors
|
|
247
|
+
client.connectors.create("name", "s3", {...})
|
|
248
|
+
client.connectors.list()
|
|
249
|
+
client.connectors.get("connector-id")
|
|
250
|
+
client.connectors.test("connector-id")
|
|
251
|
+
client.connectors.delete("connector-id")
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
---
|
|
255
|
+
|
|
256
|
+
## Error handling
|
|
257
|
+
|
|
258
|
+
```python
|
|
259
|
+
from flexorch_sdk import (
|
|
260
|
+
FlexOrchClient,
|
|
261
|
+
AuthError, # 401 — invalid or missing API key
|
|
262
|
+
QuotaError, # 402 — credit limit reached or trial expired
|
|
263
|
+
RateLimitError, # 429 — too many requests; has .retry_after (seconds)
|
|
264
|
+
NotFoundError, # 404
|
|
265
|
+
ValidationError, # 422 — bad request parameters
|
|
266
|
+
ServerError, # 5xx
|
|
267
|
+
JobFailedError, # pipeline failed; has .job_id and .failure_reason
|
|
268
|
+
TimeoutError, # Job.wait() exceeded timeout; has .job_id
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
try:
|
|
272
|
+
job = client.process("doc.pdf").wait(timeout=120)
|
|
273
|
+
except AuthError:
|
|
274
|
+
print("Invalid API key — check FLEXORCH_API_KEY")
|
|
275
|
+
except QuotaError as e:
|
|
276
|
+
print(f"Out of credits — reset at {e.reset_at}")
|
|
277
|
+
except JobFailedError as e:
|
|
278
|
+
print(f"Pipeline failed for job {e.job_id}: {e.failure_reason}")
|
|
279
|
+
except TimeoutError as e:
|
|
280
|
+
print(f"Job {e.job_id} still running after timeout — poll manually")
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
The SDK automatically retries `429` and `5xx` responses with exponential backoff (up to 3 attempts by default).
|
|
284
|
+
|
|
285
|
+
---
|
|
286
|
+
|
|
287
|
+
## Configuration
|
|
288
|
+
|
|
289
|
+
```python
|
|
290
|
+
client = FlexOrchClient(
|
|
291
|
+
api_key="fx_...",
|
|
292
|
+
base_url="https://api.flexorch.com/v1", # override for self-hosted
|
|
293
|
+
timeout=60.0, # HTTP timeout per request in seconds
|
|
294
|
+
max_retries=5, # retry attempts for transient errors
|
|
295
|
+
)
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
### Context manager
|
|
299
|
+
|
|
300
|
+
```python
|
|
301
|
+
with FlexOrchClient() as client:
|
|
302
|
+
job = client.process("report.pdf").wait()
|
|
303
|
+
job.dataset().export("jsonl", path="report.jsonl")
|
|
304
|
+
# HTTP connection pool released automatically
|
|
305
|
+
```
|
|
306
|
+
|
|
307
|
+
---
|
|
308
|
+
|
|
309
|
+
## Examples
|
|
310
|
+
|
|
311
|
+
See [`examples/`](examples/) for runnable scripts:
|
|
312
|
+
|
|
313
|
+
| File | Description |
|
|
314
|
+
|---|---|
|
|
315
|
+
| [`basic_process.py`](examples/basic_process.py) | Process a single document and export as JSONL |
|
|
316
|
+
| [`batch_process.py`](examples/batch_process.py) | Process multiple files with error handling |
|
|
317
|
+
| [`s3_import.py`](examples/s3_import.py) | Import from S3, process, export results back to S3 |
|
|
318
|
+
|
|
319
|
+
---
|
|
320
|
+
|
|
321
|
+
## Development
|
|
322
|
+
|
|
323
|
+
```bash
|
|
324
|
+
git clone https://github.com/flexorch/flexorch-sdk
|
|
325
|
+
cd flexorch-sdk
|
|
326
|
+
pip install -e ".[dev]"
|
|
327
|
+
pytest
|
|
328
|
+
```
|
|
329
|
+
|
|
330
|
+
Tests use [respx](https://lundberg.github.io/respx/) to mock httpx — no network calls, no API key needed.
|
|
331
|
+
|
|
332
|
+
---
|
|
333
|
+
|
|
334
|
+
## Links
|
|
335
|
+
|
|
336
|
+
- [Platform](https://app.flexorch.com)
|
|
337
|
+
- [API reference](https://flexorch.com/developers)
|
|
338
|
+
- [flexorch-audit](https://github.com/flexorch/flexorch-audit) — open-source PII detection library
|
|
339
|
+
|
|
340
|
+
---
|
|
341
|
+
|
|
342
|
+
## License
|
|
343
|
+
|
|
344
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
flexorch_sdk/__init__.py,sha256=oiRvyQxyjA1VPwRIMNS-SPbh9y_uyA0wOFxnRj4u9ig,1457
|
|
2
|
+
flexorch_sdk/_transport.py,sha256=Ajrqd4ow8ukmel9Mof62pfHWZpXqCJmFiTrnBAQXyx4,3936
|
|
3
|
+
flexorch_sdk/client.py,sha256=cZ2Qli5A9reDtcbgMTpt5pehWtZpQEOBteG5jJax5sM,6115
|
|
4
|
+
flexorch_sdk/errors.py,sha256=HYc4mUlAruIAR5neKznUwnHrVFP28LGJFs6l18JOVLs,2244
|
|
5
|
+
flexorch_sdk/models/__init__.py,sha256=7_raZnMhl4kK8-SiAPzyDS4Y5w_zY05twsqwflPiR8Q,219
|
|
6
|
+
flexorch_sdk/models/connector.py,sha256=BE2yJiMPla3snTabx2HuTzdFrdxqSsv2Shhlw7JSu2I,1199
|
|
7
|
+
flexorch_sdk/models/dataset.py,sha256=8NZxLk3dpO1PskL6cRFram5I8B7aGkI148GmJgg7CYs,3572
|
|
8
|
+
flexorch_sdk/models/job.py,sha256=Xy4kycl-RKFNGidzvyKjXZ18ZgSYE5VvrafrZUmEbtw,2939
|
|
9
|
+
flexorch_sdk/models/search.py,sha256=TuX9LvZYSknP9zglhenbr0OOlnXKDpBG4_rNkG_rEUQ,852
|
|
10
|
+
flexorch_sdk/resources/__init__.py,sha256=6ZadAypDV1PXH7eBWX19-Gn0aB51qt3l4jT8zTzZogc,306
|
|
11
|
+
flexorch_sdk/resources/connectors.py,sha256=Hzqswqe5G0s6fYctrXfgyKKoQk7qLmvhNPvtA9dAdt8,2118
|
|
12
|
+
flexorch_sdk/resources/datasets.py,sha256=R7Rivws-I0daVGsudmiCtWrmrvkKZcrbYBU163QWEoo,840
|
|
13
|
+
flexorch_sdk/resources/jobs.py,sha256=GKFQmwX-BhM-FanpXcjMY_NNaotwpRI1uPr4qogAzH8,788
|
|
14
|
+
flexorch_sdk/resources/usage.py,sha256=2tZcRE76G5VUXc33_3K8Ypb7oKRNFvc62aM4mS_2bEY,1343
|
|
15
|
+
flexorch_sdk/resources/webhooks.py,sha256=sjwJUTNn82TR9aforoMj1zg_0pYplA4Bmu-V2r7Ti_0,1823
|
|
16
|
+
flexorch_sdk-0.1.0.dist-info/METADATA,sha256=5pH1dBSVgZkm2RR1wGjecLeDI8WVETgt1NAVMG0LMow,9114
|
|
17
|
+
flexorch_sdk-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
18
|
+
flexorch_sdk-0.1.0.dist-info/licenses/LICENSE,sha256=8c-X4OCfQIme81CSEakvgKLVka5i6gpYaHihSWGcaRI,1076
|
|
19
|
+
flexorch_sdk-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Flexorch Technology
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|