liquid-api 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- liquid/__init__.py +60 -0
- liquid/_defaults.py +58 -0
- liquid/auth/__init__.py +8 -0
- liquid/auth/classifier.py +73 -0
- liquid/auth/manager.py +108 -0
- liquid/client.py +213 -0
- liquid/discovery/__init__.py +18 -0
- liquid/discovery/base.py +53 -0
- liquid/discovery/browser.py +175 -0
- liquid/discovery/diff.py +66 -0
- liquid/discovery/graphql.py +180 -0
- liquid/discovery/mcp.py +159 -0
- liquid/discovery/openapi.py +227 -0
- liquid/discovery/rest_heuristic.py +157 -0
- liquid/events.py +37 -0
- liquid/exceptions.py +51 -0
- liquid/mapping/__init__.py +9 -0
- liquid/mapping/learning.py +62 -0
- liquid/mapping/proposer.py +150 -0
- liquid/mapping/reviewer.py +84 -0
- liquid/models/__init__.py +36 -0
- liquid/models/adapter.py +35 -0
- liquid/models/llm.py +42 -0
- liquid/models/schema.py +84 -0
- liquid/models/sync.py +35 -0
- liquid/protocols.py +29 -0
- liquid/py.typed +0 -0
- liquid/sync/__init__.py +29 -0
- liquid/sync/auto_repair.py +64 -0
- liquid/sync/engine.py +176 -0
- liquid/sync/fetcher.py +92 -0
- liquid/sync/mapper.py +73 -0
- liquid/sync/pagination.py +102 -0
- liquid/sync/retry.py +47 -0
- liquid/sync/selector.py +32 -0
- liquid/sync/transform.py +103 -0
- liquid_api-0.2.0.dist-info/METADATA +177 -0
- liquid_api-0.2.0.dist-info/RECORD +39 -0
- liquid_api-0.2.0.dist-info/WHEEL +4 -0
liquid/sync/fetcher.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import httpx # noqa: TC002
|
|
6
|
+
|
|
7
|
+
from liquid.exceptions import (
|
|
8
|
+
AuthError,
|
|
9
|
+
EndpointGoneError,
|
|
10
|
+
RateLimitError,
|
|
11
|
+
ServiceDownError,
|
|
12
|
+
)
|
|
13
|
+
from liquid.models.schema import Endpoint # noqa: TC001
|
|
14
|
+
from liquid.protocols import Vault # noqa: TC001
|
|
15
|
+
from liquid.sync.pagination import NoPagination, PaginationStrategy
|
|
16
|
+
from liquid.sync.selector import RecordSelector
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class FetchResult:
|
|
20
|
+
__slots__ = ("next_cursor", "raw_response", "records")
|
|
21
|
+
|
|
22
|
+
def __init__(self, records: list[dict[str, Any]], next_cursor: str | None, raw_response: httpx.Response) -> None:
|
|
23
|
+
self.records = records
|
|
24
|
+
self.next_cursor = next_cursor
|
|
25
|
+
self.raw_response = raw_response
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Fetcher:
|
|
29
|
+
def __init__(
|
|
30
|
+
self,
|
|
31
|
+
http_client: httpx.AsyncClient,
|
|
32
|
+
vault: Vault,
|
|
33
|
+
pagination: PaginationStrategy | None = None,
|
|
34
|
+
selector: RecordSelector | None = None,
|
|
35
|
+
extra_headers: dict[str, str] | None = None,
|
|
36
|
+
) -> None:
|
|
37
|
+
self.http_client = http_client
|
|
38
|
+
self.vault = vault
|
|
39
|
+
self.pagination = pagination or NoPagination()
|
|
40
|
+
self.selector = selector or RecordSelector()
|
|
41
|
+
self.extra_headers = extra_headers or {}
|
|
42
|
+
|
|
43
|
+
async def fetch(
|
|
44
|
+
self,
|
|
45
|
+
endpoint: Endpoint,
|
|
46
|
+
base_url: str,
|
|
47
|
+
auth_ref: str,
|
|
48
|
+
cursor: str | None = None,
|
|
49
|
+
) -> FetchResult:
|
|
50
|
+
auth_value = await self.vault.get(auth_ref)
|
|
51
|
+
headers = {**self.extra_headers, "Authorization": f"Bearer {auth_value}"}
|
|
52
|
+
|
|
53
|
+
params = self.pagination.get_request_params(cursor)
|
|
54
|
+
|
|
55
|
+
url = f"{base_url.rstrip('/')}{endpoint.path}"
|
|
56
|
+
response = await self.http_client.request(
|
|
57
|
+
method=endpoint.method,
|
|
58
|
+
url=url,
|
|
59
|
+
params=params,
|
|
60
|
+
headers=headers,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
_check_response(response)
|
|
64
|
+
|
|
65
|
+
data = response.json()
|
|
66
|
+
records = self.selector.select(data)
|
|
67
|
+
next_cursor = self.pagination.extract_next_cursor(response)
|
|
68
|
+
|
|
69
|
+
return FetchResult(records=records, next_cursor=next_cursor, raw_response=response)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _check_response(response: httpx.Response) -> None:
|
|
73
|
+
if response.is_success:
|
|
74
|
+
return
|
|
75
|
+
|
|
76
|
+
status = response.status_code
|
|
77
|
+
text = response.text[:500]
|
|
78
|
+
|
|
79
|
+
if status == 401 or status == 403:
|
|
80
|
+
raise AuthError(f"Auth failed ({status}): {text}")
|
|
81
|
+
if status == 429:
|
|
82
|
+
retry_after = response.headers.get("retry-after")
|
|
83
|
+
raise RateLimitError(
|
|
84
|
+
f"Rate limited: {text}",
|
|
85
|
+
retry_after=float(retry_after) if retry_after else None,
|
|
86
|
+
)
|
|
87
|
+
if status == 404 or status == 410:
|
|
88
|
+
raise EndpointGoneError(f"Endpoint gone ({status}): {text}")
|
|
89
|
+
if status >= 500:
|
|
90
|
+
raise ServiceDownError(f"Server error ({status}): {text}")
|
|
91
|
+
|
|
92
|
+
response.raise_for_status()
|
liquid/sync/mapper.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from liquid.exceptions import FieldNotFoundError
|
|
6
|
+
from liquid.models import FieldMapping, MappedRecord
|
|
7
|
+
from liquid.sync.transform import UnsafeExpressionError, evaluate
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class RecordMapper:
|
|
11
|
+
def __init__(self, mappings: list[FieldMapping]) -> None:
|
|
12
|
+
self.mappings = mappings
|
|
13
|
+
|
|
14
|
+
def map_record(self, record: dict[str, Any], source_endpoint: str = "") -> MappedRecord:
|
|
15
|
+
mapped: dict[str, Any] = {}
|
|
16
|
+
errors: list[str] = []
|
|
17
|
+
|
|
18
|
+
for mapping in self.mappings:
|
|
19
|
+
try:
|
|
20
|
+
value = _extract_path(record, mapping.source_path)
|
|
21
|
+
except KeyError as e:
|
|
22
|
+
raise FieldNotFoundError(f"Field not found: {mapping.source_path}") from e
|
|
23
|
+
|
|
24
|
+
if mapping.transform:
|
|
25
|
+
try:
|
|
26
|
+
value = evaluate(mapping.transform, value)
|
|
27
|
+
except UnsafeExpressionError as e:
|
|
28
|
+
errors.append(f"Transform error for {mapping.source_path}: {e}")
|
|
29
|
+
continue
|
|
30
|
+
|
|
31
|
+
mapped[mapping.target_field] = value
|
|
32
|
+
|
|
33
|
+
return MappedRecord(
|
|
34
|
+
source_endpoint=source_endpoint,
|
|
35
|
+
source_data=record,
|
|
36
|
+
mapped_data=mapped,
|
|
37
|
+
mapping_errors=errors or None,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
def map_batch(self, records: list[dict[str, Any]], source_endpoint: str = "") -> list[MappedRecord]:
|
|
41
|
+
return [self.map_record(r, source_endpoint) for r in records]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _extract_path(data: Any, path: str) -> Any:
|
|
45
|
+
"""Extract value from nested dict using dot-notation path.
|
|
46
|
+
|
|
47
|
+
Supports:
|
|
48
|
+
- "field" -> data["field"]
|
|
49
|
+
- "nested.field" -> data["nested"]["field"]
|
|
50
|
+
- "items[].price" -> [item["price"] for item in data["items"]]
|
|
51
|
+
"""
|
|
52
|
+
parts = path.split(".")
|
|
53
|
+
current: Any = data
|
|
54
|
+
|
|
55
|
+
for part in parts:
|
|
56
|
+
if part.endswith("[]"):
|
|
57
|
+
key = part[:-2]
|
|
58
|
+
if key:
|
|
59
|
+
if not isinstance(current, dict) or key not in current:
|
|
60
|
+
raise KeyError(path)
|
|
61
|
+
current = current[key]
|
|
62
|
+
if not isinstance(current, list):
|
|
63
|
+
raise KeyError(path)
|
|
64
|
+
remaining = ".".join(parts[parts.index(part) + 1 :])
|
|
65
|
+
if remaining:
|
|
66
|
+
return [_extract_path(item, remaining) for item in current]
|
|
67
|
+
return current
|
|
68
|
+
else:
|
|
69
|
+
if not isinstance(current, dict) or part not in current:
|
|
70
|
+
raise KeyError(path)
|
|
71
|
+
current = current[part]
|
|
72
|
+
|
|
73
|
+
return current
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from typing import Any, Protocol, runtime_checkable
|
|
5
|
+
|
|
6
|
+
import httpx # noqa: TC002
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@runtime_checkable
|
|
10
|
+
class PaginationStrategy(Protocol):
|
|
11
|
+
def get_request_params(self, cursor: str | None) -> dict[str, Any]:
|
|
12
|
+
"""Return query params to add for this page."""
|
|
13
|
+
...
|
|
14
|
+
|
|
15
|
+
def extract_next_cursor(self, response: httpx.Response) -> str | None:
|
|
16
|
+
"""Extract next cursor from response. Returns None if no more pages."""
|
|
17
|
+
...
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class NoPagination:
|
|
21
|
+
def get_request_params(self, cursor: str | None) -> dict[str, Any]:
|
|
22
|
+
return {}
|
|
23
|
+
|
|
24
|
+
def extract_next_cursor(self, response: httpx.Response) -> str | None:
|
|
25
|
+
return None
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class CursorPagination:
|
|
29
|
+
def __init__(self, cursor_param: str = "cursor", response_cursor_path: str = "next_cursor") -> None:
|
|
30
|
+
self.cursor_param = cursor_param
|
|
31
|
+
self.response_cursor_path = response_cursor_path
|
|
32
|
+
|
|
33
|
+
def get_request_params(self, cursor: str | None) -> dict[str, Any]:
|
|
34
|
+
if cursor is None:
|
|
35
|
+
return {}
|
|
36
|
+
return {self.cursor_param: cursor}
|
|
37
|
+
|
|
38
|
+
def extract_next_cursor(self, response: httpx.Response) -> str | None:
|
|
39
|
+
data = response.json()
|
|
40
|
+
value = _extract_nested(data, self.response_cursor_path)
|
|
41
|
+
return str(value) if value else None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class OffsetPagination:
|
|
45
|
+
def __init__(self, offset_param: str = "offset", limit_param: str = "limit", limit: int = 100) -> None:
|
|
46
|
+
self.offset_param = offset_param
|
|
47
|
+
self.limit_param = limit_param
|
|
48
|
+
self.limit = limit
|
|
49
|
+
|
|
50
|
+
def get_request_params(self, cursor: str | None) -> dict[str, Any]:
|
|
51
|
+
offset = int(cursor) if cursor else 0
|
|
52
|
+
return {self.offset_param: offset, self.limit_param: self.limit}
|
|
53
|
+
|
|
54
|
+
def extract_next_cursor(self, response: httpx.Response) -> str | None:
|
|
55
|
+
data = response.json()
|
|
56
|
+
records = data if isinstance(data, list) else data.get("data", data.get("results", []))
|
|
57
|
+
if isinstance(records, list) and len(records) >= self.limit:
|
|
58
|
+
current_offset = int(response.request.url.params.get(self.offset_param, "0"))
|
|
59
|
+
return str(current_offset + self.limit)
|
|
60
|
+
return None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class PageNumberPagination:
|
|
64
|
+
def __init__(self, page_param: str = "page", per_page_param: str = "per_page", per_page: int = 100) -> None:
|
|
65
|
+
self.page_param = page_param
|
|
66
|
+
self.per_page_param = per_page_param
|
|
67
|
+
self.per_page = per_page
|
|
68
|
+
|
|
69
|
+
def get_request_params(self, cursor: str | None) -> dict[str, Any]:
|
|
70
|
+
page = int(cursor) if cursor else 1
|
|
71
|
+
return {self.page_param: page, self.per_page_param: self.per_page}
|
|
72
|
+
|
|
73
|
+
def extract_next_cursor(self, response: httpx.Response) -> str | None:
|
|
74
|
+
data = response.json()
|
|
75
|
+
records = data if isinstance(data, list) else data.get("data", data.get("results", []))
|
|
76
|
+
if isinstance(records, list) and len(records) >= self.per_page:
|
|
77
|
+
current_page = int(response.request.url.params.get(self.page_param, "1"))
|
|
78
|
+
return str(current_page + 1)
|
|
79
|
+
return None
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class LinkHeaderPagination:
|
|
83
|
+
_LINK_RE = re.compile(r'<([^>]+)>;\s*rel="next"')
|
|
84
|
+
|
|
85
|
+
def get_request_params(self, cursor: str | None) -> dict[str, Any]:
|
|
86
|
+
return {}
|
|
87
|
+
|
|
88
|
+
def extract_next_cursor(self, response: httpx.Response) -> str | None:
|
|
89
|
+
link_header = response.headers.get("link", "")
|
|
90
|
+
match = self._LINK_RE.search(link_header)
|
|
91
|
+
return match.group(1) if match else None
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _extract_nested(data: dict[str, Any], path: str) -> Any:
|
|
95
|
+
parts = path.split(".")
|
|
96
|
+
current: Any = data
|
|
97
|
+
for part in parts:
|
|
98
|
+
if isinstance(current, dict):
|
|
99
|
+
current = current.get(part)
|
|
100
|
+
else:
|
|
101
|
+
return None
|
|
102
|
+
return current
|
liquid/sync/retry.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import logging
|
|
5
|
+
from collections.abc import Awaitable, Callable # noqa: TC003
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
|
|
8
|
+
from liquid.exceptions import RateLimitError, ServiceDownError
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class RetryPolicy:
|
|
15
|
+
max_retries: int = 3
|
|
16
|
+
base_delay: float = 1.0
|
|
17
|
+
max_delay: float = 60.0
|
|
18
|
+
exponential_base: float = 2.0
|
|
19
|
+
retryable_exceptions: tuple[type[Exception], ...] = field(
|
|
20
|
+
default_factory=lambda: (RateLimitError, ServiceDownError)
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
async def with_retry[T](fn: Callable[[], Awaitable[T]], policy: RetryPolicy) -> T:
|
|
25
|
+
last_exception: Exception | None = None
|
|
26
|
+
|
|
27
|
+
for attempt in range(policy.max_retries + 1):
|
|
28
|
+
try:
|
|
29
|
+
return await fn()
|
|
30
|
+
except policy.retryable_exceptions as e:
|
|
31
|
+
last_exception = e
|
|
32
|
+
if attempt == policy.max_retries:
|
|
33
|
+
break
|
|
34
|
+
|
|
35
|
+
delay = _compute_delay(e, attempt, policy)
|
|
36
|
+
logger.warning("Retry %d/%d after %.1fs: %s", attempt + 1, policy.max_retries, delay, e)
|
|
37
|
+
await asyncio.sleep(delay)
|
|
38
|
+
|
|
39
|
+
raise last_exception # type: ignore[misc]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _compute_delay(exc: Exception, attempt: int, policy: RetryPolicy) -> float:
|
|
43
|
+
if isinstance(exc, RateLimitError) and exc.retry_after is not None:
|
|
44
|
+
return min(exc.retry_after, policy.max_delay)
|
|
45
|
+
|
|
46
|
+
delay = policy.base_delay * (policy.exponential_base**attempt)
|
|
47
|
+
return min(delay, policy.max_delay)
|
liquid/sync/selector.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class RecordSelector:
|
|
7
|
+
"""Extracts records from nested JSON responses by a configurable path.
|
|
8
|
+
|
|
9
|
+
Example: RecordSelector("data.orders") extracts response["data"]["orders"].
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
def __init__(self, path: str | None = None) -> None:
|
|
13
|
+
self.path = path
|
|
14
|
+
|
|
15
|
+
def select(self, data: Any) -> list[dict[str, Any]]:
|
|
16
|
+
if self.path is None:
|
|
17
|
+
if isinstance(data, list):
|
|
18
|
+
return data
|
|
19
|
+
return [data] if isinstance(data, dict) else []
|
|
20
|
+
|
|
21
|
+
current: Any = data
|
|
22
|
+
for part in self.path.split("."):
|
|
23
|
+
if isinstance(current, dict):
|
|
24
|
+
current = current.get(part)
|
|
25
|
+
else:
|
|
26
|
+
return []
|
|
27
|
+
|
|
28
|
+
if isinstance(current, list):
|
|
29
|
+
return current
|
|
30
|
+
if isinstance(current, dict):
|
|
31
|
+
return [current]
|
|
32
|
+
return []
|
liquid/sync/transform.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import ast
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
_ALLOWED_BUILTINS = {"int", "float", "str", "bool", "abs", "round", "len", "min", "max"}
|
|
7
|
+
|
|
8
|
+
_builtins_ref = __builtins__ if isinstance(__builtins__, dict) else vars(__builtins__)
|
|
9
|
+
_SAFE_BUILTINS = {name: _builtins_ref[name] for name in _ALLOWED_BUILTINS}
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class UnsafeExpressionError(Exception):
|
|
13
|
+
pass
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def evaluate(expression: str, value: Any) -> Any:
|
|
17
|
+
"""Evaluate a transform expression safely using AST whitelisting.
|
|
18
|
+
|
|
19
|
+
Only allows: arithmetic, comparisons, attribute access on `value`,
|
|
20
|
+
subscript access on `value`, and a whitelist of builtins.
|
|
21
|
+
"""
|
|
22
|
+
try:
|
|
23
|
+
tree = ast.parse(expression, mode="eval")
|
|
24
|
+
except SyntaxError as e:
|
|
25
|
+
raise UnsafeExpressionError(f"Invalid expression syntax: {expression}") from e
|
|
26
|
+
|
|
27
|
+
_validate_node(tree.body)
|
|
28
|
+
|
|
29
|
+
code = compile(tree, "<transform>", "eval")
|
|
30
|
+
try:
|
|
31
|
+
return eval(code, {"__builtins__": _SAFE_BUILTINS}, {"value": value})
|
|
32
|
+
except Exception as e:
|
|
33
|
+
raise UnsafeExpressionError(f"Expression evaluation failed: {e}") from e
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _validate_node(node: ast.AST) -> None:
|
|
37
|
+
match node:
|
|
38
|
+
case ast.Expression(body=body):
|
|
39
|
+
_validate_node(body)
|
|
40
|
+
|
|
41
|
+
case ast.Constant():
|
|
42
|
+
pass
|
|
43
|
+
|
|
44
|
+
case ast.Name(id=name):
|
|
45
|
+
if name not in {"value", *_ALLOWED_BUILTINS}:
|
|
46
|
+
raise UnsafeExpressionError(f"Forbidden name: {name}")
|
|
47
|
+
|
|
48
|
+
case ast.UnaryOp(operand=operand):
|
|
49
|
+
_validate_node(operand)
|
|
50
|
+
|
|
51
|
+
case ast.BinOp(left=left, right=right):
|
|
52
|
+
_validate_node(left)
|
|
53
|
+
_validate_node(right)
|
|
54
|
+
|
|
55
|
+
case ast.BoolOp(values=values):
|
|
56
|
+
for v in values:
|
|
57
|
+
_validate_node(v)
|
|
58
|
+
|
|
59
|
+
case ast.Compare(left=left, comparators=comparators):
|
|
60
|
+
_validate_node(left)
|
|
61
|
+
for c in comparators:
|
|
62
|
+
_validate_node(c)
|
|
63
|
+
|
|
64
|
+
case ast.IfExp(test=test, body=body, orelse=orelse):
|
|
65
|
+
_validate_node(test)
|
|
66
|
+
_validate_node(body)
|
|
67
|
+
_validate_node(orelse)
|
|
68
|
+
|
|
69
|
+
case ast.Attribute(value=attr_value):
|
|
70
|
+
_validate_node(attr_value)
|
|
71
|
+
|
|
72
|
+
case ast.Subscript(value=sub_value, slice=slice_):
|
|
73
|
+
_validate_node(sub_value)
|
|
74
|
+
_validate_node(slice_)
|
|
75
|
+
|
|
76
|
+
case ast.Index(value=idx_value):
|
|
77
|
+
_validate_node(idx_value)
|
|
78
|
+
|
|
79
|
+
case ast.Call(func=func, args=args, keywords=keywords):
|
|
80
|
+
_validate_node(func)
|
|
81
|
+
for arg in args:
|
|
82
|
+
_validate_node(arg)
|
|
83
|
+
for kw in keywords:
|
|
84
|
+
_validate_node(kw.value)
|
|
85
|
+
|
|
86
|
+
case ast.List(elts=elts) | ast.Tuple(elts=elts) | ast.Set(elts=elts):
|
|
87
|
+
for elt in elts:
|
|
88
|
+
_validate_node(elt)
|
|
89
|
+
|
|
90
|
+
case ast.Dict(keys=keys, values=values):
|
|
91
|
+
for k in keys:
|
|
92
|
+
if k is not None:
|
|
93
|
+
_validate_node(k)
|
|
94
|
+
for v in values:
|
|
95
|
+
_validate_node(v)
|
|
96
|
+
|
|
97
|
+
case ast.Slice(lower=lower, upper=upper, step=step):
|
|
98
|
+
for part in (lower, upper, step):
|
|
99
|
+
if part is not None:
|
|
100
|
+
_validate_node(part)
|
|
101
|
+
|
|
102
|
+
case _:
|
|
103
|
+
raise UnsafeExpressionError(f"Forbidden AST node: {type(node).__name__}")
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: liquid-api
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: AI discovers APIs. Code syncs data. No adapters to write.
|
|
5
|
+
Project-URL: Homepage, https://github.com/ertad-family/liquid
|
|
6
|
+
Project-URL: Documentation, https://github.com/ertad-family/liquid/blob/main/docs/QUICKSTART.md
|
|
7
|
+
Project-URL: Repository, https://github.com/ertad-family/liquid
|
|
8
|
+
Project-URL: Issues, https://github.com/ertad-family/liquid/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/ertad-family/liquid/blob/main/CHANGELOG.md
|
|
10
|
+
License-Expression: AGPL-3.0-only
|
|
11
|
+
Keywords: adapter,ai,api,discovery,graphql,llm,mcp,openapi,sync
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: GNU Affero General Public License v3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Python: >=3.12
|
|
21
|
+
Requires-Dist: httpx>=0.27
|
|
22
|
+
Requires-Dist: pydantic>=2.7
|
|
23
|
+
Requires-Dist: pyyaml>=6.0
|
|
24
|
+
Provides-Extra: browser
|
|
25
|
+
Requires-Dist: playwright>=1.40; extra == 'browser'
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pre-commit>=4.0; extra == 'dev'
|
|
28
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
|
|
29
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
30
|
+
Requires-Dist: ruff>=0.11; extra == 'dev'
|
|
31
|
+
Provides-Extra: mcp
|
|
32
|
+
Requires-Dist: mcp>=1.0; extra == 'mcp'
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
|
|
35
|
+
<p align="center">
|
|
36
|
+
<h1 align="center">Liquid</h1>
|
|
37
|
+
<p align="center"><strong>AI discovers APIs. Code syncs data. No adapters to write.</strong></p>
|
|
38
|
+
</p>
|
|
39
|
+
|
|
40
|
+
<p align="center">
|
|
41
|
+
<a href="https://github.com/ertad-family/liquid/actions"><img src="https://img.shields.io/badge/tests-210%20passed-brightgreen" alt="Tests"></a>
|
|
42
|
+
<a href="https://github.com/ertad-family/liquid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-AGPL--3.0-blue" alt="License"></a>
|
|
43
|
+
<img src="https://img.shields.io/badge/python-3.12%2B-blue" alt="Python">
|
|
44
|
+
<img src="https://img.shields.io/badge/version-0.2.0-orange" alt="Version">
|
|
45
|
+
</p>
|
|
46
|
+
|
|
47
|
+
---
|
|
48
|
+
|
|
49
|
+
Point Liquid at any URL. AI discovers the API, proposes field mappings to your data model, and generates a deterministic adapter. After human approval, sync runs on schedule with **zero LLM calls**.
|
|
50
|
+
|
|
51
|
+
```
|
|
52
|
+
URL ──→ AI discovers API ──→ Human verifies mapping ──→ Deterministic sync
|
|
53
|
+
(once) (one-time review) (forever, no LLM)
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## The Problem
|
|
57
|
+
|
|
58
|
+
Connecting to external APIs requires custom code per service. 50 services = 50 adapters. Each with unique endpoints, auth flows, pagination, and data models. Writing and maintaining them doesn't scale.
|
|
59
|
+
|
|
60
|
+
## The Solution
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
from liquid import Liquid, SyncConfig
|
|
64
|
+
from liquid._defaults import InMemoryVault, CollectorSink
|
|
65
|
+
|
|
66
|
+
client = Liquid(llm=my_llm, vault=InMemoryVault(), sink=CollectorSink())
|
|
67
|
+
|
|
68
|
+
# 1. AI discovers the API (once)
|
|
69
|
+
schema = await client.discover("https://api.shopify.com")
|
|
70
|
+
|
|
71
|
+
# 2. AI proposes field mappings → human reviews
|
|
72
|
+
review = await client.propose_mappings(schema, {"amount": "float", "date": "datetime"})
|
|
73
|
+
review.approve_all()
|
|
74
|
+
|
|
75
|
+
# 3. Create adapter config
|
|
76
|
+
config = await client.create_adapter(
|
|
77
|
+
schema=schema,
|
|
78
|
+
auth_ref="vault/shopify",
|
|
79
|
+
mappings=review.finalize(),
|
|
80
|
+
sync_config=SyncConfig(endpoints=["/orders"], schedule="0 */6 * * *"),
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
# 4. Deterministic sync — no AI, runs forever
|
|
84
|
+
result = await client.sync(config)
|
|
85
|
+
print(f"Synced {result.records_delivered} records")
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## How Discovery Works
|
|
89
|
+
|
|
90
|
+
Liquid tries the cheapest method first, falls through on failure:
|
|
91
|
+
|
|
92
|
+
| Priority | Strategy | When it works | AI needed? |
|
|
93
|
+
|----------|----------|---------------|------------|
|
|
94
|
+
| 1 | **MCP** | Service publishes an MCP server | No |
|
|
95
|
+
| 2 | **OpenAPI** | Has `/openapi.json` or `/swagger.json` | No |
|
|
96
|
+
| 3 | **GraphQL** | Has `/graphql` with introspection | No |
|
|
97
|
+
| 4 | **REST Heuristic** | REST API without spec | Yes (once) |
|
|
98
|
+
| 5 | **Browser** | No API at all — capture network traffic | Yes (once) |
|
|
99
|
+
|
|
100
|
+
## Key Features
|
|
101
|
+
|
|
102
|
+
**Progressive Discovery** — MCP → OpenAPI → GraphQL → REST → Browser. Cheapest first.
|
|
103
|
+
|
|
104
|
+
**Selective Re-mapping** — When APIs change, `repair_adapter()` diffs schemas and re-maps only broken fields. Working mappings stay untouched.
|
|
105
|
+
|
|
106
|
+
**Safe Transforms** — Field transforms like `value * -1` or `value.lower()` are evaluated via AST whitelisting. No `eval()`, no injection risk.
|
|
107
|
+
|
|
108
|
+
**Pluggable Pagination** — Cursor, offset, page number, link header. Each is a strategy, not a switch/case.
|
|
109
|
+
|
|
110
|
+
**Learning System** — Corrections improve future proposals. Connect Shopify for the 51st time → mapping is instant.
|
|
111
|
+
|
|
112
|
+
**Auth Classification** — Detects OAuth (Tier A), app registration (Tier B), or manual credentials (Tier C). Returns structured escalation info.
|
|
113
|
+
|
|
114
|
+
## Installation
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
pip install liquid # core
|
|
118
|
+
pip install liquid[mcp] # + MCP server discovery
|
|
119
|
+
pip install liquid[browser] # + Playwright browser discovery
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
## Architecture
|
|
123
|
+
|
|
124
|
+
```
|
|
125
|
+
┌─────────────┐ ┌──────────────┐ ┌────────────────┐ ┌─────────────┐
|
|
126
|
+
│ Discovery │──→│ Auth Setup │──→│ Field Mapping │──→│ Sync Engine │
|
|
127
|
+
│ (AI, once) │ │ (AI + human) │ │ (AI + human) │ │ (code, loop)│
|
|
128
|
+
└─────────────┘ └──────────────┘ └────────────────┘ └─────────────┘
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
**Liquid is a library, not a framework.** You control when to discover, how to present mappings, where to store configs, and what to do with synced data.
|
|
132
|
+
|
|
133
|
+
### Extension Points (Protocols)
|
|
134
|
+
|
|
135
|
+
| Protocol | Purpose | You provide |
|
|
136
|
+
|----------|---------|-------------|
|
|
137
|
+
| `Vault` | Credential storage | Postgres, AWS Secrets Manager, etc. |
|
|
138
|
+
| `LLMBackend` | AI provider | Claude, GPT, Llama, any LLM |
|
|
139
|
+
| `DataSink` | Where data goes | Database, queue, webhook, file |
|
|
140
|
+
| `KnowledgeStore` | Shared mappings | Redis, central registry, or disabled |
|
|
141
|
+
|
|
142
|
+
## Auto-Repair on API Changes
|
|
143
|
+
|
|
144
|
+
When an API breaks your adapter:
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
result = await client.repair_adapter(config, target_model, auto_approve=True)
|
|
148
|
+
# Re-discovers → diffs schemas → selectively re-maps broken fields
|
|
149
|
+
# Returns updated AdapterConfig or MappingReview for human review
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
## Liquid vs Alternatives
|
|
153
|
+
|
|
154
|
+
| | Liquid | Airbyte | Nango | Custom code |
|
|
155
|
+
|---|---|---|---|---|
|
|
156
|
+
| **New service** | `discover(url)` | Write connector YAML | Write TypeScript sync | Write adapter from scratch |
|
|
157
|
+
| **AI involvement** | Discovery only, then deterministic | None | AI-generated code | None |
|
|
158
|
+
| **Auth handling** | Classifies & escalates | Per-connector | Managed OAuth | Manual |
|
|
159
|
+
| **When API changes** | `repair_adapter()` | Update connector | Update sync code | Debug & fix |
|
|
160
|
+
| **Runtime LLM calls** | Zero | Zero | Zero | N/A |
|
|
161
|
+
| **Self-hosted** | Yes (library) | Yes (platform) | Yes (platform) | Yes |
|
|
162
|
+
| **License** | AGPL-3.0 | ELv2 | AGPL-3.0 | Yours |
|
|
163
|
+
|
|
164
|
+
## Documentation
|
|
165
|
+
|
|
166
|
+
- [Quick Start Guide](docs/QUICKSTART.md)
|
|
167
|
+
- [Architecture](docs/ARCHITECTURE.md)
|
|
168
|
+
- [Extending Liquid](docs/EXTENDING.md)
|
|
169
|
+
- [Contributing](CONTRIBUTING.md)
|
|
170
|
+
|
|
171
|
+
## Contributing
|
|
172
|
+
|
|
173
|
+
We welcome contributions! Check out our [contributing guide](CONTRIBUTING.md) and browse [good first issues](https://github.com/ertad-family/liquid/labels/good%20first%20issue).
|
|
174
|
+
|
|
175
|
+
## License
|
|
176
|
+
|
|
177
|
+
AGPL-3.0. Commercial licenses available — [contact us](mailto:hello@ertad.com).
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
liquid/__init__.py,sha256=P9IR_VrYoBp6CWxm74H0rOxlJDv0elkvelSo5OtBFZw,1213
|
|
2
|
+
liquid/_defaults.py,sha256=6qMXFM58BMyrfacuxKFGHz-6__X3GwQDE5-l0HHdAmY,1911
|
|
3
|
+
liquid/client.py,sha256=DPOx9emu88OJwy81Rh_BePaXQARnGq5d3puIxlBZK6A,7725
|
|
4
|
+
liquid/events.py,sha256=r7nzsObdikTulvBi_d1mH1ET2K7JheY5yq_1R225n50,795
|
|
5
|
+
liquid/exceptions.py,sha256=jmSAQnUARWHJv6c8tmxn2TrQcKXbhzilWaaY4jqsdKs,783
|
|
6
|
+
liquid/protocols.py,sha256=mcZaLcEM-flix3C-MRKCJFPQpbmH0nxEw6p_Y2ZQxuE,961
|
|
7
|
+
liquid/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
liquid/auth/__init__.py,sha256=3yWTthUbXyfQyhngDnf0x9OYNY_WNZPJKTzVqnBDlQA,188
|
|
9
|
+
liquid/auth/classifier.py,sha256=d5kU58dnqxFwF8ZlgdSDyfBGWv-JfJA5ppIejJEsPDk,2727
|
|
10
|
+
liquid/auth/manager.py,sha256=5__i4EhqaIs2K9-zr026N96UCUdCt_l2wVAb8fkiJfM,4313
|
|
11
|
+
liquid/discovery/__init__.py,sha256=zS3DV6GCE065ZL4mxbve9C8RNetF06WMzHJND3tA9qo,600
|
|
12
|
+
liquid/discovery/base.py,sha256=fGBWgeaKcRAl3zH8Q42j4b7UQSZ34epjk6SWiH-pHH4,1973
|
|
13
|
+
liquid/discovery/browser.py,sha256=tj1EderVz7L_FpBaTYYC-hNBH6UMZElWiqYJ4oWfXtM,6506
|
|
14
|
+
liquid/discovery/diff.py,sha256=ypUPIhuThb71XGT312opRwscGDY_CEkTFPYr0Ujcq78,2408
|
|
15
|
+
liquid/discovery/graphql.py,sha256=e6lwsmLn09N04CCReaSPiNVh0cxk1xPrg2KbWEI6OiI,5920
|
|
16
|
+
liquid/discovery/mcp.py,sha256=eVBA6qX2TydTXx4n6M6Fh4lVazEdmHXdbEn7YrlKRaI,5458
|
|
17
|
+
liquid/discovery/openapi.py,sha256=qO_4L3hvFzEnB0vFPTJa5CtBhhhNbSMNjBZ73eXpamc,8349
|
|
18
|
+
liquid/discovery/rest_heuristic.py,sha256=LG_OqvA7aPxjsVPFvwBmOKZDyfaD2pJytq6qrFL8Mig,4942
|
|
19
|
+
liquid/mapping/__init__.py,sha256=5Mrfdl3A94_U5HXuChIWxMZUQTKFI18Yq9SvqEF3WaM,234
|
|
20
|
+
liquid/mapping/learning.py,sha256=75R3vHlcM9DP9MDexX-PCeZaZgEWwWupikI1EkXXV5Y,1961
|
|
21
|
+
liquid/mapping/proposer.py,sha256=nd2ZeOoyS-fnrE1pkzUBkvPzyeaQQ22UHZ3F15jzYrc,5394
|
|
22
|
+
liquid/mapping/reviewer.py,sha256=5mBqFZcP1hXHbAU5FyAIqqLKFZZpmAUj_Zw_Lt7Tt4U,3027
|
|
23
|
+
liquid/models/__init__.py,sha256=w75ZYdLy1Nw3xWOVq0hG-jhmrqdAYI1pzIYBhyyPZcQ,808
|
|
24
|
+
liquid/models/adapter.py,sha256=THedZbokZhBqR9Ib9yf3TLuNf2UStJpQfczdwNpfUEQ,882
|
|
25
|
+
liquid/models/llm.py,sha256=i8WJqKOyGX2ulsQ6JDDKxnqBjgaK6dm0gh2UT3728Tc,914
|
|
26
|
+
liquid/models/schema.py,sha256=_DtqCDdBWVeDlBFmXdUo0PW9S5Pdc8ZQIku5OI3Wy0Y,2422
|
|
27
|
+
liquid/models/sync.py,sha256=pr7i9HqCLrIirOqeDb92jAEwKRMDOOh9HEw7AUbiusQ,876
|
|
28
|
+
liquid/sync/__init__.py,sha256=6DsVA608bJA8po89T02yJOHdud3HB50dITgqQhighnQ,731
|
|
29
|
+
liquid/sync/auto_repair.py,sha256=AHEkC047On6iNcSklSJ_FkZ_iQFclk7poc-kMR_Az60,2190
|
|
30
|
+
liquid/sync/engine.py,sha256=qUpHekEOH3fZuSQBlE7BDtBEgVVTXWCP4KK46swl11s,6279
|
|
31
|
+
liquid/sync/fetcher.py,sha256=e6ze0Ss3L1Kdc6NGHNodIy6XTZwj2-nwgnzLM-MqB2I,2826
|
|
32
|
+
liquid/sync/mapper.py,sha256=MFbr7G6FWpTjDTnsLY60UcaD-YfuSqbLuGtSklyf6AI,2499
|
|
33
|
+
liquid/sync/pagination.py,sha256=e6QhiPPq5LJM2BrDV9hugXnPMCWaLEGo7Y5Siq1-fdE,3728
|
|
34
|
+
liquid/sync/retry.py,sha256=aIfj67K4GHCZW0a63KzEuUeT8ekk0JbIpgJpqvuIyRU,1486
|
|
35
|
+
liquid/sync/selector.py,sha256=4uMBnfKqQSQ03WpHowGTaXjVW6xV0wcvvu8sC1dpCw4,897
|
|
36
|
+
liquid/sync/transform.py,sha256=bSiwzW-rDLDbtBbN-R77zz2P7vbJ51hgWoT2rFLw9Eo,3238
|
|
37
|
+
liquid_api-0.2.0.dist-info/METADATA,sha256=wWmA0Ef5MKwvr5oHDonrH5y9gxYsFR908kOh8sdhgoc,7641
|
|
38
|
+
liquid_api-0.2.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
39
|
+
liquid_api-0.2.0.dist-info/RECORD,,
|