liquid-api 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- liquid_api-0.2.0/.github/ISSUE_TEMPLATE/bug_report.md +31 -0
- liquid_api-0.2.0/.github/ISSUE_TEMPLATE/feature_request.md +23 -0
- liquid_api-0.2.0/.github/PULL_REQUEST_TEMPLATE.md +20 -0
- liquid_api-0.2.0/.gitignore +40 -0
- liquid_api-0.2.0/.pre-commit-config.yaml +7 -0
- liquid_api-0.2.0/CHANGELOG.md +29 -0
- liquid_api-0.2.0/CLAUDE.md +57 -0
- liquid_api-0.2.0/CODE_OF_CONDUCT.md +31 -0
- liquid_api-0.2.0/CONTRIBUTING.md +104 -0
- liquid_api-0.2.0/PKG-INFO +177 -0
- liquid_api-0.2.0/README.md +143 -0
- liquid_api-0.2.0/SECURITY.md +32 -0
- liquid_api-0.2.0/docs/ARCHITECTURE.md +316 -0
- liquid_api-0.2.0/docs/EXTENDING.md +201 -0
- liquid_api-0.2.0/docs/QUICKSTART.md +93 -0
- liquid_api-0.2.0/docs/blog/how-i-built-liquid.md +116 -0
- liquid_api-0.2.0/docs/blog/launch-posts.md +152 -0
- liquid_api-0.2.0/pyproject.toml +77 -0
- liquid_api-0.2.0/src/liquid/__init__.py +60 -0
- liquid_api-0.2.0/src/liquid/_defaults.py +58 -0
- liquid_api-0.2.0/src/liquid/auth/__init__.py +8 -0
- liquid_api-0.2.0/src/liquid/auth/classifier.py +73 -0
- liquid_api-0.2.0/src/liquid/auth/manager.py +108 -0
- liquid_api-0.2.0/src/liquid/client.py +213 -0
- liquid_api-0.2.0/src/liquid/discovery/__init__.py +18 -0
- liquid_api-0.2.0/src/liquid/discovery/base.py +53 -0
- liquid_api-0.2.0/src/liquid/discovery/browser.py +175 -0
- liquid_api-0.2.0/src/liquid/discovery/diff.py +66 -0
- liquid_api-0.2.0/src/liquid/discovery/graphql.py +180 -0
- liquid_api-0.2.0/src/liquid/discovery/mcp.py +159 -0
- liquid_api-0.2.0/src/liquid/discovery/openapi.py +227 -0
- liquid_api-0.2.0/src/liquid/discovery/rest_heuristic.py +157 -0
- liquid_api-0.2.0/src/liquid/events.py +37 -0
- liquid_api-0.2.0/src/liquid/exceptions.py +51 -0
- liquid_api-0.2.0/src/liquid/mapping/__init__.py +9 -0
- liquid_api-0.2.0/src/liquid/mapping/learning.py +62 -0
- liquid_api-0.2.0/src/liquid/mapping/proposer.py +150 -0
- liquid_api-0.2.0/src/liquid/mapping/reviewer.py +84 -0
- liquid_api-0.2.0/src/liquid/models/__init__.py +36 -0
- liquid_api-0.2.0/src/liquid/models/adapter.py +35 -0
- liquid_api-0.2.0/src/liquid/models/llm.py +42 -0
- liquid_api-0.2.0/src/liquid/models/schema.py +84 -0
- liquid_api-0.2.0/src/liquid/models/sync.py +35 -0
- liquid_api-0.2.0/src/liquid/protocols.py +29 -0
- liquid_api-0.2.0/src/liquid/py.typed +0 -0
- liquid_api-0.2.0/src/liquid/sync/__init__.py +29 -0
- liquid_api-0.2.0/src/liquid/sync/auto_repair.py +64 -0
- liquid_api-0.2.0/src/liquid/sync/engine.py +176 -0
- liquid_api-0.2.0/src/liquid/sync/fetcher.py +92 -0
- liquid_api-0.2.0/src/liquid/sync/mapper.py +73 -0
- liquid_api-0.2.0/src/liquid/sync/pagination.py +102 -0
- liquid_api-0.2.0/src/liquid/sync/retry.py +47 -0
- liquid_api-0.2.0/src/liquid/sync/selector.py +32 -0
- liquid_api-0.2.0/src/liquid/sync/transform.py +103 -0
- liquid_api-0.2.0/tests/__init__.py +0 -0
- liquid_api-0.2.0/tests/fixtures/graphql_introspection.json +106 -0
- liquid_api-0.2.0/tests/fixtures/petstore_openapi.json +104 -0
- liquid_api-0.2.0/tests/test_auth/__init__.py +0 -0
- liquid_api-0.2.0/tests/test_auth/test_classifier.py +41 -0
- liquid_api-0.2.0/tests/test_auth/test_manager.py +94 -0
- liquid_api-0.2.0/tests/test_client.py +217 -0
- liquid_api-0.2.0/tests/test_defaults.py +73 -0
- liquid_api-0.2.0/tests/test_discovery/__init__.py +0 -0
- liquid_api-0.2.0/tests/test_discovery/test_browser.py +81 -0
- liquid_api-0.2.0/tests/test_discovery/test_diff.py +156 -0
- liquid_api-0.2.0/tests/test_discovery/test_graphql.py +90 -0
- liquid_api-0.2.0/tests/test_discovery/test_mcp.py +115 -0
- liquid_api-0.2.0/tests/test_discovery/test_openapi.py +134 -0
- liquid_api-0.2.0/tests/test_discovery/test_pipeline.py +77 -0
- liquid_api-0.2.0/tests/test_discovery/test_rest_heuristic.py +78 -0
- liquid_api-0.2.0/tests/test_events.py +45 -0
- liquid_api-0.2.0/tests/test_exceptions.py +47 -0
- liquid_api-0.2.0/tests/test_mapping/__init__.py +0 -0
- liquid_api-0.2.0/tests/test_mapping/test_learning.py +66 -0
- liquid_api-0.2.0/tests/test_mapping/test_proposer.py +135 -0
- liquid_api-0.2.0/tests/test_mapping/test_reviewer.py +83 -0
- liquid_api-0.2.0/tests/test_models/__init__.py +0 -0
- liquid_api-0.2.0/tests/test_models/test_adapter.py +72 -0
- liquid_api-0.2.0/tests/test_models/test_llm.py +52 -0
- liquid_api-0.2.0/tests/test_models/test_schema.py +115 -0
- liquid_api-0.2.0/tests/test_models/test_sync.py +46 -0
- liquid_api-0.2.0/tests/test_protocols.py +54 -0
- liquid_api-0.2.0/tests/test_smoke.py +5 -0
- liquid_api-0.2.0/tests/test_sync/__init__.py +0 -0
- liquid_api-0.2.0/tests/test_sync/test_auto_repair.py +96 -0
- liquid_api-0.2.0/tests/test_sync/test_engine.py +110 -0
- liquid_api-0.2.0/tests/test_sync/test_fetcher.py +90 -0
- liquid_api-0.2.0/tests/test_sync/test_mapper.py +71 -0
- liquid_api-0.2.0/tests/test_sync/test_pagination.py +116 -0
- liquid_api-0.2.0/tests/test_sync/test_retry.py +71 -0
- liquid_api-0.2.0/tests/test_sync/test_selector.py +36 -0
- liquid_api-0.2.0/tests/test_sync/test_transform.py +53 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: Bug Report
|
|
3
|
+
about: Report a bug to help us improve Liquid
|
|
4
|
+
title: ''
|
|
5
|
+
labels: bug
|
|
6
|
+
assignees: ''
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## Describe the bug
|
|
10
|
+
A clear description of what the bug is.
|
|
11
|
+
|
|
12
|
+
## To Reproduce
|
|
13
|
+
Steps to reproduce the behavior:
|
|
14
|
+
1.
|
|
15
|
+
2.
|
|
16
|
+
3.
|
|
17
|
+
|
|
18
|
+
## Expected behavior
|
|
19
|
+
What you expected to happen.
|
|
20
|
+
|
|
21
|
+
## Actual behavior
|
|
22
|
+
What actually happened. Include error messages and tracebacks.
|
|
23
|
+
|
|
24
|
+
## Environment
|
|
25
|
+
- Python version:
|
|
26
|
+
- Liquid version:
|
|
27
|
+
- OS:
|
|
28
|
+
- LLM backend used:
|
|
29
|
+
|
|
30
|
+
## Additional context
|
|
31
|
+
Any other context about the problem.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: Feature Request
|
|
3
|
+
about: Suggest a feature for Liquid
|
|
4
|
+
title: ''
|
|
5
|
+
labels: enhancement
|
|
6
|
+
assignees: ''
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## Use case
|
|
10
|
+
Describe the problem or use case this feature would address.
|
|
11
|
+
|
|
12
|
+
## Proposed solution
|
|
13
|
+
Describe the solution you'd like, including API examples if applicable.
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
# Example usage of the proposed feature
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Alternatives considered
|
|
20
|
+
Describe any alternative solutions or workarounds you've considered.
|
|
21
|
+
|
|
22
|
+
## Additional context
|
|
23
|
+
Any other context, screenshots, or references.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
## Summary
|
|
2
|
+
|
|
3
|
+
Brief description of changes.
|
|
4
|
+
|
|
5
|
+
## Related Issue
|
|
6
|
+
|
|
7
|
+
Fixes #(issue number)
|
|
8
|
+
|
|
9
|
+
## Changes
|
|
10
|
+
|
|
11
|
+
-
|
|
12
|
+
-
|
|
13
|
+
|
|
14
|
+
## Checklist
|
|
15
|
+
|
|
16
|
+
- [ ] Tests added/updated
|
|
17
|
+
- [ ] `ruff check src/ tests/` passes
|
|
18
|
+
- [ ] `pytest tests/` passes
|
|
19
|
+
- [ ] Documentation updated (if applicable)
|
|
20
|
+
- [ ] PR targets `develop` branch
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.so
|
|
6
|
+
*.egg-info/
|
|
7
|
+
*.egg
|
|
8
|
+
dist/
|
|
9
|
+
build/
|
|
10
|
+
sdist/
|
|
11
|
+
|
|
12
|
+
# Virtual environments
|
|
13
|
+
.venv/
|
|
14
|
+
venv/
|
|
15
|
+
env/
|
|
16
|
+
|
|
17
|
+
# uv
|
|
18
|
+
uv.lock
|
|
19
|
+
|
|
20
|
+
# Testing
|
|
21
|
+
.pytest_cache/
|
|
22
|
+
.coverage
|
|
23
|
+
htmlcov/
|
|
24
|
+
.mypy_cache/
|
|
25
|
+
|
|
26
|
+
# IDEs
|
|
27
|
+
.idea/
|
|
28
|
+
.vscode/
|
|
29
|
+
*.swp
|
|
30
|
+
*.swo
|
|
31
|
+
*~
|
|
32
|
+
|
|
33
|
+
# OS
|
|
34
|
+
.DS_Store
|
|
35
|
+
Thumbs.db
|
|
36
|
+
|
|
37
|
+
# Environment
|
|
38
|
+
.env
|
|
39
|
+
.env.*
|
|
40
|
+
!.env.example
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to Liquid will be documented in this file.
|
|
4
|
+
|
|
5
|
+
## [0.2.0] - 2026-04-13
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
- `Liquid.repair_adapter()` — one-call flow for re-discovery, schema diffing, and selective re-mapping when APIs change
|
|
9
|
+
- `SchemaDiff` model and `diff_schemas()` utility for structured comparison of API schema versions
|
|
10
|
+
- `AutoRepairHandler` — opt-in event handler that triggers automatic repair on `ReDiscoveryNeeded`
|
|
11
|
+
- `AdapterRepaired` event emitted after successful repair
|
|
12
|
+
- Selective re-mapping in `MappingProposer.propose()` — keeps unchanged mappings, drops removed, LLM re-proposes broken
|
|
13
|
+
|
|
14
|
+
## [0.1.0] - 2026-04-13
|
|
15
|
+
|
|
16
|
+
### Added
|
|
17
|
+
- Initial release
|
|
18
|
+
- **Discovery Pipeline**: MCP, OpenAPI (v2+v3), GraphQL, REST heuristic, Browser (Playwright)
|
|
19
|
+
- **Auth Classification**: Tier A/B/C with structured escalation info
|
|
20
|
+
- **Auth Manager**: credential storage, header generation, OAuth token refresh
|
|
21
|
+
- **Field Mapping**: AI-powered proposals via LLM, human review workflow (approve/reject/correct), learning system
|
|
22
|
+
- **Sync Engine**: deterministic sync with zero LLM calls
|
|
23
|
+
- **Pagination**: cursor, offset, page number, link header (pluggable strategies)
|
|
24
|
+
- **Transform Evaluator**: safe AST-based expression evaluation
|
|
25
|
+
- **Retry**: exponential backoff with retry-after support
|
|
26
|
+
- **Events**: SyncCompleted, SyncFailed, ReDiscoveryNeeded
|
|
27
|
+
- **Protocols**: Vault, LLMBackend, DataSink, KnowledgeStore
|
|
28
|
+
- **Defaults**: InMemoryVault, InMemoryKnowledgeStore, CollectorSink, StdoutSink
|
|
29
|
+
- Documentation: QUICKSTART.md, EXTENDING.md, ARCHITECTURE.md
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# Liquid
|
|
2
|
+
|
|
3
|
+
## Project Overview
|
|
4
|
+
|
|
5
|
+
Python library for programmatic API discovery and adapter generation. AI discovers APIs once, then deterministic code syncs data without LLM calls.
|
|
6
|
+
|
|
7
|
+
## Tech Stack
|
|
8
|
+
|
|
9
|
+
- Python 3.12+
|
|
10
|
+
- Package manager: uv
|
|
11
|
+
- Build backend: hatchling
|
|
12
|
+
- Linter/formatter: ruff
|
|
13
|
+
- Tests: pytest + pytest-asyncio
|
|
14
|
+
- Pre-commit: ruff lint + ruff format
|
|
15
|
+
|
|
16
|
+
## Project Structure
|
|
17
|
+
|
|
18
|
+
```
|
|
19
|
+
src/liquid/ — library source code
|
|
20
|
+
tests/ — pytest tests
|
|
21
|
+
docs/ — architecture and design docs
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Development Commands
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
# Create/activate venv
|
|
28
|
+
uv venv .venv && source .venv/bin/activate
|
|
29
|
+
|
|
30
|
+
# Install with dev deps
|
|
31
|
+
uv pip install -e ".[dev]"
|
|
32
|
+
|
|
33
|
+
# Run tests
|
|
34
|
+
pytest tests/ -v
|
|
35
|
+
|
|
36
|
+
# Lint
|
|
37
|
+
ruff check src/ tests/
|
|
38
|
+
|
|
39
|
+
# Format
|
|
40
|
+
ruff format src/ tests/
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Git Workflow
|
|
44
|
+
|
|
45
|
+
GitFlow: `main` (releases) + `develop` (integration). Feature branches from `develop`.
|
|
46
|
+
|
|
47
|
+
- Branch naming: `feature/<name>`, `fix/<name>`, `release/<version>`
|
|
48
|
+
- PRs target `develop`, not `main`
|
|
49
|
+
- `main` receives merges only from `develop` via release branches
|
|
50
|
+
|
|
51
|
+
## Code Conventions
|
|
52
|
+
|
|
53
|
+
- src layout (`src/liquid/`)
|
|
54
|
+
- Async-first: use `async def` for I/O-bound operations
|
|
55
|
+
- Type hints on all public APIs
|
|
56
|
+
- Pydantic for data models
|
|
57
|
+
- Protocols for extension points (Vault, LLM, DataSink)
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# Contributor Covenant Code of Conduct
|
|
2
|
+
|
|
3
|
+
## Our Pledge
|
|
4
|
+
|
|
5
|
+
We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
|
|
6
|
+
|
|
7
|
+
## Our Standards
|
|
8
|
+
|
|
9
|
+
Examples of behavior that contributes to a positive environment:
|
|
10
|
+
|
|
11
|
+
- Using welcoming and inclusive language
|
|
12
|
+
- Being respectful of differing viewpoints and experiences
|
|
13
|
+
- Gracefully accepting constructive criticism
|
|
14
|
+
- Focusing on what is best for the community
|
|
15
|
+
- Showing empathy towards other community members
|
|
16
|
+
|
|
17
|
+
Examples of unacceptable behavior:
|
|
18
|
+
|
|
19
|
+
- The use of sexualized language or imagery, and sexual attention or advances of any kind
|
|
20
|
+
- Trolling, insulting or derogatory comments, and personal or political attacks
|
|
21
|
+
- Public or private harassment
|
|
22
|
+
- Publishing others' private information without explicit permission
|
|
23
|
+
- Other conduct which could reasonably be considered inappropriate in a professional setting
|
|
24
|
+
|
|
25
|
+
## Enforcement
|
|
26
|
+
|
|
27
|
+
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the project team at hello@ertad.com. All complaints will be reviewed and investigated promptly and fairly.
|
|
28
|
+
|
|
29
|
+
## Attribution
|
|
30
|
+
|
|
31
|
+
This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), version 2.1.
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
# Contributing to Liquid
|
|
2
|
+
|
|
3
|
+
Thank you for your interest in contributing! Liquid is an open-source project and we welcome contributions of all kinds.
|
|
4
|
+
|
|
5
|
+
## Getting Started
|
|
6
|
+
|
|
7
|
+
### Prerequisites
|
|
8
|
+
|
|
9
|
+
- Python 3.12+
|
|
10
|
+
- [uv](https://github.com/astral-sh/uv) package manager
|
|
11
|
+
|
|
12
|
+
### Setup
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
git clone https://github.com/ertad-family/liquid.git
|
|
16
|
+
cd liquid
|
|
17
|
+
uv venv .venv && source .venv/bin/activate
|
|
18
|
+
uv pip install -e ".[dev]"
|
|
19
|
+
.venv/bin/pre-commit install
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
### Run Tests
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pytest tests/ -v
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
### Lint & Format
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
ruff check src/ tests/
|
|
32
|
+
ruff format src/ tests/
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## How to Contribute
|
|
36
|
+
|
|
37
|
+
### Reporting Bugs
|
|
38
|
+
|
|
39
|
+
Open a [bug report](https://github.com/ertad-family/liquid/issues/new?template=bug_report.md) with:
|
|
40
|
+
- Steps to reproduce
|
|
41
|
+
- Expected vs actual behavior
|
|
42
|
+
- Python version and OS
|
|
43
|
+
|
|
44
|
+
### Suggesting Features
|
|
45
|
+
|
|
46
|
+
Open a [feature request](https://github.com/ertad-family/liquid/issues/new?template=feature_request.md) describing:
|
|
47
|
+
- The use case
|
|
48
|
+
- Proposed API/behavior
|
|
49
|
+
- Alternatives considered
|
|
50
|
+
|
|
51
|
+
### Good First Issues
|
|
52
|
+
|
|
53
|
+
New to the project? Check issues labeled [`good first issue`](https://github.com/ertad-family/liquid/labels/good%20first%20issue) — they're designed to be approachable without deep knowledge of the codebase.
|
|
54
|
+
|
|
55
|
+
### Pull Requests
|
|
56
|
+
|
|
57
|
+
1. Fork the repo and create a branch from `develop`:
|
|
58
|
+
```bash
|
|
59
|
+
git checkout -b feature/my-change develop
|
|
60
|
+
```
|
|
61
|
+
2. Make your changes
|
|
62
|
+
3. Add tests for new functionality
|
|
63
|
+
4. Ensure all tests pass: `pytest tests/ -v`
|
|
64
|
+
5. Ensure lint passes: `ruff check src/ tests/`
|
|
65
|
+
6. Commit with a clear message
|
|
66
|
+
7. Push and open a PR targeting `develop` (not `main`)
|
|
67
|
+
|
|
68
|
+
## Code Conventions
|
|
69
|
+
|
|
70
|
+
- **Async-first**: use `async def` for I/O-bound operations
|
|
71
|
+
- **Type hints**: required on all public APIs
|
|
72
|
+
- **Pydantic**: for data models
|
|
73
|
+
- **Protocols**: for extension points (not ABC)
|
|
74
|
+
- **Line length**: 120 characters
|
|
75
|
+
- **Tests**: pytest + pytest-asyncio
|
|
76
|
+
|
|
77
|
+
## Project Structure
|
|
78
|
+
|
|
79
|
+
```
|
|
80
|
+
src/liquid/
|
|
81
|
+
client.py — Main orchestrator (Liquid class)
|
|
82
|
+
protocols.py — Extension point interfaces
|
|
83
|
+
exceptions.py — Error hierarchy
|
|
84
|
+
events.py — Event system
|
|
85
|
+
_defaults.py — In-memory implementations for testing
|
|
86
|
+
models/ — Pydantic data models
|
|
87
|
+
discovery/ — API discovery strategies
|
|
88
|
+
auth/ — Auth classification and management
|
|
89
|
+
mapping/ — Field mapping (AI + human review)
|
|
90
|
+
sync/ — Deterministic sync engine
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## Git Workflow
|
|
94
|
+
|
|
95
|
+
We follow GitFlow:
|
|
96
|
+
- `main` — releases only
|
|
97
|
+
- `develop` — integration branch
|
|
98
|
+
- `feature/*` — new features
|
|
99
|
+
- `fix/*` — bug fixes
|
|
100
|
+
- PRs target `develop`, never `main`
|
|
101
|
+
|
|
102
|
+
## Code of Conduct
|
|
103
|
+
|
|
104
|
+
This project follows the [Contributor Covenant](CODE_OF_CONDUCT.md). By participating, you agree to uphold this code.
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: liquid-api
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: AI discovers APIs. Code syncs data. No adapters to write.
|
|
5
|
+
Project-URL: Homepage, https://github.com/ertad-family/liquid
|
|
6
|
+
Project-URL: Documentation, https://github.com/ertad-family/liquid/blob/main/docs/QUICKSTART.md
|
|
7
|
+
Project-URL: Repository, https://github.com/ertad-family/liquid
|
|
8
|
+
Project-URL: Issues, https://github.com/ertad-family/liquid/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/ertad-family/liquid/blob/main/CHANGELOG.md
|
|
10
|
+
License-Expression: AGPL-3.0-only
|
|
11
|
+
Keywords: adapter,ai,api,discovery,graphql,llm,mcp,openapi,sync
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: GNU Affero General Public License v3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Python: >=3.12
|
|
21
|
+
Requires-Dist: httpx>=0.27
|
|
22
|
+
Requires-Dist: pydantic>=2.7
|
|
23
|
+
Requires-Dist: pyyaml>=6.0
|
|
24
|
+
Provides-Extra: browser
|
|
25
|
+
Requires-Dist: playwright>=1.40; extra == 'browser'
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pre-commit>=4.0; extra == 'dev'
|
|
28
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
|
|
29
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
30
|
+
Requires-Dist: ruff>=0.11; extra == 'dev'
|
|
31
|
+
Provides-Extra: mcp
|
|
32
|
+
Requires-Dist: mcp>=1.0; extra == 'mcp'
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
|
|
35
|
+
<p align="center">
|
|
36
|
+
<h1 align="center">Liquid</h1>
|
|
37
|
+
<p align="center"><strong>AI discovers APIs. Code syncs data. No adapters to write.</strong></p>
|
|
38
|
+
</p>
|
|
39
|
+
|
|
40
|
+
<p align="center">
|
|
41
|
+
<a href="https://github.com/ertad-family/liquid/actions"><img src="https://img.shields.io/badge/tests-210%20passed-brightgreen" alt="Tests"></a>
|
|
42
|
+
<a href="https://github.com/ertad-family/liquid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-AGPL--3.0-blue" alt="License"></a>
|
|
43
|
+
<img src="https://img.shields.io/badge/python-3.12%2B-blue" alt="Python">
|
|
44
|
+
<img src="https://img.shields.io/badge/version-0.2.0-orange" alt="Version">
|
|
45
|
+
</p>
|
|
46
|
+
|
|
47
|
+
---
|
|
48
|
+
|
|
49
|
+
Point Liquid at any URL. AI discovers the API, proposes field mappings to your data model, and generates a deterministic adapter. After human approval, sync runs on schedule with **zero LLM calls**.
|
|
50
|
+
|
|
51
|
+
```
|
|
52
|
+
URL ──→ AI discovers API ──→ Human verifies mapping ──→ Deterministic sync
|
|
53
|
+
(once) (one-time review) (forever, no LLM)
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## The Problem
|
|
57
|
+
|
|
58
|
+
Connecting to external APIs requires custom code per service. 50 services = 50 adapters. Each with unique endpoints, auth flows, pagination, and data models. Writing and maintaining them doesn't scale.
|
|
59
|
+
|
|
60
|
+
## The Solution
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
from liquid import Liquid, SyncConfig
|
|
64
|
+
from liquid._defaults import InMemoryVault, CollectorSink
|
|
65
|
+
|
|
66
|
+
client = Liquid(llm=my_llm, vault=InMemoryVault(), sink=CollectorSink())
|
|
67
|
+
|
|
68
|
+
# 1. AI discovers the API (once)
|
|
69
|
+
schema = await client.discover("https://api.shopify.com")
|
|
70
|
+
|
|
71
|
+
# 2. AI proposes field mappings → human reviews
|
|
72
|
+
review = await client.propose_mappings(schema, {"amount": "float", "date": "datetime"})
|
|
73
|
+
review.approve_all()
|
|
74
|
+
|
|
75
|
+
# 3. Create adapter config
|
|
76
|
+
config = await client.create_adapter(
|
|
77
|
+
schema=schema,
|
|
78
|
+
auth_ref="vault/shopify",
|
|
79
|
+
mappings=review.finalize(),
|
|
80
|
+
sync_config=SyncConfig(endpoints=["/orders"], schedule="0 */6 * * *"),
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
# 4. Deterministic sync — no AI, runs forever
|
|
84
|
+
result = await client.sync(config)
|
|
85
|
+
print(f"Synced {result.records_delivered} records")
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## How Discovery Works
|
|
89
|
+
|
|
90
|
+
Liquid tries the cheapest method first, falls through on failure:
|
|
91
|
+
|
|
92
|
+
| Priority | Strategy | When it works | AI needed? |
|
|
93
|
+
|----------|----------|---------------|------------|
|
|
94
|
+
| 1 | **MCP** | Service publishes an MCP server | No |
|
|
95
|
+
| 2 | **OpenAPI** | Has `/openapi.json` or `/swagger.json` | No |
|
|
96
|
+
| 3 | **GraphQL** | Has `/graphql` with introspection | No |
|
|
97
|
+
| 4 | **REST Heuristic** | REST API without spec | Yes (once) |
|
|
98
|
+
| 5 | **Browser** | No API at all — capture network traffic | Yes (once) |
|
|
99
|
+
|
|
100
|
+
## Key Features
|
|
101
|
+
|
|
102
|
+
**Progressive Discovery** — MCP → OpenAPI → GraphQL → REST → Browser. Cheapest first.
|
|
103
|
+
|
|
104
|
+
**Selective Re-mapping** — When APIs change, `repair_adapter()` diffs schemas and re-maps only broken fields. Working mappings stay untouched.
|
|
105
|
+
|
|
106
|
+
**Safe Transforms** — Field transforms like `value * -1` or `value.lower()` are evaluated via AST whitelisting. No `eval()`, no injection risk.
|
|
107
|
+
|
|
108
|
+
**Pluggable Pagination** — Cursor, offset, page number, link header. Each is a strategy, not a switch/case.
|
|
109
|
+
|
|
110
|
+
**Learning System** — Corrections improve future proposals. Connect Shopify for the 51st time → mapping is instant.
|
|
111
|
+
|
|
112
|
+
**Auth Classification** — Detects OAuth (Tier A), app registration (Tier B), or manual credentials (Tier C). Returns structured escalation info.
|
|
113
|
+
|
|
114
|
+
## Installation
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
pip install liquid # core
|
|
118
|
+
pip install liquid[mcp] # + MCP server discovery
|
|
119
|
+
pip install liquid[browser] # + Playwright browser discovery
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
## Architecture
|
|
123
|
+
|
|
124
|
+
```
|
|
125
|
+
┌─────────────┐ ┌──────────────┐ ┌────────────────┐ ┌─────────────┐
|
|
126
|
+
│ Discovery │──→│ Auth Setup │──→│ Field Mapping │──→│ Sync Engine │
|
|
127
|
+
│ (AI, once) │ │ (AI + human) │ │ (AI + human) │ │ (code, loop)│
|
|
128
|
+
└─────────────┘ └──────────────┘ └────────────────┘ └─────────────┘
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
**Liquid is a library, not a framework.** You control when to discover, how to present mappings, where to store configs, and what to do with synced data.
|
|
132
|
+
|
|
133
|
+
### Extension Points (Protocols)
|
|
134
|
+
|
|
135
|
+
| Protocol | Purpose | You provide |
|
|
136
|
+
|----------|---------|-------------|
|
|
137
|
+
| `Vault` | Credential storage | Postgres, AWS Secrets Manager, etc. |
|
|
138
|
+
| `LLMBackend` | AI provider | Claude, GPT, Llama, any LLM |
|
|
139
|
+
| `DataSink` | Where data goes | Database, queue, webhook, file |
|
|
140
|
+
| `KnowledgeStore` | Shared mappings | Redis, central registry, or disabled |
|
|
141
|
+
|
|
142
|
+
## Auto-Repair on API Changes
|
|
143
|
+
|
|
144
|
+
When an API breaks your adapter:
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
result = await client.repair_adapter(config, target_model, auto_approve=True)
|
|
148
|
+
# Re-discovers → diffs schemas → selectively re-maps broken fields
|
|
149
|
+
# Returns updated AdapterConfig or MappingReview for human review
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
## Liquid vs Alternatives
|
|
153
|
+
|
|
154
|
+
| | Liquid | Airbyte | Nango | Custom code |
|
|
155
|
+
|---|---|---|---|---|
|
|
156
|
+
| **New service** | `discover(url)` | Write connector YAML | Write TypeScript sync | Write adapter from scratch |
|
|
157
|
+
| **AI involvement** | Discovery only, then deterministic | None | AI-generated code | None |
|
|
158
|
+
| **Auth handling** | Classifies & escalates | Per-connector | Managed OAuth | Manual |
|
|
159
|
+
| **When API changes** | `repair_adapter()` | Update connector | Update sync code | Debug & fix |
|
|
160
|
+
| **Runtime LLM calls** | Zero | Zero | Zero | N/A |
|
|
161
|
+
| **Self-hosted** | Yes (library) | Yes (platform) | Yes (platform) | Yes |
|
|
162
|
+
| **License** | AGPL-3.0 | ELv2 | AGPL-3.0 | Yours |
|
|
163
|
+
|
|
164
|
+
## Documentation
|
|
165
|
+
|
|
166
|
+
- [Quick Start Guide](docs/QUICKSTART.md)
|
|
167
|
+
- [Architecture](docs/ARCHITECTURE.md)
|
|
168
|
+
- [Extending Liquid](docs/EXTENDING.md)
|
|
169
|
+
- [Contributing](CONTRIBUTING.md)
|
|
170
|
+
|
|
171
|
+
## Contributing
|
|
172
|
+
|
|
173
|
+
We welcome contributions! Check out our [contributing guide](CONTRIBUTING.md) and browse [good first issues](https://github.com/ertad-family/liquid/labels/good%20first%20issue).
|
|
174
|
+
|
|
175
|
+
## License
|
|
176
|
+
|
|
177
|
+
AGPL-3.0. Commercial licenses available — [contact us](mailto:hello@ertad.com).
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<h1 align="center">Liquid</h1>
|
|
3
|
+
<p align="center"><strong>AI discovers APIs. Code syncs data. No adapters to write.</strong></p>
|
|
4
|
+
</p>
|
|
5
|
+
|
|
6
|
+
<p align="center">
|
|
7
|
+
<a href="https://github.com/ertad-family/liquid/actions"><img src="https://img.shields.io/badge/tests-210%20passed-brightgreen" alt="Tests"></a>
|
|
8
|
+
<a href="https://github.com/ertad-family/liquid/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-AGPL--3.0-blue" alt="License"></a>
|
|
9
|
+
<img src="https://img.shields.io/badge/python-3.12%2B-blue" alt="Python">
|
|
10
|
+
<img src="https://img.shields.io/badge/version-0.2.0-orange" alt="Version">
|
|
11
|
+
</p>
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
Point Liquid at any URL. AI discovers the API, proposes field mappings to your data model, and generates a deterministic adapter. After human approval, sync runs on schedule with **zero LLM calls**.
|
|
16
|
+
|
|
17
|
+
```
|
|
18
|
+
URL ──→ AI discovers API ──→ Human verifies mapping ──→ Deterministic sync
|
|
19
|
+
(once) (one-time review) (forever, no LLM)
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
## The Problem
|
|
23
|
+
|
|
24
|
+
Connecting to external APIs requires custom code per service. 50 services = 50 adapters. Each with unique endpoints, auth flows, pagination, and data models. Writing and maintaining them doesn't scale.
|
|
25
|
+
|
|
26
|
+
## The Solution
|
|
27
|
+
|
|
28
|
+
```python
|
|
29
|
+
from liquid import Liquid, SyncConfig
|
|
30
|
+
from liquid._defaults import InMemoryVault, CollectorSink
|
|
31
|
+
|
|
32
|
+
client = Liquid(llm=my_llm, vault=InMemoryVault(), sink=CollectorSink())
|
|
33
|
+
|
|
34
|
+
# 1. AI discovers the API (once)
|
|
35
|
+
schema = await client.discover("https://api.shopify.com")
|
|
36
|
+
|
|
37
|
+
# 2. AI proposes field mappings → human reviews
|
|
38
|
+
review = await client.propose_mappings(schema, {"amount": "float", "date": "datetime"})
|
|
39
|
+
review.approve_all()
|
|
40
|
+
|
|
41
|
+
# 3. Create adapter config
|
|
42
|
+
config = await client.create_adapter(
|
|
43
|
+
schema=schema,
|
|
44
|
+
auth_ref="vault/shopify",
|
|
45
|
+
mappings=review.finalize(),
|
|
46
|
+
sync_config=SyncConfig(endpoints=["/orders"], schedule="0 */6 * * *"),
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
# 4. Deterministic sync — no AI, runs forever
|
|
50
|
+
result = await client.sync(config)
|
|
51
|
+
print(f"Synced {result.records_delivered} records")
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## How Discovery Works
|
|
55
|
+
|
|
56
|
+
Liquid tries the cheapest method first, falls through on failure:
|
|
57
|
+
|
|
58
|
+
| Priority | Strategy | When it works | AI needed? |
|
|
59
|
+
|----------|----------|---------------|------------|
|
|
60
|
+
| 1 | **MCP** | Service publishes an MCP server | No |
|
|
61
|
+
| 2 | **OpenAPI** | Has `/openapi.json` or `/swagger.json` | No |
|
|
62
|
+
| 3 | **GraphQL** | Has `/graphql` with introspection | No |
|
|
63
|
+
| 4 | **REST Heuristic** | REST API without spec | Yes (once) |
|
|
64
|
+
| 5 | **Browser** | No API at all — capture network traffic | Yes (once) |
|
|
65
|
+
|
|
66
|
+
## Key Features
|
|
67
|
+
|
|
68
|
+
**Progressive Discovery** — MCP → OpenAPI → GraphQL → REST → Browser. Cheapest first.
|
|
69
|
+
|
|
70
|
+
**Selective Re-mapping** — When APIs change, `repair_adapter()` diffs schemas and re-maps only broken fields. Working mappings stay untouched.
|
|
71
|
+
|
|
72
|
+
**Safe Transforms** — Field transforms like `value * -1` or `value.lower()` are evaluated via AST whitelisting. No `eval()`, no injection risk.
|
|
73
|
+
|
|
74
|
+
**Pluggable Pagination** — Cursor, offset, page number, link header. Each is a strategy, not a switch/case.
|
|
75
|
+
|
|
76
|
+
**Learning System** — Corrections improve future proposals. Connect Shopify for the 51st time → mapping is instant.
|
|
77
|
+
|
|
78
|
+
**Auth Classification** — Detects OAuth (Tier A), app registration (Tier B), or manual credentials (Tier C). Returns structured escalation info.
|
|
79
|
+
|
|
80
|
+
## Installation
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
pip install liquid # core
|
|
84
|
+
pip install liquid[mcp] # + MCP server discovery
|
|
85
|
+
pip install liquid[browser] # + Playwright browser discovery
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## Architecture
|
|
89
|
+
|
|
90
|
+
```
|
|
91
|
+
┌─────────────┐ ┌──────────────┐ ┌────────────────┐ ┌─────────────┐
|
|
92
|
+
│ Discovery │──→│ Auth Setup │──→│ Field Mapping │──→│ Sync Engine │
|
|
93
|
+
│ (AI, once) │ │ (AI + human) │ │ (AI + human) │ │ (code, loop)│
|
|
94
|
+
└─────────────┘ └──────────────┘ └────────────────┘ └─────────────┘
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
**Liquid is a library, not a framework.** You control when to discover, how to present mappings, where to store configs, and what to do with synced data.
|
|
98
|
+
|
|
99
|
+
### Extension Points (Protocols)
|
|
100
|
+
|
|
101
|
+
| Protocol | Purpose | You provide |
|
|
102
|
+
|----------|---------|-------------|
|
|
103
|
+
| `Vault` | Credential storage | Postgres, AWS Secrets Manager, etc. |
|
|
104
|
+
| `LLMBackend` | AI provider | Claude, GPT, Llama, any LLM |
|
|
105
|
+
| `DataSink` | Where data goes | Database, queue, webhook, file |
|
|
106
|
+
| `KnowledgeStore` | Shared mappings | Redis, central registry, or disabled |
|
|
107
|
+
|
|
108
|
+
## Auto-Repair on API Changes
|
|
109
|
+
|
|
110
|
+
When an API breaks your adapter:
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
result = await client.repair_adapter(config, target_model, auto_approve=True)
|
|
114
|
+
# Re-discovers → diffs schemas → selectively re-maps broken fields
|
|
115
|
+
# Returns updated AdapterConfig or MappingReview for human review
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
## Liquid vs Alternatives
|
|
119
|
+
|
|
120
|
+
| | Liquid | Airbyte | Nango | Custom code |
|
|
121
|
+
|---|---|---|---|---|
|
|
122
|
+
| **New service** | `discover(url)` | Write connector YAML | Write TypeScript sync | Write adapter from scratch |
|
|
123
|
+
| **AI involvement** | Discovery only, then deterministic | None | AI-generated code | None |
|
|
124
|
+
| **Auth handling** | Classifies & escalates | Per-connector | Managed OAuth | Manual |
|
|
125
|
+
| **When API changes** | `repair_adapter()` | Update connector | Update sync code | Debug & fix |
|
|
126
|
+
| **Runtime LLM calls** | Zero | Zero | Zero | N/A |
|
|
127
|
+
| **Self-hosted** | Yes (library) | Yes (platform) | Yes (platform) | Yes |
|
|
128
|
+
| **License** | AGPL-3.0 | ELv2 | AGPL-3.0 | Yours |
|
|
129
|
+
|
|
130
|
+
## Documentation
|
|
131
|
+
|
|
132
|
+
- [Quick Start Guide](docs/QUICKSTART.md)
|
|
133
|
+
- [Architecture](docs/ARCHITECTURE.md)
|
|
134
|
+
- [Extending Liquid](docs/EXTENDING.md)
|
|
135
|
+
- [Contributing](CONTRIBUTING.md)
|
|
136
|
+
|
|
137
|
+
## Contributing
|
|
138
|
+
|
|
139
|
+
We welcome contributions! Check out our [contributing guide](CONTRIBUTING.md) and browse [good first issues](https://github.com/ertad-family/liquid/labels/good%20first%20issue).
|
|
140
|
+
|
|
141
|
+
## License
|
|
142
|
+
|
|
143
|
+
AGPL-3.0. Commercial licenses available — [contact us](mailto:hello@ertad.com).
|