liquid-api 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. {liquid_api-0.2.0 → liquid_api-0.3.0}/CHANGELOG.md +24 -0
  2. {liquid_api-0.2.0 → liquid_api-0.3.0}/PKG-INFO +1 -1
  3. {liquid_api-0.2.0 → liquid_api-0.3.0}/docs/EXTENDING.md +0 -1
  4. liquid_api-0.3.0/docs/pep541-request.md +50 -0
  5. {liquid_api-0.2.0 → liquid_api-0.3.0}/pyproject.toml +1 -1
  6. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/__init__.py +1 -3
  7. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/auth/manager.py +1 -2
  8. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/client.py +11 -21
  9. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/discovery/browser.py +5 -41
  10. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/discovery/graphql.py +8 -14
  11. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/discovery/mcp.py +5 -9
  12. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/discovery/openapi.py +4 -21
  13. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/discovery/rest_heuristic.py +20 -48
  14. liquid_api-0.3.0/src/liquid/discovery/utils.py +89 -0
  15. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/exceptions.py +0 -4
  16. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/sync/engine.py +63 -46
  17. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/sync/pagination.py +15 -12
  18. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_discovery/test_rest_heuristic.py +1 -1
  19. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_exceptions.py +0 -2
  20. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_smoke.py +1 -1
  21. {liquid_api-0.2.0 → liquid_api-0.3.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  22. {liquid_api-0.2.0 → liquid_api-0.3.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  23. {liquid_api-0.2.0 → liquid_api-0.3.0}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
  24. {liquid_api-0.2.0 → liquid_api-0.3.0}/.gitignore +0 -0
  25. {liquid_api-0.2.0 → liquid_api-0.3.0}/.pre-commit-config.yaml +0 -0
  26. {liquid_api-0.2.0 → liquid_api-0.3.0}/CLAUDE.md +0 -0
  27. {liquid_api-0.2.0 → liquid_api-0.3.0}/CODE_OF_CONDUCT.md +0 -0
  28. {liquid_api-0.2.0 → liquid_api-0.3.0}/CONTRIBUTING.md +0 -0
  29. {liquid_api-0.2.0 → liquid_api-0.3.0}/README.md +0 -0
  30. {liquid_api-0.2.0 → liquid_api-0.3.0}/SECURITY.md +0 -0
  31. {liquid_api-0.2.0 → liquid_api-0.3.0}/docs/ARCHITECTURE.md +0 -0
  32. {liquid_api-0.2.0 → liquid_api-0.3.0}/docs/QUICKSTART.md +0 -0
  33. {liquid_api-0.2.0 → liquid_api-0.3.0}/docs/blog/how-i-built-liquid.md +0 -0
  34. {liquid_api-0.2.0 → liquid_api-0.3.0}/docs/blog/launch-posts.md +0 -0
  35. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/_defaults.py +0 -0
  36. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/auth/__init__.py +0 -0
  37. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/auth/classifier.py +0 -0
  38. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/discovery/__init__.py +0 -0
  39. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/discovery/base.py +0 -0
  40. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/discovery/diff.py +0 -0
  41. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/events.py +0 -0
  42. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/mapping/__init__.py +0 -0
  43. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/mapping/learning.py +0 -0
  44. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/mapping/proposer.py +0 -0
  45. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/mapping/reviewer.py +0 -0
  46. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/models/__init__.py +0 -0
  47. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/models/adapter.py +0 -0
  48. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/models/llm.py +0 -0
  49. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/models/schema.py +0 -0
  50. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/models/sync.py +0 -0
  51. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/protocols.py +0 -0
  52. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/py.typed +0 -0
  53. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/sync/__init__.py +0 -0
  54. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/sync/auto_repair.py +0 -0
  55. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/sync/fetcher.py +0 -0
  56. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/sync/mapper.py +0 -0
  57. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/sync/retry.py +0 -0
  58. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/sync/selector.py +0 -0
  59. {liquid_api-0.2.0 → liquid_api-0.3.0}/src/liquid/sync/transform.py +0 -0
  60. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/__init__.py +0 -0
  61. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/fixtures/graphql_introspection.json +0 -0
  62. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/fixtures/petstore_openapi.json +0 -0
  63. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_auth/__init__.py +0 -0
  64. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_auth/test_classifier.py +0 -0
  65. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_auth/test_manager.py +0 -0
  66. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_client.py +0 -0
  67. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_defaults.py +0 -0
  68. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_discovery/__init__.py +0 -0
  69. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_discovery/test_browser.py +0 -0
  70. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_discovery/test_diff.py +0 -0
  71. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_discovery/test_graphql.py +0 -0
  72. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_discovery/test_mcp.py +0 -0
  73. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_discovery/test_openapi.py +0 -0
  74. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_discovery/test_pipeline.py +0 -0
  75. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_events.py +0 -0
  76. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_mapping/__init__.py +0 -0
  77. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_mapping/test_learning.py +0 -0
  78. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_mapping/test_proposer.py +0 -0
  79. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_mapping/test_reviewer.py +0 -0
  80. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_models/__init__.py +0 -0
  81. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_models/test_adapter.py +0 -0
  82. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_models/test_llm.py +0 -0
  83. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_models/test_schema.py +0 -0
  84. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_models/test_sync.py +0 -0
  85. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_protocols.py +0 -0
  86. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_sync/__init__.py +0 -0
  87. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_sync/test_auto_repair.py +0 -0
  88. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_sync/test_engine.py +0 -0
  89. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_sync/test_fetcher.py +0 -0
  90. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_sync/test_mapper.py +0 -0
  91. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_sync/test_pagination.py +0 -0
  92. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_sync/test_retry.py +0 -0
  93. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_sync/test_selector.py +0 -0
  94. {liquid_api-0.2.0 → liquid_api-0.3.0}/tests/test_sync/test_transform.py +0 -0
@@ -2,6 +2,30 @@
2
2
 
3
3
  All notable changes to Liquid will be documented in this file.
4
4
 
5
+ ## [0.3.0] - 2026-04-13
6
+
7
+ ### Added
8
+ - Published to PyPI as `liquid-api` (`pip install liquid-api`)
9
+ - PyPI metadata: keywords, classifiers, project URLs
10
+ - `managed_http_client()` shared context manager for discovery strategies
11
+ - `_EndpointSyncResult` TypedDict for type-safe sync results
12
+ - OSS launch materials: README revamp, CONTRIBUTING.md, CODE_OF_CONDUCT.md, SECURITY.md, CHANGELOG.md
13
+ - Issue templates (bug report, feature request), PR template
14
+ - 10 good first issues for new contributors (#3-#12)
15
+ - Blog post draft and launch post templates (HN, Reddit, Twitter)
16
+ - PEP 541 request for `liquid` package name
17
+
18
+ ### Changed
19
+ - Extracted `discovery/utils.py` with shared `infer_service_name()`, `parse_llm_endpoints_response()`, `managed_http_client()`
20
+ - Refactored `SyncEngine.run()` — extracted `_sync_endpoint()`, reduced nesting 4→2 levels
21
+ - Refactored `Liquid.repair_adapter()` — extracted `_emit_repair_event()`
22
+ - Extracted `_has_full_page()` helper in pagination, eliminating duplication
23
+ - Moved inline imports (`json`, `base64`) to top-level
24
+ - Standardized HTTP client management across all discovery strategies
25
+
26
+ ### Removed
27
+ - `ReDiscoveryNeededError` exception (dead code, use `ReDiscoveryNeeded` event instead)
28
+
5
29
  ## [0.2.0] - 2026-04-13
6
30
 
7
31
  ### Added
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: liquid-api
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Summary: AI discovers APIs. Code syncs data. No adapters to write.
5
5
  Project-URL: Homepage, https://github.com/ertad-family/liquid
6
6
  Project-URL: Documentation, https://github.com/ertad-family/liquid/blob/main/docs/QUICKSTART.md
@@ -153,7 +153,6 @@ LiquidError
153
153
  │ ├── RateLimitError — 429 response (has retry_after)
154
154
  │ ├── ServiceDownError — 5xx response
155
155
  │ └── EndpointGoneError — 404/410 response
156
- ├── ReDiscoveryNeededError — persistent failures threshold exceeded
157
156
  └── VaultError — credential storage failures
158
157
  ```
159
158
 
@@ -0,0 +1,50 @@
1
+ # PEP 541 — Request to Transfer PyPI Package Name `liquid`
2
+
3
+ ## How to Submit
4
+
5
+ File an issue at: https://github.com/pypi/support/issues/new?template=pep541-702.yml
6
+
7
+ ## Request Details
8
+
9
+ **Package name:** `liquid`
10
+
11
+ **PyPI URL:** https://pypi.org/project/liquid/
12
+
13
+ **Your PyPI username:** *(your username)*
14
+
15
+ **Your package URL:** https://pypi.org/project/liquid-api/
16
+
17
+ **Your GitHub repo:** https://github.com/ertad-family/liquid
18
+
19
+ ### Justification
20
+
21
+ The `liquid` package on PyPI meets PEP 541 criteria for an abandoned project:
22
+
23
+ 1. **No meaningful releases**: Only version 0.0.1, published September 26, 2013 (over 12 years ago)
24
+ 2. **No project description**: "The author of this package has not provided a project description"
25
+ 3. **Planning status**: Classified as "Development Status :: 1 - Planning" — never progressed beyond planning
26
+ 4. **No activity**: The associated GitHub repository (https://github.com/refnode/liquid) has had no commits or activity in years
27
+ 5. **No dependencies or functionality**: The package contains no meaningful code
28
+
29
+ ### Our Project
30
+
31
+ **liquid-api** (https://pypi.org/project/liquid-api/) is an actively developed Python library for programmatic API discovery and adapter generation. We are currently publishing under `liquid-api` but our import name is `liquid`, which creates a confusing mismatch:
32
+
33
+ ```bash
34
+ pip install liquid-api # install command
35
+ from liquid import Liquid # import name
36
+ ```
37
+
38
+ Transferring the `liquid` name would:
39
+ - Eliminate the install-vs-import name mismatch
40
+ - Give the name to an actively maintained project with 210 tests, full documentation, and regular releases
41
+ - Serve the Python community better than an abandoned placeholder
42
+
43
+ ### Contact Attempts
44
+
45
+ *(Before submitting, attempt to contact the current owner `refnode` via GitHub or email. Document your attempt here, even if there's no response after 2-4 weeks.)*
46
+
47
+ **Steps to take:**
48
+ 1. Open an issue at https://github.com/refnode/liquid asking to transfer the PyPI name
49
+ 2. Wait 4 weeks for a response
50
+ 3. If no response, submit the PEP 541 request with proof of the contact attempt
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "liquid-api"
3
- version = "0.2.0"
3
+ version = "0.3.0"
4
4
  description = "AI discovers APIs. Code syncs data. No adapters to write."
5
5
  readme = "README.md"
6
6
  license = "AGPL-3.0-only"
@@ -1,6 +1,6 @@
1
1
  """Liquid — AI discovers APIs. Code syncs data. No adapters to write."""
2
2
 
3
- __version__ = "0.2.0"
3
+ __version__ = "0.3.0"
4
4
 
5
5
  from liquid.client import Liquid
6
6
  from liquid.exceptions import (
@@ -12,7 +12,6 @@ from liquid.exceptions import (
12
12
  LiquidError,
13
13
  MappingError,
14
14
  RateLimitError,
15
- ReDiscoveryNeededError,
16
15
  ServiceDownError,
17
16
  SyncRuntimeError,
18
17
  VaultError,
@@ -50,7 +49,6 @@ __all__ = [
50
49
  "MappedRecord",
51
50
  "MappingError",
52
51
  "RateLimitError",
53
- "ReDiscoveryNeededError",
54
52
  "ServiceDownError",
55
53
  "SyncConfig",
56
54
  "SyncResult",
@@ -1,5 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import base64
3
4
  import logging
4
5
  from typing import TYPE_CHECKING, Any
5
6
 
@@ -43,8 +44,6 @@ class AuthManager:
43
44
  case "basic":
44
45
  username = await self.vault.get(f"{vault_key}/username")
45
46
  password = await self.vault.get(f"{vault_key}/password")
46
- import base64
47
-
48
47
  encoded = base64.b64encode(f"{username}:{password}".encode()).decode()
49
48
  return {"Authorization": f"Basic {encoded}"}
50
49
  case _:
@@ -2,6 +2,7 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
+ import json
5
6
  from typing import TYPE_CHECKING, Any
6
7
 
7
8
  import httpx
@@ -26,6 +27,7 @@ from liquid.sync.mapper import RecordMapper
26
27
 
27
28
  if TYPE_CHECKING:
28
29
  from liquid.events import EventHandler
30
+ from liquid.models.schema import SchemaDiff
29
31
  from liquid.models.sync import SyncResult
30
32
  from liquid.protocols import DataSink, KnowledgeStore, LLMBackend, Vault
31
33
  from liquid.sync.retry import RetryPolicy
@@ -145,21 +147,13 @@ class Liquid:
145
147
  Returns AdapterConfig if auto_approve=True and all mappings are confident,
146
148
  otherwise returns MappingReview for human review.
147
149
  """
148
- from liquid.events import AdapterRepaired
149
150
 
150
151
  new_schema = await self.discover(config.schema_.source_url)
151
152
  diff = diff_schemas(config.schema_, new_schema)
152
153
 
153
154
  if not diff.has_breaking_changes:
154
155
  updated = config.model_copy(update={"schema_": new_schema, "version": config.version + 1})
155
- if self.event_handler:
156
- await self.event_handler.handle(
157
- AdapterRepaired(
158
- adapter_id=config.config_id,
159
- diff=diff,
160
- auto_approved=True,
161
- )
162
- )
156
+ await self._emit_repair_event(config.config_id, diff)
163
157
  return updated
164
158
 
165
159
  proposals = await self._mapping_proposer.propose(
@@ -173,28 +167,26 @@ class Liquid:
173
167
 
174
168
  if auto_approve and all(m.confidence >= confidence_threshold for m in proposals):
175
169
  review.approve_all()
176
- mappings = review.finalize()
177
170
  updated = AdapterConfig(
178
171
  config_id=config.config_id,
179
172
  schema=new_schema,
180
173
  auth_ref=config.auth_ref,
181
- mappings=mappings,
174
+ mappings=review.finalize(),
182
175
  sync=config.sync,
183
176
  verified_by=config.verified_by,
184
177
  version=config.version + 1,
185
178
  )
186
- if self.event_handler:
187
- await self.event_handler.handle(
188
- AdapterRepaired(
189
- adapter_id=config.config_id,
190
- diff=diff,
191
- auto_approved=True,
192
- )
193
- )
179
+ await self._emit_repair_event(config.config_id, diff)
194
180
  return updated
195
181
 
196
182
  return review
197
183
 
184
+ async def _emit_repair_event(self, adapter_id: str, diff: SchemaDiff) -> None:
185
+ if self.event_handler:
186
+ from liquid.events import AdapterRepaired
187
+
188
+ await self.event_handler.handle(AdapterRepaired(adapter_id=adapter_id, diff=diff, auto_approved=True))
189
+
198
190
  async def learn_from_review(
199
191
  self,
200
192
  schema: APISchema,
@@ -202,8 +194,6 @@ class Liquid:
202
194
  review: MappingReview,
203
195
  ) -> None:
204
196
  """Record corrections from a mapping review for future learning."""
205
- import json
206
-
207
197
  corrections = review.corrections()
208
198
  if corrections:
209
199
  await self._mapping_learner.record_corrections(
@@ -13,7 +13,7 @@ import logging
13
13
  from typing import TYPE_CHECKING, Any
14
14
 
15
15
  from liquid.exceptions import DiscoveryError
16
- from liquid.models.schema import APISchema, AuthRequirement, Endpoint
16
+ from liquid.models.schema import APISchema
17
17
 
18
18
  if TYPE_CHECKING:
19
19
  from liquid.protocols import LLMBackend
@@ -126,50 +126,14 @@ class BrowserDiscovery:
126
126
  return self._parse_response(response.content or "{}", url, captured)
127
127
 
128
128
  def _parse_response(self, content: str, url: str, captured: list[dict[str, Any]]) -> APISchema:
129
- import json
130
- from urllib.parse import urlparse
129
+ from liquid.discovery.utils import parse_llm_endpoints_response
131
130
 
132
- try:
133
- data = json.loads(content)
134
- except json.JSONDecodeError:
135
- data = {}
136
-
137
- endpoints: list[Endpoint] = []
138
- for ep in data.get("endpoints", []):
139
- if isinstance(ep, dict) and "path" in ep:
140
- endpoints.append(
141
- Endpoint(
142
- path=ep["path"],
143
- method=ep.get("method", "GET").upper(),
144
- description=ep.get("description", ""),
145
- )
146
- )
147
-
148
- if not endpoints:
149
- # Fallback: create endpoints from captured URLs
150
- seen_paths: set[str] = set()
151
- for c in captured:
152
- parsed = urlparse(c["url"])
153
- if parsed.path not in seen_paths:
154
- seen_paths.add(parsed.path)
155
- endpoints.append(
156
- Endpoint(
157
- path=parsed.path,
158
- method=c["method"],
159
- description=f"Captured via browser ({c['status']})",
160
- )
161
- )
162
-
163
- auth_type = data.get("auth_type", "custom")
164
- valid_types = {"oauth2", "api_key", "bearer", "basic", "custom"}
165
- if auth_type not in valid_types:
166
- auth_type = "custom"
167
- tier = "A" if auth_type in ("oauth2", "bearer") else "C"
131
+ service_name, endpoints, auth = parse_llm_endpoints_response(content, url, fallback_probes=captured)
168
132
 
169
133
  return APISchema(
170
134
  source_url=url,
171
- service_name=data.get("service_name", urlparse(url).hostname or "Unknown"),
135
+ service_name=service_name,
172
136
  discovery_method="browser",
173
137
  endpoints=endpoints,
174
- auth=AuthRequirement(type=auth_type, tier=tier),
138
+ auth=auth,
175
139
  )
@@ -3,7 +3,7 @@ from __future__ import annotations
3
3
  import logging
4
4
  from typing import Any
5
5
 
6
- import httpx
6
+ import httpx # noqa: TC002
7
7
 
8
8
  from liquid.exceptions import DiscoveryError
9
9
  from liquid.models.schema import (
@@ -51,15 +51,13 @@ class GraphQLDiscovery:
51
51
  self._external_client = http_client
52
52
 
53
53
  async def discover(self, url: str) -> APISchema | None:
54
- client = self._external_client or httpx.AsyncClient()
55
- try:
54
+ from liquid.discovery.utils import managed_http_client
55
+
56
+ async with managed_http_client(self._external_client) as client:
56
57
  introspection = await self._run_introspection(client, url)
57
58
  if introspection is None:
58
59
  return None
59
60
  return self._parse_introspection(introspection, url)
60
- finally:
61
- if not self._external_client:
62
- await client.aclose()
63
61
 
64
62
  async def _run_introspection(
65
63
  self,
@@ -156,15 +154,11 @@ class GraphQLDiscovery:
156
154
  return {"type": "string", "title": name}
157
155
  return {"type": "object"}
158
156
 
159
- def _infer_service_name(self, url: str) -> str:
160
- from urllib.parse import urlparse
157
+ @staticmethod
158
+ def _infer_service_name(url: str) -> str:
159
+ from liquid.discovery.utils import infer_service_name
161
160
 
162
- parsed = urlparse(url)
163
- host = parsed.hostname or "unknown"
164
- parts = host.split(".")
165
- if len(parts) >= 2:
166
- return parts[-2].capitalize()
167
- return host.capitalize()
161
+ return infer_service_name(url)
168
162
 
169
163
 
170
164
  def _scalar_to_json_type(name: str) -> str:
@@ -148,12 +148,8 @@ class MCPDiscovery:
148
148
  )
149
149
  return params
150
150
 
151
- def _infer_service_name(self, url: str) -> str:
152
- from urllib.parse import urlparse
153
-
154
- parsed = urlparse(url)
155
- host = parsed.hostname or "unknown"
156
- parts = host.split(".")
157
- if len(parts) >= 2:
158
- return parts[-2].capitalize()
159
- return host.capitalize()
151
+ @staticmethod
152
+ def _infer_service_name(url: str) -> str:
153
+ from liquid.discovery.utils import infer_service_name
154
+
155
+ return infer_service_name(url)
@@ -3,7 +3,7 @@ from __future__ import annotations
3
3
  import logging
4
4
  from typing import Any
5
5
 
6
- import httpx
6
+ import httpx # noqa: TC002
7
7
  import yaml
8
8
 
9
9
  from liquid.exceptions import DiscoveryError
@@ -39,7 +39,9 @@ class OpenAPIDiscovery:
39
39
  self._external_client = http_client
40
40
 
41
41
  async def discover(self, url: str) -> APISchema | None:
42
- async with self._get_client() as client:
42
+ from liquid.discovery.utils import managed_http_client
43
+
44
+ async with managed_http_client(self._external_client) as client:
43
45
  spec = await self._find_spec(client, url)
44
46
  if spec is None:
45
47
  return None
@@ -206,22 +208,3 @@ class OpenAPIDiscovery:
206
208
  if "page" in param_names or "page_number" in param_names:
207
209
  return PaginationType.PAGE_NUMBER
208
210
  return None
209
-
210
- def _get_client(self) -> httpx.AsyncClient:
211
- if self._external_client:
212
-
213
- class _NoOpContext:
214
- async def __aenter__(self):
215
- return self
216
-
217
- async def __aexit__(self, *args):
218
- pass
219
-
220
- def __getattr__(self, name):
221
- return getattr(self._client, name)
222
-
223
- def __init__(self, client):
224
- self._client = client
225
-
226
- return _NoOpContext(self._external_client) # type: ignore[return-value]
227
- return httpx.AsyncClient()
@@ -3,10 +3,10 @@ from __future__ import annotations
3
3
  import logging
4
4
  from typing import TYPE_CHECKING
5
5
 
6
- import httpx
6
+ import httpx # noqa: TC002
7
7
 
8
8
  from liquid.exceptions import DiscoveryError
9
- from liquid.models.schema import APISchema, AuthRequirement, Endpoint
9
+ from liquid.models.schema import APISchema
10
10
 
11
11
  if TYPE_CHECKING:
12
12
  from liquid.protocols import LLMBackend
@@ -47,20 +47,19 @@ class RESTHeuristicDiscovery:
47
47
  self._external_client = http_client
48
48
 
49
49
  async def discover(self, url: str) -> APISchema | None:
50
- client = self._external_client or httpx.AsyncClient()
51
- try:
52
- found_endpoints = await self._probe_endpoints(client, url)
53
- if not found_endpoints:
54
- return None
55
-
56
- return await self._interpret_with_llm(url, found_endpoints)
57
- except DiscoveryError:
58
- raise
59
- except Exception as e:
60
- raise DiscoveryError(f"REST heuristic discovery failed: {e}") from e
61
- finally:
62
- if not self._external_client:
63
- await client.aclose()
50
+ from liquid.discovery.utils import managed_http_client
51
+
52
+ async with managed_http_client(self._external_client) as client:
53
+ try:
54
+ found_endpoints = await self._probe_endpoints(client, url)
55
+ if not found_endpoints:
56
+ return None
57
+
58
+ return await self._interpret_with_llm(url, found_endpoints)
59
+ except DiscoveryError:
60
+ raise
61
+ except Exception as e:
62
+ raise DiscoveryError(f"REST heuristic discovery failed: {e}") from e
64
63
 
65
64
  async def _probe_endpoints(
66
65
  self,
@@ -117,41 +116,14 @@ class RESTHeuristicDiscovery:
117
116
  return self._parse_llm_response(response.content or "{}", url, probed)
118
117
 
119
118
  def _parse_llm_response(self, content: str, url: str, probed: list[dict]) -> APISchema:
120
- import json
121
-
122
- try:
123
- data = json.loads(content)
124
- except json.JSONDecodeError:
125
- data = {}
126
-
127
- endpoints = []
128
- for ep in data.get("endpoints", []):
129
- if isinstance(ep, dict) and "path" in ep:
130
- endpoints.append(
131
- Endpoint(
132
- path=ep["path"],
133
- method=ep.get("method", "GET").upper(),
134
- description=ep.get("description", ""),
135
- )
136
- )
137
-
138
- if not endpoints:
139
- endpoints = [
140
- Endpoint(path=p["path"], method="GET", description=f"Discovered via probe ({p['status']})")
141
- for p in probed
142
- ]
143
-
144
- auth_type = data.get("auth_type", "custom")
145
- valid_auth_types = {"oauth2", "api_key", "bearer", "basic", "custom"}
146
- if auth_type not in valid_auth_types:
147
- auth_type = "custom"
148
-
149
- tier = "A" if auth_type in ("oauth2", "bearer") else "C"
119
+ from liquid.discovery.utils import parse_llm_endpoints_response
120
+
121
+ service_name, endpoints, auth = parse_llm_endpoints_response(content, url, fallback_probes=probed)
150
122
 
151
123
  return APISchema(
152
124
  source_url=url,
153
- service_name=data.get("service_name", "Unknown"),
125
+ service_name=service_name,
154
126
  discovery_method="rest_heuristic",
155
127
  endpoints=endpoints,
156
- auth=AuthRequirement(type=auth_type, tier=tier),
128
+ auth=auth,
157
129
  )
@@ -0,0 +1,89 @@
1
+ """Shared utilities for discovery strategies."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import logging
7
+ from collections.abc import AsyncIterator # noqa: TC003
8
+ from contextlib import asynccontextmanager
9
+ from typing import Any
10
+ from urllib.parse import urlparse
11
+
12
+ import httpx
13
+
14
+ from liquid.models.schema import AuthRequirement, Endpoint
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+
19
+ def infer_service_name(url: str) -> str:
20
+ """Extract a human-readable service name from a URL."""
21
+ parsed = urlparse(url)
22
+ host = parsed.hostname or "unknown"
23
+ parts = host.split(".")
24
+ if len(parts) >= 2:
25
+ return parts[-2].capitalize()
26
+ return host.capitalize()
27
+
28
+
29
+ def parse_llm_endpoints_response(
30
+ content: str,
31
+ url: str,
32
+ fallback_probes: list[dict[str, Any]] | None = None,
33
+ ) -> tuple[str, list[Endpoint], AuthRequirement]:
34
+ """Parse LLM JSON response containing service_name, endpoints, and auth_type.
35
+
36
+ Returns (service_name, endpoints, auth_requirement).
37
+ Falls back to probed endpoints if LLM response is invalid.
38
+ """
39
+ try:
40
+ data = json.loads(content)
41
+ except json.JSONDecodeError:
42
+ data = {}
43
+
44
+ endpoints: list[Endpoint] = []
45
+ for ep in data.get("endpoints", []):
46
+ if isinstance(ep, dict) and "path" in ep:
47
+ endpoints.append(
48
+ Endpoint(
49
+ path=ep["path"],
50
+ method=ep.get("method", "GET").upper(),
51
+ description=ep.get("description", ""),
52
+ )
53
+ )
54
+
55
+ if not endpoints and fallback_probes:
56
+ for probe in fallback_probes:
57
+ path = probe.get("path") or probe.get("url", "")
58
+ if "://" in path:
59
+ path = urlparse(path).path
60
+ endpoints.append(
61
+ Endpoint(
62
+ path=path,
63
+ method=probe.get("method", "GET"),
64
+ description=f"Discovered via probe ({probe.get('status', '?')})",
65
+ )
66
+ )
67
+
68
+ auth_type = data.get("auth_type", "custom")
69
+ valid_auth_types = {"oauth2", "api_key", "bearer", "basic", "custom"}
70
+ if auth_type not in valid_auth_types:
71
+ auth_type = "custom"
72
+ tier = "A" if auth_type in ("oauth2", "bearer") else "C"
73
+
74
+ service_name = data.get("service_name") or infer_service_name(url)
75
+
76
+ return service_name, endpoints, AuthRequirement(type=auth_type, tier=tier)
77
+
78
+
79
+ @asynccontextmanager
80
+ async def managed_http_client(external: httpx.AsyncClient | None = None) -> AsyncIterator[httpx.AsyncClient]:
81
+ """Yield the external client if provided, otherwise create and auto-close a new one."""
82
+ if external:
83
+ yield external
84
+ else:
85
+ client = httpx.AsyncClient()
86
+ try:
87
+ yield client
88
+ finally:
89
+ await client.aclose()
@@ -43,9 +43,5 @@ class EndpointGoneError(SyncRuntimeError):
43
43
  pass
44
44
 
45
45
 
46
- class ReDiscoveryNeededError(LiquidError):
47
- pass
48
-
49
-
50
46
  class VaultError(LiquidError):
51
47
  pass
@@ -2,7 +2,7 @@ from __future__ import annotations
2
2
 
3
3
  import logging
4
4
  from datetime import UTC, datetime
5
- from typing import TYPE_CHECKING
5
+ from typing import TYPE_CHECKING, TypedDict
6
6
 
7
7
  from liquid.events import ReDiscoveryNeeded, SyncCompleted, SyncFailed
8
8
  from liquid.exceptions import SyncRuntimeError
@@ -21,6 +21,14 @@ if TYPE_CHECKING:
21
21
  logger = logging.getLogger(__name__)
22
22
 
23
23
 
24
+ class _EndpointSyncResult(TypedDict):
25
+ fetched: int
26
+ mapped: int
27
+ delivered: int
28
+ errors: list[SyncError]
29
+ cursor: str | None
30
+
31
+
24
32
  class SyncEngine:
25
33
  def __init__(
26
34
  self,
@@ -59,51 +67,12 @@ class SyncEngine:
59
67
  )
60
68
  continue
61
69
 
62
- try:
63
- page_cursor = current_cursor
64
- while True:
65
- fetch_result = await with_retry(
66
- lambda ep=endpoint, c=page_cursor: self.fetcher.fetch(
67
- endpoint=ep,
68
- base_url=config.schema_.source_url,
69
- auth_ref=config.auth_ref,
70
- cursor=c,
71
- ),
72
- self.retry_policy,
73
- )
74
-
75
- total_fetched += len(fetch_result.records)
76
-
77
- mapped_records = self.mapper.map_batch(fetch_result.records, ep_path)
78
- total_mapped += len(mapped_records)
79
-
80
- delivery = await self.sink.deliver(mapped_records)
81
- total_delivered += delivery.delivered
82
-
83
- if delivery.errors:
84
- for err_msg in delivery.errors:
85
- errors.append(
86
- SyncError(
87
- type=SyncErrorType.DELIVERY_ERROR,
88
- message=err_msg,
89
- endpoint=ep_path,
90
- )
91
- )
92
-
93
- page_cursor = fetch_result.next_cursor
94
- current_cursor = page_cursor
95
- if page_cursor is None:
96
- break
97
-
98
- except SyncRuntimeError as e:
99
- error_type = _classify_error(e)
100
- errors.append(
101
- SyncError(
102
- type=error_type,
103
- message=str(e),
104
- endpoint=ep_path,
105
- )
106
- )
70
+ ep_result = await self._sync_endpoint(endpoint, ep_path, config, current_cursor)
71
+ total_fetched += ep_result["fetched"]
72
+ total_mapped += ep_result["mapped"]
73
+ total_delivered += ep_result["delivered"]
74
+ errors.extend(ep_result["errors"])
75
+ current_cursor = ep_result["cursor"]
107
76
 
108
77
  result = SyncResult(
109
78
  adapter_id=config.config_id,
@@ -119,6 +88,54 @@ class SyncEngine:
119
88
  await self._handle_result(result, config)
120
89
  return result
121
90
 
91
+ async def _sync_endpoint(
92
+ self,
93
+ endpoint: Endpoint,
94
+ ep_path: str,
95
+ config: AdapterConfig,
96
+ cursor: str | None,
97
+ ) -> _EndpointSyncResult:
98
+ """Sync a single endpoint with pagination."""
99
+ fetched = 0
100
+ mapped = 0
101
+ delivered = 0
102
+ errors: list[SyncError] = []
103
+ page_cursor = cursor
104
+
105
+ try:
106
+ while True:
107
+ fetch_result = await with_retry(
108
+ lambda ep=endpoint, c=page_cursor: self.fetcher.fetch(
109
+ endpoint=ep,
110
+ base_url=config.schema_.source_url,
111
+ auth_ref=config.auth_ref,
112
+ cursor=c,
113
+ ),
114
+ self.retry_policy,
115
+ )
116
+
117
+ fetched += len(fetch_result.records)
118
+ mapped_records = self.mapper.map_batch(fetch_result.records, ep_path)
119
+ mapped += len(mapped_records)
120
+
121
+ delivery = await self.sink.deliver(mapped_records)
122
+ delivered += delivery.delivered
123
+
124
+ if delivery.errors:
125
+ errors.extend(
126
+ SyncError(type=SyncErrorType.DELIVERY_ERROR, message=msg, endpoint=ep_path)
127
+ for msg in delivery.errors
128
+ )
129
+
130
+ page_cursor = fetch_result.next_cursor
131
+ if page_cursor is None:
132
+ break
133
+
134
+ except SyncRuntimeError as e:
135
+ errors.append(SyncError(type=_classify_error(e), message=str(e), endpoint=ep_path))
136
+
137
+ return {"fetched": fetched, "mapped": mapped, "delivered": delivered, "errors": errors, "cursor": page_cursor}
138
+
122
139
  async def _handle_result(self, result: SyncResult, config: AdapterConfig) -> None:
123
140
  if not self.event_handler:
124
141
  return
@@ -52,12 +52,10 @@ class OffsetPagination:
52
52
  return {self.offset_param: offset, self.limit_param: self.limit}
53
53
 
54
54
  def extract_next_cursor(self, response: httpx.Response) -> str | None:
55
- data = response.json()
56
- records = data if isinstance(data, list) else data.get("data", data.get("results", []))
57
- if isinstance(records, list) and len(records) >= self.limit:
58
- current_offset = int(response.request.url.params.get(self.offset_param, "0"))
59
- return str(current_offset + self.limit)
60
- return None
55
+ if not _has_full_page(response, self.limit):
56
+ return None
57
+ current_offset = int(response.request.url.params.get(self.offset_param, "0"))
58
+ return str(current_offset + self.limit)
61
59
 
62
60
 
63
61
  class PageNumberPagination:
@@ -71,12 +69,10 @@ class PageNumberPagination:
71
69
  return {self.page_param: page, self.per_page_param: self.per_page}
72
70
 
73
71
  def extract_next_cursor(self, response: httpx.Response) -> str | None:
74
- data = response.json()
75
- records = data if isinstance(data, list) else data.get("data", data.get("results", []))
76
- if isinstance(records, list) and len(records) >= self.per_page:
77
- current_page = int(response.request.url.params.get(self.page_param, "1"))
78
- return str(current_page + 1)
79
- return None
72
+ if not _has_full_page(response, self.per_page):
73
+ return None
74
+ current_page = int(response.request.url.params.get(self.page_param, "1"))
75
+ return str(current_page + 1)
80
76
 
81
77
 
82
78
  class LinkHeaderPagination:
@@ -91,6 +87,13 @@ class LinkHeaderPagination:
91
87
  return match.group(1) if match else None
92
88
 
93
89
 
90
+ def _has_full_page(response: httpx.Response, page_size: int) -> bool:
91
+ """Check if the response contains a full page of records (more pages likely)."""
92
+ data = response.json()
93
+ records = data if isinstance(data, list) else data.get("data", data.get("results", []))
94
+ return isinstance(records, list) and len(records) >= page_size
95
+
96
+
94
97
  def _extract_nested(data: dict[str, Any], path: str) -> Any:
95
98
  parts = path.split(".")
96
99
  current: Any = data
@@ -75,4 +75,4 @@ class TestRESTHeuristicDiscovery:
75
75
 
76
76
  assert result is not None
77
77
  assert len(result.endpoints) > 0
78
- assert result.service_name == "Unknown"
78
+ assert result.service_name == "Test" # inferred from api.test.com
@@ -9,7 +9,6 @@ from liquid.exceptions import (
9
9
  LiquidError,
10
10
  MappingError,
11
11
  RateLimitError,
12
- ReDiscoveryNeededError,
13
12
  ServiceDownError,
14
13
  SyncRuntimeError,
15
14
  VaultError,
@@ -22,7 +21,6 @@ def test_hierarchy_liquid_error_catches_all():
22
21
  AuthSetupError,
23
22
  MappingError,
24
23
  SyncRuntimeError,
25
- ReDiscoveryNeededError,
26
24
  VaultError,
27
25
  ]
28
26
  for exc_class in all_errors:
@@ -2,4 +2,4 @@ from liquid import __version__
2
2
 
3
3
 
4
4
  def test_version():
5
- assert __version__ == "0.2.0"
5
+ assert __version__ == "0.3.0"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes