fetchgraph 0.0.1__tar.gz → 0.0.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fetchgraph-0.0.3/PKG-INFO +230 -0
- fetchgraph-0.0.3/README.md +214 -0
- {fetchgraph-0.0.1 → fetchgraph-0.0.3}/pyproject.toml +14 -2
- fetchgraph-0.0.3/src/fetchgraph/__init__.py +122 -0
- {fetchgraph-0.0.1 → fetchgraph-0.0.3}/src/fetchgraph/core.py +26 -96
- fetchgraph-0.0.3/src/fetchgraph/json_types.py +12 -0
- fetchgraph-0.0.3/src/fetchgraph/models.py +93 -0
- fetchgraph-0.0.3/src/fetchgraph/parsing/__init__.py +11 -0
- fetchgraph-0.0.3/src/fetchgraph/parsing/exceptions.py +5 -0
- fetchgraph-0.0.3/src/fetchgraph/parsing/extract_json.py +1209 -0
- fetchgraph-0.0.3/src/fetchgraph/parsing/json_parser.py +63 -0
- fetchgraph-0.0.3/src/fetchgraph/parsing/plan_parser.py +88 -0
- fetchgraph-0.0.3/src/fetchgraph/prompts/plan_generic.md +140 -0
- fetchgraph-0.0.3/src/fetchgraph/protocols.py +75 -0
- fetchgraph-0.0.3/src/fetchgraph/relational_base.py +140 -0
- fetchgraph-0.0.3/src/fetchgraph/relational_composite.py +101 -0
- fetchgraph-0.0.3/src/fetchgraph/relational_models.py +226 -0
- fetchgraph-0.0.3/src/fetchgraph/relational_pandas.py +331 -0
- fetchgraph-0.0.3/src/fetchgraph/relational_provider.py +39 -0
- fetchgraph-0.0.3/src/fetchgraph/relational_sql.py +468 -0
- fetchgraph-0.0.3/src/fetchgraph/semantic_backend.py +397 -0
- fetchgraph-0.0.3/src/fetchgraph.egg-info/PKG-INFO +230 -0
- fetchgraph-0.0.3/src/fetchgraph.egg-info/SOURCES.txt +33 -0
- fetchgraph-0.0.3/src/fetchgraph.egg-info/requires.txt +5 -0
- fetchgraph-0.0.3/tests/test_relational_composite.py +86 -0
- fetchgraph-0.0.3/tests/test_relational_pandas.py +172 -0
- fetchgraph-0.0.3/tests/test_relational_sql.py +279 -0
- fetchgraph-0.0.3/tests/test_semantic_backend_csv.py +141 -0
- fetchgraph-0.0.3/tests/test_semantic_backend_pgvector.py +99 -0
- fetchgraph-0.0.1/PKG-INFO +0 -77
- fetchgraph-0.0.1/README.md +0 -64
- fetchgraph-0.0.1/src/fetchgraph/__init__.py +0 -43
- fetchgraph-0.0.1/src/fetchgraph/prompts/plan_generic.md +0 -40
- fetchgraph-0.0.1/src/fetchgraph.egg-info/PKG-INFO +0 -77
- fetchgraph-0.0.1/src/fetchgraph.egg-info/SOURCES.txt +0 -13
- fetchgraph-0.0.1/src/fetchgraph.egg-info/requires.txt +0 -1
- {fetchgraph-0.0.1 → fetchgraph-0.0.3}/LICENSE +0 -0
- {fetchgraph-0.0.1 → fetchgraph-0.0.3}/setup.cfg +0 -0
- {fetchgraph-0.0.1 → fetchgraph-0.0.3}/src/fetchgraph/prompts/synth_generic.md +0 -0
- {fetchgraph-0.0.1 → fetchgraph-0.0.3}/src/fetchgraph/utils.py +0 -0
- {fetchgraph-0.0.1 → fetchgraph-0.0.3}/src/fetchgraph.egg-info/dependency_links.txt +0 -0
- {fetchgraph-0.0.1 → fetchgraph-0.0.3}/src/fetchgraph.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fetchgraph
|
|
3
|
+
Version: 0.0.3
|
|
4
|
+
Summary: Graph-like planning → context fetching → synthesis agent (library-style).
|
|
5
|
+
Author: Alexander Onischenko
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: agents,context,langgraph,planning,llm
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Dist: pydantic>=2.6
|
|
12
|
+
Requires-Dist: pandas>=2.0
|
|
13
|
+
Provides-Extra: dev
|
|
14
|
+
Requires-Dist: pytest>=9.0; extra == "dev"
|
|
15
|
+
Dynamic: license-file
|
|
16
|
+
|
|
17
|
+
# fetchgraph
|
|
18
|
+
|
|
19
|
+
Universal, library-style agent that plans what to fetch, fetches context from pluggable providers, and synthesizes an output.
|
|
20
|
+
|
|
21
|
+
**Pipeline:** PLAN → FETCH → (ASSESS/REFETCH)* → SYNTH → VERIFY → (REFINE)* → SAVE
|
|
22
|
+
|
|
23
|
+
## Install (dev)
|
|
24
|
+
```bash
|
|
25
|
+
pip install -e .
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
# Quick Start
|
|
29
|
+
|
|
30
|
+
### Selectors are JSON-only
|
|
31
|
+
|
|
32
|
+
Providers receive a `selectors` argument that **must be JSON-serializable**. The
|
|
33
|
+
shared alias `SelectorsDict` (see `fetchgraph/json_types.py`) represents
|
|
34
|
+
`Dict[str, JSONValue]` and is used across protocols and models. The planner/LLM
|
|
35
|
+
produces this structure, so do not place runtime-only Python objects (e.g.
|
|
36
|
+
connections, DataFrames) into `selectors`; pass such hints through `**kwargs`
|
|
37
|
+
instead. Providers can publish the expected shape via `ProviderInfo.selectors_schema`
|
|
38
|
+
(a JSON Schema) and optional `examples` containing stringified JSON payloads.
|
|
39
|
+
|
|
40
|
+
Relational providers require `selectors` to include a string field `"op"` that
|
|
41
|
+
chooses the operation (e.g., `"schema"`, `"semantic_only"`, `"query"`). The
|
|
42
|
+
complete set of supported shapes is described by the schema returned from
|
|
43
|
+
`RelationalDataProvider.describe()`.
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
from fetchgraph import (
|
|
47
|
+
BaseGraphAgent, ContextPacker, BaselineSpec, ContextFetchSpec,
|
|
48
|
+
TaskProfile, RawLLMOutput
|
|
49
|
+
)
|
|
50
|
+
from fetchgraph.core import make_llm_plan_generic, make_llm_synth_generic
|
|
51
|
+
|
|
52
|
+
# Define providers (implement ContextProvider protocol)
|
|
53
|
+
class SpecProvider:
|
|
54
|
+
name = "spec"
|
|
55
|
+
def fetch(self, feature_name, selectors=None, **kw): return {"content": f"Spec for {feature_name}"}
|
|
56
|
+
def serialize(self, obj): return obj.get("content", "") if isinstance(obj, dict) else str(obj)
|
|
57
|
+
|
|
58
|
+
def dummy_llm(prompt: str, sender: str) -> str:
|
|
59
|
+
if sender == "generic_plan":
|
|
60
|
+
return '{"required_context":["spec"],"context_plan":[{"provider":"spec","mode":"full"}]}'
|
|
61
|
+
if sender == "generic_synth":
|
|
62
|
+
return "result: ok"
|
|
63
|
+
return ""
|
|
64
|
+
|
|
65
|
+
profile = TaskProfile(
|
|
66
|
+
task_name="Demo",
|
|
67
|
+
goal="Produce YAML doc from spec",
|
|
68
|
+
output_format="YAML: result: <...>"
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
agent = BaseGraphAgent(
|
|
72
|
+
llm_plan=make_llm_plan_generic(dummy_llm, profile, {"spec": SpecProvider()}),
|
|
73
|
+
llm_synth=make_llm_synth_generic(dummy_llm, profile),
|
|
74
|
+
domain_parser=lambda raw: raw.text, # RawLLMOutput -> Any
|
|
75
|
+
saver=lambda feature_name, parsed: None, # save side-effect
|
|
76
|
+
providers={"spec": SpecProvider()},
|
|
77
|
+
verifiers=[type("Ok",(),{"name":"ok","check":lambda self,out: []})()],
|
|
78
|
+
packer=ContextPacker(max_tokens=2000, summarizer_llm=lambda t: t[:200]),
|
|
79
|
+
baseline=[BaselineSpec(ContextFetchSpec(provider="spec"))],
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
print(agent.run("FeatureX"))
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
## Working with selectors
|
|
86
|
+
|
|
87
|
+
- **Plan-time inputs**: The planner/LLM crafts `selectors` (a `SelectorsDict`) for
|
|
88
|
+
each `ContextFetchSpec`. These inputs must be JSON-serializable and should be
|
|
89
|
+
validated by providers using their published JSON Schema.
|
|
90
|
+
- **Provider contract**: Implementations of `ContextProvider.fetch` should accept
|
|
91
|
+
`selectors: Optional[SelectorsDict] = None` and treat `**kwargs` as optional
|
|
92
|
+
runtime hints that may be non-serializable.
|
|
93
|
+
- **Schema + examples**: Providers can guide planners by returning
|
|
94
|
+
`ProviderInfo(selectors_schema=..., examples=[...])` from `describe()`.
|
|
95
|
+
|
|
96
|
+
Example for a relational provider that requires an `"op"` selector:
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
from fetchgraph.json_types import SelectorsDict
|
|
100
|
+
from fetchgraph.models import ProviderInfo
|
|
101
|
+
|
|
102
|
+
class RelationalDataProvider:
|
|
103
|
+
name = "relational"
|
|
104
|
+
|
|
105
|
+
def fetch(self, feature_name: str, selectors: SelectorsDict, **kwargs):
|
|
106
|
+
op = selectors.get("op")
|
|
107
|
+
if not op:
|
|
108
|
+
raise ValueError("selectors.op is required")
|
|
109
|
+
... # existing logic for schema/semantic_only/query
|
|
110
|
+
|
|
111
|
+
def describe(self) -> ProviderInfo:
|
|
112
|
+
schema = {
|
|
113
|
+
"oneOf": [
|
|
114
|
+
{"type": "object", "required": ["op"], "properties": {"op": {"const": "schema"}}},
|
|
115
|
+
{"type": "object", "required": ["op", "sql"], "properties": {"op": {"const": "query"}, "sql": {"type": "string"}}},
|
|
116
|
+
]
|
|
117
|
+
}
|
|
118
|
+
return ProviderInfo(
|
|
119
|
+
name=self.name,
|
|
120
|
+
selectors_schema=schema,
|
|
121
|
+
examples=["{\"op\":\"schema\"}", "{\"op\":\"query\",\"sql\":\"select 1\"}"],
|
|
122
|
+
)
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
During planning you can feed selectors into `ContextFetchSpec` to fix the
|
|
126
|
+
operation:
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
fetch_spec = ContextFetchSpec(provider="relational", selectors={"op": "schema"})
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
## CSV semantic backend for Pandas providers
|
|
133
|
+
|
|
134
|
+
`fetchgraph.semantic_backend` ships a lightweight TF-IDF backend that turns a CSV
|
|
135
|
+
file into semantic embeddings and reuses them across runs. The flow is:
|
|
136
|
+
|
|
137
|
+
1. Build embeddings from a CSV once using `CsvEmbeddingBuilder` and persist them
|
|
138
|
+
alongside the CSV.
|
|
139
|
+
2. Configure a `CsvSemanticBackend` with one or more `CsvSemanticSource` entries
|
|
140
|
+
(one per entity) pointing at the CSV and saved embeddings.
|
|
141
|
+
3. Pass that backend into `PandasRelationalDataProvider` so semantic clauses can
|
|
142
|
+
delegate matching to the precomputed vectors.
|
|
143
|
+
|
|
144
|
+
Example setup:
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
from pathlib import Path
|
|
148
|
+
from fetchgraph.semantic_backend import (
|
|
149
|
+
CsvEmbeddingBuilder,
|
|
150
|
+
CsvSemanticBackend,
|
|
151
|
+
CsvSemanticSource,
|
|
152
|
+
)
|
|
153
|
+
from fetchgraph.relational_models import EntityDescriptor, ColumnDescriptor
|
|
154
|
+
from fetchgraph.relational_pandas import PandasRelationalDataProvider
|
|
155
|
+
|
|
156
|
+
csv_path = Path("products.csv")
|
|
157
|
+
embedding_path = Path("products_embeddings.json")
|
|
158
|
+
|
|
159
|
+
# Build once (e.g., during deployment) to avoid recomputing embeddings at runtime.
|
|
160
|
+
CsvEmbeddingBuilder(
|
|
161
|
+
csv_path=csv_path,
|
|
162
|
+
entity="product",
|
|
163
|
+
id_column="id",
|
|
164
|
+
text_fields=["name", "description"],
|
|
165
|
+
output_path=embedding_path,
|
|
166
|
+
).build()
|
|
167
|
+
|
|
168
|
+
semantic_backend = CsvSemanticBackend(
|
|
169
|
+
{"product": CsvSemanticSource("product", csv_path, embedding_path)}
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
entities = [
|
|
173
|
+
EntityDescriptor(
|
|
174
|
+
name="product",
|
|
175
|
+
columns=[ColumnDescriptor(name="id", role="primary_key"), ColumnDescriptor(name="name"), ColumnDescriptor(name="description")],
|
|
176
|
+
)
|
|
177
|
+
]
|
|
178
|
+
|
|
179
|
+
provider = PandasRelationalDataProvider(
|
|
180
|
+
name="products", entities=entities, relations=[], frames={"product": ...}, semantic_backend=semantic_backend
|
|
181
|
+
)
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
At query time, `SemanticClause` filters sent to the relational provider will
|
|
185
|
+
call `semantic_backend.search(...)` with the requested entity, fields, and
|
|
186
|
+
query text. Fields must be a subset of the indexed CSV columns (not including
|
|
187
|
+
the reserved `__all__` combined projection). By default, field similarities are
|
|
188
|
+
summed; adjust the backend if you need a different aggregation strategy.
|
|
189
|
+
|
|
190
|
+
### pgvector / LangChain vector stores
|
|
191
|
+
|
|
192
|
+
If you already manage embeddings in PostgreSQL with ``pgvector`` via LangChain,
|
|
193
|
+
you can supply your existing vector stores directly:
|
|
194
|
+
|
|
195
|
+
```python
|
|
196
|
+
from langchain_community.vectorstores.pgvector import PGVector
|
|
197
|
+
from fetchgraph.semantic_backend import PgVectorSemanticBackend, PgVectorSemanticSource
|
|
198
|
+
|
|
199
|
+
vector_store = PGVector.from_existing_index(
|
|
200
|
+
collection_name="product_vectors", connection_string="postgresql+psycopg://..."
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
semantic_backend = PgVectorSemanticBackend(
|
|
204
|
+
{
|
|
205
|
+
"product": PgVectorSemanticSource(
|
|
206
|
+
entity="product",
|
|
207
|
+
vector_store=vector_store,
|
|
208
|
+
metadata_entity_key="entity", # optional, defaults to "entity"
|
|
209
|
+
metadata_field_key="field", # optional, defaults to "field"
|
|
210
|
+
id_metadata_keys=("id",), # optional metadata key(s) to read the row identifier
|
|
211
|
+
score_kind="distance", # convert pgvector distances into similarity scores
|
|
212
|
+
)
|
|
213
|
+
}
|
|
214
|
+
)
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
The backend will filter returned documents by entity and requested fields using
|
|
218
|
+
Document metadata before converting scores into :class:`SemanticMatch` entries.
|
|
219
|
+
|
|
220
|
+
---
|
|
221
|
+
|
|
222
|
+
## LICENSE
|
|
223
|
+
```text
|
|
224
|
+
MIT License
|
|
225
|
+
|
|
226
|
+
Copyright (c) 2025 ...
|
|
227
|
+
|
|
228
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
229
|
+
...
|
|
230
|
+
```
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
# fetchgraph
|
|
2
|
+
|
|
3
|
+
Universal, library-style agent that plans what to fetch, fetches context from pluggable providers, and synthesizes an output.
|
|
4
|
+
|
|
5
|
+
**Pipeline:** PLAN → FETCH → (ASSESS/REFETCH)* → SYNTH → VERIFY → (REFINE)* → SAVE
|
|
6
|
+
|
|
7
|
+
## Install (dev)
|
|
8
|
+
```bash
|
|
9
|
+
pip install -e .
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
# Quick Start
|
|
13
|
+
|
|
14
|
+
### Selectors are JSON-only
|
|
15
|
+
|
|
16
|
+
Providers receive a `selectors` argument that **must be JSON-serializable**. The
|
|
17
|
+
shared alias `SelectorsDict` (see `fetchgraph/json_types.py`) represents
|
|
18
|
+
`Dict[str, JSONValue]` and is used across protocols and models. The planner/LLM
|
|
19
|
+
produces this structure, so do not place runtime-only Python objects (e.g.
|
|
20
|
+
connections, DataFrames) into `selectors`; pass such hints through `**kwargs`
|
|
21
|
+
instead. Providers can publish the expected shape via `ProviderInfo.selectors_schema`
|
|
22
|
+
(a JSON Schema) and optional `examples` containing stringified JSON payloads.
|
|
23
|
+
|
|
24
|
+
Relational providers require `selectors` to include a string field `"op"` that
|
|
25
|
+
chooses the operation (e.g., `"schema"`, `"semantic_only"`, `"query"`). The
|
|
26
|
+
complete set of supported shapes is described by the schema returned from
|
|
27
|
+
`RelationalDataProvider.describe()`.
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
from fetchgraph import (
|
|
31
|
+
BaseGraphAgent, ContextPacker, BaselineSpec, ContextFetchSpec,
|
|
32
|
+
TaskProfile, RawLLMOutput
|
|
33
|
+
)
|
|
34
|
+
from fetchgraph.core import make_llm_plan_generic, make_llm_synth_generic
|
|
35
|
+
|
|
36
|
+
# Define providers (implement ContextProvider protocol)
|
|
37
|
+
class SpecProvider:
|
|
38
|
+
name = "spec"
|
|
39
|
+
def fetch(self, feature_name, selectors=None, **kw): return {"content": f"Spec for {feature_name}"}
|
|
40
|
+
def serialize(self, obj): return obj.get("content", "") if isinstance(obj, dict) else str(obj)
|
|
41
|
+
|
|
42
|
+
def dummy_llm(prompt: str, sender: str) -> str:
|
|
43
|
+
if sender == "generic_plan":
|
|
44
|
+
return '{"required_context":["spec"],"context_plan":[{"provider":"spec","mode":"full"}]}'
|
|
45
|
+
if sender == "generic_synth":
|
|
46
|
+
return "result: ok"
|
|
47
|
+
return ""
|
|
48
|
+
|
|
49
|
+
profile = TaskProfile(
|
|
50
|
+
task_name="Demo",
|
|
51
|
+
goal="Produce YAML doc from spec",
|
|
52
|
+
output_format="YAML: result: <...>"
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
agent = BaseGraphAgent(
|
|
56
|
+
llm_plan=make_llm_plan_generic(dummy_llm, profile, {"spec": SpecProvider()}),
|
|
57
|
+
llm_synth=make_llm_synth_generic(dummy_llm, profile),
|
|
58
|
+
domain_parser=lambda raw: raw.text, # RawLLMOutput -> Any
|
|
59
|
+
saver=lambda feature_name, parsed: None, # save side-effect
|
|
60
|
+
providers={"spec": SpecProvider()},
|
|
61
|
+
verifiers=[type("Ok",(),{"name":"ok","check":lambda self,out: []})()],
|
|
62
|
+
packer=ContextPacker(max_tokens=2000, summarizer_llm=lambda t: t[:200]),
|
|
63
|
+
baseline=[BaselineSpec(ContextFetchSpec(provider="spec"))],
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
print(agent.run("FeatureX"))
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## Working with selectors
|
|
70
|
+
|
|
71
|
+
- **Plan-time inputs**: The planner/LLM crafts `selectors` (a `SelectorsDict`) for
|
|
72
|
+
each `ContextFetchSpec`. These inputs must be JSON-serializable and should be
|
|
73
|
+
validated by providers using their published JSON Schema.
|
|
74
|
+
- **Provider contract**: Implementations of `ContextProvider.fetch` should accept
|
|
75
|
+
`selectors: Optional[SelectorsDict] = None` and treat `**kwargs` as optional
|
|
76
|
+
runtime hints that may be non-serializable.
|
|
77
|
+
- **Schema + examples**: Providers can guide planners by returning
|
|
78
|
+
`ProviderInfo(selectors_schema=..., examples=[...])` from `describe()`.
|
|
79
|
+
|
|
80
|
+
Example for a relational provider that requires an `"op"` selector:
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
from fetchgraph.json_types import SelectorsDict
|
|
84
|
+
from fetchgraph.models import ProviderInfo
|
|
85
|
+
|
|
86
|
+
class RelationalDataProvider:
|
|
87
|
+
name = "relational"
|
|
88
|
+
|
|
89
|
+
def fetch(self, feature_name: str, selectors: SelectorsDict, **kwargs):
|
|
90
|
+
op = selectors.get("op")
|
|
91
|
+
if not op:
|
|
92
|
+
raise ValueError("selectors.op is required")
|
|
93
|
+
... # existing logic for schema/semantic_only/query
|
|
94
|
+
|
|
95
|
+
def describe(self) -> ProviderInfo:
|
|
96
|
+
schema = {
|
|
97
|
+
"oneOf": [
|
|
98
|
+
{"type": "object", "required": ["op"], "properties": {"op": {"const": "schema"}}},
|
|
99
|
+
{"type": "object", "required": ["op", "sql"], "properties": {"op": {"const": "query"}, "sql": {"type": "string"}}},
|
|
100
|
+
]
|
|
101
|
+
}
|
|
102
|
+
return ProviderInfo(
|
|
103
|
+
name=self.name,
|
|
104
|
+
selectors_schema=schema,
|
|
105
|
+
examples=["{\"op\":\"schema\"}", "{\"op\":\"query\",\"sql\":\"select 1\"}"],
|
|
106
|
+
)
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
During planning you can feed selectors into `ContextFetchSpec` to fix the
|
|
110
|
+
operation:
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
fetch_spec = ContextFetchSpec(provider="relational", selectors={"op": "schema"})
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
## CSV semantic backend for Pandas providers
|
|
117
|
+
|
|
118
|
+
`fetchgraph.semantic_backend` ships a lightweight TF-IDF backend that turns a CSV
|
|
119
|
+
file into semantic embeddings and reuses them across runs. The flow is:
|
|
120
|
+
|
|
121
|
+
1. Build embeddings from a CSV once using `CsvEmbeddingBuilder` and persist them
|
|
122
|
+
alongside the CSV.
|
|
123
|
+
2. Configure a `CsvSemanticBackend` with one or more `CsvSemanticSource` entries
|
|
124
|
+
(one per entity) pointing at the CSV and saved embeddings.
|
|
125
|
+
3. Pass that backend into `PandasRelationalDataProvider` so semantic clauses can
|
|
126
|
+
delegate matching to the precomputed vectors.
|
|
127
|
+
|
|
128
|
+
Example setup:
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from pathlib import Path
|
|
132
|
+
from fetchgraph.semantic_backend import (
|
|
133
|
+
CsvEmbeddingBuilder,
|
|
134
|
+
CsvSemanticBackend,
|
|
135
|
+
CsvSemanticSource,
|
|
136
|
+
)
|
|
137
|
+
from fetchgraph.relational_models import EntityDescriptor, ColumnDescriptor
|
|
138
|
+
from fetchgraph.relational_pandas import PandasRelationalDataProvider
|
|
139
|
+
|
|
140
|
+
csv_path = Path("products.csv")
|
|
141
|
+
embedding_path = Path("products_embeddings.json")
|
|
142
|
+
|
|
143
|
+
# Build once (e.g., during deployment) to avoid recomputing embeddings at runtime.
|
|
144
|
+
CsvEmbeddingBuilder(
|
|
145
|
+
csv_path=csv_path,
|
|
146
|
+
entity="product",
|
|
147
|
+
id_column="id",
|
|
148
|
+
text_fields=["name", "description"],
|
|
149
|
+
output_path=embedding_path,
|
|
150
|
+
).build()
|
|
151
|
+
|
|
152
|
+
semantic_backend = CsvSemanticBackend(
|
|
153
|
+
{"product": CsvSemanticSource("product", csv_path, embedding_path)}
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
entities = [
|
|
157
|
+
EntityDescriptor(
|
|
158
|
+
name="product",
|
|
159
|
+
columns=[ColumnDescriptor(name="id", role="primary_key"), ColumnDescriptor(name="name"), ColumnDescriptor(name="description")],
|
|
160
|
+
)
|
|
161
|
+
]
|
|
162
|
+
|
|
163
|
+
provider = PandasRelationalDataProvider(
|
|
164
|
+
name="products", entities=entities, relations=[], frames={"product": ...}, semantic_backend=semantic_backend
|
|
165
|
+
)
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
At query time, `SemanticClause` filters sent to the relational provider will
|
|
169
|
+
call `semantic_backend.search(...)` with the requested entity, fields, and
|
|
170
|
+
query text. Fields must be a subset of the indexed CSV columns (not including
|
|
171
|
+
the reserved `__all__` combined projection). By default, field similarities are
|
|
172
|
+
summed; adjust the backend if you need a different aggregation strategy.
|
|
173
|
+
|
|
174
|
+
### pgvector / LangChain vector stores
|
|
175
|
+
|
|
176
|
+
If you already manage embeddings in PostgreSQL with ``pgvector`` via LangChain,
|
|
177
|
+
you can supply your existing vector stores directly:
|
|
178
|
+
|
|
179
|
+
```python
|
|
180
|
+
from langchain_community.vectorstores.pgvector import PGVector
|
|
181
|
+
from fetchgraph.semantic_backend import PgVectorSemanticBackend, PgVectorSemanticSource
|
|
182
|
+
|
|
183
|
+
vector_store = PGVector.from_existing_index(
|
|
184
|
+
collection_name="product_vectors", connection_string="postgresql+psycopg://..."
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
semantic_backend = PgVectorSemanticBackend(
|
|
188
|
+
{
|
|
189
|
+
"product": PgVectorSemanticSource(
|
|
190
|
+
entity="product",
|
|
191
|
+
vector_store=vector_store,
|
|
192
|
+
metadata_entity_key="entity", # optional, defaults to "entity"
|
|
193
|
+
metadata_field_key="field", # optional, defaults to "field"
|
|
194
|
+
id_metadata_keys=("id",), # optional metadata key(s) to read the row identifier
|
|
195
|
+
score_kind="distance", # convert pgvector distances into similarity scores
|
|
196
|
+
)
|
|
197
|
+
}
|
|
198
|
+
)
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
The backend will filter returned documents by entity and requested fields using
|
|
202
|
+
Document metadata before converting scores into :class:`SemanticMatch` entries.
|
|
203
|
+
|
|
204
|
+
---
|
|
205
|
+
|
|
206
|
+
## LICENSE
|
|
207
|
+
```text
|
|
208
|
+
MIT License
|
|
209
|
+
|
|
210
|
+
Copyright (c) 2025 ...
|
|
211
|
+
|
|
212
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
213
|
+
...
|
|
214
|
+
```
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "fetchgraph"
|
|
3
|
-
version = "0.0.
|
|
3
|
+
version = "0.0.3"
|
|
4
4
|
description = "Graph-like planning → context fetching → synthesis agent (library-style)."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.10"
|
|
@@ -9,11 +9,23 @@ authors = [{name = "Alexander Onischenko"}]
|
|
|
9
9
|
keywords = ["agents", "context", "langgraph", "planning", "llm"]
|
|
10
10
|
dependencies = [
|
|
11
11
|
"pydantic>=2.6",
|
|
12
|
+
"pandas>=2.0",
|
|
12
13
|
]
|
|
13
14
|
|
|
14
15
|
[tool.setuptools.package-data]
|
|
15
16
|
fetchgraph = ["prompts/*.md"]
|
|
16
17
|
|
|
18
|
+
[tool.setuptools]
|
|
19
|
+
package-dir = {"" = "src"}
|
|
20
|
+
|
|
21
|
+
[tool.setuptools.packages.find]
|
|
22
|
+
where = ["src"]
|
|
23
|
+
|
|
17
24
|
[build-system]
|
|
18
25
|
requires = ["setuptools>=67", "wheel"]
|
|
19
|
-
build-backend = "setuptools.build_meta"
|
|
26
|
+
build-backend = "setuptools.build_meta"
|
|
27
|
+
|
|
28
|
+
[project.optional-dependencies]
|
|
29
|
+
dev = [
|
|
30
|
+
"pytest>=9.0",
|
|
31
|
+
]
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
from pydantic import __version__ as _pydantic_version
|
|
2
|
+
|
|
3
|
+
# Fetchgraph relies on the Pydantic v2 API (model_validate/model_dump, etc.).
|
|
4
|
+
# Import errors should surface early if an incompatible version is installed.
|
|
5
|
+
if not _pydantic_version.startswith("2"):
|
|
6
|
+
raise ImportError(
|
|
7
|
+
"fetchgraph requires pydantic>=2.0; detected version %s" % _pydantic_version
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
from .core import (
|
|
11
|
+
ContextPacker,
|
|
12
|
+
BaseGraphAgent,
|
|
13
|
+
make_llm_plan_generic,
|
|
14
|
+
make_llm_synth_generic,
|
|
15
|
+
)
|
|
16
|
+
from .models import (
|
|
17
|
+
RawLLMOutput,
|
|
18
|
+
ProviderInfo,
|
|
19
|
+
TaskProfile,
|
|
20
|
+
ContextFetchSpec,
|
|
21
|
+
BaselineSpec,
|
|
22
|
+
ContextItem,
|
|
23
|
+
RefetchDecision,
|
|
24
|
+
Plan,
|
|
25
|
+
)
|
|
26
|
+
from .protocols import (
|
|
27
|
+
ContextProvider,
|
|
28
|
+
SupportsFilter,
|
|
29
|
+
SupportsDescribe,
|
|
30
|
+
Verifier,
|
|
31
|
+
Saver,
|
|
32
|
+
LLMInvoke,
|
|
33
|
+
)
|
|
34
|
+
from .relational_provider import (
|
|
35
|
+
AggregationResult,
|
|
36
|
+
AggregationSpec,
|
|
37
|
+
ColumnDescriptor,
|
|
38
|
+
CompositeRelationalProvider,
|
|
39
|
+
EntityDescriptor,
|
|
40
|
+
GroupBySpec,
|
|
41
|
+
LogicalFilter,
|
|
42
|
+
PandasRelationalDataProvider,
|
|
43
|
+
SqlRelationalDataProvider,
|
|
44
|
+
RelatedEntityData,
|
|
45
|
+
RelationDescriptor,
|
|
46
|
+
RelationJoin,
|
|
47
|
+
RelationalDataProvider,
|
|
48
|
+
RelationalQuery,
|
|
49
|
+
RelationalRequest,
|
|
50
|
+
RelationalResponse,
|
|
51
|
+
RowResult,
|
|
52
|
+
SchemaRequest,
|
|
53
|
+
SchemaResult,
|
|
54
|
+
SelectExpr,
|
|
55
|
+
SemanticBackend,
|
|
56
|
+
SemanticClause,
|
|
57
|
+
SemanticMatch,
|
|
58
|
+
SemanticOnlyRequest,
|
|
59
|
+
SemanticOnlyResult,
|
|
60
|
+
)
|
|
61
|
+
from .semantic_backend import (
|
|
62
|
+
CsvEmbeddingBuilder,
|
|
63
|
+
CsvSemanticBackend,
|
|
64
|
+
CsvSemanticSource,
|
|
65
|
+
PgVectorSemanticBackend,
|
|
66
|
+
PgVectorSemanticSource,
|
|
67
|
+
VectorStoreLike,
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
__all__ = [
|
|
71
|
+
"RawLLMOutput",
|
|
72
|
+
"ProviderInfo",
|
|
73
|
+
"TaskProfile",
|
|
74
|
+
"ContextFetchSpec",
|
|
75
|
+
"BaselineSpec",
|
|
76
|
+
"ContextItem",
|
|
77
|
+
"RefetchDecision",
|
|
78
|
+
"Plan",
|
|
79
|
+
"ContextProvider",
|
|
80
|
+
"SupportsFilter",
|
|
81
|
+
"SupportsDescribe",
|
|
82
|
+
"Verifier",
|
|
83
|
+
"Saver",
|
|
84
|
+
"LLMInvoke",
|
|
85
|
+
"ContextPacker",
|
|
86
|
+
"BaseGraphAgent",
|
|
87
|
+
"make_llm_plan_generic",
|
|
88
|
+
"make_llm_synth_generic",
|
|
89
|
+
"AggregationResult",
|
|
90
|
+
"AggregationSpec",
|
|
91
|
+
"ColumnDescriptor",
|
|
92
|
+
"CompositeRelationalProvider",
|
|
93
|
+
"EntityDescriptor",
|
|
94
|
+
"GroupBySpec",
|
|
95
|
+
"LogicalFilter",
|
|
96
|
+
"SqlRelationalDataProvider",
|
|
97
|
+
"RelatedEntityData",
|
|
98
|
+
"RelationDescriptor",
|
|
99
|
+
"RelationJoin",
|
|
100
|
+
"RelationalDataProvider",
|
|
101
|
+
"RelationalQuery",
|
|
102
|
+
"RelationalRequest",
|
|
103
|
+
"RelationalResponse",
|
|
104
|
+
"RowResult",
|
|
105
|
+
"SchemaRequest",
|
|
106
|
+
"SchemaResult",
|
|
107
|
+
"SelectExpr",
|
|
108
|
+
"SemanticBackend",
|
|
109
|
+
"SemanticClause",
|
|
110
|
+
"SemanticMatch",
|
|
111
|
+
"SemanticOnlyRequest",
|
|
112
|
+
"SemanticOnlyResult",
|
|
113
|
+
"CsvSemanticBackend",
|
|
114
|
+
"CsvSemanticSource",
|
|
115
|
+
"CsvEmbeddingBuilder",
|
|
116
|
+
"PgVectorSemanticBackend",
|
|
117
|
+
"PgVectorSemanticSource",
|
|
118
|
+
"VectorStoreLike",
|
|
119
|
+
]
|
|
120
|
+
|
|
121
|
+
if PandasRelationalDataProvider is not None:
|
|
122
|
+
__all__.append("PandasRelationalDataProvider")
|