flock-core 0.3.16__py3-none-any.whl → 0.3.18__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of flock-core might be problematic. Click here for more details.
- flock/cli/assets/release_notes.md +1 -0
- flock/core/tools/azure_tools.py +496 -0
- flock/core/tools/basic_tools.py +96 -8
- {flock_core-0.3.16.dist-info → flock_core-0.3.18.dist-info}/METADATA +3 -1
- {flock_core-0.3.16.dist-info → flock_core-0.3.18.dist-info}/RECORD +8 -7
- {flock_core-0.3.16.dist-info → flock_core-0.3.18.dist-info}/WHEEL +0 -0
- {flock_core-0.3.16.dist-info → flock_core-0.3.18.dist-info}/entry_points.txt +0 -0
- {flock_core-0.3.16.dist-info → flock_core-0.3.18.dist-info}/licenses/LICENSE +0 -0
|
@@ -69,6 +69,7 @@ Like a hummingbird, modules are small and nimble code packages. Put enough of th
|
|
|
69
69
|
- **Output Module** – Advanced output formatting and storage
|
|
70
70
|
- **Metrics Module** – Detailed performance tracking
|
|
71
71
|
- **Zep Module** – Uses Zep for Knowledge Graphs
|
|
72
|
+
- **Azure Search Tools** – Integration with Azure AI Search for vector search and document retrieval
|
|
72
73
|
|
|
73
74
|
---
|
|
74
75
|
|
|
@@ -0,0 +1,496 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
from azure.core.credentials import AzureKeyCredential
|
|
5
|
+
from azure.search.documents import SearchClient
|
|
6
|
+
from azure.search.documents.indexes import SearchIndexClient
|
|
7
|
+
from azure.search.documents.indexes.models import (
|
|
8
|
+
ExhaustiveKnnAlgorithmConfiguration,
|
|
9
|
+
HnswAlgorithmConfiguration,
|
|
10
|
+
SearchableField,
|
|
11
|
+
SearchField,
|
|
12
|
+
SearchFieldDataType,
|
|
13
|
+
SearchIndex,
|
|
14
|
+
SimpleField,
|
|
15
|
+
VectorSearch,
|
|
16
|
+
VectorSearchProfile,
|
|
17
|
+
)
|
|
18
|
+
from azure.search.documents.models import VectorizedQuery
|
|
19
|
+
|
|
20
|
+
from flock.core.logging.trace_and_logged import traced_and_logged
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _get_default_endpoint() -> str:
|
|
24
|
+
"""Get the default Azure Search endpoint from environment variables."""
|
|
25
|
+
endpoint = os.environ.get("AZURE_SEARCH_ENDPOINT")
|
|
26
|
+
if not endpoint:
|
|
27
|
+
raise ValueError(
|
|
28
|
+
"AZURE_SEARCH_ENDPOINT environment variable is not set"
|
|
29
|
+
)
|
|
30
|
+
return endpoint
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _get_default_api_key() -> str:
|
|
34
|
+
"""Get the default Azure Search API key from environment variables."""
|
|
35
|
+
api_key = os.environ.get("AZURE_SEARCH_API_KEY")
|
|
36
|
+
if not api_key:
|
|
37
|
+
raise ValueError("AZURE_SEARCH_API_KEY environment variable is not set")
|
|
38
|
+
return api_key
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _get_default_index_name() -> str:
|
|
42
|
+
"""Get the default Azure Search index name from environment variables."""
|
|
43
|
+
index_name = os.environ.get("AZURE_SEARCH_INDEX_NAME")
|
|
44
|
+
if not index_name:
|
|
45
|
+
raise ValueError(
|
|
46
|
+
"AZURE_SEARCH_INDEX_NAME environment variable is not set"
|
|
47
|
+
)
|
|
48
|
+
return index_name
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@traced_and_logged
|
|
52
|
+
def azure_search_initialize_clients(
|
|
53
|
+
endpoint: str | None = None,
|
|
54
|
+
api_key: str | None = None,
|
|
55
|
+
index_name: str | None = None,
|
|
56
|
+
) -> dict[str, Any]:
|
|
57
|
+
"""Initialize Azure AI Search clients.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
endpoint: The Azure AI Search service endpoint URL (defaults to AZURE_SEARCH_ENDPOINT env var)
|
|
61
|
+
api_key: The Azure AI Search API key (defaults to AZURE_SEARCH_API_KEY env var)
|
|
62
|
+
index_name: Optional index name for SearchClient initialization (defaults to AZURE_SEARCH_INDEX_NAME env var if not None)
|
|
63
|
+
|
|
64
|
+
Returns:
|
|
65
|
+
Dictionary containing the initialized clients
|
|
66
|
+
"""
|
|
67
|
+
# Use environment variables as defaults if not provided
|
|
68
|
+
endpoint = endpoint or _get_default_endpoint()
|
|
69
|
+
api_key = api_key or _get_default_api_key()
|
|
70
|
+
|
|
71
|
+
credential = AzureKeyCredential(api_key)
|
|
72
|
+
|
|
73
|
+
# Create the search index client
|
|
74
|
+
search_index_client = SearchIndexClient(
|
|
75
|
+
endpoint=endpoint, credential=credential
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
# Create clients dictionary
|
|
79
|
+
clients = {
|
|
80
|
+
"index_client": search_index_client,
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
# Add search client if index_name was provided or available in env
|
|
84
|
+
if index_name is None and os.environ.get("AZURE_SEARCH_INDEX_NAME"):
|
|
85
|
+
index_name = _get_default_index_name()
|
|
86
|
+
|
|
87
|
+
if index_name:
|
|
88
|
+
search_client = SearchClient(
|
|
89
|
+
endpoint=endpoint, index_name=index_name, credential=credential
|
|
90
|
+
)
|
|
91
|
+
clients["search_client"] = search_client
|
|
92
|
+
|
|
93
|
+
return clients
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@traced_and_logged
|
|
97
|
+
def azure_search_create_index(
|
|
98
|
+
index_name: str | None = None,
|
|
99
|
+
fields: list[SearchField] = None,
|
|
100
|
+
vector_search: VectorSearch | None = None,
|
|
101
|
+
endpoint: str | None = None,
|
|
102
|
+
api_key: str | None = None,
|
|
103
|
+
) -> dict[str, Any]:
|
|
104
|
+
"""Create a new search index in Azure AI Search.
|
|
105
|
+
|
|
106
|
+
Args:
|
|
107
|
+
index_name: Name of the search index to create (defaults to AZURE_SEARCH_INDEX_NAME env var)
|
|
108
|
+
fields: List of field definitions for the index
|
|
109
|
+
vector_search: Optional vector search configuration
|
|
110
|
+
endpoint: The Azure AI Search service endpoint URL (defaults to AZURE_SEARCH_ENDPOINT env var)
|
|
111
|
+
api_key: The Azure AI Search API key (defaults to AZURE_SEARCH_API_KEY env var)
|
|
112
|
+
|
|
113
|
+
Returns:
|
|
114
|
+
Dictionary containing information about the created index
|
|
115
|
+
"""
|
|
116
|
+
# Use environment variables as defaults if not provided
|
|
117
|
+
endpoint = endpoint or _get_default_endpoint()
|
|
118
|
+
api_key = api_key or _get_default_api_key()
|
|
119
|
+
index_name = index_name or _get_default_index_name()
|
|
120
|
+
|
|
121
|
+
if fields is None:
|
|
122
|
+
raise ValueError("Fields must be provided for index creation")
|
|
123
|
+
|
|
124
|
+
clients = azure_search_initialize_clients(endpoint, api_key)
|
|
125
|
+
index_client = clients["index_client"]
|
|
126
|
+
|
|
127
|
+
# Create the index
|
|
128
|
+
index = SearchIndex(
|
|
129
|
+
name=index_name, fields=fields, vector_search=vector_search
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
result = index_client.create_or_update_index(index)
|
|
133
|
+
|
|
134
|
+
return {
|
|
135
|
+
"index_name": result.name,
|
|
136
|
+
"fields": [field.name for field in result.fields],
|
|
137
|
+
"created": True,
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@traced_and_logged
|
|
142
|
+
def azure_search_upload_documents(
|
|
143
|
+
documents: list[dict[str, Any]],
|
|
144
|
+
index_name: str | None = None,
|
|
145
|
+
endpoint: str | None = None,
|
|
146
|
+
api_key: str | None = None,
|
|
147
|
+
) -> dict[str, Any]:
|
|
148
|
+
"""Upload documents to an Azure AI Search index.
|
|
149
|
+
|
|
150
|
+
Args:
|
|
151
|
+
documents: List of documents to upload (as dictionaries)
|
|
152
|
+
index_name: Name of the search index (defaults to AZURE_SEARCH_INDEX_NAME env var)
|
|
153
|
+
endpoint: The Azure AI Search service endpoint URL (defaults to AZURE_SEARCH_ENDPOINT env var)
|
|
154
|
+
api_key: The Azure AI Search API key (defaults to AZURE_SEARCH_API_KEY env var)
|
|
155
|
+
|
|
156
|
+
Returns:
|
|
157
|
+
Dictionary containing the upload results
|
|
158
|
+
"""
|
|
159
|
+
# Use environment variables as defaults if not provided
|
|
160
|
+
endpoint = endpoint or _get_default_endpoint()
|
|
161
|
+
api_key = api_key or _get_default_api_key()
|
|
162
|
+
index_name = index_name or _get_default_index_name()
|
|
163
|
+
|
|
164
|
+
clients = azure_search_initialize_clients(endpoint, api_key, index_name)
|
|
165
|
+
search_client = clients["search_client"]
|
|
166
|
+
|
|
167
|
+
result = search_client.upload_documents(documents=documents)
|
|
168
|
+
|
|
169
|
+
# Process results
|
|
170
|
+
succeeded = sum(1 for r in result if r.succeeded)
|
|
171
|
+
|
|
172
|
+
return {
|
|
173
|
+
"succeeded": succeeded,
|
|
174
|
+
"failed": len(result) - succeeded,
|
|
175
|
+
"total": len(result),
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
@traced_and_logged
|
|
180
|
+
def azure_search_query(
|
|
181
|
+
search_text: str | None = None,
|
|
182
|
+
filter: str | None = None,
|
|
183
|
+
select: list[str] | None = None,
|
|
184
|
+
top: int | None = 50,
|
|
185
|
+
vector: list[float] | None = None,
|
|
186
|
+
vector_field: str | None = None,
|
|
187
|
+
vector_k: int | None = 10,
|
|
188
|
+
index_name: str | None = None,
|
|
189
|
+
endpoint: str | None = None,
|
|
190
|
+
api_key: str | None = None,
|
|
191
|
+
) -> list[dict[str, Any]]:
|
|
192
|
+
"""Search documents in an Azure AI Search index.
|
|
193
|
+
|
|
194
|
+
Args:
|
|
195
|
+
search_text: Optional text to search for (keyword search)
|
|
196
|
+
filter: Optional OData filter expression
|
|
197
|
+
select: Optional list of fields to return
|
|
198
|
+
top: Maximum number of results to return
|
|
199
|
+
vector: Optional vector for vector search
|
|
200
|
+
vector_field: Name of the field containing vectors for vector search
|
|
201
|
+
vector_k: Number of nearest neighbors to retrieve in vector search
|
|
202
|
+
index_name: Name of the search index (defaults to AZURE_SEARCH_INDEX_NAME env var)
|
|
203
|
+
endpoint: The Azure AI Search service endpoint URL (defaults to AZURE_SEARCH_ENDPOINT env var)
|
|
204
|
+
api_key: The Azure AI Search API key (defaults to AZURE_SEARCH_API_KEY env var)
|
|
205
|
+
|
|
206
|
+
Returns:
|
|
207
|
+
List of search results as dictionaries
|
|
208
|
+
"""
|
|
209
|
+
# Use environment variables as defaults if not provided
|
|
210
|
+
endpoint = endpoint or _get_default_endpoint()
|
|
211
|
+
api_key = api_key or _get_default_api_key()
|
|
212
|
+
index_name = index_name or _get_default_index_name()
|
|
213
|
+
|
|
214
|
+
clients = azure_search_initialize_clients(endpoint, api_key, index_name)
|
|
215
|
+
search_client = clients["search_client"]
|
|
216
|
+
|
|
217
|
+
# Set up vector query if vector is provided
|
|
218
|
+
vectorized_query = None
|
|
219
|
+
if vector and vector_field:
|
|
220
|
+
vectorized_query = VectorizedQuery(
|
|
221
|
+
vector=vector, k=vector_k, fields=[vector_field]
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
# Execute the search
|
|
225
|
+
results = search_client.search(
|
|
226
|
+
search_text=search_text,
|
|
227
|
+
filter=filter,
|
|
228
|
+
select=select,
|
|
229
|
+
top=top,
|
|
230
|
+
vector_queries=[vectorized_query] if vectorized_query else None,
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
# Convert results to list of dictionaries
|
|
234
|
+
result_list = [dict(result) for result in results]
|
|
235
|
+
|
|
236
|
+
return result_list
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
@traced_and_logged
|
|
240
|
+
def azure_search_get_document(
|
|
241
|
+
key: str,
|
|
242
|
+
select: list[str] | None = None,
|
|
243
|
+
index_name: str | None = None,
|
|
244
|
+
endpoint: str | None = None,
|
|
245
|
+
api_key: str | None = None,
|
|
246
|
+
) -> dict[str, Any]:
|
|
247
|
+
"""Retrieve a specific document from an Azure AI Search index by key.
|
|
248
|
+
|
|
249
|
+
Args:
|
|
250
|
+
key: The unique key of the document to retrieve
|
|
251
|
+
select: Optional list of fields to return
|
|
252
|
+
index_name: Name of the search index (defaults to AZURE_SEARCH_INDEX_NAME env var)
|
|
253
|
+
endpoint: The Azure AI Search service endpoint URL (defaults to AZURE_SEARCH_ENDPOINT env var)
|
|
254
|
+
api_key: The Azure AI Search API key (defaults to AZURE_SEARCH_API_KEY env var)
|
|
255
|
+
|
|
256
|
+
Returns:
|
|
257
|
+
The retrieved document as a dictionary
|
|
258
|
+
"""
|
|
259
|
+
# Use environment variables as defaults if not provided
|
|
260
|
+
endpoint = endpoint or _get_default_endpoint()
|
|
261
|
+
api_key = api_key or _get_default_api_key()
|
|
262
|
+
index_name = index_name or _get_default_index_name()
|
|
263
|
+
|
|
264
|
+
clients = azure_search_initialize_clients(endpoint, api_key, index_name)
|
|
265
|
+
search_client = clients["search_client"]
|
|
266
|
+
|
|
267
|
+
result = search_client.get_document(key=key, selected_fields=select)
|
|
268
|
+
|
|
269
|
+
return dict(result)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
@traced_and_logged
|
|
273
|
+
def azure_search_delete_documents(
|
|
274
|
+
keys: list[str],
|
|
275
|
+
key_field_name: str = "id",
|
|
276
|
+
index_name: str | None = None,
|
|
277
|
+
endpoint: str | None = None,
|
|
278
|
+
api_key: str | None = None,
|
|
279
|
+
) -> dict[str, Any]:
|
|
280
|
+
"""Delete documents from an Azure AI Search index.
|
|
281
|
+
|
|
282
|
+
Args:
|
|
283
|
+
keys: List of document keys to delete
|
|
284
|
+
key_field_name: Name of the key field (defaults to "id")
|
|
285
|
+
index_name: Name of the search index (defaults to AZURE_SEARCH_INDEX_NAME env var)
|
|
286
|
+
endpoint: The Azure AI Search service endpoint URL (defaults to AZURE_SEARCH_ENDPOINT env var)
|
|
287
|
+
api_key: The Azure AI Search API key (defaults to AZURE_SEARCH_API_KEY env var)
|
|
288
|
+
|
|
289
|
+
Returns:
|
|
290
|
+
Dictionary containing the deletion results
|
|
291
|
+
"""
|
|
292
|
+
# Use environment variables as defaults if not provided
|
|
293
|
+
endpoint = endpoint or _get_default_endpoint()
|
|
294
|
+
api_key = api_key or _get_default_api_key()
|
|
295
|
+
index_name = index_name or _get_default_index_name()
|
|
296
|
+
|
|
297
|
+
clients = azure_search_initialize_clients(endpoint, api_key, index_name)
|
|
298
|
+
search_client = clients["search_client"]
|
|
299
|
+
|
|
300
|
+
# Format documents for deletion (only need the key field)
|
|
301
|
+
documents_to_delete = [{key_field_name: key} for key in keys]
|
|
302
|
+
|
|
303
|
+
result = search_client.delete_documents(documents=documents_to_delete)
|
|
304
|
+
|
|
305
|
+
# Process results
|
|
306
|
+
succeeded = sum(1 for r in result if r.succeeded)
|
|
307
|
+
|
|
308
|
+
return {
|
|
309
|
+
"succeeded": succeeded,
|
|
310
|
+
"failed": len(result) - succeeded,
|
|
311
|
+
"total": len(result),
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
@traced_and_logged
|
|
316
|
+
def azure_search_list_indexes(
|
|
317
|
+
endpoint: str | None = None, api_key: str | None = None
|
|
318
|
+
) -> list[dict[str, Any]]:
|
|
319
|
+
"""List all indexes in the Azure AI Search service.
|
|
320
|
+
|
|
321
|
+
Args:
|
|
322
|
+
endpoint: The Azure AI Search service endpoint URL (defaults to AZURE_SEARCH_ENDPOINT env var)
|
|
323
|
+
api_key: The Azure AI Search API key (defaults to AZURE_SEARCH_API_KEY env var)
|
|
324
|
+
|
|
325
|
+
Returns:
|
|
326
|
+
List of indexes as dictionaries
|
|
327
|
+
"""
|
|
328
|
+
# Use environment variables as defaults if not provided
|
|
329
|
+
endpoint = endpoint or _get_default_endpoint()
|
|
330
|
+
api_key = api_key or _get_default_api_key()
|
|
331
|
+
|
|
332
|
+
clients = azure_search_initialize_clients(endpoint, api_key)
|
|
333
|
+
index_client = clients["index_client"]
|
|
334
|
+
|
|
335
|
+
result = index_client.list_indexes()
|
|
336
|
+
|
|
337
|
+
# Convert index objects to dictionaries with basic information
|
|
338
|
+
indexes = [
|
|
339
|
+
{
|
|
340
|
+
"name": index.name,
|
|
341
|
+
"fields": [field.name for field in index.fields],
|
|
342
|
+
"field_count": len(index.fields),
|
|
343
|
+
}
|
|
344
|
+
for index in result
|
|
345
|
+
]
|
|
346
|
+
|
|
347
|
+
return indexes
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
@traced_and_logged
|
|
351
|
+
def azure_search_get_index_statistics(
|
|
352
|
+
index_name: str | None = None,
|
|
353
|
+
endpoint: str | None = None,
|
|
354
|
+
api_key: str | None = None,
|
|
355
|
+
) -> dict[str, Any]:
|
|
356
|
+
"""Get statistics for a specific Azure AI Search index.
|
|
357
|
+
|
|
358
|
+
Args:
|
|
359
|
+
index_name: Name of the search index (defaults to AZURE_SEARCH_INDEX_NAME env var)
|
|
360
|
+
endpoint: The Azure AI Search service endpoint URL (defaults to AZURE_SEARCH_ENDPOINT env var)
|
|
361
|
+
api_key: The Azure AI Search API key (defaults to AZURE_SEARCH_API_KEY env var)
|
|
362
|
+
|
|
363
|
+
Returns:
|
|
364
|
+
Dictionary containing index statistics
|
|
365
|
+
"""
|
|
366
|
+
# Use environment variables as defaults if not provided
|
|
367
|
+
endpoint = endpoint or _get_default_endpoint()
|
|
368
|
+
api_key = api_key or _get_default_api_key()
|
|
369
|
+
index_name = index_name or _get_default_index_name()
|
|
370
|
+
|
|
371
|
+
clients = azure_search_initialize_clients(endpoint, api_key, index_name)
|
|
372
|
+
search_client = clients["search_client"]
|
|
373
|
+
|
|
374
|
+
stats = search_client.get_document_count()
|
|
375
|
+
|
|
376
|
+
return {"document_count": stats}
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
@traced_and_logged
|
|
380
|
+
def azure_search_create_vector_index(
|
|
381
|
+
fields: list[dict[str, Any]],
|
|
382
|
+
vector_dimensions: int,
|
|
383
|
+
index_name: str | None = None,
|
|
384
|
+
algorithm_kind: str = "hnsw",
|
|
385
|
+
endpoint: str | None = None,
|
|
386
|
+
api_key: str | None = None,
|
|
387
|
+
) -> dict[str, Any]:
|
|
388
|
+
"""Create a vector search index in Azure AI Search.
|
|
389
|
+
|
|
390
|
+
Args:
|
|
391
|
+
fields: List of field configurations (dicts with name, type, etc.)
|
|
392
|
+
vector_dimensions: Dimensions of the vector field
|
|
393
|
+
index_name: Name of the search index (defaults to AZURE_SEARCH_INDEX_NAME env var)
|
|
394
|
+
algorithm_kind: Vector search algorithm ("hnsw" or "exhaustive")
|
|
395
|
+
endpoint: The Azure AI Search service endpoint URL (defaults to AZURE_SEARCH_ENDPOINT env var)
|
|
396
|
+
api_key: The Azure AI Search API key (defaults to AZURE_SEARCH_API_KEY env var)
|
|
397
|
+
|
|
398
|
+
Returns:
|
|
399
|
+
Dictionary with index creation result
|
|
400
|
+
"""
|
|
401
|
+
# Use environment variables as defaults if not provided
|
|
402
|
+
endpoint = endpoint or _get_default_endpoint()
|
|
403
|
+
api_key = api_key or _get_default_api_key()
|
|
404
|
+
index_name = index_name or _get_default_index_name()
|
|
405
|
+
|
|
406
|
+
clients = azure_search_initialize_clients(endpoint, api_key)
|
|
407
|
+
index_client = clients["index_client"]
|
|
408
|
+
|
|
409
|
+
# Convert field configurations to SearchField objects
|
|
410
|
+
index_fields = []
|
|
411
|
+
vector_fields = []
|
|
412
|
+
|
|
413
|
+
for field_config in fields:
|
|
414
|
+
field_name = field_config["name"]
|
|
415
|
+
field_type = field_config["type"]
|
|
416
|
+
field_searchable = field_config.get("searchable", False)
|
|
417
|
+
field_filterable = field_config.get("filterable", False)
|
|
418
|
+
field_sortable = field_config.get("sortable", False)
|
|
419
|
+
field_key = field_config.get("key", False)
|
|
420
|
+
field_vector = field_config.get("vector", False)
|
|
421
|
+
|
|
422
|
+
if field_searchable and field_type == "string":
|
|
423
|
+
field = SearchableField(
|
|
424
|
+
name=field_name,
|
|
425
|
+
type=SearchFieldDataType.String,
|
|
426
|
+
key=field_key,
|
|
427
|
+
filterable=field_filterable,
|
|
428
|
+
sortable=field_sortable,
|
|
429
|
+
)
|
|
430
|
+
else:
|
|
431
|
+
data_type = None
|
|
432
|
+
if field_type == "string":
|
|
433
|
+
data_type = SearchFieldDataType.String
|
|
434
|
+
elif field_type == "int":
|
|
435
|
+
data_type = SearchFieldDataType.Int32
|
|
436
|
+
elif field_type == "double":
|
|
437
|
+
data_type = SearchFieldDataType.Double
|
|
438
|
+
elif field_type == "boolean":
|
|
439
|
+
data_type = SearchFieldDataType.Boolean
|
|
440
|
+
elif field_type == "collection":
|
|
441
|
+
data_type = SearchFieldDataType.Collection(
|
|
442
|
+
SearchFieldDataType.String
|
|
443
|
+
)
|
|
444
|
+
|
|
445
|
+
field = SimpleField(
|
|
446
|
+
name=field_name,
|
|
447
|
+
type=data_type,
|
|
448
|
+
key=field_key,
|
|
449
|
+
filterable=field_filterable,
|
|
450
|
+
sortable=field_sortable,
|
|
451
|
+
)
|
|
452
|
+
|
|
453
|
+
index_fields.append(field)
|
|
454
|
+
|
|
455
|
+
if field_vector:
|
|
456
|
+
vector_fields.append(field_name)
|
|
457
|
+
|
|
458
|
+
# Set up vector search configuration
|
|
459
|
+
algorithm_config = None
|
|
460
|
+
if algorithm_kind.lower() == "hnsw":
|
|
461
|
+
algorithm_config = HnswAlgorithmConfiguration(
|
|
462
|
+
name="hnsw-config",
|
|
463
|
+
parameters={"m": 4, "efConstruction": 400, "efSearch": 500},
|
|
464
|
+
)
|
|
465
|
+
else:
|
|
466
|
+
algorithm_config = ExhaustiveKnnAlgorithmConfiguration(
|
|
467
|
+
name="exhaustive-config"
|
|
468
|
+
)
|
|
469
|
+
|
|
470
|
+
# Create vector search configuration
|
|
471
|
+
vector_search = VectorSearch(
|
|
472
|
+
algorithms=[algorithm_config],
|
|
473
|
+
profiles=[
|
|
474
|
+
VectorSearchProfile(
|
|
475
|
+
name="vector-profile",
|
|
476
|
+
algorithm_configuration_name=algorithm_config.name,
|
|
477
|
+
)
|
|
478
|
+
],
|
|
479
|
+
)
|
|
480
|
+
|
|
481
|
+
# Create the search index
|
|
482
|
+
index = SearchIndex(
|
|
483
|
+
name=index_name, fields=index_fields, vector_search=vector_search
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
try:
|
|
487
|
+
result = index_client.create_or_update_index(index)
|
|
488
|
+
return {
|
|
489
|
+
"index_name": result.name,
|
|
490
|
+
"vector_fields": vector_fields,
|
|
491
|
+
"vector_dimensions": vector_dimensions,
|
|
492
|
+
"algorithm": algorithm_kind,
|
|
493
|
+
"created": True,
|
|
494
|
+
}
|
|
495
|
+
except Exception as e:
|
|
496
|
+
return {"error": str(e), "created": False}
|
flock/core/tools/basic_tools.py
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
"""This module contains basic agentic tools for performing various tasks."""
|
|
2
2
|
|
|
3
3
|
import importlib
|
|
4
|
+
import json
|
|
4
5
|
import os
|
|
5
6
|
import re
|
|
6
|
-
from typing import Literal
|
|
7
|
+
from typing import Any, Literal
|
|
7
8
|
|
|
8
9
|
from flock.core.interpreter.python_interpreter import PythonInterpreter
|
|
9
10
|
from flock.core.logging.trace_and_logged import traced_and_logged
|
|
@@ -201,8 +202,6 @@ def extract_numbers(text: str) -> list[float]:
|
|
|
201
202
|
|
|
202
203
|
@traced_and_logged
|
|
203
204
|
def json_parse_safe(text: str) -> dict:
|
|
204
|
-
import json
|
|
205
|
-
|
|
206
205
|
try:
|
|
207
206
|
result = json.loads(text)
|
|
208
207
|
return result
|
|
@@ -221,9 +220,98 @@ def save_to_file(content: str, filename: str):
|
|
|
221
220
|
|
|
222
221
|
@traced_and_logged
|
|
223
222
|
def read_from_file(filename: str) -> str:
|
|
223
|
+
with open(filename, encoding="utf-8") as file:
|
|
224
|
+
return file.read()
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
@traced_and_logged
|
|
228
|
+
def json_search(
|
|
229
|
+
json_file_path: str, search_query: str, case_sensitive: bool = False
|
|
230
|
+
) -> list:
|
|
231
|
+
"""Search a JSON file for objects containing the specified search query.
|
|
232
|
+
|
|
233
|
+
Args:
|
|
234
|
+
json_file_path (str): Path to the JSON file to search
|
|
235
|
+
search_query (str): Text to search for within the JSON objects
|
|
236
|
+
case_sensitive (bool, optional): Whether to perform a case-sensitive search. Defaults to False.
|
|
237
|
+
|
|
238
|
+
Returns:
|
|
239
|
+
list: List of JSON objects (as dicts) that contain the search query
|
|
240
|
+
|
|
241
|
+
Example:
|
|
242
|
+
>>> matching_tickets = json_search("tickets.json", "error 404")
|
|
243
|
+
>>> print(
|
|
244
|
+
... f"Found {len(matching_tickets)} tickets mentioning '404 error'"
|
|
245
|
+
... )
|
|
246
|
+
"""
|
|
224
247
|
try:
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
248
|
+
# Read the JSON file
|
|
249
|
+
file_content = read_from_file(json_file_path)
|
|
250
|
+
|
|
251
|
+
# Parse the JSON content
|
|
252
|
+
json_data = json_parse_safe(file_content)
|
|
253
|
+
|
|
254
|
+
# Convert search query to lowercase if case-insensitive search
|
|
255
|
+
if not case_sensitive:
|
|
256
|
+
search_query = search_query.lower()
|
|
257
|
+
|
|
258
|
+
results = []
|
|
259
|
+
|
|
260
|
+
# Determine if the JSON root is an object or array
|
|
261
|
+
if isinstance(json_data, dict):
|
|
262
|
+
# Handle case where root is a dictionary object
|
|
263
|
+
for key, value in json_data.items():
|
|
264
|
+
if isinstance(value, list):
|
|
265
|
+
# If this key contains a list of objects, search within them
|
|
266
|
+
matching_items = _search_in_list(
|
|
267
|
+
value, search_query, case_sensitive
|
|
268
|
+
)
|
|
269
|
+
results.extend(matching_items)
|
|
270
|
+
elif _contains_text(value, search_query, case_sensitive):
|
|
271
|
+
# The entire object matches
|
|
272
|
+
results.append(json_data)
|
|
273
|
+
break
|
|
274
|
+
elif isinstance(json_data, list):
|
|
275
|
+
# Handle case where root is an array
|
|
276
|
+
matching_items = _search_in_list(
|
|
277
|
+
json_data, search_query, case_sensitive
|
|
278
|
+
)
|
|
279
|
+
results.extend(matching_items)
|
|
280
|
+
|
|
281
|
+
return results
|
|
282
|
+
|
|
283
|
+
except Exception as e:
|
|
284
|
+
return [{"error": f"Error searching JSON file: {e!s}"}]
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def _search_in_list(
|
|
288
|
+
items: list, search_query: str, case_sensitive: bool
|
|
289
|
+
) -> list:
|
|
290
|
+
"""Helper function to search for text in a list of items."""
|
|
291
|
+
matching_items = []
|
|
292
|
+
for item in items:
|
|
293
|
+
if _contains_text(item, search_query, case_sensitive):
|
|
294
|
+
matching_items.append(item)
|
|
295
|
+
return matching_items
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _contains_text(obj: Any, search_query: str, case_sensitive: bool) -> bool:
|
|
299
|
+
"""Recursively check if an object contains the search query in any of its string values."""
|
|
300
|
+
if isinstance(obj, str):
|
|
301
|
+
# For string values, check if they contain the search query
|
|
302
|
+
if case_sensitive:
|
|
303
|
+
return search_query in obj
|
|
304
|
+
else:
|
|
305
|
+
return search_query in obj.lower()
|
|
306
|
+
elif isinstance(obj, dict):
|
|
307
|
+
# For dictionaries, check each value
|
|
308
|
+
for value in obj.values():
|
|
309
|
+
if _contains_text(value, search_query, case_sensitive):
|
|
310
|
+
return True
|
|
311
|
+
elif isinstance(obj, list):
|
|
312
|
+
# For lists, check each item
|
|
313
|
+
for item in obj:
|
|
314
|
+
if _contains_text(item, search_query, case_sensitive):
|
|
315
|
+
return True
|
|
316
|
+
# For other types (numbers, booleans, None), return False
|
|
317
|
+
return False
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: flock-core
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.18
|
|
4
4
|
Summary: Declarative LLM Orchestration at Scale
|
|
5
5
|
Author-email: Andre Ratzenberger <andre.ratzenberger@whiteduck.de>
|
|
6
6
|
License-File: LICENSE
|
|
@@ -8,6 +8,7 @@ Classifier: License :: OSI Approved :: MIT License
|
|
|
8
8
|
Classifier: Operating System :: OS Independent
|
|
9
9
|
Classifier: Programming Language :: Python :: 3
|
|
10
10
|
Requires-Python: >=3.10
|
|
11
|
+
Requires-Dist: azure-search-documents>=11.5.2
|
|
11
12
|
Requires-Dist: chromadb>=0.6.3
|
|
12
13
|
Requires-Dist: cloudpickle>=3.1.1
|
|
13
14
|
Requires-Dist: devtools>=0.12.2
|
|
@@ -29,6 +30,7 @@ Requires-Dist: prometheus-client>=0.21.1
|
|
|
29
30
|
Requires-Dist: pydantic>=2.10.5
|
|
30
31
|
Requires-Dist: python-box>=7.3.2
|
|
31
32
|
Requires-Dist: python-decouple>=3.8
|
|
33
|
+
Requires-Dist: python-dotenv>=1.0.1
|
|
32
34
|
Requires-Dist: questionary>=2.1.0
|
|
33
35
|
Requires-Dist: rich>=13.9.4
|
|
34
36
|
Requires-Dist: sentence-transformers>=3.4.1
|
|
@@ -8,7 +8,7 @@ flock/cli/load_examples.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
|
|
|
8
8
|
flock/cli/load_flock.py,sha256=3JdECvt5X7uyOG2vZS3-Zk5C5SI_84_QZjcsB3oJmfA,932
|
|
9
9
|
flock/cli/load_release_notes.py,sha256=qFcgUrMddAE_TP6x1P-6ZywTUjTknfhTDW5LTxtg1yk,599
|
|
10
10
|
flock/cli/settings.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
|
|
11
|
-
flock/cli/assets/release_notes.md,sha256
|
|
11
|
+
flock/cli/assets/release_notes.md,sha256=bqnk50jxM3w5uY44Dc7MkdT8XmRREFxrVBAG9XCOSSU,4896
|
|
12
12
|
flock/core/__init__.py,sha256=mPlvKc0SxC2qCvSlgYeP_7EyV8ptmdn24NO8mlQoCSo,559
|
|
13
13
|
flock/core/flock.py,sha256=IURlcuNvdsnqKkvgXtX4v_pGWQ8Lfb60X--MT0zvxHo,19881
|
|
14
14
|
flock/core/flock_agent.py,sha256=QPyRSa1X_aAK2MSgqLNHBiL-_cnYHOSSnrFup2YTzss,12509
|
|
@@ -40,7 +40,8 @@ flock/core/mixin/prompt_parser.py,sha256=eOqI-FK3y17gVqpc_y5GF-WmK1Jv8mFlkZxTcgw
|
|
|
40
40
|
flock/core/registry/agent_registry.py,sha256=TUClh9e3eA6YzZC1CMTlsTPvQeqb9jYHewi-zPpcWM8,4987
|
|
41
41
|
flock/core/serialization/secure_serializer.py,sha256=n5-zRvvXddgJv1FFHsaQ2wuYdL3WUSGPvG_LGaffEJo,6144
|
|
42
42
|
flock/core/serialization/serializable.py,sha256=SymJ0YrjBx48mOBItYSqoRpKuzIc4vKWRS6ScTzre7s,2573
|
|
43
|
-
flock/core/tools/
|
|
43
|
+
flock/core/tools/azure_tools.py,sha256=9Bi6IrB5pzBTBhBSxpCVMgx8HBud8nl4gDp8aN0NT6c,17031
|
|
44
|
+
flock/core/tools/basic_tools.py,sha256=hEG14jNZ2itVvubCHTfsWkuJK6yuNwBtuFj2Js0VHZs,9043
|
|
44
45
|
flock/core/tools/llm_tools.py,sha256=Bdt4Dpur5dGpxd2KFEQyxjfZazvW1HCDKY6ydMj6UgQ,21811
|
|
45
46
|
flock/core/tools/markdown_tools.py,sha256=W6fGM48yGHbifVlaOk1jOtVcybfRbRmf20VbDOZv8S4,6031
|
|
46
47
|
flock/core/tools/dev_tools/github.py,sha256=a2OTPXS7kWOVA4zrZHynQDcsmEi4Pac5MfSjQOLePzA,5308
|
|
@@ -411,8 +412,8 @@ flock/workflow/activities.py,sha256=yah-lHjMW6_Ww1gt7hMXBis1cJRlcbHx0uLsMB9oNZ0,
|
|
|
411
412
|
flock/workflow/agent_activities.py,sha256=NhBZscflEf2IMfSRa_pBM_TRP7uVEF_O0ROvWZ33eDc,963
|
|
412
413
|
flock/workflow/temporal_setup.py,sha256=VWBgmBgfTBjwM5ruS_dVpA5AVxx6EZ7oFPGw4j3m0l0,1091
|
|
413
414
|
flock/workflow/workflow.py,sha256=I9MryXW_bqYVTHx-nl2epbTqeRy27CAWHHA7ZZA0nAk,1696
|
|
414
|
-
flock_core-0.3.
|
|
415
|
-
flock_core-0.3.
|
|
416
|
-
flock_core-0.3.
|
|
417
|
-
flock_core-0.3.
|
|
418
|
-
flock_core-0.3.
|
|
415
|
+
flock_core-0.3.18.dist-info/METADATA,sha256=NiwExibjtEysHFQNtBSV0KMH36KFkeU6WFh0VEohXe4,20584
|
|
416
|
+
flock_core-0.3.18.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
417
|
+
flock_core-0.3.18.dist-info/entry_points.txt,sha256=rWaS5KSpkTmWySURGFZk6PhbJ87TmvcFQDi2uzjlagQ,37
|
|
418
|
+
flock_core-0.3.18.dist-info/licenses/LICENSE,sha256=iYEqWy0wjULzM9GAERaybP4LBiPeu7Z1NEliLUdJKSc,1072
|
|
419
|
+
flock_core-0.3.18.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|