cocoindex 0.1.56__cp313-cp313t-manylinux_2_28_aarch64.whl → 0.1.57__cp313-cp313t-manylinux_2_28_aarch64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cocoindex/__init__.py +5 -2
- cocoindex/_engine.cpython-313t-aarch64-linux-gnu.so +0 -0
- cocoindex/cli.py +126 -72
- cocoindex/flow.py +72 -3
- cocoindex/functions.py +24 -7
- cocoindex/setup.py +78 -12
- {cocoindex-0.1.56.dist-info → cocoindex-0.1.57.dist-info}/METADATA +4 -2
- {cocoindex-0.1.56.dist-info → cocoindex-0.1.57.dist-info}/RECORD +11 -11
- {cocoindex-0.1.56.dist-info → cocoindex-0.1.57.dist-info}/WHEEL +0 -0
- {cocoindex-0.1.56.dist-info → cocoindex-0.1.57.dist-info}/entry_points.txt +0 -0
- {cocoindex-0.1.56.dist-info → cocoindex-0.1.57.dist-info}/licenses/LICENSE +0 -0
cocoindex/__init__.py
CHANGED
@@ -10,7 +10,8 @@ from .auth_registry import AuthEntryReference, add_auth_entry, ref_auth_entry
|
|
10
10
|
from .flow import FlowBuilder, DataScope, DataSlice, Flow, transform_flow
|
11
11
|
from .flow import flow_def
|
12
12
|
from .flow import EvaluateAndDumpOptions, GeneratedField
|
13
|
-
from .flow import
|
13
|
+
from .flow import FlowLiveUpdater, FlowLiveUpdaterOptions
|
14
|
+
from .flow import update_all_flows_async, setup_all_flows, drop_all_flows
|
14
15
|
from .lib import init, start_server, stop, main_fn
|
15
16
|
from .llm import LlmSpec, LlmApiType
|
16
17
|
from .index import VectorSimilarityMetric, VectorIndexDef, IndexOptions
|
@@ -40,9 +41,11 @@ __all__ = [
|
|
40
41
|
"flow_def",
|
41
42
|
"EvaluateAndDumpOptions",
|
42
43
|
"GeneratedField",
|
43
|
-
"update_all_flows_async",
|
44
44
|
"FlowLiveUpdater",
|
45
45
|
"FlowLiveUpdaterOptions",
|
46
|
+
"update_all_flows_async",
|
47
|
+
"setup_all_flows",
|
48
|
+
"drop_all_flows",
|
46
49
|
# Lib
|
47
50
|
"init",
|
48
51
|
"start_server",
|
Binary file
|
cocoindex/cli.py
CHANGED
@@ -7,7 +7,7 @@ import sys
|
|
7
7
|
import threading
|
8
8
|
import types
|
9
9
|
from types import FrameType
|
10
|
-
from typing import Any
|
10
|
+
from typing import Any, Iterable
|
11
11
|
|
12
12
|
import click
|
13
13
|
import watchfiles
|
@@ -17,7 +17,7 @@ from rich.panel import Panel
|
|
17
17
|
from rich.table import Table
|
18
18
|
|
19
19
|
from . import flow, lib, setting
|
20
|
-
from .setup import
|
20
|
+
from .setup import flow_names_with_setup
|
21
21
|
|
22
22
|
# Create ServerSettings lazily upon first call, as environment variables may be loaded from files, etc.
|
23
23
|
COCOINDEX_HOST = "https://cocoindex.io"
|
@@ -146,7 +146,7 @@ def cli(env_file: str | None = None) -> None:
|
|
146
146
|
|
147
147
|
if load_dotenv(dotenv_path=dotenv_path):
|
148
148
|
loaded_env_path = os.path.abspath(dotenv_path)
|
149
|
-
click.echo(f"Loaded environment variables from: {loaded_env_path}", err=True)
|
149
|
+
click.echo(f"Loaded environment variables from: {loaded_env_path}\n", err=True)
|
150
150
|
|
151
151
|
try:
|
152
152
|
_initialize_cocoindex_in_process()
|
@@ -243,6 +243,30 @@ def show(app_flow_specifier: str, color: bool, verbose: bool) -> None:
|
|
243
243
|
console.print(table)
|
244
244
|
|
245
245
|
|
246
|
+
def _setup_flows(
|
247
|
+
flow_iter: Iterable[flow.Flow],
|
248
|
+
*,
|
249
|
+
force: bool,
|
250
|
+
quiet: bool = False,
|
251
|
+
always_show_setup: bool = False,
|
252
|
+
) -> None:
|
253
|
+
setup_bundle = flow.make_setup_bundle(flow_iter)
|
254
|
+
description, is_up_to_date = setup_bundle.describe()
|
255
|
+
if always_show_setup or not is_up_to_date:
|
256
|
+
click.echo(description)
|
257
|
+
if is_up_to_date:
|
258
|
+
if not quiet:
|
259
|
+
click.echo("Setup is already up to date.")
|
260
|
+
return
|
261
|
+
if not force and not click.confirm(
|
262
|
+
"Changes need to be pushed. Continue? [yes/N]",
|
263
|
+
default=False,
|
264
|
+
show_default=False,
|
265
|
+
):
|
266
|
+
return
|
267
|
+
setup_bundle.apply(report_to_stdout=not quiet)
|
268
|
+
|
269
|
+
|
246
270
|
@cli.command()
|
247
271
|
@click.argument("app_target", type=str)
|
248
272
|
@click.option(
|
@@ -261,35 +285,12 @@ def setup(app_target: str, force: bool) -> None:
|
|
261
285
|
"""
|
262
286
|
app_ref = _get_app_ref_from_specifier(app_target)
|
263
287
|
_load_user_app(app_ref)
|
264
|
-
|
265
|
-
setup_status = sync_setup()
|
266
|
-
click.echo(setup_status)
|
267
|
-
if setup_status.is_up_to_date():
|
268
|
-
click.echo("No changes need to be pushed.")
|
269
|
-
return
|
270
|
-
if not force and not click.confirm(
|
271
|
-
"Changes need to be pushed. Continue? [yes/N]",
|
272
|
-
default=False,
|
273
|
-
show_default=False,
|
274
|
-
):
|
275
|
-
return
|
276
|
-
apply_setup_changes(setup_status)
|
288
|
+
_setup_flows(flow.flows().values(), force=force, always_show_setup=True)
|
277
289
|
|
278
290
|
|
279
291
|
@cli.command("drop")
|
280
292
|
@click.argument("app_target", type=str, required=False)
|
281
293
|
@click.argument("flow_name", type=str, nargs=-1)
|
282
|
-
@click.option(
|
283
|
-
"-a",
|
284
|
-
"--all",
|
285
|
-
"drop_all",
|
286
|
-
is_flag=True,
|
287
|
-
show_default=True,
|
288
|
-
default=False,
|
289
|
-
help="Drop the backend setup for all flows with persisted setup, "
|
290
|
-
"even if not defined in the current process."
|
291
|
-
"If used, APP_TARGET and any listed flow names are ignored.",
|
292
|
-
)
|
293
294
|
@click.option(
|
294
295
|
"-f",
|
295
296
|
"--force",
|
@@ -298,69 +299,64 @@ def setup(app_target: str, force: bool) -> None:
|
|
298
299
|
default=False,
|
299
300
|
help="Force drop without confirmation prompts.",
|
300
301
|
)
|
301
|
-
def drop(
|
302
|
-
app_target: str | None, flow_name: tuple[str, ...], drop_all: bool, force: bool
|
303
|
-
) -> None:
|
302
|
+
def drop(app_target: str | None, flow_name: tuple[str, ...], force: bool) -> None:
|
304
303
|
"""
|
305
304
|
Drop the backend setup for flows.
|
306
305
|
|
307
306
|
\b
|
308
307
|
Modes of operation:
|
309
|
-
1. Drop
|
310
|
-
2. Drop
|
311
|
-
3. Drop specific named flows: `cocoindex drop <APP_TARGET> [FLOW_NAME...]`
|
308
|
+
1. Drop all flows defined in an app: `cocoindex drop <APP_TARGET>`
|
309
|
+
2. Drop specific named flows: `cocoindex drop <APP_TARGET> [FLOW_NAME...]`
|
312
310
|
"""
|
313
311
|
app_ref = None
|
314
|
-
flow_names = []
|
315
312
|
|
316
|
-
if
|
317
|
-
if app_target or flow_name:
|
318
|
-
click.echo(
|
319
|
-
"Warning: When --all is used, APP_TARGET and any individual flow names are ignored.",
|
320
|
-
err=True,
|
321
|
-
)
|
322
|
-
flow_names = flow_names_with_setup()
|
323
|
-
elif app_target:
|
324
|
-
app_ref = _get_app_ref_from_specifier(app_target)
|
325
|
-
_load_user_app(app_ref)
|
326
|
-
if flow_name:
|
327
|
-
flow_names = list(flow_name)
|
328
|
-
click.echo(
|
329
|
-
f"Preparing to drop specified flows: {', '.join(flow_names)} (in '{app_ref}').",
|
330
|
-
err=True,
|
331
|
-
)
|
332
|
-
else:
|
333
|
-
flow_names = flow.flow_names()
|
334
|
-
if not flow_names:
|
335
|
-
click.echo(f"No flows found defined in '{app_ref}' to drop.")
|
336
|
-
return
|
337
|
-
click.echo(
|
338
|
-
f"Preparing to drop all flows defined in '{app_ref}': {', '.join(flow_names)}.",
|
339
|
-
err=True,
|
340
|
-
)
|
341
|
-
else:
|
313
|
+
if not app_target:
|
342
314
|
raise click.UsageError(
|
343
315
|
"Missing arguments. You must either provide an APP_TARGET (to target app-specific flows) "
|
344
316
|
"or use the --all flag."
|
345
317
|
)
|
346
318
|
|
347
|
-
|
319
|
+
app_ref = _get_app_ref_from_specifier(app_target)
|
320
|
+
_load_user_app(app_ref)
|
321
|
+
|
322
|
+
flows: Iterable[flow.Flow]
|
323
|
+
if flow_name:
|
324
|
+
flows = []
|
325
|
+
for name in flow_name:
|
326
|
+
try:
|
327
|
+
flows.append(flow.flow_by_name(name))
|
328
|
+
except KeyError:
|
329
|
+
click.echo(
|
330
|
+
f"Warning: Failed to get flow `{name}`. Ignored.",
|
331
|
+
err=True,
|
332
|
+
)
|
333
|
+
else:
|
334
|
+
flows = flow.flows().values()
|
335
|
+
|
336
|
+
flow_full_names = ", ".join(fl.full_name for fl in flows)
|
337
|
+
click.echo(
|
338
|
+
f"Preparing to drop specified flows: {flow_full_names} (in '{app_ref}').",
|
339
|
+
err=True,
|
340
|
+
)
|
341
|
+
|
342
|
+
if not flows:
|
348
343
|
click.echo("No flows identified for the drop operation.")
|
349
344
|
return
|
350
345
|
|
351
|
-
|
352
|
-
|
353
|
-
|
346
|
+
setup_bundle = flow.make_drop_bundle(flows)
|
347
|
+
description, is_up_to_date = setup_bundle.describe()
|
348
|
+
click.echo(description)
|
349
|
+
if is_up_to_date:
|
354
350
|
click.echo("No flows need to be dropped.")
|
355
351
|
return
|
356
352
|
if not force and not click.confirm(
|
357
|
-
f"\nThis will apply changes to drop setup for: {
|
353
|
+
f"\nThis will apply changes to drop setup for: {flow_full_names}. Continue? [yes/N]",
|
358
354
|
default=False,
|
359
355
|
show_default=False,
|
360
356
|
):
|
361
357
|
click.echo("Drop operation aborted by user.")
|
362
358
|
return
|
363
|
-
|
359
|
+
setup_bundle.apply(report_to_stdout=True)
|
364
360
|
|
365
361
|
|
366
362
|
@cli.command()
|
@@ -373,6 +369,21 @@ def drop(
|
|
373
369
|
default=False,
|
374
370
|
help="Continuously watch changes from data sources and apply to the target index.",
|
375
371
|
)
|
372
|
+
@click.option(
|
373
|
+
"--setup",
|
374
|
+
is_flag=True,
|
375
|
+
show_default=True,
|
376
|
+
default=False,
|
377
|
+
help="Automatically setup backends for the flow if it's not setup yet.",
|
378
|
+
)
|
379
|
+
@click.option(
|
380
|
+
"-f",
|
381
|
+
"--force",
|
382
|
+
is_flag=True,
|
383
|
+
show_default=True,
|
384
|
+
default=False,
|
385
|
+
help="Force setup without confirmation prompts.",
|
386
|
+
)
|
376
387
|
@click.option(
|
377
388
|
"-q",
|
378
389
|
"--quiet",
|
@@ -381,21 +392,36 @@ def drop(
|
|
381
392
|
default=False,
|
382
393
|
help="Avoid printing anything to the standard output, e.g. statistics.",
|
383
394
|
)
|
384
|
-
def update(
|
395
|
+
def update(
|
396
|
+
app_flow_specifier: str,
|
397
|
+
live: bool,
|
398
|
+
setup: bool, # pylint: disable=redefined-outer-name
|
399
|
+
force: bool,
|
400
|
+
quiet: bool,
|
401
|
+
) -> Any:
|
385
402
|
"""
|
386
403
|
Update the index to reflect the latest data from data sources.
|
387
404
|
|
388
405
|
APP_FLOW_SPECIFIER: path/to/app.py, module, path/to/app.py:FlowName, or module:FlowName.
|
389
406
|
If :FlowName is omitted, updates all flows.
|
390
407
|
"""
|
391
|
-
app_ref,
|
408
|
+
app_ref, flow_name = _parse_app_flow_specifier(app_flow_specifier)
|
392
409
|
_load_user_app(app_ref)
|
393
410
|
|
394
411
|
options = flow.FlowLiveUpdaterOptions(live_mode=live, print_stats=not quiet)
|
395
|
-
if
|
412
|
+
if flow_name is None:
|
413
|
+
if setup:
|
414
|
+
_setup_flows(
|
415
|
+
flow.flows().values(),
|
416
|
+
force=force,
|
417
|
+
quiet=quiet,
|
418
|
+
)
|
396
419
|
return flow.update_all_flows(options)
|
397
420
|
else:
|
398
|
-
|
421
|
+
fl = flow.flow_by_name(flow_name)
|
422
|
+
if setup:
|
423
|
+
_setup_flows((fl,), force=force, quiet=quiet)
|
424
|
+
with flow.FlowLiveUpdater(fl, options) as updater:
|
399
425
|
updater.wait()
|
400
426
|
return updater.update_stats()
|
401
427
|
|
@@ -485,6 +511,21 @@ def evaluate(
|
|
485
511
|
default=False,
|
486
512
|
help="Continuously watch changes from data sources and apply to the target index.",
|
487
513
|
)
|
514
|
+
@click.option(
|
515
|
+
"--setup",
|
516
|
+
is_flag=True,
|
517
|
+
show_default=True,
|
518
|
+
default=False,
|
519
|
+
help="Automatically setup backends for the flow if it's not setup yet.",
|
520
|
+
)
|
521
|
+
@click.option(
|
522
|
+
"-f",
|
523
|
+
"--force",
|
524
|
+
is_flag=True,
|
525
|
+
show_default=True,
|
526
|
+
default=False,
|
527
|
+
help="Force setup without confirmation prompts.",
|
528
|
+
)
|
488
529
|
@click.option(
|
489
530
|
"-q",
|
490
531
|
"--quiet",
|
@@ -505,6 +546,8 @@ def server(
|
|
505
546
|
app_target: str,
|
506
547
|
address: str | None,
|
507
548
|
live_update: bool,
|
549
|
+
setup: bool, # pylint: disable=redefined-outer-name
|
550
|
+
force: bool,
|
508
551
|
quiet: bool,
|
509
552
|
cors_origin: str | None,
|
510
553
|
cors_cocoindex: bool,
|
@@ -526,6 +569,8 @@ def server(
|
|
526
569
|
cors_cocoindex,
|
527
570
|
cors_local,
|
528
571
|
live_update,
|
572
|
+
setup,
|
573
|
+
force,
|
529
574
|
quiet,
|
530
575
|
)
|
531
576
|
|
@@ -568,6 +613,8 @@ def _run_server(
|
|
568
613
|
cors_cocoindex: bool = False,
|
569
614
|
cors_local: int | None = None,
|
570
615
|
live_update: bool = False,
|
616
|
+
run_setup: bool = False,
|
617
|
+
force: bool = False,
|
571
618
|
quiet: bool = False,
|
572
619
|
) -> None:
|
573
620
|
"""Helper function to run the server with specified settings."""
|
@@ -586,15 +633,22 @@ def _run_server(
|
|
586
633
|
if address is not None:
|
587
634
|
server_settings.address = address
|
588
635
|
|
589
|
-
lib.start_server(server_settings)
|
590
|
-
|
591
636
|
if COCOINDEX_HOST in cors_origins:
|
592
637
|
click.echo(f"Open CocoInsight at: {COCOINDEX_HOST}/cocoinsight")
|
593
638
|
|
639
|
+
if run_setup:
|
640
|
+
_setup_flows(
|
641
|
+
flow.flows().values(),
|
642
|
+
force=force,
|
643
|
+
quiet=quiet,
|
644
|
+
)
|
645
|
+
|
594
646
|
if live_update:
|
595
647
|
options = flow.FlowLiveUpdaterOptions(live_mode=True, print_stats=not quiet)
|
596
648
|
flow.update_all_flows(options)
|
597
649
|
|
650
|
+
lib.start_server(server_settings)
|
651
|
+
|
598
652
|
click.secho("Press Ctrl+C to stop the server.", fg="yellow")
|
599
653
|
|
600
654
|
shutdown_event = threading.Event()
|
cocoindex/flow.py
CHANGED
@@ -9,7 +9,6 @@ import re
|
|
9
9
|
import inspect
|
10
10
|
import datetime
|
11
11
|
import functools
|
12
|
-
|
13
12
|
from typing import (
|
14
13
|
Any,
|
15
14
|
Callable,
|
@@ -20,6 +19,7 @@ from typing import (
|
|
20
19
|
get_origin,
|
21
20
|
NamedTuple,
|
22
21
|
cast,
|
22
|
+
Iterable,
|
23
23
|
)
|
24
24
|
from threading import Lock
|
25
25
|
from enum import Enum
|
@@ -34,6 +34,7 @@ from . import setting
|
|
34
34
|
from .convert import dump_engine_object, encode_engine_value, make_engine_value_decoder
|
35
35
|
from .typing import encode_enriched_type
|
36
36
|
from .runtime import execution_context
|
37
|
+
from .setup import SetupChangeBundle
|
37
38
|
|
38
39
|
|
39
40
|
class _NameBuilder:
|
@@ -657,6 +658,34 @@ class Flow:
|
|
657
658
|
"""
|
658
659
|
return await asyncio.to_thread(self.internal_flow)
|
659
660
|
|
661
|
+
def setup(self, report_to_stdout: bool = False) -> None:
|
662
|
+
"""
|
663
|
+
Setup the flow.
|
664
|
+
"""
|
665
|
+
execution_context.run(self.setup_async(report_to_stdout=report_to_stdout))
|
666
|
+
|
667
|
+
async def setup_async(self, report_to_stdout: bool = False) -> None:
|
668
|
+
"""
|
669
|
+
Setup the flow. The async version.
|
670
|
+
"""
|
671
|
+
await make_setup_bundle([self]).describe_and_apply_async(
|
672
|
+
report_to_stdout=report_to_stdout
|
673
|
+
)
|
674
|
+
|
675
|
+
def drop(self, report_to_stdout: bool = False) -> None:
|
676
|
+
"""
|
677
|
+
Drop the flow.
|
678
|
+
"""
|
679
|
+
execution_context.run(self.drop_async(report_to_stdout=report_to_stdout))
|
680
|
+
|
681
|
+
async def drop_async(self, report_to_stdout: bool = False) -> None:
|
682
|
+
"""
|
683
|
+
Drop the flow. The async version.
|
684
|
+
"""
|
685
|
+
await make_drop_bundle([self]).describe_and_apply_async(
|
686
|
+
report_to_stdout=report_to_stdout
|
687
|
+
)
|
688
|
+
|
660
689
|
|
661
690
|
def _create_lazy_flow(
|
662
691
|
name: str | None, fl_def: Callable[[FlowBuilder, DataScope], None]
|
@@ -666,7 +695,7 @@ def _create_lazy_flow(
|
|
666
695
|
The flow will be built the first time when it's really needed.
|
667
696
|
"""
|
668
697
|
flow_name = _flow_name_builder.build_name(name, prefix="_flow_")
|
669
|
-
flow_full_name =
|
698
|
+
flow_full_name = get_flow_full_name(flow_name)
|
670
699
|
|
671
700
|
def _create_engine_flow() -> _engine.Flow:
|
672
701
|
flow_builder_state = _FlowBuilderState(flow_full_name)
|
@@ -685,7 +714,7 @@ _flows_lock = Lock()
|
|
685
714
|
_flows: dict[str, Flow] = {}
|
686
715
|
|
687
716
|
|
688
|
-
def
|
717
|
+
def get_flow_full_name(name: str) -> str:
|
689
718
|
"""
|
690
719
|
Get the full name of a flow.
|
691
720
|
"""
|
@@ -967,3 +996,43 @@ def transform_flow() -> Callable[[Callable[..., DataSlice[T]]], TransformFlow[T]
|
|
967
996
|
return _transform_flow
|
968
997
|
|
969
998
|
return _transform_flow_wrapper
|
999
|
+
|
1000
|
+
|
1001
|
+
def make_setup_bundle(flow_iter: Iterable[Flow]) -> SetupChangeBundle:
|
1002
|
+
"""
|
1003
|
+
Make a bundle to setup flows with the given names.
|
1004
|
+
"""
|
1005
|
+
full_names = []
|
1006
|
+
for fl in flow_iter:
|
1007
|
+
fl.internal_flow()
|
1008
|
+
full_names.append(fl.full_name)
|
1009
|
+
return SetupChangeBundle(_engine.make_setup_bundle(full_names))
|
1010
|
+
|
1011
|
+
|
1012
|
+
def make_drop_bundle(flow_iter: Iterable[Flow]) -> SetupChangeBundle:
|
1013
|
+
"""
|
1014
|
+
Make a bundle to drop flows with the given names.
|
1015
|
+
"""
|
1016
|
+
full_names = []
|
1017
|
+
for fl in flow_iter:
|
1018
|
+
fl.internal_flow()
|
1019
|
+
full_names.append(fl.full_name)
|
1020
|
+
return SetupChangeBundle(_engine.make_drop_bundle(full_names))
|
1021
|
+
|
1022
|
+
|
1023
|
+
def setup_all_flows(report_to_stdout: bool = False) -> None:
|
1024
|
+
"""
|
1025
|
+
Setup all flows registered in the current process.
|
1026
|
+
"""
|
1027
|
+
with _flows_lock:
|
1028
|
+
flow_list = list(_flows.values())
|
1029
|
+
make_setup_bundle(flow_list).describe_and_apply(report_to_stdout=report_to_stdout)
|
1030
|
+
|
1031
|
+
|
1032
|
+
def drop_all_flows(report_to_stdout: bool = False) -> None:
|
1033
|
+
"""
|
1034
|
+
Drop all flows registered in the current process.
|
1035
|
+
"""
|
1036
|
+
with _flows_lock:
|
1037
|
+
flow_list = list(_flows.values())
|
1038
|
+
make_drop_bundle(flow_list).describe_and_apply(report_to_stdout=report_to_stdout)
|
cocoindex/functions.py
CHANGED
@@ -1,16 +1,21 @@
|
|
1
1
|
"""All builtin functions."""
|
2
2
|
|
3
|
-
|
3
|
+
import dataclasses
|
4
|
+
from typing import Annotated, Any, Literal
|
5
|
+
|
4
6
|
import numpy as np
|
5
7
|
from numpy.typing import NDArray
|
6
|
-
import dataclasses
|
7
8
|
|
8
|
-
from .
|
9
|
-
from . import
|
9
|
+
from . import llm, op
|
10
|
+
from .typing import TypeAttr, Vector
|
10
11
|
|
11
|
-
#
|
12
|
-
|
13
|
-
import sentence_transformers
|
12
|
+
# Check if sentence_transformers is available
|
13
|
+
try:
|
14
|
+
import sentence_transformers # type: ignore
|
15
|
+
|
16
|
+
_SENTENCE_TRANSFORMERS_AVAILABLE = True
|
17
|
+
except ImportError:
|
18
|
+
_SENTENCE_TRANSFORMERS_AVAILABLE = False
|
14
19
|
|
15
20
|
|
16
21
|
class ParseJson(op.FunctionSpec):
|
@@ -58,6 +63,10 @@ class SentenceTransformerEmbed(op.FunctionSpec):
|
|
58
63
|
|
59
64
|
model: The name of the SentenceTransformer model to use.
|
60
65
|
args: Additional arguments to pass to the SentenceTransformer constructor. e.g. {"trust_remote_code": True}
|
66
|
+
|
67
|
+
Note:
|
68
|
+
This function requires the optional sentence-transformers dependency.
|
69
|
+
Install it with: pip install 'cocoindex[embeddings]'
|
61
70
|
"""
|
62
71
|
|
63
72
|
model: str
|
@@ -72,6 +81,14 @@ class SentenceTransformerEmbedExecutor:
|
|
72
81
|
_model: "sentence_transformers.SentenceTransformer"
|
73
82
|
|
74
83
|
def analyze(self, text: Any) -> type:
|
84
|
+
if not _SENTENCE_TRANSFORMERS_AVAILABLE:
|
85
|
+
raise ImportError(
|
86
|
+
"sentence_transformers is required for SentenceTransformerEmbed function. "
|
87
|
+
"Install it with one of these commands:\n"
|
88
|
+
" pip install 'cocoindex[embeddings]'\n"
|
89
|
+
" pip install sentence-transformers"
|
90
|
+
)
|
91
|
+
|
75
92
|
import sentence_transformers # pylint: disable=import-outside-toplevel
|
76
93
|
|
77
94
|
args = self.spec.args or {}
|
cocoindex/setup.py
CHANGED
@@ -1,26 +1,92 @@
|
|
1
|
-
|
1
|
+
"""
|
2
|
+
This module provides APIs to manage the setup of flows.
|
3
|
+
"""
|
4
|
+
|
2
5
|
from . import setting
|
3
6
|
from . import _engine # type: ignore
|
7
|
+
from .runtime import execution_context
|
8
|
+
|
9
|
+
|
10
|
+
class SetupChangeBundle:
|
11
|
+
"""
|
12
|
+
This class represents a bundle of setup changes.
|
13
|
+
"""
|
14
|
+
|
15
|
+
_engine_bundle: _engine.SetupChangeBundle
|
16
|
+
|
17
|
+
def __init__(self, _engine_bundle: _engine.SetupChangeBundle):
|
18
|
+
self._engine_bundle = _engine_bundle
|
19
|
+
|
20
|
+
def __str__(self) -> str:
|
21
|
+
desc, _ = execution_context.run(self._engine_bundle.describe_async())
|
22
|
+
return desc # type: ignore
|
23
|
+
|
24
|
+
def __repr__(self) -> str:
|
25
|
+
return self.__str__()
|
4
26
|
|
27
|
+
def apply(self, report_to_stdout: bool = False) -> None:
|
28
|
+
"""
|
29
|
+
Apply the setup changes.
|
30
|
+
"""
|
31
|
+
execution_context.run(self.apply_async(report_to_stdout=report_to_stdout))
|
5
32
|
|
6
|
-
def
|
7
|
-
|
8
|
-
|
33
|
+
async def apply_async(self, report_to_stdout: bool = False) -> None:
|
34
|
+
"""
|
35
|
+
Apply the setup changes. Async version of `apply`.
|
36
|
+
"""
|
37
|
+
await self._engine_bundle.apply_async(report_to_stdout=report_to_stdout)
|
9
38
|
|
39
|
+
def describe(self) -> tuple[str, bool]:
|
40
|
+
"""
|
41
|
+
Describe the setup changes.
|
42
|
+
"""
|
43
|
+
return execution_context.run(self.describe_async()) # type: ignore
|
10
44
|
|
11
|
-
def
|
12
|
-
|
13
|
-
|
45
|
+
async def describe_async(self) -> tuple[str, bool]:
|
46
|
+
"""
|
47
|
+
Describe the setup changes. Async version of `describe`.
|
48
|
+
"""
|
49
|
+
return await self._engine_bundle.describe_async() # type: ignore
|
50
|
+
|
51
|
+
def describe_and_apply(self, report_to_stdout: bool = False) -> None:
|
52
|
+
"""
|
53
|
+
Describe the setup changes and apply them if `report_to_stdout` is True.
|
54
|
+
Silently apply setup changes otherwise.
|
55
|
+
"""
|
56
|
+
execution_context.run(
|
57
|
+
self.describe_and_apply_async(report_to_stdout=report_to_stdout)
|
58
|
+
)
|
59
|
+
|
60
|
+
async def describe_and_apply_async(self, *, report_to_stdout: bool = False) -> None:
|
61
|
+
"""
|
62
|
+
Describe the setup changes and apply them if `report_to_stdout` is True.
|
63
|
+
Silently apply setup changes otherwise. Async version of `describe_and_apply`.
|
64
|
+
"""
|
65
|
+
if report_to_stdout:
|
66
|
+
desc, is_up_to_date = await self.describe_async()
|
67
|
+
print("Setup status:\n")
|
68
|
+
print(desc)
|
69
|
+
if is_up_to_date:
|
70
|
+
print("No setup changes to apply.")
|
71
|
+
return
|
72
|
+
await self.apply_async(report_to_stdout=report_to_stdout)
|
14
73
|
|
15
74
|
|
16
75
|
def flow_names_with_setup() -> list[str]:
|
76
|
+
"""
|
77
|
+
Get the names of all flows that have been setup.
|
78
|
+
"""
|
79
|
+
return execution_context.run(flow_names_with_setup_async()) # type: ignore
|
80
|
+
|
81
|
+
|
82
|
+
async def flow_names_with_setup_async() -> list[str]:
|
83
|
+
"""
|
84
|
+
Get the names of all flows that have been setup. Async version of `flow_names_with_setup`.
|
85
|
+
"""
|
17
86
|
result = []
|
18
|
-
|
87
|
+
all_flow_names = await _engine.flow_names_with_setup_async()
|
88
|
+
for name in all_flow_names:
|
19
89
|
app_namespace, name = setting.split_app_namespace(name, ".")
|
20
90
|
if app_namespace == setting.get_app_namespace():
|
21
91
|
result.append(name)
|
22
92
|
return result
|
23
|
-
|
24
|
-
|
25
|
-
def apply_setup_changes(setup_status: _engine.SetupStatus) -> None:
|
26
|
-
_engine.apply_setup_changes(setup_status)
|
@@ -1,16 +1,18 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: cocoindex
|
3
|
-
Version: 0.1.
|
4
|
-
Requires-Dist: sentence-transformers>=3.3.1
|
3
|
+
Version: 0.1.57
|
5
4
|
Requires-Dist: click>=8.1.8
|
6
5
|
Requires-Dist: rich>=14.0.0
|
7
6
|
Requires-Dist: python-dotenv>=1.1.0
|
8
7
|
Requires-Dist: watchfiles>=1.1.0
|
8
|
+
Requires-Dist: numpy>=1.23.2
|
9
9
|
Requires-Dist: pytest ; extra == 'test'
|
10
10
|
Requires-Dist: ruff ; extra == 'dev'
|
11
11
|
Requires-Dist: pre-commit ; extra == 'dev'
|
12
|
+
Requires-Dist: sentence-transformers>=3.3.1 ; extra == 'embeddings'
|
12
13
|
Provides-Extra: test
|
13
14
|
Provides-Extra: dev
|
15
|
+
Provides-Extra: embeddings
|
14
16
|
License-File: LICENSE
|
15
17
|
Summary: With CocoIndex, users declare the transformation, CocoIndex creates & maintains an index, and keeps the derived index up to date based on source update, with minimal computation and changes.
|
16
18
|
Author-email: CocoIndex <cocoindex.io@gmail.com>
|
@@ -1,14 +1,14 @@
|
|
1
|
-
cocoindex-0.1.
|
2
|
-
cocoindex-0.1.
|
3
|
-
cocoindex-0.1.
|
4
|
-
cocoindex-0.1.
|
5
|
-
cocoindex/__init__.py,sha256=
|
6
|
-
cocoindex/_engine.cpython-313t-aarch64-linux-gnu.so,sha256=
|
1
|
+
cocoindex-0.1.57.dist-info/METADATA,sha256=B506wZaMqr_x4PwQrv-tQZqvkukTH4nkUkreAFAZkIk,10020
|
2
|
+
cocoindex-0.1.57.dist-info/WHEEL,sha256=ToOKKGvSCIumW6I150DmcMX6hkipQtOPVDskIifqrDY,110
|
3
|
+
cocoindex-0.1.57.dist-info/entry_points.txt,sha256=_NretjYVzBdNTn7dK-zgwr7YfG2afz1u1uSE-5bZXF8,46
|
4
|
+
cocoindex-0.1.57.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
5
|
+
cocoindex/__init__.py,sha256=MFm-QJzrr0ODJCsAAsPUzJXh8KH1WdZod8F60B1iUYw,1877
|
6
|
+
cocoindex/_engine.cpython-313t-aarch64-linux-gnu.so,sha256=WmqEQosbqXUkZkKbunHzruJyVJpwDfP7ISx-Wcjd9KI,60956608
|
7
7
|
cocoindex/auth_registry.py,sha256=1XqO7ibjmBBd8i11XSJTvTgdz8p1ptW-ZpuSgo_5zzk,716
|
8
|
-
cocoindex/cli.py,sha256=
|
8
|
+
cocoindex/cli.py,sha256=8bDL-Qmd9NYtn1DsDfvUMk45xfAqNf9YTyM7H9KRuNU,21345
|
9
9
|
cocoindex/convert.py,sha256=XeSr0ykudBrB-RWRmcbdbt3WCihlLDSBIYUcDtPbTdA,10228
|
10
|
-
cocoindex/flow.py,sha256=
|
11
|
-
cocoindex/functions.py,sha256=
|
10
|
+
cocoindex/flow.py,sha256=YUDXhCEolKIMIW_0ba2bwu27-kJIo1_t3ePy_sSkYHw,32191
|
11
|
+
cocoindex/functions.py,sha256=IBwvdPpGR-S5mk53HvHpT2GVs15MI9wQznxgOdxA0ac,3202
|
12
12
|
cocoindex/index.py,sha256=j93B9jEvvLXHtpzKWL88SY6wCGEoPgpsQhEGHlyYGFg,540
|
13
13
|
cocoindex/lib.py,sha256=BeRUn3RqE_wSsVtsgCzbFFKe1LXgRyRmMOcmwWBuEXo,2940
|
14
14
|
cocoindex/llm.py,sha256=hpwvHjWiLkoV018TkC191rztq62NQeewwF3kDsHUols,430
|
@@ -16,7 +16,7 @@ cocoindex/op.py,sha256=Z7V9Fdz4qeTeozzKmp1Dk1lPUP0GBgVgD_vBEhTJS5Y,11810
|
|
16
16
|
cocoindex/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
17
17
|
cocoindex/runtime.py,sha256=bAdHYaXFWiiUWyAgzmKTeaAaRR0D_AmaqVCIdPO-v00,1056
|
18
18
|
cocoindex/setting.py,sha256=Zl8K86r8RVvG9c3pCsH0Ot8BHhDAQAQCjoBp7TnXMLQ,3590
|
19
|
-
cocoindex/setup.py,sha256=
|
19
|
+
cocoindex/setup.py,sha256=7uIHKN4FOCuoidPXcKyGTrkqpkl9luL49-6UcnMxYzw,3068
|
20
20
|
cocoindex/sources.py,sha256=JCnOhv1w4o28e03i7yvo4ESicWYAhckkBg5bQlxNH4U,1330
|
21
21
|
cocoindex/targets.py,sha256=Nfh_tpFd1goTnS_cxBjIs4j9zl3Z4Z1JomAQ1dl3Sic,2796
|
22
22
|
cocoindex/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -25,4 +25,4 @@ cocoindex/tests/test_optional_database.py,sha256=snAmkNa6wtOSaxoZE1HgjvL5v_ylitt
|
|
25
25
|
cocoindex/tests/test_typing.py,sha256=t6UCYShcfonTfjBlGRWPiFGMZ8DGFfABXo6idekPoJE,14757
|
26
26
|
cocoindex/typing.py,sha256=kPMFVKs2i4SCLzW1Tn5NP_Ev9DAc-2qW6eJ68gpLexU,12580
|
27
27
|
cocoindex/utils.py,sha256=hUhX-XV6XGCtJSEIpBOuDv6VvqImwPlgBxztBTw7u0U,598
|
28
|
-
cocoindex-0.1.
|
28
|
+
cocoindex-0.1.57.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|