cocoindex 0.3.4__cp311-abi3-manylinux_2_28_x86_64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cocoindex/__init__.py +114 -0
- cocoindex/_engine.abi3.so +0 -0
- cocoindex/auth_registry.py +44 -0
- cocoindex/cli.py +830 -0
- cocoindex/engine_object.py +214 -0
- cocoindex/engine_value.py +550 -0
- cocoindex/flow.py +1281 -0
- cocoindex/functions/__init__.py +40 -0
- cocoindex/functions/_engine_builtin_specs.py +66 -0
- cocoindex/functions/colpali.py +247 -0
- cocoindex/functions/sbert.py +77 -0
- cocoindex/index.py +50 -0
- cocoindex/lib.py +75 -0
- cocoindex/llm.py +47 -0
- cocoindex/op.py +1047 -0
- cocoindex/py.typed +0 -0
- cocoindex/query_handler.py +57 -0
- cocoindex/runtime.py +78 -0
- cocoindex/setting.py +171 -0
- cocoindex/setup.py +92 -0
- cocoindex/sources/__init__.py +5 -0
- cocoindex/sources/_engine_builtin_specs.py +120 -0
- cocoindex/subprocess_exec.py +277 -0
- cocoindex/targets/__init__.py +5 -0
- cocoindex/targets/_engine_builtin_specs.py +153 -0
- cocoindex/targets/lancedb.py +466 -0
- cocoindex/tests/__init__.py +0 -0
- cocoindex/tests/test_engine_object.py +331 -0
- cocoindex/tests/test_engine_value.py +1724 -0
- cocoindex/tests/test_optional_database.py +249 -0
- cocoindex/tests/test_transform_flow.py +300 -0
- cocoindex/tests/test_typing.py +553 -0
- cocoindex/tests/test_validation.py +134 -0
- cocoindex/typing.py +834 -0
- cocoindex/user_app_loader.py +53 -0
- cocoindex/utils.py +20 -0
- cocoindex/validation.py +104 -0
- cocoindex-0.3.4.dist-info/METADATA +288 -0
- cocoindex-0.3.4.dist-info/RECORD +42 -0
- cocoindex-0.3.4.dist-info/WHEEL +4 -0
- cocoindex-0.3.4.dist-info/entry_points.txt +2 -0
- cocoindex-0.3.4.dist-info/licenses/THIRD_PARTY_NOTICES.html +13249 -0
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import sys
|
|
3
|
+
import importlib
|
|
4
|
+
import types
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Error(Exception):
|
|
8
|
+
"""
|
|
9
|
+
Exception raised when a user app target is invalid or cannot be loaded.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
pass
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def load_user_app(app_target: str) -> types.ModuleType:
|
|
16
|
+
"""
|
|
17
|
+
Loads the user's application, which can be a file path or an installed module name.
|
|
18
|
+
Exits on failure.
|
|
19
|
+
"""
|
|
20
|
+
looks_like_path = os.sep in app_target or app_target.lower().endswith(".py")
|
|
21
|
+
|
|
22
|
+
if looks_like_path:
|
|
23
|
+
if not os.path.isfile(app_target):
|
|
24
|
+
raise Error(f"Application file path not found: {app_target}")
|
|
25
|
+
app_path = os.path.abspath(app_target)
|
|
26
|
+
app_dir = os.path.dirname(app_path)
|
|
27
|
+
module_name = os.path.splitext(os.path.basename(app_path))[0]
|
|
28
|
+
|
|
29
|
+
if app_dir not in sys.path:
|
|
30
|
+
sys.path.insert(0, app_dir)
|
|
31
|
+
try:
|
|
32
|
+
spec = importlib.util.spec_from_file_location(module_name, app_path)
|
|
33
|
+
if spec is None:
|
|
34
|
+
raise ImportError(f"Could not create spec for file: {app_path}")
|
|
35
|
+
module = importlib.util.module_from_spec(spec)
|
|
36
|
+
sys.modules[spec.name] = module
|
|
37
|
+
if spec.loader is None:
|
|
38
|
+
raise ImportError(f"Could not create loader for file: {app_path}")
|
|
39
|
+
spec.loader.exec_module(module)
|
|
40
|
+
return module
|
|
41
|
+
except (ImportError, FileNotFoundError, PermissionError) as e:
|
|
42
|
+
raise Error(f"Failed importing file '{app_path}': {e}") from e
|
|
43
|
+
finally:
|
|
44
|
+
if app_dir in sys.path and sys.path[0] == app_dir:
|
|
45
|
+
sys.path.pop(0)
|
|
46
|
+
|
|
47
|
+
# Try as module
|
|
48
|
+
try:
|
|
49
|
+
return importlib.import_module(app_target)
|
|
50
|
+
except ImportError as e:
|
|
51
|
+
raise Error(f"Failed to load module '{app_target}': {e}") from e
|
|
52
|
+
except Exception as e:
|
|
53
|
+
raise Error(f"Unexpected error importing module '{app_target}': {e}") from e
|
cocoindex/utils.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from .flow import Flow
|
|
2
|
+
from .setting import get_app_namespace
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def get_target_default_name(flow: Flow, target_name: str, delimiter: str = "__") -> str:
|
|
6
|
+
"""
|
|
7
|
+
Get the default name for a target.
|
|
8
|
+
It's used as the underlying target name (e.g. a table, a collection, etc.) followed by most targets, if not explicitly specified.
|
|
9
|
+
"""
|
|
10
|
+
return (
|
|
11
|
+
get_app_namespace(trailing_delimiter=delimiter)
|
|
12
|
+
+ flow.name
|
|
13
|
+
+ delimiter
|
|
14
|
+
+ target_name
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
get_target_storage_default_name = (
|
|
19
|
+
get_target_default_name # Deprecated: Use get_target_default_name instead
|
|
20
|
+
)
|
cocoindex/validation.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Naming validation for CocoIndex identifiers.
|
|
3
|
+
|
|
4
|
+
This module enforces naming conventions for flow names, field names,
|
|
5
|
+
target names, and app namespace names as specified in issue #779.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import re
|
|
9
|
+
from typing import Optional
|
|
10
|
+
|
|
11
|
+
_IDENTIFIER_PATTERN = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$")
|
|
12
|
+
_IDENTIFIER_WITH_DOTS_PATTERN = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_.]*$")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class NamingError(ValueError):
|
|
16
|
+
"""Exception raised for naming convention violations."""
|
|
17
|
+
|
|
18
|
+
pass
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def validate_identifier_name(
|
|
22
|
+
name: str,
|
|
23
|
+
max_length: int = 64,
|
|
24
|
+
allow_dots: bool = False,
|
|
25
|
+
identifier_type: str = "identifier",
|
|
26
|
+
) -> Optional[str]:
|
|
27
|
+
"""
|
|
28
|
+
Validate identifier names according to CocoIndex naming rules.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
name: The name to validate
|
|
32
|
+
max_length: Maximum allowed length (default 64)
|
|
33
|
+
allow_dots: Whether to allow dots in the name (for full flow names)
|
|
34
|
+
identifier_type: Type of identifier for error messages
|
|
35
|
+
|
|
36
|
+
Returns:
|
|
37
|
+
None if valid, error message string if invalid
|
|
38
|
+
"""
|
|
39
|
+
if not name:
|
|
40
|
+
return f"{identifier_type} name cannot be empty"
|
|
41
|
+
|
|
42
|
+
if len(name) > max_length:
|
|
43
|
+
return f"{identifier_type} name '{name}' exceeds maximum length of {max_length} characters"
|
|
44
|
+
|
|
45
|
+
if name.startswith("__"):
|
|
46
|
+
return f"{identifier_type} name '{name}' cannot start with double underscores (reserved for internal usage)"
|
|
47
|
+
|
|
48
|
+
# Define allowed pattern
|
|
49
|
+
if allow_dots:
|
|
50
|
+
pattern = _IDENTIFIER_WITH_DOTS_PATTERN
|
|
51
|
+
allowed_chars = "letters, digits, underscores, and dots"
|
|
52
|
+
else:
|
|
53
|
+
pattern = _IDENTIFIER_PATTERN
|
|
54
|
+
allowed_chars = "letters, digits, and underscores"
|
|
55
|
+
|
|
56
|
+
if not pattern.match(name):
|
|
57
|
+
return f"{identifier_type} name '{name}' must start with a letter or underscore and contain only {allowed_chars}"
|
|
58
|
+
|
|
59
|
+
return None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def validate_field_name(name: str) -> None:
|
|
63
|
+
"""Validate field names."""
|
|
64
|
+
error = validate_identifier_name(
|
|
65
|
+
name, max_length=64, allow_dots=False, identifier_type="Field"
|
|
66
|
+
)
|
|
67
|
+
if error:
|
|
68
|
+
raise NamingError(error)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def validate_flow_name(name: str) -> None:
|
|
72
|
+
"""Validate flow names."""
|
|
73
|
+
error = validate_identifier_name(
|
|
74
|
+
name, max_length=64, allow_dots=False, identifier_type="Flow"
|
|
75
|
+
)
|
|
76
|
+
if error:
|
|
77
|
+
raise NamingError(error)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def validate_full_flow_name(name: str) -> None:
|
|
81
|
+
"""Validate full flow names (can contain dots for namespacing)."""
|
|
82
|
+
error = validate_identifier_name(
|
|
83
|
+
name, max_length=64, allow_dots=True, identifier_type="Full flow"
|
|
84
|
+
)
|
|
85
|
+
if error:
|
|
86
|
+
raise NamingError(error)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def validate_app_namespace_name(name: str) -> None:
|
|
90
|
+
"""Validate app namespace names."""
|
|
91
|
+
error = validate_identifier_name(
|
|
92
|
+
name, max_length=64, allow_dots=False, identifier_type="App namespace"
|
|
93
|
+
)
|
|
94
|
+
if error:
|
|
95
|
+
raise NamingError(error)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def validate_target_name(name: str) -> None:
|
|
99
|
+
"""Validate target names."""
|
|
100
|
+
error = validate_identifier_name(
|
|
101
|
+
name, max_length=64, allow_dots=False, identifier_type="Target"
|
|
102
|
+
)
|
|
103
|
+
if error:
|
|
104
|
+
raise NamingError(error)
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cocoindex
|
|
3
|
+
Version: 0.3.4
|
|
4
|
+
Classifier: Development Status :: 3 - Alpha
|
|
5
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
6
|
+
Classifier: Operating System :: OS Independent
|
|
7
|
+
Classifier: Programming Language :: Rust
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
14
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
15
|
+
Classifier: Topic :: Text Processing :: Indexing
|
|
16
|
+
Classifier: Intended Audience :: Developers
|
|
17
|
+
Classifier: Natural Language :: English
|
|
18
|
+
Classifier: Typing :: Typed
|
|
19
|
+
Requires-Dist: typing-extensions>=4.12 ; python_full_version < '3.13'
|
|
20
|
+
Requires-Dist: click>=8.1.8
|
|
21
|
+
Requires-Dist: rich>=14.0.0
|
|
22
|
+
Requires-Dist: python-dotenv>=1.1.0
|
|
23
|
+
Requires-Dist: watchfiles>=1.1.0
|
|
24
|
+
Requires-Dist: numpy>=1.23.2
|
|
25
|
+
Requires-Dist: psutil>=7.0.0
|
|
26
|
+
Requires-Dist: pytest ; extra == 'dev'
|
|
27
|
+
Requires-Dist: pytest-asyncio ; extra == 'dev'
|
|
28
|
+
Requires-Dist: ruff ; extra == 'dev'
|
|
29
|
+
Requires-Dist: mypy ; extra == 'dev'
|
|
30
|
+
Requires-Dist: pre-commit ; extra == 'dev'
|
|
31
|
+
Requires-Dist: sentence-transformers>=3.3.1 ; extra == 'embeddings'
|
|
32
|
+
Requires-Dist: colpali-engine ; extra == 'colpali'
|
|
33
|
+
Requires-Dist: lancedb>=0.25.0 ; extra == 'lancedb'
|
|
34
|
+
Requires-Dist: pydantic>=2.11.9 ; extra == 'pydantic'
|
|
35
|
+
Requires-Dist: sentence-transformers>=3.3.1 ; extra == 'all'
|
|
36
|
+
Requires-Dist: colpali-engine ; extra == 'all'
|
|
37
|
+
Requires-Dist: lancedb>=0.25.0 ; extra == 'all'
|
|
38
|
+
Requires-Dist: pydantic>=2.11.9 ; extra == 'all'
|
|
39
|
+
Provides-Extra: dev
|
|
40
|
+
Provides-Extra: embeddings
|
|
41
|
+
Provides-Extra: colpali
|
|
42
|
+
Provides-Extra: lancedb
|
|
43
|
+
Provides-Extra: pydantic
|
|
44
|
+
Provides-Extra: all
|
|
45
|
+
License-File: THIRD_PARTY_NOTICES.html
|
|
46
|
+
Summary: With CocoIndex, users declare the transformation, CocoIndex creates & maintains an index, and keeps the derived index up to date based on source update, with minimal computation and changes.
|
|
47
|
+
Keywords: indexing,real-time,incremental,pipeline,search,ai,etl,rag,dataflow,context-engineering
|
|
48
|
+
Author-email: CocoIndex <cocoindex.io@gmail.com>
|
|
49
|
+
License-Expression: Apache-2.0
|
|
50
|
+
Requires-Python: >=3.11
|
|
51
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
52
|
+
Project-URL: Homepage, https://cocoindex.io/
|
|
53
|
+
|
|
54
|
+
<p align="center">
|
|
55
|
+
<img src="https://cocoindex.io/images/github.svg" alt="CocoIndex">
|
|
56
|
+
</p>
|
|
57
|
+
|
|
58
|
+
<h1 align="center">Data transformation for AI</h1>
|
|
59
|
+
|
|
60
|
+
<div align="center">
|
|
61
|
+
|
|
62
|
+
[](https://github.com/cocoindex-io/cocoindex)
|
|
63
|
+
[](https://cocoindex.io/docs/getting_started/quickstart)
|
|
64
|
+
[](https://opensource.org/licenses/Apache-2.0)
|
|
65
|
+
[](https://pypi.org/project/cocoindex/)
|
|
66
|
+
<!--[](https://pypistats.org/packages/cocoindex) -->
|
|
67
|
+
[](https://pepy.tech/projects/cocoindex)
|
|
68
|
+
[](https://github.com/cocoindex-io/cocoindex/actions/workflows/CI.yml)
|
|
69
|
+
[](https://github.com/cocoindex-io/cocoindex/actions/workflows/release.yml)
|
|
70
|
+
[](https://discord.com/invite/zpA9S2DR7s)
|
|
71
|
+
|
|
72
|
+
</div>
|
|
73
|
+
|
|
74
|
+
<div align="center">
|
|
75
|
+
<a href="https://trendshift.io/repositories/13939" target="_blank"><img src="https://trendshift.io/api/badge/repositories/13939" alt="cocoindex-io%2Fcocoindex | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
|
76
|
+
</div>
|
|
77
|
+
|
|
78
|
+
Ultra performant data transformation framework for AI, with core engine written in Rust. Support incremental processing and data lineage out-of-box. Exceptional developer velocity. Production-ready at day 0.
|
|
79
|
+
|
|
80
|
+
โญ Drop a star to help us grow!
|
|
81
|
+
|
|
82
|
+
<div align="center">
|
|
83
|
+
|
|
84
|
+
<!-- Keep these links. Translations will automatically update with the README. -->
|
|
85
|
+
[Deutsch](https://readme-i18n.com/cocoindex-io/cocoindex?lang=de) |
|
|
86
|
+
[English](https://readme-i18n.com/cocoindex-io/cocoindex?lang=en) |
|
|
87
|
+
[Espaรฑol](https://readme-i18n.com/cocoindex-io/cocoindex?lang=es) |
|
|
88
|
+
[franรงais](https://readme-i18n.com/cocoindex-io/cocoindex?lang=fr) |
|
|
89
|
+
[ๆฅๆฌ่ช](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ja) |
|
|
90
|
+
[ํ๊ตญ์ด](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ko) |
|
|
91
|
+
[Portuguรชs](https://readme-i18n.com/cocoindex-io/cocoindex?lang=pt) |
|
|
92
|
+
[ะ ัััะบะธะน](https://readme-i18n.com/cocoindex-io/cocoindex?lang=ru) |
|
|
93
|
+
[ไธญๆ](https://readme-i18n.com/cocoindex-io/cocoindex?lang=zh)
|
|
94
|
+
|
|
95
|
+
</div>
|
|
96
|
+
|
|
97
|
+
</br>
|
|
98
|
+
|
|
99
|
+
<p align="center">
|
|
100
|
+
<img src="https://cocoindex.io/images/transformation.svg" alt="CocoIndex Transformation">
|
|
101
|
+
</p>
|
|
102
|
+
|
|
103
|
+
</br>
|
|
104
|
+
|
|
105
|
+
CocoIndex makes it effortless to transform data with AI, and keep source data and target in sync. Whether youโre building a vector index for RAG, creating knowledge graphs, or performing any custom data transformations โ goes beyond SQL.
|
|
106
|
+
|
|
107
|
+
</br>
|
|
108
|
+
|
|
109
|
+
<p align="center">
|
|
110
|
+
<img alt="CocoIndex Features" src="https://cocoindex.io/images/venn2.svg" />
|
|
111
|
+
</p>
|
|
112
|
+
|
|
113
|
+
</br>
|
|
114
|
+
|
|
115
|
+
## Exceptional velocity
|
|
116
|
+
|
|
117
|
+
Just declare transformation in dataflow with ~100 lines of python
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
# import
|
|
121
|
+
data['content'] = flow_builder.add_source(...)
|
|
122
|
+
|
|
123
|
+
# transform
|
|
124
|
+
data['out'] = data['content']
|
|
125
|
+
.transform(...)
|
|
126
|
+
.transform(...)
|
|
127
|
+
|
|
128
|
+
# collect data
|
|
129
|
+
collector.collect(...)
|
|
130
|
+
|
|
131
|
+
# export to db, vector db, graph db ...
|
|
132
|
+
collector.export(...)
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
CocoIndex follows the idea of [Dataflow](https://en.wikipedia.org/wiki/Dataflow_programming) programming model. Each transformation creates a new field solely based on input fields, without hidden states and value mutation. All data before/after each transformation is observable, with lineage out of the box.
|
|
136
|
+
|
|
137
|
+
**Particularly**, developers don't explicitly mutate data by creating, updating and deleting. They just need to define transformation/formula for a set of source data.
|
|
138
|
+
|
|
139
|
+
## Plug-and-Play Building Blocks
|
|
140
|
+
|
|
141
|
+
Native builtins for different source, targets and transformations. Standardize interface, make it 1-line code switch between different components - as easy as assembling building blocks.
|
|
142
|
+
|
|
143
|
+
<p align="center">
|
|
144
|
+
<img src="https://cocoindex.io/images/components.svg" alt="CocoIndex Features">
|
|
145
|
+
</p>
|
|
146
|
+
|
|
147
|
+
## Data Freshness
|
|
148
|
+
|
|
149
|
+
CocoIndex keep source data and target in sync effortlessly.
|
|
150
|
+
|
|
151
|
+
<p align="center">
|
|
152
|
+
<img src="https://github.com/user-attachments/assets/f4eb29b3-84ee-4fa0-a1e2-80eedeeabde6" alt="Incremental Processing" width="700">
|
|
153
|
+
</p>
|
|
154
|
+
|
|
155
|
+
It has out-of-box support for incremental indexing:
|
|
156
|
+
|
|
157
|
+
- minimal recomputation on source or logic change.
|
|
158
|
+
- (re-)processing necessary portions; reuse cache when possible
|
|
159
|
+
|
|
160
|
+
## Quick Start
|
|
161
|
+
|
|
162
|
+
If you're new to CocoIndex, we recommend checking out
|
|
163
|
+
|
|
164
|
+
- ๐ [Documentation](https://cocoindex.io/docs)
|
|
165
|
+
- โก [Quick Start Guide](https://cocoindex.io/docs/getting_started/quickstart)
|
|
166
|
+
- ๐ฌ [Quick Start Video Tutorial](https://youtu.be/gv5R8nOXsWU?si=9ioeKYkMEnYevTXT)
|
|
167
|
+
|
|
168
|
+
### Setup
|
|
169
|
+
|
|
170
|
+
1. Install CocoIndex Python library
|
|
171
|
+
|
|
172
|
+
```bash
|
|
173
|
+
pip install -U cocoindex
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
2. [Install Postgres](https://cocoindex.io/docs/getting_started/installation#-install-postgres) if you don't have one. CocoIndex uses it for incremental processing.
|
|
177
|
+
|
|
178
|
+
3. (Optional) Install Claude Code skill for enhanced development experience. Run these commands in [Claude Code](https://claude.com/claude-code):
|
|
179
|
+
|
|
180
|
+
```
|
|
181
|
+
/plugin marketplace add cocoindex-io/cocoindex-claude
|
|
182
|
+
/plugin install cocoindex-skills@cocoindex
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
## Define data flow
|
|
186
|
+
|
|
187
|
+
Follow [Quick Start Guide](https://cocoindex.io/docs/getting_started/quickstart) to define your first indexing flow. An example flow looks like:
|
|
188
|
+
|
|
189
|
+
```python
|
|
190
|
+
@cocoindex.flow_def(name="TextEmbedding")
|
|
191
|
+
def text_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
|
|
192
|
+
# Add a data source to read files from a directory
|
|
193
|
+
data_scope["documents"] = flow_builder.add_source(cocoindex.sources.LocalFile(path="markdown_files"))
|
|
194
|
+
|
|
195
|
+
# Add a collector for data to be exported to the vector index
|
|
196
|
+
doc_embeddings = data_scope.add_collector()
|
|
197
|
+
|
|
198
|
+
# Transform data of each document
|
|
199
|
+
with data_scope["documents"].row() as doc:
|
|
200
|
+
# Split the document into chunks, put into `chunks` field
|
|
201
|
+
doc["chunks"] = doc["content"].transform(
|
|
202
|
+
cocoindex.functions.SplitRecursively(),
|
|
203
|
+
language="markdown", chunk_size=2000, chunk_overlap=500)
|
|
204
|
+
|
|
205
|
+
# Transform data of each chunk
|
|
206
|
+
with doc["chunks"].row() as chunk:
|
|
207
|
+
# Embed the chunk, put into `embedding` field
|
|
208
|
+
chunk["embedding"] = chunk["text"].transform(
|
|
209
|
+
cocoindex.functions.SentenceTransformerEmbed(
|
|
210
|
+
model="sentence-transformers/all-MiniLM-L6-v2"))
|
|
211
|
+
|
|
212
|
+
# Collect the chunk into the collector.
|
|
213
|
+
doc_embeddings.collect(filename=doc["filename"], location=chunk["location"],
|
|
214
|
+
text=chunk["text"], embedding=chunk["embedding"])
|
|
215
|
+
|
|
216
|
+
# Export collected data to a vector index.
|
|
217
|
+
doc_embeddings.export(
|
|
218
|
+
"doc_embeddings",
|
|
219
|
+
cocoindex.targets.Postgres(),
|
|
220
|
+
primary_key_fields=["filename", "location"],
|
|
221
|
+
vector_indexes=[
|
|
222
|
+
cocoindex.VectorIndexDef(
|
|
223
|
+
field_name="embedding",
|
|
224
|
+
metric=cocoindex.VectorSimilarityMetric.COSINE_SIMILARITY)])
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
It defines an index flow like this:
|
|
228
|
+
|
|
229
|
+
<p align="center">
|
|
230
|
+
<img width="400" alt="Data Flow" src="https://github.com/user-attachments/assets/2ea7be6d-3d94-42b1-b2bd-22515577e463" />
|
|
231
|
+
</p>
|
|
232
|
+
|
|
233
|
+
## ๐ Examples and demo
|
|
234
|
+
|
|
235
|
+
| Example | Description |
|
|
236
|
+
|---------|-------------|
|
|
237
|
+
| [Text Embedding](examples/text_embedding) | Index text documents with embeddings for semantic search |
|
|
238
|
+
| [Code Embedding](examples/code_embedding) | Index code embeddings for semantic search |
|
|
239
|
+
| [PDF Embedding](examples/pdf_embedding) | Parse PDF and index text embeddings for semantic search |
|
|
240
|
+
| [PDF Elements Embedding](examples/pdf_elements_embedding) | Extract text and images from PDFs; embed text with SentenceTransformers and images with CLIP; store in Qdrant for multimodal search |
|
|
241
|
+
| [Manuals LLM Extraction](examples/manuals_llm_extraction) | Extract structured information from a manual using LLM |
|
|
242
|
+
| [Amazon S3 Embedding](examples/amazon_s3_embedding) | Index text documents from Amazon S3 |
|
|
243
|
+
| [Azure Blob Storage Embedding](examples/azure_blob_embedding) | Index text documents from Azure Blob Storage |
|
|
244
|
+
| [Google Drive Text Embedding](examples/gdrive_text_embedding) | Index text documents from Google Drive |
|
|
245
|
+
| [Docs to Knowledge Graph](examples/docs_to_knowledge_graph) | Extract relationships from Markdown documents and build a knowledge graph |
|
|
246
|
+
| [Embeddings to Qdrant](examples/text_embedding_qdrant) | Index documents in a Qdrant collection for semantic search |
|
|
247
|
+
| [Embeddings to LanceDB](examples/text_embedding_lancedb) | Index documents in a LanceDB collection for semantic search |
|
|
248
|
+
| [FastAPI Server with Docker](examples/fastapi_server_docker) | Run the semantic search server in a Dockerized FastAPI setup |
|
|
249
|
+
| [Product Recommendation](examples/product_recommendation) | Build real-time product recommendations with LLM and graph database|
|
|
250
|
+
| [Image Search with Vision API](examples/image_search) | Generates detailed captions for images using a vision model, embeds them, enables live-updating semantic search via FastAPI and served on a React frontend|
|
|
251
|
+
| [Face Recognition](examples/face_recognition) | Recognize faces in images and build embedding index |
|
|
252
|
+
| [Paper Metadata](examples/paper_metadata) | Index papers in PDF files, and build metadata tables for each paper |
|
|
253
|
+
| [Multi Format Indexing](examples/multi_format_indexing) | Build visual document index from PDFs and images with ColPali for semantic search |
|
|
254
|
+
| [Custom Source HackerNews](examples/custom_source_hn) | Index HackerNews threads and comments, using *CocoIndex Custom Source* |
|
|
255
|
+
| [Custom Output Files](examples/custom_output_files) | Convert markdown files to HTML files and save them to a local directory, using *CocoIndex Custom Targets* |
|
|
256
|
+
| [Patient intake form extraction](examples/patient_intake_extraction) | Use LLM to extract structured data from patient intake forms with different formats |
|
|
257
|
+
| [HackerNews Trending Topics](examples/hn_trending_topics) | Extract trending topics from HackerNews threads and comments, using *CocoIndex Custom Source* and LLM |
|
|
258
|
+
| [Patient Intake Form Extraction with BAML](examples/patient_intake_extraction_baml) | Extract structured data from patient intake forms using BAML |
|
|
259
|
+
|
|
260
|
+
More coming and stay tuned ๐!
|
|
261
|
+
|
|
262
|
+
## ๐ Documentation
|
|
263
|
+
|
|
264
|
+
For detailed documentation, visit [CocoIndex Documentation](https://cocoindex.io/docs), including a [Quickstart guide](https://cocoindex.io/docs/getting_started/quickstart).
|
|
265
|
+
|
|
266
|
+
## ๐ค Contributing
|
|
267
|
+
|
|
268
|
+
We love contributions from our community โค๏ธ. For details on contributing or running the project for development, check out our [contributing guide](https://cocoindex.io/docs/about/contributing).
|
|
269
|
+
|
|
270
|
+
## ๐ฅ Community
|
|
271
|
+
|
|
272
|
+
Welcome with a huge coconut hug ๐ฅฅโ๏ฝกห๐ค. We are super excited for community contributions of all kinds - whether it's code improvements, documentation updates, issue reports, feature requests, and discussions in our Discord.
|
|
273
|
+
|
|
274
|
+
Join our community here:
|
|
275
|
+
|
|
276
|
+
- ๐ [Star us on GitHub](https://github.com/cocoindex-io/cocoindex)
|
|
277
|
+
- ๐ [Join our Discord community](https://discord.com/invite/zpA9S2DR7s)
|
|
278
|
+
- โถ๏ธ [Subscribe to our YouTube channel](https://www.youtube.com/@cocoindex-io)
|
|
279
|
+
- ๐ [Read our blog posts](https://cocoindex.io/blogs/)
|
|
280
|
+
|
|
281
|
+
## Support us
|
|
282
|
+
|
|
283
|
+
We are constantly improving, and more features and examples are coming soon. If you love this project, please drop us a star โญ at GitHub repo [](https://github.com/cocoindex-io/cocoindex) to stay tuned and help us grow.
|
|
284
|
+
|
|
285
|
+
## License
|
|
286
|
+
|
|
287
|
+
CocoIndex is Apache 2.0 licensed.
|
|
288
|
+
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
cocoindex-0.3.4.dist-info/METADATA,sha256=AcOCgitKjrE1dTsElpjai2aNzj0Bq4kLspb8KkjG2bs,14344
|
|
2
|
+
cocoindex-0.3.4.dist-info/WHEEL,sha256=O2QTG69GgK-VjUv6T5nE2QGjJc-8mS3d1MslSxOOSiY,107
|
|
3
|
+
cocoindex-0.3.4.dist-info/entry_points.txt,sha256=_NretjYVzBdNTn7dK-zgwr7YfG2afz1u1uSE-5bZXF8,46
|
|
4
|
+
cocoindex-0.3.4.dist-info/licenses/THIRD_PARTY_NOTICES.html,sha256=bkuRnofeXlmHgbT03ma4r2KDMVpLDza63oAkJ1zRZHA,750831
|
|
5
|
+
cocoindex/__init__.py,sha256=6qZWVkK4WZ01BIAg3CPh_bRRdA6Clk4d4Q6OnZ2jFa4,2630
|
|
6
|
+
cocoindex/_engine.abi3.so,sha256=gQilgITrbt9dzi85rDD9atnO38lqOQFe0temcbEQqHM,73476048
|
|
7
|
+
cocoindex/auth_registry.py,sha256=g-uLDWLYW5NMbYe7q4Y-sU5dSyrlJXBEciyWtAiP9KE,1340
|
|
8
|
+
cocoindex/cli.py,sha256=k7bl8RTUZoNNxTlQMr-Y3-9-rTNt8z1v7rJWqsajYC8,24792
|
|
9
|
+
cocoindex/engine_object.py,sha256=VzjO7mNlAWkGUw0d9_CtVe3eICsL89zz7LgPGaG3mds,7638
|
|
10
|
+
cocoindex/engine_value.py,sha256=rMbLF3Twnzbdt3ennfFbgNivLCfmxEA0epNMPQnHEo0,19819
|
|
11
|
+
cocoindex/flow.py,sha256=xDz3rOo4RhbboknvC-KnbWq8RBykEO0YsjGSBfXqIEg,40076
|
|
12
|
+
cocoindex/functions/__init__.py,sha256=V2IF4h-Cqq4OD_GN3Oqdry-FArORyRCKmqJ7g5UlJr8,1021
|
|
13
|
+
cocoindex/functions/_engine_builtin_specs.py,sha256=WpCGrjUfJBa8xZP5JiEmA8kLu7fp9Rcs7ynpuJmvSGg,1786
|
|
14
|
+
cocoindex/functions/colpali.py,sha256=pk72XCvXma18qTJeFJ87JNspklEy5GQp0M4DBblV5_w,8670
|
|
15
|
+
cocoindex/functions/sbert.py,sha256=WlZ2o_D6S-zmi_IXOnwn_IsiBriig0dlByGKx6WbduE,2708
|
|
16
|
+
cocoindex/index.py,sha256=tz5ilvmOp0BtroGehCQDqWK_pIX9m6ghkhcxsDVU8WE,982
|
|
17
|
+
cocoindex/lib.py,sha256=spfdU4IbzdffHyGdrQPIw_qGo9aX0OAAboqsjj8bTiQ,2290
|
|
18
|
+
cocoindex/llm.py,sha256=8ZdJhOmhdb2xEcCxk6rDpnj6hlhCyFBmJdhCNMqAOP4,875
|
|
19
|
+
cocoindex/op.py,sha256=GRIpVTmYpd_m0aSzK-vo2tlaLtFyOaRBAk183npZKFc,37996
|
|
20
|
+
cocoindex/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
|
+
cocoindex/query_handler.py,sha256=X-SQT71LHiOOXn6-TJlQcGodJk-iT8p_1TcIMvRLBRI,1344
|
|
22
|
+
cocoindex/runtime.py,sha256=d5NdVvsrU-o4_K-rlXRId7plh_C4yrTlsbAM6q24vgI,2426
|
|
23
|
+
cocoindex/setting.py,sha256=1Dx8ktjwf-8BiXrbsmfn5Mzudb2SQYqFdRnSNGVKaLk,4960
|
|
24
|
+
cocoindex/setup.py,sha256=7uIHKN4FOCuoidPXcKyGTrkqpkl9luL49-6UcnMxYzw,3068
|
|
25
|
+
cocoindex/sources/__init__.py,sha256=Yu9VHNaGlOEE3jpqfIseswsg25Le3HzwDr6XJAn22Ns,78
|
|
26
|
+
cocoindex/sources/_engine_builtin_specs.py,sha256=5l_-uV5JFRmOr_MMak18EVwGesTAFKEHuCBSbqURuco,3760
|
|
27
|
+
cocoindex/subprocess_exec.py,sha256=41PyvMaZ9q7B-F0YTUg1xdNVM_6_HdFErDbxDJMUFPg,9040
|
|
28
|
+
cocoindex/targets/__init__.py,sha256=HQG7I4U0xQhHiYctiUvwEBLxT2727oHP3xwrqotjmhk,78
|
|
29
|
+
cocoindex/targets/_engine_builtin_specs.py,sha256=glXUN5bj11Jxky1VPvmGnWnMHXTQWEh08INcbldo3F4,3375
|
|
30
|
+
cocoindex/targets/lancedb.py,sha256=1nzCre5p-fvKkmLOTvfpiLTfnhF3qMLqTvsTwNuGwVU,15749
|
|
31
|
+
cocoindex/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
32
|
+
cocoindex/tests/test_engine_object.py,sha256=9llG3jMyP9_hM8FT4VXagCBh8qWU8U9_U5a2mLXiQ-4,11238
|
|
33
|
+
cocoindex/tests/test_engine_value.py,sha256=hJnGxlYLJCIByqTTkmJOgl-tt7iHCJKyoylLD8OOoNM,56071
|
|
34
|
+
cocoindex/tests/test_optional_database.py,sha256=snAmkNa6wtOSaxoZE1HgjvL5v_ylitt3Jt_9df4Cgdc,8506
|
|
35
|
+
cocoindex/tests/test_transform_flow.py,sha256=dHpSvPdW9SnOViO8CEu1RNt6dkW3LcShvbfDKOD0SqM,8663
|
|
36
|
+
cocoindex/tests/test_typing.py,sha256=JoR-oMK-ZWjOGQi0pH5Etg5jp4oL_JSIreGBH247GCg,16291
|
|
37
|
+
cocoindex/tests/test_validation.py,sha256=X6AQzVs-hVKIXcrHMEMQnhfUE8at7iXQnPq8nHNhZ2Q,4543
|
|
38
|
+
cocoindex/typing.py,sha256=Nb8-ZbVsVHQ_Cecy6bR-onbim9wKx5-h-GiKvpz-McY,24785
|
|
39
|
+
cocoindex/user_app_loader.py,sha256=bc3Af-gYRxJ9GpObtpjegZY855oQBCv5FGkrkWV2yGY,1873
|
|
40
|
+
cocoindex/utils.py,sha256=hUhX-XV6XGCtJSEIpBOuDv6VvqImwPlgBxztBTw7u0U,598
|
|
41
|
+
cocoindex/validation.py,sha256=PZnJoby4sLbsmPv9fOjOQXuefjfZ7gmtsiTGU8SH-tc,3090
|
|
42
|
+
cocoindex-0.3.4.dist-info/RECORD,,
|