cocoindex 0.1.28__tar.gz → 0.1.30__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cocoindex-0.1.28 → cocoindex-0.1.30}/.env.lib_debug +1 -1
- {cocoindex-0.1.28 → cocoindex-0.1.30}/Cargo.lock +1 -1
- {cocoindex-0.1.28 → cocoindex-0.1.30}/Cargo.toml +1 -1
- {cocoindex-0.1.28 → cocoindex-0.1.30}/PKG-INFO +2 -1
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/docs/ops/storages.md +98 -64
- cocoindex-0.1.30/examples/docs_to_knowledge_graph/README.md +80 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/docs_to_knowledge_graph/main.py +16 -17
- {cocoindex-0.1.28 → cocoindex-0.1.30}/pyproject.toml +1 -1
- {cocoindex-0.1.28 → cocoindex-0.1.30}/python/cocoindex/auth_registry.py +5 -2
- {cocoindex-0.1.28 → cocoindex-0.1.30}/python/cocoindex/cli.py +45 -14
- {cocoindex-0.1.28 → cocoindex-0.1.30}/python/cocoindex/flow.py +54 -1
- {cocoindex-0.1.28 → cocoindex-0.1.30}/python/cocoindex/functions.py +7 -3
- {cocoindex-0.1.28 → cocoindex-0.1.30}/python/cocoindex/lib.py +6 -59
- {cocoindex-0.1.28 → cocoindex-0.1.30}/python/cocoindex/op.py +5 -4
- cocoindex-0.1.30/python/cocoindex/setting.py +77 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/python/cocoindex/storages.py +19 -13
- {cocoindex-0.1.28 → cocoindex-0.1.30}/python/cocoindex/typing.py +5 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/ops/storages/neo4j.rs +34 -33
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/ops/storages/spec.rs +9 -14
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/server.rs +12 -8
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/setup/driver.rs +2 -2
- cocoindex-0.1.28/examples/docs_to_knowledge_graph/README.md +0 -67
- {cocoindex-0.1.28 → cocoindex-0.1.30}/.cargo/config.toml +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/.github/ISSUE_TEMPLATE//360/237/220/233-bug-report.md" +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/.github/ISSUE_TEMPLATE//360/237/222/241-feature-request.md" +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/.github/scripts/update_version.sh +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/.github/workflows/CI.yml +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/.github/workflows/_test.yml +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/.github/workflows/docs.yml +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/.github/workflows/release.yml +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/.gitignore +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/.vscode/settings.json +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/CODE_OF_CONDUCT.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/CONTRIBUTING.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/LICENSE +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/README.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/dev/neo4j.yaml +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/dev/postgres.yaml +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/.gitignore +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/README.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/docs/about/community.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/docs/about/contributing.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/docs/ai/llm.mdx +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/docs/core/basics.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/docs/core/cli.mdx +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/docs/core/custom_function.mdx +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/docs/core/data_example.svg +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/docs/core/data_types.mdx +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/docs/core/flow_def.mdx +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/docs/core/flow_example.svg +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/docs/core/flow_methods.mdx +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/docs/core/initialization.mdx +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/docs/getting_started/installation.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/docs/getting_started/markdown_files.zip +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/docs/getting_started/overview.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/docs/getting_started/quickstart.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/docs/ops/functions.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/docs/ops/sources.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/docusaurus.config.ts +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/package.json +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/sidebars.ts +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/src/components/HomepageFeatures/index.tsx +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/src/components/HomepageFeatures/styles.module.css +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/src/css/custom.css +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/src/theme/Root.js +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/static/.nojekyll +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/static/img/docusaurus.png +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/static/img/favicon.ico +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/static/img/icon.svg +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/static/robots.txt +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/tsconfig.json +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/docs/yarn.lock +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/code_embedding/.env +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/code_embedding/README.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/code_embedding/main.py +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/code_embedding/pyproject.toml +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/docs_to_knowledge_graph/.env +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/docs_to_knowledge_graph/markdown_files/1706.03762v7.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/docs_to_knowledge_graph/markdown_files/1810.04805v2.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/docs_to_knowledge_graph/markdown_files/rfc8259.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/docs_to_knowledge_graph/pyproject.toml +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/gdrive_text_embedding/.env.example +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/gdrive_text_embedding/.gitignore +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/gdrive_text_embedding/README.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/gdrive_text_embedding/data/1706.03762v7.docx +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/gdrive_text_embedding/data/1810.04805v2.docx +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/gdrive_text_embedding/main.py +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/gdrive_text_embedding/pyproject.toml +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/manuals_llm_extraction/.env +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/manuals_llm_extraction/README.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/manuals_llm_extraction/main.py +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/manuals_llm_extraction/manuals/array.pdf +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/manuals_llm_extraction/manuals/base64.pdf +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/manuals_llm_extraction/manuals/copy.pdf +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/manuals_llm_extraction/manuals/glob.pdf +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/manuals_llm_extraction/pyproject.toml +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/pdf_embedding/.env +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/pdf_embedding/README.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/pdf_embedding/main.py +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/pdf_embedding/pdf_files/1706.03762v7.pdf +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/pdf_embedding/pdf_files/1810.04805v2.pdf +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/pdf_embedding/pdf_files/rfc8259.pdf +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/pdf_embedding/pyproject.toml +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/text_embedding/.env +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/text_embedding/README.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/text_embedding/Text_Embedding.ipynb +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/text_embedding/main.py +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/text_embedding/markdown_files/1706.03762v7.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/text_embedding/markdown_files/1810.04805v2.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/text_embedding/markdown_files/rfc8259.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/text_embedding/pyproject.toml +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/text_embedding_qdrant/.env +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/text_embedding_qdrant/README.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/text_embedding_qdrant/main.py +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/text_embedding_qdrant/markdown_files/rfc8259.md +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/examples/text_embedding_qdrant/pyproject.toml +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/python/cocoindex/__init__.py +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/python/cocoindex/convert.py +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/python/cocoindex/index.py +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/python/cocoindex/llm.py +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/python/cocoindex/py.typed +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/python/cocoindex/query.py +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/python/cocoindex/runtime.py +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/python/cocoindex/setup.py +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/python/cocoindex/sources.py +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/python/cocoindex/tests/__init__.py +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/python/cocoindex/tests/test_convert.py +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/base/field_attrs.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/base/json_schema.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/base/mod.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/base/schema.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/base/spec.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/base/value.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/builder/analyzed_flow.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/builder/analyzer.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/builder/flow_builder.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/builder/mod.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/builder/plan.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/execution/db_tracking.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/execution/db_tracking_setup.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/execution/dumper.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/execution/evaluator.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/execution/live_updater.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/execution/memoization.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/execution/mod.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/execution/query.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/execution/row_indexer.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/execution/source_indexer.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/execution/stats.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/lib.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/lib_context.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/llm/anthropic.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/llm/gemini.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/llm/mod.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/llm/ollama.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/llm/openai.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/ops/factory_bases.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/ops/functions/extract_by_llm.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/ops/functions/mod.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/ops/functions/parse_json.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/ops/functions/split_recursively.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/ops/interface.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/ops/mod.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/ops/py_factory.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/ops/registration.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/ops/registry.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/ops/sdk.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/ops/sources/google_drive.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/ops/sources/local_file.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/ops/sources/mod.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/ops/storages/mod.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/ops/storages/postgres.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/ops/storages/qdrant.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/prelude.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/py/convert.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/py/mod.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/service/error.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/service/flows.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/service/mod.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/service/search.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/settings.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/setup/auth_registry.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/setup/components.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/setup/db_metadata.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/setup/mod.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/setup/states.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/utils/db.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/utils/fingerprint.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/utils/immutable.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/utils/mod.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/utils/retriable.rs +0 -0
- {cocoindex-0.1.28 → cocoindex-0.1.30}/src/utils/yaml_ser.rs +0 -0
@@ -2,7 +2,7 @@
|
|
2
2
|
name = "cocoindex"
|
3
3
|
# Version used for local development is always higher than others to take precedence.
|
4
4
|
# Will be overridden for specific release versions.
|
5
|
-
version = "0.1.
|
5
|
+
version = "0.1.30"
|
6
6
|
edition = "2021"
|
7
7
|
|
8
8
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
@@ -1,8 +1,9 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: cocoindex
|
3
|
-
Version: 0.1.
|
3
|
+
Version: 0.1.30
|
4
4
|
Requires-Dist: sentence-transformers>=3.3.1
|
5
5
|
Requires-Dist: click>=8.1.8
|
6
|
+
Requires-Dist: rich>=14.0.0
|
6
7
|
Requires-Dist: pytest ; extra == 'test'
|
7
8
|
Provides-Extra: test
|
8
9
|
License-File: LICENSE
|
@@ -87,24 +87,34 @@ You can find an end-to-end example [here](https://github.com/cocoindex-io/cocoin
|
|
87
87
|
|
88
88
|
## Property Graph Targets
|
89
89
|
|
90
|
-
Property graph is a
|
90
|
+
Property graph is a widely-adopted model for knowledge graphs, where both nodes and relationships can have properties.
|
91
|
+
[Graph database concepts](https://neo4j.com/docs/getting-started/appendix/graphdb-concepts/) has a good introduction to basic concepts of property graphs.
|
92
|
+
|
93
|
+
The following concepts will be used in the following sections:
|
94
|
+
* [Node](https://neo4j.com/docs/getting-started/appendix/graphdb-concepts/#graphdb-node)
|
95
|
+
* [Node label](https://neo4j.com/docs/getting-started/appendix/graphdb-concepts/#graphdb-labels), which represents a type of nodes.
|
96
|
+
* [Relationship](https://neo4j.com/docs/getting-started/appendix/graphdb-concepts/#graphdb-relationship), which describes a connection between two nodes.
|
97
|
+
* [Relationship type](https://neo4j.com/docs/getting-started/appendix/graphdb-concepts/#graphdb-relationship-type)
|
98
|
+
* [Properties](https://neo4j.com/docs/getting-started/appendix/graphdb-concepts/#graphdb-properties), which are key-value pairs associated with nodes and relationships.
|
91
99
|
|
92
100
|
### Data Mapping
|
93
101
|
|
94
|
-
|
95
|
-
|
96
|
-
|
102
|
+
Data from collectors are mapped to graph elements in various types:
|
103
|
+
|
104
|
+
1. Rows from collectors → Nodes in the graph
|
105
|
+
2. Rows from collectors → Relationships in the graph (including source and target nodes of the relationship)
|
97
106
|
|
98
|
-
|
99
|
-
2. You can export rows from some other collectors to relationships in the graph.
|
100
|
-
3. Some nodes referenced by relationships exported in 2 may not exist as nodes exported in 1.
|
101
|
-
CocoIndex will automatically create and keep these nodes, as long as they're still referenced by at least one relationship.
|
102
|
-
This guarantees that all relationships exported in 2 are valid.
|
107
|
+
This is what you need to provide to define these mappings:
|
103
108
|
|
104
|
-
|
105
|
-
|
109
|
+
* Specify [nodes to export](#nodes-to-export).
|
110
|
+
* [Declare extra node labels](#declare-extra-node-labels), for labels to appear as source/target nodes of relationships but not exported as nodes.
|
111
|
+
* Specify [relationships to export](#relationships-to-export).
|
106
112
|
|
107
|
-
|
113
|
+
In addition, the same node may appear multiple times, from exported nodes and various relationships.
|
114
|
+
They should appear as the same node in the target graph database.
|
115
|
+
CocoIndex automatically [matches and deduplicates nodes](#nodes-matching-and-deduplicating) based on their primary key values.
|
116
|
+
|
117
|
+
#### Nodes to Export
|
108
118
|
|
109
119
|
Here's how CocoIndex data elements map to nodes in the graph:
|
110
120
|
|
@@ -114,9 +124,9 @@ Here's how CocoIndex data elements map to nodes in the graph:
|
|
114
124
|
| a collected row | a node |
|
115
125
|
| a field | a property of node |
|
116
126
|
|
117
|
-
Note that the label used in different `
|
127
|
+
Note that the label used in different `Nodes`s should be unique.
|
118
128
|
|
119
|
-
`cocoindex.storages.
|
129
|
+
`cocoindex.storages.Nodes` is to describe mapping to nodes. It has the following fields:
|
120
130
|
|
121
131
|
* `label` (type: `str`): The label of the node.
|
122
132
|
|
@@ -138,7 +148,7 @@ document_collector.export(
|
|
138
148
|
...
|
139
149
|
cocoindex.storages.Neo4j(
|
140
150
|
...
|
141
|
-
mapping=cocoindex.storages.
|
151
|
+
mapping=cocoindex.storages.Nodes(label="Document"),
|
142
152
|
),
|
143
153
|
primary_key_fields=["filename"],
|
144
154
|
)
|
@@ -167,7 +177,32 @@ graph TD
|
|
167
177
|
classDef node font-size:8pt,text-align:left,stroke-width:2;
|
168
178
|
```
|
169
179
|
|
170
|
-
####
|
180
|
+
#### Declare Extra Node Labels
|
181
|
+
|
182
|
+
If a node label needs to appear as source or target of a relationship, but not exported as a node, you need to [declare](../core/flow_def#target-declarations) the label with necessary configuration.
|
183
|
+
|
184
|
+
The dataclass to describe the declaration is specific to each target storage (e.g. `cocoindex.storages.Neo4jDeclarations`),
|
185
|
+
while they share the following common fields:
|
186
|
+
|
187
|
+
* `nodes_label` (required): The label of the node.
|
188
|
+
* Options for [storage indexes](../core/flow_def#storage-indexes).
|
189
|
+
* `primary_key_fields` (required)
|
190
|
+
* `vector_indexes` (optional)
|
191
|
+
|
192
|
+
Continuing the same example above.
|
193
|
+
Considering we want to extract relationships from `Document` to `Place` later (i.e. a document mentions a place), but the `Place` label isn't exported as a node, we need to declare it:
|
194
|
+
|
195
|
+
```python
|
196
|
+
flow_builder.declare(
|
197
|
+
cocoindex.storages.Neo4jDeclarations(
|
198
|
+
connection = ...,
|
199
|
+
nodes_label="Place",
|
200
|
+
primary_key_fields=["name"],
|
201
|
+
),
|
202
|
+
)
|
203
|
+
```
|
204
|
+
|
205
|
+
#### Relationships to Export
|
171
206
|
|
172
207
|
Here's how CocoIndex data elements map to relationships in the graph:
|
173
208
|
|
@@ -177,12 +212,12 @@ Here's how CocoIndex data elements map to relationships in the graph:
|
|
177
212
|
| a collected row | a relationship |
|
178
213
|
| a field | a property of relationship, or a property of source/target node, based on configuration |
|
179
214
|
|
180
|
-
Note that the type used in different `
|
215
|
+
Note that the type used in different `Relationships`s should be unique.
|
181
216
|
|
182
|
-
`cocoindex.storages.
|
217
|
+
`cocoindex.storages.Relationships` is to describe mapping to relationships. It has the following fields:
|
183
218
|
|
184
219
|
* `rel_type` (type: `str`): The type of the relationship.
|
185
|
-
* `source`/`target` (type: `cocoindex.storages.
|
220
|
+
* `source`/`target` (type: `cocoindex.storages.NodeFromFields`): Specify how to extract source/target node information from specific fields in the collected row. It has the following fields:
|
186
221
|
* `label` (type: `str`): The label of the node.
|
187
222
|
* `fields` (type: `Sequence[cocoindex.storages.TargetFieldMapping]`): Specify field mappings from the collected rows to node properties, with the following fields:
|
188
223
|
* `source` (type: `str`): The name of the field in the collected row.
|
@@ -218,13 +253,13 @@ doc_place_collector.export(
|
|
218
253
|
...
|
219
254
|
cocoindex.storages.Neo4j(
|
220
255
|
...
|
221
|
-
mapping=cocoindex.storages.
|
256
|
+
mapping=cocoindex.storages.Relationships(
|
222
257
|
rel_type="MENTION",
|
223
|
-
source=cocoindex.storages.
|
258
|
+
source=cocoindex.storages.NodeFromFields(
|
224
259
|
label="Document",
|
225
260
|
fields=[cocoindex.storages.TargetFieldMapping(source="doc_filename", target="filename")],
|
226
261
|
),
|
227
|
-
target=cocoindex.storages.
|
262
|
+
target=cocoindex.storages.NodeFromFields(
|
228
263
|
label="Place",
|
229
264
|
fields=[
|
230
265
|
cocoindex.storages.TargetFieldMapping(source="place_name", target="name"),
|
@@ -250,19 +285,26 @@ graph TD
|
|
250
285
|
classDef: nodeRef
|
251
286
|
}
|
252
287
|
|
253
|
-
|
288
|
+
Doc_Chapter2_a@{
|
254
289
|
shape: rounded
|
255
290
|
label: "**[Document]**
|
256
291
|
**filename\\*: chapter2.md**"
|
257
292
|
classDef: nodeRef
|
258
293
|
}
|
259
294
|
|
260
|
-
|
295
|
+
Doc_Chapter2_b@{
|
296
|
+
shape: rounded
|
297
|
+
label: "**[Document]**
|
298
|
+
**filename\\*: chapter2.md**"
|
299
|
+
classDef: nodeRef
|
300
|
+
}
|
301
|
+
|
302
|
+
Place_CrystalPalace_a@{
|
261
303
|
shape: rounded
|
262
304
|
label: "**[Place]**
|
263
305
|
**name\\*: Crystal Palace**
|
264
306
|
embedding: [0.1, 0.5, ...]"
|
265
|
-
classDef:
|
307
|
+
classDef: node
|
266
308
|
}
|
267
309
|
|
268
310
|
Place_MagicForest@{
|
@@ -270,38 +312,43 @@ graph TD
|
|
270
312
|
label: "**[Place]**
|
271
313
|
**name\\*: Magic Forest**
|
272
314
|
embedding: [0.4, 0.2, ...]"
|
273
|
-
classDef:
|
315
|
+
classDef: node
|
316
|
+
}
|
317
|
+
|
318
|
+
Place_CrystalPalace_b@{
|
319
|
+
shape: rounded
|
320
|
+
label: "**[Place]**
|
321
|
+
**name\\*: Crystal Palace**
|
322
|
+
embedding: [0.1, 0.5, ...]"
|
323
|
+
classDef: node
|
274
324
|
}
|
275
325
|
|
276
|
-
|
277
|
-
|
278
|
-
|
326
|
+
|
327
|
+
Doc_Chapter1:::nodeRef -- **:MENTION** (location:12) --> Place_CrystalPalace_a:::node
|
328
|
+
Doc_Chapter2_a:::nodeRef -- **:MENTION** (location:23) --> Place_MagicForest:::node
|
329
|
+
Doc_Chapter2_b:::nodeRef -- **:MENTION** (location:56) --> Place_CrystalPalace_b:::node
|
279
330
|
|
280
331
|
classDef nodeRef font-size:8pt,text-align:left,fill:transparent,stroke-width:1,stroke-dasharray:5 5;
|
332
|
+
classDef node font-size:8pt,text-align:left,stroke-width:2;
|
281
333
|
|
282
334
|
```
|
283
335
|
|
336
|
+
#### Nodes Matching and Deduplicating
|
284
337
|
|
285
|
-
|
338
|
+
The nodes and relationships we got above are discrete elements.
|
339
|
+
To fit them into a connected property graph, CocoIndex will match and deduplicate nodes automatically:
|
286
340
|
|
287
|
-
|
341
|
+
* Match nodes based on their primary key values. Nodes with the same primary key values are considered as the same node.
|
342
|
+
* For non-primary key fields (a.k.a. value fields), CocoIndex will pick the values from an arbitrary one.
|
343
|
+
If multiple nodes (before deduplication) with the same primary key provide value fields, an arbitrary one will be picked.
|
288
344
|
|
289
|
-
:::note
|
345
|
+
:::note
|
290
346
|
|
291
|
-
If the same node (as identified by primary key values) appears multiple times (e.g. they're referenced by different relationships),
|
292
|
-
CocoIndex uses value fields provided by an arbitrary one of them.
|
293
347
|
The best practice is to make the value fields consistent across different appearances of the same node, to avoid non-determinism in the exported graph.
|
294
348
|
|
295
349
|
:::
|
296
350
|
|
297
|
-
|
298
|
-
The following options are supported:
|
299
|
-
|
300
|
-
* `primary_key_fields` (required)
|
301
|
-
* `vector_indexes` (optional)
|
302
|
-
|
303
|
-
Using the same example above.
|
304
|
-
After combining exported nodes and relationships, we get the knowledge graph with all information:
|
351
|
+
After matching and deduplication, we get the final graph:
|
305
352
|
|
306
353
|
```mermaid
|
307
354
|
graph TD
|
@@ -326,7 +373,7 @@ graph TD
|
|
326
373
|
label: "**[Place]**
|
327
374
|
**name\\*: Crystal Palace**
|
328
375
|
embedding: [0.1, 0.5, ...]"
|
329
|
-
classDef:
|
376
|
+
classDef: node
|
330
377
|
}
|
331
378
|
|
332
379
|
Place_MagicForest@{
|
@@ -334,30 +381,14 @@ graph TD
|
|
334
381
|
label: "**[Place]**
|
335
382
|
**name\\*: Magic Forest**
|
336
383
|
embedding: [0.4, 0.2, ...]"
|
337
|
-
classDef:
|
384
|
+
classDef: node
|
338
385
|
}
|
339
386
|
|
340
|
-
Doc_Chapter1:::node --
|
341
|
-
Doc_Chapter2:::node --
|
342
|
-
Doc_Chapter2:::node --
|
387
|
+
Doc_Chapter1:::node -- **:MENTION** (location:12) --> Place_CrystalPalace:::node
|
388
|
+
Doc_Chapter2:::node -- **:MENTION** (location:23) --> Place_MagicForest:::node
|
389
|
+
Doc_Chapter2:::node -- **:MENTION** (location:56) --> Place_CrystalPalace:::node
|
343
390
|
|
344
391
|
classDef node font-size:8pt,text-align:left,stroke-width:2;
|
345
|
-
classDef nodeRef font-size:8pt,text-align:left,fill:transparent,stroke-width:1,stroke-dasharray:5 5;
|
346
|
-
|
347
|
-
```
|
348
|
-
|
349
|
-
Nodes with `Place` label in the example aren't exported explicitly using `NodeMapping`, so CocoIndex will automatically create them as long as they're still referenced by any relationship.
|
350
|
-
You need to declare a `ReferencedNode`:
|
351
|
-
|
352
|
-
```python
|
353
|
-
flow_builder.declare(
|
354
|
-
cocoindex.storages.Neo4jDeclarations(
|
355
|
-
...
|
356
|
-
referenced_nodes=[
|
357
|
-
cocoindex.storages.ReferencedNode(label="Place", primary_key_fields=["name"]),
|
358
|
-
],
|
359
|
-
),
|
360
|
-
)
|
361
392
|
```
|
362
393
|
|
363
394
|
### Neo4j
|
@@ -388,6 +419,9 @@ The `Neo4j` storage exports each row as a relationship to Neo4j Knowledge Graph.
|
|
388
419
|
Neo4j also provides a declaration spec `Neo4jDeclaration`, to configure indexing options for nodes only referenced by relationships. It has the following fields:
|
389
420
|
|
390
421
|
* `connection` (type: auth reference to `Neo4jConnectionSpec`)
|
391
|
-
*
|
422
|
+
* Fields for [nodes to declare](#nodes-to-declare), including
|
423
|
+
* `nodes_label` (required)
|
424
|
+
* `primary_key_fields` (required)
|
425
|
+
* `vector_indexes` (optional)
|
392
426
|
|
393
427
|
You can find an end-to-end example [here](https://github.com/cocoindex-io/cocoindex/tree/main/examples/docs_to_knowledge_graph).
|
@@ -0,0 +1,80 @@
|
|
1
|
+
# Build Real-Time Knowledge Graph For Documents with LLM
|
2
|
+
|
3
|
+
We will process a list of documents and use LLM to extract relationships between the concepts in each document.
|
4
|
+
We will generate two kinds of relationships:
|
5
|
+
|
6
|
+
1. Relationships between subjects and objects. E.g., "CocoIndex supports Incremental Processing"
|
7
|
+
2. Mentions of entities in a document. E.g., "core/basics.mdx" mentions `CocoIndex` and `Incremental Processing`.
|
8
|
+
|
9
|
+
You can find a step by step blog for this project [here](https://cocoindex.io/blogs/knowledge-graph-for-docs)
|
10
|
+
|
11
|
+
Please drop [Cocoindex on Github](https://github.com/cocoindex-io/cocoindex) a star to support us if you like our work. Thank you so much with a warm coconut hug 🥥🤗. [](https://github.com/cocoindex-io/cocoindex)
|
12
|
+
|
13
|
+

|
14
|
+
|
15
|
+
|
16
|
+
## Prerequisite
|
17
|
+
* [Install Postgres](https://cocoindex.io/docs/getting_started/installation#-install-postgres) if you don't have one.
|
18
|
+
* [Install Neo4j](https://cocoindex.io/docs/ops/storages#neo4j) if you don't have one.
|
19
|
+
* [Configure your OpenAI API key](https://cocoindex.io/docs/ai/llm#openai).
|
20
|
+
|
21
|
+
## Documentation
|
22
|
+
You can read the official CocoIndex Documentation for Property Graph Targets [here](https://cocoindex.io/docs/ops/storages#property-graph-targets).
|
23
|
+
|
24
|
+
## Run
|
25
|
+
|
26
|
+
### Build the index
|
27
|
+
|
28
|
+
Install dependencies:
|
29
|
+
|
30
|
+
```bash
|
31
|
+
pip install -e .
|
32
|
+
```
|
33
|
+
|
34
|
+
Setup:
|
35
|
+
|
36
|
+
```bash
|
37
|
+
python main.py cocoindex setup
|
38
|
+
```
|
39
|
+
|
40
|
+
Update index:
|
41
|
+
|
42
|
+
```bash
|
43
|
+
python main.py cocoindex update
|
44
|
+
```
|
45
|
+
|
46
|
+
### Browse the knowledge graph
|
47
|
+
|
48
|
+
After the knowledge graph is build, you can explore the knowledge graph you built in Neo4j Browser.
|
49
|
+
|
50
|
+
For the dev enviroment, you can connect neo4j browser using credentials:
|
51
|
+
- username: `neo4j`
|
52
|
+
- password: `cocoindex`
|
53
|
+
which is pre-configured in the our docker compose [config.yaml](https://raw.githubusercontent.com/cocoindex-io/cocoindex/refs/heads/main/dev/neo4j.yaml).
|
54
|
+
|
55
|
+
You can open it at [http://localhost:7474](http://localhost:7474), and run the following Cypher query to get all relationships:
|
56
|
+
|
57
|
+
```cypher
|
58
|
+
MATCH p=()-->() RETURN p
|
59
|
+
```
|
60
|
+
<img width="1366" alt="neo4j-for-coco-docs" src="https://github.com/user-attachments/assets/3c8b6329-6fee-4533-9480-571399b57e57" />
|
61
|
+
|
62
|
+
|
63
|
+
|
64
|
+
## CocoInsight
|
65
|
+
I used CocoInsight (Free beta now) to troubleshoot the index generation and understand the data lineage of the pipeline.
|
66
|
+
It just connects to your local CocoIndex server, with Zero pipeline data retention. Run following command to start CocoInsight:
|
67
|
+
|
68
|
+
```bash
|
69
|
+
python3 main.py cocoindex server -c https://cocoindex.io
|
70
|
+
```
|
71
|
+
|
72
|
+
And then open the url https://cocoindex.io/cocoinsight.
|
73
|
+
|
74
|
+
```
|
75
|
+
python main.py cocoindex server -c https://cocoindex.io
|
76
|
+
```
|
77
|
+
|
78
|
+
<img width="1430" alt="cocoinsight" src="https://github.com/user-attachments/assets/d5ada581-cceb-42bf-a949-132df674f3dd" />
|
79
|
+
|
80
|
+
|
@@ -13,7 +13,10 @@ class DocumentSummary:
|
|
13
13
|
|
14
14
|
@dataclasses.dataclass
|
15
15
|
class Relationship:
|
16
|
-
"""
|
16
|
+
"""
|
17
|
+
Describe a relationship between two entities.
|
18
|
+
Subject and object should be Core CocoIndex concepts only, should be nouns. For example, `CocoIndex`, `Incremental Processing`, `ETL`, `Data` etc.
|
19
|
+
"""
|
17
20
|
subject: str
|
18
21
|
predicate: str
|
19
22
|
object: str
|
@@ -62,8 +65,8 @@ def docs_to_kg_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.D
|
|
62
65
|
output_type=list[Relationship],
|
63
66
|
instruction=(
|
64
67
|
"Please extract relationships from CocoIndex documents. "
|
65
|
-
"Focus on concepts and
|
66
|
-
|
68
|
+
"Focus on concepts and ignore examples and code. "
|
69
|
+
)))
|
67
70
|
|
68
71
|
with doc["relationships"].row() as relationship:
|
69
72
|
# relationship between two entities
|
@@ -90,35 +93,31 @@ def docs_to_kg_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.D
|
|
90
93
|
"document_node",
|
91
94
|
cocoindex.storages.Neo4j(
|
92
95
|
connection=conn_spec,
|
93
|
-
mapping=cocoindex.storages.
|
96
|
+
mapping=cocoindex.storages.Nodes(label="Document")),
|
94
97
|
primary_key_fields=["filename"],
|
95
98
|
)
|
96
99
|
# Declare reference Node to reference entity node in a relationship
|
97
100
|
flow_builder.declare(
|
98
|
-
cocoindex.storages.
|
101
|
+
cocoindex.storages.Neo4jDeclaration(
|
99
102
|
connection=conn_spec,
|
100
|
-
|
101
|
-
|
102
|
-
label="Entity",
|
103
|
-
primary_key_fields=["value"],
|
104
|
-
)
|
105
|
-
]
|
103
|
+
nodes_label="Entity",
|
104
|
+
primary_key_fields=["value"],
|
106
105
|
)
|
107
106
|
)
|
108
107
|
entity_relationship.export(
|
109
108
|
"entity_relationship",
|
110
109
|
cocoindex.storages.Neo4j(
|
111
110
|
connection=conn_spec,
|
112
|
-
mapping=cocoindex.storages.
|
111
|
+
mapping=cocoindex.storages.Relationships(
|
113
112
|
rel_type="RELATIONSHIP",
|
114
|
-
source=cocoindex.storages.
|
113
|
+
source=cocoindex.storages.NodeFromFields(
|
115
114
|
label="Entity",
|
116
115
|
fields=[
|
117
116
|
cocoindex.storages.TargetFieldMapping(
|
118
117
|
source="subject", target="value"),
|
119
118
|
]
|
120
119
|
),
|
121
|
-
target=cocoindex.storages.
|
120
|
+
target=cocoindex.storages.NodeFromFields(
|
122
121
|
label="Entity",
|
123
122
|
fields=[
|
124
123
|
cocoindex.storages.TargetFieldMapping(
|
@@ -133,13 +132,13 @@ def docs_to_kg_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.D
|
|
133
132
|
"entity_mention",
|
134
133
|
cocoindex.storages.Neo4j(
|
135
134
|
connection=conn_spec,
|
136
|
-
mapping=cocoindex.storages.
|
135
|
+
mapping=cocoindex.storages.Relationships(
|
137
136
|
rel_type="MENTION",
|
138
|
-
source=cocoindex.storages.
|
137
|
+
source=cocoindex.storages.NodeFromFields(
|
139
138
|
label="Document",
|
140
139
|
fields=[cocoindex.storages.TargetFieldMapping("filename")],
|
141
140
|
),
|
142
|
-
target=cocoindex.storages.
|
141
|
+
target=cocoindex.storages.NodeFromFields(
|
143
142
|
label="Entity",
|
144
143
|
fields=[cocoindex.storages.TargetFieldMapping(
|
145
144
|
source="entity", target="value")],
|
@@ -9,7 +9,7 @@ description = "With CocoIndex, users declare the transformation, CocoIndex creat
|
|
9
9
|
authors = [{ name = "CocoIndex", email = "cocoindex.io@gmail.com" }]
|
10
10
|
readme = "README.md"
|
11
11
|
requires-python = ">=3.11"
|
12
|
-
dependencies = ["sentence-transformers>=3.3.1", "click>=8.1.8"]
|
12
|
+
dependencies = ["sentence-transformers>=3.3.1", "click>=8.1.8", "rich>=14.0.0"]
|
13
13
|
license = "Apache-2.0"
|
14
14
|
urls = { Homepage = "https://cocoindex.io/" }
|
15
15
|
|
@@ -3,16 +3,19 @@ Auth registry is used to register and reference auth entries.
|
|
3
3
|
"""
|
4
4
|
|
5
5
|
from dataclasses import dataclass
|
6
|
+
from typing import Generic, TypeVar
|
6
7
|
|
7
8
|
from . import _engine
|
8
9
|
from .convert import dump_engine_object
|
9
10
|
|
11
|
+
T = TypeVar("T")
|
12
|
+
|
10
13
|
@dataclass
|
11
|
-
class AuthEntryReference:
|
14
|
+
class AuthEntryReference(Generic[T]):
|
12
15
|
"""Reference an auth entry by its key."""
|
13
16
|
key: str
|
14
17
|
|
15
|
-
def add_auth_entry(key: str, value) -> AuthEntryReference:
|
18
|
+
def add_auth_entry(key: str, value: T) -> AuthEntryReference[T]:
|
16
19
|
"""Add an auth entry to the registry. Returns its reference."""
|
17
20
|
_engine.add_auth_entry(key, dump_engine_object(value))
|
18
21
|
return AuthEntryReference(key)
|
@@ -1,8 +1,9 @@
|
|
1
|
-
import asyncio
|
2
1
|
import click
|
3
2
|
import datetime
|
4
3
|
|
5
|
-
from . import
|
4
|
+
from rich.console import Console
|
5
|
+
|
6
|
+
from . import flow, lib, setting
|
6
7
|
from .setup import sync_setup, drop_setup, flow_names_with_setup, apply_setup_changes
|
7
8
|
from .runtime import execution_context
|
8
9
|
|
@@ -20,7 +21,7 @@ def ls(show_all: bool):
|
|
20
21
|
"""
|
21
22
|
List all flows.
|
22
23
|
"""
|
23
|
-
current_flow_names =
|
24
|
+
current_flow_names = flow.flow_names()
|
24
25
|
persisted_flow_names = flow_names_with_setup()
|
25
26
|
remaining_persisted_flow_names = set(persisted_flow_names)
|
26
27
|
|
@@ -52,11 +53,14 @@ def ls(show_all: bool):
|
|
52
53
|
|
53
54
|
@cli.command()
|
54
55
|
@click.argument("flow_name", type=str, required=False)
|
55
|
-
|
56
|
+
@click.option("--color/--no-color", default=True)
|
57
|
+
def show(flow_name: str | None, color: bool):
|
56
58
|
"""
|
57
|
-
Show the flow spec.
|
59
|
+
Show the flow spec in a readable format with colored output.
|
58
60
|
"""
|
59
|
-
|
61
|
+
fl = _flow_by_name(flow_name)
|
62
|
+
console = Console(no_color=not color)
|
63
|
+
console.print(fl._render_text())
|
60
64
|
|
61
65
|
@cli.command()
|
62
66
|
def setup():
|
@@ -145,32 +149,59 @@ def evaluate(flow_name: str | None, output_dir: str | None, cache: bool = True):
|
|
145
149
|
options = flow.EvaluateAndDumpOptions(output_dir=output_dir, use_cache=cache)
|
146
150
|
fl.evaluate_and_dump(options)
|
147
151
|
|
148
|
-
|
152
|
+
# Create ServerSettings lazily upon first call, as environment variables may be loaded from files, etc.
|
153
|
+
COCOINDEX_HOST = 'https://cocoindex.io'
|
149
154
|
|
150
155
|
@cli.command()
|
151
156
|
@click.option(
|
152
|
-
"-a", "--address", type=str,
|
153
|
-
help="The address to bind the server to, in the format of IP:PORT."
|
157
|
+
"-a", "--address", type=str,
|
158
|
+
help="The address to bind the server to, in the format of IP:PORT. "
|
159
|
+
"If unspecified, the address specified in COCOINDEX_SERVER_ADDRESS will be used.")
|
160
|
+
@click.option(
|
161
|
+
"-c", "--cors-origin", type=str,
|
162
|
+
help="The origins of the clients (e.g. CocoInsight UI) to allow CORS from. "
|
163
|
+
"Multiple origins can be specified as a comma-separated list. "
|
164
|
+
"e.g. `https://cocoindex.io,http://localhost:3000`. "
|
165
|
+
"Origins specified in COCOINDEX_SERVER_CORS_ORIGINS will also be included.")
|
166
|
+
@click.option(
|
167
|
+
"-ci", "--cors-cocoindex", is_flag=True, show_default=True, default=False,
|
168
|
+
help=f"Allow {COCOINDEX_HOST} to access the server.")
|
154
169
|
@click.option(
|
155
|
-
"-
|
156
|
-
help="
|
157
|
-
"e.g. `http://cocoindex.io` if you want to allow CocoInsight to access the server.")
|
170
|
+
"-cl", "--cors-local", type=int,
|
171
|
+
help="Allow http://localhost:<port> to access the server.")
|
158
172
|
@click.option(
|
159
173
|
"-L", "--live-update", is_flag=True, show_default=True, default=False,
|
160
174
|
help="Continuously watch changes from data sources and apply to the target index.")
|
161
175
|
@click.option(
|
162
176
|
"-q", "--quiet", is_flag=True, show_default=True, default=False,
|
163
177
|
help="Avoid printing anything to the standard output, e.g. statistics.")
|
164
|
-
def server(address: str, live_update: bool, quiet: bool, cors_origin: str | None
|
178
|
+
def server(address: str | None, live_update: bool, quiet: bool, cors_origin: str | None,
|
179
|
+
cors_cocoindex: bool, cors_local: int | None):
|
165
180
|
"""
|
166
181
|
Start a HTTP server providing REST APIs.
|
167
182
|
|
168
183
|
It will allow tools like CocoInsight to access the server.
|
169
184
|
"""
|
170
|
-
|
185
|
+
server_settings = setting.ServerSettings.from_env()
|
186
|
+
cors_origins: set[str] = set(server_settings.cors_origins or [])
|
187
|
+
if cors_origin is not None:
|
188
|
+
cors_origins.update(setting.ServerSettings.parse_cors_origins(cors_origin))
|
189
|
+
if cors_cocoindex:
|
190
|
+
cors_origins.add(COCOINDEX_HOST)
|
191
|
+
if cors_local is not None:
|
192
|
+
cors_origins.add(f"http://localhost:{cors_local}")
|
193
|
+
server_settings.cors_origins = list(cors_origins)
|
194
|
+
|
195
|
+
if address is not None:
|
196
|
+
server_settings.address = address
|
197
|
+
|
198
|
+
lib.start_server(server_settings)
|
199
|
+
|
171
200
|
if live_update:
|
172
201
|
options = flow.FlowLiveUpdaterOptions(live_mode=True, print_stats=not quiet)
|
173
202
|
execution_context.run(flow.update_all_flows(options))
|
203
|
+
if COCOINDEX_HOST in cors_origins:
|
204
|
+
click.echo(f"Open CocoInsight at: {COCOINDEX_HOST}/cocoinsight")
|
174
205
|
input("Press Enter to stop...")
|
175
206
|
|
176
207
|
|