cocoindex 0.1.53__tar.gz → 0.1.54__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (252) hide show
  1. {cocoindex-0.1.53 → cocoindex-0.1.54}/.github/ISSUE_TEMPLATE//360/237/222/241-feature-request.md +1 -1
  2. {cocoindex-0.1.53 → cocoindex-0.1.54}/.github/scripts/update_version.sh +1 -1
  3. cocoindex-0.1.54/.pre-commit-config.yaml +71 -0
  4. {cocoindex-0.1.53 → cocoindex-0.1.54}/.vscode/settings.json +1 -1
  5. {cocoindex-0.1.53 → cocoindex-0.1.54}/CONTRIBUTING.md +1 -1
  6. {cocoindex-0.1.53 → cocoindex-0.1.54}/Cargo.lock +1 -1
  7. {cocoindex-0.1.53 → cocoindex-0.1.54}/Cargo.toml +1 -1
  8. {cocoindex-0.1.53 → cocoindex-0.1.54}/PKG-INFO +11 -10
  9. {cocoindex-0.1.53 → cocoindex-0.1.54}/README.md +9 -9
  10. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docs/about/contributing.md +19 -10
  11. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docs/ai/llm.mdx +3 -3
  12. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docs/core/basics.md +2 -2
  13. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docs/core/cli.mdx +1 -1
  14. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docs/core/custom_function.mdx +1 -1
  15. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docs/core/flow_def.mdx +1 -1
  16. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docs/core/flow_methods.mdx +3 -3
  17. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docs/core/settings.mdx +1 -1
  18. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docs/getting_started/installation.md +2 -3
  19. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docs/getting_started/overview.md +3 -4
  20. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docs/getting_started/quickstart.md +3 -3
  21. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docs/ops/functions.md +1 -1
  22. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docs/ops/sources.md +4 -4
  23. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docs/ops/targets.md +1 -1
  24. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docs/query.mdx +0 -1
  25. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/sidebars.ts +1 -1
  26. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/src/css/custom.css +7 -7
  27. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/src/theme/Root.js +1 -1
  28. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/static/robots.txt +1 -1
  29. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/amazon_s3_embedding/.env.example +1 -1
  30. cocoindex-0.1.54/examples/amazon_s3_embedding/.gitignore +1 -0
  31. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/amazon_s3_embedding/README.md +2 -2
  32. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/code_embedding/README.md +6 -7
  33. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/code_embedding/main.py +16 -4
  34. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/docs_to_knowledge_graph/README.md +4 -6
  35. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/fastapi_server_docker/README.md +2 -2
  36. {cocoindex-0.1.53/examples/text_embedding/markdown_files → cocoindex-0.1.54/examples/fastapi_server_docker/files}/1810.04805v2.md +1 -1
  37. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/fastapi_server_docker/requirements.txt +1 -1
  38. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/gdrive_text_embedding/.env.example +2 -2
  39. cocoindex-0.1.54/examples/gdrive_text_embedding/.gitignore +1 -0
  40. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/gdrive_text_embedding/README.md +4 -4
  41. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/image_search/.env +1 -1
  42. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/image_search/README.md +0 -1
  43. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/image_search/main.py +28 -26
  44. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/image_search/pyproject.toml +2 -0
  45. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/image_search/requirements.txt +1 -1
  46. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/manuals_llm_extraction/README.md +1 -1
  47. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/pdf_embedding/README.md +1 -1
  48. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/product_recommendation/README.md +4 -6
  49. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/product_recommendation/products/p1.json +1 -1
  50. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/product_recommendation/products/p2.json +1 -1
  51. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/product_recommendation/products/p3.json +1 -1
  52. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/product_recommendation/products/p4.json +1 -1
  53. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/product_recommendation/products/p6.json +1 -1
  54. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/product_recommendation/products/p7.json +1 -1
  55. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/product_recommendation/products/p8.json +1 -1
  56. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/product_recommendation/products/p9.json +1 -1
  57. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/text_embedding/README.md +3 -4
  58. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/text_embedding/main.py +11 -3
  59. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/text_embedding/markdown_files/1706.03762v7.md +1 -1
  60. {cocoindex-0.1.53/examples/fastapi_server_docker/files → cocoindex-0.1.54/examples/text_embedding/markdown_files}/1810.04805v2.md +1 -1
  61. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/text_embedding/markdown_files/rfc8259.md +1 -1
  62. cocoindex-0.1.54/examples/text_embedding_qdrant/.env +2 -0
  63. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/text_embedding_qdrant/README.md +3 -5
  64. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/text_embedding_qdrant/markdown_files/rfc8259.md +1 -1
  65. {cocoindex-0.1.53 → cocoindex-0.1.54}/pyproject.toml +2 -1
  66. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/cli.py +6 -6
  67. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/convert.py +80 -60
  68. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/flow.py +3 -2
  69. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/functions.py +10 -0
  70. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/llm.py +1 -0
  71. cocoindex-0.1.54/python/cocoindex/tests/__init__.py +0 -0
  72. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/tests/test_convert.py +134 -35
  73. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/tests/test_typing.py +26 -7
  74. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/typing.py +31 -12
  75. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/base/json_schema.rs +12 -0
  76. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/base/schema.rs +19 -0
  77. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/base/value.rs +60 -3
  78. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/execution/db_tracking_setup.rs +3 -3
  79. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/llm/anthropic.rs +13 -6
  80. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/llm/gemini.rs +81 -15
  81. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/llm/litellm.rs +4 -4
  82. cocoindex-0.1.54/src/llm/mod.rs +137 -0
  83. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/llm/ollama.rs +13 -10
  84. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/llm/openai.rs +46 -10
  85. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/llm/openrouter.rs +4 -4
  86. cocoindex-0.1.54/src/llm/voyage.rs +109 -0
  87. cocoindex-0.1.54/src/ops/functions/embed_text.rs +97 -0
  88. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/functions/extract_by_llm.rs +5 -1
  89. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/functions/mod.rs +1 -0
  90. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/registration.rs +1 -0
  91. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/sdk.rs +1 -0
  92. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/targets/kuzu.rs +2 -2
  93. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/targets/neo4j.rs +11 -0
  94. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/targets/postgres.rs +7 -0
  95. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/py/convert.rs +25 -0
  96. cocoindex-0.1.53/check.sh +0 -12
  97. cocoindex-0.1.53/examples/amazon_s3_embedding/.gitignore +0 -1
  98. cocoindex-0.1.53/examples/gdrive_text_embedding/.gitignore +0 -1
  99. cocoindex-0.1.53/examples/product_recommendation/.env +0 -3
  100. cocoindex-0.1.53/python/cocoindex/tests/__init__.py +0 -1
  101. cocoindex-0.1.53/src/llm/mod.rs +0 -88
  102. {cocoindex-0.1.53 → cocoindex-0.1.54}/.cargo/config.toml +0 -0
  103. {cocoindex-0.1.53 → cocoindex-0.1.54}/.env.lib_debug +0 -0
  104. {cocoindex-0.1.53 → cocoindex-0.1.54}/.github/ISSUE_TEMPLATE//360/237/220/233-bug-report.md" +0 -0
  105. {cocoindex-0.1.53 → cocoindex-0.1.54}/.github/workflows/CI.yml +0 -0
  106. {cocoindex-0.1.53 → cocoindex-0.1.54}/.github/workflows/_doc_release.yml +0 -0
  107. {cocoindex-0.1.53 → cocoindex-0.1.54}/.github/workflows/_test.yml +0 -0
  108. {cocoindex-0.1.53 → cocoindex-0.1.54}/.github/workflows/docs.yml +0 -0
  109. {cocoindex-0.1.53 → cocoindex-0.1.54}/.github/workflows/release.yml +0 -0
  110. {cocoindex-0.1.53 → cocoindex-0.1.54}/.gitignore +0 -0
  111. {cocoindex-0.1.53 → cocoindex-0.1.54}/CODE_OF_CONDUCT.md +0 -0
  112. {cocoindex-0.1.53 → cocoindex-0.1.54}/LICENSE +0 -0
  113. {cocoindex-0.1.53 → cocoindex-0.1.54}/dev/neo4j.yaml +0 -0
  114. {cocoindex-0.1.53 → cocoindex-0.1.54}/dev/postgres.yaml +0 -0
  115. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/.gitignore +0 -0
  116. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/README.md +0 -0
  117. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docs/about/community.md +0 -0
  118. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docs/core/data_example.svg +0 -0
  119. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docs/core/data_types.mdx +0 -0
  120. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docs/core/flow_example.svg +0 -0
  121. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docs/getting_started/markdown_files.zip +0 -0
  122. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/docusaurus.config.ts +0 -0
  123. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/package.json +0 -0
  124. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/src/components/HomepageFeatures/index.tsx +0 -0
  125. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/src/components/HomepageFeatures/styles.module.css +0 -0
  126. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/static/.nojekyll +0 -0
  127. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/static/img/docusaurus.png +0 -0
  128. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/static/img/favicon.ico +0 -0
  129. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/static/img/icon.svg +0 -0
  130. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/static/img/incremental-etl.gif +0 -0
  131. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/tsconfig.json +0 -0
  132. {cocoindex-0.1.53 → cocoindex-0.1.54}/docs/yarn.lock +0 -0
  133. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/amazon_s3_embedding/main.py +0 -0
  134. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/amazon_s3_embedding/pyproject.toml +0 -0
  135. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/code_embedding/.env +0 -0
  136. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/code_embedding/pyproject.toml +0 -0
  137. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/docs_to_knowledge_graph/.env +0 -0
  138. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/docs_to_knowledge_graph/main.py +0 -0
  139. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/docs_to_knowledge_graph/pyproject.toml +0 -0
  140. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/fastapi_server_docker/.dockerignore +0 -0
  141. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/fastapi_server_docker/.env +0 -0
  142. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/fastapi_server_docker/compose.yaml +0 -0
  143. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/fastapi_server_docker/dockerfile +0 -0
  144. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/fastapi_server_docker/main.py +0 -0
  145. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/gdrive_text_embedding/main.py +0 -0
  146. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/gdrive_text_embedding/pyproject.toml +0 -0
  147. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/image_search/frontend/.gitignore +0 -0
  148. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/image_search/frontend/index.html +0 -0
  149. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/image_search/frontend/package-lock.json +0 -0
  150. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/image_search/frontend/package.json +0 -0
  151. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/image_search/frontend/src/App.jsx +0 -0
  152. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/image_search/frontend/src/main.jsx +0 -0
  153. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/image_search/frontend/src/style.css +0 -0
  154. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/image_search/frontend/vite.config.js +0 -0
  155. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/image_search/img/cat1.jpeg +0 -0
  156. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/image_search/img/dog1.jpeg +0 -0
  157. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/image_search/img/elephant1.jpg +0 -0
  158. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/image_search/img/giraffe.jpg +0 -0
  159. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/manuals_llm_extraction/.env +0 -0
  160. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/manuals_llm_extraction/main.py +0 -0
  161. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/manuals_llm_extraction/manuals/array.pdf +0 -0
  162. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/manuals_llm_extraction/manuals/base64.pdf +0 -0
  163. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/manuals_llm_extraction/manuals/copy.pdf +0 -0
  164. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/manuals_llm_extraction/manuals/glob.pdf +0 -0
  165. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/manuals_llm_extraction/pyproject.toml +0 -0
  166. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/pdf_embedding/.env +0 -0
  167. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/pdf_embedding/main.py +0 -0
  168. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/pdf_embedding/pdf_files/1706.03762v7.pdf +0 -0
  169. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/pdf_embedding/pdf_files/1810.04805v2.pdf +0 -0
  170. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/pdf_embedding/pdf_files/rfc8259.pdf +0 -0
  171. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/pdf_embedding/pyproject.toml +0 -0
  172. {cocoindex-0.1.53/examples/text_embedding → cocoindex-0.1.54/examples/product_recommendation}/.env +0 -0
  173. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/product_recommendation/img/cocoinsight.png +0 -0
  174. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/product_recommendation/img/neo4j.png +0 -0
  175. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/product_recommendation/main.py +0 -0
  176. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/product_recommendation/products/p5.json +0 -0
  177. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/product_recommendation/pyproject.toml +0 -0
  178. {cocoindex-0.1.53/examples/text_embedding_qdrant → cocoindex-0.1.54/examples/text_embedding}/.env +0 -0
  179. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/text_embedding/Text_Embedding.ipynb +0 -0
  180. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/text_embedding/pyproject.toml +0 -0
  181. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/text_embedding_qdrant/main.py +0 -0
  182. {cocoindex-0.1.53 → cocoindex-0.1.54}/examples/text_embedding_qdrant/pyproject.toml +0 -0
  183. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/__init__.py +0 -0
  184. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/auth_registry.py +0 -0
  185. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/index.py +0 -0
  186. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/lib.py +0 -0
  187. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/op.py +0 -0
  188. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/py.typed +0 -0
  189. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/runtime.py +0 -0
  190. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/setting.py +0 -0
  191. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/setup.py +0 -0
  192. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/sources.py +0 -0
  193. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/targets.py +0 -0
  194. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/tests/test_optional_database.py +0 -0
  195. {cocoindex-0.1.53 → cocoindex-0.1.54}/python/cocoindex/utils.py +0 -0
  196. {cocoindex-0.1.53 → cocoindex-0.1.54}/ruff.toml +0 -0
  197. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/base/duration.rs +0 -0
  198. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/base/field_attrs.rs +0 -0
  199. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/base/mod.rs +0 -0
  200. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/base/spec.rs +0 -0
  201. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/builder/analyzed_flow.rs +0 -0
  202. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/builder/analyzer.rs +0 -0
  203. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/builder/flow_builder.rs +0 -0
  204. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/builder/mod.rs +0 -0
  205. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/builder/plan.rs +0 -0
  206. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/execution/db_tracking.rs +0 -0
  207. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/execution/dumper.rs +0 -0
  208. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/execution/evaluator.rs +0 -0
  209. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/execution/indexing_status.rs +0 -0
  210. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/execution/live_updater.rs +0 -0
  211. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/execution/memoization.rs +0 -0
  212. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/execution/mod.rs +0 -0
  213. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/execution/row_indexer.rs +0 -0
  214. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/execution/source_indexer.rs +0 -0
  215. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/execution/stats.rs +0 -0
  216. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/lib.rs +0 -0
  217. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/lib_context.rs +0 -0
  218. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/factory_bases.rs +0 -0
  219. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/functions/parse_json.rs +0 -0
  220. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/functions/split_recursively.rs +0 -0
  221. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/interface.rs +0 -0
  222. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/mod.rs +0 -0
  223. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/py_factory.rs +0 -0
  224. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/registry.rs +0 -0
  225. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/sources/amazon_s3.rs +0 -0
  226. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/sources/google_drive.rs +0 -0
  227. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/sources/local_file.rs +0 -0
  228. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/sources/mod.rs +0 -0
  229. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/targets/mod.rs +0 -0
  230. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/targets/qdrant.rs +0 -0
  231. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/targets/shared/mod.rs +0 -0
  232. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/targets/shared/property_graph.rs +0 -0
  233. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/ops/targets/shared/table_columns.rs +0 -0
  234. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/prelude.rs +0 -0
  235. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/py/mod.rs +0 -0
  236. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/server.rs +0 -0
  237. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/service/error.rs +0 -0
  238. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/service/flows.rs +0 -0
  239. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/service/mod.rs +0 -0
  240. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/settings.rs +0 -0
  241. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/setup/auth_registry.rs +0 -0
  242. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/setup/components.rs +0 -0
  243. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/setup/db_metadata.rs +0 -0
  244. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/setup/driver.rs +0 -0
  245. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/setup/mod.rs +0 -0
  246. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/setup/states.rs +0 -0
  247. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/utils/db.rs +0 -0
  248. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/utils/fingerprint.rs +0 -0
  249. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/utils/immutable.rs +0 -0
  250. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/utils/mod.rs +0 -0
  251. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/utils/retryable.rs +0 -0
  252. {cocoindex-0.1.53 → cocoindex-0.1.54}/src/utils/yaml_ser.rs +0 -0
@@ -17,4 +17,4 @@ assignees: ''
17
17
 
18
18
  ---
19
19
  ❤️ Contributors, please refer to 📙[Contributing Guide](https://cocoindex.io/docs/about/contributing).
20
- Unless the PR can be sent immediately (e.g. just a few lines of code), we recommend you to leave a comment on the issue like **`I'm working on it`** or **`Can I work on this issue?`** to avoid duplicating work. Our [Discord server](https://discord.com/invite/zpA9S2DR7s) is always open and friendly.
20
+ Unless the PR can be sent immediately (e.g. just a few lines of code), we recommend you to leave a comment on the issue like **`I'm working on it`** or **`Can I work on this issue?`** to avoid duplicating work. Our [Discord server](https://discord.com/invite/zpA9S2DR7s) is always open and friendly.
@@ -19,4 +19,4 @@ else
19
19
  fi
20
20
 
21
21
  # Update Cargo.toml
22
- sed "${SED_INLINE[@]}" "s/^version = .*/version = \"$VERSION\"/" Cargo.toml
22
+ sed "${SED_INLINE[@]}" "s/^version = .*/version = \"$VERSION\"/" Cargo.toml
@@ -0,0 +1,71 @@
1
+ ci:
2
+ autofix_prs: false
3
+ autoupdate_schedule: 'monthly'
4
+
5
+ repos:
6
+ - repo: https://github.com/pre-commit/pre-commit-hooks
7
+ rev: v5.0.0
8
+ hooks:
9
+ - id: check-case-conflict
10
+ # Check for files with names that would conflict on a case-insensitive
11
+ # filesystem like MacOS HFS+ or Windows FAT.
12
+ - id: check-merge-conflict
13
+ # Check for files that contain merge conflict strings.
14
+ - id: check-symlinks
15
+ # Checks for symlinks which do not point to anything.
16
+ exclude: ".*(.github.*)$"
17
+ - id: detect-private-key
18
+ # Checks for the existence of private keys.
19
+ - id: end-of-file-fixer
20
+ # Makes sure files end in a newline and only a newline.
21
+ exclude: ".*(data.*|licenses.*|_static.*|\\.ya?ml|\\.jpe?g|\\.png|\\.svg|\\.webp)$"
22
+ - id: trailing-whitespace
23
+ # Trims trailing whitespace.
24
+ exclude_types: [python] # Covered by Ruff W291.
25
+ exclude: ".*(data.*|licenses.*|_static.*|\\.ya?ml|\\.jpe?g|\\.png|\\.svg|\\.webp)$"
26
+
27
+ - repo: local
28
+ hooks:
29
+ - id: maturin-develop
30
+ name: maturin develop
31
+ entry: maturin develop
32
+ language: system
33
+ files: ^(python/|src/|Cargo\.toml|pyproject\.toml)
34
+ pass_filenames: false
35
+
36
+ - id: cargo-fmt
37
+ name: cargo fmt
38
+ entry: cargo fmt
39
+ language: system
40
+ types: [rust]
41
+ pass_filenames: false
42
+
43
+ - id: cargo-test
44
+ name: cargo test
45
+ entry: cargo test
46
+ language: system
47
+ types: [rust]
48
+ pass_filenames: false
49
+
50
+ - id: mypy-check
51
+ name: mypy type check
52
+ entry: mypy
53
+ language: system
54
+ types: [python]
55
+ pass_filenames: false
56
+
57
+ - repo: https://github.com/astral-sh/ruff-pre-commit
58
+ rev: v0.12.0
59
+ hooks:
60
+ - id: ruff-format
61
+ types: [python]
62
+ pass_filenames: true
63
+
64
+ - repo: https://github.com/christophmeissner/pytest-pre-commit
65
+ rev: 1.0.0
66
+ hooks:
67
+ - id: pytest
68
+ language: system
69
+ types: [python]
70
+ pass_filenames: false
71
+ always_run: false
@@ -6,4 +6,4 @@
6
6
  ],
7
7
  "editor.formatOnSave": true,
8
8
  "python.formatting.provider": "ruff"
9
- }
9
+ }
@@ -1 +1 @@
1
- We love contributions from our community ❤️. Please check out our [contributing guide](https://cocoindex.io/docs/about/contributing).
1
+ We love contributions from our community ❤️. Please check out our [contributing guide](https://cocoindex.io/docs/about/contributing).
@@ -1040,7 +1040,7 @@ dependencies = [
1040
1040
 
1041
1041
  [[package]]
1042
1042
  name = "cocoindex"
1043
- version = "0.1.53"
1043
+ version = "0.1.54"
1044
1044
  dependencies = [
1045
1045
  "anyhow",
1046
1046
  "async-openai",
@@ -2,7 +2,7 @@
2
2
  name = "cocoindex"
3
3
  # Version used for local development is always higher than others to take precedence.
4
4
  # Will be overridden for specific release versions.
5
- version = "0.1.53"
5
+ version = "0.1.54"
6
6
  edition = "2024"
7
7
  rust-version = "1.86"
8
8
 
@@ -1,12 +1,13 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cocoindex
3
- Version: 0.1.53
3
+ Version: 0.1.54
4
4
  Requires-Dist: sentence-transformers>=3.3.1
5
5
  Requires-Dist: click>=8.1.8
6
6
  Requires-Dist: rich>=14.0.0
7
7
  Requires-Dist: python-dotenv>=1.1.0
8
8
  Requires-Dist: pytest ; extra == 'test'
9
9
  Requires-Dist: ruff ; extra == 'dev'
10
+ Requires-Dist: pre-commit ; extra == 'dev'
10
11
  Provides-Extra: test
11
12
  Provides-Extra: dev
12
13
  License-File: LICENSE
@@ -51,10 +52,10 @@ Unlike a workflow orchestration framework where data is usually opaque, in CocoI
51
52
 
52
53
  ```python
53
54
  # import
54
- data['content'] = flow_builder.add_source(...)
55
+ data['content'] = flow_builder.add_source(...)
55
56
 
56
57
  # transform
57
- data['out'] = data['content']
58
+ data['out'] = data['content']
58
59
  .transform(...)
59
60
  .transform(...)
60
61
 
@@ -75,17 +76,17 @@ As a data framework, CocoIndex takes it to the next level on data freshness. **I
75
76
  The frameworks takes care of
76
77
  - Change data capture.
77
78
  - Figure out what exactly needs to be updated, and only updating that without having to recompute everything.
78
-
79
+
79
80
  This makes it fast to reflect any source updates to the target store. If you have concerns with surfacing stale data to AI agents and are spending lots of efforts working on infra piece to optimize the latency, the framework actually handles it for you.
80
81
 
81
82
 
82
83
  ## Quick Start:
83
- If you're new to CocoIndex, we recommend checking out
84
+ If you're new to CocoIndex, we recommend checking out
84
85
  - 📖 [Documentation](https://cocoindex.io/docs)
85
86
  - ⚡ [Quick Start Guide](https://cocoindex.io/docs/getting_started/quickstart)
86
- - 🎬 [Quick Start Video Tutorial](https://youtu.be/gv5R8nOXsWU?si=9ioeKYkMEnYevTXT)
87
+ - 🎬 [Quick Start Video Tutorial](https://youtu.be/gv5R8nOXsWU?si=9ioeKYkMEnYevTXT)
87
88
 
88
- ### Setup
89
+ ### Setup
89
90
 
90
91
  1. Install CocoIndex Python library
91
92
 
@@ -155,8 +156,8 @@ It defines an index flow like this:
155
156
  | [Google Drive Text Embedding](examples/gdrive_text_embedding) | Index text documents from Google Drive |
156
157
  | [Docs to Knowledge Graph](examples/docs_to_knowledge_graph) | Extract relationships from Markdown documents and build a knowledge graph |
157
158
  | [Embeddings to Qdrant](examples/text_embedding_qdrant) | Index documents in a Qdrant collection for semantic search |
158
- | [FastAPI Server with Docker](examples/fastapi_server_docker) | Run the semantic search server in a Dockerized FastAPI setup |
159
- | [Product Recommendation](examples/product_recommendation) | Build real-time product recommendations with LLM and graph database|
159
+ | [FastAPI Server with Docker](examples/fastapi_server_docker) | Run the semantic search server in a Dockerized FastAPI setup |
160
+ | [Product Recommendation](examples/product_recommendation) | Build real-time product recommendations with LLM and graph database|
160
161
  | [Image Search with Vision API](examples/image_search) | Generates detailed captions for images using a vision model, embeds them, enables live-updating semantic search via FastAPI and served on a React frontend|
161
162
 
162
163
  More coming and stay tuned 👀!
@@ -178,7 +179,7 @@ Join our community here:
178
179
  - 📜 [Read our blog posts](https://cocoindex.io/blogs/)
179
180
 
180
181
  ## Support us:
181
- We are constantly improving, and more features and examples are coming soon. If you love this project, please drop us a star ⭐ at GitHub repo [![GitHub](https://img.shields.io/github/stars/cocoindex-io/cocoindex?color=5B5BD6)](https://github.com/cocoindex-io/cocoindex) to stay tuned and help us grow.
182
+ We are constantly improving, and more features and examples are coming soon. If you love this project, please drop us a star ⭐ at GitHub repo [![GitHub](https://img.shields.io/github/stars/cocoindex-io/cocoindex?color=5B5BD6)](https://github.com/cocoindex-io/cocoindex) to stay tuned and help us grow.
182
183
 
183
184
  ## License
184
185
  CocoIndex is Apache 2.0 licensed.
@@ -32,10 +32,10 @@ Unlike a workflow orchestration framework where data is usually opaque, in CocoI
32
32
 
33
33
  ```python
34
34
  # import
35
- data['content'] = flow_builder.add_source(...)
35
+ data['content'] = flow_builder.add_source(...)
36
36
 
37
37
  # transform
38
- data['out'] = data['content']
38
+ data['out'] = data['content']
39
39
  .transform(...)
40
40
  .transform(...)
41
41
 
@@ -56,17 +56,17 @@ As a data framework, CocoIndex takes it to the next level on data freshness. **I
56
56
  The frameworks takes care of
57
57
  - Change data capture.
58
58
  - Figure out what exactly needs to be updated, and only updating that without having to recompute everything.
59
-
59
+
60
60
  This makes it fast to reflect any source updates to the target store. If you have concerns with surfacing stale data to AI agents and are spending lots of efforts working on infra piece to optimize the latency, the framework actually handles it for you.
61
61
 
62
62
 
63
63
  ## Quick Start:
64
- If you're new to CocoIndex, we recommend checking out
64
+ If you're new to CocoIndex, we recommend checking out
65
65
  - 📖 [Documentation](https://cocoindex.io/docs)
66
66
  - ⚡ [Quick Start Guide](https://cocoindex.io/docs/getting_started/quickstart)
67
- - 🎬 [Quick Start Video Tutorial](https://youtu.be/gv5R8nOXsWU?si=9ioeKYkMEnYevTXT)
67
+ - 🎬 [Quick Start Video Tutorial](https://youtu.be/gv5R8nOXsWU?si=9ioeKYkMEnYevTXT)
68
68
 
69
- ### Setup
69
+ ### Setup
70
70
 
71
71
  1. Install CocoIndex Python library
72
72
 
@@ -136,8 +136,8 @@ It defines an index flow like this:
136
136
  | [Google Drive Text Embedding](examples/gdrive_text_embedding) | Index text documents from Google Drive |
137
137
  | [Docs to Knowledge Graph](examples/docs_to_knowledge_graph) | Extract relationships from Markdown documents and build a knowledge graph |
138
138
  | [Embeddings to Qdrant](examples/text_embedding_qdrant) | Index documents in a Qdrant collection for semantic search |
139
- | [FastAPI Server with Docker](examples/fastapi_server_docker) | Run the semantic search server in a Dockerized FastAPI setup |
140
- | [Product Recommendation](examples/product_recommendation) | Build real-time product recommendations with LLM and graph database|
139
+ | [FastAPI Server with Docker](examples/fastapi_server_docker) | Run the semantic search server in a Dockerized FastAPI setup |
140
+ | [Product Recommendation](examples/product_recommendation) | Build real-time product recommendations with LLM and graph database|
141
141
  | [Image Search with Vision API](examples/image_search) | Generates detailed captions for images using a vision model, embeds them, enables live-updating semantic search via FastAPI and served on a React frontend|
142
142
 
143
143
  More coming and stay tuned 👀!
@@ -159,7 +159,7 @@ Join our community here:
159
159
  - 📜 [Read our blog posts](https://cocoindex.io/blogs/)
160
160
 
161
161
  ## Support us:
162
- We are constantly improving, and more features and examples are coming soon. If you love this project, please drop us a star ⭐ at GitHub repo [![GitHub](https://img.shields.io/github/stars/cocoindex-io/cocoindex?color=5B5BD6)](https://github.com/cocoindex-io/cocoindex) to stay tuned and help us grow.
162
+ We are constantly improving, and more features and examples are coming soon. If you love this project, please drop us a star ⭐ at GitHub repo [![GitHub](https://img.shields.io/github/stars/cocoindex-io/cocoindex?color=5B5BD6)](https://github.com/cocoindex-io/cocoindex) to stay tuned and help us grow.
163
163
 
164
164
  ## License
165
165
  CocoIndex is Apache 2.0 licensed.
@@ -15,22 +15,22 @@ We use [GitHub Issues](https://github.com/cocoindex-io/cocoindex/issues) to trac
15
15
 
16
16
  We tag issues with the ["good first issue"](https://github.com/cocoindex-io/cocoindex/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) label for beginner contributors.
17
17
 
18
- ## How to Contribute
18
+ ## How to Contribute
19
19
  - If you decide to work on an issue, unless the PR can be sent immediately (e.g. just a few lines of code), we recommend you to leave a comment on the issue like **`I'm working on it`** or **`Can I work on this issue?`** to avoid duplicating work.
20
20
  - For larger features, we recommend you to discuss with us first in our [Discord server](https://discord.com/invite/zpA9S2DR7s) to coordinate the design and work.
21
21
  - Our [Discord server](https://discord.com/invite/zpA9S2DR7s) are constantly open. If you are unsure about anything, it is a good place to discuss! We'd love to collaborate and will always be friendly.
22
22
 
23
- ## Start hacking! Setting Up Development Environment
23
+ ## Start hacking! Setting Up Development Environment
24
24
  Following the steps below to get cocoindex build on latest codebase locally - if you are making changes to cocoindex funcionality and want to test it out.
25
25
 
26
26
  - 🦀 [Install Rust](https://rust-lang.org/tools/install)
27
-
27
+
28
28
  If you don't have Rust installed, run
29
29
  ```sh
30
30
  curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
31
31
  ```
32
- Already have Rust? Make sure it's up to date
33
- ```sh
32
+ Already have Rust? Make sure it's up to date
33
+ ```sh
34
34
  rustup update
35
35
  ```
36
36
 
@@ -46,7 +46,7 @@ Following the steps below to get cocoindex build on latest codebase locally - if
46
46
 
47
47
  - Install required tools:
48
48
  ```sh
49
- pip install maturin mypy ruff
49
+ pip install maturin mypy pre-commit
50
50
  ```
51
51
 
52
52
  - Build the library. Run at the root of cocoindex directory:
@@ -54,6 +54,11 @@ Following the steps below to get cocoindex build on latest codebase locally - if
54
54
  maturin develop
55
55
  ```
56
56
 
57
+ - Install and enable pre-commit hooks. This ensures all checks run automatically before each commit:
58
+ ```sh
59
+ pre-commit install
60
+ ```
61
+
57
62
  - Before running a specific example, set extra environment variables, for exposing extra traces, allowing dev UI, etc.
58
63
  ```sh
59
64
  . ./.env.lib_debug
@@ -67,10 +72,14 @@ To submit your code:
67
72
  1. Fork the [CocoIndex repository](https://github.com/cocoindex-io/cocoindex)
68
73
  2. [Create a new branch](https://docs.github.com/en/desktop/making-changes-in-a-branch/managing-branches-in-github-desktop) on your fork
69
74
  3. Make your changes
70
- 4. Make sure all tests and linting pass by running
71
- ```sh
72
- ./check.sh
73
- ```
75
+ 4. Run the pre-commit checks (automatically triggered on `git commit`)
76
+
77
+ :::tip
78
+ To run them manually (same as CI):
79
+ ```sh
80
+ pre-commit run --all-files
81
+ ```
82
+ :::
74
83
 
75
84
  5. [Open a Pull Request (PR)](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork) when your work is ready for review
76
85
 
@@ -136,9 +136,9 @@ pip install 'litellm[proxy]'
136
136
  **Example for OpenAI:**
137
137
  ```yaml
138
138
  model_list:
139
- - model_name: "*"
139
+ - model_name: "*"
140
140
  litellm_params:
141
- model: openai/*
141
+ model: openai/*
142
142
  api_key: os.environ/LITELLM_API_KEY
143
143
  ```
144
144
 
@@ -176,7 +176,7 @@ litellm --config config.yml
176
176
  ```python
177
177
  cocoindex.LlmSpec(
178
178
  api_type=cocoindex.LlmApiType.LITE_LLM,
179
- model="deepseek-r1",
179
+ model="deepseek-r1",
180
180
  address="http://127.0.0.1:4000", # default url of LiteLLM
181
181
  )
182
182
  ```
@@ -71,7 +71,7 @@ An indexing flow, once set up, maintains a long-lived relationship between data
71
71
 
72
72
  * **One time update**: Once triggered, CocoIndex updates the target data to reflect the version of source data up to the current moment.
73
73
  * **Live update**: CocoIndex continuously reacts to changes of source data and updates the target data accordingly, based on various **change capture mechanisms** for the source.
74
-
74
+
75
75
  See more details in the [build / update target data](flow_methods#build--update-target-data) section.
76
76
 
77
77
  3. CocoIndex intelligently reprocesses to propagate source changes to target by:
@@ -101,4 +101,4 @@ As an indexing flow is long-lived, it needs to store intermediate data to keep t
101
101
  CocoIndex uses internal storage for this purpose.
102
102
 
103
103
  Currently, CocoIndex uses Postgres database as the internal storage.
104
- See [Settings](settings#databaseconnectionspec) for configuring its location, and `cocoindex setup` CLI command (see [CocoIndex CLI](cli)) creates tables for the internal storage.
104
+ See [Settings](settings#databaseconnectionspec) for configuring its location, and `cocoindex setup` CLI command (see [CocoIndex CLI](cli)) creates tables for the internal storage.
@@ -72,4 +72,4 @@ Use `--help` to see the full list of subcommands, and `subcommand --help` to see
72
72
  ```sh
73
73
  cocoindex --help # Show all subcommands
74
74
  cocoindex show --help # Show usage of "show" subcommand
75
- ```
75
+ ```
@@ -133,7 +133,7 @@ The cocoindex repository contains the following examples of custom functions def
133
133
  * In the [pdf_embedding](https://github.com/cocoindex-io/cocoindex/blob/main/examples/pdf_embedding/main.py) example, we define a custom function `PdfToMarkdown`
134
134
  * The `SentenceTransformerEmbed` function shipped with the CocoIndex Python package is defined by Python SDK.
135
135
  Search for [`SentenceTransformerEmbedExecutor`](https://github.com/search?q=repo%3Acocoindex-io%2Fcocoindex+lang%3Apython+SentenceTransformerEmbedExecutor&type=code) to see the code.
136
-
136
+
137
137
  ## Parameters for custom functions
138
138
 
139
139
  Custom functions take the following additional parameters:
@@ -56,7 +56,7 @@ A data scope has a bunch of fields and collectors, and users can add new fields
56
56
 
57
57
  ### Get or Add a Field
58
58
 
59
- You can get or add a field of a data scope (which is a data slice).
59
+ You can get or add a field of a data scope (which is a data slice).
60
60
 
61
61
  :::note
62
62
 
@@ -182,10 +182,10 @@ CocoIndex also provides asynchronous versions of APIs for blocking operations, i
182
182
  my_updater = cocoindex.FlowLiveUpdater(demo_flow)
183
183
  # Start the updater.
184
184
  await my_updater.start_async()
185
-
185
+
186
186
  # Perform your own logic (e.g. a query loop).
187
187
  ...
188
-
188
+
189
189
  # Print the update stats.
190
190
  print(my_updater.update_stats())
191
191
  # Abort the updater.
@@ -245,4 +245,4 @@ demo_flow.evaluate_and_dump(EvaluateAndDumpOptions(output_dir="./eval_output"))
245
245
  ```
246
246
 
247
247
  </TabItem>
248
- </Tabs>
248
+ </Tabs>
@@ -113,4 +113,4 @@ This is the list of environment variables, each of which has a corresponding fie
113
113
  | `COCOINDEX_DATABASE_URL` | `database.url` | Yes |
114
114
  | `COCOINDEX_DATABASE_USER` | `database.user` | No |
115
115
  | `COCOINDEX_DATABASE_PASSWORD` | `database.password` | No |
116
- | `COCOINDEX_APP_NAMESPACE` | `app_namespace` | No |
116
+ | `COCOINDEX_APP_NAMESPACE` | `app_namespace` | No |
@@ -1,5 +1,5 @@
1
1
  ---
2
- title: Installation
2
+ title: Installation
3
3
  description: Setup the CocoIndex environment in 0-3 min
4
4
  ---
5
5
 
@@ -17,7 +17,7 @@ pip install -U cocoindex
17
17
 
18
18
  ## 📦 Install Postgres
19
19
 
20
- You can skip this step if you already have a Postgres database with pgvector extension installed.
20
+ You can skip this step if you already have a Postgres database with pgvector extension installed.
21
21
 
22
22
  If you don't have a Postgres database:
23
23
 
@@ -31,4 +31,3 @@ docker compose -f <(curl -L https://raw.githubusercontent.com/cocoindex-io/cocoi
31
31
  ## 🎉 All set!
32
32
 
33
33
  You can now start using CocoIndex.
34
-
@@ -5,7 +5,7 @@ slug: /
5
5
 
6
6
  # Welcome to CocoIndex
7
7
 
8
- CocoIndex is an ultra-performant real-time data transformation framework for AI, with incremental processing.
8
+ CocoIndex is an ultra-performant real-time data transformation framework for AI, with incremental processing.
9
9
 
10
10
  As a data framework, CocoIndex takes it to the next level on data freshness. **Incremental processing** is one of the core values provided by CocoIndex.
11
11
 
@@ -17,10 +17,10 @@ CocoIndex follows the idea of [Dataflow programming](https://en.wikipedia.org/wi
17
17
  The gist of an example data transformation:
18
18
  ```python
19
19
  # import
20
- data['content'] = flow_builder.add_source(...)
20
+ data['content'] = flow_builder.add_source(...)
21
21
 
22
22
  # transform
23
- data['out'] = data['content']
23
+ data['out'] = data['content']
24
24
  .transform(...)
25
25
  .transform(...)
26
26
 
@@ -33,4 +33,3 @@ collector.export(...)
33
33
 
34
34
  Get Started:
35
35
  - [Quick Start](https://cocoindex.io/docs/getting_started/quickstart)
36
-
@@ -19,7 +19,7 @@ This guide will help you get up and running with CocoIndex in just a few minutes
19
19
  We'll need to install a bunch of dependencies for this project.
20
20
 
21
21
  1. Install CocoIndex:
22
-
22
+
23
23
  ```bash
24
24
  pip install -U cocoindex
25
25
  ```
@@ -149,7 +149,7 @@ documents: 3 added, 0 removed, 0 updated
149
149
 
150
150
  ## Step 4 (optional): Run queries against the index
151
151
 
152
- CocoIndex excels at transforming your data and storing it (a.k.a. indexing).
152
+ CocoIndex excels at transforming your data and storing it (a.k.a. indexing).
153
153
  The goal of transforming your data is usually to query against it.
154
154
  Once you already have your index built, you can directly access the transformed data in the target database.
155
155
  CocoIndex also provides utilities for you to do this more seamlessly.
@@ -291,4 +291,4 @@ Next, you may want to:
291
291
  * Learn about [CocoIndex Basics](../core/basics.md).
292
292
  * Learn about other examples in the [examples](https://github.com/cocoindex-io/cocoindex/tree/main/examples) directory.
293
293
  * The `text_embedding` example is this quickstart.
294
- * Pick other examples to learn upon your interest.
294
+ * Pick other examples to learn upon your interest.
@@ -53,7 +53,7 @@ Input data:
53
53
  :::note
54
54
 
55
55
  We use the `language` field to determine how to split the input text, following these rules:
56
-
56
+
57
57
  * We'll match the input `language` field against the `language_name` or `aliases` of each custom language specification, and use the matched one. If value of `language` is null, it'll be treated as empty string when matching `language_name` or `aliases`.
58
58
  * If no match is found, we'll match the `language` field against the builtin language configurations.
59
59
  For all supported builtin language names and aliases (extensions), see [the code](https://github.com/search?q=org%3Acocoindex-io+lang%3Arust++%22static+TREE_SITTER_LANGUAGE_BY_LANG%22&type=code).
@@ -22,9 +22,9 @@ The spec takes the following fields:
22
22
  If not specified, no files will be excluded.
23
23
 
24
24
  :::info
25
-
25
+
26
26
  `included_patterns` and `excluded_patterns` are using Unix-style glob syntax. See [globset syntax](https://docs.rs/globset/latest/globset/index.html#syntax) for the details.
27
-
27
+
28
28
  :::
29
29
 
30
30
  ### Schema
@@ -131,9 +131,9 @@ The spec takes the following fields:
131
131
  If not specified, no files will be excluded.
132
132
 
133
133
  :::info
134
-
134
+
135
135
  `included_patterns` and `excluded_patterns` are using Unix-style glob syntax. See [globset syntax](https://docs.rs/globset/latest/globset/index.html#syntax) for the details.
136
-
136
+
137
137
  :::
138
138
 
139
139
  * `sqs_queue_url` (type: `str`, optional): if provided, the source will receive change event notifications from Amazon S3 via this SQS queue.
@@ -52,7 +52,7 @@ Here's how CocoIndex data elements map to Qdrant elements during export:
52
52
 
53
53
  | CocoIndex Element | Qdrant Element |
54
54
  |-------------------|------------------|
55
- | an export target | a unique collection |
55
+ | an export target | a unique collection |
56
56
  | a collected row | a point |
57
57
  | a field | a named vector, if fits into Qdrant vector; or a field within payload otherwise |
58
58
 
@@ -99,4 +99,3 @@ query = f"SELECT filename, text FROM {table_name} ORDER BY embedding <=> %s DESC
99
99
 
100
100
  </TabItem>
101
101
  </Tabs>
102
-
@@ -61,4 +61,4 @@ const sidebars: SidebarsConfig = {
61
61
  ],
62
62
  };
63
63
 
64
- export default sidebars;
64
+ export default sidebars;
@@ -18,17 +18,17 @@
18
18
  --ifm-color-primary-lighter: #8F8FE6;
19
19
  --ifm-color-primary-lightest: #ABABEF;
20
20
 
21
-
21
+
22
22
  --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.1);
23
23
  --my-color-text-black: #111827;
24
-
24
+
25
25
  /* Additional colors */
26
26
  --ifm-navbar-background-color: var(--ifm-background-color);
27
27
  --ifm-background-color: #ffffff;
28
28
  --ifm-footer-background-color: #ffffff;
29
29
  --ifm-menu-color: #374151;
30
30
  --ifm-toc-link-color: #374151;
31
-
31
+
32
32
  /* Theme colors for breadcrumbs */
33
33
  --theme-color-text-light: #6b7280;
34
34
  --theme-color-text-default: #111827;
@@ -44,17 +44,17 @@
44
44
  --ifm-color-primary-light: #ABABEF;
45
45
  --ifm-color-primary-lighter: #C4C4F5;
46
46
  --ifm-color-primary-lightest: #E1E1FF;
47
-
47
+
48
48
  --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.3);
49
49
  --my-color-text-black: #f9fafb;
50
-
50
+
51
51
  /* Dark mode specific colors */
52
52
  --ifm-navbar-background-color: var(--ifm-background-color);
53
53
  --ifm-background-color: #111827;
54
54
  --ifm-footer-background-color: #111827;
55
55
  --ifm-menu-color: #f3f4f6;
56
56
  --ifm-toc-link-color: #f3f4f6;
57
-
57
+
58
58
  /* Dark mode theme colors for breadcrumbs */
59
59
  --theme-color-text-light: #9ca3af;
60
60
  --theme-color-text-default: #f9fafb;
@@ -113,7 +113,7 @@
113
113
 
114
114
  .footer__title {
115
115
  font-family: 'Questrial', sans-serif;
116
- font-size: 1rem;
116
+ font-size: 1rem;
117
117
  font-weight: 600;
118
118
  margin-bottom: 1rem;
119
119
  color: var(--my-color-text-black);
@@ -15,4 +15,4 @@ export default function Root({ children }) {
15
15
  }, []);
16
16
 
17
17
  return <>{children}</>;
18
- }
18
+ }
@@ -1,2 +1,2 @@
1
1
  User-agent: *
2
- Disallow:
2
+ Disallow:
@@ -8,4 +8,4 @@ AMAZON_S3_BUCKET_NAME=your-bucket-name
8
8
  # AMAZON_S3_PREFIX=
9
9
 
10
10
  # Optional
11
- # AMAZON_S3_SQS_QUEUE_URL=
11
+ # AMAZON_S3_SQS_QUEUE_URL=
@@ -53,7 +53,7 @@ During running, it will keep observing changes in the Amazon S3 bucket and updat
53
53
  At the same time, it accepts queries from the terminal, and performs search on top of the up-to-date index.
54
54
 
55
55
 
56
- ## CocoInsight
56
+ ## CocoInsight
57
57
  CocoInsight is in Early Access now (Free) 😊 You found us! A quick 3 minute video tutorial about CocoInsight: [Watch on YouTube](https://youtu.be/ZnmyoHslBSc?si=pPLXWALztkA710r9).
58
58
 
59
59
  Run CocoInsight to understand your RAG data pipeline:
@@ -68,4 +68,4 @@ You can also add a `-L` flag to make the server keep updating the index to refle
68
68
  cocoindex server -ci -L main.py
69
69
  ```
70
70
 
71
- Then open the CocoInsight UI at [https://cocoindex.io/cocoinsight](https://cocoindex.io/cocoinsight).
71
+ Then open the CocoInsight UI at [https://cocoindex.io/cocoinsight](https://cocoindex.io/cocoinsight).