cocoindex 0.1.49__tar.gz → 0.1.51__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (249) hide show
  1. cocoindex-0.1.51/.github/workflows/_doc_release.yml +31 -0
  2. {cocoindex-0.1.49 → cocoindex-0.1.51}/.github/workflows/_test.yml +3 -2
  3. cocoindex-0.1.51/.github/workflows/docs.yml +34 -0
  4. {cocoindex-0.1.49 → cocoindex-0.1.51}/.github/workflows/release.yml +6 -0
  5. {cocoindex-0.1.49 → cocoindex-0.1.51}/.gitignore +1 -1
  6. {cocoindex-0.1.49 → cocoindex-0.1.51}/Cargo.lock +145 -38
  7. {cocoindex-0.1.49 → cocoindex-0.1.51}/Cargo.toml +9 -6
  8. {cocoindex-0.1.49 → cocoindex-0.1.51}/PKG-INFO +1 -1
  9. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docs/core/data_types.mdx +3 -2
  10. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docs/getting_started/overview.md +2 -0
  11. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docs/getting_started/quickstart.md +11 -6
  12. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docs/query.mdx +3 -3
  13. cocoindex-0.1.51/docs/static/img/incremental-etl.gif +0 -0
  14. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/code_embedding/pyproject.toml +1 -1
  15. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/docs_to_knowledge_graph/pyproject.toml +1 -1
  16. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/fastapi_server_docker/main.py +16 -10
  17. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/manuals_llm_extraction/pyproject.toml +1 -1
  18. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/pdf_embedding/pyproject.toml +1 -1
  19. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/product_recommendation/pyproject.toml +1 -1
  20. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/text_embedding/Text_Embedding.ipynb +8 -4
  21. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/text_embedding/main.py +6 -2
  22. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/text_embedding/pyproject.toml +3 -1
  23. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/text_embedding_qdrant/pyproject.toml +1 -1
  24. {cocoindex-0.1.49 → cocoindex-0.1.51}/pyproject.toml +0 -1
  25. cocoindex-0.1.51/python/cocoindex/__init__.py +68 -0
  26. {cocoindex-0.1.49 → cocoindex-0.1.51}/python/cocoindex/cli.py +22 -4
  27. {cocoindex-0.1.49 → cocoindex-0.1.51}/python/cocoindex/convert.py +41 -1
  28. {cocoindex-0.1.49 → cocoindex-0.1.51}/python/cocoindex/functions.py +6 -4
  29. {cocoindex-0.1.49 → cocoindex-0.1.51}/python/cocoindex/lib.py +1 -2
  30. {cocoindex-0.1.49 → cocoindex-0.1.51}/python/cocoindex/setting.py +10 -6
  31. cocoindex-0.1.51/python/cocoindex/tests/test_convert.py +814 -0
  32. cocoindex-0.1.51/python/cocoindex/tests/test_optional_database.py +249 -0
  33. cocoindex-0.1.51/python/cocoindex/tests/test_typing.py +505 -0
  34. {cocoindex-0.1.49 → cocoindex-0.1.51}/python/cocoindex/typing.py +92 -17
  35. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/base/value.rs +18 -6
  36. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/builder/analyzer.rs +3 -4
  37. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/builder/flow_builder.rs +10 -6
  38. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/builder/plan.rs +0 -1
  39. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/execution/db_tracking_setup.rs +3 -4
  40. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/execution/live_updater.rs +2 -2
  41. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/execution/mod.rs +0 -1
  42. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/lib_context.rs +92 -27
  43. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/factory_bases.rs +4 -15
  44. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/interface.rs +2 -65
  45. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/py_factory.rs +4 -5
  46. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/sources/google_drive.rs +31 -46
  47. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/storages/kuzu.rs +2 -10
  48. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/storages/neo4j.rs +8 -11
  49. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/storages/postgres.rs +7 -201
  50. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/storages/qdrant.rs +14 -45
  51. cocoindex-0.1.51/src/py/convert.rs +518 -0
  52. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/py/mod.rs +25 -87
  53. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/server.rs +8 -12
  54. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/service/error.rs +12 -4
  55. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/service/flows.rs +29 -10
  56. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/service/mod.rs +0 -1
  57. cocoindex-0.1.51/src/settings.rs +99 -0
  58. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/setup/components.rs +0 -8
  59. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/setup/db_metadata.rs +3 -4
  60. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/setup/states.rs +2 -14
  61. cocoindex-0.1.49/.github/workflows/docs.yml +0 -74
  62. cocoindex-0.1.49/python/cocoindex/__init__.py +0 -17
  63. cocoindex-0.1.49/python/cocoindex/query.py +0 -115
  64. cocoindex-0.1.49/python/cocoindex/tests/test_convert.py +0 -539
  65. cocoindex-0.1.49/src/execution/query.rs +0 -124
  66. cocoindex-0.1.49/src/py/convert.rs +0 -232
  67. cocoindex-0.1.49/src/service/search.rs +0 -58
  68. cocoindex-0.1.49/src/settings.rs +0 -13
  69. {cocoindex-0.1.49 → cocoindex-0.1.51}/.cargo/config.toml +0 -0
  70. {cocoindex-0.1.49 → cocoindex-0.1.51}/.env.lib_debug +0 -0
  71. {cocoindex-0.1.49 → cocoindex-0.1.51}/.github/ISSUE_TEMPLATE//360/237/220/233-bug-report.md" +0 -0
  72. {cocoindex-0.1.49 → cocoindex-0.1.51}/.github/ISSUE_TEMPLATE//360/237/222/241-feature-request.md" +0 -0
  73. {cocoindex-0.1.49 → cocoindex-0.1.51}/.github/scripts/update_version.sh +0 -0
  74. {cocoindex-0.1.49 → cocoindex-0.1.51}/.github/workflows/CI.yml +0 -0
  75. {cocoindex-0.1.49 → cocoindex-0.1.51}/.vscode/settings.json +0 -0
  76. {cocoindex-0.1.49 → cocoindex-0.1.51}/CODE_OF_CONDUCT.md +0 -0
  77. {cocoindex-0.1.49 → cocoindex-0.1.51}/CONTRIBUTING.md +0 -0
  78. {cocoindex-0.1.49 → cocoindex-0.1.51}/LICENSE +0 -0
  79. {cocoindex-0.1.49 → cocoindex-0.1.51}/README.md +0 -0
  80. {cocoindex-0.1.49 → cocoindex-0.1.51}/dev/neo4j.yaml +0 -0
  81. {cocoindex-0.1.49 → cocoindex-0.1.51}/dev/postgres.yaml +0 -0
  82. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/.gitignore +0 -0
  83. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/README.md +0 -0
  84. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docs/about/community.md +0 -0
  85. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docs/about/contributing.md +0 -0
  86. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docs/ai/llm.mdx +0 -0
  87. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docs/core/basics.md +0 -0
  88. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docs/core/cli.mdx +0 -0
  89. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docs/core/custom_function.mdx +0 -0
  90. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docs/core/data_example.svg +0 -0
  91. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docs/core/flow_def.mdx +0 -0
  92. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docs/core/flow_example.svg +0 -0
  93. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docs/core/flow_methods.mdx +0 -0
  94. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docs/core/settings.mdx +0 -0
  95. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docs/getting_started/installation.md +0 -0
  96. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docs/getting_started/markdown_files.zip +0 -0
  97. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docs/ops/functions.md +0 -0
  98. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docs/ops/sources.md +0 -0
  99. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docs/ops/storages.md +0 -0
  100. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/docusaurus.config.ts +0 -0
  101. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/package.json +0 -0
  102. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/sidebars.ts +0 -0
  103. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/src/components/HomepageFeatures/index.tsx +0 -0
  104. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/src/components/HomepageFeatures/styles.module.css +0 -0
  105. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/src/css/custom.css +0 -0
  106. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/src/theme/Root.js +0 -0
  107. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/static/.nojekyll +0 -0
  108. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/static/img/docusaurus.png +0 -0
  109. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/static/img/favicon.ico +0 -0
  110. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/static/img/icon.svg +0 -0
  111. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/static/robots.txt +0 -0
  112. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/tsconfig.json +0 -0
  113. {cocoindex-0.1.49 → cocoindex-0.1.51}/docs/yarn.lock +0 -0
  114. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/amazon_s3_embedding/.env.example +0 -0
  115. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/amazon_s3_embedding/.gitignore +0 -0
  116. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/amazon_s3_embedding/README.md +0 -0
  117. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/amazon_s3_embedding/main.py +0 -0
  118. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/amazon_s3_embedding/pyproject.toml +0 -0
  119. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/code_embedding/.env +0 -0
  120. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/code_embedding/README.md +0 -0
  121. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/code_embedding/main.py +0 -0
  122. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/docs_to_knowledge_graph/.env +0 -0
  123. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/docs_to_knowledge_graph/README.md +0 -0
  124. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/docs_to_knowledge_graph/main.py +0 -0
  125. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/fastapi_server_docker/.dockerignore +0 -0
  126. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/fastapi_server_docker/.env +0 -0
  127. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/fastapi_server_docker/README.md +0 -0
  128. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/fastapi_server_docker/compose.yaml +0 -0
  129. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/fastapi_server_docker/dockerfile +0 -0
  130. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/fastapi_server_docker/files/1810.04805v2.md +0 -0
  131. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/fastapi_server_docker/requirements.txt +0 -0
  132. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/gdrive_text_embedding/.env.example +0 -0
  133. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/gdrive_text_embedding/.gitignore +0 -0
  134. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/gdrive_text_embedding/README.md +0 -0
  135. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/gdrive_text_embedding/main.py +0 -0
  136. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/gdrive_text_embedding/pyproject.toml +0 -0
  137. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/image_search/.env +0 -0
  138. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/image_search/README.md +0 -0
  139. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/image_search/frontend/.gitignore +0 -0
  140. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/image_search/frontend/index.html +0 -0
  141. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/image_search/frontend/package-lock.json +0 -0
  142. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/image_search/frontend/package.json +0 -0
  143. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/image_search/frontend/src/App.jsx +0 -0
  144. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/image_search/frontend/src/main.jsx +0 -0
  145. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/image_search/frontend/src/style.css +0 -0
  146. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/image_search/frontend/vite.config.js +0 -0
  147. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/image_search/img/cat1.jpeg +0 -0
  148. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/image_search/img/dog1.jpeg +0 -0
  149. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/image_search/img/elephant1.jpg +0 -0
  150. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/image_search/img/giraffe.jpg +0 -0
  151. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/image_search/main.py +0 -0
  152. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/image_search/pyproject.toml +0 -0
  153. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/image_search/requirements.txt +0 -0
  154. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/manuals_llm_extraction/.env +0 -0
  155. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/manuals_llm_extraction/README.md +0 -0
  156. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/manuals_llm_extraction/main.py +0 -0
  157. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/manuals_llm_extraction/manuals/array.pdf +0 -0
  158. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/manuals_llm_extraction/manuals/base64.pdf +0 -0
  159. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/manuals_llm_extraction/manuals/copy.pdf +0 -0
  160. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/manuals_llm_extraction/manuals/glob.pdf +0 -0
  161. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/pdf_embedding/.env +0 -0
  162. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/pdf_embedding/README.md +0 -0
  163. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/pdf_embedding/main.py +0 -0
  164. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/pdf_embedding/pdf_files/1706.03762v7.pdf +0 -0
  165. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/pdf_embedding/pdf_files/1810.04805v2.pdf +0 -0
  166. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/pdf_embedding/pdf_files/rfc8259.pdf +0 -0
  167. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/product_recommendation/.env +0 -0
  168. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/product_recommendation/README.md +0 -0
  169. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/product_recommendation/img/cocoinsight.png +0 -0
  170. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/product_recommendation/img/neo4j.png +0 -0
  171. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/product_recommendation/main.py +0 -0
  172. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/product_recommendation/products/p1.json +0 -0
  173. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/product_recommendation/products/p2.json +0 -0
  174. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/product_recommendation/products/p3.json +0 -0
  175. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/product_recommendation/products/p4.json +0 -0
  176. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/product_recommendation/products/p5.json +0 -0
  177. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/product_recommendation/products/p6.json +0 -0
  178. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/product_recommendation/products/p7.json +0 -0
  179. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/product_recommendation/products/p8.json +0 -0
  180. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/product_recommendation/products/p9.json +0 -0
  181. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/text_embedding/.env +0 -0
  182. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/text_embedding/README.md +0 -0
  183. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/text_embedding/markdown_files/1706.03762v7.md +0 -0
  184. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/text_embedding/markdown_files/1810.04805v2.md +0 -0
  185. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/text_embedding/markdown_files/rfc8259.md +0 -0
  186. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/text_embedding_qdrant/.env +0 -0
  187. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/text_embedding_qdrant/README.md +0 -0
  188. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/text_embedding_qdrant/main.py +0 -0
  189. {cocoindex-0.1.49 → cocoindex-0.1.51}/examples/text_embedding_qdrant/markdown_files/rfc8259.md +0 -0
  190. {cocoindex-0.1.49 → cocoindex-0.1.51}/python/cocoindex/auth_registry.py +0 -0
  191. {cocoindex-0.1.49 → cocoindex-0.1.51}/python/cocoindex/flow.py +0 -0
  192. {cocoindex-0.1.49 → cocoindex-0.1.51}/python/cocoindex/index.py +0 -0
  193. {cocoindex-0.1.49 → cocoindex-0.1.51}/python/cocoindex/llm.py +0 -0
  194. {cocoindex-0.1.49 → cocoindex-0.1.51}/python/cocoindex/op.py +0 -0
  195. {cocoindex-0.1.49 → cocoindex-0.1.51}/python/cocoindex/py.typed +0 -0
  196. {cocoindex-0.1.49 → cocoindex-0.1.51}/python/cocoindex/runtime.py +0 -0
  197. {cocoindex-0.1.49 → cocoindex-0.1.51}/python/cocoindex/setup.py +0 -0
  198. {cocoindex-0.1.49 → cocoindex-0.1.51}/python/cocoindex/sources.py +0 -0
  199. {cocoindex-0.1.49 → cocoindex-0.1.51}/python/cocoindex/storages.py +0 -0
  200. {cocoindex-0.1.49 → cocoindex-0.1.51}/python/cocoindex/tests/__init__.py +0 -0
  201. {cocoindex-0.1.49 → cocoindex-0.1.51}/python/cocoindex/utils.py +0 -0
  202. {cocoindex-0.1.49 → cocoindex-0.1.51}/ruff.toml +0 -0
  203. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/base/duration.rs +0 -0
  204. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/base/field_attrs.rs +0 -0
  205. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/base/json_schema.rs +0 -0
  206. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/base/mod.rs +0 -0
  207. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/base/schema.rs +0 -0
  208. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/base/spec.rs +0 -0
  209. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/builder/analyzed_flow.rs +0 -0
  210. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/builder/mod.rs +0 -0
  211. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/execution/db_tracking.rs +0 -0
  212. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/execution/dumper.rs +0 -0
  213. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/execution/evaluator.rs +0 -0
  214. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/execution/indexing_status.rs +0 -0
  215. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/execution/memoization.rs +0 -0
  216. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/execution/row_indexer.rs +0 -0
  217. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/execution/source_indexer.rs +0 -0
  218. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/execution/stats.rs +0 -0
  219. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/lib.rs +0 -0
  220. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/llm/anthropic.rs +0 -0
  221. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/llm/gemini.rs +0 -0
  222. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/llm/mod.rs +0 -0
  223. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/llm/ollama.rs +0 -0
  224. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/llm/openai.rs +0 -0
  225. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/functions/extract_by_llm.rs +0 -0
  226. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/functions/mod.rs +0 -0
  227. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/functions/parse_json.rs +0 -0
  228. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/functions/split_recursively.rs +0 -0
  229. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/mod.rs +0 -0
  230. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/registration.rs +0 -0
  231. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/registry.rs +0 -0
  232. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/sdk.rs +0 -0
  233. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/sources/amazon_s3.rs +0 -0
  234. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/sources/local_file.rs +0 -0
  235. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/sources/mod.rs +0 -0
  236. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/storages/mod.rs +0 -0
  237. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/storages/shared/mod.rs +0 -0
  238. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/storages/shared/property_graph.rs +0 -0
  239. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/ops/storages/shared/table_columns.rs +0 -0
  240. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/prelude.rs +0 -0
  241. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/setup/auth_registry.rs +0 -0
  242. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/setup/driver.rs +0 -0
  243. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/setup/mod.rs +0 -0
  244. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/utils/db.rs +0 -0
  245. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/utils/fingerprint.rs +0 -0
  246. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/utils/immutable.rs +0 -0
  247. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/utils/mod.rs +0 -0
  248. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/utils/retryable.rs +0 -0
  249. {cocoindex-0.1.49 → cocoindex-0.1.51}/src/utils/yaml_ser.rs +0 -0
@@ -0,0 +1,31 @@
1
+ name: Release Docs
2
+
3
+ on:
4
+ workflow_call:
5
+
6
+ jobs:
7
+ deploy:
8
+ runs-on: ubuntu-latest
9
+ environment: docs-release
10
+ steps:
11
+ - uses: actions/checkout@v4
12
+ - uses: actions/setup-node@v4
13
+ with:
14
+ node-version: 18
15
+ cache: yarn
16
+ cache-dependency-path: docs/yarn.lock
17
+ - uses: webfactory/ssh-agent@v0.5.0
18
+ with:
19
+ ssh-private-key: ${{ secrets.GH_PAGES_DEPLOY }}
20
+ - name: Deploy to GitHub Pages
21
+ env:
22
+ USE_SSH: true
23
+ run: |
24
+ export COCOINDEX_DOCS_POSTHOG_API_KEY=${{ vars.COCOINDEX_DOCS_POSTHOG_API_KEY }}
25
+ export COCOINDEX_DOCS_MIXPANEL_API_KEY=${{ vars.COCOINDEX_DOCS_MIXPANEL_API_KEY }}
26
+ export COCOINDEX_DOCS_ALGOLIA_APP_ID=${{ vars.COCOINDEX_DOCS_ALGOLIA_APP_ID }}
27
+ export COCOINDEX_DOCS_ALGOLIA_API_KEY=${{ vars.COCOINDEX_DOCS_ALGOLIA_API_KEY }}
28
+ git config --global user.email "${{ vars.COCOINDEX_DOCS_DEPLOY_USER_EMAIL }}"
29
+ git config --global user.name "${{ vars.COCOINDEX_DOCS_DEPLOY_USER_NAME }}"
30
+ yarn --cwd docs install --frozen-lockfile
31
+ yarn --cwd docs deploy
@@ -27,15 +27,16 @@ jobs:
27
27
  run: cargo test --verbose
28
28
 
29
29
  - uses: actions/setup-python@v5
30
+ id: setup_python
30
31
  with:
31
32
  python-version: ${{ matrix.python-version }}
32
33
  cache: 'pip'
33
34
  - uses: actions/cache@v4
34
35
  with:
35
36
  path: .venv
36
- key: ${{ runner.os }}-pythonenv-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}
37
+ key: ${{ runner.os }}-pyenv-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('pyproject.toml') }}
37
38
  restore-keys: |
38
- ${{ runner.os }}-pythonenv-${{ matrix.python-version }}-
39
+ ${{ runner.os }}-pyenv-${{ steps.setup_python.outputs.python-version }}-
39
40
  - name: Setup venv
40
41
  run: |
41
42
  python -m venv .venv
@@ -0,0 +1,34 @@
1
+ name: docs
2
+
3
+ on:
4
+ pull_request:
5
+ branches: [main]
6
+ paths:
7
+ - docs/**
8
+ - ".github/workflows/docs.yml"
9
+ workflow_dispatch:
10
+
11
+ permissions:
12
+ contents: write
13
+
14
+ jobs:
15
+ test-deploy:
16
+ if: github.event_name == 'pull_request'
17
+ runs-on: ubuntu-latest
18
+ steps:
19
+ - uses: actions/checkout@v4
20
+ - uses: actions/setup-node@v4
21
+ with:
22
+ node-version: 18
23
+ cache: yarn
24
+ cache-dependency-path: docs/yarn.lock
25
+ - name: Install dependencies
26
+ run: yarn --cwd docs install --frozen-lockfile
27
+ - name: Test build website
28
+ run: yarn --cwd docs build
29
+
30
+ deploy:
31
+ name: Release Docs
32
+ if: ${{ github.event_name == 'workflow_dispatch' }}
33
+ uses: ./.github/workflows/_doc_release.yml
34
+ secrets: inherit
@@ -107,3 +107,9 @@ jobs:
107
107
  with:
108
108
  command: upload
109
109
  args: --non-interactive --skip-existing wheels-*/*
110
+
111
+ release-docs:
112
+ name: Release Docs
113
+ needs: [release]
114
+ uses: ./.github/workflows/_doc_release.yml
115
+ secrets: inherit
@@ -9,7 +9,7 @@ __pycache__/
9
9
  *.so
10
10
 
11
11
  # Distribution / packaging
12
- .venv/
12
+ .venv*/
13
13
  dist/
14
14
 
15
15
  .DS_Store
@@ -676,16 +676,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
676
676
  checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
677
677
  dependencies = [
678
678
  "async-trait",
679
- "axum-core",
679
+ "axum-core 0.4.5",
680
680
  "bytes",
681
681
  "futures-util",
682
682
  "http 1.3.1",
683
683
  "http-body 1.0.1",
684
684
  "http-body-util",
685
+ "itoa",
686
+ "matchit 0.7.3",
687
+ "memchr",
688
+ "mime",
689
+ "percent-encoding",
690
+ "pin-project-lite",
691
+ "rustversion",
692
+ "serde",
693
+ "sync_wrapper",
694
+ "tower 0.5.2",
695
+ "tower-layer",
696
+ "tower-service",
697
+ ]
698
+
699
+ [[package]]
700
+ name = "axum"
701
+ version = "0.8.4"
702
+ source = "registry+https://github.com/rust-lang/crates.io-index"
703
+ checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5"
704
+ dependencies = [
705
+ "axum-core 0.5.2",
706
+ "bytes",
707
+ "form_urlencoded",
708
+ "futures-util",
709
+ "http 1.3.1",
710
+ "http-body 1.0.1",
711
+ "http-body-util",
685
712
  "hyper 1.6.0",
686
713
  "hyper-util",
687
714
  "itoa",
688
- "matchit",
715
+ "matchit 0.8.4",
689
716
  "memchr",
690
717
  "mime",
691
718
  "percent-encoding",
@@ -721,28 +748,48 @@ dependencies = [
721
748
  "sync_wrapper",
722
749
  "tower-layer",
723
750
  "tower-service",
751
+ ]
752
+
753
+ [[package]]
754
+ name = "axum-core"
755
+ version = "0.5.2"
756
+ source = "registry+https://github.com/rust-lang/crates.io-index"
757
+ checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6"
758
+ dependencies = [
759
+ "bytes",
760
+ "futures-core",
761
+ "http 1.3.1",
762
+ "http-body 1.0.1",
763
+ "http-body-util",
764
+ "mime",
765
+ "pin-project-lite",
766
+ "rustversion",
767
+ "sync_wrapper",
768
+ "tower-layer",
769
+ "tower-service",
724
770
  "tracing",
725
771
  ]
726
772
 
727
773
  [[package]]
728
774
  name = "axum-extra"
729
- version = "0.9.6"
775
+ version = "0.10.1"
730
776
  source = "registry+https://github.com/rust-lang/crates.io-index"
731
- checksum = "c794b30c904f0a1c2fb7740f7df7f7972dfaa14ef6f57cb6178dc63e5dca2f04"
777
+ checksum = "45bf463831f5131b7d3c756525b305d40f1185b688565648a92e1392ca35713d"
732
778
  dependencies = [
733
- "axum",
734
- "axum-core",
779
+ "axum 0.8.4",
780
+ "axum-core 0.5.2",
735
781
  "bytes",
736
- "fastrand",
782
+ "form_urlencoded",
737
783
  "futures-util",
738
784
  "http 1.3.1",
739
785
  "http-body 1.0.1",
740
786
  "http-body-util",
741
787
  "mime",
742
- "multer",
743
788
  "pin-project-lite",
789
+ "rustversion",
744
790
  "serde",
745
791
  "serde_html_form",
792
+ "serde_path_to_error",
746
793
  "tower 0.5.2",
747
794
  "tower-layer",
748
795
  "tower-service",
@@ -993,7 +1040,7 @@ dependencies = [
993
1040
 
994
1041
  [[package]]
995
1042
  name = "cocoindex"
996
- version = "0.1.49"
1043
+ version = "0.1.51"
997
1044
  dependencies = [
998
1045
  "anyhow",
999
1046
  "async-openai",
@@ -1002,7 +1049,7 @@ dependencies = [
1002
1049
  "aws-config",
1003
1050
  "aws-sdk-s3",
1004
1051
  "aws-sdk-sqs",
1005
- "axum",
1052
+ "axum 0.8.4",
1006
1053
  "axum-extra",
1007
1054
  "base64 0.22.1",
1008
1055
  "blake2",
@@ -1026,8 +1073,10 @@ dependencies = [
1026
1073
  "json5",
1027
1074
  "log",
1028
1075
  "neo4rs",
1076
+ "numpy",
1029
1077
  "owo-colors",
1030
1078
  "pgvector",
1079
+ "phf",
1031
1080
  "pyo3",
1032
1081
  "pyo3-async-runtimes",
1033
1082
  "pythonize",
@@ -2610,6 +2659,22 @@ version = "0.7.3"
2610
2659
  source = "registry+https://github.com/rust-lang/crates.io-index"
2611
2660
  checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
2612
2661
 
2662
+ [[package]]
2663
+ name = "matchit"
2664
+ version = "0.8.4"
2665
+ source = "registry+https://github.com/rust-lang/crates.io-index"
2666
+ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
2667
+
2668
+ [[package]]
2669
+ name = "matrixmultiply"
2670
+ version = "0.3.10"
2671
+ source = "registry+https://github.com/rust-lang/crates.io-index"
2672
+ checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08"
2673
+ dependencies = [
2674
+ "autocfg",
2675
+ "rawpointer",
2676
+ ]
2677
+
2613
2678
  [[package]]
2614
2679
  name = "md-5"
2615
2680
  version = "0.10.6"
@@ -2678,20 +2743,18 @@ dependencies = [
2678
2743
  ]
2679
2744
 
2680
2745
  [[package]]
2681
- name = "multer"
2682
- version = "3.1.0"
2746
+ name = "ndarray"
2747
+ version = "0.16.1"
2683
2748
  source = "registry+https://github.com/rust-lang/crates.io-index"
2684
- checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b"
2749
+ checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841"
2685
2750
  dependencies = [
2686
- "bytes",
2687
- "encoding_rs",
2688
- "futures-util",
2689
- "http 1.3.1",
2690
- "httparse",
2691
- "memchr",
2692
- "mime",
2693
- "spin",
2694
- "version_check",
2751
+ "matrixmultiply",
2752
+ "num-complex",
2753
+ "num-integer",
2754
+ "num-traits",
2755
+ "portable-atomic",
2756
+ "portable-atomic-util",
2757
+ "rawpointer",
2695
2758
  ]
2696
2759
 
2697
2760
  [[package]]
@@ -2759,6 +2822,15 @@ dependencies = [
2759
2822
  "zeroize",
2760
2823
  ]
2761
2824
 
2825
+ [[package]]
2826
+ name = "num-complex"
2827
+ version = "0.4.6"
2828
+ source = "registry+https://github.com/rust-lang/crates.io-index"
2829
+ checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
2830
+ dependencies = [
2831
+ "num-traits",
2832
+ ]
2833
+
2762
2834
  [[package]]
2763
2835
  name = "num-conv"
2764
2836
  version = "0.1.0"
@@ -2814,6 +2886,22 @@ dependencies = [
2814
2886
  "libc",
2815
2887
  ]
2816
2888
 
2889
+ [[package]]
2890
+ name = "numpy"
2891
+ version = "0.25.0"
2892
+ source = "registry+https://github.com/rust-lang/crates.io-index"
2893
+ checksum = "29f1dee9aa8d3f6f8e8b9af3803006101bb3653866ef056d530d53ae68587191"
2894
+ dependencies = [
2895
+ "libc",
2896
+ "ndarray",
2897
+ "num-complex",
2898
+ "num-integer",
2899
+ "num-traits",
2900
+ "pyo3",
2901
+ "pyo3-build-config",
2902
+ "rustc-hash 2.1.1",
2903
+ ]
2904
+
2817
2905
  [[package]]
2818
2906
  name = "object"
2819
2907
  version = "0.36.7"
@@ -2993,6 +3081,7 @@ version = "0.11.3"
2993
3081
  source = "registry+https://github.com/rust-lang/crates.io-index"
2994
3082
  checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
2995
3083
  dependencies = [
3084
+ "phf_macros",
2996
3085
  "phf_shared",
2997
3086
  ]
2998
3087
 
@@ -3016,6 +3105,19 @@ dependencies = [
3016
3105
  "rand 0.8.5",
3017
3106
  ]
3018
3107
 
3108
+ [[package]]
3109
+ name = "phf_macros"
3110
+ version = "0.11.3"
3111
+ source = "registry+https://github.com/rust-lang/crates.io-index"
3112
+ checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216"
3113
+ dependencies = [
3114
+ "phf_generator",
3115
+ "phf_shared",
3116
+ "proc-macro2",
3117
+ "quote",
3118
+ "syn 2.0.101",
3119
+ ]
3120
+
3019
3121
  [[package]]
3020
3122
  name = "phf_shared"
3021
3123
  version = "0.11.3"
@@ -3177,11 +3279,10 @@ dependencies = [
3177
3279
 
3178
3280
  [[package]]
3179
3281
  name = "pyo3"
3180
- version = "0.24.2"
3282
+ version = "0.25.0"
3181
3283
  source = "registry+https://github.com/rust-lang/crates.io-index"
3182
- checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219"
3284
+ checksum = "f239d656363bcee73afef85277f1b281e8ac6212a1d42aa90e55b90ed43c47a4"
3183
3285
  dependencies = [
3184
- "cfg-if",
3185
3286
  "chrono",
3186
3287
  "indoc",
3187
3288
  "libc",
@@ -3196,9 +3297,9 @@ dependencies = [
3196
3297
 
3197
3298
  [[package]]
3198
3299
  name = "pyo3-async-runtimes"
3199
- version = "0.24.0"
3300
+ version = "0.25.0"
3200
3301
  source = "registry+https://github.com/rust-lang/crates.io-index"
3201
- checksum = "dd0b83dc42f9d41f50d38180dad65f0c99763b65a3ff2a81bf351dd35a1df8bf"
3302
+ checksum = "d73cc6b1b7d8b3cef02101d37390dbdfe7e450dfea14921cae80a9534ba59ef2"
3202
3303
  dependencies = [
3203
3304
  "futures",
3204
3305
  "once_cell",
@@ -3209,9 +3310,9 @@ dependencies = [
3209
3310
 
3210
3311
  [[package]]
3211
3312
  name = "pyo3-build-config"
3212
- version = "0.24.2"
3313
+ version = "0.25.0"
3213
3314
  source = "registry+https://github.com/rust-lang/crates.io-index"
3214
- checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999"
3315
+ checksum = "755ea671a1c34044fa165247aaf6f419ca39caa6003aee791a0df2713d8f1b6d"
3215
3316
  dependencies = [
3216
3317
  "once_cell",
3217
3318
  "target-lexicon",
@@ -3219,9 +3320,9 @@ dependencies = [
3219
3320
 
3220
3321
  [[package]]
3221
3322
  name = "pyo3-ffi"
3222
- version = "0.24.2"
3323
+ version = "0.25.0"
3223
3324
  source = "registry+https://github.com/rust-lang/crates.io-index"
3224
- checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33"
3325
+ checksum = "fc95a2e67091e44791d4ea300ff744be5293f394f1bafd9f78c080814d35956e"
3225
3326
  dependencies = [
3226
3327
  "libc",
3227
3328
  "pyo3-build-config",
@@ -3229,9 +3330,9 @@ dependencies = [
3229
3330
 
3230
3331
  [[package]]
3231
3332
  name = "pyo3-macros"
3232
- version = "0.24.2"
3333
+ version = "0.25.0"
3233
3334
  source = "registry+https://github.com/rust-lang/crates.io-index"
3234
- checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9"
3335
+ checksum = "a179641d1b93920829a62f15e87c0ed791b6c8db2271ba0fd7c2686090510214"
3235
3336
  dependencies = [
3236
3337
  "proc-macro2",
3237
3338
  "pyo3-macros-backend",
@@ -3241,9 +3342,9 @@ dependencies = [
3241
3342
 
3242
3343
  [[package]]
3243
3344
  name = "pyo3-macros-backend"
3244
- version = "0.24.2"
3345
+ version = "0.25.0"
3245
3346
  source = "registry+https://github.com/rust-lang/crates.io-index"
3246
- checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a"
3347
+ checksum = "9dff85ebcaab8c441b0e3f7ae40a6963ecea8a9f5e74f647e33fcf5ec9a1e89e"
3247
3348
  dependencies = [
3248
3349
  "heck",
3249
3350
  "proc-macro2",
@@ -3254,9 +3355,9 @@ dependencies = [
3254
3355
 
3255
3356
  [[package]]
3256
3357
  name = "pythonize"
3257
- version = "0.24.0"
3358
+ version = "0.25.0"
3258
3359
  source = "registry+https://github.com/rust-lang/crates.io-index"
3259
- checksum = "d5bcac0d0b71821f0d69e42654f1e15e5c94b85196446c4de9588951a2117e7b"
3360
+ checksum = "597907139a488b22573158793aa7539df36ae863eba300c75f3a0d65fc475e27"
3260
3361
  dependencies = [
3261
3362
  "pyo3",
3262
3363
  "serde",
@@ -3411,6 +3512,12 @@ dependencies = [
3411
3512
  "getrandom 0.3.2",
3412
3513
  ]
3413
3514
 
3515
+ [[package]]
3516
+ name = "rawpointer"
3517
+ version = "0.2.1"
3518
+ source = "registry+https://github.com/rust-lang/crates.io-index"
3519
+ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
3520
+
3414
3521
  [[package]]
3415
3522
  name = "redox_syscall"
3416
3523
  version = "0.5.11"
@@ -4653,7 +4760,7 @@ checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52"
4653
4760
  dependencies = [
4654
4761
  "async-stream",
4655
4762
  "async-trait",
4656
- "axum",
4763
+ "axum 0.7.9",
4657
4764
  "base64 0.22.1",
4658
4765
  "bytes",
4659
4766
  "flate2",
@@ -2,8 +2,9 @@
2
2
  name = "cocoindex"
3
3
  # Version used for local development is always higher than others to take precedence.
4
4
  # Will be overridden for specific release versions.
5
- version = "0.1.49"
5
+ version = "0.1.51"
6
6
  edition = "2024"
7
+ rust-version = "1.86"
7
8
 
8
9
  [profile.release]
9
10
  codegen-units = 1
@@ -14,14 +15,14 @@ name = "cocoindex_engine"
14
15
  crate-type = ["cdylib"]
15
16
 
16
17
  [dependencies]
17
- pyo3 = { version = "0.24.1", features = ["chrono"] }
18
- pythonize = "0.24.0"
19
- pyo3-async-runtimes = { version = "0.24.0", features = ["tokio-runtime"] }
18
+ pyo3 = { version = "0.25.0", features = ["chrono", "auto-initialize"] }
19
+ pythonize = "0.25.0"
20
+ pyo3-async-runtimes = { version = "0.25.0", features = ["tokio-runtime"] }
20
21
 
21
22
  anyhow = { version = "1.0.97", features = ["std"] }
22
23
  async-trait = "0.1.88"
23
- axum = "0.7.9"
24
- axum-extra = { version = "0.9.6", features = ["query"] }
24
+ axum = "0.8.4"
25
+ axum-extra = { version = "0.10.1", features = ["query"] }
25
26
  base64 = "0.22.1"
26
27
  chrono = "0.4.40"
27
28
  config = "0.14.1"
@@ -50,6 +51,7 @@ tower-http = { version = "0.6.2", features = ["cors", "trace"] }
50
51
  indexmap = { version = "2.8.0", features = ["serde"] }
51
52
  blake2 = "0.10.6"
52
53
  pgvector = { version = "0.4.0", features = ["sqlx"] }
54
+ phf = { version = "0.11.3", features = ["macros"] }
53
55
  indenter = "0.3.3"
54
56
  itertools = "0.14.0"
55
57
  derivative = "2.2.0"
@@ -112,3 +114,4 @@ json5 = "0.4.1"
112
114
  aws-config = "1.6.2"
113
115
  aws-sdk-s3 = "1.85.0"
114
116
  aws-sdk-sqs = "1.67.0"
117
+ numpy = "0.25.0"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cocoindex
3
- Version: 0.1.49
3
+ Version: 0.1.51
4
4
  Requires-Dist: sentence-transformers>=3.3.1
5
5
  Requires-Dist: click>=8.1.8
6
6
  Requires-Dist: rich>=14.0.0
@@ -36,16 +36,17 @@ This is the list of all basic types supported by CocoIndex:
36
36
  | LocalDatetime | Date and time without timezone | `cocoindex.LocalDateTime` | `datetime.datetime` |
37
37
  | OffsetDatetime | Date and time with a timezone offset | `cocoindex.OffsetDateTime` | `datetime.datetime` |
38
38
  | TimeDelta | A duration of time | `datetime.timedelta` | `datetime.timedelta` |
39
- | Vector[*T*, *Dim*?] | *T* must be basic type. *Dim* is a positive integer and optional. |`cocoindex.Vector[T]` or `cocoindex.Vector[T, Dim]` | `list[T]` |
40
39
  | Json | | `cocoindex.Json` | Any data convertible to JSON by `json` package |
40
+ | Vector[*T*, *Dim*?] | *T* can be a basic type or a numeric type. *Dim* is a positive integer and optional. | `cocoindex.Vector[T]` or `cocoindex.Vector[T, Dim]` | `numpy.typing.NDArray[T]` or `list[T]` |
41
41
 
42
42
  Values of all data types can be represented by values in Python's native types (as described under the Native Python Type column).
43
43
  However, the underlying execution engine and some storage system (like Postgres) has finer distinctions for some types, specifically:
44
44
 
45
45
  * *Float32* and *Float64* for `float`, with different precision.
46
46
  * *LocalDateTime* and *OffsetDateTime* for `datetime.datetime`, with different timezone awareness.
47
- * *Vector* has optional dimension information.
48
47
  * *Range* and *Json* provide a clear tag for the type, to clearly distinguish the type in CocoIndex.
48
+ * *Vector* holds elements of type *T*. If *T* is numeric (e.g., `np.float32` or `np.float64`), it's represented as `NDArray[T]`; otherwise, as `list[T]`.
49
+ * *Vector* also has optional dimension information.
49
50
 
50
51
  The native Python type is always more permissive and can represent a superset of possible values.
51
52
  * Only when you annotate the return type of a custom function, you should use the specific type,
@@ -9,6 +9,8 @@ CocoIndex is an ultra-performant real-time data transformation framework for AI,
9
9
 
10
10
  As a data framework, CocoIndex takes it to the next level on data freshness. **Incremental processing** is one of the core values provided by CocoIndex.
11
11
 
12
+ ![Incremental Processing](/img/incremental-etl.gif)
13
+
12
14
  ## Programming Model
13
15
  CocoIndex follows the idea of [Dataflow programming](https://en.wikipedia.org/wiki/Dataflow_programming) model. Each transformation creates a new field solely based on input fields, without hidden states and value mutation. All data before/after each transformation is observable, with lineage out of the box.
14
16
 
@@ -154,11 +154,11 @@ The goal of transforming your data is usually to query against it.
154
154
  Once you already have your index built, you can directly access the transformed data in the target database.
155
155
  CocoIndex also provides utilities for you to do this more seamlessly.
156
156
 
157
- In this example, we'll use the [`psycopg` library](https://www.psycopg.org/) to connect to the database and run queries.
158
- Please make sure it's installed:
157
+ In this example, we'll use the [`psycopg` library](https://www.psycopg.org/) along with pgvector to connect to the database and run queries on vector data.
158
+ Please make sure the required packages are installed:
159
159
 
160
160
  ```bash
161
- pip install psycopg[binary,pool]
161
+ pip install numpy "psycopg[binary,pool]" pgvector
162
162
  ```
163
163
 
164
164
  ### Step 4.1: Extract common transformations
@@ -169,8 +169,11 @@ i.e. they should use exactly the same embedding model and parameters.
169
169
  Let's extract that into a function:
170
170
 
171
171
  ```python title="quickstart.py"
172
+ from numpy.typing import NDArray
173
+ import numpy as np
174
+
172
175
  @cocoindex.transform_flow()
173
- def text_to_embedding(text: cocoindex.DataSlice[str]) -> cocoindex.DataSlice[list[float]]:
176
+ def text_to_embedding(text: cocoindex.DataSlice[str]) -> cocoindex.DataSlice[NDArray[np.float32]]:
174
177
  return text.transform(
175
178
  cocoindex.functions.SentenceTransformerEmbed(
176
179
  model="sentence-transformers/all-MiniLM-L6-v2"))
@@ -207,6 +210,7 @@ Now we can create a function to query the index upon a given input query:
207
210
 
208
211
  ```python title="quickstart.py"
209
212
  from psycopg_pool import ConnectionPool
213
+ from pgvector.psycopg import register_vector
210
214
 
211
215
  def search(pool: ConnectionPool, query: str, top_k: int = 5):
212
216
  # Get the table name, for the export target in the text_embedding_flow above.
@@ -215,9 +219,10 @@ def search(pool: ConnectionPool, query: str, top_k: int = 5):
215
219
  query_vector = text_to_embedding.eval(query)
216
220
  # Run the query and get the results.
217
221
  with pool.connection() as conn:
222
+ register_vector(conn)
218
223
  with conn.cursor() as cur:
219
224
  cur.execute(f"""
220
- SELECT filename, text, embedding <=> %s::vector AS distance
225
+ SELECT filename, text, embedding <=> %s AS distance
221
226
  FROM {table_name} ORDER BY distance LIMIT %s
222
227
  """, (query_vector, top_k))
223
228
  return [
@@ -236,7 +241,7 @@ There're two CocoIndex-specific logic:
236
241
 
237
242
  2. Evaluate the transform flow defined above with the input query, to get the embedding.
238
243
  It's done by the `eval()` method of the transform flow `text_to_embedding`.
239
- The return type of this method is `list[float]` as declared in the `text_to_embedding()` function (`cocoindex.DataSlice[list[float]]`).
244
+ The return type of this method is `NDArray[np.float32]` as declared in the `text_to_embedding()` function (`cocoindex.DataSlice[NDArray[np.float32]]`).
240
245
 
241
246
  ### Step 4.3: Add the main script logic
242
247
 
@@ -41,7 +41,7 @@ The [quickstart](getting_started/quickstart#step-41-extract-common-transformatio
41
41
 
42
42
  ```python
43
43
  @cocoindex.transform_flow()
44
- def text_to_embedding(text: cocoindex.DataSlice[str]) -> cocoindex.DataSlice[list[float]]:
44
+ def text_to_embedding(text: cocoindex.DataSlice[str]) -> cocoindex.DataSlice[NDArray[np.float32]]:
45
45
  return text.transform(
46
46
  cocoindex.functions.SentenceTransformerEmbed(
47
47
  model="sentence-transformers/all-MiniLM-L6-v2"))
@@ -61,7 +61,7 @@ with doc["chunks"].row() as chunk:
61
61
  chunk["embedding"] = chunk["text"].call(text_to_embedding)
62
62
  ```
63
63
 
64
- Any time, you can call the `eval()` method with specific string, which will return a `list[float]`:
64
+ Any time, you can call the `eval()` method with specific string, which will return a `NDArray[np.float32]`:
65
65
 
66
66
  ```python
67
67
  print(text_to_embedding.eval("Hello, world!"))
@@ -93,7 +93,7 @@ For example:
93
93
 
94
94
  ```python
95
95
  table_name = cocoindex.utils.get_target_storage_default_name(text_embedding_flow, "doc_embeddings")
96
- query = f"SELECT filename, text FROM {table_name} ORDER BY embedding <=> %s::vector DESC LIMIT 5"
96
+ query = f"SELECT filename, text FROM {table_name} ORDER BY embedding <=> %s DESC LIMIT 5"
97
97
  ...
98
98
  ```
99
99
 
@@ -2,7 +2,7 @@
2
2
  name = "code-embedding"
3
3
  version = "0.1.0"
4
4
  description = "Simple example for cocoindex: build embedding index based on source code."
5
- requires-python = ">=3.10"
5
+ requires-python = ">=3.11"
6
6
  dependencies = ["cocoindex>=0.1.42", "python-dotenv>=1.0.1"]
7
7
 
8
8
  [tool.setuptools]
@@ -2,7 +2,7 @@
2
2
  name = "manuals-to-kg"
3
3
  version = "0.1.0"
4
4
  description = "Simple example for cocoindex: extract triples from files and build knowledge graph."
5
- requires-python = ">=3.10"
5
+ requires-python = ">=3.11"
6
6
  dependencies = ["cocoindex>=0.1.42"]
7
7
 
8
8
  [tool.setuptools]