cocoindex 0.1.75__tar.gz → 0.1.77__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (315) hide show
  1. {cocoindex-0.1.75 → cocoindex-0.1.77}/Cargo.lock +1 -1
  2. {cocoindex-0.1.75 → cocoindex-0.1.77}/Cargo.toml +1 -1
  3. {cocoindex-0.1.75 → cocoindex-0.1.77}/PKG-INFO +5 -1
  4. {cocoindex-0.1.75 → cocoindex-0.1.77}/README.md +1 -0
  5. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/core/data_types.mdx +20 -0
  6. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/core/flow_methods.mdx +3 -3
  7. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/ops/functions.md +59 -9
  8. cocoindex-0.1.77/docs/docs/tutorials/manage_flow_dynamically.mdx +302 -0
  9. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/sidebars.ts +1 -0
  10. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/amazon_s3_embedding/pyproject.toml +1 -1
  11. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/azure_blob_embedding/pyproject.toml +1 -1
  12. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/code_embedding/pyproject.toml +1 -1
  13. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/custom_output_files/pyproject.toml +1 -1
  14. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/docs_to_knowledge_graph/pyproject.toml +1 -1
  15. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/face_recognition/pyproject.toml +1 -1
  16. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/fastapi_server_docker/requirements.txt +1 -1
  17. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/gdrive_text_embedding/pyproject.toml +1 -1
  18. cocoindex-0.1.77/examples/image_search/.env +1 -0
  19. cocoindex-0.1.77/examples/image_search/README.md +105 -0
  20. cocoindex-0.1.77/examples/image_search/colpali_main.py +161 -0
  21. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/image_search/frontend/src/App.jsx +2 -2
  22. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/image_search/frontend/vite.config.js +1 -0
  23. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/image_search/pyproject.toml +3 -3
  24. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/live_updates/pyproject.toml +1 -4
  25. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/manuals_llm_extraction/pyproject.toml +1 -1
  26. cocoindex-0.1.77/examples/multi_format_indexing/README.md +71 -0
  27. cocoindex-0.1.77/examples/multi_format_indexing/main.py +135 -0
  28. cocoindex-0.1.77/examples/multi_format_indexing/pyproject.toml +14 -0
  29. cocoindex-0.1.77/examples/multi_format_indexing/source_files/cat1.jpeg +0 -0
  30. cocoindex-0.1.77/examples/multi_format_indexing/source_files/dog1.jpeg +0 -0
  31. cocoindex-0.1.77/examples/multi_format_indexing/source_files/elephant1.jpg +0 -0
  32. cocoindex-0.1.77/examples/multi_format_indexing/source_files/giraffe.jpg +0 -0
  33. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/paper_metadata/pyproject.toml +1 -1
  34. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/patient_intake_extraction/pyproject.toml +1 -1
  35. cocoindex-0.1.77/examples/pdf_embedding/pdf_files/1706.03762v7.pdf +0 -0
  36. cocoindex-0.1.77/examples/pdf_embedding/pdf_files/1810.04805v2.pdf +0 -0
  37. cocoindex-0.1.77/examples/pdf_embedding/pdf_files/rfc8259.pdf +0 -0
  38. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/pdf_embedding/pyproject.toml +1 -1
  39. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/product_recommendation/pyproject.toml +1 -1
  40. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/text_embedding/pyproject.toml +1 -1
  41. cocoindex-0.1.77/examples/text_embedding_qdrant/.env +2 -0
  42. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/text_embedding_qdrant/pyproject.toml +1 -1
  43. {cocoindex-0.1.75 → cocoindex-0.1.77}/pyproject.toml +2 -1
  44. cocoindex-0.1.77/python/cocoindex/functions.py +298 -0
  45. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/op.py +65 -39
  46. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/tests/test_transform_flow.py +41 -0
  47. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/base/schema.rs +5 -0
  48. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/execution/evaluator.rs +11 -0
  49. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/factory_bases.rs +55 -8
  50. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/functions/embed_text.rs +6 -5
  51. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/functions/extract_by_llm.rs +58 -19
  52. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/functions/parse_json.rs +5 -4
  53. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/functions/split_recursively.rs +104 -47
  54. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/functions/test_utils.rs +15 -27
  55. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/sdk.rs +1 -1
  56. cocoindex-0.1.75/examples/image_search/.env +0 -1
  57. cocoindex-0.1.75/examples/image_search/README.md +0 -52
  58. cocoindex-0.1.75/python/cocoindex/functions.py +0 -101
  59. {cocoindex-0.1.75 → cocoindex-0.1.77}/.cargo/config.toml +0 -0
  60. {cocoindex-0.1.75 → cocoindex-0.1.77}/.env.lib_debug +0 -0
  61. {cocoindex-0.1.75 → cocoindex-0.1.77}/.github/ISSUE_TEMPLATE//360/237/220/233-bug-report.md" +0 -0
  62. {cocoindex-0.1.75 → cocoindex-0.1.77}/.github/ISSUE_TEMPLATE//360/237/222/241-feature-request.md" +0 -0
  63. {cocoindex-0.1.75 → cocoindex-0.1.77}/.github/scripts/update_version.sh +0 -0
  64. {cocoindex-0.1.75 → cocoindex-0.1.77}/.github/workflows/CI.yml +0 -0
  65. {cocoindex-0.1.75 → cocoindex-0.1.77}/.github/workflows/_doc_release.yml +0 -0
  66. {cocoindex-0.1.75 → cocoindex-0.1.77}/.github/workflows/_test.yml +0 -0
  67. {cocoindex-0.1.75 → cocoindex-0.1.77}/.github/workflows/docs.yml +0 -0
  68. {cocoindex-0.1.75 → cocoindex-0.1.77}/.github/workflows/format.yml +0 -0
  69. {cocoindex-0.1.75 → cocoindex-0.1.77}/.github/workflows/release.yml +0 -0
  70. {cocoindex-0.1.75 → cocoindex-0.1.77}/.gitignore +0 -0
  71. {cocoindex-0.1.75 → cocoindex-0.1.77}/.pre-commit-config.yaml +0 -0
  72. {cocoindex-0.1.75 → cocoindex-0.1.77}/CODE_OF_CONDUCT.md +0 -0
  73. {cocoindex-0.1.75 → cocoindex-0.1.77}/CONTRIBUTING.md +0 -0
  74. {cocoindex-0.1.75 → cocoindex-0.1.77}/LICENSE +0 -0
  75. {cocoindex-0.1.75 → cocoindex-0.1.77}/dev/neo4j.yaml +0 -0
  76. {cocoindex-0.1.75 → cocoindex-0.1.77}/dev/postgres.yaml +0 -0
  77. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/.gitignore +0 -0
  78. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/README.md +0 -0
  79. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/about/community.md +0 -0
  80. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/about/contributing.md +0 -0
  81. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/ai/llm.mdx +0 -0
  82. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/core/basics.md +0 -0
  83. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/core/cli.mdx +0 -0
  84. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/core/data_example.svg +0 -0
  85. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/core/flow_def.mdx +0 -0
  86. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/core/flow_example.svg +0 -0
  87. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/core/settings.mdx +0 -0
  88. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/custom_ops/custom_functions.mdx +0 -0
  89. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/custom_ops/custom_targets.mdx +0 -0
  90. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/getting_started/installation.md +0 -0
  91. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/getting_started/markdown_files.zip +0 -0
  92. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/getting_started/overview.md +0 -0
  93. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/getting_started/quickstart.md +0 -0
  94. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/ops/sources.md +0 -0
  95. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/ops/targets.md +0 -0
  96. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/query.mdx +0 -0
  97. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docs/tutorials/live_updates.md +0 -0
  98. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/docusaurus.config.ts +0 -0
  99. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/package.json +0 -0
  100. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/src/components/HomepageFeatures/index.tsx +0 -0
  101. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/src/components/HomepageFeatures/styles.module.css +0 -0
  102. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/src/css/custom.css +0 -0
  103. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/src/theme/Root.js +0 -0
  104. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/static/.nojekyll +0 -0
  105. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/static/img/docusaurus.png +0 -0
  106. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/static/img/favicon.ico +0 -0
  107. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/static/img/icon.svg +0 -0
  108. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/static/img/incremental-etl.gif +0 -0
  109. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/static/robots.txt +0 -0
  110. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/tsconfig.json +0 -0
  111. {cocoindex-0.1.75 → cocoindex-0.1.77}/docs/yarn.lock +0 -0
  112. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/amazon_s3_embedding/.env.example +0 -0
  113. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/amazon_s3_embedding/.gitignore +0 -0
  114. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/amazon_s3_embedding/README.md +0 -0
  115. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/amazon_s3_embedding/main.py +0 -0
  116. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/azure_blob_embedding/.env.example +0 -0
  117. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/azure_blob_embedding/.gitignore +0 -0
  118. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/azure_blob_embedding/README.md +0 -0
  119. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/azure_blob_embedding/main.py +0 -0
  120. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/code_embedding/.env +0 -0
  121. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/code_embedding/README.md +0 -0
  122. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/code_embedding/main.py +0 -0
  123. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/custom_output_files/.env +0 -0
  124. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/custom_output_files/.gitignore +0 -0
  125. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/custom_output_files/README.md +0 -0
  126. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/custom_output_files/data/bizarre_animals.md +0 -0
  127. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/custom_output_files/data/chunk_norris.md +0 -0
  128. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/custom_output_files/main.py +0 -0
  129. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/docs_to_knowledge_graph/.env +0 -0
  130. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/docs_to_knowledge_graph/README.md +0 -0
  131. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/docs_to_knowledge_graph/main.py +0 -0
  132. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/face_recognition/.env +0 -0
  133. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/face_recognition/README.md +0 -0
  134. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/face_recognition/images/Carter_welcomes_Reagan.jpg +0 -0
  135. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/face_recognition/images/Solvay_conference_1927.jpg +0 -0
  136. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/face_recognition/images/Steve_Jobs_and_Bill_Gates_(522695099).jpg +0 -0
  137. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/face_recognition/images/einplanck3.jpg +0 -0
  138. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/face_recognition/main.py +0 -0
  139. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/fastapi_server_docker/.dockerignore +0 -0
  140. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/fastapi_server_docker/.env +0 -0
  141. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/fastapi_server_docker/README.md +0 -0
  142. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/fastapi_server_docker/compose.yaml +0 -0
  143. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/fastapi_server_docker/dockerfile +0 -0
  144. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/fastapi_server_docker/files/1810.04805v2.md +0 -0
  145. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/fastapi_server_docker/main.py +0 -0
  146. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/gdrive_text_embedding/.env.example +0 -0
  147. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/gdrive_text_embedding/.gitignore +0 -0
  148. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/gdrive_text_embedding/README.md +0 -0
  149. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/gdrive_text_embedding/main.py +0 -0
  150. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/image_search/frontend/.gitignore +0 -0
  151. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/image_search/frontend/index.html +0 -0
  152. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/image_search/frontend/package-lock.json +0 -0
  153. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/image_search/frontend/package.json +0 -0
  154. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/image_search/frontend/src/main.jsx +0 -0
  155. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/image_search/frontend/src/style.css +0 -0
  156. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/image_search/img/cat1.jpeg +0 -0
  157. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/image_search/img/dog1.jpeg +0 -0
  158. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/image_search/img/elephant1.jpg +0 -0
  159. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/image_search/img/giraffe.jpg +0 -0
  160. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/image_search/main.py +0 -0
  161. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/live_updates/.env +0 -0
  162. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/live_updates/README.md +0 -0
  163. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/live_updates/data/bizarre_animals.md +0 -0
  164. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/live_updates/data/chunk_norris.md +0 -0
  165. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/live_updates/main.py +0 -0
  166. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/manuals_llm_extraction/.env +0 -0
  167. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/manuals_llm_extraction/README.md +0 -0
  168. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/manuals_llm_extraction/main.py +0 -0
  169. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/manuals_llm_extraction/manuals/array.pdf +0 -0
  170. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/manuals_llm_extraction/manuals/base64.pdf +0 -0
  171. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/manuals_llm_extraction/manuals/copy.pdf +0 -0
  172. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/manuals_llm_extraction/manuals/glob.pdf +0 -0
  173. {cocoindex-0.1.75/examples/pdf_embedding → cocoindex-0.1.77/examples/multi_format_indexing}/.env +0 -0
  174. {cocoindex-0.1.75/examples/paper_metadata/papers → cocoindex-0.1.77/examples/multi_format_indexing/source_files}/1706.03762v7.pdf +0 -0
  175. {cocoindex-0.1.75/examples/paper_metadata/papers → cocoindex-0.1.77/examples/multi_format_indexing/source_files}/1810.04805v2.pdf +0 -0
  176. {cocoindex-0.1.75/examples/pdf_embedding/pdf_files → cocoindex-0.1.77/examples/multi_format_indexing/source_files}/rfc8259.pdf +0 -0
  177. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/paper_metadata/.env.example +0 -0
  178. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/paper_metadata/.gitignore +0 -0
  179. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/paper_metadata/README.md +0 -0
  180. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/paper_metadata/main.py +0 -0
  181. {cocoindex-0.1.75/examples/pdf_embedding/pdf_files → cocoindex-0.1.77/examples/paper_metadata/papers}/1706.03762v7.pdf +0 -0
  182. {cocoindex-0.1.75/examples/pdf_embedding/pdf_files → cocoindex-0.1.77/examples/paper_metadata/papers}/1810.04805v2.pdf +0 -0
  183. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/paper_metadata/papers/2502.06786v3.pdf +0 -0
  184. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/paper_metadata/papers/2502.20346v1.pdf +0 -0
  185. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/patient_intake_extraction/.env.example +0 -0
  186. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/patient_intake_extraction/README.md +0 -0
  187. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/patient_intake_extraction/data/README.md +0 -0
  188. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/patient_intake_extraction/data/patient_forms/Patient_Intake_Form_David_Artificial.docx +0 -0
  189. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/patient_intake_extraction/data/patient_forms/Patient_Intake_Form_Emily_Artificial.pdf +0 -0
  190. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/patient_intake_extraction/data/patient_forms/Patient_Intake_Form_Joe_Artificial.pdf +0 -0
  191. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/patient_intake_extraction/data/patient_forms/Patient_Intake_From_Jane_Artificial.docx +0 -0
  192. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/patient_intake_extraction/main.py +0 -0
  193. {cocoindex-0.1.75/examples/product_recommendation → cocoindex-0.1.77/examples/pdf_embedding}/.env +0 -0
  194. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/pdf_embedding/README.md +0 -0
  195. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/pdf_embedding/main.py +0 -0
  196. {cocoindex-0.1.75/examples/text_embedding → cocoindex-0.1.77/examples/product_recommendation}/.env +0 -0
  197. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/product_recommendation/README.md +0 -0
  198. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/product_recommendation/img/cocoinsight.png +0 -0
  199. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/product_recommendation/img/neo4j.png +0 -0
  200. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/product_recommendation/main.py +0 -0
  201. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/product_recommendation/products/p1.json +0 -0
  202. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/product_recommendation/products/p2.json +0 -0
  203. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/product_recommendation/products/p3.json +0 -0
  204. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/product_recommendation/products/p4.json +0 -0
  205. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/product_recommendation/products/p5.json +0 -0
  206. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/product_recommendation/products/p6.json +0 -0
  207. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/product_recommendation/products/p7.json +0 -0
  208. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/product_recommendation/products/p8.json +0 -0
  209. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/product_recommendation/products/p9.json +0 -0
  210. {cocoindex-0.1.75/examples/text_embedding_qdrant → cocoindex-0.1.77/examples/text_embedding}/.env +0 -0
  211. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/text_embedding/README.md +0 -0
  212. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/text_embedding/Text_Embedding.ipynb +0 -0
  213. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/text_embedding/main.py +0 -0
  214. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/text_embedding/markdown_files/1706.03762v7.md +0 -0
  215. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/text_embedding/markdown_files/1810.04805v2.md +0 -0
  216. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/text_embedding/markdown_files/rfc8259.md +0 -0
  217. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/text_embedding_qdrant/README.md +0 -0
  218. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/text_embedding_qdrant/main.py +0 -0
  219. {cocoindex-0.1.75 → cocoindex-0.1.77}/examples/text_embedding_qdrant/markdown_files/rfc8259.md +0 -0
  220. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/__init__.py +0 -0
  221. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/auth_registry.py +0 -0
  222. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/cli.py +0 -0
  223. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/convert.py +0 -0
  224. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/flow.py +0 -0
  225. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/index.py +0 -0
  226. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/lib.py +0 -0
  227. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/llm.py +0 -0
  228. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/py.typed +0 -0
  229. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/runtime.py +0 -0
  230. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/setting.py +0 -0
  231. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/setup.py +0 -0
  232. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/sources.py +0 -0
  233. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/targets.py +0 -0
  234. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/tests/__init__.py +0 -0
  235. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/tests/test_convert.py +0 -0
  236. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/tests/test_optional_database.py +0 -0
  237. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/tests/test_typing.py +0 -0
  238. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/tests/test_validation.py +0 -0
  239. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/typing.py +0 -0
  240. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/utils.py +0 -0
  241. {cocoindex-0.1.75 → cocoindex-0.1.77}/python/cocoindex/validation.py +0 -0
  242. {cocoindex-0.1.75 → cocoindex-0.1.77}/ruff.toml +0 -0
  243. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/base/duration.rs +0 -0
  244. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/base/field_attrs.rs +0 -0
  245. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/base/json_schema.rs +0 -0
  246. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/base/mod.rs +0 -0
  247. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/base/spec.rs +0 -0
  248. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/base/value.rs +0 -0
  249. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/builder/analyzed_flow.rs +0 -0
  250. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/builder/analyzer.rs +0 -0
  251. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/builder/exec_ctx.rs +0 -0
  252. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/builder/flow_builder.rs +0 -0
  253. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/builder/mod.rs +0 -0
  254. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/builder/plan.rs +0 -0
  255. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/execution/db_tracking.rs +0 -0
  256. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/execution/db_tracking_setup.rs +0 -0
  257. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/execution/dumper.rs +0 -0
  258. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/execution/indexing_status.rs +0 -0
  259. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/execution/live_updater.rs +0 -0
  260. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/execution/memoization.rs +0 -0
  261. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/execution/mod.rs +0 -0
  262. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/execution/row_indexer.rs +0 -0
  263. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/execution/source_indexer.rs +0 -0
  264. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/execution/stats.rs +0 -0
  265. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/lib.rs +0 -0
  266. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/lib_context.rs +0 -0
  267. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/llm/anthropic.rs +0 -0
  268. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/llm/gemini.rs +0 -0
  269. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/llm/litellm.rs +0 -0
  270. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/llm/mod.rs +0 -0
  271. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/llm/ollama.rs +0 -0
  272. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/llm/openai.rs +0 -0
  273. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/llm/openrouter.rs +0 -0
  274. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/llm/vllm.rs +0 -0
  275. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/llm/voyage.rs +0 -0
  276. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/functions/mod.rs +0 -0
  277. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/interface.rs +0 -0
  278. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/mod.rs +0 -0
  279. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/py_factory.rs +0 -0
  280. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/registration.rs +0 -0
  281. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/registry.rs +0 -0
  282. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/sources/amazon_s3.rs +0 -0
  283. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/sources/azure_blob.rs +0 -0
  284. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/sources/google_drive.rs +0 -0
  285. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/sources/local_file.rs +0 -0
  286. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/sources/mod.rs +0 -0
  287. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/targets/kuzu.rs +0 -0
  288. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/targets/mod.rs +0 -0
  289. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/targets/neo4j.rs +0 -0
  290. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/targets/postgres.rs +0 -0
  291. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/targets/qdrant.rs +0 -0
  292. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/targets/shared/mod.rs +0 -0
  293. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/targets/shared/property_graph.rs +0 -0
  294. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/ops/targets/shared/table_columns.rs +0 -0
  295. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/prelude.rs +0 -0
  296. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/py/convert.rs +0 -0
  297. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/py/mod.rs +0 -0
  298. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/server.rs +0 -0
  299. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/service/error.rs +0 -0
  300. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/service/flows.rs +0 -0
  301. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/service/mod.rs +0 -0
  302. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/settings.rs +0 -0
  303. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/setup/auth_registry.rs +0 -0
  304. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/setup/components.rs +0 -0
  305. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/setup/db_metadata.rs +0 -0
  306. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/setup/driver.rs +0 -0
  307. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/setup/mod.rs +0 -0
  308. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/setup/states.rs +0 -0
  309. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/utils/concur_control.rs +0 -0
  310. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/utils/db.rs +0 -0
  311. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/utils/fingerprint.rs +0 -0
  312. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/utils/immutable.rs +0 -0
  313. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/utils/mod.rs +0 -0
  314. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/utils/retryable.rs +0 -0
  315. {cocoindex-0.1.75 → cocoindex-0.1.77}/src/utils/yaml_ser.rs +0 -0
@@ -1297,7 +1297,7 @@ dependencies = [
1297
1297
 
1298
1298
  [[package]]
1299
1299
  name = "cocoindex"
1300
- version = "0.1.75"
1300
+ version = "0.1.77"
1301
1301
  dependencies = [
1302
1302
  "anyhow",
1303
1303
  "async-openai",
@@ -2,7 +2,7 @@
2
2
  name = "cocoindex"
3
3
  # Version used for local development is always higher than others to take precedence.
4
4
  # Will be overridden for specific release versions.
5
- version = "0.1.75"
5
+ version = "0.1.77"
6
6
  edition = "2024"
7
7
  rust-version = "1.88"
8
8
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cocoindex
3
- Version: 0.1.75
3
+ Version: 0.1.77
4
4
  Requires-Dist: click>=8.1.8
5
5
  Requires-Dist: rich>=14.0.0
6
6
  Requires-Dist: python-dotenv>=1.1.0
@@ -12,9 +12,12 @@ Requires-Dist: ruff ; extra == 'dev'
12
12
  Requires-Dist: mypy ; extra == 'dev'
13
13
  Requires-Dist: pre-commit ; extra == 'dev'
14
14
  Requires-Dist: sentence-transformers>=3.3.1 ; extra == 'embeddings'
15
+ Requires-Dist: colpali-engine ; extra == 'colpali'
15
16
  Requires-Dist: sentence-transformers>=3.3.1 ; extra == 'all'
17
+ Requires-Dist: colpali-engine ; extra == 'all'
16
18
  Provides-Extra: dev
17
19
  Provides-Extra: embeddings
20
+ Provides-Extra: colpali
18
21
  Provides-Extra: all
19
22
  License-File: LICENSE
20
23
  Summary: With CocoIndex, users declare the transformation, CocoIndex creates & maintains an index, and keeps the derived index up to date based on source update, with minimal computation and changes.
@@ -211,6 +214,7 @@ It defines an index flow like this:
211
214
  | [Image Search with Vision API](examples/image_search) | Generates detailed captions for images using a vision model, embeds them, enables live-updating semantic search via FastAPI and served on a React frontend|
212
215
  | [Face Recognition](examples/face_recognition) | Recognize faces in images and build embedding index |
213
216
  | [Paper Metadata](examples/paper_metadata) | Index papers in PDF files, and build metadata tables for each paper |
217
+ | [Multi Format Indexing](examples/multi_format_indexing) | Build visual document index from PDFs and images with ColPali for semantic search |
214
218
  | [Custom Output Files](examples/custom_output_files) | Convert markdown files to HTML files and save them to a local directory, using *CocoIndex Custom Targets* |
215
219
 
216
220
  More coming and stay tuned 👀!
@@ -185,6 +185,7 @@ It defines an index flow like this:
185
185
  | [Image Search with Vision API](examples/image_search) | Generates detailed captions for images using a vision model, embeds them, enables live-updating semantic search via FastAPI and served on a React frontend|
186
186
  | [Face Recognition](examples/face_recognition) | Recognize faces in images and build embedding index |
187
187
  | [Paper Metadata](examples/paper_metadata) | Index papers in PDF files, and build metadata tables for each paper |
188
+ | [Multi Format Indexing](examples/multi_format_indexing) | Build visual document index from PDFs and images with ColPali for semantic search |
188
189
  | [Custom Output Files](examples/custom_output_files) | Convert markdown files to HTML files and save them to a local directory, using *CocoIndex Custom Targets* |
189
190
 
190
191
  More coming and stay tuned 👀!
@@ -199,3 +199,23 @@ Currently, the following types are key types
199
199
  - *Uuid*
200
200
  - *Date*
201
201
  - *Struct* with all fields being key types (using `@dataclass(frozen=True)` or `NamedTuple`)
202
+
203
+ ## *Null* Values
204
+
205
+ CocoIndex supports *Null* values. A *Null* value represents the absence of data or an unknown value, distinct from empty strings, zero numbers, or false boolean values.
206
+
207
+ ### Nullable Type
208
+
209
+ For any data (e.g. a field of a *Struct*, an argument or return value of a CocoIndex function), if it is nullable, it means its value can be *Null*.
210
+ We use a `?` suffix to indicate a nullable type, e.g. *Str?*, *Person?*.
211
+
212
+ In Python, *Null* is represented as `None`, so a nullable type can be represented by `T | None` or `typing.Optional[T]`.
213
+
214
+ ### *Null* propagating on CocoIndex functions
215
+
216
+ A function may specify whether each input argument is nullable or not.
217
+ Non-nullable argument means the function needs a known value for the argument to work.
218
+ However, it doesn't forbid the argument to be *Null* at runtime.
219
+ When a non-nullable argument receives a *Null* value, the function execution is skipped and the result is *Null*.
220
+
221
+ For example, for [`SplitRecursively` function](/docs/ops/functions#splitrecursively), the `text` and `chunk_size` arguments are not nullable. If the input value of either of them is *Null*, the function will return *Null*.
@@ -1,13 +1,13 @@
1
1
  ---
2
- title: Run a Flow
2
+ title: Operate a Flow
3
3
  toc_max_heading_level: 4
4
- description: Run a CocoIndex Flow, including build / update data in the target and evaluate the flow without changing the target.
4
+ description: Operate a CocoIndex Flow, including build / update data in the target and evaluate the flow without changing the target.
5
5
  ---
6
6
 
7
7
  import Tabs from '@theme/Tabs';
8
8
  import TabItem from '@theme/TabItem';
9
9
 
10
- # Run a CocoIndex Flow
10
+ # Operate a CocoIndex Flow
11
11
 
12
12
  After a flow is defined as discussed in [Flow Definition](/docs/core/flow_def), you can start to transform data with it.
13
13
 
@@ -9,12 +9,12 @@ description: CocoIndex Built-in Functions
9
9
 
10
10
  `ParseJson` parses a given text to JSON.
11
11
 
12
- The spec takes the following fields:
12
+ Input data:
13
13
 
14
- * `text` (`str`): The source text to parse.
15
- * `language` (`str`, optional): The language of the source text. Only `json` is supported now. Default to `json`.
14
+ * `text` (*Str*): The source text to parse.
15
+ * `language` (*Str?*, default: `"json"`): The language of the source text. Only `json` is supported now.
16
16
 
17
- Return: *Json*
17
+ Return: *Json*, the parsed JSON object.
18
18
 
19
19
  ## SplitRecursively
20
20
 
@@ -37,7 +37,7 @@ Input data:
37
37
 
38
38
  * `text` (*Str*): The text to split.
39
39
  * `chunk_size` (*Int64*): The maximum size of each chunk, in bytes.
40
- * `min_chunk_size` (*Int64*, optional): The minimum size of each chunk, in bytes. If not provided, default to `chunk_size / 2`.
40
+ * `min_chunk_size` (*Int64*, default: `chunk_size / 2`): The minimum size of each chunk, in bytes.
41
41
 
42
42
  :::note
43
43
 
@@ -48,8 +48,8 @@ Input data:
48
48
 
49
49
  :::
50
50
 
51
- * `chunk_overlap` (*Int64*, optional): The maximum overlap size between adjacent chunks, in bytes.
52
- * `language` (*Str*, optional): The language of the document.
51
+ * `chunk_overlap` (*Int64?*, default: *Null*): The maximum overlap size between adjacent chunks, in bytes.
52
+ * `language` (*Str*, default: `""`): The language of the document.
53
53
  Can be a language name (e.g. `Python`, `Javascript`, `Markdown`) or a file extension (e.g. `.py`, `.js`, `.md`).
54
54
 
55
55
 
@@ -61,7 +61,7 @@ Input data:
61
61
  * `custom_languages` in the spec, against the `language_name` or `aliases` field of each entry.
62
62
  * Builtin languages (see [Supported Languages](#supported-languages) section below), against the language, aliases or file extensions of each entry.
63
63
 
64
- All matches are in a case-insensitive manner. If the value of `language` is null, it'll be treated as empty string.
64
+ All matches are in a case-insensitive manner.
65
65
 
66
66
  * If no match is found, the input will be treated as plain text.
67
67
 
@@ -185,6 +185,56 @@ Not all LLM APIs support text embedding. See the [LLM API Types table](/docs/ai/
185
185
 
186
186
  Input data:
187
187
 
188
- * `text` (*Str*, required): The text to embed.
188
+ * `text` (*Str*): The text to embed.
189
189
 
190
190
  Return: *Vector[Float32, N]*, where *N* is the dimension of the embedding vector determined by the model.
191
+
192
+ ## ColPaliEmbedImage
193
+
194
+ `ColPaliEmbedImage` embeds images using the ColPali multimodal model.
195
+
196
+ ColPali (Contextual Late-interaction over Patches) uses late interaction between image patch embeddings and text token embeddings for retrieval.
197
+
198
+ :::note Optional Dependency Required
199
+
200
+ This function requires the `colpali-engine` library, which is an optional dependency. Install CocoIndex with:
201
+
202
+ ```bash
203
+ pip install 'cocoindex[colpali]'
204
+ ```
205
+ :::
206
+
207
+ The spec takes the following fields:
208
+
209
+ * `model` (`str`): The ColPali model name to use (e.g., "vidore/colpali-v1.2")
210
+
211
+ Input data:
212
+
213
+ * `img_bytes` (*Bytes*): The image data in bytes format.
214
+
215
+ Return: *Vector[Vector[Float32, N]]*, where *N* is the hidden dimension determined by the model. This returns a multi-vector format with variable patches and fixed hidden dimension.
216
+
217
+ ## ColPaliEmbedQuery
218
+
219
+ `ColPaliEmbedQuery` embeds text queries using the ColPali multimodal model.
220
+
221
+ This produces query embeddings compatible with ColPali image embeddings for late interaction scoring (MaxSim).
222
+
223
+ :::note Optional Dependency Required
224
+
225
+ This function requires the `colpali-engine` library, which is an optional dependency. Install CocoIndex with:
226
+
227
+ ```bash
228
+ pip install 'cocoindex[colpali]'
229
+ ```
230
+ :::
231
+
232
+ The spec takes the following fields:
233
+
234
+ * `model` (`str`): The ColPali model name to use (e.g., "vidore/colpali-v1.2")
235
+
236
+ Input data:
237
+
238
+ * `query` (*Str*): The text query to embed.
239
+
240
+ Return: *Vector[Vector[Float32, N]]*, where *N* is the hidden dimension determined by the model. This returns a multi-vector format with variable tokens and fixed hidden dimension.
@@ -0,0 +1,302 @@
1
+ ---
2
+ title: Manage Flows Dynamically
3
+ description: "Learn how to dynamically manage multiple flow instances in CocoIndex. Create parameterized data indexing pipelines, handle persistent resources, perform updates, and implement memory-efficient caching for scalable applications."
4
+ ---
5
+
6
+ # Manage Flows Dynamically
7
+
8
+ You write a function, a.k.a. *flow definition*, to define indexing logic.
9
+ Sometimes you want to reuse the same flow definition for multiple *flow instances* (a.k.a. *flow*), e.g. each takes input from different sources, exports to different targets, and even with slightly different parameters for transformation logic.
10
+
11
+ ## States of a flow instance
12
+
13
+ A flow instance has states from two aspects:
14
+
15
+ * *In-process object*, of type `cocoindex.Flow`.
16
+ * *Persistent resource*, including states in the [internal storage](/docs/core/basics#internal-storage) and backend resources that are owned by the flow instance.
17
+
18
+ A flow instance is ultimately a persistent resource. Its in-process object is a handle to operate on it. Consider file handles and database connections.
19
+ CocoIndex provides APIs to *open* and *close* flow instances, and *setup* and *drop* the persistent resource.
20
+
21
+ ## Parameterize the flow definition
22
+
23
+ In the example from the [Quickstart Guide](/docs/getting_started/quickstart), we decorate the flow definition function with a `@cocoindex.flow_def(name="DemoFlow")` decorator:
24
+
25
+ ```python title="Example in Quickstart Guide"
26
+ @cocoindex.flow_def(name="TextEmbedding")
27
+ def text_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
28
+ ...
29
+ ```
30
+
31
+ This immediately creates the in-process object of the flow instance, using the given function as the flow definition.
32
+ This is a shortcut of:
33
+
34
+ ```python
35
+ def _text_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
36
+ ...
37
+
38
+ text_embedding_flow = cocoindex.open_flow("TextEmbedding", _text_embedding_flow)
39
+ ```
40
+
41
+ Here, `cocoindex.open_flow()` is the function that creates the in-process object of the flow instance, with the given name and flow definition function.
42
+ You can directly call it dynamically with flow name created programmatically.
43
+
44
+ Oftentimes, you also want to parameterize the flow definition function.
45
+ For example, we may have a dataclass like this to hold the parameters of the flow:
46
+
47
+ ```python
48
+ @dataclass
49
+ class TextEmbeddingFlowParameters:
50
+ source_path: str
51
+ target_table_name: str
52
+ ```
53
+
54
+ And consider we have a registry of parameters for all flow instances somewhere.
55
+ For simplicity, we use a hardcoded `dict` here, and provide a simple function to get the parameters for a given flow name.
56
+ In reality, the source of truth may come from a configuration file, a database, etc., and the function can be replaced by your own implementation.
57
+
58
+ ```python
59
+ FLOW_PARAMETERS: dict[str, TextEmbeddingFlowParameters] = {
60
+ "foo": TextEmbeddingFlowParameters(source_path="/path/to/foo", target_table_name="foo_embeddings"),
61
+ "bar": TextEmbeddingFlowParameters(source_path="/path/to/bar", target_table_name="bar_embeddings"),
62
+ }
63
+
64
+ def get_flow_parameters(name: str) -> TextEmbeddingFlowParameters:
65
+ return FLOW_PARAMETERS[name]
66
+ ```
67
+
68
+ Then you can have a function that returns the flow definition function for the given parameters:
69
+
70
+ ```python
71
+ def text_embedding_flow_def(params: TextEmbeddingFlowParameters):
72
+ def _flow_def(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
73
+ # Add a data source to read files from the specified directory
74
+ data_scope["documents"] = flow_builder.add_source(
75
+ cocoindex.sources.LocalFile(path=params.source_path))
76
+
77
+ doc_embeddings = data_scope.add_collector()
78
+ ...
79
+
80
+ # Export the collected data to a Postgres table, with the specified table name.
81
+ doc_embeddings.export(
82
+ "doc_embeddings",
83
+ cocoindex.targets.Postgres(table_name=params.target_table_name),
84
+ primary_key_fields=["filename", "location"],
85
+ )
86
+
87
+ return _flow_def
88
+ ```
89
+
90
+ With this, you can open flow instances dynamically with its parameters:
91
+
92
+ ```python
93
+ text_embedding_flows: dict[str, cocoindex.Flow] = {}
94
+
95
+ def get_text_embedding_flow(name: str) -> cocoindex.Flow:
96
+ flow = text_embedding_flows.get(name)
97
+
98
+ if flow is None:
99
+ params = get_flow_parameters(name)
100
+ flow = text_embedding_flows[name] = cocoindex.open_flow(f"TextEmbedding_{name}", text_embedding_flow_def(params))
101
+
102
+ return flow
103
+ ```
104
+
105
+ ## Operate on the flow instances
106
+
107
+ ### Setup the persistent resource
108
+
109
+ After you instantiated and open flow instances dynamically, before you can perform any data updates, you need to make sure the persistent resource is ready.
110
+ You can use the `setup()` method, e.g. modify the above code to:
111
+
112
+ ```python
113
+ text_embedding_flows: dict[str, cocoindex.Flow] = {}
114
+
115
+ def get_text_embedding_flow(name: str) -> cocoindex.Flow:
116
+ flow = text_embedding_flows.get(name)
117
+
118
+ if flow is None:
119
+ params = get_flow_parameters(name)
120
+ flow = text_embedding_flows[name] = cocoindex.open_flow(f"TextEmbedding_{name}", text_embedding_flow_def(params))
121
+ flow.setup(report_to_stdout=True)
122
+
123
+ return flow
124
+ ```
125
+
126
+ `setup()` method synchronizes the persistent resource to a state that is consistent with the in-process object. For example,
127
+ * If the persistent resource is not there yet, it will create the backend resources for new targets.
128
+ * If your flow definition changed and a new target has been added since the last time of setup, it will create the backend resources for new targets.
129
+ * If an existing target is removed from the flow definition, it will drop the backend resources for the removed target.
130
+ * If nothing changed since the last time of setup, it will be a no-op. i.e. the `setup()` method is idempotent.
131
+
132
+ `setup()` takes a `report_to_stdout` parameter to control whether to print the setup progress to the standard output.
133
+
134
+ `setup()` takes care of all scenarios and makes sure the persistent resource is in the right state.
135
+ It's generally safe to call it after you open a flow instance, even if you don't know whether the persistent resource already exists.
136
+
137
+
138
+ ### Perform data updates
139
+
140
+ After you make sure the persistent resource is ready, you can perform data updates using the flow.
141
+
142
+ The `update()` method updates the target defined by the flow.
143
+
144
+ ```python
145
+ flow.update()
146
+ ```
147
+
148
+ This performs a one-time data update. After the function returns, the target is up-to-date as of the moment when the function is called. For example, we can call `update()` to update the target after the flow is setup:
149
+
150
+ ```python
151
+ def update_text_embedding_index(name: str):
152
+ flow = get_text_embedding_flow(name)
153
+ flow.update()
154
+ ```
155
+
156
+ You can also do a live update.
157
+ See the [Live Updates](/docs/tutorials/live_updates) tutorial for more details.
158
+
159
+
160
+ ### Close the flow object
161
+
162
+ Sometimes you don't want to hold the in-process object forever.
163
+ You can free up the memory resources by closing the flow instances with the `close()` method.
164
+
165
+ For example, the `dict` we managed above behaves like a cache to hold the flow instances.
166
+ If a specific flow isn't used for a while, we may close it.
167
+ The `TTLCache` from [`cachetools`](https://pypi.org/project/cachetools/) package provides exactly this functionality.
168
+ We can rewrite the above code a little bit.
169
+ First, we bring in necessary imports:
170
+
171
+ ```python
172
+ from cachetools import cached, TTLCache
173
+ ```
174
+
175
+ Then we define our own version of `TTLCache` to make it call the `close()` method when the flow instance is evicted from the cache:
176
+
177
+ ```python
178
+ class MyTTLCache(TTLCache):
179
+ def popitem(self):
180
+ # Close the flow instance when it is evicted from the cache
181
+ key, flow = super().popitem()
182
+ flow.close()
183
+ return key, flow
184
+ ```
185
+
186
+ With this, we can modify our `get_text_embedding_flow()` function to use `MyTTLCache` to cache the flow instances, instead of managing our own `dict`:
187
+
188
+ ```python
189
+ @cached(cache=MyTTLCache(maxsize=20, ttl=600))
190
+ def get_text_embedding_flow(name: str) -> cocoindex.Flow:
191
+ params = get_flow_parameters(name)
192
+ flow = cocoindex.open_flow(f"TextEmbedding_{name}", text_embedding_flow_def(params))
193
+ flow.setup(report_to_stdout=True)
194
+ return flow
195
+ ```
196
+
197
+ The `@cached()` decorator from `cachetools` package automatically manages the cache for us (and it also offers thread safety!).
198
+ Once a flow is not touched for 10 minutes, it will call the `popitem()` method, which will close the in-memory flow object.
199
+
200
+
201
+ ### Drop the persistent resource
202
+
203
+ Occasionally, you may want to drop the persistent resource of a flow.
204
+ The `drop()` method is for this purpose.
205
+
206
+ ```python
207
+ def drop_text_embedding_index(name: str):
208
+ flow = get_text_embedding_flow(name)
209
+ flow.drop()
210
+ ```
211
+
212
+ This will drop the persistent resource of the flow.
213
+ The in-memory flow object is still alive, and can be reused until it's closed.
214
+ For example, you can still call `setup()` again.
215
+
216
+ ## Put it all together
217
+
218
+ ```python
219
+ import cocoindex
220
+ from cachetools import cached, TTLCache
221
+ from dataclasses import dataclass
222
+
223
+ @dataclass
224
+ class TextEmbeddingFlowParameters:
225
+ source_path: str
226
+ target_table_name: str
227
+
228
+ FLOW_PARAMETERS: dict[str, TextEmbeddingFlowParameters] = {
229
+ "foo": TextEmbeddingFlowParameters(source_path="/path/to/foo", target_table_name="foo_embeddings"),
230
+ "bar": TextEmbeddingFlowParameters(source_path="/path/to/bar", target_table_name="bar_embeddings"),
231
+ }
232
+
233
+ # Placeholder to get the parameters for a given flow name. You can replace this with your own implementation.
234
+ def get_flow_parameters(name: str) -> TextEmbeddingFlowParameters:
235
+ return FLOW_PARAMETERS[name]
236
+
237
+
238
+ def text_embedding_flow_def(params: TextEmbeddingFlowParameters):
239
+ def _flow_def(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
240
+ # Add a data source to read files from a directory
241
+ data_scope["documents"] = flow_builder.add_source(
242
+ cocoindex.sources.LocalFile(path=params.source_path))
243
+
244
+ doc_embeddings = data_scope.add_collector()
245
+ ...
246
+
247
+ # Export the collected data to a Postgres table, with the specified table name.
248
+ doc_embeddings.export(
249
+ "doc_embeddings",
250
+ cocoindex.targets.Postgres(table_name=params.target_table_name),
251
+ primary_key_fields=["filename", "location"],
252
+ )
253
+
254
+ return _flow_def
255
+
256
+ class MyTTLCache(TTLCache):
257
+ def popitem(self):
258
+ # Close the flow instance when it is evicted from the cache
259
+ key, flow = super().popitem()
260
+ flow.close()
261
+ return key, flow
262
+
263
+ @cached(cache=MyTTLCache(maxsize=20, ttl=600))
264
+ def get_text_embedding_flow(name: str) -> cocoindex.Flow:
265
+ params = get_flow_parameters(name)
266
+ flow = cocoindex.open_flow(f"TextEmbedding_{name}", text_embedding_flow_def(params))
267
+ flow.setup(report_to_stdout=True)
268
+ return flow
269
+
270
+ def update_text_embedding_index(name: str):
271
+ flow = get_text_embedding_flow(name)
272
+ flow.update()
273
+
274
+ def drop_text_embedding_index(name: str):
275
+ flow = get_text_embedding_flow(name)
276
+ flow.drop()
277
+ ```
278
+
279
+ This provides a skeleton.
280
+ With this, you can trigger `update_text_embedding_index()` and `drop_text_embedding_index()` from your application, e.g. a web server API.
281
+
282
+ ## Takeaways
283
+
284
+ From this tutorial, we walked through major flow management / operation APIs provided by CocoIndex. These APIs can be categorized into three aspects:
285
+
286
+ | Aspect | APIs | Description |
287
+ |--------|------|-------------|
288
+ | Life of in-process flow object | `open_flow()`, `Flow.close()` | Create and destroy the in-memory handle to operate on flow instances |
289
+ | Life of persistent resource | `Flow.setup()`, `Flow.drop()` | Create and destroy the backend resources and internal storage |
290
+ | Data updates | `Flow.update()`, `FlowLiveUpdater` | Execute the indexing logic to update targets. *Requires persistent resource to be up-to-date first.* |
291
+
292
+
293
+ For simplicity, we use an in-memory `dict` as source of truth for the flow parameters.
294
+ You can replace it with your own mechanism, e.g. table from a database, a configuration file, etc.
295
+ You can trigger these APIs from your applications specific to your use case, e.g. from a specific API endpoint of a web server.
296
+
297
+ ## Further readings
298
+
299
+ You can see the following documents for more details:
300
+
301
+ * [CocoIndex Flow Definition: Entry Point](/docs/core/flow_def#entry-point)
302
+ * [Operate a Flow](/docs/core/flow_methods)
@@ -18,6 +18,7 @@ const sidebars: SidebarsConfig = {
18
18
  collapsed: false,
19
19
  items: [
20
20
  'tutorials/live_updates',
21
+ 'tutorials/manage_flow_dynamically',
21
22
  ],
22
23
  },
23
24
  {
@@ -4,7 +4,7 @@ version = "0.1.0"
4
4
  description = "Simple example for cocoindex: build embedding index based on Amazon S3 files."
5
5
  requires-python = ">=3.11"
6
6
  dependencies = [
7
- "cocoindex[embeddings]>=0.1.67",
7
+ "cocoindex[embeddings]>=0.1.75",
8
8
  "python-dotenv>=1.0.1",
9
9
  "psycopg[binary,pool]",
10
10
  ]
@@ -4,7 +4,7 @@ version = "0.1.0"
4
4
  description = "Simple example for cocoindex: build embedding index based on Azure Blob Storage files."
5
5
  requires-python = ">=3.11"
6
6
  dependencies = [
7
- "cocoindex[embeddings]>=0.1.67",
7
+ "cocoindex[embeddings]>=0.1.75",
8
8
  "python-dotenv>=1.0.1",
9
9
  "psycopg[binary,pool]",
10
10
  ]
@@ -4,7 +4,7 @@ version = "0.1.0"
4
4
  description = "Simple example for cocoindex: build embedding index based on source code."
5
5
  requires-python = ">=3.11"
6
6
  dependencies = [
7
- "cocoindex[embeddings]>=0.1.67",
7
+ "cocoindex[embeddings]>=0.1.75",
8
8
  "python-dotenv>=1.0.1",
9
9
  "pgvector>=0.4.1",
10
10
  "psycopg[binary,pool]",
@@ -3,7 +3,7 @@ name = "custom-output-files"
3
3
  version = "0.1.0"
4
4
  description = "Simple example for cocoindex: convert markdown files to HTML files and save them to a local directory."
5
5
  requires-python = ">=3.11"
6
- dependencies = ["cocoindex>=0.1.74", "markdown-it-py[linkify,plugins]"]
6
+ dependencies = ["cocoindex>=0.1.75", "markdown-it-py[linkify,plugins]"]
7
7
 
8
8
  [tool.setuptools]
9
9
  packages = []
@@ -3,7 +3,7 @@ name = "manuals-to-kg"
3
3
  version = "0.1.0"
4
4
  description = "Simple example for cocoindex: extract triples from files and build knowledge graph."
5
5
  requires-python = ">=3.11"
6
- dependencies = ["cocoindex>=0.1.67"]
6
+ dependencies = ["cocoindex>=0.1.75"]
7
7
 
8
8
  [tool.setuptools]
9
9
  packages = []
@@ -4,7 +4,7 @@ version = "0.1.0"
4
4
  description = "Build index for papers with both metadata and content embeddings"
5
5
  requires-python = ">=3.11"
6
6
  dependencies = [
7
- "cocoindex>=0.1.71",
7
+ "cocoindex>=0.1.75",
8
8
  "face-recognition>=1.3.0",
9
9
  "pillow>=10.0.0",
10
10
  "numpy>=1.26.0",
@@ -1,4 +1,4 @@
1
- cocoindex[embeddings]>=0.1.63
1
+ cocoindex[embeddings]>=0.1.75
2
2
  python-dotenv>=1.0.1
3
3
  fastapi==0.115.12
4
4
  fastapi-cli==0.0.7
@@ -4,7 +4,7 @@ version = "0.1.0"
4
4
  description = "Simple example for cocoindex: build embedding index based on Google Drive files."
5
5
  requires-python = ">=3.11"
6
6
  dependencies = [
7
- "cocoindex[embeddings]>=0.1.67",
7
+ "cocoindex[embeddings]>=0.1.75",
8
8
  "python-dotenv>=1.0.1",
9
9
  "psycopg[binary,pool]",
10
10
  ]
@@ -0,0 +1 @@
1
+ export COCOINDEX_DATABASE_URL="postgres://cocoindex:cocoindex@localhost/cocoindex"