cocoindex 0.1.31__tar.gz → 0.1.33__tar.gz

{cocoindex-0.1.31 → cocoindex-0.1.33}/Cargo.lock

@@ -533,7 +533,7 @@ dependencies = [
 
 [[package]]
 name = "cocoindex"
-version = "0.1.31"
+version = "0.1.33"
 dependencies = [
  "anyhow",
  "async-openai",
@@ -546,7 +546,6 @@ dependencies = [
  "bytes",
  "chrono",
  "config",
- "console-subscriber",
  "const_format",
  "derivative",
  "env_logger",
@@ -649,45 +648,6 @@ dependencies = [
  "yaml-rust2 0.8.1",
 ]
 
-[[package]]
-name = "console-api"
-version = "0.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8030735ecb0d128428b64cd379809817e620a40e5001c54465b99ec5feec2857"
-dependencies = [
- "futures-core",
- "prost",
- "prost-types",
- "tonic",
- "tracing-core",
-]
-
-[[package]]
-name = "console-subscriber"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6539aa9c6a4cd31f4b1c040f860a1eac9aa80e7df6b05d506a6e7179936d6a01"
-dependencies = [
- "console-api",
- "crossbeam-channel",
- "crossbeam-utils",
- "futures-task",
- "hdrhistogram",
- "humantime",
- "hyper-util",
- "prost",
- "prost-types",
- "serde",
- "serde_json",
- "thread_local",
- "tokio",
- "tokio-stream",
- "tonic",
- "tracing",
- "tracing-core",
- "tracing-subscriber",
-]
-
 [[package]]
 name = "const-oid"
 version = "0.9.6"
@@ -802,15 +762,6 @@ dependencies = [
  "cfg-if",
 ]
 
-[[package]]
-name = "crossbeam-channel"
-version = "0.5.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
-dependencies = [
- "crossbeam-utils",
-]
-
 [[package]]
 name = "crossbeam-queue"
 version = "0.3.12"
@@ -1328,8 +1279,8 @@ dependencies = [
  "aho-corasick",
  "bstr",
  "log",
- "regex-automata 0.4.9",
- "regex-syntax 0.8.5",
+ "regex-automata",
+ "regex-syntax",
 ]
 
 [[package]]
@@ -1439,19 +1390,6 @@ dependencies = [
  "hashbrown 0.15.2",
 ]
 
-[[package]]
-name = "hdrhistogram"
-version = "7.5.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d"
-dependencies = [
- "base64 0.21.7",
- "byteorder",
- "flate2",
- "nom",
- "num-traits",
-]
-
 [[package]]
 name = "heck"
 version = "0.5.0"
@@ -1543,12 +1481,6 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
-[[package]]
-name = "humantime"
-version = "2.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f"
-
 [[package]]
 name = "hyper"
 version = "1.6.0"
@@ -2016,15 +1948,6 @@ version = "0.4.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
 
-[[package]]
-name = "matchers"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
-dependencies = [
- "regex-automata 0.1.10",
-]
-
 [[package]]
 name = "matchit"
 version = "0.7.3"
@@ -2822,17 +2745,8 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-automata 0.4.9",
- "regex-syntax 0.8.5",
-]
-
-[[package]]
-name = "regex-automata"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
-dependencies = [
- "regex-syntax 0.6.29",
+ "regex-automata",
+ "regex-syntax",
 ]
 
 [[package]]
@@ -2843,15 +2757,9 @@ checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-syntax 0.8.5",
+ "regex-syntax",
 ]
 
-[[package]]
-name = "regex-syntax"
-version = "0.6.29"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
-
 [[package]]
 name = "regex-syntax"
 version = "0.8.5"
@@ -3348,15 +3256,6 @@ dependencies = [
  "digest",
 ]
 
-[[package]]
-name = "sharded-slab"
-version = "0.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
-dependencies = [
- "lazy_static",
-]
-
 [[package]]
 name = "shlex"
 version = "1.3.0"
@@ -3768,16 +3667,6 @@ dependencies = [
  "syn 2.0.101",
 ]
 
-[[package]]
-name = "thread_local"
-version = "1.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c"
-dependencies = [
- "cfg-if",
- "once_cell",
-]
-
 [[package]]
 name = "time"
 version = "0.3.41"
@@ -4078,22 +3967,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c"
 dependencies = [
  "once_cell",
- "valuable",
-]
-
-[[package]]
-name = "tracing-subscriber"
-version = "0.3.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008"
-dependencies = [
- "matchers",
- "once_cell",
- "regex",
- "sharded-slab",
- "thread_local",
- "tracing",
- "tracing-core",
 ]
 
 [[package]]
@@ -4104,7 +3977,7 @@ checksum = "b9ac5ea5e7f2f1700842ec071401010b9c59bf735295f6e9fa079c3dc035b167"
 dependencies = [
  "cc",
  "regex",
- "regex-syntax 0.8.5",
+ "regex-syntax",
  "serde_json",
  "streaming-iterator",
  "tree-sitter-language",
@@ -4476,12 +4349,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "valuable"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
-
 [[package]]
 name = "vcpkg"
 version = "0.2.15"

{cocoindex-0.1.31 → cocoindex-0.1.33}/Cargo.toml

@@ -2,10 +2,13 @@
 name = "cocoindex"
 # Version used for local development is always higher than others to take precedence.
 # Will be overridden for specific release versions.
-version = "0.1.31"
+version = "0.1.33"
 edition = "2021"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[profile.release]
+codegen-units = 1
+lto = true
+
 [lib]
 name = "cocoindex_engine"
 crate-type = ["cdylib"]
@@ -52,7 +55,6 @@ itertools = "0.14.0"
 derivative = "2.2.0"
 hex = "0.4.3"
 schemars = "0.8.22"
-console-subscriber = "0.4.1"
 env_logger = "0.11.7"
 reqwest = { version = "0.12.15", default-features = false, features = [
     "json",

{cocoindex-0.1.31 → cocoindex-0.1.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cocoindex
-Version: 0.1.31
+Version: 0.1.33
 Requires-Dist: sentence-transformers>=3.3.1
 Requires-Dist: click>=8.1.8
 Requires-Dist: rich>=14.0.0
@@ -120,6 +120,7 @@ Go to the [examples directory](examples) to try out with any of the examples, fo
 | [Google Drive Text Embedding](examples/gdrive_text_embedding) | Index text documents from Google Drive |
 | [Docs to Knowledge Graph](examples/docs_to_knowledge_graph) | Extract relationships from Markdown documents and build a knowledge graph |
 | [Embeddings to Qdrant](examples/text_embedding_qdrant) | Index documents in a Qdrant collection for semantic search |
+| [FastAPI Server with Docker](examples/fastapi_server_docker) | Run the semantic search server in a Dockerized FastAPI setup | 
 
 More coming and stay tuned! If there's any specific examples you would like to see, please let us know in our [Discord community](https://discord.com/invite/zpA9S2DR7s) 🌱.
 

{cocoindex-0.1.31 → cocoindex-0.1.33}/README.md

@@ -104,6 +104,7 @@ Go to the [examples directory](examples) to try out with any of the examples, fo
 | [Google Drive Text Embedding](examples/gdrive_text_embedding) | Index text documents from Google Drive |
 | [Docs to Knowledge Graph](examples/docs_to_knowledge_graph) | Extract relationships from Markdown documents and build a knowledge graph |
 | [Embeddings to Qdrant](examples/text_embedding_qdrant) | Index documents in a Qdrant collection for semantic search |
+| [FastAPI Server with Docker](examples/fastapi_server_docker) | Run the semantic search server in a Dockerized FastAPI setup | 
 
 More coming and stay tuned! If there's any specific examples you would like to see, please let us know in our [Discord community](https://discord.com/invite/zpA9S2DR7s) 🌱.
 

{cocoindex-0.1.31 → cocoindex-0.1.33}/examples/code_embedding/pyproject.toml

@@ -3,4 +3,4 @@ name = "code-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on source code."
 requires-python = ">=3.10"
-dependencies = ["cocoindex>=0.1.30", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex>=0.1.32", "python-dotenv>=1.0.1"]

{cocoindex-0.1.31 → cocoindex-0.1.33}/examples/docs_to_knowledge_graph/pyproject.toml

@@ -3,4 +3,4 @@ name = "manuals-to-kg"
 version = "0.1.0"
 description = "Simple example for cocoindex: extract triples from files and build knowledge graph."
 requires-python = ">=3.10"
-dependencies = ["cocoindex>=0.1.30", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex>=0.1.32", "python-dotenv>=1.0.1"]

cocoindex-0.1.33/examples/fastapi_server_docker/.dockerignore

@@ -0,0 +1,2 @@
+.venv
+__pycache__

cocoindex-0.1.33/examples/fastapi_server_docker/.env

@@ -0,0 +1 @@
+COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@coco_db:5432/cocoindex

cocoindex-0.1.33/examples/fastapi_server_docker/README.md

@@ -0,0 +1,10 @@
+## Run cocoindex docker container with a simple query endpoint via fastapi
+In this example, we provide a simple docker container using docker compose to build pgvector17 along with a simple python fastapi script than runs a simple query endpoint. This example uses the code from the code embedding example.
+
+## How to run
+Edit the sample code directory to include the code you want to query over in
+```sample_code/``` 
+
+Edit the configuration code from the file ```src/cocoindex_funs.py``` line 23 to 25.
+
+Finally build the docker container via: ```docker compose up``` while inside the directory of the example.

cocoindex-0.1.33/examples/fastapi_server_docker/compose.yaml

@@ -0,0 +1,18 @@
+services:
+  coco_db:
+    image: pgvector/pgvector:pg17
+    restart: always
+    environment:
+      POSTGRES_USER: cocoindex
+      POSTGRES_PASSWORD: cocoindex
+      POSTGRES_DB: cocoindex
+    ports:
+      - "5432:5432"
+
+  coco_api:
+    build:
+      context: .
+    ports:
+      - 8080:8080
+    depends_on:
+      - coco_db

cocoindex-0.1.33/examples/fastapi_server_docker/dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+
+RUN pip install -r requirements.txt
+
+COPY . .
+
+RUN cat .env
+
+CMD ["sh", "-c", "echo yes | python main.py cocoindex setup && python main.py cocoindex update && python main.py"]

cocoindex-0.1.33/examples/fastapi_server_docker/main.py

@@ -0,0 +1,30 @@
+import cocoindex
+import uvicorn
+
+from fastapi import FastAPI
+from dotenv import load_dotenv
+
+from src.cocoindex_funs import code_embedding_flow, code_to_embedding
+
+fastapi_app = FastAPI()
+    
+query_handler = cocoindex.query.SimpleSemanticsQueryHandler(
+    name="SemanticsSearch",
+    flow=code_embedding_flow,
+    target_name="code_embeddings",
+    query_transform_flow=code_to_embedding,
+    default_similarity_metric=cocoindex.VectorSimilarityMetric.COSINE_SIMILARITY
+)
+
+@fastapi_app.get("/query")
+def query_endpoint(string: str):
+    results, _ = query_handler.search(string, 10)
+    return results
+
+@cocoindex.main_fn()
+def _run():
+    uvicorn.run(fastapi_app, host="0.0.0.0", port=8080)
+    
+if __name__ == "__main__":
+    load_dotenv(override=True)
+    _run()

cocoindex-0.1.33/examples/fastapi_server_docker/requirements.txt

@@ -0,0 +1,5 @@
+cocoindex>=0.1.26
+python-dotenv>=1.0.1
+fastapi==0.115.12
+fastapi-cli==0.0.7
+uvicorn==0.34.2

cocoindex-0.1.33/examples/fastapi_server_docker/sample_code/main.py

@@ -0,0 +1,113 @@
+class Button:
+    def __init__(self, label, on_click):
+        self.label = label
+        self.on_click = on_click
+
+    def click(self):
+        print(f"Button '{self.label}' clicked.")
+        if callable(self.on_click):
+            self.on_click()
+
+
+class ProgressBar:
+    def __init__(self, max_value):
+        self.max_value = max_value
+        self.current_value = 0
+
+    def update(self, value):
+        self.current_value = min(value, self.max_value)
+        self.display()
+
+    def display(self):
+        percent = (self.current_value / self.max_value) * 100
+        print(f"Progress: {percent:.2f}%")
+
+
+class Slider:
+    def __init__(self, min_value, max_value, initial_value=None):
+        self.min_value = min_value
+        self.max_value = max_value
+        self.value = initial_value if initial_value is not None else min_value
+
+    def set_value(self, new_value):
+        if self.min_value <= new_value <= self.max_value:
+            self.value = new_value
+            print(f"Slider set to {self.value}")
+        else:
+            print("Value out of range.")
+
+
+class Dropdown:
+    def __init__(self, options):
+        self.options = options
+        self.selected = None
+
+    def select(self, option):
+        if option in self.options:
+            self.selected = option
+            print(f"Dropdown selected: {option}")
+        else:
+            print("Option not available.")
+
+
+class TextField:
+    def __init__(self, placeholder=''):
+        self.placeholder = placeholder
+        self.text = ''
+
+    def input(self, new_text):
+        self.text = new_text
+        print(f"Text field updated: {self.text}")
+
+
+class Checkbox:
+    def __init__(self, label):
+        self.label = label
+        self.checked = False
+
+    def toggle(self):
+        self.checked = not self.checked
+        print(f"{self.label}: {'Checked' if self.checked else 'Unchecked'}")
+
+
+class RadioButton:
+    def __init__(self, group, label):
+        self.group = group
+        self.label = label
+        self.selected = False
+
+    def select(self):
+        self.selected = True
+        print(f"Radio button '{self.label}' selected in group '{self.group}'.")
+
+
+class ToggleSwitch:
+    def __init__(self, state=False):
+        self.state = state
+
+    def toggle(self):
+        self.state = not self.state
+        print(f"ToggleSwitch is now {'On' if self.state else 'Off'}")
+
+
+class Tooltip:
+    def __init__(self, text):
+        self.text = text
+
+    def show(self):
+        print(f"Tooltip: {self.text}")
+
+
+class Modal:
+    def __init__(self, content):
+        self.content = content
+        self.visible = False
+
+    def open(self):
+        self.visible = True
+        print("Modal opened.")
+        print(f"Content: {self.content}")
+
+    def close(self):
+        self.visible = False
+        print("Modal closed.")

cocoindex-0.1.33/examples/fastapi_server_docker/src/cocoindex_funs.py

@@ -0,0 +1,45 @@
+import cocoindex
+import os
+
+@cocoindex.op.function()
+def extract_extension(filename: str) -> str:
+    """Extract the extension of a filename."""
+    return os.path.splitext(filename)[1]
+
+def code_to_embedding(text: cocoindex.DataSlice) -> cocoindex.DataSlice:
+    """
+    Embed the text using a SentenceTransformer model.
+    """
+    return text.transform(
+        cocoindex.functions.SentenceTransformerEmbed(
+            model="sentence-transformers/all-MiniLM-L6-v2"))
+
+@cocoindex.flow_def(name="CodeEmbedding")
+def code_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
+    """
+    Define an example flow that embeds files into a vector database.
+    """
+    data_scope["files"] = flow_builder.add_source(
+        cocoindex.sources.LocalFile(path="sample_code",
+                                    included_patterns=["*.py", "*.rs", "*.toml", "*.md", "*.mdx", "*.ts", "*.tsx"],
+                                    excluded_patterns=[".*", "target", "**/node_modules"]))
+    code_embeddings = data_scope.add_collector()
+
+    with data_scope["files"].row() as file:
+        file["extension"] = file["filename"].transform(extract_extension)
+        file["chunks"] = file["content"].transform(
+            cocoindex.functions.SplitRecursively(),
+            language=file["extension"], chunk_size=1000, chunk_overlap=300)
+        with file["chunks"].row() as chunk:
+            chunk["embedding"] = chunk["text"].call(code_to_embedding)
+            code_embeddings.collect(filename=file["filename"], location=chunk["location"],
+                                    code=chunk["text"], embedding=chunk["embedding"])
+
+    code_embeddings.export(
+        "code_embeddings",
+        cocoindex.storages.Postgres(),
+        primary_key_fields=["filename", "location"],
+        vector_indexes=[
+            cocoindex.VectorIndexDef(
+                field_name="embedding",
+                metric=cocoindex.VectorSimilarityMetric.COSINE_SIMILARITY)])

{cocoindex-0.1.31 → cocoindex-0.1.33}/examples/gdrive_text_embedding/pyproject.toml

@@ -3,4 +3,4 @@ name = "gdrive-text-embedding"
 version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on Google Drive files."
 requires-python = ">=3.11"
-dependencies = ["cocoindex>=0.1.30", "python-dotenv>=1.0.1"]
+dependencies = ["cocoindex>=0.1.32", "python-dotenv>=1.0.1"]

{cocoindex-0.1.31 → cocoindex-0.1.33}/examples/manuals_llm_extraction/pyproject.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 description = "Simple example for cocoindex: extract structured information from a Markdown file using LLM."
 requires-python = ">=3.10"
 dependencies = [
-    "cocoindex>=0.1.30",
+    "cocoindex>=0.1.32",
     "python-dotenv>=1.0.1",
     "marker-pdf>=1.5.2",
 ]

{cocoindex-0.1.31 → cocoindex-0.1.33}/examples/pdf_embedding/pyproject.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 description = "Simple example for cocoindex: build embedding index based on local PDF files."
 requires-python = ">=3.10"
 dependencies = [
-    "cocoindex>=0.1.30",
+    "cocoindex>=0.1.32",
     "python-dotenv>=1.0.1",
     "marker-pdf>=1.5.2",
 ]

cocoindex-0.1.33/examples/product_taxonomy_knowledge_graph/.env

@@ -0,0 +1,3 @@
+# Postgres database address for cocoindex
+COCOINDEX_DATABASE_URL=postgres://cocoindex:cocoindex@localhost/cocoindex
+

cocoindex-0.1.33/examples/product_taxonomy_knowledge_graph/README.md

@@ -0,0 +1,66 @@
+# Build Real-Time Product Recommendation based on LLM Taxonomy Extraction and Knowledge Graph
+
+We will process a list of products and use LLM to extract the taxonomy and complimentary taxonomy for each product.
+
+Please drop [CocoIndex on Github](https://github.com/cocoindex-io/cocoindex) a star to support us and stay tuned for more updates. Thank you so much 🥥🤗. [![GitHub](https://img.shields.io/github/stars/cocoindex-io/cocoindex?color=5B5BD6)](https://github.com/cocoindex-io/cocoindex)
+
+
+## Prerequisite
+*   [Install Postgres](https://cocoindex.io/docs/getting_started/installation#-install-postgres) if you don't have one.
+*   [Install Neo4j](https://cocoindex.io/docs/ops/storages#neo4j) if you don't have one.
+*   [Configure your OpenAI API key](https://cocoindex.io/docs/ai/llm#openai).
+
+## Documentation
+You can read the official CocoIndex Documentation for Property Graph Targets [here](https://cocoindex.io/docs/ops/storages#property-graph-targets).
+
+## Run
+
+### Build the index
+
+Install dependencies:
+
+```bash
+pip install -e .
+```
+
+Setup:
+
+```bash
+python main.py cocoindex setup
+```
+
+Update index:
+
+```bash
+python main.py cocoindex update
+```
+
+### Browse the knowledge graph
+
+After the knowledge graph is built, you can explore the knowledge graph you built in Neo4j Browser.
+
+For the dev enviroment, you can connect neo4j browser using credentials:
+- username: `neo4j`
+- password: `cocoindex`
+which is pre-configured in the our docker compose [config.yaml](https://raw.githubusercontent.com/cocoindex-io/cocoindex/refs/heads/main/dev/neo4j.yaml).
+
+You can open it at [http://localhost:7474](http://localhost:7474), and run the following Cypher query to get all relationships:
+
+```cypher
+MATCH p=()-->() RETURN p
+```
+![Neo4j Browser Screenshot](img/neo4j.png)
+
+## CocoInsight 
+I used CocoInsight (Free beta now) to troubleshoot the index generation and understand the data lineage of the pipeline. 
+It just connects to your local CocoIndex server, with Zero pipeline data retention. Run following command to start CocoInsight:
+
+```bash
+python main.py cocoindex server -ci
+```
+
+And then open the url https://cocoindex.io/cocoinsight. 
+
+![CocoInsight Screenshot](img/cocoinsight.png)
+
+