symbolicai 0.21.0__tar.gz → 1.1.0__tar.gz

{symbolicai-0.21.0 → symbolicai-1.1.0}/.gitignore

@@ -162,7 +162,3 @@ symsh
 .bash_history
 .conversation_state
 .symai
-zagent-review
-zagent-research
-.codex/context
-llama.cpp

symbolicai-1.1.0/MANIFEST.in

@@ -0,0 +1,3 @@
+global-exclude test_*.py
+include symai/**/*
+exclude **/*.pyc

{symbolicai-0.21.0 → symbolicai-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: symbolicai
-Version: 0.21.0
+Version: 1.1.0
 Summary: A Neurosymbolic Perspective on Large Language Models
 Author-email: Marius-Constantin Dinu <marius@extensity.ai>, Leoveanu-Condrei Claudiu <leo@extensity.ai>
 License: BSD 3-Clause License
@@ -84,6 +84,8 @@ Requires-Dist: httpx>=0.27.2
 Requires-Dist: nest-asyncio>=1.6.0
 Requires-Dist: rich>=13.9.4
 Requires-Dist: playwright>=1.55.0
+Requires-Dist: llvmlite>=0.45.1
+Requires-Dist: cerebras-cloud-sdk>=1.59.0
 Provides-Extra: bitsandbytes
 Requires-Dist: bitsandbytes>=0.43.1; extra == "bitsandbytes"
 Provides-Extra: blip2
@@ -96,18 +98,22 @@ Requires-Dist: accelerate>=0.33.0; extra == "hf"
 Requires-Dist: peft>=0.13.1; extra == "hf"
 Requires-Dist: datasets>=3.0.1; extra == "hf"
 Requires-Dist: trl>=0.11.3; extra == "hf"
-Provides-Extra: webscraping
-Requires-Dist: beautifulsoup4>=4.12.3; extra == "webscraping"
-Requires-Dist: trafilatura>=2.0.0; extra == "webscraping"
-Requires-Dist: pdfminer.six; extra == "webscraping"
-Requires-Dist: playwright>=1.45.0; extra == "webscraping"
+Provides-Extra: scrape
+Requires-Dist: beautifulsoup4>=4.12.3; extra == "scrape"
+Requires-Dist: trafilatura>=2.0.0; extra == "scrape"
+Requires-Dist: pdfminer.six; extra == "scrape"
+Requires-Dist: playwright>=1.45.0; extra == "scrape"
+Requires-Dist: parallel-web>=0.3.3; extra == "scrape"
 Provides-Extra: llama-cpp
 Requires-Dist: llama-cpp-python[server]>=0.3.7; extra == "llama-cpp"
 Provides-Extra: wolframalpha
 Requires-Dist: wolframalpha>=5.0.0; extra == "wolframalpha"
 Provides-Extra: whisper
 Requires-Dist: openai-whisper>=20240930; extra == "whisper"
-Requires-Dist: numba>=0.60.0; extra == "whisper"
+Requires-Dist: numba>=0.62.1; extra == "whisper"
+Requires-Dist: llvmlite>=0.45.1; extra == "whisper"
+Provides-Extra: search
+Requires-Dist: parallel-web>=0.3.3; extra == "search"
 Provides-Extra: serpapi
 Requires-Dist: google_search_results>=2.4.2; extra == "serpapi"
 Provides-Extra: services
@@ -116,14 +122,20 @@ Requires-Dist: redis>=5.0.2; extra == "services"
 Requires-Dist: uvicorn>=0.27.1; extra == "services"
 Provides-Extra: solver
 Requires-Dist: z3-solver>=4.12.6.0; extra == "solver"
+Provides-Extra: qdrant
+Requires-Dist: qdrant-client; extra == "qdrant"
+Requires-Dist: chonkie[all]>=0.4.1; extra == "qdrant"
+Requires-Dist: tokenizers; extra == "qdrant"
 Provides-Extra: all
 Requires-Dist: symbolicai[hf]; extra == "all"
 Requires-Dist: symbolicai[wolframalpha]; extra == "all"
 Requires-Dist: symbolicai[whisper]; extra == "all"
-Requires-Dist: symbolicai[webscraping]; extra == "all"
+Requires-Dist: symbolicai[scrape]; extra == "all"
+Requires-Dist: symbolicai[search]; extra == "all"
 Requires-Dist: symbolicai[serpapi]; extra == "all"
 Requires-Dist: symbolicai[services]; extra == "all"
 Requires-Dist: symbolicai[solver]; extra == "all"
+Requires-Dist: symbolicai[qdrant]; extra == "all"
 Dynamic: license-file
 
 # **SymbolicAI: A neuro-symbolic perspective on LLMs**
@@ -293,7 +305,7 @@ pip install "symbolicai[llamacpp]",
 pip install "symbolicai[bitsandbytes]",
 pip install "symbolicai[wolframalpha]",
 pip install "symbolicai[whisper]",
-pip install "symbolicai[webscraping]",
+pip install "symbolicai[scrape]",
 pip install "symbolicai[serpapi]",
 pip install "symbolicai[services]",
 pip install "symbolicai[solver]"
@@ -313,7 +325,7 @@ uv sync --frozen
 To install optional extras via uv:
 ```bash
 uv sync --extra all # all optional extras
-uv sync --extra webscraping # only webscraping
+uv sync --extra scrape # only scrape
 ```
 
 > ❗️**NOTE**❗️Please note that some of these optional dependencies may require additional installation steps. Additionally, some are only experimentally supported now and may not work as expected. If a feature is extremely important to you, please consider contributing to the project or reaching out to us.

{symbolicai-0.21.0 → symbolicai-1.1.0}/README.md

@@ -165,7 +165,7 @@ pip install "symbolicai[llamacpp]",
 pip install "symbolicai[bitsandbytes]",
 pip install "symbolicai[wolframalpha]",
 pip install "symbolicai[whisper]",
-pip install "symbolicai[webscraping]",
+pip install "symbolicai[scrape]",
 pip install "symbolicai[serpapi]",
 pip install "symbolicai[services]",
 pip install "symbolicai[solver]"
@@ -185,7 +185,7 @@ uv sync --frozen
 To install optional extras via uv:
 ```bash
 uv sync --extra all # all optional extras
-uv sync --extra webscraping # only webscraping
+uv sync --extra scrape # only scrape
 ```
 
 > ❗️**NOTE**❗️Please note that some of these optional dependencies may require additional installation steps. Additionally, some are only experimentally supported now and may not work as expected. If a feature is extremely important to you, please consider contributing to the project or reaching out to us.

symbolicai-1.1.0/docs/source/ENGINES/indexing_engine.md

@@ -0,0 +1,283 @@
+# Indexing Engine
+
+SymbolicAI supports multiple indexing engines for vector search and RAG (Retrieval-Augmented Generation) operations. This document covers both the default naive vector engine and the production-ready Qdrant engine.
+
+## Naive Vector Engine (Default)
+
+By default, text indexing and retrieval is performed with the local naive vector engine using the `Interface` abstraction:
+
+```python
+from symai.interfaces import Interface
+
+db = Interface('naive_vectordb', index_name="my_index")
+db("Hello world", operation="add")
+result = db("Hello", operation="search", top_k=1)
+print(result.value)  # most relevant match
+```
+
+You can also add or search multiple documents at once, and perform save/load/purge operations:
+
+```python
+docs = ["Alpha document", "Beta entry", "Gamma text"]
+db = Interface('naive_vectordb', index_name="my_index")
+db(docs, operation="add")
+db("save", operation="config")
+# Load or purge as needed
+```
+
+## Qdrant RAG Engine
+
+The Qdrant engine provides a production-ready vector database for scalable RAG applications. It supports both local and cloud deployments, advanced document chunking, and comprehensive collection management.
+
+### Setup
+
+#### Option 1: Local Qdrant Server
+
+Start a local Qdrant server using the built-in wrapper:
+
+```bash
+# Using Docker (default)
+python -m symai.server.qdrant_server
+
+# Using Qdrant binary
+python -m symai.server.qdrant_server --mode binary --binary-path /path/to/qdrant
+
+# Custom configuration
+python -m symai.server.qdrant_server --host 0.0.0.0 --port 6333 --storage-path ./qdrant_storage
+```
+
+#### Option 2: Cloud Qdrant
+
+Configure your cloud Qdrant instance:
+
+```python
+import os
+os.environ["INDEXING_ENGINE_URL"] = "https://your-cluster.qdrant.io"
+os.environ["INDEXING_ENGINE_API_KEY"] = "your-api-key"
+```
+
+### Basic Usage
+
+The Qdrant engine automatically registers when a Qdrant server is available. Use it through the `Interface` abstraction:
+
+```python
+from symai.interfaces import Interface
+
+# Qdrant engine is used automatically when server is running
+db = Interface('naive_vectordb', index_name="my_collection")
+db("Hello world", operation="add")
+result = db("Hello", operation="search", top_k=5)
+print(result.value)  # list of relevant matches
+```
+
+### Advanced Usage: Direct Engine Access
+
+For more control, use the Qdrant engine directly:
+
+```python
+from symai.backend.engines.index.engine_qdrant import QdrantIndexEngine
+
+# Initialize engine
+engine = QdrantIndexEngine(
+    url="http://localhost:6333",  # or your cloud URL
+    api_key=None,  # optional, for cloud instances
+    index_name="my_collection",
+    index_dims=1536,  # embedding dimension
+    index_top_k=5,  # default top-k for searches
+    index_metric="Cosine"  # Cosine, Dot, or Euclidean
+)
+```
+
+### Collection Management
+
+Create and manage collections programmatically:
+
+```python
+import asyncio
+
+async def manage_collections():
+    # Create a new collection
+    await engine.create_collection(
+        collection_name="documents",
+        vector_size=1536,
+        distance="Cosine"
+    )
+
+    # Check if collection exists
+    exists = await engine.collection_exists("documents")
+
+    # List all collections
+    collections = await engine.list_collections()
+
+    # Get collection info
+    info = await engine.get_collection_info("documents")
+    print(f"Points: {info['points_count']}")
+
+    # Delete collection
+    await engine.delete_collection("documents")
+
+asyncio.run(manage_collections())
+```
+
+### Document Chunking and RAG
+
+The Qdrant engine includes built-in document chunking for RAG workflows:
+
+```python
+import asyncio
+
+async def add_documents():
+    # Chunk and index text directly
+    num_chunks = await engine.chunk_and_upsert(
+        collection_name="documents",
+        text="Your long document text here...",
+        metadata={"source": "manual_input", "author": "John Doe"}
+    )
+    print(f"Indexed {num_chunks} chunks")
+
+    # Chunk and index from a file (PDF, DOCX, etc.)
+    num_chunks = await engine.chunk_and_upsert(
+        collection_name="documents",
+        document_path="/path/to/document.pdf",
+        metadata={"source": "document.pdf"}
+    )
+
+    # Chunk and index from a URL
+    num_chunks = await engine.chunk_and_upsert(
+        collection_name="documents",
+        document_url="https://example.com/document.pdf",
+        metadata={"source": "url"}
+    )
+
+    # Custom chunker configuration
+    num_chunks = await engine.chunk_and_upsert(
+        collection_name="documents",
+        text="Your text...",
+        chunker_name="RecursiveChunker",
+        chunker_kwargs={"chunk_size": 512, "chunk_overlap": 50}
+    )
+
+asyncio.run(add_documents())
+```
+
+### Point Operations
+
+For fine-grained control over individual vectors:
+
+```python
+import asyncio
+import numpy as np
+
+async def point_operations():
+    # Upsert points with embeddings
+    points = [
+        {
+            "id": 1,
+            "vector": [0.1] * 1536,  # your embedding vector
+            "payload": {"text": "Document 1", "category": "tech"}
+        },
+        {
+            "id": 2,
+            "vector": [0.2] * 1536,
+            "payload": {"text": "Document 2", "category": "science"}
+        }
+    ]
+    await engine.upsert(collection_name="documents", points=points)
+
+    # Retrieve points by ID
+    retrieved = await engine.retrieve(
+        collection_name="documents",
+        ids=[1, 2],
+        with_payload=True,
+        with_vectors=False
+    )
+
+    # Search for similar vectors
+    query_vector = [0.15] * 1536
+    results = await engine.search(
+        collection_name="documents",
+        query_vector=query_vector,
+        limit=10,
+        score_threshold=0.7  # optional minimum similarity
+    )
+
+    # Delete points
+    await engine.delete(collection_name="documents", points_selector=[1, 2])
+
+asyncio.run(point_operations())
+```
+
+### Configuration Options
+
+The Qdrant engine supports extensive configuration:
+
+```python
+engine = QdrantIndexEngine(
+    # Connection settings
+    url="http://localhost:6333",
+    api_key="your-api-key",  # for cloud instances
+
+    # Collection settings
+    index_name="default_collection",
+    index_dims=1536,  # must match your embedding model
+    index_top_k=5,
+    index_metric="Cosine",  # Cosine, Dot, or Euclidean
+
+    # Chunking settings
+    chunker_name="RecursiveChunker",
+    tokenizer_name="gpt2",
+    embedding_model_name="minishlab/potion-base-8M",
+
+    # Retry settings
+    tries=20,
+    delay=0.5,
+    max_delay=-1,
+    backoff=1,
+    jitter=0
+)
+```
+
+### Environment Variables
+
+Configure Qdrant via environment variables:
+
+```bash
+# Qdrant connection
+export INDEXING_ENGINE_URL="http://localhost:6333"
+export INDEXING_ENGINE_API_KEY="your-api-key"  # optional
+
+# Embedding model (shared with other engines)
+export EMBEDDING_ENGINE_API_KEY=""  # empty for local, or API key for cloud
+export EMBEDDING_ENGINE_MODEL="all-mpnet-base-v2"  # or your preferred model
+```
+
+### Embedding Model & API Key Behavior
+
+- **If `EMBEDDING_ENGINE_API_KEY` is empty (`""`, the default),** SymbolicAI will use a local, lightweight embedding engine based on SentenceTransformers. You can specify any supported model name via `EMBEDDING_ENGINE_MODEL` (e.g. `"all-mpnet-base-v2"`).
+- **If you DO provide an `EMBEDDING_ENGINE_API_KEY`**, then the respective remote embedding engine will be used (e.g. OpenAI). The model is selected according to the `EMBEDDING_ENGINE_MODEL` key where applicable.
+
+This allows you to easily experiment locally for free, and switch to more powerful cloud backends when ready.
+
+### Installation
+
+Install Qdrant support using the package extra (recommended):
+
+```bash
+pip install symai[qdrant]
+```
+
+This installs all required dependencies:
+- `qdrant-client` - Qdrant Python client
+- `chonkie[all]` - Document chunking library
+- `tokenizers` - Tokenization support
+
+Alternatively, install dependencies individually:
+
+```bash
+pip install qdrant-client chonkie tokenizers
+```
+
+### See Also
+
+- See `tests/engines/index/test_qdrant_engine.py` for comprehensive usage examples
+- Qdrant documentation: https://qdrant.tech/documentation/

{symbolicai-0.21.0 → symbolicai-1.1.0}/docs/source/ENGINES/neurosymbolic_engine.md

@@ -1,12 +1,13 @@
 # Neuro-Symbolic Engine
 
 The **neuro-symbolic** engine is our generic wrapper around large language models (LLMs) that support prompts, function/tool calls, vision tokens, token‐counting/truncation, etc.
-Depending on which backend you configure (OpenAI/GPT, Claude, Gemini, Deepseek, Groq, llama.cpp, HuggingFace, …), a few things must be handled differently:
+Depending on which backend you configure (OpenAI/GPT, Claude, Gemini, Deepseek, Groq, Cerebras, llama.cpp, HuggingFace, …), a few things must be handled differently:
 
 * GPT-family (OpenAI) and most backends accept the usual `max_tokens`, `temperature`, etc., out of the box.
-* Claude (Anthropic), Gemini (Google), Deepseek, and Qwen (Groq) can return an internal "thinking trace" when you enable it.
+* Claude (Anthropic), Gemini (Google), Deepseek, Cerebras, and Qwen (Groq) can return an internal "thinking trace" when you enable it.
 * Local engines (llamacpp, HuggingFace) do *not* yet support token counting, JSON format enforcement, or vision inputs in the same way.
 * Groq engine requires a special format for the `NEUROSYMBOLIC_ENGINE_MODEL` key: `groq:model_id`. E.g., `groq:qwen/qwen3-32b`.
+* Cerebras engine requires a special format for the `NEUROSYMBOLIC_ENGINE_MODEL` key: `cerebras:model_id`. E.g., `cerebras:gpt-oss-120b`.
 * Token‐truncation and streaming are handled automatically but may vary in behavior by engine.
 
 > ❗️**NOTE**❗️the most accurate documentation is the _code_, so be sure to check out the tests. Look for the `mandatory` mark since those are the features that were tested and are guaranteed to work.
@@ -170,6 +171,25 @@ print(res)
 print(metadata["thinking"])
 ```
 
+### Cerebras
+
+```python
+from symai import Symbol
+
+# cerebras:gpt-oss-120b
+res, metadata = Symbol("Topic: Disneyland") \
+    .query(
+      "Write a dystopic take on the topic.",
+      return_metadata=True,
+      reasoning_effort="medium",   # forwarded as Cerebras reasoning_effort
+      disable_reasoning=False      # forwarded as disable_reasoning
+    )
+print(res)
+print(metadata["thinking"])
+```
+
+For Cerebras backends, `symai` collects the reasoning trace from either the dedicated `reasoning` field on the message (when present) or from `<think>…</think>` blocks embedded in the content. In both cases the trace is exposed as `metadata["thinking"]` and removed from the final user-facing text.
+
 ---
 
 ## JSON‐Only Responses

symbolicai-1.1.0/docs/source/ENGINES/scrape_engine.md

@@ -0,0 +1,43 @@
+# Scrape Engine
+
+## Naive Scrape
+
+To access data from the web, we can use the `naive_scrape` interface. The engine underneath is very lightweight and can be used to scrape data from websites. It is based on the `requests` library, as well as `trafilatura` for output formatting, and `bs4` for HTML parsing. `trafilatura` currently supports the following output formats: `json`, `csv`, `html`, `markdown`, `text`, `xml`
+
+```python
+from symai.interfaces import Interface
+
+scraper = Interface("naive_scrape")
+url = "https://docs.astral.sh/uv/guides/scripts/#next-steps"
+res = scraper(url)
+```
+
+## Parallel (Parallel.ai)
+
+The Parallel.ai integration routes scrape calls through the official `parallel-web` SDK and can handle PDFs, JavaScript-heavy feeds, and standard HTML pages in the same workflow. Instantiate the Parallel interface and call `.scrape(...)` with the target URL. The engine detects scrape requests automatically whenever a URL is supplied.
+
+```python
+from symai.extended import Interface
+
+scraper = Interface("parallel")
+article = scraper.scrape(
+    "https://trafilatura.readthedocs.io/en/latest/crawls.html",
+    full_content=True,           # optional: request full document text
+    excerpts=True,               # optional: default True, retain excerpt snippets
+    objective="Summarize crawl guidance for internal notes."
+)
+print(str(article))
+```
+
+Configuration requires a Parallel API key and the Parallel model token. Add the following to your settings:
+
+```bash
+{
+    …
+    "SEARCH_ENGINE_API_KEY": "…",
+    "SEARCH_ENGINE_MODEL": "parallel"
+    …
+}
+```
+
+When invoked with a URL, the engine hits Parallel's Extract API and returns an `ExtractResult`. The result string joins excerpts or the full content if requested. Because processing is offloaded to Parallel's hosted infrastructure, the engine remains reliable on dynamic pages that the naive scraper cannot render. Install the dependency with `pip install parallel-web` before enabling this engine.

{symbolicai-0.21.0 → symbolicai-1.1.0}/docs/source/ENGINES/search_engine.md

@@ -1,5 +1,54 @@
 # Search Engine
 
+## Parallel (Parallel.ai)
+Parallel.ai provides unified search and scrape capabilities through the `parallel-web` SDK. When you invoke `.search(...)` on the Parallel interface, responses are normalized into a `SearchResult` that flattens excerpt text and appends inline citation markers. All URLs are deduplicated and cleaned of tracking parameters, and you can access structured citations via `result.get_citations()`.
+
+```python
+from symai.extended import Interface
+
+engine = Interface("parallel")
+result = engine.search(
+    "Latest advances in quantum error correction",
+    max_results=5,
+    allowed_domains=["arxiv.org", ".gov"],
+    objective="Find peer-reviewed summaries.",
+)
+
+print(str(result))
+print(result.get_citations())
+```
+
+The engine accepts either a single string or a list of query strings. Domain filters are normalized to apex domains and capped at 10 entries, mirroring the underlying Parallel API requirements. You can also change the search `mode` (default `"one-shot"`) or adjust `max_chars_per_result` to constrain excerpt length.
+
+### Task processors via the search route
+
+If you pass a `processor` kwarg to `engine.search(...)`, SymbolicAI automatically switches to Parallel's task route while keeping the familiar `SearchResult` interface. The query (or prepared input) becomes the task input, and any citations or reasoning returned by the processor are converted back into inline search excerpts. This allows LLM-style task processors to power the same downstream consumers that expect search responses.
+
+```python
+task_result = engine.search(
+    "Timisoara housing price index 2010-2025",
+    processor="base",
+    task_api_timeout=600,   # optional, forwarded to task_run.result
+    task_output_schema={"type": "json"},  # optional schema/task spec hint
+)
+
+print(task_result.value)          # flattened text with citations
+print(task_result.raw["task_output"])  # raw processor output payload
+```
+
+Any `allowed_domains` filters are forwarded via `source_policy` to the task run. You can also provide `task_timeout` (client-side polling window) or `task_api_timeout` (server-side execution window) for long-running tasks. When no processor is supplied, the engine uses the standard Parallel search route described above.
+
+Enable the engine by installing `parallel-web` and configuring the Parallel credentials in your settings:
+
+```bash
+{
+    …
+    "SEARCH_ENGINE_API_KEY": "…",
+    "SEARCH_ENGINE_MODEL": "parallel"
+    …
+}
+```
+
 ## SerpApi (Google)
 To obtain fact-based content, we can perform search queries via `SerpApi` with a `Google` backend. The following example demonstrates how to search for a query and return the results:
 
@@ -14,7 +63,7 @@ Here's a quick example for how to set it up:
 ```bash
 {
     …
-    "SEARCH_ENGINE_API_KEY": …,
+    "SEARCH_ENGINE_API_KEY": "…",
     "SEARCH_ENGINE_ENGINE": "google",
     …
 }
@@ -34,7 +83,7 @@ Please note that the system_message is optional and can be used to provide conte
 ```bash
 {
     …
-    "SEARCH_ENGINE_API_KEY": "pplx-…",
+    "SEARCH_ENGINE_API_KEY": "…",
     "SEARCH_ENGINE_MODEL": "sonar",
     …
 }
@@ -96,7 +145,7 @@ Here's how to configure the OpenAI search engine:
 ```bash
 {
     …
-    "SEARCH_ENGINE_API_KEY": "sk-…",
+    "SEARCH_ENGINE_API_KEY": "…",
     "SEARCH_ENGINE_MODEL": "gpt-4.1-mini",
     …
 }