gnosisllm-knowledge 0.2.0__py3-none-any.whl

gnosisllm_knowledge-0.2.0.dist-info/METADATA

@@ -0,0 +1,382 @@
+Metadata-Version: 2.4
+Name: gnosisllm-knowledge
+Version: 0.2.0
+Summary: Enterprise-grade knowledge loading, indexing, and search for Python
+License: MIT
+Keywords: knowledge-base,rag,semantic-search,vector-search,opensearch,llm,embeddings,enterprise
+Author: David Marsa
+Author-email: david.marsa@neomanex.com
+Requires-Python: >=3.11,<4.0
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing :: Indexing
+Classifier: Typing :: Typed
+Classifier: Framework :: AsyncIO
+Requires-Dist: aiohttp (>=3.13.2,<4.0.0)
+Requires-Dist: httpx (>=0.28.1,<0.29.0)
+Requires-Dist: opensearch-py (>=3.1.0,<4.0.0)
+Requires-Dist: pydantic (>=2.12.5,<3.0.0)
+Requires-Dist: pydantic-settings (>=2.12.0,<3.0.0)
+Requires-Dist: python-dotenv (>=1.2.1,<2.0.0)
+Requires-Dist: rich (>=13.7.0,<14.0.0)
+Requires-Dist: typer (>=0.12.0,<1.0.0)
+Project-URL: Changelog, https://github.com/gnosisllm/gnosisllm-knowledge/blob/main/CHANGELOG.md
+Project-URL: Documentation, https://gnosisllm-knowledge.readthedocs.io
+Project-URL: Homepage, https://github.com/gnosisllm/gnosisllm-knowledge
+Project-URL: Repository, https://github.com/gnosisllm/gnosisllm-knowledge
+Description-Content-Type: text/markdown
+
+# GnosisLLM Knowledge
+
+Enterprise-grade knowledge loading, indexing, and semantic search library for Python.
+
+## Features
+
+- **Semantic Search**: Vector-based similarity search using OpenAI embeddings
+- **Hybrid Search**: Combine semantic and keyword (BM25) search for best results
+- **Multiple Loaders**: Load content from websites, sitemaps, and files
+- **Intelligent Chunking**: Sentence-aware text splitting with configurable overlap
+- **OpenSearch Backend**: Production-ready with k-NN vector search
+- **Multi-Tenancy**: Built-in support for account and collection isolation
+- **Event-Driven**: Observer pattern for progress tracking and monitoring
+- **SOLID Architecture**: Clean, maintainable, and extensible codebase
+
+## Installation
+
+```bash
+pip install gnosisllm-knowledge
+
+# With OpenSearch backend
+pip install gnosisllm-knowledge[opensearch]
+
+# With all optional dependencies
+pip install gnosisllm-knowledge[all]
+```
+
+## Quick Start (CLI)
+
+```bash
+# Install
+pip install gnosisllm-knowledge
+
+# Set OpenAI API key for embeddings
+export OPENAI_API_KEY=sk-...
+
+# Setup OpenSearch with ML model
+gnosisllm-knowledge setup --host localhost --port 9200
+# ✓ Created connector, model, pipelines, index
+# Model ID: abc123  →  Add to .env: OPENSEARCH_MODEL_ID=abc123
+
+export OPENSEARCH_MODEL_ID=abc123
+
+# Load content from a sitemap
+gnosisllm-knowledge load https://docs.example.com/sitemap.xml
+# ✓ Loaded 247 documents (1,248 chunks) in 45.3s
+
+# Search
+gnosisllm-knowledge search "how to configure authentication"
+# Found 42 results (23.4ms)
+# 1. Authentication Guide (92.3%)
+#    To configure authentication, set AUTH_PROVIDER...
+
+# Interactive search mode
+gnosisllm-knowledge search --interactive
+```
+
+## Quick Start (Python API)
+
+```python
+from gnosisllm_knowledge import Knowledge
+
+# Create instance with OpenSearch backend
+knowledge = Knowledge.from_opensearch(
+    host="localhost",
+    port=9200,
+)
+
+# Setup backend (creates indices)
+await knowledge.setup()
+
+# Load and index a sitemap
+await knowledge.load(
+    "https://docs.example.com/sitemap.xml",
+    collection_id="docs",
+)
+
+# Search
+results = await knowledge.search("how to configure authentication")
+for item in results.items:
+    print(f"{item.title}: {item.score}")
+```
+
+## CLI Commands
+
+### Setup
+
+Configure OpenSearch with neural search capabilities:
+
+```bash
+gnosisllm-knowledge setup [OPTIONS]
+
+Options:
+  --host        OpenSearch host (default: localhost)
+  --port        OpenSearch port (default: 9200)
+  --use-ssl     Enable SSL connection
+  --force       Clean up existing resources first
+  --no-hybrid   Skip hybrid search pipeline
+```
+
+### Load
+
+Load and index content from URLs or sitemaps:
+
+```bash
+gnosisllm-knowledge load <URL> [OPTIONS]
+
+Options:
+  --type         Source type: website, sitemap (auto-detects)
+  --index        Target index name (default: knowledge)
+  --account-id   Multi-tenant account ID
+  --collection-id Collection grouping ID
+  --batch-size   Documents per batch (default: 100)
+  --max-urls     Max URLs from sitemap (default: 1000)
+  --dry-run      Preview without indexing
+```
+
+### Search
+
+Search indexed content with multiple modes:
+
+```bash
+gnosisllm-knowledge search <QUERY> [OPTIONS]
+
+Options:
+  --mode         Search mode: semantic, keyword, hybrid, agentic
+  --index        Index to search (default: knowledge)
+  --limit        Max results (default: 5)
+  --account-id   Filter by account
+  --collection-ids Filter by collections (comma-separated)
+  --json         Output as JSON for scripting
+  --interactive  Interactive search session
+```
+
+## Architecture
+
+```
+gnosisllm-knowledge/
+├── api/                 # High-level Knowledge facade
+├── core/
+│   ├── domain/          # Document, SearchQuery, SearchResult models
+│   ├── interfaces/      # Protocol definitions (IContentLoader, etc.)
+│   ├── events/          # Event system for progress tracking
+│   └── exceptions.py    # Exception hierarchy
+├── loaders/             # Content loaders (website, sitemap)
+├── fetchers/            # Content fetchers (HTTP, Neoreader)
+├── chunking/            # Text chunking strategies
+├── backends/
+│   ├── opensearch/      # OpenSearch implementation
+│   └── memory/          # In-memory backend for testing
+└── services/            # Indexing and search orchestration
+```
+
+## Search Modes
+
+```python
+from gnosisllm_knowledge import SearchMode
+
+# Semantic search (vector similarity)
+results = await knowledge.search(query, mode=SearchMode.SEMANTIC)
+
+# Keyword search (BM25)
+results = await knowledge.search(query, mode=SearchMode.KEYWORD)
+
+# Hybrid search (default - combines both)
+results = await knowledge.search(query, mode=SearchMode.HYBRID)
+```
+
+## Agentic Search
+
+AI-powered search with reasoning and natural language answers using OpenSearch ML agents.
+
+**Requirements:** OpenSearch 3.4+ for conversational memory support.
+
+### Setup
+
+```bash
+# 1. First run standard setup (creates embedding model)
+gnosisllm-knowledge setup --port 9201
+
+# 2. Setup agentic agents (creates LLM connector, VectorDBTool, MLModelTool, agents)
+gnosisllm-knowledge agentic setup
+# ✓ Flow Agent ID: abc123
+# ✓ Conversational Agent ID: def456
+
+# 3. Add agent IDs to environment
+export OPENSEARCH_FLOW_AGENT_ID=abc123
+export OPENSEARCH_CONVERSATIONAL_AGENT_ID=def456
+```
+
+### Usage
+
+```bash
+# Single-turn agentic search (uses flow agent)
+gnosisllm-knowledge search --mode agentic "What is Typer?"
+
+# Interactive multi-turn chat (uses conversational agent with memory)
+gnosisllm-knowledge agentic chat
+# You: What is Typer?
+# Assistant: Typer is a library for building CLI applications...
+# You: What did you just say about it?
+# Assistant: I told you that Typer is a library for building CLI...
+```
+
+### How It Works
+
+```
+┌─────────────────────────────────────────────────────────────────────┐
+│                         User Query                                   │
+│                    "What is Typer?"                                  │
+└─────────────────────────────────────────────────────────────────────┘
+                              │
+                              ▼
+┌─────────────────────────────────────────────────────────────────────┐
+│                    OpenSearch ML Agent                               │
+│              (Flow or Conversational)                                │
+└─────────────────────────────────────────────────────────────────────┘
+                              │
+            ┌─────────────────┴─────────────────┐
+            ▼                                   ▼
+┌───────────────────────┐           ┌───────────────────────┐
+│     VectorDBTool      │           │   Conversation Memory │
+│  (Knowledge Search)   │           │   (Conversational     │
+│                       │           │    Agent Only)        │
+│  - Searches index     │           │                       │
+│  - Returns context    │           │  - Stores Q&A pairs   │
+│                       │           │  - Injects chat_history│
+└───────────────────────┘           └───────────────────────┘
+            │                                   │
+            └─────────────────┬─────────────────┘
+                              ▼
+┌─────────────────────────────────────────────────────────────────────┐
+│                      MLModelTool (answer_generator)                  │
+│                                                                      │
+│  Prompt Template:                                                    │
+│  ┌────────────────────────────────────────────────────────────────┐ │
+│  │ Context from knowledge base:                                    │ │
+│  │ ${parameters.knowledge_search.output}                           │ │
+│  │                                                                 │ │
+│  │ Previous conversation:        ← Only for conversational agent  │ │
+│  │ ${parameters.chat_history:-}                                    │ │
+│  │                                                                 │ │
+│  │ Question: ${parameters.question}                                │ │
+│  └────────────────────────────────────────────────────────────────┘ │
+└─────────────────────────────────────────────────────────────────────┘
+                              │
+                              ▼
+┌─────────────────────────────────────────────────────────────────────┐
+│                      AI-Generated Answer                             │
+│  "Typer is a library for building CLI applications in Python..."    │
+└─────────────────────────────────────────────────────────────────────┘
+```
+
+### Agent Types
+
+| Agent | Type | Use Case | Memory |
+|-------|------|----------|--------|
+| Flow | `flow` | Fast single-turn RAG, API calls | No |
+| Conversational | `conversational_flow` | Multi-turn dialogue, chat | Yes |
+
+### Key Configuration
+
+The conversational agent requires these settings for memory to work:
+
+```python
+# In agent registration (setup.py)
+agent_body = {
+    "type": "conversational_flow",
+    "app_type": "rag",  # Required for memory injection
+    "llm": {
+        "model_id": llm_model_id,
+        "parameters": {
+            "message_history_limit": 10,  # Include last N messages
+        },
+    },
+    "memory": {"type": "conversation_index"},
+}
+
+# MLModelTool prompt must include:
+# ${parameters.chat_history:-}  ← Receives conversation history
+```
+
+## Multi-Tenancy
+
+```python
+# Load with tenant isolation
+await knowledge.load(
+    source="https://docs.example.com/sitemap.xml",
+    account_id="tenant-123",
+    collection_id="docs",
+)
+
+# Search within tenant
+results = await knowledge.search(
+    "query",
+    account_id="tenant-123",
+    collection_ids=["docs"],
+)
+```
+
+## Event Tracking
+
+```python
+from gnosisllm_knowledge import EventType
+
+# Subscribe to events
+@knowledge.events.on(EventType.DOCUMENT_INDEXED)
+def on_indexed(event):
+    print(f"Indexed: {event.document_id}")
+
+@knowledge.events.on(EventType.BATCH_COMPLETED)
+def on_batch(event):
+    print(f"Batch complete: {event.documents_indexed} docs")
+```
+
+## Configuration
+
+```python
+from gnosisllm_knowledge import OpenSearchConfig
+
+# From environment variables
+config = OpenSearchConfig.from_env()
+
+# Explicit configuration
+config = OpenSearchConfig(
+    host="search.example.com",
+    port=443,
+    use_ssl=True,
+    username="admin",
+    password="secret",
+    embedding_model="text-embedding-3-small",
+    embedding_dimension=1536,
+)
+
+knowledge = Knowledge.from_opensearch(config=config)
+```
+
+## Requirements
+
+- Python 3.11+
+- OpenSearch 2.0+ (for production use)
+- OpenSearch 3.4+ (for agentic search with conversation memory)
+
+## License
+
+MIT
+

gnosisllm_knowledge-0.2.0.dist-info/RECORD

@@ -0,0 +1,64 @@
+gnosisllm_knowledge/__init__.py,sha256=4pqB289D-9CG7nTnzLfGLwZFTkmnl9REtOAhyDhKE-g,3675
+gnosisllm_knowledge/api/__init__.py,sha256=YOiYXDvA1UPFKpZve_F-wKxN_NNyyiftHEzqsYmRd0I,129
+gnosisllm_knowledge/api/knowledge.py,sha256=on00xHGBe80xq1q2qtmAFTLrdeWGRUKGP8EUiTKVk0o,16791
+gnosisllm_knowledge/backends/__init__.py,sha256=8O0Lsi7gDVbviUmHFjy2rV4eZ2R354Amq-G2EL2eZtk,708
+gnosisllm_knowledge/backends/memory/__init__.py,sha256=Q_2Fh0-Xs1f7CsNn9V7z8NLW0P7F1owmxZQ4k3fTSXo,238
+gnosisllm_knowledge/backends/memory/indexer.py,sha256=R_Yd1gEplzS01o7QfPxjaAZ21VxjfdvatbgrNsyp5lM,10517
+gnosisllm_knowledge/backends/memory/searcher.py,sha256=k6OGtppC-UsHB8aIhlK3wSASVOPEokfe6LLV1jbmE5M,15547
+gnosisllm_knowledge/backends/opensearch/__init__.py,sha256=wwg3QPVVMLkq5qHUGw_9m6gvXO3hxa-xaBj3_BRJlN4,679
+gnosisllm_knowledge/backends/opensearch/agentic.py,sha256=UJdiVDaxQARigOQz0cVfPrduxF8Oa7EH96lorjwji98,25462
+gnosisllm_knowledge/backends/opensearch/config.py,sha256=P30mOAVpo4bEKNHHP04lX2ttzNXgDbu1i-SfobcURNk,7256
+gnosisllm_knowledge/backends/opensearch/indexer.py,sha256=fkfpCwg7Ka2UwDl2L0V1z9sXlvnQbyYMiFeKk1Q1D1Q,15101
+gnosisllm_knowledge/backends/opensearch/mappings.py,sha256=ssev3HFLf82lB1guzqZ0iDQE1C6ppvG1HNGVjG4vwAU,7873
+gnosisllm_knowledge/backends/opensearch/queries.py,sha256=IEZ_teGqc80OFrzD4XIwUArXRre-UhQWIH2oZnRGvC8,13301
+gnosisllm_knowledge/backends/opensearch/searcher.py,sha256=j6j0kzLSxgDRCk1fkqZ642rAXONLa-hnzZr1qEQ1CKI,11594
+gnosisllm_knowledge/backends/opensearch/setup.py,sha256=mZoHBxMK_WZA63F5PviLvGLeH8CqmAn9XM0FnuJXfrQ,52319
+gnosisllm_knowledge/chunking/__init__.py,sha256=XKrk04DqwhhPxxP_SLGnQrVUo2fexmvzuEN9YzO4Jxk,225
+gnosisllm_knowledge/chunking/fixed.py,sha256=LXZhRbYbtg4hQzqrAKyGrM_zmDhqJWXDi6cHOiJlGrM,3905
+gnosisllm_knowledge/chunking/sentence.py,sha256=EqY2Y1dpfNK_qukXQC9QYrKN8b1dqyyX9TZN5bUYGqs,7271
+gnosisllm_knowledge/cli/__init__.py,sha256=G8AO-LwNQqXSKZCFxLiXggOQf-xmNITWVs25xWl_-Cg,564
+gnosisllm_knowledge/cli/app.py,sha256=4Nm5ccxjNHQw9jy9VH7EJ7-rn138i_fPU1yzaef8ELE,15132
+gnosisllm_knowledge/cli/commands/__init__.py,sha256=TGrGl_AxVVXpAv01iO4JCUtWW8wzSQTBtQFj4vPeE0A,303
+gnosisllm_knowledge/cli/commands/agentic.py,sha256=XGZyDKu2TkNNhetzbDIp5zFbBeLwgSyfFfR3VZgZRQM,19836
+gnosisllm_knowledge/cli/commands/load.py,sha256=RZhLtw2Ba6LCceLTD665QE4QcOm-HBc9D-lEX1ReGLM,13117
+gnosisllm_knowledge/cli/commands/search.py,sha256=lj3kUXmvSZRA_z8utbN2Ph_2NczOuzGkuxILreJuj_o,15326
+gnosisllm_knowledge/cli/commands/setup.py,sha256=7NtGGulAggVVQVvzcDq-YLPZlqpGtdKDnYVXd1KPpiQ,7596
+gnosisllm_knowledge/cli/display/__init__.py,sha256=hLQ3XCBxw8C5WOT-wu1lqFyrD7FJVPHSWXUZM24Qnjc,155
+gnosisllm_knowledge/cli/display/service.py,sha256=klGJtc-w7LhNWUaG530QQIazc6c5t8laIM6bnwdyhlg,17537
+gnosisllm_knowledge/cli/utils/__init__.py,sha256=BlzSMhfXM_x9EAv304krVb_DHvdJaKLrlbb716LKkQI,106
+gnosisllm_knowledge/cli/utils/config.py,sha256=8iGzJfInv83lucAHSDfxA5wbTIzh9MlqB2-xa4vcwJg,7292
+gnosisllm_knowledge/core/__init__.py,sha256=yeZYSnaypTdStzt3_FERALGUgfC5fuudTCGixUm1_z0,1827
+gnosisllm_knowledge/core/domain/__init__.py,sha256=15IWkFNV-VKQnInFxMtY4FYh7pNmjYXDq4MPpSsAoPk,938
+gnosisllm_knowledge/core/domain/document.py,sha256=FYvM4Vi1DQeDIq4OEY94dX9eNMpQCmkDQRHkyYqq_WY,7765
+gnosisllm_knowledge/core/domain/result.py,sha256=xyPfselaG7EMhvmcYjepFOEseM4Mfc15Q1Ciq7KxQFE,5843
+gnosisllm_knowledge/core/domain/search.py,sha256=uMJZhGCQ_kS_R5kb1hN5VNEyGbWkuRL_ON_BR4BoqSM,10569
+gnosisllm_knowledge/core/domain/source.py,sha256=kKmWr3U01sJkfG8jtzAQDzxtVVzUOTSqEJZT5gGlYU8,4493
+gnosisllm_knowledge/core/events/__init__.py,sha256=pqWmPSuZIV3_teyYxGgNOLu8yaEw5jTO7A0tKDTydpE,540
+gnosisllm_knowledge/core/events/emitter.py,sha256=qFANzBtgdnQMVgLFE76rA-4i6Es_7ItaI-JfOl9vrNc,6940
+gnosisllm_knowledge/core/events/types.py,sha256=UtmCxkhSfgpHaeFtl43aarmw-eg84KdLuXMBAttICh0,6636
+gnosisllm_knowledge/core/exceptions.py,sha256=fOnE_w2fqX6uS7PGkyXJxiIWcySXpRhfQtUJVA4AFdw,10367
+gnosisllm_knowledge/core/interfaces/__init__.py,sha256=LpBUW0ABmIMUscNIaX-dVkkarqK2pXr0Gf1rg0TSrtU,775
+gnosisllm_knowledge/core/interfaces/agentic.py,sha256=OrS-3YRJ6Z6j1yAngsAv3B9fN9otmwlEyZ_G_fjDJPY,3600
+gnosisllm_knowledge/core/interfaces/chunker.py,sha256=FUu3B6Ent5Zofs1XLWZ0RqevHBbGd7iNRO7pAg2JqdE,1862
+gnosisllm_knowledge/core/interfaces/fetcher.py,sha256=19u2TKE6rDjX-8l7xIRKUqa6DLmZz_RncPy1p0sN3tk,3318
+gnosisllm_knowledge/core/interfaces/indexer.py,sha256=4a33oyhYetAiQ6TF2u7ICPyjjidGwsrjILJ0i0YZCqY,6166
+gnosisllm_knowledge/core/interfaces/loader.py,sha256=NKq5Yj1Pmbv1I8AUMDtleUCZO5URZFw6kCdx6uLG7Bk,2903
+gnosisllm_knowledge/core/interfaces/searcher.py,sha256=c76SH2DD3EEVmaD6TuVjZu5hP6HjoyawUoiT9aANZz0,4634
+gnosisllm_knowledge/core/interfaces/setup.py,sha256=9DMKVqoNUrWKbS58QCnfwlGk4sD9R5iGe-WLwlW7z-Q,4457
+gnosisllm_knowledge/fetchers/__init__.py,sha256=CiupP4NdIPzWme6SKp--a_wDX1qAy-QBUO_bjoVDFBQ,393
+gnosisllm_knowledge/fetchers/config.py,sha256=msUHhTzCGa-cASzoXeAhXFNrDpY9E1XaDo_gqs2CtrU,2611
+gnosisllm_knowledge/fetchers/http.py,sha256=nQjH1nKTIR65tmbfW6RWaP3pR5JJe4j33cTzSBWHM-E,5133
+gnosisllm_knowledge/fetchers/neoreader.py,sha256=Gc5SqHr9e3zvP9TE_xHmW_iwXcYq0UqDCzob7frkaqM,6637
+gnosisllm_knowledge/loaders/__init__.py,sha256=4XlrhFapi9uWsHUAG2eeQEfc47T75a9GRUXde1dmom4,387
+gnosisllm_knowledge/loaders/base.py,sha256=T5m0zrtn-4lA3nYR6mzswQfPwCnYiRhVogzNBvYK2xA,13089
+gnosisllm_knowledge/loaders/factory.py,sha256=gmUqYT3yTijMLP0XRldFWlgLFWdmdvKrj5WvD3RMZw0,6374
+gnosisllm_knowledge/loaders/sitemap.py,sha256=sbsLYs_EugwJRWXOOawpCy7iB2lu46ZW9_3eNymp33I,9524
+gnosisllm_knowledge/loaders/website.py,sha256=i54S7X0wTYNw9jJQfaoWFqMkGJkbAJeBv-OFBNjPQ_g,1587
+gnosisllm_knowledge/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+gnosisllm_knowledge/services/__init__.py,sha256=qOn1CjsB25UQ2K1TQIGMBe9-CUrGxAKhBrNh9Awo8n8,273
+gnosisllm_knowledge/services/indexing.py,sha256=ZVDDV2_Zlv6zmCWg-VSNeczieR7LyNaqzXcaFP-uvsE,12471
+gnosisllm_knowledge/services/search.py,sha256=ft8Q7rtRKqXVZ0c_bVqa01jDaV5UQF51LnrY2ICfC1Q,10156
+gnosisllm_knowledge-0.2.0.dist-info/METADATA,sha256=Q5bOjQAt93BXqPFdSF_3XlZh_6ystmXCX8-hcSIuwpg,14152
+gnosisllm_knowledge-0.2.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+gnosisllm_knowledge-0.2.0.dist-info/entry_points.txt,sha256=-4wtWCTX7bT8WessWJyqfkQ99-2aCVuSoTL1S_QpaOo,68
+gnosisllm_knowledge-0.2.0.dist-info/RECORD,,

gnosisllm_knowledge-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.2.1
+Root-Is-Purelib: true
+Tag: py3-none-any

gnosisllm_knowledge-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+gnosisllm-knowledge=gnosisllm_knowledge.cli:main
+