zerda 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zerda-0.1.0/CHANGELOG.md +46 -0
- zerda-0.1.0/LICENSE +21 -0
- zerda-0.1.0/MANIFEST.in +21 -0
- zerda-0.1.0/PKG-INFO +309 -0
- zerda-0.1.0/README.md +168 -0
- zerda-0.1.0/pyproject.toml +161 -0
- zerda-0.1.0/setup.cfg +4 -0
- zerda-0.1.0/zerda/__init__.py +64 -0
- zerda-0.1.0/zerda/cache/__init__.py +30 -0
- zerda-0.1.0/zerda/cache/cache_entry.py +102 -0
- zerda-0.1.0/zerda/cache/cache_manager.py +711 -0
- zerda-0.1.0/zerda/cache/cache_metrics.py +124 -0
- zerda-0.1.0/zerda/cache/cache_strategies.py +9 -0
- zerda-0.1.0/zerda/cache/config_cache.py +284 -0
- zerda-0.1.0/zerda/cache/cost_model.py +287 -0
- zerda-0.1.0/zerda/cache/embedding_index.py +489 -0
- zerda-0.1.0/zerda/cache/metrics.py +301 -0
- zerda-0.1.0/zerda/cache/models.py +354 -0
- zerda-0.1.0/zerda/cache/multi_level_cache.py +978 -0
- zerda-0.1.0/zerda/cache/normalization.py +169 -0
- zerda-0.1.0/zerda/cache/policy_learner.py +227 -0
- zerda-0.1.0/zerda/cache/security.py +218 -0
- zerda-0.1.0/zerda/cache/storage/__init__.py +28 -0
- zerda-0.1.0/zerda/cache/storage/base_storage.py +38 -0
- zerda-0.1.0/zerda/cache/storage/factury_fun.py +24 -0
- zerda-0.1.0/zerda/cache/storage/redis.py +172 -0
- zerda-0.1.0/zerda/cache/storage/sqlite.py +150 -0
- zerda-0.1.0/zerda/cache/storage/storage_memory.py +64 -0
- zerda-0.1.0/zerda/cache/tenant_manager.py +269 -0
- zerda-0.1.0/zerda/chunks/__init__.py +92 -0
- zerda-0.1.0/zerda/chunks/adaptive_chunker.py +210 -0
- zerda-0.1.0/zerda/chunks/arabic_chunker.py +452 -0
- zerda-0.1.0/zerda/chunks/arabic_preprocessor.py +329 -0
- zerda-0.1.0/zerda/chunks/base.py +131 -0
- zerda-0.1.0/zerda/chunks/chunk_config.py +132 -0
- zerda-0.1.0/zerda/chunks/context_aware_chunker.py +233 -0
- zerda-0.1.0/zerda/chunks/deduplicator.py +118 -0
- zerda-0.1.0/zerda/chunks/doc_model.py +137 -0
- zerda-0.1.0/zerda/chunks/document_structure_parser.py +288 -0
- zerda-0.1.0/zerda/chunks/embeddings.py +182 -0
- zerda-0.1.0/zerda/chunks/metadata_enricher.py +187 -0
- zerda-0.1.0/zerda/chunks/multi_chunker.py +377 -0
- zerda-0.1.0/zerda/chunks/query_pattern_analyzer.py +176 -0
- zerda-0.1.0/zerda/chunks/semantic_chunker.py +165 -0
- zerda-0.1.0/zerda/chunks/structure_aware_chunker.py +231 -0
- zerda-0.1.0/zerda/chunks/text_splitter.py +210 -0
- zerda-0.1.0/zerda/context/__init__.py +168 -0
- zerda-0.1.0/zerda/context/config.py +143 -0
- zerda-0.1.0/zerda/context/engine.py +669 -0
- zerda-0.1.0/zerda/context/guard_and_learning.py +365 -0
- zerda-0.1.0/zerda/context/legacy_compat.py +146 -0
- zerda-0.1.0/zerda/context/models.py +291 -0
- zerda-0.1.0/zerda/context/pipeline.py +417 -0
- zerda-0.1.0/zerda/context/query_analyzer.py +237 -0
- zerda-0.1.0/zerda/context/retriever.py +609 -0
- zerda-0.1.0/zerda/context/strategy.py +149 -0
- zerda-0.1.0/zerda/document_loaders/__init__.py +102 -0
- zerda-0.1.0/zerda/document_loaders/auto_loader.py +192 -0
- zerda-0.1.0/zerda/document_loaders/base_loader.py +129 -0
- zerda-0.1.0/zerda/document_loaders/config_loader.py +193 -0
- zerda-0.1.0/zerda/document_loaders/csv_loader.py +286 -0
- zerda-0.1.0/zerda/document_loaders/directory_loader.py +208 -0
- zerda-0.1.0/zerda/document_loaders/docx_loader.py +143 -0
- zerda-0.1.0/zerda/document_loaders/graph_loader.py +430 -0
- zerda-0.1.0/zerda/document_loaders/html_loader.py +227 -0
- zerda-0.1.0/zerda/document_loaders/json_loader.py +260 -0
- zerda-0.1.0/zerda/document_loaders/pdf_loader.py +251 -0
- zerda-0.1.0/zerda/document_loaders/table_loader.py +547 -0
- zerda-0.1.0/zerda/document_loaders/text_loader.py +250 -0
- zerda-0.1.0/zerda/document_loaders/web_loader.py +169 -0
- zerda-0.1.0/zerda/embeddings/__init__.py +128 -0
- zerda-0.1.0/zerda/embeddings/arabic_embedder.py +887 -0
- zerda-0.1.0/zerda/embeddings/base_embedder.py +492 -0
- zerda-0.1.0/zerda/embeddings/config_embedder.py +44 -0
- zerda-0.1.0/zerda/embeddings/gemini_embedder.py +785 -0
- zerda-0.1.0/zerda/embeddings/hugginface_embedder.py +476 -0
- zerda-0.1.0/zerda/embeddings/mistral_embedder.py +596 -0
- zerda-0.1.0/zerda/embeddings/ollama_embedder.py +654 -0
- zerda-0.1.0/zerda/embeddings/openai_embedder.py +734 -0
- zerda-0.1.0/zerda/llm/__init__.py +47 -0
- zerda-0.1.0/zerda/llm/anthropic_interface.py +163 -0
- zerda-0.1.0/zerda/llm/base_llm_interface.py +130 -0
- zerda-0.1.0/zerda/llm/config_llm.py +29 -0
- zerda-0.1.0/zerda/llm/gemini_interface.py +334 -0
- zerda-0.1.0/zerda/llm/groq_interface.py +184 -0
- zerda-0.1.0/zerda/llm/mistral_interface.py +193 -0
- zerda-0.1.0/zerda/llm/ollama_interface.py +389 -0
- zerda-0.1.0/zerda/llm/openai_interface.py +163 -0
- zerda-0.1.0/zerda/llm/tools.py +188 -0
- zerda-0.1.0/zerda/memory/__init__.py +80 -0
- zerda-0.1.0/zerda/memory/agents/__init__.py +9 -0
- zerda-0.1.0/zerda/memory/agents/agent_memory.py +417 -0
- zerda-0.1.0/zerda/memory/core/__init__.py +18 -0
- zerda-0.1.0/zerda/memory/core/base.py +94 -0
- zerda-0.1.0/zerda/memory/core/config.py +147 -0
- zerda-0.1.0/zerda/memory/core/memory_entry.py +231 -0
- zerda-0.1.0/zerda/memory/core/memory_type.py +117 -0
- zerda-0.1.0/zerda/memory/core/models.py +56 -0
- zerda-0.1.0/zerda/memory/dashboard/__init__.py +21 -0
- zerda-0.1.0/zerda/memory/dashboard/__main__.py +4 -0
- zerda-0.1.0/zerda/memory/dashboard/app.py +427 -0
- zerda-0.1.0/zerda/memory/dashboard/static/app.js +396 -0
- zerda-0.1.0/zerda/memory/dashboard/static/index.html +114 -0
- zerda-0.1.0/zerda/memory/dashboard/static/style.css +626 -0
- zerda-0.1.0/zerda/memory/intelligent/__init__.py +24 -0
- zerda-0.1.0/zerda/memory/intelligent/forgetting_mechanism.py +145 -0
- zerda-0.1.0/zerda/memory/intelligent/memory_compressor.py +266 -0
- zerda-0.1.0/zerda/memory/intelligent/memory_manager.py +415 -0
- zerda-0.1.0/zerda/memory/intelligent/memory_selector.py +161 -0
- zerda-0.1.0/zerda/memory/intelligent/short_term_memory.py +85 -0
- zerda-0.1.0/zerda/memory/intelligent/user_profile_manager.py +241 -0
- zerda-0.1.0/zerda/memory/intelligent/working_memory.py +103 -0
- zerda-0.1.0/zerda/memory/memories/__init__.py +14 -0
- zerda-0.1.0/zerda/memory/memories/buffer_memory.py +106 -0
- zerda-0.1.0/zerda/memory/memories/entity_memory.py +217 -0
- zerda-0.1.0/zerda/memory/memories/summary_memory.py +192 -0
- zerda-0.1.0/zerda/memory/memories/window_memory.py +60 -0
- zerda-0.1.0/zerda/memory/rag/__init__.py +2 -0
- zerda-0.1.0/zerda/memory/rag/context_builder.py +223 -0
- zerda-0.1.0/zerda/memory/security/__init__.py +2 -0
- zerda-0.1.0/zerda/memory/security/privacy_security.py +233 -0
- zerda-0.1.0/zerda/memory/semantic_memory/__init__.py +2 -0
- zerda-0.1.0/zerda/memory/semantic_memory/semantic_memory.py +302 -0
- zerda-0.1.0/zerda/memory/storage/__init__.py +2 -0
- zerda-0.1.0/zerda/memory/storage/long_term_memory.py +237 -0
- zerda-0.1.0/zerda/observability/__init__.py +129 -0
- zerda-0.1.0/zerda/observability/exporters/__init__.py +12 -0
- zerda-0.1.0/zerda/observability/exporters/base.py +37 -0
- zerda-0.1.0/zerda/observability/exporters/console.py +48 -0
- zerda-0.1.0/zerda/observability/exporters/json.py +59 -0
- zerda-0.1.0/zerda/observability/exporters/langsmith.py +134 -0
- zerda-0.1.0/zerda/observability/span.py +191 -0
- zerda-0.1.0/zerda/observability/tracing.py +291 -0
- zerda-0.1.0/zerda/output_parser/__init__.py +76 -0
- zerda-0.1.0/zerda/output_parser/base_parser.py +255 -0
- zerda-0.1.0/zerda/output_parser/csv_parser.py +175 -0
- zerda-0.1.0/zerda/output_parser/fixer.py +338 -0
- zerda-0.1.0/zerda/output_parser/format_detector.py +280 -0
- zerda-0.1.0/zerda/output_parser/json_parser.py +180 -0
- zerda-0.1.0/zerda/output_parser/list_parser.py +182 -0
- zerda-0.1.0/zerda/output_parser/parser.py +658 -0
- zerda-0.1.0/zerda/output_parser/pydantic_parser.py +172 -0
- zerda-0.1.0/zerda/output_parser/retry_handler.py +307 -0
- zerda-0.1.0/zerda/output_parser/schema.py +269 -0
- zerda-0.1.0/zerda/output_parser/structured_parser.py +250 -0
- zerda-0.1.0/zerda/output_parser/validator.py +313 -0
- zerda-0.1.0/zerda/output_parser/yaml_parser.py +260 -0
- zerda-0.1.0/zerda/plugins/__init__.py +98 -0
- zerda-0.1.0/zerda/plugins/base_plugin.py +392 -0
- zerda-0.1.0/zerda/plugins/hot_reload.py +346 -0
- zerda-0.1.0/zerda/plugins/loader.py +227 -0
- zerda-0.1.0/zerda/plugins/metadata.py +306 -0
- zerda-0.1.0/zerda/plugins/plugin_manager.py +698 -0
- zerda-0.1.0/zerda/plugins/registry.py +225 -0
- zerda-0.1.0/zerda/plugins/security.py +303 -0
- zerda-0.1.0/zerda/plugins/types/__init__.py +35 -0
- zerda-0.1.0/zerda/plugins/types/action_plugin.py +52 -0
- zerda-0.1.0/zerda/plugins/types/processing_plugin.py +35 -0
- zerda-0.1.0/zerda/plugins/types/retrieval_plugin.py +42 -0
- zerda-0.1.0/zerda/plugins/types/tool_plugin.py +24 -0
- zerda-0.1.0/zerda/prompt/__init__.py +104 -0
- zerda-0.1.0/zerda/prompt/ab_testing.py +512 -0
- zerda-0.1.0/zerda/prompt/builder.py +144 -0
- zerda-0.1.0/zerda/prompt/context_manager.py +285 -0
- zerda-0.1.0/zerda/prompt/guardrails.py +328 -0
- zerda-0.1.0/zerda/prompt/optimizer.py +161 -0
- zerda-0.1.0/zerda/prompt/prompt_engine.py +541 -0
- zerda-0.1.0/zerda/prompt/strategies/__init__.py +425 -0
- zerda-0.1.0/zerda/prompt/templates.py +791 -0
- zerda-0.1.0/zerda/prompt/types.py +176 -0
- zerda-0.1.0/zerda/rag/__init__.py +24 -0
- zerda-0.1.0/zerda/rag/core/__init__.py +42 -0
- zerda-0.1.0/zerda/rag/core/base_rag_system.py +246 -0
- zerda-0.1.0/zerda/rag/core/document_versioning.py +510 -0
- zerda-0.1.0/zerda/rag/core/evaluation.py +640 -0
- zerda-0.1.0/zerda/rag/core/exceptions.py +459 -0
- zerda-0.1.0/zerda/rag/core/logger.py +282 -0
- zerda-0.1.0/zerda/rag/core/prompt_router.py +299 -0
- zerda-0.1.0/zerda/rag/core/query_expansion.py +241 -0
- zerda-0.1.0/zerda/rag/core/rag_config.py +137 -0
- zerda-0.1.0/zerda/rag/core/rag_system.py +793 -0
- zerda-0.1.0/zerda/rag/core/reranker.py +405 -0
- zerda-0.1.0/zerda/rag/core/retrieval_cache.py +146 -0
- zerda-0.1.0/zerda/rag/core/synonm_table.py +1232 -0
- zerda-0.1.0/zerda/rag/types/agentic_rag/__init__.py +10 -0
- zerda-0.1.0/zerda/rag/types/agentic_rag/action_step.py +15 -0
- zerda-0.1.0/zerda/rag/types/agentic_rag/agentic_config.py +45 -0
- zerda-0.1.0/zerda/rag/types/agentic_rag/agentic_rag.py +779 -0
- zerda-0.1.0/zerda/rag/types/agentic_rag/agentic_result.py +37 -0
- zerda-0.1.0/zerda/rag/types/agentic_rag/cache.py +33 -0
- zerda-0.1.0/zerda/rag/types/agentic_rag/document.py +23 -0
- zerda-0.1.0/zerda/rag/types/conversational_rag/__init__.py +11 -0
- zerda-0.1.0/zerda/rag/types/conversational_rag/conversation_config.py +28 -0
- zerda-0.1.0/zerda/rag/types/conversational_rag/conversation_history.py +179 -0
- zerda-0.1.0/zerda/rag/types/conversational_rag/conversation_turn.py +41 -0
- zerda-0.1.0/zerda/rag/types/conversational_rag/conversational_rag.py +364 -0
- zerda-0.1.0/zerda/rag/types/domain_rag/__init__.py +12 -0
- zerda-0.1.0/zerda/rag/types/domain_rag/domain_defination.py +210 -0
- zerda-0.1.0/zerda/rag/types/domain_rag/domain_metrics.py +30 -0
- zerda-0.1.0/zerda/rag/types/domain_rag/domain_specific_rag.py +626 -0
- zerda-0.1.0/zerda/rag/types/federated_rag/__init__.py +4 -0
- zerda-0.1.0/zerda/rag/types/federated_rag/circuitbreaker.py +60 -0
- zerda-0.1.0/zerda/rag/types/federated_rag/federat.py +1151 -0
- zerda-0.1.0/zerda/rag/types/federated_rag/federat_cache.py +39 -0
- zerda-0.1.0/zerda/rag/types/graph_rag/__init__.py +7 -0
- zerda-0.1.0/zerda/rag/types/graph_rag/config.py +142 -0
- zerda-0.1.0/zerda/rag/types/graph_rag/edges.py +135 -0
- zerda-0.1.0/zerda/rag/types/graph_rag/graph_rag.py +595 -0
- zerda-0.1.0/zerda/rag/types/graph_rag/knowledge_graph.py +330 -0
- zerda-0.1.0/zerda/rag/types/graph_rag/node.py +136 -0
- zerda-0.1.0/zerda/rag/types/hybrid_search/__init__.py +10 -0
- zerda-0.1.0/zerda/rag/types/hybrid_search/bm25.py +121 -0
- zerda-0.1.0/zerda/rag/types/hybrid_search/hybrid_search.py +680 -0
- zerda-0.1.0/zerda/rag/types/hybrid_search/hybrid_search_config.py +15 -0
- zerda-0.1.0/zerda/rag/types/hybrid_search/stanze_.py +143 -0
- zerda-0.1.0/zerda/rag/types/hybrid_search/tfidf.py +93 -0
- zerda-0.1.0/zerda/rag/types/multi_doc_rag/__init__.py +4 -0
- zerda-0.1.0/zerda/rag/types/multi_doc_rag/multi_doc_rag.py +637 -0
- zerda-0.1.0/zerda/rag/types/multi_hop/__init__.py +7 -0
- zerda-0.1.0/zerda/rag/types/multi_hop/multi_hop.py +703 -0
- zerda-0.1.0/zerda/rag/types/multi_hop/multihop_dataclass.py +29 -0
- zerda-0.1.0/zerda/rag/types/multi_hop/multihop_strategy.py +11 -0
- zerda-0.1.0/zerda/rag/types/multi_hop/query_decomposer.py +293 -0
- zerda-0.1.0/zerda/rag/types/self_improving_rag/__init__.py +24 -0
- zerda-0.1.0/zerda/rag/types/self_improving_rag/hyde.py +154 -0
- zerda-0.1.0/zerda/rag/types/self_improving_rag/recursive.py +183 -0
- zerda-0.1.0/zerda/rag/types/self_improving_rag/self_config.py +118 -0
- zerda-0.1.0/zerda/rag/types/self_improving_rag/self_improving_rag.py +389 -0
- zerda-0.1.0/zerda/rag/types/self_improving_rag/self_rag_dataclass.py +158 -0
- zerda-0.1.0/zerda/rag/types/streaming_rag/__init__.py +11 -0
- zerda-0.1.0/zerda/rag/types/streaming_rag/protocols.py +24 -0
- zerda-0.1.0/zerda/rag/types/streaming_rag/streaming_config.py +29 -0
- zerda-0.1.0/zerda/rag/types/streaming_rag/streaming_enum.py +9 -0
- zerda-0.1.0/zerda/rag/types/streaming_rag/streaming_rag.py +445 -0
- zerda-0.1.0/zerda/router/__init__.py +40 -0
- zerda-0.1.0/zerda/router/cache/__init__.py +3 -0
- zerda-0.1.0/zerda/router/cache/manager.py +215 -0
- zerda-0.1.0/zerda/router/config.py +159 -0
- zerda-0.1.0/zerda/router/core/__init__.py +7 -0
- zerda-0.1.0/zerda/router/core/base.py +221 -0
- zerda-0.1.0/zerda/router/core/result.py +193 -0
- zerda-0.1.0/zerda/router/core/route.py +284 -0
- zerda-0.1.0/zerda/router/core/route_group.py +285 -0
- zerda-0.1.0/zerda/router/execution/__init__.py +14 -0
- zerda-0.1.0/zerda/router/execution/executor.py +468 -0
- zerda-0.1.0/zerda/router/execution/streaming_executor.py +391 -0
- zerda-0.1.0/zerda/router/feedback/__init__.py +7 -0
- zerda-0.1.0/zerda/router/feedback/engine.py +271 -0
- zerda-0.1.0/zerda/router/observability/__init__.py +7 -0
- zerda-0.1.0/zerda/router/observability/observability.py +359 -0
- zerda-0.1.0/zerda/router/routing/__init__.py +15 -0
- zerda-0.1.0/zerda/router/routing/hierarchical.py +602 -0
- zerda-0.1.0/zerda/router/routing/pipeline.py +315 -0
- zerda-0.1.0/zerda/router/routing/scorers.py +342 -0
- zerda-0.1.0/zerda/vector_database/__init__.py +13 -0
- zerda-0.1.0/zerda/vector_database/base.py +360 -0
- zerda-0.1.0/zerda/vector_database/chroma.py +742 -0
- zerda-0.1.0/zerda/vector_database/faiss.py +966 -0
- zerda-0.1.0/zerda/vector_database/pinecone.py +1189 -0
- zerda-0.1.0/zerda.egg-info/PKG-INFO +309 -0
- zerda-0.1.0/zerda.egg-info/SOURCES.txt +262 -0
- zerda-0.1.0/zerda.egg-info/dependency_links.txt +1 -0
- zerda-0.1.0/zerda.egg-info/requires.txt +142 -0
- zerda-0.1.0/zerda.egg-info/top_level.txt +1 -0
zerda-0.1.0/CHANGELOG.md
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html)
|
|
7
|
+
once it reaches `1.0.0`. Before `1.0.0`, minor versions may include breaking changes.
|
|
8
|
+
|
|
9
|
+
## [0.1.0] - Unreleased
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
- Initial public release of `zerda`.
|
|
13
|
+
- Core RAG system (`zerda.rag.core.RAGSystem`) plus specialized RAG types
|
|
14
|
+
under `zerda.rag.types`: agentic, graph, hybrid-search, multi-hop,
|
|
15
|
+
conversational, multi-document, federated, self-improving
|
|
16
|
+
(HyDE / Recursive / Self-RAG), streaming, and domain-specific RAG.
|
|
17
|
+
- Embedding provider clients: OpenAI, Gemini, Mistral, HuggingFace/local
|
|
18
|
+
models, Ollama, and an Arabic-specialized embedder.
|
|
19
|
+
- LLM client interfaces: OpenAI, Anthropic, Groq, Mistral, Gemini, Ollama.
|
|
20
|
+
- Vector database backends: Chroma, FAISS, Pinecone.
|
|
21
|
+
- Document loaders: PDF, DOCX, HTML, CSV/Excel, JSON, plain text.
|
|
22
|
+
- Chunking strategies, including an Arabic text preprocessor.
|
|
23
|
+
- Context intelligence engine (query analysis, hybrid retrieval fusion,
|
|
24
|
+
composite ranking, budget-aware context composition).
|
|
25
|
+
- Multi-level semantic cache with pluggable storage (memory, SQLite,
|
|
26
|
+
Redis) and a security guard (prompt-injection detection, PII scrubbing,
|
|
27
|
+
HMAC content integrity).
|
|
28
|
+
- Conversation/long-term memory subsystem.
|
|
29
|
+
- Prompt engine, structured output parser, request router, plugin
|
|
30
|
+
system, and tracing/observability exporters (console, JSON, LangSmith).
|
|
31
|
+
|
|
32
|
+
### Fixed (packaging)
|
|
33
|
+
- Added missing `zerda/__init__.py`, `zerda/rag/__init__.py`, and
|
|
34
|
+
`zerda/rag/types/__init__.py` — without these, `zerda.rag` and
|
|
35
|
+
`zerda.rag.types` were not regular Python packages and were silently
|
|
36
|
+
excluded by `setuptools.find_packages()`.
|
|
37
|
+
- Reworked `zerda/embeddings/__init__.py` and `zerda/llm/__init__.py` to
|
|
38
|
+
import each provider defensively, matching the rest of the codebase.
|
|
39
|
+
Previously these two sub-packages imported every provider SDK
|
|
40
|
+
unconditionally at import time, so `import zerda.embeddings` / `import
|
|
41
|
+
zerda.llm` raised `ModuleNotFoundError` unless *every* optional provider
|
|
42
|
+
SDK was installed, defeating the purpose of the optional extras.
|
|
43
|
+
- Removed a dangling `"EmbedderMode"` entry from
|
|
44
|
+
`zerda.embeddings.__all__` (no such class exists in the codebase).
|
|
45
|
+
|
|
46
|
+
[0.1.0]: https://github.com/your-org/zerda/releases/tag/v0.1.0
|
zerda-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Zerda Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
zerda-0.1.0/MANIFEST.in
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Files to include in the sdist in addition to what setuptools picks up
|
|
2
|
+
# automatically from the packages it finds.
|
|
3
|
+
|
|
4
|
+
include README.md
|
|
5
|
+
include LICENSE
|
|
6
|
+
include CHANGELOG.md
|
|
7
|
+
include pyproject.toml
|
|
8
|
+
|
|
9
|
+
# PEP 561 typing marker
|
|
10
|
+
include zerda/py.typed
|
|
11
|
+
|
|
12
|
+
# Static assets used by the Flask memory dashboard (zerda.memory.dashboard)
|
|
13
|
+
recursive-include zerda/memory/dashboard/static *
|
|
14
|
+
|
|
15
|
+
# Never ship the standalone pytest suite that lives inside the package tree
|
|
16
|
+
exclude zerda/document_loaders/test.py
|
|
17
|
+
|
|
18
|
+
# Housekeeping
|
|
19
|
+
global-exclude *.pyc
|
|
20
|
+
global-exclude __pycache__
|
|
21
|
+
global-exclude .DS_Store
|
zerda-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: zerda
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Advanced Retrieval-Augmented Generation (RAG) toolkit with first-class Arabic language support.
|
|
5
|
+
Author: Yousef Khalil
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://zerdaco.vercel.app/
|
|
8
|
+
Keywords: rag,retrieval-augmented-generation,llm,nlp,arabic,embeddings,vector-database,langchain-alternative,agent,agents,agentic,retrieval,retrieval-augmented,retrieval-augmented-generation,retrieval-augmented-generation-toolkit,retrieval-augmented-generation-framework
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
20
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
21
|
+
Classifier: Natural Language :: Arabic
|
|
22
|
+
Classifier: Natural Language :: English
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
Requires-Dist: numpy>=1.24
|
|
27
|
+
Provides-Extra: openai
|
|
28
|
+
Requires-Dist: openai>=1.0; extra == "openai"
|
|
29
|
+
Requires-Dist: tiktoken>=0.5; extra == "openai"
|
|
30
|
+
Provides-Extra: anthropic
|
|
31
|
+
Requires-Dist: anthropic>=0.25; extra == "anthropic"
|
|
32
|
+
Provides-Extra: groq
|
|
33
|
+
Requires-Dist: groq>=0.4; extra == "groq"
|
|
34
|
+
Provides-Extra: mistral
|
|
35
|
+
Requires-Dist: mistralai>=0.1; extra == "mistral"
|
|
36
|
+
Provides-Extra: gemini
|
|
37
|
+
Requires-Dist: google-generativeai>=0.4; extra == "gemini"
|
|
38
|
+
Provides-Extra: ollama
|
|
39
|
+
Requires-Dist: requests>=2.28; extra == "ollama"
|
|
40
|
+
Requires-Dist: aiohttp>=3.9; extra == "ollama"
|
|
41
|
+
Provides-Extra: huggingface
|
|
42
|
+
Requires-Dist: sentence-transformers>=2.2; extra == "huggingface"
|
|
43
|
+
Requires-Dist: transformers>=4.30; extra == "huggingface"
|
|
44
|
+
Requires-Dist: torch>=2.0; extra == "huggingface"
|
|
45
|
+
Provides-Extra: arabic
|
|
46
|
+
Requires-Dist: sentence-transformers>=2.2; extra == "arabic"
|
|
47
|
+
Requires-Dist: transformers>=4.30; extra == "arabic"
|
|
48
|
+
Requires-Dist: torch>=2.0; extra == "arabic"
|
|
49
|
+
Requires-Dist: stanza>=1.7; extra == "arabic"
|
|
50
|
+
Provides-Extra: chroma
|
|
51
|
+
Requires-Dist: chromadb>=0.4; extra == "chroma"
|
|
52
|
+
Provides-Extra: faiss
|
|
53
|
+
Requires-Dist: faiss-cpu>=1.7; extra == "faiss"
|
|
54
|
+
Provides-Extra: pinecone
|
|
55
|
+
Requires-Dist: pinecone>=3.0; extra == "pinecone"
|
|
56
|
+
Provides-Extra: redis
|
|
57
|
+
Requires-Dist: redis>=5.0; extra == "redis"
|
|
58
|
+
Provides-Extra: pdf
|
|
59
|
+
Requires-Dist: pypdf>=3.0; extra == "pdf"
|
|
60
|
+
Requires-Dist: PyPDF2>=3.0; extra == "pdf"
|
|
61
|
+
Requires-Dist: pdfplumber>=0.10; extra == "pdf"
|
|
62
|
+
Requires-Dist: PyMuPDF>=1.23; extra == "pdf"
|
|
63
|
+
Provides-Extra: docx
|
|
64
|
+
Requires-Dist: python-docx>=1.0; extra == "docx"
|
|
65
|
+
Provides-Extra: excel
|
|
66
|
+
Requires-Dist: openpyxl>=3.1; extra == "excel"
|
|
67
|
+
Requires-Dist: pandas>=2.0; extra == "excel"
|
|
68
|
+
Provides-Extra: html
|
|
69
|
+
Requires-Dist: beautifulsoup4>=4.12; extra == "html"
|
|
70
|
+
Provides-Extra: tables
|
|
71
|
+
Requires-Dist: camelot-py[cv]>=0.11; extra == "tables"
|
|
72
|
+
Provides-Extra: legacy-docs
|
|
73
|
+
Requires-Dist: textract>=1.6.5; extra == "legacy-docs"
|
|
74
|
+
Requires-Dist: chardet>=5.0; extra == "legacy-docs"
|
|
75
|
+
Provides-Extra: json-query
|
|
76
|
+
Requires-Dist: jq>=1.6; extra == "json-query"
|
|
77
|
+
Provides-Extra: documents
|
|
78
|
+
Requires-Dist: pypdf>=3.0; extra == "documents"
|
|
79
|
+
Requires-Dist: PyPDF2>=3.0; extra == "documents"
|
|
80
|
+
Requires-Dist: pdfplumber>=0.10; extra == "documents"
|
|
81
|
+
Requires-Dist: PyMuPDF>=1.23; extra == "documents"
|
|
82
|
+
Requires-Dist: python-docx>=1.0; extra == "documents"
|
|
83
|
+
Requires-Dist: openpyxl>=3.1; extra == "documents"
|
|
84
|
+
Requires-Dist: pandas>=2.0; extra == "documents"
|
|
85
|
+
Requires-Dist: beautifulsoup4>=4.12; extra == "documents"
|
|
86
|
+
Requires-Dist: chardet>=5.0; extra == "documents"
|
|
87
|
+
Requires-Dist: jq>=1.6; extra == "documents"
|
|
88
|
+
Provides-Extra: output
|
|
89
|
+
Requires-Dist: pydantic>=2.0; extra == "output"
|
|
90
|
+
Requires-Dist: PyYAML>=6.0; extra == "output"
|
|
91
|
+
Provides-Extra: observability
|
|
92
|
+
Requires-Dist: langsmith>=0.1; extra == "observability"
|
|
93
|
+
Provides-Extra: security
|
|
94
|
+
Requires-Dist: cryptography>=42.0; extra == "security"
|
|
95
|
+
Provides-Extra: dashboard
|
|
96
|
+
Requires-Dist: flask>=3.0; extra == "dashboard"
|
|
97
|
+
Provides-Extra: hot-reload
|
|
98
|
+
Requires-Dist: watchdog>=4.0; extra == "hot-reload"
|
|
99
|
+
Provides-Extra: progress
|
|
100
|
+
Requires-Dist: tqdm>=4.66; extra == "progress"
|
|
101
|
+
Provides-Extra: all
|
|
102
|
+
Requires-Dist: openai>=1.0; extra == "all"
|
|
103
|
+
Requires-Dist: tiktoken>=0.5; extra == "all"
|
|
104
|
+
Requires-Dist: anthropic>=0.25; extra == "all"
|
|
105
|
+
Requires-Dist: groq>=0.4; extra == "all"
|
|
106
|
+
Requires-Dist: mistralai>=0.1; extra == "all"
|
|
107
|
+
Requires-Dist: google-generativeai>=0.4; extra == "all"
|
|
108
|
+
Requires-Dist: requests>=2.28; extra == "all"
|
|
109
|
+
Requires-Dist: aiohttp>=3.9; extra == "all"
|
|
110
|
+
Requires-Dist: sentence-transformers>=2.2; extra == "all"
|
|
111
|
+
Requires-Dist: transformers>=4.30; extra == "all"
|
|
112
|
+
Requires-Dist: torch>=2.0; extra == "all"
|
|
113
|
+
Requires-Dist: stanza>=1.7; extra == "all"
|
|
114
|
+
Requires-Dist: chromadb>=0.4; extra == "all"
|
|
115
|
+
Requires-Dist: faiss-cpu>=1.7; extra == "all"
|
|
116
|
+
Requires-Dist: pinecone>=3.0; extra == "all"
|
|
117
|
+
Requires-Dist: redis>=5.0; extra == "all"
|
|
118
|
+
Requires-Dist: pypdf>=3.0; extra == "all"
|
|
119
|
+
Requires-Dist: PyPDF2>=3.0; extra == "all"
|
|
120
|
+
Requires-Dist: pdfplumber>=0.10; extra == "all"
|
|
121
|
+
Requires-Dist: PyMuPDF>=1.23; extra == "all"
|
|
122
|
+
Requires-Dist: python-docx>=1.0; extra == "all"
|
|
123
|
+
Requires-Dist: openpyxl>=3.1; extra == "all"
|
|
124
|
+
Requires-Dist: pandas>=2.0; extra == "all"
|
|
125
|
+
Requires-Dist: beautifulsoup4>=4.12; extra == "all"
|
|
126
|
+
Requires-Dist: chardet>=5.0; extra == "all"
|
|
127
|
+
Requires-Dist: jq>=1.6; extra == "all"
|
|
128
|
+
Requires-Dist: pydantic>=2.0; extra == "all"
|
|
129
|
+
Requires-Dist: PyYAML>=6.0; extra == "all"
|
|
130
|
+
Requires-Dist: langsmith>=0.1; extra == "all"
|
|
131
|
+
Requires-Dist: cryptography>=42.0; extra == "all"
|
|
132
|
+
Requires-Dist: flask>=3.0; extra == "all"
|
|
133
|
+
Requires-Dist: watchdog>=4.0; extra == "all"
|
|
134
|
+
Requires-Dist: tqdm>=4.66; extra == "all"
|
|
135
|
+
Provides-Extra: dev
|
|
136
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
137
|
+
Requires-Dist: pyflakes>=3.2; extra == "dev"
|
|
138
|
+
Requires-Dist: build>=1.0; extra == "dev"
|
|
139
|
+
Requires-Dist: twine>=5.0; extra == "dev"
|
|
140
|
+
Dynamic: license-file
|
|
141
|
+
|
|
142
|
+
# zerda
|
|
143
|
+
|
|
144
|
+
**Advanced Retrieval-Augmented Generation (RAG) toolkit for Python, with first-class Arabic language support.**
|
|
145
|
+
|
|
146
|
+
`zerda` is a modular collection of building blocks for RAG applications —
|
|
147
|
+
embeddings, vector stores, document loaders, chunking, caching, prompt
|
|
148
|
+
construction, output parsing, routing, memory, and several ready-made RAG
|
|
149
|
+
system architectures (agentic, graph, hybrid, multi-hop, conversational,
|
|
150
|
+
federated, self-improving, streaming, domain-specific).
|
|
151
|
+
|
|
152
|
+
Every third-party integration (OpenAI, Anthropic, Groq, Mistral, Gemini,
|
|
153
|
+
Chroma, FAISS, Pinecone, Redis, ...) is **optional**. Installing `zerda`
|
|
154
|
+
itself only pulls in `numpy`; you install the extras you actually need.
|
|
155
|
+
|
|
156
|
+
> **Status:** Alpha. APIs may change between minor versions until `1.0`.
|
|
157
|
+
|
|
158
|
+
---
|
|
159
|
+
|
|
160
|
+
## Features
|
|
161
|
+
|
|
162
|
+
- **Multiple RAG architectures** out of the box: `RAGSystem` (core), plus
|
|
163
|
+
`AgenticRAG`, `GraphRAG`, `HybridSearchRAG`, `MultiHopRAG`,
|
|
164
|
+
`ConversationalRAG`, `MultiDocumentRAGSystem`, `FederatedRAG`,
|
|
165
|
+
`SelfIRAG` / `RecursiveRAG` / `HyDERAG`, `StreamingRAG`, and
|
|
166
|
+
`DomainSpecificRAG`.
|
|
167
|
+
- **Embeddings**: OpenAI, Gemini, Mistral, HuggingFace/local models, Ollama,
|
|
168
|
+
and an Arabic-specialized embedder with text normalization.
|
|
169
|
+
- **LLM clients**: OpenAI, Anthropic, Groq, Mistral, Gemini, Ollama —
|
|
170
|
+
sync/async, streaming, and tool-calling support.
|
|
171
|
+
- **Vector databases**: Chroma, FAISS, Pinecone behind a common interface.
|
|
172
|
+
- **Document loaders**: PDF, DOCX, HTML, CSV/Excel, JSON, plain text.
|
|
173
|
+
- **Chunking**: multiple strategies plus an Arabic-aware preprocessor.
|
|
174
|
+
- **Context engine**: query analysis, hybrid (vector + BM25) retrieval
|
|
175
|
+
fusion, composite ranking, budget-aware context composition.
|
|
176
|
+
- **Multi-level semantic cache**: exact + semantic + persistent storage
|
|
177
|
+
(memory, SQLite, Redis) with tenant isolation and a security guard layer
|
|
178
|
+
(prompt-injection detection, PII scrubbing, HMAC integrity checks).
|
|
179
|
+
- **Memory**: short/long-term, episodic, semantic, and procedural
|
|
180
|
+
conversation memory.
|
|
181
|
+
- **Prompt engine, structured output parsing, request router, plugin
|
|
182
|
+
system, and tracing/observability exporters** (console, JSON, LangSmith).
|
|
183
|
+
|
|
184
|
+
## Installation
|
|
185
|
+
|
|
186
|
+
```bash
|
|
187
|
+
pip install zerda
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
This installs the core package (`numpy` only). Add extras for the pieces
|
|
191
|
+
you actually use:
|
|
192
|
+
|
|
193
|
+
```bash
|
|
194
|
+
# LLM providers
|
|
195
|
+
pip install "zerda[openai]"
|
|
196
|
+
pip install "zerda[anthropic]"
|
|
197
|
+
pip install "zerda[groq]"
|
|
198
|
+
pip install "zerda[mistral]"
|
|
199
|
+
pip install "zerda[gemini]"
|
|
200
|
+
pip install "zerda[ollama]"
|
|
201
|
+
|
|
202
|
+
# Local / HuggingFace embedding models (incl. the Arabic embedder)
|
|
203
|
+
pip install "zerda[huggingface]"
|
|
204
|
+
pip install "zerda[arabic]"
|
|
205
|
+
|
|
206
|
+
# Vector databases
|
|
207
|
+
pip install "zerda[chroma]"
|
|
208
|
+
pip install "zerda[faiss]"
|
|
209
|
+
pip install "zerda[pinecone]"
|
|
210
|
+
|
|
211
|
+
# Cache storage
|
|
212
|
+
pip install "zerda[redis]"
|
|
213
|
+
|
|
214
|
+
# Document loaders
|
|
215
|
+
pip install "zerda[documents]" # pdf + docx + excel + html + json
|
|
216
|
+
pip install "zerda[pdf]"
|
|
217
|
+
pip install "zerda[tables]" # camelot (needs system Ghostscript/OpenCV)
|
|
218
|
+
pip install "zerda[legacy-docs]" # textract for legacy formats
|
|
219
|
+
|
|
220
|
+
# Misc
|
|
221
|
+
pip install "zerda[output]" # pydantic + PyYAML for output parsing
|
|
222
|
+
pip install "zerda[observability]" # LangSmith exporter
|
|
223
|
+
pip install "zerda[security]" # cryptography
|
|
224
|
+
pip install "zerda[dashboard]" # Flask memory dashboard
|
|
225
|
+
pip install "zerda[hot-reload]" # watchdog for plugin hot-reload
|
|
226
|
+
|
|
227
|
+
# Everything
|
|
228
|
+
pip install "zerda[all]"
|
|
229
|
+
|
|
230
|
+
# Multiple extras at once
|
|
231
|
+
pip install "zerda[openai,faiss,documents]"
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
Every `zerda.*` sub-package can always be imported even without its
|
|
235
|
+
extras installed — only the specific classes that need a missing library
|
|
236
|
+
will be unavailable until you install the matching extra.
|
|
237
|
+
|
|
238
|
+
## Quickstart
|
|
239
|
+
|
|
240
|
+
```python
|
|
241
|
+
from zerda.llm import OpenAIInterface
|
|
242
|
+
from zerda.embeddings import OpenAIEmbedder
|
|
243
|
+
from zerda.vector_database import ChromaVectorDB # name depends on the backend module
|
|
244
|
+
from zerda.chunks import TextSplitter # example chunker
|
|
245
|
+
from zerda.context import ContextManager # example context builder
|
|
246
|
+
from zerda.rag import RAGSystem, RAGConfig
|
|
247
|
+
|
|
248
|
+
llm = OpenAIInterface(api_key="sk-...")
|
|
249
|
+
embedder = OpenAIEmbedder(api_key="sk-...")
|
|
250
|
+
vector_db = ChromaVectorDB(embedder=embedder, collection_name="docs")
|
|
251
|
+
chunker = TextSplitter()
|
|
252
|
+
context_manager = ContextManager()
|
|
253
|
+
|
|
254
|
+
rag = RAGSystem(
|
|
255
|
+
vector_db=vector_db,
|
|
256
|
+
llm=llm,
|
|
257
|
+
chunker=chunker,
|
|
258
|
+
context_manager=context_manager,
|
|
259
|
+
config=RAGConfig(),
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
answer = rag.query("ما هي فوائد الطاقة الشمسية؟")
|
|
263
|
+
print(answer)
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
`RAGSystem` is intentionally composable: it accepts any objects that
|
|
267
|
+
implement the small `vector_db` / `llm` / `chunker` / `context_manager`
|
|
268
|
+
interfaces described in its docstring, so you can mix and match `zerda`'s
|
|
269
|
+
own implementations with your own. For a fully worked example with a
|
|
270
|
+
specific vector store and embedder, check the docstrings in
|
|
271
|
+
`zerda.vector_database` and `zerda.embeddings`.
|
|
272
|
+
|
|
273
|
+
### Using a specialized RAG type
|
|
274
|
+
|
|
275
|
+
```python
|
|
276
|
+
from zerda.rag.types import GraphRAG, ConfigGraphRAG
|
|
277
|
+
|
|
278
|
+
graph_rag = GraphRAG(llm=llm, embedder=embedder, config=ConfigGraphRAG())
|
|
279
|
+
graph_rag.add_document("...")
|
|
280
|
+
result = graph_rag.query("How are these two entities related?")
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
## Module map
|
|
284
|
+
|
|
285
|
+
| Module | Purpose |
|
|
286
|
+
| -------------------------- | ---------------------------------------------------- |
|
|
287
|
+
| `zerda.rag` | Core `RAGSystem` + all specialized RAG types (`zerda.rag.types`) |
|
|
288
|
+
| `zerda.embeddings` | Embedding provider clients |
|
|
289
|
+
| `zerda.llm` | LLM chat-completion clients |
|
|
290
|
+
| `zerda.vector_database` | Chroma / FAISS / Pinecone backends |
|
|
291
|
+
| `zerda.document_loaders` | PDF / DOCX / HTML / CSV / JSON / text loaders |
|
|
292
|
+
| `zerda.chunks` | Text chunking & Arabic preprocessing |
|
|
293
|
+
| `zerda.context` | Query analysis, retrieval fusion, context building |
|
|
294
|
+
| `zerda.cache` | Multi-level semantic cache with security guard |
|
|
295
|
+
| `zerda.memory` | Conversation & long-term memory |
|
|
296
|
+
| `zerda.prompt` | Prompt templating & strategies |
|
|
297
|
+
| `zerda.output_parser` | Structured output parsing & validation |
|
|
298
|
+
| `zerda.router` | Intent routing & handler dispatch |
|
|
299
|
+
| `zerda.observability` | Tracing & metrics exporters |
|
|
300
|
+
| `zerda.plugins` | Plugin system |
|
|
301
|
+
|
|
302
|
+
## Contributing
|
|
303
|
+
|
|
304
|
+
Issues and pull requests are welcome. Please open an issue describing the
|
|
305
|
+
change before submitting a large PR.
|
|
306
|
+
|
|
307
|
+
## License
|
|
308
|
+
|
|
309
|
+
MIT — see [LICENSE](LICENSE).
|
zerda-0.1.0/README.md
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
# zerda
|
|
2
|
+
|
|
3
|
+
**Advanced Retrieval-Augmented Generation (RAG) toolkit for Python, with first-class Arabic language support.**
|
|
4
|
+
|
|
5
|
+
`zerda` is a modular collection of building blocks for RAG applications —
|
|
6
|
+
embeddings, vector stores, document loaders, chunking, caching, prompt
|
|
7
|
+
construction, output parsing, routing, memory, and several ready-made RAG
|
|
8
|
+
system architectures (agentic, graph, hybrid, multi-hop, conversational,
|
|
9
|
+
federated, self-improving, streaming, domain-specific).
|
|
10
|
+
|
|
11
|
+
Every third-party integration (OpenAI, Anthropic, Groq, Mistral, Gemini,
|
|
12
|
+
Chroma, FAISS, Pinecone, Redis, ...) is **optional**. Installing `zerda`
|
|
13
|
+
itself only pulls in `numpy`; you install the extras you actually need.
|
|
14
|
+
|
|
15
|
+
> **Status:** Alpha. APIs may change between minor versions until `1.0`.
|
|
16
|
+
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
## Features
|
|
20
|
+
|
|
21
|
+
- **Multiple RAG architectures** out of the box: `RAGSystem` (core), plus
|
|
22
|
+
`AgenticRAG`, `GraphRAG`, `HybridSearchRAG`, `MultiHopRAG`,
|
|
23
|
+
`ConversationalRAG`, `MultiDocumentRAGSystem`, `FederatedRAG`,
|
|
24
|
+
`SelfIRAG` / `RecursiveRAG` / `HyDERAG`, `StreamingRAG`, and
|
|
25
|
+
`DomainSpecificRAG`.
|
|
26
|
+
- **Embeddings**: OpenAI, Gemini, Mistral, HuggingFace/local models, Ollama,
|
|
27
|
+
and an Arabic-specialized embedder with text normalization.
|
|
28
|
+
- **LLM clients**: OpenAI, Anthropic, Groq, Mistral, Gemini, Ollama —
|
|
29
|
+
sync/async, streaming, and tool-calling support.
|
|
30
|
+
- **Vector databases**: Chroma, FAISS, Pinecone behind a common interface.
|
|
31
|
+
- **Document loaders**: PDF, DOCX, HTML, CSV/Excel, JSON, plain text.
|
|
32
|
+
- **Chunking**: multiple strategies plus an Arabic-aware preprocessor.
|
|
33
|
+
- **Context engine**: query analysis, hybrid (vector + BM25) retrieval
|
|
34
|
+
fusion, composite ranking, budget-aware context composition.
|
|
35
|
+
- **Multi-level semantic cache**: exact + semantic + persistent storage
|
|
36
|
+
(memory, SQLite, Redis) with tenant isolation and a security guard layer
|
|
37
|
+
(prompt-injection detection, PII scrubbing, HMAC integrity checks).
|
|
38
|
+
- **Memory**: short/long-term, episodic, semantic, and procedural
|
|
39
|
+
conversation memory.
|
|
40
|
+
- **Prompt engine, structured output parsing, request router, plugin
|
|
41
|
+
system, and tracing/observability exporters** (console, JSON, LangSmith).
|
|
42
|
+
|
|
43
|
+
## Installation
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install zerda
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
This installs the core package (`numpy` only). Add extras for the pieces
|
|
50
|
+
you actually use:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
# LLM providers
|
|
54
|
+
pip install "zerda[openai]"
|
|
55
|
+
pip install "zerda[anthropic]"
|
|
56
|
+
pip install "zerda[groq]"
|
|
57
|
+
pip install "zerda[mistral]"
|
|
58
|
+
pip install "zerda[gemini]"
|
|
59
|
+
pip install "zerda[ollama]"
|
|
60
|
+
|
|
61
|
+
# Local / HuggingFace embedding models (incl. the Arabic embedder)
|
|
62
|
+
pip install "zerda[huggingface]"
|
|
63
|
+
pip install "zerda[arabic]"
|
|
64
|
+
|
|
65
|
+
# Vector databases
|
|
66
|
+
pip install "zerda[chroma]"
|
|
67
|
+
pip install "zerda[faiss]"
|
|
68
|
+
pip install "zerda[pinecone]"
|
|
69
|
+
|
|
70
|
+
# Cache storage
|
|
71
|
+
pip install "zerda[redis]"
|
|
72
|
+
|
|
73
|
+
# Document loaders
|
|
74
|
+
pip install "zerda[documents]" # pdf + docx + excel + html + json
|
|
75
|
+
pip install "zerda[pdf]"
|
|
76
|
+
pip install "zerda[tables]" # camelot (needs system Ghostscript/OpenCV)
|
|
77
|
+
pip install "zerda[legacy-docs]" # textract for legacy formats
|
|
78
|
+
|
|
79
|
+
# Misc
|
|
80
|
+
pip install "zerda[output]" # pydantic + PyYAML for output parsing
|
|
81
|
+
pip install "zerda[observability]" # LangSmith exporter
|
|
82
|
+
pip install "zerda[security]" # cryptography
|
|
83
|
+
pip install "zerda[dashboard]" # Flask memory dashboard
|
|
84
|
+
pip install "zerda[hot-reload]" # watchdog for plugin hot-reload
|
|
85
|
+
|
|
86
|
+
# Everything
|
|
87
|
+
pip install "zerda[all]"
|
|
88
|
+
|
|
89
|
+
# Multiple extras at once
|
|
90
|
+
pip install "zerda[openai,faiss,documents]"
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Every `zerda.*` sub-package can always be imported even without its
|
|
94
|
+
extras installed — only the specific classes that need a missing library
|
|
95
|
+
will be unavailable until you install the matching extra.
|
|
96
|
+
|
|
97
|
+
## Quickstart
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
from zerda.llm import OpenAIInterface
|
|
101
|
+
from zerda.embeddings import OpenAIEmbedder
|
|
102
|
+
from zerda.vector_database import ChromaVectorDB # name depends on the backend module
|
|
103
|
+
from zerda.chunks import TextSplitter # example chunker
|
|
104
|
+
from zerda.context import ContextManager # example context builder
|
|
105
|
+
from zerda.rag import RAGSystem, RAGConfig
|
|
106
|
+
|
|
107
|
+
llm = OpenAIInterface(api_key="sk-...")
|
|
108
|
+
embedder = OpenAIEmbedder(api_key="sk-...")
|
|
109
|
+
vector_db = ChromaVectorDB(embedder=embedder, collection_name="docs")
|
|
110
|
+
chunker = TextSplitter()
|
|
111
|
+
context_manager = ContextManager()
|
|
112
|
+
|
|
113
|
+
rag = RAGSystem(
|
|
114
|
+
vector_db=vector_db,
|
|
115
|
+
llm=llm,
|
|
116
|
+
chunker=chunker,
|
|
117
|
+
context_manager=context_manager,
|
|
118
|
+
config=RAGConfig(),
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
answer = rag.query("ما هي فوائد الطاقة الشمسية؟")
|
|
122
|
+
print(answer)
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
`RAGSystem` is intentionally composable: it accepts any objects that
|
|
126
|
+
implement the small `vector_db` / `llm` / `chunker` / `context_manager`
|
|
127
|
+
interfaces described in its docstring, so you can mix and match `zerda`'s
|
|
128
|
+
own implementations with your own. For a fully worked example with a
|
|
129
|
+
specific vector store and embedder, check the docstrings in
|
|
130
|
+
`zerda.vector_database` and `zerda.embeddings`.
|
|
131
|
+
|
|
132
|
+
### Using a specialized RAG type
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
from zerda.rag.types import GraphRAG, ConfigGraphRAG
|
|
136
|
+
|
|
137
|
+
graph_rag = GraphRAG(llm=llm, embedder=embedder, config=ConfigGraphRAG())
|
|
138
|
+
graph_rag.add_document("...")
|
|
139
|
+
result = graph_rag.query("How are these two entities related?")
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## Module map
|
|
143
|
+
|
|
144
|
+
| Module | Purpose |
|
|
145
|
+
| -------------------------- | ---------------------------------------------------- |
|
|
146
|
+
| `zerda.rag` | Core `RAGSystem` + all specialized RAG types (`zerda.rag.types`) |
|
|
147
|
+
| `zerda.embeddings` | Embedding provider clients |
|
|
148
|
+
| `zerda.llm` | LLM chat-completion clients |
|
|
149
|
+
| `zerda.vector_database` | Chroma / FAISS / Pinecone backends |
|
|
150
|
+
| `zerda.document_loaders` | PDF / DOCX / HTML / CSV / JSON / text loaders |
|
|
151
|
+
| `zerda.chunks` | Text chunking & Arabic preprocessing |
|
|
152
|
+
| `zerda.context` | Query analysis, retrieval fusion, context building |
|
|
153
|
+
| `zerda.cache` | Multi-level semantic cache with security guard |
|
|
154
|
+
| `zerda.memory` | Conversation & long-term memory |
|
|
155
|
+
| `zerda.prompt` | Prompt templating & strategies |
|
|
156
|
+
| `zerda.output_parser` | Structured output parsing & validation |
|
|
157
|
+
| `zerda.router` | Intent routing & handler dispatch |
|
|
158
|
+
| `zerda.observability` | Tracing & metrics exporters |
|
|
159
|
+
| `zerda.plugins` | Plugin system |
|
|
160
|
+
|
|
161
|
+
## Contributing
|
|
162
|
+
|
|
163
|
+
Issues and pull requests are welcome. Please open an issue describing the
|
|
164
|
+
change before submitting a large PR.
|
|
165
|
+
|
|
166
|
+
## License
|
|
167
|
+
|
|
168
|
+
MIT — see [LICENSE](LICENSE).
|