ignis-observability 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ignis_observability-0.2.0/.env.example +130 -0
- ignis_observability-0.2.0/LICENSE +21 -0
- ignis_observability-0.2.0/MANIFEST.in +3 -0
- ignis_observability-0.2.0/PKG-INFO +549 -0
- ignis_observability-0.2.0/README.md +497 -0
- ignis_observability-0.2.0/examples/00_langwatch.py +26 -0
- ignis_observability-0.2.0/examples/00_test.py +35 -0
- ignis_observability-0.2.0/examples/01_basic_example.py +88 -0
- ignis_observability-0.2.0/examples/02_streaming_example.py +88 -0
- ignis_observability-0.2.0/examples/03_multi_call_example.py +135 -0
- ignis_observability-0.2.0/examples/04_complete_example.py +227 -0
- ignis_observability-0.2.0/examples/05_provider_streaming_example.py +103 -0
- ignis_observability-0.2.0/examples/06_anthropic_example.py +91 -0
- ignis_observability-0.2.0/examples/07_gemini_example.py +91 -0
- ignis_observability-0.2.0/examples/08_multi_model_example.py +149 -0
- ignis_observability-0.2.0/examples/09_openai_dedicated_example.py +128 -0
- ignis_observability-0.2.0/examples/10_multi_platform_example.py +293 -0
- ignis_observability-0.2.0/examples/11_langwatch_multi_model_example.py +301 -0
- ignis_observability-0.2.0/examples/12_advanced_tracing_example.py +206 -0
- ignis_observability-0.2.0/examples/13_configurable_tracing_example.py +178 -0
- ignis_observability-0.2.0/examples/14_otel_tracing_example.py +228 -0
- ignis_observability-0.2.0/examples/15_custom_metadata_example.py +98 -0
- ignis_observability-0.2.0/examples/15_langsmith_tracer_example.py +173 -0
- ignis_observability-0.2.0/examples/16_decorator_with_hierarchical_tracing.py +115 -0
- ignis_observability-0.2.0/examples/16_installation_example.py +52 -0
- ignis_observability-0.2.0/examples/17_err_example.py +73 -0
- ignis_observability-0.2.0/examples/18_multi_error_example.py +115 -0
- ignis_observability-0.2.0/examples/19_tool_call_example.py +302 -0
- ignis_observability-0.2.0/examples/20_metadata_test_example.py +82 -0
- ignis_observability-0.2.0/examples/21_fastapi_example.py +203 -0
- ignis_observability-0.2.0/examples/_test_scheduler.py +96 -0
- ignis_observability-0.2.0/examples/app.py +71 -0
- ignis_observability-0.2.0/examples/interval_trace_runner.py +271 -0
- ignis_observability-0.2.0/examples/rapid_trace_seeder.py +166 -0
- ignis_observability-0.2.0/examples/seed_and_verify.py +269 -0
- ignis_observability-0.2.0/examples/test.py +74 -0
- ignis_observability-0.2.0/examples/test_db.py +77 -0
- ignis_observability-0.2.0/examples/test_db_out.py +288 -0
- ignis_observability-0.2.0/examples/test_langsmith_observability.py +145 -0
- ignis_observability-0.2.0/pyproject.toml +94 -0
- ignis_observability-0.2.0/setup.cfg +4 -0
- ignis_observability-0.2.0/src/ignis_observability/__init__.py +34 -0
- ignis_observability-0.2.0/src/ignis_observability/config.py +99 -0
- ignis_observability-0.2.0/src/ignis_observability/database.py +135 -0
- ignis_observability-0.2.0/src/ignis_observability/engine.py +787 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/__init__.py +1 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/anthropic.py +138 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/gemini.py +138 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/ingestion/__init__.py +23 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/ingestion/arize.py +375 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/ingestion/base.py +14 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/ingestion/field_mapping.py +27 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/ingestion/laminar.py +47 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/ingestion/langfuse.py +46 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/ingestion/langsmith.py +641 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/ingestion/langwatch.py +294 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/ingestion/otel.py +232 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/openai.py +139 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/sync/__init__.py +1 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/sync/__main__.py +4 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/sync/arize_adapter.py +149 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/sync/base_adapter.py +130 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/sync/db_writer.py +319 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/sync/field_mapping.py +116 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/sync/langsmith_adapter.py +197 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/sync/langsmith_sync.py +304 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/sync/langwatch_adapter.py +259 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/sync/pull_from_tools.py +1186 -0
- ignis_observability-0.2.0/src/ignis_observability/providers/sync/run_pull_scheduler.py +28 -0
- ignis_observability-0.2.0/src/ignis_observability/py.typed +0 -0
- ignis_observability-0.2.0/src/ignis_observability/trace_builder.py +240 -0
- ignis_observability-0.2.0/src/ignis_observability.egg-info/PKG-INFO +549 -0
- ignis_observability-0.2.0/src/ignis_observability.egg-info/SOURCES.txt +74 -0
- ignis_observability-0.2.0/src/ignis_observability.egg-info/dependency_links.txt +1 -0
- ignis_observability-0.2.0/src/ignis_observability.egg-info/requires.txt +34 -0
- ignis_observability-0.2.0/src/ignis_observability.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
# =============================================================================
|
|
2
|
+
# AI OBSERVABILITY HUB - ENVIRONMENT CONFIGURATION
|
|
3
|
+
# =============================================================================
|
|
4
|
+
# Copy this file to .env and fill in your actual API keys
|
|
5
|
+
#
|
|
6
|
+
# Quick Setup:
|
|
7
|
+
# 1. Get OpenAI API Key: https://platform.openai.com/api-keys
|
|
8
|
+
# 2. Get LangSmith API Key: https://smith.langchain.com/settings
|
|
9
|
+
# 3. Create a LangSmith Project: https://smith.langchain.com/
|
|
10
|
+
# =============================================================================
|
|
11
|
+
|
|
12
|
+
# -----------------------------------------------------------------------------
|
|
13
|
+
# OpenAI Configuration
|
|
14
|
+
# -----------------------------------------------------------------------------
|
|
15
|
+
# Your OpenAI API key for making LLM requests
|
|
16
|
+
# Get it from: https://platform.openai.com/api-keys
|
|
17
|
+
OPENAI_API_KEY=sk-proj-YOUR_OPENAI_API_KEY_HERE
|
|
18
|
+
|
|
19
|
+
# -----------------------------------------------------------------------------
|
|
20
|
+
# LangSmith Configuration
|
|
21
|
+
# -----------------------------------------------------------------------------
|
|
22
|
+
# Your LangSmith API key for tracing and observability
|
|
23
|
+
# Get it from: https://smith.langchain.com/settings
|
|
24
|
+
LANGCHAIN_API_KEY=lsv2_pt_YOUR_LANGSMITH_API_KEY_HERE
|
|
25
|
+
|
|
26
|
+
# LangSmith API endpoint (usually don't need to change this)
|
|
27
|
+
LANGCHAIN_ENDPOINT=https://api.smith.langchain.com
|
|
28
|
+
|
|
29
|
+
# LangSmith project name - traces will be sent to this project
|
|
30
|
+
# Create projects at: https://smith.langchain.com/
|
|
31
|
+
LANGCHAIN_PROJECT=default
|
|
32
|
+
LANGCHAIN_PROJECT_ID=your-langsmith-project-id
|
|
33
|
+
|
|
34
|
+
# Enable LangSmith tracing (true/false)
|
|
35
|
+
LANGCHAIN_TRACING_V2=true
|
|
36
|
+
|
|
37
|
+
# -----------------------------------------------------------------------------
|
|
38
|
+
# Database Configuration
|
|
39
|
+
# -----------------------------------------------------------------------------
|
|
40
|
+
# Local SQLite database for storing traces and metrics
|
|
41
|
+
# You can change this to use a different path or database
|
|
42
|
+
DATABASE_URL=sqlite+aiosqlite:///./obs_hub.db
|
|
43
|
+
|
|
44
|
+
# -----------------------------------------------------------------------------
|
|
45
|
+
# Sync Configuration
|
|
46
|
+
# -----------------------------------------------------------------------------
|
|
47
|
+
# How often (in seconds) to sync data from external tools back to local DB
|
|
48
|
+
SYNC_INTERVAL=60
|
|
49
|
+
|
|
50
|
+
# -----------------------------------------------------------------------------
|
|
51
|
+
# Logging Configuration
|
|
52
|
+
# -----------------------------------------------------------------------------
|
|
53
|
+
# Log level: debug, info, warning, error, critical
|
|
54
|
+
LOG_LEVEL=info
|
|
55
|
+
|
|
56
|
+
# -----------------------------------------------------------------------------
|
|
57
|
+
# Additional Observability Platform Configuration
|
|
58
|
+
# -----------------------------------------------------------------------------
|
|
59
|
+
# LangWatch
|
|
60
|
+
LANGWATCH_API_KEY=
|
|
61
|
+
LANGWATCH_PROJECT=
|
|
62
|
+
LANGWATCH_ENDPOINT=
|
|
63
|
+
LANGWATCH_ENABLED=false
|
|
64
|
+
|
|
65
|
+
# LangFuse
|
|
66
|
+
LANGFUSE_API_KEY=
|
|
67
|
+
LANGFUSE_DATASET=
|
|
68
|
+
LANGFUSE_ENDPOINT=
|
|
69
|
+
LANGFUSE_ENABLED=false
|
|
70
|
+
|
|
71
|
+
# Arize
|
|
72
|
+
ARIZE_API_KEY=
|
|
73
|
+
ARIZE_SPACE_KEY=
|
|
74
|
+
ARIZE_MODEL_NAME=
|
|
75
|
+
ARIZE_ENDPOINT=
|
|
76
|
+
ARIZE_ENABLED=false
|
|
77
|
+
|
|
78
|
+
# Laminar
|
|
79
|
+
LAMINAR_API_KEY=
|
|
80
|
+
LAMINAR_PROJECT=
|
|
81
|
+
LAMINAR_ENDPOINT=
|
|
82
|
+
LAMINAR_ENABLED=false
|
|
83
|
+
|
|
84
|
+
# OpenTelemetry (OTel) - Send traces to Jaeger, DataDog, Honeycomb, etc.
|
|
85
|
+
# Install: pip install opentelemetry-api opentelemetry-exporter-otlp
|
|
86
|
+
# Start Jaeger: docker run -p 4317:4317 jaegertracing/jaeger:latest
|
|
87
|
+
# Access Jaeger UI: http://localhost:16686
|
|
88
|
+
OTEL_ENABLED=false
|
|
89
|
+
OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
|
|
90
|
+
OTEL_SERVICE_NAME=llm-app
|
|
91
|
+
OTEL_ENVIRONMENT=development
|
|
92
|
+
|
|
93
|
+
# -----------------------------------------------------------------------------
|
|
94
|
+
# Bi-Directional Sync Configuration (NEW)
|
|
95
|
+
# -----------------------------------------------------------------------------
|
|
96
|
+
# Enable syncing enriched data (feedback, scores) from providers to local DB
|
|
97
|
+
# This allows you to pull data from LangSmith/LangWatch back to local database
|
|
98
|
+
SYNC_ENABLED=false
|
|
99
|
+
|
|
100
|
+
# How often to sync from providers (in minutes)
|
|
101
|
+
SYNC_INTERVAL_MINUTES=15
|
|
102
|
+
|
|
103
|
+
# How far back to look for new data (in hours)
|
|
104
|
+
SYNC_LOOKBACK_HOURS=24
|
|
105
|
+
|
|
106
|
+
# -----------------------------------------------------------------------------
|
|
107
|
+
# Local Dashboard API Configuration (NEW)
|
|
108
|
+
# -----------------------------------------------------------------------------
|
|
109
|
+
# Enable FastAPI server for local dashboard
|
|
110
|
+
# This provides REST API endpoints to query traces, feedback, and metrics
|
|
111
|
+
API_ENABLED=false
|
|
112
|
+
|
|
113
|
+
# API server settings
|
|
114
|
+
API_HOST=127.0.0.1
|
|
115
|
+
API_PORT=8000
|
|
116
|
+
|
|
117
|
+
# CORS settings for local dashboard (comma-separated origins)
|
|
118
|
+
# Examples: React dev server, Vite dev server
|
|
119
|
+
API_CORS_ORIGINS=http://localhost:3000,http://localhost:5173
|
|
120
|
+
|
|
121
|
+
# Optional other LLM provider keys
|
|
122
|
+
ANTHROPIC_API_KEY=
|
|
123
|
+
GEMINI_API_KEY=
|
|
124
|
+
|
|
125
|
+
# -----------------------------------------------------------------------------
|
|
126
|
+
# Scheduler JSON Multi-Key Config (Optional)
|
|
127
|
+
# -----------------------------------------------------------------------------
|
|
128
|
+
# Backward compatible: default false keeps existing SCHEDULER_* behavior unchanged.
|
|
129
|
+
SCHEDULER_TOOLS_CONFIG_ENABLED=false
|
|
130
|
+
SCHEDULER_TOOLS_CONFIG_PATH=./scheduler_tools_config.json
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Infogain-GenAI
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,549 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ignis_observability
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Non-Blocking AI Observability Hub for multi-model LLM applications
|
|
5
|
+
Author-email: Infogain-GenAI <genai@infogain.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Infogain-GenAI/ignis_observability
|
|
8
|
+
Project-URL: Repository, https://github.com/Infogain-GenAI/ignis_observability.git
|
|
9
|
+
Project-URL: Documentation, https://github.com/Infogain-GenAI/ignis_observability#readme
|
|
10
|
+
Project-URL: Bug Tracker, https://github.com/Infogain-GenAI/ignis_observability/issues
|
|
11
|
+
Keywords: observability,llm,tracing,langsmith,openai,anthropic,gemini,monitoring,async
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
19
|
+
Classifier: Topic :: System :: Monitoring
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: openai>=1.0.0
|
|
24
|
+
Requires-Dist: langsmith>=0.1.0
|
|
25
|
+
Requires-Dist: sqlalchemy>=2.0.0
|
|
26
|
+
Requires-Dist: aiosqlite>=0.19.0
|
|
27
|
+
Requires-Dist: tiktoken>=0.5.0
|
|
28
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
29
|
+
Requires-Dist: pydantic-settings>=2.0.0
|
|
30
|
+
Requires-Dist: pydantic>=2.0.0
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
33
|
+
Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
|
|
34
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
|
|
35
|
+
Requires-Dist: black>=23.0.0; extra == "dev"
|
|
36
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
37
|
+
Requires-Dist: mypy>=1.0.0; extra == "dev"
|
|
38
|
+
Provides-Extra: anthropic
|
|
39
|
+
Requires-Dist: anthropic>=0.7.0; extra == "anthropic"
|
|
40
|
+
Provides-Extra: gemini
|
|
41
|
+
Requires-Dist: google-generativeai>=0.3.0; extra == "gemini"
|
|
42
|
+
Provides-Extra: langwatch
|
|
43
|
+
Requires-Dist: langwatch>=0.2.11; extra == "langwatch"
|
|
44
|
+
Provides-Extra: arize
|
|
45
|
+
Requires-Dist: arize-phoenix-otel>=0.6.1; extra == "arize"
|
|
46
|
+
Provides-Extra: all
|
|
47
|
+
Requires-Dist: anthropic>=0.7.0; extra == "all"
|
|
48
|
+
Requires-Dist: google-generativeai>=0.3.0; extra == "all"
|
|
49
|
+
Requires-Dist: langwatch>=0.2.11; extra == "all"
|
|
50
|
+
Requires-Dist: arize-phoenix-otel>=0.6.1; extra == "all"
|
|
51
|
+
Dynamic: license-file
|
|
52
|
+
|
|
53
|
+
# Ignis Observability: AI Observability Hub
|
|
54
|
+
|
|
55
|
+
[](https://pypi.org/project/ignis_observability/)
|
|
56
|
+
[](https://www.python.org/downloads/)
|
|
57
|
+
[](LICENSE)
|
|
58
|
+
|
|
59
|
+
A **non-blocking AI observability hub** that provides comprehensive monitoring, tracing, and cost tracking for LLM applications. Built on a producer-consumer pattern, it ensures zero impact on your application's response time while capturing detailed metrics across multiple observability platforms.
|
|
60
|
+
|
|
61
|
+
## Installation
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
pip install ignis_observability
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
With optional provider integrations:
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
# Anthropic Claude support
|
|
71
|
+
pip install ignis_observability[anthropic]
|
|
72
|
+
|
|
73
|
+
# Google Gemini support
|
|
74
|
+
pip install ignis_observability[gemini]
|
|
75
|
+
|
|
76
|
+
# LangWatch integration
|
|
77
|
+
pip install ignis_observability[langwatch]
|
|
78
|
+
|
|
79
|
+
# Arize Phoenix / OpenTelemetry integration
|
|
80
|
+
pip install ignis_observability[arize]
|
|
81
|
+
|
|
82
|
+
# All optional integrations
|
|
83
|
+
pip install ignis_observability[all]
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## Key Features
|
|
87
|
+
|
|
88
|
+
- **Zero-Latency Observability**: Fire-and-forget pattern ensures LLM responses reach users instantly
|
|
89
|
+
- **Multi-Platform Tracing**: Send traces to LangSmith, LangWatch, Arize Phoenix, LangFuse, Laminar, or OpenTelemetry collectors
|
|
90
|
+
- **Multi-Model Support**: OpenAI GPT, Anthropic Claude, and Google Gemini out of the box
|
|
91
|
+
- **Local Persistence**: SQLite database for offline trace storage and cost analysis
|
|
92
|
+
- **Automatic Cost Tracking**: Built-in pricing tables calculate costs per request
|
|
93
|
+
- **Streaming Support**: Full support for streaming responses with TTFT (Time-to-First-Token) metrics
|
|
94
|
+
- **Universal Decorator**: Single `@observe` decorator works with both streaming and non-streaming functions
|
|
95
|
+
- **Modern Tech Stack**: SQLAlchemy 2.0, asyncio, type-safe with Mapped columns
|
|
96
|
+
|
|
97
|
+
## 📋 Table of Contents
|
|
98
|
+
|
|
99
|
+
- [Architecture](#architecture)
|
|
100
|
+
- [Supported Platforms](#supported-platforms)
|
|
101
|
+
- [Quick Start](#quick-start)
|
|
102
|
+
- [Configuration](#configuration)
|
|
103
|
+
- [Usage Examples](#usage-examples)
|
|
104
|
+
- [Database Schema](#database-schema)
|
|
105
|
+
- [Cost Tracking](#cost-tracking)
|
|
106
|
+
- [API Reference](#api-reference)
|
|
107
|
+
- [Troubleshooting](#troubleshooting)
|
|
108
|
+
|
|
109
|
+
## 🏛️ Architecture
|
|
110
|
+
|
|
111
|
+
### Producer-Consumer Pattern
|
|
112
|
+
|
|
113
|
+
```
|
|
114
|
+
┌─────────────┐ ┌──────────────┐ ┌─────────────────┐
|
|
115
|
+
│ │ │ │ │ │
|
|
116
|
+
│ Your App │───────▶│ asyncio.Queue │───────▶│ Background │
|
|
117
|
+
│ (Producer) │ fast! │ │ │ Worker │
|
|
118
|
+
│ │ │ │ │ (Consumer) │
|
|
119
|
+
└─────────────┘ └──────────────┘ └────────┬────────┘
|
|
120
|
+
│
|
|
121
|
+
┌───────────────────────────┴─────────┐
|
|
122
|
+
▼ ▼
|
|
123
|
+
┌──────────────────┐ ┌──────────────┐
|
|
124
|
+
│ LangSmith │ │ SQLite │
|
|
125
|
+
│ (Cloud) │ │ (Local) │
|
|
126
|
+
└──────────────────┘ └──────────────┘
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
### Tech Stack
|
|
130
|
+
|
|
131
|
+
| Layer | Technology | Purpose |
|
|
132
|
+
|-------|------------|---------|
|
|
133
|
+
| **Language** | Python 3.10+ | Modern async/await support |
|
|
134
|
+
| **Concurrency** | asyncio | Non-blocking I/O operations |
|
|
135
|
+
| **Database** | SQLite + aiosqlite | Lightweight local storage |
|
|
136
|
+
| **ORM** | SQLAlchemy 2.0 | Type-safe database models |
|
|
137
|
+
| **Tracing Platforms** | LangSmith, LangWatch, Arize, OTel | Cloud observability targets |
|
|
138
|
+
| **Tokenization** | tiktoken | Accurate token counting |
|
|
139
|
+
| **Configuration** | pydantic-settings | Type-safe environment config |
|
|
140
|
+
|
|
141
|
+
## Supported Platforms
|
|
142
|
+
|
|
143
|
+
| Platform | Install Extra | Notes |
|
|
144
|
+
|---|---|---|
|
|
145
|
+
| LangSmith | *(core)* | Built-in, no extra needed |
|
|
146
|
+
| LangWatch | `[langwatch]` | `pip install ignis_observability[langwatch]` |
|
|
147
|
+
| Arize Phoenix | `[arize]` | Includes OpenTelemetry |
|
|
148
|
+
| OpenTelemetry | `[arize]` | OTLP-compatible collectors |
|
|
149
|
+
| LangFuse | `[all]` | Included in full install |
|
|
150
|
+
| Laminar | `[all]` | Included in full install |
|
|
151
|
+
|
|
152
|
+
## Quick Start
|
|
153
|
+
|
|
154
|
+
### 1. Installation
|
|
155
|
+
|
|
156
|
+
```bash
|
|
157
|
+
pip install ignis_observability
|
|
158
|
+
|
|
159
|
+
# For development (includes testing tools)
|
|
160
|
+
pip install ignis_observability[dev]
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
### 2. Get Your API Keys
|
|
164
|
+
|
|
165
|
+
#### OpenAI API Key
|
|
166
|
+
1. Go to https://platform.openai.com/api-keys
|
|
167
|
+
2. Click "Create new secret key"
|
|
168
|
+
3. Copy the key (starts with `sk-proj-...`)
|
|
169
|
+
|
|
170
|
+
#### LangSmith API Key
|
|
171
|
+
1. Go to https://smith.langchain.com (sign up if needed)
|
|
172
|
+
2. Navigate to **Settings** → **API Keys**
|
|
173
|
+
3. Click "Create API Key"
|
|
174
|
+
4. Copy the key (starts with `lsv2_pt_...`)
|
|
175
|
+
|
|
176
|
+
#### Create a LangSmith Project
|
|
177
|
+
1. In LangSmith, go to **Projects**
|
|
178
|
+
2. Click "New Project"
|
|
179
|
+
3. Name it (e.g., `my-llm-app` or `production`)
|
|
180
|
+
4. Copy the project name
|
|
181
|
+
|
|
182
|
+
### 3. Configure Environment
|
|
183
|
+
|
|
184
|
+
Create a `.env` file in your project root:
|
|
185
|
+
|
|
186
|
+
```bash
|
|
187
|
+
touch .env # Linux/Mac
|
|
188
|
+
New-Item .env # Windows PowerShell
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
Add the following values:
|
|
192
|
+
|
|
193
|
+
```ini
|
|
194
|
+
# Required: Your OpenAI API key
|
|
195
|
+
OPENAI_API_KEY=sk-proj-YOUR_ACTUAL_KEY_HERE
|
|
196
|
+
|
|
197
|
+
# Required: Your LangSmith API key
|
|
198
|
+
LANGCHAIN_API_KEY=lsv2_pt_YOUR_ACTUAL_KEY_HERE
|
|
199
|
+
|
|
200
|
+
# Required: Your LangSmith project name
|
|
201
|
+
LANGCHAIN_PROJECT=my-llm-app
|
|
202
|
+
|
|
203
|
+
# Optional: Other settings (defaults usually fine)
|
|
204
|
+
LANGCHAIN_ENDPOINT=https://api.smith.langchain.com
|
|
205
|
+
LANGCHAIN_TRACING_V2=true
|
|
206
|
+
DATABASE_URL=sqlite+aiosqlite:///./obs_hub.db
|
|
207
|
+
LOG_LEVEL=info
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
### 4. Run Your First Trace
|
|
211
|
+
|
|
212
|
+
Create a `main.py` with the quick-start code from the [Usage Examples](#usage-examples) section below and run it:
|
|
213
|
+
|
|
214
|
+
```bash
|
|
215
|
+
python main.py
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
You should see:
|
|
219
|
+
|
|
220
|
+
```
|
|
221
|
+
Database ready
|
|
222
|
+
Worker running
|
|
223
|
+
Response: Why do programmers prefer dark mode? Because light attracts bugs!
|
|
224
|
+
Done! Check your LangSmith dashboard and local database.
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
### 5. View Your Data
|
|
228
|
+
|
|
229
|
+
#### LangSmith Dashboard
|
|
230
|
+
1. Go to https://smith.langchain.com
|
|
231
|
+
2. Click on your project (e.g., `my-llm-app`)
|
|
232
|
+
3. See your traced LLM calls with full details!
|
|
233
|
+
|
|
234
|
+
#### Local Database
|
|
235
|
+
```powershell
|
|
236
|
+
# Query your local data
|
|
237
|
+
python
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
```python
|
|
241
|
+
import asyncio
|
|
242
|
+
from sqlalchemy import select
|
|
243
|
+
from ignis_observability import DBManager, UnifiedTrace
|
|
244
|
+
|
|
245
|
+
async def view_traces():
|
|
246
|
+
db = DBManager()
|
|
247
|
+
await db.init_db()
|
|
248
|
+
async with db.session_factory() as session:
|
|
249
|
+
result = await session.execute(
|
|
250
|
+
select(UnifiedTrace).order_by(UnifiedTrace.timestamp.desc()).limit(5)
|
|
251
|
+
)
|
|
252
|
+
for trace in result.scalars():
|
|
253
|
+
print(f"{trace.name}: {trace.latency_ms:.2f}ms, {trace.prompt_tokens + trace.completion_tokens} tokens, ${trace.total_cost:.6f}")
|
|
254
|
+
|
|
255
|
+
asyncio.run(view_traces())
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
## Configuration
|
|
259
|
+
|
|
260
|
+
All configuration is managed through environment variables (loaded from `.env` file).
|
|
261
|
+
|
|
262
|
+
### Required Variables
|
|
263
|
+
|
|
264
|
+
| Variable | Description | Example | Where to Get |
|
|
265
|
+
|----------|-------------|---------|--------------|
|
|
266
|
+
| `OPENAI_API_KEY` | OpenAI API key for LLM calls | `sk-proj-abc123...` | [OpenAI Platform](https://platform.openai.com/api-keys) |
|
|
267
|
+
| `LANGCHAIN_API_KEY` | LangSmith API key for tracing | `lsv2_pt_xyz789...` | [LangSmith Settings](https://smith.langchain.com/settings) |
|
|
268
|
+
| `LANGCHAIN_PROJECT` | LangSmith project name | `my-production-app` | [LangSmith Projects](https://smith.langchain.com/) |
|
|
269
|
+
|
|
270
|
+
### Optional Variables
|
|
271
|
+
|
|
272
|
+
| Variable | Default | Description |
|
|
273
|
+
|----------|---------|-------------|
|
|
274
|
+
| `LANGCHAIN_ENDPOINT` | `https://api.smith.langchain.com` | LangSmith API endpoint |
|
|
275
|
+
| `LANGCHAIN_TRACING_V2` | `true` | Enable/disable LangSmith tracing |
|
|
276
|
+
| `DATABASE_URL` | `sqlite+aiosqlite:///./obs_hub.db` | Local database connection string |
|
|
277
|
+
| `LOG_LEVEL` | `info` | Logging level (debug/info/warning/error) |
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
## Usage Examples
|
|
281
|
+
|
|
282
|
+
### Basic Usage (OpenAI + LangSmith)
|
|
283
|
+
|
|
284
|
+
```python
|
|
285
|
+
import asyncio
|
|
286
|
+
from dotenv import load_dotenv
|
|
287
|
+
from openai import AsyncOpenAI
|
|
288
|
+
from ignis_observability import ObsHub, DBManager, settings
|
|
289
|
+
|
|
290
|
+
load_dotenv()
|
|
291
|
+
|
|
292
|
+
async def main():
|
|
293
|
+
# Initialize database
|
|
294
|
+
db = DBManager(settings.DATABASE_URL)
|
|
295
|
+
await db.init_db()
|
|
296
|
+
await db.seed_prices()
|
|
297
|
+
|
|
298
|
+
# Initialize hub
|
|
299
|
+
hub = ObsHub(
|
|
300
|
+
langsmith_api_key=settings.LANGCHAIN_API_KEY,
|
|
301
|
+
db_manager=db,
|
|
302
|
+
project_name=settings.LANGCHAIN_PROJECT,
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
worker_task = asyncio.create_task(hub.worker())
|
|
306
|
+
client = AsyncOpenAI(api_key=settings.OPENAI_API_KEY)
|
|
307
|
+
|
|
308
|
+
# Decorate your LLM function — observability happens automatically
|
|
309
|
+
@hub.observe(name="my_llm_call")
|
|
310
|
+
async def ask_llm(question: str):
|
|
311
|
+
return await client.chat.completions.create(
|
|
312
|
+
model="gpt-4o-mini",
|
|
313
|
+
messages=[{"role": "user", "content": question}],
|
|
314
|
+
)
|
|
315
|
+
|
|
316
|
+
response = await ask_llm("What is observability?")
|
|
317
|
+
print(response.choices[0].message.content)
|
|
318
|
+
|
|
319
|
+
# Allow background worker to flush traces
|
|
320
|
+
await asyncio.sleep(2)
|
|
321
|
+
worker_task.cancel()
|
|
322
|
+
|
|
323
|
+
asyncio.run(main())
|
|
324
|
+
```
|
|
325
|
+
|
|
326
|
+
### Streaming Example
|
|
327
|
+
|
|
328
|
+
```python
|
|
329
|
+
@hub.observe(name="streaming_response")
|
|
330
|
+
async def stream_response():
|
|
331
|
+
return await client.chat.completions.create(
|
|
332
|
+
model="gpt-3.5-turbo",
|
|
333
|
+
messages=[{"role": "user", "content": "Write a poem"}],
|
|
334
|
+
stream=True,
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
stream = await stream_response()
|
|
338
|
+
async for chunk in stream:
|
|
339
|
+
if chunk.choices[0].delta.content:
|
|
340
|
+
print(chunk.choices[0].delta.content, end="", flush=True)
|
|
341
|
+
```
|
|
342
|
+
|
|
343
|
+
### Anthropic Claude Example
|
|
344
|
+
|
|
345
|
+
```python
|
|
346
|
+
pip install ignis_observability[anthropic]
|
|
347
|
+
```
|
|
348
|
+
|
|
349
|
+
```python
|
|
350
|
+
import asyncio
|
|
351
|
+
from ignis_observability import ObsHub, DBManager, settings
|
|
352
|
+
from ignis_observability.providers.anthropic import AnthropicProvider
|
|
353
|
+
|
|
354
|
+
# Initialize with Anthropic provider
|
|
355
|
+
hub = ObsHub(langsmith_api_key=settings.LANGCHAIN_API_KEY, db_manager=db)
|
|
356
|
+
|
|
357
|
+
@hub.observe(name="claude_call")
|
|
358
|
+
async def ask_claude(prompt: str):
|
|
359
|
+
provider = AnthropicProvider(api_key=settings.ANTHROPIC_API_KEY)
|
|
360
|
+
return await provider.complete(prompt)
|
|
361
|
+
```
|
|
362
|
+
|
|
363
|
+
### Google Gemini Example
|
|
364
|
+
|
|
365
|
+
```python
|
|
366
|
+
pip install ignis_observability[gemini]
|
|
367
|
+
```
|
|
368
|
+
|
|
369
|
+
```python
|
|
370
|
+
from ignis_observability import ObsHub, DBManager, settings
|
|
371
|
+
from ignis_observability.providers.gemini import GeminiProvider
|
|
372
|
+
|
|
373
|
+
@hub.observe(name="gemini_call")
|
|
374
|
+
async def ask_gemini(prompt: str):
|
|
375
|
+
provider = GeminiProvider(api_key=settings.GEMINI_API_KEY)
|
|
376
|
+
return await provider.complete(prompt)
|
|
377
|
+
```
|
|
378
|
+
|
|
379
|
+
## 🗄️ Database Schema
|
|
380
|
+
|
|
381
|
+
### `llm_traces` Table
|
|
382
|
+
|
|
383
|
+
| Column | Type | Description |
|
|
384
|
+
|--------|------|-------------|
|
|
385
|
+
| `id` | String (PK) | Unique trace identifier |
|
|
386
|
+
| `timestamp` | DateTime | When the trace was created |
|
|
387
|
+
| `name` | String | Function/operation name |
|
|
388
|
+
| `model` | String | LLM model used (e.g., gpt-3.5-turbo) |
|
|
389
|
+
| `latency_ms` | Float | Total request latency in milliseconds |
|
|
390
|
+
| `prompt_tokens` | Integer | Number of input tokens |
|
|
391
|
+
| `completion_tokens` | Integer | Number of output tokens |
|
|
392
|
+
| `total_cost` | Float | Calculated cost in USD |
|
|
393
|
+
| `metadata_json` | JSON | Additional metadata (TTFT, streaming, etc.) |
|
|
394
|
+
|
|
395
|
+
### `model_prices` Table
|
|
396
|
+
|
|
397
|
+
| Column | Type | Description |
|
|
398
|
+
|--------|------|-------------|
|
|
399
|
+
| `model` | String (PK) | Model name |
|
|
400
|
+
| `input_1k_price` | Float | Cost per 1K input tokens (USD) |
|
|
401
|
+
| `output_1k_price` | Float | Cost per 1K output tokens (USD) |
|
|
402
|
+
|
|
403
|
+
## Cost Tracking
|
|
404
|
+
|
|
405
|
+
The system automatically calculates costs based on:
|
|
406
|
+
|
|
407
|
+
1. **Token Counts**: Captured from API responses or calculated via tiktoken
|
|
408
|
+
2. **Pricing Table**: Local `model_prices` table with current rates
|
|
409
|
+
3. **Formula**: `cost = (prompt_tokens/1000 * input_price) + (completion_tokens/1000 * output_price)`
|
|
410
|
+
|
|
411
|
+
### Current Pricing (2026 estimates)
|
|
412
|
+
|
|
413
|
+
| Model | Input (per 1K tokens) | Output (per 1K tokens) |
|
|
414
|
+
|-------|----------------------|------------------------|
|
|
415
|
+
| gpt-3.5-turbo | $0.0005 | $0.0015 |
|
|
416
|
+
| gpt-4o | $0.0025 | $0.010 |
|
|
417
|
+
| gpt-4o-mini | $0.00015 | $0.0006 |
|
|
418
|
+
|
|
419
|
+
**Note**: Update prices in `database.py` `seed_prices()` method as OpenAI pricing changes.
|
|
420
|
+
|
|
421
|
+
## 📚 API Reference
|
|
422
|
+
|
|
423
|
+
### `ObsHub`
|
|
424
|
+
|
|
425
|
+
Main orchestration hub for observability.
|
|
426
|
+
|
|
427
|
+
```python
|
|
428
|
+
hub = ObsHub(
|
|
429
|
+
langsmith_api_key: str, # Your LangSmith API key
|
|
430
|
+
db_manager: DBManager # Database manager instance
|
|
431
|
+
)
|
|
432
|
+
```
|
|
433
|
+
|
|
434
|
+
**Methods:**
|
|
435
|
+
- `observe(name: str)` - Decorator to instrument functions
|
|
436
|
+
- `capture(data: dict)` - Manually capture trace data
|
|
437
|
+
- `worker()` - Background worker (run as async task)
|
|
438
|
+
|
|
439
|
+
### `DBManager`
|
|
440
|
+
|
|
441
|
+
Database operations manager.
|
|
442
|
+
|
|
443
|
+
```python
|
|
444
|
+
db = DBManager(db_url: str = "sqlite+aiosqlite:///./obs_hub.db")
|
|
445
|
+
```
|
|
446
|
+
|
|
447
|
+
**Methods:**
|
|
448
|
+
- `init_db()` - Initialize database schema
|
|
449
|
+
- `seed_prices()` - Seed pricing table
|
|
450
|
+
- `save_trace(data: dict)` - Save trace (simple)
|
|
451
|
+
- `calculate_and_save(data: dict)` - Calculate cost and save
|
|
452
|
+
|
|
453
|
+
### `@observe` Decorator
|
|
454
|
+
|
|
455
|
+
Automatically instruments async functions.
|
|
456
|
+
|
|
457
|
+
```python
|
|
458
|
+
@hub.observe(name="function_name")
|
|
459
|
+
async def my_function():
|
|
460
|
+
# Your code here
|
|
461
|
+
pass
|
|
462
|
+
```
|
|
463
|
+
|
|
464
|
+
**Captured Metrics:**
|
|
465
|
+
- Latency (wall-clock time)
|
|
466
|
+
- TTFT (for streaming)
|
|
467
|
+
- Token counts (prompt + completion)
|
|
468
|
+
- Cost (calculated automatically)
|
|
469
|
+
- Model name
|
|
470
|
+
- Custom metadata
|
|
471
|
+
|
|
472
|
+
## 🐛 Troubleshooting
|
|
473
|
+
|
|
474
|
+
### Import Errors
|
|
475
|
+
|
|
476
|
+
**Problem**: `ModuleNotFoundError: No module named 'ignis_observability'`
|
|
477
|
+
|
|
478
|
+
**Solution**: Make sure the package is installed:
|
|
479
|
+
```bash
|
|
480
|
+
pip install ignis_observability
|
|
481
|
+
```
|
|
482
|
+
|
|
483
|
+
### Authentication Errors
|
|
484
|
+
|
|
485
|
+
**Problem**: `AuthenticationError` from OpenAI or LangSmith
|
|
486
|
+
|
|
487
|
+
**Solution**:
|
|
488
|
+
1. Check `.env` file has correct API keys
|
|
489
|
+
2. Verify keys are active (not revoked)
|
|
490
|
+
3. Ensure OpenAI account has credits
|
|
491
|
+
4. Test keys independently
|
|
492
|
+
|
|
493
|
+
### No Traces in LangSmith
|
|
494
|
+
|
|
495
|
+
**Problem**: Traces not appearing in LangSmith dashboard
|
|
496
|
+
|
|
497
|
+
**Solutions**:
|
|
498
|
+
1. Verify `LANGCHAIN_TRACING_V2=true` in `.env`
|
|
499
|
+
2. Check `LANGCHAIN_PROJECT` name matches existing project
|
|
500
|
+
3. Wait a few seconds (traces are batched)
|
|
501
|
+
4. Check console for error messages
|
|
502
|
+
5. Verify API key has write permissions
|
|
503
|
+
|
|
504
|
+
### Database Errors
|
|
505
|
+
|
|
506
|
+
**Problem**: SQLAlchemy or aiosqlite errors
|
|
507
|
+
|
|
508
|
+
**Solutions**:
|
|
509
|
+
1. Delete existing `obs_hub.db` file
|
|
510
|
+
2. Run `init_db()` again
|
|
511
|
+
3. Check file permissions
|
|
512
|
+
4. Verify `aiosqlite` is installed
|
|
513
|
+
|
|
514
|
+
### Streaming Not Working
|
|
515
|
+
|
|
516
|
+
**Problem**: Streaming metrics not captured
|
|
517
|
+
|
|
518
|
+
**Solution**:
|
|
519
|
+
- Ensure you're using `async for` to consume the stream completely
|
|
520
|
+
- The decorator wraps the stream, so all chunks must be consumed
|
|
521
|
+
- Check that `stream=True` is passed to OpenAI API
|
|
522
|
+
|
|
523
|
+
## Contributing
|
|
524
|
+
|
|
525
|
+
This library is published as a closed-source PyPI package. For bug reports or feature requests, please use the [Bug Tracker](https://github.com/Infogain-GenAI/ignis_observability/issues).
|
|
526
|
+
|
|
527
|
+
## 📄 License
|
|
528
|
+
|
|
529
|
+
MIT License — see [LICENSE](LICENSE) for details.
|
|
530
|
+
|
|
531
|
+
## References
|
|
532
|
+
|
|
533
|
+
Built with:
|
|
534
|
+
- [OpenAI](https://openai.com) - LLM provider
|
|
535
|
+
- [Anthropic](https://www.anthropic.com) - Claude LLM provider
|
|
536
|
+
- [Google Gemini](https://deepmind.google/technologies/gemini/) - Gemini LLM provider
|
|
537
|
+
- [LangSmith](https://smith.langchain.com) - Cloud-based LLM tracing
|
|
538
|
+
- [LangWatch](https://langwatch.ai) - LLM observability platform
|
|
539
|
+
- [Arize Phoenix](https://phoenix.arize.com) - ML observability & OpenTelemetry
|
|
540
|
+
- [SQLAlchemy](https://www.sqlalchemy.org/) - Database ORM
|
|
541
|
+
- [tiktoken](https://github.com/openai/tiktoken) - Token counting
|
|
542
|
+
- [pydantic](https://docs.pydantic.dev) - Data validation and settings
|
|
543
|
+
|
|
544
|
+
---
|
|
545
|
+
|
|
546
|
+
**Questions?** Open an issue or check the [examples](./examples/) directory for more details.
|
|
547
|
+
|
|
548
|
+
**Ready to monitor your LLM applications?** Start with `examples/01_basic_example.py`!
|
|
549
|
+
|