netra-sdk 0.1.18__py3-none-any.whl → 0.1.20__py3-none-any.whl

netra/instrumentation/__init__.py

@@ -1,4 +1,6 @@
 import logging
+from contextlib import redirect_stderr, redirect_stdout
+from io import StringIO
 from typing import Any, Callable, Optional, Set
 
 from traceloop.sdk import Instruments, Telemetry
@@ -55,12 +57,13 @@ def init_instrumentations(
         }
     )
 
-    init_instrumentations(
-        should_enrich_metrics=should_enrich_metrics,
-        base64_image_uploader=base64_image_uploader,
-        instruments=traceloop_instruments,
-        block_instruments=traceloop_block_instruments,
-    )
+    with redirect_stdout(StringIO()), redirect_stderr(StringIO()):
+        init_instrumentations(
+            should_enrich_metrics=should_enrich_metrics,
+            base64_image_uploader=base64_image_uploader,
+            instruments=traceloop_instruments,
+            block_instruments=traceloop_block_instruments,
+        )
 
     netra_custom_instruments = netra_custom_instruments or set(CustomInstruments)
     netra_custom_instruments = netra_custom_instruments - netra_custom_block_instruments

netra/pii.py

@@ -577,7 +577,7 @@ class PresidioPIIDetector(PIIDetector):
     call Presidio's Analyzer + Anonymizer on a string.
 
     Examples:
-        # Using default hash function
+        # Using default configuration
         detector = PresidioPIIDetector()
         result = detector.detect("My email is john@example.com")
 
@@ -592,6 +592,41 @@ class PresidioPIIDetector(PIIDetector):
             action_type="MASK",
             score_threshold=0.8
         )
+
+        # Using custom spaCy model configuration
+        spacy_config = {
+            "nlp_engine_name": "spacy",
+            "models": [{"lang_code": "en", "model_name": "en_core_web_lg"}]
+        }
+        detector = PresidioPIIDetector(nlp_configuration=spacy_config)
+
+        # Using Stanza model configuration
+        stanza_config = {
+            "nlp_engine_name": "stanza",
+            "models": [{"lang_code": "en", "model_name": "en"}]
+        }
+        detector = PresidioPIIDetector(nlp_configuration=stanza_config)
+
+        # Using transformers model configuration
+        transformers_config = {
+            "nlp_engine_name": "transformers",
+            "models": [{
+                "lang_code": "en",
+                "model_name": {
+                    "spacy": "en_core_web_sm",
+                    "transformers": "dbmdz/bert-large-cased-finetuned-conll03-english"
+                }
+            }],
+            "ner_model_configuration": {
+                "labels_to_ignore": ["O"],
+                "model_to_presidio_entity_mapping": {
+                    "PER": "PERSON",
+                    "LOC": "LOCATION",
+                    "ORG": "ORGANIZATION"
+                }
+            }
+        }
+        detector = PresidioPIIDetector(nlp_configuration=transformers_config)
     """
 
     def __init__(
@@ -602,7 +637,35 @@ class PresidioPIIDetector(PIIDetector):
         action_type: Optional[Literal["BLOCK", "FLAG", "MASK"]] = None,
         anonymizer_cache_size: int = 1000,
         hash_function: Optional[Callable[[str], str]] = None,
+        nlp_configuration: Optional[Dict[str, Any]] = None,
     ) -> None:
+        """
+        Initialize the Presidio PII detector.
+
+        Args:
+            entities: List of entity types to detect. If None, uses DEFAULT_ENTITIES.
+            language: Language code for detection (default: "en").
+            score_threshold: Minimum confidence score for detections (default: 0.6).
+            action_type: Action to take when PII is detected ("BLOCK", "FLAG", "MASK").
+            anonymizer_cache_size: Size of the anonymizer cache (default: 1000).
+            hash_function: Custom hash function for anonymization.
+            nlp_configuration: Dictionary containing NLP engine configuration.
+                Format: {
+                    "nlp_engine_name": "spacy|stanza|transformers",
+                    "models": [{"lang_code": "en", "model_name": "model_name"}],
+                    "ner_model_configuration": {...}  # Optional, for transformers
+                }
+
+                For spaCy and Stanza:
+                - model_name should be a string (e.g., "en_core_web_lg", "en")
+
+                For transformers:
+                - model_name should be a dict with "spacy" and "transformers" keys
+                - Example: {"spacy": "en_core_web_sm", "transformers": "model_path"}
+
+        Raises:
+            ImportError: If presidio-analyzer is not installed or required NLP library is missing.
+        """
         if action_type is None:
             action_type = "FLAG"
             env_action = os.getenv("NETRA_ACTION_TYPE", "FLAG")
@@ -610,18 +673,99 @@ class PresidioPIIDetector(PIIDetector):
             if env_action in ["BLOCK", "FLAG", "MASK"]:
                 action_type = cast(Literal["BLOCK", "FLAG", "MASK"], env_action)
         super().__init__(action_type=action_type)
+
+        # Import presidio-analyzer
         try:
             from presidio_analyzer import AnalyzerEngine  # noqa: F401
         except ImportError as exc:
-            raise ImportError("Presidio-based PII detection requires: presidio-analyzer. " "Install via pip.") from exc
+            raise ImportError("Presidio-based PII detection requires: presidio-analyzer. Install via pip.") from exc
 
         self.language: str = language
         self.entities: Optional[List[str]] = entities if entities else DEFAULT_ENTITIES
         self.score_threshold: float = score_threshold
 
-        self.analyzer = AnalyzerEngine()
+        # Initialize AnalyzerEngine with custom or default NLP engine
+        if nlp_configuration is not None:
+            self.analyzer = self._create_analyzer_with_custom_nlp(nlp_configuration)
+        else:
+            # Use default AnalyzerEngine
+            self.analyzer = AnalyzerEngine()
+
         self.anonymizer = Anonymizer(hash_function=hash_function, cache_size=anonymizer_cache_size)
 
+    def _create_analyzer_with_custom_nlp(self, nlp_configuration: Dict[str, Any]) -> Any:
+        """
+        Create an AnalyzerEngine with custom NLP configuration.
+
+        Args:
+            nlp_configuration: Dictionary containing NLP engine configuration.
+
+        Returns:
+            AnalyzerEngine instance with custom NLP engine.
+
+        Raises:
+            ImportError: If required NLP library is not available.
+        """
+        try:
+            from presidio_analyzer import AnalyzerEngine
+            from presidio_analyzer.nlp_engine import NlpEngineProvider
+        except ImportError as exc:
+            raise ImportError("Presidio-based PII detection requires: presidio-analyzer. Install via pip.") from exc
+
+        # Validate and prepare configuration
+        engine_name = nlp_configuration.get("nlp_engine_name", "").lower()
+
+        # Perform lazy imports based on engine type
+        if engine_name == "spacy":
+            self._ensure_spacy_available()
+        elif engine_name == "stanza":
+            self._ensure_stanza_available()
+        elif engine_name == "transformers":
+            self._ensure_transformers_available()
+        else:
+            # Default behavior - let Presidio handle it
+            pass
+
+        # Create NLP engine from configuration
+        provider = NlpEngineProvider(nlp_configuration=nlp_configuration)
+        custom_nlp_engine = provider.create_engine()
+
+        # Extract supported languages from configuration
+        supported_languages = [self.language]
+        if "models" in nlp_configuration:
+            supported_languages = [model["lang_code"] for model in nlp_configuration["models"]]
+
+        return AnalyzerEngine(nlp_engine=custom_nlp_engine, supported_languages=supported_languages)
+
+    def _ensure_spacy_available(self) -> None:
+        """Ensure spaCy is available when needed."""
+        try:
+            import spacy  # noqa: F401
+        except ImportError as exc:
+            raise ImportError(
+                "spaCy is required for spaCy-based PII detection. Install via: pip install spacy"
+            ) from exc
+
+    def _ensure_stanza_available(self) -> None:
+        """Ensure Stanza is available when needed."""
+        try:
+            import stanza  # noqa: F401
+        except ImportError as exc:
+            raise ImportError(
+                "Stanza is required for Stanza-based PII detection. Install via: pip install stanza"
+            ) from exc
+
+    def _ensure_transformers_available(self) -> None:
+        """Ensure transformers is available when needed."""
+        try:
+            import torch  # noqa: F401
+            import transformers  # noqa: F401
+        except ImportError as exc:
+            raise ImportError(
+                "Transformers and PyTorch are required for transformers-based PII detection. "
+                "Install via: pip install transformers torch"
+            ) from exc
+
     def _detect_pii(self, text: str) -> Tuple[bool, Counter[str], str, Dict[str, str]]:
         """
         Detect PII in a single message.
@@ -666,6 +810,7 @@ def get_default_detector(
     action_type: Optional[Literal["BLOCK", "FLAG", "MASK"]] = None,
     entities: Optional[List[str]] = None,
     hash_function: Optional[Callable[[str], str]] = None,
+    nlp_configuration: Optional[Dict[str, Any]] = None,
 ) -> PIIDetector:
     """
     Returns a default PII detector instance (Presidio-based by default).
@@ -678,8 +823,11 @@ def get_default_detector(
             - "MASK": Replace PII with mask tokens (default)
         entities: Optional list of entity types to detect. If None, uses Presidio's default entities
         hash_function: Optional custom hash function for anonymization. If None, uses default hash function.
+        nlp_configuration: Dictionary containing NLP engine configuration for custom models.
     """
-    return PresidioPIIDetector(action_type=action_type, entities=entities, hash_function=hash_function)
+    return PresidioPIIDetector(
+        action_type=action_type, entities=entities, hash_function=hash_function, nlp_configuration=nlp_configuration
+    )
 
 
 # ---------------------------------------------------------------------------- #

netra/version.py

@@ -1 +1 @@
-__version__ = "0.1.18"
+__version__ = "0.1.20"

{netra_sdk-0.1.18.dist-info → netra_sdk-0.1.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: netra-sdk
-Version: 0.1.18
+Version: 0.1.20
 Summary: A Python SDK for AI application observability that provides OpenTelemetry-based monitoring, tracing, and PII protection for LLM and vector database applications. Enables easy instrumentation, session tracking, and privacy-focused data collection for AI systems in production environments.
 License: Apache-2.0
 Keywords: netra,tracing,observability,sdk,ai,llm,vector,database
@@ -69,7 +69,9 @@ Requires-Dist: opentelemetry-instrumentation-urllib3 (>=0.55b1,<1.0.0)
 Requires-Dist: opentelemetry-sdk (>=1.34.0,<2.0.0)
 Requires-Dist: presidio-analyzer (==2.2.358) ; extra == "presidio"
 Requires-Dist: presidio-anonymizer (==2.2.358) ; extra == "presidio"
+Requires-Dist: stanza (>=1.10.1,<2.0.0) ; extra == "presidio"
 Requires-Dist: traceloop-sdk (>=0.40.7,<0.43.0)
+Requires-Dist: transformers (==4.51.3) ; extra == "presidio"
 Project-URL: Bug Tracker, https://github.com/KeyValueSoftwareSystems/netra-sdk-py/issues
 Project-URL: Documentation, https://github.com/KeyValueSoftwareSystems/netra-sdk-py/blob/main/README.md
 Project-URL: Homepage, https://github.com/KeyValueSoftwareSystems/netra-sdk-py
@@ -331,6 +333,119 @@ print(f"Masked text: {result.masked_text}")
 print(f"PII entities: {result.pii_entities}")
 ```
 
+#### Custom Models for PII Detection
+
+The `PresidioPIIDetector` supports custom NLP models through the `nlp_configuration` parameter, allowing you to use specialized models for improved PII detection accuracy. You can configure custom spaCy, Stanza, or transformers models:
+
+##### NLP Configuration Example
+
+Follow this configuration structure to provide your custom models.
+```python
+nlp_configuration = {
+    "nlp_engine_name": "spacy|stanza|transformers",
+    "models": [
+        {
+            "lang_code": "en",  # Language code
+            "model_name": "model_identifier"  # Varies by engine type
+        }
+    ],
+    "ner_model_configuration": {  # Optional, mainly for transformers
+        # Additional configuration options
+    }
+}
+```
+
+##### Using Custom spaCy Models
+
+```python
+from netra.pii import PresidioPIIDetector
+
+# Configure custom spaCy model
+spacy_config = {
+    "nlp_engine_name": "spacy",
+    "models": [{"lang_code": "en", "model_name": "en_core_web_lg"}]
+}
+
+detector = PresidioPIIDetector(
+    nlp_configuration=spacy_config,
+    action_type="MASK",
+    score_threshold=0.8
+)
+
+text = "Dr. Sarah Wilson works at 123 Main St, New York"
+result = detector.detect(text)
+print(f"Detected entities: {result.pii_entities}")
+```
+
+##### Using Stanza Models
+
+```python
+from netra.pii import PresidioPIIDetector
+
+# Configure Stanza model
+stanza_config = {
+    "nlp_engine_name": "stanza",
+    "models": [{"lang_code": "en", "model_name": "en"}]
+}
+
+detector = PresidioPIIDetector(
+    nlp_configuration=stanza_config,
+    action_type="FLAG"
+)
+
+text = "Contact Alice Smith at alice@company.com"
+result = detector.detect(text)
+print(f"PII detected: {result.has_pii}")
+```
+
+##### Using Transformers Models
+
+For advanced NER capabilities, you can use transformer-based models:
+
+```python
+from netra.pii import PresidioPIIDetector
+
+# Configure transformers model with entity mapping
+transformers_config = {
+    "nlp_engine_name": "transformers",
+    "models": [{
+        "lang_code": "en",
+        "model_name": {
+            "spacy": "en_core_web_sm",
+            "transformers": "dbmdz/bert-large-cased-finetuned-conll03-english"
+        }
+    }],
+    "ner_model_configuration": {
+        "labels_to_ignore": ["O"],
+        "model_to_presidio_entity_mapping": {
+            "PER": "PERSON",
+            "LOC": "LOCATION",
+            "ORG": "ORGANIZATION",
+            "MISC": "MISC"
+        },
+        "low_confidence_score_multiplier": 0.4,
+        "low_score_entity_names": ["ORG"]
+    }
+}
+
+detector = PresidioPIIDetector(
+    nlp_configuration=transformers_config,
+    action_type="MASK"
+)
+
+text = "Microsoft Corporation is located in Redmond, Washington"
+result = detector.detect(text)
+print(f"Masked text: {result.masked_text}")
+```
+
+
+
+**Note**: Custom model configuration allows for:
+- **Better accuracy** with domain-specific models
+- **Multi-language support** by specifying different language codes
+- **Fine-tuned models** trained on your specific data
+- **Performance optimization** by choosing models suited to your use case
+
 #### Regex-based Detection
 ```python
 from netra.pii import RegexPIIDetector
@@ -555,102 +670,6 @@ Configuration values are resolved in the following order (highest to lowest prec
 4. **Default Values**: Fallback values defined in the SDK
 
 This allows you to:
-- Override any setting directly in code for maximum control
-- Use Netra-specific environment variables for Netra-specific settings
-- Fall back to standard OpenTelemetry variables for compatibility
-- Rely on sensible defaults when no other configuration is provided
-
-**Example**:
-```bash
-export NETRA_APP_NAME="my-ai-service"
-export NETRA_OTLP_ENDPOINT="https://collector.example.com:4318"
-export NETRA_API_KEY="your-api-key-here"
-export NETRA_ENV="production"
-export NETRA_RESOURCE_ATTRS='{"team":"ai", "version":"1.0.0"}'
-```
-
-### Programmatic Configuration
-
-You can also configure the SDK programmatically when initializing:
-
-```python
-from netra import Netra
-from netra.instrumentation.instruments import InstrumentSet
-
-Netra.init(
-    app_name="my-ai-service",
-    environment="production",
-    resource_attributes={"team": "ai", "version": "1.0.0"},
-    trace_content=True,
-    disable_batch=False,
-    instruments={InstrumentSet.OPENAI}
-)
-```
-
-### Custom Instrumentation Selection
-
-Control which instrumentations are enabled:
-
-```python
-from netra import Netra
-from netra.instrumentation.instruments import InstrumentSet
-
-# Enable specific instruments
-Netra.init(
-    app_name="Selective App",
-    instruments={
-        InstrumentSet.OPENAI,
-        InstrumentSet.WEAVIATEDB,
-        InstrumentSet.FASTAPI
-    }
-)
-
-# Block specific instruments
-Netra.init(
-    app_name="Blocked App",
-    block_instruments={
-        InstrumentSet.HTTPX,  # Don't trace HTTPX calls
-        InstrumentSet.REDIS   # Don't trace Redis operations
-    }
-)
-```
-
-### 🌐 Custom Endpoint Integration
-
-Since Netra SDK follows the **OpenTelemetry standard**, you can integrate it with any OpenTelemetry-compatible observability backend:
-
-#### Popular OpenTelemetry Backends
-- **Jaeger** - Distributed tracing platform
-- **Zipkin** - Distributed tracing system
-- **Prometheus** - Monitoring and alerting toolkit
-- **Grafana** - Observability and data visualization
-- **New Relic** - Full-stack observability platform
-- **Datadog** - Monitoring and analytics platform
-- **Honeycomb** - Observability for complex systems
-- **Lightstep** - Distributed tracing and observability
-- **AWS X-Ray** - Distributed tracing service
-- **Google Cloud Trace** - Distributed tracing system
-
-#### Custom Endpoint Configuration
-
-**Recommended: Environment Variable Configuration (No Code Changes Required)**
-```bash
-# Set custom OTLP endpoint via environment variables
-export NETRA_OTLP_ENDPOINT="https://your-custom-backend.com/v1/traces"
-export NETRA_HEADERS="authorization=Bearer your-token"
-
-```
-
-```python
-from netra import Netra
-from netra.instrumentation.instruments import InstrumentSet
-
-# Simple initialization - SDK automatically picks up environment variables
-Netra.init(app_name="Your App", instruments={InstrumentSet})
-# No endpoint configuration needed in code!
-```
-
-#### Benefits of OpenTelemetry Compatibility
 - **🔄 Vendor Agnostic**: Switch between observability platforms without code changes
 - **📊 Standard Format**: Consistent telemetry data across all tools
 - **🔧 Flexible Integration**: Works with existing observability infrastructure

{netra_sdk-0.1.18.dist-info → netra_sdk-0.1.20.dist-info}/RECORD

@@ -9,7 +9,7 @@ netra/exceptions/__init__.py,sha256=uDgcBxmC4WhdS7HRYQk_TtJyxH1s1o6wZmcsnSHLAcM,
 netra/exceptions/injection.py,sha256=ke4eUXRYUFJkMZgdSyPPkPt5PdxToTI6xLEBI0hTWUQ,1332
 netra/exceptions/pii.py,sha256=MT4p_x-zH3VtYudTSxw1Z9qQZADJDspq64WrYqSWlZc,2438
 netra/input_scanner.py,sha256=bzP3s7YudGHQrIbUgQGrcIBEJ6CmOewzuYNSu75cVXM,4988
-netra/instrumentation/__init__.py,sha256=ZORhcso04LPD6-Y7oPsh74rY-jwz1unsfYZBcP--SzM,40081
+netra/instrumentation/__init__.py,sha256=ckV_tYPCQhEQ03tT0NU0ZrPD0o_1x0RnxLja3Esi97Q,40252
 netra/instrumentation/aiohttp/__init__.py,sha256=M1kuF0R3gKY5rlbhEC1AR13UWHelmfokluL2yFysKWc,14398
 netra/instrumentation/aiohttp/version.py,sha256=Zy-0Aukx-HS_Mo3NKPWg-hlUoWKDzS0w58gLoVtJec8,24
 netra/instrumentation/cohere/__init__.py,sha256=3XwmCAZwZiMkHdNN3YvcBOLsNCx80ymbU31TyMzv1IY,17685
@@ -32,15 +32,15 @@ netra/instrumentation/openai/version.py,sha256=_J-N1qG50GykJDM356BSQf0E8LoLbB8Aa
 netra/instrumentation/openai/wrappers.py,sha256=4VQwIBLYaGovO9gE5TSMC-Ot84IaDuDhGqHndgR-Am4,21637
 netra/instrumentation/weaviate/__init__.py,sha256=EOlpWxobOLHYKqo_kMct_7nu26x1hr8qkeG5_h99wtg,4330
 netra/instrumentation/weaviate/version.py,sha256=PiCZHjonujPbnIn0KmD3Yl68hrjPRG_oKe5vJF3mmG8,24
-netra/pii.py,sha256=S7GnVzoNJEzKiUWnqN9bOCKPeNLsriztgB2E6Rx-yJU,27023
+netra/pii.py,sha256=Rn4SjgTJW_aw9LcbjLuMqF3fKd9b1ndlYt1CaK51Ge0,33125
 netra/processors/__init__.py,sha256=wfnSskRBtMT90hO7LqFJoEW374LgoH_gnTxhynqtByI,109
 netra/processors/session_span_processor.py,sha256=qcsBl-LnILWefsftI8NQhXDGb94OWPc8LvzhVA0JS_c,2432
 netra/scanner.py,sha256=wqjMZnEbVvrGMiUSI352grUyHpkk94oBfHfMiXPhpGU,3866
 netra/session_manager.py,sha256=EVcnWcSj4NdkH--HmqHx0mmzivQiM4GCyFLu6lwi33M,6252
 netra/span_wrapper.py,sha256=DA5jjXkHBUJ8_mdlYP06rcZzFoSih4gdP71Wwr3btcQ,8104
 netra/tracer.py,sha256=In5QPVLz_6BxrolWpav9EuR9_hirD2UUIlyY75QUaKk,3450
-netra/version.py,sha256=6BiuMUkhwQp6bzUZSF8np8F1NwCltEtK0sPBF__tepU,23
-netra_sdk-0.1.18.dist-info/LICENCE,sha256=8B_UoZ-BAl0AqiHAHUETCgd3I2B9yYJ1WEQtVb_qFMA,11359
-netra_sdk-0.1.18.dist-info/METADATA,sha256=xANrdroAsDTktKVUyaTO6-c6r9Pja6XIMda5-Zhn0Zo,26133
-netra_sdk-0.1.18.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-netra_sdk-0.1.18.dist-info/RECORD,,
+netra/version.py,sha256=8XalsVoLEfXslFvdtUEmkNOuYShzOzYOcFbgmOz1oSk,23
+netra_sdk-0.1.20.dist-info/LICENCE,sha256=8B_UoZ-BAl0AqiHAHUETCgd3I2B9yYJ1WEQtVb_qFMA,11359
+netra_sdk-0.1.20.dist-info/METADATA,sha256=l4zeyWaf_45aIOdY884MG0EVcqHCs4tS99sz7ifuqIw,26416
+netra_sdk-0.1.20.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+netra_sdk-0.1.20.dist-info/RECORD,,