contentintelpy 0.1.0__tar.gz

contentintelpy-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 ContentIntelPy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

contentintelpy-0.1.0/PKG-INFO

@@ -0,0 +1,156 @@
+Metadata-Version: 2.4
+Name: contentintelpy
+Version: 0.1.0
+Summary: Production-grade NLP library for unified content intelligence.
+Author-email: Ronit Fulari <ronitfulari31@gmail.com>
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: tqdm>=4.66.0
+Provides-Extra: core
+Requires-Dist: transformers<5.0.0,>=4.30.0; extra == "core"
+Requires-Dist: torch<3.0.0,>=2.0.0; extra == "core"
+Requires-Dist: sentence-transformers>=2.2.0; extra == "core"
+Provides-Extra: ner
+Requires-Dist: spacy>=3.7.0; extra == "ner"
+Requires-Dist: gliner>=0.1.0; extra == "ner"
+Provides-Extra: translation
+Requires-Dist: argostranslate>=1.9.0; extra == "translation"
+Provides-Extra: summarization
+Requires-Dist: sumy>=0.11.0; extra == "summarization"
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: black; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Dynamic: license-file
+
+# contentintelpy
+
+**Production-grade NLP library for unified content intelligence.**
+
+`contentintelpy` provides a unified, DAG-based engine for multilingual sentiment analysis, NER, translation, and summarization using real transformer models (RoBERTa, GLiNER, NLLB).
+
+## Features
+
+- **Real Models**: No heuristics. Uses State-of-the-Art Transformers.
+    - Sentiment: RoBERTa
+    - NER: GLiNER
+    - Translation: NLLB (GPU) + ArgosTranslate (Offline CPU)
+- **Hybrid Execution**: Models download on first run (lazy-loaded). Offline fallback available.
+- **Deterministic Pipelines**: DAG-based execution guarantees order.
+- **Dual API**: 
+    - **Pipeline-first** for complex workflows.
+    - **Service-first** for quick scripts.
+- **Production Ready**: Thread-safe, standard error handling, sparse outputs.
+
+## Installation
+
+Install the base library:
+```bash
+pip install contentintelpy
+```
+
+### Optional Dependencies (Recommended)
+Since the library uses heavy ML models, you should install the specific components you need:
+
+```bash
+# For all core features
+pip install "contentintelpy[core,ner,translation,summarization]"
+
+# For development
+pip install "contentintelpy[dev]"
+```
+
+> [!IMPORTANT]
+> **spaCy Model Requirement**
+> If you use NER or language features, you must install a spaCy model manually:
+> ```bash
+> python -m spacy download en_core_web_sm
+> ```
+
+---
+
+## Quick Start
+
+Ideal for simple tasks in notebooks or scripts.
+
+```python
+from contentintelpy import SentimentService, TranslationService
+
+# Sentiment
+service = SentimentService()
+result = service.analyze("This library is amazing!")
+print(result) 
+# {'value': 'positive', 'confidence': 0.99, ...}
+
+# Translation
+translator = TranslationService()
+text = translator.translate("Hola mundo", target="en")
+print(text)
+# "Hello world"
+```
+
+## Production Usage (Pipeline-First)
+
+Recommended for Backends, APIs, and Data Pipelines.
+
+```python
+import contentintelpy as ci
+
+# 1. Create the canonical pipeline
+pipeline = ci.create_default_pipeline()
+
+# 2. Run it (Thread-safe)
+result = pipeline.run({
+    "text": "गूगल ने बेंगलुरु में नया कार्यालय खोला"
+})
+
+# 3. Access Sparse Output
+print(result)
+```
+
+**Output Example:**
+```json
+{
+  "text": "...",
+  "text_translated": "Google opened a new office in Bengaluru",
+  "language": "hi",
+  "entities": [
+    {"text": "Google", "label": "ORG"},
+    {"text": "Bengaluru", "label": "LOC"}
+  ],
+  "sentiment": {
+    "value": "neutral",
+    "value_en": "neutral",
+    "confidence": 0.95
+  },
+  "summary": "..."
+}
+```
+
+## Error Handling
+
+Nodes **never crash** the pipeline. Errors are collected in `errors` dict.
+
+```python
+{
+    "text": "...",
+    "errors": {
+        "TranslationNode": "Model download failed: Connection error"
+    }
+}
+```
+
+## Architecture
+
+This library is pure logic. It does **NOT** contain:
+- Flask / FastAPI routes
+- Database models
+- Authentication
+
+It is designed to be **consumed** by your backend application.

contentintelpy-0.1.0/README.md

@@ -0,0 +1,125 @@
+# contentintelpy
+
+**Production-grade NLP library for unified content intelligence.**
+
+`contentintelpy` provides a unified, DAG-based engine for multilingual sentiment analysis, NER, translation, and summarization using real transformer models (RoBERTa, GLiNER, NLLB).
+
+## Features
+
+- **Real Models**: No heuristics. Uses State-of-the-Art Transformers.
+    - Sentiment: RoBERTa
+    - NER: GLiNER
+    - Translation: NLLB (GPU) + ArgosTranslate (Offline CPU)
+- **Hybrid Execution**: Models download on first run (lazy-loaded). Offline fallback available.
+- **Deterministic Pipelines**: DAG-based execution guarantees order.
+- **Dual API**: 
+    - **Pipeline-first** for complex workflows.
+    - **Service-first** for quick scripts.
+- **Production Ready**: Thread-safe, standard error handling, sparse outputs.
+
+## Installation
+
+Install the base library:
+```bash
+pip install contentintelpy
+```
+
+### Optional Dependencies (Recommended)
+Since the library uses heavy ML models, you should install the specific components you need:
+
+```bash
+# For all core features
+pip install "contentintelpy[core,ner,translation,summarization]"
+
+# For development
+pip install "contentintelpy[dev]"
+```
+
+> [!IMPORTANT]
+> **spaCy Model Requirement**
+> If you use NER or language features, you must install a spaCy model manually:
+> ```bash
+> python -m spacy download en_core_web_sm
+> ```
+
+---
+
+## Quick Start
+
+Ideal for simple tasks in notebooks or scripts.
+
+```python
+from contentintelpy import SentimentService, TranslationService
+
+# Sentiment
+service = SentimentService()
+result = service.analyze("This library is amazing!")
+print(result) 
+# {'value': 'positive', 'confidence': 0.99, ...}
+
+# Translation
+translator = TranslationService()
+text = translator.translate("Hola mundo", target="en")
+print(text)
+# "Hello world"
+```
+
+## Production Usage (Pipeline-First)
+
+Recommended for Backends, APIs, and Data Pipelines.
+
+```python
+import contentintelpy as ci
+
+# 1. Create the canonical pipeline
+pipeline = ci.create_default_pipeline()
+
+# 2. Run it (Thread-safe)
+result = pipeline.run({
+    "text": "गूगल ने बेंगलुरु में नया कार्यालय खोला"
+})
+
+# 3. Access Sparse Output
+print(result)
+```
+
+**Output Example:**
+```json
+{
+  "text": "...",
+  "text_translated": "Google opened a new office in Bengaluru",
+  "language": "hi",
+  "entities": [
+    {"text": "Google", "label": "ORG"},
+    {"text": "Bengaluru", "label": "LOC"}
+  ],
+  "sentiment": {
+    "value": "neutral",
+    "value_en": "neutral",
+    "confidence": 0.95
+  },
+  "summary": "..."
+}
+```
+
+## Error Handling
+
+Nodes **never crash** the pipeline. Errors are collected in `errors` dict.
+
+```python
+{
+    "text": "...",
+    "errors": {
+        "TranslationNode": "Model download failed: Connection error"
+    }
+}
+```
+
+## Architecture
+
+This library is pure logic. It does **NOT** contain:
+- Flask / FastAPI routes
+- Database models
+- Authentication
+
+It is designed to be **consumed** by your backend application.

contentintelpy-0.1.0/contentintelpy/__init__.py

@@ -0,0 +1,60 @@
+from .pipeline.pipeline import Pipeline
+from .pipeline.context import PipelineContext
+from .pipeline.base_node import Node
+
+# Import Nodes for the default pipeline
+from .nodes.language_node import LanguageDetectionNode
+from .nodes.translation_node import TranslationNode
+from .nodes.classification_node import CategoryClassificationNode
+from .nodes.ner_node import NERNode
+from .nodes.location_node import LocationExtractionNode
+from .nodes.sentiment_node import SentimentNode
+from .nodes.keyword_extract_node import KeywordExtractionNode
+from .nodes.summarization_node import SummarizationNode
+
+# Import Services for public use
+from .services.sentiment_service import SentimentService
+from .services.translation_service import TranslationService
+from .services.ner_service import NERService
+from .services.summarization_service import SummarizationService
+
+def create_default_pipeline() -> Pipeline:
+    """
+    Creates the canonical ContentIntelPy pipeline.
+    
+    Execution Order:
+    1. Language Detection (Detects source lang)
+    2. Translation (Normalizes to English)
+    3. Classification (Broad categorization)
+    4. NER (Entity discovery)
+    5. Location Extraction (Refines location entities)
+    6. Sentiment (Analyzes tone)
+    7. Keyword Extraction (Highlights)
+    8. Summarization (Reduces content)
+    
+    Returns:
+        A configured Pipeline instance ready to run.
+    """
+    nodes = [
+        LanguageDetectionNode(),
+        TranslationNode(target_lang="en"), # Normalize to English
+        CategoryClassificationNode(),
+        NERNode(),
+        LocationExtractionNode(),
+        SentimentNode(),
+        KeywordExtractionNode(),
+        SummarizationNode()
+    ]
+    
+    return Pipeline(nodes)
+
+__all__ = [
+    "Pipeline",
+    "create_default_pipeline",
+    "PipelineContext",
+    "Node",
+    "SentimentService",
+    "TranslationService",
+    "NERService",
+    "SummarizationService"
+]

contentintelpy-0.1.0/contentintelpy/nodes/classification_node.py

@@ -0,0 +1,49 @@
+from ..pipeline.base_node import Node
+from ..pipeline.context import PipelineContext
+from ..utils.model_registry import registry
+import logging
+
+logger = logging.getLogger("contentintelpy.nodes.classification")
+
+class CategoryClassificationNode(Node):
+    """
+    Classifies text into categories using Zero-Shot Classification (BART).
+    Default labels: Business, Politics, Sports, Technology, Entertainment, Health, Science.
+    """
+    DEFAULT_LABELS = [
+        "Business", "Politics", "Sports", "Technology", 
+        "Entertainment", "Health", "Science", "World"
+    ]
+
+    def __init__(self, candidate_labels: list = None):
+        super().__init__("CategoryClassificationNode")
+        self.candidate_labels = candidate_labels or self.DEFAULT_LABELS
+
+    def process(self, context: PipelineContext) -> PipelineContext:
+        # Prefer translated text if available, else original
+        text = context.get("text_translated") or context.get("text")
+        
+        if not text or not isinstance(text, str):
+            logger.warning("No text available for category classification.")
+            return context
+
+        try:
+            classifier = registry.get_classifier_pipeline()
+            # Multi-label=False ensures scores sum to 1
+            result = classifier(text, self.candidate_labels, multi_label=False)
+            
+            # Result format: {'labels': ['Sports', ...], 'scores': [0.99, ...]}
+            if result and 'labels' in result and 'scores' in result:
+                top_label = result['labels'][0]
+                top_score = result['scores'][0]
+                
+                context["category"] = top_label
+                context["category_score"] = top_score
+                context["all_categories"] = dict(zip(result['labels'], result['scores']))
+                logger.debug(f"Classified as: {top_label} ({top_score:.2f})")
+            
+        except Exception as e:
+            logger.error(f"Classification failed: {e}")
+            context.add_error("CategoryClassificationNode", str(e))
+
+        return context

contentintelpy-0.1.0/contentintelpy/nodes/keyword_extract_node.py

@@ -0,0 +1,78 @@
+from ..pipeline.base_node import Node
+from ..pipeline.context import PipelineContext
+from ..utils.model_registry import registry
+from sklearn.metrics.pairwise import cosine_similarity
+from sklearn.feature_extraction.text import CountVectorizer
+import numpy as np
+import logging
+import itertools
+
+logger = logging.getLogger("contentintelpy.nodes.keywords")
+
+class KeywordExtractionNode(Node):
+    """
+    Extracts keywords using semantic embeddings (KeyBERT-style logic).
+    
+    Algorithm:
+    1. Generate candidate n-grams (1-2 words).
+    2. Embed document and candidates using SentenceTransformer.
+    3. Calculate cosine similarity.
+    4. Return top N candidates.
+    """
+    def __init__(self, top_n: int = 5):
+        super().__init__("KeywordExtractionNode")
+        self.top_n = top_n
+
+    def process(self, context: PipelineContext) -> PipelineContext:
+        text = context.get("text_translated") or context.get("text")
+        
+        if not text or len(text.split()) < 3:
+            return context
+
+        try:
+            model = registry.get_embedding_model()
+
+            # 1. Candidate Generation (using simple CountVectorizer)
+            # Remove stopwords usually handled by vectorizer, but we use 'english'
+            n_gram_range = (1, 2)
+            count = CountVectorizer(ngram_range=n_gram_range, stop_words="english").fit([text])
+            candidates = count.get_feature_names_out()
+
+            if len(candidates) == 0:
+                return context
+
+            # 2. Embeddings
+            doc_embedding = model.encode([text])
+            candidate_embeddings = model.encode(candidates)
+
+            # 3. Similarity
+            distances = cosine_similarity(doc_embedding, candidate_embeddings)
+            
+            # 4. Top N
+            keywords = []
+            # flatten distances
+            distances = distances[0]
+            
+            # Get indices of top_n
+            # partition works nicely to get top elements, then we sort them
+            keywords_idx = np.argpartition(distances, -self.top_n)[-self.top_n:]
+            # Sort by score descending
+            keywords_idx = keywords_idx[np.argsort(distances[keywords_idx])][::-1]
+
+            for idx in keywords_idx:
+                keywords.append({
+                    "text": candidates[idx],
+                    "score": float(distances[idx])
+                })
+
+            context["keywords"] = keywords
+            logger.debug(f"Extracted {len(keywords)} keywords.")
+
+        except Exception as e:
+            # Fallback? Maybe just log error. 
+            # Ideally could fallback to rake-nltk but we banned it.
+            # So we just fail softly.
+            logger.error(f"Keyword extraction failed: {e}")
+            context.add_error("KeywordExtractionNode", str(e))
+
+        return context

contentintelpy-0.1.0/contentintelpy/nodes/language_node.py

@@ -0,0 +1,51 @@
+from ..pipeline.base_node import Node
+from ..pipeline.context import PipelineContext
+from ..utils.model_registry import registry
+import logging
+
+logger = logging.getLogger("contentintelpy.nodes.language")
+
+class LanguageDetectionNode(Node):
+    """
+    Detects the language of the input text.
+    Writes 'language' and 'language_score' to context.
+    """
+    def __init__(self):
+        super().__init__("LanguageDetectionNode")
+
+    def process(self, context: PipelineContext) -> PipelineContext:
+        text = context.get("text")
+        if not text or not isinstance(text, str):
+            logger.warning("No text found in context for LanguageDetectionNode.")
+            context["language"] = "en" # Default to English if no text
+            context["language_score"] = 0.0
+            return context
+
+        try:
+            detector = registry.get_language_detector()
+            # Truncate text for detection speed/limit
+            snippet = text[:512] 
+            
+            result = detector(snippet)
+            # Result format: [{'label': 'en', 'score': 0.99}]
+            
+            if result and len(result) > 0:
+                top_result = result[0]
+                lang_code = top_result['label']
+                score = top_result['score']
+                
+                context["language"] = lang_code
+                context["language_score"] = score
+                logger.info(f"Detected language: {lang_code} ({score:.2f})")
+            else:
+                context["language"] = "unknown"
+                context["language_score"] = 0.0
+                
+        except Exception as e:
+            # Fallback if model fails
+            logger.error(f"Language detection model error: {e}")
+            context["language"] = "en" # Fallback safe default
+            context["language_score"] = 0.0
+            raise e # Re-raise to trigger BaseNode error logging
+
+        return context

contentintelpy-0.1.0/contentintelpy/nodes/location_node.py

@@ -0,0 +1,47 @@
+from ..pipeline.base_node import Node
+from ..pipeline.context import PipelineContext
+import logging
+
+logger = logging.getLogger("contentintelpy.nodes.location")
+
+class LocationExtractionNode(Node):
+    """
+    Refines location data from NER results.
+    Extracts 'Location', 'City', 'Country' entities into a dedicated 'locations' key.
+    Future upgrade: Add actual Geocoding (Lat/Lon) here.
+    """
+    LOCATION_LABELS = {"Location", "City", "Country", "GPE"}
+
+    def __init__(self):
+        super().__init__("LocationExtractionNode")
+
+    def process(self, context: PipelineContext) -> PipelineContext:
+        entities = context.get("entities", [])
+        
+        if not entities:
+            return context
+
+        locations = []
+        seen = set()
+
+        for ent in entities:
+            label = ent.get("label")
+            text = ent.get("text")
+            
+            if label in self.LOCATION_LABELS and text:
+                # Basic deduplication
+                clean_text = text.lower().strip()
+                if clean_text not in seen:
+                    locations.append({
+                        "name": text,
+                        "type": label,
+                        # Placeholder for future geocoding extension:
+                        # "coordinates": None 
+                    })
+                    seen.add(clean_text)
+
+        if locations:
+            context["locations"] = locations
+            logger.debug(f"Extracted {len(locations)} unique locations.")
+        
+        return context

contentintelpy-0.1.0/contentintelpy/nodes/ner_node.py

@@ -0,0 +1,46 @@
+from ..pipeline.base_node import Node
+from ..pipeline.context import PipelineContext
+from ..utils.model_registry import registry
+import logging
+
+logger = logging.getLogger("contentintelpy.nodes.ner")
+
+class NERNode(Node):
+    """
+    Named Entity Recognition using GLiNER.
+    Extracts: Person, Organization, Location, Date, etc.
+    """
+    LABELS = ["Person", "Organization", "Location", "City", "Country", "Date"]
+
+    def __init__(self, labels: list = None):
+        super().__init__("NERNode")
+        self.labels = labels or self.LABELS
+
+    def process(self, context: PipelineContext) -> PipelineContext:
+        text = context.get("text_translated") or context.get("text")
+        
+        if not text:
+            return context
+
+        try:
+            gliner = registry.get_gliner_model()
+            # GLiNER predict_entities returns list of dicts: {'text': '', 'label': '', 'score': float}
+            entities = gliner.predict_entities(text, self.labels)
+            
+            # Normalize and serialize
+            serialized_entities = []
+            for ent in entities:
+                serialized_entities.append({
+                    "text": ent["text"],
+                    "label": ent["label"],
+                    "score": float(ent.get("score", 0.0))
+                })
+            
+            context["entities"] = serialized_entities
+            logger.debug(f"Found {len(serialized_entities)} entities.")
+
+        except Exception as e:
+            logger.error(f"NER failed: {e}")
+            context.add_error("NERNode", str(e))
+
+        return context