content-core 1.10.0__py3-none-any.whl

content_core/__init__.py

@@ -0,0 +1,216 @@
+import argparse
+import asyncio
+import json
+import os
+import sys
+from xml.etree import ElementTree as ET
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+from dicttoxml import dicttoxml  # type: ignore
+
+from content_core.common import ProcessSourceInput
+from content_core.content.cleanup import cleanup_content
+from content_core.content.extraction import extract_content
+from content_core.content.summary import summarize
+from content_core.logging import configure_logging, logger
+
+# Exposing functions for direct access when importing content_core as cc
+extract = extract_content
+clean = cleanup_content
+
+
+# Configure loguru logger using centralized configuration
+configure_logging(debug=False)
+
+
+def parse_content_format(content: str) -> str:
+    """Parse content that might be JSON or XML, extracting the 'content' field if present."""
+    try:
+        # Try JSON first
+        try:
+            json_data = json.loads(content)
+            if isinstance(json_data, dict) and "content" in json_data:
+                extracted = json_data["content"]
+                return str(extracted) if extracted is not None else content
+        except json.JSONDecodeError:
+            # Try XML
+            try:
+                root = ET.fromstring(content)
+                content_elem = root.find(".//content")
+                if content_elem is not None and content_elem.text is not None:
+                    return content_elem.text
+            except ET.ParseError:
+                pass
+        return content
+    except Exception as e:
+        logger.error(f"Error parsing content: {e}")
+        return content
+
+
+def get_content(args, parser, allow_empty=False):
+    """Helper to get content from args or stdin."""
+    if args.content is None:
+        if sys.stdin.isatty():
+            parser.error("No content provided. Provide content or pipe input.")
+        else:
+            content = sys.stdin.read().strip()
+    else:
+        content = args.content
+
+    if not content and not allow_empty:
+        parser.error("Empty input provided.")
+    return content
+
+
+async def process_input_content(content: str) -> str:
+    """Process input content, handling URLs and file paths."""
+    if "http" in content:
+        result = await extract_content(ProcessSourceInput(url=content))
+        content = result.content if result.content else str(result)
+    elif os.path.exists(content):
+        result = await extract_content(ProcessSourceInput(file_path=content))
+        content = result.content if result.content else str(result)
+    return content
+
+
+async def ccore_main():
+    """CLI logic for ccore (extract)."""
+    parser = argparse.ArgumentParser(
+        description="Content Core CLI: Extract content with formatting options."
+    )
+    parser.add_argument(
+        "-f",
+        "--format",
+        choices=["xml", "json", "text"],
+        default="text",
+        help="Output format (xml, json, or text). Default: text",
+    )
+    parser.add_argument(
+        "-d", "--debug", action="store_true", help="Enable debug logging."
+    )
+    parser.add_argument(
+        "content",
+        nargs="?",
+        help="Content to process (URL, file path, or text). If not provided, reads from stdin.",
+    )
+
+    args = parser.parse_args()
+
+    # Adjust logging level based on debug flag using centralized configuration
+    configure_logging(debug=args.debug)
+    if args.debug:
+        logger.debug("Debug logging enabled")
+
+    content = get_content(args, parser)
+
+    content = await process_input_content(content)
+
+    try:
+        result = await extract_content(ProcessSourceInput(content=content))
+        if args.format == "xml":
+            result = dicttoxml(
+                result.model_dump(), custom_root="result", attr_type=False
+            ).decode('utf-8')
+        elif args.format == "json":
+            result = result.model_dump_json()
+        else:  # text
+            result = result.content
+        print(result)
+    except Exception as e:
+        logger.error(f"Error extracting content: {e}")
+        sys.exit(1)
+
+
+async def cclean_main():
+    """CLI logic for cclean."""
+    parser = argparse.ArgumentParser(
+        description="Content Core CLI: Clean content string."
+    )
+    parser.add_argument(
+        "-d", "--debug", action="store_true", help="Enable debug logging."
+    )
+    parser.add_argument(
+        "content",
+        nargs="?",
+        help="Content to clean (URL, file path, text, JSON, or XML). If not provided, reads from stdin.",
+    )
+
+    args = parser.parse_args()
+
+    # Adjust logging level based on debug flag using centralized configuration
+    configure_logging(debug=args.debug)
+    if args.debug:
+        logger.debug("Debug logging enabled")
+
+    content = get_content(args, parser)
+
+    content = await process_input_content(content)
+    content = parse_content_format(content)
+
+    try:
+        result = await cleanup_content(content)
+        print(result)
+    except Exception as e:
+        logger.error(f"Error cleaning content: {e}")
+        sys.exit(1)
+
+
+async def csum_main():
+    """CLI logic for csum."""
+    parser = argparse.ArgumentParser(
+        description="Content Core CLI: Summarize content with optional context."
+    )
+    parser.add_argument(
+        "--context",
+        default="",
+        help="Optional context for summarization (e.g., 'summarize as if explaining to a child').",
+    )
+    parser.add_argument(
+        "-d", "--debug", action="store_true", help="Enable debug logging."
+    )
+    parser.add_argument(
+        "content",
+        nargs="?",
+        help="Content to summarize (URL, file path, text, JSON, or XML). If not provided, reads from stdin.",
+    )
+
+    args = parser.parse_args()
+
+    # Adjust logging level based on debug flag using centralized configuration
+    configure_logging(debug=args.debug)
+    if args.debug:
+        logger.debug("Debug logging enabled")
+
+    content = get_content(args, parser)
+
+    content = await process_input_content(content)
+    content = parse_content_format(content)
+
+    try:
+        result = await summarize(content, args.context)
+        print(result)
+    except Exception as e:
+        logger.error(f"Error summarizing content: {e}")
+        sys.exit(1)
+
+
+def ccore():
+    """Synchronous wrapper for ccore."""
+    asyncio.run(ccore_main())
+
+
+def cclean():
+    """Synchronous wrapper for cclean."""
+    asyncio.run(cclean_main())
+
+
+def csum():
+    """Synchronous wrapper for csum."""
+    asyncio.run(csum_main())
+
+
+if __name__ == "__main__":
+    ccore()

content_core/cc_config.yaml

@@ -0,0 +1,86 @@
+# Content Core main configuration
+# Copy this file to your project root or set CCORE_CONFIG_PATH to its location
+
+speech_to_text:
+  provider: openai
+  model_name: gpt-4o-transcribe-diarize
+  timeout: 3600  # 1 hour - for processing very long audio files
+
+default_model:
+  provider: openai
+  model_name: gpt-4o-mini
+  config:
+    temperature: 0.5
+    top_p: 1
+    max_tokens: 2000
+    timeout: 300  # 5 minutes - for general language model operations
+
+cleanup_model:
+  provider: openai
+  model_name: gpt-4o-mini
+  config:
+    temperature: 0
+    max_tokens: 8000
+    output_format: json
+    timeout: 600  # 10 minutes - for complex content cleaning with large inputs
+
+summary_model:
+  provider: openai
+  model_name: gpt-4o-mini
+  config:
+    temperature: 0
+    top_p: 1
+    max_tokens: 2000
+    timeout: 300  # 5 minutes - for content summarization
+
+extraction:
+  document_engine: auto  # auto | simple | docling - for files/documents
+  url_engine: auto  # auto | simple | firecrawl | jina | crawl4ai | docling - for URLs
+  audio:
+    concurrency: 3              # Number of concurrent audio transcriptions (1-10)
+  docling:
+    output_format: markdown  # markdown | html | json
+  pymupdf:
+    enable_formula_ocr: false    # Enable OCR for formula-heavy pages (requires Tesseract)
+    formula_threshold: 3         # Minimum formulas per page to trigger OCR
+    ocr_fallback: true          # Gracefully fallback to standard extraction if OCR fails
+
+youtube_transcripts:
+  preferred_languages: ["en", "es", "pt"]
+
+# Proxy configuration for HTTP/HTTPS requests
+# Environment variables CCORE_HTTP_PROXY, HTTP_PROXY, or HTTPS_PROXY take precedence
+proxy:
+  url: null  # Proxy URL (e.g., "http://proxy.example.com:8080" or "http://user:pass@proxy:8080")
+  no_proxy:  # List of hosts to bypass proxy
+    - "localhost"
+    - "127.0.0.1"
+
+# Retry configuration for transient failures
+# Each operation type can be configured independently
+# Environment variables override these settings (e.g., CCORE_YOUTUBE_MAX_RETRIES)
+retry:
+  youtube:
+    max_attempts: 5       # Number of retry attempts (YouTube has aggressive rate limiting)
+    base_delay: 2         # Base delay in seconds for exponential backoff
+    max_delay: 60         # Maximum delay between retries
+  url_api:
+    max_attempts: 3       # For API-based URL extraction (Jina, Firecrawl) and Crawl4AI
+    base_delay: 1
+    max_delay: 30
+  url_network:
+    max_attempts: 3       # For network-only operations (BeautifulSoup, HEAD requests)
+    base_delay: 0.5
+    max_delay: 10
+  audio:
+    max_attempts: 3       # For speech-to-text API calls
+    base_delay: 2
+    max_delay: 30
+  llm:
+    max_attempts: 3       # For LLM API calls (summary, cleanup)
+    base_delay: 1
+    max_delay: 30
+  download:
+    max_attempts: 3       # For remote file downloads
+    base_delay: 1
+    max_delay: 15

content_core/common/__init__.py

@@ -0,0 +1,38 @@
+"""Common utilities and shared code for content-core."""
+
+from .exceptions import (
+    ContentCoreError,
+    InvalidInputError,
+    NotFoundError,
+    UnsupportedTypeException,
+)
+from .retry import (
+    RetryError,
+    retry_audio_transcription,
+    retry_download,
+    retry_llm,
+    retry_url_api,
+    retry_url_network,
+    retry_youtube,
+)
+from .state import ProcessSourceInput, ProcessSourceOutput, ProcessSourceState
+from .utils import process_input_content
+
+__all__ = [
+    "ContentCoreError",
+    "UnsupportedTypeException",
+    "InvalidInputError",
+    "NotFoundError",
+    "ProcessSourceInput",
+    "ProcessSourceState",
+    "ProcessSourceOutput",
+    "process_input_content",
+    # Retry decorators
+    "retry_youtube",
+    "retry_url_api",
+    "retry_url_network",
+    "retry_audio_transcription",
+    "retry_llm",
+    "retry_download",
+    "RetryError",
+]

content_core/common/exceptions.py

@@ -0,0 +1,70 @@
+class ContentCoreError(Exception):
+    """Base exception class for Open Notebook errors."""
+
+    pass
+
+
+class DatabaseOperationError(ContentCoreError):
+    """Raised when a database operation fails."""
+
+    pass
+
+
+class UnsupportedTypeException(ContentCoreError):
+    """Raised when an unsupported type is provided."""
+
+    pass
+
+
+class InvalidInputError(ContentCoreError):
+    """Raised when invalid input is provided."""
+
+    pass
+
+
+class NotFoundError(ContentCoreError):
+    """Raised when a requested resource is not found."""
+
+    pass
+
+
+class AuthenticationError(ContentCoreError):
+    """Raised when there's an authentication problem."""
+
+    pass
+
+
+class ConfigurationError(ContentCoreError):
+    """Raised when there's a configuration problem."""
+
+    pass
+
+
+class ExternalServiceError(ContentCoreError):
+    """Raised when an external service (e.g., AI model) fails."""
+
+    pass
+
+
+class RateLimitError(ContentCoreError):
+    """Raised when a rate limit is exceeded."""
+
+    pass
+
+
+class FileOperationError(ContentCoreError):
+    """Raised when a file operation fails."""
+
+    pass
+
+
+class NetworkError(ContentCoreError):
+    """Raised when a network operation fails."""
+
+    pass
+
+
+class NoTranscriptFound(ContentCoreError):
+    """Raised when no transcript is found for a video."""
+
+    pass