content-core 0.1.0__py3-none-any.whl

content_core/tools/__init__.py

@@ -0,0 +1,9 @@
+from .cleanup import cleanup_content_tool
+from .extract import extract_content_tool
+from .summarize import summarize_content_tool
+
+__all__ = [
+    "cleanup_content_tool",
+    "extract_content_tool",
+    "summarize_content_tool",
+]

content_core/tools/cleanup.py

@@ -0,0 +1,15 @@
+from langchain_core.tools import tool
+
+from content_core.content_cleanup import cleanup_content
+from content_core.common import process_input_content
+
+
+@tool
+async def cleanup_content_tool(content: str) -> str:
+    """
+    Clean content. Rewrite paragraphs. Fix grammar and spelling.
+    Accepts direct text, URLs, or file paths. If a URL or file path is provided,
+    the content will be extracted first before cleaning.
+    """
+    content = await process_input_content(content)
+    return await cleanup_content(content)

content_core/tools/extract.py

@@ -0,0 +1,21 @@
+from typing import Dict
+
+from langchain_core.tools import tool
+
+from content_core.extraction import extract_content
+
+
+@tool
+async def extract_content_tool(file_path_or_url: str) -> Dict:
+    """
+    Extract title, content and metadata from URLs and Links.
+
+    Args:
+        file_path_or_url: URL or file path to extract content from.
+
+    Returns:
+        Dict: Extracted content and metadata.
+    """
+    if file_path_or_url.startswith("http"):
+        return await extract_content({"url": file_path_or_url})
+    return await extract_content({"file_path": file_path_or_url})

content_core/tools/summarize.py

@@ -0,0 +1,17 @@
+from typing import Optional
+
+from langchain_core.tools import tool
+
+from content_core.content_summary import summarize
+from content_core.common import process_input_content
+
+
+@tool
+async def summarize_content_tool(content: str, context: Optional[str] = None) -> str:
+    """
+    Summarize content according to instructions provided via context.
+    Accepts direct text, URLs, or file paths. If a URL or file path is provided,
+    the content will be extracted first before summarizing.
+    """
+    content = await process_input_content(content)
+    return await summarize(content, context or "")

content_core-0.1.0.dist-info/METADATA

@@ -0,0 +1,250 @@
+Metadata-Version: 2.4
+Name: content-core
+Version: 0.1.0
+Summary: Extract what matters from any media source
+Author-email: LUIS NOVO <lfnovo@gmail.com>
+Requires-Python: >=3.10
+Requires-Dist: aiohttp>=3.11.16
+Requires-Dist: bs4>=0.0.2
+Requires-Dist: dicttoxml>=1.7.16
+Requires-Dist: esperanto>=1.2.0
+Requires-Dist: google-genai>=1.10.0
+Requires-Dist: jinja2>=3.1.6
+Requires-Dist: langdetect>=1.0.9
+Requires-Dist: langgraph>=0.3.29
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: openai>=1.73.0
+Requires-Dist: openpyxl>=3.1.5
+Requires-Dist: pandas>=2.2.3
+Requires-Dist: pydub>=0.25.1
+Requires-Dist: pymupdf>=1.25.5
+Requires-Dist: python-docx>=1.1.2
+Requires-Dist: python-dotenv>=1.1.0
+Requires-Dist: python-magic>=0.4.27
+Requires-Dist: python-pptx>=1.0.2
+Requires-Dist: validators>=0.34.0
+Requires-Dist: youtube-transcript-api>=1.0.3
+Description-Content-Type: text/markdown
+
+# Content Core
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+**Content Core** is a versatile Python library designed to extract and process content from various sources, providing a unified interface for handling text, web pages, and local files.
+
+## Overview
+
+The primary goal of Content Core is to simplify the process of ingesting content from diverse origins. Whether you have raw text, a URL pointing to an article, or a local file like a video or markdown document, Content Core aims to extract the meaningful content for further use.
+
+## Key Features
+
+*   **Multi-Source Extraction:** Handles content from:
+    *   Direct text strings.
+    *   Web URLs (using robust extraction methods).
+    *   Local files (including automatic transcription for video/audio files and parsing for text-based formats).
+*   **Intelligent Processing:** Applies appropriate extraction techniques based on the source type.
+*   **Content Cleaning (Optional):** Likely integrates with LLMs (via `prompter.py` and Jinja templates) to refine and clean the extracted content.
+*   **Asynchronous:** Built with `asyncio` for efficient I/O operations.
+
+## Getting Started
+
+### Installation
+
+Install Content Core using `pip`:
+
+```bash
+# Install the package
+pip install content-core
+```
+
+Alternatively, if you’re developing locally:
+
+```bash
+# Clone the repository
+git clone https://github.com/lfnovo/content-core
+cd content-core
+
+# Install with uv
+uv sync
+```
+
+### Command-Line Interface
+
+Content Core provides three CLI commands for extracting, cleaning, and summarizing content: 
+ccore, cclean, and csum. These commands support input from text, URLs, files, or piped data (e.g., via cat file | command).
+
+#### ccore - Extract Content
+
+Extracts content from text, URLs, or files, with optional formatting.
+Usage:
+```bash
+ccore [-f|--format xml|json|text] [-d|--debug] [content]
+```
+Options:
+- `-f`, `--format`: Output format (xml, json, or text). Default: text.
+- `-d`, `--debug`: Enable debug logging.
+- `content`: Input content (text, URL, or file path). If omitted, reads from stdin.
+
+Examples:
+
+```bash
+# Extract from a URL as text
+ccore https://example.com
+
+# Extract from a file as JSON
+ccore -f json document.pdf
+
+# Extract from piped text as XML
+echo "Sample text" | ccore --format xml
+```
+
+#### cclean - Clean Content
+Cleans content by removing unnecessary formatting, spaces, or artifacts. Accepts text, JSON, XML input, URLs, or file paths.
+Usage:
+
+```bash
+cclean [-d|--debug] [content]
+```
+
+Options:
+- `-d`, `--debug`: Enable debug logging.
+- `content`: Input content to clean (text, URL, file path, JSON, or XML). If omitted, reads from stdin.
+
+Examples:
+
+```bash
+# Clean a text string
+cclean "  messy   text   "
+
+# Clean piped JSON
+echo '{"content": "  messy   text   "}' | cclean
+
+# Clean content from a URL
+cclean https://example.com
+
+# Clean a file’s content
+cclean document.txt
+```
+
+### csum - Summarize Content
+
+Summarizes content with an optional context to guide the summary style. Accepts text, JSON, XML input, URLs, or file paths.
+
+Usage:
+
+```bash
+csum [--context "context text"] [-d|--debug] [content]
+```
+
+Options:
+- `--context`: Context for summarization (e.g., "explain to a child"). Default: none.
+- `-d`, `--debug`: Enable debug logging.
+- `content`: Input content to summarize (text, URL, file path, JSON, or XML). If omitted, reads from stdin.
+
+Examples:
+
+```bash
+# Summarize text
+csum "AI is transforming industries."
+
+# Summarize with context
+csum --context "in bullet points" "AI is transforming industries."
+
+# Summarize piped content
+cat article.txt | csum --context "one sentence"
+
+# Summarize content from URL
+csum https://example.com
+
+# Summarize a file's content
+csum document.txt
+```
+
+## Using with Langchain
+
+For users integrating with the [Langchain](https://python.langchain.com/) framework, `content-core` exposes a set of compatible tools. These tools, located in the `src/content_core/tools` directory, allow you to leverage `content-core` extraction, cleaning, and summarization capabilities directly within your Langchain agents and chains.
+
+You can import and use these tools like any other Langchain tool. For example:
+
+```python
+from content_core.tools import extract_content_tool, cleanup_content_tool, summarize_content_tool
+from langchain.agents import initialize_agent, AgentType
+
+tools = [extract_content_tool, cleanup_content_tool, summarize_content_tool]
+agent = initialize_agent(tools, llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, verbose=True)
+agent.run("Extract the content from https://example.com and then summarize it.") 
+```
+
+Refer to the source code in `src/content_core/tools` for specific tool implementations and usage details.
+
+## Basic Usage
+
+The core functionality revolves around the extract_content function.
+
+```python
+import asyncio
+from content_core.extraction import extract_content
+
+async def main():
+    # Extract from raw text
+    text_data = await extract_content({"content": "This is my sample text content."})
+    print(text_data)
+
+    # Extract from a URL
+    url_data = await extract_content({"url": "https://www.example.com"})
+    print(url_data)
+
+    # Extract from a local video file (gets transcript)
+    video_data = await extract_content({"file_path": "path/to/your/video.mp4"})
+    print(video_data)
+
+    # Extract from a local markdown file
+    md_data = await extract_content({"file_path": "path/to/your/document.md"})
+    print(md_data)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+(See `src/content_core/notebooks/run.ipynb` for more detailed examples.)
+
+## Configuration
+
+Configuration settings (like API keys for external services, logging levels) can be managed through environment variables or `.env` files, loaded automatically via `python-dotenv`.
+
+Example `.env`:
+
+```plaintext
+OPENAI_API_KEY=your-key-here
+GOOGLE_API_KEY=your-key-here
+```
+
+## Development
+
+To set up a development environment:
+
+```bash
+# Clone the repository
+git clone <repository-url>
+cd content-core
+
+# Create virtual environment and install dependencies
+uv venv
+source .venv/bin/activate
+uv sync --group dev
+
+# Run tests
+make test
+
+# Lint code
+make lint
+
+# See all commands
+make help
+```
+
+## Contributing
+Contributions are welcome! Please follow standard practices (fork, feature branch, pull request).
+
+## License
+This project is licensed under the MIT License - see the LICENSE file for details.

content_core-0.1.0.dist-info/RECORD

@@ -0,0 +1,32 @@
+content_core/__init__.py,sha256=CAPVVm3mDl8dH3j7Pn7t7UJqdwwpNjy77SzF7acssFw,6352
+content_core/config.py,sha256=5nFWb-g7DG__OuxSvwK4yCFEC1YCKdE6-rX5Z7c6JSo,794
+content_core/prompter.py,sha256=aUm_Bz_pkQuXIMKB1Xe6OEE-y4AUNoNsfFy82fAU-Ss,4049
+content_core/py.typed,sha256=pLuU3XTTeVpXo4UomOjcvAIQqOrzIotlWlJ3KFo2lxQ,154
+content_core/templated_message.py,sha256=NSttaX1jL5LYIDlJnabx7baDuySIFIPjFjAX5NZt9pM,1704
+content_core/common/__init__.py,sha256=SjDp-0QRjX9PMubyTjv77_GrUqm6eC4gBuXr593JVK4,525
+content_core/common/exceptions.py,sha256=NpYedVbckIq4kP2wek7bicMVgGGn0fkhCvid5cIxfy4,1304
+content_core/common/state.py,sha256=GrBtXKCz3c9SlMBfVxcTZ9szCrtQtz8PEo6E7Ihy6vY,867
+content_core/common/utils.py,sha256=0o4jovPEw_6wu7EcPPbDNZskbhhfLUBJBvRmp0Yc4R4,1182
+content_core/content/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+content_core/content/cleanup/__init__.py,sha256=wymD24WLDDdsZrv-5WhparSiHBK9SJCcqBHmokuZqk4,121
+content_core/content/cleanup/core.py,sha256=FJupJGiIcZfAEX8Usn8ob2REfOVYFhcL3JmGovdnJOM,506
+content_core/content/extraction/__init__.py,sha256=TaYw6CAcG62GZfsJxeZ6VJDLP85BU2a7_G271v6WWPk,446
+content_core/content/extraction/graph.py,sha256=qrTEl9YDUJJJg7TbBqPjueSvV9oo4_WwAJ-VpWKOYec,4621
+content_core/content/summary/__init__.py,sha256=ReKCZWKfDtqlInKeh87Y1DEfiNzVWabGybEz3hS2FrI,114
+content_core/content/summary/core.py,sha256=BTzTYhB-5sQmvvDaxnnWwBYcVinMHcnG-fBApcf2cyg,517
+content_core/notebooks/run.ipynb,sha256=MmZGVl62b8S7FpsFPAUKbB7ndEtyFXY-JeihaeT5CII,375888
+content_core/processors/audio.py,sha256=ox-ScigfbBrN9B4MCvdgbYn2d3GBYqf6His0HPrzXDs,3459
+content_core/processors/office.py,sha256=13qNAfqqLwXUT6HNmF8OnxjbfvkhnTRCPoVUctw4w1k,12139
+content_core/processors/pdf.py,sha256=yndt8EGvV5_IxcFbFp4lb4g9T84w6cJ4LPdiTduO7aM,5296
+content_core/processors/text.py,sha256=MiXNILDKcLO5sWTzp-LNJ8yC764_gmUEvDB7GUd7Wys,1145
+content_core/processors/url.py,sha256=LcDB_FcmJMcwqM75DX2ERXcOry98e63WEjwd6l8u3ho,6268
+content_core/processors/video.py,sha256=aEe3M_POENPwI1tK4mUNxSeGewsmjex7lvMmELdcQeo,5183
+content_core/processors/youtube.py,sha256=RdkMWVV3iy4HMZfkT_eq_fUfSbwbimN8LrKTcdti0JA,5700
+content_core/tools/__init__.py,sha256=DuJmd7fE-NpDvLP8IW1XY5MUkAQcdks52rn2jk4N8jQ,231
+content_core/tools/cleanup.py,sha256=5IdKedsFyRQMdYzgFSKtsfyxJldbroXQXHesHICNENI,523
+content_core/tools/extract.py,sha256=-r2_jsuMMXyXxGVqWhh1ilNPo_UMYAbw3Pkp1FzPy5g,577
+content_core/tools/summarize.py,sha256=DPfeglLWB08q8SvHrsKpOKZ35XjduUDs2J02ISwjdj0,596
+content_core-0.1.0.dist-info/METADATA,sha256=tkjNmmkdSfe3Mt64qOFicBwmHDAnHCEC0fbZW7wFMJo,7305
+content_core-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+content_core-0.1.0.dist-info/entry_points.txt,sha256=9fGQUk6bxBVXj9PRwfWVPn54ClSEJV7J-KBLXtjOhQw,99
+content_core-0.1.0.dist-info/RECORD,,

content_core-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

content_core-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,4 @@
+[console_scripts]
+cclean = content_core:cclean
+ccore = content_core:ccore
+csum = content_core:csum