content-core 0.1.0__tar.gz

content_core-0.1.0/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,22 @@
+<!--
+Thanks for contributing a pull request! Please ensure you have taken a look at CONTRIBUTING.md
+-->
+
+#### Reference Issues/PRs
+<!--
+Example: Fixes #1234. See also #3456.
+Please use keywords (e.g., Fixes) to create link to the issues or pull requests
+you resolved, so that they will automatically be closed when your pull request
+is merged. See https://github.com/blog/1506-closing-issues-via-pull-requests
+-->
+
+
+#### What does this implement/fix? Explain your changes.
+
+
+#### Any other comments?
+
+
+<!--
+Thanks for contributing!
+-->

content_core-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,38 @@
+name: Publish
+
+on:
+  push:
+    tags:
+      - 'v*'
+      - '[0-9]*.[0-9]*.[0-9]*'
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.10"
+    
+    - name: Install uv
+      run: |
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+        
+    - name: Install dependencies and build tools
+      run: |
+        uv venv
+        rm -rf dist
+        uv sync
+
+    - name: Build package
+      run: uv build
+    
+    - name: Publish to PyPI
+      env:
+        PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+      run: uv publish --token "$PYPI_TOKEN"

content_core-0.1.0/.gitignore

@@ -0,0 +1,21 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+.env
+
+**/tmp
+
+**/*.DS_Store
+.vscode/
+
+ai_docs/
+
+todo.md
+WIP/

content_core-0.1.0/.python-version

@@ -0,0 +1 @@
+3.10

content_core-0.1.0/.windsurfrules

@@ -0,0 +1 @@
+Also use uv as the package manager: uv run, uv sync, uv add.

content_core-0.1.0/Makefile

@@ -0,0 +1,8 @@
+test:
+	uv run pytest -v
+    
+build-docs:
+	repomix . --include "**/*.py,**/*.yaml" --compress --style xml -o ai_docs/core.txt
+
+ruff:
+	ruff check . --fix

content_core-0.1.0/PKG-INFO

@@ -0,0 +1,250 @@
+Metadata-Version: 2.4
+Name: content-core
+Version: 0.1.0
+Summary: Extract what matters from any media source
+Author-email: LUIS NOVO <lfnovo@gmail.com>
+Requires-Python: >=3.10
+Requires-Dist: aiohttp>=3.11.16
+Requires-Dist: bs4>=0.0.2
+Requires-Dist: dicttoxml>=1.7.16
+Requires-Dist: esperanto>=1.2.0
+Requires-Dist: google-genai>=1.10.0
+Requires-Dist: jinja2>=3.1.6
+Requires-Dist: langdetect>=1.0.9
+Requires-Dist: langgraph>=0.3.29
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: openai>=1.73.0
+Requires-Dist: openpyxl>=3.1.5
+Requires-Dist: pandas>=2.2.3
+Requires-Dist: pydub>=0.25.1
+Requires-Dist: pymupdf>=1.25.5
+Requires-Dist: python-docx>=1.1.2
+Requires-Dist: python-dotenv>=1.1.0
+Requires-Dist: python-magic>=0.4.27
+Requires-Dist: python-pptx>=1.0.2
+Requires-Dist: validators>=0.34.0
+Requires-Dist: youtube-transcript-api>=1.0.3
+Description-Content-Type: text/markdown
+
+# Content Core
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+**Content Core** is a versatile Python library designed to extract and process content from various sources, providing a unified interface for handling text, web pages, and local files.
+
+## Overview
+
+The primary goal of Content Core is to simplify the process of ingesting content from diverse origins. Whether you have raw text, a URL pointing to an article, or a local file like a video or markdown document, Content Core aims to extract the meaningful content for further use.
+
+## Key Features
+
+*   **Multi-Source Extraction:** Handles content from:
+    *   Direct text strings.
+    *   Web URLs (using robust extraction methods).
+    *   Local files (including automatic transcription for video/audio files and parsing for text-based formats).
+*   **Intelligent Processing:** Applies appropriate extraction techniques based on the source type.
+*   **Content Cleaning (Optional):** Likely integrates with LLMs (via `prompter.py` and Jinja templates) to refine and clean the extracted content.
+*   **Asynchronous:** Built with `asyncio` for efficient I/O operations.
+
+## Getting Started
+
+### Installation
+
+Install Content Core using `pip`:
+
+```bash
+# Install the package
+pip install content-core
+```
+
+Alternatively, if you’re developing locally:
+
+```bash
+# Clone the repository
+git clone https://github.com/lfnovo/content-core
+cd content-core
+
+# Install with uv
+uv sync
+```
+
+### Command-Line Interface
+
+Content Core provides three CLI commands for extracting, cleaning, and summarizing content: 
+ccore, cclean, and csum. These commands support input from text, URLs, files, or piped data (e.g., via cat file | command).
+
+#### ccore - Extract Content
+
+Extracts content from text, URLs, or files, with optional formatting.
+Usage:
+```bash
+ccore [-f|--format xml|json|text] [-d|--debug] [content]
+```
+Options:
+- `-f`, `--format`: Output format (xml, json, or text). Default: text.
+- `-d`, `--debug`: Enable debug logging.
+- `content`: Input content (text, URL, or file path). If omitted, reads from stdin.
+
+Examples:
+
+```bash
+# Extract from a URL as text
+ccore https://example.com
+
+# Extract from a file as JSON
+ccore -f json document.pdf
+
+# Extract from piped text as XML
+echo "Sample text" | ccore --format xml
+```
+
+#### cclean - Clean Content
+Cleans content by removing unnecessary formatting, spaces, or artifacts. Accepts text, JSON, XML input, URLs, or file paths.
+Usage:
+
+```bash
+cclean [-d|--debug] [content]
+```
+
+Options:
+- `-d`, `--debug`: Enable debug logging.
+- `content`: Input content to clean (text, URL, file path, JSON, or XML). If omitted, reads from stdin.
+
+Examples:
+
+```bash
+# Clean a text string
+cclean "  messy   text   "
+
+# Clean piped JSON
+echo '{"content": "  messy   text   "}' | cclean
+
+# Clean content from a URL
+cclean https://example.com
+
+# Clean a file’s content
+cclean document.txt
+```
+
+### csum - Summarize Content
+
+Summarizes content with an optional context to guide the summary style. Accepts text, JSON, XML input, URLs, or file paths.
+
+Usage:
+
+```bash
+csum [--context "context text"] [-d|--debug] [content]
+```
+
+Options:
+- `--context`: Context for summarization (e.g., "explain to a child"). Default: none.
+- `-d`, `--debug`: Enable debug logging.
+- `content`: Input content to summarize (text, URL, file path, JSON, or XML). If omitted, reads from stdin.
+
+Examples:
+
+```bash
+# Summarize text
+csum "AI is transforming industries."
+
+# Summarize with context
+csum --context "in bullet points" "AI is transforming industries."
+
+# Summarize piped content
+cat article.txt | csum --context "one sentence"
+
+# Summarize content from URL
+csum https://example.com
+
+# Summarize a file's content
+csum document.txt
+```
+
+## Using with Langchain
+
+For users integrating with the [Langchain](https://python.langchain.com/) framework, `content-core` exposes a set of compatible tools. These tools, located in the `src/content_core/tools` directory, allow you to leverage `content-core` extraction, cleaning, and summarization capabilities directly within your Langchain agents and chains.
+
+You can import and use these tools like any other Langchain tool. For example:
+
+```python
+from content_core.tools import extract_content_tool, cleanup_content_tool, summarize_content_tool
+from langchain.agents import initialize_agent, AgentType
+
+tools = [extract_content_tool, cleanup_content_tool, summarize_content_tool]
+agent = initialize_agent(tools, llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, verbose=True)
+agent.run("Extract the content from https://example.com and then summarize it.") 
+```
+
+Refer to the source code in `src/content_core/tools` for specific tool implementations and usage details.
+
+## Basic Usage
+
+The core functionality revolves around the extract_content function.
+
+```python
+import asyncio
+from content_core.extraction import extract_content
+
+async def main():
+    # Extract from raw text
+    text_data = await extract_content({"content": "This is my sample text content."})
+    print(text_data)
+
+    # Extract from a URL
+    url_data = await extract_content({"url": "https://www.example.com"})
+    print(url_data)
+
+    # Extract from a local video file (gets transcript)
+    video_data = await extract_content({"file_path": "path/to/your/video.mp4"})
+    print(video_data)
+
+    # Extract from a local markdown file
+    md_data = await extract_content({"file_path": "path/to/your/document.md"})
+    print(md_data)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+(See `src/content_core/notebooks/run.ipynb` for more detailed examples.)
+
+## Configuration
+
+Configuration settings (like API keys for external services, logging levels) can be managed through environment variables or `.env` files, loaded automatically via `python-dotenv`.
+
+Example `.env`:
+
+```plaintext
+OPENAI_API_KEY=your-key-here
+GOOGLE_API_KEY=your-key-here
+```
+
+## Development
+
+To set up a development environment:
+
+```bash
+# Clone the repository
+git clone <repository-url>
+cd content-core
+
+# Create virtual environment and install dependencies
+uv venv
+source .venv/bin/activate
+uv sync --group dev
+
+# Run tests
+make test
+
+# Lint code
+make lint
+
+# See all commands
+make help
+```
+
+## Contributing
+Contributions are welcome! Please follow standard practices (fork, feature branch, pull request).
+
+## License
+This project is licensed under the MIT License - see the LICENSE file for details.

content_core-0.1.0/README.md

@@ -0,0 +1,222 @@
+# Content Core
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+**Content Core** is a versatile Python library designed to extract and process content from various sources, providing a unified interface for handling text, web pages, and local files.
+
+## Overview
+
+The primary goal of Content Core is to simplify the process of ingesting content from diverse origins. Whether you have raw text, a URL pointing to an article, or a local file like a video or markdown document, Content Core aims to extract the meaningful content for further use.
+
+## Key Features
+
+*   **Multi-Source Extraction:** Handles content from:
+    *   Direct text strings.
+    *   Web URLs (using robust extraction methods).
+    *   Local files (including automatic transcription for video/audio files and parsing for text-based formats).
+*   **Intelligent Processing:** Applies appropriate extraction techniques based on the source type.
+*   **Content Cleaning (Optional):** Likely integrates with LLMs (via `prompter.py` and Jinja templates) to refine and clean the extracted content.
+*   **Asynchronous:** Built with `asyncio` for efficient I/O operations.
+
+## Getting Started
+
+### Installation
+
+Install Content Core using `pip`:
+
+```bash
+# Install the package
+pip install content-core
+```
+
+Alternatively, if you’re developing locally:
+
+```bash
+# Clone the repository
+git clone https://github.com/lfnovo/content-core
+cd content-core
+
+# Install with uv
+uv sync
+```
+
+### Command-Line Interface
+
+Content Core provides three CLI commands for extracting, cleaning, and summarizing content: 
+ccore, cclean, and csum. These commands support input from text, URLs, files, or piped data (e.g., via cat file | command).
+
+#### ccore - Extract Content
+
+Extracts content from text, URLs, or files, with optional formatting.
+Usage:
+```bash
+ccore [-f|--format xml|json|text] [-d|--debug] [content]
+```
+Options:
+- `-f`, `--format`: Output format (xml, json, or text). Default: text.
+- `-d`, `--debug`: Enable debug logging.
+- `content`: Input content (text, URL, or file path). If omitted, reads from stdin.
+
+Examples:
+
+```bash
+# Extract from a URL as text
+ccore https://example.com
+
+# Extract from a file as JSON
+ccore -f json document.pdf
+
+# Extract from piped text as XML
+echo "Sample text" | ccore --format xml
+```
+
+#### cclean - Clean Content
+Cleans content by removing unnecessary formatting, spaces, or artifacts. Accepts text, JSON, XML input, URLs, or file paths.
+Usage:
+
+```bash
+cclean [-d|--debug] [content]
+```
+
+Options:
+- `-d`, `--debug`: Enable debug logging.
+- `content`: Input content to clean (text, URL, file path, JSON, or XML). If omitted, reads from stdin.
+
+Examples:
+
+```bash
+# Clean a text string
+cclean "  messy   text   "
+
+# Clean piped JSON
+echo '{"content": "  messy   text   "}' | cclean
+
+# Clean content from a URL
+cclean https://example.com
+
+# Clean a file’s content
+cclean document.txt
+```
+
+### csum - Summarize Content
+
+Summarizes content with an optional context to guide the summary style. Accepts text, JSON, XML input, URLs, or file paths.
+
+Usage:
+
+```bash
+csum [--context "context text"] [-d|--debug] [content]
+```
+
+Options:
+- `--context`: Context for summarization (e.g., "explain to a child"). Default: none.
+- `-d`, `--debug`: Enable debug logging.
+- `content`: Input content to summarize (text, URL, file path, JSON, or XML). If omitted, reads from stdin.
+
+Examples:
+
+```bash
+# Summarize text
+csum "AI is transforming industries."
+
+# Summarize with context
+csum --context "in bullet points" "AI is transforming industries."
+
+# Summarize piped content
+cat article.txt | csum --context "one sentence"
+
+# Summarize content from URL
+csum https://example.com
+
+# Summarize a file's content
+csum document.txt
+```
+
+## Using with Langchain
+
+For users integrating with the [Langchain](https://python.langchain.com/) framework, `content-core` exposes a set of compatible tools. These tools, located in the `src/content_core/tools` directory, allow you to leverage `content-core` extraction, cleaning, and summarization capabilities directly within your Langchain agents and chains.
+
+You can import and use these tools like any other Langchain tool. For example:
+
+```python
+from content_core.tools import extract_content_tool, cleanup_content_tool, summarize_content_tool
+from langchain.agents import initialize_agent, AgentType
+
+tools = [extract_content_tool, cleanup_content_tool, summarize_content_tool]
+agent = initialize_agent(tools, llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, verbose=True)
+agent.run("Extract the content from https://example.com and then summarize it.") 
+```
+
+Refer to the source code in `src/content_core/tools` for specific tool implementations and usage details.
+
+## Basic Usage
+
+The core functionality revolves around the extract_content function.
+
+```python
+import asyncio
+from content_core.extraction import extract_content
+
+async def main():
+    # Extract from raw text
+    text_data = await extract_content({"content": "This is my sample text content."})
+    print(text_data)
+
+    # Extract from a URL
+    url_data = await extract_content({"url": "https://www.example.com"})
+    print(url_data)
+
+    # Extract from a local video file (gets transcript)
+    video_data = await extract_content({"file_path": "path/to/your/video.mp4"})
+    print(video_data)
+
+    # Extract from a local markdown file
+    md_data = await extract_content({"file_path": "path/to/your/document.md"})
+    print(md_data)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+(See `src/content_core/notebooks/run.ipynb` for more detailed examples.)
+
+## Configuration
+
+Configuration settings (like API keys for external services, logging levels) can be managed through environment variables or `.env` files, loaded automatically via `python-dotenv`.
+
+Example `.env`:
+
+```plaintext
+OPENAI_API_KEY=your-key-here
+GOOGLE_API_KEY=your-key-here
+```
+
+## Development
+
+To set up a development environment:
+
+```bash
+# Clone the repository
+git clone <repository-url>
+cd content-core
+
+# Create virtual environment and install dependencies
+uv venv
+source .venv/bin/activate
+uv sync --group dev
+
+# Run tests
+make test
+
+# Lint code
+make lint
+
+# See all commands
+make help
+```
+
+## Contributing
+Contributions are welcome! Please follow standard practices (fork, feature branch, pull request).
+
+## License
+This project is licensed under the MIT License - see the LICENSE file for details.

content_core-0.1.0/prompts/content/cleanup.jinja

@@ -0,0 +1,16 @@
+# GOAL
+
+Adjust the content below to make it clean and readable:
+Remove repeated strings that do not add value to the text.
+
+Remove any content unrelated to the text itself (e.g., metadata, artifacts, or extraction errors).
+
+Format the output as unstructured but clear text.
+
+Do not add extra text, introductions, conclusions, or commentary—only rewrite the provided content as it is.
+
+Do not interpret, analyze, or alter the meaning, intent, or narrative of the text—just reformat it for clarity and readability.
+
+Do not change the text structure, do not write conclusions about it. Your only job is to make it readable. 
+
+Keep the text in its original language, regardless of what it is.

content_core-0.1.0/prompts/content/summarize.jinja

@@ -0,0 +1,25 @@
+You are an AI assistant for a personal study platform.
+
+In this platform, your user collects various articles and content from the Internet for reference and study.
+
+Your role is to summarize the selected content as densely as possible, helping the reader extract maximum value from it without reading the full text.
+Focus solely on the content's value, avoiding unnecessary comments or messages.
+
+The summary should be dense, rich in characters, and designed to create a powerful vector representation.
+If the user provided additional context, follow its instructions. Otherwise, summary the whole content.
+
+Do not return any acknowledgments or greetings—only the summary.
+
+CONTENT:
+
+{{ content }}
+
+{% if context %}
+CONTEXT:
+
+User has provided the aditional context for your task:
+{{context}}
+{% endif%}
+
+
+SUMMARY:

content_core-0.1.0/pyproject.toml

@@ -0,0 +1,60 @@
+[project]
+name = "content-core"
+version = "0.1.0"
+description = "Extract what matters from any media source"
+readme = "README.md"
+authors = [
+    { name = "LUIS NOVO", email = "lfnovo@gmail.com" }
+]
+requires-python = ">=3.10"
+dependencies = [
+    "aiohttp>=3.11.16",
+    "bs4>=0.0.2",
+    "esperanto>=1.2.0",
+    "google-genai>=1.10.0",
+    "jinja2>=3.1.6",
+    "langdetect>=1.0.9",
+    "loguru>=0.7.3",
+    "openai>=1.73.0",
+    "openpyxl>=3.1.5",
+    "pandas>=2.2.3",
+    "pydub>=0.25.1",
+    "pymupdf>=1.25.5",
+    "python-docx>=1.1.2",
+    "python-dotenv>=1.1.0",
+    "python-magic>=0.4.27",
+    "python-pptx>=1.0.2",
+    "youtube-transcript-api>=1.0.3",
+    "langgraph>=0.3.29",
+    "dicttoxml>=1.7.16",
+    "validators>=0.34.0",
+]
+
+[project.scripts]
+ccore = "content_core:ccore"
+cclean = "content_core:cclean"
+csum = "content_core:csum"
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[build-system]
+requires = ["hatchling", "pip"]
+build-backend = "hatchling.build"
+
+[tool.setuptools]
+package-dir = {"content_core" = "src/content_core"}
+
+[dependency-groups]
+dev = [
+    "ipykernel>=4.0.1",
+    "ipywidgets>=4.0.0",
+    "pyperclip>=1.9.0",
+    "pytest>=7.2.0",
+    "pytest-asyncio>=0.21.0",
+]
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"