content-extraction 0.1.0__tar.gz → 0.3.0__tar.gz

{content_extraction-0.1.0 → content_extraction-0.3.0}/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.4
 Name: content_extraction
-Version: 0.1.0
+Version: 0.3.0
 Summary: Project dedicated to content extraction from unstructured files that contain some useful information.
-Requires-Python: >=3.13
+Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 Requires-Dist: beautifulsoup4>=4.13.4
 Requires-Dist: lxml>=6.0.0

{content_extraction-0.1.0 → content_extraction-0.3.0}/pyproject.toml

@@ -5,18 +5,22 @@ build-backend = "setuptools.build_meta"
 [tool.setuptools.packages.find]
 where = ["src"]
 
+[tool.setuptools.package-data]
+"content_extraction" = ["process_document.sh"]
+
 [project]
 name = "content_extraction"
-version = "0.1.0"
+version = "0.3.0"
 description = "Project dedicated to content extraction from unstructured files that contain some useful information."
 readme = "README.md"
-requires-python = ">=3.13"
+requires-python = ">=3.12"
 dependencies = ["beautifulsoup4>=4.13.4", "lxml>=6.0.0", "python-pptx>=1.0.2"]
 
 [dependency-groups]
 dev = [
     "dspy>=2.6.27",
     "jupyterlab>=4.4.5",
+    "mypy>=1.17.0",
     "pre-commit>=4.2.0",
     "pyright>=1.1.403",
     "pytest>=8.4.1",

{content_extraction-0.1.0 → content_extraction-0.3.0}/src/content_extraction/file_handlers.py

@@ -50,7 +50,7 @@ def process_pdf(file_path: str, output_dir: str):
     """
     logger.info(f'[Processing PDF file] started for: "{file_path}"')
     # This path assumes the script is located at src/scripts/process_document.sh
-    script_path = os.path.join(os.path.dirname(__file__), '..', 'scripts', 'process_document.sh')
+    script_path = os.path.join(os.path.dirname(__file__), 'process_document.sh')
     output_html_path = os.path.join(output_dir, 'index.html')  # Define output_html_path
     logger.debug(f'[Processing PDF file] script path: "{script_path}"; output_html_path: "{output_html_path}"')
 

content_extraction-0.3.0/src/content_extraction/process.py

@@ -0,0 +1,59 @@
+import argparse
+import sys
+import logging
+
+from content_extraction.logging_config import setup_logging
+from content_extraction.file_handlers import process_file
+
+
+logger = logging.getLogger(__name__)
+
+
+def main():
+    """
+    Main function to process the input file/URL and generate structured output.
+    """
+    setup_logging()
+    parser = argparse.ArgumentParser(
+        description='Process various document types (PDF, PPTX, DOCX, MD, HTML, URL) and extract content into a structured HTML format.',
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Run from the project root directory (`content-extraction/`)
+
+  # Process a local PDF file
+  python src/process.py my_document.pdf -o ./output_folder
+
+  # Process a PowerPoint file silently
+  python src/process.py my_slides.pptx --silent
+
+  # Process a remote URL, letting the script determine the type
+  python src/process.py https://example.com/document.pdf
+
+  # Process a remote URL, forcing the type to be treated as HTML
+  python src/process.py https://example.com/some-page --force-ext html
+""",
+    )
+
+    parser.add_argument('input_path', help='Path to the input file or a URL to process.')
+    parser.add_argument(
+        '-o',
+        '--output',
+        default='output',
+        help="Path to the output directory (defaults to 'output').",
+    )
+    parser.add_argument(
+        '--force-ext',
+        default=None,
+        help="Force the handler for a specific file extension (e.g., 'pdf', 'pptx') when auto-detection is ambiguous or incorrect.",
+    )
+
+    args = parser.parse_args()
+
+    logger.info(f'Processing file: {args.input_path}')
+    process_file(args.input_path, args.output, args.force_ext)
+    logger.info('Processing complete.')
+
+
+if __name__ == '__main__':
+    sys.exit(main())

content_extraction-0.3.0/src/content_extraction/process_document.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Fail immediately if any command fails
+set -e
+
+# Check if the correct number of arguments is provided
+if [ "$#" -ne 2 ]; then
+  echo "Usage: $0 <input_file_path> <output_directory>"
+  exit 1
+fi
+
+INPUT_FILE="$1"
+OUTPUT_DIR="$2"
+
+# Step 1: Perform OCR and save the result to the directory
+echo "Performing OCR on $INPUT_FILE..."
+python -m content_extraction.do_ocr "$INPUT_FILE" -o "$OUTPUT_DIR"
+
+# Step 2: Combine the OCR pages into a single file
+echo "Combining pages into a single Markdown file..."
+cd "$OUTPUT_DIR"
+ls page-*.md | sort | xargs -I{} sh -c 'cat "{}"; echo; echo' > combined.md
+
+# Step 3: Extract headings from the combined Markdown file
+echo "Extracting headings from combined.md..."
+grep "^#" combined.md > headings.md
+
+# Step 4: Fix any OCR errors using provided script
+echo "Fixing OCR errors..."
+fixed_text=$(python -m content_extraction.fix_ocr combined.md headings.md)
+echo "$fixed_text" > fixed.md
+
+# Step 5: Render the markdown file to HTML
+pandoc fixed.md -s -f markdown -t html -o index.html
+
+echo "All processes completed successfully. Output saved in $OUTPUT_DIR"

{content_extraction-0.1.0 → content_extraction-0.3.0}/src/content_extraction.egg-info/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.4
 Name: content_extraction
-Version: 0.1.0
+Version: 0.3.0
 Summary: Project dedicated to content extraction from unstructured files that contain some useful information.
-Requires-Python: >=3.13
+Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 Requires-Dist: beautifulsoup4>=4.13.4
 Requires-Dist: lxml>=6.0.0

{content_extraction-0.1.0 → content_extraction-0.3.0}/src/content_extraction.egg-info/SOURCES.txt

@@ -9,6 +9,8 @@ src/content_extraction/file_handlers.py
 src/content_extraction/fix_ocr.py
 src/content_extraction/logging_config.py
 src/content_extraction/parse_html.py
+src/content_extraction/process.py
+src/content_extraction/process_document.sh
 src/content_extraction/semantic_chunk_html.py
 src/content_extraction/split_and_create_digest.py
 src/content_extraction.egg-info/PKG-INFO

{content_extraction-0.1.0 → content_extraction-0.3.0}/src/content_extraction.egg-info/top_level.txt

@@ -1,2 +1 @@
 content_extraction
-scripts