content-core 0.3.1__tar.gz → 0.4.0__tar.gz

{content_core-0.3.1 → content_core-0.4.0}/.windsurfrules

@@ -4,10 +4,10 @@ All documentation (code or readmes) must be in english.
 Whenever I ask you to tag and release, make sure to run `make test` as part of the process. 
 
 The full release process is: 
-- Run `make test` to make sure everything is working
+- Run `make test` to make sure everything is working (if we changed any code or import)
 - Update version on pyproject.toml
 - Run `uv sync` to update the lock file
 - Commit all that's needed
-- Merge to main
+- Merge to main (if in a branch)
 - Tag the release
 - Push to GitHub

{content_core-0.3.1 → content_core-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: content-core
-Version: 0.3.1
+Version: 0.4.0
 Summary: Extract what matters from any media source
 Author-email: LUIS NOVO <lfnovo@gmail.com>
 License-File: LICENSE

{content_core-0.3.1 → content_core-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "content-core"
-version = "0.3.1"
+version = "0.4.0"
 description = "Extract what matters from any media source"
 readme = "README.md"
 homepage = "https://github.com/lfnovo/content-core"

{content_core-0.3.1 → content_core-0.4.0}/src/content_core/content/extraction/graph.py

@@ -21,6 +21,10 @@ from content_core.processors.url import extract_url, url_provider
 from content_core.processors.video import extract_best_audio_from_video
 from content_core.processors.youtube import extract_youtube_transcript
 
+import aiohttp
+import tempfile
+from urllib.parse import urlparse
+
 
 async def source_identification(state: ProcessSourceState) -> Dict[str, str]:
     """
@@ -91,6 +95,21 @@ async def source_type_router(x: ProcessSourceState) -> Optional[str]:
     return x.source_type
 
 
+async def download_remote_file(state: ProcessSourceState) -> Dict[str, Any]:
+    url = state.url
+    assert url, "No URL provided"
+    async with aiohttp.ClientSession() as session:
+        async with session.get(url) as resp:
+            resp.raise_for_status()
+            mime = resp.headers.get("content-type", "").split(";", 1)[0]
+            suffix = os.path.splitext(urlparse(url).path)[1] if urlparse(url).path else ""
+            fd, tmp = tempfile.mkstemp(suffix=suffix)
+            os.close(fd)
+            with open(tmp, "wb") as f:
+                f.write(await resp.read())
+    return {"file_path": tmp, "identified_type": mime}
+
+
 # Create workflow
 workflow = StateGraph(
     ProcessSourceState, input=ProcessSourceInput, output=ProcessSourceState
@@ -108,6 +127,7 @@ workflow.add_node("extract_best_audio_from_video", extract_best_audio_from_video
 workflow.add_node("extract_audio", extract_audio)
 workflow.add_node("extract_youtube_transcript", extract_youtube_transcript)
 workflow.add_node("delete_file", delete_file)
+workflow.add_node("download_remote_file", download_remote_file)
 
 # Add edges
 workflow.add_edge(START, "source")
@@ -127,7 +147,7 @@ workflow.add_conditional_edges(
 workflow.add_conditional_edges(
     "url_provider",
     url_type_router,
-    {"article": "extract_url", "youtube": "extract_youtube_transcript"},
+    {**{m: "download_remote_file" for m in SUPPORTED_FITZ_TYPES}, "article": "extract_url", "youtube": "extract_youtube_transcript"},
 )
 workflow.add_edge("url_provider", END)
 workflow.add_edge("file_type", END)
@@ -140,6 +160,7 @@ workflow.add_edge("extract_office_content", "delete_file")
 workflow.add_edge("extract_best_audio_from_video", "extract_audio")
 workflow.add_edge("extract_audio", "delete_file")
 workflow.add_edge("delete_file", END)
+workflow.add_edge("download_remote_file", "file_type")
 
 # Compile graph
 graph = workflow.compile()

{content_core-0.3.1 → content_core-0.4.0}/src/content_core/processors/url.py

@@ -6,6 +6,7 @@ from bs4 import BeautifulSoup, Comment
 
 from content_core.common import ProcessSourceState
 from content_core.logging import logger
+from content_core.processors.pdf import SUPPORTED_FITZ_TYPES
 
 # future: better extraction methods
 # https://github.com/buriy/python-readability
@@ -20,12 +21,20 @@ async def url_provider(state: ProcessSourceState):
     url = state.url
     if url:
         if "youtube.com" in url or "youtu.be" in url:
-            return_dict["identified_type"] = (
-                "youtube"  # future: playlists, channels in the future
-            )
+            return_dict["identified_type"] = "youtube"
         else:
-            return_dict["identified_type"] = "article"
-            # future: article providers in the future
+            # remote URL: check content-type to catch PDFs
+            try:
+                async with aiohttp.ClientSession() as session:
+                    async with session.head(url, timeout=10, allow_redirects=True) as resp:
+                        mime = resp.headers.get("content-type", "").split(";", 1)[0]
+            except Exception as e:
+                logger.debug(f"HEAD check failed for {url}: {e}")
+                mime = "article"
+            if mime in SUPPORTED_FITZ_TYPES:
+                return_dict["identified_type"] = mime
+            else:
+                return_dict["identified_type"] = "article"
     return return_dict
 
 

{content_core-0.3.1 → content_core-0.4.0}/tests/integration/test_extraction.py

@@ -200,3 +200,13 @@ async def test_extract_content_from_xlsx(fixture_path):
     )
     assert result.title is not None  # Attempt to extract title/metadata
     assert len(result.content) > 0  # Check that some content was extracted
+
+
+@pytest.mark.asyncio
+async def test_extract_content_from_pdf_url():
+    """Tests extracting content from a remote PDF URL."""
+    url = "https://arxiv.org/pdf/2408.09869"
+    result = await extract_content({"url": url})
+    assert result.source_type == "url"
+    assert result.identified_type == "application/pdf"
+    assert len(result.content) > 100  # Expect substantial extracted text

{content_core-0.3.1 → content_core-0.4.0}/uv.lock

@@ -354,7 +354,7 @@ wheels = [
 
 [[package]]
 name = "content-core"
-version = "0.3.1"
+version = "0.4.0"
 source = { editable = "." }
 dependencies = [
     { name = "aiohttp" },
@@ -1018,7 +1018,7 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/8c/de/8eb6fffecd9c5f129461edcdd7e1ac944f9de15783e3d89c84ed6e0374bc/lxml-5.3.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:aa837e6ee9534de8d63bc4c1249e83882a7ac22bd24523f83fad68e6ffdf41ae", size = 5652903 },
     { url = "https://files.pythonhosted.org/packages/95/79/80f4102a08495c100014593680f3f0f7bd7c1333b13520aed855fc993326/lxml-5.3.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:da4c9223319400b97a2acdfb10926b807e51b69eb7eb80aad4942c0516934858", size = 5491813 },
     { url = "https://files.pythonhosted.org/packages/15/f5/9b1f7edf6565ee31e4300edb1bcc61eaebe50a3cff4053c0206d8dc772f2/lxml-5.3.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:dc0e9bdb3aa4d1de703a437576007d366b54f52c9897cae1a3716bb44fc1fc85", size = 5227837 },
-    { url = "https://files.pythonhosted.org/packages/5c/17/c31d94364c02e3492215658917f5590c00edce8074aeb06d05b7771465d9/lxml-5.3.2-cp310-cp310-win32.whl", hash = "sha256:5f94909a1022c8ea12711db7e08752ca7cf83e5b57a87b59e8a583c5f35016ad", size = 3477533 },
+    { url = "https://files.pythonhosted.org/packages/dd/53/a187c4ccfcd5fbfca01e6c96da39499d8b801ab5dcf57717db95d7a968a8/lxml-5.3.2-cp310-cp310-win32.win32.whl", hash = "sha256:dd755a0a78dd0b2c43f972e7b51a43be518ebc130c9f1a7c4480cf08b4385486", size = 3477533 },
     { url = "https://files.pythonhosted.org/packages/f2/2c/397c5a9d76a7a0faf9e5b13143ae1a7e223e71d2197a45da71c21aacb3d4/lxml-5.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:d64ea1686474074b38da13ae218d9fde0d1dc6525266976808f41ac98d9d7980", size = 3805160 },
     { url = "https://files.pythonhosted.org/packages/84/b8/2b727f5a90902f7cc5548349f563b60911ca05f3b92e35dfa751349f265f/lxml-5.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9d61a7d0d208ace43986a92b111e035881c4ed45b1f5b7a270070acae8b0bfb4", size = 8163457 },
     { url = "https://files.pythonhosted.org/packages/91/84/23135b2dc72b3440d68c8f39ace2bb00fe78e3a2255f7c74f7e76f22498e/lxml-5.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:856dfd7eda0b75c29ac80a31a6411ca12209183e866c33faf46e77ace3ce8a79", size = 4433445 },