spiderforce4ai 1.8__py3-none-any.whl → 1.9__py3-none-any.whl

spiderforce4ai/__init__.py

@@ -22,7 +22,7 @@ from multiprocessing import Pool
 
 console = Console()
 
-def extract_metadata_headers(markdown: str) -> str:
+def extract_metadata_headers(markdown: str, url: str = '') -> str:
     """Extract metadata and headers from markdown content with SEO formatting."""
     lines = markdown.split('\n')
     extracted = []
@@ -62,10 +62,10 @@ def extract_metadata_headers(markdown: str) -> str:
                     metadata['language'] = value
     
     # Add formatted metadata section with URL first
-    extracted.append(f"URL: {metadata.get('url', '')}")
-    extracted.append(f"Title: {metadata['title']}")
+    extracted.append(f"URL: {url}")  # Use the actual crawled URL
+    extracted.append(f"Title: {metadata['title'] or url.split('/')[-2].replace('-', ' ').title()}")  # Fallback to URL segment
     extracted.append(f"Description: {metadata['description']}")
-    extracted.append(f"CanonicalUrl: {metadata['canonical_url']}")
+    extracted.append(f"CanonicalUrl: {metadata['canonical_url'] or url}")  # Fallback to crawled URL
     extracted.append(f"Language: {metadata['language'] or 'en'}")  # Default to 'en' if not specified
     extracted.append("")  # Empty line after metadata
     
@@ -247,7 +247,7 @@ def _process_url_parallel(args: Tuple[str, str, CrawlConfig]) -> CrawlResult:
             
             # Handle combined markdown file
             if config.combine_to_one_markdown:
-                content = markdown if config.combine_to_one_markdown == 'full' else extract_metadata_headers(markdown)
+                content = markdown if config.combine_to_one_markdown == 'full' else extract_metadata_headers(markdown, url)
                 combined_content = f"\n----PAGE----\n{url}\n\n{content}\n----PAGE END----\n"
                 
                 with open(config.combined_markdown_file, 'a', encoding='utf-8') as f:

{spiderforce4ai-1.8.dist-info → spiderforce4ai-1.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: spiderforce4ai
-Version: 1.8
+Version: 1.9
 Summary: Python wrapper for SpiderForce4AI HTML-to-Markdown conversion service
 Home-page: https://petertam.pro
 Author: Piotr Tamulewicz

spiderforce4ai-1.9.dist-info/RECORD

@@ -0,0 +1,5 @@
+spiderforce4ai/__init__.py,sha256=JL3APcu08DJWqeVBssPODQ8zqZdislI-qiOah_7xnus,35564
+spiderforce4ai-1.9.dist-info/METADATA,sha256=St2DWVpNEWX22A9x7aizkUtRtTOk8tnva0izcXRNL5o,7183
+spiderforce4ai-1.9.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+spiderforce4ai-1.9.dist-info/top_level.txt,sha256=Kth7A21Js7DCp0j5XBBi-FE45SCLouZkeNZU__Yr9Yk,15
+spiderforce4ai-1.9.dist-info/RECORD,,

spiderforce4ai-1.8.dist-info/RECORD

@@ -1,5 +0,0 @@
-spiderforce4ai/__init__.py,sha256=GaW2zVESi1pi13pD0Dky4g0Yuj9hEj7_4eP_eyoBnWM,35425
-spiderforce4ai-1.8.dist-info/METADATA,sha256=T1K4wWbagvh0ZW_vsYNAAhSAqRH7bLDOF6lr7Yy1pfg,7183
-spiderforce4ai-1.8.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-spiderforce4ai-1.8.dist-info/top_level.txt,sha256=Kth7A21Js7DCp0j5XBBi-FE45SCLouZkeNZU__Yr9Yk,15
-spiderforce4ai-1.8.dist-info/RECORD,,