spiderforce4ai 1.8__tar.gz → 2.0__tar.gz

{spiderforce4ai-1.8 → spiderforce4ai-2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: spiderforce4ai
-Version: 1.8
+Version: 2.0
 Summary: Python wrapper for SpiderForce4AI HTML-to-Markdown conversion service
 Home-page: https://petertam.pro
 Author: Piotr Tamulewicz

{spiderforce4ai-1.8 → spiderforce4ai-2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "spiderforce4ai"
-version = "1.8"
+version = "2.0"
 description = "Python wrapper for SpiderForce4AI HTML-to-Markdown conversion service"
 readme = "README.md"
 authors = [{name = "Piotr Tamulewicz", email = "pt@petertam.pro"}]

{spiderforce4ai-1.8 → spiderforce4ai-2.0}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="spiderforce4ai",
-    version="1.8",
+    version="2.0",
     author="Piotr Tamulewicz",
     author_email="pt@petertam.pro",
     description="Python wrapper for SpiderForce4AI HTML-to-Markdown conversion service",

{spiderforce4ai-1.8 → spiderforce4ai-2.0}/spiderforce4ai/__init__.py

@@ -22,7 +22,7 @@ from multiprocessing import Pool
 
 console = Console()
 
-def extract_metadata_headers(markdown: str) -> str:
+def extract_metadata_headers(markdown: str, url: str = '') -> str:
     """Extract metadata and headers from markdown content with SEO formatting."""
     lines = markdown.split('\n')
     extracted = []
@@ -48,24 +48,31 @@ def extract_metadata_headers(markdown: str) -> str:
         # Extract metadata within the block
         if in_metadata:
             if ':' in line:
-                key, value = line.split(':', 1)
-                key = key.strip().lower()
-                value = value.strip()
+                key, value = [part.strip() for part in line.split(':', 1)]
+                key = key.lower()
+                
+                # Handle multi-line values
+                if value.startswith('>'):
+                    value = value[1:].strip()
+                    j = i + 1
+                    while j < len(lines) and lines[j].strip() and not lines[j].strip() == '---':
+                        value += ' ' + lines[j].strip()
+                        j += 1
                 
                 if key == 'title':
                     metadata['title'] = value
-                elif key == 'description':
+                elif key == 'description' or key == 'meta_description':
                     metadata['description'] = value
-                elif key == 'canonical_url':
+                elif key == 'canonical_url' or key == 'canonical':
                     metadata['canonical_url'] = value
-                elif key == 'language':
+                elif key == 'language' or key == 'lang':
                     metadata['language'] = value
     
     # Add formatted metadata section with URL first
-    extracted.append(f"URL: {metadata.get('url', '')}")
-    extracted.append(f"Title: {metadata['title']}")
+    extracted.append(f"URL: {url}")  # Use the actual crawled URL
+    extracted.append(f"Title: {metadata['title'] or url.split('/')[-2].replace('-', ' ').title()}")  # Fallback to URL segment
     extracted.append(f"Description: {metadata['description']}")
-    extracted.append(f"CanonicalUrl: {metadata['canonical_url']}")
+    extracted.append(f"CanonicalUrl: {metadata['canonical_url'] or url}")  # Fallback to crawled URL
     extracted.append(f"Language: {metadata['language'] or 'en'}")  # Default to 'en' if not specified
     extracted.append("")  # Empty line after metadata
     
@@ -247,7 +254,7 @@ def _process_url_parallel(args: Tuple[str, str, CrawlConfig]) -> CrawlResult:
             
             # Handle combined markdown file
             if config.combine_to_one_markdown:
-                content = markdown if config.combine_to_one_markdown == 'full' else extract_metadata_headers(markdown)
+                content = markdown if config.combine_to_one_markdown == 'full' else extract_metadata_headers(markdown, url)
                 combined_content = f"\n----PAGE----\n{url}\n\n{content}\n----PAGE END----\n"
                 
                 with open(config.combined_markdown_file, 'a', encoding='utf-8') as f:

{spiderforce4ai-1.8 → spiderforce4ai-2.0}/spiderforce4ai.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: spiderforce4ai
-Version: 1.8
+Version: 2.0
 Summary: Python wrapper for SpiderForce4AI HTML-to-Markdown conversion service
 Home-page: https://petertam.pro
 Author: Piotr Tamulewicz