spiderforce4ai 1.6__tar.gz → 1.8__tar.gz

{spiderforce4ai-1.6 → spiderforce4ai-1.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: spiderforce4ai
-Version: 1.6
+Version: 1.8
 Summary: Python wrapper for SpiderForce4AI HTML-to-Markdown conversion service
 Home-page: https://petertam.pro
 Author: Piotr Tamulewicz

{spiderforce4ai-1.6 → spiderforce4ai-1.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "spiderforce4ai"
-version = "1.6"
+version = "1.8"
 description = "Python wrapper for SpiderForce4AI HTML-to-Markdown conversion service"
 readme = "README.md"
 authors = [{name = "Piotr Tamulewicz", email = "pt@petertam.pro"}]

{spiderforce4ai-1.6 → spiderforce4ai-1.8}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="spiderforce4ai",
-    version="1.6",
+    version="1.8",
     author="Piotr Tamulewicz",
     author_email="pt@petertam.pro",
     description="Python wrapper for SpiderForce4AI HTML-to-Markdown conversion service",

{spiderforce4ai-1.6 → spiderforce4ai-1.8}/spiderforce4ai/__init__.py

@@ -23,26 +23,72 @@ from multiprocessing import Pool
 console = Console()
 
 def extract_metadata_headers(markdown: str) -> str:
-    """Extract metadata and headers from markdown content."""
+    """Extract metadata and headers from markdown content with SEO formatting."""
     lines = markdown.split('\n')
     extracted = []
     in_metadata = False
+    metadata = {
+        'title': '',
+        'description': '',
+        'canonical_url': '',
+        'language': ''
+    }
     
-    for line in lines:
-        # Check for metadata block
+    # First pass - collect metadata
+    for i, line in enumerate(lines):
+        # Check for metadata block boundaries
         if line.strip() == '---':
-            in_metadata = not in_metadata
-            extracted.append(line)
-            continue
+            if not in_metadata:
+                in_metadata = True
+                continue
+            else:
+                in_metadata = False
+                break
         
-        # Include metadata
+        # Extract metadata within the block
         if in_metadata:
-            extracted.append(line)
-            continue
-            
-        # Include headers (lines starting with #)
+            if ':' in line:
+                key, value = line.split(':', 1)
+                key = key.strip().lower()
+                value = value.strip()
+                
+                if key == 'title':
+                    metadata['title'] = value
+                elif key == 'description':
+                    metadata['description'] = value
+                elif key == 'canonical_url':
+                    metadata['canonical_url'] = value
+                elif key == 'language':
+                    metadata['language'] = value
+    
+    # Add formatted metadata section with URL first
+    extracted.append(f"URL: {metadata.get('url', '')}")
+    extracted.append(f"Title: {metadata['title']}")
+    extracted.append(f"Description: {metadata['description']}")
+    extracted.append(f"CanonicalUrl: {metadata['canonical_url']}")
+    extracted.append(f"Language: {metadata['language'] or 'en'}")  # Default to 'en' if not specified
+    extracted.append("")  # Empty line after metadata
+    
+    # Second pass - process headers
+    for line in lines:
         if line.strip().startswith('#'):
-            extracted.append(line)
+            # Count the number of # symbols
+            level = len(line) - len(line.lstrip('#'))
+            text = line.lstrip('#').strip()
+            
+            # Format header according to level
+            if level == 1:
+                extracted.append(f"H1: {text}")
+            elif level == 2:
+                extracted.append(f"H2: {text}")
+            elif level == 3:
+                extracted.append(f"H3: {text}")
+            elif level == 4:
+                extracted.append(f"H4: {text}")
+            elif level == 5:
+                extracted.append(f"H5: {text}")
+            elif level == 6:
+                extracted.append(f"H6: {text}")
     
     return '\n'.join(extracted)
 

{spiderforce4ai-1.6 → spiderforce4ai-1.8}/spiderforce4ai.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: spiderforce4ai
-Version: 1.6
+Version: 1.8
 Summary: Python wrapper for SpiderForce4AI HTML-to-Markdown conversion service
 Home-page: https://petertam.pro
 Author: Piotr Tamulewicz