spiderforce4ai 2.1__tar.gz → 2.3.1__tar.gz

{spiderforce4ai-2.1 → spiderforce4ai-2.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: spiderforce4ai
-Version: 2.1
+Version: 2.3.1
 Summary: Python wrapper for SpiderForce4AI HTML-to-Markdown conversion service
 Home-page: https://petertam.pro
 Author: Piotr Tamulewicz

{spiderforce4ai-2.1 → spiderforce4ai-2.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "spiderforce4ai"
-version = "2.1"
+version = "2.3.1"
 description = "Python wrapper for SpiderForce4AI HTML-to-Markdown conversion service"
 readme = "README.md"
 authors = [{name = "Piotr Tamulewicz", email = "pt@petertam.pro"}]

{spiderforce4ai-2.1 → spiderforce4ai-2.3.1}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="spiderforce4ai",
-    version="2.1",
+    version="2.3.1",
     author="Piotr Tamulewicz",
     author_email="pt@petertam.pro",
     description="Python wrapper for SpiderForce4AI HTML-to-Markdown conversion service",

{spiderforce4ai-2.1 → spiderforce4ai-2.3.1}/spiderforce4ai/__init__.py

@@ -23,76 +23,55 @@ from multiprocessing import Pool
 console = Console()
 
 def extract_metadata_headers(markdown: str, url: str = '') -> str:
-    """Extract metadata and headers from markdown content with enhanced SEO formatting."""
+    """Extract metadata and headers from markdown content."""
     lines = markdown.split('\n')
-    extracted = []
-    in_metadata = False
-    metadata = {
-        'title': '',
-        'description': '',
-        'canonical_url': '',
-        'language': ''
-    }
-    first_paragraph = ''
+    metadata = {}
+    headers = []
     
-    # First pass - collect metadata and first paragraph
-    for i, line in enumerate(lines):
-        # Check for metadata block boundaries
-        if line.strip() == '---':
-            if not in_metadata:
-                in_metadata = True
-                continue
-            else:
-                in_metadata = False
-                break
+    def parse_metadata_line(line):
+        """Parse a single metadata line correctly."""
+        first_colon = line.find(':')
+        if first_colon == -1:
+            return None, None
+            
+        key = line[:first_colon].strip()
+        value = line[first_colon + 1:].strip()
         
-        # Extract metadata within the block
-        if in_metadata:
-            if ':' in line:
-                key, value = [part.strip() for part in line.split(':', 1)]
-                key = key.lower()
-                
-                # Handle multi-line values
-                if value.startswith('>'):
-                    value = value[1:].strip()
-                    j = i + 1
-                    while j < len(lines) and lines[j].strip() and not lines[j].strip() == '---':
-                        value += ' ' + lines[j].strip()
-                        j += 1
-                
-                if key == 'title':
-                    metadata['title'] = value
-                elif key in ['description', 'meta_description', 'og:description', 'meta-description']:
-                    metadata['description'] = value
-                elif key in ['canonical_url', 'canonical']:
-                    metadata['canonical_url'] = value
-                elif key in ['language', 'lang']:
-                    metadata['language'] = value
-        elif not in_metadata and not first_paragraph and line.strip() and not line.startswith('#'):
-            first_paragraph = line.strip()
-    
-    # Use first paragraph as fallback description if none found
-    if not metadata['description'] and first_paragraph:
-        metadata['description'] = first_paragraph[:160] + ('...' if len(first_paragraph) > 160 else '')
-    
-    # Add formatted metadata section
-    extracted.append(f"URL: {url}")
-    extracted.append(f"Title: {metadata['title'] or url.split('/')[-2].replace('-', ' ').title()}")
-    extracted.append(f"Description: {metadata['description']}")
-    extracted.append(f"CanonicalUrl: {metadata['canonical_url'] or url}")
-    extracted.append(f"Language: {metadata['language'] or 'en'}")
-    extracted.append("")  # Empty line after metadata
+        # Handle the case where value starts with "URL:" - this means it's a missing description
+        if value.startswith('URL:'):
+            return key, ''
+            
+        return key, value
     
-    # Second pass - process headers
+    # Process each line
     for line in lines:
-        if line.strip().startswith('#'):
+        line = line.strip()
+        if not line:
+            continue
+            
+        # Check if it's a metadata line (contains : but isn't a header)
+        if ':' in line and not line.startswith('#'):
+            key, value = parse_metadata_line(line)
+            if key:
+                metadata[key] = value
+        # Check if it's a header
+        elif line.startswith('#'):
             level = len(line) - len(line.lstrip('#'))
             text = line.lstrip('#').strip()
             if 1 <= level <= 6:
-                extracted.append(f"H{level}: {text}")
+                headers.append(f"H{level}: {text}")
     
-    return '\n'.join(extracted)
-
+    # Construct output
+    output = []
+    output.append(f"URL: {url}")
+    output.append(f"Title: {metadata.get('Title', url.split('/')[-2].replace('-', ' ').title())}")
+    output.append(f"Description: {metadata.get('Description', '')}")  # Now this will be empty string for missing descriptions
+    output.append(f"CanonicalUrl: {metadata.get('CanonicalUrl', url)}")
+    output.append(f"Language: {metadata.get('Language', 'en')}")
+    output.append("")  # Empty line
+    output.extend(headers)
+    
+    return '\n'.join(output)
 def slugify(url: str) -> str:
     """Convert URL to a valid filename."""
     parsed = urlparse(url)

{spiderforce4ai-2.1 → spiderforce4ai-2.3.1}/spiderforce4ai.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: spiderforce4ai
-Version: 2.1
+Version: 2.3.1
 Summary: Python wrapper for SpiderForce4AI HTML-to-Markdown conversion service
 Home-page: https://petertam.pro
 Author: Piotr Tamulewicz