spiderforce4ai 2.0__py3-none-any.whl → 2.1__py3-none-any.whl

spiderforce4ai/__init__.py

@@ -23,7 +23,7 @@ from multiprocessing import Pool
 console = Console()
 
 def extract_metadata_headers(markdown: str, url: str = '') -> str:
-    """Extract metadata and headers from markdown content with SEO formatting."""
+    """Extract metadata and headers from markdown content with enhanced SEO formatting."""
     lines = markdown.split('\n')
     extracted = []
     in_metadata = False
@@ -33,8 +33,9 @@ def extract_metadata_headers(markdown: str, url: str = '') -> str:
         'canonical_url': '',
         'language': ''
     }
+    first_paragraph = ''
     
-    # First pass - collect metadata
+    # First pass - collect metadata and first paragraph
     for i, line in enumerate(lines):
         # Check for metadata block boundaries
         if line.strip() == '---':
@@ -61,41 +62,34 @@ def extract_metadata_headers(markdown: str, url: str = '') -> str:
                 
                 if key == 'title':
                     metadata['title'] = value
-                elif key == 'description' or key == 'meta_description':
+                elif key in ['description', 'meta_description', 'og:description', 'meta-description']:
                     metadata['description'] = value
-                elif key == 'canonical_url' or key == 'canonical':
+                elif key in ['canonical_url', 'canonical']:
                     metadata['canonical_url'] = value
-                elif key == 'language' or key == 'lang':
+                elif key in ['language', 'lang']:
                     metadata['language'] = value
+        elif not in_metadata and not first_paragraph and line.strip() and not line.startswith('#'):
+            first_paragraph = line.strip()
     
-    # Add formatted metadata section with URL first
-    extracted.append(f"URL: {url}")  # Use the actual crawled URL
-    extracted.append(f"Title: {metadata['title'] or url.split('/')[-2].replace('-', ' ').title()}")  # Fallback to URL segment
+    # Use first paragraph as fallback description if none found
+    if not metadata['description'] and first_paragraph:
+        metadata['description'] = first_paragraph[:160] + ('...' if len(first_paragraph) > 160 else '')
+    
+    # Add formatted metadata section
+    extracted.append(f"URL: {url}")
+    extracted.append(f"Title: {metadata['title'] or url.split('/')[-2].replace('-', ' ').title()}")
     extracted.append(f"Description: {metadata['description']}")
-    extracted.append(f"CanonicalUrl: {metadata['canonical_url'] or url}")  # Fallback to crawled URL
-    extracted.append(f"Language: {metadata['language'] or 'en'}")  # Default to 'en' if not specified
+    extracted.append(f"CanonicalUrl: {metadata['canonical_url'] or url}")
+    extracted.append(f"Language: {metadata['language'] or 'en'}")
     extracted.append("")  # Empty line after metadata
     
     # Second pass - process headers
     for line in lines:
         if line.strip().startswith('#'):
-            # Count the number of # symbols
             level = len(line) - len(line.lstrip('#'))
             text = line.lstrip('#').strip()
-            
-            # Format header according to level
-            if level == 1:
-                extracted.append(f"H1: {text}")
-            elif level == 2:
-                extracted.append(f"H2: {text}")
-            elif level == 3:
-                extracted.append(f"H3: {text}")
-            elif level == 4:
-                extracted.append(f"H4: {text}")
-            elif level == 5:
-                extracted.append(f"H5: {text}")
-            elif level == 6:
-                extracted.append(f"H6: {text}")
+            if 1 <= level <= 6:
+                extracted.append(f"H{level}: {text}")
     
     return '\n'.join(extracted)
 
@@ -317,7 +311,7 @@ class SpiderForce4AI:
 
         # Handle combined markdown file
         if self.config.combine_to_one_markdown:
-            content = markdown if self.config.combine_to_one_markdown == 'full' else extract_metadata_headers(markdown)
+            content = markdown if config.combine_to_one_markdown == 'full' else extract_metadata_headers(markdown, url)
             combined_content = f"\n----PAGE----\n{url}\n\n{content}\n----PAGE END----\n"
             
             async with aiofiles.open(self.config.combined_markdown_file, 'a', encoding='utf-8') as f:

{spiderforce4ai-2.0.dist-info → spiderforce4ai-2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: spiderforce4ai
-Version: 2.0
+Version: 2.1
 Summary: Python wrapper for SpiderForce4AI HTML-to-Markdown conversion service
 Home-page: https://petertam.pro
 Author: Piotr Tamulewicz

spiderforce4ai-2.1.dist-info/RECORD

@@ -0,0 +1,5 @@
+spiderforce4ai/__init__.py,sha256=6WskofS5vOJuPhdwoCRvcOmWVimCKJxtkkP_pshrrlo,35805
+spiderforce4ai-2.1.dist-info/METADATA,sha256=bK_85RBFEAmDTZgo2oCPKgDNd-dqfYvRJoBl92Zk-i8,7183
+spiderforce4ai-2.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+spiderforce4ai-2.1.dist-info/top_level.txt,sha256=Kth7A21Js7DCp0j5XBBi-FE45SCLouZkeNZU__Yr9Yk,15
+spiderforce4ai-2.1.dist-info/RECORD,,

spiderforce4ai-2.0.dist-info/RECORD

@@ -1,5 +0,0 @@
-spiderforce4ai/__init__.py,sha256=CiZBCoRGCfu8587NbW_rtU6kFZEC0R7i_lZwJLesH3M,35975
-spiderforce4ai-2.0.dist-info/METADATA,sha256=nOYUQWRl46UwW3HybozwqcFdc2JPotPLXsEBiL4GuqI,7183
-spiderforce4ai-2.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-spiderforce4ai-2.0.dist-info/top_level.txt,sha256=Kth7A21Js7DCp0j5XBBi-FE45SCLouZkeNZU__Yr9Yk,15
-spiderforce4ai-2.0.dist-info/RECORD,,