givememd 0.1.0__py3-none-any.whl

givememd/__init__.py

@@ -0,0 +1,39 @@
+import re
+
+from .extract import convert
+from .fetch import TIERS, UnsafeURL, fetch
+
+
+def read(target, links=True, tiers=None):
+    """URL -> (markdown, Result). Raises UnsafeURL for private/invalid targets."""
+    r = fetch(target, tiers=tiers)
+    if r.partial:
+        return convert(r.body, r.url, links=False, meta_only=True), r
+    if not r.ok:
+        return None, r
+    if 'markdown' in r.content_type:
+        return clean_site_markdown(r.body), r
+    md = convert(r.body, r.url, links=links)
+    later = [n for n, _ in TIERS[[n for n, _ in TIERS].index(r.tier) + 1:]] if r.tier in [n for n, _ in TIERS] else []
+    if later and len(body_text(md)) < 200 and r.body.count('<script') >= 3:
+        # the page "loaded" but its content is rendered by JavaScript (streamed payloads, app shells): render it
+        r2 = fetch(target, tiers=later, use_cache=False)
+        if r2.ok and 'markdown' not in r2.content_type:
+            md2 = convert(r2.body, r2.url, links=links)
+            if len(body_text(md2)) > len(body_text(md)):
+                r2.trace = r.trace + ['extraction was empty; rendered'] + r2.trace
+                return md2, r2
+    return md, r
+
+
+def clean_site_markdown(md):
+    """Site-served markdown is often MDX: drop heading anchors and bare component tags, keep their content."""
+    md = re.sub(r'\s*\{/\*.*?\*/\}', '', md)
+    md = re.sub(r'(?m)^[ \t]*</?[A-Z][\w.]*(\s[^>]*)?/?>[ \t]*\n', '', md)
+    return re.sub(r'\n{3,}', '\n\n', md)
+
+
+def body_text(md):
+    """Text after the frontmatter, without markdown punctuation."""
+    parts = md.split('---', 2)
+    return re.sub(r'[#*_`>|\-\s]+', ' ', parts[2] if len(parts) == 3 else md).strip()

givememd/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

givememd/cli.py

@@ -0,0 +1,23 @@
+"""givememd <url> [--no-links] [--trace]   -- print LLM-ready markdown for a web page (fetched from this machine)."""
+import sys
+
+from . import UnsafeURL, read
+
+
+def main():
+    args = [a for a in sys.argv[1:] if not a.startswith('--')]
+    if not args or '--help' in sys.argv or '-h' in sys.argv:
+        print(__doc__.strip()); sys.exit(0 if args or '--help' in sys.argv or '-h' in sys.argv else 2)
+    try:
+        md, r = read(args[0], links='--no-links' not in sys.argv)
+    except UnsafeURL as e:
+        sys.exit(f'refused: {e}')
+    if '--trace' in sys.argv:
+        print('\n'.join(r.trace), file=sys.stderr)
+    if md is None:
+        sys.exit(f'could not read {r.url}')
+    print(md)
+
+
+if __name__ == '__main__':
+    main()

givememd/components.py

@@ -0,0 +1,189 @@
+"""Component catalog: every section of a web page is matched to one of these, filled, and rendered.
+
+Each component has a category (first-stage choice), a short description (the decision model's criterion),
+a shape (how it is filled from the DOM and rendered), the fields it carries, and the JSON-LD types that can
+fill it directly. Shapes:
+
+  header   title + one line of facts          kv       key: value pairs
+  bullets  list of short items                prose    paragraphs
+  steps    numbered list                      qa       question / answer pairs
+  table    rows and columns                   cards    repeated items -> one table
+  code     code with surrounding text         links    titled links (rendered only if asked)
+  drop     not content; never rendered
+"""
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True)
+class Component:
+    name: str
+    category: str
+    desc: str
+    shape: str
+    fields: tuple = ()
+    ld: tuple = ()               # JSON-LD @types that fill this component
+    heading: str = ''            # section heading when rendered ('' = the section's own heading)
+
+
+def C(name, category, desc, shape, fields='', ld='', heading=''):
+    return Component(name, category, desc, shape, tuple(fields.split()), tuple(ld.split()), heading)
+
+
+CATEGORIES = {
+    'commerce': 'a product for sale: its name, price, options, specs, shipping, reviews',
+    'listings': 'many items listed together: search results, category pages, homes, jobs, events, people',
+    'article': 'news, blog, essay or story content: headline, byline, body text, quotes, images',
+    'docs': 'technical documentation, API references, code, tutorials, changelogs',
+    'community': 'forum threads, questions and answers, comments, social posts',
+    'profile': 'a person, company, place or channel: name, bio, stats, contact',
+    'media': 'video, podcast, music, film, book or game details',
+    'events_travel': 'events, tickets, hotels, flights, rentals, itineraries',
+    'food_health': 'recipes, nutrition, menus, medical or fitness information',
+    'data_finance': 'tables, statistics, prices, charts, financial quotes, comparisons',
+    'business': 'company marketing: features, pricing plans, testimonials, FAQs, calls to action',
+    'reference': 'encyclopedia or knowledge content: infobox, definitions, timelines, citations',
+    'chrome': 'not content: navigation, footers, ads, cookie banners, login prompts, UI text',
+}
+
+CATALOG = [
+    # ---- commerce
+    C('product_header', 'commerce', 'main product name, brand, price, rating', 'header', 'name brand price list_price rating rating_count availability', 'Product'),
+    C('price_offer', 'commerce', 'price, discount, sellers, deal or offer details', 'kv', 'price list_price discount seller condition', 'Offer AggregateOffer'),
+    C('variant_picker', 'commerce', 'color, size, style or configuration options', 'bullets', 'option values', heading='Options'),
+    C('spec_table', 'commerce', 'product specifications as key/value pairs', 'kv', 'specs', heading='Specs'),
+    C('feature_bullets', 'commerce', 'bullet list of product features or highlights', 'bullets', 'items', heading='Features'),
+    C('product_description', 'commerce', 'paragraphs describing the product', 'prose', 'text', heading='Description'),
+    C('whats_in_box', 'commerce', "list of what's included in the box", 'bullets', 'items', heading="What's in the box"),
+    C('shipping_returns', 'commerce', 'delivery, shipping, returns or warranty terms', 'kv', 'shipping returns warranty', heading='Shipping & returns'),
+    C('stock_status', 'commerce', 'in stock, sold out, quantity left, bought recently', 'kv', 'availability'),
+    C('review_summary', 'commerce', 'average rating, rating histogram, review topics', 'kv', 'rating count distribution topics', 'AggregateRating', 'Reviews'),
+    C('review_item', 'commerce', 'one customer review: author, stars, date, text', 'prose', 'author rating date title text', 'Review', 'Review'),
+    C('size_guide', 'commerce', 'size chart or fit guidance', 'table', 'rows', heading='Size guide'),
+    C('product_comparison', 'commerce', 'table comparing several products side by side', 'table', 'rows', heading='Comparison'),
+    C('bundle_offer', 'commerce', 'frequently bought together or bundle deal', 'links', 'items'),
+    C('seller_info', 'commerce', 'seller or store name, ratings, policies', 'kv', 'name rating', 'Store', 'Seller'),
+    # ---- listings
+    C('listing_grid', 'listings', 'grid or list of many products, homes, results', 'cards', 'items', 'ItemList', 'Results'),
+    C('listing_card', 'listings', 'one item in a listing: title, price, key facts', 'kv', 'title price facts'),
+    C('result_count', 'listings', 'number of results, sort order, page info', 'kv', 'count sort'),
+    C('filter_panel', 'listings', 'search filters and facets', 'drop'),
+    C('pagination', 'listings', 'page numbers, next/previous links', 'drop'),
+    C('real_estate_listing', 'listings', 'one home or rental: price, beds, baths, area, address', 'kv', 'address price beds baths area status', 'SingleFamilyResidence Apartment House Residence RealEstateListing'),
+    C('job_posting', 'listings', 'job title, company, location, salary, description', 'kv', 'title company location salary type posted', 'JobPosting'),
+    C('search_box', 'listings', 'search input and suggestions', 'drop'),
+    C('map_widget', 'listings', 'map with pins or location picker', 'drop'),
+    # ---- article
+    C('article_header', 'article', 'headline, byline, date, standfirst of an article', 'header', 'headline author date standfirst', 'NewsArticle Article BlogPosting Report'),
+    C('article_body', 'article', 'paragraphs of article or blog text', 'prose', 'text'),
+    C('subheading_section', 'article', 'subheading followed by its paragraphs', 'prose', 'text'),
+    C('pull_quote', 'article', 'highlighted quotation from the article', 'prose', 'text'),
+    C('image_caption', 'article', 'caption or credit for a photo or figure', 'prose', 'text'),
+    C('author_bio', 'article', 'short biography of the author', 'prose', 'text', heading='About the author'),
+    C('key_takeaways', 'article', 'summary bullets, TL;DR, key points', 'bullets', 'items', heading='Key points'),
+    C('table_of_contents', 'article', 'list of sections in this page', 'drop'),
+    C('related_articles', 'article', 'links to other stories: most viewed, read next, related', 'links', 'items'),
+    C('newsletter_signup', 'article', 'subscribe to a newsletter prompt', 'drop'),
+    C('correction_note', 'article', 'correction, update or editor note', 'prose', 'text'),
+    C('live_blog_entry', 'article', 'timestamped live blog update', 'prose', 'time text', 'LiveBlogPosting'),
+    C('embedded_post', 'article', 'embedded social media post inside an article', 'prose', 'author text'),
+    # ---- docs
+    C('doc_header', 'docs', 'title and summary of a documentation page', 'header', 'title summary', 'TechArticle'),
+    C('api_signature', 'docs', 'function, method or endpoint signature', 'code', 'signature'),
+    C('parameter_list', 'docs', 'parameters or arguments with descriptions', 'kv', 'params', heading='Parameters'),
+    C('return_value', 'docs', 'what a function returns', 'prose', 'text', heading='Returns'),
+    C('code_example', 'docs', 'code snippet, possibly with explanation', 'code', 'code text'),
+    C('doc_prose', 'docs', 'explanatory documentation text', 'prose', 'text'),
+    C('callout_note', 'docs', 'note, warning, tip or pitfall box', 'prose', 'kind text'),
+    C('install_instructions', 'docs', 'install or setup commands', 'code', 'code text', heading='Install'),
+    C('changelog_entry', 'docs', 'release notes for a version', 'bullets', 'version date items'),
+    C('compat_table', 'docs', 'browser or version compatibility table', 'table', 'rows', heading='Compatibility'),
+    C('readme_badges', 'docs', 'status badges and shields', 'drop'),
+    C('repo_header', 'docs', 'code repository name, description, stars, forks', 'header', 'name description stars forks language', 'SoftwareSourceCode'),
+    C('file_tree', 'docs', 'list of files and folders in a repository', 'drop'),
+    C('cli_reference', 'docs', 'command line usage and flags', 'code', 'code text'),
+    # ---- community
+    C('thread_header', 'community', 'forum or Q&A question title, author, stats', 'header', 'title author date votes answers views', 'QAPage DiscussionForumPosting'),
+    C('post_body', 'community', 'body of a forum post or question', 'prose', 'author date text'),
+    C('answer', 'community', 'one answer or reply with author and votes', 'prose', 'author votes date text', 'Answer Comment'),
+    C('comment_list', 'community', 'comments under a post', 'prose', 'items', heading='Comments'),
+    C('social_post', 'community', 'tweet, post or status with author and counts', 'prose', 'author date text likes reposts replies', 'SocialMediaPosting'),
+    C('poll', 'community', 'poll question and options with votes', 'kv', 'question options'),
+    C('community_rules', 'community', 'rules or guidelines of a forum or group', 'bullets', 'items', heading='Rules'),
+    C('subreddit_header', 'community', 'community or group name, members, description', 'header', 'name members description'),
+    # ---- profile
+    C('profile_header', 'profile', 'person or account name, handle, bio, follower counts', 'header', 'name handle bio followers following posts', 'Person ProfilePage'),
+    C('org_header', 'profile', 'company or organization name, industry, size, HQ', 'header', 'name industry size headquarters founded website', 'Organization Corporation'),
+    C('place_header', 'profile', 'business or place name, address, hours, rating', 'header', 'name address phone hours rating', 'LocalBusiness Restaurant Place'),
+    C('experience_list', 'profile', 'jobs or roles held with dates', 'bullets', 'items', heading='Experience'),
+    C('education_list', 'profile', 'schools and degrees', 'bullets', 'items', heading='Education'),
+    C('skills_list', 'profile', 'skills, tags or specialties', 'bullets', 'items', heading='Skills'),
+    C('contact_info', 'profile', 'email, phone, address, social links', 'kv', 'email phone address', heading='Contact'),
+    C('opening_hours', 'profile', 'opening hours by day', 'kv', 'hours', heading='Hours'),
+    C('post_grid', 'profile', "grid of an account's posts or videos", 'cards', 'items', heading='Posts'),
+    C('people_list', 'profile', 'list of people, employees, followers or members', 'cards', 'items', heading='People'),
+    # ---- media
+    C('video_header', 'media', 'video title, channel, views, date, description', 'header', 'title channel views date duration description', 'VideoObject'),
+    C('podcast_episode', 'media', 'podcast or episode title, show, date, summary', 'header', 'title show date duration summary', 'PodcastEpisode'),
+    C('film_header', 'media', 'film or show title, year, rating, director, cast', 'header', 'title year rating director cast genre', 'Movie TVSeries'),
+    C('book_header', 'media', 'book title, author, rating, pages, published', 'header', 'title author rating pages published', 'Book'),
+    C('music_header', 'media', 'song or album title, artist, release', 'header', 'title artist release', 'MusicRecording MusicAlbum'),
+    C('game_header', 'media', 'game or app name, developer, rating, platforms', 'header', 'name developer rating platforms', 'VideoGame SoftwareApplication MobileApplication'),
+    C('cast_list', 'media', 'cast and crew list', 'bullets', 'items', heading='Cast'),
+    C('episode_list', 'media', 'list of episodes or tracks', 'cards', 'items', heading='Episodes'),
+    C('transcript', 'media', 'transcript or lyrics text', 'prose', 'text', heading='Transcript'),
+    C('genre_tags', 'media', 'genres, tags or categories', 'bullets', 'items', heading='Genres'),
+    C('media_player', 'media', 'player controls, timestamps, playback UI', 'drop'),
+    # ---- events / travel
+    C('event_card', 'events_travel', 'event name, date, time, venue, price', 'kv', 'name date venue price', 'Event'),
+    C('ticket_options', 'events_travel', 'ticket types and prices', 'table', 'rows', heading='Tickets'),
+    C('hotel_header', 'events_travel', 'hotel or rental name, rating, price per night, location', 'header', 'name rating price location', 'Hotel LodgingBusiness VacationRental'),
+    C('amenities_list', 'events_travel', 'amenities or facilities', 'bullets', 'items', heading='Amenities'),
+    C('flight_option', 'events_travel', 'flight times, airline, stops, price', 'kv', 'airline depart arrive stops price', 'Flight'),
+    C('itinerary', 'events_travel', 'day-by-day plan or schedule', 'steps', 'items', heading='Itinerary'),
+    C('booking_policy', 'events_travel', 'cancellation, check-in or house rules', 'kv', 'rules', heading='Policies'),
+    # ---- food / health
+    C('recipe', 'food_health', 'recipe title, time, servings, rating', 'header', 'name time servings rating', 'Recipe'),
+    C('ingredients', 'food_health', 'list of recipe ingredients', 'bullets', 'items', heading='Ingredients'),
+    C('instructions', 'food_health', 'numbered cooking or how-to steps', 'steps', 'items', 'HowTo', 'Steps'),
+    C('nutrition_facts', 'food_health', 'calories and nutrients', 'kv', 'nutrients', 'NutritionInformation', 'Nutrition'),
+    C('menu', 'food_health', 'restaurant menu items and prices', 'cards', 'items', 'Menu', 'Menu'),
+    C('medical_info', 'food_health', 'symptoms, dosage, treatment or health facts', 'prose', 'text', 'MedicalCondition Drug'),
+    # ---- data / finance
+    C('data_table', 'data_finance', 'table of data with rows and columns', 'table', 'rows'),
+    C('stat_highlight', 'data_finance', 'big number or key statistic with label', 'kv', 'label value'),
+    C('stock_quote', 'data_finance', 'ticker, price, change, market data', 'kv', 'symbol price change volume market_cap'),
+    C('chart', 'data_finance', 'chart or graph with its labels', 'prose', 'title text'),
+    C('pricing_plans', 'business', 'subscription plans with prices and features', 'table', 'rows', heading='Pricing'),
+    # ---- business
+    C('hero', 'business', 'headline and tagline at the top of a landing page', 'header', 'headline tagline'),
+    C('feature_grid', 'business', 'grid of product features with short descriptions', 'kv', 'features', heading='Features'),
+    C('testimonial', 'business', 'customer quote or case study snippet', 'prose', 'author text'),
+    C('logo_wall', 'business', 'customer or partner logos', 'drop'),
+    C('faq', 'business', 'frequently asked questions and answers', 'qa', 'items', 'FAQPage Question', 'FAQ'),
+    C('call_to_action', 'business', 'sign up, get started, contact sales buttons', 'drop'),
+    C('team_section', 'business', 'team members with roles', 'cards', 'items', heading='Team'),
+    C('timeline', 'reference', 'dated events in order', 'bullets', 'items', heading='Timeline'),
+    # ---- reference
+    C('infobox', 'reference', 'summary box of key facts about a subject', 'kv', 'facts', heading='Facts'),
+    C('definition', 'reference', 'term and its definition', 'kv', 'term definition'),
+    C('encyclopedia_section', 'reference', 'section of encyclopedia text under a heading', 'prose', 'text'),
+    C('see_also', 'reference', 'see also list of related topics', 'links', 'items', heading='See also'),
+    C('citations', 'reference', 'references, footnotes, sources', 'drop'),
+    C('disambiguation', 'reference', 'list of meanings for an ambiguous term', 'bullets', 'items'),
+    # ---- chrome
+    C('navigation', 'chrome', 'site menus, tabs, category lists', 'drop'),
+    C('breadcrumb', 'chrome', 'category path like Home > Electronics > Headphones', 'kv', 'path', 'BreadcrumbList'),
+    C('footer', 'chrome', 'site footer, copyright, legal and policy links', 'drop'),
+    C('promo_ad', 'chrome', 'ads, sponsored items, credit card or upsell offers', 'drop'),
+    C('cookie_banner', 'chrome', 'cookie or consent notice', 'drop'),
+    C('login_prompt', 'chrome', 'sign in, create account or app download prompt', 'drop'),
+    C('share_buttons', 'chrome', 'share, save, like buttons', 'drop'),
+    C('ui_chrome', 'chrome', 'loading text, image placeholders, empty widgets, error text', 'drop'),
+    C('recommendations', 'chrome', 'customers also bought, you may also like, trending', 'links', 'items'),
+]
+
+BY_NAME = {c.name: c for c in CATALOG}
+BY_CATEGORY = {cat: [c for c in CATALOG if c.category == cat] for cat in CATEGORIES}
+BY_LD = {t: c for c in CATALOG for t in c.ld}
+assert len(BY_NAME) == len(CATALOG), 'duplicate component name'
+assert all(len(v) <= 255 for v in BY_CATEGORY.values())

givememd/compose.py

@@ -0,0 +1,337 @@
+"""Component pipeline: segment the page finely, label each section with a catalog component, fill, render.
+
+    segments(root)          -> Segment list in reading order (small, single-purpose sections)
+    page_type(...)          -> one catalog category for the whole page (JSON-LD, else Jev, else rules)
+    label(...)              -> a component per segment: confident rules first, Jev for the rest
+    render(...)             -> markdown: components drawn with fixed layouts, junk dropped, same-kind runs merged
+
+Turned on with convert(..., mode='components') or GIVEMEMD_COMPONENTS=1. Without a Jev key it still runs, with
+rules-only labels (coarser: anything unclear is kept as generic prose).
+"""
+import hashlib, re
+from collections import Counter
+from dataclasses import dataclass, field
+
+from . import datatables, jev
+from .components import BY_CATEGORY, BY_LD, BY_NAME, CATEGORIES, CATALOG
+
+DROP_CONFIDENCE = float(__import__('os').environ.get('GIVEMEMD_DROP_CONFIDENCE', '0.7'))            # a model must be this sure before content is thrown away; otherwise it is kept
+MAXSEG = 700                      # a section bigger than this is split into its children
+LEAF = {'table', 'pre', 'ul', 'ol', 'dl'}
+LEAF_MAX = 6000
+HEADING = re.compile(r'^#{1,6} ')
+FACT = re.compile(r'^[-\s]*([$€£¥₹]\s?[\d,.]+|[\d.]+ out of 5|\(?[\d,]+\)? ?(ratings|reviews)|[\d.]+ ?stars?)\b', re.I)
+KV = re.compile(r'^- ?[^:]{1,40}: \S')
+GENERIC = ['article_body', 'doc_prose', 'encyclopedia_section', 'data_table', 'image_caption', 'related_articles',
+           'see_also', 'faq', 'breadcrumb', 'key_takeaways', 'callout_note', 'code_example']
+PROSE_FOR = {'article': 'article_body', 'docs': 'doc_prose', 'reference': 'encyclopedia_section', 'community': 'post_body',
+             'commerce': 'product_description', 'food_health': 'article_body', 'business': 'article_body'}
+KV_FOR = {'commerce': 'spec_table', 'reference': 'infobox', 'docs': 'parameter_list', 'profile': 'contact_info'}
+LD_CATEGORY = {'Product': 'commerce', 'NewsArticle': 'article', 'Article': 'article', 'BlogPosting': 'article',
+               'Recipe': 'food_health', 'Event': 'events_travel', 'JobPosting': 'listings', 'ItemList': 'listings',
+               'QAPage': 'community', 'DiscussionForumPosting': 'community', 'VideoObject': 'media', 'Movie': 'media',
+               'Book': 'media', 'Person': 'profile', 'ProfilePage': 'profile', 'Organization': 'profile',
+               'LocalBusiness': 'profile', 'SoftwareSourceCode': 'docs', 'TechArticle': 'docs', 'FAQPage': 'business',
+               'Hotel': 'events_travel', 'SingleFamilyResidence': 'listings'}
+
+
+# when a page carries several JSON-LD entities, the main one is the most specific
+LD_PRIORITY = ['Product', 'Recipe', 'Event', 'JobPosting', 'NewsArticle', 'BlogPosting', 'Article', 'TechArticle', 'QAPage',
+               'DiscussionForumPosting', 'Movie', 'Book', 'Hotel', 'SingleFamilyResidence', 'SoftwareSourceCode', 'LocalBusiness',
+               'Person', 'ProfilePage', 'VideoObject', 'FAQPage', 'ItemList', 'Organization']
+
+
+def _types(o):
+    t = o.get('@type')
+    return t if isinstance(t, list) else [t]
+
+
+def main_entities(ld):
+    rank = lambda o: min((LD_PRIORITY.index(t) for t in _types(o) if t in LD_PRIORITY), default=len(LD_PRIORITY))
+    return sorted(ld, key=rank)
+
+
+# types that sites attach to almost any page; they don't say what the page is
+WEAK_LD = {'Organization', 'ItemList', 'Person', 'VideoObject', 'Article', 'FAQPage', 'WebPage', 'WebSite', 'BreadcrumbList'}
+PROSE_KINDS = ['article_body', 'subheading_section', 'doc_prose', 'encyclopedia_section', 'post_body', 'answer', 'review_item',
+               'product_description', 'callout_note', 'author_bio', 'testimonial', 'pull_quote', 'image_caption', 'medical_info',
+               'api_signature', 'parameter_list', 'return_value', 'transcript', 'related_articles', 'promo_ad', 'login_prompt', 'ui_chrome']
+
+
+@dataclass
+class Segment:
+    blocks: list                  # markdown blocks (from extract.MD)
+    tag: str = ''
+    links: list = field(default_factory=list)
+    label: str = ''
+    how: str = ''                 # 'rule:<name>' | 'jev' | 'default'
+    confidence: float = 1.0
+
+    @property
+    def text(self):
+        return ' '.join(re.sub(r'^#+ |^- ', '', b) for b in self.blocks)
+
+
+# ---------- segmentation ----------
+def segments(root, url):
+    from .extract import MD
+    out = []
+
+    def emit(nodes, tag):
+        md = MD(url)
+        for n in nodes:
+            md.handle(n)
+        md.flush()
+        if md.blocks:
+            out.append(Segment(md.blocks, tag, md.links))
+
+    def rec(node):
+        run = []                                   # consecutive inline children (text, <span>, <a>...)
+        for c in node.iter(include_text=True):
+            if c.tag == '-text' or c.tag in ('span', 'a', 'b', 'strong', 'em', 'i', 'code', 'small', 'label', 'sup', 'sub', 'br'):
+                run.append(c); continue
+            if run:
+                emit(run, 'inline'); run = []
+            text = c.text(separator=' ')
+            n = len(re.sub(r'\s+', ' ', text).strip())
+            if n == 0:
+                continue
+            if c.tag in LEAF and n <= LEAF_MAX or c.tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6'):
+                emit([c], c.tag); continue
+            if n <= MAXSEG and not _mixed(c, url):
+                emit([c], c.tag); continue
+            rec(c)
+        if run:
+            emit(run, 'inline')
+
+    rec(root)
+    return _attach_headings(out)
+
+
+def _mixed(node, url):
+    """A small box that holds a price/rating fact next to other text (Amazon: price + Visa promo) is split."""
+    from .extract import MD
+    md = MD(url); md.walk(node); md.flush()
+    return len(md.blocks) > 1 and any(FACT.match(b) for b in md.blocks) and any(len(b) > 60 for b in md.blocks) \
+        and node.css_first('div,span,p,li') is not None
+
+
+def _attach_headings(segs):
+    """A heading on its own belongs to the section that follows it."""
+    out, pending = [], []
+    for s in segs:
+        if all(HEADING.match(b) for b in s.blocks):
+            pending += s.blocks; continue
+        if pending:
+            if len(pending) >= 3:              # a stack of empty headings (tab labels, "step 1..11") says nothing
+                pending = pending[-1:]
+            s.blocks = pending + s.blocks; pending = []
+        out.append(s)
+    if pending:
+        out.append(Segment(pending, 'h'))
+    for s in out:                              # the same rule inside one section
+        blocks, run = [], []
+        for b in s.blocks + ['']:
+            if b and HEADING.match(b):
+                run.append(b); continue
+            blocks += run[-1:] if len(run) >= 3 else run
+            run = []
+            if b: blocks.append(b)
+        s.blocks = blocks
+    return out
+
+
+# ---------- labeling ----------
+def page_type(meta, ld, segs, tables, judge):
+    counts = Counter(t for o in ld for t in _types(o) if t)
+    repeated = [t for t, n in counts.items() if n >= 3 and t not in ('ListItem', 'BreadcrumbList', 'Question', 'Answer')]
+    if repeated:                                   # many Events / Products / homes on one page = a listing
+        return 'listings', f'ld:{repeated[0]}x{counts[repeated[0]]}'
+    for o in main_entities(ld):
+        for t in _types(o):
+            if t in LD_CATEGORY and t not in WEAK_LD:
+                return LD_CATEGORY[t], f'ld:{t}'
+    if judge:
+        state = f"Web page: {meta.get('title', '')}\nURL: {meta.get('url', '')}\nOpening text: " + ' '.join(s.text for s in segs)[:1500]
+        a = judge.ask(state, {'page': {'type': 'choice', 'instructions': 'What kind of web page is this?',
+                                       'criteria': {k: v for k, v in CATEGORIES.items() if k != 'chrome'}}})
+        if a: return a['page']['choice'], 'jev'
+    if tables and len(tables[0].recs) >= 8: return 'listings', 'rule:table'
+    if sum(1 for s in segs if s.tag == 'pre') >= 3: return 'docs', 'rule:code'
+    return 'article', 'default'
+
+
+def _rule(s, ptype):
+    """Confident labels that need no model. Returns a component name or None."""
+    from .extract import UI_TEXT, DATA_MARK
+    body = [b for b in s.blocks if not HEADING.match(b)]
+    if any(DATA_MARK in b for b in s.blocks): return 'listing_grid'
+    if not body: return None
+    if all(UI_TEXT.match(b) for b in body): return 'ui_chrome'
+    if s.tag == 'pre' or all(b.startswith('```') for b in body): return 'code_example'
+    if sum(1 for b in body if KV.match(b)) >= max(2, 0.7 * len(body)): return KV_FOR.get(ptype, 'spec_table')
+    text = ' '.join(body)
+    link_chars = sum(len(t) for t, _ in s.links)
+    if len(text) >= 250 and link_chars < 0.25 * len(text) and not any(FACT.match(b) for b in body):
+        return 'prose?'                            # prose for sure; which kind is asked (cheaply) below
+    return None
+
+
+def options(ptype):
+    names = [c.name for c in BY_CATEGORY.get(ptype, [])] + GENERIC + [c.name for c in BY_CATEGORY['chrome']]
+    return {n: BY_NAME[n].desc for n in dict.fromkeys(names)}
+
+
+def label(segs, ptype, meta, judge):
+    ask, prose = [], []
+    for s in segs:
+        r = _rule(s, ptype)
+        if r == 'prose?':
+            s.label, s.how = PROSE_FOR.get(ptype, 'article_body'), 'rule'
+            prose.append(s)
+        elif r:
+            s.label, s.how = r, 'rule'
+        else:
+            ask.append(s)
+    if judge and (ask or prose):
+        crit, pcrit = options(ptype), {n: BY_NAME[n].desc for n in PROSE_KINDS}
+        state = f"Web page ({ptype}): {meta.get('title', '')}\nURL: {meta.get('url', '')}"
+        q = lambda s, c: {'type': 'choice', 'criteria': c, 'instructions': 'Which component is this section of the page?\nSection:\n' + s.text[:600]}
+        qs = {f's{i}': q(s, crit) for i, s in enumerate(ask)}
+        qs.update({f'p{i}': q(s, pcrit) for i, s in enumerate(prose)})
+        a = judge.ask(state, qs)
+        if a:
+            for i, s in enumerate(prose):
+                s.label, s.how, s.confidence = a[f'p{i}']['choice'], 'jev-prose', a[f'p{i}'].get('confidence', 0.0)
+                if BY_NAME[s.label].shape == 'drop' and s.confidence < DROP_CONFIDENCE:
+                    s.label, s.how = PROSE_FOR.get(ptype, 'article_body'), 'kept'
+            for i, s in enumerate(ask):
+                s.label, s.how, s.confidence = a[f's{i}']['choice'], 'jev', a[f's{i}'].get('confidence', 0.0)
+                if BY_NAME[s.label].shape == 'drop' and s.confidence < DROP_CONFIDENCE:
+                    s.label, s.how = PROSE_FOR.get(ptype, 'article_body'), f'kept (jev said {s.label} {s.confidence:.2f})'
+    for s in ask:
+        if not s.label:                            # no model: keep it, as generic content
+            s.label, s.how = PROSE_FOR.get(ptype, 'article_body'), 'default'
+    return segs
+
+
+# ---------- filling + rendering ----------
+ALIASES = {'name': ['name', 'headline', 'title'], 'headline': ['headline', 'name'], 'title': ['name', 'headline', 'title'],
+           'price': ['offers.price', 'offers.lowPrice', 'price'], 'brand': ['brand.name', 'brand'],
+           'rating': ['aggregateRating.ratingValue'], 'rating_count': ['aggregateRating.reviewCount', 'aggregateRating.ratingCount'],
+           'author': ['author.name', 'author'], 'date': ['datePublished', 'startDate', 'uploadDate'],
+           'availability': ['offers.availability'], 'standfirst': ['description'], 'description': ['description'],
+           'time': ['totalTime'], 'servings': ['recipeYield'], 'venue': ['location.name'], 'channel': ['author.name'],
+           'views': ['interactionStatistic.userInteractionCount'], 'followers': ['interactionStatistic.userInteractionCount']}
+
+
+def ld_header(ld, ptype):
+    """The page's main entity from JSON-LD, filled into its header component."""
+    for o in main_entities(ld):
+        types = _types(o)
+        comp = next((BY_LD[t] for t in types if t in BY_LD and BY_LD[t].shape == 'header'), None)
+        if not comp: continue
+        flat = datatables.flatten(o)
+        vals = {}
+        for f in comp.fields:
+            for k in ALIASES.get(f, [f]):
+                v = flat.get(k)
+                if v not in (None, '') and not str(v).startswith('http'):
+                    vals[f] = re.sub(r'^https?://schema.org/', '', str(v)); break
+        if vals.get('name') or vals.get('headline') or vals.get('title'):
+            return comp, vals
+    return None, {}
+
+
+def render_header(comp, vals):
+    title = vals.get('name') or vals.get('headline') or vals.get('title')
+    facts = [f'**{k.replace("_", " ").title()}:** {v}' for k, v in vals.items() if k not in ('name', 'headline', 'title', 'description', 'standfirst')]
+    out = [f'# {title}']
+    if facts: out.append(' · '.join(facts))
+    if vals.get('standfirst') or vals.get('description'): out.append(vals.get('standfirst') or vals.get('description'))
+    return out
+
+
+STEP_HEADING = re.compile(r'^#+ (step|stage|part)?\s*\d+\.?$', re.I)
+
+
+def render_segment(s, comp, prev):
+    """Blocks keep their page order; only the section's leading headings are treated as its title."""
+    lead = 0
+    while lead < len(s.blocks) and HEADING.match(s.blocks[lead]):
+        lead += 1
+    out, rest = list(s.blocks[:lead]), s.blocks[lead:]
+    if not out and comp.heading and prev != comp.name:
+        out.append(f'## {comp.heading}')
+    if comp.shape == 'header':
+        short = [b for b in rest if len(b) <= 80 and not HEADING.match(b)]
+        out += ([' · '.join(re.sub(r'^- ', '', b) for b in short)] if short else []) + [b for b in rest if b not in short]
+    elif comp.shape in ('bullets', 'links'):
+        out += [b if b.startswith(('- ', '|', '```', '#')) else f'- {b}' for b in rest[:15 if comp.shape == 'links' else None]]
+    elif comp.shape == 'steps':        # "step 1".."step 11" headings carry nothing once the steps are numbered
+        out = [b for b in out if not STEP_HEADING.match(b)]
+        items = [re.sub(r'^- ', '', b) for b in rest if not STEP_HEADING.match(b)]
+        out += [b if HEADING.match(b) else f'{i}. {b}' for i, b in enumerate(items, 1)]
+    else:                              # kv, prose, qa, table, code: the blocks already have the right form
+        out += rest
+    return out
+
+
+# short facts that belong in the main header's facts line, keyed by the field they fill
+FACT_FIELD = {'price_offer': 'Price', 'review_summary': 'Rating', 'stock_status': 'Availability', 'shipping_returns': 'Shipping',
+              'variant_picker': 'Options', 'seller_info': 'Seller', 'result_count': 'Results', 'stat_highlight': 'Stats',
+              'contact_info': 'Contact', 'opening_hours': 'Hours', 'genre_tags': 'Genres', 'breadcrumb': 'Category'}
+NEAR = 8                          # how far from the header a fact may sit
+
+
+def header_facts(segs):
+    """Fill the main header component: index of the header section, and the nearby short fact sections that
+    fill its fields (price, rating, availability...)."""
+    h = next((i for i, s in enumerate(segs) if BY_NAME.get(s.label) and BY_NAME[s.label].shape == 'header'), None)
+    if h is None: return None, {}
+    facts = {}
+    for i in range(max(0, h - NEAR), min(len(segs), h + NEAR + 1)):
+        s = segs[i]
+        if s.label in FACT_FIELD and len(s.text) <= 120:
+            facts[i] = (FACT_FIELD[s.label], re.sub(r'^#+ ', '', s.text))
+    return h, facts
+
+
+def render(segs, ptype, meta, ld, cands):
+    from .extract import DATA_MARK
+    lines, prev, dropped = [], None, Counter()
+    comp, vals = ld_header(ld, ptype)
+    header_text = ''
+    if comp:
+        h = render_header(comp, vals)
+        lines += h; header_text = ' '.join(h).lower(); prev = comp.name
+    tables = [datatables.render([c]) for c in cands]
+    h, facts = header_facts(segs)
+    for i, s in enumerate(segs):
+        c = BY_NAME.get(s.label) or BY_NAME[PROSE_FOR.get(ptype, 'article_body')]
+        if c.shape == 'drop':
+            dropped[c.name] += 1; continue
+        if i in facts: continue                  # rendered inside the header's facts line
+        if i == h and facts:
+            block = render_segment(s, c, prev)
+            by_field = {}
+            for f, v in facts.values(): by_field.setdefault(f, []).append(v)
+            block.insert(1 if block and HEADING.match(block[0]) else 0,
+                         ' · '.join(f'**{f}:** ' + '; '.join(dict.fromkeys(v)) for f, v in by_field.items()))
+            if comp and c.name == comp.name:
+                block = [b for b in block if b.lower().lstrip('# ') not in header_text]
+            lines += block; prev = c.name; continue
+        if any(DATA_MARK in b for b in s.blocks) and tables:
+            lines += tables.pop(0); prev = 'listing_grid'; continue
+        block = render_segment(s, c, prev)
+        if comp and c.name == comp.name:         # DOM copy of the JSON-LD header: keep only what it adds
+            block = [b for b in block if b.lower().lstrip('# ') not in header_text]
+        lines += block
+        prev = c.name
+    for t in tables:
+        lines += t
+    return lines, dropped
+
+
+def signature(s):
+    return hashlib.sha1(s.text[:200].encode()).hexdigest()[:12]