rebly-sections 1.0.1 → 1.2.0

package/assets/scripts/kb-extractor-helpers.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+"""Helpers for kb-extractor.py — zip handling, flat extraction, dedup, manifest, report."""
+
+import hashlib
+import json
+import re
+import shutil
+import zipfile
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+# ─── Zip Extraction Logic ────────────────────────────────────────────────────
+
+def find_shopify_root(base: Path) -> Path | None:
+    """Find the directory that contains sections/*.liquid files."""
+    if (base / 'sections').exists() and list((base / 'sections').glob('*.liquid')):
+        return base
+    for child in sorted(base.iterdir()):
+        if child.is_dir() and (child / 'sections').exists():
+            if list((child / 'sections').glob('*.liquid')):
+                return child
+    return None
+
+
+def extract_zip_safe(zip_path: Path, dest: Path) -> bool:
+    """Extract a zip file safely, filtering dangerous paths."""
+    try:
+        dest.mkdir(parents=True, exist_ok=True)
+        with zipfile.ZipFile(zip_path, 'r') as zf:
+            members = [m for m in zf.namelist()
+                       if not m.startswith('__MACOSX')
+                       and '..' not in m
+                       and not m.startswith('/')]
+            zf.extractall(dest, members=members)
+        return True
+    except (zipfile.BadZipFile, Exception):
+        return False
+
+
+def find_inner_zip(base: Path) -> Path | None:
+    """Find most likely theme zip: install-me-*.zip > largest non-demo zip."""
+    for f in base.rglob('install-me-*.zip'):
+        return f
+    zips = [f for f in base.rglob('*.zip') if '__MACOSX' not in str(f)]
+    if not zips:
+        return None
+    excluded = {'demo', 'assets', 'preview', 'documentation', 'docs', 'samples'}
+    candidates = [z for z in zips if not any(e in z.stem.lower() for e in excluded)]
+    if not candidates:
+        candidates = zips
+    return max(candidates, key=lambda z: z.stat().st_size)
+
+
+def extract_theme(zip_path: Path, tmp_dir: Path) -> Path | None:
+    """Extract theme zip handling up to 3 levels of nesting."""
+    theme_name = zip_path.stem.lower().replace(' ', '-')[:50]
+    dest = tmp_dir / theme_name
+    if dest.exists():
+        shutil.rmtree(dest)
+
+    current_zip = zip_path
+    for level in range(3):
+        level_dest = dest / f'_level{level}'
+        if not extract_zip_safe(current_zip, level_dest):
+            return None
+        root = find_shopify_root(level_dest)
+        if root:
+            return root
+        inner = find_inner_zip(level_dest)
+        if not inner:
+            return None
+        current_zip = inner
+    return None
+
+
+def theme_slug(zip_path: Path) -> str:
+    """Stable, filesystem-safe slug from zip filename."""
+    name = zip_path.stem.lower()
+    name = re.sub(r'[^a-z0-9]+', '-', name).strip('-')
+    return name[:60]
+
+
+def content_hash(path: Path) -> str:
+    """SHA-256 first 16 hex chars for dedup."""
+    return hashlib.sha256(path.read_bytes()).hexdigest()[:16]
+
+
+def already_extracted(out_dir: Path, slug: str) -> bool:
+    """Check if theme already extracted (has manifest.json)."""
+    theme_dir = out_dir / slug
+    return theme_dir.exists() and (theme_dir / 'manifest.json').exists()
+
+
+def extract_sections_flat(theme_root: Path, out_dir: Path, slug: str,
+                          source_zip: str, zip_type: str) -> dict:
+    """Copy unique sections from theme_root to out_dir/{slug}/sections/.
+
+    Returns per-theme stats dict.
+    """
+    sections_src = theme_root / 'sections'
+    theme_out = out_dir / slug / 'sections'
+    theme_out.mkdir(parents=True, exist_ok=True)
+
+    seen_hashes = set()
+    found = 0
+    unique = 0
+    deduped = 0
+
+    for liquid in sorted(sections_src.glob('*.liquid')):
+        found += 1
+        h = content_hash(liquid)
+        if h in seen_hashes:
+            deduped += 1
+            continue
+        seen_hashes.add(h)
+        shutil.copy2(liquid, theme_out / liquid.name)
+        unique += 1
+
+    # Write per-theme manifest
+    manifest = {
+        "slug": slug,
+        "source_zip": source_zip,
+        "zip_type": zip_type,
+        "sections_found": found,
+        "sections_unique": unique,
+        "sections_deduped": deduped,
+        "extracted_at": datetime.now(timezone.utc).isoformat(),
+    }
+    manifest_path = out_dir / slug / 'manifest.json'
+    manifest_path.write_text(json.dumps(manifest, indent=2), encoding='utf-8')
+
+    return manifest
+
+
+def write_extraction_report(out_dir: Path, stats: dict, theme_manifests: list):
+    """Write themes/extraction-report.md with summary table."""
+    report = out_dir / 'extraction-report.md'
+
+    # Count by type
+    type_counts = {}
+    for m in theme_manifests:
+        t = m.get('zip_type', '?')
+        type_counts[t] = type_counts.get(t, 0) + 1
+
+    total_sections = sum(m.get('sections_unique', 0) for m in theme_manifests)
+    total_deduped = sum(m.get('sections_deduped', 0) for m in theme_manifests)
+
+    lines = [
+        "# Extraction Report",
+        f"\nGenerated: {datetime.now(timezone.utc).isoformat()}",
+        "",
+        "## Summary",
+        f"- Total zips processed: {stats.get('total', 0)}",
+        f"- Extracted: {stats.get('extracted', 0)}",
+        f"- Skipped (no sections): {stats.get('skipped', 0)}",
+        f"- Skipped (resume): {stats.get('resumed', 0)}",
+        f"- Failed: {stats.get('failed', 0)}",
+        f"- Unique sections: {total_sections}",
+        f"- Deduped (within-theme): {total_deduped}",
+        "",
+        "## Type Breakdown",
+    ]
+    for t, c in sorted(type_counts.items()):
+        lines.append(f"- Type {t}: {c}")
+
+    lines += [
+        "",
+        "## Top Themes by Section Count",
+        "| Theme | Sections | Type |",
+        "|-------|----------|------|",
+    ]
+    top = sorted(theme_manifests, key=lambda m: m.get('sections_unique', 0), reverse=True)[:20]
+    for m in top:
+        lines.append(f"| {m['slug']} | {m['sections_unique']} | {m['zip_type']} |")
+
+    report.write_text("\n".join(lines), encoding='utf-8')
+    return str(report)

package/assets/scripts/kb-extractor.py

@@ -19,91 +19,21 @@ import importlib.util
 import json
 import shutil
 import sys
-import zipfile
 from pathlib import Path
 
+# Import all helpers (extraction logic, dedup, manifest, report)
+_helpers_path = Path(__file__).parent / 'kb-extractor-helpers.py'
+_hspec = importlib.util.spec_from_file_location('kb_extractor_helpers', _helpers_path)
+_helpers = importlib.util.module_from_spec(_hspec)
+_hspec.loader.exec_module(_helpers)
+theme_slug_fn = _helpers.theme_slug
+already_extracted = _helpers.already_extracted
+extract_sections_flat = _helpers.extract_sections_flat
+write_extraction_report = _helpers.write_extraction_report
+extract_theme = _helpers.extract_theme
 
 TMP_DIR = Path('themes/tmp')
 
-# ─── Extraction Logic ─────────────────────────────────────────────────────────
-
-def find_shopify_root(base: Path) -> Path | None:
-    """Find the directory that contains sections/*.liquid files."""
-    # Check base itself
-    if (base / 'sections').exists() and list((base / 'sections').glob('*.liquid')):
-        return base
-    # Check one level deep
-    for child in sorted(base.iterdir()):
-        if child.is_dir() and (child / 'sections').exists():
-            if list((child / 'sections').glob('*.liquid')):
-                return child
-    return None
-
-
-def extract_zip_safe(zip_path: Path, dest: Path) -> bool:
-    """Extract a zip file safely, return True if successful."""
-    try:
-        dest.mkdir(parents=True, exist_ok=True)
-        with zipfile.ZipFile(zip_path, 'r') as zf:
-            # Filter out dangerous paths and MACOSX junk
-            members = [m for m in zf.namelist()
-                       if not m.startswith('__MACOSX')
-                       and '..' not in m
-                       and not m.startswith('/')]
-            zf.extractall(dest, members=members)
-        return True
-    except zipfile.BadZipFile:
-        return False
-    except Exception as e:
-        print(f"  ⚠️  Extract error: {e}")
-        return False
-
-
-def find_inner_zip(base: Path) -> Path | None:
-    """Find the first inner zip file (install-me pattern or any zip)."""
-    # Prefer install-me-*.zip pattern
-    for f in base.rglob('install-me-*.zip'):
-        return f
-    # Fallback: any zip not in __MACOSX
-    for f in base.rglob('*.zip'):
-        if '__MACOSX' not in str(f):
-            return f
-    return None
-
-
-def extract_theme(zip_path: Path, tmp_dir: Path) -> Path | None:
-    """
-    Extract a theme zip to tmp_dir, handling all 3 types.
-    Returns the path to the Shopify theme root (containing sections/) or None.
-    """
-    theme_name = zip_path.stem.lower().replace(' ', '-')[:50]
-    dest = tmp_dir / theme_name
-    # Clean previous attempt
-    if dest.exists():
-        shutil.rmtree(dest)
-
-    # Level 1 extract
-    if not extract_zip_safe(zip_path, dest):
-        return None
-
-    # Check if already a Shopify theme (Type A)
-    root = find_shopify_root(dest)
-    if root:
-        return root
-
-    # Level 2: find and extract inner zip (Type B & C)
-    inner = find_inner_zip(dest)
-    if not inner:
-        return None
-
-    inner_dest = dest / '_inner'
-    if not extract_zip_safe(inner, inner_dest):
-        return None
-
-    root = find_shopify_root(inner_dest)
-    return root
-
-
 # ─── KB Builder Integration ───────────────────────────────────────────────────
 
 def load_kb_builder(scripts_dir: Path):
@@ -119,46 +49,31 @@ def load_kb_builder(scripts_dir: Path):
 
 # ─── Main ─────────────────────────────────────────────────────────────────────
 
-def process_zip(zip_path: Path, kb_dir: Path, tmp_dir: Path, kb_builder, stats: dict, dry_run: bool = False):
-    """Extract a single zip, build KB from it, cleanup."""
-    print(f"\n📦 {zip_path.name}")
+def build_queue(args, allowed_types, skip_collections):
+    """Build the zip processing queue from manifest, source, or single zip."""
+    queue = []  # list of (zip_path, zip_type, source_str)
 
-    if dry_run:
-        print(f"  [DRY RUN] Would extract: {zip_path.name}")
-        stats['dry_run'] += 1
-        return
-
-    theme_root = extract_theme(zip_path, tmp_dir)
+    if args.zip:
+        p = Path(args.zip)
+        queue.append((p, '?', str(p)))
 
-    if not theme_root:
-        print(f"  ❌ No valid Shopify sections found — skipping")
-        stats['skipped'] += 1
-        # Cleanup failed extraction
-        theme_tmp = tmp_dir / zip_path.stem.lower().replace(' ', '-')[:50]
-        if theme_tmp.exists():
-            shutil.rmtree(theme_tmp, ignore_errors=True)
-        return
+    elif args.manifest:
+        manifest = json.loads(Path(args.manifest).read_text(encoding='utf-8'))
+        source_base = Path(manifest.get('source', '.'))
+        for coll_name, coll_data in manifest.get('collections', {}).items():
+            if coll_name in skip_collections:
+                continue
+            for entry in coll_data.get('files', []):
+                if entry.get('type') in allowed_types:
+                    zip_path = source_base / entry['path']
+                    if zip_path.exists():
+                        queue.append((zip_path, entry['type'], entry['path']))
 
-    sections = list((theme_root / 'sections').glob('*.liquid'))
-    print(f"  ✅ Found {len(sections)} sections → building KB...")
-    stats['found'] += 1
-
-    # Run kb-builder
-    if kb_builder:
-        bstats = {'added': 0, 'updated': 0, 'skipped': 0}
-        kb_builder.process_theme(theme_root, kb_dir, bstats)
-        stats['sections_added'] += bstats['added']
-        stats['sections_updated'] += bstats['updated']
-        print(f"  📝 +{bstats['added']} new, ↑{bstats['updated']} updated, ⏭ {bstats['skipped']} skipped")
-    else:
-        print(f"  ⚠️  kb-builder not found — extracted to {theme_root} (not cleaning up)")
-        return
+    elif args.source:
+        source = Path(args.source)
+        queue = [(f, '?', str(f)) for f in source.rglob('*.zip')]
 
-    # Cleanup to save disk space
-    parent = theme_tmp = tmp_dir / zip_path.stem.lower().replace(' ', '-')[:50]
-    if parent.exists():
-        shutil.rmtree(parent, ignore_errors=True)
-        print(f"  🗑️  Cleaned up temp files")
+    return queue
 
 
 def main():
@@ -167,80 +82,101 @@ def main():
     parser.add_argument('--zip', help='Path to single zip file to process')
     parser.add_argument('--source', help='Path to shopify-theme-data/ (process all zips directly)')
     parser.add_argument('--kb', default='src/rebly-sections/data/', help='Path to KB data dir')
+    parser.add_argument('--out', default='', help='Flat extraction output dir (e.g. themes/extracted/)')
     parser.add_argument('--tmp', default='themes/tmp/', help='Temp extraction dir')
     parser.add_argument('--batch-size', type=int, default=10, help='Process N zips at a time')
     parser.add_argument('--dry-run', action='store_true', help='Preview without extracting')
     parser.add_argument('--types', default='A,B,C', help='Zip types to process (A,B,C)')
+    parser.add_argument('--skip-types', nargs='*', default=['WP', 'RAR', 'INVALID'],
+                        help='Zip types to skip')
+    parser.add_argument('--skip-collections', nargs='*', default=[
+        '150+ Shopify Codes', '150+ Shopify Liquid Codes'],
+                        help='Collection names to skip entirely')
+    parser.add_argument('--resume', action='store_true', default=True,
+                        help='Skip already-extracted themes (default: on)')
     args = parser.parse_args()
 
     kb_dir = Path(args.kb)
     tmp_dir = Path(args.tmp)
     tmp_dir.mkdir(parents=True, exist_ok=True)
+    out_dir = Path(args.out) if args.out else None
+    if out_dir:
+        out_dir.mkdir(parents=True, exist_ok=True)
 
-    # Load kb-builder
+    # Load kb-builder (only needed for legacy KB mode, not flat extraction)
     scripts_dir = Path(__file__).parent
-    kb_builder = load_kb_builder(scripts_dir)
-    if not kb_builder and not args.dry_run:
-        print("❌ kb-builder.py not found in scripts dir. Run from correct directory.")
-        sys.exit(1)
-
-    allowed_types = set(args.types.upper().split(','))
-    stats = {'found': 0, 'skipped': 0, 'dry_run': 0, 'sections_added': 0, 'sections_updated': 0}
+    kb_builder = load_kb_builder(scripts_dir) if not out_dir else None
 
-    # Build zip queue
-    queue = []
+    allowed_types = set(args.types.upper().split(',')) - set(args.skip_types)
+    skip_colls = set(args.skip_collections) if args.skip_collections else set()
+    queue = build_queue(args, allowed_types, skip_colls)
 
-    if args.zip:
-        queue = [Path(args.zip)]
-
-    elif args.manifest:
-        manifest = json.loads(Path(args.manifest).read_text(encoding='utf-8'))
-        source_base = Path(manifest.get('source', '.'))
-        for coll_name, coll_data in manifest.get('collections', {}).items():
-            for entry in coll_data.get('files', []):
-                if entry.get('type') in allowed_types:
-                    zip_path = source_base / entry['path']
-                    if zip_path.exists():
-                        queue.append(zip_path)
-
-    elif args.source:
-        source = Path(args.source)
-        queue = [f for f in source.rglob('*.zip')]
-
-    else:
+    if not queue:
         parser.print_help()
         return
 
     total = len(queue)
-    print(f"🚀 Processing {total} zip files (batch size: {args.batch_size})")
-
-    # Process in batches
-    for i in range(0, total, args.batch_size):
-        batch = queue[i:i + args.batch_size]
-        print(f"\n{'='*50}")
-        print(f"Batch {i // args.batch_size + 1} of {(total + args.batch_size - 1) // args.batch_size}")
-        print(f"{'='*50}")
-
-        for zip_path in batch:
-            process_zip(zip_path, kb_dir, tmp_dir, kb_builder, stats, args.dry_run)
+    stats = {'total': total, 'extracted': 0, 'skipped': 0, 'resumed': 0,
+             'failed': 0, 'sections_added': 0, 'sections_updated': 0}
+    theme_manifests = []
+
+    print(f"Processing {total} zip files (batch size: {args.batch_size})")
+
+    for i, (zip_path, zip_type, source_str) in enumerate(queue, 1):
+        slug = theme_slug_fn(zip_path)
+        print(f"\n[{i}/{total}] {zip_path.name} [{zip_type}]")
+
+        if args.dry_run:
+            print(f"  [DRY RUN] Would extract: {zip_path.name}")
+            continue
+
+        # Resume: skip already-extracted themes
+        if out_dir and args.resume and already_extracted(out_dir, slug):
+            print(f"  -- Already extracted, skipping")
+            stats['resumed'] += 1
+            continue
+
+        try:
+            theme_root = extract_theme(zip_path, tmp_dir)
+        except Exception as e:
+            print(f"  FAIL: {e}")
+            stats['failed'] += 1
+            continue
+
+        if not theme_root:
+            print(f"  No sections found, skipping")
+            stats['skipped'] += 1
+        elif out_dir:
+            # Flat extraction mode
+            m = extract_sections_flat(theme_root, out_dir, slug, source_str, zip_type)
+            theme_manifests.append(m)
+            stats['extracted'] += 1
+            print(f"  OK: {m['sections_unique']} unique sections ({m['sections_deduped']} deduped)")
+        elif kb_builder:
+            # Legacy KB builder mode
+            bstats = {'added': 0, 'updated': 0, 'skipped': 0}
+            kb_builder.process_theme(theme_root, kb_dir, bstats)
+            stats['extracted'] += 1
+            stats['sections_added'] += bstats['added']
+            stats['sections_updated'] += bstats['updated']
+            print(f"  +{bstats['added']} new, ^{bstats['updated']} updated")
+
+        # Cleanup tmp for this theme
+        theme_tmp = tmp_dir / slug[:50]
+        if theme_tmp.exists():
+            shutil.rmtree(theme_tmp, ignore_errors=True)
 
     # Final cleanup
     if tmp_dir.exists() and not args.dry_run:
-        remaining = list(tmp_dir.iterdir())
-        if remaining:
-            print(f"\n⚠️  {len(remaining)} unclean dirs in tmp — cleaning up...")
-            shutil.rmtree(tmp_dir, ignore_errors=True)
-
-    print(f"""
-╔══════════════════════════════════════╗
-║        KB Extractor Complete         ║
-╠══════════════════════════════════════╣
-║  ✅ Themes processed: {stats['found']:>4}            ║
-║  ❌ Skipped (no sections): {stats['skipped']:>4}      ║
-║  🆕 Sections added: {stats['sections_added']:>4}            ║
-║  ⬆️  Sections updated: {stats['sections_updated']:>4}         ║
-╚══════════════════════════════════════╝
-""")
+        shutil.rmtree(tmp_dir, ignore_errors=True)
+
+    # Write extraction report
+    if out_dir and theme_manifests:
+        report_path = write_extraction_report(out_dir, stats, theme_manifests)
+        print(f"\nReport: {report_path}")
+
+    print(f"\nExtracted: {stats['extracted']} | Skipped: {stats['skipped']} | "
+          f"Resumed: {stats['resumed']} | Failed: {stats['failed']}")
 
 
 if __name__ == '__main__':