insta-collect-data 0.2.0__py3-none-any.whl

insta_collect/cli.py

@@ -0,0 +1,246 @@
+#!/usr/bin/env python3
+
+import argparse
+import sys
+import time
+import itertools
+import threading
+import contextlib
+import os
+
+from insta_collect.scraper import scrape_hashtag, scrape_profile, scrape_profiles
+from insta_collect.saver import save_data
+
+
+PREVIEW_LIMIT = 5
+
+
+# ================= SPINNER =================
+def spinner_task(message, stop_event):
+    spinner = itertools.cycle(["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"])
+    while not stop_event.is_set():
+        sys.stdout.write(f"\r{message} {next(spinner)}")
+        sys.stdout.flush()
+        time.sleep(0.1)
+
+    sys.stdout.write(f"\r{message} done\n")
+    sys.stdout.flush()
+
+
+# ================= PREVIEW =================
+def preview_posts(data, n=PREVIEW_LIMIT):
+    print("\n--- SAMPLE POSTS ---")
+
+    for i, post in enumerate(data[:n], start=1):
+        url = post.get("url", "[no url]")
+
+        caption = post.get("caption") or ""
+        caption = caption.replace("\n", " ").strip()
+        caption_preview = caption[:120] + ("..." if len(caption) > 120 else "")
+
+        hashtags = post.get("hashtags", [])
+        hashtag_preview = " ".join(f"#{h}" for h in hashtags[:5])
+
+        likes = post.get("likes")
+        comments = post.get("comments")
+        views = post.get("views")
+
+        print(f"[{i}] {url}")
+
+        if caption_preview:
+            print(f"    caption : {caption_preview}")
+        else:
+            print("    caption : [no caption]")
+
+        metrics = []
+        if likes is not None:
+            metrics.append(f"likes={likes}")
+        if comments is not None:
+            metrics.append(f"comments={comments}")
+        if views is not None:
+            metrics.append(f"views={views}")
+        if metrics:
+            print(f"    metrics : {', '.join(metrics)}")
+
+        if hashtag_preview:
+            print(f"    hashtags: {hashtag_preview}")
+
+        print()
+
+
+def preview_profile(profile):
+    print(f"\n--- PROFILE @{profile.get('handle')} ---")
+    print(f"    followers : {profile.get('followers')}")
+    print(f"    posts     : {profile.get('posts_scraped')} scraped")
+    print(f"    avg eng   : {profile.get('avg_engagement')}")
+    print(f"    ER %      : {profile.get('engagement_rate_pct')}")
+    preview_posts(profile.get("posts") or [], n=PREVIEW_LIMIT)
+
+
+@contextlib.contextmanager
+def suppress_stdout():
+    with open(os.devnull, "w") as devnull:
+        old_stdout = sys.stdout
+        sys.stdout = devnull
+        try:
+            yield
+        finally:
+            sys.stdout = old_stdout
+
+
+def load_handles_from_file(path):
+    handles = []
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            handles.append(line.lstrip("@"))
+    return handles
+
+
+# ================= MAIN =================
+def main():
+    parser = argparse.ArgumentParser(
+        description="Instagram hashtag / profile scraper"
+    )
+
+    parser.add_argument(
+        "--tag",
+        default=None,
+        help="Hashtag mode: hashtag without #"
+    )
+    parser.add_argument(
+        "--user",
+        default=None,
+        help="Profile mode: single handle/username (with or without @)"
+    )
+    parser.add_argument(
+        "--users-file",
+        default=None,
+        help="Profile mode: text file with one handle per line"
+    )
+    parser.add_argument(
+        "--limit",
+        type=int,
+        default=10,
+        help="Max posts per target (use 6-12 for engagement)"
+    )
+    parser.add_argument(
+        "--profile",
+        default="ig_profile",
+        help="Folder profile Playwright (persistent login)"
+    )
+    parser.add_argument(
+        "--headed",
+        action="store_true",
+        help="Open a visible browser window (for first-time login / debugging). Default is headless."
+    )
+
+    args = parser.parse_args()
+
+    profile_mode = bool(args.user or args.users_file)
+    if not profile_mode and not args.tag:
+        parser.error("Provide --tag (hashtag) OR --user / --users-file (profile)")
+    if profile_mode and args.tag:
+        parser.error("Use either --tag or profile mode (--user/--users-file), not both")
+
+    handles = []
+    if args.user:
+        handles = [h.strip() for h in args.user.split(",") if h.strip()]
+    if args.users_file:
+        handles.extend(load_handles_from_file(args.users_file))
+
+    mode = "profile" if profile_mode else "hashtag"
+    print(f"▶ Mode            : {mode}")
+    if profile_mode:
+        print(f"▶ Handles         : {', '.join('@' + h.lstrip('@') for h in handles)}")
+    else:
+        print(f"▶ Target hashtag  : #{args.tag}")
+    print(f"▶ Max posts       : {args.limit}")
+    print(f"▶ Browser         : {'headed' if args.headed else 'headless'}\n")
+
+    stop_event = threading.Event()
+    label = "Collecting profiles..." if profile_mode else "Collecting posts..."
+    spinner_thread = threading.Thread(
+        target=spinner_task,
+        args=(f"[SCRAPING] {label}", stop_event),
+        daemon=True,
+    )
+    spinner_thread.start()
+
+    try:
+        with suppress_stdout():
+            if profile_mode:
+                if len(handles) == 1:
+                    data = scrape_profile(
+                        handles[0],
+                        limit=args.limit,
+                        profile_dir=args.profile,
+                        headless=not args.headed,
+                    )
+                    data = [data]
+                else:
+                    data = scrape_profiles(
+                        handles,
+                        limit=args.limit,
+                        profile_dir=args.profile,
+                        headless=not args.headed,
+                    )
+            else:
+                data = scrape_hashtag(
+                    hashtag=args.tag,
+                    limit=args.limit,
+                    profile_dir=args.profile,
+                    headless=not args.headed,
+                )
+    except Exception as e:
+        stop_event.set()
+        spinner_thread.join()
+        print(f"[!] {e}")
+        sys.exit(1)
+
+    stop_event.set()
+    spinner_thread.join()
+
+    if profile_mode:
+        print(f"[DONE] {len(data)} profile(s) processed")
+        if not data:
+            print("[!] No profiles scraped.")
+            return
+
+        for profile in data:
+            if profile.get("error"):
+                print(f"[!] @{profile['handle']}: {profile['error']}")
+            else:
+                preview_profile(profile)
+
+        # Flat summary always useful for engagement work
+        print("\n--- ENGAGEMENT SUMMARY ---")
+        print(f"{'handle':<20} {'followers':>12} {'posts':>6} {'avg_eng':>12} {'ER%':>10}")
+        for profile in data:
+            print(
+                f"@{profile.get('handle', '?'):<19} "
+                f"{str(profile.get('followers') or '-'):>12} "
+                f"{profile.get('posts_scraped', 0):>6} "
+                f"{str(profile.get('avg_engagement') or '-'):>12} "
+                f"{str(profile.get('engagement_rate_pct') or '-'):>10}"
+            )
+
+        tag = handles[0] if len(handles) == 1 else "profiles"
+        filename = f"result_{tag}"
+        save_data(data, filename=filename)
+        print(f"\n[✓] Data saved in {filename}.json / {filename}.csv")
+        return
+
+    print(f"[DONE] {len(data)} posts successfully scraped")
+
+    if not data:
+        print("[!] No data collected.")
+        return
+
+    preview_posts(data)
+
+    filename = f"result_{args.tag}"
+    save_data(data, filename=filename)
+    print(f"[✓] Data saved in {filename}.json / {filename}.csv")

insta_collect/exporter.py

@@ -0,0 +1,6 @@
+import pandas as pd
+
+def save_to_xlsx(data, filename="instagram_comments.xlsx"):
+    df = pd.DataFrame(data)
+    df.to_excel(filename, index=False)
+    return filename

insta_collect/html_parser.py

@@ -0,0 +1,76 @@
+from bs4 import BeautifulSoup
+import re
+
+
+def parse_instagram_html(file_path, min_len=10):
+
+    with open(file_path, "r", encoding="utf-8") as f:
+        soup = BeautifulSoup(f, "html.parser")
+
+    results = []
+    current_user = None
+    caption_done = False
+
+    UI_TEXTS = {
+        "Instagram Lite",
+        "Log in",
+        "Sign up",
+        "Open in app",
+        "Reply",
+        "Like",
+        "See translation",
+        "View all replies",
+        "View replies",
+        "More",
+        "Follow",
+        "Suggested for you"
+    }
+
+    def is_username(text):
+        return (
+            text.islower()
+            and " " not in text
+            and 3 <= len(text) <= 30
+            and text not in UI_TEXTS
+        )
+
+    def is_timestamp(text):
+        return re.match(r"^\d+[smhdw]$", text) is not None
+
+    for span in soup.find_all("span"):
+
+        text = span.get_text(strip=True)
+
+        if not text:
+            continue
+
+        if text in UI_TEXTS:
+            continue
+
+        if is_timestamp(text):
+            continue
+
+        if is_username(text):
+            current_user = text
+            continue
+
+        if current_user and len(text) >= min_len:
+
+            entry_type = "caption" if not caption_done else "comment"
+            caption_done = True
+
+            clean_text = re.sub(r"Verified\d+[smhdw]", "", text)
+
+            results.append({
+                "username": current_user,
+                "text": clean_text,
+                "type": entry_type,
+                "char_len": len(clean_text)
+            })
+
+            current_user = None
+
+    print(f"[DEBUG] Parsed {len(results)} entries")
+
+    return results
+  

insta_collect/insta-collect.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+import argparse
+import json
+from html_parser import parse_instagram_html
+from exporter import save_to_xlsx
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Instagram HTML Comment Extractor")
+    parser.add_argument("html_file", help="Path to Instagram HTML file")
+    parser.add_argument(
+        "--preview",
+        type=int,
+        default=0,
+        help="Show N comments as preview in terminal"
+    )
+
+    args = parser.parse_args()
+
+    
+    data = parse_instagram_html(args.html_file)
+
+    
+    json_output = "instagram_comments.json"
+    with open(json_output, "w", encoding="utf-8") as f:
+        json.dump(data, f, ensure_ascii=False, indent=2)
+
+    
+    xlsx_output = save_to_xlsx(data)
+
+    print(f"[+] Total entries saved: {len(data)}")
+    print(f"[+] JSON output: {json_output}")
+    print(f"[+] XLSX output: {xlsx_output}")
+
+    
+    if args.preview > 0:
+        print("\n--- PREVIEW ---")
+        for i, d in enumerate(data[:args.preview], 1):
+            print(f"{i}. @{d['username']}: {d['text'][:100]}")
+
+
+if __name__ == "__main__":
+    main()

insta_collect/main.py

@@ -0,0 +1,9 @@
+from src.scraper import scrape_hashtag
+from src.saver import save_to_csv, save_to_json
+
+if __name__ == "__main__":
+    hashtag = input("Enter hashtag (without #): ")
+    limit = int(input("Limit posts: "))
+    results = scrape_hashtag(hashtag, limit)
+    save_to_csv(results, hashtag)
+    save_to_json(results, hashtag)

insta_collect/saver.py

@@ -0,0 +1,20 @@
+import json
+import csv
+
+def save_data(data, filename="results"):
+    json_path = f"{filename}.json"
+    csv_path = f"{filename}.csv"
+
+
+    with open(json_path, "w", encoding="utf-8") as f:
+        json.dump(data, f, ensure_ascii=False, indent=2)
+
+
+    keys = data[0].keys()
+    with open(csv_path, "w", encoding="utf-8", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=keys)
+        writer.writeheader()
+        writer.writerows(data)
+
+    print(f"[✓] Saved to {json_path} & {csv_path}")
+

insta_collect/scraper.py

@@ -0,0 +1,454 @@
+from playwright.sync_api import sync_playwright
+import time
+import random
+import re
+
+# ================= UTIL =================
+def count_sentences(text):
+    if not text:
+        return 0
+    sentences = re.split(r"[.!?]+", text)
+    return len([s for s in sentences if s.strip()])
+
+def extract_hashtags_mentions(text):
+    if not text:
+        return [], []
+    hashtags = re.findall(r"#(\w+)", text)
+    mentions = re.findall(r"@(\w+)", text)
+    return hashtags, mentions
+
+def parse_count(text):
+    """Parse Instagram-style counts: '1,234', '12K', '1.5M', '2B' -> int."""
+    if text is None:
+        return None
+    if isinstance(text, (int, float)):
+        return int(text)
+    s = str(text).strip().replace(",", "").replace(" ", "")
+    if not s:
+        return None
+    mult = 1
+    if s[-1] in "KkMmBb":
+        suffix = s[-1].upper()
+        s = s[:-1]
+        mult = {"K": 1_000, "M": 1_000_000, "B": 1_000_000_000}[suffix]
+    try:
+        return int(float(s) * mult)
+    except ValueError:
+        return None
+
+def parse_post_metrics(meta_desc):
+    """Extract likes/comments/views from og:description prefix like '679K likes, 6,719 comments - user...'."""
+    likes = comments = views = None
+    if not meta_desc:
+        return likes, comments, views
+    m = re.search(r"([\d,.]+[KMB]?)\s*likes?", meta_desc, re.I)
+    if m:
+        likes = parse_count(m.group(1))
+    m = re.search(r"([\d,.]+[KMB]?)\s*comments?", meta_desc, re.I)
+    if m:
+        comments = parse_count(m.group(1))
+    m = re.search(r"([\d,.]+[KMB]?)\s*views?", meta_desc, re.I)
+    if m:
+        views = parse_count(m.group(1))
+    return likes, comments, views
+
+def parse_profile_og(meta_desc):
+    """Parse '104M Followers, 94 Following, 4,966 Posts - ...' -> followers, following, posts_count."""
+    followers = following = posts_count = None
+    if not meta_desc:
+        return followers, following, posts_count
+    m = re.search(r"([\d,.]+[KMB]?)\s*Followers", meta_desc, re.I)
+    if m:
+        followers = parse_count(m.group(1))
+    m = re.search(r"([\d,.]+[KMB]?)\s*Following", meta_desc, re.I)
+    if m:
+        following = parse_count(m.group(1))
+    m = re.search(r"([\d,.]+[KMB]?)\s*Posts", meta_desc, re.I)
+    if m:
+        posts_count = parse_count(m.group(1))
+    return followers, following, posts_count
+
+def engagement_rate(posts, followers):
+    """
+    Average ER% = mean(likes+comments per post, plus views for posts
+    that report them) / followers * 100. Returns None rates when data
+    is insufficient.
+    """
+    scores = []
+    for p in posts:
+        likes = p.get("likes") or 0
+        comments = p.get("comments") or 0
+        views = p.get("views") or 0
+        score = likes + comments
+        # only fold views in when the post actually reported them
+        if p.get("views") is not None:
+            score += views
+        scores.append(score)
+    if not scores:
+        return None, None
+    avg = sum(scores) / len(scores)
+    er = None
+    if followers and followers > 0:
+        er = round(avg / followers * 100, 4)
+    return round(avg, 2), er
+
+def is_login_wall(page):
+    return (
+        page.locator("input[name='username']").count() > 0
+        or "login" in page.url
+    )
+
+def dismiss_cookie_banner(page):
+    for sel in (
+        "button:has-text('Allow all cookies')",
+        "button:has-text('Accept all')",
+        "button:has-text('Allow all')",
+    ):
+        btn = page.locator(sel)
+        if btn.count() > 0:
+            try:
+                btn.first.click(timeout=3000)
+                time.sleep(1)
+                return True
+            except Exception:
+                pass
+    return False
+
+def dismiss_signup_modal(page):
+    """Best-effort close of the 'Sign up / Log in' modal on post pages."""
+    for sel in (
+        "div[role='dialog'] svg[aria-label='Close']",
+        "div[role='dialog'] button:has-text('Not now')",
+        "svg[aria-label='Close']",
+    ):
+        el = page.locator(sel)
+        if el.count() > 0:
+            try:
+                el.first.click(timeout=2000)
+                time.sleep(0.5)
+                return True
+            except Exception:
+                pass
+    return False
+
+# ================= DETAIL SCRAPER =================
+def get_post_details(page, url):
+    data = {
+        "caption": None,
+        "caption_status": None,
+        "timestamp": None,
+        "is_video": False,
+        "hashtags": [],
+        "mentions": [],
+        "likes": None,
+        "comments": None,
+        "views": None,
+    }
+
+    try:
+        print(f"   [..] Visiting: {url}")
+        page.goto(url, wait_until="domcontentloaded", timeout=20000)
+        time.sleep(1.5)
+
+        if is_login_wall(page):
+            data["caption_status"] = "login_required"
+            return data
+
+        # Signup modal is non-blocking for meta extraction; try to close it.
+        dismiss_signup_modal(page)
+
+        # ---------- META ----------
+        # og:description looks like:
+        #   12K likes, 30 comments - username on January 1, 2026: "caption text".
+        caption_text = None
+        username = None
+        meta_desc = None
+        try:
+            meta_desc = page.locator('meta[property="og:description"]').get_attribute("content")
+            if meta_desc:
+                # username sits before " on <date>: "
+                m_user = re.search(r"-\s*([A-Za-z0-9._]+)\s+on\s+", meta_desc)
+                if m_user:
+                    username = m_user.group(1)
+                likes, comments, views = parse_post_metrics(meta_desc)
+                data["likes"] = likes
+                data["comments"] = comments
+                data["views"] = views
+                if ": " in meta_desc:
+                    caption_text = meta_desc.split(": ", 1)[1].strip()
+                    # og:description wraps the caption in quotes: ...: "text".
+                    if caption_text.startswith('"'):
+                        caption_text = caption_text[1:]
+                    if caption_text.endswith('".'):
+                        caption_text = caption_text[:-2]
+                    elif caption_text.endswith('"'):
+                        caption_text = caption_text[:-1]
+                    caption_text = caption_text.strip()
+                else:
+                    caption_text = meta_desc.strip()
+        except:
+            pass
+
+        # ---------- DOM FALLBACK ----------
+        if not caption_text:
+            try:
+                candidates = page.locator("article span").all()
+                longest = None
+                for el in candidates:
+                    txt = el.inner_text().strip()
+                    if len(txt) < 80:
+                        continue
+                    if not longest or len(txt) > len(longest):
+                        longest = txt
+                caption_text = longest
+            except:
+                pass
+
+        # ---------- VALIDASI ----------
+        # Accept any non-empty caption (short reel captions often have no sentence punctuation).
+        if not caption_text or not caption_text.strip():
+            data["caption_status"] = "no_text"
+            return data
+
+        sentence_count = count_sentences(caption_text)
+        data["caption"] = caption_text
+        data["caption_status"] = "long_text" if sentence_count >= 8 else "short_text"
+        if username:
+            data["username"] = username
+
+        # ---------- HASHTAG & MENTION ----------
+        hashtags, mentions = extract_hashtags_mentions(caption_text)
+        data["hashtags"] = hashtags
+        data["mentions"] = mentions
+
+        # ---------- TIMESTAMP ----------
+        try:
+            time_el = page.locator("time").first
+            if time_el.count() > 0:
+                data["timestamp"] = time_el.get_attribute("datetime")
+        except:
+            pass
+
+        # ---------- VIDEO FLAG ----------
+        if page.locator("video").count() > 0 or "/reel/" in url:
+            data["is_video"] = True
+
+    except Exception as e:
+        data["caption_status"] = "error"
+        print(f"   [!] Error: {e}")
+
+    return data
+
+# ================= HASHTAG SCRAPER =================
+def scrape_hashtag(hashtag, limit=10, profile_dir="ig_profile", headless=True):
+    results = []
+
+    with sync_playwright() as p:
+        context = p.chromium.launch_persistent_context(
+            user_data_dir=profile_dir,
+            headless=headless,
+            viewport={"width": 1280, "height": 800}
+        )
+        page = context.new_page()
+
+        print(f"[1/3] Scanning hashtag #{hashtag}")
+        page.goto(f"https://www.instagram.com/explore/tags/{hashtag}/", wait_until="domcontentloaded")
+        time.sleep(3)
+        dismiss_cookie_banner(page)
+
+        if is_login_wall(page):
+            if headless:
+                context.close()
+                raise RuntimeError(
+                    "Instagram login wall blocked the hashtag page in headless mode. "
+                    "Re-run with --headed and log in once (session is saved in --profile), "
+                    "or try again later."
+                )
+            print("[!] Login is required. Please log in in an open browser....")
+            input("Press Enter after login is complete to continue....")
+
+        collected_links = set()
+        # Logged-out grids often only expose /reel/ links; accept both.
+        max_scrolls = 25
+        for _ in range(max_scrolls):
+            for selector in ("a[href*='/p/']", "a[href*='/reel/']"):
+                for a in page.locator(selector).all():
+                    href = a.get_attribute("href")
+                    if href:
+                        # strip query params
+                        path = href.split("?")[0]
+                        collected_links.add("https://www.instagram.com" + path)
+            if len(collected_links) >= limit * 2:
+                break
+            page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+            time.sleep(1.5)
+
+        print(f"[INFO] Found {len(collected_links)} links")
+
+        print("[2/3] Scraping detail posts")
+        for i, link in enumerate(collected_links):
+            if len(results) >= limit:
+                break
+            print(f"   -> {i+1}/{len(collected_links)}")
+            d = get_post_details(page, link)
+            # Keep reels/videos: logged-out grids only serve reels.
+            if d["caption_status"] in ("login_required", "error"):
+                continue
+            results.append({"url": link, **d, "source_tag": hashtag})
+            time.sleep(random.uniform(1.5, 2.5))
+
+        context.close()
+
+    print(f"[DONE] Collected {len(results)} posts from #{hashtag}")
+    return results
+
+
+# ================= PROFILE SCRAPER =================
+def _collect_profile_links(page, handle, limit):
+    """Scroll profile grid; return post URLs in DOM order (recent first)."""
+    handle_l = handle.lower().lstrip("@")
+    collected = []
+    seen = set()
+    max_scrolls = 20
+    for _ in range(max_scrolls):
+        for selector in ("a[href*='/p/']", "a[href*='/reel/']"):
+            for a in page.locator(selector).all():
+                href = a.get_attribute("href")
+                if not href:
+                    continue
+                path = href.split("?")[0]
+                if not path.startswith("/"):
+                    path = "/" + path
+                # Prefer posts on this handle's paths; still allow collab posts later.
+                url = "https://www.instagram.com" + path
+                if url in seen:
+                    continue
+                seen.add(url)
+                collected.append(url)
+        if len(collected) >= limit * 2:
+            break
+        page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+        time.sleep(1.2)
+
+    # Prefer same-handle posts first, preserve discovery order within each group
+    same, other = [], []
+    for u in collected:
+        if f"/{handle_l}/" in u.lower():
+            same.append(u)
+        else:
+            other.append(u)
+    return (same + other)[: limit * 2]
+
+
+def scrape_profile(handle, limit=10, profile_dir="ig_profile", headless=True):
+    """
+    Fetch recent posts for a profile and compute engagement rate.
+    Returns a dict: handle, followers, posts, avg_engagement, engagement_rate_pct, ...
+    """
+    handle = handle.strip().lstrip("@")
+
+    with sync_playwright() as p:
+        context = p.chromium.launch_persistent_context(
+            user_data_dir=profile_dir,
+            headless=headless,
+            viewport={"width": 1280, "height": 800},
+        )
+        page = context.new_page()
+        try:
+            profile = _scrape_profile_with_page(page, handle, limit, headless)
+        finally:
+            context.close()
+
+    print(f"[DONE] @{handle}: {len(profile['posts'])} posts, "
+          f"ER={profile.get('engagement_rate_pct')}%")
+    return profile
+
+
+def _scrape_profile_with_page(page, handle, limit, headless):
+    handle = handle.strip().lstrip("@")
+    posts = []
+    print(f"[1/3] Opening profile @{handle}")
+    page.goto(f"https://www.instagram.com/{handle}/", wait_until="domcontentloaded", timeout=30000)
+    time.sleep(3)
+    dismiss_cookie_banner(page)
+
+    if is_login_wall(page):
+        if headless:
+            raise RuntimeError(
+                f"Login wall blocked @{handle} in headless mode. "
+                "Re-run with --headed and log in once, or try later."
+            )
+        print("[!] Login is required. Please log in in an open browser....")
+        input("Press Enter after login is complete to continue....")
+
+    meta_desc = None
+    try:
+        meta_desc = page.locator('meta[property="og:description"]').get_attribute("content")
+    except Exception:
+        pass
+    followers, following, posts_count = parse_profile_og(meta_desc)
+    print(f"[INFO] followers={followers} following={following} posts={posts_count}")
+
+    links = _collect_profile_links(page, handle, limit)
+    print(f"[INFO] Found {len(links)} grid links")
+
+    print("[2/3] Scraping recent posts")
+    for i, link in enumerate(links):
+        if len(posts) >= limit:
+            break
+        print(f"   -> {i+1}/{len(links)}")
+        d = get_post_details(page, link)
+        if d["caption_status"] in ("login_required", "error"):
+            continue
+        posts.append({"url": link, **d, "source_handle": handle})
+        time.sleep(random.uniform(1.5, 2.5))
+
+    avg_eng, er = engagement_rate(posts, followers)
+    return {
+        "handle": handle,
+        "url": f"https://www.instagram.com/{handle}/",
+        "followers": followers,
+        "following": following,
+        "posts_total": posts_count,
+        "posts_scraped": len(posts),
+        "avg_engagement": avg_eng,
+        "engagement_rate_pct": er,
+        "posts": posts,
+    }
+
+
+def scrape_profiles(handles, limit=10, profile_dir="ig_profile", headless=True):
+    """Scrape multiple profiles in one browser session."""
+    results = []
+    handles = [h.strip().lstrip("@") for h in handles if h and h.strip()]
+
+    with sync_playwright() as p:
+        context = p.chromium.launch_persistent_context(
+            user_data_dir=profile_dir,
+            headless=headless,
+            viewport={"width": 1280, "height": 800},
+        )
+        page = context.new_page()
+        try:
+            for handle in handles:
+                try:
+                    profile = _scrape_profile_with_page(page, handle, limit, headless)
+                    results.append(profile)
+                except Exception as e:
+                    print(f"[!] @{handle} failed: {e}")
+                    results.append({
+                        "handle": handle,
+                        "error": str(e),
+                        "posts": [],
+                        "posts_scraped": 0,
+                        "engagement_rate_pct": None,
+                        "avg_engagement": None,
+                        "followers": None,
+                    })
+                time.sleep(random.uniform(2.0, 3.5))
+        finally:
+            context.close()
+
+    return results
+
+

insta_collect_data-0.2.0.dist-info/METADATA

@@ -0,0 +1,313 @@
+Metadata-Version: 2.4
+Name: insta-collect-data
+Version: 0.2.0
+Summary:  Modular Instagram Data Collector
+Author-email: Alam Mahadika <alam.mahadika.psm@umy.ac.id>
+License: MIT
+Project-URL: Homepage, https://github.com/Alammahadika/insta-collect
+Keywords: instagram,scraper,hashtag,profile,engagement,analytics,comments
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: beautifulsoup4>=4.12.2
+Requires-Dist: lxml>=4.9.3
+Requires-Dist: pandas>=2.1.0
+Requires-Dist: openpyxl>=3.1.2
+Requires-Dist: requests>=2.31.0
+Requires-Dist: tqdm>=4.65.0
+Requires-Dist: playwright>=1.40.0
+
+# insta-collect-data: Modular Instagram Data Collector
+
+## Project Status: In Development
+
+insta-collect-data is a Playwright-based collector for Instagram **hashtags** and **profiles**. It supports photo and carousel posts as well as Reels, extracts post-level engagement metrics, and can compute per-profile engagement rates for influencer research.
+
+The live scraper is functional but depends on Instagram's public markup. Expect occasional breakage when Instagram changes its HTML or tightens its login wall. The HTML comment parser is still experimental.
+
+***
+
+## Project Overview
+
+insta-collect-data is a small, modular Python project that automates Instagram in a real browser via **Playwright**. It is intended for structured data collection for analysis and research: hashtag sampling, creator profiling, engagement comparisons, and small-batch archival work.
+
+***
+
+## Current Key Features
+
+* **Hashtag Scraping:** Collect posts from an explore/tag page.
+* **Profile Scraping:** Collect recent posts for one handle or a list of handles in a single browser session.
+* **Engagement Metrics:** Extract `likes`, `comments`, and `views` (when Instagram exposes them), plus profile `followers`, `following`, and total `posts`.
+* **Engagement Rate:** Compute average engagement per post and ER% for each scraped profile.
+* **Reels Included:** Both `/p/` posts and `/reel/` links are collected. Older versions filtered videos out; that is no longer the case.
+* **Rich Post Data:** `url`, `username`, `caption`, `caption_status`, `timestamp`, `is_video`, `hashtags`, `mentions`, `likes`, `comments`, `views`.
+* **Session Reuse:** Log in once with a visible browser and reuse the saved Playwright profile on later headless runs.
+* **Login-Wall Handling:** Detects Instagram login walls, dismisses cookie/sign-up modals when possible, and gives an actionable error in headless mode.
+* **Dual Output:** Every run writes both **JSON** and **CSV**.
+* **HTML Comment Extraction:** Parse comments from a previously saved Instagram HTML page into JSON and XLSX.
+
+***
+
+## Installation
+
+```bash
+pip install insta-collect-data
+```
+
+### or install directly from GitHub
+
+```bash
+pip install git+https://github.com/Alammahadika/insta-collect.git
+```
+
+This distribution installs the `insta_collect` Python package and the `insta-collect` CLI command.
+
+> `insta-collect-data` is the continuation of the earlier `insta-collect` package (last release 0.1.2) under a new PyPI name.
+
+Requirements: **Python 3.10+** and a Chromium browser for Playwright.
+
+### Playwright Setup (Required)
+
+Install the browser binaries once after installing the package:
+
+```bash
+playwright install chromium
+```
+
+### First-Time Login (Recommended)
+
+Instagram often blocks logged-out scraping with a login wall. Log in once using a visible browser; the session is stored in the Playwright profile directory (`ig_profile` by default) and reused afterwards.
+
+```bash
+insta-collect --user natgeo --limit 6 --headed
+```
+
+Log in manually in the window that opens, then let the run finish. Later runs work headless by default.
+
+> Do not commit the `ig_profile/` folder — it contains your authenticated session.
+
+***
+
+## Usage
+
+The installed CLI is `insta-collect`. It has two modes, and they are mutually exclusive.
+
+### Arguments
+
+| Argument | Description | Default |
+|---|---|---|
+| `--tag` | Hashtag to scrape (without `#`) | — |
+| `--user` | One handle, or a comma-separated list (`natgeo,nasa`) | — |
+| `--users-file` | Text file with one handle per line; `#` starts a comment | — |
+| `--limit` | Max posts per target (6–12 is a good range for engagement analysis) | `10` |
+| `--profile` | Playwright persistent profile directory (session storage) | `ig_profile` |
+| `--headed` | Show the browser window (first-time login / debugging) | headless |
+
+Provide either `--tag` **or** `--user` / `--users-file`.
+
+### Hashtag mode
+
+```bash
+insta-collect --tag donaldtrump --limit 10
+```
+
+### Single profile
+
+```bash
+insta-collect --user natgeo --limit 10
+```
+
+### Multiple profiles
+
+```bash
+insta-collect --user natgeo,nasa --limit 8
+```
+
+Or from a file (`handles.txt`):
+
+```text
+# one handle per line
+natgeo
+nasa
+@somecreator
+```
+
+```bash
+insta-collect --users-file handles.txt --limit 8
+```
+
+### Visible browser
+
+```bash
+insta-collect --tag donaldtrump --limit 10 --headed
+```
+
+***
+
+## Output
+
+Each run writes two files in the current directory.
+
+| Mode | Files |
+|---|---|
+| Hashtag | `result_<tag>.json`, `result_<tag>.csv` |
+| Single profile | `result_<handle>.json`, `result_<handle>.csv` |
+| Multiple profiles | `result_profiles.json`, `result_profiles.csv` |
+
+### Post fields (hashtag mode)
+
+```json
+[
+  {
+    "url": "https://www.instagram.com/p/DS3UWMfCRA5/",
+    "username": "cdn37",
+    "caption": "El presidente de Estados Unidos...",
+    "caption_status": "long_text",
+    "timestamp": "2025-12-29T22:35:50.000Z",
+    "is_video": false,
+    "hashtags": ["CDN", "Noticias", "DonaldTrump"],
+    "mentions": [],
+    "likes": 1200,
+    "comments": 34,
+    "views": null,
+    "source_tag": "donaldtrump"
+  }
+]
+```
+
+`caption_status` is one of `long_text`, `short_text`, `no_text`, `login_required`, or `error`.
+
+### Profile fields (profile mode)
+
+```json
+[
+  {
+    "handle": "natgeo",
+    "url": "https://www.instagram.com/natgeo/",
+    "followers": 283000000,
+    "following": 120,
+    "posts_total": 31000,
+    "posts_scraped": 8,
+    "avg_engagement": 45210.5,
+    "engagement_rate_pct": 0.016,
+    "posts": [
+      {
+        "url": "https://www.instagram.com/p/...",
+        "username": "natgeo",
+        "caption": "...",
+        "likes": 40000,
+        "comments": 5210,
+        "views": null,
+        "source_handle": "natgeo"
+      }
+    ]
+  }
+]
+```
+
+### Engagement rate definition
+
+For each scraped post, engagement is `likes + comments`, plus `views` when that post reported a view count. Then:
+
+```text
+avg_engagement     = mean(per-post engagement)
+engagement_rate_%  = avg_engagement / followers * 100
+```
+
+Instagram displays rounded counts (`12K`, `1.5M`), so parsed metrics are approximations, not exact values.
+
+### CSV note
+
+CSV output is flat. In profile mode the nested `posts` list is written as a single serialized cell per profile. Use the JSON file when you need post-level analysis.
+
+***
+
+## How It Works
+
+1. **Session resumption:** Opens Chromium with the persistent `--profile` directory, reusing any saved login.
+2. **Link collection:** Navigates to the hashtag or profile page, dismisses the cookie banner, and scrolls until enough `/p/` and `/reel/` links are found (query strings stripped, duplicates removed).
+3. **Post extraction:** Visits each post, closes the sign-up modal when present, and reads `og:description` plus page selectors for caption, username, timestamp, and engagement metrics.
+4. **Filtering:** Skips posts that hit a login wall or throw an error. Reels and videos are kept.
+5. **Aggregation:** In profile mode, computes average engagement and ER% from the scraped posts.
+6. **Export:** Writes JSON and CSV.
+
+***
+
+## Limitations
+
+* Instagram's login wall is the most common failure. If a headless run reports it, rerun once with `--headed` and log in.
+* Logged-out hashtag grids often expose only Reels; results may be Reel-heavy.
+* Metrics are rounded by Instagram (`K`/`M`/`B`) and may be missing entirely.
+* Keep `--limit` modest and expect random delays between requests. This is not a high-volume crawler.
+* The persistent profile directory contains credentials; exclude it from version control.
+
+***
+
+## Naming Conventions: `_` vs `-` in Python Projects
+
+| Context | Use | Example |
+|---|---|---|
+| Package name (PyPI / GitHub) | `-` (hyphen) | insta-collect-data |
+| Python package directory | `_` (underscore) | insta_collect |
+| Python import statement | `_` (underscore) | import insta_collect |
+| CLI execution via module | `_` (underscore) | python -m insta_collect.cli |
+| Script / file name | flexible | insta-collect.py |
+
+***
+
+## Collecting Comments from an HTML File
+
+In addition to live scraping, insta-collect-data can parse comments from a saved Instagram HTML page made in a logged-in browser.
+
+This is useful when:
+
+* You already saved the page and want structured comments.
+* You want to avoid live requests.
+* You need an XLSX deliverable.
+
+### Example
+
+```bash
+cd path/to/insta_collect   # the package directory in a repo checkout
+python3 insta-collect.py ~/Downloads/obama.html --preview 20
+```
+
+The script writes `instagram_comments.json` and `instagram_comments.xlsx` to the current directory.
+
+### Output
+
+```text
+[+] Total entries saved: 239
+[+] JSON output: instagram_comments.json
+[+] XLSX output: instagram_comments.xlsx
+```
+
+### Comment fields
+
+* `username`
+* `text` (`comment_text` in the README of older releases)
+* `timestamp` when available
+* `source_file`
+
+### Notes
+
+* Output filenames are fixed; existing files are overwritten.
+* `--preview` only affects terminal output, not the saved data.
+* This feature is experimental and may change.
+
+***
+
+## Roadmap
+
+* Live comment scraping.
+* XLSX export for live post/profile runs.
+* Retry/backoff and proxy support for larger batches.
+* Optional pagination beyond the recent grid.
+
+Completed in this release: profile scraping, engagement metrics/ER%, CSV output, Reels support, headless-by-default with persistent login.
+
+***
+
+## Ethical Use Notice
+
+This tool is intended strictly for academic research, journalism, and public-interest analysis.
+Users are responsible for complying with Instagram's Terms of Service and applicable laws.

insta_collect_data-0.2.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+insta_collect/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+insta_collect/cli.py,sha256=XzgxLfB9nPjwJKD33l5BxN60ZQIvr48y7NRcL9sWXCk,7624
+insta_collect/exporter.py,sha256=RYqxiPublZBcvBcjVxMAGT7AXn0JYk00XULW3dRk7zo,168
+insta_collect/html_parser.py,sha256=2RGJx1OjigrQEarIvQvbdiNM4moO216eu_5SsxfQI3A,1665
+insta_collect/insta-collect.py,sha256=eUcl7jLQixFqirSSutuaddWsW5sf7qliQEn9oRNfcz4,1103
+insta_collect/main.py,sha256=WR1xddYoSTCyNM-bLJ0yU6aCLfH_MAuTybAGmxLQf8A,320
+insta_collect/saver.py,sha256=dwNanzImiRPn0ptoGQa9IeUFJSxSNlrGsB2bjgQkrMs,507
+insta_collect/scraper.py,sha256=xHSFzuSqrfPwyYBb4m1ddxcP0v_GymUkNDsYyJiovis,16011
+insta_collect_data-0.2.0.dist-info/METADATA,sha256=XF4A61iu8A_LX7JBw37FKtcMU1iRlKZobOBDn1uHoaA,10143
+insta_collect_data-0.2.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+insta_collect_data-0.2.0.dist-info/entry_points.txt,sha256=fGFkxnBYUF4ZSgOjozw6ZH3KsopfXicuhEDverV8GJE,57
+insta_collect_data-0.2.0.dist-info/top_level.txt,sha256=wtLMtis7WD-MS4fzNzNZbm1iJeNkKp3mX_PZwnEmD1U,14
+insta_collect_data-0.2.0.dist-info/RECORD,,

insta_collect_data-0.2.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

insta_collect_data-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+insta-collect = insta_collect.cli:main

insta_collect_data-0.2.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+insta_collect