suur-data 1.0.0__py3-none-any.whl

suur_data/__init__.py

@@ -0,0 +1,233 @@
+"""
+suur_data
+========
+A smart data ingestion and tokenization pipeline.
+
+Python API usage:
+    from suur_data import suur_data
+
+    tokens = suur_data("https://example.com/article.html")
+    tokens = suur_data("my_corpus.pdf", topic="quantum computing")
+    tokens = suur_data("data.txt", tokenizer="custom", vocab_size=4000)
+
+CLI usage:
+    suur_data fetch <source> [OPTIONS]
+
+    Options:
+      --topic TEXT            Topic/subject to filter content by
+      --tokenizer [pretrained|custom]
+      --model TEXT            Pretrained model name (default: gpt2)
+      --vocab-size INT        BPE vocab size for custom mode (default: 8000)
+      --threshold FLOAT       Relevance threshold 0.0–1.0 (default: 0.05)
+      --save-dir PATH         Where to save tokenizer artifacts
+      --no-filter             Skip the relevance filter
+      --verbose / --quiet
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import json
+import tempfile
+from typing import List, Optional
+
+import click
+
+from .ingest import ingest
+from .filter import filter_chunks
+from .tokenizer import tokenize_pretrained, tokenize_custom
+
+
+# ---------------------------------------------------------------------------
+# Python API
+# ---------------------------------------------------------------------------
+
+def suur_data(
+    data_location: str,
+    topic: str = "",
+    tokenizer: str = "pretrained",
+    model: str = "gpt2",
+    vocab_size: int = 8000,
+    threshold: float = 0.05,
+    save_dir: Optional[str] = None,
+    no_filter: bool = False,
+    verbose: bool = True,
+) -> List[int]:
+    """
+    End-to-end pipeline: ingest → filter → tokenize.
+
+    Parameters
+    ----------
+    data_location : str
+        URL or local file path. Supports .txt, .pdf, .docx, .csv,
+        .json, .html, .htm, .epub, .md, .rst
+    topic : str
+        Subject/keyword for the neural relevance filter.
+        Leave empty to skip filtering.
+    tokenizer : str
+        "pretrained" (default) or "custom".
+    model : str
+        HuggingFace model name for pretrained mode (default: "gpt2").
+    vocab_size : int
+        BPE vocabulary size for custom mode (default: 8000).
+    threshold : float
+        Cosine similarity cutoff for relevance filter (default: 0.05).
+    save_dir : str | None
+        Directory to save tokenizer files. None = don't save.
+    no_filter : bool
+        If True, skip the relevance filter and tokenize everything.
+    verbose : bool
+        Print progress information.
+
+    Returns
+    -------
+    List[int]
+        List of integer token IDs.
+    """
+    tmp = tempfile.mkdtemp(prefix="suur_data_")
+
+    # --- Stage 1: Ingest ---
+    if verbose:
+        print(f"\n[suur_data] Stage 1 — Ingesting: {data_location}")
+    raw_text = ingest(data_location, tmp_dir=tmp)
+
+    # --- Stage 2: Filter ---
+    if no_filter or not topic.strip():
+        if verbose:
+            if no_filter:
+                print("[suur_data] Stage 2 — Filter: SKIPPED (--no-filter)")
+            else:
+                print("[suur_data] Stage 2 — Filter: SKIPPED (no topic given)")
+        filtered_text = raw_text
+    else:
+        if verbose:
+            print(f"[suur_data] Stage 2 — Neural Filter (topic: '{topic}')")
+        filtered_text, _ = filter_chunks(
+            raw_text,
+            topic=topic,
+            threshold=threshold,
+            verbose=verbose,
+        )
+
+    # --- Stage 3: Tokenize ---
+    if verbose:
+        print(f"[suur_data] Stage 3 — Tokenizing ({tokenizer} mode)")
+
+    if tokenizer == "custom":
+        tokens = tokenize_custom(filtered_text, vocab_size=vocab_size, save_dir=save_dir)
+    else:
+        tokens = tokenize_pretrained(filtered_text, model_name=model, save_dir=save_dir)
+
+    if verbose:
+        print(f"\n[suur_data] Done. Total tokens: {len(tokens):,}\n")
+
+    return tokens
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+@click.group()
+@click.version_option("1.0.0", prog_name="suur-data")
+def cli():
+    """
+    \b
+    Suur Data — intelligent data ingestion and tokenization pipeline.
+
+    Fetch any text source, filter it by topic using TF-IDF relevance
+    scoring, then tokenize with a pretrained or custom BPE tokenizer.
+    """
+
+
+@cli.command("fetch")
+@click.argument("source")
+@click.option("--topic",      default="",           show_default=True,  help="Topic for relevance filtering.")
+@click.option("--tokenizer",  default="pretrained", show_default=True,  type=click.Choice(["pretrained", "custom"]), help="Tokenizer mode.")
+@click.option("--model",      default="gpt2",       show_default=True,  help="Pretrained model name (HuggingFace).")
+@click.option("--vocab-size", default=8000,         show_default=True,  help="BPE vocab size for custom tokenizer.")
+@click.option("--threshold",  default=0.05,         show_default=True,  help="Relevance filter threshold (0.0–1.0).")
+@click.option("--save-dir",   default=None,                             help="Directory to save tokenizer artifacts.")
+@click.option("--no-filter",  is_flag=True,         default=False,      help="Skip the relevance filter.")
+@click.option("--output",     default=None,                             help="Save token IDs to a JSON file.")
+@click.option("--quiet",      is_flag=True,         default=False,      help="Suppress progress output.")
+def fetch_cmd(source, topic, tokenizer, model, vocab_size, threshold,
+              save_dir, no_filter, output, quiet):
+    """
+    Ingest SOURCE (URL or file path), filter by topic, and tokenize.
+
+    \b
+    Examples:
+      suur_data fetch https://en.wikipedia.org/wiki/Neuroscience --topic "brain"
+      suur_data fetch corpus.pdf --topic "machine learning" --tokenizer custom
+      suur_data fetch data.txt --no-filter --tokenizer pretrained --model bert
+    """
+    tokens = suur_data(
+        data_location=source,
+        topic=topic,
+        tokenizer=tokenizer,
+        model=model,
+        vocab_size=vocab_size,
+        threshold=threshold,
+        save_dir=save_dir,
+        no_filter=no_filter,
+        verbose=not quiet,
+    )
+
+    if output:
+        with open(output, "w") as f:
+            json.dump(tokens, f)
+        click.echo(f"Tokens saved to {output}")
+    else:
+        click.echo(f"Token count: {len(tokens):,}")
+        click.echo(f"First 50 tokens: {tokens[:50]}")
+
+
+@cli.command("models")
+def models_cmd():
+    """List supported pretrained model shortcuts."""
+    rows = [
+        ("gpt2",       "GPT-2 (OpenAI)"),
+        ("bert",       "BERT base uncased"),
+        ("roberta",    "RoBERTa base"),
+        ("distilbert", "DistilBERT base uncased"),
+        ("t5",         "T5 small"),
+    ]
+    click.echo("\nSupported pretrained model shortcuts:\n")
+    for key, name in rows:
+        click.echo(f"  {key:<14} {name}")
+    click.echo("\nYou can also pass any HuggingFace model ID directly.")
+    click.echo('Example: --model "facebook/opt-125m"\n')
+
+
+@cli.command("formats")
+def formats_cmd():
+    """List supported input file formats."""
+    fmts = [
+        (".txt / .md / .rst", "Plain text, Markdown, reStructuredText"),
+        (".pdf",              "PDF documents (requires pdfminer.six)"),
+        (".docx",             "Word documents (requires python-docx)"),
+        (".csv / .tsv",       "Comma/tab-separated values"),
+        (".json",             "JSON — recursively flattens key-value pairs"),
+        (".html / .htm",      "HTML pages (requires beautifulsoup4)"),
+        (".epub",             "E-books (requires ebooklib + beautifulsoup4)"),
+        ("URL",               "Any HTTP/HTTPS URL — auto-downloaded"),
+    ]
+    click.echo("\nSupported input formats:\n")
+    for ext, desc in fmts:
+        click.echo(f"  {ext:<22} {desc}")
+    click.echo()
+
+
+# ---------------------------------------------------------------------------
+# Allow running as: python -m suur_data
+# ---------------------------------------------------------------------------
+
+def main():
+    cli()
+
+
+if __name__ == "__main__":
+    main()
+

suur_data/filter.py

@@ -0,0 +1,183 @@
+"""
+suur_data.filter
+---------------
+Stage 2 — Neural relevance filter.
+
+Architecture:
+  - Splits raw text into paragraph-level chunks
+  - Embeds chunks using TF-IDF (fast, no GPU needed)
+  - Scores each chunk against a topic query using cosine similarity
+  - Keeps only chunks above a configurable threshold
+  - Falls back to keyword-density scoring if sklearn is missing
+
+This is intentionally lightweight so it runs on CPU with no external
+model downloads. For better quality, swap in a sentence-transformers
+embedding in the future.
+"""
+
+import re
+import math
+from typing import List, Tuple
+
+
+# ---------------------------------------------------------------------------
+# 1. Text splitter
+# ---------------------------------------------------------------------------
+
+def split_chunks(text: str, min_words: int = 20) -> List[str]:
+    """
+    Split text into chunks (paragraphs or groups of sentences).
+    Chunks shorter than min_words are merged with the next one.
+    """
+    # Split on blank lines first (paragraph boundaries)
+    raw = re.split(r"\n{2,}", text)
+
+    chunks = []
+    buffer = ""
+    for para in raw:
+        para = para.strip()
+        if not para:
+            continue
+        combined = (buffer + " " + para).strip() if buffer else para
+        if len(combined.split()) >= min_words:
+            chunks.append(combined)
+            buffer = ""
+        else:
+            buffer = combined
+
+    if buffer:
+        chunks.append(buffer)
+
+    return chunks
+
+
+# ---------------------------------------------------------------------------
+# 2. TF-IDF based relevance scorer
+# ---------------------------------------------------------------------------
+
+def _tokenize(text: str) -> List[str]:
+    return re.findall(r"[a-z]+", text.lower())
+
+
+def _tf(tokens: List[str]) -> dict:
+    counts: dict = {}
+    for t in tokens:
+        counts[t] = counts.get(t, 0) + 1
+    total = len(tokens) or 1
+    return {t: c / total for t, c in counts.items()}
+
+
+def _idf(docs: List[List[str]], vocab: set) -> dict:
+    N = len(docs)
+    idf: dict = {}
+    for word in vocab:
+        df = sum(1 for doc in docs if word in doc)
+        idf[word] = math.log((N + 1) / (df + 1)) + 1
+    return idf
+
+
+def _cosine(a: dict, b: dict) -> float:
+    dot = sum(a.get(k, 0) * v for k, v in b.items())
+    norm_a = math.sqrt(sum(v * v for v in a.values())) or 1e-9
+    norm_b = math.sqrt(sum(v * v for v in b.values())) or 1e-9
+    return dot / (norm_a * norm_b)
+
+
+def _tfidf_score(query_tokens: List[str],
+                 chunks: List[str]) -> List[float]:
+    """Score each chunk against the query using TF-IDF cosine similarity."""
+    tokenized = [_tokenize(c) for c in chunks]
+    all_docs = [query_tokens] + tokenized
+    vocab = set(t for doc in all_docs for t in doc)
+
+    idf = _idf(all_docs, vocab)
+
+    def tfidf_vec(tokens):
+        tf = _tf(tokens)
+        return {t: tf[t] * idf.get(t, 1) for t in tf}
+
+    q_vec = tfidf_vec(query_tokens)
+    scores = [_cosine(q_vec, tfidf_vec(tok)) for tok in tokenized]
+    return scores
+
+
+# ---------------------------------------------------------------------------
+# 3. Optional: sklearn TF-IDF (higher quality)
+# ---------------------------------------------------------------------------
+
+def _sklearn_score(query: str, chunks: List[str]) -> List[float]:
+    from sklearn.feature_extraction.text import TfidfVectorizer
+    from sklearn.metrics.pairwise import cosine_similarity
+    import numpy as np
+
+    docs = [query] + chunks
+    vec = TfidfVectorizer(stop_words="english", ngram_range=(1, 2))
+    matrix = vec.fit_transform(docs)
+    q_vec = matrix[0]
+    chunk_vecs = matrix[1:]
+    sims = cosine_similarity(q_vec, chunk_vecs)[0]
+    return sims.tolist()
+
+
+# ---------------------------------------------------------------------------
+# 4. Main filter function
+# ---------------------------------------------------------------------------
+
+def filter_chunks(
+    text: str,
+    topic: str,
+    threshold: float = 0.05,
+    min_words: int = 20,
+    verbose: bool = True,
+) -> Tuple[str, List[Tuple[str, float]]]:
+    """
+    Filter raw text to keep only chunks relevant to `topic`.
+
+    Returns:
+        filtered_text  — joined string of kept chunks
+        scored         — list of (chunk, score) for all chunks (sorted desc)
+    """
+    chunks = split_chunks(text, min_words=min_words)
+    if not chunks:
+        return text, []
+
+    if verbose:
+        print(f"  Scoring {len(chunks)} chunks against topic: '{topic}' ...")
+
+    # Use sklearn if available for better quality
+    try:
+        scores = _sklearn_score(topic, chunks)
+        method = "sklearn TF-IDF"
+    except ImportError:
+        scores = _tfidf_score(_tokenize(topic), chunks)
+        method = "built-in TF-IDF"
+
+    if verbose:
+        print(f"  Scoring method: {method}")
+
+    scored = sorted(zip(chunks, scores), key=lambda x: x[1], reverse=True)
+
+    # Dynamic threshold: if topic is empty/generic, keep everything
+    if not topic.strip():
+        kept = chunks
+        if verbose:
+            print("  No topic given — keeping all chunks.")
+    else:
+        kept = [c for c, s in scored if s >= threshold]
+        dropped = len(chunks) - len(kept)
+        if verbose:
+            top = scored[:3]
+            print(f"  Kept {len(kept)}/{len(chunks)} chunks (dropped {dropped} below threshold {threshold})")
+            print(f"  Top scores: {[round(s,4) for _, s in top]}")
+
+    # If filter is too aggressive, relax automatically
+    if len(kept) == 0 and len(chunks) > 0:
+        # Keep top 30% regardless
+        n = max(1, len(chunks) // 3)
+        kept = [c for c, _ in scored[:n]]
+        if verbose:
+            print(f"  Threshold too strict — auto-relaxed, keeping top {n} chunks.")
+
+    filtered_text = "\n\n".join(kept)
+    return filtered_text, scored
+

suur_data/ingest.py

@@ -0,0 +1,165 @@
+"""
+suur_data.ingest
+---------------
+Stage 1 — Data ingestion from URLs and local files.
+Supported: .txt, .pdf, .docx, .csv, .json, .html, .epub, plain URLs
+"""
+
+import os
+import re
+import csv
+import json
+import tempfile
+import urllib.request
+import urllib.parse
+from pathlib import Path
+
+
+def _is_url(source: str) -> bool:
+    parsed = urllib.parse.urlparse(source)
+    return parsed.scheme in ("http", "https", "ftp")
+
+
+def _download(url: str, dest_dir: str) -> str:
+    """Download a URL to a temp file. Returns local path."""
+    parsed = urllib.parse.urlparse(url)
+    ext = Path(parsed.path).suffix or ".html"
+    fd, path = tempfile.mkstemp(suffix=ext, dir=dest_dir)
+    os.close(fd)
+    print(f"  Downloading {url} ...")
+    headers = {"User-Agent": "Mozilla/5.0 (suur_data/1.0)"}
+    req = urllib.request.Request(url, headers=headers)
+    with urllib.request.urlopen(req, timeout=30) as resp, open(path, "wb") as f:
+        f.write(resp.read())
+    print(f"  Saved to {path}")
+    return path
+
+
+def _read_txt(path: str) -> str:
+    try:
+        import chardet
+        raw = open(path, "rb").read()
+        enc = chardet.detect(raw)["encoding"] or "utf-8"
+    except ImportError:
+        enc = "utf-8"
+    return open(path, encoding=enc, errors="replace").read()
+
+
+def _read_pdf(path: str) -> str:
+    try:
+        from pdfminer.high_level import extract_text
+        return extract_text(path)
+    except ImportError:
+        raise RuntimeError("pdfminer.six is required for PDF support: pip install pdfminer.six")
+
+
+def _read_docx(path: str) -> str:
+    try:
+        import docx
+        doc = docx.Document(path)
+        return "\n".join(p.text for p in doc.paragraphs)
+    except ImportError:
+        raise RuntimeError("python-docx is required for .docx support: pip install python-docx")
+
+
+def _read_csv(path: str) -> str:
+    rows = []
+    with open(path, newline="", encoding="utf-8", errors="replace") as f:
+        reader = csv.reader(f)
+        for row in reader:
+            rows.append(" ".join(row))
+    return "\n".join(rows)
+
+
+def _read_json(path: str) -> str:
+    with open(path, encoding="utf-8", errors="replace") as f:
+        obj = json.load(f)
+
+    def _flatten(o, depth=0):
+        if isinstance(o, str):
+            return o
+        if isinstance(o, (int, float, bool)):
+            return str(o)
+        if isinstance(o, list):
+            return "\n".join(_flatten(i, depth) for i in o)
+        if isinstance(o, dict):
+            parts = []
+            for k, v in o.items():
+                parts.append(f"{k}: {_flatten(v, depth+1)}")
+            return "\n".join(parts)
+        return ""
+
+    return _flatten(obj)
+
+
+def _read_html(path: str) -> str:
+    try:
+        from bs4 import BeautifulSoup
+    except ImportError:
+        raise RuntimeError("beautifulsoup4 is required for HTML support: pip install beautifulsoup4")
+    with open(path, encoding="utf-8", errors="replace") as f:
+        soup = BeautifulSoup(f.read(), "html.parser")
+    for tag in soup(["script", "style", "nav", "footer", "header"]):
+        tag.decompose()
+    return soup.get_text(separator="\n")
+
+
+def _read_epub(path: str) -> str:
+    try:
+        import ebooklib
+        from ebooklib import epub
+        from bs4 import BeautifulSoup
+    except ImportError:
+        raise RuntimeError("ebooklib + beautifulsoup4 required for .epub: pip install ebooklib beautifulsoup4")
+    book = epub.read_epub(path)
+    chapters = []
+    for item in book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
+        soup = BeautifulSoup(item.get_content(), "html.parser")
+        chapters.append(soup.get_text(separator="\n"))
+    return "\n\n".join(chapters)
+
+
+READERS = {
+    ".txt":  _read_txt,
+    ".md":   _read_txt,
+    ".rst":  _read_txt,
+    ".pdf":  _read_pdf,
+    ".docx": _read_docx,
+    ".csv":  _read_csv,
+    ".tsv":  _read_csv,
+    ".json": _read_json,
+    ".html": _read_html,
+    ".htm":  _read_html,
+    ".epub": _read_epub,
+}
+
+
+def ingest(source: str, tmp_dir: str | None = None) -> str:
+    """
+    Ingest text from a URL or local file.
+    Returns a single raw text string.
+    """
+    if tmp_dir is None:
+        tmp_dir = tempfile.mkdtemp(prefix="suur_data_")
+
+    if _is_url(source):
+        local_path = _download(source, tmp_dir)
+    else:
+        local_path = os.path.expanduser(source)
+        if not os.path.exists(local_path):
+            raise FileNotFoundError(f"File not found: {local_path}")
+
+    ext = Path(local_path).suffix.lower()
+
+    if ext not in READERS:
+        # fallback: try plain text
+        print(f"  Unknown extension '{ext}', attempting plain-text read.")
+        reader = _read_txt
+    else:
+        reader = READERS[ext]
+
+    print(f"  Reading {ext or 'file'} ...")
+    text = reader(local_path)
+    print(f"  Ingested {len(text):,} characters.")
+    return text
+

suur_data/tokenizer.py

@@ -0,0 +1,142 @@
+
+import re
+import os
+import json
+import collections
+from typing import List, Optional
+
+
+def _simple_pretokenize(text: str) -> List[str]:
+    return re.findall(r"\b\w+\b|[^\w\s]", text)
+
+
+def _load_hf_tokenizer(model_name: str):
+    try:
+        from transformers import AutoTokenizer
+    except ImportError:
+        raise ImportError("Install transformers: pip install transformers")
+    models = {
+        "gpt2": "gpt2",
+        "bert": "bert-base-uncased",
+        "roberta": "roberta-base",
+        "distilbert": "distilbert-base-uncased",
+        "t5": "t5-small",
+    }
+    resolved = models.get(model_name.lower(), model_name)
+    print(f"  Loading pretrained tokenizer: {resolved} ...")
+    return AutoTokenizer.from_pretrained(resolved)
+
+
+def _fallback_word_tokenize(text: str) -> List[int]:
+    words = _simple_pretokenize(text)
+    vocab: dict = {}
+    ids: List[int] = []
+    for w in words:
+        if w not in vocab:
+            vocab[w] = len(vocab)
+        ids.append(vocab[w])
+    print(f"  Fallback tokenizer: vocab_size={len(vocab)}, tokens={len(ids):,}")
+    return ids
+
+
+def tokenize_pretrained(text: str, model_name: str = "gpt2", save_dir: Optional[str] = None) -> List[int]:
+    try:
+        tok = _load_hf_tokenizer(model_name)
+        ids = tok.encode(text, add_special_tokens=False)
+        if save_dir:
+            os.makedirs(save_dir, exist_ok=True)
+            tok.save_pretrained(save_dir)
+        print(f"  Pretrained tokenizer produced {len(ids):,} tokens.")
+        return ids
+    except ImportError:
+        print("  [WARN] transformers not available, using fallback word tokenizer.")
+        return _fallback_word_tokenize(text)
+
+
+def _get_vocab(text: str) -> dict:
+    vocab: dict = collections.Counter()
+    for word in _simple_pretokenize(text):
+        chars = " ".join(list(word)) + " </w>"
+        vocab[chars] += 1
+    return dict(vocab)
+
+
+def _get_pairs(vocab: dict) -> dict:
+    pairs: dict = collections.Counter()
+    for word, freq in vocab.items():
+        symbols = word.split()
+        for i in range(len(symbols) - 1):
+            pairs[(symbols[i], symbols[i + 1])] += freq
+    return pairs
+
+
+def _merge_vocab(pair, vocab: dict) -> dict:
+    new_vocab: dict = {}
+    bigram = re.escape(" ".join(pair))
+    pattern = re.compile(r"(?<!\S)" + bigram + r"(?!\S)")
+    for word in vocab:
+        new_vocab[pattern.sub("".join(pair), word)] = vocab[word]
+    return new_vocab
+
+
+def _pure_python_bpe(text: str, vocab_size: int = 1000, save_dir: Optional[str] = None) -> List[int]:
+    print(f"  Training pure-Python BPE (vocab_size={vocab_size}) ...")
+    vocab = _get_vocab(text)
+    symbols: set = set()
+    for word in vocab:
+        symbols.update(word.split())
+    merges = []
+    while len(symbols) < vocab_size:
+        pairs = _get_pairs(vocab)
+        if not pairs:
+            break
+        best = max(pairs, key=pairs.get)
+        vocab = _merge_vocab(best, vocab)
+        symbols.add("".join(best))
+        merges.append(best)
+    word_to_id = {s: i for i, s in enumerate(sorted(symbols))}
+    ids = []
+    for word in _simple_pretokenize(text):
+        chars = list(word)
+        for a, b in merges:
+            i = 0
+            new_chars = []
+            while i < len(chars):
+                if i < len(chars) - 1 and chars[i] == a and chars[i+1] == b:
+                    new_chars.append(a + b)
+                    i += 2
+                else:
+                    new_chars.append(chars[i])
+                    i += 1
+            chars = new_chars
+        for c in chars:
+            ids.append(word_to_id.get(c, 0))
+    if save_dir:
+        os.makedirs(save_dir, exist_ok=True)
+        with open(os.path.join(save_dir, "bpe_vocab.json"), "w") as f:
+            json.dump(word_to_id, f, indent=2)
+    print(f"  Pure-Python BPE: vocab={len(symbols)}, tokens={len(ids):,}")
+    return ids
+
+
+def tokenize_custom(text: str, vocab_size: int = 8000, save_dir: Optional[str] = None) -> List[int]:
+    try:
+        import tokenizers  # noqa
+        from tokenizers import Tokenizer
+        from tokenizers.models import BPE
+        from tokenizers.trainers import BpeTrainer
+        from tokenizers.pre_tokenizers import Whitespace
+        print(f"  Training HF BPE tokenizer (vocab_size={vocab_size}) ...")
+        tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
+        tokenizer.pre_tokenizer = Whitespace()
+        trainer = BpeTrainer(vocab_size=vocab_size, special_tokens=["[UNK]"], min_frequency=2)
+        tokenizer.train_from_iterator([text], trainer=trainer)
+        if save_dir:
+            os.makedirs(save_dir, exist_ok=True)
+            tokenizer.save(os.path.join(save_dir, "tokenizer.json"))
+        ids = tokenizer.encode(text).ids
+        print(f"  HF BPE: vocab={tokenizer.get_vocab_size()}, tokens={len(ids):,}")
+        return ids
+    except ImportError:
+        print("  [WARN] tokenizers library not found, using pure-Python BPE.")
+        return _pure_python_bpe(text, vocab_size=vocab_size, save_dir=save_dir)

suur_data-1.0.0.dist-info/METADATA

@@ -0,0 +1,209 @@
+Metadata-Version: 2.4
+Name: suur_data
+Version: 1.0.0
+Summary: Intelligent data ingestion and tokenization pipeline
+Author: Your Name
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: requests
+Requires-Dist: beautifulsoup4
+Requires-Dist: scikit-learn
+Requires-Dist: numpy
+Requires-Dist: click
+Requires-Dist: chardet
+Provides-Extra: pdf
+Requires-Dist: pdfminer.six; extra == "pdf"
+Provides-Extra: docx
+Requires-Dist: python-docx; extra == "docx"
+Provides-Extra: epub
+Requires-Dist: ebooklib; extra == "epub"
+Provides-Extra: hf
+Requires-Dist: transformers; extra == "hf"
+Requires-Dist: tokenizers; extra == "hf"
+Provides-Extra: all
+Requires-Dist: pdfminer.six; extra == "all"
+Requires-Dist: python-docx; extra == "all"
+Requires-Dist: ebooklib; extra == "all"
+Requires-Dist: transformers; extra == "all"
+Requires-Dist: tokenizers; extra == "all"
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Suur Data
+
+**Intelligent data ingestion and tokenization pipeline.**
+
+Suur Data fetches text from any source, filters it by topic using a neural relevance scorer, then tokenizes it using either a pretrained HuggingFace tokenizer or a custom-trained BPE tokenizer.
+
+---
+
+## Installation
+
+```bash
+# Core (URLs, .txt, .csv, .json, .html)
+pip install -e .
+
+# With all optional formats + HuggingFace tokenizers
+pip install -e ".[all]"
+```
+
+---
+
+## Python API
+
+```python
+from suur_data import suur_data
+
+# Minimal — fetches URL, no filter, GPT-2 tokenizer
+tokens = suur_data("https://en.wikipedia.org/wiki/Neuroscience")
+
+# Filter by topic, custom BPE tokenizer
+tokens = suur_data(
+    "research_paper.pdf",
+    topic="quantum computing",
+    tokenizer="custom",
+    vocab_size=4000,
+    save_dir="./my_tokenizer",
+)
+
+# Local file, pretrained BERT tokenizer, strict filter
+tokens = suur_data(
+    "~/corpus/biology.txt",
+    topic="cell biology",
+    tokenizer="pretrained",
+    model="bert",
+    threshold=0.10,
+)
+
+# Skip the filter entirely
+tokens = suur_data("data.csv", no_filter=True)
+
+print(tokens[:20])   # list of integer token IDs
+print(len(tokens))   # total token count
+```
+
+### Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `data_location` | str | — | URL or local file path |
+| `topic` | str | `""` | Subject for relevance filtering (empty = skip filter) |
+| `tokenizer` | str | `"pretrained"` | `"pretrained"` or `"custom"` |
+| `model` | str | `"gpt2"` | HuggingFace model shortcut or full ID |
+| `vocab_size` | int | `8000` | BPE vocab size for custom tokenizer |
+| `threshold` | float | `0.05`` | Cosine similarity cutoff (0.0–1.0) |
+| `save_dir` | str | `None` | Path to save tokenizer files |
+| `no_filter` | bool | `False` | Skip the relevance filter |
+| `verbose` | bool | `True` | Show progress output |
+
+### Returns
+`List[int]` — flat list of integer token IDs.
+
+---
+
+## CLI
+
+```bash
+# Basic URL fetch
+suur_data fetch https://example.com/article --topic "machine learning"
+
+# PDF with custom BPE tokenizer
+suur_data fetch paper.pdf --topic "protein folding" --tokenizer custom --vocab-size 6000
+
+# Local file, pretrained BERT, save tokenizer
+suur_data fetch corpus.txt --tokenizer pretrained --model bert --save-dir ./bert_tok
+
+# Skip filter, save tokens to file
+suur_data fetch data.json --no-filter --output tokens.json
+
+# See supported models
+suur_data models
+
+# See supported file formats
+suur_data formats
+```
+
+---
+
+## Supported Input Formats
+
+| Format | Notes |
+|--------|-------|
+| `.txt`, `.md`, `.rst` | Plain text |
+| `.pdf` | Requires `pdfminer.six` |
+| `.docx` | Requires `python-docx` |
+| `.csv`, `.tsv` | All cells joined as text |
+| `.json` | Recursively flattened key-value pairs |
+| `.html`, `.htm` | Scripts/styles stripped (requires `beautifulsoup4`) |
+| `.epub` | E-books (requires `ebooklib` + `beautifulsoup4`) |
+| HTTP/HTTPS URL | Auto-downloaded, then parsed by extension |
+
+---
+
+## Pretrained Model Shortcuts
+
+| Shortcut | Model |
+|----------|-------|
+| `gpt2` | GPT-2 (OpenAI) |
+| `bert` | BERT base uncased |
+| `roberta` | RoBERTa base |
+| `distilbert` | DistilBERT base uncased |
+| `t5` | T5 small |
+
+You can also pass any HuggingFace Hub model ID directly:
+```
+--model "facebook/opt-125m"
+```
+
+---
+
+## Architecture
+
+```
+Source (URL / file)
+        │
+        ▼
+  Stage 1: Ingest
+  Handles 8 file types + HTTP download
+        │
+        ▼
+  Stage 2: Neural Filter
+  Splits into paragraph chunks
+  Scores each chunk against topic via TF-IDF cosine similarity
+  Drops chunks below threshold
+        │
+        ▼
+  Stage 3: Tokenize
+  ┌─────────────────────┐  ┌────────────────────────────┐
+  │  Pretrained mode    │  │  Custom mode               │
+  │  HuggingFace        │  │  BPE trainer (HF library   │
+  │  AutoTokenizer      │  │  or pure-Python fallback)  │
+  └─────────────────────┘  └────────────────────────────┘
+        │
+        ▼
+  List[int]  ←  token IDs
+```
+
+---
+
+## Dependency Matrix
+
+| Feature | Required packages |
+|---------|------------------|
+| Core pipeline | `requests`, `beautifulsoup4`, `scikit-learn`, `numpy`, `click`, `chardet` |
+| PDF support | `pdfminer.six` |
+| .docx support | `python-docx` |
+| .epub support | `ebooklib` |
+| Pretrained tokenizers | `transformers` |
+| Fast BPE training | `tokenizers` |
+
+All optional — the tool degrades gracefully with built-in fallbacks when optional packages are missing.
+

suur_data-1.0.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+suur_data/__init__.py,sha256=NN4R90oZcJIzHtwn8L3sksZ0bFAVQTbvQF-6--hwIvU,8064
+suur_data/filter.py,sha256=HCp_j35kC2UeKV_QthbW5SV5NUwS2F9UKF72lxIWljM,5804
+suur_data/ingest.py,sha256=c2d5d4wVawxdd2FG_Ecsxb5bRLvGz-PiFAHcIoPvxZo,4811
+suur_data/tokenizer.py,sha256=uDnGTKBigXdMQkPcJm-ebC62Sim7fHQnsTOavy6Ir5k,4989
+suur_data-1.0.0.dist-info/METADATA,sha256=rmBT0QEdEvx6A_EdMrQYM7e8LIl-Tz6VL83tFFFC9A0,5856
+suur_data-1.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+suur_data-1.0.0.dist-info/entry_points.txt,sha256=TKKBMSX0wa1BroERW5ftHglvR2-Fy_8fxij3XmqnO7o,45
+suur_data-1.0.0.dist-info/top_level.txt,sha256=8ooHxzPfqZ56sFPb_U6tRTKdxXBlsBrZOhDHWO9g_x4,10
+suur_data-1.0.0.dist-info/RECORD,,

suur_data-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

suur_data-1.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+suur-data = suur_data:main

suur_data-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+suur_data