topiclayers 0.2.0__py3-none-any.whl

topiclayers/__init__.py

@@ -0,0 +1,3 @@
+"""topiclayers — from posts to topical multilayer networks."""
+
+__version__ = "0.2.0"

topiclayers/adapters/__init__.py

@@ -0,0 +1,7 @@
+"""Input adapters: convert various formats into a list of Post dataclasses."""
+
+from __future__ import annotations
+
+from .registry import AdapterRegistry, resolve_adapter
+
+__all__ = ["AdapterRegistry", "resolve_adapter"]

topiclayers/adapters/csv.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import List
+
+import pandas as pd
+
+from topiclayers.adapters.registry import register
+from topiclayers.schema import InteractionType, Post
+
+_INTERACTION_MAP = {
+    "original": None,
+    "repost": InteractionType.REPOST,
+    "quote": InteractionType.QUOTE,
+    "reply": InteractionType.REPLY,
+}
+
+
+def _parse_interaction(value) -> InteractionType | None:
+    if value is None:
+        return None
+    if isinstance(value, float) and pd.isna(value):
+        return None
+    s = str(value).strip().lower()
+    if not s or s in ("nan", "none", ""):
+        return None
+    return _INTERACTION_MAP.get(s, None)
+
+
+def _parse_mentions(value) -> list[str]:
+    if value is None:
+        return []
+    if isinstance(value, float) and pd.isna(value):
+        return []
+    s = str(value).strip()
+    if not s or s.lower() in ("nan", "none"):
+        return []
+    return [m.strip() for m in s.split(";") if m.strip()]
+
+
+def _row_to_post(row, extra_cols: list[str]) -> Post:
+    extra = {}
+    for col in extra_cols:
+        val = row.get(col)
+        if isinstance(val, float) and pd.isna(val):
+            val = None
+        key = col.removeprefix("extra_")
+        extra[key] = val
+
+    return Post(
+        post_id=str(row["post_id"]),
+        user_id=str(row["user_id"]),
+        text=str(row.get("text", "")),
+        created_at=row.get("created_at") if not (isinstance(row.get("created_at"), float) and pd.isna(row.get("created_at"))) else None,
+        lang=row.get("lang") if not (isinstance(row.get("lang"), float) and pd.isna(row.get("lang"))) else None,
+        interaction_type=_parse_interaction(row.get("interaction_type")),
+        target_post_id=str(row["target_post_id"]) if row.get("target_post_id") and not (isinstance(row.get("target_post_id"), float) and pd.isna(row.get("target_post_id"))) else None,
+        mentions=_parse_mentions(row.get("mentions")),
+        extra=extra,
+    )
+
+
+def load_csv(path: Path) -> List[Post]:
+    df = pd.read_csv(path)
+    required = {"post_id", "user_id", "text"}
+    missing = required - set(df.columns)
+    if missing:
+        raise ValueError(
+            f"CSV file at {path} is missing required columns: {missing}. "
+            f"Required columns are: post_id, user_id, text. "
+            f"Optional: created_at, lang, interaction_type, target_post_id, target_user_id, mentions, and extra_* columns."
+        )
+    extra_cols = [c for c in df.columns if c.startswith("extra_")]
+    return [_row_to_post(row, extra_cols) for _, row in df.iterrows()]
+
+
+def load_parquet(path: Path) -> List[Post]:
+    df = pd.read_parquet(path)
+    required = {"post_id", "user_id", "text"}
+    missing = required - set(df.columns)
+    if missing:
+        raise ValueError(
+            f"Parquet file at {path} is missing required columns: {missing}. "
+            f"Required columns are: post_id, user_id, text."
+        )
+    extra_cols = [c for c in df.columns if c.startswith("extra_")]
+    return [_row_to_post(row, extra_cols) for _, row in df.iterrows()]
+
+
+register("csv_posts", load_csv)
+register("parquet_posts", load_parquet)

topiclayers/adapters/registry.py

@@ -0,0 +1,47 @@
+"""Input adapter registry — maps format strings to loader functions."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Callable, List
+
+from topiclayers.schema import Post
+
+
+AdapterFn = Callable[[Path], List[Post]]
+
+_registry: dict[str, AdapterFn] = {}
+
+
+def register(format_name: str, fn: AdapterFn) -> None:
+    _registry[format_name] = fn
+
+
+def resolve_adapter(path: Path) -> tuple[str, AdapterFn]:
+    """Auto-detect format from file extension and return (format_name, loader)."""
+    ext = path.suffix.lower()
+    if ext in (".jsonl", ".json", ".dat"):
+        fmt = "twitter_jsonl"
+    elif ext == ".csv":
+        fmt = "csv_posts"
+    elif ext == ".parquet":
+        fmt = "parquet_posts"
+    else:
+        raise ValueError(
+            f"Cannot auto-detect format for extension '{ext}'. "
+            f"Supported: .jsonl, .json, .dat, .csv, .parquet. "
+            f"Use explicit `format:` in config for other formats."
+        )
+    if fmt not in _registry:
+        raise KeyError(
+            f"No adapter registered for format '{fmt}'. "
+            f"Available adapters: {list(_registry.keys())}. "
+            f"Did you import topiclayers.adapters.{fmt.split('_')[0]}?"
+        )
+    return fmt, _registry[fmt]
+
+
+class AdapterRegistry:
+    """Explicit registry for manual adapter lookups."""
+    pass

topiclayers/adapters/twitter.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import List
+
+from topiclayers.adapters.registry import register
+from topiclayers.schema import InteractionType, Post
+
+
+def _map_interaction_type(ref_type: str | None) -> InteractionType | None:
+    if ref_type == "retweeted":
+        return InteractionType.REPOST
+    elif ref_type == "quoted":
+        return InteractionType.QUOTE
+    elif ref_type == "replied_to":
+        return InteractionType.REPLY
+    return None
+
+
+def load(path: Path) -> List[Post]:
+    posts = []
+    with open(path) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            obj = json.loads(line)
+
+            post_id = str(obj.get("id", ""))
+            user_id = str(obj.get("author_id", ""))
+            text = obj.get("text", "")
+            created_at = obj.get("created_at")
+            lang = obj.get("lang")
+
+            ref_tweets = obj.get("referenced_tweets")
+            if ref_tweets and isinstance(ref_tweets, list) and len(ref_tweets) > 0:
+                ref = ref_tweets[0]
+                interaction_type = _map_interaction_type(ref.get("type"))
+                target_post_id = str(ref.get("id")) if ref.get("id") is not None else None
+            else:
+                interaction_type = None
+                target_post_id = None
+
+            entities = obj.get("entities", {})
+            mentions_raw = entities.get("mentions", []) if isinstance(entities, dict) else []
+            mentions = [m.get("username", "") for m in mentions_raw if isinstance(m, dict)]
+
+            extra = {
+                "conversation_id": str(obj.get("conversation_id")) if obj.get("conversation_id") is not None else None,
+                "context_annotations": obj.get("context_annotations", []),
+                "public_metrics": obj.get("public_metrics", {}),
+                "possibly_sensitive": obj.get("possibly_sensitive"),
+                "edit_history_tweet_ids": obj.get("edit_history_tweet_ids", []),
+            }
+
+            post = Post(
+                post_id=post_id,
+                user_id=user_id,
+                text=text,
+                created_at=created_at,
+                lang=lang,
+                interaction_type=interaction_type,
+                target_post_id=target_post_id,
+                mentions=mentions,
+                extra=extra,
+            )
+            posts.append(post)
+
+    return posts
+
+
+register("twitter_jsonl", load)

topiclayers/cli.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+import hashlib
+import sys
+from pathlib import Path
+
+from topiclayers.config import RunConfig
+
+
+def _adapters_loaded() -> None:
+    import topiclayers.adapters.csv
+    import topiclayers.adapters.twitter
+
+
+def run_pipeline(config: RunConfig) -> None:
+    from topiclayers.adapters.registry import resolve_adapter
+    from topiclayers.core.data import Dataset
+    from topiclayers.manifest import RunManifest
+    from topiclayers.core.topic import TopicModeler
+    from topiclayers.status import RunStatus
+
+    _adapters_loaded()
+
+    input_path = Path(config.input.posts)
+    if not input_path.exists():
+        print(f"Input file not found: {input_path}", file=sys.stderr)
+        sys.exit(1)
+
+    config.output_dir.mkdir(parents=True, exist_ok=True)
+    status = RunStatus(config.output_dir, config.name)
+    status.start()
+
+    try:
+        _run_pipeline_stages(config, input_path, status)
+    except BaseException:
+        status.finish("failed")
+        raise
+
+
+def _run_pipeline_stages(config: RunConfig, input_path: Path, status) -> None:
+    from topiclayers.adapters.registry import resolve_adapter
+    from topiclayers.core.data import Dataset
+    from topiclayers.manifest import RunManifest
+    from topiclayers.core.topic import TopicModeler
+
+    status.stage("load")
+    fmt, loader = resolve_adapter(input_path)
+    print(f"Loading posts via '{fmt}' adapter from {input_path}")
+    posts = loader(input_path)
+    print(f"  {len(posts)} posts loaded")
+
+    row_counts = {"raw_posts": len(posts)}
+    tm = None
+
+    cache_dir = config.output_dir / "cache"
+
+    status.stage("data", total=len(posts))
+    ds = Dataset(posts, n_cop=config.name, file_user=config.input.users)
+    ds.process(cache_dir=cache_dir, adapter_name=fmt, reporter=status)
+    row_counts["tweets"] = len(ds.df_tweets)
+    row_counts["original"] = len(ds.df_original)
+    row_counts["retweets"] = len(ds.df_retweets)
+    print(f"  {len(ds.df_original)} original, {len(ds.df_retweets)} retweets, "
+          f"{len(ds.df_quotes)} quotes, {len(ds.df_reply)} replies")
+
+    if len(ds.df_original) == 0:
+        print("  No original posts found — skipping topic modeling. All retweets will be assigned topic -1.")
+        df_labeled = ds.df_original.copy()
+        df_labeled["topic"] = -1
+        df_labeled["topic_prob"] = None
+        row_counts["labeled_originals"] = len(df_labeled)
+        topic_dist = {"-1": len(df_labeled)}
+    else:
+        tm = TopicModeler(
+            ds.df_original,
+            embedder_name=config.topic_model.embedder,
+            path_cache=cache_dir,
+            name=config.name,
+        )
+        try:
+            df_labeled = tm.get_topics(
+                min_topic_size=config.topic_model.min_topic_size,
+                nr_topics=config.topic_model.nr_topics,
+                random_state=config.topic_model.random_state,
+                reporter=status,
+                exact_probabilities=config.topic_model.exact_probabilities,
+                repro_mode=config.topic_model.repro_mode,
+                umap_init=config.topic_model.umap_init,
+            )
+        except RuntimeError as e:
+            print(f"  Topic modeling failed: {e}")
+            print("  Assigning topic -1 to all originals and continuing.")
+            df_labeled = ds.df_original.copy()
+            df_labeled["topic"] = -1
+            df_labeled["topic_prob"] = None
+        row_counts["labeled_originals"] = len(df_labeled)
+        topic_counts = df_labeled["topic"].value_counts().to_dict()
+        topic_dist = {str(k): int(v) for k, v in topic_counts.items()}
+        print(f"  {len(topic_dist)} topics found (including -1 outlier)")
+
+    status.stage("label")
+    if config.topic_model.label_model and tm is not None and tm.model is not None:
+        tm.label_topics(
+            model=config.topic_model.label_model,
+            api_base=config.topic_model.label_api_base,
+            label_context=config.topic_model.label_context,
+        )
+    df_labeled = ds.update_df(df_labeled)
+    ds._save_labeled_dataframe(cache_dir)
+    row_counts["retweets_labeled"] = len(df_labeled)
+
+    from topiclayers.core.network import NetworkCreator
+    nw = NetworkCreator(df_labeled, config.name, config.output_dir)
+    dropped_edges = 0
+
+    status.stage("network", total=len(df_labeled))
+    network_type = config.network.type
+    if network_type == "multilayer_repost":
+        nw.create_retweet_network(reporter=status)
+        try:
+            nw.create_retweet_ml(reporter=status)
+        except ValueError as e:
+            print(f"  Skipping multilayer network: {e}")
+    elif network_type == "retweet":
+        nw.create_retweet_network(reporter=status)
+    elif network_type == "ttn":
+        nw.create_ttnetwork(project=config.network.project_ttn)
+    elif network_type == "all":
+        nw.create_retweet_network(reporter=status)
+        try:
+            nw.create_retweet_ml(reporter=status)
+        except ValueError as e:
+            print(f"  Skipping multilayer network: {e}")
+        nw.create_ttnetwork(project=config.network.project_ttn)
+    else:
+        print(f"Unknown network type: '{network_type}'. "
+              f"Use one of: multilayer_repost, retweet, ttn, all.", file=sys.stderr)
+        sys.exit(1)
+
+    status.stage("manifest")
+
+    params_hash = hashlib.sha256(
+        str(config).encode()
+    ).hexdigest()[:16]
+
+    manifest = RunManifest.create(
+        params={
+            "name": config.name,
+            "input_format": config.input.format,
+            "input_posts": str(config.input.posts),
+            "embedder": config.topic_model.embedder,
+            "min_topic_size": config.topic_model.min_topic_size,
+            "nr_topics": config.topic_model.nr_topics,
+            "random_state": config.topic_model.random_state,
+            "network_type": config.network.type,
+            "exact_probabilities": config.topic_model.exact_probabilities,
+            "repro_mode": config.topic_model.repro_mode,
+            "umap_init": config.topic_model.umap_init,
+        },
+        row_counts=row_counts,
+        topic_dist=topic_dist,
+        dropped=dropped_edges,
+        status="success",
+    )
+    manifest.write(config.output_dir / "run_manifest.json")
+    status.finish("success")
+    print(f"\nDone. Output in {config.output_dir.resolve()}")
+    print(f"  Run manifest: {config.output_dir / 'run_manifest.json'}")
+
+
+def main() -> None:
+    if len(sys.argv) < 3 or sys.argv[1] not in ("run", "status"):
+        print("Usage: topiclayers run <config.yaml>", file=sys.stderr)
+        print("       topiclayers status <output_dir>", file=sys.stderr)
+        sys.exit(1)
+
+    if sys.argv[1] == "status":
+        from topiclayers.status import print_status
+
+        sys.exit(print_status(Path(sys.argv[2])))
+
+    config_path = Path(sys.argv[2])
+    if not config_path.exists():
+        print(f"Config file not found: {config_path}", file=sys.stderr)
+        sys.exit(1)
+
+    cfg = RunConfig.from_yaml(config_path)
+    run_pipeline(cfg)
+
+
+if __name__ == "__main__":
+    main()

topiclayers/config.py

@@ -0,0 +1,84 @@
+"""YAML run configuration schema."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+
+
+@dataclass
+class InputConfig:
+    format: str = "twitter_jsonl"
+    posts: Path = Path(".")
+    users: Optional[Path] = None
+
+
+@dataclass
+class TopicModelConfig:
+    embedder: str = "all-MiniLM-L6-v2"
+    min_topic_size: int = 50
+    nr_topics: str = "auto"
+    random_state: int = 42
+    umap_n_neighbors: int = 15
+    umap_n_components: int = 5
+    exact_probabilities: bool = False
+    repro_mode: str = "strict"
+    # "pca" (seeded) is deterministic and ~20x faster than "spectral"
+    # with equal or better clustering quality; see docs/UMAP_INIT_BENCHMARK.md
+    umap_init: str = "pca"
+    # LLM used to label topics (any LiteLLM model string). Requires the
+    # 'labeling' extra: pip install 'topiclayers[labeling]'. Set to "" to
+    # disable. Non-fatal: if unreachable, topics keep their keyword labels.
+    label_model: str = "ollama/qwen2.5:7b"
+    # Base URL of an OpenAI-compatible endpoint for label_model (e.g.
+    # "http://localhost:8888/v1" for a local Unsloth/llama.cpp/vLLM server).
+    # When set, prefix the model with "openai/", e.g.
+    #   label_model: openai/unsloth/Qwen3.8-27B-GGUF
+    # The API key is read from LABEL_API_KEY, OPENAI_LIKE_API_KEY or
+    # OPENAI_API_KEY (in that order). Leave "" for Ollama/OpenAI defaults.
+    label_api_base: str = ""
+    # Dataset-level context for labeling ("all documents are ..."), so the LLM
+    # labels the specific subtopic instead of the general domain. Empty = auto
+    # (derived from `name` when it matches cop<number>).
+    label_context: str = ""
+
+
+@dataclass
+class NetworkConfig:
+    type: str = "multilayer_repost"
+    project_ttn: bool = True
+
+
+@dataclass
+class RunConfig:
+    input: InputConfig = field(default_factory=InputConfig)
+    name: str = "default"
+    topic_model: TopicModelConfig = field(default_factory=TopicModelConfig)
+    network: NetworkConfig = field(default_factory=NetworkConfig)
+    output_dir: Path = Path("./out")
+
+    @classmethod
+    def from_yaml(cls, path: Path) -> "RunConfig":
+        import yaml
+
+        path = Path(path)
+        with open(path) as f:
+            raw = yaml.safe_load(f)
+
+        input_raw = dict(raw.get("input", {}))
+        # input paths are resolved relative to the config file's directory
+        for key in ("posts", "users"):
+            if input_raw.get(key) is not None:
+                p = Path(input_raw[key])
+                if not p.is_absolute():
+                    p = (path.parent / p).resolve()
+                input_raw[key] = p
+
+        return cls(
+            input=InputConfig(**input_raw),
+            name=raw.get("name", "default"),
+            topic_model=TopicModelConfig(**raw.get("topic_model", {})),
+            network=NetworkConfig(**raw.get("network", {})),
+            output_dir=Path(raw.get("output_dir", "./out")),
+        )

topiclayers/core/__init__.py

@@ -0,0 +1,26 @@
+"""Core pipeline stages: data, topic modeling, network construction."""
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from topiclayers.core.data import Dataset
+    from topiclayers.core.network import NetworkCreator
+    from topiclayers.core.topic import TopicModeler
+
+__all__ = ["Dataset", "NetworkCreator", "TopicModeler"]
+
+
+def __getattr__(name: str):
+    if name == "Dataset":
+        from topiclayers.core.data import Dataset
+
+        return Dataset
+    if name == "NetworkCreator":
+        from topiclayers.core.network import NetworkCreator
+
+        return NetworkCreator
+    if name == "TopicModeler":
+        from topiclayers.core.topic import TopicModeler
+
+        return TopicModeler
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")