netgamut 0.4.0__py3-none-any.whl

netgamut/__init__.py

@@ -0,0 +1,486 @@
+"""Browse, download and cache network datasets.
+
+Each repository lives in its own subpackage:
+
+* :mod:`netgamut.snap` -- the Stanford Large Network Dataset Collection, 109
+  datasets. Large, well known, and skewed to social, web and technological
+  systems.
+* :mod:`netgamut.netzschleuder` -- networks.skewed.de, 286 networks tagged by
+  domain, including the biological, transport and multilayer networks SNAP has
+  none of.
+* :mod:`netgamut.konect` -- KONECT, the Koblenz Network Collection, 300
+  networks described in one fixed vocabulary of format, edge type and loops.
+  A network Netzschleuder or SNAP also holds is listed there but used from the
+  other source here, so nothing is counted twice.
+* :mod:`netgamut.ogb` -- Open Graph Benchmark, 12 datasets whose node
+  categories are *named* rather than left as anonymous codes.
+* :mod:`netgamut.pyg` -- the PyTorch Geometric corpus, 99 datasets: citation,
+  web, social and heterophilous graphs.
+* :mod:`netgamut.paths` -- curated path data (flight itineraries, navigation
+  trails) plus the conversion into second-order state networks for Infomap's
+  memory model, which no edge list can express.
+* :mod:`netgamut.ahorn` -- the Aachen Higher-Order Repository of Networks, 83
+  hypergraphs and simplicial complexes. Its unit is a *set* of nodes rather than
+  a pair, so it returns a
+  :class:`~netgamut.Hypergraph` rather than a ``Network``.
+
+Metadata for every source ships inside the package, so browsing is offline and
+instant; only ``load()`` touches the network, and it caches into a shared
+user-level directory so a dataset is fetched once per machine.
+
+Beyond the datasets, :mod:`netgamut.generate` samples synthetic benchmark
+networks with planted communities from any blueprint in the pool (requires
+the ``generate`` extra).
+
+Every network -- downloaded or generated, first-order, state or multilayer --
+converts and writes the same way: ``to_networkx()``, ``to_infomap()``, and
+``to_pajek(directory)`` / ``to_link_list(directory)``, which return the path
+of the file they wrote.
+
+All sources are mapped onto one domain vocabulary, so a comparison can span
+kinds of system rather than kinds of provenance::
+
+    import netgamut
+
+    netgamut.domains()
+    # {'Social': 201, 'Informational': 68, 'Biological': 36, ...}
+
+    for dataset in netgamut.search(domain="Biological", max_nodes=1000):
+        print(dataset.source, dataset.name)
+
+Please cite the datasets you use -- each source's records carry the reference
+its site asks for.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from . import ahorn, generate, konect, netzschleuder, ogb, paths, pyg, snap
+from ._core.cache import (
+    CACHE_ENV_VAR,
+    CachedDataset,
+    CacheInfo,
+    cache_dir,
+    cache_info,
+    clear_cache,
+    set_cache_dir,
+)
+from ._core.columns import Columns
+from ._core.domains import DOMAINS
+from ._core.hypergraph import Hypergraph
+from ._core.in_memory import InMemoryNetwork
+from ._core.multilayer import MultilayerNetwork
+from ._core.network import Network
+from ._core.state_network import StateNetwork
+
+__all__ = [
+    "CACHE_ENV_VAR",
+    "DOMAINS",
+    "CacheInfo",
+    "CachedDataset",
+    "Columns",
+    "Hypergraph",
+    "Match",
+    "MultilayerNetwork",
+    "Network",
+    "StateNetwork",
+    "InMemoryNetwork",
+    "__version__",
+    "ahorn",
+    "cache_dir",
+    "cache_info",
+    "clear_cache",
+    "domains",
+    "generate",
+    "konect",
+    "load",
+    "netzschleuder",
+    "ogb",
+    "paths",
+    "pyg",
+    "search",
+    "set_cache_dir",
+    "snap",
+    "sources",
+]
+
+__version__ = "0.4.0"
+
+#: Source name -> subpackage, in the order results are reported.
+_SOURCES: dict[str, Any] = {
+    "snap": snap,
+    "netzschleuder": netzschleuder,
+    "konect": konect,
+    "ogb": ogb,
+    "pyg": pyg,
+    "paths": paths,
+    "ahorn": ahorn,
+}
+
+#: The kinds of network :class:`Match` can say yes, no or unknown to, which
+#: :func:`search`, the explorer's index and the web mirror's manifest share.
+_KINDS = ("weighted", "bipartite", "temporal", "multilayer", "signed", "multigraph")
+
+
+def sources() -> tuple[str, ...]:
+    """The available data sources."""
+    return tuple(_SOURCES)
+
+
+def _shadowed(source: str, dataset: Any) -> bool:
+    """Whether cross-source queries skip *dataset* for another source's copy.
+
+    Only KONECT defers: a third of it is Netzschleuder's or SNAP's data. See
+    :func:`netgamut.konect.duplicates`.
+    """
+    return source == "konect" and dataset.duplicate_of is not None
+
+
+def _owners(name: str) -> list[str]:
+    """The sources holding *name*, leaving out a KONECT copy of another's."""
+    found = [key for key, module in _SOURCES.items() if name in module.catalog()]
+    if len(found) > 1 and "konect" in found and _shadowed("konect", konect.info(name)):
+        found.remove("konect")
+    return found
+
+
+@dataclass(frozen=True, slots=True)
+class Match:
+    """One dataset found by a cross-source query.
+
+    Keeps the source alongside the record, because the two catalogs use
+    different names for the same idea and a few names appear in both.
+    """
+
+    source: str
+    dataset: Any
+
+    @property
+    def name(self) -> str:
+        name: str = self.dataset.name
+        return name
+
+    @property
+    def domains(self) -> tuple[str, ...]:
+        result: tuple[str, ...] = self.dataset.domains
+        return result
+
+    @property
+    def nodes(self) -> int | None:
+        return self.dataset.nodes  # type: ignore[no-any-return]
+
+    @property
+    def edges(self) -> int | None:
+        return self.dataset.edges  # type: ignore[no-any-return]
+
+    @property
+    def directed(self) -> bool | None:
+        return self.dataset.directed  # type: ignore[no-any-return]
+
+    @property
+    def node_metadata(self) -> bool | None:
+        """Whether :meth:`Network.nodes` returns per-node attributes; ``None`` if unknown.
+
+        Each source answers from its own catalog, before any download: KONECT and
+        Netzschleuder from their file listings and analyses, SNAP from its label
+        files, OGB and PyG from their class labels. AHORN keeps node lines inside
+        the dataset file, so it cannot say.
+        """
+        value = getattr(self.dataset, "node_metadata", None)
+        return None if value is None else bool(value)
+
+    @property
+    def node_attributes(self) -> tuple[str, ...] | None:
+        """The per-node columns the source's catalog can name, ``None`` if unknown.
+
+        Exact for Netzschleuder, KONECT and SNAP. OGB names only its extra files
+        (``year``, ``species``) and PyG none, since their other column names come
+        with the download; ``nodes()`` returns those too.
+        """
+        value = getattr(self.dataset, "node_attributes", None)
+        return None if value is None else tuple(value)
+
+    @property
+    def loops(self) -> bool | None:
+        """Whether any node links to itself, where some source has checked.
+
+        KONECT checks every network; for another source's network, KONECT's
+        check is used when it holds the same edge list. ``None`` means unknown.
+        """
+        value = getattr(self.dataset, "loops", None)
+        if value is not None:
+            return bool(value)
+        return konect.loops_for(self.key)
+
+    def _flag(self, name: str) -> bool | None:
+        value = getattr(self.dataset, name, None)
+        return None if value is None else bool(value)
+
+    @property
+    def weighted(self) -> bool | None:
+        """Whether links carry weights; ``None`` where the source cannot say.
+
+        From Netzschleuder's tags, KONECT's edge type, SNAP's graph type and
+        AHORN's catalog. OGB, PyG and the path data cannot say.
+        """
+        return self._flag("weighted")
+
+    @property
+    def bipartite(self) -> bool | None:
+        """Whether links only join two kinds of node; ``None`` where the source cannot say.
+
+        KONECT and SNAP say so in their catalogs. For Netzschleuder it is the
+        site's structural check, no odd cycle, which every two-mode network
+        passes and so do a few trees and grids.
+        """
+        return self._flag("bipartite")
+
+    @property
+    def temporal(self) -> bool | None:
+        """Whether links carry timestamps; ``None`` where the source cannot say.
+
+        From Netzschleuder's ``Temporal`` and ``Timestamps`` tags, KONECT's
+        timestamp column, SNAP's graph type and AHORN's catalog.
+        """
+        return self._flag("temporal")
+
+    @property
+    def multilayer(self) -> bool | None:
+        """Whether Netzschleuder publishes it as a multilayer or multiplex network.
+
+        ``None`` for every other source, whose catalogs do not say.
+        """
+        return self._flag("multilayer")
+
+    @property
+    def signed(self) -> bool | None:
+        """Whether links are positive or negative; ``None`` where the source cannot say.
+
+        From Netzschleuder's ``Signed`` tag, KONECT's edge type and SNAP's graph type.
+        """
+        return self._flag("signed")
+
+    @property
+    def multigraph(self) -> bool | None:
+        """Whether some pair of nodes links more than once; ``None`` if the source cannot say.
+
+        From Netzschleuder's ``Multigraph`` tag and KONECT's edge type.
+        """
+        return self._flag("multigraph")
+
+    @property
+    def key(self) -> str:
+        """``"<source>/<name>"`` -- unique across sources."""
+        return f"{self.source}/{self.name}"
+
+    def load(self, **kwargs: Any) -> Network:
+        """Download this dataset via its own source."""
+        result: Network = _SOURCES[self.source].load(self.name, **kwargs)
+        return result
+
+    def to_dict(self) -> dict[str, Any]:
+        """A JSON-serialisable summary, the same shape for every source.
+
+        Holds the fields :func:`search` filters on plus what a reader needs to
+        choose and cite a dataset: its description, citations, its page at the
+        source (``url``) and, where the catalog records it, the original
+        publisher (``upstream_url``). ``weighted``, ``bipartite``, ``temporal``,
+        ``multilayer``, ``signed`` and ``multigraph`` are the properties of the
+        same names, ``None`` where the source cannot say. ``duplicate_of`` is
+        the key of the network a KONECT copy defers to (see
+        :func:`netgamut.konect.duplicates`), else ``None``.
+        ``subgraph_count`` is how many networks the name holds: a Netzschleuder
+        entry can hold several, the first few named in ``subgraphs``, and the
+        statistics describe the first, which :meth:`load` fetches by default.
+        Other sources hold one and name none.
+        """
+        dataset = self.dataset
+        attributes = self.node_attributes
+        upstream = getattr(dataset, "upstream_url", "") or getattr(dataset, "source_url", "")
+        return {
+            "key": self.key,
+            "source": self.source,
+            "name": self.name,
+            "title": getattr(dataset, "title", ""),
+            "domains": list(self.domains),
+            "nodes": self.nodes,
+            "edges": self.edges,
+            "directed": self.directed,
+            "loops": self.loops,
+            "node_metadata": self.node_metadata,
+            "node_attributes": None if attributes is None else list(attributes),
+            **{name: getattr(self, name) for name in _KINDS},
+            "subgraph_count": getattr(dataset, "subgraph_count", 1),
+            "subgraphs": list(getattr(dataset, "subgraphs", ())),
+            "description": getattr(dataset, "description", ""),
+            "citations": [_citation(entry) for entry in getattr(dataset, "citations", ())],
+            "url": dataset.url,
+            "upstream_url": upstream,
+            "duplicate_of": getattr(dataset, "duplicate_of", None),
+        }
+
+    def __str__(self) -> str:
+        shape = []
+        if self.nodes is not None:
+            shape.append(f"{self.nodes:,} nodes")
+        if self.edges is not None:
+            shape.append(f"{self.edges:,} edges")
+        domains = "/".join(self.domains) or "uncategorised"
+        tail = f" ({', '.join(shape)})" if shape else ""
+        return f"{self.key} [{domains}]{tail}"
+
+
+def _citation(entry: Any) -> dict[str, str]:
+    """One citation as ``{"text", "url"}``: Netzschleuder pairs each text with a link."""
+    if isinstance(entry, str):
+        return {"text": entry, "url": ""}
+    text, *link = entry
+    return {"text": str(text), "url": str(link[0]) if link else ""}
+
+
+def domains() -> dict[str, int]:
+    """Dataset counts per domain, summed over every source.
+
+    A dataset listed under two domains counts once in each, and a network held
+    by two sources counts once.
+    """
+    counts = dict.fromkeys(DOMAINS, 0)
+    for source, module in _SOURCES.items():
+        for dataset in module.datasets():
+            if _shadowed(source, dataset):
+                continue
+            for domain in dataset.domains:
+                counts[domain] += 1
+    return counts
+
+
+def search(
+    query: str = "",
+    *,
+    domain: str | None = None,
+    source: str | None = None,
+    directed: bool | None = None,
+    max_nodes: int | None = None,
+    min_nodes: int | None = None,
+    max_edges: int | None = None,
+    min_edges: int | None = None,
+    loops: bool | None = None,
+    node_metadata: bool | None = None,
+    node_attribute: str | None = None,
+    weighted: bool | None = None,
+    bipartite: bool | None = None,
+    temporal: bool | None = None,
+    multilayer: bool | None = None,
+    signed: bool | None = None,
+    multigraph: bool | None = None,
+    unknown: bool = False,
+) -> tuple[Match, ...]:
+    """Search every source at once, filtering on the shared domain vocabulary.
+
+    Only the fields every source agrees on are accepted here. For anything
+    source-specific -- SNAP's ground-truth partitions, Netzschleuder's subtypes,
+    OGB's loadability, PyG's class -- use that source's own ``search()``.
+
+    Size filters exclude datasets whose size the source does not publish rather than
+    assuming anything: most PyG entries and the largest OGB ones have no counts.
+    ``loops`` does the same with datasets no source has checked for self-links
+    (see :attr:`Match.loops`), and ``node_metadata`` with datasets whose catalog
+    cannot say (see :attr:`Match.node_metadata`), and so do ``weighted``,
+    ``bipartite``, ``temporal``, ``multilayer``, ``signed`` and ``multigraph``
+    (see :attr:`Match.weighted` and the rest). With ``unknown=True`` a dataset
+    passes every filter its source cannot answer instead, so
+    ``search(weighted=False, unknown=True)`` keeps the networks no catalog calls
+    weighted. ``node_attribute`` finds
+    datasets with a per-node column of that name, case-insensitively, among the
+    names each catalog knows -- ``node_attribute="gender"`` for something to score
+    a partition against.
+
+    A network held by both KONECT and Netzschleuder or SNAP appears once, from
+    the other source; ``source="konect"`` lists KONECT's copy too.
+
+    Raises:
+        KeyError: Unknown *domain* or *source*.
+    """
+    if domain is not None and domain not in DOMAINS:
+        raise KeyError(f"Unknown domain {domain!r}. Available: {', '.join(DOMAINS)}")
+    if source is not None and source not in _SOURCES:
+        raise KeyError(f"Unknown source {source!r}. Available: {', '.join(_SOURCES)}")
+
+    needle = query.casefold().strip()
+    results: list[Match] = []
+    wanted = (weighted, bipartite, temporal, multilayer, signed, multigraph)
+    kinds = dict(zip(_KINDS, wanted, strict=True))
+
+    def within(value: int | None, low: int | None, high: int | None) -> bool:
+        if low is None and high is None:
+            return True
+        if value is None:
+            return unknown
+        return (low is None or value >= low) and (high is None or value <= high)
+
+    def says(value: bool | None, want: bool | None) -> bool:
+        if want is None:
+            return True
+        return unknown if value is None else value is want
+
+    for name, module in _SOURCES.items():
+        if source is not None and name != source:
+            continue
+        for dataset in module.datasets():
+            if source is None and _shadowed(name, dataset):
+                continue
+            if domain is not None and domain not in dataset.domains:
+                continue
+            text = f"{dataset.name} {getattr(dataset, 'description', '')}".casefold()
+            if needle and needle not in text:
+                continue
+            if not says(dataset.directed, directed):
+                continue
+            if not within(dataset.nodes, min_nodes, max_nodes):
+                continue
+            if not within(dataset.edges, min_edges, max_edges):
+                continue
+            match = Match(source=name, dataset=dataset)
+            if not says(match.loops, loops) or not says(match.node_metadata, node_metadata):
+                continue
+            if not all(says(getattr(match, name), want) for name, want in kinds.items()):
+                continue
+            columns = match.node_attributes
+            if node_attribute is not None and not (
+                unknown
+                if columns is None
+                else any(column.casefold() == node_attribute.casefold() for column in columns)
+            ):
+                continue
+            results.append(match)
+
+    return tuple(results)
+
+
+def load(name: str, *, source: str | None = None, **kwargs: Any) -> Network:
+    """Download a dataset by name, searching every source.
+
+    Pass *source* to disambiguate. Without it, a name present in more than one
+    source is an error rather than an arbitrary choice -- except that KONECT's
+    copy of another source's network defers to that source.
+
+    Raises:
+        KeyError: No source has *name*, or several do and *source* was omitted.
+    """
+    if source is not None:
+        if source not in _SOURCES:
+            raise KeyError(f"Unknown source {source!r}. Available: {', '.join(_SOURCES)}")
+        result: Network = _SOURCES[source].load(name, **kwargs)
+        return result
+
+    found = _owners(name)
+    if not found:
+        raise KeyError(f"No dataset named {name!r} in any source: {', '.join(_SOURCES)}.")
+    if len(found) > 1:
+        raise KeyError(
+            f"{name!r} exists in several sources ({', '.join(found)}). Pass source= to choose."
+        )
+    loaded: Network = _SOURCES[found[0]].load(name, **kwargs)
+    return loaded