mattergraph-connectors 0.1.0__tar.gz

mattergraph_connectors-0.1.0/.gitignore

@@ -0,0 +1,48 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+dist/
+build/
+.venv/
+venv/
+.mypy_cache/
+.ruff_cache/
+.pytest_cache/
+.hypothesis/
+.coverage
+coverage.xml
+htmlcov/
+
+# Docs build output
+site/
+
+# Env
+.env
+.env.local
+*.local
+
+# Node
+node_modules/
+apps/web/dist/
+apps/web/playwright-report/
+apps/web/test-results/
+*.tsbuildinfo
+.next/
+out/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+
+# OS
+.DS_Store
+
+# Data artifacts (keep demo/ tracked)
+data/cache/
+*.sqlite3
+
+.claude
+apps/private-platform-ui/

mattergraph_connectors-0.1.0/PKG-INFO

@@ -0,0 +1,83 @@
+Metadata-Version: 2.4
+Name: mattergraph-connectors
+Version: 0.1.0
+Summary: Connectors to public materials databases for MatterGraph.
+Project-URL: Homepage, https://github.com/cyrusmo/MatterGraph
+Project-URL: Repository, https://github.com/cyrusmo/MatterGraph
+Project-URL: Issues, https://github.com/cyrusmo/MatterGraph/issues
+Project-URL: Changelog, https://github.com/cyrusmo/MatterGraph/blob/main/CHANGELOG.md
+Author: MatterGraph contributors
+License-Expression: Apache-2.0
+Keywords: jarvis,lematerial,materials-project,materials-science,nomad
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Chemistry
+Classifier: Topic :: Scientific/Engineering :: Physics
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: mattergraph-core~=0.1.0
+Requires-Dist: pandas>=2.0
+Requires-Dist: pydantic>=2.5
+Requires-Dist: pymatgen>=2024.1.1
+Provides-Extra: all
+Requires-Dist: jarvis-tools>=2024.1.1; extra == 'all'
+Requires-Dist: mp-api>=0.40.0; extra == 'all'
+Provides-Extra: jarvis
+Requires-Dist: jarvis-tools>=2024.1.1; extra == 'jarvis'
+Provides-Extra: mp
+Requires-Dist: mp-api>=0.40.0; extra == 'mp'
+Description-Content-Type: text/markdown
+
+# mattergraph-connectors
+
+Connectors to public materials databases for [MatterGraph](https://github.com/cyrusmo/MatterGraph).
+
+Each connector returns normalized `Material` records from `mattergraph-core`, so downstream filtering, graph export, and scoring work identically regardless of source.
+
+## Sources
+
+| Connector | Status |
+|---|---|
+| Materials Project (`mp-api`) | Supported — requires `MP_API_KEY` |
+| JARVIS-DFT (`jarvis-tools`) | Supported |
+| NOMAD | Public metadata reads, no API key needed. Metadata-only in v0.1 — does not fetch archives or derived scalar properties. |
+| LeMat-Bulk | Companion adapter (records, parquet, or Hugging Face `datasets`) |
+| Local CSV | Supported |
+| OQMD | Stub — preserves the API surface, returns no records |
+
+Heavy source SDKs are imported lazily, so importing the package stays cheap and a missing optional dependency produces a readable install hint rather than an `ImportError` traceback.
+
+## Install
+
+```bash
+pip install mattergraph-connectors
+```
+
+## Example
+
+```python
+import json
+from pathlib import Path
+
+from mattergraph_connectors import LeMatBulk
+
+artifact = json.loads(Path("data/demo/spc_real_snapshot.json").read_text())
+dataset = LeMatBulk.from_records(artifact["records"], subset="compatible_pbe")
+
+candidates = (
+    dataset
+    .filter_elements(include=["Ti", "Al", "N"])
+    .filter_complexity(max_nsites=16, max_nelements=3)
+    .create_slice("spc_tialn_candidates_v1", target="energy_above_hull")
+)
+print(candidates.report())
+```
+
+## License
+
+Apache-2.0

mattergraph_connectors-0.1.0/README.md

@@ -0,0 +1,48 @@
+# mattergraph-connectors
+
+Connectors to public materials databases for [MatterGraph](https://github.com/cyrusmo/MatterGraph).
+
+Each connector returns normalized `Material` records from `mattergraph-core`, so downstream filtering, graph export, and scoring work identically regardless of source.
+
+## Sources
+
+| Connector | Status |
+|---|---|
+| Materials Project (`mp-api`) | Supported — requires `MP_API_KEY` |
+| JARVIS-DFT (`jarvis-tools`) | Supported |
+| NOMAD | Public metadata reads, no API key needed. Metadata-only in v0.1 — does not fetch archives or derived scalar properties. |
+| LeMat-Bulk | Companion adapter (records, parquet, or Hugging Face `datasets`) |
+| Local CSV | Supported |
+| OQMD | Stub — preserves the API surface, returns no records |
+
+Heavy source SDKs are imported lazily, so importing the package stays cheap and a missing optional dependency produces a readable install hint rather than an `ImportError` traceback.
+
+## Install
+
+```bash
+pip install mattergraph-connectors
+```
+
+## Example
+
+```python
+import json
+from pathlib import Path
+
+from mattergraph_connectors import LeMatBulk
+
+artifact = json.loads(Path("data/demo/spc_real_snapshot.json").read_text())
+dataset = LeMatBulk.from_records(artifact["records"], subset="compatible_pbe")
+
+candidates = (
+    dataset
+    .filter_elements(include=["Ti", "Al", "N"])
+    .filter_complexity(max_nsites=16, max_nelements=3)
+    .create_slice("spc_tialn_candidates_v1", target="energy_above_hull")
+)
+print(candidates.report())
+```
+
+## License
+
+Apache-2.0

mattergraph_connectors-0.1.0/mattergraph_connectors/__init__.py

@@ -0,0 +1,213 @@
+from __future__ import annotations
+
+from importlib import import_module
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+  from mattergraph_connectors.base import (
+    Connector,
+    ConnectorQuery,
+    apply_property_filter,
+    coerce_query,
+    connector_provenance,
+  )
+  from mattergraph_connectors.http_policy import ConnectorHTTPPolicy, MemoryResponseCache
+  from mattergraph_connectors.jarvis import JarvisConnector
+  from mattergraph_connectors.lematerial import LeMatBulk
+  from mattergraph_connectors.local_csv import load_materials_from_csv
+  from mattergraph_connectors.local_import import (
+    DatasetImportMapping,
+    ImportIssue,
+    ImportLimitError,
+    ImportReport,
+    ImportResult,
+    ImportValidationError,
+    NormalizedImport,
+    PropertyColumnMapping,
+    import_local_content,
+    inspect_local_content,
+  )
+  from mattergraph_connectors.materials_project import MaterialsProjectConnector
+  from mattergraph_connectors.nomad import (
+    NOMADConnector,
+    NOMADConnectorError,
+    NOMADHTTPError,
+    NOMADMappingError,
+    NOMADPayloadError,
+    NOMADStubConnector,
+  )
+  from mattergraph_connectors.optimade import (
+    PROVIDERS,
+    OptimadeConnector,
+    OptimadeConnectorError,
+    OptimadeHTTPError,
+    OptimadeMappingError,
+    OptimadePayloadError,
+  )
+  from mattergraph_connectors.oqmd import OQMDStubConnector
+
+_EXPORTS: dict[str, tuple[str, str, str | None]] = {
+  "Connector": ("mattergraph_connectors.base", "Connector", None),
+  "ConnectorQuery": ("mattergraph_connectors.base", "ConnectorQuery", None),
+  "apply_property_filter": ("mattergraph_connectors.base", "apply_property_filter", None),
+  "coerce_query": ("mattergraph_connectors.base", "coerce_query", None),
+  "connector_provenance": ("mattergraph_connectors.base", "connector_provenance", None),
+  "MaterialsProjectConnector": (
+    "mattergraph_connectors.materials_project",
+    "MaterialsProjectConnector",
+    (
+      "MaterialsProjectConnector needs the optional `mp-api` dependency: install the extra "
+      "with `pip install 'mattergraph-connectors[mp]'`, or run "
+      "`uv sync --all-packages --group dev --extra all` for a full workspace environment."
+    ),
+  ),
+  "JarvisConnector": (
+    "mattergraph_connectors.jarvis",
+    "JarvisConnector",
+    (
+      "JarvisConnector needs the optional `jarvis-tools` dependency: install the extra "
+      "with `pip install 'mattergraph-connectors[jarvis]'`, or run "
+      "`uv sync --all-packages --group dev --extra all` for a full workspace environment."
+    ),
+  ),
+  "load_materials_from_csv": (
+    "mattergraph_connectors.local_csv",
+    "load_materials_from_csv",
+    None,
+  ),
+  "ConnectorHTTPPolicy": (
+    "mattergraph_connectors.http_policy",
+    "ConnectorHTTPPolicy",
+    None,
+  ),
+  "MemoryResponseCache": (
+    "mattergraph_connectors.http_policy",
+    "MemoryResponseCache",
+    None,
+  ),
+  "DatasetImportMapping": ("mattergraph_connectors.local_import", "DatasetImportMapping", None),
+  "ImportIssue": ("mattergraph_connectors.local_import", "ImportIssue", None),
+  "ImportLimitError": ("mattergraph_connectors.local_import", "ImportLimitError", None),
+  "ImportReport": ("mattergraph_connectors.local_import", "ImportReport", None),
+  "ImportResult": ("mattergraph_connectors.local_import", "ImportResult", None),
+  "ImportValidationError": (
+    "mattergraph_connectors.local_import",
+    "ImportValidationError",
+    None,
+  ),
+  "NormalizedImport": ("mattergraph_connectors.local_import", "NormalizedImport", None),
+  "PropertyColumnMapping": (
+    "mattergraph_connectors.local_import",
+    "PropertyColumnMapping",
+    None,
+  ),
+  "import_local_content": ("mattergraph_connectors.local_import", "import_local_content", None),
+  "inspect_local_content": ("mattergraph_connectors.local_import", "inspect_local_content", None),
+  "LeMatBulk": (
+    "mattergraph_connectors.lematerial",
+    "LeMatBulk",
+    None,
+  ),
+  "OQMDStubConnector": (
+    "mattergraph_connectors.oqmd",
+    "OQMDStubConnector",
+    None,
+  ),
+  "NOMADConnector": (
+    "mattergraph_connectors.nomad",
+    "NOMADConnector",
+    None,
+  ),
+  "NOMADConnectorError": (
+    "mattergraph_connectors.nomad",
+    "NOMADConnectorError",
+    None,
+  ),
+  "NOMADHTTPError": (
+    "mattergraph_connectors.nomad",
+    "NOMADHTTPError",
+    None,
+  ),
+  "NOMADMappingError": (
+    "mattergraph_connectors.nomad",
+    "NOMADMappingError",
+    None,
+  ),
+  "NOMADPayloadError": (
+    "mattergraph_connectors.nomad",
+    "NOMADPayloadError",
+    None,
+  ),
+  "NOMADStubConnector": (
+    "mattergraph_connectors.nomad",
+    "NOMADStubConnector",
+    None,
+  ),
+  "OptimadeConnector": ("mattergraph_connectors.optimade", "OptimadeConnector", None),
+  "OptimadeConnectorError": (
+    "mattergraph_connectors.optimade",
+    "OptimadeConnectorError",
+    None,
+  ),
+  "OptimadeHTTPError": ("mattergraph_connectors.optimade", "OptimadeHTTPError", None),
+  "OptimadeMappingError": ("mattergraph_connectors.optimade", "OptimadeMappingError", None),
+  "OptimadePayloadError": ("mattergraph_connectors.optimade", "OptimadePayloadError", None),
+  "PROVIDERS": ("mattergraph_connectors.optimade", "PROVIDERS", None),
+}
+
+__all__ = [
+  "Connector",
+  "ConnectorQuery",
+  "apply_property_filter",
+  "coerce_query",
+  "connector_provenance",
+  "MaterialsProjectConnector",
+  "JarvisConnector",
+  "LeMatBulk",
+  "load_materials_from_csv",
+  "ConnectorHTTPPolicy",
+  "MemoryResponseCache",
+  "DatasetImportMapping",
+  "ImportIssue",
+  "ImportLimitError",
+  "ImportReport",
+  "ImportResult",
+  "ImportValidationError",
+  "NormalizedImport",
+  "PropertyColumnMapping",
+  "import_local_content",
+  "inspect_local_content",
+  "OQMDStubConnector",
+  "NOMADConnector",
+  "NOMADConnectorError",
+  "NOMADHTTPError",
+  "NOMADMappingError",
+  "NOMADPayloadError",
+  "NOMADStubConnector",
+  "OptimadeConnector",
+  "OptimadeConnectorError",
+  "OptimadeHTTPError",
+  "OptimadeMappingError",
+  "OptimadePayloadError",
+  "PROVIDERS",
+]
+
+
+def __getattr__(name: str) -> Any:
+  if name not in _EXPORTS:
+    msg = f"module {__name__!r} has no attribute {name!r}"
+    raise AttributeError(msg)
+  module_name, attr_name, hint = _EXPORTS[name]
+  try:
+    module = import_module(module_name)
+  except ImportError as e:
+    if hint is None:
+      raise
+    raise ImportError(hint) from e
+  value = getattr(module, attr_name)
+  globals()[name] = value
+  return value
+
+
+def __dir__() -> list[str]:
+  return sorted(set(globals()) | set(__all__))

mattergraph_connectors-0.1.0/mattergraph_connectors/base.py

@@ -0,0 +1,183 @@
+"""The contract every MatterGraph connector implements.
+
+Before this module the connector "interface" was a convention: a class with a ``fetch``
+returning ``list[Material]``. Nothing enforced it, and the signatures had already drifted
+four ways — Materials Project took ``material_ids``/``properties``/``num_chunks``/``chunk_size``,
+JARVIS took ``max_records``, NOMAD took ``max_records``/``page_size``, and the OQMD stub took
+``*args, **kwargs`` and ignored all of them. :class:`ConnectorQuery` gives those one shape and
+:class:`Connector` makes conformance testable.
+"""
+
+from __future__ import annotations
+
+import warnings
+from typing import Any, Protocol, runtime_checkable
+
+from mattergraph.normalization.properties import canonical_property_name
+from mattergraph.schema.material import Material
+from mattergraph.schema.property import PropertyMethod
+from mattergraph.schema.provenance import ProvenanceRecord
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+# Legacy keyword names accepted by the pre-contract fetch() signatures, mapped onto their
+# ConnectorQuery field. Callers using these get a DeprecationWarning, not a break.
+_LEGACY_ALIASES: dict[str, str] = {
+  "material_ids": "source_ids",
+  "chunk_size": "page_size",
+}
+
+
+class ConnectorQuery(BaseModel):
+  """A source-independent description of what to fetch.
+
+  Every field is a *request*, not a guarantee: a connector that cannot honor one must say so
+  by raising, never by silently ignoring it.
+  """
+
+  model_config = ConfigDict(extra="forbid", validate_assignment=True)
+
+  elements: list[str] | None = Field(
+    default=None,
+    description="Restrict to records containing these element symbols",
+  )
+  source_ids: list[str] | None = Field(
+    default=None,
+    description="Fetch these upstream identifiers directly, e.g. mp-149 or JVASP-1002",
+  )
+  properties: list[str] | None = Field(
+    default=None,
+    description="Canonical property names to return; connectors that cannot filter must raise",
+  )
+  max_records: int = Field(default=50, description="Upper bound on returned materials")
+  page_size: int = Field(default=25, description="Records per upstream request where paged")
+
+  @field_validator("elements", "source_ids", "properties")
+  @classmethod
+  def _clean_list(cls, value: list[str] | None) -> list[str] | None:
+    if value is None:
+      return None
+    cleaned = [item.strip() for item in value if str(item).strip()]
+    return cleaned or None
+
+  @field_validator("max_records", "page_size")
+  @classmethod
+  def _positive(cls, value: int, info: Any) -> int:
+    if value < 1:
+      msg = f"{info.field_name} must be at least 1"
+      raise ValueError(msg)
+    return value
+
+
+@runtime_checkable
+class Connector(Protocol):
+  """A source of :class:`Material` records.
+
+  ``isinstance(obj, Connector)`` checks that the members exist, not that their signatures
+  match — see ``tests/test_connector_contract.py``, which also exercises the call itself.
+  """
+
+  source_name: str
+
+  def fetch(self, query: ConnectorQuery | None = None, **legacy: Any) -> list[Material]:
+    """Return at most ``query.max_records`` materials matching ``query``."""
+    ...
+
+
+def coerce_query(
+  query: ConnectorQuery | None,
+  legacy: dict[str, Any],
+  *,
+  source_name: str,
+) -> ConnectorQuery:
+  """Accept either a :class:`ConnectorQuery` or the pre-contract keyword form.
+
+  The keyword form is kept working because ``scripts/ingest_*.py`` and ``examples/`` use it;
+  it warns rather than breaking. Passing both forms at once is a caller bug and raises.
+  """
+  if query is not None and legacy:
+    msg = (
+      f"{source_name}.fetch() got both a ConnectorQuery and keyword arguments "
+      f"({', '.join(sorted(legacy))}); pass one or the other"
+    )
+    raise TypeError(msg)
+  if query is not None:
+    return query
+  if not legacy:
+    return ConnectorQuery()
+
+  warnings.warn(
+    f"Calling {source_name}.fetch() with keyword arguments is deprecated; "
+    f"pass a ConnectorQuery instead, e.g. fetch(ConnectorQuery(elements=['Ti'])).",
+    DeprecationWarning,
+    stacklevel=3,
+  )
+  mapped: dict[str, Any] = {}
+  for key, value in legacy.items():
+    field = _LEGACY_ALIASES.get(key, key)
+    if field not in ConnectorQuery.model_fields:
+      msg = f"{source_name}.fetch() got an unexpected keyword argument {key!r}"
+      raise TypeError(msg)
+    if value is not None:
+      mapped[field] = value
+  return ConnectorQuery(**mapped)
+
+
+def apply_property_filter(
+  materials: list[Material],
+  query: ConnectorQuery,
+  *,
+  supported: frozenset[str],
+  source_name: str,
+) -> list[Material]:
+  """Restrict each material to the requested properties, or raise if it cannot be done.
+
+  A connector that accepts ``properties`` and returns everything anyway is the failure this
+  guards against: the caller has no way to tell that the filter did nothing.
+  """
+  if not query.properties:
+    return materials
+
+  requested = {canonical_property_name(name) for name in query.properties}
+  unsupported = sorted(requested - supported)
+  if unsupported:
+    msg = (
+      f"{source_name} cannot return {', '.join(unsupported)}; "
+      f"supported properties are {', '.join(sorted(supported))}"
+    )
+    raise ValueError(msg)
+
+  for material in materials:
+    material.properties = [p for p in material.properties if p.name in requested]
+  return materials
+
+
+def connector_provenance(
+  source: str,
+  *,
+  source_id: str | None = None,
+  method: PropertyMethod = PropertyMethod.DFT,
+  notes: str | None = None,
+  parameters: dict[str, Any] | None = None,
+) -> ProvenanceRecord:
+  """Build the lineage record every ingested material carries.
+
+  ``constraint.provenance`` asks that a value say where it came from. Connectors previously
+  put lineage in ``Material.metadata``, which is untyped and invisible to anything that
+  reasons about provenance, leaving ``Material.provenance`` empty on every ingested record.
+  """
+  return ProvenanceRecord(
+    source=source,
+    method=method,
+    source_id=source_id,
+    notes=notes,
+    parameters=parameters,
+  )
+
+
+__all__ = [
+  "Connector",
+  "ConnectorQuery",
+  "apply_property_filter",
+  "coerce_query",
+  "connector_provenance",
+]