optimade-maker 0.3.0__tar.gz

optimade_maker-0.3.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Materials Cloud & Matthew Evans
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

optimade_maker-0.3.0/PKG-INFO

@@ -0,0 +1,164 @@
+Metadata-Version: 2.1
+Name: optimade-maker
+Version: 0.3.0
+Summary: Tools for making OPTIMADE APIs from raw structural data.
+License: MIT
+Keywords: optimade,jsonapi,materials
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Database
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pydantic~=2.2
+Requires-Dist: optimade[ase,server]~=1.1
+Requires-Dist: pyyaml~=6.0
+Requires-Dist: pymatgen>=2023.9
+Requires-Dist: pandas~=2.1
+Requires-Dist: pybtex~=0.24
+Requires-Dist: tqdm~=4.65
+Requires-Dist: requests~=2.31
+Requires-Dist: numpy~=1.26
+Requires-Dist: click~=8.1
+Provides-Extra: tests
+Requires-Dist: pytest~=7.4; extra == "tests"
+Requires-Dist: pytest-cov~=4.0; extra == "tests"
+Provides-Extra: dev
+Requires-Dist: black; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: types-all; extra == "dev"
+
+<div align="center" style="padding: 2em;">
+<span style="padding: 1em">
+<img height="70px" align="center" src="https://matsci.org/uploads/default/original/2X/b/bd2f59b3bf14fb046b74538750699d7da4c19ac1.svg">
+</span>
+</div>
+
+# <div align="center">optimade-maker</div>
+
+[![PyPI - Version](https://img.shields.io/pypi/v/optimade-maker?color=4CC61E)](https://pypi.org/project/optimade-maker/)
+
+Tools for making [OPTIMADE APIs](https://optimade.org) from various formats of structural data (e.g. an archive of CIF files).
+
+This repository contains the `src/optimade-maker` Python package and the corresponding CLI tool `optimake` that work towards this aim. Features include
+
+- definition of a config file format (`optimade.yaml`) for annotating data archives to be used in the OPTIMADE ecosystem;
+- conversion of the raw data into corresponding OPTIMADE types using pre-existing parsers (e.g., ASE for structures);
+- conversion of the annotated data archive into an intermediate JSONLines file format that can be ingested into a database and used to serve a full OPTIMADE API.
+- serving either an annotated data archive or a JSONLines file as an OPTIMADE API (using the [`optimade-python-tools`](https://github.com/Materials-Consortia/optimade-python-tools/)
+  reference server implementation).
+
+## Usage
+
+See `./examples` for a more complete set of supported formats and corresponding `optimade.yaml` config files.
+
+### Annotating with `optimade.yaml`
+
+To annotate your structural data for `optimade-maker`, the data archive needs to be accompanied by an `optimade.yaml` config file. The following is a simple example for a zip archive (`structures.zip`) of cif files together with an optional property file (`data.csv`):
+
+```yaml
+config_version: 0.1.0
+database_description: Simple database
+
+entries:
+  - entry_type: structures
+    entry_paths:
+      - file: structures.zip
+        matches:
+          - cifs/*/*.cif
+    # (optional) property file and definitions:
+    property_paths:
+      - file: data.csv
+    property_definitions:
+      - name: energy
+        title: Total energy per atom
+        description: The total energy per atom as computed by DFT
+        unit: eV/atom
+        type: float
+```
+
+### Structure `id`s and property files
+
+`optimade-maker` will assign an `id` for each structure based on its full path in the archive, following a simple deterministic rule: from the set of all archive paths, the maximum common path prefix and postfix (including file extensions) are removed. E.g.
+
+```
+structures.zip/cifs/set1/101.cif
+structures.zip/cifs/set2/102.cif
+```
+
+produces `["set1/101", "set2/102"]`.
+
+The property files need to either refer to these `id`s or the full path in the archive to be associated with a structure. E.g. a possible property `csv` file could be
+
+```csv
+id,energy
+set1/101,2.5
+structures.zip/cifs/set2/102.cif,3.2
+```
+
+### Installing and running `optimake`
+
+Install with
+
+```bash
+pip install optimade-maker
+```
+
+this will also make the `optimake` CLI utility available.
+
+For a folder containing the data archive and the `optimade.yaml` file (such as in `/examples`), run
+
+- `optimake convert .` to just convert the entry into the JSONL format (see below).
+- `optimake serve .` to start the OPTIMADE API (this also first converts the entry, if needed);
+
+For more detailed information see also `optimake --help`.
+
+## `optimade-maker` JSONLines Format
+
+As described above, `optimade-maker` works via an intermediate JSONLines file representation of an OPTIMADE API (see also the [corresponding issue in the specification](https://github.com/Materials-Consortia/OPTIMADE/issues/471)).
+This file should provide enough metadata to spin up an OPTIMADE API with many different entry types.
+The format is as follows:
+
+- First line must be a dictionary with the key `x-optimade`, containing a sub-dictionary of metadata (such as the OPTIMADE API version).
+- Second line contains the `info/structures` endpoint.
+- Third line contains the `info/references` endpoint, if present.
+- Then each line contains an entry from the corresponding individual structure/reference endpoints.
+
+```json
+{"x-optimade": {"meta": {"api_version": "1.1.0"}}}
+{"type": "info", "id": "structures", "properties": {...}}
+{"type": "info", "id": "references", "properties": {...}}
+{"type": "structures", "id": "1234", "attributes": {...}}
+{"type": "structures", "id": "1235", "attributes": {...}}
+{"type": "references", "id": "sfdas", "attributes": {...}}
+```
+
+NOTE: the `info/` endpoints in [OPTIMADE v1.2.0](https://www.optimade.org/specification/#entry-listing-info-endpoints) will include `type` and `id` as well.
+
+## Relevant links
+
+- [Roadmap and meeting notes](https://docs.google.com/document/d/1cIpwuX6Ty5d3ZHKYWktQaBBQcI9fYmgG_hsD1P1UpO4/edit)
+- [OPTIMADE serialization format notes](https://docs.google.com/document/d/1vf8_qxSRP5lCSb0P3M9gTr6nqkERxgOoSDno6YLcCjo/edit)
+- [Flow diagram](https://excalidraw.com/#json=MBNl66sARCQekVrKZXDg8,K35f5FwmiS46vlsYGMJdrw)
+
+## Contributors
+
+Initial prototype was created at the Paul Scherrer Institute, Switzerland in the week of
+12th-16th June 2023.
+
+Authors (alphabetical):
+
+- Kristjan Eimre
+- Matthew Evans
+- Giovanni Pizzi
+- Gian-Marco Rignanese
+- Jusong Yu
+- Xing Wang

optimade_maker-0.3.0/README.md

@@ -0,0 +1,126 @@
+<div align="center" style="padding: 2em;">
+<span style="padding: 1em">
+<img height="70px" align="center" src="https://matsci.org/uploads/default/original/2X/b/bd2f59b3bf14fb046b74538750699d7da4c19ac1.svg">
+</span>
+</div>
+
+# <div align="center">optimade-maker</div>
+
+[![PyPI - Version](https://img.shields.io/pypi/v/optimade-maker?color=4CC61E)](https://pypi.org/project/optimade-maker/)
+
+Tools for making [OPTIMADE APIs](https://optimade.org) from various formats of structural data (e.g. an archive of CIF files).
+
+This repository contains the `src/optimade-maker` Python package and the corresponding CLI tool `optimake` that work towards this aim. Features include
+
+- definition of a config file format (`optimade.yaml`) for annotating data archives to be used in the OPTIMADE ecosystem;
+- conversion of the raw data into corresponding OPTIMADE types using pre-existing parsers (e.g., ASE for structures);
+- conversion of the annotated data archive into an intermediate JSONLines file format that can be ingested into a database and used to serve a full OPTIMADE API.
+- serving either an annotated data archive or a JSONLines file as an OPTIMADE API (using the [`optimade-python-tools`](https://github.com/Materials-Consortia/optimade-python-tools/)
+  reference server implementation).
+
+## Usage
+
+See `./examples` for a more complete set of supported formats and corresponding `optimade.yaml` config files.
+
+### Annotating with `optimade.yaml`
+
+To annotate your structural data for `optimade-maker`, the data archive needs to be accompanied by an `optimade.yaml` config file. The following is a simple example for a zip archive (`structures.zip`) of cif files together with an optional property file (`data.csv`):
+
+```yaml
+config_version: 0.1.0
+database_description: Simple database
+
+entries:
+  - entry_type: structures
+    entry_paths:
+      - file: structures.zip
+        matches:
+          - cifs/*/*.cif
+    # (optional) property file and definitions:
+    property_paths:
+      - file: data.csv
+    property_definitions:
+      - name: energy
+        title: Total energy per atom
+        description: The total energy per atom as computed by DFT
+        unit: eV/atom
+        type: float
+```
+
+### Structure `id`s and property files
+
+`optimade-maker` will assign an `id` for each structure based on its full path in the archive, following a simple deterministic rule: from the set of all archive paths, the maximum common path prefix and postfix (including file extensions) are removed. E.g.
+
+```
+structures.zip/cifs/set1/101.cif
+structures.zip/cifs/set2/102.cif
+```
+
+produces `["set1/101", "set2/102"]`.
+
+The property files need to either refer to these `id`s or the full path in the archive to be associated with a structure. E.g. a possible property `csv` file could be
+
+```csv
+id,energy
+set1/101,2.5
+structures.zip/cifs/set2/102.cif,3.2
+```
+
+### Installing and running `optimake`
+
+Install with
+
+```bash
+pip install optimade-maker
+```
+
+this will also make the `optimake` CLI utility available.
+
+For a folder containing the data archive and the `optimade.yaml` file (such as in `/examples`), run
+
+- `optimake convert .` to just convert the entry into the JSONL format (see below).
+- `optimake serve .` to start the OPTIMADE API (this also first converts the entry, if needed);
+
+For more detailed information see also `optimake --help`.
+
+## `optimade-maker` JSONLines Format
+
+As described above, `optimade-maker` works via an intermediate JSONLines file representation of an OPTIMADE API (see also the [corresponding issue in the specification](https://github.com/Materials-Consortia/OPTIMADE/issues/471)).
+This file should provide enough metadata to spin up an OPTIMADE API with many different entry types.
+The format is as follows:
+
+- First line must be a dictionary with the key `x-optimade`, containing a sub-dictionary of metadata (such as the OPTIMADE API version).
+- Second line contains the `info/structures` endpoint.
+- Third line contains the `info/references` endpoint, if present.
+- Then each line contains an entry from the corresponding individual structure/reference endpoints.
+
+```json
+{"x-optimade": {"meta": {"api_version": "1.1.0"}}}
+{"type": "info", "id": "structures", "properties": {...}}
+{"type": "info", "id": "references", "properties": {...}}
+{"type": "structures", "id": "1234", "attributes": {...}}
+{"type": "structures", "id": "1235", "attributes": {...}}
+{"type": "references", "id": "sfdas", "attributes": {...}}
+```
+
+NOTE: the `info/` endpoints in [OPTIMADE v1.2.0](https://www.optimade.org/specification/#entry-listing-info-endpoints) will include `type` and `id` as well.
+
+## Relevant links
+
+- [Roadmap and meeting notes](https://docs.google.com/document/d/1cIpwuX6Ty5d3ZHKYWktQaBBQcI9fYmgG_hsD1P1UpO4/edit)
+- [OPTIMADE serialization format notes](https://docs.google.com/document/d/1vf8_qxSRP5lCSb0P3M9gTr6nqkERxgOoSDno6YLcCjo/edit)
+- [Flow diagram](https://excalidraw.com/#json=MBNl66sARCQekVrKZXDg8,K35f5FwmiS46vlsYGMJdrw)
+
+## Contributors
+
+Initial prototype was created at the Paul Scherrer Institute, Switzerland in the week of
+12th-16th June 2023.
+
+Authors (alphabetical):
+
+- Kristjan Eimre
+- Matthew Evans
+- Giovanni Pizzi
+- Gian-Marco Rignanese
+- Jusong Yu
+- Xing Wang

optimade_maker-0.3.0/pyproject.toml

@@ -0,0 +1,58 @@
+[project]
+name = "optimade-maker"
+description = "Tools for making OPTIMADE APIs from raw structural data."
+readme = "README.md"
+version = "0.3.0"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+keywords = ["optimade", "jsonapi", "materials"]
+
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Intended Audience :: Science/Research",
+    "Topic :: Database",
+    "Topic :: Scientific/Engineering",
+]
+
+dependencies = [
+    "pydantic~=2.2",
+    "optimade[server,ase]~=1.1",
+    "pyyaml~=6.0",
+    "pymatgen>=2023.9",
+    "pandas~=2.1",
+    "pybtex~=0.24",
+    "tqdm~=4.65",
+    "requests~=2.31",
+    "numpy~=1.26",
+    "click~=8.1"
+]
+
+[project.optional-dependencies]
+tests = ["pytest~=7.4", "pytest-cov~=4.0"]
+dev = ["black", "ruff", "pre-commit", "mypy", "isort", "types-all"]
+
+[tool.ruff]
+select = ["E", "F", "I", "W", "Q"]
+ignore = ["E501", "E402"]
+fixable = ["A", "B", "C", "D", "E", "F", "I"]
+unfixable = []
+target-version = "py311"
+per-file-ignores = {}
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+[tool.mypy]
+plugins = "pydantic.mypy"
+ignore_missing_imports = true
+follow_imports = "skip"
+
+[tool.isort]
+known_first_party = "optimade_maker"
+profile = "black"
+
+[project.scripts]
+optimake = "optimade_maker.cli:cli"

optimade_maker-0.3.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

optimade_maker-0.3.0/src/optimade_maker/__init__.py

@@ -0,0 +1,4 @@
+from optimade_maker.config import Config
+from optimade_maker.convert import convert_archive
+
+__all__ = ("Config", "convert_archive")

optimade_maker-0.3.0/src/optimade_maker/archive/archive_record.py

@@ -0,0 +1,183 @@
+import json
+import os
+from urllib.error import HTTPError, URLError
+
+import requests
+
+from optimade_maker.config import Config
+
+DEFAULT_ARCHIVE_URL = "https://archive.materialscloud.org"
+
+
+class ArchiveRecord:
+    """An class for Materials Cloud Archive record.
+    The class have the following methods:
+    1. get the url of a record by its id
+    2. get the metadata of a record by request the url
+    3. check if the record has a config file called "optimade.yaml"
+    4. if so, parse the config file, get the file list to be download.
+    5. download the files in the file list
+    6. convert the structure to OPTIMADE format (in another script)
+
+    Parameters:
+
+    id: int
+        id of the record. In MC archive, on the right panel,
+        "Export" --> "JSON", then find the "id" value.
+    archive_url: str
+        url of the archive.
+    dir: str
+        directory to save the downloaded files.
+    """
+
+    def __init__(self, id: int, archive_url: str = DEFAULT_ARCHIVE_URL) -> None:
+        self.id = id
+        self.archive_url = archive_url
+        self.url = self.get_record_url(id)
+
+        self.metadata = self.get_record_metadata()
+        self.doi_id = self.get_doi_id()
+        self.files_w_checksums = self.get_files_w_checksums()
+
+        self.default_path = os.path.join("/tmp/archive", self.doi_id)
+
+        self.optimade_config_name = self.check_optimade_config_name()
+
+    def check_optimade_config_name(self):
+        """
+        Check if optimade config file exists. If it doesn't, return None
+        """
+        optimade_yml_name = None
+        for name_candidate in ["optimade.yaml", "optimade.yml"]:
+            if name_candidate in self.files_w_checksums:
+                optimade_yml_name = name_candidate
+                break
+        return optimade_yml_name
+
+    def process(self):
+        if not self.is_optimade_record():
+            return
+        self.load_optimade_config()
+        self.download_files()
+        # self.convert_to_optimade()
+
+    def download_optimade_files(self, path=None):
+        if not self.is_optimade_record():
+            return
+        self.load_optimade_config()
+        self.download_files(path)
+
+    def get_record_url(self, record_id: int) -> str:
+        return self.archive_url + "api/records/" + str(record_id)
+
+    def get_file_url(self, filename: str) -> str:
+        # checksum = self.files_w_checksums[filename]
+        filename = filename.replace(" ", "+")
+        # original version, failing for
+        # https://staging-archive.materialscloud.org//record/file_stats?record_id=1412&checksum=md5:81b5fefab6bfa8e516d313b9cea39c66&filename=structures.zip
+        # url = (
+        #     self.archive_url
+        #     + f"/record/file_stats?record_id={record_id}&checksum={checksum}&filename={filename}"
+        # )
+        url = self.archive_url + f"/record/file?record_id={self.id}&filename={filename}"
+        return url
+
+    def get_record_metadata(self):
+        """
+        Get the metadata of a record by request the url.
+        """
+        try:
+            r = requests.get(self.url, allow_redirects=True, verify=False)
+            s = json.loads(r.content.decode("utf-8"))
+            return s["metadata"]
+        except HTTPError as e:
+            print("The server couldn't fulfill the request.")
+            print("Error code: ", e.code)
+        except URLError as e:
+            print("We failed to reach a server.")
+            print("Reason: ", e.reason)
+
+    def get_doi_id(self):
+        """
+        Get the DOI identifier of the record, e.g.
+        "10.24435/materialscloud:jq-0s" -> "jq-0s"
+        "10.24435/materialscloud:2020.0040/v1" -> "2020.0040/v1"
+
+        NOTE: the slash in the old format currently unsupported (e.g. can't make a folder,
+        or docker container), but these entries any way don't contain optimade.yml, so it
+        should be safe to ignore this for now.
+        """
+        return self.metadata["doi"].split(":")[-1]
+
+    def get_files_w_checksums(self):
+        """
+        Get the file list with checksums of a record.
+        """
+        files = {f["key"]: f["checksum"] for f in self.metadata["_files"]}
+        return files
+
+    def is_optimade_record(self):
+        """
+        return if the record has the optimade config file.
+        """
+        return self.optimade_config_name is not None
+
+    def download_optimade_config_file(self):
+        """
+        Try to download the optimade.yaml/yml file.
+        """
+        filename = self.optimade_config_name
+        url = self.get_file_url(filename)
+        response = requests.get(url, allow_redirects=True)
+        if not response.status_code == 200:
+            raise RuntimeError(f"Could not download {filename} file.")
+        return response
+
+    def load_optimade_config(self):
+        """
+        Download and parse the optimade.yaml/yml file.
+        """
+        response = self.download_optimade_config_file()
+        self.mc_config = Config.from_string(response.content.decode("utf-8"))
+
+    def download_files(self, path=None):
+        """
+        Download all files from the optimade file list.
+        """
+        import os
+        import shutil
+
+        from .utils import download_file
+
+        if not path:
+            path = self.default_path
+
+        # remove the directory if it exists
+        if os.path.exists(path) and os.path.isdir(path):
+            shutil.rmtree(path)
+        os.makedirs(path)
+
+        # download optimade.yml/yaml and rename to "yml->yaml"
+        file_url = self.get_file_url(self.optimade_config_name)
+        download_file(file_url, path, rename="optimade.yaml")
+
+        # download files in record
+        if hasattr(self.mc_config.entries, "jsonl_path"):
+            # case 1: jsonl file specified (either via `file: jsonl.gz` or `jsonl_path:`)
+            if hasattr(self.mc_config.entries, "file"):
+                # download `file:`, if specified
+                file_url = self.get_file_url(self.mc_config.entries.file)
+                download_file(file_url, path)
+            else:
+                # otherwise download the `jsonl_path:`
+                file_url = self.get_file_url(self.mc_config.entries.jsonl_path)
+                download_file(file_url, path)
+        else:
+            # case 2: files specified as entry_paths/property_paths
+            for entry in self.mc_config.entries:
+                list_of_files = [path.file for path in entry.entry_paths]
+                if hasattr(entry, "property_paths"):
+                    list_of_files += [path.file for path in entry.property_paths]
+                for fname in list_of_files:
+                    file_url = self.get_file_url(fname)
+                    download_file(file_url, path)

optimade_maker-0.3.0/src/optimade_maker/archive/cli.py

@@ -0,0 +1,13 @@
+import click
+
+from .scan_records import scan_records
+
+
+@click.group()
+def cli():
+    pass
+
+
+@cli.command()
+def scan():
+    scan_records()

optimade_maker-0.3.0/src/optimade_maker/archive/scan_records.py

@@ -0,0 +1,36 @@
+import tqdm
+
+from optimade_maker.archive.archive_record import ArchiveRecord
+from optimade_maker.archive.utils import get_all_records, get_parsed_records
+
+DEFAULT_ARCHIVE_URL = "https://archive.materialscloud.org/"
+
+
+def process_records(records: list, archive_url: str = DEFAULT_ARCHIVE_URL):
+    """
+    Scan the Materials Cloud Archive entries, read the file info
+    and check if there is a file called "optimade.y(ml|aml)".
+    If so, triger the conversion step.
+    """
+    # get the old records by looping through the optimade_id.json files in the folders
+    old_record_ids = get_parsed_records()
+    for record in tqdm.tqdm(records, desc="Processing records"):
+        record_id = record["id"]
+        if record_id in old_record_ids:
+            continue
+        record = ArchiveRecord(record_id, archive_url=archive_url)
+        if record.is_optimade_record():
+            print(f"Record {record_id} is a OPTIMADE record.")
+            record.process()
+
+
+def scan_records(archive_url=DEFAULT_ARCHIVE_URL):
+    """This script can be run as a cron job to check for new optimade entries in the Materials Cloud Archive, and convert them to OPTIMADE format."""
+    print("Start scanning the Materials Cloud Archive for new OPTIMADE entries...")
+    records = get_all_records(archive_url)
+    process_records(records, archive_url)
+
+
+if __name__ == "__main__":
+    url = "https://staging-archive.materialscloud.org/"
+    scan_records(url)