nomad-ml-workflows 0.0.6__tar.gz

nomad_ml_workflows-0.0.6/.devcontainer/devcontainer.json

@@ -0,0 +1,25 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/python
+{
+  "name": "Python 3",
+  // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+  "image": "mcr.microsoft.com/devcontainers/python:1-3.12-bookworm",
+  // Features to add to the dev container. More info: https://containers.dev/features.
+  // "features": {},
+  // Use 'forwardPorts' to make a list of ports inside the container available locally.
+  // "forwardPorts": [],
+  // Use 'postCreateCommand' to run commands after the container is created.
+  "postCreateCommand": "pipx install cruft",
+  // Configure tool-specific properties.
+  "customizations": {
+    // Configure properties specific to VS Code.
+    "vscode": {
+      // Add the IDs of extensions you want installed when the container is created.
+      "extensions": [
+        "charliermarsh.ruff"
+      ]
+    }
+  }
+  // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+  // "remoteUser": "root"
+}

nomad_ml_workflows-0.0.6/.github/dependabot.yml

@@ -0,0 +1,12 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for more information:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+# https://containers.dev/guide/dependabot
+
+version: 2
+updates:
+ - package-ecosystem: "devcontainers"
+   directory: "/"
+   schedule:
+     interval: weekly

nomad_ml_workflows-0.0.6/.github/workflows/actions.yml

@@ -0,0 +1,43 @@
+name: install-and-test-workflow
+
+on: [push]
+
+env:
+  UV_VERSION: 0.9
+  PYTHON_VERSION: 3.12
+  UV_SYSTEM_PYTHON: true
+
+jobs:
+  install-and-test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+      with:
+          version: ${{ env.UV_VERSION }}
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+          python-version: ${{ env.PYTHON_VERSION }}
+    - name: Install dependencies
+      run: |
+        uv sync --all-extras
+    - name: Test with pytest
+      run: |
+        uv run --with coverage coverage run -m pytest -sv
+    - name: Submit to coveralls
+      continue-on-error: true
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        uv run --with coveralls coveralls --service=github
+
+  ruff-linting:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/ruff-action@v3
+      - run: ruff check .
+      # Uncomment the following line to enable format check
+      # - run: ruff format --check .

nomad_ml_workflows-0.0.6/.github/workflows/mkdocs-deploy.yml

@@ -0,0 +1,34 @@
+name: Deploy MkDocs Site
+
+on:
+  push:
+    branches:
+      - main  # Triggers deployment on push to the main branch
+
+env:
+  REGISTRY: ghcr.io
+  UV_VERSION: 0.9
+  PYTHON_VERSION: 3.12
+  UV_SYSTEM_PYTHON: true
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: ${{ env.UV_VERSION }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Build and Deploy
+        run: |
+          uv run --extra dev mkdocs gh-deploy --force --remote-branch gh-pages

nomad_ml_workflows-0.0.6/.github/workflows/publish-pypi.yml

@@ -0,0 +1,39 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Publish Python Package to PyPI
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build
+    - name: Build package
+      run: python -m build
+    - name: Publish package
+      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_API_TOKEN }}

nomad_ml_workflows-0.0.6/.gitignore

@@ -0,0 +1,130 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+.pyenv
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

nomad_ml_workflows-0.0.6/.vscode/settings.json

@@ -0,0 +1,21 @@
+{
+  "editor.rulers": [
+    88
+  ],
+  "editor.renderWhitespace": "all",
+  "editor.tabSize": 4,
+  "files.trimTrailingWhitespace": true,
+  "[python]": {
+    "editor.formatOnSave": true,
+    "editor.codeActionsOnSave": {
+      "source.fixAll": "explicit",
+      "source.organizeImports": "explicit"
+    },
+    "editor.defaultFormatter": "charliermarsh.ruff"
+  },
+  "python.testing.pytestArgs": [
+    "tests"
+  ],
+  "python.testing.unittestEnabled": false,
+  "python.testing.pytestEnabled": true
+}

nomad_ml_workflows-0.0.6/LICENSE

@@ -0,0 +1,22 @@
+
+The MIT License (MIT)
+
+Copyright (c) 2025 Sarthak Kapoor
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

nomad_ml_workflows-0.0.6/PKG-INFO

@@ -0,0 +1,197 @@
+Metadata-Version: 2.4
+Name: nomad-ml-workflows
+Version: 0.0.6
+Summary: A NOMAD plugin for managing ML workflows.
+Author-email: Sarthak Kapoor <sarthak.kapoor@physik.hu-berlin.de>
+Maintainer-email: Sarthak Kapoor <sarthak.kapoor@physik.hu-berlin.de>
+License: 
+        The MIT License (MIT)
+        
+        Copyright (c) 2025 Sarthak Kapoor
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in
+        all copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+        THE SOFTWARE.
+        
+Project-URL: Repository, https://github.com/FAIRmat-NFDI/nomad-ml-workflows
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: nomad-lab>=1.4.0
+Requires-Dist: json-stream
+Requires-Dist: pydantic
+Requires-Dist: temporalio
+Provides-Extra: dev
+Requires-Dist: nomad-lab[infrastructure]>=1.4.0; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: structlog; extra == "dev"
+Requires-Dist: mkdocs; extra == "dev"
+Requires-Dist: mkdocs-material>=8.1.1; extra == "dev"
+Requires-Dist: pymdown-extensions; extra == "dev"
+Requires-Dist: mkdocs-click; extra == "dev"
+Requires-Dist: pytest-asyncio; extra == "dev"
+Provides-Extra: cpu-action
+Requires-Dist: pyarrow; extra == "cpu-action"
+Dynamic: license-file
+
+# nomad-ml-workflows
+
+A NOMAD plugin for managing ML workflows. Currently, it provides an action to export large number of entries from NOMAD database as tabular data files. Other ML workflow related actions and schemas will be added in future.
+
+## 📦 Installation
+You can install the plugin using pip:
+```sh
+pip install nomad-ml-workflows @ git+https://github.com/FAIRmat-NFDI/nomad-ml-workflows.git
+```
+
+However, to fully utilize the plugin, you need to add it to your NOMAD instance as described [below](#-adding-this-plugin-to-nomad).
+
+## ✨ Features
+
+- Export a large number of NOMAD entries as tabular data files (CSV, Parquet) using NOMAD Actions. Once the action is triggered, it will:
+  - Search entries based on user-defined criteria.
+  - Optionally include or exclude data fields from the entries.
+  - Package the entries into tabular data files like CSV or Parquet (or as JSON)
+  - Export the files to a specified Project (or previously known as Upload) in NOAMD.
+
+  These can then be downloaded from the NOMAD web interface for local use.
+
+## ⚙️ Configuration
+The Export Entries action can be configured using the following parameters in
+the `nomad.yaml` configuration file of your NOMAD Oasis instance:
+
+```yaml
+plugins:
+  entry_points:
+    options:
+      nomad_ml_workflows.actions:export_entries:
+        search_batch_timeout: 7200
+        # Timeout (in seconds) for each search batch in the Export Entries
+        # action. Set this accordingly to time out longer searches.
+        max_entries_export_limit: 100000
+        # Maximum number of entries that can be exported in a single
+        # Export Entries action.
+```
+
+
+## 🚀 Adding this plugin to NOMAD
+
+Currently, NOMAD has two distinct flavors that are relevant depending on your role as an user:
+1. [A NOMAD Oasis](#adding-this-plugin-in-your-nomad-oasis): any user with a NOMAD Oasis instance.
+2. [Local NOMAD installation and the source code of NOMAD](#adding-this-plugin-in-your-local-nomad-installation-and-the-source-code-of-nomad): internal developers.
+
+### Adding this plugin in your NOMAD Oasis
+
+Read the [NOMAD plugin documentation](https://nomad-lab.eu/prod/v1/staging/docs/howto/oasis/plugins_install.html) for all details on how to deploy the plugin on your NOMAD instance.
+
+### Adding this plugin in your local NOMAD installation and the source code of NOMAD
+
+We now recommend using the dedicated [`nomad-distro-dev`](https://github.com/FAIRmat-NFDI/nomad-distro-dev) repository to simplify the process. Please refer to that repository for detailed instructions.
+
+
+## 🛠️ Development
+
+If you want to develop locally this plugin, clone the project and in the plugin folder, create a virtual environment (you can use Python 3.10, 3.11 or 3.12):
+```sh
+git clone https://github.com/FAIRmat-NFDI/nomad-ml-workflows.git
+cd nomad-ml-workflows
+python3.11 -m venv .pyenv
+. .pyenv/bin/activate
+```
+
+Make sure to have `pip` upgraded:
+```sh
+pip install --upgrade pip
+```
+
+We recommend installing `uv` for fast pip installation of the packages:
+```sh
+pip install uv
+```
+
+Install the `nomad-lab` package:
+```sh
+uv pip install -e '.[dev]'
+```
+
+### Run linting and auto-formatting
+
+We use [Ruff](https://docs.astral.sh/ruff/) for linting and formatting the code. Ruff auto-formatting is also a part of the GitHub workflow actions. You can run locally:
+```sh
+ruff check .
+ruff format . --check
+```
+
+### Debugging
+
+For interactive debugging of the tests, use `pytest` with the `--pdb` flag. We recommend using an IDE for debugging, e.g., _VSCode_. If that is the case, add the following snippet to your `.vscode/launch.json`:
+```json
+{
+  "configurations": [
+      {
+        "name": "<descriptive tag>",
+        "type": "debugpy",
+        "request": "launch",
+        "cwd": "${workspaceFolder}",
+        "program": "${workspaceFolder}/.pyenv/bin/pytest",
+        "justMyCode": true,
+        "env": {
+            "_PYTEST_RAISE": "1"
+        },
+        "args": [
+            "-sv",
+            "--pdb",
+            "<path-to-plugin-tests>",
+        ]
+    }
+  ]
+}
+```
+
+where `<path-to-plugin-tests>` must be changed to the local path to the test module to be debugged.
+
+The settings configuration file `.vscode/settings.json` automatically applies the linting and formatting upon saving the modified file.
+
+### Documentation on Github pages
+
+To view the documentation locally, install the related packages using:
+```sh
+uv pip install -r requirements_docs.txt
+```
+
+Run the documentation server:
+```sh
+mkdocs serve
+```
+
+
+## 👥 Main contributors
+| Name | E-mail     |
+|------|------------|
+| Sarthak Kapoor | [sarthak.kapoor@physik.hu-berlin.de](mailto:sarthak.kapoor@physik.hu-berlin.de)
+
+
+## 📄 License
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

nomad_ml_workflows-0.0.6/README.md

@@ -0,0 +1,139 @@
+# nomad-ml-workflows
+
+A NOMAD plugin for managing ML workflows. Currently, it provides an action to export large number of entries from NOMAD database as tabular data files. Other ML workflow related actions and schemas will be added in future.
+
+## 📦 Installation
+You can install the plugin using pip:
+```sh
+pip install nomad-ml-workflows @ git+https://github.com/FAIRmat-NFDI/nomad-ml-workflows.git
+```
+
+However, to fully utilize the plugin, you need to add it to your NOMAD instance as described [below](#-adding-this-plugin-to-nomad).
+
+## ✨ Features
+
+- Export a large number of NOMAD entries as tabular data files (CSV, Parquet) using NOMAD Actions. Once the action is triggered, it will:
+  - Search entries based on user-defined criteria.
+  - Optionally include or exclude data fields from the entries.
+  - Package the entries into tabular data files like CSV or Parquet (or as JSON)
+  - Export the files to a specified Project (or previously known as Upload) in NOAMD.
+
+  These can then be downloaded from the NOMAD web interface for local use.
+
+## ⚙️ Configuration
+The Export Entries action can be configured using the following parameters in
+the `nomad.yaml` configuration file of your NOMAD Oasis instance:
+
+```yaml
+plugins:
+  entry_points:
+    options:
+      nomad_ml_workflows.actions:export_entries:
+        search_batch_timeout: 7200
+        # Timeout (in seconds) for each search batch in the Export Entries
+        # action. Set this accordingly to time out longer searches.
+        max_entries_export_limit: 100000
+        # Maximum number of entries that can be exported in a single
+        # Export Entries action.
+```
+
+
+## 🚀 Adding this plugin to NOMAD
+
+Currently, NOMAD has two distinct flavors that are relevant depending on your role as an user:
+1. [A NOMAD Oasis](#adding-this-plugin-in-your-nomad-oasis): any user with a NOMAD Oasis instance.
+2. [Local NOMAD installation and the source code of NOMAD](#adding-this-plugin-in-your-local-nomad-installation-and-the-source-code-of-nomad): internal developers.
+
+### Adding this plugin in your NOMAD Oasis
+
+Read the [NOMAD plugin documentation](https://nomad-lab.eu/prod/v1/staging/docs/howto/oasis/plugins_install.html) for all details on how to deploy the plugin on your NOMAD instance.
+
+### Adding this plugin in your local NOMAD installation and the source code of NOMAD
+
+We now recommend using the dedicated [`nomad-distro-dev`](https://github.com/FAIRmat-NFDI/nomad-distro-dev) repository to simplify the process. Please refer to that repository for detailed instructions.
+
+
+## 🛠️ Development
+
+If you want to develop locally this plugin, clone the project and in the plugin folder, create a virtual environment (you can use Python 3.10, 3.11 or 3.12):
+```sh
+git clone https://github.com/FAIRmat-NFDI/nomad-ml-workflows.git
+cd nomad-ml-workflows
+python3.11 -m venv .pyenv
+. .pyenv/bin/activate
+```
+
+Make sure to have `pip` upgraded:
+```sh
+pip install --upgrade pip
+```
+
+We recommend installing `uv` for fast pip installation of the packages:
+```sh
+pip install uv
+```
+
+Install the `nomad-lab` package:
+```sh
+uv pip install -e '.[dev]'
+```
+
+### Run linting and auto-formatting
+
+We use [Ruff](https://docs.astral.sh/ruff/) for linting and formatting the code. Ruff auto-formatting is also a part of the GitHub workflow actions. You can run locally:
+```sh
+ruff check .
+ruff format . --check
+```
+
+### Debugging
+
+For interactive debugging of the tests, use `pytest` with the `--pdb` flag. We recommend using an IDE for debugging, e.g., _VSCode_. If that is the case, add the following snippet to your `.vscode/launch.json`:
+```json
+{
+  "configurations": [
+      {
+        "name": "<descriptive tag>",
+        "type": "debugpy",
+        "request": "launch",
+        "cwd": "${workspaceFolder}",
+        "program": "${workspaceFolder}/.pyenv/bin/pytest",
+        "justMyCode": true,
+        "env": {
+            "_PYTEST_RAISE": "1"
+        },
+        "args": [
+            "-sv",
+            "--pdb",
+            "<path-to-plugin-tests>",
+        ]
+    }
+  ]
+}
+```
+
+where `<path-to-plugin-tests>` must be changed to the local path to the test module to be debugged.
+
+The settings configuration file `.vscode/settings.json` automatically applies the linting and formatting upon saving the modified file.
+
+### Documentation on Github pages
+
+To view the documentation locally, install the related packages using:
+```sh
+uv pip install -r requirements_docs.txt
+```
+
+Run the documentation server:
+```sh
+mkdocs serve
+```
+
+
+## 👥 Main contributors
+| Name | E-mail     |
+|------|------------|
+| Sarthak Kapoor | [sarthak.kapoor@physik.hu-berlin.de](mailto:sarthak.kapoor@physik.hu-berlin.de)
+
+
+## 📄 License
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

nomad_ml_workflows-0.0.6/docs/assets/.gitignore

@@ -0,0 +1 @@
+nomad-oasis*.zip

nomad_ml_workflows-0.0.6/docs/explanation/explanation.md

@@ -0,0 +1,4 @@
+# Explanation
+
+!!! note "Attention"
+    TODO

nomad_ml_workflows-0.0.6/docs/how_to/contribute_to_the_documentation.md

@@ -0,0 +1,4 @@
+# Contribute to the documentation
+
+!!! note "Attention"
+    TODO