dcp-tools 1.0.0__tar.gz

dcp_tools-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 ONE Campaign
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dcp_tools-1.0.0/PKG-INFO

@@ -0,0 +1,168 @@
+Metadata-Version: 2.4
+Name: dcp-tools
+Version: 1.0.0
+Summary: Tools to work with the Data Commons Platform.
+Keywords: data-commons,datacommons,dcp,etl,statistics
+Author: ONE Campaign, Luca Picci, Jorge Rivera
+Author-email: Luca Picci <lpicci96@gmail.com>, Jorge Rivera <jorge.rivera@one.org>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Requires-Dist: pydantic>=2.11.3,<3.0.0
+Requires-Dist: pandas>=2.2.3,<4.0.0
+Requires-Dist: pydantic-settings>=2.9.1,<3.0.0
+Requires-Dist: google-cloud-storage>=3.1.0,<4.0.0
+Requires-Dist: datacommons-admin>=1.1.1
+Requires-Python: >=3.13
+Project-URL: homepage, https://docs.one.org/tools/dcp-tools/
+Project-URL: documentation, https://docs.one.org/tools/dcp-tools/
+Project-URL: repository, https://github.com/ONEcampaign/dcp-tools
+Project-URL: issues, https://github.com/ONEcampaign/dcp-tools/issues
+Description-Content-Type: text/markdown
+
+# dcp-tools
+
+__Manage and load data to Data Commons Platform instances__
+
+[![PyPI](https://img.shields.io/pypi/v/dcp-tools.svg)](https://pypi.org/project/dcp-tools/)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/dcp-tools.svg)](https://pypi.org/project/dcp-tools/)
+[![Docs](https://img.shields.io/badge/docs-dcp--tools-blue)](https://docs.one.org/tools/dcp-tools/)
+[![Lint/format: Ruff](https://img.shields.io/badge/lint%2Fformat-ruff-46a758.svg)](https://github.com/astral-sh/ruff)
+
+A [Data Commons Platform](https://docs.datacommons.org/custom_dc/custom_data.html) instance takes
+your data as CSVs in a fixed variable-per-row shape, plus a `config.json` that maps each CSV's
+columns onto the Data Commons schema. If you're defining your own statistical variables,
+entities, or properties rather than reusing existing ones, you also need MCF (Meta Content
+Framework) files describing them. `dcp-tools` builds and validates that bundle in Python (or from
+the CLI) and uploads it to Cloud Storage to trigger the platform's ingestion workflow.
+
+This package was published as `bblocks-datacommons-tools` until version 0.1.1, and imported as
+`bblocks.datacommons_tools`. Installing the old distribution now pulls in `dcp-tools` and
+redirects those imports with a `DeprecationWarning`, so existing code keeps working. Update
+imports to `dcp_tools` when convenient.
+
+## Install
+
+```bash
+pip install dcp-tools
+```
+
+Or from GitHub:
+
+```bash
+pip install git+https://github.com/ONEcampaign/dcp-tools
+```
+
+## Quickstart
+
+This builds a config and MCF for a source, a provenance, one input file, and one statistical
+variable, then exports the bundle to disk.
+
+```python
+from pathlib import Path
+
+import pandas as pd
+from dcp_tools import CustomDataManager
+
+manager = CustomDataManager()
+manager.add_source(dcid="ONEData", url="https://data.one.org")
+manager.add_provenance(
+    dcid="ONEClimateFinance",
+    url="https://datacommons.one.org/data/climate-finance-files",
+    source="ONEData",
+)
+
+data = pd.DataFrame({
+    "country": ["Kenya", "Kenya", "Vietnam"],
+    "year": [2022, 2023, 2023],
+    "variable": ["climateFinanceProvidedCommitments"] * 3,
+    "value": [12.4, 15.1, 8.7],
+})
+manager.add_input_file(
+    file_name="climate_finance/one_cf_provider_commitments.csv",
+    provenance="ONEClimateFinance",
+    data=data,
+    column_mappings={
+        "observationAbout": "country",
+        "date": "year",
+        "variable": "variable",
+        "value": "value",
+    },
+    observation_properties={"unit": "USDollar"},
+)
+
+manager.add_variable_to_mcf(
+    dcid="climateFinanceProvidedCommitments",
+    name="Climate finance commitments (bilateral)",
+    description="Funding committed for climate adaptation and mitigation projects",
+    stat_type="dcid:measuredValue",
+)
+
+manager.add_mcf_file("*.mcf", provenance="ONEClimateFinance")
+
+out_dir = Path("export/climate_finance")
+out_dir.mkdir(parents=True, exist_ok=True)
+manager.export_all(out_dir)
+```
+
+`export_all` writes `config.json`, the CSV, and both MCF files under `out_dir`. Since we
+never called `set_import_name`, `config.json` defaults `importName` to the export directory's
+name, and column mappings and the provenance name are resolved to full dcids:
+
+```json
+{
+    "importName": "climate_finance",
+    "inputFiles": [
+        {
+            "filename": "climate_finance/one_cf_provider_commitments.csv",
+            "provenance": "dcid:provenance/ONEClimateFinance",
+            "columnMappings": {
+                "dcid:variableMeasured": "variable",
+                "dcid:observationDate": "year",
+                "dcid:value": "value",
+                "dcid:observationAbout": "country"
+            },
+            "observationProperties": {"unit": "USDollar"},
+            "format": "variablePerRow"
+        },
+        {
+            "pattern": "*.mcf",
+            "provenance": "dcid:provenance/ONEClimateFinance"
+        }
+    ]
+}
+```
+
+## Loading it
+
+Once you have a bundle on disk, `dcp_tools.gcp_utilities` uploads it and triggers ingestion:
+
+```python
+from dcp_tools.gcp_utilities import (
+    get_kg_settings,
+    run_ingestion_workflow,
+    upload_to_cloud_storage,
+)
+
+settings = get_kg_settings(source="env", env_file="customDC.env")
+upload_to_cloud_storage(settings=settings, directory="export/climate_finance")
+run_ingestion_workflow(settings=settings)
+```
+
+`run_ingestion_workflow` triggers the DCP (Data Commons Platform) ingestion workflow, which
+ingests the new data and serves it. There's no separate redeploy step to run. See the
+[loading-data docs](https://docs.one.org/tools/dcp-tools/loading-data/) for the
+full settings reference, and the `dcp-tools` CLI (`upload`, `ingest`, `pipeline`), which wraps
+this same flow.
+
+## Contributing
+
+Contributions are welcome! See [CONTRIBUTING](https://github.com/ONEcampaign/dcp-tools/blob/main/CONTRIBUTING.MD)
+for how to get started, report bugs, and submit changes.

dcp_tools-1.0.0/README.md

@@ -0,0 +1,139 @@
+# dcp-tools
+
+__Manage and load data to Data Commons Platform instances__
+
+[![PyPI](https://img.shields.io/pypi/v/dcp-tools.svg)](https://pypi.org/project/dcp-tools/)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/dcp-tools.svg)](https://pypi.org/project/dcp-tools/)
+[![Docs](https://img.shields.io/badge/docs-dcp--tools-blue)](https://docs.one.org/tools/dcp-tools/)
+[![Lint/format: Ruff](https://img.shields.io/badge/lint%2Fformat-ruff-46a758.svg)](https://github.com/astral-sh/ruff)
+
+A [Data Commons Platform](https://docs.datacommons.org/custom_dc/custom_data.html) instance takes
+your data as CSVs in a fixed variable-per-row shape, plus a `config.json` that maps each CSV's
+columns onto the Data Commons schema. If you're defining your own statistical variables,
+entities, or properties rather than reusing existing ones, you also need MCF (Meta Content
+Framework) files describing them. `dcp-tools` builds and validates that bundle in Python (or from
+the CLI) and uploads it to Cloud Storage to trigger the platform's ingestion workflow.
+
+This package was published as `bblocks-datacommons-tools` until version 0.1.1, and imported as
+`bblocks.datacommons_tools`. Installing the old distribution now pulls in `dcp-tools` and
+redirects those imports with a `DeprecationWarning`, so existing code keeps working. Update
+imports to `dcp_tools` when convenient.
+
+## Install
+
+```bash
+pip install dcp-tools
+```
+
+Or from GitHub:
+
+```bash
+pip install git+https://github.com/ONEcampaign/dcp-tools
+```
+
+## Quickstart
+
+This builds a config and MCF for a source, a provenance, one input file, and one statistical
+variable, then exports the bundle to disk.
+
+```python
+from pathlib import Path
+
+import pandas as pd
+from dcp_tools import CustomDataManager
+
+manager = CustomDataManager()
+manager.add_source(dcid="ONEData", url="https://data.one.org")
+manager.add_provenance(
+    dcid="ONEClimateFinance",
+    url="https://datacommons.one.org/data/climate-finance-files",
+    source="ONEData",
+)
+
+data = pd.DataFrame({
+    "country": ["Kenya", "Kenya", "Vietnam"],
+    "year": [2022, 2023, 2023],
+    "variable": ["climateFinanceProvidedCommitments"] * 3,
+    "value": [12.4, 15.1, 8.7],
+})
+manager.add_input_file(
+    file_name="climate_finance/one_cf_provider_commitments.csv",
+    provenance="ONEClimateFinance",
+    data=data,
+    column_mappings={
+        "observationAbout": "country",
+        "date": "year",
+        "variable": "variable",
+        "value": "value",
+    },
+    observation_properties={"unit": "USDollar"},
+)
+
+manager.add_variable_to_mcf(
+    dcid="climateFinanceProvidedCommitments",
+    name="Climate finance commitments (bilateral)",
+    description="Funding committed for climate adaptation and mitigation projects",
+    stat_type="dcid:measuredValue",
+)
+
+manager.add_mcf_file("*.mcf", provenance="ONEClimateFinance")
+
+out_dir = Path("export/climate_finance")
+out_dir.mkdir(parents=True, exist_ok=True)
+manager.export_all(out_dir)
+```
+
+`export_all` writes `config.json`, the CSV, and both MCF files under `out_dir`. Since we
+never called `set_import_name`, `config.json` defaults `importName` to the export directory's
+name, and column mappings and the provenance name are resolved to full dcids:
+
+```json
+{
+    "importName": "climate_finance",
+    "inputFiles": [
+        {
+            "filename": "climate_finance/one_cf_provider_commitments.csv",
+            "provenance": "dcid:provenance/ONEClimateFinance",
+            "columnMappings": {
+                "dcid:variableMeasured": "variable",
+                "dcid:observationDate": "year",
+                "dcid:value": "value",
+                "dcid:observationAbout": "country"
+            },
+            "observationProperties": {"unit": "USDollar"},
+            "format": "variablePerRow"
+        },
+        {
+            "pattern": "*.mcf",
+            "provenance": "dcid:provenance/ONEClimateFinance"
+        }
+    ]
+}
+```
+
+## Loading it
+
+Once you have a bundle on disk, `dcp_tools.gcp_utilities` uploads it and triggers ingestion:
+
+```python
+from dcp_tools.gcp_utilities import (
+    get_kg_settings,
+    run_ingestion_workflow,
+    upload_to_cloud_storage,
+)
+
+settings = get_kg_settings(source="env", env_file="customDC.env")
+upload_to_cloud_storage(settings=settings, directory="export/climate_finance")
+run_ingestion_workflow(settings=settings)
+```
+
+`run_ingestion_workflow` triggers the DCP (Data Commons Platform) ingestion workflow, which
+ingests the new data and serves it. There's no separate redeploy step to run. See the
+[loading-data docs](https://docs.one.org/tools/dcp-tools/loading-data/) for the
+full settings reference, and the `dcp-tools` CLI (`upload`, `ingest`, `pipeline`), which wraps
+this same flow.
+
+## Contributing
+
+Contributions are welcome! See [CONTRIBUTING](https://github.com/ONEcampaign/dcp-tools/blob/main/CONTRIBUTING.MD)
+for how to get started, report bugs, and submit changes.

dcp_tools-1.0.0/pyproject.toml

@@ -0,0 +1,163 @@
+[project]
+name = "dcp-tools"
+version = "1.0.0"
+description = "Tools to work with the Data Commons Platform."
+license = "MIT"
+license-files = ["LICENSE"]
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "pydantic>=2.11.3,<3.0.0",
+    "pandas>=2.2.3,<4.0.0",
+    "pydantic-settings>=2.9.1,<3.0.0",
+    "google-cloud-storage>=3.1.0,<4.0.0",
+    "datacommons-admin>=1.1.1",
+]
+keywords = [
+    "data-commons",
+    "datacommons",
+    "dcp",
+    "etl",
+    "statistics",
+]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Scientific/Engineering :: Information Analysis",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Typing :: Typed",
+]
+
+[[project.authors]]
+name = "ONE Campaign"
+
+[[project.authors]]
+name = "Luca Picci"
+email = "lpicci96@gmail.com"
+
+[[project.authors]]
+name = "Jorge Rivera"
+email = "jorge.rivera@one.org"
+
+[project.scripts]
+dcp-tools = "dcp_tools.cli:main"
+
+[project.urls]
+homepage = "https://docs.one.org/tools/dcp-tools/"
+documentation = "https://docs.one.org/tools/dcp-tools/"
+repository = "https://github.com/ONEcampaign/dcp-tools"
+issues = "https://github.com/ONEcampaign/dcp-tools/issues"
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0.0",
+    "pytest-cov>=4.1.0",
+    "ty>=0.0.33",
+    "ruff>=0.8.0",
+    "pre-commit>=3.0.0",
+    "mkdocs-material>=9.6.14,<10.0.0",
+]
+
+[build-system]
+requires = ["uv_build>=0.11.8,<0.12"]
+build-backend = "uv_build"
+
+[tool.ruff]
+line-length = 88
+target-version = "py313"
+extend-include = ["*.ipynb"]
+exclude = [
+    ".git",
+    ".venv",
+    "venv",
+    "__pycache__",
+    "build",
+    "dist",
+    "*.egg-info",
+    ".pytest_cache",
+    "htmlcov",
+]
+
+[tool.ruff.lint]
+select = [
+    "E",
+    "W",
+    "F",
+    "I",
+    "UP",
+    "B",
+    "C4",
+    "SIM",
+    "PL",
+    "RUF",
+    "ERA",
+    "PERF",
+    "ANN",
+]
+ignore = [
+    "E501",
+    "PLR0913",
+    "PLR0912",
+    "PLR0914",
+    "PLR0915",
+    "PLR2004",
+    "ANN401",
+]
+fixable = ["ALL"]
+unfixable = []
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**/*.py" = [
+    "ANN",
+    "PLR2004",
+]
+"**/*.ipynb" = [
+    "ANN",
+    "F401",
+    "F811",
+    "E402",
+]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+skip-magic-trailing-comma = false
+line-ending = "auto"
+
+[tool.ty.environment]
+python-version = "3.13"
+
+[tool.ty.src]
+include = ["src/dcp_tools"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+addopts = "-v --cov=src/dcp_tools --cov-report=term"
+
+[tool.coverage.run]
+source = ["src"]
+branch = true
+omit = [
+    "*/tests/*",
+    "*/__pycache__/*",
+]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "def __str__",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+]
+show_missing = true
+precision = 2

dcp_tools-1.0.0/pyproject.toml.orig

@@ -0,0 +1,151 @@
+[project]
+name = "dcp-tools"
+version = "1.0.0"
+description = "Tools to work with the Data Commons Platform."
+authors = [
+    { name = "ONE Campaign" },
+    { name = "Luca Picci", email = "lpicci96@gmail.com" },
+    { name = "Jorge Rivera", email = "jorge.rivera@one.org" },
+]
+license = "MIT"
+license-files = ["LICENSE"]
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "pydantic>=2.11.3,<3.0.0",
+    "pandas>=2.2.3,<4.0.0",
+    "pydantic-settings>=2.9.1,<3.0.0",
+    "google-cloud-storage>=3.1.0,<4.0.0",
+    "datacommons-admin>=1.1.1",
+]
+keywords = ["data-commons", "datacommons", "dcp", "etl", "statistics"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Scientific/Engineering :: Information Analysis",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Typing :: Typed",
+]
+
+[project.scripts]
+dcp-tools = "dcp_tools.cli:main"
+
+[project.urls]
+homepage = "https://docs.one.org/tools/dcp-tools/"
+documentation = "https://docs.one.org/tools/dcp-tools/"
+repository = "https://github.com/ONEcampaign/dcp-tools"
+issues = "https://github.com/ONEcampaign/dcp-tools/issues"
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0.0",
+    "pytest-cov>=4.1.0",
+    "ty>=0.0.33",
+    "ruff>=0.8.0",
+    "pre-commit>=3.0.0",
+    "mkdocs-material>=9.6.14,<10.0.0",
+]
+
+[build-system]
+requires = ["uv_build>=0.11.8,<0.12"]
+build-backend = "uv_build"
+
+[tool.ruff]
+line-length = 88
+target-version = "py313"
+extend-include = ["*.ipynb"]
+
+exclude = [
+    ".git",
+    ".venv",
+    "venv",
+    "__pycache__",
+    "build",
+    "dist",
+    "*.egg-info",
+    ".pytest_cache",
+    "htmlcov",
+]
+
+[tool.ruff.lint]
+# "N" (pep8-naming) is intentionally omitted: the public API mirrors the Data
+# Commons config.json schema, which is camelCase (e.g. ``importName``,
+# ``set_includeInputSubdirs``). Those names are a serialization contract, not a
+# style choice, so the naming rules would fight the domain.
+select = [
+    "E", "W", "F",      # pycodestyle, pyflakes
+    "I",                 # isort
+    "UP",                # pyupgrade
+    "B",                 # bugbear
+    "C4",                # comprehensions
+    "SIM",               # simplify
+    "PL",                # pylint
+    "RUF",               # ruff-specific
+    "ERA",               # eradicate (commented-out code)
+    "PERF",              # perflint
+    "ANN",               # flake8-annotations (require type hints on public APIs)
+]
+
+ignore = [
+    "E501",     # Line length (formatter handles)
+    "PLR0913",  # Too many arguments
+    "PLR0912",  # Too many branches
+    "PLR0914",  # Too many locals
+    "PLR0915",  # Too many statements
+    "PLR2004",  # Magic values
+    "ANN401",   # Allow `Any` in typed signatures
+]
+
+# Allow autofix for all enabled rules
+fixable = ["ALL"]
+unfixable = []
+
+[tool.ruff.lint.per-file-ignores]
+# Tests don't need annotations or pylint-complexity gates.
+"tests/**/*.py" = ["ANN", "PLR2004"]
+# Notebooks: don't require annotations; allow REPL-style patterns.
+"**/*.ipynb" = ["ANN", "F401", "F811", "E402"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+skip-magic-trailing-comma = false
+line-ending = "auto"
+
+[tool.ty.environment]
+python-version = "3.13"
+
+[tool.ty.src]
+include = ["src/dcp_tools"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+addopts = "-v --cov=src/dcp_tools --cov-report=term"
+
+[tool.coverage.run]
+source = ["src"]
+branch = true
+omit = [
+    "*/tests/*",
+    "*/__pycache__/*",
+]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "def __str__",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+]
+show_missing = true
+precision = 2

dcp_tools-1.0.0/src/dcp_tools/__init__.py

@@ -0,0 +1,15 @@
+from importlib.metadata import version
+
+from .custom_data.config_utils import merge_configs_from_directory
+from .custom_data.data_management import CustomDataManager
+from .custom_data.schema_tools import csv_metadata_to_mcf_file, csv_metadata_to_nodes
+
+__version__ = version("dcp-tools")
+
+
+__all__ = [
+    "CustomDataManager",
+    "csv_metadata_to_mcf_file",
+    "csv_metadata_to_nodes",
+    "merge_configs_from_directory",
+]

dcp_tools-1.0.0/src/dcp_tools/__main__.py

@@ -0,0 +1,6 @@
+"""Enable ``python -m dcp_tools``."""
+
+from dcp_tools.cli.main import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

dcp_tools-1.0.0/src/dcp_tools/cli/__init__.py

@@ -0,0 +1,5 @@
+"""Command line interface for ``dcp_tools`` package."""
+
+from dcp_tools.cli.main import main
+
+__all__ = ["main"]

dcp_tools-1.0.0/src/dcp_tools/cli/common.py

@@ -0,0 +1,9 @@
+import argparse
+
+from dcp_tools.gcp_utilities import KGSettings, get_kg_settings
+
+
+def load_settings_from_args(args: argparse.Namespace) -> KGSettings:
+    if args.settings_file:
+        return get_kg_settings(source="json", file=args.settings_file)
+    return get_kg_settings(env_file=args.env_file)