dcp-tools 1.0.0__py3-none-any.whl

dcp_tools/__init__.py

@@ -0,0 +1,15 @@
+from importlib.metadata import version
+
+from .custom_data.config_utils import merge_configs_from_directory
+from .custom_data.data_management import CustomDataManager
+from .custom_data.schema_tools import csv_metadata_to_mcf_file, csv_metadata_to_nodes
+
+__version__ = version("dcp-tools")
+
+
+__all__ = [
+    "CustomDataManager",
+    "csv_metadata_to_mcf_file",
+    "csv_metadata_to_nodes",
+    "merge_configs_from_directory",
+]

dcp_tools/__main__.py

@@ -0,0 +1,6 @@
+"""Enable ``python -m dcp_tools``."""
+
+from dcp_tools.cli.main import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

dcp_tools/cli/__init__.py

@@ -0,0 +1,5 @@
+"""Command line interface for ``dcp_tools`` package."""
+
+from dcp_tools.cli.main import main
+
+__all__ = ["main"]

dcp_tools/cli/common.py

@@ -0,0 +1,9 @@
+import argparse
+
+from dcp_tools.gcp_utilities import KGSettings, get_kg_settings
+
+
+def load_settings_from_args(args: argparse.Namespace) -> KGSettings:
+    if args.settings_file:
+        return get_kg_settings(source="json", file=args.settings_file)
+    return get_kg_settings(env_file=args.env_file)

dcp_tools/cli/csv2mcf.py

@@ -0,0 +1,91 @@
+"""CSV to MCF command implementation."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from dcp_tools.custom_data.schema_tools import (
+    NodeTypes,
+    csv_metadata_to_mcf_file,
+)
+
+__all__ = ["add_parser", "run"]
+
+
+def _kv_pair(value: str) -> tuple[str, str]:
+    """Parse a KEY=VALUE string into a tuple."""
+    if "=" not in value:
+        raise ValueError(f"Invalid key-value pair: {value}")
+    key, val = value.split("=", 1)
+    return key.strip(), val.strip()
+
+
+def add_parser(subparsers: argparse._SubParsersAction) -> None:
+    """Register the ``csv2mcf`` subcommand with the CLI parser."""
+    parser = subparsers.add_parser(
+        "csv2mcf", help="Convert a CSV of Node metadata to an MCF file"
+    )
+
+    parser.add_argument("csv", type=Path, help="Path to the input CSV file")
+    parser.add_argument("mcf", type=Path, help="Path to write the generated MCF")
+
+    parser.add_argument(
+        "--node-type",
+        choices=[node_type.value for node_type in NodeTypes],
+        default="Node",
+        help="Type of node to create (default: %(default)s)",
+    )
+
+    parser.add_argument(
+        "--column-mapping",
+        metavar="CSV_COL=MCF_PROP",
+        type=_kv_pair,
+        action="append",
+        help=(
+            "Map CSV column names to MCF properties. "
+            "May be used multiple times, eg: "
+            "--column-mapping description=searchDescription --column-mapping indicator=Node"
+        ),
+    )
+
+    parser.add_argument(
+        "--csv-option",
+        metavar="KEY=VALUE",
+        type=_kv_pair,
+        action="append",
+        help=(
+            "Extra keyword arguments forwarded to pandas.read_csv, "
+            'e.g. --csv-option delimiter=";" --csv-option encoding=UTF-8'
+        ),
+    )
+
+    parser.add_argument(
+        "--ignore-column",
+        metavar="COLUMN",
+        action="append",
+        help="Name of a CSV column to ignore. May be specified multiple times.",
+    )
+
+    parser.add_argument(
+        "--append", action="store_true", help="Append to the output file if it exists."
+    )
+
+    parser.set_defaults(func=run)
+
+
+def run(args: argparse.Namespace) -> int:
+    """Execute the ``csv2mcf`` command."""
+    column_mapping = dict(args.column_mapping) if args.column_mapping else None
+    csv_options = dict(args.csv_option) if args.csv_option else None
+
+    csv_metadata_to_mcf_file(
+        csv_path=args.csv,
+        mcf_path=args.mcf,
+        node_type=args.node_type,
+        column_to_property_mapping=column_mapping,
+        csv_options=csv_options,
+        ignore_columns=args.ignore_column,
+        overwrite=not args.append,
+    )
+    return 0

dcp_tools/cli/ingest.py

@@ -0,0 +1,35 @@
+"""Run the ingestion workflow via CLI."""
+
+import argparse
+from pathlib import Path
+
+from dcp_tools.cli.common import load_settings_from_args
+from dcp_tools.gcp_utilities.ingestion import run_ingestion_workflow
+
+__all__ = ["add_parser", "run"]
+
+
+def add_parser(subparsers: argparse._SubParsersAction) -> None:
+    """Register the ``ingest`` subcommand."""
+    parser = subparsers.add_parser(
+        "ingest", help="Run the Knowledge Graph ingestion workflow"
+    )
+    parser.add_argument(
+        "--settings-file", type=Path, help="Path to the KG settings JSON file"
+    )
+    parser.add_argument(
+        "--env-file", type=Path, help="Optional .env file containing KG settings"
+    )
+    parser.add_argument(
+        "--imports",
+        type=str,
+        help="Comma-separated list of imports to ingest, defaults to all imports",
+    )
+    parser.set_defaults(func=run)
+
+
+def run(args: argparse.Namespace) -> int:
+    """Execute the ``ingest`` command."""
+    settings = load_settings_from_args(args)
+    run_ingestion_workflow(settings=settings, imports=args.imports)
+    return 0

dcp_tools/cli/main.py

@@ -0,0 +1,42 @@
+"""Entry point for the ``dcp_tools`` command line interface."""
+
+import argparse
+from collections.abc import Iterable
+
+from dcp_tools.cli import (
+    csv2mcf,
+    ingest,
+    pipeline,
+    upload,
+)
+
+__all__ = ["main"]
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    """Create the CLI argument parser and register commands."""
+    parser = argparse.ArgumentParser(
+        description="Utilities for working with Data Commons files"
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    csv2mcf.add_parser(subparsers)
+    upload.add_parser(subparsers)
+    ingest.add_parser(subparsers)
+    pipeline.add_parser(subparsers)
+    return parser
+
+
+def main(argv: Iterable[str] | None = None) -> int:
+    """Parse ``argv`` and execute the selected command."""
+    parser = _build_parser()
+    args = parser.parse_args(list(argv) if argv is not None else None)
+
+    if hasattr(args, "func"):
+        return args.func(args)
+
+    parser.error(f"Unknown command: {args.command}")
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

dcp_tools/cli/pipeline.py

@@ -0,0 +1,45 @@
+"""Run the full Knowledge Graph pipeline via CLI."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from dcp_tools.cli.common import load_settings_from_args
+from dcp_tools.gcp_utilities.ingestion import run_ingestion_workflow
+from dcp_tools.gcp_utilities.pipeline import upload_to_cloud_storage
+
+__all__ = ["add_parser", "run"]
+
+
+def add_parser(subparsers: argparse._SubParsersAction) -> None:
+    """Register the ``pipeline`` subcommand."""
+    parser = subparsers.add_parser(
+        "pipeline", help="Run the full Knowledge Graph loading pipeline"
+    )
+    parser.add_argument(
+        "--settings-file", type=Path, help="Path to the KG settings JSON file"
+    )
+    parser.add_argument(
+        "--env-file", type=Path, help="Optional .env file containing KG settings"
+    )
+    parser.add_argument(
+        "--directory",
+        type=Path,
+        help="Local directory to upload. Defaults to settings.LOCAL_PATH",
+    )
+    parser.add_argument(
+        "--sync",
+        action="store_true",
+        default=False,
+        help="Delete remote blobs that no longer have a local counterpart after uploading",
+    )
+    parser.set_defaults(func=run)
+
+
+def run(args: argparse.Namespace) -> int:
+    """Execute the ``pipeline`` command."""
+    settings = load_settings_from_args(args)
+    upload_to_cloud_storage(settings=settings, directory=args.directory, sync=args.sync)
+    run_ingestion_workflow(settings=settings)
+    return 0

dcp_tools/cli/upload.py

@@ -0,0 +1,43 @@
+"""Upload data to Cloud Storage via CLI."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from dcp_tools.cli.common import load_settings_from_args
+from dcp_tools.gcp_utilities.pipeline import upload_to_cloud_storage
+
+__all__ = ["add_parser", "run"]
+
+
+def add_parser(subparsers: argparse._SubParsersAction) -> None:
+    """Register the ``upload`` subcommand."""
+    parser = subparsers.add_parser(
+        "upload", help="Upload local data to Google Cloud Storage"
+    )
+    parser.add_argument(
+        "--settings-file", type=Path, help="Path to the KG settings JSON file"
+    )
+    parser.add_argument(
+        "--env-file", type=Path, help="Optional .env file containing KG settings"
+    )
+    parser.add_argument(
+        "--directory",
+        type=Path,
+        help="Local directory to upload. Defaults to settings.LOCAL_PATH",
+    )
+    parser.add_argument(
+        "--sync",
+        action="store_true",
+        default=False,
+        help="Delete remote blobs that no longer have a local counterpart after uploading",
+    )
+    parser.set_defaults(func=run)
+
+
+def run(args: argparse.Namespace) -> int:
+    """Execute the ``upload`` command."""
+    settings = load_settings_from_args(args)
+    upload_to_cloud_storage(settings=settings, directory=args.directory, sync=args.sync)
+    return 0

dcp_tools/custom_data/__init__.py

@@ -0,0 +1,5 @@
+from dcp_tools.custom_data.data_management import CustomDataManager
+
+__all__ = [
+    "CustomDataManager",
+]

dcp_tools/custom_data/config_utils.py

@@ -0,0 +1,188 @@
+"""Utilities for merging config files."""
+
+from collections.abc import Iterator
+from os import PathLike
+from pathlib import Path
+from typing import Any, Literal
+
+from dcp_tools.custom_data.models.config_file import Config
+from dcp_tools.logger import logger
+
+DuplicatePolicy = Literal["error", "override", "ignore"]
+
+
+def iter_config_files(directory: Path, pattern: str = "config.json") -> Iterator[Path]:
+    """Yield all ``config.json`` files under ``directory`` recursively.
+
+    Files are yielded in sorted order so that merge results are deterministic
+    across platforms.
+    """
+    for path in sorted(directory.rglob(pattern)):
+        if path.is_file():
+            yield path
+
+
+def _merge_simple_attrs(existing: Config, new: Config, policy: DuplicatePolicy) -> None:
+    """Merge simple attributes that are booleans, strings, or None."""
+    for attr in (
+        "import_name",
+        "include_input_subdirs",
+        "group_stat_vars_by_property",
+        "default_custom_root_stat_var_group_name",
+        "custom_id_namespace",
+        "custom_svg_prefix",
+        "vertical_specs_file",
+    ):
+        _merge_attribute(existing, new, attr, policy)
+
+    _merge_sequence_attribute(
+        existing=existing,
+        new=new,
+        attribute="sv_hierarchy_props_blocklist",
+        policy=policy,
+    )
+
+    _merge_sequence_attribute(
+        existing=existing,
+        new=new,
+        attribute="data_download_url",
+        policy=policy,
+    )
+
+
+def _handle_conflict(
+    field: str, target_value: Any, source_value: Any, policy: DuplicatePolicy
+) -> None:
+    """Log or raise depending on *policy* when a conflict is detected."""
+    if policy == "override":
+        logger.warning(f"Overriding {field}: {target_value} -> {source_value}")
+    elif policy == "ignore":
+        logger.info(f"Ignoring {field}: {target_value} -> {source_value}")
+    else:
+        raise ValueError(f"Conflicting {field}: {target_value!r} vs {source_value!r}")
+
+
+def _merge_input_files(existing: Config, new: Config, policy: DuplicatePolicy) -> None:
+    """Merge input file entries from *new* into *existing* in-place.
+
+    Entries are keyed by ``filename`` or ``pattern`` (whichever is set).
+    Conflict semantics follow *policy*: ``"error"`` raises, ``"override"``
+    replaces, ``"ignore"`` and equal entries are skipped.
+    """
+    existing_by_key: dict[str, int] = {}
+    for i, e in enumerate(existing.input_files):
+        k = e.filename or e.pattern
+        if k is not None:
+            existing_by_key[k] = i
+
+    for entry in new.input_files:
+        key = entry.filename or entry.pattern
+        if key is None:
+            # Should not happen: InputFile enforces filename XOR pattern.
+            continue
+        if key not in existing_by_key:
+            logger.info(f"Added input file '{key}'")
+            existing.input_files.append(entry)
+            existing_by_key[key] = len(existing.input_files) - 1
+            continue
+
+        idx = existing_by_key[key]
+        tgt = existing.input_files[idx]
+        if tgt == entry:
+            continue
+
+        _handle_conflict(
+            field=f"Input file '{key}'",
+            target_value=tgt,
+            source_value=entry,
+            policy=policy,
+        )
+        if policy == "override":
+            existing.input_files[idx] = entry
+
+
+def _merge_attribute(
+    existing: Config, new: Config, attribute: str, policy: DuplicatePolicy
+) -> None:
+    """Merge a single attribute, handling collisions with *policy*."""
+    src_val = getattr(new, attribute)
+    if src_val is None:
+        return
+
+    tgt_val = getattr(existing, attribute)
+    if tgt_val is None or tgt_val == src_val:
+        setattr(existing, attribute, src_val if tgt_val is None else tgt_val)
+        return
+
+    _handle_conflict(
+        field=attribute, target_value=tgt_val, source_value=src_val, policy=policy
+    )
+    if policy == "override":
+        setattr(existing, attribute, src_val)
+
+
+def _merge_sequence_attribute(
+    existing: Config, new: Config, attribute: str, policy: DuplicatePolicy
+) -> None:
+    """Merge a sequence attribute, normalising duplicates when overriding."""
+    src_val = getattr(new, attribute)
+    if not src_val:
+        return
+
+    tgt_val = getattr(existing, attribute)
+    if not tgt_val:
+        setattr(existing, attribute, list(dict.fromkeys(src_val)))
+        return
+
+    if tgt_val == src_val:
+        return
+
+    _handle_conflict(
+        field=attribute, target_value=tgt_val, source_value=src_val, policy=policy
+    )
+
+    if policy == "override":
+        # Preserve the order of the incoming values but drop duplicates
+        setattr(existing, attribute, list(dict.fromkeys(src_val)))
+
+
+def merge_configs(
+    existing: Config, new: Config, *, policy: DuplicatePolicy = "error"
+) -> None:
+    """Merge ``new`` into ``existing`` in-place.
+
+    Args:
+        existing: target config.
+        new: the config to be merged with *existing*.
+        policy: How to resolve collisions.
+
+    Raises:
+        ValueError: If policy is `"error" and conflicting
+            values are encountered.
+    """
+    # Merge attributes that are booleans or None
+    _merge_simple_attrs(existing=existing, new=new, policy=policy)
+
+    # Merge the input file list
+    _merge_input_files(existing=existing, new=new, policy=policy)
+
+
+def merge_configs_from_directory(
+    directory: str | PathLike[str], *, policy: DuplicatePolicy = "error"
+) -> Config:
+    """Return a ``Config`` merging all configs found under ``directory``.
+
+    Args:
+        directory: Directory to search for config files.
+        policy: How to resolve collisions.
+    Raises:
+        ValueError: If policy is `"error"` and conflicting
+            values are encountered.
+
+    """
+    base = Config(input_files=[])
+    for path in iter_config_files(Path(directory)):
+        logger.info(f"Merging config file {path}")
+        config = Config.from_json(str(path))
+        merge_configs(existing=base, new=config, policy=policy)
+    return base