makegis 0.1.0__py3-none-any.whl

makegis/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

makegis/cli.py

@@ -0,0 +1,235 @@
+import argparse
+import logging
+from pathlib import Path
+import sys
+
+import dotenv
+from rich.console import Console
+from rich.logging import RichHandler
+
+from . import __version__
+from .config import RootConfig
+from .dag.builder import Builder
+from .targets import Target
+from . import errors
+
+console = Console()
+
+log = logging.getLogger("makegis")
+
+
+def cli():
+
+    # Handle general -v and -d options outside of argparse.
+    args = sys.argv[1:]
+    verbose_flags = ["-v", "--verbose"]
+    debug_flags = ["--debug"]
+    debug = any([flag in args for flag in debug_flags])
+    verbose = not debug and any([flag in args for flag in verbose_flags])
+    args = [a for a in args if a not in verbose_flags + debug_flags]
+
+    # Configure logger
+    level = logging.WARN
+    format = "%(message)s"
+    datefmt = "[%X]"
+    show_time = False
+    show_path = False
+    if debug:
+        level = logging.DEBUG
+        show_path = True
+    elif verbose:
+        level = logging.INFO
+    logging.basicConfig(
+        level=level,
+        format=format,
+        datefmt=datefmt,
+        handlers=[
+            RichHandler(
+                console=console,
+                rich_tracebacks=True,
+                show_path=show_path,
+                show_time=show_time,
+            )
+        ],
+    )
+    log.info(f"makegis {__version__}")
+
+    parser = argparse.ArgumentParser(prog="mkgs")
+
+    # The --verbose and --debug options are parsed outside of argparse but we still declare
+    # them here so they show up as general options in the generated help.
+    parser.add_argument("-v", "--verbose", action="store_true", help="verbose messages")
+    parser.add_argument("--debug", action="store_true", help="debug messages")
+
+    subparsers = parser.add_subparsers(dest="command", help="commands")
+
+    def add_target_argument(parser):
+        parser.add_argument(
+            "-t",
+            "--target",
+            action="store",
+            type=str,
+            default=None,
+            help="db instance to target",
+        )
+
+    init_parser = subparsers.add_parser("init", help="create schemas and journal table")
+    add_target_argument(init_parser)
+    init_parser.set_defaults(func=init)
+
+    list_parser = subparsers.add_parser("ls", help="list nodes")
+    list_parser.add_argument("pattern", type=str, help="DAG selection pattern")
+    list_parser.set_defaults(func=show)
+
+    outdated_parser = subparsers.add_parser("outdated", help="report outdated nodes")
+    add_target_argument(outdated_parser)
+    outdated_parser.set_defaults(func=outdated)
+
+    run_parser = subparsers.add_parser("run", help="run nodes")
+    run_parser.add_argument("pattern", type=str, help="DAG selection pattern")
+    add_target_argument(run_parser)
+    run_parser.add_argument(
+        "-d",
+        "--dry-run",
+        action="store_true",
+        help="process nodes without actually running them",
+    )
+    run_parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        help="also run fresh nodes",
+    )
+    run_parser.set_defaults(func=run)
+
+    # Load .env
+    dotenv.load_dotenv(".env")
+
+    # Parse preprocessed args
+    args = parser.parse_args(args)
+
+    # Inject verbose or debug option
+    args.verbose = verbose
+    args.debug = debug
+
+    # Call handler
+    if hasattr(args, "func"):
+        args.func(args)
+    else:
+        parser.print_help()
+
+
+def init(args):
+    cfg = load_root_config()
+    target_id = args.target or cfg.defaults.target
+    assert target_id is not None
+    log.info(f"using target {target_id}")
+    target = Target(cfg.targets[target_id])
+
+    dag = Builder(cfg).build()
+    target.ensure_schemas(dag.list_schemas())
+    target.init_journal()
+
+
+def outdated(args):
+    cfg = load_root_config()
+    target_id = args.target or cfg.defaults.target
+    assert target_id is not None
+    log.info(f"using target {target_id}")
+    target = Target(cfg.targets[target_id])
+
+    dag = Builder(cfg).build()
+    node_ids = dag.get_outdated(target)
+    if not node_ids:
+        print("All nodes are up to date.")
+
+    for node_id in node_ids:
+        print(dag.render_node(node_id))
+
+
+def run(args):
+    cfg = load_root_config()
+
+    target_id = args.target or cfg.defaults.target
+    assert target_id is not None
+    log.info(f"using target {target_id}")
+    target = Target(cfg.targets[target_id])
+
+    dry_run = args.dry_run == True
+    if args.dry_run:
+        log.info("dry run - target will not be modified")
+
+    dag = Builder(cfg).build()
+    node_ids = dag.select_nodes(args.pattern)
+    if not node_ids:
+        print("No nodes matching selection pattern.")
+        return
+
+    if not args.force:
+        outdated = dag.get_outdated(target, limit_to=node_ids)
+        node_ids = outdated
+
+        if not node_ids:
+            print("All selected nodes are up to date. Use --force to run anyways.")
+            return
+
+    n = len(node_ids)
+    with console.status("") as status:
+        for inode, node_id in enumerate(node_ids):
+            status.update(f"Running node {inode + 1}/{n}: {node_id}")
+            if args.dry_run:
+                log.info(f"dry running node '{node_id}'")
+                continue
+            log.info(f"running node '{node_id}'")
+            try:
+                dag.run_node(node_id, target)
+            except errors.FailedNodeRun as e:
+                log.error(e.message)
+                return
+            except Exception:
+                log.exception(f"node '{node_id}' run failed!")
+                return
+        if args.dry_run:
+            print(f"Dry run done. Would've run {n} node(s).")
+        else:
+            print(f"Done. Ran {n} node(s).")
+
+
+def show(args):
+    cfg = load_root_config()
+    dag = Builder(cfg).build()
+    node_ids = dag.select_nodes(args.pattern)
+    if not node_ids:
+        print("No matching nodes.")
+        return
+
+    for node_id in node_ids:
+        print(dag.render_node(node_id))
+
+
+def load_root_config():
+    cfg_path = find_root_config()
+
+    # Load .env in same dir as makegis.root.yml
+    cfg_dir = cfg_path.parent
+    dotenv.load_dotenv(cfg_dir / ".env")
+
+    return RootConfig.from_file(cfg_path)
+
+
+def find_root_config(cwd: Path = Path(".").resolve()):
+    """
+    Returns path to first makegis.root.yml file found in current dir or parents.
+    """
+    path = cwd / "makegis.root.yml"
+    if path.exists():
+        return path
+    parent = cwd.parent
+    if parent == cwd:
+        log.error("Found no makegis root file in current directory or its parents.")
+        exit(1)
+    return find_root_config(cwd=parent)
+
+
+if __name__ == "__main__":
+    cli()

makegis/config/__init__.py

@@ -0,0 +1,2 @@
+from .root import RootConfig
+from .root import TargetConfig

makegis/config/makegis.py

@@ -0,0 +1,253 @@
+import logging
+from pathlib import Path
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Literal
+
+from pydantic import BaseModel
+from pydantic import model_validator
+from pydantic import ValidationError
+import yaml
+
+try:
+    from yaml import CLoader as Loader
+except ImportError:
+    from yaml import Loader
+
+from .utils import expand_dict_strings
+
+log = logging.getLogger("makegis")
+
+
+class LoadDefaults(BaseModel):
+    epsg: int | str | None = None
+    geom_index: bool | None = None
+    geom_column: str | None = None
+    raster_index: bool | None = None
+    raster_column: str | None = None
+    raster_constraints: bool | None = None
+    tile_size: int | None = None
+
+
+class BaseSourceBlock(BaseModel):
+    epsg: int | str | None = None
+    # Name of column to use as primary key
+    pk: str | None = None
+
+
+class VectorSourceBlock(BaseModel):
+    geom_index: bool | None = None
+    geom_column: str | None = None
+
+
+class CSVSourceBlock(BaseSourceBlock, VectorSourceBlock):
+    type: Literal["csv"] = "csv"
+    path: Path
+    # TODO:
+    # x_column: str | None = None
+    # y_column: str | None = None
+    # keep_xy_columns: bool = False
+
+
+class EsriSourceBlock(BaseSourceBlock, VectorSourceBlock):
+    type: Literal["esri"] = "esri"
+    url: str
+    f: Literal["pjson", "pgeojson"] = "pjson"
+
+
+class DuckDBSourceBlock(BaseSourceBlock, VectorSourceBlock):
+    type: Literal["duckdb"] = "duckdb"
+    path: Path
+    table: Optional[str] = None
+
+
+class FileSourceBlock(BaseSourceBlock, VectorSourceBlock):
+    type: Literal["file"] = "file"
+    path: Path
+    layer: str | None = None
+
+
+class RasterSourceBlock(BaseSourceBlock):
+    type: Literal["raster"] = "raster"
+    path: Path
+    raster_index: bool | None = None
+    raster_column: str | None = None
+    raster_constraints: bool | None = None
+    tile_size: int | None = None
+
+
+class WFSSourceBlock(BaseSourceBlock, VectorSourceBlock):
+    type: Literal["wfs"] = "wfs"
+    url: str
+
+
+type SourceBlock = CSVSourceBlock | EsriSourceBlock | DuckDBSourceBlock | FileSourceBlock | RasterSourceBlock | WFSSourceBlock
+
+SOURCE_KEYS = set(["csv", "esri", "duckdb", "file", "raster", "wfs"])
+
+
+class LoadItem(BaseModel):
+    name: str
+    src: SourceBlock
+    meta: Dict[str, str | int | float | None]
+
+    @classmethod
+    def from_kv(cls, k: str, v: Dict):
+        name = k
+        meta = v.pop("meta", {})
+        matched_source_keys = [sk for sk in SOURCE_KEYS if sk in v]
+        if len(matched_source_keys) == 0:
+            raise RuntimeError(
+                f"Missing source key in load block item, execting one of {SOURCE_KEYS}"
+            )
+        elif len(matched_source_keys) > 1:
+            raise RuntimeError(
+                f"Too many source keys in load block item, expecting exactly one of {SOURCE_KEYS}"
+            )
+        if "csv" in matched_source_keys:
+            path = v.pop("csv")
+            src = CSVSourceBlock(path=path, **v)
+        elif "esri" in matched_source_keys:
+            url = v.pop("esri")
+            src = EsriSourceBlock(url=url, **v)
+        elif "duckdb" in matched_source_keys:
+            path = v.pop("duckdb")
+            src = DuckDBSourceBlock(path=path, **v)
+        elif "file" in matched_source_keys:
+            path = v.pop("file")
+            src = FileSourceBlock(path=path, **v)
+        elif "raster" in matched_source_keys:
+            path = v.pop("raster")
+            src = RasterSourceBlock(path=path, **v)
+        elif "wfs" in matched_source_keys:
+            url = v.pop("wfs")
+            src = WFSSourceBlock(url=url, **v)
+        else:
+            raise NotImplementedError("Unhandled source key in load block item")
+
+        return LoadItem(name=name, src=src, meta=meta)
+
+
+class SQLTransform(BaseModel):
+    path: Path
+
+
+class LoadBlock(BaseModel):
+    defaults: LoadDefaults
+    items: List[LoadItem]
+
+    @classmethod
+    def from_dict(cls, d: Dict):
+        defaults = LoadDefaults(**d.pop("defaults", {}))
+        items = [LoadItem.from_kv(k, v) for k, v in d.items()]
+        return LoadBlock(defaults=defaults, items=items)
+
+
+class TransformBlock(BaseModel):
+    transforms: List[SQLTransform]
+
+    @classmethod
+    def from_sequence(cls, s: List):
+        transforms = [SQLTransform(path=p) for p in s]
+        return TransformBlock(transforms=transforms)
+
+
+class DatabaseItem(BaseModel):
+    type: Literal["table", "function"]
+    name: str
+
+    @classmethod
+    def from_dict(cls, d: Dict):
+        assert (
+            len(d) == 1
+        ), "each item in a 'creates' or 'deps' block must have exactly 1 key e.g. - table: name"
+        k, v = next(iter(d.items()))
+        return DatabaseItem(type=k, name=v)
+
+
+class RunTask(BaseModel):
+    cmd: str
+    creates: List[DatabaseItem]
+
+    @classmethod
+    def from_dict(cls, d: Dict):
+        creates = [DatabaseItem.from_dict(item) for item in d.pop("creates", [])]
+        return RunTask(creates=creates, **d)
+
+
+class DoBlock(BaseModel):
+    """The 'do' key in a 'node' block"""
+
+    load: Optional[LoadBlock] = None
+    run: Optional[List[RunTask]] = None
+
+    @model_validator(mode="after")
+    def at_least_one(self):
+        if self.load is None and self.run is None:
+            raise ValidationError(
+                "A node's do block must have a 'load' and/or a 'run' key"
+            )
+        return self
+
+    @classmethod
+    def from_dict(cls, d: Dict):
+        load, tasks = None, None
+        if "load" in d:
+            load = LoadBlock.from_dict(d["load"])
+        if "run" in d:
+            tasks = [RunTask.from_dict(t) for t in d["run"]]
+        return DoBlock(load=load, run=tasks)
+
+
+class NodeBlock(BaseModel):
+    """Top-level 'node' block in a makegis.yml file"""
+
+    deps: List[DatabaseItem] | None = []
+    prep: List[str] | None = []
+    do: DoBlock
+    post: List[str] | None = []
+    cleanup: List[str] | None = []
+
+    @classmethod
+    def from_dict(cls, d: Dict):
+        deps = [DatabaseItem.from_dict(d) for d in d.pop("deps", [])]
+        do = DoBlock.from_dict(d.pop("do"))
+        return NodeBlock(deps=deps, do=do, **d)
+
+
+class MakeGISConfig(BaseModel):
+    block: LoadBlock | TransformBlock | NodeBlock
+    type: Literal["load", "transform", "node"]
+
+    @classmethod
+    def from_file(cls, path: Path):
+        log.debug(f"reading {path}")
+        with open(path) as f:
+            d = yaml.load(f, Loader)
+        return cls.from_dict(d)
+
+    @classmethod
+    def from_yaml(cls, s: str):
+        d = yaml.load(s, Loader)
+        return cls.from_dict(d)
+
+    @classmethod
+    def from_dict(cls, d: Dict):
+        expand_dict_strings(d)
+        assert len(d) == 1
+        key = list(d)[0]
+        if key == "load":
+            typ = "load"
+            block = LoadBlock.from_dict(d["load"])
+        elif key == "transform":
+            typ = "transform"
+            block = TransformBlock.from_sequence(d["transform"])
+        elif key == "node":
+            typ = "node"
+            block = NodeBlock.from_dict(d["node"])
+        else:
+            raise RuntimeError(
+                f"Unknown makegis file key '{key}', should be one of load, transform or node"
+            )
+        return MakeGISConfig(type=typ, block=block)

makegis/config/root.py

@@ -0,0 +1,69 @@
+import logging
+from pathlib import Path
+from typing import Dict
+
+from pydantic import BaseModel
+import yaml
+
+try:
+    from yaml import CLoader as Loader
+except ImportError:
+    from yaml import Loader
+
+from .makegis import LoadDefaults
+from .utils import expand_dict_strings
+
+log = logging.getLogger("makegis")
+
+
+class RootDefaults(BaseModel):
+    load: LoadDefaults = LoadDefaults()
+    target: str | None = None
+
+
+class TargetConfig(BaseModel):
+    """Describes a target database"""
+
+    # optional description
+    description: str | None = None
+    host: str = "localhost"
+    port: int = 5432
+    user: str = "postgres"
+    # database name
+    db: str
+
+    def conn_uri(self) -> str:
+        s = self
+        return f"postgresql://{s.user}@{s.host}:{s.port}/{s.db}"
+
+    def conn_str(self) -> str:
+        s = self
+        return f"host={s.host} port={s.port} dbname={s.db} user={s.user}"
+
+
+class RootConfig(BaseModel):
+    src_dir: Path
+    defaults: RootDefaults
+    targets: Dict[str, TargetConfig]
+
+    @classmethod
+    def from_file(cls, path: Path):
+        log.debug(f"reading {path}")
+        with open(path) as f:
+            d = yaml.load(f, Loader)
+        rc = cls.from_dict(d)
+        # Resolve path of src dir
+        if not rc.src_dir.is_absolute():
+            rc.src_dir = (path.parent / rc.src_dir).resolve()
+        return rc
+
+    @classmethod
+    def from_yaml(cls, s: str):
+        d = yaml.load(s, Loader)
+        return cls.from_dict(d)
+
+    @classmethod
+    def from_dict(cls, d: Dict):
+        expand_dict_strings(d)
+        defaults = RootDefaults(**d.pop("defaults", {}))
+        return RootConfig(defaults=defaults, **d)

makegis/config/utils.py

@@ -0,0 +1,24 @@
+import os
+from typing import Dict
+import re
+
+
+def expand_dict_strings(raw_dict: Dict):
+    """
+    Replaces {{variables}} found in strings in place.
+    """
+    pattern = re.compile(r"\{\{\s*(\w+)\s*\}\}")
+
+    def expand_string_values(d: dict):
+        for k, v in d.items():
+            if isinstance(v, dict):
+                expand_string_values(v)
+            if isinstance(v, str):
+                vars = re.findall(pattern, v)
+                for var in vars:
+                    if var not in os.environ:
+                        raise RuntimeError(f"unmatched env var {var}")
+                    v = re.sub(rf"\{{\{{\s*{var}\s*\}}\}}", os.environ[var], v)
+                d[k] = v
+
+    expand_string_values(raw_dict)

makegis/core/commands.py

@@ -0,0 +1,7 @@
+from pathlib import Path
+
+from pydantic import BaseModel
+
+
+class Command(BaseModel):
+    path: Path

makegis/core/load.py

@@ -0,0 +1,77 @@
+from pathlib import Path
+from dataclasses import dataclass
+from typing import Literal
+
+
+@dataclass(frozen=True)
+class BaseSource:
+    # Explicit srid of geometry in source dataset
+    epsg: int | None
+    # Name of column to use as primary key
+    pk: str | None
+
+
+@dataclass(frozen=True)
+class CSVSource(BaseSource):
+    path: Path
+
+
+@dataclass(frozen=True)
+class EsriSource(BaseSource):
+    url: str
+    f: Literal["pgeojson", "pjson"]
+
+
+@dataclass(frozen=True)
+class DuckDBSource(BaseSource):
+    # Path to database file
+    path: Path
+    # Fully qualified name of table to import
+    table: str
+
+
+@dataclass(frozen=True)
+class FileSource(BaseSource):
+    path: Path
+    # Optional layer name for file formats supporting it
+    layer: str | None
+
+
+@dataclass(frozen=True)
+class RasterSource(BaseSource):
+    path: Path
+
+
+@dataclass(frozen=True)
+class WFSSource(BaseSource):
+    url: str
+
+
+type Source = EsriSource | DuckDBSource | FileSource | WFSSource
+
+
+@dataclass(frozen=True)
+class Destination:
+    schema: str
+    table: str
+    # Desired srid of geometry column in destination table
+    epsg: int | None
+    # Name to assign to geometry column. Keep original if None.
+    geom_column: str | None
+    # Wether to index geometries or not.
+    # Defaults to False to be conservative.
+    geom_index: bool
+    # Name to assign to raster column.
+    raster_column: str
+    # Wether to index raster bounds
+    raster_index: bool
+    # Wether to set the standard set of raster constraints
+    raster_constraints: bool
+    # Raster tile size
+    tile_size: int | None 
+
+
+@dataclass(frozen=True)
+class LoadJob:
+    src: Source
+    dst: Destination

makegis/core/transforms.py

@@ -0,0 +1,7 @@
+from pathlib import Path
+
+from pydantic import BaseModel
+
+
+class Transform(BaseModel):
+    sql: Path

makegis/dag/__init__.py

@@ -0,0 +1 @@
+from .dag import DAG