gluekit 1.0.1.dev1__py3-none-any.whl

gluekit/local/local_fixtures.py

@@ -0,0 +1,337 @@
+from __future__ import annotations
+
+import json
+import posixpath
+import shutil
+from pathlib import Path
+from typing import Any
+
+LOCAL_RUN_CONFIG_FILE = Path(".gluekit/local.json")
+
+
+def empty_local_run_config() -> dict[str, Any]:
+    return {
+        "version": 1,
+        "s3": {"buckets": {}, "objects": {}, "directories": []},
+        "ssm": {"parameters": {}},
+    }
+
+
+def parse_s3_uri(raw: str) -> tuple[str, str]:
+    if not raw.startswith("s3://"):
+        raise ValueError(f"S3 URI must start with s3://. Received: {raw!r}")
+    bucket_and_key = raw[5:]
+    bucket, _, key = bucket_and_key.partition("/")
+    bucket = bucket.strip()
+    if not bucket:
+        raise ValueError(f"S3 URI must include a bucket name. Received: {raw!r}")
+    return bucket, key
+
+
+def is_s3_uri(raw: str) -> bool:
+    return raw.startswith("s3://")
+
+
+def normalize_s3_uri(raw: str) -> str:
+    bucket, key = parse_s3_uri(raw)
+    normalized_key = posixpath.normpath(key.strip("/")) if key.strip("/") else ""
+    if normalized_key == ".":
+        normalized_key = ""
+    if normalized_key:
+        return f"s3://{bucket}/{normalized_key}"
+    return f"s3://{bucket}"
+
+
+def normalize_s3_object_uri(raw: str) -> str:
+    bucket, key = parse_s3_uri(raw)
+    normalized = normalize_s3_uri(raw)
+    if not key.strip("/"):
+        raise ValueError(f"S3 object URI must include a key. Received: {raw!r}")
+    return normalized
+
+
+def s3_destination_uri(destination: str, source_name: str) -> str:
+    bucket, key = parse_s3_uri(destination)
+    if not key or destination.endswith("/"):
+        return normalize_s3_object_uri(
+            f"s3://{bucket}/{posixpath.join(key, source_name)}"
+        )
+    return normalize_s3_object_uri(destination)
+
+
+def s3_prefix(raw: str) -> str:
+    bucket, key = parse_s3_uri(raw)
+    normalized_key = key.strip("/")
+    if normalized_key:
+        return f"s3://{bucket}/{normalized_key}/"
+    return f"s3://{bucket}/"
+
+
+def s3_directory_destination(raw: str) -> str:
+    return s3_prefix(normalize_s3_uri(raw))
+
+
+def s3_uri_matches_prefix(uri: str, prefix: str) -> bool:
+    return uri.startswith(prefix)
+
+
+def normalize_s3_bucket_name(raw: str) -> str:
+    if raw.startswith("s3://"):
+        bucket, key = parse_s3_uri(raw)
+        if key.strip("/"):
+            raise ValueError(f"S3 bucket mapping must not include a key: {raw!r}")
+        return bucket
+    bucket = raw.strip()
+    if not bucket:
+        raise ValueError("S3 bucket mapping must include a bucket name.")
+    if "/" in bucket:
+        raise ValueError(f"S3 bucket mapping must be a bucket name: {raw!r}")
+    return bucket
+
+
+def normalize_s3_root_path(raw: str) -> str:
+    root = raw.strip()
+    if not root:
+        raise ValueError("S3 root mapping must include a local directory.")
+    return root
+
+
+def load_local_fixture_config(
+    config_file: str | Path | None = None,
+) -> dict[str, Any]:
+    path = Path(config_file) if config_file is not None else LOCAL_RUN_CONFIG_FILE
+    if not path.exists():
+        return empty_local_run_config()
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"Invalid JSON in local run config {path}: {exc}") from exc
+    if not isinstance(data, dict):
+        raise ValueError(f"Local run config {path} must contain a JSON object.")
+    return normalize_local_fixture_config(data, path=path)
+
+
+def normalize_local_fixture_config(
+    data: dict[str, Any], *, path: Path | None = None
+) -> dict[str, Any]:
+    location = f" in {path}" if path else ""
+    if data.get("version") != 1:
+        raise ValueError(f"Local run config{location} must use version 1.")
+    raw_profile = data.get("profile")
+    if raw_profile is not None and not isinstance(raw_profile, str):
+        raise ValueError(f"Local run config{location} has invalid profile.")
+    profile = raw_profile.strip() if isinstance(raw_profile, str) else None
+
+    s3 = data.get("s3", {})
+    if not isinstance(s3, dict):
+        raise ValueError(f"Local run config{location} has invalid s3 section.")
+    raw_buckets = s3.get("buckets", {})
+    raw_root = s3.get("root")
+    raw_objects = s3.get("objects", {})
+    raw_directories = s3.get("directories", [])
+    if raw_root is not None and not isinstance(raw_root, str):
+        raise ValueError(f"Local run config{location} has invalid s3.root.")
+    if not isinstance(raw_buckets, dict) or not all(
+        isinstance(bucket, str) and isinstance(local_path, str)
+        for bucket, local_path in raw_buckets.items()
+    ):
+        raise ValueError(f"Local run config{location} has invalid s3.buckets.")
+    if not isinstance(raw_objects, dict) or not all(
+        isinstance(uri, str) and isinstance(local_path, str)
+        for uri, local_path in raw_objects.items()
+    ):
+        raise ValueError(f"Local run config{location} has invalid s3.objects.")
+    if not isinstance(raw_directories, list):
+        raise ValueError(f"Local run config{location} has invalid s3.directories.")
+
+    buckets: dict[str, str] = {}
+    for bucket, local_path in raw_buckets.items():
+        normalized_bucket = normalize_s3_bucket_name(bucket)
+        if not local_path.strip():
+            raise ValueError(
+                f"Local run config{location} has invalid local path for S3 bucket {bucket!r}."
+            )
+        buckets[normalized_bucket] = local_path
+
+    root = normalize_s3_root_path(raw_root) if isinstance(raw_root, str) else None
+
+    objects: dict[str, str] = {}
+    for uri, local_path in raw_objects.items():
+        normalized_uri = normalize_s3_object_uri(uri)
+        objects[normalized_uri] = local_path
+    directories: list[dict[str, str]] = []
+    seen_directories: set[tuple[str, str]] = set()
+    for entry in raw_directories:
+        if not isinstance(entry, dict):
+            raise ValueError(
+                f"Local run config{location} has invalid s3.directories entry."
+            )
+        raw_local_path = entry.get("local_path")
+        raw_destination = entry.get("destination")
+        if not isinstance(raw_local_path, str) or not raw_local_path.strip():
+            raise ValueError(
+                f"Local run config{location} has invalid local_path in s3.directories."
+            )
+        if not isinstance(raw_destination, str) or not raw_destination.strip():
+            raise ValueError(
+                f"Local run config{location} has invalid destination in s3.directories."
+            )
+        normalized_destination = s3_directory_destination(raw_destination)
+        directory_key = (raw_local_path, normalized_destination)
+        if directory_key in seen_directories:
+            continue
+        seen_directories.add(directory_key)
+        directories.append(
+            {
+                "local_path": raw_local_path,
+                "destination": normalized_destination,
+            }
+        )
+
+    ssm = data.get("ssm", {})
+    if not isinstance(ssm, dict):
+        raise ValueError(f"Local run config{location} has invalid ssm section.")
+    raw_parameters = ssm.get("parameters", {})
+    if not isinstance(raw_parameters, dict):
+        raise ValueError(f"Local run config{location} has invalid ssm.parameters.")
+
+    parameters: dict[str, str] = {}
+    for name, value in raw_parameters.items():
+        if not isinstance(name, str) or not name.strip() or " " in name:
+            raise ValueError(
+                f"Local run config{location} has invalid SSM parameter name."
+            )
+        if not isinstance(value, str):
+            raise ValueError(
+                f"Local run config{location} has invalid value for SSM parameter {name!r}."
+            )
+        parameters[name.strip()] = value
+
+    s3_config: dict[str, Any] = {
+        "buckets": dict(sorted(buckets.items())),
+        "objects": dict(sorted(objects.items())),
+        "directories": sorted(
+            directories,
+            key=lambda entry: (entry["destination"], entry["local_path"]),
+        ),
+    }
+    if root is not None:
+        s3_config["root"] = root
+
+    normalized: dict[str, Any] = {
+        "version": 1,
+        "s3": s3_config,
+        "ssm": {"parameters": dict(sorted(parameters.items()))},
+    }
+    if profile:
+        normalized["profile"] = profile
+    return normalized
+
+
+def save_local_fixture_config(
+    config: dict[str, Any], config_file: str | Path | None = None
+) -> None:
+    path = Path(config_file) if config_file is not None else LOCAL_RUN_CONFIG_FILE
+    normalized = normalize_local_fixture_config(config, path=path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(normalized, indent=4) + "\n", encoding="utf-8")
+
+
+def config_to_runtime_kwargs(config: dict[str, Any]) -> dict[str, Any]:
+    s3_objects = expand_s3_root_mapping(config)
+    s3_objects.update(expand_bucket_mappings(config))
+    s3_objects.update(config["s3"]["objects"])
+    s3_objects.update(expand_directory_mappings(config))
+    bucket_names = {parse_s3_uri(object_uri)[0] for object_uri in s3_objects}
+    return {
+        "create_buckets": sorted(bucket_names),
+        "s3_objects": s3_objects,
+        "ssm_parameters": dict(config["ssm"]["parameters"]),
+    }
+
+
+def load_runtime_config(config_file: str | Path | None = None) -> dict[str, Any]:
+    path = Path(config_file) if config_file is not None else LOCAL_RUN_CONFIG_FILE
+    if not path.exists():
+        return {}
+    return config_to_runtime_kwargs(load_local_fixture_config(path))
+
+
+def expand_s3_root_mapping(config: dict[str, Any]) -> dict[str, str]:
+    root = config.get("s3", {}).get("root")
+    if not isinstance(root, str):
+        return {}
+    root_path = Path(root)
+    if not root_path.exists():
+        raise ValueError(f"Mapped local S3 root directory does not exist: {root_path}")
+    if not root_path.is_dir():
+        raise ValueError(f"Mapped local S3 root path is not a directory: {root_path}")
+
+    expanded: dict[str, str] = {}
+    for bucket_dir in sorted(filter(Path.is_dir, root_path.iterdir())):
+        bucket = normalize_s3_bucket_name(bucket_dir.name)
+        for candidate in sorted(filter(Path.is_file, bucket_dir.rglob("*"))):
+            relative_path = candidate.relative_to(bucket_dir).as_posix()
+            target_uri = normalize_s3_object_uri(f"s3://{bucket}/{relative_path}")
+            expanded[target_uri] = str(candidate)
+    return expanded
+
+
+def expand_bucket_mappings(config: dict[str, Any]) -> dict[str, str]:
+    expanded: dict[str, str] = {}
+    buckets = config.get("s3", {}).get("buckets", {})
+    if not isinstance(buckets, dict):
+        return expanded
+    for bucket, local_path in buckets.items():
+        if not isinstance(bucket, str) or not isinstance(local_path, str):
+            continue
+        source_dir = Path(local_path)
+        if not source_dir.exists():
+            raise ValueError(
+                f"Mapped local S3 bucket directory does not exist: {source_dir}"
+            )
+        if not source_dir.is_dir():
+            raise ValueError(
+                f"Mapped local S3 bucket path is not a directory: {source_dir}"
+            )
+        for candidate in sorted(filter(Path.is_file, source_dir.rglob("*"))):
+            relative_path = candidate.relative_to(source_dir).as_posix()
+            target_uri = normalize_s3_object_uri(f"s3://{bucket}/{relative_path}")
+            expanded[target_uri] = str(candidate)
+    return expanded
+
+
+def expand_directory_mappings(config: dict[str, Any]) -> dict[str, str]:
+    expanded: dict[str, str] = {}
+    directories = config.get("s3", {}).get("directories", [])
+    if not isinstance(directories, list):
+        return expanded
+    for entry in directories:
+        if not isinstance(entry, dict):
+            continue
+        local_path = entry.get("local_path")
+        destination = entry.get("destination")
+        if not isinstance(local_path, str) or not isinstance(destination, str):
+            continue
+        source_dir = Path(local_path)
+        if not source_dir.exists():
+            raise ValueError(f"Mapped local directory does not exist: {source_dir}")
+        if not source_dir.is_dir():
+            raise ValueError(f"Mapped local path is not a directory: {source_dir}")
+        for candidate in sorted(filter(Path.is_file, source_dir.rglob("*"))):
+            relative_path = candidate.relative_to(source_dir).as_posix()
+            target_uri = normalize_s3_object_uri(f"{destination}{relative_path}")
+            expanded[target_uri] = str(candidate)
+    return expanded
+
+
+def copy_mapped_file(source_path: str, destination: str | Path) -> Path:
+    source = Path(source_path)
+    if not source.is_file():
+        raise ValueError(f"Mapped local file does not exist: {source}")
+    destination_path = Path(destination)
+    if str(destination).endswith(("/", "\\")) or destination_path.is_dir():
+        destination_path = destination_path / source.name
+    destination_path.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copyfile(source, destination_path)
+    return destination_path

gluekit/local/pyspark/__init__.py

@@ -0,0 +1,7 @@
+"""pyspark top-level module"""
+
+from . import sql,context
+
+
+
+__all__ = ["context", "sql"]

gluekit/local/pyspark/context.py

@@ -0,0 +1,31 @@
+from sqlframe.duckdb import DuckDBSession
+
+
+class SparkContext:
+    """
+
+    Minimal local placeholder for Glue bootstrap compatibility.
+
+    Example usage in Glue job or notebook:
+
+        from awsglue.context import GlueContext
+        from awsglue.job import Job
+        from awsglue.utils import getResolvedOptions
+        from pyspark.context import SparkContext
+
+        sc = SparkContext()
+        glueContext = GlueContext(sc)
+        spark = glueContext.spark_session
+        args = getResolvedOptions(sys.argv, ['JOB_NAME'])
+        job = Job(glueContext)
+        job.init(args['JOB_NAME'], args)
+
+    """
+
+    def __init__(self):
+
+        self._builder = DuckDBSession.builder
+
+    @classmethod
+    def getOrCreate(cls):
+        return cls()

gluekit/local/pyspark/sql/__init__.py

@@ -0,0 +1,6 @@
+from sqlframe import activate
+
+activate(engine="duckdb")
+
+
+from .session import SparkSession

gluekit/local/pyspark/sql/session.py

@@ -0,0 +1,29 @@
+from sqlframe.duckdb import DuckDBSession
+
+
+class SparkSession(DuckDBSession):
+    """
+
+
+    Minimal local placeholder for Glue bootstrap compatibility.
+    Probably do not need this as this is redundant with
+    sqlframe but using but adding here just in case.
+
+    Example usage in Glue job or notebook:
+
+    from pyspark.sql import SparkSession
+
+    spark = (
+        SparkSession.builder.appName("DataMigrationCrossTrackProfiling"
+    ).config("spark.sql.caseSensitive", "true").getOrCreate()
+    )
+
+
+    """
+
+    def __init__(self):
+        self._builder = DuckDBSession.builder
+        
+    def getOrCreate(self):
+        session = self._builder.getOrCreate()
+        return session