bedrock-ge 0.2.4__py3-none-any.whl → 0.3.0__py3-none-any.whl

bedrock_ge/gi/geospatial.py

@@ -0,0 +1,349 @@
+import geopandas as gpd
+import numpy as np
+import pandas as pd
+from pandera.typing import DataFrame
+from pyproj import CRS, Transformer
+from pyproj.crs.crs import CompoundCRS
+from shapely.geometry import LineString, Point
+
+from bedrock_ge.gi.schemas import (
+    BedrockGIDatabase,
+    BedrockGIGeospatialDatabase,
+    InSituTestSchema,
+    LocationSchema,
+    SampleSchema,
+)
+
+
+def create_brgi_geodb(
+    brgi_db: BedrockGIDatabase,
+) -> BedrockGIGeospatialDatabase:
+    """Creates a Bedrock GI geospatial database from a Bedrock GI database.
+
+    Creates a Bedrock GI geospatial database by performing the following steps:
+    1. Creates a geospatial DataFrame for the Location table using the
+       `create_location_geodf` function.
+    2. Creates a geospatial DataFrame for the LonLatHeight table using the
+       `create_lon_lat_height_geodf` function.
+    3. Creates a dictionary of geospatial DataFrames for the In-Situ test tables
+       using the `interpolate_gi_geometry` function.
+    4. Creates a geospatial DataFrame for the Sample table using the
+       `interpolate_gi_geometry` function, if the Sample table exists.
+    5. Returns a BedrockGIGeospatialDatabase object.
+
+    Args:
+        brgi_db (BedrockGIDatabase): The Bedrock GI database to be converted.
+
+    Returns:
+        BedrockGIGeospatialDatabase: The resulting Bedrock GI geospatial database.
+    """
+    location_geodf = create_location_geodf(brgi_db)
+    lon_lat_height_geodf = create_lon_lat_height_geodf(brgi_db)
+    insitu_test_geodfs = {}
+    for insitu_test_name, insitu_test_data in brgi_db.InSituTests.items():
+        insitu_test_geodfs[insitu_test_name] = interpolate_gi_geometry(  # type: ignore
+            insitu_test_data,  # type: ignore
+            location_geodf,  # type: ignore
+        )  # type: ignore
+
+    if brgi_db.Sample is not None:
+        sample_geodf = interpolate_gi_geometry(brgi_db.Sample, location_geodf)  # type: ignore
+    else:
+        sample_geodf = None
+
+    return BedrockGIGeospatialDatabase(
+        Project=brgi_db.Project,
+        Location=location_geodf,
+        LonLatHeight=lon_lat_height_geodf,
+        InSituTests=insitu_test_geodfs,
+        Sample=sample_geodf,
+        LabTests=brgi_db.LabTests,
+        Other=brgi_db.Other,
+    )
+
+
+def create_location_geodf(brgi_db: BedrockGIDatabase) -> gpd.GeoDataFrame:
+    """Creates a geospatial DataFrame for the Location table from a Bedrock GI database.
+
+    This function generates a GeoDataFrame for the Location table using the input
+    Bedrock GI database. It assumes the boreholes are vertical (for now) and calculates
+    elevation at the base of each borehole. It raises an error if multiple
+    horizontal or vertical coordinate reference systems (CRS) are found in the
+    project data.
+
+    Args:
+        brgi_db (BedrockGIDatabase): The Bedrock GI database containing location
+            data and project CRS information.
+
+    Returns:
+        gpd.GeoDataFrame: A GeoDataFrame with LineString geometries representing
+            vertical boreholes, using the compound CRS derived from the project's
+            horizontal and vertical CRS.
+    """
+    # TODO: Implement logic to handle multiple CRS'es in the input GI data:
+    #       1. Create WKT geometry for each location in original CRS
+    #       2. Convert to WGS84 + EGM2008 orthometric height EPSG:9518
+    #       3. Interpolate InSituTest and Sample geospatial vector geometry from active geometry column
+    hor_crs_series = brgi_db.Project["horizontal_crs_wkt"]
+    vert_crs_series = brgi_db.Project["vertical_crs_wkt"]
+    if hor_crs_series.nunique() > 1 or vert_crs_series.nunique() > 1:
+        raise ValueError(
+            "All projects must have the same horizontal and vertical CRS (Coordinate Reference System).\n"
+            "Raise an issue on GitHub in case you need to be able to combine GI data that was acquired in multiple different CRSes."
+        )
+
+    horizontal_crs = CRS.from_wkt(hor_crs_series.iat[0])
+    vertical_crs = CRS.from_wkt(vert_crs_series.iat[0])
+    compound_crs = CompoundCRS(
+        name=f"{horizontal_crs.name} + {vertical_crs.name}",
+        components=[horizontal_crs, vertical_crs],
+    )
+
+    # TODO: Implement logic such that inclined borholes are handled correctly.
+    #       All boreholes are now assumed to be vertical.
+    location_df = brgi_db.Location.copy()
+    location_df["elevation_at_base"] = (
+        location_df["ground_level_elevation"] - location_df["depth_to_base"]
+    )
+    return gpd.GeoDataFrame(
+        brgi_db.Location.copy(),
+        geometry=location_df.apply(
+            lambda row: LineString(
+                [
+                    (row["easting"], row["northing"], row["ground_level_elevation"]),
+                    (row["easting"], row["northing"], row["elevation_at_base"]),
+                ]
+            ),
+            axis=1,
+        ),
+        crs=compound_crs,
+    )
+
+
+def create_lon_lat_height_geodf(brgi_db: BedrockGIDatabase) -> gpd.GeoDataFrame:
+    """Creates GeoDataFrame with (lon, lat, height) for each location in a Bedrock GI database.
+
+    This function processes all GI locations in a Bedrock GI database, transforming the
+    (easting, northing, ground level elevation) coordinates to WGS84 (lon, lat)
+    + EGM2008 orthometric height coordinates, which have coordinate reference system EPSG:9518.
+    It returns a GeoDataFrame with the transformed longitude, latitude, and
+    EGM2008 ground level height, along with the corresponding point geometries in EPSG:9518.
+
+    Args:
+        brgi_db (BedrockGIDatabase): The source Bedrock Ground Investigation database
+            containing location and project information.
+
+    Returns:
+        gpd.GeoDataFrame: A GeoDataFrame with the transformed longitude, latitude,
+            and EGM2008 ground level height, along with the corresponding point
+            geometries in EPSG:9518.
+    """
+    wgs84_egm2008_crs = CRS("EPSG:9518")
+    crs_lookup = brgi_db.Project.set_index("project_uid")
+    dfs = []
+    for project_uid, location_df in brgi_db.Location.groupby("project_uid"):
+        horizontal_crs = CRS.from_wkt(crs_lookup.at[project_uid, "horizontal_crs_wkt"])
+        vertical_crs = CRS.from_wkt(crs_lookup.at[project_uid, "vertical_crs_wkt"])
+        compound_crs = CompoundCRS(
+            name=f"{horizontal_crs.name} + {vertical_crs.name}",
+            components=[horizontal_crs, vertical_crs],
+        )
+        transformer = Transformer.from_crs(
+            compound_crs, wgs84_egm2008_crs, always_xy=True
+        )
+        lon, lat, egm2008_height = transformer.transform(
+            location_df["easting"],
+            location_df["northing"],
+            location_df["ground_level_elevation"],
+        )
+        dfs.append(
+            pd.DataFrame(
+                {
+                    "project_uid": project_uid,
+                    "location_uid": location_df["location_uid"],
+                    "longitude": lon,
+                    "latitude": lat,
+                    "egm2008_ground_level_height": egm2008_height,
+                }
+            )
+        )
+
+    lon_lat_height_df = pd.concat(dfs, ignore_index=True)
+    return gpd.GeoDataFrame(
+        lon_lat_height_df,
+        geometry=gpd.points_from_xy(
+            lon_lat_height_df["longitude"],
+            lon_lat_height_df["latitude"],
+            lon_lat_height_df["egm2008_ground_level_height"],
+        ),
+        crs=wgs84_egm2008_crs,
+    )
+
+
+def interpolate_gi_geometry(
+    insitu_test_df: DataFrame[InSituTestSchema] | DataFrame[SampleSchema],
+    location_geodf: gpd.GeoDataFrame,
+) -> gpd.GeoDataFrame:
+    """Interpolates the geospatial geometry for a given In-Situ test DataFrame using the corresponding GI Location GeoDataFrame.
+
+    This function takes an In-Situ test or Sample DataFrame and a GI Location GeoDataFrame and
+    returns a GeoDataFrame with its geometry interpolated from the Location GeoDataFrame.
+    The In-Situ test geometry is always a LineString or Point, depending on whether the
+    In-Situ test is performed at a specific depth or over a depth interval inside a borehole.
+    The geometry is calculated by linearly interpolating the depth values for each row
+    in a In-Situ test DataFrame along the corresponding location's LineString geometry.
+
+    Args:
+        insitu_test_df: The In-Situ test or Sample DataFrame containing the depth values to be interpolated.
+        location_geodf: The location GeoDataFrame containing the location LineStrings to be used for interpolation.
+
+    Returns:
+        gpd.GeoDataFrame: A GeoDataFrame containing the interpolated geospatial geometry
+            for the In-Situ test DataFrame.
+    """
+    # TODO: implement a warning when interpolating GI geospatial geometry when
+    # TODO: a single GI location has waaay too many rows in a certain In-Situ test.
+    geodf = location_geodf[["location_uid", "geometry"]].merge(
+        insitu_test_df,
+        how="right",
+        on="location_uid",
+    )
+    return gpd.GeoDataFrame(
+        insitu_test_df.copy(),
+        geometry=geodf.apply(
+            _interpolate_gi_geometry_row,
+            axis=1,
+        ),
+        crs=str(geodf.crs),
+    )
+
+
+def _interpolate_gi_geometry_row(row: pd.Series) -> LineString | Point:
+    """Process geometry based on available depth values for each row."""
+    has_top = pd.notna(row.get("depth_to_top"))
+    has_base = pd.notna(row.get("depth_to_base"))
+
+    if has_top and has_base:
+        return substring_3d(
+            row["geometry"],
+            start_dist=row["depth_to_top"],
+            end_dist=row["depth_to_base"],
+        )
+    elif has_top:
+        return interpolate_3d(
+            row["geometry"],
+            distance=row["depth_to_top"],
+        )
+    elif has_base:
+        return interpolate_3d(
+            row["geometry"],
+            distance=row["depth_to_base"],
+        )
+    else:
+        raise KeyError(
+            "An In-Situ test must either have a 'depth_to_top' or a 'depth_to_base', or both."
+        )
+
+
+def calc_distances_along_3d_linestring(linestring: LineString) -> np.ndarray:
+    """Calculate cumulative distances along a 3D LineString."""
+    coords = np.array(linestring.coords)
+    if coords.shape[1] < 3:
+        raise ValueError("Coordinates must be 3D (x, y, z)")
+
+    # Calculate 3D distances between consecutive points
+    diffs = np.diff(coords, axis=0)
+    distances = np.sqrt(np.sum(diffs**2, axis=1))
+
+    # Return cumulative distances (starting with 0)
+    return np.concatenate([[0], np.cumsum(distances)])
+
+
+def interpolate_3d(linestring: LineString, distance: float) -> Point:
+    """Interpolate a point along a 3D LineString using true 3D distance.
+
+    Return the first point if the distance is less than 0 or the last point if
+    the distance is greater than the total length. This behavior is different than
+    the shapely.LineString.interpolate method.
+
+    Args:
+        linestring: A 3D LineString geometry
+        distance: Distance along the line in 3D space
+
+    Returns:
+        Point: The interpolated 3D point
+    """
+    if distance <= 0:
+        return Point(linestring.coords[0])
+
+    cumulative_distances = calc_distances_along_3d_linestring(linestring)
+    total_length = cumulative_distances[-1]
+
+    if distance >= total_length:
+        return Point(linestring.coords[-1])
+
+    # Find the segment where the distance falls
+    segment_end_idx = int(np.searchsorted(cumulative_distances, distance))
+    segment_end_dist = cumulative_distances[segment_end_idx]
+    segment_start_idx = max(0, segment_end_idx - 1)  # Ensure non-negative
+    segment_start_dist = cumulative_distances[segment_start_idx]
+
+    # Get the coordinates of the point at the start of the segment
+    p1 = np.array(linestring.coords[segment_start_idx])
+    segment_length = segment_end_dist - segment_start_dist
+    if segment_length == 0:
+        return Point(p1)
+    p2 = np.array(linestring.coords[segment_end_idx])
+
+    # Calculate the ratio of how far along the segment the distance of interest falls
+    ratio = (distance - segment_start_dist) / segment_length
+
+    return Point(p1 + ratio * (p2 - p1))
+
+
+def substring_3d(
+    linestring: LineString, start_dist: float, end_dist: float
+) -> LineString | Point:
+    """Extract a substring of a 3D LineString using true 3D distances.
+
+    Args:
+        linestring: A 3D LineString geometry
+        start_dist: Start distance along the line in 3D space
+        end_dist: End distance along the line in 3D space
+
+    Returns:
+        LineString: The extracted 3D LineString segment
+    """
+    # Ensure start_dist <= end_dist
+    if start_dist > end_dist:
+        start_dist, end_dist = end_dist, start_dist
+
+    # Calculate cumulative 3D distances
+    cumulative_distances = calc_distances_along_3d_linestring(linestring)
+    total_length = cumulative_distances[-1]
+
+    # Handle edge cases
+    start_dist = max(0, min(start_dist, total_length))
+    end_dist = max(0, min(end_dist, total_length))
+
+    if start_dist == end_dist:
+        return interpolate_3d(linestring, start_dist)
+
+    # Find segments that intersect with our range
+    result_coords = []
+
+    # Add start point if it's not at a linestring vertex
+    start_point = interpolate_3d(linestring, start_dist)
+    result_coords.append(start_point.coords[0])
+
+    # Add all vertices that fall within the range
+    for i, dist in enumerate(cumulative_distances):
+        if start_dist < dist < end_dist:
+            result_coords.append(linestring.coords[i])
+
+    # Add end point if it's not at a vertex
+    end_point = interpolate_3d(linestring, end_dist)
+    if end_point.coords[0] != result_coords[-1]:  # Avoid duplicate points
+        result_coords.append(end_point.coords[0])
+
+    return LineString(result_coords)

bedrock_ge/gi/io_utils.py

@@ -0,0 +1,271 @@
+"""Utility functions for reading, parsing and writing data."""
+
+import codecs
+import io
+from contextlib import contextmanager, nullcontext
+from pathlib import Path
+from typing import IO, ContextManager
+
+import chardet
+import geopandas as gpd
+import pandas as pd
+
+from bedrock_ge.gi.schemas import BedrockGIDatabase, BedrockGIGeospatialDatabase
+
+DEFAULT_ENCODING = "utf-8"
+
+
+def detect_encoding(source: str | Path | IO[str] | IO[bytes] | bytes) -> str:
+    """Detect the character encoding of various input types.
+
+    Args:
+        source (str | Path | IO[str] | IO[bytes] | bytes): The source to detect encoding from.
+            - str or Path: File path.
+            - IO[str]: Already decoded text stream (returns `DEFAULT_ENCODING`)
+            - IO[bytes]: Binary stream to detect encoding from
+            - bytes: Binary data to detect encoding from
+
+    Returns:
+        str: The detected encoding name (e.g., 'utf-8', 'iso-8859-1', 'ascii', etc.)
+
+    Raises:
+        TypeError: If the source type is unsupported
+        FileNotFoundError: If a file path doesn't exist
+    """
+    # Set number of bytes to read for detection and required confidence
+    SAMPLE_SIZE = 1_000_000
+    REQUIRED_CONFIDENCE = 0.7
+
+    def _detect_from_bytes(data: bytes) -> str:
+        """Detect encoding from bytes data."""
+        sample = data[: min(len(data), SAMPLE_SIZE)]
+        result = chardet.detect(sample)
+        encoding = result.get("encoding", DEFAULT_ENCODING)
+        confidence = result.get("confidence", 0.0)
+
+        if not encoding or confidence < REQUIRED_CONFIDENCE:
+            return DEFAULT_ENCODING
+
+        if encoding.lower() == "ascii":
+            return "utf-8"
+
+        return encoding
+
+    def _read_from_path(path: Path):
+        """Read contents from path."""
+        if path.exists() and path.is_file():
+            with open(path, "rb") as file:
+                sample = file.read(SAMPLE_SIZE)
+                return _detect_from_bytes(sample)
+        else:
+            raise FileNotFoundError(
+                f"Path does not exist or is not a file: {path.__str__()[0:40]}"
+            )
+
+    # bytes
+    if isinstance(source, bytes):
+        return _detect_from_bytes(source)
+
+    # String, if not a path, still returns DEFAULT_ENCODING
+    if isinstance(source, str):
+        path = Path(source)
+        try:
+            return _read_from_path(path)
+        except FileNotFoundError:
+            return DEFAULT_ENCODING
+
+    # Path object
+    if isinstance(source, Path):
+        return _read_from_path(source)
+
+    # IO[str] object
+    if hasattr(source, "encoding"):
+        if source.encoding:
+            # Could be `None`, e.g. io.StringIO has an encoding attribute which is None.
+            return source.encoding
+        else:
+            return DEFAULT_ENCODING
+
+    # IO[bytes]
+    if isinstance(source, io.BufferedIOBase):
+        try:
+            if not source.seekable():
+                # For non-seekable streams, read what we can without seeking
+                sample = source.read(SAMPLE_SIZE)
+                if isinstance(sample, bytes):
+                    return _detect_from_bytes(sample)
+                else:
+                    return DEFAULT_ENCODING
+
+            # For seekable streams, preserve position
+            original_position = source.tell()
+            try:
+                source.seek(0)
+                sample = source.read(SAMPLE_SIZE)
+                if isinstance(sample, bytes):
+                    encoding = _detect_from_bytes(sample)
+                else:
+                    # if not bytes, then its a custom string-like type that was not caught
+                    encoding = DEFAULT_ENCODING
+                return encoding
+            finally:
+                source.seek(original_position)
+        except (AttributeError, IOError, OSError):
+            return DEFAULT_ENCODING
+
+    raise TypeError(f"Unsupported input type for encoding detection: {type(source)}")
+
+
+def open_text_data_source(
+    source: str | Path | IO[str] | IO[bytes] | bytes, encoding: str | None = None
+) -> ContextManager[io.TextIOBase]:
+    """Opens or wraps a given source for reading AGS (text-based) data.
+
+    Args:
+        source (str | Path | IO[str] | IO[bytes] | bytes): The source to read from.
+            - str or Path: File path or direct string content.
+            - IO[str]: A file-like text stream.
+            - IO[bytes]: Byte stream
+            - bytes: Binary content or stream (will be decoded).
+        encoding (str | None): Encoding to use for decoding bytes. Default is None.
+
+    Returns:
+        ContextManager[TextIOBase]: A context manager yielding a text stream.
+
+    Raises:
+        TypeError: If the source type is unsupported or binary streams are not decoded.
+    """
+    try:
+        codecs.lookup(encoding)
+    except LookupError:
+        raise ValueError(f"Unsupported encoding: {encoding}")
+
+    @contextmanager
+    def _bytes_source(bytes_content: bytes):
+        string_io = io.StringIO(bytes_content.decode(encoding))
+        try:
+            yield string_io
+        finally:
+            string_io.close()
+
+    if isinstance(source, (str, Path)):
+        path = Path(source)
+        if path.exists() and path.is_file():
+            return open(path, "r", encoding=encoding)
+        raise FileNotFoundError(f"Path does not exist or is not a file: {source}")
+
+    elif isinstance(source, io.TextIOBase):
+        source.seek(0)
+        return nullcontext(source)
+
+    elif isinstance(source, io.BufferedIOBase):
+        text_stream = io.TextIOWrapper(source, encoding=encoding)
+        text_stream.seek(0)
+        return nullcontext(text_stream)
+
+    elif isinstance(source, bytes):
+        return _bytes_source(source)
+
+    else:
+        raise TypeError(
+            f"Unsupported source type: {type(source)}. "
+            "Expected str, Path, IO[str], IO[bytes], or bytes."
+        )
+
+
+def coerce_string(string: str) -> None | bool | float | str:
+    """Converts a string to an appropriate Python data type.
+
+    Args:
+        string (str): The input string to be converted.
+
+    Returns:
+        None: If the string is 'none', 'null', or empty.
+        bool: If the string is 'true' or 'false' (case insensitive).
+        int: If the string can be converted to a float and has no decimal part.
+        float: If the string can be converted to a float with a decimal part.
+        str: If the string cannot be converted to any of the above types.
+
+    """
+    if string.lower() in {"none", "null", ""}:
+        return None
+    elif string.lower() == "true":
+        return True
+    elif string.lower() == "false":
+        return False
+    else:
+        try:
+            value = float(string)
+            if value.is_integer():
+                return int(value)
+            else:
+                return value
+        except ValueError:
+            return string
+
+
+def brgi_db_to_dfs(
+    brgi_db: BedrockGIDatabase | BedrockGIGeospatialDatabase,
+) -> dict[str, pd.DataFrame | gpd.GeoDataFrame]:
+    """Converts a Bedrock GI (geospatial) database to a dictionary of DataFrames.
+
+    Args:
+        brgi_db (BedrockGIDatabase | BedrockGIGeospatialDatabase): The Bedrock GI (geospatial) database.
+
+    Returns:
+        dict[str, pd.DataFrame | gpd.GeoDataFrame]: A dictionary where the keys are
+            the Bedrock GI table names and the values are the DataFrames that contain
+            the data for each table.
+    """
+    dict_of_dfs = {
+        "Project": brgi_db.Project,
+        "Location": brgi_db.Location,
+    }
+
+    if hasattr(brgi_db, "LonLatHeight"):
+        dict_of_dfs["LonLatHeight"] = brgi_db.LonLatHeight
+
+    if brgi_db.Sample is not None:
+        dict_of_dfs["Sample"] = brgi_db.Sample
+
+    insitu_dfs = {k: v for k, v in brgi_db.InSituTests.items()}
+    lab_dfs = {k: v for k, v in brgi_db.LabTests.items()}
+    other_dfs = {k: v for k, v in brgi_db.Other.items()}
+
+    return dict_of_dfs | insitu_dfs | lab_dfs | other_dfs
+
+
+def convert_object_col_content_to_string(
+    df: pd.DataFrame, in_place: bool = True
+) -> pd.DataFrame:
+    """Converts the data in columns with the object dtype to strings.
+
+    The real reason that this is necessary is that pandas and marimo are a little finicky about strings:
+    1. The built-in pd.Dataframe.convert_dtypes() method doesn't convert the dtype of
+      columns that contain multiple types in that same column to string.
+    2. marimo cannot handle pd.DataFrames with nullable strings (and other nullable pandas dtypes)
+      very well, see https://github.com/marimo-team/marimo/issues/5445.
+
+    Therefore, this function converts all the data in columns with the object dtype to strings,
+    and then back to the object dtype.
+
+    Args:
+        df: The DataFrame to modify.
+        in_place: Whether to modify the DataFrame in-place (default) or return a new DataFrame.
+
+    Returns:
+        pd.DataFrame: The modified DataFrame with object dtypes converted to string dtypes.
+
+    """
+    if not in_place:
+        df = df.copy()
+    object_cols = df.select_dtypes(include=["object"]).columns
+    df[object_cols] = df[object_cols].astype("string")
+    df[object_cols] = df[object_cols].astype("object")
+    return df
+
+
+def geodf_to_df(geodf: gpd.GeoDataFrame) -> pd.DataFrame:
+    """Convenience function to convert GeoDataFrames to DataFrames for nicer display in notebook environments like marimo."""
+    df = pd.DataFrame(geodf.copy())
+    return df.assign(geometry=df.geometry.astype(str))