bedrock-ge 0.2.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

bedrock_ge/gi/ags/validate.py

@@ -0,0 +1,26 @@
+import pandas as pd
+
+
+def check_ags_proj_group(ags_proj: pd.DataFrame) -> bool:
+    """Checks if the AGS 3 or AGS 4 PROJ group is correct.
+
+    Args:
+        proj_df (pd.DataFrame): The DataFrame with the PROJ group.
+
+    Raises:
+        ValueError: If AGS 3 of AGS 4 PROJ group is not correct.
+
+    Returns:
+        bool: Returns True if the AGS 3 or AGS 4 PROJ group is correct.
+    """
+
+    if len(ags_proj) != 1:
+        raise ValueError("The PROJ group must contain exactly one row.")
+
+    project_id = ags_proj["PROJ_ID"].iloc[0]
+    if not project_id:
+        raise ValueError(
+            'The project ID ("PROJ_ID" in the "PROJ" group) is missing from the AGS data.'
+        )
+
+    return True

bedrock_ge/gi/bedrock-gi-schema.json

@@ -0,0 +1,36 @@
+{
+  "Location": {
+    "attributes": {},
+    "geometry_type": "Point / 3D LineString",
+    "children": {
+      "MaterialClassification": {
+        "attributes": {},
+        "geometry_type": "3D LineString"
+      },
+      "SPT": {
+        "attributes": {},
+        "geometry_type": "3D Point"
+      },
+      "RQD": {
+        "attributes": {},
+        "geometry_type": "3D LineString"
+      },
+      "OtherInSituTests": {
+        "attributes": {},
+        "geometry_type": "3D Point or 3D LineString"
+      },
+      "Sample": {
+        "attributes": {},
+        "geometry_type": "3D Point",
+        "children": {
+          "grainSizeDistribution": {},
+          "atterbergLimits": {},
+          "oedometerTest": {},
+          "triaxialTest": {},
+          "unconfinedCompressiveStrength": {},
+          "otherLabTests": {}
+        }
+      }
+    }
+  }
+}

bedrock_ge/gi/concatenate.py

@@ -0,0 +1,38 @@
+from typing import Dict, Union
+
+import geopandas as gpd
+import pandas as pd
+
+
+def concatenate_databases(
+    db1: Dict[str, Union[pd.DataFrame, gpd.GeoDataFrame]],
+    db2: Dict[str, Union[pd.DataFrame, gpd.GeoDataFrame]],
+) -> Dict[str, pd.DataFrame]:
+    """
+    Concatenate two dictionaries of pandas dataframes into one dict of dfs.
+
+    The function concatenates the pandas dataframes of the second dict of dfs to the first dict of df for the keys that they have in common.
+    Keys that don't occur in either of the dict of dfs need to end up in the final dict of dfs.
+
+    Args:
+        db1 (Dict[str, pd.DataFrame]): A dictionary of pandas DataFrames, i.e. a database.
+        db2 (Dict[str, pd.DataFrame]): A dictionary of pandas DataFrames, i.e. a database.
+
+    Returns:
+        dict: A dictionary of concatenated pandas DataFrames.
+    """
+
+    # Create a new dict to store the concatenated dataframes
+    concatenated_dict = {key: df.dropna(axis=1, how="all") for key, df in db1.items()}
+
+    # Iterate over the keys in the second dict
+    for key, df in db2.items():
+        df = df.dropna(axis=1, how="all")
+        # If the key is also in the first dict, concatenate the dataframes
+        if key in db1:
+            concatenated_dict[key] = pd.concat([db1[key], df], ignore_index=True)
+        # If the key is not in the first dict, just add it to the new dict
+        else:
+            concatenated_dict[key] = df
+
+    return concatenated_dict

bedrock_ge/gi/gis_geometry.py

@@ -0,0 +1,235 @@
+from typing import Dict, Tuple, Union
+
+import geopandas as gpd
+import numpy as np
+import pandas as pd
+from pyproj import Transformer
+from pyproj.crs import CRS
+from shapely.geometry import LineString, Point
+
+# TODO: change function type hints, such that pandera checks the dataframes against the Bedrock schemas
+
+
+def calculate_gis_geometry(
+    no_gis_brgi_db: Dict[str, Union[pd.DataFrame, gpd.GeoDataFrame]],
+) -> Dict[str, gpd.GeoDataFrame]:
+    # Make sure that the Bedrock database is not changed outside this function.
+    brgi_db = no_gis_brgi_db.copy()
+
+    print("Calculating GIS geometry for the Bedrock GI database tables...")
+
+    # Check if all projects have the same CRS
+    if not brgi_db["Project"]["crs_wkt"].nunique() == 1:
+        raise ValueError(
+            "All projects must have the same CRS (Coordinate Reference System).\n"
+            "Raise an issue on GitHub in case you need to be able to combine GI data that was acquired in multiple different CRS's."
+        )
+
+    crs = CRS.from_wkt(brgi_db["Project"]["crs_wkt"].iloc[0])
+
+    # Calculate GIS geometry for the 'Location' table
+    print("Calculating GIS geometry for the Bedrock GI 'Location' table...")
+    brgi_db["Location"] = calculate_location_gis_geometry(brgi_db["Location"], crs)
+
+    # Create the 'LonLatHeight' table.
+    # The 'LonLatHeight' table makes it easier to visualize the GIS geometry on 2D maps,
+    # because vertical lines are often very small or completely hidden in 2D.
+    # This table only contains the 3D of the GI locations at ground level,
+    # in WGS84 (Longitude, Latitude, Height) coordinates.
+    print(
+        "Creating 'LonLatHeight' table with GI locations in WGS84 geodetic coordinates...",
+        "    WGS84 geodetic coordinates: (Longitude, Latitude, Ground Level Ellipsoidal Height)",
+        sep="\n",
+    )
+    brgi_db["LonLatHeight"] = create_lon_lat_height_table(brgi_db["Location"], crs)
+
+    # Create GIS geometry for tables that have In-Situ GIS geometry.
+    # These are the 'Sample' table and 'InSitu_...' tables.
+    # These tables are children of the Location table,
+    # i.e. have the 'Location' table as the parent table.
+    if "Sample" in brgi_db.keys():
+        print("Calculating GIS geometry for the Bedrock GI 'Sample' table...")
+        brgi_db["Sample"] = calculate_in_situ_gis_geometry(
+            brgi_db["Sample"], brgi_db["Location"], crs
+        )
+
+    for table_name, table in brgi_db.items():
+        if table_name.startswith("InSitu_"):
+            print(
+                f"Calculating GIS geometry for the Bedrock GI '{table_name}' table..."
+            )
+            brgi_db[table_name] = calculate_in_situ_gis_geometry(
+                table, brgi_db["Location"], crs
+            )
+
+    return brgi_db
+
+
+def calculate_location_gis_geometry(
+    brgi_location: Union[pd.DataFrame, gpd.GeoDataFrame], crs: CRS
+) -> gpd.GeoDataFrame:
+    """
+    Calculate GIS geometry for a set of Ground Investigation locations.
+
+    Args:
+        brgi_location (Union[pd.DataFrame, gpd.GeoDataFrame]): The GI locations to calculate GIS geometry for.
+        crs (pyproj.CRS): The Coordinate Reference System (CRS) to use for the GIS geometry.
+
+    Returns:
+        gpd.GeoDataFrame: The GIS geometry for the given GI locations, with *additional* columns:
+            longitude: The longitude of the location in the WGS84 CRS.
+            latitude: The latitude of the location in the WGS84 CRS.
+            wgs84_ground_level_height: The height of the ground level of the location in the WGS84 CRS.
+            elevation_at_base: The elevation at the base of the location.
+            geometry: The GIS geometry of the location.
+    """
+    # Calculate Elevation at base of GI location
+    brgi_location["elevation_at_base"] = (
+        brgi_location["ground_level_elevation"] - brgi_location["depth_to_base"]
+    )
+
+    # Make a gpd.GeoDataFrame from the pd.DataFrame by creating GIS geometry
+    brgi_location = gpd.GeoDataFrame(
+        brgi_location,
+        geometry=brgi_location.apply(
+            lambda row: LineString(
+                [
+                    (row["easting"], row["northing"], row["ground_level_elevation"]),
+                    (row["easting"], row["northing"], row["elevation_at_base"]),
+                ]
+            ),
+            axis=1,
+        ),
+        crs=crs,
+    )
+
+    # Calculate WGS84 geodetic coordinates
+    brgi_location[["longitude", "latitude", "wgs84_ground_level_height"]] = (
+        brgi_location.apply(
+            lambda row: calculate_wgs84_coordinates(
+                from_crs=crs,
+                easting=row["easting"],
+                northing=row["northing"],
+                elevation=row["ground_level_elevation"],
+            ),
+            axis=1,
+            result_type="expand",
+        )
+    )
+
+    return brgi_location
+
+
+def calculate_wgs84_coordinates(
+    from_crs: CRS, easting: float, northing: float, elevation: Union[float, None] = None
+) -> Tuple:
+    """Transform coordinates from an arbitrary Coordinate Reference System (CRS) to
+    the WGS84 CRS, which is the standard for geodetic coordinates.
+
+    Args:
+        from_crs (pyproj.CRS): The pyproj.CRS object of the CRS to transform from.
+        easting (float): The easting coordinate of the point to transform.
+        northing (float): The northing coordinate of the point to transform.
+        elevation (float or None, optional): The elevation of the point to
+            transform. Defaults to None.
+
+    Returns:
+        tuple: A tuple containing the longitude, latitude and WGS84 height of the
+            transformed point, in that order. The height is None if no elevation was
+            given, or if the provided CRS doesn't have a proper datum defined.
+    """
+    transformer = Transformer.from_crs(from_crs, 4326, always_xy=True)
+    if elevation:
+        lon, lat, wgs84_height = transformer.transform(easting, northing, elevation)
+    else:
+        lon, lat = transformer.transform(easting, northing)
+        wgs84_height = None
+
+    return lon, lat, wgs84_height
+
+
+def create_lon_lat_height_table(
+    brgi_location: gpd.GeoDataFrame, crs: CRS
+) -> gpd.GeoDataFrame:
+    """Create a GeoDataFrame with GI locations in WGS84 (lon, lat, height) coordinates.
+
+    The 'LonLatHeight' table makes it easier to visualize the GIS geometry on 2D maps,
+    because vertical lines are often very small or completely hidden in 2D. This table
+    only contains the 3D point of the GI locations at ground level, in WGS84 (Longitude,
+    Latitude, Height) coordinates. Other attributes, such as the location type, sample
+    type, geology description, etc., can be attached to this table by joining, i.e.
+    merging those tables on the location_uid key.
+
+    Args:
+        brgi_location (GeoDataFrame): The GeoDataFrame with the GI locations.
+        crs (CRS): The Coordinate Reference System of the GI locations.
+
+    Returns:
+        GeoDataFrame: The 'LonLatHeight' GeoDataFrame.
+    """
+    lon_lat_height = gpd.GeoDataFrame(
+        brgi_location[
+            [
+                "project_uid",
+                "location_uid",
+            ]
+        ],
+        geometry=brgi_location.apply(
+            lambda row: Point(
+                row["longitude"], row["latitude"], row["wgs84_ground_level_height"]
+            ),
+            axis=1,
+        ),
+        crs=4326,
+    )
+    return lon_lat_height
+
+
+def calculate_in_situ_gis_geometry(
+    brgi_in_situ: Union[pd.DataFrame, gpd.GeoDataFrame],
+    brgi_location: Union[pd.DataFrame, gpd.GeoDataFrame],
+    crs: CRS,
+) -> gpd.GeoDataFrame:
+    location_child = brgi_in_situ.copy()
+
+    # Merge the location data into the in-situ data to get the location coordinates
+    location_child = pd.merge(
+        location_child,
+        brgi_location[
+            ["location_uid", "easting", "northing", "ground_level_elevation"]
+        ],
+        on="location_uid",
+        how="left",
+    )
+
+    # Calculate the elevation at the top of the Sample or in-situ test
+    location_child["elevation_at_top"] = (
+        location_child["ground_level_elevation"] - location_child["depth_to_top"]
+    )
+    brgi_in_situ["elevation_at_top"] = location_child["elevation_at_top"]
+
+    # Calculate the elevation at the base of the Sample or in-situ test
+    if "depth_to_base" in location_child.columns:
+        location_child["elevation_at_base"] = (
+            location_child["ground_level_elevation"] - location_child["depth_to_base"]
+        )
+        brgi_in_situ["elevation_at_base"] = location_child["elevation_at_base"]
+
+    # Create the in-situ data as a GeoDataFrame with LineString GIS geometry for
+    # Samples or in-situ tests that have an elevation at the base of the Sample or in-situ test.
+    brgi_in_situ = gpd.GeoDataFrame(
+        brgi_in_situ,
+        geometry=location_child.apply(
+            lambda row: LineString(
+                [
+                    (row["easting"], row["northing"], row["elevation_at_top"]),
+                    (row["easting"], row["northing"], row["elevation_at_base"]),
+                ]
+            )
+            if "elevation_at_base" in row and not np.isnan(row["elevation_at_base"])
+            else Point((row["easting"], row["northing"], row["elevation_at_top"])),
+            axis=1,
+        ),
+        crs=crs,
+    )
+    return brgi_in_situ

bedrock_ge/gi/schemas.py

@@ -0,0 +1,95 @@
+"""pandera schemas for Bedrock GI data. Base schemas refer to schemas that have no calculated GIS geometry or values."""
+
+from typing import Optional
+
+import pandera as pa
+from pandera.typing import Series
+from pandera.typing.geopandas import GeoSeries
+
+
+class Project(pa.DataFrameModel):
+    project_uid: Series[str] = pa.Field(
+        # primary_key=True,
+        unique=True,
+    )
+    crs_wkt: Series[str] = pa.Field(description="Coordinate Reference System")
+    # datum: Series[str] = pa.Field(description="Datum used for measurement of the ground level elevation.")
+
+
+class BaseLocation(pa.DataFrameModel):
+    location_uid: Series[str] = pa.Field(
+        # primary_key=True,
+        unique=True,
+    )
+    project_uid: Series[str] = pa.Field(
+        # foreign_key="project.project_uid"
+    )
+    location_source_id: Series[str]
+    location_type: Series[str]
+    easting: Series[float] = pa.Field(coerce=True)
+    northing: Series[float] = pa.Field(coerce=True)
+    ground_level_elevation: Series[float] = pa.Field(
+        coerce=True,
+        description="Elevation w.r.t. a local datum. Usually the orthometric height from the geoid, i.e. mean sea level, to the ground level.",
+    )
+    depth_to_base: Series[float]
+
+
+class Location(BaseLocation):
+    elevation_at_base: Series[float]
+    longitude: Series[float]
+    latitude: Series[float]
+    wgs84_ground_level_height: Series[float] = pa.Field(
+        description="Ground level height w.r.t. the WGS84 (World Geodetic System 1984) ellipsoid.",
+        nullable=True,
+    )
+    geometry: GeoSeries
+
+
+class BaseInSitu(pa.DataFrameModel):
+    project_uid: Series[str] = pa.Field(
+        # foreign_key="project.project_uid"
+    )
+    location_uid: Series[str] = pa.Field(
+        # foreign_key="location.location_uid"
+    )
+    depth_to_top: Series[float] = pa.Field(coerce=True)
+    depth_to_base: Optional[Series[float]] = pa.Field(coerce=True, nullable=True)
+
+
+class BaseSample(BaseInSitu):
+    sample_uid: Series[str] = pa.Field(
+        # primary_key=True,
+        unique=True,
+    )
+    sample_source_id: Series[str]
+
+
+class Sample(BaseSample):
+    elevation_at_top: Series[float]
+    elevation_at_base: Optional[Series[float]] = pa.Field(nullable=True)
+    geometry: GeoSeries
+
+
+class InSitu(BaseInSitu):
+    elevation_at_top: Series[float]
+    elevation_at_base: Optional[Series[float]] = pa.Field(nullable=True)
+    geometry: GeoSeries
+
+
+class BaseLab(pa.DataFrameModel):
+    project_uid: Series[str] = pa.Field(
+        # foreign_key="project.project_uid"
+    )
+    location_uid: Series[str] = pa.Field(
+        # foreign_key="location.location_uid"
+    )
+    sample_uid: Series[str] = pa.Field(
+        # foreign_key="sample.sample_uid"
+    )
+
+
+class Lab(BaseLab):
+    geometry: GeoSeries = pa.Field(
+        description="GIS geometry of the sample on which this lab test was performed."
+    )

bedrock_ge/gi/sqlmodels.py

@@ -0,0 +1,74 @@
+from typing import Optional
+
+from sqlmodel import Field, SQLModel
+
+
+class Project(SQLModel, table=True):
+    project_uid: str = Field(primary_key=True)
+    crs: str = Field(description="Coordinate Reference System")
+
+
+class Location(SQLModel, table=True):
+    location_uid: str = Field(primary_key=True)
+    project_uid: str = Field(foreign_key="project.project_uid")
+    source_id: str
+    location_type: str
+    easting: float
+    northing: float
+    ground_level: float
+    depth_to_base: float
+    elevation_at_base: float
+    latitude: float
+    longitude: float
+
+
+class DepthInformation(SQLModel):
+    depth_to_top: float
+    depth_to_base: Optional[float] = None
+    elevation_at_top: float
+    elevation_at_base: Optional[float] = None
+
+
+class Sample(DepthInformation, table=True):
+    sample_uid: Optional[int] = Field(default=None, primary_key=True)
+    project_uid: str = Field(foreign_key="project.project_uid")
+    location_uid: str = Field(foreign_key="location.location_uid")
+    source_id: str
+
+
+class InSitu(DepthInformation):
+    project_uid: str = Field(foreign_key="project.project_uid")
+    location_uid: str = Field(foreign_key="location.location_uid")
+
+
+class Lab(SQLModel):
+    project_uid: str = Field(foreign_key="project.project_uid")
+    location_uid: str = Field(foreign_key="location.location_uid")
+    sample_uid: str = Field(foreign_key="sample.sample_uid")
+
+
+class Material(InSitu, table=True):
+    """Material descriptions from the field. GEOL group in AGS 3 and AGS 4."""
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    material_name: str
+    material_description: Optional[str] = None
+
+
+class SPT(InSitu, table=True):
+    id: Optional[int] = Field(default=None, primary_key=True)
+    spt_count: int
+
+
+class RockCore(InSitu, table=True):
+    id: Optional[int] = Field(default=None, primary_key=True)
+    tcr: Optional[float] = Field(default=None, description="Total Core Recovery (%)")
+    scr: Optional[float] = Field(default=None, description="Solid Core Recovery (%)")
+    rqd: Optional[float] = Field(
+        default=None, description="Rock Quality Designation (%)"
+    )
+
+
+class Weathering(InSitu, table=True):
+    id: Optional[int] = Field(default=None, primary_key=True)
+    weathering: str

bedrock_ge/gi/validate.py

@@ -0,0 +1,116 @@
+from typing import Dict, Union
+
+import geopandas as gpd  # type: ignore
+import pandas as pd
+
+from bedrock_ge.gi.schemas import (
+    BaseInSitu,
+    BaseLocation,
+    BaseSample,
+    InSitu,
+    Location,
+    Project,
+    Sample,
+)
+
+
+def check_brgi_database(brgi_db: Dict):
+    for table_name, table in brgi_db.items():
+        if table_name == "Project":
+            Project.validate(table)
+            print("'Project' table aligns with Bedrock's 'Project' table schema.")
+        elif table_name == "Location":
+            Location.validate(table)
+            check_foreign_key("project_uid", brgi_db["Project"], table)
+            print("'Location' table aligns with Bedrock's 'Location' table schema.")
+        elif table_name == "Sample":
+            Sample.validate(table)
+            check_foreign_key("project_uid", brgi_db["Project"], table)
+            check_foreign_key("location_uid", brgi_db["Location"], table)
+            print("'Sample' table aligns with Bedrock's 'Sample' table schema.")
+        elif table_name == "InSitu":
+            InSitu.validate(table)
+            check_foreign_key("project_uid", brgi_db["Project"], table)
+            check_foreign_key("location_uid", brgi_db["Location"], table)
+            print(
+                f"'{table_name}' table aligns with Bedrock's table schema for In-Situ measurements."
+            )
+        elif table_name.startswith("Lab_"):
+            print(
+                "🚨 !NOT IMPLEMENTED! We haven't come across Lab data yet. !NOT IMPLEMENTED!"
+            )
+
+    return True
+
+
+def check_no_gis_brgi_database(brgi_db: Dict):
+    for table_name, table in brgi_db.items():
+        if table_name == "Project":
+            Project.validate(table)
+            print("'Project' table aligns with Bedrock's 'Project' table schema.")
+        elif table_name == "Location":
+            BaseLocation.validate(table)
+            check_foreign_key("project_uid", brgi_db["Project"], table)
+            print(
+                "'Location' table aligns with Bedrock's 'Location' table schema without GIS geometry."
+            )
+        elif table_name == "Sample":
+            BaseSample.validate(table)
+            check_foreign_key("project_uid", brgi_db["Project"], table)
+            check_foreign_key("location_uid", brgi_db["Location"], table)
+            print(
+                "'Sample' table aligns with Bedrock's 'Sample' table schema without GIS geometry."
+            )
+        elif table_name.startswith("InSitu_"):
+            BaseInSitu.validate(table)
+            check_foreign_key("project_uid", brgi_db["Project"], table)
+            check_foreign_key("location_uid", brgi_db["Location"], table)
+            print(
+                f"'{table_name}' table aligns with Bedrock's '{table_name}' table schema without GIS geometry."
+            )
+        elif table_name.startswith("Lab_"):
+            print(
+                "🚨 !NOT IMPLEMENTED! We haven't come across Lab data yet. !NOT IMPLEMENTED!"
+            )
+
+    return True
+
+
+def check_foreign_key(
+    foreign_key: str,
+    parent_table: Union[pd.DataFrame, gpd.GeoDataFrame],
+    table_with_foreign_key: Union[pd.DataFrame, gpd.GeoDataFrame],
+) -> bool:
+    """
+    Checks if a foreign key in a table exists in the parent table.
+
+    Foreign keys describe the relationship between tables in a relational database.
+    For example, all GI Locations belong to a project.
+    All GI Locations are related to a project with the project_uid (Project Unique IDentifier).
+    The project_uid is the foreign key in the Location table.
+    This implies that the project_uid in the foreign key in the Location table must exist in the Project parent table.
+    That is what this function checks.
+
+    Args:
+        foreign_key (str): The name of the column of the foreign key.
+        parent_table (Union[pd.DataFrame, gpd.GeoDataFrame]): The parent table.
+        table_with_foreign_key (Union[pd.DataFrame, gpd.GeoDataFrame]): The table with the foreign key.
+
+    Raises:
+        ValueError: If the table with the foreign key contains foreign keys that don't occur in the parent table.
+
+    Returns:
+        bool: True if the foreign keys all exist in the parent table.
+    """
+    # Get the foreign keys that are missing in the parent group
+    missing_foreign_keys = table_with_foreign_key[
+        ~table_with_foreign_key[foreign_key].isin(parent_table[foreign_key])
+    ]
+
+    # Raise an error if there are missing foreign keys
+    if len(missing_foreign_keys) > 0:
+        raise ValueError(
+            f"This table contains '{foreign_key}'s that don't occur in the parent table:\n{missing_foreign_keys}"
+        )
+
+    return True