bedrock-ge 0.2.4__py3-none-any.whl → 0.3.0__py3-none-any.whl

bedrock_ge/gi/mapper.py

@@ -0,0 +1,221 @@
+import base64
+import hashlib
+import json
+
+import pandas as pd
+
+from bedrock_ge.gi.mapping_models import BedrockGIMapping
+from bedrock_ge.gi.schemas import (
+    BedrockGIDatabase,
+    InSituTestSchema,
+    LabTestSchema,
+    LocationSchema,
+    ProjectSchema,
+    SampleSchema,
+)
+
+
+def map_to_brgi_db(brgi_db_mapping: BedrockGIMapping) -> BedrockGIDatabase:
+    """Creates a Bedrock GI Database for a single project from a BedrockGIMapping.
+
+    This function takes a BedrockGIDatabaseMapping, which contains various table mappings
+    for project, location, in-situ tests, samples, lab tests, and other tables, and
+    converts it into a BedrockGIDatabase object. It creates pandas DataFrames for each
+    table, validates them against their respective schemas, and constructs the final
+    BedrockGIDatabase object.
+
+    Examples:
+        ```python
+        from pyproj import CRS
+        from bedrock_ge.gi.mapping_models import BedrockGIMapping
+
+        brgi_db_mapping = BedrockGIMapping(
+            ProjectTableMapping={
+                "data": {
+                    "project_name: "Test Project",
+                    "project_description": "Project description. Add more data about the project here if you please."
+                }
+                "project_id": "project-1",
+                "horizontal_crs": CRS("EPSG:2193"),
+                "vertical_crs": CRS("EPSG:7839"),
+            },
+            LocationTableMapping={
+                "data": location_df,
+                "location_id_column": "LocationID",
+                "easting_column": "Easting",
+                "northing_column": "Northing",
+                "ground_level_elevation_column": "GroundLevel",
+                "depth_to_base_column": "FinalDepth",
+            },
+            InSituTestTableMapping=[
+                {
+                    "table_name": "Geol",
+                    "data": geology_df,
+                    "location_id_column": "LocationID",
+                    "depth_to_top_column": "from",
+                    "depth_to_base_column": "to",
+                },
+                {
+                    "table_name": "SPT",
+                    "data": spt_df,
+                    "location_id_column": "LocationID",
+                    "depth_to_top"column": "from",
+                }
+            ],
+            SampleTableMapping=None,
+            LabTestTableMapping=[],
+            OtherTable=[],
+        )
+        ```
+
+    Args:
+        brgi_db_mapping (BedrockGIDatabaseMapping): The mapping object containing GI
+            data and metadata for mapping to Bedrock's schema.
+
+    Returns:
+        BedrockGIDatabase: The transformed Bedrock GI database containing validated
+            DataFrames for each table type.
+    """
+    # Create a base64 hash from the project data, such that a project Unique ID
+    # can be created from the project_id and the hash of the project data.
+    project_data_jsons = json.dumps(brgi_db_mapping.Project.data, sort_keys=True)
+    project_data_bytes_hash = hashlib.blake2b(
+        project_data_jsons.encode("utf-8"), digest_size=9
+    ).digest()
+    project_data_b64_hash = base64.b64encode(project_data_bytes_hash).decode()
+    project_uid = brgi_db_mapping.Project.project_id + "-" + project_data_b64_hash
+
+    # Create the project table
+    project_df = pd.DataFrame(
+        {
+            "project_uid": project_uid,
+            "project_source_id": brgi_db_mapping.Project.project_id,
+            "horizontal_crs": brgi_db_mapping.Project.horizontal_crs.to_string(),
+            "horizontal_crs_wkt": brgi_db_mapping.Project.horizontal_crs.to_wkt(),
+            "vertical_crs": brgi_db_mapping.Project.vertical_crs.to_string(),
+            "vertical_crs_wkt": brgi_db_mapping.Project.vertical_crs.to_wkt(),
+            **brgi_db_mapping.Project.data,
+        },
+        index=[0],
+    )
+    project_df = project_df.loc[:, ~project_df.columns.duplicated()]
+    ProjectSchema.validate(project_df)
+
+    # Create the location table
+    location_df = pd.DataFrame(
+        {
+            "location_uid": brgi_db_mapping.Location.data[
+                brgi_db_mapping.Location.location_id_column
+            ]
+            + f"_{project_uid}",
+            "location_source_id": brgi_db_mapping.Location.data[
+                brgi_db_mapping.Location.location_id_column
+            ],
+            "project_uid": project_uid,
+            "easting": brgi_db_mapping.Location.data[
+                brgi_db_mapping.Location.easting_column
+            ],
+            "northing": brgi_db_mapping.Location.data[
+                brgi_db_mapping.Location.northing_column
+            ],
+            "ground_level_elevation": brgi_db_mapping.Location.data[
+                brgi_db_mapping.Location.ground_level_elevation_column
+            ],
+            "depth_to_base": brgi_db_mapping.Location.data[
+                brgi_db_mapping.Location.depth_to_base_column
+            ],
+        }
+    )
+    location_df = pd.concat([location_df, brgi_db_mapping.Location.data], axis=1)
+    location_df = location_df.loc[:, ~location_df.columns.duplicated()]
+    location_df = LocationSchema.validate(location_df)
+
+    # Create the in-situ test tables
+    insitu_tests = {}
+    for insitu_mapping in brgi_db_mapping.InSitu:
+        insitu_df = pd.DataFrame(
+            {
+                "project_uid": project_uid,
+                "location_uid": insitu_mapping.data[insitu_mapping.location_id_column]
+                + f"_{project_uid}",
+            }
+        )
+        if insitu_mapping.depth_to_top_column:
+            insitu_df["depth_to_top"] = insitu_mapping.data[
+                insitu_mapping.depth_to_top_column
+            ]
+        if insitu_mapping.depth_to_base_column:
+            insitu_df["depth_to_base"] = insitu_mapping.data[
+                insitu_mapping.depth_to_base_column
+            ]
+        insitu_df = pd.concat([insitu_df, insitu_mapping.data], axis=1)
+        insitu_df = insitu_df.loc[:, ~insitu_df.columns.duplicated()]
+        insitu_df = InSituTestSchema.validate(insitu_df)
+        insitu_tests[insitu_mapping.table_name] = insitu_df.copy()
+
+    # Create the sample table
+    sample_df = None
+    if brgi_db_mapping.Sample:
+        sample_df = pd.DataFrame(
+            {
+                "sample_uid": brgi_db_mapping.Sample.data[
+                    brgi_db_mapping.Sample.sample_id_column
+                ]
+                + f"_{project_uid}",
+                "sample_source_id": brgi_db_mapping.Sample.data[
+                    brgi_db_mapping.Sample.sample_id_column
+                ],
+                "project_uid": project_uid,
+                "location_uid": brgi_db_mapping.Sample.data[
+                    brgi_db_mapping.Sample.location_id_column
+                ]
+                + f"_{project_uid}",
+                "depth_to_top": brgi_db_mapping.Sample.data[
+                    brgi_db_mapping.Sample.depth_to_top_column
+                ],
+            }
+        )
+        if brgi_db_mapping.Sample.depth_to_base_column:
+            sample_df["depth_to_base"] = brgi_db_mapping.Sample.data[
+                brgi_db_mapping.Sample.depth_to_top_column
+            ]
+        sample_df = pd.concat([sample_df, brgi_db_mapping.Sample.data], axis=1)
+        sample_df = sample_df.loc[:, ~sample_df.columns.duplicated()]
+        sample_df = SampleSchema.validate(sample_df)
+
+    # Create the lab test tables
+    lab_tests = {}
+    if brgi_db_mapping.Lab:
+        for lab_mapping in brgi_db_mapping.Lab:
+            lab_df = pd.DataFrame(
+                {
+                    "project_uid": project_uid,
+                    "sample_uid": lab_mapping.data[lab_mapping.sample_id_column]
+                    + f"_{project_uid}",
+                }
+            )
+            if lab_mapping.location_id_column:
+                lab_df["location_uid"] = lab_mapping.data[
+                    lab_mapping.location_id_column
+                ]
+            lab_df = pd.concat([lab_df, lab_mapping.data.copy()], axis=1)
+            LabTestSchema.validate(lab_df)
+            lab_tests[lab_mapping.table_name] = lab_df.copy()
+
+    # Create the other tables
+    other_tables = {}
+    if brgi_db_mapping.Other:
+        for other_table_mapping in brgi_db_mapping.Other:
+            other_table_df = other_table_mapping.data
+            other_table_df.insert(0, "project_uid", project_uid)
+            other_tables[other_table_mapping.table_name] = other_table_df
+
+    # Create and return the Bedrock GI database
+    return BedrockGIDatabase(
+        Project=project_df,
+        Location=location_df,
+        InSituTests=insitu_tests,
+        Sample=sample_df,
+        LabTests=lab_tests,
+        Other=other_tables,
+    )

bedrock_ge/gi/mapping_models.py

@@ -0,0 +1,69 @@
+from typing import Optional, Union
+
+import pandas as pd
+import pyproj
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class ProjectTableMapping(BaseModel):
+    data: dict = {}
+    project_id: str
+    horizontal_crs: pyproj.CRS
+    vertical_crs: pyproj.CRS = Field(default=pyproj.CRS(3855))
+    # "compound_crs": Optional[CRS] = None
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+class LocationTableMapping(BaseModel):
+    data: pd.DataFrame
+    location_id_column: str
+    easting_column: str
+    northing_column: str
+    ground_level_elevation_column: str
+    depth_to_base_column: str
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+class SampleTableMapping(BaseModel):
+    data: pd.DataFrame
+    sample_id_column: str
+    location_id_column: str
+    depth_to_top_column: str
+    depth_to_base_column: Optional[str] = None
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+class OtherTable(BaseModel):
+    table_name: str
+    data: pd.DataFrame
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+class InSituTestTableMapping(OtherTable):
+    location_id_column: str
+    depth_to_top_column: Optional[str] = None
+    depth_to_base_column: Optional[str] = None
+
+    @model_validator(mode="after")
+    def validate_at_least_one_depth_column(self):
+        if not (self.depth_to_top_column or self.depth_to_base_column):
+            raise ValueError("At least one depth column must be specified.")
+        return self
+
+
+class LabTestTableMapping(OtherTable):
+    sample_id_column: str
+    location_id_column: Optional[str] = None
+
+
+class BedrockGIMapping(BaseModel):
+    Project: ProjectTableMapping
+    Location: LocationTableMapping
+    InSitu: list[InSituTestTableMapping]
+    Sample: Union[SampleTableMapping, None] = None
+    Lab: list[LabTestTableMapping] = []
+    Other: list[OtherTable] = []

bedrock_ge/gi/schemas.py

@@ -2,21 +2,33 @@
 
 from typing import Optional
 
-import pandera as pa
+import geopandas as gpd
+import pandas as pd
+import pandera.pandas as pa
 from pandera.typing import Series
-from pandera.typing.geopandas import GeoSeries
+from pydantic import BaseModel, ConfigDict
 
 
-class Project(pa.DataFrameModel):
+class ProjectSchema(pa.DataFrameModel):
     project_uid: Series[str] = pa.Field(
         # primary_key=True,
         unique=True,
     )
-    crs_wkt: Series[str] = pa.Field(description="Coordinate Reference System")
-    # datum: Series[str] = pa.Field(description="Datum used for measurement of the ground level elevation.")
+    horizontal_crs: Series[str] = pa.Field(
+        description="Horizontal Coordinate Reference System (CRS)."
+    )
+    horizontal_crs_wkt: Series[str] = pa.Field(
+        description="Horizontal CRS in Well-known Text (WKT) format."
+    )
+    vertical_crs: Series[str] = pa.Field(
+        description="Vertical Coordinate Reference System (CRS)."
+    )
+    vertical_crs_wkt: Series[str] = pa.Field(
+        description="Vertical CRS in Well-known Text (WKT) format."
+    )
 
 
-class BaseLocation(pa.DataFrameModel):
+class LocationSchema(pa.DataFrameModel):
     location_uid: Series[str] = pa.Field(
         # primary_key=True,
         unique=True,
@@ -25,39 +37,122 @@ class BaseLocation(pa.DataFrameModel):
         # foreign_key="project.project_uid"
     )
     location_source_id: Series[str]
-    location_type: Series[str]
     easting: Series[float] = pa.Field(coerce=True)
     northing: Series[float] = pa.Field(coerce=True)
     ground_level_elevation: Series[float] = pa.Field(
         coerce=True,
         description="Elevation w.r.t. a local datum. Usually the orthometric height from the geoid, i.e. mean sea level, to the ground level.",
     )
-    depth_to_base: Series[float]
+    depth_to_base: Series[float] = pa.Field(coerce=True, gt=0)
 
 
-class Location(BaseLocation):
-    elevation_at_base: Series[float]
+class LonLatHeightSchema(pa.DataFrameModel):
+    project_uid: Series[str] = pa.Field(
+        # foreign_key="project.project_uid"
+    )
+    location_uid: Series[str] = pa.Field(
+        # foreign_key="location.location_uid",
+        unique=True,
+    )
     longitude: Series[float]
     latitude: Series[float]
-    wgs84_ground_level_height: Series[float] = pa.Field(
-        description="Ground level height w.r.t. the WGS84 (World Geodetic System 1984) ellipsoid.",
+    egm2008_ground_level_height: Series[float] = pa.Field(
+        description="Ground level orthometric height w.r.t. the EGM2008 (Earth Gravitational Model 2008).",
         nullable=True,
     )
-    geometry: GeoSeries
 
 
-class BaseInSitu(pa.DataFrameModel):
+class InSituTestSchema(pa.DataFrameModel):
     project_uid: Series[str] = pa.Field(
         # foreign_key="project.project_uid"
     )
     location_uid: Series[str] = pa.Field(
         # foreign_key="location.location_uid"
     )
-    depth_to_top: Series[float] = pa.Field(coerce=True)
-    depth_to_base: Optional[Series[float]] = pa.Field(coerce=True, nullable=True)
-
-
-class BaseSample(BaseInSitu):
+    depth_to_top: Optional[Series[float]] = pa.Field(nullable=True, coerce=True, ge=0)
+    depth_to_base: Optional[Series[float]] = pa.Field(nullable=True, coerce=True, gt=0)
+
+    # https://pandera.readthedocs.io/en/stable/dataframe_models.html#dataframe-checks
+    # Check depth column completeness such that either shapely.Point's or
+    # shapely.LineString's can be created.
+    @pa.dataframe_check
+    def depth_column_completeness(cls, df: pd.DataFrame) -> pd.Series:
+        has_top = "depth_to_top" in df.columns
+        has_base = "depth_to_base" in df.columns
+
+        # If neither column exists, this check should fail
+        if not has_top and not has_base:
+            return pd.Series([False] * len(df), index=df.index)
+
+        # If only one column exists, check that it's all non-null
+        if has_top and not has_base:
+            return df["depth_to_top"].notna()
+        if has_base and not has_top:
+            return df["depth_to_base"].notna()
+
+        # If both columns exist:
+        #   Either depth_to_top or depth_to_base must be non-null => Point
+        #   OR
+        #   Both depth_to_top and depth_to_base must be non-null => LineString
+        # ! Commented out, because some In-Situ tests have a mix of
+        # ! Point's and LineString's, such as IPRM
+        # top_has_value = df["depth_to_top"].notna()
+        # base_has_value = df["depth_to_base"].notna()
+        # either_has_value = top_has_value ^ base_has_value
+        # both_have_values = top_has_value & base_has_value
+
+        # if either_has_value.all():
+        #     return either_has_value
+        # elif both_have_values.all():
+        #     return both_have_values
+        # else:
+        #     if either_has_value.sum() < both_have_values.sum():
+        #         return either_has_value
+        #     else:
+        #         return both_have_values
+
+        # ! Incorrect check
+        # If both columns exist, at least one must be non-null
+        return ~(df["depth_to_top"].isna() & df["depth_to_base"].isna())
+
+    @pa.dataframe_check
+    def top_above_base(cls, df: pd.DataFrame) -> pd.Series:
+        """Check that depth_to_top <= depth_to_base when both columns are present.
+
+        If either column is missing, this check passes (nothing to compare).
+        If both columns are present, the check fails if any row has
+        depth_to_top > depth_to_base.
+
+        Returns:
+            pd.Series: pandas.Series of bools indicating successful checks.
+        """
+        has_top = "depth_to_top" in df.columns
+        has_base = "depth_to_base" in df.columns
+
+        # If either column is missing, this check passes (nothing to compare)
+        if not has_top or not has_base:
+            return pd.Series([True] * len(df), index=df.index)
+
+        # Only compare when both values are non-null
+        mask = df["depth_to_top"].notna() & df["depth_to_base"].notna()
+        # Use where() to conditionally apply the comparison
+        result = (~mask) | (df["depth_to_top"] <= df["depth_to_base"])
+
+        # Debug: Show failing cases
+        failing_mask = mask & ~result
+        if failing_mask.any():
+            print("🚨 ERROR: depth_to_top > depth_to_base:")
+            print(
+                df.loc[
+                    failing_mask,
+                    ["location_uid", "depth_to_top", "depth_to_base", df.columns[5]],
+                ]
+            )
+
+        return result
+
+
+class SampleSchema(InSituTestSchema):
     sample_uid: Series[str] = pa.Field(
         # primary_key=True,
         unique=True,
@@ -65,19 +160,7 @@ class BaseSample(BaseInSitu):
     sample_source_id: Series[str]
 
 
-class Sample(BaseSample):
-    elevation_at_top: Series[float]
-    elevation_at_base: Optional[Series[float]] = pa.Field(nullable=True)
-    geometry: GeoSeries
-
-
-class InSitu(BaseInSitu):
-    elevation_at_top: Series[float]
-    elevation_at_base: Optional[Series[float]] = pa.Field(nullable=True)
-    geometry: GeoSeries
-
-
-class BaseLab(pa.DataFrameModel):
+class LabTestSchema(pa.DataFrameModel):
     project_uid: Series[str] = pa.Field(
         # foreign_key="project.project_uid"
     )
@@ -89,7 +172,24 @@ class BaseLab(pa.DataFrameModel):
     )
 
 
-class Lab(BaseLab):
-    geometry: GeoSeries = pa.Field(
-        description="GIS geometry of the sample on which this lab test was performed."
-    )
+class BedrockGIDatabase(BaseModel):
+    Project: pd.DataFrame
+    Location: pd.DataFrame
+    InSituTests: dict[str, pd.DataFrame]
+    Sample: pd.DataFrame | None = None
+    LabTests: dict[str, pd.DataFrame] = {}
+    Other: dict[str, pd.DataFrame] = {}
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+class BedrockGIGeospatialDatabase(BaseModel):
+    Project: pd.DataFrame
+    Location: gpd.GeoDataFrame
+    LonLatHeight: gpd.GeoDataFrame
+    InSituTests: dict[str, gpd.GeoDataFrame]
+    Sample: gpd.GeoDataFrame | None = None
+    LabTests: dict[str, pd.DataFrame] = {}
+    Other: dict[str, pd.DataFrame] = {}
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)