bedrock-ge 0.2.3__py3-none-any.whl → 0.3.0__py3-none-any.whl

bedrock_ge/__init__.py

@@ -1,3 +1,3 @@
 """Bedrock, the open source foundation for ground engineering."""
 
-__version__ = "0.2.3"
+__version__ = "0.3.0"

bedrock_ge/gi/ags.py

@@ -0,0 +1,103 @@
+from pathlib import Path
+from typing import IO
+
+from pyproj import CRS
+
+from bedrock_ge.gi.ags3 import ags3_to_brgi_db_mapping
+from bedrock_ge.gi.io_utils import detect_encoding, open_text_data_source
+from bedrock_ge.gi.mapping_models import BedrockGIMapping
+
+
+def ags_to_brgi_db_mapping(
+    source: str | Path | IO[str] | IO[bytes] | bytes,
+    projected_crs: CRS,
+    vertical_crs: CRS = CRS(3855),
+    encoding: str | None = None,
+) -> BedrockGIMapping:
+    """Map AGS 3 or AGS 4 data to the Bedrock GI data model.
+
+    Args:
+        source (str | Path | IO[str] | IO[bytes] | bytes): The AGS file (str or Path)
+            or a file-like object that represents the AGS file.
+        projected_crs (CRS): Projected Coordinate Reference System (CRS). For example:
+            - OSGB36 / British National Grid: `pyproj.CRS("EPSG:27700")`
+            - Hong Kong 1980 Grid System: `pyproj.CRS("EPSG:2326")`
+        vertical_crs (CRS, optional): Vertical CRS. Defaults to EGM2008 height, EPSG:3855
+            which measures the orthometric height w.r.t. the Earth Gravitational Model 2008.
+            - Ordnance Datum Newlyn (ODN) Height: `pyproj.CRS("EPSG:5701")`
+            - Hong Kong Principle Datum (HKPD) Height: `pyproj.CRS("EPSG:5738")`
+        encoding (str | None, optional): Encoding of the text file or bytes stream.
+            Defaults to None. An attempt at detecting the encoding will be made if None.
+
+    Raises:
+        ValueError: If the data does not match AGS 3 or AGS 4 format.
+
+    Returns:
+        BedrockGIDatabaseMapping: Object that maps AGS 3 or AGS 4 data to Bedrock GI data model.
+    """
+    if not encoding:
+        encoding = detect_encoding(source)
+
+    # Get first non-blank line, None if all lines are blank
+    with open_text_data_source(source, encoding=encoding) as f:
+        first_line = next((line.strip() for line in f if line.strip()), None)
+
+    if first_line:
+        if first_line.startswith('"**'):
+            ags_version = 3
+            brgi_db_mapping = ags3_to_brgi_db_mapping(
+                source, projected_crs, vertical_crs, encoding
+            )
+        elif first_line.startswith('"GROUP"'):
+            ags_version = 4
+            # brgi_db_mapping = ags4_to_brgi_db_mapping(
+            #     source, projected_crs, vertical_crs, encoding
+            # )
+        else:
+            # If first non-empty line doesn't match AGS 3 or AGS 4 format
+            raise ValueError("The data provided is not valid AGS 3 or AGS 4 data.")
+    else:
+        raise ValueError("The file provided has only blank lines")
+
+    # Put CPT data into brgi_db.Other table, because CPT data has too many rows
+    # that would generate geospatial geometry.
+    # "STCN" and "SCPT" are the group names for CPT data in AGS 3 and AGS 3 respectively.
+    # TODO: implement a warning when interpolating GI geospatial geometry when
+    # TODO: a single GI location has waaay too many rows in a certain In-Situ test,
+    # TODO: rather than removing specific groups here.
+    insitu_test_names = {
+        insitu_test.table_name: i
+        for i, insitu_test in enumerate(brgi_db_mapping.InSitu)
+    }
+    cpt_key = (
+        "STCN"
+        if "STCN" in insitu_test_names
+        else "SCPT"
+        if "SCPT" in insitu_test_names
+        else None
+    )
+    if cpt_key is not None:
+        cpt_data_mapping = brgi_db_mapping.InSitu.pop(insitu_test_names[cpt_key])
+        del insitu_test_names[cpt_key]
+        brgi_db_mapping.Other.append(cpt_data_mapping)
+
+    # Log information about the mapped AGS 3 or AGS 4 data
+    project_id = brgi_db_mapping.Project.project_id
+    n_gi_locations = len(brgi_db_mapping.Location.data)
+    n_samples = len(brgi_db_mapping.Sample.data) if brgi_db_mapping.Sample else 0
+    print_args = [
+        f"AGS {ags_version} data was read for Project {project_id}",
+        f"This GI data contains {n_gi_locations} GI locations, {n_samples} samples and:",
+        f"  - In-Situ Tests: {list(insitu_test_names.keys())}",
+    ]
+    if brgi_db_mapping.Lab:
+        print_args.append(
+            f"  - Lab Tests: {[lab_test.table_name for lab_test in brgi_db_mapping.Lab]}"
+        )
+    if brgi_db_mapping.Other:
+        print_args.append(
+            f"  - Other Tables: {[other_table.table_name for other_table in brgi_db_mapping.Other]}"
+        )
+    print(*print_args, sep="\n", end="\n\n")
+
+    return brgi_db_mapping

bedrock_ge/gi/ags3.py

@@ -0,0 +1,275 @@
+from pathlib import Path
+from typing import IO, Any
+
+import pandas as pd
+from pyproj import CRS
+
+from bedrock_ge.gi.ags_schemas import Ags3HOLE, Ags3SAMP, check_ags_proj_group
+from bedrock_ge.gi.io_utils import coerce_string, open_text_data_source
+from bedrock_ge.gi.mapping_models import (
+    BedrockGIMapping,
+    InSituTestTableMapping,
+    LabTestTableMapping,
+    LocationTableMapping,
+    OtherTable,
+    ProjectTableMapping,
+    SampleTableMapping,
+)
+
+
+def ags3_to_dfs(
+    source: str | Path | IO[str] | IO[bytes] | bytes, encoding: str
+) -> dict[str, pd.DataFrame]:
+    """Converts AGS 3 data to a dictionary of pandas DataFrames.
+
+    Also strips '?' from non-standard AGS 3 group and header names, in order to
+    make the rest of the code more generic.
+
+    Args:
+        source (str | Path | IO[str] | IO[bytes] | bytes): The AGS 3 file (str or Path)
+            or a file-like object that represents the AGS 3 file.
+        encoding (str):  Encoding of file or object.
+
+    Returns:
+        dict[str, pd.DataFrame]: A dictionary of pandas DataFrames, i.e. a database,
+            where each key is an AGS 3 group, and the corresponding value is
+            a pandas DataFrame containing the data for that group.
+    """
+    # Initialize dictionary and variables used in the AGS 3 read loop
+    ags3_dfs = {}
+    line_type = "line_0"
+    group = ""
+    headers: list[str] = ["", "", ""]
+    group_data: list[list[Any]] = [[], [], []]
+
+    with open_text_data_source(source, encoding=encoding) as file:
+        for i, line in enumerate(file):
+            line = line.strip()
+            last_line_type = line_type
+
+            # In AGS 3.1 group names are prefixed with **
+            if line.startswith('"**'):
+                line_type = "group_name"
+                if group:
+                    ags3_dfs[group] = pd.DataFrame(group_data, columns=headers)
+
+                group = line.strip(' ,"*?')
+                group_data = []
+
+            # In AGS 3 header names are prefixed with "*
+            elif line.startswith('"*'):
+                line_type = "headers"
+                new_headers = line.split('","')
+                new_headers = [h.strip(' ,"*?') for h in new_headers]
+
+                # Some groups have so many headers that they span multiple lines.
+                # Therefore we need to check whether the new headers are
+                # a continuation of the previous headers from the last line.
+                if line_type == last_line_type:
+                    headers = headers + new_headers
+                else:
+                    headers = new_headers
+
+            # Skip lines where group units are defined, these are defined in the AGS 3 data dictionary.
+            elif line.startswith('"<UNITS>"'):
+                line_type = "units"
+                continue
+
+            # The rest of the lines contain:
+            # 1. GI data
+            # 2. a continuation of the previous line. These lines contain "<CONT>" in the first column.
+            # 3. are empty or contain worthless data
+            else:
+                line_type = "data_row"
+                data_row = line.split('","')
+                if len("".join(data_row)) == 0:
+                    # print(f"Line {i} is empty. Last Group: {group}")
+                    continue
+                elif len(data_row) != len(headers):
+                    print(
+                        f"\n🚨 CAUTION: The number of columns ({len(data_row)}) on line {i + 1} doesn't match the number of columns ({len(headers)}) of group {group}!",
+                        f"{group} headers: {headers}",
+                        f"Line {i + 1}:      {data_row}",
+                        sep="\n",
+                        end="\n\n",
+                    )
+                    continue
+                # Append continued lines (<CONT>) to the last data_row
+                elif data_row[0] == '"<CONT>':
+                    last_data_row = group_data[-1]
+                    for j, data in enumerate(data_row):
+                        data = data.strip(' "')
+                        if data and data != "<CONT>":
+                            if last_data_row[j] is None:
+                                # Last data row didn't contain data for this column
+                                last_data_row[j] = coerce_string(data)
+                            else:
+                                # Last data row already contains data for this column
+                                last_data_row[j] = str(last_data_row[j]) + data
+                # Lines that are assumed to contain valid data are added to the group data
+                else:
+                    cleaned_data_row = []
+                    for data in data_row:
+                        cleaned_data_row.append(coerce_string(data.strip(' "')))
+                    group_data.append(cleaned_data_row)
+
+    # Also add the last group's df to the dictionary of AGS dfs
+    ags3_dfs[group] = pd.DataFrame(group_data, columns=headers)
+
+    if not group:
+        print(
+            '🚨 ERROR: The provided AGS 3 data does not contain any groups, i.e. lines starting with "**'
+        )
+
+    return ags3_dfs
+
+
+# TODO: AGS 3 table validation based on the AGS 3 data dictionary.
+def ags3_to_brgi_db_mapping(
+    source: str | Path | IO[str] | IO[bytes] | bytes,
+    projected_crs: CRS,
+    vertical_crs: CRS,
+    encoding: str,
+) -> BedrockGIMapping:
+    """Map AGS 3 data to the Bedrock GI data model.
+
+    Args:
+        ags3_db (dict[str, pd.DataFrame]): A dictionary of pandas DataFrames, i.e. database,
+            where each key is an AGS 3 group, and the corresponding value is
+            a pandas DataFrame containing the data for that group.
+        projected_crs (CRS): Projected coordinate reference system (CRS).
+        vertical_crs (CRS, optional): Vertical CRS. Defaults to EGM2008 height, EPSG:3855
+            which measures the orthometric height w.r.t. the Earth Gravitational Model 2008.
+        encoding (str): Encoding of the text file or bytes stream.
+
+    Returns:
+        BedrockGIDatabaseMapping: Object that maps AGS 3 data to Bedrock GI data model.
+    """
+    ags3_dfs = ags3_to_dfs(source, encoding)
+
+    check_ags_proj_group(ags3_dfs["PROJ"])
+    ags3_project = ProjectTableMapping(
+        data=ags3_dfs["PROJ"].to_dict(orient="records")[0],
+        project_id=ags3_dfs["PROJ"].at[0, "PROJ_ID"],
+        horizontal_crs=projected_crs,
+        vertical_crs=vertical_crs,
+    )
+    del ags3_dfs["PROJ"]
+
+    Ags3HOLE.validate(ags3_dfs["HOLE"])
+    ags3_location = LocationTableMapping(
+        data=ags3_dfs["HOLE"],
+        location_id_column="HOLE_ID",
+        easting_column="HOLE_NATE",
+        northing_column="HOLE_NATN",
+        ground_level_elevation_column="HOLE_GL",
+        depth_to_base_column="HOLE_FDEP",
+    )
+    del ags3_dfs["HOLE"]
+
+    if "SAMP" in ags3_dfs.keys():
+        Ags3SAMP.validate(ags3_dfs["SAMP"])
+        samp_df = ags3_dfs["SAMP"]
+        samp_df = _add_sample_source_id(samp_df)
+        ags3_sample = SampleTableMapping(
+            data=samp_df,
+            location_id_column="HOLE_ID",
+            sample_id_column="sample_source_id",
+            depth_to_top_column="SAMP_TOP",
+        )
+        del ags3_dfs["SAMP"]
+    else:
+        ags3_sample = None
+
+    ags3_lab_tests = []
+    ags3_insitu_tests = []
+    ags3_other_tables = []
+
+    for group, df in ags3_dfs.items():
+        # Non-standard group names contain the "?" prefix.
+        # => checking that "SAMP_TOP" / "HOLE_ID" is in the columns is too restrictive.
+        if "SAMP_TOP" in df.columns:
+            df = _add_sample_source_id(df)
+            ags3_lab_tests.append(
+                LabTestTableMapping(
+                    table_name=group,
+                    data=df,
+                    location_id_column="HOLE_ID",
+                    sample_id_column="sample_source_id",
+                )
+            )
+        elif "HOLE_ID" in df.columns:
+            top_depth, base_depth = _get_depth_columns(group, list(df.columns))
+            ags3_insitu_tests.append(
+                InSituTestTableMapping(
+                    table_name=group,
+                    data=df,
+                    location_id_column="HOLE_ID",
+                    depth_to_top_column=top_depth,
+                    depth_to_base_column=base_depth,
+                )
+            )
+        else:
+            ags3_other_tables.append(OtherTable(table_name=group, data=df))
+
+    brgi_db_mapping = BedrockGIMapping(
+        Project=ags3_project,
+        Location=ags3_location,
+        InSitu=ags3_insitu_tests,
+        Sample=ags3_sample,
+        Lab=ags3_lab_tests,
+        Other=ags3_other_tables,
+    )
+    return brgi_db_mapping
+
+
+def _add_sample_source_id(df: pd.DataFrame) -> pd.DataFrame:
+    df["sample_source_id"] = (
+        df["SAMP_REF"].astype(str)
+        + "-"
+        + df["SAMP_TYPE"].astype(str)
+        + "-"
+        + df["SAMP_TOP"].astype(str)
+        + "-"
+        + df["HOLE_ID"].astype(str)
+    )
+    return df
+
+
+def _get_depth_columns(group: str, headers: list[str]) -> tuple[str | None, str | None]:
+    top_depth: str | None = f"{group}_TOP"
+    base_depth: str | None = f"{group}_BASE"
+
+    match group:
+        case "CDIA":
+            top_depth = "CDIA_CDEP"
+        case "FLSH":
+            top_depth = "FLSH_FROM"
+            base_depth = "FLSH_TO"
+        case "CORE":
+            base_depth = "CORE_BOT"
+        case "HDIA":
+            top_depth = "HDIA_HDEP"
+        case "PTIM":
+            top_depth = "PTIM_DEP"
+        case "IVAN":
+            top_depth = "IVAN_DPTH"
+        case "STCN":
+            top_depth = "STCN_DPTH"
+        case "POBS" | "PREF":
+            top_depth = "PREF_TDEP"
+        case "DREM":
+            top_depth = "DREM_DPTH"
+        case "PRTD" | "PRTG" | "PRTL":
+            top_depth = "PRTD_DPTH"
+
+    if top_depth not in headers:
+        top_depth = None
+    if base_depth not in headers:
+        base_depth = None
+    if not top_depth and not base_depth:
+        raise ValueError(
+            'The in-situ test group "{group}" group in this AGS 3 file does not contain a top or base depth heading!'
+        )
+
+    return top_depth, base_depth

bedrock_ge/gi/ags4.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+from typing import IO
+
+import pandas as pd
+from python_ags4 import AGS4
+
+
+def ags4_to_dfs(
+    source: str | Path | IO[str] | IO[bytes] | bytes,
+) -> dict[str, pd.DataFrame]:
+    """Converts AGS 4 data to a dictionary of pandas DataFrames.
+
+    Args:
+        source (str | Path | IO[str] | IO[bytes] | bytes): The AGS4 file (str or Path) or a file-like
+            object that represents and AGS4 file.
+
+    Returns:
+        dict[str, pd.DataFrame]: A dictionary of pandas DataFrames, where each key
+            represents a group name from AGS 4 data, and the corresponding value is a
+            pandas DataFrame containing the data for that group.
+    """
+    ags4_tups = AGS4.AGS4_to_dataframe(source)
+
+    ags4_dfs = {}
+    for group, df in ags4_tups[0].items():
+        df = df.loc[2:].drop(columns=["HEADING"]).reset_index(drop=True)
+        ags4_dfs[group] = df
+
+    return ags4_dfs

bedrock_ge/gi/{ags/schemas.py → ags_schemas.py}

@@ -1,7 +1,35 @@
-import pandera as pa
+import pandas as pd
+import pandera.pandas as pa
 from pandera.typing import Series
 
 
+def check_ags_proj_group(ags_proj: pd.DataFrame) -> bool:
+    """Checks if the AGS 3 or AGS 4 PROJ group is correct.
+
+    Args:
+        ags_proj (pd.DataFrame): The DataFrame with the PROJ group.
+
+    Raises:
+        ValueError: If AGS 3 of AGS 4 PROJ group is not correct.
+
+    Returns:
+        bool: Returns True if the AGS 3 or AGS 4 PROJ group is correct.
+    """
+    if len(ags_proj) != 1:
+        raise ValueError("The PROJ group must contain exactly one row.")
+
+    msg = 'The project ID ("PROJ_ID" in the "PROJ" group) is missing from the AGS data.'
+    try:
+        project_id = ags_proj.at[ags_proj.index[0], "PROJ_ID"]
+    except KeyError:
+        raise ValueError(msg)
+
+    if pd.isna(project_id) or str(project_id).strip() == "":
+        raise ValueError(msg)
+
+    return True
+
+
 class Ags3HOLE(pa.DataFrameModel):
     HOLE_ID: Series[str] = pa.Field(
         # primary_key=True,
@@ -56,13 +84,6 @@ class BaseSAMP(pa.DataFrameModel):
 
 
 class Ags3SAMP(BaseSAMP):
-    sample_id: Series[str] = pa.Field(
-        # primary_key=True,
-        unique=True,
-        coerce=True,
-        description="Sample unique identifier",
-        # example="REF_TYPE_TOP_HOLE_ID",
-    )
     HOLE_ID: Series[str] = pa.Field(
         # foreign_key="Ags3HOLE.HOLE_ID",
         description="Exploratory hole or location equivalent",

bedrock_ge/gi/db_operations.py

@@ -0,0 +1,128 @@
+from collections.abc import Iterable
+
+import pandas as pd
+
+from bedrock_ge.gi.io_utils import convert_object_col_content_to_string
+from bedrock_ge.gi.schemas import (
+    BedrockGIDatabase,
+    InSituTestSchema,
+    LabTestSchema,
+    LocationSchema,
+    ProjectSchema,
+    SampleSchema,
+)
+from bedrock_ge.gi.validate import check_foreign_key
+
+
+def merge_dbs(
+    brgi_dbs: Iterable[BedrockGIDatabase],
+) -> BedrockGIDatabase:
+    """Merges the incoming Bedrock GI database into the target Bedrock GI database.
+
+    The function concatenates the pandas DataFrames of the second dict of
+    DataFrames to the first dict of DataFrames for the keys they have in common.
+    Keys that are unique to either dictionary will be included in the final
+    concatenated dictionary.
+
+    Args:
+        brgi_dbs: The Bedrock GI databases containing the data to be merged.
+
+    Returns:
+        BedrockGIDatabase: Merged Bedrock GI database.
+    """
+    dbs = list(brgi_dbs)
+
+    if not dbs:
+        raise ValueError("Cannot merge an empty list of Bedrock GI databases.")
+    elif len(dbs) == 1 and isinstance(dbs[0], BedrockGIDatabase):
+        return dbs[0]
+
+    project_dataframes = _filter_dataframes([db.Project for db in dbs])
+    merged_project = pd.concat(project_dataframes, ignore_index=True)
+    merged_project = merged_project.drop_duplicates().reset_index(drop=True)
+    merged_project = convert_object_col_content_to_string(merged_project)
+    ProjectSchema.validate(merged_project)
+
+    location_dataframes = _filter_dataframes([db.Location for db in dbs])
+    merged_location = pd.concat(location_dataframes, ignore_index=True)
+    merged_location = merged_location.drop_duplicates().reset_index(drop=True)
+    merged_location = convert_object_col_content_to_string(merged_location)
+    LocationSchema.validate(merged_location)
+    check_foreign_key("project_uid", merged_project, merged_location)
+
+    insitu_tables: set[str] = set()
+    lab_tables: set[str] = set()
+    other_tables: set[str] = set()
+    for db in dbs:
+        insitu_tables.update(db.InSituTests.keys())
+        if db.LabTests:
+            lab_tables.update(db.LabTests.keys())
+        if db.Other:
+            other_tables.update(db.Other.keys())
+
+    merged_insitu: dict[str, pd.DataFrame] = {}
+    for table_name in insitu_tables:
+        insitu_dataframes = _filter_dataframes(
+            [db.InSituTests.get(table_name) for db in dbs]
+        )
+        insitu_df = pd.concat(insitu_dataframes, ignore_index=True)
+        insitu_df = insitu_df.drop_duplicates().reset_index(drop=True)
+        insitu_df = convert_object_col_content_to_string(insitu_df)
+        InSituTestSchema.validate(insitu_df)
+        check_foreign_key("project_uid", merged_project, insitu_df)
+        check_foreign_key("location_uid", merged_location, insitu_df)
+        merged_insitu[table_name] = insitu_df
+
+    sample_dfs = _filter_dataframes([db.Sample for db in dbs])
+    merged_sample = None
+    if sample_dfs:
+        merged_sample = pd.concat(sample_dfs, ignore_index=True)
+        merged_sample = merged_sample.drop_duplicates().reset_index(drop=True)
+        merged_sample = convert_object_col_content_to_string(merged_sample)
+        SampleSchema.validate(merged_sample)
+        check_foreign_key("project_uid", merged_project, merged_sample)
+
+    merged_lab: dict[str, pd.DataFrame] = {}
+    for table_name in lab_tables:
+        lab_dataframes = _filter_dataframes([db.LabTests.get(table_name) for db in dbs])
+        lab_df = pd.concat(lab_dataframes, ignore_index=True)
+        lab_df = lab_df.drop_duplicates().reset_index(drop=True)
+        lab_df = convert_object_col_content_to_string(lab_df)
+        LabTestSchema.validate(lab_df)
+        check_foreign_key("project_uid", merged_project, lab_df)
+        check_foreign_key("sample_uid", merged_sample, lab_df)
+        merged_lab[table_name] = lab_df
+
+    merged_other: dict[str, pd.DataFrame] = {}
+    for table_name in other_tables:
+        other_dataframes = _filter_dataframes([db.Other.get(table_name) for db in dbs])
+        other_df = pd.concat(other_dataframes, ignore_index=True)
+        other_df = other_df.drop_duplicates().reset_index(drop=True)
+        other_df = convert_object_col_content_to_string(other_df)
+        check_foreign_key("project_uid", merged_project, other_df)
+        merged_other[table_name] = other_df
+
+    return BedrockGIDatabase(
+        Project=merged_project,
+        Location=merged_location,
+        InSituTests=merged_insitu,
+        Sample=merged_sample,
+        LabTests=merged_lab,
+        Other=merged_other,
+    )
+
+
+def _filter_dataframes(dataframes: list[pd.DataFrame | None]) -> list[pd.DataFrame]:
+    """Filter out empty or all-NA DataFrames to avoid FutureWarnings."""
+    valid_dfs = []
+    for df in dataframes:
+        if df is not None and not df.empty and not df.isna().all().all():
+            if df.columns.duplicated().any():
+                raise ValueError(
+                    f"Duplicate column names found in dataframe:\n{list(df.columns)}"
+                )
+
+            df.dropna(axis=1, how="all", inplace=True)
+
+            valid_dfs.append(df)
+    return valid_dfs