EOValidator 0.1__tar.gz

eovalidator-0.1/EOValidator.egg-info/PKG-INFO

@@ -0,0 +1,3 @@
+Metadata-Version: 2.4
+Name: EOValidator
+Version: 0.1

eovalidator-0.1/EOValidator.egg-info/SOURCES.txt

@@ -0,0 +1,93 @@
+pyproject.toml
+EOValidator.egg-info/PKG-INFO
+EOValidator.egg-info/SOURCES.txt
+EOValidator.egg-info/dependency_links.txt
+EOValidator.egg-info/top_level.txt
+eo_validator_main/__init__.py
+eo_validator_main/setup.py
+eo_validator_main/eo_validator_old_package/__init__.py
+eo_validator_main/eo_validator_old_package/__main__.py
+eo_validator_main/eo_validator_old_package/main.py
+eo_validator_main/eo_validator_old_package/validate_netcdf.py
+eo_validator_main/eo_validator_old_package/validate_shapefile.py
+eo_validator_main/eo_validator_old_package/validator.py
+eo_validator_main/excluded/__init__.py
+eo_validator_main/excluded/base_validator.py
+eo_validator_main/excluded/csv_validator.py
+eo_validator_main/excluded/failed_validator_wrapper.py
+eo_validator_main/excluded/file.py
+eo_validator_main/excluded/geodatabase_validator.py
+eo_validator_main/excluded/move_2_loc_validator.py
+eo_validator_main/excluded/validate_geojson.py
+eo_validator_main/excluded/validate_geopackage.py
+eo_validator_main/excluded/errors/__init__.py
+eo_validator_main/excluded/errors/file.py
+eo_validator_main/excluded/errors/unsupported_file_type.py
+eo_validator_main/excluded/errors/validation.py
+eo_validator_main/src/__init__.py
+eo_validator_main/src/eo_validator_refactored/__init__.py
+eo_validator_main/src/eo_validator_refactored/api/__init__.py
+eo_validator_main/src/eo_validator_refactored/api/file_format_validator.py
+eo_validator_main/src/eo_validator_refactored/api/raster_file_validator.py
+eo_validator_main/src/eo_validator_refactored/api/validation_mode.py
+eo_validator_main/src/eo_validator_refactored/api/validation_request.py
+eo_validator_main/src/eo_validator_refactored/api/vector_file_validator.py
+eo_validator_main/src/eo_validator_refactored/core/__init__.py
+eo_validator_main/src/eo_validator_refactored/core/file_input.py
+eo_validator_main/src/eo_validator_refactored/core/format.py
+eo_validator_main/src/eo_validator_refactored/core/multi_file_input.py
+eo_validator_main/src/eo_validator_refactored/core/shape_type.py
+eo_validator_main/src/eo_validator_refactored/core/validator_registry.py
+eo_validator_main/src/eo_validator_refactored/integration/__init__.py
+eo_validator_main/src/eo_validator_refactored/integration/cwltool_facade.py
+eo_validator_main/src/eo_validator_refactored/integration/validation_manager.py
+eo_validator_main/src/eo_validator_refactored/integration/builder/__init__.py
+eo_validator_main/src/eo_validator_refactored/integration/builder/cli_request_builder.py
+eo_validator_main/src/eo_validator_refactored/integration/builder/cwl_utils_request_builder.py
+eo_validator_main/src/eo_validator_refactored/integration/builder/cwltool_request_builder.py
+eo_validator_main/src/eo_validator_refactored/integration/builder/http_request_builder.py
+eo_validator_main/src/eo_validator_refactored/integration/builder/validation_request_builder.py
+eo_validator_main/src/eo_validator_refactored/integration/handler/__init__.py
+eo_validator_main/src/eo_validator_refactored/integration/handler/cli_response_handler.py
+eo_validator_main/src/eo_validator_refactored/integration/handler/cwl_utils_response_handler.py
+eo_validator_main/src/eo_validator_refactored/integration/handler/cwltool_response_handler.py
+eo_validator_main/src/eo_validator_refactored/integration/handler/http_response_handler.py
+eo_validator_main/src/eo_validator_refactored/integration/handler/validation_response_handler.py
+eo_validator_main/src/eo_validator_refactored/main/__main__.py
+eo_validator_main/src/eo_validator_refactored/main/eo_validator.py
+eo_validator_main/src/eo_validator_refactored/main/main.py
+eo_validator_main/src/eo_validator_refactored/result/__init__.py
+eo_validator_main/src/eo_validator_refactored/result/issue_severity.py
+eo_validator_main/src/eo_validator_refactored/result/validation_issue.py
+eo_validator_main/src/eo_validator_refactored/result/validaton_result.py
+eo_validator_main/src/eo_validator_refactored/utils/__init__.py
+eo_validator_main/src/eo_validator_refactored/utils/crs_operations.py
+eo_validator_main/src/eo_validator_refactored/utils/loader.py
+eo_validator_main/src/eo_validator_refactored/utils/open_operations.py
+eo_validator_main/src/eo_validator_refactored/utils/shape_operations.py
+eo_validator_main/src/eo_validator_refactored/utils/structural_checks.py
+eo_validator_main/src/eo_validator_refactored/utils/temp_path.py
+eo_validator_main/src/eo_validator_refactored/validators/__init__.py
+eo_validator_main/src/eo_validator_refactored/validators/cog_validator.py
+eo_validator_main/src/eo_validator_refactored/validators/dem_validator.py
+eo_validator_main/src/eo_validator_refactored/validators/dsm_validator.py
+eo_validator_main/src/eo_validator_refactored/validators/geojson_validator.py
+eo_validator_main/src/eo_validator_refactored/validators/geopackage_validator.py
+eo_validator_main/src/eo_validator_refactored/validators/geoparquet_validator.py
+eo_validator_main/src/eo_validator_refactored/validators/geotiff_validator.py
+eo_validator_main/src/eo_validator_refactored/validators/grib_validator.py
+eo_validator_main/src/eo_validator_refactored/validators/kml_validator.py
+eo_validator_main/src/eo_validator_refactored/validators/shapefile_validator.py
+eo_validator_main/src/eo_validator_refactored/validators/wkt_validator.py
+eo_validator_main/src/eo_validator_refactored/validators/xdmf_validator.py
+eo_validator_main/src/eo_validator_refactored/validators/TODO/__init__.py
+eo_validator_main/src/eo_validator_refactored/validators/TODO/geozarr_validator.py
+eo_validator_main/src/eo_validator_refactored/validators/TODO/gml_validator.py
+eo_validator_main/src/eo_validator_refactored/validators/TODO/gpx_validator.py
+eo_validator_main/src/eo_validator_refactored/validators/TODO/netcdf_validator.py
+eo_validator_main/src/eo_validator_refactored/validators/TODO/osm_json_validator.py
+eo_validator_main/src/eo_validator_refactored/validators/TODO/osm_xml_validator.py
+eo_validator_main/src/eo_validator_refactored/validators/TODO/pbf_validator.py
+eo_validator_main/src/eo_validator_refactored/validators/TODO/vrt_validator.py
+eo_validator_main/src/eo_validator_refactored/validators/TODO/wkb_validator.py
+eo_validator_main/tests/__init__.py

eovalidator-0.1/EOValidator.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

eovalidator-0.1/EOValidator.egg-info/top_level.txt

@@ -0,0 +1 @@
+eo_validator_main

eovalidator-0.1/PKG-INFO

@@ -0,0 +1,3 @@
+Metadata-Version: 2.4
+Name: EOValidator
+Version: 0.1

eovalidator-0.1/eo_validator_main/eo_validator_old_package/__init__.py

@@ -0,0 +1,4 @@
+"""EO file formats validator."""
+
+__author__ = "moritz.pataky@gmx.de"
+__version__ = "0.1.0"

eovalidator-0.1/eo_validator_main/eo_validator_old_package/__main__.py

@@ -0,0 +1,5 @@
+"""Default entrypoint for the cwltool module."""
+
+from . import main
+
+main.run()

eovalidator-0.1/eo_validator_main/eo_validator_old_package/main.py

@@ -0,0 +1,25 @@
+from eo_validator_main.eo_validator.validate_geotiff import validate_geotiff_strong
+from eo_validator_main.eo_validator.validate_geotiff import validate_geotiff_simple
+from pathlib import Path
+import json, sys
+#@build_sniff_from_prefix
+#class Tiff(Text):
+
+#@build_sniff_from_prefix
+#class GeoTiff(Tiff):
+#    file_ext = "tif"
+
+
+def run():
+    return None
+
+def test():
+    return None
+if __name__ == '__main__':
+    p = "C:/Tests/WSF2019_v1_8_48.tif"
+    r = validate_geotiff_strong(p, require_tags=False, require_crs=True)
+    print(json.dumps(r, indent=2))
+
+    #p2 = "C:/Tests/OntologyEx1.png"
+    #r = validate_geotiff_strong(p2, require_tags=False, require_crs=True)
+    #print(json.dumps(r, indent=2))

eovalidator-0.1/eo_validator_main/eo_validator_old_package/validate_netcdf.py

@@ -0,0 +1,345 @@
+from pathlib import Path
+from typing import Dict, Any
+
+try:
+    import netCDF4
+    NETCDF4_INSTALLED = True
+except Exception:
+    NETCDF4_INSTALLED = False
+
+try:
+    import xarray as xr
+    XARRAY_INSTALLED = True
+except Exception:
+    XARRAY_INSTALLED = False
+
+def validate_netcdf_simple(path, require_tags, require_crs):
+    path = Path(path)
+    #report = {"path": str(path), "exists": path.exists(), "warnings": [], "errors": []}
+    report: Dict[str, Any] = {
+        "path": path,
+        "exists": False,
+        "extension_ok": False,
+        "header": None,
+        "open_info": None,
+        "cf_checks": None,
+        "is_valid": False,
+        "errors": [],
+        "warnings": []
+    }
+    # simple validation using path and file extension
+    # check path
+    if not path.exists():
+        report["errors"].append("File not found")
+        return report
+    else:
+        report["exists"] = True
+    # check file extension
+    if path.suffix.lower() not in ('.cdf', ".nc4", '.nc'):
+        report["warnings"] = ["extension not .cdf/.nc4/.nc"]
+    else:
+        report["extension_ok"] = True
+
+    return report
+
+def sniff_header(path) -> Dict[str, Any]:
+    report = {"magic": "None", "is_netcdf3_magic": False, "is_hdf5_magic": False}
+    try:
+        with open(path, "rb") as fh:
+            header = fh.read(16)
+        report["magic"] = header[:16].hex()
+        if header[:3] == b"CDF":
+            report["is_netcdf3_magic"] = True
+        if header.startswith(b"\x89HDF\r\n\x1a\n"):
+            report["is_hdf5_magic"] = True
+    except Exception as e:
+        report["error"] = f"Header read error: {e}"
+    return report
+
+def open_with_netcdf4(path: str) -> Dict[str, Any]:
+    report: Dict[str, Any] = {
+        "opened": False,
+        "error": None,
+        "file_format": None,
+        "global_attributes": {},
+        "dimensions": {},
+        "variables": {}
+
+    }
+    try:
+        dataset = netCDF4.Dataset(path, mode="r")
+    except Exception as e:
+        report["error"] = f"netCDF4 open failed: {e}"
+        return report
+    report["opened"] = True
+    # global attributes
+    try:
+        for k in dataset.ncattrs():
+            try:
+                report["global_attributes"][k] = getattr(dataset, k)
+            except Exception:
+                report["global_attributes"][k] = None
+    except Exception:
+        pass
+    # dimensions
+    try:
+        for d_name, dim in dataset.dimensions.items():
+            report["dimensions"][d_name] = {"size": len(dim), "isunlimited": bool(dim.isunlimited())}
+    except Exception:
+        pass
+    # variables
+    try:
+        for v_name, var in dataset.variables.items():
+            try:
+                attributes = {k: getattr(var, k) for k in var.ncattrs()}
+            except Exception:
+                attributes = {}
+            report["variables"][v_name] = {
+                "dtype": str(var.dtype),
+                "dimensions": tuple(var.dimensions),
+                "shape": tuple(var.shape),
+                "attributes": attributes
+            }
+    except Exception:
+        pass
+    # file format hint
+    try:
+        report["file_format"] = getattr(dataset, "file_format", None)
+    except Exception:
+        report["file_format"] = None
+    try:
+        dataset.close()
+    except Exception:
+        pass
+    return report
+
+def open_with_xarray(path: str) -> Dict[str, Any]:
+    report: Dict[str, Any] = {
+        "opened": False,
+        "error": None,
+        "file_format": None,
+        "global_attributes": {},
+        "dimensions": {},
+        "variables": {}
+    }
+
+    try:
+        dataset = xr.open_dataset(path, decode_times=False, mask_and_scale=False, autoclose=True)
+    except Exception as e:
+        report["error"] = f"xarray open failed: {e}"
+        return report
+    report["opened"] = True
+    try:
+        for k, v in dataset.attrs.items():
+            report["global_attributes"][k] = v
+    except Exception:
+        pass
+    try:
+        for d_name, length in dataset.sizes.items():
+            report["dimensions"][d_name] = {"size": int(length), "isunlimited": False}
+    except Exception:
+        pass
+    try:
+        for v_name, var in dataset.data_vars.items():
+            report["variables"][v_name] = {
+                "dtype": str(var.dtype),
+                "dimensions": tuple(var.dims),
+                "shape": tuple(var.shape),
+                "attributes": dict(var.attrs)
+            }
+        # coordinate variables
+        for c_name, coord in dataset.coords.items():
+            if c_name not in report["variables"]:
+                report["variables"][c_name] = {
+                    "dtype": str(coord.dtype),
+                    "dimensions": tuple(coord.dims),
+                    "shape": tuple(coord.shape),
+                    "attributes": dict(coord.attrs)
+                }
+    except Exception:
+        pass
+    try:
+        dataset.close()
+    except Exception:
+        pass
+    return report
+
+def collect_netcdf_info(path) -> Dict[str, Any]:
+    info = {
+        "path": path,
+        "header": sniff_header(path),
+        "reader": None,
+        "file_format": None,
+        "global_attributes": {},
+        "dimensions": {},
+        "variables": {},
+        "errors": [],
+        "warnings": []
+    }
+
+    # try netCDF4
+    if NETCDF4_INSTALLED:
+        report = open_with_netcdf4(path)
+        if report.get("opened"):
+            info["reader"] = "netCDF4"
+            info["file_format"] = report.get("file_format")
+            info["global_attributes"] = report.get("global_attributes", {})
+            info["dimensions"] = report.get("dimensions", {})
+            info["variables"] = report.get("variables", {})
+            return info
+        else:
+            info["errors"].append(report.get("error") or "netCDF4 open failed")
+
+    # fallback xarray
+    if XARRAY_INSTALLED:
+        report = open_with_xarray(path)
+        if report.get("opened"):
+            info["reader"] = "xarray"
+            info["file_format"] = report.get("file_format")
+            info["global_attributes"] = report.get("global_attributes", {})
+            info["dimensions"] = report.get("dimensions", {})
+            info["variables"] = report.get("variables", {})
+            return info
+        else:
+            info["errors"].append(report.get("error") or "xarray open failed")
+    else:
+        info["errors"].append("No netCDF4 or xarray available to open file")
+    return info
+
+def cf_heuristic_checks(info: Dict[str, Any]) -> Dict[str, Any]:
+    vars = info.get("variables", {})
+    dims = info.get("dimensions", {})
+    global_attributes = info.get("global_attributes", {})
+    out = {"checks": {}, "warnings": [], "errors": []}
+
+    # Conventions attribute
+    conv = None
+    for k in ("Conventions", "convention", "conventions"):
+        if k in global_attributes:
+            conv = global_attributes[k]
+            break
+    if conv:
+        out["checks"]["Conventions_present"] = True
+        out["checks"]["Conventions_value"] = conv
+    else:
+        out["checks"]["Conventions_present"] = False
+        out["warnings"].append("Global attribute 'Conventions' missing (can't detect CF version).")
+
+    # coordinate variables: dims that lack a variable with the same name
+    missing_coord_vars = [d for d in dims.keys() if d not in vars]
+    if missing_coord_vars:
+        out["checks"]["missing_coord_vars"] = missing_coord_vars
+        out["warnings"].append(f"No coordinate variable(s) for dimension(s): {missing_coord_vars}")
+
+    # lat/lon/time candidates
+    lat_candidates = [n for n in ("lat", "latitude", "y") if n in vars]
+    lon_candidates = [n for n in ("lon", "longitude", "x") if n in vars]
+    time_candidates = [n for n in ("time", "times") if n in vars]
+
+    out["checks"]["lat_candidates"] = lat_candidates
+    out["checks"]["lon_candidates"] = lon_candidates
+    out["checks"]["time_candidates"] = time_candidates
+
+    if not lat_candidates or not lon_candidates:
+        out["warnings"].append("No obvious latitude/longitude coordinate variables found (lat/lon).")
+    else:
+        for name in lat_candidates + lon_candidates:
+            v = vars.get(name, {})
+            if len(v.get("dimensions", ())) != 1:
+                out["warnings"].append(f"Coordinate candidate '{name}' is not 1D (dims={v.get('dimensions')}).")
+            attributes = v.get("attributes", {})
+            if "units" not in attributes:
+                out["warnings"].append(f"Coordinate variable '{name}' missing 'units' attribute.")
+
+    # time checks
+    if not time_candidates:
+        out["warnings"].append("No obvious time variable found.")
+    else:
+        for t in time_candidates:
+            attributes = vars[t].get("attributes", {})
+            if "units" not in attributes:
+                out["warnings"].append(f"Time variable '{t}' missing 'units' attribute (CF expects units like 'days since ...').")
+
+    # variable dimension consistency
+    var_shape_issues = []
+    for v_name, vinfo in vars.items():
+        for dim in vinfo.get("dimensions", ()):
+            if dim not in dims:
+                var_shape_issues.append((v_name, dim))
+    if var_shape_issues:
+        out["errors"].append(f"Variables reference undefined dimensions: {var_shape_issues}")
+
+    # missing _FillValue / missing_value for data vars (warn)
+    suspect = []
+    for v_name, v_info in vars.items():
+        attrs = v_info.get("attributes", {})
+        # treat coordinate vars as less important (if var name == dim and 1D)
+        if v_name in dims and len(v_info.get("dimensions", ())) == 1:
+            continue
+        if not any(k in attrs for k in ("_FillValue", "missing_value")):
+            suspect.append(v_name)
+    if suspect:
+        out["warnings"].append(f"Data variables without _FillValue/missing_value: {suspect[:10]} (up to 10 shown)")
+
+    return out
+
+def validate_netcdf_strong(path, require_tags, require_crs):
+    #resultGeoPackage = {"path": str(path), "exists": path.exists(),
+    #          "extension_ok": False, "sqlite_header_ok": False, "sqlite_open_ok": False,
+    #          "integrity_ok": None, "required_tables_present": False, "missing_required_tables": [],
+    #          "recommended_tables_present": [], "missing_recommended_tables": [], "gpkg_contents": [],
+    #          "gpkg_spatial_ref_sys": [], "srs_reference_ok": None, "geometry_columns_ok": None,
+    #          "table_sample_checks": {}, "warnings": [], "errors": []}
+
+    #resultGeoTIFF = {"path": str(path), "exists": path.exists(), "is_tiff": False,
+    #          "driver": None, "crs": None, "transform": None,
+    #          "bands": None, "dtypes": None, "nodatavals": None,
+    #          "geo_tags": {}, "warnings": [], "errors": []}
+
+    path = Path(path)
+    #report = {"path": str(path), "exists": path.exists(), "extension_ok": False, "warnings": [], "errors": []}
+
+    # simple validation using path and file extension
+    # check path
+    #if not path.exists():
+    #    report["errors"].append("File not found")
+    #    return report
+
+    # check file extension
+    #if path.suffix.lower() not in ('.cdf', '.nc'):
+    #    report["warnings"] = ["extension not .cdf/.nc"]
+    #    return report
+    report = validate_netcdf_simple(path, require_tags, require_crs)
+
+    # strong checks
+    report["header"] = sniff_header(path)
+
+    info = collect_netcdf_info(path)
+    report["open_info"] = info
+    if info.get("errors"):
+        report["errors"].extend(info.get("errors", []))
+
+    cf = cf_heuristic_checks(info)
+    report["cf_checks"] = cf
+    if cf.get("warnings"):
+        report["warnings"].extend(cf.get("warnings", []))
+    if cf.get("errors"):
+        report["errors"].extend(cf.get("errors", []))
+
+    # decide validity
+    # For strong validation Conventions present and at least lat/lon/time candidates required
+
+    conv_ok = cf["checks"].get("Conventions_present", False)
+    coords_ok = bool(cf["checks"].get("lat_candidates")) and bool(cf["checks"].get("lon_candidates"))
+    time_ok = bool(cf["checks"].get("time_candidates"))
+    report["is_valid"] = (info.get("reader") is not None) and conv_ok and coords_ok and time_ok and not bool(
+        cf.get("errors", []))
+    if not conv_ok:
+        report["errors"].append("Strong validation mode: missing 'Conventions' global attribute.")
+    if not coords_ok:
+        report["errors"].append("Strong validation mode: missing lat/lon coordinate variables.")
+    if not time_ok:
+        report["errors"].append("Strong validation mode: missing time variable.")
+
+
+    return report