raster-integrity 0.0.2__py2.py3-none-any.whl

raster_integrity/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.2"

raster_integrity/integrity.py

@@ -0,0 +1,45 @@
+import argparse
+import os
+from osgeo import gdal
+from technique_factory import TechniqueFactory
+
+
+def parse_tech_args(item):
+    if "=" not in item:
+        raise argparse.ArgumentTypeError(f"Expected KEY=VALUE, given: {item}")
+    key, value = item.split("=", 1)
+    return key.strip(), value.strip()
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Execute integrity technique on raster file",
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    parser.add_argument("--input", "-i", required=True, help="Path of raster file")
+    parser.add_argument(
+        "--arg",
+        "-a",
+        action="append",
+        type=parse_tech_args,
+        metavar="KEY=VALUE",
+        help="Extra argument for the technique, repeatable (e.g. -a precision=2)",
+    )
+    parser.add_argument(
+        "--technique", "-t", required=True, help=TechniqueFactory.help_text()
+    )
+    args = parser.parse_args()
+
+    if not os.path.exists(args.input):
+        parser.error(f"File not found: {args.input}")
+
+    try:
+        method = TechniqueFactory.create(args.technique, file=args.input)
+    except (ValueError, TypeError) as e:
+        parser.error(str(e))
+
+    print(method.create_integrity(**dict(args.arg or [])))
+
+
+if __name__ == "__main__":
+    main()

raster_integrity/setup.py

@@ -0,0 +1,122 @@
+from osgeo import gdal
+from pathlib import Path
+import argparse, os, sys
+
+gdal.UseExceptions()
+
+
+def create_metadata_file(file_path, metadata=None):
+    if os.path.exists(f"{file_path}.aux.xml"):
+        os.remove(f"{file_path}.aux.xml")
+    if metadata is not None:
+        for key, value in metadata.items():
+            if not isinstance(key, str) or not key.strip():
+                raise ValueError(f"Invalid metadata-key: {key!r}")
+            if value is None:
+                raise ValueError(f"Metadata-value for '{key}' is empty")
+
+    gdalPamEnabled = gdal.GetConfigOption("GDAL_PAM_ENABLED")
+    gdalGeorefSrcs = gdal.GetConfigOption("GDAL_GEOREF_SOURCES")
+
+    gdal.SetConfigOption("GDAL_PAM_ENABLED", "YES")
+    gdal.SetConfigOption("GDAL_GEOREF_SOURCES", "PAM,INTERNAL")
+
+    ds = gdal.Open(file_path, gdal.GA_ReadOnly)
+
+    srs = ds.GetProjection()
+    gt = ds.GetGeoTransform()
+    band = ds.GetRasterBand(1)
+    nodata = band.GetNoDataValue()
+
+    if nodata == None:
+        max_val = band.GetMaximum()
+        min_val = band.GetMinimum()
+        # Compute if not available
+        if max_val is None or min_val is None:
+            # False means calculate on full resolution
+            min_val, max_val = band.ComputeRasterMinMax(False)
+
+        # make decision - first max at last min (min is prior)
+        if max_val > 99999:
+            nodata = max_val
+        if min_val < -9999:
+            nodata = min_val
+
+    ds = None
+
+    ds2 = gdal.OpenEx(
+        file_path,
+        gdal.OF_RASTER,
+        open_options=["GEOREF_SOURCES=PAM"],  # PAM has priority
+    )
+    if metadata:
+        for key, value in metadata.items():
+            ds2.SetMetadataItem(key, str(value))
+    ds2.SetProjection(srs)
+    ds2.SetGeoTransform(gt)
+    if nodata is not None:
+        ds2.GetRasterBand(1).SetNoDataValue(nodata)
+    ds2.GetRasterBand(1).ComputeStatistics(False)
+
+    ds2.FlushCache()
+    ds2 = None
+    gdal.SetConfigOption("GDAL_PAM_ENABLED", gdalPamEnabled)
+    gdal.SetConfigOption("GDAL_GEOREF_SOURCES", gdalGeorefSrcs)
+
+
+def parse_meta(item):
+    if "=" not in item:
+        raise argparse.ArgumentTypeError(f"Expected KEY=VALUE, given: {item}")
+    key, value = item.split("=", 1)
+    return key.strip(), value.strip()
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=(
+            "Enforce raster data to include metadata,"
+            "\n"
+            "ie: raster_integrity.setup -f raster.tif -m Corporation='CORP' Survey_Year=2025 Datum=LAT"
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+
+    parser.add_argument(
+        "--meta",
+        "-m",
+        action="append",
+        type=parse_meta,
+        help="Metadata as KEY=VALUE (repeatable ie: -m Corporation='CORP' Survey_Year=2025 Datum=LAT)",
+    )
+
+    parser.add_argument(
+        "--file", "-f", required=True, help="Raster file to create a metadatafile"
+    )
+
+    args = parser.parse_args()
+
+    if not os.path.exists(args.file):
+        print(f"ERROR: File not found: {args.file}")
+        sys.exit(1)
+    metadata = dict(args.meta) if args.meta else None
+    create_metadata_file(args.file, metadata)
+
+    gdal.SetConfigOption("GDAL_PAM_ENABLED", "YES")
+    ds = gdal.Open(args.file, gdal.GA_ReadOnly)
+    band = ds.GetRasterBand(1)
+    band_metadata = band.GetMetadata()
+    print("+" + "-" * len(ds.GetDriver().GetName()) + "+")
+    print("|" + ds.GetDriver().GetName() + "|")
+    print("+" + "-" * len(ds.GetDriver().GetName()) + "+")
+    print(args.file)
+    print("-" * len(args.file))
+    print("DATASET METADATA")
+    print(ds.GetMetadata())
+    print("BAND 1 METADATA")
+    print(band_metadata)
+    ds.FlushCache()
+    ds = None
+
+
+if __name__ == "__main__":
+    main()

raster_integrity/technique_factory.py

@@ -0,0 +1,92 @@
+"""
+technique_factory.py
+=====================
+Factory-implementation for creating of different types of
+hashing- and validation techniques based on short codes.
+"""
+
+from enum import Enum
+from techniques.file_hash import FileHash
+from techniques.merkle_tree_hash import *
+from techniques.archiving import *
+from techniques.timestamping import *
+
+
+class TechniqueType(Enum):
+    FILE_HASH = "fh"
+    MERKLE = "me"
+    MERKLE_ROBUST = "mr"
+    MERKLE_SQLITE = "ms"
+    BAGIT_ARCHIVING = "ba"
+    TIMESTAMPING = "ts"
+
+
+class TechniqueFactory:
+    _mapping = {
+        TechniqueType.FILE_HASH: FileHash,
+        TechniqueType.MERKLE: MerkleTreeInmemoryHash,
+        TechniqueType.MERKLE_ROBUST: MerkleTreeInmemoryRobustHash,
+        TechniqueType.MERKLE_SQLITE: MerkleTreeSqliteHash,
+        TechniqueType.BAGIT_ARCHIVING: BagItArchive,
+        TechniqueType.TIMESTAMPING: Timestamping,
+    }
+
+    @classmethod
+    def create(self, code: str, **kwargs):
+        try:
+            hash_type = TechniqueType(code)
+        except ValueError:
+            raise ValueError(
+                f"Unknown technique code '{code}'.\n"
+                f"Available codes:\n{self.help_text()}"
+            )
+
+        return self._mapping[hash_type](kwargs)
+
+
+class TechniqueFactory:
+    _mapping = {
+        TechniqueType.FILE_HASH: FileHash,
+        TechniqueType.MERKLE: MerkleTreeInmemoryHash,
+        TechniqueType.MERKLE_ROBUST: MerkleTreeInmemoryRobustHash,
+        TechniqueType.MERKLE_SQLITE: MerkleTreeSqliteHash,
+        TechniqueType.BAGIT_ARCHIVING: BagItArchive,
+        TechniqueType.TIMESTAMPING: Timestamping,
+    }
+
+    @classmethod
+    def create(cls, code: str, **kwargs):
+        try:
+            technique_type = TechniqueType(code)
+        except ValueError:
+            raise ValueError(
+                f"Unknown technique code '{code}'.\n"
+                f"Available codes:\n{cls.help_text()}"
+            ) from None
+
+        return cls._mapping[technique_type](**kwargs)
+
+    @staticmethod
+    def help_text():
+        return (
+            "Techniques and their extra arguments (use -a KEY=VALUE):\n"
+            "\n"
+            "  fh  File hashing (digest of the file bytes)\n"
+            "        algorithm   hashlib algorithm            [default: sha256]\n"
+            "\n"
+            "  me  Merkle tree with hashing (in memory)\n"
+            "        algorithm   hashlib algorithm            [default: sha256]\n"
+            "\n"
+            "  mr  Merkle tree with robust hashing\n"
+            "        algorithm   hashlib algorithm            [default: sha256]\n"
+            "        precision   decimals to round (int)      [default: 3]\n"
+            "\n"
+            "  ms  Merkle tree with hashing, stored in SQLite (<file>.db)\n"
+            "        algorithm   hashlib algorithm            [default: sha256]\n"
+            "\n"
+            "  ba  RFC 8493 BagIt archiving\n"
+            "        root_hash   fingerprint stored in the bag [required]\n"
+            "\n"
+            "  ts  RFC 3161 trusted timestamping\n"
+            "        file        file used for timestamping    [required]\n"
+        )

raster_integrity/techniques/archiving.py

@@ -0,0 +1,99 @@
+"""
+archiving.py
+===================
+BagIt packaging for long-term archiving.
+
+BagIt (RFC 8493) is a hierarchical file packaging format for the storage and
+transfer of digital content. A "bag" is a directory containing the payload
+files together with a manifest of checksums, so that the completeness and
+integrity of the content can be verified at any time. This provides:
+
+    - Completeness check: all files in the manifest are present in the bag
+    - Integrity verification: checksums confirm that no file was altered
+    - Self-describing packages: metadata travels with the data
+    - Portability: plain files and text manifests, no special software required
+
+Structure of a bag:
+    bag/
+    ├── bagit.txt              BagIt version and tag file encoding
+    ├── bag-info.txt           descriptive metadata (source, date, contact, ...)
+    ├── manifest-sha256.txt    checksums of all payload files
+    ├── tagmanifest-sha256.txt checksums of the tag files
+    └── data/                  the payload (e.g. the bathymetric datasets)
+
+Difference from timestamping:
+    BagIt:        checksums + metadata → proves what a package contains
+    Timestamping: independent third party → proves when it existed
+    The two are complementary: a bag manifest can be timestamped to prove
+    both the content and the time of archiving.
+
+Use at Hydrographic Offices:
+    - Packaging bathymetric datasets (e.g. GeoTIFF) for long-term archiving
+    - Detecting bit rot or accidental modification of archived surveys
+    - Transferring datasets to other parties with verifiable integrity
+
+Usage:
+    # Create a bag from a directory
+    bag = bagit.make_bag("survey_2024/", {"Source-Organization": "Flemish Hydrography"})
+
+    # Validate an existing bag
+    bag = bagit.Bag("survey_2024/")
+    bag.validate()
+"""
+
+import bagit
+import os
+import shutil
+from osgeo import gdal
+from .technique import Technique
+
+gdal.UseExceptions()
+
+
+class BagItArchive(Technique):
+    def label(self):
+        return f"archive_bagit"
+
+    def _house_holding(self):
+        directory = self._get_directory(self._file)
+
+        if os.path.exists(directory):
+            shutil.rmtree(directory)
+        os.makedirs(directory)
+
+        filename = os.path.basename(self._file)
+        shutil.copy(self._file, os.path.join(directory, filename))
+
+        aux = f"{self._file}.aux.xml"
+        if os.path.exists(aux):
+            shutil.copy(aux, os.path.join(directory, f"{filename}.aux.xml"))
+
+    def create_integrity(self, **kwargs):
+        """
+        Parameters
+        ----------
+        root_hash : str
+            Integrity hash key for storing into bag, for validation on data/info content
+        """
+        if "root_hash" not in kwargs:
+            raise TypeError("Missing required keyword argument: 'root_hash'")
+
+        self._root_hash = kwargs["root_hash"]
+        self._house_holding()
+        bag = bagit.make_bag(
+            self._get_directory(self._file),
+            {
+                "fingerprint": self._root_hash,
+            },
+        )
+        return bag
+
+    def _get_directory(self, file):
+        root, _ = os.path.splitext(file)
+        return f"{root}_bagit"
+
+    def _create_directory(self, file):
+        dir = self._get_directory(file)
+        if not os.path.exists(dir):
+            os.mkdir(dir)
+        return dir

raster_integrity/techniques/file_hash.py

@@ -0,0 +1,73 @@
+"""
+file_hash.py
+============
+Cryptographic file hashing for integrity verification.
+
+A cryptographic hash function maps a file of arbitrary size to a fixed-length
+digest (the "fingerprint" of the file). Any change to the file, however small,
+results in a completely different digest. This provides:
+
+    - Integrity verification: detect accidental corruption or modification
+    - Change detection: compare a file against a previously stored digest
+    - Deduplication: identical content yields an identical digest
+
+The default algorithm is SHA-256 (SHA-2 family, specified in NIST FIPS 180-4).
+Any algorithm supported by Python's hashlib can be selected instead.
+Files are read in 64 KB chunks, so large datasets (e.g. GeoTIFF rasters or
+point clouds) can be hashed without loading them into memory.
+
+Difference from HMAC and timestamping:
+    File hash:    digest of the content only -> proves the file is unchanged
+                  relative to a known digest
+    HMAC:         digest + shared secret key -> also proves who created it
+    Timestamping: digest + independent third party -> proves when it existed
+    A plain hash offers no authenticity: anyone can recompute it. It is
+    therefore only trustworthy when the reference digest itself is stored
+    or transmitted through a trusted channel (e.g. in a signed or
+    timestamped manifest).
+
+Used for:
+    - Fixity checks on archived bathymetric datasets
+    - Verifying that a dataset was transferred without corruption
+
+Usage:
+    hasher = FileHash(algorithm="sha256")
+
+    # Create the digest of a file
+    digest = hasher.create_integrity("dataset.tif")
+
+    # Verify a file against a stored digest
+    ok = hasher.verify("dataset.tif", integrity=digest)
+"""
+
+import os
+import hashlib
+from osgeo import gdal
+from .technique import Technique
+
+gdal.UseExceptions()
+
+
+class FileHash(Technique):
+    def label(self):
+        return "file_hash"
+
+    def create_integrity(self, **kwargs):
+        """
+        Parameters
+        ----------
+        algorithm : str
+            kind of algorithm to used for creating hash key
+            standard: "sha256"
+        """
+        if "algorithm" in kwargs:
+            self._algorithm = kwargs["algorithm"]
+        else:
+            self._algorithm = "sha256"
+
+        hasher = hashlib.new(self._algorithm)
+        with open(self._file, "rb") as f:
+            while chunk := f.read(65536):  # 64KB chunks
+                hasher.update(chunk)
+        digest = hasher.hexdigest()
+        return digest

raster_integrity/techniques/merkle_tree_hash.py

@@ -0,0 +1,262 @@
+"""
+merkle_tree_hash.py
+===================
+Content-based integrity hashing of raster datasets with a Merkle tree.
+
+Where a plain file hash (see FileHash) fingerprints the stored bytes of a
+file, the classes in this module fingerprint the *content* of a raster: the
+pixel values of the first band, read through GDAL. Every raster row becomes
+one leaf of a Merkle tree (a binary tree in which every node is the hash of
+its children). The root of the tree, hex-encoded, is the integrity value of
+the dataset. This provides:
+
+    - Format independence: the same values stored with a different
+      compression or tiling yield the same fingerprint
+    - Low memory use: the raster is read and hashed row by row
+    - Change localisation: a persisted tree can be used to find which rows
+      changed, or to prove that a row is part of the dataset
+    - Tolerance (robust variant): insensitive to small rounding differences
+
+Classes:
+    MerkleTreeInmemoryHash
+        Tree kept in memory. Nothing is written to disk.
+    MerkleTreeSqliteHash
+        Same fingerprint, but the tree is persisted in a SQLite database next
+        to the dataset ("<file>.db"). Suited when the tree is needed after
+        the calculation, for instance to generate proofs.
+    MerkleTreeInmemoryRobustHash
+        Rounds values before hashing, so that small numerical differences do
+        not change the fingerprint (see "Robust hashing").
+
+What is hashed (in this order):
+    1. Dataset metadata items whose key starts with "STATISTICS" or
+       "Corporation", sorted by key and stored as "key=value"
+    2. The same selection from the metadata of band 1
+    3. For each raster row: the values with NoData removed, as float32 bytes
+
+    Not part of the fingerprint: georeferencing (geotransform, CRS), other
+    bands, other metadata, and the position of NoData cells within a row.
+
+Robust hashing:
+    Before hashing, values are rounded to `precision` decimals (default 3)
+    and converted to absolute values. The sign-insensitivity makes the
+    fingerprint independent of the sign convention (e.g. depths stored as
+    positive or negative numbers). For datasets with a negative mean,
+    STATISTICS_MINIMUM and STATISTICS_MAXIMUM are swapped accordingly.
+
+        12.123 == 12.1234555    (precision=3)
+        12.123 != 12.456
+
+    Limitations:
+    - Rounding is not a true tolerance: two values that differ by less than
+      10**-precision can still fall on different sides of a rounding
+      boundary and produce different fingerprints.
+    - A value cannot be distinguished from its negative, so the robust
+      variant is only meaningful for datasets whose values share one sign.
+
+Difference from a file hash:
+    File hash:   digest of the stored bytes -> proves the file is unchanged
+    Merkle hash: digest of the raster values -> proves the data is unchanged,
+                 even if the file was rewritten (e.g. recompressed)
+
+Use at Flemish Hydrography:
+    - Fixity checks on archived bathymetric rasters (e.g. GeoTIFF)
+    - Verifying that a dataset kept its values after format conversion
+    - Producing the fingerprints that are timestamped (RFC 3161) or listed
+      in a BagIt manifest
+
+Requires: GDAL (osgeo), NumPy and pymerkle.
+
+Usage:
+    hasher = MerkleTreeInmemoryHash(algorithm="sha256")
+
+    # Create the fingerprint of a raster
+    fingerprint = hasher.create_integrity("dataset.tif")
+
+    # Verify a raster against a stored fingerprint
+    ok = hasher.verify("dataset.tif", integrity=fingerprint)
+
+    # Tolerant of rounding differences beyond the third decimal
+    robust = MerkleTreeInmemoryRobustHash(precision=3)
+    fingerprint = robust.create_integrity("dataset.tif")
+"""
+
+from osgeo import gdal
+import numpy as np
+from pymerkle import SqliteTree, InmemoryTree
+import hashlib
+import sys
+import os
+from .technique import Technique
+
+gdal.UseExceptions()
+
+
+class MerkleTreeInmemoryHash(Technique):
+    def _getDatasource(self, file):
+        ds = gdal.Open(file)
+        if ds is None:
+            raise FileNotFoundError(f"GDAL cannot open file: {file}")
+        return ds
+
+    def _getMerkleTree(self):
+        self._house_holding()
+        return InmemoryTree(algorithm=self._algorithm)
+
+    def _normalize_to_prec(self, value):
+        return value
+
+    def _addMetadataToTree(self, metadata, tree):
+        # ADDING metadata
+        # Sorted, be sure all metadata items are stored in the same way
+        # Dictionary sort is not guaranteed
+        filtered = sorted(
+            (
+                (k, self._normalize_to_prec(v))
+                for k, v in metadata
+                if k.startswith("STATISTICS") or k.startswith("Corporation")
+            ),
+            key=lambda x: x[0],  # sorteer op key
+        )
+
+        for meta_key, meta_value in filtered:
+            tree.append_entry(f"{meta_key}={meta_value}".encode("utf-8"))
+
+    def label(self):
+        return "merkle_tree_inmemory_hash"
+
+    def _house_holding(self):
+        db_path = f"{self._file}.db"
+        if os.path.exists(db_path):
+            os.remove(db_path)
+
+    def create_integrity(self, **kwargs):
+        """
+        Parameters
+        ----------
+        algorithm : str
+            kind of algorithm to used for creating hash key
+            standard: "sha256"
+        """
+        self._algorithm = "sha256"
+        if "algorithm" in kwargs:
+            self._algorithm = kwargs["algorithm"]
+
+        ds = self._getDatasource(self._file)
+        band = ds.GetRasterBand(1)
+        nodata = band.GetNoDataValue()
+
+        cols = ds.RasterXSize
+        rows = ds.RasterYSize
+
+        tree = self._getMerkleTree()
+
+        # ADDING metadata
+        self._addMetadataToTree(ds.GetMetadata().items(), tree)
+        self._addMetadataToTree(band.GetMetadata().items(), tree)
+
+        for y in range(rows):
+            row = band.ReadAsArray(0, y, cols, 1)[0]
+            # mask nodata
+            if nodata is not None:
+                row = row[row != nodata]
+            # No X/Y, no geotransform, only structure + value
+            packed = row.astype(np.float32).tobytes()
+            tree.append_entry(packed)
+        root = tree.get_state()
+        # band.SetMetadataItem("fingerprint", root.hex())
+        ds = None
+        band = None
+        return root.hex()
+
+
+class MerkleTreeSqliteHash(MerkleTreeInmemoryHash):
+    def label(self):
+        return "merkle_tree_sqlite_hash"
+
+    def _getMerkleTree(self):
+        self._house_holding()
+        return SqliteTree(f"{self._file}.db", algorithm=self._algorithm)
+
+
+class MerkleTreeInmemoryRobustHash(MerkleTreeInmemoryHash):
+    """
+    Merkle Tree integriteitscontrole with robust hashing.
+
+    The difference with regular hashing:
+        - Values will be rounded on `precision` decimals before hashing.
+          It makes the hash tolerant for non-content changes such as:
+              12.123     == 12.1234555  (precision=3)
+              -5.25      == -5.2499999  (precision=2)
+          It detects real content changes:
+              12.123     != 12.456
+    """
+
+    def label(self):
+        return f"merkle_tree_robust_hash_p{self._precision}"
+
+    def _normalize_to_prec(self, value):
+        try:
+            return abs(round(float(value), self._precision))
+        except ValueError:
+            return value
+
+    def _robust_metadata(self, metadata):
+        if "STATISTICS_MEAN" in metadata and float(metadata["STATISTICS_MEAN"]) < 0:
+            new_min = metadata["STATISTICS_MAXIMUM"]
+            metadata["STATISTICS_MAXIMUM"] = metadata["STATISTICS_MINIMUM"]
+            metadata["STATISTICS_MINIMUM"] = new_min
+        return metadata
+
+    def create_integrity(self, **kwargs):
+        """
+        Parameters
+        ----------
+        algorithm : str
+            kind of algorithm to used for creating hash key
+            standard: "sha256"
+        precision : int
+            Number of decimals to be rounded before hashing.
+            High value = sensitive for small changes.
+            Low value  = more tolerant for rounding differences.
+
+            Recommendation for raster data:
+                precision=2 -> tolerant up to 0.01m   (ie. after lossy compression)
+                precision=3 -> tolerant up to 0.001m  (recommended standard)
+                precision=4 -> tolerant up to 0.0001m (strict)
+        """
+        self._algorithm = "sha256"
+        if "algorithm" in kwargs:
+            self._algorithm = kwargs["algorithm"]
+
+        self._precision = 3
+        if "precision" in kwargs:
+            self._precision = kwargs["precision"]
+
+        ds = self._getDatasource(self._file)
+        band = ds.GetRasterBand(1)
+        nodata = band.GetNoDataValue()
+
+        cols = ds.RasterXSize
+        rows = ds.RasterYSize
+
+        tree = self._getMerkleTree()
+
+        # ADDING metadata
+        self._addMetadataToTree(self._robust_metadata(ds.GetMetadata()).items(), tree)
+        self._addMetadataToTree(self._robust_metadata(band.GetMetadata()).items(), tree)
+
+        for y in range(rows):
+            row = band.ReadAsArray(0, y, cols, 1)[0]
+            # mask nodata
+            if nodata is not None:
+                row = row[row != nodata]
+
+            row = np.abs(np.round(row, decimals=self._precision))
+            packed = row.astype(np.float32).tobytes()
+            tree.append_entry(packed)
+
+        root = tree.get_state()
+        band = None
+        ds = None
+        return root.hex()

raster_integrity/techniques/technique.py

@@ -0,0 +1,17 @@
+import os
+
+
+class Technique:
+    def __init__(self, **kwargs):
+        """
+        Parameters
+        ----------
+        file : str
+            Raster file for creating integrity
+        """
+        if "file" not in kwargs:
+            raise TypeError("Missing required keyword argument: 'file'")
+
+        self._file = kwargs["file"]
+        if not os.path.exists(self._file):
+            raise ValueError(f"File '{self._file}' does not exist")

raster_integrity/techniques/timestamping.py

@@ -0,0 +1,174 @@
+"""
+timestamping.py
+===============
+RFC 3161 Trusted Timestamping.
+
+RFC 3161 is an internet standard for trusted timestamping. A
+Timestamp Authority (TSA) cryptographically certifies that a given
+hash value existed at a specific point in time. This provides:
+
+    - Integrity verification: the data has not been modified since the timestamp
+    - Non-repudiation: the time has been confirmed by an independent party
+    - Legal evidentiary value: relevant for liability and archiving
+
+Difference from HMAC:
+    HMAC:         shared key between two parties → internal use
+    Timestamping: independent third party → external proof, legally enforceable
+
+Use at Hydrographic Offices:
+    - Long-term archiving of bathymetric datasets
+    - Legal proof of the state of a dataset at a specific point in time
+    - Demonstrating when a survey was completed
+
+Usage:
+    # Compute timestamp (online via FreeTSA)
+    openssl ts -reply -in file.tsr -text
+    openssl ts -verify -in file.tsr -data file.txt -CAfile tsa_cert/cacert.pem
+
+    # Verify a timestamp
+    python timestamping.py --verify dataset.tif --timestamp dataset.tif.tsr
+
+"""
+
+from rfc3161_client import (
+    TimestampRequestBuilder,
+    decode_timestamp_response,
+    VerifierBuilder,
+    VerificationError,
+)
+import secrets
+import requests
+from cryptography import x509
+from cryptography.hazmat.primitives import hashes
+import certifi
+import os
+import subprocess
+from pathlib import Path
+import urllib.request
+
+TSA_EINDPUNTEN = {
+    "freetsa": "https://freetsa.org/tsr",
+    "digicert": "https://timestamp.digicert.com",
+    "globalsign": "http://timestamp.globalsign.com/tsa/r45standard",
+}
+CACERT_URL = "https://freetsa.org/files/cacert.pem"
+
+
+class Timestamping:
+
+    def __init__(self, **kwargs):
+        """
+        Parameters
+        ----------
+        file : str or Path
+            File used for timestamping
+        """
+        if "file" not in kwargs:
+            raise TypeError("Missing required keyword argument: 'file'")
+
+        self._file = Path(kwargs["file"])
+        if not self._file.is_file():
+            raise FileNotFoundError(f"File not found for Timestamping: {self._file}")
+
+    def label(self):
+        return f"timestamping"
+
+    # def _house_holding(self, file=""):
+    #     path = f"{file}.tsr"
+    #     if os.path.exists(path):
+    #         os.remove(path)
+
+    # def verify(self, url="output.xyz", integrity=""):
+    #     """
+    #     Clones this functionality
+    #     openssl ts -verify -in file.tsr -data file.txt -CAfile tsa_cert/cacert.pem
+    #     """
+    #     ca_path = "data/tsa_cert/cacert.pem"
+    #     self._ensure_file(ca_path)
+    #     data_path = self._file
+    #     try:
+    #         with open(url, "rb") as f:
+    #             tsr_bytes = f.read()
+
+    #         tsr_object = decode_timestamp_response(tsr_bytes)
+
+    #         with open(data_path, "rb") as f:
+    #             message_data = f.read()
+
+    #         with open(ca_path, "rb") as f:
+    #             ca_certs = x509.load_pem_x509_certificates(f.read())
+
+    #         builder = VerifierBuilder()
+    #         for cert in ca_certs:
+    #             builder.add_root_certificate(cert)
+
+    #         verifier = builder.build()
+
+    #         verifier.verify_message(tsr_object, message_data)
+
+    #         print(" Verification: OK")
+    #         return True
+
+    #     except VerificationError as ve:
+    #         print(f" Verification FAILED: Cryptografische fout of mismatch: {ve}")
+    #         return False
+    #     except Exception as e:
+    #         print(f" Fout bij het uitvoeren van de verificatie: {e}")
+    #         return False
+
+    def create_integrity(self, **kwargs):
+        """
+        Parameters
+        ----------
+        tsr_name : str
+            Name given when timestamp is done and a timestamp is made
+        """
+        if "tsr_name" not in kwargs:
+            raise TypeError("Missing required keyword argument: 'tsr_name'")
+
+        tsr_name = kwargs["tsr_name"]
+        tsa_name = "freetsa"
+        tsa_url = TSA_EINDPUNTEN.get(tsa_name)
+
+        with open(self._file, "rb") as f:
+            message = f.read()
+
+        # Verzoek bouwen
+        request = TimestampRequestBuilder().data(message).nonce().build()
+        with open(f"{tsr_name}.tsq", "wb") as f:
+            f.write(request.as_bytes())
+        # print(request.nonce)
+        # Sturen naar TSA
+        response = requests.post(
+            tsa_url,
+            data=request.as_bytes(),
+            headers={"Content-Type": "application/timestamp-query"},
+        )
+        timestamp_response = decode_timestamp_response(response.content)
+        tsr = timestamp_response.as_bytes()
+        with open(f"{tsr_name}.tsr", "wb") as f:
+            f.write(tsr)
+
+        return f"{tsr_name}.tsr"
+
+    def _ensure_file(self, path, url=CACERT_URL):
+        """Download `url` to `path` if the file does not exist yet."""
+        if os.path.exists(path):
+            return path
+
+        os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
+
+        tmp_path = f"{path}.part"
+        try:
+            with urllib.request.urlopen(url, timeout=30) as response, open(
+                tmp_path, "wb"
+            ) as out:
+                out.write(response.read())
+            os.replace(
+                tmp_path, path
+            )  # atomair: nooit een half bestand onder de echte naam
+        except Exception:
+            if os.path.exists(tmp_path):
+                os.remove(tmp_path)
+            raise
+        return path

raster_integrity/verify.py

@@ -0,0 +1,55 @@
+import argparse
+import os
+import sys
+from technique_factory import TechniqueFactory, TechniqueType
+
+
+def parse_tech_args(item):
+    if "=" not in item:
+        raise argparse.ArgumentTypeError(f"Expected KEY=VALUE, given: {item}")
+    key, value = item.split("=", 1)
+    return key.strip(), value.strip()
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Raster data verification",
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    parser.add_argument(
+        "--source",
+        "-s",
+        required=True,
+        help="Hash key or Timestamp checking the intregrity",
+    )
+    parser.add_argument(
+        "--arg",
+        "-a",
+        action="append",
+        type=parse_tech_args,
+        metavar="KEY=VALUE",
+        help="Extra argument for the technique, repeatable (e.g. -a precision=2)",
+    )
+    parser.add_argument(
+        "--technique",
+        "-t",
+        default="fh",
+        required=True,
+        help=f"Used technique for verification: \n {TechniqueFactory.help_text()}",
+    )
+
+    args = parser.parse_args()
+
+    if not os.path.exists(args.source):
+        print(f"ERROR: File not found: {args.source}")
+        sys.exit(1)
+    try:
+        techn = TechniqueFactory.create(args.technique, **dict(args.arg or []))
+        print(f"Verification result:{techn.verify(args.source)}")
+    except ValueError as e:
+        print(f"FOUT: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

raster_integrity-0.0.2.dist-info/METADATA

@@ -0,0 +1,85 @@
+Metadata-Version: 2.4
+Name: raster_integrity
+Version: 0.0.2
+Summary: Technical methods for checking the integrity of raster data
+Author-email: Dedulle Wim <wim.dedulle@proton.me>
+License: MIT License
+Project-URL: Homepage, https://github.com/wdedulle/raster_integrity
+Keywords: raster_integrity
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: AUTHORS.rst
+Requires-Dist: bagit
+Requires-Dist: cryptography
+Requires-Dist: numpy
+Requires-Dist: pymerkle
+Requires-Dist: rfc3161_client
+Provides-Extra: all
+Requires-Dist: raster_integrity[extra]; extra == "all"
+Provides-Extra: extra
+Requires-Dist: pandas; extra == "extra"
+Dynamic: license-file
+
+# raster_integrity
+
+
+[![image](https://img.shields.io/pypi/v/raster_integrity.svg)](https://pypi.python.org/pypi/raster_integrity)
+[![image](https://img.shields.io/conda/vn/conda-forge/raster_integrity.svg)](https://anaconda.org/conda-forge/raster_integrity)
+
+[![image](https://pyup.io/repos/github/wdedulle/raster_integrity/shield.svg)](https://pyup.io/repos/github/wdedulle/raster_integrity)
+
+
+**Technical methods for checking the integrity of raster data**
+
+
+-   Free software: MIT License
+-   Documentation: https://wdedulle.github.io/raster_integrity
+
+
+## Features
+
+-   Setting up metadata for raster files: it includes common and custom raster information.
+```shell
+python .\raster_integrity\setup.py -f .\data\classified.xyz -m "used_for"="classification for AI"
+```
+
+-   Creating integrity hash
+```shell
+python .\raster_integrity\integrity.py -i .\data\classified.jpg -t me
+8eb3bdddc021b045622f9811ff925d132ad6274f0bba6546dcbf6014dac19bdf
+```
+-   Archive all artefacts with its integrity hash
+```shell
+python .\raster_integrity\integrity.py -i .\data\classified.jpg -t ba -a root_hash=8eb3bdddc021b045622f9811ff925d132ad6274f0bba6546dcbf6014dac19bdf
+.\data\classified_bagit
+```
+-   Seal the tagmanifest with timestamping
+```shell
+python .\raster_integrity\integrity.py -i .\data\classified_bagit\tagmanifest-sha256.txt -t ts -a tsr_name=.\data\tagmanifest_cassified_jpg
+.\data\tagmanifest_cassified_jpg.tsr
+```
+
+-   Verification - To Be
+```shell
+python .\raster_integrity\verify.py -a file=.\data\classified_bagit\tagmanifest-sha256.txt -t ts -s .\data\classified.jpg.tsr
+python .\raster_integrity\verify.py -a root_hash=8eb3bdddc021b045622f9811ff925d132ad6274f0bba6546dcbf6014dac19bdf -t me -s .\data\classified.jpg
+```
+
+## Required package
+GDAL: Geospatial Data Abstraction Library version 3.13.3
+### Installation for Python 3.15
+Wheel packages can be found at https://github.com/cgohlke/geospatial-wheels/releases
+
+This a specific GDAL version 3.13.3 installation for Windows AMD64 with Python version 3.15.*
+```
+pip install https://github.com/cgohlke/geospatial-wheels/releases/download/v2026.8.20/gdal-3.13.3-cp315-cp315-win_amd64.whl
+```

raster_integrity-0.0.2.dist-info/RECORD

@@ -0,0 +1,18 @@
+raster_integrity/__init__.py,sha256=QvlVh4JTl3JL7jQAja76yKtT-IvF4631ASjWY1wS6AQ,22
+raster_integrity/integrity.py,sha256=jARfjBikXtqtvDiKM-fdy4aS2nLckSjGiLhkhA-LUKs,1282
+raster_integrity/setup.py,sha256=ZS68cjz3ptqr7LRpNFHUsUYwid_eVkQv3-Iom5ulp6o,3720
+raster_integrity/technique_factory.py,sha256=QVySneWWYlSRpWaH74tefh9V7ZIInQrgNFlX63GgPfY,3137
+raster_integrity/verify.py,sha256=u4nDoyhRK7s9wpQj0lGuE8ydLUwz8rAt7ZLXBPXnu_0,1461
+raster_integrity/techniques/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+raster_integrity/techniques/archiving.py,sha256=V2WZ58zWuA53Iw5vIHBing4fAasgTsbmtDpHM78-XmY,3355
+raster_integrity/techniques/file_hash.py,sha256=Z4fPRKBk933ZbYWufC1sOaU7WURsqdUYkDSdRpv7cTo,2506
+raster_integrity/techniques/merkle_tree_hash.py,sha256=NRitljjMpxECYPJs2pxaSg31QqSNj8gxwvjJh_F9LUI,9405
+raster_integrity/techniques/technique.py,sha256=qu761tiiPq-w_Fq8yXDRIIBrFo7mTbDqna_MjayFzu4,445
+raster_integrity/techniques/timestamping.py,sha256=OWz2gA-u1XA8tXQuYeKjkiHfnauvdJ799fmBUwgFwW0,5522
+raster_integrity-0.0.2.dist-info/licenses/AUTHORS.rst,sha256=GGH30hXv6Zfjg7QDp6TkeVbAnuCksKxBgb2kqXnfBtw,158
+raster_integrity-0.0.2.dist-info/licenses/LICENSE,sha256=MRMIkJs8bKcYa_8_GHofsYEfcCVb_0I8y2kUQyeI2_E,1070
+raster_integrity-0.0.2.dist-info/METADATA,sha256=o3HIQmZJkkCaEw5KIXedCdAaU1wxqQ8_K_lKWWXqgpk,3230
+raster_integrity-0.0.2.dist-info/WHEEL,sha256=Uvlg2aNlEKdilsyFSJt1QX5me0eEbCjQzR3nZeFkvy4,109
+raster_integrity-0.0.2.dist-info/entry_points.txt,sha256=fAMaL9chR--gr6a4LHdx2M-pPYUrGRbG7CyTPmj89as,63
+raster_integrity-0.0.2.dist-info/top_level.txt,sha256=D9igPYFLAm8QOh9rTevgj95GiGzQq7BzS9EMqmAxSTA,17
+raster_integrity-0.0.2.dist-info/RECORD,,

raster_integrity-0.0.2.dist-info/WHEEL

@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py2-none-any
+Tag: py3-none-any
+

raster_integrity-0.0.2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+raster_integrity = raster_integrity.cli:main

raster_integrity-0.0.2.dist-info/licenses/AUTHORS.rst

@@ -0,0 +1,13 @@
+=======
+Credits
+=======
+
+Development Lead
+----------------
+
+* Dedulle Wim <wim.dedulle@proton.me>
+
+Contributors
+------------
+
+None yet. Why not be the first?

raster_integrity-0.0.2.dist-info/licenses/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2026, Dedulle Wim
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

raster_integrity-0.0.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+raster_integrity