toshi-hazard-post 0.7.1__tar.gz

toshi_hazard_post-0.7.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022, Chris B Chamberlain
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

toshi_hazard_post-0.7.1/PKG-INFO

@@ -0,0 +1,65 @@
+Metadata-Version: 2.3
+Name: toshi-hazard-post
+Version: 0.7.1
+Summary: Hazard post-processing pipeline as serverless AWS infrastructure.
+License: AGPL-3.0
+Author: Chris DiCaprio
+Author-email: christopher.dicaprio@gmail.com
+Requires-Python: >=3.10,<3.13
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Requires-Dist: click (>=8.1.8)
+Requires-Dist: numba (<0.61.0)
+Requires-Dist: numpy (<2)
+Requires-Dist: nzshm-common (>=0.8.4)
+Requires-Dist: nzshm-model (>=0.13.5)
+Requires-Dist: pandas (>=2.0.3)
+Requires-Dist: pyarrow[pandas] (>=15.0)
+Requires-Dist: pydantic (>=2.10.6)
+Requires-Dist: python-dotenv (>=1.0.1)
+Requires-Dist: tomlkit (>=0.13.2)
+Requires-Dist: toshi-hazard-store (>=1.1.0)
+Requires-Dist: tzdata (>=2025.2,<2026.0)
+Description-Content-Type: text/markdown
+
+# toshi-hazard-post
+
+
+[![pypi](https://img.shields.io/pypi/v/toshi-hazard-post.svg)](https://pypi.org/project/toshi-hazard-post/)
+[![python](https://img.shields.io/pypi/pyversions/toshi-hazard-post.svg)](https://pypi.org/project/toshi-hazard-post/)
+[![Build Status](https://github.com/gns-science/toshi-hazard-post/actions/workflows/dev.yml/badge.svg)](https://github.com/gns-science/toshi-hazard-post/actions/workflows/dev.yml)
+[![codecov](https://codecov.io/gh/gns-science/toshi-hazard-post/branch/main/graphs/badge.svg)](https://codecov.io/github/gns-science/toshi-hazard-post)
+
+
+Seismic hazard from pre-calculated relizations. This application impliments the hazard calculation method described in [Calculation of National Seismic Hazard Models with Large Logic Trees: Application to the NZ NSHM 2022](https://doi.org/10.1785/0220230226).
+
+
+* Documentation: <https://gns-science.github.io/toshi-hazard-post>
+* GitHub: <https://github.com/gns-science/toshi-hazard-post>
+* PyPI: <https://pypi.org/project/toshi-hazard-post/>
+* Free software: AGPL-3.0
+
+## Requirements
+
+toshi-hazard-post requires python 3.10 - 3.11. It relies on other GNS-Science NSHM libraries:
+
+- nzshm-model: <https://github.com/GNS-Science/nzshm-model>
+- nzshm-common: <https://github.com/GNS-Science/nzshm-common-py>
+- toshi-hazard-store: <https://github.com/GNS-Science/toshi-hazard-store>
+
+In addition, it requires a small number of scientific computation packages such as numba, numpy, and pandas. See [installation notes](docs/installation.md) for more details on how to install the application.
+
+
+## Usage
+
+Getting started
+```
+thp --help
+```
+
+See [usage documentation page](docs/usage.md) for more info.
+

toshi_hazard_post-0.7.1/README.md

@@ -0,0 +1,36 @@
+# toshi-hazard-post
+
+
+[![pypi](https://img.shields.io/pypi/v/toshi-hazard-post.svg)](https://pypi.org/project/toshi-hazard-post/)
+[![python](https://img.shields.io/pypi/pyversions/toshi-hazard-post.svg)](https://pypi.org/project/toshi-hazard-post/)
+[![Build Status](https://github.com/gns-science/toshi-hazard-post/actions/workflows/dev.yml/badge.svg)](https://github.com/gns-science/toshi-hazard-post/actions/workflows/dev.yml)
+[![codecov](https://codecov.io/gh/gns-science/toshi-hazard-post/branch/main/graphs/badge.svg)](https://codecov.io/github/gns-science/toshi-hazard-post)
+
+
+Seismic hazard from pre-calculated relizations. This application impliments the hazard calculation method described in [Calculation of National Seismic Hazard Models with Large Logic Trees: Application to the NZ NSHM 2022](https://doi.org/10.1785/0220230226).
+
+
+* Documentation: <https://gns-science.github.io/toshi-hazard-post>
+* GitHub: <https://github.com/gns-science/toshi-hazard-post>
+* PyPI: <https://pypi.org/project/toshi-hazard-post/>
+* Free software: AGPL-3.0
+
+## Requirements
+
+toshi-hazard-post requires python 3.10 - 3.11. It relies on other GNS-Science NSHM libraries:
+
+- nzshm-model: <https://github.com/GNS-Science/nzshm-model>
+- nzshm-common: <https://github.com/GNS-Science/nzshm-common-py>
+- toshi-hazard-store: <https://github.com/GNS-Science/toshi-hazard-store>
+
+In addition, it requires a small number of scientific computation packages such as numba, numpy, and pandas. See [installation notes](docs/installation.md) for more details on how to install the application.
+
+
+## Usage
+
+Getting started
+```
+thp --help
+```
+
+See [usage documentation page](docs/usage.md) for more info.

toshi_hazard_post-0.7.1/pyproject.toml

@@ -0,0 +1,115 @@
+[project]
+name = "toshi-hazard-post"
+version = "0.7.1"
+readme = "README.md"
+description = "Hazard post-processing pipeline as serverless AWS infrastructure."
+authors = [
+    {name = "Chris DiCaprio", email = "christopher.dicaprio@gmail.com"},
+    {name = "Chris Chamberlain", email = "chrisbc@artisan.co.nz"},
+]
+license =  "AGPL-3.0"
+requires-python = ">=3.10,<3.13"
+classifiers=[
+    'Development Status :: 2 - Pre-Alpha',
+    'Intended Audience :: Developers',
+    'License :: OSI Approved :: MIT License',
+    'Natural Language :: English',
+    'Programming Language :: Python :: 3',
+    'Programming Language :: Python :: 3.10',
+]
+dependencies = [
+    "pandas (>=2.0.3)",
+    "click (>=8.1.8)",
+    "python-dotenv (>=1.0.1)",
+    "pyarrow[pandas] (>=15.0)",
+    "nzshm-common (>=0.8.4)",
+    "nzshm-model (>=0.13.5)",
+    "pydantic (>=2.10.6)",
+    "tomlkit (>=0.13.2)",
+    "numba (<0.61.0)",
+    "toshi-hazard-store (>=1.1.0)",
+    "numpy (<2)",
+    "tzdata (>=2025.2,<2026.0)",
+]
+
+[tool.poetry]
+packages = [
+    { include = "toshi_hazard_post" },
+    { include = "tests", format = "sdist" },
+    { include = "scripts" },
+]
+
+[tool.poetry.scripts]
+thp = 'scripts.cli:thp'
+
+[tool.mypy]
+plugins = "numpy.typing.mypy_plugin"
+ignore_missing_imports = true
+
+
+[tool.poetry.group.dev.dependencies]
+black = "^25.1.0"
+isort = "^6.0.0"
+flake8 = "^7.1.2"
+pytest = "^8.3.4"
+pytest-cov = "^6.0.0"
+twine = "^6.1.0"
+bump2version = "^1.0.1"
+types-toml = "^0.10.8.20240310"
+types-requests = "^2.32.0.20241016"
+tox = "^4.24.1"
+mypy = "^1.15.0"
+flake8-docstrings = "^1.7.0"
+
+
+[tool.poetry.group.doc.dependencies]
+mkdocs = "^1.6.1"
+mkdocs-include-markdown-plugin = "^7.1.4"
+mkdocs-material = "^9.6.5"
+mkdocs-material-extensions = "^1.3.1"
+mkdocs-autorefs = "^1.4.0"
+mkdocs-pymdownx-material-extras = "^2.7"
+pymdown-extensions = "^10.14.3"
+mkdocstrings = "^0.28.2"
+mkdocstrings-python = "^1.16.2"
+
+[tool.poetry.extras]
+aws = ["boto3"]
+
+[tool.black]
+line-length = 120
+skip-string-normalization = true
+target-version = ['py310', 'py311']
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.eggs
+  | \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+)/
+'''
+
+[tool.isort]
+multi_line_output = 3
+include_trailing_comma = true
+force_grid_wrap = 0
+use_parentheses = true
+ensure_newline_before_comments = true
+line_length = 120
+skip_gitignore = true
+# you can skip files as below
+#skip_glob = docs/conf.py
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.pytest.ini_options]
+addopts = "--ignore=scratch"

toshi_hazard_post-0.7.1/scripts/calc_time.py

@@ -0,0 +1,34 @@
+from pathlib import Path
+import sys
+
+def main(log_filepath):
+    log_filepath = Path(log_filepath)
+
+    times_jobtable = []
+    times_batchtable = []
+    times_calc = []
+    with log_filepath.open() as f:
+        for line in f.readlines():
+            if "starting" in line and "calculations with" in line:
+                nproc = int(line.split(' ')[-2])
+            elif "time to create batch table" in line:
+                times_batchtable.append(float(line.split(' ')[-2]))
+            elif "time to create job table" in line:
+                times_jobtable.append(float(line.split(' ')[-2]))
+            elif "time to perform one aggregation" in line:
+                times_calc.append(float(line.split(' ')[-2]))
+            elif "time to perform aggregations after job setup" in line:
+                total_time = float(line.split(' ')[-1])
+
+    print(f"Number of workers: {nproc}")
+    print(f"Total number of partitions used: {len(times_batchtable)}")
+    print(f"Total number of aggregate calculations: {len(times_calc)}")
+    print(f"Mean Partition Table Creation Time: {sum(times_batchtable)/len(times_batchtable):.02f} seconds")
+    print(f"Mean Job Table Creation Time: {sum(times_jobtable)/len(times_jobtable):.02f} seconds")
+    print(f"Mean Agg Calc Time: {sum(times_calc)/len(times_calc):.02f} seconds")
+    print(f"Total Time: {total_time} seconds")
+    print(f"Mean time per calculation (including I/O): {nproc * total_time/len(times_calc):.02f} CPU seconds")
+
+if __name__ == "__main__":
+    filepath = sys.argv[1]
+    main(filepath)

toshi_hazard_post-0.7.1/scripts/cli.py

@@ -0,0 +1,36 @@
+import logging
+import os
+
+import click
+
+from toshi_hazard_post.aggregation import run_aggregation
+from toshi_hazard_post.aggregation_args import load_input_args
+import toshi_hazard_store
+
+logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logging.getLogger('toshi_hazard_post').setLevel(logging.INFO)
+# logging.getLogger('toshi_hazard_post.aggregation_calc').setLevel(logging.DEBUG)
+# logging.getLogger('toshi_hazard_post.aggregation').setLevel(logging.DEBUG)
+# logging.getLogger('toshi_hazard_post.aggregation_calc').setLevel(logging.DEBUG)
+# logging.getLogger('toshi_hazard_post.logic_tree').setLevel(logging.DEBUG)
+logging.getLogger('toshi_hazard_store').setLevel(logging.ERROR)
+logging.getLogger('toshi_hazard_post').setLevel(logging.INFO)
+
+
+@click.group()
+def thp():
+    pass
+
+
+@thp.command(name='aggregate', help='aggregate hazard curves')
+@click.argument('input_file', type=click.Path(exists=True))
+def aggregate(input_file):
+
+    args = load_input_args(input_file)
+    click.echo("Toshi Hazard Post: hazard curve aggregation")
+    click.echo("=================================================")
+    run_aggregation(args)
+
+
+if __name__ == "__main__":
+    thp()  # pragma: no cover

toshi_hazard_post-0.7.1/scripts/cli_old.py

@@ -0,0 +1,213 @@
+"""Console script for toshi_hazard_post."""
+
+import logging
+import os
+import sys
+
+import click
+import toml
+from toshi_hazard_store.model import migrate as migrate_ths
+
+# from toshi_hazard_post.hazard_aggregation import AggregationConfig, process_aggregation, process_deaggregation
+from toshi_hazard_post.hazard_aggregation.aggregation import process_aggregation
+from toshi_hazard_post.hazard_aggregation.aggregation_config import AggregationConfig
+from toshi_hazard_post.hazard_aggregation.aws_aggregation import distribute_aggregation, push_test_message
+from toshi_hazard_post.hazard_aggregation.aws_deaggregation import distribute_deaggregation
+from toshi_hazard_post.hazard_aggregation.deaggregation import process_deaggregation
+from toshi_hazard_post.hazard_grid.aws_gridded_hazard import distribute_gridded_hazard
+from toshi_hazard_post.hazard_grid.gridded_hazard import calc_gridded_hazard
+from toshi_hazard_post.hazard_grid.misc import get_filter_locations, get_site_lists, migrate
+
+log = logging.getLogger()
+logging.basicConfig(level=logging.INFO)
+logging.getLogger('nshm_toshi_client.toshi_client_base').setLevel(logging.INFO)
+logging.getLogger('urllib3').setLevel(logging.INFO)
+logging.getLogger('botocore').setLevel(logging.INFO)
+logging.getLogger('pynamodb').setLevel(logging.INFO)
+logging.getLogger('toshi_hazard_post').setLevel(logging.INFO)
+logging.getLogger('toshi_hazard_store').setLevel(logging.INFO)
+logging.getLogger('gql.transport.requests').setLevel(logging.WARN)
+
+formatter = logging.Formatter(fmt='%(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
+screen_handler = logging.StreamHandler(stream=sys.stdout)
+screen_handler.setFormatter(formatter)
+log.addHandler(screen_handler)
+
+
+@click.group()
+def thp():
+    pass
+
+
+@thp.command(name='build-grid', help='calculate hazard grids used to create maps')
+@click.option('-H', '--hazard_model_ids', help='comma-delimted list of hazard model ids.')
+@click.option('-L', '--site-list', help='A site list ENUM.')
+@click.option('-I', '--imts', help='comma-delimited list of imts.')
+@click.option('-A', '--aggs', help='comma-delimited list of aggs.')
+@click.option('-V', '--vs30s', help='comma-delimited list of vs30s.')
+@click.option('-P', '--poes', help='comma-delimited list of poe_levels.')
+@click.option('-c', '--config', type=click.Path(exists=True))  # help="path to a valid THU configuration file."
+@click.option('-lsl', '--list-site-lists', help='print the list of sites list ENUMs and exit', is_flag=True)
+@click.option('-v', '--verbose', is_flag=True)
+@click.option('-d', '--dry-run', is_flag=True)
+@click.option('-m', '--migrate-tables', is_flag=True)
+@click.option('-f', '--force', is_flag=True)
+@click.option(
+    '--mode',
+    '-m',
+    default='LOCAL',
+    type=click.Choice(['AWS_BATCH', 'LOCAL'], case_sensitive=True),
+)
+@click.option(
+    '--iter-method',
+    '-i',
+    default='product',
+    type=click.Choice(['product', 'zip'], case_sensitive=True),
+)
+@click.option('-w', '--num-workers', default=4, show_default=True)
+def cli_gridded_hazard(
+    hazard_model_ids,
+    site_list,
+    imts,
+    aggs,
+    vs30s,
+    poes,
+    config,
+    list_site_lists,
+    verbose,
+    dry_run,
+    migrate_tables,
+    force,
+    mode,
+    iter_method,
+    num_workers,
+):
+    """Process gridded hazard for a given set of arguments."""
+
+    if list_site_lists:
+        click.echo("ENUM name\tDetails")
+        click.echo("===============\t======================================================================")
+        for rg in get_site_lists():
+            click.echo(f"{rg.name}\t{rg.value}")
+        return
+
+    # site_lists = site_lists.split(',') if site_lists else None
+
+    hazard_model_ids = hazard_model_ids.split(',') if hazard_model_ids else None
+    vs30s = vs30s.split(',') if vs30s else None
+    imts = imts.split(',') if imts else None
+    aggs = aggs.split(',') if aggs else None
+    poes = poes.split(',') if poes else None
+    filter_sites = None
+
+    if config:
+        conf = toml.load(config)
+        if verbose:
+            click.echo(f"using settings in {config} for export")
+
+        site_list = site_list or conf.get('site_list')
+        hazard_model_ids = hazard_model_ids or conf.get('hazard_model_ids')
+        imts = imts or conf.get('imts')
+        vs30s = vs30s or conf.get('vs30s')
+        aggs = aggs or conf.get('aggs')
+        poes = poes or conf.get('poes')
+        filter_sites = filter_sites or conf.get('filter_sites')
+
+    if verbose:
+        click.echo(f"{hazard_model_ids} {imts} {vs30s}")
+
+    if dry_run:
+        click.echo(f"dry-run {site_list} {hazard_model_ids} {imts} {vs30s}")
+        return
+
+    if migrate_tables:
+        click.echo("Ensuring that dynamodb tables are available in target region & stage.")
+        migrate()
+
+    if mode == "LOCAL":
+        try:
+            click.echo(filter_sites)
+            filter_locations = get_filter_locations(filter_sites)
+            click.echo(filter_locations)
+            calc_gridded_hazard(
+                location_grid_id=site_list,
+                poe_levels=poes,
+                hazard_model_ids=hazard_model_ids,
+                vs30s=vs30s,
+                imts=imts,
+                aggs=aggs,
+                num_workers=num_workers,
+                force=force,
+                filter_locations=filter_locations,
+                iter_method=iter_method,
+            )
+        except Exception as err:
+            click.echo(err)
+            raise click.UsageError('An error occurred, pls check usage.')
+
+        # haggs = query_v3.get_hazard_curves(locations, vs30s, hazard_model_ids, imts=imts, aggs=aggs)
+        click.echo('Done!')
+    elif mode == "AWS_BATCH":
+        try:
+            click.echo(filter_sites)
+            filter_locations = get_filter_locations(filter_sites)
+            click.echo(filter_locations)
+            distribute_gridded_hazard(
+                location_grid_id=site_list,
+                poe_levels=poes,
+                hazard_model_ids=hazard_model_ids,
+                vs30s=vs30s,
+                imts=imts,
+                aggs=aggs,
+                force=force,
+                filter_locations=filter_locations,
+                iter_method=iter_method,
+            )
+        except Exception as err:
+            click.echo(err)
+            raise click.UsageError('An error occurred, pls check usage.')
+
+
+@thp.command(name='aggregate', help='aggregate hazard curves or disaggregations')
+@click.option(
+    '--mode',
+    '-m',
+    default=lambda: os.environ.get("NZSHM22_THP_MODE", 'LOCAL'),
+    type=click.Choice(['AWS', 'AWS_BATCH', 'LOCAL'], case_sensitive=True),
+)
+@click.option('--deagg', '-d', is_flag=True)
+@click.option('--push-sns-test', '-pt', is_flag=True)
+@click.option('--migrate-tables', '-M', is_flag=True)
+@click.argument('config', type=click.Path(exists=True))  # help="path to a valid THP configuration file."
+def aggregate(config, mode, deagg, push_sns_test, migrate_tables):
+    """Main entrypoint."""
+    click.echo("Hazard post-processing pipeline as serverless AWS infrastructure.")
+    click.echo(f"mode: {mode}")
+
+    agconf = AggregationConfig(config)
+    # click.echo(agconf)
+    # log.info("Doit")
+
+    if push_sns_test:
+        push_test_message()
+        return
+
+    if mode == 'LOCAL':
+        if deagg:
+            process_deaggregation(agconf)
+        else:
+            process_aggregation(agconf)
+        return
+    if 'AWS_BATCH' in mode:  # TODO: multiple vs30s
+        if migrate_tables:
+            click.echo("Ensuring that dynamodb tables are available in target region & stage.")
+            migrate_ths()
+        if deagg:
+            distribute_deaggregation(agconf, mode)
+        else:
+            distribute_aggregation(agconf, mode)
+        return
+
+
+if __name__ == "__main__":
+    thp()  # pragma: no cover

toshi_hazard_post-0.7.1/scripts/container_task.sh

@@ -0,0 +1,5 @@
+# This is the AWS CONTAINER SCRIPT
+
+cd /app
+python3 -m ${PYTHON_TASK_MODULE} ${TASK_CONFIG_JSON_QUOTED}
+#END_OF_SCRIPT

toshi_hazard_post-0.7.1/tests/__init__.py

@@ -0,0 +1 @@
+"""The tests package."""

toshi_hazard_post-0.7.1/tests/fixtures/calculators/rates.json

@@ -0,0 +1 @@
+[0.019156457392332, 0.040571554784402, 0.064849989807042, 0.092877386139271, 0.126026760101808, 0.16659831488621, 0.218904146241079, 0.292625074988018, 0.418651835089826]

toshi_hazard_post-0.7.1/tests/fixtures/calculators/weights_and_values.json

@@ -0,0 +1,24 @@
+{
+    "weights": [
+        0.447077835475027,
+        0.058470204772802,
+        0.869074768834212,
+        0.77857878253347,
+        0.588413020406674,
+        0.849791015005617,
+        0.743044887843583,
+        0.273707026202851,
+        0.82405809555373
+    ],
+    "values": [
+        6.90142303832663,
+        1.54115132087442,
+        1.64338018395884,
+        4.82202052695663,
+        9.43809367161881,
+        1.0831467357305,
+        6.01064665810539,
+        6.51800796017942,
+        3.98688564815937
+    ]
+}