azul-bedrock 9.0.32__tar.gz

azul_bedrock-9.0.32/.editorconfig

@@ -0,0 +1,25 @@
+# Check http://editorconfig.org for more information
+# This is the main config file for this project:
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 2
+trim_trailing_whitespace = true
+
+[*.{py, pyi}]
+indent_style = space
+indent_size = 4
+max_line_length = 119
+
+[Makefile]
+indent_style = tab
+
+[*.md]
+trim_trailing_whitespace = true
+
+[*.{diff,patch}]
+trim_trailing_whitespace = false

azul_bedrock-9.0.32/.github/CODEOWNERS

@@ -0,0 +1 @@
+* @AustralianCyberSecurityCentre/azul-admins

azul_bedrock-9.0.32/.gitignore

@@ -0,0 +1,274 @@
+yara/
+# Created by https://www.toptal.com/developers/gitignore/api/python,vim,visualstudiocode,intellij+all
+# Edit at https://www.toptal.com/developers/gitignore?templates=python,vim,visualstudiocode,intellij+all
+
+### Intellij+all ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### Intellij+all Patch ###
+# Ignores the whole .idea folder and all .iml files
+# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360
+
+.idea/
+
+# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023
+
+*.iml
+modules.xml
+.idea/misc.xml
+*.ipr
+
+# Sonarlint plugin
+.idea/sonarlint
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+### Vim ###
+# Swap
+[._]*.s[a-v][a-z]
+!*.svg  # comment out if you don't need vector files
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+Sessionx.vim
+
+# Temporary
+.netrwhist
+*~
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
+
+### VisualStudioCode ###
+.vscode/*
+#!.vscode/settings.json
+#!.vscode/tasks.json
+#!.vscode/launch.json
+#!.vscode/extensions.json
+*.code-workspace
+
+# Local History for Visual Studio Code
+.history/
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+
+# End of https://www.toptal.com/developers/gitignore/api/python,vim,visualstudiocode,intellij+all

azul_bedrock-9.0.32/.golangci.yml

@@ -0,0 +1,32 @@
+version: "2"
+run:
+  tests: false
+linters:
+  exclusions:
+    generated: lax
+    presets:
+      - comments
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+    rules:
+      # Ignore forcing switch statements everywhere there is an if else statement
+      - linters:
+          - staticcheck
+        text: "QF1003:"
+formatters:
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+  enable:
+    # - gci
+    - gofmt
+    # - gofumpt
+    - goimports

azul_bedrock-9.0.32/.mockery.yaml

@@ -0,0 +1,23 @@
+with-expecter: true
+packages:
+  github.com/AustralianCyberSecurityCentre/azul-bedrock/v9/gosrc/store:
+    # place your package-specific config here
+    config:
+      inpackage: true
+      dir: "{{.InterfaceDir}}"
+    interfaces:
+      # select the interfaces you want mocked
+      FileStorage:
+        # Modify package-level config for this specific interface (if applicable)
+        config:
+          filename: store_mock.go
+  github.com/AustralianCyberSecurityCentre/azul-bedrock/v9/gosrc/client:
+    # place your package-specific config here
+    config:
+      inpackage: true
+      dir: "{{.InterfaceDir}}"
+    interfaces:
+      # select the interfaces you want mocked
+      ClientInterface:
+        # Modify package-level config for this specific interface (if applicable)
+        config:

azul_bedrock-9.0.32/PKG-INFO

@@ -0,0 +1,197 @@
+Metadata-Version: 2.4
+Name: azul-bedrock
+Version: 9.0.32
+Summary: Shared library for Azul models and operations.
+Home-page: https://www.asd.gov.au/
+Author: Azul
+Author-email: azul@asd.gov.au
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: cachetools
+Requires-Dist: fastapi
+Requires-Dist: httpx
+Requires-Dist: multipart
+Requires-Dist: pendulum
+Requires-Dist: pydantic
+Requires-Dist: python-magic>=0.4.15
+Requires-Dist: pyyaml
+Requires-Dist: starlette
+Requires-Dist: yara-python
+Provides-Extra: mock
+Requires-Dist: cart; extra == "mock"
+Requires-Dist: fastapi[all]>=0.60.0; extra == "mock"
+Requires-Dist: azure-storage-blob; extra == "mock"
+Requires-Dist: pytest; extra == "mock"
+Requires-Dist: tox; extra == "mock"
+Requires-Dist: urllib3; extra == "mock"
+Provides-Extra: test-utils
+Requires-Dist: cart; extra == "test-utils"
+Requires-Dist: fastapi[all]>=0.60.0; extra == "test-utils"
+Requires-Dist: azure-storage-blob; extra == "test-utils"
+Requires-Dist: pytest; extra == "test-utils"
+Requires-Dist: tox; extra == "test-utils"
+Requires-Dist: urllib3; extra == "test-utils"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Azul Restapi Common
+
+Shared library for Azul models and operations.
+
+## Design
+
+The models are not in a nested folder as this gives nicer import names.
+
+e.g.
+
+```python
+# much clearer naming of models on import, prevents confusion
+from azul_thingo import api
+from azul_bedrock import model_api
+
+api.send_thingo(model_api.Thingo())
+
+# annoying and depends on developer, likely will fragment into different names on import
+from azul_thingo import api
+from azul_bedrock.models import api as mapi
+
+api.send_thingo(mapi.Thingo())
+```
+
+## Requirements
+
+### libmagic
+
+# default libmagic for debian can get out of date
+contains a number of bugs for office and archive file types
+
+```bash
+git clone --depth 1 --branch FILE5_46 https://github.com/file/file
+cd file/
+autoreconf -f -i
+./configure --disable-silent-rules
+make -j4
+sudo make install
+sudo cp ./magic/magic.mgc /etc/magic
+cd -
+# check it worked
+file --version
+```
+
+### yara
+
+```bash
+sudo apt-get install automake libtool make gcc pkg-config git flex bison -y
+mkdir -p ./yara
+git clone --branch v4.3.2 https://github.com/VirusTotal/yara ./yara
+cd ./yara
+./bootstrap.sh
+./configure
+make
+sudo make install
+cd -
+## check it worked
+yara -v
+```
+
+```bash
+# you should put this in bashrc
+export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib"
+```
+
+## Install
+
+```bash
+pip install azul-bedrock
+```
+
+## Mocks
+
+Using Mockery to generate mocks for golang client.
+
+This is for use in upstream packages.
+
+```bash
+go install github.com/vektra/mockery/v2@v2.53.5
+mockery
+```
+
+## Test Files
+
+To run tests based off of bedrock you will need to set environment variables to allow you to download and use test files.
+
+This requires you to be able to download files from Virustotal.
+To save on download quotas files are also cached to an S3 cluster and your local file system.
+
+The full set of environment variables for configuring this setup are here:
+
+```bash
+# Timeout for requests made by the file manager.
+export file_manager_request_timeout: int = 30
+# The URL to Virustotal's V3 API (guarantee no trailing slash).
+export file_manager_virustotal_api_url="https://www.virustotal.com/api/v3"
+# Virustotal API key used to download files from Virustotal
+export file_manager_virustotal_api_key = ""
+# whether to attempt to download files from virustotal or not.
+export file_manager_virustotal_enabled="True"
+# Directory where files are cached on the local file system when downloaded. (stored as carts)
+export file_manager_file_cache_dir="/var/tmp/azul"
+# Flag used to enable/disable the caching of test files.
+export file_manager_file_caching_enabled="True"
+# URL from Azure storage blob (storage account name address)
+export file_manager_azure_storage_account_address=""
+# Storage account Access key. (SAS key) used to access the azure storage.
+export file_manager_azure_storage_access_key=""
+# Name of the storage container within the blob storage.
+export file_manager_azure_container_name="azul-test-file-cache"
+# Flag used to enable/disable bucket caching.
+export file_manager_azure_blob_cache_enabled="True"
+```
+
+It is recommended when you set these environment variables you add them to your `~/.bashrc` as it makes it easier when running tests.
+
+## Avro Schema changes
+
+If you modify the Avro schema you need to ensure you consider legal schema changes to maintain backwards compatibility.
+
+Refer here for a guide: https://docs.confluent.io/platform/current/schema-registry/fundamentals/schema-evolution.html#compatibility-types
+
+Azul must maintain BACKWARD_TRANSITIVE compatibility at all times with it's avro models.
+This means fields can be deleted from the schema and OPTIONAL fields added.
+
+If this is not achievable for a change the model version must be incremented and an upgrade path for old events needs to
+be added to `msginflight/conversion_avro.go` for the model type that has been upgraded.
+
+Test cases for the upgrade path will needed to be added.
+
+Also if enough old versions build up the following release of Azul will need to require a kafka reprocess to be run
+and assuming that has been run the old upgrade code and previous schemas could then be cleared.
+
+## Integration tests
+
+To run the golang integration test suite the docker-compose.yaml file must first be used to stand up,
+a minio pod to act as an S3 datastore.
+
+With the command:
+
+` docker compose up`
+
+Integration tests should be run with the script `test_integration.sh` with the
+missing environment variables set to run the azure storage related tests.

azul_bedrock-9.0.32/README.md

@@ -0,0 +1,144 @@
+# Azul Restapi Common
+
+Shared library for Azul models and operations.
+
+## Design
+
+The models are not in a nested folder as this gives nicer import names.
+
+e.g.
+
+```python
+# much clearer naming of models on import, prevents confusion
+from azul_thingo import api
+from azul_bedrock import model_api
+
+api.send_thingo(model_api.Thingo())
+
+# annoying and depends on developer, likely will fragment into different names on import
+from azul_thingo import api
+from azul_bedrock.models import api as mapi
+
+api.send_thingo(mapi.Thingo())
+```
+
+## Requirements
+
+### libmagic
+
+# default libmagic for debian can get out of date
+contains a number of bugs for office and archive file types
+
+```bash
+git clone --depth 1 --branch FILE5_46 https://github.com/file/file
+cd file/
+autoreconf -f -i
+./configure --disable-silent-rules
+make -j4
+sudo make install
+sudo cp ./magic/magic.mgc /etc/magic
+cd -
+# check it worked
+file --version
+```
+
+### yara
+
+```bash
+sudo apt-get install automake libtool make gcc pkg-config git flex bison -y
+mkdir -p ./yara
+git clone --branch v4.3.2 https://github.com/VirusTotal/yara ./yara
+cd ./yara
+./bootstrap.sh
+./configure
+make
+sudo make install
+cd -
+## check it worked
+yara -v
+```
+
+```bash
+# you should put this in bashrc
+export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib"
+```
+
+## Install
+
+```bash
+pip install azul-bedrock
+```
+
+## Mocks
+
+Using Mockery to generate mocks for golang client.
+
+This is for use in upstream packages.
+
+```bash
+go install github.com/vektra/mockery/v2@v2.53.5
+mockery
+```
+
+## Test Files
+
+To run tests based off of bedrock you will need to set environment variables to allow you to download and use test files.
+
+This requires you to be able to download files from Virustotal.
+To save on download quotas files are also cached to an S3 cluster and your local file system.
+
+The full set of environment variables for configuring this setup are here:
+
+```bash
+# Timeout for requests made by the file manager.
+export file_manager_request_timeout: int = 30
+# The URL to Virustotal's V3 API (guarantee no trailing slash).
+export file_manager_virustotal_api_url="https://www.virustotal.com/api/v3"
+# Virustotal API key used to download files from Virustotal
+export file_manager_virustotal_api_key = ""
+# whether to attempt to download files from virustotal or not.
+export file_manager_virustotal_enabled="True"
+# Directory where files are cached on the local file system when downloaded. (stored as carts)
+export file_manager_file_cache_dir="/var/tmp/azul"
+# Flag used to enable/disable the caching of test files.
+export file_manager_file_caching_enabled="True"
+# URL from Azure storage blob (storage account name address)
+export file_manager_azure_storage_account_address=""
+# Storage account Access key. (SAS key) used to access the azure storage.
+export file_manager_azure_storage_access_key=""
+# Name of the storage container within the blob storage.
+export file_manager_azure_container_name="azul-test-file-cache"
+# Flag used to enable/disable bucket caching.
+export file_manager_azure_blob_cache_enabled="True"
+```
+
+It is recommended when you set these environment variables you add them to your `~/.bashrc` as it makes it easier when running tests.
+
+## Avro Schema changes
+
+If you modify the Avro schema you need to ensure you consider legal schema changes to maintain backwards compatibility.
+
+Refer here for a guide: https://docs.confluent.io/platform/current/schema-registry/fundamentals/schema-evolution.html#compatibility-types
+
+Azul must maintain BACKWARD_TRANSITIVE compatibility at all times with it's avro models.
+This means fields can be deleted from the schema and OPTIONAL fields added.
+
+If this is not achievable for a change the model version must be incremented and an upgrade path for old events needs to
+be added to `msginflight/conversion_avro.go` for the model type that has been upgraded.
+
+Test cases for the upgrade path will needed to be added.
+
+Also if enough old versions build up the following release of Azul will need to require a kafka reprocess to be run
+and assuming that has been run the old upgrade code and previous schemas could then be cleared.
+
+## Integration tests
+
+To run the golang integration test suite the docker-compose.yaml file must first be used to stand up,
+a minio pod to act as an S3 datastore.
+
+With the command:
+
+` docker compose up`
+
+Integration tests should be run with the script `test_integration.sh` with the
+missing environment variables set to run the azure storage related tests.