azul-runner 9.0.38__tar.gz

azul_runner-9.0.38/.github/CODEOWNERS

@@ -0,0 +1 @@
+* @AustralianCyberSecurityCentre/azul-admins

azul_runner-9.0.38/.gitignore

@@ -0,0 +1,247 @@
+docker/tmp/
+
+# Created by https://www.gitignore.io/api/vim,python,pycharm
+# Edit at https://www.gitignore.io/?templates=vim,python,pycharm
+
+### PyCharm ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+.idea
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### PyCharm Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+# .idea/misc.xml
+# *.ipr
+
+# Sonarlint plugin
+.idea/sonarlint
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+### Python Patch ###
+.venv/
+
+### Vim ###
+# Swap
+[._]*.s[a-v][a-z]
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+
+# Temporary
+.netrwhist
+*~
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
+
+# End of https://www.gitignore.io/api/vim,python,pycharm
+
+# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode
+# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode
+
+### VisualStudioCode ###
+.vscode/*
+#!.vscode/settings.json
+#!.vscode/tasks.json
+#!.vscode/launch.json
+#!.vscode/extensions.json
+*.code-workspace
+
+# Local History for Visual Studio Code
+.history/
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+
+# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode

azul_runner-9.0.38/PKG-INFO

@@ -0,0 +1,178 @@
+Metadata-Version: 2.4
+Name: azul-runner
+Version: 9.0.38
+Summary: Core framework for writing Python plugins for Azul.
+Home-page: https://www.asd.gov.au/
+Author: Azul
+Author-email: azul@asd.gov.au
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: azul-bedrock
+Requires-Dist: bitarray>=0.9
+Requires-Dist: cart
+Requires-Dist: httpx
+Requires-Dist: pydantic>2
+Requires-Dist: pydantic-settings
+Requires-Dist: watchdog
+Requires-Dist: psutil
+Provides-Extra: test-utils
+Requires-Dist: azul-bedrock[test_utils]; extra == "test-utils"
+Requires-Dist: black; extra == "test-utils"
+Requires-Dist: fastapi; extra == "test-utils"
+Requires-Dist: pytest; extra == "test-utils"
+Requires-Dist: tox; extra == "test-utils"
+Requires-Dist: uvicorn[standard]; extra == "test-utils"
+Requires-Dist: pytest-timeout; extra == "test-utils"
+Requires-Dist: yara-x; extra == "test-utils"
+Requires-Dist: urllib3; extra == "test-utils"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Azul Runner
+
+Core framework for writing Python plugins for Azul.
+
+It handles filtering and fetching events, setting up data streams, and
+parsing and validating plugin results before posting output events
+back to the dispatcher.
+
+Additionally, it allows plugins to be run
+locally from the commandline, with local files or folders as input data.
+
+## Usage
+
+To run a plugin (after installing azul-runner and the plugin as Python packages):
+
+```bash
+# against local samples
+$ azul-plugin-(name) local_file.bin
+$ azul-plugin-(name) local_folder/
+
+# against remote dispatcher
+$ azul-plugin-(name) --server http://server.address/path
+
+# with custom config
+$ azul-plugin-(name) -c KEY VALUE -c KEY2 VALUE2
+```
+
+Plugins may define multiple entrypoints, check the plugins setup.py for details.
+
+Plugin configuration may be passed in as environment variables.
+
+Check ./azul_runner/settings.py for specific options
+
+- PLUGIN_RUN_TIMEOUT=600
+- PLUGIN_SERVER=http://widgets.co.internal
+
+## Plugin Development
+
+Use the project `azure-generator` to generate boilerplate code for a new plugin.
+
+For most common use cases, inherit from `azul_runner.BinaryPlugin` and implement the `execute` method.
+It is recommended to look at existing plugins in order to implement your plugin.
+You can also check binary_plugin.py and plugin.py for more properties that can be configured.
+
+Advanced use cases may inherit from `azul_runner.Plugin` instead.
+
+For more documentation on development process, see [here](./docs/index.md)
+
+For more documentation on specific api usage and plugin code, see [here](./docs/coding/index.md)
+
+See [structure](./docs/structure.md) for more information about the structure of azul-runner.
+
+See [migration guide](./docs/migration.md) to update your plugin in line with api changes in azul-runner.
+
+## Example Plugin
+
+```python
+from azul_runner import BinaryPlugin, Feature, Job, cmdline_run, State, FeatureType
+
+
+class LookForThings(BinaryPlugin):
+    """Look for things."""
+
+    VERSION = "1.0"
+    SETTINGS = add_settings(
+      # You can filter assemblyline file types as below.
+      # These are the files that your plugin will process.
+      # Note: you can filter by prefix (document/).
+      # Check azul-bedrock/identify.yaml for valid file types.
+        filter_data_types={
+            "content": [
+                "text/plain",
+                "document/",
+            ]
+        }
+    )
+    FEATURES = [
+        Feature("tag", "Custom tag", FeatureType.String),
+    ]
+
+    def execute(self, job: Job):
+        """Find peanuts."""
+        data = job.get_data()
+        # 'data' is a file-like object that supports seeking and being read from
+        # (The content may be retrieved in parts if the file is large and non-local)
+        header: bytes = data.read(7)
+        if header == b"PEANUT:":
+            # create a tag
+            self.add_feature_values("tag", "may contain nuts")
+            # add the next 24 bytes as a child
+            c = self.add_child_with_data(
+                relationship={"label": "peanut"},
+                data=data.read(24),
+            )
+            c.add_feature_values("tag", "may be hard to crack")
+        else:
+            return State.Label.OPT_OUT
+
+if __name__ == '__main__':
+    cmdline_run(plugin=LookForThings)
+```
+
+## Execution
+
+As the example plugin was not constructed in `azure-generator`, running them requires directly executing
+the script they are contained in. A copy can be found in `tests/example_plugins.py`.
+
+```bash
+azul-runner$ python tests/example_plugins.py tests/data/peanut.txt
+----- LookForThings results -----
+COMPLETED
+
+events (2)
+
+event for cmdline_entity:None
+  {}
+  output features:
+    tag: may contain nuts
+
+event for 7c4cd5274277dde41aa3f5e06cfca8c6cc703951a642b6c268cb43a2b345780a:None
+  {'label': 'peanut'}
+  child of cmdline_entity
+  output data streams (1):
+    24 bytes - EventData(hash='7c4cd5274277dde41aa3f5e06cfca8c6cc703951a642b6c268cb43a2b345780a', label='content')
+  output features:
+    tag: may be hard to crack
+
+Feature key:
+  tag:  Custom tag
+
+```

azul_runner-9.0.38/README.md

@@ -0,0 +1,131 @@
+# Azul Runner
+
+Core framework for writing Python plugins for Azul.
+
+It handles filtering and fetching events, setting up data streams, and
+parsing and validating plugin results before posting output events
+back to the dispatcher.
+
+Additionally, it allows plugins to be run
+locally from the commandline, with local files or folders as input data.
+
+## Usage
+
+To run a plugin (after installing azul-runner and the plugin as Python packages):
+
+```bash
+# against local samples
+$ azul-plugin-(name) local_file.bin
+$ azul-plugin-(name) local_folder/
+
+# against remote dispatcher
+$ azul-plugin-(name) --server http://server.address/path
+
+# with custom config
+$ azul-plugin-(name) -c KEY VALUE -c KEY2 VALUE2
+```
+
+Plugins may define multiple entrypoints, check the plugins setup.py for details.
+
+Plugin configuration may be passed in as environment variables.
+
+Check ./azul_runner/settings.py for specific options
+
+- PLUGIN_RUN_TIMEOUT=600
+- PLUGIN_SERVER=http://widgets.co.internal
+
+## Plugin Development
+
+Use the project `azure-generator` to generate boilerplate code for a new plugin.
+
+For most common use cases, inherit from `azul_runner.BinaryPlugin` and implement the `execute` method.
+It is recommended to look at existing plugins in order to implement your plugin.
+You can also check binary_plugin.py and plugin.py for more properties that can be configured.
+
+Advanced use cases may inherit from `azul_runner.Plugin` instead.
+
+For more documentation on development process, see [here](./docs/index.md)
+
+For more documentation on specific api usage and plugin code, see [here](./docs/coding/index.md)
+
+See [structure](./docs/structure.md) for more information about the structure of azul-runner.
+
+See [migration guide](./docs/migration.md) to update your plugin in line with api changes in azul-runner.
+
+## Example Plugin
+
+```python
+from azul_runner import BinaryPlugin, Feature, Job, cmdline_run, State, FeatureType
+
+
+class LookForThings(BinaryPlugin):
+    """Look for things."""
+
+    VERSION = "1.0"
+    SETTINGS = add_settings(
+      # You can filter assemblyline file types as below.
+      # These are the files that your plugin will process.
+      # Note: you can filter by prefix (document/).
+      # Check azul-bedrock/identify.yaml for valid file types.
+        filter_data_types={
+            "content": [
+                "text/plain",
+                "document/",
+            ]
+        }
+    )
+    FEATURES = [
+        Feature("tag", "Custom tag", FeatureType.String),
+    ]
+
+    def execute(self, job: Job):
+        """Find peanuts."""
+        data = job.get_data()
+        # 'data' is a file-like object that supports seeking and being read from
+        # (The content may be retrieved in parts if the file is large and non-local)
+        header: bytes = data.read(7)
+        if header == b"PEANUT:":
+            # create a tag
+            self.add_feature_values("tag", "may contain nuts")
+            # add the next 24 bytes as a child
+            c = self.add_child_with_data(
+                relationship={"label": "peanut"},
+                data=data.read(24),
+            )
+            c.add_feature_values("tag", "may be hard to crack")
+        else:
+            return State.Label.OPT_OUT
+
+if __name__ == '__main__':
+    cmdline_run(plugin=LookForThings)
+```
+
+## Execution
+
+As the example plugin was not constructed in `azure-generator`, running them requires directly executing
+the script they are contained in. A copy can be found in `tests/example_plugins.py`.
+
+```bash
+azul-runner$ python tests/example_plugins.py tests/data/peanut.txt
+----- LookForThings results -----
+COMPLETED
+
+events (2)
+
+event for cmdline_entity:None
+  {}
+  output features:
+    tag: may contain nuts
+
+event for 7c4cd5274277dde41aa3f5e06cfca8c6cc703951a642b6c268cb43a2b345780a:None
+  {'label': 'peanut'}
+  child of cmdline_entity
+  output data streams (1):
+    24 bytes - EventData(hash='7c4cd5274277dde41aa3f5e06cfca8c6cc703951a642b6c268cb43a2b345780a', label='content')
+  output features:
+    tag: may be hard to crack
+
+Feature key:
+  tag:  Custom tag
+
+```

azul_runner-9.0.38/azul_runner/__init__.py

@@ -0,0 +1,58 @@
+# flake8: noqa - Flake8 doesn't detect these exported functions and classes properly.
+import contextlib
+
+from azul_bedrock.models_network import DataLabel, FeatureType
+from azul_bedrock.models_network import FeatureValue as APIFeatureValue
+
+from .binary_plugin import BinaryPlugin
+from .main import cmdline_run
+from .models import (
+    FV,
+    Event,
+    EventData,
+    EventParent,
+    Feature,
+    FeatureValue,
+    Filepath,
+    Job,
+    JobResult,
+    State,
+    Uri,
+)
+from .plugin import Plugin
+from .settings import add_settings
+from .storage import DATA_HASH, DATA_HASH_NAME, StorageProxyFile
+
+append_all: list[str] = []
+with contextlib.suppress(ImportError):
+    from .test_utils import test_template
+    from .test_utils.test_template import TestPlugin
+
+    print("You have the azul-runner[test_utils] installed, this should only be used for development or testing.")
+    append_all = ["TestPlugin", "test_template"]
+
+EXPORTS = append_all + [
+    "add_settings",
+    "APIFeatureValue",
+    "BinaryPlugin",
+    "cmdline_run",
+    "DATA_HASH_NAME",
+    "DATA_HASH",
+    "DataLabel",
+    "Event",
+    "EventData",
+    "EventParent",
+    "Feature",
+    "FeatureType",
+    "FeatureValue",
+    "Filepath",
+    "FV",
+    "Job",
+    "JobResult",
+    "Plugin",
+    "State",
+    "StorageProxyFile",
+    "Uri",
+]
+
+__all__ = list(EXPORTS)

azul_runner-9.0.38/azul_runner/binary_plugin.py

@@ -0,0 +1,22 @@
+"""Subclass of Plugin designed for processing binary files."""
+
+from azul_bedrock import models_network
+
+from . import plugin, settings
+
+
+class BinaryPlugin(plugin.Plugin):
+    """Base class for plugins that want to run on new sightings of a binary.
+
+    See readme.md for usage information.
+    """
+
+    SETTINGS = settings.add_settings(
+        # plugin code generally assumes it can access data
+        assume_streams_available=True,
+        # event must be a new sighting of a label='content' binary
+        filter_allow_event_types=[
+            models_network.BinaryAction.Sourced,
+            models_network.BinaryAction.Extracted,
+        ],
+    )