howler-client 3.1.0.dev404__tar.gz → 3.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- howler_client-3.2.0/PKG-INFO +293 -0
- howler_client-3.2.0/README.md +260 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/__init__.py +11 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/client.py +2 -0
- howler_client-3.2.0/howler_client/common/exceptions.py +2 -0
- howler_client-3.2.0/howler_client/module/mapping.py +313 -0
- howler_client-3.2.0/howler_client/schemas/__init__.py +1 -0
- howler_client-3.2.0/howler_client/schemas/row_mapping_v1.schema.json +130 -0
- howler_client-3.2.0/howler_client/utils/transforms.py +135 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/pyproject.toml +7 -2
- howler_client-3.1.0.dev404/PKG-INFO +0 -73
- howler_client-3.1.0.dev404/README.md +0 -42
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/LICENSE +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/common/__init__.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/common/dict_utils.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/common/utils.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/connection.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/logger.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/__init__.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/action.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/comment.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/help.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/hit.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/search/__init__.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/search/chunk.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/search/facet.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/search/fields.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/search/grouped.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/search/histogram.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/search/stats.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/search/stream.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/user.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/v2/__init__.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/v2/case.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/v2/ingest.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/module/v2/search.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/utils/__init__.py +0 -0
- {howler_client-3.1.0.dev404 → howler_client-3.2.0}/howler_client/utils/json_encoders.py +0 -0
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: howler-client
|
|
3
|
+
Version: 3.2.0
|
|
4
|
+
Summary: The Howler client library facilitates issuing requests to Howler
|
|
5
|
+
License: MIT
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Keywords: howler,alerting,gc,canada,cse-cst,cse,cst,cyber,cccs
|
|
8
|
+
Author: Canadian Centre for Cyber Security
|
|
9
|
+
Author-email: howler@cyber.gc.ca
|
|
10
|
+
Requires-Python: >=3.10,<4.0
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
22
|
+
Requires-Dist: coverage[toml] (>=7.6.1,<8.0.0)
|
|
23
|
+
Requires-Dist: jsonschema (>=4.23,<5.0)
|
|
24
|
+
Requires-Dist: pycryptodome (>=3.20.0,<4.0.0)
|
|
25
|
+
Requires-Dist: python-baseconv (>=1.2.2,<2.0.0)
|
|
26
|
+
Requires-Dist: pyyaml (>=6.0,<7)
|
|
27
|
+
Requires-Dist: requests[security] (>=2.32.0,<3.0.0)
|
|
28
|
+
Project-URL: Documentation, https://cybercentrecanada.github.io/howler/developer/client/
|
|
29
|
+
Project-URL: Homepage, https://cybercentrecanada.github.io/howler/
|
|
30
|
+
Project-URL: Repository, https://github.com/CybercentreCanada/howler-client
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
|
|
33
|
+
# Howler Client Library
|
|
34
|
+
|
|
35
|
+
`howler-client` is the Python client for the Howler API. Use it to:
|
|
36
|
+
|
|
37
|
+
- create, search, update, and delete hits
|
|
38
|
+
- manage comments, actions, users, and cases
|
|
39
|
+
- turn raw table rows into hit records offline, before you ingest them
|
|
40
|
+
|
|
41
|
+
It handles authentication, request formatting, retries, and response parsing.
|
|
42
|
+
|
|
43
|
+
- **Package:** [howler-client on PyPI](https://pypi.org/project/howler-client/)
|
|
44
|
+
- **User guide:** [Howler client documentation](https://cybercentrecanada.github.io/howler/ingestion/client/)
|
|
45
|
+
- **Contributor guide:** [Client development](https://cybercentrecanada.github.io/howler/developer/client/)
|
|
46
|
+
|
|
47
|
+
## Installation
|
|
48
|
+
|
|
49
|
+
The client requires Python 3.10 or newer.
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pip install howler-client
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Connecting to Howler
|
|
56
|
+
|
|
57
|
+
`get_client()` returns a `Client` connected to a Howler server. It checks that the server supports the v1 API when
|
|
58
|
+
it connects, so the server must be reachable.
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
from howler_client import get_client
|
|
62
|
+
|
|
63
|
+
# API key (recommended): create one in the Howler UI under Settings -> User Security -> API Keys
|
|
64
|
+
client = get_client("https://howler.example.com", apikey=("my_username", "my_key_name:my_key_secret"))
|
|
65
|
+
|
|
66
|
+
# JWT, for example one obtained through an OAuth flow
|
|
67
|
+
client = get_client("https://howler.example.com", auth="eyJhbGciOi...")
|
|
68
|
+
|
|
69
|
+
# Username and password (only if the server allows internal authentication)
|
|
70
|
+
client = get_client("https://howler.example.com", auth=("my_username", "my_password"))
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Other useful `get_client()` options:
|
|
74
|
+
|
|
75
|
+
- `verify`: whether to verify TLS certificates.
|
|
76
|
+
- `cert`: a client certificate.
|
|
77
|
+
- `timeout`: the request timeout.
|
|
78
|
+
- `retries`: how many times to retry a request. The default `0` retries forever.
|
|
79
|
+
- `throw_on_bad_request` and `throw_on_max_retries`: whether to raise `ClientError` on failures.
|
|
80
|
+
- `authenticate`: a callback that returns a bearer token.
|
|
81
|
+
|
|
82
|
+
## What the client covers
|
|
83
|
+
|
|
84
|
+
| Attribute | Purpose |
|
|
85
|
+
| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
86
|
+
| `client.hit` | Create hits (`create`), fetch one by ID (`client.hit(hit_id)`), `overwrite`, `update`, `update_by_query`, `delete`, `generate_hash`. Comments are under `client.hit.comment` (`add`, `edit`, `delete`). |
|
|
87
|
+
| `client.search` | Lucene search over hits (`hit`) and actions (`action`), plus `facet`, `fields`, `grouped`, `histogram`, `stats`, and `stream` helpers, each with a `.hit(...)` method. |
|
|
88
|
+
| `client.action` | List, create, and execute saved actions. |
|
|
89
|
+
| `client.user` | Add, delete, list, and update users; `whoami`. |
|
|
90
|
+
| `client.help` | Server metadata such as `classification_definition()`. |
|
|
91
|
+
| `client.v2.ingest` | v2 ingestion for the `hit` and `event` indexes: `create`, `validate` (check records without storing them), `overwrite`, `update_by_query`, `delete`. |
|
|
92
|
+
| `client.v2.search` | Search across several indexes at once (`client.v2.search("hit,event", query)`), plus `count`, `facet`, and `explain`. |
|
|
93
|
+
| `client.v2.case` | Fetch, create, update, hide, and delete cases; manage case items and correlation rules. |
|
|
94
|
+
| `client.mapping` | `map_rows(...)`, the same as the top-level `howler_client.map_rows`. See below. |
|
|
95
|
+
|
|
96
|
+
For example:
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
results = client.search.hit("howler.status:open", rows=10, sort="event.created desc")
|
|
100
|
+
for hit in results["items"]:
|
|
101
|
+
print(hit["howler"]["id"], hit["howler"]["analytic"])
|
|
102
|
+
|
|
103
|
+
client.hit.create({"howler": {"analytic": "My Analytic", "score": 0.5}})
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
Failed requests raise `howler_client.common.utils.ClientError`. It has `status_code` and `api_response` attributes. See
|
|
107
|
+
the [user guide](https://cybercentrecanada.github.io/howler/ingestion/client/) for more examples.
|
|
108
|
+
|
|
109
|
+
## Mapping raw table rows to hit records
|
|
110
|
+
|
|
111
|
+
`RowMapping` turns raw rows into Howler hit dictionaries using a YAML mapping file. It runs entirely offline:
|
|
112
|
+
|
|
113
|
+
- It does not need a Howler connection.
|
|
114
|
+
- It never stores or ingests records.
|
|
115
|
+
- It never sets server-owned IDs such as `howler.id`.
|
|
116
|
+
|
|
117
|
+
To store the results, pass them to `client.hit.create` or `client.v2.ingest.create`.
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
from howler_client import RowMapping, map_rows
|
|
121
|
+
|
|
122
|
+
mapping = RowMapping.from_file("mapping.yaml")
|
|
123
|
+
|
|
124
|
+
for hit, error in map_rows(query_result_rows, mapping):
|
|
125
|
+
if error is not None:
|
|
126
|
+
handle_mapping_error(error) # a MappingError naming the row number and target field
|
|
127
|
+
continue
|
|
128
|
+
process_hit(hit)
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
How `map_rows` behaves:
|
|
132
|
+
|
|
133
|
+
- It is lazy: it reads each source row only when you ask for the next result.
|
|
134
|
+
- For each row, in order, it yields either `(hit, None)` or `(None, MappingError)`.
|
|
135
|
+
- A bad row produces an error result, and the following rows are still processed.
|
|
136
|
+
- A bad mapping raises as soon as you call `map_rows`, before any row is read.
|
|
137
|
+
- Exceptions raised by your row iterator are passed through unchanged.
|
|
138
|
+
|
|
139
|
+
To map a single row, `mapping.map_row(row)` returns the hit or raises `MappingError`.
|
|
140
|
+
|
|
141
|
+
### Mapping file format
|
|
142
|
+
|
|
143
|
+
```yaml
|
|
144
|
+
version: 1
|
|
145
|
+
on_null: error # default for every target: error (default) or omit
|
|
146
|
+
mappings:
|
|
147
|
+
howler.analytic:
|
|
148
|
+
value: Directory User Inventory # constant
|
|
149
|
+
user.id:
|
|
150
|
+
column: user_identifier # copy a column
|
|
151
|
+
user.name:
|
|
152
|
+
column: account_name
|
|
153
|
+
transforms: [trim, lower]
|
|
154
|
+
user.email:
|
|
155
|
+
column: primary_email
|
|
156
|
+
on_null: omit # leave the field out when the value is null
|
|
157
|
+
organization.name:
|
|
158
|
+
column: profile
|
|
159
|
+
path: [organization, name] # read a value nested inside the column
|
|
160
|
+
user.roles:
|
|
161
|
+
concat: [directory_role_names, elevated_role_names] # join list columns
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
Each key under `mappings` is a dotted Howler field name. Each target needs exactly one source:
|
|
165
|
+
|
|
166
|
+
| Expression | Result |
|
|
167
|
+
| -------------------------------- | -------------------------------------------------------------------------------------------------------------------------- |
|
|
168
|
+
| `column: name` | The value of that column. A missing column is a row error. |
|
|
169
|
+
| `column` + `path: [key, 0, ...]` | A value nested inside the column, found by dictionary keys and list indexes. A missing nested key or index counts as null. |
|
|
170
|
+
| `concat: [a, b, ...]` | Two or more list columns joined into one list. |
|
|
171
|
+
| `value: ...` | A constant, computed once when the mapping is loaded. |
|
|
172
|
+
|
|
173
|
+
A target can also set:
|
|
174
|
+
|
|
175
|
+
- `on_null`, which overrides the document-wide setting. With `error`, a null value fails the row. With `omit`, the
|
|
176
|
+
field is left out.
|
|
177
|
+
- `transforms`, which are applied in order:
|
|
178
|
+
|
|
179
|
+
| Transform | Effect |
|
|
180
|
+
| ------------------------------ | ------------------------------------------------------------------------------------------------- |
|
|
181
|
+
| `string` | Converts a scalar to a string. Use `json` for objects and lists. |
|
|
182
|
+
| `integer`, `number`, `boolean` | Strict conversions. `boolean` accepts only booleans or `"true"`/`"false"`. |
|
|
183
|
+
| `trim`, `lower`, `upper` | String clean-up. |
|
|
184
|
+
| `list` | Wraps a scalar in a list. |
|
|
185
|
+
| `json` | Converts the value to compact JSON text with sorted keys. |
|
|
186
|
+
| `{op: timestamp}` | Converts an ISO-8601 string to UTC. Add `assume_timezone: UTC` to accept values with no timezone. |
|
|
187
|
+
|
|
188
|
+
Mapping files are parsed with `yaml.safe_load`. Quote constants that must stay strings, such as `"yes"`, `"off"`, or
|
|
189
|
+
`"2026-01-01"`.
|
|
190
|
+
|
|
191
|
+
Mapping documents must contain JSON-compatible values: object keys must be strings, and YAML-only values such as dates,
|
|
192
|
+
sets, and binary data (as well as non-finite numbers and cyclic structures) are rejected. The client validates mappings
|
|
193
|
+
offline with its bundled [Draft 2020-12 schema](howler_client/schemas/row_mapping_v1.schema.json), which is included in
|
|
194
|
+
the wheel and source distribution.
|
|
195
|
+
|
|
196
|
+
You can also build a mapping from YAML text with `RowMapping.from_yaml(text)`, or from an already-loaded dictionary
|
|
197
|
+
with `RowMapping(document)`. `map_rows` accepts a `RowMapping`, a dictionary, or a path to a YAML file. To use YAML
|
|
198
|
+
text, go through `from_yaml` so it isn't mistaken for a path.
|
|
199
|
+
[`examples/row_mapping/example_mapping.yaml`](examples/row_mapping/example_mapping.yaml) is a complete example.
|
|
200
|
+
|
|
201
|
+
### Checking targets against Howler's fields
|
|
202
|
+
|
|
203
|
+
You can pass Howler's field metadata to check targets before you ingest anything. Use the output of
|
|
204
|
+
`client.search.fields.hit()` or of the MCP `get_hit_fields` tool:
|
|
205
|
+
|
|
206
|
+
```python
|
|
207
|
+
fields = client.search.fields.hit()
|
|
208
|
+
mapping = RowMapping.from_file("mapping.yaml", fields=fields)
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
With field metadata, `RowMapping` checks that:
|
|
212
|
+
|
|
213
|
+
- every target exists and is not deprecated
|
|
214
|
+
- every `concat` target is a list field
|
|
215
|
+
|
|
216
|
+
`RowMapping` does not validate mapped values against target field types or list shape. Validate records with
|
|
217
|
+
`client.v2.ingest.validate("hit", hits)` or let Howler validate them during ingestion.
|
|
218
|
+
|
|
219
|
+
## Development
|
|
220
|
+
|
|
221
|
+
The client lives in the `client/` folder of the Howler monorepo. It uses its own [Poetry](https://python-poetry.org/)
|
|
222
|
+
environment, so run every command below from `client/`.
|
|
223
|
+
|
|
224
|
+
### Setup
|
|
225
|
+
|
|
226
|
+
```bash
|
|
227
|
+
cd client
|
|
228
|
+
poetry install --with dev,test,types
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
The repository's pre-commit hooks run Ruff and check commit messages. From the repository root, install them with
|
|
232
|
+
`pre-commit install`.
|
|
233
|
+
|
|
234
|
+
### Code quality
|
|
235
|
+
|
|
236
|
+
These are the checks CI runs:
|
|
237
|
+
|
|
238
|
+
```bash
|
|
239
|
+
poetry run ruff format howler_client --diff
|
|
240
|
+
poetry run ruff check howler_client --output-format=github
|
|
241
|
+
poetry run type_check # mypy over howler_client
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
### Unit tests
|
|
245
|
+
|
|
246
|
+
Unit tests in `test/unit/` run offline: they need no Howler server, Elasticsearch, or Redis.
|
|
247
|
+
|
|
248
|
+
```bash
|
|
249
|
+
poetry run test test/unit # all unit tests
|
|
250
|
+
poetry run test test/unit/test_row_mapping.py -k timestamp # focused run; arguments are passed to pytest
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
### Integration tests
|
|
254
|
+
|
|
255
|
+
Integration tests in `test/integration/` call a running Howler API at `http://localhost:5000`. Set `UI_HOST` to use a
|
|
256
|
+
different address. The API needs Elasticsearch, Redis, and seeded test data. To set this up the same way CI does,
|
|
257
|
+
starting from the repository root:
|
|
258
|
+
|
|
259
|
+
```bash
|
|
260
|
+
# 1. Prepare the API configuration
|
|
261
|
+
sudo mkdir -p /etc/howler/conf /etc/howler/lookups /var/log/howler
|
|
262
|
+
sudo chown -R "$USER" /etc/howler /var/log/howler
|
|
263
|
+
cp api/build_scripts/classification.yml /etc/howler/conf/classification.yml
|
|
264
|
+
cp api/build_scripts/mappings.yml /etc/howler/conf/mappings.yml
|
|
265
|
+
cp client/test/config/config.ci.yml /etc/howler/conf/config.yml
|
|
266
|
+
|
|
267
|
+
# 2. Start Elasticsearch and Redis
|
|
268
|
+
docker compose -f client/test/docker-compose.yml up -d
|
|
269
|
+
|
|
270
|
+
# 3. Install and start the API (leave it running in its own terminal)
|
|
271
|
+
cd api
|
|
272
|
+
poetry install --with test,dev
|
|
273
|
+
poetry run mitre /etc/howler/lookups
|
|
274
|
+
poetry run sigma
|
|
275
|
+
TESTING=true poetry run flask --app howler.app run --no-reload
|
|
276
|
+
|
|
277
|
+
# 4. In another terminal, seed users and random data (this creates the admin API key the tests use)
|
|
278
|
+
cd api
|
|
279
|
+
poetry run python -m howler.odm.random_data
|
|
280
|
+
|
|
281
|
+
# 5. Run the full client test suite with coverage
|
|
282
|
+
cd ../client
|
|
283
|
+
poetry run test
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
`poetry run test` with no arguments runs everything under `test/` with branch coverage. Integration tests need the
|
|
287
|
+
running services. To run only part of the suite, pass paths or pytest options.
|
|
288
|
+
|
|
289
|
+
### Releases
|
|
290
|
+
|
|
291
|
+
Releases are automated with Release Please using Conventional Commits. Use the `client` scope, for example
|
|
292
|
+
`feat(client): ...`. `CHANGELOG.md` is generated, so don't edit it by hand.
|
|
293
|
+
|
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
# Howler Client Library
|
|
2
|
+
|
|
3
|
+
`howler-client` is the Python client for the Howler API. Use it to:
|
|
4
|
+
|
|
5
|
+
- create, search, update, and delete hits
|
|
6
|
+
- manage comments, actions, users, and cases
|
|
7
|
+
- turn raw table rows into hit records offline, before you ingest them
|
|
8
|
+
|
|
9
|
+
It handles authentication, request formatting, retries, and response parsing.
|
|
10
|
+
|
|
11
|
+
- **Package:** [howler-client on PyPI](https://pypi.org/project/howler-client/)
|
|
12
|
+
- **User guide:** [Howler client documentation](https://cybercentrecanada.github.io/howler/ingestion/client/)
|
|
13
|
+
- **Contributor guide:** [Client development](https://cybercentrecanada.github.io/howler/developer/client/)
|
|
14
|
+
|
|
15
|
+
## Installation
|
|
16
|
+
|
|
17
|
+
The client requires Python 3.10 or newer.
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
pip install howler-client
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
## Connecting to Howler
|
|
24
|
+
|
|
25
|
+
`get_client()` returns a `Client` connected to a Howler server. It checks that the server supports the v1 API when
|
|
26
|
+
it connects, so the server must be reachable.
|
|
27
|
+
|
|
28
|
+
```python
|
|
29
|
+
from howler_client import get_client
|
|
30
|
+
|
|
31
|
+
# API key (recommended): create one in the Howler UI under Settings -> User Security -> API Keys
|
|
32
|
+
client = get_client("https://howler.example.com", apikey=("my_username", "my_key_name:my_key_secret"))
|
|
33
|
+
|
|
34
|
+
# JWT, for example one obtained through an OAuth flow
|
|
35
|
+
client = get_client("https://howler.example.com", auth="eyJhbGciOi...")
|
|
36
|
+
|
|
37
|
+
# Username and password (only if the server allows internal authentication)
|
|
38
|
+
client = get_client("https://howler.example.com", auth=("my_username", "my_password"))
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Other useful `get_client()` options:
|
|
42
|
+
|
|
43
|
+
- `verify`: whether to verify TLS certificates.
|
|
44
|
+
- `cert`: a client certificate.
|
|
45
|
+
- `timeout`: the request timeout.
|
|
46
|
+
- `retries`: how many times to retry a request. The default `0` retries forever.
|
|
47
|
+
- `throw_on_bad_request` and `throw_on_max_retries`: whether to raise `ClientError` on failures.
|
|
48
|
+
- `authenticate`: a callback that returns a bearer token.
|
|
49
|
+
|
|
50
|
+
## What the client covers
|
|
51
|
+
|
|
52
|
+
| Attribute | Purpose |
|
|
53
|
+
| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
54
|
+
| `client.hit` | Create hits (`create`), fetch one by ID (`client.hit(hit_id)`), `overwrite`, `update`, `update_by_query`, `delete`, `generate_hash`. Comments are under `client.hit.comment` (`add`, `edit`, `delete`). |
|
|
55
|
+
| `client.search` | Lucene search over hits (`hit`) and actions (`action`), plus `facet`, `fields`, `grouped`, `histogram`, `stats`, and `stream` helpers, each with a `.hit(...)` method. |
|
|
56
|
+
| `client.action` | List, create, and execute saved actions. |
|
|
57
|
+
| `client.user` | Add, delete, list, and update users; `whoami`. |
|
|
58
|
+
| `client.help` | Server metadata such as `classification_definition()`. |
|
|
59
|
+
| `client.v2.ingest` | v2 ingestion for the `hit` and `event` indexes: `create`, `validate` (check records without storing them), `overwrite`, `update_by_query`, `delete`. |
|
|
60
|
+
| `client.v2.search` | Search across several indexes at once (`client.v2.search("hit,event", query)`), plus `count`, `facet`, and `explain`. |
|
|
61
|
+
| `client.v2.case` | Fetch, create, update, hide, and delete cases; manage case items and correlation rules. |
|
|
62
|
+
| `client.mapping` | `map_rows(...)`, the same as the top-level `howler_client.map_rows`. See below. |
|
|
63
|
+
|
|
64
|
+
For example:
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
results = client.search.hit("howler.status:open", rows=10, sort="event.created desc")
|
|
68
|
+
for hit in results["items"]:
|
|
69
|
+
print(hit["howler"]["id"], hit["howler"]["analytic"])
|
|
70
|
+
|
|
71
|
+
client.hit.create({"howler": {"analytic": "My Analytic", "score": 0.5}})
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Failed requests raise `howler_client.common.utils.ClientError`. It has `status_code` and `api_response` attributes. See
|
|
75
|
+
the [user guide](https://cybercentrecanada.github.io/howler/ingestion/client/) for more examples.
|
|
76
|
+
|
|
77
|
+
## Mapping raw table rows to hit records
|
|
78
|
+
|
|
79
|
+
`RowMapping` turns raw rows into Howler hit dictionaries using a YAML mapping file. It runs entirely offline:
|
|
80
|
+
|
|
81
|
+
- It does not need a Howler connection.
|
|
82
|
+
- It never stores or ingests records.
|
|
83
|
+
- It never sets server-owned IDs such as `howler.id`.
|
|
84
|
+
|
|
85
|
+
To store the results, pass them to `client.hit.create` or `client.v2.ingest.create`.
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
from howler_client import RowMapping, map_rows
|
|
89
|
+
|
|
90
|
+
mapping = RowMapping.from_file("mapping.yaml")
|
|
91
|
+
|
|
92
|
+
for hit, error in map_rows(query_result_rows, mapping):
|
|
93
|
+
if error is not None:
|
|
94
|
+
handle_mapping_error(error) # a MappingError naming the row number and target field
|
|
95
|
+
continue
|
|
96
|
+
process_hit(hit)
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
How `map_rows` behaves:
|
|
100
|
+
|
|
101
|
+
- It is lazy: it reads each source row only when you ask for the next result.
|
|
102
|
+
- For each row, in order, it yields either `(hit, None)` or `(None, MappingError)`.
|
|
103
|
+
- A bad row produces an error result, and the following rows are still processed.
|
|
104
|
+
- A bad mapping raises as soon as you call `map_rows`, before any row is read.
|
|
105
|
+
- Exceptions raised by your row iterator are passed through unchanged.
|
|
106
|
+
|
|
107
|
+
To map a single row, `mapping.map_row(row)` returns the hit or raises `MappingError`.
|
|
108
|
+
|
|
109
|
+
### Mapping file format
|
|
110
|
+
|
|
111
|
+
```yaml
|
|
112
|
+
version: 1
|
|
113
|
+
on_null: error # default for every target: error (default) or omit
|
|
114
|
+
mappings:
|
|
115
|
+
howler.analytic:
|
|
116
|
+
value: Directory User Inventory # constant
|
|
117
|
+
user.id:
|
|
118
|
+
column: user_identifier # copy a column
|
|
119
|
+
user.name:
|
|
120
|
+
column: account_name
|
|
121
|
+
transforms: [trim, lower]
|
|
122
|
+
user.email:
|
|
123
|
+
column: primary_email
|
|
124
|
+
on_null: omit # leave the field out when the value is null
|
|
125
|
+
organization.name:
|
|
126
|
+
column: profile
|
|
127
|
+
path: [organization, name] # read a value nested inside the column
|
|
128
|
+
user.roles:
|
|
129
|
+
concat: [directory_role_names, elevated_role_names] # join list columns
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Each key under `mappings` is a dotted Howler field name. Each target needs exactly one source:
|
|
133
|
+
|
|
134
|
+
| Expression | Result |
|
|
135
|
+
| -------------------------------- | -------------------------------------------------------------------------------------------------------------------------- |
|
|
136
|
+
| `column: name` | The value of that column. A missing column is a row error. |
|
|
137
|
+
| `column` + `path: [key, 0, ...]` | A value nested inside the column, found by dictionary keys and list indexes. A missing nested key or index counts as null. |
|
|
138
|
+
| `concat: [a, b, ...]` | Two or more list columns joined into one list. |
|
|
139
|
+
| `value: ...` | A constant, computed once when the mapping is loaded. |
|
|
140
|
+
|
|
141
|
+
A target can also set:
|
|
142
|
+
|
|
143
|
+
- `on_null`, which overrides the document-wide setting. With `error`, a null value fails the row. With `omit`, the
|
|
144
|
+
field is left out.
|
|
145
|
+
- `transforms`, which are applied in order:
|
|
146
|
+
|
|
147
|
+
| Transform | Effect |
|
|
148
|
+
| ------------------------------ | ------------------------------------------------------------------------------------------------- |
|
|
149
|
+
| `string` | Converts a scalar to a string. Use `json` for objects and lists. |
|
|
150
|
+
| `integer`, `number`, `boolean` | Strict conversions. `boolean` accepts only booleans or `"true"`/`"false"`. |
|
|
151
|
+
| `trim`, `lower`, `upper` | String clean-up. |
|
|
152
|
+
| `list` | Wraps a scalar in a list. |
|
|
153
|
+
| `json` | Converts the value to compact JSON text with sorted keys. |
|
|
154
|
+
| `{op: timestamp}` | Converts an ISO-8601 string to UTC. Add `assume_timezone: UTC` to accept values with no timezone. |
|
|
155
|
+
|
|
156
|
+
Mapping files are parsed with `yaml.safe_load`. Quote constants that must stay strings, such as `"yes"`, `"off"`, or
|
|
157
|
+
`"2026-01-01"`.
|
|
158
|
+
|
|
159
|
+
Mapping documents must contain JSON-compatible values: object keys must be strings, and YAML-only values such as dates,
|
|
160
|
+
sets, and binary data (as well as non-finite numbers and cyclic structures) are rejected. The client validates mappings
|
|
161
|
+
offline with its bundled [Draft 2020-12 schema](howler_client/schemas/row_mapping_v1.schema.json), which is included in
|
|
162
|
+
the wheel and source distribution.
|
|
163
|
+
|
|
164
|
+
You can also build a mapping from YAML text with `RowMapping.from_yaml(text)`, or from an already-loaded dictionary
|
|
165
|
+
with `RowMapping(document)`. `map_rows` accepts a `RowMapping`, a dictionary, or a path to a YAML file. To use YAML
|
|
166
|
+
text, go through `from_yaml` so it isn't mistaken for a path.
|
|
167
|
+
[`examples/row_mapping/example_mapping.yaml`](examples/row_mapping/example_mapping.yaml) is a complete example.
|
|
168
|
+
|
|
169
|
+
### Checking targets against Howler's fields
|
|
170
|
+
|
|
171
|
+
You can pass Howler's field metadata to check targets before you ingest anything. Use the output of
|
|
172
|
+
`client.search.fields.hit()` or of the MCP `get_hit_fields` tool:
|
|
173
|
+
|
|
174
|
+
```python
|
|
175
|
+
fields = client.search.fields.hit()
|
|
176
|
+
mapping = RowMapping.from_file("mapping.yaml", fields=fields)
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
With field metadata, `RowMapping` checks that:
|
|
180
|
+
|
|
181
|
+
- every target exists and is not deprecated
|
|
182
|
+
- every `concat` target is a list field
|
|
183
|
+
|
|
184
|
+
`RowMapping` does not validate mapped values against target field types or list shape. Validate records with
|
|
185
|
+
`client.v2.ingest.validate("hit", hits)` or let Howler validate them during ingestion.
|
|
186
|
+
|
|
187
|
+
## Development
|
|
188
|
+
|
|
189
|
+
The client lives in the `client/` folder of the Howler monorepo. It uses its own [Poetry](https://python-poetry.org/)
|
|
190
|
+
environment, so run every command below from `client/`.
|
|
191
|
+
|
|
192
|
+
### Setup
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
cd client
|
|
196
|
+
poetry install --with dev,test,types
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
The repository's pre-commit hooks run Ruff and check commit messages. From the repository root, install them with
|
|
200
|
+
`pre-commit install`.
|
|
201
|
+
|
|
202
|
+
### Code quality
|
|
203
|
+
|
|
204
|
+
These are the checks CI runs:
|
|
205
|
+
|
|
206
|
+
```bash
|
|
207
|
+
poetry run ruff format howler_client --diff
|
|
208
|
+
poetry run ruff check howler_client --output-format=github
|
|
209
|
+
poetry run type_check # mypy over howler_client
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
### Unit tests
|
|
213
|
+
|
|
214
|
+
Unit tests in `test/unit/` run offline: they need no Howler server, Elasticsearch, or Redis.
|
|
215
|
+
|
|
216
|
+
```bash
|
|
217
|
+
poetry run test test/unit # all unit tests
|
|
218
|
+
poetry run test test/unit/test_row_mapping.py -k timestamp # focused run; arguments are passed to pytest
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
### Integration tests
|
|
222
|
+
|
|
223
|
+
Integration tests in `test/integration/` call a running Howler API at `http://localhost:5000`. Set `UI_HOST` to use a
|
|
224
|
+
different address. The API needs Elasticsearch, Redis, and seeded test data. To set this up the same way CI does,
|
|
225
|
+
starting from the repository root:
|
|
226
|
+
|
|
227
|
+
```bash
|
|
228
|
+
# 1. Prepare the API configuration
|
|
229
|
+
sudo mkdir -p /etc/howler/conf /etc/howler/lookups /var/log/howler
|
|
230
|
+
sudo chown -R "$USER" /etc/howler /var/log/howler
|
|
231
|
+
cp api/build_scripts/classification.yml /etc/howler/conf/classification.yml
|
|
232
|
+
cp api/build_scripts/mappings.yml /etc/howler/conf/mappings.yml
|
|
233
|
+
cp client/test/config/config.ci.yml /etc/howler/conf/config.yml
|
|
234
|
+
|
|
235
|
+
# 2. Start Elasticsearch and Redis
|
|
236
|
+
docker compose -f client/test/docker-compose.yml up -d
|
|
237
|
+
|
|
238
|
+
# 3. Install and start the API (leave it running in its own terminal)
|
|
239
|
+
cd api
|
|
240
|
+
poetry install --with test,dev
|
|
241
|
+
poetry run mitre /etc/howler/lookups
|
|
242
|
+
poetry run sigma
|
|
243
|
+
TESTING=true poetry run flask --app howler.app run --no-reload
|
|
244
|
+
|
|
245
|
+
# 4. In another terminal, seed users and random data (this creates the admin API key the tests use)
|
|
246
|
+
cd api
|
|
247
|
+
poetry run python -m howler.odm.random_data
|
|
248
|
+
|
|
249
|
+
# 5. Run the full client test suite with coverage
|
|
250
|
+
cd ../client
|
|
251
|
+
poetry run test
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
`poetry run test` with no arguments runs everything under `test/` with branch coverage. Integration tests need the
|
|
255
|
+
running services. To run only part of the suite, pass paths or pytest options.
|
|
256
|
+
|
|
257
|
+
### Releases
|
|
258
|
+
|
|
259
|
+
Releases are automated with Release Please using Conventional Commits. Use the `client` scope, for example
|
|
260
|
+
`feat(client): ...`. `CHANGELOG.md` is generated, so don't edit it by hand.
|
|
@@ -2,6 +2,7 @@ from importlib.metadata import PackageNotFoundError, version
|
|
|
2
2
|
|
|
3
3
|
from howler_client.client import Client
|
|
4
4
|
from howler_client.connection import Connection
|
|
5
|
+
from howler_client.module.mapping import MappingError, RowMapping, RowMappingResult, map_rows
|
|
5
6
|
|
|
6
7
|
try:
|
|
7
8
|
__version__ = version("howler-client")
|
|
@@ -11,6 +12,16 @@ except PackageNotFoundError:
|
|
|
11
12
|
RETRY_FOREVER = 0
|
|
12
13
|
SUPPORTED_APIS = {"v1"}
|
|
13
14
|
|
|
15
|
+
__all__ = [
|
|
16
|
+
"Client",
|
|
17
|
+
"Connection",
|
|
18
|
+
"MappingError",
|
|
19
|
+
"RowMapping",
|
|
20
|
+
"RowMappingResult",
|
|
21
|
+
"get_client",
|
|
22
|
+
"map_rows",
|
|
23
|
+
]
|
|
24
|
+
|
|
14
25
|
|
|
15
26
|
def get_client(
|
|
16
27
|
server,
|
|
@@ -5,6 +5,7 @@ from howler_client.connection import Connection
|
|
|
5
5
|
from howler_client.module.action import Action
|
|
6
6
|
from howler_client.module.help import Help
|
|
7
7
|
from howler_client.module.hit import Hit
|
|
8
|
+
from howler_client.module.mapping import HowlerMapping
|
|
8
9
|
from howler_client.module.search import Search
|
|
9
10
|
from howler_client.module.user import User
|
|
10
11
|
from howler_client.module.v2 import V2
|
|
@@ -27,6 +28,7 @@ class Client(object):
|
|
|
27
28
|
self.hit = Hit(self._connection, self.search)
|
|
28
29
|
self.user = User(self._connection)
|
|
29
30
|
self.v2 = V2(self._connection)
|
|
31
|
+
self.mapping = HowlerMapping()
|
|
30
32
|
|
|
31
33
|
paths: list[str] = []
|
|
32
34
|
walk_api_path(self, [""], paths)
|